requests: Add chunked response body support.

python-ecosys/requests/README.md

@@ -5,7 +5,8 @@ This module provides a lightweight version of the Python
 
 It includes support for all HTTP verbs, https, json decoding of responses,
 redirects, basic authentication, HTTP/1.1 requests, and reading response
-bodies with Content-Length via streaming ``.raw`` or lazy ``.content``.
+bodies with Content-Length or Transfer-Encoding: chunked via streaming
+``.raw`` or lazy ``.content``.
 
 ### Limitations
 
@@ -14,11 +15,9 @@ bodies with Content-Length via streaming ``.raw`` or lazy ``.content``.
   multipart-form encoding of post data (this can be done manually).
 * Compressed requests/responses are not currently supported.
 * File upload is not supported.
-* Chunked encoding in responses is not supported.
 * HTTP keep-alive connection reuse is not supported (Connection: close by default).
 
 ### Follow-up work
 
-* Chunked response bodies.
 * TLS certificate verification (see micropython-lib issue #838).
 * ``stream=True`` incremental body reads (see issue #777).

python-ecosys/requests/requests/__init__.py

@@ -2,30 +2,58 @@
 
 
 class BodyStream:
-    def __init__(self, sock, remaining):
+    def __init__(self, sock, remaining=0, chunked=False):
         self._sock = sock
         self._remaining = remaining
+        self._chunked = chunked
 
     def read(self, n=-1):
-        if self._remaining == 0:
-            return b""
-        if n < 0 or n > self._remaining:
-            n = self._remaining
-        data = self._sock.read(n)
-        self._remaining -= len(data)
-        if not data:
-            raise ValueError("Connection closed before Content-Length satisfied")
-        return data
+        if not self._chunked:
+            if self._remaining == 0:
+                return b""
+            if n < 0 or n > self._remaining:
+                n = self._remaining
+            data = self._sock.read(n)
+            self._remaining -= len(data)
+            if not data:
+                raise ValueError("Connection closed before Content-Length satisfied")
+            return data
+        if n >= 0:
+            buf = bytearray(n)
+            return bytes(buf[: self.readinto(buf)])
+        buf = bytearray(256)
+        chunks = []
+        while True:
+            got = self.readinto(buf)
+            if not got:
+                break
+            chunks.append(bytes(buf[:got]))
+        return b"".join(chunks)
 
     def readinto(self, buf):
         if self._remaining == 0:
-            return 0
+            if self._chunked:
+                l = self._sock.readline()
+                self._remaining = int(l.split(b";", 1)[0], 16)
+                if self._remaining == 0:
+                    while True:
+                        l = self._sock.readline()
+                        if not l or l == b"\r\n":
+                            break
+                    self._chunked = False
+                    return 0
+            else:
+                return 0
         if len(buf) > self._remaining:
             buf = memoryview(buf)[: self._remaining]
         got = self._sock.readinto(buf)
-        self._remaining -= got
         if not got:
+            if self._chunked:
+                raise ValueError("Connection closed before chunk was complete")
             raise ValueError("Connection closed before Content-Length satisfied")
+        self._remaining -= got
+        if self._remaining == 0 and self._chunked:
+            self._sock.readline()
         return got
 
     def close(self):
@@ -193,14 +221,15 @@ def request(
         if len(l) > 2:
             reason = l[2].rstrip()
         remaining = None
+        chunked = False
         while True:
             l = s.readline()
             if not l or l == b"\r\n":
                 break
             # print(l)
             if l.startswith(b"Transfer-Encoding:"):
                 if b"chunked" in l:
-                    raise ValueError("Unsupported " + str(l, "utf-8"))
+                    chunked = True
             elif l.startswith(b"Location:") and not 200 <= status <= 299:
                 if status in [301, 302, 303, 307, 308]:
                     redirect = str(l[10:-2], "utf-8")
@@ -230,7 +259,9 @@ def request(
         else:
             return request(method, redirect, data, json, headers, stream)
     else:
-        if remaining is not None:
+        if chunked:
+            resp = Response(BodyStream(s, chunked=True))
+        elif remaining is not None:
             resp = Response(BodyStream(s, remaining))
         else:
             resp = Response(s)

python-ecosys/requests/test_requests.py

@@ -172,19 +172,28 @@ def test_content_length_via_content():
     socket.socket = lambda *a, **k: Socket()
 
 
-def test_chunked_response_raises():
+def test_chunked_response_via_content():
     socket.socket = lambda *a, **k: Socket(
-        read_data=b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n5\r\nhello\r\n0\r\n\r\n"
+        read_data=b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n5\r\nhello\r\n6\r\n world\r\n0\r\n\r\n"
     )
-    raised = False
-    try:
-        requests.request("GET", "http://example.com")
-    except ValueError as e:
-        raised = True
-        if "Unsupported" not in str(e):
-            raise
-    if not raised:
-        raise AssertionError("expected ValueError for chunked response")
+    response = requests.request("GET", "http://example.com")
+    assert response.content == b"hello world"
+    socket.socket = lambda *a, **k: Socket()
+
+
+def test_chunked_response_readinto():
+    socket.socket = lambda *a, **k: Socket(
+        read_data=b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n3\r\nabc\r\n4\r\ndefg\r\n0\r\n\r\n"
+    )
+    response = requests.request("GET", "http://example.com")
+    buf = bytearray(2)
+    result = b""
+    while True:
+        n = response.raw.readinto(buf)
+        if n == 0:
+            break
+        result += buf if n == 2 else buf[:n]
+    assert result == b"abcdefg"
     socket.socket = lambda *a, **k: Socket()
 
 
@@ -236,7 +245,8 @@ def test_raw_readinto_content_length():
 test_overwrite_post_chunked_data_headers()
 test_do_not_modify_headers_argument()
 test_content_length_via_content()
-test_chunked_response_raises()
+test_chunked_response_via_content()
+test_chunked_response_readinto()
 test_raw_open_before_content()
 test_raw_incremental_content_length()
 test_raw_readinto_content_length()