Decode chunked request bodies before passing them to httpbin

pytest_httpbin/serve.py

@@ -1,11 +1,14 @@
+import io
 import os
+import re
 import ssl
 import threading
 from urllib.parse import urljoin
 from wsgiref.handlers import SimpleHandler
 from wsgiref.simple_server import WSGIRequestHandler, WSGIServer, make_server
 
 CERT_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "certs")
+HEX_DIGITS = re.compile(rb"[0-9A-Fa-f]+")
 
 
 class ServerHandler(SimpleHandler):
@@ -33,12 +36,49 @@ def handle(self):
         if not self.parse_request():  # An error code has been sent, just exit
             return
 
-        handler = ServerHandler(
-            self.rfile, self.wfile, self.get_stderr(), self.get_environ()
-        )
+        stdin = self.rfile
+        environ = self.get_environ()
+        if self.headers.get("Transfer-Encoding", "").strip().lower() == "chunked":
+            # wsgiref doesn't decode chunked request bodies, and httpbin answers
+            # them with a 501 without reading them, so a client still sending the
+            # body gets a broken pipe. Decode the body here and give it to the
+            # application with a Content-Length instead, see issue #33
+            try:
+                body = self.read_chunked_body()
+            except ValueError:
+                self.send_error(400, "Bad chunked request body")
+                return
+            stdin = io.BytesIO(body)
+            environ["CONTENT_LENGTH"] = str(len(body))
+            del environ["HTTP_TRANSFER_ENCODING"]
+
+        handler = ServerHandler(stdin, self.wfile, self.get_stderr(), environ)
         handler.request_handler = self  # backpointer for logging
         handler.run(self.server.get_app())
 
+    def read_chunked_body(self):
+        """
+        Read a request body sent with Transfer-Encoding: chunked, and return it
+        decoded. Chunk extensions and trailer fields are ignored.
+        """
+        chunks = []
+        while True:
+            size_line = self.rfile.readline(65537)
+            size = size_line.split(b";", 1)[0].strip()
+            if not size_line.endswith(b"\n") or not HEX_DIGITS.fullmatch(size):
+                raise ValueError(f"bad chunk size line: {size_line!r}")
+            size = int(size, 16)
+            if size == 0:
+                break
+            chunk = self.rfile.read(size)
+            if len(chunk) != size or self.rfile.readline(3) not in (b"\r\n", b"\n"):
+                raise ValueError("chunk data doesn't match its size")
+            chunks.append(chunk)
+        # Skip the trailer section, which ends with an empty line
+        while self.rfile.readline(65537) not in (b"\r\n", b"\n", b""):
+            pass
+        return b"".join(chunks)
+
     def get_environ(self):
         """
         wsgiref simple server adds content-type text/plain to everything, this

tests/test_server.py

@@ -1,7 +1,10 @@
 import contextlib
+import http.client
+import json
 import os
 import re
 import socket
+import time
 
 import pytest
 import requests.exceptions
@@ -34,6 +37,69 @@ def test_unicode_data(httpbin):
     assert resp.json()["data"] == "оживлённым"
 
 
+def test_chunked_request_body(httpbin_both):
+    """
+    httpbin answered chunked request bodies with a 501 instead of echoing them,
+    see issue #33
+    """
+
+    def body():
+        yield b"ham"
+        yield b"spam"
+
+    resp = requests.post(httpbin_both + "/post", data=body())
+    assert resp.status_code == 200
+    assert resp.json()["data"] == "hamspam"
+    # The application gets the decoded body with a Content-Length
+    assert resp.json()["headers"]["Content-Length"] == "7"
+
+
+def test_chunked_request_body_is_read_before_responding(httpbin):
+    """
+    The server answered chunked requests without reading their body, so a client
+    still sending it got a broken pipe, see issue #33
+    """
+
+    def slow_body():
+        yield b"ham"
+        time.sleep(0.2)
+        yield b"spam"
+
+    conn = http.client.HTTPConnection(httpbin.host, httpbin.port, timeout=10)
+    try:
+        conn.request("POST", "/post", body=slow_body())
+        resp = conn.getresponse()
+        assert resp.status == 200
+        assert json.loads(resp.read())["data"] == "hamspam"
+    finally:
+        conn.close()
+
+
+def test_bad_chunked_request_body(httpbin):
+    # Send nothing after the bad chunk size line, so the server has read all of the
+    # request when it answers and closes the connection
+    request = b"\r\n".join(
+        [
+            b"POST /post HTTP/1.1",
+            b"Host: " + httpbin.host.encode("ascii"),
+            b"Transfer-Encoding: chunked",
+            b"Connection: close",
+            b"",
+            b"not-a-size",
+            b"",
+        ]
+    )
+    with socket.create_connection((httpbin.host, httpbin.port), timeout=10) as sock:
+        sock.sendall(request)
+        response = b""
+        buffer = sock.recv(4096)
+        while buffer:
+            response += buffer
+            buffer = sock.recv(4096)
+
+    assert response.split(b"\r\n", 1)[0].split(b" ")[1] == b"400"
+
+
 def test_server_should_be_http_1_1(httpbin):
     """
     The server should speak HTTP/1.1 since we live in the future, see issue #6