Repository: encode/starlette

Issue:
MultiPart and URLEncoded form parsing
Introduce multipart and URL encoded parsing to support `form = await request.form()`

See https://github.com/pallets/werkzeug/pull/1330



Discussion:

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -46,6 +46,7 @@ def get_packages(package):
     extras_require = {
         'full': [
             'aiofiles',
+            'python-multipart',
             'requests',
             'ujson'
         ]
diff --git a/starlette/formparsers.py b/starlette/formparsers.py
new file mode 100644
--- /dev/null
+++ b/starlette/formparsers.py
@@ -0,0 +1,247 @@
+from enum import Enum
+from starlette.datastructures import Headers
+import asyncio
+import tempfile
+import typing
+
+try:
+    from multipart.multipart import parse_options_header
+    import multipart
+except ImportError:  # pragma: nocover
+    parse_options_header = None  # type: ignore
+    multipart = None  # type: ignore
+
+
+class FormMessage(Enum):
+    FIELD_START = 1
+    FIELD_NAME = 2
+    FIELD_DATA = 3
+    FIELD_END = 4
+    END = 5
+
+
+class MultiPartMessage(Enum):
+    PART_BEGIN = 1
+    PART_DATA = 2
+    PART_END = 3
+    HEADER_FIELD = 4
+    HEADER_VALUE = 5
+    HEADER_END = 6
+    HEADERS_FINISHED = 7
+    END = 8
+
+
+class UploadFile:
+    def __init__(self, filename: str) -> None:
+        self.filename = filename
+        self._file = None
+        self._loop = asyncio.get_event_loop()
+
+    async def setup(self) -> None:
+        self._file = await self._loop.run_in_executor(
+            None, tempfile.SpooledTemporaryFile
+        )
+
+    async def write(self, data: bytes) -> None:
+        await self._loop.run_in_executor(None, self._file.write, data)
+
+    async def read(self, size: int = None) -> bytes:
+        return await self._loop.run_in_executor(None, self._file.read, size)
+
+    async def seek(self, offset: int) -> None:
+        await self._loop.run_in_executor(None, self._file.seek, offset)
+
+    async def close(self) -> None:
+        await self._loop.run_in_executor(None, self._file.close)
+
+
+class FormParser:
+    def __init__(
+        self, headers: Headers, stream: typing.AsyncGenerator[bytes, None]
+    ) -> None:
+        assert (
+            multipart is not None
+        ), "The `python-multipart` library must be installed to use form parsing."
+        self.headers = headers
+        self.stream = stream
+        self.messages = []  # type: typing.List[typing.Tuple[MultiPartMessage, bytes]]
+
+    def on_field_start(self) -> None:
+        print("on_field_start")
+        message = (FormMessage.FIELD_START, b"")
+        self.messages.append(message)
+
+    def on_field_name(self, data: bytes, start: int, end: int) -> None:
+        print("on_field_name")
+        message = (FormMessage.FIELD_NAME, data[start:end])
+        self.messages.append(message)
+
+    def on_field_data(self, data: bytes, start: int, end: int) -> None:
+        print("on_field_data")
+        message = (FormMessage.FIELD_DATA, data[start:end])
+        self.messages.append(message)
+
+    def on_field_end(self) -> None:
+        print("on_field_end")
+        message = (FormMessage.FIELD_END, b"")
+        self.messages.append(message)
+
+    def on_end(self) -> None:
+        print("on_end")
+        message = (FormMessage.END, b"")
+        self.messages.append(message)
+
+    async def parse(self) -> typing.Dict[str, typing.Union[str, UploadFile]]:
+        # Callbacks dictionary.
+        callbacks = {
+            "on_field_start": self.on_field_start,
+            "on_field_name": self.on_field_name,
+            "on_field_data": self.on_field_data,
+            "on_field_end": self.on_field_end,
+            "on_end": self.on_end,
+        }
+
+        # Create the parser.
+        parser = multipart.QuerystringParser(callbacks)
+        field_name = b""
+        field_value = b""
+
+        result = {}  # type: typing.Dict[str, typing.Union[str, UploadFile]]
+
+        # Feed the parser with data from the request.
+        async for chunk in self.stream():
+            if chunk:
+                parser.write(chunk)
+            else:
+                parser.finalize()
+            messages = list(self.messages)
+            self.messages.clear()
+            for message_type, message_bytes in messages:
+                if message_type == FormMessage.FIELD_START:
+                    field_name = b""
+                    field_value = b""
+                elif message_type == FormMessage.FIELD_NAME:
+                    field_name += message_bytes
+                elif message_type == FormMessage.FIELD_DATA:
+                    field_value += message_bytes
+                elif message_type == FormMessage.FIELD_END:
+                    result[field_name.decode("latin-1")] = field_value.decode("latin-1")
+                elif message_type == FormMessage.END:
+                    pass
+
+        return result
+
+
+class MultiPartParser:
+    def __init__(
+        self, headers: Headers, stream: typing.AsyncGenerator[bytes, None]
+    ) -> None:
+        assert (
+            multipart is not None
+        ), "The `python-multipart` library must be installed to use form parsing."
+        self.headers = headers
+        self.stream = stream
+        self.messages = []  # type: typing.List[typing.Tuple[MultiPartMessage, bytes]]
+
+    def on_part_begin(self) -> None:
+        message = (MultiPartMessage.PART_BEGIN, b"")
+        self.messages.append(message)
+
+    def on_part_data(self, data: bytes, start: int, end: int) -> None:
+        message = (MultiPartMessage.PART_DATA, data[start:end])
+        self.messages.append(message)
+
+    def on_part_end(self) -> None:
+        message = (MultiPartMessage.PART_END, b"")
+        self.messages.append(message)
+
+    def on_header_field(self, data: bytes, start: int, end: int) -> None:
+        message = (MultiPartMessage.HEADER_FIELD, data[start:end])
+        self.messages.append(message)
+
+    def on_header_value(self, data: bytes, start: int, end: int) -> None:
+        message = (MultiPartMessage.HEADER_VALUE, data[start:end])
+        self.messages.append(message)
+
+    def on_header_end(self) -> None:
+        message = (MultiPartMessage.HEADER_END, b"")
+        self.messages.append(message)
+
+    def on_headers_finished(self) -> None:
+        message = (MultiPartMessage.HEADERS_FINISHED, b"")
+        self.messages.append(message)
+
+    def on_end(self) -> None:
+        message = (MultiPartMessage.END, b"")
+        self.messages.append(message)
+
+    async def parse(self) -> typing.Dict[str, typing.Union[str, UploadFile]]:
+        # Parse the Content-Type header to get the multipart boundary.
+        content_type, params = parse_options_header(self.headers["Content-Type"])
+        boundary = params.get(b"boundary")
+
+        # Callbacks dictionary.
+        callbacks = {
+            "on_part_begin": self.on_part_begin,
+            "on_part_data": self.on_part_data,
+            "on_part_end": self.on_part_end,
+            "on_header_field": self.on_header_field,
+            "on_header_value": self.on_header_value,
+            "on_header_end": self.on_header_end,
+            "on_headers_finished": self.on_headers_finished,
+            "on_end": self.on_end,
+        }
+
+        # Create the parser.
+        parser = multipart.MultipartParser(boundary, callbacks)
+        header_field = b""
+        header_value = b""
+        raw_headers = []  # type: typing.List[typing.Tuple[bytes, bytes]]
+        field_name = ""
+        data = b""
+        file = None  # type: UploadFile
+
+        result = {}  # type: typing.Dict[str, typing.Union[str, UploadFile]]
+
+        # Feed the parser with data from the request.
+        async for chunk in self.stream():
+            parser.write(chunk)
+            messages = list(self.messages)
+            self.messages.clear()
+            for message_type, message_bytes in messages:
+                if message_type == MultiPartMessage.PART_BEGIN:
+                    raw_headers = []
+                    data = b""
+                elif message_type == MultiPartMessage.HEADER_FIELD:
+                    header_field += message_bytes
+                elif message_type == MultiPartMessage.HEADER_VALUE:
+                    header_value += message_bytes
+                elif message_type == MultiPartMessage.HEADER_END:
+                    raw_headers.append((header_field.lower(), header_value))
+                    header_field = b""
+                    header_value = b""
+                elif message_type == MultiPartMessage.HEADERS_FINISHED:
+                    headers = Headers(raw_headers)
+                    content_disposition = headers.get("Content-Disposition")
+                    disposition, options = parse_options_header(content_disposition)
+                    field_name = options[b"name"].decode("latin-1")
+                    if b"filename" in options:
+                        filename = options[b"filename"].decode("latin-1")
+                        file = UploadFile(filename=filename)
+                        await file.setup()
+                elif message_type == MultiPartMessage.PART_DATA:
+                    if file is None:
+                        data += message_bytes
+                    else:
+                        await file.write(message_bytes)
+                elif message_type == MultiPartMessage.PART_END:
+                    if file is None:
+                        result[field_name] = data.decode("latin-1")
+                    else:
+                        await file.seek(0)
+                        result[field_name] = file
+                elif message_type == MultiPartMessage.END:
+                    pass
+
+        parser.finalize()
+        return result
diff --git a/starlette/requests.py b/starlette/requests.py
--- a/starlette/requests.py
+++ b/starlette/requests.py
@@ -1,10 +1,17 @@
 import typing
 import json
 from collections.abc import Mapping
+from urllib.parse import unquote
 import http.cookies
 from starlette.datastructures import URL, Headers, QueryParams
+from starlette.formparsers import FormParser, MultiPartParser
 from starlette.types import Scope, Receive
 
+try:
+    from multipart.multipart import parse_options_header
+except ImportError:  # pragma: nocover
+    parse_options_header = None  # type: ignore
+
 
 class ClientDisconnect(Exception):
     pass
@@ -77,11 +84,14 @@ async def stream(self) -> typing.AsyncGenerator[bytes, None]:
         while True:
             message = await self._receive()
             if message["type"] == "http.request":
-                yield message.get("body", b"")
+                body = message.get("body", b"")
+                if body:
+                    yield body
                 if not message.get("more_body", False):
                     break
             elif message["type"] == "http.disconnect":
                 raise ClientDisconnect()
+        yield b""
 
     async def body(self) -> bytes:
         if not hasattr(self, "_body"):
@@ -96,3 +106,26 @@ async def json(self) -> typing.Any:
             body = await self.body()
             self._json = json.loads(body)
         return self._json
+
+    async def form(self):
+        if not hasattr(self, "_form"):
+            assert (
+                parse_options_header is not None
+            ), "The `python-multipart` library must be installed to use form parsing."
+            content_type_header = self.headers.get("Content-Type")
+            content_type, options = parse_options_header(content_type_header)
+            if content_type == b"multipart/form-data":
+                parser = MultiPartParser(self.headers, self.stream)
+                self._form = await parser.parse()
+            elif content_type == b"application/x-www-form-urlencoded":
+                parser = FormParser(self.headers, self.stream)
+                self._form = await parser.parse()
+            else:
+                self._form = {}
+        return self._form
+
+    async def close(self):
+        if hasattr(self, "_form"):
+            for item in self._form.values():
+                if hasattr(item, "close"):
+                    await item.close()


Test patch:
diff --git a/scripts/test b/scripts/test
--- a/scripts/test
+++ b/scripts/test
@@ -11,7 +11,7 @@ export PYTHON_VERSION=`python -c "$VERSION_SCRIPT"`
 set -x
 
 PYTHONPATH=. ${PREFIX}pytest --ignore venv --cov=starlette --cov=tests --cov-fail-under=100 --cov-report=term-missing ${@}
-${PREFIX}mypy starlette --ignore-missing-imports
+#${PREFIX}mypy starlette --ignore-missing-imports
 if [ "${PYTHON_VERSION}" = '3.7' ]; then
     echo "Skipping 'black' on 3.7. See issue https://github.com/ambv/black/issues/494"
 else
diff --git a/tests/test_formparsers.py b/tests/test_formparsers.py
new file mode 100644
--- /dev/null
+++ b/tests/test_formparsers.py
@@ -0,0 +1,62 @@
+from starlette.formparsers import UploadFile
+from starlette.responses import JSONResponse
+from starlette.requests import Request
+from starlette.testclient import TestClient
+import os
+
+
+class ForceMultipartDict(dict):
+    def __bool__(self):
+        return True
+
+
+# FORCE_MULTIPART is an empty dict that boolean-evaluates as `True`.
+FORCE_MULTIPART = ForceMultipartDict()
+
+
+def app(scope):
+    async def asgi(receive, send):
+        request = Request(scope, receive)
+        data = await request.form()
+        output = {}
+        for key, value in data.items():
+            if isinstance(value, UploadFile):
+                content = await value.read()
+                output[key] = {"filename": value.filename, "content": content.decode()}
+            else:
+                output[key] = value
+        await request.close()
+        response = JSONResponse(output)
+        await response(receive, send)
+
+    return asgi
+
+
+def test_multipart_request_data(tmpdir):
+    client = TestClient(app)
+    response = client.post("/", data={"some": "data"}, files=FORCE_MULTIPART)
+    assert response.json() == {"some": "data"}
+
+
+def test_multipart_request_files(tmpdir):
+    path = os.path.join(tmpdir, "test.txt")
+    with open(path, "wb") as file:
+        file.write(b"<file content>")
+
+    client = TestClient(app)
+    response = client.post("/", files={"test": open(path, "rb")})
+    assert response.json() == {
+        "test": {"filename": "test.txt", "content": "<file content>"}
+    }
+
+
+def test_urlencoded_request_data(tmpdir):
+    client = TestClient(app)
+    response = client.post("/", data={"some": "data"})
+    assert response.json() == {"some": "data"}
+
+
+def test_no_request_data(tmpdir):
+    client = TestClient(app)
+    response = client.post("/")
+    assert response.json() == {}