Repository: aio-libs/aiohttp

Issue:
Some servers don't understand quoted boundaries in multipart documents
## Long story short

After a prolonged debugging session I discovered that the usage of quotes in the boundary of a multipart document caused my request to be rejected by the server since it didn't support this.
Preventing the quotation marks from being added in the code makes this work.

The server in question is using nginx, but I don't know which version yet (asked their support).

## Specification

https://tools.ietf.org/html/rfc7231#section-3.1.1.5
https://tools.ietf.org/html/rfc7231#appendix-D
https://tools.ietf.org/html/rfc7230#section-3.2.6

    Content-Type = media-type

    media-type = type "/" subtype *( OWS ";" OWS parameter )
    parameter = token "=" ( token / quoted-string )

    token          = 1*tchar
    quoted-string  = DQUOTE *( qdtext / quoted-pair ) DQUOTE
    qdtext         = HTAB / SP /%x21 / %x23-5B / %x5D-7E / obs-text
    obs-text       = %x80-FF
    quoted-pair    = "\" ( HTAB / SP / VCHAR / obs-text )
    tchar          = "!" / "#" / "$" / "%" / "&" / "'" / "*"
                       / "+" / "-" / "." / "^" / "_" / "`" / "|" / "~"
                       / DIGIT / ALPHA
                       ; any VCHAR, except delimiters

## Expected behaviour

Specify the multipart boundary without quotes, unless they are required.

## Actual behaviour

Multipart boundary is always quoted.

## Steps to reproduce

Since I don't know which nginx version is affected, the only reliable way to reproduce right now is by filing a request at [share-online's upload API](http://www.share-online.biz/uploadapi).

A sample script is provided below (fill in your data):

<details>

```py
import asyncio
import pathlib

import aiohttp

SESSION_URL = "http://www.share-online.biz/upv3_session.php"
USER = ""
PASS = ""
path = pathlib.Path(".gitignore")


async def async_main():
    async with aiohttp.ClientSession() as session:
        data = {'username': USER, 'password': PASS}
        async with session.post(SESSION_URL, data=data) as resp:
            text = await resp.text()
        session_name, session_endpoint = text.split(";")
        session_endpoint = f"http://{session_endpoint}"

        file_size = path.stat().st_size
        with path.open('rb') as fp:
            data = aiohttp.FormData()
            data.add_field('username', USER)
            data.add_field('password', PASS)
            data.add_field('upload_session', session_name)
            data.add_field('chunk_no', "1")
            data.add_field('chunk_number', "1")
            data.add_field('filesize', str(file_size))
            data.add_field('fn', fp, filename=path.name, content_type="application/octet-steam")
            data.add_field('finalize', "1")

            print(f"Starting upload of {path.name}; size: {file_size}")
            async with session.post(session_endpoint, data=data) as resp:
                # print(f"response headers: {resp.headers}")
                resp.raise_for_status()
                text = await resp.text()
                print(f"reply: {text}")


def main():
    loop = asyncio.get_event_loop()
    task = asyncio.ensure_future(async_main())
    try:
        loop.run_until_complete(task)
    finally:
        loop.close()


if __name__ == '__main__':
    main()
```

</details>

With this script, the upload will fail with `*** EXCEPTION session creation/reuse failed - 11-21-2017, 3:13 pm ***`.

Removing the quotes at https://github.com/aio-libs/aiohttp/blob/439c732f5bf29a83779dce6da4a55e7efd433cf8/aiohttp/multipart.py#L640 makes the request work.

## Your environment

Arch Linux, Python 3.6, aiohttp 2.3.3 (client)


Discussion:

Resolving patch:
diff --git a/aiohttp/multipart.py b/aiohttp/multipart.py
--- a/aiohttp/multipart.py
+++ b/aiohttp/multipart.py
@@ -633,11 +633,18 @@ class MultipartWriter(Payload):
 
     def __init__(self, subtype='mixed', boundary=None):
         boundary = boundary if boundary is not None else uuid.uuid4().hex
+        # The underlying Payload API demands a str (utf-8), not bytes,
+        # so we need to ensure we don't lose anything during conversion.
+        # As a result, require the boundary to be ASCII only.
+        # In both situations.
+
         try:
-            self._boundary = boundary.encode('us-ascii')
+            self._boundary = boundary.encode('ascii')
         except UnicodeEncodeError:
-            raise ValueError('boundary should contains ASCII only chars')
-        ctype = 'multipart/{}; boundary="{}"'.format(subtype, boundary)
+            raise ValueError('boundary should contain ASCII only chars') \
+                from None
+        ctype = ('multipart/{}; boundary={}'
+                 .format(subtype, self._boundary_value))
 
         super().__init__(None, content_type=ctype)
 
@@ -657,6 +664,41 @@ def __iter__(self):
     def __len__(self):
         return len(self._parts)
 
+    _valid_tchar_regex = re.compile(br"\A[!#$%&'*+\-.^_`|~\w]+\Z")
+    _invalid_qdtext_char_regex = re.compile(br"[\x00-\x08\x0A-\x1F\x7F]")
+
+    @property
+    def _boundary_value(self):
+        """Wrap boundary parameter value in quotes, if necessary.
+
+        Reads self.boundary and returns a unicode sting.
+        """
+        # Refer to RFCs 7231, 7230, 5234.
+        #
+        # parameter      = token "=" ( token / quoted-string )
+        # token          = 1*tchar
+        # quoted-string  = DQUOTE *( qdtext / quoted-pair ) DQUOTE
+        # qdtext         = HTAB / SP / %x21 / %x23-5B / %x5D-7E / obs-text
+        # obs-text       = %x80-FF
+        # quoted-pair    = "\" ( HTAB / SP / VCHAR / obs-text )
+        # tchar          = "!" / "#" / "$" / "%" / "&" / "'" / "*"
+        #                  / "+" / "-" / "." / "^" / "_" / "`" / "|" / "~"
+        #                  / DIGIT / ALPHA
+        #                  ; any VCHAR, except delimiters
+        # VCHAR           = %x21-7E
+        value = self.boundary
+        if re.match(self._valid_tchar_regex, value):
+            return value.decode('ascii')  # cannot fail
+
+        if re.search(self._invalid_qdtext_char_regex, value):
+            raise ValueError("boundary value contains invalid characters")
+
+        # escape %x5C and %x22
+        quoted_value_content = value.replace(b'\\', b'\\\\')
+        quoted_value_content = quoted_value_content.replace(b'"', b'\\"')
+
+        return '"' + quoted_value_content.decode('ascii') + '"'
+
     @property
     def boundary(self):
         return self._boundary


Test patch:
diff --git a/tests/test_multipart.py b/tests/test_multipart.py
--- a/tests/test_multipart.py
+++ b/tests/test_multipart.py
@@ -945,9 +945,21 @@ def test_default_subtype(self):
         self.assertEqual('multipart', mimetype.type)
         self.assertEqual('mixed', mimetype.subtype)
 
+    def test_unquoted_boundary(self):
+        writer = aiohttp.multipart.MultipartWriter(boundary='abc123')
+        self.assertEqual({CONTENT_TYPE: 'multipart/mixed; boundary=abc123'},
+                         writer.headers)
+
+    def test_quoted_boundary(self):
+        writer = aiohttp.multipart.MultipartWriter(boundary=R'\"')
+        self.assertEqual({CONTENT_TYPE: R'multipart/mixed; boundary="\\\""'},
+                         writer.headers)
+
     def test_bad_boundary(self):
         with self.assertRaises(ValueError):
             aiohttp.multipart.MultipartWriter(boundary='тест')
+        with self.assertRaises(ValueError):
+            aiohttp.multipart.MultipartWriter(boundary='test\n')
 
     def test_default_headers(self):
         self.assertEqual({CONTENT_TYPE: 'multipart/mixed; boundary=":"'},