Repository: encode/httpx
Pull request #569 (2019-11-30 11:22:42)

Don't use background task in HTTP/1.1 dispatcher
As mentioned in https://github.com/encode/httpx/issues/565#issuecomment-559798817, sending the request and receiving the response concurrently is not necessary for HTTP/1.1


Patch:
diff --git a/docs/advanced.md b/docs/advanced.md
index 7f1fc52..bc1d0f8 100644
--- a/docs/advanced.md
+++ b/docs/advanced.md
@@ -75,7 +75,7 @@ For a list of all available client-level parameters, see the [`Client` API refer
 ## Calling into Python Web Apps
 
 You can configure an `httpx` client to call directly into a Python web
-application using either the ASGI protocol.
+application using the ASGI protocol.
 
 This is particularly useful for two main use-cases:
 
@@ -107,22 +107,25 @@ We can make requests directly against the application, like so:
 ...     assert r.text == "Hello World!"
 ```
 
-For some more complex cases you might need to customize the WSGI or ASGI
-dispatch. This allows you to:
+For some more complex cases you might need to customise the ASGI dispatch. This allows you to:
 
 * Inspect 500 error responses rather than raise exceptions by setting `raise_app_exceptions=False`.
-* Mount the WSGI or ASGI application at a subpath by setting `root_path`.
+* Mount the ASGI application at a subpath by setting `root_path`.
 * Use a given client address for requests by setting `client`.
 
 For example:
 
 ```python
-# Instantiate a client that makes WSGI requests with a client IP of "1.2.3.4".
-dispatch = httpx.dispatch.ASGIDispatch(app=app, remote_addr="1.2.3.4")
+# Instantiate a client that makes ASGI requests with a client IP of "1.2.3.4",
+# on port 123.
+dispatch = httpx.dispatch.ASGIDispatch(app=app, client=("1.2.3.4", 123))
 async with httpx.Client(dispatch=dispatch) as client:
     ...
 ```
 
+See [the ASGI documentation](https://asgi.readthedocs.io/en/latest/specs/www.html#connection-scope) for more details on the `client` and `root_path`
+keys.
+
 ## Build Request
 
 You can use `Client.build_request()` to build a request and
@@ -360,6 +363,7 @@ name of the payloads as keys and a tuple of elements as values.
 ```
 
 More specifically, this tuple must have at least two elements and maximum of three:
+
 - The first one is an optional file name which can be set to `None`.
 - The second may be a file-like object or a string which will be automatically
 encoded in UTF-8.
diff --git a/docs/compatibility.md b/docs/compatibility.md
index a95c12c..d652a0a 100644
--- a/docs/compatibility.md
+++ b/docs/compatibility.md
@@ -14,6 +14,7 @@ to the API in our own documentation. The following exceptions apply:
 but also provide lower-cased versions for API compatibility with `requests`.
 * `stream=True`. - Streaming responses provide the `.stream()` and `.raw()` byte iterator interfaces, rather than the `.iter_content()` method and the `.raw` socket interface.
 * `.get`, `.delete`, `.head`, `.options` -  These methods do not support `files`, `data`, or `json` arguments. Use `.request` if you need to need to send data using these http methods.
+* We don't support `response.is_ok` since the naming is ambiguous there, and might incorrectly imply an equivalence to `response.status_code == codes.OK`. Instead we provide an `.is_error` property.
 
 ## Advanced Usage
 
diff --git a/httpx/client.py b/httpx/client.py
index 4c594f2..1fe4f6c 100644
--- a/httpx/client.py
+++ b/httpx/client.py
@@ -384,7 +384,7 @@ class Client:
             auth = BasicAuth(username=username, password=password)
             return auth(request)
 
-        if trust_env:
+        if trust_env and "Authorization" not in request.headers:
             credentials = self.netrc.get_credentials(request.url.authority)
             if credentials is not None:
                 auth = BasicAuth(username=credentials[0], password=credentials[1])
diff --git a/httpx/decoders.py b/httpx/decoders.py
index 3ae822b..22c6104 100644
--- a/httpx/decoders.py
+++ b/httpx/decoders.py
@@ -211,6 +211,67 @@ class TextDecoder:
         return result
 
 
+class LineDecoder:
+    """
+    Handles incrementally reading lines from text.
+
+    Uses universal line decoding, supporting any of `\n`, `\r`, or `\r\n`
+    as line endings, normalizing to `\n`.
+    """
+
+    def __init__(self) -> None:
+        self.buffer = ""
+
+    def decode(self, text: str) -> typing.List[str]:
+        lines = []
+
+        if text.startswith("\n") and self.buffer and self.buffer[-1] == "\r":
+            # Handle the case where we have an "\r\n" split across
+            # our previous input, and our new chunk.
+            lines.append(self.buffer[:-1] + "\n")
+            self.buffer = ""
+            text = text[1:]
+
+        while text:
+            num_chars = len(text)
+            for idx in range(num_chars):
+                char = text[idx]
+                next_char = None if idx + 1 == num_chars else text[idx + 1]
+                if char == "\n":
+                    lines.append(self.buffer + text[: idx + 1])
+                    self.buffer = ""
+                    text = text[idx + 1 :]
+                    break
+                elif char == "\r" and next_char == "\n":
+                    lines.append(self.buffer + text[:idx] + "\n")
+                    self.buffer = ""
+                    text = text[idx + 2 :]
+                    break
+                elif char == "\r" and next_char is not None:
+                    lines.append(self.buffer + text[:idx] + "\n")
+                    self.buffer = ""
+                    text = text[idx + 1 :]
+                    break
+                elif next_char is None:
+                    self.buffer = text
+                    text = ""
+                    break
+
+        return lines
+
+    def flush(self) -> typing.List[str]:
+        if self.buffer.endswith("\r"):
+            # Handle the case where we had a trailing '\r', which could have
+            # been a '\r\n' pair.
+            lines = [self.buffer[:-1] + "\n"]
+        elif self.buffer:
+            lines = [self.buffer]
+        else:
+            lines = []
+        self.buffer = ""
+        return lines
+
+
 SUPPORTED_DECODERS = {
     "identity": IdentityDecoder,
     "gzip": GZipDecoder,
diff --git a/httpx/dispatch/asgi.py b/httpx/dispatch/asgi.py
index f9cbb35..ee75deb 100644
--- a/httpx/dispatch/asgi.py
+++ b/httpx/dispatch/asgi.py
@@ -4,11 +4,8 @@ from ..concurrency.asyncio import AsyncioBackend
 from ..concurrency.base import ConcurrencyBackend
 from ..config import CertTypes, TimeoutTypes, VerifyTypes
 from ..models import Request, Response
-from ..utils import MessageLoggerASGIMiddleware, get_logger
 from .base import Dispatcher
 
-logger = get_logger(__name__)
-
 
 class ASGIDispatch(Dispatcher):
     """
@@ -81,7 +78,6 @@ class ASGIDispatch(Dispatcher):
             "client": self.client,
             "root_path": self.root_path,
         }
-        app = MessageLoggerASGIMiddleware(self.app, logger=logger)
         status_code = None
         headers = None
         body_parts = []
@@ -119,7 +115,7 @@ class ASGIDispatch(Dispatcher):
                     response_complete = True
 
         try:
-            await app(scope, receive, send)
+            await self.app(scope, receive, send)
         except Exception:
             if self.raise_app_exceptions or not response_complete:
                 raise
diff --git a/httpx/dispatch/http11.py b/httpx/dispatch/http11.py
index 058691a..8202781 100644
--- a/httpx/dispatch/http11.py
+++ b/httpx/dispatch/http11.py
@@ -46,10 +46,8 @@ class HTTP11Connection:
         timeout = None if timeout is None else TimeoutConfig(timeout)
 
         await self._send_request(request, timeout)
-
-        task, args = self._send_request_data, [request.stream(), timeout]
-        async with self.backend.background_manager(task, *args):
-            http_version, status_code, headers = await self._receive_response(timeout)
+        await self._send_request_body(request, timeout)
+        http_version, status_code, headers = await self._receive_response(timeout)
         content = self._receive_response_data(timeout)
 
         return Response(
@@ -89,15 +87,15 @@ class HTTP11Connection:
         event = h11.Request(method=method, target=target, headers=headers)
         await self._send_event(event, timeout)
 
-    async def _send_request_data(
-        self, data: typing.AsyncIterator[bytes], timeout: TimeoutConfig = None
+    async def _send_request_body(
+        self, request: Request, timeout: TimeoutConfig = None
     ) -> None:
         """
         Send the request body to the network.
         """
         try:
             # Send the request body.
-            async for chunk in data:
+            async for chunk in request.stream():
                 logger.trace(f"send_data data=Data(<{len(chunk)} bytes>)")
                 event = h11.Data(data=chunk)
                 await self._send_event(event, timeout)
diff --git a/httpx/models.py b/httpx/models.py
index 0b14734..d702903 100644
--- a/httpx/models.py
+++ b/httpx/models.py
@@ -17,6 +17,7 @@ from .decoders import (
     SUPPORTED_DECODERS,
     Decoder,
     IdentityDecoder,
+    LineDecoder,
     MultiDecoder,
     TextDecoder,
 )
@@ -80,11 +81,18 @@ RequestData = typing.Union[dict, str, bytes, typing.AsyncIterator[bytes]]
 RequestFiles = typing.Dict[
     str,
     typing.Union[
-        typing.IO[typing.AnyStr],  # file
-        typing.Tuple[str, typing.IO[typing.AnyStr]],  # (filename, file)
+        # file (or str)
+        typing.Union[typing.IO[typing.AnyStr], typing.AnyStr],
+        # (filename, file (or str))
         typing.Tuple[
-            str, typing.IO[typing.AnyStr], str
-        ],  # (filename, file, content_type)
+            typing.Optional[str], typing.Union[typing.IO[typing.AnyStr], typing.AnyStr],
+        ],
+        # (filename, file (or str), content_type)
+        typing.Tuple[
+            typing.Optional[str],
+            typing.Union[typing.IO[typing.AnyStr], typing.AnyStr],
+            typing.Optional[str],
+        ],
     ],
 ]
 
@@ -345,7 +353,12 @@ class QueryParams(typing.Mapping[str, str]):
 
         params = QueryParams(params)
         for param in params:
-            self[param] = params[param]
+            item, *extras = params.getlist(param)
+            self[param] = item
+            if extras:
+                self._list.extend((param, e) for e in extras)
+                # ensure getter matches merged QueryParams getter
+                self._dict[param] = params[param]
 
     def __getitem__(self, key: typing.Any) -> str:
         return self._dict[key]
@@ -841,6 +854,10 @@ class Response:
 
         return self._decoder
 
+    @property
+    def is_error(self) -> bool:
+        return StatusCode.is_error(self.status_code)
+
     @property
     def is_redirect(self) -> bool:
         return StatusCode.is_redirect(self.status_code) and "location" in self.headers
@@ -856,11 +873,9 @@ class Response:
 
         if StatusCode.is_client_error(self.status_code):
             message = message.format(self, error_type="Client Error")
+            raise HTTPError(message, response=self)
         elif StatusCode.is_server_error(self.status_code):
             message = message.format(self, error_type="Server Error")
-        else:
-            message = ""
-        if message:
             raise HTTPError(message, response=self)
 
     def json(self, **kwargs: typing.Any) -> typing.Union[dict, list]:
@@ -929,6 +944,14 @@ class Response:
             yield decoder.decode(chunk)
         yield decoder.flush()
 
+    async def stream_lines(self) -> typing.AsyncIterator[str]:
+        decoder = LineDecoder()
+        async for text in self.stream_text():
+            for line in decoder.decode(text):
+                yield line
+        for line in decoder.flush():
+            yield line
+
     async def raw(self) -> typing.AsyncIterator[bytes]:
         """
         A byte-iterator over the raw response content.
diff --git a/httpx/multipart.py b/httpx/multipart.py
index fcfd256..a464279 100644
--- a/httpx/multipart.py
+++ b/httpx/multipart.py
@@ -58,19 +58,21 @@ class FileField(Field):
                 value[2] if len(value) > 2 else self.guess_content_type()
             )
 
-    def guess_content_type(self) -> str:
+    def guess_content_type(self) -> typing.Optional[str]:
         if self.filename:
             return mimetypes.guess_type(self.filename)[0] or "application/octet-stream"
         else:
-            return "application/octet-stream"
+            return None
 
     def render_headers(self) -> bytes:
         parts = [b"Content-Disposition: form-data; ", _format_param("name", self.name)]
         if self.filename:
             filename = _format_param("filename", self.filename)
             parts.extend([b"; ", filename])
-        content_type = self.content_type.encode()
-        parts.extend([b"\r\nContent-Type: ", content_type, b"\r\n\r\n"])
+        if self.content_type is not None:
+            content_type = self.content_type.encode()
+            parts.extend([b"\r\nContent-Type: ", content_type])
+        parts.append(b"\r\n\r\n")
         return b"".join(parts)
 
     def render_data(self) -> bytes:
diff --git a/httpx/status_codes.py b/httpx/status_codes.py
index 3be26ca..eda7c56 100644
--- a/httpx/status_codes.py
+++ b/httpx/status_codes.py
@@ -49,6 +49,10 @@ class StatusCode(IntEnum):
             StatusCode.PERMANENT_REDIRECT,
         )
 
+    @classmethod
+    def is_error(cls, value: int) -> bool:
+        return 400 <= value <= 599
+
     @classmethod
     def is_client_error(cls, value: int) -> bool:
         return 400 <= value <= 499
diff --git a/httpx/utils.py b/httpx/utils.py
index 07277a3..9d3aa22 100644
--- a/httpx/utils.py
+++ b/httpx/utils.py
@@ -339,64 +339,3 @@ class ElapsedTimer:
         if self.end is None:
             return timedelta(seconds=perf_counter() - self.start)
         return timedelta(seconds=self.end - self.start)
-
-
-ASGI_PLACEHOLDER_FORMAT = {
-    "body": "<{length} bytes>",
-    "bytes": "<{length} bytes>",
-    "text": "<{length} chars>",
-}
-
-
-def asgi_message_with_placeholders(message: dict) -> dict:
-    """
-    Return an ASGI message, with any body-type content omitted and replaced
-    with a placeholder.
-    """
-    new_message = message.copy()
-
-    for attr in ASGI_PLACEHOLDER_FORMAT:
-        if attr in message:
-            content = message[attr]
-            placeholder = ASGI_PLACEHOLDER_FORMAT[attr].format(length=len(content))
-            new_message[attr] = placeholder
-
-    if "headers" in message:
-        new_message["headers"] = list(obfuscate_sensitive_headers(message["headers"]))
-
-    return new_message
-
-
-class MessageLoggerASGIMiddleware:
-    def __init__(self, app: typing.Callable, logger: Logger) -> None:
-        self.app = app
-        self.logger = logger
-
-    async def __call__(
-        self, scope: dict, receive: typing.Callable, send: typing.Callable
-    ) -> None:
-        async def inner_receive() -> dict:
-            message = await receive()
-            logged_message = asgi_message_with_placeholders(message)
-            self.logger.trace(f"sent {kv_format(**logged_message)}")
-            return message
-
-        async def inner_send(message: dict) -> None:
-            logged_message = asgi_message_with_placeholders(message)
-            self.logger.trace(f"received {kv_format(**logged_message)}")
-            await send(message)
-
-        logged_scope = dict(scope)
-        if "headers" in scope:
-            logged_scope["headers"] = list(
-                obfuscate_sensitive_headers(scope["headers"])
-            )
-        self.logger.trace(f"started {kv_format(**logged_scope)}")
-
-        try:
-            await self.app(scope, inner_receive, inner_send)
-        except BaseException as exc:
-            self.logger.trace("raised_exception")
-            raise exc from None
-        else:
-            self.logger.trace("completed")


Test patch:
diff --git a/tests/client/test_auth.py b/tests/client/test_auth.py
index 58999cf..d81ddb1 100644
--- a/tests/client/test_auth.py
+++ b/tests/client/test_auth.py
@@ -156,6 +156,18 @@ async def test_netrc_auth():
     }
 
 
+@pytest.mark.asyncio
+async def test_auth_header_has_priority_over_netrc():
+    os.environ["NETRC"] = "tests/.netrc"
+    url = "http://netrcexample.org"
+
+    client = Client(dispatch=MockDispatch())
+    response = await client.get(url, headers={"Authorization": "Override"})
+
+    assert response.status_code == 200
+    assert response.json() == {"auth": "Override"}
+
+
 @pytest.mark.asyncio
 async def test_trust_env_auth():
     os.environ["NETRC"] = "tests/.netrc"
diff --git a/tests/models/test_queryparams.py b/tests/models/test_queryparams.py
index 1303170..0b0101c 100644
--- a/tests/models/test_queryparams.py
+++ b/tests/models/test_queryparams.py
@@ -62,6 +62,9 @@ def test_queryparam_types():
     q = QueryParams({"a": 123})
     assert str(q) == "a=123"
 
+    q = QueryParams({"a": [1, 2]})
+    assert str(q) == "a=1&a=2"
+
 
 def test_queryparam_setters():
     q = QueryParams({"a": 1})
@@ -72,3 +75,10 @@ def test_queryparam_setters():
     q = QueryParams([("a", 1), ("a", 2)])
     q["a"] = "3"
     assert str(q) == "a=3"
+
+    q = QueryParams([("a", 1), ("b", 1)])
+    u = QueryParams([("b", 2), ("b", 3)])
+    q.update(u)
+
+    assert str(q) == "a=1&b=2&b=3"
+    assert q["b"] == u["b"]
diff --git a/tests/models/test_responses.py b/tests/models/test_responses.py
index 012d2b7..9b60c67 100644
--- a/tests/models/test_responses.py
+++ b/tests/models/test_responses.py
@@ -23,6 +23,7 @@ def test_response():
     assert response.reason_phrase == "OK"
     assert response.text == "Hello, world!"
     assert response.elapsed == datetime.timedelta(0)
+    assert not response.is_error
 
 
 def test_response_repr():
@@ -163,6 +164,18 @@ async def test_stream_text():
     assert content == "Hello, world!"
 
 
+@pytest.mark.asyncio
+async def test_stream_lines():
+    response = httpx.Response(200, content=b"Hello,\nworld!")
+
+    await response.read()
+
+    content = []
+    async for line in response.stream_lines():
+        content.append(line)
+    assert content == ["Hello,\n", "world!"]
+
+
 @pytest.mark.asyncio
 async def test_stream_interface_after_read():
     response = httpx.Response(200, content=b"Hello, world!")
diff --git a/tests/test_decoders.py b/tests/test_decoders.py
index 7525239..a599ce0 100644
--- a/tests/test_decoders.py
+++ b/tests/test_decoders.py
@@ -9,6 +9,7 @@ from httpx.decoders import (
     DeflateDecoder,
     GZipDecoder,
     IdentityDecoder,
+    LineDecoder,
     TextDecoder,
 )
 
@@ -167,6 +168,48 @@ def test_text_decoder_empty_cases():
     assert decoder.flush() == ""
 
 
+def test_line_decoder_nl():
+    decoder = LineDecoder()
+    assert decoder.decode("") == []
+    assert decoder.decode("a\n\nb\nc") == ["a\n", "\n", "b\n"]
+    assert decoder.flush() == ["c"]
+
+    decoder = LineDecoder()
+    assert decoder.decode("") == []
+    assert decoder.decode("a\n\nb\nc\n") == ["a\n", "\n", "b\n", "c\n"]
+    assert decoder.flush() == []
+
+
+def test_line_decoder_cr():
+    decoder = LineDecoder()
+    assert decoder.decode("") == []
+    assert decoder.decode("a\r\rb\rc") == ["a\n", "\n", "b\n"]
+    assert decoder.flush() == ["c"]
+
+    decoder = LineDecoder()
+    assert decoder.decode("") == []
+    assert decoder.decode("a\r\rb\rc\r") == ["a\n", "\n", "b\n"]
+    assert decoder.flush() == ["c\n"]
+
+
+def test_line_decoder_crnl():
+    decoder = LineDecoder()
+    assert decoder.decode("") == []
+    assert decoder.decode("a\r\n\r\nb\r\nc") == ["a\n", "\n", "b\n"]
+    assert decoder.flush() == ["c"]
+
+    decoder = LineDecoder()
+    assert decoder.decode("") == []
+    assert decoder.decode("a\r\n\r\nb\r\nc\r\n") == ["a\n", "\n", "b\n", "c\n"]
+    assert decoder.flush() == []
+
+    decoder = LineDecoder()
+    assert decoder.decode("") == []
+    assert decoder.decode("a\r") == []
+    assert decoder.decode("\n\r\nb\r\nc") == ["a\n", "\n", "b\n"]
+    assert decoder.flush() == ["c"]
+
+
 def test_invalid_content_encoding_header():
     headers = [(b"Content-Encoding", b"invalid-header")]
     body = b"test 123"
diff --git a/tests/test_multipart.py b/tests/test_multipart.py
index 8772702..98f8468 100644
--- a/tests/test_multipart.py
+++ b/tests/test_multipart.py
@@ -141,13 +141,34 @@ def test_multipart_encode_files_allows_filenames_as_none():
 
         assert content_type == f"multipart/form-data; boundary={boundary}"
         assert body == (
-            '--{0}\r\nContent-Disposition: form-data; name="file"\r\n'
-            "Content-Type: application/octet-stream\r\n\r\n<file content>\r\n"
-            "--{0}--\r\n"
+            '--{0}\r\nContent-Disposition: form-data; name="file"\r\n\r\n'
+            "<file content>\r\n--{0}--\r\n"
             "".format(boundary).encode("ascii")
         )
 
 
+@pytest.mark.parametrize(
+    "file_name,expected_content_type",
+    [("example.json", "application/json"), ("example.log", "application/octet-stream")],
+)
+def test_multipart_encode_files_guesses_correct_content_type(
+    file_name, expected_content_type
+):
+    files = {"file": (file_name, io.BytesIO(b"<file content>"))}
+    with mock.patch("os.urandom", return_value=os.urandom(16)):
+        boundary = binascii.hexlify(os.urandom(16)).decode("ascii")
+
+        body, content_type = multipart.multipart_encode(data={}, files=files)
+
+        assert content_type == f"multipart/form-data; boundary={boundary}"
+        assert body == (
+            f'--{boundary}\r\nContent-Disposition: form-data; name="file"; '
+            f'filename="{file_name}"\r\nContent-Type: '
+            f"{expected_content_type}\r\n\r\n<file content>\r\n--{boundary}--\r\n"
+            "".encode("ascii")
+        )
+
+
 def test_multipart_encode_files_allows_str_content():
     files = {"file": ("test.txt", "<string content>", "text/plain")}
     with mock.patch("os.urandom", return_value=os.urandom(16)):