Repository: encode/httpx

Issue:
Refine headers behaviour for `Request()` and `client.build_request()`
Currently `Request()` auto-generates a bunch of headers on `__init__`. Having thought about it for a bit I can see some refinements we could make here to:

* Give users a bit more control over the default headers, such as allowing them to remove the `User-Agent` header.
* Make `Request()` more generally useful, so that it only auto sets mandatory headers.

Rather than set the following headings on `Request`, we should instead include them on `client.headers`...

* `User-Agent: python-httpx/1.2.3`
* `Accept: */*`
* `Accept-Encoding: gzip, deflate`
* `Connection: keep-alive`

When passing `Client(headers={...})` or setting `client.headers = {}` we should give precedence to explicitly set headers, but should *include* these defaults if they're not otherwise set.

```python
# Create a client, with a custom Accept header.
# Other automatic headers are still included.
client = httpx.Client(headers={'Accept': 'application/json, */*;q=0.8'}
```

Or...

```python
client = httpx.Client()

# Set a custom Accept header.
# Other automatic headers are still included.
client.headers = {'Accept': 'application/json, */*;q=0.8'}
```

If a user wants to *actually* remove one of these, they *can* do so by explicitly deleting it, eg...

```python
# Create a client, without a User-Agent header.
client = httpx.Client()
del client.headers['User-Agent']
```

The recommended way of creating a request instance will continue to be `client.build_request(...)`, but we'll want to highlight that more clearly, as instantiating a plain `Request()` instance will then give you a more basic set of behaviour by only auto-setting mandatory headers, and leaving everything else up to the user...

* Set a `Host` header, if the request URL is an absolute URL.
* Requests that mandate a body will always include either a `Content-Length` or `Transfer-Encoding` header.

Both of those cases will defer to any explicitly included header if one exists, and only auto set if those mandatory headers are omitted.

Also `.prepare_headers()` should become a private method.

This all ends up being beneficial, because users still get all the regular convenience of `client.build_request()`, but there's also a plainer style for creating `Request()` instances with complete control over the headers which doesn't make any assumptions about auto-including a set of default headers.

(Somewhat refs #1091)


Discussion:

Resolving patch:
diff --git a/httpx/_client.py b/httpx/_client.py
--- a/httpx/_client.py
+++ b/httpx/_client.py
@@ -5,6 +5,7 @@
 
 import httpcore
 
+from .__version__ import __version__
 from ._auth import Auth, BasicAuth, FunctionAuth
 from ._config import (
     DEFAULT_LIMITS,
@@ -18,6 +19,7 @@
     create_ssl_context,
 )
 from ._content_streams import ContentStream
+from ._decoders import SUPPORTED_DECODERS
 from ._exceptions import (
     HTTPCORE_EXC_MAP,
     InvalidURL,
@@ -55,6 +57,10 @@
 logger = get_logger(__name__)
 
 KEEPALIVE_EXPIRY = 5.0
+USER_AGENT = f"python-httpx/{__version__}"
+ACCEPT_ENCODING = ", ".join(
+    [key for key in SUPPORTED_DECODERS.keys() if key != "identity"]
+)
 
 
 class BaseClient:
@@ -74,7 +80,7 @@ def __init__(
 
         self._auth = self._build_auth(auth)
         self._params = QueryParams(params)
-        self._headers = Headers(headers)
+        self.headers = Headers(headers)
         self._cookies = Cookies(cookies)
         self._timeout = Timeout(timeout)
         self.max_redirects = max_redirects
@@ -161,7 +167,16 @@ def headers(self) -> Headers:
 
     @headers.setter
     def headers(self, headers: HeaderTypes) -> None:
-        self._headers = Headers(headers)
+        client_headers = Headers(
+            {
+                b"Accept": b"*/*",
+                b"Accept-Encoding": ACCEPT_ENCODING.encode("ascii"),
+                b"Connection": b"keep-alive",
+                b"User-Agent": USER_AGENT.encode("ascii"),
+            }
+        )
+        client_headers.update(headers)
+        self._headers = client_headers
 
     @property
     def cookies(self) -> Cookies:
@@ -299,11 +314,9 @@ def _merge_headers(
         Merge a headers argument together with any headers on the client,
         to create the headers used for the outgoing request.
         """
-        if headers or self.headers:
-            merged_headers = Headers(self.headers)
-            merged_headers.update(headers)
-            return merged_headers
-        return headers
+        merged_headers = Headers(self.headers)
+        merged_headers.update(headers)
+        return merged_headers
 
     def _merge_queryparams(
         self, params: QueryParamTypes = None
diff --git a/httpx/_models.py b/httpx/_models.py
--- a/httpx/_models.py
+++ b/httpx/_models.py
@@ -14,7 +14,6 @@
 import rfc3986
 import rfc3986.exceptions
 
-from .__version__ import __version__
 from ._content_streams import ByteStream, ContentStream, encode
 from ._decoders import (
     SUPPORTED_DECODERS,
@@ -103,13 +102,11 @@ def userinfo(self) -> str:
 
     @property
     def username(self) -> str:
-        userinfo = self._uri_reference.userinfo or ""
-        return unquote(userinfo.partition(":")[0])
+        return unquote(self.userinfo.partition(":")[0])
 
     @property
     def password(self) -> str:
-        userinfo = self._uri_reference.userinfo or ""
-        return unquote(userinfo.partition(":")[2])
+        return unquote(self.userinfo.partition(":")[2])
 
     @property
     def host(self) -> str:
@@ -580,12 +577,6 @@ def getlist(self, key: str, split_commas: bool = False) -> typing.List[str]:
         return self.get_list(key, split_commas=split_commas)
 
 
-USER_AGENT = f"python-httpx/{__version__}"
-ACCEPT_ENCODING = ", ".join(
-    [key for key in SUPPORTED_DECODERS.keys() if key != "identity"]
-)
-
-
 class Request:
     def __init__(
         self,
@@ -627,26 +618,12 @@ def prepare(self) -> None:
         has_content_length = (
             "content-length" in self.headers or "transfer-encoding" in self.headers
         )
-        has_user_agent = "user-agent" in self.headers
-        has_accept = "accept" in self.headers
-        has_accept_encoding = "accept-encoding" in self.headers
-        has_connection = "connection" in self.headers
-
-        if not has_host:
-            url = self.url
-            if url.userinfo:
-                url = url.copy_with(username=None, password=None)
-            auto_headers.append((b"host", url.authority.encode("ascii")))
+
+        if not has_host and self.url.authority:
+            host = self.url.copy_with(username=None, password=None).authority
+            auto_headers.append((b"host", host.encode("ascii")))
         if not has_content_length and self.method in ("POST", "PUT", "PATCH"):
             auto_headers.append((b"content-length", b"0"))
-        if not has_user_agent:
-            auto_headers.append((b"user-agent", USER_AGENT.encode("ascii")))
-        if not has_accept:
-            auto_headers.append((b"accept", b"*/*"))
-        if not has_accept_encoding:
-            auto_headers.append((b"accept-encoding", ACCEPT_ENCODING.encode()))
-        if not has_connection:
-            auto_headers.append((b"connection", b"keep-alive"))
 
         self.headers = Headers(auto_headers + self.headers.raw)
 


Test patch:
diff --git a/tests/client/test_headers.py b/tests/client/test_headers.py
--- a/tests/client/test_headers.py
+++ b/tests/client/test_headers.py
@@ -124,6 +124,28 @@ def test_header_update():
     }
 
 
+def test_remove_default_header():
+    """
+    Remove a default header from the Client.
+    """
+    url = "http://example.org/echo_headers"
+
+    client = httpx.Client(transport=MockTransport())
+    del client.headers["User-Agent"]
+
+    response = client.get(url)
+
+    assert response.status_code == 200
+    assert response.json() == {
+        "headers": {
+            "accept": "*/*",
+            "accept-encoding": "gzip, deflate, br",
+            "connection": "keep-alive",
+            "host": "example.org",
+        }
+    }
+
+
 def test_header_does_not_exist():
     headers = httpx.Headers({"foo": "bar"})
     with pytest.raises(KeyError):
diff --git a/tests/models/test_requests.py b/tests/models/test_requests.py
--- a/tests/models/test_requests.py
+++ b/tests/models/test_requests.py
@@ -34,6 +34,18 @@ def test_json_encoded_data():
     assert request.content == b'{"test": 123}'
 
 
+def test_headers():
+    request = httpx.Request("POST", "http://example.org", json={"test": 123})
+
+    assert request.headers == httpx.Headers(
+        {
+            "Host": "example.org",
+            "Content-Type": "application/json",
+            "Content-Length": "13",
+        }
+    )
+
+
 def test_read_and_stream_data():
     # Ensure a request may still be streamed if it has been read.
     # Needed for cases such as authentication classes that read the request body.