Repository: encode/httpx

Issue:
DecodingError with zlib-compressed responses
Using the following minimal WSGI app:

```python
import zlib

def app(environ, start_response):
    start_response("200 OK", [("Content-Encoding", "deflate")])
    return [zlib.compress(b"hello world")]
```

This works fine in a web browser.

Requests is quite happy:

```
>>> import requests
>>> requests.get("http://localhost:9999")
<Response [200]>
>>> requests.get("http://localhost:9999").text
'hello world'
```

httpx says:

```
>>> httpx.get('http://localhost:9999')
Traceback (most recent call last):
  File "/Users/jamie/code/httpx/httpx/decoders.py", line 52, in decode
    return self.decompressor.decompress(data)
zlib.error: Error -3 while decompressing data: invalid stored block lengths

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/Users/jamie/code/httpx/httpx/api.py", line 170, in get
    trust_env=trust_env,
  File "/Users/jamie/code/httpx/httpx/api.py", line 96, in request
    allow_redirects=allow_redirects,
  File "/Users/jamie/code/httpx/httpx/client.py", line 568, in request
    request, auth=auth, allow_redirects=allow_redirects, timeout=timeout,
  File "/Users/jamie/code/httpx/httpx/client.py", line 593, in send
    response.read()
  File "/Users/jamie/code/httpx/httpx/models.py", line 900, in read
    self._content = b"".join([part for part in self.iter_bytes()])
  File "/Users/jamie/code/httpx/httpx/models.py", line 900, in <listcomp>
    self._content = b"".join([part for part in self.iter_bytes()])
  File "/Users/jamie/code/httpx/httpx/models.py", line 912, in iter_bytes
    yield self.decoder.decode(chunk)
  File "/Users/jamie/code/httpx/httpx/decoders.py", line 54, in decode
    raise DecodingError from exc
httpx.exceptions.DecodingError
```

The reason this is an issue and not a PR is because I don't understand the code in `decoders.py` - it very explicitly says `self.decompressor = zlib.decompressobj(-zlib.MAX_WBITS)`, which according to [the Python docs](https://docs.python.org/3/library/zlib.html#zlib.compressobj) means "Uses the absolute value of wbits as the window size logarithm, while producing a raw output stream with no header or trailing checksum." I don't know enough about zlib to understand this, but it looks very deliberate so I thought I'd better ask.

Discussion:

Patch:
diff --git a/docs/quickstart.md b/docs/quickstart.md
index 25c1cea..a147b04 100644
--- a/docs/quickstart.md
+++ b/docs/quickstart.md
@@ -255,7 +255,7 @@ httpx.exceptions.HttpError: 404 Not Found
 
 Any successful response codes will simply return `None` rather than raising an exception.
 
-``` python
+```python
 >>> r.raise_for_status()
 ```
 
@@ -296,7 +296,7 @@ For large downloads you may want to use streaming responses that do not load the
 
 You can stream the binary content of the response...
 
-```
+```python
 >>> with httpx.stream("GET", "https://www.example.com") as r:
 ...     for data in r.iter_bytes():
 ...         print(data)
@@ -304,7 +304,7 @@ You can stream the binary content of the response...
 
 Or the text of the response...
 
-```
+```python
 >>> with httpx.stream("GET", "https://www.example.com") as r:
 ...     for text in r.iter_text():
 ...         print(text)
@@ -312,7 +312,7 @@ Or the text of the response...
 
 Or stream the text, on a line-by-line basis...
 
-```
+```python
 >>> with httpx.stream("GET", "https://www.example.com") as r:
 ...     for line in r.iter_lines():
 ...         print(line)
@@ -322,7 +322,7 @@ HTTPX will use universal line endings, normalising all cases to `\n`.
 
 In some cases you might want to access the raw bytes on the response without applying any HTTP content decoding. In this case any content encoding that the web server has applied such as `gzip`, `deflate`, or `brotli` will not be automatically decoded.
 
-```
+```python
 >>> with httpx.stream("GET", "https://www.example.com") as r:
 ...     for chunk in r.iter_raw():
 ...         print(chunk)
diff --git a/httpx/decoders.py b/httpx/decoders.py
index 75d980d..454ec4a 100644
--- a/httpx/decoders.py
+++ b/httpx/decoders.py
@@ -45,12 +45,18 @@ class DeflateDecoder(Decoder):
     """
 
     def __init__(self) -> None:
-        self.decompressor = zlib.decompressobj(-zlib.MAX_WBITS)
+        self.first_attempt = True
+        self.decompressor = zlib.decompressobj()
 
     def decode(self, data: bytes) -> bytes:
+        was_first_attempt = self.first_attempt
+        self.first_attempt = False
         try:
             return self.decompressor.decompress(data)
         except zlib.error as exc:
+            if was_first_attempt:
+                self.decompressor = zlib.decompressobj(-zlib.MAX_WBITS)
+                return self.decode(data)
             raise DecodingError from exc
 
     def flush(self) -> bytes:
diff --git a/setup.py b/setup.py
index 4d92524..31ad5ea 100644
--- a/setup.py
+++ b/setup.py
@@ -40,6 +40,10 @@ setup(
     python_requires=">=3.6",
     version=get_version("httpx"),
     url="https://github.com/encode/httpx",
+    project_urls={
+        "Documentation": "https://www.python-httpx.org",
+        "Source": "https://github.com/encode/httpx",
+    },
     license="BSD",
     description="The next generation HTTP client.",
     long_description=get_long_description(),
@@ -74,5 +78,6 @@ setup(
         "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3 :: Only",
     ],
 )


Test patch:
diff --git a/tests/test_decoders.py b/tests/test_decoders.py
index f320d94..d9e82f7 100644
--- a/tests/test_decoders.py
+++ b/tests/test_decoders.py
@@ -18,6 +18,11 @@ REQUEST = httpx.Request("GET", "https://example.org")
 
 
 def test_deflate():
+    """
+    Deflate encoding may use either 'zlib' or 'deflate' in the wild.
+
+    https://stackoverflow.com/questions/1838699/how-can-i-decompress-a-gzip-stream-with-zlib#answer-22311297
+    """
     body = b"test 123"
     compressor = zlib.compressobj(9, zlib.DEFLATED, -zlib.MAX_WBITS)
     compressed_body = compressor.compress(body) + compressor.flush()
@@ -29,6 +34,22 @@ def test_deflate():
     assert response.content == body
 
 
+def test_zlib():
+    """
+    Deflate encoding may use either 'zlib' or 'deflate' in the wild.
+
+    https://stackoverflow.com/questions/1838699/how-can-i-decompress-a-gzip-stream-with-zlib#answer-22311297
+    """
+    body = b"test 123"
+    compressed_body = zlib.compress(body)
+
+    headers = [(b"Content-Encoding", b"deflate")]
+    response = httpx.Response(
+        200, headers=headers, content=compressed_body, request=REQUEST
+    )
+    assert response.content == body
+
+
 def test_gzip():
     body = b"test 123"
     compressor = zlib.compressobj(9, zlib.DEFLATED, zlib.MAX_WBITS | 16)