Repository: urllib3/urllib3

Issue:
[v2] Add support for Content-Encoding: zstd
See: https://tools.ietf.org/html/rfc8478

### Minimum requirements

:moneybag: **You can get paid to complete this issue! [Please read the docs for more information](https://urllib3.readthedocs.io/en/latest/contributing.html#getting-paid-for-your-contributions).**

- [ ] Evaluate all zstandard packages that are available for Python, which makes sense for us to support? One, multiple?
- [ ] Add support for a `ZstandardDecoder`, don't enable `zstd` in `Accept-Encoding` by default though.
- [ ] Add documentation for the feature
- [ ] Add test cases for the feature, see `BrotliDecoder` and `Content-Encoding: brotli` tests for inspiration.


Discussion:

Resolving patch:
diff --git a/noxfile.py b/noxfile.py
--- a/noxfile.py
+++ b/noxfile.py
@@ -16,7 +16,7 @@
 
 def tests_impl(
     session: nox.Session,
-    extras: str = "socks,secure,brotli",
+    extras: str = "socks,secure,brotli,zstd",
     byte_string_comparisons: bool = True,
 ) -> None:
     # Install deps and the package itself.
@@ -168,7 +168,7 @@ def mypy(session: nox.Session) -> None:
 @nox.session
 def docs(session: nox.Session) -> None:
     session.install("-r", "docs/requirements.txt")
-    session.install(".[socks,secure,brotli]")
+    session.install(".[socks,secure,brotli,zstd]")
 
     session.chdir("docs")
     if os.path.exists("_build"):
diff --git a/src/urllib3/response.py b/src/urllib3/response.py
--- a/src/urllib3/response.py
+++ b/src/urllib3/response.py
@@ -1,6 +1,7 @@
 import io
 import json as _json
 import logging
+import re
 import zlib
 from contextlib import contextmanager
 from http.client import HTTPMessage as _HttplibHTTPMessage
@@ -27,6 +28,22 @@
 except ImportError:
     brotli = None
 
+try:
+    import zstandard as zstd  # type: ignore[import]
+
+    # The package 'zstandard' added the 'eof' property starting
+    # in v0.18.0 which we require to ensure a complete and
+    # valid zstd stream was fed into the ZstdDecoder.
+    # See: https://github.com/urllib3/urllib3/pull/2624
+    _zstd_version = _zstd_version = tuple(
+        map(int, re.search(r"^([0-9]+)\.([0-9]+)", zstd.__version__).groups())  # type: ignore[union-attr]
+    )
+    if _zstd_version < (0, 18):  # Defensive:
+        zstd = None
+
+except (AttributeError, ImportError, ValueError):  # Defensive:
+    zstd = None
+
 from ._collections import HTTPHeaderDict
 from .connection import _TYPE_BODY, BaseSSLError, HTTPConnection, HTTPException
 from .exceptions import (
@@ -148,6 +165,24 @@ def flush(self) -> bytes:
             return b""
 
 
+if zstd is not None:
+
+    class ZstdDecoder(ContentDecoder):
+        def __init__(self) -> None:
+            self._obj = zstd.ZstdDecompressor().decompressobj()
+
+        def decompress(self, data: bytes) -> bytes:
+            if not data:
+                return b""
+            return self._obj.decompress(data)  # type: ignore[no-any-return]
+
+        def flush(self) -> bytes:
+            ret = self._obj.flush()
+            if not self._obj.eof:
+                raise DecodeError("Zstandard data is incomplete")
+            return ret  # type: ignore[no-any-return]
+
+
 class MultiDecoder(ContentDecoder):
     """
     From RFC7231:
@@ -179,6 +214,9 @@ def _get_decoder(mode: str) -> ContentDecoder:
     if brotli is not None and mode == "br":
         return BrotliDecoder()
 
+    if zstd is not None and mode == "zstd":
+        return ZstdDecoder()
+
     return DeflateDecoder()
 
 
@@ -186,12 +224,17 @@ class BaseHTTPResponse(io.IOBase):
     CONTENT_DECODERS = ["gzip", "deflate"]
     if brotli is not None:
         CONTENT_DECODERS += ["br"]
+    if zstd is not None:
+        CONTENT_DECODERS += ["zstd"]
     REDIRECT_STATUSES = [301, 302, 303, 307, 308]
 
     DECODER_ERROR_CLASSES: Tuple[Type[Exception], ...] = (IOError, zlib.error)
     if brotli is not None:
         DECODER_ERROR_CLASSES += (brotli.error,)
 
+    if zstd is not None:
+        DECODER_ERROR_CLASSES += (zstd.ZstdError,)
+
     def __init__(
         self,
         *,
diff --git a/src/urllib3/util/request.py b/src/urllib3/util/request.py
--- a/src/urllib3/util/request.py
+++ b/src/urllib3/util/request.py
@@ -24,6 +24,12 @@
     pass
 else:
     ACCEPT_ENCODING += ",br"
+try:
+    import zstandard as _unused_module_zstd  # type: ignore[import] # noqa: F401
+except ImportError:
+    pass
+else:
+    ACCEPT_ENCODING += ",zstd"
 
 
 class _TYPE_FAILEDTELL(Enum):


Test patch:
diff --git a/test/__init__.py b/test/__init__.py
--- a/test/__init__.py
+++ b/test/__init__.py
@@ -31,6 +31,11 @@
 except ImportError:
     brotli = None
 
+try:
+    import zstandard as zstd  # type: ignore[import]
+except ImportError:
+    zstd = None
+
 import functools
 
 from urllib3 import util
@@ -146,6 +151,19 @@ def notBrotli() -> Callable[[_TestFuncT], _TestFuncT]:
     )
 
 
+def onlyZstd() -> Callable[[_TestFuncT], _TestFuncT]:
+    return pytest.mark.skipif(
+        zstd is None, reason="only run if a python-zstandard library is installed"
+    )
+
+
+def notZstd() -> Callable[[_TestFuncT], _TestFuncT]:
+    return pytest.mark.skipif(
+        zstd is not None,
+        reason="only run if a python-zstandard library is not installed",
+    )
+
+
 # Hack to make pytest evaluate a condition at test runtime instead of collection time.
 def lazy_condition(condition: Callable[[], bool]) -> bool:
     class LazyCondition:
diff --git a/test/test_response.py b/test/test_response.py
--- a/test/test_response.py
+++ b/test/test_response.py
@@ -6,7 +6,7 @@
 from base64 import b64decode
 from http.client import IncompleteRead as httplib_IncompleteRead
 from io import BufferedReader, BytesIO, TextIOWrapper
-from test import onlyBrotli
+from test import onlyBrotli, onlyZstd
 from typing import Any, Generator, List, Optional
 from unittest import mock
 
@@ -22,7 +22,7 @@
     ResponseNotChunked,
     SSLError,
 )
-from urllib3.response import HTTPResponse, brotli  # type: ignore[attr-defined]
+from urllib3.response import HTTPResponse, brotli, zstd  # type: ignore[attr-defined]
 from urllib3.util.response import is_fp_closed
 from urllib3.util.retry import RequestHistory, Retry
 
@@ -254,6 +254,39 @@ def test_decode_brotli_error(self) -> None:
         with pytest.raises(DecodeError):
             HTTPResponse(fp, headers={"content-encoding": "br"})
 
+    @onlyZstd()
+    def test_decode_zstd(self) -> None:
+        data = zstd.compress(b"foo")
+
+        fp = BytesIO(data)
+        r = HTTPResponse(fp, headers={"content-encoding": "zstd"})
+        assert r.data == b"foo"
+
+    @onlyZstd()
+    def test_chunked_decoding_zstd(self) -> None:
+        data = zstd.compress(b"foobarbaz")
+
+        fp = BytesIO(data)
+        r = HTTPResponse(
+            fp, headers={"content-encoding": "zstd"}, preload_content=False
+        )
+
+        ret = b""
+
+        for _ in range(100):
+            ret += r.read(1)
+            if r.closed:
+                break
+        assert ret == b"foobarbaz"
+
+    @onlyZstd()
+    @pytest.mark.parametrize("data", [b"foo", b"x" * 100])
+    def test_decode_zstd_error(self, data: bytes) -> None:
+        fp = BytesIO(data)
+
+        with pytest.raises(DecodeError):
+            HTTPResponse(fp, headers={"content-encoding": "zstd"})
+
     def test_multi_decoding_deflate_deflate(self) -> None:
         data = zlib.compress(zlib.compress(b"foo"))
 
diff --git a/test/test_util.py b/test/test_util.py
--- a/test/test_util.py
+++ b/test/test_util.py
@@ -5,7 +5,7 @@
 import sys
 import warnings
 from itertools import chain
-from test import ImportBlocker, ModuleStash, notBrotli, onlyBrotli
+from test import ImportBlocker, ModuleStash, notBrotli, notZstd, onlyBrotli, onlyZstd
 from typing import TYPE_CHECKING, Dict, List, NoReturn, Optional, Tuple, Union
 from unittest import mock
 from unittest.mock import MagicMock, Mock, patch
@@ -515,27 +515,47 @@ def test_parse_url_bytes_type_error(self) -> None:
     @pytest.mark.parametrize(
         "kwargs, expected",
         [
+            pytest.param(
+                {"accept_encoding": True},
+                {"accept-encoding": "gzip,deflate,br,zstd"},
+                marks=[onlyBrotli(), onlyZstd()],  # type: ignore[list-item]
+            ),
             pytest.param(
                 {"accept_encoding": True},
                 {"accept-encoding": "gzip,deflate,br"},
-                marks=onlyBrotli(),  # type: ignore[arg-type]
+                marks=[onlyBrotli(), notZstd()],  # type: ignore[list-item]
+            ),
+            pytest.param(
+                {"accept_encoding": True},
+                {"accept-encoding": "gzip,deflate,zstd"},
+                marks=[notBrotli(), onlyZstd()],  # type: ignore[list-item]
             ),
             pytest.param(
                 {"accept_encoding": True},
                 {"accept-encoding": "gzip,deflate"},
-                marks=notBrotli(),  # type: ignore[arg-type]
+                marks=[notBrotli(), notZstd()],  # type: ignore[list-item]
             ),
             ({"accept_encoding": "foo,bar"}, {"accept-encoding": "foo,bar"}),
             ({"accept_encoding": ["foo", "bar"]}, {"accept-encoding": "foo,bar"}),
+            pytest.param(
+                {"accept_encoding": True, "user_agent": "banana"},
+                {"accept-encoding": "gzip,deflate,br,zstd", "user-agent": "banana"},
+                marks=[onlyBrotli(), onlyZstd()],  # type: ignore[list-item]
+            ),
             pytest.param(
                 {"accept_encoding": True, "user_agent": "banana"},
                 {"accept-encoding": "gzip,deflate,br", "user-agent": "banana"},
-                marks=onlyBrotli(),  # type: ignore[arg-type]
+                marks=[onlyBrotli(), notZstd()],  # type: ignore[list-item]
+            ),
+            pytest.param(
+                {"accept_encoding": True, "user_agent": "banana"},
+                {"accept-encoding": "gzip,deflate,zstd", "user-agent": "banana"},
+                marks=[notBrotli(), onlyZstd()],  # type: ignore[list-item]
             ),
             pytest.param(
                 {"accept_encoding": True, "user_agent": "banana"},
                 {"accept-encoding": "gzip,deflate", "user-agent": "banana"},
-                marks=notBrotli(),  # type: ignore[arg-type]
+                marks=[notBrotli(), notZstd()],  # type: ignore[list-item]
             ),
             ({"user_agent": "banana"}, {"user-agent": "banana"}),
             ({"keep_alive": True}, {"connection": "keep-alive"}),