Repository: kevin1024/vcrpy

Issue:
[>=4.3.1] Chunked uploads are broken with urllib3 v2
With vcrpy 5.0.0, when [updating urllib3 from `1.26.16`  to `2.0.3`](https://gitlab.com/gitlabracadabra/gitlabracadabra/-/merge_requests/531), chunked uploads [are failing](https://gitlab.com/gitlabracadabra/gitlabracadabra/-/jobs/4563346377).

This is because the body appears to vcrpy unchunked:

```python
(Pdb) self.cassette.requests[18].url == self._vcr_request.url
True
(Pdb) self.cassette.requests[18].body                         
b'45\r\nBlob bf9500cc6aa4fac1a75c958496c9e9caeceacb256113e28836fc5b3bd448f706\r\n0\r\n\r\n'
(Pdb) [c for c in self._vcr_request.body]
[b'Blob bf9500cc6aa4fac1a75c958496c9e9caeceacb256113e28836fc5b3bd448f706']
(Pdb) self._vcr_request.headers          
{'User-Agent': 'gitlabracadabra/1.13.0', 'Docker-Distribution-Api-Version': 'registry/2.0', 'Authorization': 'Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IlJJWTM6SUhGMjpZUlhVOlJGTVI6NEVRRDpJWVZDOjZMSzM6MzVJTDpGRFJGOklKUE46Q0MySjpMTEIzIiwidHlwIjoiSldUIn0.eyJhY2Nlc3MiOlt7InR5cGUiOiJyZXBvc2l0b3J5IiwibmFtZSI6InJvb3QvdGVzdF9yZWdpc3RyeS9vcGVyYXRvci1mcmFtZXdvcmsvb2xtIiwiYWN0aW9ucyI6WyJwdXNoIiwicHVsbCJdfV0sImp0aSI6ImM5ZGQ1YWQxLWQ0M2ItNDI0Yi1iZDUwLWU4NzJlYjYxZjYwNSIsImF1ZCI6ImNvbnRhaW5lcl9yZWdpc3RyeSIsInN1YiI6InJvb3QiLCJpc3MiOiJnaXRsYWItaXNzdWVyIiwiaWF0IjoxNjE3MTkzMjE5LCJuYmYiOjE2MTcxOTMyMTQsImV4cCI6MTYxNzE5MzUxOX0.b6PKFISAEPnDnfTKR8XVIHUOc6JYomUOB-u_w3QQIqF8gkb92n199zvFSwcpOZolcXjk1DvCfMtUkOvklzAgXyIwNcBx7t_o_SO4mCqWEnbfWxzhJKKSPm6YLAC0IoQbxCsPfA4CK7XEOCJU7eTLaCTqLvq6GvmpjtyLF-KgZV8eUV4zPDNYkhvUqcJXSQR_UHqQ4k32h8itBGYzfhykHSP2B2ZvbBDHLYmBlsDFtc7LJVUEEvKKhCR1VQppwgIoHwJwu7OR3R6I5_XO23NwBbPYvAQZC9twPBKNvoPfcToMzSF_keuAVodUgAyYMUHmnFPw5jX0-PcPt0L9j6a9rA', 'Cookie': 'perf_bar_enabled=true; experimentation_subject_id=eyJfcmFpbHMiOnsibWVzc2FnZSI6IkltSmpaRFZrTXpRMkxXWXlPR010TkRZek1DMWhNMll5TFRNMlpEYzNZVFJrWWpsa09DST0iLCJleHAiOm51bGwsInB1ciI6ImNvb2tpZS5leHBlcmltZW50YXRpb25fc3ViamVjdF9pZCJ9fQ%3D%3D--403d642cd6f4aaa4365664b26bec713ff089b313', 'Transfer-Encoding': 'chunked'}
(Pdb) self.cassette.requests[18].headers                     
{'Authorization': 'Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IlJJWTM6SUhGMjpZUlhVOlJGTVI6NEVRRDpJWVZDOjZMSzM6MzVJTDpGRFJGOklKUE46Q0MySjpMTEIzIiwidHlwIjoiSldUIn0.eyJhY2Nlc3MiOlt7InR5cGUiOiJyZXBvc2l0b3J5IiwibmFtZSI6InJvb3QvdGVzdF9yZWdpc3RyeS9vcGVyYXRvci1mcmFtZXdvcmsvb2xtIiwiYWN0aW9ucyI6WyJwdXNoIiwicHVsbCJdfV0sImp0aSI6ImM5ZGQ1YWQxLWQ0M2ItNDI0Yi1iZDUwLWU4NzJlYjYxZjYwNSIsImF1ZCI6ImNvbnRhaW5lcl9yZWdpc3RyeSIsInN1YiI6InJvb3QiLCJpc3MiOiJnaXRsYWItaXNzdWVyIiwiaWF0IjoxNjE3MTkzMjE5LCJuYmYiOjE2MTcxOTMyMTQsImV4cCI6MTYxNzE5MzUxOX0.b6PKFISAEPnDnfTKR8XVIHUOc6JYomUOB-u_w3QQIqF8gkb92n199zvFSwcpOZolcXjk1DvCfMtUkOvklzAgXyIwNcBx7t_o_SO4mCqWEnbfWxzhJKKSPm6YLAC0IoQbxCsPfA4CK7XEOCJU7eTLaCTqLvq6GvmpjtyLF-KgZV8eUV4zPDNYkhvUqcJXSQR_UHqQ4k32h8itBGYzfhykHSP2B2ZvbBDHLYmBlsDFtc7LJVUEEvKKhCR1VQppwgIoHwJwu7OR3R6I5_XO23NwBbPYvAQZC9twPBKNvoPfcToMzSF_keuAVodUgAyYMUHmnFPw5jX0-PcPt0L9j6a9rA', 'Cookie': 'experimentation_subject_id=eyJfcmFpbHMiOnsibWVzc2FnZSI6IkltSmpaRFZrTXpRMkxXWXlPR010TkRZek1DMWhNMll5TFRNMlpEYzNZVFJrWWpsa09DST0iLCJleHAiOm51bGwsInB1ciI6ImNvb2tpZS5leHBlcmltZW50YXRpb25fc3ViamVjdF9pZCJ9fQ%3D%3D--403d642cd6f4aaa4365664b26bec713ff089b313; perf_bar_enabled=true', 'Docker-Distribution-Api-Version': 'registry/2.0', 'Transfer-Encoding': 'chunked'}
```

Notes:

- The fix is probably similar to https://github.com/kevin1024/vcrpy/pull/723.
- The affected cassette is [here](https://gitlab.com/gitlabracadabra/gitlabracadabra/-/blob/6130c586f1cd4d9dfb9055dd033ac59e9251c124/gitlabracadabra/tests/unit/fixtures/ProjectImageMirrors_image_mirror_digest.yaml#L687-719)
- I use [a custom body matcher](https://gitlab.com/gitlabracadabra/gitlabracadabra/-/blob/6130c586f1cd4d9dfb9055dd033ac59e9251c124/gitlabracadabra/tests/vcrfuncs.py#L54-68), but I don't think this is relevant here (except that this could be a way to workaround)

Discussion:

Patch:
diff --git a/vcr/matchers.py b/vcr/matchers.py
index 98395b6..70e7891 100644
--- a/vcr/matchers.py
+++ b/vcr/matchers.py
@@ -2,9 +2,13 @@ import json
 import logging
 import urllib
 import xmlrpc.client
+from string import hexdigits
+from typing import List, Set
 
 from .util import read_body
 
+_HEXDIG_CODE_POINTS: Set[int] = {ord(s.encode("ascii")) for s in hexdigits}
+
 log = logging.getLogger(__name__)
 
 
@@ -49,11 +53,17 @@ def raw_body(r1, r2):
 
 
 def body(r1, r2):
-    transformer = _get_transformer(r1)
-    r2_transformer = _get_transformer(r2)
-    if transformer != r2_transformer:
-        transformer = _identity
-    if transformer(read_body(r1)) != transformer(read_body(r2)):
+    transformers = list(_get_transformers(r1))
+    if transformers != list(_get_transformers(r2)):
+        transformers = []
+
+    b1 = read_body(r1)
+    b2 = read_body(r2)
+    for transform in transformers:
+        b1 = transform(b1)
+        b2 = transform(b2)
+
+    if b1 != b2:
         raise AssertionError
 
 
@@ -72,6 +82,62 @@ def _header_checker(value, header="Content-Type"):
     return checker
 
 
+def _dechunk(body):
+    if isinstance(body, str):
+        body = body.encode("utf-8")
+    elif isinstance(body, bytearray):
+        body = bytes(body)
+    elif hasattr(body, "__iter__"):
+        body = list(body)
+        if body:
+            if isinstance(body[0], str):
+                body = ("".join(body)).encode("utf-8")
+            elif isinstance(body[0], bytes):
+                body = b"".join(body)
+            elif isinstance(body[0], int):
+                body = bytes(body)
+            else:
+                raise ValueError(f"Body chunk type {type(body[0])} not supported")
+        else:
+            body = None
+
+    if not isinstance(body, bytes):
+        return body
+
+    # Now decode chunked data format (https://en.wikipedia.org/wiki/Chunked_transfer_encoding)
+    # Example input: b"45\r\n<69 bytes>\r\n0\r\n\r\n" where int(b"45", 16) == 69.
+    CHUNK_GAP = b"\r\n"
+    BODY_LEN: int = len(body)
+
+    chunks: List[bytes] = []
+    pos: int = 0
+
+    while True:
+        for i in range(pos, BODY_LEN):
+            if body[i] not in _HEXDIG_CODE_POINTS:
+                break
+
+        if i == 0 or body[i : i + len(CHUNK_GAP)] != CHUNK_GAP:
+            if pos == 0:
+                return body  # i.e. assume non-chunk data
+            raise ValueError("Malformed chunked data")
+
+        size_bytes = int(body[pos:i], 16)
+        if size_bytes == 0:  # i.e. well-formed ending
+            return b"".join(chunks)
+
+        chunk_data_first = i + len(CHUNK_GAP)
+        chunk_data_after_last = chunk_data_first + size_bytes
+
+        if body[chunk_data_after_last : chunk_data_after_last + len(CHUNK_GAP)] != CHUNK_GAP:
+            raise ValueError("Malformed chunked data")
+
+        chunk_data = body[chunk_data_first:chunk_data_after_last]
+        chunks.append(chunk_data)
+
+        pos = chunk_data_after_last + len(CHUNK_GAP)
+
+
 def _transform_json(body):
     # Request body is always a byte string, but json.loads() wants a text
     # string. RFC 7159 says the default encoding is UTF-8 (although UTF-16
@@ -83,6 +149,7 @@ def _transform_json(body):
 _xml_header_checker = _header_checker("text/xml")
 _xmlrpc_header_checker = _header_checker("xmlrpc", header="User-Agent")
 _checker_transformer_pairs = (
+    (_header_checker("chunked", header="Transfer-Encoding"), _dechunk),
     (
         _header_checker("application/x-www-form-urlencoded"),
         lambda body: urllib.parse.parse_qs(body.decode("ascii")),
@@ -92,16 +159,10 @@ _checker_transformer_pairs = (
 )
 
 
-def _identity(x):
-    return x
-
-
-def _get_transformer(request):
+def _get_transformers(request):
     for checker, transformer in _checker_transformer_pairs:
         if checker(request.headers):
-            return transformer
-    else:
-        return _identity
+            yield transformer
 
 
 def requests_match(r1, r2, matchers):


Test patch:
diff --git a/tests/unit/test_matchers.py b/tests/unit/test_matchers.py
index 971cb66..f8686eb 100644
--- a/tests/unit/test_matchers.py
+++ b/tests/unit/test_matchers.py
@@ -63,6 +63,9 @@ boto3_bytes_headers = {
     "Expect": b"100-continue",
     "Content-Length": "21",
 }
+chunked_headers = {
+    "Transfer-Encoding": "chunked",
+}
 
 
 @pytest.mark.parametrize(
@@ -151,6 +154,36 @@ boto3_bytes_headers = {
             request.Request("POST", "http://aws.custom.com/", b"123", boto3_bytes_headers),
             request.Request("POST", "http://aws.custom.com/", b"123", boto3_bytes_headers),
         ),
+        (
+            # chunked transfer encoding: decoded bytes versus encoded bytes
+            request.Request("POST", "scheme1://host1.test/", b"123456789_123456", chunked_headers),
+            request.Request(
+                "GET",
+                "scheme2://host2.test/",
+                b"10\r\n123456789_123456\r\n0\r\n\r\n",
+                chunked_headers,
+            ),
+        ),
+        (
+            # chunked transfer encoding: bytes iterator versus string iterator
+            request.Request(
+                "POST",
+                "scheme1://host1.test/",
+                iter([b"123456789_", b"123456"]),
+                chunked_headers,
+            ),
+            request.Request("GET", "scheme2://host2.test/", iter(["123456789_", "123456"]), chunked_headers),
+        ),
+        (
+            # chunked transfer encoding: bytes iterator versus single byte iterator
+            request.Request(
+                "POST",
+                "scheme1://host1.test/",
+                iter([b"123456789_", b"123456"]),
+                chunked_headers,
+            ),
+            request.Request("GET", "scheme2://host2.test/", iter(b"123456789_123456"), chunked_headers),
+        ),
     ],
 )
 def test_body_matcher_does_match(r1, r2):