Repository: pallets/werkzeug
Pull request #2767 (2023-08-14 09:22:32)

Merge 2.3.x


Patch:
diff --git a/.github/workflows/lock.yaml b/.github/workflows/lock.yaml
index e962fd04..c790fae5 100644
--- a/.github/workflows/lock.yaml
+++ b/.github/workflows/lock.yaml
@@ -19,7 +19,7 @@ jobs:
   lock:
     runs-on: ubuntu-latest
     steps:
-      - uses: dessant/lock-threads@be8aa5be94131386884a6da4189effda9b14aa21
+      - uses: dessant/lock-threads@c1b35aecc5cdb1a34539d14196df55838bb2f836
         with:
           issue-inactive-days: 14
           pr-inactive-days: 14
diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml
index 3f368ebe..f5209f4e 100644
--- a/.github/workflows/publish.yaml
+++ b/.github/workflows/publish.yaml
@@ -9,8 +9,8 @@ jobs:
     outputs:
       hash: ${{ steps.hash.outputs.hash }}
     steps:
-      - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9
-      - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1
+      - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3
+      - uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435
         with:
           python-version: '3.x'
           cache: 'pip'
@@ -33,7 +33,7 @@ jobs:
       id-token: write
       contents: write
     # Can't pin with hash due to how this workflow works.
-    uses: slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@v1.7.0
+    uses: slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@v1.5.0
     with:
       base64-subjects: ${{ needs.build.outputs.hash }}
   create-release:
@@ -63,10 +63,10 @@ jobs:
     steps:
       - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a
       # Try uploading to Test PyPI first, in case something fails.
-      - uses: pypa/gh-action-pypi-publish@f8c70e705ffc13c3b4d1221169b84f12a75d6ca8
+      - uses: pypa/gh-action-pypi-publish@29930c9cf57955dc1b98162d0d8bc3ec80d9e75c
         with:
           repository-url: https://test.pypi.org/legacy/
           packages-dir: artifact/
-      - uses: pypa/gh-action-pypi-publish@f8c70e705ffc13c3b4d1221169b84f12a75d6ca8
+      - uses: pypa/gh-action-pypi-publish@29930c9cf57955dc1b98162d0d8bc3ec80d9e75c
         with:
           packages-dir: artifact/
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 7ab3a413..6425015c 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -3,7 +3,7 @@ ci:
   autoupdate_schedule: monthly
 repos:
   - repo: https://github.com/asottile/pyupgrade
-    rev: v3.7.0
+    rev: v3.10.1
     hooks:
       - id: pyupgrade
         args: ["--py38-plus"]
@@ -19,11 +19,11 @@ repos:
         files: "^examples/"
         args: ["--application-directories", "examples"]
   - repo: https://github.com/psf/black
-    rev: 23.3.0
+    rev: 23.7.0
     hooks:
       - id: black
   - repo: https://github.com/PyCQA/flake8
-    rev: 6.0.0
+    rev: 6.1.0
     hooks:
       - id: flake8
         additional_dependencies:
diff --git a/CHANGES.rst b/CHANGES.rst
index f3784c23..5601921a 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,11 +1,24 @@
 .. currentmodule:: werkzeug
 
-Version 2.3.7
+Version 2.3.8
 -------------
 
 Unreleased
 
+
+Version 2.3.7
+-------------
+
+Released 2023-08-14
+
 -   Use ``flit_core`` instead of ``setuptools`` as build backend.
+-   Fix parsing of multipart bodies. :issue:`2734`
+    Adjust index of last newline in data start. :issue:`2761`
+-   ``_plain_int`` and ``_plain_float`` strip whitespace before type
+    enforcement. :issue:`2734`
+-   Fix empty file streaming when testing. :issue:`2740`
+-   Clearer error message when URL rule does not start with slash. :pr:`2750`
+-   ``Accept`` ``q`` value can be a float without a decimal part. :issue:`2751`
 
 
 Version 2.3.6
diff --git a/src/werkzeug/_internal.py b/src/werkzeug/_internal.py
index 4981f811..6ed4d302 100644
--- a/src/werkzeug/_internal.py
+++ b/src/werkzeug/_internal.py
@@ -313,7 +313,6 @@ def _decode_idna(domain: str) -> str:
 
 
 _plain_int_re = re.compile(r"-?\d+", re.ASCII)
-_plain_float_re = re.compile(r"-?\d+\.\d+", re.ASCII)
 
 
 def _plain_int(value: str) -> int:
@@ -321,21 +320,11 @@ def _plain_int(value: str) -> int:
 
     This disallows ``+``, ``_``, and non-ASCII digits, which are accepted by ``int`` but
     are not allowed in HTTP header values.
+
+    Any leading or trailing whitespace is stripped
     """
+    value = value.strip()
     if _plain_int_re.fullmatch(value) is None:
         raise ValueError
 
     return int(value)
-
-
-def _plain_float(value: str) -> float:
-    """Parse a float only if it is only ASCII digits and ``-``, and contains digits
-    before and after the ``.``.
-
-    This disallows ``+``, ``_``, non-ASCII digits, and ``.123``, which are accepted by
-    ``float`` but are not allowed in HTTP header values.
-    """
-    if _plain_float_re.fullmatch(value) is None:
-        raise ValueError
-
-    return float(value)
diff --git a/src/werkzeug/debug/shared/debugger.js b/src/werkzeug/debug/shared/debugger.js
index 2354f030..f463e9c7 100644
--- a/src/werkzeug/debug/shared/debugger.js
+++ b/src/werkzeug/debug/shared/debugger.js
@@ -305,7 +305,8 @@ function handleConsoleSubmit(e, command, frameID) {
           wrapperSpan.append(spanToWrap);
           spanToWrap.hidden = true;
 
-          expansionButton.addEventListener("click", () => {
+          expansionButton.addEventListener("click", (event) => {
+            event.preventDefault();
             spanToWrap.hidden = !spanToWrap.hidden;
             expansionButton.classList.toggle("open");
             return false;
diff --git a/src/werkzeug/http.py b/src/werkzeug/http.py
index 72dc3f3e..07d1fd49 100644
--- a/src/werkzeug/http.py
+++ b/src/werkzeug/http.py
@@ -18,7 +18,6 @@ from urllib.parse import unquote
 from urllib.request import parse_http_list as _parse_list_header
 
 from ._internal import _dt_as_utc
-from ._internal import _plain_float
 from ._internal import _plain_int
 
 if t.TYPE_CHECKING:
@@ -614,6 +613,7 @@ def parse_options_header(value: str | None) -> tuple[str, dict[str, str]]:
     return value, options
 
 
+_q_value_re = re.compile(r"-?\d+(\.\d+)?", re.ASCII)
 _TAnyAccept = t.TypeVar("_TAnyAccept", bound="ds.Accept")
 
 
@@ -656,13 +656,15 @@ def parse_accept_header(
         item, options = parse_options_header(item)
 
         if "q" in options:
-            try:
-                # pop q, remaining options are reconstructed
-                q = _plain_float(options.pop("q"))
-            except ValueError:
+            # pop q, remaining options are reconstructed
+            q_str = options.pop("q").strip()
+
+            if _q_value_re.fullmatch(q_str) is None:
                 # ignore an invalid q
                 continue
 
+            q = float(q_str)
+
             if q < 0 or q > 1:
                 # ignore an invalid q
                 continue
diff --git a/src/werkzeug/routing/rules.py b/src/werkzeug/routing/rules.py
index 63e62c94..904a0225 100644
--- a/src/werkzeug/routing/rules.py
+++ b/src/werkzeug/routing/rules.py
@@ -454,7 +454,8 @@ class Rule(RuleFactory):
         websocket: bool = False,
     ) -> None:
         if not string.startswith("/"):
-            raise ValueError("urls must start with a leading slash")
+            raise ValueError(f"URL rule '{string}' must start with a slash.")
+
         self.rule = string
         self.is_leaf = not string.endswith("/")
         self.is_branch = string.endswith("/")
diff --git a/src/werkzeug/sansio/multipart.py b/src/werkzeug/sansio/multipart.py
index 11e65ed0..380993af 100644
--- a/src/werkzeug/sansio/multipart.py
+++ b/src/werkzeug/sansio/multipart.py
@@ -256,7 +256,7 @@ class MultipartDecoder:
             # a partial boundary at the end. As the boundary
             # starts with either a nl or cr find the earliest and
             # return up to that as data.
-            data_end = del_index = self.last_newline(data[data_start:])
+            data_end = del_index = self.last_newline(data[data_start:]) + data_start
             more_data = True
         else:
             match = self.boundary_re.search(data)
@@ -268,7 +268,7 @@ class MultipartDecoder:
                 data_end = match.start()
                 del_index = match.end()
             else:
-                data_end = del_index = self.last_newline(data[data_start:])
+                data_end = del_index = self.last_newline(data[data_start:]) + data_start
             more_data = match is None
 
         return bytes(data[data_start:data_end]), del_index, more_data


Test patch:
diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
index 8dbd601a..f29fe5f7 100644
--- a/.github/workflows/tests.yaml
+++ b/.github/workflows/tests.yaml
@@ -34,8 +34,8 @@ jobs:
           - {name: 'PyPy', python: 'pypy-3.10', os: ubuntu-latest, tox: pypy310}
           - {name: Typing, python: '3.11', os: ubuntu-latest, tox: typing}
     steps:
-      - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9
-      - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1
+      - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3
+      - uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435
         with:
           python-version: ${{ matrix.python }}
           cache: 'pip'
diff --git a/docs/test.rst b/docs/test.rst
index d31ac593..704eb5f5 100644
--- a/docs/test.rst
+++ b/docs/test.rst
@@ -18,8 +18,8 @@ requests.
 >>> response = c.get("/")
 >>> response.status_code
 200
->>> response.headers
-Headers([('Content-Type', 'text/html; charset=utf-8'), ('Content-Length', '5211')])
+>>> resp.headers
+Headers([('Content-Type', 'text/html; charset=utf-8'), ('Content-Length', '6658')])
 >>> response.get_data(as_text=True)
 '<!doctype html>...'
 
diff --git a/src/werkzeug/test.py b/src/werkzeug/test.py
index f5b4f2ff..968553f2 100644
--- a/src/werkzeug/test.py
+++ b/src/werkzeug/test.py
@@ -136,6 +136,7 @@ def stream_encode_multipart(
                 chunk = reader(16384)
 
                 if not chunk:
+                    write_binary(encoder.send_event(Data(data=chunk, more_data=False)))
                     break
 
                 write_binary(encoder.send_event(Data(data=chunk, more_data=True)))
diff --git a/tests/sansio/test_multipart.py b/tests/sansio/test_multipart.py
index 1d836d64..35109d4b 100644
--- a/tests/sansio/test_multipart.py
+++ b/tests/sansio/test_multipart.py
@@ -1,3 +1,5 @@
+import pytest
+
 from werkzeug.datastructures import Headers
 from werkzeug.sansio.multipart import Data
 from werkzeug.sansio.multipart import Epilogue
@@ -56,6 +58,57 @@ asdasd
     assert data == result
 
 
+@pytest.mark.parametrize(
+    "data_start",
+    [
+        b"A",
+        b"\n",
+        b"\r",
+        b"\r\n",
+        b"\n\r",
+        b"A\n",
+        b"A\r",
+        b"A\r\n",
+        b"A\n\r",
+    ],
+)
+@pytest.mark.parametrize("data_end", [b"", b"\r\n--foo"])
+def test_decoder_data_start_with_different_newline_positions(
+    data_start: bytes, data_end: bytes
+) -> None:
+    boundary = b"foo"
+    data = (
+        b"\r\n--foo\r\n"
+        b'Content-Disposition: form-data; name="test"; filename="testfile"\r\n'
+        b"Content-Type: application/octet-stream\r\n\r\n"
+        b"" + data_start + b"\r\nBCDE" + data_end
+    )
+    decoder = MultipartDecoder(boundary)
+    decoder.receive_data(data)
+    events = [decoder.next_event()]
+    # We want to check up to data start event
+    while not isinstance(events[-1], Data):
+        events.append(decoder.next_event())
+    expected = data_start if data_end == b"" else data_start + b"\r\nBCDE"
+    assert events == [
+        Preamble(data=b""),
+        File(
+            name="test",
+            filename="testfile",
+            headers=Headers(
+                [
+                    (
+                        "Content-Disposition",
+                        'form-data; name="test"; filename="testfile"',
+                    ),
+                    ("Content-Type", "application/octet-stream"),
+                ]
+            ),
+        ),
+        Data(data=expected, more_data=True),
+    ]
+
+
 def test_chunked_boundaries() -> None:
     boundary = b"--boundary"
     decoder = MultipartDecoder(boundary)
diff --git a/tests/sansio/test_utils.py b/tests/sansio/test_utils.py
index 8c8faa61..04d02e44 100644
--- a/tests/sansio/test_utils.py
+++ b/tests/sansio/test_utils.py
@@ -1,7 +1,10 @@
+from __future__ import annotations
+
 import typing as t
 
 import pytest
 
+from werkzeug.sansio.utils import get_content_length
 from werkzeug.sansio.utils import get_host
 
 
@@ -30,3 +33,23 @@ def test_get_host(
     expected: str,
 ) -> None:
     assert get_host(scheme, host_header, server) == expected
+
+
+@pytest.mark.parametrize(
+    ("http_content_length", "http_transfer_encoding", "expected"),
+    [
+        ("2", None, 2),
+        (" 2", None, 2),
+        ("2 ", None, 2),
+        (None, None, None),
+        (None, "chunked", None),
+        ("a", None, 0),
+        ("-2", None, 0),
+    ],
+)
+def test_get_content_length(
+    http_content_length: str | None,
+    http_transfer_encoding: str | None,
+    expected: int | None,
+) -> None:
+    assert get_content_length(http_content_length, http_transfer_encoding) == expected
diff --git a/tests/test_datastructures.py b/tests/test_datastructures.py
index 6d2f9fe5..5206aa6a 100644
--- a/tests/test_datastructures.py
+++ b/tests/test_datastructures.py
@@ -63,7 +63,7 @@ class _MutableMultiDictTests:
             d = create_instance()
             s = pickle.dumps(d, protocol)
             ud = pickle.loads(s)
-            assert type(ud) == type(d)
+            assert type(ud) == type(d)  # noqa: E721
             assert ud == d
             alternative = pickle.dumps(create_instance("werkzeug"), protocol)
             assert pickle.loads(alternative) == d
diff --git a/tests/test_http.py b/tests/test_http.py
index 2c6c0b29..bbd51ba3 100644
--- a/tests/test_http.py
+++ b/tests/test_http.py
@@ -776,6 +776,12 @@ def test_accept_invalid_float(value):
     assert list(a.values()) == ["en"]
 
 
+def test_accept_valid_int_one_zero():
+    assert http.parse_accept_header("en;q=1") == http.parse_accept_header("en;q=1.0")
+    assert http.parse_accept_header("en;q=0") == http.parse_accept_header("en;q=0.0")
+    assert http.parse_accept_header("en;q=5") == http.parse_accept_header("en;q=5.0")
+
+
 @pytest.mark.parametrize("value", ["🯱🯲🯳", "+1-", "1-1_23"])
 def test_range_invalid_int(value):
     assert http.parse_range_header(value) is None
diff --git a/tests/test_routing.py b/tests/test_routing.py
index 15bcc0af..65d2a5f9 100644
--- a/tests/test_routing.py
+++ b/tests/test_routing.py
@@ -748,7 +748,7 @@ def test_uuid_converter():
     m = r.Map([r.Rule("/a/<uuid:a_uuid>", endpoint="a")])
     a = m.bind("example.org", "/")
     route, kwargs = a.match("/a/a8098c1a-f86e-11da-bd1a-00112444be1e")
-    assert type(kwargs["a_uuid"]) == uuid.UUID
+    assert type(kwargs["a_uuid"]) == uuid.UUID  # noqa: E721
 
 
 def test_converter_with_tuples():
diff --git a/tests/test_test.py b/tests/test_test.py
index a7390008..c7f21fa1 100644
--- a/tests/test_test.py
+++ b/tests/test_test.py
@@ -357,6 +357,23 @@ def test_environ_builder_unicode_file_mix():
         files["f"].close()
 
 
+def test_environ_builder_empty_file():
+    f = FileStorage(BytesIO(rb""), "empty.txt")
+    d = MultiDict(dict(f=f, s=""))
+    stream, length, boundary = stream_encode_multipart(d)
+    _, form, files = parse_form_data(
+        {
+            "wsgi.input": stream,
+            "CONTENT_LENGTH": str(length),
+            "CONTENT_TYPE": f'multipart/form-data; boundary="{boundary}"',
+        }
+    )
+    assert form["s"] == ""
+    assert files["f"].read() == rb""
+    stream.close()
+    files["f"].close()
+
+
 def test_create_environ():
     env = create_environ("/foo?bar=baz", "http://example.org/")
     expected = {
@@ -409,7 +426,7 @@ def test_file_closing():
 
     class SpecialInput:
         def read(self, size):
-            return ""
+            return b""
 
         def close(self):
             closed.append(self)