Repository: twisted/twisted

Issue:
Speed up http_headers operations
Handling HTTP headers is a significant percentage of time spent in the HTTP client (and probably the HTTP server as well). In particular, all the normalization (which happens both incoming and outgoing!) adds up.


Discussion:

Resolving patch:
diff --git a/src/twisted/web/client.py b/src/twisted/web/client.py
--- a/src/twisted/web/client.py
+++ b/src/twisted/web/client.py
@@ -1530,7 +1530,7 @@ def _handleResponse(self, response):
         return response
 
 
-_canonicalHeaderName = Headers()._canonicalNameCaps
+_canonicalHeaderName = Headers()._encodeName
 _defaultSensitiveHeaders = frozenset(
     [
         b"Authorization",
diff --git a/src/twisted/web/http_headers.py b/src/twisted/web/http_headers.py
--- a/src/twisted/web/http_headers.py
+++ b/src/twisted/web/http_headers.py
@@ -6,9 +6,9 @@
 An API for storing HTTP header names and values.
 """
 
-from collections.abc import Sequence as _Sequence
 from typing import (
     AnyStr,
+    ClassVar,
     Dict,
     Iterator,
     List,
@@ -26,17 +26,6 @@
 _T = TypeVar("_T")
 
 
-def _dashCapitalize(name: bytes) -> bytes:
-    """
-    Return a byte string which is capitalized using '-' as a word separator.
-
-    @param name: The name of the header to capitalize.
-
-    @return: The given header capitalized using '-' as a word separator.
-    """
-    return b"-".join([word.capitalize() for word in name.split(b"-")])
-
-
 def _sanitizeLinearWhitespace(headerComponent: bytes) -> bytes:
     r"""
     Replace linear whitespace (C{\n}, C{\r\n}, C{\r}) in a header key
@@ -65,13 +54,17 @@ class Headers:
     and values as opaque byte strings.
 
     @cvar _caseMappings: A L{dict} that maps lowercase header names
-        to their canonicalized representation.
+        to their canonicalized representation, for headers with unconventional
+        capitalization.
+
+    @cvar _canonicalHeaderCache: A L{dict} that maps header names to their
+        canonicalized representation.
 
     @ivar _rawHeaders: A L{dict} mapping header names as L{bytes} to L{list}s of
         header values as L{bytes}.
     """
 
-    _caseMappings = {
+    _caseMappings: ClassVar[Dict[bytes, bytes]] = {
         b"content-md5": b"Content-MD5",
         b"dnt": b"DNT",
         b"etag": b"ETag",
@@ -81,6 +74,12 @@ class Headers:
         b"x-xss-protection": b"X-XSS-Protection",
     }
 
+    _canonicalHeaderCache: ClassVar[Dict[Union[bytes, str], bytes]] = {}
+
+    _MAX_CACHED_HEADERS: ClassVar[int] = 10_000
+
+    __slots__ = ["_rawHeaders"]
+
     def __init__(
         self,
         rawHeaders: Optional[Mapping[AnyStr, Sequence[AnyStr]]] = None,
@@ -112,16 +111,36 @@ def __cmp__(self, other):
 
     def _encodeName(self, name: Union[str, bytes]) -> bytes:
         """
-        Encode the name of a header (eg 'Content-Type') to an ISO-8859-1 encoded
-        bytestring if required.
+        Encode the name of a header (eg 'Content-Type') to an ISO-8859-1
+        encoded bytestring if required.  It will be canonicalized and
+        whitespace-sanitized.
 
         @param name: A HTTP header name
 
         @return: C{name}, encoded if required, lowercased
         """
-        if isinstance(name, str):
-            return name.lower().encode("iso-8859-1")
-        return name.lower()
+        if canonicalName := self._canonicalHeaderCache.get(name, None):
+            return canonicalName
+
+        bytes_name = name.encode("iso-8859-1") if isinstance(name, str) else name
+
+        if bytes_name.lower() in self._caseMappings:
+            # Some headers have special capitalization:
+            result = self._caseMappings[bytes_name.lower()]
+        else:
+            result = _sanitizeLinearWhitespace(
+                b"-".join([word.capitalize() for word in bytes_name.split(b"-")])
+            )
+
+        # In general, we should only see a very small number of header
+        # variations in the real world, so caching them is fine. However, an
+        # attacker could generate infinite header variations to fill up RAM, so
+        # we cap how many we cache. The performance degradation from lack of
+        # caching won't be that bad, and legit traffic won't hit it.
+        if len(self._canonicalHeaderCache) < self._MAX_CACHED_HEADERS:
+            self._canonicalHeaderCache[name] = result
+
+        return result
 
     def copy(self):
         """
@@ -151,21 +170,9 @@ def removeHeader(self, name: AnyStr) -> None:
         """
         self._rawHeaders.pop(self._encodeName(name), None)
 
-    @overload
-    def setRawHeaders(self, name: Union[str, bytes], values: Sequence[bytes]) -> None:
-        ...
-
-    @overload
-    def setRawHeaders(self, name: Union[str, bytes], values: Sequence[str]) -> None:
-        ...
-
-    @overload
     def setRawHeaders(
         self, name: Union[str, bytes], values: Sequence[Union[str, bytes]]
     ) -> None:
-        ...
-
-    def setRawHeaders(self, name: Union[str, bytes], values: object) -> None:
         """
         Sets the raw representation of the given header.
 
@@ -179,29 +186,7 @@ def setRawHeaders(self, name: Union[str, bytes], values: object) -> None:
 
         @return: L{None}
         """
-        if not isinstance(values, _Sequence):
-            raise TypeError(
-                "Header entry %r should be sequence but found "
-                "instance of %r instead" % (name, type(values))
-            )
-
-        if not isinstance(name, (bytes, str)):
-            raise TypeError(
-                f"Header name is an instance of {type(name)!r}, not bytes or str"
-            )
-
-        for count, value in enumerate(values):
-            if not isinstance(value, (bytes, str)):
-                raise TypeError(
-                    "Header value at position %s is an instance of %r, not "
-                    "bytes or str"
-                    % (
-                        count,
-                        type(value),
-                    )
-                )
-
-        _name = _sanitizeLinearWhitespace(self._encodeName(name))
+        _name = self._encodeName(name)
         encodedValues: List[bytes] = []
         for v in values:
             if isinstance(v, str):
@@ -220,20 +205,7 @@ def addRawHeader(self, name: Union[str, bytes], value: Union[str, bytes]) -> Non
 
         @param value: The value to set for the named header.
         """
-        if not isinstance(name, (bytes, str)):
-            raise TypeError(
-                f"Header name is an instance of {type(name)!r}, not bytes or str"
-            )
-
-        if not isinstance(value, (bytes, str)):
-            raise TypeError(
-                "Header value is an instance of %r, not "
-                "bytes or str" % (type(value),)
-            )
-
-        self._rawHeaders.setdefault(
-            _sanitizeLinearWhitespace(self._encodeName(name)), []
-        ).append(
+        self._rawHeaders.setdefault(self._encodeName(name), []).append(
             _sanitizeLinearWhitespace(
                 value.encode("utf8") if isinstance(value, str) else value
             )
@@ -277,19 +249,7 @@ def getAllRawHeaders(self) -> Iterator[Tuple[bytes, Sequence[bytes]]]:
         object, as L{bytes}.  The keys are capitalized in canonical
         capitalization.
         """
-        for k, v in self._rawHeaders.items():
-            yield self._canonicalNameCaps(k), v
-
-    def _canonicalNameCaps(self, name: bytes) -> bytes:
-        """
-        Return the canonical name for the given header.
-
-        @param name: The all-lowercase header name to capitalize in its
-            canonical form.
-
-        @return: The canonical name of the header.
-        """
-        return self._caseMappings.get(name, _dashCapitalize(name))
+        return iter(self._rawHeaders.items())
 
 
 __all__ = ["Headers"]


Test patch:
diff --git a/src/twisted/web/test/test_http_headers.py b/src/twisted/web/test/test_http_headers.py
--- a/src/twisted/web/test/test_http_headers.py
+++ b/src/twisted/web/test/test_http_headers.py
@@ -86,38 +86,6 @@ def test_setRawHeaders(self) -> None:
         self.assertTrue(h.hasHeader(b"Test"))
         self.assertEqual(h.getRawHeaders(b"test"), rawValue)
 
-    def test_rawHeadersTypeCheckingValuesIterable(self) -> None:
-        """
-        L{Headers.setRawHeaders} requires values to be of type list.
-        """
-        h = Headers()
-        self.assertRaises(TypeError, h.setRawHeaders, b"key", {b"Foo": b"bar"})
-
-    def test_rawHeadersTypeCheckingName(self) -> None:
-        """
-        L{Headers.setRawHeaders} requires C{name} to be a L{bytes} or
-        L{str} string.
-        """
-        h = Headers()
-        e = self.assertRaises(TypeError, h.setRawHeaders, None, [b"foo"])
-        self.assertEqual(
-            e.args[0],
-            "Header name is an instance of <class 'NoneType'>, " "not bytes or str",
-        )
-
-    def test_rawHeadersTypeCheckingValuesAreString(self) -> None:
-        """
-        L{Headers.setRawHeaders} requires values to a L{list} of L{bytes} or
-        L{str} strings.
-        """
-        h = Headers()
-        e = self.assertRaises(TypeError, h.setRawHeaders, b"key", [b"bar", None])
-        self.assertEqual(
-            e.args[0],
-            "Header value at position 1 is an instance of <class 'NoneType'>, "
-            "not bytes or str",
-        )
-
     def test_addRawHeader(self) -> None:
         """
         L{Headers.addRawHeader} adds a new value for a given header.
@@ -128,30 +96,6 @@ def test_addRawHeader(self) -> None:
         h.addRawHeader(b"test", b"panda")
         self.assertEqual(h.getRawHeaders(b"test"), [b"lemur", b"panda"])
 
-    def test_addRawHeaderTypeCheckName(self) -> None:
-        """
-        L{Headers.addRawHeader} requires C{name} to be a L{bytes} or L{str}
-        string.
-        """
-        h = Headers()
-        e = self.assertRaises(TypeError, h.addRawHeader, None, b"foo")
-        self.assertEqual(
-            e.args[0],
-            "Header name is an instance of <class 'NoneType'>, " "not bytes or str",
-        )
-
-    def test_addRawHeaderTypeCheckValue(self) -> None:
-        """
-        L{Headers.addRawHeader} requires value to be a L{bytes} or L{str}
-        string.
-        """
-        h = Headers()
-        e = self.assertRaises(TypeError, h.addRawHeader, b"key", None)
-        self.assertEqual(
-            e.args[0],
-            "Header value is an instance of <class 'NoneType'>, " "not bytes or str",
-        )
-
     def test_getRawHeadersNoDefault(self) -> None:
         """
         L{Headers.getRawHeaders} returns L{None} if the header is not found and
@@ -232,21 +176,23 @@ def test_removeHeaderDoesntExist(self) -> None:
         h.removeHeader(b"test")
         self.assertEqual(list(h.getAllRawHeaders()), [])
 
-    def test_canonicalNameCaps(self) -> None:
+    def test_encodeName(self) -> None:
         """
-        L{Headers._canonicalNameCaps} returns the canonical capitalization for
+        L{Headers._encodeName} returns the canonical capitalization for
         the given header.
         """
         h = Headers()
-        self.assertEqual(h._canonicalNameCaps(b"test"), b"Test")
-        self.assertEqual(h._canonicalNameCaps(b"test-stuff"), b"Test-Stuff")
-        self.assertEqual(h._canonicalNameCaps(b"content-md5"), b"Content-MD5")
-        self.assertEqual(h._canonicalNameCaps(b"dnt"), b"DNT")
-        self.assertEqual(h._canonicalNameCaps(b"etag"), b"ETag")
-        self.assertEqual(h._canonicalNameCaps(b"p3p"), b"P3P")
-        self.assertEqual(h._canonicalNameCaps(b"te"), b"TE")
-        self.assertEqual(h._canonicalNameCaps(b"www-authenticate"), b"WWW-Authenticate")
-        self.assertEqual(h._canonicalNameCaps(b"x-xss-protection"), b"X-XSS-Protection")
+        self.assertEqual(h._encodeName(b"test"), b"Test")
+        self.assertEqual(h._encodeName(b"test-stuff"), b"Test-Stuff")
+        self.assertEqual(h._encodeName(b"content-md5"), b"Content-MD5")
+        self.assertEqual(h._encodeName(b"dnt"), b"DNT")
+        self.assertEqual(h._encodeName(b"etag"), b"ETag")
+        self.assertEqual(h._encodeName(b"p3p"), b"P3P")
+        self.assertEqual(h._encodeName(b"te"), b"TE")
+        self.assertEqual(h._encodeName(b"www-authenticate"), b"WWW-Authenticate")
+        self.assertEqual(h._encodeName(b"WWW-authenticate"), b"WWW-Authenticate")
+        self.assertEqual(h._encodeName(b"Www-Authenticate"), b"WWW-Authenticate")
+        self.assertEqual(h._encodeName(b"x-xss-protection"), b"X-XSS-Protection")
 
     def test_getAllRawHeaders(self) -> None:
         """
@@ -300,7 +246,7 @@ def test_repr(self) -> None:
         baz = b"baz"
         self.assertEqual(
             repr(Headers({foo: [bar, baz]})),
-            f"Headers({{{foo!r}: [{bar!r}, {baz!r}]}})",
+            f"Headers({{{foo.capitalize()!r}: [{bar!r}, {baz!r}]}})",
         )
 
     def test_reprWithRawBytes(self) -> None:
@@ -317,7 +263,7 @@ def test_reprWithRawBytes(self) -> None:
         baz = b"baz\xe1"
         self.assertEqual(
             repr(Headers({foo: [bar, baz]})),
-            f"Headers({{{foo!r}: [{bar!r}, {baz!r}]}})",
+            f"Headers({{{foo.capitalize()!r}: [{bar!r}, {baz!r}]}})",
         )
 
     def test_subclassRepr(self) -> None:
@@ -334,7 +280,7 @@ class FunnyHeaders(Headers):
 
         self.assertEqual(
             repr(FunnyHeaders({foo: [bar, baz]})),
-            f"FunnyHeaders({{{foo!r}: [{bar!r}, {baz!r}]}})",
+            f"FunnyHeaders({{{foo.capitalize()!r}: [{bar!r}, {baz!r}]}})",
         )
 
     def test_copy(self) -> None:
@@ -352,6 +298,17 @@ def test_copy(self) -> None:
         i.addRawHeader(b"test", b"baz")
         self.assertEqual(h.getRawHeaders(b"test"), [b"foo", b"bar"])
 
+    def test_max_cached_headers(self) -> None:
+        """
+        Only a limited number of HTTP header names get cached.
+        """
+        headers = Headers()
+        for i in range(Headers._MAX_CACHED_HEADERS + 200):
+            headers.addRawHeader(f"hello-{i}", "value")
+        self.assertEqual(
+            len(Headers._canonicalHeaderCache), Headers._MAX_CACHED_HEADERS
+        )
+
 
 class UnicodeHeadersTests(TestCase):
     """
@@ -435,13 +392,6 @@ def test_rawHeadersValueEncoding(self) -> None:
         self.assertTrue(h.hasHeader(b"\xe1"))
         self.assertEqual(h.getRawHeaders(b"\xe1"), [b"\xe2\x98\x83", b"foo"])
 
-    def test_rawHeadersTypeChecking(self) -> None:
-        """
-        L{Headers.setRawHeaders} requires values to be of type sequence
-        """
-        h = Headers()
-        self.assertRaises(TypeError, h.setRawHeaders, "key", {"Foo": "bar"})
-
     def test_addRawHeader(self) -> None:
         """
         L{Headers.addRawHeader} adds a new value for a given header.
@@ -614,7 +564,7 @@ def test_repr(self) -> None:
         foo = "foo\u00E1"
         bar = "bar\u2603"
         baz = "baz"
-        fooEncoded = "'foo\\xe1'"
+        fooEncoded = "'Foo\\xe1'"
         barEncoded = "'bar\\xe2\\x98\\x83'"
         fooEncoded = "b" + fooEncoded
         barEncoded = "b" + barEncoded
@@ -633,7 +583,7 @@ def test_subclassRepr(self) -> None:
         foo = "foo\u00E1"
         bar = "bar\u2603"
         baz = "baz"
-        fooEncoded = "b'foo\\xe1'"
+        fooEncoded = "b'Foo\\xe1'"
         barEncoded = "b'bar\\xe2\\x98\\x83'"
 
         class FunnyHeaders(Headers):