Repository: mitmproxy/mitmproxy

Issue:
Warn when sending uppercase HTTP/2 headers
The following addon will make some requests fail with HTTP/2 protocol errors:

```python
def request(flow):
    flow.request.headers["Content-Type"] = "application/json"
```

It's not immediately obvious what's the problem here, but HTTP/2 headers may only be lowercase. We are smart enough to automatically fix that for requests that we received as HTTP/1, but otherwise we intentionally just keep things as-is. This is hard to debug for addon authors, we should emit a warning if we see uppercase HTTP/2 headers being sent.
Warn when sending uppercase HTTP/2 headers
The following addon will make some requests fail with HTTP/2 protocol errors:

```python
def request(flow):
    flow.request.headers["Content-Type"] = "application/json"
```

It's not immediately obvious what's the problem here, but HTTP/2 headers may only be lowercase. We are smart enough to automatically fix that for requests that we received as HTTP/1, but otherwise we intentionally just keep things as-is. This is hard to debug for addon authors, we should emit a warning if we see uppercase HTTP/2 headers being sent.


Discussion:
Why do we *intentionally just keep things as-is*?
Why not apply the same smartness by default, and maybe provide a `--strict` mode or similar? Less surprises for most users, and for the advanced users, they can opt into a strict mode for tinkering with the protocols.
Fair point. Let's add an option and normalize by default.
Why do we *intentionally just keep things as-is*?
Why not apply the same smartness by default, and maybe provide a `--strict` mode or similar? Less surprises for most users, and for the advanced users, they can opt into a strict mode for tinkering with the protocols.
Fair point. Let's add an option and normalize by default.

Resolving patch:
diff --git a/mitmproxy/addons/proxyserver.py b/mitmproxy/addons/proxyserver.py
--- a/mitmproxy/addons/proxyserver.py
+++ b/mitmproxy/addons/proxyserver.py
@@ -90,6 +90,14 @@ def load(self, loader):
             "proxy_debug", bool, False,
             "Enable debug logs in the proxy core.",
         )
+        loader.add_option(
+            "normalize_outbound_headers", bool, True,
+            """
+            Normalize outgoing HTTP/2 header names, but emit a warning when doing so.
+            HTTP/2 does not allow uppercase header names. This option makes sure that HTTP/2 headers set
+            in custom scripts are lowercased before they are sent.
+            """,
+        )
 
     def running(self):
         self.master = ctx.master
diff --git a/mitmproxy/proxy/layers/http/_http2.py b/mitmproxy/proxy/layers/http/_http2.py
--- a/mitmproxy/proxy/layers/http/_http2.py
+++ b/mitmproxy/proxy/layers/http/_http2.py
@@ -269,6 +269,13 @@ def normalize_h1_headers(headers: List[Tuple[bytes, bytes]], is_client: bool) ->
     return headers
 
 
+def normalize_h2_headers(headers: List[Tuple[bytes, bytes]]) -> CommandGenerator[None]:
+    for i in range(len(headers)):
+        if not headers[i][0].islower():
+            yield Log(f"Lowercased {repr(headers[i][0]).lstrip('b')} header as uppercase is not allowed with HTTP/2.")
+            headers[i] = (headers[i][0].lower(), headers[i][1])
+
+
 class Http2Server(Http2Connection):
     h2_conf = h2.config.H2Configuration(
         **Http2Connection.h2_conf_defaults,
@@ -290,7 +297,10 @@ def _handle_event(self, event: Event) -> CommandGenerator[None]:
                     (b":status", b"%d" % event.response.status_code),
                     *event.response.headers.fields
                 ]
-                if not event.response.is_http2:
+                if event.response.is_http2:
+                    if self.context.options.normalize_outbound_headers:
+                        yield from normalize_h2_headers(headers)
+                else:
                     headers = normalize_h1_headers(headers, False)
 
                 self.h2_conn.send_headers(
@@ -407,6 +417,8 @@ def _handle_event2(self, event: Event) -> CommandGenerator[None]:
 
             if event.request.is_http2:
                 hdrs = list(event.request.headers.fields)
+                if self.context.options.normalize_outbound_headers:
+                    yield from normalize_h2_headers(hdrs)
             else:
                 headers = event.request.headers
                 if not event.request.authority and "host" in headers:


Test patch:
diff --git a/test/mitmproxy/proxy/layers/http/test_http2.py b/test/mitmproxy/proxy/layers/http/test_http2.py
--- a/test/mitmproxy/proxy/layers/http/test_http2.py
+++ b/test/mitmproxy/proxy/layers/http/test_http2.py
@@ -10,7 +10,7 @@
 from mitmproxy.flow import Error
 from mitmproxy.http import HTTPFlow, Headers, Request
 from mitmproxy.net.http import status_codes
-from mitmproxy.proxy.commands import CloseConnection, OpenConnection, SendData
+from mitmproxy.proxy.commands import CloseConnection, Log, OpenConnection, SendData
 from mitmproxy.proxy.context import Context
 from mitmproxy.proxy.events import ConnectionClosed, DataReceived
 from mitmproxy.proxy.layers import http
@@ -353,19 +353,19 @@ def enable_response_streaming(flow: HTTPFlow):
 
 
 @pytest.mark.xfail(reason="inbound validation turned on to protect against request smuggling")
-def test_no_normalization(tctx):
+@pytest.mark.parametrize("normalize", [True, False])
+def test_no_normalization(tctx, normalize):
     """Test that we don't normalize headers when we just pass them through."""
+    tctx.options.normalize_outbound_headers = normalize
 
     server = Placeholder(Server)
     flow = Placeholder(HTTPFlow)
     playbook, cff = start_h2_client(tctx)
 
-    request_headers = example_request_headers + (
-        (b"Should-Not-Be-Capitalized! ", b" :) "),
-    )
-    response_headers = example_response_headers + (
-        (b"Same", b"Here"),
-    )
+    request_headers = list(example_request_headers) + [(b"Should-Not-Be-Capitalized! ", b" :) ")]
+    request_headers_lower = [(k.lower(), v) for (k, v) in request_headers]
+    response_headers = list(example_response_headers) + [(b"Same", b"Here")]
+    response_headers_lower = [(k.lower(), v) for (k, v) in response_headers]
 
     initial = Placeholder(bytes)
     assert (
@@ -385,18 +385,22 @@ def test_no_normalization(tctx):
         hyperframe.frame.SettingsFrame,
         hyperframe.frame.HeadersFrame,
     ]
-    assert hpack.hpack.Decoder().decode(frames[1].data, True) == list(request_headers)
+    assert hpack.hpack.Decoder().decode(frames[1].data, True) == request_headers_lower if normalize else request_headers
 
     sff = FrameFactory()
-    assert (
+    (
             playbook
             >> DataReceived(server, sff.build_headers_frame(response_headers, flags=["END_STREAM"]).serialize())
             << http.HttpResponseHeadersHook(flow)
             >> reply()
             << http.HttpResponseHook(flow)
             >> reply()
-            << SendData(tctx.client, cff.build_headers_frame(response_headers, flags=["END_STREAM"]).serialize())
     )
+    if normalize:
+        playbook << Log("Lowercased 'Same' header as uppercase is not allowed with HTTP/2.")
+    hdrs = response_headers_lower if normalize else response_headers
+    assert playbook << SendData(tctx.client, cff.build_headers_frame(hdrs, flags=["END_STREAM"]).serialize())
+
     assert flow().request.headers.fields == ((b"Should-Not-Be-Capitalized! ", b" :) "),)
     assert flow().response.headers.fields == ((b"Same", b"Here"),)