Repository: mitmproxy/mitmproxy

Issue:
Crash when proxying HTTP/2
#### Problem Description
Looks like we have a bug in our HTTP/2 code:

#### Steps to reproduce the behavior:
1. `mitmdump --set proxy_debug -vvv`
3. Browse openstreetmap.org with Firefox

```
127.0.0.1:49187: mitmproxy has crashed!
Traceback (most recent call last):
  File "/mnt/c/Users/user/git/mitmproxy/mitmproxy/proxy/server.py", line 276, in server_event
    for command in layer_commands:
  File "/mnt/c/Users/user/git/mitmproxy/mitmproxy/proxy/layer.py", line 114, in handle_event
    yield from self.__process(command_generator)
  File "/mnt/c/Users/user/git/mitmproxy/mitmproxy/proxy/layer.py", line 142, in __process
    command = next(command_generator)
  File "/mnt/c/Users/user/git/mitmproxy/mitmproxy/proxy/layer.py", line 186, in handle_event
    yield from self._handle(event)
  File "/mnt/c/Users/user/git/mitmproxy/mitmproxy/proxy/layer.py", line 114, in handle_event
    yield from self.__process(command_generator)
  File "/mnt/c/Users/user/git/mitmproxy/mitmproxy/proxy/layer.py", line 142, in __process
    command = next(command_generator)
  File "/mnt/c/Users/user/git/mitmproxy/mitmproxy/proxy/layers/http/__init__.py", line 570, in _handle_event
    yield from self.event_to_child(stream, event)
  File "/mnt/c/Users/user/git/mitmproxy/mitmproxy/proxy/layers/http/__init__.py", line 586, in event_to_child
    for command in child.handle_event(event):
  File "/mnt/c/Users/user/git/mitmproxy/mitmproxy/proxy/layer.py", line 114, in handle_event
    yield from self.__process(command_generator)
  File "/mnt/c/Users/user/git/mitmproxy/mitmproxy/proxy/layer.py", line 142, in __process
    command = next(command_generator)
  File "/mnt/c/Users/user/git/mitmproxy/mitmproxy/proxy/layers/http/__init__.py", line 488, in passthrough
    for command in self.child_layer.handle_event(event):
  File "/mnt/c/Users/user/git/mitmproxy/mitmproxy/proxy/layer.py", line 114, in handle_event
    yield from self.__process(command_generator)
  File "/mnt/c/Users/user/git/mitmproxy/mitmproxy/proxy/layer.py", line 142, in __process
    command = next(command_generator)
  File "/mnt/c/Users/user/git/mitmproxy/mitmproxy/proxy/tunnel.py", line 85, in _handle_event
    yield from self.event_to_child(event)
  File "/mnt/c/Users/user/git/mitmproxy/mitmproxy/proxy/tunnel.py", line 104, in event_to_child
    for command in self.child_layer.handle_event(event):
  File "/mnt/c/Users/user/git/mitmproxy/mitmproxy/proxy/layer.py", line 114, in handle_event
    yield from self.__process(command_generator)
  File "/mnt/c/Users/user/git/mitmproxy/mitmproxy/proxy/layer.py", line 142, in __process
    command = next(command_generator)
  File "/mnt/c/Users/user/git/mitmproxy/mitmproxy/proxy/tunnel.py", line 85, in _handle_event
    yield from self.event_to_child(event)
  File "/mnt/c/Users/user/git/mitmproxy/mitmproxy/proxy/tunnel.py", line 104, in event_to_child
    for command in self.child_layer.handle_event(event):
  File "/mnt/c/Users/user/git/mitmproxy/mitmproxy/proxy/layer.py", line 114, in handle_event
    yield from self.__process(command_generator)
  File "/mnt/c/Users/user/git/mitmproxy/mitmproxy/proxy/layer.py", line 142, in __process
    command = next(command_generator)
  File "/mnt/c/Users/user/git/mitmproxy/mitmproxy/proxy/layers/http/__init__.py", line 570, in _handle_event
    yield from self.event_to_child(stream, event)
  File "/mnt/c/Users/user/git/mitmproxy/mitmproxy/proxy/layers/http/__init__.py", line 601, in event_to_child
    yield from self.event_to_child(conn, command.event)
  File "/mnt/c/Users/user/git/mitmproxy/mitmproxy/proxy/layers/http/__init__.py", line 586, in event_to_child
    for command in child.handle_event(event):
  File "/mnt/c/Users/user/git/mitmproxy/mitmproxy/proxy/layer.py", line 114, in handle_event
    yield from self.__process(command_generator)
  File "/mnt/c/Users/user/git/mitmproxy/mitmproxy/proxy/layer.py", line 123, in __process
    command = command_generator.send(send)
  File "/mnt/c/Users/user/git/mitmproxy/mitmproxy/proxy/layers/http/_http2.py", line 266, in _handle_event
    yield from super()._handle_event(event)
  File "/mnt/c/Users/user/git/mitmproxy/mitmproxy/proxy/layers/http/_http2.py", line 91, in _handle_event
    if stream.state_machine.state is not h2.stream.StreamState.CLOSED:
AttributeError: 'NoneType' object has no attribute 'state_machine'
```

#### System Information
```
Mitmproxy: 7.0.0.dev (+330, commit b5d9dcd)
Python:    3.9.1
OpenSSL:   OpenSSL 1.1.1i  8 Dec 2020
Platform:  Linux-4.4.0-19041-Microsoft-x86_64-with-glibc2.27
```

Crash when proxying HTTP/2
#### Problem Description
Looks like we have a bug in our HTTP/2 code:

#### Steps to reproduce the behavior:
1. `mitmdump --set proxy_debug -vvv`
3. Browse openstreetmap.org with Firefox

```
127.0.0.1:49187: mitmproxy has crashed!
Traceback (most recent call last):
  File "/mnt/c/Users/user/git/mitmproxy/mitmproxy/proxy/server.py", line 276, in server_event
    for command in layer_commands:
  File "/mnt/c/Users/user/git/mitmproxy/mitmproxy/proxy/layer.py", line 114, in handle_event
    yield from self.__process(command_generator)
  File "/mnt/c/Users/user/git/mitmproxy/mitmproxy/proxy/layer.py", line 142, in __process
    command = next(command_generator)
  File "/mnt/c/Users/user/git/mitmproxy/mitmproxy/proxy/layer.py", line 186, in handle_event
    yield from self._handle(event)
  File "/mnt/c/Users/user/git/mitmproxy/mitmproxy/proxy/layer.py", line 114, in handle_event
    yield from self.__process(command_generator)
  File "/mnt/c/Users/user/git/mitmproxy/mitmproxy/proxy/layer.py", line 142, in __process
    command = next(command_generator)
  File "/mnt/c/Users/user/git/mitmproxy/mitmproxy/proxy/layers/http/__init__.py", line 570, in _handle_event
    yield from self.event_to_child(stream, event)
  File "/mnt/c/Users/user/git/mitmproxy/mitmproxy/proxy/layers/http/__init__.py", line 586, in event_to_child
    for command in child.handle_event(event):
  File "/mnt/c/Users/user/git/mitmproxy/mitmproxy/proxy/layer.py", line 114, in handle_event
    yield from self.__process(command_generator)
  File "/mnt/c/Users/user/git/mitmproxy/mitmproxy/proxy/layer.py", line 142, in __process
    command = next(command_generator)
  File "/mnt/c/Users/user/git/mitmproxy/mitmproxy/proxy/layers/http/__init__.py", line 488, in passthrough
    for command in self.child_layer.handle_event(event):
  File "/mnt/c/Users/user/git/mitmproxy/mitmproxy/proxy/layer.py", line 114, in handle_event
    yield from self.__process(command_generator)
  File "/mnt/c/Users/user/git/mitmproxy/mitmproxy/proxy/layer.py", line 142, in __process
    command = next(command_generator)
  File "/mnt/c/Users/user/git/mitmproxy/mitmproxy/proxy/tunnel.py", line 85, in _handle_event
    yield from self.event_to_child(event)
  File "/mnt/c/Users/user/git/mitmproxy/mitmproxy/proxy/tunnel.py", line 104, in event_to_child
    for command in self.child_layer.handle_event(event):
  File "/mnt/c/Users/user/git/mitmproxy/mitmproxy/proxy/layer.py", line 114, in handle_event
    yield from self.__process(command_generator)
  File "/mnt/c/Users/user/git/mitmproxy/mitmproxy/proxy/layer.py", line 142, in __process
    command = next(command_generator)
  File "/mnt/c/Users/user/git/mitmproxy/mitmproxy/proxy/tunnel.py", line 85, in _handle_event
    yield from self.event_to_child(event)
  File "/mnt/c/Users/user/git/mitmproxy/mitmproxy/proxy/tunnel.py", line 104, in event_to_child
    for command in self.child_layer.handle_event(event):
  File "/mnt/c/Users/user/git/mitmproxy/mitmproxy/proxy/layer.py", line 114, in handle_event
    yield from self.__process(command_generator)
  File "/mnt/c/Users/user/git/mitmproxy/mitmproxy/proxy/layer.py", line 142, in __process
    command = next(command_generator)
  File "/mnt/c/Users/user/git/mitmproxy/mitmproxy/proxy/layers/http/__init__.py", line 570, in _handle_event
    yield from self.event_to_child(stream, event)
  File "/mnt/c/Users/user/git/mitmproxy/mitmproxy/proxy/layers/http/__init__.py", line 601, in event_to_child
    yield from self.event_to_child(conn, command.event)
  File "/mnt/c/Users/user/git/mitmproxy/mitmproxy/proxy/layers/http/__init__.py", line 586, in event_to_child
    for command in child.handle_event(event):
  File "/mnt/c/Users/user/git/mitmproxy/mitmproxy/proxy/layer.py", line 114, in handle_event
    yield from self.__process(command_generator)
  File "/mnt/c/Users/user/git/mitmproxy/mitmproxy/proxy/layer.py", line 123, in __process
    command = command_generator.send(send)
  File "/mnt/c/Users/user/git/mitmproxy/mitmproxy/proxy/layers/http/_http2.py", line 266, in _handle_event
    yield from super()._handle_event(event)
  File "/mnt/c/Users/user/git/mitmproxy/mitmproxy/proxy/layers/http/_http2.py", line 91, in _handle_event
    if stream.state_machine.state is not h2.stream.StreamState.CLOSED:
AttributeError: 'NoneType' object has no attribute 'state_machine'
```

#### System Information
```
Mitmproxy: 7.0.0.dev (+330, commit b5d9dcd)
Python:    3.9.1
OpenSSL:   OpenSSL 1.1.1i  8 Dec 2020
Platform:  Linux-4.4.0-19041-Microsoft-x86_64-with-glibc2.27
```



Discussion:

Resolving patch:
diff --git a/mitmproxy/proxy/layers/http/_http2.py b/mitmproxy/proxy/layers/http/_http2.py
--- a/mitmproxy/proxy/layers/http/_http2.py
+++ b/mitmproxy/proxy/layers/http/_http2.py
@@ -64,11 +64,29 @@ def __init__(self, context: Context, conn: Connection):
     def is_closed(self, stream_id: int) -> bool:
         """Check if a non-idle stream is closed"""
         stream = self.h2_conn.streams.get(stream_id, None)
-        if stream is not None:
-            return stream.closed
+        if (
+            stream is not None
+            and
+            stream.state_machine.state is not h2.stream.StreamState.CLOSED
+        ):
+            return False
         else:
             return True
 
+    def is_open_for_us(self, stream_id: int) -> bool:
+        """Check if we can write to a non-idle stream."""
+        stream = self.h2_conn.streams.get(stream_id, None)
+        if (
+            stream is not None
+            and
+            stream.state_machine.state is not h2.stream.StreamState.HALF_CLOSED_LOCAL
+            and
+            stream.state_machine.state is not h2.stream.StreamState.CLOSED
+        ):
+            return True
+        else:
+            return False
+
     def _handle_event(self, event: Event) -> CommandGenerator[None]:
         if isinstance(event, Start):
             self.h2_conn.initiate_connection()
@@ -77,24 +95,18 @@ def _handle_event(self, event: Event) -> CommandGenerator[None]:
         elif isinstance(event, HttpEvent):
             if isinstance(event, self.SendData):
                 assert isinstance(event, (RequestData, ResponseData))
-                self.h2_conn.send_data(event.stream_id, event.data)
+                if self.is_open_for_us(event.stream_id):
+                    self.h2_conn.send_data(event.stream_id, event.data)
             elif isinstance(event, self.SendEndOfMessage):
-                stream = self.h2_conn.streams.get(event.stream_id)
-                if stream.state_machine.state not in (h2.stream.StreamState.HALF_CLOSED_LOCAL,
-                                                      h2.stream.StreamState.CLOSED):
+                if self.is_open_for_us(event.stream_id):
                     self.h2_conn.end_stream(event.stream_id)
-                if self.is_closed(event.stream_id):
-                    self.streams.pop(event.stream_id, None)
             elif isinstance(event, self.SendProtocolError):
                 assert isinstance(event, (RequestProtocolError, ResponseProtocolError))
-                stream = self.h2_conn.streams.get(event.stream_id)
-                if stream.state_machine.state is not h2.stream.StreamState.CLOSED:
+                if not self.is_closed(event.stream_id):
                     code = {
                         status_codes.CLIENT_CLOSED_REQUEST: h2.errors.ErrorCodes.CANCEL,
                     }.get(event.code, h2.errors.ErrorCodes.INTERNAL_ERROR)
                     self.h2_conn.reset_stream(event.stream_id, code)
-                if self.is_closed(event.stream_id):
-                    self.streams.pop(event.stream_id, None)
             else:
                 raise AssertionError(f"Unexpected event: {event}")
             data_to_send = self.h2_conn.data_to_send()
@@ -250,18 +262,19 @@ def __init__(self, context: Context):
 
     def _handle_event(self, event: Event) -> CommandGenerator[None]:
         if isinstance(event, ResponseHeaders):
-            headers = [
-                (b":status", b"%d" % event.response.status_code),
-                *event.response.headers.fields
-            ]
-            if not event.response.is_http2:
-                headers = normalize_h1_headers(headers, False)
-
-            self.h2_conn.send_headers(
-                event.stream_id,
-                headers,
-            )
-            yield SendData(self.conn, self.h2_conn.data_to_send())
+            if self.is_open_for_us(event.stream_id):
+                headers = [
+                    (b":status", b"%d" % event.response.status_code),
+                    *event.response.headers.fields
+                ]
+                if not event.response.is_http2:
+                    headers = normalize_h1_headers(headers, False)
+
+                self.h2_conn.send_headers(
+                    event.stream_id,
+                    headers,
+                )
+                yield SendData(self.conn, self.h2_conn.data_to_send())
         else:
             yield from super()._handle_event(event)
 


Test patch: