Repository: mitmproxy/mitmproxy

Issue:
HttpStream.state_wait_for_request_headers error
#### Problem Description
```
mit_server          | 2021-10-19T07:17:01.079759751Z 37.103.81.174:57654: mitmproxy has crashed!
mit_server          | 2021-10-19T07:17:01.079788763Z Traceback (most recent call last):
mit_server          | 2021-10-19T07:17:01.079794934Z   File "/usr/local/lib/python3.9/site-packages/mitmproxy/proxy/server.py", line 279, in server_event
mit_server          | 2021-10-19T07:17:01.079799991Z     for command in layer_commands:
mit_server          | 2021-10-19T07:17:01.079804504Z   File "/usr/local/lib/python3.9/site-packages/mitmproxy/proxy/layer.py", line 144, in handle_event
mit_server          | 2021-10-19T07:17:01.079809056Z     command = command_generator.send(send)
mit_server          | 2021-10-19T07:17:01.079813637Z   File "/usr/local/lib/python3.9/site-packages/mitmproxy/proxy/layer.py", line 255, in handle_event
mit_server          | 2021-10-19T07:17:01.079818387Z     yield from self._handle(event)
mit_server          | 2021-10-19T07:17:01.079826095Z   File "/usr/local/lib/python3.9/site-packages/mitmproxy/proxy/layer.py", line 144, in handle_event
mit_server          | 2021-10-19T07:17:01.079830842Z     command = command_generator.send(send)
mit_server          | 2021-10-19T07:17:01.079835186Z   File "/usr/local/lib/python3.9/site-packages/mitmproxy/proxy/layers/http/__init__.py", line 740, in _handle_event
mit_server          | 2021-10-19T07:17:01.079839927Z     yield from self.event_to_child(handler, event)
mit_server          | 2021-10-19T07:17:01.079844281Z   File "/usr/local/lib/python3.9/site-packages/mitmproxy/proxy/layers/http/__init__.py", line 761, in event_to_child
mit_server          | 2021-10-19T07:17:01.079849161Z     yield from self.event_to_child(stream, command.event)
mit_server          | 2021-10-19T07:17:01.079853589Z   File "/usr/local/lib/python3.9/site-packages/mitmproxy/proxy/layers/http/__init__.py", line 749, in event_to_child
mit_server          | 2021-10-19T07:17:01.079858633Z     for command in child.handle_event(event):
mit_server          | 2021-10-19T07:17:01.079863297Z   File "/usr/local/lib/python3.9/site-packages/mitmproxy/proxy/layer.py", line 144, in handle_event
mit_server          | 2021-10-19T07:17:01.079902048Z     command = command_generator.send(send)
mit_server          | 2021-10-19T07:17:01.079906606Z   File "/usr/local/lib/python3.9/site-packages/mitmproxy/proxy/layers/http/__init__.py", line 136, in _handle_event
mit_server          | 2021-10-19T07:17:01.079910826Z     yield from self.client_state(event)
mit_server          | 2021-10-19T07:17:01.079914867Z   File "/usr/local/lib/python3.9/site-packages/mitmproxy/proxy/utils.py", line 23, in _check_event_type
mit_server          | 2021-10-19T07:17:01.079919260Z     raise AssertionError(
mit_server          | 2021-10-19T07:17:01.079925280Z AssertionError: Unexpected event type at HttpStream.state_wait_for_request_headers: Expected RequestHeaders, got RequestData(stream_id=1, data=b"\x16\x03\x03\x00\xbe\x01\x00\x00\xba\x03\x03anp\xedr'\xd2\xa1\xe5\xf0 9\xf7\r\xef\x9dd/\x8d\xf8r*&K\xf7(\xe8\xd6\xf6\x96\x16\xa3\x00\x00*\xc0,\xc0+\xc00\xc0/\x00\x9f\x00\x9e\xc0$\xc0#\xc0(\xc0'\xc0\n\xc0\t\xc0\x14\xc0\x13\x00\x9d\x00\x9c\x00=\x00<\x005\x00/\x00\n\x01\x00\x00g\x00\x00\x00&\x00$\x00\x00!authentication-prod.ar.indazn.com\x00\n\x00\x08\x00\x06\x00\x1d\x00\x17\x00\x18\x00\x0b\x00\x02\x01\x00\x00\r\x00\x1a\x00\x18\x08\x04\x08\x05\x08\x06\x04\x01\x05\x01\x02\x01\x04\x03\x05\x03\x02\x03\x02\x02\x06\x01\x06\x03\x00#\x00\x00\x00\x17\x00\x00\xff\x01\x00\x01\x00").
mit_server          | 2021-10-19T07:17:01.079939109Z 
mit_server          | 2021-10-19T07:17:04.149705128Z 37.103.81.174:49416: mitmproxy has crashed!
mit_server          | 2021-10-19T07:17:04.149733496Z Traceback (most recent call last):
mit_server          | 2021-10-19T07:17:04.149738250Z   File "/usr/local/lib/python3.9/site-packages/mitmproxy/proxy/server.py", line 279, in server_event
mit_server          | 2021-10-19T07:17:04.149742511Z     for command in layer_commands:
mit_server          | 2021-10-19T07:17:04.149746107Z   File "/usr/local/lib/python3.9/site-packages/mitmproxy/proxy/layer.py", line 144, in handle_event
mit_server          | 2021-10-19T07:17:04.149749966Z     command = command_generator.send(send)
mit_server          | 2021-10-19T07:17:04.149753507Z   File "/usr/local/lib/python3.9/site-packages/mitmproxy/proxy/layer.py", line 255, in handle_event
mit_server          | 2021-10-19T07:17:04.149757131Z     yield from self._handle(event)
mit_server          | 2021-10-19T07:17:04.149760429Z   File "/usr/local/lib/python3.9/site-packages/mitmproxy/proxy/layer.py", line 144, in handle_event
mit_server          | 2021-10-19T07:17:04.149763878Z     command = command_generator.send(send)
mit_server          | 2021-10-19T07:17:04.149767309Z   File "/usr/local/lib/python3.9/site-packages/mitmproxy/proxy/layers/http/__init__.py", line 740, in _handle_event
mit_server          | 2021-10-19T07:17:04.149770905Z     yield from self.event_to_child(handler, event)
mit_server          | 2021-10-19T07:17:04.149774329Z   File "/usr/local/lib/python3.9/site-packages/mitmproxy/proxy/layers/http/__init__.py", line 749, in event_to_child
mit_server          | 2021-10-19T07:17:04.149777989Z     for command in child.handle_event(event):
mit_server          | 2021-10-19T07:17:04.149781325Z   File "/usr/local/lib/python3.9/site-packages/mitmproxy/proxy/layer.py", line 135, in handle_event
mit_server          | 2021-10-19T07:17:04.149784903Z     command_generator = self._handle_event(event)
mit_server          | 2021-10-19T07:17:04.149788233Z   File "/usr/local/lib/python3.9/site-packages/mitmproxy/proxy/utils.py", line 23, in _check_event_type
mit_server          | 2021-10-19T07:17:04.149805983Z     raise AssertionError(
mit_server          | 2021-10-19T07:17:04.149809318Z AssertionError: Unexpected event type at HttpStream._handle_event: Expected Start|HttpEvent, got ConnectionClosed(connection=Server({'id': '…a2bdcc', 'address': ('api-global.netflix.com', 443), 'state': <ConnectionState.CAN_WRITE: 2>, 'timestamp_start': 1634627762.836605, 'timestamp_tcp_setup': 1634627763.5006588, 'peername': ('34.214.223.171', 443), 'sockname': ('172.23.0.4', 57018)})).
```

#### Steps to reproduce the behavior:
1. After a period of use, the client shuts down the proxy and the server keeps printing error messages
2.The client is iphone

#### System Information
Mitmproxy: 7.0.4
Python:    3.9.7
OpenSSL:   OpenSSL 1.1.1l  24 Aug 2021
Platform:  Linux-3.10.0-1127.18.2.el7.x86_64-x86_64-with-glibc2.28



Discussion:
Thanks! How hard is this to reproduce? Could you reproduce it with `--set proxy_debug` and send me the corresponding log either here or by email?
Thanks for your reply, this is my log,Because the log is too large, I only intercepted part of it




------------------&nbsp;原始邮件&nbsp;------------------
发件人:                                                                                                                        "mitmproxy/mitmproxy"                                                                                    ***@***.***&gt;;
发送时间:&nbsp;2021年10月19日(星期二) 下午3:50
***@***.***&gt;;
***@***.******@***.***&gt;;
主题:&nbsp;Re: [mitmproxy/mitmproxy] HttpStream.state_wait_for_request_headers error (Issue #4870)





 
Thanks! How hard is this to reproduce? Could you reproduce it with --set proxy_debug and send me the corresponding log either here or by email?
 
—
You are receiving this because you authored the thread.
Reply to this email directly, view it on GitHub, or unsubscribe.
Triage notifications on the go with GitHub Mobile for iOS or Android.
Thanks for your reply, this is my log,Because the log is too large, I only intercepted part of it
[mitm.log](https://github.com/mitmproxy/mitmproxy/files/7371894/mitm.log)


Resolving patch:
diff --git a/mitmproxy/proxy/layers/http/__init__.py b/mitmproxy/proxy/layers/http/__init__.py
--- a/mitmproxy/proxy/layers/http/__init__.py
+++ b/mitmproxy/proxy/layers/http/__init__.py
@@ -592,11 +592,11 @@ def handle_connect_finish(self):
             )
 
         if 200 <= self.flow.response.status_code < 300:
-            yield SendHttp(ResponseHeaders(self.stream_id, self.flow.response, True), self.context.client)
-            yield SendHttp(ResponseEndOfMessage(self.stream_id), self.context.client)
             self.child_layer = self.child_layer or layer.NextLayer(self.context)
             yield from self.child_layer.handle_event(events.Start())
             self._handle_event = self.passthrough
+            yield SendHttp(ResponseHeaders(self.stream_id, self.flow.response, True), self.context.client)
+            yield SendHttp(ResponseEndOfMessage(self.stream_id), self.context.client)
         else:
             yield from self.send_response()
 
diff --git a/mitmproxy/proxy/layers/http/_http1.py b/mitmproxy/proxy/layers/http/_http1.py
--- a/mitmproxy/proxy/layers/http/_http1.py
+++ b/mitmproxy/proxy/layers/http/_http1.py
@@ -122,7 +122,10 @@ def make_pipe(self) -> layer.CommandGenerator[None]:
         self.state = self.passthrough
         if self.buf:
             already_received = self.buf.maybe_extract_at_most(len(self.buf))
-            yield from self.state(events.DataReceived(self.conn, already_received))
+            # Some clients send superfluous newlines after CONNECT, we want to eat those.
+            already_received = already_received.lstrip(b"\r\n")
+            if already_received:
+                yield from self.state(events.DataReceived(self.conn, already_received))
 
     def passthrough(self, event: events.Event) -> layer.CommandGenerator[None]:
         assert self.stream_id


Test patch:
diff --git a/test/mitmproxy/proxy/layers/http/test_http.py b/test/mitmproxy/proxy/layers/http/test_http.py
--- a/test/mitmproxy/proxy/layers/http/test_http.py
+++ b/test/mitmproxy/proxy/layers/http/test_http.py
@@ -1324,3 +1324,23 @@ def make_chunked(flow: HTTPFlow):
                                      b"Transfer-Encoding: chunked\r\n\r\n"
                                      b"0\r\n\r\n")
     )
+
+
+def test_connect_more_newlines(tctx):
+    """Ignore superfluous \r\n in CONNECT request, https://github.com/mitmproxy/mitmproxy/issues/4870"""
+    server = Placeholder(Server)
+    playbook = Playbook(http.HttpLayer(tctx, HTTPMode.regular))
+    nl = Placeholder(layer.NextLayer)
+
+    assert (
+        playbook
+        >> DataReceived(tctx.client, b"CONNECT example.com:80 HTTP/1.1\r\n\r\n\r\n")
+        << http.HttpConnectHook(Placeholder())
+        >> reply()
+        << OpenConnection(server)
+        >> reply(None)
+        << SendData(tctx.client, b'HTTP/1.1 200 Connection established\r\n\r\n')
+        >> DataReceived(tctx.client, b"\x16\x03\x03\x00\xb3\x01\x00\x00\xaf\x03\x03")
+        << layer.NextLayerHook(nl)
+    )
+    assert nl().data_client() == b"\x16\x03\x03\x00\xb3\x01\x00\x00\xaf\x03\x03"