Repository: mitmproxy/mitmproxy

Issue:
Does not handle upstream closing the connection
##### Steps to reproduce the problem:

1. Use a setup such as redirect_requests.py
2. Send a keepalive request into MITM
3. This is passed onto the upstream server and response returned (fine)
4. Wait a little while - upstream closes the connection with a FIN
5. (MITMproxy did not respond to this)
6. Wait a bit longer and send another request to MITM
7. MITM attempts to send this to the upstream which responds with a RST
8. Client sees a 502 bad gateway error

##### Any other comments? What have you tried so far?
In our case we were using https://www.npmjs.com/package/json-server#cli-usage as the upstream server but it appears to be complying with keepalive docs.  Just very surprised we are the only people seeing this :-)

Workaround for us was to add flow.request.headers['Connection'] = 'close', this closed the upstream connection after each request.


```
GET /config HTTP/1.1
Host: 127.0.0.1
Connection: keep-alive
Cache-Control: max-age=0
Upgrade-Insecure-Requests: 1
User-Agent: Mozilla/5.0 (Linux; Android 7.1.1; Android SDK built for x86 Build/NYC) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.91 Mobile Safari/537.36
Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8
Accept-Encoding: identity
Accept-Language: en-US,en;q=0.8

HTTP/1.1 200 OK
X-Powered-By: Express
Vary: Origin, Accept-Encoding
Access-Control-Allow-Credentials: true
Cache-Control: no-cache
Pragma: no-cache
Expires: -1
X-Content-Type-Options: nosniff
Content-Type: application/json; charset=utf-8
Content-Length: 4094
ETag: W/"ffe-YXiFVkdRXmM5eFcp/fDfl4O1X+w"
Date: Wed, 04 Apr 2018 12:53:09 GMT
Connection: keep-alive

{
  "content": {
....
}

[ NB: FIN is sent by upstream and received by MITM here]

GET /config HTTP/1.1
Host: 127.0.0.1
Connection: keep-alive
Cache-Control: max-age=0
Upgrade-Insecure-Requests: 1
User-Agent: Mozilla/5.0 (Linux; Android 7.1.1; Android SDK built for x86 Build/NYC) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.91 Mobile Safari/537.36
Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8
Accept-Encoding: identity
Accept-Language: en-US,en;q=0.8
If-None-Match: W/"ffe-YXiFVkdRXmM5eFcp/fDfl4O1X+w"
 
[ RST now received]
```

##### System information

Mitmproxy: 3.0.3 
Python:    3.6.5
OpenSSL:   OpenSSL 1.0.2o  27 Mar 2018
Platform:  Darwin-17.5.0-x86_64-i386-64bit



Discussion:
I thought this was a problem with my servers. Thanks for this workaround 👍 
Thanks for the super useful bug report! We should re-connect in these cases, not sure why we don't do this anymore.
Ok, I found some easy steps to reproduce this:

```
$ pathod
$ mitmdump --mode reverse:http://localhost:9999 -vvv
$ pathoc localhost:8080 "get:'/p/200:da'" get:/p/200
```
`git bisect` reveals this was broken in https://github.com/mitmproxy/mitmproxy/commit/9e1902be62ba5a6c8d6d1496b3cb18f7deb0e583.

Resolving patch:
diff --git a/mitmproxy/proxy/protocol/http.py b/mitmproxy/proxy/protocol/http.py
--- a/mitmproxy/proxy/protocol/http.py
+++ b/mitmproxy/proxy/protocol/http.py
@@ -333,8 +333,20 @@ def _process_flow(self, f):
                     f.request.scheme
                 )
 
-                try:
+                def get_response():
                     self.send_request_headers(f.request)
+                    if f.request.stream:
+                        chunks = self.read_request_body(f.request)
+                        if callable(f.request.stream):
+                            chunks = f.request.stream(chunks)
+                        self.send_request_body(f.request, chunks)
+                    else:
+                        self.send_request_body(f.request, [f.request.data.content])
+
+                    f.response = self.read_response_headers()
+
+                try:
+                    get_response()
                 except exceptions.NetlibException as e:
                     self.log(
                         "server communication error: %s" % repr(e),
@@ -357,22 +369,17 @@ def _process_flow(self, f):
                         raise exceptions.ProtocolException(
                             "First and only attempt to get response via HTTP2 failed."
                         )
+                    elif f.request.stream:
+                        # We may have already consumed some request chunks already,
+                        # so all we can do is signal downstream that upstream closed the connection.
+                        self.send_error_response(408, "Request Timeout")
+                        f.error = flow.Error(repr(e))
+                        self.channel.ask("error", f)
+                        return False
 
                     self.disconnect()
                     self.connect()
-                    self.send_request_headers(f.request)
-
-                # This is taken out of the try except block because when streaming
-                # we can't send the request body while retrying as the generator gets exhausted
-                if f.request.stream:
-                    chunks = self.read_request_body(f.request)
-                    if callable(f.request.stream):
-                        chunks = f.request.stream(chunks)
-                    self.send_request_body(f.request, chunks)
-                else:
-                    self.send_request_body(f.request, [f.request.data.content])
-
-                f.response = self.read_response_headers()
+                    get_response()
 
                 # call the appropriate script hook - this is an opportunity for
                 # an inline script to set f.stream = True


Test patch:
diff --git a/test/mitmproxy/proxy/test_server.py b/test/mitmproxy/proxy/test_server.py
--- a/test/mitmproxy/proxy/test_server.py
+++ b/test/mitmproxy/proxy/test_server.py
@@ -229,28 +229,14 @@ def test_connection_close(self):
                 p.request("get:'%s'" % response)
 
     def test_reconnect(self):
-        req = "get:'%s/p/200:b@1'" % self.server.urlbase
+        req = "get:'%s/p/200:b@1:da'" % self.server.urlbase
         p = self.pathoc()
 
-        class MockOnce:
-            call = 0
-
-            def mock_once(self, http1obj, req):
-                self.call += 1
-                if self.call == 1:
-                    raise exceptions.TcpDisconnect
-                else:
-                    headers = http1.assemble_request_head(req)
-                    http1obj.server_conn.wfile.write(headers)
-                    http1obj.server_conn.wfile.flush()
-
         with p.connect():
-            with mock.patch("mitmproxy.proxy.protocol.http1.Http1Layer.send_request_headers",
-                            side_effect=MockOnce().mock_once, autospec=True):
-                # Server disconnects while sending headers but mitmproxy reconnects
-                resp = p.request(req)
-                assert resp
-                assert resp.status_code == 200
+            assert p.request(req)
+            # Server has disconnected. Mitmproxy should detect this, and reconnect.
+            assert p.request(req)
+            assert p.request(req)
 
     def test_get_connection_switching(self):
         req = "get:'%s/p/200:b@1'"
@@ -1045,22 +1031,6 @@ def test_reconnect(self):
         request again.
         """
 
-        class MockOnce:
-            call = 0
-
-            def mock_once(self, http1obj, req):
-                self.call += 1
-
-                if self.call == 2:
-                    headers = http1.assemble_request_head(req)
-                    http1obj.server_conn.wfile.write(headers)
-                    http1obj.server_conn.wfile.flush()
-                    raise exceptions.TcpDisconnect
-                else:
-                    headers = http1.assemble_request_head(req)
-                    http1obj.server_conn.wfile.write(headers)
-                    http1obj.server_conn.wfile.flush()
-
         self.chain[0].set_addons(RequestKiller([1, 2]))
         self.chain[1].set_addons(RequestKiller([1]))
 
@@ -1075,9 +1045,7 @@ def mock_once(self, http1obj, req):
             assert len(self.chain[0].tmaster.state.flows) == 1
             assert len(self.chain[1].tmaster.state.flows) == 1
 
-            with mock.patch("mitmproxy.proxy.protocol.http1.Http1Layer.send_request_headers",
-                            side_effect=MockOnce().mock_once, autospec=True):
-                req = p.request("get:'/p/418:b\"content2\"'")
+            req = p.request("get:'/p/418:b\"content2\"'")
 
             assert req.status_code == 502