Repository: mitmproxy/mitmproxy

Issue:
Client TLS handshake failed. The client may not trust the proxy's certificate for static-assets.highwebmedia.com (connection closed)
#### Problem Description

v6 does not log these errors. The thing is that directly connecting to these hosts works too without error. And it seems like everything is working, so what's going on here? Does it actually fail?

```
Client TLS handshake failed. The client may not trust the proxy's certificate for static-assets.highwebmedia.com (connection closed)
Client TLS handshake failed. The client may not trust the proxy's certificate for roomimg.stream.highwebmedia.com (connection closed)
```

#### Steps to reproduce the behavior:
1. `mitmproxy`
2. Open Chromium via proxy
3. Visit https://chaturbate.com (obligatory https://hackerone.com/chaturbate)

You'll instantly see the error.

Here's `-set proxy_debug -vvv` for v6 and v7

[v6.log](https://github.com/mitmproxy/mitmproxy/files/6814791/v6.log)
[v7.log](https://github.com/mitmproxy/mitmproxy/files/6814792/v7.log)

#### System Information

main, does not happen on 6.0.2



Discussion:
I might look into this some more later, I'll assign the issue to myself if I do so.

Could this _somehow_ be related to me using the same `.config/mitmproxy` certs and switching between v6 and v7? The ca was generated with v6 I guess.
Thanks, this is useful. Tracking this down:

1. I first grep the log for "The client may not trust the proxy's certificate" to see that the connection with source port 58954 is affected.
2. Next, I run `test/helper_tools/loggrep.py 58954 v7.log` to only see events relating to that connection.
3. Refine with `test/helper_tools/loggrep.py 58954 v7.log | rg "SendData|ConnectionClosed" -C 1000` to highlight specific keywords.

Using that approach we see that the client unexpectedly disconnects while mitmproxy is negotiating TLS with the server. The error by itself is not wrong, but in this particular setting we can be more precise: We haven't presented a cert yet, so we can be sure that it's not because the CA is untrusted. The general problem is that some TLS clients just hang up after being presented with an untrusted cert, so we can't entirely avoid false-positives here.

Resolving patch:
diff --git a/mitmproxy/proxy/layers/tls.py b/mitmproxy/proxy/layers/tls.py
--- a/mitmproxy/proxy/layers/tls.py
+++ b/mitmproxy/proxy/layers/tls.py
@@ -1,7 +1,7 @@
 import struct
 import time
 from dataclasses import dataclass
-from typing import Iterator, Optional, Tuple
+from typing import Iterator, Literal, Optional, Tuple
 
 from OpenSSL import SSL
 from mitmproxy import certs, connection
@@ -22,10 +22,10 @@ def is_tls_handshake_record(d: bytes) -> bool:
     # TLS 1.3 mandates legacy_record_version to be 0x0301.
     # http://www.moserware.com/2009/06/first-few-milliseconds-of-https.html#client-hello
     return (
-            len(d) >= 3 and
-            d[0] == 0x16 and
-            d[1] == 0x03 and
-            0x0 <= d[2] <= 0x03
+        len(d) >= 3 and
+        d[0] == 0x16 and
+        d[1] == 0x03 and
+        0x0 <= d[2] <= 0x03
     )
 
 
@@ -161,6 +161,8 @@ def __repr__(self):
 
     def start_tls(self) -> layer.CommandGenerator[None]:
         assert not self.tls
+        if not self.conn.connected:
+            return
 
         tls_start = TlsStartData(self.conn, self.context)
         if tls_start.conn == tls_start.context.client:
@@ -379,6 +381,8 @@ def receive_handshake_data(self, data: bytes) -> layer.CommandGenerator[Tuple[bo
                                    f"Trying to establish TLS with client anyway.")
 
         yield from self.start_tls()
+        if not self.conn.connected:
+            return False, "connection closed early"
 
         ret = yield from super().receive_handshake_data(bytes(self.recv_buffer))
         self.recv_buffer.clear()
@@ -399,13 +403,23 @@ def on_handshake_error(self, err: str) -> layer.CommandGenerator[None]:
             dest = self.conn.sni
         else:
             dest = human.format_address(self.context.server.address)
+        level: Literal["warn", "info"] = "warn"
         if err.startswith("Cannot parse ClientHello"):
             pass
         elif "unknown ca" in err or "bad certificate" in err:
             err = f"The client does not trust the proxy's certificate for {dest} ({err})"
+        elif err == "connection closed":
+            err = (
+                f"The client disconnected during the handshake. If this happens consistently for {dest}, "
+                f"this may indicate that the client does not trust the proxy's certificate."
+            )
+            level = "info"
+        elif err == "connection closed early":
+            pass
         else:
             err = f"The client may not trust the proxy's certificate for {dest} ({err})"
-        yield commands.Log(f"Client TLS handshake failed. {err}", level="warn")
+        if err != "connection closed early":
+            yield commands.Log(f"Client TLS handshake failed. {err}", level=level)
         yield from super().on_handshake_error(err)
         self.event_to_child = self.errored  # type: ignore
 


Test patch:
diff --git a/test/mitmproxy/proxy/layers/test_tls.py b/test/mitmproxy/proxy/layers/test_tls.py
--- a/test/mitmproxy/proxy/layers/test_tls.py
+++ b/test/mitmproxy/proxy/layers/test_tls.py
@@ -520,20 +520,45 @@ def test_mitmproxy_ca_is_untrusted(self, tctx: context.Context):
         )
         assert not tctx.client.tls_established
 
-    def test_mitmproxy_ca_is_untrusted_immediate_disconnect(self, tctx: context.Context):
-        """Test the scenario where the client doesn't trust the mitmproxy CA."""
+    @pytest.mark.parametrize("close_at", ["tls_clienthello", "tls_start_client", "handshake"])
+    def test_immediate_disconnect(self, tctx: context.Context, close_at):
+        """Test the scenario where the client is disconnecting during the handshake.
+        This may happen because they are not interested in the connection anymore, or because they do not like
+        the proxy certificate."""
         playbook, client_layer, tssl_client = make_client_tls_layer(tctx, sni=b"wrong.host.mitmproxy.org")
+        playbook.logs = True
 
-        assert (
+        playbook >> events.DataReceived(tctx.client, tssl_client.bio_read())
+        playbook << tls.TlsClienthelloHook(tutils.Placeholder())
+
+        if close_at == "tls_clienthello":
+            assert (
                 playbook
-                >> events.DataReceived(tctx.client, tssl_client.bio_read())
-                << tls.TlsClienthelloHook(tutils.Placeholder())
-                >> tutils.reply()
-                << tls.TlsStartClientHook(tutils.Placeholder())
-                >> reply_tls_start_client()
-                << commands.SendData(tctx.client, tutils.Placeholder())
                 >> events.ConnectionClosed(tctx.client)
-                << commands.Log("Client TLS handshake failed. The client may not trust the proxy's certificate "
-                                "for wrong.host.mitmproxy.org (connection closed)", "warn")
+                >> tutils.reply(to=-2)
                 << commands.CloseConnection(tctx.client)
+            )
+            return
+
+        playbook >> tutils.reply()
+        playbook << tls.TlsStartClientHook(tutils.Placeholder())
+
+        if close_at == "tls_start_client":
+            assert (
+                playbook
+                >> events.ConnectionClosed(tctx.client)
+                >> reply_tls_start_client(to=-2)
+                << commands.CloseConnection(tctx.client)
+            )
+            return
+
+        assert (
+            playbook
+            >> reply_tls_start_client()
+            << commands.SendData(tctx.client, tutils.Placeholder())
+            >> events.ConnectionClosed(tctx.client)
+            << commands.Log("Client TLS handshake failed. The client disconnected during the handshake. "
+                            "If this happens consistently for wrong.host.mitmproxy.org, this may indicate that the "
+                            "client does not trust the proxy's certificate.", "info")
+            << commands.CloseConnection(tctx.client)
         )