Repository: mitmproxy/mitmproxy

Issue:
Error parsing TLSv1.2 ClientHello with no extensions
```
⇩ [0/0]    [showhost]                                                                                                                                                                        ?:help [*:8001]
Traceback (most recent call last):
  File "/Users/yifanlu/Downloads/mitmproxy/venv3.5/lib/python3.5/site-packages/construct/core.py", line 460, in _parse
⇩ [0/0]    [showhost]                                                                                                                                                                        ?:help [*:8001]
Error: 192.168.2.28:54194: Traceback (most recent call last):
  File "/Users/yifanlu/Downloads/mitmproxy/venv3.5/lib/python3.5/site-packages/construct/core.py", line 460, in _parse
    return packer.unpack(self.fmtstr, _read_stream(stream, self.sizeof()))[0]
  File "/Users/yifanlu/Downloads/mitmproxy/venv3.5/lib/python3.5/site-packages/construct/core.py", line 73, in _read_stream
    raise FieldError("could not read enough bytes, expected %d, found %d" % (length, len(data)))
construct.core.FieldError: could not read enough bytes, expected 2, found 0
Traceback (most recent call last):
During handling of the above exception, another exception occurred:
    return self.subcon._parse(stream, context, path)
Traceback (most recent call last):
  File "/Users/yifanlu/Downloads/mitmproxy/venv3.5/lib/python3.5/site-packages/construct/core.py", line 2696, in _parse
    return self.subcon._parse(stream, context, path)
  File "/Users/yifanlu/Downloads/mitmproxy/venv3.5/lib/python3.5/site-packages/construct/core.py", line 859, in _parse
    subobj = sc._parse(stream, context, path)
  File "/Users/yifanlu/Downloads/mitmproxy/venv3.5/lib/python3.5/site-packages/construct/core.py", line 462, in _parse
    raise FieldError("packer %r error during parsing" % self.fmtstr)
construct.core.FieldError: packer '>H' error during parsing
Traceback (most recent call last):
During handling of the above exception, another exception occurred:
    return cls(raw_client_hello)
Traceback (most recent call last):
  File "/Users/yifanlu/Downloads/mitmproxy/mitmproxy/proxy/protocol/tls.py", line 293, in from_client_conn
    return cls(raw_client_hello)
  File "/Users/yifanlu/Downloads/mitmproxy/mitmproxy/proxy/protocol/tls.py", line 251, in __init__
    self._client_hello = _constructs.ClientHello.parse(raw_client_hello)
  File "/Users/yifanlu/Downloads/mitmproxy/venv3.5/lib/python3.5/site-packages/construct/core.py", line 175, in parse
    return self.parse_stream(BytesIO(data), context, **kw)
  File "/Users/yifanlu/Downloads/mitmproxy/venv3.5/lib/python3.5/site-packages/construct/core.py", line 186, in parse_stream
    return self._parse(stream, context, "parsing")
  File "/Users/yifanlu/Downloads/mitmproxy/venv3.5/lib/python3.5/site-packages/construct/core.py", line 2696, in _parse
    return self.subcon._parse(stream, context, path)
  File "/Users/yifanlu/Downloads/mitmproxy/venv3.5/lib/python3.5/site-packages/construct/core.py", line 859, in _parse
    subobj = sc._parse(stream, context, path)
  File "/Users/yifanlu/Downloads/mitmproxy/venv3.5/lib/python3.5/site-packages/construct/core.py", line 2700, in _parse
    raise e.__class__("%s\n    %s" % (e, path))
construct.core.FieldError: packer '>H' error during parsing
    parsing -> ClientHello -> extensions
Traceback (most recent call last):
During handling of the above exception, another exception occurred:
    root_layer()
Traceback (most recent call last):
  File "/Users/yifanlu/Downloads/mitmproxy/mitmproxy/proxy/server.py", line 119, in handle
    root_layer()
  File "/Users/yifanlu/Downloads/mitmproxy/mitmproxy/proxy/modes/transparent_proxy.py", line 19, in __call__
    layer()
  File "/Users/yifanlu/Downloads/mitmproxy/mitmproxy/proxy/protocol/tls.py", line 338, in __call__
    self._client_hello = TlsClientHello.from_client_conn(self.client_conn)
  File "/Users/yifanlu/Downloads/mitmproxy/mitmproxy/proxy/protocol/tls.py", line 297, in from_client_conn
    (repr(e), raw_client_hello.encode("hex"))
AttributeError: 'bytes' object has no attribute 'encode'
```

Here is my ClientHello packet

```
Secure Sockets Layer
    SSL Record Layer: Handshake Protocol: Client Hello
        Content Type: Handshake (22)
        Version: TLS 1.1 (0x0302)
        Length: 55
        Handshake Protocol: Client Hello
            Handshake Type: Client Hello (1)
            Length: 51
            Version: TLS 1.1 (0x0302)
            Random
                GMT Unix Time: Dec  4, 2016 22:44:19.000000000 PST
                Random Bytes: 17991bbc1b718f75650857ad440fa366eae285337fa07c76...
            Session ID Length: 0
            Cipher Suites Length: 12
            Cipher Suites (6 suites)
                Cipher Suite: TLS_RSA_WITH_AES_256_CBC_SHA (0x0035)
                Cipher Suite: TLS_RSA_WITH_AES_128_CBC_SHA (0x002f)
                Cipher Suite: TLS_RSA_WITH_3DES_EDE_CBC_SHA (0x000a)
                Cipher Suite: TLS_RSA_WITH_RC4_128_SHA (0x0005)
                Cipher Suite: TLS_RSA_WITH_RC4_128_MD5 (0x0004)
                Cipher Suite: TLS_EMPTY_RENEGOTIATION_INFO_SCSV (0x00ff)
            Compression Methods Length: 1
            Compression Methods (1 method)
                Compression Method: null (0)

0000   16 03 02 00 37 01 00 00 33 03 02 58 45 0c c3 17  ....7...3..XE...
0010   99 1b bc 1b 71 8f 75 65 08 57 ad 44 0f a3 66 ea  ....q.ue.W.D..f.
0020   e2 85 33 7f a0 7c 76 ce b5 79 99 00 00 0c 00 35  ..3..|v..y.....5
0030   00 2f 00 0a 00 05 00 04 00 ff 01 00              ./..........
```

Seems like in `_constructs.py`, `ClientHello` and `ServerHello` assumes the existence of extensions whereas the RFC allows for no extensions

```
      struct {
          ProtocolVersion client_version;
          Random random;
          SessionID session_id;
          CipherSuite cipher_suites<2..2^16-2>;
          CompressionMethod compression_methods<1..2^8-1>;
          select (extensions_present) {
              case false:
                  struct {};
              case true:
                  Extension extensions<0..2^16-1>;
          };
      } ClientHello;
```


Discussion:

Resolving patch:
diff --git a/mitmproxy/contrib/tls/__init__.py b/mitmproxy/contrib/tls/__init__.py
deleted file mode 100644
--- a/mitmproxy/contrib/tls/__init__.py
+++ /dev/null
@@ -1,4 +0,0 @@
-# This file is dual licensed under the terms of the Apache License, Version
-# 2.0, and the BSD License. See the LICENSE file in the root of this repository
-# for complete details.
-
diff --git a/mitmproxy/contrib/tls/_constructs.py b/mitmproxy/contrib/tls_parser.py
similarity index 94%
rename from mitmproxy/contrib/tls/_constructs.py
rename to mitmproxy/contrib/tls_parser.py
--- a/mitmproxy/contrib/tls/_constructs.py
+++ b/mitmproxy/contrib/tls_parser.py
@@ -1,3 +1,6 @@
+# This file originally comes from https://github.com/pyca/tls/blob/master/tls/_constructs.py.
+# Modified by the mitmproxy team.
+
 # This file is dual licensed under the terms of the Apache License, Version
 # 2.0, and the BSD License. See the LICENSE file in the root of this repository
 # for complete details.
@@ -113,9 +116,11 @@
     )
 )
 
-extensions = "extensions" / Struct(
-    Int16ub,
-    "extensions" / GreedyRange(Extension)
+extensions = "extensions" / Optional(
+    Struct(
+        Int16ub,
+        "extensions" / GreedyRange(Extension)
+    )
 )
 
 ClientHello = "ClientHello" / Struct(
diff --git a/mitmproxy/proxy/protocol/tls.py b/mitmproxy/proxy/protocol/tls.py
--- a/mitmproxy/proxy/protocol/tls.py
+++ b/mitmproxy/proxy/protocol/tls.py
@@ -4,7 +4,7 @@
 
 import construct
 from mitmproxy import exceptions
-from mitmproxy.contrib.tls import _constructs
+from mitmproxy.contrib import tls_parser
 from mitmproxy.proxy.protocol import base
 from mitmproxy.net import check
 
@@ -248,7 +248,7 @@ def get_client_hello(client_conn):
 class TlsClientHello:
 
     def __init__(self, raw_client_hello):
-        self._client_hello = _constructs.ClientHello.parse(raw_client_hello)
+        self._client_hello = tls_parser.ClientHello.parse(raw_client_hello)
 
     def raw(self):
         return self._client_hello
@@ -259,21 +259,25 @@ def cipher_suites(self):
 
     @property
     def sni(self):
-        for extension in self._client_hello.extensions.extensions:
-            is_valid_sni_extension = (
-                extension.type == 0x00 and
-                len(extension.server_names) == 1 and
-                extension.server_names[0].name_type == 0 and
-                check.is_valid_host(extension.server_names[0].host_name)
-            )
-            if is_valid_sni_extension:
-                return extension.server_names[0].host_name.decode("idna")
+        if self._client_hello.extensions:
+            for extension in self._client_hello.extensions.extensions:
+                is_valid_sni_extension = (
+                    extension.type == 0x00 and
+                    len(extension.server_names) == 1 and
+                    extension.server_names[0].name_type == 0 and
+                    check.is_valid_host(extension.server_names[0].host_name)
+                )
+                if is_valid_sni_extension:
+                    return extension.server_names[0].host_name.decode("idna")
+        return None
 
     @property
     def alpn_protocols(self):
-        for extension in self._client_hello.extensions.extensions:
-            if extension.type == 0x10:
-                return list(extension.alpn_protocols)
+        if self._client_hello.extensions:
+            for extension in self._client_hello.extensions.extensions:
+                if extension.type == 0x10:
+                    return list(extension.alpn_protocols)
+        return []
 
     @classmethod
     def from_client_conn(cls, client_conn):


Test patch:
diff --git a/test/mitmproxy/contrib/test_tls_parser.py b/test/mitmproxy/contrib/test_tls_parser.py
new file mode 100644
--- /dev/null
+++ b/test/mitmproxy/contrib/test_tls_parser.py
@@ -0,0 +1,38 @@
+from mitmproxy.contrib import tls_parser
+
+
+def test_parse_chrome():
+    """
+    Test if we properly parse a ClientHello sent by Chrome 54.
+    """
+    data = bytes.fromhex(
+        "03033b70638d2523e1cba15f8364868295305e9c52aceabda4b5147210abc783e6e1000022c02bc02fc02cc030"
+        "cca9cca8cc14cc13c009c013c00ac014009c009d002f0035000a0100006cff0100010000000010000e00000b65"
+        "78616d706c652e636f6d0017000000230000000d00120010060106030501050304010403020102030005000501"
+        "00000000001200000010000e000c02683208687474702f312e3175500000000b00020100000a00080006001d00"
+        "170018"
+    )
+    c = tls_parser.ClientHello.parse(data)
+    assert c.version.major == 3
+    assert c.version.minor == 3
+
+    alpn = [a for a in c.extensions.extensions if a.type == 16]
+    assert len(alpn) == 1
+    assert alpn[0].alpn_protocols == [b"h2", b"http/1.1"]
+
+    sni = [a for a in c.extensions.extensions if a.type == 0]
+    assert len(sni) == 1
+    assert sni[0].server_names[0].name_type == 0
+    assert sni[0].server_names[0].host_name == b"example.com"
+
+
+def test_parse_no_extensions():
+    data = bytes.fromhex(
+        "03015658a756ab2c2bff55f636814deac086b7ca56b65058c7893ffc6074f5245f70205658a75475103a152637"
+        "78e1bb6d22e8bbd5b6b0a3a59760ad354e91ba20d353001a0035002f000a000500040009000300060008006000"
+        "61006200640100"
+    )
+    c = tls_parser.ClientHello.parse(data)
+    assert c.version.major == 3
+    assert c.version.minor == 1
+    assert c.extensions is None
diff --git a/test/mitmproxy/protocol/test_tls.py b/test/mitmproxy/protocol/test_tls.py
new file mode 100644
--- /dev/null
+++ b/test/mitmproxy/protocol/test_tls.py
@@ -0,0 +1,26 @@
+from mitmproxy.proxy.protocol.tls import TlsClientHello
+
+
+class TestClientHello:
+
+    def test_no_extensions(self):
+        data = bytes.fromhex(
+            "03015658a756ab2c2bff55f636814deac086b7ca56b65058c7893ffc6074f5245f70205658a75475103a152637"
+            "78e1bb6d22e8bbd5b6b0a3a59760ad354e91ba20d353001a0035002f000a000500040009000300060008006000"
+            "61006200640100"
+        )
+        c = TlsClientHello(data)
+        assert c.sni is None
+        assert c.alpn_protocols == []
+
+    def test_extensions(self):
+        data = bytes.fromhex(
+            "03033b70638d2523e1cba15f8364868295305e9c52aceabda4b5147210abc783e6e1000022c02bc02fc02cc030"
+            "cca9cca8cc14cc13c009c013c00ac014009c009d002f0035000a0100006cff0100010000000010000e00000b65"
+            "78616d706c652e636f6d0017000000230000000d00120010060106030501050304010403020102030005000501"
+            "00000000001200000010000e000c02683208687474702f312e3175500000000b00020100000a00080006001d00"
+            "170018"
+        )
+        c = TlsClientHello(data)
+        assert c.sni == 'example.com'
+        assert c.alpn_protocols == [b'h2', b'http/1.1']