Repository: abhinavsingh/proxy.py

Issue:
Broken tests on Python 3.7+ (urllib.parse behavior has changed)
**Describe the bug**
Broken TLS interception tests, which were working fine up until Jan 7th.
<img width="983" alt="Screen Shot 2020-01-20 at 8 10 19 AM" src="https://user-images.githubusercontent.com/126065/72741064-bef4d300-3b5b-11ea-8680-983ad8ea8c91.png">


**To Reproduce**
Steps to reproduce the behavior:
1. Run tests as make
2. See errors

```
================================================================== FAILURES ===================================================================
____________________________________________________ TestHttpProxyTlsInterception.test_e2e ____________________________________________________

self = <tests.http.test_http_proxy_tls_interception.TestHttpProxyTlsInterception testMethod=test_e2e>
mock_fromfd = <MagicMock name='fromfd' id='4545990224'>, mock_selector = <MagicMock name='DefaultSelector' id='4712344720'>
mock_popen = <MagicMock name='Popen' id='4712357776'>, mock_server_conn = <MagicMock name='TcpServerConnection' id='4546528400'>
mock_ssl_context = <MagicMock name='create_default_context' id='4712396432'>, mock_ssl_wrap = <MagicMock name='wrap_socket' id='4712405584'>

    @mock.patch('ssl.wrap_socket')
    @mock.patch('ssl.create_default_context')
    @mock.patch('proxy.http.proxy.server.TcpServerConnection')
    @mock.patch('subprocess.Popen')
    @mock.patch('selectors.DefaultSelector')
    @mock.patch('socket.fromfd')
    def test_e2e(
            self,
            mock_fromfd: mock.Mock,
            mock_selector: mock.Mock,
            mock_popen: mock.Mock,
            mock_server_conn: mock.Mock,
            mock_ssl_context: mock.Mock,
            mock_ssl_wrap: mock.Mock) -> None:
        host, port = uuid.uuid4().hex, 443
        netloc = '{0}:{1}'.format(host, port)
    
        self.mock_fromfd = mock_fromfd
        self.mock_selector = mock_selector
        self.mock_popen = mock_popen
        self.mock_server_conn = mock_server_conn
        self.mock_ssl_context = mock_ssl_context
        self.mock_ssl_wrap = mock_ssl_wrap
    
        ssl_connection = mock.MagicMock(spec=ssl.SSLSocket)
        self.mock_ssl_context.return_value.wrap_socket.return_value = ssl_connection
        self.mock_ssl_wrap.return_value = mock.MagicMock(spec=ssl.SSLSocket)
        plain_connection = mock.MagicMock(spec=socket.socket)
    
        def mock_connection() -> Any:
            if self.mock_ssl_context.return_value.wrap_socket.called:
                return ssl_connection
            return plain_connection
    
        type(self.mock_server_conn.return_value).connection = \
            mock.PropertyMock(side_effect=mock_connection)
    
        self.fileno = 10
        self._addr = ('127.0.0.1', 54382)
        self.flags = Flags(
            ca_cert_file='ca-cert.pem',
            ca_key_file='ca-key.pem',
            ca_signing_key_file='ca-signing-key.pem',
        )
        self.plugin = mock.MagicMock()
        self.proxy_plugin = mock.MagicMock()
        self.flags.plugins = {
            b'HttpProtocolHandlerPlugin': [self.plugin, HttpProxyPlugin],
            b'HttpProxyBasePlugin': [self.proxy_plugin],
        }
        self._conn = mock_fromfd.return_value
        self.protocol_handler = HttpProtocolHandler(
            TcpClientConnection(self._conn, self._addr),
            flags=self.flags)
        self.protocol_handler.initialize()
    
        self.plugin.assert_called()
        self.assertEqual(self.plugin.call_args[0][1], self.flags)
        self.assertEqual(self.plugin.call_args[0][2].connection, self._conn)
        self.proxy_plugin.assert_called()
        self.assertEqual(self.proxy_plugin.call_args[0][1], self.flags)
        self.assertEqual(
            self.proxy_plugin.call_args[0][2].connection,
            self._conn)
    
        connect_request = build_http_request(
            httpMethods.CONNECT, bytes_(netloc),
            headers={
                b'Host': bytes_(netloc),
            })
        self._conn.recv.return_value = connect_request
    
        # Prepare mocked HttpProtocolHandlerPlugin
        self.plugin.return_value.get_descriptors.return_value = ([], [])
        self.plugin.return_value.write_to_descriptors.return_value = False
        self.plugin.return_value.read_from_descriptors.return_value = False
        self.plugin.return_value.on_client_data.side_effect = lambda raw: raw
        self.plugin.return_value.on_request_complete.return_value = False
        self.plugin.return_value.on_response_chunk.side_effect = lambda chunk: chunk
        self.plugin.return_value.on_client_connection_close.return_value = None
    
        # Prepare mocked HttpProxyBasePlugin
        self.proxy_plugin.return_value.before_upstream_connection.side_effect = lambda r: r
        self.proxy_plugin.return_value.handle_client_request.side_effect = lambda r: r
    
        self.mock_selector.return_value.select.side_effect = [
            [(selectors.SelectorKey(
                fileobj=self._conn,
                fd=self._conn.fileno,
                events=selectors.EVENT_READ,
                data=None), selectors.EVENT_READ)], ]
        self.protocol_handler.run_once()
    
        # Assert our mocked plugins invocations
        self.plugin.return_value.get_descriptors.assert_called()
        self.plugin.return_value.write_to_descriptors.assert_called_with([])
        self.plugin.return_value.on_client_data.assert_called_with(
            connect_request)
        self.plugin.return_value.on_request_complete.assert_called()
        self.plugin.return_value.read_from_descriptors.assert_called_with([
>                                                                         self._conn])

tests/http/test_http_proxy_tls_interception.py:130: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

_mock_self = <MagicMock name='mock().read_from_descriptors' id='4712797392'>, args = ([<MagicMock name='fromfd()' id='4712524560'>],)
kwargs = {}, expected = "read_from_descriptors([<MagicMock name='fromfd()' id='4712524560'>])"

    def assert_called_with(_mock_self, *args, **kwargs):
        """assert that the mock was called with the specified arguments.
    
        Raises an AssertionError if the args and keyword args passed in are
        different to the last call to the mock."""
        self = _mock_self
        if self.call_args is None:
            expected = self._format_mock_call_signature(args, kwargs)
>           raise AssertionError('Expected call: %s\nNot called' % (expected,))
E           AssertionError: Expected call: read_from_descriptors([<MagicMock name='fromfd()' id='4712524560'>])
E           Not called

/usr/local/Cellar/python/3.7.6_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/unittest/mock.py:864: AssertionError
------------------------------------------------------------ Captured stderr call -------------------------------------------------------------
2020-01-20 08:05:57,738 - pid:22876 [D] initialize:145 - Handling connection <MagicMock name='fromfd()' id='4712524560'>
2020-01-20 08:05:57,740 - pid:22876 [D] handle_readables:302 - Client is ready for reads, reading
2020-01-20 08:05:57,740 - pid:22876 [D] recv:65 - received 101 bytes from client
2020-01-20 08:05:57,741 - pid:22876 [E] connect_upstream:430 - Both host and port must exist
NoneType: None
2020-01-20 08:05:57,741 - pid:22876 [D] handle_readables:361 - HttpProtocolException type raised
-------------------------------------------------------------- Captured log call --------------------------------------------------------------
DEBUG    proxy.http.handler:handler.py:145 Handling connection <MagicMock name='fromfd()' id='4712524560'>
DEBUG    proxy.http.handler:handler.py:302 Client is ready for reads, reading
DEBUG    proxy.core.connection.connection:connection.py:65 received 101 bytes from client
ERROR    proxy.http.proxy.server:server.py:430 Both host and port must exist
NoneType: None
DEBUG    proxy.http.handler:handler.py:361 HttpProtocolException type raised
________________________________________ TestHttpProtocolHandler.test_authenticated_proxy_http_tunnel _________________________________________

self = <tests.http.test_protocol_handler.TestHttpProtocolHandler testMethod=test_authenticated_proxy_http_tunnel>
mock_server_connection = <MagicMock name='TcpServerConnection' id='4713238608'>, mock_fromfd = <MagicMock name='fromfd' id='4713790672'>
mock_selector = <MagicMock name='DefaultSelector' id='4713807824'>

    @mock.patch('selectors.DefaultSelector')
    @mock.patch('socket.fromfd')
    @mock.patch('proxy.http.proxy.server.TcpServerConnection')
    def test_authenticated_proxy_http_tunnel(
            self, mock_server_connection: mock.Mock,
            mock_fromfd: mock.Mock,
            mock_selector: mock.Mock) -> None:
        server = mock_server_connection.return_value
        server.connect.return_value = True
        server.buffer_size.return_value = 0
        self._conn = mock_fromfd.return_value
        self.mock_selector_for_client_read_read_server_write(
            mock_selector, server)
    
        flags = Flags(
            auth_code=b'Basic %s' %
                      base64.b64encode(b'user:pass'))
        flags.plugins = Flags.load_plugins(
            b'proxy.http.proxy.HttpProxyPlugin,proxy.http.server.HttpWebServerPlugin')
    
        self.protocol_handler = HttpProtocolHandler(
            TcpClientConnection(self._conn, self._addr), flags=flags)
        self.protocol_handler.initialize()
    
        assert self.http_server_port is not None
        self._conn.recv.return_value = CRLF.join([
            b'CONNECT localhost:%d HTTP/1.1' % self.http_server_port,
            b'Host: localhost:%d' % self.http_server_port,
            b'User-Agent: proxy.py/%s' % bytes_(__version__),
            b'Proxy-Connection: Keep-Alive',
            b'Proxy-Authorization: Basic dXNlcjpwYXNz',
            CRLF
        ])
>       self.assert_tunnel_response(mock_server_connection, server)

tests/http/test_protocol_handler.py:272: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
tests/http/test_protocol_handler.py:88: in assert_tunnel_response
    cast(HttpProxyPlugin, self.protocol_handler.plugins['HttpProxyPlugin']).server is not None)
E   AssertionError: False is not true
------------------------------------------------------------ Captured stderr call -------------------------------------------------------------
2020-01-20 08:05:57,866 - pid:22876 [I] load_plugins:525 - Loaded plugin proxy.http.proxy.HttpProxyPlugin
2020-01-20 08:05:57,866 - pid:22876 [I] load_plugins:525 - Loaded plugin proxy.http.server.HttpWebServerPlugin
2020-01-20 08:05:57,867 - pid:22876 [D] initialize:145 - Handling connection <MagicMock name='fromfd()' id='4713703888'>
2020-01-20 08:05:57,869 - pid:22876 [I] load_plugins:525 - Loaded plugin proxy.http.proxy.HttpProxyPlugin
2020-01-20 08:05:57,869 - pid:22876 [I] load_plugins:525 - Loaded plugin proxy.http.server.HttpWebServerPlugin
2020-01-20 08:05:57,869 - pid:22876 [D] initialize:145 - Handling connection <MagicMock name='fromfd()' id='4713876688'>
2020-01-20 08:05:57,870 - pid:22876 [D] handle_readables:302 - Client is ready for reads, reading
2020-01-20 08:05:57,870 - pid:22876 [D] recv:65 - received 158 bytes from client
2020-01-20 08:05:57,870 - pid:22876 [E] connect_upstream:430 - Both host and port must exist
NoneType: None
2020-01-20 08:05:57,870 - pid:22876 [D] handle_readables:361 - HttpProtocolException type raised
-------------------------------------------------------------- Captured log call --------------------------------------------------------------
INFO     proxy.common.flags:flags.py:525 Loaded plugin proxy.http.proxy.HttpProxyPlugin
INFO     proxy.common.flags:flags.py:525 Loaded plugin proxy.http.server.HttpWebServerPlugin
DEBUG    proxy.http.handler:handler.py:145 Handling connection <MagicMock name='fromfd()' id='4713703888'>
INFO     proxy.common.flags:flags.py:525 Loaded plugin proxy.http.proxy.HttpProxyPlugin
INFO     proxy.common.flags:flags.py:525 Loaded plugin proxy.http.server.HttpWebServerPlugin
DEBUG    proxy.http.handler:handler.py:145 Handling connection <MagicMock name='fromfd()' id='4713876688'>
DEBUG    proxy.http.handler:handler.py:302 Client is ready for reads, reading
DEBUG    proxy.core.connection.connection:connection.py:65 received 158 bytes from client
ERROR    proxy.http.proxy.server:server.py:430 Both host and port must exist
NoneType: None
DEBUG    proxy.http.handler:handler.py:361 HttpProtocolException type raised
__________________________________________________ TestHttpProtocolHandler.test_http_tunnel ___________________________________________________

self = <tests.http.test_protocol_handler.TestHttpProtocolHandler testMethod=test_http_tunnel>
mock_server_connection = <MagicMock name='TcpServerConnection' id='4713947600'>

    @mock.patch('proxy.http.proxy.server.TcpServerConnection')
    def test_http_tunnel(self, mock_server_connection: mock.Mock) -> None:
        server = mock_server_connection.return_value
        server.connect.return_value = True
    
        def has_buffer() -> bool:
            return cast(bool, server.queue.called)
    
        server.has_buffer.side_effect = has_buffer
        self.mock_selector.return_value.select.side_effect = [
            [(selectors.SelectorKey(
                fileobj=self._conn,
                fd=self._conn.fileno,
                events=selectors.EVENT_READ,
                data=None), selectors.EVENT_READ), ],
            [(selectors.SelectorKey(
                fileobj=self._conn,
                fd=self._conn.fileno,
                events=0,
                data=None), selectors.EVENT_WRITE), ],
            [(selectors.SelectorKey(
                fileobj=self._conn,
                fd=self._conn.fileno,
                events=selectors.EVENT_READ,
                data=None), selectors.EVENT_READ), ],
            [(selectors.SelectorKey(
                fileobj=server.connection,
                fd=server.connection.fileno,
                events=0,
                data=None), selectors.EVENT_WRITE), ],
        ]
    
        assert self.http_server_port is not None
        self._conn.recv.return_value = CRLF.join([
            b'CONNECT localhost:%d HTTP/1.1' % self.http_server_port,
            b'Host: localhost:%d' % self.http_server_port,
            b'User-Agent: proxy.py/%s' % bytes_(__version__),
            b'Proxy-Connection: Keep-Alive',
            CRLF
        ])
>       self.assert_tunnel_response(mock_server_connection, server)

tests/http/test_protocol_handler.py:143: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
tests/http/test_protocol_handler.py:88: in assert_tunnel_response
    cast(HttpProxyPlugin, self.protocol_handler.plugins['HttpProxyPlugin']).server is not None)
E   AssertionError: False is not true
------------------------------------------------------------ Captured stderr call -------------------------------------------------------------
2020-01-20 08:05:57,911 - pid:22876 [I] load_plugins:525 - Loaded plugin proxy.http.proxy.HttpProxyPlugin
2020-01-20 08:05:57,911 - pid:22876 [I] load_plugins:525 - Loaded plugin proxy.http.server.HttpWebServerPlugin
2020-01-20 08:05:57,911 - pid:22876 [D] initialize:145 - Handling connection <MagicMock name='fromfd()' id='4543913232'>
2020-01-20 08:05:57,913 - pid:22876 [D] handle_readables:302 - Client is ready for reads, reading
2020-01-20 08:05:57,913 - pid:22876 [D] recv:65 - received 117 bytes from client
2020-01-20 08:05:57,913 - pid:22876 [E] connect_upstream:430 - Both host and port must exist
NoneType: None
2020-01-20 08:05:57,913 - pid:22876 [D] handle_readables:361 - HttpProtocolException type raised
-------------------------------------------------------------- Captured log call --------------------------------------------------------------
INFO     proxy.common.flags:flags.py:525 Loaded plugin proxy.http.proxy.HttpProxyPlugin
INFO     proxy.common.flags:flags.py:525 Loaded plugin proxy.http.server.HttpWebServerPlugin
DEBUG    proxy.http.handler:handler.py:145 Handling connection <MagicMock name='fromfd()' id='4543913232'>
DEBUG    proxy.http.handler:handler.py:302 Client is ready for reads, reading
DEBUG    proxy.core.connection.connection:connection.py:65 received 117 bytes from client
ERROR    proxy.http.proxy.server:server.py:430 Both host and port must exist
NoneType: None
DEBUG    proxy.http.handler:handler.py:361 HttpProtocolException type raised
________________________________ TestHttpProxyPluginExamplesWithTlsInterception.test_man_in_the_middle_plugin _________________________________

self = <tests.plugin.test_http_proxy_plugins_with_tls_interception.TestHttpProxyPluginExamplesWithTlsInterception testMethod=test_man_in_the_middle_plugin>
mock_fromfd = <MagicMock name='fromfd' id='4542520592'>, mock_selector = <MagicMock name='DefaultSelector' id='4712442832'>
mock_popen = <MagicMock name='Popen' id='4544256144'>, mock_server_conn = <MagicMock name='TcpServerConnection' id='4543959824'>
mock_ssl_context = <MagicMock name='create_default_context' id='4544330640'>, mock_ssl_wrap = <MagicMock name='wrap_socket' id='4712634000'>

    @mock.patch('ssl.wrap_socket')
    @mock.patch('ssl.create_default_context')
    @mock.patch('proxy.http.proxy.server.TcpServerConnection')
    @mock.patch('subprocess.Popen')
    @mock.patch('selectors.DefaultSelector')
    @mock.patch('socket.fromfd')
    def setUp(self,
              mock_fromfd: mock.Mock,
              mock_selector: mock.Mock,
              mock_popen: mock.Mock,
              mock_server_conn: mock.Mock,
              mock_ssl_context: mock.Mock,
              mock_ssl_wrap: mock.Mock) -> None:
        self.mock_fromfd = mock_fromfd
        self.mock_selector = mock_selector
        self.mock_popen = mock_popen
        self.mock_server_conn = mock_server_conn
        self.mock_ssl_context = mock_ssl_context
        self.mock_ssl_wrap = mock_ssl_wrap
    
        self.fileno = 10
        self._addr = ('127.0.0.1', 54382)
        self.flags = Flags(
            ca_cert_file='ca-cert.pem',
            ca_key_file='ca-key.pem',
            ca_signing_key_file='ca-signing-key.pem',)
        self.plugin = mock.MagicMock()
    
        plugin = get_plugin_by_test_name(self._testMethodName)
    
        self.flags.plugins = {
            b'HttpProtocolHandlerPlugin': [HttpProxyPlugin],
            b'HttpProxyBasePlugin': [plugin],
        }
        self._conn = mock.MagicMock(spec=socket.socket)
        mock_fromfd.return_value = self._conn
        self.protocol_handler = HttpProtocolHandler(
            TcpClientConnection(self._conn, self._addr), flags=self.flags)
        self.protocol_handler.initialize()
    
        self.server = self.mock_server_conn.return_value
    
        self.server_ssl_connection = mock.MagicMock(spec=ssl.SSLSocket)
        self.mock_ssl_context.return_value.wrap_socket.return_value = self.server_ssl_connection
        self.client_ssl_connection = mock.MagicMock(spec=ssl.SSLSocket)
        self.mock_ssl_wrap.return_value = self.client_ssl_connection
    
        def has_buffer() -> bool:
            return cast(bool, self.server.queue.called)
    
        def closed() -> bool:
            return not self.server.connect.called
    
        def mock_connection() -> Any:
            if self.mock_ssl_context.return_value.wrap_socket.called:
                return self.server_ssl_connection
            return self._conn
    
        self.server.has_buffer.side_effect = has_buffer
        type(self.server).closed = mock.PropertyMock(side_effect=closed)
        type(
            self.server).connection = mock.PropertyMock(
            side_effect=mock_connection)
    
        self.mock_selector.return_value.select.side_effect = [
            [(selectors.SelectorKey(
                fileobj=self._conn,
                fd=self._conn.fileno,
                events=selectors.EVENT_READ,
                data=None), selectors.EVENT_READ)],
            [(selectors.SelectorKey(
                fileobj=self.client_ssl_connection,
                fd=self.client_ssl_connection.fileno,
                events=selectors.EVENT_READ,
                data=None), selectors.EVENT_READ)],
            [(selectors.SelectorKey(
                fileobj=self.server_ssl_connection,
                fd=self.server_ssl_connection.fileno,
                events=selectors.EVENT_WRITE,
                data=None), selectors.EVENT_WRITE)],
            [(selectors.SelectorKey(
                fileobj=self.server_ssl_connection,
                fd=self.server_ssl_connection.fileno,
                events=selectors.EVENT_READ,
                data=None), selectors.EVENT_READ)], ]
    
        # Connect
        def send(raw: bytes) -> int:
            return len(raw)
    
        self._conn.send.side_effect = send
        self._conn.recv.return_value = build_http_request(
            httpMethods.CONNECT, b'uni.corn:443'
        )
        self.protocol_handler.run_once()
    
>       self.mock_popen.assert_called()

tests/plugin/test_http_proxy_plugins_with_tls_interception.py:129: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

_mock_self = <MagicMock name='Popen' id='4544256144'>

    def assert_called(_mock_self):
        """assert that the mock was called at least once
        """
        self = _mock_self
        if self.call_count == 0:
            msg = ("Expected '%s' to have been called." %
                   self._mock_name or 'mock')
>           raise AssertionError(msg)
E           AssertionError: Expected 'Popen' to have been called.

/usr/local/Cellar/python/3.7.6_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/unittest/mock.py:845: AssertionError
------------------------------------------------------------ Captured stderr call -------------------------------------------------------------
2020-01-20 08:05:58,057 - pid:22876 [D] initialize:145 - Handling connection <MagicMock name='fromfd()' spec='socket' id='4712239760'>
2020-01-20 08:05:58,059 - pid:22876 [D] handle_readables:302 - Client is ready for reads, reading
2020-01-20 08:05:58,059 - pid:22876 [D] recv:65 - received 33 bytes from client
2020-01-20 08:05:58,060 - pid:22876 [E] connect_upstream:430 - Both host and port must exist
NoneType: None
2020-01-20 08:05:58,060 - pid:22876 [D] handle_readables:361 - HttpProtocolException type raised
-------------------------------------------------------------- Captured log call --------------------------------------------------------------
DEBUG    proxy.http.handler:handler.py:145 Handling connection <MagicMock name='fromfd()' spec='socket' id='4712239760'>
DEBUG    proxy.http.handler:handler.py:302 Client is ready for reads, reading
DEBUG    proxy.core.connection.connection:connection.py:65 received 33 bytes from client
ERROR    proxy.http.proxy.server:server.py:430 Both host and port must exist
NoneType: None
DEBUG    proxy.http.handler:handler.py:361 HttpProtocolException type raised
_________________________________ TestHttpProxyPluginExamplesWithTlsInterception.test_modify_post_data_plugin _________________________________

self = <tests.plugin.test_http_proxy_plugins_with_tls_interception.TestHttpProxyPluginExamplesWithTlsInterception testMethod=test_modify_post_data_plugin>
mock_fromfd = <MagicMock name='fromfd' id='4544021648'>, mock_selector = <MagicMock name='DefaultSelector' id='4712673680'>
mock_popen = <MagicMock name='Popen' id='4712577040'>, mock_server_conn = <MagicMock name='TcpServerConnection' id='4543962832'>
mock_ssl_context = <MagicMock name='create_default_context' id='4712282000'>, mock_ssl_wrap = <MagicMock name='wrap_socket' id='4712567056'>

    @mock.patch('ssl.wrap_socket')
    @mock.patch('ssl.create_default_context')
    @mock.patch('proxy.http.proxy.server.TcpServerConnection')
    @mock.patch('subprocess.Popen')
    @mock.patch('selectors.DefaultSelector')
    @mock.patch('socket.fromfd')
    def setUp(self,
              mock_fromfd: mock.Mock,
              mock_selector: mock.Mock,
              mock_popen: mock.Mock,
              mock_server_conn: mock.Mock,
              mock_ssl_context: mock.Mock,
              mock_ssl_wrap: mock.Mock) -> None:
        self.mock_fromfd = mock_fromfd
        self.mock_selector = mock_selector
        self.mock_popen = mock_popen
        self.mock_server_conn = mock_server_conn
        self.mock_ssl_context = mock_ssl_context
        self.mock_ssl_wrap = mock_ssl_wrap
    
        self.fileno = 10
        self._addr = ('127.0.0.1', 54382)
        self.flags = Flags(
            ca_cert_file='ca-cert.pem',
            ca_key_file='ca-key.pem',
            ca_signing_key_file='ca-signing-key.pem',)
        self.plugin = mock.MagicMock()
    
        plugin = get_plugin_by_test_name(self._testMethodName)
    
        self.flags.plugins = {
            b'HttpProtocolHandlerPlugin': [HttpProxyPlugin],
            b'HttpProxyBasePlugin': [plugin],
        }
        self._conn = mock.MagicMock(spec=socket.socket)
        mock_fromfd.return_value = self._conn
        self.protocol_handler = HttpProtocolHandler(
            TcpClientConnection(self._conn, self._addr), flags=self.flags)
        self.protocol_handler.initialize()
    
        self.server = self.mock_server_conn.return_value
    
        self.server_ssl_connection = mock.MagicMock(spec=ssl.SSLSocket)
        self.mock_ssl_context.return_value.wrap_socket.return_value = self.server_ssl_connection
        self.client_ssl_connection = mock.MagicMock(spec=ssl.SSLSocket)
        self.mock_ssl_wrap.return_value = self.client_ssl_connection
    
        def has_buffer() -> bool:
            return cast(bool, self.server.queue.called)
    
        def closed() -> bool:
            return not self.server.connect.called
    
        def mock_connection() -> Any:
            if self.mock_ssl_context.return_value.wrap_socket.called:
                return self.server_ssl_connection
            return self._conn
    
        self.server.has_buffer.side_effect = has_buffer
        type(self.server).closed = mock.PropertyMock(side_effect=closed)
        type(
            self.server).connection = mock.PropertyMock(
            side_effect=mock_connection)
    
        self.mock_selector.return_value.select.side_effect = [
            [(selectors.SelectorKey(
                fileobj=self._conn,
                fd=self._conn.fileno,
                events=selectors.EVENT_READ,
                data=None), selectors.EVENT_READ)],
            [(selectors.SelectorKey(
                fileobj=self.client_ssl_connection,
                fd=self.client_ssl_connection.fileno,
                events=selectors.EVENT_READ,
                data=None), selectors.EVENT_READ)],
            [(selectors.SelectorKey(
                fileobj=self.server_ssl_connection,
                fd=self.server_ssl_connection.fileno,
                events=selectors.EVENT_WRITE,
                data=None), selectors.EVENT_WRITE)],
            [(selectors.SelectorKey(
                fileobj=self.server_ssl_connection,
                fd=self.server_ssl_connection.fileno,
                events=selectors.EVENT_READ,
                data=None), selectors.EVENT_READ)], ]
    
        # Connect
        def send(raw: bytes) -> int:
            return len(raw)
    
        self._conn.send.side_effect = send
        self._conn.recv.return_value = build_http_request(
            httpMethods.CONNECT, b'uni.corn:443'
        )
        self.protocol_handler.run_once()
    
>       self.mock_popen.assert_called()

tests/plugin/test_http_proxy_plugins_with_tls_interception.py:129: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

_mock_self = <MagicMock name='Popen' id='4712577040'>

    def assert_called(_mock_self):
        """assert that the mock was called at least once
        """
        self = _mock_self
        if self.call_count == 0:
            msg = ("Expected '%s' to have been called." %
                   self._mock_name or 'mock')
>           raise AssertionError(msg)
E           AssertionError: Expected 'Popen' to have been called.

/usr/local/Cellar/python/3.7.6_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/unittest/mock.py:845: AssertionError
------------------------------------------------------------ Captured stderr call -------------------------------------------------------------
2020-01-20 08:05:58,100 - pid:22876 [D] initialize:145 - Handling connection <MagicMock name='fromfd()' spec='socket' id='4712355344'>
2020-01-20 08:05:58,102 - pid:22876 [D] handle_readables:302 - Client is ready for reads, reading
2020-01-20 08:05:58,102 - pid:22876 [D] recv:65 - received 33 bytes from client
2020-01-20 08:05:58,103 - pid:22876 [E] connect_upstream:430 - Both host and port must exist
NoneType: None
2020-01-20 08:05:58,103 - pid:22876 [D] handle_readables:361 - HttpProtocolException type raised
-------------------------------------------------------------- Captured log call --------------------------------------------------------------
DEBUG    proxy.http.handler:handler.py:145 Handling connection <MagicMock name='fromfd()' spec='socket' id='4712355344'>
DEBUG    proxy.http.handler:handler.py:302 Client is ready for reads, reading
DEBUG    proxy.core.connection.connection:connection.py:65 received 33 bytes from client
ERROR    proxy.http.proxy.server:server.py:430 Both host and port must exist
NoneType: None
DEBUG    proxy.http.handler:handler.py:361 HttpProtocolException type raised
======================================================== 5 failed, 114 passed in 5.60s ========================================================
make: *** [lib-test] Error 1
```

**Version information**
 - OS: Mac
 - Device: Macbook
 - proxy.py Development


Discussion:
abhinavsingh: @Benouare jFYI
codecov[bot]: # [Codecov](https://codecov.io/gh/abhinavsingh/proxy.py/pull/277?src=pr&el=h1) Report
> Merging [#277](https://codecov.io/gh/abhinavsingh/proxy.py/pull/277?src=pr&el=desc) into [develop](https://codecov.io/gh/abhinavsingh/proxy.py/commit/f75e21c4b159f2ef7ecb421768fbf3a15bf2bee3?src=pr&el=desc) will **not change** coverage.
> The diff coverage is `100%`.

[![Impacted file tree graph](https://codecov.io/gh/abhinavsingh/proxy.py/pull/277/graphs/tree.svg?width=650&token=Zh9J7b4la2&height=150&src=pr)](https://codecov.io/gh/abhinavsingh/proxy.py/pull/277?src=pr&el=tree)

```diff
@@           Coverage Diff            @@
##           develop     #277   +/-   ##
========================================
  Coverage    73.06%   73.06%           
========================================
  Files           74       74           
  Lines         2840     2840           
========================================
  Hits          2075     2075           
  Misses         765      765
```


| [Impacted Files](https://codecov.io/gh/abhinavsingh/proxy.py/pull/277?src=pr&el=tree) | Coverage Δ | |
|---|---|---|
| [proxy/testing/\_\_init\_\_.py](https://codecov.io/gh/abhinavsingh/proxy.py/pull/277/diff?src=pr&el=tree#diff-cHJveHkvdGVzdGluZy9fX2luaXRfXy5weQ==) | `100% <ø> (ø)` | :arrow_up: |
| [proxy/testing/test\_case.py](https://codecov.io/gh/abhinavsingh/proxy.py/pull/277/diff?src=pr&el=tree#diff-cHJveHkvdGVzdGluZy90ZXN0X2Nhc2UucHk=) | `60% <ø> (ø)` | :arrow_up: |
| [proxy/http/parser.py](https://codecov.io/gh/abhinavsingh/proxy.py/pull/277/diff?src=pr&el=tree#diff-cHJveHkvaHR0cC9wYXJzZXIucHk=) | `95.42% <100%> (ø)` | :arrow_up: |

------

[Continue to review full report at Codecov](https://codecov.io/gh/abhinavsingh/proxy.py/pull/277?src=pr&el=continue).
> **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta)
> `Δ = absolute <relative> (impact)`, `ø = not affected`, `? = missing data`
> Powered by [Codecov](https://codecov.io/gh/abhinavsingh/proxy.py/pull/277?src=pr&el=footer). Last update [f75e21c...e258902](https://codecov.io/gh/abhinavsingh/proxy.py/pull/277?src=pr&el=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments).


Patch:
diff --git a/proxy/common/version.py b/proxy/common/version.py
index 1a23f79..170fcbc 100644
--- a/proxy/common/version.py
+++ b/proxy/common/version.py
@@ -8,5 +8,5 @@
     :copyright: (c) 2013-present by Abhinav Singh and contributors.
     :license: BSD, see LICENSE for more details.
 """
-VERSION = (2, 1, 0)
+VERSION = (2, 1, 1)
 __version__ = '.'.join(map(str, VERSION[0:3]))
diff --git a/proxy/http/parser.py b/proxy/http/parser.py
index ece1ce0..1f07bfe 100644
--- a/proxy/http/parser.py
+++ b/proxy/http/parser.py
@@ -111,8 +111,9 @@ class HttpParser:
     def set_line_attributes(self) -> None:
         if self.type == httpParserTypes.REQUEST_PARSER:
             if self.method == httpMethods.CONNECT and self.url:
-                u = urlparse.urlsplit(b'//' + self.url.path)
-                self.host, self.port = u.hostname, u.port
+                self.host = self.url.scheme
+                self.port = 443 if self.url.path == b'' else \
+                    int(self.url.path)
             elif self.url:
                 self.host, self.port = self.url.hostname, self.url.port \
                     if self.url.port else DEFAULT_HTTP_PORT
diff --git a/setup.py b/setup.py
index 8ff5ec1..5d376b4 100644
--- a/setup.py
+++ b/setup.py
@@ -10,7 +10,7 @@
 """
 from setuptools import setup, find_packages
 
-VERSION = (2, 1, 0)
+VERSION = (2, 1, 1)
 __version__ = '.'.join(map(str, VERSION[0:3]))
 __description__ = '''⚡⚡⚡Fast, Lightweight, Pluggable, TLS interception capable proxy server
     focused on Network monitoring, controls & Application development, testing, debugging.'''


Test patch:
diff --git a/proxy/testing/__init__.py b/proxy/testing/__init__.py
index ba03413..232621f 100644
--- a/proxy/testing/__init__.py
+++ b/proxy/testing/__init__.py
@@ -2,7 +2,8 @@
 """
     proxy.py
     ~~~~~~~~
-    ⚡⚡⚡ Fast, Lightweight, Programmable Proxy Server in a single Python file.
+    ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on
+    Network monitoring, controls & Application development, testing, debugging.
 
     :copyright: (c) 2013-present by Abhinav Singh and contributors.
     :license: BSD, see LICENSE for more details.
diff --git a/proxy/testing/test_case.py b/proxy/testing/test_case.py
index cc261f0..b1cd9ce 100644
--- a/proxy/testing/test_case.py
+++ b/proxy/testing/test_case.py
@@ -2,7 +2,8 @@
 """
     proxy.py
     ~~~~~~~~
-    ⚡⚡⚡ Fast, Lightweight, Programmable Proxy Server in a single Python file.
+    ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on
+    Network monitoring, controls & Application development, testing, debugging.
 
     :copyright: (c) 2013-present by Abhinav Singh and contributors.
     :license: BSD, see LICENSE for more details.
diff --git a/tests/http/test_http_parser.py b/tests/http/test_http_parser.py
index 6867749..896b778 100644
--- a/tests/http/test_http_parser.py
+++ b/tests/http/test_http_parser.py
@@ -22,6 +22,11 @@ class TestHttpParser(unittest.TestCase):
     def setUp(self) -> None:
         self.parser = HttpParser(httpParserTypes.REQUEST_PARSER)
 
+    def test_urlparse(self) -> None:
+        self.parser.parse(b'CONNECT httpbin.org:443 HTTP/1.1\r\n')
+        self.assertEqual(self.parser.host, b'httpbin.org')
+        self.assertEqual(self.parser.port, 443)
+
     def test_build_request(self) -> None:
         self.assertEqual(
             build_http_request(