Repository: abhinavsingh/proxy.py
Pull request #125 (2019-10-11 02:38:09)

Invoke HttpWebServerBasePlugin.handle_request for each request in HTTP/1.1 pipeline
Example start with `WebServerPlugin` plugin:

```
./proxy.py \
    --enable-web-server \
    --plugins plugin_examples.WebServerPlugin
```

Send HTTP/1.1 pipeline request using `telnet`.  Our plugin.handle_request will now be invoked each time a new request completes on the pipelined connection.

```
$ telnet localhost 8899
Trying ::1...
Connected to localhost.
Escape character is '^]'.
GET http://localhost:8899/http-route-example HTTP/1.1

HTTP/1.1 200
Content-Length: 19

HTTP route responseGET http://localhost:8899/http-route-example HTTP/1.1

HTTP/1.1 200
Content-Length: 19

HTTP route responseGET http://localhost:8899/http-route-example HTTP/1.1

HTTP/1.1 200
Content-Length: 19

HTTP route responseGET http://localhost:8899/https-route-example HTTP/1.1   

HTTP/1.1 200
Content-Length: 20

HTTPS route response
```


Patch:
diff --git a/README.md b/README.md
index e7d9e50..4bd53a6 100644
--- a/README.md
+++ b/README.md
@@ -36,6 +36,8 @@ Table of Contents
 * [Start proxy.py](#start-proxypy)
     * [Command Line](#command-line)
     * [Docker Image](#docker-image)
+        * [Stable version](#stable-version-from-docker-hub)
+        * [Development version](#build-development-version-locally)
 * [Plugin Examples](#plugin-examples)
     * [ProposedRestApiPlugin](#proposedrestapiplugin)
     * [RedirectToCustomServerPlugin](#redirecttocustomserverplugin)
@@ -66,7 +68,7 @@ Features
 ========
 
 - Lightweight
-    - Distributed as a single file module `~50KB`
+    - Distributed as a single file module `~100KB`
     - Uses only `~5-20MB` RAM
     - No external dependency other than standard Python library
 - Programmable
@@ -104,11 +106,12 @@ or from GitHub `master` branch
 
     $ pip install git+https://github.com/abhinavsingh/proxy.py.git@master
 
-or download from here [proxy.py](https://raw.githubusercontent.com/abhinavsingh/proxy.py/master/proxy.py)
 or simply `wget` it:
 
     $ wget -q https://raw.githubusercontent.com/abhinavsingh/proxy.py/master/proxy.py
 
+or download from here [proxy.py](https://raw.githubusercontent.com/abhinavsingh/proxy.py/master/proxy.py)
+
 ## Development version
 
     $ pip install git+https://github.com/abhinavsingh/proxy.py.git@develop
@@ -166,19 +169,28 @@ See [flags](#flags) for full list of available configuration options.
 
 ## Docker image
 
+#### Stable Version from Docker Hub
+
     $ docker run -it -p 8899:8899 --rm abhinavsingh/proxy.py:latest
 
+#### Build Development Version Locally
+
+    $ git clone https://github.com/abhinavsingh/proxy.py.git
+    $ cd proxy.py
+    $ make container
+    $ docker run -it -p 8899:8899 --rm abhinavsingh/proxy.py:v$(./proxy.py -v)
+
 By default `docker` binary is started with IPv4 networking flags:
 
     --hostname 0.0.0.0 --port 8899
 
 To override input flags, start docker image as follows.
-For example, to check `proxy.py --version`:
+For example, to check `proxy.py` version within Docker image:
 
     $ docker run -it \
         -p 8899:8899 \
         --rm abhinavsingh/proxy.py:latest \
-        --version
+        -v
 
 [![WARNING](https://img.shields.io/static/v1?label=MacOS&message=warning&color=red)](https://github.com/moby/vpnkit/issues/469)
 `docker` image is currently broken on `macOS` due to incompatibility with [vpnkit](https://github.com/moby/vpnkit/issues/469).
diff --git a/plugin_examples.py b/plugin_examples.py
index a9af97e..1751dd7 100644
--- a/plugin_examples.py
+++ b/plugin_examples.py
@@ -108,7 +108,9 @@ class RedirectToCustomServerPlugin(proxy.HttpProxyBasePlugin):
         # Redirect all non-https requests to inbuilt WebServer.
         if self.request.method != proxy.httpMethods.CONNECT:
             self.request.url = urlparse.urlsplit(self.UPSTREAM_SERVER)
-            self.request.set_host_port()
+            # This command will re-parse modified url and
+            # update host, port, path fields
+            self.request.set_line_attributes()
         return False
 
     def on_upstream_connection(self) -> None:
@@ -201,10 +203,9 @@ class WebServerPlugin(proxy.HttpWebServerBasePlugin):
         ]
 
     def handle_request(self, request: proxy.HttpParser) -> None:
-        path = request.build_url()
-        if path == b'/http-route-example':
+        if request.path == b'/http-route-example':
             self.client.queue(proxy.build_http_response(200, body=b'HTTP route response'))
-        elif path == b'/https-route-example':
+        elif request.path == b'/https-route-example':
             self.client.queue(proxy.build_http_response(200, body=b'HTTPS route response'))
 
     def on_websocket_open(self) -> None:
diff --git a/proxy.py b/proxy.py
index dd914b4..0ba62f2 100755
--- a/proxy.py
+++ b/proxy.py
@@ -121,7 +121,7 @@ def bytes_(s: Any, encoding: str = 'utf-8', errors: str = 'strict') -> Any:
 
 
 version = bytes_(__version__)
-CRLF, COLON, WHITESPACE, COMMA, DOT = b'\r\n', b':', b' ', b',', b'.'
+CRLF, COLON, WHITESPACE, COMMA, DOT, HTTP_1_1 = b'\r\n', b':', b' ', b',', b'.', b'HTTP/1.1'
 PROXY_AGENT_HEADER_KEY = b'Proxy-agent'
 PROXY_AGENT_HEADER_VALUE = b'proxy.py v' + version
 PROXY_AGENT_HEADER = PROXY_AGENT_HEADER_KEY + \
@@ -192,7 +192,7 @@ websocketOpcodes = WebsocketOpcodes(0x0, 0x1, 0x2, 0x8, 0x9, 0xA)
 
 
 def build_http_request(method: bytes, url: bytes,
-                       protocol_version: bytes = b'HTTP/1.1',
+                       protocol_version: bytes = HTTP_1_1,
                        headers: Optional[Dict[bytes, bytes]] = None,
                        body: Optional[bytes] = None) -> bytes:
     """Build and returns a HTTP request packet."""
@@ -203,7 +203,7 @@ def build_http_request(method: bytes, url: bytes,
 
 
 def build_http_response(status_code: int,
-                        protocol_version: bytes = b'HTTP/1.1',
+                        protocol_version: bytes = HTTP_1_1,
                         reason: Optional[bytes] = None,
                         headers: Optional[Dict[bytes, bytes]] = None,
                         body: Optional[bytes] = None) -> bytes:
@@ -537,6 +537,7 @@ class HttpParser:
         # which is broken.
         self.host: Optional[bytes] = None
         self.port: Optional[int] = None
+        self.path: Optional[bytes] = None
 
     def header(self, key: bytes) -> bytes:
         if key.lower() not in self.headers:
@@ -561,7 +562,7 @@ class HttpParser:
         for key in headers:
             self.del_header(key.lower())
 
-    def set_host_port(self) -> None:
+    def set_line_attributes(self) -> None:
         if self.type == httpParserTypes.REQUEST_PARSER:
             if self.method == httpMethods.CONNECT and self.url:
                 u = urlparse.urlsplit(b'//' + self.url.path)
@@ -571,6 +572,7 @@ class HttpParser:
                     if self.url.port else 80
             else:
                 raise KeyError('Invalid request\n%s' % self.bytes)
+            self.path = self.build_url()
 
     def is_chunked_encoded(self) -> bool:
         return b'transfer-encoding' in self.headers and \
@@ -668,7 +670,7 @@ class HttpParser:
             self.version = line[0]
             self.code = line[1]
             self.reason = WHITESPACE.join(line[2:])
-        self.set_host_port()
+        self.set_line_attributes()
 
     def process_header(self, raw: bytes) -> None:
         parts = raw.split(COLON)
@@ -679,7 +681,6 @@ class HttpParser:
     def build_url(self) -> bytes:
         if not self.url:
             return b'/None'
-
         url = self.url.path
         if url == b'':
             url = b'/'
@@ -690,16 +691,14 @@ class HttpParser:
         return url
 
     def build(self, disable_headers: Optional[List[bytes]] = None) -> bytes:
-        assert self.method and self.version
+        assert self.method and self.version and self.path
         if disable_headers is None:
             disable_headers = DEFAULT_DISABLE_HEADERS
-        body: Optional[bytes] = None
-        if self.is_chunked_encoded() and self.body:
-            body = ChunkParser.to_chunks(self.body)
-        else:
-            body = self.body
+        body: Optional[bytes] = ChunkParser.to_chunks(self.body) \
+            if self.is_chunked_encoded() and self.body else \
+            self.body
         return build_http_request(
-            self.method, self.build_url(), self.version,
+            self.method, self.path, self.version,
             headers={} if not self.headers else {self.headers[k][0]: self.headers[k][1] for k in self.headers if
                                                  k.lower() not in disable_headers},
             body=body
@@ -709,6 +708,11 @@ class HttpParser:
         """Host field SHOULD be None for incoming local WebServer requests."""
         return True if self.host is not None else False
 
+    def is_http_1_1_keep_alive(self) -> bool:
+        return self.version == HTTP_1_1 and \
+            (not self.has_header(b'Connection') or
+             self.header(b'Connection').lower() == b'keep-alive')
+
 
 class AcceptorPool:
     """AcceptorPool.
@@ -864,7 +868,7 @@ class HttpRequestRejected(ProtocolException):
     def response(self, _request: HttpParser) -> Optional[bytes]:
         pkt = []
         if self.status_code is not None:
-            line = b'HTTP/1.1 ' + bytes_(self.status_code)
+            line = HTTP_1_1 + WHITESPACE + bytes_(self.status_code)
             if self.reason:
                 line += WHITESPACE + self.reason
             pkt.append(line)
@@ -1206,7 +1210,7 @@ class HttpProxyPlugin(ProtocolHandlerPlugin):
                 (self.client.addr[0], self.client.addr[1],
                  text_(self.request.method),
                  text_(server_host), server_port,
-                 text_(self.request.build_url()),
+                 text_(self.request.path),
                  text_(self.response.code),
                  text_(self.response.reason),
                  self.response.total_size))
@@ -1245,7 +1249,11 @@ class HttpProxyPlugin(ProtocolHandlerPlugin):
         else:
             return raw
 
-    def generate_upstream_certificate(self, _certificate: Optional[Dict[str, Any]]) -> Optional[str]:
+    @staticmethod
+    def generated_cert_file_path(ca_cert_dir: str, host: str) -> str:
+        return os.path.join(ca_cert_dir, '%s.pem' % host)
+
+    def generate_upstream_certificate(self, _certificate: Optional[Dict[str, Any]]) -> str:
         if not (self.config.ca_cert_dir and self.config.ca_signing_key_file and
                 self.config.ca_cert_file and self.config.ca_key_file):
             raise ProtocolException(
@@ -1253,12 +1261,9 @@ class HttpProxyPlugin(ProtocolHandlerPlugin):
                 f'--ca-cert-file:{ self.config.ca_cert_file }, '
                 f'--ca-key-file:{ self.config.ca_key_file }, '
                 f'--ca-signing-key-file:{ self.config.ca_signing_key_file }')
+        cert_file_path = HttpProxyPlugin.generated_cert_file_path(
+            self.config.ca_cert_dir, text_(self.request.host))
         with self.lock:
-            cert_file_path = os.path.join(
-                self.config.ca_cert_dir,
-                '%s.pem' %
-                text_(
-                    self.request.host))
             if not os.path.isfile(cert_file_path):
                 logger.debug('Generating certificates %s', cert_file_path)
                 # TODO: Parse subject from certificate
@@ -1275,7 +1280,7 @@ class HttpProxyPlugin(ProtocolHandlerPlugin):
                     stderr=subprocess.PIPE)
                 # TODO: Ensure sign_cert success.
                 sign_cert.communicate(timeout=10)
-            return cert_file_path
+        return cert_file_path
 
     def on_request_complete(self) -> Union[socket.socket, bool]:
         if not self.request.has_upstream_server():
@@ -1378,9 +1383,6 @@ class WebsocketFrame:
     GUID = b'258EAFA5-E914-47DA-95CA-C5AB0DC85B11'
 
     def __init__(self) -> None:
-        self.reset()
-
-    def reset(self) -> None:
         self.fin: bool = False
         self.rsv1: bool = False
         self.rsv2: bool = False
@@ -1391,6 +1393,17 @@ class WebsocketFrame:
         self.mask: Optional[bytes] = None
         self.data: Optional[bytes] = None
 
+    def reset(self) -> None:
+        self.fin = False
+        self.rsv1 = False
+        self.rsv2 = False
+        self.rsv3 = False
+        self.opcode = 0
+        self.masked = False
+        self.payload_length = None
+        self.mask = None
+        self.data = None
+
     def parse_fin_and_rsv(self, byte: int) -> None:
         self.fin = bool(byte & 1 << 7)
         self.rsv1 = bool(byte & 1 << 6)
@@ -1800,14 +1813,14 @@ class HttpWebServerPlugin(ProtocolHandlerPlugin):
             client: TcpClientConnection,
             request: HttpParser):
         super().__init__(config, client, request)
-
+        self.pipeline_request: Optional[HttpParser] = None
         self.switched_protocol: Optional[int] = None
-
         self.routes: Dict[int, Dict[bytes, HttpWebServerBasePlugin]] = {
             httpProtocolTypes.HTTP: {},
             httpProtocolTypes.HTTPS: {},
             httpProtocolTypes.WEBSOCKET: {},
         }
+        self.route: Optional[HttpWebServerBasePlugin] = None
 
         if b'HttpWebServerBasePlugin' in self.config.plugins:
             for klass in self.config.plugins[b'HttpWebServerBasePlugin']:
@@ -1815,7 +1828,7 @@ class HttpWebServerPlugin(ProtocolHandlerPlugin):
                 for (protocol, path) in instance.routes():
                     self.routes[protocol][path] = instance
 
-    def serve_file_or_404(self, path: str) -> None:
+    def serve_file_or_404(self, path: str) -> bool:
         try:
             with open(path, 'rb') as f:
                 content = f.read()
@@ -1828,8 +1841,10 @@ class HttpWebServerPlugin(ProtocolHandlerPlugin):
                     b'Connection': b'close'
                 }, body=content
             ))
+            return False
         except IOError:
             self.client.queue(self.DEFAULT_404_RESPONSE)
+        return True
 
     def try_upgrade(self) -> bool:
         if self.request.has_header(b'connection') and \
@@ -1850,10 +1865,10 @@ class HttpWebServerPlugin(ProtocolHandlerPlugin):
         if self.request.has_upstream_server():
             return False
 
-        url = self.request.build_url()
-
         # If a websocket route exists for the path, try upgrade
-        if url in self.routes[httpProtocolTypes.WEBSOCKET]:
+        if self.request.path in self.routes[httpProtocolTypes.WEBSOCKET]:
+            self.route = self.routes[httpProtocolTypes.WEBSOCKET][self.request.path]
+
             # Connection upgrade
             teardown = self.try_upgrade()
             if teardown:
@@ -1862,24 +1877,24 @@ class HttpWebServerPlugin(ProtocolHandlerPlugin):
             # For upgraded connections, nothing more to do
             if self.switched_protocol:
                 # Invoke plugin.on_websocket_open
-                for r in self.routes[httpProtocolTypes.WEBSOCKET]:
-                    if r == url:
-                        self.routes[httpProtocolTypes.WEBSOCKET][r].on_websocket_open()
+                self.route.on_websocket_open()
                 return False
 
         # Routing for Http(s) requests
-        protocol = httpProtocolTypes.HTTPS if self.config.certfile and self.config.keyfile else httpProtocolTypes.HTTP
+        protocol = httpProtocolTypes.HTTPS \
+            if self.config.certfile and self.config.keyfile else \
+            httpProtocolTypes.HTTP
         for r in self.routes[protocol]:
-            if r == url:
-                self.routes[protocol][r].handle_request(self.request)
-                return True
+            if r == self.request.path:
+                self.route = self.routes[protocol][r]
+                self.route.handle_request(self.request)
+                return False
 
         # No-route found, try static serving if enabled
         if self.config.enable_static_server:
-            path = text_(url).split('?')[0]
+            path = text_(self.request.path).split('?')[0]
             if os.path.isfile(self.config.static_server_dir + path):
-                self.serve_file_or_404(self.config.static_server_dir + path)
-                return True
+                return self.serve_file_or_404(self.config.static_server_dir + path)
 
         # Catch all unhandled web server requests, return 404
         self.client.queue(self.DEFAULT_404_RESPONSE)
@@ -1893,17 +1908,28 @@ class HttpWebServerPlugin(ProtocolHandlerPlugin):
 
     def on_client_data(self, raw: bytes) -> Optional[bytes]:
         if self.switched_protocol == httpProtocolTypes.WEBSOCKET:
-            url = self.request.build_url()
             remaining = raw
             frame = WebsocketFrame()
             while remaining != b'':
                 # TODO: Teardown if invalid protocol exception
                 remaining = frame.parse(remaining)
                 for r in self.routes[httpProtocolTypes.WEBSOCKET]:
-                    if r == url:
+                    if r == self.request.path:
                         self.routes[httpProtocolTypes.WEBSOCKET][r].on_websocket_message(frame)
                 frame.reset()
             return None
+        # If 1st valid request was completed and it's a HTTP/1.1 keep-alive
+        elif self.request.state == httpParserStates.COMPLETE and \
+                self.request.is_http_1_1_keep_alive():
+            if self.pipeline_request is None:
+                self.pipeline_request = HttpParser(httpParserTypes.REQUEST_PARSER)
+            self.pipeline_request.parse(raw)
+            if self.pipeline_request.state == httpParserStates.COMPLETE:
+                assert self.route is not None
+                self.route.handle_request(self.pipeline_request)
+                if not self.pipeline_request.is_http_1_1_keep_alive():
+                    raise ProtocolException()
+                self.pipeline_request = None
         return raw
 
     def on_response_chunk(self, chunk: bytes) -> bytes:
@@ -1915,13 +1941,16 @@ class HttpWebServerPlugin(ProtocolHandlerPlugin):
         if self.switched_protocol:
             # Invoke plugin.on_websocket_close
             for r in self.routes[httpProtocolTypes.WEBSOCKET]:
-                if r == self.request.build_url():
+                if r == self.request.path:
                     self.routes[httpProtocolTypes.WEBSOCKET][r].on_websocket_close()
+        self.access_log()
+
+    def access_log(self) -> None:
         logger.info(
             '%s:%s - %s %s' %
             (self.client.addr[0], self.client.addr[1], text_(
                 self.request.method), text_(
-                self.request.build_url())))
+                self.request.path)))
 
     def get_descriptors(
             self) -> Tuple[List[socket.socket], List[socket.socket]]:
@@ -1989,12 +2018,6 @@ class ProtocolHandler(threading.Thread):
             conn = ctx.wrap_socket(conn, server_side=True)
         return conn
 
-    def send_server_error(self, e: Exception) -> None:
-        logger.exception('Server error', exc_info=e)
-        self.client.queue(build_http_response(
-            500, b'Server Error'
-        ))
-
     def connection_inactive_for(self) -> int:
         return (self.now() - self.last_activity).seconds
 
@@ -2031,17 +2054,22 @@ class ProtocolHandler(threading.Thread):
                 self.client.closed = True
                 return True
 
-            # ProtocolHandlerPlugin.on_client_data
-            plugin_index = 0
-            plugins = list(self.plugins.values())
-            while plugin_index < len(plugins) and client_data:
-                client_data = plugins[plugin_index].on_client_data(client_data)
-                if client_data is None:
-                    break
-                plugin_index += 1
-
-            if client_data:
-                try:
+            try:
+                # ProtocolHandlerPlugin.on_client_data
+                # Can raise ProtocolException to teardown the connection
+                plugin_index = 0
+                plugins = list(self.plugins.values())
+                while plugin_index < len(plugins) and client_data:
+                    client_data = plugins[plugin_index].on_client_data(client_data)
+                    if client_data is None:
+                        break
+                    plugin_index += 1
+
+                # Don't parse request any further after 1st request has completed.
+                # This specially does happen for pipeline requests.
+                # Plugins can utilize on_client_data for such cases and
+                # apply custom logic to handle request data sent after 1st valid request.
+                if client_data and self.request.state != httpParserStates.COMPLETE:
                     # Parse http request
                     self.request.parse(client_data)
                     if self.request.state == httpParserStates.COMPLETE:
@@ -2059,17 +2087,13 @@ class ProtocolHandler(threading.Thread):
                                             'Upgraded client conn for plugin %s', str(plugin_))
                             elif isinstance(upgraded_sock, bool) and upgraded_sock is True:
                                 return True
-                except ProtocolException as e:
-                    logger.exception(
-                        'ProtocolException type raised', exc_info=e)
-                    response = e.response(self.request)
-                    if response:
-                        self.client.queue(response)
-                    else:
-                        # If ProtocolException doesn't return a response,
-                        # its a server error
-                        self.send_server_error(e)
-                    return True
+            except ProtocolException as e:
+                logger.exception(
+                    'ProtocolException type raised', exc_info=e)
+                response = e.response(self.request)
+                if response:
+                    self.client.queue(response)
+                return True
         return False
 
     def get_events(self) -> Dict[socket.socket, int]:
@@ -2246,7 +2270,7 @@ class DevtoolsProtocolPlugin(ProtocolHandlerPlugin):
 
     def on_request_complete(self) -> Union[socket.socket, bool]:
         if not self.request.has_upstream_server() and \
-                self.request.build_url() == self.config.devtools_ws_path:
+                self.request.path == self.config.devtools_ws_path:
             return False
 
         # Handle devtool frontend websocket upgrade
@@ -2259,7 +2283,7 @@ class DevtoolsProtocolPlugin(ProtocolHandlerPlugin):
 
     def on_response_chunk(self, chunk: bytes) -> bytes:
         if not self.request.has_upstream_server() and \
-                self.request.build_url() == self.config.devtools_ws_path:
+                self.request.path == self.config.devtools_ws_path:
             return chunk
 
         if self.config.devtools_event_queue:
@@ -2292,10 +2316,10 @@ class DevtoolsProtocolPlugin(ProtocolHandlerPlugin):
             'documentURL': 'http://proxy-py',
             'request': {
                 'url': text_(
-                    self.request.build_url()
+                    self.request.path
                     if self.request.has_upstream_server() else
                     b'http://' + bytes_(str(self.config.hostname)) +
-                    COLON + bytes_(self.config.port) + self.request.build_url()
+                    COLON + bytes_(self.config.port) + self.request.path
                 ),
                 'urlFragment': '',
                 'method': text_(self.request.method),


Test patch:
diff --git a/tests.py b/tests.py
index 3ab61ee..8d39c3c 100644
--- a/tests.py
+++ b/tests.py
@@ -18,6 +18,7 @@ import socket
 import ssl
 import tempfile
 import unittest
+import uuid
 from contextlib import closing
 from http.server import HTTPServer, BaseHTTPRequestHandler
 from threading import Thread
@@ -542,7 +543,7 @@ class TestHttpParser(unittest.TestCase):
 
     def test_set_host_port_raises(self) -> None:
         with self.assertRaises(KeyError):
-            self.parser.set_host_port()
+            self.parser.set_line_attributes()
 
     def test_find_line(self) -> None:
         self.assertEqual(
@@ -1231,6 +1232,76 @@ class TestHttpProtocolHandler(unittest.TestCase):
                 data=None), selectors.EVENT_WRITE), ],
         ]
 
+    def assert_data_queued(
+            self, mock_server_connection: mock.Mock, server: mock.Mock) -> None:
+        self.proxy.run_once()
+        self.assertEqual(
+            self.proxy.request.state,
+            proxy.httpParserStates.COMPLETE)
+        mock_server_connection.assert_called_once()
+        server.connect.assert_called_once()
+        server.closed = False
+        assert self.http_server_port is not None
+        pkt = proxy.CRLF.join([
+            b'GET / HTTP/1.1',
+            b'User-Agent: proxy.py/%s' % proxy.version,
+            b'Host: localhost:%d' % self.http_server_port,
+            b'Accept: */*',
+            b'Via: %s' % b'1.1 proxy.py v%s' % proxy.version,
+            proxy.CRLF
+        ])
+        server.queue.assert_called_once_with(pkt)
+        server.buffer_size.return_value = len(pkt)
+
+    def assert_data_queued_to_server(self, server: mock.Mock) -> None:
+        assert self.http_server_port is not None
+        self.assertEqual(
+            self._conn.send.call_args[0][0],
+            proxy.HttpProxyPlugin.PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT)
+
+        self._conn.recv.return_value = proxy.CRLF.join([
+            b'GET / HTTP/1.1',
+            b'Host: localhost:%d' % self.http_server_port,
+            b'User-Agent: proxy.py/%s' % proxy.version,
+            proxy.CRLF
+        ])
+        self.proxy.run_once()
+
+        pkt = proxy.CRLF.join([
+            b'GET / HTTP/1.1',
+            b'Host: localhost:%d' % self.http_server_port,
+            b'User-Agent: proxy.py/%s' % proxy.version,
+            proxy.CRLF
+        ])
+        server.queue.assert_called_once_with(pkt)
+        server.buffer_size.return_value = len(pkt)
+        server.flush.assert_not_called()
+
+    def mock_selector_for_client_read(self, mock_selector: mock.Mock) -> None:
+        mock_selector.return_value.select.return_value = [(
+            selectors.SelectorKey(
+                fileobj=self._conn,
+                fd=self._conn.fileno,
+                events=selectors.EVENT_READ,
+                data=None), selectors.EVENT_READ), ]
+
+
+class TestWebServerPlugin(unittest.TestCase):
+
+    @mock.patch('selectors.DefaultSelector')
+    @mock.patch('socket.fromfd')
+    def setUp(self, mock_fromfd: mock.Mock, mock_selector: mock.Mock) -> None:
+        self.fileno = 10
+        self._addr = ('127.0.0.1', 54382)
+        self._conn = mock_fromfd.return_value
+        self.mock_selector = mock_selector
+        self.config = proxy.ProtocolConfig()
+        self.config.plugins = proxy.load_plugins(
+            b'proxy.HttpProxyPlugin,proxy.HttpWebServerPlugin')
+        self.proxy = proxy.ProtocolHandler(
+            self.fileno, self._addr, config=self.config)
+        self.proxy.initialize()
+
     @mock.patch('selectors.DefaultSelector')
     @mock.patch('socket.fromfd')
     def test_pac_file_served_from_disk(
@@ -1270,14 +1341,6 @@ class TestHttpProtocolHandler(unittest.TestCase):
             }, body=pac_file_content
         ))
 
-    def mock_selector_for_client_read(self, mock_selector: mock.Mock) -> None:
-        mock_selector.return_value.select.return_value = [(
-            selectors.SelectorKey(
-                fileobj=self._conn,
-                fd=self._conn.fileno,
-                events=selectors.EVENT_READ,
-                data=None), selectors.EVENT_READ), ]
-
     @mock.patch('selectors.DefaultSelector')
     @mock.patch('socket.fromfd')
     def test_default_web_server_returns_404(
@@ -1427,50 +1490,13 @@ class TestHttpProtocolHandler(unittest.TestCase):
             proxy.CRLF,
         ])
 
-    def assert_data_queued(
-            self, mock_server_connection: mock.Mock, server: mock.Mock) -> None:
-        self.proxy.run_once()
-        self.assertEqual(
-            self.proxy.request.state,
-            proxy.httpParserStates.COMPLETE)
-        mock_server_connection.assert_called_once()
-        server.connect.assert_called_once()
-        server.closed = False
-        assert self.http_server_port is not None
-        pkt = proxy.CRLF.join([
-            b'GET / HTTP/1.1',
-            b'User-Agent: proxy.py/%s' % proxy.version,
-            b'Host: localhost:%d' % self.http_server_port,
-            b'Accept: */*',
-            b'Via: %s' % b'1.1 proxy.py v%s' % proxy.version,
-            proxy.CRLF
-        ])
-        server.queue.assert_called_once_with(pkt)
-        server.buffer_size.return_value = len(pkt)
-
-    def assert_data_queued_to_server(self, server: mock.Mock) -> None:
-        assert self.http_server_port is not None
-        self.assertEqual(
-            self._conn.send.call_args[0][0],
-            proxy.HttpProxyPlugin.PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT)
-
-        self._conn.recv.return_value = proxy.CRLF.join([
-            b'GET / HTTP/1.1',
-            b'Host: localhost:%d' % self.http_server_port,
-            b'User-Agent: proxy.py/%s' % proxy.version,
-            proxy.CRLF
-        ])
-        self.proxy.run_once()
-
-        pkt = proxy.CRLF.join([
-            b'GET / HTTP/1.1',
-            b'Host: localhost:%d' % self.http_server_port,
-            b'User-Agent: proxy.py/%s' % proxy.version,
-            proxy.CRLF
-        ])
-        server.queue.assert_called_once_with(pkt)
-        server.buffer_size.return_value = len(pkt)
-        server.flush.assert_not_called()
+    def mock_selector_for_client_read(self, mock_selector: mock.Mock) -> None:
+        mock_selector.return_value.select.return_value = [(
+            selectors.SelectorKey(
+                fileobj=self._conn,
+                fd=self._conn.fileno,
+                events=selectors.EVENT_READ,
+                data=None), selectors.EVENT_READ), ]
 
 
 class TestHttpProxyPlugin(unittest.TestCase):
@@ -1551,20 +1577,23 @@ class TestHttpProxyTlsInterception(unittest.TestCase):
     @mock.patch('ssl.wrap_socket')
     @mock.patch('ssl.create_default_context')
     @mock.patch('proxy.TcpServerConnection')
-    @mock.patch('proxy.HttpProxyPlugin.generate_upstream_certificate')
+    @mock.patch('subprocess.Popen')
     @mock.patch('selectors.DefaultSelector')
     @mock.patch('socket.fromfd')
     def test_e2e(
             self,
             mock_fromfd: mock.Mock,
             mock_selector: mock.Mock,
-            mock_generate_certificate: mock.Mock,
+            mock_popen: mock.Mock,
             mock_server_conn: mock.Mock,
             mock_ssl_context: mock.Mock,
             mock_ssl_wrap: mock.Mock) -> None:
+        host, port = uuid.uuid4().hex, 443
+        netloc = '{0}:{1}'.format(host, port)
+
         self.mock_fromfd = mock_fromfd
         self.mock_selector = mock_selector
-        self.mock_generate_certificate = mock_generate_certificate
+        self.mock_popen = mock_popen
         self.mock_server_conn = mock_server_conn
         self.mock_ssl_context = mock_ssl_context
         self.mock_ssl_wrap = mock_ssl_wrap
@@ -1608,9 +1637,9 @@ class TestHttpProxyTlsInterception(unittest.TestCase):
         self.assertEqual(self.proxy_plugin.call_args[0][1].connection, self._conn)
 
         connect_request = proxy.build_http_request(
-            proxy.httpMethods.CONNECT, b'super.secure:443',
+            proxy.httpMethods.CONNECT, proxy.bytes_(netloc),
             headers={
-                b'Host': b'super.secure:443',
+                b'Host': proxy.bytes_(netloc),
             })
         self._conn.recv.return_value = connect_request
 
@@ -1643,7 +1672,7 @@ class TestHttpProxyTlsInterception(unittest.TestCase):
         self.proxy_plugin.return_value.before_upstream_connection.assert_called()
         self.proxy_plugin.return_value.on_upstream_connection.assert_called()
 
-        self.mock_server_conn.assert_called_with('super.secure', 443)
+        self.mock_server_conn.assert_called_with(host, port)
         self.mock_server_conn.return_value.connection.setblocking.assert_called_with(False)
 
         self.mock_ssl_context.assert_called_with(ssl.Purpose.SERVER_AUTH)
@@ -1651,26 +1680,27 @@ class TestHttpProxyTlsInterception(unittest.TestCase):
         #                  ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3 | ssl.OP_NO_TLSv1 | ssl.OP_NO_TLSv1_1)
         self.assertEqual(plain_connection.setblocking.call_count, 2)
         self.mock_ssl_context.return_value.wrap_socket.assert_called_with(
-            plain_connection, server_hostname='super.secure')
+            plain_connection, server_hostname=host)
+        # TODO: Assert Popen arguments, piping, success condition
+        self.assertEqual(self.mock_popen.call_count, 2)
         self.assertEqual(ssl_connection.setblocking.call_count, 1)
         self.assertEqual(self.mock_server_conn.return_value._conn, ssl_connection)
-        self.mock_generate_certificate.assert_called_with(
-            self.mock_server_conn.return_value.connection.getpeercert.return_value)
         self._conn.send.assert_called_with(proxy.HttpProxyPlugin.PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT)
+        assert self.config.ca_cert_dir is not None
         self.mock_ssl_wrap.assert_called_with(
             self._conn,
             server_side=True,
             keyfile=self.config.ca_signing_key_file,
-            certfile=self.mock_generate_certificate.return_value
+            certfile=proxy.HttpProxyPlugin.generated_cert_file_path(
+                self.config.ca_cert_dir, host)
         )
         self.assertEqual(self._conn.setblocking.call_count, 2)
         self.assertEqual(self.proxy.client.connection, self.mock_ssl_wrap.return_value)
 
         # Assert connection references for all other plugins is updated
         self.assertEqual(self.plugin.return_value.client._conn, self.mock_ssl_wrap.return_value)
-
-        # Currently proxy doesn't update it's own plugin
-        # self.assertEqual(self.proxy_plugin.return_value.client._conn, self._conn)
+        # Currently proxy doesn't update it's own plugin connection reference
+        # self.assertEqual(self.proxy_plugin.return_value.client._conn, self.mock_ssl_wrap.return_value)
 
 
 class TestHttpRequestRejected(unittest.TestCase):