Repository: abhinavsingh/proxy.py
Pull request #96 (2019-09-26 01:17:20)

Add HttpParser.build_response utility


Patch:
diff --git a/README.md b/README.md
index 0b002e2..f4d918f 100644
--- a/README.md
+++ b/README.md
@@ -15,6 +15,7 @@
 [![Tested With Ubuntu](https://img.shields.io/static/v1?label=tested%20with&message=Ubuntu%20%F0%9F%96%A5&color=brightgreen)](https://developer.apple.com/library/archive/documentation/IDEs/Conceptual/iOS_Simulator_Guide/Introduction/Introduction.html)
 [![Tested With Windows](https://img.shields.io/static/v1?label=tested%20with&message=Windows%20%F0%9F%92%BB&color=brightgreen)](https://developer.apple.com/library/archive/documentation/IDEs/Conceptual/iOS_Simulator_Guide/Introduction/Introduction.html)
 
+[![Gitter](https://badges.gitter.im/proxy-py/community.svg)](https://gitter.im/proxy-py/community)
 [![Maintenance](https://img.shields.io/static/v1?label=maintained%3F&message=yes&color=green)](https://gitHub.com/abhinavsingh/proxy.py/graphs/commit-activity)
 [![Ask Me Anything](https://img.shields.io/static/v1?label=need%20help%3F&message=ask&color=green)](https://twitter.com/imoracle)
 [![Contributions Welcome](https://img.shields.io/static/v1?label=contributions&message=welcome%20%F0%9F%91%8D&color=green)](https://github.com/abhinavsingh/proxy.py/issues)
@@ -396,18 +397,59 @@ To enable TLS interception first generate CA certificates:
 make ca-certificates
 ```
 
-Start `proxy.py` as:
+Lets also enable `CacheResponsePlugin` so that we can verify decrypted
+response from the server. Start `proxy.py` as:
 
 ```
 $ proxy.py \
+    --plugins plugin_examples.CacheResponsesPlugin \
     --ca-key-file ca-key.pem \
     --ca-cert-file ca-cert.pem \
     --ca-signing-key-file ca-signing-key.pem
 ```
 
-Verify using `curl -x localhost:8899 --cacert ca-cert.pem https://httpbin.org/get`
+Verify using `curl -v -x localhost:8899 --cacert ca-cert.pem https://httpbin.org/get`
+
+```
+*  issuer: C=US; ST=CA; L=SanFrancisco; O=proxy.py; OU=CA; CN=Proxy PY CA; emailAddress=proxyca@mailserver.com
+*  SSL certificate verify ok.
+> GET /get HTTP/1.1
+... [redacted] ...
+< Connection: keep-alive
+< 
+{
+  "args": {}, 
+  "headers": {
+    "Accept": "*/*", 
+    "Host": "httpbin.org", 
+    "User-Agent": "curl/7.54.0"
+  }, 
+  "origin": "1.2.3.4, 5.6.7.8", 
+  "url": "https://httpbin.org/get"
+}
+```
+
+The `issuer` line confirms that response was intercepted.
+
+Also verify the contents of cached response file.  Get path to the cache
+file from `proxy.py` logs.
+
+`$ cat /path/to/your/tmp/directory/httpbin.org-1569452863.924174.txt`
 
 ```
+HTTP/1.1 200 OK
+Access-Control-Allow-Credentials: true
+Access-Control-Allow-Origin: *
+Content-Type: application/json
+Date: Wed, 25 Sep 2019 23:07:05 GMT
+Referrer-Policy: no-referrer-when-downgrade
+Server: nginx
+X-Content-Type-Options: nosniff
+X-Frame-Options: DENY
+X-XSS-Protection: 1; mode=block
+Content-Length: 202
+Connection: keep-alive
+
 {
   "args": {}, 
   "headers": {
@@ -418,9 +460,13 @@ Verify using `curl -x localhost:8899 --cacert ca-cert.pem https://httpbin.org/ge
   "origin": "1.2.3.4, 5.6.7.8", 
   "url": "https://httpbin.org/get"
 }
+
 ```
 
-Now you can use CA flags with 
+Viola!!!  If you remove CA flags, encrypted data will be found in the
+cached file instead of plain text.
+
+Now use CA flags other 
 [plugin examples](#plugin-examples) to make them work for `https` traffic.
 
 Plugin Developer and Contributor Guide
diff --git a/plugin_examples.py b/plugin_examples.py
index 24e34cf..f9dbd2f 100644
--- a/plugin_examples.py
+++ b/plugin_examples.py
@@ -97,13 +97,7 @@ class ManInTheMiddlePlugin(proxy.HttpProxyBasePlugin):
         pass
 
     def handle_upstream_response(self, raw: bytes) -> bytes:
-        body = b'Hello from man in the middle'
-        response = proxy.CRLF.join([
-            b'HTTP/1.1 200 OK',
-            b'Content-Length: ' + proxy.bytes_(str(len(body))),
-            proxy.CRLF,
-        ]) + body
-        return response
+        return proxy.HttpParser.build_response(200, reason=b'OK', body=b'Hello from man in the middle')
 
     def on_upstream_connection_close(self) -> None:
         pass
diff --git a/proxy.py b/proxy.py
index 7778434..1f65767 100755
--- a/proxy.py
+++ b/proxy.py
@@ -449,7 +449,9 @@ class Worker(multiprocessing.Process):
                 conn = socket.fromfd(
                     fileno, family=self.config.family, type=socket.SOCK_STREAM)
                 # TODO(abhinavsingh): Move handshake logic within
-                # HttpProtocolHandler.
+                # HttpProtocolHandler or should this go under TcpServer directly?
+                # Rationale behind deferring ssl wrap is that plugins can custom wrap
+                # sockets if necessary.
                 if self.config.certfile and self.config.keyfile:
                     try:
                         ctx = ssl.create_default_context(
@@ -567,6 +569,21 @@ class HttpParser:
         self.host: Optional[bytes] = None
         self.port: Optional[int] = None
 
+    def add_header(self, key: bytes, value: bytes) -> None:
+        self.headers[key.lower()] = (key, value)
+
+    def add_headers(self, headers: List[Tuple[bytes, bytes]]) -> None:
+        for (key, value) in headers:
+            self.add_header(key, value)
+
+    def del_header(self, header: bytes) -> None:
+        if header.lower() in self.headers:
+            del self.headers[header.lower()]
+
+    def del_headers(self, headers: List[bytes]) -> None:
+        for key in headers:
+            self.del_header(key.lower())
+
     def set_host_port(self) -> None:
         if self.type == httpParserTypes.REQUEST_PARSER:
             if self.method == b'CONNECT' and self.url:
@@ -699,27 +716,55 @@ class HttpParser:
         return url
 
     def build(self, disable_headers: Optional[List[bytes]] = None) -> bytes:
+        assert self.method and self.version
         if disable_headers is None:
             disable_headers = DEFAULT_DISABLE_HEADERS
+        return HttpParser.build_request(
+            self.method, self.build_url(), self.version,
+            headers={} if not self.headers else {self.headers[k][0]: self.headers[k][1] for k in self.headers if
+                                                 k.lower() not in disable_headers},
+            body=self.body
+        )
 
-        assert self.method and self.version
-        req = b' '.join([self.method, self.build_url(), self.version])
-        req += CRLF
+    @staticmethod
+    def build_request(method: bytes, url: bytes, protocol_version: bytes,
+                      headers: Optional[Dict[bytes, bytes]] = None,
+                      body: Optional[bytes] = None) -> bytes:
+        if headers is None:
+            headers = {}
+        return HttpParser.build_pkt([method, url, protocol_version], headers, body)
 
-        for k in self.headers:
-            if k.lower() not in disable_headers:
-                req += self.build_header(self.headers[k]
-                                         [0], self.headers[k][1]) + CRLF
+    @staticmethod
+    def build_response(status_code: int,
+                       protocol_version: bytes = b'HTTP/1.1',
+                       reason: Optional[bytes] = None,
+                       headers: Optional[Dict[bytes, bytes]] = None,
+                       body: Optional[bytes] = None) -> bytes:
+        line = [protocol_version, bytes_(str(status_code))]
+        if reason:
+            line.append(reason)
+        if headers is None:
+            headers = {}
+        if body is not None and not any(k.lower() == b'content-length' for k in headers):
+            headers[b'Content-Length'] = bytes_(str(len(body)))
+        return HttpParser.build_pkt(line, headers, body)
 
+    @staticmethod
+    def build_pkt(line: List[bytes],
+                  headers: Optional[Dict[bytes, bytes]] = None,
+                  body: Optional[bytes] = None) -> bytes:
+        req = WHITESPACE.join(line) + CRLF
+        if headers is not None:
+            for k in headers:
+                req += HttpParser.build_header(k, headers[k]) + CRLF
         req += CRLF
-        if self.body:
-            req += self.body
-
+        if body:
+            req += body
         return req
 
     @staticmethod
     def build_header(k: bytes, v: bytes) -> bytes:
-        return k + b': ' + v
+        return k + COLON + WHITESPACE + v
 
     @staticmethod
     def find_line(raw: bytes) -> Tuple[Optional[bytes], bytes]:
@@ -744,21 +789,6 @@ class HttpParser:
         """Host field SHOULD be None for incoming local WebServer requests."""
         return True if self.host is not None else False
 
-    def add_header(self, key: bytes, value: bytes) -> None:
-        self.headers[key.lower()] = (key, value)
-
-    def add_headers(self, headers: List[Tuple[bytes, bytes]]) -> None:
-        for (key, value) in headers:
-            self.add_header(key, value)
-
-    def del_header(self, header: bytes) -> None:
-        if header.lower() in self.headers:
-            del self.headers[header.lower()]
-
-    def del_headers(self, headers: List[bytes]) -> None:
-        for key in headers:
-            self.del_header(key.lower())
-
 
 class HttpProtocolException(Exception):
     """Top level HttpProtocolException exception class.
@@ -1127,7 +1157,7 @@ class HttpProxyPlugin(HttpProtocolBasePlugin):
                             self.server.conn, server_hostname=text_(
                                 self.request.host))
                         logger.info(
-                            'Intercepting traffic using %s', generated_cert)
+                            'TLS interception using %s', generated_cert)
                         return self.client.conn
         # for general http requests, re-build request packet
         # and queue for the server with appropriate headers
@@ -1450,7 +1480,7 @@ class HttpProtocolHandler(threading.Thread):
                     self.client.conn.shutdown(socket.SHUT_RDWR)
                     self.client.close()
                 except OSError as e:
-                    logger.warning('OSError: %s', str(e))
+                    logger.exception('OSError: %s', str(e), exc_info=e)
 
             # Invoke plugin.on_client_connection_close
             for plugin in self.plugins.values():


Test patch:
diff --git a/tests.py b/tests.py
index fee2754..c1d7cb8 100644
--- a/tests.py
+++ b/tests.py
@@ -369,6 +369,60 @@ class TestHttpParser(unittest.TestCase):
     def setUp(self) -> None:
         self.parser = proxy.HttpParser(proxy.httpParserTypes.REQUEST_PARSER)
 
+    def test_build_request(self) -> None:
+        self.assertEqual(
+            proxy.HttpParser.build_request(b'GET', b'http://localhost:12345', b'HTTP/1.1'),
+            proxy.CRLF.join([
+                b'GET http://localhost:12345 HTTP/1.1',
+                proxy.CRLF
+            ]))
+        self.assertEqual(
+            proxy.HttpParser.build_request(b'GET', b'http://localhost:12345', b'HTTP/1.1',
+                                           headers={b'key': b'value'}),
+            proxy.CRLF.join([
+                b'GET http://localhost:12345 HTTP/1.1',
+                b'key: value',
+                proxy.CRLF
+            ]))
+        self.assertEqual(
+            proxy.HttpParser.build_request(b'GET', b'http://localhost:12345', b'HTTP/1.1',
+                                           headers={b'key': b'value'},
+                                           body=b'Hello from proxy.py'),
+            proxy.CRLF.join([
+                b'GET http://localhost:12345 HTTP/1.1',
+                b'key: value',
+                proxy.CRLF
+            ]) + b'Hello from proxy.py')
+
+    def test_build_response(self) -> None:
+        self.assertEqual(
+            proxy.HttpParser.build_response(200, reason=b'OK', protocol_version=b'HTTP/1.1'),
+            proxy.CRLF.join([
+                b'HTTP/1.1 200 OK',
+                proxy.CRLF
+            ]))
+        self.assertEqual(
+            proxy.HttpParser.build_response(200, reason=b'OK', protocol_version=b'HTTP/1.1',
+                                            headers={b'key': b'value'}),
+            proxy.CRLF.join([
+                b'HTTP/1.1 200 OK',
+                b'key: value',
+                proxy.CRLF
+            ]))
+
+    def test_build_response_adds_content_length_header(self) -> None:
+        body = b'Hello world!!!'
+        self.assertEqual(
+            proxy.HttpParser.build_response(200, reason=b'OK', protocol_version=b'HTTP/1.1',
+                                            headers={b'key': b'value'},
+                                            body=body),
+            proxy.CRLF.join([
+                b'HTTP/1.1 200 OK',
+                b'key: value',
+                b'Content-Length: ' + proxy.bytes_(str(len(body))),
+                proxy.CRLF
+            ]) + body)
+
     def test_build_header(self) -> None:
         self.assertEqual(
             proxy.HttpParser.build_header(
@@ -379,7 +433,7 @@ class TestHttpParser(unittest.TestCase):
             proxy.HttpParser.find_line(
                 b'CONNECT python.org:443 HTTP/1.0\r\n\r\n'),
             (b'CONNECT python.org:443 HTTP/1.0',
-             b'\r\n'))
+             proxy.CRLF))
 
     def test_find_line_returns_None(self) -> None:
         self.assertEqual(