Repository: abhinavsingh/proxy.py
Pull request #3 (2015-03-17 18:54:49)

v0.2 with support for python3


Patch:
diff --git a/.gitignore b/.gitignore
index 0a1c1ac..7a42f48 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,7 +1,8 @@
 .project
 .pydevproject
 .settings
-
+.coverage
 *.pyc
+cover
 dist
 proxy.py.egg-info
diff --git a/Makefile b/Makefile
new file mode 100644
index 0000000..422ea38
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,17 @@
+.PHONY: all clean package test
+
+all: clean test
+
+clean:
+	find . -name '*.pyc' -exec rm -f {} +
+	find . -name '*.pyo' -exec rm -f {} +
+	find . -name '*~' -exec rm -f {} +
+
+test:
+	python tests.py -v
+
+package:
+	python setup.py sdist
+
+release:
+	python setup.py sdist register upload
diff --git a/proxy.py b/proxy.py
old mode 100644
new mode 100755
index a40d3e8..d67e226
--- a/proxy.py
+++ b/proxy.py
@@ -9,7 +9,7 @@
     :copyright: (c) 2013 by Abhinav Singh.
     :license: BSD, see LICENSE for more details.
 """
-VERSION = (0, 1)
+VERSION = (0, 2)
 __version__ = '.'.join(map(str, VERSION[0:2]))
 __description__ = 'HTTP Proxy Server in Python'
 __author__ = 'Abhinav Singh'
@@ -17,9 +17,9 @@ __author_email__ = 'mailsforabhinav@gmail.com'
 __homepage__ = 'https://github.com/abhinavsingh/proxy.py'
 __license__ = 'BSD'
 
+import sys
 import multiprocessing
 import datetime
-import urlparse
 import argparse
 import logging
 import socket
@@ -27,7 +27,37 @@ import select
 
 logger = logging.getLogger(__name__)
 
-CRLF, COLON, SP = '\r\n', ':', ' '
+# True if we are running on Python 3.
+PY3 = sys.version_info[0] == 3
+
+if PY3:
+    text_type = str
+    binary_type = bytes
+    from urllib import parse as urlparse
+else:
+    text_type = unicode
+    binary_type = str
+    import urlparse
+
+
+def text_(s, encoding='utf-8', errors='strict'):
+    """ If ``s`` is an instance of ``binary_type``, return
+    ``s.decode(encoding, errors)``, otherwise return ``s``"""
+    if isinstance(s, binary_type):
+        return s.decode(encoding, errors)
+    return s  # pragma: no cover
+
+
+def bytes_(s, encoding='utf-8', errors='strict'):
+    """ If ``s`` is an instance of ``text_type``, return
+    ``s.encode(encoding, errors)``, otherwise return ``s``"""
+    if isinstance(s, text_type):  # pragma: no cover
+        return s.encode(encoding, errors)
+    return s
+
+version = bytes_(__version__)
+
+CRLF, COLON, SP = b'\r\n', b':', b' '
 
 HTTP_REQUEST_PARSER = 1
 HTTP_RESPONSE_PARSER = 2
@@ -43,13 +73,14 @@ CHUNK_PARSER_STATE_WAITING_FOR_SIZE = 1
 CHUNK_PARSER_STATE_WAITING_FOR_DATA = 2
 CHUNK_PARSER_STATE_COMPLETE = 3
 
+
 class ChunkParser(object):
     """HTTP chunked encoding response parser."""
     
     def __init__(self):
         self.state = CHUNK_PARSER_STATE_WAITING_FOR_SIZE
-        self.body = ''
-        self.chunk = ''
+        self.body = b''
+        self.chunk = b''
         self.size = None
     
     def parse(self, data):
@@ -72,7 +103,7 @@ class ChunkParser(object):
                     self.state = CHUNK_PARSER_STATE_COMPLETE
                 else:
                     self.state = CHUNK_PARSER_STATE_WAITING_FOR_SIZE
-                self.chunk = ''
+                self.chunk = b''
                 self.size = None
         return len(data) > 0, data
 
@@ -83,8 +114,8 @@ class HttpParser(object):
         self.state = HTTP_PARSER_STATE_INITIALIZED
         self.type = type if type else HTTP_REQUEST_PARSER
         
-        self.raw = ''
-        self.buffer = ''
+        self.raw = b''
+        self.buffer = b''
         
         self.headers = dict()
         self.body = None
@@ -100,7 +131,7 @@ class HttpParser(object):
     def parse(self, data):
         self.raw += data
         data = self.buffer + data
-        self.buffer = ''
+        self.buffer = b''
         
         more = True if len(data) > 0 else False
         while more: 
@@ -109,16 +140,16 @@ class HttpParser(object):
     
     def process(self, data):
         if self.state >= HTTP_PARSER_STATE_HEADERS_COMPLETE and \
-        (self.method == "POST" or self.type == HTTP_RESPONSE_PARSER):
+        (self.method == b"POST" or self.type == HTTP_RESPONSE_PARSER):
             if not self.body:
-                self.body = ''
-            
-            if 'content-length' in self.headers:
+                self.body = b''
+
+            if b'content-length' in self.headers:
                 self.state = HTTP_PARSER_STATE_RCVING_BODY
                 self.body += data
-                if len(self.body) >= int(self.headers['content-length'][1]):
+                if len(self.body) >= int(self.headers[b'content-length'][1]):
                     self.state = HTTP_PARSER_STATE_COMPLETE
-            elif 'transfer-encoding' in self.headers and self.headers['transfer-encoding'][1].lower() == 'chunked':
+            elif b'transfer-encoding' in self.headers and self.headers[b'transfer-encoding'][1].lower() == b'chunked':
                 if not self.chunker:
                     self.chunker = ChunkParser()
                 self.chunker.parse(data)
@@ -126,7 +157,7 @@ class HttpParser(object):
                     self.body = self.chunker.body
                     self.state = HTTP_PARSER_STATE_COMPLETE
             
-            return False, ''
+            return False, b''
         
         line, data = HttpParser.split(data)
         if line == False: return line, data
@@ -138,7 +169,7 @@ class HttpParser(object):
         
         if self.state == HTTP_PARSER_STATE_HEADERS_COMPLETE and \
         self.type == HTTP_REQUEST_PARSER and \
-        not self.method == "POST" and \
+        not self.method == b"POST" and \
         self.raw.endswith(CRLF*2):
             self.state = HTTP_PARSER_STATE_COMPLETE
         
@@ -153,7 +184,7 @@ class HttpParser(object):
         else:
             self.version = line[0]
             self.code = line[1]
-            self.reason = ' '.join(line[2:])
+            self.reason = b' '.join(line[2:])
         self.state = HTTP_PARSER_STATE_LINE_RCVD
     
     def process_header(self, data):
@@ -171,19 +202,19 @@ class HttpParser(object):
     
     def build_url(self):
         if not self.url:
-            return '/None'
+            return b'/None'
         
         url = self.url.path
-        if url == '': url = '/'
-        if not self.url.query == '': url += '?' + self.url.query
-        if not self.url.fragment == '': url += '#' + self.url.fragment
+        if url == b'': url = b'/'
+        if not self.url.query == b'': url += b'?' + self.url.query
+        if not self.url.fragment == b'': url += b'#' + self.url.fragment
         return url
     
     def build_header(self, k, v):
-        return '%s: %s%s' % (k, v, CRLF)
+        return k + b": " + v + CRLF
     
     def build(self, del_headers=None, add_headers=None):
-        req = '%s %s %s' % (self.method, self.build_url(), self.version)
+        req = b" ".join([self.method, self.build_url(), self.version])
         req += CRLF
         
         if not del_headers: del_headers = []
@@ -210,12 +241,12 @@ class HttpParser(object):
         return line, data
 
 class Connection(object):
-    """TCP connection abstraction."""
+    """TCP server/client connection abstraction."""
     
     def __init__(self, what):
-        self.buffer = ''
+        self.buffer = b''
         self.closed = False
-        self.what = what
+        self.what = what # server or client
     
     def send(self, data):
         return self.conn.send(data)
@@ -251,10 +282,10 @@ class Connection(object):
         logger.debug('flushed %d bytes to %s' % (sent, self.what))
 
 class Server(Connection):
-    """Established connection to destination server."""
+    """Establish connection to destination server."""
     
     def __init__(self, host, port):
-        super(Server, self).__init__('server')
+        super(Server, self).__init__(b'server')
         self.addr = (host, int(port))
     
     def connect(self):
@@ -265,7 +296,7 @@ class Client(Connection):
     """Accepted client connection."""
     
     def __init__(self, conn, addr):
-        super(Client, self).__init__('client')
+        super(Client, self).__init__(b'client')
         self.conn = conn
         self.addr = addr
 
@@ -299,6 +330,12 @@ class Proxy(multiprocessing.Process):
         
         self.request = HttpParser()
         self.response = HttpParser(HTTP_RESPONSE_PARSER)
+        
+        self.connection_established_pkt = CRLF.join([
+            b'HTTP/1.1 200 Connection established',
+            b'Proxy-agent: proxy.py v' + version,
+            CRLF
+        ])
     
     def _now(self):
         return datetime.datetime.utcnow()
@@ -326,7 +363,7 @@ class Proxy(multiprocessing.Process):
         if self.request.state == HTTP_PARSER_STATE_COMPLETE:
             logger.debug('request parser is in state complete')
             
-            if self.request.method == "CONNECT":
+            if self.request.method == b"CONNECT":
                 host, port = self.request.url.path.split(COLON)
             elif self.request.url:
                 host, port = self.request.url.hostname, self.request.url.port if self.request.url.port else 80
@@ -343,24 +380,20 @@ class Proxy(multiprocessing.Process):
             # for http connect methods (https requests)
             # queue appropriate response for client 
             # notifying about established connection
-            if self.request.method == "CONNECT":
-                self.client.queue(CRLF.join([
-                    'HTTP/1.1 200 Connection established',
-                    'Proxy-agent: proxy.py v%s' % __version__,
-                    CRLF
-                ]))
+            if self.request.method == b"CONNECT":
+                self.client.queue(self.connection_established_pkt)
             # for usual http requests, re-build request packet
             # and queue for the server with appropriate headers
             else:
                 self.server.queue(self.request.build(
-                    del_headers=['proxy-connection', 'connection', 'keep-alive'], 
-                    add_headers=[('Connection', 'Close')]
+                    del_headers=[b'proxy-connection', b'connection', b'keep-alive'], 
+                    add_headers=[(b'Connection', b'Close')]
                 ))
     
     def _process_response(self, data):
         # parse incoming response packet
         # only for non-https requests
-        if not self.request.method == "CONNECT":
+        if not self.request.method == b"CONNECT":
             self.response.parse(data)
         
         # queue data for client
@@ -368,11 +401,11 @@ class Proxy(multiprocessing.Process):
     
     def _access_log(self):
         host, port = self.server.addr if self.server else (None, None)
-        if self.request.method == "CONNECT":
+        if self.request.method == b"CONNECT":
             logger.info("%s:%s - %s %s:%s" % (self.client.addr[0], self.client.addr[1], self.request.method, host, port))
-        else:
+        elif self.request.method:
             logger.info("%s:%s - %s %s:%s%s - %s %s - %s bytes" % (self.client.addr[0], self.client.addr[1], self.request.method, host, port, self.request.build_url(), self.response.code, self.response.reason, len(self.response.raw)))
-    
+        
     def _get_waitable_lists(self):
         rlist, wlist, xlist = [self.client.conn], [], []
         logger.debug('*** watching client for read ready')
@@ -415,12 +448,12 @@ class Proxy(multiprocessing.Process):
             except ProxyConnectionFailed as e:
                 logger.exception(e)
                 self.client.queue(CRLF.join([
-                    'HTTP/1.1 502 Bad Gateway',
-                    'Proxy-agent: proxy.py v%s' % __version__,
-                    'Content-Length: 11',
-                    'Connection: close',
+                    b'HTTP/1.1 502 Bad Gateway',
+                    b'Proxy-agent: proxy.py v' + version,
+                    b'Content-Length: 11',
+                    b'Connection: close',
                     CRLF
-                ]) + 'Bad Gateway')
+                ]) + b'Bad Gateway')
                 self.client.flush()
                 return True
         
@@ -471,22 +504,20 @@ class Proxy(multiprocessing.Process):
             self._access_log()
             logger.debug('Closing proxy for connection %r at address %r' % (self.client.conn, self.client.addr))
 
-class Http(object):
-    """HTTP server implementation.
-    
-    Listens on configured (host, port) and spawns a new process
-    for handling every accepted HTTP connection. Spawned process
-    takes care of proxying the HTTP request.
-    """
+class TCP(object):
+    """TCP server implementation."""
     
     def __init__(self, hostname='127.0.0.1', port=8899, backlog=100):
         self.hostname = hostname
         self.port = port
         self.backlog = backlog
     
+    def handle(self, client):
+        raise NotImplementedError()
+    
     def run(self):
         try:
-            logger.info('Starting proxy server on port %d' % self.port)
+            logger.info('Starting server on port %d' % self.port)
             self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
             self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
             self.socket.bind((self.hostname, self.port))
@@ -495,33 +526,44 @@ class Http(object):
                 conn, addr = self.socket.accept()
                 logger.debug('Accepted connection %r at address %r' % (conn, addr))
                 client = Client(conn, addr)
-                proc = Proxy(client)
-                proc.daemon = True
-                proc.start()
-                logger.debug('Started process %r to handle connection %r' % (proc, conn))
+                self.handle(client)
         except Exception as e:
             logger.exception('Exception while running the server %r' % e)
         finally:
             logger.info('Closing server socket')
             self.socket.close()
 
+class HTTP(TCP):
+    """HTTP proxy server implementation.
+    
+    Spawns new process to proxy accepted client connection.
+    """
+    
+    def handle(self, client):
+        proc = Proxy(client)
+        proc.daemon = True
+        proc.start()
+        logger.debug('Started process %r to handle connection %r' % (proc, client.conn))
+
 def main():
     parser = argparse.ArgumentParser(
         description='proxy.py v%s' % __version__,
         epilog='Having difficulty using proxy.py? Report at: %s/issues/new' % __homepage__
     )
+    
     parser.add_argument('--hostname', default='127.0.0.1', help='Default: 127.0.0.1')
     parser.add_argument('--port', default='8899', help='Default: 8899')
     parser.add_argument('--log-level', default='INFO', help='DEBUG, INFO, WARNING, ERROR, CRITICAL')
     args = parser.parse_args()
     
+    logging.basicConfig(level=getattr(logging, args.log_level), format='%(asctime)s - %(levelname)s - pid:%(process)d - %(message)s')
+    
     hostname = args.hostname
     port = int(args.port)
-    logging.basicConfig(level=getattr(logging, args.log_level), format='%(asctime)s - %(process)d - %(message)s')
     
     try:
-        http = Http(hostname, port)
-        http.run()
+        proxy = HTTP(hostname, port)
+        proxy.run()
     except KeyboardInterrupt:
         pass
 


Test patch:
diff --git a/tests.py b/tests.py
new file mode 100644
index 0000000..dabd485
--- /dev/null
+++ b/tests.py
@@ -0,0 +1,329 @@
+import unittest
+import proxy
+from proxy import *
+from proxy import bytes_, text_
+
+
+class TestChunkParser(unittest.TestCase):
+
+    def setUp(self):
+        self.parser = ChunkParser()
+
+    def test_chunk_parse(self):
+        self.parser.parse(b''.join([
+            b'4\r\n',
+            b'Wiki\r\n',
+            b'5\r\n',
+            b'pedia\r\n',
+            b'E\r\n',
+            b' in\r\n\r\nchunks.\r\n',
+            b'0\r\n',
+            b'\r\n'
+        ]))
+        self.assertEqual(self.parser.chunk, b'')
+        self.assertEqual(self.parser.size, None)
+        self.assertEqual(self.parser.body, b'Wikipedia in\r\n\r\nchunks.')
+        self.assertEqual(self.parser.state, CHUNK_PARSER_STATE_COMPLETE)
+
+class TestHttpParser(unittest.TestCase):
+
+    def setUp(self):
+        self.parser = HttpParser()
+
+    def test_get_full_parse(self):
+        raw = text_(CRLF, encoding="utf-8").join([
+            "GET %s HTTP/1.1",
+            "Host: %s",
+            text_(CRLF, encoding="utf-8")
+        ])
+        self.parser.parse(bytes_(raw % ('https://example.com/path/dir/?a=b&c=d#p=q', 'example.com')))
+        self.assertEqual(self.parser.build_url(), b'/path/dir/?a=b&c=d#p=q')
+        self.assertEqual(self.parser.method, b"GET")
+        self.assertEqual(self.parser.url.hostname, b"example.com")
+        self.assertEqual(self.parser.url.port, None)
+        self.assertEqual(self.parser.version, b"HTTP/1.1")
+        self.assertEqual(self.parser.state, HTTP_PARSER_STATE_COMPLETE)
+        self.assertDictContainsSubset({b'host': (b'Host', b'example.com')}, self.parser.headers)
+        self.assertEqual(bytes_(raw % ('/path/dir/?a=b&c=d#p=q', 'example.com')), self.parser.build(del_headers=[b'host'], add_headers=[(b'Host', b'example.com')]))
+
+    def test_build_url_none(self):
+        self.assertEqual(self.parser.build_url(), b'/None')
+
+    def test_line_rcvd_to_rcving_headers_state_change(self):
+        self.parser.parse(b"GET http://localhost HTTP/1.1")
+        self.assertEqual(self.parser.state, HTTP_PARSER_STATE_INITIALIZED)
+        self.parser.parse(CRLF)
+        self.assertEqual(self.parser.state, HTTP_PARSER_STATE_LINE_RCVD)
+        self.parser.parse(CRLF)
+        self.assertEqual(self.parser.state, HTTP_PARSER_STATE_RCVING_HEADERS)
+
+    def test_get_partial_parse1(self):
+        self.parser.parse(CRLF.join([
+            b"GET http://localhost:8080 HTTP/1.1"
+        ]))
+        self.assertEqual(self.parser.method, None)
+        self.assertEqual(self.parser.url, None)
+        self.assertEqual(self.parser.version, None)
+        self.assertEqual(self.parser.state, HTTP_PARSER_STATE_INITIALIZED)
+
+        self.parser.parse(CRLF)
+        self.assertEqual(self.parser.method, b"GET")
+        self.assertEqual(self.parser.url.hostname, b"localhost")
+        self.assertEqual(self.parser.url.port, 8080)
+        self.assertEqual(self.parser.version, b"HTTP/1.1")
+        self.assertEqual(self.parser.state, HTTP_PARSER_STATE_LINE_RCVD)
+
+        self.parser.parse(b"Host: localhost:8080")
+        self.assertDictEqual(self.parser.headers, dict())
+        self.assertEqual(self.parser.buffer, b"Host: localhost:8080")
+        self.assertEqual(self.parser.state, HTTP_PARSER_STATE_LINE_RCVD)
+
+        self.parser.parse(CRLF*2)
+        self.assertDictContainsSubset({b'host':(b'Host', b'localhost:8080')}, self.parser.headers)
+        self.assertEqual(self.parser.state, HTTP_PARSER_STATE_COMPLETE)
+
+    def test_get_partial_parse2(self):
+        self.parser.parse(CRLF.join([
+            b"GET http://localhost:8080 HTTP/1.1",
+            b"Host: "
+        ]))
+        self.assertEqual(self.parser.method, b"GET")
+        self.assertEqual(self.parser.url.hostname, b"localhost")
+        self.assertEqual(self.parser.url.port, 8080)
+        self.assertEqual(self.parser.version, b"HTTP/1.1")
+        self.assertEqual(self.parser.buffer, b"Host: ")
+        self.assertEqual(self.parser.state, HTTP_PARSER_STATE_LINE_RCVD)
+
+        self.parser.parse(b"localhost:8080" + CRLF)
+        self.assertDictContainsSubset({b'host': (b'Host', b'localhost:8080')}, self.parser.headers)
+        self.assertEqual(self.parser.buffer, b"")
+        self.assertEqual(self.parser.state, HTTP_PARSER_STATE_RCVING_HEADERS)
+
+        self.parser.parse(b"Content-Type: text/plain" + CRLF)
+        self.assertEqual(self.parser.buffer, b"")
+        self.assertDictContainsSubset({b'content-type': (b'Content-Type', b'text/plain')}, self.parser.headers)
+        self.assertEqual(self.parser.state, HTTP_PARSER_STATE_RCVING_HEADERS)
+
+        self.parser.parse(CRLF)
+        self.assertEqual(self.parser.state, HTTP_PARSER_STATE_COMPLETE)
+
+    def test_post_full_parse(self):
+        raw = text_(CRLF).join([
+            "POST %s HTTP/1.1",
+            "Host: localhost",
+            "Content-Length: 7",
+            "Content-Type: application/x-www-form-urlencoded" + text_(CRLF, encoding="utf-8"),
+            "a=b&c=d"
+        ])
+        self.parser.parse(bytes_(raw % 'http://localhost'))
+        self.assertEqual(self.parser.method, b"POST")
+        self.assertEqual(self.parser.url.hostname, b"localhost")
+        self.assertEqual(self.parser.url.port, None)
+        self.assertEqual(self.parser.version, b"HTTP/1.1")
+        self.assertDictContainsSubset({b'content-type': (b'Content-Type', b'application/x-www-form-urlencoded')}, self.parser.headers)
+        self.assertDictContainsSubset({b'content-length': (b'Content-Length', b'7')}, self.parser.headers)
+        self.assertEqual(self.parser.body, b"a=b&c=d")
+        self.assertEqual(self.parser.buffer, b"")
+        self.assertEqual(self.parser.state, HTTP_PARSER_STATE_COMPLETE)
+        self.assertEqual(len(self.parser.build()), len(raw % '/'))
+
+    def test_post_partial_parse(self):
+        self.parser.parse(CRLF.join([
+            b"POST http://localhost HTTP/1.1",
+            b"Host: localhost",
+            b"Content-Length: 7",
+            b"Content-Type: application/x-www-form-urlencoded"
+        ]))
+        self.assertEqual(self.parser.method, b"POST")
+        self.assertEqual(self.parser.url.hostname, b"localhost")
+        self.assertEqual(self.parser.url.port, None)
+        self.assertEqual(self.parser.version, b"HTTP/1.1")
+        self.assertEqual(self.parser.state, HTTP_PARSER_STATE_RCVING_HEADERS)
+
+        self.parser.parse(CRLF)
+        self.assertEqual(self.parser.state, HTTP_PARSER_STATE_RCVING_HEADERS)
+
+        self.parser.parse(CRLF)
+        self.assertEqual(self.parser.state, HTTP_PARSER_STATE_HEADERS_COMPLETE)
+
+        self.parser.parse(b"a=b")
+        self.assertEqual(self.parser.state, HTTP_PARSER_STATE_RCVING_BODY)
+        self.assertEqual(self.parser.body, b"a=b")
+        self.assertEqual(self.parser.buffer, b"")
+
+        self.parser.parse(b"&c=d")
+        self.assertEqual(self.parser.state, HTTP_PARSER_STATE_COMPLETE)
+        self.assertEqual(self.parser.body, b"a=b&c=d")
+        self.assertEqual(self.parser.buffer, b"")
+
+    def test_response_parse(self):
+        self.parser.type = HTTP_RESPONSE_PARSER
+        self.parser.parse(b''.join([
+            b'HTTP/1.1 301 Moved Permanently\r\n',
+            b'Location: http://www.google.com/\r\n',
+            b'Content-Type: text/html; charset=UTF-8\r\n',
+            b'Date: Wed, 22 May 2013 14:07:29 GMT\r\n',
+            b'Expires: Fri, 21 Jun 2013 14:07:29 GMT\r\n',
+            b'Cache-Control: public, max-age=2592000\r\n',
+            b'Server: gws\r\n',
+            b'Content-Length: 219\r\n',
+            b'X-XSS-Protection: 1; mode=block\r\n',
+            b'X-Frame-Options: SAMEORIGIN\r\n\r\n',
+            b'<HTML><HEAD><meta http-equiv="content-type" content="text/html;charset=utf-8">\n<TITLE>301 Moved</TITLE></HEAD>',
+            b'<BODY>\n<H1>301 Moved</H1>\nThe document has moved\n<A HREF="http://www.google.com/">here</A>.\r\n</BODY></HTML>\r\n'
+        ]))
+        self.assertEqual(self.parser.code, b'301')
+        self.assertEqual(self.parser.reason, b'Moved Permanently')
+        self.assertEqual(self.parser.version, b'HTTP/1.1')
+        self.assertEqual(self.parser.body, b'<HTML><HEAD><meta http-equiv="content-type" content="text/html;charset=utf-8">\n<TITLE>301 Moved</TITLE></HEAD><BODY>\n<H1>301 Moved</H1>\nThe document has moved\n<A HREF="http://www.google.com/">here</A>.\r\n</BODY></HTML>\r\n')
+        self.assertDictContainsSubset({b'content-length': (b'Content-Length', b'219')}, self.parser.headers)
+        self.assertEqual(self.parser.state, HTTP_PARSER_STATE_COMPLETE)
+
+    def test_response_partial_parse(self):
+        self.parser.type = HTTP_RESPONSE_PARSER
+        self.parser.parse(b''.join([
+            b'HTTP/1.1 301 Moved Permanently\r\n',
+            b'Location: http://www.google.com/\r\n',
+            b'Content-Type: text/html; charset=UTF-8\r\n',
+            b'Date: Wed, 22 May 2013 14:07:29 GMT\r\n',
+            b'Expires: Fri, 21 Jun 2013 14:07:29 GMT\r\n',
+            b'Cache-Control: public, max-age=2592000\r\n',
+            b'Server: gws\r\n',
+            b'Content-Length: 219\r\n',
+            b'X-XSS-Protection: 1; mode=block\r\n',
+            b'X-Frame-Options: SAMEORIGIN\r\n'
+        ]))
+        self.assertDictContainsSubset({b'x-frame-options': (b'X-Frame-Options', b'SAMEORIGIN')}, self.parser.headers)
+        self.assertEqual(self.parser.state, HTTP_PARSER_STATE_RCVING_HEADERS)
+        self.parser.parse(b'\r\n')
+        self.assertEqual(self.parser.state, HTTP_PARSER_STATE_HEADERS_COMPLETE)
+        self.parser.parse(b'<HTML><HEAD><meta http-equiv="content-type" content="text/html;charset=utf-8">\n<TITLE>301 Moved</TITLE></HEAD>')
+        self.assertEqual(self.parser.state, HTTP_PARSER_STATE_RCVING_BODY)
+        self.parser.parse(b'<BODY>\n<H1>301 Moved</H1>\nThe document has moved\n<A HREF="http://www.google.com/">here</A>.\r\n</BODY></HTML>\r\n')
+        self.assertEqual(self.parser.state, HTTP_PARSER_STATE_COMPLETE)
+
+    def test_chunked_response_parse(self):
+        self.parser.type = HTTP_RESPONSE_PARSER
+        self.parser.parse(b''.join([
+            b'HTTP/1.1 200 OK\r\n',
+            b'Content-Type: application/json\r\n',
+            b'Date: Wed, 22 May 2013 15:08:15 GMT\r\n',
+            b'Server: gunicorn/0.16.1\r\n',
+            b'transfer-encoding: chunked\r\n',
+            b'Connection: keep-alive\r\n\r\n',
+            b'4\r\n',
+            b'Wiki\r\n',
+            b'5\r\n',
+            b'pedia\r\n',
+            b'E\r\n',
+            b' in\r\n\r\nchunks.\r\n',
+            b'0\r\n',
+            b'\r\n'
+        ]))
+        self.assertEqual(self.parser.body, b'Wikipedia in\r\n\r\nchunks.')
+        self.assertEqual(self.parser.state, HTTP_PARSER_STATE_COMPLETE)
+
+class MockConnection(object):
+    
+    def __init__(self, buffer=b''):
+        self.buffer = buffer
+    
+    def recv(self, bytes=8192):
+        data = self.buffer[:bytes]
+        self.buffer = self.buffer[bytes:]
+        return data
+    
+    def send(self, data):
+        return len(data)
+    
+    def queue(self, data):
+        self.buffer += data
+
+class TestProxy(unittest.TestCase):
+
+    def setUp(self):
+        self._conn = MockConnection()
+        self._addr = ('127.0.0.1', 54382)
+        self.proxy = Proxy(Client(self._conn, self._addr))
+
+    def test_http_get(self):
+        self.proxy.client.conn.queue(b"GET http://httpbin.org/get HTTP/1.1" + CRLF)
+        self.proxy._process_request(self.proxy.client.recv())
+        self.assertNotEqual(self.proxy.request.state, HTTP_PARSER_STATE_COMPLETE)
+
+        self.proxy.client.conn.queue(CRLF.join([
+            b"User-Agent: curl/7.27.0",
+            b"Host: httpbin.org",
+            b"Accept: */*",
+            b"Proxy-Connection: Keep-Alive",
+            CRLF
+        ]))
+        
+        self.proxy._process_request(self.proxy.client.recv())
+        self.assertEqual(self.proxy.request.state, HTTP_PARSER_STATE_COMPLETE)
+        self.assertEqual(self.proxy.server.addr, (b"httpbin.org", 80))
+        
+        self.proxy.server.flush()
+        self.assertEqual(self.proxy.server.buffer_size(), 0)
+        
+        data = self.proxy.server.recv()
+        while data:
+            self.proxy._process_response(data)
+            if self.proxy.response.state == HTTP_PARSER_STATE_COMPLETE:
+                break
+            data = self.proxy.server.recv()
+
+        self.assertEqual(self.proxy.response.state, HTTP_PARSER_STATE_COMPLETE)
+        self.assertEqual(int(self.proxy.response.code), 200)
+
+    def test_https_get(self):
+        self.proxy.client.conn.queue(CRLF.join([
+            b"CONNECT httpbin.org:80 HTTP/1.1",
+            b"Host: httpbin.org:80",
+            b"User-Agent: curl/7.27.0",
+            b"Proxy-Connection: Keep-Alive",
+            CRLF
+        ]))
+        self.proxy._process_request(self.proxy.client.recv())
+        self.assertFalse(self.proxy.server == None)
+        self.assertEqual(self.proxy.client.buffer, self.proxy.connection_established_pkt)
+        
+        parser = HttpParser(HTTP_RESPONSE_PARSER)
+        parser.parse(self.proxy.client.buffer)
+        self.assertEqual(parser.state, HTTP_PARSER_STATE_HEADERS_COMPLETE)
+        self.assertEqual(int(parser.code), 200)
+        
+        self.proxy.client.flush()
+        self.assertEqual(self.proxy.client.buffer_size(), 0)
+        
+        self.proxy.client.conn.queue(CRLF.join([
+            b"GET /user-agent HTTP/1.1",
+            b"Host: httpbin.org",
+            b"User-Agent: curl/7.27.0",
+            CRLF
+        ]))
+        self.proxy._process_request(self.proxy.client.recv())
+        self.proxy.server.flush()
+        self.assertEqual(self.proxy.server.buffer_size(), 0)
+
+        parser = HttpParser(HTTP_RESPONSE_PARSER)
+        data = self.proxy.server.recv()
+        while data:
+            parser.parse(data)
+            if parser.state == HTTP_PARSER_STATE_COMPLETE:
+                break
+            data = self.proxy.server.recv()
+
+        self.assertEqual(parser.state, HTTP_PARSER_STATE_COMPLETE)
+        self.assertEqual(int(parser.code), 200)
+
+    def test_proxy_connection_failed(self):
+        with self.assertRaises(ProxyConnectionFailed):
+            self.proxy._process_request(CRLF.join([
+                b"GET http://unknown.domain HTTP/1.1",
+                b"Host: unknown.domain",
+                CRLF
+            ]))
+
+if __name__ == '__main__':
+    unittest.main()