Repository: abhinavsingh/proxy.py
Pull request #1065 (2022-01-26 11:22:42)

[Jupyter] Request creation notebook
- Also added missing tests to the plugin
- `build_http_request` utility changes:
  - Now always adds `User-Agent` header, unless `no_ua` or a custom header has not already been provided
  - Now adds/update the `Content-Length` header if body is present


Patch:
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index aa09dfa..7b7eb48 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -127,7 +127,7 @@ repos:
   - id: codespell
     exclude: >
       (?x)^(
-        tutorial/responses.ipynb|
+        ^.+\.ipynb$|
         tests/http/test_responses\.py|
         ^.+\.min\.js$
       )$
diff --git a/README.md b/README.md
index 234c915..e38d107 100644
--- a/README.md
+++ b/README.md
@@ -2297,7 +2297,7 @@ usage: -m [-h] [--tunnel-hostname TUNNEL_HOSTNAME] [--tunnel-port TUNNEL_PORT]
           [--filtered-client-ips FILTERED_CLIENT_IPS]
           [--filtered-url-regex-config FILTERED_URL_REGEX_CONFIG]
 
-proxy.py v2.4.0rc8.dev17+g59a4335.d20220123
+proxy.py v2.4.0rc9.dev8+gea0253d.d20220126
 
 options:
   -h, --help            show this help message and exit
diff --git a/proxy/common/utils.py b/proxy/common/utils.py
index cb2cb1d..d157775 100644
--- a/proxy/common/utils.py
+++ b/proxy/common/utils.py
@@ -26,7 +26,7 @@ from typing import Any, Dict, List, Type, Tuple, Callable, Optional
 from .types import HostPort
 from .constants import (
     CRLF, COLON, HTTP_1_1, IS_WINDOWS, WHITESPACE, DEFAULT_TIMEOUT,
-    DEFAULT_THREADLESS,
+    DEFAULT_THREADLESS, PROXY_AGENT_HEADER_VALUE,
 )
 
 
@@ -84,14 +84,30 @@ def bytes_(s: Any, encoding: str = 'utf-8', errors: str = 'strict') -> Any:
 def build_http_request(
     method: bytes, url: bytes,
     protocol_version: bytes = HTTP_1_1,
+    content_type: Optional[bytes] = None,
     headers: Optional[Dict[bytes, bytes]] = None,
     body: Optional[bytes] = None,
     conn_close: bool = False,
+    no_ua: bool = False,
 ) -> bytes:
     """Build and returns a HTTP request packet."""
+    headers = headers or {}
+    if content_type is not None:
+        headers[b'Content-Type'] = content_type
+    has_transfer_encoding = False
+    has_user_agent = False
+    for k, _ in headers.items():
+        if k.lower() == b'transfer-encoding':
+            has_transfer_encoding = True
+        elif k.lower() == b'user-agent':
+            has_user_agent = True
+    if body and not has_transfer_encoding:
+        headers[b'Content-Length'] = bytes_(len(body))
+    if not has_user_agent and not no_ua:
+        headers[b'User-Agent'] = PROXY_AGENT_HEADER_VALUE
     return build_http_pkt(
         [method, url, protocol_version],
-        headers or {},
+        headers,
         body,
         conn_close,
     )
@@ -109,8 +125,7 @@ def build_http_response(
     line = [protocol_version, bytes_(status_code)]
     if reason:
         line.append(reason)
-    if headers is None:
-        headers = {}
+    headers = headers or {}
     has_transfer_encoding = False
     for k, _ in headers.items():
         if k.lower() == b'transfer-encoding':
diff --git a/proxy/http/parser/parser.py b/proxy/http/parser/parser.py
index 7d29f76..4877667 100644
--- a/proxy/http/parser/parser.py
+++ b/proxy/http/parser/parser.py
@@ -270,6 +270,7 @@ class HttpParser:
                 k.lower() not in disable_headers
             },
             body=body,
+            no_ua=True,
         )
 
     def build_response(self) -> bytes:
diff --git a/proxy/plugin/__init__.py b/proxy/plugin/__init__.py
index e76a253..f75dcbb 100644
--- a/proxy/plugin/__init__.py
+++ b/proxy/plugin/__init__.py
@@ -11,6 +11,8 @@
     .. spelling::
 
        Cloudflare
+       ws
+       onmessage
 """
 from .cache import CacheResponsesPlugin, BaseCacheResponsesPlugin
 from .shortlink import ShortLinkPlugin
diff --git a/proxy/plugin/modify_post_data.py b/proxy/plugin/modify_post_data.py
index d4b5ba6..a290ed5 100644
--- a/proxy/plugin/modify_post_data.py
+++ b/proxy/plugin/modify_post_data.py
@@ -12,7 +12,7 @@ from typing import Optional
 
 from ..http import httpMethods
 from ..http.proxy import HttpProxyBasePlugin
-from ..http.parser import HttpParser
+from ..http.parser import HttpParser, ChunkParser
 from ..common.utils import bytes_
 
 
@@ -30,14 +30,20 @@ class ModifyPostDataPlugin(HttpProxyBasePlugin):
             self, request: HttpParser,
     ) -> Optional[HttpParser]:
         if request.method == httpMethods.POST:
-            request.body = ModifyPostDataPlugin.MODIFIED_BODY
-            # Update Content-Length header only when request is NOT chunked
-            # encoded
-            if not request.is_chunked_encoded:
+            # If request data is compressed, compress the body too
+            body = ModifyPostDataPlugin.MODIFIED_BODY
+            # If the request is of type chunked encoding
+            # add post data as chunk
+            if request.is_chunked_encoded:
+                body = ChunkParser.to_chunks(
+                    ModifyPostDataPlugin.MODIFIED_BODY,
+                )
+            else:
                 request.add_header(
                     b'Content-Length',
-                    bytes_(len(request.body)),
+                    bytes_(len(body)),
                 )
+            request.body = body
             # Enforce content-type json
             if request.has_header(b'Content-Type'):
                 request.del_header(b'Content-Type')
diff --git a/proxy/plugin/web_server_route.py b/proxy/plugin/web_server_route.py
index 205a8f9..3b927cd 100644
--- a/proxy/plugin/web_server_route.py
+++ b/proxy/plugin/web_server_route.py
@@ -7,6 +7,11 @@
 
     :copyright: (c) 2013-present by Abhinav Singh and contributors.
     :license: BSD, see LICENSE for more details.
+
+    .. spelling::
+
+       ws
+       onmessage
 """
 import logging
 from typing import List, Tuple
@@ -14,6 +19,7 @@ from typing import List, Tuple
 from ..http.parser import HttpParser
 from ..http.server import HttpWebServerBasePlugin, httpProtocolTypes
 from ..http.responses import okResponse
+from ..http.websocket.frame import WebsocketFrame
 
 
 logger = logging.getLogger(__name__)
@@ -37,3 +43,15 @@ class WebServerPlugin(HttpWebServerBasePlugin):
             self.client.queue(HTTP_RESPONSE)
         elif request.path == b'/https-route-example':
             self.client.queue(HTTPS_RESPONSE)
+
+    def on_websocket_message(self, frame: WebsocketFrame) -> None:
+        """Open chrome devtools and try using following commands:
+
+        Example:
+
+            ws = new WebSocket("ws://localhost:8899/ws-route-example")
+            ws.onmessage = function(m) { console.log(m); }
+            ws.send('hello')
+
+        """
+        self.client.queue(memoryview(WebsocketFrame.text(frame.data or b'')))
diff --git a/tutorial/requests.ipynb b/tutorial/requests.ipynb
new file mode 100644
index 0000000..fb2cc36
--- /dev/null
+++ b/tutorial/requests.ipynb
@@ -0,0 +1,273 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Http Requests\n",
+    "\n",
+    "## Usage\n",
+    "\n",
+    "To construct a HTTP request packet you have a variety of facilities available.\n",
+    "\n",
+    "Previously we saw how to parse HTTP responses using `HttpParser`.  We also saw how `HttpParser` class is capable of parsing various type of HTTP protocols.  Remember the _take away_ from that tutorial."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "b'GET / HTTP/1.1\\r\\nHost: jaxl.com\\r\\nUser-Agent: proxy.py v2.4.0rc9.dev8+gea0253d.d20220126\\r\\n\\r\\n'\n"
+     ]
+    }
+   ],
+   "source": [
+    "from proxy.http.parser import HttpParser, httpParserTypes\n",
+    "from proxy.http import httpMethods\n",
+    "from proxy.common.utils import HTTP_1_1\n",
+    "\n",
+    "request = HttpParser(httpParserTypes.REQUEST_PARSER)\n",
+    "request.path, request.method, request.version = b'/', httpMethods.GET, HTTP_1_1\n",
+    "request.add_header(b'Host', b'jaxl.com')\n",
+    "\n",
+    "print(request.build())"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "But, this is a painful way to construct request packets.  Hence, other high level abstractions are available.\n",
+    "\n",
+    "Example, following one liner will give us the same request packet."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "b'GET / HTTP/1.1\\r\\nHost: jaxl.com\\r\\nUser-Agent: proxy.py v2.4.0rc9.dev8+gea0253d.d20220126\\r\\n\\r\\n'"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from proxy.common.utils import build_http_request\n",
+    "\n",
+    "build_http_request(\n",
+    "    method=httpMethods.GET,\n",
+    "    url=b'/',\n",
+    "    headers={b'Host': b'jaxl.com'},\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "`build_http_request` ensures a `User-Agent` header.  You can provide your own too:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "b'GET / HTTP/1.1\\r\\nHost: jaxl.com\\r\\nUser-Agent: my app v1\\r\\n\\r\\n'"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "build_http_request(\n",
+    "    method=httpMethods.GET,\n",
+    "    url=b'/',\n",
+    "    headers={\n",
+    "        b'Host': b'jaxl.com',\n",
+    "        b'User-Agent': b'my app v1'\n",
+    "    },\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Or, if you don't want a `User-Agent` header at all"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "b'GET / HTTP/1.1\\r\\nHost: jaxl.com\\r\\n\\r\\n'"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "build_http_request(\n",
+    "    method=httpMethods.GET,\n",
+    "    url=b'/',\n",
+    "    headers={b'Host': b'jaxl.com'},\n",
+    "    no_ua=True,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "To add a connection close header, simply pass `conn_close=True`"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "b'GET / HTTP/1.1\\r\\nHost: jaxl.com\\r\\nUser-Agent: proxy.py v2.4.0rc9.dev8+gea0253d.d20220126\\r\\nConnection: close\\r\\n\\r\\n'"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "build_http_request(\n",
+    "    method=httpMethods.GET,\n",
+    "    url=b'/',\n",
+    "    headers={b'Host': b'jaxl.com'},\n",
+    "    conn_close=True,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "For `POST` requests, provide the `body` attribute"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "b'POST / HTTP/1.1\\r\\nHost: jaxl.com\\r\\nContent-Type: application/x-www-form-urlencoded\\r\\nContent-Length: 21\\r\\nUser-Agent: proxy.py v2.4.0rc9.dev8+gea0253d.d20220126\\r\\nConnection: close\\r\\n\\r\\nkey=value&hello=world'"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "build_http_request(\n",
+    "    method=httpMethods.POST,\n",
+    "    url=b'/',\n",
+    "    headers={b'Host': b'jaxl.com'},\n",
+    "    body=b'key=value&hello=world',\n",
+    "    content_type=b'application/x-www-form-urlencoded',\n",
+    "    conn_close=True,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "For chunked data, simply include a `Transfer-Encoding` header.  This will omit the `Content-Length` header then:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "b'POST / HTTP/1.1\\r\\nHost: jaxl.com\\r\\nTransfer-Encoding: chunked\\r\\nContent-Type: application/x-www-form-urlencoded\\r\\nUser-Agent: proxy.py v2.4.0rc9.dev8+gea0253d.d20220126\\r\\nConnection: close\\r\\n\\r\\n5\\r\\nkey=v\\r\\n5\\r\\nalue&\\r\\n5\\r\\nhello\\r\\n5\\r\\n=worl\\r\\n1\\r\\nd\\r\\n0\\r\\n\\r\\n'"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from proxy.http.parser import ChunkParser\n",
+    "\n",
+    "build_http_request(\n",
+    "    method=httpMethods.POST,\n",
+    "    url=b'/',\n",
+    "    headers={\n",
+    "        b'Host': b'jaxl.com',\n",
+    "        b'Transfer-Encoding': b'chunked',\n",
+    "    },\n",
+    "    body=ChunkParser.to_chunks(b'key=value&hello=world', chunk_size=5),\n",
+    "    content_type=b'application/x-www-form-urlencoded',\n",
+    "    conn_close=True,\n",
+    ")"
+   ]
+  }
+ ],
+ "metadata": {
+  "interpreter": {
+   "hash": "da9d6927d62b2b95bde149eedfbd5367cb7f465aad65a736f49c99ee3db39df7"
+  },
+  "kernelspec": {
+   "display_name": "Python 3.10.0 64-bit ('venv310': venv)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.0"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/tutorial/responses.ipynb b/tutorial/responses.ipynb
index 7b8de55..f74f2eb 100644
--- a/tutorial/responses.ipynb
+++ b/tutorial/responses.ipynb
@@ -8,9 +8,9 @@
     "\n",
     "## Usage\n",
     "\n",
-    "To construct a response packet you have a variety of facilities available.  We previously experienced how to parse HTTP responses using `HttpParser`.  Of-course, we can also construct a response packet using `HttpParser` class.\n",
+    "To construct a response packet you have a variety of facilities available.\n",
     "\n",
-    "Let's construct a HTTP response packet:"
+    "Previously we saw how to parse HTTP responses using `HttpParser`.  Of-course, we can also construct a response packet using `HttpParser` class."
    ]
   },
   {
@@ -42,14 +42,14 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "But it is a little painful to construct responses like above.  Hence, other high level abstractions are available.\n",
+    "But, this is a painful way to construct responses.  Hence, other high level abstractions are available.\n",
     "\n",
     "Example, following one liner will give us the same response packet."
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [
     {


Test patch:
diff --git a/tests/http/parser/test_http_parser.py b/tests/http/parser/test_http_parser.py
index 95c052d..427553c 100644
--- a/tests/http/parser/test_http_parser.py
+++ b/tests/http/parser/test_http_parser.py
@@ -124,7 +124,7 @@ class TestHttpParser(unittest.TestCase):
     def test_build_request(self) -> None:
         self.assertEqual(
             build_http_request(
-                b'GET', b'http://localhost:12345', b'HTTP/1.1',
+                b'GET', b'http://localhost:12345', b'HTTP/1.1', no_ua=True,
             ),
             CRLF.join([
                 b'GET http://localhost:12345 HTTP/1.1',
@@ -135,6 +135,7 @@ class TestHttpParser(unittest.TestCase):
             build_http_request(
                 b'GET', b'http://localhost:12345', b'HTTP/1.1',
                 headers={b'key': b'value'},
+                no_ua=True,
             ),
             CRLF.join([
                 b'GET http://localhost:12345 HTTP/1.1',
@@ -147,10 +148,12 @@ class TestHttpParser(unittest.TestCase):
                 b'GET', b'http://localhost:12345', b'HTTP/1.1',
                 headers={b'key': b'value'},
                 body=b'Hello from py',
+                no_ua=True,
             ),
             CRLF.join([
                 b'GET http://localhost:12345 HTTP/1.1',
                 b'key: value',
+                b'Content-Length: 13',
                 CRLF,
             ]) + b'Hello from py',
         )
diff --git a/tests/http/proxy/test_http_proxy_tls_interception.py b/tests/http/proxy/test_http_proxy_tls_interception.py
index 800cfb8..b3734ca 100644
--- a/tests/http/proxy/test_http_proxy_tls_interception.py
+++ b/tests/http/proxy/test_http_proxy_tls_interception.py
@@ -119,6 +119,7 @@ class TestHttpProxyTlsInterception(Assertions):
         connect_request = build_http_request(
             httpMethods.CONNECT, bytes_(netloc),
             headers=headers,
+            no_ua=True,
         )
         self._conn.recv.return_value = connect_request
         get_request = build_http_request(
diff --git a/tests/plugin/test_http_proxy_plugins.py b/tests/plugin/test_http_proxy_plugins.py
index 1f9d43d..1a04c28 100644
--- a/tests/plugin/test_http_proxy_plugins.py
+++ b/tests/plugin/test_http_proxy_plugins.py
@@ -28,6 +28,9 @@ from proxy.http.proxy import HttpProxyPlugin
 from proxy.common.flag import FlagParser
 from proxy.http.parser import HttpParser, httpParserTypes
 from proxy.common.utils import bytes_, build_http_request, build_http_response
+from proxy.http.responses import (
+    NOT_FOUND_RESPONSE_PKT, PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT,
+)
 from proxy.common.constants import DEFAULT_HTTP_PORT, PROXY_AGENT_HEADER_VALUE
 from .utils import get_plugin_by_test_name
 from ..test_assertions import Assertions
@@ -90,6 +93,7 @@ class TestHttpProxyPluginExamples(Assertions):
                 b'Content-Length': bytes_(len(original)),
             },
             body=original,
+            no_ua=True,
         )
         self.mock_selector.return_value.select.side_effect = [
             [(
@@ -117,6 +121,7 @@ class TestHttpProxyPluginExamples(Assertions):
                     b'Via': b'1.1 %s' % PROXY_AGENT_HEADER_VALUE,
                 },
                 body=modified,
+                no_ua=True,
             ),
         )
 
@@ -186,6 +191,7 @@ class TestHttpProxyPluginExamples(Assertions):
             headers={
                 b'Host': b'example.org',
             },
+            no_ua=True,
         )
         self._conn.recv.return_value = request
         self.mock_selector.return_value.select.side_effect = [
@@ -212,9 +218,44 @@ class TestHttpProxyPluginExamples(Assertions):
                     b'Host': upstream.netloc,
                     b'Via': b'1.1 %s' % PROXY_AGENT_HEADER_VALUE,
                 },
+                no_ua=True,
             ),
         )
 
+    @pytest.mark.asyncio    # type: ignore[misc]
+    @pytest.mark.parametrize(
+        "_setUp",
+        (
+            ('test_redirect_to_custom_server_plugin'),
+        ),
+        indirect=True,
+    )   # type: ignore[misc]
+    async def test_redirect_to_custom_server_plugin_skips_https(self) -> None:
+        request = build_http_request(
+            b'CONNECT', b'jaxl.com:443',
+            headers={
+                b'Host': b'jaxl.com:443',
+            },
+        )
+        self._conn.recv.return_value = request
+        self.mock_selector.return_value.select.side_effect = [
+            [(
+                selectors.SelectorKey(
+                    fileobj=self._conn.fileno(),
+                    fd=self._conn.fileno(),
+                    events=selectors.EVENT_READ,
+                    data=None,
+                ),
+                selectors.EVENT_READ,
+            )],
+        ]
+        await self.protocol_handler._run_once()
+        self.mock_server_conn.assert_called_with('jaxl.com', 443)
+        self.assertEqual(
+            self.protocol_handler.work.buffer[0].tobytes(),
+            PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT,
+        )
+
     @pytest.mark.asyncio    # type: ignore[misc]
     @pytest.mark.parametrize(
         "_setUp",
@@ -268,6 +309,7 @@ class TestHttpProxyPluginExamples(Assertions):
             headers={
                 b'Host': b'super.secure',
             },
+            no_ua=True,
         )
         self._conn.recv.return_value = request
 
@@ -326,8 +368,12 @@ class TestHttpProxyPluginExamples(Assertions):
                     b'Host': b'super.secure',
                     b'Via': b'1.1 %s' % PROXY_AGENT_HEADER_VALUE,
                 },
+                no_ua=True,
             )
-        server.queue.assert_called_once_with(queued_request)
+        server.queue.assert_called_once()
+        print(server.queue.call_args_list[0][0][0].tobytes())
+        print(queued_request)
+        self.assertEqual(server.queue.call_args_list[0][0][0], queued_request)
 
         # Server write
         await self.protocol_handler._run_once()
@@ -386,3 +432,122 @@ class TestHttpProxyPluginExamples(Assertions):
                 conn_close=True,
             ),
         )
+
+    @pytest.mark.asyncio    # type: ignore[misc]
+    @pytest.mark.parametrize(
+        "_setUp",
+        (
+            ('test_shortlink_plugin'),
+        ),
+        indirect=True,
+    )   # type: ignore[misc]
+    async def test_shortlink_plugin(self) -> None:
+        request = build_http_request(
+            b'GET', b'http://t/',
+            headers={
+                b'Host': b't',
+            },
+        )
+        self._conn.recv.return_value = request
+
+        self.mock_selector.return_value.select.side_effect = [
+            [(
+                selectors.SelectorKey(
+                    fileobj=self._conn.fileno(),
+                    fd=self._conn.fileno(),
+                    events=selectors.EVENT_READ,
+                    data=None,
+                ),
+                selectors.EVENT_READ,
+            )],
+        ]
+
+        await self.protocol_handler._run_once()
+        self.assertEqual(
+            self.protocol_handler.work.buffer[0].tobytes(),
+            build_http_response(
+                status_code=httpStatusCodes.SEE_OTHER,
+                reason=b'See Other',
+                headers={
+                    b'Location': b'http://twitter.com/',
+                },
+                conn_close=True,
+            ),
+        )
+
+    @pytest.mark.asyncio    # type: ignore[misc]
+    @pytest.mark.parametrize(
+        "_setUp",
+        (
+            ('test_shortlink_plugin'),
+        ),
+        indirect=True,
+    )   # type: ignore[misc]
+    async def test_shortlink_plugin_unknown(self) -> None:
+        request = build_http_request(
+            b'GET', b'http://unknown/',
+            headers={
+                b'Host': b'unknown',
+            },
+        )
+        self._conn.recv.return_value = request
+
+        self.mock_selector.return_value.select.side_effect = [
+            [(
+                selectors.SelectorKey(
+                    fileobj=self._conn.fileno(),
+                    fd=self._conn.fileno(),
+                    events=selectors.EVENT_READ,
+                    data=None,
+                ),
+                selectors.EVENT_READ,
+            )],
+        ]
+        await self.protocol_handler._run_once()
+        self.assertEqual(
+            self.protocol_handler.work.buffer[0].tobytes(),
+            NOT_FOUND_RESPONSE_PKT,
+        )
+
+    @pytest.mark.asyncio    # type: ignore[misc]
+    @pytest.mark.parametrize(
+        "_setUp",
+        (
+            ('test_shortlink_plugin'),
+        ),
+        indirect=True,
+    )   # type: ignore[misc]
+    async def test_shortlink_plugin_external(self) -> None:
+        request = build_http_request(
+            b'GET', b'http://jaxl.com/',
+            headers={
+                b'Host': b'jaxl.com',
+            },
+            no_ua=True,
+        )
+        self._conn.recv.return_value = request
+
+        self.mock_selector.return_value.select.side_effect = [
+            [(
+                selectors.SelectorKey(
+                    fileobj=self._conn.fileno(),
+                    fd=self._conn.fileno(),
+                    events=selectors.EVENT_READ,
+                    data=None,
+                ),
+                selectors.EVENT_READ,
+            )],
+        ]
+        await self.protocol_handler._run_once()
+        self.mock_server_conn.assert_called_once_with('jaxl.com', 80)
+        self.mock_server_conn.return_value.queue.assert_called_with(
+            build_http_request(
+                b'GET', b'/',
+                headers={
+                    b'Host': b'jaxl.com',
+                    b'Via': b'1.1 %s' % PROXY_AGENT_HEADER_VALUE,
+                },
+                no_ua=True,
+            ),
+        )
+        self.assertFalse(self.protocol_handler.work.has_buffer())
diff --git a/tests/plugin/test_http_proxy_plugins_with_tls_interception.py b/tests/plugin/test_http_proxy_plugins_with_tls_interception.py
index d48cccd..64dd944 100644
--- a/tests/plugin/test_http_proxy_plugins_with_tls_interception.py
+++ b/tests/plugin/test_http_proxy_plugins_with_tls_interception.py
@@ -152,7 +152,9 @@ class TestHttpProxyPluginExamplesWithTlsInterception(Assertions):
 
         self._conn.send.side_effect = send
         self._conn.recv.return_value = build_http_request(
-            httpMethods.CONNECT, b'uni.corn:443',
+            httpMethods.CONNECT,
+            b'uni.corn:443',
+            no_ua=True,
         )
 
     @pytest.mark.asyncio    # type: ignore[misc]
@@ -193,6 +195,7 @@ class TestHttpProxyPluginExamplesWithTlsInterception(Assertions):
                 b'Content-Type': b'application/x-www-form-urlencoded',
             },
             body=original,
+            no_ua=True,
         )
         await self.protocol_handler._run_once()
         self.server.queue.assert_called_once()
@@ -242,6 +245,7 @@ class TestHttpProxyPluginExamplesWithTlsInterception(Assertions):
             headers={
                 b'Host': b'uni.corn',
             },
+            no_ua=True,
         )
         self.client_ssl_connection.recv.return_value = request
 
diff --git a/tests/plugin/utils.py b/tests/plugin/utils.py
index 7e63350..400e874 100644
--- a/tests/plugin/utils.py
+++ b/tests/plugin/utils.py
@@ -11,9 +11,9 @@
 from typing import Type
 
 from proxy.plugin import (
-    CacheResponsesPlugin, ManInTheMiddlePlugin, ModifyPostDataPlugin,
-    ProposedRestApiPlugin, FilterByURLRegexPlugin, FilterByUpstreamHostPlugin,
-    RedirectToCustomServerPlugin,
+    ShortLinkPlugin, CacheResponsesPlugin, ManInTheMiddlePlugin,
+    ModifyPostDataPlugin, ProposedRestApiPlugin, FilterByURLRegexPlugin,
+    FilterByUpstreamHostPlugin, RedirectToCustomServerPlugin,
 )
 from proxy.http.proxy import HttpProxyBasePlugin
 
@@ -34,4 +34,6 @@ def get_plugin_by_test_name(test_name: str) -> Type[HttpProxyBasePlugin]:
         plugin = ManInTheMiddlePlugin
     elif test_name == 'test_filter_by_url_regex_plugin':
         plugin = FilterByURLRegexPlugin
+    elif test_name == 'test_shortlink_plugin':
+        plugin = ShortLinkPlugin
     return plugin