Repository: matrix-org/synapse

Issue:
HTTP HEAD and GET requests return different headers for media/r0/download endpoints
### Description

Synapse returns different headers for HTTP HEAD and GET requests, for media/r0/download endpoints.

According to HTTP/1.1 spec https://www.w3.org/Protocols/rfc2616/rfc2616-sec9.html section 9.4 HEAD:

"The metainformation contained in the HTTP headers in response to a HEAD request SHOULD be identical to the information sent in response to a GET request."

### Steps to reproduce

- make a plain text file, e.g. "echo hello > hello.txt".
- using Riot-web 1.3.3, in room #test:matrix.org, send that file using the "Upload file" paperclip button.
- copy the URL from the displayed "Download hello.txt" link.
- compare the results of "curl --head <URL>" and "curl -i <URL>"

```
$ curl --head https://matrix.org/_matrix/media/r0/download/matrix.org/WozzzqyaQXYsxVYlRePaGtIZ
HTTP/2 500 
date: Tue, 10 Sep 2019 12:17:55 GMT
content-type: text/html
set-cookie: __cfduid=d8bae399428628ffcc58058da4277761f1568117875; expires=Wed, 09-Sep-20 12:17:55 GMT; path=/; domain=.matrix.org; HttpOnly
expect-ct: max-age=604800, report-uri="https://report-uri.cloudflare.com/cdn-cgi/beacon/expect-ct"
server: cloudflare
cf-ray: 5141576ec83ece17-LHR

```
versus
```
$ curl -i https://matrix.org/_matrix/media/r0/download/matrix.org/WozzzqyaQXYsxVYlRePaGtIZ
HTTP/2 200 
date: Tue, 10 Sep 2019 12:18:00 GMT
content-type: text/plain
set-cookie: __cfduid=d2ea5ebdf19d7719715ee5c80be8dcb341568117880; expires=Wed, 09-Sep-20 12:18:00 GMT; path=/; domain=.matrix.org; HttpOnly
access-control-allow-origin: *
access-control-allow-methods: GET, POST, PUT, DELETE, OPTIONS
access-control-allow-headers: Origin, X-Requested-With, Content-Type, Accept, Authorization
content-security-policy: sandbox; default-src 'none'; script-src 'none'; plugin-types application/pdf; style-src 'unsafe-inline'; media-src 'self'; object-src 'self';
content-disposition: inline; filename=hello.txt
cache-control: public,max-age=86400,s-maxage=86400
expect-ct: max-age=604800, report-uri="https://report-uri.cloudflare.com/cdn-cgi/beacon/expect-ct"
server: cloudflare
cf-ray: 5141578e3aadbbca-LHR

hello
```

### Version information

- **Homeserver**: matrix.org
- **Version**: Synapse "1.3.1 (b=matrix-org-hotfixes,9240622c1,dirty)"

HEAD requests seem to be broken
```
rav@faith:~$ curl -I http://localhost:8008/_synapse/admin/v1/server_version
HTTP/1.1 400 Bad Request
Content-Type: application/json
Access-Control-Allow-Origin: *
Access-Control-Allow-Methods: GET, POST, PUT, DELETE, OPTIONS
Content-Length: 73
Cache-Control: no-cache, no-store, must-revalidate
Date: Thu, 30 Jul 2020 15:06:09 GMT
Access-Control-Allow-Headers: Origin, X-Requested-With, Content-Type, Accept, Authorization
Server: Synapse/1.18.0rc2 (b=rav/fix_reject_no_extrems,b744eb3)

```


Discussion:

I went back to 1.6.1 (earliest I could easily get to run) and this seemed broken there.

I think it is straightforward though so I'll take a quick look.

Resolving patch:
diff --git a/synapse/http/server.py b/synapse/http/server.py
--- a/synapse/http/server.py
+++ b/synapse/http/server.py
@@ -242,10 +242,12 @@ async def _async_render(self, request: Request):
         no appropriate method exists. Can be overriden in sub classes for
         different routing.
         """
+        # Treat HEAD requests as GET requests.
+        request_method = request.method.decode("ascii")
+        if request_method == "HEAD":
+            request_method = "GET"
 
-        method_handler = getattr(
-            self, "_async_render_%s" % (request.method.decode("ascii"),), None
-        )
+        method_handler = getattr(self, "_async_render_%s" % (request_method,), None)
         if method_handler:
             raw_callback_return = method_handler(request)
 
@@ -362,11 +364,15 @@ def _get_handler_for_request(
             A tuple of the callback to use, the name of the servlet, and the
             key word arguments to pass to the callback
         """
+        # Treat HEAD requests as GET requests.
         request_path = request.path.decode("ascii")
+        request_method = request.method
+        if request_method == b"HEAD":
+            request_method = b"GET"
 
         # Loop through all the registered callbacks to check if the method
         # and path regex match
-        for path_entry in self.path_regexs.get(request.method, []):
+        for path_entry in self.path_regexs.get(request_method, []):
             m = path_entry.pattern.match(request_path)
             if m:
                 # We found a match!
@@ -579,7 +585,7 @@ def set_cors_headers(request: Request):
     """
     request.setHeader(b"Access-Control-Allow-Origin", b"*")
     request.setHeader(
-        b"Access-Control-Allow-Methods", b"GET, POST, PUT, DELETE, OPTIONS"
+        b"Access-Control-Allow-Methods", b"GET, HEAD, POST, PUT, DELETE, OPTIONS"
     )
     request.setHeader(
         b"Access-Control-Allow-Headers",


Test patch:
diff --git a/tests/test_server.py b/tests/test_server.py
--- a/tests/test_server.py
+++ b/tests/test_server.py
@@ -157,6 +157,29 @@ def _callback(request, **kwargs):
         self.assertEqual(channel.json_body["error"], "Unrecognized request")
         self.assertEqual(channel.json_body["errcode"], "M_UNRECOGNIZED")
 
+    def test_head_request(self):
+        """
+        JsonResource.handler_for_request gives correctly decoded URL args to
+        the callback, while Twisted will give the raw bytes of URL query
+        arguments.
+        """
+
+        def _callback(request, **kwargs):
+            return 200, {"result": True}
+
+        res = JsonResource(self.homeserver)
+        res.register_paths(
+            "GET", [re.compile("^/_matrix/foo$")], _callback, "test_servlet",
+        )
+
+        # The path was registered as GET, but this is a HEAD request.
+        request, channel = make_request(self.reactor, b"HEAD", b"/_matrix/foo")
+        render(request, res, self.reactor)
+
+        self.assertEqual(channel.result["code"], b"200")
+        self.assertNotIn("body", channel.result)
+        self.assertEqual(channel.headers.getRawHeaders(b"Content-Length"), [b"15"])
+
 
 class OptionsResourceTests(unittest.TestCase):
     def setUp(self):
@@ -255,7 +278,7 @@ def setUp(self):
         self.reactor = ThreadedMemoryReactorClock()
 
     def test_good_response(self):
-        def callback(request):
+        async def callback(request):
             request.write(b"response")
             request.finish()
 
@@ -275,7 +298,7 @@ def test_redirect_exception(self):
         with the right location.
         """
 
-        def callback(request, **kwargs):
+        async def callback(request, **kwargs):
             raise RedirectException(b"/look/an/eagle", 301)
 
         res = WrapHtmlRequestHandlerTests.TestResource()
@@ -295,7 +318,7 @@ def test_redirect_exception_with_cookie(self):
         returned too
         """
 
-        def callback(request, **kwargs):
+        async def callback(request, **kwargs):
             e = RedirectException(b"/no/over/there", 304)
             e.cookies.append(b"session=yespls")
             raise e
@@ -312,3 +335,19 @@ def callback(request, **kwargs):
         self.assertEqual(location_headers, [b"/no/over/there"])
         cookies_headers = [v for k, v in headers if k == b"Set-Cookie"]
         self.assertEqual(cookies_headers, [b"session=yespls"])
+
+    def test_head_request(self):
+        """A head request should work by being turned into a GET request."""
+
+        async def callback(request):
+            request.write(b"response")
+            request.finish()
+
+        res = WrapHtmlRequestHandlerTests.TestResource()
+        res.callback = callback
+
+        request, channel = make_request(self.reactor, b"HEAD", b"/path")
+        render(request, res, self.reactor)
+
+        self.assertEqual(channel.result["code"], b"200")
+        self.assertNotIn("body", channel.result)