Repository: readthedocs/readthedocs.org
Issue: Serve custom 404 pages from Django

Currently, we are serving the `404.html` from NGINX returning a `Response` that includes the `X-Accel-Redirect` header:

https://github.com/rtfd/readthedocs.org/blob/537f7b8d2e8d81f8243d1d4a562968e472bc6337/readthedocs/core/views/__init__.py#L208-L210

We found that even marking the `response.status_code = 404`, the response is 200.

```
$ curl -IL https://docs.readthedocs.io/notfound
HTTP/1.1 200 OK
Content-Length: 14167
Content-Type: text/html
Last-Modified: Mon, 22 Apr 2019 15:30:57 GMT
Accept-Ranges: bytes
ETag: "5cbdde31-3757"
Vary: Accept-Encoding
Server: nginx
X-Served: Nginx-Sendfile
X-Deity: web01
Strict-Transport-Security: max-age=1800; includeSubDomains
Date: Mon, 29 Apr 2019 07:59:21 GMT
```

To fix this, we have to serve this file directly from Django instead of using this header.

Changed files:
readthedocs/core/views/__init__.py, readthedocs/rtd_tests/tests/test_doc_serving.py

diff --git a/readthedocs/core/views/__init__.py b/readthedocs/core/views/__init__.py
index 61e5b6e71..8fcccb749 100644
--- a/readthedocs/core/views/__init__.py
+++ b/readthedocs/core/views/__init__.py
@@ -14,13 +14,12 @@ from django.conf import settings
 from django.http import HttpResponseRedirect, Http404, JsonResponse
 from django.shortcuts import render, get_object_or_404, redirect
 from django.views.generic import TemplateView
-
+from django.views.static import serve as static_serve
 
 from readthedocs.builds.models import Version
 from readthedocs.core.utils.general import wipe_version_via_slugs
 from readthedocs.core.resolver import resolve_path
 from readthedocs.core.symlink import PrivateSymlink, PublicSymlink
-from readthedocs.core.views.serve import _serve_file
 from readthedocs.projects.constants import PRIVATE
 from readthedocs.projects.models import HTMLFile, Project
 from readthedocs.redirects.utils import (
@@ -205,7 +204,7 @@ def server_error_404_subdomain(request, template_name='404.html'):
                     project.slug,
                     slug,
                 )
-                r = _serve_file(request, filename, basepath)
+                r = static_serve(request, filename, basepath)
                 r.status_code = 404
                 return r
 
diff --git a/readthedocs/rtd_tests/tests/test_doc_serving.py b/readthedocs/rtd_tests/tests/test_doc_serving.py
index d2ed7adf1..0eb938a70 100644
--- a/readthedocs/rtd_tests/tests/test_doc_serving.py
+++ b/readthedocs/rtd_tests/tests/test_doc_serving.py
@@ -201,14 +201,11 @@ class TestPublicDocs(BaseDocServing):
         ROOT_URLCONF=settings.SUBDOMAIN_URLCONF,
     )
 
-    @patch('readthedocs.core.views.serve.os')
+    @patch('readthedocs.core.views.static_serve')
     @patch('readthedocs.core.views.os')
-    def test_custom_404_page(self, os_view_mock, os_serve_mock):
+    def test_custom_404_page(self, os_view_mock, static_serve_mock):
         os_view_mock.path.exists.return_value = True
 
-        os_serve_mock.path.join.side_effect = os.path.join
-        os_serve_mock.path.exists.return_value = True
-
         self.public.versions.update(active=True, built=True)
 
         factory = RequestFactory()
@@ -221,7 +218,6 @@ class TestPublicDocs(BaseDocServing):
         middleware.process_request(request)
         response = server_error_404_subdomain(request)
         self.assertEqual(response.status_code, 404)
-        self.assertTrue(response['X-Accel-Redirect'].endswith('/public/en/latest/404.html'))
 
     @override_settings(
         USE_SUBDOMAIN=True,