Repository: readthedocs/readthedocs.org

Issue:
Handle serving README.html as default index file with Proxito
One of the rules we support in nginx is serving of README.html after index.html is tried. We need to be able to support this either in code or with nginx. Opening the issue here, but we can move if it is going to be addressed with nginx configuration.


Discussion:

Resolving patch:
diff --git a/readthedocs/proxito/middleware.py b/readthedocs/proxito/middleware.py
--- a/readthedocs/proxito/middleware.py
+++ b/readthedocs/proxito/middleware.py
@@ -8,14 +8,11 @@
 import logging
 
 from django.conf import settings
-from django.http import HttpResponseBadRequest
 from django.shortcuts import render
 from django.utils.deprecation import MiddlewareMixin
-from django.utils.translation import ugettext_lazy as _
 
 from readthedocs.projects.models import Domain
 
-
 log = logging.getLogger(__name__)  # noqa
 
 
@@ -55,7 +52,9 @@ def map_host_to_project_slug(request):
             # TODO: This can catch some possibly valid domains (docs.readthedocs.io.com) for example
             # But these feel like they might be phishing, etc. so let's block them for now.
             log.warning('Weird variation on our hostname: host=%s', host)
-            return HttpResponseBadRequest(_('Invalid hostname'))
+            return render(
+                request, 'core/dns-404.html', context={'host': host}, status=400
+            )
 
     # Serve CNAMEs
     else:
diff --git a/readthedocs/proxito/urls.py b/readthedocs/proxito/urls.py
--- a/readthedocs/proxito/urls.py
+++ b/readthedocs/proxito/urls.py
@@ -30,18 +30,28 @@
 """
 
 from django.conf.urls import url
+from django.views import defaults
 
 from readthedocs.constants import pattern_opts
-from readthedocs.proxito.views import redirect_page_with_filename, serve_docs
+from readthedocs.proxito import views
 
 
 urlpatterns = [
+    # Serve custom 404 pages
+    url(
+        r'^_proxito_404_(?P<proxito_path>.*)$',
+        views.serve_error_404,
+        name='serve_error_404',
+    ),
+    url(r'robots\.txt$', views.robots_txt, name='robots_txt'),
+    url(r'sitemap\.xml$', views.sitemap_xml, name='sitemap_xml'),
+
     # # TODO: Support this?
     # (Sub)project `page` redirect
     url(
         r'^(?:projects/(?P<subproject_slug>{project_slug})/)?'
         r'page/(?P<filename>.*)$'.format(**pattern_opts),
-        redirect_page_with_filename,
+        views.redirect_page_with_filename,
         name='redirect_page_with_filename',
     ),
 
@@ -53,7 +63,7 @@
             r'(?P<version_slug>{version_slug})/'
             r'(?P<filename>{filename_slug})$'.format(**pattern_opts)
         ),
-        serve_docs,
+        views.serve_docs,
         name='docs_detail',
     ),
 
@@ -75,7 +85,11 @@
             r'^(?:projects/(?P<subproject_slug>{project_slug})/)?'
             r'(?P<filename>{filename_slug})$'.format(**pattern_opts)
         ),
-        serve_docs,
+        views.serve_docs,
         name='docs_detail_singleversion_subproject',
     ),
 ]
+
+# Use Django default error handlers to make things simpler
+handler404 = views.fast_404
+handler500 = defaults.server_error
diff --git a/readthedocs/proxito/views.py b/readthedocs/proxito/views.py
--- a/readthedocs/proxito/views.py
+++ b/readthedocs/proxito/views.py
@@ -1,5 +1,6 @@
 """Views for doc serving."""
 
+import itertools
 import logging
 import mimetypes
 import os
@@ -10,16 +11,31 @@
 from django.core.files.storage import get_storage_class
 from django.http import Http404, HttpResponse, HttpResponseRedirect
 from django.shortcuts import get_object_or_404, render
+from django.urls import resolve as url_resolve
 from django.utils.encoding import iri_to_uri
+from django.views.decorators.cache import cache_page
 from django.views.static import serve
 
+from readthedocs.builds.constants import LATEST, STABLE
+from readthedocs.builds.models import Version
 from readthedocs.core.resolver import resolve
+from readthedocs.projects import constants
 from readthedocs.projects.models import Project, ProjectRelationship
-
+from readthedocs.projects.templatetags.projects_tags import sort_version_aware
 
 log = logging.getLogger(__name__)  # noqa
 
 
+def fast_404(request, *args, **kwargs):
+    """
+    A fast error page handler.
+
+    This stops us from running RTD logic in our error handling. We already do
+    this in RTD prod when we fallback to it.
+    """
+    return HttpResponse('Not Found.', status=404)
+
+
 def _serve_401(request, project):
     res = render(request, '401.html')
     res.status_code = 401
@@ -109,7 +125,11 @@ def inner_view(  # noqa
 @map_project_slug
 @map_subproject_slug
 def redirect_page_with_filename(request, project, subproject, filename):  # pylint: disable=unused-argument  # noqa
-    """Redirect /page/file.html to /<default-lang>/<default-version>/file.html."""
+    """
+    Redirect /page/file.html to.
+
+    /<default-lang>/<default-version>/file.html.
+    """
 
     urlparse_result = urlparse(request.get_full_path())
     return HttpResponseRedirect(
@@ -137,7 +157,7 @@ def redirect_project_slug(request, project, subproject):  # pylint: disable=unus
 
 @map_project_slug
 @map_subproject_slug
-def serve_docs(
+def _get_project_data_from_request(
         request,
         project,
         subproject,
@@ -145,30 +165,14 @@ def serve_docs(
         version_slug=None,
         filename='',
 ):
-    """Take the incoming parsed URL's and figure out what file to serve."""
-
-    log.debug(
-        'project=%s, subproject=%s, lang_slug=%s, version_slug=%s, filename=%s',
-        project, subproject, lang_slug, version_slug, filename
-    )
+    """
+    Get the proper project based on the request and URL.
 
+    This is used in a few places and so we break out into a utility function.
+    """
     # Take the most relevant project so far
     current_project = subproject or project
 
-    # Handle a / redirect when we aren't a single version
-    if all([lang_slug is None, version_slug is None, filename == '',
-            not current_project.single_version]):
-        log.info('Proxito redirect: slug=%s', current_project.slug)
-        return redirect_project_slug(
-            request,
-            project=current_project,
-            subproject=None,
-        )
-
-    if (lang_slug is None or version_slug is None) and not current_project.single_version:
-        log.info('Invalid URL for project with versions. url=%s', filename)
-        raise Http404('Invalid URL for project with versions')
-
     # Handle single-version projects that have URLs like a real project
     if current_project.single_version:
         if lang_slug and version_slug:
@@ -183,22 +187,70 @@ def serve_docs(
             current_project.translations.all(), language=lang_slug
         )
 
+    # Handle single version by grabbing the default version
+    if final_project.single_version:
+        version_slug = final_project.get_default_version()
+
     # ``final_project`` is now the actual project we want to serve docs on,
     # accounting for:
     # * Project
     # * Subproject
     # * Translations
 
+    return final_project, lang_slug, version_slug, filename
+
+
+def serve_docs(
+        request,
+        project_slug=None,
+        subproject_slug=None,
+        lang_slug=None,
+        version_slug=None,
+        filename='',
+):
+    """Take the incoming parsed URL's and figure out what file to serve."""
+
+    final_project, lang_slug, version_slug, filename = _get_project_data_from_request(  # noqa
+        request,
+        project_slug=project_slug,
+        subproject_slug=subproject_slug,
+        lang_slug=lang_slug,
+        version_slug=version_slug,
+        filename=filename,
+    )
+
+    log.debug(
+        'Serving docs: project=%s, subproject=%s, lang_slug=%s, version_slug=%s, filename=%s',
+        final_project.slug, subproject_slug, lang_slug, version_slug, filename
+    )
+
+    # Handle a / redirect when we aren't a single version
+    if all([lang_slug is None, version_slug is None, filename == '',
+            not final_project.single_version]):
+        redirect_to = redirect_project_slug(
+            request,
+            project=final_project,
+            subproject=None,
+        )
+        log.info(
+            'Proxito redirect: from=%s, to=%s, project=%s', filename,
+            redirect_to, final_project.slug
+        )
+        return redirect_to
+
+    if (lang_slug is None or version_slug is None) and not final_project.single_version:
+        log.info(
+            'Invalid URL for project with versions. url=%s, project=%s',
+            filename, final_project.slug
+        )
+        raise Http404('Invalid URL for project with versions')
+
     # TODO: Redirects need to be refactored before we can turn them on
     # They currently do 1 request per redirect that exists for the project
     # path, http_status = final_project.redirects.get_redirect_path_with_status(
     #     language=lang_slug, version_slug=version_slug, path=filename
     # )
 
-    # Handle single version by grabbing the default version
-    if final_project.single_version:
-        version_slug = final_project.get_default_version()
-
     # Don't do auth checks
     # try:
     #     Version.objects.public(user=request.user, project=final_project).get(slug=version_slug)
@@ -212,36 +264,59 @@ def serve_docs(
     storage_path = final_project.get_storage_path(
         type_='html', version_slug=version_slug, include_file=False
     )
-    path = f'{storage_path}/{filename}'
+    path = os.path.join(storage_path, filename)
 
-    # Handle out backend storage not supporting directory indexes,
+    # Handle our backend storage not supporting directory indexes,
     # so we need to append index.html when appropriate.
-    # TODO: We don't currently support `docs.example.com/en/latest/install`
-    # where the user actually wants `docs.example.com/en/latest/install/index.html`
-    # We would need to emulate nginx's try_files in order to handle this.
     if path[-1] == '/':
         path += 'index.html'
 
-    # Serve from the filesystem if using PYTHON_MEDIA
-    # We definitely shouldn't do this in production,
-    # but I don't want to force a check for DEBUG.
+    return _serve_docs(request, final_project=final_project, path=path)
+
+
+def _serve_docs(request, final_project, path):
+    """
+    Serve documentation in the way specified by settings.
+
+    Serve from the filesystem if using PYTHON_MEDIA We definitely shouldn't do
+    this in production, but I don't want to force a check for DEBUG.
+    """
+
     if settings.PYTHON_MEDIA:
-        log.info('[Django serve] path=%s, project=%s', path, final_project)
-        storage = get_storage_class(settings.RTD_BUILD_MEDIA_STORAGE)()
-        root_path = storage.path('')
-        # Serve from Python
-        return serve(request, path, root_path)
-
-    # Serve via nginx
-    log.info('[Nginx serve] path=%s, project=%s', path, final_project)
-    return _serve_docs_nginx(
-        request, final_project=final_project, path=f'/proxito/{path}'
-    )
+        return _serve_docs_python(
+            request, final_project=final_project, path=path
+        )
+    return _serve_docs_nginx(request, final_project=final_project, path=path)
+
+
+def _serve_docs_python(request, final_project, path):
+    """
+    Serve docs from Python.
+
+    .. warning:: Don't do this in production!
+    """
+    log.info('[Django serve] path=%s, project=%s', path, final_project.slug)
+
+    storage = get_storage_class(settings.RTD_BUILD_MEDIA_STORAGE)()
+    root_path = storage.path('')
+    # Serve from Python
+    return serve(request, path, root_path)
 
 
 def _serve_docs_nginx(request, final_project, path):
+    """
+    Serve docs from nginx.
+
+    Returns a response with ``X-Accel-Redirect``, which will cause nginx to
+    serve it directly as an internal redirect.
+    """
+    log.info('[Nginx serve] path=%s, project=%s', path, final_project.slug)
+
+    if not path.startswith('/proxito/'):
+        if path[0] == '/':
+            path = path[1:]
+        path = f'/proxito/{path}'
 
-    # Serve from Nginx
     content_type, encoding = mimetypes.guess_type(path)
     content_type = content_type or 'application/octet-stream'
     response = HttpResponse(
@@ -259,3 +334,269 @@ def _serve_docs_nginx(request, final_project, path):
     response['X-Accel-Redirect'] = x_accel_redirect
 
     return response
+
+
+def serve_error_404(request, proxito_path, template_name='404.html'):
+    """
+    Handler for 404 pages on subdomains.
+
+    This does a couple things:
+
+    * Handles directory indexing for URLs that don't end in a slash
+    * Handles directory indexing for README.html (for now)
+    * Handles custom 404 serving
+
+    For 404's, first search for a 404 page in the current version, then continues
+    with the default version and finally, if none of them are found, the Read
+    the Docs default page (Maze Found) is rendered by Django and served.
+    """
+    # pylint: disable=too-many-locals
+
+    # Parse the URL using the normal urlconf, so we get proper subdomain/translation data
+    _, __, kwargs = url_resolve(
+        proxito_path, urlconf='readthedocs.proxito.urls'
+    )
+    final_project, lang_slug, version_slug, filename = _get_project_data_from_request(  # noqa
+        request,
+        project_slug=kwargs.get('project_slug'),
+        subproject_slug=kwargs.get('subproject_slug'),
+        lang_slug=kwargs.get('lang_slug'),
+        version_slug=kwargs.get('version_slug'),
+        filename=kwargs.get('filename', ''),
+    )
+
+    storage_root_path = final_project.get_storage_path(
+        type_='html',
+        version_slug=version_slug,
+        include_file=False,
+    )
+    storage = get_storage_class(settings.RTD_BUILD_MEDIA_STORAGE)()
+
+    # First, check for dirhtml with slash
+    for tryfile in ('index.html', 'README.html'):
+        storage_filename_path = os.path.join(
+            storage_root_path, filename, tryfile
+        )
+        log.debug(
+            'Trying index filename: project=%s version=%s, file=%s',
+            final_project.slug,
+            version_slug,
+            storage_filename_path,
+        )
+        if storage.exists(storage_filename_path):
+            log.info(
+                'Serving index file: project=%s version=%s, url=%s',
+                final_project.slug,
+                version_slug,
+                storage_filename_path,
+            )
+            resp = HttpResponse(storage.open(storage_filename_path).read())
+            return resp
+
+    # If that doesn't work, attempt to serve the 404 of the current version (version_slug)
+    # Secondly, try to serve the 404 page for the default version
+    # (project.get_default_version())
+    for version_slug_404 in [version_slug, final_project.get_default_version()]:
+        for tryfile in ('404.html', '404/index.html'):
+            storage_root_path = final_project.get_storage_path(
+                type_='html',
+                version_slug=version_slug_404,
+                include_file=False,
+            )
+            storage_filename_path = os.path.join(storage_root_path, tryfile)
+            if storage.exists(storage_filename_path):
+                log.debug(
+                    'Serving custom 404.html page: [project: %s] [version: %s]',
+                    final_project.slug,
+                    version_slug_404,
+                )
+                resp = HttpResponse(storage.open(storage_filename_path).read())
+                resp.status_code = 404
+                return resp
+
+    # Finally, return the default 404 page generated by Read the Docs
+    resp = render(request, template_name)
+    resp.status_code = 404
+    return resp
+
+
+@map_project_slug
+def robots_txt(request, project):
+    """
+    Serve custom user's defined ``/robots.txt``.
+
+    If the user added a ``robots.txt`` in the "default version" of the project,
+    we serve it directly.
+    """
+    # Use the ``robots.txt`` file from the default version configured
+    version_slug = project.get_default_version()
+    version = project.versions.get(slug=version_slug)
+
+    no_serve_robots_txt = any([
+        # If project is private or,
+        project.privacy_level == constants.PRIVATE,
+        # default version is private or,
+        version.privacy_level == constants.PRIVATE,
+        # default version is not active or,
+        not version.active,
+        # default version is not built
+        not version.built,
+    ])
+
+    if no_serve_robots_txt:
+        # ... we do return a 404
+        raise Http404()
+
+    storage_path = project.get_storage_path(
+        type_='html', version_slug=version_slug, include_file=False
+    )
+    path = os.path.join(storage_path, 'robots.txt')
+
+    storage = get_storage_class(settings.RTD_BUILD_MEDIA_STORAGE)()
+    if storage.exists(path):
+        return _serve_docs(request, project, path)
+
+    sitemap_url = '{scheme}://{domain}/sitemap.xml'.format(
+        scheme='https',
+        domain=project.subdomain(),
+    )
+    return HttpResponse(
+        'User-agent: *\nAllow: /\nSitemap: {}\n'.format(sitemap_url),
+        content_type='text/plain',
+    )
+
+
+@map_project_slug
+@cache_page(60 * 60 * 24 * 3)  # 3 days
+def sitemap_xml(request, project):
+    """
+    Generate and serve a ``sitemap.xml`` for a particular ``project``.
+
+    The sitemap is generated from all the ``active`` and public versions of
+    ``project``. These versions are sorted by using semantic versioning
+    prepending ``latest`` and ``stable`` (if they are enabled) at the beginning.
+
+    Following this order, the versions are assigned priorities and change
+    frequency. Starting from 1 and decreasing by 0.1 for priorities and starting
+    from daily, weekly to monthly for change frequency.
+
+    If the project is private, the view raises ``Http404``. On the other hand,
+    if the project is public but a version is private, this one is not included
+    in the sitemap.
+
+    :param request: Django request object
+    :param project: Project instance to generate the sitemap
+
+    :returns: response with the ``sitemap.xml`` template rendered
+
+    :rtype: django.http.HttpResponse
+    """
+    # pylint: disable=too-many-locals
+
+    def priorities_generator():
+        """
+        Generator returning ``priority`` needed by sitemap.xml.
+
+        It generates values from 1 to 0.1 by decreasing in 0.1 on each
+        iteration. After 0.1 is reached, it will keep returning 0.1.
+        """
+        priorities = [1, 0.9, 0.8, 0.7, 0.6, 0.5, 0.4, 0.3, 0.2]
+        yield from itertools.chain(priorities, itertools.repeat(0.1))
+
+    def hreflang_formatter(lang):
+        """
+        sitemap hreflang should follow correct format.
+
+        Use hyphen instead of underscore in language and country value.
+        ref: https://en.wikipedia.org/wiki/Hreflang#Common_Mistakes
+        """
+        if '_' in lang:
+            return lang.replace('_', '-')
+        return lang
+
+    def changefreqs_generator():
+        """
+        Generator returning ``changefreq`` needed by sitemap.xml.
+
+        It returns ``weekly`` on first iteration, then ``daily`` and then it
+        will return always ``monthly``.
+
+        We are using ``monthly`` as last value because ``never`` is too
+        aggressive. If the tag is removed and a branch is created with the same
+        name, we will want bots to revisit this.
+        """
+        changefreqs = ['weekly', 'daily']
+        yield from itertools.chain(changefreqs, itertools.repeat('monthly'))
+
+    if project.privacy_level == constants.PRIVATE:
+        raise Http404
+
+    sorted_versions = sort_version_aware(
+        Version.internal.public(
+            project=project,
+            only_active=True,
+        ),
+    )
+
+    # This is a hack to swap the latest version with
+    # stable version to get the stable version first in the sitemap.
+    # We want stable with priority=1 and changefreq='weekly' and
+    # latest with priority=0.9 and changefreq='daily'
+    # More details on this: https://github.com/rtfd/readthedocs.org/issues/5447
+    if (len(sorted_versions) >= 2 and sorted_versions[0].slug == LATEST and
+            sorted_versions[1].slug == STABLE):
+        sorted_versions[0], sorted_versions[1] = sorted_versions[1], sorted_versions[0]
+
+    versions = []
+    for version, priority, changefreq in zip(
+            sorted_versions,
+            priorities_generator(),
+            changefreqs_generator(),
+    ):
+        element = {
+            'loc': version.get_subdomain_url(),
+            'priority': priority,
+            'changefreq': changefreq,
+            'languages': [],
+        }
+
+        # Version can be enabled, but not ``built`` yet. We want to show the
+        # link without a ``lastmod`` attribute
+        last_build = version.builds.order_by('-date').first()
+        if last_build:
+            element['lastmod'] = last_build.date.isoformat()
+
+        if project.translations.exists():
+            for translation in project.translations.all():
+                translation_versions = (
+                    Version.internal.public(project=translation
+                                            ).values_list('slug', flat=True)
+                )
+                if version.slug in translation_versions:
+                    href = project.get_docs_url(
+                        version_slug=version.slug,
+                        lang_slug=translation.language,
+                        private=False,
+                    )
+                    element['languages'].append({
+                        'hreflang': hreflang_formatter(translation.language),
+                        'href': href,
+                    })
+
+            # Add itself also as protocol requires
+            element['languages'].append({
+                'hreflang': project.language,
+                'href': element['loc'],
+            })
+
+        versions.append(element)
+
+    context = {
+        'versions': versions,
+    }
+    return render(
+        request,
+        'sitemap.xml',
+        context,
+        content_type='application/xml',
+    )


Test patch:
diff --git a/readthedocs/proxito/tests/test_full.py b/readthedocs/proxito/tests/test_full.py
--- a/readthedocs/proxito/tests/test_full.py
+++ b/readthedocs/proxito/tests/test_full.py
@@ -2,10 +2,18 @@
 
 import os
 
+import django_dynamic_fixture as fixture
 import mock
 from django.conf import settings
+from django.core.files.storage import Storage
 from django.http import HttpResponse
 from django.test.utils import override_settings
+from django.urls import reverse
+
+from readthedocs.builds.constants import EXTERNAL, INTERNAL
+from readthedocs.builds.models import Version
+from readthedocs.projects import constants
+from readthedocs.projects.models import Project
 
 from .base import BaseDocServing
 
@@ -144,3 +152,212 @@ def test_project_nginx_serving_unicode_filename(self):
             resp['x-accel-redirect'],
             '/proxito/html/project/latest/%C3%BA%C3%B1%C3%AD%C4%8D%C3%B3d%C3%A9.html',
         )
+
+
+@override_settings(
+    PYTHON_MEDIA=False,
+    PUBLIC_DOMAIN='readthedocs.io',
+)
+class TestAdditionalDocViews(BaseDocServing):
+    # Test that robots.txt and sitemap.xml work
+
+    def test_default_robots_txt(self):
+        self.project.versions.update(active=True, built=True)
+        response = self.client.get(
+            reverse('robots_txt'),
+            HTTP_HOST='project.readthedocs.io',
+        )
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(
+            response.content,
+            b'User-agent: *\nAllow: /\nSitemap: https://project.readthedocs.io/sitemap.xml\n'
+        )
+
+    @mock.patch('readthedocs.proxito.views.get_storage_class')
+    def test_custom_robots_txt(self, storage_mock):
+        self.project.versions.update(active=True, built=True)
+        storage_mock()().exists.return_value = True
+        response = self.client.get(
+            reverse('robots_txt'),
+            HTTP_HOST='project.readthedocs.io',
+        )
+        self.assertEqual(
+            response['x-accel-redirect'], '/proxito/html/project/latest/robots.txt',
+        )
+
+    @mock.patch('readthedocs.proxito.views.get_storage_class')
+    def test_directory_indexes(self, storage_mock):
+        self.project.versions.update(active=True, built=True)
+        storage_mock()().exists.return_value = True
+        storage_mock()().open().read.return_value = 'foo'
+        # Confirm we've serving from storage for the `index-exists/index.html` file
+        response = self.client.get(
+            reverse('serve_error_404', kwargs={'proxito_path': '/en/latest/index-exists'}),
+            HTTP_HOST='project.readthedocs.io',
+        )
+        self.assertEqual(
+            response.content, b'foo'
+        )
+        self.assertEqual(
+            response.status_code, 200
+        )
+
+    @mock.patch('readthedocs.proxito.views.get_storage_class')
+    def test_404_storage_serves_404(self, storage_mock):
+        self.project.versions.update(active=True, built=True)
+
+        storage_mock()().exists.side_effect = [False, False, True]
+        response = self.client.get(
+            reverse('serve_error_404', kwargs={'proxito_path': '/en/fancy-version/not-found'}),
+            HTTP_HOST='project.readthedocs.io',
+        )
+        storage_mock()().exists.assert_has_calls(
+            [
+                mock.call('html/project/fancy-version/not-found/index.html'),
+                mock.call('html/project/fancy-version/not-found/README.html'),
+                mock.call('html/project/fancy-version/404.html'),
+            ]
+        )
+        self.assertEqual(
+            response.status_code, 404
+        )
+
+    @mock.patch('readthedocs.proxito.views.get_storage_class')
+    def test_404_storage_paths_checked(self, storage_mock):
+        self.project.versions.update(active=True, built=True)
+        storage_mock()().exists.return_value = False
+        self.client.get(
+            reverse('serve_error_404', kwargs={'proxito_path': '/en/fancy-version/not-found'}),
+            HTTP_HOST='project.readthedocs.io',
+        )
+        storage_mock()().exists.assert_has_calls(
+            [
+                mock.call('html/project/fancy-version/not-found/index.html'),
+                mock.call('html/project/fancy-version/not-found/README.html'),
+                mock.call('html/project/fancy-version/404.html'),
+                mock.call('html/project/fancy-version/404/index.html'),
+                mock.call('html/project/latest/404.html'),
+                mock.call('html/project/latest/404/index.html')
+            ]
+        )
+
+    def test_sitemap_xml(self):
+        self.project.versions.update(active=True)
+        private_version = fixture.get(
+            Version,
+            privacy_level=constants.PRIVATE,
+            project=self.project,
+        )
+        not_translated_public_version = fixture.get(
+            Version,
+            identifier='not-translated-version',
+            verbose_name='not-translated-version',
+            slug='not-translated-version',
+            privacy_level=constants.PUBLIC,
+            project=self.project,
+            active=True
+        )
+        stable_version = fixture.get(
+            Version,
+            identifier='stable',
+            verbose_name='stable',
+            slug='stable',
+            privacy_level=constants.PUBLIC,
+            project=self.project,
+            active=True
+        )
+        # This is a EXTERNAL Version
+        external_version = fixture.get(
+            Version,
+            identifier='pr-version',
+            verbose_name='pr-version',
+            slug='pr-9999',
+            project=self.project,
+            active=True,
+            type=EXTERNAL
+        )
+        # This also creates a Version `latest` Automatically for this project
+        translation = fixture.get(
+            Project,
+            main_language_project=self.project,
+            language='translation-es'
+        )
+        # sitemap hreflang should follow correct format.
+        # ref: https://en.wikipedia.org/wiki/Hreflang#Common_Mistakes
+        hreflang_test_translation_project = fixture.get(
+            Project,
+            main_language_project=self.project,
+            language='zh_CN'
+        )
+        response = self.client.get(
+            reverse('sitemap_xml'),
+            HTTP_HOST='project.readthedocs.io',
+        )
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response['Content-Type'], 'application/xml')
+        for version in self.project.versions(manager=INTERNAL).filter(privacy_level=constants.PUBLIC):
+            self.assertContains(
+                response,
+                self.project.get_docs_url(
+                    version_slug=version.slug,
+                    lang_slug=self.project.language,
+                    private=False,
+                ),
+            )
+
+        # PRIVATE version should not appear here
+        self.assertNotContains(
+            response,
+            self.project.get_docs_url(
+                version_slug=private_version.slug,
+                lang_slug=self.project.language,
+                private=True,
+            ),
+        )
+        # The `translation` project doesn't have a version named `not-translated-version`
+        # so, the sitemap should not have a doc url for
+        # `not-translated-version` with `translation-es` language.
+        # ie: http://project.readthedocs.io/translation-es/not-translated-version/
+        self.assertNotContains(
+            response,
+            self.project.get_docs_url(
+                version_slug=not_translated_public_version.slug,
+                lang_slug=translation.language,
+                private=False,
+            ),
+        )
+        # hreflang should use hyphen instead of underscore
+        # in language and country value. (zh_CN should be zh-CN)
+        self.assertContains(response, 'zh-CN')
+
+        # External Versions should not be in the sitemap_xml.
+        self.assertNotContains(
+            response,
+            self.project.get_docs_url(
+                version_slug=external_version.slug,
+                lang_slug=self.project.language,
+                private=True,
+            ),
+        )
+
+        # Check if STABLE version has 'priority of 1 and changefreq of weekly.
+        self.assertEqual(
+            response.context['versions'][0]['loc'],
+            self.project.get_docs_url(
+                version_slug=stable_version.slug,
+                lang_slug=self.project.language,
+                private=False,
+            ),)
+        self.assertEqual(response.context['versions'][0]['priority'], 1)
+        self.assertEqual(response.context['versions'][0]['changefreq'], 'weekly')
+
+        # Check if LATEST version has priority of 0.9 and changefreq of daily.
+        self.assertEqual(
+            response.context['versions'][1]['loc'],
+            self.project.get_docs_url(
+                version_slug='latest',
+                lang_slug=self.project.language,
+                private=False,
+            ),)
+        self.assertEqual(response.context['versions'][1]['priority'], 0.9)
+        self.assertEqual(response.context['versions'][1]['changefreq'], 'daily')