Repository: UAlbertaALTLab/morphodict (language: python)

Issue:
Dynamically generate a sitemap.xml with ALL THE WORDFORMS!
> **User story**: As a Cree language learner, when I search for a Cree word in my search engine of choice, it should link to a page on itwêwina.

How do we let search engines like Google, Bing, Duck Duck Go, etc. know that we're a Cree dictionary? By having them crawl our site! But since web crawlers need _links_ to crawl, we need to provide links to all of our _stored_ wordforms. The solution: `sitemap.xml`.

The sitemap.xml should list both all general pages (e.g., `about`, `contact-us`), but also every `/word/XXX` page.

@aarppe, I know you don't want Google to just show the definition on their search page. You can disable this by using the [`nosnippet` directive](https://support.google.com/webmasters/answer/35624#nosnippet).

We'll probably want to provide a `robots.txt` as well with a link to the `sitemap.xml`.
Dynamically generate a sitemap.xml with ALL THE WORDFORMS!
> **User story**: As a Cree language learner, when I search for a Cree word in my search engine of choice, it should link to a page on itwêwina.

How do we let search engines like Google, Bing, Duck Duck Go, etc. know that we're a Cree dictionary? By having them crawl our site! But since web crawlers need _links_ to crawl, we need to provide links to all of our _stored_ wordforms. The solution: `sitemap.xml`.

The sitemap.xml should list both all general pages (e.g., `about`, `contact-us`), but also every `/word/XXX` page.

@aarppe, I know you don't want Google to just show the definition on their search page. You can disable this by using the [`nosnippet` directive](https://support.google.com/webmasters/answer/35624#nosnippet).

We'll probably want to provide a `robots.txt` as well with a link to the `sitemap.xml`.


Discussion / hints:
I highly suspect Google will truncate an extra long sitemap for a domain (with no evidence, just doubting)
I can do this later. I have experience with Django's sitemap generation before
I highly suspect Google will truncate an extra long sitemap for a domain (with no evidence, just doubting)
I can do this later. I have experience with Django's sitemap generation before

Fix patch:
diff --git a/CreeDictionary/CreeDictionary/migrations/0002_change_example_site_to_cree_dictionary_site.py b/CreeDictionary/CreeDictionary/migrations/0002_change_example_site_to_cree_dictionary_site.py
new file mode 100644
index 000000000..f0f443f4d
--- /dev/null
+++ b/CreeDictionary/CreeDictionary/migrations/0002_change_example_site_to_cree_dictionary_site.py
@@ -0,0 +1,39 @@
+"""
+The Django "sites" framework is annoying because it automatically creates a site called
+"example.com" instead of... you know... the name of the current site AS A POST-MIGRATION
+HOOK!
+
+This means the "example" _might_ be in the database. Or it might not.
+
+Regardless, this migration changes it to the site configured in settings.py.
+"""
+
+from django.conf import settings
+from django.db import migrations
+
+
+def change_example_to_production_site(apps, schema_editor):
+    """
+    Changes the example.com site to the current site domain.
+    As of 2021-03-11, this is itwewina.altlab.app.
+    """
+    Site = apps.get_model("sites", "Site")
+    site, created = Site.objects.get_or_create(pk=1)
+
+    if not created:
+        # See: https://github.com/django/django/blob/876dc0c1a7dbf569782eb64f62f339c1daeb75e0/django/contrib/sites/management.py#L11-L38
+        assert site.name == "example.com"
+        print("overwriting the default site")
+
+    site.name = site.domain = settings.PRODUCTION_HOST
+    site.save()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("sites", "0002_alter_domain_unique"),
+        ("CreeDictionary", "0001_insert_bibliographic_data"),
+    ]
+
+    operations = [migrations.RunPython(change_example_to_production_site)]
diff --git a/CreeDictionary/CreeDictionary/settings.py b/CreeDictionary/CreeDictionary/settings.py
index 1f5ab5f70..432772c6b 100644
--- a/CreeDictionary/CreeDictionary/settings.py
+++ b/CreeDictionary/CreeDictionary/settings.py
@@ -53,6 +53,8 @@
     "django.contrib.contenttypes",
     "django.contrib.sessions",
     "django.contrib.messages",
+    "django.contrib.sites",
+    "django.contrib.sitemaps",
     # WhiteNoise nostatic HAS to come before Django's staticfiles
     # See: http://whitenoise.evans.io/en/stable/django.html#using-whitenoise-in-development
     "whitenoise.runserver_nostatic",
@@ -186,6 +188,11 @@ def defaultDatabasePath():
         "default": env.dj_db_url("DATABASE_URL", default=defaultDatabasePath())
     }
 
+################################ Django sites framework ################################
+
+# See: https://docs.djangoproject.com/en/2.2/ref/contrib/sites/#enabling-the-sites-framework
+SITE_ID = 1
+
 ################################## SecurityMiddleware ##################################
 
 # Send X-Content-Type-Options: nosniff header
diff --git a/CreeDictionary/CreeDictionary/sitemaps.py b/CreeDictionary/CreeDictionary/sitemaps.py
new file mode 100644
index 000000000..0a02ab92a
--- /dev/null
+++ b/CreeDictionary/CreeDictionary/sitemaps.py
@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+
+from API.models import Wordform
+from django.contrib.sitemaps import Sitemap
+from django.urls import reverse
+
+
+class WordformSitemap(Sitemap):
+    """
+    Make heads available in the sitemap.
+    This way, web crawlers can index ALL THE WORDS!
+    """
+
+    def items(self):
+        return Wordform.objects.filter(is_lemma=True)
+
+
+class StaticViewSitemap(Sitemap):
+    """
+    Index static pages too!
+
+    Note that the 'items' list needs to be manually maintained :/
+
+    See: https://docs.djangoproject.com/en/2.2/ref/contrib/sitemaps/#sitemap-for-static-views
+    """
+
+    def items(self):
+        return ["index", "about", "contact-us"]
+
+    def location(self, item):
+        return reverse(f"cree-dictionary-{item}")
+
+
+sitemaps = {
+    "static": StaticViewSitemap,
+    "words": WordformSitemap,
+}
diff --git a/CreeDictionary/CreeDictionary/urls.py b/CreeDictionary/CreeDictionary/urls.py
index 7cbaa479c..cbcabb58b 100644
--- a/CreeDictionary/CreeDictionary/urls.py
+++ b/CreeDictionary/CreeDictionary/urls.py
@@ -9,11 +9,13 @@
 from django.conf.urls import url
 from django.conf.urls.static import static
 from django.contrib import admin
+from django.contrib.sitemaps.views import sitemap
 from django.contrib.staticfiles.urls import staticfiles_urlpatterns
 from django.urls import include, path
 from django_js_reverse.views import urls_js
 
 from CreeDictionary import views
+from CreeDictionary.sitemaps import sitemaps
 
 urlpatterns = [
     ################################# Primary URLs #################################
@@ -50,6 +52,12 @@
     path("admin/", admin.site.urls),
     path("search-quality/", include("search_quality.urls")),
     path("", include("morphodict.urls")),
+    path(
+        "sitemap.xml",
+        sitemap,
+        {"sitemaps": sitemaps},
+        name="django.contrib.sitemaps.views.sitemap",
+    ),
     ################################# Special URLS #################################
     # Reverse URLs in JavaScript:  https://github.com/ierror/django-js-reverse
     path("jsreverse", urls_js, name="js_reverse"),


Test patch:
diff --git a/CreeDictionary/tests/CreeDictionary_tests/test_sitemap.py b/CreeDictionary/tests/CreeDictionary_tests/test_sitemap.py
new file mode 100644
index 000000000..2642b2f3c
--- /dev/null
+++ b/CreeDictionary/tests/CreeDictionary_tests/test_sitemap.py
@@ -0,0 +1,69 @@
+"""
+Test that the sitemap is accessible and has expected results.
+"""
+
+import random
+import xml.etree.ElementTree as ET
+from http import HTTPStatus
+from urllib.parse import urlparse
+
+import pytest
+from django.urls import reverse
+
+
+@pytest.mark.django_db
+def test_can_access_sitemap(client, sitemap_url):
+    assert sitemap_url == "/sitemap.xml"
+
+    r = client.get(sitemap_url)
+    assert r.status_code == HTTPStatus.OK
+    assert "xml" in r["Content-Type"]
+
+
+@pytest.mark.django_db
+def test_sitemap_has_valid_locations(client, sitemap_url):
+    r = client.get(sitemap_url)
+
+    root = ET.fromstring(r.content)
+    urls = [loc.text for loc in find_all_location_elements(root)]
+
+    # Try visiting a page:
+    url = random.choice(urls)
+    r = client.get(urlparse(url).path, follow=True)
+    assert r.status_code == HTTPStatus.OK, f"unexpected status for {url}"
+
+
+@pytest.fixture
+def sitemap_url():
+    return reverse("django.contrib.sitemaps.views.sitemap")
+
+
+def find_all_location_elements(root):
+    """
+    Given the following markup:
+
+        <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+          <!-- primary site urls -->
+          <url>
+            <loc>http://itwewina.altlab.app/</loc>
+          </url>
+
+          <!-- head word forms -->
+          <url>
+            <loc>http://itwewina.altlab.app/word/atim/</loc>
+          </url>
+        </urlset>
+
+    Returns all <loc> elements:
+    <loc>http://itwewina.altlab.app/</loc>
+    <loc>http://itwewina.altlab.app/word/atim/</loc>
+    """
+    # The ElementTree API is... not great.
+    # Whenever I get an XML document and it uses namespaces,
+    # and I try to query it, I always forget that the tagname has to be QName
+    # like: <http://www.sitemaps.org/schemas/sitemap/0.9:url>
+    # or: <http://www.sitemaps.org/schemas/sitemap/0.9:loc>
+    # this namespace
+    namespace_aliases = {"sitemap": "http://www.sitemaps.org/schemas/sitemap/0.9"}
+    # XPath to return all <loc>s within all <url>s from the current node.
+    return root.findall("./sitemap:url/sitemap:loc", namespace_aliases)