Repository: DDMAL/CantusDB

Issue:
Search should ignore diacritics
In an email from Debra:

> I'm in a session at IAML and just realized that the books from Kraków (that the speaker is highlighting as being in Cantus) only come up when I search with ó.
> 
> I was lazy in my first search and just entered Krakow.
> Is a search that finds both (or all) versions something easy to add to the functionality of the site?
> 
> There are other place names with accents, so we would want to apply this type of search to more than just the Krakow sources.

We should make sure that searches throughout the site ignore diacritic marks (i.e. accents) on characters.


Discussion:
https://docs.djangoproject.com/en/4.2/ref/contrib/postgres/lookups/#unaccent
@annamorphism - we'll never need to do this for chants or sequences, correct? It seems that when you're working with really big database tables (like our 500000 chants), this unaccent tool can affect performance. So if we only need to worry about sources, that will be a good thing for keeping other parts of the site loading quickly.
This is a good question actually.  Cantus Index has a (still very small) number of texts with lots of accents, taken from a Czech database (see e.g. https://cantusindex.org/text-search?text=svat) . DACT is interested in vernacular translations of chant, so there's a possibility that texts with accents will enter CantusDB in the future. (Will there ever be non-Roman alphabets? Hmmm.)

HOWEVER I think it is probably (?) fair to expect people searching for Czech chants to use correct Czech accents in their search terms, whereas one-word place names are maybe less fair game. (In this case "Krakow" is a valid alternate spelling, too--in an ideal world we would have it in a place index along with Cracow, Krakau, Cracovia, which would come closer to solving the real problem.) So,  I don't think we need it for chants--there may come a time when it is needed for dealing with chants in other languages, but that is not any time soon.

Resolving patch:
diff --git a/django/cantusdb_project/cantusdb/settings.py b/django/cantusdb_project/cantusdb/settings.py
--- a/django/cantusdb_project/cantusdb/settings.py
+++ b/django/cantusdb_project/cantusdb/settings.py
@@ -56,6 +56,7 @@
     "django.contrib.sites",
     "django.contrib.flatpages",
     "django.contrib.humanize",
+    "django.contrib.postgres",
     "extra_views",
     "main_app",
     "articles",
diff --git a/django/cantusdb_project/main_app/migrations/0002_unaccent.py b/django/cantusdb_project/main_app/migrations/0002_unaccent.py
new file mode 100644
--- /dev/null
+++ b/django/cantusdb_project/main_app/migrations/0002_unaccent.py
@@ -0,0 +1,12 @@
+from django.contrib.postgres.operations import UnaccentExtension
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("main_app", "0001_initial"),
+    ]
+
+    operations = [
+        UnaccentExtension(),
+    ]
diff --git a/django/cantusdb_project/main_app/views/source.py b/django/cantusdb_project/main_app/views/source.py
--- a/django/cantusdb_project/main_app/views/source.py
+++ b/django/cantusdb_project/main_app/views/source.py
@@ -122,13 +122,13 @@ def get_queryset(self):
             # field, allowing for a more flexible search, and a field needs
             # to match only one of the terms
             for term in general_search_terms:
-                title_q |= Q(title__icontains=term)
-                siglum_q |= Q(siglum__icontains=term)
-                rism_siglum_q |= Q(rism_siglum__name__icontains=term) | Q(
-                    rism_siglum__description__icontains=term
+                title_q |= Q(title__unaccent__icontains=term)
+                siglum_q |= Q(siglum__unaccent__icontains=term)
+                rism_siglum_q |= Q(rism_siglum__name__unaccent__icontains=term) | Q(
+                    rism_siglum__description__unaccent__icontains=term
                 )
-                description_q |= Q(description__icontains=term)
-                summary_q |= Q(summary__icontains=term)
+                description_q |= Q(description__unaccent__icontains=term)
+                summary_q |= Q(summary__unaccent__icontains=term)
                 # provenance_q |= Q(provenance__name__icontains=term)
             # All the Q objects are put together with OR.
             # The end result is that at least one term has to match in at least one


Test patch:
diff --git a/django/cantusdb_project/main_app/tests/make_fakes.py b/django/cantusdb_project/main_app/tests/make_fakes.py
--- a/django/cantusdb_project/main_app/tests/make_fakes.py
+++ b/django/cantusdb_project/main_app/tests/make_fakes.py
@@ -43,6 +43,36 @@ def make_random_string(
     return "".join(characters[random.randrange(len(characters))] for _ in range(length))
 
 
+def add_accents_to_string(s: str) -> str:
+    """Replace some letters in a string
+    with their accented versions.
+
+    Args:
+        s (str): A string
+
+    Returns:
+        str: The same string, but with vowels, c's and n's
+            replaced with accented versions of the same letter
+    """
+    accented_string = (
+        s.replace("a", "à")
+        .replace("e", "é")
+        .replace("i", "ï")
+        .replace("o", "ô")
+        .replace("u", "ū")
+        .replace("c", "ç")
+        .replace("n", "ñ")
+        .replace("A", "À")
+        .replace("E", "É")
+        .replace("I", "Ï")
+        .replace("O", "Ô")
+        .replace("U", "Ū")
+        .replace("C", "Ç")
+        .replace("N", "Ñ")
+    )
+    return accented_string
+
+
 def make_fake_volpiano(
     words: int = 5,
     syllables_per_word: int = 2,
diff --git a/django/cantusdb_project/main_app/tests/test_views.py b/django/cantusdb_project/main_app/tests/test_views.py
--- a/django/cantusdb_project/main_app/tests/test_views.py
+++ b/django/cantusdb_project/main_app/tests/test_views.py
@@ -29,6 +29,7 @@
     make_fake_source,
     make_fake_volpiano,
     make_random_string,
+    add_accents_to_string,
 )
 
 from main_app.models import Century
@@ -4198,6 +4199,14 @@ def test_search_by_title(self):
         response = self.client.get(reverse("source-list"), {"general": search_term})
         self.assertIn(source, response.context["sources"])
 
+        # Test that postgres searches unaccented version of title
+        unaccented_title = source.title
+        accented_title = add_accents_to_string(unaccented_title)
+        source.title = accented_title
+        source.save()
+        response = self.client.get(reverse("source-list"), {"general": search_term})
+        self.assertIn(source, response.context["sources"])
+
     def test_search_by_siglum(self):
         source = make_fake_source(
             siglum=make_random_string(6),
@@ -4208,6 +4217,14 @@ def test_search_by_siglum(self):
         response = self.client.get(reverse("source-list"), {"general": search_term})
         self.assertIn(source, response.context["sources"])
 
+        # Test that postgres searches unaccented version of siglum
+        unaccented_siglum = source.siglum
+        accented_siglum = add_accents_to_string(unaccented_siglum)
+        source.siglum = accented_siglum
+        source.save()
+        response = self.client.get(reverse("source-list"), {"general": search_term})
+        self.assertIn(source, response.context["sources"])
+
     def test_search_by_rism_siglum_name(self):
         rism_siglum = make_fake_rism_siglum()
         source = make_fake_source(
@@ -4219,6 +4236,14 @@ def test_search_by_rism_siglum_name(self):
         response = self.client.get(reverse("source-list"), {"general": search_term})
         self.assertIn(source, response.context["sources"])
 
+        # Test that postgres searches unaccented version of RISM siglum name
+        unaccented_name = rism_siglum.name
+        accented_name = add_accents_to_string(unaccented_name)
+        rism_siglum.name = accented_name
+        rism_siglum.save()
+        response = self.client.get(reverse("source-list"), {"general": search_term})
+        self.assertIn(source, response.context["sources"])
+
     def test_search_by_rism_siglum_description(self):
         rism_siglum = make_fake_rism_siglum()
         source = make_fake_source(
@@ -4230,6 +4255,14 @@ def test_search_by_rism_siglum_description(self):
         response = self.client.get(reverse("source-list"), {"general": search_term})
         self.assertIn(source, response.context["sources"])
 
+        # Test that postgres searches unaccented version of RISM siglum description
+        unaccented_description = rism_siglum.description
+        accented_description = add_accents_to_string(unaccented_description)
+        rism_siglum.description = accented_description
+        rism_siglum.save()
+        response = self.client.get(reverse("source-list"), {"general": search_term})
+        self.assertIn(source, response.context["sources"])
+
     def test_search_by_description(self):
         source = make_fake_source(
             description=faker.sentence(),
@@ -4240,6 +4273,14 @@ def test_search_by_description(self):
         response = self.client.get(reverse("source-list"), {"general": search_term})
         self.assertIn(source, response.context["sources"])
 
+        # Test that postgres searches unaccented version of description
+        unaccented_description = source.description
+        accented_description = add_accents_to_string(unaccented_description)
+        source.title = accented_description
+        source.save()
+        response = self.client.get(reverse("source-list"), {"general": search_term})
+        self.assertIn(source, response.context["sources"])
+
     def test_search_by_summary(self):
         source = make_fake_source(
             summary=faker.sentence(),
@@ -4250,6 +4291,14 @@ def test_search_by_summary(self):
         response = self.client.get(reverse("source-list"), {"general": search_term})
         self.assertIn(source, response.context["sources"])
 
+        # Test that postgres searches unaccented version of summary
+        unaccented_summary = source.summary
+        accented_summary = add_accents_to_string(unaccented_summary)
+        source.title = accented_summary
+        source.save()
+        response = self.client.get(reverse("source-list"), {"general": search_term})
+        self.assertIn(source, response.context["sources"])
+
     def test_search_by_indexing_notes(self):
         """The "indexing notes" field searches in `indexing_notes` and indexer/editor related fields"""
         source = make_fake_source(
@@ -4261,6 +4310,14 @@ def test_search_by_indexing_notes(self):
         response = self.client.get(reverse("source-list"), {"indexing": search_term})
         self.assertIn(source, response.context["sources"])
 
+        # Test that postgres searches unaccented version of indexing_notes
+        unaccented_indexing_notes = source.indexing_notes
+        accented_indexing_notes = add_accents_to_string(unaccented_indexing_notes)
+        source.title = accented_indexing_notes
+        source.save()
+        response = self.client.get(reverse("source-list"), {"general": search_term})
+        self.assertIn(source, response.context["sources"])
+
 
 class SourceCreateViewTest(TestCase):
     @classmethod