Repository: Princeton-CDH/geniza

Issue:
Server Error 500 - seemingly related to saving documents with footnotes
Many PGP researchers have had issues over the last few days/weeks saving documents on the backend. @rlskoeser posited it might have to do with existing footnotes and that does seem to be the case: 

I was not able to save ID 5650 (with one footnote), but was able to save 20616 when I added a new footnote. 

Within a footnote page, I was able to save.


Discussion:

Resolving patch:
diff --git a/geniza/corpus/models.py b/geniza/corpus/models.py
--- a/geniza/corpus/models.py
+++ b/geniza/corpus/models.py
@@ -640,38 +640,8 @@ def merge_with(self, merge_docs, rationale, user=None):
                 if textblock.fragment not in self.fragments.all():
                     self.textblock_set.add(textblock)
 
-            # combine footnotes
-            current_footnotes = self.footnotes.all()
-            for footnote in doc.footnotes.all():
-                # if footnote is not present (based on footnote equality check), add it
-                if footnote not in current_footnotes:
-                    self.footnotes.add(footnote)
-
-            # reassociate log entries
-            # make a list of currently associated log entries to skip duplicates
-            current_logs = [
-                "%s_%s" % (le.user_id, le.action_time.isoformat())
-                for le in self.log_entries.all()
-            ]
-            for log_entry in doc.log_entries.all():
-                # check duplicate log entries, based on user id and time
-                # (likely only applies to historic input & revision)
-                if (
-                    "%s_%s" % (log_entry.user_id, log_entry.action_time.isoformat())
-                    in current_logs
-                ):
-                    # skip if it's a duplicate
-                    continue
-
-                # otherwise annotate and reassociate
-                # - modify change message to document which object this event applied to
-                log_entry.change_message = "%s [PGPID %d]" % (
-                    log_entry.change_message,
-                    doc.pk,
-                )
-                log_entry.save()
-                # - associate with the primary document
-                self.log_entries.add(log_entry)
+            self._merge_footnotes(doc)
+            self._merge_logentries(doc)
 
         # combine text fields
         self.description = "\n".join(description_chunks)
@@ -698,6 +668,58 @@ def merge_with(self, merge_docs, rationale, user=None):
             action_flag=CHANGE,
         )
 
+    def _merge_footnotes(self, doc):
+        # combine footnotes; footnote logic for merge_with
+        for footnote in doc.footnotes.all():
+            # first, check for an exact match
+            equiv_fn = self.footnotes.includes_footnote(footnote)
+            # if there is no exact match, check again ignoring content
+            if not equiv_fn:
+                equiv_fn = self.footnotes.includes_footnote(
+                    footnote, include_content=False
+                )
+                # if there's a partial match (everything but content)
+                if equiv_fn:
+                    # if the new footnote has content, add it
+                    if footnote.content:
+                        self.footnotes.add(footnote)
+                    # if the partial match has no content, remove it
+                    # (if it has any content, then it is different from the new one
+                    # and should be preserved)
+                    if not equiv_fn.content:
+                        self.footnotes.remove(equiv_fn)
+
+                # if neither an exact or partial match, add the new footnote
+                else:
+                    self.footnotes.add(footnote)
+
+    def _merge_logentries(self, doc):
+        # reassociate log entries; logic for merge_with
+        # make a list of currently associated log entries to skip duplicates
+        current_logs = [
+            "%s_%s" % (le.user_id, le.action_time.isoformat())
+            for le in self.log_entries.all()
+        ]
+        for log_entry in doc.log_entries.all():
+            # check duplicate log entries, based on user id and time
+            # (likely only applies to historic input & revision)
+            if (
+                "%s_%s" % (log_entry.user_id, log_entry.action_time.isoformat())
+                in current_logs
+            ):
+                # skip if it's a duplicate
+                continue
+
+            # otherwise annotate and reassociate
+            # - modify change message to document which object this event applied to
+            log_entry.change_message = "%s [PGPID %d]" % (
+                log_entry.change_message,
+                doc.pk,
+            )
+            log_entry.save()
+            # - associate with the primary document
+            self.log_entries.add(log_entry)
+
 
 class TextBlock(models.Model):
     """The portion of a document that appears on a particular fragment."""
diff --git a/geniza/footnotes/models.py b/geniza/footnotes/models.py
--- a/geniza/footnotes/models.py
+++ b/geniza/footnotes/models.py
@@ -174,6 +174,32 @@ def all_authors(self):
     all_authors.admin_order_field = "first_author"  # set in admin queryset
 
 
+class FootnoteQuerySet(models.QuerySet):
+    def includes_footnote(self, other, include_content=True):
+        """Check if the current queryset includes a match for the
+        specified footnotes. Matches are made by comparing content source,
+        location, document relation type, notes, and content (ignores
+        associated content object). To ignore content when comparing
+        footnotes, specify `include_content=False`.
+        Returns the matching object if there was one, or False if not."""
+
+        compare_fields = ["source", "location", "notes"]
+        # optionally include content when comparing; include by default
+        if include_content:
+            compare_fields.append("content")
+
+        for fn in self.all():
+            if (
+                all(getattr(fn, val) == getattr(other, val) for val in compare_fields)
+                # NOTE: fn.doc_relation seems to be different on queryset footnote
+                # than newly created object; check by display value to avoid problems
+                and fn.get_doc_relation_display() == other.get_doc_relation_display()
+            ):
+                return fn
+
+        return False
+
+
 class Footnote(models.Model):
     source = models.ForeignKey(Source, on_delete=models.CASCADE)
     location = models.CharField(
@@ -214,6 +240,9 @@ class Footnote(models.Model):
     object_id = GfkLookupField("content_type")
     content_object = GenericForeignKey()
 
+    # replace default queryset with customized version
+    objects = FootnoteQuerySet.as_manager()
+
     class Meta:
         ordering = ["source", "location"]
 
@@ -223,15 +252,6 @@ def __str__(self):
         rel = " and ".join([str(choices[c]) for c in self.doc_relation]) or "Footnote"
         return f"{rel} of {self.content_object}"
 
-    def __eq__(self, other):
-        """Footnotes should be considered equal if content source,
-        location, document relation type, notes, and content all match.
-        Does not include associated content object in comparison check."""
-        return all(
-            getattr(self, val) == getattr(other, val)
-            for val in ["source", "location", "doc_relation", "notes", "content"]
-        )
-
     def has_transcription(self):
         """Admin display field indicating presence of digitized transcription."""
         return bool(self.content)


Test patch:
diff --git a/geniza/corpus/tests/test_corpus_models.py b/geniza/corpus/tests/test_corpus_models.py
--- a/geniza/corpus/tests/test_corpus_models.py
+++ b/geniza/corpus/tests/test_corpus_models.py
@@ -615,6 +615,23 @@ def test_document_merge_with_footnotes(document, join, source):
     assert document.footnotes.count() == 2
 
 
+def test_document_merge_with_footnotes_transcription(document, join, source):
+    # create some footnotes
+    Footnote.objects.create(content_object=document, source=source, location="p. 3")
+    # page 3 footnote is a near duplicate but adds content
+    Footnote.objects.create(
+        content_object=join, source=source, location="p. 3", content="{'foo': 'bar'}"
+    )
+
+    assert document.footnotes.count() == 1
+    assert join.footnotes.count() == 1
+    document.merge_with([join], "combine footnotes")
+    # should only have one footnotes after the merge
+    assert document.footnotes.count() == 1
+    # should preserve the footnote with content and remove the one without
+    assert document.footnotes.first().content
+
+
 def test_document_merge_with_log_entries(document, join):
     # create some log entries
     document_contenttype = ContentType.objects.get_for_model(Document)
diff --git a/geniza/footnotes/tests/test_footnote_models.py b/geniza/footnotes/tests/test_footnote_models.py
--- a/geniza/footnotes/tests/test_footnote_models.py
+++ b/geniza/footnotes/tests/test_footnote_models.py
@@ -122,35 +122,83 @@ def test_has_url(self, source):
         footnote.url = "http://example.com/"
         assert footnote.has_url()
 
+
+class TestFootnoteQuerySet:
     @pytest.mark.django_db
-    def test_eq(self, source, twoauthor_source, document):
+    def test_includes_footnote(self, source, twoauthor_source, document):
         # same source, content object, location
-        footnote1 = Footnote(source=source, content_object=document, location="p.1")
-        footnote2 = Footnote(source=source, content_object=document, location="p.1")
-        assert footnote1 == footnote2
+        footnote1 = Footnote.objects.create(
+            source=source,
+            content_object=document,
+            location="p.1",
+            doc_relation=Footnote.EDITION,
+        )
+        footnote2 = Footnote.objects.create(
+            source=source,
+            content_object=document,
+            location="p.1",
+            doc_relation=Footnote.EDITION,
+        )
+        assert (
+            Footnote.objects.filter(pk=footnote1.pk).includes_footnote(footnote2)
+            == footnote1
+        )
 
         # different location
         footnote2.location = "p.5"
+        footnote2.save()
         assert footnote1 != footnote2
+        assert not Footnote.objects.filter(pk=footnote1.pk).includes_footnote(footnote2)
 
         # different source
         footnote2.location = "p.1"
         footnote2.source = twoauthor_source
-        assert footnote1 != footnote2
+        footnote2.save()
+        assert not Footnote.objects.filter(pk=footnote1.pk).includes_footnote(footnote2)
 
         # different content object -- considered equal if all else is equal!
         footnote2.source = source
         footnote2.content_object = twoauthor_source
-        assert footnote1 == footnote2
+        footnote2.save()
+        assert (
+            Footnote.objects.filter(pk=footnote1.pk).includes_footnote(footnote2)
+            == footnote1
+        )
 
         # different notes
         footnote2.notes = "some extra info"
-        assert footnote1 != footnote2
+        footnote2.save()
+        assert not Footnote.objects.filter(pk=footnote1.pk).includes_footnote(footnote2)
 
         # different content
         footnote2.notes = ""
         footnote2.content = "{}"
-        assert footnote1 != footnote2
+        footnote2.save()
+        assert not Footnote.objects.filter(pk=footnote1.pk).includes_footnote(footnote2)
+
+    @pytest.mark.django_db
+    def test_includes_footnote_ignore_content(self, source, twoauthor_source, document):
+        # same source, content object, location; one with content
+        footnote1 = Footnote.objects.create(
+            source=source,
+            content_object=document,
+            location="p.1",
+            doc_relation=Footnote.EDITION,
+        )
+        footnote2 = Footnote.objects.create(
+            source=source,
+            content_object=document,
+            location="p.1",
+            doc_relation=Footnote.EDITION,
+            content="{'foo': 'bar'}",
+        )
+        assert not Footnote.objects.filter(pk=footnote1.pk).includes_footnote(footnote2)
+        assert (
+            Footnote.objects.filter(pk=footnote1.pk).includes_footnote(
+                footnote2, include_content=False
+            )
+            == footnote1
+        )
 
 
 class TestCreator: