Repository: internetarchive/openlibrary

Issue:
Imports should de-duplicate fields on import
<!-- IMPORTANT: Before posting, be sure to redact or remove sensitive data, such as passwords, secret keys, session cookies, etc. -->
<!-- What problem are we solving? What does the experience look like today? What are the symptoms? -->
Imports should (case insensitively) de-duplicate all fields on import. Currently, they do not, so it's possible that, for example, re-import will add duplicate subjects, which accumulate in a `Work`. Then, when manually saved through the UI, the `Work` *is* de-duplicated.

This makes the edit history look quite strange, as someone might add an ISBN via the UI, save, and then notice a number of subjects (which were actually duplicates) were removed from the record.

### Proposal & Constraints
<!-- What is the proposed solution / implementation? Is there a precedent of this approach succeeding elsewhere? -->
Make the UI and the import system use shared logic for de-duplicating fields on save/import.

### Stakeholders
<!-- @ tag stakeholders of this bug -->
@mekarpeles 



Discussion:

Resolving patch:
diff --git a/openlibrary/catalog/add_book/__init__.py b/openlibrary/catalog/add_book/__init__.py
--- a/openlibrary/catalog/add_book/__init__.py
+++ b/openlibrary/catalog/add_book/__init__.py
@@ -22,6 +22,7 @@
     response = load(record)
 
 """
+import itertools
 import re
 from typing import TYPE_CHECKING, Any
 
@@ -849,7 +850,7 @@ def update_edition_with_rec_data(
         edition['ocaid'] = rec['ocaid']
         need_edition_save = True
 
-    # Add list fields to edition as needed
+    # Fields which have their VALUES added if absent.
     edition_list_fields = [
         'local_id',
         'lccn',
@@ -863,14 +864,16 @@ def update_edition_with_rec_data(
         # ensure values is a list
         values = rec[f] if isinstance(rec[f], list) else [rec[f]]
         if f in edition:
-            # get values from rec that are not currently on the edition
-            to_add = [v for v in values if v not in edition[f]]
+            # get values from rec field that are not currently on the edition
+            case_folded_values = {v.casefold() for v in edition[f]}
+            to_add = [v for v in values if v.casefold() not in case_folded_values]
             edition[f] += to_add
         else:
             edition[f] = to_add = values
         if to_add:
             need_edition_save = True
 
+    # Fields that are added as a whole if absent. (Individual values are not added.)
     other_edition_fields = [
         'description',
         'number_of_pages',
@@ -908,13 +911,15 @@ def update_work_with_rec_data(
     """
     # Add subjects to work, if not already present
     if 'subjects' in rec:
-        work_subjects = list(work.get('subjects', []))
-        for s in rec['subjects']:
-            if s not in work_subjects:
-                work_subjects.append(s)
-                need_work_save = True
-        if need_work_save and work_subjects:
-            work['subjects'] = work_subjects
+        work_subjects: list[str] = list(work.get('subjects', []))
+        rec_subjects: list[str] = rec.get('subjects', [])
+        deduped_subjects = uniq(
+            itertools.chain(work_subjects, rec_subjects), lambda item: item.casefold()
+        )
+
+        if work_subjects != deduped_subjects:
+            work['subjects'] = deduped_subjects
+            need_work_save = True
 
     # Add cover to work, if needed
     if not work.get('covers') and edition.get_covers():
diff --git a/openlibrary/plugins/upstream/addbook.py b/openlibrary/plugins/upstream/addbook.py
--- a/openlibrary/plugins/upstream/addbook.py
+++ b/openlibrary/plugins/upstream/addbook.py
@@ -753,9 +753,9 @@ def read_subject(subjects):
             f = io.StringIO(subjects.replace('\r\n', ''))
             dedup = set()
             for s in next(csv.reader(f, dialect='excel', skipinitialspace=True)):
-                if s.lower() not in dedup:
+                if s.casefold() not in dedup:
                     yield s
-                    dedup.add(s.lower())
+                    dedup.add(s.casefold())
 
         work.subjects = list(read_subject(work.get('subjects', '')))
         work.subject_places = list(read_subject(work.get('subject_places', '')))


Test patch:
diff --git a/openlibrary/catalog/add_book/tests/test_add_book.py b/openlibrary/catalog/add_book/tests/test_add_book.py
--- a/openlibrary/catalog/add_book/tests/test_add_book.py
+++ b/openlibrary/catalog/add_book/tests/test_add_book.py
@@ -1129,6 +1129,72 @@ def test_add_description_to_work(mock_site) -> None:
     assert e.works[0]['description'] == 'An added description from an existing edition'
 
 
+def test_add_subjects_to_work_deduplicates(mock_site) -> None:
+    """
+    Ensure a rec's subjects, after a case insensitive check, are added to an
+    existing Work if not already present.
+    """
+    author = {
+        'type': {'key': '/type/author'},
+        'name': 'John Smith',
+        'key': '/authors/OL1A',
+    }
+
+    existing_work = {
+        'authors': [{'author': '/authors/OL1A', 'type': {'key': '/type/author_role'}}],
+        'key': '/works/OL1W',
+        'subjects': ['granite', 'GRANITE', 'Straße', 'ΠΑΡΆΔΕΙΣΟΣ'],
+        'title': 'Some Title',
+        'type': {'key': '/type/work'},
+    }
+
+    existing_edition = {
+        'key': '/books/OL1M',
+        'title': 'Some Title',
+        'publishers': ['Black Spot'],
+        'type': {'key': '/type/edition'},
+        'source_records': ['non-marc:test'],
+        'publish_date': 'Jan 09, 2011',
+        'isbn_10': ['1250144051'],
+        'works': [{'key': '/works/OL1W'}],
+    }
+
+    mock_site.save(author)
+    mock_site.save(existing_work)
+    mock_site.save(existing_edition)
+
+    rec = {
+        'authors': [{'name': 'John Smith'}],
+        'isbn_10': ['1250144051'],
+        'publish_date': 'Jan 09, 2011',
+        'publishers': ['Black Spot'],
+        'source_records': 'non-marc:test',
+        'subjects': [
+            'granite',
+            'Granite',
+            'SANDSTONE',
+            'sandstone',
+            'strasse',
+            'παράδεισος',
+        ],
+        'title': 'Some Title',
+    }
+
+    reply = load(rec)
+    assert reply['success'] is True
+    assert reply['edition']['status'] == 'matched'
+    assert reply['work']['status'] == 'modified'
+    assert reply['work']['key'] == '/works/OL1W'
+    w = mock_site.get(reply['work']['key'])
+
+    def get_casefold(item_list: list[str]):
+        return [item.casefold() for item in item_list]
+
+    expected = ['granite', 'Straße', 'ΠΑΡΆΔΕΙΣΟΣ', 'sandstone']
+    got = w.subjects
+    assert get_casefold(got) == get_casefold(expected)
+
+
 def test_add_identifiers_to_edition(mock_site) -> None:
     """
     Ensure a rec's identifiers that are not present in a matched edition are
@@ -1182,6 +1248,61 @@ def test_add_identifiers_to_edition(mock_site) -> None:
     assert e.identifiers._data == {'goodreads': ['1234'], 'librarything': ['5678']}
 
 
+def test_adding_list_field_items_to_edition_deduplicates_input(mock_site) -> None:
+    """
+    Ensure a rec's edition_list_fields that are not present in a matched
+    edition are added to that matched edition.
+    """
+    author = {
+        'type': {'key': '/type/author'},
+        'name': 'John Smith',
+        'key': '/authors/OL1A',
+    }
+
+    existing_work = {
+        'authors': [{'author': '/authors/OL1A', 'type': {'key': '/type/author_role'}}],
+        'key': '/works/OL1W',
+        'title': 'Some Title',
+        'type': {'key': '/type/work'},
+    }
+
+    existing_edition = {
+        'isbn_10': ['1250144051'],
+        'key': '/books/OL1M',
+        'lccn': ['agr25000003'],
+        'publish_date': 'Jan 09, 2011',
+        'publishers': ['Black Spot'],
+        'source_records': ['non-marc:test'],
+        'title': 'Some Title',
+        'type': {'key': '/type/edition'},
+        'works': [{'key': '/works/OL1W'}],
+    }
+
+    mock_site.save(author)
+    mock_site.save(existing_work)
+    mock_site.save(existing_edition)
+
+    rec = {
+        'authors': [{'name': 'John Smith'}],
+        'isbn_10': ['1250144051'],
+        'lccn': ['AGR25000003', 'AGR25-3'],
+        'publish_date': 'Jan 09, 2011',
+        'publishers': ['Black Spot', 'Second Publisher'],
+        'source_records': ['NON-MARC:TEST', 'ia:someid'],
+        'title': 'Some Title',
+    }
+
+    reply = load(rec)
+    assert reply['success'] is True
+    assert reply['edition']['status'] == 'modified'
+    assert reply['work']['status'] == 'matched'
+    assert reply['work']['key'] == '/works/OL1W'
+    e = mock_site.get(reply['edition']['key'])
+    assert e.works[0]['key'] == '/works/OL1W'
+    assert e.lccn == ['agr25000003']
+    assert e.source_records == ['non-marc:test', 'ia:someid']
+
+
 @pytest.mark.parametrize(
     'name, rec, error',
     [