Repository: internetarchive/openlibrary

Issue:
Don't import from Amazon or BWB promise items if `publish_date` is 1900
<!-- IMPORTANT: Before posting, be sure to redact or remove sensitive data, such as passwords, secret keys, session cookies, etc. -->

<!-- A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] -->
Too many Amazon and BWB promise item imports seem to have made up `publish_date` entries of the year 1900.

### Describe the problem that you'd like solved
<!-- A clear and concise description of what you want to happen. -->
We should not add made up publication dates when importing records.

### Proposal & Constraints
<!-- What is the proposed solution / implementation? Is there a precedent of this approach succeeding elsewhere? -->
To avoid adding made up dates to records, for all Amazon and BWB promise item imports, if the `publish_date` is 1900, we should drop the publish date from the record before importing.

<!-- Which suggestions or requirements should be considered for how feature needs to appear or be implemented? -->

### Additional context
<!-- Add any other context or screenshots about the feature request here. -->

### Stakeholders
<!-- @ tag stakeholders of this bug -->
@mekarpeles 




Discussion:

Resolving patch:
diff --git a/openlibrary/catalog/add_book/__init__.py b/openlibrary/catalog/add_book/__init__.py
--- a/openlibrary/catalog/add_book/__init__.py
+++ b/openlibrary/catalog/add_book/__init__.py
@@ -25,7 +25,7 @@
 
 import itertools
 import re
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, Final
 
 import web
 
@@ -67,6 +67,8 @@
 re_normalize = re.compile('[^[:alphanum:] ]', re.U)
 re_lang = re.compile('^/languages/([a-z]{3})$')
 ISBD_UNIT_PUNCT = ' : '  # ISBD cataloging title-unit separator punctuation
+SUSPECT_PUBLICATION_DATES: Final = ["1900", "January 1, 1900"]
+SOURCE_RECORDS_REQUIRING_DATE_SCRUTINY: Final = ["amazon", "bwb", "promise"]
 
 
 type_map = {
@@ -747,6 +749,7 @@ def normalize_import_record(rec: dict) -> None:
         - Cleaning all ISBN and LCCN fields ('bibids');
         - Deduplicate authors; and
         - Remove throw-away data used for validation.
+        - Remove publication years of 1900 for AMZ/BWB/Promise.
 
         NOTE: This function modifies the passed-in rec in place.
     """
@@ -789,6 +792,14 @@ def normalize_import_record(rec: dict) -> None:
     if rec.get('publish_date') == "????":
         rec.pop('publish_date')
 
+    # Remove suspect publication dates from certain sources (e.g. 1900 from Amazon).
+    if any(
+        source_record.split(":")[0] in SOURCE_RECORDS_REQUIRING_DATE_SCRUTINY
+        and rec.get('publish_date') in SUSPECT_PUBLICATION_DATES
+        for source_record in rec['source_records']
+    ):
+        rec.pop('publish_date')
+
 
 def validate_record(rec: dict) -> None:
     """


Test patch:
diff --git a/openlibrary/catalog/add_book/tests/test_add_book.py b/openlibrary/catalog/add_book/tests/test_add_book.py
--- a/openlibrary/catalog/add_book/tests/test_add_book.py
+++ b/openlibrary/catalog/add_book/tests/test_add_book.py
@@ -1617,3 +1617,131 @@ def test_future_publication_dates_are_deleted(self, year, expected):
     def test_dummy_data_to_satisfy_parse_data_is_removed(self, rec, expected):
         normalize_import_record(rec=rec)
         assert rec == expected
+
+    @pytest.mark.parametrize(
+        ["rec", "expected"],
+        [
+            (
+                # 1900 publication from non AMZ/BWB is okay.
+                {
+                    'title': 'a title',
+                    'source_records': ['ia:someid'],
+                    'publishers': ['a publisher'],
+                    'authors': [{'name': 'an author'}],
+                    'publish_date': '1900',
+                },
+                {
+                    'title': 'a title',
+                    'source_records': ['ia:someid'],
+                    'publishers': ['a publisher'],
+                    'authors': [{'name': 'an author'}],
+                    'publish_date': '1900',
+                },
+            ),
+            (
+                # 1900 publication from AMZ disappears.
+                {
+                    'title': 'a title',
+                    'source_records': ['amazon:someid'],
+                    'publishers': ['a publisher'],
+                    'authors': [{'name': 'an author'}],
+                    'publish_date': '1900',
+                },
+                {
+                    'title': 'a title',
+                    'source_records': ['amazon:someid'],
+                    'publishers': ['a publisher'],
+                    'authors': [{'name': 'an author'}],
+                },
+            ),
+            (
+                # 1900 publication from bwb item disappears.
+                {
+                    'title': 'a title',
+                    'source_records': ['bwb:someid'],
+                    'publishers': ['a publisher'],
+                    'authors': [{'name': 'an author'}],
+                    'publish_date': '1900',
+                },
+                {
+                    'title': 'a title',
+                    'source_records': ['bwb:someid'],
+                    'publishers': ['a publisher'],
+                    'authors': [{'name': 'an author'}],
+                },
+            ),
+            (
+                # 1900 publication from promise item disappears.
+                {
+                    'title': 'a title',
+                    'source_records': ['promise:someid'],
+                    'publishers': ['a publisher'],
+                    'authors': [{'name': 'an author'}],
+                    'publish_date': 'January 1, 1900',
+                },
+                {
+                    'title': 'a title',
+                    'source_records': ['promise:someid'],
+                    'publishers': ['a publisher'],
+                    'authors': [{'name': 'an author'}],
+                },
+            ),
+            (
+                # An otherwise valid date from AMZ is okay.
+                {
+                    'title': 'a title',
+                    'source_records': ['amazon:someid'],
+                    'publishers': ['a publisher'],
+                    'authors': [{'name': 'an author'}],
+                    'publish_date': 'January 2, 1900',
+                },
+                {
+                    'title': 'a title',
+                    'source_records': ['amazon:someid'],
+                    'publishers': ['a publisher'],
+                    'authors': [{'name': 'an author'}],
+                    'publish_date': 'January 2, 1900',
+                },
+            ),
+            (
+                # An otherwise valid date from promise is okay.
+                {
+                    'title': 'a title',
+                    'source_records': ['promise:someid'],
+                    'publishers': ['a publisher'],
+                    'authors': [{'name': 'an author'}],
+                    'publish_date': 'January 2, 1900',
+                },
+                {
+                    'title': 'a title',
+                    'source_records': ['promise:someid'],
+                    'publishers': ['a publisher'],
+                    'authors': [{'name': 'an author'}],
+                    'publish_date': 'January 2, 1900',
+                },
+            ),
+            (
+                # Handle records without publish_date.
+                {
+                    'title': 'a title',
+                    'source_records': ['promise:someid'],
+                    'publishers': ['a publisher'],
+                    'authors': [{'name': 'an author'}],
+                },
+                {
+                    'title': 'a title',
+                    'source_records': ['promise:someid'],
+                    'publishers': ['a publisher'],
+                    'authors': [{'name': 'an author'}],
+                },
+            ),
+        ],
+    )
+    def test_year_1900_removed_from_amz_and_bwb_promise_items(self, rec, expected):
+        """
+        A few import sources (e.g. promise items, BWB, and Amazon) have `publish_date`
+        values that are known to be inaccurate, so those `publish_date` values are
+        removed.
+        """
+        normalize_import_record(rec=rec)
+        assert rec == expected