Repository: internetarchive/openlibrary

Issue:
Adds pre-check logic to affiliate-server amz imports
Only attempt to queue amz records with new data or for editions we don't yet have OL records for import

<!-- What issue does this PR close? -->
Closes #7245  which is required for #7184 
Depends on #7316 which closes #7244 


<!-- What does this PR achieve? [feature|hotfix|fix|refactor] -->


### Technical
<!-- What should be noted about the implementation? -->

This code likely needs to be cleaned up, moved to a function, and possible done in different thread if there are concerns about blocking time.

### Testing
<!-- Steps for reviewer to reproduce/verify what this PR does/fixes. -->

See https://github.com/internetarchive/openlibrary/issues/7184#issuecomment-1362439106 which has very thorough test instructions.

### Screenshot
<!-- If this PR touches UI, please post evidence (screenshots) of it behaving correctly. -->

### Stakeholders
<!-- @ tag stakeholders of this bug -->


<!-- Attribution Disclaimer: By proposing this pull request, I affirm to have made a best-effort and exercised my discretion to make sure relevant sections of this code which substantially leverage code suggestions, code generation, or code snippets from sources (e.g. Stack Overflow, GitHub) have been annotated with basic attribution so reviewers & contributors may have confidence and access to the correct context to evaluate and use this code. -->



Discussion:

Resolving patch:
diff --git a/scripts/affiliate_server.py b/scripts/affiliate_server.py
--- a/scripts/affiliate_server.py
+++ b/scripts/affiliate_server.py
@@ -17,6 +17,7 @@
 
 """
 
+import itertools
 import json
 import logging
 import os
@@ -55,6 +56,14 @@
 
 API_MAX_ITEMS_PER_CALL = 10
 API_MAX_WAIT_SECONDS = 0.9
+AZ_OL_MAP = {
+    'cover': 'covers',
+    'title': 'title',
+    'authors': 'authors',
+    'publishers': 'publishers',
+    'publish_date': 'publish_date',
+    'number_of_pages': 'number_of_pages',
+}
 
 batch_name = ""
 batch: Batch | None = None
@@ -77,12 +86,71 @@ def get_current_amazon_batch() -> Batch:
     return batch
 
 
+def get_isbns_from_book(book: dict) -> list[str]:  # Singular: book
+    return [str(isbn) for isbn in book.get('isbn_10', []) + book.get('isbn_13', [])]
+
+
+def get_isbns_from_books(books: list[dict]) -> list[str]:  # Plural: books
+    return sorted(set(itertools.chain(*[get_isbns_from_book(book) for book in books])))
+
+
+def is_book_needed(book: dict, edition: dict) -> bool:
+    """
+    Should an OL edition's metadata be updated with Amazon book data?
+
+    :param book: dict from openlibrary.core.vendors.clean_amazon_metadata_for_load()
+    :param edition: dict from web.ctx.site.get_many(edition_ids)
+    """
+    needed_book_fields = []  # book fields that should be copied to the edition
+    for book_field, edition_field in AZ_OL_MAP.items():
+        if field_value := book.get(book_field) and not edition.get(edition_field):
+            needed_book_fields.append(book_field)
+    if needed_book_fields:  # Log book fields that should to be copied to the edition
+        fields = ", ".join(needed_book_fields)
+        logger.debug(f"{edition.get('key', None) or 'New Edition'} needs {fields}")
+    return bool(needed_book_fields)
+
+
+def get_editions_for_books(books: list[dict]) -> list[dict]:
+    """
+    Get the OL editions for a list of ISBNs.
+
+    :param isbns: list of book dicts
+    :return: list of OL editions dicts
+    """
+    isbns = get_isbns_from_books(books)
+    unique_edition_ids = set(
+        web.ctx.site.things({'type': '/type/edition', 'isbn_': isbns})
+    )
+    return web.ctx.site.get_many(list(unique_edition_ids))
+
+
+def get_pending_books(books):
+    pending_books = []
+    editions = get_editions_for_books(books)  # Make expensive call just once
+    # For each amz book, check that we need its data
+    for book in books:
+        ed = next(
+            (
+                ed
+                for ed in editions
+                if set(book.get('isbn_13')).intersection(set(ed.isbn_13))
+                or set(book.get('isbn_10')).intersection(set(ed.isbn_10))
+            ),
+            {},
+        )
+        if is_book_needed(book, ed):
+            pending_books.append(book)
+    return pending_books
+
+
 def process_amazon_batch(isbn_10s: list[str]) -> None:
     """
     Call the Amazon API to get the products for a list of isbn_10s and store
     each product in memcache using amazon_product_{isbn_13} as the cache key.
     """
     logger.info(f"process_amazon_batch(): {len(isbn_10s)} items")
+    assert cache  # mypy  workaround for `cache or None`
     try:
         products = web.amazon_api.get_products(isbn_10s, serialize=True)
     except Exception:
@@ -95,7 +163,6 @@ def process_amazon_batch(isbn_10s: list[str]) -> None:
             and product.get('isbn_13')[0]
             or isbn_10_to_isbn_13(product.get('isbn_10')[0])
         )
-        assert cache
         cache.memcache_cache.set(  # Add each product to memcache
             f'amazon_product_{cache_key}', product, expires=WEEK_SECS
         )
@@ -105,11 +172,11 @@ def process_amazon_batch(isbn_10s: list[str]) -> None:
         logger.debug("DB parameters missing from affiliate-server infobase")
         return
 
-    # XXX temporarily disable until covers + substantive changes added
-    # if books := [clean_amazon_metadata_for_load(product) for product in products]:
-    #    get_current_amazon_batch().add_items(
-    #        [{'ia_id': b['source_records'][0], 'data': b} for b in books]
-    #    )
+    if books := [clean_amazon_metadata_for_load(product) for product in products]:
+        if pending_books := get_pending_books(books):
+            get_current_amazon_batch().add_items(
+                [{'ia_id': b['source_records'][0], 'data': b} for b in pending_books]
+            )
 
 
 def seconds_remaining(start_time: float) -> float:
@@ -137,7 +204,8 @@ def amazon_lookup() -> None:
             process_amazon_batch(list(isbn_10s))
 
 
-threading.Thread(target=amazon_lookup).start()
+if "pytest" not in sys.modules:
+    threading.Thread(target=amazon_lookup).start()
 
 
 class Status:
@@ -179,7 +247,8 @@ def GET(self, isbn: str) -> str:
         If isbn is in memcache then return the `hit`.  If not then queue the isbn to be
         looked up and return the equivalent of a promise as `submitted`
         """
-        if not web.amazon_api:
+        # cache could be None if reached before initialized (mypy)
+        if not web.amazon_api or not cache:
             return json.dumps({"error": "not_configured"})
 
         isbn10, isbn13 = self.unpack_isbn(isbn)
@@ -189,7 +258,6 @@ def GET(self, isbn: str) -> str:
             )
 
         # Cache lookup by isbn13. If there's a hit return the product to the caller
-        assert cache
         if product := cache.memcache_cache.get(f'amazon_product_{isbn13}'):
             return json.dumps({"status": "success", "hit": product})
 


Test patch:
diff --git a/scripts/tests/test_affiliate_server.py b/scripts/tests/test_affiliate_server.py
new file mode 100644
--- /dev/null
+++ b/scripts/tests/test_affiliate_server.py
@@ -0,0 +1,120 @@
+"""
+Requires pytest-mock to be installed: `pip install pytest-mock`
+for access to the mocker fixture.
+
+# docker-compose run --rm home pytest scripts/tests/test_affiliate_server.py
+"""
+import sys
+from unittest.mock import MagicMock
+
+# TODO: Can we remove _init_path someday :(
+sys.modules['_init_path'] = MagicMock()
+
+from openlibrary.mocks.mock_infobase import mock_site  # noqa: F401
+from scripts.affiliate_server import (  # noqa: E402
+    get_isbns_from_book,
+    get_isbns_from_books,
+    get_editions_for_books,
+    get_pending_books,
+)
+
+ol_editions = {
+    f"123456789{i}": {
+        "type": "/type/edition",
+        "key": f"/books/OL{i}M",
+        "isbn_10": [f"123456789{i}"],
+        "isbn_13": [f"123456789012{i}"],
+        "covers": [int(f"1234567{i}")],
+        "title": f"Book {i}",
+        "authors": [{"key": f"/authors/OL{i}A"}],
+        "publishers": [f"Publisher {i}"],
+        "publish_date": f"Aug 0{i}, 2023",
+        "number_of_pages": int(f"{i}00"),
+    }
+    for i in range(8)
+}
+ol_editions["1234567891"].pop("covers")
+ol_editions["1234567892"].pop("title")
+ol_editions["1234567893"].pop("authors")
+ol_editions["1234567894"].pop("publishers")
+ol_editions["1234567895"].pop("publish_date")
+ol_editions["1234567896"].pop("number_of_pages")
+
+amz_books = {
+    f"123456789{i}": {
+        "isbn_10": [f"123456789{i}"],
+        "isbn_13": [f"12345678901{i}"],
+        "cover": [int(f"1234567{i}")],
+        "title": f"Book {i}",
+        "authors": [{'name': f"Last_{i}a, First"}, {'name': f"Last_{i}b, First"}],
+        "publishers": [f"Publisher {i}"],
+        "publish_date": f"Aug 0{i}, 2023",
+        "number_of_pages": int(f"{i}00"),
+    }
+    for i in range(8)
+}
+
+
+def test_ol_editions_and_amz_books():
+    assert len(ol_editions) == len(amz_books) == 8
+
+
+def test_get_editions_for_books(mock_site):  # noqa: F811
+    """
+    Attempting to save many ol editions and then get them back...
+    """
+    start = len(mock_site.docs)
+    mock_site.save_many(ol_editions.values())
+    assert len(mock_site.docs) - start == len(ol_editions)
+    editions = get_editions_for_books(amz_books.values())
+    assert len(editions) == len(ol_editions)
+    assert sorted(edition.key for edition in editions) == [
+        f"/books/OL{i}M" for i in range(8)
+    ]
+
+
+def test_get_pending_books(mock_site):  # noqa: F811
+    """
+    Testing get_pending_books() with no ol editions saved and then with ol editions.
+    """
+    # All books will be pending if they have no corresponding ol editions
+    assert len(get_pending_books(amz_books.values())) == len(amz_books)
+    # Save corresponding ol editions into the mock site
+    start = len(mock_site.docs)
+    mock_site.save_many(ol_editions.values())  # Save the ol editions
+    assert len(mock_site.docs) - start == len(ol_editions)
+    books = get_pending_books(amz_books.values())
+    assert len(books) == 6  # Only 6 books are missing covers, titles, authors, etc.
+
+
+def test_get_isbns_from_book():
+    """
+    Testing get_isbns_from_book() with a book that has both isbn_10 and isbn_13.
+    """
+    book = {
+        "isbn_10": ["1234567890"],
+        "isbn_13": ["1234567890123"],
+    }
+    assert get_isbns_from_book(book) == ["1234567890", "1234567890123"]
+
+
+def test_get_isbns_from_books():
+    """
+    Testing get_isbns_from_books() with a list of books that have both isbn_10 and isbn_13.
+    """
+    books = [
+        {
+            "isbn_10": ["1234567890"],
+            "isbn_13": ["1234567890123"],
+        },
+        {
+            "isbn_10": ["1234567891"],
+            "isbn_13": ["1234567890124"],
+        },
+    ]
+    assert get_isbns_from_books(books) == [
+        '1234567890',
+        '1234567890123',
+        '1234567890124',
+        '1234567891',
+    ]