Repository: internetarchive/openlibrary

Issue:
Affiliate Server: Data Fetch Thread Dies 
<!-- What problem are we solving? What does the experience look like today? What are the symptoms? -->

If run long enough (could be an hour, could be a week), the affiliate server's data-fetch thread can seize up and result in a never-ending queue of of isbns which never get cleared + processed.

This issue calls for a modification to the affiliate-sever which checks the state of the auxiliary data processing thread(s) to check whether it's healthy/alive when isbns are submitted. We can use this check to bootstrap/fix/restart any hung thread(s).

### Related files
<!-- Files related to this issue; this is super useful for new contributors who might want to help! If you're not sure, leave this blank; a maintainer will add them. -->

https://github.com/internetarchive/openlibrary/blob/master/scripts/affiliate_server.py

### Stakeholders
<!-- @ tag stakeholders of this bug -->



Discussion:
If possible, let's try to have a PoC fix for next Thursday mtg (1wk)

Resolving patch:
diff --git a/scripts/affiliate_server.py b/scripts/affiliate_server.py
--- a/scripts/affiliate_server.py
+++ b/scripts/affiliate_server.py
@@ -86,6 +86,7 @@
 batch: Batch | None = None
 
 web.amazon_queue = queue.Queue()  # a thread-safe multi-producer, multi-consumer queue
+web.amazon_lookup_thread = None
 
 
 def get_current_amazon_batch() -> Batch:
@@ -214,7 +215,7 @@ def seconds_remaining(start_time: float) -> float:
     return max(API_MAX_WAIT_SECONDS - (time.time() - start_time), 0)
 
 
-def amazon_lookup(site, stats_client) -> None:
+def amazon_lookup(site, stats_client, logger) -> None:
     """
     A separate thread of execution that uses the time up to API_MAX_WAIT_SECONDS to
     create a list of isbn_10s that is not larger than API_MAX_ITEMS_PER_CALL and then
@@ -233,15 +234,35 @@ def amazon_lookup(site, stats_client) -> None:
                 )
             except queue.Empty:
                 pass
+        logger.info(f"Before amazon_lookup(): {len(isbn_10s)} items")
         if isbn_10s:
             time.sleep(seconds_remaining(start_time))
-            process_amazon_batch(list(isbn_10s))
+            try:
+                process_amazon_batch(list(isbn_10s))
+                logger.info(f"After amazon_lookup(): {len(isbn_10s)} items")
+            except Exception:
+                logger.exception("Amazon Lookup Thread died")
+                stats_client.incr("ol.affiliate.amazon.lookup_thread_died")
+
+
+def make_amazon_lookup_thread() -> threading.Thread:
+    """Called from start_server() and assigned to web.amazon_lookup_thread."""
+    thread = threading.Thread(
+        target=amazon_lookup,
+        args=(web.ctx.site, stats.client, logger),
+        daemon=True,
+    )
+    thread.start()
+    return thread
 
 
 class Status:
     def GET(self) -> str:
         return json.dumps(
             {
+                "thread_is_alive": bool(
+                    web.amazon_lookup_thread and web.amazon_lookup_thread.is_alive()
+                ),
                 "queue_size": web.amazon_queue.qsize(),
                 "queue": list(web.amazon_queue.queue),
             }
@@ -345,9 +366,13 @@ def start_server():
     infogami._setup()
 
     if "pytest" not in sys.modules:
-        threading.Thread(
-            target=amazon_lookup, args=(web.ctx.site, stats.client)
-        ).start()
+        web.amazon_lookup_thread = make_amazon_lookup_thread()
+        thread_is_alive = bool(
+            web.amazon_lookup_thread and web.amazon_lookup_thread.is_alive()
+        )
+        logger.critical(f"web.amazon_lookup_thread.is_alive() is {thread_is_alive}")
+    else:
+        logger.critical("Not starting amazon_lookup_thread in pytest")
 
     sys.argv = [sys.argv[0]] + list(args)
     app.run()


Test patch: