Repository: internetarchive/openlibrary

Issue:
Open Library Amazon API usage exceeding limit
Dup of #3967

Received email from Amazon API:

> Hello,
> 
> Your Associates account is at risk of closure.
> 
> Why?
> You are trying to submit requests that exceed your account’s usage limit. You must not exceed, or if you build and release an application that calls PA API, each copy of that application that is installed by an end user will not exceed, any limit on calls per second set forth in any Specifications.
> 
> What’s next?
> Within five business days please correct the violations and notify us when you are in compliance through our customer service contact page: https://affiliate-program.amazon.com/home/contact. Please choose the subject 'Warning/Information Request Response' from the dropdown menu, and be sure to reference Issue Code 83441-US in the comments field.
> 
> If not done within five business days, we will close your account.

## Stakeholders
@mekarpeles @bfalling 


Discussion:
I temporarily disabled Amazon calls on ol-web[1,2] & ol-dev1 to reduce traffic. It's still running on ol-dev0.

I also emailed Amazon w/ the info above and will await their reply.

Next steps: (can now wait until this week)
* I think we should consider making prices lazy load (like Related Books Carousels) cc: @tabshaikh, @jamesachamp 
* We might consider moving amazon fetch code into a single service/API running on ol-www1 and setting an nginx rule to reverse proxy to that service (or make it an internal service -- may be a better idea) @mekarpeles 
* Presumably we can batch 10 ASIN/ISBN requests to Amazon. Not sure the best way to queue those. Maybe some global list which gets flushed every 1 seconds @mekarpeles, @cclauss 

Resolving patch:
diff --git a/openlibrary/core/lending.py b/openlibrary/core/lending.py
--- a/openlibrary/core/lending.py
+++ b/openlibrary/core/lending.py
@@ -56,7 +56,6 @@
 config_loanstatus_url = None
 config_bookreader_host = None
 config_internal_tests_api_key = None
-config_amz_api = None
 
 def setup(config):
     """Initializes this module from openlibrary config.
@@ -114,7 +113,7 @@ def cached_work_authors_and_subjects(work_id):
 def compose_ia_url(limit=None, page=1, subject=None, query=None, work_id=None,
                    _type=None, sorts=None, advanced=True):
     """This needs to be exposed by a generalized API endpoint within
-    plugins/openlibrary/api/browse which lets lazy-load more items for
+    plugins/api/browse which lets lazy-load more items for
     the homepage carousel and support the upcoming /browse view
     (backed by archive.org search, so we don't have to send users to
     archive.org to see more books)
@@ -174,7 +173,7 @@ def compose_ia_url(limit=None, page=1, subject=None, query=None, work_id=None,
                     'sorts': sorts,
                     'advanced': advanced,
                 })
-                return ''  # TODO: Should we just raise an excpetion instead?
+                return ''  # TODO: Should we just raise an exception instead?
             q += ' AND (%s) AND !openlibrary_work:(%s)' % (_q, work_id.split('/')[-1])
 
     if not advanced:
@@ -261,6 +260,7 @@ def get_available(limit=None, page=1, subject=None, query=None,
     used in such things as 'Staff Picks' carousel to retrieve a list
     of unique available books.
     """
+
     url = url or compose_ia_url(
         limit=limit, page=page, subject=subject, query=query,
         work_id=work_id, _type=_type, sorts=sorts
@@ -279,6 +279,7 @@ def get_available(limit=None, page=1, subject=None, query=None,
             '_type': _type,
             'sorts': sorts,
         })
+        return {'error': 'no_url'}
     try:
         # Internet Archive Elastic Search (which powers some of our
         # carousel queries) needs Open Library to forward user IPs so
diff --git a/openlibrary/core/stats.py b/openlibrary/core/stats.py
--- a/openlibrary/core/stats.py
+++ b/openlibrary/core/stats.py
@@ -37,12 +37,12 @@ def create_stats_client(cfg = config):
         logger.critical("Couldn't create stats client - %s", e, exc_info = True)
         return False
 
-def put(key, value):
+def put(key, value, rate=1.0):
     "Records this ``value`` with the given ``key``. It is stored as a millisecond count"
     global client
     if client:
         l.debug("Putting %s as %s" % (value, key))
-        client.timing(key, value)
+        client.timing(key, value, rate)
 
 
 def increment(key, n=1, rate=1.0):
diff --git a/openlibrary/core/vendors.py b/openlibrary/core/vendors.py
--- a/openlibrary/core/vendors.py
+++ b/openlibrary/core/vendors.py
@@ -21,27 +21,18 @@
 
 logger = logging.getLogger("openlibrary.vendors")
 
-amazon_api = None
-config_amz_api = None
-
 BETTERWORLDBOOKS_BASE_URL = 'https://betterworldbooks.com'
 BETTERWORLDBOOKS_API_URL = ('https://products.betterworldbooks.com/service.aspx?'
                             'IncludeAmazon=True&ItemId=')
+affiliate_server_url = None
 BWB_AFFILIATE_LINK = 'http://www.anrdoezrs.net/links/{}/type/dlg/http://www.betterworldbooks.com/-id-%s'.format(h.affiliate_id('betterworldbooks'))
 AMAZON_FULL_DATE_RE = re.compile(r'\d{4}-\d\d-\d\d')
 ISBD_UNIT_PUNCT = ' : '  # ISBD cataloging title-unit separator punctuation
 
 
 def setup(config):
-    global config_amz_api, amazon_api
-    config_amz_api = config.get('amazon_api')
-    try:
-        amazon_api = AmazonAPI(
-            config_amz_api.key, config_amz_api.secret,
-            config_amz_api.id, throttling=0.9)
-    except AttributeError:
-        amazon_api = None
-
+    global affiliate_server_url
+    affiliate_server_url = config.get('affiliate_server')
 
 class AmazonAPI:
     """Amazon Product Advertising API 5.0 wrapper for Python"""
@@ -124,7 +115,9 @@ def get_products(self, asins, serialize=False, marketplace='www.amazon.com',
                          exc_info=True)
             return None
         response = self.api.get_items(request)
-        products = response.items_result.items
+        products = [
+            p for p in response.items_result.items if p
+        ] if response.items_result else []
         return (products if not serialize else
                 [self.serialize(p) for p in products])
 
@@ -258,17 +251,20 @@ def _get_amazon_metadata(id_, id_type='isbn', resources=None):
     :return: A single book item's metadata, or None.
     :rtype: dict or None
     """
+    if not affiliate_server_url:
+        return None
+
     if id_type == 'isbn':
         id_ = normalize_isbn(id_)
         if len(id_) == 13 and id_.startswith('978'):
             id_ = isbn_13_to_isbn_10(id_)
 
-    if amazon_api:
-        try:
-            return amazon_api.get_product(id_, serialize=True, resources=resources)
-        except Exception:
-            logger.exception("amazon_api.get_product({}, serialize=True)".format(id_))
-            return None
+    try:
+        r = requests.get('http://%s/isbn/%s' % (affiliate_server_url, id_))
+        return r.json().get('hit') or None
+    except requests.exceptions.ConnectionError:
+        logger.exception("Affiliate Server unreachable")
+        return None
 
 
 def split_amazon_title(full_title):


Test patch: