Repository: internetarchive/openlibrary

Issue:
Decouple OpenLibrary from Civi CRM
<!-- A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] -->

### Describe the problem that you'd like solved
<!-- A clear and concise description of what you want to happen. -->
Open Library should not be calling Civi directly.  The application team reports an unexpectedly high volume of traffic to Civi, which is likely due to recent changes to the "My Books" page.  Civi calls are used to find patron's sponsorships, of which a count is displayed in the sidebar.

### Proposal & Constraints
<!-- What is the proposed solution / implementation? Is there a precedent of this approach succeeding elsewhere? -->
Either remove code related to the sponsorship program, or find another way to get the sponsorship data.

<!-- Which suggestions or requirements should be considered for how feature needs to appear or be implemented? -->

### Additional context
<!-- Add any other context or screenshots about the feature request here. -->

### Stakeholders
<!-- @ tag stakeholders of this bug -->
@mekarpeles 




Discussion:
we can a query of this flavor
https://archive.org/search.php?query=donor%3A%22%40mekarpeles%22%20collection%3Aopenlibraryscanningteam

Resolving patch:
diff --git a/openlibrary/core/sponsorships.py b/openlibrary/core/sponsorships.py
--- a/openlibrary/core/sponsorships.py
+++ b/openlibrary/core/sponsorships.py
@@ -33,19 +33,51 @@ def eligibility_check(edition, patron=None):
 PAGE_COST_CENTS = 12
 
 
-def get_sponsored_editions(user):
+def get_sponsored_editions_civi(user):
     """
+    Deprecated by get_sponsored_editions but worth maintaining as we
+    may periodically have to programmatically access data from civi
+    since it is the ground-truth of this data.
+
     Gets a list of books from the civi API which internet archive
     @archive_username has sponsored
 
     :param user user: infogami user
     :rtype: list
-    :return: list of editions sponsored by user
+    :return: list of archive.org items sponsored by user
     """
     archive_id = get_internet_archive_id(user.key if 'key' in user else user._key)
     # MyBooks page breaking on local environments without archive_id check
-    contact_id = get_contact_id_by_username(archive_id) if archive_id else None
-    return get_sponsorships_by_contact_id(contact_id) if contact_id else []
+    if archive_id:
+        contact_id = get_contact_id_by_username(archive_id) if archive_id else None
+        return get_sponsorships_by_contact_id(contact_id) if contact_id else []
+    return []
+
+
+def get_sponsored_editions(user, page=1):
+    """
+    Gets a list of books from archive.org elasticsearch
+    @archive_username has sponsored
+
+    :param user user: infogami user
+    :rtype: list
+    :return: list of archive.org editions sponsored by user
+    """
+    archive_id = get_internet_archive_id(user.key if 'key' in user else user._key)
+    if archive_id:
+        url = 'https://archive.org/advancedsearch.php'
+        params = urlencode({
+            'fl[]': ['identifier', 'openlibrary_edition'],
+            'sort[]': 'date',
+            'output': 'json',
+            'page': page,
+            'rows': 50,
+            'q': f'donor:{archive_id}'
+        }, doseq=True)
+        r = requests.get(f'{url}?{params}')
+        # e.g. [{'openlibrary_edition': 'OL24896084M', 'identifier': 'isbn_9780691160191'}]
+        return r.json()['response'].get('docs')
+    return []
 
 
 def do_we_want_it(isbn):
diff --git a/openlibrary/plugins/upstream/mybooks.py b/openlibrary/plugins/upstream/mybooks.py
--- a/openlibrary/plugins/upstream/mybooks.py
+++ b/openlibrary/plugins/upstream/mybooks.py
@@ -193,7 +193,12 @@ def render(self, page=1, sort='desc', list=None):
             self.counts['sponsorships'] = len(sponsorships)
 
             if self.key == 'sponsorships':
-                data = self._prepare_data(logged_in_user, sponsorships=sponsorships)
+                data = add_availability(
+                    web.ctx.site.get_many([
+                        '/books/%s' % doc['openlibrary_edition']
+                        for doc in sponsorships
+                    ])
+                ) if sponsorships else None
             elif self.key in self.READING_LOG_KEYS:
                 data = add_availability(
                     self.readlog.get_works(
@@ -231,20 +236,9 @@ def render(self, page=1, sort='desc', list=None):
     def _prepare_data(
         self,
         logged_in_user,
-        sponsorships=None,
         page=1,
         username=None,
     ):
-        if sponsorships:
-            return (
-                web.ctx.site.get(
-                    web.ctx.site.things(
-                        {'type': '/type/edition', 'isbn_%s' % len(s['isbn']): s['isbn']}
-                    )[0]
-                )
-                for s in sponsorships
-            )
-
         if self.key == 'loans':
             logged_in_user.update_loan_status()
             return borrow.get_loans(logged_in_user)


Test patch:
diff --git a/openlibrary/tests/core/test_sponsors.py b/openlibrary/tests/core/test_sponsors.py
--- a/openlibrary/tests/core/test_sponsors.py
+++ b/openlibrary/tests/core/test_sponsors.py
@@ -4,9 +4,33 @@
 
 
 class TestSponsorship:
+
     def test_get_sponsored_editions(self, monkeypatch):
         user = storage(key='/person/mekBot')
 
+        monkeypatch.setattr(
+            sponsorships, 'get_internet_archive_id', lambda user_key: '@username'
+        )
+        assert sponsorships.get_internet_archive_id(user) == '@username'
+
+        class RequestMock(dict):
+            def __init__(self, *arg):
+                super().__init__(*arg)
+
+            def json(self):
+                return self
+
+        monkeypatch.setattr(
+            sponsorships.requests, 'get', lambda url, **kwargs:
+            RequestMock({'response': {'docs': []}})
+        )
+
+        assert sponsorships.get_sponsored_editions(user) == []
+
+
+    def test_get_sponsored_editions_civi(self, monkeypatch):
+        user = storage(key='/person/mekBot')
+
         monkeypatch.setattr(
             sponsorships, 'get_internet_archive_id', lambda user_key: '@username'
         )
@@ -15,7 +39,7 @@ def test_get_sponsored_editions(self, monkeypatch):
             sponsorships, 'get_contact_id_by_username', lambda archive_id: None
         )
         assert sponsorships.get_contact_id_by_username('@username') is None
-        assert sponsorships.get_sponsored_editions(user) == []
+        assert sponsorships.get_sponsored_editions_civi(user) == []
 
         monkeypatch.setattr(
             sponsorships, 'get_contact_id_by_username', lambda archive_id: '123'
@@ -26,7 +50,7 @@ def test_get_sponsored_editions(self, monkeypatch):
             'get_sponsorships_by_contact_id',
             lambda contact_id: ['fake_data'],
         )
-        assert sponsorships.get_sponsored_editions(user) == ['fake_data']
+        assert sponsorships.get_sponsored_editions_civi(user) == ['fake_data']
 
     def test_do_we_want_it(self, monkeypatch, mock_site):
         isbn = '0123456789'