Repository: internetarchive/openlibrary

Issue:
Improve solr_updater solr POST error handling
During solr catch up, we often have errors like:

`Error with solr POST update: {"delete": ["/works/OL36792267M", "/works/OL35623783M", "/works/OL36340245M", "/works/OL37644115M", "/works/OL35974137M", "/works/OL35620087M", "/works/OL37486121M", "/works/OL37643824M", "/works/OL36791677M", "/works/OL36955347M", "/works/OL36870114M", "/works/OL37643345M", "/works/OL36792359M", "/works/OL36197272M", "/works/OL35986808M", "/works/OL37644119M", "/works/OL35965122M", "/works/OL36792352M", "/works/OL36976381M", "/works/OL36795315M", "/works/OL28049383M", "/work...`

### Proposal & Constraints
- Improve the logged message to not log the full command but include it is an error param
- Differentiate better between the three types of failures that can happen here
- Most importantly: Add retrying!

### Additional context
See `update_work.py` file.

### Stakeholders
<!-- @ tag stakeholders of this bug -->





Discussion:

Resolving patch:
diff --git a/openlibrary/conftest.py b/openlibrary/conftest.py
--- a/openlibrary/conftest.py
+++ b/openlibrary/conftest.py
@@ -24,6 +24,42 @@ def mock_request(*args, **kwargs):
     monkeypatch.setattr("requests.sessions.Session.request", mock_request)
 
 
+@pytest.fixture(autouse=True)
+def no_sleep(monkeypatch):
+    def mock_sleep(*args, **kwargs):
+        raise Warning(
+            '''
+            Sleeping is blocked in the testing environment.
+            Use monkeytime instead; it stubs time.time() and time.sleep().
+
+            Eg:
+                def test_foo(monkeytime):
+                    assert time.time() == 1
+                    time.sleep(1)
+                    assert time.time() == 2
+
+            If you need more methods stubbed, edit monkeytime in openlibrary/conftest.py
+            '''
+        )
+
+    monkeypatch.setattr("time.sleep", mock_sleep)
+
+
+@pytest.fixture
+def monkeytime(monkeypatch):
+    cur_time = 1
+
+    def time():
+        return cur_time
+
+    def sleep(secs):
+        nonlocal cur_time
+        cur_time += secs
+
+    monkeypatch.setattr("time.time", time)
+    monkeypatch.setattr("time.sleep", sleep)
+
+
 @pytest.fixture
 def wildcard():
     return Wildcard()
diff --git a/openlibrary/solr/update_work.py b/openlibrary/solr/update_work.py
--- a/openlibrary/solr/update_work.py
+++ b/openlibrary/solr/update_work.py
@@ -14,7 +14,7 @@
 import sys
 import time
 
-from httpx import HTTPError, TimeoutException
+from httpx import HTTPError, HTTPStatusError, TimeoutException
 from six.moves.urllib.parse import urlparse
 from collections import defaultdict
 from unicodedata import normalize
@@ -37,6 +37,7 @@
 from openlibrary.utils.ddc import normalize_ddc, choose_sorting_ddc
 from openlibrary.utils.isbn import opposite_isbn
 from openlibrary.utils.lcc import short_lcc_to_sortable_lcc, choose_sorting_lcc
+from openlibrary.utils.retry import MaxRetriesExceeded, RetryStrategy
 
 logger = logging.getLogger("openlibrary.solr")
 
@@ -1177,29 +1178,57 @@ def solr_update(
         params['commitWithin'] = commit_within
     if skip_id_check:
         params['overwrite'] = 'false'
-    logger.debug(f"POSTing update to {solr_base_url}/update {params}")
-    try:
-        resp = httpx.post(
-            f'{solr_base_url}/update',
-            # Large batches especially can take a decent chunk of time
-            timeout=300,
-            params=params,
-            headers={'Content-Type': 'application/json'},
-            content=content,
-        )
+
+    def make_request():
+        logger.debug(f"POSTing update to {solr_base_url}/update {params}")
         try:
-            resp_json = resp.json()
-            errors = resp_json['responseHeader'].get('errors', [])
-            if errors:
-                for e in errors:
-                    logger.error(f'Error with solr POST update: {e}')
-        except JSONDecodeError:
-            logger.error('Error with solr POST update: ' + resp.text)
-        resp.raise_for_status()
-    except TimeoutException:
-        logger.error('Timeout Error with solr POST update: ' + content)
-    except HTTPError:
-        logger.error('Error with solr POST update: ' + content)
+            resp = httpx.post(
+                f'{solr_base_url}/update',
+                # Large batches especially can take a decent chunk of time
+                timeout=300,
+                params=params,
+                headers={'Content-Type': 'application/json'},
+                content=content,
+            )
+
+            if resp.status_code == 400:
+                resp_json = resp.json()
+
+                indiv_errors = resp_json.get('responseHeader', {}).get('errors', [])
+                if indiv_errors:
+                    for e in indiv_errors:
+                        logger.error(f'Individual Solr POST Error: {e}')
+
+                global_error = resp_json.get('error')
+                if global_error:
+                    logger.error(f'Global Solr POST Error: {global_error.get("msg")}')
+
+                if not (indiv_errors or global_error):
+                    # We can handle the above errors. Any other 400 status codes
+                    # are fatal and should cause a retry
+                    resp.raise_for_status()
+            else:
+                resp.raise_for_status()
+        except HTTPStatusError as e:
+            logger.error(f'HTTP Status Solr POST Error: {e}')
+            raise
+        except TimeoutException:
+            logger.error(f'Timeout Solr POST Error: {content}')
+            raise
+        except HTTPError as e:
+            logger.error(f'HTTP Solr POST Error: {e}')
+            raise
+
+    retry = RetryStrategy(
+        [HTTPStatusError, TimeoutException, HTTPError],
+        max_retries=5,
+        delay=8,
+    )
+
+    try:
+        return retry(make_request)
+    except MaxRetriesExceeded as e:
+        logger.error(f'Max retries exceeded for Solr POST: {e.last_exception}')
 
 
 def get_subject(key):
diff --git a/openlibrary/utils/retry.py b/openlibrary/utils/retry.py
new file mode 100644
--- /dev/null
+++ b/openlibrary/utils/retry.py
@@ -0,0 +1,35 @@
+import time
+from typing import Optional
+
+
+class MaxRetriesExceeded(Exception):
+    last_exception: Exception
+
+    def __init__(self, last_exception: Exception):
+        self.last_exception = last_exception
+
+
+class RetryStrategy:
+    retry_count = 0
+    last_exception: Optional[Exception] = None
+
+    def __init__(self, exceptions: list[type[Exception]], max_retries=3, delay=1):
+        self.exceptions = exceptions
+        self.max_retries = max_retries
+        self.delay = delay
+
+    def __call__(self, func):
+        try:
+            return func()
+        except Exception as e:
+            if not any(isinstance(e, exc) for exc in self.exceptions):
+                raise e
+
+            self.last_exception = e
+
+            if self.retry_count < self.max_retries:
+                self.retry_count += 1
+                time.sleep(self.delay)
+                return self(func)
+            else:
+                raise MaxRetriesExceeded(self.last_exception)


Test patch:
diff --git a/openlibrary/tests/core/test_cache.py b/openlibrary/tests/core/test_cache.py
--- a/openlibrary/tests/core/test_cache.py
+++ b/openlibrary/tests/core/test_cache.py
@@ -60,16 +60,20 @@ def test_update_async(self):
 
         assert m.memcache_get([20], {})[0] == 400
 
-    def test_timeout(self):
+    def test_timeout(self, monkeytime):
         m = self.square_memoize()
         m.timeout = 0.1
         s = m.stats
 
         assert m(10) == 100
+
         time.sleep(0.1)
+        assert m(10) == 100
+        assert [s.calls, s.hits, s.updates, s.async_updates] == [2, 1, 1, 0]
 
+        time.sleep(0.01)
         assert m(10) == 100
-        assert [s.calls, s.hits, s.updates, s.async_updates] == [2, 1, 1, 1]
+        assert [s.calls, s.hits, s.updates, s.async_updates] == [3, 2, 1, 1]
 
     def test_delete(self):
         m = self.square_memoize()
diff --git a/openlibrary/tests/core/test_init.py b/openlibrary/tests/core/test_init.py
--- a/openlibrary/tests/core/test_init.py
+++ b/openlibrary/tests/core/test_init.py
@@ -47,7 +47,7 @@ def test_poll_and_wait(self, tmpdir):
         assert s.wait() == 0
         assert s.poll() == 0
 
-    def test_stop(self, tmpdir):
+    def test_stop(self, monkeytime, tmpdir):
         s = self.create_service(
             "sleep",
             {
diff --git a/openlibrary/tests/solr/test_update_work.py b/openlibrary/tests/solr/test_update_work.py
--- a/openlibrary/tests/solr/test_update_work.py
+++ b/openlibrary/tests/solr/test_update_work.py
@@ -1,12 +1,18 @@
+import json
+import httpx
+from httpx import ConnectError, Response
 import pytest
+from unittest.mock import MagicMock
 
 from openlibrary.solr import update_work
 from openlibrary.solr.data_provider import DataProvider
 from openlibrary.solr.update_work import (
+    CommitRequest,
     SolrProcessor,
     build_data,
     pick_cover_edition,
     pick_number_of_pages_median,
+    solr_update,
 )
 
 author_counter = 0
@@ -736,3 +742,143 @@ def test_goog_deprioritized(self):
         ]
         assert SolrProcessor.get_ebook_info(editions)['ia'] == ["foobarblah", "foobargoog"]
 
+
+class TestSolrUpdate:
+    def sample_response_200(self):
+        return Response(
+            200,
+            request=MagicMock(),
+            content=json.dumps(
+                {
+                    "responseHeader": {
+                        "errors": [],
+                        "maxErrors": -1,
+                        "status": 0,
+                        "QTime": 183,
+                    }
+                }
+            ),
+        )
+
+    def sample_global_error(self):
+        return Response(
+            400,
+            request=MagicMock(),
+            content=json.dumps(
+                {
+                    'responseHeader': {
+                        'errors': [],
+                        'maxErrors': -1,
+                        'status': 400,
+                        'QTime': 76,
+                    },
+                    'error': {
+                        'metadata': [
+                            'error-class',
+                            'org.apache.solr.common.SolrException',
+                            'root-error-class',
+                            'org.apache.solr.common.SolrException',
+                        ],
+                        'msg': "Unknown key 'key' at [14]",
+                        'code': 400,
+                    },
+                }
+            ),
+        )
+
+    def sample_individual_error(self):
+        return Response(
+            400,
+            request=MagicMock(),
+            content=json.dumps(
+                {
+                    'responseHeader': {
+                        'errors': [
+                            {
+                                'type': 'ADD',
+                                'id': '/books/OL1M',
+                                'message': '[doc=/books/OL1M] missing required field: type',
+                            }
+                        ],
+                        'maxErrors': -1,
+                        'status': 0,
+                        'QTime': 10,
+                    }
+                }
+            ),
+        )
+
+    def sample_response_503(self):
+        return Response(
+            503,
+            request=MagicMock(),
+            content=b"<html><body><h1>503 Service Unavailable</h1>",
+        )
+
+    def test_successful_response(self, monkeypatch, monkeytime):
+        mock_post = MagicMock(return_value=self.sample_response_200())
+        monkeypatch.setattr(httpx, "post", mock_post)
+
+        solr_update(
+            [CommitRequest()],
+            solr_base_url="http://localhost:8983/solr/foobar",
+        )
+
+        assert mock_post.call_count == 1
+
+    def test_non_json_solr_503(self, monkeypatch, monkeytime):
+        mock_post = MagicMock(return_value=self.sample_response_503())
+        monkeypatch.setattr(httpx, "post", mock_post)
+
+        solr_update(
+            [CommitRequest()],
+            solr_base_url="http://localhost:8983/solr/foobar",
+        )
+
+        assert mock_post.call_count > 1
+
+    def test_solr_offline(self, monkeypatch, monkeytime):
+        mock_post = MagicMock(side_effect=ConnectError('', request=None))
+        monkeypatch.setattr(httpx, "post", mock_post)
+
+        solr_update(
+            [CommitRequest()],
+            solr_base_url="http://localhost:8983/solr/foobar",
+        )
+
+        assert mock_post.call_count > 1
+
+    def test_invalid_solr_request(self, monkeypatch, monkeytime):
+        mock_post = MagicMock(return_value=self.sample_global_error())
+        monkeypatch.setattr(httpx, "post", mock_post)
+
+        solr_update(
+            [CommitRequest()],
+            solr_base_url="http://localhost:8983/solr/foobar",
+        )
+
+        assert mock_post.call_count == 1
+
+    def test_bad_apple_in_solr_request(self, monkeypatch, monkeytime):
+        mock_post = MagicMock(return_value=self.sample_individual_error())
+        monkeypatch.setattr(httpx, "post", mock_post)
+
+        solr_update(
+            [CommitRequest()],
+            solr_base_url="http://localhost:8983/solr/foobar",
+        )
+
+        assert mock_post.call_count == 1
+
+    def test_other_non_ok_status(self, monkeypatch, monkeytime):
+        mock_post = MagicMock(
+            return_value=Response(500, request=MagicMock(), content="{}")
+        )
+        monkeypatch.setattr(httpx, "post", mock_post)
+
+        solr_update(
+            [CommitRequest()],
+            solr_base_url="http://localhost:8983/solr/foobar",
+        )
+
+        assert mock_post.call_count > 1
diff --git a/openlibrary/utils/tests/test_retry.py b/openlibrary/utils/tests/test_retry.py
new file mode 100644
--- /dev/null
+++ b/openlibrary/utils/tests/test_retry.py
@@ -0,0 +1,40 @@
+from unittest.mock import Mock
+import pytest
+from openlibrary.utils.retry import MaxRetriesExceeded, RetryStrategy
+
+
+class TestRetryStrategy:
+    def test_exception_filter(self, monkeytime):
+        foo = Mock(side_effect=ZeroDivisionError)
+        retry = RetryStrategy([ZeroDivisionError], max_retries=3)
+        with pytest.raises(MaxRetriesExceeded):
+            retry(foo)
+        assert foo.call_count == 4
+
+    def test_no_retry(self):
+        foo = Mock(return_value=1)
+        retry = RetryStrategy([ZeroDivisionError], max_retries=3)
+        assert retry(foo) == 1
+        assert foo.call_count == 1
+
+    def test_retry(self, monkeytime):
+        foo = Mock(side_effect=[ZeroDivisionError, 1])
+        retry = RetryStrategy([ZeroDivisionError], max_retries=3)
+        assert retry(foo) == 1
+        assert foo.call_count == 2
+
+    def test_unhandled_error(self):
+        foo = Mock(side_effect=ZeroDivisionError)
+        retry = RetryStrategy([ValueError], max_retries=3)
+        with pytest.raises(ZeroDivisionError):
+            retry(foo)
+        assert foo.call_count == 1
+
+    def test_last_exception(self, monkeytime):
+        retry = RetryStrategy([ZeroDivisionError], max_retries=3)
+        with pytest.raises(MaxRetriesExceeded):
+            try:
+                retry(lambda: 1 / 0)
+            except MaxRetriesExceeded as e:
+                assert isinstance(e.last_exception, ZeroDivisionError)
+                raise