Repository: internetarchive/openlibrary

Issue:
Solrbuilder workers failing partway
Solrbuilder getting stuck partway during full reindex process.

### Details

NA

### Proposal & Constraints
<!-- What is the proposed solution / implementation? Is there a precedent of this approach succeeding elsewhere? -->

### Related files
<!-- Files related to this issue; this is super useful for new contributors who might want to help! If you're not sure, leave this blank; a maintainer will add them. -->

### Stakeholders
<!-- @ tag stakeholders of this bug -->



Discussion:

Resolving patch:
diff --git a/openlibrary/solr/update_work.py b/openlibrary/solr/update_work.py
--- a/openlibrary/solr/update_work.py
+++ b/openlibrary/solr/update_work.py
@@ -513,31 +513,25 @@ def add_list(name, values):
                            for v in e[db_key])
             add_list(solr_key, values)
 
-        # Need this to enable in pytest, which have config.runtime_config = {}
-        conf = config.runtime_config or {
-            'plugin_worksearch': {
-                'has_classifications': True
-            }}
-        if conf.get('plugin_worksearch', {}).get('has_classifications', False):
-            raw_lccs = set(lcc
-                           for ed in editions
-                           for lcc in ed.get('lc_classifications', []))
-            lccs = set(lcc for lcc in map(short_lcc_to_sortable_lcc, raw_lccs) if lcc)
-            if lccs:
-                add_list("lcc", lccs)
-                # Choose the... idk, longest for sorting?
-                add("lcc_sort", sorted(lccs, key=len, reverse=True)[0])
-
-            raw_ddcs = set(ddc
-                           for ed in editions
-                           for ddc in ed.get('dewey_decimal_class', []))
-            ddcs = set(ddc
-                       for raw_ddc in raw_ddcs
-                       for ddc in normalize_ddc(raw_ddc))
-            if ddcs:
-                add_list("ddc", ddcs)
-                # Choose longest, since has most precision?
-                add("ddc_sort", sorted(ddcs, key=len, reverse=True)[0])
+        raw_lccs = set(lcc
+                       for ed in editions
+                       for lcc in ed.get('lc_classifications', []))
+        lccs = set(lcc for lcc in map(short_lcc_to_sortable_lcc, raw_lccs) if lcc)
+        if lccs:
+            add_list("lcc", lccs)
+            # Choose the... idk, longest for sorting?
+            add("lcc_sort", sorted(lccs, key=len, reverse=True)[0])
+
+        raw_ddcs = set(ddc
+                       for ed in editions
+                       for ddc in ed.get('dewey_decimal_class', []))
+        ddcs = set(ddc
+                   for raw_ddc in raw_ddcs
+                   for ddc in normalize_ddc(raw_ddc))
+        if ddcs:
+            add_list("ddc", ddcs)
+            # Choose longest, since has most precision?
+            add("ddc_sort", sorted(ddcs, key=len, reverse=True)[0])
 
         add_list("isbn", self.get_isbns(editions))
         add("last_modified_i", self.get_last_modified(w, editions))
diff --git a/scripts/solr_builder/solr_builder/solr_builder_main.py b/scripts/solr_builder/solr_builder/solr_builder_main.py
--- a/scripts/solr_builder/solr_builder/solr_builder_main.py
+++ b/scripts/solr_builder/solr_builder/solr_builder_main.py
@@ -32,18 +32,24 @@ def config_section_to_dict(config_file, section):
     return result
 
 
-def batches(itrble, size):
+def batch_until_len(items, max_batch_len):
     """
     :param collections.Sized itrble:
     :param int size:
     :rtype: typings.Generator[list, None, None]
     """
-    start = 0
-    end = 0
-    while end < len(itrble):
-        end = start + size
-        yield itrble[start:end]
-        start = end
+    batch_len = 0
+    batch = []
+    for item in items:
+        if batch_len + len(item) > max_batch_len and batch:
+            yield batch
+            batch = [item]
+            batch_len = len(item)
+        else:
+            batch.append(item)
+            batch_len += len(item)
+    if batch:
+        yield batch
 
 
 class LocalPostgresDataProvider(DataProvider):
@@ -130,30 +136,50 @@ def query_batched(self, query, size, cursor_name=None, cache_json=False):
         cur.close()
 
     @staticmethod
-    def _get_lite_metadata(ocaids, rows=1000):
+    def _get_lite_metadata(ocaids, _recur_depth=0, _max_recur_depth=3):
+        """
+        For bulk fetch, some of the ocaids in Open Library may be bad
+        and break archive.org ES fetches. When this happens, we (up to
+        3 times) recursively split up the pool of ocaids to do as many
+        successful sub-bulk fetches as we can and then when limit is
+        reached, downstream code will fetch remaining ocaids individually
+        (and skip bad ocaids) 
+        """
+        if not ocaids or _recur_depth > _max_recur_depth:
+            return []
+
         r = requests.get("https://archive.org/advancedsearch.php", params={
             'q': "identifier:(%s)" % ' OR '.join(ocaids),
-            'rows': rows,
+            'rows': len(ocaids),
             'fl': 'identifier,boxid,collection',
             'page': 1,
             'output': 'json',
             'save': 'yes',
         })
-        r.raise_for_status()
-        return r.json()['response']
 
-    def cache_ia_metadata(self, ocaids, batch_size=750):
+        try:
+            r.raise_for_status()
+            return r.json()['response']['docs']
+        except (RequestException, ValueError, KeyError):
+            logger.exception("Error while fetching IA data")
+            # there's probably a bad apple; try splitting the batch
+            mid = len(ocaids) // 2
+            h1, h2 = ocaids[:mid], ocaids[mid:]
+            f = LocalPostgresDataProvider._get_lite_metadata
+            return (
+                f(h1, _recur_depth=_recur_depth + 1) +
+                f(h2, _recur_depth=_recur_depth + 1)
+            )
+
+    def cache_ia_metadata(self, ocaids):
         """
         :param list of str ocaids:
         :param int batch_size:
         :return: None
         """
-        for b in batches(ocaids, batch_size):
-            try:
-                for doc in self._get_lite_metadata(b, rows=batch_size)['docs']:
-                    self.ia_cache[doc['identifier']] = doc
-            except (RequestException, ValueError):
-                logger.error("Error while caching IA", exc_info=True)
+        for b in batch_until_len(ocaids, 3000):
+            for doc in self._get_lite_metadata(b):
+                self.ia_cache[doc['identifier']] = doc
 
     def cache_edition_works(self, lo_key, hi_key):
         q = """
@@ -199,9 +225,10 @@ def cache_work_authors(self, lo_key, hi_key):
         self.query_all(q, cache_json=True)
 
     def cache_cached_editions_ia_metadata(self):
-        # Limit length of OCAIDs to avoid really long URLs. They just won't be cached.
-        ocaids = [doc['ocaid'] for doc in self.cache.itervalues() if
-                  'ocaid' in doc and len(doc['ocaid']) < 50]
+        ocaids = [
+            doc['ocaid'] for doc in self.cache.itervalues()
+            if 'ocaid' in doc]
+        ocaids = list(set(ocaids))
         self.cache_ia_metadata(ocaids)
 
     def find_redirects(self, key):
@@ -222,19 +249,20 @@ def get_editions_of_work(self, work):
         return [r[0] for r in self.query_iter(q)]
 
     def get_metadata(self, identifier):
-        logger.info("find_metadata %s", identifier)
-
         if identifier in self.ia_cache:
+            logger.info("IA metadata cache hit")
             return self.ia_cache[identifier]
-
-        return ia.get_metadata(identifier)
+        else:
+            logger.info("IA metadata cache miss")
+            return ia.get_metadata(identifier)
 
     def get_document(self, key):
-        logger.info("get_document %s", key)
-
         if key in self.cache:
+            logger.info("get_document cache hit %s", key)
             return self.cache[key]
 
+        logger.info("get_document cache miss %s", key)
+
         q = """
         SELECT "JSON" FROM test
         WHERE "Key" = '%s'


Test patch: