Repository: internetarchive/openlibrary

Issue:
Import pipeline erroring/failing recently
TODO @mekarpeles 

Some sort of unicode issue is preventing the importer from opening the dump file -- something something character encoding.


Discussion:

Resolving patch:
diff --git a/scripts/partner_batch_imports.py b/scripts/partner_batch_imports.py
--- a/scripts/partner_batch_imports.py
+++ b/scripts/partner_batch_imports.py
@@ -4,6 +4,10 @@
 `import_item` table (http://openlibrary.org/admin/imports)
 which queues items to be imported via the
 Open Library JSON import API: https://openlibrary.org/api/import
+
+To Run:
+
+PYTHONPATH=. python ./scripts/partner_batch_imports.py /olsystem/etc/openlibrary.yml
 """
 
 import os
@@ -15,10 +19,10 @@
 import logging
 import requests
 
-# Add openlibrary into our path so we can process config + batch functions
-from openlibrary.core.imports import Batch
-from infogami import config
+from infogami import config  # noqa: F401
 from openlibrary.config import load_config
+from openlibrary.core.imports import Batch
+from scripts.solr_builder.solr_builder.fn_to_cli import FnToCLI
 
 logger = logging.getLogger("openlibrary.importer.bwb")
 
@@ -94,10 +98,9 @@ def __init__(self, data):
         self.length, self.width, self.height = data[40:43]
 
         # Assert importable
-        assert self.isbn_13
-        assert self.primary_format not in self.NONBOOK
-        for field in self.REQUIRED_FIELDS:
+        for field in self.REQUIRED_FIELDS + ['isbn_13']:
             assert getattr(self, field)
+        assert self.primary_format not in self.NONBOOK
 
     @staticmethod
     def contributors(data):
@@ -161,7 +164,12 @@ def update_state(logfile, fname, line_num=0):
 
 def csv_to_ol_json_item(line):
     """converts a line to a book item"""
-    b = Biblio(line.strip().split('|'))
+    try:
+        data = line.decode().strip().split('|')
+    except UnicodeDecodeError:
+        data = line.decode('ISO-8859-1').strip().split('|')
+
+    b = Biblio(data)
     return {'ia_id': b.source_id, 'data': b.json()}
 
 
@@ -171,7 +179,7 @@ def batch_import(path, batch, batch_size=5000):
 
     for fname in filenames:
         book_items = []
-        with open(fname) as f:
+        with open(fname, 'rb') as f:
             logger.info(f"Processing: {fname} from line {offset}")
             for line_num, line in enumerate(f):
 
@@ -183,8 +191,8 @@ def batch_import(path, batch, batch_size=5000):
 
                 try:
                     book_items.append(csv_to_ol_json_item(line))
-                except UnicodeDecodeError:
-                    pass
+                except AssertionError as e:
+                    logger.info(f"Error: {e} from {line}")
 
                 # If we have enough items, submit a batch
                 if not ((line_num + 1) % batch_size):
@@ -198,10 +206,9 @@ def batch_import(path, batch, batch_size=5000):
             update_state(logfile, fname, line_num)
 
 
-def main():
-    load_config(
-        os.path.abspath(os.path.join(os.sep, 'olsystem', 'etc', 'openlibrary.yml'))
-    )
+def main(ol_config: str):
+    load_config(ol_config)
+
     # Partner data is offset ~15 days from start of month
     date = datetime.date.today() - timedelta(days=15)
     batch_name = "%s-%04d%02d" % ('bwb', date.year, date.month)
@@ -210,4 +217,4 @@ def main():
 
 
 if __name__ == '__main__':
-    main()
+    FnToCLI(main).run()
diff --git a/scripts/solr_builder/solr_builder/fn_to_cli.py b/scripts/solr_builder/solr_builder/fn_to_cli.py
--- a/scripts/solr_builder/solr_builder/fn_to_cli.py
+++ b/scripts/solr_builder/solr_builder/fn_to_cli.py
@@ -30,11 +30,12 @@ def __init__(self, fn: typing.Callable):
         self.fn = fn
         arg_names = fn.__code__.co_varnames[: fn.__code__.co_argcount]
         annotations = fn.__annotations__
-        num_required = len(arg_names) - len(fn.__defaults__)  # type: ignore
+        defaults = fn.__defaults__ or []  # type: ignore
+        num_required = len(arg_names) - len(defaults)
         default_args = arg_names[num_required:]
         defaults = {
             arg: default
-            for [arg, default] in zip(default_args, fn.__defaults__)  # type: ignore
+            for [arg, default] in zip(default_args, defaults)
         }
 
         docs = fn.__doc__ or ''


Test patch: