Repository: internetarchive/openlibrary

Issue:
Data Dump keys sometimes have wrong format
The format of some of the keys is wrong though. E.g. instead of /authors/OL123A, it sometimes lists /a/OL123A
This looks like it was a regression between the 2021-10-13  data dump and 2021-11-30 data dump

### Evidence / Screenshot (if possible)

```sh
$ curl -L 'https://archive.org/download/ol_dump_2022-03-02/ol_dump_authors_2022-03-02.txt.gz' | zcat | grep -F 'OL3965446A' | head -n1
/type/author    /authors/OL3965446A     1       2008-04-30T20:50:18.033121      {"name": "I. Meyerson", "last_modified": {"type": "/type/datetime", "value": "2008-04-30 20:50:18.033121"}, "key": "/a/OL3965446A", "type": {"key": "/type/author"}, "id": 16613668, "revision": 1}

$ curl -L 'https://archive.org/download/ol_dump_2022-03-24/ol_dump_authors_2022-03-24.txt.gz' | zcat | grep -F 'OL3965446A' | head -n1
/type/author    /authors/OL3965446A     1       2008-04-30T20:50:18.033121      {"name": "I. Meyerson", "last_modified": {"type": "/type/datetime", "value": "2008-04-30 20:50:18.033121"}, "key": "/a/OL3965446A", "type": {"key": "/type/author"}, "id": 16613668, "revision": 1}

$ curl -L 'https://archive.org/download/ol_dump_2021-11-30/ol_dump_authors_2021-11-30.txt.gz' | zcat | grep -F 'OL3965446A' | head -n1
/type/author    /authors/OL3965446A     1       2008-04-30T20:50:18.033121      {"name": "I. Meyerson", "last_modified": {"type": "/type/datetime", "value": "2008-04-30 20:50:18.033121"}, "key": "/a/OL3965446A", "type": {"key": "/type/author"}, "id": 16613668, "revision": 1}

$ curl -L 'https://archive.org/download/ol_dump_2021-10-13/ol_dump_authors_2021-10-13.txt.gz' | zcat | grep -F 'OL3965446A' | head -n1
/type/author    /authors/OL3965446A     1       2008-04-30T20:50:18.033121      {"name": "I. Meyerson", "last_modified": {"type": "/type/datetime", "value": "2008-04-30T20:50:18.033121"}, "key": "/authors/OL3965446A", "type": {"key": "/type/author"}, "revision": 1}
```

### Proposal & Constraints
<!-- What is the proposed solution / implementation? Is there a precedent of this approach succeeding elsewhere? -->

### Related files
<!-- Files related to this issue; this is super useful for new contributors who might want to help! If you're not sure, leave this blank; a maintainer will add them. -->

### Stakeholders
<!-- @ tag stakeholders of this bug -->



Discussion:

Resolving patch:
diff --git a/openlibrary/data/dump.py b/openlibrary/data/dump.py
--- a/openlibrary/data/dump.py
+++ b/openlibrary/data/dump.py
@@ -30,18 +30,15 @@
 logger.setLevel(logging.DEBUG)
 
 
-def print_dump(json_records, filter=None):
+def print_dump(json_records, filter=None, print=print):
     """Print the given json_records in the dump format."""
-    for i, json_data in enumerate(json_records):
+    for i, raw_json_data in enumerate(json_records):
         if i % 1_000_000 == 0:
             log(f"{i:,}")
-        d = json.loads(json_data)
+        d = json.loads(raw_json_data)
         d.pop("id", None)
         d = _process_data(d)
 
-        if filter and filter(d) is False:
-            continue
-
         key = web.safestr(d["key"])
 
         # skip user and admin pages
@@ -53,8 +50,12 @@ def print_dump(json_records, filter=None):
         if key.startswith(("/b/", "/scan", "/old/")) or not key.startswith("/"):
             continue
 
+        if filter and filter(d) is False:
+            continue
+
         type_key = d["type"]["key"]
         timestamp = d["last_modified"]["value"]
+        json_data = json.dumps(d)
 
         print("\t".join([type_key, key, str(d["revision"]), timestamp, json_data]))
 
@@ -265,17 +266,13 @@ def make_bsddb(dbfile, dump_file):
 
 
 def _process_key(key):
-    mapping = (
-        "/l/",
-        "/languages/",
-        "/a/",
-        "/authors/",
-        "/b/",
-        "/books/",
-        "/user/",
-        "/people/",
-    )
-    for old, new in web.group(mapping, 2):
+    mapping = {
+        "/l/": "/languages/",
+        "/a/": "/authors/",
+        "/b/": "/books/",
+        "/user/": "/people/",
+    }
+    for old, new in mapping.items():
         if key.startswith(old):
             return new + key[len(old) :]
     return key


Test patch:
diff --git a/openlibrary/tests/data/dump.py b/openlibrary/tests/data/dump.py
new file mode 100644
--- /dev/null
+++ b/openlibrary/tests/data/dump.py
@@ -0,0 +1,61 @@
+import json
+
+from openlibrary.data.dump import print_dump
+
+
+class SpyPrint:
+    def __init__(self):
+        self.output = []
+
+    def __call__(self, msg: str):
+        self.output.append(msg)
+
+
+class TestPrintDump:
+    def test_fixes_prefixes(self):
+        records = [
+            {
+                "key": "/b/OL1M",
+                "type": {"key": "/type/edition"},
+                "revision": 1,
+                "last_modified": {"value": "2019-01-01T00:00:00.000"},
+            },
+        ]
+        spy_print = SpyPrint()
+
+        print_dump(map(json.dumps, records), print=spy_print)
+        assert spy_print.output == [
+            "\t".join([
+                "/type/edition",
+                "/books/OL1M",
+                "1",
+                "2019-01-01T00:00:00.000",
+                json.dumps({
+                    "key": "/books/OL1M",
+                    "type": {"key": "/type/edition"},
+                    "revision": 1,
+                    "last_modified": {"value": "2019-01-01T00:00:00.000"},
+                }),
+            ]),
+        ]
+
+    def test_excludes_sensitive_pages(self):
+        records = [
+            {"key": "/people/foo"},
+            {"key": "/user/foo"},
+            {"key": "/admin/foo"},
+        ]
+        spy_print = SpyPrint()
+
+        print_dump(map(json.dumps, records), print=spy_print)
+        assert spy_print.output == []
+
+    def test_excludes_obsolete_pages(self):
+        records = [
+            {"key": "/scan_record/foo"},
+            {"key": "/old/what"},
+        ]
+        spy_print = SpyPrint()
+
+        print_dump(map(json.dumps, records), print=spy_print)
+        assert spy_print.output == []
\ No newline at end of file