Repository: internetarchive/openlibrary

Issue:
fixes remaining archival and sitemaps bugs
- Removes the `$TMPDIR/sitemaps` dir as part of `cleanup`
- Fixes the archive step which was failing because of a syntax issue only using `[]` instead of `[[ ]]`

good riddance :rofl: 
#5892 #6358  #6253 

<!-- What does this PR achieve? [feature|hotfix|fix|refactor] -->



Discussion:

Resolving patch:
diff --git a/openlibrary/data/dump.py b/openlibrary/data/dump.py
--- a/openlibrary/data/dump.py
+++ b/openlibrary/data/dump.py
@@ -18,21 +18,17 @@
 
 import web
 
-from infogami import config
-from openlibrary.config import load_config
 from openlibrary.data import db
 from openlibrary.data.sitemap import generate_html_index, generate_sitemaps
 from openlibrary.plugins.openlibrary.processors import urlsafe
-from openlibrary.utils.sentry import Sentry
 
 logger = logging.getLogger(__file__)
 logger.setLevel(logging.DEBUG)
 
 
 def log(*args) -> None:
-    msg = f"{datetime.now():%Y-%m-%d %H:%M:%S} [openlibrary.dump] " + " ".join(
-        str(a) for a in args
-    )
+    args_str = " ".join(str(a) for a in args)
+    msg = f"{datetime.now():%Y-%m-%d %H:%M:%S} [openlibrary.dump] {args_str}"
     logger.info(msg)
     print(msg, file=sys.stderr)
 
@@ -67,7 +63,7 @@ def print_dump(json_records, filter=None):
 
         print("\t".join([type_key, key, str(d["revision"]), timestamp, json_data]))
     minutes = (datetime.now() - start_time).seconds // 60
-    log(f"print_dump() processed {i} records in {minutes} minutes.")
+    log(f"    print_dump() processed {i:,} records in {minutes:,} minutes.")
 
 
 def read_data_file(filename: str, max_lines: int = 0):
@@ -83,7 +79,7 @@ def read_data_file(filename: str, max_lines: int = 0):
         if max_lines and i >= max_lines:
             break
     minutes = (datetime.now() - start_time).seconds // 60
-    log(f"read_data_file() processed {i} records in {minutes} minutes.")
+    log(f"read_data_file() processed {i:,} records in {minutes:,} minutes.")
 
 
 def xopen(path: str, mode: str):
@@ -107,7 +103,7 @@ def read_tsv(file, strip=True):
             line = line.strip()
         yield line.split("\t")
     minutes = (datetime.now() - start_time).seconds // 60
-    log(f"read_tsv() processed {i} records in {minutes} minutes.")
+    log(f" read_tsv() processed {i:,} records in {minutes:,} minutes.")
 
 
 def generate_cdump(data_file, date=None):
@@ -165,7 +161,7 @@ def sort_dump(dump_file=None, tmpdir="/tmp/", buffer_size="1G"):
         if status != 0:
             raise Exception("sort failed with status %d" % status)
     minutes = (datetime.now() - start_time).seconds // 60
-    log(f"sort_dump() processed {i} records in {minutes} minutes.")
+    log(f"sort_dump() processed {i:,} records in {minutes:,} minutes.")
 
 
 def generate_dump(cdump_file=None):
@@ -175,7 +171,7 @@ def generate_dump(cdump_file=None):
     """
 
     def process(data):
-        revision = lambda cols: int(cols[2])
+        revision = lambda cols: int(cols[2])  # noqa: E731
         for key, rows in itertools.groupby(data, key=lambda cols: cols[1]):
             row = max(rows, key=revision)
             yield row
@@ -186,7 +182,7 @@ def process(data):
     # group by key and find the max by revision
     sys.stdout.writelines(tjoin(row) for row in process(data))
     minutes = (datetime.now() - start_time).seconds // 60
-    log(f"generate_dump({cdump_file}) ran in {minutes} minutes.")
+    log(f"generate_dump({cdump_file}) ran in {minutes:,} minutes.")
 
 
 def generate_idump(day, **db_parameters):
@@ -208,6 +204,7 @@ def generate_idump(day, **db_parameters):
 
 def split_dump(dump_file=None, format="oldump_%s.txt"):
     """Split dump into authors, editions and works."""
+    log(f"split_dump({dump_file}, format={format})")
     start_time = datetime.now()
     types = ("/type/edition", "/type/author", "/type/work", "/type/redirect")
     files = {}
@@ -226,11 +223,12 @@ def split_dump(dump_file=None, format="oldump_%s.txt"):
     for f in files.values():
         f.close()
     minutes = (datetime.now() - start_time).seconds // 60
-    log(f"split_dump() processed {i} records in {minutes} minutes.")
+    log(f"split_dump() processed {i:,} records in {minutes:,} minutes.")
 
 
 def make_index(dump_file):
     """Make index with "path", "title", "created" and "last_modified" columns."""
+    log(f"make_index({dump_file})")
     start_time = datetime.now()
     for i, type, key, revision, timestamp, json_data in enumerate(read_tsv(dump_file)):
         data = json.loads(json_data)
@@ -252,7 +250,7 @@ def make_index(dump_file):
             created = "-"
         print("\t".join([web.safestr(path), web.safestr(title), created, timestamp]))
     minutes = (datetime.now() - start_time).seconds // 60
-    log(f"split_dump() processed {i} records in {minutes} minutes.")
+    log(f"make_index() processed {i:,} records in {minutes:,} minutes.")
 
 
 def _process_key(key):
@@ -347,13 +345,4 @@ def main(cmd, args):
 
 
 if __name__ == "__main__":
-    ol_config = os.getenv("OL_CONFIG")
-    if ol_config:
-        logger.info(f"loading config from {ol_config}")
-        load_config(ol_config)
-        sentry = Sentry(getattr(config, "sentry_cron_jobs", {}))
-        if sentry.enabled:
-            sentry.init()
-
-    log(f"sentry.enabled = {bool(ol_config and sentry.enabled)}")
     main(sys.argv[1], sys.argv[2:])
diff --git a/scripts/oldump.py b/scripts/oldump.py
--- a/scripts/oldump.py
+++ b/scripts/oldump.py
@@ -1,16 +1,38 @@
 #!/usr/bin/env python
 
+import logging
+import os
 import sys
 from datetime import datetime
 
 import _init_path  # noqa: F401  Imported for its side effect of setting PYTHONPATH
 
+logger = logging.getLogger(__file__)
+logger.setLevel(logging.DEBUG)
+
+
+def log(*args) -> None:
+    args_str = " ".join(str(a) for a in args)
+    msg = f"{datetime.now():%Y-%m-%d %H:%M:%S} [openlibrary.dump] {args_str}"
+    logger.info(msg)
+    print(msg, file=sys.stderr)
 
-if __name__ == "__main__":
-    now = f"{datetime.now():%Y-%m-%d %H:%M:%S}"  # 2022-06-01 07:18:34
-    py_ver = "Python {}.{}.{}".format(*sys.version_info)  # Python 3.10.4
-    print(f"{now} [openlibrary.dump] {sys.argv} on {py_ver}", file=sys.stderr)
 
+if __name__ == "__main__":
+    from infogami import config
+    from openlibrary.config import load_config
     from openlibrary.data import dump
+    from openlibrary.utils.sentry import Sentry
+
+    log("{} on Python {}.{}.{}".format(sys.argv, *sys.version_info))  # Python 3.10.4
+
+    ol_config = os.getenv("OL_CONFIG")
+    if ol_config:
+        logger.info(f"loading config from {ol_config}")
+        load_config(ol_config)
+        sentry = Sentry(getattr(config, "sentry_cron_jobs", {}))
+        if sentry.enabled:
+            sentry.init()
+    log(f"sentry.enabled = {bool(ol_config and sentry.enabled)}")
 
     dump.main(sys.argv[1], sys.argv[2:])
diff --git a/scripts/sitemaps/sitemap.py b/scripts/sitemaps/sitemap.py
--- a/scripts/sitemaps/sitemap.py
+++ b/scripts/sitemaps/sitemap.py
@@ -6,13 +6,13 @@
     python sitemaps.py suffix dump.txt.gz
 """
 
-import datetime
 import gzip
 import itertools
 import json
+import logging
 import os
 import re
-import time
+from datetime import datetime
 
 import web
 
@@ -38,10 +38,20 @@
 </sitemapindex>
 """
 
+logger = logging.getLogger(__file__)
+logger.setLevel(logging.DEBUG)
+
 sitemap = web.template.Template(t_sitemap)
 siteindex = web.template.Template(t_siteindex)
 
 
+def log(*args) -> None:
+    args_str = " ".join(str(a) for a in args)
+    msg = f"{datetime.now():%Y-%m-%d %H:%M:%S} [openlibrary.dump] {args_str}"
+    logger.info(msg)
+    print(msg, file=sys.stderr)
+
+
 def xopen(filename):
     return gzip.open(filename) if filename.endswith(".gz") else open(filename)
 
@@ -60,7 +70,7 @@ def urlsafe(name):
     space = ' \n\r'
 
     unsafe = reserved + delims + unwise + space
-    pattern = '[%s]+' % "".join(re.escape(c) for c in unsafe)
+    pattern = f"[{''.join(re.escape(c) for c in unsafe)}]+"
     safepath_re = re.compile(pattern)
     return safepath_re.sub('_', name).replace(' ', '-').strip('_')[:100]
 
@@ -116,14 +126,14 @@ def generate_sitemaps(filename):
             things.append(web.storage(path=path, last_modified=last_modified))
 
         if things:
-            write("sitemaps/sitemap_%s.xml.gz" % sortkey, sitemap(things))
+            write(f"sitemaps/sitemap_{sortkey}.xml.gz", sitemap(things))
 
 
 def generate_siteindex():
     filenames = sorted(os.listdir("sitemaps"))
     if "siteindex.xml.gz" in filenames:
         filenames.remove("siteindex.xml.gz")
-    timestamp = datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S') + 'Z'
+    timestamp = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S') + 'Z'
     index = siteindex(filenames, timestamp)
     write("sitemaps/siteindex.xml.gz", index)
 
@@ -135,7 +145,7 @@ def write(path, text):
         with gzip.open(path, 'w') as f:
             f.write(text.encode())
     except Exception as e:
-        print(f'write fail {e}')
+        log(f'write fail {e}')
     # os.system("gzip " + path)
 
 
@@ -163,11 +173,6 @@ def system(cmd):
         raise Exception("%r failed with exit status: %d" % (cmd, status))
 
 
-def log(*args):
-    msg = " ".join(map(str, args))
-    print(f"{time.asctime()} {msg}")
-
-
 def main(dumpfile):
     system("rm -rf sitemaps sitemaps_data.txt*; mkdir sitemaps")
 


Test patch: