Repository: internetarchive/openlibrary

Issue:
Need to get oldump cron job running in Docker
<!-- What problem are we solving? What does the experience look like today? What are the symptoms? -->

### Evidence / Screenshot (if possible)
Subtask of #4383
Fixed by ~#4627~ + internetarchive/olsystem#118

### Stakeholders
@cdrini



Discussion:

Resolving patch:
diff --git a/openlibrary/data/dump.py b/openlibrary/data/dump.py
--- a/openlibrary/data/dump.py
+++ b/openlibrary/data/dump.py
@@ -8,19 +8,18 @@
 """
 from __future__ import print_function
 
-import sys
+import gzip
+import itertools
+import json
 import os
-import web
 import re
+import sys
 import time
-import json
-import itertools
-import gzip
 
 import six
+import web
 
 from infogami.infobase.utils import flatten_dict
-
 from openlibrary.data import db
 from openlibrary.data.sitemap import generate_html_index, generate_sitemaps
 from openlibrary.plugins.openlibrary.processors import urlsafe
@@ -42,11 +41,11 @@ def print_dump(json_records, filter=None):
         json_data = json.dumps(d)
 
         # skip user and admin pages
-        if key.startswith("/people/") or key.startswith("/admin/"):
+        if key.startswith(("/people/", "/admin/")):
             continue
 
         # skip obsolete pages. Obsolete pages include volumes, scan_records and users marked as spam.
-        if key.startswith("/b/") or key.startswith("/scan") or key.startswith("/old/") or not key.startswith("/"):
+        if key.startswith(("/b/", "/scan", "/old/")) or not key.startswith("/"):
             continue
 
         if filter and filter(d) is False:
@@ -56,6 +55,8 @@ def print_dump(json_records, filter=None):
 
 def read_data_file(filename):
     for line in xopen(filename):
+        if six.PY3 and not isinstance(line, str):
+            line = line.decode("utf-8")
         thing_id, revision, json_data = line.strip().split("\t")
         yield pgdecode(json_data)
 
@@ -75,6 +76,8 @@ def read_tsv(file, strip=True):
         file = xopen(file)
 
     for i, line in enumerate(file):
+        if six.PY3 and not isinstance(line, str):
+            line = line.decode("utf-8")
         if i % 1000000 == 0:
             log(i)
         if strip:
@@ -105,20 +108,18 @@ def sort_dump(dump_file=None, tmpdir="/tmp/", buffer_size="1G"):
     M = 1024*1024
 
     filenames = [os.path.join(tmpdir, "%02x.txt.gz" % i) for i in range(256)]
-    files = [gzip.open(f, 'w') for f in filenames]
-
-    if dump_file is None:
-        stdin = sys.stdin
-    else:
-        stdin = xopen(dump_file)
+    files = [gzip.open(f, 'wb') for f in filenames]
+    stdin = xopen(dump_file) if dump_file else sys.stdin
 
     # split the file into 256 chunks using hash of key
     log("splitting", dump_file)
     for i, line in enumerate(stdin):
+        if six.PY3 and not isinstance(line, bytes):
+            line = line.encode("utf-8")
         if i % 1000000 == 0:
             log(i)
 
-        type, key, revision, timestamp, json_data = line.strip().split("\t")
+        type, key, revision, timestamp, json_data = line.strip().split(b"\t")
         findex = hash(key) % 256
         files[findex].write(line)
 
@@ -164,21 +165,19 @@ def generate_idump(day, **db_parameters):
 
 def split_dump(dump_file=None, format="oldump_%s.txt"):
     """Split dump into authors, editions and works."""
-    types = ["/type/edition", "/type/author", "/type/work", "/type/redirect"]
+    types = ("/type/edition", "/type/author", "/type/work", "/type/redirect")
     files = {}
     for t in types:
         tname = t.split("/")[-1] + "s"
-        files[t] = xopen(format % tname, "w")
-
-    if dump_file is None:
-        stdin = sys.stdin
-    else:
-        stdin = xopen(dump_file)
+        files[t] = xopen(format % tname, "wb")
 
+    stdin = xopen(dump_file) if dump_file else sys.stdin
     for i, line in enumerate(stdin):
+        if six.PY3 and not isinstance(line, bytes):
+            line = line.encode("utf-8")
         if i % 1000000 == 0:
             log(i)
-        type, rest = line.split("\t", 1)
+        type, rest = line.split(b"\t", 1)
         if type in files:
             files[type].write(line)
 
@@ -190,7 +189,7 @@ def make_index(dump_file):
 
     for type, key, revision, timestamp, json_data in read_tsv(dump_file):
         data = json.loads(json_data)
-        if type == '/type/edition' or type == '/type/work':
+        if type in ('/type/edition', '/type/work'):
             title = data.get('title', 'untitled')
             path = key + '/' + urlsafe(title)
         elif type == '/type/author':
@@ -255,7 +254,7 @@ def _process_data(data):
             data['key'] = _process_key(data['key'])
 
         # convert date to ISO format
-        if 'type' in data and data['type'] == '/type/datetime':
+        if data.get('type') == '/type/datetime':
             data['value'] = data['value'].replace(' ', 'T')
 
         return dict((k, _process_data(v)) for k, v in data.items())
diff --git a/scripts/sitemaps/sitemap.py b/scripts/sitemaps/sitemap.py
--- a/scripts/sitemaps/sitemap.py
+++ b/scripts/sitemaps/sitemap.py
@@ -5,15 +5,16 @@
     python sitemaps.py suffix dump.txt.gz
 """
 
-import web
-import os
-import itertools
 import datetime
 import gzip
-import re
+import itertools
 import json
+import os
+import re
 import time
 
+import web
+
 t_sitemap = """$def with (things)
 <?xml version="1.0" encoding="UTF-8"?>
 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
@@ -40,10 +41,7 @@
 siteindex = web.template.Template(t_siteindex)
 
 def xopen(filename):
-    if filename.endswith(".gz"):
-        return gzip.open(filename)
-    else:
-        return open(filename)
+    return gzip.open(filename) if filename.endswith(".gz") else open(filename)
 
 def urlsafe(name):
     """Slugifies the name to produce OL url slugs
@@ -68,7 +66,7 @@ def process_dump(dumpfile):
 
     The summary file contains: sort-key, path and last_modified columns.
     """
-    rows = (line.strip().split("\t") for line in xopen(dumpfile))
+    rows = (line.decode().strip().split("\t") for line in xopen(dumpfile))
     for type, key, revision, last_modified, jsontext in rows:
         if type not in ['/type/work', '/type/author']:
             continue
@@ -77,7 +75,7 @@ def process_dump(dumpfile):
         title = doc.get('name', '') if type == '/type/author' \
                 else doc.get('title', '')
 
-        path = key + "/" + urlsafe(title.strip()).encode('utf-8')
+        path = key + "/" + urlsafe(title.strip())
 
         last_modified = last_modified.replace(' ', 'T') + 'Z'
         sortkey = get_sort_key(key)
@@ -126,25 +124,23 @@ def write(path, text):
     try:
         text = web.safestr(text)
         log('writing', path, text.count('\n'))
-        f = gzip.open(path, 'w')
-        f.write(text)
-        f.close()
-    except:
-        print('write fail')
+        with gzip.open(path, 'w') as f:
+            f.write(text.encode())
+    except Exception as e:
+        print('write fail {}'.format(e))
     #os.system("gzip " + path)
 
 def write_tsv(path, rows):
     lines = ("\t".join(row) + "\n" for row in rows)
-    f = open(path, "w")
-    f.writelines(lines)
-    f.close()
+    with open(path, "w") as f:
+        f.writelines(lines)
 
 def system_memory():
     """Returns system memory in MB."""
     try:
         x = os.popen("cat /proc/meminfo | grep MemTotal | sed 's/[^0-9]//g'").read()
         # proc gives memory in KB, converting it to MB
-        return int(x)/1024
+        return int(x) // 1024
     except IOError:
         # default to 1024MB
         return 1024


Test patch: