Repository: internetarchive/openlibrary

Issue:
scripts/copydocs.py fails on Python 3 w/ TypeError: POST data should be bytes
<!-- What problem are we solving? What does the experience look like today? What are the symptoms? -->

### Evidence / Screenshot (if possible)

At first blush, this one seems easy to fix but it quickly bumps into #4212 !!

my Mac % `docker-compose exec web bash`
openlibrary@500abfd9a786:/openlibrary$ `./scripts/copydocs.py /books/OL21692921M`
```
Traceback (most recent call last):
  File "./scripts/copydocs.py", line 328, in <module>
    main()
  File "./scripts/copydocs.py", line 315, in main
    dest.login("admin", "admin123")
  File "/openlibrary/openlibrary/api.py", line 99, in login
    response = self._request('/account/login', method='POST', data=data, headers=headers)
  File "/openlibrary/openlibrary/api.py", line 55, in _request
    return urllib.request.urlopen(req)
  File "/home/openlibrary/.pyenv/versions/3.8.6/lib/python3.8/urllib/request.py", line 222, in urlopen
    return opener.open(url, data, timeout)
  File "/home/openlibrary/.pyenv/versions/3.8.6/lib/python3.8/urllib/request.py", line 522, in open
    req = meth(req)
  File "/home/openlibrary/.pyenv/versions/3.8.6/lib/python3.8/urllib/request.py", line 1281, in do_request_
    raise TypeError(msg)
 an iterable of bytes, or a file object. It cannot be of type str.
```
### Relevant url?
<!-- `https://openlibrary.org/...` -->

### Steps to Reproduce
<!-- What steps caused you to find the bug? -->
1. Go to ...
2. Do ...

<!-- What actually happened after these steps? What did you expect to happen? -->
* Actual: 
* Expected: 

### Details

- **Logged in (Y/N)?** 
- **Browser type/version?** 
- **Operating system?** 
- **Environment (prod/dev/local)?** prod
<!-- If not sure, put prod -->

### Proposal & Constraints
<!-- What is the proposed solution / implementation? Is there a precedent of this approach succeeding elsewhere? -->

### Related files
<!-- Files related to this issue; this is super useful for new contributors who might want to help! If you're not sure, leave this blank; a maintainer will add them. -->

### Stakeholders
<!-- @ tag stakeholders of this bug -->



Discussion:
@SouthGoingZax Could you please look into this one?

Resolving patch:
diff --git a/openlibrary/api.py b/openlibrary/api.py
--- a/openlibrary/api.py
+++ b/openlibrary/api.py
@@ -22,19 +22,18 @@
 import json
 import web
 import logging
+import requests
 
 import six
-from six.moves import urllib
 from six.moves.configparser import ConfigParser
 
 logger = logging.getLogger("openlibrary.api")
 
 class OLError(Exception):
-    def __init__(self, http_error):
-        self.code = http_error.code
-        self.headers = http_error.headers
-        msg = http_error.msg + ": " + http_error.read()
-        Exception.__init__(self, msg)
+    def __init__(self, e):
+        self.code = e.response.status_code
+        self.headers = e.response.headers
+        Exception.__init__(self, "{}. Response: {}".format(e, e.response.text))
 
 
 class OpenLibrary:
@@ -42,18 +41,20 @@ def __init__(self, base_url="https://openlibrary.org"):
         self.base_url = base_url.rstrip('/') if base_url else "https://openlibrary.org"
         self.cookie = None
 
-    def _request(self, path, method='GET', data=None, headers=None):
+    def _request(self, path, method='GET', data=None, headers=None, params=None):
         logger.info("%s %s", method, path)
         url = self.base_url + path
         headers = headers or {}
+        params = params or {}
         if self.cookie:
             headers['Cookie'] = self.cookie
 
         try:
-            req = urllib.request.Request(url, data, headers)
-            req.get_method = lambda: method
-            return urllib.request.urlopen(req)
-        except urllib.error.HTTPError as e:
+            response = requests.request(method, url, data=data, headers=headers,
+                                        params=params)
+            response.raise_for_status()
+            return response
+        except requests.HTTPError as e:
             raise OLError(e)
 
     def autologin(self, section=None):
@@ -97,7 +98,7 @@ def login(self, username, password):
         try:
             data = json.dumps(dict(username=username, password=password))
             response = self._request('/account/login', method='POST', data=data, headers=headers)
-        except urllib.error.HTTPError as e:
+        except OLError as e:
             response = e
 
         if 'Set-Cookie' in response.headers:
@@ -105,8 +106,8 @@ def login(self, username, password):
             self.cookie =  ';'.join([c.split(';')[0] for c in cookies])
 
     def get(self, key, v=None):
-        data = self._request(key + '.json' + ('?v=%d' % v if v else '')).read()
-        return unmarshal(json.loads(data))
+        response = self._request(key + '.json', params={'v': v} if v else {})
+        return unmarshal(response.json())
 
     def get_many(self, keys):
         """Get multiple documents in a single request as a dictionary.
@@ -121,8 +122,8 @@ def get_many(self, keys):
             return self._get_many(keys)
 
     def _get_many(self, keys):
-        response = self._request("/api/get_many?" + urllib.parse.urlencode({"keys": json.dumps(keys)}))
-        return json.loads(response.read())['result']
+        response = self._request("/api/get_many", params={"keys": json.dumps(keys)})
+        return response.json()['result']
 
     def save(self, key, data, comment=None):
         headers = {'Content-Type': 'application/json'}
@@ -131,7 +132,7 @@ def save(self, key, data, comment=None):
             headers['Opt'] = '"%s/dev/docs/api"; ns=42' % self.base_url
             headers['42-comment'] = comment
         data = json.dumps(data)
-        return self._request(key, method="PUT", data=data, headers=headers).read()
+        return self._request(key, method="PUT", data=data, headers=headers).content
 
     def _call_write(self, name, query, comment, action):
         headers = {'Content-Type': 'application/json'}
@@ -146,7 +147,7 @@ def _call_write(self, name, query, comment, action):
             headers['42-action'] = action
 
         response = self._request('/api/' + name, method="POST", data=json.dumps(query), headers=headers)
-        return json.loads(response.read())
+        return response.json()
 
     def save_many(self, query, comment=None, action=None):
         return self._call_write('save_many', query, comment, action)
@@ -192,13 +193,12 @@ def unlimited_query(q):
         if 'limit' in q and q['limit'] == False:
             return unlimited_query(q)
         else:
-            q = json.dumps(q)
-            response = self._request("/query.json?" + urllib.parse.urlencode(dict(query=q)))
-            return unmarshal(json.loads(response.read()))
+            response = self._request("/query.json", params=dict(query=json.dumps(q)))
+            return unmarshal(response.json())
 
     def import_ocaid(self, ocaid, require_marc=True):
         data = {'identifier': ocaid, 'require_marc': 'true' if require_marc else 'false'}
-        return self._request('/api/import/ia', method='POST', data=urllib.parse.urlencode(data)).read()
+        return self._request('/api/import/ia', method='POST', data=data).content
 
 
 def marshal(data):
diff --git a/openlibrary/plugins/openlibrary/connection.py b/openlibrary/plugins/openlibrary/connection.py
--- a/openlibrary/plugins/openlibrary/connection.py
+++ b/openlibrary/plugins/openlibrary/connection.py
@@ -86,6 +86,11 @@ def save(self, sitename, path, data):
         return self.conn.request(sitename, path, 'POST', data)
 
     def save_many(self, sitename, data):
+        # Work-around for https://github.com/internetarchive/openlibrary/issues/4285
+        # Infogami seems to read encoded bytes as a string with a byte literal inside
+        # of it, which is invalid JSON and also can't be decode()'d.
+        if isinstance(data.get('query'), bytes):
+            data['query'] = data['query'].decode()
         return self.conn.request(sitename, '/save_many', 'POST', data)
 
     def store_get(self, sitename, path):
diff --git a/scripts/copydocs.py b/scripts/copydocs.py
--- a/scripts/copydocs.py
+++ b/scripts/copydocs.py
@@ -69,7 +69,7 @@ def parse_args():
     parser.add_option("-c", "--comment", dest="comment", default="", help="comment")
     parser.add_option("--src", dest="src", metavar="SOURCE_URL", default="http://openlibrary.org/", help="URL of the source server (default: %default)")
     parser.add_option("--dest", dest="dest", metavar="DEST_URL", default="http://localhost", help="URL of the destination server (default: %default)")
-    parser.add_option("-r", "--recursive", dest="recursive", action='store_true', default=False, help="Recursively fetch all the referred docs.")
+    parser.add_option("-r", "--recursive", dest="recursive", action='store_true', default=True, help="Recursively fetch all the referred docs.")
     parser.add_option("-l", "--list", dest="lists", action="append", default=[], help="copy docs from a list.")
     return parser.parse_args()
 


Test patch: