Repository: internetarchive/openlibrary

Issue:
Add unit tests for GoodReads import
GoodReads imports broke without any automatic indication (#4216 ). Let's add some unit tests that test this code (code in question: https://github.com/internetarchive/openlibrary/pull/3597 )

## Summary of requirements

- [ ] The tests should have caught #4216 had they been in place.

## Stakeholders

@SouthGoingZax @tabshaikh 


Discussion:
@SouthGoingZax Would you be able to work on this one? That CSV file you used for testing would be mighty helpful :) 

Resolving patch:
diff --git a/openlibrary/plugins/upstream/account.py b/openlibrary/plugins/upstream/account.py
--- a/openlibrary/plugins/upstream/account.py
+++ b/openlibrary/plugins/upstream/account.py
@@ -865,24 +865,7 @@ def GET(self):
 
     @require_login
     def POST(self):
-        import csv
-        i = web.input(csv={})
-        csv_payload = i.csv.value if isinstance(i.csv.value, str) else i.csv.value.decode()
-        csv_file = csv.reader(csv_payload.splitlines(), delimiter=',', quotechar='"')
-        header = next(csv_file)
-        books = {}
-        books_wo_isbns = {}
-        for book in list(csv_file):
-            _book = dict(zip(header, book))
-            _book['ISBN'] = _book['ISBN'].replace('"','').replace('=','')
-            _book['ISBN13'] = _book['ISBN13'].replace('"','').replace('=','')
-            if _book['ISBN'] != '':
-                books[_book['ISBN']] = _book
-            elif _book['ISBN13'] != '':
-                books[_book['ISBN13']] = _book
-                books[_book['ISBN13']]['ISBN'] = _book['ISBN13']
-            else:
-                books_wo_isbns[_book['Book Id']] = _book
+        books, books_wo_isbns = process_goodreads_csv(web.input())
         return render['account/import'](books, books_wo_isbns)
 
 class export_books(delegate.page):
@@ -959,3 +942,24 @@ def g(*a, **kw):
         finally:
             web.ctx.headers = []
     return g
+
+
+def process_goodreads_csv(i):
+    import csv
+    csv_payload = i.csv if isinstance(i.csv, str) else i.csv.decode()
+    csv_file = csv.reader(csv_payload.splitlines(), delimiter=',', quotechar='"')
+    header = next(csv_file)
+    books = {}
+    books_wo_isbns = {}
+    for book in list(csv_file):
+        _book = dict(zip(header, book))
+        isbn = _book['ISBN'] = _book['ISBN'].replace('"', '').replace('=', '')
+        isbn_13 = _book['ISBN13'] = _book['ISBN13'].replace('"', '').replace('=', '')
+        if isbn != '':
+            books[isbn] = _book
+        elif isbn_13 != '':
+            books[isbn_13] = _book
+            books[isbn_13]['ISBN'] = isbn_13
+        else:
+            books_wo_isbns[_book['Book Id']] = _book
+    return books, books_wo_isbns


Test patch:
diff --git a/openlibrary/plugins/upstream/tests/test_account.py b/openlibrary/plugins/upstream/tests/test_account.py
--- a/openlibrary/plugins/upstream/tests/test_account.py
+++ b/openlibrary/plugins/upstream/tests/test_account.py
@@ -1,7 +1,20 @@
+# -*- coding: utf-8 -*-
+
 from .. import account
 import web
+import os
 import re
 import pytest
+import sys
+
+
+def open_test_data(filename):
+    """Returns a file handle to file with specified filename inside test_data directory.
+    """
+    root = os.path.dirname(__file__)
+    fullpath = os.path.join(root, 'test_data', filename)
+    return open(fullpath, mode='rb')
+
 
 def test_create_list_doc(wildcard):
     key = "account/foo/verify"
@@ -21,6 +34,134 @@ def test_create_list_doc(wildcard):
         "expires_on": wildcard
     }
 
+
+class TestGoodReadsImport:
+    def setup_method(self, method):
+        with open_test_data('goodreads_library_export.csv') as reader:
+            self.csv_data = reader.read()
+
+        self.expected_books = {
+            "0142402494": {
+                "Additional Authors": "Florence Lamborn, Louis S. Glanzman",
+                "Author": "Astrid Lindgren",
+                "Author l-f": "Lindgren, Astrid",
+                "Average Rating": "4.13",
+                "BCID": "",
+                "Binding": "Mass Market Paperback",
+                "Book Id": "19302",
+                "Bookshelves": "to-read",
+                "Bookshelves with positions": "to-read (#2)",
+                "Condition": "",
+                "Condition Description": "",
+                "Date Added": "2020/12/13",
+                "Date Read": "",
+                "Exclusive Shelf": "to-read",
+                "ISBN": "0142402494",
+                "ISBN13": "9780142402498",
+                "My Rating": "0",
+                "My Review": "",
+                "Number of Pages": "160",
+                "Original Publication Year": "1945",
+                "Original Purchase Date": "",
+                "Original Purchase Location": "",
+                "Owned Copies": "0",
+                "Private Notes": "",
+                "Publisher": "Puffin Books",
+                "Read Count": "0",
+                "Recommended By": "",
+                "Recommended For": "",
+                "Spoiler": "",
+                "Title": "Pippi Longstocking (Pippi Långstrump, #1)",
+                "Year Published": "2005"
+            },
+            "0735214484": {
+                "Additional Authors": "",
+                "Author": "David   Epstein",
+                "Author l-f": "Epstein, David",
+                "Average Rating": "4.16",
+                "BCID": "",
+                "Binding": "Hardcover",
+                "Book Id": "41795733",
+                "Bookshelves": "to-read",
+                "Bookshelves with positions": "to-read (#1)",
+                "Condition": "",
+                "Condition Description": "",
+                "Date Added": "2020/12/13",
+                "Date Read": "",
+                "Exclusive Shelf": "to-read",
+                "ISBN": "0735214484",
+                "ISBN13": "9780735214484",
+                "My Rating": "0",
+                "My Review": "",
+                "Number of Pages": "352",
+                "Original Publication Year": "2019",
+                "Original Purchase Date": "",
+                "Original Purchase Location": "",
+                "Owned Copies": "0",
+                "Private Notes": "",
+                "Publisher": "Riverhead Books",
+                "Read Count": "0",
+                "Recommended By": "",
+                "Recommended For": "",
+                "Spoiler": "",
+                "Title": "Range: Why Generalists Triumph in a Specialized World",
+                "Year Published": "2019"
+            }
+        }
+
+        self.expected_books_wo_isbns = {
+            "99999999999": {
+                "Additional Authors": "",
+                "Author": "AuthorWith NoISBN",
+                "Author l-f": "NoISBN, AuthorWith",
+                "Average Rating": "4.16",
+                "BCID": "",
+                "Binding": "Hardcover",
+                "Book Id": "99999999999",
+                "Bookshelves": "to-read",
+                "Bookshelves with positions": "to-read (#1)",
+                "Condition": "",
+                "Condition Description": "",
+                "Date Added": "2020/12/13",
+                "Date Read": "",
+                "Exclusive Shelf": "to-read",
+                "ISBN": "",
+                "ISBN13": "",
+                "My Rating": "0",
+                "My Review": "",
+                "Number of Pages": "352",
+                "Original Publication Year": "2019",
+                "Original Purchase Date": "",
+                "Original Purchase Location": "",
+                "Owned Copies": "0",
+                "Private Notes": "",
+                "Publisher": "Test Publisher",
+                "Read Count": "0",
+                "Recommended By": "",
+                "Recommended For": "",
+                "Spoiler": "",
+                "Title": "Test Book Title With No ISBN",
+                "Year Published": "2019"
+            }
+        }
+
+    @pytest.mark.skipif(sys.version_info < (3, 0),
+                        reason="Python2's csv module doesn't support Unicode")
+    def test_process_goodreads_csv_with_utf8(self):
+        books, books_wo_isbns = account.process_goodreads_csv(
+            web.storage({'csv': self.csv_data.decode('utf-8')}))
+        assert books == self.expected_books
+        assert books_wo_isbns == self.expected_books_wo_isbns
+
+    def test_process_goodreads_csv_with_bytes(self):
+        # Note: In Python2, reading data as bytes returns a string, which should
+        # also be supported by account.process_goodreads_csv()
+        books, books_wo_isbns = account.process_goodreads_csv(
+            web.storage({'csv': self.csv_data}))
+        assert books == self.expected_books
+        assert books_wo_isbns == self.expected_books_wo_isbns
+
+
 @pytest.mark.xfail
 class TestAccount:
     def signup(self, b, displayname, username, password, email):
diff --git a/openlibrary/plugins/upstream/tests/test_data/goodreads_library_export.csv b/openlibrary/plugins/upstream/tests/test_data/goodreads_library_export.csv
new file mode 100644
--- /dev/null
+++ b/openlibrary/plugins/upstream/tests/test_data/goodreads_library_export.csv
@@ -0,0 +1,4 @@
+Book Id,Title,Author,Author l-f,Additional Authors,ISBN,ISBN13,My Rating,Average Rating,Publisher,Binding,Number of Pages,Year Published,Original Publication Year,Date Read,Date Added,Bookshelves,Bookshelves with positions,Exclusive Shelf,My Review,Spoiler,Private Notes,Read Count,Recommended For,Recommended By,Owned Copies,Original Purchase Date,Original Purchase Location,Condition,Condition Description,BCID
+19302,"Pippi Longstocking (Pippi Långstrump, #1)",Astrid Lindgren,"Lindgren, Astrid","Florence Lamborn, Louis S. Glanzman","=""0142402494""","=""9780142402498""",0,4.13,Puffin Books,Mass Market Paperback,160,2005,1945,,2020/12/13,to-read,to-read (#2),to-read,,,,0,,,0,,,,,
+41795733,Range: Why Generalists Triumph in a Specialized World,David   Epstein,"Epstein, David",,"=""0735214484""","=""9780735214484""",0,4.16,Riverhead Books,Hardcover,352,2019,2019,,2020/12/13,to-read,to-read (#1),to-read,,,,0,,,0,,,,,
+99999999999,Test Book Title With No ISBN,AuthorWith NoISBN,"NoISBN, AuthorWith",,,,0,4.16,Test Publisher,Hardcover,352,2019,2019,,2020/12/13,to-read,to-read (#1),to-read,,,,0,,,0,,,,,