Repository: internetarchive/openlibrary

Issue:
Enriching Reading Log data exports (more fields)
* Currently Reading Log dumps (https://openlibrary.org/account/import) are limited to IDs because large Reading Logs (500+ items) can cause performance issues aggregating results in memory and and serving as large files. Currently, Lists impose an export limit if there are more than ~1k items
* The manual work around is to look over each Reading Log entry (e.g. OL16801714W) and to fetch the json for the corresponding item from Open Library API or the data dumps: https://openlibrary.org/works/OL16801714W.json
* 2 possible solutions to this problem are:
  * Give an option to download all work data but limit to e.g. 500 results (and show a notice, similar to lists page)
  * Offer a js option which allows one (from the OL website) to download pages of ~500 results from Reading Log (calculated based on total number of patron's books on Reading Log)
  
Questions for community:
* What format of data would you prefer/expect? CSV? JSON?

cc: @jimchamp @mheiman @seabelis @jimman2003 


Discussion:
My sense is that anyone able to manipulate a JSON file to do anything useful is capable of using the existing API.

If a non-programmer wants their reading log data for something, they're probably hoping to manage it in a spreadsheet or import it to another service, which suggests CSV. But without having some clear use cases, it's hard to know what fields would be helpful to include beyond the obvious title, author, year.
csv could be useful to make statistics.
The number of books read in a year, languages, authors, theme ...

so I would suggest for the columns: 

1. title
2. author
3. isbn ?
4. my rating
5. number of pages
6. year published
7. date read
8. read count
9. series ?


Personally, I like to export my data in csv format, for future proofing purposes.
@Fl0rent would like to give it a try?
@mekarpeles what do you think about the points mentioned by @mheiman and @Fl0rent 
@tuminzee a try for ?

Resolving patch:
diff --git a/openlibrary/core/bookshelves.py b/openlibrary/core/bookshelves.py
--- a/openlibrary/core/bookshelves.py
+++ b/openlibrary/core/bookshelves.py
@@ -1,6 +1,8 @@
 import logging
 from datetime import date
 from typing import Literal, Optional, cast
+from collections.abc import Iterable
+
 from openlibrary.utils.dateutil import DATE_ONE_MONTH_AGO, DATE_ONE_WEEK_AGO
 
 from . import db
@@ -94,17 +96,8 @@ def most_logged_books(
         query = f'select work_id, count(*) as cnt from bookshelves_books {where}'
         query += ' group by work_id order by cnt desc limit $limit offset $offset'
         logger.info("Query: %s", query)
-        logged_books = list(
-            oldb.query(
-                query,
-                vars={
-                    'shelf_id': shelf_id,
-                    'limit': limit,
-                    'offset': offset,
-                    'since': since,
-                },
-            )
-        )
+        data = {'shelf_id': shelf_id, 'limit': limit, 'offset': offset, 'since': since}
+        logged_books = list(oldb.query(query, vars=data))
         return cls.fetch(logged_books) if fetch else logged_books
 
     @classmethod
@@ -180,21 +173,17 @@ def get_users_logged_books(
         cls,
         username: str,
         bookshelf_id: str = None,
-        limit=100,
-        page=1,
+        limit: int = 100,
+        page: int = 1,  # Not zero-based counting!
         sort: Literal['created asc', 'created desc'] = 'created desc',
-    ) -> list:
+    ) -> list[dict]:
         """Returns a list of Reading Log database records for books which
         the user has logged. Records are described in core/schema.py
         and include:
 
         :param username: who logged this book
-        :param work_id: the Open Library work ID as an int (e.g. OL123W becomes 123)
         :param bookshelf_id: the ID of the bookshelf, see: PRESET_BOOKSHELVES.
             If bookshelf_id is None, return books from all bookshelves.
-        :param edition_id: the specific edition logged, if applicable
-        :param created: date the book was logged
-
         """
         oldb = db.get_db()
         page = int(page or 1)
@@ -225,9 +214,38 @@ def get_users_logged_books(
             data = {'username': username}
         return list(oldb.query(query, vars=data))
 
+    @classmethod
+    def iterate_users_logged_books(cls, username: str) -> Iterable[dict]:
+        """
+        Heavy users will have long lists of books which consume lots of memory and
+        cause performance issues.  So, instead of creating a big list, let's repeatedly
+        get small lists like get_users_logged_books() and yield one book at a time.
+        """
+        if not username or not isinstance(username, str):
+            raise ValueError(f"username must be a string, not {username}.")
+        oldb = db.get_db()
+        block = 0
+        LIMIT = 100  # Is there an ideal block size?!?
+
+        def get_a_block_of_books() -> list:
+            data = {"username": username, "limit": LIMIT, "offset": LIMIT * block}
+            query = (
+                "SELECT * from bookshelves_books WHERE username=$username "
+                "ORDER BY created DESC LIMIT $limit OFFSET $offset"
+            )
+            return list(oldb.query(query, vars=data))
+
+        while books := get_a_block_of_books():
+            block += 1
+            yield from books
+
     @classmethod
     def get_recently_logged_books(
-        cls, bookshelf_id=None, limit=50, page=1, fetch=False
+        cls,
+        bookshelf_id=None,
+        limit=50,
+        page=1,
+        fetch=False,
     ) -> list:
         oldb = db.get_db()
         page = int(page or 1)
@@ -324,7 +342,7 @@ def remove(cls, username: str, work_id: str, bookshelf_id: str = None):
                 where=('work_id=$work_id AND username=$username'),
                 vars=where,
             )
-        except:  # we want to catch no entry exists
+        except Exception:  # we want to catch no entry exists
             return None
 
     @classmethod
@@ -335,7 +353,7 @@ def get_works_shelves(cls, work_id: str, lazy: bool = False):
         try:
             result = oldb.query(query, vars={'work_id': work_id})
             return result if lazy else list(result)
-        except:
+        except Exception:
             return None
 
     @classmethod
diff --git a/openlibrary/plugins/upstream/account.py b/openlibrary/plugins/upstream/account.py
--- a/openlibrary/plugins/upstream/account.py
+++ b/openlibrary/plugins/upstream/account.py
@@ -1,9 +1,10 @@
-from typing import Any
-from collections.abc import Callable, Iterable, Mapping
-import web
 import logging
 import json
 import re
+from typing import Any, Callable
+from collections.abc import Iterable, Mapping
+
+import web
 
 from infogami.utils import delegate
 from infogami import config
@@ -13,9 +14,7 @@
     render_template,
     add_flash_message,
 )
-
 from infogami.infobase.client import ClientException
-from infogami.utils.context import context
 import infogami.core.code as core
 
 from openlibrary import accounts
@@ -37,8 +36,6 @@
 )
 from openlibrary.plugins.upstream import borrow, forms, utils
 
-import urllib
-
 
 logger = logging.getLogger("openlibrary.account")
 
@@ -758,7 +755,7 @@ class account_my_books(delegate.page):
     def GET(self):
         user = accounts.get_current_user()
         username = user.key.split('/')[-1]
-        raise web.seeother('/people/%s/books' % (username))
+        raise web.seeother(f'/people/{username}/books')
 
 
 # This would be by the civi backend which would require the api keys
@@ -885,19 +882,49 @@ def GET(self):
         web.header('Content-disposition', f'attachment; filename={filename}')
         return delegate.RawText('' or data, content_type="text/csv")
 
-    def generate_reading_log(self, username):
-        books = Bookshelves.get_users_logged_books(username, limit=10000)
-        csv = []
-        csv.append('Work Id,Edition Id,Bookshelf\n')
-        mapping = {1: 'Want to Read', 2: 'Currently Reading', 3: 'Already Read'}
-        for book in books:
-            row = [
-                'OL{}W'.format(book['work_id']),
-                'OL{}M'.format(book['edition_id']) if book['edition_id'] else '',
-                '{}\n'.format(mapping[book['bookshelf_id']]),
-            ]
-            csv.append(','.join(row))
-        return ''.join(csv)
+    def generate_reading_log(self, username: str) -> str:
+        from openlibrary.plugins.upstream.models import Work  # Avoid a circular import
+
+        bookshelf_map = {1: 'Want to Read', 2: 'Currently Reading', 3: 'Already Read'}
+
+        def get_subjects(work: Work, subject_type: str) -> str:
+            return " | ".join(
+                s.title.replace(",", ";") for s in work.get_subject_links(subject_type)
+            )
+
+        def format_reading_log(book: dict) -> dict:
+            """
+            Adding, deleting, renaming, or reordering the fields of the dict returned
+            below will automatically be reflected in the CSV that is generated.
+            """
+            work_key = f"/works/OL{book['work_id']}W"
+            work: Work = web.ctx.site.get(work_key)
+            if not work:
+                raise ValueError(f"No Work found for {work_key}.")
+            if edition_id := book.get("edition_id") or "":
+                edition_id = f"OL{edition_id}M"
+            ratings = work.get_rating_stats() or {"average": "", "count": ""}
+            ratings_average, ratings_count = ratings.values()
+            return {
+                "work_id": work_key.split("/")[-1],
+                "title": work.title,
+                "authors": " | ".join(work.get_author_names()),
+                "first_publish_year": work.first_publish_year,
+                "edition_id": edition_id,
+                "edition_count": work.edition_count,
+                "bookshelf": bookshelf_map[work.get_users_read_status(username)],
+                "my_ratings": work.get_users_rating(username) or "",
+                "ratings_average": ratings_average,
+                "ratings_count": ratings_count,
+                "has_ebook": work.has_ebook(),
+                "subjects": get_subjects(work=work, subject_type="subject"),
+                "subject_people": get_subjects(work=work, subject_type="person"),
+                "subject_places": get_subjects(work=work, subject_type="place"),
+                "subject_times": get_subjects(work=work, subject_type="time"),
+            }
+
+        books = Bookshelves.iterate_users_logged_books(username)
+        return csv_string(books, format_reading_log)
 
     def generate_book_notes(self, username: str) -> str:
         def format_booknote(booknote: Mapping) -> dict:
@@ -930,19 +957,14 @@ def generate_list_overview(self, lists):
         for list in lists:
             list_id = list.key.split('/')[-1]
             created_on = list.created.strftime(self.date_format)
-            last_updated = (
-                list.last_modified.strftime(self.date_format)
-                if list.last_modified
-                else ''
-            )
+            if last_updated := list.last_modified or "":
+                last_updated = last_updated.strftime(self.date_format)
             for seed in list.seeds:
                 entry = seed
                 if not isinstance(seed, str):
                     entry = seed.key
-                list_name = list.name.replace('"', '""') if list.name else ''
-                list_desc = (
-                    list.description.replace('"', '""') if list.description else ''
-                )
+                list_name = (list.name or '').replace('"', '""')
+                list_desc = (list.description or '').replace('"', '""')
                 row = [
                     list_id,
                     f'"{list_name}"',
@@ -1013,7 +1035,7 @@ def GET(self):
 
 
 def send_forgot_password_email(username, email):
-    key = "account/%s/password" % username
+    key = f"account/{username}/password"
 
     doc = create_link_doc(key, username, email)
     web.ctx.site.store[key] = doc


Test patch: