Repository: LMFDB/lmfdb

Issue:
many api json downloads fail with internal server errors
The following pages all give internal server errors; the problem appears to occur on collections whose documents contain attributes that are object ids (other than _id, which is handled separately), which causes errors like:

TypeError: ObjectId('4f7052dc5009fb18ef000811') is not JSON serializable

The following pages all have this problem:

http://www.lmfdb.org/api/MaassWaveForms/Coefficients?_format=json
http://www.lmfdb.org/api/MaassWaveForms/Coefficients.chunks?_format=json
http://www.lmfdb.org/api/MaassWaveForms/Coefficients.files?_format=json
http://www.lmfdb.org/api/MaassWaveForms/FS?_format=json
http://www.lmfdb.org/api/MaassWaveForms/Table.chunks?_format=json
http://www.lmfdb.org/api/MaassWaveForms/maassform_plots?_format=json
http://www.lmfdb.org/api/SL2Zsubgroups/groups?_format=json
http://www.lmfdb.org/api/modularforms2/webchar.chunks?_format=json
http://www.lmfdb.org/api/modularforms2/webeigenvalues.chunks?_format=json
http://www.lmfdb.org/api/modularforms2/webmodformspace.chunks?_format=json
http://www.lmfdb.org/api/modularforms2/webnewforms.chunks?_format=json
http://www.lmfdb.org/api/siegel_modular_forms/samples?_format=json



Discussion:
I'm looking at this now


Resolving patch:
diff --git a/lmfdb/api/api.py b/lmfdb/api/api.py
--- a/lmfdb/api/api.py
+++ b/lmfdb/api/api.py
@@ -4,13 +4,15 @@
 import urllib2
 ASC = pymongo.ASCENDING
 DESC = pymongo.DESCENDING
-import flask
 import yaml
+import json
+import flask
 import lmfdb.base as base
 from lmfdb.utils import flash_error
 from datetime import datetime
-from flask import render_template, request, url_for
+from flask import render_template, request, url_for, app, current_app
 from lmfdb.api import api_page, api_logger
+from bson import json_util
 from bson.objectid import ObjectId
 
 # caches the database information
@@ -26,6 +28,14 @@ def quote_string(value):
         return "\"ObjectId('%s')\""%value
     return value
 
+def oids_to_strings(doc):
+    """ recursively replace all ObjectId values in dictionary doc with strings encoding the ObjectId values"""
+    for k,v in doc.items():
+        if isinstance(v,ObjectId):
+            doc[k] = "ObjectId('%s')" % v
+        elif isinstance(v,dict):
+            oids_to_strings(doc[k])
+
 def pretty_document(rec,sep=", ",id=True):
     # sort keys and remove _id for html display
     attrs = sorted([(key,quote_string(rec[key])) for key in rec.keys() if (id or key != '_id')])
@@ -215,12 +225,9 @@ def api_query(db, collection, id = None):
             flash_error("no document with id %s found in collection %s.%s.", id, db, collection)
             return flask.redirect(url_for(".api_query", db=db, collection=collection))
     
+    # fixup object ids for display and json/yaml encoding
     for document in data:
-        oid = document["_id"]
-        if type(oid) == ObjectId:
-            document["_id"] = "ObjectId('%s')" % oid
-        elif isinstance(oid, basestring):
-            document["_id"] = str(oid)
+        oids_to_strings(document)
 
     # preparing the datastructure
     start = offset
@@ -244,9 +251,9 @@ def api_query(db, collection, id = None):
         "next": next
     }
 
-    # display of the result (default html)
     if format.lower() == "json":
-        return flask.jsonify(**data)
+        #return flask.jsonify(**data) # can't handle binary data
+        return current_app.response_class(json.dumps(data, encoding='ISO-8859-1', indent=2, default=json_util.default), mimetype='application/json')
     elif format.lower() == "yaml":
         y = yaml.dump(data,
                       default_flow_style=False,
@@ -254,7 +261,7 @@ def api_query(db, collection, id = None):
                       allow_unicode=True)
         return flask.Response(y, mimetype='text/plain')
     else:
-        # sort displayed records by key (as json and yaml do)
+        # sort displayed records by key (as jsonify and yaml_dump do)
         data["pretty"] = pretty_document
         location = "%s/%s" % (db, collection)
         title = "API - " + location


Test patch: