Repository: zauberzeug/nicegui
Pull request #532 (2023-03-16 01:34:32)

Fallback to Python's built-in json module if orjson unavailable
`orjson` does not support all architectures. For those cases, if the `orjson` module import fails, the json serializer abstraction falls back to Python's built-in `json` module.

The two implementations have been adjusted/extended in order to output equal JSON strings, e.g. for date/datetime/numpy arrays etc. To ensure this, a new unit test `test_json.py` has been added.


Patch:
diff --git a/nicegui/json/__init__.py b/nicegui/json/__init__.py
index a2cb7449..0734d061 100644
--- a/nicegui/json/__init__.py
+++ b/nicegui/json/__init__.py
@@ -1,15 +1,29 @@
 """
 Custom json module. Provides dumps and loads implementations
-wrapping the `orjson` package.
+wrapping the orjson package. If the orjson package is not available,
+the standard Python json module is used.
 
-Custom module required in order to override json-module used
+This custom module is required in order to override the json-module used
 in socketio.AsyncServer, which expects a module as parameter
 to override Python's default json module.
 """
 
-from nicegui.json.orjson_wrapper import dumps, loads
+try:
+    # orjson not available on all platforms, fallback to Python's json module if not available
+    import orjson
+    has_orjson = True
+except ImportError:
+    has_orjson = False
+
+
+if has_orjson:
+    from nicegui.json.orjson_wrapper import NiceGUIJSONResponse, dumps, loads
+else:
+    from nicegui.json.builtin_wrapper import NiceGUIJSONResponse, dumps, loads
+
 
 __all__ = [
     'dumps',
-    'loads'
+    'loads',
+    'NiceGUIJSONResponse'
 ]
diff --git a/nicegui/json/builtin_wrapper.py b/nicegui/json/builtin_wrapper.py
new file mode 100644
index 00000000..4a715580
--- /dev/null
+++ b/nicegui/json/builtin_wrapper.py
@@ -0,0 +1,55 @@
+import json
+from datetime import date, datetime
+from typing import Any, Optional, Tuple
+
+import numpy as np
+from fastapi import Response
+
+
+def dumps(obj: Any, sort_keys: bool = False, separators: Optional[Tuple[str, str]] = None):
+    """Serializes a Python object to a JSON-encoded string.
+
+    This implementation uses Python's default json module, but extends it
+    in order to support numpy arrays.
+    """
+    if separators is None:
+        separators = (',', ':')
+    return json.dumps(
+        obj,
+        sort_keys=sort_keys,
+        separators=separators,
+        indent=None,
+        allow_nan=False,
+        ensure_ascii=False,
+        cls=NumpyJsonEncoder)
+
+
+def loads(value: str) -> Any:
+    """Deserialize a JSON-encoded string to a corresponding Python object/value.
+
+    Uses Python's default json module internally.
+    """
+    return json.loads(value)
+
+
+class NiceGUIJSONResponse(Response):
+    """FastAPI response class to support our custom json serializer implementation."""
+    media_type = 'application/json'
+
+    def render(self, content: Any) -> bytes:
+        return dumps(content).encode('utf-8')
+
+
+class NumpyJsonEncoder(json.JSONEncoder):
+    """Special json encoder that supports numpy arrays and date/datetime objects."""
+
+    def default(self, obj):
+        if isinstance(obj, np.integer):
+            return int(obj)
+        if isinstance(obj, np.floating):
+            return float(obj)
+        if isinstance(obj, np.ndarray):
+            return obj.tolist()
+        if isinstance(obj, (datetime, date)):
+            return obj.isoformat()
+        return json.JSONEncoder.default(self, obj)
diff --git a/nicegui/json/fastapi.py b/nicegui/json/fastapi.py
deleted file mode 100644
index 58c81a4b..00000000
--- a/nicegui/json/fastapi.py
+++ /dev/null
@@ -1,13 +0,0 @@
-from typing import Any
-
-import orjson
-from fastapi import Response
-
-from nicegui.json.orjson_wrapper import ORJSON_OPTS
-
-
-class NiceGUIJSONResponse(Response):
-    media_type = 'application/json'
-
-    def render(self, content: Any) -> bytes:
-        return orjson.dumps(content, option=ORJSON_OPTS)
diff --git a/nicegui/json/orjson_wrapper.py b/nicegui/json/orjson_wrapper.py
index a9fd95d3..b56f92ab 100644
--- a/nicegui/json/orjson_wrapper.py
+++ b/nicegui/json/orjson_wrapper.py
@@ -1,6 +1,7 @@
 from typing import Any, Optional, Tuple
 
 import orjson
+from fastapi import Response
 
 ORJSON_OPTS = orjson.OPT_SERIALIZE_NUMPY | orjson.OPT_NON_STR_KEYS
 
@@ -34,3 +35,14 @@ def loads(value: str) -> Any:
     Uses package `orjson` internally.
     """
     return orjson.loads(value)
+
+
+class NiceGUIJSONResponse(Response):
+    """FastAPI response class to support our custom json serializer implementation.
+
+    Uses package `orjson` internally.
+    """
+    media_type = 'application/json'
+
+    def render(self, content: Any) -> bytes:
+        return orjson.dumps(content, option=ORJSON_OPTS)
diff --git a/nicegui/nicegui.py b/nicegui/nicegui.py
index f73b06e5..fbf4aacf 100644
--- a/nicegui/nicegui.py
+++ b/nicegui/nicegui.py
@@ -11,7 +11,7 @@ from fastapi.staticfiles import StaticFiles
 from fastapi_socketio import SocketManager
 
 from nicegui import json
-from nicegui.json.fastapi import NiceGUIJSONResponse
+from nicegui.json import NiceGUIJSONResponse
 
 from . import background_tasks, binding, globals, outbox
 from .app import App
diff --git a/pyproject.toml b/pyproject.toml
index 4adbae39..3f4b2121 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -25,7 +25,7 @@ watchfiles = "^0.18.1"
 jinja2 = "^3.1.2"
 python-multipart = "^0.0.6"
 plotly = "^5.13.0"
-orjson = "^3.8.6"
+orjson = {version = "^3.8.6", markers = "platform_machine != 'i386' and platform_machine != 'i686'"} # orjson does not support 32bit
 pywebview = "^4.0.2"
 
 [tool.poetry.group.dev.dependencies]


Test patch:
diff --git a/tests/test_json.py b/tests/test_json.py
new file mode 100644
index 00000000..5a434e06
--- /dev/null
+++ b/tests/test_json.py
@@ -0,0 +1,51 @@
+"""
+Test our two json serializers (orjson, and Python's built-in json module).
+
+Need to ensure that we get the same output regardless of the serializer used.
+"""
+
+import sys
+from datetime import date, datetime
+
+import numpy as np
+import pytest
+
+try:
+    # try to import module, only run test if succeeded
+    import orjson
+except ImportError:
+    pass
+
+
+@pytest.mark.skipif('orjson' not in sys.modules, reason='requires the orjson library.')
+def test_json():
+    # only run test if orjson is available to not break it on 32 bit systems
+    # or architectures where orjson is not supported.
+
+    from nicegui.json.builtin_wrapper import dumps as builtin_dumps
+    from nicegui.json.orjson_wrapper import dumps as orjson_dumps
+
+    # test different scalar and array types
+    tests = [
+        None,
+        'text',
+        True,
+        1.0,
+        1,
+        [],
+        dict(),
+        dict(key1='value1', key2=1),
+        date(2020, 1, 31),
+        datetime(2020, 1, 31, 12, 59, 59, 123456),
+        [1.0, -3, 0],
+        ['test', '€'],
+        [0, None, False, np.pi, 'text', date(2020, 1, 31), datetime(2020, 1, 31, 12, 59, 59, 123456), np.array([1.0])],
+        np.array([1.0, 0]),
+        np.array([0, False, np.pi]),
+        np.array(['2010-10-17 07:15:30', '2011-05-13 08:20:35', '2013-01-15 09:09:09'], dtype=np.datetime64),
+    ]
+
+    for test in tests:
+        orjson_str = orjson_dumps(test)
+        builtin_str = builtin_dumps(test)
+        assert orjson_str == builtin_str, f'json serializer implementations do not match: orjson={orjson_str}, built-in={builtin_str}'
diff --git a/tests/test_plotly.py b/tests/test_plotly.py
index ae086609..51af6c50 100644
--- a/tests/test_plotly.py
+++ b/tests/test_plotly.py
@@ -1,3 +1,4 @@
+import numpy as np
 import plotly.graph_objects as go
 
 from nicegui import ui
@@ -11,7 +12,8 @@ def test_plotly(screen: Screen):
     plot = ui.plotly(fig)
 
     ui.button('Add trace', on_click=lambda: (
-        fig.add_trace(go.Scatter(x=[0, 1, 2], y=[2, 1, 0], name='Trace 2')),
+        # test numpy array support for value arrays
+        fig.add_trace(go.Scatter(x=np.array([0, 1, 2]), y=np.array([2, 1, 0]), name='Trace 2')),
         plot.update()
     ))