Repository: reflex-dev/reflex
Pull request: fix serialization as a whole for list/dict/Base containing custom items to serialize
Improve the serialization logic to handle the case where we want to serialize a list of custom objects.

Before PR, a datetime can be serialized,  but not a list of datetime or a dict containing datetime as value.
After PR, they can all be serialized.

Include the changes suggested by @ElijahAhianyo in #1977.

Closes #1976 

Problem statement:
Need for custom json serializer
During compile time, any non-serializable object encountered during the JSON serialization process is treated as a list, This means the code below:

```python
import reflex as rx
from datetime import datetime
from sqlmodel import Field

class User(rx.Model, table=True):
      first_name: str = Field()
      last_name: str = Field()
      created_at: datetime = Field(default=datetime.now())
      updated_at: datetime = Field(default=datetime.now())


class State(rx.State):
    @rx.var
     def user(self) -> User:
         with rx.session as session:
             user = session.exec(User.select.where(User.id == 1)).first()
             return user


```
Fails with error:

```
  File "/Users/eli/Documents/work/Pynecone-io/pynecone-forks/main/reflex/reflex/app.py", line 951, in on_event
    await self.emit_update(update=update, sid=sid)
  File "/Users/eli/Documents/work/Pynecone-io/pynecone-forks/main/reflex/reflex/app.py", line 921, in emit_update
    self.emit(str(constants.SocketEvent.EVENT), update.json(), to=sid)
                                                ^^^^^^^^^^^^^
  File "/Users/eli/Documents/work/Pynecone-io/pynecone-forks/main/reflex/reflex/base.py", line 33, in json
    return self.__config__.json_dumps(self.dict(), default=list)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/json/__init__.py", line 238, in dumps
    **kw).encode(obj)
          ^^^^^^^^^^^
  File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/json/encoder.py", line 200, in encode
    chunks = self.iterencode(o, _one_shot=True)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/json/encoder.py", line 258, in iterencode
    return _iterencode(o, 0)
           ^^^^^^^^^^^^^^^^^
TypeError: 'datetime.datetime' object is not iterable

```
Ideally, we should have a custom serializer to handle this

Hints from issue discussion:

Commit message: serialization is fixed

Patch to review:
diff --git a/reflex/base.py b/reflex/base.py
index 53510786b36..6a8d9cac158 100644
--- a/reflex/base.py
+++ b/reflex/base.py
@@ -30,7 +30,9 @@ def json(self) -> str:
         Returns:
             The object as a json string.
         """
-        return self.__config__.json_dumps(self.dict(), default=list)
+        from reflex.utils.serializers import serialize
+
+        return self.__config__.json_dumps(self.dict(), default=serialize)
 
     def set(self, **kwargs):
         """Set multiple fields and return the object.
diff --git a/reflex/utils/format.py b/reflex/utils/format.py
index dd0948fdcc0..315f7307d20 100644
--- a/reflex/utils/format.py
+++ b/reflex/utils/format.py
@@ -588,7 +588,7 @@ def json_dumps(obj: Any) -> str:
     Returns:
         A string
     """
-    return json.dumps(obj, ensure_ascii=False, default=list)
+    return json.dumps(obj, ensure_ascii=False, default=serialize)
 
 
 def unwrap_vars(value: str) -> str:
diff --git a/reflex/utils/serializers.py b/reflex/utils/serializers.py
index 41ffec0a408..b8646d615f6 100644
--- a/reflex/utils/serializers.py
+++ b/reflex/utils/serializers.py
@@ -93,7 +93,7 @@ def get_serializer(type_: Type) -> Serializer | None:
         return serializer
 
     # If the type is not registered, check if it is a subclass of a registered type.
-    for registered_type, serializer in SERIALIZERS.items():
+    for registered_type, serializer in reversed(SERIALIZERS.items()):
         if types._issubclass(type_, registered_type):
             return serializer
 
@@ -127,18 +127,31 @@ def serialize_str(value: str) -> str:
 
 
 @serializer
-def serialize_primitive(value: Union[bool, int, float, Base, None]) -> str:
+def serialize_primitive(value: Union[bool, int, float, None]) -> str:
     """Serialize a primitive type.
 
     Args:
-        value: The number to serialize.
+        value: The number/bool/None to serialize.
 
     Returns:
-        The serialized number.
+        The serialized number/bool/None.
     """
     return format.json_dumps(value)
 
 
+@serializer
+def serialize_base(value: Base) -> str:
+    """Serialize a Base instance.
+
+    Args:
+        value : The Base to serialize.
+
+    Returns:
+        The serialized Base.
+    """
+    return value.json()
+
+
 @serializer
 def serialize_list(value: Union[List, Tuple, Set]) -> str:
     """Serialize a list to a JSON string.
diff --git a/tests/utils/test_format.py b/tests/utils/test_format.py
index 61c7427eae8..29aba20bb10 100644
--- a/tests/utils/test_format.py
+++ b/tests/utils/test_format.py
@@ -1,3 +1,4 @@
+import datetime
 from typing import Any
 
 import pytest
@@ -572,6 +573,14 @@ def test_format_library_name(input: str, output: str):
         ([1, 2, 3], "[1, 2, 3]"),
         ({}, "{}"),
         ({"k1": False, "k2": True}, '{"k1": false, "k2": true}'),
+        (
+            [datetime.timedelta(1, 1, 1), datetime.timedelta(1, 1, 2)],
+            '["1 day, 0:00:01.000001", "1 day, 0:00:01.000002"]',
+        ),
+        (
+            {"key1": datetime.timedelta(1, 1, 1), "key2": datetime.timedelta(1, 1, 2)},
+            '{"key1": "1 day, 0:00:01.000001", "key2": "1 day, 0:00:01.000002"}',
+        ),
     ],
 )
 def test_json_dumps(input, output):
diff --git a/tests/utils/test_serializers.py b/tests/utils/test_serializers.py
index 778e55c069e..8dcce1b4257 100644
--- a/tests/utils/test_serializers.py
+++ b/tests/utils/test_serializers.py
@@ -1,8 +1,10 @@
 import datetime
+from enum import Enum
 from typing import Any, Dict, List, Type
 
 import pytest
 
+from reflex.base import Base
 from reflex.utils import serializers
 from reflex.vars import Var
 
@@ -93,6 +95,31 @@ def serialize_foo(value: Foo) -> str:
     assert not serializers.has_serializer(Foo)
 
 
+class TestEnum(str, Enum):
+    """TestEnum."""
+
+    FOO = "foo"
+    BAR = "bar"
+
+
+class TestEnum2(Enum):
+    """TestEnum2."""
+
+    FOO = "foo"
+    BAR = "bar"
+
+
+@serializers.serializer
+def serialize_TestEnum2(enum: TestEnum2) -> str:
+    return "prefix_" + enum.value
+
+
+class TestBase(Base):
+    """A class inheriting from Base for testing."""
+
+    ts: datetime.timedelta = datetime.timedelta(1, 1, 1)
+
+
 @pytest.mark.parametrize(
     "value,expected",
     [
@@ -104,6 +131,20 @@ def serialize_foo(value: Foo) -> str:
         (None, "null"),
         ([1, 2, 3], "[1, 2, 3]"),
         ([1, "2", 3.0], '[1, "2", 3.0]'),
+        ([{"key": 1}, {"key": 2}], '[{"key": 1}, {"key": 2}]'),
+        (TestEnum.FOO, "foo"),
+        ([TestEnum.FOO, TestEnum.BAR], '["foo", "bar"]'),
+        (
+            {"key1": [1, 2, 3], "key2": [TestEnum.FOO, TestEnum.BAR]},
+            '{"key1": [1, 2, 3], "key2": ["foo", "bar"]}',
+        ),
+        (TestEnum2.FOO, "prefix_foo"),
+        ([TestEnum2.FOO, TestEnum2.BAR], '["prefix_foo", "prefix_bar"]'),
+        (
+            {"key1": TestEnum2.FOO, "key2": TestEnum2.BAR},
+            '{"key1": "prefix_foo", "key2": "prefix_bar"}',
+        ),
+        (TestBase(ts=datetime.timedelta(1, 1, 1)), '{"ts": "1 day, 0:00:01.000001"}'),
         (
             [1, Var.create_safe("hi"), Var.create_safe("bye", is_local=False)],
             '[1, "hi", bye]',
@@ -121,6 +162,10 @@ def serialize_foo(value: Foo) -> str:
         (datetime.date(2021, 1, 1), "2021-01-01"),
         (datetime.time(1, 1, 1, 1), "01:01:01.000001"),
         (datetime.timedelta(1, 1, 1), "1 day, 0:00:01.000001"),
+        (
+            [datetime.timedelta(1, 1, 1), datetime.timedelta(1, 1, 2)],
+            '["1 day, 0:00:01.000001", "1 day, 0:00:01.000002"]',
+        ),
     ],
 )
 def test_serialize(value: Any, expected: str):


Code review comments:
diff_hunk: @@ -93,6 +95,31 @@ def serialize_foo(value: Foo) -> str:
     assert not serializers.has_serializer(Foo)
 
 
+class TestEnum(str, Enum):
+    """TestEnum."""
+
+    FOO = "foo"
+    BAR = "bar"
+
+
+class TestEnum2(Enum):
path: tests/utils/test_serializers.py
text: @user1:
```suggestion
class FooEnumWithPrefix(Enum):
```

2 suggestions here

* remove the `Test` prefix from the name, so pytest doesn't try to think this is a test class (also do this for `TestEnum` -> `FooEnum`)
* Add `WithPrefix` to the end to help visually distinguish why these otherwise identical classes are different (this would particularly help in the parametrizations, which I had to re-read like 3 times before I noticed the `2`)

diff_hunk: @@ -93,6 +95,31 @@ def serialize_foo(value: Foo) -> str:
     assert not serializers.has_serializer(Foo)
 
 
+class TestEnum(str, Enum):
+    """TestEnum."""
+
+    FOO = "foo"
+    BAR = "bar"
+
+
+class TestEnum2(Enum):
+    """TestEnum2."""
+
+    FOO = "foo"
+    BAR = "bar"
+
+
+@serializers.serializer
+def serialize_TestEnum2(enum: TestEnum2) -> str:
+    return "prefix_" + enum.value
+
+
+class TestBase(Base):
path: tests/utils/test_serializers.py
text: @user1:
```suggestion
class BaseSubclass(Base):
```

Again, avoiding the `Test` prefix to not get picked up in pytest collection routine (and avoid warning)

@user1:
![image](https://github.com/reflex-dev/reflex/assets/1524005/19b920e4-a86a-46e3-a002-49202c692f53)


@author:
So THAT was the reason for this warnings, thanks!