Repository: ermakov-oleg/serpyco-rs (language: python)

Issue:
Add a performance comparison with other libraries



Discussion / hints:

Fix patch:
diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
index 414a823..9c05ae4 100644
--- a/.github/workflows/CI.yml
+++ b/.github/workflows/CI.yml
@@ -87,9 +87,9 @@ jobs:
           python -m pip install --upgrade pip
           pip install --no-index --no-deps --find-links wheels/ serpyco-rs
           pip install --find-links wheels/ serpyco-rs
-          pip install -r requirements.txt
+          pip install -r requirements/lint.txt
       - name: tests
-        run: pytest -vss
+        run: pytest -vss tests/
       - name: fmt
         working-directory: ./python/serpyco_rs
         run: black --check --diff .
diff --git a/README.md b/README.md
index 5a8967a..a9a5d1a 100644
--- a/README.md
+++ b/README.md
@@ -60,3 +60,33 @@ There is support for generic types from the standard typing module:
 * Mapping
 * Sequence
 * Tuple (fixed size)
+
+
+## Benchmark
+
+#### dump
+
+| Library     |   Median latency (milliseconds) |   Operations per second |   Relative (latency) |
+|-------------|---------------------------------|-------------------------|----------------------|
+| serpyco_rs  |                            0.05 |                 22193.4 |                 1    |
+| serpyco     |                            0.05 |                 20643.6 |                 1.07 |
+| pydantic    |                            2.64 |                   377.6 |                58.57 |
+| marshmallow |                            1.04 |                   958.9 |                23.18 |
+
+#### load with validate
+
+| Library     |   Median latency (milliseconds) |   Operations per second |   Relative (latency) |
+|-------------|---------------------------------|-------------------------|----------------------|
+| serpyco_rs  |                            0.24 |                  4231.7 |                 1    |
+| serpyco     |                            0.29 |                  3505.4 |                 1.21 |
+| pydantic    |                            2.05 |                   487.5 |                 8.72 |
+| marshmallow |                            4.63 |                   215.4 |                19.69 |
+
+#### load
+
+| Library     |   Median latency (milliseconds) |   Operations per second |   Relative (latency) |
+|-------------|---------------------------------|-------------------------|----------------------|
+| serpyco_rs  |                            0.07 |                 13766.2 |                 1    |
+| serpyco     |                            0.08 |                 12263.8 |                 1.12 |
+| pydantic    |                            2.04 |                   490   |                27.97 |
+| marshmallow |                            4.59 |                   217.9 |                63.14 |
diff --git a/bench/__init__.py b/bench/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/bench/graph.py b/bench/graph.py
new file mode 100644
index 0000000..8b4d017
--- /dev/null
+++ b/bench/graph.py
@@ -0,0 +1,66 @@
+import collections
+import io
+import json
+import os
+
+from tabulate import tabulate
+
+
+LIBRARIES = (
+    "serpyco_rs",
+    "serpyco",
+    "pydantic",
+    "marshmallow",
+)
+
+
+def aggregate():
+    benchmarks_dir = os.path.join(".benchmarks", os.listdir(".benchmarks")[0])
+    res = collections.defaultdict(dict)
+    for filename in os.listdir(benchmarks_dir):
+        with open(os.path.join(benchmarks_dir, filename), "r") as fileh:
+            data = json.loads(fileh.read())
+
+        for each in data["benchmarks"]:
+            res[each["group"]][each["extra_info"]["lib"]] = {
+                "data": [val * 1000 for val in each["stats"]["data"]],
+                "median": each["stats"]["median"] * 1000,
+                "ops": each["stats"]["ops"],
+                "correct": each["extra_info"]["correct"],
+            }
+    return res
+
+
+def tab(obj):
+    buf = io.StringIO()
+    headers = (
+        "Library",
+        "Median latency (milliseconds)",
+        "Operations per second",
+        "Relative (latency)",
+    )
+    for group, val in sorted(obj.items(), reverse=True):
+        buf.write("\n" + "#### " + group + "\n\n")
+        table = []
+        for lib in LIBRARIES:
+            correct = val[lib]["correct"]
+            table.append(
+                [
+                    lib,
+                    val[lib]["median"] if correct else None,
+                    "%.1f" % val[lib]["ops"] if correct else None,
+                    0,
+                ]
+            )
+        baseline = table[0][1]
+        for each in table:
+            each[3] = (
+                "%.2f" % (each[1] / baseline) if isinstance(each[1], float) else None
+            )
+            each[1] = "%.2f" % each[1] if isinstance(each[1], float) else None
+        buf.write(tabulate(table, headers, tablefmt="github") + "\n")
+
+    print(buf.getvalue())
+
+
+tab(aggregate())
diff --git a/bench/libs/__init__.py b/bench/libs/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/bench/libs/base.py b/bench/libs/base.py
new file mode 100644
index 0000000..6b295d6
--- /dev/null
+++ b/bench/libs/base.py
@@ -0,0 +1,36 @@
+from dataclasses import dataclass
+from typing import Optional
+
+
+@dataclass
+class Nested:
+    """
+    A nested type for Dataclass
+    """
+
+    name: str
+
+
+@dataclass
+class Dataclass:
+    """
+    A Dataclass class
+    """
+
+    name: str
+    value: int
+    f: float
+    b: bool
+    nest: list[Nested]
+    many: list[int]
+    option: Optional[str] = None
+
+
+test_object = Dataclass(
+    name="Foo",
+    value=42,
+    f=12.34,
+    b=True,
+    nest=[Nested(name="Bar_{}".format(index)) for index in range(0, 1000)],
+    many=[1, 2, 3],
+)
diff --git a/bench/libs/marshmallow.py b/bench/libs/marshmallow.py
new file mode 100644
index 0000000..e5e90de
--- /dev/null
+++ b/bench/libs/marshmallow.py
@@ -0,0 +1,18 @@
+from typing import Any
+
+import marshmallow_dataclass
+
+from .base import Dataclass, test_object
+
+
+_schema = marshmallow_dataclass.class_schema(Dataclass)()
+
+test_object = test_object
+
+
+def load(data: dict[str, Any], validate: bool = True) -> Dataclass:
+    return _schema.load(data)
+
+
+def dump(obj: Dataclass) -> dict[str, Any]:
+    return _schema.dump(obj)
diff --git a/bench/libs/pydantic.py b/bench/libs/pydantic.py
new file mode 100644
index 0000000..a128291
--- /dev/null
+++ b/bench/libs/pydantic.py
@@ -0,0 +1,35 @@
+from typing import Optional, Any
+
+from pydantic import BaseModel
+
+
+class Nested(BaseModel):
+    name: str
+
+
+class Dataclass(BaseModel):
+    name: str
+    value: int
+    f: float
+    b: bool
+    nest: list[Nested]
+    many: list[int]
+    option: Optional[str] = None
+
+
+test_object = Dataclass(
+    name="Foo",
+    value=42,
+    f=12.34,
+    b=True,
+    nest=[Nested(name="Bar_{}".format(index)) for index in range(0, 1000)],
+    many=[1, 2, 3],
+)
+
+
+def load(data: dict[str, Any], validate: bool = True) -> Dataclass:
+    return Dataclass(**data)
+
+
+def dump(obj: Dataclass) -> dict[str, Any]:
+    return obj.dict()
diff --git a/bench/libs/serpyco.py b/bench/libs/serpyco.py
new file mode 100644
index 0000000..49885e4
--- /dev/null
+++ b/bench/libs/serpyco.py
@@ -0,0 +1,16 @@
+from typing import Any
+
+import serpyco
+from .base import Dataclass, test_object
+
+_serializer = serpyco.Serializer(Dataclass)
+
+test_object = test_object
+
+
+def load(data: dict[str, Any], validate: bool = True) -> Dataclass:
+    return _serializer.load(data, validate=validate)
+
+
+def dump(obj: Dataclass) -> dict[str, Any]:
+    return _serializer.dump(obj)
diff --git a/bench/libs/serpyco_rs.py b/bench/libs/serpyco_rs.py
new file mode 100644
index 0000000..4d326f1
--- /dev/null
+++ b/bench/libs/serpyco_rs.py
@@ -0,0 +1,17 @@
+from typing import Any
+
+import serpyco_rs
+from .base import Dataclass, test_object
+
+
+_serializer = serpyco_rs.Serializer(Dataclass)
+
+test_object = test_object
+
+
+def load(data: dict[str, Any], validate: bool = True) -> Dataclass:
+    return _serializer.load(data, validate=validate)
+
+
+def dump(obj: Dataclass) -> dict[str, Any]:
+    return _serializer.dump(obj)
diff --git a/noxfile.py b/noxfile.py
new file mode 100644
index 0000000..2e28993
--- /dev/null
+++ b/noxfile.py
@@ -0,0 +1,24 @@
+import platform
+
+import nox
+
+
+@nox.session(reuse_venv=True)
+def bench(session):
+    session.install('-r', 'requirements/bench.txt')
+
+    if platform.machine() == 'arm64' and platform.system() == 'Darwin':
+        # https://github.com/Stranger6667/jsonschema-rs/issues/409
+        session.install('jsonschema_rs', '--no-binary', ':all:')
+
+    session.run('maturin', 'develop', '-r')
+    session.run(
+        'pytest',
+        '--verbose',
+        '--benchmark-min-time=1',
+        '--benchmark-max-time=5',
+        '--benchmark-disable-gc',
+        '--benchmark-autosave',
+        '--benchmark-save-data',
+        'bench',
+    )
diff --git a/requirements.txt b/requirements.txt
deleted file mode 100644
index 31788ee..0000000
--- a/requirements.txt
+++ /dev/null
@@ -1,12 +0,0 @@
-pytest
-maturin
-serpyco
-pytest-benchmark
-pytz
-black
-ruff
-mypy
-pyright
-isort
-tzdata; sys_platform == 'win32'
-types-python-dateutil
diff --git a/requirements/bench.txt b/requirements/bench.txt
new file mode 100644
index 0000000..57084a5
--- /dev/null
+++ b/requirements/bench.txt
@@ -0,0 +1,8 @@
+-r dev.txt
+
+pytest-benchmark[histogram]
+tabulate
+serpyco
+pydantic
+marshmallow
+marshmallow-dataclass
diff --git a/requirements/dev.txt b/requirements/dev.txt
new file mode 100644
index 0000000..18c0b76
--- /dev/null
+++ b/requirements/dev.txt
@@ -0,0 +1,6 @@
+pytest
+maturin
+pytz
+python-dateutil
+tzdata; sys_platform == 'win32'
+nox
diff --git a/requirements/lint.txt b/requirements/lint.txt
new file mode 100644
index 0000000..2747d0f
--- /dev/null
+++ b/requirements/lint.txt
@@ -0,0 +1,8 @@
+-r dev.txt
+
+black
+ruff
+mypy
+pyright
+isort
+types-python-dateutil
diff --git a/src/serializer/encoders.rs b/src/serializer/encoders.rs
index c38e93b..e484ad7 100644
--- a/src/serializer/encoders.rs
+++ b/src/serializer/encoders.rs
@@ -87,7 +87,7 @@ impl Encoder for DecimalEncoder {
     #[inline]
     fn load(&self, value: *mut PyObject) -> PyResult<*mut PyObject> {
         to_decimal(value).map_err(|e| {
-            ValidationError::new_err(format!("invalid Decimal value: {:?} error: {:?}", value, e))
+            ValidationError::new_err(format!("invalid Decimal value: {value:?} error: {e:?}"))
         })
     }
 }


Test patch:
diff --git a/bench/test_benchmarks.py b/bench/test_benchmarks.py
new file mode 100644
index 0000000..8c98a6f
--- /dev/null
+++ b/bench/test_benchmarks.py
@@ -0,0 +1,52 @@
+import pytest
+
+from .libs import (
+    serpyco,
+    serpyco_rs,
+    pydantic,
+    marshmallow,
+)
+
+
+serializers = {
+    "serpyco": serpyco,
+    "serpyco_rs": serpyco_rs,
+    "pydantic": pydantic,
+    "marshmallow": marshmallow,
+}
+
+
+@pytest.mark.parametrize("lib", serializers.keys())
+def test_dump(benchmark, lib):
+    serializer = serializers[lib]
+    serializer.dump(serializer.test_object)  # warmup
+
+    benchmark.group = "dump"
+    benchmark.extra_info["lib"] = lib
+    benchmark.extra_info["correct"] = serializer.load(serializer.dump(serializer.test_object)) == serializer.test_object
+    benchmark(serializer.dump, serializer.test_object)
+
+
+@pytest.mark.parametrize("lib", serializers.keys())
+def test_load(benchmark, lib):
+    serializer = serializers[lib]
+    test_dict = serializer.dump(serializer.test_object)
+    serializer.load(test_dict, validate=False)  # warmup
+
+    benchmark.group = "load"
+    benchmark.extra_info["lib"] = lib
+    benchmark.extra_info["correct"] = serializer.load(serializer.dump(serializer.test_object)) == serializer.test_object
+    benchmark(serializer.load, test_dict, validate=False)
+
+
+@pytest.mark.parametrize("lib", serializers.keys())
+def test_load_validate(benchmark, lib):
+    serializer = serializers[lib]
+    test_dict = serializer.dump(serializer.test_object)
+    serializer.load(test_dict, validate=True)  # warmup
+
+    benchmark.group = "load with validate"
+    benchmark.extra_info["lib"] = lib
+    benchmark.extra_info["correct"] = serializer.load(serializer.dump(serializer.test_object)) == serializer.test_object
+
+    benchmark(serializer.load, test_dict, validate=True)
diff --git a/tests/test_benchmarks.py b/tests/test_benchmarks.py
deleted file mode 100644
index d12ba0b..0000000
--- a/tests/test_benchmarks.py
+++ /dev/null
@@ -1,85 +0,0 @@
-import dataclasses
-import typing
-
-import pytest
-import serpyco
-import serpyco_rs
-
-
-@dataclasses.dataclass
-class Nested(object):
-    """
-    A nested type for Dataclass
-    """
-
-    name: str
-
-
-@dataclasses.dataclass
-class Dataclass(object):
-    """
-    A Dataclass class
-    """
-
-    name: str
-    value: int
-    f: float
-    b: bool
-    nest: typing.List[Nested]
-    many: typing.List[int]
-    option: typing.Optional[str] = None
-
-
-serializer_cython = serpyco.Serializer(Dataclass)
-serializer_rs = serpyco_rs.Serializer(Dataclass)
-
-test_object = Dataclass(
-    name="Foo",
-    value=42,
-    f=12.34,
-    b=True,
-    nest=[Nested(name="Bar_{}".format(index)) for index in range(0, 1000)],
-    many=[1, 2, 3],
-)
-
-
-serializers = {
-    "cython": serializer_cython,
-    "rust": serializer_rs,
-}
-
-
-@pytest.mark.parametrize("impl", ["cython", "rust"])
-def test_dump(benchmark, impl):
-    serializer = serializers[impl]
-    serializer.dump(test_object)  # warmup
-
-    benchmark.group = "dump"
-    benchmark.extra_info["impl"] = impl
-    benchmark.extra_info["correct"] = serializer.load(serializer.dump(test_object)) == test_object
-    benchmark(serializer.dump, test_object)
-
-
-@pytest.mark.parametrize("impl", ["cython", "rust"])
-def test_load(benchmark, impl):
-    serializer = serializers[impl]
-    test_dict = serializer.dump(test_object)
-    serializer.load(test_dict, validate=False)  # warmup
-
-    benchmark.group = "load"
-    benchmark.extra_info["impl"] = impl
-    benchmark.extra_info["correct"] = serializer.load(serializer.dump(test_object)) == test_object
-    benchmark(serializer.load, test_dict, validate=False)
-
-
-@pytest.mark.parametrize("impl", ["rust", "cython"])
-def test_load_validate(benchmark, impl):
-    serializer = serializers[impl]
-    test_dict = serializer.dump(test_object)
-    serializer.load(test_dict, validate=True)  # warmup
-
-    benchmark.group = "load with validate"
-    benchmark.extra_info["impl"] = impl
-    benchmark.extra_info["correct"] = serializer.load(serializer.dump(test_object)) == test_object
-
-    benchmark(serializer.load, test_dict, validate=True)