Repository: yukinarit/pyserde
Pull request #250 (2022-07-13 13:29:02)

feat: Add type coercion feature

codecov[bot]: # [Codecov](https://codecov.io/gh/yukinarit/pyserde/pull/250?src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=yukinarit) Report
> Merging [#250](https://codecov.io/gh/yukinarit/pyserde/pull/250?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=yukinarit) (f424195) into [master](https://codecov.io/gh/yukinarit/pyserde/commit/11b4df67974354d263909148e212b9f3f9033a15?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=yukinarit) (11b4df6) will **increase** coverage by `0.19%`.
> The diff coverage is `100.00%`.

```diff
@@            Coverage Diff             @@
##           master     #250      +/-   ##
==========================================
+ Coverage   89.71%   89.91%   +0.19%     
==========================================
  Files          11       11              
  Lines        1595     1616      +21     
  Branches      352      355       +3     
==========================================
+ Hits         1431     1453      +22     
+ Misses        115      114       -1     
  Partials       49       49              
```


| [Impacted Files](https://codecov.io/gh/yukinarit/pyserde/pull/250?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=yukinarit) | Coverage Δ | |
|---|---|---|
| [serde/compat.py](https://codecov.io/gh/yukinarit/pyserde/pull/250/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=yukinarit#diff-c2VyZGUvY29tcGF0LnB5) | `83.14% <100.00%> (+0.14%)` | :arrow_up: |
| [serde/core.py](https://codecov.io/gh/yukinarit/pyserde/pull/250/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=yukinarit#diff-c2VyZGUvY29yZS5weQ==) | `91.98% <100.00%> (+0.46%)` | :arrow_up: |
| [serde/de.py](https://codecov.io/gh/yukinarit/pyserde/pull/250/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=yukinarit#diff-c2VyZGUvZGUucHk=) | `95.71% <100.00%> (+0.05%)` | :arrow_up: |
| [serde/se.py](https://codecov.io/gh/yukinarit/pyserde/pull/250/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=yukinarit#diff-c2VyZGUvc2UucHk=) | `92.30% <100.00%> (+0.13%)` | :arrow_up: |

------

[Continue to review full report at Codecov](https://codecov.io/gh/yukinarit/pyserde/pull/250?src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=yukinarit).
> **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=yukinarit)
> `Δ = absolute <relative> (impact)`, `ø = not affected`, `? = missing data`
> Powered by [Codecov](https://codecov.io/gh/yukinarit/pyserde/pull/250?src=pr&el=footer&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=yukinarit). Last update [5a7fa3c...f424195](https://codecov.io/gh/yukinarit/pyserde/pull/250?src=pr&el=lastupdated&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=yukinarit). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=yukinarit).


Patch:
diff --git a/serde/compat.py b/serde/compat.py
index 66a4358..199947d 100644
--- a/serde/compat.py
+++ b/serde/compat.py
@@ -225,6 +225,11 @@ def typename(typ, with_typing_module: bool = False) -> str:
     elif typ is Any:
         return f'{mod}Any'
     else:
+        # Get super type for NewType
+        inner = getattr(typ, '__supertype__', None)
+        if inner:
+            return typename(typ.__supertype__)
+
         name = getattr(typ, '_name', None)
         if name:
             return name
diff --git a/serde/core.py b/serde/core.py
index 2d50954..48bc163 100644
--- a/serde/core.py
+++ b/serde/core.py
@@ -750,3 +750,15 @@ NoCheck = TypeCheck(kind=TypeCheck.Kind.NoCheck)
 Coerce = TypeCheck(kind=TypeCheck.Kind.Coerce)
 
 Strict = TypeCheck(kind=TypeCheck.Kind.Strict)
+
+
+def coerce(typ: Type, obj: Any, type_check: TypeCheck) -> Any:
+    return typ(obj) if is_coercible(typ, obj, type_check) else obj
+
+
+def is_coercible(typ: Type, obj: Any, type_check: TypeCheck) -> bool:
+    if not type_check.is_coerce():
+        return False
+    if obj is None:
+        return False
+    return True
diff --git a/serde/de.py b/serde/de.py
index f33ceef..f0c8ee7 100644
--- a/serde/de.py
+++ b/serde/de.py
@@ -58,10 +58,10 @@ from .core import (
     Field,
     NoCheck,
     SerdeScope,
-    Strict,
     Tagging,
     TypeCheck,
     add_func,
+    coerce,
     ensure,
     fields,
     is_dict_instance,
@@ -232,6 +232,9 @@ def deserialize(
         g['from_obj'] = from_obj
         g['get_generic_arg'] = get_generic_arg
         g['is_instance'] = is_instance
+        g['TypeCheck'] = TypeCheck
+        g['Coerce'] = Coerce
+        g['coerce'] = coerce
         if deserialize:
             g['serde_custom_class_deserializer'] = functools.partial(
                 serde_custom_class_deserializer, custom=deserializer
@@ -341,7 +344,9 @@ def from_obj(c: Type, o: Any, named: bool, reuse_instances: bool, type_check: Ty
         if is_deserializable(c):
             serde_scope: SerdeScope = getattr(c, SERDE_SCOPE)
             func_name = FROM_DICT if named else FROM_ITER
-            res = serde_scope.funcs[func_name](c, maybe_generic=maybe_generic, data=o, reuse_instances=reuse_instances)
+            res = serde_scope.funcs[func_name](
+                c, maybe_generic=maybe_generic, data=o, reuse_instances=reuse_instances, type_check=type_check
+            )
             if strict:
                 serde_scope.funcs[TYPE_CHECK](res)
             return res
@@ -550,6 +555,7 @@ class Renderer:
     cls: Optional[Type] = None
     custom: Optional[DeserializeFunc] = None  # Custom class level deserializer.
     import_numpy: bool = False
+    suppress_coerce: bool = False
 
     def render(self, arg: DeField) -> str:
         """
@@ -654,13 +660,13 @@ class Renderer:
 
         >>> from typing import List
         >>> Renderer('foo').render(DeField(Optional[int], 'o', datavar='data'))
-        '(data["o"]) if data.get("o") is not None else None'
+        '(coerce(int, data["o"], type_check)) if data.get("o") is not None else None'
 
         >>> Renderer('foo').render(DeField(Optional[List[int]], 'o', datavar='data'))
-        '([v for v in data["o"]]) if data.get("o") is not None else None'
+        '([coerce(int, v, type_check) for v in data["o"]]) if data.get("o") is not None else None'
 
         >>> Renderer('foo').render(DeField(Optional[List[int]], 'o', datavar='data'))
-        '([v for v in data["o"]]) if data.get("o") is not None else None'
+        '([coerce(int, v, type_check) for v in data["o"]]) if data.get("o") is not None else None'
 
         >>> @deserialize
         ... class Foo:
@@ -685,10 +691,10 @@ if data.get("f") is not None else None'
 
         >>> from typing import List
         >>> Renderer('foo').render(DeField(List[int], 'l', datavar='data'))
-        '[v for v in data["l"]]'
+        '[coerce(int, v, type_check) for v in data["l"]]'
 
         >>> Renderer('foo').render(DeField(List[List[int]], 'l', datavar='data'))
-        '[[v for v in v] for v in data["l"]]'
+        '[[coerce(int, v, type_check) for v in v] for v in data["l"]]'
         """
         if is_bare_list(arg.type):
             return f'list({arg.data})'
@@ -701,10 +707,10 @@ if data.get("f") is not None else None'
 
         >>> from typing import Set
         >>> Renderer('foo').render(DeField(Set[int], 'l', datavar='data'))
-        'set(v for v in data["l"])'
+        'set(coerce(int, v, type_check) for v in data["l"])'
 
         >>> Renderer('foo').render(DeField(Set[Set[int]], 'l', datavar='data'))
-        'set(set(v for v in v) for v in data["l"])'
+        'set(set(coerce(int, v, type_check) for v in v) for v in data["l"])'
         """
         if is_bare_set(arg.type):
             return f'set({arg.data})'
@@ -719,13 +725,15 @@ if data.get("f") is not None else None'
         >>> @deserialize
         ... class Foo: pass
         >>> Renderer('foo').render(DeField(Tuple[str, int, List[int], Foo], 'd', datavar='data'))
-        '(data["d"][0], data["d"][1], [v for v in data["d"][2]], \
+        '(coerce(str, data["d"][0], type_check), coerce(int, data["d"][1], type_check), \
+[coerce(int, v, type_check) for v in data["d"][2]], \
 Foo.__serde__.funcs[\\'foo\\'](data=data["d"][3], reuse_instances=reuse_instances),)'
 
         >>> field = DeField(Tuple[str, int, List[int], Foo], 'd', datavar='data', index=0, iterbased=True)
         >>> Renderer('foo').render(field)
-        "(data[0][0], data[0][1], [v for v in data[0][2]], \
-Foo.__serde__.funcs['foo'](data=data[0][3], reuse_instances=reuse_instances),)"
+        "(coerce(str, data[0][0], type_check), coerce(int, data[0][1], type_check), \
+[coerce(int, v, type_check) for v in data[0][2]], Foo.__serde__.funcs['foo'](data=data[0][3], \
+reuse_instances=reuse_instances),)"
         """
         if is_bare_tuple(arg.type):
             return f'tuple({arg.data})'
@@ -742,7 +750,7 @@ Foo.__serde__.funcs['foo'](data=data[0][3], reuse_instances=reuse_instances),)"
 
         >>> from typing import List
         >>> Renderer('foo').render(DeField(Dict[str, int], 'd', datavar='data'))
-        '{k: v for k, v in data["d"].items()}'
+        '{coerce(str, k, type_check): coerce(int, v, type_check) for k, v in data["d"].items()}'
 
         >>> @deserialize
         ... class Foo: pass
@@ -760,20 +768,25 @@ Foo.__serde__.funcs['foo'](data=data[0][3], reuse_instances=reuse_instances),)"
     def enum(self, arg: DeField) -> str:
         return f'{typename(arg.type)}({self.primitive(arg)})'
 
-    def primitive(self, arg: DeField) -> str:
+    def primitive(self, arg: DeField, suppress_coerce: bool = False) -> str:
         """
         Render rvalue for primitives.
 
         >>> Renderer('foo').render(DeField(int, 'i', datavar='data'))
-        'data["i"]'
+        'coerce(int, data["i"], type_check)'
 
         >>> Renderer('foo').render(DeField(int, 'int_field', datavar='data', case='camelcase'))
-        'data["intField"]'
+        'coerce(int, data["intField"], type_check)'
 
         >>> Renderer('foo').render(DeField(int, 'i', datavar='data', index=1, iterbased=True))
-        'data[1]'
+        'coerce(int, data[1], type_check)'
         """
-        return arg.data
+        typ = typename(arg.type)
+        dat = arg.data
+        if self.suppress_coerce:
+            return dat
+        else:
+            return f'coerce({typ}, {dat}, type_check)'
 
     def c_tor(self, arg: DeField) -> str:
         return f"{typename(arg.type)}({arg.data})"
@@ -816,7 +829,8 @@ def to_iter_arg(f: DeField, *args, **kwargs) -> DeField:
 
 def render_from_iter(cls: Type, custom: Optional[DeserializeFunc] = None) -> str:
     template = """
-def {{func}}(cls=cls, maybe_generic=None, data=None, reuse_instances = {{serde_scope.reuse_instances_default}}):
+def {{func}}(cls=cls, maybe_generic=None, data=None, reuse_instances = {{serde_scope.reuse_instances_default}},
+             type_check: TypeCheck=Coerce):
   if reuse_instances is Ellipsis:
     reuse_instances = {{serde_scope.reuse_instances_default}}
 
@@ -851,7 +865,9 @@ def {{func}}(cls=cls, maybe_generic=None, data=None, reuse_instances = {{serde_s
 
 def render_from_dict(cls: Type, rename_all: Optional[str] = None, custom: Optional[DeserializeFunc] = None) -> str:
     template = """
-def {{func}}(cls=cls, maybe_generic=None, data=None, reuse_instances = {{serde_scope.reuse_instances_default}}):
+def {{func}}(cls=cls, maybe_generic=None, data=None,
+             reuse_instances = {{serde_scope.reuse_instances_default}},
+             type_check: TypeCheck = Coerce):
   if reuse_instances is Ellipsis:
     reuse_instances = {{serde_scope.reuse_instances_default}}
 
@@ -888,7 +904,8 @@ def {{func}}(cls=cls, maybe_generic=None, data=None, reuse_instances = {{serde_s
 
 def render_union_func(cls: Type, union_args: List[Type], tagging: Tagging = DefaultTagging) -> str:
     template = """
-def {{func}}(cls=cls, maybe_generic=None, data=None, reuse_instances = {{serde_scope.reuse_instances_default}}):
+def {{func}}(cls=cls, maybe_generic=None, data=None, reuse_instances = {{serde_scope.reuse_instances_default}},
+             type_check: TypeCheck=Coerce):
   errors = []
   {% for t in union_args %}
   try:
@@ -925,7 +942,7 @@ def {{func}}(cls=cls, maybe_generic=None, data=None, reuse_instances = {{serde_s
     """
     union_name = f"Union[{', '.join([typename(a) for a in union_args])}]"
 
-    renderer = Renderer(FROM_DICT, cls=cls)
+    renderer = Renderer(FROM_DICT, cls=cls, suppress_coerce=True)
     env = jinja2.Environment(loader=jinja2.DictLoader({'dict': template}))
     env.filters.update(
         {'arg': lambda x: DeField(x, datavar="fake_dict", name="fake_key")}
diff --git a/serde/se.py b/serde/se.py
index 35691e3..94dde18 100644
--- a/serde/se.py
+++ b/serde/se.py
@@ -54,10 +54,10 @@ from .core import (
     DefaultTagging,
     Field,
     SerdeScope,
-    Strict,
     Tagging,
     TypeCheck,
     add_func,
+    coerce,
     conv,
     fields,
     is_dict_instance,
@@ -236,6 +236,9 @@ def serialize(
         g["to_obj"] = to_obj
         g["typing"] = typing
         g["Literal"] = Literal
+        g["TypeCheck"] = TypeCheck
+        g["Coerce"] = Coerce
+        g["coerce"] = coerce
         if serialize:
             g["serde_custom_class_serializer"] = functools.partial(serde_custom_class_serializer, custom=serializer)
 
@@ -308,7 +311,9 @@ def to_obj(o, named: bool, reuse_instances: bool, convert_sets: bool, c: Type =
             func_name = TO_DICT if named else TO_ITER
             if strict:
                 serde_scope.funcs[TYPE_CHECK](o)
-            return serde_scope.funcs[func_name](o, reuse_instances=reuse_instances, convert_sets=convert_sets)
+            return serde_scope.funcs[func_name](
+                o, reuse_instances=reuse_instances, convert_sets=convert_sets, type_check=type_check
+            )
         elif is_dataclass(o):
             if strict:
                 serde_scope.funcs[TYPE_CHECK](o)
@@ -446,7 +451,8 @@ def sefields(cls: Type) -> Iterator[SeField]:
 def render_to_tuple(cls: Type, custom: Optional[SerializeFunc] = None) -> str:
     template = """
 def {{func}}(obj, reuse_instances = {{serde_scope.reuse_instances_default}},
-             convert_sets = {{serde_scope.convert_sets_default}}):
+             convert_sets = {{serde_scope.convert_sets_default}},
+             type_check: TypeCheck=Coerce):
   if reuse_instances is Ellipsis:
     reuse_instances = {{serde_scope.reuse_instances_default}}
   if convert_sets is Ellipsis:
@@ -473,7 +479,8 @@ def {{func}}(obj, reuse_instances = {{serde_scope.reuse_instances_default}},
 def render_to_dict(cls: Type, case: Optional[str] = None, custom: Optional[SerializeFunc] = None) -> str:
     template = """
 def {{func}}(obj, reuse_instances = {{serde_scope.reuse_instances_default}},
-             convert_sets = {{serde_scope.convert_sets_default}}):
+             convert_sets = {{serde_scope.convert_sets_default}},
+             type_check: TypeCheck = Coerce):
   if reuse_instances is Ellipsis:
     reuse_instances = {{serde_scope.reuse_instances_default}}
   if convert_sets is Ellipsis:
@@ -510,7 +517,7 @@ def render_union_func(cls: Type, union_args: List[Type], tagging: Tagging = Defa
     Render function that serializes a field with union type.
     """
     template = """
-def {{func}}(obj, reuse_instances, convert_sets):
+def {{func}}(obj, reuse_instances, convert_sets, type_check: TypeCheck=Coerce):
   union_args = serde_scope.union_se_args['{{func}}']
 
   {% for t in union_args %}
@@ -536,7 +543,7 @@ def {{func}}(obj, reuse_instances, convert_sets):
     """
     union_name = f"Union[{', '.join([typename(a) for a in union_args])}]"
 
-    renderer = Renderer(TO_DICT)
+    renderer = Renderer(TO_DICT, suppress_coerce=True)
     env = jinja2.Environment(loader=jinja2.DictLoader({"dict": template}))
     env.filters.update({"arg": lambda x: SeField(x, "obj")})
     env.filters.update({"rvalue": renderer.render})
@@ -586,6 +593,7 @@ class Renderer:
 
     func: str
     custom: Optional[SerializeFunc] = None  # Custom class level serializer.
+    suppress_coerce: bool = False
 
     def render(self, arg: SeField) -> str:
         """
@@ -593,10 +601,10 @@ class Renderer:
 
         >>> from typing import Tuple
         >>> Renderer(TO_ITER).render(SeField(int, 'i'))
-        'i'
+        'coerce(int, i, type_check)'
 
         >>> Renderer(TO_ITER).render(SeField(List[int], 'l'))
-        '[v for v in l]'
+        '[coerce(int, v, type_check) for v in l]'
 
         >>> @serialize
         ... @dataclass(unsafe_hash=True)
@@ -609,7 +617,7 @@ class Renderer:
         "[v.__serde__.funcs['to_iter'](v, reuse_instances=reuse_instances, convert_sets=convert_sets) for v in foo]"
 
         >>> Renderer(TO_ITER).render(SeField(Dict[str, Foo], 'foo'))
-        "{k: v.__serde__.funcs['to_iter'](v, reuse_instances=reuse_instances, \
+        "{coerce(str, k, type_check): v.__serde__.funcs['to_iter'](v, reuse_instances=reuse_instances, \
 convert_sets=convert_sets) for k, v in foo.items()}"
 
         >>> Renderer(TO_ITER).render(SeField(Dict[Foo, Foo], 'foo'))
@@ -618,8 +626,8 @@ convert_sets=convert_sets): v.__serde__.funcs['to_iter'](v, reuse_instances=reus
 convert_sets=convert_sets) for k, v in foo.items()}"
 
         >>> Renderer(TO_ITER).render(SeField(Tuple[str, Foo, int], 'foo'))
-        "(foo[0], foo[1].__serde__.funcs['to_iter'](foo[1], reuse_instances=reuse_instances, \
-convert_sets=convert_sets), foo[2],)"
+        "(coerce(str, foo[0], type_check), foo[1].__serde__.funcs['to_iter'](foo[1], reuse_instances=reuse_instances, \
+convert_sets=convert_sets), coerce(int, foo[2], type_check),)"
         """
         if arg.serializer and arg.serializer.inner is not default_serializer:
             res = self.custom_field_serializer(arg)
@@ -762,7 +770,12 @@ convert_sets=convert_sets), foo[2],)"
         """
         Render rvalue for primitives.
         """
-        return f"{arg.varname}"
+        typ = typename(arg.type)
+        var = arg.varname
+        if self.suppress_coerce:
+            return var
+        else:
+            return f'coerce({typ}, {var}, type_check)'
 
     def string(self, arg: SeField) -> str:
         return f"str({arg.varname})"


Test patch:
diff --git a/tests/test_basics.py b/tests/test_basics.py
index 155ae5c..488eaf3 100644
--- a/tests/test_basics.py
+++ b/tests/test_basics.py
@@ -337,7 +337,7 @@ def test_default(se, de):
     assert p == from_dict(PriDefault, {'i': 10, 's': 'foo', 'f': 100.0, 'b': True})
     assert p == from_tuple(PriDefault, (10, 'foo', 100.0, True))
 
-    if se is not serde.toml.to_toml:
+    if se is not serde.toml.to_toml:  # Toml doesn't support None.
         o = OptDefault()
         assert o == de(OptDefault, se(o))
         o = OptDefault()
@@ -754,7 +754,7 @@ test_cases = [
     (Dict[str, data.Int], {"foo": data.Int(1.0)}, True),  # type: ignore
     (Set[int], {10}, False),
     (Set[int], {10.0}, True),
-    (Set[int], [10], False),  # List is coerced into Set
+    (Set[int], [10], True),
     (Tuple[int], (10,), False),
     (Tuple[int], (10.0,), True),
     (Tuple[int, str], (10, "foo"), False),
@@ -767,7 +767,7 @@ test_cases = [
     (Union[int, data.Int], data.Int(10), False),
     (datetime.date, datetime.date.today(), False),
     (pathlib.Path, pathlib.Path(), False),
-    (pathlib.Path, "foo", False),  # str is coerced into Path
+    (pathlib.Path, "foo", True),
 ]
 
 
@@ -779,8 +779,52 @@ def test_type_check(T, data, exc):
 
     if exc:
         with pytest.raises(serde.SerdeError):
-            d = serde.to_dict(C(data))
+            d = serde.to_dict(C(data), type_check=Strict)
             serde.from_dict(C, d, type_check=Strict)
     else:
-        d = serde.to_dict(C(data))
+        d = serde.to_dict(C(data), type_check=Strict)
         serde.from_dict(C, d, type_check=Strict)
+
+
+def test_uncoercible():
+    @serde.serde
+    class Foo:
+        i: int
+
+    with pytest.raises(serde.SerdeError):
+        serde.to_dict(Foo("foo"))
+
+    with pytest.raises(serde.SerdeError):
+        serde.from_dict(Foo, {"i": "foo"})
+
+
+def test_coerce():
+    d = {"i": "10", "s": 100, "f": 1000, "b": "True"}
+    p = serde.from_dict(data.Pri, d)
+    assert p.i == 10
+    assert p.s == "100"
+    assert p.f == 1000.0
+    assert p.b
+
+    p = data.Pri("10", 100, 1000, "True")
+    d = serde.to_dict(p)
+    assert d["i"] == 10
+    assert d["s"] == "100"
+    assert d["f"] == 1000.0
+    assert d["b"]
+
+    # Couldn't coerce
+    with pytest.raises(serde.SerdeError):
+        d = {"i": "foo", "s": 100, "f": "bar", "b": "True"}
+        p = serde.from_dict(data.Pri, d)
+
+    # Nested structure
+    p = data.NestedPri(data.Int("10"), data.Str(100), data.Float(1000), data.Bool("True"))
+    d = serde.to_dict(p)
+    assert d["i"]["i"] == 10
+    assert d["s"]["s"] == "100"
+    assert d["f"]["f"] == 1000.0
+    assert d["b"]["b"]
+
+    d = {"i": "10", "s": 100, "f": 1000, "b": "True"}
+    p = serde.from_dict(data.Pri, d)