Repository: yukinarit/pyserde

Issue:
Pyserde does not use default_factory during deserialization
I want to define a dict field that is optional but non-nullable, and this can be done by using default_factory. But I found that pyserde cannot deserialize data if a value for this field is not present.

```python
from dataclasses import dataclass, field
from typing import Dict
from serde import deserialize, from_dict

@deserialize
@dataclass
class Foo:
    foo: str
    items: Dict[str, int] = field(default_factory=dict)

# check that default_factory really works
assert Foo(foo="bar").items == {}

# but it fails
foo = from_dict(Foo, {"foo": "bar"})
```

```
Traceback (most recent call last):
  File "serde_test.py", line 15, in <module>
    foo = from_dict(Foo, {"foo": "bar"})
  File "/serde/de.py", line 209, in from_dict
    return cls.__serde_from_dict__(o)
  File "<string>", line 9, in __serde_from_dict__
KeyError: 'items'
```

It seems the renderer completely ignores the existence of `default_factory`. My experiments with pyserde code have shown that this can in principle be fixed by adding `serde.core.Field.default_factory` and modifying the renderer (warning, ugly pseudocode):

```python
@dataclass
class Renderer:
    # ...
    def render(self, arg: DeField) -> str:
        if is_dataclass(arg.type):
            result = self.dataclass(arg)
        # ...
        if arg.default_factory is None or isinstance(arg.default_factory, dataclasses._MISSING_TYPE):
            return result
        return f'{result} if "{arg.name}" in {arg.datavar} else cls.__dataclass_fields__["{arg.name}"].default_factory()'
```

But I'm not smart enough to test all cases and make a proper pull request, so I just leave this issue here.

(Not sure if this is a bug report or feature request...)

Discussion:

Patch:
diff --git a/serde/compat.py b/serde/compat.py
index e4b4702..0e32883 100644
--- a/serde/compat.py
+++ b/serde/compat.py
@@ -1,6 +1,7 @@
 """
 Module for compatibility.
 """
+import dataclasses
 import enum
 from dataclasses import fields, is_dataclass
 from itertools import zip_longest
@@ -190,6 +191,39 @@ def is_primitive(typ) -> bool:
         return any(isinstance(typ, ty) for ty in PRIMITIVES)
 
 
+def has_default(field) -> bool:
+    """
+    Test if the field has default value.
+
+    >>> @dataclasses.dataclass
+    ... class C:
+    ...     a: int
+    ...     d: int = 10
+    >>> has_default(fields(C)[0])
+    False
+    >>> has_default(fields(C)[1])
+    True
+    """
+    return not isinstance(field.default, dataclasses._MISSING_TYPE)
+
+
+def has_default_factory(field) -> bool:
+    """
+    Test if the field has default factory.
+
+    >>> from typing import Dict
+    >>> @dataclasses.dataclass
+    ... class C:
+    ...     a: int
+    ...     d: Dict = dataclasses.field(default_factory=dict)
+    >>> has_default_factory(fields(C)[0])
+    False
+    >>> has_default_factory(fields(C)[1])
+    True
+    """
+    return not isinstance(field.default_factory, dataclasses._MISSING_TYPE)
+
+
 def assert_type(typ: Type, obj, throw=False) -> None:
     if not isinstance(obj, typ):
         if throw:
diff --git a/serde/core.py b/serde/core.py
index 94e8d93..a24da6f 100644
--- a/serde/core.py
+++ b/serde/core.py
@@ -161,6 +161,7 @@ class Field:
     type: Type
     name: Optional[str]
     default: Any = field(default_factory=dataclasses._MISSING_TYPE)
+    default_factory: Any = field(default_factory=dataclasses._MISSING_TYPE)
     case: Optional[str] = None
     rename: Optional[str] = None
     skip: Optional[bool] = None
@@ -183,6 +184,7 @@ class Field:
             f.type,
             f.name,
             default=f.default,
+            default_factory=f.default_factory,  # type: ignore
             rename=f.metadata.get('serde_rename'),
             skip=f.metadata.get('serde_skip'),
             skip_if=skip_if or skip_if_false_func,
diff --git a/serde/de.py b/serde/de.py
index bca8233..80eaa0f 100644
--- a/serde/de.py
+++ b/serde/de.py
@@ -19,7 +19,8 @@ from typing import Any, Callable, Dict, List, Optional, Type
 
 import jinja2
 
-from .compat import is_dict, is_enum, is_list, is_opt, is_primitive, is_tuple, is_union, iter_types, type_args
+from .compat import (has_default, has_default_factory, is_dict, is_enum, is_list, is_opt, is_primitive, is_tuple,
+                     is_union, iter_types, type_args)
 from .core import FROM_DICT, FROM_ITER, HIDDEN_NAME, SETTINGS, Field, Hidden, SerdeError, T, conv, fields, gen, logger
 from .more_types import deserialize as custom
 
@@ -62,7 +63,7 @@ def deserialize(_cls=None, rename_all: Optional[str] = None):
     """
 
     def wrap(cls):
-        g: Dict[str, Any] = globals().copy()
+        g: Dict[str, Any] = {}
 
         # Create hidden object used by serde.
         if not hasattr(cls, HIDDEN_NAME):
@@ -79,6 +80,15 @@ def deserialize(_cls=None, rename_all: Optional[str] = None):
         for typ in iter_types(cls):
             if is_dataclass(typ) or is_enum(typ):
                 getattr(cls, '__serde_scope__')[typ.__name__] = typ
+
+        # Collect default values and default factories used in the generated code.
+        # To avoid name conflicts, name the variables like "__default_<NAME>__".
+        for f in dataclasses.fields(cls):
+            if has_default(f):
+                g[f'__default_{f.name}__'] = f.default
+            elif has_default_factory(f):
+                g[f'__default_{f.name}__'] = f.default_factory
+
         logger.debug(f'{cls.__name__}: __serde_scope__ {scope}')
 
         cls = de_func(cls, FROM_ITER, render_from_iter(cls, custom), g)
@@ -299,22 +309,34 @@ class Renderer:
         Render rvalue
         """
         if is_dataclass(arg.type):
-            return self.dataclass(arg)
+            res = self.dataclass(arg)
         elif is_opt(arg.type):
-            return self.opt(arg)
+            res = self.opt(arg)
         elif is_list(arg.type):
-            return self.list(arg)
+            res = self.list(arg)
         elif is_dict(arg.type):
-            return self.dict(arg)
+            res = self.dict(arg)
         elif is_tuple(arg.type):
-            return self.tuple(arg)
+            res = self.tuple(arg)
         elif is_enum(arg.type):
-            return self.enum(arg)
+            res = self.enum(arg)
         elif any(f(arg.type) for f in (is_primitive, is_union)):
-            return self.primitive(arg)
+            res = self.primitive(arg)
         else:
             return f'__custom_deserializer__(fs[{arg.index}], {arg.data})'
 
+        if has_default(arg) or has_default_factory(arg):
+            if arg.iterbased:
+                exists = f'{arg.data} is not None'
+            else:
+                exists = f'"{arg.name}" in {arg.datavar}'
+            if has_default(arg):
+                return f'{res} if {exists} else __default_{arg.name}__'
+            elif has_default_factory(arg):
+                return f'{res} if {exists} else __default_{arg.name}__()'
+
+        return res
+
     def dataclass(self, arg: DeField) -> str:
         return f'{arg.type.__name__}.{self.func}({arg.data})'
 
@@ -415,11 +437,8 @@ class Renderer:
         >>> Renderer('foo').render(DeField(int, 'i', datavar='data', index=1, iterbased=True))
         'data[1]'
         """
-        if not arg.iterbased and not isinstance(arg.default, dataclasses._MISSING_TYPE):
-            default = arg.default
-            if isinstance(default, str):
-                default = f'"{default}"'
-            return f'{arg.datavar}.get("{arg.name}", {default})'
+        if not arg.iterbased and has_default(arg):
+            return f'{arg.datavar}.get("{arg.name}", __default_{arg.name}__)'
         else:
             return arg.data
 


Test patch:
diff --git a/tests/test_basics.py b/tests/test_basics.py
index 795f200..90f3d75 100644
--- a/tests/test_basics.py
+++ b/tests/test_basics.py
@@ -59,6 +59,7 @@ def test_primitive(se, de, opt):
         """
         Primitives.
         """
+
         i: int
         s: str
         f: float
@@ -247,17 +248,20 @@ def test_optional_nested(se, de):
     assert p == de(NestedPriOpt, se(p))
 
 
-def test_field_default():
+@pytest.mark.parametrize('se,de', all_formats)
+def test_dataclass_default_factory(se, de):
+    @deserialize
+    @serialize
     @dataclass
     class Foo:
-        i: int = 10
+        foo: str
+        items: Dict[str, int] = field(default_factory=dict)
 
-    @dataclass
-    class Bar:
-        i: int = field(default=10)
+    f = Foo('bar')
+    assert f == de(Foo, se(f))
 
-    assert 10 == dataclasses.fields(Foo)[0].default
-    assert 10 == dataclasses.fields(Bar())[0].default
+    assert {'foo': 'bar', 'items': {}} == asdict(f)
+    assert f == from_dict(Foo, {'foo': 'bar'})
 
 
 @pytest.mark.parametrize('se,de', all_formats)
@@ -265,6 +269,18 @@ def test_default(se, de):
     p = PriDefault()
     assert p == de(PriDefault, se(p))
 
+    p = PriDefault()
+    assert p == from_dict(PriDefault, {})
+    assert p == from_dict(PriDefault, {'i': 10})
+    assert p == from_dict(PriDefault, {'i': 10, 's': 'foo'})
+    assert p == from_dict(PriDefault, {'i': 10, 's': 'foo', 'f': 100.0})
+    assert p == from_dict(PriDefault, {'i': 10, 's': 'foo', 'f': 100.0, 'b': True})
+
+    assert 10 == dataclasses.fields(PriDefault)[0].default
+    assert 'foo' == dataclasses.fields(PriDefault)[1].default
+    assert 100.0 == dataclasses.fields(PriDefault)[2].default
+    assert True is dataclasses.fields(PriDefault)[3].default
+
 
 @pytest.mark.parametrize('se,de', (format_dict + format_tuple + format_json + format_msgpack + format_yaml))
 def test_list_pri(se, de):
@@ -284,59 +300,6 @@ def test_dict_pri(se, de):
     assert p == de(data.DictPri, se(p))
 
 
-def test_complex():
-    @deserialize
-    @serialize
-    @dataclass
-    class Baz:
-        v: List[int] = field(default_factory=list)
-        d: Dict[str, int] = field(default_factory=dict)
-
-    @deserialize
-    @serialize
-    @dataclass
-    class Bar:
-        i: int
-
-    @deserialize
-    @serialize
-    @dataclass
-    class Foo:
-        i: int
-        s: str
-        f: float
-        b: bool
-        foo: Baz
-        lst: List[Bar] = field(default_factory=list)
-        lst2: List[Dict[str, Bar]] = field(default_factory=list)
-        dct: Dict[str, List[List[Bar]]] = field(default_factory=dict)
-
-    f = Baz(v=[1, 2, 3, 4, 5], d={'foo': 10, 'fuga': 20})
-    lst = [Bar(10), Bar(20)]
-    lst2 = [{'bar1': Bar(10)}, {'bar2': Bar(10), 'bar3': Bar(20)}]
-    dct = {'foo': [[Bar(10), Bar(20)], [Bar(20), Bar(30)]]}
-    h = Foo(i=10, s='foo', f=100.0, b=True, foo=f, lst=lst, lst2=lst2, dct=dct)
-    s = """
-               {"i": 10,
-                "s": "foo",
-                "f": 100.0,
-                "b": true,
-                "foo" : {
-                    "v": [1, 2, 3, 4, 5],
-                    "d": {"foo": 10, "fuga": 20}
-                },
-                "lst": [{"i": 10}, {"i": 20}],
-                "lst2": [{"bar1": {"i": 10}}, {"bar2": {"i": 10}, "bar3": {"i": 20}}],
-                "dct": {"foo": [[{"i": 10}, {"i": 20}], [{"i": 20}, {"i": 30}]]}
-                }
-                """
-    assert json.loads(s) == json.loads(to_json(h))
-    hh = from_json(Foo, s)
-    assert h.foo == hh.foo
-    assert h.lst == hh.lst
-    assert h.dct == hh.dct
-
-
 def test_json():
     p = Pri(10, 'foo', 100.0, True)
     s = '{"i": 10, "s": "foo", "f": 100.0, "b": true}'
@@ -445,8 +408,8 @@ def test_skip_if(se, de):
 
     f = Foo([])
     ff = de(Foo, se(f))
-    assert ff.comments is None  # TODO format_tuple gets [] instaed of None
-    assert ff.attrs is None  # TODO format_tuple gets {} instaed of None
+    assert ff.comments == []
+    assert ff.attrs == {}
 
 
 @pytest.mark.parametrize('se,de', all_formats)
@@ -473,4 +436,4 @@ def test_skip_if_overrides_skip_if_false(se, de):
 
     f = Foo(['foo'])
     ff = de(Foo, se(f))
-    assert ff.comments is None
+    assert ff.comments == []
diff --git a/tests/test_more_types.py b/tests/test_more_types.py
index 8da673c..ba3ad9c 100644
--- a/tests/test_more_types.py
+++ b/tests/test_more_types.py
@@ -13,6 +13,7 @@ from tests.test_basics import format_json, format_toml, format_yaml
 @dataclass
 class Foo:
     d: Decimal
+    d2: Decimal = Decimal('10.0')
 
 
 @deserialize
@@ -20,6 +21,7 @@ class Foo:
 @dataclass
 class Bar:
     p: Path
+    p2: Path = Path('/etc')
 
 
 @pytest.mark.parametrize('se,de', format_json + format_yaml + format_toml)  # type: ignore
@@ -28,13 +30,16 @@ def test_decimal(se, de):
     assert f == de(Foo, se(f))
 
     f = Foo(d=Decimal(0.1))
-    assert f == de(Foo, se(f))
+    ff = de(Foo, se(f))
+    assert f == ff
+    assert f.d == Decimal(0.1)
+    assert f.d2 == Decimal(10.0)
 
 
 @pytest.mark.parametrize('se,de', format_json + format_yaml + format_toml)  # type: ignore
 def test_path(se, de):
     f = Bar(p=Path('/tmp'))
-    assert f == de(Bar, se(f))
-
-    f = Bar(p=Path('/tmp'))
-    assert f == de(Bar, se(f))
+    ff = de(Bar, se(f))
+    assert f == ff
+    assert Path('/tmp') == ff.p
+    assert Path('/etc') == ff.p2