Repository: yukinarit/pyserde

Issue:
Missing types with custom class deserializers
The code generation doesn't put `Optional` or `Union` in scope when calling a custom class deserializer. This results in a `NameError` during deserialization.

Example
```py
from dataclasses import dataclass
from typing import Optional

from serde import deserialize, from_dict


def deserialize_maybe_str(_typ, value):
    return value


@deserialize(deserializer=deserialize_maybe_str)
@dataclass
class CustomDe:
    maybe_str: Optional[str]


if __name__ == "__main__":
    data = {"maybe_str": "bye"}
    print(f"from_dict({data}) = {from_dict(CustomDe, data)}")
```
Output
```
Traceback (most recent call last):
  File "C:\Users\mauve\dev\modlunky2\src\union_demo.py", line 19, in <module>
    print(f"from_dict({data}) = {from_dict(CustomDe, data)}")
  File "C:\Users\mauve\dev\modlunky2\.venv\lib\site-packages\serde\de.py", line 319, in from_dict
    return from_obj(cls, o, named=True, reuse_instances=reuse_instances)
  File "C:\Users\mauve\dev\modlunky2\.venv\lib\site-packages\serde\de.py", line 253, in from_obj
    return serde_scope.funcs[FROM_DICT](o, reuse_instances=reuse_instances)
  File "<string>", line 10, in from_dict
NameError: name 'Optional' is not defined
```

Discussion:
yukinarit: This MR fixes the #167 but I would like to find a more elegant way 🤔 
codecov[bot]: # [Codecov](https://codecov.io/gh/yukinarit/pyserde/pull/169?src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=yukinarit) Report
> Merging [#169](https://codecov.io/gh/yukinarit/pyserde/pull/169?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=yukinarit) (f423d65) into [master](https://codecov.io/gh/yukinarit/pyserde/commit/2ae8db139d83497b671a7295f0b6e9896ddc8d0e?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=yukinarit) (2ae8db1) will **increase** coverage by `0.01%`.
> The diff coverage is `100.00%`.

[![Impacted file tree graph](https://codecov.io/gh/yukinarit/pyserde/pull/169/graphs/tree.svg?width=650&height=150&src=pr&token=fF9S7dP4eH&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=yukinarit)](https://codecov.io/gh/yukinarit/pyserde/pull/169?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=yukinarit)

```diff
@@            Coverage Diff             @@
##           master     #169      +/-   ##
==========================================
+ Coverage   93.90%   93.91%   +0.01%     
==========================================
  Files          10       10              
  Lines        1050     1052       +2     
  Branches      239      239              
==========================================
+ Hits          986      988       +2     
  Misses         38       38              
  Partials       26       26              
```


| [Impacted Files](https://codecov.io/gh/yukinarit/pyserde/pull/169?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=yukinarit) | Coverage Δ | |
|---|---|---|
| [serde/de.py](https://codecov.io/gh/yukinarit/pyserde/pull/169/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=yukinarit#diff-c2VyZGUvZGUucHk=) | `97.46% <100.00%> (+<0.01%)` | :arrow_up: |
| [serde/se.py](https://codecov.io/gh/yukinarit/pyserde/pull/169/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=yukinarit#diff-c2VyZGUvc2UucHk=) | `96.80% <100.00%> (+0.01%)` | :arrow_up: |

------

[Continue to review full report at Codecov](https://codecov.io/gh/yukinarit/pyserde/pull/169?src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=yukinarit).
> **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=yukinarit)
> `Δ = absolute <relative> (impact)`, `ø = not affected`, `? = missing data`
> Powered by [Codecov](https://codecov.io/gh/yukinarit/pyserde/pull/169?src=pr&el=footer&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=yukinarit). Last update [2ae8db1...f423d65](https://codecov.io/gh/yukinarit/pyserde/pull/169?src=pr&el=lastupdated&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=yukinarit). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=yukinarit).


Patch:
diff --git a/serde/compat.py b/serde/compat.py
index e6bd1ca..09a1452 100644
--- a/serde/compat.py
+++ b/serde/compat.py
@@ -72,9 +72,17 @@ def typename(typ) -> str:
     'Any'
     """
     if is_opt(typ):
-        return f'Optional[{typename(type_args(typ)[0])}]'
+        args = type_args(typ)
+        if args:
+            return f'Optional[{typename(type_args(typ)[0])}]'
+        else:
+            return 'Optional'
     elif is_union(typ):
-        return f'Union[{", ".join([typename(e) for e in union_args(typ)])}]'
+        args = union_args(typ)
+        if args:
+            return f'Union[{", ".join([typename(e) for e in args])}]'
+        else:
+            return 'Union'
     elif is_list(typ):
         args = type_args(typ)
         if args:
@@ -98,7 +106,11 @@ def typename(typ) -> str:
         else:
             return 'Dict'
     elif is_tuple(typ):
-        return f'Tuple[{", ".join([typename(e) for e in type_args(typ)])}]'
+        args = type_args(typ)
+        if args:
+            return f'Tuple[{", ".join([typename(e) for e in args])}]'
+        else:
+            return 'Tuple'
     elif typ is Any:
         return 'Any'
     else:
@@ -170,9 +182,12 @@ def dataclass_fields(cls: Type) -> Iterator:
     return iter(raw_fields)
 
 
-def iter_types(cls: Type) -> Iterator[Type]:
+def iter_types(cls: Type) -> Iterator[Union[Type, typing.Any]]:
     """
     Iterate field types recursively.
+
+    The correct return type is `Iterator[Union[Type, typing._specialform]],
+    but `typing._specialform` doesn't exist for python 3.6. Use `Any` instead.
     """
     if is_dataclass(cls):
         yield cls
@@ -181,20 +196,30 @@ def iter_types(cls: Type) -> Iterator[Type]:
     elif isinstance(cls, str):
         yield cls
     elif is_opt(cls):
+        yield Optional
         arg = type_args(cls)
         if arg:
             yield from iter_types(arg[0])
     elif is_union(cls):
+        yield Union
         for arg in type_args(cls):
             yield from iter_types(arg)
     elif is_list(cls) or is_set(cls):
+        yield List
+        arg = type_args(cls)
+        if arg:
+            yield from iter_types(arg[0])
+    elif is_set(cls):
+        yield Set
         arg = type_args(cls)
         if arg:
             yield from iter_types(arg[0])
     elif is_tuple(cls):
+        yield Tuple
         for arg in type_args(cls):
             yield from iter_types(arg)
     elif is_dict(cls):
+        yield Dict
         arg = type_args(cls)
         if arg and len(arg) >= 2:
             yield from iter_types(arg[0])
@@ -242,9 +267,32 @@ def is_union(typ) -> bool:
 def is_opt(typ) -> bool:
     """
     Test if the type is `typing.Optional`.
+
+    >>> is_opt(Optional[int])
+    True
+    >>> is_opt(Optional)
+    True
+    >>> is_opt(None.__class__)
+    False
+    """
+    args = type_args(typ)
+    if args:
+        return typing_inspect.is_optional_type(typ) and len(args) == 2 and not is_none(args[0]) and is_none(args[1])
+    else:
+        return typ is Optional
+
+
+def is_bare_opt(typ) -> bool:
+    """
+    Test if the type is `typing.Optional` without type args.
+    >>> is_bare_opt(Optional[int])
+    False
+    >>> is_bare_opt(Optional)
+    True
+    >>> is_bare_opt(None.__class__)
+    False
     """
-    args = get_args(typ)
-    return typing_inspect.is_optional_type(typ) and len(args) == 2 and not is_none(args[0]) and is_none(args[1])
+    return not type_args(typ) and typ is Optional
 
 
 def is_list(typ) -> bool:
diff --git a/serde/core.py b/serde/core.py
index c5474ea..a7558ae 100644
--- a/serde/core.py
+++ b/serde/core.py
@@ -455,3 +455,10 @@ def union_func_name(prefix: str, union_args: List[Type]) -> str:
     'union_se_int_List_str__IPv4Address'
     """
     return re.sub(r"[ ,\[\]]+", "_", f"{prefix}_{'_'.join([typename(e) for e in union_args])}")
+
+
+def filter_scope(scope: Dict[str, Any]) -> Iterator[str]:
+    for k, v in scope.items():
+        if v.__module__ == "typing":
+            continue
+        yield k
diff --git a/serde/de.py b/serde/de.py
index d34eddf..9255dce 100644
--- a/serde/de.py
+++ b/serde/de.py
@@ -46,6 +46,7 @@ from .core import (
     StrSerializableTypes,
     add_func,
     fields,
+    filter_scope,
     logger,
     raise_unsupported_type,
     union_func_name,
@@ -169,14 +170,11 @@ def deserialize(
 
         # Collect types used in the generated code.
         for typ in iter_types(cls):
-            if typ is cls:
+            if typ is cls or (is_primitive(typ) and not is_enum(typ)):
                 continue
 
-            if typ is Any:
-                continue
-
-            if is_dataclass(typ) or is_enum(typ) or not is_primitive(typ):
-                scope.types[typ.__name__] = typ
+            scope.types[typename(typ)] = typ
+            g[typename(typ)] = typ
 
         # render all union functions
         for union in iter_unions(cls):
@@ -473,7 +471,7 @@ class Renderer:
         if self.custom and not arg.deserializer:
             # The function takes a closure in order to execute the default value lazily.
             return (
-                f'serde_custom_class_deserializer({arg.type.__name__}, {arg.datavar}, {arg.data}, '
+                f'serde_custom_class_deserializer({typename(arg.type)}, {arg.datavar}, {arg.data}, '
                 f'default=lambda: {res})'
             )
         else:
@@ -665,7 +663,7 @@ def {{func}}(data, reuse_instances = {{serde_scope.reuse_instances_default}}):
     reuse_instances = {{serde_scope.reuse_instances_default}}
 
   {# List up all classes used by this class. -#}
-  {% for name in serde_scope.types.keys() -%}
+  {% for name in serde_scope.types|filter_scope -%}
   {{name}} = serde_scope.types['{{name}}']
   {% endfor -%}
 
@@ -683,6 +681,7 @@ def {{func}}(data, reuse_instances = {{serde_scope.reuse_instances_default}}):
     env = jinja2.Environment(loader=jinja2.DictLoader({'iter': template}))
     env.filters.update({'rvalue': renderer.render})
     env.filters.update({'arg': to_iter_arg})
+    env.filters.update({'filter_scope': filter_scope})
     return env.get_template('iter').render(func=FROM_ITER, serde_scope=getattr(cls, SERDE_SCOPE), fields=defields(cls))
 
 
@@ -693,7 +692,7 @@ def {{func}}(data, reuse_instances = {{serde_scope.reuse_instances_default}}):
     reuse_instances = {{serde_scope.reuse_instances_default}}
 
   {# List up all classes used by this class. #}
-  {% for name in serde_scope.types.keys() %}
+  {% for name in serde_scope.types|filter_scope %}
   {{name}} = serde_scope.types['{{name}}']
   {% endfor %}
 
@@ -711,13 +710,14 @@ def {{func}}(data, reuse_instances = {{serde_scope.reuse_instances_default}}):
     env = jinja2.Environment(loader=jinja2.DictLoader({'dict': template}))
     env.filters.update({'rvalue': renderer.render})
     env.filters.update({'arg': functools.partial(to_arg, rename_all=rename_all)})
+    env.filters.update({'filter_scope': filter_scope})
     return env.get_template('dict').render(func=FROM_DICT, serde_scope=getattr(cls, SERDE_SCOPE), fields=defields(cls))
 
 
 def render_union_func(cls: Type, union_args: List[Type]) -> str:
     template = """
 def {{func}}(data, reuse_instances):
-  {% for name in serde_scope.types.keys() %}
+  {% for name in serde_scope.types|filter_scope %}
   {{name}} = serde_scope.types['{{name}}']
   {% endfor %}
 
@@ -751,6 +751,7 @@ def {{func}}(data, reuse_instances):
     env.filters.update({'rvalue': renderer.render})
     env.filters.update({'is_primitive': is_primitive})
     env.filters.update({'is_none': is_none})
+    env.filters.update({'filter_scope': filter_scope})
     return env.get_template('dict').render(
         func=union_func_name(UNION_DE_PREFIX, union_args),
         serde_scope=getattr(cls, SERDE_SCOPE),
diff --git a/serde/se.py b/serde/se.py
index 59e21c9..861461a 100644
--- a/serde/se.py
+++ b/serde/se.py
@@ -18,6 +18,7 @@ from .compat import (
     SerdeSkip,
     is_bare_dict,
     is_bare_list,
+    is_bare_opt,
     is_bare_set,
     is_bare_tuple,
     is_dict,
@@ -46,6 +47,7 @@ from .core import (
     add_func,
     conv,
     fields,
+    filter_scope,
     is_instance,
     logger,
     raise_unsupported_type,
@@ -179,14 +181,11 @@ def serialize(
 
         # Collect types used in the generated code.
         for typ in iter_types(cls):
-            if typ is cls:
+            if typ is cls or (is_primitive(typ) and not is_enum(typ)):
                 continue
 
-            if typ is Any:
-                continue
-
-            if is_dataclass(typ) or is_enum(typ) or not is_primitive(typ):
-                scope.types[typ.__name__] = typ
+            scope.types[typename(typ)] = typ
+            g[typename(typ)] = typ
 
         # render all union functions
         for union in iter_unions(cls):
@@ -376,7 +375,7 @@ def {{func}}(obj, reuse_instances = {{serde_scope.reuse_instances_default}},
     return copy.deepcopy(obj)
 
   {# List up all classes used by this class. #}
-  {% for name in serde_scope.types.keys() %}
+  {% for name in serde_scope.types|filter_scope %}
   {{name}} = serde_scope.types['{{name}}']
   {% endfor %}
 
@@ -392,6 +391,7 @@ def {{func}}(obj, reuse_instances = {{serde_scope.reuse_instances_default}},
     renderer = Renderer(TO_ITER, custom)
     env = jinja2.Environment(loader=jinja2.DictLoader({'iter': template}))
     env.filters.update({'rvalue': renderer.render})
+    env.filters.update({'filter_scope': filter_scope})
     return env.get_template('iter').render(func=TO_ITER, serde_scope=getattr(cls, SERDE_SCOPE), fields=sefields(cls))
 
 
@@ -408,7 +408,7 @@ def {{func}}(obj, reuse_instances = {{serde_scope.reuse_instances_default}},
     return copy.deepcopy(obj)
 
   {# List up all classes used by this class. #}
-  {% for name in serde_scope.types.keys() -%}
+  {% for name in serde_scope.types|filter_scope -%}
   {{name}} = serde_scope.types['{{name}}']
   {% endfor -%}
 
@@ -432,13 +432,14 @@ def {{func}}(obj, reuse_instances = {{serde_scope.reuse_instances_default}},
     env.filters.update({'rvalue': renderer.render})
     env.filters.update({'lvalue': lrenderer.render})
     env.filters.update({'case': functools.partial(conv, case=case)})
+    env.filters.update({'filter_scope': filter_scope})
     return env.get_template('dict').render(func=TO_DICT, serde_scope=getattr(cls, SERDE_SCOPE), fields=sefields(cls))
 
 
 def render_union_func(cls: Type, union_args: List[Type]) -> str:
     template = """
 def {{func}}(obj, reuse_instances, convert_sets):
-  {% for name in serde_scope.types.keys() %}
+  {% for name in serde_scope.types|filter_scope %}
   {{name}} = serde_scope.types['{{name}}']
   {% endfor %}
 
@@ -456,6 +457,7 @@ def {{func}}(obj, reuse_instances, convert_sets):
     env = jinja2.Environment(loader=jinja2.DictLoader({'dict': template}))
     env.filters.update({'arg': lambda x: SeField(x, "obj")})
     env.filters.update({'rvalue': renderer.render})
+    env.filters.update({'filter_scope': filter_scope})
     return env.get_template('dict').render(
         func=union_func_name(UNION_SE_PREFIX, union_args),
         serde_scope=getattr(cls, SERDE_SCOPE),
@@ -567,7 +569,7 @@ convert_sets=convert_sets), foo[2],)"
 
         # Custom field serializer overrides custom class serializer.
         if self.custom and not arg.serializer:
-            return f'serde_custom_class_serializer({arg.type.__name__}, {arg.varname}, default=lambda: {res})'
+            return f'serde_custom_class_serializer({typename(arg.type)}, {arg.varname}, default=lambda: {res})'
         else:
             return res
 
@@ -598,9 +600,12 @@ convert_sets=convert_sets), foo[2],)"
         """
         Render rvalue for optional.
         """
-        inner = arg[0]
-        inner.name = arg.varname
-        return f'({self.render(inner)}) if {arg.varname} is not None else None'
+        if is_bare_opt(arg.type):
+            return f'{arg.varname} if {arg.varname} is not None else None'
+        else:
+            inner = arg[0]
+            inner.name = arg.varname
+            return f'({self.render(inner)}) if {arg.varname} is not None else None'
 
     def list(self, arg: SeField) -> str:
         """


Test patch:
diff --git a/tests/test_compat.py b/tests/test_compat.py
index ced8dcb..e88211a 100644
--- a/tests/test_compat.py
+++ b/tests/test_compat.py
@@ -13,6 +13,7 @@ from serde.compat import (
     iter_types,
     iter_unions,
     type_args,
+    typename,
     union_args,
 )
 from serde.core import is_instance
@@ -52,12 +53,16 @@ def test_types():
         assert is_dict(dict[str, int])
 
 
+def test_typename():
+    assert typename(Optional) == "Optional"
+
+
 def test_iter_types():
     assert [Pri, int, str, float, bool] == list(iter_types(Pri))
-    assert [str, Pri, int, str, float, bool] == list(iter_types(Dict[str, Pri]))
-    assert [str] == list(iter_types(List[str]))
-    assert [int, str, bool, float] == list(iter_types(Tuple[int, str, bool, float]))
-    assert [PriOpt, int, str, float, bool] == list(iter_types(PriOpt))
+    assert [Dict, str, Pri, int, str, float, bool] == list(iter_types(Dict[str, Pri]))
+    assert [List, str] == list(iter_types(List[str]))
+    assert [Tuple, int, str, bool, float] == list(iter_types(Tuple[int, str, bool, float]))
+    assert [PriOpt, Optional, int, Optional, str, Optional, float, Optional, bool] == list(iter_types(PriOpt))
 
 
 def test_iter_unions():
diff --git a/tests/test_custom.py b/tests/test_custom.py
index 653c757..b13e870 100644
--- a/tests/test_custom.py
+++ b/tests/test_custom.py
@@ -3,7 +3,7 @@ Tests for custom serializer/deserializer.
 """
 from dataclasses import dataclass, field
 from datetime import datetime
-from typing import Optional
+from typing import Optional, Union
 
 import pytest
 
@@ -93,14 +93,16 @@ def test_custom_class_serializer():
         i: int
         dt1: datetime
         dt2: datetime
+        s: Optional[str] = None
+        u: Union[str, int] = 10
 
     dt = datetime(2021, 1, 1, 0, 0, 0)
     f = Foo(10, dt, dt)
 
-    assert to_json(f) == '{"i": 10, "dt1": "01/01/21", "dt2": "01/01/21"}'
+    assert to_json(f) == '{"i": 10, "dt1": "01/01/21", "dt2": "01/01/21", "s": null, "u": 10}'
     assert f == from_json(Foo, to_json(f))
 
-    assert to_tuple(f) == (10, '01/01/21', '01/01/21')
+    assert to_tuple(f) == (10, '01/01/21', '01/01/21', None, 10)
     assert f == from_tuple(Foo, to_tuple(f))