Repository: s-knibbs/dataclasses-jsonschema (2019-07-10 ~ 2019-07-12)

problem_statement: Optional type check fails with optional union type
The following type does not currently get identified as optional:
```python
Optional[Union[int, float]]
```

This is because the code is looking for `type(None)` as the second value in the `__args__` array but does not take into account the flattening of `Union` types (`Optional` is an alias of `Union[T, type(None)]`).
date: 2019-07-10
patch: diff --git a/dataclasses_jsonschema/__init__.py b/dataclasses_jsonschema/__init__.py
index 383d591..493c311 100644
--- a/dataclasses_jsonschema/__init__.py
+++ b/dataclasses_jsonschema/__init__.py
@@ -57,7 +57,7 @@ def issubclass_safe(klass: Any, base: Type):
 
 def is_optional(field: Any) -> bool:
     try:
-        return field.__origin__ == Union and issubclass(field.__args__[1], type(None))
+        return field.__origin__ == Union and type(None) in field.__args__
     except AttributeError:
         return False
 
@@ -80,10 +80,15 @@ def is_literal(field: Any) -> bool:
         return False
 
 
-def final_wrapped_type(final_type: Any) -> Any:
+def unwrap_final(final_type: Any) -> Any:
     return final_type.__args__[0] if sys.version_info[:2] >= (3, 7) else final_type.__type__
 
 
+def unwrap_optional(optional_type: Any) -> Any:
+    idx = optional_type.__args__.index(type(None))
+    return Union[optional_type.__args__[:idx] + optional_type.__args__[idx+1:]]
+
+
 def schema_reference(schema_type: SchemaType, schema_name: str) -> Dict[str, str]:
     ref_path = '#/components/schemas' if schema_type == SchemaType.SWAGGER_V3 else '#/definitions'
     return {"$ref": '{}/{}'.format(ref_path, schema_name)}
@@ -193,9 +198,9 @@ class JsonSchemaMixin:
             if field_type in cls._field_encoders:
                 def encoder(ft, v, __): return cls._field_encoders[ft].to_wire(v)
             elif is_optional(field_type):
-                def encoder(ft, val, o): return cls._encode_field(ft.__args__[0], val, o)
+                def encoder(ft, val, o): return cls._encode_field(unwrap_optional(ft), val, o)
             elif is_final(field_type):
-                def encoder(ft, val, o): return cls._encode_field(final_wrapped_type(ft), val, o)
+                def encoder(ft, val, o): return cls._encode_field(unwrap_final(ft), val, o)
             elif is_enum(field_type):
                 def encoder(_, v, __): return v.value
             elif field_type_name == 'Union':
@@ -296,9 +301,9 @@ class JsonSchemaMixin:
             if cls._is_json_schema_subclass(field_type):
                 def decoder(_, ft, val): return ft.from_dict(val, validate=False)
             elif is_optional(field_type):
-                def decoder(f, ft, val): return cls._decode_field(f, ft.__args__[0], val)
+                def decoder(f, ft, val): return cls._decode_field(f, unwrap_optional(ft), val)
             elif is_final(field_type):
-                def decoder(f, ft, val): return cls._decode_field(f, final_wrapped_type(ft), val)
+                def decoder(f, ft, val): return cls._decode_field(f, unwrap_final(ft), val)
             elif field_type_name == 'Union':
                 # Attempt to decode the value using each decoder in turn
                 decoded = None
@@ -400,9 +405,9 @@ class JsonSchemaMixin:
             values = init_values if field.init else non_init_values
             ft = field.type
             if is_optional(ft):
-                ft = ft.__args__[0]
+                ft = unwrap_optional(ft)
             field_type_name = cls._get_field_type_name(ft)
-            if cls._is_json_schema_subclass(field.type):
+            if cls._is_json_schema_subclass(ft):
                 values[field.name] = ft.from_object(getattr(obj, field.name), exclude=sub_exclude)
             elif field_type_name == "List" and cls._is_json_schema_subclass(ft.__args__[0]):
                 values[field.name] = [
@@ -471,10 +476,10 @@ class JsonSchemaMixin:
         else:
             # If is optional[...]
             if is_optional(field_type):
-                field_schema = cls._get_field_schema(field_type.__args__[0], schema_type)[0]
+                field_schema = cls._get_field_schema(unwrap_optional(field_type), schema_type)[0]
                 required = False
             elif is_final(field_type):
-                field_schema, required = cls._get_field_schema(final_wrapped_type(field_type), schema_type)
+                field_schema, required = cls._get_field_schema(unwrap_final(field_type), schema_type)
             elif is_literal(field_type):
                 field_schema = {
                     'enum': list(field_type.__args__ if sys.version_info[:2] >= (3, 7) else field_type.__values__)

test_patch: diff --git a/tests/test_core.py b/tests/test_core.py
index b341235..8df892a 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -2,7 +2,7 @@ import datetime
 from _decimal import Decimal
 from dataclasses import dataclass, field
 from ipaddress import IPv4Address, IPv6Address
-from typing import List, NewType, Optional
+from typing import List, NewType, Optional, Union
 from typing_extensions import Final, Literal
 from uuid import UUID
 
@@ -524,3 +524,19 @@ def test_inherited_schema():
         }
     })
     assert Cat.json_schema() == expected_schema
+
+
+def test_optional_union():
+    @dataclass
+    class Baz(JsonSchemaMixin):
+        """Class with optional union"""
+        a: Optional[Union[int, str]]
+
+    expected_schema = compose_schema({
+        "description": "Class with optional union",
+        "type": "object",
+        "properties": {
+            "a": {"oneOf": [{"type": "integer"}, {"type": "string"}]}
+        }
+    })
+    assert Baz.json_schema() == expected_schema


----

problem_statement: Base classes may only be used by one derived class
If I create a base class with some common fields I can create a subclass and it works, but if I create more than one subclass from the same base class the validation fails.

I added `tests/test_inheritance.py`
```
from dataclasses import dataclass
from dataclasses_jsonschema import JsonSchemaMixin


def test_inheritance():
    @dataclass
    class Base(JsonSchemaMixin):
        base: str

    @dataclass
    class Derived1(Base):
        derived1: str

    @dataclass
    class Derived2(Base):
        derived2: str

    data1 = {"derived1": "d", "base": "b"}
    d = Derived1.from_dict(data1)
    assert d.to_dict() == data1

    data2 = {"derived2": "d", "base": "b"}
    d = Derived2.from_dict(data2)
    assert d.to_dict() == data2
```
Expected output: the test should pass.
Actual output:
```
GLOB sdist-make: /Users/duncan.booth/github/dataclasses-jsonschema/setup.py
py37 inst-nodeps: /Users/duncan.booth/github/dataclasses-jsonschema/.tox/.tmp/package/1/dataclasses-jsonschema-2.6.2.dev0+ge79e8e1.d20190712.zip
py37 installed: apispec==2.0.2,apispec-webframeworks==0.4.0,atomicwrites==1.3.0,attrs==19.1.0,Click==7.0,dataclasses-jsonschema==2.6.2.dev0+ge79e8e1.d20190712,entrypoints==0.3,flake8==3.7.8,Flask==1.1.1,importlib-metadata==0.18,itsdangerous==1.1.0,Jinja2==2.10.1,jsonschema==3.0.1,MarkupSafe==1.1.1,mccabe==0.6.1,more-itertools==7.1.0,mypy==0.711,mypy-extensions==0.4.1,packaging==19.0,pluggy==0.12.0,py==1.8.0,pycodestyle==2.5.0,pyflakes==2.1.1,pyparsing==2.4.0,pyrsistent==0.15.3,pytest==5.0.1,pytest-ordering==0.6,python-dateutil==2.8.0,PyYAML==5.1.1,six==1.12.0,typed-ast==1.4.0,typing-extensions==3.7.4,wcwidth==0.1.7,Werkzeug==0.15.4,zipp==0.5.2
py37 run-test-pre: PYTHONHASHSEED='2578755219'
py37 run-test: commands[0] | pytest tests
==================================================================== test session starts =====================================================================
platform darwin -- Python 3.7.2, pytest-5.0.1, py-1.8.0, pluggy-0.12.0
cachedir: .tox/py37/.pytest_cache
rootdir: /Users/duncan.booth/github/dataclasses-jsonschema
plugins: ordering-0.6
collected 28 items                                                                                                                                           

tests/test_core.py ..........................                                                                                                          [ 92%]
tests/test_inheritance.py F                                                                                                                            [ 96%]
tests/test_apispec_plugin.py .                                                                                                                         [100%]

========================================================================== FAILURES ==========================================================================
______________________________________________________________________ test_inheritance ______________________________________________________________________

self = <jsonschema.validators.RefResolver object at 0x107a187b8>
document = {'$schema': 'http://json-schema.org/draft-06/schema#', 'allOf': [{'$ref': '#/definitions/Base'}, {'properties': {'deri...{'type': 'string'}}, 'required': ['derived1'], 'type': 'object'}], 'description': 'Derived1(base: str, derived1: str)'}
fragment = 'definitions/Base'

    def resolve_fragment(self, document, fragment):
        """
        Resolve a ``fragment`` within the referenced ``document``.
    
        Arguments:
    
            document:
    
                The referent document
    
            fragment (str):
    
                a URI fragment to resolve within it
        """
    
        fragment = fragment.lstrip(u"/")
        parts = unquote(fragment).split(u"/") if fragment else []
    
        for part in parts:
            part = part.replace(u"~1", u"/").replace(u"~0", u"~")
    
            if isinstance(document, Sequence):
                # Array indexes should be turned into integers
                try:
                    part = int(part)
                except ValueError:
                    pass
            try:
>               document = document[part]
E               KeyError: 'definitions'

.tox/py37/lib/python3.7/site-packages/jsonschema/validators.py:776: KeyError

During handling of the above exception, another exception occurred:

    def test_inheritance():
        @dataclass
        class Base(JsonSchemaMixin):
            base: str
    
        @dataclass
        class Derived1(Base):
            derived1: str
    
        @dataclass
        class Derived2(Base):
            derived2: str
    
        data1 = {"derived1": "d", "base": "b"}
        d = Derived1.from_dict(data1)
        assert d.to_dict() == data1
    
        data2 = {"derived2": "d", "base": "b"}
>       d = Derived2.from_dict(data2)

tests/test_inheritance.py:24: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
dataclasses_jsonschema/__init__.py:396: in from_dict
    cls._validate(data)
dataclasses_jsonschema/__init__.py:383: in _validate
    validate_func(data, cls.json_schema())
.tox/py37/lib/python3.7/site-packages/jsonschema/validators.py:897: in validate
    error = exceptions.best_match(validator.iter_errors(instance))
.tox/py37/lib/python3.7/site-packages/jsonschema/exceptions.py:293: in best_match
    best = next(errors, None)
.tox/py37/lib/python3.7/site-packages/jsonschema/validators.py:323: in iter_errors
    for error in errors:
.tox/py37/lib/python3.7/site-packages/jsonschema/_validators.py:303: in allOf
    for error in validator.descend(instance, subschema, schema_path=index):
.tox/py37/lib/python3.7/site-packages/jsonschema/validators.py:339: in descend
    for error in self.iter_errors(instance, schema):
.tox/py37/lib/python3.7/site-packages/jsonschema/validators.py:323: in iter_errors
    for error in errors:
.tox/py37/lib/python3.7/site-packages/jsonschema/_validators.py:247: in ref
    scope, resolved = validator.resolver.resolve(ref)
.tox/py37/lib/python3.7/site-packages/jsonschema/validators.py:734: in resolve
    return url, self._remote_cache(url)
.tox/py37/lib/python3.7/site-packages/jsonschema/validators.py:746: in resolve_from_url
    return self.resolve_fragment(document, fragment)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <jsonschema.validators.RefResolver object at 0x107a187b8>
document = {'$schema': 'http://json-schema.org/draft-06/schema#', 'allOf': [{'$ref': '#/definitions/Base'}, {'properties': {'deri...{'type': 'string'}}, 'required': ['derived1'], 'type': 'object'}], 'description': 'Derived1(base: str, derived1: str)'}
fragment = 'definitions/Base'

    def resolve_fragment(self, document, fragment):
        """
        Resolve a ``fragment`` within the referenced ``document``.
    
        Arguments:
    
            document:
    
                The referent document
    
            fragment (str):
    
                a URI fragment to resolve within it
        """
    
        fragment = fragment.lstrip(u"/")
        parts = unquote(fragment).split(u"/") if fragment else []
    
        for part in parts:
            part = part.replace(u"~1", u"/").replace(u"~0", u"~")
    
            if isinstance(document, Sequence):
                # Array indexes should be turned into integers
                try:
                    part = int(part)
                except ValueError:
                    pass
            try:
                document = document[part]
            except (TypeError, LookupError):
                raise exceptions.RefResolutionError(
>                   "Unresolvable JSON pointer: %r" % fragment
                )
E               jsonschema.exceptions.RefResolutionError: Unresolvable JSON pointer: 'definitions/Base'

.tox/py37/lib/python3.7/site-packages/jsonschema/validators.py:779: RefResolutionError
====================================================================== warnings summary ======================================================================
.tox/py37/lib/python3.7/site-packages/jinja2/utils.py:485
  /Users/duncan.booth/github/dataclasses-jsonschema/.tox/py37/lib/python3.7/site-packages/jinja2/utils.py:485: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated, and in 3.8 it will stop working
    from collections import MutableMapping

.tox/py37/lib/python3.7/site-packages/jinja2/runtime.py:318
  /Users/duncan.booth/github/dataclasses-jsonschema/.tox/py37/lib/python3.7/site-packages/jinja2/runtime.py:318: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated, and in 3.8 it will stop working
    from collections import Mapping

.tox/py37/lib/python3.7/site-packages/_pytest/mark/structures.py:332
  /Users/duncan.booth/github/dataclasses-jsonschema/.tox/py37/lib/python3.7/site-packages/_pytest/mark/structures.py:332: PytestUnknownMarkWarning: Unknown pytest.mark.last - is this a typo?  You can register custom marks to avoid this warning - for details, see https://docs.pytest.org/en/latest/mark.html
    PytestUnknownMarkWarning,

-- Docs: https://docs.pytest.org/en/latest/warnings.html
====================================================== 1 failed, 27 passed, 3 warnings in 0.42 seconds =======================================================
ERROR: InvocationError for command /Users/duncan.booth/github/dataclasses-jsonschema/.tox/py37/bin/pytest tests (exited with code 1)
__________________________________________________________________________ summary ___________________________________________________________________________
ERROR:   py37: commands failed
```

The code fails when attempting to do `Derived2.from_dict(...)` but only if we have previously called `Derived1.from_dict(...)`. Whichever order I try to validate the data the first one succeeds and the second one fails.

The problem appears to be that the JsonSchemaMixin fields _schema, _compiled_schema and so on are shared between all classes in the same hierarchy. If I change the code so that `Base` no longer inherits from `JsonSchemaMixin` the two derived classes no longer share the mixin's attributes and everything works.
date: 2019-07-11
patch: diff --git a/dataclasses_jsonschema/__init__.py b/dataclasses_jsonschema/__init__.py
index 290b1a1..92bbf2e 100644
--- a/dataclasses_jsonschema/__init__.py
+++ b/dataclasses_jsonschema/__init__.py
@@ -172,13 +172,22 @@ class JsonSchemaMixin:
     }
 
     # Cache of the generated schema
-    _schema: Optional[Dict[SchemaType, JsonDict]] = None
-    _compiled_schema: Optional[Dict[SchemaType, Callable]] = None
-    _definitions: Optional[Dict[SchemaType, JsonDict]] = None
+    _schema: Dict[SchemaType, JsonDict]
+    _compiled_schema: Dict[SchemaType, Callable]
+    _definitions: Dict[SchemaType, JsonDict]
     # Cache of field encode / decode functions
-    _encode_cache: Optional[Dict[Any, _ValueEncoder]] = None
-    _decode_cache: Optional[Dict[Any, _ValueDecoder]] = None
-    _mapped_fields: Optional[List[Tuple[Field, str]]] = None
+    _encode_cache: Dict[Any, _ValueEncoder]
+    _decode_cache: Dict[Any, _ValueDecoder]
+    _mapped_fields: List[Tuple[Field, str]]
+
+    def __init_subclass__(cls):
+        # Initialise caches
+        cls._schema = {}
+        cls._compiled_schema = {}
+        cls._definitions = {}
+        cls._encode_cache = {}
+        cls._decode_cache = {}
+        cls._mapped_fields = []
 
     @classmethod
     def field_mapping(cls) -> Dict[str, str]:
@@ -206,9 +215,6 @@ class JsonSchemaMixin:
         try:
             encoder = cls._encode_cache[field_type]  # type: ignore
         except (KeyError, TypeError):
-            if cls._encode_cache is None:
-                cls._encode_cache = {}
-
             # TODO: Use field_type.__origin__ instead of the type name.
             # This has different behaviour between 3.6 & 3.7 however
             field_type_name = cls._get_field_type_name(field_type)
@@ -283,7 +289,7 @@ class JsonSchemaMixin:
         if not base_fields:
             return _get_fields_uncached()
 
-        if cls._mapped_fields is None:
+        if not cls._mapped_fields:
             cls._mapped_fields = _get_fields_uncached()
         return cls._mapped_fields  # type: ignore
 
@@ -312,8 +318,6 @@ class JsonSchemaMixin:
         try:
             decoder = cls._decode_cache[field_type]  # type: ignore
         except (KeyError, TypeError):
-            if cls._decode_cache is None:
-                cls._decode_cache = {}
             # Note: Only literal types composed of primitive values are currently supported
             if type(value) in JSON_ENCODABLE_TYPES and (field_type in JSON_ENCODABLE_TYPES or is_literal(field_type)):
                 return value
@@ -371,8 +375,6 @@ class JsonSchemaMixin:
     def _validate(cls, data: JsonDict):
         try:
             if fast_validation:
-                if cls._compiled_schema is None:
-                    cls._compiled_schema = {}
                 # TODO: Support validating with other schema types
                 schema_validator = cls._compiled_schema.get(DEFAULT_SCHEMA_TYPE)
                 if schema_validator is None:
@@ -617,9 +619,7 @@ class JsonSchemaMixin:
             return cls.all_json_schemas(schema_type)
 
         definitions: JsonDict = {}
-        if cls._definitions is None:
-            cls._definitions = {schema_type: definitions}
-        elif schema_type not in cls._definitions:
+        if schema_type not in cls._definitions:
             cls._definitions[schema_type] = definitions
         else:
             definitions = cls._definitions[schema_type]
@@ -661,9 +661,6 @@ class JsonSchemaMixin:
             if cls.__doc__:
                 schema['description'] = cls.__doc__
 
-            if cls._schema is None:
-                cls._schema = {}
-
             cls._schema[schema_type] = schema
 
         if embeddable:

test_patch: diff --git a/tests/test_core.py b/tests/test_core.py
index 4e6e817..977660a 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -505,7 +505,12 @@ def test_inherited_schema():
         """A cat"""
         hunting_skill: str
 
-    expected_schema = compose_schema({
+    @dataclass
+    class Dog(Pet):
+        """A dog"""
+        breed: str
+
+    expected_cat_schema = compose_schema({
         "description": "A cat",
         "allOf": [
             {"$ref": "#/definitions/Pet"},
@@ -523,7 +528,26 @@ def test_inherited_schema():
             "required": ["name"]
         }
     })
-    assert Cat.json_schema() == expected_schema
+    expected_dog_schema = compose_schema({
+        "description": "A dog",
+        "allOf": [
+            {"$ref": "#/definitions/Pet"},
+            {
+                "type": "object",
+                "properties": {"breed": {"type": "string"}},
+                "required": ["breed"]
+            },
+        ]
+    }, {
+        "Pet": {
+            "description": "A generic pet",
+            "type": "object",
+            "properties": {"name": {"type": "string"}},
+            "required": ["name"]
+        }
+    })
+    assert Cat.json_schema() == expected_cat_schema
+    assert Dog.json_schema() == expected_dog_schema
 
 
 def test_optional_union():


----

problem_statement: Improve handling of enum fields in `from_object`
We should automatically convert enum fields from the source field by calling the enum constructor.

The enum constructor accepts both enums and enum values.
date: 2019-07-12
patch: diff --git a/dataclasses_jsonschema/__init__.py b/dataclasses_jsonschema/__init__.py
index 92bbf2e..3c01cc8 100644
--- a/dataclasses_jsonschema/__init__.py
+++ b/dataclasses_jsonschema/__init__.py
@@ -409,7 +409,7 @@ class JsonSchemaMixin:
         return instance
 
     @classmethod
-    def from_object(cls: Type[T], obj: Any, exclude: FieldExcludeList = tuple()):
+    def from_object(cls: Type[T], obj: Any, exclude: FieldExcludeList = tuple()) -> T:
         """Returns a dataclass instance from another object (typically an ORM model).
         The `exclude` parameter is a tuple of field names or (field.name, nested_exclude)
         to exclude from the conversion. For example `exclude=('artist_name', ('albums', ('tracks',))` will exclude
@@ -434,6 +434,8 @@ class JsonSchemaMixin:
             field_type_name = cls._get_field_type_name(ft)
             if cls._is_json_schema_subclass(ft):
                 values[field.name] = ft.from_object(getattr(obj, field.name), exclude=sub_exclude)
+            elif is_enum(ft):
+                values[field.name] = ft(getattr(obj, field.name))
             elif field_type_name == "List" and cls._is_json_schema_subclass(ft.__args__[0]):
                 values[field.name] = [
                     ft.__args__[0].from_object(v, exclude=sub_exclude) for v in getattr(obj, field.name)

test_patch: diff --git a/tests/test_core.py b/tests/test_core.py
index 977660a..55cd03b 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -1,5 +1,7 @@
 import datetime
 from _decimal import Decimal
+from enum import Enum
+
 from dataclasses import dataclass, field
 from ipaddress import IPv4Address, IPv6Address
 from typing import List, NewType, Optional, Union
@@ -453,6 +455,11 @@ def test_literal_types():
 
 def test_from_object():
 
+    class Genre(Enum):
+        HORROR = "horror"
+        BIOGRAPHY = "biography"
+        THRILLER = "thriller"
+
     class AuthorModel:
 
         def __init__(self, name, age, books):
@@ -462,15 +469,17 @@ def test_from_object():
 
     class BookModel:
 
-        def __init__(self, name, first_print, publisher):
+        def __init__(self, name, first_print, publisher, genre):
             self.name = name
             self.first_print = first_print
             self.publisher = publisher
+            self.genre = genre
 
     @dataclass
     class Book(JsonSchemaMixin):
         name: str
         publisher: str
+        genre: Genre
         first_print: Optional[datetime.datetime] = None
 
     @dataclass
@@ -479,8 +488,10 @@ def test_from_object():
         age: Optional[int] = None
         books: Optional[List[Book]] = None
 
-    sample_author = AuthorModel("Joe Bloggs", 32, [BookModel("Hello World!", datetime.datetime.utcnow(), "ACME Corp")])
-    expected_author = Author("Joe Bloggs", books=[Book("Hello World!", "ACME Corp")])
+    sample_author = AuthorModel(
+        "Joe Bloggs", 32, [BookModel("Hello World!", datetime.datetime.utcnow(), "ACME Corp", "biography")]
+    )
+    expected_author = Author("Joe Bloggs", books=[Book("Hello World!", "ACME Corp", Genre.BIOGRAPHY)])
     assert Author.from_object(sample_author, exclude=('age', ('books', ('first_print',)))) == expected_author
 
     with pytest.raises(ValueError):