Repository: samuelcolvin/pydantic
Pull request #3542 (2021-12-19 17:23:00)

fix: keep old behaviour of `json()` by default
<!-- Thank you for your contribution! -->
<!-- Unless your change is trivial, please create an issue to discuss the change before creating a PR -->
<!-- See https://pydantic-docs.helpmanual.io/contributing/ for help on Contributing -->

## Change Summary
fix regression on 1.9 introduced in #2650 (see https://github.com/samuelcolvin/pydantic/discussions/3538#discussioncomment-1843005)

<!-- Please give a short summary of the changes. -->

## Related issue number

<!-- Are there any issues opened that will be resolved by merging this change? -->
<!-- WARNING: please use "fix #123" style references so the issue is closed when this PR is merged. -->

## Checklist

* [x] Unit tests for the changes exist
* [x] Tests pass on CI and coverage remains at 100%
* [ ] ~Documentation reflects the changes where applicable~
* [ ] ~`changes/<pull request or issue id>-<github username>.md` file added describing change~ (I didn't add a change file since it's due to a new feature of 1.9)
~(see [changes/README.md](https://github.com/samuelcolvin/pydantic/blob/master/changes/README.md) for details)~
* [x] My PR is ready to review, **please add a comment including the phrase "please review" to assign reviewers**


PrettyWood: please review
layday: I'm not sure if this is a good workaround. IIUC the (broader) issue with #2650 is that the transformation is no longer recursive. This will manifest in other ways (and it already has: #3544 :) It'd probably make more sense to expose `to_dict` in `json`, but adding more options always comes at a cost.
layday: If you don't want to delay v1.9 then reverting #2650 might be the best course of action for the time being?
PrettyWood: Yeah I saw new issues pop up. I will take some time tomorrow to check properly. But if no easy workaround is possible, I'll revert #2650 
samuelcolvin: @PrettyWood did you have time to think about this?

I've haven't dug into it in detail, but I will try to find time soon.

Patch:
diff --git a/docs/examples/exporting_models_json_forward_ref.py b/docs/examples/exporting_models_json_forward_ref.py
index 540221f3a..0ae9eb947 100644
--- a/docs/examples/exporting_models_json_forward_ref.py
+++ b/docs/examples/exporting_models_json_forward_ref.py
@@ -31,4 +31,4 @@ wolfgang = User(
         User(name='John', address=Address(city='London', country='UK')),
     ],
 )
-print(wolfgang.json())
+print(wolfgang.json(models_as_dict=False))
diff --git a/docs/examples/schema_ad_hoc.py b/docs/examples/schema_ad_hoc.py
index 6e93f913d..10862e162 100644
--- a/docs/examples/schema_ad_hoc.py
+++ b/docs/examples/schema_ad_hoc.py
@@ -2,7 +2,7 @@ from typing import Literal, Union
 
 from typing_extensions import Annotated
 
-from pydantic import BaseModel, Field, schema_json
+from pydantic import BaseModel, Field, schema_json_of
 
 
 class Cat(BaseModel):
@@ -17,4 +17,4 @@ class Dog(BaseModel):
 
 Pet = Annotated[Union[Cat, Dog], Field(discriminator='pet_type')]
 
-print(schema_json(Pet, title='The Pet Schema', indent=2))
+print(schema_json_of(Pet, title='The Pet Schema', indent=2))
diff --git a/docs/requirements.txt b/docs/requirements.txt
index 2bc1b7dc1..88f06ea3f 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -1,12 +1,12 @@
 ansi2html==1.6.0
 flake8==4.0.1
 flake8-quotes==3.3.1
-hypothesis==6.31.4
+hypothesis==6.31.6
 markdown-include==0.6.0
 mdx-truly-sane-lists==1.2
 mkdocs==1.2.3
 mkdocs-exclude==1.0.2
-mkdocs-material==8.1.1
+mkdocs-material==8.1.3
 sqlalchemy
 orjson
 ujson
diff --git a/docs/usage/exporting_models.md b/docs/usage/exporting_models.md
index f5770daa4..1673c45d5 100644
--- a/docs/usage/exporting_models.md
+++ b/docs/usage/exporting_models.md
@@ -109,6 +109,9 @@ _(This script is complete, it should run "as is")_
 
 ### Serialising self-reference or other models
 
+By default, models are serialised as dictionaries.
+If you want to serialise them differently, you can add `models_as_dict=False` when calling `json()` method
+and add the classes of the model in `json_encoders`.
 In case of forward references, you can use a string with the class name instead of the class itself
 ```py
 {!.tmp_examples/exporting_models_json_forward_ref.py!}
diff --git a/docs/usage/schema.md b/docs/usage/schema.md
index 9928b2782..458014a41 100644
--- a/docs/usage/schema.md
+++ b/docs/usage/schema.md
@@ -39,7 +39,7 @@ will be replaced with the model naming using `str.format()`.
 
 ## Getting schema of a specified type
 
-_Pydantic_ includes two standalone utility functions `schema` and `schema_json` that can be used to
+_Pydantic_ includes two standalone utility functions `schema_of` and `schema_json_of` that can be used to
 apply the schema generation logic used for _pydantic_ models in a more ad-hoc way.
 These functions behave similarly to `BaseModel.schema` and `BaseModel.schema_json`,
 but work with arbitrary pydantic-compatible types.
diff --git a/pydantic/__init__.py b/pydantic/__init__.py
index 0443acef6..982ea4755 100644
--- a/pydantic/__init__.py
+++ b/pydantic/__init__.py
@@ -67,8 +67,8 @@ __all__ = [
     'parse_file_as',
     'parse_obj_as',
     'parse_raw_as',
-    'schema',
-    'schema_json',
+    'schema_of',
+    'schema_json_of',
     # types
     'NoneStr',
     'NoneBytes',
diff --git a/pydantic/dataclasses.py b/pydantic/dataclasses.py
index f353de2e2..12d4c588a 100644
--- a/pydantic/dataclasses.py
+++ b/pydantic/dataclasses.py
@@ -169,9 +169,8 @@ def _process_class(
 
         if field.default is not dataclasses.MISSING:
             default = field.default
-        # mypy issue 7020 and 708
-        elif field.default_factory is not dataclasses.MISSING:  # type: ignore
-            default_factory = field.default_factory  # type: ignore
+        elif field.default_factory is not dataclasses.MISSING:
+            default_factory = field.default_factory
         else:
             default = Required
 
diff --git a/pydantic/datetime_parse.py b/pydantic/datetime_parse.py
index e4dff67d5..024a899d0 100644
--- a/pydantic/datetime_parse.py
+++ b/pydantic/datetime_parse.py
@@ -245,4 +245,4 @@ def parse_duration(value: StrBytesIntFloat) -> timedelta:
 
     kw_ = {k: float(v) for k, v in kw.items() if v is not None}
 
-    return sign * timedelta(**kw_)  # type: ignore
+    return sign * timedelta(**kw_)
diff --git a/pydantic/fields.py b/pydantic/fields.py
index ff7b962f7..44a3aa10b 100644
--- a/pydantic/fields.py
+++ b/pydantic/fields.py
@@ -50,6 +50,7 @@ from .utils import (
     ValueItems,
     get_discriminator_alias_and_values,
     get_unique_discriminator_alias,
+    lenient_isinstance,
     lenient_issubclass,
     sequence_like,
     smart_deepcopy,
@@ -1048,7 +1049,7 @@ class ModelField(Representation):
                             return v, None
                     except TypeError:
                         # compound type
-                        if isinstance(v, get_origin(field.outer_type_)):
+                        if lenient_isinstance(v, get_origin(field.outer_type_)):
                             value, error = field.validate(v, values, loc=loc, cls=cls)
                             if not error:
                                 return value, None
diff --git a/pydantic/generics.py b/pydantic/generics.py
index 1fee636c3..a712d26f2 100644
--- a/pydantic/generics.py
+++ b/pydantic/generics.py
@@ -258,6 +258,7 @@ def replace_types(type_: Any, type_map: Mapping[Any, Any]) -> Any:
             # `type` or `collections.abc.Callable` need to be translated.
             # See: https://www.python.org/dev/peps/pep-0585
             origin_type = getattr(typing, type_._name)
+        assert origin_type is not None
         return origin_type[resolved_type_args]
 
     # We handle pydantic generic models separately as they don't have the same
diff --git a/pydantic/main.py b/pydantic/main.py
index fa501701f..a00cc0a3e 100644
--- a/pydantic/main.py
+++ b/pydantic/main.py
@@ -454,6 +454,7 @@ class BaseModel(Representation, metaclass=ModelMetaclass):
         exclude_defaults: bool = False,
         exclude_none: bool = False,
         encoder: Optional[Callable[[Any], Any]] = None,
+        models_as_dict: bool = True,
         **dumps_kwargs: Any,
     ) -> str:
         """
@@ -469,11 +470,12 @@ class BaseModel(Representation, metaclass=ModelMetaclass):
             exclude_unset = skip_defaults
         encoder = cast(Callable[[Any], Any], encoder or self.__json_encoder__)
 
-        # We don't directly call `self.dict()`, which does exactly the same thing but
-        # with `to_dict = True` because we want to keep raw `BaseModel` instances and not as `dict`.
+        # We don't directly call `self.dict()`, which does exactly this with `to_dict=True`
+        # because we want to be able to keep raw `BaseModel` instances and not as `dict`.
         # This allows users to write custom JSON encoders for given `BaseModel` classes.
         data = dict(
             self._iter(
+                to_dict=models_as_dict,
                 by_alias=by_alias,
                 include=include,
                 exclude=exclude,
diff --git a/pydantic/mypy.py b/pydantic/mypy.py
index c2120992f..d8ad93db1 100644
--- a/pydantic/mypy.py
+++ b/pydantic/mypy.py
@@ -15,13 +15,6 @@ except ImportError:  # pragma: no cover
         warnings.warn('No TOML parser installed, cannot read configuration from `pyproject.toml`.')
         toml = None  # type: ignore
 
-try:
-    from mypy.types import TypeVarDef
-except ImportError:  # pragma: no cover
-    # Backward-compatible with TypeVarDef from Mypy 0.910.
-    from mypy.types import TypeVarType as TypeVarDef  # type: ignore[misc]
-
-
 from mypy.errorcodes import ErrorCode
 from mypy.nodes import (
     ARG_NAMED,
@@ -67,6 +60,7 @@ from mypy.types import (
     Type,
     TypeOfAny,
     TypeType,
+    TypeVarLikeType,
     TypeVarType,
     UnionType,
     get_proper_type,
@@ -363,23 +357,17 @@ class PydanticModelTransformer:
         obj_type = ctx.api.named_type('__builtins__.object')
         self_tvar_name = '_PydanticBaseModel'  # Make sure it does not conflict with other names in the class
         tvar_fullname = ctx.cls.fullname + '.' + self_tvar_name
-        tvd = TypeVarDef(self_tvar_name, tvar_fullname, -1, [], obj_type)
+        self_type = TypeVarType(self_tvar_name, tvar_fullname, -1, [], obj_type)
         self_tvar_expr = TypeVarExpr(self_tvar_name, tvar_fullname, [], obj_type)
         ctx.cls.info.names[self_tvar_name] = SymbolTableNode(MDEF, self_tvar_expr)
 
-        # Backward-compatible with TypeVarDef from Mypy 0.910.
-        if isinstance(tvd, TypeVarType):  # pragma: no cover
-            self_type = tvd
-        else:
-            self_type = TypeVarType(tvd)
-
         add_method(
             ctx,
             'construct',
             construct_arguments,
             return_type=self_type,
             self_type=self_type,
-            tvar_def=tvd,
+            tvar_like_type=self_type,
             is_classmethod=True,
         )
 
@@ -631,7 +619,7 @@ def add_method(
     args: List[Argument],
     return_type: Type,
     self_type: Optional[Type] = None,
-    tvar_def: Optional[TypeVarDef] = None,
+    tvar_like_type: Optional[TypeVarLikeType] = None,
     is_classmethod: bool = False,
     is_new: bool = False,
     # is_staticmethod: bool = False,
@@ -668,8 +656,8 @@ def add_method(
 
     function_type = ctx.api.named_type('__builtins__.function')
     signature = CallableType(arg_types, arg_kinds, arg_names, return_type, function_type)
-    if tvar_def:
-        signature.variables = [tvar_def]
+    if tvar_like_type:
+        signature.variables = [tvar_like_type]
 
     func = FuncDef(name, args, Block([PassStmt()]))
     func.info = info
diff --git a/pydantic/networks.py b/pydantic/networks.py
index 53b4643b0..18a042ca9 100644
--- a/pydantic/networks.py
+++ b/pydantic/networks.py
@@ -285,6 +285,7 @@ class AnyUrl(str):
             tld = d.group('tld')
             if tld is None and not is_international:
                 d = int_domain_regex().fullmatch(host)
+                assert d is not None
                 tld = d.group('tld')
                 is_international = True
 
diff --git a/pydantic/tools.py b/pydantic/tools.py
index f1d651cd4..9cdb4538e 100644
--- a/pydantic/tools.py
+++ b/pydantic/tools.py
@@ -7,7 +7,7 @@ from .parse import Protocol, load_file, load_str_bytes
 from .types import StrBytes
 from .typing import display_as_type
 
-__all__ = ('parse_file_as', 'parse_obj_as', 'parse_raw_as', 'schema', 'schema_json')
+__all__ = ('parse_file_as', 'parse_obj_as', 'parse_raw_as', 'schema_of', 'schema_json_of')
 
 NameFactory = Union[str, Callable[[Type[Any]], str]]
 
@@ -82,11 +82,11 @@ def parse_raw_as(
     return parse_obj_as(type_, obj, type_name=type_name)
 
 
-def schema(type_: Any, *, title: Optional[NameFactory] = None, **schema_kwargs: Any) -> 'DictStrAny':
+def schema_of(type_: Any, *, title: Optional[NameFactory] = None, **schema_kwargs: Any) -> 'DictStrAny':
     """Generate a JSON schema (as dict) for the passed model or dynamically generated one"""
     return _get_parsing_type(type_, type_name=title).schema(**schema_kwargs)
 
 
-def schema_json(type_: Any, *, title: Optional[NameFactory] = None, **schema_json_kwargs: Any) -> str:
+def schema_json_of(type_: Any, *, title: Optional[NameFactory] = None, **schema_json_kwargs: Any) -> str:
     """Generate a JSON schema (as JSON) for the passed model or dynamically generated one"""
     return _get_parsing_type(type_, type_name=title).schema_json(**schema_json_kwargs)
diff --git a/pydantic/typing.py b/pydantic/typing.py
index b2742bf09..c00d504aa 100644
--- a/pydantic/typing.py
+++ b/pydantic/typing.py
@@ -110,7 +110,7 @@ if sys.version_info < (3, 8):
 else:
     from typing import get_origin as _typing_get_origin
 
-    def get_origin(tp: Type[Any]) -> Type[Any]:
+    def get_origin(tp: Type[Any]) -> Optional[Type[Any]]:
         """
         We can't directly use `typing.get_origin` since we need a fallback to support
         custom generic classes like `ConstrainedList`
@@ -188,7 +188,7 @@ else:
 
 if sys.version_info < (3, 10):
 
-    def is_union(tp: Type[Any]) -> bool:
+    def is_union(tp: Optional[Type[Any]]) -> bool:
         return tp is Union
 
     WithArgsTypes = (TypingGenericAlias,)
@@ -197,7 +197,7 @@ else:
     import types
     import typing
 
-    def is_union(tp: Type[Any]) -> bool:
+    def is_union(tp: Optional[Type[Any]]) -> bool:
         return tp is Union or tp is types.UnionType  # noqa: E721
 
     WithArgsTypes = (typing._GenericAlias, types.GenericAlias, types.UnionType)
diff --git a/pydantic/utils.py b/pydantic/utils.py
index ec94f76c7..f0a594c88 100644
--- a/pydantic/utils.py
+++ b/pydantic/utils.py
@@ -53,6 +53,7 @@ __all__ = (
     'import_string',
     'sequence_like',
     'validate_field_name',
+    'lenient_isinstance',
     'lenient_issubclass',
     'in_ipython',
     'deep_update',
@@ -163,9 +164,16 @@ def validate_field_name(bases: List[Type['BaseModel']], field_name: str) -> None
             )
 
 
-def lenient_issubclass(cls: Any, class_or_tuple: Union[Type[Any], Tuple[Type[Any], ...]]) -> bool:
+def lenient_isinstance(o: Any, class_or_tuple: Union[Type[Any], Tuple[Type[Any], ...], None]) -> bool:
     try:
-        return isinstance(cls, type) and issubclass(cls, class_or_tuple)
+        return isinstance(o, class_or_tuple)  # type: ignore[arg-type]
+    except TypeError:
+        return False
+
+
+def lenient_issubclass(cls: Any, class_or_tuple: Union[Type[Any], Tuple[Type[Any], ...], None]) -> bool:
+    try:
+        return isinstance(cls, type) and issubclass(cls, class_or_tuple)  # type: ignore[arg-type]
     except TypeError:
         if isinstance(cls, WithArgsTypes):
             return False
diff --git a/requirements.txt b/requirements.txt
index 5a2b15ce9..95aef0852 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,6 @@
 # requirements for compilation and from setup.py so dependabot prompts us to test with latest version of these packages
 
-Cython==0.29.25;sys_platform!='win32'
+Cython==0.29.26;sys_platform!='win32'
 devtools==0.8.0
 email-validator==1.1.3
 dataclasses==0.6; python_version < '3.7'


Test patch:
diff --git a/tests/requirements-linting.txt b/tests/requirements-linting.txt
index 81614a46e..259fa7679 100644
--- a/tests/requirements-linting.txt
+++ b/tests/requirements-linting.txt
@@ -1,9 +1,9 @@
 black==21.12b0
 flake8==4.0.1
 flake8-quotes==3.3.1
-hypothesis==6.31.4
+hypothesis==6.31.6
 isort==5.10.1
-mypy==0.910
+mypy==0.920
 pre-commit==2.16.0
 pycodestyle==2.8.0
 pyflakes==2.4.0
diff --git a/tests/requirements-testing.txt b/tests/requirements-testing.txt
index faa291edf..892dba137 100644
--- a/tests/requirements-testing.txt
+++ b/tests/requirements-testing.txt
@@ -1,8 +1,8 @@
 coverage==6.2
-hypothesis==6.31.4
+hypothesis==6.31.6
 # pin importlib-metadata as upper versions need typing-extensions to work if on python < 3.8
 importlib-metadata==3.1.0;python_version<"3.8"
-mypy==0.910
+mypy==0.920
 pytest==6.2.5
 pytest-cov==3.0.0
 pytest-mock==3.6.1
diff --git a/tests/test_json.py b/tests/test_json.py
index 9ef389283..5871446b7 100644
--- a/tests/test_json.py
+++ b/tests/test_json.py
@@ -281,7 +281,7 @@ def test_custom_decode_encode():
     assert m.json() == '{\n  "a": 1,\n  "b": "foo"\n}'
 
 
-def test_json_nested_encode():
+def test_json_nested_encode_models():
     class Phone(BaseModel):
         manufacturer: str
         number: int
@@ -314,8 +314,58 @@ def test_json_nested_encode():
 
     timon.friend = pumbaa
 
-    assert iphone.json() == '{"manufacturer": "Apple", "number": 18002752273}'
+    assert iphone.json(models_as_dict=False) == '{"manufacturer": "Apple", "number": 18002752273}'
     assert (
-        pumbaa.json() == '{"name": "Pumbaa", "SSN": 234, "birthday": 737424000.0, "phone": 18007267864, "friend": null}'
+        pumbaa.json(models_as_dict=False)
+        == '{"name": "Pumbaa", "SSN": 234, "birthday": 737424000.0, "phone": 18007267864, "friend": null}'
     )
-    assert timon.json() == '{"name": "Timon", "SSN": 123, "birthday": 738892800.0, "phone": 18002752273, "friend": 234}'
+    assert (
+        timon.json(models_as_dict=False)
+        == '{"name": "Timon", "SSN": 123, "birthday": 738892800.0, "phone": 18002752273, "friend": 234}'
+    )
+
+
+def test_custom_encode_fallback_basemodel():
+    class MyExoticType:
+        pass
+
+    def custom_encoder(o):
+        if isinstance(o, MyExoticType):
+            return 'exo'
+        raise TypeError('not serialisable')
+
+    class Foo(BaseModel):
+        x: MyExoticType
+
+        class Config:
+            arbitrary_types_allowed = True
+
+    class Bar(BaseModel):
+        foo: Foo
+
+    assert Bar(foo=Foo(x=MyExoticType())).json(encoder=custom_encoder) == '{"foo": {"x": "exo"}}'
+
+
+def test_custom_encode_error():
+    class MyExoticType:
+        pass
+
+    def custom_encoder(o):
+        raise TypeError('not serialisable')
+
+    class Foo(BaseModel):
+        x: MyExoticType
+
+        class Config:
+            arbitrary_types_allowed = True
+
+    with pytest.raises(TypeError, match='not serialisable'):
+        Foo(x=MyExoticType()).json(encoder=custom_encoder)
+
+
+def test_recursive():
+    class Model(BaseModel):
+        value: Optional[str]
+        nested: Optional[BaseModel]
+
+    assert Model(value=None, nested=Model(value=None)).json(exclude_none=True) == '{"nested": {}}'
diff --git a/tests/test_tools.py b/tests/test_tools.py
index 97784d552..689b32d4c 100644
--- a/tests/test_tools.py
+++ b/tests/test_tools.py
@@ -5,7 +5,7 @@ import pytest
 
 from pydantic import BaseModel, ValidationError
 from pydantic.dataclasses import dataclass
-from pydantic.tools import parse_file_as, parse_obj_as, parse_raw_as, schema, schema_json
+from pydantic.tools import parse_file_as, parse_obj_as, parse_raw_as, schema_json_of, schema_of
 
 
 @pytest.mark.parametrize('obj,type_,parsed', [('1', int, 1), (['1'], List[int], [1])])
@@ -101,11 +101,11 @@ def test_raw_as():
 
 
 def test_schema():
-    assert schema(Union[int, str], title='IntOrStr') == {
+    assert schema_of(Union[int, str], title='IntOrStr') == {
         'title': 'IntOrStr',
         'anyOf': [{'type': 'integer'}, {'type': 'string'}],
     }
-    assert schema_json(Union[int, str], title='IntOrStr', indent=2) == (
+    assert schema_json_of(Union[int, str], title='IntOrStr', indent=2) == (
         '{\n'
         '  "title": "IntOrStr",\n'
         '  "anyOf": [\n'
diff --git a/tests/test_types.py b/tests/test_types.py
index 2a3161316..bbf4c23b1 100644
--- a/tests/test_types.py
+++ b/tests/test_types.py
@@ -27,7 +27,7 @@ from typing import (
 from uuid import UUID
 
 import pytest
-from typing_extensions import Literal
+from typing_extensions import Literal, TypedDict
 
 from pydantic import (
     UUID1,
@@ -3120,6 +3120,22 @@ def test_smart_union_compouned_types_edge_case():
     assert Model(x=[1, '2']).x == ['1', '2']
 
 
+def test_smart_union_typeddict():
+    class Dict1(TypedDict):
+        foo: str
+
+    class Dict2(TypedDict):
+        bar: str
+
+    class M(BaseModel):
+        d: Union[Dict2, Dict1]
+
+        class Config:
+            smart_union = True
+
+    assert M(d=dict(foo='baz')).d == {'foo': 'baz'}
+
+
 @pytest.mark.parametrize(
     'value,result',
     (