Repository: samuelcolvin/pydantic (python)

Issue:
python 3.9 generic alias Callable from collections module does not behave correctly in generics

# Bug

Output of `python -c "import pydantic.utils; print(pydantic.utils.version_info())"`:
```
             pydantic version: 1.8
            pydantic compiled: False
                 install path: /home/user/Documents/Code/pydantic/pydantic
               python version: 3.9.1 (default, Dec 11 2020, 14:32:07)  [GCC 7.3.0]
                     platform: Linux-5.4.0-66-generic-x86_64-with-glibc2.31
     optional deps. installed: ['devtools', 'dotenv', 'email-validator', 'typing-extensions']
```

```
from typing import Any, Callable, ClassVar, Dict, Generic, List, Literal, Optional, Sequence, Tuple, Type, TypeVar, Union
```

For example, the `test_iter_contained_typevars` test with replacing `typing.Callable` with `collections.abc.Callable`:

```py
from collections.abc import Callable
from typing import Dict, Generic, List, Optional, Union

@skip_36
def test_iter_contained_typevars():
    T = TypeVar('T')
    T2 = TypeVar('T2')

    class Model(GenericModel, Generic[T]):
        a: T

    assert list(iter_contained_typevars(Optional[List[Union[str, Model[T], Callable[[T2, T], str]]]])) == [T, T2, T]
```
results in 

```
----> 1 assert list(iter_contained_typevars(Optional[List[Union[str, Model[T], Callable[[T2, T], str]]]])) == [T, T2, T]

pydantic-39/lib/python3.9/typing.py in inner(*args, **kwds)
    260             except TypeError:
    261                 pass  # All real errors (not unhashable args) are raised below.
--> 262             return func(*args, **kwds)
    263         return inner
    264 

pydantic-39/lib/python3.9/typing.py in __getitem__(self, parameters)
    337     @_tp_cache
    338     def __getitem__(self, parameters):
--> 339         return self._getitem(self, parameters)
    340 
    341 

pydantic-39/lib/python3.9/typing.py in Union(self, parameters)
    449     msg = "Union[arg, ...]: each arg must be a type."
    450     parameters = tuple(_type_check(p, msg) for p in parameters)
--> 451     parameters = _remove_dups_flatten(parameters)
    452     if len(parameters) == 1:
    453         return parameters[0]

pydantic-39/lib/python3.9/typing.py in _remove_dups_flatten(parameters)
    229             params.append(p)
    230 
--> 231     return tuple(_deduplicate(params))
    232 
    233 

pydantic-39/lib/python3.9/typing.py in _deduplicate(params)
    203 def _deduplicate(params):
    204     # Weed out strict duplicates, preserving the first of each occurrence.
--> 205     all_params = set(params)
    206     if len(all_params) < len(params):
    207         new_params = []

TypeError: unhashable type: 'list'
```

This came up while reading the release notes for python 3.10 where this is changed back: https://docs.python.org/3.10/whatsnew/3.10.html#collections-abc 
So in 3.10 this problem won't be present.

However, since I don't think we're testing any of the "collection" module types particularly well, the assumption is that there might be other problems as well.
We should therefore probably add some tests that check these python >= 3.9 features / generic alias cases in case they are missing.

Pull request:
Allow collections.abc.Callable to be used as type in python 3.9
## Change Summary

Allows using python 3.9 generic alias version of `Callable` (`collections.abc.Callable`)

Fixes #2465

## Related issue number

Address #2465

## Checklist

* [x] Unit tests for the changes exist
* [x] Tests pass on CI and coverage remains at 100%
* [x] Documentation reflects the changes where applicable
* [x] `changes/<pull request or issue id>-<github username>.md` file added describing change
  (see [changes/README.md](https://github.com/samuelcolvin/pydantic/blob/master/changes/README.md) for details)


Patch:
diff --git a/changes/2519-daviskirk.md b/changes/2519-daviskirk.md
new file mode 100644
index 000000000..419646e3b
--- /dev/null
+++ b/changes/2519-daviskirk.md
@@ -0,0 +1,1 @@
+Allow `collections.abc.Callable` to be used as type in python 3.9.
diff --git a/pydantic/fields.py b/pydantic/fields.py
index 93887e8f0..e0eb1d8b3 100644
--- a/pydantic/fields.py
+++ b/pydantic/fields.py
@@ -30,7 +30,6 @@ from .error_wrappers import ErrorWrapper
 from .errors import ConfigError, NoneIsNotAllowedError
 from .types import Json, JsonWrapper
 from .typing import (
-    NONE_TYPES,
     Callable,
     ForwardRef,
     NoArgAnyCallable,
@@ -40,6 +39,7 @@ from .typing import (
     get_origin,
     is_literal_type,
     is_new_type,
+    is_none_type,
     is_typeddict,
     is_union_origin,
     new_type_supertype,
@@ -739,7 +739,7 @@ class ModelField(Representation):
                 return v, errors
 
         if v is None:
-            if self.type_ in NONE_TYPES:
+            if is_none_type(self.type_):
                 # keep validating
                 pass
             elif self.allow_none:
diff --git a/pydantic/schema.py b/pydantic/schema.py
index 333b1ce72..d640bc163 100644
--- a/pydantic/schema.py
+++ b/pydantic/schema.py
@@ -63,7 +63,6 @@ from .types import (
     constr,
 )
 from .typing import (
-    NONE_TYPES,
     ForwardRef,
     all_literal_values,
     get_args,
@@ -71,6 +70,7 @@ from .typing import (
     is_callable_type,
     is_literal_type,
     is_namedtuple,
+    is_none_type,
     is_union_origin,
 )
 from .utils import ROOT_KEY, get_model, lenient_issubclass, sequence_like
@@ -788,7 +788,7 @@ def field_singleton_schema(  # noqa: C901 (ignore complexity)
         )
     if field_type is Any or field_type.__class__ == TypeVar:
         return {}, definitions, nested_models  # no restrictions
-    if field_type in NONE_TYPES:
+    if is_none_type(field_type):
         return {'type': 'null'}, definitions, nested_models
     if is_callable_type(field_type):
         raise SkipField(f'Callable {field.name} was excluded from schema since JSON schema has no equivalent type.')
diff --git a/pydantic/typing.py b/pydantic/typing.py
index 7004c7ba3..fff0b89ac 100644
--- a/pydantic/typing.py
+++ b/pydantic/typing.py
@@ -227,7 +227,7 @@ __all__ = (
     'AnyCallable',
     'NoArgAnyCallable',
     'NoneType',
-    'NONE_TYPES',
+    'is_none_type',
     'display_as_type',
     'resolve_annotations',
     'is_callable_type',
@@ -260,7 +260,29 @@ __all__ = (
 
 
 NoneType = None.__class__
-NONE_TYPES: Set[Any] = {None, NoneType, Literal[None]}
+
+
+NONE_TYPES: Tuple[Any, Any, Any] = (None, NoneType, Literal[None])
+
+
+if sys.version_info < (3, 8):  # noqa: C901 (ignore complexity)
+    # Even though this implementation is slower, we need it for python 3.6/3.7:
+    # In python 3.6/3.7 "Literal" is not a builtin type and uses a different
+    # mechanism.
+    # for this reason `Literal[None] is Literal[None]` evaluates to `False`,
+    # breaking the faster implementation used for the other python versions.
+
+    def is_none_type(type_: Any) -> bool:
+        return type_ in NONE_TYPES
+
+
+else:
+
+    def is_none_type(type_: Any) -> bool:
+        for none_type in NONE_TYPES:
+            if type_ is none_type:
+                return True
+        return False
 
 
 def display_as_type(v: Type[Any]) -> str:
diff --git a/pydantic/validators.py b/pydantic/validators.py
index 6d14c537a..82061110e 100644
--- a/pydantic/validators.py
+++ b/pydantic/validators.py
@@ -31,7 +31,6 @@ from typing_extensions import Literal
 from . import errors
 from .datetime_parse import parse_date, parse_datetime, parse_duration, parse_time
 from .typing import (
-    NONE_TYPES,
     AnyCallable,
     ForwardRef,
     all_literal_values,
@@ -40,6 +39,7 @@ from .typing import (
     is_callable_type,
     is_literal_type,
     is_namedtuple,
+    is_none_type,
     is_typeddict,
 )
 from .utils import almost_equal_floats, lenient_issubclass, sequence_like
@@ -657,7 +657,8 @@ def find_validators(  # noqa: C901 (ignore complexity)
     type_type = type_.__class__
     if type_type == ForwardRef or type_type == TypeVar:
         return
-    if type_ in NONE_TYPES:
+
+    if is_none_type(type_):
         yield none_validator
         return
     if type_ is Pattern:


Test patch:
diff --git a/tests/test_callable.py b/tests/test_callable.py
index 037b692e1..cf31241a2 100644
--- a/tests/test_callable.py
+++ b/tests/test_callable.py
@@ -1,11 +1,18 @@
+import sys
 from typing import Callable
 
 import pytest
 
 from pydantic import BaseModel, ValidationError
 
+collection_callable_types = [Callable, Callable[[int], int]]
+if sys.version_info >= (3, 9):
+    from collections.abc import Callable as CollectionsCallable
 
-@pytest.mark.parametrize('annotation', [Callable, Callable[[int], int]])
+    collection_callable_types += [CollectionsCallable, CollectionsCallable[[int], int]]
+
+
+@pytest.mark.parametrize('annotation', collection_callable_types)
 def test_callable(annotation):
     class Model(BaseModel):
         callback: annotation
@@ -14,7 +21,7 @@ def test_callable(annotation):
     assert callable(m.callback)
 
 
-@pytest.mark.parametrize('annotation', [Callable, Callable[[int], int]])
+@pytest.mark.parametrize('annotation', collection_callable_types)
 def test_non_callable(annotation):
     class Model(BaseModel):
         callback: annotation
diff --git a/tests/test_typing.py b/tests/test_typing.py
index d0d99125e..af0b8d969 100644
--- a/tests/test_typing.py
+++ b/tests/test_typing.py
@@ -1,9 +1,9 @@
 from collections import namedtuple
-from typing import NamedTuple
+from typing import Callable as TypingCallable, NamedTuple
 
 import pytest
 
-from pydantic.typing import is_namedtuple, is_typeddict
+from pydantic.typing import Literal, is_namedtuple, is_none_type, is_typeddict
 
 try:
     from typing import TypedDict as typing_TypedDict
@@ -54,3 +54,15 @@ def test_is_typeddict_typing(TypedDict):
         id: int
 
     assert is_typeddict(Other) is False
+
+
+def test_is_none_type():
+    assert is_none_type(Literal[None]) is True
+    assert is_none_type(None) is True
+    assert is_none_type(type(None)) is True
+    assert is_none_type(6) is False
+    assert is_none_type({}) is False
+    # WARNING: It's important to test `typing.Callable` not
+    # `collections.abc.Callable` (even with python >= 3.9) as they behave
+    # differently
+    assert is_none_type(TypingCallable) is False