Repository: pydantic/pydantic

Issue:
python 3.9 generic alias Callable from collections module does not behave correctly in generics

# Bug

Output of `python -c "import pydantic.utils; print(pydantic.utils.version_info())"`:
```
             pydantic version: 1.8
            pydantic compiled: False
                 install path: /home/user/Documents/Code/pydantic/pydantic
               python version: 3.9.1 (default, Dec 11 2020, 14:32:07)  [GCC 7.3.0]
                     platform: Linux-5.4.0-66-generic-x86_64-with-glibc2.31
     optional deps. installed: ['devtools', 'dotenv', 'email-validator', 'typing-extensions']
```

```
from typing import Any, Callable, ClassVar, Dict, Generic, List, Literal, Optional, Sequence, Tuple, Type, TypeVar, Union
```

For example, the `test_iter_contained_typevars` test with replacing `typing.Callable` with `collections.abc.Callable`:

```py
from collections.abc import Callable
from typing import Dict, Generic, List, Optional, Union

@skip_36
def test_iter_contained_typevars():
    T = TypeVar('T')
    T2 = TypeVar('T2')

    class Model(GenericModel, Generic[T]):
        a: T

    assert list(iter_contained_typevars(Optional[List[Union[str, Model[T], Callable[[T2, T], str]]]])) == [T, T2, T]
```
results in 

```
----> 1 assert list(iter_contained_typevars(Optional[List[Union[str, Model[T], Callable[[T2, T], str]]]])) == [T, T2, T]

pydantic-39/lib/python3.9/typing.py in inner(*args, **kwds)
    260             except TypeError:
    261                 pass  # All real errors (not unhashable args) are raised below.
--> 262             return func(*args, **kwds)
    263         return inner
    264 

pydantic-39/lib/python3.9/typing.py in __getitem__(self, parameters)
    337     @_tp_cache
    338     def __getitem__(self, parameters):
--> 339         return self._getitem(self, parameters)
    340 
    341 

pydantic-39/lib/python3.9/typing.py in Union(self, parameters)
    449     msg = "Union[arg, ...]: each arg must be a type."
    450     parameters = tuple(_type_check(p, msg) for p in parameters)
--> 451     parameters = _remove_dups_flatten(parameters)
    452     if len(parameters) == 1:
    453         return parameters[0]

pydantic-39/lib/python3.9/typing.py in _remove_dups_flatten(parameters)
    229             params.append(p)
    230 
--> 231     return tuple(_deduplicate(params))
    232 
    233 

pydantic-39/lib/python3.9/typing.py in _deduplicate(params)
    203 def _deduplicate(params):
    204     # Weed out strict duplicates, preserving the first of each occurrence.
--> 205     all_params = set(params)
    206     if len(all_params) < len(params):
    207         new_params = []

TypeError: unhashable type: 'list'
```

This came up while reading the release notes for python 3.10 where this is changed back: https://docs.python.org/3.10/whatsnew/3.10.html#collections-abc 
So in 3.10 this problem won't be present.

However, since I don't think we're testing any of the "collection" module types particularly well, the assumption is that there might be other problems as well.
We should therefore probably add some tests that check these python >= 3.9 features / generic alias cases in case they are missing.


Comments:
I can probably look at this this weekend... I don't think many people are using generics in 3.9 yet
Yes you're right but supporting generic types seems to be rational, According to [PEP585](https://www.python.org/dev/peps/pep-0585/) legacy typing.List , typing.Dict , ... are **Deprecated** since python3.9 and will be removed from python.

Patch:
diff --git a/pydantic/fields.py b/pydantic/fields.py
--- a/pydantic/fields.py
+++ b/pydantic/fields.py
@@ -30,7 +30,6 @@
 from .errors import ConfigError, NoneIsNotAllowedError
 from .types import Json, JsonWrapper
 from .typing import (
-    NONE_TYPES,
     Callable,
     ForwardRef,
     NoArgAnyCallable,
@@ -40,6 +39,7 @@
     get_origin,
     is_literal_type,
     is_new_type,
+    is_none_type,
     is_typeddict,
     is_union_origin,
     new_type_supertype,
@@ -739,7 +739,7 @@ def validate(
                 return v, errors
 
         if v is None:
-            if self.type_ in NONE_TYPES:
+            if is_none_type(self.type_):
                 # keep validating
                 pass
             elif self.allow_none:
diff --git a/pydantic/schema.py b/pydantic/schema.py
--- a/pydantic/schema.py
+++ b/pydantic/schema.py
@@ -63,7 +63,6 @@
     constr,
 )
 from .typing import (
-    NONE_TYPES,
     ForwardRef,
     all_literal_values,
     get_args,
@@ -71,6 +70,7 @@
     is_callable_type,
     is_literal_type,
     is_namedtuple,
+    is_none_type,
     is_union_origin,
 )
 from .utils import ROOT_KEY, get_model, lenient_issubclass, sequence_like
@@ -788,7 +788,7 @@ def field_singleton_schema(  # noqa: C901 (ignore complexity)
         )
     if field_type is Any or field_type.__class__ == TypeVar:
         return {}, definitions, nested_models  # no restrictions
-    if field_type in NONE_TYPES:
+    if is_none_type(field_type):
         return {'type': 'null'}, definitions, nested_models
     if is_callable_type(field_type):
         raise SkipField(f'Callable {field.name} was excluded from schema since JSON schema has no equivalent type.')
diff --git a/pydantic/typing.py b/pydantic/typing.py
--- a/pydantic/typing.py
+++ b/pydantic/typing.py
@@ -227,7 +227,7 @@ def is_union_origin(origin: Type[Any]) -> bool:
     'AnyCallable',
     'NoArgAnyCallable',
     'NoneType',
-    'NONE_TYPES',
+    'is_none_type',
     'display_as_type',
     'resolve_annotations',
     'is_callable_type',
@@ -260,7 +260,29 @@ def is_union_origin(origin: Type[Any]) -> bool:
 
 
 NoneType = None.__class__
-NONE_TYPES: Set[Any] = {None, NoneType, Literal[None]}
+
+
+NONE_TYPES: Tuple[Any, Any, Any] = (None, NoneType, Literal[None])
+
+
+if sys.version_info < (3, 8):  # noqa: C901 (ignore complexity)
+    # Even though this implementation is slower, we need it for python 3.6/3.7:
+    # In python 3.6/3.7 "Literal" is not a builtin type and uses a different
+    # mechanism.
+    # for this reason `Literal[None] is Literal[None]` evaluates to `False`,
+    # breaking the faster implementation used for the other python versions.
+
+    def is_none_type(type_: Any) -> bool:
+        return type_ in NONE_TYPES
+
+
+else:
+
+    def is_none_type(type_: Any) -> bool:
+        for none_type in NONE_TYPES:
+            if type_ is none_type:
+                return True
+        return False
 
 
 def display_as_type(v: Type[Any]) -> str:
diff --git a/pydantic/validators.py b/pydantic/validators.py
--- a/pydantic/validators.py
+++ b/pydantic/validators.py
@@ -31,7 +31,6 @@
 from . import errors
 from .datetime_parse import parse_date, parse_datetime, parse_duration, parse_time
 from .typing import (
-    NONE_TYPES,
     AnyCallable,
     ForwardRef,
     all_literal_values,
@@ -40,6 +39,7 @@
     is_callable_type,
     is_literal_type,
     is_namedtuple,
+    is_none_type,
     is_typeddict,
 )
 from .utils import almost_equal_floats, lenient_issubclass, sequence_like
@@ -657,7 +657,8 @@ def find_validators(  # noqa: C901 (ignore complexity)
     type_type = type_.__class__
     if type_type == ForwardRef or type_type == TypeVar:
         return
-    if type_ in NONE_TYPES:
+
+    if is_none_type(type_):
         yield none_validator
         return
     if type_ is Pattern:


Test patch:
diff --git a/tests/test_callable.py b/tests/test_callable.py
--- a/tests/test_callable.py
+++ b/tests/test_callable.py
@@ -1,11 +1,18 @@
+import sys
 from typing import Callable
 
 import pytest
 
 from pydantic import BaseModel, ValidationError
 
+collection_callable_types = [Callable, Callable[[int], int]]
+if sys.version_info >= (3, 9):
+    from collections.abc import Callable as CollectionsCallable
 
-@pytest.mark.parametrize('annotation', [Callable, Callable[[int], int]])
+    collection_callable_types += [CollectionsCallable, CollectionsCallable[[int], int]]
+
+
+@pytest.mark.parametrize('annotation', collection_callable_types)
 def test_callable(annotation):
     class Model(BaseModel):
         callback: annotation
@@ -14,7 +21,7 @@ class Model(BaseModel):
     assert callable(m.callback)
 
 
-@pytest.mark.parametrize('annotation', [Callable, Callable[[int], int]])
+@pytest.mark.parametrize('annotation', collection_callable_types)
 def test_non_callable(annotation):
     class Model(BaseModel):
         callback: annotation
diff --git a/tests/test_typing.py b/tests/test_typing.py
--- a/tests/test_typing.py
+++ b/tests/test_typing.py
@@ -1,9 +1,9 @@
 from collections import namedtuple
-from typing import NamedTuple
+from typing import Callable as TypingCallable, NamedTuple
 
 import pytest
 
-from pydantic.typing import is_namedtuple, is_typeddict
+from pydantic.typing import Literal, is_namedtuple, is_none_type, is_typeddict
 
 try:
     from typing import TypedDict as typing_TypedDict
@@ -54,3 +54,15 @@ class Other(dict):
         id: int
 
     assert is_typeddict(Other) is False
+
+
+def test_is_none_type():
+    assert is_none_type(Literal[None]) is True
+    assert is_none_type(None) is True
+    assert is_none_type(type(None)) is True
+    assert is_none_type(6) is False
+    assert is_none_type({}) is False
+    # WARNING: It's important to test `typing.Callable` not
+    # `collections.abc.Callable` (even with python >= 3.9) as they behave
+    # differently
+    assert is_none_type(TypingCallable) is False