Repository: pydantic/pydantic

Issue:
Error in mypy plugin reading config from pyproject.toml with mypy >= 0.920
### Checks

* [x] I added a descriptive title to this issue
* [x] I have searched (google, github) for similar issues and couldn't find anything
* [x] I have read and followed [the docs](https://pydantic-docs.helpmanual.io/) and still think this is a bug

<!-- Sorry to sound so draconian, but every second saved replying to issues is time spend improving pydantic :-) -->

# Bug

Output of `python -c "import pydantic.utils; print(pydantic.utils.version_info())"`:
```
             pydantic version: 1.9.0a2
            pydantic compiled: True
                 install path: /Users/paul/.cache/pre-commit/repo8u79aah5/py_env-python3.9/lib/python3.9/site-packages/pydantic
               python version: 3.9.6 (default, Aug 18 2021, 18:03:48)  [Clang 11.0.3 (clang-1103.0.32.62)]
                     platform: macOS-10.16-x86_64-i386-64bit
     optional deps. installed: ['typing-extensions']
```
<!-- or if you're using pydantic prior to v1.3, manually include: OS, python version and pydantic version -->

<!-- Please read the [docs](https://pydantic-docs.helpmanual.io/) and search through issues to
confirm your bug hasn't already been reported. -->

<!-- Where possible please include a self-contained code snippet describing your bug: -->

I'm testing out the new `pyproject.toml` config support in alpha. I tried to set the config in my `pyproject.toml` exactly like in the docs: https://pydantic-docs.helpmanual.io/mypy_plugin/#configuring-the-plugin . However, this doesn't seem to work with `mypy` `0.930`, when I run it I get the folllowing traceback:
```
Error constructing plugin instance of PydanticPlugin

Traceback (most recent call last):
  File "/Users/paul/.cache/pre-commit/repo8u79aah5/py_env-python3.9/bin/mypy", line 8, in <module>
    sys.exit(console_entry())
  File "/Users/paul/.cache/pre-commit/repo8u79aah5/py_env-python3.9/lib/python3.9/site-packages/mypy/__main__.py", line 12, in console_entry
    main(None, sys.stdout, sys.stderr)
  File "mypy/main.py", line 96, in main
  File "mypy/main.py", line 173, in run_build
  File "mypy/build.py", line 180, in build
  File "mypy/build.py", line 231, in _build
  File "mypy/build.py", line 478, in load_plugins
  File "mypy/build.py", line 456, in load_plugins_from_config
  File "pydantic/mypy.py", line 91, in pydantic.mypy.PydanticPlugin.__init__
  File "pydantic/mypy.py", line 129, in pydantic.mypy.PydanticPluginConfig.__init__
  File "pydantic/mypy.py", line 130, in pydantic.mypy.PydanticPluginConfig.__init__
  File "/Users/paul/.cache/pre-commit/repo8u79aah5/py_env-python3.9/lib/python3.9/site-packages/tomli/_parser.py", line 56, in load
    s = __fp.read().decode()
AttributeError: 'str' object has no attribute 'decode'
```

The issue is here: https://github.com/samuelcolvin/pydantic/blob/ef4678999f94625819ebad61b44ea264479aeb0a/pydantic/mypy.py#L129

Newer `mypy` versions (`>=0.920`) use `tomli` for parsing, and `tomli`'s version of `load` expects the file handle in binary mode explaining the error.

Related:
https://github.com/samuelcolvin/pydantic/issues/2895
https://github.com/samuelcolvin/pydantic/pull/2908



Comments:

Patch:
diff --git a/pydantic/class_validators.py b/pydantic/class_validators.py
--- a/pydantic/class_validators.py
+++ b/pydantic/class_validators.py
@@ -9,6 +9,9 @@
 from .typing import AnyCallable
 from .utils import ROOT_KEY, in_ipython
 
+if TYPE_CHECKING:
+    from .typing import AnyClassMethod
+
 
 class Validator:
     __slots__ = 'func', 'pre', 'each_item', 'always', 'check_fields', 'skip_on_failure'
@@ -54,7 +57,7 @@ def validator(
     check_fields: bool = True,
     whole: bool = None,
     allow_reuse: bool = False,
-) -> Callable[[AnyCallable], classmethod]:  # type: ignore[type-arg]
+) -> Callable[[AnyCallable], 'AnyClassMethod']:
     """
     Decorate methods on the class indicating that they should be used to validate fields
     :param fields: which field(s) the method should be called on
@@ -81,7 +84,7 @@ def validator(
         assert each_item is False, '"each_item" and "whole" conflict, remove "whole"'
         each_item = not whole
 
-    def dec(f: AnyCallable) -> classmethod:  # type: ignore[type-arg]
+    def dec(f: AnyCallable) -> 'AnyClassMethod':
         f_cls = _prepare_validator(f, allow_reuse)
         setattr(
             f_cls,
@@ -97,20 +100,20 @@ def dec(f: AnyCallable) -> classmethod:  # type: ignore[type-arg]
 
 
 @overload
-def root_validator(_func: AnyCallable) -> classmethod:  # type: ignore[type-arg]
+def root_validator(_func: AnyCallable) -> 'AnyClassMethod':
     ...
 
 
 @overload
 def root_validator(
     *, pre: bool = False, allow_reuse: bool = False, skip_on_failure: bool = False
-) -> Callable[[AnyCallable], classmethod]:  # type: ignore[type-arg]
+) -> Callable[[AnyCallable], 'AnyClassMethod']:
     ...
 
 
 def root_validator(
     _func: Optional[AnyCallable] = None, *, pre: bool = False, allow_reuse: bool = False, skip_on_failure: bool = False
-) -> Union[classmethod, Callable[[AnyCallable], classmethod]]:  # type: ignore[type-arg]
+) -> Union['AnyClassMethod', Callable[[AnyCallable], 'AnyClassMethod']]:
     """
     Decorate methods on a model indicating that they should be used to validate (and perhaps modify) data either
     before or after standard model parsing/validation is performed.
@@ -122,7 +125,7 @@ def root_validator(
         )
         return f_cls
 
-    def dec(f: AnyCallable) -> classmethod:  # type: ignore[type-arg]
+    def dec(f: AnyCallable) -> 'AnyClassMethod':
         f_cls = _prepare_validator(f, allow_reuse)
         setattr(
             f_cls, ROOT_VALIDATOR_CONFIG_KEY, Validator(func=f_cls.__func__, pre=pre, skip_on_failure=skip_on_failure)
@@ -132,7 +135,7 @@ def dec(f: AnyCallable) -> classmethod:  # type: ignore[type-arg]
     return dec
 
 
-def _prepare_validator(function: AnyCallable, allow_reuse: bool) -> classmethod:  # type: ignore[type-arg]
+def _prepare_validator(function: AnyCallable, allow_reuse: bool) -> 'AnyClassMethod':
     """
     Avoid validators with duplicated names since without this, validators can be overwritten silently
     which generally isn't the intended behaviour, don't run in ipython (see #312) or if allow_reuse is False.
@@ -325,7 +328,7 @@ def _generic_validator_basic(validator: AnyCallable, sig: 'Signature', args: Set
         return lambda cls, v, values, field, config: validator(v, values=values, field=field, config=config)
 
 
-def gather_all_validators(type_: 'ModelOrDc') -> Dict[str, classmethod]:  # type: ignore[type-arg]
+def gather_all_validators(type_: 'ModelOrDc') -> Dict[str, 'AnyClassMethod']:
     all_attributes = ChainMap(*[cls.__dict__ for cls in type_.__mro__])
     return {
         k: v
diff --git a/pydantic/main.py b/pydantic/main.py
--- a/pydantic/main.py
+++ b/pydantic/main.py
@@ -66,6 +66,7 @@
     from .types import ModelOrDc
     from .typing import (
         AbstractSetIntStr,
+        AnyClassMethod,
         CallableGenerator,
         DictAny,
         DictStrAny,
@@ -890,7 +891,7 @@ def create_model(
     __config__: Optional[Type[BaseConfig]] = None,
     __base__: None = None,
     __module__: str = __name__,
-    __validators__: Dict[str, classmethod] = None,  # type: ignore[type-arg]
+    __validators__: Dict[str, 'AnyClassMethod'] = None,
     **field_definitions: Any,
 ) -> Type['BaseModel']:
     ...
@@ -903,7 +904,7 @@ def create_model(
     __config__: Optional[Type[BaseConfig]] = None,
     __base__: Union[Type['Model'], Tuple[Type['Model'], ...]],
     __module__: str = __name__,
-    __validators__: Dict[str, classmethod] = None,  # type: ignore[type-arg]
+    __validators__: Dict[str, 'AnyClassMethod'] = None,
     **field_definitions: Any,
 ) -> Type['Model']:
     ...
@@ -915,7 +916,7 @@ def create_model(
     __config__: Optional[Type[BaseConfig]] = None,
     __base__: Union[None, Type['Model'], Tuple[Type['Model'], ...]] = None,
     __module__: str = __name__,
-    __validators__: Dict[str, classmethod] = None,  # type: ignore[type-arg]
+    __validators__: Dict[str, 'AnyClassMethod'] = None,
     **field_definitions: Any,
 ) -> Type['Model']:
     """
diff --git a/pydantic/mypy.py b/pydantic/mypy.py
--- a/pydantic/mypy.py
+++ b/pydantic/mypy.py
@@ -1,20 +1,6 @@
 from configparser import ConfigParser
 from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type as TypingType, Union
 
-from pydantic.utils import is_valid_field
-
-try:
-    import toml
-except ImportError:  # pragma: no cover
-    # future-proofing for upcoming `mypy` releases which will switch dependencies
-    try:
-        import tomli as toml  # type: ignore
-    except ImportError:
-        import warnings
-
-        warnings.warn('No TOML parser installed, cannot read configuration from `pyproject.toml`.')
-        toml = None  # type: ignore
-
 from mypy.errorcodes import ErrorCode
 from mypy.nodes import (
     ARG_NAMED,
@@ -60,13 +46,21 @@
     Type,
     TypeOfAny,
     TypeType,
-    TypeVarLikeType,
     TypeVarType,
     UnionType,
     get_proper_type,
 )
 from mypy.typevars import fill_typevars
 from mypy.util import get_unique_redefinition_name
+from mypy.version import __version__ as mypy_version
+
+from pydantic.utils import is_valid_field
+
+try:
+    from mypy.types import TypeVarDef  # type: ignore[attr-defined]
+except ImportError:  # pragma: no cover
+    # Backward-compatible with TypeVarDef from Mypy 0.910.
+    from mypy.types import TypeVarType as TypeVarDef
 
 CONFIGFILE_KEY = 'pydantic-mypy'
 METADATA_KEY = 'pydantic-mypy-metadata'
@@ -74,6 +68,7 @@
 BASESETTINGS_FULLNAME = 'pydantic.env_settings.BaseSettings'
 FIELD_FULLNAME = 'pydantic.fields.Field'
 DATACLASS_FULLNAME = 'pydantic.dataclasses.dataclass'
+BUILTINS_NAME = 'builtins' if float(mypy_version) >= 0.930 else '__builtins__'
 
 
 def plugin(version: str) -> 'TypingType[Plugin]':
@@ -125,9 +120,9 @@ def __init__(self, options: Options) -> None:
         if options.config_file is None:  # pragma: no cover
             return
 
-        if toml and options.config_file.endswith('.toml'):
-            with open(options.config_file, 'r') as rf:
-                config = toml.load(rf).get('tool', {}).get('pydantic-mypy', {})
+        toml_config = parse_toml(options.config_file)
+        if toml_config is not None:
+            config = toml_config.get('tool', {}).get('pydantic-mypy', {})
             for key in self.__slots__:
                 setting = config.get(key, False)
                 if not isinstance(setting, bool):
@@ -348,26 +343,32 @@ def add_construct_method(self, fields: List['PydanticModelField']) -> None:
         and does not treat settings fields as optional.
         """
         ctx = self._ctx
-        set_str = ctx.api.named_type('builtins.set', [ctx.api.named_type('builtins.str')])
+        set_str = ctx.api.named_type(f'{BUILTINS_NAME}.set', [ctx.api.named_type(f'{BUILTINS_NAME}.str')])
         optional_set_str = UnionType([set_str, NoneType()])
         fields_set_argument = Argument(Var('_fields_set', optional_set_str), optional_set_str, None, ARG_OPT)
         construct_arguments = self.get_field_arguments(fields, typed=True, force_all_optional=False, use_alias=False)
         construct_arguments = [fields_set_argument] + construct_arguments
 
-        obj_type = ctx.api.named_type('builtins.object')
+        obj_type = ctx.api.named_type(f'{BUILTINS_NAME}.object')
         self_tvar_name = '_PydanticBaseModel'  # Make sure it does not conflict with other names in the class
         tvar_fullname = ctx.cls.fullname + '.' + self_tvar_name
-        self_type = TypeVarType(self_tvar_name, tvar_fullname, -1, [], obj_type)
+        tvd = TypeVarDef(self_tvar_name, tvar_fullname, -1, [], obj_type)
         self_tvar_expr = TypeVarExpr(self_tvar_name, tvar_fullname, [], obj_type)
         ctx.cls.info.names[self_tvar_name] = SymbolTableNode(MDEF, self_tvar_expr)
 
+        # Backward-compatible with TypeVarDef from Mypy 0.910.
+        if isinstance(tvd, TypeVarType):
+            self_type = tvd
+        else:
+            self_type = TypeVarType(tvd)  # type: ignore[call-arg]
+
         add_method(
             ctx,
             'construct',
             construct_arguments,
             return_type=self_type,
             self_type=self_type,
-            tvar_like_type=self_type,
+            tvar_def=tvd,
             is_classmethod=True,
         )
 
@@ -619,7 +620,7 @@ def add_method(
     args: List[Argument],
     return_type: Type,
     self_type: Optional[Type] = None,
-    tvar_like_type: Optional[TypeVarLikeType] = None,
+    tvar_def: Optional[TypeVarDef] = None,
     is_classmethod: bool = False,
     is_new: bool = False,
     # is_staticmethod: bool = False,
@@ -654,10 +655,10 @@ def add_method(
         arg_names.append(get_name(arg.variable))
         arg_kinds.append(arg.kind)
 
-    function_type = ctx.api.named_type('builtins.function')
+    function_type = ctx.api.named_type(f'{BUILTINS_NAME}.function')
     signature = CallableType(arg_types, arg_kinds, arg_names, return_type, function_type)
-    if tvar_like_type:
-        signature.variables = [tvar_like_type]
+    if tvar_def:
+        signature.variables = [tvar_def]
 
     func = FuncDef(name, args, Block([PassStmt()]))
     func.info = info
@@ -714,3 +715,25 @@ def get_name(x: Union[FuncBase, SymbolNode]) -> str:
     if callable(fn):  # pragma: no cover
         return fn()
     return fn
+
+
+def parse_toml(config_file: str) -> Optional[Dict[str, Any]]:
+    if not config_file.endswith('.toml'):
+        return None
+
+    read_mode = 'rb'
+    try:
+        import tomli as toml_
+    except ImportError:
+        # older versions of mypy have toml as a dependency, not tomli
+        read_mode = 'r'
+        try:
+            import toml as toml_  # type: ignore[no-redef]
+        except ImportError:  # pragma: no cover
+            import warnings
+
+            warnings.warn('No TOML parser installed, cannot read configuration from `pyproject.toml`.')
+            return None
+
+    with open(config_file, read_mode) as rf:
+        return toml_.load(rf)  # type: ignore[arg-type]
diff --git a/pydantic/typing.py b/pydantic/typing.py
--- a/pydantic/typing.py
+++ b/pydantic/typing.py
@@ -228,6 +228,7 @@ def is_union(tp: Optional[Type[Any]]) -> bool:
     MappingIntStrAny = Mapping[IntStr, Any]
     CallableGenerator = Generator[AnyCallable, None, None]
     ReprArgs = Sequence[Tuple[Optional[str], Any]]
+    AnyClassMethod = classmethod[Any]
 
 __all__ = (
     'ForwardRef',
@@ -258,6 +259,7 @@ def is_union(tp: Optional[Type[Any]]) -> bool:
     'DictIntStrAny',
     'CallableGenerator',
     'ReprArgs',
+    'AnyClassMethod',
     'CallableGenerator',
     'WithArgsTypes',
     'get_args',
diff --git a/pydantic/utils.py b/pydantic/utils.py
--- a/pydantic/utils.py
+++ b/pydantic/utils.py
@@ -574,7 +574,8 @@ def _coerce_items(items: Union['AbstractSetIntStr', 'MappingIntStrAny']) -> 'Map
         elif isinstance(items, AbstractSet):
             items = dict.fromkeys(items, ...)
         else:
-            raise TypeError(f'Unexpected type of exclude value {items.__class__}')  # type: ignore[attr-defined]
+            class_name = getattr(items, '__class__', '???')
+            raise TypeError(f'Unexpected type of exclude value {class_name}')
         return items
 
     @classmethod


Test patch:
diff --git a/tests/mypy/test_mypy.py b/tests/mypy/test_mypy.py
--- a/tests/mypy/test_mypy.py
+++ b/tests/mypy/test_mypy.py
@@ -7,8 +7,10 @@
 
 try:
     from mypy import api as mypy_api
+    from mypy.version import __version__ as mypy_version
 except ImportError:
     mypy_api = None  # type: ignore
+    mypy_version = '0'
 
 try:
     import dotenv
@@ -72,6 +74,15 @@ def test_mypy_results(config_filename: str, python_filename: str, output_filenam
         raise RuntimeError(f'wrote actual output to {output_path} since file did not exist')
 
     expected_out = Path(output_path).read_text() if output_path else ''
+
+    # fix for compatibility between mypy versions: (this can be dropped once we drop support for mypy<0.930)
+    if actual_out and float(mypy_version) < 0.930:
+        actual_out = actual_out.lower()
+        expected_out = expected_out.lower()
+        actual_out = actual_out.replace('variant:', 'variants:')
+        actual_out = re.sub(r'^(\d+: note: {4}).*', r'\1...', actual_out, flags=re.M)
+        expected_out = re.sub(r'^(\d+: note: {4}).*', r'\1...', expected_out, flags=re.M)
+
     assert actual_out == expected_out, actual_out
 
 
diff --git a/tests/requirements-linting.txt b/tests/requirements-linting.txt
--- a/tests/requirements-linting.txt
+++ b/tests/requirements-linting.txt
@@ -8,4 +8,3 @@ pre-commit==2.16.0
 pycodestyle==2.8.0
 pyflakes==2.4.0
 twine==3.7.1
-types-toml==0.10.1