Repository: pydantic/pydantic

Issue:
mypy plugin
# Feature Request

I believe that this project could benefit from having a small mypy plugin built for it. Errors like #400 or #280 could be addressed in this plugin.

## References

- [Mypy blog post about plugins](http://mypy-lang.blogspot.com/2019/03/extending-mypy-with-plugins.html)
- [django-mypy-plugin](https://github.com/mkurnikov/django-stubs/tree/master/mypy_django_plugin)
- [SQLAlchemy-stubs](https://github.com/dropbox/sqlalchemy-stubs/blob/master/sqlmypy.py)
- [dataclasses mypy plugin](https://github.com/python/mypy/blob/master/mypy/plugins/dataclasses.py)


Discussion:
great idea, yes please.

I won't have time to work on it for a while but happy to review a PR if anyone else is keen.

Resolving patch:
diff --git a/docs/build/exec_examples.py b/docs/build/exec_examples.py
--- a/docs/build/exec_examples.py
+++ b/docs/build/exec_examples.py
@@ -119,37 +119,45 @@ def error(desc: str):
             error('no trailing new line')
         if re.search('^ *# *>', file_text, flags=re.M):
             error('contains comments with print output, please remove')
-        no_print_intercept_re = re.compile(r'^# no-print-intercept\n', flags=re.M)
-        no_print_intercept = bool(no_print_intercept_re.search(file_text))
-        if no_print_intercept:
-            file_text = no_print_intercept_re.sub('', file_text)
-
-        mp = MockPrint(file)
-        with patch('builtins.print') as mock_print:
-            if not no_print_intercept:
-                mock_print.side_effect = mp
-            try:
-                importlib.import_module(file.stem)
-            except Exception:
-                tb = traceback.format_exception(*sys.exc_info())
-                error(''.join(e for e in tb if '/pydantic/docs/examples/' in e or not e.startswith('  File ')))
-
-        lines = file_text.split('\n')
-
-        to_json_line = '# output-json'
-        if to_json_line in lines:
-            lines = [line for line in lines if line != to_json_line]
-            if len(mp.statements) != 1:
-                error('should only have one print statement')
-            new_files[file.stem + '.json'] = '\n'.join(mp.statements[0][1]) + '\n'
 
+        dont_execute_re = re.compile(r'^# dont-execute\n', flags=re.M)
+        if dont_execute_re.search(file_text):
+            lines = dont_execute_re.sub('', file_text).split('\n')
         else:
-            for line_no, print_lines in reversed(mp.statements):
-                if len(print_lines) > 2:
-                    text = '"""\n{}\n"""'.format('\n'.join(print_lines))
-                else:
-                    text = '\n'.join('#> ' + l for l in print_lines)
-                lines.insert(line_no, text)
+            no_print_intercept_re = re.compile(r'^# no-print-intercept\n', flags=re.M)
+            no_print_intercept = bool(no_print_intercept_re.search(file_text))
+            if no_print_intercept:
+                file_text = no_print_intercept_re.sub('', file_text)
+
+            mp = MockPrint(file)
+            with patch('builtins.print') as mock_print:
+                if not no_print_intercept:
+                    mock_print.side_effect = mp
+                try:
+                    mod = importlib.import_module(file.stem)
+                except Exception:
+                    tb = traceback.format_exception(*sys.exc_info())
+                    error(''.join(e for e in tb if '/pydantic/docs/examples/' in e or not e.startswith('  File ')))
+
+            if not mod.__file__.startswith(str(EXAMPLES_DIR)):
+                error(f'module path not inside "{EXAMPLES_DIR}", name may shadow another module?')
+
+            lines = file_text.split('\n')
+
+            to_json_line = '# output-json'
+            if to_json_line in lines:
+                lines = [line for line in lines if line != to_json_line]
+                if len(mp.statements) != 1:
+                    error('should only have one print statement')
+                new_files[file.stem + '.json'] = '\n'.join(mp.statements[0][1]) + '\n'
+
+            else:
+                for line_no, print_lines in reversed(mp.statements):
+                    if len(print_lines) > 2:
+                        text = '"""\n{}\n"""'.format('\n'.join(print_lines))
+                    else:
+                        text = '\n'.join('#> ' + l for l in print_lines)
+                    lines.insert(line_no, text)
 
         try:
             ignore_above = lines.index('# ignore-above')
diff --git a/docs/examples/mypy.py b/docs/examples/mypy_example.py
similarity index 71%
rename from docs/examples/mypy.py
rename to docs/examples/mypy_example.py
--- a/docs/examples/mypy.py
+++ b/docs/examples/mypy_example.py
@@ -1,3 +1,4 @@
+# dont-execute
 from datetime import datetime
 from typing import List, Optional
 from pydantic import BaseModel, NoneStr
@@ -10,6 +11,5 @@ class Model(BaseModel):
     list_of_ints: List[int]
 
 m = Model(age=42, list_of_ints=[1, '2', b'3'])
-print(m.age)
-Model()
-# will raise a validation error for age and list_of_ints
+print(m.middle_name)  # not a model field!
+Model()  # will raise a validation error for age and list_of_ints
diff --git a/pydantic/mypy.py b/pydantic/mypy.py
new file mode 100644
--- /dev/null
+++ b/pydantic/mypy.py
@@ -0,0 +1,656 @@
+from configparser import ConfigParser
+from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type as TypingType
+
+from mypy.errorcodes import ErrorCode
+from mypy.nodes import (
+    ARG_NAMED,
+    ARG_NAMED_OPT,
+    ARG_OPT,
+    ARG_POS,
+    ARG_STAR2,
+    MDEF,
+    Argument,
+    AssignmentStmt,
+    Block,
+    CallExpr,
+    ClassDef,
+    Context,
+    Decorator,
+    EllipsisExpr,
+    FuncDef,
+    JsonDict,
+    MemberExpr,
+    NameExpr,
+    PassStmt,
+    PlaceholderNode,
+    RefExpr,
+    StrExpr,
+    SymbolTableNode,
+    TempNode,
+    TypeInfo,
+    TypeVarExpr,
+    Var,
+)
+from mypy.options import Options
+from mypy.plugin import CheckerPluginInterface, ClassDefContext, MethodContext, Plugin, SemanticAnalyzerPluginInterface
+from mypy.semanal import set_callable_name  # type: ignore
+from mypy.server.trigger import make_wildcard_trigger
+from mypy.types import (
+    AnyType,
+    CallableType,
+    Instance,
+    NoneType,
+    Type,
+    TypeOfAny,
+    TypeType,
+    TypeVarDef,
+    TypeVarType,
+    UnionType,
+)
+from mypy.typevars import fill_typevars
+from mypy.util import get_unique_redefinition_name
+
+CONFIGFILE_KEY = 'pydantic-mypy'
+METADATA_KEY = 'pydantic-mypy-metadata'
+BASEMODEL_FULLNAME = 'pydantic.main.BaseModel'
+BASESETTINGS_FULLNAME = 'pydantic.env_settings.BaseSettings'
+FIELD_FULLNAME = 'pydantic.fields.Field'
+
+
+def plugin(version: str) -> 'TypingType[Plugin]':
+    """
+    `version` is the mypy version string
+
+    We might want to use this to print a warning if the mypy version being used is
+    newer, or especially older, than we expect (or need).
+    """
+    return PydanticPlugin
+
+
+class PydanticPlugin(Plugin):
+    def __init__(self, options: Options) -> None:
+        self.plugin_config = PydanticPluginConfig(options)
+        super().__init__(options)
+
+    def get_base_class_hook(self, fullname: str) -> 'Optional[Callable[[ClassDefContext], None]]':
+        sym = self.lookup_fully_qualified(fullname)
+        if sym and isinstance(sym.node, TypeInfo):  # pragma: no branch
+            # No branching may occur if the mypy cache has not been cleared
+            if any(base.fullname() == BASEMODEL_FULLNAME for base in sym.node.mro):
+                return self._pydantic_model_class_maker_callback
+        return None
+
+    def get_method_hook(self, fullname: str) -> Optional[Callable[[MethodContext], Type]]:
+        if fullname.endswith('.from_orm'):
+            return from_orm_callback
+        return None
+
+    def _pydantic_model_class_maker_callback(self, ctx: ClassDefContext) -> None:
+        transformer = PydanticModelTransformer(ctx, self.plugin_config)
+        transformer.transform()
+
+
+class PydanticPluginConfig:
+    __slots__ = ('init_forbid_extra', 'init_typed', 'warn_required_dynamic_aliases', 'warn_untyped_fields')
+    init_forbid_extra: bool
+    init_typed: bool
+    warn_required_dynamic_aliases: bool
+    warn_untyped_fields: bool
+
+    def __init__(self, options: Options) -> None:
+        if options.config_file is None:  # pragma: no cover
+            return
+        plugin_config = ConfigParser()
+        plugin_config.read(options.config_file)
+        for key in self.__slots__:
+            setting = plugin_config.getboolean(CONFIGFILE_KEY, key, fallback=False)
+            setattr(self, key, setting)
+
+
+def from_orm_callback(ctx: MethodContext) -> Type:
+    """
+    Raise an error if orm_mode is not enabled
+    """
+    model_type: Instance
+    if isinstance(ctx.type, CallableType) and isinstance(ctx.type.ret_type, Instance):
+        model_type = ctx.type.ret_type  # called on the class
+    elif isinstance(ctx.type, Instance):
+        model_type = ctx.type  # called on an instance (unusual, but still valid)
+    else:  # pragma: no cover
+        detail = f'ctx.type: {ctx.type} (of type {type(ctx.type).__name__})'
+        error_unexpected_behavior(detail, ctx.api, ctx.context)
+        return ctx.default_return_type
+    pydantic_metadata = model_type.type.metadata.get(METADATA_KEY)
+    if pydantic_metadata is None:
+        return ctx.default_return_type
+    orm_mode = pydantic_metadata.get('config', {}).get('orm_mode')
+    if orm_mode is not True:
+        error_from_orm(model_type.type.name(), ctx.api, ctx.context)
+    return ctx.default_return_type
+
+
+class PydanticModelTransformer:
+    tracked_config_fields: Set[str] = {
+        'extra',
+        'allow_mutation',
+        'orm_mode',
+        'allow_population_by_field_name',
+        'alias_generator',
+    }
+
+    def __init__(self, ctx: ClassDefContext, plugin_config: PydanticPluginConfig) -> None:
+        self._ctx = ctx
+        self.plugin_config = plugin_config
+
+    def transform(self) -> None:
+        """
+        Configures the BaseModel subclass according to the plugin settings.
+
+        In particular:
+        * determines the model config and fields,
+        * adds a fields-aware signature for the initializer and construct methods
+        * freezes the class if allow_mutation = False
+        * stores the fields, config, and if the class is settings in the mypy metadata for access by subclasses
+        """
+        ctx = self._ctx
+        info = self._ctx.cls.info
+
+        config = self.collect_config()
+        fields = self.collect_fields(config)
+        for field in fields:
+            if info[field.name].type is None:
+                if not ctx.api.final_iteration:
+                    ctx.api.defer()
+        is_settings = any(base.fullname() == BASESETTINGS_FULLNAME for base in info.mro[:-1])
+        self.add_initializer(fields, config, is_settings)
+        self.add_construct_method(fields)
+        self.set_frozen(fields, frozen=config.allow_mutation is False)
+        info.metadata[METADATA_KEY] = {
+            'fields': {field.name: field.serialize() for field in fields},
+            'config': config.set_values_dict(),
+        }
+
+    def collect_config(self) -> 'ModelConfigData':
+        """
+        Collects the values of the config attributes that are used by the plugin, accounting for parent classes.
+        """
+        ctx = self._ctx
+        cls = ctx.cls
+        config = ModelConfigData()
+        for stmt in cls.defs.body:
+            if not isinstance(stmt, ClassDef):
+                continue
+            if stmt.name == 'Config':
+                for substmt in stmt.defs.body:
+                    if not isinstance(substmt, AssignmentStmt):
+                        continue
+                    config.update(self.get_config_update(substmt))
+                if (
+                    config.has_alias_generator
+                    and not config.allow_population_by_field_name
+                    and self.plugin_config.warn_required_dynamic_aliases
+                ):
+                    error_required_dynamic_aliases(ctx.api, stmt)
+        for info in cls.info.mro[1:]:  # 0 is the current class
+            if METADATA_KEY not in info.metadata:
+                continue
+
+            # Each class depends on the set of fields in its ancestors
+            ctx.api.add_plugin_dependency(make_wildcard_trigger(info.fullname()))
+            for name, value in info.metadata[METADATA_KEY]['config'].items():
+                config.setdefault(name, value)
+        return config
+
+    def collect_fields(self, model_config: 'ModelConfigData') -> List['PydanticModelField']:
+        """
+        Collects the fields for the model, accounting for parent classes
+        """
+        # First, collect fields belonging to the current class.
+        ctx = self._ctx
+        cls = self._ctx.cls
+        fields = []  # type: List[PydanticModelField]
+        known_fields = set()  # type: Set[str]
+        for stmt in cls.defs.body:
+            if not isinstance(stmt, AssignmentStmt):  # `and stmt.new_syntax` to require annotation
+                continue
+
+            lhs = stmt.lvalues[0]
+            if not isinstance(lhs, NameExpr):
+                continue
+
+            if not stmt.new_syntax and self.plugin_config.warn_untyped_fields:
+                error_untyped_fields(ctx.api, stmt)
+
+            # if lhs.name == '__config__':  # BaseConfig not well handled; I'm not sure why yet
+            #     continue
+
+            sym = cls.info.names.get(lhs.name)
+            if sym is None:  # pragma: no cover
+                # This is likely due to a star import (see the dataclasses plugin for a more detailed explanation)
+                # This is the same logic used in the dataclasses plugin
+                continue
+
+            node = sym.node
+            if isinstance(node, PlaceholderNode):  # pragma: no cover
+                # See the PlaceholderNode docstring for more detail about how this can occur
+                # Basically, it is an edge case when dealing with complex import logic
+                # This is the same logic used in the dataclasses plugin
+                continue
+            assert isinstance(node, Var)
+
+            # x: ClassVar[int] is ignored by dataclasses.
+            if node.is_classvar:
+                continue
+
+            is_required = self.get_is_required(cls, stmt, lhs)
+            alias, has_dynamic_alias = self.get_alias_info(stmt)
+            if (
+                has_dynamic_alias
+                and not model_config.allow_population_by_field_name
+                and self.plugin_config.warn_required_dynamic_aliases
+            ):
+                error_required_dynamic_aliases(ctx.api, stmt)
+            fields.append(
+                PydanticModelField(
+                    name=lhs.name,
+                    is_required=is_required,
+                    alias=alias,
+                    has_dynamic_alias=has_dynamic_alias,
+                    line=stmt.line,
+                    column=stmt.column,
+                )
+            )
+            known_fields.add(lhs.name)
+        all_fields = fields.copy()
+        for info in cls.info.mro[1:]:  # 0 is the current class, -2 is BaseModel, -1 is object
+            if METADATA_KEY not in info.metadata:
+                continue
+
+            superclass_fields = []
+            # Each class depends on the set of fields in its ancestors
+            ctx.api.add_plugin_dependency(make_wildcard_trigger(info.fullname()))
+
+            for name, data in info.metadata[METADATA_KEY]['fields'].items():
+                if name not in known_fields:
+                    field = PydanticModelField.deserialize(info, data)
+                    known_fields.add(name)
+                    superclass_fields.append(field)
+                else:
+                    (field,) = [a for a in all_fields if a.name == name]
+                    all_fields.remove(field)
+                    superclass_fields.append(field)
+            all_fields = superclass_fields + all_fields
+        return all_fields
+
+    def add_initializer(self, fields: List['PydanticModelField'], config: 'ModelConfigData', is_settings: bool) -> None:
+        """
+        Adds a fields-aware `__init__` method to the class.
+
+        The added `__init__` will be annotated with types vs. all `Any` depending on the plugin settings.
+        """
+        ctx = self._ctx
+        typed = self.plugin_config.init_typed
+        use_alias = config.allow_population_by_field_name is not True
+        force_all_optional = is_settings or bool(
+            config.has_alias_generator and not config.allow_population_by_field_name
+        )
+        init_arguments = self.get_field_arguments(
+            fields, typed=typed, force_all_optional=force_all_optional, use_alias=use_alias
+        )
+        if not self.should_init_forbid_extra(fields, config):
+            var = Var('kwargs')
+            init_arguments.append(Argument(var, AnyType(TypeOfAny.explicit), None, ARG_STAR2))
+        add_method(ctx, '__init__', init_arguments, NoneType())
+
+    def add_construct_method(self, fields: List['PydanticModelField']) -> None:
+        """
+        Adds a fully typed `construct` classmethod to the class.
+
+        Similar to the fields-aware __init__ method, but always uses the field names (not aliases),
+        and does not treat settings fields as optional.
+        """
+        ctx = self._ctx
+        set_str = ctx.api.named_type('__builtins__.set', [ctx.api.named_type('__builtins__.str')])
+        optional_set_str = UnionType([set_str, NoneType()])
+        fields_set_argument = Argument(Var('_fields_set', optional_set_str), optional_set_str, None, ARG_OPT)
+        construct_arguments = self.get_field_arguments(fields, typed=True, force_all_optional=False, use_alias=False)
+        construct_arguments = [fields_set_argument] + construct_arguments
+
+        obj_type = ctx.api.named_type('__builtins__.object')
+        self_tvar_name = 'Model'
+        tvar_fullname = ctx.cls.fullname + '.' + self_tvar_name
+        tvd = TypeVarDef(self_tvar_name, tvar_fullname, -1, [], obj_type)
+        self_tvar_expr = TypeVarExpr(self_tvar_name, tvar_fullname, [], obj_type)
+        ctx.cls.info.names[self_tvar_name] = SymbolTableNode(MDEF, self_tvar_expr)
+        self_type = TypeVarType(tvd)
+        add_method(
+            ctx,
+            'construct',
+            construct_arguments,
+            return_type=self_type,
+            self_type=self_type,
+            tvar_def=tvd,
+            is_classmethod=True,
+        )
+
+    def set_frozen(self, fields: List['PydanticModelField'], frozen: bool) -> None:
+        """
+        Marks all fields as properties so that attempts to set them trigger mypy errors.
+
+        This is the same approach used by the attrs and dataclasses plugins.
+        """
+        info = self._ctx.cls.info
+        for field in fields:
+            sym_node = info.names.get(field.name)
+            if sym_node is not None:
+                var = sym_node.node
+                assert isinstance(var, Var)
+                var.is_property = frozen
+            else:
+                var = field.to_var(info, use_alias=False)
+                var.info = info
+                var.is_property = frozen
+                var._fullname = info.fullname() + '.' + var.name()
+                info.names[var.name()] = SymbolTableNode(MDEF, var)
+
+    def get_config_update(self, substmt: AssignmentStmt) -> Optional['ModelConfigData']:
+        """
+        Determines the config update due to a single statement in the Config class definition.
+
+        Warns if a tracked config attribute is set to a value the plugin doesn't know how to interpret (e.g., an int)
+        """
+        lhs = substmt.lvalues[0]
+        if not (isinstance(lhs, NameExpr) and lhs.name in self.tracked_config_fields):
+            return None
+        if lhs.name == 'extra':
+            if isinstance(substmt.rvalue, StrExpr):
+                forbid_extra = substmt.rvalue.value == 'forbid'
+            elif isinstance(substmt.rvalue, MemberExpr):
+                forbid_extra = substmt.rvalue.name == 'forbid'
+            else:
+                error_invalid_config_value(lhs.name, self._ctx.api, substmt)
+                return None
+            return ModelConfigData(forbid_extra=forbid_extra)
+        if lhs.name == 'alias_generator':
+            has_alias_generator = True
+            if isinstance(substmt.rvalue, NameExpr) and substmt.rvalue.fullname == 'builtins.None':
+                has_alias_generator = False
+            return ModelConfigData(has_alias_generator=has_alias_generator)
+        if isinstance(substmt.rvalue, NameExpr) and substmt.rvalue.fullname in ('builtins.True', 'builtins.False'):
+            return ModelConfigData(**{lhs.name: substmt.rvalue.fullname == 'builtins.True'})
+        error_invalid_config_value(lhs.name, self._ctx.api, substmt)
+        return None
+
+    @staticmethod
+    def get_is_required(cls: ClassDef, stmt: AssignmentStmt, lhs: NameExpr) -> bool:
+        """
+        Returns a boolean indicating whether the field defined in `stmt` is a required field.
+        """
+        expr = stmt.rvalue
+        if isinstance(expr, TempNode):
+            # TempNode means annotation-only, so only non-required if Optional
+            value_type = cls.info[lhs.name].type
+            if isinstance(value_type, UnionType) and any(isinstance(item, NoneType) for item in value_type.items):
+                # Annotated as Optional, or otherwise having NoneType in the union
+                return False
+            return True
+        if isinstance(expr, CallExpr) and isinstance(expr.callee, RefExpr) and expr.callee.fullname == FIELD_FULLNAME:
+            # The "default value" is a call to `Field`; at this point, the field is
+            # only required if default is Ellipsis (i.e., `field_name: Annotation = Field(...)`)
+            return len(expr.args) > 0 and type(expr.args[0]) is EllipsisExpr
+        # Only required if the "default value" is Ellipsis (i.e., `field_name: Annotation = ...`)
+        return isinstance(expr, EllipsisExpr)
+
+    @staticmethod
+    def get_alias_info(stmt: AssignmentStmt) -> Tuple[Optional[str], bool]:
+        """
+        Returns a pair (alias, has_dynamic_alias), extracted from the declaration of the field defined in `stmt`.
+
+        `has_dynamic_alias` is True if and only if an alias is provided, but not as a string literal.
+        If `has_dynamic_alias` is True, `alias` will be None.
+        """
+        expr = stmt.rvalue
+        if isinstance(expr, TempNode):
+            # TempNode means annotation-only
+            return None, False
+
+        if not (
+            isinstance(expr, CallExpr) and isinstance(expr.callee, RefExpr) and expr.callee.fullname == FIELD_FULLNAME
+        ):
+            # Assigned value is not a call to pydantic.fields.Field
+            return None, False
+
+        for i, arg_name in enumerate(expr.arg_names):
+            if arg_name != 'alias':
+                continue
+            arg = expr.args[i]
+            if isinstance(arg, StrExpr):
+                return arg.value, False
+            else:
+                return None, True
+        return None, False
+
+    def get_field_arguments(
+        self, fields: List['PydanticModelField'], typed: bool, force_all_optional: bool, use_alias: bool
+    ) -> List[Argument]:
+        """
+        Helper function used during the construction of the `__init__` and `construct` method signatures.
+
+        Returns a list of mypy Argument instances for use in the generated signatures.
+        """
+        info = self._ctx.cls.info
+        arguments = [
+            field.to_argument(info, typed=typed, force_optional=force_all_optional, use_alias=use_alias)
+            for field in fields
+            if not (use_alias and field.has_dynamic_alias)
+        ]
+        return arguments
+
+    def should_init_forbid_extra(self, fields: List['PydanticModelField'], config: 'ModelConfigData') -> bool:
+        """
+        Indicates whether the generated `__init__` should get a `**kwargs` at the end of its signature
+
+        We disallow arbitrary kwargs if the extra config setting is "forbid", or if the plugin config says to,
+        *unless* a required dynamic alias is present (since then we can't determine a valid signature).
+        """
+        if not config.allow_population_by_field_name:
+            if self.is_dynamic_alias_present(fields, bool(config.has_alias_generator)):
+                return False
+        if config.forbid_extra:
+            return True
+        return self.plugin_config.init_forbid_extra
+
+    @staticmethod
+    def is_dynamic_alias_present(fields: List['PydanticModelField'], has_alias_generator: bool) -> bool:
+        """
+        Returns whether any fields on the model have a "dynamic alias", i.e., an alias that cannot be
+        determined during static analysis.
+        """
+        for field in fields:
+            if field.has_dynamic_alias:
+                return True
+        if has_alias_generator:
+            for field in fields:
+                if field.alias is None:
+                    return True
+        return False
+
+
+class PydanticModelField:
+    def __init__(
+        self, name: str, is_required: bool, alias: Optional[str], has_dynamic_alias: bool, line: int, column: int
+    ):
+        self.name = name
+        self.is_required = is_required
+        self.alias = alias
+        self.has_dynamic_alias = has_dynamic_alias
+        self.line = line
+        self.column = column
+
+    def to_var(self, info: TypeInfo, use_alias: bool) -> Var:
+        name = self.name
+        if use_alias and self.alias is not None:
+            name = self.alias
+        return Var(name, info[self.name].type)
+
+    def to_argument(self, info: TypeInfo, typed: bool, force_optional: bool, use_alias: bool) -> Argument:
+        if typed and info[self.name].type is not None:
+            type_annotation = info[self.name].type
+        else:
+            type_annotation = AnyType(TypeOfAny.explicit)
+        return Argument(
+            variable=self.to_var(info, use_alias),
+            type_annotation=type_annotation,
+            initializer=None,
+            kind=ARG_NAMED_OPT if force_optional or not self.is_required else ARG_NAMED,
+        )
+
+    def serialize(self) -> JsonDict:
+        return self.__dict__
+
+    @classmethod
+    def deserialize(cls, info: TypeInfo, data: JsonDict) -> 'PydanticModelField':
+        return cls(**data)
+
+
+class ModelConfigData:
+    def __init__(
+        self,
+        forbid_extra: Optional[bool] = None,
+        allow_mutation: Optional[bool] = None,
+        orm_mode: Optional[bool] = None,
+        allow_population_by_field_name: Optional[bool] = None,
+        has_alias_generator: Optional[bool] = None,
+    ):
+        self.forbid_extra = forbid_extra
+        self.allow_mutation = allow_mutation
+        self.orm_mode = orm_mode
+        self.allow_population_by_field_name = allow_population_by_field_name
+        self.has_alias_generator = has_alias_generator
+
+    def set_values_dict(self) -> Dict[str, Any]:
+        return {k: v for k, v in self.__dict__.items() if v is not None}
+
+    def update(self, config: Optional['ModelConfigData']) -> None:
+        if config is None:
+            return
+        for k, v in config.set_values_dict().items():
+            setattr(self, k, v)
+
+    def setdefault(self, key: str, value: Any) -> None:
+        if getattr(self, key) is None:
+            setattr(self, key, value)
+
+
+ERROR_ORM = ErrorCode('pydantic-orm', 'Invalid from_orm call', 'Pydantic')
+ERROR_CONFIG = ErrorCode('pydantic-config', 'Invalid config value', 'Pydantic')
+ERROR_ALIAS = ErrorCode('pydantic-alias', 'Dynamic alias disallowed', 'Pydantic')
+ERROR_UNEXPECTED = ErrorCode('pydantic-unexpected', 'Unexpected behavior', 'Pydantic')
+ERROR_UNTYPED = ErrorCode('pydantic-field', 'Untyped field disallowed', 'Pydantic')
+
+
+def error_from_orm(model_name: str, api: CheckerPluginInterface, context: Context) -> None:
+    api.fail(f'"{model_name}" does not have orm_mode=True', context, code=ERROR_ORM)
+
+
+def error_invalid_config_value(name: str, api: SemanticAnalyzerPluginInterface, context: Context) -> None:
+    api.fail(f'Invalid value for "Config.{name}"', context, code=ERROR_CONFIG)
+
+
+def error_required_dynamic_aliases(api: SemanticAnalyzerPluginInterface, context: Context) -> None:
+    api.fail('Required dynamic aliases disallowed', context, code=ERROR_ALIAS)
+
+
+def error_unexpected_behavior(detail: str, api: CheckerPluginInterface, context: Context) -> None:  # pragma: no cover
+    # Can't think of a good way to test this, but I confirmed it renders as desired by adding to a non-error path
+    link = 'https://github.com/samuelcolvin/pydantic/issues/new/choose'
+    full_message = f'The pydantic mypy plugin ran into unexpected behavior: {detail}\n'
+    full_message += f'Please consider reporting this bug at {link} so we can try to fix it!'
+    api.fail(full_message, context, code=ERROR_UNEXPECTED)
+
+
+def error_untyped_fields(api: SemanticAnalyzerPluginInterface, context: Context) -> None:
+    api.fail('Untyped fields disallowed', context, code=ERROR_UNTYPED)
+
+
+def add_method(
+    ctx: ClassDefContext,
+    name: str,
+    args: List[Argument],
+    return_type: Type,
+    self_type: Optional[Type] = None,
+    tvar_def: Optional[TypeVarDef] = None,
+    is_classmethod: bool = False,
+    is_new: bool = False,
+    # is_staticmethod: bool = False,
+) -> None:
+    """
+    Adds a new method to a class.
+
+    This can be dropped if/when https://github.com/python/mypy/issues/7301 is merged
+    """
+    info = ctx.cls.info
+
+    # First remove any previously generated methods with the same name
+    # to avoid clashes and problems in the semantic analyzer.
+    if name in info.names:
+        sym = info.names[name]
+        if sym.plugin_generated and isinstance(sym.node, FuncDef):
+            ctx.cls.defs.body.remove(sym.node)
+
+    self_type = self_type or fill_typevars(info)
+    if is_classmethod or is_new:
+        first = [Argument(Var('_cls'), TypeType.make_normalized(self_type), None, ARG_POS)]
+    # elif is_staticmethod:
+    #     first = []
+    else:
+        self_type = self_type or fill_typevars(info)
+        first = [Argument(Var('self'), self_type, None, ARG_POS)]
+    args = first + args
+    arg_types, arg_names, arg_kinds = [], [], []
+    for arg in args:
+        assert arg.type_annotation, 'All arguments must be fully typed.'
+        arg_types.append(arg.type_annotation)
+        arg_names.append(arg.variable.name())
+        arg_kinds.append(arg.kind)
+
+    function_type = ctx.api.named_type('__builtins__.function')
+    signature = CallableType(arg_types, arg_kinds, arg_names, return_type, function_type)
+    if tvar_def:
+        signature.variables = [tvar_def]
+
+    func = FuncDef(name, args, Block([PassStmt()]))
+    func.info = info
+    func.type = set_callable_name(signature, func)
+    func.is_class = is_classmethod
+    # func.is_static = is_staticmethod
+    func._fullname = info.fullname() + '.' + name
+    func.line = info.line
+
+    # NOTE: we would like the plugin generated node to dominate, but we still
+    # need to keep any existing definitions so they get semantically analyzed.
+    if name in info.names:
+        # Get a nice unique name instead.
+        r_name = get_unique_redefinition_name(name, info.names)
+        info.names[r_name] = info.names[name]
+
+    if is_classmethod:  # or is_staticmethod:
+        func.is_decorated = True
+        v = Var(name, func.type)
+        v.info = info
+        v._fullname = func._fullname
+        # if is_classmethod:
+        v.is_classmethod = True
+        dec = Decorator(func, [NameExpr('classmethod')], v)
+        # else:
+        #     v.is_staticmethod = True
+        #     dec = Decorator(func, [NameExpr('staticmethod')], v)
+
+        dec.line = info.line
+        sym = SymbolTableNode(MDEF, dec)
+    else:
+        sym = SymbolTableNode(MDEF, func)
+    sym.plugin_generated = True
+
+    info.names[name] = sym
+    info.defn.defs.body.append(func)


Test patch:
diff --git a/tests/mypy/mypy-default.ini b/tests/mypy/configs/mypy-default.ini
similarity index 100%
rename from tests/mypy/mypy-default.ini
rename to tests/mypy/configs/mypy-default.ini
diff --git a/tests/mypy/configs/mypy-plugin-strict.ini b/tests/mypy/configs/mypy-plugin-strict.ini
new file mode 100644
--- /dev/null
+++ b/tests/mypy/configs/mypy-plugin-strict.ini
@@ -0,0 +1,19 @@
+[mypy]
+plugins = pydantic.mypy
+
+follow_imports = silent
+strict_optional = True
+warn_redundant_casts = True
+warn_unused_ignores = True
+disallow_any_generics = True
+check_untyped_defs = True
+;no_implicit_reexport = True
+
+# for strict mypy: (this is the tricky one :-))
+disallow_untyped_defs = True
+
+[pydantic-mypy]
+init_forbid_extra = True
+init_typed = True
+warn_required_dynamic_aliases = True
+warn_untyped_fields = True
diff --git a/tests/mypy/configs/mypy-plugin.ini b/tests/mypy/configs/mypy-plugin.ini
new file mode 100644
--- /dev/null
+++ b/tests/mypy/configs/mypy-plugin.ini
@@ -0,0 +1,13 @@
+[mypy]
+plugins = pydantic.mypy
+
+follow_imports = silent
+strict_optional = True
+warn_redundant_casts = True
+warn_unused_ignores = True
+disallow_any_generics = True
+check_untyped_defs = True
+;no_implicit_reexport = True
+
+# for strict mypy: (this is the tricky one :-))
+disallow_untyped_defs = True
diff --git a/tests/mypy/fail1.py b/tests/mypy/modules/fail1.py
similarity index 100%
rename from tests/mypy/fail1.py
rename to tests/mypy/modules/fail1.py
diff --git a/tests/mypy/fail2.py b/tests/mypy/modules/fail2.py
similarity index 100%
rename from tests/mypy/fail2.py
rename to tests/mypy/modules/fail2.py
diff --git a/tests/mypy/fail3.py b/tests/mypy/modules/fail3.py
similarity index 100%
rename from tests/mypy/fail3.py
rename to tests/mypy/modules/fail3.py
diff --git a/tests/mypy/modules/plugin_fail.py b/tests/mypy/modules/plugin_fail.py
new file mode 100644
--- /dev/null
+++ b/tests/mypy/modules/plugin_fail.py
@@ -0,0 +1,193 @@
+from typing import Any, Generic, Optional, Set, TypeVar, Union
+
+from pydantic import BaseModel, BaseSettings, Extra, Field
+from pydantic.generics import GenericModel
+
+
+class Model(BaseModel):
+    x: int
+    y: str
+
+    def method(self) -> None:
+        pass
+
+    class Config:
+        alias_generator = None
+        allow_mutation = False
+        extra = Extra.forbid
+
+        def config_method(self) -> None:
+            ...
+
+
+model = Model(x=1, y='y', z='z')
+model = Model(x=1)
+model.y = 'a'
+Model.from_orm({})
+Model.from_orm({})  # type: ignore[pydantic-orm]  # noqa F821
+
+
+class ForbidExtraModel(BaseModel):
+    class Config:
+        extra = 'forbid'
+
+
+ForbidExtraModel(x=1)
+
+
+class ForbidExtraModel2(BaseModel):
+    class Config:
+        extra = 'forbid'
+        validate_all = False
+
+    Config.validate_all = True
+
+
+ForbidExtraModel2(x=1)
+
+
+class BadExtraModel(BaseModel):
+    class Config:
+        extra = 1  # type: ignore[pydantic-config]  # noqa F821
+        extra = 1
+
+
+class BadConfig1(BaseModel):
+    class Config:
+        orm_mode: Any = {}  # not sensible, but should still be handled gracefully
+
+
+class BadConfig2(BaseModel):
+    class Config:
+        orm_mode = list  # not sensible, but should still be handled gracefully
+
+
+class InheritingModel(Model):
+    class Config:
+        allow_mutation = True
+
+
+class DefaultTestingModel(BaseModel):
+    # Required
+    a: int
+    b: int = ...
+    c: int = Field(...)
+    d: Union[int, str]
+    e = ...
+
+    # Not required
+    f: Optional[int]
+    g: int = 1
+    h: int = Field(1)
+    i: int = Field(None)
+    j = 1
+
+
+DefaultTestingModel()
+
+
+class UndefinedAnnotationModel(BaseModel):
+    undefined: Undefined  # noqa F821
+
+
+UndefinedAnnotationModel()
+
+
+class Settings(BaseSettings):
+    x: int
+
+
+Model.construct(x=1)
+Model.construct(_fields_set={'x'}, x=1, y='2')
+Model.construct(x='1', y='2')
+
+Settings()  # should pass here due to possibly reading from environment
+
+# Strict mode fails
+inheriting = InheritingModel(x='1', y='1')
+Settings(x='1')
+Model(x='1', y='2')
+
+
+class Blah(BaseModel):
+    fields_set: Optional[Set[str]] = None
+
+
+# Need to test generic checking here since generics don't work in 3.6, and plugin-success.py is executed
+T = TypeVar('T')
+
+
+class Response(GenericModel, Generic[T]):
+    data: T
+    error: Optional[str]
+
+
+response = Response[Model](data=model, error=None)
+response = Response[Model](data=1, error=None)
+
+
+class AliasModel(BaseModel):
+    x: str = Field(..., alias='y')
+    z: int
+
+
+AliasModel(y=1, z=2)
+
+x_alias = 'y'
+
+
+class DynamicAliasModel(BaseModel):
+    x: str = Field(..., alias=x_alias)
+    z: int
+
+
+DynamicAliasModel(y='y', z='1')
+
+
+class DynamicAliasModel2(BaseModel):
+    x: str = Field(..., alias=x_alias)
+    z: int
+
+    class Config:
+        allow_population_by_field_name = True
+
+
+DynamicAliasModel2(y='y', z=1)
+DynamicAliasModel2(x='y', z=1)
+
+
+class AliasGeneratorModel(BaseModel):
+    x: int
+
+    class Config:
+        alias_generator = lambda x: x + '_'  # noqa E731
+
+
+AliasGeneratorModel(x=1)
+AliasGeneratorModel(x_=1)
+AliasGeneratorModel(z=1)
+
+
+class AliasGeneratorModel2(BaseModel):
+    x: int = Field(..., alias='y')
+
+    class Config:  # type: ignore[pydantic-alias]  # noqa F821
+        alias_generator = lambda x: x + '_'  # noqa E731
+
+
+class UntypedFieldModel(BaseModel):
+    x: int = 1
+    y = 2
+    z = 2  # type: ignore[pydantic-field]  # noqa F821
+
+
+AliasGeneratorModel2(x=1)
+AliasGeneratorModel2(y=1, z=1)
+
+
+class CoverageTester(Missing):  # noqa F821
+    def from_orm(self) -> None:
+        pass
+
+
+CoverageTester().from_orm()
diff --git a/tests/mypy/modules/plugin_success.py b/tests/mypy/modules/plugin_success.py
new file mode 100644
--- /dev/null
+++ b/tests/mypy/modules/plugin_success.py
@@ -0,0 +1,106 @@
+from typing import ClassVar, Optional
+
+from pydantic import BaseModel, Field
+
+
+class Model(BaseModel):
+    x: float
+    y: str
+
+    class Config:
+        orm_mode = True
+
+    class NotConfig:
+        allow_mutation = False
+
+
+class SelfReferencingModel(BaseModel):
+    submodel: Optional['SelfReferencingModel']
+
+    @property
+    def prop(self) -> None:
+        ...
+
+
+SelfReferencingModel.update_forward_refs()
+
+model = Model(x=1, y='y')
+Model(x=1, y='y', z='z')
+model.x = 2
+model.from_orm(model)
+
+self_referencing_model = SelfReferencingModel(submodel=SelfReferencingModel(submodel=None))
+
+
+class InheritingModel(Model):
+    z: int = 1
+
+
+InheritingModel.from_orm(model)
+
+
+class ForwardReferencingModel(Model):
+    future: 'FutureModel'
+
+
+class FutureModel(Model):
+    pass
+
+
+ForwardReferencingModel.update_forward_refs()
+future_model = FutureModel(x=1, y='a')
+forward_model = ForwardReferencingModel(x=1, y='a', future=future_model)
+
+
+class NoMutationModel(BaseModel):
+    x: int
+
+    class Config:
+        allow_mutation = False
+
+
+class MutationModel(NoMutationModel):
+    a = 1
+
+    class Config:
+        allow_mutation = True
+        orm_mode = True
+
+
+MutationModel(x=1).x = 2
+MutationModel.from_orm(model)
+
+
+class OverrideModel(Model):
+    x: int
+
+
+OverrideModel(x=1.5, y='b')
+
+
+class Mixin:
+    def f(self) -> None:
+        pass
+
+
+class MultiInheritanceModel(BaseModel, Mixin):
+    pass
+
+
+MultiInheritanceModel().f()
+
+
+class AliasModel(BaseModel):
+    x: str = Field(..., alias='y')
+
+
+alias_model = AliasModel(y='hello')
+assert alias_model.x == 'hello'
+
+
+class ClassVarModel(BaseModel):
+    x: int
+    y: ClassVar[int] = 1
+
+
+ClassVarModel(x=1)
diff --git a/tests/mypy/success.py b/tests/mypy/modules/success.py
similarity index 100%
rename from tests/mypy/success.py
rename to tests/mypy/modules/success.py
diff --git a/tests/mypy/outputs/fail1.txt b/tests/mypy/outputs/fail1.txt
new file mode 100644
--- /dev/null
+++ b/tests/mypy/outputs/fail1.txt
@@ -0,0 +1 @@
+20: error: Unsupported operand types for + ("int" and "str")  [operator]
\ No newline at end of file
diff --git a/tests/mypy/outputs/fail2.txt b/tests/mypy/outputs/fail2.txt
new file mode 100644
--- /dev/null
+++ b/tests/mypy/outputs/fail2.txt
@@ -0,0 +1 @@
+20: error: "Model" has no attribute "foobar"  [attr-defined]
\ No newline at end of file
diff --git a/tests/mypy/outputs/fail3.txt b/tests/mypy/outputs/fail3.txt
new file mode 100644
--- /dev/null
+++ b/tests/mypy/outputs/fail3.txt
@@ -0,0 +1 @@
+22: error: Argument 1 to "append" of "list" has incompatible type "str"; expected "int"  [arg-type]
\ No newline at end of file
diff --git a/tests/mypy/outputs/plugin-fail-strict.txt b/tests/mypy/outputs/plugin-fail-strict.txt
new file mode 100644
--- /dev/null
+++ b/tests/mypy/outputs/plugin-fail-strict.txt
@@ -0,0 +1,34 @@
+23: error: Unexpected keyword argument "z" for "Model"  [call-arg]
+24: error: Missing named argument "y" for "Model"  [call-arg]
+25: error: Property "y" defined in "Model" is read-only  [misc]
+26: error: "Model" does not have orm_mode=True  [pydantic-orm]
+35: error: Unexpected keyword argument "x" for "ForbidExtraModel"  [call-arg]
+46: error: Unexpected keyword argument "x" for "ForbidExtraModel2"  [call-arg]
+52: error: Invalid value for "Config.extra"  [pydantic-config]
+57: error: Invalid value for "Config.orm_mode"  [pydantic-config]
+62: error: Invalid value for "Config.orm_mode"  [pydantic-config]
+73: error: Incompatible types in assignment (expression has type "ellipsis", variable has type "int")  [assignment]
+76: error: Untyped fields disallowed  [pydantic-field]
+83: error: Untyped fields disallowed  [pydantic-field]
+86: error: Missing named argument "a" for "DefaultTestingModel"  [call-arg]
+86: error: Missing named argument "b" for "DefaultTestingModel"  [call-arg]
+86: error: Missing named argument "c" for "DefaultTestingModel"  [call-arg]
+86: error: Missing named argument "d" for "DefaultTestingModel"  [call-arg]
+86: error: Missing named argument "e" for "DefaultTestingModel"  [call-arg]
+90: error: Name 'Undefined' is not defined  [name-defined]
+93: error: Missing named argument "undefined" for "UndefinedAnnotationModel"  [call-arg]
+100: error: Missing named argument "y" for "construct" of "Model"  [call-arg]
+102: error: Argument "x" to "construct" of "Model" has incompatible type "str"; expected "int"  [arg-type]
+107: error: Argument "x" to "InheritingModel" has incompatible type "str"; expected "int"  [arg-type]
+108: error: Argument "x" to "Settings" has incompatible type "str"; expected "int"  [arg-type]
+109: error: Argument "x" to "Model" has incompatible type "str"; expected "int"  [arg-type]
+126: error: Argument "data" to "Response" has incompatible type "int"; expected "Model"  [arg-type]
+134: error: Argument "y" to "AliasModel" has incompatible type "int"; expected "str"  [arg-type]
+140: error: Required dynamic aliases disallowed  [pydantic-alias]
+144: error: Argument "z" to "DynamicAliasModel" has incompatible type "str"; expected "int"  [arg-type]
+155: error: Unexpected keyword argument "y" for "DynamicAliasModel2"  [call-arg]
+162: error: Required dynamic aliases disallowed  [pydantic-alias]
+180: error: Untyped fields disallowed  [pydantic-field]
+184: error: Unexpected keyword argument "x" for "AliasGeneratorModel2"  [call-arg]
+185: error: Unexpected keyword argument "z" for "AliasGeneratorModel2"  [call-arg]
+188: error: Name 'Missing' is not defined  [name-defined]
\ No newline at end of file
diff --git a/tests/mypy/outputs/plugin-fail.txt b/tests/mypy/outputs/plugin-fail.txt
new file mode 100644
--- /dev/null
+++ b/tests/mypy/outputs/plugin-fail.txt
@@ -0,0 +1,23 @@
+23: error: Unexpected keyword argument "z" for "Model"  [call-arg]
+24: error: Missing named argument "y" for "Model"  [call-arg]
+25: error: Property "y" defined in "Model" is read-only  [misc]
+26: error: "Model" does not have orm_mode=True  [pydantic-orm]
+35: error: Unexpected keyword argument "x" for "ForbidExtraModel"  [call-arg]
+46: error: Unexpected keyword argument "x" for "ForbidExtraModel2"  [call-arg]
+52: error: Invalid value for "Config.extra"  [pydantic-config]
+57: error: Invalid value for "Config.orm_mode"  [pydantic-config]
+62: error: Invalid value for "Config.orm_mode"  [pydantic-config]
+73: error: Incompatible types in assignment (expression has type "ellipsis", variable has type "int")  [assignment]
+86: error: Missing named argument "a" for "DefaultTestingModel"  [call-arg]
+86: error: Missing named argument "b" for "DefaultTestingModel"  [call-arg]
+86: error: Missing named argument "c" for "DefaultTestingModel"  [call-arg]
+86: error: Missing named argument "d" for "DefaultTestingModel"  [call-arg]
+86: error: Missing named argument "e" for "DefaultTestingModel"  [call-arg]
+90: error: Name 'Undefined' is not defined  [name-defined]
+93: error: Missing named argument "undefined" for "UndefinedAnnotationModel"  [call-arg]
+100: error: Missing named argument "y" for "construct" of "Model"  [call-arg]
+102: error: Argument "x" to "construct" of "Model" has incompatible type "str"; expected "int"  [arg-type]
+155: error: Missing named argument "x" for "DynamicAliasModel2"  [call-arg]
+174: error: unused 'type: ignore' comment
+181: error: unused 'type: ignore' comment
+188: error: Name 'Missing' is not defined  [name-defined]
\ No newline at end of file
diff --git a/tests/mypy/outputs/plugin-success-strict.txt b/tests/mypy/outputs/plugin-success-strict.txt
new file mode 100644
--- /dev/null
+++ b/tests/mypy/outputs/plugin-success-strict.txt
@@ -0,0 +1,3 @@
+28: error: Unexpected keyword argument "z" for "Model"  [call-arg]
+63: error: Untyped fields disallowed  [pydantic-field]
+78: error: Argument "x" to "OverrideModel" has incompatible type "float"; expected "int"  [arg-type]
\ No newline at end of file
diff --git a/tests/mypy/test_mypy.py b/tests/mypy/test_mypy.py
new file mode 100644
--- /dev/null
+++ b/tests/mypy/test_mypy.py
@@ -0,0 +1,79 @@
+import importlib
+import os
+from pathlib import Path
+
+import pytest
+from mypy import api
+
+try:
+    import typing_extensions
+except ImportError:
+    typing_extensions = None
+
+# This ensures mypy can find the test files, no matter where tests are run from:
+os.chdir(Path(__file__).parent.parent.parent)
+
+# You can change the following variable to True during development to overwrite expected output with generated output
+GENERATE = False
+
+cases = [
+    ('mypy-plugin.ini', 'plugin_success.py', None),
+    ('mypy-plugin.ini', 'plugin_fail.py', 'plugin-fail.txt'),
+    ('mypy-plugin-strict.ini', 'plugin_success.py', 'plugin-success-strict.txt'),
+    ('mypy-plugin-strict.ini', 'plugin_fail.py', 'plugin-fail-strict.txt'),
+    ('mypy-default.ini', 'success.py', None),
+    ('mypy-default.ini', 'fail1.py', 'fail1.txt'),
+    ('mypy-default.ini', 'fail2.py', 'fail2.txt'),
+    ('mypy-default.ini', 'fail3.py', 'fail3.txt'),
+    ('mypy-default.ini', 'plugin_success.py', None),
+]
+executable_modules = list({fname[:-3] for _, fname, out_fname in cases if out_fname is None})
+
+
+@pytest.mark.skipif(not typing_extensions, reason='typing_extensions not installed')
+@pytest.mark.parametrize('config_filename,python_filename,output_filename', cases)
+def test_mypy_results(config_filename, python_filename, output_filename):
+    full_config_filename = f'tests/mypy/configs/{config_filename}'
+    full_filename = f'tests/mypy/modules/{python_filename}'
+    full_output_filename = None if output_filename is None else f'tests/mypy/outputs/{output_filename}'
+
+    expected_out = ''
+    expected_err = ''
+    expected_returncode = 0 if output_filename is None else 1
+    if full_output_filename is not None:
+        with open(full_output_filename, 'r') as f:
+            expected_out = f.read()
+
+    # Specifying a different cache dir for each configuration dramatically speeds up subsequent execution
+    # It also prevents cache-invalidation-related bugs in the tests
+    cache_dir = f'.mypy_cache/test-{config_filename[:-4]}'
+    actual_result = api.run(
+        [full_filename, '--config-file', full_config_filename, '--cache-dir', cache_dir, '--show-error-codes']
+    )
+    actual_out, actual_err, actual_returncode = actual_result
+    # Need to strip filenames due to differences in formatting by OS
+    actual_out = '\n'.join(['.py:'.join(line.split('.py:')[1:]) for line in actual_out.split('\n')]).strip()
+
+    if GENERATE and output_filename is not None:
+        with open(full_output_filename, 'w') as f:
+            f.write(actual_out)
+    else:
+        assert actual_out == expected_out, actual_out
+
+    assert actual_err == expected_err
+    assert actual_returncode == expected_returncode
+
+
+@pytest.mark.parametrize('module', executable_modules)
+def test_success_cases_run(module):
+    """
+    Ensure the "success" files can actually be executed
+    """
+    importlib.import_module(f'tests.mypy.modules.{module}')
+
+
+def test_generation_is_disabled():
+    """
+    Makes sure we don't accidentally leave generation on
+    """
+    assert not GENERATE
diff --git a/tests/test_mypy.py b/tests/test_mypy.py
deleted file mode 100644
--- a/tests/test_mypy.py
+++ /dev/null
@@ -1,61 +0,0 @@
-import importlib
-import os
-import re
-from pathlib import Path
-
-import pytest
-from mypy import api
-
-try:
-    import typing_extensions
-except ImportError:
-    typing_extensions = None
-
-# This ensures mypy can find the test files, no matter where tests are run from:
-os.chdir(Path(__file__).parent.parent)
-
-expected_fails = [
-    (
-        'tests/mypy/fail1.py',
-        ('tests/mypy/fail1.py:20: error: Unsupported operand types for + ("int" and "str")\n', '', 1),
-    ),
-    ('tests/mypy/fail2.py', ('tests/mypy/fail2.py:20: error: "Model" has no attribute "foobar"\n', '', 1)),
-    (
-        'tests/mypy/fail3.py',
-        (
-            'tests/mypy/fail3.py:22: error: '
-            'Argument 1 to "append" of "list" has incompatible type "str"; expected "int"\n',
-            '',
-            1,
-        ),
-    ),
-]
-
-expected_successes = [('tests/mypy/success.py', ('', '', 0))]
-
-
-@pytest.mark.skipif(not typing_extensions, reason='typing_extensions not installed')
-@pytest.mark.parametrize('filename,expected_result', expected_successes + expected_fails)
-def test_mypy_results(filename, expected_result):
-    actual_result = api.run([filename, '--config-file', 'tests/mypy/mypy-default.ini', '--no-error-summary'])
-
-    expected_out, expected_err, expected_returncode = expected_result
-    actual_out, actual_err, actual_returncode = actual_result
-    assert (expected_err, expected_returncode) == (actual_err, actual_returncode)
-
-    # Need to remove filenames, as they render differently on mac/linux and windows:
-    actual_out_lines = actual_out.split('\n')
-    expected_out_lines = expected_out.split('\n')
-    assert len(actual_out_lines) == len(expected_out_lines)
-    for actual_line, expected_line in zip(actual_out_lines, expected_out_lines):
-        actual_line_without_filename = '.py'.join(actual_line.split('.py')[1:])
-        expected_line_without_filename = '.py'.join(expected_line.split('.py')[1:])
-        assert actual_line_without_filename == expected_line_without_filename
-
-
-success_imports = [re.sub('/', '.', filename)[:-3] for filename, _ in expected_successes]
-
-
-@pytest.mark.parametrize('module', success_imports)
-def test_mypy_successes_run(module):
-    importlib.import_module(module)