Repository: samuelcolvin/pydantic
Pull request #2908 (2021-06-14 14:35:02)

Toml support for `pydantic-mypy` plugin config
<!-- Thank you for your contribution! -->
<!-- Unless your change is trivial, please create an issue to discuss the change before creating a PR -->
<!-- See https://pydantic-docs.helpmanual.io/contributing/ for help on Contributing -->

## Change Summary

This PR adds support for configuring the `pydantic-mypy` plugin in `pyproject.toml`, as well as the previous `mypy.ini` config file, enabling full compatibility with `mypy>0.900`. 

Configuration behavior should be consistent with prior behavior from `mypy.ini`, with enumerated slots populated from the `pydantic-mypy` configuration section, and with non-boolean options resulting in a `ValueError`.

Note that this increases the pinned `mypy` version in the testing suite to 0.901, and includes some formatting changes for compatibility to the test cases. However, the plugin behavior is the same in earlier versions of `mypy`.

## Related issue number

Resolves https://github.com/samuelcolvin/pydantic/issues/2895

## Checklist

* [x] Unit tests for the changes exist
* [x] Tests pass on CI and coverage remains at 100%
* [x] Documentation reflects the changes where applicable
* [x] `changes/<pull request or issue id>-<github username>.md` file added describing change
  (see [changes/README.md](https://github.com/samuelcolvin/pydantic/blob/master/changes/README.md) for details)
* [x] My PR is ready to review, **please add a comment including the phrase "please review" to assign reviewers**


jrwalk: This should be ready for review!
jrwalk: This should be ready, please review!
SnoopJeDi: @samuelcolvin @PrettyWood is anything other than resolving the implicit dependency needed to get this merged? I'd love to stop organizing project configuration around the plugin's idiosyncratic parsing, and the PR looks great otherwise.
PrettyWood: Please update.
jrwalk: @PrettyWood should be good to go!

Patch:
diff --git a/changes/2908-jrwalk.md b/changes/2908-jrwalk.md
new file mode 100644
index 000000000..8a5096501
--- /dev/null
+++ b/changes/2908-jrwalk.md
@@ -0,0 +1,1 @@
+Make `pydantic-mypy` plugin compatible with `pyproject.toml` configuration, consistent with `mypy` changes. See the [doc](https://pydantic-docs.helpmanual.io/mypy_plugin/#configuring-the-plugin) for more information.
diff --git a/docs/mypy_plugin.md b/docs/mypy_plugin.md
index b444d2d55..842078056 100644
--- a/docs/mypy_plugin.md
+++ b/docs/mypy_plugin.md
@@ -138,3 +138,28 @@ init_typed = True
 warn_required_dynamic_aliases = True
 warn_untyped_fields = True
 ```
+
+As of `mypy>=0.900`, mypy config may also be included in the `pyproject.toml` file rather than `mypy.ini`.
+The same configuration as above would be:
+```toml
+[tool.mypy]
+plugins = [
+  "pydantic.mypy"
+]
+
+follow_imports = "silent"
+warn_redundant_casts = true
+warn_unused_ignores = true
+disallow_any_generics = true
+check_untyped_defs = true
+no_implicit_reexport = true
+
+# for strict mypy: (this is the tricky one :-))
+disallow_untyped_defs = true
+
+[tool.pydantic-mypy]
+init_forbid_extra = true
+init_typed = true
+warn_required_dynamic_aliases = true
+warn_untyped_fields = true
+```
diff --git a/pydantic/mypy.py b/pydantic/mypy.py
index 8107b54b4..1dfd49e60 100644
--- a/pydantic/mypy.py
+++ b/pydantic/mypy.py
@@ -1,6 +1,17 @@
 from configparser import ConfigParser
 from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type as TypingType, Union
 
+try:
+    import toml
+except ImportError:  # pragma: no cover
+    # future-proofing for upcoming `mypy` releases which will switch dependencies
+    try:
+        import tomli as toml  # type: ignore
+    except ImportError:
+        import warnings
+
+        warnings.warn('No TOML parser installed, cannot read configuration from `pyproject.toml`.')
+        toml = None  # type: ignore
 from mypy.errorcodes import ErrorCode
 from mypy.nodes import (
     ARG_NAMED,
@@ -110,11 +121,21 @@ class PydanticPluginConfig:
     def __init__(self, options: Options) -> None:
         if options.config_file is None:  # pragma: no cover
             return
-        plugin_config = ConfigParser()
-        plugin_config.read(options.config_file)
-        for key in self.__slots__:
-            setting = plugin_config.getboolean(CONFIGFILE_KEY, key, fallback=False)
-            setattr(self, key, setting)
+
+        if toml and options.config_file.endswith('.toml'):
+            with open(options.config_file, 'r') as rf:
+                config = toml.load(rf).get('tool', {}).get('pydantic-mypy', {})
+            for key in self.__slots__:
+                setting = config.get(key, False)
+                if not isinstance(setting, bool):
+                    raise ValueError(f'Configuration value must be a boolean for key: {key}')
+                setattr(self, key, setting)
+        else:
+            plugin_config = ConfigParser()
+            plugin_config.read(options.config_file)
+            for key in self.__slots__:
+                setting = plugin_config.getboolean(CONFIGFILE_KEY, key, fallback=False)
+                setattr(self, key, setting)
 
 
 def from_orm_callback(ctx: MethodContext) -> Type:


Test patch:
diff --git a/tests/mypy/configs/pyproject-default.toml b/tests/mypy/configs/pyproject-default.toml
new file mode 100644
index 000000000..63efc7a93
--- /dev/null
+++ b/tests/mypy/configs/pyproject-default.toml
@@ -0,0 +1,27 @@
+[build-system]
+requires = ["poetry>=0.12"]
+build_backend = "poetry.masonry.api"
+
+[tool.poetry]
+name = "test"
+version = "0.0.1"
+readme = "README.md"
+authors = [
+    "author@example.com"
+]
+
+[tool.poetry.dependencies]
+python = "*"
+
+[tool.pytest.ini_options]
+addopts = "-v -p no:warnings"
+
+[tool.mypy]
+follow_imports = "silent"
+strict_optional = true
+warn_redundant_casts = true
+warn_unused_ignores = true
+disallow_any_generics = true
+check_untyped_defs = true
+no_implicit_reexport = true
+disallow_untyped_defs = true
diff --git a/tests/mypy/configs/pyproject-plugin-bad-param.toml b/tests/mypy/configs/pyproject-plugin-bad-param.toml
new file mode 100644
index 000000000..9e58a4b69
--- /dev/null
+++ b/tests/mypy/configs/pyproject-plugin-bad-param.toml
@@ -0,0 +1,33 @@
+[build-system]
+requires = ["poetry>=0.12"]
+build_backend = "poetry.masonry.api"
+
+[tool.poetry]
+name = "test"
+version = "0.0.1"
+readme = "README.md"
+authors = [
+    "author@example.com"
+]
+
+[tool.poetry.dependencies]
+python = "*"
+
+[tool.pytest.ini_options]
+addopts = "-v -p no:warnings"
+
+[tool.mypy]
+plugins = [
+    "pydantic.mypy"
+]
+follow_imports = "silent"
+strict_optional = true
+warn_redundant_casts = true
+warn_unused_ignores = true
+disallow_any_generics = true
+check_untyped_defs = true
+no_implicit_reexport = true
+disallow_untyped_defs = true
+
+[tool.pydantic-mypy]
+init_forbid_extra = "foo"  # this will raise a ValueError for the config
diff --git a/tests/mypy/configs/pyproject-plugin-strict.toml b/tests/mypy/configs/pyproject-plugin-strict.toml
new file mode 100644
index 000000000..99f5df074
--- /dev/null
+++ b/tests/mypy/configs/pyproject-plugin-strict.toml
@@ -0,0 +1,36 @@
+[build-system]
+requires = ["poetry>=0.12"]
+build_backend = "poetry.masonry.api"
+
+[tool.poetry]
+name = "test"
+version = "0.0.1"
+readme = "README.md"
+authors = [
+    "author@example.com"
+]
+
+[tool.poetry.dependencies]
+python = "*"
+
+[tool.pytest.ini_options]
+addopts = "-v -p no:warnings"
+
+[tool.mypy]
+plugins = [
+    "pydantic.mypy"
+]
+follow_imports = "silent"
+strict_optional = true
+warn_redundant_casts = true
+warn_unused_ignores = true
+disallow_any_generics = true
+check_untyped_defs = true
+no_implicit_reexport = true
+disallow_untyped_defs = true
+
+[tool.pydantic-mypy]
+init_forbid_extra = true
+init_typed = true
+warn_required_dynamic_aliases = true
+warn_untyped_fields = true
diff --git a/tests/mypy/configs/pyproject-plugin.toml b/tests/mypy/configs/pyproject-plugin.toml
new file mode 100644
index 000000000..e8e13525c
--- /dev/null
+++ b/tests/mypy/configs/pyproject-plugin.toml
@@ -0,0 +1,30 @@
+[build-system]
+requires = ["poetry>=0.12"]
+build_backend = "poetry.masonry.api"
+
+[tool.poetry]
+name = "test"
+version = "0.0.1"
+readme = "README.md"
+authors = [
+    "author@example.com"
+]
+
+[tool.poetry.dependencies]
+python = "*"
+
+[tool.pytest.ini_options]
+addopts = "-v -p no:warnings"
+
+[tool.mypy]
+plugins = [
+    "pydantic.mypy"
+]
+follow_imports = "silent"
+strict_optional = true
+warn_redundant_casts = true
+warn_unused_ignores = true
+disallow_any_generics = true
+check_untyped_defs = true
+no_implicit_reexport = true
+disallow_untyped_defs = true
diff --git a/tests/mypy/test_mypy.py b/tests/mypy/test_mypy.py
index a81cfe0f8..ea472ba98 100644
--- a/tests/mypy/test_mypy.py
+++ b/tests/mypy/test_mypy.py
@@ -8,12 +8,12 @@ import pytest
 try:
     from mypy import api as mypy_api
 except ImportError:
-    mypy_api = None
+    mypy_api = None  # type: ignore
 
 try:
     import dotenv
 except ImportError:
-    dotenv = None
+    dotenv = None  # type: ignore
 
 # This ensures mypy can find the test files, no matter where tests are run from:
 os.chdir(Path(__file__).parent.parent.parent)
@@ -29,20 +29,29 @@ cases = [
     ('mypy-default.ini', 'fail3.py', 'fail3.txt'),
     ('mypy-default.ini', 'fail4.py', 'fail4.txt'),
     ('mypy-default.ini', 'plugin_success.py', 'plugin_success.txt'),
+    ('pyproject-default.toml', 'success.py', None),
+    ('pyproject-default.toml', 'fail1.py', 'fail1.txt'),
+    ('pyproject-default.toml', 'fail2.py', 'fail2.txt'),
+    ('pyproject-default.toml', 'fail3.py', 'fail3.txt'),
+    ('pyproject-default.toml', 'fail4.py', 'fail4.txt'),
+    ('pyproject-plugin.toml', 'plugin_success.py', None),
+    ('pyproject-plugin.toml', 'plugin_fail.py', 'plugin-fail.txt'),
+    ('pyproject-plugin-strict.toml', 'plugin_success.py', 'plugin-success-strict.txt'),
+    ('pyproject-plugin-strict.toml', 'plugin_fail.py', 'plugin-fail-strict.txt'),
 ]
 executable_modules = list({fname[:-3] for _, fname, out_fname in cases if out_fname is None})
 
 
 @pytest.mark.skipif(not (dotenv and mypy_api), reason='dotenv or mypy are not installed')
 @pytest.mark.parametrize('config_filename,python_filename,output_filename', cases)
-def test_mypy_results(config_filename, python_filename, output_filename):
+def test_mypy_results(config_filename: str, python_filename: str, output_filename: str) -> None:
     full_config_filename = f'tests/mypy/configs/{config_filename}'
     full_filename = f'tests/mypy/modules/{python_filename}'
     output_path = None if output_filename is None else Path(f'tests/mypy/outputs/{output_filename}')
 
     # Specifying a different cache dir for each configuration dramatically speeds up subsequent execution
     # It also prevents cache-invalidation-related bugs in the tests
-    cache_dir = f'.mypy_cache/test-{config_filename[:-4]}'
+    cache_dir = f'.mypy_cache/test-{os.path.splitext(config_filename)[0]}'
     command = [full_filename, '--config-file', full_config_filename, '--cache-dir', cache_dir, '--show-error-codes']
     print(f"\nExecuting: mypy {' '.join(command)}")  # makes it easier to debug as necessary
     actual_result = mypy_api.run(command)
@@ -66,15 +75,31 @@ def test_mypy_results(config_filename, python_filename, output_filename):
     assert actual_out == expected_out, actual_out
 
 
+@pytest.mark.skipif(not (dotenv and mypy_api), reason='dotenv or mypy are not installed')
+def test_bad_toml_config() -> None:
+    full_config_filename = 'tests/mypy/configs/pyproject-plugin-bad-param.toml'
+    full_filename = 'tests/mypy/modules/success.py'
+
+    # Specifying a different cache dir for each configuration dramatically speeds up subsequent execution
+    # It also prevents cache-invalidation-related bugs in the tests
+    cache_dir = '.mypy_cache/test-pyproject-plugin-bad-param'
+    command = [full_filename, '--config-file', full_config_filename, '--cache-dir', cache_dir, '--show-error-codes']
+    print(f"\nExecuting: mypy {' '.join(command)}")  # makes it easier to debug as necessary
+    with pytest.raises(ValueError) as e:
+        mypy_api.run(command)
+
+    assert str(e.value) == 'Configuration value must be a boolean for key: init_forbid_extra'
+
+
 @pytest.mark.parametrize('module', executable_modules)
-def test_success_cases_run(module):
+def test_success_cases_run(module: str) -> None:
     """
     Ensure the "success" files can actually be executed
     """
     importlib.import_module(f'tests.mypy.modules.{module}')
 
 
-def test_explicit_reexports():
+def test_explicit_reexports() -> None:
     from pydantic import __all__ as root_all
     from pydantic.main import __all__ as main
     from pydantic.networks import __all__ as networks
diff --git a/tests/requirements-linting.txt b/tests/requirements-linting.txt
index e99dad38f..b06f22806 100644
--- a/tests/requirements-linting.txt
+++ b/tests/requirements-linting.txt
@@ -4,6 +4,7 @@ flake8-quotes==3.3.0
 hypothesis==6.17.4
 isort==5.9.3
 mypy==0.910
+types-toml==0.1.5
 pycodestyle==2.7.0
 pyflakes==2.3.1
 twine==3.4.2
\ No newline at end of file