Repository: pantsbuild/pants

Issue:
Fix PythonProtobufSubsystem mypy-protobuf interpreter selection.
The PythonProtobufSubsystem currently allows turning on generation of type stubs for generated protobuf code as well as selecting the version of the mypy-protobuf protoc plugin to use:
https://github.com/pantsbuild/pants/blob/652df2cb5138eb01a6be7935023c56e36563b1cd/src/python/pants/backend/codegen/protobuf/python/python_protobuf_subsystem.py#L34-L52

It does not, however allow setting the interpreter constraints required by the mypy-protobuf plugin and we hard code them:
https://github.com/pantsbuild/pants/blob/652df2cb5138eb01a6be7935023c56e36563b1cd/src/python/pants/backend/codegen/protobuf/python/rules.py#L108-L111

The comment there is misleading, since the arbitrary choice of interpreter constraint may not in fact work with the user configured mypy-protobuf version and since the python selected does have everything to do with which interpreter is used to run the plugin here:
https://github.com/pantsbuild/pants/blob/652df2cb5138eb01a6be7935023c56e36563b1cd/src/python/pants/backend/codegen/protobuf/python/rules.py#L146

That bin script will have the absolute path of the interpreter used to build the mypy pex as its shebang since that's how pip handles installation of console scripts.


Discussion:
This bit https://github.com/pantsbuild/pants/pull/11662 with failures like: https://github.com/pantsbuild/pants/runs/2104789484?check_suite_focus=true#step:10:241

Resolving patch:
diff --git a/src/python/pants/backend/codegen/protobuf/python/python_protobuf_subsystem.py b/src/python/pants/backend/codegen/protobuf/python/python_protobuf_subsystem.py
--- a/src/python/pants/backend/codegen/protobuf/python/python_protobuf_subsystem.py
+++ b/src/python/pants/backend/codegen/protobuf/python/python_protobuf_subsystem.py
@@ -4,11 +4,14 @@
 from typing import cast
 
 from pants.backend.codegen.protobuf.target_types import ProtobufDependencies
+from pants.backend.python.subsystems.python_tool_base import PythonToolRequirementsBase
+from pants.base.deprecated import deprecated_conditional
 from pants.engine.addresses import Addresses, UnparsedAddressInputs
 from pants.engine.rules import Get, collect_rules, rule
 from pants.engine.target import InjectDependenciesRequest, InjectedDependencies
 from pants.engine.unions import UnionRule
 from pants.option.custom_types import target_option
+from pants.option.ranked_value import Rank
 from pants.option.subsystem import Subsystem
 from pants.util.docutil import docs_url
 
@@ -43,7 +46,6 @@ def register_options(cls, register):
         register(
             "--mypy-plugin-version",
             type=str,
-            default="mypy-protobuf==2.4",
             advanced=True,
             help=(
                 "The pip-style requirement string for `mypy-protobuf`. You must still set "
@@ -59,9 +61,42 @@ def runtime_dependencies(self) -> UnparsedAddressInputs:
     def mypy_plugin(self) -> bool:
         return cast(bool, self.options.mypy_plugin)
 
-    @property
-    def mypy_plugin_version(self) -> str:
-        return cast(str, self.options.mypy_plugin_version)
+
+class PythonProtobufMypyPlugin(PythonToolRequirementsBase):
+    options_scope = PythonProtobufSubsystem.subscope("mypy-plugin")
+    help = (
+        "Configuration of the mypy-protobuf type stub generation plugin for the Protobuf Python "
+        "backend."
+    )
+
+    default_version = "mypy-protobuf==2.4"
+    register_interpreter_constraints = True
+    default_interpreter_constraints = ["CPython>=3.6"]
+
+    def plugin_requirement(self, python_protobuf_subsystem: PythonProtobufSubsystem) -> str:
+        deprecated_key = "mypy_plugin_version"
+        mypy_plugin_requirement = python_protobuf_subsystem.options.get(deprecated_key)
+        if mypy_plugin_requirement:
+            # TODO(John Sirois): When removing this deprecation, its safe to remove the
+            #  `--mypy-plugin-version` option registration in PythonProtobufSubsystem above since
+            #  the `version` option registered by this subsystem is in a subscope picked to shadow
+            #  the deprecated option in flag and environment variable namespaces; i.e.: both
+            #  --python-protobuf-mypy-plugin-version and PANTS_PYTHON_PROTOBUF_MYPY_PLUGIN_VERSION
+            #  will continue to work."
+            deprecated_conditional(
+                predicate=lambda: (
+                    python_protobuf_subsystem.options.get_rank(deprecated_key) == Rank.CONFIG
+                ),
+                removal_version="2.5.0.dev0",
+                entity_description=f"[{PythonProtobufSubsystem.options_scope}] {deprecated_key}",
+                hint_message=(
+                    f"Instead of configuring the `{deprecated_key}` in the "
+                    f"`[{PythonProtobufSubsystem.options_scope}]` options section, configure the "
+                    f"`version` in the `[{self.options_scope}]` options section."
+                ),
+            )
+            return cast(str, mypy_plugin_requirement)
+        return self.requirement
 
 
 class InjectPythonProtobufDependencies(InjectDependenciesRequest):
diff --git a/src/python/pants/backend/codegen/protobuf/python/rules.py b/src/python/pants/backend/codegen/protobuf/python/rules.py
--- a/src/python/pants/backend/codegen/protobuf/python/rules.py
+++ b/src/python/pants/backend/codegen/protobuf/python/rules.py
@@ -6,7 +6,10 @@
 from pants.backend.codegen.protobuf.protoc import Protoc
 from pants.backend.codegen.protobuf.python.additional_fields import PythonSourceRootField
 from pants.backend.codegen.protobuf.python.grpc_python_plugin import GrpcPythonPlugin
-from pants.backend.codegen.protobuf.python.python_protobuf_subsystem import PythonProtobufSubsystem
+from pants.backend.codegen.protobuf.python.python_protobuf_subsystem import (
+    PythonProtobufMypyPlugin,
+    PythonProtobufSubsystem,
+)
 from pants.backend.codegen.protobuf.target_types import ProtobufGrpcToggle, ProtobufSources
 from pants.backend.python.target_types import PythonSources
 from pants.backend.python.util_rules import pex
@@ -56,6 +59,7 @@ async def generate_python_from_protobuf(
     protoc: Protoc,
     grpc_python_plugin: GrpcPythonPlugin,
     python_protobuf_subsystem: PythonProtobufSubsystem,
+    python_protobuf_mypy_plugin: PythonProtobufMypyPlugin,
 ) -> GeneratedSources:
     download_protoc_request = Get(
         DownloadedExternalTool, ExternalToolRequest, protoc.get_request(Platform.current)
@@ -102,14 +106,12 @@ async def generate_python_from_protobuf(
     mypy_request = PexRequest(
         output_filename="mypy_protobuf.pex",
         internal_only=True,
-        requirements=PexRequirements([python_protobuf_subsystem.mypy_plugin_version]),
-        # TODO(John Sirois): Fix these interpreter constraints to track the actual
-        #  python requirement of the mypy_plugin_version or else plumb an option for
-        #  manually setting the constraint to track what mypy_plugin_version needs:
-        #  https://github.com/pantsbuild/pants/issues/11565
-        # Here we guess a constraint that will likely work with any mypy_plugin_version
-        # selected.
-        interpreter_constraints=PexInterpreterConstraints(["CPython>=3.5"]),
+        requirements=PexRequirements(
+            [python_protobuf_mypy_plugin.plugin_requirement(python_protobuf_subsystem)]
+        ),
+        interpreter_constraints=PexInterpreterConstraints(
+            python_protobuf_mypy_plugin.interpreter_constraints
+        ),
     )
 
     if python_protobuf_subsystem.mypy_plugin:
diff --git a/src/python/pants/backend/python/subsystems/python_tool_base.py b/src/python/pants/backend/python/subsystems/python_tool_base.py
--- a/src/python/pants/backend/python/subsystems/python_tool_base.py
+++ b/src/python/pants/backend/python/subsystems/python_tool_base.py
@@ -16,6 +16,9 @@ class PythonToolRequirementsBase(Subsystem):
     # Subclasses do not need to override.
     default_extra_requirements: ClassVar[Sequence[str]] = []
 
+    default_interpreter_constraints: ClassVar[Sequence[str]] = []
+    register_interpreter_constraints: ClassVar[bool] = False
+
     @classmethod
     def register_options(cls, register):
         super().register_options(register)
@@ -36,9 +39,24 @@ def register_options(cls, register):
             "tool allows you to install plugins or if you need to constrain a dependency to "
             "a certain version.",
         )
+        if cls.default_interpreter_constraints and not cls.register_interpreter_constraints:
+            raise ValueError(
+                f"`default_interpreter_constraints` are configured for `{cls.options_scope}`, but "
+                "`register_interpreter_constraints` is not set to `True`, so the "
+                "`--interpreter-constraints` option will not be registered. Did you mean to set "
+                "this?"
+            )
+        if cls.register_interpreter_constraints:
+            register(
+                "--interpreter-constraints",
+                type=list,
+                advanced=True,
+                default=cls.default_interpreter_constraints,
+                help="Python interpreter constraints for this tool.",
+            )
 
     @property
-    def version(self) -> str:
+    def requirement(self) -> str:
         return cast(str, self.options.version)
 
     @property
@@ -47,7 +65,11 @@ def extra_requirements(self) -> Tuple[str, ...]:
 
     @property
     def all_requirements(self) -> Tuple[str, ...]:
-        return (self.options.version, *self.options.extra_requirements)
+        return (self.requirement, *self.extra_requirements)
+
+    @property
+    def interpreter_constraints(self) -> Tuple[str, ...]:
+        return tuple(self.options.interpreter_constraints)
 
 
 class PythonToolBase(PythonToolRequirementsBase):
@@ -55,9 +77,6 @@ class PythonToolBase(PythonToolRequirementsBase):
 
     # Subclasses must set.
     default_main: ClassVar[MainSpecification]
-    # Subclasses do not need to override.
-    default_interpreter_constraints: ClassVar[Sequence[str]] = []
-    register_interpreter_constraints: ClassVar[bool] = False
 
     @classmethod
     def register_options(cls, register):
@@ -86,26 +105,6 @@ def register_options(cls, register):
             ),
         )
 
-        if cls.default_interpreter_constraints and not cls.register_interpreter_constraints:
-            raise ValueError(
-                f"`default_interpreter_constraints` are configured for `{cls.options_scope}`, but "
-                "`register_interpreter_constraints` is not set to `True`, so the "
-                "`--interpreter-constraints` option will not be registered. Did you mean to set "
-                "this?"
-            )
-        if cls.register_interpreter_constraints:
-            register(
-                "--interpreter-constraints",
-                type=list,
-                advanced=True,
-                default=cls.default_interpreter_constraints,
-                help="Python interpreter constraints for this tool.",
-            )
-
-    @property
-    def interpreter_constraints(self) -> Tuple[str, ...]:
-        return tuple(self.options.interpreter_constraints)
-
     @property
     def main(self) -> MainSpecification:
         is_default_console_script = self.options.is_default("console_script")


Test patch: