Repository: pantsbuild/pants

Issue:
Configurable tool versions should probably have a hardcoded constraint range.
Any tool Pants uses must have the CLI syntax Pants expects when Pants is released. If a user works around a tool issue by configuring the tool version, that can leave a custom tool configuration that works at the moment, but fails mysteriously when Pants is upgraded to a version that expects the tool to be able to support new arguments. We should probably support a floor for this sort of case and at least log a warning. More generally we probably need to support a range.

This surfaced in the wild with a user that had configured the following when migrating from Pants 2.2 to 2.3:
```toml
[download-pex-bin]
version = "v2.1.33"
known_versions = [
    "v2.1.33|darwin|7f9f6168691fe83f38fbc248bd8629a152cf2a8833be1afdc06219f70fbb6064|3596348",
    "v2.1.33|linux|7f9f6168691fe83f38fbc248bd8629a152cf2a8833be1afdc06219f70fbb6064|3596348"
]
```
Upon upgrade to Pants 2.5.x which ships using Pex 2.1.42 they hit mysterious Pex errors:
```
21:27:43.95 [INFO] Starting: Resolving plugins: toolchain.pants.plugin==0.11.0
21:27:44.31 [INFO] Completed: Resolving plugins: toolchain.pants.plugin==0.11.0
21:27:44.31 [ERROR] 1 Exception encountered:
  ProcessExecutionFailure: Process 'Resolving plugins: toolchain.pants.plugin==0.11.0' failed with exit code 2.
stdout:
stderr:
usage: pex [-o OUTPUT.PEX] [options] [-- arg1 arg2 ...]
pex builds a PEX (Python Executable) file based on the given specifications: sources, requirements, their dependencies and other options.
pex: error: unrecognized arguments: toolchain.pants.plugin==0.11.0
```

This error, in the end, is because Pex 2.1.33 does not understand arguments added in Pex 2.1.42.


Discussion:

Resolving patch:
diff --git a/src/python/pants/backend/python/util_rules/pex_cli.py b/src/python/pants/backend/python/util_rules/pex_cli.py
--- a/src/python/pants/backend/python/util_rules/pex_cli.py
+++ b/src/python/pants/backend/python/util_rules/pex_cli.py
@@ -39,6 +39,7 @@ class PexBinary(TemplatedExternalTool):
 
     default_version = "v2.1.42"
     default_url_template = "https://github.com/pantsbuild/pex/releases/download/{version}/pex"
+    version_constraints = ">=2.1.42, <3.0"
 
     @classproperty
     def default_known_versions(cls):
diff --git a/src/python/pants/core/util_rules/external_tool.py b/src/python/pants/core/util_rules/external_tool.py
--- a/src/python/pants/core/util_rules/external_tool.py
+++ b/src/python/pants/core/util_rules/external_tool.py
@@ -1,11 +1,15 @@
 # Copyright 2020 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
+import logging
 import textwrap
 from abc import ABCMeta, abstractmethod
 from dataclasses import dataclass
+from enum import Enum
 from typing import Dict, List, Optional, Tuple, cast
 
+from pkg_resources import Requirement
+
 from pants.core.util_rules import archive
 from pants.core.util_rules.archive import ExtractedArchive
 from pants.engine.fs import Digest, DownloadFile, FileDigest
@@ -15,6 +19,8 @@
 from pants.util.logging import LogLevel
 from pants.util.meta import classproperty
 
+logger = logging.getLogger(__name__)
+
 
 class UnknownVersion(Exception):
     pass
@@ -24,6 +30,18 @@ class ExternalToolError(Exception):
     pass
 
 
+class UnsupportedVersion(ExternalToolError):
+    """The specified version of the tool is not supported, according to the given version
+    constraints."""
+
+
+class UnsupportedVersionUsage(Enum):
+    """What action to take in case the requested version of the tool is not supported."""
+
+    RaiseError = "error"
+    LogWarning = "warning"
+
+
 @dataclass(frozen=True)
 class ExternalToolRequest:
     download_file_request: DownloadFile
@@ -52,6 +70,8 @@ class MyExternalTool(ExternalTool):
           "1.2.3|linux |1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd|333333",
         ]
 
+        version_constraints = ">=1.2.3, <2.0"
+
         def generate_url(self, plat: Platform) -> str:
             ...
 
@@ -68,10 +88,19 @@ def my_rule(my_external_tool: MyExternalTool) -> Foo:
         ...
     """
 
-    # The default values for --version and --known-versions.
+    # The default values for --version and --known-versions, and the supported versions.
     # Subclasses must set appropriately.
     default_version: str
     default_known_versions: List[str]
+    version_constraints: Optional[str] = None
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        if self.version_constraints:
+            constraints = Requirement.parse(f"{self.name}{self.version_constraints}")
+            self.check_version_constraints(
+                self.version, constraints, self.options.use_unsupported_version
+            )
 
     @classproperty
     def name(cls):
@@ -89,7 +118,12 @@ def register_options(cls, register):
             type=str,
             default=cls.default_version,
             advanced=True,
-            help=f"Use this version of {cls.name}.",
+            help=f"Use this version of {cls.name}."
+            + (
+                f"\n\nSupported {cls.name} versions: {cls.version_constraints}"
+                if cls.version_constraints
+                else ""
+            ),
         )
 
         help_str = textwrap.dedent(
@@ -121,6 +155,20 @@ def register_options(cls, register):
             help=help_str,
         )
 
+        register(
+            "--use-unsupported-version",
+            advanced=True,
+            type=UnsupportedVersionUsage,
+            help=textwrap.dedent(
+                f"""
+                What action to take in case the requested version of {cls.name} is not supported.
+
+                Supported {cls.name} versions: {cls.version_constraints if cls.version_constraints else "unspecified"}
+                """
+            ),
+            default=UnsupportedVersionUsage.RaiseError,
+        )
+
     @property
     def version(self) -> str:
         return cast(str, self.options.version)
@@ -175,6 +223,36 @@ def get_request(self, plat: Platform) -> ExternalToolRequest:
             f"{self.known_versions}"
         )
 
+    @classmethod
+    def check_version_constraints(
+        cls, version: str, constraints: Requirement, action: UnsupportedVersionUsage
+    ) -> None:
+        if constraints.specifier.contains(version):  # type: ignore[attr-defined]
+            # all ok
+            return None
+
+        msg = [
+            f"The option [{cls.options_scope}].version is set to {version}, which is not compatible",
+            f"with what this release of Pants expects: {constraints}.",
+            "Please update the version to a supported value, or consider using a different Pants",
+            "release if you cannot change the version.",
+        ]
+
+        if action is UnsupportedVersionUsage.LogWarning:
+            msg.extend(
+                [
+                    "Alternatively, you can ignore this warning (at your own peril) by adding this",
+                    "to the GLOBAL section of pants.toml:",
+                    f"""ignore_warnings = ["The option [{cls.options_scope}].version is set to"].""",
+                ]
+            )
+            logger.warning(" ".join(msg))
+        elif action is UnsupportedVersionUsage.RaiseError:
+            msg.append(
+                f"Alternatively, update [{cls.options_scope}].use_unsupported_version to be 'warning'."
+            )
+            raise UnsupportedVersion(" ".join(msg))
+
 
 class TemplatedExternalTool(ExternalTool):
     """Extends the ExternalTool to allow url templating for custom/self-hosted source.
diff --git a/src/python/pants/testutil/pytest_util.py b/src/python/pants/testutil/pytest_util.py
new file mode 100644
--- /dev/null
+++ b/src/python/pants/testutil/pytest_util.py
@@ -0,0 +1,22 @@
+# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from contextlib import contextmanager
+
+
+@contextmanager
+def no_exception():
+    """Useful replacement for `pytest.raises()`, when there is no exception to be expected.
+
+    When declaring parametrized tests, the test function can take a exceptions
+    expectation as input, and always use a with-block for the code under test.
+
+        @pytest.mark.parametrize('answer, expect_raises', [
+            (42, no_exception()),
+            (12, pytest.raises(WrongAnswer)),
+        ])
+        def test_search_for_the_meaning_of_life_universe_and_everything(answer, expect_raises):
+            with expect_raises:
+                computer.validate_result(answer)
+    """
+    yield None


Test patch:
diff --git a/src/python/pants/core/util_rules/external_tool_test.py b/src/python/pants/core/util_rules/external_tool_test.py
--- a/src/python/pants/core/util_rules/external_tool_test.py
+++ b/src/python/pants/core/util_rules/external_tool_test.py
@@ -1,6 +1,9 @@
 # Copyright 2020 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
+import logging
+import re
+
 import pytest
 
 from pants.core.util_rules.external_tool import (
@@ -9,10 +12,13 @@
     ExternalToolRequest,
     TemplatedExternalTool,
     UnknownVersion,
+    UnsupportedVersion,
+    UnsupportedVersionUsage,
 )
 from pants.engine.fs import DownloadFile, FileDigest
 from pants.engine.platform import Platform
 from pants.testutil.option_util import create_subsystem
+from pants.testutil.pytest_util import no_exception
 
 
 class FooBar(ExternalTool):
@@ -105,3 +111,112 @@ def do_test(
         create_subsystem(
             FooBar, version="9.9.9", known_versions=FooBar.default_known_versions
         ).get_request(Platform.darwin)
+
+
+class ConstrainedTool(TemplatedExternalTool):
+    name = "foobar"
+    options_scope = "foobar"
+    version_constraints = ">3.2.1, <3.8"
+    default_version = "v3.4.7"
+    default_known_versions = [
+        "v3.2.0|darwin   |1102324cdaacd589e50b8b7770595f220f54e18a1d76ee3c445198f80ab865b8|123346",
+        "v3.2.0|linux_ppc|39e5d64b0f31117c94651c880d0a776159e49eab42b2066219569934b936a5e7|124443",
+        "v3.2.0|linux    |c0c667fb679a8221bed01bffeed1f80727c6c7827d0cbd8f162195efb12df9e4|121212",
+        "v3.4.7|darwin   |9d0e18cd74b918c7b3edd0203e75569e0c8caecb1367b3be409b45e28514f5be|123321",
+        "v3.4.7|linux    |a019dfc4b32d63c1392aa264aed2253c1e0c2fb09216f8e2cc269bbfb8bb49b5|134213",
+    ]
+    default_url_template = "https://foobar.org/bin/v{version}/foobar-{version}-{platform}.tgz"
+    default_url_platform_mapping = {
+        "darwin": "osx-x86_64",
+        "linux": "linux-x86_64",
+    }
+
+    def generate_exe(self, plat: Platform) -> str:
+        return f"foobar-{self.version}/bin/foobar"
+
+
+@pytest.mark.parametrize(
+    "version, action, assert_expectation, expect_logged",
+    [
+        (
+            "v1.2.3",
+            UnsupportedVersionUsage.RaiseError,
+            pytest.raises(
+                UnsupportedVersion,
+                match=re.escape(
+                    "The option [foobar].version is set to v1.2.3, which is not compatible with what this release of Pants expects: foobar<3.8,>3.2.1. "
+                    "Please update the version to a supported value, or consider using a different Pants release if you cannot change the version. "
+                    "Alternatively, update [foobar].use_unsupported_version to be 'warning'."
+                ),
+            ),
+            None,
+        ),
+        (
+            "v3.2.2",
+            UnsupportedVersionUsage.RaiseError,
+            pytest.raises(
+                UnknownVersion, match="No known version of foobar v3.2.2 for darwin found in"
+            ),
+            None,
+        ),
+        (
+            "v3.4.7",
+            UnsupportedVersionUsage.RaiseError,
+            no_exception(),
+            None,
+        ),
+        (
+            "v3.8.0",
+            UnsupportedVersionUsage.RaiseError,
+            pytest.raises(
+                UnsupportedVersion,
+                match=re.escape(
+                    "The option [foobar].version is set to v3.8.0, which is not compatible with what this release of Pants expects: foobar<3.8,>3.2.1. "
+                    "Please update the version to a supported value, or consider using a different Pants release if you cannot change the version. "
+                    "Alternatively, update [foobar].use_unsupported_version to be 'warning'."
+                ),
+            ),
+            None,
+        ),
+        (
+            "v3.8.0",
+            UnsupportedVersionUsage.LogWarning,
+            pytest.raises(
+                UnknownVersion, match="No known version of foobar v3.8.0 for darwin found in"
+            ),
+            [
+                (
+                    logging.WARNING,
+                    (
+                        "The option [foobar].version is set to v3.8.0, which is not compatible with what this release of Pants expects: foobar<3.8,>3.2.1. "
+                        "Please update the version to a supported value, or consider using a different Pants release if you cannot change the version. "
+                        "Alternatively, you can ignore this warning (at your own peril) by adding this to the GLOBAL section of pants.toml: "
+                        'ignore_warnings = ["The option [foobar].version is set to"].'
+                    ),
+                )
+            ],
+        ),
+    ],
+)
+def test_version_constraints(caplog, version, action, assert_expectation, expect_logged) -> None:
+    caplog.set_level(logging.DEBUG)
+    caplog.clear()
+
+    with assert_expectation:
+        create_subsystem(
+            ConstrainedTool,
+            version=version,
+            use_unsupported_version=action,
+            known_versions=ConstrainedTool.default_known_versions,
+            url_template=ConstrainedTool.default_url_template,
+            url_platform_mapping=ConstrainedTool.default_url_platform_mapping,
+        ).get_request(Platform.darwin)
+
+    if expect_logged:
+        assert len(caplog.records) == len(expect_logged)
+        for idx, (lvl, msg) in enumerate(expect_logged):
+            log_record = caplog.records[idx]
+            assert msg in log_record.message
+            assert lvl == log_record.levelno
+    else:
+        assert not caplog.records