Repository: pex-tool/pex

Issue:
The `--resolve-local-platforms` option does not work with `--complete-platforms`
The `--resolve-local-platforms` option allows Pex to substitute a local interpreter for a `--platform` whenever there is a local interpreter available that fits the abbreviated `--platform` description. The option also applies to `--complete-platform` and in either case the test of a match is here:
https://github.com/pantsbuild/pex/blob/b1343ffbc4ef58c456d072c4f700ad23f1f51743/pex/resolve/target_configuration.py#L139-L161

The test is that there is an intersection between the local interpreter's compatibility tags and those of the foreign platform. Intersection is problematic when the local interpreter supports newer ABIs than the foreign platform. In this case, the local interpreter is used as a substitute for the foreign platform which allows wheels with newer ABIs than the foreign platform can support to be resolved. 

To correct this problem, a local interpreter should only be substituted for a foreign platform if its compatibility tags form a subset of the foreign platform tags. In the case of abbreviated `--platform` there is not much that can be done differently than the current intersection test since the full accurate set of tags is unknown. For the `--complete-platform` case, however, a subset relationship can be checked instead.


Discussion:

Resolving patch:
diff --git a/pex/resolve/target_configuration.py b/pex/resolve/target_configuration.py
--- a/pex/resolve/target_configuration.py
+++ b/pex/resolve/target_configuration.py
@@ -20,7 +20,7 @@
 from pex.variables import ENV
 
 if TYPE_CHECKING:
-    from typing import Iterator, Optional, Tuple
+    from typing import FrozenSet, Iterator, Optional, Set, Tuple
 
     import attr  # vendor:skip
 else:
@@ -100,6 +100,73 @@ class InterpreterConstraintsNotSatisfied(TargetConfigurationError):
     """Indicates no interpreter meeting the requested constraints could be found."""
 
 
+def _interpreter_compatible_platforms(
+    all_platforms,  # type: OrderedDict[Optional[Platform], Optional[CompletePlatform]]
+    candidate_interpreter,  # type: PythonInterpreter
+):
+    # type: (...) -> FrozenSet[Platform]
+    resolved_platforms = candidate_interpreter.supported_platforms.intersection(
+        all_platforms
+    )  # type: FrozenSet[Platform]
+    incompatible_platforms = set()  # type: Set[Platform]
+
+    for resolved_platform in resolved_platforms:
+        requested_complete = all_platforms[resolved_platform]
+        if requested_complete is not None:
+            # if there was an explicit complete platform specified, only use the local interpreter
+            # when the interpreter's tags are a subset of the complete platform's: tags supported by
+            # the interpreter but not the complete platform may result in incompatible wheels being
+            # chosen, if the interpreter was used directly
+            candidate_complete = CompletePlatform.from_interpreter(candidate_interpreter)
+            requested_tags = set(requested_complete.supported_tags)
+            missing_tags = OrderedSet(
+                t for t in candidate_complete.supported_tags if t not in requested_tags
+            )
+            if missing_tags:
+                TRACER.log(
+                    "Rejected resolution of {} for platform {} due to supporting {} extra tags".format(
+                        candidate_interpreter,
+                        resolved_platform,
+                        len(missing_tags),
+                    ),
+                    V=3,
+                )
+                TRACER.log(
+                    "Extra tags supported by {} for platform {} but not supported by specified complete platform: {}".format(
+                        candidate_interpreter,
+                        resolved_platform,
+                        ", ".join(map(str, missing_tags)),
+                    ),
+                    V=9,
+                )
+                # keep iterating to give information about each of the relevant platforms
+                incompatible_platforms.add(resolved_platform)
+                continue
+
+        TRACER.log(
+            "Provisionally accepted resolution of {} for platform {} due to matching {}".format(
+                candidate_interpreter,
+                resolved_platform,
+                "tags"
+                if requested_complete is not None
+                else "platform (no complete platform and thus no tags to check)",
+            ),
+            V=3,
+        )
+
+    if incompatible_platforms:
+        TRACER.log(
+            "Rejected interpreter {} due to being incompatible with {}: {}".format(
+                candidate_interpreter,
+                "a platform" if len(incompatible_platforms) == 1 else "some platforms",
+                ", ".join(sorted(map(str, incompatible_platforms))),
+            )
+        )
+        return frozenset()
+
+    return resolved_platforms
+
+
 @attr.s(frozen=True)
 class TargetConfiguration(object):
     interpreter_configuration = attr.ib(
@@ -151,8 +218,8 @@ def resolve_targets(self):
                 )  # type: OrderedSet[PythonInterpreter]
                 candidate_interpreters.add(PythonInterpreter.get())
                 for candidate_interpreter in candidate_interpreters:
-                    resolved_platforms = candidate_interpreter.supported_platforms.intersection(
-                        all_platforms
+                    resolved_platforms = _interpreter_compatible_platforms(
+                        all_platforms, candidate_interpreter
                     )
                     if resolved_platforms:
                         for resolved_platform in resolved_platforms:


Test patch:
diff --git a/tests/resolve/test_target_options.py b/tests/resolve/test_target_options.py
--- a/tests/resolve/test_target_options.py
+++ b/tests/resolve/test_target_options.py
@@ -22,7 +22,7 @@
 from pex.variables import ENV
 
 if TYPE_CHECKING:
-    from typing import Any, Iterable, List, Optional, Tuple
+    from typing import Any, Dict, Iterable, List, Optional, Tuple
 
 
 def compute_target_configuration(
@@ -384,3 +384,148 @@ def assert_local_platforms(
         expected_interpreter=py27,
         expected_interpreters=(py310, py27),
     )
+
+
+def test_configure_resolve_local_platforms_with_complete_platforms(
+    tmpdir,  # type: Any
+    parser,  # type: ArgumentParser
+    py27,  # type: PythonInterpreter
+    py38,  # type: PythonInterpreter
+    py310,  # type: PythonInterpreter
+):
+    # type: (...) -> None
+    target_options.register(parser)
+
+    path_env_var = path_for(py27, py38, py310)
+
+    def dump_complete_platform(
+        name,  # type: str
+        marker_environment,  # type: Dict[str, str]
+        compatible_tags,  # type: List[str]
+        **extra_fields  # type: Any
+    ):
+        # type: (...) -> str
+        path = os.path.join(str(tmpdir), name)
+        with open(path, "w") as fp:
+            json.dump(
+                dict(
+                    marker_environment=marker_environment,
+                    compatible_tags=compatible_tags,
+                    **extra_fields
+                ),
+                fp,
+            )
+        return path
+
+    def assert_local_platforms(
+        complete_platforms,  # type: Iterable[str]
+        expected_platforms,  # type: Iterable[str]
+        expected_interpreter,  # type: Optional[PythonInterpreter]
+        expected_interpreters=None,  # type: Optional[Tuple[PythonInterpreter, ...]]
+    ):
+        # type: (...) -> None
+        args = ["--python-path", path_env_var, "--resolve-local-platforms"]
+        args.extend(
+            itertools.chain.from_iterable(("--complete-platform", p) for p in complete_platforms)
+        )
+        targets = compute_target_configuration(parser, args)
+        # assert tuple(Platform.create(ep) for ep in expected_platforms) == targets.platforms
+        assert_interpreters_configured(targets, expected_interpreter, expected_interpreters)
+
+    py38_complete = dump_complete_platform(
+        "py38",
+        py38.identity.env_markers.as_dict(),
+        py38.identity.supported_tags.to_string_list(),
+    )
+    py38_extra_complete = dump_complete_platform(
+        "py38_extra",
+        py38.identity.env_markers.as_dict(),
+        py38.identity.supported_tags.to_string_list() + ["py3-none-manylinux_2_9999_x86_64"],
+    )
+
+    py38_subset_tags = py38.identity.supported_tags.to_string_list()[:-10]
+    # make the platform different
+    py38_subset_tags[0:2] = py38_subset_tags[0:2:-1]
+    py38_subset_complete = dump_complete_platform(
+        "py38_subset",
+        py38.identity.env_markers.as_dict(),
+        py38_subset_tags,
+    )
+    py310_complete = dump_complete_platform(
+        "py310",
+        py310.identity.env_markers.as_dict(),
+        py310.identity.supported_tags.to_string_list(),
+    )
+    py39999_env_markers = py310.identity.env_markers.as_dict()
+    py39999_env_markers.update(
+        implementation_version="3.9999.0",
+        python_full_version="3.9999.0",
+        python_version="3.9999",
+        sys_platform="linux",
+    )
+    py39999_complete = dump_complete_platform(
+        "other",
+        py39999_env_markers,
+        [
+            "py39999-none-any",
+            "py3-none-any",
+            "py39998-none-any",
+            "py39997-none-any",
+            # you get the idea...
+            "py310-none-any",
+            "py39-none-any",
+            "py38-none-any",
+            "py37-none-any",
+            "py36-none-any",
+            "py35-none-any",
+            "py34-none-any",
+            "py33-none-any",
+            "py32-none-any",
+            "py31-none-any",
+            "py30-none-any",
+        ],
+    )
+
+    # exact match, yay
+    assert_local_platforms(
+        complete_platforms=[py38_complete],
+        expected_platforms=[str(py38.platform)],
+        expected_interpreter=py38,
+    )
+
+    # the interpreter doesn't support some tags, but that's fine
+    assert_local_platforms(
+        complete_platforms=[py38_extra_complete],
+        expected_platforms=[str(py38.platform)],
+        expected_interpreter=py38,
+    )
+
+    # # the interpreter has some tags it supports that this complete platform does not
+    assert_local_platforms(
+        complete_platforms=[py38_subset_complete],
+        expected_platforms=[],
+        expected_interpreter=None,
+    )
+
+    # as above, but now with multiple complete platforms that apply to one interpreter (two
+    # compatible, one not)
+    assert_local_platforms(
+        complete_platforms=[py38_subset_complete, py38_complete, py38_extra_complete],
+        expected_platforms=[],
+        expected_interpreter=None,
+    )
+
+    # wildly different
+    assert_local_platforms(
+        complete_platforms=[py39999_complete],
+        expected_platforms=[],
+        expected_interpreter=None,
+    )
+
+    # multiple
+    assert_local_platforms(
+        complete_platforms=[py38_complete, py310_complete],
+        expected_platforms=[str(py38.platform), str(py310.platform)],
+        expected_interpreter=py38,
+        expected_interpreters=(py38, py310),
+    )