Repository: pex-tool/pex

Issue:
Pex lock creation does not handle wheels with non {`cp`,`pp`,`py`} pyver tag.
The error looks like so:
```
$ python -mpex.cli lock create lexid --style universal --resolver-version pip-2020-resolver --interpreter-constraint ">=3.7,<4"
pid 24464 -> /home/jsirois/.pex/venvs/2e77638397a501cfdb2b7c33808a3a7f5fe57413/5985ed09b49a653d6596b0e14d134c5456cf1a9f/bin/python -sE /home/jsirois/.pex/venvs/2e77638397a501cfdb2b7c33808a3a7f5fe57413/5985ed09b49a653d6596b0e14d134c5456cf1a9f/pex --disable-pip-version-check --no-python-version-warning --exists-action a --no-input --isolated -q --cache-dir /home/jsirois/.pex/pip_cache --log /tmp/pex-pip-log.n0c8ko88/pip.log download --dest /tmp/tmp5ga3zanm/usr.bin.python3.10 lexid --index-url https://pypi.org/simple --retries 5 --timeout 15 exited with 2 and STDERR:
>>> pyversion: pypy27
>>> pyversions: ['py38', 'pypy27']
>>> self: <pip._internal.models.wheel.Wheel object at 0x7f7917559de0>
ERROR: Exception:
Traceback (most recent call last):
  File "/home/jsirois/.pex/venvs/2e77638397a501cfdb2b7c33808a3a7f5fe57413/5985ed09b49a653d6596b0e14d134c5456cf1a9f/lib/python3.10/site-packages/pip/_vendor/resolvelib/resolvers.py", line 171, in _merge_into_criterion
    crit = self.state.criteria[name]
KeyError: 'lexid'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/jsirois/.pex/venvs/2e77638397a501cfdb2b7c33808a3a7f5fe57413/5985ed09b49a653d6596b0e14d134c5456cf1a9f/lib/python3.10/site-packages/pip/_internal/cli/base_command.py", line 223, in _main
    status = self.run(options, args)
  File "/home/jsirois/.pex/venvs/2e77638397a501cfdb2b7c33808a3a7f5fe57413/5985ed09b49a653d6596b0e14d134c5456cf1a9f/lib/python3.10/site-packages/pip/_internal/cli/req_command.py", line 180, in wrapper
    return func(self, options, args)
  File "/home/jsirois/.pex/venvs/2e77638397a501cfdb2b7c33808a3a7f5fe57413/5985ed09b49a653d6596b0e14d134c5456cf1a9f/lib/python3.10/site-packages/pip/_internal/commands/download.py", line 130, in run
    requirement_set = resolver.resolve(
  File "/home/jsirois/.pex/venvs/2e77638397a501cfdb2b7c33808a3a7f5fe57413/5985ed09b49a653d6596b0e14d134c5456cf1a9f/lib/python3.10/site-packages/pip/_internal/resolution/resolvelib/resolver.py", line 121, in resolve
    self._result = resolver.resolve(
  File "/home/jsirois/.pex/venvs/2e77638397a501cfdb2b7c33808a3a7f5fe57413/5985ed09b49a653d6596b0e14d134c5456cf1a9f/lib/python3.10/site-packages/pip/_vendor/resolvelib/resolvers.py", line 453, in resolve
    state = resolution.resolve(requirements, max_rounds=max_rounds)
  File "/home/jsirois/.pex/venvs/2e77638397a501cfdb2b7c33808a3a7f5fe57413/5985ed09b49a653d6596b0e14d134c5456cf1a9f/lib/python3.10/site-packages/pip/_vendor/resolvelib/resolvers.py", line 318, in resolve
    name, crit = self._merge_into_criterion(r, parent=None)
  File "/home/jsirois/.pex/venvs/2e77638397a501cfdb2b7c33808a3a7f5fe57413/5985ed09b49a653d6596b0e14d134c5456cf1a9f/lib/python3.10/site-packages/pip/_vendor/resolvelib/resolvers.py", line 173, in _merge_into_criterion
    crit = Criterion.from_requirement(self._p, requirement, parent)
  File "/home/jsirois/.pex/venvs/2e77638397a501cfdb2b7c33808a3a7f5fe57413/5985ed09b49a653d6596b0e14d134c5456cf1a9f/lib/python3.10/site-packages/pip/_vendor/resolvelib/resolvers.py", line 82, in from_requirement
    if not cands:
  File "/home/jsirois/.pex/venvs/2e77638397a501cfdb2b7c33808a3a7f5fe57413/5985ed09b49a653d6596b0e14d134c5456cf1a9f/lib/python3.10/site-packages/pip/_vendor/resolvelib/structs.py", line 124, in __bool__
    return bool(self._sequence)
  File "/home/jsirois/.pex/venvs/2e77638397a501cfdb2b7c33808a3a7f5fe57413/5985ed09b49a653d6596b0e14d134c5456cf1a9f/lib/python3.10/site-packages/pip/_internal/resolution/resolvelib/found_candidates.py", line 99, in __bool__
    return any(self)
  File "/home/jsirois/.pex/venvs/2e77638397a501cfdb2b7c33808a3a7f5fe57413/5985ed09b49a653d6596b0e14d134c5456cf1a9f/lib/python3.10/site-packages/pip/_internal/resolution/resolvelib/factory.py", line 220, in iter_index_candidates
    result = self._finder.find_best_candidate(
  File "/home/jsirois/.pex/venvs/2e77638397a501cfdb2b7c33808a3a7f5fe57413/5985ed09b49a653d6596b0e14d134c5456cf1a9f/lib/python3.10/site-packages/pip/_internal/index/package_finder.py", line 882, in find_best_candidate
    candidates = self.find_all_candidates(project_name)
  File "/home/jsirois/.pex/venvs/2e77638397a501cfdb2b7c33808a3a7f5fe57413/5985ed09b49a653d6596b0e14d134c5456cf1a9f/lib/python3.10/site-packages/pip/_internal/index/package_finder.py", line 825, in find_all_candidates
    package_links = self.process_project_url(
  File "/home/jsirois/.pex/venvs/2e77638397a501cfdb2b7c33808a3a7f5fe57413/5985ed09b49a653d6596b0e14d134c5456cf1a9f/lib/python3.10/site-packages/pip/_internal/index/package_finder.py", line 796, in process_project_url
    package_links = self.evaluate_links(
  File "/home/jsirois/.pex/venvs/2e77638397a501cfdb2b7c33808a3a7f5fe57413/5985ed09b49a653d6596b0e14d134c5456cf1a9f/lib/python3.10/site-packages/pip/_internal/index/package_finder.py", line 778, in evaluate_links
    candidate = self.get_install_candidate(link_evaluator, link)
  File "/home/jsirois/.pex/venvs/2e77638397a501cfdb2b7c33808a3a7f5fe57413/5985ed09b49a653d6596b0e14d134c5456cf1a9f/lib/python3.10/site-packages/pip/_internal/index/package_finder.py", line 757, in get_install_candidate
    is_candidate, result = link_evaluator.evaluate_link(link)
  File "/home/jsirois/.pex/venvs/2e77638397a501cfdb2b7c33808a3a7f5fe57413/5985ed09b49a653d6596b0e14d134c5456cf1a9f/lib/python3.10/site-packages/pip/_internal/index/package_finder.py", line 197, in evaluate_link
    if not wheel.supported(supported_tags):
  File "/tmp/tmplos6r6ee/_pex_pip_patches.py", line 158, in <lambda>
    Wheel.supported = lambda *args, **kwargs: all(
  File "/tmp/tmplos6r6ee/_pex_pip_patches.py", line 159, in <genexpr>
    check(*args, **kwargs) for check in supported_checks
  File "/tmp/tmplos6r6ee/_pex_pip_patches.py", line 127, in supported_version
    major = int(match.group("major"))
AttributeError: 'NoneType' object has no attribute 'group'
```

The issue here is that the two oldest releases of `lexid` have wheels with `pypy27` pyver tags. For example: https://pypi.org/project/lexid/2020.1003/#files
Which has `https://files.pythonhosted.org/packages/d0/29/c6ca27464a82787ffd9be6f19be6c2715cc660957a39ec689bc1654fd89d/lexid-2020.1003-py38.pypy27-none-any.whl`.

This patching code here is both incorrect and unhelpful when it is incorrect:
https://github.com/pantsbuild/pex/blob/06570eb41d6681a537bc561aa1a670dfb6dd2c3a/pex/resolve/locker_patches.py#L114-L122



Discussion:
The relevant spec: https://peps.python.org/pep-0425/#python-tag

The `pypy` tag is unexpected (`pp` is expected for PyPy instead), but certainly permissible.

Resolving patch:
diff --git a/pex/resolve/locker_patches.py b/pex/resolve/locker_patches.py
--- a/pex/resolve/locker_patches.py
+++ b/pex/resolve/locker_patches.py
@@ -112,21 +112,23 @@ def supported_version(self, *_args, **_kwargs):
                 versions = set()
                 is_abi3 = ["abi3"] == list(self.abis)
                 for pyversion in self.pyversions:
-                    if pyversion[:2] in ("cp", "pp", "py"):
-                        version_str = pyversion[2:]
-                        # N.B.: This overblown seeming use of an re
-                        # is necessitated by distributions like
-                        # pydantic 0.18.* which incorrectly use
-                        # `py36+`.
-                        match = re.search(r"^(?P<major>\d)(?P<minor>\d+)?", version_str)
-                        major = int(match.group("major"))
-                        minor = match.group("minor")
-                        if is_abi3 and major == 3:
-                            versions.add(major)
-                        elif minor:
-                            versions.add((major, int(minor)))
-                        else:
-                            versions.add(major)
+                    # For the format, see: https://peps.python.org/pep-0425/#python-tag
+                    match = re.search(r"^(?P<impl>\D{2,})(?P<major>\d)(?P<minor>\d+)?", pyversion)
+                    if not match:
+                        continue
+
+                    impl = match.group("impl")
+                    if impl not in ("cp", "pp", "py", "cpython", "pypy"):
+                        continue
+
+                    major = int(match.group("major"))
+                    minor = match.group("minor")
+                    if is_abi3 and major == 3:
+                        versions.add(major)
+                    elif minor:
+                        versions.add((major, int(minor)))
+                    else:
+                        versions.add(major)
 
                 self._versions = versions
 


Test patch:
diff --git a/tests/integration/cli/commands/test_issue_2059.py b/tests/integration/cli/commands/test_issue_2059.py
new file mode 100644
--- /dev/null
+++ b/tests/integration/cli/commands/test_issue_2059.py
@@ -0,0 +1,54 @@
+# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+import os.path
+import sys
+
+from pex.cli.testing import run_pex3
+from pex.compatibility import commonpath
+from pex.testing import PY310, ensure_python_interpreter, run_pex_command
+from pex.typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from typing import Any
+
+
+def test_pypy_impl_tag_handling(tmpdir):
+    # type: (Any) -> None
+
+    pex_root = os.path.realpath(os.path.join(str(tmpdir), "pex_root"))
+    lock = os.path.join(str(tmpdir), "lock.json")
+    run_pex3(
+        "lock",
+        "create",
+        "--pex-root",
+        pex_root,
+        "lexid",
+        "--style",
+        "universal",
+        "--resolver-version",
+        "pip-2020-resolver",
+        "--interpreter-constraint",
+        ">=3.7,<4",
+        "-o",
+        lock,
+        "--indent",
+        "2",
+    ).assert_success()
+
+    python = sys.executable if sys.version_info[:2] >= (3, 7) else ensure_python_interpreter(PY310)
+    result = run_pex_command(
+        args=[
+            "--pex-root",
+            pex_root,
+            "--runtime-pex-root",
+            pex_root,
+            "--lock",
+            lock,
+            "--",
+            "-c",
+            "import lexid; print(lexid.__file__)",
+        ],
+        python=python,
+    )
+    result.assert_success()
+    assert pex_root == commonpath([pex_root, os.path.realpath(result.output.strip())])