Repository: pex-tool/pex

Issue:
Add official CPython 3.11 support
The beta / feature freeze is slated to land Friday May 6th: https://peps.python.org/pep-0664/
It would be good to get CI shards set up for 3.11 shortly thereafter to set the stage for blessing 3.11 the day the Python folks do.


Discussion:

Resolving patch:
diff --git a/pex/interpreter_constraints.py b/pex/interpreter_constraints.py
--- a/pex/interpreter_constraints.py
+++ b/pex/interpreter_constraints.py
@@ -189,10 +189,10 @@ def iter_compatible_versions(
     # N.B.: Pex does not support the missing 3.x versions here.
     PythonVersion(Lifecycle.EOL, 3, 5, 10),
     PythonVersion(Lifecycle.EOL, 3, 6, 15),
-    PythonVersion(Lifecycle.STABLE, 3, 7, 12),
-    PythonVersion(Lifecycle.STABLE, 3, 8, 12),
-    PythonVersion(Lifecycle.STABLE, 3, 9, 10),
-    PythonVersion(Lifecycle.STABLE, 3, 10, 2),
+    PythonVersion(Lifecycle.STABLE, 3, 7, 13),
+    PythonVersion(Lifecycle.STABLE, 3, 8, 13),
+    PythonVersion(Lifecycle.STABLE, 3, 9, 12),
+    PythonVersion(Lifecycle.STABLE, 3, 10, 4),
     PythonVersion(Lifecycle.DEV, 3, 11, 0),
 )
 
diff --git a/pex/result.py b/pex/result.py
--- a/pex/result.py
+++ b/pex/result.py
@@ -56,7 +56,9 @@ def __init__(
         super(Error, self).__init__(exit_code=exit_code, message=message)
 
 
-@attr.s(frozen=True)
+# Frozen exception types don't work under 3.11.0b1 which sets the `__traceback__` attribute after
+# construction in some cases.
+@attr.s
 class ResultError(Exception):
     """Wraps an Error in an exception for use in short-circuiting via `try_` / `catch`."""
 


Test patch:
diff --git a/tests/bin/test_sh_boot.py b/tests/bin/test_sh_boot.py
--- a/tests/bin/test_sh_boot.py
+++ b/tests/bin/test_sh_boot.py
@@ -64,6 +64,7 @@ def expected(*names):
         all_names.add(current_interpreter_identity.binary_name(version_components=2))
     all_names.update(
         [
+            "python3.11",
             "python3.10",
             "python3.9",
             "python3.8",
@@ -71,6 +72,7 @@ def expected(*names):
             "python3.6",
             "python3.5",
             "python2.7",
+            "pypy3.11",
             "pypy3.10",
             "pypy3.9",
             "pypy3.8",
diff --git a/tests/integration/cli/commands/test_issue_1711.py b/tests/integration/cli/commands/test_issue_1711.py
--- a/tests/integration/cli/commands/test_issue_1711.py
+++ b/tests/integration/cli/commands/test_issue_1711.py
@@ -11,7 +11,7 @@
 from pex.resolve import lockfile
 from pex.resolve.locked_resolve import Artifact, LockedRequirement
 from pex.resolve.resolved_requirement import Fingerprint
-from pex.testing import run_pex_command
+from pex.testing import PY_VER, run_pex_command
 from pex.typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
@@ -88,7 +88,9 @@ def assert_psutil_basics():
     # differently - likely because of some collection implementation difference.
     if PY3:
         run_pex_command(
-            args=["pex==2.1.77", "-c", "pex3", "--"] + create_lock_args
+            args=["pex==2.1.77", "-c", "pex3", "--"] + create_lock_args,
+            # N.B.: Pex 2.1.77 only works on Python 3.10 and older.
+            python=py37.binary if PY_VER > (3, 10) else None,
         ).assert_success()
         psutil_old = assert_psutil_basics()
         assert 0 == len(psutil_old.additional_artifacts), (
diff --git a/tests/integration/cli/commands/test_issue_1741.py b/tests/integration/cli/commands/test_issue_1741.py
--- a/tests/integration/cli/commands/test_issue_1741.py
+++ b/tests/integration/cli/commands/test_issue_1741.py
@@ -7,6 +7,7 @@
 import pytest
 
 from pex.cli.testing import run_pex3
+from pex.interpreter import PythonInterpreter
 from pex.pep_440 import Version
 from pex.pep_503 import ProjectName
 from pex.resolve.lockfile import json_codec
@@ -22,6 +23,7 @@
 )
 def test_prereleases(
     tmpdir,  # type: Any
+    py310,  # type: PythonInterpreter
 ):
     # type: (...) -> None
 
@@ -58,6 +60,8 @@ def test_prereleases(
     # 1st prove this does the wrong thing on prior broken versions of Pex.
     result = run_pex_command(
         args=["pex==2.1.83", "-c", "pex", "--"] + use_lock_args,
+        # N.B.: Pex 2.1.88 only works on Python 3.10 and older.
+        python=py310.binary if PY_VER > (3, 10) else None,
         quiet=True,
     )
     result.assert_failure()
diff --git a/tests/integration/cli/commands/test_lock.py b/tests/integration/cli/commands/test_lock.py
--- a/tests/integration/cli/commands/test_lock.py
+++ b/tests/integration/cli/commands/test_lock.py
@@ -1,7 +1,6 @@
 # Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-import hashlib
 import os
 import re
 import subprocess
@@ -39,7 +38,7 @@
 from pex.version import __version__
 
 if TYPE_CHECKING:
-    from typing import Any, Optional
+    from typing import Any
 
     import attr  # vendor:skip
 else:
@@ -175,30 +174,40 @@ def test_create_local_unsupported(pex_project_dir):
     ) == result.error
 
 
-def test_create_vcs(tmpdir):
-    # type: (Any) -> None
+def test_create_vcs(
+    tmpdir,  # type: Any
+    py310,  # type: str
+):
+    # type: (...) -> None
+
+    # N.B.: The VCS Pex below only works on Python 3.10 and older.
+    python = py310 if PY_VER > (3, 10) else sys.executable
 
     lock = os.path.join(str(tmpdir), "lock")
     run_pex3(
         "lock",
         "create",
+        "--python",
+        python,
         "pex @ git+https://github.com/pantsbuild/pex@473c6ac7",
         "git+https://github.com/VaasuDevanS/cowsay-python@v3.0#egg=cowsay",
         "-o",
         lock,
     ).assert_success()
     pex_file = os.path.join(str(tmpdir), "pip-pex.pex")
-    run_pex_command(args=["--lock", lock, "-o", pex_file]).assert_success()
+    run_pex_command(args=["--lock", lock, "-o", pex_file], python=python).assert_success()
 
-    assert (
-        "3.0"
-        == subprocess.check_output(args=[pex_file, "--version"], env=make_env(PEX_SCRIPT="cowsay"))
+    version_output = (
+        subprocess.check_output(
+            args=[python, pex_file, "--version"], env=make_env(PEX_SCRIPT="cowsay")
+        )
         .decode("utf-8")
         .strip()
     )
+    assert "3.0" == version_output, version_output
 
     process = subprocess.Popen(
-        args=[pex_file, "-V"],
+        args=[python, pex_file, "-V"],
         env=make_env(PEX_SCRIPT="pex"),
         stdout=subprocess.PIPE,
         stderr=subprocess.PIPE,
diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py
--- a/tests/integration/test_integration.py
+++ b/tests/integration/test_integration.py
@@ -31,6 +31,7 @@
     PY27,
     PY37,
     PY310,
+    PY_VER,
     IntegResults,
     WheelBuilder,
     built_wheel,
@@ -1333,7 +1334,7 @@ def test_tmpdir_file(tmp_workdir):
 
 EXAMPLE_PYTHON_REQUIREMENTS_URL = (
     "https://raw.githubusercontent.com/pantsbuild/example-python/"
-    "c6052498f25a436f2639ccd0bc846cec1a55d7d5"
+    "68387a9f5f1a1cb288820f8ebb5d6f66d95c888a"
     "/requirements.txt"
 )
 
@@ -1369,10 +1370,9 @@ def req(
             ),
         )
         assert [
-            req("ansicolors>=1.0.2", 4),
-            req("setuptools>=42.0.0", 5),
-            req("translate>=3.2.1", 6),
-            req("protobuf>=3.11.3", 7),
+            req("ansicolors==1.1.8", 4),
+            req("setuptools>=56.2.0,<57", 5),
+            req("types-setuptools>=56.2.0,<58", 6),
         ] == list(reqs)
 
 
@@ -1551,24 +1551,17 @@ def test_pip_issues_9420_workaround():
     )
 
 
+@pytest.mark.skipif(
+    PY_VER <= (3, 5),
+    reason="The example python requirements URL has requirements that only work with Python 3.6+.",
+)
 def test_requirement_file_from_url(tmpdir):
     # type: (Any) -> None
 
-    constraints = os.path.join(str(tmpdir), "constraints.txt")
-    with open(constraints, "w") as fp:
-        print("translate>=3.2.1,<3.6.0", file=fp)
-        print("protobuf<=3.17.3", file=fp)
-        print("setuptools<60", file=fp)
-
     pex_file = os.path.join(str(tmpdir), "pex")
-
-    results = run_pex_command(
-        args=["-r", EXAMPLE_PYTHON_REQUIREMENTS_URL, "--constraints", constraints, "-o", pex_file]
-    )
+    results = run_pex_command(args=["-r", EXAMPLE_PYTHON_REQUIREMENTS_URL, "-o", pex_file])
     results.assert_success()
-    output, returncode = run_simple_pex(
-        pex_file, args=["-c", "import colors, google.protobuf, setuptools, translate"]
-    )
+    output, returncode = run_simple_pex(pex_file, args=["-c", "import colors, setuptools"])
     assert 0 == returncode, output
     assert b"" == output
 
diff --git a/tests/integration/test_issue_1597.py b/tests/integration/test_issue_1597.py
--- a/tests/integration/test_issue_1597.py
+++ b/tests/integration/test_issue_1597.py
@@ -20,7 +20,7 @@
 
 def build_pex(
     args,  # type: List[str]
-    marker='python_full_version == "{python_full_version}"'.format(
+    marker='python_full_version >= "{python_full_version}"'.format(
         python_full_version=platform.python_version()
     ),  # type: str
 ):
diff --git a/tests/integration/test_issue_1656.py b/tests/integration/test_issue_1656.py
--- a/tests/integration/test_issue_1656.py
+++ b/tests/integration/test_issue_1656.py
@@ -9,7 +9,7 @@
 
 from pex.interpreter import PythonInterpreter
 from pex.pex_info import PexInfo
-from pex.testing import make_env, run_pex_command
+from pex.testing import PY_VER, make_env, run_pex_command
 from pex.typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
@@ -39,6 +39,15 @@ def old_pex(tmpdir_factory):
     return pex_file
 
 
+skip_for_newer_than_310 = pytest.mark.skipif(
+    PY_VER > (3, 10),
+    reason=(
+        "This test exercises compatibility with old Pex versions but those do not work on "
+        "anything newer than Python 3.10."
+    ),
+)
+
+
 def run_current_pex_tool(
     subject_pex,  # type: str
     subcommand,  # type: str
@@ -63,6 +72,7 @@ def run_pex_tool(
     ).decode("utf-8")
 
 
+@skip_for_newer_than_310
 def test_old_venv_tool_vs_new_pex(
     tmpdir,  # type: Any
     old_pex,  # type: str
@@ -77,6 +87,7 @@ def test_old_venv_tool_vs_new_pex(
     assert b"4.0\n" == subprocess.check_output(args=[os.path.join(venv, "pex"), "--version"])
 
 
+@skip_for_newer_than_310
 def test_new_venv_tool_vs_old_pex(
     tmpdir,  # type: Any
     old_pex,  # type: str
@@ -91,6 +102,7 @@ def test_new_venv_tool_vs_old_pex(
     assert b"4.0\n" == subprocess.check_output(args=[os.path.join(venv, "pex"), "--version"])
 
 
+@skip_for_newer_than_310
 def test_mixed_pex_root(
     tmpdir,  # type: Any
     old_pex,  # type: str
diff --git a/tests/integration/test_issue_1726.py b/tests/integration/test_issue_1726.py
--- a/tests/integration/test_issue_1726.py
+++ b/tests/integration/test_issue_1726.py
@@ -10,7 +10,8 @@
 import pytest
 
 from pex.common import safe_open
-from pex.testing import run_pex_command
+from pex.interpreter import PythonInterpreter
+from pex.testing import PY_VER, run_pex_command
 from pex.typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
@@ -21,8 +22,11 @@
     sys.version_info[:2] < (3, 7),
     reason="The jaraco-collections 3.5.1 distribution requires Python >=3.7",
 )
-def test_check_install_issue_1726(tmpdir):
-    # type: (Any) -> None
+def test_check_install_issue_1726(
+    tmpdir,  # type: Any
+    py310,  # type: PythonInterpreter
+):
+    # type: (...) -> None
 
     src = os.path.join(str(tmpdir), "src")
     with safe_open(os.path.join(src, "setup.py"), "w") as fp:
@@ -53,7 +57,11 @@ def test_check_install_issue_1726(tmpdir):
         "-c",
         "from jaraco import collections; print(collections.__file__)",
     ]
-    old_result = run_pex_command(args=["pex==2.1.80", "-c", "pex", "--"] + pex_args)
+    old_result = run_pex_command(
+        args=["pex==2.1.80", "-c", "pex", "--"] + pex_args,
+        # N.B.: Pex 2.1.80 only works on Python 3.10 and older.
+        python=py310.binary if PY_VER > (3, 10) else None,
+    )
     old_result.assert_failure()
     assert (
         "Failed to resolve compatible distributions:\n"
diff --git a/tests/integration/test_issue_539.py b/tests/integration/test_issue_539.py
--- a/tests/integration/test_issue_539.py
+++ b/tests/integration/test_issue_539.py
@@ -9,14 +9,16 @@
 
 from pex.common import temporary_dir
 from pex.pip.tool import get_pip
-from pex.testing import IS_PYPY, run_pex_command
+from pex.testing import IS_PYPY, PY_VER, run_pex_command
 
 
 @pytest.mark.skipif(
-    IS_PYPY,
-    reason="The cryptography 2.6.1 project only has pre-built wheels for CPython "
-    "available on PyPI and this test relies upon a pre-built wheel being "
-    "available.",
+    IS_PYPY or PY_VER > (3, 10),
+    reason=(
+        "The cryptography 2.6.1 project only has pre-built wheels for CPython <= 3.10 "
+        "available on PyPI and this test relies upon a pre-built wheel being "
+        "available."
+    ),
 )
 def test_abi3_resolution():
     # type: () -> None
@@ -27,7 +29,7 @@ def test_abi3_resolution():
     # cryptography-2.6.1-cp34-abi3-macosx_10_6_intel.whl
     # cryptography-2.6.1-cp34-abi3-manylinux1_x86_64.whl
     # With pex in --no-build mode, we force a test that pex abi3 resolution works when this test is
-    # run under CPython>3.4,<4 on OSX and linux.
+    # run under CPython>3.4,<3.11 on OSX and linux.
 
     with temporary_dir() as td:
         # The dependency graph for cryptography-2.6.1 includes pycparser which is only released as an
diff --git a/tests/integration/test_lock_resolver.py b/tests/integration/test_lock_resolver.py
--- a/tests/integration/test_lock_resolver.py
+++ b/tests/integration/test_lock_resolver.py
@@ -7,6 +7,7 @@
 import re
 import shutil
 import subprocess
+import sys
 from textwrap import dedent
 
 import colors
@@ -22,7 +23,7 @@
 from pex.pex_info import PexInfo
 from pex.resolve import lockfile
 from pex.resolve.locked_resolve import LockedRequirement
-from pex.testing import built_wheel, make_env, run_pex_command
+from pex.testing import PY_VER, built_wheel, make_env, run_pex_command
 from pex.typing import TYPE_CHECKING
 from pex.util import CacheHelper
 
@@ -394,6 +395,7 @@ def test_issue_1413_portable_find_links(tmpdir):
 
 def test_issue_1717_transitive_extras(
     tmpdir,  # type: Any
+    py310,  # type: PythonInterpreter
 ):
     # type: (...) -> None
 
@@ -481,12 +483,16 @@ def assert_dists(
             ProjectNameAndVersion.from_filename(d).project_name for d in pex_info.distributions
         }
 
-    run_pex_command(args=["pex==2.1.78", "-cpex", "--"] + create_pex_args).assert_success()
+    # N.B.: Pex 2.1.78 only works on Python 3.10 and older.
+    python = py310.binary if PY_VER > (3, 10) else sys.executable
+    run_pex_command(
+        args=["pex==2.1.78", "-cpex", "--"] + create_pex_args, python=python
+    ).assert_success()
     pex_info = PexInfo.from_pex(pex)
     assert_requirements(pex_info)
     assert_dists(pex_info, "root", "middle_man_with_extras", "A", "B", "C")
 
-    process = subprocess.Popen(args=test_pex_args, stderr=subprocess.PIPE)
+    process = subprocess.Popen(args=[python] + test_pex_args, stderr=subprocess.PIPE)
     _, stderr = process.communicate()
     assert 0 != process.returncode
 
@@ -510,7 +516,7 @@ def assert_dists(
             stderr.decode("utf-8"),
         )
         is not None
-    )
+    ), stderr.decode("utf-8")
 
     run_pex_command(args=create_pex_args).assert_success()
     pex_info = PexInfo.from_pex(pex)
diff --git a/tests/integration/test_reproducible.py b/tests/integration/test_reproducible.py
--- a/tests/integration/test_reproducible.py
+++ b/tests/integration/test_reproducible.py
@@ -7,12 +7,15 @@
 from textwrap import dedent
 from zipfile import ZipFile
 
+import pytest
+
 from pex.common import temporary_dir
 from pex.compatibility import PY2
 from pex.testing import (
     PY27,
     PY37,
     PY310,
+    PY_VER,
     create_pex_command,
     ensure_python_interpreter,
     run_command_with_jitter,
@@ -93,6 +96,15 @@ def test_reproducible_build_no_args():
     assert_reproducible_build([], pythons=MIXED_MAJOR_PYTHONS)
 
 
+@pytest.mark.skipif(
+    PY_VER > (3, 10),
+    reason=(
+        "There are no pre-built binaries for the cryptograph transitive dependency graph (cffi in "
+        "particular); so this test fails under Python 3.11+ since it requires building an sdist "
+        "and that leads to an underlying C `.so` build that we have insufficient control over to "
+        "make reproducible."
+    ),
+)
 def test_reproducible_build_bdist_requirements():
     # type: () -> None
     # We test both a pure Python wheel (six) and a platform-specific wheel (cryptography).
diff --git a/tests/test_targets.py b/tests/test_targets.py
--- a/tests/test_targets.py
+++ b/tests/test_targets.py
@@ -113,6 +113,7 @@ def test_requires_python_current():
 
     current_target = targets.current()
     major, minor, patch = current_target.interpreter.version
+    patch_is_zero = 0 == patch
 
     def requires_python(template):
         # type: (str) -> str
@@ -132,25 +133,25 @@ def assert_requires_python(
     assert_requires_python(True, "~={major}.{minor}")
 
     assert_requires_python(True, "=={major}.{minor}.*")
-    assert_requires_python(False, "=={major}.{minor}")
+    assert_requires_python(patch_is_zero, "=={major}.{minor}")
     assert_requires_python(True, "=={major}.{minor}.{patch}")
 
     assert_requires_python(True, "!={major}")
     assert_requires_python(False, "!={major}.*")
-    assert_requires_python(True, "!={major}.{minor}")
+    assert_requires_python(not patch_is_zero, "!={major}.{minor}")
     assert_requires_python(False, "!={major}.{minor}.*")
     assert_requires_python(False, "!={major}.{minor}.{patch}")
 
     assert_requires_python(False, "<{major}")
     assert_requires_python(False, "<={major}")
     assert_requires_python(False, "<{major}.{minor}")
-    assert_requires_python(False, "<={major}.{minor}")
+    assert_requires_python(patch_is_zero, "<={major}.{minor}")
     assert_requires_python(False, "<{major}.{minor}.{patch}")
     assert_requires_python(True, "<={major}.{minor}.{patch}")
 
     assert_requires_python(True, ">{major}")
     assert_requires_python(True, ">={major}")
-    assert_requires_python(True, ">{major}.{minor}")
+    assert_requires_python(not patch_is_zero, ">{major}.{minor}")
     assert_requires_python(True, ">={major}.{minor}")
     assert_requires_python(False, ">{major}.{minor}.{patch}")
     assert_requires_python(True, ">={major}.{minor}.{patch}")