Repository: pex-tool/pex

Issue:
Support python3.10
Python3.10 is out and pex does not support it.

This is a request to support pex on python3.10


Discussion:

Resolving patch:
diff --git a/pex/vendor/__init__.py b/pex/vendor/__init__.py
--- a/pex/vendor/__init__.py
+++ b/pex/vendor/__init__.py
@@ -5,8 +5,11 @@
 
 import collections
 import os
+import subprocess
+import sys
+from textwrap import dedent
 
-from pex.common import filter_pyc_dirs, filter_pyc_files, touch
+from pex.common import filter_pyc_dirs, filter_pyc_files, safe_mkdtemp, touch
 from pex.compatibility import urlparse
 from pex.tracer import TRACER
 from pex.typing import TYPE_CHECKING
@@ -56,18 +59,31 @@ def pinned(cls, key, version, rewrite=True):
         )
 
     @classmethod
-    def vcs(cls, url, rewrite=True):
-        result = urlparse.urlparse(url)
-        fragment_params = urlparse.parse_qs(result.fragment)
-        values = fragment_params.get("egg")
-        if not values or len(values) != 1:
-            raise ValueError(
-                "Expected the vcs requirement url to have an #egg=<name> fragment. "
-                "Got: {}".format(url)
-            )
-
-        # N.B.: Constraints do not work for vcs urls.
-        return cls(key=values[0], requirement=url, rewrite=rewrite, constrain=False)
+    def git(cls, repo, commit, project_name, prep_command=None, rewrite=True):
+        requirement = "git+{repo}@{commit}#egg={project_name}".format(
+            repo=repo, commit=commit, project_name=project_name
+        )
+        if not prep_command:
+            return cls(key=project_name, requirement=requirement, rewrite=rewrite, constrain=False)
+
+        class PreparedGit(VendorSpec):
+            def prepare(self):
+                clone_dir = safe_mkdtemp()
+                subprocess.check_call(["git", "clone", "--depth", "1", repo, clone_dir])
+                subprocess.check_call(
+                    ["git", "fetch", "--depth", "1", "origin", commit], cwd=clone_dir
+                )
+                subprocess.check_call(["git", "checkout", commit], cwd=clone_dir)
+                if prep_command:
+                    subprocess.check_call(prep_command, cwd=clone_dir)
+                return clone_dir
+
+        return PreparedGit(
+            key=project_name,
+            requirement=requirement,
+            rewrite=rewrite,
+            constrain=False,
+        )
 
     @property
     def _subpath_components(self):
@@ -81,6 +97,9 @@ def relpath(self):
     def target_dir(self):
         return os.path.join(self.ROOT, self.relpath)
 
+    def prepare(self):
+        return self.requirement
+
     def create_packages(self):
         """Create missing packages joining the vendor root to the base of the vendored distribution.
 
@@ -124,15 +143,50 @@ def iter_vendor_specs():
     # in `pip download --constraint...` tracked at https://github.com/pypa/pip/issues/9283 and fixed
     # by https://github.com/pypa/pip/pull/9301 there and https://github.com/pantsbuild/pip/pull/8 in
     # our fork.
-    yield VendorSpec.vcs(
-        "git+https://github.com/pantsbuild/pip@de1c91261f2b54d60fdf2a17fba756ef0decb146#egg=pip",
+    yield VendorSpec.git(
+        repo="https://github.com/pantsbuild/pip",
+        commit="de1c91261f2b54d60fdf2a17fba756ef0decb146",
+        project_name="pip",
         rewrite=False,
     )
 
     # We expose this to pip at buildtime for legacy builds, but we also use pkg_resources via
     # pex.third_party at runtime in various ways.
-    # N.B.: 44.0.0 is the last setuptools version compatible with Python 2.
-    yield VendorSpec.pinned("setuptools", "44.0.0")
+    # N.B.: 44.0.0 is the last setuptools version compatible with Python 2 and we use a fork of that
+    # with patches needed to support Pex on the v44.0.0/patches/pex-2.x branch.
+    pantsbuild_setuptools_commit = "3acb925dd708430aeaf197ea53ac8a752f7c1863"
+    yield VendorSpec.git(
+        repo="https://github.com/pantsbuild/setuptools",
+        commit=pantsbuild_setuptools_commit,
+        project_name="setuptools",
+        # Setuptools from source requires running bootstrap.py 1st manually due to circularity in
+        # needing setuptools to build setuptools. The bootstrap runs `setup.py egg_info` which
+        # generates a version containing a date stamp. We override setup.cfg's egg_info section to
+        # avoid this instability and instead force using the commit as the stable version
+        # modifier.
+        # N.B.: This code assumes its run under Python 3.5+.
+        prep_command=[
+            sys.executable,
+            "-c",
+            dedent(
+                """\
+                import configparser
+                import subprocess
+                import sys
+
+
+                parser = configparser.ConfigParser()
+                parser.read("setup.cfg")
+                parser["egg_info"]["tag_build"] = "+{commit}"
+                del parser["egg_info"]["tag_date"]
+                with open("setup.cfg", "w") as fp:
+                    parser.write(fp)
+
+                subprocess.check_call([sys.executable, "bootstrap.py"])
+                """
+            ).format(commit=pantsbuild_setuptools_commit),
+        ],
+    )
 
     # We expose this to pip at buildtime for legacy builds.
     yield VendorSpec.pinned("wheel", "0.36.2", rewrite=False)
diff --git a/pex/vendor/__main__.py b/pex/vendor/__main__.py
--- a/pex/vendor/__main__.py
+++ b/pex/vendor/__main__.py
@@ -210,6 +210,7 @@ def vendorize(root_dir, vendor_specs, prefix):
         # pex.vendor. Since we document that pex.vendor should be run via tox, that environment will
         # contain pinned versions of setuptools and wheel. As a result, vendoring (at least via tox)
         # is hermetic.
+        requirement = vendor_spec.prepare()
         cmd = [
             "pip",
             "install",
@@ -218,7 +219,7 @@ def vendorize(root_dir, vendor_specs, prefix):
             "--no-compile",
             "--target",
             vendor_spec.target_dir,
-            vendor_spec.requirement,
+            requirement,
         ]
 
         constraints_file = os.path.join(vendor_spec.target_dir, "constraints.txt")
@@ -245,12 +246,20 @@ def vendorize(root_dir, vendor_specs, prefix):
         safe_rmtree(os.path.join(vendor_spec.target_dir, "bin"))
         safe_delete(os.path.join(vendor_spec.target_dir, "easy_install.py"))
 
-        # The RECORD contains file hashes of all installed files and is unfortunately unstable in the
-        # case of scripts which get a shebang added with a system-specific path to the python
+        # The RECORD contains file hashes of all installed files and is unfortunately unstable in
+        # the case of scripts which get a shebang added with a system-specific path to the python
         # interpreter to execute.
         for record_file in glob.glob(os.path.join(vendor_spec.target_dir, "*-*.dist-info/RECORD")):
             safe_delete(record_file)
 
+        # The direct_url.json contains an url if a requirement was installed from one. We install
+        # certain vendored projects via git clones we do prep-work on and these clones have a local
+        # temporary directory path that will change on each re-vendor.
+        for direct_url_file in glob.glob(
+            os.path.join(vendor_spec.target_dir, "*-*.dist-info/direct_url.json")
+        ):
+            safe_delete(direct_url_file)
+
         vendor_spec.create_packages()
 
     vendored_path = [vendor_spec.target_dir for vendor_spec in vendor_specs if vendor_spec.rewrite]
diff --git a/pex/vendor/_vendored/setuptools/setuptools/command/install_scripts.py b/pex/vendor/_vendored/setuptools/setuptools/command/install_scripts.py
--- a/pex/vendor/_vendored/setuptools/setuptools/command/install_scripts.py
+++ b/pex/vendor/_vendored/setuptools/setuptools/command/install_scripts.py
@@ -40,8 +40,11 @@ def run(self):
         )
         bs_cmd = self.get_finalized_command('build_scripts')
         exec_param = getattr(bs_cmd, 'executable', None)
-        bw_cmd = self.get_finalized_command("bdist_wininst")
-        is_wininst = getattr(bw_cmd, '_is_running', False)
+        is_wininst = False
+        # The distutils bdist_wininst command was removed in Python 3.10.
+        if sys.version_info[:2] < (3, 10):
+            bw_cmd = self.get_finalized_command("bdist_wininst")
+            is_wininst = getattr(bw_cmd, '_is_running', False)
         writer = ei.ScriptWriter
         if is_wininst:
             exec_param = "python.exe"


Test patch:
diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py
--- a/tests/integration/test_integration.py
+++ b/tests/integration/test_integration.py
@@ -614,7 +614,7 @@ def test_fail():
     with temporary_content({"tester.py": test_stub}) as output_dir:
         pex_path = os.path.join(output_dir, "test.pex")
         tester_path = os.path.join(output_dir, "tester.py")
-        results = run_pex_command(["pytest==3.9.1", "-e", "pytest:main", "-o", pex_path])
+        results = run_pex_command(["pytest", "-c", "pytest", "-o", pex_path])
         results.assert_success()
 
         assert subprocess.call([pex_path, os.path.realpath(tester_path)]) == 1
@@ -734,7 +734,7 @@ def test_pex_resource_bundling():
                     resources_input_dir,
                     "-e",
                     "exe",
-                    "setuptools==17.0",
+                    "setuptools==44.0",
                 ]
             )
             res.assert_success()
@@ -802,7 +802,7 @@ def test_pex_console_script_custom_setuptools_useable():
             version='0.0.0',
             zip_safe=True,
             packages=[''],
-            install_requires=['setuptools==36.2.7'],
+            install_requires=['setuptools==43.0.0'],
             entry_points={'console_scripts': ['my_app_function = my_app:do_something']},
         )
   """
@@ -810,14 +810,9 @@ def test_pex_console_script_custom_setuptools_useable():
 
     my_app = dedent(
         """
-        import sys
-
         def do_something():
-            try:
-                from setuptools.sandbox import run_setup
-                return 0
-            except:
-                return 1
+            import setuptools
+            assert '43.0.0' == setuptools.__version__
   """
     )
 
@@ -844,8 +839,15 @@ def pex_with_no_entrypoints():
     # type: () -> Iterator[Tuple[str, bytes, str]]
     with temporary_dir() as out:
         pex = os.path.join(out, "pex.pex")
-        run_pex_command(["setuptools==36.2.7", "-o", pex])
-        test_script = b"from setuptools.sandbox import run_setup; print(str(run_setup))"
+        run_pex_command(["setuptools==43.0.0", "-o", pex])
+        test_script = dedent(
+            """\
+            import sys
+            import setuptools
+
+            sys.exit(0 if '43.0.0' == setuptools.__version__ else 1)
+            """
+        ).encode("utf-8")
         yield pex, test_script, out
 
 
diff --git a/tests/integration/test_reproducible.py b/tests/integration/test_reproducible.py
--- a/tests/integration/test_reproducible.py
+++ b/tests/integration/test_reproducible.py
@@ -1,5 +1,6 @@
 # Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
+
 import filecmp
 import os
 import sys
@@ -95,7 +96,12 @@ def test_reproducible_build_no_args():
 def test_reproducible_build_bdist_requirements():
     # type: () -> None
     # We test both a pure Python wheel (six) and a platform-specific wheel (cryptography).
-    assert_reproducible_build(["six==1.12.0", "cryptography==2.6.1"])
+    assert_reproducible_build(
+        [
+            "six==1.12.0",
+            "cryptography=={version}".format(version="2.6.1" if PY2 else "3.4.8"),
+        ]
+    )
 
 
 def test_reproducible_build_sdist_requirements():
diff --git a/tests/test_bdist_pex.py b/tests/test_bdist_pex.py
--- a/tests/test_bdist_pex.py
+++ b/tests/test_bdist_pex.py
@@ -38,7 +38,7 @@ def bdist_pex_pythonpath():
         BDIST_PEX_PYTHONPATH.extend(
             installed_distribution.distribution.location
             for installed_distribution in resolver.resolve(
-                ["setuptools==36.2.7"]
+                ["setuptools==43.0.0"]
             ).installed_distributions
         )
     return BDIST_PEX_PYTHONPATH
diff --git a/tests/test_pex.py b/tests/test_pex.py
--- a/tests/test_pex.py
+++ b/tests/test_pex.py
@@ -714,12 +714,12 @@ def test_pex_run_strip_env():
 
 def test_pex_run_custom_setuptools_useable():
     # type: () -> None
-    result = resolver.resolve(["setuptools==36.2.7"])
+    result = resolver.resolve(["setuptools==43.0.0"])
     dists = [installed_dist.distribution for installed_dist in result.installed_distributions]
     with temporary_dir() as temp_dir:
         pex = write_simple_pex(
             temp_dir,
-            "from setuptools.sandbox import run_setup",
+            "import setuptools, sys; sys.exit(0 if '43.0.0' == setuptools.__version__ else 1)",
             dists=dists,
         )
         rc = PEX(pex.path()).run()
@@ -729,15 +729,9 @@ def test_pex_run_custom_setuptools_useable():
 def test_pex_run_conflicting_custom_setuptools_useable():
     # type: () -> None
     # Here we use our vendored, newer setuptools to build the pex which has an older setuptools
-    # requirement. These setuptools dists have different pkg_resources APIs:
-    # $ diff \
-    #   <(zipinfo -1 setuptools-20.3.1-py2.py3-none-any.whl | grep pkg_resources/ | sort) \
-    #   <(zipinfo -1 setuptools-40.6.2-py2.py3-none-any.whl | grep pkg_resources/ | sort)
-    # 2a3,4
-    # > pkg_resources/py31compat.py
-    # > pkg_resources/_vendor/appdirs.py
-
-    result = resolver.resolve(["setuptools==20.3.1"])
+    # requirement.
+
+    result = resolver.resolve(["setuptools==43.0.0"])
     dists = [installed_dist.distribution for installed_dist in result.installed_distributions]
     with temporary_dir() as temp_dir:
         pex = write_simple_pex(
@@ -745,19 +739,9 @@ def test_pex_run_conflicting_custom_setuptools_useable():
             exe_contents=textwrap.dedent(
                 """
                 import sys
-                import pkg_resources
-
-                try:
-                    from pkg_resources import appdirs
-                    sys.exit(1)
-                except ImportError:
-                    pass
+                import setuptools
 
-                try:
-                    from pkg_resources import py31compat
-                    sys.exit(2)
-                except ImportError:
-                    pass
+                sys.exit(0 if '43.0.0' == setuptools.__version__ else 1)
                 """
             ),
             dists=dists,
@@ -800,6 +784,7 @@ def test_pex_run_custom_pex_useable():
         assert old_pex_version != __version__
 
 
+@pytest.mark.skipif(PY2, reason="ResourceWarning was only introduced in Python 3.2.")
 def test_interpreter_teardown_dev_null_unclosed_resource_warning_suppressed():
     # type: () -> None
 
@@ -810,6 +795,7 @@ def test_interpreter_teardown_dev_null_unclosed_resource_warning_suppressed():
         pex_builder.freeze()
 
         output = subprocess.check_output(
-            args=[sys.executable, "-W" "all", pex_chroot, "-c", ""], stderr=subprocess.STDOUT
+            args=[sys.executable, "-W", "error::ResourceWarning", pex_chroot, "-c", ""],
+            stderr=subprocess.STDOUT,
         )
         assert b"" == output
diff --git a/tests/test_pex_builder.py b/tests/test_pex_builder.py
--- a/tests/test_pex_builder.py
+++ b/tests/test_pex_builder.py
@@ -16,7 +16,7 @@
 from pex.layout import Layout
 from pex.pex import PEX
 from pex.pex_builder import CopyMode, PEXBuilder
-from pex.testing import built_wheel, make_bdist, make_env
+from pex.testing import PY_VER, built_wheel, make_bdist, make_env
 from pex.testing import write_simple_pex as write_pex
 from pex.typing import TYPE_CHECKING
 from pex.variables import ENV
@@ -67,6 +67,13 @@ def test_pex_builder():
             assert fp.read() == "success"
 
 
+@pytest.mark.skipif(
+    PY_VER >= (3, 10),
+    reason=(
+        "The pyparsing 2.1.10 distribution imports collections.MutableMapping which was (re)moved "
+        "in Python 3.10."
+    ),
+)
 def test_pex_builder_wheeldep():
     # type: () -> None
     """Repeat the pex_builder test, but this time include an import of something from a wheel that
diff --git a/tests/test_vendor.py b/tests/test_vendor.py
--- a/tests/test_vendor.py
+++ b/tests/test_vendor.py
@@ -1,10 +1,17 @@
 # Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-import pytest
+import os
+import subprocess
+import sys
 
+from pex.common import touch
+from pex.typing import TYPE_CHECKING
 from pex.vendor import VendorSpec
 
+if TYPE_CHECKING:
+    from typing import Any
+
 
 def test_pinned():
     # type: () -> None
@@ -13,20 +20,48 @@ def test_pinned():
     assert "foo==1.2.3" == vendor_spec.requirement
 
 
-def test_vcs_valid():
+def test_git():
     # type: () -> None
-    vendor_spec = VendorSpec.vcs("git+https://github.com/foo.git@da39a3ee#egg=bar")
+    vendor_spec = VendorSpec.git(
+        repo="https://github.com/foo.git", commit="da39a3ee", project_name="bar"
+    )
     assert "bar" == vendor_spec.key
     assert "git+https://github.com/foo.git@da39a3ee#egg=bar" == vendor_spec.requirement
+    assert "git+https://github.com/foo.git@da39a3ee#egg=bar" == vendor_spec.prepare()
 
 
-def test_vcs_invalid_no_egg():
-    # type: () -> None
-    with pytest.raises(ValueError):
-        VendorSpec.vcs("git+https://github.com/foo.git@da39a3ee")
+def test_git_prep_command(tmpdir):
+    # type: (Any) -> None
+    repo = os.path.join(str(tmpdir), "repo")
+    subprocess.check_call(["git", "init", repo])
+    assert os.path.isdir(repo)
 
+    subprocess.check_call(["git", "config", "user.email", "you@example.com"], cwd=repo)
+    subprocess.check_call(["git", "config", "user.name", "Your Name"], cwd=repo)
 
-def test_vcs_invalid_multiple_egg():
-    # type: () -> None
-    with pytest.raises(ValueError):
-        VendorSpec.vcs("git+https://github.com/foo.git@da39a3ee#egg=bar&egg=foo")
+    touch(os.path.join(repo, "README"))
+    subprocess.check_call(["git", "add", "README"], cwd=repo)
+    subprocess.check_call(["git", "commit", "-m", "Initial Commit."], cwd=repo)
+    commit = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=repo).decode("utf-8").strip()
+
+    prep_file = os.path.join(repo, "prep")
+    assert not os.path.exists(prep_file)
+
+    vendor_spec = VendorSpec.git(
+        repo=repo,
+        commit=commit,
+        project_name="bar",
+        prep_command=[sys.executable, "-c", "fp = open('prep', 'w'); fp.close()"],
+    )
+    assert not os.path.exists(prep_file)
+
+    assert "git+{repo}@{commit}#egg=bar".format(repo=repo, commit=commit) == vendor_spec.requirement
+    assert not os.path.exists(prep_file)
+
+    clone = vendor_spec.prepare()
+    assert (
+        commit
+        == subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=clone).decode("utf-8").strip()
+    )
+    assert not os.path.exists(prep_file)
+    assert os.path.isfile(os.path.join(clone, "prep"))