Repository: pex-tool/pex

Issue:
The deprecated `pex.pex_bootstrapper.bootstrap_pex_env` API does not scrub `pex.third_party`.
The `pex.pex_bootstrapper.bootstrap_pex_env` is maintained for legacy consumers, including https://github.com/pantsbuild/lambdex and https://github.com/kwlzn/pyuwsgi_pex. As such, despite the `__pex__` magic import package prefix which obviates the need for the entrypoint generally and the `pex3 venv create --prefix` tool for the Lambdex use case in particular, the API should be fixed to scrub `pex.third_party` vendored imports and the Pex bootstrap from the `sys.path` like the `pex.pex.PEX.execute` standard entry point does. Without this, errors like the following are seen for user-code that depends on the latest versions of attrs - which Pex vendors at ~21.5.0 (this is from the Lambdex use case with a user code dependency on attrs 23.1.0 and using Pex 2.1.134):
```
Unable to import module 'lambdex_handler': cannot import name 'AttrsInstance' from 'pex.vendor._vendored.attrs.attr' (/var/task/.bootstrap/pex/vendor/_vendored/attrs/attr/__init__.py)
```


Discussion:
The issue is here:
https://github.com/pantsbuild/pex/blob/ae83e326c2ba42a6d56ceb6beb4ce134aded844b/pex/pex_bootstrapper.py#L693-L702

Which fails to do the scrubbing found here before returning from the function:
https://github.com/pantsbuild/pex/blob/ae83e326c2ba42a6d56ceb6beb4ce134aded844b/pex/pex.py#L607-L634

The example failing lambdex in the OP above was fixed with this patch:
```diff
+++ .bootstrap/pex/pex_bootstrapper.py.orig     2023-07-22 16:43:47.570107642 +0000
--- .bootstrap/pex/pex_bootstrapper.py  2023-07-22 16:52:07.720031735 +0000
@@ -700,11 +700,3 @@
     from .environment import PEXEnvironment

     PEXEnvironment.mount(entry_point, pex_info).activate()
+
+    from . import third_party
+    from .bootstrap import Bootstrap
+    third_party.uninstall()
+    bootstrap = Bootstrap.locate()
+    bootstrap.demote()
+    import pex
+
```

Resolving patch:
diff --git a/pex/bootstrap.py b/pex/bootstrap.py
--- a/pex/bootstrap.py
+++ b/pex/bootstrap.py
@@ -2,7 +2,10 @@
 # Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
+from __future__ import absolute_import, print_function
+
 import os
+import sys
 
 
 class Bootstrap(object):
@@ -88,3 +91,38 @@ def __repr__(self):
         return "{cls}(sys_path_entry={sys_path_entry!r})".format(
             cls=type(self).__name__, sys_path_entry=self._sys_path_entry
         )
+
+
+def demote():
+    # type: () -> None
+    """Demote PEX bootstrap code to the end of `sys.path` and uninstall all PEX vendored code."""
+
+    from . import third_party
+    from .tracer import TRACER
+
+    TRACER.log("Bootstrap complete, performing final sys.path modifications...")
+
+    should_log = {level: TRACER.should_log(V=level) for level in range(1, 10)}
+
+    def log(msg, V=1):
+        if should_log.get(V, False):
+            print("pex: {}".format(msg), file=sys.stderr)
+
+    # Remove the third party resources pex uses and demote pex bootstrap code to the end of
+    # sys.path for the duration of the run to allow conflicting versions supplied by user
+    # dependencies to win during the course of the execution of user code.
+    third_party.uninstall()
+
+    bootstrap = Bootstrap.locate()
+    log("Demoting code from %s" % bootstrap, V=2)
+    for module in bootstrap.demote():
+        log("un-imported {}".format(module), V=9)
+
+    import pex
+
+    log("Re-imported pex from {}".format(pex.__path__), V=3)
+
+    log("PYTHONPATH contains:")
+    for element in sys.path:
+        log("  %c %s" % (" " if os.path.exists(element) else "*", element))
+    log("  * - paths that do not exist or will be imported via zipimport")
diff --git a/pex/pex.py b/pex/pex.py
--- a/pex/pex.py
+++ b/pex/pex.py
@@ -9,7 +9,7 @@
 from site import USER_SITE
 from types import ModuleType
 
-from pex import third_party
+from pex import bootstrap
 from pex.bootstrap import Bootstrap
 from pex.common import die
 from pex.dist_metadata import CallableEntryPoint, Distribution, EntryPoint, find_distributions
@@ -604,35 +604,6 @@ def _execute(self):
                 EntryPoint.parse("run = {}".format(self._pex_info.entry_point))
             )
 
-    @classmethod
-    def demote_bootstrap(cls):
-        TRACER.log("Bootstrap complete, performing final sys.path modifications...")
-
-        should_log = {level: TRACER.should_log(V=level) for level in range(1, 10)}
-
-        def log(msg, V=1):
-            if should_log.get(V, False):
-                print("pex: {}".format(msg), file=sys.stderr)
-
-        # Remove the third party resources pex uses and demote pex bootstrap code to the end of
-        # sys.path for the duration of the run to allow conflicting versions supplied by user
-        # dependencies to win during the course of the execution of user code.
-        third_party.uninstall()
-
-        bootstrap = Bootstrap.locate()
-        log("Demoting code from %s" % bootstrap, V=2)
-        for module in bootstrap.demote():
-            log("un-imported {}".format(module), V=9)
-
-        import pex
-
-        log("Re-imported pex from {}".format(pex.__path__), V=3)
-
-        log("PYTHONPATH contains:")
-        for element in sys.path:
-            log("  %c %s" % (" " if os.path.exists(element) else "*", element))
-        log("  * - paths that do not exist or will be imported via zipimport")
-
     def execute_interpreter(self):
         # type: () -> Any
 
@@ -695,7 +666,7 @@ def execute_interpreter(self):
 
                 atexit.register(readline.write_history_file, histfile)
 
-            self.demote_bootstrap()
+            bootstrap.demote()
 
             import code
 
@@ -781,7 +752,7 @@ def execute_ast(
         argv0=None,  # type: Optional[str]
     ):
         # type: (...) -> Optional[str]
-        cls.demote_bootstrap()
+        bootstrap.demote()
 
         from pex.compatibility import exec_function
 
@@ -801,7 +772,7 @@ def execute_entry(self, entry_point):
 
     def execute_module(self, module_name):
         # type: (str) -> None
-        self.demote_bootstrap()
+        bootstrap.demote()
 
         import runpy
 
@@ -810,7 +781,7 @@ def execute_module(self, module_name):
     @classmethod
     def execute_entry_point(cls, entry_point):
         # type: (CallableEntryPoint) -> Any
-        cls.demote_bootstrap()
+        bootstrap.demote()
 
         runner = entry_point.resolve()
         return runner()
diff --git a/pex/pex_bootstrapper.py b/pex/pex_bootstrapper.py
--- a/pex/pex_bootstrapper.py
+++ b/pex/pex_bootstrapper.py
@@ -700,3 +700,7 @@ def bootstrap_pex_env(entry_point):
     from .environment import PEXEnvironment
 
     PEXEnvironment.mount(entry_point, pex_info).activate()
+
+    from . import bootstrap
+
+    bootstrap.demote()


Test patch:
diff --git a/tests/integration/test_issue_2183.py b/tests/integration/test_issue_2183.py
new file mode 100644
--- /dev/null
+++ b/tests/integration/test_issue_2183.py
@@ -0,0 +1,83 @@
+# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+import os.path
+import subprocess
+from textwrap import dedent
+
+import pytest
+
+from pex.common import safe_open
+from pex.compatibility import commonpath
+from pex.pep_440 import Version
+from pex.pep_503 import ProjectName
+from pex.pex import PEX
+from pex.testing import PY_VER, make_env, run_pex_command
+from pex.typing import TYPE_CHECKING
+from pex.venv.virtualenv import Virtualenv
+
+if TYPE_CHECKING:
+    from typing import Any
+
+
+@pytest.mark.skipif(
+    PY_VER < (3, 7) or PY_VER >= (3, 12),
+    reason=(
+        "The test requires use of attrs 23.1.0 which requires Python >= 3.7 and Lambdex further "
+        "requires a released version of Pex that supports Python 3.12."
+    ),
+)
+def test_lambdex_with_incompatible_attrs(tmpdir):
+    # type: (Any) -> None
+
+    src = os.path.join(str(tmpdir), "src")
+    with safe_open(os.path.join(src, "example.py"), "w") as fp:
+        fp.write(
+            dedent(
+                """\
+                import sys
+
+                from attr import AttrsInstance
+
+                def run():
+                    print(sys.modules[AttrsInstance.__module__].__file__)
+                """
+            )
+        )
+
+    pex = os.path.join(str(tmpdir), "pex")
+    run_pex_command(args=["-D", src, "attrs==23.1.0", "-o", pex]).assert_success()
+
+    pex_distributions_by_project_name = {
+        dist.metadata.project_name: dist for dist in PEX(pex).resolve()
+    }
+    user_attrs = pex_distributions_by_project_name[ProjectName("attrs")]
+    assert Version("23.1.0") == user_attrs.metadata.version
+
+    lambda_zip = os.path.join(str(tmpdir), "lambda.zip")
+    run_pex_command(
+        args=[
+            "lambdex",
+            "-c",
+            "lambdex",
+            "--",
+            "build",
+            "-e",
+            "example:run",
+            "-o",
+            lambda_zip,
+            pex,
+        ]
+    ).assert_success()
+
+    venv_dir = os.path.join(str(tmpdir), "venv_dir")
+    venv = Virtualenv.create(venv_dir=venv_dir)
+    output = (
+        subprocess.check_output(
+            args=[venv.interpreter.binary, "-c", "from lambdex_handler import handler; handler()"],
+            env=make_env(PYTHONPATH=lambda_zip),
+        )
+        .decode("utf-8")
+        .strip()
+    )
+    assert user_attrs.location == commonpath((user_attrs.location, output)), output