Repository: pex-tool/pex

Issue:
Pex Pip code fails to handle PEX_ env vars being present in the ambient environment.
The upshot is a "multiple values for keyword argument PEX_..." error.

The code that needs to be fixed is here:
https://github.com/pantsbuild/pex/blob/ad450399a22a9a36fa84aff231fbb9adee1c0d28/pex/pip.py#L462-L472

For example, in Pants CI on an upgrade to Pex 2.1.43:
```
=================================== FAILURES ===================================
________________________________ test_platforms ________________________________

rule_runner = RuleRunner(build_root=/tmp/_BUILD_ROOTxtgh_sck)

    def test_platforms(rule_runner: RuleRunner) -> None:
        # We use Python 2.7, rather than Python 3, to ensure that the specified platform is
        # actually used.
        platforms = PexPlatforms(["linux-x86_64-cp-27-cp27mu"])
        constraints = InterpreterConstraints(["CPython>=2.7,<3", "CPython>=3.6"])
        pex_output = create_pex_and_get_all_data(
            rule_runner,
            requirements=PexRequirements(["cryptography==2.9"]),
            platforms=platforms,
            interpreter_constraints=constraints,
>           internal_only=False,  # Internal only PEXes do not support (foreign) platforms.
        )

src/python/pants/backend/python/util_rules/pex_test.py:415: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
src/python/pants/backend/python/util_rules/pex_test.py:112: in create_pex_and_get_all_data
    pex = rule_runner.request(pex_type, [request])
src/python/pants/testutil/rule_runner.py:212: in request
    self.scheduler.product_request(output_type, [Params(*inputs)])
src/python/pants/engine/internals/scheduler.py:561: in product_request
    self._raise_on_error([t for _, t in throws])
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <pants.engine.internals.scheduler.SchedulerSession object at 0x7f4145d4a550>
throws = [Throw(exc=ProcessExecutionFailure("Process 'Building test.pex with 1 requirement: cryptography==2.9' failed with exit...d.python.util_rules.pex.build_pex (test.pex)', 'pants.backend.python.util_rules.pex.create_pex (test.pex)', 'select'])]

    def _raise_on_error(self, throws: list[Throw]) -> NoReturn:
        exception_noun = pluralize(len(throws), "Exception")
    
        if self._scheduler.include_trace_on_error:
            throw = throws[0]
            etb = throw.engine_traceback
            python_traceback_str = throw.python_traceback or ""
            engine_traceback_str = ""
            others_msg = f"\n(and {len(throws) - 1} more)" if len(throws) > 1 else ""
            if etb:
                sep = "\n  in "
                engine_traceback_str = "Engine traceback:" + sep + sep.join(reversed(etb)) + "\n"
            raise ExecutionError(
                f"{exception_noun} encountered:\n\n"
                f"{engine_traceback_str}"
                f"{python_traceback_str}"
                f"{others_msg}",
>               wrapped_exceptions=tuple(t.exc for t in throws),
            )
E           pants.engine.internals.scheduler.ExecutionError: 1 Exception encountered:
E           
E           Engine traceback:
E             in select
E             in pants.backend.python.util_rules.pex.create_pex (test.pex)
E             in pants.backend.python.util_rules.pex.build_pex (test.pex)
E             in pants.engine.process.fallible_to_exec_result_or_raise
E           Traceback (most recent call last):
E             File "/tmp/process-executionZ3I85W/src/python/pants/engine/process.py", line 262, in fallible_to_exec_result_or_raise
E               description.value,
E           pants.engine.process.ProcessExecutionFailure: Process 'Building test.pex with 1 requirement: cryptography==2.9' failed with exit code 1.
E           stdout:
E           
E           stderr:
E           Failed to spawn a job for DistributionTarget(platform=Platform(platform='linux_x86_64', impl='cp', version='27', abi='cp27mu')): patch() got multiple values for keyword argument 'PEX_ROOT'

src/python/pants/engine/internals/scheduler.py:501: ExecutionError
- generated xml file: /tmp/process-executionZ3I85W/src.python.pants.backend.python.util_rules.pex_test.py.tests.xml -


=========================== short test summary info ============================
FAILED src/python/pants/backend/python/util_rules/pex_test.py::test_platforms
======================== 1 failed, 13 passed in 55.51s =========================
```


Discussion:

Resolving patch:
diff --git a/pex/pip.py b/pex/pip.py
--- a/pex/pip.py
+++ b/pex/pip.py
@@ -410,6 +410,7 @@ def _spawn_pip_isolated(
         cache=None,  # type: Optional[str]
         interpreter=None,  # type: Optional[PythonInterpreter]
         pip_verbosity=0,  # type: int
+        extra_env=None,  # type: Optional[Dict[str, str]]
         **popen_kwargs  # type: Any
     ):
         # type: (...) -> Tuple[List[str], subprocess.Popen]
@@ -459,16 +460,15 @@ def _spawn_pip_isolated(
         if package_index_configuration:
             command.extend(package_index_configuration.args)
 
-        extra_env = popen_kwargs.pop("env", None)
-        env = dict(extra_env) if extra_env else {}
+        extra_env = extra_env or {}
         if package_index_configuration:
-            env.update(package_index_configuration.env)
+            extra_env.update(package_index_configuration.env)
 
         with ENV.strip().patch(
             PEX_ROOT=cache or ENV.PEX_ROOT,
             PEX_VERBOSE=str(ENV.PEX_VERBOSE),
             __PEX_UNVENDORED__="1",
-            **env
+            **extra_env
         ) as env:
             # Guard against API calls from environment with ambient PYTHONPATH preventing pip PEX
             # bootstrapping. See: https://github.com/pantsbuild/pex/issues/892
@@ -498,6 +498,7 @@ def _spawn_pip_isolated_job(
         interpreter=None,  # type: Optional[PythonInterpreter]
         pip_verbosity=0,  # type: int
         finalizer=None,  # type: Optional[Callable[[], None]]
+        extra_env=None,  # type: Optional[Dict[str, str]]
         **popen_kwargs  # type: Any
     ):
         # type: (...) -> Job
@@ -507,6 +508,7 @@ def _spawn_pip_isolated_job(
             cache=cache,
             interpreter=interpreter,
             pip_verbosity=pip_verbosity,
+            extra_env=extra_env,
             **popen_kwargs
         )
         return Job(command=command, process=process, finalizer=finalizer)
@@ -620,7 +622,7 @@ def spawn_download_distributions(
         if requirements:
             download_cmd.extend(requirements)
 
-        env = None
+        extra_env = None
         log_analyzers = []  # type: List[_LogAnalyzer]
 
         # Pip evaluates environment markers in the context of the ambient interpreter instead of
@@ -639,8 +641,7 @@ def spawn_download_distributions(
                 os.path.join(env_markers_dir, "env_markers.{}.json".format(platform)), "w"
             ) as fp:
                 json.dump(patched_environment, fp)
-            env = os.environ.copy()
-            env[self._PATCHED_MARKERS_FILE_ENV_VAR_NAME] = fp.name
+            extra_env = {self._PATCHED_MARKERS_FILE_ENV_VAR_NAME: fp.name}
             log_analyzers.append(_Issue10050Analyzer(platform=platform))
             TRACER.log(
                 "Patching environment markers for {} with {}".format(target, patched_environment),
@@ -668,7 +669,7 @@ def spawn_download_distributions(
             package_index_configuration=package_index_configuration,
             cache=cache,
             interpreter=target.get_interpreter(),
-            env=env,
+            extra_env=extra_env,
         )
         if log:
             return _LogScrapeJob(command, process, log, log_analyzers)


Test patch:
diff --git a/tests/test_pip.py b/tests/test_pip.py
--- a/tests/test_pip.py
+++ b/tests/test_pip.py
@@ -15,7 +15,7 @@
 from pex.jobs import Job
 from pex.pip import PackageIndexConfiguration, Pip, ResolverVersion
 from pex.platforms import Platform
-from pex.testing import PY38, ensure_python_interpreter
+from pex.testing import PY38, ensure_python_interpreter, environment_as
 from pex.typing import TYPE_CHECKING
 from pex.variables import ENV
 
@@ -32,8 +32,17 @@ def current_interpreter():
 
 
 @pytest.fixture
-def create_pip(tmpdir):
-    # type: (Any) -> Iterator[CreatePip]
+def pex_root(tmpdir):
+    # type: (Any) -> str
+    return os.path.join(str(tmpdir), "pex_root")
+
+
+@pytest.fixture
+def create_pip(
+    pex_root,  # type: str
+    tmpdir,  # type: Any
+):
+    # type: (...) -> Iterator[CreatePip]
     pex_root = os.path.join(str(tmpdir), "pex_root")
     pip_root = os.path.join(str(tmpdir), "pip_root")
 
@@ -126,7 +135,7 @@ def assert_ansicolors_downloaded(target=None):
     assert [os.path.basename(ansicolors_wheel)] == os.listdir(download_dir)
 
 
-def test_download_platform_markers_issue_1366(
+def assert_download_platform_markers_issue_1366(
     create_pip,  # type: CreatePip
     tmpdir,  # type: Any
 ):
@@ -146,6 +155,30 @@ def test_download_platform_markers_issue_1366(
     assert ["typing_extensions-3.7.4.2-py2-none-any.whl"] == os.listdir(download_dir)
 
 
+def test_download_platform_markers_issue_1366(
+    create_pip,  # type: CreatePip
+    tmpdir,  # type: Any
+):
+    # type: (...) -> None
+    assert_download_platform_markers_issue_1366(create_pip, tmpdir)
+
+
+def test_download_platform_markers_issue_1366_issue_1387(
+    create_pip,  # type: CreatePip
+    pex_root,  # type: str
+    tmpdir,  # type: Any
+):
+    # type: (...) -> None
+
+    # As noted in https://github.com/pantsbuild/pex/issues/1387, previously, internal env vars were
+    # passed by 1st cloning the ambient environment and then adding internal env vars for
+    # subprocesses to see. This could lead to duplicate keyword argument errors when env vars we
+    # patch - like PEX_ROOT - are also present in the ambient environment. This test verifies we
+    # are not tripped up by such ambient environment variables.
+    with environment_as(PEX_ROOT=pex_root):
+        assert_download_platform_markers_issue_1366(create_pip, tmpdir)
+
+
 def test_download_platform_markers_issue_1366_indeterminate(
     create_pip,  # type: CreatePip
     tmpdir,  # type: Any