Repository: pex-tool/pex

Issue:
Lockfile: missing `pip.log`
Using Pex 2.71 and the Docker image `quay.io/pypa/manylinux2014_x86_64:latest`, I get this failure:

```
PEX_SCRIPT=pex3 /opt/python/cp37-cp37m/bin/python ./pex lock create --resolver-version=pip-2020-resolver ansicolors
Traceback (most recent call last):
  File "/root/.pex/venvs/685195326480c4d9d521e9cc3994b066950eeb1e/f084cae1efd893aa7ecfef66d45dea1a5e619caa/pex", line 126, in <module>
    os.execv(script_path, [script_path] + sys.argv[1:])
FileNotFoundError: [Errno 2] No such file or directory
Traceback (most recent call last):
  File "/root/.pex/unzipped_pexes/3c037481d92e64c70b4c4d598b63e24e6c2bb819/.bootstrap/pex/pex.py", line 503, in execute
    exit_value = self._wrap_coverage(self._wrap_profiling, self._execute)
  File "/root/.pex/unzipped_pexes/3c037481d92e64c70b4c4d598b63e24e6c2bb819/.bootstrap/pex/pex.py", line 408, in _wrap_coverage
    return runner(*args)
  File "/root/.pex/unzipped_pexes/3c037481d92e64c70b4c4d598b63e24e6c2bb819/.bootstrap/pex/pex.py", line 439, in _wrap_profiling
    return runner(*args)
  File "/root/.pex/unzipped_pexes/3c037481d92e64c70b4c4d598b63e24e6c2bb819/.bootstrap/pex/pex.py", line 553, in _execute
    return self.execute_script(self._pex_info_overrides.script)
  File "/root/.pex/unzipped_pexes/3c037481d92e64c70b4c4d598b63e24e6c2bb819/.bootstrap/pex/pex.py", line 635, in execute_script
    return self.execute_entry(entry_point)
  File "/root/.pex/unzipped_pexes/3c037481d92e64c70b4c4d598b63e24e6c2bb819/.bootstrap/pex/pex.py", line 695, in execute_entry
    return self.execute_pkg_resources(entry_point)
  File "/root/.pex/unzipped_pexes/3c037481d92e64c70b4c4d598b63e24e6c2bb819/.bootstrap/pex/pex.py", line 727, in execute_pkg_resources
    return runner()
  File "/root/.pex/installed_wheels/9fb4bc59f7085c6133bed415fd8806a166d6f472798f2350158f7f0a710ec0f6/pex-2.1.71-py2.py3-none-any.whl/pex/cli/pex.py", line 30, in main
    result = catch(command.run)
  File "/root/.pex/installed_wheels/9fb4bc59f7085c6133bed415fd8806a166d6f472798f2350158f7f0a710ec0f6/pex-2.1.71-py2.py3-none-any.whl/pex/result.py", line 101, in catch
    return func(*args, **kwargs)
  File "/root/.pex/installed_wheels/9fb4bc59f7085c6133bed415fd8806a166d6f472798f2350158f7f0a710ec0f6/pex-2.1.71-py2.py3-none-any.whl/pex/cli/command.py", line 84, in run
    return subcommand_func(self)
  File "/root/.pex/installed_wheels/9fb4bc59f7085c6133bed415fd8806a166d6f472798f2350158f7f0a710ec0f6/pex-2.1.71-py2.py3-none-any.whl/pex/cli/commands/lock.py", line 248, in _create
    pip_configuration=pip_configuration,
  File "/root/.pex/installed_wheels/9fb4bc59f7085c6133bed415fd8806a166d6f472798f2350158f7f0a710ec0f6/pex-2.1.71-py2.py3-none-any.whl/pex/resolve/lockfile/__init__.py", line 253, in create
    dest=dest,
  File "/root/.pex/installed_wheels/9fb4bc59f7085c6133bed415fd8806a166d6f472798f2350158f7f0a710ec0f6/pex-2.1.71-py2.py3-none-any.whl/pex/resolver.py", line 1200, in download
    lock_configuration=lock_configuration,
  File "/root/.pex/installed_wheels/9fb4bc59f7085c6133bed415fd8806a166d6f472798f2350158f7f0a710ec0f6/pex-2.1.71-py2.py3-none-any.whl/pex/resolver.py", line 1086, in _download_internal
    dest=dest, max_parallel_jobs=max_parallel_jobs
  File "/root/.pex/installed_wheels/9fb4bc59f7085c6133bed415fd8806a166d6f472798f2350158f7f0a710ec0f6/pex-2.1.71-py2.py3-none-any.whl/pex/resolver.py", line 141, in download_distributions
    max_jobs=max_parallel_jobs,
  File "/root/.pex/installed_wheels/9fb4bc59f7085c6133bed415fd8806a166d6f472798f2350158f7f0a710ec0f6/pex-2.1.71-py2.py3-none-any.whl/pex/jobs.py", line 512, in execute_parallel
    yield spawn_result.spawned_job.await_result()
  File "/root/.pex/installed_wheels/9fb4bc59f7085c6133bed415fd8806a166d6f472798f2350158f7f0a710ec0f6/pex-2.1.71-py2.py3-none-any.whl/pex/jobs.py", line 216, in await_result
    job.wait()
  File "/root/.pex/installed_wheels/9fb4bc59f7085c6133bed415fd8806a166d6f472798f2350158f7f0a710ec0f6/pex-2.1.71-py2.py3-none-any.whl/pex/jobs.py", line 75, in wait
    self._check_returncode(stderr)
  File "/root/.pex/installed_wheels/9fb4bc59f7085c6133bed415fd8806a166d6f472798f2350158f7f0a710ec0f6/pex-2.1.71-py2.py3-none-any.whl/pex/pip.py", line 450, in _check_returncode
    with open(self._log, "r") as fp:
FileNotFoundError: [Errno 2] No such file or directory: '/tmp/tmp8czrmzg8/pip.log'
```

It's deterministic, but I can't get it to happen on my mac. Using the old resolver returns this error. 

```
Traceback (most recent call last):
  File "/root/.pex/venvs/685195326480c4d9d521e9cc3994b066950eeb1e/f084cae1efd893aa7ecfef66d45dea1a5e619caa/pex", line 126, in <module>
    os.execv(script_path, [script_path] + sys.argv[1:])
FileNotFoundError: [Errno 2] No such file or directory
pid 751 -> /root/.pex/venvs/685195326480c4d9d521e9cc3994b066950eeb1e/f084cae1efd893aa7ecfef66d45dea1a5e619caa/pex --disable-pip-version-check --no-python-version-warning --exists-action a --use-deprecated legacy-resolver --isolated -q --cache-dir /root/.pex --log /tmp/tmpu7jsac5m/pip.log download --dest /tmp/tmpdpb9zdq_/opt._internal.cpython-3.7.12.bin.python3.7 ansicolors --index-url https://pypi.org/simple --retries 5 --timeout 15 exited with 1 and STDERR:
None
```


Discussion:
Noting this is specifically a Pex PEX issue. Eric's odd looking choice of means to execute Pex here is intended (I think) and the issue. Pants executes Pex this way. It's also worth pointing out this has nothing to do with `pip.log` or locking. The error is earlier trying to re-direct to the `pex3` script. It's also worth noting this has nothing to do with docker, I can get the error on a bare Arch machine. It's also sensitive to how the `~/.pex` cache was populated. I can make the error go away by populating the `~/.pex` cache else-wise in hand-wavy ways. If I nuke the cache though and run Eric's command, I can get failure every time; i.e.: when unzipping and venv creation are initiated from the Pex PEX.
Specifically, this is the Pex dogfood Pip venv PEX being used internally looking for and not finding `pex3`. That Pip venv definitely doesn't and shouldn't have `pex3`; so that much is good. The only question is how `PEX_SCRIPT` is making it down there now when it apparently wasn't before.
I'm not sure how this ever worked:
```
^jsirois@gill ~/dev/pantsbuild/pex ((v2.1.60)|BISECTING) $ rm -rf .tox dist /tmp/pex_root && tox -epackage -- --local && PEX_ROOT=/tmp/pex_root PEX_SCRIPT=pex3 /usr/bin/python3.7 dist/pex lock create --resolver-version=pip-2020-resolver ansicolors --indent 2
package create: /home/jsirois/dev/pantsbuild/pex/.tox/package
package installdeps: flit, pygments, toml
package installed: certifi==2021.10.8,charset-normalizer==2.0.12,docutils==0.18.1,flit==3.7.1,flit_core==3.7.1,idna==3.3,Pygments==2.11.2,requests==2.27.1,toml==0.10.2,tomli==2.0.1,tomli_w==1.0.0,urllib3==1.26.8
package run-test-pre: PYTHONHASHSEED='2745318215'
package run-test: commands[0] | python scripts/package.py --local
Building Pex PEX to `dist/pex` ...
Built Pex PEX @ v2.1.60:
sha256: 7df92784aa8a8b0ed2ce4bbc99388fb2cb5b6e7966ab6a1295ff25d399d25fb2
  size: 3693748
____________________________________________________________________________________________________________________________________ summary ____________________________________________________________________________________________________________________________________
  package: commands succeeded
  congratulations :)
Traceback (most recent call last):
  File "/tmp/pex_root/venvs/d2793a662f3a73ad3be3ab7149bad4426b7468b5/8526d9ba87349150aa1391090e89d679ed25a846/pex", line 111, in <module>
    os.execv(script_path, [script_path] + sys.argv[1:])
FileNotFoundError: [Errno 2] No such file or directory
```

Here Pex 2.1.60 was the version the `PEX_SCRIPT=pex3` means of invoking the `pex.cli` module was 1st introduced in Pants.
Yeah, this never worked. Iff you 1st ran the Pex PEX without `PEX_SCRIPT=...` and the PEX_ROOT got populated with a Pip venv clear of that, then things would work. 
Ah ha. To try to reproduce a bug, I indeed was using a Docker container and the first thing I ran was `./pants --python-lockfile-generator=pex generate-lockfiles`.

Resolving patch:
diff --git a/pex/pex.py b/pex/pex.py
--- a/pex/pex.py
+++ b/pex/pex.py
@@ -79,11 +79,12 @@ def __init__(
         if verify_entry_point:
             self._do_entry_point_verification()
 
-    def pex_info(self):
-        # type: () -> PexInfo
+    def pex_info(self, include_env_overrides=True):
+        # type: (bool) -> PexInfo
         pex_info = self._pex_info.copy()
-        pex_info.update(self._pex_info_overrides)
-        pex_info.merge_pex_path(self._vars.PEX_PATH)
+        if include_env_overrides:
+            pex_info.update(self._pex_info_overrides)
+            pex_info.merge_pex_path(self._vars.PEX_PATH)
         return pex_info
 
     @property
diff --git a/pex/venv/pex.py b/pex/venv/pex.py
--- a/pex/venv/pex.py
+++ b/pex/venv/pex.py
@@ -286,9 +286,14 @@ def _populate_sources(
 ):
     # type: (...) -> Iterator[Tuple[str, str]]
 
+    # We want the venv at rest to reflect the PEX it was created from at rest; as such we use the
+    # PEX's at-rest PEX-INFO to perform the layout. The venv can then be executed with various PEX
+    # environment variables in-play that it respects (e.g.: PEX_EXTRA_SYS_PATH, PEX_INTERPRETER,
+    # PEX_MODULE, etc.).
+    pex_info = pex.pex_info(include_env_overrides=False)
+
     # Since the pex.path() is ~always outside our control (outside ~/.pex), we copy all PEX user
     # sources into the venv.
-    pex_info = pex.pex_info()
     for src, dst in _copytree(
         src=PEXEnvironment.mount(pex.path()).path,
         dst=venv.site_packages_dir,


Test patch:
diff --git a/tests/integration/venv_ITs/test_issue_1668.py b/tests/integration/venv_ITs/test_issue_1668.py
new file mode 100644
--- /dev/null
+++ b/tests/integration/venv_ITs/test_issue_1668.py
@@ -0,0 +1,74 @@
+# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+import os.path
+import subprocess
+
+from pex.testing import PY37, ensure_python_interpreter, make_env, pex_project_dir, run_pex_command
+from pex.typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from typing import Any
+
+
+def assert_venv_runtime_env_vars_ignored_during_create(
+    tmpdir,  # type: Any
+    pex_venv,  # type: bool
+):
+    # type: (...) -> None
+
+    pex_pex = os.path.join(str(tmpdir), "pex.pex")
+    args = [pex_project_dir(), "-c", "pex", "-o", pex_pex, "--no-strip-pex-env", "--disable-cache"]
+    if pex_venv:
+        args.append("--venv")
+    run_pex_command(args=args).assert_success()
+
+    py37 = ensure_python_interpreter(PY37)
+    pex_root = os.path.join(str(tmpdir), "pex_root")
+    lock = os.path.join(str(tmpdir), "lock.json")
+    subprocess.check_call(
+        args=[
+            py37,
+            pex_pex,
+            "lock",
+            "create",
+            "ansicolors==1.1.8",
+            "-o",
+            lock,
+            "--pex-root",
+            pex_root,
+        ],
+        env=make_env(PEX_SCRIPT="pex3"),
+    )
+    ansicolors_path = (
+        subprocess.check_output(
+            args=[
+                py37,
+                pex_pex,
+                "--pex-root",
+                pex_root,
+                "--runtime-pex-root",
+                pex_root,
+                "--lock",
+                lock,
+                "--",
+                "-c",
+                "import colors; print(colors.__file__)",
+            ]
+        )
+        .decode("utf-8")
+        .strip()
+    )
+    assert ansicolors_path.startswith(pex_root)
+
+
+def test_venv_runtime_env_vars_ignored_during_create_nested(tmpdir):
+    # type: (Any) -> None
+
+    # N.B.: The venv being created here is the internal Pip venv Pex uses to execute Pip. It's that
+    # venv that used to get PEX env vars like PEX_MODULE and PEX_SCRIPT sealed in at creation time.
+    assert_venv_runtime_env_vars_ignored_during_create(tmpdir, pex_venv=False)
+
+
+def test_venv_runtime_env_vars_ignored_during_create_top_level(tmpdir):
+    assert_venv_runtime_env_vars_ignored_during_create(tmpdir, pex_venv=True)
diff --git a/tests/test_pip.py b/tests/test_pip.py
--- a/tests/test_pip.py
+++ b/tests/test_pip.py
@@ -21,9 +21,16 @@
 from pex.variables import ENV
 
 if TYPE_CHECKING:
-    from typing import Any, Callable, Iterator, Optional
+    from typing import Any, Iterator, Optional, Protocol
 
-    CreatePip = Callable[[Optional[PythonInterpreter]], Pip]
+    class CreatePip(Protocol):
+        def __call__(
+            self,
+            interpreter,  # type: Optional[PythonInterpreter]
+            **extra_env  # type: str
+        ):
+            # type: (...) -> Pip
+            pass
 
 
 @pytest.fixture
@@ -47,13 +54,15 @@ def create_pip(
     pex_root = os.path.join(str(tmpdir), "pex_root")
     pip_root = os.path.join(str(tmpdir), "pip_root")
 
-    with ENV.patch(PEX_ROOT=pex_root):
-
-        def create_pip(interpreter):
-            # type: (Optional[PythonInterpreter]) -> Pip
+    def create_pip(
+        interpreter,  # type: Optional[PythonInterpreter]
+        **extra_env  # type: str
+    ):
+        # type: (...) -> Pip
+        with ENV.patch(PEX_ROOT=pex_root, **extra_env):
             return Pip.create(path=pip_root, interpreter=interpreter)
 
-        yield create_pip
+    yield create_pip
 
 
 def test_no_duplicate_constraints_pex_warnings(
@@ -239,3 +248,16 @@ def test_download_platform_markers_issue_1488(
         )
         == sorted(os.listdir(download_dir))
     )
+
+
+def test_create_confounding_env_vars_issue_1668(
+    create_pip,  # type: CreatePip
+    tmpdir,  # type: Any
+):
+    # type: (...) -> None
+
+    download_dir = os.path.join(str(tmpdir), "downloads")
+    create_pip(None, PEX_SCRIPT="pex3").spawn_download_distributions(
+        requirements=["ansicolors==1.1.8"], download_dir=download_dir
+    ).wait()
+    assert ["ansicolors-1.1.8-py2.py3-none-any.whl"] == os.listdir(download_dir)