Repository: pantsbuild/pants

Issue:
v2 pex creation and execution should disable the Pex cache
Discovered running in a more restrictive remote execution environment than RBE where the filesystem is mounted read-only saving for /tmp and the remote execution chroot. In this case the default cache root of `~/.pex` lands on a read-only portion of the FS and leads to a write failure. We never wanted Pex to use it's own cache in these contexts, we just had forgotten it was doing so. 


Discussion:

Resolving patch:
diff --git a/src/python/pants/backend/python/rules/create_requirements_pex.py b/src/python/pants/backend/python/rules/create_requirements_pex.py
--- a/src/python/pants/backend/python/rules/create_requirements_pex.py
+++ b/src/python/pants/backend/python/rules/create_requirements_pex.py
@@ -2,16 +2,16 @@
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
 from pants.backend.python.rules.download_pex_bin import DownloadedPexBin
-from pants.backend.python.subsystems.python_native_code import PexBuildEnvironment, PythonNativeCode
+from pants.backend.python.rules.hermetic_pex import HermeticPex
+from pants.backend.python.subsystems.python_native_code import PexBuildEnvironment
 from pants.backend.python.subsystems.python_setup import PythonSetup
+from pants.backend.python.subsystems.subprocess_environment import SubprocessEncodingEnvironment
 from pants.engine.fs import Digest
-from pants.engine.isolated_process import (ExecuteProcessRequest, ExecuteProcessResult,
-                                           MultiPlatformExecuteProcessRequest)
+from pants.engine.isolated_process import ExecuteProcessResult, MultiPlatformExecuteProcessRequest
 from pants.engine.platform import Platform, PlatformConstraint
 from pants.engine.rules import optionable_rule, rule
 from pants.engine.selectors import Get
 from pants.util.objects import datatype, hashable_string_list, string_optional, string_type
-from pants.util.strutil import create_path_env_var
 
 
 class RequirementsPexRequest(datatype([
@@ -23,7 +23,7 @@ class RequirementsPexRequest(datatype([
   pass
 
 
-class RequirementsPex(datatype([('directory_digest', Digest)])):
+class RequirementsPex(HermeticPex, datatype([('directory_digest', Digest)])):
   pass
 
 
@@ -34,51 +34,50 @@ def create_requirements_pex(
     request: RequirementsPexRequest,
     pex_bin: DownloadedPexBin,
     python_setup: PythonSetup,
+    subprocess_encoding_environment: SubprocessEncodingEnvironment,
     pex_build_environment: PexBuildEnvironment,
     platform: Platform
 ) -> RequirementsPex:
   """Returns a PEX with the given requirements, optional entry point, and optional
   interpreter constraints."""
 
-  interpreter_search_paths = create_path_env_var(python_setup.interpreter_search_paths)
-  env = {"PATH": interpreter_search_paths, **pex_build_environment.invocation_environment_dict}
-
   interpreter_constraint_args = []
   for constraint in request.interpreter_constraints:
     interpreter_constraint_args.extend(["--interpreter-constraint", constraint])
 
-  # NB: we use the hardcoded and generic bin name `python`, rather than something dynamic like
-  # `sys.executable`, to ensure that the interpreter may be discovered both locally and in remote
-  # execution (so long as `env` is populated with a `PATH` env var and `python` is discoverable
-  # somewhere on that PATH). This is only used to run the downloaded PEX tool; it is not
-  # necessarily the interpreter that PEX will use to execute the generated .pex file.
-  # TODO(#7735): Set --python-setup-interpreter-search-paths differently for the host and target
-  # platforms, when we introduce platforms in https://github.com/pantsbuild/pants/issues/7735.
-  argv = ["python", f"./{pex_bin.executable}", "--output-file", request.output_filename]
+  argv = ["--output-file", request.output_filename]
   if request.entry_point is not None:
     argv.extend(["--entry-point", request.entry_point])
   argv.extend(interpreter_constraint_args + list(request.requirements))
-  # NOTE
-  # PEX outputs are platform dependent so in order to get a PEX that we can use locally, without cross-building
-  # we specify that out PEX command be run on the current local platform. When we support cross-building
-  # through CLI flags we can configure requests that build a PEX for out local platform that are
-  # able to execute on a different platform, but for now in order to guarantee correct build we need
-  # to restrict this command to execute on the same platform type that the output is intended for.
-  # The correct way to interpret the keys (execution_platform_constraint, target_platform_constraint)
-  # of this dictionary is "The output of this command is intended for `target_platform_constraint` iff
-  # it is run on `execution_platform_constraint`".
+
+  # NB: PEX outputs are platform dependent so in order to get a PEX that we can use locally, without
+  # cross-building we specify that out PEX command be run on the current local platform. When we
+  # support cross-building through CLI flags we can configure requests that build a PEX for out
+  # local platform that are able to execute on a different platform, but for now in order to
+  # guarantee correct build we need to restrict this command to execute on the same platform type
+  # that the output is intended for. The correct way to interpret the keys
+  # (execution_platform_constraint, target_platform_constraint) of this dictionary is "The output of
+  # this command is intended for `target_platform_constraint` iff it is run on `execution_platform
+  # constraint`".
   execute_process_request = MultiPlatformExecuteProcessRequest(
     {
-      (PlatformConstraint(platform.value), PlatformConstraint(platform.value)): ExecuteProcessRequest(
-        argv=tuple(argv),
-        env=env,
-        input_files=pex_bin.directory_digest,
-        description=f"Create a requirements PEX: {', '.join(request.requirements)}",
-        output_files=(request.output_filename,))
+      (PlatformConstraint(platform.value), PlatformConstraint(platform.value)):
+        pex_bin.create_execute_request(
+          python_setup=python_setup,
+          subprocess_encoding_environment=subprocess_encoding_environment,
+          pex_build_environment=pex_build_environment,
+          pex_args=argv,
+          description=f"Create a requirements PEX: {', '.join(request.requirements)}",
+          output_files=(request.output_filename,)
+        )
     }
   )
 
-  result = yield Get(ExecuteProcessResult, MultiPlatformExecuteProcessRequest, execute_process_request)
+  result = yield Get(
+    ExecuteProcessResult,
+    MultiPlatformExecuteProcessRequest,
+    execute_process_request
+  )
   yield RequirementsPex(directory_digest=result.output_directory_digest)
 
 
@@ -86,5 +85,4 @@ def rules():
   return [
     create_requirements_pex,
     optionable_rule(PythonSetup),
-    optionable_rule(PythonNativeCode),
   ]
diff --git a/src/python/pants/backend/python/rules/download_pex_bin.py b/src/python/pants/backend/python/rules/download_pex_bin.py
--- a/src/python/pants/backend/python/rules/download_pex_bin.py
+++ b/src/python/pants/backend/python/rules/download_pex_bin.py
@@ -1,14 +1,57 @@
 # Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
+from typing import Any, Iterable
+
+from pants.backend.python.rules.hermetic_pex import HermeticPex
+from pants.backend.python.subsystems.python_native_code import PexBuildEnvironment
+from pants.backend.python.subsystems.python_setup import PythonSetup
+from pants.backend.python.subsystems.subprocess_environment import SubprocessEncodingEnvironment
 from pants.engine.fs import Digest, Snapshot, UrlToFetch
+from pants.engine.isolated_process import ExecuteProcessRequest
 from pants.engine.rules import rule
 from pants.engine.selectors import Get
 from pants.util.objects import datatype
 
 
-class DownloadedPexBin(datatype([('executable', str), ('directory_digest', Digest)])):
-  pass
+class DownloadedPexBin(HermeticPex, datatype([('executable', str), ('directory_digest', Digest)])):
+
+  def create_execute_request(self,
+    python_setup: PythonSetup,
+    subprocess_encoding_environment: SubprocessEncodingEnvironment,
+    pex_build_environment: PexBuildEnvironment,
+    *,
+    pex_args: Iterable[str],
+    description: str,
+    input_files: Digest = None,
+    **kwargs: Any
+  ) -> ExecuteProcessRequest:
+    """Creates an ExecuteProcessRequest that will run the pex CLI tool hermetically.
+
+    :param python_setup: The parameters for selecting python interpreters to use when invoking the
+                         pex tool.
+    :param subprocess_encoding_environment: The locale settings to use for the pex tool invocation.
+    :param pex_path: The path within `input_files` of the pex tool.
+    :param pex_args: The arguments to pass to the pex CLI tool.
+    :param description: A description of the process execution to be performed.
+    :param input_files: The files that contain the pex CLI tool itself and any input files it needs
+                        to run against. By default just the files that contain the pex CLI tool
+                        itself. To merge in additional files, include the `directory_digest` in
+                        `DirectoriesToMerge` request.
+    :param env: The environment to run the pex CLI tool in.
+    :param **kwargs: Any additional :class:`ExecuteProcessRequest` kwargs to pass through.
+    """
+
+    return super().create_execute_request(
+      python_setup=python_setup,
+      subprocess_encoding_environment=subprocess_encoding_environment,
+      pex_path=self.executable,
+      pex_args=["--disable-cache"] + list(pex_args),
+      description=description,
+      input_files=input_files or self.directory_digest,
+      env=pex_build_environment.invocation_environment_dict,
+      **kwargs
+    )
 
 
 @rule
diff --git a/src/python/pants/backend/python/rules/hermetic_pex.py b/src/python/pants/backend/python/rules/hermetic_pex.py
new file mode 100644
--- /dev/null
+++ b/src/python/pants/backend/python/rules/hermetic_pex.py
@@ -0,0 +1,66 @@
+# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from typing import Any, Dict, Iterable, Optional
+
+from pants.backend.python.subsystems.python_setup import PythonSetup
+from pants.backend.python.subsystems.subprocess_environment import SubprocessEncodingEnvironment
+from pants.engine.fs import Digest
+from pants.engine.isolated_process import ExecuteProcessRequest
+from pants.util.strutil import create_path_env_var
+
+
+class HermeticPex:
+  """A mixin for types that provide an executable Pex that should be executed hermetically."""
+
+  def create_execute_request(
+    self,
+    python_setup: PythonSetup,
+    subprocess_encoding_environment: SubprocessEncodingEnvironment,
+    *,
+    pex_path: str,
+    pex_args: Iterable[str],
+    description: str,
+    input_files: Digest,
+    env: Optional[Dict[str, str]] = None,
+    **kwargs: Any
+  ) -> ExecuteProcessRequest:
+    """Creates an ExecuteProcessRequest that will run a PEX hermetically.
+
+    :param python_setup: The parameters for selecting python interpreters to use when invoking the
+                         PEX.
+    :param subprocess_encoding_environment: The locale settings to use for the PEX invocation.
+    :param pex_path: The path within `input_files` of the PEX file (or directory if a loose pex).
+    :param pex_args: The arguments to pass to the PEX executable.
+    :param description: A description of the process execution to be performed.
+    :param input_files: The files that contain the pex itself and any input files it needs to run
+                        against.
+    :param env: The environment to run the PEX in.
+    :param **kwargs: Any additional :class:`ExecuteProcessRequest` kwargs to pass through.
+    """
+
+    # NB: we use the hardcoded and generic bin name `python`, rather than something dynamic like
+    # `sys.executable`, to ensure that the interpreter may be discovered both locally and in remote
+    # execution (so long as `env` is populated with a `PATH` env var and `python` is discoverable
+    # somewhere on that PATH). This is only used to run the downloaded PEX tool; it is not
+    # necessarily the interpreter that PEX will use to execute the generated .pex file.
+    # TODO(#7735): Set --python-setup-interpreter-search-paths differently for the host and target
+    # platforms, when we introduce platforms in https://github.com/pantsbuild/pants/issues/7735.
+    argv = ('python', pex_path, *pex_args)
+
+    hermetic_env = env.copy() if env else {}
+    hermetic_env.update(
+      PATH=create_path_env_var(python_setup.interpreter_search_paths),
+      PEX_ROOT='./pex_root',
+      PEX_INHERIT_PATH='false',
+      PEX_IGNORE_RCFILES='true',
+      **subprocess_encoding_environment.invocation_environment_dict
+    )
+
+    return ExecuteProcessRequest(
+      argv=argv,
+      input_files=input_files,
+      description=description,
+      env=hermetic_env,
+      **kwargs
+    )


Test patch:
diff --git a/src/python/pants/backend/python/rules/python_test_runner.py b/src/python/pants/backend/python/rules/python_test_runner.py
--- a/src/python/pants/backend/python/rules/python_test_runner.py
+++ b/src/python/pants/backend/python/rules/python_test_runner.py
@@ -16,7 +16,6 @@
 from pants.engine.selectors import Get
 from pants.rules.core.core_test_model import Status, TestResult, TestTarget
 from pants.rules.core.strip_source_root import SourceRootStrippedSources
-from pants.util.strutil import create_path_env_var
 
 
 @rule
@@ -98,25 +97,17 @@ def run_python_test(
     DirectoriesToMerge(directories=tuple(all_input_digests)),
   )
 
-  interpreter_search_paths = create_path_env_var(python_setup.interpreter_search_paths)
-  pex_exe_env = {
-    'PATH': interpreter_search_paths,
-    **subprocess_encoding_environment.invocation_environment_dict
-  }
-
   test_target_sources_file_names = sorted(source_root_stripped_test_target_sources.snapshot.files)
   # NB: we use the hardcoded and generic bin name `python`, rather than something dynamic like
   # `sys.executable`, to ensure that the interpreter may be discovered both locally and in remote
   # execution (so long as `env` is populated with a `PATH` env var and `python` is discoverable
   # somewhere on that PATH). This is only used to run the downloaded PEX tool; it is not
   # necessarily the interpreter that PEX will use to execute the generated .pex file.
-  request = ExecuteProcessRequest(
-    argv=(
-      "python",
-      f'./{output_pytest_requirements_pex_filename}',
-      *test_target_sources_file_names
-    ),
-    env=pex_exe_env,
+  request = resolved_requirements_pex.create_execute_request(
+    python_setup=python_setup,
+    subprocess_encoding_environment=subprocess_encoding_environment,
+    pex_path=f'./{output_pytest_requirements_pex_filename}',
+    pex_args=test_target_sources_file_names,
     input_files=merged_input_files,
     description=f'Run Pytest for {test_target.address.reference()}',
   )
diff --git a/tests/python/pants_test/backend/python/rules/test_create_requirements_pex.py b/tests/python/pants_test/backend/python/rules/test_create_requirements_pex.py
--- a/tests/python/pants_test/backend/python/rules/test_create_requirements_pex.py
+++ b/tests/python/pants_test/backend/python/rules/test_create_requirements_pex.py
@@ -12,6 +12,8 @@
 from pants.backend.python.subsystems.python_native_code import (PythonNativeCode,
                                                                 create_pex_native_build_environment)
 from pants.backend.python.subsystems.python_setup import PythonSetup
+from pants.backend.python.subsystems.subprocess_environment import (
+  SubprocessEnvironment, create_subprocess_encoding_environment)
 from pants.engine.fs import DirectoryToMaterialize
 from pants.engine.rules import RootRule
 from pants.engine.selectors import Params
@@ -28,15 +30,17 @@ def rules(cls):
     return super().rules() + [
       create_requirements_pex,
       create_pex_native_build_environment,
+      create_subprocess_encoding_environment,
       download_pex_bin,
       RootRule(RequirementsPexRequest),
       RootRule(PythonSetup),
       RootRule(PythonNativeCode),
+      RootRule(SubprocessEnvironment)
     ]
 
   def setUp(self):
     super().setUp()
-    init_subsystems([PythonSetup, PythonNativeCode])
+    init_subsystems([PythonSetup, PythonNativeCode, SubprocessEnvironment])
 
   def create_pex_and_get_pex_info(
     self, *, requirements=None, entry_point=None, interpreter_constraints=None
@@ -54,6 +58,7 @@ def hashify_optional_collection(iterable):
       self.scheduler.product_request(RequirementsPex, [Params(
         request,
         PythonSetup.global_instance(),
+        SubprocessEnvironment.global_instance(),
         PythonNativeCode.global_instance()
       )])
     )