Repository: pantsbuild/pants

Issue:
`[python-native-code]` breaks Pantsd cache invalidation
We should not be using `os.environ`. It needs to use the engine to request `Environment` instead.

https://github.com/pantsbuild/pants/blob/8f62a1c0016a19908be658d950ebdeab30a431bc/src/python/pants/backend/python/subsystems/python_native_code.py#L19-L20

We might need to add a new magic string like `<CPPFLAGS>`, similar to how we handle `<PATH>`.

https://github.com/pantsbuild/pants/blob/8f62a1c0016a19908be658d950ebdeab30a431bc/src/python/pants/backend/go/subsystems/golang.py#L65-L75

While fixing, if we don't use a magic string like `<CPPFLAGS>`, we should also use `default_str` so that our docs don't put whatever the value of the docs release is.

--

Even better would be removing `[python-native-code]` and consolidating it into something like `[subprocess-environment]` or `[python]`. The subsystem is confusing and a holdover from v1. I tried consolidating, but gave up because I couldn't avoid breaking the deprecation policy.

> consolidating it into something like `[subprocess-environment]` or `[python]`

It seems like we've gone in the direction of env vars per language, so `[subprocess-environment]` is probably a misnomer.


Discussion:
For deprecation purposes, I'm proposing adding a deprecation message for the envvar `CPPFLAGS` being set if the environment target is a local environment, and an error if not. The magic string approach seems OK for a new method that should explicitly only work on local environments.

Consolidating the various Python subsystems looks like a good idea, probably worth undertaking as a separate ticket though.

Resolving patch:
diff --git a/src/python/pants/backend/python/subsystems/python_native_code.py b/src/python/pants/backend/python/subsystems/python_native_code.py
--- a/src/python/pants/backend/python/subsystems/python_native_code.py
+++ b/src/python/pants/backend/python/subsystems/python_native_code.py
@@ -1,9 +1,13 @@
 # Copyright 2017 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-import os
-from typing import Dict
+from __future__ import annotations
 
+from dataclasses import dataclass
+from typing import Dict, Sequence
+
+from pants.engine.env_vars import EnvironmentVars, EnvironmentVarsRequest
+from pants.engine.rules import Get, collect_rules, rule
 from pants.option.option_types import StrListOption
 from pants.option.subsystem import Subsystem
 from pants.util.strutil import safe_shlex_join, safe_shlex_split
@@ -13,22 +17,61 @@ class PythonNativeCodeSubsystem(Subsystem):
     options_scope = "python-native-code"
     help = "Options for building native code using Python, e.g. when resolving distributions."
 
-    class EnvironmentAware:
+    class EnvironmentAware(Subsystem.EnvironmentAware):
         # TODO(#7735): move the --cpp-flags and --ld-flags to a general subprocess support subsystem.
-        cpp_flags = StrListOption(
-            default=safe_shlex_split(os.environ.get("CPPFLAGS", "")),
-            help="Override the `CPPFLAGS` environment variable for any forked subprocesses.",
+        _cpp_flags = StrListOption(
+            default=["<CPPFLAGS>"],
+            help=(
+                "Override the `CPPFLAGS` environment variable for any forked subprocesses. "
+                "Use the value `['<CPPFLAGS>']` to inherit the value of the `CPPFLAGS` "
+                "environment variable from your runtime environment target."
+            ),
             advanced=True,
         )
-        ld_flags = StrListOption(
-            default=safe_shlex_split(os.environ.get("LDFLAGS", "")),
-            help="Override the `LDFLAGS` environment variable for any forked subprocesses.",
+        _ld_flags = StrListOption(
+            default=["<LDFLAGS>"],
+            help=(
+                "Override the `LDFLAGS` environment variable for any forked subprocesses. "
+                "Use the value `['<LDFLAGS>']` to inherit the value of the `LDFLAGS` environment "
+                "variable from your runtime environment target."
+            ),
             advanced=True,
         )
 
-        @property
-        def environment_dict(self) -> Dict[str, str]:
-            return {
-                "CPPFLAGS": safe_shlex_join(self.cpp_flags),
-                "LDFLAGS": safe_shlex_join(self.ld_flags),
-            }
+
+@dataclass(frozen=True)
+class PythonNativeCodeEnvironment:
+
+    cpp_flags: tuple[str, ...]
+    ld_flags: tuple[str, ...]
+
+    @property
+    def environment_dict(self) -> Dict[str, str]:
+        return {
+            "CPPFLAGS": safe_shlex_join(self.cpp_flags),
+            "LDFLAGS": safe_shlex_join(self.ld_flags),
+        }
+
+
+@rule
+async def resolve_python_native_code_environment(
+    env_aware: PythonNativeCodeSubsystem.EnvironmentAware,
+) -> PythonNativeCodeEnvironment:
+
+    env_vars = await Get(EnvironmentVars, EnvironmentVarsRequest(("CPPFLAGS", "LDFLAGS")))
+
+    def iter_values(env_var: str, values: Sequence[str]):
+        for value in values:
+            if value == f"<{env_var}>":
+                yield from safe_shlex_split(env_vars.get(env_var, ""))
+            else:
+                yield value
+
+    return PythonNativeCodeEnvironment(
+        cpp_flags=tuple(iter_values("CPPFLAGS", env_aware._cpp_flags)),
+        ld_flags=tuple(iter_values("LDFLAGS", env_aware._ld_flags)),
+    )
+
+
+def rules():
+    return [*collect_rules()]
diff --git a/src/python/pants/backend/python/util_rules/pex_cli.py b/src/python/pants/backend/python/util_rules/pex_cli.py
--- a/src/python/pants/backend/python/util_rules/pex_cli.py
+++ b/src/python/pants/backend/python/util_rules/pex_cli.py
@@ -9,7 +9,8 @@
 from pathlib import Path
 from typing import Iterable, List, Mapping, Optional, Tuple
 
-from pants.backend.python.subsystems.python_native_code import PythonNativeCodeSubsystem
+from pants.backend.python.subsystems import python_native_code
+from pants.backend.python.subsystems.python_native_code import PythonNativeCodeEnvironment
 from pants.backend.python.util_rules import pex_environment
 from pants.backend.python.util_rules.pex_environment import (
     PexEnvironment,
@@ -124,7 +125,7 @@ async def setup_pex_cli_process(
     request: PexCliProcess,
     pex_pex: PexPEX,
     pex_env: PexEnvironment,
-    python_native_code_subsystem: PythonNativeCodeSubsystem.EnvironmentAware,
+    python_native_code_environment: PythonNativeCodeEnvironment,
     global_options: GlobalOptions,
     pex_subsystem: PexSubsystem,
 ) -> Process:
@@ -190,7 +191,7 @@ async def setup_pex_cli_process(
     normalized_argv = complete_pex_env.create_argv(pex_pex.exe, *args, python=request.python)
     env = {
         **complete_pex_env.environment_dict(python_configured=request.python is not None),
-        **python_native_code_subsystem.environment_dict,
+        **python_native_code_environment.environment_dict,
         **(request.extra_env or {}),
         # If a subcommand is used, we need to use the `pex3` console script.
         **({"PEX_SCRIPT": "pex3"} if request.subcommand else {}),
@@ -211,4 +212,9 @@ async def setup_pex_cli_process(
 
 
 def rules():
-    return [*collect_rules(), *external_tool.rules(), *pex_environment.rules()]
+    return [
+        *collect_rules(),
+        *external_tool.rules(),
+        *pex_environment.rules(),
+        *python_native_code.rules(),
+    ]


Test patch: