Repository: pantsbuild/pants

Issue:
Failure to build an AWS Lambda on macOS, even without native deps
A lambda which used to build successfully with `2.9.x` (with `resolve_all_constraints`) fails to build in `2.10.0` and `2.11.0rc0` (with `enable_resolves`).
```
stderr:
ERROR: Exception:
Traceback (most recent call last):
File "$HOME/.cache/pants/named_caches/pex_root/venvs/s/ed918e81/venv/lib/python3.6/site-packages/pip/_internal/cli/base_command.py", line 223, in _main
status = self.run(options, args)
File "$HOME/.cache/pants/named_caches/pex_root/venvs/s/ed918e81/venv/lib/python3.6/site-packages/pip/_internal/cli/req_command.py", line 180, in wrapper
return func(self, options, args)
File "$HOME/.cache/pants/named_caches/pex_root/venvs/s/ed918e81/venv/lib/python3.6/site-packages/pip/_internal/commands/download.py", line 131, in run
reqs, check_supported_wheels=True
File "$HOME/.cache/pants/named_caches/pex_root/venvs/s/ed918e81/venv/lib/python3.6/site-packages/pip/_internal/resolution/resolvelib/resolver.py", line 101, in resolve
req, requested_extras=(),
File "$HOME/.cache/pants/named_caches/pex_root/venvs/s/ed918e81/venv/lib/python3.6/site-packages/pip/_internal/resolution/resolvelib/factory.py", line 291, in make_requirement_from_install_req
if not ireq.match_markers(requested_extras):
File "$HOME/.cache/pants/named_caches/pex_root/venvs/s/ed918e81/venv/lib/python3.6/site-packages/pip/_internal/req/req_install.py", line 292, in match_markers
for extra in extras_requested)
File "$HOME/.cache/pants/named_caches/pex_root/venvs/s/ed918e81/venv/lib/python3.6/site-packages/pip/_internal/req/req_install.py", line 292, in <genexpr>
for extra in extras_requested)
File "$HOME/.cache/pants/named_caches/pex_root/venvs/s/ed918e81/venv/lib/python3.6/site-packages/pip/_vendor/packaging/markers.py", line 336, in evaluate
return _evaluate_markers(self._markers, current_environment)
File "$HOME/.cache/pants/named_caches/pex_root/venvs/s/ed918e81/venv/lib/python3.6/site-packages/pip/_vendor/packaging/markers.py", line 252, in _evaluate_markers
lhs_value = _get_env(environment, lhs.value)
File "$HOME/.cache/pants/named_caches/pex_root/venvs/s/ed918e81/venv/lib/python3.6/site-packages/pip/_vendor/packaging/markers.py", line 233, in _get_env
"{0!r} does not exist in evaluation environment.".format(name)
pip._vendor.packaging.markers.UndefinedEnvironmentName: 'python_full_version' does not exist in evaluation environment.
pid 86188 -> $HOME/.cache/pants/named_caches/pex_root/venvs/b295182ab4c550677e62422803f6a58e63ee0a33/a58bfc017a488e495e46564ebdf4aa8aa8f2c0fb/pex --disable-pip-version-check --no-python-version-warning --exists-action a --isolated -q --cache-dir $HOME/.cache/pants/named_caches/pex_root --log /private/var/folders/hv/p6g7p3p95d19gtm5cfkrk5w00000gn/T/process-executionzr9mEY/.tmp/tmpavs4y06b/pip.log download --dest /private/var/folders/hv/p6g7p3p95d19gtm5cfkrk5w00000gn/T/process-executionzr9mEY/.tmp/tmply2cuzf5/cp38-cp38-linux_x86_64 --only-binary :all: --no-deps --requirement 3rdparty/python/default_$ORG_lock.txt --platform manylinux2014_x86_64 --platform linux_x86_64 --implementation cp --python-version 38 --abi cp38 --index-url https://pypi.org/simple/ --retries 5 --timeout 15 exited with 2 and STDERR:
Failed to resolve for platform linux_x86_64-cp-38-cp38. Resolve requires evaluation of unknown environment marker: 'python_full_version' does not exist in evaluation environment.
```

The process that is failing is the building of a repository PEX... which seems strange to me, since I didn't think that we could build a repository PEX in this case (where a PEX is being built for another platform than the current one). So possibly that special casing broke.


Discussion:
This is also busted on 2.10, my guess is this has more to do with the usage of lockfiles (as opposed to constraint file)
Ok, good to know. Have expanded the description.

This should probably be fixed independently from #14980 and #14991 then, since those only need picking to `2.11.x`.
It turns out that this is exactly described by #12222. Working on it now.

Resolving patch:
diff --git a/src/python/pants/backend/python/target_types.py b/src/python/pants/backend/python/target_types.py
--- a/src/python/pants/backend/python/target_types.py
+++ b/src/python/pants/backend/python/target_types.py
@@ -1025,8 +1025,9 @@ def parse_requirements_file(content: str, *, rel_path: str) -> Iterator[PipRequi
     VCS requirements will fail, with a helpful error message describing how to use PEP 440.
     """
     for i, line in enumerate(content.splitlines()):
-        line = line.strip()
-        if not line or line.startswith("#") or line.startswith("-"):
+        line, _, _ = line.partition("--")
+        line = line.strip().rstrip("\\")
+        if not line or line.startswith("#"):
             continue
         try:
             yield PipRequirement.parse(line)
diff --git a/src/python/pants/backend/python/util_rules/pex_from_targets.py b/src/python/pants/backend/python/util_rules/pex_from_targets.py
--- a/src/python/pants/backend/python/util_rules/pex_from_targets.py
+++ b/src/python/pants/backend/python/util_rules/pex_from_targets.py
@@ -393,6 +393,9 @@ async def create_pex_from_targets(
 
         pex_native_subsetting_supported = False
         if python_setup.enable_resolves:
+            # TODO: Once `requirement_constraints` is removed in favor of `enable_resolves`,
+            # `ChosenPythonResolveRequest` and `_PexRequirementsRequest` should merge and
+            # do a single transitive walk to replace this method.
             chosen_resolve = await Get(
                 ChosenPythonResolve, ChosenPythonResolveRequest(request.addresses)
             )
@@ -400,6 +403,10 @@ async def create_pex_from_targets(
                 LoadedLockfile, LoadedLockfileRequest(chosen_resolve.lockfile)
             )
             pex_native_subsetting_supported = loaded_lockfile.is_pex_native
+            if loaded_lockfile.constraints_strings:
+                requirements = dataclasses.replace(
+                    requirements, constraints_strings=loaded_lockfile.constraints_strings
+                )
 
         should_return_entire_lockfile = (
             python_setup.run_against_entire_lockfile and request.internal_only
@@ -520,6 +527,10 @@ async def create_pex_from_targets(
 async def get_repository_pex(
     request: _RepositoryPexRequest, python_setup: PythonSetup
 ) -> OptionalPexRequest:
+    # NB: It isn't safe to resolve against an entire lockfile or constraints file if
+    # platforms are in use. See https://github.com/pantsbuild/pants/issues/12222.
+    if request.platforms or request.complete_platforms:
+        return OptionalPexRequest(None)
 
     interpreter_constraints = await Get(
         InterpreterConstraints,
@@ -570,9 +581,7 @@ async def _setup_constraints_repository_pex(
     global_requirement_constraints: GlobalRequirementConstraints,
 ) -> OptionalPexRequest:
     request = constraints_request.repository_pex_request
-    # NB: it isn't safe to resolve against the whole constraints file if
-    # platforms are in use. See https://github.com/pantsbuild/pants/issues/12222.
-    if not python_setup.resolve_all_constraints or request.platforms or request.complete_platforms:
+    if not python_setup.resolve_all_constraints:
         return OptionalPexRequest(None)
 
     constraints_path = python_setup.requirement_constraints
diff --git a/src/python/pants/backend/python/util_rules/pex_requirements.py b/src/python/pants/backend/python/util_rules/pex_requirements.py
--- a/src/python/pants/backend/python/util_rules/pex_requirements.py
+++ b/src/python/pants/backend/python/util_rules/pex_requirements.py
@@ -9,7 +9,7 @@
 
 from pants.backend.python.pip_requirement import PipRequirement
 from pants.backend.python.subsystems.setup import InvalidLockfileBehavior, PythonSetup
-from pants.backend.python.target_types import PythonRequirementsField
+from pants.backend.python.target_types import PythonRequirementsField, parse_requirements_file
 from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints
 from pants.backend.python.util_rules.lockfile_metadata import (
     InvalidPythonLockfileReason,
@@ -87,8 +87,11 @@ class LoadedLockfile:
     requirement_estimate: int
     # True if the loaded lockfile is in PEX's native format.
     is_pex_native: bool
+    # If !is_pex_native, the lockfile parsed as constraints strings, for use when the lockfile
+    # needs to be subsetted (see #15031, ##12222).
+    constraints_strings: FrozenOrderedSet[str] | None
     # The original file or file content (which may not have identical content to the output
-    # `lockfile_digest).
+    # `lockfile_digest`).
     original_lockfile: Lockfile | LockfileContent
 
 
@@ -167,11 +170,16 @@ async def load_lockfile(
             ),
         )
         requirement_estimate = _pex_lockfile_requirement_count(lock_bytes)
+        constraints_strings = None
     else:
         header_delimiter = "#"
+        lock_string = lock_bytes.decode()
         # Note: this is a very naive heuristic. It will overcount because entries often
         # have >1 line due to `--hash`.
-        requirement_estimate = len(lock_bytes.decode().splitlines())
+        requirement_estimate = len(lock_string.splitlines())
+        constraints_strings = FrozenOrderedSet(
+            str(req) for req in parse_requirements_file(lock_string, rel_path=lockfile_path)
+        )
 
     metadata: PythonLockfileMetadata | None = None
     if should_validate_metadata(lockfile, python_setup):
@@ -188,6 +196,7 @@ async def load_lockfile(
         metadata,
         requirement_estimate,
         is_pex_native,
+        constraints_strings,
         original_lockfile=lockfile,
     )
 


Test patch:
diff --git a/src/python/pants/backend/python/target_types_test.py b/src/python/pants/backend/python/target_types_test.py
--- a/src/python/pants/backend/python/target_types_test.py
+++ b/src/python/pants/backend/python/target_types_test.py
@@ -358,13 +358,17 @@ def test_requirements_field() -> None:
 
 def test_parse_requirements_file() -> None:
     content = dedent(
-        """\
+        r"""\
         # Comment.
         --find-links=https://duckduckgo.com
         ansicolors>=1.18.0
         Django==3.2 ; python_version>'3'
         Un-Normalized-PROJECT  # Inline comment.
         pip@ git+https://github.com/pypa/pip.git
+        setuptools==54.1.2; python_version >= "3.6" \
+            --hash=sha256:dd20743f36b93cbb8724f4d2ccd970dce8b6e6e823a13aa7e5751bb4e674c20b \
+            --hash=sha256:ebd0148faf627b569c8d2a1b20f5d3b09c873f12739d71c7ee88f037d5be82ff
+        wheel==1.2.3 --hash=sha256:dd20743f36b93cbb8724f4d2ccd970dce8b6e6e823a13aa7e5751bb4e674c20b
         """
     )
     assert set(parse_requirements_file(content, rel_path="foo.txt")) == {
@@ -372,6 +376,8 @@ def test_parse_requirements_file() -> None:
         PipRequirement.parse("Django==3.2 ; python_version>'3'"),
         PipRequirement.parse("Un-Normalized-PROJECT"),
         PipRequirement.parse("pip@ git+https://github.com/pypa/pip.git"),
+        PipRequirement.parse("setuptools==54.1.2; python_version >= '3.6'"),
+        PipRequirement.parse("wheel==1.2.3"),
     }
 
 
diff --git a/src/python/pants/backend/python/util_rules/pex_from_targets_test.py b/src/python/pants/backend/python/util_rules/pex_from_targets_test.py
--- a/src/python/pants/backend/python/util_rules/pex_from_targets_test.py
+++ b/src/python/pants/backend/python/util_rules/pex_from_targets_test.py
@@ -175,7 +175,7 @@ class Project:
 build_deps = ["setuptools==54.1.2", "wheel==0.36.2"]
 
 
-setuptools_poetry_lockfile = """
+setuptools_poetry_lockfile = r"""
 # This lockfile was autogenerated by Pants. To regenerate, run:
 #
 #    ./pants generate-lockfiles --resolve=setuptools
@@ -430,11 +430,16 @@ def test_exclude_requirements(
     assert len(pex_request.requirements.req_strings) == (1 if include_requirements else 0)
 
 
-def test_issue_12222(rule_runner: RuleRunner) -> None:
+@pytest.mark.parametrize("enable_resolves", [False, True])
+def test_cross_platform_pex_disables_subsetting(
+    rule_runner: RuleRunner, enable_resolves: bool
+) -> None:
+    # See https://github.com/pantsbuild/pants/issues/12222.
+    lockfile = "3rdparty/python/default.lock"
     constraints = ["foo==1.0", "bar==1.0"]
     rule_runner.write_files(
         {
-            "constraints.txt": "\n".join(constraints),
+            lockfile: "\n".join(constraints),
             "a.py": "",
             "BUILD": dedent(
                 """
@@ -445,19 +450,26 @@ def test_issue_12222(rule_runner: RuleRunner) -> None:
             ),
         }
     )
+
+    if enable_resolves:
+        options = [
+            "--python-enable-resolves",
+            # NB: Because this is a synthetic lockfile without a header.
+            "--python-invalid-lockfile-behavior=ignore",
+        ]
+    else:
+        options = [
+            f"--python-requirement-constraints={lockfile}",
+            "--python-resolve-all-constraints",
+        ]
+    rule_runner.set_options(options, env_inherit={"PATH"})
+
     request = PexFromTargetsRequest(
         [Address("", target_name="lib")],
         output_filename="demo.pex",
         internal_only=False,
         platforms=PexPlatforms(["some-platform-x86_64"]),
     )
-    rule_runner.set_options(
-        [
-            "--python-requirement-constraints=constraints.txt",
-            "--python-resolve-all-constraints",
-        ],
-        env_inherit={"PATH"},
-    )
     result = rule_runner.request(PexRequest, [request])
 
     assert result.requirements == PexRequirements(["foo"], constraints_strings=constraints)