Repository: pantsbuild/pants

Issue:
Failure to build an AWS Lambda on macOS, even without native deps
A lambda which used to build successfully with `2.9.x` (with `resolve_all_constraints`) fails to build in `2.10.0` and `2.11.0rc0` (with `enable_resolves`).
```
stderr:
ERROR: Exception:
Traceback (most recent call last):
File "$HOME/.cache/pants/named_caches/pex_root/venvs/s/ed918e81/venv/lib/python3.6/site-packages/pip/_internal/cli/base_command.py", line 223, in _main
status = self.run(options, args)
File "$HOME/.cache/pants/named_caches/pex_root/venvs/s/ed918e81/venv/lib/python3.6/site-packages/pip/_internal/cli/req_command.py", line 180, in wrapper
return func(self, options, args)
File "$HOME/.cache/pants/named_caches/pex_root/venvs/s/ed918e81/venv/lib/python3.6/site-packages/pip/_internal/commands/download.py", line 131, in run
reqs, check_supported_wheels=True
File "$HOME/.cache/pants/named_caches/pex_root/venvs/s/ed918e81/venv/lib/python3.6/site-packages/pip/_internal/resolution/resolvelib/resolver.py", line 101, in resolve
req, requested_extras=(),
File "$HOME/.cache/pants/named_caches/pex_root/venvs/s/ed918e81/venv/lib/python3.6/site-packages/pip/_internal/resolution/resolvelib/factory.py", line 291, in make_requirement_from_install_req
if not ireq.match_markers(requested_extras):
File "$HOME/.cache/pants/named_caches/pex_root/venvs/s/ed918e81/venv/lib/python3.6/site-packages/pip/_internal/req/req_install.py", line 292, in match_markers
for extra in extras_requested)
File "$HOME/.cache/pants/named_caches/pex_root/venvs/s/ed918e81/venv/lib/python3.6/site-packages/pip/_internal/req/req_install.py", line 292, in <genexpr>
for extra in extras_requested)
File "$HOME/.cache/pants/named_caches/pex_root/venvs/s/ed918e81/venv/lib/python3.6/site-packages/pip/_vendor/packaging/markers.py", line 336, in evaluate
return _evaluate_markers(self._markers, current_environment)
File "$HOME/.cache/pants/named_caches/pex_root/venvs/s/ed918e81/venv/lib/python3.6/site-packages/pip/_vendor/packaging/markers.py", line 252, in _evaluate_markers
lhs_value = _get_env(environment, lhs.value)
File "$HOME/.cache/pants/named_caches/pex_root/venvs/s/ed918e81/venv/lib/python3.6/site-packages/pip/_vendor/packaging/markers.py", line 233, in _get_env
"{0!r} does not exist in evaluation environment.".format(name)
pip._vendor.packaging.markers.UndefinedEnvironmentName: 'python_full_version' does not exist in evaluation environment.
pid 86188 -> $HOME/.cache/pants/named_caches/pex_root/venvs/b295182ab4c550677e62422803f6a58e63ee0a33/a58bfc017a488e495e46564ebdf4aa8aa8f2c0fb/pex --disable-pip-version-check --no-python-version-warning --exists-action a --isolated -q --cache-dir $HOME/.cache/pants/named_caches/pex_root --log /private/var/folders/hv/p6g7p3p95d19gtm5cfkrk5w00000gn/T/process-executionzr9mEY/.tmp/tmpavs4y06b/pip.log download --dest /private/var/folders/hv/p6g7p3p95d19gtm5cfkrk5w00000gn/T/process-executionzr9mEY/.tmp/tmply2cuzf5/cp38-cp38-linux_x86_64 --only-binary :all: --no-deps --requirement 3rdparty/python/default_$ORG_lock.txt --platform manylinux2014_x86_64 --platform linux_x86_64 --implementation cp --python-version 38 --abi cp38 --index-url https://pypi.org/simple/ --retries 5 --timeout 15 exited with 2 and STDERR:
Failed to resolve for platform linux_x86_64-cp-38-cp38. Resolve requires evaluation of unknown environment marker: 'python_full_version' does not exist in evaluation environment.
```

The process that is failing is the building of a repository PEX... which seems strange to me, since I didn't think that we could build a repository PEX in this case (where a PEX is being built for another platform than the current one). So possibly that special casing broke.


Discussion:
This is also busted on 2.10, my guess is this has more to do with the usage of lockfiles (as opposed to constraint file)
Ok, good to know. Have expanded the description.

This should probably be fixed independently from #14980 and #14991 then, since those only need picking to `2.11.x`.
It turns out that this is exactly described by #12222. Working on it now.

Resolving patch:
diff --git a/src/python/pants/backend/python/target_types.py b/src/python/pants/backend/python/target_types.py
--- a/src/python/pants/backend/python/target_types.py
+++ b/src/python/pants/backend/python/target_types.py
@@ -1011,8 +1011,9 @@ def parse_requirements_file(content: str, *, rel_path: str) -> Iterator[PipRequi
     VCS requirements will fail, with a helpful error message describing how to use PEP 440.
     """
     for i, line in enumerate(content.splitlines()):
-        line = line.strip()
-        if not line or line.startswith("#") or line.startswith("-"):
+        line, _, _ = line.partition("--")
+        line = line.strip().rstrip("\\")
+        if not line or line.startswith("#"):
             continue
         try:
             yield PipRequirement.parse(line)
diff --git a/src/python/pants/backend/python/util_rules/pex_from_targets.py b/src/python/pants/backend/python/util_rules/pex_from_targets.py
--- a/src/python/pants/backend/python/util_rules/pex_from_targets.py
+++ b/src/python/pants/backend/python/util_rules/pex_from_targets.py
@@ -46,6 +46,7 @@
 from pants.util.docutil import doc_url
 from pants.util.logging import LogLevel
 from pants.util.meta import frozen_after_init
+from pants.util.ordered_set import FrozenOrderedSet
 from pants.util.strutil import bullet_list, path_safe
 
 logger = logging.getLogger(__name__)
@@ -376,7 +377,10 @@ class _ConstraintsRepositoryPexRequest:
 
 
 @rule(level=LogLevel.DEBUG)
-async def create_pex_from_targets(request: PexFromTargetsRequest) -> PexRequest:
+async def create_pex_from_targets(
+    request: PexFromTargetsRequest,
+    python_setup: PythonSetup,
+) -> PexRequest:
     interpreter_constraints = await Get(
         InterpreterConstraints,
         InterpreterConstraintsRequest,
@@ -444,6 +448,29 @@ async def create_pex_from_targets(request: PexFromTargetsRequest) -> PexRequest:
             ),
         )
         requirements = dataclasses.replace(requirements, repository_pex=repository_pex.maybe_pex)
+        if python_setup.enable_resolves and not repository_pex.maybe_pex:
+            # If we cannot use a PEX-repository for whatever reason, we still need to parse and
+            # apply the lockfile as constraints.
+            chosen_resolve = await Get(
+                ChosenPythonResolve, ChosenPythonResolveRequest(request.addresses)
+            )
+            lockfile_contents = await Get(
+                DigestContents,
+                PathGlobs(
+                    [chosen_resolve.lockfile_path],
+                    glob_match_error_behavior=GlobMatchErrorBehavior.error,
+                    description_of_origin=f"the resolve `{chosen_resolve.name}` (from `[python].resolves`)",
+                ),
+            )
+            constraints_strings = FrozenOrderedSet(
+                str(req)
+                for req in parse_requirements_file(
+                    lockfile_contents[0].content.decode(), rel_path=chosen_resolve.lockfile_path
+                )
+            )
+            requirements = dataclasses.replace(
+                requirements, constraints_strings=constraints_strings
+            )
 
     return PexRequest(
         output_filename=request.output_filename,
@@ -465,6 +492,10 @@ async def create_pex_from_targets(request: PexFromTargetsRequest) -> PexRequest:
 async def get_repository_pex(
     request: _RepositoryPexRequest, python_setup: PythonSetup
 ) -> OptionalPexRequest:
+    # NB: It isn't safe to resolve against an entire lockfile or constraints file if
+    # platforms are in use. See https://github.com/pantsbuild/pants/issues/12222.
+    if request.platforms or request.complete_platforms:
+        return OptionalPexRequest(None)
 
     interpreter_constraints = await Get(
         InterpreterConstraints,
@@ -520,9 +551,7 @@ async def _setup_constraints_repository_pex(
     global_requirement_constraints: GlobalRequirementConstraints,
 ) -> OptionalPexRequest:
     request = constraints_request.repository_pex_request
-    # NB: it isn't safe to resolve against the whole constraints file if
-    # platforms are in use. See https://github.com/pantsbuild/pants/issues/12222.
-    if not python_setup.resolve_all_constraints or request.platforms or request.complete_platforms:
+    if not python_setup.resolve_all_constraints:
         return OptionalPexRequest(None)
 
     constraints_path = python_setup.requirement_constraints


Test patch:
diff --git a/src/python/pants/backend/python/target_types_test.py b/src/python/pants/backend/python/target_types_test.py
--- a/src/python/pants/backend/python/target_types_test.py
+++ b/src/python/pants/backend/python/target_types_test.py
@@ -362,13 +362,17 @@ def test_requirements_field() -> None:
 
 def test_parse_requirements_file() -> None:
     content = dedent(
-        """\
+        r"""\
         # Comment.
         --find-links=https://duckduckgo.com
         ansicolors>=1.18.0
         Django==3.2 ; python_version>'3'
         Un-Normalized-PROJECT  # Inline comment.
         pip@ git+https://github.com/pypa/pip.git
+        setuptools==54.1.2; python_version >= "3.6" \
+            --hash=sha256:dd20743f36b93cbb8724f4d2ccd970dce8b6e6e823a13aa7e5751bb4e674c20b \
+            --hash=sha256:ebd0148faf627b569c8d2a1b20f5d3b09c873f12739d71c7ee88f037d5be82ff
+        wheel==1.2.3 --hash=sha256:dd20743f36b93cbb8724f4d2ccd970dce8b6e6e823a13aa7e5751bb4e674c20b
         """
     )
     assert set(parse_requirements_file(content, rel_path="foo.txt")) == {
@@ -376,6 +380,8 @@ def test_parse_requirements_file() -> None:
         PipRequirement.parse("Django==3.2 ; python_version>'3'"),
         PipRequirement.parse("Un-Normalized-PROJECT"),
         PipRequirement.parse("pip@ git+https://github.com/pypa/pip.git"),
+        PipRequirement.parse("setuptools==54.1.2; python_version >= '3.6'"),
+        PipRequirement.parse("wheel==1.2.3"),
     }
 
 
diff --git a/src/python/pants/backend/python/util_rules/pex_from_targets_test.py b/src/python/pants/backend/python/util_rules/pex_from_targets_test.py
--- a/src/python/pants/backend/python/util_rules/pex_from_targets_test.py
+++ b/src/python/pants/backend/python/util_rules/pex_from_targets_test.py
@@ -395,32 +395,46 @@ def test_exclude_requirements(
     assert len(pex_request.requirements.req_strings) == (1 if include_requirements else 0)
 
 
-def test_issue_12222(rule_runner: RuleRunner) -> None:
+@pytest.mark.parametrize("enable_resolves", [False, True])
+def test_cross_platform_pex_disables_subsetting(
+    rule_runner: RuleRunner, enable_resolves: bool
+) -> None:
+    # See https://github.com/pantsbuild/pants/issues/12222.
+    lockfile = "3rdparty/python/default.lock"
     constraints = ["foo==1.0", "bar==1.0"]
     rule_runner.write_files(
         {
-            "constraints.txt": "\n".join(constraints),
+            lockfile: "\n".join(constraints),
+            "a.py": "",
             "BUILD": dedent(
                 """
                 python_requirement(name="foo",requirements=["foo"])
                 python_requirement(name="bar",requirements=["bar"])
-                python_sources(name="lib",sources=[],dependencies=[":foo"])
+                python_sources(name="lib",dependencies=[":foo"])
                 """
             ),
         }
     )
+
+    if enable_resolves:
+        options = [
+            "--python-enable-resolves",
+            # NB: Because this is a synthetic lockfile without a header.
+            "--python-invalid-lockfile-behavior=ignore",
+        ]
+    else:
+        options = [
+            f"--python-requirement-constraints={lockfile}",
+            "--python-resolve-all-constraints",
+        ]
+    rule_runner.set_options(options, env_inherit={"PATH"})
+
     request = PexFromTargetsRequest(
         [Address("", target_name="lib")],
         output_filename="demo.pex",
         internal_only=False,
         platforms=PexPlatforms(["some-platform-x86_64"]),
     )
-    rule_runner.set_options(
-        [
-            "--python-requirement-constraints=constraints.txt",
-            "--python-resolve-all-constraints",
-        ]
-    )
     result = rule_runner.request(PexRequest, [request])
 
     assert result.requirements == PexRequirements(["foo"], constraints_strings=constraints)