Repository: pantsbuild/pants

Issue:
Internal pants test with coverage option fails with requirements parse error
**Describe the bug**
I'm getting this error ValueError: Invalid requirement '//3rdparty/python:pytest': Parse error at "'//3rdpar'": Expected W:(0-9A-Za-z) running test with coverage  ./pants test --test-use-coverage src/python/pants/backend/docker/:  in the pants repo. If I comment out [this line](https://github.com/pantsbuild/pants/blob/d5812af250f13477588de610c5e59aeb56101eaf/pants.toml#L179), it works.

**Pants version**
Latest `main` as of about three days ago. 

**OS**
MacOS

**Additional info**

Stack trace:
```
./pants test --test-use-coverage src/python/pants/backend/docker/:
There is no pantsd metadata at /Users/rhys.madigan/repos/pants/.pids/d54692e05ced/pantsd.
18:17:49.39 [INFO] waiting for pantsd to start...
18:17:49.60 [INFO] pantsd started
18:17:56.46 [INFO] Initializing scheduler...
18:18:07.55 [INFO] Scheduler initialized.
18:18:18.58 [ERROR] 1 Exception encountered:

Engine traceback:
  in select
    ..
  in pants.core.goals.test.run_tests
    `test` goal

Traceback (most recent call last):
  File "/Users/rhys.madigan/.cache/pants/pants_dev_deps/5709c1a054ca5899853199a432062f92bb7571eb.venv/lib/python3.9/site-packages/pkg_resources/_vendor/packaging/requirements.py", line 102, in __init__
    req = REQUIREMENT.parseString(requirement_string)
  File "/Users/rhys.madigan/.cache/pants/pants_dev_deps/5709c1a054ca5899853199a432062f92bb7571eb.venv/lib/python3.9/site-packages/pkg_resources/_vendor/pyparsing/core.py", line 1141, in parse_string
    raise exc.with_traceback(None)
pkg_resources._vendor.pyparsing.exceptions.ParseException: Expected W:(0-9A-Za-z), found '/'  (at char 0), (line:1, col:1)

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/Users/rhys.madigan/repos/pants/src/python/pants/backend/python/pip_requirement.py", line 21, in parse
    return cls(pkg_resources.Requirement.parse(line))
  File "/Users/rhys.madigan/.cache/pants/pants_dev_deps/5709c1a054ca5899853199a432062f92bb7571eb.venv/lib/python3.9/site-packages/pkg_resources/__init__.py", line 3147, in parse
    req, = parse_requirements(s)
  File "/Users/rhys.madigan/.cache/pants/pants_dev_deps/5709c1a054ca5899853199a432062f92bb7571eb.venv/lib/python3.9/site-packages/pkg_resources/__init__.py", line 3102, in __init__
    super(Requirement, self).__init__(requirement_string)
  File "/Users/rhys.madigan/.cache/pants/pants_dev_deps/5709c1a054ca5899853199a432062f92bb7571eb.venv/lib/python3.9/site-packages/pkg_resources/_vendor/packaging/requirements.py", line 104, in __init__
    raise InvalidRequirement(
pkg_resources.extern.packaging.requirements.InvalidRequirement: Parse error at "'//3rdpar'": Expected W:(0-9A-Za-z)

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/Users/rhys.madigan/repos/pants/src/python/pants/engine/internals/selectors.py", line 626, in native_engine_generator_send
    res = rule.send(arg) if err is None else rule.throw(throw or err)
  File "/Users/rhys.madigan/repos/pants/src/python/pants/core/goals/test.py", line 889, in run_tests
    results = await MultiGet(
  File "/Users/rhys.madigan/repos/pants/src/python/pants/engine/internals/selectors.py", line 361, in MultiGet
    return await _MultiGet(tuple(__arg0))
  File "/Users/rhys.madigan/repos/pants/src/python/pants/engine/internals/selectors.py", line 168, in __await__
    result = yield self.gets
  File "/Users/rhys.madigan/repos/pants/src/python/pants/engine/internals/selectors.py", line 626, in native_engine_generator_send
    res = rule.send(arg) if err is None else rule.throw(throw or err)
  File "/Users/rhys.madigan/repos/pants/src/python/pants/backend/python/goals/pytest_runner.py", line 503, in run_python_tests
    setup = await Get(
  File "/Users/rhys.madigan/repos/pants/src/python/pants/engine/internals/selectors.py", line 118, in __await__
    result = yield self
  File "/Users/rhys.madigan/repos/pants/src/python/pants/engine/internals/selectors.py", line 626, in native_engine_generator_send
    res = rule.send(arg) if err is None else rule.throw(throw or err)
  File "/Users/rhys.madigan/repos/pants/src/python/pants/backend/python/goals/pytest_runner.py", line 373, in setup_pytest_for_target
    await validate_pytest_cov_included(pytest)
  File "/Users/rhys.madigan/repos/pants/src/python/pants/backend/python/goals/pytest_runner.py", line 220, in validate_pytest_cov_included
    requirements = {PipRequirement.parse(req) for req in _pytest.requirements}
  File "/Users/rhys.madigan/repos/pants/src/python/pants/backend/python/goals/pytest_runner.py", line 220, in <setcomp>
    requirements = {PipRequirement.parse(req) for req in _pytest.requirements}
  File "/Users/rhys.madigan/repos/pants/src/python/pants/backend/python/pip_requirement.py", line 49, in parse
    raise ValueError(f"Invalid requirement '{line}'{origin_str}: {e}")
ValueError: Invalid requirement '//3rdparty/python:pytest': Parse error at "'//3rdpar'": Expected W:(0-9A-Za-z)
```



Discussion:

Resolving patch:
diff --git a/src/python/pants/backend/python/goals/pytest_runner.py b/src/python/pants/backend/python/goals/pytest_runner.py
--- a/src/python/pants/backend/python/goals/pytest_runner.py
+++ b/src/python/pants/backend/python/goals/pytest_runner.py
@@ -29,8 +29,9 @@
     PythonLockfileMetadataV2,
     PythonLockfileMetadataV3,
 )
-from pants.backend.python.util_rules.pex import Pex, PexRequest, VenvPex, VenvPexProcess
+from pants.backend.python.util_rules.pex import Pex, PexRequest, ReqStrings, VenvPex, VenvPexProcess
 from pants.backend.python.util_rules.pex_from_targets import RequirementsPexRequest
+from pants.backend.python.util_rules.pex_requirements import PexRequirements
 from pants.backend.python.util_rules.python_sources import (
     PythonSourceFiles,
     PythonSourceFilesRequest,
@@ -217,7 +218,8 @@ def _count_pytest_tests(contents: DigestContents) -> int:
 async def validate_pytest_cov_included(_pytest: PyTest):
     if _pytest.requirements:
         # We'll only be using this subset of the lockfile.
-        requirements = {PipRequirement.parse(req) for req in _pytest.requirements}
+        req_strings = (await Get(ReqStrings, PexRequirements(_pytest.requirements))).req_strings
+        requirements = {PipRequirement.parse(req_string) for req_string in req_strings}
     else:
         # We'll be using the entire lockfile.
         lockfile_metadata = await get_lockfile_metadata(_pytest)


Test patch:
diff --git a/src/python/pants/backend/python/goals/pytest_runner_test.py b/src/python/pants/backend/python/goals/pytest_runner_test.py
--- a/src/python/pants/backend/python/goals/pytest_runner_test.py
+++ b/src/python/pants/backend/python/goals/pytest_runner_test.py
@@ -1,8 +1,30 @@
 # Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-from pants.backend.python.goals.pytest_runner import _count_pytest_tests
+from __future__ import annotations
+
+import pytest
+
+from pants.backend.python.goals.pytest_runner import (
+    _count_pytest_tests,
+    validate_pytest_cov_included,
+)
+from pants.backend.python.pip_requirement import PipRequirement
+from pants.backend.python.subsystems.pytest import PyTest
+from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints
+from pants.backend.python.util_rules.lockfile_metadata import PythonLockfileMetadataV3
+from pants.backend.python.util_rules.pex import ReqStrings
+from pants.backend.python.util_rules.pex_requirements import (
+    LoadedLockfile,
+    LoadedLockfileRequest,
+    Lockfile,
+    PexRequirements,
+    Resolve,
+)
 from pants.engine.fs import DigestContents, FileContent
+from pants.engine.internals.native_engine import EMPTY_DIGEST
+from pants.testutil.option_util import create_subsystem
+from pants.testutil.rule_runner import MockGet, run_rule_with_mocks
 
 EXAMPLE_TEST1 = b"""
 def test_foo():
@@ -54,3 +76,56 @@ def test_count_pytest_tests_multiple() -> None:
     )
     test_count = _count_pytest_tests(digest_contents)
     assert test_count == 3
+
+
+@pytest.mark.parametrize("entire_lockfile", [False, True])
+def test_validate_pytest_cov_included(entire_lockfile: bool) -> None:
+    def validate(reqs: list[str]) -> None:
+        if entire_lockfile:
+            tool = create_subsystem(
+                PyTest,
+                lockfile="dummy.lock",
+                install_from_resolve="dummy_resolve",
+                requirements=[],
+            )
+        else:
+            tool = create_subsystem(
+                PyTest,
+                lockfile="dummy.lock",
+                install_from_resolve="dummy_resolve",
+                requirements=reqs,
+            )
+        lockfile = Lockfile("dummy_url", "dummy_description_of_origin", "dummy_resolve")
+        metadata = PythonLockfileMetadataV3(
+            valid_for_interpreter_constraints=InterpreterConstraints(),
+            requirements={PipRequirement.parse(req) for req in reqs},
+            manylinux=None,
+            requirement_constraints=set(),
+            only_binary=set(),
+            no_binary=set(),
+        )
+        loaded_lockfile = LoadedLockfile(EMPTY_DIGEST, "", metadata, 0, True, None, lockfile)
+        run_rule_with_mocks(
+            validate_pytest_cov_included,
+            rule_args=[tool],
+            mock_gets=[
+                MockGet(ReqStrings, (PexRequirements,), lambda x: ReqStrings(tuple(reqs))),
+                MockGet(Lockfile, (Resolve,), lambda x: lockfile),
+                MockGet(
+                    LoadedLockfile,
+                    (LoadedLockfileRequest,),
+                    lambda x: loaded_lockfile if x.lockfile == lockfile else None,
+                ),
+            ],
+        )
+
+    # Canonicalize project name.
+    validate(["PyTeST_cOV"])
+
+    with pytest.raises(ValueError) as exc:
+        validate([])
+    assert "missing `pytest-cov`" in str(exc.value)
+
+    with pytest.raises(ValueError) as exc:
+        validate(["custom-plugin"])
+    assert "missing `pytest-cov`" in str(exc.value)
diff --git a/src/python/pants/backend/python/subsystems/pytest_runner_test.py b/src/python/pants/backend/python/subsystems/pytest_runner_test.py
deleted file mode 100644
--- a/src/python/pants/backend/python/subsystems/pytest_runner_test.py
+++ /dev/null
@@ -1,73 +0,0 @@
-# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md).
-# Licensed under the Apache License, Version 2.0 (see LICENSE).
-
-from __future__ import annotations
-
-import pytest
-
-from pants.backend.python.goals.pytest_runner import validate_pytest_cov_included
-from pants.backend.python.pip_requirement import PipRequirement
-from pants.backend.python.subsystems.pytest import PyTest
-from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints
-from pants.backend.python.util_rules.lockfile_metadata import PythonLockfileMetadataV3
-from pants.backend.python.util_rules.pex_requirements import (
-    LoadedLockfile,
-    LoadedLockfileRequest,
-    Lockfile,
-    Resolve,
-)
-from pants.engine.internals.native_engine import EMPTY_DIGEST
-from pants.testutil.option_util import create_subsystem
-from pants.testutil.rule_runner import MockGet, run_rule_with_mocks
-
-
-@pytest.mark.parametrize("entire_lockfile", [False, True])
-def test_validate_pytest_cov_included(entire_lockfile: bool) -> None:
-    def validate(reqs: list[str]) -> None:
-        if entire_lockfile:
-            tool = create_subsystem(
-                PyTest,
-                lockfile="dummy.lock",
-                install_from_resolve="dummy_resolve",
-                requirements=[],
-            )
-        else:
-            tool = create_subsystem(
-                PyTest,
-                lockfile="dummy.lock",
-                install_from_resolve="dummy_resolve",
-                requirements=reqs,
-            )
-        lockfile = Lockfile("dummy_url", "dummy_description_of_origin", "dummy_resolve")
-        metadata = PythonLockfileMetadataV3(
-            valid_for_interpreter_constraints=InterpreterConstraints(),
-            requirements={PipRequirement.parse(req) for req in reqs},
-            manylinux=None,
-            requirement_constraints=set(),
-            only_binary=set(),
-            no_binary=set(),
-        )
-        loaded_lockfile = LoadedLockfile(EMPTY_DIGEST, "", metadata, 0, True, None, lockfile)
-        run_rule_with_mocks(
-            validate_pytest_cov_included,
-            rule_args=[tool],
-            mock_gets=[
-                MockGet(Lockfile, (Resolve,), lambda x: lockfile),
-                MockGet(
-                    LoadedLockfile,
-                    (LoadedLockfileRequest,),
-                    lambda x: loaded_lockfile if x.lockfile == lockfile else None,
-                ),
-            ],
-        )
-
-    # Canonicalize project name.
-    validate(["PyTeST_cOV"])
-
-    with pytest.raises(ValueError) as exc:
-        validate([])
-    assert "missing `pytest-cov`" in str(exc.value)
-
-    with pytest.raises(ValueError) as exc:
-        validate(["custom-plugin"])
-    assert "missing `pytest-cov`" in str(exc.value)