Repository: tox-dev/tox

Issue:
Bug with poetry and overlapping extras
## Issue

I'm not sure if this is a poetry bug or a tox bug, but I figured I'd start here.  The issue seems to be with how:
https://github.com/tox-dev/tox/blob/c83819262b4b739899af8a0bd7b6f32442344e4a/src/tox/tox_env/python/virtual_env/package/util.py#L31
is parsing extras that fall into multiple extra groups.  That is if you have the same requirement in 2+ extra groups, poetry will e.g., produce the line:

```
Requires-Dist: filelock (>=3.9.0,<4.0.0) ; extra == "extras1" or extra == "extras2"
```
in the `PKG-INFO`

Then when `tox` is run with e.g., `extras = extras1`, you get:

```
py-extras1: install_package_deps> python -I -m pip install 'filelock<4.0.0,>=3.9.0; or extra == "extras2"'
...
pip._vendor.packaging.markers.InvalidMarker: Invalid marker: 'or extra == "extras2"', parse error at 'or extra'
```

## Environment

Provide at least:

- OS: macos
- `pip list` of the host Python where `tox` is installed:

```console
./.venv/bin/pip list
Package       Version
------------- -------
cachetools    5.2.0
chardet       5.1.0
colorama      0.4.6
distlib       0.3.6
filelock      3.9.0
packaging     22.0
pip           22.3.1
platformdirs  2.6.2
pluggy        1.0.0
pyproject_api 1.2.1
setuptools    65.5.0
tox           4.1.0
virtualenv    20.17.1
```

## Minimal example

I've created a minimal poetry project that reproduces this bug: https://github.com/dconathan/poetry-tox-bug

Discussion:
dconathan: Resorted to some `type: ignore` fixes for mypy.  I don't have much mypy experience so I'm open to feedback/suggestions for alternatives for those cases...

Patch:
diff --git a/docs/changelog/2791.bugfix.rst b/docs/changelog/2791.bugfix.rst
new file mode 100644
index 00000000..cc2e9f9c
--- /dev/null
+++ b/docs/changelog/2791.bugfix.rst
@@ -0,0 +1,1 @@
+Fix extracting extras from markers with more than 2 extras in an or chain - by :user:`dconathan`.
diff --git a/src/tox/tox_env/python/virtual_env/package/util.py b/src/tox/tox_env/python/virtual_env/package/util.py
index e57efee9..948de48c 100644
--- a/src/tox/tox_env/python/virtual_env/package/util.py
+++ b/src/tox/tox_env/python/virtual_env/package/util.py
@@ -1,8 +1,9 @@
 from __future__ import annotations
 
 from copy import deepcopy
+from typing import Optional, Set, cast
 
-from packaging.markers import Variable  # type: ignore[attr-defined]
+from packaging.markers import Marker, Op, Value, Variable  # type: ignore[attr-defined]
 from packaging.requirements import Requirement
 
 
@@ -29,25 +30,42 @@ def dependencies_with_extras(deps: list[Requirement], extras: set[str], package_
 
 
 def extract_extra_markers(deps: list[Requirement]) -> list[tuple[Requirement, set[str | None]]]:
-    # extras might show up as markers, move them into extras property
-    result: list[tuple[Requirement, set[str | None]]] = []
-    for req in deps:
-        req = deepcopy(req)
-        markers: list[str | tuple[Variable, Variable, Variable]] = getattr(req.marker, "_markers", []) or []
-        _at: int | None = None
-        extra_markers = set()
-        for _at, (marker_key, op, marker_value) in (
-            (_at_marker, marker)
-            for _at_marker, marker in enumerate(markers)
-            if isinstance(marker, tuple) and len(marker) == 3
-        ):
-            if marker_key.value == "extra" and op.value == "==":  # pragma: no branch
-                extra_markers.add(marker_value.value)
-                del markers[_at]
-                _at -= 1
-                if _at >= 0 and (isinstance(markers[_at], str) and markers[_at] in ("and", "or")):
-                    del markers[_at]
-                if len(markers) == 0:
-                    req.marker = None
-        result.append((req, extra_markers or {None}))
+    """
+    Extract extra markers from dependencies.
+
+    :param deps: the dependencies
+    :return: a list of requirement, extras set
+    """
+    result = [_extract_extra_markers(d) for d in deps]
     return result
+
+
+def _extract_extra_markers(req: Requirement) -> tuple[Requirement, set[str | None]]:
+    req = deepcopy(req)
+    markers: list[str | tuple[Variable, Op, Variable]] = getattr(req.marker, "_markers", []) or []
+    new_markers: list[str | tuple[Variable, Op, Variable]] = []
+    extra_markers: set[str] = set()  # markers that have a key of extra
+    marker = markers.pop(0) if markers else None
+    while marker:
+        extra = _get_extra(marker)
+        if extra is not None:
+            extra_markers.add(extra)
+            if new_markers and new_markers[-1] in ("and", "or"):
+                del new_markers[-1]
+            marker = markers.pop(0) if markers else None
+            if marker in ("and", "or"):
+                marker = markers.pop(0) if markers else None
+        else:
+            new_markers.append(marker)
+            marker = markers.pop(0) if markers else None
+    if new_markers:
+        cast(Marker, req.marker)._markers = new_markers
+    else:
+        req.marker = None
+    return req, cast(Set[Optional[str]], extra_markers) or {None}
+
+
+def _get_extra(_marker: str | tuple[Variable, Op, Value]) -> str | None:
+    if isinstance(_marker, tuple) and len(_marker) == 3 and _marker[0].value == "extra" and _marker[1].value == "==":
+        return cast(str, _marker[2].value)
+    return None


Test patch:
diff --git a/tests/tox_env/python/virtual_env/package/test_python_package_util.py b/tests/tox_env/python/virtual_env/package/test_python_package_util.py
index 5cb18812..1e2e1881 100644
--- a/tests/tox_env/python/virtual_env/package/test_python_package_util.py
+++ b/tests/tox_env/python/virtual_env/package/test_python_package_util.py
@@ -67,5 +67,13 @@ def test_loads_deps_recursive_extras() -> None:
 
 def test_load_dependency_requirement_or_extras() -> None:
     requires = [Requirement('filelock<4.0.0,>=3.9.0; extra == "extras1" or extra == "extras2"')]
-    result = dependencies_with_extras(requires, {"extras1"}, "")
+    for extras in ["extras1", "extras2"]:
+        result = dependencies_with_extras(requires, {extras}, "")
+        assert [str(r) for r in result] == ["filelock<4.0.0,>=3.9.0"]
+
+
+@pytest.mark.parametrize("extra", ["extras1", "extras2", "extras3"])
+def test_load_dependency_requirement_many_or_extras(extra: str) -> None:
+    requires = [Requirement('filelock<4.0.0,>=3.9.0; extra == "extras1" or extra == "extras2" or extra == "extras3"')]
+    result = dependencies_with_extras(requires, {extra}, "")
     assert [str(r) for r in result] == ["filelock<4.0.0,>=3.9.0"]