Repository: jazzband/pip-tools

Issue:
latest pip (23.3) causes dependency oddity
When running `pip-compile` + `pip` 23.3 with a requirements.in containing `pytest-cov`, it shows a weird dependency list with `coverage[toml]` being installed because of a dependency on `coverage`. Note that actual package versions are completely fine, this seems to be purely cosmetic.

Things seem better with the previous `pip` 23.2.1.

Not sure what causes this -- it could pip, pip-tools, pytest-cov or coverage, but `pip-tools` has the best community vibes so I'm trying here first ;)

#### Environment Versions

MacOS 13.6, python 3.11.6 managed through pyenv, fresh virtualenv with latest pip and pip-tools:
```
$ pip freeze --all
build==1.0.3
click==8.1.7
packaging==23.2
pip==23.3
pip-tools==7.3.0
pyproject_hooks==1.0.0
setuptools==65.5.0
wheel==0.41.2
```

#### Steps to replicate
$ cat requirements.in
pytest-cov
$ pip-compile requirements.in

#### Expected result
```
$ pip-compile requirements.in
WARNING: --strip-extras is becoming the default in version 8.0.0. To silence this warning, either use --strip-extras to opt into the new default or use --no-strip-extras to retain the existing behavior.
#
# This file is autogenerated by pip-compile with Python 3.11
# by the following command:
#
#    pip-compile requirements.in
#
coverage[toml]==7.3.2
    # via pytest-cov
iniconfig==2.0.0
    # via pytest
packaging==23.2
    # via pytest
pluggy==1.3.0
    # via pytest
pytest==7.4.2
    # via pytest-cov
pytest-cov==4.1.0
    # via -r requirements.in
```

#### Actual result
```
$ pip-compile requirements.in
WARNING: --strip-extras is becoming the default in version 8.0.0. To silence this warning, either use --strip-extras to opt into the new default or use --no-strip-extras to retain the existing behavior.
#
# This file is autogenerated by pip-compile with Python 3.11
# by the following command:
#
#    pip-compile requirements.in
#
coverage[toml]==7.3.2
    # via
    #   coverage
    #   pytest-cov
iniconfig==2.0.0
    # via pytest
packaging==23.2
    # via pytest
pluggy==1.3.0
    # via pytest
pytest==7.4.2
    # via pytest-cov
pytest-cov==4.1.0
    # via -r requirements.in
```
[FIXME] Nightly CI failures (against pip latest)
Here's an example: https://github.com/jazzband/pip-tools/actions/runs/6595090364/job/17919708295.

```console
=================================== FAILURES ===================================
___ test_combine_different_extras_of_the_same_package[backtracking resolver] ___
[gw3] darwin -- Python 3.10.13 /Users/runner/work/pip-tools/pip-tools/.tox/piplatest-coverage/bin/python

pip_conf = '/private/var/folders/3s/vfzpb5r51gs6y328rmlgzm7c0000gn/T/pytest-of-runner/pytest-0/popen-gw3/test_combine_different_extras_1/pip.conf'
runner = <click.testing.CliRunner object at 0x111ac1330>
tmpdir = local('/private/var/folders/3s/vfzpb5r51gs6y328rmlgzm7c0000gn/T/pytest-of-runner/pytest-0/popen-gw3/test_combine_different_extras_1')
make_package = <function make_package.<locals>._make_package at 0x1106fab00>
make_wheel = <function make_wheel.<locals>._make_wheel at 0x11054d5a0>

    def test_combine_different_extras_of_the_same_package(
        pip_conf, runner, tmpdir, make_package, make_wheel
    ):
        """
        Loosely based on the example from [https://github.com/jazzband/pip-tools/issues/1511.](https://github.com/jazzband/pip-tools/issues/1511.?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc)
        """
        pkgs = [
            make_package(
                "fake-colorful",
                version="0.3",
            ),
            make_package(
                "fake-tensorboardX",
                version="0.5",
            ),
            make_package(
                "fake-ray",
                version="0.1",
                extras_require={
                    "default": ["fake-colorful==0.3"],
                    "tune": ["fake-tensorboardX==0.5"],
                },
            ),
            make_package(
                "fake-tune-sklearn",
                version="0.7",
                install_requires=[
                    "fake-ray[tune]==0.1",
                ],
            ),
        ]
    
        dists_dir = tmpdir / "dists"
        for pkg in pkgs:
            make_wheel(pkg, dists_dir)
    
        with open("requirements.in", "w") as req_in:
            req_in.writelines(
                [
                    "fake-ray[default]==0.1\n",
                    "fake-tune-sklearn==0.7\n",
                ]
            )
    
        out = runner.invoke(
            cli,
            [
                "--output-file",
                "-",
                "--quiet",
                "--find-links",
                str(dists_dir),
                "--no-header",
                "--no-emit-options",
            ],
        )
        assert out.exit_code == 0
>       assert (
            dedent(
                """\
            fake-colorful==0.3
                # via fake-ray
            fake-ray[default,tune]==0.1
                # via
                #   -r requirements.in
                #   fake-tune-sklearn
            fake-tensorboardx==0.5
                # via fake-ray
            fake-tune-sklearn==0.7
                # via -r requirements.in
            """
            )
            == out.stdout
        )
E       AssertionError: assert 'fake-colorfu...irements.in\n' == 'fake-colorfu...irements.in\n'
E         Skipping 106 identical leading characters in diff, use -v to show
E         -  #   fake-ray
E         -     #   fake-tune-sklearn
E         ? ---
E         +  #   fake-tune-sklearn
E           fake-tensorboardx==0.5
E               # via fake-ray
E           fake-tune-sklearn==0.7
E               # via -r requirements.in

/Users/runner/work/pip-tools/pip-tools/tests/test_cli_compile.py:2354: AssertionError
----------------------------- Captured stderr call -----------------------------
/Users/runner/work/pip-tools/pip-tools/.tox/piplatest-coverage/lib/python3.10/site-packages/setuptools/_distutils/cmd.py:66: SetuptoolsDeprecationWarning: setup.py install is deprecated.
!!

        ********************************************************************************
        Please avoid running ``setup.py`` directly.
        Instead, use pypa/build, pypa/installer or other
        standards-based tools.

        See https://blog.ganssle.io/articles/2021/10/setup-py-deprecated.html for details.
        ********************************************************************************

!!
  self.initialize_options()
/Users/runner/work/pip-tools/pip-tools/.tox/piplatest-coverage/lib/python3.10/site-packages/setuptools/_distutils/cmd.py:66: SetuptoolsDeprecationWarning: setup.py install is deprecated.
!!

        ********************************************************************************
        Please avoid running ``setup.py`` directly.
        Instead, use pypa/build, pypa/installer or other
        standards-based tools.

        See https://blog.ganssle.io/articles/2021/10/setup-py-deprecated.html for details.
        ********************************************************************************

!!
  self.initialize_options()
/Users/runner/work/pip-tools/pip-tools/.tox/piplatest-coverage/lib/python3.10/site-packages/setuptools/_distutils/cmd.py:66: SetuptoolsDeprecationWarning: setup.py install is deprecated.
!!

        ********************************************************************************
        Please avoid running ``setup.py`` directly.
        Instead, use pypa/build, pypa/installer or other
        standards-based tools.

        See https://blog.ganssle.io/articles/2021/10/setup-py-deprecated.html for details.
        ********************************************************************************

!!
  self.initialize_options()
/Users/runner/work/pip-tools/pip-tools/.tox/piplatest-coverage/lib/python3.10/site-packages/setuptools/_distutils/cmd.py:66: SetuptoolsDeprecationWarning: setup.py install is deprecated.
!!

        ********************************************************************************
        Please avoid running ``setup.py`` directly.
        Instead, use pypa/build, pypa/installer or other
        standards-based tools.

        See https://blog.ganssle.io/articles/2021/10/setup-py-deprecated.html for details.
        ********************************************************************************

!!
  self.initialize_options()
=============================== warnings summary ===============================
tests/conftest.py:29
tests/conftest.py:29
tests/conftest.py:29
tests/conftest.py:29
tests/conftest.py:29
  /Users/runner/work/pip-tools/pip-tools/tests/conftest.py:29: DeprecationWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html
    from pip._vendor.pkg_resources import Requirement

.tox/piplatest-coverage/lib/python3.10/site-packages/xdist/plugin.py:252
.tox/piplatest-coverage/lib/python3.10/site-packages/xdist/plugin.py:252
.tox/piplatest-coverage/lib/python3.10/site-packages/xdist/plugin.py:252
.tox/piplatest-coverage/lib/python3.10/site-packages/xdist/plugin.py:252
.tox/piplatest-coverage/lib/python3.10/site-packages/xdist/plugin.py:252
  /Users/runner/work/pip-tools/pip-tools/.tox/piplatest-coverage/lib/python3.10/site-packages/xdist/plugin.py:252: DeprecationWarning: The --rsyncdir command line argument and rsyncdirs config variable are deprecated.
  The rsync feature will be removed in pytest-xdist 4.0.
    config.issue_config_time_warning(warning, 2)

tests/test_resolver.py: 1 warning
tests/test_cli_compile.py: 66 warnings
  /Users/runner/work/pip-tools/pip-tools/.tox/piplatest-coverage/lib/python3.10/site-packages/pip/_vendor/packaging/specifiers.py:255: DeprecationWarning: Creating a LegacyVersion has been deprecated and will be removed in the next major release
    warnings.warn(

tests/test_resolver.py: 24 warnings
tests/test_cli_compile.py: 132 warnings
  /Users/runner/work/pip-tools/pip-tools/.tox/piplatest-coverage/lib/python3.10/site-packages/pip/_vendor/packaging/version.py:111: DeprecationWarning: Creating a LegacyVersion has been deprecated and will be removed in the next major release
    warnings.warn(

tests/test_resolver.py::test_resolver[input16-expected16-False-unsafe_constraints16]
  /Users/runner/work/pip-tools/pip-tools/.tox/piplatest-coverage/lib/python3.10/site-packages/pip/_internal/models/link.py:445: PipDeprecationWarning: DEPRECATION: git+https://github.com/celery/billiard#egg=billiard==3.5.9999 contains an egg fragment with a non-PEP 508 name pip 25.0 will enforce this behaviour change. A possible replacement is to use the req @ url syntax, and remove the egg fragment. Discussion can be found at https://github.com/pypa/pip/issues/11617
    deprecated(

tests/test_sync.py::test_merge_urls
  /Users/runner/work/pip-tools/pip-tools/.tox/piplatest-coverage/lib/python3.10/site-packages/pip/_internal/models/link.py:445: PipDeprecationWarning: DEPRECATION: file:///example.zip#egg=example==1.0 contains an egg fragment with a non-PEP 508 name pip 25.0 will enforce this behaviour change. A possible replacement is to use the req @ url syntax, and remove the egg fragment. Discussion can be found at https://github.com/pypa/pip/issues/11617
    deprecated(

-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
```

It would also be useful to look into the file descriptor leaks per
```console
sys:1: ResourceWarning: unclosed <socket.socket fd=11, family=AddressFamily.AF_INET, type=SocketKind.SOCK_STREAM, proto=0, laddr=('127.0.0.1', 49217), raddr=('127.0.0.1', 49210)>
sys:1: ResourceWarning: unclosed <socket.socket fd=11, family=AddressFamily.AF_INET, type=SocketKind.SOCK_STREAM, proto=0, laddr=('127.0.0.1', 49218), raddr=('127.0.0.1', 49210)>
sys:1: ResourceWarning: unclosed <socket.socket fd=11, family=AddressFamily.AF_INET, type=SocketKind.SOCK_STREAM, proto=0, laddr=('127.0.0.1', 49215), raddr=('127.0.0.1', 49210)>
sys:1: ResourceWarning: unclosed <socket.socket fd=11, family=AddressFamily.AF_INET, type=SocketKind.SOCK_STREAM, proto=0, laddr=('127.0.0.1', 49216), raddr=('127.0.0.1', 49210)>
```


Discussion:
Hi @jap!

`[toml]` is called an extra and is a way to provide optional dependencies of distribution packages.
`pytest-cov == 4.1.0` specifies a dependency on `coverage[toml] >= 5.2.1` @ https://github.com/pytest-dev/pytest-cov/blob/v4.1.0/setup.py#L133. The `toml` extra itself is declared in `coverage[toml] == 7.3.2` @ https://github.com/nedbat/coveragepy/blob/7.3.2/setup.py#L106 and brings a conditional dependency `tomli` under older CPython versions under v3.11.0a6 and older. Since you're running `pip-tools` under Python v3.11.6 and `3.11.6 > 3.11.0a6`, the `tomli` dependency is not pulled into the tree by `coverage`.
To summarize, `coverage[toml]` is indeed met in your dependency tree, but it doesn't contribute any actual dependency to the resulting lockfile.

What may feel weird to you is probably a representation issue in the resulting constraint files. You can explicitly use `--strip-extras` on CLI or `strip-extras = true` in `.pip-tools.toml` to have the extras stripped off, which is going to become a default at some point per https://github.com/jazzband/pip-tools/issues/1613.
@webknjaz I think the report is not about the inclusion of `[toml]`, but the fact that the annotation now adds that `coverage[toml]` is installed `via coverage`, where it didn't before.

This can be seen in current main when testing with the latest pip release, where, at least locally, `test_combine_different_extras_of_the_same_package` is failing because now `fake-ray[default,tune]` is annotated as `via fake-ray` (other `via` items are still there, I'm paraphrasing to highlight the new change).
Note: this only happens with the backtracking resolver.
> @webknjaz I think the report is not about the inclusion of `[toml]`, but the fact that the annotation now adds that `coverage[toml]` is installed `via coverage`, where it didn't before.

Exactly, that is the only difference between the actual and expected result.
Oh, should we close this in favor of #2004, then?
That seems like a different issue to me. That one is about the non-comment requirement format, and this one is about dependency annotations.
Another phrasing that may or may not help: 

For #2004, extras need to be canonicalized/combined (and possibly restored, though I hope not), for proper ireq line format.

Here in #2003, we should probably ensure an ireq with extras doesn't depend on itself in extra-less form, at least according to annotations.
Looking into this.
Is this #2003?
Likely somewhat related. I haven't looked deep. Just wanted to document it explicitly.
I think it's exactly that issue. Quoting from my comment there:

> This can be seen in current main when testing with the latest pip release, where, at least locally, `test_combine_different_extras_of_the_same_package` is failing because now `fake-ray[default,tune]` is annotated as `via fake-ray` (other `via` items are still there, I'm paraphrasing to highlight the new change).

Resolving patch:
diff --git a/piptools/writer.py b/piptools/writer.py
--- a/piptools/writer.py
+++ b/piptools/writer.py
@@ -295,7 +295,11 @@ def _format_requirement(
                 if src_ireq.comes_from
             }
 
-        if ireq.comes_from:
+        # Filter out the origin install requirements for extras.
+        # See https://github.com/jazzband/pip-tools/issues/2003
+        if ireq.comes_from and (
+            isinstance(ireq.comes_from, str) or ireq.comes_from.name != ireq.name
+        ):
             required_by.add(_comes_from_as_string(ireq.comes_from))
 
         required_by |= set(getattr(ireq, "_required_by", set()))


Test patch:
diff --git a/tests/test_cli_compile.py b/tests/test_cli_compile.py
--- a/tests/test_cli_compile.py
+++ b/tests/test_cli_compile.py
@@ -3290,3 +3290,53 @@ def test_do_not_show_warning_on_explicit_strip_extras_option(
 
     assert out.exit_code == 0
     assert strip_extras_warning not in out.stderr
+
+
+def test_origin_of_extra_requirement_not_written_to_annotations(
+    pip_conf, runner, make_package, make_wheel, tmp_path, tmpdir
+):
+    req_in = tmp_path / "requirements.in"
+    package_with_extras = make_package(
+        "package_with_extras",
+        version="0.1",
+        extras_require={
+            "extra1": ["small-fake-a==0.1"],
+            "extra2": ["small-fake-b==0.1"],
+        },
+    )
+
+    dists_dir = tmpdir / "dists"
+    make_wheel(package_with_extras, dists_dir)
+
+    with open(req_in, "w") as req_out:
+        req_out.write("package-with-extras[extra1,extra2]")
+
+    out = runner.invoke(
+        cli,
+        [
+            "--output-file",
+            "-",
+            "--quiet",
+            "--no-header",
+            "--find-links",
+            str(dists_dir),
+            "--no-emit-options",
+            "--no-build-isolation",
+            req_in.as_posix(),
+        ],
+    )
+
+    assert out.exit_code == 0, out
+    assert (
+        dedent(
+            f"""\
+        package-with-extras[extra1,extra2]==0.1
+            # via -r {req_in.as_posix()}
+        small-fake-a==0.1
+            # via package-with-extras
+        small-fake-b==0.1
+            # via package-with-extras
+        """
+        )
+        == out.stdout
+    )