Repository: jazzband/pip-tools

Issue:
Adding a nesting structure so that pip-compile can pin the same versions across nested extras.
#### What's the problem this feature will solve?

I am using `setup.cfg` to manage my dependencies and extra dependencies, and I am nesting some extra dependencies in each other. Currently, pip-compile does not guarantee resolving to the same version when I compile the requirements for nested extras.

```cfg
[options.extras_require]
server =
    pydantic~=1.10.7, >1.9.0

dev =
    %(server)s
    wget
```

#### Describe the solution you'd like

I would like to constrain the extended extras to the versions pinned by the `nested` extra. Here `server` should pin a pydantic version that `dev` must follow, and all should be generated into a single independent `requirements.txt` for each extra.

<!-- Provide examples of real-world use cases that this would enable and how it solves the problem described above. -->

This is especially important in building smaller containers, since I can guarantee version pinning and smaller diffs between layers of nested extras.

```Dockerfile
FROM base as builder-1
pip install --no-cache-dir server-requirements.txt

FROM builder
pip install --no-cache-dir dev-requirements.txt
```
#### Alternative Solutions
I can use multiple `requirements.txt` and nest them in each other and introduce dynamic constraints by symlinking the outputs of base extra.

#### Additional context

I use a single `setup.cfg` so that I can nest `extra` pip dependencies easily using `setuptools` magic. This gives me great flexibility to add requirements and does not make my repo much more crowded. If I had to migrate to having multiple `requirments.in` and dynamic constrains, it would add a lot of intermediary files, and it is harder to maintain the repository. I think having a single file in which I can define all packages is way more superior to any alternative IMO.



Discussion:

Resolving patch:
diff --git a/piptools/scripts/compile.py b/piptools/scripts/compile.py
--- a/piptools/scripts/compile.py
+++ b/piptools/scripts/compile.py
@@ -326,6 +326,20 @@ def _determine_linesep(
     help="Do not read any config file.",
     is_eager=True,
 )
+@click.option(
+    "-c",
+    "--constraint",
+    type=click.Path(
+        exists=True,
+        file_okay=True,
+        dir_okay=False,
+        readable=True,
+        allow_dash=False,
+        path_type=str,
+    ),
+    multiple=True,
+    help="Constrain versions using the given constraints file; may be used more than once.",
+)
 def cli(
     ctx: click.Context,
     verbose: int,
@@ -366,6 +380,7 @@ def cli(
     unsafe_package: tuple[str, ...],
     config: Path | None,
     no_config: bool,
+    constraint: tuple[str, ...],
 ) -> None:
     """
     Compiles requirements.txt from requirements.in, pyproject.toml, setup.cfg,
@@ -561,6 +576,18 @@ def cli(
                 )
             )
 
+    # Parse all constraints from `--constraint` files
+    for filename in constraint:
+        constraints.extend(
+            parse_requirements(
+                filename,
+                constraint=True,
+                finder=repository.finder,
+                options=repository.options,
+                session=repository.session,
+            )
+        )
+
     if upgrade_packages:
         constraints_file = tempfile.NamedTemporaryFile(mode="wt", delete=False)
         constraints_file.write("\n".join(upgrade_packages))


Test patch:
diff --git a/tests/conftest.py b/tests/conftest.py
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -10,7 +10,7 @@
 from functools import partial
 from pathlib import Path
 from textwrap import dedent
-from typing import Any
+from typing import Any, cast
 
 import pytest
 import tomli_w
@@ -234,7 +234,7 @@ def runner():
 @pytest.fixture
 def tmpdir_cwd(tmpdir):
     with tmpdir.as_cwd():
-        yield tmpdir
+        yield Path(tmpdir)
 
 
 @pytest.fixture
@@ -455,12 +455,12 @@ def _maker(
         pyproject_param: str, new_default: Any, config_file_name: str = CONFIG_FILE_NAME
     ) -> Path:
         # Make a config file with this one config default override
-        config_path = Path(tmpdir_cwd) / pyproject_param
+        config_path = tmpdir_cwd / pyproject_param
         config_file = config_path / config_file_name
         config_path.mkdir(exist_ok=True)
 
         config_to_dump = {"tool": {"pip-tools": {pyproject_param: new_default}}}
         config_file.write_text(tomli_w.dumps(config_to_dump))
-        return config_file.relative_to(tmpdir_cwd)
+        return cast(Path, config_file.relative_to(tmpdir_cwd))
 
     return _maker
diff --git a/tests/test_cli_compile.py b/tests/test_cli_compile.py
--- a/tests/test_cli_compile.py
+++ b/tests/test_cli_compile.py
@@ -3019,6 +3019,38 @@ def test_raise_error_on_invalid_config_option(
     assert "Invalid value for config key 'dry_run': ['invalid', 'value']" in out.stderr
 
 
+@pytest.mark.parametrize("option", ("-c", "--constraint"))
+def test_constraint_option(pip_conf, runner, tmpdir_cwd, make_config_file, option):
+    req_in = tmpdir_cwd / "requirements.in"
+    req_in.write_text("small-fake-a")
+
+    constraints_txt = tmpdir_cwd / "constraints.txt"
+    constraints_txt.write_text("small-fake-a==0.1")
+
+    out = runner.invoke(
+        cli,
+        [
+            req_in.name,
+            option,
+            constraints_txt.name,
+            "--output-file",
+            "-",
+            "--no-header",
+            "--no-emit-options",
+        ],
+    )
+
+    assert out.exit_code == 0
+    assert out.stdout == dedent(
+        """\
+        small-fake-a==0.1
+            # via
+            #   -c constraints.txt
+            #   -r requirements.in
+        """
+    )
+
+
 def test_allow_in_config_pip_sync_option(pip_conf, runner, tmp_path, make_config_file):
     config_file = make_config_file("--ask", True)  # pip-sync's option
 
diff --git a/tests/test_utils.py b/tests/test_utils.py
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -634,6 +634,7 @@ def test_callback_config_file_defaults(pyproject_param, new_default, make_config
         ("trusted_host", "not-a-list"),
         ("annotate", "not-a-bool"),
         ("max_rounds", "not-an-int"),
+        ("constraint", "not-an-list"),
     ),
 )
 def test_callback_config_file_defaults_multi_validate_value(