Repository: pdm-project/pdm

Issue:
Incorrect resolution for JAX
- [x] I have searched the issue tracker and believe that this is not a duplicate.

I am trying to specify JAX as a dependency for my project, but currently it is not resolved correctly by PDM.

For some additional context. JAX depends on a native dependency `jaxlib` for correct operation. This requires the user to specify the extra (e.g., jax[cpu], jax[cuda]) to install the correct jaxlib _variant_, where the CPU variant uses a version and the GPU wheels use a local version identifier to differentiate from the CPU one (and the user needs to use JAX's find link).

## Steps to reproduce

The following pyproject resolves incorrectly
```toml
[project]
name = "pdm-example"
version = "0.1.0"
description = ""
authors = [
    {name = "Yicheng Luo", email = "ethanluoyc@gmail.com"},
]
dependencies = [
    "jax==0.4.17",
]
requires-python = ">=3.10,<3.11"
readme = "README.md"
license = {text = "MIT"}

[project.optional-dependencies]
cpu = ["jax[cpu]"]
cuda = ["jax[cuda12_pip]"]

[build-system]
requires = ["pdm-backend"]
build-backend = "pdm.backend"

# [[tool.pdm.source]]
# name = "jax_cuda"
# url = "https://storage.googleapis.com/jax-releases/jax_cuda_releases.html"
# verify_ssl = true
# type = "find_links"

[tool.pdm.resolution]
respect-source-order = true
```
If I run ` pdm lock -G cpu` and the lockfile looks like

```
# This file is @generated by PDM.
# It is not intended for manual editing.

[metadata]
groups = ["default", "cpu"]
strategy = ["cross_platform"]
lock_version = "4.4"
content_hash = "sha256:c7c792eac140bf630ef85a5d708263e9598ace33bba32675d6aade1ac6a881b4"

[[package]]
name = "jax"
version = "0.4.17"
requires_python = ">=3.9"
summary = "Differentiate, compile, and transform Numpy code."
dependencies = [
    "ml-dtypes>=0.2.0",
    "numpy>=1.22",
    "opt-einsum",
    "scipy>=1.7",
]
files = [
    {file = "jax-0.4.17-py3-none-any.whl", hash = "sha256:c3ab72ea2f1c5d8ccf2561e79f6562fb2964629f3e55b3ac1c11c48b64c20336"},
    {file = "jax-0.4.17.tar.gz", hash = "sha256:d7508a69e87835f534cb07a2f21d79cc1cb8c4cfdcf7fb010927267ef7355f1d"},
]

[[package]]
name = "jax"
version = "0.4.17"
extras = ["cpu"]
requires_python = ">=3.9"
summary = "Differentiate, compile, and transform Numpy code."
dependencies = [
    "jax==0.4.17",
    "jaxlib==0.4.17",
]
files = [
    {file = "jax-0.4.17-py3-none-any.whl", hash = "sha256:c3ab72ea2f1c5d8ccf2561e79f6562fb2964629f3e55b3ac1c11c48b64c20336"},
    {file = "jax-0.4.17.tar.gz", hash = "sha256:d7508a69e87835f534cb07a2f21d79cc1cb8c4cfdcf7fb010927267ef7355f1d"},
]

[[package]]
name = "jaxlib"
version = "0.4.17"
requires_python = ">=3.9"
summary = "XLA library for JAX"
dependencies = [
    "ml-dtypes>=0.2.0",
    "numpy>=1.22",
    "scipy>=1.7",
]
files = [
    {file = "jaxlib-0.4.17-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:d4be1ac4bf1be1ae1cd8f5f4da414a6d0de8de36cf2effdb5758d4d677896078"},
    {file = "jaxlib-0.4.17-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:392c779f902c43e1a0af49159daffef9b5af952aba001463f98cf95a59ef17ff"},
    {file = "jaxlib-0.4.17-cp310-cp310-manylinux2014_x86_64.whl", hash = "sha256:160fce68b82a79a6c522652e8dd9a10aac9c00d1599cb7e166671ad909aa139e"},
    {file = "jaxlib-0.4.17-cp310-cp310-win_amd64.whl", hash = "sha256:61b3788c6cfe46f307e6e67d4a942de72cf34711ff349f4f11500cdf6dc67199"},
]

[[package]]
name = "ml-dtypes"
version = "0.3.1"
requires_python = ">=3.9"
summary = ""
dependencies = [
    "numpy>1.20",
    "numpy>=1.21.2; python_version > \"3.9\"",
]
files = [
    {file = "ml_dtypes-0.3.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:510d249a91face47211762eb294d6fe64f325356b965fb6388c1bf51bd339267"},
    {file = "ml_dtypes-0.3.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f83ff080df8910c0f987f615b03e4f8198638e0c00c6e679ea8892dda909763b"},
    {file = "ml_dtypes-0.3.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fcae2c69715410d96906e1dfe8f017d9f78a0d10e0df91aae52e91f51fdfe45e"},
    {file = "ml_dtypes-0.3.1-cp310-cp310-win_amd64.whl", hash = "sha256:da274599e4950a9b488d21571061f49a185537cc77f2d3f8121151d58a9e9f16"},
    {file = "ml_dtypes-0.3.1.tar.gz", hash = "sha256:60778f99194b4c4f36ba42da200b35ef851ce4d4af698aaf70f5b91fe70fc611"},
]

[[package]]
name = "numpy"
version = "1.26.1"
requires_python = "<3.13,>=3.9"
summary = "Fundamental package for array computing in Python"
files = [
    {file = "numpy-1.26.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:82e871307a6331b5f09efda3c22e03c095d957f04bf6bc1804f30048d0e5e7af"},
    {file = "numpy-1.26.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cdd9ec98f0063d93baeb01aad472a1a0840dee302842a2746a7a8e92968f9575"},
    {file = "numpy-1.26.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d78f269e0c4fd365fc2992c00353e4530d274ba68f15e968d8bc3c69ce5f5244"},
    {file = "numpy-1.26.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8ab9163ca8aeb7fd32fe93866490654d2f7dda4e61bc6297bf72ce07fdc02f67"},
    {file = "numpy-1.26.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:78ca54b2f9daffa5f323f34cdf21e1d9779a54073f0018a3094ab907938331a2"},
    {file = "numpy-1.26.1-cp310-cp310-win32.whl", hash = "sha256:d1cfc92db6af1fd37a7bb58e55c8383b4aa1ba23d012bdbba26b4bcca45ac297"},
    {file = "numpy-1.26.1-cp310-cp310-win_amd64.whl", hash = "sha256:d2984cb6caaf05294b8466966627e80bf6c7afd273279077679cb010acb0e5ab"},
    {file = "numpy-1.26.1-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:06934e1a22c54636a059215d6da99e23286424f316fddd979f5071093b648668"},
    {file = "numpy-1.26.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:76ff661a867d9272cd2a99eed002470f46dbe0943a5ffd140f49be84f68ffc42"},
    {file = "numpy-1.26.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:6965888d65d2848e8768824ca8288db0a81263c1efccec881cb35a0d805fcd2f"},
    {file = "numpy-1.26.1.tar.gz", hash = "sha256:c8c6c72d4a9f831f328efb1312642a1cafafaa88981d9ab76368d50d07d93cbe"},
]

[[package]]
name = "opt-einsum"
version = "3.3.0"
requires_python = ">=3.5"
summary = "Optimizing numpys einsum function"
dependencies = [
    "numpy>=1.7",
]
files = [
    {file = "opt_einsum-3.3.0-py3-none-any.whl", hash = "sha256:2455e59e3947d3c275477df7f5205b30635e266fe6dc300e3d9f9646bfcea147"},
    {file = "opt_einsum-3.3.0.tar.gz", hash = "sha256:59f6475f77bbc37dcf7cd748519c0ec60722e91e63ca114e68821c0c54a46549"},
]

[[package]]
name = "scipy"
version = "1.11.3"
requires_python = "<3.13,>=3.9"
summary = "Fundamental algorithms for scientific computing in Python"
dependencies = [
    "numpy<1.28.0,>=1.21.6",
]
files = [
    {file = "scipy-1.11.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:370f569c57e1d888304052c18e58f4a927338eafdaef78613c685ca2ea0d1fa0"},
    {file = "scipy-1.11.3-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:9885e3e4f13b2bd44aaf2a1a6390a11add9f48d5295f7a592393ceb8991577a3"},
    {file = "scipy-1.11.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e04aa19acc324a1a076abb4035dabe9b64badb19f76ad9c798bde39d41025cdc"},
    {file = "scipy-1.11.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3e1a8a4657673bfae1e05e1e1d6e94b0cabe5ed0c7c144c8aa7b7dbb774ce5c1"},
    {file = "scipy-1.11.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7abda0e62ef00cde826d441485e2e32fe737bdddee3324e35c0e01dee65e2a88"},
    {file = "scipy-1.11.3-cp310-cp310-win_amd64.whl", hash = "sha256:033c3fd95d55012dd1148b201b72ae854d5086d25e7c316ec9850de4fe776929"},
    {file = "scipy-1.11.3.tar.gz", hash = "sha256:bba4d955f54edd61899776bad459bf7326e14b9fa1c552181f0479cc60a568cd"},
]

```

For `pdm export -G cpu` I get
```
# This file is @generated by PDM.
# Please do not edit it manually.

jax==0.4.17
jax==0.4.17
ml-dtypes==0.3.1
numpy==1.26.1
opt-einsum==3.3.0
scipy==1.11.3
```
## Actual behavior
There are a few issues with the resolution above 

1. Duplicate entries of JAX. I saw duplicate entries in the exported requirements.
2. When I selectively export the CPU group, jax should include jaxlib as a dependency, but it is currently missing from the exported requirements. This also causes problems with `pdm install` as jaxlib is also not installed.

This is not expected.

## Expected behavior

I expect that when using the CPU group to install dependencies, jaxlib should be included as a dependnecy.

## Environment Information

```bash
# Paste the output of `pdm info && pdm info --env` below:
PDM version:
  2.10.0
Python Interpreter:
  /workspace/.venv/bin/python (3.10)
Project Root:
  /workspace
Local Packages:

{
  "implementation_name": "cpython",
  "implementation_version": "3.10.13",
  "os_name": "posix",
  "platform_machine": "x86_64",
  "platform_release": "5.15.0-87-generic",
  "platform_system": "Linux",
  "platform_version": "#97~20.04.1-Ubuntu SMP Thu Oct 5 08:25:28 UTC 2023",
  "python_full_version": "3.10.13",
  "platform_python_implementation": "CPython",
  "python_version": "3.10",
  "sys_platform": "linux"
}
```

## Updates
I tried a few things, it seems that if you also pin the version in extra then things work, but I find that to be surprising.

Discussion:
codecov[bot]: ## [Codecov](https://app.codecov.io/gh/pdm-project/pdm/pull/2371?src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=pdm-project) Report
Attention: `2 lines` in your changes are missing coverage. Please review.

| [Files](https://app.codecov.io/gh/pdm-project/pdm/pull/2371?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=pdm-project) | Coverage Δ | |
|---|---|---|
| [src/pdm/models/candidates.py](https://app.codecov.io/gh/pdm-project/pdm/pull/2371?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=pdm-project#diff-c3JjL3BkbS9tb2RlbHMvY2FuZGlkYXRlcy5weQ==) | `86.06% <100.00%> (+0.31%)` | :arrow_up: |
| [src/pdm/models/repositories.py](https://app.codecov.io/gh/pdm-project/pdm/pull/2371?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=pdm-project#diff-c3JjL3BkbS9tb2RlbHMvcmVwb3NpdG9yaWVzLnB5) | `73.94% <100.00%> (-0.08%)` | :arrow_down: |
| [src/pdm/resolver/providers.py](https://app.codecov.io/gh/pdm-project/pdm/pull/2371?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=pdm-project#diff-c3JjL3BkbS9yZXNvbHZlci9wcm92aWRlcnMucHk=) | `92.94% <100.00%> (ø)` | |
| [src/pdm/formats/requirements.py](https://app.codecov.io/gh/pdm-project/pdm/pull/2371?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=pdm-project#diff-c3JjL3BkbS9mb3JtYXRzL3JlcXVpcmVtZW50cy5weQ==) | `77.98% <60.00%> (-0.59%)` | :arrow_down: |

... and [11 files with indirect coverage changes](https://app.codecov.io/gh/pdm-project/pdm/pull/2371/indirect-changes?src=pr&el=tree-more&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=pdm-project)


:loudspeaker: Thoughts on this report? [Let us know!](https://about.codecov.io/pull-request-comment-report/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=pdm-project).


Patch:
diff --git a/news/2273.enhancement.md b/news/2273.enhancement.md
deleted file mode 100644
index c740a83f..00000000
--- a/news/2273.enhancement.md
+++ /dev/null
@@ -1,2 +0,0 @@
-Add test coverage entrypoint in `pyproject.toml`
-Improve test coverage for untested functions in `pdm.utils`
\ No newline at end of file
diff --git a/news/2286.refactor.md b/news/2286.refactor.md
deleted file mode 100644
index affeceec..00000000
--- a/news/2286.refactor.md
+++ /dev/null
@@ -1,1 +0,0 @@
-Refactor `find_project_root()` and `find_pypackage()` to look for the project root recursively, and remove the `project_max_depth` configuration variable and the corresponding `PDM_PROJECT_MAX_DEPTH` environment variable.
diff --git a/news/2369.bugfix.md b/news/2369.bugfix.md
new file mode 100644
index 00000000..e903cda3
--- /dev/null
+++ b/news/2369.bugfix.md
@@ -0,0 +1,1 @@
+Fix a resolution issue that extra dependencies are not resolved when the bare dependency has more specific version constraint.
diff --git a/src/pdm/formats/requirements.py b/src/pdm/formats/requirements.py
index 1da87490..6cdfc89b 100644
--- a/src/pdm/formats/requirements.py
+++ b/src/pdm/formats/requirements.py
@@ -183,13 +183,18 @@ def export(
     from pdm.models.candidates import Candidate
 
     lines = ["# This file is @generated by PDM.\n# Please do not edit it manually.\n\n"]
+    collected_req: set[str] = set()
     for candidate in sorted(candidates, key=lambda x: x.identify()):  # type: ignore[attr-defined]
         if isinstance(candidate, Candidate):
             req = dataclasses.replace(candidate.req, specifier=get_specifier(f"=={candidate.version}"), marker=None)
         else:
             assert isinstance(candidate, Requirement)
             req = candidate
+        line = project.backend.expand_line(req.as_line(), options.expandvars)
+        if line in collected_req:
+            continue
         lines.append(project.backend.expand_line(req.as_line(), options.expandvars))
+        collected_req.add(line)
         if options.hashes and getattr(candidate, "hashes", None):
             for item in sorted({row["hash"] for row in candidate.hashes}):  # type: ignore[attr-defined]
                 lines.append(f" \\\n    --hash={item}")
diff --git a/src/pdm/models/candidates.py b/src/pdm/models/candidates.py
index bd7213bb..434df1ac 100644
--- a/src/pdm/models/candidates.py
+++ b/src/pdm/models/candidates.py
@@ -175,6 +175,16 @@ class Candidate:
     def identify(self) -> str:
         return self.req.identify()
 
+    def copy_with(self, requirement: Requirement) -> Candidate:
+        can = Candidate(requirement, name=self.name, version=self.version, link=self.link)
+        can.summary = self.summary
+        can.hashes = self.hashes
+        can._requires_python = self._requires_python
+        can._prepared = self._prepared
+        if can._prepared:
+            can._prepared.req = requirement
+        return can
+
     @property
     def dep_key(self) -> tuple[str, str | None]:
         """Key for retrieving and storing dependencies from the provider.
diff --git a/src/pdm/models/repositories.py b/src/pdm/models/repositories.py
index bb0c6e88..6fd07fbc 100644
--- a/src/pdm/models/repositories.py
+++ b/src/pdm/models/repositories.py
@@ -601,8 +601,7 @@ class LockedRepository(BaseRepository):
             if not requirement.name:
                 # make sure can.identify() won't return a randomly-generated name
                 requirement.name = can.name
-            can.req = requirement
-            yield can
+            yield can.copy_with(requirement)
 
     def get_hashes(self, candidate: Candidate) -> list[FileHash]:
         return candidate.hashes
diff --git a/src/pdm/resolver/providers.py b/src/pdm/resolver/providers.py
index 757e87b1..4d3345c8 100644
--- a/src/pdm/resolver/providers.py
+++ b/src/pdm/resolver/providers.py
@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import itertools
 import os
 from typing import TYPE_CHECKING, Callable, cast
 
@@ -159,15 +158,21 @@ class BaseProvider(AbstractProvider):
                 return (c for c in candidates if c not in incompat)
             elif identifier in self.overrides:
                 return iter(self.get_override_candidates(identifier))
-            reqs_iter = requirements[identifier]
+            reqs = sorted(requirements[identifier], key=self.requirement_preference)
+            original_req = reqs[0]
             bare_name, extras = strip_extras(identifier)
             if extras and bare_name in requirements:
                 # We should consider the requirements for both foo and foo[extra]
-                reqs_iter = itertools.chain(reqs_iter, requirements[bare_name])
-            reqs = sorted(reqs_iter, key=self.requirement_preference)
+                reqs.extend(requirements[bare_name])
+                reqs.sort(key=self.requirement_preference)
             candidates = self._find_candidates(reqs[0])
             return (
-                can for can in candidates if can not in incompat and all(self.is_satisfied_by(r, can) for r in reqs)
+                # In some cases we will use candidates from the bare requirement,
+                # this will miss the extra dependencies if any. So we associate the original
+                # requirement back with the candidate since it is used by `get_dependencies()`.
+                can.copy_with(original_req) if extras else can
+                for can in candidates
+                if can not in incompat and all(self.is_satisfied_by(r, can) for r in reqs)
             )
 
         return matches_gen


Test patch:
diff --git a/tests/cli/test_install.py b/tests/cli/test_install.py
index 853b0659..26f5cea7 100644
--- a/tests/cli/test_install.py
+++ b/tests/cli/test_install.py
@@ -287,3 +287,11 @@ def test_install_groups_and_lock(project, pdm, working_set):
     assert project.lockfile.groups == ["tz"]
     assert "pytz" in project.locked_repository.all_candidates
     assert "urllib3" not in project.locked_repository.all_candidates
+
+
+def test_install_requirement_with_extras(project, pdm, working_set):
+    project.add_dependencies({"requests": parse_requirement("requests==2.19.1")})
+    project.add_dependencies({"requests[socks]": parse_requirement("requests[socks]")}, to_group="socks")
+    pdm(["lock", "-Gsocks"], obj=project, strict=True)
+    pdm(["sync", "-Gsocks"], obj=project, strict=True)
+    assert "pysocks" in working_set