Repository: pdm-project/pdm
Pull request #2061 (2023-06-28 03:23:58)

fix: correctly update dependencies for out-of-order table
Signed-off-by: Frost Ming <me@frostming.com>

## Pull Request Checklist

- [x] A news fragment is added in `news/` describing what is new.
- [ ] Test cases added for changed code.

## Describe what you have changed in this PR.



Patch:
diff --git a/news/2056.bugfix.md b/news/2056.bugfix.md
new file mode 100644
index 00000000..bc358699
--- /dev/null
+++ b/news/2056.bugfix.md
@@ -0,0 +1,1 @@
+Fix a bug that dependencies can't be updated when the table is separated by another table.
diff --git a/news/2060.bugfix.md b/news/2060.bugfix.md
new file mode 100644
index 00000000..ed586f6e
--- /dev/null
+++ b/news/2060.bugfix.md
@@ -0,0 +1,1 @@
+Fix a bug that `*_lock` hooks are always emitted with dry_run=True in `pdm update`.
diff --git a/src/pdm/cli/commands/remove.py b/src/pdm/cli/commands/remove.py
index 25d4cb66..760bc536 100644
--- a/src/pdm/cli/commands/remove.py
+++ b/src/pdm/cli/commands/remove.py
@@ -78,7 +78,7 @@ class Command(BaseCommand):
         group = selection.one()
         lock_groups = project.lockfile.groups
 
-        deps, _ = project.get_pyproject_dependencies(group, selection.dev or False)
+        deps, setter = project.use_pyproject_dependencies(group, selection.dev or False)
         project.core.ui.echo(
             f"Removing packages from [primary]{group}[/] "
             f"{'dev-' if selection.dev else ''}dependencies: " + ", ".join(f"[req]{name}[/]" for name in packages)
@@ -91,7 +91,7 @@ class Command(BaseCommand):
                     raise ProjectError(f"[req]{name}[/] does not exist in [primary]{group}[/] dependencies.")
                 for i in matched_indexes:
                     del deps[i]
-        cast(Array, deps).multiline(True)
+        setter(cast(Array, deps).multiline(True))
 
         if not dry_run:
             project.pyproject.write()
diff --git a/src/pdm/cli/commands/update.py b/src/pdm/cli/commands/update.py
index 87f2fd99..ceb0d54c 100644
--- a/src/pdm/cli/commands/update.py
+++ b/src/pdm/cli/commands/update.py
@@ -154,15 +154,20 @@ class Command(BaseCommand):
                 for dep in deps.values():
                     dep.specifier = get_specifier("")
         reqs = [r for deps in all_dependencies.values() for r in deps.values()]
-        resolved = do_lock(
-            project,
-            strategy,
-            chain.from_iterable(updated_deps.values()),
-            reqs,
-            dry_run=True,
-            hooks=hooks,
-            groups=locked_groups,
-        )
+        # Since dry run is always true in the locking,
+        # we need to emit the hook manually with the real dry_run value
+        hooks.try_emit("pre_lock", requirements=reqs, dry_run=dry_run)
+        with hooks.skipping("pre_lock", "post_lock"):
+            resolved = do_lock(
+                project,
+                strategy,
+                chain.from_iterable(updated_deps.values()),
+                reqs,
+                dry_run=True,
+                hooks=hooks,
+                groups=locked_groups,
+            )
+        hooks.try_emit("post_lock", resolution=resolved, dry_run=dry_run)
         for deps in updated_deps.values():
             populate_requirement_names(deps)
         if unconstrained:
diff --git a/src/pdm/project/core.py b/src/pdm/project/core.py
index 4860c575..a16bf7ba 100644
--- a/src/pdm/project/core.py
+++ b/src/pdm/project/core.py
@@ -7,7 +7,7 @@ import re
 import shutil
 import sys
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Iterable, Mapping, cast
+from typing import TYPE_CHECKING, Any, Callable, Iterable, Mapping, cast
 
 import tomlkit
 from tomlkit.items import Array
@@ -524,31 +524,40 @@ class Project:
         content_hash = self.pyproject.content_hash(algo)
         return content_hash == hash_value
 
-    def get_pyproject_dependencies(self, group: str, dev: bool = False) -> tuple[list[str], bool]:
-        """Get the dependencies array in the pyproject.toml
+    def use_pyproject_dependencies(
+        self, group: str, dev: bool = False
+    ) -> tuple[list[str], Callable[[list[str]], None]]:
+        """Get the dependencies array and setter in the pyproject.toml
         Return a tuple of two elements, the first is the dependencies array,
-        and the second tells whether it is a dev-dependencies group.
+        and the second value is a callable to set the dependencies array back.
         """
+
+        def update_dev_dependencies(deps: list[str]) -> None:
+            from tomlkit.container import OutOfOrderTableProxy
+
+            settings.setdefault("dev-dependencies", {})[group] = deps
+            if isinstance(self.pyproject._data["tool"], OutOfOrderTableProxy):
+                # In case of a separate table, we have to remove and re-add it to make the write correct.
+                # This may change the order of tables in the TOML file, but it's the best we can do.
+                # see bug pdm-project/pdm#2056 for details
+                del self.pyproject._data["tool"]["pdm"]
+                self.pyproject._data["tool"]["pdm"] = settings
+
         metadata, settings = self.pyproject.metadata, self.pyproject.settings
         if group == "default":
-            return metadata.setdefault("dependencies", []), False
-        deps_dict = {
-            False: metadata.get("optional-dependencies", {}),
-            True: settings.get("dev-dependencies", {}),
-        }
-        for is_dev, deps in deps_dict.items():
+            return metadata.get("dependencies", tomlkit.array()), lambda x: metadata.__setitem__("dependencies", x)
+        deps_setter = [
+            (
+                metadata.get("optional-dependencies", {}),
+                lambda x: metadata.setdefault("optional-dependencies", {}).__setitem__(group, x),
+            ),
+            (settings.get("dev-dependencies", {}), update_dev_dependencies),
+        ]
+        for deps, setter in deps_setter:
             if group in deps:
-                return deps[group], is_dev
-        if dev:
-            return (
-                settings.setdefault("dev-dependencies", {}).setdefault(group, []),
-                dev,
-            )
-        else:
-            return (
-                metadata.setdefault("optional-dependencies", {}).setdefault(group, []),
-                dev,
-            )
+                return deps[group], setter
+        # If not found, return an empty list and a setter to add the group
+        return tomlkit.array(), deps_setter[int(dev)][1]
 
     def add_dependencies(
         self,
@@ -557,8 +566,7 @@ class Project:
         dev: bool = False,
         show_message: bool = True,
     ) -> None:
-        deps, is_dev = self.get_pyproject_dependencies(to_group, dev)
-        cast(Array, deps).multiline(True)
+        deps, setter = self.use_pyproject_dependencies(to_group, dev)
         for _, dep in requirements.items():
             matched_index = next(
                 (i for i, r in enumerate(deps) if dep.matches(r)),
@@ -569,6 +577,7 @@ class Project:
                 deps.append(req)
             else:
                 deps[matched_index] = req
+        setter(cast(Array, deps).multiline(True))
         self.pyproject.write(show_message)
 
     def init_global_project(self) -> None:


Test patch:
diff --git a/tests/cli/test_add.py b/tests/cli/test_add.py
index ec942dcb..98336081 100644
--- a/tests/cli/test_add.py
+++ b/tests/cli/test_add.py
@@ -256,7 +256,7 @@ def test_add_editable_package_with_extras(project, working_set, pdm):
     project.environment.python_requires = PySpecSet(">=3.6")
     dep_path = FIXTURES.joinpath("projects/demo").as_posix()
     pdm(["add", "-dGdev", "-e", f"{dep_path}[security]"], obj=project, strict=True)
-    assert f"-e {path_to_url(dep_path)}#egg=demo[security]" in project.get_pyproject_dependencies("dev", True)[0]
+    assert f"-e {path_to_url(dep_path)}#egg=demo[security]" in project.use_pyproject_dependencies("dev", True)[0]
     assert "demo" in working_set
     assert "requests" in working_set
     assert "urllib3" in working_set
@@ -266,7 +266,7 @@ def test_add_package_with_local_version(project, repository, working_set, pdm):
     repository.add_candidate("foo", "1.0-alpha.0+local")
     pdm(["add", "foo"], obj=project, strict=True)
     assert working_set["foo"].version == "1.0-alpha.0+local"
-    dependencies, _ = project.get_pyproject_dependencies("default")
+    dependencies, _ = project.use_pyproject_dependencies("default")
     assert dependencies[0] == "foo>=1.0a0"
 
 
diff --git a/tests/cli/test_build.py b/tests/cli/test_build.py
index 1edd5cda..f0a8523c 100644
--- a/tests/cli/test_build.py
+++ b/tests/cli/test_build.py
@@ -142,7 +142,7 @@ def test_cli_build_with_config_settings(fixture_project, pdm):
     assert (project.root / "dist/demo_package-0.1.0-py3-none-win_amd64.whl").exists()
 
 
-@pytest.mark.network
+@pytest.mark.usefixtures("local_finder")
 @pytest.mark.parametrize("isolated", (True, False))
 def test_build_with_no_isolation(fixture_project, pdm, isolated):
     project = fixture_project("demo-failure")