Repository: rapidsai/dependency-file-generator

Issue:
Rendering a nested pyproject.toml file to stdout is broken
Discovered this today while working on https://github.com/rapidsai/cudf/pull/15245.

On the latest version of `dependency-file-generator` (v1.13.4), rendering a nested `pyproject.toml` to stdout fails.

## Reproducible Example

```shell
git clone git@github.com:rapidsai/cudf.git
cd cudf
git checkout b810113d

rapids-dependency-file-generator \
  --output pyproject \
  --file-key py_run_cudf \
  --matrix "cuda=12.2"
```

Fails like this:

```text
Traceback (most recent call last):
  File "/Users/jlamb/miniforge3/envs/rapids-dev/lib/python3.10/site-packages/rapids_dependency_file_generator/_rapids_dependency_file_generator.py", line 183, in make_dependency_file
    table = table[section]
  File "/Users/jlamb/miniforge3/envs/rapids-dev/lib/python3.10/site-packages/tomlkit/container.py", line 624, in __getitem__
    item = self.item(key)
  File "/Users/jlamb/miniforge3/envs/rapids-dev/lib/python3.10/site-packages/tomlkit/container.py", line 466, in item
    raise NonExistentKey(key)
tomlkit.exceptions.NonExistentKey: 'Key "project" does not exist.'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/Users/jlamb/miniforge3/envs/rapids-dev/bin/rapids-dependency-file-generator", line 10, in <module>
    sys.exit(main())
  File "/Users/jlamb/miniforge3/envs/rapids-dev/lib/python3.10/site-packages/rapids_dependency_file_generator/_cli.py", line 151, in main
    make_dependency_files(parsed_config, file_keys, output, matrix, args.prepend_channels, to_stdout)
  File "/Users/jlamb/miniforge3/envs/rapids-dev/lib/python3.10/site-packages/rapids_dependency_file_generator/_rapids_dependency_file_generator.py", line 419, in make_dependency_files
    contents = make_dependency_file(
  File "/Users/jlamb/miniforge3/envs/rapids-dev/lib/python3.10/site-packages/rapids_dependency_file_generator/_rapids_dependency_file_generator.py", line 188, in make_dependency_file
    if not table.is_super_table():
AttributeError: 'TOMLDocument' object has no attribute 'is_super_table'
```

## Notes

I believe the specific issue is right here:

https://github.com/rapidsai/dependency-file-generator/blob/ec3d043e7ab0abb5acb111ebdeea0c72b691d0a5/src/rapids_dependency_file_generator/_rapids_dependency_file_generator.py#L418

Discussion:
@jameslamb: I have a fix for this, will put up a PR shortly. Just wanted to get the bug into writing.@GPUtester: :tada: This issue has been resolved in version 1.13.5 :tada:

The release is available on:
- [GitHub release](https://github.com/rapidsai/dependency-file-generator/releases/tag/v1.13.5)
- [PyPI release](https://pypi.org/project/rapids-dependency-file-generator/)
- [Conda release - rapidsai](https://anaconda.org/rapidsai/rapids-dependency-file-generator)
- [Conda release - rapidsai-nightly](https://anaconda.org/rapidsai-nightly/rapids-dependency-file-generator)

Your **[semantic-release](https://github.com/semantic-release/semantic-release)** bot :package::rocket:

Solution patch:
diff --git a/src/rapids_dependency_file_generator/_rapids_dependency_file_generator.py b/src/rapids_dependency_file_generator/_rapids_dependency_file_generator.py
index 96d9931..724b8ee 100644
--- a/src/rapids_dependency_file_generator/_rapids_dependency_file_generator.py
+++ b/src/rapids_dependency_file_generator/_rapids_dependency_file_generator.py
@@ -244,8 +244,8 @@ def get_filename(file_type: _config.Output, file_key: str, matrix_combo: dict[st
     return filename + file_ext
 
 
-def get_output_dir(file_type: _config.Output, config_file_path: os.PathLike, file_config: _config.File):
-    """Get the directory to which to write a generated dependency file.
+def get_output_dir(*, file_type: _config.Output, config_file_path: os.PathLike, file_config: _config.File):
+    """Get the directory containing a generated dependency file's contents.
 
     The output directory is determined by the `file_type` and the corresponding
     key in the `file_config`. The path provided in `file_config` will be taken
@@ -255,17 +255,16 @@ def get_output_dir(file_type: _config.Output, config_file_path: os.PathLike, fil
     ----------
     file_type : Output
         An Output value used to determine the file type.
-    output_root : PathLike
-        The path to the root directory where dependency files are placed.
+    config_file_path : PathLike
+        Path to the dependency-file-generator config file (e.g. dependencies.yaml).
     file_config : File
         A dictionary corresponding to one of the [files.$FILENAME] sections of
-        the dependencies.yaml file. May contain `conda_dir` or
-        `requirements_dir`.
+        the dependencies.yaml file. May contain `conda_dir`, `pyproject_dir`, or `requirements_dir`.
 
     Returns
     -------
     str
-        The directory to write the file to.
+        The directory containing the dependency file's contents.
     """
     path = [os.path.dirname(config_file_path)]
     if file_type == _config.Output.CONDA:
@@ -415,7 +414,11 @@ def make_dependency_files(
                 full_file_name = get_filename(file_type, file_key, matrix_combo)
                 deduped_deps = dedupe(dependencies)
 
-                output_dir = "." if to_stdout else get_output_dir(file_type, parsed_config.path, file_config)
+                output_dir = get_output_dir(
+                    file_type=file_type,
+                    config_file_path=parsed_config.path,
+                    file_config=file_config,
+                )
                 contents = make_dependency_file(
                     file_type=file_type,
                     name=full_file_name,


Test patch:
diff --git a/tests/examples/nested-pyproject/dependencies.yaml b/tests/examples/nested-pyproject/dependencies.yaml
new file mode 100644
index 0000000..5386b14
--- /dev/null
+++ b/tests/examples/nested-pyproject/dependencies.yaml
@@ -0,0 +1,24 @@
+files:
+  sparkly_unicorn:
+    output: pyproject
+    includes:
+      - run_deps
+    extras:
+      table: project
+    pyproject_dir: some/cool/code
+dependencies:
+  run_deps:
+    common:
+      - output_types: [pyproject]
+        packages:
+          - fsspec>=0.6.0
+    specific:
+      - output_types: [pyproject]
+        matrices:
+          - matrix: {"cuda": "100.*"}
+            packages:
+              - cuda-python>=100.1,<101.0a0
+          - matrix: {"cuda": "11.*"}
+            packages:
+              - cuda-python>=11.7.1,<12.0a0
+          - {matrix: null, packages: ["should-not-be-found-by-test"]}
diff --git a/tests/examples/nested-pyproject/pyproject.toml b/tests/examples/nested-pyproject/pyproject.toml
new file mode 100644
index 0000000..22ff89f
--- /dev/null
+++ b/tests/examples/nested-pyproject/pyproject.toml
@@ -0,0 +1,2 @@
+[tool.some-nonsense]
+should_dfg_update_this = "no"
diff --git a/tests/examples/nested-pyproject/some/cool/code/pyproject.toml b/tests/examples/nested-pyproject/some/cool/code/pyproject.toml
new file mode 100644
index 0000000..5119e18
--- /dev/null
+++ b/tests/examples/nested-pyproject/some/cool/code/pyproject.toml
@@ -0,0 +1,7 @@
+[project]
+name = "beep-boop"
+version = "1.2.3"
+dependencies = [
+    "fsspec>=0.6.0",
+    "should-not-be-found-by-test",
+] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../../dependencies.yaml and run `rapids-dependency-file-generator`.
diff --git a/tests/test_rapids_dependency_file_generator.py b/tests/test_rapids_dependency_file_generator.py
index 1a35a4e..2f34a40 100644
--- a/tests/test_rapids_dependency_file_generator.py
+++ b/tests/test_rapids_dependency_file_generator.py
@@ -1,12 +1,15 @@
 from unittest import mock
 
 import yaml
+import tomlkit
+import pathlib
 
 from rapids_dependency_file_generator import _config
 from rapids_dependency_file_generator._constants import cli_name
 from rapids_dependency_file_generator._rapids_dependency_file_generator import (
     dedupe,
     make_dependency_file,
+    make_dependency_files,
     should_use_specific_entry,
 )
 
@@ -67,6 +70,30 @@ def test_make_dependency_file(mock_relpath):
     assert env == header + "dep1\ndep2\n"
 
 
+def test_make_dependency_files_should_choose_correct_pyproject_toml(capsys):
+
+    current_dir = pathlib.Path(__file__).parent
+    make_dependency_files(
+        parsed_config=_config.load_config_from_file(current_dir / "examples" / "nested-pyproject" / "dependencies.yaml"),
+        file_keys=["sparkly_unicorn"],
+        output={_config.Output.PYPROJECT},
+        matrix={"cuda": ["100.17"]},
+        prepend_channels=[],
+        to_stdout=True
+    )
+    captured_stdout = capsys.readouterr().out
+
+    # should be valid TOML, containing the expected dependencies and the other contents of
+    # the nested pyproject.toml file
+    doc = tomlkit.loads(captured_stdout)
+    assert doc["project"]["name"] == "beep-boop"
+    assert doc["project"]["version"] == "1.2.3"
+    assert sorted(doc["project"]["dependencies"]) == ["cuda-python>=100.1,<101.0a0", "fsspec>=0.6.0"]
+
+    # and should NOT contain anything from the root-level pyproject.toml
+    assert set(dict(doc).keys()) == {"project"}
+
+
 def test_should_use_specific_entry():
     # no match
     matrix_combo = {"cuda": "11.5", "arch": "x86_64"}