Repository: iterative/dvc

Issue:
exp run/save: track top-level params/metrics/plots 
In both `exp run` and `exp save`, it's up to the user to track the params/metrics/plots even if they are already specified in `dvc.yaml`.

For example, take the setup in https://github.com/iterative/dvclive/issues/435#issuecomment-1410966511. 

I tried to update `dvc.yaml` to capture the metrics like:


```yaml
# dvc.yaml
stages:
  train:
    cmd: python train.py
    metrics:
      - dvclive/metrics.json
```

or:

```yaml
# dvc.yaml
metrics:
  - dvclive/metrics.json
stages:
  train:
    cmd: python train.py
```

Neither one captures the metrics until `dvclive/metrics.json` (or the `dvc.lock` file in the first case) is committed to Git. I thought `dvc.lock` was being added automatically, but apparently not? Also, is there any reason not to auto add params/metrics/plots?


Discussion:
@dberenbaum Could you revisit or try again the setup?

After the very first `exp run` I see the metrics and `dvc.lock` committed in the experiment ref even if they are untracked in the workspace.
It seems partially resolved but not for top-level metrics:

```
# dvc.yaml
stages:
  train:
    cmd: python train.py
    metrics:
      - dvclive/metrics.json
```

For this example, it's tracking tracking `dvclive/metrics.json` and `dvc.lock`.

```
# dvc.yaml
metrics:
  - dvclive/metrics.json
stages:
  train:
    cmd: python train.py
```

For this example, `dvclive/metrics.json` is not tracked. I don't expect it in `dvc.lock`, but I would expect it to be added to the Git commit.
@dberenbaum, having something in `metrics` does not mean it is git-tracked. At least, we are not interpreting as that at the moment.
I guess it depends on what we are doing already for `cache: false`.
If I'm using `dvc exp`, I'd expect that any paths referenced by `dvc.yaml` are being tracked by either Git or DVC.
> I guess it depends on what we are doing already for cache: false.

cache: false get included in the experiment commit
The metrics/params/plots that are not top-level are included in the experiment by:

https://github.com/iterative/dvc/blob/1de37556d5a5eb0dfefc1820c4359664d6312c2a/dvc/repo/reproduce.py#L68-L74

Which is run as a callback once per stage during `exp run`.

 For the top-level files, we need to do something like that but only once and after all the stages have been reproduced.

Resolving patch:
diff --git a/dvc/repo/experiments/executor/base.py b/dvc/repo/experiments/executor/base.py
--- a/dvc/repo/experiments/executor/base.py
+++ b/dvc/repo/experiments/executor/base.py
@@ -34,6 +34,8 @@
     ExpRefInfo,
 )
 from dvc.repo.experiments.utils import to_studio_params
+from dvc.repo.metrics.show import _collect_top_level_metrics
+from dvc.repo.params.show import _collect_top_level_params
 from dvc.stage.serialize import to_lockfile
 from dvc.ui import ui
 from dvc.utils import dict_sha256, env2bool, relpath
@@ -275,6 +277,13 @@ def save(
         else:
             os.chdir(dvc.root_dir)
 
+        include_untracked = include_untracked or []
+        include_untracked.extend(_collect_top_level_metrics(dvc))
+        include_untracked.extend(_collect_top_level_params(dvc))
+        include_untracked.extend(
+            dvc.index._plot_sources  # pylint: disable=protected-access
+        )
+
         try:
             stages = dvc.commit([], force=True, relink=False)
             exp_hash = cls.hash_exp(stages)
diff --git a/dvc/repo/experiments/utils.py b/dvc/repo/experiments/utils.py
--- a/dvc/repo/experiments/utils.py
+++ b/dvc/repo/experiments/utils.py
@@ -340,6 +340,6 @@ def to_studio_params(dvc_params):
         return result
     for rev_data in dvc_params.values():
         for file_name, file_data in rev_data.get("data", {}).items():
-            result[file_name] = file_data["data"]
+            result[file_name] = file_data.get("data", {})
 
     return result
diff --git a/dvc/repo/reproduce.py b/dvc/repo/reproduce.py
--- a/dvc/repo/reproduce.py
+++ b/dvc/repo/reproduce.py
@@ -3,6 +3,8 @@
 from typing import TYPE_CHECKING, Iterator, List
 
 from dvc.exceptions import DvcException, ReproductionError
+from dvc.repo.metrics.show import _collect_top_level_metrics
+from dvc.repo.params.show import _collect_top_level_params
 from dvc.repo.scm_context import scm_context
 from dvc.stage.exceptions import CheckpointKilledError
 
@@ -74,6 +76,17 @@ def _track_stage(stage: "Stage") -> None:
     return context.track_changed_files()
 
 
+def _track_top_level(repo: "Repo") -> None:
+    context = repo.scm_context
+    for metric_file in _collect_top_level_metrics(repo):
+        context.track_file(metric_file)
+    for param_file in _collect_top_level_params(repo):
+        context.track_file(param_file)
+    for plot_file in repo.index._plot_sources:  # pylint: disable=protected-access
+        context.track_file(plot_file)
+    context.track_changed_files()
+
+
 @locked
 @scm_context
 def reproduce(  # noqa: C901
@@ -125,7 +138,9 @@ def reproduce(  # noqa: C901
                 )
             )
 
-    return _reproduce_stages(self.index.graph, list(stages), **kwargs)
+    result = _reproduce_stages(self.index.graph, list(stages), **kwargs)
+    _track_top_level(self)
+    return result
 
 
 def _reproduce_stages(  # noqa: C901


Test patch:
diff --git a/tests/func/experiments/test_experiments.py b/tests/func/experiments/test_experiments.py
--- a/tests/func/experiments/test_experiments.py
+++ b/tests/func/experiments/test_experiments.py
@@ -683,3 +683,24 @@ def test_local_config_is_propagated_to_tmp(tmp_dir, scm, dvc):
     with fs.open("file") as fobj:
         conf_obj = ConfigObj(fobj)
         assert conf_obj["cache"]["type"] == "hardlink"
+
+
+@pytest.mark.parametrize("tmp", [True, False])
+def test_untracked_top_level_files_are_included_in_exp(tmp_dir, scm, dvc, tmp):
+    (tmp_dir / "dvc.yaml").dump(
+        {
+            "metrics": ["metrics.json"],
+            "params": ["params.yaml"],
+            "plots": ["plots.csv"],
+        }
+    )
+    stage = dvc.stage.add(
+        cmd="touch metrics.json && touch params.yaml && touch plots.csv",
+        name="top-level",
+    )
+    scm.add_commit(["dvc.yaml"], message="add dvc.yaml")
+    results = dvc.experiments.run(stage.addressing, tmp_dir=tmp)
+    exp = first(results)
+    fs = scm.get_fs(exp)
+    for file in ["metrics.json", "params.yaml", "plots.csv"]:
+        assert fs.exists(file)
diff --git a/tests/func/experiments/test_save.py b/tests/func/experiments/test_save.py
--- a/tests/func/experiments/test_save.py
+++ b/tests/func/experiments/test_save.py
@@ -116,3 +116,23 @@ def test_exp_save_include_untracked_warning(tmp_dir, dvc, scm, mocker):
 
     dvc.experiments.save(name="exp", include_untracked=["new_dir"])
     assert not logger.warning.called
+
+
+def test_untracked_top_level_files_are_included_in_exp(tmp_dir, scm, dvc):
+    (tmp_dir / "dvc.yaml").dump(
+        {
+            "metrics": ["metrics.json"],
+            "params": ["params.yaml"],
+            "plots": ["plots.csv"],
+        }
+    )
+    stage = dvc.stage.add(
+        cmd="touch metrics.json && touch params.yaml && touch plots.csv",
+        name="top-level",
+    )
+    scm.add_commit(["dvc.yaml"], message="add dvc.yaml")
+    dvc.reproduce(stage.addressing)
+    exp = dvc.experiments.save()
+    fs = scm.get_fs(exp)
+    for file in ["metrics.json", "params.yaml", "plots.csv"]:
+        assert fs.exists(file)
diff --git a/tests/unit/repo/experiments/test_utils.py b/tests/unit/repo/experiments/test_utils.py
--- a/tests/unit/repo/experiments/test_utils.py
+++ b/tests/unit/repo/experiments/test_utils.py
@@ -62,6 +62,10 @@ def test_run_check_ref_format(scm, name, result):
             {"workspace": {"data": {"params.yaml": {"data": {"foo": 1}}}}},
             {"params.yaml": {"foo": 1}},
         ),
+        (
+            {"workspace": {"data": {"params.yaml": {"error": "FileNotFound"}}}},
+            {"params.yaml": {}},
+        ),
         (
             {"workspace": {"error": "something went wrong"}},
             {},