internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: iterative/dvc

Issue:
exp run: local config settings are not propagated into temp workspace
When generating the temp workspaces we do not respect the parent repo's `config.local`, so `--local` overrides for link types or state/index location will not be propagated and used during exp execution.


Discussion:

Resolving patch:
diff --git a/dvc/repo/experiments/executor/local.py b/dvc/repo/experiments/executor/local.py
--- a/dvc/repo/experiments/executor/local.py
+++ b/dvc/repo/experiments/executor/local.py
@@ -4,6 +4,7 @@
 from tempfile import mkdtemp
 from typing import TYPE_CHECKING, Optional, Union
 
+from configobj import ConfigObj
 from funcy import retry
 from shortuuid import uuid
 
@@ -134,22 +135,29 @@ def init_git(
         merge_rev = self.scm.get_ref(EXEC_MERGE)
 
         self.scm.stash.apply(merge_rev)
+        self._update_config(repo.config.read("local"))
 
-    def _config(self, cache_dir):
+    def _update_config(self, update):
         local_config = os.path.join(
             self.root_dir,
             self.dvc_dir,
             "config.local",
         )
         logger.debug("Writing experiments local config '%s'", local_config)
-        with open(local_config, "w", encoding="utf-8") as fobj:
-            fobj.write(f"[cache]\n    dir = {cache_dir}")
+        if os.path.exists(local_config):
+            conf_obj = ConfigObj(local_config)
+            conf_obj.merge(update)
+        else:
+            conf_obj = ConfigObj(update)
+        if conf_obj:
+            with open(local_config, "wb") as fobj:
+                conf_obj.write(fobj)
 
     def init_cache(
         self, repo: "Repo", rev: str, run_cache: bool = True  # noqa: ARG002
     ):
         """Initialize DVC cache."""
-        self._config(repo.cache.repo.path)
+        self._update_config({"cache": {"dir": repo.cache.repo.path}})
 
     def cleanup(self, infofile: Optional[str] = None):
         super().cleanup(infofile)


Test patch:
diff --git a/tests/func/experiments/test_experiments.py b/tests/func/experiments/test_experiments.py
--- a/tests/func/experiments/test_experiments.py
+++ b/tests/func/experiments/test_experiments.py
@@ -5,6 +5,7 @@
 from textwrap import dedent
 
 import pytest
+from configobj import ConfigObj
 from funcy import first
 
 from dvc.dvcfile import PROJECT_FILE
@@ -664,3 +665,21 @@ def test_experiment_no_commit(tmp_dir):
             repo.experiments.ls()
     finally:
         repo.close()
+
+
+def test_local_config_is_propagated_to_tmp(tmp_dir, scm, dvc):
+    with dvc.config.edit("local") as conf:
+        conf["cache"]["type"] = "hardlink"
+
+    stage = dvc.stage.add(
+        cmd="cat .dvc/config.local > file", name="foo", outs_no_cache=["file"]
+    )
+    scm.add_commit(["dvc.yaml"], message="add dvc.yaml")
+
+    results = dvc.experiments.run(stage.addressing, tmp_dir=True)
+    exp = first(results)
+    fs = scm.get_fs(exp)
+
+    with fs.open("file") as fobj:
+        conf_obj = ConfigObj(fobj)
+        assert conf_obj["cache"]["type"] == "hardlink"
Repository: iterative/dvc

Issue:
Being able to call  `exp show` with multiple branches (`--rev`) and a number of commits (`-n`)
In the VS Code extension, we would like to show multiple branches at the same time in the table. Each with its own number of commits. We're currently doing this by calling `exp show` once per branch (`dvc exp show --rev main -n 5`, `dvc exp show --rev other-branch -n 3`...). We'd like to be able to do this in one call only (something like `dvc exp show --rev main -n 5 --rev other-branch -n 3...`). Would it be possible?


Discussion:
> Would it be possible?

The internal API already supports multiple `revs`.

This should be a simple change to make for the CLI (TBH I thought the CLI already supported it)

Resolving patch:
diff --git a/dvc/commands/experiments/__init__.py b/dvc/commands/experiments/__init__.py
--- a/dvc/commands/experiments/__init__.py
+++ b/dvc/commands/experiments/__init__.py
@@ -82,6 +82,7 @@ def add_rev_selection_flags(
     experiments_subcmd_parser.add_argument(
         "--rev",
         type=str,
+        action="append",
         default=None,
         help=msg,
         metavar="<commit>",


Test patch:
diff --git a/tests/unit/command/test_experiments.py b/tests/unit/command/test_experiments.py
--- a/tests/unit/command/test_experiments.py
+++ b/tests/unit/command/test_experiments.py
@@ -116,7 +116,7 @@ def test_experiments_show(dvc, scm, mocker):
         hide_queued=True,
         hide_failed=True,
         num=1,
-        revs="foo",
+        revs=["foo"],
         sha_only=True,
         param_deps=True,
         fetch_running=True,
@@ -226,7 +226,7 @@ def test_experiments_list(dvc, scm, mocker):
     m.assert_called_once_with(
         cmd.repo,
         git_remote="origin",
-        rev="foo",
+        rev=["foo"],
         all_commits=True,
         num=-1,
     )
@@ -265,7 +265,7 @@ def test_experiments_push(dvc, scm, mocker):
         cmd.repo,
         "origin",
         ["experiment1", "experiment2"],
-        rev="foo",
+        rev=["foo"],
         all_commits=True,
         num=2,
         force=True,
@@ -322,7 +322,7 @@ def test_experiments_pull(dvc, scm, mocker):
         cmd.repo,
         "origin",
         ["experiment"],
-        rev="foo",
+        rev=["foo"],
         all_commits=True,
         num=1,
         force=True,
@@ -371,7 +371,7 @@ def test_experiments_remove_flag(dvc, scm, mocker, capsys, caplog):
         cmd.repo,
         exp_names=[],
         all_commits=True,
-        rev="foo",
+        rev=["foo"],
         num=2,
         queue=False,
         git_remote="myremote",
Repository: iterative/dvc

Issue:
add/import-url: fix cloud versioning
https://github.com/iterative/dvc-s3/actions/runs/5241699655/jobs/9464141699?pr=43


Discussion:

Resolving patch:
diff --git a/dvc/repo/index.py b/dvc/repo/index.py
--- a/dvc/repo/index.py
+++ b/dvc/repo/index.py
@@ -136,7 +136,11 @@ def _load_data_from_outs(index, prefix, outs):
         if out.stage.is_import and not out.stage.is_repo_import:
             dep = out.stage.deps[0]
             entry.meta = dep.meta
-            entry.hash_info = dep.hash_info
+            if out.hash_info:
+                entry.hash_info = out.hash_info
+            else:
+                # partial import
+                entry.hash_info = dep.hash_info
 
         # FIXME PyGTrie-based DataIndex doesn't remove entry.key during
         # index.add, so we have to set the entry manually here to make


Test patch:
diff --git a/dvc/testing/workspace_tests.py b/dvc/testing/workspace_tests.py
--- a/dvc/testing/workspace_tests.py
+++ b/dvc/testing/workspace_tests.py
@@ -150,6 +150,8 @@ def test_import_no_download(self, tmp_dir, dvc, remote_version_aware):
 
         dvc.pull()
         assert (tmp_dir / "data_dir" / "subdir" / "file").read_text() == "file"
+
+        dvc.commit(force=True)
         assert dvc.status() == {}
Repository: iterative/dvc

Issue:
`data status --not-in-remote`: not working
# Bug Report

## Description

`dvc data status --not-in-remote` doesn't report what is not in remote. This looks like a regression introduced [here](https://github.com/iterative/dvc/commit/654358010076d1c20d90edbab39733cc27c598e0) (cc @efiop).

### Reproduce

```
git init -q repo
cd repo
dvc init -q
dvc remote add -q -d storage /tmp/remote
touch test
dvc add -q test
git add .
git commit -q -m "init"
dvc data status --not-in-remote
```

This outputs no changes.


### Expected

An output that shows data is missing from the remote.

### Environment information

**Output of `dvc doctor`:**

```console
$ dvc doctor
DVC version: 3.0.0a2.dev11+g78637713c
-------------------------------------
Platform: Python 3.10.10 on macOS-13.3.1-arm64-arm-64bit
Subprojects:
        dvc_data = 0.54.3
        dvc_objects = 0.22.0
        dvc_render = 0.5.3
        dvc_task = 0.2.1
        scmrepo = 1.0.3
Supports:
        azure (adlfs = 2023.4.0, knack = 0.10.1, azure-identity = 1.12.0),
        gdrive (pydrive2 = 1.15.3),
        gs (gcsfs = 2022.11.0),
        hdfs (fsspec = 2022.11.0, pyarrow = 11.0.0),
        http (aiohttp = 3.7.4.post0, aiohttp-retry = 2.8.3),
        https (aiohttp = 3.7.4.post0, aiohttp-retry = 2.8.3),
        oss (ossfs = 2021.8.0),
        s3 (s3fs = 2022.11.0, boto3 = 1.24.59),
        ssh (sshfs = 2023.4.1),
        webdav (webdav4 = 0.9.8),
        webdavs (webdav4 = 0.9.8),
        webhdfs (fsspec = 2022.11.0)
Config:
        Global: /Users/dave/Library/Application Support/dvc
        System: /Library/Application Support/dvc
```


Discussion:
Besides being a regression, this is a blocker for #5369
One of the changes in dvc-data caused this. Preparing a fix.

Resolving patch:
diff --git a/dvc/repo/index.py b/dvc/repo/index.py
--- a/dvc/repo/index.py
+++ b/dvc/repo/index.py
@@ -670,7 +670,7 @@ def build_data_index(  # noqa: C901
                 hash_name=hash_name,
             )
         except FileNotFoundError:
-            continue
+            out_entry = DataIndexEntry()
 
         out_entry.key = key
         data.add(out_entry)


Test patch:
Repository: iterative/dvc

Issue:
plots: fail silently on errors
# Bug Report

## Description

`dvc plots show` has started failing silently when there are errors instead of reporting an error was found. It looks like this was introduced in https://github.com/iterative/dvc/commit/43195a806d690b65f9429b82444e48992b7442d9 (cc @skshetry).

### Reproduce

Clone https://github.com/iterative/vscode-dvc-demo and make these changes to `dvc.yaml`:

```
$ git diff
diff --git a/dvc.yaml b/dvc.yaml
index 3a4fa26..cd91913 100644
--- a/dvc.yaml
+++ b/dvc.yaml
@@ -2,8 +2,8 @@ plots:
   - Accuracy:
       x: step
       y:
-        training/plots/metrics/train/acc.tsv: acc
-        training/plots/metrics/test/acc.tsv: acc
+        training/plots/metrics/train/acc.tsv: train/acc
+        training/plots/metrics/test/acc.tsv: test/acc
       y_label: accuracy
   - Loss:
       x: step
```

Then run `dvc plots show`.

### Expected

Prior to this change, `dvc plots show` would return an error message like this:

```
$ dvc plots show
ERROR: Could not find provided field ('train/acc') in data fields ('step, acc').
```


Discussion:
I think I did it in this way to fix https://github.com/iterative/dvc/issues/9025. 
Before we just used to fail, but now we reuse the same merged spec over multiple revisions data.

So, you may see multiple messages in this case if we logged them:
```console
$ dvc plots diff workspace $(git log --oneline --pretty=format:"%h" -n2)
WARNING: Could not find provided field ('acc/loss') in data fields ('step, loss').                                                                                                                                   
WARNING: Could not find provided field ('acc/loss') in data fields ('step, loss').
WARNING: Could not find provided field ('acc/loss') in data fields ('step, loss').
```


Makes sense for `diff` but what about for `show`?

Resolving patch:
diff --git a/dvc/render/match.py b/dvc/render/match.py
--- a/dvc/render/match.py
+++ b/dvc/render/match.py
@@ -1,3 +1,4 @@
+import logging
 import os
 from collections import defaultdict
 from typing import TYPE_CHECKING, DefaultDict, Dict, List, NamedTuple, Optional
@@ -17,6 +18,7 @@
 
 
 dpath.options.ALLOW_EMPTY_STRING_KEYS = True
+logger = logging.getLogger(__name__)
 
 
 def _squash_plots_properties(data: List) -> Dict:
@@ -113,6 +115,7 @@ def match_defs_renderers(  # noqa: C901, PLR0912
             try:
                 dps, rev_props = converter.flat_datapoints(rev)
             except Exception as e:  # noqa: BLE001, pylint: disable=broad-except
+                logger.warning("In %r, %s", rev, str(e).lower())
                 def_errors[rev] = e
                 continue
 


Test patch:
Repository: iterative/dvc

Issue:
plots diff/show: caching is broken for files in legacy cache format
```console
git clone https://github.com/iterative/vscode-dvc-demo
cd vscode-dvc-demo
dvc plots diff
ls .dvc/cache -la
dvc plots diff
```

The `plots diff` refetches them from remote, and is not being cached.
See https://github.com/iterative/dvc/pull/9183.


Discussion:
@skshetry This is another outstanding 3.0 regression.
Previously @efiop was against introducing caching built into `dvc_data.fs.DataFileSystem`. 
It is a bit difficult to introspect the appropriate odb/cache from the outside. I'll try to look into this again.

@efiop, if you have any ideas? 
I decided to look into adding support for caching to DataFileSystem.

Resolving patch:
diff --git a/dvc/fs/dvc.py b/dvc/fs/dvc.py
--- a/dvc/fs/dvc.py
+++ b/dvc/fs/dvc.py
@@ -261,15 +261,12 @@ def _open(
         try:
             return self.repo.fs.open(fs_path, mode=mode)
         except FileNotFoundError:
-            repo, dvc_fs, subkey = self._get_subrepo_info(key)
+            _, dvc_fs, subkey = self._get_subrepo_info(key)
             if not dvc_fs:
                 raise
 
         dvc_path = _get_dvc_path(dvc_fs, subkey)
-        kw = {}
-        if kwargs.get("cache_remote_stream", False):
-            kw["cache_odb"] = repo.cache.local
-        return dvc_fs.open(dvc_path, mode=mode, **kw)
+        return dvc_fs.open(dvc_path, mode=mode, cache=kwargs.get("cache", False))
 
     def isdvc(self, path, **kwargs) -> bool:
         """Is this entry dvc-tracked?"""
diff --git a/dvc/repo/plots/__init__.py b/dvc/repo/plots/__init__.py
--- a/dvc/repo/plots/__init__.py
+++ b/dvc/repo/plots/__init__.py
@@ -212,7 +212,7 @@ def show(
             short_rev = "workspace"
             if rev := getattr(self.repo.fs, "rev", None):
                 short_rev = rev[:7]
-            _resolve_data_sources(data, short_rev, cache_remote_stream=True)
+            _resolve_data_sources(data, short_rev, cache=True)
             result.update(data)
 
         errored = errored_revisions(result)
@@ -281,9 +281,7 @@ def _is_plot(out: "Output") -> bool:
     return bool(out.plot)
 
 
-def _resolve_data_sources(
-    plots_data: Dict, rev: str, cache_remote_stream: bool = False
-):
+def _resolve_data_sources(plots_data: Dict, rev: str, cache: bool = False):
     from dvc.progress import Tqdm
 
     values = list(plots_data.values())
@@ -298,7 +296,7 @@ def _resolve_data_sources(
     def resolve(value):
         data_source = value.pop("data_source")
         assert callable(data_source)
-        value.update(data_source(cache_remote_stream=cache_remote_stream))
+        value.update(data_source(cache=cache))
 
     if not to_resolve:
         return


Test patch:
Repository: iterative/dvc

Issue:
`DVCFileSystem` does not support `fsspec`'s chaining API
I have a repository hosted on Gitlab and tracked using DVC. I would like to load a file using a library that uses `fsspec` under the hood, so I need to provide a `fsspec` compliant URL to load the file. I tried to chain DVC and the Gitlab URL like so: `fsspec.core.url_to_fs("dvc::https://gitlab.com/repository/folder/file.json")` but I get the following exception:

```
TypeError: Repo.__init__() got an unexpected keyword argument 'target_options'
```

It appears that `_DVCFileSystem` does not comply with the API described in [`fsspec` URL chaining documentation](https://filesystem-spec.readthedocs.io/en/latest/features.html#url-chaining).

I have a fix for my specific use case that involves patching `_DVCFileSystem`'s constructor like so:

```python
if repo is None:
    url = url if url is not None else repo_kwargs.pop("fo")
    repo_kwargs.pop("target_options")
    repo_kwargs.pop("target_protocol")
    repo = self._make_repo(url=url, rev=rev, subrepos=subrepos, **repo_kwargs)
```

but I'm not entirely confident that it will work with all cases.


Discussion:
@bilelomrani1 If that solves it for you, please don't hesitate to submit a PR. We should go from the need there and your implementation seems reasonable, at least to start with.

Resolving patch:
diff --git a/dvc/fs/dvc.py b/dvc/fs/dvc.py
--- a/dvc/fs/dvc.py
+++ b/dvc/fs/dvc.py
@@ -83,6 +83,11 @@ def __init__(
         repo: Optional["Repo"] = None,
         subrepos: bool = False,
         repo_factory: Optional[RepoFactory] = None,
+        fo: Optional[str] = None,
+        # pylint:disable-next=unused-argument
+        target_options: Optional[Dict[str, Any]] = None,  # noqa: ARG002
+        # pylint:disable-next=unused-argument
+        target_protocol: Optional[str] = None,  # noqa: ARG002
         **repo_kwargs: Any,
     ) -> None:
         """DVC + git-tracked files fs.
@@ -125,6 +130,7 @@ def __init__(
         super().__init__()
         self._repo_stack = ExitStack()
         if repo is None:
+            url = url if url is not None else fo
             repo = self._make_repo(url=url, rev=rev, subrepos=subrepos, **repo_kwargs)
             assert repo is not None
             # pylint: disable=protected-access


Test patch:
Repository: iterative/dvc

Issue:
Typo in this log message dvc/rwlock.py ? remove -> removing
Looks so to me, but I am unsure.

https://github.com/iterative/dvc/blob/65d412bfa356d974fa1e1969346878b0f11284b3/dvc/rwlock.py#L107


Discussion:
Thanks @thisiswhereitype! The context is in https://github.com/iterative/dvc/pull/8623. It looks like you are right, although I would suggest `removed` since I think it's already done at this point. Feel free to submit a PR with the change.

Resolving patch:
diff --git a/dvc/rwlock.py b/dvc/rwlock.py
--- a/dvc/rwlock.py
+++ b/dvc/rwlock.py
@@ -104,7 +104,7 @@ def _check_blockers(tmp_dir, lock, info, *, mode, waiters):  # noqa: C901, PLR09
                 logger.warning(
                     (
                         "Process '%s' with (Pid %s), in RWLock-file '%s'"
-                        " had been killed. Auto remove it from the lock file."
+                        " had been killed. Auto removed it from the lock file."
                     ),
                     cmd,
                     pid,


Test patch:
Repository: iterative/dvc

Issue:
unnecessary remote repository fetches
DVC has started cloning external repositories even on cases where it's not strictly required. 
Anything that uses `outs_graph` has started cloning external repositories.

Commands like `add/plots modify/pull/push/fetch/move` are affected, and maybe potentially others.

### Reproduce

```bash
git clone https://github.com/iterative/example-get-started.git
cd example-get-started
dvc add foo
```

### Expected

No cloning occurs.





Discussion:

Resolving patch:
diff --git a/dvc/repo/graph.py b/dvc/repo/graph.py
--- a/dvc/repo/graph.py
+++ b/dvc/repo/graph.py
@@ -158,10 +158,9 @@ def build_outs_graph(graph, outs_trie):
 
     outs_graph.add_nodes_from(outs_trie.values())
     for stage in graph.nodes():
+        if stage.is_repo_import:
+            continue
         for dep in stage.deps:
-            if dep.fs_path is None:
-                # RepoDependency don't have a path
-                continue
             dep_key = dep.fs.path.parts(dep.fs_path)
             overlapping = [n.value for n in outs_trie.prefixes(dep_key)]
             if outs_trie.has_subtrie(dep_key):


Test patch:
Repository: iterative/dvc

Issue:
exp save: Short option for --message is -M, but for dvc exp run it is -m
It would be nice if the short options of `dvc exp run` and `dvc exp save` for specifying a commit message would be identical. Also, best to use the same options as one would use for `git commit`, i.e., `-m` instead of `-M`.

```
usage: dvc experiments save [-h] [-q | -v] [-f] [--json] [-n <name>] [-I <path>] [-M MESSAGE]

Save current workspace as an experiment.
Documentation: <https://man.dvc.org/exp/save>

options:
  -h, --help            show this help message and exit
  -q, --quiet           Be quiet.
  -v, --verbose         Be verbose.
  -f, --force           Replace experiment if it already exists.
  --json                Show output in JSON format.
  -n <name>, --name <name>
                        Human-readable experiment name. If not specified, a name will be auto-generated.
  -I <path>, --include-untracked <path>
                        List of untracked paths to include in the experiment.
  -M MESSAGE, --message MESSAGE
                        Custom commit message to use when committing the experiment.
```

DVC CLI v3.22.1


Discussion:

Resolving patch:
diff --git a/dvc/commands/experiments/exec_run.py b/dvc/commands/experiments/exec_run.py
--- a/dvc/commands/experiments/exec_run.py
+++ b/dvc/commands/experiments/exec_run.py
@@ -49,7 +49,7 @@ def add_parser(experiments_subparsers, parent_parser):
         ),
     )
     exec_run_parser.add_argument(
-        "-M",
+        "-m",
         "--message",
         type=str,
         default=None,
diff --git a/dvc/commands/experiments/save.py b/dvc/commands/experiments/save.py
--- a/dvc/commands/experiments/save.py
+++ b/dvc/commands/experiments/save.py
@@ -72,7 +72,7 @@ def add_parser(experiments_subparsers, parent_parser):
         metavar="<path>",
     )
     save_parser.add_argument(
-        "-M",
+        "-m",
         "--message",
         type=str,
         default=None,


Test patch:
diff --git a/tests/unit/command/test_experiments.py b/tests/unit/command/test_experiments.py
--- a/tests/unit/command/test_experiments.py
+++ b/tests/unit/command/test_experiments.py
@@ -479,6 +479,24 @@ def test_experiments_save(dvc, scm, mocker):
     )
 
 
+def test_experiments_save_message(dvc, scm, mocker):
+    cli_args = parse_args(["exp", "save", "-m", "custom commit message"])
+    assert cli_args.func == CmdExperimentsSave
+
+    cmd = cli_args.func(cli_args)
+    m = mocker.patch("dvc.repo.experiments.save.save", return_value="acabb")
+
+    assert cmd.run() == 0
+
+    m.assert_called_once_with(
+        cmd.repo,
+        name=None,
+        force=False,
+        include_untracked=[],
+        message="custom commit message",
+    )
+
+
 def test_experiments_clean(dvc, scm, mocker):
     cli_args = parse_args(["experiments", "clean"])
     assert cli_args.func == CmdExperimentsClean
Repository: iterative/dvc

Issue:
`params_show`: locking issues
`params_show` requires the lock, which breaks in multi-gpu/multi-node training. Need to reconsider whether it's needed and/or recommend another method for reading params, at least in multi-gpu/multi-node scenarios (see https://dvc.org/doc/dvclive/ml-frameworks/pytorch#distributeddataparallel).


Discussion:
Is it a git lock? can we add Git no lock option like @mattseddon did in VS Code?

Resolving patch:
diff --git a/dvc/repo/metrics/show.py b/dvc/repo/metrics/show.py
--- a/dvc/repo/metrics/show.py
+++ b/dvc/repo/metrics/show.py
@@ -17,7 +17,6 @@
 from scmrepo.exceptions import SCMError
 
 from dvc.log import logger
-from dvc.repo import locked
 from dvc.scm import NoSCMError
 from dvc.utils import as_posix, expand_paths
 from dvc.utils.collections import ensure_list
@@ -185,7 +184,6 @@ def _hide_workspace(
     return res
 
 
-@locked
 def show(
     repo: "Repo",
     targets: Optional[List[str]] = None,
diff --git a/dvc/repo/params/show.py b/dvc/repo/params/show.py
--- a/dvc/repo/params/show.py
+++ b/dvc/repo/params/show.py
@@ -5,7 +5,6 @@
 
 from dvc.dependency.param import ParamsDependency, read_param_file
 from dvc.log import logger
-from dvc.repo import locked
 from dvc.repo.metrics.show import FileResult, Result
 from dvc.stage import PipelineStage
 from dvc.utils import as_posix, expand_paths
@@ -158,7 +157,6 @@ def _gather_params(
     return data
 
 
-@locked
 def show(
     repo: "Repo",
     targets: Optional[List[str]] = None,


Test patch: