internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: iterative/dvc

Issue:
dvc repro: Tuples in parameters are always marked as modified
# Bug Report

<!--
## Issue name

Issue names must follow the pattern `command: description` where the command is the dvc command that you are trying to run. The description should describe the consequence of the bug.

Example: `repro: doesn't detect input changes`
-->

## Description

<!--
A clear and concise description of what the bug is.
-->
When tuples are used in parameters, dvc always marks them as modified.

```sh
dvc status
```
outputs
```yaml
train_model:
  changed deps:
    model/parameters.py:    
      modified: NGRAM_VECTORIZER_RANGE
```

regardless of whether `NGRAM_VECTORIZER_RANGE` has been changed



Discussion:
Can reproduce. Taking a look
When serializing the ParamsDependency to the lock file (`dvc.lock`), the tuple gets converted to a list (because `dvc.lock` is in YAML format).

On the subsequent `dvc repro` execution, the value in the params file gets correctly loaded as a tuple but compared against the serialized list (https://github.com/iterative/dvc/blob/main/dvc/dependency/param.py#L127), always reporting it as modified.

Resolving patch:
diff --git a/dvc/dependency/param.py b/dvc/dependency/param.py
--- a/dvc/dependency/param.py
+++ b/dvc/dependency/param.py
@@ -125,9 +125,14 @@ def workspace_status(self):
             elif param not in info:
                 st = "new"
             elif actual[param] != info[param]:
-                st = "modified"
+                if (
+                    isinstance(actual[param], tuple)
+                    and list(actual[param]) == info[param]
+                ):
+                    continue
+                else:
+                    st = "modified"
             else:
-                assert actual[param] == info[param]
                 continue
 
             status[str(self)][param] = st


Test patch:
diff --git a/tests/unit/dependency/test_params.py b/tests/unit/dependency/test_params.py
--- a/tests/unit/dependency/test_params.py
+++ b/tests/unit/dependency/test_params.py
@@ -207,6 +207,22 @@ def test_read_params_py(tmp_dir, dvc):
     assert dep.read_params() == {"x": 4, "Klass.a": "val1"}
 
 
+def test_params_py_tuple_status(tmp_dir, dvc):
+    """https://github.com/iterative/dvc/issues/8803"""
+    parameters_file = "parameters.py"
+    tmp_dir.gen(parameters_file, "TUPLE = (10, 100)\n")
+    dep = ParamsDependency(Stage(dvc), parameters_file, ["TUPLE"])
+    # lock file uses YAML so the tuple will be loaded as a list
+    dep.fill_values({"TUPLE": [10, 100]})
+    assert dep.status() == {}
+    dep.fill_values({"TUPLE": [11, 100]})
+    assert dep.status() == {"parameters.py": {"TUPLE": "modified"}}
+    dep.fill_values({"TUPLE": [10]})
+    assert dep.status() == {"parameters.py": {"TUPLE": "modified"}}
+    dep.fill_values({"TUPLE": {10: "foo", 100: "bar"}})
+    assert dep.status() == {"parameters.py": {"TUPLE": "modified"}}
+
+
 def test_get_hash_missing_config(dvc):
     dep = ParamsDependency(Stage(dvc), None, ["foo"])
     with pytest.raises(MissingParamsError):
Repository: iterative/dvc

Issue:
exp: improve ui for empty git repos
`dvc exp` commands should fail fast and provide a meaningful error when the git repo is empty. Related to https://github.com/iterative/vscode-dvc/issues/3057.


Discussion:

Resolving patch:
diff --git a/dvc/repo/experiments/__init__.py b/dvc/repo/experiments/__init__.py
--- a/dvc/repo/experiments/__init__.py
+++ b/dvc/repo/experiments/__init__.py
@@ -65,6 +65,11 @@ def __init__(self, repo):
 
     @property
     def scm(self):
+        from dvc.scm import SCMError
+
+        if self.repo.scm.no_commits:
+            raise SCMError("Empty Git repo. Add a commit to use experiments.")
+
         return self.repo.scm
 
     @cached_property


Test patch:
diff --git a/tests/func/experiments/test_diff.py b/tests/func/experiments/test_diff.py
--- a/tests/func/experiments/test_diff.py
+++ b/tests/func/experiments/test_diff.py
@@ -1,16 +1,6 @@
 from funcy import first
 
 
-def test_diff_no_commits(tmp_dir):
-    from dvc.repo import Repo
-    from dvc.scm import Git
-
-    git = Git.init(tmp_dir.fs_path)
-    assert git.no_commits
-
-    assert Repo.init().experiments.diff() == {}
-
-
 def test_diff_empty(tmp_dir, scm, dvc, exp_stage):
     assert dvc.experiments.diff() == {"params": {}, "metrics": {}}
 
diff --git a/tests/func/experiments/test_experiments.py b/tests/func/experiments/test_experiments.py
--- a/tests/func/experiments/test_experiments.py
+++ b/tests/func/experiments/test_experiments.py
@@ -13,7 +13,7 @@
 from dvc.repo.experiments.queue.base import BaseStashQueue
 from dvc.repo.experiments.refs import CELERY_STASH
 from dvc.repo.experiments.utils import exp_refs_by_rev
-from dvc.scm import resolve_rev
+from dvc.scm import SCMError, resolve_rev
 from dvc.stage.exceptions import StageFileDoesNotExistError
 from dvc.utils.serialize import PythonFileCorruptedError
 from tests.scripts import COPY_SCRIPT
@@ -646,3 +646,20 @@ def test_clean(tmp_dir, scm, dvc, mocker):
     clean = mocker.spy(dvc.experiments.celery_queue.celery, "clean")
     dvc.experiments.clean()
     clean.assert_called_once_with()
+
+
+def test_experiment_no_commit(tmp_dir):
+    from scmrepo.git import Git
+
+    from dvc.repo import Repo
+
+    Git.init(tmp_dir.fs_path).close()
+
+    repo = Repo.init()
+    assert repo.scm.no_commits
+
+    try:
+        with pytest.raises(SCMError):  # noqa: PT011
+            repo.experiments.ls()
+    finally:
+        repo.close()
diff --git a/tests/func/experiments/test_show.py b/tests/func/experiments/test_show.py
--- a/tests/func/experiments/test_show.py
+++ b/tests/func/experiments/test_show.py
@@ -41,17 +41,6 @@ def make_executor_info(**kwargs):
     return ExecutorInfo(**kwargs)
 
 
-@pytest.mark.vscode
-def test_show_no_commits(tmp_dir):
-    from dvc.repo import Repo
-    from dvc.scm import Git
-
-    git = Git.init(tmp_dir.fs_path)
-    assert git.no_commits
-
-    assert Repo.init().experiments.show() == {}
-
-
 @pytest.mark.vscode
 def test_show_branch_and_tag_name(tmp_dir, scm, dvc, exp_stage):
     with tmp_dir.branch("new/branch", new=True):
Repository: iterative/dvc

Issue:
Custom header for dvc-webdav
This refers to a [dvc-webdav feature request](https://github.com/iterative/dvc-webdav/issues/19). The feature is support for custom auth headers for dvc webdav. I have been in contact with @skshetry will add a related pull request for dvc shortly. Thanks a lot for your support!

fyi @conrad-stork


Discussion:

Resolving patch:
diff --git a/dvc/config_schema.py b/dvc/config_schema.py
--- a/dvc/config_schema.py
+++ b/dvc/config_schema.py
@@ -106,6 +106,7 @@ class RelPath(str):
     "password": str,
     "ask_password": Bool,
     "token": str,
+    "custom_auth_header": str,
     "cert_path": str,
     "key_path": str,
     "timeout": Coerce(int),


Test patch:
diff --git a/tests/unit/remote/test_webdav.py b/tests/unit/remote/test_webdav.py
--- a/tests/unit/remote/test_webdav.py
+++ b/tests/unit/remote/test_webdav.py
@@ -9,6 +9,7 @@
 user = "username"
 password = "password"
 token = "4MgjsNM5aSJjxIKM"
+custom_auth_header = "Custom-Header"
 
 
 def test_common():
@@ -23,6 +24,7 @@ def test_common():
         password=None,
         ask_password=False,
         token=None,
+        custom_auth_header=None,
     )
     assert issubset(
         {
@@ -87,6 +89,46 @@ def test_ask_password(mocker):
     ask_password_mocked.assert_called_once_with(host, user)
 
 
+def test_custom_auth_header():
+    config = {
+        "url": url,
+        "custom_auth_header": custom_auth_header,
+        "password": password,
+    }
+    fs = WebDAVFileSystem(**config)
+    assert issubset(
+        {"headers": {custom_auth_header: password}, "auth": None},
+        fs.fs_args,
+    )
+
+
+def test_ask_password_custom_auth_header(mocker):
+    ask_password_mocked = mocker.patch(
+        "dvc_webdav.ask_password", return_value="pass"
+    )
+    host = "host"
+
+    # it should not ask for password as password is set
+    config = {
+        "url": url,
+        "custom_auth_header": custom_auth_header,
+        "password": password,
+        "ask_password": True,
+        "host": host,
+    }
+    fs = WebDAVFileSystem(**config)
+    assert issubset(
+        {"headers": {custom_auth_header: password}, "auth": None}, fs.fs_args
+    )
+
+    config.pop("password")
+    fs = WebDAVFileSystem(**config)
+    assert issubset(
+        {"headers": {custom_auth_header: "pass"}, "auth": None}, fs.fs_args
+    )
+    ask_password_mocked.assert_called_once_with(host, custom_auth_header)
+
+
 def test_ssl_verify_custom_cert():
     config = {
         "url": url,
Repository: iterative/dvc

Issue:
studio: Send `metrics` on `done` event.
Allow DVC to send metrics updates outside the stage where DVCLive is used. 

For example, the following metrics are not currently present in the "live experiment" row:

https://github.com/iterative/example-get-started-experiments/blob/982941e239db6a7628ab9d327e7424fdef09429e/dvc.yaml#L38-L40

Requires https://github.com/iterative/dvc-studio-client/pull/10


Discussion:

Resolving patch:
diff --git a/dvc/repo/experiments/executor/base.py b/dvc/repo/experiments/executor/base.py
--- a/dvc/repo/experiments/executor/base.py
+++ b/dvc/repo/experiments/executor/base.py
@@ -21,6 +21,7 @@
     Union,
 )
 
+from funcy import get_in
 from scmrepo.exceptions import SCMError
 
 from dvc.env import DVC_EXP_AUTO_PUSH, DVC_EXP_GIT_REMOTE
@@ -590,6 +591,7 @@ def _repro_dvc(  # noqa: C901
                     info.name,
                     "dvc",
                     experiment_rev=dvc.experiments.scm.get_ref(EXEC_BRANCH),
+                    metrics=get_in(dvc.metrics.show(), ["", "data"]),
                 )
 
                 if infofile is not None:


Test patch:
diff --git a/tests/integration/test_studio_live_experiments.py b/tests/integration/test_studio_live_experiments.py
--- a/tests/integration/test_studio_live_experiments.py
+++ b/tests/integration/test_studio_live_experiments.py
@@ -38,4 +38,5 @@ def test_post_to_studio(tmp_dir, dvc, scm, exp_stage, mocker, monkeypatch, tmp):
         "name": name,
         "client": "dvc",
         "experiment_rev": exp_rev,
+        "metrics": {"metrics.yaml": {"data": {"foo": 1}}},
     }
Repository: iterative/dvc

Issue:
data status doesn't use dvcignore
`dvc diff` is respecting the `.dvcignore` but not `dvc data status --granular`. 

_Originally posted by @yathomasi in https://github.com/iterative/vscode-dvc/issues/3237#issuecomment-1425238480_
            


Discussion:

Resolving patch:
diff --git a/dvc/repo/index.py b/dvc/repo/index.py
--- a/dvc/repo/index.py
+++ b/dvc/repo/index.py
@@ -589,6 +589,10 @@ def build_data_index(
     from dvc_data.index.build import build_entries, build_entry
     from dvc_data.index.save import build_tree
 
+    ignore = None
+    if workspace == "repo" and isinstance(fs, LocalFileSystem):
+        ignore = index.repo.dvcignore
+
     data = DataIndex()
     for key in index.data_keys.get(workspace, set()):
         out_path = fs.path.join(path, *key)
@@ -610,7 +614,11 @@ def build_data_index(
             continue
 
         for entry in build_entries(
-            out_path, fs, compute_hash=compute_hash, state=index.repo.state
+            out_path,
+            fs,
+            compute_hash=compute_hash,
+            state=index.repo.state,
+            ignore=ignore,
         ):
             if not entry.key or entry.key == ("",):
                 # NOTE: whether the root will be returned by build_entries


Test patch:
Repository: iterative/dvc

Issue:
exp save: Incorrect warnings about untracked files
The current check:

https://github.com/iterative/dvc/blob/6fbbbeccc530125e21aa5c9555f4b9ada7002b59/dvc/repo/experiments/save.py#L61-L62

Logs warnings when there is not an exact string match between the untracked in `scm.status()` and any item in `include_untracked`.

This causes incorrect warnings in scenarios like passing a folder to `--include-untracked`:

```console
$ git init
$ dvc init
$ git commit -m "init"
$ mkdir data
$ touch data/foo
$ touch data/bar
$ dvc exp save --include-untracked data
WARNING: The following untracked files were present in the workspace before saving but will not be included in the experiment commit:
	data/foo, data/bar
Experiment has been saved as: paler-tams                              

To promote an experiment to a Git branch run:

	dvc exp branch <exp> <branch>
```

Where `data/foo` and `data/bar` are actually included in the experiment:

```console
$ git show f312d6d
commit f312d6d00ab0ff7d6bb1d2d0fb8eca12e2a8ffb0 (refs/exps/2e/95a6fc1d0347b35f21d03051555f55bd4cf2af/paler-tams)
Author: David de la Iglesia Castro <daviddelaiglesiacastro@gmail.com>
Date:   Mon Feb 20 15:15:31 2023 +0100

    dvc: commit experiment paler-tams

diff --git a/data/bar b/data/bar
new file mode 100644
index 0000000..e69de29
diff --git a/data/foo b/data/foo
new file mode 100644
index 0000000..e69de29
```



Discussion:

Resolving patch:
diff --git a/dvc/repo/experiments/save.py b/dvc/repo/experiments/save.py
--- a/dvc/repo/experiments/save.py
+++ b/dvc/repo/experiments/save.py
@@ -2,6 +2,8 @@
 import os
 from typing import TYPE_CHECKING, List, Optional
 
+from pathspec import PathSpec
+
 from dvc.scm import Git
 
 from .exceptions import ExperimentExistsError
@@ -59,7 +61,8 @@ def save(
 
     _, _, untracked = repo.scm.status()
     if include_untracked:
-        untracked = [file for file in untracked if file not in include_untracked]
+        spec = PathSpec.from_lines("gitwildmatch", include_untracked)
+        untracked = [file for file in untracked if not spec.match_file(file)]
     if untracked:
         logger.warning(
             (


Test patch:
diff --git a/tests/func/experiments/test_save.py b/tests/func/experiments/test_save.py
--- a/tests/func/experiments/test_save.py
+++ b/tests/func/experiments/test_save.py
@@ -1,3 +1,5 @@
+import logging
+
 import pytest
 from funcy import first
 
@@ -91,3 +93,15 @@ def test_exp_save_include_untracked(tmp_dir, dvc, scm, exp_stage):
 
     dvc.experiments.apply("exp-0")
     assert new_file.read_text() == "exp-0"
+
+
+def test_exp_save_include_untracked_warning(tmp_dir, dvc, scm, caplog, exp_stage):
+    """Regression test for https://github.com/iterative/dvc/issues/9061"""
+    new_dir = tmp_dir / "new_dir"
+    new_dir.mkdir()
+    (new_dir / "foo").write_text("foo")
+    (new_dir / "bar").write_text("bar")
+
+    with caplog.at_level(logging.WARNING, logger="dvc.repo.experiments.save"):
+        dvc.experiments.save(name="exp", include_untracked=["new_dir"])
+        assert not caplog.records
Repository: iterative/dvc

Issue:
exp push/pull: don't include baseline commit
`dvc exp push/pull` transfer all cached outputs for all experiment commits up to and including the baseline commit on which the experiment is based. The baseline commit is a regular Git commit, so it might be unexpected that it's data is pushed/pulled by `exp push/pull`, and if no `dvc.lock` exists in the baseline commit, this will generate a warning that could be confusing.

@iesahin @pmrowla @daavoo 


Discussion:
This should be simple to fix, we just need to remove
https://github.com/iterative/dvc/blob/049a856cd93c14e6a2efdbf6f5e02b028885a544/dvc/repo/experiments/utils.py#L90-L91

Resolving patch:
diff --git a/dvc/repo/experiments/utils.py b/dvc/repo/experiments/utils.py
--- a/dvc/repo/experiments/utils.py
+++ b/dvc/repo/experiments/utils.py
@@ -215,8 +215,6 @@ def exp_commits(
     refs = ref_infos if ref_infos else exp_refs(scm)
     for ref_info in refs:
         shas.update(scm.branch_revs(str(ref_info), ref_info.baseline_sha))
-        if ref_info.baseline_sha:
-            shas.add(ref_info.baseline_sha)
     yield from shas
 
 


Test patch:
diff --git a/tests/func/test_gc.py b/tests/func/test_gc.py
--- a/tests/func/test_gc.py
+++ b/tests/func/test_gc.py
@@ -276,8 +276,7 @@ def test_gc_all_experiments(tmp_dir, scm, dvc):
     (foo,) = tmp_dir.dvc_gen("foo", "foo", commit="foo")
     foo_hash = foo.outs[0].hash_info.value
 
-    (bar,) = tmp_dir.dvc_gen("foo", "bar", commit="bar")
-    bar_hash = bar.outs[0].hash_info.value
+    tmp_dir.dvc_gen("foo", "bar", commit="bar")
     baseline = scm.get_rev()
 
     (baz,) = tmp_dir.dvc_gen("foo", "baz", commit="baz")
@@ -288,12 +287,7 @@ def test_gc_all_experiments(tmp_dir, scm, dvc):
 
     dvc.gc(all_experiments=True, force=True)
 
-    # all_experiments will include the experiment commit (baz) plus baseline
-    # commit (bar)
     assert not (tmp_dir / ".dvc" / "cache" / foo_hash[:2] / foo_hash[2:]).exists()
-    assert (
-        tmp_dir / ".dvc" / "cache" / bar_hash[:2] / bar_hash[2:]
-    ).read_text() == "bar"
     assert (
         tmp_dir / ".dvc" / "cache" / baz_hash[:2] / baz_hash[2:]
     ).read_text() == "baz"
diff --git a/tests/unit/scm/test_scm.py b/tests/unit/scm/test_scm.py
--- a/tests/unit/scm/test_scm.py
+++ b/tests/unit/scm/test_scm.py
@@ -64,7 +64,6 @@ def test_iter_revs(
     assert gen == {
         rev_new: [rev_new],
         rev_old: [rev_old],
-        rev_root: [rev_root],
     }
 
     def _resolve_commit(rev):
Repository: iterative/dvc

Issue:
Invalid permisions of /var/tmp/dvc/repo
# Bug Report

## dvc ls: creates `/var/tmp/dvc` with incorrect permissions

## Description

Since https://github.com/iterative/dvc/releases/tag/2.48.0 the dvc is using `/var/tmp/dvc` as a "git cache" for unix based operating systems.
When the `/var/tmp/dvc` does not exist the dvc creates site cache folder using `os.makedirs` with default `777` permissions, but these permissions are masked by UMASK (https://stackoverflow.com/a/5231994).
This can cause an unexpected behavior of `dvc ls` (an other commands) for multi-user environment where other users can not write to originally created site-cache folder 

https://github.com/iterative/dvc/blob/612f3039e05b609d02d96c063763f8c093c2779d/dvc/repo/__init__.py#L212

### Reproduce

1) remove site cache dir `rm -r -f /var/tmp/dvc`
2) run `dvc ls dvc ls git@github.com:iterative/dvc-test.git`
3) check permissions `ls -al /var/tmp/dvc` or run 2) as a different user

### Expected

I believe that the site cache dir should be created with permissions that allows other users to list dvc repositories.

### Environment information

Centos8

```
╰─❯ dvc doctor
DVC version: 2.48.0 (pip)
-------------------------
Platform: Python 3.8.13 on Linux-4.18.0-408.el8.x86_64-x86_64-with-glibc2.2.5
Subprojects:
	dvc_data = 0.43.0
	dvc_objects = 0.21.1
	dvc_render = 0.2.0
	dvc_task = 0.2.0
	scmrepo = 0.1.15
Supports:
	http (aiohttp = 3.8.1, aiohttp-retry = 2.8.3),
	https (aiohttp = 3.8.1, aiohttp-retry = 2.8.3),
	s3 (s3fs = 2023.1.0, boto3 = 1.24.59)
Cache types: reflink, hardlink, symlink
Cache directory: xfs on /dev/md0
Caches: local
Remotes: s3
Workspace directory: xfs on /dev/md0
Repo: dvc, git
Repo.site_cache_dir: /var/tmp/dvc/repo/8e5a699173beb5035173f8252c5bcbec

```



Discussion:

Resolving patch:
diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py
--- a/dvc/repo/__init__.py
+++ b/dvc/repo/__init__.py
@@ -572,9 +572,17 @@ def site_cache_dir(self) -> str:
         else:
             root_dir = self.root_dir
 
+        repos_dir = os.path.join(cache_dir, "repo")
+
+        umask = os.umask(0)
+        try:
+            os.makedirs(repos_dir, mode=0o777, exist_ok=True)
+        finally:
+            os.umask(umask)
+
         repo_token = hashlib.md5(os.fsencode(root_dir)).hexdigest()  # noqa: S324
 
-        return os.path.join(cache_dir, "repo", repo_token)
+        return os.path.join(repos_dir, repo_token)
 
     @contextmanager
     def open_by_relpath(self, path, remote=None, mode="r", encoding=None):


Test patch:
Repository: iterative/dvc

Issue:
exp save: Include untracked `dvc.lock`
              Can't remember if it's a new issue or whether this should be a separate follow-up, but an untracked `dvc.lock` is not being included automatically:

```
$ dvc exp save
WARNING: The following untracked files were present in the workspace before saving but will not be included in the experiment commit:
        dvc.lock
```


Discussion:

Resolving patch:
diff --git a/dvc/repo/experiments/executor/base.py b/dvc/repo/experiments/executor/base.py
--- a/dvc/repo/experiments/executor/base.py
+++ b/dvc/repo/experiments/executor/base.py
@@ -265,6 +265,7 @@ def save(
         force: bool = False,
         include_untracked: Optional[List[str]] = None,
     ) -> ExecutorResult:
+        from dvc.dvcfile import LOCK_FILE
         from dvc.repo import Repo
 
         exp_hash: Optional[str] = None
@@ -283,6 +284,10 @@ def save(
         include_untracked.extend(
             dvc.index._plot_sources  # pylint: disable=protected-access
         )
+        # dvc repro automatically stages dvc.lock. Running redundant `git add`
+        # on it causes an error when exiting the detached head context.
+        if LOCK_FILE in dvc.scm.untracked_files():
+            include_untracked.append(LOCK_FILE)
 
         try:
             stages = dvc.commit([], force=True, relink=False)


Test patch:
diff --git a/tests/func/experiments/test_save.py b/tests/func/experiments/test_save.py
--- a/tests/func/experiments/test_save.py
+++ b/tests/func/experiments/test_save.py
@@ -134,5 +134,22 @@ def test_untracked_top_level_files_are_included_in_exp(tmp_dir, scm, dvc):
     dvc.reproduce(stage.addressing)
     exp = dvc.experiments.save()
     fs = scm.get_fs(exp)
-    for file in ["metrics.json", "params.yaml", "plots.csv"]:
+    for file in ["metrics.json", "params.yaml", "plots.csv", "dvc.lock"]:
         assert fs.exists(file)
+
+
+def test_untracked_dvclock_is_included_in_exp(tmp_dir, scm, dvc):
+    stage = dvc.stage.add(
+        cmd="echo foo",
+        name="foo",
+    )
+    scm.add_commit(["dvc.yaml"], message="add dvc.yaml")
+    dvc.reproduce(stage.addressing)
+
+    # dvc.reproduce automatically stages `dvc.lock`
+    # force it to be untracked
+    scm.reset()
+
+    exp = dvc.experiments.save()
+    fs = scm.get_fs(exp)
+    assert fs.exists("dvc.lock")