Repository: iterative/dvc
Pull request #9790 (2023-08-02 13:04:08)

ruff: enable autofix
Let's enable autofix in ruff as an experiment.

If it breaks code too often or gets too aggressive, we can rollback or set `unfixable` in the pyproject.toml file as follows:

```toml
[tool.ruff]
unfixable = ["B", "SIM", "TRY", "RUF"]
```

codecov[bot]: ## [Codecov](https://app.codecov.io/gh/iterative/dvc/pull/9790?src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=iterative) Report
Patch coverage has no change and project coverage change: **`-0.43%`** :warning:
> Comparison is base [(`502b3df`)](https://app.codecov.io/gh/iterative/dvc/commit/502b3df6c995cff889b23979e62741add4c136ea?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=iterative) 90.46% compared to head [(`6c02e91`)](https://app.codecov.io/gh/iterative/dvc/pull/9790?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=iterative) 90.03%.

<details><summary>Additional details and impacted files</summary>


```diff
@@            Coverage Diff             @@
##             main    #9790      +/-   ##
==========================================
- Coverage   90.46%   90.03%   -0.43%     
==========================================
  Files         480      480              
  Lines       36615    36615              
  Branches     5277     4313     -964     
==========================================
- Hits        33124    32967     -157     
- Misses       2892     3005     +113     
- Partials      599      643      +44     
```


[see 42 files with indirect coverage changes](https://app.codecov.io/gh/iterative/dvc/pull/9790/indirect-changes?src=pr&el=tree-more&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=iterative)


</details>

[:umbrella: View full report in Codecov by Sentry](https://app.codecov.io/gh/iterative/dvc/pull/9790?src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=iterative).   
:loudspeaker: Have feedback on the report? [Share it here](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=iterative).


Patch:
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 362bc4824..305837a4c 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -35,6 +35,7 @@ repos:
     rev: 'v0.0.282'
     hooks:
       - id: ruff
+        args: [--fix, --exit-non-zero-on-fix]
   - repo: https://github.com/codespell-project/codespell
     rev: v2.2.5
     hooks:
diff --git a/dvc/cli/utils.py b/dvc/cli/utils.py
index 6b794d7b2..1dfd83536 100644
--- a/dvc/cli/utils.py
+++ b/dvc/cli/utils.py
@@ -1,3 +1,27 @@
+import argparse
+
+
+class DictAction(argparse.Action):
+    def __call__(self, parser, args, values, option_string=None):  # noqa: ARG002
+        d = getattr(args, self.dest) or {}
+
+        if isinstance(values, list):
+            kvs = values
+        else:
+            kvs = [values]
+
+        for kv in kvs:
+            key, value = kv.split("=")
+            if not value:
+                raise argparse.ArgumentError(
+                    self,
+                    f'Could not parse argument "{values}" as k1=v1 k2=v2 ... format',
+                )
+            d[key] = value
+
+        setattr(args, self.dest, d)
+
+
 def fix_subparsers(subparsers):
     """Workaround for bug in Python 3. See more info at:
     https://bugs.python.org/issue16308
diff --git a/dvc/commands/get.py b/dvc/commands/get.py
index 016a23415..c1c4ad2d6 100644
--- a/dvc/commands/get.py
+++ b/dvc/commands/get.py
@@ -3,7 +3,7 @@ import logging
 
 from dvc.cli import completion
 from dvc.cli.command import CmdBaseNoRepo
-from dvc.cli.utils import append_doc_link
+from dvc.cli.utils import DictAction, append_doc_link
 from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)
@@ -38,6 +38,8 @@ class CmdGet(CmdBaseNoRepo):
                 jobs=self.args.jobs,
                 force=self.args.force,
                 config=self.args.config,
+                remote=self.args.remote,
+                remote_config=self.args.remote_config,
             )
             return 0
         except CloneError:
@@ -111,4 +113,19 @@ def add_parser(subparsers, parent_parser):
             "in the target repository."
         ),
     )
+    get_parser.add_argument(
+        "--remote",
+        type=str,
+        help="Remote name to set as a default in the target repository.",
+    )
+    get_parser.add_argument(
+        "--remote-config",
+        type=str,
+        nargs="*",
+        action=DictAction,
+        help=(
+            "Remote config options to merge with a remote's config (default or one "
+            "specified by '--remote') in the target repository."
+        ),
+    )
     get_parser.set_defaults(func=CmdGet)
diff --git a/dvc/commands/imp.py b/dvc/commands/imp.py
index 217df87eb..51874a76b 100644
--- a/dvc/commands/imp.py
+++ b/dvc/commands/imp.py
@@ -3,7 +3,7 @@ import logging
 
 from dvc.cli import completion
 from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link
+from dvc.cli.utils import DictAction, append_doc_link
 from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)
@@ -23,6 +23,8 @@ class CmdImport(CmdBase):
                 no_download=self.args.no_download,
                 jobs=self.args.jobs,
                 config=self.args.config,
+                remote=self.args.remote,
+                remote_config=self.args.remote_config,
             )
         except CloneError:
             logger.exception("failed to import '%s'", self.args.path)
@@ -103,4 +105,19 @@ def add_parser(subparsers, parent_parser):
             "in the target repository."
         ),
     )
+    import_parser.add_argument(
+        "--remote",
+        type=str,
+        help="Remote name to set as a default in the target repository.",
+    )
+    import_parser.add_argument(
+        "--remote-config",
+        type=str,
+        nargs="*",
+        action=DictAction,
+        help=(
+            "Remote config options to merge with a remote's config (default or one "
+            "specified by '--remote') in the target repository."
+        ),
+    )
     import_parser.set_defaults(func=CmdImport)
diff --git a/dvc/commands/ls/__init__.py b/dvc/commands/ls/__init__.py
index ecb345064..c044e3571 100644
--- a/dvc/commands/ls/__init__.py
+++ b/dvc/commands/ls/__init__.py
@@ -3,7 +3,7 @@ import logging
 
 from dvc.cli import completion
 from dvc.cli.command import CmdBaseNoRepo
-from dvc.cli.utils import append_doc_link
+from dvc.cli.utils import DictAction, append_doc_link
 from dvc.commands.ls.ls_colors import LsColors
 from dvc.exceptions import DvcException
 from dvc.ui import ui
@@ -35,6 +35,8 @@ class CmdList(CmdBaseNoRepo):
                 recursive=self.args.recursive,
                 dvc_only=self.args.dvc_only,
                 config=self.args.config,
+                remote=self.args.remote,
+                remote_config=self.args.remote_config,
             )
             if self.args.json:
                 ui.write_json(entries)
@@ -89,6 +91,21 @@ def add_parser(subparsers, parent_parser):
             "in the target repository."
         ),
     )
+    list_parser.add_argument(
+        "--remote",
+        type=str,
+        help="Remote name to set as a default in the target repository.",
+    )
+    list_parser.add_argument(
+        "--remote-config",
+        type=str,
+        nargs="*",
+        action=DictAction,
+        help=(
+            "Remote config options to merge with a remote's config (default or one "
+            "specified by '--remote') in the target repository."
+        ),
+    )
     list_parser.add_argument(
         "path",
         nargs="?",
diff --git a/dvc/dependency/repo.py b/dvc/dependency/repo.py
index 7c6fbb43b..dfe6a8b72 100644
--- a/dvc/dependency/repo.py
+++ b/dvc/dependency/repo.py
@@ -1,6 +1,6 @@
 from typing import TYPE_CHECKING, Dict, Optional, Union
 
-from voluptuous import Required
+from voluptuous import Any, Required
 
 from dvc.utils import as_posix
 
@@ -17,6 +17,7 @@ class RepoDependency(Dependency):
     PARAM_REV = "rev"
     PARAM_REV_LOCK = "rev_lock"
     PARAM_CONFIG = "config"
+    PARAM_REMOTE = "remote"
 
     REPO_SCHEMA = {
         PARAM_REPO: {
@@ -24,6 +25,7 @@ class RepoDependency(Dependency):
             PARAM_REV: str,
             PARAM_REV_LOCK: str,
             PARAM_CONFIG: str,
+            PARAM_REMOTE: Any(str, dict),
         }
     }
 
@@ -76,6 +78,10 @@ class RepoDependency(Dependency):
         if config:
             repo[self.PARAM_CONFIG] = config
 
+        remote = self.def_repo.get(self.PARAM_REMOTE)
+        if remote:
+            repo[self.PARAM_REMOTE] = remote
+
         return {
             self.PARAM_PATH: self.def_path,
             self.PARAM_REPO: repo,
@@ -99,6 +105,14 @@ class RepoDependency(Dependency):
         from dvc.config import Config
         from dvc.fs import DVCFileSystem
 
+        rem = self.def_repo.get("remote")
+        if isinstance(rem, dict):
+            remote = None  # type: ignore[unreachable]
+            remote_config = rem
+        else:
+            remote = rem
+            remote_config = None
+
         conf = self.def_repo.get("config")
         if conf:
             config = Config.load_file(conf)
@@ -113,6 +127,8 @@ class RepoDependency(Dependency):
             rev=rev or self._get_rev(locked=locked),
             subrepos=True,
             config=config,
+            remote=remote,
+            remote_config=remote_config,
         )
 
     def _get_rev(self, locked: bool = True):
diff --git a/dvc/repo/get.py b/dvc/repo/get.py
index 239a08318..4617e0cd3 100644
--- a/dvc/repo/get.py
+++ b/dvc/repo/get.py
@@ -19,7 +19,17 @@ class GetDVCFileError(DvcException):
         )
 
 
-def get(url, path, out=None, rev=None, jobs=None, force=False, config=None):
+def get(
+    url,
+    path,
+    out=None,
+    rev=None,
+    jobs=None,
+    force=False,
+    config=None,
+    remote=None,
+    remote_config=None,
+):
     from dvc.config import Config
     from dvc.dvcfile import is_valid_filename
     from dvc.repo import Repo
@@ -38,6 +48,8 @@ def get(url, path, out=None, rev=None, jobs=None, force=False, config=None):
         subrepos=True,
         uninitialized=True,
         config=config,
+        remote=remote,
+        remote_config=remote_config,
     ) as repo:
         from dvc.fs import download
         from dvc.fs.data import DataFileSystem
diff --git a/dvc/repo/imp.py b/dvc/repo/imp.py
index 558fa56da..8f4e2dcaf 100644
--- a/dvc/repo/imp.py
+++ b/dvc/repo/imp.py
@@ -1,4 +1,14 @@
-def imp(self, url, path, out=None, rev=None, config=None, **kwargs):
+def imp(
+    self,
+    url,
+    path,
+    out=None,
+    rev=None,
+    config=None,
+    remote=None,
+    remote_config=None,
+    **kwargs,
+):
     erepo = {"url": url}
     if rev is not None:
         erepo["rev"] = rev
@@ -6,4 +16,17 @@ def imp(self, url, path, out=None, rev=None, config=None, **kwargs):
     if config is not None:
         erepo["config"] = config
 
+    if remote is not None and remote_config is not None:
+        conf = erepo.get("config") or {}
+        remotes = conf.get("remote") or {}
+        remote_conf = remotes.get(remote) or {}
+        remote_conf.update(remote_config)
+        remotes[remote] = remote_conf
+        conf["remote"] = remotes
+        erepo["config"] = conf
+    elif remote is not None:
+        erepo["remote"] = remote
+    elif remote_config is not None:
+        erepo["remote"] = remote_config
+
     return self.imp_url(path, out=out, erepo=erepo, frozen=True, **kwargs)
diff --git a/dvc/repo/ls.py b/dvc/repo/ls.py
index ce408bbb5..5e6005ea3 100644
--- a/dvc/repo/ls.py
+++ b/dvc/repo/ls.py
@@ -14,6 +14,8 @@ def ls(
     recursive: Optional[bool] = None,
     dvc_only: bool = False,
     config: Optional[str] = None,
+    remote: Optional[str] = None,
+    remote_config: Optional[dict] = None,
 ):
     """Methods for getting files and outputs for the repo.
 
@@ -23,7 +25,9 @@ def ls(
         rev (str, optional): SHA commit, branch or tag name
         recursive (bool, optional): recursively walk the repo
         dvc_only (bool, optional): show only DVC-artifacts
-        config (bool, optional): path to config file
+        config (str, optional): path to config file
+        remote (str, optional): remote name to set as a default remote in the repo
+        remote_config (str, dict): remote config to merge with a remote in the repo
 
     Returns:
         list of `entry`
@@ -47,7 +51,13 @@ def ls(
         config_dict = None
 
     with Repo.open(
-        url, rev=rev, subrepos=True, uninitialized=True, config=config_dict
+        url,
+        rev=rev,
+        subrepos=True,
+        uninitialized=True,
+        config=config_dict,
+        remote=remote,
+        remote_config=remote_config,
     ) as repo:
         path = path or ""
 
diff --git a/pyproject.toml b/pyproject.toml
index 244dd4255..ab6388cb7 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -34,7 +34,7 @@ dependencies = [
     "configobj>=5.0.6",
     "distro>=1.3",
     "dpath<3,>=2.1.0",
-    "dvc-data>=2.8.1,<2.9.0",
+    "dvc-data>=2.9.0,<2.10.0",
     "dvc-http>=2.29.0",
     "dvc-render>=0.3.1,<1",
     "dvc-studio-client>=0.9.2,<1",


Test patch:
diff --git a/tests/unit/command/ls/test_ls.py b/tests/unit/command/ls/test_ls.py
index 1d6702624..884dae137 100644
--- a/tests/unit/command/ls/test_ls.py
+++ b/tests/unit/command/ls/test_ls.py
@@ -19,7 +19,14 @@ def test_list(mocker):
     url = "local_dir"
     m = _test_cli(mocker, url)
     m.assert_called_once_with(
-        url, None, recursive=False, rev=None, dvc_only=False, config=None
+        url,
+        None,
+        recursive=False,
+        rev=None,
+        dvc_only=False,
+        config=None,
+        remote=None,
+        remote_config=None,
     )
 
 
@@ -27,7 +34,14 @@ def test_list_recursive(mocker):
     url = "local_dir"
     m = _test_cli(mocker, url, "-R")
     m.assert_called_once_with(
-        url, None, recursive=True, rev=None, dvc_only=False, config=None
+        url,
+        None,
+        recursive=True,
+        rev=None,
+        dvc_only=False,
+        config=None,
+        remote=None,
+        remote_config=None,
     )
 
 
@@ -35,7 +49,14 @@ def test_list_git_ssh_rev(mocker):
     url = "git@github.com:repo"
     m = _test_cli(mocker, url, "--rev", "123")
     m.assert_called_once_with(
-        url, None, recursive=False, rev="123", dvc_only=False, config=None
+        url,
+        None,
+        recursive=False,
+        rev="123",
+        dvc_only=False,
+        config=None,
+        remote=None,
+        remote_config=None,
     )
 
 
@@ -44,7 +65,14 @@ def test_list_targets(mocker):
     target = "subdir"
     m = _test_cli(mocker, url, target)
     m.assert_called_once_with(
-        url, target, recursive=False, rev=None, dvc_only=False, config=None
+        url,
+        target,
+        recursive=False,
+        rev=None,
+        dvc_only=False,
+        config=None,
+        remote=None,
+        remote_config=None,
     )
 
 
@@ -52,15 +80,40 @@ def test_list_outputs_only(mocker):
     url = "local_dir"
     m = _test_cli(mocker, url, None, "--dvc-only")
     m.assert_called_once_with(
-        url, None, recursive=False, rev=None, dvc_only=True, config=None
+        url,
+        None,
+        recursive=False,
+        rev=None,
+        dvc_only=True,
+        config=None,
+        remote=None,
+        remote_config=None,
     )
 
 
 def test_list_config(mocker):
     url = "local_dir"
-    m = _test_cli(mocker, url, None, "--config", "myconfig")
+    m = _test_cli(
+        mocker,
+        url,
+        None,
+        "--config",
+        "myconfig",
+        "--remote",
+        "myremote",
+        "--remote-config",
+        "k1=v1",
+        "k2=v2",
+    )
     m.assert_called_once_with(
-        url, None, recursive=False, rev=None, dvc_only=False, config="myconfig"
+        url,
+        None,
+        recursive=False,
+        rev=None,
+        dvc_only=False,
+        config="myconfig",
+        remote="myremote",
+        remote_config={"k1": "v1", "k2": "v2"},
     )
 
 
diff --git a/tests/unit/command/test_get.py b/tests/unit/command/test_get.py
index 3e2cfad50..05e77b747 100644
--- a/tests/unit/command/test_get.py
+++ b/tests/unit/command/test_get.py
@@ -16,6 +16,11 @@ def test_get(mocker):
             "4",
             "--config",
             "myconfig",
+            "--remote",
+            "myremote",
+            "--remote-config",
+            "k1=v1",
+            "k2=v2",
         ]
     )
     assert cli_args.func == CmdGet
@@ -33,6 +38,8 @@ def test_get(mocker):
         jobs=4,
         config="myconfig",
         force=False,
+        remote="myremote",
+        remote_config={"k1": "v1", "k2": "v2"},
     )
 
 
diff --git a/tests/unit/command/test_imp.py b/tests/unit/command/test_imp.py
index 83b49d8bd..b6aa9bc7b 100644
--- a/tests/unit/command/test_imp.py
+++ b/tests/unit/command/test_imp.py
@@ -16,6 +16,11 @@ def test_import(mocker, dvc):
             "3",
             "--config",
             "myconfig",
+            "--remote",
+            "myremote",
+            "--remote-config",
+            "k1=v1",
+            "k2=v2",
         ]
     )
     assert cli_args.func == CmdImport
@@ -34,6 +39,8 @@ def test_import(mocker, dvc):
         no_download=False,
         jobs=3,
         config="myconfig",
+        remote="myremote",
+        remote_config={"k1": "v1", "k2": "v2"},
     )
 
 
@@ -65,6 +72,8 @@ def test_import_no_exec(mocker, dvc):
         no_download=False,
         jobs=None,
         config=None,
+        remote=None,
+        remote_config=None,
     )
 
 
@@ -96,4 +105,6 @@ def test_import_no_download(mocker, dvc):
         no_download=True,
         jobs=None,
         config=None,
+        remote=None,
+        remote_config=None,
     )