Repository: iterative/dvc

Issue:
repro: use build cache for deterministic stages
In my experiment I run a few different preprocessing steps which create a different CSV file, then I am modeling this data and also checking different parameters.
When I want to run the same experiment on 4 different machines (`dvc` is connected to the same remote cache).
Running every type of preprocessing will be done on every machine which takes a lot of time and could be omitted by running `dvc pull` before `dvc repro` and `dvc push` after it. 
It could work with one command like `dvc repro --remote`

Discussion:

Patch:
diff --git a/dvc/command/update.py b/dvc/command/update.py
index c3ef60ec9..718f08cda 100644
--- a/dvc/command/update.py
+++ b/dvc/command/update.py
@@ -10,12 +10,15 @@ logger = logging.getLogger(__name__)
 class CmdUpdate(CmdBase):
     def run(self):
         ret = 0
-        for target in self.args.targets:
-            try:
-                self.repo.update(target, self.args.rev)
-            except DvcException:
-                logger.exception("failed to update '{}'.".format(target))
-                ret = 1
+        try:
+            self.repo.update(
+                targets=self.args.targets,
+                rev=self.args.rev,
+                recursive=self.args.recursive,
+            )
+        except DvcException:
+            logger.exception("failed update data")
+            ret = 1
         return ret
 
 
@@ -37,4 +40,11 @@ def add_parser(subparsers, parent_parser):
         help="Git revision (e.g. SHA, branch, tag)",
         metavar="<commit>",
     )
+    update_parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        default=False,
+        help="Update all stages in the specified directory.",
+    )
     update_parser.set_defaults(func=CmdUpdate)
diff --git a/dvc/data_cloud.py b/dvc/data_cloud.py
index 0fceabb4a..0b594f182 100644
--- a/dvc/data_cloud.py
+++ b/dvc/data_cloud.py
@@ -48,12 +48,7 @@ class DataCloud(object):
         return Remote(self.repo, name=name)
 
     def push(
-        self,
-        cache,
-        jobs=None,
-        remote=None,
-        show_checksums=False,
-        run_cache=False,
+        self, cache, jobs=None, remote=None, show_checksums=False,
     ):
         """Push data items in a cloud-agnostic way.
 
@@ -67,20 +62,12 @@ class DataCloud(object):
         """
         remote = self.get_remote(remote, "push")
 
-        if run_cache:
-            self.repo.stage_cache.push(remote)
-
         return self.repo.cache.local.push(
             cache, jobs=jobs, remote=remote, show_checksums=show_checksums,
         )
 
     def pull(
-        self,
-        cache,
-        jobs=None,
-        remote=None,
-        show_checksums=False,
-        run_cache=False,
+        self, cache, jobs=None, remote=None, show_checksums=False,
     ):
         """Pull data items in a cloud-agnostic way.
 
@@ -94,9 +81,6 @@ class DataCloud(object):
         """
         remote = self.get_remote(remote, "pull")
 
-        if run_cache:
-            self.repo.stage_cache.pull(remote)
-
         downloaded_items_num = self.repo.cache.local.pull(
             cache, jobs=jobs, remote=remote, show_checksums=show_checksums
         )
diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py
index 88618cf49..f90a22e0c 100644
--- a/dvc/repo/__init__.py
+++ b/dvc/repo/__init__.py
@@ -248,6 +248,7 @@ class Repo(object):
         force=False,
         jobs=None,
         recursive=False,
+        used_run_cache=None,
     ):
         """Get the stages related to the given target and collect
         the `info` of its outputs.
@@ -291,6 +292,12 @@ class Repo(object):
                 )
                 cache.update(used_cache, suffix=suffix)
 
+        if used_run_cache:
+            used_cache = self.stage_cache.get_used_cache(
+                used_run_cache, remote=remote, force=force, jobs=jobs,
+            )
+            cache.update(used_cache)
+
         return cache
 
     def _collect_graph(self, stages):
diff --git a/dvc/repo/fetch.py b/dvc/repo/fetch.py
index efbd93beb..cf2a01b76 100644
--- a/dvc/repo/fetch.py
+++ b/dvc/repo/fetch.py
@@ -34,6 +34,9 @@ def _fetch(
         config.NoRemoteError: thrown when downloading only local files and no
             remote is configured
     """
+
+    used_run_cache = self.stage_cache.pull(remote) if run_cache else []
+
     used = self.used_cache(
         targets,
         all_branches=all_branches,
@@ -44,6 +47,7 @@ def _fetch(
         remote=remote,
         jobs=jobs,
         recursive=recursive,
+        used_run_cache=used_run_cache,
     )
 
     downloaded = 0
@@ -51,11 +55,7 @@ def _fetch(
 
     try:
         downloaded += self.cloud.pull(
-            used,
-            jobs,
-            remote=remote,
-            show_checksums=show_checksums,
-            run_cache=run_cache,
+            used, jobs, remote=remote, show_checksums=show_checksums,
         )
     except NoRemoteError:
         if not used.external and used["local"]:
diff --git a/dvc/repo/push.py b/dvc/repo/push.py
index b98714338..81e07ce6a 100644
--- a/dvc/repo/push.py
+++ b/dvc/repo/push.py
@@ -14,6 +14,8 @@ def push(
     all_commits=False,
     run_cache=False,
 ):
+    used_run_cache = self.stage_cache.push(remote) if run_cache else []
+
     used = self.used_cache(
         targets,
         all_branches=all_branches,
@@ -24,6 +26,7 @@ def push(
         remote=remote,
         jobs=jobs,
         recursive=recursive,
+        used_run_cache=used_run_cache,
     )
 
-    return self.cloud.push(used, jobs, remote=remote, run_cache=run_cache)
+    return self.cloud.push(used, jobs, remote=remote)
diff --git a/dvc/repo/update.py b/dvc/repo/update.py
index 5ade23366..d0dac9a97 100644
--- a/dvc/repo/update.py
+++ b/dvc/repo/update.py
@@ -1,14 +1,23 @@
+from ..dvcfile import Dvcfile
 from . import locked
 
 
 @locked
-def update(self, target, rev=None):
-    from ..dvcfile import Dvcfile
+def update(self, targets, rev=None, recursive=False):
+    if not targets:
+        targets = [None]
 
-    dvcfile = Dvcfile(self, target)
-    stage = dvcfile.stage
-    stage.update(rev)
+    if isinstance(targets, str):
+        targets = [targets]
 
-    dvcfile.dump(stage)
+    stages = set()
+    for target in targets:
+        stages.update(self.collect(target, recursive=recursive))
 
-    return stage
+    for stage in stages:
+        stage.update(rev)
+        dvcfile = Dvcfile(self, stage.path)
+        dvcfile.dump(stage)
+        stages.add(stage)
+
+    return list(stages)
diff --git a/dvc/stage/cache.py b/dvc/stage/cache.py
index af6528f51..c383d8725 100644
--- a/dvc/stage/cache.py
+++ b/dvc/stage/cache.py
@@ -106,9 +106,11 @@ class StageCache:
 
     @staticmethod
     def _transfer(func, from_remote, to_remote):
+        ret = []
+
         runs = from_remote.path_info / "runs"
         if not from_remote.exists(runs):
-            return
+            return []
 
         for src in from_remote.walk_files(runs):
             rel = src.relative_to(from_remote.path_info)
@@ -118,9 +120,36 @@ class StageCache:
             if to_remote.exists(key) and first(to_remote.walk_files(key)):
                 continue
             func(src, dst)
+            ret.append((src.parent.name, src.name))
+
+        return ret
 
     def push(self, remote):
+        remote = self.repo.cloud.get_remote(remote)
         return self._transfer(remote.upload, self.repo.cache.local, remote)
 
     def pull(self, remote):
+        remote = self.repo.cloud.get_remote(remote)
         return self._transfer(remote.download, remote, self.repo.cache.local)
+
+    def get_used_cache(self, used_run_cache, *args, **kwargs):
+        from dvc.cache import NamedCache
+        from dvc.stage import create_stage, PipelineStage
+
+        cache = NamedCache()
+
+        for key, value in used_run_cache:
+            entry = self._load_cache(key, value)
+            if not entry:
+                continue
+            stage = create_stage(
+                PipelineStage,
+                repo=self.repo,
+                path="dvc.yaml",
+                cmd=entry["cmd"],
+                deps=[dep["path"] for dep in entry["deps"]],
+                outs=[out["path"] for out in entry["outs"]],
+            )
+            StageLoader.fill_from_lock(stage, entry)
+            cache.update(stage.get_used_cache(*args, **kwargs))
+        return cache


Test patch:
diff --git a/tests/func/test_repro.py b/tests/func/test_repro.py
index 10652b14d..bc6bbcbeb 100644
--- a/tests/func/test_repro.py
+++ b/tests/func/test_repro.py
@@ -956,12 +956,12 @@ class TestReproExternalBase(SingleStageRun, TestDvc):
 
         self.assertNotEqual(self.dvc.status(), {})
 
-        self.dvc.update(import_stage.path)
+        self.dvc.update([import_stage.path])
         self.assertTrue(os.path.exists("import"))
         self.assertTrue(filecmp.cmp("import", self.BAR, shallow=False))
         self.assertEqual(self.dvc.status([import_stage.path]), {})
 
-        self.dvc.update(import_remote_stage.path)
+        self.dvc.update([import_remote_stage.path])
         self.assertEqual(self.dvc.status([import_remote_stage.path]), {})
 
         stages = self.dvc.reproduce(cmd_stage.addressing)
diff --git a/tests/func/test_update.py b/tests/func/test_update.py
index 2da66f347..6a91df6ff 100644
--- a/tests/func/test_update.py
+++ b/tests/func/test_update.py
@@ -26,7 +26,7 @@ def test_update_import(tmp_dir, dvc, erepo_dir, cached):
 
     assert old_rev != new_rev
 
-    dvc.update(stage.path)
+    dvc.update([stage.path])
     assert (tmp_dir / "version").read_text() == "updated"
 
     stage = Dvcfile(dvc, stage.path).stage
@@ -65,7 +65,7 @@ def test_update_import_after_remote_updates_to_dvc(tmp_dir, dvc, erepo_dir):
     assert changed_dep[0].startswith("version ")
     assert changed_dep[1] == "update available"
 
-    dvc.update(stage.path)
+    dvc.update([stage.path])
 
     assert dvc.status([stage.path]) == {}
 
@@ -106,7 +106,7 @@ def test_update_before_and_after_dvc_init(tmp_dir, dvc, git_dir):
         ]
     }
 
-    dvc.update(stage.path)
+    dvc.update([stage.path])
 
     assert (tmp_dir / "file").read_text() == "second version"
     assert dvc.status([stage.path]) == {}
@@ -127,7 +127,7 @@ def test_update_import_url(tmp_dir, dvc, tmp_path_factory):
     src.write_text("updated file content")
 
     assert dvc.status([stage.path]) == {}
-    dvc.update(stage.path)
+    dvc.update([stage.path])
     assert dvc.status([stage.path]) == {}
 
     assert dst.is_file()
@@ -149,7 +149,7 @@ def test_update_rev(tmp_dir, dvc, scm, git_dir):
         git_dir.scm_gen({"foo": "foobar foo"}, commit="branch2 commit")
         branch2_head = git_dir.scm.get_rev()
 
-    stage = dvc.update("foo.dvc", rev="branch1")
+    stage = dvc.update(["foo.dvc"], rev="branch1")[0]
     assert stage.deps[0].def_repo == {
         "url": fspath(git_dir),
         "rev": "branch1",
@@ -158,7 +158,7 @@ def test_update_rev(tmp_dir, dvc, scm, git_dir):
     with open(fspath_py35(tmp_dir / "foo")) as f:
         assert "foobar" == f.read()
 
-    stage = dvc.update("foo.dvc", rev="branch2")
+    stage = dvc.update(["foo.dvc"], rev="branch2")[0]
     assert stage.deps[0].def_repo == {
         "url": fspath(git_dir),
         "rev": "branch2",
@@ -166,3 +166,60 @@ def test_update_rev(tmp_dir, dvc, scm, git_dir):
     }
     with open(fspath_py35(tmp_dir / "foo")) as f:
         assert "foobar foo" == f.read()
+
+
+def test_update_recursive(tmp_dir, dvc, erepo_dir):
+    with erepo_dir.branch("branch", new=True), erepo_dir.chdir():
+        erepo_dir.scm_gen(
+            {"foo1": "text1", "foo2": "text2", "foo3": "text3"},
+            commit="add foo files",
+        )
+        old_rev = erepo_dir.scm.get_rev()
+
+    tmp_dir.gen({"dir": {"subdir": {}}})
+    stage1 = dvc.imp(
+        fspath(erepo_dir), "foo1", os.path.join("dir", "foo1"), rev="branch",
+    )
+    stage2 = dvc.imp(
+        fspath(erepo_dir),
+        "foo2",
+        os.path.join("dir", "subdir", "foo2"),
+        rev="branch",
+    )
+    stage3 = dvc.imp(
+        fspath(erepo_dir),
+        "foo3",
+        os.path.join("dir", "subdir", "foo3"),
+        rev="branch",
+    )
+
+    assert (tmp_dir / os.path.join("dir", "foo1")).read_text() == "text1"
+    assert (
+        tmp_dir / os.path.join("dir", "subdir", "foo2")
+    ).read_text() == "text2"
+    assert (
+        tmp_dir / os.path.join("dir", "subdir", "foo3")
+    ).read_text() == "text3"
+
+    assert stage1.deps[0].def_repo["rev_lock"] == old_rev
+    assert stage2.deps[0].def_repo["rev_lock"] == old_rev
+    assert stage3.deps[0].def_repo["rev_lock"] == old_rev
+
+    with erepo_dir.branch("branch", new=False), erepo_dir.chdir():
+        erepo_dir.scm_gen(
+            {"foo1": "updated1", "foo2": "updated2", "foo3": "updated3"},
+            "",
+            "update foo content",
+        )
+        new_rev = erepo_dir.scm.get_rev()
+
+    assert old_rev != new_rev
+
+    dvc.update(["dir"], recursive=True)
+
+    stage1 = Dvcfile(dvc, stage1.path).stage
+    stage2 = Dvcfile(dvc, stage2.path).stage
+    stage3 = Dvcfile(dvc, stage3.path).stage
+    assert stage1.deps[0].def_repo["rev_lock"] == new_rev
+    assert stage2.deps[0].def_repo["rev_lock"] == new_rev
+    assert stage3.deps[0].def_repo["rev_lock"] == new_rev
diff --git a/tests/unit/command/test_update.py b/tests/unit/command/test_update.py
index ff8e77942..087ab4483 100644
--- a/tests/unit/command/test_update.py
+++ b/tests/unit/command/test_update.py
@@ -1,20 +1,17 @@
-import pytest
-
 from dvc.cli import parse_args
 from dvc.command.update import CmdUpdate
 
 
-@pytest.mark.parametrize(
-    "command,rev", [(["update"], None), (["update", "--rev", "REV"], "REV")]
-)
-def test_update(dvc, mocker, command, rev):
-    targets = ["target1", "target2", "target3"]
-    cli_args = parse_args(command + targets)
+def test_update(dvc, mocker):
+    cli_args = parse_args(
+        ["update", "target1", "target2", "--rev", "REV", "--recursive"]
+    )
     assert cli_args.func == CmdUpdate
     cmd = cli_args.func(cli_args)
     m = mocker.patch("dvc.repo.Repo.update")
 
     assert cmd.run() == 0
 
-    calls = [mocker.call(target, rev) for target in targets]
-    m.assert_has_calls(calls)
+    m.assert_called_once_with(
+        targets=["target1", "target2"], rev="REV", recursive=True,
+    )