Repository: iterative/dvc

Issue:
gc: support used cache collection for several repositories using the same remote storage
E.g. you have 2 projects that have somewhat similar cache files, that you decided to put on the same remote. Currently, calling `dvc gc -c` in one of the projects will wipe out remote cache for the second project, since used cache is only collected for the current project. Need to add ability to collect used cache for several projects.


Discussion:
For the record: @ynop from discord volunteered to work on this. 

Resolving patch:
diff --git a/dvc/cli.py b/dvc/cli.py
--- a/dvc/cli.py
+++ b/dvc/cli.py
@@ -591,6 +591,13 @@ def parse_args(argv=None):
                         type=int,
                         default=None,
                         help='Number of jobs to run simultaneously.')
+    gc_parser.add_argument(
+                        '-p',
+                        '--projects',
+                        type=str,
+                        nargs='*',
+                        default=None,
+                        help='Collect garbage for all given projects.')
     gc_parser.set_defaults(func=CmdGC)
 
     # Config
diff --git a/dvc/command/gc.py b/dvc/command/gc.py
--- a/dvc/command/gc.py
+++ b/dvc/command/gc.py
@@ -1,3 +1,5 @@
+import os
+
 from dvc.command.base import CmdBase
 
 
@@ -5,13 +7,21 @@ class CmdGC(CmdBase):
     def run(self):
         msg = 'This will remove all cache except the cache that is used in '
         if not self.args.all_branches and not self.args.all_tags:
-            msg += 'the current git branch.'
+            msg += 'the current git branch'
         elif self.args.all_branches and not self.args.all_tags:
-            msg += 'all git branches.'
+            msg += 'all git branches'
         elif not self.args.all_branches and self.args.all_tags:
-            msg += 'all git tags.'
+            msg += 'all git tags'
+        else:
+            msg += 'all git branches and all git tags'
+
+        if self.args.projects is not None and len(self.args.projects) > 0:
+            msg += ' of the current and the following projects:'
+
+            for project_path in self.args.projects:
+                msg += '\n  - %s' % os.path.abspath(project_path)
         else:
-            msg += 'all git branches and all git tags.'
+            msg += ' of the current project.'
 
         self.project.logger.warn(msg)
 
@@ -24,5 +34,6 @@ def run(self):
                         cloud=self.args.cloud,
                         remote=self.args.remote,
                         force=self.args.force,
-                        jobs=self.args.jobs)
+                        jobs=self.args.jobs,
+                        projects=self.args.projects)
         return 0
diff --git a/dvc/project.py b/dvc/project.py
--- a/dvc/project.py
+++ b/dvc/project.py
@@ -1,3 +1,4 @@
+import collections
 import os
 
 from dvc.exceptions import DvcException, MoveNotDataSourceError
@@ -108,6 +109,19 @@ def init(root_dir=os.curdir, no_scm=False, force=False):
 
         return proj
 
+    @staticmethod
+    def load_all(projects_paths):
+        """
+        Instantiate all projects in the given list of paths.
+
+        Args:
+            projects_paths: List of paths to projects.
+
+        Returns:
+            List of Project instances in the same order of the given paths.
+        """
+        return [Project(path) for path in projects_paths]
+
     def destroy(self):
         import shutil
 
@@ -602,6 +616,45 @@ def _used_cache(self,
 
         return cache
 
+    @staticmethod
+    def merge_cache_lists(clists):
+        merged_cache = collections.defaultdict(list)
+
+        for cache_list in clists:
+            for scheme, cache in cache_list.items():
+                for item in cache:
+                    if item not in merged_cache[scheme]:
+                        merged_cache[scheme].append(item)
+
+        return merged_cache
+
+    @staticmethod
+    def load_all_used_cache(projects,
+                            target=None,
+                            all_branches=False,
+                            active=True,
+                            with_deps=False,
+                            all_tags=False,
+                            remote=None,
+                            force=False,
+                            jobs=None):
+        clists = []
+
+        for project in projects:
+            with project.state:
+                project_clist = project._used_cache(target=None,
+                                                    all_branches=all_branches,
+                                                    active=False,
+                                                    with_deps=with_deps,
+                                                    all_tags=all_tags,
+                                                    remote=remote,
+                                                    force=force,
+                                                    jobs=jobs)
+
+                clists.append(project_clist)
+
+        return clists
+
     def _do_gc(self, typ, func, clist):
         removed = func(clist)
         if not removed:
@@ -614,16 +667,30 @@ def gc(self,
            with_deps=False,
            all_tags=False,
            force=False,
-           jobs=None):
+           jobs=None,
+           projects=None):
+
+        all_projects = [self]
+
+        if projects is not None and len(projects) > 0:
+            all_projects.extend(Project.load_all(projects))
+
+        all_clists = Project.load_all_used_cache(all_projects,
+                                                 target=None,
+                                                 all_branches=all_branches,
+                                                 active=False,
+                                                 with_deps=with_deps,
+                                                 all_tags=all_tags,
+                                                 remote=remote,
+                                                 force=force,
+                                                 jobs=jobs)
+
+        if len(all_clists) > 1:
+            clist = Project.merge_cache_lists(all_clists)
+        else:
+            clist = all_clists[0]
+
         with self.state:
-            clist = self._used_cache(target=None,
-                                     all_branches=all_branches,
-                                     active=False,
-                                     with_deps=with_deps,
-                                     all_tags=all_tags,
-                                     remote=remote,
-                                     force=force,
-                                     jobs=jobs)
             self._do_gc('local', self.cache.local.gc, clist)
 
             if self.cache.s3:


Test patch:
diff --git a/tests/test_cli.py b/tests/test_cli.py
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -129,6 +129,20 @@ def test(self):
         self.assertIsInstance(args.func(args), CmdGC)
 
 
+class TestGCMultipleProjects(TestDvc):
+    def test(self):
+        args = parse_args([
+            'gc',
+            '-p',
+            '/tmp/asdf',
+            '/tmp/xyz'
+        ])
+
+        self.assertIsInstance(args.func(args), CmdGC)
+
+        self.assertEqual(args.projects, ['/tmp/asdf', '/tmp/xyz'])
+
+
 class TestConfig(TestDvc):
     def test(self):
         name = 'param'
diff --git a/tests/test_gc.py b/tests/test_gc.py
--- a/tests/test_gc.py
+++ b/tests/test_gc.py
@@ -1,13 +1,18 @@
 import os
 import shutil
 import filecmp
+import configobj
+
+from git import Repo
 
 from dvc.main import main
 from dvc.utils import file_md5
 from dvc.stage import Stage
+from dvc.project import Project
 from dvc.command.gc import CmdGC
 
 from tests.basic_env import TestDvc
+from tests.basic_env import TestDir
 
 
 class TestGC(TestDvc):
@@ -16,7 +21,8 @@ def setUp(self):
 
         self.dvc.add(self.FOO)
         self.dvc.add(self.DATA_DIR)
-        self.good_cache = [self.dvc.cache.local.get(md5) for md5 in self.dvc.cache.local.all()]
+        self.good_cache = [self.dvc.cache.local.get(
+            md5) for md5 in self.dvc.cache.local.all()]
 
         self.bad_cache = []
         for i in ['123', '234', '345']:
@@ -100,3 +106,75 @@ def test(self):
         self.dvc.gc(all_tags=True, all_branches=False)
 
         self._check_cache(1)
+
+
+class TestGCMultipleProjects(TestDvc):
+    def _check_cache(self, num):
+        total = 0
+        for root, dirs, files in os.walk(os.path.join('.dvc', 'cache')):
+            total += len(files)
+        self.assertEqual(total, num)
+
+    def setUp(self):
+        super(TestGCMultipleProjects, self).setUp()
+        self.additional_path = TestDir.mkdtemp()
+        self.additional_git = Repo.init(self.additional_path)
+        self.additional_dvc = Project.init(self.additional_path)
+
+        cache_path = os.path.join(self._root_dir, '.dvc', 'cache')
+        config_path = os.path.join(self.additional_path, '.dvc',
+                                   'config.local')
+        cfg = configobj.ConfigObj()
+        cfg.filename = config_path
+        cfg['cache'] = {
+            'dir': cache_path
+        }
+        cfg.write()
+
+        self.additional_dvc = Project(self.additional_path)
+
+    def test(self):
+
+        # ADD FILE ONLY IN MAIN PROJECT
+        fname = 'only_in_first'
+        with open(fname, 'w+') as fobj:
+            fobj.write('only in main project')
+
+        stages = self.dvc.add(fname)
+        self.assertEqual(len(stages), 1)
+
+        # ADD FILE IN MAIN PROJECT THAT IS ALSO IN SECOND PROJECT
+        fname = 'in_both'
+        with open(fname, 'w+') as fobj:
+            fobj.write('in both projects')
+
+        stages = self.dvc.add(fname)
+        self.assertEqual(len(stages), 1)
+
+        cwd = os.getcwd()
+        os.chdir(self.additional_path)
+        # ADD FILE ONLY IN SECOND PROJECT
+        fname = os.path.join(self.additional_path, 'only_in_second')
+        with open(fname, 'w+') as fobj:
+            fobj.write('only in additional project')
+
+        stages = self.additional_dvc.add(fname)
+        self.assertEqual(len(stages), 1)
+
+        # ADD FILE IN SECOND PROJECT THAT IS ALSO IN MAIN PROJECT
+        fname = os.path.join(self.additional_path, 'in_both')
+        with open(fname, 'w+') as fobj:
+            fobj.write('in both projects')
+
+        stages = self.additional_dvc.add(fname)
+        self.assertEqual(len(stages), 1)
+
+        os.chdir(cwd)
+
+        self._check_cache(3)
+
+        self.dvc.gc(projects=[self.additional_path])
+        self._check_cache(3)
+
+        self.dvc.gc()
+        self._check_cache(2)