Repository: iterative/dvc

Issue:
gc: support used cache collection for several repositories using the same remote storage
E.g. you have 2 projects that have somewhat similar cache files, that you decided to put on the same remote. Currently, calling `dvc gc -c` in one of the projects will wipe out remote cache for the second project, since used cache is only collected for the current project. Need to add ability to collect used cache for several projects.

Discussion:

Patch:
diff --git a/dvc/cli.py b/dvc/cli.py
index 39ec74680..efc406f0d 100644
--- a/dvc/cli.py
+++ b/dvc/cli.py
@@ -25,7 +25,7 @@ from dvc.command.metrics import CmdMetricsRemove, CmdMetricsModify
 from dvc.command.install import CmdInstall
 from dvc.command.root import CmdRoot
 from dvc.command.lock import CmdLock, CmdUnlock
-from dvc.command.pipeline import CmdPipelineShow, CmdPipelineList
+from dvc.command.pipeline import CmdPipelineShow
 from dvc.logger import Logger
 from dvc import VERSION
 
@@ -591,6 +591,13 @@ def parse_args(argv=None):
                         type=int,
                         default=None,
                         help='Number of jobs to run simultaneously.')
+    gc_parser.add_argument(
+                        '-p',
+                        '--projects',
+                        type=str,
+                        nargs='*',
+                        default=None,
+                        help='Collect garbage for all given projects.')
     gc_parser.set_defaults(func=CmdGC)
 
     # Config
@@ -927,14 +934,6 @@ def parse_args(argv=None):
                         help="DVC files. 'Dvcfile' by default.")
     pipeline_show_parser.set_defaults(func=CmdPipelineShow)
 
-    PIPELINE_LIST_HELP = 'List pipelines.'
-    pipeline_list_parser = pipeline_subparsers.add_parser(
-                        'list',
-                        parents=[parent_parser],
-                        description=PIPELINE_LIST_HELP,
-                        help=PIPELINE_LIST_HELP)
-    pipeline_list_parser.set_defaults(func=CmdPipelineList)
-
     args = parser.parse_args(argv)
 
     if (issubclass(args.func, CmdRepro)
diff --git a/dvc/command/gc.py b/dvc/command/gc.py
index f0b25ae49..a802bf08c 100644
--- a/dvc/command/gc.py
+++ b/dvc/command/gc.py
@@ -1,3 +1,5 @@
+import os
+
 from dvc.command.base import CmdBase
 
 
@@ -5,13 +7,21 @@ class CmdGC(CmdBase):
     def run(self):
         msg = 'This will remove all cache except the cache that is used in '
         if not self.args.all_branches and not self.args.all_tags:
-            msg += 'the current git branch.'
+            msg += 'the current git branch'
         elif self.args.all_branches and not self.args.all_tags:
-            msg += 'all git branches.'
+            msg += 'all git branches'
         elif not self.args.all_branches and self.args.all_tags:
-            msg += 'all git tags.'
+            msg += 'all git tags'
+        else:
+            msg += 'all git branches and all git tags'
+
+        if self.args.projects is not None and len(self.args.projects) > 0:
+            msg += ' of the current and the following projects:'
+
+            for project_path in self.args.projects:
+                msg += '\n  - %s' % os.path.abspath(project_path)
         else:
-            msg += 'all git branches and all git tags.'
+            msg += ' of the current project.'
 
         self.project.logger.warn(msg)
 
@@ -24,5 +34,6 @@ class CmdGC(CmdBase):
                         cloud=self.args.cloud,
                         remote=self.args.remote,
                         force=self.args.force,
-                        jobs=self.args.jobs)
+                        jobs=self.args.jobs,
+                        projects=self.args.projects)
         return 0
diff --git a/dvc/command/pipeline.py b/dvc/command/pipeline.py
index 83eb40ebc..ac682b282 100644
--- a/dvc/command/pipeline.py
+++ b/dvc/command/pipeline.py
@@ -110,19 +110,3 @@ class CmdPipelineShow(CmdBase):
                 self.project.logger.error(msg, ex)
                 return 1
         return 0
-
-
-class CmdPipelineList(CmdBase):
-    def run(self):
-        import networkx
-
-        pipelines = self.project.pipelines()
-        for p in pipelines:
-            stages = networkx.get_node_attributes(p, 'stage')
-            for stage in stages:
-                self.project.logger.info(stage)
-            if len(stages) != 0:
-                self.project.logger.info("="*80)
-        self.project.logger.info("{} pipeline(s) total".format(len(pipelines)))
-
-        return 0
diff --git a/dvc/logger.py b/dvc/logger.py
index 4685a2b1d..41b2b6f2e 100644
--- a/dvc/logger.py
+++ b/dvc/logger.py
@@ -221,10 +221,8 @@ class Logger(object):
             ) for line in lines
         ]
 
-        box = "{margin}{padding}{content}{padding}{margin}".format(
-                margin=cls.colorize(margin, border_color),
-                padding=''.join(padding_lines),
-                content=''.join(content_lines),
-            )
-
-        print(box)
+        return "{margin}{padding}{content}{padding}{margin}".format(
+            margin=cls.colorize(margin, border_color),
+            padding=''.join(padding_lines),
+            content=''.join(content_lines),
+        )
diff --git a/dvc/project.py b/dvc/project.py
index 0828dbce4..1c62a3a81 100644
--- a/dvc/project.py
+++ b/dvc/project.py
@@ -1,3 +1,4 @@
+import collections
 import os
 
 from dvc.exceptions import DvcException, MoveNotDataSourceError
@@ -108,6 +109,19 @@ class Project(object):
 
         return proj
 
+    @staticmethod
+    def load_all(projects_paths):
+        """
+        Instantiate all projects in the given list of paths.
+
+        Args:
+            projects_paths: List of paths to projects.
+
+        Returns:
+            List of Project instances in the same order of the given paths.
+        """
+        return [Project(path) for path in projects_paths]
+
     def destroy(self):
         import shutil
 
@@ -602,6 +616,45 @@ class Project(object):
 
         return cache
 
+    @staticmethod
+    def merge_cache_lists(clists):
+        merged_cache = collections.defaultdict(list)
+
+        for cache_list in clists:
+            for scheme, cache in cache_list.items():
+                for item in cache:
+                    if item not in merged_cache[scheme]:
+                        merged_cache[scheme].append(item)
+
+        return merged_cache
+
+    @staticmethod
+    def load_all_used_cache(projects,
+                            target=None,
+                            all_branches=False,
+                            active=True,
+                            with_deps=False,
+                            all_tags=False,
+                            remote=None,
+                            force=False,
+                            jobs=None):
+        clists = []
+
+        for project in projects:
+            with project.state:
+                project_clist = project._used_cache(target=None,
+                                                    all_branches=all_branches,
+                                                    active=False,
+                                                    with_deps=with_deps,
+                                                    all_tags=all_tags,
+                                                    remote=remote,
+                                                    force=force,
+                                                    jobs=jobs)
+
+                clists.append(project_clist)
+
+        return clists
+
     def _do_gc(self, typ, func, clist):
         removed = func(clist)
         if not removed:
@@ -614,16 +667,30 @@ class Project(object):
            with_deps=False,
            all_tags=False,
            force=False,
-           jobs=None):
+           jobs=None,
+           projects=None):
+
+        all_projects = [self]
+
+        if projects is not None and len(projects) > 0:
+            all_projects.extend(Project.load_all(projects))
+
+        all_clists = Project.load_all_used_cache(all_projects,
+                                                 target=None,
+                                                 all_branches=all_branches,
+                                                 active=False,
+                                                 with_deps=with_deps,
+                                                 all_tags=all_tags,
+                                                 remote=remote,
+                                                 force=force,
+                                                 jobs=jobs)
+
+        if len(all_clists) > 1:
+            clist = Project.merge_cache_lists(all_clists)
+        else:
+            clist = all_clists[0]
+
         with self.state:
-            clist = self._used_cache(target=None,
-                                     all_branches=all_branches,
-                                     active=False,
-                                     with_deps=with_deps,
-                                     all_tags=all_tags,
-                                     remote=remote,
-                                     force=force,
-                                     jobs=jobs)
             self._do_gc('local', self.cache.local.gc, clist)
 
             if self.cache.s3:
diff --git a/dvc/updater.py b/dvc/updater.py
index 98748c383..c3f42cde6 100644
--- a/dvc/updater.py
+++ b/dvc/updater.py
@@ -1,4 +1,3 @@
-import sys
 import os
 import time
 import requests
@@ -18,9 +17,14 @@ class Updater(object):  # pragma: no cover
     def __init__(self, dvc_dir):
         self.dvc_dir = dvc_dir
         self.updater_file = os.path.join(dvc_dir, self.UPDATER_FILE)
-        self.current = VERSION_BASE
+
+    @staticmethod
+    def init(dvc_dir):
+        return Updater(dvc_dir)
 
     def check(self):
+        current = VERSION_BASE
+
         if os.getenv('CI'):
             return
 
@@ -36,121 +40,36 @@ class Updater(object):  # pragma: no cover
         Logger.info('Checking for updates...')
 
         try:
-            self._get_latest_version()
+            r = requests.get(self.URL, timeout=self.TIMEOUT_GET)
+            j = r.json()
+            latest = j['version']
+            open(self.updater_file, 'w+').close()
         except Exception as exc:
             msg = 'Failed to obtain latest version: {}'.format(str(exc))
             Logger.debug(msg)
             return
 
-        if self._is_outdated():
-            self._notify()
+        l_major, l_minor, l_patch = [int(x) for x in latest.split('.')]
+        c_major, c_minor, c_patch = [int(x) for x in current.split('.')]
 
-    def _get_latest_version(self):
-        r = requests.get(self.URL, timeout=self.TIMEOUT_GET)
-        j = r.json()
-        self.latest = j['version']
-        open(self.updater_file, 'w+').close()
-
-    def _is_outdated(self):
-        l_major, l_minor, l_patch = [int(x) for x in self.latest.split('.')]
-        c_major, c_minor, c_patch = [int(x) for x in self.current.split('.')]
-
-        return (l_major > c_major or
-                l_minor > c_minor or
-                l_patch > c_patch)
+        if l_major <= c_major and \
+           l_minor <= c_minor and \
+           l_patch <= c_patch:
+            return
 
-    def _notify(self):
-        message = (
+        msg = (
             'Update available {red}{current}{reset} -> {green}{latest}{reset}'
-            + '\n'
-            + self._get_update_instructions()
-        ).format(red=colorama.Fore.RED,
-                 reset=colorama.Fore.RESET,
-                 green=colorama.Fore.GREEN,
-                 yellow=colorama.Fore.YELLOW,
-                 blue=colorama.Fore.BLUE,
-                 current=self.current,
-                 latest=self.latest)
-
-        if sys.stdout.isatty():
-            Logger.box(message, border_color='yellow')
-
-    def _get_update_instructions(self):
-        instructions = {
-            'pip': 'Run {yellow}pip{reset} install dvc {blue}--upgrade{reset}',
-            'yum': 'Run {yellow}yum{reset} update dvc',
-            'yay': 'Run {yellow}yay{reset} {blue}-S{reset} dvc',
-            'formula': 'Run {yellow}brew{reset} upgrade dvc',
-            'cask': 'Run {yellow}brew cask{reset} upgrade dvc',
-            'apt': ('Run {yellow}apt-get{reset} install'
-                    ' {blue}--only-upgrade{reset} dvc'),
-            'binary': ('To upgrade follow this steps:\n'
-                       '1. Uninstall dvc binary\n'
-                       '2. Go to {blue}https://dvc.org{reset}\n'
-                       '3. Download and install new binary'),
-            None: ('Find the latest release at\n{blue}'
-                   'https://github.com/iterative/dvc/releases/latest'
-                   '{reset}'),
-        }
-
-        package_manager = self._get_package_manager()
-
-        return instructions[package_manager]
-
-    @staticmethod
-    def _is_binary():
-        return getattr(sys, 'frozen', False)
-
-    def _get_linux(self):
-        import distro
-
-        if not self._is_binary():
-            return 'pip'
-
-        package_managers = {
-            'rhel':     'yum',
-            'centos':   'yum',
-            'fedora':   'yum',
-            'amazon':   'yum',
-            'opensuse': 'yum',
-            'ubuntu':   'apt',
-            'debian':   'apt',
-        }
-
-        return package_managers.get(distro.id())
-
-    def _get_darwin(self):
-        if not self._is_binary():
-            if __file__.startswith('/usr/local/Cellar'):
-                return 'formula'
-            else:
-                return 'pip'
-
-        # NOTE: both pkg and cask put dvc binary into /usr/local/bin,
-        # so in order to know which method of installation was used,
-        # we need to actually call `brew cask`
-        ret = os.system('brew cask ls dvc')
-        if ret == 0:
-            return 'cask'
-
-        return None
-
-    def _get_windows(self):
-        return None if self._is_binary() else 'pip'
-
-    def _get_package_manager(self):
-        import platform
-        from dvc.exceptions import DvcException
-
-        m = {
-            'Windows': self._get_windows,
-            'Darwin': self._get_darwin,
-            'Linux': self._get_linux,
-        }
-
-        system = platform.system()
-        func = m.get(system)
-        if func is None:
-            raise DvcException("Not supported system '{}'".format(system))
-
-        return func()
+            '\n'
+            'Run {yellow}pip{reset} install dvc {blue}--upgrade{reset}'
+            .format(
+                red=colorama.Fore.RED,
+                reset=colorama.Fore.RESET,
+                green=colorama.Fore.GREEN,
+                yellow=colorama.Fore.YELLOW,
+                blue=colorama.Fore.BLUE,
+                current=current,
+                latest=latest,
+            )
+        )
+
+        Logger.box(msg, border_color='yellow')
diff --git a/requirements.txt b/requirements.txt
index a21c5e263..f5fe48990 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -23,4 +23,3 @@ futures>=3.2.0; python_version == "2.7"
 grandalf==0.6
 pydot>=1.2.4
 asciimatics>=1.10.0
-distro>=1.3.0
diff --git a/setup.py b/setup.py
index 70e30693f..a2612cafb 100644
--- a/setup.py
+++ b/setup.py
@@ -23,7 +23,6 @@ install_requires = [
     "requests>=2.18.4",
     "grandalf==0.6",
     "asciimatics>=1.10.0",
-    "distro>=1.3.0",
 ]
 
 # Extra dependencies for remote integrations


Test patch:
diff --git a/tests/test_cli.py b/tests/test_cli.py
index ee438b4e3..4aa22d47c 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -129,6 +129,20 @@ class TestGC(TestDvc):
         self.assertIsInstance(args.func(args), CmdGC)
 
 
+class TestGCMultipleProjects(TestDvc):
+    def test(self):
+        args = parse_args([
+            'gc',
+            '-p',
+            '/tmp/asdf',
+            '/tmp/xyz'
+        ])
+
+        self.assertIsInstance(args.func(args), CmdGC)
+
+        self.assertEqual(args.projects, ['/tmp/asdf', '/tmp/xyz'])
+
+
 class TestConfig(TestDvc):
     def test(self):
         name = 'param'
diff --git a/tests/test_gc.py b/tests/test_gc.py
index dcafd6005..4467f0986 100644
--- a/tests/test_gc.py
+++ b/tests/test_gc.py
@@ -1,13 +1,18 @@
 import os
 import shutil
 import filecmp
+import configobj
+
+from git import Repo
 
 from dvc.main import main
 from dvc.utils import file_md5
 from dvc.stage import Stage
+from dvc.project import Project
 from dvc.command.gc import CmdGC
 
 from tests.basic_env import TestDvc
+from tests.basic_env import TestDir
 
 
 class TestGC(TestDvc):
@@ -16,7 +21,8 @@ class TestGC(TestDvc):
 
         self.dvc.add(self.FOO)
         self.dvc.add(self.DATA_DIR)
-        self.good_cache = [self.dvc.cache.local.get(md5) for md5 in self.dvc.cache.local.all()]
+        self.good_cache = [self.dvc.cache.local.get(
+            md5) for md5 in self.dvc.cache.local.all()]
 
         self.bad_cache = []
         for i in ['123', '234', '345']:
@@ -100,3 +106,75 @@ class TestGCBranchesTags(TestDvc):
         self.dvc.gc(all_tags=True, all_branches=False)
 
         self._check_cache(1)
+
+
+class TestGCMultipleProjects(TestDvc):
+    def _check_cache(self, num):
+        total = 0
+        for root, dirs, files in os.walk(os.path.join('.dvc', 'cache')):
+            total += len(files)
+        self.assertEqual(total, num)
+
+    def setUp(self):
+        super(TestGCMultipleProjects, self).setUp()
+        self.additional_path = TestDir.mkdtemp()
+        self.additional_git = Repo.init(self.additional_path)
+        self.additional_dvc = Project.init(self.additional_path)
+
+        cache_path = os.path.join(self._root_dir, '.dvc', 'cache')
+        config_path = os.path.join(self.additional_path, '.dvc',
+                                   'config.local')
+        cfg = configobj.ConfigObj()
+        cfg.filename = config_path
+        cfg['cache'] = {
+            'dir': cache_path
+        }
+        cfg.write()
+
+        self.additional_dvc = Project(self.additional_path)
+
+    def test(self):
+
+        # ADD FILE ONLY IN MAIN PROJECT
+        fname = 'only_in_first'
+        with open(fname, 'w+') as fobj:
+            fobj.write('only in main project')
+
+        stages = self.dvc.add(fname)
+        self.assertEqual(len(stages), 1)
+
+        # ADD FILE IN MAIN PROJECT THAT IS ALSO IN SECOND PROJECT
+        fname = 'in_both'
+        with open(fname, 'w+') as fobj:
+            fobj.write('in both projects')
+
+        stages = self.dvc.add(fname)
+        self.assertEqual(len(stages), 1)
+
+        cwd = os.getcwd()
+        os.chdir(self.additional_path)
+        # ADD FILE ONLY IN SECOND PROJECT
+        fname = os.path.join(self.additional_path, 'only_in_second')
+        with open(fname, 'w+') as fobj:
+            fobj.write('only in additional project')
+
+        stages = self.additional_dvc.add(fname)
+        self.assertEqual(len(stages), 1)
+
+        # ADD FILE IN SECOND PROJECT THAT IS ALSO IN MAIN PROJECT
+        fname = os.path.join(self.additional_path, 'in_both')
+        with open(fname, 'w+') as fobj:
+            fobj.write('in both projects')
+
+        stages = self.additional_dvc.add(fname)
+        self.assertEqual(len(stages), 1)
+
+        os.chdir(cwd)
+
+        self._check_cache(3)
+
+        self.dvc.gc(projects=[self.additional_path])
+        self._check_cache(3)
+
+        self.dvc.gc()
+        self._check_cache(2)
diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py
index 7fe287298..ec1d829c8 100644
--- a/tests/test_pipeline.py
+++ b/tests/test_pipeline.py
@@ -165,15 +165,3 @@ class TestPipelineShowDeep(TestReproChangedDeepData):
     def test_non_existing(self):
         ret = main(['pipeline', 'show', 'non-existing'])
         self.assertNotEqual(ret, 0)
-
-
-class TestPipelineListEmpty(TestDvc):
-    def test(self):
-        ret = main(['pipeline', 'list'])
-        self.assertEqual(ret, 0)
-
-
-class TestPipelineListSingle(TestPipelineShowDeep):
-    def test(self):
-        ret = main(['pipeline', 'list'])
-        self.assertEqual(ret, 0)