Repository: iterative/dvc

Issue:
Optimize stage collection for large repos
I posted this issue #2666 yesterday which was closed but the patch did not solve my underlying issue. The DAG is constructed using the stages from the `repo.collect_stages()` method. This method is what is very slow for me with a large repo. So I would like to ask again if you could make the collection of every stage before any operation optional so it doesn't take hours to `dvc add` a single file with no dependencies.

Discussion:
Suor: Should be good to go.

Patch:
diff --git a/dvc/command/data_sync.py b/dvc/command/data_sync.py
index 7be18cc74..a7decd18b 100644
--- a/dvc/command/data_sync.py
+++ b/dvc/command/data_sync.py
@@ -102,7 +102,7 @@ def add_parser(subparsers, _parent_parser):
     from dvc.command.status import CmdDataStatus
 
     # Pull
-    PULL_HELP = "Pull data files from a DVC remote storage."
+    PULL_HELP = "Download tracked files or directories from remote storage."
 
     pull_parser = subparsers.add_parser(
         "pull",
@@ -152,8 +152,7 @@ def add_parser(subparsers, _parent_parser):
     pull_parser.set_defaults(func=CmdDataPull)
 
     # Push
-    PUSH_HELP = "Uploads files or directories tracked by DVC"
-    " to remote storage."
+    PUSH_HELP = "Upload tracked files or directories to remote storage."
 
     push_parser = subparsers.add_parser(
         "push",
@@ -196,7 +195,7 @@ def add_parser(subparsers, _parent_parser):
     push_parser.set_defaults(func=CmdDataPush)
 
     # Fetch
-    FETCH_HELP = "Get files or directories tracked by DVC from remote storage"
+    FETCH_HELP = "Get tracked files or directories from remote storage"
     " into the cache."
 
     fetch_parser = subparsers.add_parser(
diff --git a/dvc/command/metrics.py b/dvc/command/metrics.py
index 038e2f266..04c2f80ee 100644
--- a/dvc/command/metrics.py
+++ b/dvc/command/metrics.py
@@ -159,7 +159,7 @@ class CmdMetricsDiff(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    METRICS_HELP = "Commands to add, manage, collect and display metrics."
+    METRICS_HELP = "Commands to add, manage, collect, and display metrics."
 
     metrics_parser = subparsers.add_parser(
         "metrics",
@@ -176,7 +176,7 @@ def add_parser(subparsers, parent_parser):
 
     fix_subparsers(metrics_subparsers)
 
-    METRICS_SHOW_HELP = "Output metric values."
+    METRICS_SHOW_HELP = "Print metrics, with optional formatting."
     metrics_show_parser = metrics_subparsers.add_parser(
         "show",
         parents=[parent_parser],
@@ -227,7 +227,7 @@ def add_parser(subparsers, parent_parser):
     )
     metrics_show_parser.set_defaults(func=CmdMetricsShow)
 
-    METRICS_ADD_HELP = "Tag file as a metric file."
+    METRICS_ADD_HELP = "Mark a DVC-tracked file as a metric."
     metrics_add_parser = metrics_subparsers.add_parser(
         "add",
         parents=[parent_parser],
@@ -244,7 +244,7 @@ def add_parser(subparsers, parent_parser):
     metrics_add_parser.add_argument("path", help="Path to a metric file.")
     metrics_add_parser.set_defaults(func=CmdMetricsAdd)
 
-    METRICS_MODIFY_HELP = "Modify metric file options."
+    METRICS_MODIFY_HELP = "Modify metric default formatting."
     metrics_modify_parser = metrics_subparsers.add_parser(
         "modify",
         parents=[parent_parser],
@@ -261,7 +261,7 @@ def add_parser(subparsers, parent_parser):
     metrics_modify_parser.add_argument("path", help="Path to a metric file.")
     metrics_modify_parser.set_defaults(func=CmdMetricsModify)
 
-    METRICS_REMOVE_HELP = "Remove files's metric tag."
+    METRICS_REMOVE_HELP = "Remove metric mark on a DVC-tracked file."
     metrics_remove_parser = metrics_subparsers.add_parser(
         "remove",
         parents=[parent_parser],
diff --git a/dvc/command/pipeline.py b/dvc/command/pipeline.py
index 6aedb086a..a265f10e1 100644
--- a/dvc/command/pipeline.py
+++ b/dvc/command/pipeline.py
@@ -175,7 +175,7 @@ def add_parser(subparsers, parent_parser):
 
     fix_subparsers(pipeline_subparsers)
 
-    PIPELINE_SHOW_HELP = "Show pipelines."
+    PIPELINE_SHOW_HELP = "Show stages in a pipeline."
     pipeline_show_parser = pipeline_subparsers.add_parser(
         "show",
         parents=[parent_parser],
@@ -231,7 +231,7 @@ def add_parser(subparsers, parent_parser):
     )
     pipeline_show_parser.set_defaults(func=CmdPipelineShow)
 
-    PIPELINE_LIST_HELP = "List pipelines."
+    PIPELINE_LIST_HELP = "List connected groups of stages (pipelines)."
     pipeline_list_parser = pipeline_subparsers.add_parser(
         "list",
         parents=[parent_parser],
diff --git a/dvc/command/remote.py b/dvc/command/remote.py
index 364ee7db6..14d089a1a 100644
--- a/dvc/command/remote.py
+++ b/dvc/command/remote.py
@@ -100,7 +100,7 @@ class CmdRemoteList(CmdRemote):
 def add_parser(subparsers, parent_parser):
     from dvc.command.config import parent_config_parser
 
-    REMOTE_HELP = "Manage remote storage configuration."
+    REMOTE_HELP = "Set up and manage data remotes."
     remote_parser = subparsers.add_parser(
         "remote",
         parents=[parent_parser],
@@ -116,7 +116,7 @@ def add_parser(subparsers, parent_parser):
 
     fix_subparsers(remote_subparsers)
 
-    REMOTE_ADD_HELP = "Add remote."
+    REMOTE_ADD_HELP = "Add a new data remote."
     remote_add_parser = remote_subparsers.add_parser(
         "add",
         parents=[parent_config_parser, parent_parser],
@@ -147,7 +147,7 @@ def add_parser(subparsers, parent_parser):
     )
     remote_add_parser.set_defaults(func=CmdRemoteAdd)
 
-    REMOTE_DEFAULT_HELP = "Set/unset default remote."
+    REMOTE_DEFAULT_HELP = "Set/unset the default data remote."
     remote_default_parser = remote_subparsers.add_parser(
         "default",
         parents=[parent_config_parser, parent_parser],
@@ -167,7 +167,7 @@ def add_parser(subparsers, parent_parser):
     )
     remote_default_parser.set_defaults(func=CmdRemoteDefault)
 
-    REMOTE_REMOVE_HELP = "Remove remote."
+    REMOTE_REMOVE_HELP = "Remove a data remote."
     remote_remove_parser = remote_subparsers.add_parser(
         "remove",
         parents=[parent_config_parser, parent_parser],
@@ -180,7 +180,7 @@ def add_parser(subparsers, parent_parser):
     )
     remote_remove_parser.set_defaults(func=CmdRemoteRemove)
 
-    REMOTE_MODIFY_HELP = "Modify remote."
+    REMOTE_MODIFY_HELP = "Modify the configuration of a data remote."
     remote_modify_parser = remote_subparsers.add_parser(
         "modify",
         parents=[parent_config_parser, parent_parser],
@@ -204,7 +204,7 @@ def add_parser(subparsers, parent_parser):
     )
     remote_modify_parser.set_defaults(func=CmdRemoteModify)
 
-    REMOTE_LIST_HELP = "List available remotes."
+    REMOTE_LIST_HELP = "List all available data remotes."
     remote_list_parser = remote_subparsers.add_parser(
         "list",
         parents=[parent_config_parser, parent_parser],
diff --git a/dvc/command/remove.py b/dvc/command/remove.py
index f93b4d265..24c7c1ce0 100644
--- a/dvc/command/remove.py
+++ b/dvc/command/remove.py
@@ -42,7 +42,7 @@ class CmdRemove(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    REMOVE_HELP = "Remove DVC-file outputs."
+    REMOVE_HELP = "Remove DVC-tracked files or directories."
     remove_parser = subparsers.add_parser(
         "remove",
         parents=[parent_parser],
diff --git a/dvc/command/repro.py b/dvc/command/repro.py
index 6fd1b6573..63f532f4e 100644
--- a/dvc/command/repro.py
+++ b/dvc/command/repro.py
@@ -57,7 +57,8 @@ class CmdRepro(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    REPRO_HELP = "Check for changes and reproduce stages and dependencies."
+    REPRO_HELP = "Reproduce complete or partial pipelines"
+    " by executing their stages."
     repro_parser = subparsers.add_parser(
         "repro",
         parents=[parent_parser],
diff --git a/dvc/command/root.py b/dvc/command/root.py
index c17dbf914..f9dadc2ed 100644
--- a/dvc/command/root.py
+++ b/dvc/command/root.py
@@ -17,7 +17,7 @@ class CmdRoot(CmdBaseNoRepo):
 
 
 def add_parser(subparsers, parent_parser):
-    ROOT_HELP = "Relative path to the repository's directory."
+    ROOT_HELP = "Return the relative path to the root of the DVC project."
     root_parser = subparsers.add_parser(
         "root",
         parents=[parent_parser],
diff --git a/dvc/command/version.py b/dvc/command/version.py
index 539c7b1ef..8356cb402 100644
--- a/dvc/command/version.py
+++ b/dvc/command/version.py
@@ -142,7 +142,8 @@ def _get_dvc_repo_info(repo):
 
 
 def add_parser(subparsers, parent_parser):
-    VERSION_HELP = "Show DVC version and system/environment information."
+    VERSION_HELP = "Display the DVC version"
+    " and system/environment information."
 
     version_parser = subparsers.add_parser(
         "version",
diff --git a/dvc/external_repo.py b/dvc/external_repo.py
index ec48a43f8..a598eb669 100644
--- a/dvc/external_repo.py
+++ b/dvc/external_repo.py
@@ -190,6 +190,7 @@ class ExternalGitRepo:
 
     @contextmanager
     def open_by_relpath(self, path, mode="r", encoding=None, **kwargs):
+        """Opens a specified resource as a file object."""
         try:
             abs_path = os.path.join(self.root_dir, path)
             with open(abs_path, mode, encoding=encoding) as fd:
diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py
index e7db95646..e7f2b4dfd 100644
--- a/dvc/repo/__init__.py
+++ b/dvc/repo/__init__.py
@@ -174,7 +174,20 @@ class Repo(object):
 
     def check_modified_graph(self, new_stages):
         """Generate graph including the new stage to check for errors"""
-        self._collect_graph(self.stages + new_stages)
+        # Building graph might be costly for the ones with many DVC-files,
+        # so we provide this undocumented hack to skip it. See [1] for
+        # more details. The hack can be used as:
+        #
+        #     repo = Repo(...)
+        #     repo._skip_graph_checks = True
+        #     repo.add(...)
+        #
+        # A user should care about not duplicating outs and not adding cycles,
+        # otherwise DVC might have an undefined behaviour.
+        #
+        # [1] https://github.com/iterative/dvc/issues/2671
+        if not getattr(self, "_skip_graph_checks", False):
+            self._collect_graph(self.stages + new_stages)
 
     def collect(self, target, with_deps=False, recursive=False, graph=None):
         import networkx as nx
diff --git a/dvc/scm/git/__init__.py b/dvc/scm/git/__init__.py
index 8f05ad029..e70d6f7b0 100644
--- a/dvc/scm/git/__init__.py
+++ b/dvc/scm/git/__init__.py
@@ -3,6 +3,7 @@
 import logging
 import os
 import yaml
+import shlex
 
 from funcy import cached_property
 from pathspec.patterns import GitWildMatchPattern
@@ -321,11 +322,13 @@ class Git(Base):
         if not self.files_to_track:
             return
 
+        files = " ".join(shlex.quote(path) for path in self.files_to_track)
+
         logger.info(
             "\n"
             "To track the changes with git, run:\n"
             "\n"
-            "\tgit add {files}".format(files=" ".join(self.files_to_track))
+            "\tgit add {files}".format(files=files)
         )
 
     def track_file(self, path):


Test patch:
diff --git a/tests/unit/repo/test_repo.py b/tests/unit/repo/test_repo.py
index 4638ff011..985c113ef 100644
--- a/tests/unit/repo/test_repo.py
+++ b/tests/unit/repo/test_repo.py
@@ -82,3 +82,30 @@ def test_collect_optimization(tmp_dir, dvc, mocker):
     # Should read stage directly instead of collecting the whole graph
     dvc.collect(stage.path)
     dvc.collect_granular(stage.path)
+
+
+def test_skip_graph_checks(tmp_dir, dvc, mocker, run_copy):
+    # See https://github.com/iterative/dvc/issues/2671 for more info
+    mock_collect_graph = mocker.patch("dvc.repo.Repo._collect_graph")
+
+    # sanity check
+    tmp_dir.gen("foo", "foo text")
+    dvc.add("foo")
+    run_copy("foo", "bar")
+    assert mock_collect_graph.called
+
+    # check that our hack can be enabled
+    mock_collect_graph.reset_mock()
+    dvc._skip_graph_checks = True
+    tmp_dir.gen("baz", "baz text")
+    dvc.add("baz")
+    run_copy("baz", "qux")
+    assert not mock_collect_graph.called
+
+    # check that our hack can be disabled
+    mock_collect_graph.reset_mock()
+    dvc._skip_graph_checks = False
+    tmp_dir.gen("quux", "quux text")
+    dvc.add("quux")
+    run_copy("quux", "quuz")
+    assert mock_collect_graph.called
diff --git a/tests/unit/scm/test_scm.py b/tests/unit/scm/test_scm.py
index 513653139..4fb590fc5 100644
--- a/tests/unit/scm/test_scm.py
+++ b/tests/unit/scm/test_scm.py
@@ -37,3 +37,9 @@ class TestScmContext(TestCase):
 
         self.assertEqual(0, self.scm_mock.reset_ignores.call_count)
         self.assertEqual(0, self.scm_mock.remind_to_track.call_count)
+
+
+def test_remind_to_track(scm, caplog):
+    scm.files_to_track = ["fname with spaces.txt", "тест", "foo"]
+    scm.remind_to_track()
+    assert "git add 'fname with spaces.txt' 'тест' foo" in caplog.text