Repository: iterative/dvc

Issue:
add: Suggest to use `dvc add folder/` instead of irregular `dvc add -R` for large amount of files
Related #1756


Discussion:
Or maybe it is worth renaming it into something else than `--recursive` to break association with `cp`.
But is it needed at all?
@ei-grad that is also a great question :) We do have a user that indirectly requested it https://github.com/iterative/dvc/issues/1026 . Need to re-think this case.
That one could be better solved by `dvc add -u` as of my opinion, to be honest. But anyway just removing the `-R` option is not an option now.
The point is more about to don't touch the `dvc add -R`, but add a suggestion to use `dvc add folder` where it worth.
Could you elaborate on `dvc add -u`, please?

I agree about not removing it. I just threw in a thought that if it is indeed needed, maybe it is worth thinking about renaming it. A suggestion to use `dvc add directory` also seems good to me.
>>> We do have a user that indirectly requested it #1026

>> That one could be better solved by dvc add -u

> Could you elaborate on dvc add -u, please?

The actual request in #1026 was to provide an ability to `dvc add` all the changed files in one command. This is what `git add -u` does for git repository.

Resolving patch:
diff --git a/dvc/repo/add.py b/dvc/repo/add.py
--- a/dvc/repo/add.py
+++ b/dvc/repo/add.py
@@ -1,11 +1,16 @@
 import os
+import logging
+import colorama
 
 from dvc.repo.scm_context import scm_context
 from dvc.stage import Stage
-from dvc.utils import walk_files
+from dvc.utils import walk_files, LARGE_DIR_SIZE
 from dvc.exceptions import RecursiveAddingWhileUsingFilename
 
 
+logger = logging.getLogger(__name__)
+
+
 @scm_context
 def add(repo, target, recursive=False, no_commit=False, fname=None):
     if recursive and fname:
@@ -13,6 +18,19 @@ def add(repo, target, recursive=False, no_commit=False, fname=None):
 
     targets = _find_all_targets(repo, target, recursive)
 
+    if os.path.isdir(target) and len(targets) > LARGE_DIR_SIZE:
+        logger.warning(
+            "You are adding a large directory '{target}' recursively,"
+            " consider tracking it as a whole instead.\n"
+            "{purple}HINT:{nc} Remove the generated stage files and then"
+            " run {cyan}dvc add {target}{nc}".format(
+                purple=colorama.Fore.MAGENTA,
+                cyan=colorama.Fore.CYAN,
+                nc=colorama.Style.RESET_ALL,
+                target=target,
+            )
+        )
+
     stages = _create_stages(repo, targets, fname, no_commit)
 
     repo.check_dag(repo.stages() + stages)
diff --git a/dvc/utils/compat.py b/dvc/utils/compat.py
--- a/dvc/utils/compat.py
+++ b/dvc/utils/compat.py
@@ -108,6 +108,7 @@ def _makedirs(name, mode=0o777, exist_ok=False):
     input = raw_input  # noqa: F821
     cast_bytes_py2 = cast_bytes
     makedirs = _makedirs
+    range = xrange  # noqa: F821
 
     import StringIO
     import io
@@ -148,3 +149,4 @@ def __exit__(self, *args):
     input = input  # noqa: F821
     open = open  # noqa: F821
     cast_bytes_py2 = no_code
+    range = range  # noqa: F821


Test patch:
diff --git a/tests/func/test_add.py b/tests/func/test_add.py
--- a/tests/func/test_add.py
+++ b/tests/func/test_add.py
@@ -7,13 +7,16 @@
 import shutil
 import filecmp
 import posixpath
+import logging
+import colorama
 
 from dvc.system import System
 from mock import patch
 
 from dvc.main import main
-from dvc.utils import file_md5
+from dvc.utils import file_md5, LARGE_DIR_SIZE
 from dvc.utils.stage import load_stage_file
+from dvc.utils.compat import range
 from dvc.stage import Stage
 from dvc.exceptions import DvcException, RecursiveAddingWhileUsingFilename
 from dvc.output.base import OutputAlreadyTrackedError
@@ -87,6 +90,30 @@ def test(self):
         ret = main(["add", "--recursive", self.DATA_DIR])
         self.assertEqual(ret, 0)
 
+    def test_warn_about_large_directories(self):
+        warning = (
+            "You are adding a large directory 'large-dir' recursively,"
+            " consider tracking it as a whole instead.\n"
+            "{purple}HINT:{nc} Remove the generated stage files and then"
+            " run {cyan}dvc add large-dir{nc}".format(
+                purple=colorama.Fore.MAGENTA,
+                cyan=colorama.Fore.CYAN,
+                nc=colorama.Style.RESET_ALL,
+            )
+        )
+
+        os.mkdir("large-dir")
+
+        # Create a lot of files
+        for iteration in range(LARGE_DIR_SIZE + 1):
+            path = os.path.join("large-dir", str(iteration))
+            with open(path, "w") as fobj:
+                fobj.write(path)
+
+        with self._caplog.at_level(logging.WARNING, logger="dvc"):
+            assert main(["add", "--recursive", "large-dir"]) == 0
+            assert warning in self._caplog.text
+
 
 class TestAddDirectoryWithForwardSlash(TestDvc):
     def test(self):