Repository: iterative/dvc

Issue:
dvc add -R dir
Thanks for the great project !

Let's say I have the following workflow:

data/dataset1/dataset1.csv
data/dataset2/dataset2.csv

`dvc add data/dataset1 && git add data/dataset1.dvc` --> creates data/dataset1.dvc and tracks with git
`dvc add data/dataset2 && git add data/dataset2.dvc` --> creates data/dataset2.dvc and tracks with git

Now let's say I modify both dataset1.csv and dataset2.csv I want to add both updates to dvc in a single command
`dvc add data` --> this fails because `data` contains files tracked by git.

Would it be possible to support that use case (i.e. ignore the git-tracked files when running dvc add)


Discussion:
Hi @tdeboissiere !

This happens because `dvc add data` tries to track the directory as a whole and not as a set of files. We should definitely implement something like `dvc add -R dir` which will recursively walk through the directory and add each file in it that is not tracked by git. Turning this issue into a feature request, will be sure to implement this shortly.

Thanks,
Ruslan

Resolving patch:
diff --git a/dvc/cli.py b/dvc/cli.py
--- a/dvc/cli.py
+++ b/dvc/cli.py
@@ -139,6 +139,12 @@ def parse_args(argv=None):
                         parents=[parent_parser],
                         description=ADD_HELP,
                         help=ADD_HELP)
+    add_parser.add_argument(
+                        '-R',
+                        '--recursive',
+                        action='store_true',
+                        default=False,
+                        help='Recursively add each file under the directory.')
     add_parser.add_argument(
                         'targets',
                         nargs='+',
diff --git a/dvc/command/add.py b/dvc/command/add.py
--- a/dvc/command/add.py
+++ b/dvc/command/add.py
@@ -7,7 +7,7 @@ class CmdAdd(CmdBase):
     def run(self):
         for target in self.args.targets:
             try:
-                self.project.add(target)
+                self.project.add(target, recursive=self.args.recursive)
             except DvcException as ex:
                 Logger.error('Failed to add \'{}\''.format(target), ex)
                 return 1
diff --git a/dvc/project.py b/dvc/project.py
--- a/dvc/project.py
+++ b/dvc/project.py
@@ -132,16 +132,36 @@ def _check_output_duplication(self, outs):
                         stages = [o.stage.relpath, out.stage.relpath]
                         raise OutputDuplicationError(o.path, stages)
 
-    def add(self, fname):
-        stage = Stage.loads(project=self,
-                            outs=[fname],
-                            add=True)
+    def add(self, fname, recursive=False):
+        fnames = []
+        if recursive and os.path.isdir(fname):
+            fnames = []
+            for root, dirs, files in os.walk(fname):
+                for f in files:
+                    path = os.path.join(root, f)
+                    if Stage.is_stage_file(path):
+                        continue
+                    if os.path.basename(path) == self.scm.ignore_file():
+                        continue
+                    if self.scm.is_tracked(path):
+                        continue
+                    fnames.append(path)
+        else:
+            fnames = [fname]
 
-        self._check_output_duplication(stage.outs)
+        stages = []
+        for f in fnames:
+            stage = Stage.loads(project=self,
+                                outs=[f],
+                                add=True)
 
-        stage.save()
-        stage.dump()
-        return stage
+            self._check_output_duplication(stage.outs)
+
+            stage.save()
+            stage.dump()
+            stages.append(stage)
+
+        return stages
 
     def remove(self, target, outs_only=False):
         stage = Stage.load(self, target)


Test patch:
diff --git a/tests/test_add.py b/tests/test_add.py
--- a/tests/test_add.py
+++ b/tests/test_add.py
@@ -19,7 +19,10 @@ class TestAdd(TestDvc):
     def test(self):
         md5 = file_md5(self.FOO)[0]
 
-        stage = self.dvc.add(self.FOO)
+        stages = self.dvc.add(self.FOO)
+        self.assertEqual(len(stages), 1)
+        stage = stages[0]
+        self.assertTrue(stage is not None)
 
         self.assertIsInstance(stage, Stage)
         self.assertTrue(os.path.isfile(stage.path))
@@ -40,8 +43,10 @@ def test(self):
         dname = 'directory'
         os.mkdir(dname)
         self.create(os.path.join(dname, 'file'), 'file')
-        stage = self.dvc.add(dname)
-        self.assertNotEqual(stage, None)
+        stages = self.dvc.add(dname)
+        self.assertEqual(len(stages), 1)
+        stage = stages[0]
+        self.assertTrue(stage is not None)
         self.assertEqual(len(stage.deps), 0)
         self.assertEqual(len(stage.outs), 1)
 
@@ -52,12 +57,29 @@ def test(self):
             self.assertTrue('\\' not in info['relpath'])
 
 
+class TestAddDirectoryRecursive(TestDvc):
+    def test(self):
+        stages = self.dvc.add(self.DATA_DIR, recursive=True)
+        self.assertEqual(len(stages), 2)
+
+
+class TestAddCmdDirectoryRecursive(TestDvc):
+    def test(self):
+        ret = main(['add',
+                    '--recursive',
+                    self.DATA_DIR])
+        self.assertEqual(ret, 0)
+
+
 class TestAddDirectoryWithForwardSlash(TestDvc):
     def test(self):
         dname = 'directory/'
         os.mkdir(dname)
         self.create(os.path.join(dname, 'file'), 'file')
-        stage = self.dvc.add(dname)
+        stages = self.dvc.add(dname)
+        self.assertEqual(len(stages), 1)
+        stage = stages[0]
+        self.assertTrue(stage is not None)
         self.assertEquals(os.path.abspath('directory.dvc'), stage.path)
 
 class TestAddTrackedFile(TestDvc):
@@ -78,20 +100,30 @@ def test(self):
         os.mkdir(dname)
         shutil.copyfile(self.FOO, fname)
 
-        self.dvc.add(self.FOO)
-        self.dvc.add(dname)
+        stages = self.dvc.add(self.FOO)
+        self.assertEqual(len(stages), 1)
+        self.assertTrue(stages[0] is not None)
+        stages = self.dvc.add(dname)
+        self.assertEqual(len(stages), 1)
+        self.assertTrue(stages[0] is not None)
 
 
 class TestAddModifiedDir(TestDvc):
     def test(self):
-        self.dvc.add(self.DATA_DIR)
+        stages = self.dvc.add(self.DATA_DIR)
+        self.assertEqual(len(stages), 1)
+        self.assertTrue(stages[0] is not None)
         os.unlink(self.DATA)
-        self.dvc.add(self.DATA_DIR)
+        stages = self.dvc.add(self.DATA_DIR)
+        self.assertEqual(len(stages), 1)
+        self.assertTrue(stages[0] is not None)
 
 
 class TestAddFileInDir(TestDvc):
     def test(self):
-        stage = self.dvc.add(self.DATA_SUB)
+        stages = self.dvc.add(self.DATA_SUB)
+        self.assertEqual(len(stages), 1)
+        stage = stages[0]
         self.assertNotEqual(stage, None)
         self.assertEqual(len(stage.deps), 0)
         self.assertEqual(len(stage.outs), 1)
@@ -104,7 +136,9 @@ def test(self):
         fname = os.path.join(dname, 'foo')
         shutil.copyfile(self.FOO, fname)
 
-        stage = self.dvc.add(fname)
+        stages = self.dvc.add(fname)
+        self.assertEqual(len(stages), 1)
+        stage = stages[0]
         self.assertNotEqual(stage, None)
         self.assertEqual(len(stage.deps), 0)
         self.assertEqual(len(stage.outs), 1)
diff --git a/tests/test_checkout.py b/tests/test_checkout.py
--- a/tests/test_checkout.py
+++ b/tests/test_checkout.py
@@ -16,7 +16,11 @@ class TestCheckout(TestRepro):
     def setUp(self):
         super(TestCheckout, self).setUp()
 
-        self.data_dir_stage = self.dvc.add(self.DATA_DIR)
+
+        stages = self.dvc.add(self.DATA_DIR)
+        self.assertEqual(len(stages), 1)
+        self.data_dir_stage = stages[0]
+        self.assertTrue(self.data_dir_stage is not None)
 
         self.orig = 'orig'
         shutil.copy(self.FOO, self.orig)
@@ -74,7 +78,9 @@ class CheckoutBase(TestDvc):
     def commit_data_file(self, fname, content='random text'):
         with open(fname, 'w') as fd:
             fd.write(content)
-        self.dvc.add(fname)
+        stages = self.dvc.add(fname)
+        self.assertEqual(len(stages), 1)
+        self.assertTrue(stages[0] is not None)
         self.dvc.scm.add([fname + '.dvc', '.gitignore'])
         self.dvc.scm.commit('adding ' + fname)
 
@@ -181,7 +187,10 @@ def test(self):
         dname = 'empty_dir'
         os.mkdir(dname)
 
-        stage = self.dvc.add(dname)
+        stages = self.dvc.add(dname)
+        self.assertEqual(len(stages), 1)
+        stage = stages[0]
+        self.assertTrue(stage is not None)
         self.assertEqual(len(stage.outs), 1)
 
         stage.outs[0].remove()
diff --git a/tests/test_data_cloud.py b/tests/test_data_cloud.py
--- a/tests/test_data_cloud.py
+++ b/tests/test_data_cloud.py
@@ -204,12 +204,19 @@ def _setup_cloud(self):
     def _test_cloud(self):
         self._setup_cloud()
 
-        stage = self.dvc.add(self.FOO)
+        stages = self.dvc.add(self.FOO)
+        self.assertEqual(len(stages), 1)
+        stage = stages[0]
+        self.assertTrue(stage is not None)
         cache = stage.outs[0].cache
         info = stage.outs[0].dumpd()
         md5 = info['md5']
 
-        stage_dir = self.dvc.add(self.DATA_DIR)
+        stages = self.dvc.add(self.DATA_DIR)
+        self.assertEqual(len(stages), 1)
+        stage_dir = stages[0]
+        self.assertTrue(stage_dir is not None)
+
         cache_dir = stage_dir.outs[0].cache
         info_dir = stage_dir.outs[0].dumpd()
         md5_dir = info_dir['md5']
@@ -358,10 +365,16 @@ def _test_cloud(self, remote=None):
         else:
             args += []
 
-        stage = self.dvc.add(self.FOO)
+        stages = self.dvc.add(self.FOO)
+        self.assertEqual(len(stages), 1)
+        stage = stages[0]
+        self.assertTrue(stage is not None)
         cache = stage.outs[0].cache
 
-        stage_dir = self.dvc.add(self.DATA_DIR)
+        stages = self.dvc.add(self.DATA_DIR)
+        self.assertEqual(len(stages), 1)
+        stage_dir = stages[0]
+        self.assertTrue(stage_dir is not None)
         cache_dir = stage_dir.outs[0].cache
 
         # FIXME check status output
diff --git a/tests/test_metrics.py b/tests/test_metrics.py
--- a/tests/test_metrics.py
+++ b/tests/test_metrics.py
@@ -219,7 +219,9 @@ def _do_write(self, branch):
         with open('metrics.json', 'w+') as fd:
             json.dump({'metrics': branch}, fd)
 
-        stage = self.dvc.add('metrics.json')
+        stages = self.dvc.add('metrics.json')
+        self.assertEqual(len(stages), 1)
+        stage = stages[0]
         self.assertNotEqual(stage, None)
 
         self.dvc.scm.add(['.gitignore', 'metrics.json.dvc'])
diff --git a/tests/test_remove.py b/tests/test_remove.py
--- a/tests/test_remove.py
+++ b/tests/test_remove.py
@@ -9,7 +9,10 @@
 
 class TestRemove(TestDvc):
     def test(self):
-        stage = self.dvc.add(self.FOO)
+        stages = self.dvc.add(self.FOO)
+        self.assertEqual(len(stages), 1)
+        stage = stages[0]
+        self.assertTrue(stage is not None)
         stage_removed = self.dvc.remove(stage.path, outs_only=True)
 
         self.assertIsInstance(stage_removed, Stage)
@@ -31,7 +34,10 @@ def test(self):
 
 class TestRemoveDirectory(TestDvc):
     def test(self):
-        stage_add = self.dvc.add(self.DATA_DIR)
+        stages = self.dvc.add(self.DATA_DIR)
+        self.assertEqual(len(stages), 1)
+        stage_add = stages[0]
+        self.assertTrue(stage_add is not None)
         stage_removed = self.dvc.remove(stage_add.path)
         self.assertEqual(stage_add.path, stage_removed.path)
         self.assertFalse(os.path.exists(self.DATA_DIR))
@@ -40,7 +46,10 @@ def test(self):
 
 class TestCmdRemove(TestDvc):
     def test(self):
-        stage = self.dvc.add(self.FOO)
+        stages = self.dvc.add(self.FOO)
+        self.assertEqual(len(stages), 1)
+        stage = stages[0]
+        self.assertTrue(stage is not None)
         ret = main(['remove',
                     stage.path])
         self.assertEqual(ret, 0)
diff --git a/tests/test_repro.py b/tests/test_repro.py
--- a/tests/test_repro.py
+++ b/tests/test_repro.py
@@ -30,7 +30,10 @@ class TestRepro(TestDvc):
     def setUp(self):
         super(TestRepro, self).setUp()
 
-        self.foo_stage = self.dvc.add(self.FOO)
+        stages = self.dvc.add(self.FOO)
+        self.assertEqual(len(stages), 1)
+        self.foo_stage = stages[0]
+        self.assertTrue(self.foo_stage is not None)
 
         self.file1 = 'file1'
         self.file1_stage = self.file1 + '.dvc'
@@ -50,7 +53,10 @@ def test(self):
 
 class TestReproDepUnderDir(TestDvc):
     def test(self):
-        self.dir_stage = self.dvc.add(self.DATA_DIR)
+        stages = self.dvc.add(self.DATA_DIR)
+        self.assertEqual(len(stages), 1)
+        self.dir_stage = stages[0]
+        self.assertTrue(self.dir_stage is not None)
 
         sleep()