Repository: iterative/dvc

Issue:
Data sources: how to import file in a new repo
When I clone DVC repo and do `repro` my source file does not exist and I see a strange message from the next stage:

```
Failed to reproduce 'Posts.xml.dvc': Exec command error (tar zxf data/Posts.xml.tgz -C data/):

tar: Error opening archive: Failed to open 'data/Posts.xml.tgz'
```

How to fix - add data source file back:

```
$ dvc remove data/Posts.xml.tgz
$ cp ../stackoverflow_small_xml/100K/Posts.xml.tgz data/
$ dvc add data/Posts.xml.tgz # this command is actually not required!!!
```

What to do? Basically, you should add the source file again. Let's just write this in an error message: `Failed to reproduce 'Posts.xml.dvc': data source file data/Posts.xml.tgz does not exist. Copy source file in this place and run 'dvc add data/Posts.xml.tgz'`

Questions:
1. Do we need a special support for this scenario like `dvc add --source-stage data/Posts.xml.tgz ~/Download/reuters-dataset.zip` where a file from a random place will be correctly deployed into a correct place?
2. why should we ask to add the file if it works without this command?


Discussion:
It still does not look good:
```
Failed to reproduce 'data/Posts.xml.tgz.dvc': Output file 'data/Posts.xml.tgz' error: does not exist
```

Output file does not exist - is what I'd expect from any stage. The only exception is data source stages. We should to distinguish these things and provide a proper error message.
Update: ideally it should be clear that you require to put a data source file in the right place.

Resolving patch:
diff --git a/dvc/executor.py b/dvc/executor.py
--- a/dvc/executor.py
+++ b/dvc/executor.py
@@ -58,6 +58,5 @@ def exec_cmd_only_success(cmd, stdout_file=None, stderr_file=None, cwd=None, she
         code, out, err = Executor.exec_cmd(cmd, stdout_file=stdout_file,
                                            stderr_file=stderr_file, cwd=cwd, shell=shell)
         if code != 0:
-            cmd_str = ' '.join(cmd) if type(cmd) == list else cmd
-            raise ExecutorError('Exec command error ({}):\n{}\n{}'.format(cmd_str, out, err))
+            raise ExecutorError('Exec command error:\n{}'.format(err))
         return out
diff --git a/dvc/stage.py b/dvc/stage.py
--- a/dvc/stage.py
+++ b/dvc/stage.py
@@ -10,33 +10,49 @@
 
 
 class OutputError(DvcException):
+    pass
+
+
+class MissingDataSource(OutputError):
+    def __init__(self, missing_files):
+        assert len(missing_files) > 0
+
+        source = 'source'
+        if len(missing_files) > 1:
+            source += 's'
+
+        msg = u'missing data {}: {}'.format(source, ', '.join(missing_files))
+        super(MissingDataSource, self).__init__(msg)
+
+
+class CmdOutputError(DvcException):
     def __init__(self, path, msg):
-        super(OutputError, self).__init__('Output file \'{}\' error: {}'.format(path, msg))
+        super(CmdOutputError, self).__init__('Output file \'{}\' error: {}'.format(path, msg))
 
 
-class OutputNoCacheError(OutputError):
+class CmdOutputNoCacheError(CmdOutputError):
     def __init__(self, path):
-        super(OutputNoCacheError, self).__init__(path, 'no cache')
+        super(CmdOutputNoCacheError, self).__init__(path, 'no cache')
 
 
-class OutputOutsideOfRepoError(OutputError):
+class CmdOutputOutsideOfRepoError(CmdOutputError):
     def __init__(self, path):
-        super(OutputOutsideOfRepoError, self).__init__(path, 'outside of repository')
+        super(CmdOutputOutsideOfRepoError, self).__init__(path, 'outside of repository')
 
 
-class OutputDoesNotExistError(OutputError):
+class CmdOutputDoesNotExistError(CmdOutputError):
     def __init__(self, path):
-        super(OutputDoesNotExistError, self).__init__(path, 'does not exist')
+        super(CmdOutputDoesNotExistError, self).__init__(path, 'does not exist')
 
 
-class OutputIsNotFileError(OutputError):
+class CmdOutputIsNotFileError(CmdOutputError):
     def __init__(self, path):
-        super(OutputIsNotFileError, self).__init__(path, 'not a file')
+        super(CmdOutputIsNotFileError, self).__init__(path, 'not a file')
 
 
-class OutputAlreadyTrackedError(OutputError):
+class CmdOutputAlreadyTrackedError(CmdOutputError):
     def __init__(self, path):
-        super(OutputAlreadyTrackedError, self).__init__(path, 'already tracked by scm(e.g. git)')
+        super(CmdOutputAlreadyTrackedError, self).__init__(path, 'already tracked by scm(e.g. git)')
 
 
 class Dependency(object):
@@ -48,7 +64,7 @@ def __init__(self, project, path, md5=None):
         self.path = os.path.abspath(os.path.realpath(path))
 
         if not self.path.startswith(self.project.root_dir):
-            raise OutputOutsideOfRepoError(self.path)
+            raise CmdOutputOutsideOfRepoError(self.path)
 
         self.md5 = md5
 
@@ -56,6 +72,10 @@ def __init__(self, project, path, md5=None):
     def dvc_path(self):
         return os.path.relpath(self.path, self.project.root_dir)
 
+    @property
+    def rel_path(self):
+        return os.path.relpath(self.path, '.')
+
     def _changed_md5(self):
         if not os.path.exists(self.path):
             return True
@@ -76,9 +96,9 @@ def mtime(self):
 
     def update(self):
         if not os.path.exists(self.path):
-            raise OutputDoesNotExistError(self.path)
+            raise CmdOutputDoesNotExistError(self.rel_path)
         if not os.path.isfile(self.path):
-            raise OutputIsNotFileError(self.path)
+            raise CmdOutputIsNotFileError(self.path)
 
         state = self.project.state.get(self.path)
         if state and state.mtime == self.mtime():
@@ -164,10 +184,10 @@ def changed(self):
 
     def link(self, checkout=False):
         if not self.use_cache:
-            raise OutputNoCacheError(self.path)
+            raise CmdOutputNoCacheError(self.path)
 
         if not os.path.exists(self.path) and not os.path.exists(self.cache):
-            raise OutputNoCacheError(self.path)
+            raise CmdOutputNoCacheError(self.path)
 
         if os.path.exists(self.path) and \
            os.path.exists(self.cache) and \
@@ -187,7 +207,7 @@ def link(self, checkout=False):
             src = self.path
             link = self.cache
         else:
-            raise OutputNoCacheError(self.path)
+            raise CmdOutputNoCacheError(self.path)
 
         System.hardlink(src, link)
 
@@ -207,7 +227,7 @@ def save(self):
             return
 
         if self.project.scm.is_tracked(self.path):
-            raise OutputAlreadyTrackedError(self.path)
+            raise CmdOutputAlreadyTrackedError(self.path)
 
         self.project.scm.ignore(self.path)
         self.link()
@@ -249,6 +269,10 @@ def relpath(self):
     def dvc_path(self):
         return os.path.relpath(self.path, self.project.root_dir)
 
+    @property
+    def is_data_source(self):
+        return self.cmd is None
+
     @staticmethod
     def is_stage_file(path):
         if not os.path.isfile(path):
@@ -282,9 +306,7 @@ def reproduce(self, force=False):
             # Removing outputs only if we actually have command to reproduce
             self.remove_outs()
 
-        self.project.logger.info("Reproducing {}:\n\t{}".format(self.relpath, self.cmd))
         self.run()
-        self.project.logger.debug("{} was reproduced".format(self.relpath))
 
     @staticmethod
     def loadd(project, d, path):
@@ -335,9 +357,22 @@ def save(self):
             out.save()
 
     def run(self):
-        if self.cmd:
+        if not self.is_data_source:
+            self.project.logger.info(u'Reproducing \'{}\':\n\t{}'.format(self.relpath, self.cmd))
+
             Executor.exec_cmd_only_success(self.cmd, cwd=str(self.cwd), shell=True)
-        self.save()
+            self.save()
+
+            self.project.logger.debug(u'\'{}\' was reproduced'.format(self.relpath))
+        else:
+            self.project.logger.info(u'Verifying data sources in \'{}\''.format(self.relpath))
+            self.check_missing_outputs()
+            self.save()
+
+    def check_missing_outputs(self):
+        missing_outs = [out.rel_path for out in self.outs if not os.path.exists(out.rel_path)]
+        if missing_outs:
+            raise MissingDataSource(missing_outs)
 
     def checkout(self):
         for out in self.outs:


Test patch:
diff --git a/tests/test_add.py b/tests/test_add.py
--- a/tests/test_add.py
+++ b/tests/test_add.py
@@ -4,9 +4,9 @@
 
 from dvc.main import main
 from dvc.data_cloud import file_md5
-from dvc.stage import Stage, OutputNoCacheError, OutputOutsideOfRepoError
-from dvc.stage import OutputDoesNotExistError, OutputIsNotFileError
-from dvc.stage import OutputAlreadyTrackedError
+from dvc.stage import Stage, CmdOutputNoCacheError, CmdOutputOutsideOfRepoError
+from dvc.stage import CmdOutputDoesNotExistError, CmdOutputIsNotFileError
+from dvc.stage import CmdOutputAlreadyTrackedError
 from dvc.project import StageNotFoundError
 from dvc.command.add import CmdAdd
 
@@ -30,13 +30,13 @@ def test(self):
 
 class TestAddNonExistentFile(TestDvc):
     def test(self):
-        with self.assertRaises(OutputDoesNotExistError) as cx:
+        with self.assertRaises(CmdOutputDoesNotExistError) as cx:
             self.dvc.add('non_existent_file')
 
 
 class TestAddFileOutsideOfRepo(TestDvc):
     def test(self):
-        with self.assertRaises(OutputOutsideOfRepoError) as cx:
+        with self.assertRaises(CmdOutputOutsideOfRepoError) as cx:
             self.dvc.add(os.path.join(os.path.dirname(self.dvc.root_dir), self.FOO))
 
 
@@ -44,7 +44,7 @@ class TestAddDirectory(TestDvc):
     def test(self):
         dname = 'directory'
         os.mkdir(dname)
-        with self.assertRaises(OutputIsNotFileError) as cx:
+        with self.assertRaises(CmdOutputIsNotFileError) as cx:
             self.dvc.add(dname)
 
 
@@ -55,7 +55,7 @@ def test(self):
         self.dvc.scm.add([fname])
         self.dvc.scm.commit('add {}'.format(fname))
 
-        with self.assertRaises(OutputAlreadyTrackedError) as cx:
+        with self.assertRaises(CmdOutputAlreadyTrackedError) as cx:
             self.dvc.add(fname)
 
 
diff --git a/tests/test_checkout.py b/tests/test_checkout.py
--- a/tests/test_checkout.py
+++ b/tests/test_checkout.py
@@ -5,8 +5,8 @@
 
 from dvc.main import main
 from dvc.data_cloud import file_md5
-from dvc.stage import Stage, OutputNoCacheError, OutputOutsideOfRepoError
-from dvc.stage import OutputDoesNotExistError, OutputIsNotFileError
+from dvc.stage import Stage, CmdOutputNoCacheError, CmdOutputOutsideOfRepoError
+from dvc.stage import CmdOutputDoesNotExistError, CmdOutputIsNotFileError
 from dvc.project import StageNotFoundError
 
 from tests.test_repro import TestRepro
diff --git a/tests/test_gc.py b/tests/test_gc.py
--- a/tests/test_gc.py
+++ b/tests/test_gc.py
@@ -4,8 +4,8 @@
 
 from dvc.main import main
 from dvc.data_cloud import file_md5
-from dvc.stage import Stage, OutputNoCacheError, OutputOutsideOfRepoError
-from dvc.stage import OutputDoesNotExistError, OutputIsNotFileError
+from dvc.stage import Stage, CmdOutputNoCacheError, CmdOutputOutsideOfRepoError
+from dvc.stage import CmdOutputDoesNotExistError, CmdOutputIsNotFileError
 from dvc.project import StageNotFoundError
 from dvc.command.gc import CmdGC
 
diff --git a/tests/test_remove.py b/tests/test_remove.py
--- a/tests/test_remove.py
+++ b/tests/test_remove.py
@@ -1,7 +1,7 @@
 import os
 
 from dvc.main import main
-from dvc.stage import Stage, OutputOutsideOfRepoError
+from dvc.stage import Stage, CmdOutputOutsideOfRepoError
 from dvc.project import StageNotFoundError
 from dvc.command.remove import CmdRemove
 
@@ -26,7 +26,7 @@ def test(self):
 
 class TestRemoveFileOutsideOfRepo(TestDvc):
     def test(self):
-        with self.assertRaises(OutputOutsideOfRepoError) as cx:
+        with self.assertRaises(CmdOutputOutsideOfRepoError) as cx:
             self.dvc.remove(os.path.join(os.path.dirname(self.dvc.root_dir), self.FOO))
 
 
diff --git a/tests/test_repro.py b/tests/test_repro.py
--- a/tests/test_repro.py
+++ b/tests/test_repro.py
@@ -6,6 +6,7 @@
 from dvc.main import main
 from dvc.command.repro import CmdRepro
 from dvc.project import ReproductionError
+from dvc.data_cloud import file_md5
 
 from tests.basic_env import TestDvc
 
@@ -14,7 +15,8 @@ class TestRepro(TestDvc):
     def setUp(self):
         super(TestRepro, self).setUp()
 
-        self.dvc.add(self.FOO)
+        self.foo_stage = self.dvc.add(self.FOO)
+
         self.file1 = 'file1'
         self.file1_stage = self.file1 + '.dvc'
         self.dvc.run(fname=self.file1_stage,
@@ -85,6 +87,16 @@ def test(self):
             self.dvc.reproduce(self.file1_stage)
 
 
+class TestReproDataSource(TestReproChangedData):
+    def test(self):
+        self.swap_foo_with_bar()
+
+        stages = self.dvc.reproduce(self.foo_stage.path)
+
+        self.assertTrue(filecmp.cmp(self.FOO, self.BAR))
+        self.assertEqual(stages[0].outs[0].md5, file_md5(self.BAR)[0])
+
+
 class TestCmdRepro(TestRepro):
     def test(self):
         ret = main(['repro',