Repository: iterative/dvc
Pull request #1463 (2019-01-02 13:41:37)

 local: support remote:// syntax


Patch:
diff --git a/dvc/dependency/base.py b/dvc/dependency/base.py
index 7a519f9cf..a1224ac7a 100644
--- a/dvc/dependency/base.py
+++ b/dvc/dependency/base.py
@@ -20,26 +20,27 @@ class DependencyBase(object):
 
     PARAM_PATH = 'path'
 
-    def __init__(self, stage, path):
+    def __init__(self, stage, path, info=None):
         self.stage = stage
         self.project = stage.project
-        self.path = path
+        self.url = path
+        self.info = info
 
     def __repr__(self):
-        return "{class_name}: '{path}'".format(
+        return "{class_name}: '{url}'".format(
             class_name=type(self).__name__,
-            path=(self.path or 'No path')
+            url=(self.url or 'No url')
         )
 
     def __str__(self):
-        return self.path
+        return self.url
 
     @classmethod
     def match(cls, url):
         return re.match(cls.REGEX, url)
 
     def group(self, name):
-        match = self.match(self.path)
+        match = self.match(self.url)
         if not match:
             return None
         return match.group(name)
@@ -48,6 +49,14 @@ class DependencyBase(object):
     def supported(cls, url):
         return cls.match(url) is not None
 
+    @property
+    def scheme(self):
+        return self.path_info['scheme']
+
+    @property
+    def path(self):
+        return self.path_info['path']
+
     @property
     def sep(self):
         return '/'
@@ -62,18 +71,16 @@ class DependencyBase(object):
     def status(self):
         if self.changed():
             # FIXME better msgs
-            if self.path_info['scheme'] == 'local':
-                p = self.rel_path
-            else:
-                p = self.path
-            return {p: 'changed'}
+            return {str(self): 'changed'}
         return {}
 
     def save(self):
         raise NotImplementedError
 
     def dumpd(self):
-        return {self.PARAM_PATH: self.path}
+        ret = self.info.copy()
+        ret[self.PARAM_PATH] = self.url
+        return ret
 
     def download(self, to_info):
         self.remote.download([self.path_info], [to_info])
diff --git a/dvc/dependency/gs.py b/dvc/dependency/gs.py
index 6f31f3d9e..0bcc14d70 100644
--- a/dvc/dependency/gs.py
+++ b/dvc/dependency/gs.py
@@ -13,8 +13,7 @@ class DependencyGS(DependencyS3):
     REGEX = RemoteGS.REGEX
 
     def __init__(self, stage, path, info=None, remote=None):
-        super(DependencyGS, self).__init__(stage, path)
-        self.info = info
+        super(DependencyGS, self).__init__(stage, path, info=info)
         self.remote = remote if remote else RemoteGS(stage.project, {})
         bucket = remote.bucket if remote else urlparse(path).netloc
         path = urlparse(path).path.lstrip('/')
diff --git a/dvc/dependency/hdfs.py b/dvc/dependency/hdfs.py
index 81506ba97..dc4a7c41a 100644
--- a/dvc/dependency/hdfs.py
+++ b/dvc/dependency/hdfs.py
@@ -13,8 +13,7 @@ class DependencyHDFS(DependencyBase):
     REGEX = RemoteHDFS.REGEX
 
     def __init__(self, stage, path, info=None, remote=None):
-        super(DependencyHDFS, self).__init__(stage, path)
-        self.info = info
+        super(DependencyHDFS, self).__init__(stage, path, info=info)
         self.remote = remote if remote else RemoteHDFS(stage.project, {})
         if remote:
             path = posixpath.join(remote.url, urlparse(path).path.lstrip('/'))
@@ -31,8 +30,3 @@ class DependencyHDFS(DependencyBase):
 
     def save(self):
         self.info = self.remote.save_info(self.path_info)
-
-    def dumpd(self):
-        ret = self.info
-        ret[self.PARAM_PATH] = self.path
-        return ret
diff --git a/dvc/dependency/http.py b/dvc/dependency/http.py
index a2cccf454..df2951ae9 100644
--- a/dvc/dependency/http.py
+++ b/dvc/dependency/http.py
@@ -11,9 +11,8 @@ class DependencyHTTP(DependencyBase):
     REGEX = RemoteHTTP.REGEX
 
     def __init__(self, stage, path, info=None, remote=None):
-        super(DependencyHTTP, self).__init__(stage, path)
+        super(DependencyHTTP, self).__init__(stage, path, info=info)
 
-        self.info = info
         self.remote = remote or RemoteHTTP(stage.project, {})
 
         if path.startswith('remote'):
@@ -27,11 +26,6 @@ class DependencyHTTP(DependencyBase):
     def save(self):
         self.info = self.remote.save_info(self.path_info)
 
-    def dumpd(self):
-        ret = self.info
-        ret[self.PARAM_PATH] = self.path
-        return ret
-
     def changed(self):
         if not self.exists:
             return True
diff --git a/dvc/dependency/local.py b/dvc/dependency/local.py
index f944e5e6a..4ba7e9bdf 100644
--- a/dvc/dependency/local.py
+++ b/dvc/dependency/local.py
@@ -19,33 +19,33 @@ class DependencyLOCAL(DependencyBase):
     IsNotFileOrDirError = DependencyIsNotFileOrDirError
 
     def __init__(self, stage, path, info=None, remote=None):
-        self.stage = stage
-        self.project = stage.project
-        self.info = info
+        super(DependencyLOCAL, self).__init__(stage, path, info)
         if remote is not None:
             self.remote = remote
         else:
             self.remote = RemoteLOCAL(stage.project, {})
 
         if remote:
-            path = os.path.join(remote.prefix, urlparse(path).path.lstrip('/'))
+            p = os.path.join(remote.prefix,
+                             urlparse(self.url).path.lstrip('/'))
+        else:
+            p = path
 
-        if not os.path.isabs(path):
-            path = self.remote.to_ospath(path)
-            path = os.path.join(stage.cwd, path)
-        self.path = os.path.abspath(os.path.normpath(path))
+        if not os.path.isabs(p):
+            p = self.remote.to_ospath(p)
+            p = os.path.join(stage.cwd, p)
+        p = os.path.abspath(os.path.normpath(p))
 
         self.path_info = {'scheme': 'local',
-                          'path': self.path}
+                          'path': p}
 
     def __str__(self):
         return self.rel_path
 
     @property
     def is_local(self):
-        assert os.path.isabs(self.path)
-        assert os.path.isabs(self.project.root_dir)
-        return self.path.startswith(self.project.root_dir)
+        return (urlparse(self.url).scheme != 'remote'
+                and not os.path.isabs(self.url))
 
     @property
     def sep(self):
@@ -83,7 +83,7 @@ class DependencyLOCAL(DependencyBase):
             path = self.remote.unixpath(os.path.relpath(self.path,
                                                         self.stage.cwd))
         else:
-            path = self.path
+            path = self.url
 
         info = self.info.copy()
         info[self.PARAM_PATH] = path
diff --git a/dvc/dependency/s3.py b/dvc/dependency/s3.py
index d2f0c4eef..62b9330bc 100644
--- a/dvc/dependency/s3.py
+++ b/dvc/dependency/s3.py
@@ -13,8 +13,7 @@ class DependencyS3(DependencyBase):
     REGEX = RemoteS3.REGEX
 
     def __init__(self, stage, path, info=None, remote=None):
-        super(DependencyS3, self).__init__(stage, path)
-        self.info = info
+        super(DependencyS3, self).__init__(stage, path, info=info)
         self.remote = remote or RemoteS3(stage.project, {})
 
         bucket = remote.bucket if remote else urlparse(path).netloc
@@ -33,8 +32,3 @@ class DependencyS3(DependencyBase):
 
     def save(self):
         self.info = self.remote.save_info(self.path_info)
-
-    def dumpd(self):
-        ret = self.info
-        ret[self.PARAM_PATH] = self.path
-        return ret
diff --git a/dvc/dependency/ssh.py b/dvc/dependency/ssh.py
index f4463213f..2f25a9707 100644
--- a/dvc/dependency/ssh.py
+++ b/dvc/dependency/ssh.py
@@ -13,8 +13,7 @@ class DependencySSH(DependencyBase):
     REGEX = RemoteSSH.REGEX
 
     def __init__(self, stage, path, info=None, remote=None):
-        super(DependencySSH, self).__init__(stage, path)
-        self.info = info
+        super(DependencySSH, self).__init__(stage, path, info=info)
         self.remote = remote if remote else RemoteSSH(stage.project, {})
 
         host = remote.host if remote else self.group('host')
@@ -24,7 +23,7 @@ class DependencySSH(DependencyBase):
             path = posixpath.join(remote.prefix,
                                   urlparse(path).path.lstrip('/'))
         else:
-            path = self.match(self.path).group('path')
+            path = self.match(self.url).group('path')
 
         self.path_info = {'scheme': 'ssh',
                           'host': host,
@@ -40,8 +39,3 @@ class DependencySSH(DependencyBase):
 
     def save(self):
         self.info = self.remote.save_info(self.path_info)
-
-    def dumpd(self):
-        ret = self.info
-        ret[self.PARAM_PATH] = self.path
-        return ret
diff --git a/dvc/output/local.py b/dvc/output/local.py
index eda2aa6b0..bd9ff504d 100644
--- a/dvc/output/local.py
+++ b/dvc/output/local.py
@@ -127,7 +127,7 @@ class OutputLOCAL(DependencyLOCAL):
             self.project.scm.ignore_remove(self.path)
 
         self.remote.move(self.path_info, out.path_info)
-        self.path = out.path
+        self.url = out.url
         self.path_info = out.path_info
         self.save()
 
diff --git a/dvc/project.py b/dvc/project.py
index 5eebf9d39..a6f605770 100644
--- a/dvc/project.py
+++ b/dvc/project.py
@@ -653,7 +653,7 @@ class Project(object):
         info['branch'] = branch
         ret = [info]
 
-        if out.path_info['scheme'] != 'local':
+        if out.scheme != 'local':
             return ret
 
         md5 = info[out.remote.PARAM_CHECKSUM]
@@ -1098,7 +1098,7 @@ class Project(object):
             msg = "unable to find file '{}' in the pipeline".format(path)
             raise DvcException(msg)
 
-        if out.path_info['scheme'] != 'local':
+        if out.scheme != 'local':
             msg = "output '{}' scheme '{}' is not supported for metrics"
             raise DvcException(msg.format(out.path, out.path_info['scheme']))
 
diff --git a/dvc/stage.py b/dvc/stage.py
index 59da076c1..73e0abfee 100644
--- a/dvc/stage.py
+++ b/dvc/stage.py
@@ -224,7 +224,7 @@ class Stage(object):
 
     def unprotect_outs(self):
         for out in self.outs:
-            if out.path_info['scheme'] != 'local' or not out.exists:
+            if out.scheme != 'local' or not out.exists:
                 continue
             self.project.unprotect(out.path)
 
@@ -292,7 +292,7 @@ class Stage(object):
             return (cls.STAGE_FILE, cwd if cwd else os.getcwd())
 
         out = outs[0]
-        if out.path_info['scheme'] == 'local':
+        if out.scheme == 'local':
             path = os.path
         else:
             path = posixpath
@@ -583,7 +583,7 @@ class Stage(object):
 
     def check_missing_outputs(self):
         outs = [out for out in self.outs if not out.exists]
-        paths = [out.path if out.path_info['scheme'] != 'local' else
+        paths = [out.path if out.scheme != 'local' else
                  out.rel_path for out in outs]
         if paths:
             raise MissingDataSource(paths)


Test patch:
diff --git a/tests/test_add.py b/tests/test_add.py
index 444264ce7..e7621b116 100644
--- a/tests/test_add.py
+++ b/tests/test_add.py
@@ -1,4 +1,5 @@
 import os
+import yaml
 import time
 import shutil
 import filecmp
@@ -8,6 +9,7 @@ from dvc.utils import file_md5
 from dvc.stage import Stage
 from dvc.exceptions import DvcException
 from dvc.output.base import OutputAlreadyTrackedError
+from dvc.project import Project
 
 from tests.basic_env import TestDvc
 
@@ -149,6 +151,36 @@ class TestAddExternalLocalFile(TestDvc):
         self.assertTrue(filecmp.cmp(fname, 'foo', shallow=False))
 
 
+class TestAddLocalRemoteFile(TestDvc):
+    def test(self):
+        """
+        Making sure that 'remote' syntax is handled properly for local outs.
+        """
+        cwd = os.getcwd()
+        remote = 'myremote'
+
+        ret = main(['remote', 'add', remote, cwd])
+        self.assertEqual(ret, 0)
+
+        self.dvc = Project()
+
+        foo = 'remote://{}/{}'.format(remote, self.FOO)
+        ret = main(['add', foo])
+        self.assertEqual(ret, 0)
+
+        with open('foo.dvc', 'r') as fobj:
+            d = yaml.safe_load(fobj)
+            self.assertEqual(d['outs'][0]['path'], foo)
+
+        bar = os.path.join(cwd, self.BAR)
+        ret = main(['add', bar])
+        self.assertEqual(ret, 0)
+
+        with open('bar.dvc', 'r') as fobj:
+            d = yaml.safe_load(fobj)
+            self.assertEqual(d['outs'][0]['path'], bar)
+
+
 class TestCmdAdd(TestDvc):
     def test(self):
         ret = main(['add',