Repository: iterative/dvc

Issue:
import: downloads repository every time
```bash
dvc init --no-scm
dvc import https://github.com/iterative/dvc scripts/innosetup/dvc.ico

# Adding '../../../tmp/tmpp4_abkehdvc-repo/.dvc/tmp' to '../../../tmp/tmpp4_abkehdvc-repo/.dvc/.gitignore'.

dvc unlock dvc.ico.dvc
dvc repro dvc.ico.dvc

# Adding '../../../tmp/tmpqx36r24_dvc-repo/.dvc/tmp' to '../../../tmp/tmpqx36r24_dvc-repo/.dvc/.gitignore'.
```

Why not using `.dvc/repositories` to store the imported git repository?


Discussion:
For the record, this is the main cause for https://github.com/iterative/dvc/issues/2484
Need to figure out possible options to solve it. E.g. possible solutions right now are:

1) Cache cloned repos (will need to implement switching branches though); E.g. in https://github.com/iterative/dvc/blob/master/dvc/external_repo.py#L18 ;

2) Cache credentials (so need to ask them and then pass them to git clone)


Resolving patch:
diff --git a/dvc/external_repo.py b/dvc/external_repo.py
--- a/dvc/external_repo.py
+++ b/dvc/external_repo.py
@@ -1,50 +1,91 @@
 from __future__ import unicode_literals
 
 import os
-import logging
 import tempfile
+from distutils.dir_util import copy_tree
 
 from funcy import retry
 from contextlib import contextmanager
 
-from dvc.config import Config
-from dvc.cache import CacheConfig
 from dvc.utils import remove
 
 
-logger = logging.getLogger(__name__)
+REPO_CACHE = {}
 
 
-def _clone(cache_dir=None, url=None, rev=None, rev_lock=None):
+@contextmanager
+def external_repo(url=None, rev=None, rev_lock=None, cache_dir=None):
     from dvc.repo import Repo
 
-    _path = tempfile.mkdtemp("dvc-repo")
+    path = _external_repo(url=url, rev=rev_lock or rev, cache_dir=cache_dir)
+    repo = Repo(path)
+    yield repo
+    repo.close()
+
+
+def _external_repo(url=None, rev=None, cache_dir=None):
+    from dvc.config import Config
+    from dvc.cache import CacheConfig
+    from dvc.repo import Repo
+
+    key = (url, rev, cache_dir)
+    if key in REPO_CACHE:
+        return REPO_CACHE[key]
+
+    new_path = tempfile.mkdtemp("dvc-erepo")
+
+    # Copy and adjust existing clone
+    if (url, None, None) in REPO_CACHE:
+        old_path = REPO_CACHE[url, None, None]
 
-    repo = Repo.clone(url, _path, rev=(rev_lock or rev))
+        # This one unlike shutil.copytree() works with an existing dir
+        copy_tree(old_path, new_path)
+    else:
+        # Create a new clone
+        _clone_repo(url, new_path)
+
+        # Save clean clone dir so that we will have access to a default branch
+        clean_clone_path = tempfile.mkdtemp("dvc-erepo")
+        copy_tree(new_path, clean_clone_path)
+        REPO_CACHE[url, None, None] = clean_clone_path
 
+    # Adjust new clone/copy to fit rev and cache_dir
+    repo = Repo(new_path)
     try:
-        if cache_dir:
+        if rev is not None:
+            repo.scm.checkout(rev)
+
+        if cache_dir is not None:
             cache_config = CacheConfig(repo.config)
             cache_config.set_dir(cache_dir, level=Config.LEVEL_LOCAL)
     finally:
+        # Need to close/reopen repo to force config reread
         repo.close()
 
-    return Repo(_path)
+    REPO_CACHE[key] = new_path
+    return new_path
 
 
-def _remove(repo):
-    repo.close()
+def clean_repos():
+    # Outside code should not see cache while we are removing
+    repo_paths = list(REPO_CACHE.values())
+    REPO_CACHE.clear()
+
+    for path in repo_paths:
+        _remove(path)
 
+
+def _remove(path):
     if os.name == "nt":
         # git.exe may hang for a while not permitting to remove temp dir
         os_retry = retry(5, errors=OSError, timeout=0.1)
-        os_retry(remove)(repo.root_dir)
+        os_retry(remove)(path)
     else:
-        remove(repo.root_dir)
+        remove(path)
 
 
-@contextmanager
-def external_repo(**kwargs):
-    repo = _clone(**kwargs)
-    yield repo
-    _remove(repo)
+def _clone_repo(url, path):
+    from dvc.scm.git import Git
+
+    git = Git.clone(url, path)
+    git.close()
diff --git a/dvc/main.py b/dvc/main.py
--- a/dvc/main.py
+++ b/dvc/main.py
@@ -10,6 +10,7 @@
 from dvc.analytics import Analytics
 from dvc.exceptions import NotDvcRepoError, DvcParserError
 from dvc.remote.pool import close_pools
+from dvc.external_repo import clean_repos
 from dvc.utils.compat import is_py2
 
 
@@ -65,10 +66,15 @@ def main(argv=None):
         ret = 255
     finally:
         logger.setLevel(outerLogLevel)
+
         # Python 2 fails to close these clean occasionally and users see
         # weird error messages, so we do it manually
         close_pools()
 
+        # Remove cached repos in the end of the call, these are anonymous
+        # so won't be reused by any other subsequent run anyway.
+        clean_repos()
+
     Analytics().send_cmd(cmd, args, ret)
 
     return ret
diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py
--- a/dvc/repo/__init__.py
+++ b/dvc/repo/__init__.py
@@ -489,15 +489,6 @@ def dvcignore(self):
     def close(self):
         self.scm.close()
 
-    @staticmethod
-    def clone(url, to_path, rev=None):
-        from dvc.scm.git import Git
-
-        git = Git.clone(url, to_path, rev=rev)
-        git.close()
-
-        return Repo(to_path)
-
     @locked
     def checkout(self, *args, **kwargs):
         return self._checkout(*args, **kwargs)


Test patch:
diff --git a/tests/func/test_external_repo.py b/tests/func/test_external_repo.py
new file mode 100644
--- /dev/null
+++ b/tests/func/test_external_repo.py
@@ -0,0 +1,28 @@
+import os
+from mock import patch
+
+from dvc.external_repo import external_repo
+from dvc.scm.git import Git
+
+
+def test_external_repo(erepo):
+    url = erepo.root_dir
+    # We will share cache dir, to fetch version file
+    cache_dir = erepo.dvc.cache.local.cache_dir
+
+    with patch.object(Git, "clone", wraps=Git.clone) as mock:
+        with external_repo(url, cache_dir=cache_dir) as repo:
+            with repo.open(os.path.join(repo.root_dir, "version")) as fd:
+                assert fd.read() == "master"
+
+        with external_repo(url, rev="branch", cache_dir=cache_dir) as repo:
+            with repo.open(os.path.join(repo.root_dir, "version")) as fd:
+                assert fd.read() == "branch"
+
+        # Check cache_dir is unset
+        with external_repo(url) as repo:
+            assert repo.cache.local.cache_dir.startswith(
+                repo.root_dir + os.sep
+            )
+
+        assert mock.call_count == 1
diff --git a/tests/func/test_import.py b/tests/func/test_import.py
--- a/tests/func/test_import.py
+++ b/tests/func/test_import.py
@@ -1,8 +1,8 @@
 import os
 import filecmp
-import shutil
 
 import pytest
+from mock import patch
 from tests.utils import trees_equal
 
 from dvc.stage import Stage
@@ -73,9 +73,10 @@ def test_download_error_pulling_imported_stage(dvc_repo, erepo):
     dst_stage = Stage.load(dvc_repo, "foo_imported.dvc")
     dst_cache = dst_stage.outs[0].cache_path
 
-    shutil.rmtree(erepo.root_dir)
     os.remove(dst)
     os.remove(dst_cache)
 
-    with pytest.raises(DownloadError):
+    with patch(
+        "dvc.remote.RemoteLOCAL._download", side_effect=Exception
+    ), pytest.raises(DownloadError):
         dvc_repo.pull(["foo_imported.dvc"])
diff --git a/tests/func/test_update.py b/tests/func/test_update.py
--- a/tests/func/test_update.py
+++ b/tests/func/test_update.py
@@ -3,6 +3,7 @@
 import filecmp
 
 from dvc.repo import Repo
+from dvc.external_repo import clean_repos
 
 
 def test_update_import(dvc_repo, erepo):
@@ -34,6 +35,10 @@ def test_update_import(dvc_repo, erepo):
 
     os.chdir(saved_dir)
 
+    # Caching in external repos doesn't see upstream updates within single
+    # cli call, so we need to clean the caches to see the changes.
+    clean_repos()
+
     assert dvc_repo.status([stage.path]) == {}
     dvc_repo.update(stage.path)
     assert dvc_repo.status([stage.path]) == {}