Repository: iterative/dvc

Issue:
DVC status fails after using add -R
Hey :)

I have a folder of files I added with 
$$ dvc add -R raw_data

Then I've imported the folder in other git repo using
$$ dvc import -o data --rev raw-v1 https://...@bitbucket.org/.../raw_data_reg.git raw_data

Then when execute dvc status I get an error message

```
$$ dvc status -v

2020-06-29 13:54:52,056 DEBUG: fetched: [(3,)]                                  
2020-06-29 13:54:52,059 DEBUG: Creating external repo https://...@bitbucket.org/.../raw_data_reg.git@43fb50fb519d58415bba4903b480f44811fefec1
2020-06-29 13:54:52,059 DEBUG: erepo: git clone https://...@bitbucket.org/.../raw_data_reg.git to a temporary dir
Cloning                                                                     |0.00 [00:00,      ?obj/s]Password for 'https://...@bitbucket.org':
2020-06-29 13:54:58,709 DEBUG: fetched: [(17,)]                                                      
2020-06-29 13:54:58,728 ERROR: failed to obtain data status - get_hash() takes 2 positional arguments but 3 were given
------------------------------------------------------------
Traceback (most recent call last):
  File "/home/moshe/.local/lib/python3.6/site-packages/dvc/dependency/repo.py", line 57, in _get_checksum
    return repo.find_out_by_relpath(self.def_path).info["md5"]
  File "/home/moshe/.local/lib/python3.6/site-packages/dvc/repo/__init__.py", line 566, in find_out_by_relpath
    (out,) = self.find_outs_by_path(path)
  File "/home/moshe/.local/lib/python3.6/site-packages/dvc/repo/__init__.py", line 560, in find_outs_by_path
    raise OutputNotFoundError(path, self)
dvc.exceptions.OutputNotFoundError: Unable to find DVC-file with output '../../../tmp/tmpq5w4msxhdvc-clone/raw_data'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/moshe/.local/lib/python3.6/site-packages/dvc/command/status.py", line 51, in run
    recursive=self.args.recursive,
  File "/home/moshe/.local/lib/python3.6/site-packages/dvc/repo/__init__.py", line 35, in wrapper
    ret = f(repo, *args, **kwargs)
  File "/home/moshe/.local/lib/python3.6/site-packages/dvc/repo/status.py", line 146, in status
    self, targets, with_deps=with_deps, recursive=recursive
  File "/home/moshe/.local/lib/python3.6/site-packages/dvc/repo/status.py", line 37, in _local_status
    return _joint_status(stages)
  File "/home/moshe/.local/lib/python3.6/site-packages/dvc/repo/status.py", line 23, in _joint_status
    status.update(stage.status(check_updates=True))
  File "/home/moshe/.local/lib/python3.6/site-packages/funcy/decorators.py", line 39, in wrapper
    return deco(call, *dargs, **dkwargs)
  File "/home/moshe/.local/lib/python3.6/site-packages/dvc/stage/decorators.py", line 35, in rwlocked
    return call()
  File "/home/moshe/.local/lib/python3.6/site-packages/funcy/decorators.py", line 60, in __call__
    return self._func(*self._args, **self._kwargs)
  File "/home/moshe/.local/lib/python3.6/site-packages/dvc/stage/__init__.py", line 469, in status
    self._status_deps(ret)
  File "/home/moshe/.local/lib/python3.6/site-packages/dvc/stage/__init__.py", line 485, in _status_deps
    deps_status = self._status(self.deps)
  File "/home/moshe/.local/lib/python3.6/site-packages/dvc/stage/__init__.py", line 480, in _status
    ret.update(entry.status())
  File "/home/moshe/.local/lib/python3.6/site-packages/dvc/dependency/repo.py", line 71, in status
    current_checksum = self._get_checksum(locked=True)
  File "/home/moshe/.local/lib/python3.6/site-packages/dvc/dependency/repo.py", line 67, in _get_checksum
    return self.repo.cache.local.get_hash(path, tree)
TypeError: get_hash() takes 2 positional arguments but 3 were given
------------------------------------------------------------

$$ dvc version
DVC version: 1.0.1
Python version: 3.6.9
Platform: Linux-5.3.0-61-generic-x86_64-with-Ubuntu-18.04-bionic
Binary: False
Package: pip
Supported remotes: azure, gdrive, gs, hdfs, http, https, s3, ssh, oss
Cache: reflink - not supported, hardlink - supported, symlink - supported
Filesystem type (cache directory): ('ext4', '/dev/sda1')
Repo: dvc, git
Filesystem type (workspace): ('ext4', '/dev/sda1')
```

Do you know what could be the problem?
Thanks.



Discussion:
Looks like
https://github.com/iterative/dvc/blob/07da9f1e5c0be1662dbce7b0667c38c45e2c6b1a/dvc/dependency/repo.py#L67

should be
`get_hash(path, tree=tree)`

We should make sure to add test coverage for this case w/the bugfix

Resolving patch:
diff --git a/dvc/dependency/repo.py b/dvc/dependency/repo.py
--- a/dvc/dependency/repo.py
+++ b/dvc/dependency/repo.py
@@ -64,7 +64,7 @@ def _get_checksum(self, locked=True):
 
                 # We are polluting our repo cache with some dir listing here
                 if tree.isdir(path):
-                    return self.repo.cache.local.get_hash(path, tree)
+                    return self.repo.cache.local.get_hash(path, tree=tree)
                 return tree.get_file_hash(path)
 
     def status(self):


Test patch:
diff --git a/tests/dir_helpers.py b/tests/dir_helpers.py
--- a/tests/dir_helpers.py
+++ b/tests/dir_helpers.py
@@ -221,6 +221,17 @@ def branch(self, name, new=False):
         finally:
             self.scm.checkout(old)
 
+    def read_text(self, *args, **kwargs):  # pylint: disable=signature-differs
+        # NOTE: on windows we'll get PermissionError instead of
+        # IsADirectoryError when we try to `open` a directory, so we can't
+        # rely on exception flow control
+        if self.is_dir():
+            return {
+                path.name: path.read_text(*args, **kwargs)
+                for path in self.iterdir()
+            }
+        return super().read_text(*args, **kwargs)
+
 
 def _coerce_filenames(filenames):
     if isinstance(filenames, (str, bytes, pathlib.PurePath)):
diff --git a/tests/func/test_get.py b/tests/func/test_get.py
--- a/tests/func/test_get.py
+++ b/tests/func/test_get.py
@@ -10,7 +10,6 @@
 from dvc.repo.get import GetDVCFileError
 from dvc.system import System
 from dvc.utils.fs import makedirs
-from tests.utils import trees_equal
 
 
 def test_get_repo_file(tmp_dir, erepo_dir):
@@ -29,8 +28,7 @@ def test_get_repo_dir(tmp_dir, erepo_dir):
 
     Repo.get(os.fspath(erepo_dir), "dir", "dir_imported")
 
-    assert os.path.isdir("dir_imported")
-    trees_equal(erepo_dir / "dir", "dir_imported")
+    assert (tmp_dir / "dir_imported").read_text() == {"file": "contents"}
 
 
 @pytest.mark.parametrize(
@@ -44,7 +42,6 @@ def test_get_git_file(tmp_dir, erepo):
 
     Repo.get(os.fspath(erepo), src, dst)
 
-    assert (tmp_dir / dst).is_file()
     assert (tmp_dir / dst).read_text() == "hello"
 
 
@@ -61,8 +58,7 @@ def test_get_git_dir(tmp_dir, erepo):
 
     Repo.get(os.fspath(erepo), src, dst)
 
-    assert (tmp_dir / dst).is_dir()
-    trees_equal(erepo / src, tmp_dir / dst)
+    assert (tmp_dir / dst).read_text() == {"dir": {"file.txt": "hello"}}
 
 
 def test_cache_type_is_properly_overridden(tmp_dir, erepo_dir):
diff --git a/tests/func/test_import.py b/tests/func/test_import.py
--- a/tests/func/test_import.py
+++ b/tests/func/test_import.py
@@ -11,7 +11,6 @@
 from dvc.exceptions import DownloadError, PathMissingError
 from dvc.system import System
 from dvc.utils.fs import makedirs, remove
-from tests.utils import trees_equal
 
 
 def test_import(tmp_dir, scm, dvc, erepo_dir):
@@ -73,8 +72,7 @@ def test_import_git_dir(tmp_dir, scm, dvc, git_dir, src_is_dvc):
 
     stage = dvc.imp(os.fspath(git_dir), "src", "dst")
 
-    assert (tmp_dir / "dst").is_dir()
-    trees_equal(git_dir / "src", tmp_dir / "dst")
+    assert (tmp_dir / "dst").read_text() == {"file.txt": "hello"}
     assert tmp_dir.scm.repo.git.check_ignore(os.fspath(tmp_dir / "dst"))
     assert stage.deps[0].def_repo == {
         "url": os.fspath(git_dir),
@@ -88,8 +86,7 @@ def test_import_dir(tmp_dir, scm, dvc, erepo_dir):
 
     stage = dvc.imp(os.fspath(erepo_dir), "dir", "dir_imported")
 
-    assert os.path.isdir("dir_imported")
-    trees_equal(erepo_dir / "dir", "dir_imported")
+    assert (tmp_dir / "dir_imported").read_text() == {"foo": "foo content"}
     assert scm.repo.git.check_ignore("dir_imported")
     assert stage.deps[0].def_repo == {
         "url": os.fspath(erepo_dir),
@@ -222,8 +219,7 @@ def test_pull_imported_directory_stage(tmp_dir, dvc, erepo_dir):
 
     dvc.pull(["dir_imported.dvc"])
 
-    assert os.path.isdir("dir_imported")
-    trees_equal(erepo_dir / "dir", "dir_imported")
+    assert (tmp_dir / "dir_imported").read_text() == {"foo": "foo content"}
 
 
 def test_download_error_pulling_imported_stage(tmp_dir, dvc, erepo_dir):
diff --git a/tests/func/test_update.py b/tests/func/test_update.py
--- a/tests/func/test_update.py
+++ b/tests/func/test_update.py
@@ -10,26 +10,69 @@ def test_update_import(tmp_dir, dvc, erepo_dir, cached):
     gen = erepo_dir.dvc_gen if cached else erepo_dir.scm_gen
 
     with erepo_dir.branch("branch", new=True), erepo_dir.chdir():
-        gen("version", "branch", "add version file")
+        gen(
+            {
+                "version": "branch",
+                "dir": {"version": "branch", "subdir": {"file": "file"}},
+            },
+            commit="add version file",
+        )
         old_rev = erepo_dir.scm.get_rev()
 
     stage = dvc.imp(os.fspath(erepo_dir), "version", "version", rev="branch")
+    dir_stage = dvc.imp(os.fspath(erepo_dir), "dir", "dir", rev="branch")
+    assert dvc.status() == {}
 
     assert (tmp_dir / "version").read_text() == "branch"
+    assert (tmp_dir / "dir").read_text() == {
+        "version": "branch",
+        "subdir": {"file": "file"},
+    }
     assert stage.deps[0].def_repo["rev_lock"] == old_rev
+    assert dir_stage.deps[0].def_repo["rev_lock"] == old_rev
 
     # Update version file
     with erepo_dir.branch("branch", new=False), erepo_dir.chdir():
-        gen("version", "updated", "update version content")
+        gen(
+            {
+                "version": "updated",
+                "dir": {"version": "updated", "subdir": {"file": "file"}},
+            },
+            commit="update version content",
+        )
         new_rev = erepo_dir.scm.get_rev()
 
     assert old_rev != new_rev
 
-    dvc.update([stage.path])
+    assert dvc.status() == {
+        "dir.dvc": [
+            {
+                "changed deps": {
+                    f"dir ({os.fspath(erepo_dir)})": "update available"
+                }
+            }
+        ],
+        "version.dvc": [
+            {
+                "changed deps": {
+                    f"version ({os.fspath(erepo_dir)})": "update available"
+                }
+            }
+        ],
+    }
+
+    (stage,) = dvc.update(stage.path)
+    (dir_stage,) = dvc.update(dir_stage.path)
+    assert dvc.status() == {}
+
     assert (tmp_dir / "version").read_text() == "updated"
+    assert (tmp_dir / "dir").read_text() == {
+        "version": "updated",
+        "subdir": {"file": "file"},
+    }
 
-    stage = Dvcfile(dvc, stage.path).stage
     assert stage.deps[0].def_repo["rev_lock"] == new_rev
+    assert dir_stage.deps[0].def_repo["rev_lock"] == new_rev
 
 
 def test_update_import_after_remote_updates_to_dvc(tmp_dir, dvc, erepo_dir):
diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py
--- a/tests/utils/__init__.py
+++ b/tests/utils/__init__.py
@@ -1,6 +1,5 @@
 import os
 from contextlib import contextmanager
-from filecmp import dircmp
 
 from dvc.scm import Git
 
@@ -20,15 +19,5 @@ def cd(newdir):
         os.chdir(prevdir)
 
 
-def trees_equal(dir_path_1, dir_path_2):
-
-    comparison = dircmp(dir_path_1, dir_path_2)
-
-    assert set(comparison.left_only) == set(comparison.right_only) == set()
-
-    for d in comparison.common_dirs:
-        trees_equal(os.path.join(dir_path_1, d), os.path.join(dir_path_2, d))
-
-
 def to_posixpath(path):
     return path.replace("\\", "/")