Repository: iterative/dvc

Issue:
Unexpected error when run dvc gc in git repo with submodules
I run the command: 
```
dvc gc -a -v
```
I get some traceback:
```
2020-03-16 09:53:37,900 ERROR: unexpected error - 'be23d78966ad0171e87879e051edf6eb3f446e12'
------------------------------------------------------------
Traceback (most recent call last):
  File "/home/my_user/data/env/my_project/lib/python3.6/site-packages/dvc/main.py", line 50, in main
    ret = cmd.run()
  File "/home/my_user/data/env/my_project/lib/python3.6/site-packages/dvc/command/gc.py", line 60, in run
    workspace=self.args.workspace,
  File "/home/my_user/data/env/my_project/lib/python3.6/site-packages/dvc/repo/__init__.py", line 27, in wrapper
    ret = f(repo, *args, **kwargs)
  File "/home/my_user/data/env/my_project/lib/python3.6/site-packages/dvc/repo/gc.py", line 75, in gc
    jobs=jobs,
  File "/home/my_user/data/env/my_project/lib/python3.6/site-packages/dvc/repo/__init__.py", line 256, in used_cache
    for stage, filter_info in pairs:
  File "/home/my_user/data/env/my_project/lib/python3.6/site-packages/dvc/repo/__init__.py", line 252, in <genexpr>
    for target in targets
  File "/home/my_user/data/env/my_project/lib/python3.6/site-packages/dvc/repo/__init__.py", line 202, in collect_granular
    return [(stage, None) for stage in self.stages]
  File "/home/my_user/data/env/my_project/lib/python3.6/site-packages/funcy/objects.py", line 28, in __get__
    res = instance.__dict__[self.fget.__name__] = self.fget(instance)
  File "/home/my_user/data/env/my_project/lib/python3.6/site-packages/dvc/repo/__init__.py", line 397, in stages
    for root, dirs, files in self.tree.walk(self.root_dir):
  File "/home/my_user/data/env/my_project/lib/python3.6/site-packages/dvc/ignore.py", line 135, in walk
    dirs[:], files[:] = self.dvcignore(root, dirs, files)
  File "/home/my_user/data/env/my_project/lib/python3.6/site-packages/funcy/objects.py", line 28, in __get__
    res = instance.__dict__[self.fget.__name__] = self.fget(instance)
  File "/home/my_user/data/env/my_project/lib/python3.6/site-packages/dvc/ignore.py", line 115, in dvcignore
    return DvcIgnoreFilter(self.tree)
  File "/home/my_user/data/env/my_project/lib/python3.6/site-packages/dvc/ignore.py", line 93, in __init__
    for root, dirs, files in self.tree.walk(self.tree.tree_root):
  File "/home/my_user/data/env/my_project/lib/python3.6/site-packages/dvc/scm/git/tree.py", line 148, in walk
    yield from self._walk(tree, topdown=topdown)
  File "/home/my_user/data/env/my_project/lib/python3.6/site-packages/dvc/scm/git/tree.py", line 131, in _walk
    yield from self._walk(tree[i], topdown=topdown)
  File "/home/my_user/data/env/my_project/lib/python3.6/site-packages/dvc/scm/git/tree.py", line 121, in _walk
    for i in _iter_tree(tree):
  File "/home/my_user/data/env/my_project/lib/python3.6/site-packages/dvc/scm/git/tree.py", line 26, in _iter_tree
    node = submodules[node.hexsha]
KeyError: 'be23d78966ad0171e87879e051edf6eb3f446e12'
```

**My setup**
- DVC version: 0.88.0
- Platform: Ubuntu 18.04
- Method of installation: pip




Discussion:
Discord context https://discordapp.com/channels/485586884165107732/563406153334128681/689046497446199296
Can reproduce with git repo that has submodules, e.g. with `git clone https://github.com/githubtraining/example-dependency`:
```
>>> from dvc.scm.git import Git
>>> git = Git(".")
>>> tree = git.get_tree("HEAD~1")
>>> for root, dnames, fnames in tree.walk("."):
...     print(root)
...     print(dnames)
...     print(fnames)
...
/home/efiop/git/feedstocks
['feedstocks']
['.gitmodules', 'LICENSE', 'README.md']
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/home/efiop/git/dvc/dvc/scm/git/tree.py", line 148, in walk
    yield from self._walk(tree, topdown=topdown)
  File "/home/efiop/git/dvc/dvc/scm/git/tree.py", line 131, in _walk
    yield from self._walk(tree[i], topdown=topdown)
  File "/home/efiop/git/dvc/dvc/scm/git/tree.py", line 121, in _walk
    for i in _iter_tree(tree):
  File "/home/efiop/git/dvc/dvc/scm/git/tree.py", line 26, in _iter_tree
    node = submodules[node.hexsha]
KeyError: 'ba4d4c3b302af35049508e381fdff85072caa200'
```

so this affects all git repos with submodules, which is really bad.

Resolving patch:
diff --git a/dvc/scm/git/tree.py b/dvc/scm/git/tree.py
--- a/dvc/scm/git/tree.py
+++ b/dvc/scm/git/tree.py
@@ -12,19 +12,11 @@
 GIT_MODE_FILE = 0o644
 
 
-def _iter_tree(tree):
-    # During usual tree iteration with ``for .. in ..`` GitPython returns each
-    # submodule instance without correctly filled ``name`` property. It raises
-    # an exception during accessing such submodule ``name`` later.
-    # The same time repo's submodules property contains correctly initialized
-    # submodules list.
-    # Here we just replace submodule instance from iterator with the
-    # corresponding instance from ``repo.submodules.``
-    submodules = {x.hexsha: x for x in tree.repo.submodules}
-    for node in tree:
-        if node.type == "submodule":
-            node = submodules[node.hexsha]
-        yield node
+def _item_basename(item):
+    # NOTE: `item.name` is not always a basename. See [1] for more details.
+    #
+    # [1] https://github.com/iterative/dvc/issues/3481#issuecomment-600693884
+    return os.path.basename(item.path)
 
 
 class GitTree(BaseTree):
@@ -87,8 +79,8 @@ def _is_tree_and_contains(obj, path):
             return False
         # see https://github.com/gitpython-developers/GitPython/issues/851
         # `return (i in tree)` doesn't work so here is a workaround:
-        for i in _iter_tree(obj):
-            if i.name == path:
+        for item in obj:
+            if _item_basename(item) == path:
                 return True
         return False
 
@@ -118,11 +110,12 @@ def git_object_by_path(self, path):
 
     def _walk(self, tree, topdown=True):
         dirs, nondirs = [], []
-        for i in _iter_tree(tree):
-            if i.mode == GIT_MODE_DIR:
-                dirs.append(i.name)
+        for item in tree:
+            name = _item_basename(item)
+            if item.mode == GIT_MODE_DIR:
+                dirs.append(name)
             else:
-                nondirs.append(i.name)
+                nondirs.append(name)
 
         if topdown:
             yield os.path.normpath(tree.abspath), dirs, nondirs


Test patch:
diff --git a/tests/unit/scm/test_git.py b/tests/unit/scm/test_git.py
--- a/tests/unit/scm/test_git.py
+++ b/tests/unit/scm/test_git.py
@@ -1,5 +1,7 @@
 import os
 
+from dvc.compat import fspath
+
 from tests.basic_env import TestDvcGit
 
 
@@ -15,3 +17,23 @@ def test_belongs_to_scm_true_on_git_internal(self):
     def test_belongs_to_scm_false(self):
         path = os.path.join("some", "non-.git", "file")
         self.assertFalse(self.dvc.scm.belongs_to_scm(path))
+
+
+def test_walk_with_submodules(tmp_dir, scm, git_dir):
+    git_dir.scm_gen(
+        {"foo": "foo", "bar": "bar", "dir": {"data": "data"}},
+        commit="add dir and files",
+    )
+    scm.repo.create_submodule("submodule", "submodule", url=fspath(git_dir))
+    scm.commit("added submodule")
+
+    files = []
+    dirs = []
+    tree = scm.get_tree("HEAD")
+    for _, dnames, fnames in tree.walk("."):
+        dirs.extend(dnames)
+        files.extend(fnames)
+
+    # currently we don't walk through submodules
+    assert not dirs
+    assert set(files) == {".gitmodules", "submodule"}