Repository: python/mypy

Issue:
Crash in fine-grained incremental mode if __init__.py is deleted
Deleting package `__init__.py` causes a fine-grained incremental mode crash.

Test case:

```
[case testDeletePackageInitFile]
# cmd: mypy p/__init__.py p/a.py
[file p/__init__.py]
[file p/a.py]
[delete p/__init__.py.2]
```

Traceback:

```
/Users/jukka/src/mypy/mypy/server/update.py:378: in update_single_isolated
    graph = load_graph(sources, manager)
/Users/jukka/src/mypy/mypy/build.py:2187: in load_graph
    ancestor_for=st)
/Users/jukka/src/mypy/mypy/build.py:1610: in __init__
    raise CompileError(["mypy: can't find module '%s'" % id])
E   mypy.errors.CompileError: mypy: can't find module 'p'

During handling of the above exception, another exception occurred:
/Users/jukka/src/mypy/mypy/test/testfinegrained.py:73: in run_case
    new_messages = fine_grained_manager.update(modules)
/Users/jukka/src/mypy/mypy/server/update.py:222: in update
    result = self.update_single(next_id, next_path)
/Users/jukka/src/mypy/mypy/server/update.py:265: in update_single
    result = update_single_isolated(module, path, manager, previous_modules)
/Users/jukka/src/mypy/mypy/server/update.py:381: in update_single_isolated
    assert err.module_with_blocker
E   AssertionError
```


Discussion:
This can also happen if a newly created package is missing `__init__.py`:

```py
[case testAddPackage4]
import p.a
p.a.f(1)
[file p/a.py.2]
def f(x: str) -> None: pass
[file p/__init__.py.3]
```
I think the fix here is that whenever an `__init__.py` is added, we need to remove and re-add any modules underneath it, since their names change.
To really get it right, we also need to do this for __init__ files that aren't in the build, unfortunately

Resolving patch:
diff --git a/mypy/dmypy_server.py b/mypy/dmypy_server.py
--- a/mypy/dmypy_server.py
+++ b/mypy/dmypy_server.py
@@ -318,9 +318,12 @@ def update_sources(self, sources: List[mypy.build.BuildSource]) -> None:
     def find_changed(self, sources: List[mypy.build.BuildSource]) -> Tuple[List[Tuple[str, str]],
                                                                            List[Tuple[str, str]]]:
         changed_paths = self.fswatcher.find_changed()
+        # Find anything that has been added or modified
         changed = [(source.module, source.path)
                    for source in sources
                    if source.path in changed_paths]
+
+        # Now find anything that has been removed from the build
         modules = {source.module for source in sources}
         omitted = [source for source in self.previous_sources if source.module not in modules]
         removed = []
@@ -328,6 +331,16 @@ def find_changed(self, sources: List[mypy.build.BuildSource]) -> Tuple[List[Tupl
             path = source.path
             assert path
             removed.append((source.module, path))
+
+        # Find anything that has had its module path change because of added or removed __init__s
+        last = {s.path: s.module for s in self.previous_sources}
+        for s in sources:
+            assert s.path
+            if s.path in last and last[s.path] != s.module:
+                # Mark it as removed from its old name and changed at its new name
+                removed.append((last[s.path], s.path))
+                changed.append((s.module, s.path))
+
         return changed, removed
 
     def cmd_hang(self) -> Dict[str, object]:
diff --git a/mypy/server/update.py b/mypy/server/update.py
--- a/mypy/server/update.py
+++ b/mypy/server/update.py
@@ -428,6 +428,10 @@ def update_module_isolated(module: str,
     else:
         manager.log_fine_grained('new module %r' % module)
 
+    if not manager.fscache.isfile(path) or force_removed:
+        delete_module(module, graph, manager)
+        return NormalUpdate(module, path, [], None)
+
     old_modules = dict(manager.modules)
     sources = get_sources(manager.fscache, previous_modules, [(module, path)])
 
@@ -453,10 +457,6 @@ def update_module_isolated(module: str,
             remaining_modules = []
         return BlockedUpdate(err.module_with_blocker, path, remaining_modules, err.messages)
 
-    if not manager.fscache.isfile(path) or force_removed:
-        delete_module(module, graph, manager)
-        return NormalUpdate(module, path, [], None)
-
     # Find any other modules brought in by imports.
     changed_modules = get_all_changed_modules(module, path, previous_modules, graph)
     # If there are multiple modules to process, only process one of them and return
@@ -612,28 +612,11 @@ def verify_dependencies(state: State, manager: BuildManager) -> None:
     for dep in dependencies + state.suppressed:  # TODO: ancestors?
         if dep not in manager.modules and not manager.options.ignore_missing_imports:
             assert state.tree
-            line = find_import_line(state.tree, dep) or 1
+            line = state.dep_line_map.get(dep, 1)
             assert state.path
             manager.module_not_found(state.path, state.id, line, dep)
 
 
-def find_import_line(node: MypyFile, target: str) -> Optional[int]:
-    for imp in node.imports:
-        if isinstance(imp, Import):
-            for name, _ in imp.ids:
-                if name == target:
-                    return imp.line
-        if isinstance(imp, ImportFrom):
-            if imp.id == target:
-                return imp.line
-            # TODO: Relative imports
-            for name, _ in imp.names:
-                if '%s.%s' % (imp.id, name) == target:
-                    return imp.line
-        # TODO: ImportAll
-    return None
-
-
 def collect_dependencies(new_modules: Mapping[str, Optional[MypyFile]],
                          deps: Dict[str, Set[str]],
                          graph: Dict[str, State]) -> None:


Test patch:
diff --git a/test-data/unit/fine-grained-modules.test b/test-data/unit/fine-grained-modules.test
--- a/test-data/unit/fine-grained-modules.test
+++ b/test-data/unit/fine-grained-modules.test
@@ -237,6 +237,73 @@ def foo(x: str) -> None: pass
 ==
 foo/a.py:2: error: Argument 1 to "foo" has incompatible type "int"; expected "str"
 
+[case testAddPackage5]
+# cmd: mypy main p/a.py
+# cmd2: mypy main p/a.py
+# cmd3: mypy main p/a.py p/__init__.py
+import p.a
+p.a.f(1)
+[file p/a.py]
+[file p/a.py.2]
+def f(x: str) -> None: pass
+[file p/__init__.py.3]
+[out]
+main:4: error: Cannot find module named 'p'
+main:4: note: (Perhaps setting MYPYPATH or using the "--ignore-missing-imports" flag would help)
+main:4: error: Cannot find module named 'p.a'
+==
+main:4: error: Cannot find module named 'p'
+main:4: note: (Perhaps setting MYPYPATH or using the "--ignore-missing-imports" flag would help)
+main:4: error: Cannot find module named 'p.a'
+==
+main:5: error: Argument 1 to "f" has incompatible type "int"; expected "str"
+
+[case testAddPackage6]
+# flags: --follow-imports=skip --ignore-missing-imports
+# cmd: mypy x.py
+# cmd2: mypy x.py p/a.py
+[file x.py]
+import p.a
+p.a.f(1)
+[file p/a.py.2]
+def f(x: str) -> None: pass
+[file p/__init__.py.2]
+[out]
+==
+-- It is a bug (#4797) that this isn't an error, but not a fine-grained specific one
+
+[case testAddPackage7]
+# flags: --follow-imports=skip
+# cmd: mypy x.py
+# cmd2: mypy x.py p/a.py
+[file x.py]
+from p.a import f
+f(1)
+[file p/a.py.2]
+def f(x: str) -> None: pass
+[file p/__init__.py.2]
+[out]
+x.py:1: error: Cannot find module named 'p.a'
+x.py:1: note: (Perhaps setting MYPYPATH or using the "--ignore-missing-imports" flag would help)
+==
+x.py:2: error: Argument 1 to "f" has incompatible type "int"; expected "str"
+
+
+[case testAddPackage8]
+# cmd: mypy x.py p/a.py
+# cmd2: mypy x.py p/a.py p/__init__.py
+# cmd3: mypy x.py p/a.py p/__init__.py
+[file x.py]
+[file p/a.py]
+1+'hi'
+[file p/__init__.py.2]
+[file p/a.py.3]
+'1'+'hi'
+[out]
+p/a.py:1: error: Unsupported operand types for + ("int" and "str")
+==
+p/a.py:1: error: Unsupported operand types for + ("int" and "str")
+==
 
 
 -- Delete file
@@ -713,8 +780,30 @@ main:1: error: Cannot find module named 'p'
 main:1: note: (Perhaps setting MYPYPATH or using the "--ignore-missing-imports" flag would help)
 main:1: error: Cannot find module named 'p.a'
 
-[case testDeletePackage4-skip]
-# TODO: Currently crashing (https://github.com/python/mypy/issues/4477)
+[case testDeletePackage4]
+import p.a
+p.a.f(1)
+[file p/a.py]
+def f(x: str) -> None: pass
+[file p/__init__.py]
+[delete p/__init__.py.2]
+[delete p/a.py.3]
+[out]
+main:2: error: Argument 1 to "f" has incompatible type "int"; expected "str"
+==
+main:1: error: Cannot find module named 'p'
+main:1: note: (Perhaps setting MYPYPATH or using the "--ignore-missing-imports" flag would help)
+main:1: error: Cannot find module named 'p.a'
+==
+main:1: error: Cannot find module named 'p'
+main:1: note: (Perhaps setting MYPYPATH or using the "--ignore-missing-imports" flag would help)
+main:1: error: Cannot find module named 'p.a'
+
+[case testDeletePackage5]
+# cmd1: mypy main p/a.py p/__init__.py
+# cmd2: mypy main p/a.py
+# cmd3: mypy main
+
 import p.a
 p.a.f(1)
 [file p/a.py]
@@ -723,7 +812,39 @@ def f(x: str) -> None: pass
 [delete p/__init__.py.2]
 [delete p/a.py.3]
 [out]
-TODO
+main:6: error: Argument 1 to "f" has incompatible type "int"; expected "str"
+==
+main:5: error: Cannot find module named 'p'
+main:5: note: (Perhaps setting MYPYPATH or using the "--ignore-missing-imports" flag would help)
+main:5: error: Cannot find module named 'p.a'
+==
+main:5: error: Cannot find module named 'p'
+main:5: note: (Perhaps setting MYPYPATH or using the "--ignore-missing-imports" flag would help)
+main:5: error: Cannot find module named 'p.a'
+
+
+[case testDeletePackage6]
+# cmd1: mypy p/a.py p/b.py p/__init__.py
+# cmd2: mypy p/a.py p/b.py
+# cmd3: mypy p/a.py p/b.py
+[file p/a.py]
+def f(x: str) -> None: pass
+[file p/b.py]
+from p.a import f
+f(12)
+[file p/__init__.py]
+[delete p/__init__.py.2]
+[file p/b.py.3]
+from a import f
+f(12)
+[out]
+p/b.py:2: error: Argument 1 to "f" has incompatible type "int"; expected "str"
+==
+p/b.py:1: error: Cannot find module named 'p.a'
+p/b.py:1: note: (Perhaps setting MYPYPATH or using the "--ignore-missing-imports" flag would help)
+==
+p/b.py:2: error: Argument 1 to "f" has incompatible type "int"; expected "str"
+
 
 -- TODO:
 -- - add one file which imports another new file, blocking error in new file