Repository: python/mypy

Issue:
Fine-grained: Errors reported for a file removed from the build
This fine-grained incremental mode test case continues to report errors after `a.py` has been removed from the build:

```
[case testDeleteFileWithErrors]
# cmd: mypy main a.py
# cmd2: mypy main
# flags: --follow-imports=skip --ignore-missing-imports
import a
[file a.py]
def f() -> None:
    1()
''()
[file b.py.2]
# unrelated change
[out]
a.py:2: error: "int" not callable
a.py:3: error: "str" not callable
==
```

Mypy produces the same 2 errors on the second run as well, which is incorrect since `a.py` is no longer in the build.


Discussion:

Resolving patch:
diff --git a/mypy/dmypy_server.py b/mypy/dmypy_server.py
--- a/mypy/dmypy_server.py
+++ b/mypy/dmypy_server.py
@@ -318,7 +318,7 @@ def initialize_fine_grained(self, sources: List[mypy.build.BuildSource]) -> Dict
                     FileData(st_mtime=float(meta.mtime), st_size=meta.size, md5=meta.hash))
 
             # Run an update
-            changed = self.find_changed(sources)
+            changed, removed = self.find_changed(sources)
 
             # Find anything that has had its dependency list change
             for state in self.fine_grained_manager.graph.values():
@@ -326,8 +326,8 @@ def initialize_fine_grained(self, sources: List[mypy.build.BuildSource]) -> Dict
                     assert state.path is not None
                     changed.append((state.id, state.path))
 
-            if changed:
-                messages = self.fine_grained_manager.update(changed)
+            if changed or removed:
+                messages = self.fine_grained_manager.update(changed, removed)
         else:
             # Stores the initial state of sources as a side effect.
             self.fswatcher.find_changed()
@@ -343,13 +343,13 @@ def fine_grained_increment(self, sources: List[mypy.build.BuildSource]) -> Dict[
 
         t0 = time.time()
         self.update_sources(sources)
-        changed = self.find_changed(sources)
+        changed, removed = self.find_changed(sources)
         t1 = time.time()
-        if not changed:
+        if not (changed or removed):
             # Nothing changed -- just produce the same result as before.
             messages = self.previous_messages
         else:
-            messages = self.fine_grained_manager.update(changed)
+            messages = self.fine_grained_manager.update(changed, removed)
         t2 = time.time()
         self.fine_grained_manager.manager.log(
             "fine-grained increment: find_changed: {:.3f}s, update: {:.3f}s".format(
@@ -364,20 +364,20 @@ def update_sources(self, sources: List[mypy.build.BuildSource]) -> None:
         paths = [source.path for source in sources if source.path is not None]
         self.fswatcher.add_watched_paths(paths)
 
-    def find_changed(self, sources: List[mypy.build.BuildSource]) -> List[Tuple[str, str]]:
+    def find_changed(self, sources: List[mypy.build.BuildSource]) -> Tuple[List[Tuple[str, str]],
+                                                                           List[Tuple[str, str]]]:
         changed_paths = self.fswatcher.find_changed()
         changed = [(source.module, source.path)
                    for source in sources
                    if source.path in changed_paths]
         modules = {source.module for source in sources}
         omitted = [source for source in self.previous_sources if source.module not in modules]
+        removed = []
         for source in omitted:
             path = source.path
             assert path
-            # Note that a file could be removed from the list of root sources but have no changes.
-            if path in changed_paths:
-                changed.append((source.module, path))
-        return changed
+            removed.append((source.module, path))
+        return changed, removed
 
     def cmd_hang(self) -> Dict[str, object]:
         """Hang for 100 seconds, as a debug hack."""
diff --git a/mypy/server/update.py b/mypy/server/update.py
--- a/mypy/server/update.py
+++ b/mypy/server/update.py
@@ -183,7 +183,9 @@ def __init__(self,
         # Modules processed during the last update
         self.updated_modules = []  # type: List[str]
 
-    def update(self, changed_modules: List[Tuple[str, str]]) -> List[str]:
+    def update(self,
+               changed_modules: List[Tuple[str, str]],
+               removed_modules: List[Tuple[str, str]]) -> List[str]:
         """Update previous build result by processing changed modules.
 
         Also propagate changes to other modules as needed, but only process
@@ -194,14 +196,18 @@ def update(self, changed_modules: List[Tuple[str, str]]) -> List[str]:
 
         Args:
             changed_modules: Modules changed since the previous update/build; each is
-                a (module id, path) tuple. Includes modified, added and deleted modules.
+                a (module id, path) tuple. Includes modified and added modules.
                 Assume this is correct; it's not validated here.
+            removed_modules: Modules that have been deleted since the previous update
+                or removed from the build.
 
         Returns:
             A list of errors.
         """
-        assert changed_modules, 'No changed modules'
+        changed_modules = changed_modules + removed_modules
+        assert changed_modules or removed_modules, 'No changed modules'
 
+        removed_set = {module for module, _ in removed_modules}
         self.changed_modules = changed_modules
 
         # Reset global caches for the new build.
@@ -230,7 +236,7 @@ def update(self, changed_modules: List[Tuple[str, str]]) -> List[str]:
             if next_id not in self.previous_modules and next_id not in initial_set:
                 self.manager.log_fine_grained('skip %r (module not in import graph)' % next_id)
                 continue
-            result = self.update_single(next_id, next_path)
+            result = self.update_single(next_id, next_path, next_id in removed_set)
             messages, remaining, (next_id, next_path), blocker = result
             changed_modules = [(id, path) for id, path in changed_modules
                                if id != next_id]
@@ -247,15 +253,24 @@ def update(self, changed_modules: List[Tuple[str, str]]) -> List[str]:
 
         return messages
 
-    def update_single(self, module: str, path: str) -> Tuple[List[str],
-                                                             List[Tuple[str, str]],
-                                                             Tuple[str, str],
-                                                             bool]:
+    def update_single(self,
+                      module: str,
+                      path: str,
+                      force_removed: bool) -> Tuple[List[str],
+                                                    List[Tuple[str, str]],
+                                                    Tuple[str, str],
+                                                    bool]:
         """Update a single modified module.
 
         If the module contains imports of previously unseen modules, only process one of
         the new modules and return the remaining work to be done.
 
+        Args:
+            module: Id of the module
+            path: File system path of the module
+            force_removed: If True, consider module removed from the build even if path
+                exists (used for removing an existing file from the build)
+
         Returns:
             Tuple with these items:
 
@@ -279,7 +294,8 @@ def update_single(self, module: str, path: str) -> Tuple[List[str],
             old_snapshots[module] = snapshot
 
         manager.errors.reset()
-        result = update_single_isolated(module, path, manager, previous_modules, graph)
+        result = update_single_isolated(module, path, manager, previous_modules, graph,
+                                        force_removed)
         if isinstance(result, BlockedUpdate):
             # Blocking error -- just give up
             module, path, remaining, errors = result
@@ -344,7 +360,8 @@ def update_single_isolated(module: str,
                            path: str,
                            manager: BuildManager,
                            previous_modules: Dict[str, str],
-                           graph: Graph) -> UpdateResult:
+                           graph: Graph,
+                           force_removed: bool) -> UpdateResult:
     """Build a new version of one changed module only.
 
     Don't propagate changes to elsewhere in the program. Raise CompleError on
@@ -355,6 +372,8 @@ def update_single_isolated(module: str,
         path: Path of the changed module
         manager: Build manager
         graph: Build graph
+        force_removed: If True, consider the module removed from the build even it the
+            file exists
 
     Returns a named tuple describing the result (see above for details).
     """
@@ -388,7 +407,7 @@ def update_single_isolated(module: str,
             remaining_modules = []
         return BlockedUpdate(err.module_with_blocker, path, remaining_modules, err.messages)
 
-    if not os.path.isfile(path):
+    if not os.path.isfile(path) or force_removed:
         delete_module(module, graph, manager)
         return NormalUpdate(module, path, [], None)
 


Test patch:
diff --git a/mypy/test/testmerge.py b/mypy/test/testmerge.py
--- a/mypy/test/testmerge.py
+++ b/mypy/test/testmerge.py
@@ -124,7 +124,7 @@ def build(self, source: str) -> Tuple[List[str], Optional[BuildManager], Dict[st
     def build_increment(self, manager: FineGrainedBuildManager,
                         module_id: str, path: str) -> Tuple[MypyFile,
                                                             Dict[Expression, Type]]:
-        manager.update([(module_id, path)])
+        manager.update([(module_id, path)], [])
         module = manager.manager.modules[module_id]
         type_map = manager.graph[module_id].type_map()
         return module, type_map
diff --git a/test-data/unit/fine-grained-modules.test b/test-data/unit/fine-grained-modules.test
--- a/test-data/unit/fine-grained-modules.test
+++ b/test-data/unit/fine-grained-modules.test
@@ -975,7 +975,7 @@ x = Foo()
 main:2: error: Too few arguments for "foo" of "Foo"
 
 -- This series of tests is designed to test adding a new module that
--- doesn't appear in the cache, for cache mode. They are run in
+-- does not appear in the cache, for cache mode. They are run in
 -- cache mode only because stale and rechecked differ heavily between
 -- the modes.
 [case testAddModuleAfterCache1-skip-nocache]
@@ -1033,6 +1033,7 @@ a.py:2: error: Too many arguments for "foo"
 [case testAddModuleAfterCache3-skip-nocache]
 # cmd: mypy main a.py
 # cmd2: mypy main a.py b.py c.py d.py e.py f.py g.py
+# cmd3: mypy main a.py b.py c.py d.py e.py f.py g.py
 # flags: --ignore-missing-imports --follow-imports=skip
 import a
 [file a.py]
@@ -1345,3 +1346,19 @@ x = ''
 [out]
 ==
 ==
+
+[case testDeleteFileWithErrors]
+# cmd: mypy main a.py
+# cmd2: mypy main
+# flags: --follow-imports=skip --ignore-missing-imports
+import a
+[file a.py]
+def f() -> None:
+    1()
+''()
+[file b.py.2]
+# unrelated change
+[out]
+a.py:2: error: "int" not callable
+a.py:3: error: "str" not callable
+==