Repository: python/mypy
Pull request: Fix file reloading in dmypy with --export-types
Fixes https://github.com/python/mypy/issues/15794

Unfortunately, this requires to pass `--export-types` to `dmypy run`  if one wants to inspect a file that was previously kicked out of the build.


Problem statement:
(🐞) `dmypy inspect` says `Unknown module` with multiple `run`s
`test1.py`
```py
a: int
```
`test2.py`
```py
a: str
```
```
👉 dmypy run test1.py
Daemon started
Success: no issues found in 42 source files
👉 dmypy run test2.py
Success: no issues found in 1 source file
👉 dmypy run test1.py --export types
Success: no issues found in 1 source file
👉 dmypy inspect test1.py:1:1:1:1
Unknown module: test
```

Hints from issue discussion:

Commit message: Fix fine-grained tests

Patch to review:
diff --git a/mypy/dmypy_server.py b/mypy/dmypy_server.py
index 9cc0888fc208..6e801792bbc3 100644
--- a/mypy/dmypy_server.py
+++ b/mypy/dmypy_server.py
@@ -393,15 +393,21 @@ def cmd_recheck(
         t1 = time.time()
         manager = self.fine_grained_manager.manager
         manager.log(f"fine-grained increment: cmd_recheck: {t1 - t0:.3f}s")
-        self.options.export_types = export_types
+        old_export_types = self.options.export_types
+        self.options.export_types = self.options.export_types or export_types
         if not self.following_imports():
-            messages = self.fine_grained_increment(sources, remove, update)
+            messages = self.fine_grained_increment(
+                sources, remove, update, explicit_export_types=export_types
+            )
         else:
             assert remove is None and update is None
-            messages = self.fine_grained_increment_follow_imports(sources)
+            messages = self.fine_grained_increment_follow_imports(
+                sources, explicit_export_types=export_types
+            )
         res = self.increment_output(messages, sources, is_tty, terminal_width)
         self.flush_caches()
         self.update_stats(res)
+        self.options.export_types = old_export_types
         return res
 
     def check(
@@ -412,17 +418,21 @@ def check(
         If is_tty is True format the output nicely with colors and summary line
         (unless disabled in self.options). Also pass the terminal_width to formatter.
         """
-        self.options.export_types = export_types
+        old_export_types = self.options.export_types
+        self.options.export_types = self.options.export_types or export_types
         if not self.fine_grained_manager:
             res = self.initialize_fine_grained(sources, is_tty, terminal_width)
         else:
             if not self.following_imports():
-                messages = self.fine_grained_increment(sources)
+                messages = self.fine_grained_increment(sources, explicit_export_types=export_types)
             else:
-                messages = self.fine_grained_increment_follow_imports(sources)
+                messages = self.fine_grained_increment_follow_imports(
+                    sources, explicit_export_types=export_types
+                )
             res = self.increment_output(messages, sources, is_tty, terminal_width)
         self.flush_caches()
         self.update_stats(res)
+        self.options.export_types = old_export_types
         return res
 
     def flush_caches(self) -> None:
@@ -461,6 +471,7 @@ def initialize_fine_grained(
         messages = result.errors
         self.fine_grained_manager = FineGrainedBuildManager(result)
 
+        original_sources_len = len(sources)
         if self.following_imports():
             sources = find_all_sources_in_build(self.fine_grained_manager.graph, sources)
             self.update_sources(sources)
@@ -525,7 +536,8 @@ def initialize_fine_grained(
 
         __, n_notes, __ = count_stats(messages)
         status = 1 if messages and n_notes < len(messages) else 0
-        messages = self.pretty_messages(messages, len(sources), is_tty, terminal_width)
+        # We use explicit sources length to match the logic in non-incremental mode.
+        messages = self.pretty_messages(messages, original_sources_len, is_tty, terminal_width)
         return {"out": "".join(s + "\n" for s in messages), "err": "", "status": status}
 
     def fine_grained_increment(
@@ -533,6 +545,7 @@ def fine_grained_increment(
         sources: list[BuildSource],
         remove: list[str] | None = None,
         update: list[str] | None = None,
+        explicit_export_types: bool = False,
     ) -> list[str]:
         """Perform a fine-grained type checking increment.
 
@@ -543,6 +556,8 @@ def fine_grained_increment(
             sources: sources passed on the command line
             remove: paths of files that have been removed
             update: paths of files that have been changed or created
+            explicit_export_types: --export-type was passed in a check command
+              (as opposite to being set in dmypy start)
         """
         assert self.fine_grained_manager is not None
         manager = self.fine_grained_manager.manager
@@ -557,6 +572,14 @@ def fine_grained_increment(
             # Use the remove/update lists to update fswatcher.
             # This avoids calling stat() for unchanged files.
             changed, removed = self.update_changed(sources, remove or [], update or [])
+        if explicit_export_types:
+            # If --export-types is given, we need to force full re-checking of all
+            # explicitly passed files, since we need to visit each expression.
+            changed += [
+                (bs.module, bs.path)
+                for bs in sources
+                if bs.path and (bs.module, bs.path) not in changed
+            ]
         changed += self.find_added_suppressed(
             self.fine_grained_manager.graph, set(), manager.search_paths
         )
@@ -575,7 +598,9 @@ def fine_grained_increment(
         self.previous_sources = sources
         return messages
 
-    def fine_grained_increment_follow_imports(self, sources: list[BuildSource]) -> list[str]:
+    def fine_grained_increment_follow_imports(
+        self, sources: list[BuildSource], explicit_export_types: bool = False
+    ) -> list[str]:
         """Like fine_grained_increment, but follow imports."""
         t0 = time.time()
 
@@ -601,6 +626,13 @@ def fine_grained_increment_follow_imports(self, sources: list[BuildSource]) -> l
         changed, new_files = self.find_reachable_changed_modules(
             sources, graph, seen, changed_paths
         )
+        if explicit_export_types:
+            # Same as in fine_grained_increment().
+            changed += [
+                (bs.module, bs.path)
+                for bs in sources
+                if bs.path and (bs.module, bs.path) not in changed
+            ]
         sources.extend(new_files)
 
         # Process changes directly reachable from roots.
diff --git a/mypy/test/testfinegrained.py b/mypy/test/testfinegrained.py
index ba0526d32558..3000cd66edc2 100644
--- a/mypy/test/testfinegrained.py
+++ b/mypy/test/testfinegrained.py
@@ -149,6 +149,7 @@ def get_options(self, source: str, testcase: DataDrivenTestCase, build_cache: bo
         options.use_fine_grained_cache = self.use_cache and not build_cache
         options.cache_fine_grained = self.use_cache
         options.local_partial_types = True
+        options.export_types = True
         options.enable_incomplete_feature = [TYPE_VAR_TUPLE, UNPACK]
         # Treat empty bodies safely for these test cases.
         options.allow_empty_bodies = not testcase.name.endswith("_no_empty")
@@ -164,7 +165,7 @@ def get_options(self, source: str, testcase: DataDrivenTestCase, build_cache: bo
         return options
 
     def run_check(self, server: Server, sources: list[BuildSource]) -> list[str]:
-        response = server.check(sources, export_types=True, is_tty=False, terminal_width=-1)
+        response = server.check(sources, export_types=False, is_tty=False, terminal_width=-1)
         out = response["out"] or response["err"]
         assert isinstance(out, str)
         return out.splitlines()
diff --git a/test-data/unit/daemon.test b/test-data/unit/daemon.test
index 18a03a92207d..99dcf1ccd941 100644
--- a/test-data/unit/daemon.test
+++ b/test-data/unit/daemon.test
@@ -360,6 +360,33 @@ def bar() -> None:
     x = foo('abc')  # type: str
     foo(arg='xyz')
 
+[case testDaemonInspectCheck]
+$ dmypy start
+Daemon started
+$ dmypy check foo.py
+Success: no issues found in 1 source file
+$ dmypy check foo.py --export-types
+Success: no issues found in 1 source file
+$ dmypy inspect foo.py:1:1
+"int"
+[file foo.py]
+x = 1
+
+[case testDaemonInspectRun]
+$ dmypy run test1.py
+Daemon started
+Success: no issues found in 1 source file
+$ dmypy run test2.py
+Success: no issues found in 1 source file
+$ dmypy run test1.py --export-types
+Success: no issues found in 1 source file
+$ dmypy inspect test1.py:1:1
+"int"
+[file test1.py]
+a: int
+[file test2.py]
+a: str
+
 [case testDaemonGetType]
 $ dmypy start --log-file log.txt -- --follow-imports=error --no-error-summary --python-version 3.8
 Daemon started


Code review comments:
diff_hunk: @@ -557,6 +572,14 @@ def fine_grained_increment(
             # Use the remove/update lists to update fswatcher.
             # This avoids calling stat() for unchanged files.
             changed, removed = self.update_changed(sources, remove or [], update or [])
+        if explicit_export_types:
+            # If --export-types is given, we need to force full re-checking of all
+            # explicitly passed files, since we need to visit each expression.
+            changed += [
+                (bs.module, bs.path)
+                for bs in sources
+                if bs.path and (bs.module, bs.path) not in changed
path: mypy/dmypy_server.py
text: @user1:
This is quadratic, and it could be slow if there are many files and many that are changed. Maybe construct a temporary set from `changed` and use it in the `not in changed` check?

@author:
Yes, good point, fixed now.

diff_hunk: @@ -601,6 +626,13 @@ def fine_grained_increment_follow_imports(self, sources: list[BuildSource]) -> l
         changed, new_files = self.find_reachable_changed_modules(
             sources, graph, seen, changed_paths
         )
+        if explicit_export_types:
+            # Same as in fine_grained_increment().
+            changed += [
+                (bs.module, bs.path)
+                for bs in sources
+                if bs.path and (bs.module, bs.path) not in changed
path: mypy/dmypy_server.py
text: @user1:
Similar to above. Maybe move this to a helper function to avoid code duplication?

@author:
Moved to a helper function.