Repository: python/mypy

Issue:
Use consistent order for messages from mypy daemon
If mypy daemon generates errors in multiple files, successive daemon runs can change the order of messages even if they are otherwise identical (e.g. the first run produces messages from `mod1` first followed by messages from `mod2`, and the second runs produces errors from `mod2` first and `mod1` next). Try to keep the order of error messages consistent across runs, since the varying order can be distracting when you are trying to fix errors one by one.

Suggested approach:
* Store the order of files where we reported errors after each daemon run as a list
* If we had errors in the previous run, sort the errors in the next run so that the order of files matches the previous run (and errors in additional files are at the end, for example)

Errors are already sorted consistently within a single file, but not across multiple files.

Another way would be to sort the messages by the file before reporting them, but this would be a bigger change conceptually and might have some issues (e.g. can't stream errors during the initial run). Preserving the order of errors from the immediate previous run shouldn't cause any additional issues. Related issue: #2144

Test cases to cover:
* Initial, non-daemon run generates errors.
* Two successive incremental runs generate errors.
* There is a successful run between runs that generate errors. In this case the errors can be reported in an arbitrary order.




Discussion:
I'm working on this right now, but i'm wondering what the best way to store state in-between function calls is, since the client is composed by functions and not by a class. how should i deal with this? Should i just store the file list in args? I also tried changing `do_run` into a callable class, but the tests didn't like it @JukkaL  

Resolving patch:
diff --git a/mypy/server/update.py b/mypy/server/update.py
--- a/mypy/server/update.py
+++ b/mypy/server/update.py
@@ -115,6 +115,7 @@
 from __future__ import annotations
 
 import os
+import re
 import sys
 import time
 from typing import Callable, NamedTuple, Sequence, Union
@@ -182,7 +183,7 @@ def __init__(self, result: BuildResult) -> None:
         # Merge in any root dependencies that may not have been loaded
         merge_dependencies(manager.load_fine_grained_deps(FAKE_ROOT_MODULE), self.deps)
         self.previous_targets_with_errors = manager.errors.targets()
-        self.previous_messages = result.errors[:]
+        self.previous_messages: list[str] = result.errors[:]
         # Module, if any, that had blocking errors in the last run as (id, path) tuple.
         self.blocking_error: tuple[str, str] | None = None
         # Module that we haven't processed yet but that are known to be stale.
@@ -290,6 +291,7 @@ def update(
                     messages = self.manager.errors.new_messages()
                     break
 
+        messages = sort_messages_preserving_file_order(messages, self.previous_messages)
         self.previous_messages = messages[:]
         return messages
 
@@ -1260,3 +1262,61 @@ def refresh_suppressed_submodules(
                                 state.suppressed.append(submodule)
                                 state.suppressed_set.add(submodule)
     return messages
+
+
+def extract_fnam_from_message(message: str) -> str | None:
+    m = re.match(r"([^:]+):[0-9]+: (error|note): ", message)
+    if m:
+        return m.group(1)
+    return None
+
+
+def extract_possible_fnam_from_message(message: str) -> str:
+    # This may return non-path things if there is some random colon on the line
+    return message.split(":", 1)[0]
+
+
+def sort_messages_preserving_file_order(
+    messages: list[str], prev_messages: list[str]
+) -> list[str]:
+    """Sort messages so that the order of files is preserved.
+
+    An update generates messages so that the files can be in a fairly
+    arbitrary order.  Preserve the order of files to avoid messages
+    getting reshuffled continuously.  If there are messages in
+    additional files, sort them towards the end.
+    """
+    # Calculate file order from the previous messages
+    n = 0
+    order = {}
+    for msg in prev_messages:
+        fnam = extract_fnam_from_message(msg)
+        if fnam and fnam not in order:
+            order[fnam] = n
+            n += 1
+
+    # Related messages must be sorted as a group of successive lines
+    groups = []
+    i = 0
+    while i < len(messages):
+        msg = messages[i]
+        maybe_fnam = extract_possible_fnam_from_message(msg)
+        group = [msg]
+        if maybe_fnam in order:
+            # This looks like a file name. Collect all lines related to this message.
+            while (
+                i + 1 < len(messages)
+                and extract_possible_fnam_from_message(messages[i + 1]) not in order
+                and extract_fnam_from_message(messages[i + 1]) is None
+                and not messages[i + 1].startswith("mypy: ")
+            ):
+                i += 1
+                group.append(messages[i])
+        groups.append((order.get(maybe_fnam, n), group))
+        i += 1
+
+    groups = sorted(groups, key=lambda g: g[0])
+    result = []
+    for key, group in groups:
+        result.extend(group)
+    return result


Test patch:
diff --git a/mypy/test/testfinegrained.py b/mypy/test/testfinegrained.py
--- a/mypy/test/testfinegrained.py
+++ b/mypy/test/testfinegrained.py
@@ -17,6 +17,7 @@
 import os
 import re
 import sys
+import unittest
 from typing import Any, cast
 
 import pytest
@@ -30,6 +31,7 @@
 from mypy.modulefinder import BuildSource
 from mypy.options import Options
 from mypy.server.mergecheck import check_consistency
+from mypy.server.update import sort_messages_preserving_file_order
 from mypy.test.config import test_temp_dir
 from mypy.test.data import DataDrivenTestCase, DataSuite, DeleteFile, UpdateFile
 from mypy.test.helpers import (
@@ -369,3 +371,70 @@ def get_inspect(self, program_text: str, incremental_step: int) -> list[tuple[st
 
 def normalize_messages(messages: list[str]) -> list[str]:
     return [re.sub("^tmp" + re.escape(os.sep), "", message) for message in messages]
+
+
+class TestMessageSorting(unittest.TestCase):
+    def test_simple_sorting(self) -> None:
+        msgs = ['x.py:1: error: "int" not callable', 'foo/y.py:123: note: "X" not defined']
+        old_msgs = ['foo/y.py:12: note: "Y" not defined', 'x.py:8: error: "str" not callable']
+        assert sort_messages_preserving_file_order(msgs, old_msgs) == list(reversed(msgs))
+        assert sort_messages_preserving_file_order(list(reversed(msgs)), old_msgs) == list(
+            reversed(msgs)
+        )
+
+    def test_long_form_sorting(self) -> None:
+        # Multi-line errors should be sorted together and not split.
+        msg1 = [
+            'x.py:1: error: "int" not callable',
+            "and message continues (x: y)",
+            "    1()",
+            "    ^~~",
+        ]
+        msg2 = [
+            'foo/y.py: In function "f":',
+            'foo/y.py:123: note: "X" not defined',
+            "and again message continues",
+        ]
+        old_msgs = ['foo/y.py:12: note: "Y" not defined', 'x.py:8: error: "str" not callable']
+        assert sort_messages_preserving_file_order(msg1 + msg2, old_msgs) == msg2 + msg1
+        assert sort_messages_preserving_file_order(msg2 + msg1, old_msgs) == msg2 + msg1
+
+    def test_mypy_error_prefix(self) -> None:
+        # Some errors don't have a file and start with "mypy: ". These
+        # shouldn't be sorted together with file-specific errors.
+        msg1 = 'x.py:1: error: "int" not callable'
+        msg2 = 'foo/y:123: note: "X" not defined'
+        msg3 = "mypy: Error not associated with a file"
+        old_msgs = [
+            "mypy: Something wrong",
+            'foo/y:12: note: "Y" not defined',
+            'x.py:8: error: "str" not callable',
+        ]
+        assert sort_messages_preserving_file_order([msg1, msg2, msg3], old_msgs) == [
+            msg2,
+            msg1,
+            msg3,
+        ]
+        assert sort_messages_preserving_file_order([msg3, msg2, msg1], old_msgs) == [
+            msg2,
+            msg1,
+            msg3,
+        ]
+
+    def test_new_file_at_the_end(self) -> None:
+        msg1 = 'x.py:1: error: "int" not callable'
+        msg2 = 'foo/y.py:123: note: "X" not defined'
+        new1 = "ab.py:3: error: Problem: error"
+        new2 = "aaa:3: error: Bad"
+        old_msgs = ['foo/y.py:12: note: "Y" not defined', 'x.py:8: error: "str" not callable']
+        assert sort_messages_preserving_file_order([msg1, msg2, new1], old_msgs) == [
+            msg2,
+            msg1,
+            new1,
+        ]
+        assert sort_messages_preserving_file_order([new1, msg1, msg2, new2], old_msgs) == [
+            msg2,
+            msg1,
+            new1,
+            new2,
+        ]
diff --git a/test-data/unit/fine-grained-blockers.test b/test-data/unit/fine-grained-blockers.test
--- a/test-data/unit/fine-grained-blockers.test
+++ b/test-data/unit/fine-grained-blockers.test
@@ -317,8 +317,8 @@ a.py:1: error: invalid syntax
 ==
 a.py:1: error: invalid syntax
 ==
-b.py:3: error: Too many arguments for "f"
 a.py:3: error: Too many arguments for "g"
+b.py:3: error: Too many arguments for "f"
 
 [case testDeleteFileWithBlockingError-only_when_nocache]
 -- Different cache/no-cache tests because:
diff --git a/test-data/unit/fine-grained-follow-imports.test b/test-data/unit/fine-grained-follow-imports.test
--- a/test-data/unit/fine-grained-follow-imports.test
+++ b/test-data/unit/fine-grained-follow-imports.test
@@ -587,8 +587,8 @@ def f() -> None:
 main.py:2: error: Cannot find implementation or library stub for module named "p"
 main.py:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
 ==
-p/m.py:1: error: "str" not callable
 p/__init__.py:1: error: "int" not callable
+p/m.py:1: error: "str" not callable
 
 [case testFollowImportsNormalPackageInitFileStub]
 # flags: --follow-imports=normal
@@ -610,11 +610,11 @@ x x x
 main.py:1: error: Cannot find implementation or library stub for module named "p"
 main.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
 ==
-p/m.pyi:1: error: "str" not callable
 p/__init__.pyi:1: error: "int" not callable
-==
 p/m.pyi:1: error: "str" not callable
+==
 p/__init__.pyi:1: error: "int" not callable
+p/m.pyi:1: error: "str" not callable
 
 [case testFollowImportsNormalNamespacePackages]
 # flags: --follow-imports=normal --namespace-packages
@@ -638,12 +638,12 @@ main.py:2: error: Cannot find implementation or library stub for module named "p
 main.py:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
 main.py:2: error: Cannot find implementation or library stub for module named "p2"
 ==
-p2/m2.py:1: error: "str" not callable
 p1/m1.py:1: error: "int" not callable
+p2/m2.py:1: error: "str" not callable
 ==
+p1/m1.py:1: error: "int" not callable
 main.py:2: error: Cannot find implementation or library stub for module named "p2.m2"
 main.py:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
-p1/m1.py:1: error: "int" not callable
 
 [case testFollowImportsNormalNewFileOnCommandLine]
 # flags: --follow-imports=normal
@@ -659,8 +659,8 @@ p1/m1.py:1: error: "int" not callable
 [out]
 main.py:1: error: "int" not callable
 ==
-x.py:1: error: "str" not callable
 main.py:1: error: "int" not callable
+x.py:1: error: "str" not callable
 
 [case testFollowImportsNormalSearchPathUpdate-only_when_nocache]
 # flags: --follow-imports=normal
@@ -678,8 +678,8 @@ import bar
 
 [out]
 ==
-src/bar.py:1: error: "int" not callable
 src/foo.py:2: error: "str" not callable
+src/bar.py:1: error: "int" not callable
 
 [case testFollowImportsNormalSearchPathUpdate2-only_when_cache]
 # flags: --follow-imports=normal
diff --git a/test-data/unit/fine-grained-modules.test b/test-data/unit/fine-grained-modules.test
--- a/test-data/unit/fine-grained-modules.test
+++ b/test-data/unit/fine-grained-modules.test
@@ -38,8 +38,8 @@ def f(x: int) -> None: pass
 ==
 a.py:2: error: Incompatible return value type (got "int", expected "str")
 ==
-b.py:2: error: Too many arguments for "f"
 a.py:2: error: Incompatible return value type (got "int", expected "str")
+b.py:2: error: Too many arguments for "f"
 ==
 
 [case testAddFileFixesError]
diff --git a/test-data/unit/fine-grained.test b/test-data/unit/fine-grained.test
--- a/test-data/unit/fine-grained.test
+++ b/test-data/unit/fine-grained.test
@@ -1814,9 +1814,9 @@ def f() -> Iterator[None]:
 [out]
 main:2: note: Revealed type is "contextlib.GeneratorContextManager[None]"
 ==
+main:2: note: Revealed type is "contextlib.GeneratorContextManager[None]"
 a.py:3: error: Cannot find implementation or library stub for module named "b"
 a.py:3: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
-main:2: note: Revealed type is "contextlib.GeneratorContextManager[None]"
 ==
 main:2: note: Revealed type is "contextlib.GeneratorContextManager[None]"
 
@@ -8689,8 +8689,8 @@ main:2: note: Revealed type is "builtins.int"
 ==
 main:2: note: Revealed type is "Literal[1]"
 ==
-mod.py:2: error: Incompatible types in assignment (expression has type "Literal[2]", variable has type "Literal[1]")
 main:2: note: Revealed type is "Literal[1]"
+mod.py:2: error: Incompatible types in assignment (expression has type "Literal[2]", variable has type "Literal[1]")
 
 [case testLiteralFineGrainedFunctionConversion]
 from mod import foo
@@ -9178,10 +9178,10 @@ a.py:1: error: Type signature has too few arguments
 a.py:5: error: Type signature has too few arguments
 a.py:11: error: Type signature has too few arguments
 ==
+c.py:1: error: Type signature has too few arguments
 a.py:1: error: Type signature has too few arguments
 a.py:5: error: Type signature has too few arguments
 a.py:11: error: Type signature has too few arguments
-c.py:1: error: Type signature has too few arguments
 
 [case testErrorReportingNewAnalyzer]
 # flags: --disallow-any-generics
@@ -10072,3 +10072,53 @@ class Base(Protocol):
 [out]
 ==
 main:6: error: Call to abstract method "meth" of "Base" with trivial body via super() is unsafe
+
+[case testPrettyMessageSorting]
+# flags: --pretty
+import a
+
+[file a.py]
+1 + ''
+import b
+
+[file b.py]
+object + 1
+
+[file b.py.2]
+object + 1
+1()
+
+[out]
+b.py:1: error: Unsupported left operand type for + ("Type[object]")
+    object + 1
+    ^
+a.py:1: error: Unsupported operand types for + ("int" and "str")
+    1 + ''
+        ^
+==
+b.py:1: error: Unsupported left operand type for + ("Type[object]")
+    object + 1
+    ^
+b.py:2: error: "int" not callable
+    1()
+    ^
+a.py:1: error: Unsupported operand types for + ("int" and "str")
+    1 + ''
+        ^
+[out version>=3.8]
+b.py:1: error: Unsupported left operand type for + ("Type[object]")
+    object + 1
+    ^~~~~~~~~~
+a.py:1: error: Unsupported operand types for + ("int" and "str")
+    1 + ''
+        ^~
+==
+b.py:1: error: Unsupported left operand type for + ("Type[object]")
+    object + 1
+    ^~~~~~~~~~
+b.py:2: error: "int" not callable
+    1()
+    ^~~
+a.py:1: error: Unsupported operand types for + ("int" and "str")
+    1 + ''
+        ^~