Repository: python/mypy

Issue:
New semantic analyzer: should we collect functions from AST instead of symbol tables?
Currently, functions are collected for processing from symbol tables, not form ASTs. This causes some issues, for example, if there is method name clash in class-based named tuple, then the user provided method will be not analyzed, which will cause a crash later, see `testNewNamedTupleIllegalNames`.

This is a follow-up for https://github.com/python/mypy/pull/6452


Discussion:
This is something I started to wonder about as well. This may also affect fine-grained incremental mode, and it would be nice if the solution we come up would support that as well. I see three plausible alternatives for how to deal with this:

* Rename definitions with name clashes. For example, we can use dummy suffixes, as we currently do for certain things. We'd need to extend this to deal with methods at least, and to make sure the renamed functions work consistently in every context.
* Traverse the module top level to find all functions, as suggested above. This may not work well in the context of mypy daemon, however.
* Treat functions with name conflicts as part of the module top level instead of as independent functions.
#6783 has a concrete example of a crash with dataclasses that's caused by this.

Resolving patch:
diff --git a/mypy/newsemanal/semanal_namedtuple.py b/mypy/newsemanal/semanal_namedtuple.py
--- a/mypy/newsemanal/semanal_namedtuple.py
+++ b/mypy/newsemanal/semanal_namedtuple.py
@@ -20,6 +20,7 @@
 )
 from mypy.options import Options
 from mypy.exprtotype import expr_to_unanalyzed_type, TypeTranslationError
+from mypy.util import get_unique_redefinition_name
 
 MYPY = False
 if MYPY:
@@ -493,10 +494,15 @@ def save_namedtuple_body(self, named_tuple_info: TypeInfo) -> Iterator[None]:
         # Restore the names in the original symbol table. This ensures that the symbol
         # table contains the field objects created by build_namedtuple_typeinfo. Exclude
         # __doc__, which can legally be overwritten by the class.
-        named_tuple_info.names.update({
-            key: value for key, value in nt_names.items()
-            if key not in named_tuple_info.names or key != '__doc__'
-        })
+        for key, value in nt_names.items():
+            if key in named_tuple_info.names:
+                if key == '__doc__':
+                    continue
+                # Keep existing (user-provided) definitions under mangled names, so they
+                # get semantically analyzed.
+                r_key = get_unique_redefinition_name(key, named_tuple_info.names)
+                named_tuple_info.names[r_key] = named_tuple_info.names[key]
+            named_tuple_info.names[key] = value
 
     # Helpers
 
diff --git a/mypy/plugins/common.py b/mypy/plugins/common.py
--- a/mypy/plugins/common.py
+++ b/mypy/plugins/common.py
@@ -8,6 +8,7 @@
 from mypy.semanal import set_callable_name
 from mypy.types import CallableType, Overloaded, Type, TypeVarDef, LiteralType, Instance
 from mypy.typevars import fill_typevars
+from mypy.util import get_unique_redefinition_name
 
 
 def _get_decorator_bool_argument(
@@ -117,6 +118,13 @@ def add_method(
     func._fullname = info.fullname() + '.' + name
     func.line = info.line
 
+    # NOTE: we would like the plugin generated node to dominate, but we still
+    # need to keep any existing definitions so they get semantically analyzed.
+    if name in info.names:
+        # Get a nice unique name instead.
+        r_name = get_unique_redefinition_name(name, info.names)
+        info.names[r_name] = info.names[name]
+
     info.names[name] = SymbolTableNode(MDEF, func, plugin_generated=True)
     info.defn.defs.body.append(func)
 
diff --git a/mypy/util.py b/mypy/util.py
--- a/mypy/util.py
+++ b/mypy/util.py
@@ -4,7 +4,7 @@
 import re
 import subprocess
 import sys
-from typing import TypeVar, List, Tuple, Optional, Dict, Sequence, Iterable
+from typing import TypeVar, List, Tuple, Optional, Dict, Sequence, Iterable, Container
 
 MYPY = False
 if MYPY:
@@ -290,6 +290,22 @@ def unmangle(name: str) -> str:
     return name.rstrip("'")
 
 
+def get_unique_redefinition_name(name: str, existing: Container[str]) -> str:
+    """Get a simple redefinition name not present among existing.
+
+    For example, for name 'foo' we try 'foo-redefinition', 'foo-redefinition2',
+    'foo-redefinition3', etc. until we find one that is not in existing.
+    """
+    r_name = name + '-redefinition'
+    if r_name not in existing:
+        return r_name
+
+    i = 2
+    while r_name + str(i) in existing:
+        i += 1
+    return r_name + str(i)
+
+
 def check_python_version(program: str) -> None:
     """Report issues with the Python used to run mypy, dmypy, or stubgen"""
     # Check for known bad Python versions.


Test patch:
diff --git a/test-data/unit/check-class-namedtuple.test b/test-data/unit/check-class-namedtuple.test
--- a/test-data/unit/check-class-namedtuple.test
+++ b/test-data/unit/check-class-namedtuple.test
@@ -587,9 +587,7 @@ reveal_type(takes_base(Base(1)))  # E: Revealed type is 'builtins.int'
 reveal_type(takes_base(Child(1)))  # E: Revealed type is 'builtins.int'
 [builtins fixtures/tuple.pyi]
 
--- Depends on collecting functions via AST, not symbol tables.
 [case testNewNamedTupleIllegalNames]
-# flags: --no-new-semantic-analyzer
 from typing import Callable, NamedTuple
 
 class XMethBad(NamedTuple):
@@ -614,16 +612,16 @@ class AnnotationsAsAMethod(NamedTuple):
 
 class ReuseNames(NamedTuple):
     x: int
-    def x(self) -> str:  # E: Name 'x' already defined on line 23
+    def x(self) -> str:  # E: Name 'x' already defined on line 22
         return ''
 
     def y(self) -> int:
         return 0
-    y: str  # E: Name 'y' already defined on line 27
+    y: str  # E: Name 'y' already defined on line 26
 
 class ReuseCallableNamed(NamedTuple):
     z: Callable[[ReuseNames], int]
-    def z(self) -> int:  # E: Name 'z' already defined on line 32
+    def z(self) -> int:  # E: Name 'z' already defined on line 31
         return 0
 
 [builtins fixtures/dict.pyi]
diff --git a/test-data/unit/check-dataclasses.test b/test-data/unit/check-dataclasses.test
--- a/test-data/unit/check-dataclasses.test
+++ b/test-data/unit/check-dataclasses.test
@@ -393,9 +393,8 @@ class Application:  # E: eq must be True if order is True
 
 [builtins fixtures/list.pyi]
 
--- Blocked by #6454
 [case testDataclassOrderingWithCustomMethods]
-# flags: --python-version 3.6  --no-new-semantic-analyzer
+# flags: --python-version 3.6
 from dataclasses import dataclass
 
 @dataclass(order=True)