Repository: python/mypy

Issue:
Two issues with subclassing namedtuple-generated class and overriding __new__
Python 3.5.1, mypy 2f89c79.

Test code:

```
from typing import NamedTuple

Base = NamedTuple('Base', [('param', int)])


class Child(Base):

    def __new__(cls, param: int = 1) -> 'Child':
        return Base.__new__(cls, param)


print(Child())
```

Python result:

```
% python3 test2.py 
Child(param=1)
```

mypy output:

```
test2.py: note: In member "__new__" of class "Child":
test2.py:9: error: Too many arguments for "__new__" of "object"
test2.py: note: At top level:
test2.py:12: error: Too few arguments for "Child"
```

I poked around in the code a bit and I believe I managed to narrow this down to the following difference:
- The code above depends on the namedtuple-generated class to have `__new__` defined (CPython: https://github.com/python/cpython/blob/9967d6c1053f186f3b3d118ac336a03752efec32/Lib/collections/__init__.py#L316)
- mypy treats namedtuple definitions as if they were overriding `__init__` instead (https://github.com/python/mypy/blob/2f89c79a5b030343879f2f3c7172f0ad2b00c4de/mypy/semanal.py#L1444)



Discussion:
Yeah, mypy should probably define NamedTuple in terms of `__new__`, not `__init__`. There are some deficiencies in mypy's `__new__` support though that may make this not simple until they are first fixed.

In particular #1021 looks like a dependency for this. (And we haven't assigned that a milestone yet, so we can't assign a milestone here either.)

I propose to bump the priority here. This seems a real problem to some of our users.
> I propose to bump the priority here. This seems a real problem to some of our users.

Sure, although TBH this will not help much, we are already overloaded with work.
Indeed. :-(

And aggressively critical users don't motivate me.

Resolving patch:
diff --git a/mypy/checker.py b/mypy/checker.py
--- a/mypy/checker.py
+++ b/mypy/checker.py
@@ -1563,8 +1563,8 @@ def check_compatibility(self, name: str, base1: TypeInfo,
         a direct subclass relationship (i.e., the compatibility requirement only derives from
         multiple inheritance).
         """
-        if name == '__init__':
-            # __init__ can be incompatible -- it's a special case.
+        if name in ('__init__', '__new__', '__init_subclass__'):
+            # __init__ and friends can be incompatible -- it's a special case.
             return
         first = base1[name]
         second = base2[name]
diff --git a/mypy/semanal.py b/mypy/semanal.py
--- a/mypy/semanal.py
+++ b/mypy/semanal.py
@@ -776,7 +776,7 @@ def analyze_class_body(self, defn: ClassDef) -> Iterator[bool]:
                 nt_names = named_tuple_info.names
                 named_tuple_info.names = SymbolTable()
                 # This is needed for the cls argument to classmethods to get bound correctly.
-                named_tuple_info.names['__init__'] = nt_names['__init__']
+                named_tuple_info.names['__new__'] = nt_names['__new__']
 
                 self.enter_class(named_tuple_info)
 
diff --git a/mypy/semanal_namedtuple.py b/mypy/semanal_namedtuple.py
--- a/mypy/semanal_namedtuple.py
+++ b/mypy/semanal_namedtuple.py
@@ -347,8 +347,9 @@ def add_method(funcname: str,
                        args: List[Argument],
                        name: Optional[str] = None,
                        is_classmethod: bool = False,
+                       is_new: bool = False,
                        ) -> None:
-            if is_classmethod:
+            if is_classmethod or is_new:
                 first = [Argument(Var('cls'), TypeType.make_normalized(selftype), None, ARG_POS)]
             else:
                 first = [Argument(Var('self'), selftype, None, ARG_POS)]
@@ -384,8 +385,9 @@ def make_init_arg(var: Var) -> Argument:
             kind = ARG_POS if default is None else ARG_OPT
             return Argument(var, var.type, default, kind)
 
-        add_method('__init__', ret=NoneTyp(), name=info.name(),
-                   args=[make_init_arg(var) for var in vars])
+        add_method('__new__', ret=selftype, name=info.name(),
+                   args=[make_init_arg(var) for var in vars],
+                   is_new=True)
         add_method('_asdict', args=[], ret=ordereddictype)
         special_form_any = AnyType(TypeOfAny.special_form)
         add_method('_make', ret=selftype, is_classmethod=True,


Test patch:
diff --git a/test-data/unit/check-class-namedtuple.test b/test-data/unit/check-class-namedtuple.test
--- a/test-data/unit/check-class-namedtuple.test
+++ b/test-data/unit/check-class-namedtuple.test
@@ -587,7 +587,7 @@ class XMethBad(NamedTuple):
 class MagicalFields(NamedTuple):
     x: int
     def __slots__(self) -> None: pass  # E: Cannot overwrite NamedTuple attribute "__slots__"
-    def __new__(cls) -> None: pass  # E: Cannot overwrite NamedTuple attribute "__new__"
+    def __new__(cls) -> None: pass  # E: Name '__new__' already defined (possibly by an import)
     def _source(self) -> int: pass  # E: Cannot overwrite NamedTuple attribute "_source"
     __annotations__ = {'x': float}  # E: NamedTuple field name cannot start with an underscore: __annotations__ \
         # E: Invalid statement in NamedTuple definition; expected "field_name: field_type [= default]" \
diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test
--- a/test-data/unit/check-namedtuple.test
+++ b/test-data/unit/check-namedtuple.test
@@ -748,3 +748,15 @@ def f(o: Base) -> None:
         reveal_type(o.value)  # E: Revealed type is 'builtins.float'
 [builtins fixtures/isinstance.pyi]
 [out]
+
+[case testNamedTupleNew]
+from typing import NamedTuple
+
+Base = NamedTuple('Base', [('param', int)])
+
+class Child(Base):
+    def __new__(cls, param: int = 1) -> 'Child':
+        return Base.__new__(cls, param)
+
+Base(param=10)
+Child(param=10)
diff --git a/test-data/unit/cmdline.test b/test-data/unit/cmdline.test
--- a/test-data/unit/cmdline.test
+++ b/test-data/unit/cmdline.test
@@ -1182,7 +1182,7 @@ warn_unused_configs = True
 [[mypy-emarg.hatch]
 -- Currently we don't treat an unstructured pattern like a.*.b as unused
 -- if it matches another section (like a.x.b). This would be reasonable
--- to change.
+-- to change. '
 [[mypy-a.*.b]
 [[mypy-a.*.c]
 [[mypy-a.x.b]
@@ -1257,6 +1257,6 @@ import d
 [case testCacheMap]
 -- This just checks that a valid --cache-map triple is accepted.
 -- (Errors are too verbose to check.)
-# cmd: mypy a.py --cache-map a.py a.meta.json a.data.json 
+# cmd: mypy a.py --cache-map a.py a.meta.json a.data.json
 [file a.py]
 [out]
diff --git a/test-data/unit/diff.test b/test-data/unit/diff.test
--- a/test-data/unit/diff.test
+++ b/test-data/unit/diff.test
@@ -275,7 +275,7 @@ M = NamedTuple('M', [('x', int), ('y', str)])
 [out]
 __main__.A
 __main__.N
-__main__.N.__init__
+__main__.N.__new__
 __main__.N._asdict
 __main__.N._make
 __main__.N._replace
diff --git a/test-data/unit/merge.test b/test-data/unit/merge.test
--- a/test-data/unit/merge.test
+++ b/test-data/unit/merge.test
@@ -668,7 +668,7 @@ TypeInfo<2>(
   Names(
     __annotations__<4> (builtins.object<1>)
     __doc__<5> (builtins.str<6>)
-    __init__<7>
+    __new__<7>
     _asdict<8>
     _field_defaults<9> (builtins.object<1>)
     _field_types<10> (builtins.object<1>)
@@ -690,7 +690,7 @@ TypeInfo<2>(
   Names(
     __annotations__<4> (builtins.object<1>)
     __doc__<5> (builtins.str<6>)
-    __init__<7>
+    __new__<7>
     _asdict<8>
     _field_defaults<9> (builtins.object<1>)
     _field_types<10> (builtins.object<1>)
diff --git a/test-data/unit/pythoneval.test b/test-data/unit/pythoneval.test
--- a/test-data/unit/pythoneval.test
+++ b/test-data/unit/pythoneval.test
@@ -26,6 +26,7 @@ print(list(reversed(A())))
 [[3, 2, 1]
 [['c', 'b', 'a']
 [['f', 'o', 'o']
+-- ]]]]]
 
 [case testIntAndFloatConversion]
 from typing import SupportsInt, SupportsFloat
@@ -90,6 +91,7 @@ import typing
 print(list.__add__([1, 2], [3, 4]))
 [out]
 [[1, 2, 3, 4]
+-- ]
 
 [case testInheritedClassAttribute]
 import typing
@@ -1047,7 +1049,7 @@ _testTypedDictGet.py:9: error: TypedDict "D" has no key 'z'
 _testTypedDictGet.py:10: error: All overload variants of "get" of "Mapping" require at least one argument
 _testTypedDictGet.py:10: note: Possible overload variants:
 _testTypedDictGet.py:10: note:     def get(self, k: str) -> object
-_testTypedDictGet.py:10: note:     def [_T] get(self, k: str, default: object) -> object 
+_testTypedDictGet.py:10: note:     def [_T] get(self, k: str, default: object) -> object
 _testTypedDictGet.py:12: error: Revealed type is 'builtins.object*'
 
 [case testTypedDictMappingMethods]
@@ -1317,3 +1319,24 @@ def g(ms: 'T[M]') -> None:
     reduce(f, ms)
 T = Iterable
 [out]
+
+[case testNamedTupleNew]
+# This is an eval test because there was a snag found only with full stubs
+from typing import NamedTuple
+
+Base = NamedTuple('Base', [('param', int)])
+
+class Child(Base):
+    def __new__(cls, param: int = 1) -> 'Child':
+        return Base.__new__(cls, param)
+
+Base(param=10)
+Child(param=10)
+reveal_type(Child())
+
+from collections import namedtuple
+X = namedtuple('X', ['a', 'b'])
+x = X(a=1, b='s')
+
+[out]
+_testNamedTupleNew.py:12: error: Revealed type is 'Tuple[builtins.int, fallback=_testNamedTupleNew.Child]'