Repository: python/mypy

Issue:
Make tuple types subtypes of Sized, Container, Reversible, etc.
Currently `(1, 1)` is compatible with `Sequence[int]` and `Iterable[int]` but not with `Sized` or `Container[int]`, which is inconsistent.

This code should be fine, but currently mypy complains about it:

```
from typing import Sized, Container
def f(x: Sized) -> None: pass
f((1, 2)) # Argument 1 to "f" has incompatible type "Tuple[int, int]"; expected "Sized"
def g(x: Container[int]) -> None: pass
g((1, 2)) # Argument 1 to "g" has incompatible type "Tuple[int, int]"; expected Container[int]
```

Once this has been implemented, also simplify the signature of `len()` to not have `tuple` explicitly (`Sized` should be enough for the argument).



Discussion:

Resolving patch:
diff --git a/mypyc/genops.py b/mypyc/genops.py
--- a/mypyc/genops.py
+++ b/mypyc/genops.py
@@ -86,7 +86,8 @@ def f(x: int) -> int:
     py_call_op, py_call_with_kwargs_op, py_method_call_op,
     fast_isinstance_op, bool_op, new_slice_op,
     type_op, pytype_from_template_op, import_op, get_module_dict_op,
-    ellipsis_op, method_new_op, type_is_op, type_object_op, py_calc_meta_op
+    ellipsis_op, method_new_op, type_is_op, type_object_op, py_calc_meta_op,
+    dataclass_sleight_of_hand,
 )
 from mypyc.ops_exc import (
     raise_exception_op, raise_exception_with_tb_op, reraise_exception_op,
@@ -202,8 +203,32 @@ def build_ir(modules: List[MypyFile],
     return result
 
 
+def is_trait_decorator(d: Expression) -> bool:
+    return isinstance(d, RefExpr) and d.fullname == 'mypy_extensions.trait'
+
+
+def is_trait(cdef: ClassDef) -> bool:
+    return any(is_trait_decorator(d) for d in cdef.decorators)
+
+
+def is_dataclass_decorator(d: Expression) -> bool:
+    return (
+        (isinstance(d, RefExpr) and d.fullname == 'dataclasses.dataclass')
+        or (
+            isinstance(d, CallExpr)
+            and isinstance(d.callee, RefExpr)
+            and d.callee.fullname == 'dataclasses.dataclass'
+        )
+    )
+
+
+def is_dataclass(cdef: ClassDef) -> bool:
+    return any(is_dataclass_decorator(d) for d in cdef.decorators)
+
+
 def is_extension_class(cdef: ClassDef) -> bool:
-    if any(not (isinstance(d, RefExpr) and d.fullname == 'mypy_extensions.trait')
+
+    if any(not is_trait_decorator(d) and not is_dataclass_decorator(d)
            for d in cdef.decorators):
         return False
     elif (cdef.info.metaclass_type and cdef.info.metaclass_type.type.fullname() not in (
@@ -229,6 +254,7 @@ def mark_non_ext_classes(class_map: Dict[TypeInfo, ClassIR]) -> None:
         if not ir.is_ext_class:
             visit_second.append(typ)
 
+    # FIXME: Just reject these?
     # Second pass to propagate non-extension markings up the base class
     # chains of classes marked as non-extension classes during the first pass.
     for typ in visit_second:
@@ -244,11 +270,6 @@ def mark_non_ext_classes(class_map: Dict[TypeInfo, ClassIR]) -> None:
                     todo.append(parent.type)
 
 
-def is_trait(cdef: ClassDef) -> bool:
-    return any(d.fullname == 'mypy_extensions.trait' for d in cdef.decorators
-               if isinstance(d, NameExpr))
-
-
 def get_func_def(op: Union[FuncDef, Decorator, OverloadedFuncDef]) -> FuncDef:
     if isinstance(op, OverloadedFuncDef):
         assert op.impl
@@ -536,6 +557,10 @@ def prepare_class_def(path: str, module_name: str, cdef: ClassDef,
 
     # We sort the table for determinism here on Python 3.5
     for name, node in sorted(info.names.items()):
+        # Currenly all plugin generated methods are dummies and not included.
+        if node.plugin_generated:
+            continue
+
         if isinstance(node.node, Var):
             assert node.node.type, "Class member %s missing type" % name
             if not node.node.is_classvar and name != '__slots__':
@@ -585,7 +610,8 @@ def prepare_class_def(path: str, module_name: str, cdef: ClassDef,
         # If there is a nontrivial __init__ that wasn't defined in an
         # extension class, we need to make the constructor take *args,
         # **kwargs so it can call tp_init.
-        if ((defining_ir is None or not defining_ir.is_ext_class)
+        if ((defining_ir is None or not defining_ir.is_ext_class
+             or cdef.info['__init__'].plugin_generated)
                 and init_node.info.fullname() != 'builtins.object'):
             init_sig = FuncSignature(
                 [init_sig.args[0],
@@ -630,6 +656,9 @@ def prepare_class_def(path: str, module_name: str, cdef: ClassDef,
     for base in bases:
         base.children.append(ir)
 
+    if is_dataclass(cdef):
+        ir.is_augmented = True
+
 
 def prepare_non_ext_class_def(path: str, module_name: str, cdef: ClassDef,
                               errors: Errors, mapper: Mapper) -> None:
@@ -1233,6 +1262,10 @@ def generate_attr_defaults(self, cdef: ClassDef) -> None:
                     if stmt.lvalues[0].name == '__slots__':
                         continue
 
+                    # Skip type annotated assignments in dataclasses
+                    if is_dataclass(cdef) and stmt.type:
+                        continue
+
                     default_assignments.append(stmt)
 
         if not default_assignments:
@@ -1271,9 +1304,7 @@ def generate_attr_defaults(self, cdef: ClassDef) -> None:
         self.functions.append(ir)
         cls.methods[ir.name] = ir
 
-    def load_non_ext_class(self, ir: ClassIR, non_ext: NonExtClassInfo, line: int) -> Value:
-        cls_name = self.load_static_unicode(ir.name)
-
+    def finish_non_ext_dict(self, non_ext: NonExtClassInfo, line: int) -> None:
         # Add __annotations__ to the class dict.
         self.primitive_op(dict_set_item_op,
                           [non_ext.dict, self.load_static_unicode('__annotations__'),
@@ -1282,11 +1313,17 @@ def load_non_ext_class(self, ir: ClassIR, non_ext: NonExtClassInfo, line: int) -
         # We add a __doc__ attribute so if the non-extension class is decorated with the
         # dataclass decorator, dataclass will not try to look for __text_signature__.
         # https://github.com/python/cpython/blob/3.7/Lib/dataclasses.py#L957
-        filler_doc_str = 'filler docstring for classes decorated with dataclass'
+        filler_doc_str = 'mypyc filler docstring'
         self.add_to_non_ext_dict(
             non_ext, '__doc__', self.load_static_unicode(filler_doc_str), line)
         self.add_to_non_ext_dict(
             non_ext, '__module__', self.load_static_unicode(self.module_name), line)
+
+    def load_non_ext_class(self, ir: ClassIR, non_ext: NonExtClassInfo, line: int) -> Value:
+        cls_name = self.load_static_unicode(ir.name)
+
+        self.finish_non_ext_dict(non_ext, line)
+
         metaclass = self.primitive_op(type_object_op, [], line)
         metaclass = self.primitive_op(py_calc_meta_op, [metaclass, non_ext.bases], line)
 
@@ -1399,6 +1436,46 @@ def cache_class_attrs(self, attrs_to_cache: List[Lvalue], cdef: ClassDef) -> Non
             rval = self.py_get_attr(typ, lval.name, cdef.line)
             self.init_final_static(lval, rval, cdef.name)
 
+    def dataclass_non_ext_info(self, cdef: ClassDef) -> Optional[NonExtClassInfo]:
+        """Set up a NonExtClassInfo to track dataclass attributes.
+
+        In addition to setting up a normal extension class for dataclasses,
+        we also collect its class attributes like a non-extension class so
+        that we can hand them to the dataclass decorator.
+        """
+        if is_dataclass(cdef):
+            return NonExtClassInfo(
+                self.primitive_op(new_dict_op, [], cdef.line),
+                self.add(TupleSet([], cdef.line)),
+                self.primitive_op(new_dict_op, [], cdef.line),
+            )
+        else:
+            return None
+
+    def dataclass_finalize(
+            self, cdef: ClassDef, non_ext: NonExtClassInfo, type_obj: Value) -> None:
+        """Generate code to finish instantiating a dataclass.
+
+        This works by replacing all of the attributes on the class
+        (which will be descriptors) with whatever they would be in a
+        non-extension class, calling dataclass, then switching them back.
+
+        The resulting class is an extension class and instances of it do not
+        have a __dict__ (unless something else requires it).
+        All methods written explicitly in the source are compiled and
+        may be called through the vtable while the methods generated
+        by dataclasses are interpreted and may not be.
+
+        (If we just called dataclass without doing this, it would think that all
+        of the descriptors for our attributes are default values and generate an
+        incorrect constructor. We need to do the switch so that dataclass gets the
+        appropriate defaults.)
+        """
+        self.finish_non_ext_dict(non_ext, cdef.line)
+        dec = self.accept(next(d for d in cdef.decorators if is_dataclass_decorator(d)))
+        self.primitive_op(
+            dataclass_sleight_of_hand, [dec, type_obj, non_ext.dict, non_ext.anns], cdef.line)
+
     def visit_class_def(self, cdef: ClassDef) -> None:
         ir = self.mapper.type_to_ir[cdef.info]
         # Currently, we only create non-extension classes for classes that are
@@ -1406,8 +1483,9 @@ def visit_class_def(self, cdef: ClassDef) -> None:
         # apply here, and are handled in a different way.
         if ir.is_ext_class:
             # If the class is not decorated, generate an extension class for it.
-            self.allocate_class(cdef)
+            type_obj = self.allocate_class(cdef)  # type: Optional[Value]
             non_ext = None  # type: Optional[NonExtClassInfo]
+            dataclass_non_ext = self.dataclass_non_ext_info(cdef)
         else:
             non_ext_bases = self.populate_non_ext_bases(cdef)
             non_ext_dict = self.setup_non_ext_dict(cdef, non_ext_bases)
@@ -1416,8 +1494,10 @@ def visit_class_def(self, cdef: ClassDef) -> None:
             # TODO: Maybe generate more precise types for annotations
             non_ext_anns = self.primitive_op(new_dict_op, [], cdef.line)
             non_ext = NonExtClassInfo(non_ext_dict, non_ext_bases, non_ext_anns)
+            dataclass_non_ext = None
+            type_obj = None
 
-            attrs_to_cache = []  # type: List[Lvalue]
+        attrs_to_cache = []  # type: List[Lvalue]
 
         for stmt in cdef.defs.body:
             if isinstance(stmt, OverloadedFuncDef) and stmt.is_property:
@@ -1430,16 +1510,16 @@ def visit_class_def(self, cdef: ClassDef) -> None:
                     with self.catch_errors(stmt.line):
                         self.visit_method(cdef, non_ext, get_func_def(item))
             elif isinstance(stmt, (FuncDef, Decorator, OverloadedFuncDef)):
-                if cdef.info.names[stmt.name()].plugin_generated and not ir.is_ext_class:
-                    # Ignore plugin generated methods when creating non-extension classes
+                # Ignore plugin generated methods (since they have no
+                # bodies to compile and will need to have the bodies
+                # provided by some other mechanism.)
+                if cdef.info.names[stmt.name()].plugin_generated:
                     continue
                 with self.catch_errors(stmt.line):
                     self.visit_method(cdef, non_ext, get_func_def(stmt))
             elif isinstance(stmt, PassStmt):
                 continue
             elif isinstance(stmt, AssignmentStmt):
-                # Variable declaration with no body
-
                 if len(stmt.lvalues) != 1:
                     self.error("Multiple assignment in class bodies not supported", stmt.line)
                     continue
@@ -1448,10 +1528,13 @@ def visit_class_def(self, cdef: ClassDef) -> None:
                     self.error("Only assignment to variables is supported in class bodies",
                                stmt.line)
                     continue
-                if non_ext:
-                    self.add_non_ext_class_attr(
-                        non_ext, lvalue, stmt, cdef, attrs_to_cache)
-                    continue
+                # We want to collect class variables in a dictionary for both real
+                # non-extension classes and fake dataclass ones.
+                var_non_ext = non_ext or dataclass_non_ext
+                if var_non_ext:
+                    self.add_non_ext_class_attr(var_non_ext, lvalue, stmt, cdef, attrs_to_cache)
+                    if non_ext:
+                        continue
                 # Variable declaration with no body
                 if isinstance(stmt.rvalue, TempNode):
                     continue
@@ -1473,6 +1556,9 @@ def visit_class_def(self, cdef: ClassDef) -> None:
         if not non_ext:  # That is, an extension class
             self.generate_attr_defaults(cdef)
             self.create_ne_from_eq(cdef)
+            if dataclass_non_ext:
+                assert type_obj
+                self.dataclass_finalize(cdef, dataclass_non_ext, type_obj)
         else:
             # Dynamically create the class via the type constructor
             non_ext_class = self.load_non_ext_class(ir, non_ext, cdef.line)
@@ -1500,7 +1586,7 @@ def create_mypyc_attrs_tuple(self, ir: ClassIR, line: int) -> Value:
                                  [self.load_static_unicode(attr) for attr in attrs],
                                  line)
 
-    def allocate_class(self, cdef: ClassDef) -> None:
+    def allocate_class(self, cdef: ClassDef) -> Value:
         # OK AND NOW THE FUN PART
         base_exprs = cdef.base_type_exprs + cdef.removed_base_type_exprs
         if base_exprs:
@@ -1535,6 +1621,8 @@ def allocate_class(self, cdef: ClassDef) -> None:
                           [self.load_globals_dict(), self.load_static_unicode(cdef.name),
                            tp], cdef.line)
 
+        return tp
+
     def gen_import(self, id: str, line: int) -> None:
         self.imports[id] = None
 
@@ -2624,9 +2712,15 @@ def translate_eq_cmp(self,
             return None
 
         class_ir = ltype.class_ir
-        # Check whether any subclasses of the operand redefines __eq__.
-        cmp_varies_at_runtime = (not class_ir.is_method_final('__eq__')
-            or not class_ir.is_method_final('__ne__'))
+        # Check whether any subclasses of the operand redefines __eq__
+        # or it might be redefined in a Python parent class or by
+        # dataclasses
+        cmp_varies_at_runtime = (
+            not class_ir.is_method_final('__eq__')
+            or not class_ir.is_method_final('__ne__')
+            or class_ir.inherits_python
+            or class_ir.is_augmented
+        )
 
         if cmp_varies_at_runtime:
             # We might need to call left.__eq__(right) or right.__eq__(left)
diff --git a/mypyc/ops.py b/mypyc/ops.py
--- a/mypyc/ops.py
+++ b/mypyc/ops.py
@@ -1745,6 +1745,9 @@ def __init__(self, name: str, module_name: str, is_trait: bool = False,
         self.is_generated = is_generated
         self.is_abstract = is_abstract
         self.is_ext_class = is_ext_class
+        # An augmented class has additional methods separate from what mypyc generates.
+        # Right now the only one is dataclasses.
+        self.is_augmented = False
         self.inherits_python = False
         self.has_dict = False
         # If this a subclass of some built-in python class, the name
@@ -1895,6 +1898,7 @@ def serialize(self) -> JsonDict:
             'is_ext_class': self.is_ext_class,
             'is_abstract': self.is_abstract,
             'is_generated': self.is_generated,
+            'is_augmented': self.is_augmented,
             'inherits_python': self.inherits_python,
             'has_dict': self.has_dict,
             'builtin_base': self.builtin_base,
@@ -1940,6 +1944,7 @@ def deserialize(cls, data: JsonDict, ctx: DeserMaps) -> 'ClassIR':
         ir.is_generated = data['is_generated']
         ir.is_abstract = data['is_abstract']
         ir.is_ext_class = data['is_ext_class']
+        ir.is_augmented = data['is_augmented']
         ir.inherits_python = data['inherits_python']
         ir.has_dict = data['has_dict']
         ir.builtin_base = data['builtin_base']
diff --git a/mypyc/ops_misc.py b/mypyc/ops_misc.py
--- a/mypyc/ops_misc.py
+++ b/mypyc/ops_misc.py
@@ -423,3 +423,12 @@
     format_str='{dest} = pytype_from_template({comma_args})',
     emit=simple_emit(
         '{dest} = CPyType_FromTemplate((PyTypeObject *){args[0]}, {args[1]}, {args[2]});'))
+
+# Create a dataclass from an extension class. See
+# CPyDataclass_SleightOfHand for more docs.
+dataclass_sleight_of_hand = custom_op(
+    arg_types=[object_rprimitive, object_rprimitive, dict_rprimitive, dict_rprimitive],
+    result_type=bool_rprimitive,
+    error_kind=ERR_FALSE,
+    format_str='{dest} = dataclass_sleight_of_hand({comma_args})',
+    emit=call_emit('CPyDataclass_SleightOfHand'))


Test patch:
diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test
--- a/mypyc/test-data/run-classes.test
+++ b/mypyc/test-data/run-classes.test
@@ -126,7 +126,7 @@ assert TestEnum.a.value == 1
 assert TestEnum.b.name == 'b'
 assert TestEnum.b.value == 2
 
-[case testDataClass]
+[case testRunDataclass]
 from dataclasses import dataclass, field
 from typing import Set, List, Callable, Any
 
@@ -144,10 +144,14 @@ def testBool(p: Person1) -> bool:
     else:
         return False
 
+@dataclass
+class Person1b(Person1):
+    id: str = '000'
+
 @dataclass
 class Person2:
     age : int
-    name : str = 'robot'
+    name : str = field(default='robot')
 
 @dataclass(order = True)
 class Person3:
@@ -184,20 +188,22 @@ class Person4:
     def name(self) -> str:
         return self._name
 
-[file driver.py]
-import sys
-
-# Dataclasses introduced in 3.7
-version = sys.version_info[:2]
-if version[0] < 3 or version[1] < 7:
-    exit()
-
-from native import Person1, Person2, Person3, Person4, testBool
+[file other.py]
+from native import Person1, Person1b, Person2, Person3, Person4, testBool
 i1 = Person1(age = 5, name = 'robot')
 assert i1.age == 5
 assert i1.name == 'robot'
 assert testBool(i1) == True
 assert testBool(Person1(age = 5, name = 'robo')) == False
+i1b = Person1b(age = 5, name = 'robot')
+assert i1b.age == 5
+assert i1b.name == 'robot'
+assert testBool(i1b) == True
+assert testBool(Person1b(age = 5, name = 'robo')) == False
+i1c = Person1b(age = 20, name = 'robot', id = 'test')
+assert i1c.age == 20
+assert i1c.id == 'test'
+
 i2 = Person2(age = 5)
 assert i2.age == 5
 assert i2.name == 'robot'
@@ -229,6 +235,39 @@ assert i8 == i9
 i8.age = 2
 assert i8 > i9
 
+
+[file driver.py]
+import sys
+
+# Dataclasses introduced in 3.7
+version = sys.version_info[:2]
+if version[0] < 3 or version[1] < 7:
+    exit()
+
+# Run the tests in both interpreted and compiled mode
+import other
+import other_interpreted
+
+# Test for an exceptional cases
+from testutil import assertRaises
+from native import Person1, Person1b, Person3
+from types import BuiltinMethodType
+
+with assertRaises(TypeError, "missing 1 required positional argument"):
+    Person1(0)
+
+with assertRaises(TypeError, "missing 2 required positional arguments"):
+    Person1b()
+
+with assertRaises(TypeError, "int object expected; got str"):
+    Person1('nope', 'test')
+
+p = Person1(0, 'test')
+with assertRaises(TypeError, "int object expected; got str"):
+    p.age = 'nope'
+
+assert isinstance(Person3().get_age, BuiltinMethodType)
+
 [case testGetAttribute]
 class C:
     x: int