Repository: cython/cython

Issue:
[BUG] classes with `__weakref__` don't compile using the limited api
### Describe the bug

Hello everyone,

I was trying to compile a class with a `cdef object __weakref__` attribute in the limited api and it crashes with:

```
clang -Wsign-compare -Wunreachable-code -fno-common -dynamic -DNDEBUG -g -fwrapv -O3 -Wall -isysroot /Library/Developer/CommandLineTools/SDKs/MacOSX14.sdk -DCYTHON_LIMITED_API=1 -DPy_LIMITED_API=0x030B0000 -I/private/var/folders/l4/gpyzwv455hzbmbv78x05555h0000gn/T/build-env-x8r7d7re/include -I/opt/homebrew/opt/python@3.11/Frameworks/Python.framework/Versions/3.11/include/python3.11 -c limited_weakref.cpp -o build/temp.macosx-14-arm64-cpython-311/limited_weakref.o
limited_weakref.cpp:3707:38: error: member access into incomplete type 'PyTypeObject' (aka '_typeobject')
  if (__pyx_ptype_15limited_weakref_A->tp_weaklistoffset == 0) __pyx_ptype_15limited_weakref_A->tp_weaklistoffset = offsetof(struct __pyx_obj_15limited_weakref_A, __weakref__);
                                     ^
/opt/homebrew/opt/python@3.11/Frameworks/Python.framework/Versions/3.11/include/python3.11/pytypedefs.h:20:16: note: forward declaration of '_typeobject'
typedef struct _typeobject PyTypeObject;
               ^
limited_weakref.cpp:3707:95: error: member access into incomplete type 'PyTypeObject' (aka '_typeobject')
  if (__pyx_ptype_15limited_weakref_A->tp_weaklistoffset == 0) __pyx_ptype_15limited_weakref_A->tp_weaklistoffset = offsetof(struct __pyx_obj_15limited_weakref_A, __weakref__);
                                                                                              ^
/opt/homebrew/opt/python@3.11/Frameworks/Python.framework/Versions/3.11/include/python3.11/pytypedefs.h:20:16: note: forward declaration of '_typeobject'
typedef struct _typeobject PyTypeObject;
               ^
2 errors generated.
error: command '/usr/bin/clang' failed with exit code 1
```

-Andreas

### Code to reproduce the behaviour:

```cython
# limited_weakref.pyx

cdef class A:
    cdef object __weakref__
```

### Expected behaviour

It should compile.

### OS

_No response_

### Python version

_No response_

### Cython version

_No response_

### Additional context

The relevant compiler code seems to be here: https://github.com/cython/cython/blob/bc32b7ec3f15f532c0ae59b930544c23172e8c50/Cython/Compiler/Nodes.py#L5817-L5831


Discussion:
I think the solution here would be to add `__weakref__` to `Py_tp_members` (or possibly to use `[Py_TPFLAGS_MANAGED_WEAKREF](https://docs.python.org/3/c-api/typeobj.html#c.Py_TPFLAGS_MANAGED_WEAKREF)` if the version of Python is high enough).

> If it is not possible to switch to a MANAGED flag (for example, for vectorcall or to support Python older than 3.12), specify the offset in [Py_tp_members](https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_members).

(https://docs.python.org/3/c-api/type.html#c.PyType_Slot.slot)

Unfortunately that doesn't do the "already set" check that our current code does so it isn't identical, but that's an edge case that I don't think we can cover with the limited API.

-----------------------------------------------

It's probably worth using the MANAGED flags (for both `__dict__` and `__weakref__`) in the regular API too on suitable Python versions and in the common case where we aren't inheriting from an extern class or builtin. In this case we know that we don't need the "already set" check. And I think Python is hoping that this'll be the optimized structure in future.

--------------------------------------------

It's all slightly fiddly though because it's generating code in multiple places.

Resolving patch:
diff --git a/Cython/Compiler/ModuleNode.py b/Cython/Compiler/ModuleNode.py
--- a/Cython/Compiler/ModuleNode.py
+++ b/Cython/Compiler/ModuleNode.py
@@ -26,7 +26,7 @@
 from . import PyrexTypes
 from . import Pythran
 
-from .Errors import error, warning, CompileError
+from .Errors import error, warning, CompileError, format_position
 from .PyrexTypes import py_object_type
 from ..Utils import open_new_file, replace_suffix, decode_filename, build_hex_version, is_cython_generated_file
 from .Code import UtilityCode, IncludeCode, TempitaUtilityCode
@@ -2675,16 +2675,55 @@ def generate_typeobj_spec(self, entry, code):
             buffer_slot.generate_substructure(scope, code)
             code.putln("#endif")
 
+        if ext_type.typedef_flag:
+            objstruct = ext_type.objstruct_cname
+        else:
+            objstruct = "struct %s" % ext_type.objstruct_cname
+
+        weakref_entry = scope.lookup_here("__weakref__") if not scope.is_closure_class_scope else None
+        if weakref_entry and weakref_entry.is_inherited:
+            weakref_entry = None  # only generate it for the defining class
+        generate_members = bool(weakref_entry)
+        if generate_members:
+            code.globalstate.use_utility_code(
+                UtilityCode.load_cached("IncludeStructmemberH", "ModuleSetupCode.c"))
+            code.putln("static PyMemberDef %s_members[] = {" % ext_type.typeobj_cname)
+            code.putln("#if !CYTHON_USE_TYPE_SLOTS")
+            if weakref_entry:
+                # Note that unlike the assignment of tp_weaklistoffset in the type-ready code
+                # used in the non-limited API case, this doesn't preserve the weaklistoffset
+                # from base classes.
+                # Practically that doesn't matter, but it isn't exactly the identical.
+                code.putln('{"__weaklistoffset__", T_PYSSIZET, offsetof(%s, %s), READONLY},'
+                           % (objstruct, weakref_entry.cname))
+            code.putln("#endif")
+            code.putln("{0, 0, 0, 0}")
+            code.putln("};")
+
+            if weakref_entry:
+                position = format_position(weakref_entry.pos)
+                weakref_warn_mesage = (
+                    f"{position}: __weakref__ is unsupported in the Limited API when "
+                    "running on Python <3.9.")
+                # Note: Limited API rather than USE_TYPE_SPECS - we work round the issue
+                # with USE_TYPE_SPECS outside the limited API
+                code.putln("#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x03090000")
+                code.putln("#if defined(__GNUC__) || defined(__clang__)")
+                code.putln(f'#warning "{weakref_warn_mesage}"')
+                code.putln("#elif defined(_MSC_VER)")
+                code.putln(f'#pragma message("{weakref_warn_mesage}")')
+                code.putln("#endif")
+                code.putln("#endif")
+
+
         code.putln("static PyType_Slot %s_slots[] = {" % ext_type.typeobj_cname)
         for slot in TypeSlots.get_slot_table(code.globalstate.directives):
             slot.generate_spec(scope, code)
+        if generate_members:
+            code.putln("{Py_tp_members, (void*)%s_members}," % ext_type.typeobj_cname)
         code.putln("{0, 0},")
         code.putln("};")
 
-        if ext_type.typedef_flag:
-            objstruct = ext_type.objstruct_cname
-        else:
-            objstruct = "struct %s" % ext_type.objstruct_cname
         classname = scope.class_name.as_c_string_literal()
         code.putln("static PyType_Spec %s_spec = {" % ext_type.typeobj_cname)
         code.putln('"%s.%s",' % (self.full_module_name, classname.replace('"', '')))
diff --git a/Cython/Compiler/Nodes.py b/Cython/Compiler/Nodes.py
--- a/Cython/Compiler/Nodes.py
+++ b/Cython/Compiler/Nodes.py
@@ -5842,11 +5842,13 @@ def generate_type_ready_code(entry, code, bases_tuple_cname=None, check_heap_typ
                         objstruct = type.objstruct_cname
                     else:
                         objstruct = "struct %s" % type.objstruct_cname
+                    code.putln("#if CYTHON_USE_TYPE_SLOTS")
                     code.putln("if (%s == 0) %s = offsetof(%s, %s);" % (
                         tp_weaklistoffset,
                         tp_weaklistoffset,
                         objstruct,
                         weakref_entry.cname))
+                    code.putln("#endif")
                 else:
                     error(weakref_entry.pos, "__weakref__ slot must be of type 'object'")
 


Test patch:
diff --git a/tests/run/isolated_limited_api_tests.srctree b/tests/run/isolated_limited_api_tests.srctree
--- a/tests/run/isolated_limited_api_tests.srctree
+++ b/tests/run/isolated_limited_api_tests.srctree
@@ -24,6 +24,8 @@ setup(
 import decimal
 
 import limited
+import weakref
+import sys
 
 limited.fib(11)
 
@@ -41,6 +43,12 @@ limited.catches()
 limited.C()
 limited.D()
 limited.E()
+if sys.version_info >= (3, 9):
+    # Weak-referenceable classes won't work in Python <3.9 because
+    # of an interpreter bug (https://github.com/python/cpython/issues/82321).
+    # This depends on the interpreter version, not the limited API version.
+    weakref.ref(limited.D())
+    weakref.ref(limited.E())
 
 assert limited.C.cm() == limited.C().cm() == limited.C
 assert limited.D.cm() == limited.D().cm() == limited.D
@@ -113,6 +121,8 @@ class C:
         return cls
 
 cdef class D:
+    cdef object __weakref__
+
     @classmethod
     def cm(cls):
         return cls