Repository: cython/cython

Issue:
variable 'referenced before assignment' warning, error in all() with .pxd but not .pyx
The following is a standalone example of a problem I've run into trying to accelerate some code:

given allExt.py (takes a set of words and checks if 3rd letter in each is a vowel):
```
class a3v(object):
    def __init__(self, inset):
        self.myStrSet = inset   # set input word list as an attribute on  class instance

    def all3Vowels(self):     # test if 3rd letter of each word is a vowel
        vndx = 2                    # local variable for 3rd letter index
        inStrSet = self.myStrSet    # local variable for class instance word list
        if all(
            ltr in ("a", "e", "i", "o", "u")
            for ltr in (w[vndx] for w in inStrSet)
        ):
            return True
        return False
```
and tst.py:
```
#!/usr/local/bin/python3
import allExt

a3vL = allExt.a3v(["the", "bright", "flower"])

if a3vL.all3Vowels():
   print('yes')
else:
   print('no')
```
The code runs without issue, and fits with examples I have found in the cython docs.  However, when augmented with allExt.pxd: 
```
import cython

cdef class a3v:
     cdef public list myStrSet

     @cython.locals(vndx=int, inStrSet=list, ltr=str)
     cpdef bint all3Vowels(self)
```
Upon compiling as shown I get:
```
rob@nova-370:01 14:45:55 atmNdx $ cythonize -3 -f -i allExt.py allExt.pxd
[1/1] Cythonizing /Users/rob/sync/rgithub/cython/atmNdx/allExt.py
warning: allExt.py:12:26: local variable 'vndx' referenced before assignment

Error compiling Cython file:
------------------------------------------------------------
...
    def all3Vowels(self):
        vndx = 2
        inStrSet = self.myStrSet
        if all(
            ltr in ("a", "e", "i", "o", "u")
            for ltr in (w[vndx] for w in inStrSet)
                                        ^
------------------------------------------------------------

allExt.py:12:41: local variable 'inStrSet' referenced before assignment
Traceback (most recent call last):
```
OTOH, the equivalent (?) pyx version allExt.pyx:
```
class a3v(object):
    def __init__(self, inset):
        self.myStrSet = inset

    def all3Vowels(self):
        cdef int vndx = 2
        cdef list inStrSet = self.myStrSet
        cdef str ltr
        if all(
            ltr in ("a", "e", "i", "o", "u")
            for ltr in (w[vndx] for w in inStrSet)
        ):
            return True
        return False
```
compiles and runs without issue.

Have I missed something in the docs or some other simple issue?
Thank you.



Discussion:
Thanks for the report. There seems to be something wrong with the nested generator expressions here and the way the type information is applied from the decorator (or maybe _when_ it is applied).

You didn't mention which Cython version you were using, so I guess it's a recent release. Could you try the latest master branch, and also PR #3323?

BTW, you don't need to declare `inStrSet` here because its type should get inferred from the assignment, since the attribute is typed.
Thanks for the follow up.
This was with Cython-0.29.16, but I have now reproduced with both 3.0a0
master and https://github.com/da-woods/cython.git (#3323).


On Fri, Apr 3, 2020 at 8:45 AM Stefan Behnel <notifications@github.com>
wrote:

> Thanks for the report. There seems to be something wrong with the nested
> generator expressions here and the way the type information is applied from
> the decorator (or maybe *when* it is applied).
>
> You didn't mention which Cython version you were using, so I guess it's a
> recent release. Could you try the latest master branch, and also PR #3323
> <https://github.com/cython/cython/pull/3323>?
>
> BTW, you don't need to declare inStrSet here because its type should get
> inferred from the assignment, since the attribute is typed.
>
> —
> You are receiving this because you authored the thread.
> Reply to this email directly, view it on GitHub
> <https://github.com/cython/cython/issues/3477#issuecomment-608283260>, or
> unsubscribe
> <https://github.com/notifications/unsubscribe-auth/AAZW3QDL4WTTE5FOZWIMHALRKWHZRANCNFSM4LY7IUIA>
> .
>


Resolving patch:
diff --git a/Cython/Compiler/ParseTreeTransforms.py b/Cython/Compiler/ParseTreeTransforms.py
--- a/Cython/Compiler/ParseTreeTransforms.py
+++ b/Cython/Compiler/ParseTreeTransforms.py
@@ -2496,8 +2496,6 @@ class AlignFunctionDefinitions(CythonTransform):
 
     def visit_ModuleNode(self, node):
         self.scope = node.scope
-        self.directives = node.directives
-        self.imported_names = set()  # hack, see visit_FromImportStatNode()
         self.visitchildren(node)
         return node
 
@@ -2535,15 +2533,45 @@ def visit_DefNode(self, node):
                     error(pxd_def.pos, "previous declaration here")
                 return None
             node = node.as_cfunction(pxd_def)
-        elif (self.scope.is_module_scope and self.directives['auto_cpdef']
-              and node.name not in self.imported_names
-              and node.is_cdef_func_compatible()):
-            # FIXME: cpdef-ing should be done in analyse_declarations()
-            node = node.as_cfunction(scope=self.scope)
         # Enable this when nested cdef functions are allowed.
         # self.visitchildren(node)
         return node
 
+    def visit_ExprNode(self, node):
+        # ignore lambdas and everything else that appears in expressions
+        return node
+
+
+class AutoCpdefFunctionDefinitions(CythonTransform):
+
+    def visit_ModuleNode(self, node):
+        self.directives = node.directives
+        self.imported_names = set()  # hack, see visit_FromImportStatNode()
+        self.scope = node.scope
+        self.visitchildren(node)
+        return node
+
+    def visit_DefNode(self, node):
+        if (self.scope.is_module_scope and self.directives['auto_cpdef']
+                and node.name not in self.imported_names
+                and node.is_cdef_func_compatible()):
+            # FIXME: cpdef-ing should be done in analyse_declarations()
+            node = node.as_cfunction(scope=self.scope)
+        return node
+
+    def visit_CClassDefNode(self, node, pxd_def=None):
+        if pxd_def is None:
+            pxd_def = self.scope.lookup(node.class_name)
+        if pxd_def:
+            if not pxd_def.defined_in_pxd:
+                return node
+            outer_scope = self.scope
+            self.scope = pxd_def.type.scope
+        self.visitchildren(node)
+        if pxd_def:
+            self.scope = outer_scope
+        return node
+
     def visit_FromImportStatNode(self, node):
         # hack to prevent conditional import fallback functions from
         # being cdpef-ed (global Python variables currently conflict
diff --git a/Cython/Compiler/Pipeline.py b/Cython/Compiler/Pipeline.py
--- a/Cython/Compiler/Pipeline.py
+++ b/Cython/Compiler/Pipeline.py
@@ -149,7 +149,7 @@ def create_pipeline(context, mode, exclude_classes=()):
     from .ParseTreeTransforms import ExpandInplaceOperators, ParallelRangeTransform
     from .ParseTreeTransforms import CalculateQualifiedNamesTransform
     from .TypeInference import MarkParallelAssignments, MarkOverflowingArithmetic
-    from .ParseTreeTransforms import AdjustDefByDirectives, AlignFunctionDefinitions
+    from .ParseTreeTransforms import AdjustDefByDirectives, AlignFunctionDefinitions, AutoCpdefFunctionDefinitions
     from .ParseTreeTransforms import RemoveUnreachableCode, GilCheck
     from .FlowControl import ControlFlowAnalysis
     from .AnalysedTreeTransforms import AutoTestDictTransform
@@ -186,10 +186,11 @@ def create_pipeline(context, mode, exclude_classes=()):
         TrackNumpyAttributes(),
         InterpretCompilerDirectives(context, context.compiler_directives),
         ParallelRangeTransform(context),
-        AdjustDefByDirectives(context),
         WithTransform(context),
-        MarkClosureVisitor(context),
+        AdjustDefByDirectives(context),
         _align_function_definitions,
+        MarkClosureVisitor(context),
+        AutoCpdefFunctionDefinitions(context),
         RemoveUnreachableCode(context),
         ConstantFolding(),
         FlattenInListTransform(),


Test patch:
diff --git a/tests/run/pure_pxd.srctree b/tests/run/pure_pxd.srctree
--- a/tests/run/pure_pxd.srctree
+++ b/tests/run/pure_pxd.srctree
@@ -55,6 +55,17 @@ def func(a, b, c):
     """
     return a + b + c
 
+def sum_generator_expression(a):
+    # GH-3477 - closure variables incorrectly captured in functions transformed to cdef
+    return sum(i for i in range(a))
+
+def run_sum_generator_expression(a):
+    """
+    >>> run_sum_generator_expression(5)
+    10
+    """
+    return sum_generator_expression(a)
+
 
 def test(module):
     import os.path
@@ -95,3 +106,6 @@ cdef class TypedMethod:
 
 
 cpdef int func(x, int y, z) except? -1  # argument names should not matter, types should
+
+
+cdef int sum_generator_expression(int a)