Repository: cython/cython

Issue:
Allow genexprs in @cfunc functions.
This fails to compile:

```
@cython.cfunc
def f():
    ','.join(x for x in '123')
```

The problem is that the cfunc directive forces an AdjustDefByDirectives visit. This is ok for the f node but not for the genexpr DefNode (i.e. node.body.stats[0].expr.args[0].def_node, where node is the f node) which inherits the cfunc directive from f and is also (but wrongly) transformed to a CFuncDefNode.

I'm not sure whether there is a valid use case for inheriting cfunc or not. The current implementation just set the visitor directives attribute to the directives in a CompilerDirectivesNode, so the same directives are active for the entire recursive descent. An alternative is to pop the directive after applying it, in cases for which directive inheritance is undesirable. This PR does what I've just described, but only for cfunc. I consider it a dirty hack until more general criteria are discussed and adopted (for example, it seems inconsistent to me doing this for cfunc and not for ccall).



Discussion:
This doesn't look like the best fix, but
a) this is definitely a bug, thanks for your investigation
b) it's even more general as it also applies to other "directives" that shouldn't get inherited


Resolving patch:
diff --git a/Cython/Compiler/Nodes.py b/Cython/Compiler/Nodes.py
--- a/Cython/Compiler/Nodes.py
+++ b/Cython/Compiler/Nodes.py
@@ -2340,7 +2340,7 @@ def analyse_declarations(self, env):
         self.is_c_class_method = env.is_c_class_scope
         if self.directive_locals is None:
             self.directive_locals = {}
-        self.directive_locals.update(env.directives['locals'])
+        self.directive_locals.update(env.directives.get('locals', {}))
         if self.directive_returns is not None:
             base_type = self.directive_returns.analyse_as_type(env)
             if base_type is None:
@@ -2907,7 +2907,7 @@ def analyse_declarations(self, env):
         self.py_wrapper.analyse_declarations(env)
 
     def analyse_argument_types(self, env):
-        self.directive_locals = env.directives['locals']
+        self.directive_locals = env.directives.get('locals', {})
         allow_none_for_extension_args = env.directives['allow_none_for_extension_args']
 
         f2s = env.fused_to_specific
@@ -4959,7 +4959,7 @@ def generate_type_ready_code(entry, code, heap_type_bases=False):
                     code.putln("if (__Pyx_MergeVtables(&%s) < 0) %s" % (
                         typeobj_cname,
                         code.error_goto(entry.pos)))
-            if not type.scope.is_internal and not type.scope.directives['internal']:
+            if not type.scope.is_internal and not type.scope.directives.get('internal'):
                 # scope.is_internal is set for types defined by
                 # Cython (such as closures), the 'internal'
                 # directive is set by users
diff --git a/Cython/Compiler/Options.py b/Cython/Compiler/Options.py
--- a/Cython/Compiler/Options.py
+++ b/Cython/Compiler/Options.py
@@ -170,8 +170,6 @@ def get_directive_defaults():
     'nonecheck' : False,
     'initializedcheck' : True,
     'embedsignature' : False,
-    'locals' : {},
-    'exceptval' : None,  # (except value=None, check=True)
     'auto_cpdef': False,
     'auto_pickle': None,
     'cdivision': False,  # was True before 0.12
@@ -183,12 +181,8 @@ def get_directive_defaults():
     'wraparound' : True,
     'ccomplex' : False,  # use C99/C++ for complex types and arith
     'callspec' : "",
-    'final' : False,
     'nogil' : False,
-    'internal' : False,
     'profile': False,
-    'no_gc_clear': False,
-    'no_gc': False,
     'linetrace': False,
     'emit_code_comments': True,  # copy original source code into C code comments
     'annotation_typing': True,  # read type declarations from Python function annotations
@@ -241,7 +235,6 @@ def get_directive_defaults():
 
 # experimental, subject to change
     'binding': None,
-    'freelist': 0,
 
     'formal_grammar': False,
 }
@@ -301,7 +294,9 @@ def normalise_encoding_name(option_name, encoding):
 directive_types = {
     'language_level': int,  # values can be None/2/3, where None == 2+warning
     'auto_pickle': bool,
+    'locals': dict,
     'final' : bool,  # final cdef classes and methods
+    'nogil' : bool,
     'internal' : bool,  # cdef class visibility in the module dict
     'infer_types' : bool,  # values can be True/None/False
     'binding' : bool,
@@ -310,7 +305,10 @@ def normalise_encoding_name(option_name, encoding):
     'inline' : None,
     'staticmethod' : None,
     'cclass' : None,
+    'no_gc_clear' : bool,
+    'no_gc' : bool,
     'returns' : type,
+    'exceptval': type,  # actually (type, check=True/False), but has its own parser
     'set_initial_path': str,
     'freelist': int,
     'c_string_type': one_of('bytes', 'bytearray', 'str', 'unicode'),
@@ -325,8 +323,10 @@ def normalise_encoding_name(option_name, encoding):
     # 'module', 'function', 'class', 'with statement'
     'auto_pickle': ('module', 'cclass'),
     'final' : ('cclass', 'function'),
-    'nogil' : ('function',),
+    'nogil' : ('function', 'with statement'),
     'inline' : ('function',),
+    'cfunc' : ('function', 'with statement'),
+    'ccall' : ('function', 'with statement'),
     'returns' : ('function',),
     'exceptval' : ('function',),
     'locals' : ('function',),
@@ -334,6 +334,7 @@ def normalise_encoding_name(option_name, encoding):
     'no_gc_clear' : ('cclass',),
     'no_gc' : ('cclass',),
     'internal' : ('cclass',),
+    'cclass' : ('class', 'cclass', 'with statement'),
     'autotestdict' : ('module',),
     'autotestdict.all' : ('module',),
     'autotestdict.cdef' : ('module',),
diff --git a/Cython/Compiler/ParseTreeTransforms.py b/Cython/Compiler/ParseTreeTransforms.py
--- a/Cython/Compiler/ParseTreeTransforms.py
+++ b/Cython/Compiler/ParseTreeTransforms.py
@@ -958,30 +958,33 @@ def try_to_parse_directive(self, optname, args, kwds, pos):
         else:
             assert False
 
-    def visit_with_directives(self, body, directives):
-        olddirectives = self.directives
-        newdirectives = copy.copy(olddirectives)
-        newdirectives.update(directives)
-        self.directives = newdirectives
-        assert isinstance(body, Nodes.StatListNode), body
-        retbody = self.visit_Node(body)
-        directive = Nodes.CompilerDirectivesNode(pos=retbody.pos, body=retbody,
-                                                 directives=newdirectives)
-        self.directives = olddirectives
-        return directive
+    def visit_with_directives(self, node, directives):
+        if not directives:
+            return self.visit_Node(node)
+
+        old_directives = self.directives
+        new_directives = dict(old_directives)
+        new_directives.update(directives)
+
+        if new_directives == old_directives:
+            return self.visit_Node(node)
+
+        self.directives = new_directives
+        retbody = self.visit_Node(node)
+        self.directives = old_directives
+
+        if not isinstance(retbody, Nodes.StatListNode):
+            retbody = Nodes.StatListNode(node.pos, stats=[retbody])
+        return Nodes.CompilerDirectivesNode(
+            pos=retbody.pos, body=retbody, directives=new_directives)
 
     # Handle decorators
     def visit_FuncDefNode(self, node):
         directives = self._extract_directives(node, 'function')
-        if not directives:
-            return self.visit_Node(node)
-        body = Nodes.StatListNode(node.pos, stats=[node])
-        return self.visit_with_directives(body, directives)
+        return self.visit_with_directives(node, directives)
 
     def visit_CVarDefNode(self, node):
         directives = self._extract_directives(node, 'function')
-        if not directives:
-            return self.visit_Node(node)
         for name, value in directives.items():
             if name == 'locals':
                 node.directive_locals = value
@@ -990,29 +993,19 @@ def visit_CVarDefNode(self, node):
                     node.pos,
                     "Cdef functions can only take cython.locals(), "
                     "staticmethod, or final decorators, got %s." % name))
-        body = Nodes.StatListNode(node.pos, stats=[node])
-        return self.visit_with_directives(body, directives)
+        return self.visit_with_directives(node, directives)
 
     def visit_CClassDefNode(self, node):
         directives = self._extract_directives(node, 'cclass')
-        if not directives:
-            return self.visit_Node(node)
-        body = Nodes.StatListNode(node.pos, stats=[node])
-        return self.visit_with_directives(body, directives)
+        return self.visit_with_directives(node, directives)
 
     def visit_CppClassNode(self, node):
         directives = self._extract_directives(node, 'cppclass')
-        if not directives:
-            return self.visit_Node(node)
-        body = Nodes.StatListNode(node.pos, stats=[node])
-        return self.visit_with_directives(body, directives)
+        return self.visit_with_directives(node, directives)
 
     def visit_PyClassDefNode(self, node):
         directives = self._extract_directives(node, 'class')
-        if not directives:
-            return self.visit_Node(node)
-        body = Nodes.StatListNode(node.pos, stats=[node])
-        return self.visit_with_directives(body, directives)
+        return self.visit_with_directives(node, directives)
 
     def _extract_directives(self, node, scope_name):
         if not node.decorators:
@@ -1059,7 +1052,7 @@ def _extract_directives(self, node, scope_name):
                 optdict[name] = value
         return optdict
 
-    # Handle with statements
+    # Handle with-statements
     def visit_WithStatNode(self, node):
         directive_dict = {}
         for directive in self.try_to_parse_directives(node.manager) or []:
@@ -2385,6 +2378,10 @@ def visit_DefNode(self, node):
         self.visitchildren(node)
         return node
 
+    def visit_LambdaNode(self, node):
+        # No directives should modify lambdas or generator expressions (and also nothing in them).
+        return node
+
     def visit_PyClassDefNode(self, node):
         if 'cclass' in self.directives:
             node = node.as_cclass()


Test patch:
diff --git a/tests/run/exttype_freelist.pyx b/tests/run/exttype_freelist.pyx
--- a/tests/run/exttype_freelist.pyx
+++ b/tests/run/exttype_freelist.pyx
@@ -451,3 +451,19 @@ cdef class ExtTypeWithRefCycle:
 
     def __init__(self, obj=None):
         self.attribute = obj
+
+
+@cython.freelist(3)
+@cython.cclass
+class DecoratedPyClass(object):
+    """
+    >>> obj1 = DecoratedPyClass()
+    >>> obj2 = DecoratedPyClass()
+    >>> obj3 = DecoratedPyClass()
+    >>> obj4 = DecoratedPyClass()
+
+    >>> obj1 = DecoratedPyClass()
+    >>> obj2 = DecoratedPyClass()
+    >>> obj3 = DecoratedPyClass()
+    >>> obj4 = DecoratedPyClass()
+    """
diff --git a/tests/run/purecdef.py b/tests/run/purecdef.py
--- a/tests/run/purecdef.py
+++ b/tests/run/purecdef.py
@@ -28,6 +28,17 @@ def fwith1(a):
     def fwith2(a):
         return a*4
 
+    @cython.test_assert_path_exists(
+        '//CFuncDefNode',
+        '//LambdaNode',
+        '//GeneratorDefNode',
+        '//GeneratorBodyDefNode',
+    )
+    def f_with_genexpr(a):
+        f = lambda x: x+1
+        return (f(x) for x in a)
+
+
 with cclass:
     @cython.test_assert_path_exists('//CClassDefNode')
     class Egg(object):
@@ -123,3 +134,14 @@ def test_typed_return():
     """
     x = cython.declare(int, 5)
     assert typed_return(cython.address(x))[0] is x
+
+
+def test_genexpr_in_cdef(l):
+    """
+    >>> gen = test_genexpr_in_cdef([1, 2, 3])
+    >>> list(gen)
+    [2, 3, 4]
+    >>> list(gen)
+    []
+    """
+    return f_with_genexpr(l)