Repository: cython/cython

Issue:
Conditional `with gil` and threading
A nogil block with a loop that uses an unlikely `with gil` statement e.g. approximately,
```pyx
cpdef float test_gil(float[:] x):
    cdef int i = 0
    cdef int size = x1.shape[0]
    cdef float out = 0
    with nogil:
        while i < size:
            out_sum += x[i]
            if i > size +1:
                with gil:
                    raise ValueError
   return out 
```
appears work similarly to the while function not releasing gil, when using in multi-threaded code.

See more complete minimal example in https://github.com/scikit-learn/scikit-learn/pull/17038#issuecomment-619476846
  
 Naively I though that since the `with gil` is never executed, performance wise this would be equivalent to the same function without the `with gil` block (or roughly that threading performance would be directly impacted by the fraction of run time where the gil is released). However it does not seem to be the case.

Is this behavior expected? If so maybe the documentation section https://cython.readthedocs.io/en/latest/src/userguide/external_C_code.html#acquiring-and-releasing-the-gil could be updated to mention conditional acquiring of GIL.
   


Discussion:
Note that you do not need to write the `with gil` in this specific example since it's implicit in the `raise` statement (i.e. you can raise exceptions also from within nogil sections). That generates the same code as `with gil`, though, so will run into the same problem.

Looking at the C code that Cython generates from the example that you linked to, the difference is that the function that uses `with gil` has a Python temp variable, which needs to be cleaned up at the end in the exception case. For that, it needs to acquire the GIL. However, it wouldn't need the GIL in the success case, and still acquires it. That makes it uselessly hammer on the GIL in very fast functions like this.

The GIL acquisition is unconditionally inserted here, as a try-finally statement wrapping the whole function body:
https://github.com/cython/cython/blob/f09e61ab721ad51526ec7a6798fc01d8346f539d/Cython/Compiler/ParseTreeTransforms.py#L1861-L1872

and gets released here, after generating the function code:
https://github.com/cython/cython/blob/f09e61ab721ad51526ec7a6798fc01d8346f539d/Cython/Compiler/Nodes.py#L2145-L2148

I don't know how difficult it would be to detect the needless acquire and avoid it, but in any case, I consider it a bug that this happens unconditionally. in the success case, it should run through the function without touching the GIL at all.
Thanks for the detailed answer and investigation @scoder !
I think the best solution is to get rid of the try-finally node and do the cleanup in `FuncDefNode.generate_function_definitions()`, acquiring the GIL only when necessary for a given exit case (error or success).

Resolving patch:
diff --git a/Cython/Compiler/Code.py b/Cython/Compiler/Code.py
--- a/Cython/Compiler/Code.py
+++ b/Cython/Compiler/Code.py
@@ -2401,8 +2401,8 @@ def put_setup_refcount_context(self, name, acquire_gil=False):
                 UtilityCode.load_cached("ForceInitThreads", "ModuleSetupCode.c"))
         self.putln('__Pyx_RefNannySetupContext(%s, %d);' % (name, acquire_gil and 1 or 0))
 
-    def put_finish_refcount_context(self):
-        self.putln("__Pyx_RefNannyFinishContext();")
+    def put_finish_refcount_context(self, nogil=False):
+        self.putln("__Pyx_RefNannyFinishContextNogil()" if nogil else "__Pyx_RefNannyFinishContext();")
 
     def put_add_traceback(self, qualified_name, include_cline=True):
         """
diff --git a/Cython/Compiler/Nodes.py b/Cython/Compiler/Nodes.py
--- a/Cython/Compiler/Nodes.py
+++ b/Cython/Compiler/Nodes.py
@@ -1847,11 +1847,10 @@ def generate_function_definitions(self, env, code):
 
         use_refnanny = not lenv.nogil or lenv.has_with_gil_block
 
+        gilstate_decl = code.insertion_point()
         if acquire_gil or acquire_gil_for_var_decls_only:
             code.put_ensure_gil()
             code.funcstate.gil_owned = True
-        elif lenv.nogil and lenv.has_with_gil_block:
-            code.declare_gilstate()
 
         if profile or linetrace:
             if not self.is_generator:
@@ -1974,6 +1973,19 @@ def generate_function_definitions(self, env, code):
         code.putln("")
         code.putln("/* function exit code */")
 
+        gil_owned = {
+            'success': code.funcstate.gil_owned,
+            'error': code.funcstate.gil_owned,
+            'gil_state_declared': False,
+        }
+        def assure_gil(code_path):
+            if not gil_owned[code_path]:
+                if not gil_owned['gil_state_declared']:
+                    gilstate_decl.declare_gilstate()
+                    gil_owned['gil_state_declared'] = True
+                code.put_ensure_gil(declare_gilstate=False)
+                gil_owned[code_path] = True
+
         # ----- Default return value
         return_type = self.return_type
         if not self.body.is_terminator:
@@ -1982,6 +1994,7 @@ def generate_function_definitions(self, env, code):
                 #    lhs = "(PyObject *)%s" % Naming.retval_cname
                 #else:
                 lhs = Naming.retval_cname
+                assure_gil('success')
                 code.put_init_to_py_none(lhs, return_type)
             elif not return_type.is_memoryviewslice:
                 # memory view structs receive their default value on initialisation
@@ -2006,6 +2019,7 @@ def generate_function_definitions(self, env, code):
                 code.globalstate.use_utility_code(restore_exception_utility_code)
                 code.putln("{ PyObject *__pyx_type, *__pyx_value, *__pyx_tb;")
                 code.putln("__Pyx_PyThreadState_declare")
+                assure_gil('error')
                 code.putln("__Pyx_PyThreadState_assign")
                 code.putln("__Pyx_ErrFetch(&__pyx_type, &__pyx_value, &__pyx_tb);")
                 for entry in used_buffer_entries:
@@ -2026,20 +2040,14 @@ def generate_function_definitions(self, env, code):
                 # code.globalstate.use_utility_code(get_exception_tuple_utility_code)
                 # code.put_trace_exception()
 
-                if lenv.nogil and not lenv.has_with_gil_block:
-                    code.putln("{")
-                    code.put_ensure_gil()
-
+                assure_gil('error')
                 code.put_add_traceback(self.entry.qualified_name)
-
-                if lenv.nogil and not lenv.has_with_gil_block:
-                    code.put_release_ensured_gil()
-                    code.putln("}")
             else:
                 warning(self.entry.pos,
                         "Unraisable exception in function '%s'." %
                         self.entry.qualified_name, 0)
-                code.put_unraisable(self.entry.qualified_name, lenv.nogil)
+                assure_gil('error')
+                code.put_unraisable(self.entry.qualified_name)
             default_retval = return_type.default_value
             if err_val is None and default_retval:
                 err_val = default_retval
@@ -2050,19 +2058,33 @@ def generate_function_definitions(self, env, code):
                 code.putln("__Pyx_pretend_to_initialize(&%s);" % Naming.retval_cname)
 
             if is_getbuffer_slot:
+                assure_gil('error')
                 self.getbuffer_error_cleanup(code)
 
             # If we are using the non-error cleanup section we should
             # jump past it if we have an error. The if-test below determine
             # whether this section is used.
             if buffers_present or is_getbuffer_slot or return_type.is_memoryviewslice:
+                # In the buffer cases, we already called assure_gil('error') and own the GIL.
+                assert gil_owned['error'] or return_type.is_memoryviewslice
                 code.put_goto(code.return_from_error_cleanup_label)
+            else:
+                # align error and success GIL state
+                if gil_owned['success']:
+                    assure_gil('error')
+                elif gil_owned['error']:
+                    code.put_release_ensured_gil()
+                    gil_owned['error'] = False
 
         # ----- Non-error return cleanup
         code.put_label(code.return_label)
+        assert gil_owned['error'] == gil_owned['success'], "%s != %s" % (gil_owned['error'], gil_owned['success'])
+
         for entry in used_buffer_entries:
+            assure_gil('success')
             Buffer.put_release_buffer_code(code, entry)
         if is_getbuffer_slot:
+            assure_gil('success')
             self.getbuffer_normal_cleanup(code)
 
         if return_type.is_memoryviewslice:
@@ -2072,17 +2094,22 @@ def generate_function_definitions(self, env, code):
             cond = code.unlikely(return_type.error_condition(Naming.retval_cname))
             code.putln(
                 'if (%s) {' % cond)
-            if env.nogil:
+            if not gil_owned['success']:
                 code.put_ensure_gil()
             code.putln(
                 'PyErr_SetString(PyExc_TypeError, "Memoryview return value is not initialized");')
-            if env.nogil:
+            if not gil_owned['success']:
                 code.put_release_ensured_gil()
             code.putln(
                 '}')
 
         # ----- Return cleanup for both error and no-error return
-        code.put_label(code.return_from_error_cleanup_label)
+        if code.label_used(code.return_from_error_cleanup_label):
+            # If we came through the success path, then we took the same GIL decisions as for jumping here.
+            # If we came through the error path and did not jump, then we aligned both paths above.
+            # In the end, all paths are aligned from this point on.
+            assert gil_owned['error'] == gil_owned['success'], "%s != %s" % (gil_owned['error'], gil_owned['success'])
+            code.put_label(code.return_from_error_cleanup_label)
 
         for entry in lenv.var_entries:
             if not entry.used or entry.in_closure:
@@ -2110,6 +2137,7 @@ def generate_function_definitions(self, env, code):
             # FIXME use entry.xdecref_cleanup - del arg seems to be the problem
             code.put_var_xdecref(entry, have_gil=not lenv.nogil)
         if self.needs_closure:
+            assure_gil('success')
             code.put_decref(Naming.cur_scope_cname, lenv.scope_class.type)
 
         # ----- Return
@@ -2124,6 +2152,7 @@ def generate_function_definitions(self, env, code):
         if self.entry.is_special and self.entry.name == "__hash__":
             # Returning -1 for __hash__ is supposed to signal an error
             # We do as Python instances and coerce -1 into -2.
+            assure_gil('success')
             code.putln("if (unlikely(%s == -1) && !PyErr_Occurred()) %s = -2;" % (
                 Naming.retval_cname, Naming.retval_cname))
 
@@ -2133,16 +2162,16 @@ def generate_function_definitions(self, env, code):
                 # generators are traced when iterated, not at creation
                 if return_type.is_pyobject:
                     code.put_trace_return(
-                        Naming.retval_cname, nogil=not code.funcstate.gil_owned)
+                        Naming.retval_cname, nogil=not gil_owned['success'])
                 else:
                     code.put_trace_return(
-                        "Py_None", nogil=not code.funcstate.gil_owned)
+                        "Py_None", nogil=not gil_owned['success'])
 
         if not lenv.nogil:
             # GIL holding function
-            code.put_finish_refcount_context()
+            code.put_finish_refcount_context(nogil=not gil_owned['success'])
 
-        if acquire_gil or (lenv.nogil and lenv.has_with_gil_block):
+        if acquire_gil or (lenv.nogil and gil_owned['success']):
             # release the GIL (note that with-gil blocks acquire it on exit in their EnsureGILNode)
             code.put_release_ensured_gil()
             code.funcstate.gil_owned = False
diff --git a/Cython/Compiler/ParseTreeTransforms.py b/Cython/Compiler/ParseTreeTransforms.py
--- a/Cython/Compiler/ParseTreeTransforms.py
+++ b/Cython/Compiler/ParseTreeTransforms.py
@@ -1858,19 +1858,6 @@ def _create_fused_function(self, env, node):
 
         return node
 
-    def _handle_nogil_cleanup(self, lenv, node):
-        "Handle cleanup for 'with gil' blocks in nogil functions."
-        if lenv.nogil and lenv.has_with_gil_block:
-            # Acquire the GIL for cleanup in 'nogil' functions, by wrapping
-            # the entire function body in try/finally.
-            # The corresponding release will be taken care of by
-            # Nodes.FuncDefNode.generate_function_definitions()
-            node.body = Nodes.NogilTryFinallyStatNode(
-                node.body.pos,
-                body=node.body,
-                finally_clause=Nodes.EnsureGILNode(node.body.pos),
-                finally_except_clause=Nodes.EnsureGILNode(node.body.pos))
-
     def _handle_fused(self, node):
         if node.is_generator and node.has_fused_arguments:
             node.has_fused_arguments = False
@@ -1913,7 +1900,6 @@ def visit_FuncDefNode(self, node):
             node = self._create_fused_function(env, node)
         else:
             node.body.analyse_declarations(lenv)
-            self._handle_nogil_cleanup(lenv, node)
             self._super_visit_FuncDefNode(node)
 
         self.seen_vars_stack.pop()


Test patch: