Repository: cython/cython

Issue:
[BUG] segmentation fault when numpy is not available
**Describe the bug**
When using `cimport numpy` and `numpy` is not available at runtime, a segmentation fault is thrown.

**To Reproduce**
I used the following code
```cython
# distutils: language=c++
# cython: language_level=3, binding=True

cimport numpy as np

def test():
    pass
```
and uninstalled numpy before importing the module. For Cython 3.0a1 and Cython 0.29.24 this leads to an import error, while for any version newer than Cython 3.0a1 this causes a segmentation fault.

**Expected behavior**
No segmentation fault should be thrown.

**Environment (please complete the following information):**
 - OS: tested on Windows and Fedora
 - Python version: tested on 3.7 and 3.9
 - Cython version >3.0a1

**Additional context**
running the import in gdb leads to the following backtrace:
```
Program received signal SIGSEGV, Segmentation fault.
__Pyx_CLineForTraceback (c_line=3988, tstate=0x55555555afa0) at src/test.cpp:5923
5923	        __PYX_PY_DICT_LOOKUP_IF_MODIFIED(
(gdb) bt
#0  __Pyx_CLineForTraceback (c_line=3988, tstate=0x55555555afa0) at src/test.cpp:5923
#1  __Pyx_AddTraceback (filename=0x7ffff7fba89b "test.pyx", py_line=2, c_line=3988, 
    funcname=0x7ffff7fba886 "init test") at src/test.cpp:6105
#2  __pyx_pymod_exec_test (__pyx_pyinit_module=<optimized out>) at src/test.cpp:4057
#3  0x00007ffff7ddcdc3 in PyModule_ExecDef (module=<module at remote 0x7fffea250090>, def=<optimized out>)
    at /usr/src/debug/python3.9-3.9.7-1.fc34.x86_64/Objects/moduleobject.c:399
#4  0x00007ffff7ddcd34 in exec_builtin_or_dynamic (mod=<module at remote 0x7fffea250090>)
    at /usr/src/debug/python3.9-3.9.7-1.fc34.x86_64/Python/import.c:2248
#5  _imp_exec_builtin_impl (mod=<module at remote 0x7fffea250090>, module=<optimized out>)
    at /usr/src/debug/python3.9-3.9.7-1.fc34.x86_64/Python/import.c:2341
#6  _imp_exec_builtin (module=<optimized out>, mod=<module at remote 0x7fffea250090>)
    at /usr/src/debug/python3.9-3.9.7-1.fc34.x86_64/Python/clinic/import.c.h:388
#7  0x00007ffff7d6a36b in cfunction_vectorcall_O (
    func=<built-in method exec_dynamic of module object at remote 0x7fffea3746d0>, args=0x7fffea2de8f8, 
    nargsf=<optimized out>, kwnames=<optimized out>)
    at /usr/src/debug/python3.9-3.9.7-1.fc34.x86_64/Objects/methodobject.c:516
#8  0x00007ffff7d63fc7 in do_call_core (kwdict={}, callargs=(<module at remote 0x7fffea250090>,), 
    func=<built-in method exec_dynamic of module object at remote 0x7fffea3746d0>, tstate=<optimized out>)
    at /usr/src/debug/python3.9-3.9.7-1.fc34.x86_64/Python/ceval.c:5095
#9  _PyEval_EvalFrameDefault (tstate=<optimized out>, f=<optimized out>, throwflag=<optimized out>)
    at /usr/src/debug/python3.9-3.9.7-1.fc34.x86_64/Python/ceval.c:3580
#10 0x00007ffff7d5d00d in _PyEval_EvalFrame (throwflag=0, 
    f=Frame 0x7fffea2d7ac0, for file <frozen importlib._bootstrap>, line 228, in _call_with_frames_removed (f=<built-in method exec_dynamic of module object at remote 0x7fffea3746d0>, args=(<module at remote 0x7fffea250090>,), kwds={}), tstate=0x55555555afa0) at /usr/src/debug/python3.9-3.9.7-1.fc34.x86_64/Include/internal/pycore_ceval.h:40
#11 _PyEval_EvalCode (tstate=<optimized out>, _co=<optimized out>, globals=<optimized out>, locals=<optimized out>, 
    args=<optimized out>, argcount=<optimized out>, kwnames=0x0, kwargs=0x7fffea2d7e18, kwcount=0, kwstep=1, 
    defs=0x0, defcount=0, kwdefs=0x0, closure=0x0, name='_call_with_frames_removed', 
    qualname='_call_with_frames_removed') at /usr/src/debug/python3.9-3.9.7-1.fc34.x86_64/Python/ceval.c:4327
#12 0x00007ffff7d6acee in _PyFunction_Vectorcall (func=<optimized out>, stack=<optimized out>, 
    nargsf=<optimized out>, kwnames=<optimized out>)
    at /usr/src/debug/python3.9-3.9.7-1.fc34.x86_64/Objects/call.c:396
#13 0x00007ffff7d6305e in _PyObject_VectorcallTstate (kwnames=0x0, nargsf=<optimized out>, args=0x7fffea2d7e08, 
    callable=<function at remote 0x7fffea38b430>, tstate=0x55555555afa0)
    at /usr/src/debug/python3.9-3.9.7-1.fc34.x86_64/Include/cpython/abstract.h:118
#14 PyObject_Vectorcall (kwnames=0x0, nargsf=<optimized out>, args=0x7fffea2d7e08, 
    callable=<function at remote 0x7fffea38b430>)
    at /usr/src/debug/python3.9-3.9.7-1.fc34.x86_64/Include/cpython/abstract.h:127
#15 call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=0x55555555afa0)
    at /usr/src/debug/python3.9-3.9.7-1.fc34.x86_64/Python/ceval.c:5075
#16 _PyEval_EvalFrameDefault (tstate=<optimized out>, f=<optimized out>, throwflag=<optimized out>)
    at /usr/src/debug/python3.9-3.9.7-1.fc34.x86_64/Python/ceval.c:3487
#17 0x00007ffff7d6afe3 in _PyEval_EvalFrame (throwflag=0, 
    f=Frame 0x7fffea2d7c80, for file <frozen importlib._bootstrap_external>, line 1181, in exec_module (self=<ExtensionFileLoader(name='rapidfuzz.test', path='/home/max/RapidFuzz/.venv/lib64/python3.9/site-packages/rapidfuzz/test.cpython-39-x86_64-linux-gnu.so') at remote 0x7fffea2498b0>, module=<module at remote 0x7fffea250090>), 
    tstate=0x55555555afa0) at /usr/src/debug/python3.9-3.9.7-1.fc34.x86_64/Include/internal/pycore_ceval.h:40
#18 function_code_fastcall (tstate=0x55555555afa0, co=<optimized out>, args=<optimized out>, nargs=2, 
    globals=<optimized out>) at /usr/src/debug/python3.9-3.9.7-1.fc34.x86_64/Objects/call.c:330
#19 0x00007ffff7d5e5eb in _PyObject_VectorcallTstate (kwnames=0x0, nargsf=<optimized out>, args=0x7fffea310f50, 
    callable=<function at remote 0x7fffea34f790>, tstate=0x55555555afa0)
    at /usr/src/debug/python3.9-3.9.7-1.fc34.x86_64/Include/cpython/abstract.h:118
#20 PyObject_Vectorcall (kwnames=0x0, nargsf=<optimized out>, args=0x7fffea310f50, 
    callable=<function at remote 0x7fffea34f790>)
    at /usr/src/debug/python3.9-3.9.7-1.fc34.x86_64/Include/cpython/abstract.h:127
#21 call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=0x55555555afa0)
    at /usr/src/debug/python3.9-3.9.7-1.fc34.x86_64/Python/ceval.c:5075
--Type <RET> for more, q to quit, c to continue without paging--
#22 _PyEval_EvalFrameDefault (tstate=<optimized out>, f=<optimized out>, throwflag=<optimized out>)
    at /usr/src/debug/python3.9-3.9.7-1.fc34.x86_64/Python/ceval.c:3504
#23 0x00007ffff7d6afe3 in _PyEval_EvalFrame (throwflag=0, 
    f=Frame 0x7fffea310dd0, for file <frozen importlib._bootstrap>, line 680, in _load_unlocked (spec=<ModuleSpec(name='rapidfuzz.test', loader=<ExtensionFileLoader(name='rapidfuzz.test', path='/home/max/RapidFuzz/.venv/lib64/python3.9/site-packages/rapidfuzz/test.cpython-39-x86_64-linux-gnu.so') at remote 0x7fffea2498b0>, origin='/home/max/RapidFuzz/.venv/lib64/python3.9/site-packages/rapidfuzz/test.cpython-39-x86_64-linux-gnu.so', loader_state=None, submodule_search_locations=None, _set_fileattr=True, _cached=None, _initializing=True) at remote 0x7fffea2492b0>, module=<module at remote 0x7fffea250090>), tstate=0x55555555afa0)
    at /usr/src/debug/python3.9-3.9.7-1.fc34.x86_64/Include/internal/pycore_ceval.h:40
#24 function_code_fastcall (tstate=0x55555555afa0, co=<optimized out>, args=<optimized out>, nargs=1, 
    globals=<optimized out>) at /usr/src/debug/python3.9-3.9.7-1.fc34.x86_64/Objects/call.c:330
```
For the full backtrace check the following file:
[backtrace.txt](https://github.com/cython/cython/files/7169134/backtrace.txt)



Discussion:
It looks to be f18e3c9f9ce35ebf2cf2713775f35c47351e9c15 (i.e. my fault).
I think this is kind of intentional (partly).

If you're doing `cimport numpy` then you *should* be calling the Numpy C function `import_array` too. A lot of people were forgetting to do it (and getting odd crashes later because of it), so we started adding it in automatically.

To disable this autogeneration you can do:

```
# at global scope
<void>np.import_array
```

(i.e. just "use" the symbol from Cython's point of view).

It looks like `np.import_array` crashes when the Numpy module isn't present, so possibly we should guard against that. But I'm not sure how right now.
What I'm a little puzzled by is that the first thing `import_array` does is to try to import the Python module and raise an exception if that fails. So it should be pretty safe:

https://github.com/numpy/numpy/blob/1f95d7914ff23ffeb95767de79c55b4f702e6f2f/numpy/core/code_generators/generate_numpy_api.py#L44-L52

-------------------

Edit: OK - the problem isn't in `import_array` - it actually happens if you raise any exception in `__Pyx_InitGlobals`
> If you're doing cimport numpy then you should be calling the Numpy C function import_array too. A lot of people were forgetting to do it (and getting odd crashes later because of it), so we started adding it in automatically.

Yes I received a bug report yesterday, about my library crashing on arm since I started to use numpy: https://github.com/maxbachmann/Levenshtein/issues/12
My first thought was that I forgot to call import_array (which I did forget :smile:) and was very surprised to find, that this is automatically done by Cython.
```
__Pyx_PyDict_GetItemStr(*cython_runtime_dict, __pyx_n_s_cline_in_traceback)
```

but we import Numpy before initializing the string-tab... Should be fairly easily fixed I think
> I received a bug report yesterday, about my library crashing on arm since I started to use numpy: maxbachmann/Levenshtein#12
> My first thought was that I forgot to call import_array (which I did forget smile) and was very surprised to find, that this is automatically done by Cython.

I attempted to fix this by manually running `np.import_array` https://github.com/maxbachmann/RapidFuzz/blob/7cc1e5a053c6f2e7d84dd7348b766a2044a48c71/src/cpp_process_cdist.pyx#L44. This did indeed fix the segmentation fault. Now I am down to being binary incompatible, since pibuildwheels does not correctly use `oldest-supported-numpy` https://github.com/piwheels/packages/issues/212 ...


Resolving patch:
diff --git a/Cython/Compiler/Code.py b/Cython/Compiler/Code.py
--- a/Cython/Compiler/Code.py
+++ b/Cython/Compiler/Code.py
@@ -1138,7 +1138,8 @@ class GlobalState(object):
         'pystring_table',
         'cached_builtins',
         'cached_constants',
-        'init_globals',
+        'init_constants',
+        'init_globals',  # (utility code called at init-time)
         'init_module',
         'cleanup_globals',
         'cleanup_module',
@@ -1209,6 +1210,11 @@ def initialize_main_c_code(self):
         w.putln("")
         w.putln("static CYTHON_SMALL_CODE int __Pyx_InitGlobals(void) {")
 
+        w = self.parts['init_constants']
+        w.enter_cfunc_scope()
+        w.putln("")
+        w.putln("static CYTHON_SMALL_CODE int __Pyx_InitConstants(void) {")
+
         if not Options.generate_cleanup_code:
             del self.parts['cleanup_globals']
         else:
@@ -1284,13 +1290,14 @@ def close_global_decls(self):
         w.putln("}")
         w.exit_cfunc_scope()
 
-        w = self.parts['init_globals']
-        w.putln("return 0;")
-        if w.label_used(w.error_label):
-            w.put_label(w.error_label)
-            w.putln("return -1;")
-        w.putln("}")
-        w.exit_cfunc_scope()
+        for part in ['init_globals', 'init_constants']:
+            w = self.parts[part]
+            w.putln("return 0;")
+            if w.label_used(w.error_label):
+                w.put_label(w.error_label)
+                w.putln("return -1;")
+            w.putln("}")
+            w.exit_cfunc_scope()
 
         if Options.generate_cleanup_code:
             w = self.parts['cleanup_globals']
@@ -1510,7 +1517,7 @@ def generate_cached_methods_decls(self):
             return
 
         decl = self.parts['decls']
-        init = self.parts['init_globals']
+        init = self.parts['init_constants']
         cnames = []
         for (type_cname, method_name), cname in sorted(self.cached_cmethods.items()):
             cnames.append(cname)
@@ -1560,7 +1567,7 @@ def generate_string_constants(self):
                 decls_writer.putln("static Py_UNICODE %s[] = { %s };" % (cname, utf16_array))
                 decls_writer.putln("#endif")
 
-        init_globals = self.parts['init_globals']
+        init_constants = self.parts['init_constants']
         if py_strings:
             self.use_utility_code(UtilityCode.load_cached("InitStrings", "StringTools.c"))
             py_strings.sort()
@@ -1575,9 +1582,9 @@ def generate_string_constants(self):
             decls_writer.putln("#if !CYTHON_USE_MODULE_STATE")
             not_limited_api_decls_writer = decls_writer.insertion_point()
             decls_writer.putln("#endif")
-            init_globals.putln("#if CYTHON_USE_MODULE_STATE")
-            init_globals_in_module_state = init_globals.insertion_point()
-            init_globals.putln("#endif")
+            init_constants.putln("#if CYTHON_USE_MODULE_STATE")
+            init_constants_in_module_state = init_constants.insertion_point()
+            init_constants.putln("#endif")
             for idx, py_string_args in enumerate(py_strings):
                 c_cname, _, py_string = py_string_args
                 if not py_string.is_str or not py_string.encoding or \
@@ -1627,20 +1634,20 @@ def generate_string_constants(self):
                     py_string.is_str,
                     py_string.intern
                     ))
-                init_globals_in_module_state.putln("if (__Pyx_InitString(%s[%d], &%s) < 0) %s;" % (
+                init_constants_in_module_state.putln("if (__Pyx_InitString(%s[%d], &%s) < 0) %s;" % (
                     Naming.stringtab_cname,
                     idx,
                     py_string.cname,
-                    init_globals.error_goto(self.module_pos)))
+                    init_constants.error_goto(self.module_pos)))
             w.putln("{0, 0, 0, 0, 0, 0, 0}")
             w.putln("};")
 
-            init_globals.putln("#if !CYTHON_USE_MODULE_STATE")
-            init_globals.putln(
+            init_constants.putln("#if !CYTHON_USE_MODULE_STATE")
+            init_constants.putln(
                 "if (__Pyx_InitStrings(%s) < 0) %s;" % (
                     Naming.stringtab_cname,
-                    init_globals.error_goto(self.module_pos)))
-            init_globals.putln("#endif")
+                    init_constants.error_goto(self.module_pos)))
+            init_constants.putln("#endif")
 
     def generate_num_constants(self):
         consts = [(c.py_type, c.value[0] == '-', len(c.value), c.value, c.value_code, c)
@@ -1648,7 +1655,7 @@ def generate_num_constants(self):
         consts.sort()
         decls_writer = self.parts['decls']
         decls_writer.putln("#if !CYTHON_USE_MODULE_STATE")
-        init_globals = self.parts['init_globals']
+        init_constants = self.parts['init_constants']
         for py_type, _, _, value, value_code, c in consts:
             cname = c.cname
             self.parts['module_state'].putln("PyObject *%s;" % cname)
@@ -1669,9 +1676,9 @@ def generate_num_constants(self):
                 function = "PyInt_FromLong(%sL)"
             else:
                 function = "PyInt_FromLong(%s)"
-            init_globals.putln('%s = %s; %s' % (
+            init_constants.putln('%s = %s; %s' % (
                 cname, function % value_code,
-                init_globals.error_goto_if_null(cname, self.module_pos)))
+                init_constants.error_goto_if_null(cname, self.module_pos)))
         decls_writer.putln("#endif")
 
     # The functions below are there in a transition phase only
diff --git a/Cython/Compiler/ModuleNode.py b/Cython/Compiler/ModuleNode.py
--- a/Cython/Compiler/ModuleNode.py
+++ b/Cython/Compiler/ModuleNode.py
@@ -2939,7 +2939,7 @@ def generate_module_init_func(self, imported_modules, env, code):
 
         # start of module init/exec function (pre/post PEP 489)
         code.putln("{")
-
+        code.putln('int stringtab_initialized = 0;')
         tempdecl_code = code.insertion_point()
 
         profile = code.globalstate.directives['profile']
@@ -3012,7 +3012,10 @@ def generate_module_init_func(self, imported_modules, env, code):
         code.putln("#endif")
 
         code.putln("/*--- Initialize various global constants etc. ---*/")
-        code.put_error_if_neg(self.pos, "__Pyx_InitGlobals()")
+        code.put_error_if_neg(self.pos, "__Pyx_InitConstants()")
+        code.putln("stringtab_initialized = 1;")
+        code.put_error_if_neg(self.pos, "__Pyx_InitGlobals()")  # calls any utility code
+
 
         code.putln("#if PY_MAJOR_VERSION < 3 && (__PYX_DEFAULT_STRING_ENCODING_IS_ASCII || "
                    "__PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT)")
@@ -3095,7 +3098,9 @@ def generate_module_init_func(self, imported_modules, env, code):
         for cname, type in code.funcstate.all_managed_temps():
             code.put_xdecref(cname, type)
         code.putln('if (%s) {' % env.module_cname)
-        code.putln('if (%s) {' % env.module_dict_cname)
+        code.putln('if (%s && stringtab_initialized) {' % env.module_dict_cname)
+        # We can run into errors before the module or stringtab are initialized.
+        # In this case it is not safe to add a traceback (because it uses the stringtab)
         code.put_add_traceback(EncodedString("init %s" % env.qualified_name))
         code.globalstate.use_utility_code(Nodes.traceback_utility_code)
         # Module reference and module dict are in global variables which might still be needed


Test patch:
diff --git a/tests/run/numpy_import_array_error.srctree b/tests/run/numpy_import_array_error.srctree
new file mode 100644
--- /dev/null
+++ b/tests/run/numpy_import_array_error.srctree
@@ -0,0 +1,40 @@
+PYTHON setup.py build_ext -i
+PYTHON main.py
+
+############# setup.py ############
+
+from distutils.core import setup
+from Cython.Build import cythonize
+
+setup(ext_modules = cythonize('cimport_numpy.pyx'))
+
+############# numpy.pxd ############
+
+# A fake Numpy module. This defines a version of _import_array
+# that always fails. The Cython-generated call to _import_array
+# happens quite early (before the stringtab is initialized)
+# and thus the error itself handling could cause a segmentation fault
+# https://github.com/cython/cython/issues/4377
+
+cdef extern from *:
+    """
+    #define NPY_NDARRAYOBJECT_H
+    static int _import_array(void) {
+        PyErr_SetString(PyExc_ValueError, "Oh no!");
+        return -1;
+    }
+    """
+    int _import_array() except -1
+    
+############# cimport_numpy.pyx ###########
+
+cimport numpy
+
+############# main.py ####################
+
+try:
+    import cimport_numpy
+except ImportError as e:
+    print(e)
+else:
+    assert(False)