Repository: scipy/scipy

Issue:
scipy.interpolate.RectBivariateSpline construction locks PyQt event loop
Construction of a RectBivariateSpline is time-consuming, and I want to create it in a background PyQt thread and then pass to my GUI. Alas, its construction even in dedicated thread freezes PyQt event loop. Why does it happen? AFAIK, scipy calls do release GIL lock?

Here is my sample code

```
# -*- coding: utf-8 -*-

import numpy as np
import scipy as sp
import scipy.interpolate

import os
import sys
import time
import pickle
from PyQt4 import QtCore, QtGui


class LongRunningWorker(QtCore.QObject):

    result_ready = QtCore.pyqtSignal(tuple)

    @QtCore.pyqtSlot()
    def generate_uber_mega_data(self):
        print "Oy!"
        # time.sleep(5)

        r_ticks = np.linspace(0, 1000, 5000)
        phi_ticks = np.linspace(0, 2 * np.pi, 5000)

        r_grid, phi_grid = np.meshgrid(r_ticks, phi_ticks)

        print u"Starting interpolation"
        spline = sp.interpolate.RectBivariateSpline(r_ticks, phi_ticks, r_grid * phi_grid)
        print u"Интерполяция всё / Interpolation complete"
        time.sleep(5) # And here (while the sleep is running) GUI starts working
        print "Vey!"
        self.result_ready.emit((7, 40))

@QtCore.pyqtSlot(tuple)
def process_mega_data(mega_data):
    print mega_data

if __name__ == "__main__":
    app = QtGui.QApplication(sys.argv)
    mega_button = QtGui.QPushButton("Press__________________________________________here")
    mega_button.show()

    background_thread = QtCore.QThread()
    worker = LongRunningWorker()
    worker.moveToThread(background_thread)

    background_thread.start()

    # Press here -- and interpolation will start and GUI events on SpinBox will freeze
    mega_button.clicked.connect(worker.generate_uber_mega_data)
    worker.result_ready.connect(process_mega_data)

    # This little scum doesn't change values, when interpolation is running!
    mega_spinbox = QtGui.QSpinBox()
    mega_spinbox.show()

    sys.exit(app.exec_())
```



Discussion:
Seems like Fortran functions in interpolate/src/fitpack.pyf are not marked as thread safe. Is there a reason for that?

There is an affirmative reason to make them thread-safe.

For example, this interpolator may be created for several seconds or even more. While it's being created, without GIL release, we have only three ways:

1) Suffer from freezed GUI

2) Do various sodomic perversions, creating the interpolator in other process and them calling it from main process and getting little pieces of data in main process (point by point): its serialization and deserialization is extremely slow.

3) Make it release GIL.

I don't remember this being asked or looked into before, could be that there's no good reason.

Fitpack code doesn't seem to have `save` or `common` blocks, so that should be enough for Fortran77 code to be thread-safe (?). (disclaimer: my Fortran knowledge is poor)

Fortran didn't use to have stack variables, and after they were introduced they were't the default. Hmm... <checks> See http://stackoverflow.com/questions/2582409/are-local-variables-in-fortran-77-static-or-stack-dynamic </check>

Note that LAPACK threadsafety also relies on the fortran compiler having
stack variables. The default behavior in gfortran is IIRC dangerous, as
it puts only "small" arrays on stack. There was a LAPACK bug reported
some years back on threadsafety, don't remember how they addressed it.
.
However, be it as it may, there's no specific need to use the _GIL_ for
this purpose. Each of the fortran codes could as well use their own
locks. This in principle could be implemented inside f2py, but we might
also work around it manually by declaring routines threadsafe or not.
.
The main question here is dealing with potential deadlocks on recursive
calls from Python callbacks.

> There was a LAPACK bug reported some years back on threadsafety, don't remember how they addressed it.

You must be referring to this one: [bug112 :: DSYEVR does not seem to be thread-safe](http://www.netlib.org/lapack/bug_list.html#_strong_span_class_green_bug112_span_strong_dsyevr_does_not_seem_to_be_thread_safe).

The problem was solved by compiling LAPACK with`-frecursive`. Perhaps we should enable it by default for `gfortran`, in `numpy/distutils/fcompiler/gnu.py`.

Anyway, I've looked through FITPACK sources and haven't found any local arrays larger than 160 bytes. The default value for `-fmax-stack-var-size` is 32768, so the code must be thread safe with `gfortran` even without `-frecursive`.


Resolving patch:
diff --git a/scipy/_lib/_testutils.py b/scipy/_lib/_testutils.py
new file mode 100644
--- /dev/null
+++ b/scipy/_lib/_testutils.py
@@ -0,0 +1,27 @@
+"""
+Generic test utilities and decorators.
+
+"""
+
+from __future__ import division, print_function, absolute_import
+
+import os
+from numpy.testing import dec
+
+
+__all__ = ['knownfailure_overridable']
+
+
+def knownfailure_overridable(msg=None):
+    if not msg:
+        msg = "Undiagnosed issues (corner cases, wrong comparison values, or otherwise)"
+    msg = msg + " [Set environment variable SCIPY_XFAIL=1 to run this test nevertheless.]"
+
+    def deco(func):
+        try:
+            if bool(os.environ['SCIPY_XFAIL']):
+                return func
+        except (ValueError, KeyError):
+            pass
+        return dec.knownfailureif(True, msg)(func)
+    return deco


Test patch:
diff --git a/scipy/interpolate/tests/test_gil.py b/scipy/interpolate/tests/test_gil.py
new file mode 100644
--- /dev/null
+++ b/scipy/interpolate/tests/test_gil.py
@@ -0,0 +1,71 @@
+from __future__ import division, print_function, absolute_import
+
+import itertools
+import threading
+import time
+
+import numpy as np
+from numpy.testing import TestCase, assert_equal, run_module_suite
+from numpy.testing.decorators import slow
+import scipy.interpolate
+from scipy._lib._testutils import knownfailure_overridable
+
+
+class TestGIL(TestCase):
+    """Check if the GIL is properly released by scipy.interpolate functions."""
+
+    def setUp(self):
+        self.messages = []
+
+    def log(self, message):
+        self.messages.append(message)
+
+    def make_worker_thread(self, target, args):
+        log = self.log
+
+        class WorkerThread(threading.Thread):
+            def run(self):
+                log('interpolation started')
+                target(*args)
+                log('interpolation complete')
+
+        return WorkerThread()
+
+    @slow
+    @knownfailure_overridable('race conditions, may depend on system load')
+    def test_rectbivariatespline(self):
+        def generate_params(n_points):
+            x = y = np.linspace(0, 1000, n_points)
+            x_grid, y_grid = np.meshgrid(x, y)
+            z = x_grid * y_grid
+            return x, y, z
+
+        def calibrate_delay(requested_time):
+            for n_points in itertools.count(5000, 1000):
+                args = generate_params(n_points)
+                time_started = time.time()
+                interpolate(*args)
+                if time.time() - time_started > requested_time:
+                    return args
+
+        def interpolate(x, y, z):
+            scipy.interpolate.RectBivariateSpline(x, y, z)
+
+        args = calibrate_delay(requested_time=3)
+        worker_thread = self.make_worker_thread(interpolate, args)
+        worker_thread.start()
+        for i in range(3):
+            time.sleep(0.5)
+            self.log('working')
+        worker_thread.join()
+        assert_equal(self.messages, [
+            'interpolation started',
+            'working',
+            'working',
+            'working',
+            'interpolation complete',
+        ])
+
+
+if __name__ == "__main__":
+    run_module_suite()
diff --git a/scipy/special/tests/test_mpmath.py b/scipy/special/tests/test_mpmath.py
--- a/scipy/special/tests/test_mpmath.py
+++ b/scipy/special/tests/test_mpmath.py
@@ -5,7 +5,6 @@
 from __future__ import division, print_function, absolute_import
 
 import sys
-import os
 import time
 
 from distutils.version import LooseVersion
@@ -16,6 +15,7 @@
 
 import scipy.special as sc
 from scipy._lib.six import reraise, with_metaclass
+from scipy._lib._testutils import knownfailure_overridable
 from scipy.special._testutils import FuncData, assert_func_equal
 
 try:
@@ -496,21 +496,6 @@ def nonfunctional_tooslow(func):
     return dec.skipif(True, "    Test not yet functional (too slow), needs more work.")(func)
 
 
-def knownfailure_overridable(msg=None):
-    if not msg:
-        msg = "Undiagnosed issues (corner cases, wrong comparison values, or otherwise)"
-    msg = msg + " [Set environment variable SCIPY_XFAIL=1 to run this test nevertheless.]"
-
-    def deco(func):
-        try:
-            if bool(os.environ['SCIPY_XFAIL']):
-                return func
-        except (ValueError, KeyError):
-            pass
-        return dec.knownfailureif(True, msg)(func)
-    return deco
-
-
 class _SystematicMeta(type):
     """
     Metaclass which decorates all of the test_* methods with