Repository: qutip/qutip

Issue:
OpenMP broken in current master, OpenMP tests worrying
OpenMP handling is broken in the current `master`.  It is likely caused by changes made in #1289.  Trying to `import qutip` with OpenMP built produces errors
```python
In [1]: import qutip
Calibrating OPENMP threshold...
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-1-e01d5608f34f> in <module>
----> 1 import qutip

~/code/qutip/qutip/qutip/__init__.py in <module>
    277         print('Calibrating OPENMP threshold...')
    278         thrsh = calculate_openmp_thresh()
--> 279         qutip.configrc.write_rc_key(rc_file, 'openmp_thresh', thrsh)
    280
    281 # Load the config file

~/code/qutip/qutip/qutip/configrc.py in write_rc_key(key, value, section, rc_file)
    169     config.read(rc_file)
    170     if not config.has_section(section):
--> 171         config.add_section(section)
    172     config.set(section, key, str(value))
    173

~/.anaconda3/envs/qutip-dev/lib/python3.8/configparser.py in add_section(self, section)
   1205         RawConfigParser.add_section by validating if the section name is
   1206         a string."""
-> 1207         self._validate_value_types(section=section)
   1208         super().add_section(section)
   1209

~/.anaconda3/envs/qutip-dev/lib/python3.8/configparser.py in _validate_value_types(self, section, option, value)
   1178         """
   1179         if not isinstance(section, str):
-> 1180             raise TypeError("section names must be strings")
   1181         if not isinstance(option, str):
   1182             raise TypeError("option keys must be strings")

TypeError: section names must be strings
```

The issue is probably due to `__init__.py` being out-of-date with regards to the change to how the `qutiprc` file is written.  The immediate fix should be very simple, however it highlights a larger problem with our testing - this should have been caught by the OpenMP tests.

In this case, I believe that the tests don't see the issue because there is a test in `__init__.py` that doesn't trigger the OpenMP calibration if only there is only 1 CPU.  Presumably, we only ever get allocated single-CPU VMs on Travis, so it hasn't got caught.  That makes me worry a lot that the OpenMP code isn't even being tested correctly.


Discussion:

Resolving patch:
diff --git a/qutip/__init__.py b/qutip/__init__.py
--- a/qutip/__init__.py
+++ b/qutip/__init__.py
@@ -30,65 +30,21 @@
 #    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 #    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ###############################################################################
-from __future__ import division, print_function, absolute_import
 import os
-import sys
 import warnings
 
 import qutip.settings
 import qutip.version
 from qutip.version import version as __version__
-from qutip.utilities import _version2int
 
 # -----------------------------------------------------------------------------
 # Check if we're in IPython.
 try:
     __IPYTHON__
     qutip.settings.ipython = True
-except:
+except NameError:
     qutip.settings.ipython = False
 
-# -----------------------------------------------------------------------------
-# Check for minimum requirements of dependencies, give the user a warning
-# if the requirements aren't fulfilled
-#
-
-numpy_requirement = "1.12.0"
-try:
-    import numpy
-    if _version2int(numpy.__version__) < _version2int(numpy_requirement):
-        print("QuTiP warning: old version of numpy detected " +
-              ("(%s), requiring %s." %
-               (numpy.__version__, numpy_requirement)))
-except:
-    warnings.warn("numpy not found.")
-
-scipy_requirement = "1.0.0"
-try:
-    import scipy
-    if _version2int(scipy.__version__) < _version2int(scipy_requirement):
-        print("QuTiP warning: old version of scipy detected " +
-              ("(%s), requiring %s." %
-               (scipy.__version__, scipy_requirement)))
-except:
-    warnings.warn("scipy not found.")
-
-# -----------------------------------------------------------------------------
-# check to see if running from install directory for released versions.
-#
-top_path = os.path.dirname(os.path.dirname(__file__))
-try:
-    setup_file = open(top_path + '/setup.py', 'r')
-except:
-    pass
-else:
-    if ('QuTiP' in setup_file.readlines()[1][3:]) and qutip.version.release:
-        print("You are in the installation directory. " +
-              "Change directories before running QuTiP.")
-    setup_file.close()
-
-del top_path
-
 
 # -----------------------------------------------------------------------------
 # Look to see if we are running with OPENMP
@@ -99,7 +55,7 @@
 
 try:
     from qutip.cy.openmp.parfuncs import spmv_csr_openmp
-except:
+except ImportError:
     qutip.settings.has_openmp = False
 else:
     qutip.settings.has_openmp = True
@@ -107,27 +63,29 @@
     os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'
 
 import platform
-from .utilities import _blas_info
+from qutip.utilities import _blas_info
 qutip.settings.eigh_unsafe = (_blas_info() == "OPENBLAS" and
                               platform.system() == 'Darwin')
+del platform, _blas_info
 # -----------------------------------------------------------------------------
 # setup the cython environment
 #
-_cython_requirement = "0.21.0"
 try:
-    import Cython
-    if _version2int(Cython.__version__) < _version2int(_cython_requirement):
-        print("QuTiP warning: old version of cython detected " +
-              ("(%s), requiring %s." %
-               (Cython.__version__, _cython_requirement)))
-    # Setup pyximport
-    import qutip.cy.pyxbuilder as pbldr
-    pbldr.install(setup_args={'include_dirs': [numpy.get_include()]})
-    del pbldr
-except Exception as e:
+    import Cython as _Cython
+except ImportError:
     pass
 else:
-    del Cython
+    from qutip.utilities import _version2int
+    _cy_require = "0.29.20"
+    if _version2int(_Cython.__version__) < _version2int(_cy_require):
+        warnings.warn(
+            "Old version of Cython detected: needed {}, got {}."
+            .format(_cy_require, _Cython.__version__)
+        )
+    # Setup pyximport
+    import qutip.cy.pyxbuilder as _pyxbuilder
+    _pyxbuilder.install()
+    del _pyxbuilder, _Cython, _version2int
 
 
 # -----------------------------------------------------------------------------
@@ -144,15 +102,17 @@
 if qutip.settings.num_cpus == 0:
     # if num_cpu is 0 set it to the available number of cores
     import qutip.hardware_info
-    info =  qutip.hardware_info.hardware_info()
+    info = qutip.hardware_info.hardware_info()
     if 'cpus' in info:
         qutip.settings.num_cpus = info['cpus']
     else:
         try:
             qutip.settings.num_cpus = multiprocessing.cpu_count()
-        except:
+        except NotImplementedError:
             qutip.settings.num_cpus = 1
 
+del multiprocessing
+
 
 # Find MKL library if it exists
 import qutip._mkl
@@ -165,7 +125,7 @@
 # Check for Matplotlib
 try:
     import matplotlib
-except:
+except ImportError:
     warnings.warn("matplotlib not found: Graphics will not work.")
 else:
     del matplotlib
@@ -247,36 +207,30 @@
 from qutip.about import *
 from qutip.cite import *
 
-# Remove -Wstrict-prototypes from cflags
-import distutils.sysconfig
-cfg_vars = distutils.sysconfig.get_config_vars()
-if "CFLAGS" in cfg_vars:
-    cfg_vars["CFLAGS"] = cfg_vars["CFLAGS"].replace("-Wstrict-prototypes", "")
-
 # -----------------------------------------------------------------------------
 # Load user configuration if present: override defaults.
 #
 import qutip.configrc
 has_rc, rc_file = qutip.configrc.has_qutip_rc()
 
-# Make qutiprc and benchmark OPENMP if has_rc = False
-if qutip.settings.has_openmp and (not has_rc):
-    qutip.configrc.generate_qutiprc()
-    has_rc, rc_file = qutip.configrc.has_qutip_rc()
-    if has_rc and qutip.settings.num_cpus > 1:
-        from qutip.cy.openmp.bench_openmp import calculate_openmp_thresh
-        #bench OPENMP
-        print('Calibrating OPENMP threshold...')
-        thrsh = calculate_openmp_thresh()
-        qutip.configrc.write_rc_key(rc_file, 'openmp_thresh', thrsh)
-# Make OPENMP if has_rc but 'openmp_thresh' not in keys
-elif qutip.settings.has_openmp and has_rc:
-    has_omp_key = qutip.configrc.has_rc_key(rc_file, 'openmp_thresh')
-    if not has_omp_key and qutip.settings.num_cpus > 1:
+# Read the OpenMP threshold out if it already exists, or calibrate and save it
+# if it doesn't.
+if qutip.settings.has_openmp:
+    _calibrate_openmp = qutip.settings.num_cpus > 1
+    if has_rc:
+        _calibrate_openmp = (
+            _calibrate_openmp
+            and not qutip.configrc.has_rc_key('openmp_thresh', rc_file=rc_file)
+        )
+    else:
+        qutip.configrc.generate_qutiprc()
+        has_rc, rc_file = qutip.configrc.has_qutip_rc()
+    if _calibrate_openmp:
+        print('Calibrating OpenMP threshold...')
         from qutip.cy.openmp.bench_openmp import calculate_openmp_thresh
-        print('Calibrating OPENMP threshold...')
-        thrsh = calculate_openmp_thresh()
-        qutip.configrc.write_rc_key(rc_file, 'openmp_thresh', thrsh)
+        thresh = calculate_openmp_thresh()
+        qutip.configrc.write_rc_key('openmp_thresh', thresh, rc_file=rc_file)
+        del calculate_openmp_thresh
 
 # Load the config file
 if has_rc:
@@ -285,4 +239,4 @@
 # -----------------------------------------------------------------------------
 # Clean name space
 #
-del os, sys, numpy, scipy, multiprocessing, distutils
+del os, warnings
diff --git a/qutip/cy/pyxbuilder.py b/qutip/cy/pyxbuilder.py
--- a/qutip/cy/pyxbuilder.py
+++ b/qutip/cy/pyxbuilder.py
@@ -30,33 +30,42 @@
 #    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 #    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ###############################################################################
-import sys, os
-try:
-    import pyximport    
-    from pyximport import install
-
-    old_get_distutils_extension = pyximport.pyximport.get_distutils_extension
-
-    def new_get_distutils_extension(modname, pyxfilename, language_level=None):
-        extension_mod, setup_args = old_get_distutils_extension(modname,
-                                                                pyxfilename,
-                                                                language_level)
-        extension_mod.language='c++'
-        # If on Win and Python version >= 3.5 and not in MSYS2
-        # (i.e. Visual studio compile)
-        if sys.platform == 'win32' and \
-                int(str(sys.version_info[0]) +
-                    str(sys.version_info[1])) >= 35 and \
-                os.environ.get('MSYSTEM') is None:
-            extension_mod.extra_compile_args = ['/w', '/O1']
-        else:
-            extension_mod.extra_compile_args = ['-w', '-O1']
-            if sys.platform == 'darwin':
-                extension_mod.extra_compile_args.append(
-                                                '-mmacosx-version-min=10.9')
-                extension_mod.extra_link_args = ['-mmacosx-version-min=10.9']
-        return extension_mod,setup_args
-
-    pyximport.pyximport.get_distutils_extension = new_get_distutils_extension
-except Exception:
-    pass
+import sys
+import os
+
+import numpy as np
+import pyximport
+
+old_get_distutils_extension = pyximport.pyximport.get_distutils_extension
+
+
+def new_get_distutils_extension(modname, pyxfilename, language_level=None):
+    # Remove -Wstrict-prototypes from cflags; we build in C++ mode, where the
+    # flag is invalid, but for some reason distutils still has it as a default,
+    # and tries to append CFLAGS to the compile even in C++ mode.
+    import distutils.sysconfig
+    cfg_vars = distutils.sysconfig.get_config_vars()
+    if "CFLAGS" in cfg_vars:
+        cfg_vars["CFLAGS"] = cfg_vars["CFLAGS"].replace("-Wstrict-prototypes",
+                                                        "")
+    extension_mod, setup_args =\
+        old_get_distutils_extension(modname, pyxfilename, language_level)
+    extension_mod.language = 'c++'
+    # If on Win and Python version >= 3.5 and not in MSYS2
+    # (i.e. Visual studio compile)
+    if sys.platform == 'win32' and os.environ.get('MSYSTEM') is None:
+        extension_mod.extra_compile_args = ['/w', '/O1']
+    else:
+        extension_mod.extra_compile_args = ['-w', '-O1']
+    if sys.platform == 'darwin':
+        extension_mod.extra_compile_args.append('-mmacosx-version-min=10.9')
+        extension_mod.extra_link_args.append('-mmacosx-version-min=10.9')
+    return extension_mod, setup_args
+
+
+pyximport.pyximport.get_distutils_extension = new_get_distutils_extension
+
+
+def install():
+    """Install the pyximport interface."""
+    return pyximport.install(setup_args={'include_dirs': [np.get_include()]})
diff --git a/qutip/steadystate.py b/qutip/steadystate.py
--- a/qutip/steadystate.py
+++ b/qutip/steadystate.py
@@ -61,7 +61,6 @@
 from qutip.graph import weighted_bipartite_matching
 from qutip import (mat2vec, tensor, identity, operator_to_vector)
 import qutip.settings as settings
-from qutip.utilities import _version2int
 from qutip.cy.spconvert import dense2D_to_fastcsr_fmode
 
 import qutip.logging_utils
@@ -72,9 +71,6 @@
 if settings.has_mkl:
     from qutip._mkl.spsolve import (mkl_splu, mkl_spsolve)
 
-# test if scipy is recent enought to get L & U factors from superLU
-_scipy_check = _version2int(scipy.__version__) >= _version2int('0.14.0')
-
 
 def _empty_info_dict():
     def_info = {'perm': [], 'solution_time': None,
@@ -423,7 +419,7 @@ def _steadystate_direct_sparse(L, ss_args):
         v = lu.solve(b)
         _direct_end = time.time()
         ss_args['info']['solution_time'] = _direct_end - _direct_start
-        if (settings.debug or ss_args['return_info']) and _scipy_check:
+        if (settings.debug or ss_args['return_info']):
             L_nnz = lu.L.nnz
             U_nnz = lu.U.nnz
             ss_args['info']['l_nnz'] = L_nnz
@@ -566,20 +562,18 @@ def _iterative_precondition(A, n, ss_args):
                 logger.debug('Preconditioning succeeded.')
                 logger.debug('Precond. time: %f' %
                              (_precond_end - _precond_start))
-
-            if _scipy_check:
-                L_nnz = P.L.nnz
-                U_nnz = P.U.nnz
-                ss_args['info']['l_nnz'] = L_nnz
-                ss_args['info']['u_nnz'] = U_nnz
-                ss_args['info']['ilu_fill_factor'] = (L_nnz+U_nnz)/A.nnz
-                e = np.ones(n ** 2, dtype=int)
-                condest = la.norm(M*e, np.inf)
-                ss_args['info']['ilu_condest'] = condest
-                if settings.debug:
-                    logger.debug('L NNZ: %i ; U NNZ: %i' % (L_nnz, U_nnz))
-                    logger.debug('Fill factor: %f' % ((L_nnz+U_nnz)/A.nnz))
-                    logger.debug('iLU condest: %f' % condest)
+            L_nnz = P.L.nnz
+            U_nnz = P.U.nnz
+            ss_args['info']['l_nnz'] = L_nnz
+            ss_args['info']['u_nnz'] = U_nnz
+            ss_args['info']['ilu_fill_factor'] = (L_nnz+U_nnz)/A.nnz
+            e = np.ones(n ** 2, dtype=int)
+            condest = la.norm(M*e, np.inf)
+            ss_args['info']['ilu_condest'] = condest
+            if settings.debug:
+                logger.debug('L NNZ: %i ; U NNZ: %i' % (L_nnz, U_nnz))
+                logger.debug('Fill factor: %f' % ((L_nnz+U_nnz)/A.nnz))
+                logger.debug('iLU condest: %f' % condest)
 
     except:
         raise Exception("Failed to build preconditioner. Try increasing " +
@@ -863,7 +857,7 @@ def _iter_count(r):
                       diag_pivot_thresh=ss_args['diag_pivot_thresh'],
                       options=dict(ILU_MILU=ss_args['ILU_MILU']))
 
-            if settings.debug and _scipy_check:
+            if settings.debug:
                 L_nnz = lu.L.nnz
                 U_nnz = lu.U.nnz
                 logger.debug('L NNZ: %i ; U NNZ: %i' % (L_nnz, U_nnz))


Test patch:
diff --git a/qutip/tests/test_sp_eigs.py b/qutip/tests/test_sp_eigs.py
--- a/qutip/tests/test_sp_eigs.py
+++ b/qutip/tests/test_sp_eigs.py
@@ -37,11 +37,8 @@
 import unittest
 
 from qutip import num, rand_herm, expect, rand_unitary
-from qutip import _version2int
 
 
-@unittest.skipIf(_version2int(scipy.__version__) < _version2int('0.10'),
-                 'Known to fail on SciPy ' + scipy.__version__)
 def test_SparseHermValsVecs():
     """
     Sparse eigs Hermitian
@@ -105,8 +102,6 @@ def test_SparseValsVecs():
     assert_equal(len(spvals), 9)
 
 
-@unittest.skipIf(_version2int(scipy.__version__) < _version2int('0.10'),
-                 'Known to fail on SciPy ' + scipy.__version__)
 def test_SparseValsOnly():
     """
     Sparse eigvals only Hermitian.