Repository: qutip/qutip

Issue:
Without warning, coefficient functions are not compiled
### Describe the Issue!

The `filelock` package is required for cython-compiling coefficient functions. However, QuTiP currently does not warn if `filelock` is missing, and the dependency does not seem to be documented anywhere. Instead, QuTiP silently defaults to a `StrFunctionCoefficient`, i.e., to using `eval`. This behavior comes from here:

https://github.com/qutip/qutip/blob/7892453dd3ae03ce8ac17caffc0457df200a69ec/qutip/core/coefficient.py#L256-L264

The default value of `use_cython` is therefore set to `False` if `filelock` is missing. The result is the following:

```python
$ git clone git@github.com:qutip/qutip.git .
$ conda create -n qutip-tmp python cython numpy scipy setuptools packaging wheel pytest ipython matplotlib
$ conda activate qutip-tmp
$ python setup.py install
$ cd ~
$ ipython
	import qutip
	coeff = qutip.coefficient("sin(t)")
	type(coeff)
	> qutip.core.cy.coefficient.StrFunctionCoefficient
	exit()
```

Since cython is installed and there was no warning or error, I think it would be reasonable for the user to assume that the coefficient would be compiled, but it is not. Only when `use_cython` is explicitly enabled, we get an error:

```shell
In [5]: coeff = qutip.coefficient("sin(t)", compile_opt=qutip.CompilationOptions(use_cython=True))
---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
Cell In[5], line 1
----> 1 coeff = qutip.coefficient("sin(t)", compile_opt=qutip.CompilationOptions(use_cython=True))

File ~/miniconda3/envs/qutip-tmp/lib/python3.11/site-packages/qutip-5.0.0.dev0+7892453-py3.11-linux-x86_64.egg/qutip/core/coefficient.py:170, in coefficient(base, tlist, args, args_ctypes, order, compile_opt, function_style, boundary_conditions, **kwargs)
    168 for type_ in coefficient_builders:
    169     if isinstance(base, type_):
--> 170         return coefficient_builders[type_](base, **kwargs)
    172 if callable(base):
    173     op = FunctionCoefficient(base, args.copy(), style=function_style)

File ~/miniconda3/envs/qutip-tmp/lib/python3.11/site-packages/qutip-5.0.0.dev0+7892453-py3.11-linux-x86_64.egg/qutip/core/coefficient.py:388, in coeff_from_str(base, args, args_ctypes, compile_opt, **_)
    385 code = make_cy_code(parsed, variables, constants,
    386                     raw, compile_opt)
    387 try:
--> 388     coeff = compile_code(code, file_name, parsed, compile_opt)
    389 except PermissionError:
    390     pass

File ~/miniconda3/envs/qutip-tmp/lib/python3.11/site-packages/qutip-5.0.0.dev0+7892453-py3.11-linux-x86_64.egg/qutip/core/coefficient.py:528, in compile_code(code, file_name, parsed, c_opt)
    524 os.chdir(qset.coeffroot)
    525 # Files with the same name, but differents extension than the pyx file, are
    526 # erased during cythonization process, breaking filelock.
    527 # Adding a prefix make them safe to use.
--> 528 lock = filelock.FileLock("compile_lock_" + file_name + ".lock")
    529 try:
    530     lock.acquire(timeout=0)

NameError: name 'filelock' is not defined
```

However, I would expect it to be rare that users call `qutip.coefficient` directly including compilation options. The only indication that something might be wrong then comes from two failing tests, but it is very hard to deduce the actual problem from the pytest output:

```shell
$ pytest --disable-pytest-warnings --pyargs qutip.tests.core -k test_coefficient
========================================================================= test session starts =========================================================================
platform linux -- Python 3.11.3, pytest-7.1.2, pluggy-1.0.0
rootdir: /home/paul
collected 5917 items / 5700 deselected / 217 selected

miniconda3/envs/qutip-tmp/lib/python3.11/site-packages/qutip-5.0.0.dev0+7892453-py3.11-linux-x86_64.egg/qutip/tests/core/test_coefficient.py .................. [  8%]
................................................................................................................F........F..................................... [ 81%]
.......................................s                                                                                                                        [100%]

============================================================================== FAILURES ===============================================================================
__________________________________________________________________________ test_CoeffOptions __________________________________________________________________________

    @pytest.mark.requires_cython
    def test_CoeffOptions():
        from itertools import combinations
        base = "1 + 1. + 1j"
        options = []
        options.append(CompilationOptions(accept_int=True))
        options.append(CompilationOptions(accept_float=False))
        options.append(CompilationOptions(static_types=True))
        options.append(CompilationOptions(try_parse=False))
        options.append(CompilationOptions(use_cython=False))
        coeffs = [coefficient(base, compile_opt=opt) for opt in options]
        for coeff in coeffs:
            assert coeff(0) == 2+1j
        for coeff1, coeff2 in combinations(coeffs, 2):
>           assert not isinstance(coeff1, coeff2.__class__)
E           AssertionError: assert not True
E            +  where True = isinstance(<qutip.core.cy.coefficient.StrFunctionCoefficient object at 0x7f86f86a7d40>, <class 'qutip.core.cy.coefficient.StrFunctionCoefficient'>)
E            +    where <class 'qutip.core.cy.coefficient.StrFunctionCoefficient'> = <qutip.core.cy.coefficient.StrFunctionCoefficient object at 0x7f86f86a7f00>.__class__

miniconda3/envs/qutip-tmp/lib/python3.11/site-packages/qutip-5.0.0.dev0+7892453-py3.11-linux-x86_64.egg/qutip/tests/core/test_coefficient.py:239: AssertionError
__________________________________________________________________________ test_advance_use ___________________________________________________________________________

    @pytest.mark.requires_cython
    def test_advance_use():
        opt = CompilationOptions(recompile=True, extra_import="""
    from qutip import basis
    from qutip.core.data cimport CSR
    from qutip.core.data.expect cimport expect_csr
    """)
        csr = qutip.num(3).data
        coeff = coefficient("expect_csr(op, op)",
                            args={"op": csr},
                            args_ctypes={"op": "CSR"},
                            compile_opt=opt)
>       assert coeff(0) == 5.

miniconda3/envs/qutip-tmp/lib/python3.11/site-packages/qutip-5.0.0.dev0+7892453-py3.11-linux-x86_64.egg/qutip/tests/core/test_coefficient.py:292:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
qutip/core/cy/coefficient.pyx:126: in qutip.core.cy.coefficient.Coefficient.__call__
    ???
qutip/core/cy/coefficient.pyx:346: in qutip.core.cy.coefficient.StrFunctionCoefficient._call
    ???
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

t = 0.0, args = {'op': CSR(shape=(3, 3), nnz=3)}

>   ???
E   NameError: name 'expect_csr' is not defined

<string>:4: NameError
======================================================================= short test summary info =======================================================================
FAILED miniconda3/envs/qutip-tmp/lib/python3.11/site-packages/qutip-5.0.0.dev0+7892453-py3.11-linux-x86_64.egg/qutip/tests/core/test_coefficient.py::test_CoeffOptions
FAILED miniconda3/envs/qutip-tmp/lib/python3.11/site-packages/qutip-5.0.0.dev0+7892453-py3.11-linux-x86_64.egg/qutip/tests/core/test_coefficient.py::test_advance_use
=============================================== 2 failed, 214 passed, 1 skipped, 5700 deselected, 13 warnings in 1.17s ================================================
```

(Note that the second failing test only fails because without cython, there is no `cimport`.)

---

If we install filelock, everything works as expected:

```shell
$ conda install filelock
$ ipython
	import qutip
	coeff = qutip.coefficient("sin(t)")
	> [1/1] Cythonizing qtcoeff_f230db93af3a399e4166e330cf4663.pyx
	> [...]
	type(coeff)
	> qtcoeff_f230db93af3a399e4166e330cf4663.StrCoefficient
	exit()
$ pytest --disable-pytest-warnings --pyargs qutip.tests.core -k test_coefficient
========================================================================= test session starts =========================================================================
platform linux -- Python 3.11.3, pytest-7.1.2, pluggy-1.0.0
rootdir: /home/paul
collected 5917 items / 5700 deselected / 217 selected

miniconda3/envs/qutip-tmp/lib/python3.11/site-packages/qutip-5.0.0.dev0+7892453-py3.11-linux-x86_64.egg/qutip/tests/core/test_coefficient.py .................. [  8%]
............................................................................................................................................................... [ 81%]
.......................................s                                                                                                                        [100%]

==================================================== 216 passed, 1 skipped, 5700 deselected, 14 warnings in 59.58s ====================================================
```

---

I suggest documenting this dependence in the documentation.

Regarding the tests, the concerned tests should probably be skipped if there is a dependency missing? (Just like they already are if cython is missing.)  
In case they are not skipped, the reason for the failure should somehow become more clear.

Regarding the actual behavior, qutip could print a warning if `cython` is installed and `filelock` is not, but I am not sure whether that would be a good idea.

---

For completeness, my `qutip.about()` output:

```
QuTiP Version:      5.0.0.dev0+7892453
Numpy Version:      1.24.3
Scipy Version:      1.10.1
Cython Version:     0.29.33
Matplotlib Version: 3.7.1
Python Version:     3.11.3
Number of CPUs:     16
BLAS Info:          INTEL MKL
INTEL MKL Ext:      True
Platform Info:      Linux (x86_64)
Installation path:  /home/paul/miniconda3/envs/qutip-tmp/lib/python3.11/site-packages/qutip-5.0.0.dev0+7892453-py3.11-linux-x86_64.egg/qutip
```


Discussion:

Resolving patch:
diff --git a/qutip/core/coefficient.py b/qutip/core/coefficient.py
--- a/qutip/core/coefficient.py
+++ b/qutip/core/coefficient.py
@@ -199,6 +199,9 @@ def const(value):
 # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 # %%%%%%%%%      Everything under this is for string compilation      %%%%%%%%%
 # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+WARN_MISSING_MODULE = [0]
+
+
 class CompilationOptions(QutipOptions):
     """
     Compilation options:
@@ -259,6 +262,7 @@ class CompilationOptions(QutipOptions):
         _use_cython = True
     except ImportError:
         _use_cython = False
+        WARN_MISSING_MODULE[0] = 1
 
     _options = {
         "use_cython": _use_cython,
@@ -369,6 +373,12 @@ def coeff_from_str(base, args, args_ctypes, compile_opt=None, **_):
     coeff = None
     # Do we compile?
     if not compile_opt['use_cython']:
+        if WARN_MISSING_MODULE[0]:
+            warnings.warn(
+                "Both `cython` and `filelock` are required for compilation of "
+                "string coefficents. Falling back on `eval`.")
+            # Only warns once.
+            WARN_MISSING_MODULE[0] = 0
         return StrFunctionCoefficient(base, args)
     # Parsing tries to make the code in common pattern
     parsed, variables, constants, raw = try_parse(base, args,


Test patch:
diff --git a/qutip/tests/conftest.py b/qutip/tests/conftest.py
--- a/qutip/tests/conftest.py
+++ b/qutip/tests/conftest.py
@@ -28,6 +28,7 @@ def _skip_cython_tests_if_unavailable(item):
         # importorskip rather than mark.skipif because this way we get pytest's
         # version-handling semantics.
         pytest.importorskip('Cython', minversion='0.14')
+        pytest.importorskip('filelock')
 
 
 @pytest.hookimpl(trylast=True)
diff --git a/qutip/tests/core/test_coefficient.py b/qutip/tests/core/test_coefficient.py
--- a/qutip/tests/core/test_coefficient.py
+++ b/qutip/tests/core/test_coefficient.py
@@ -6,9 +6,11 @@
 from functools import partial
 from qutip.core.coefficient import (coefficient, norm, conj, const,
                                     CompilationOptions, Coefficient,
-                                    clean_compiled_coefficient
+                                    clean_compiled_coefficient,
+                                    WARN_MISSING_MODULE,
                                     )
 
+
 # Ensure the latest version is tested
 clean_compiled_coefficient(True)
 
@@ -239,6 +241,14 @@ def test_CoeffOptions():
         assert not isinstance(coeff1, coeff2.__class__)
 
 
+def test_warn_no_cython():
+    option = CompilationOptions(use_cython=False)
+    WARN_MISSING_MODULE[0] = 1
+    with pytest.warns(
+        UserWarning, match="`cython` and `filelock` are required"
+    ):
+        coefficient("t", compile_opt=option)
+
 @pytest.mark.requires_cython
 @pytest.mark.parametrize(['codestring', 'args', 'reference'], [
     pytest.param("cos(2*t)*cos(t*w1) + sin(w1*w2/2*t)*sin(t*w2)"
diff --git a/qutip/tests/solver/test_integrator.py b/qutip/tests/solver/test_integrator.py
--- a/qutip/tests/solver/test_integrator.py
+++ b/qutip/tests/solver/test_integrator.py
@@ -8,6 +8,10 @@
 from numpy.testing import assert_allclose
 import pytest
 
+# Deactivate warning for test without cython
+from qutip.core.coefficient import WARN_MISSING_MODULE
+WARN_MISSING_MODULE[0] = 0
+
 
 class TestIntegratorCte():
     _analytical_se = lambda _, t: np.cos(t * np.pi)
diff --git a/qutip/tests/solver/test_mesolve.py b/qutip/tests/solver/test_mesolve.py
--- a/qutip/tests/solver/test_mesolve.py
+++ b/qutip/tests/solver/test_mesolve.py
@@ -7,6 +7,11 @@
 import pickle
 import pytest
 
+# Deactivate warning for test without cython
+from qutip.core.coefficient import WARN_MISSING_MODULE
+WARN_MISSING_MODULE[0] = 0
+
+
 all_ode_method = [
     method for method, integrator in MESolver.avail_integrators().items()
     if integrator.support_time_dependant
diff --git a/qutip/tests/solver/test_sesolve.py b/qutip/tests/solver/test_sesolve.py
--- a/qutip/tests/solver/test_sesolve.py
+++ b/qutip/tests/solver/test_sesolve.py
@@ -6,6 +6,11 @@
 from qutip.solver.krylovsolve import krylovsolve
 from qutip.solver.solver_base import Solver
 
+# Deactivate warning for test without cython
+from qutip.core.coefficient import WARN_MISSING_MODULE
+WARN_MISSING_MODULE[0] = 0
+
+
 all_ode_method = [
     method for method, integrator in SESolver.avail_integrators().items()
     if integrator.support_time_dependant