Repository: alan-turing-institute/mogp-emulator (language: python)

Issue:
Fitting tests break due to some change in Numpy/Scipy
On my macbook, I find that the fitting tests seem to have broken (though they seem to be fine on Travis at the moment). This seems to be due to some random number behavior that I can't really figure out (it seems to have cropped up in another project around the same time), as my tests for the fitting routine just confirm that the fitting runs to completion and returns expected values. I ran into some similar trouble with tests running in series/parallel, and fixed it just by making the tests run in series.

One possible fix, that I think is probably the "correct" method from a unit testing standpoint, is to mock the Scipy minimization routine such that it always returns the same set of hyperparameters independent of the starting point. This will remove any future dependence on such changes in random number generation, and will still test the code to the same level of scrutiny as is currently implemented, and in some ways be more robust as the routine won't ever fail (as it occasionally does). I think this is pretty straightforward using the `mock.patch` decorator provided with Python.


Discussion / hints:

Fix patch:
diff --git a/setup.py b/setup.py
index 0e7dbd82..f3039e86 100644
--- a/setup.py
+++ b/setup.py
@@ -4,7 +4,7 @@
 MAJOR = 0
 MINOR = 5
 MICRO = 0
-PRERELEASE = 1
+PRERELEASE = 2
 ISRELEASED = False
 version = "{}.{}.{}".format(MAJOR, MINOR, MICRO)
 


Test patch:
diff --git a/mogp_emulator/tests/test_fitting.py b/mogp_emulator/tests/test_fitting.py
index 34239e8f..8af1df0e 100644
--- a/mogp_emulator/tests/test_fitting.py
+++ b/mogp_emulator/tests/test_fitting.py
@@ -5,9 +5,15 @@
 from ..MultiOutputGP import MultiOutputGP
 from ..fitting import fit_GP_MAP, _fit_single_GP_MAP, _fit_MOGP_MAP
 
-def test_fit_GP_MAP():
+def minimize_mock(fun, x0, args=(), method=None, jac=None, hess=None, hessp=None, bounds=None, constraints=(), tol=None, callback=None, options=None):
+    return {"x": np.array([1.6, -2.1, -0.8]),
+            "fun": 0.}
+
+def test_fit_GP_MAP(monkeypatch):
     "test the fit_GP_MAP function"
 
+    monkeypatch.setattr("mogp_emulator.fitting.minimize", minimize_mock)
+    
     # test correct basic functioning
 
     x = np.linspace(0., 1.)
@@ -15,9 +21,8 @@ def test_fit_GP_MAP():
 
     gp = GaussianProcess(x, y)
 
-    theta_exp = np.array([ 1.6030532031342832, -2.090511425471982 , -0.7803960307137198])
-    logpost_exp = -296.0297245831661
-    np.random.seed(4335)
+    theta_exp = np.array([ 1.6, -2.1 , -0.8])
+    logpost_exp = gp.logposterior(theta_exp)
 
     gp = fit_GP_MAP(gp)
 
@@ -27,13 +32,20 @@ def test_fit_GP_MAP():
 
     # same test, but pass args and kwargs rather than gp
 
-    np.random.seed(4335)
-
     gp = fit_GP_MAP(x, y, mean="0.", use_patsy=False, method="L-BFGS-B")
     assert isinstance(gp, GaussianProcess)
     assert_allclose(gp.theta, theta_exp)
     assert_allclose(gp.current_logpost, logpost_exp)
 
+
+def test_fit_GP_MAP_failures():
+    "test failures of fit_GP_MAP"
+
+    x = np.linspace(0., 1.)
+    y = x**2
+
+    gp = GaussianProcess(x, y)
+    
     # minimization fails
 
     with pytest.raises(RuntimeError):
@@ -72,35 +84,21 @@ def test_fit_GP_MAP_MOGP():
 
     gp = MultiOutputGP(x, y)
 
-    theta_exp = np.zeros((2, 3))
-    theta_exp[0] = np.array([ 1.6030532031342832, -2.090511425471982 , -0.7803960307137198])
-    theta_exp[1] = np.array([ 1.414112951818647 , -0.5974688573393576,  0.6857536842773265])
-    logpost_exp = np.zeros(2)
-    logpost_exp[0] = -296.0297245831661
-    logpost_exp[1] = -250.06025683867367
     np.random.seed(4335)
 
-    gp = fit_GP_MAP(gp, processes=1)
+    gp = fit_GP_MAP(gp)
 
     assert isinstance(gp, MultiOutputGP)
-    for i in range(2):
-        assert_allclose(gp.emulators[i].theta, theta_exp[i])
-        assert_allclose(gp.emulators[i].current_logpost, logpost_exp[i])
-
-    # same test, but pass args and kwargs rather than gp
 
     np.random.seed(4335)
+    
+    # same test, but pass args and kwargs rather than gp
 
     gp = fit_GP_MAP(x, y, mean="0.", use_patsy=False, method="L-BFGS-B", processes=1)
     assert isinstance(gp, MultiOutputGP)
-    for i in range(2):
-        assert_allclose(gp.emulators[i].theta, theta_exp[i])
-        assert_allclose(gp.emulators[i].current_logpost, logpost_exp[i])
 
     # pass processes argument
 
-    np.random.seed(4335)
-
     gp = fit_GP_MAP(x, y, mean="0.", use_patsy=False, method="L-BFGS-B", processes=1)
     assert isinstance(gp, MultiOutputGP)
 
@@ -112,6 +110,17 @@ def test_fit_GP_MAP_MOGP():
 
     gp = fit_GP_MAP(x, y, theta0=[None, np.zeros(3)])
 
+
+def test_fit_GP_MAP_MOGP_failures():
+    "test situations where mogp fitting should fail"
+    
+    x = np.linspace(0., 1.)
+    y = np.zeros((2, 50))
+    y[0] = x**2
+    y[1] = 2. + x**3
+
+    gp = MultiOutputGP(x, y)
+    
     # minimization fails
 
     with pytest.raises(RuntimeError):
@@ -154,17 +163,18 @@ def test_fit_GP_MAP_MOGP():
     with pytest.raises(AssertionError):
         fit_GP_MAP(gp, theta0=[np.zeros(1), np.zeros(3)], processes=1)
 
-def test_fit_single_GP_MAP():
+def test_fit_single_GP_MAP(monkeypatch):
     "test the method to run the minimization algorithm on a GP class"
 
+    monkeypatch.setattr("mogp_emulator.fitting.minimize", minimize_mock)
+    
     x = np.linspace(0., 1.)
     y = x**2
 
     gp = GaussianProcess(x, y)
 
-    theta_exp = np.array([ 1.6030532031342832, -2.090511425471982 , -0.7803960307137198])
-    logpost_exp = -296.0297245831661
-    np.random.seed(4335)
+    theta_exp = np.array([ 1.6, -2.1 , -0.8])
+    logpost_exp = gp.logposterior(theta_exp)
 
     gp = _fit_single_GP_MAP(gp)
 
@@ -172,6 +182,15 @@ def test_fit_single_GP_MAP():
     assert_allclose(gp.theta, theta_exp)
     assert_allclose(gp.current_logpost, logpost_exp)
 
+
+def test_fit_single_GP_MAP_failures():
+    "test situation where fitting one emulator should fail"
+    
+    x = np.linspace(0., 1.)
+    y = x**2
+
+    gp = GaussianProcess(x, y)
+
     # minimization fails
 
     with pytest.raises(RuntimeError):
@@ -196,7 +215,7 @@ def test_fit_single_GP_MAP():
     with pytest.raises(AssertionError):
         fit_GP_MAP(gp, theta0=np.ones(1))
 
-def test_fit_MOGP_MAP_MOGP():
+def test_fit_MOGP_MAP():
     "test the fit_GP_MAP function with multiple outputs"
 
     x = np.linspace(0., 1.)
@@ -206,20 +225,11 @@ def test_fit_MOGP_MAP_MOGP():
 
     gp = MultiOutputGP(x, y)
 
-    theta_exp = np.zeros((2, 3))
-    theta_exp[0] = np.array([ 1.6030532031342832, -2.090511425471982 , -0.7803960307137198])
-    theta_exp[1] = np.array([ 1.414112951818647 , -0.5974688573393576,  0.6857536842773265])
-    logpost_exp = np.zeros(2)
-    logpost_exp[0] = -296.0297245831661
-    logpost_exp[1] = -250.06025683867367
     np.random.seed(4335)
 
-    gp = _fit_MOGP_MAP(gp, processes=1)
+    gp = _fit_MOGP_MAP(gp)
 
     assert isinstance(gp, MultiOutputGP)
-    for i in range(2):
-        assert_allclose(gp.emulators[i].theta, theta_exp[i])
-        assert_allclose(gp.emulators[i].current_logpost, logpost_exp[i])
 
     # pass processes argument
 
@@ -236,6 +246,17 @@ def test_fit_MOGP_MAP_MOGP():
 
     gp = fit_GP_MAP(gp, theta0=[None, np.zeros(3)])
 
+
+def test_fit_MOGP_MAP_failures():
+    "test situations where fitting should fail"
+
+    x = np.linspace(0., 1.)
+    y = np.zeros((2, 50))
+    y[0] = x**2
+    y[1] = 2. + x**3
+
+    gp = MultiOutputGP(x, y)
+    
     # minimization fails
 
     with pytest.raises(RuntimeError):
@@ -276,4 +297,4 @@ def test_fit_MOGP_MAP_MOGP():
         _fit_MOGP_MAP(gp, theta0=[np.zeros(1), None], processes=1)
 
     with pytest.raises(AssertionError):
-        _fit_MOGP_MAP(gp, processes=-1)
\ No newline at end of file
+        _fit_MOGP_MAP(gp, processes=-1)