Repository: GPflow/GPflow

Issue:
autoflow fails after reset_default_graph
Sometimes it's necessary to reseat tensorflow's graph for memory reasons. This can be done with `tf.reset_default_graph()`. 

Problem is, this breaks all out AutoFlow functions! Here's a MWE:

``` python
tf.reset_default_graph()
k = GPflow.kernels.Matern32(1)
X, Y = np.random.randn(2, 10, 1)
m = GPflow.gpr.GPR(X, Y, kern=k)
m.predict_f(m.X)
```

This is because the Autoflow constructor takes placeholders as arguments. After reset_default_graph, those placeholders are not on the same graph as everything else.  I'm going to propose a fix where we just give AutoFlow enough information to _make_ the placeholders, not the placeholders themselves. 



Discussion:
Could this be related to the following bug? If I recompile a model, suddenly the predictions go out the window!

```
import numpy as np
import numpy.random as rnd
import matplotlib.pyplot as plt
import GPflow

X = rnd.rand(100, 1)
Y = np.sin(X*4) + rnd.randn(100, 1)*0.1

# mod = GPflow.sgpr.GPRFITC(X, Y, GPflow.kernels.RBF(1), rnd.rand(10, 1))
mod = GPflow.sgpr.SGPR(X, Y, GPflow.kernels.RBF(1), rnd.rand(10, 1))
mod.optimize(max_iters=100)

pX = np.linspace(0, 1, 100)[:, None]
pY1, pYv1 = mod.predict_y(pX)

mod._compile()
pY2, pYv2 = mod.predict_y(pX)

plt.plot(X, Y, 'x')
plt.plot(pX, pY1)
plt.plot(pX, pY2)
```

Oh dear, what horrible behaviour. 

You'll be pleased to know that it seems to be fixed by #62 (try git checkout autoflow_graph_bug_bugfix)

I've not merged #62 yet because the tests timeout on the codeship server. I could really do with some help making the tests run faster. 

Do you have any pointers as to where speedups could be made? And how much faster are we talking?

I have a hypothesis: using an InteractiveSession() might help.

If that worked, I'd be really happy. If not, we might have to either look
at merging some tests (Alex has done this a bit in the past) or just
running the tests on a bigger machine, or in parallel. eek.

On Tue, 24 May 2016 at 10:55 Mark van der Wilk notifications@github.com
wrote:

> Do you have any pointers as to where speedups could be made?
> 
> —
> You are receiving this because you authored the thread.
> Reply to this email directly or view it on GitHub
> https://github.com/GPflow/GPflow/issues/61#issuecomment-221221672

@markvdw I'm talking to @jameshensman about the slow tests on #67 

@markvd @jameshensman I think that the problem might be that _compile() calls tf.initialize_all_variables() which will set all the variables back to their pre-optimized values. 

This refers specifically to Mark's code snippet. 

I don't think that an InteractiveSession will help. Effectively that just sets up the one session object which is used by default and automatically under the hood.


Resolving patch:
diff --git a/GPflow/model.py b/GPflow/model.py
--- a/GPflow/model.py
+++ b/GPflow/model.py
@@ -39,8 +39,7 @@ class AutoFlow:
     The syntax looks like:
 
     >>> class MyModel(Model):
-    >>>
-    >>>   @AutoFlow(tf.placeholder(tf.float64), tf.placeholder(tf.float64))
+    >>>   @AutoFlow((tf.float64), (tf.float64))
     >>>   def my_method(self, x, y):
     >>>       #compute something, returning a tf graph.
     >>>       return tf.foo(self.baz, x, y)
@@ -65,17 +64,21 @@ class AutoFlow:
     result in the graph being constructed only once.
 
     """
-    def __init__(self, *tf_args):
-        self.tf_args = tf_args
+    def __init__(self, *tf_arg_tuples):
+        # NB. TF arg_tuples is a list of tuples, each of which can be used to
+        # construct a tf placeholder.
+        self.tf_arg_tuples = tf_arg_tuples
+
     def __call__(self, tf_method):
         @wraps(tf_method)
         def runnable(instance, *np_args):
             graph_name = '_' + tf_method.__name__ + '_graph'
             if not hasattr(instance, graph_name):
+                instance._compile()
+                self.tf_args = [tf.placeholder(*a) for a in self.tf_arg_tuples]
                 with instance.tf_mode():
-                    instance.make_tf_array(instance._free_vars)
                     graph = tf_method(instance, *self.tf_args)
-                    setattr(instance, graph_name, graph)
+                setattr(instance, graph_name, graph)
             feed_dict = dict(zip(self.tf_args, np_args))
             feed_dict[instance._free_vars] = instance.get_free_state()
             graph = getattr(instance, graph_name)
@@ -83,7 +86,6 @@ def runnable(instance, *np_args):
         return runnable
 
 
-
 class Model(Parameterized):
     """
     The Model base class.
@@ -183,7 +185,7 @@ def obj(x):
 
     def __setattr__(self, key, value):
         Parameterized.__setattr__(self, key, value)
-        #delete any AutoFlow related graphs
+        # delete any AutoFlow related graphs
         if key=='_needs_recompile' and value:
             for key in filter(lambda x : x[0]=='_' and x[-6:]=='_graph', dir(self)):
                 delattr(self, key)
@@ -350,14 +352,14 @@ def __init__(self, X, Y, kern, likelihood, mean_function, name='model'):
     def build_predict(self):
         raise NotImplementedError
 
-    @AutoFlow(tf.placeholder(tf.float64, [None, None]))
+    @AutoFlow((tf.float64, [None, None]))
     def predict_f(self, Xnew):
         """
         Compute the mean and variance of the latent function(s) at the points Xnew
         """
         return self.build_predict(Xnew)
 
-    @AutoFlow(tf.placeholder(tf.float64, [None, None]))
+    @AutoFlow((tf.float64, [None, None]))
     def predict_f_full_cov(self, Xnew):
         """
         Compute the mean and covariance matrix of the latent function(s) at the
@@ -365,7 +367,7 @@ def predict_f_full_cov(self, Xnew):
         """
         return self.build_predict(Xnew, full_cov=True)
 
-    @AutoFlow(tf.placeholder(tf.float64), tf.placeholder(tf.int32, []))
+    @AutoFlow((tf.float64, [None, None]), (tf.int32, []))
     def predict_f_samples(self, Xnew, num_samples):
         """
         Produce samples from the posterior latent function(s) at the points
@@ -378,7 +380,7 @@ def predict_f_samples(self, Xnew, num_samples):
             samples.append(mu[:,i:i+1] + tf.matmul(L, tf.random_normal(tf.pack([tf.shape(L)[0], num_samples]), dtype=tf.float64)))
         return tf.transpose(tf.pack(samples))
 
-    @AutoFlow(tf.placeholder(tf.float64, [None, None]))
+    @AutoFlow((tf.float64, [None, None]))
     def predict_y(self, Xnew):
         """
         Compute the mean and variance of held-out data at the points Xnew
@@ -386,8 +388,7 @@ def predict_y(self, Xnew):
         pred_f_mean, pred_f_var = self.build_predict(Xnew)
         return self.likelihood.predict_mean_and_var(pred_f_mean, pred_f_var)
 
-
-    @AutoFlow(tf.placeholder(tf.float64, [None, None]), tf.placeholder(tf.float64, [None, None]))
+    @AutoFlow((tf.float64, [None, None]), (tf.float64, [None, None]))
     def predict_density(self, Xnew, Ynew):
         """
         Compute the (log) density of the data Ynew at the points Xnew


Test patch:
diff --git a/testing/test_autoflow.py b/testing/test_autoflow.py
--- a/testing/test_autoflow.py
+++ b/testing/test_autoflow.py
@@ -3,6 +3,7 @@
 import numpy as np
 import unittest
 
+
 class DumbModel(GPflow.model.Model):
     def __init__(self):
         GPflow.model.Model.__init__(self)
@@ -11,55 +12,75 @@ def __init__(self):
     def build_likelihood(self):
         return -tf.square(self.a)
 
+
 class NoArgsModel(DumbModel):
 
     @GPflow.model.AutoFlow()
     def function(self):
         return self.a
 
+
 class TestNoArgs(unittest.TestCase):
     def setUp(self):
+        tf.reset_default_graph()
         self.m = NoArgsModel()
         self.m._compile()
+
     def test_return(self):
         self.failUnless(np.allclose(self.m.function(), 3.))
 
 
-x_tf = tf.placeholder(tf.float64)
-y_tf = tf.placeholder(tf.float64)
 class AddModel(DumbModel):
-    @GPflow.model.AutoFlow(x_tf, y_tf)
+    @GPflow.model.AutoFlow((tf.float64,), (tf.float64,))
     def add(self, x, y):
         return tf.add(x, y)
 
+
 class TestAdd(unittest.TestCase):
     def setUp(self):
+        tf.reset_default_graph()
         self.m = AddModel()
         self.m._compile()
         rng = np.random.RandomState(0)
-        self.x = rng.randn(10,20)
-        self.y = rng.randn(10,20)
+        self.x = rng.randn(10, 20)
+        self.y = rng.randn(10, 20)
+
     def test_add(self):
         self.failUnless(np.allclose(self.x + self.y, self.m.add(self.x, self.y)))
 
 
 class TestGPmodel(unittest.TestCase):
     def setUp(self):
+        tf.reset_default_graph()
         rng = np.random.RandomState(0)
         X, Y = rng.randn(2, 10, 1)
         self.m = GPflow.svgp.SVGP(X, Y, kern=GPflow.kernels.Matern32(1), likelihood=GPflow.likelihoods.StudentT(), Z=X[::2].copy())
-        self.Xtest = np.random.randn(100,1)
-        self.Ytest = np.random.randn(100,1)
+        self.Xtest = np.random.randn(100, 1)
+        self.Ytest = np.random.randn(100, 1)
+
     def test_predict_f(self):
         mu, var = self.m.predict_f(self.Xtest)
+
     def test_predict_y(self):
         mu, var = self.m.predict_y(self.Xtest)
+
     def test_predict_density(self):
         d = self.m.predict_density(self.Xtest, self.Ytest)
 
-    
+
+class TestResetGraph(unittest.TestCase):
+    def setUp(self):
+        tf.reset_default_graph()
+        k = GPflow.kernels.Matern32(1)
+        X, Y = np.random.randn(2, 10, 1)
+        self.Xnew = np.random.randn(5, 1)
+        self.m = GPflow.gpr.GPR(X, Y, kern=k)
+
+    def test(self):
+        mu, var = self.m.predict_f(self.Xnew)
+        tf.reset_default_graph()
+        mu1, var1 = self.m.predict_f(self.Xnew)
 
 
 if __name__ == "__main__":
     unittest.main()
-
diff --git a/testing/test_conditionals.py b/testing/test_conditionals.py
--- a/testing/test_conditionals.py
+++ b/testing/test_conditionals.py
@@ -12,6 +12,7 @@ class DiagsTest(unittest.TestCase):
     Here we make sure the behaviours overlap.
     """
     def setUp(self):
+        tf.reset_default_graph()
         self.num_latent = 2
         self.k = GPflow.kernels.Matern32(1) + GPflow.kernels.White(1)
         self.k.white.variance = 0.01
@@ -70,6 +71,7 @@ def test_nonwhiten(self):
 
 class WhitenTest(unittest.TestCase):
     def setUp(self):
+        tf.reset_default_graph()
         self.k = GPflow.kernels.Matern32(1) + GPflow.kernels.White(1)
         self.k.white.variance = 0.01
         self.num_data = 10
@@ -118,6 +120,7 @@ def test_whiten(self):
 
 class WhitenTestGaussian(WhitenTest):
     def setUp(self):
+        tf.reset_default_graph()
         WhitenTest.setUp(self)
         self.F_sqrt = tf.placeholder('float64', [self.num_data, 1])
         self.F_sqrt_data = self.rng.rand(self.num_data,1)
diff --git a/testing/test_hmc.py b/testing/test_hmc.py
--- a/testing/test_hmc.py
+++ b/testing/test_hmc.py
@@ -5,6 +5,7 @@
 
 class SampleGaussianTest(unittest.TestCase):
     def setUp(self):
+        tf.reset_default_graph()
         self.f = lambda x : (0.5*np.sum(np.square(x)), x)
         self.x0 = np.zeros(3)
 
@@ -45,6 +46,7 @@ class SampleModelTest(unittest.TestCase):
     Create a very simple model and make sure samples form is make sense. 
     """
     def setUp(self):
+        tf.reset_default_graph()
         rng = np.random.RandomState(0)
         class Quadratic(GPflow.model.Model):
             def __init__(self):
diff --git a/testing/test_kerns.py b/testing/test_kerns.py
--- a/testing/test_kerns.py
+++ b/testing/test_kerns.py
@@ -27,6 +27,7 @@ def test_1d(self):
 
 class TestKernSymmetry(unittest.TestCase):
     def setUp(self):
+        tf.reset_default_graph()
         self.kernels = GPflow.kernels.Stationary.__subclasses__() + [GPflow.kernels.Constant, GPflow.kernels.Linear]
         self.rng = np.random.RandomState()
 
@@ -60,6 +61,7 @@ def test_5d(self):
 
 class TestKernDiags(unittest.TestCase):
     def setUp(self):
+        tf.reset_default_graph()
         inputdim = 3
         rng = np.random.RandomState(1)
         self.X = tf.placeholder(tf.float64, [30, inputdim])
@@ -88,6 +90,7 @@ class TestAdd(unittest.TestCase):
     the result of the kernels separaetely
     """
     def setUp(self):
+        tf.reset_default_graph()
         self.rbf = GPflow.kernels.RBF(1)
         self.lin = GPflow.kernels.Linear(1)
         self.k = GPflow.kernels.RBF(1) + GPflow.kernels.Linear(1)
@@ -124,6 +127,7 @@ class TestWhite(unittest.TestCase):
     k(X, X)
     """
     def setUp(self):
+        tf.reset_default_graph()
         self.k = GPflow.kernels.White(1)
         self.rng = np.random.RandomState(0)
 
@@ -145,6 +149,7 @@ class TestSlice(unittest.TestCase):
     with correctly sliced data...
     """
     def setUp(self):
+        tf.reset_default_graph()
         self.rng = np.random.RandomState(0)
         self.k1 = GPflow.kernels.RBF(1, active_dims=[0])
         self.k2 = GPflow.kernels.RBF(1, active_dims=[1])
@@ -191,6 +196,7 @@ def test_asymm(self):
 
 class TestProd(unittest.TestCase):
     def setUp(self):
+        tf.reset_default_graph()
         self.k1 = GPflow.kernels.Matern32(2)
         self.k2 = GPflow.kernels.Matern52(2, lengthscales=0.3)
         self.k3 = self.k1 * self.k2
@@ -219,6 +225,7 @@ def test_prod(self):
 
 class TestARDActiveProd(unittest.TestCase):
     def setUp(self):
+        tf.reset_default_graph()
         self.rng = np.random.RandomState(0)
 
         # k3 = k1 * k2
diff --git a/testing/test_likelihoods.py b/testing/test_likelihoods.py
--- a/testing/test_likelihoods.py
+++ b/testing/test_likelihoods.py
@@ -11,6 +11,7 @@ class TestPredictConditional(unittest.TestCase):
     has no uncertainty.
     """
     def setUp(self):
+        tf.reset_default_graph()
         self.liks = [L() for L in GPflow.likelihoods.Likelihood.__subclasses__()]
 
         #some likelihoods are additionally tested with non-standard links
@@ -63,6 +64,7 @@ class TestQuadrature(unittest.TestCase):
      does something close to the quadrature
     """
     def setUp(self):
+        tf.reset_default_graph()
 
         self.rng = np.random.RandomState()
         self.Fmu, self.Fvar, self.Y = self.rng.randn(3, 10, 2)
diff --git a/testing/test_mean_functions.py b/testing/test_mean_functions.py
--- a/testing/test_mean_functions.py
+++ b/testing/test_mean_functions.py
@@ -10,6 +10,7 @@ class TestMeanFuncs(unittest.TestCase):
     check that the combination of mean functions returns the correct clas
     """
     def setUp(self):
+        tf.reset_default_graph()
         self.input_dim = 3
         self.output_dim = 2
         self.N = 20
@@ -58,6 +59,7 @@ class TestModelCompositionOperations(unittest.TestCase):
     change the mean function
     """
     def setUp(self):
+        tf.reset_default_graph()
         self.input_dim = 3
         self.output_dim = 2
         self.N = 20
@@ -164,6 +166,7 @@ class TestModelsWithMeanFuncs(unittest.TestCase):
     """
 
     def setUp(self):
+        tf.reset_default_graph()
         self.input_dim = 3
         self.output_dim = 2
         self.N = 20
diff --git a/testing/test_method_equivalence.py b/testing/test_method_equivalence.py
--- a/testing/test_method_equivalence.py
+++ b/testing/test_method_equivalence.py
@@ -2,6 +2,7 @@
 import GPflow
 import numpy as np
 import unittest
+import tensorflow as tf
 
 class TestEquivalence(unittest.TestCase):
     """
@@ -18,6 +19,7 @@ class TestEquivalence(unittest.TestCase):
     5) Sparse variational GP Regression (as above, but there the inducing variables are 'collapsed' out, as in Titsias 2009)
     """
     def setUp(self):
+        tf.reset_default_graph()
         rng = np.random.RandomState(0)
         X = rng.rand(20,1)*10
         Y = np.sin(X) + 0.9 * np.cos(X*1.6) + rng.randn(*X.shape)* 0.8
diff --git a/testing/test_methods.py b/testing/test_methods.py
--- a/testing/test_methods.py
+++ b/testing/test_methods.py
@@ -1,74 +1,84 @@
 import GPflow
-import tensorflow as tf
 import numpy as np
 import unittest
+import tensorflow as tf
 
 
 class TestMethods(unittest.TestCase):
     def setUp(self):
+        tf.reset_default_graph()
         self.rng = np.random.RandomState(0)
-        self.X = self.rng.randn(100,2)
-        self.Y = self.rng.randn(100,1)
-        self.Z = self.rng.randn(10,2)
+        self.X = self.rng.randn(100, 2)
+        self.Y = self.rng.randn(100, 1)
+        self.Z = self.rng.randn(10, 2)
         self.lik = GPflow.likelihoods.Gaussian()
         self.kern = GPflow.kernels.Matern32(2)
-        self.Xs = self.rng.randn(10,2)
+        self.Xs = self.rng.randn(10, 2)
 
-        #make one of each model
+        # make one of each model
         self.ms = []
+        #for M in (GPflow.gpmc.GPMC, GPflow.vgp.VGP):
         for M in (GPflow.vgp.VGP, GPflow.gpmc.GPMC):
-            self.ms.append( M(self.X, self.Y, self.kern, self.lik) )
+            self.ms.append(M(self.X, self.Y, self.kern, self.lik))
         for M in (GPflow.sgpmc.SGPMC, GPflow.svgp.SVGP):
-            self.ms.append( M(self.X, self.Y, self.kern, self.lik, self.Z) )
+            self.ms.append(M(self.X, self.Y, self.kern, self.lik, self.Z))
         self.ms.append(GPflow.gpr.GPR(self.X, self.Y, self.kern))
         self.ms.append(GPflow.sgpr.SGPR(self.X, self.Y, self.kern, Z=self.Z))
         self.ms.append(GPflow.sgpr.GPRFITC(self.X, self.Y, self.kern, Z=self.Z))
 
     def test_all(self):
-        #test sizes.
+        # test sizes.
         for m in self.ms:
             m._compile()
-            f,g = m._objective(m.get_free_state())
+            f, g = m._objective(m.get_free_state())
             self.failUnless(f.size == 1)
             self.failUnless(g.size == m.get_free_state().size)
-        #test predict f.        
+
+    def test_predict_f(self):    
         for m in self.ms:
             mf, vf = m.predict_f(self.Xs)
             self.failUnless(mf.shape == vf.shape)
             self.failUnless(mf.shape == (10, 1))
             self.failUnless(np.all(vf >= 0.0))
-        # test predict y.
+
+    def test_predict_y(self):    
         for m in self.ms:
             mf, vf = m.predict_y(self.Xs)
             self.failUnless(mf.shape == vf.shape)
             self.failUnless(mf.shape == (10, 1))
             self.failUnless(np.all(vf >= 0.0))
 
-        #test predict density.
-        self.Ys = self.rng.randn(10,1)
+    def test_predict_density(self):
+        self.Ys = self.rng.randn(10, 1)
         for m in self.ms:
             d = m.predict_density(self.Xs, self.Ys)
             self.failUnless(d.shape == (10, 1))
 
 
-
 class TestSVGP(unittest.TestCase):
     """
-    The SVGP has four modes of operation. with and without whitening, with and without diagonals.
+    The SVGP has four modes of operation. with and without whitening, with and
+    without diagonals.
 
-    Here we make sure thet the bound on the likelihood is the same when using both representations (as far as possible)
+    Here we make sure thet the bound on the likelihood is the same when using
+    both representations (as far as possible)
     """
     def setUp(self):
+        tf.reset_default_graph()
         self.rng = np.random.RandomState(0)
         self.X = self.rng.randn(20, 1)
-        self.Y = self.rng.randn(20,2)
-        self.Z = self.rng.randn(3,1)
+        self.Y = self.rng.randn(20, 2)
+        self.Z = self.rng.randn(3, 1)
 
     def test_white(self):
-        m1 = GPflow.svgp.SVGP(self.X, self.Y, kern=GPflow.kernels.RBF(1)+GPflow.kernels.White(1), likelihood=GPflow.likelihoods.Exponential(),
-                           Z=self.Z, q_diag=True, whiten=True)
-        m2 = GPflow.svgp.SVGP(self.X, self.Y, kern=GPflow.kernels.RBF(1)+GPflow.kernels.White(1), likelihood=GPflow.likelihoods.Exponential(),
-                           Z=self.Z, q_diag=False, whiten=True)
+        m1 = GPflow.svgp.SVGP(self.X, self.Y,
+                              kern=GPflow.kernels.RBF(1),
+                              likelihood=GPflow.likelihoods.Exponential(),
+                              Z=self.Z, q_diag=True, whiten=True)
+        m2 = GPflow.svgp.SVGP(self.X, self.Y,
+                              kern=GPflow.kernels.RBF(1),
+                              likelihood=GPflow.likelihoods.Exponential(),
+                              Z=self.Z, q_diag=False, whiten=True)
         m1._compile()
         m2._compile()
 
@@ -76,14 +86,16 @@ def test_white(self):
         qsqrt = (qsqrt**2)*0.01
         m1.q_sqrt = qsqrt
         m1.q_mu = qmean
-        m2.q_sqrt = np.array([np.diag(qsqrt[:,0]), np.diag(qsqrt[:,1])]).swapaxes(0,2)
+        m2.q_sqrt = np.array([np.diag(qsqrt[:, 0]),
+                              np.diag(qsqrt[:, 1])]).swapaxes(0, 2)
         m2.q_mu = qmean
-        self.failUnless(np.allclose(m1._objective(m1.get_free_state())[0], m2._objective(m2.get_free_state())[0]))
+        self.failUnless(np.allclose(m1._objective(m1.get_free_state())[0],
+                                    m2._objective(m2.get_free_state())[0]))
 
     def test_notwhite(self):
         m1 = GPflow.svgp.SVGP(self.X,
                               self.Y,
-                              kern=GPflow.kernels.RBF(1) + \
+                              kern=GPflow.kernels.RBF(1) +
                                    GPflow.kernels.White(1),
                               likelihood=GPflow.likelihoods.Exponential(),
                               Z=self.Z,
@@ -91,7 +103,7 @@ def test_notwhite(self):
                               whiten=False)
         m2 = GPflow.svgp.SVGP(self.X,
                               self.Y,
-                              kern=GPflow.kernels.RBF(1) + \
+                              kern=GPflow.kernels.RBF(1) +
                                    GPflow.kernels.White(1),
                               likelihood=GPflow.likelihoods.Exponential(),
                               Z=self.Z,
@@ -104,9 +116,10 @@ def test_notwhite(self):
         qsqrt = (qsqrt**2)*0.01
         m1.q_sqrt = qsqrt
         m1.q_mu = qmean
-        m2.q_sqrt = np.array([np.diag(qsqrt[:,0]), np.diag(qsqrt[:,1])]).swapaxes(0,2)
+        m2.q_sqrt = np.array([np.diag(qsqrt[:, 0]), np.diag(qsqrt[:, 1])]).swapaxes(0, 2)
         m2.q_mu = qmean
-        self.failUnless(np.allclose(m1._objective(m1.get_free_state())[0], m2._objective(m2.get_free_state())[0]))
+        self.failUnless(np.allclose(m1._objective(m1.get_free_state())[0],
+                                    m2._objective(m2.get_free_state())[0]))
 
     def test_q_sqrt_fixing(self):
         """
@@ -120,18 +133,17 @@ def test_q_sqrt_fixing(self):
         m1._compile()
 
 
-
-
 class TestSparseMCMC(unittest.TestCase):
     """
     This test makes sure that when the inducing points are the same as the data
     points, the sparse mcmc is the same as full mcmc
     """
     def setUp(self):
+        tf.reset_default_graph()
         rng = np.random.RandomState(0)
-        X = rng.randn(10,1)
-        Y = rng.randn(10,1)
-        v_vals = rng.randn(10,1)
+        X = rng.randn(10, 1)
+        Y = rng.randn(10, 1)
+        v_vals = rng.randn(10, 1)
 
         l = GPflow.likelihoods.StudentT()
         self.m1 = GPflow.gpmc.GPMC(X=X, Y=Y, kern=GPflow.kernels.Exponential(1), likelihood=l)
@@ -151,7 +163,8 @@ def test_likelihoods_and_gradients(self):
         f1, _ = self.m1._objective(self.m1.get_free_state())
         f2, _ = self.m2._objective(self.m2.get_free_state())
         self.failUnless(np.allclose(f1, f2))
-        #the parameters might not be in the same order, so sort the gradients before checking they're the same
+        # the parameters might not be in the same order, so
+        # sort the gradients before checking they're the same
         _, g1 = self.m1._objective(self.m1.get_free_state())
         _, g2 = self.m2._objective(self.m2.get_free_state())
         g1 = np.sort(g1)
@@ -161,4 +174,3 @@ def test_likelihoods_and_gradients(self):
 
 if __name__ == "__main__":
     unittest.main()
-
diff --git a/testing/test_model.py b/testing/test_model.py
--- a/testing/test_model.py
+++ b/testing/test_model.py
@@ -7,6 +7,7 @@
 
 class TestOptimize(unittest.TestCase):
     def setUp(self):
+        tf.reset_default_graph()
         rng = np.random.RandomState(0)
         class Quadratic(GPflow.model.Model):
             def __init__(self):
@@ -40,6 +41,7 @@ def __call__(self, *a, **kw):
 
 class TestKeyboardCatching(unittest.TestCase):
     def setUp(self):
+        tf.reset_default_graph()
         X = np.random.randn(1000, 3)
         Y = np.random.randn(1000, 3)
         Z = np.random.randn(100, 3)
@@ -63,6 +65,7 @@ def test_optimize_tf(self):
 
 class TestLikelihoodAutoflow(unittest.TestCase):
     def setUp(self):
+        tf.reset_default_graph()
         X = np.random.randn(1000, 3)
         Y = np.random.randn(1000, 3)
         Z = np.random.randn(100, 3)
diff --git a/testing/test_param.py b/testing/test_param.py
--- a/testing/test_param.py
+++ b/testing/test_param.py
@@ -6,6 +6,7 @@
 
 class ParamTestsScalar(unittest.TestCase):
     def setUp(self):
+        tf.reset_default_graph()
         self.m = GPflow.param.Parameterized()
         self.m.p = GPflow.param.Param(1.0)
     
@@ -60,6 +61,7 @@ def testTheanoMode(self):
 
 class ParamTestsDeeper(unittest.TestCase):
     def setUp(self):
+        tf.reset_default_graph()
         self.m = GPflow.param.Parameterized()
         self.m.foo = GPflow.param.Parameterized()
         self.m.foo.bar = GPflow.param.Parameterized()
@@ -122,6 +124,7 @@ def testTheanoMode(self):
 
 class ParamTestswider(unittest.TestCase):
     def setUp(self):
+        tf.reset_default_graph()
         self.m = GPflow.param.Parameterized()
         self.m.foo = GPflow.param.Param(1.0)
         self.m.bar = GPflow.param.Param(np.arange(10))
diff --git a/testing/test_predict.py b/testing/test_predict.py
--- a/testing/test_predict.py
+++ b/testing/test_predict.py
@@ -1,21 +1,22 @@
 import GPflow
-import tensorflow as tf
 import numpy as np
 import unittest
+import tensorflow as tf
+
 
 class TestGaussian(unittest.TestCase):
     def setUp(self):
+        tf.reset_default_graph()
         self.rng = np.random.RandomState(0)
         self.X = self.rng.randn(100,2)
-        self.Y = self.rng.randn(100,1)
+        self.Y = self.rng.randn(100, 1)
         self.kern = GPflow.kernels.Matern32(2) + GPflow.kernels.White(1)
-        self.Xtest = self.rng.randn(10,2)
-        self.Ytest = self.rng.randn(10,1)
+        self.Xtest = self.rng.randn(10, 2)
+        self.Ytest = self.rng.randn(10, 1)
 
-        #make a Gaussian model
+        # make a Gaussian model
         self.m = GPflow.gpr.GPR(self.X, self.Y, kern=self.kern)
 
-
     def test_all(self):
         mu_f, var_f = self.m.predict_f(self.Xtest)
         mu_y, var_y = self.m.predict_y(self.Xtest)
@@ -46,45 +47,111 @@ def test_recompile(self):
 
 
 class TestFullCov(unittest.TestCase):
+    """ 
+    this base class requires inherriting to specify the model.
+
+    This test structure is more complex that, say, looping over the models, but
+    makses all the tests much smaller and so less prone to erroring out. Also,
+    if a test fails, it should be clearer where the error is. 
+    """
     def setUp(self):
-        self.input_dim=3
-        self.output_dim=2
-        self.N=20
-        self.Ntest=30
-        self.M=5
+        tf.reset_default_graph()
+        self.input_dim = 3
+        self.output_dim = 2
+        self.N = 20
+        self.Ntest = 30
+        self.M = 5
         rng = np.random.RandomState(0)
-        X, Y, Z, self.Xtest = rng.randn(self.N, self.input_dim),\
+        self.num_samples = 5
+        self.samples_shape = (self.num_samples, self.Ntest, self.output_dim)
+        self.covar_shape = (self.Ntest, self.Ntest, self.output_dim)
+        self.X, self.Y, self.Z, self.Xtest = rng.randn(self.N, self.input_dim),\
                               rng.randn(self.N, self.output_dim),\
                               rng.randn(self.M, self.input_dim),\
                               rng.randn(self.Ntest, self.input_dim)
-        k = lambda : GPflow.kernels.Matern32(self.input_dim)
-        self.models = [GPflow.gpr.GPR(X, Y, kern=k()),
-                  GPflow.sgpr.SGPR(X, Y, Z=Z, kern=k()),
-                  GPflow.sgpr.GPRFITC(X, Y, Z=Z, kern=k()),
-                  GPflow.svgp.SVGP(X, Y, Z=Z, kern=k(), likelihood=GPflow.likelihoods.Gaussian(), whiten=False, q_diag=True),
-                  GPflow.svgp.SVGP(X, Y, Z=Z, kern=k(), likelihood=GPflow.likelihoods.Gaussian(), whiten=True, q_diag=False),
-                  GPflow.svgp.SVGP(X, Y, Z=Z, kern=k(), likelihood=GPflow.likelihoods.Gaussian(), whiten=True, q_diag=True),
-                  GPflow.svgp.SVGP(X, Y, Z=Z, kern=k(), likelihood=GPflow.likelihoods.Gaussian(), whiten=True, q_diag=False),
-                  GPflow.vgp.VGP(X, Y, kern=k(), likelihood=GPflow.likelihoods.Gaussian()),
-                  GPflow.vgp.VGP(X, Y, kern=k(), likelihood=GPflow.likelihoods.Gaussian()),
-                  GPflow.gpmc.GPMC(X, Y, kern=k(), likelihood=GPflow.likelihoods.Gaussian()),
-                  GPflow.sgpmc.SGPMC(X, Y, kern=k(), likelihood=GPflow.likelihoods.Gaussian(), Z=Z)]
-
-    def test_cov_and_samples(self):
-        for m in self.models:
-            mu1, var = m.predict_f(self.Xtest)
-            mu2, covar = m.predict_f_full_cov(self.Xtest)
-            self.failUnless(np.all(mu1==mu2))
-            self.failUnless(covar.shape == (self.Ntest, self.Ntest, self.output_dim))
-            self.failUnless(var.shape == (self.Ntest, self.output_dim))
-            for i in range(self.output_dim):
-                self.failUnless(np.allclose(var[:,i] , np.diag(covar[:,:,i])))
-    
-        for m in self.models:
-            samples = m.predict_f_samples(self.Xtest, 5)
-            self.failUnless(samples.shape==(5, self.Xtest.shape[0], self.output_dim))
+        self.k = lambda: GPflow.kernels.Matern32(self.input_dim)
+        self.model = GPflow.gpr.GPR(self.X, self.Y, kern=self.k())
+
+    def test_cov(self):
+        mu1, var = self.model.predict_f(self.Xtest)
+        mu2, covar = self.model.predict_f_full_cov(self.Xtest)
+        self.failUnless(np.all(mu1 == mu2))
+        self.failUnless(covar.shape == self.covar_shape)
+        self.failUnless(var.shape == (self.Ntest, self.output_dim))
+        for i in range(self.output_dim):
+            self.failUnless(np.allclose(var[:, i], np.diag(covar[:, :, i])))
+
+    def test_samples(self):
+        samples = self.model.predict_f_samples(self.Xtest, self.num_samples)
+        self.failUnless(samples.shape == self.samples_shape)
+
+
+class TestFullCovSGPR(TestFullCov):
+    def setUp(self):
+        TestFullCov.setUp(self)
+        self.model = GPflow.sgpr.SGPR(self.X, self.Y, Z=self.Z, kern=self.k())
+
+
+class TestFullCovGPRFITC(TestFullCov):
+    def setUp(self):
+        TestFullCov.setUp(self)
+        self.model = GPflow.sgpr.GPRFITC(self.X, self.Y,
+                                         Z=self.Z, kern=self.k())
+
+
+class TestFullCovSVGP1(TestFullCov):
+    def setUp(self):
+        TestFullCov.setUp(self)
+        self.model = GPflow.svgp.SVGP(self.X, self.Y, Z=self.Z, kern=self.k(),
+                                      likelihood=GPflow.likelihoods.Gaussian(),
+                                      whiten=False, q_diag=True)
+
+
+class TestFullCovSVGP2(TestFullCov):
+    def setUp(self):
+        TestFullCov.setUp(self)
+        self.model = GPflow.svgp.SVGP(self.X, self.Y, Z=self.Z, kern=self.k(),
+                                      likelihood=GPflow.likelihoods.Gaussian(),
+                                      whiten=True, q_diag=False)
+
+
+class TestFullCovSVGP3(TestFullCov):
+    def setUp(self):
+        TestFullCov.setUp(self)
+        self.model = GPflow.svgp.SVGP(self.X, self.Y, Z=self.Z, kern=self.k(),
+                                      likelihood=GPflow.likelihoods.Gaussian(),
+                                      whiten=True, q_diag=True)
+
+
+class TestFullCovSVGP4(TestFullCov):
+    def setUp(self):
+        TestFullCov.setUp(self)
+        self.model = GPflow.svgp.SVGP(self.X, self.Y, Z=self.Z, kern=self.k(),
+                                      likelihood=GPflow.likelihoods.Gaussian(),
+                                      whiten=True, q_diag=False)
+
+
+class TestFullCovVGP(TestFullCov):
+    def setUp(self):
+        TestFullCov.setUp(self)
+        self.model = GPflow.vgp.VGP(self.X, self.Y, kern=self.k(),
+                                    likelihood=GPflow.likelihoods.Gaussian())
+
+
+class TestFullCovGPMC(TestFullCov):
+    def setUp(self):
+        TestFullCov.setUp(self)
+        self.model = GPflow.gpmc.GPMC(self.X, self.Y, kern=self.k(),
+                                      likelihood=GPflow.likelihoods.Gaussian())
+
+
+class TestFullCovSGPMC(TestFullCov):
+    def setUp(self):
+        TestFullCov.setUp(self)
+        self.model = GPflow.sgpmc.SGPMC(self.X, self.Y, kern=self.k(),
+                                        likelihood=GPflow.likelihoods.Gaussian(),
+                                        Z=self.Z)
 
 
 if __name__ == "__main__":
     unittest.main()
-
diff --git a/testing/test_priors.py b/testing/test_priors.py
--- a/testing/test_priors.py
+++ b/testing/test_priors.py
@@ -1,6 +1,8 @@
 import unittest
 import GPflow
 import numpy as np
+import tensorflow as tf
+
 
 class PriorModeTests(unittest.TestCase):
     """
@@ -8,6 +10,8 @@ class PriorModeTests(unittest.TestCase):
     mode is the same as the known mode.
     """
     def setUp(self):
+        tf.reset_default_graph()
+
         class FlatModel(GPflow.model.Model):
             def build_likelihood(self):
                 return 0
@@ -15,15 +19,15 @@ def build_likelihood(self):
 
     def testGaussianMode(self):
         self.m.x = GPflow.param.Param(1.0)
-        self.m.x.prior = GPflow.priors.Gaussian(3,1)
+        self.m.x.prior = GPflow.priors.Gaussian(3, 1)
         self.m.optimize(display=0)
 
         xmax = self.m.get_free_state()
         self.failUnless(np.allclose(xmax, 3))
 
     def testGaussianModeMatrix(self):
-        self.m.x = GPflow.param.Param(np.random.randn(4,4))
-        self.m.x.prior = GPflow.priors.Gaussian(-1,10)
+        self.m.x = GPflow.param.Param(np.random.randn(4, 4))
+        self.m.x.prior = GPflow.priors.Gaussian(-1, 10)
         self.m.optimize(display=0)
 
         xmax = self.m.get_free_state()
@@ -36,12 +40,8 @@ def testGammaMode(self):
         self.m.optimize(display=0)
 
         print(self.m.x._array, (shape - 1) * scale)
-        self.failUnless(np.allclose(self.m.x._array, (shape -1.) * scale, 1e-3))
-
-
-
-
-
+        true_mode = (shape - 1.) * scale
+        self.failUnless(np.allclose(self.m.x._array, true_mode, 1e-3))
 
 
 if __name__ == "__main__":
diff --git a/testing/test_transforms.py b/testing/test_transforms.py
--- a/testing/test_transforms.py
+++ b/testing/test_transforms.py
@@ -5,6 +5,7 @@
 
 class TransformTests(unittest.TestCase):
     def setUp(self):
+        tf.reset_default_graph()
         self.x = tf.placeholder(tf.float64)
         self.x_np = np.random.randn(10)
         self.session = tf.Session()
diff --git a/testing/test_variational.py b/testing/test_variational.py
--- a/testing/test_variational.py
+++ b/testing/test_variational.py
@@ -38,6 +38,7 @@ def kernel(kernelVariance=1,lengthScale=1.):
 
 class VariationalUnivariateTest(unittest.TestCase):
     def setUp(self):
+        tf.reset_default_graph()
         #def __init__(self, X, Y, kern, likelihood, Z, mean_function=Zero(), num_latent=None, q_diag=False, whiten=True):
         self.y_real = 2.
         self.K = 1.
@@ -109,6 +110,7 @@ def testUnivariateConditionals(self):
 
 class VariationalMultivariateTest(unittest.TestCase):
     def setUp( self ):
+        tf.reset_default_graph()
         self.nDimensions = 3
         self.rng = np.random.RandomState(1)
         self.Y = self.rng.randn( self.nDimensions, 1 )