Repository: GPflow/GPflow

Issue:
Improve test coverage for stochastic gradients with tf optimizer. 
#281 suggests we need better test coverage of this part of the code. 


Discussion:

Resolving patch:
diff --git a/GPflow/minibatch.py b/GPflow/minibatch.py
new file mode 100644
--- /dev/null
+++ b/GPflow/minibatch.py
@@ -0,0 +1,124 @@
+# Copyright 2016 James Hensman, Valentine Svensson, alexggmatthews, Mark van der Wilk
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+from .param import DataHolder
+
+
+class IndexManager(object):
+    """
+    Base clase for methods of batch indexing data.
+    rng is an instance of np.random.RandomState, defaults to seed 0.
+    """
+    def __init__(self, minibatch_size, total_points, rng=None):
+        self.minibatch_size = minibatch_size
+        self.total_points = total_points
+        self.rng = rng or np.random.RandomState(0)
+
+    def nextIndices(self):
+        raise NotImplementedError
+
+
+class ReplacementSampling(IndexManager):
+    def nextIndices(self):
+        return self.rng.randint(self.total_points,
+                                size=self.minibatch_size)
+
+
+class NoReplacementSampling(IndexManager):
+    def __init__(self, minibatch_size, total_points, rng=None):
+        # Can't sample without replacement is minibatch_size is larger
+        # than total_points
+        assert(minibatch_size <= total_points)
+        IndexManager.__init__(self, minibatch_size, total_points, rng)
+
+    def nextIndices(self):
+        permutation = self.rng.permutation(self.total_points)
+        return permutation[:self.minibatch_size]
+
+
+class SequenceIndices(IndexManager):
+    """
+    A class that maintains the state necessary to manage
+    sequential indexing of data holders.
+    """
+    def __init__(self, minibatch_size, total_points, rng=None):
+        self.counter = 0
+        IndexManager.__init__(self, minibatch_size, total_points, rng)
+
+    def nextIndices(self):
+        """
+        Written so that if total_points
+        changes this will still work
+        """
+        firstIndex = self.counter
+        lastIndex = self.counter + self.minibatch_size
+        self.counter = lastIndex % self.total_points
+        return np.arange(firstIndex, lastIndex) % self.total_points
+
+
+class MinibatchData(DataHolder):
+    """
+    A special DataHolder class which feeds a minibatch
+    to tensorflow via update_feed_dict().
+    """
+
+    # List of valid specifiers for generation methods.
+    _generation_methods = ['replace', 'noreplace', 'sequential']
+
+    def __init__(self, array, minibatch_size, rng=None, batch_manager=None):
+        """
+        array is a numpy array of data.
+        minibatch_size (int) is the size of the minibatch
+
+        batch_manager specified data sampling scheme and is a subclass
+        of IndexManager.
+
+        Note: you may want to randomize the order of the data
+        if using sequential generation.
+        """
+        DataHolder.__init__(self, array, on_shape_change='pass')
+        total_points = self._array.shape[0]
+        self.parseGenerationMethod(batch_manager,
+                                   minibatch_size,
+                                   total_points,
+                                   rng)
+
+    def parseGenerationMethod(self,
+                              input_batch_manager,
+                              minibatch_size,
+                              total_points,
+                              rng):
+        # Logic for default behaviour.
+        # When minibatch_size is a small fraction of total_point
+        # ReplacementSampling should give similar results to
+        # NoReplacementSampling and the former can be much faster.
+        if input_batch_manager is None:
+            fraction = float(minibatch_size) / float(total_points)
+            if fraction < 0.5:
+                self.index_manager = ReplacementSampling(minibatch_size,
+                                                         total_points,
+                                                         rng)
+            else:
+                self.index_manager = NoReplacementSampling(minibatch_size,
+                                                           total_points,
+                                                           rng)
+        else:  # Explicitly specified behaviour.
+            if input_batch_manager.__class__ not in IndexManager.__subclasses__():
+                raise NotImplementedError
+            self.index_manager = input_batch_manager
+
+    def update_feed_dict(self, key_dict, feed_dict):
+        next_indices = self.index_manager.nextIndices()
+        feed_dict[key_dict[self]] = self._array[next_indices]
diff --git a/GPflow/svgp.py b/GPflow/svgp.py
--- a/GPflow/svgp.py
+++ b/GPflow/svgp.py
@@ -16,41 +16,13 @@
 from __future__ import absolute_import
 import tensorflow as tf
 import numpy as np
-from .param import Param, DataHolder
+from .param import Param
 from .model import GPModel
 from . import transforms, conditionals, kullback_leiblers
 from .mean_functions import Zero
 from .tf_wraps import eye
 from ._settings import settings
-
-
-class MinibatchData(DataHolder):
-    """
-    A special DataHolder class which feeds a minibatch to tensorflow via update_feed_dict().
-    """
-    def __init__(self, array, minibatch_size, rng=None):
-        """
-        array is a numpy array of data.
-        minibatch_size (int) is the size of the minibatch
-        rng is an instance of np.random.RandomState(), defaults to seed 0.
-        """
-        DataHolder.__init__(self, array, on_shape_change='pass')
-        self.minibatch_size = minibatch_size
-        self.rng = rng or np.random.RandomState(0)
-
-    def generate_index(self):
-        if float(self.minibatch_size) / float(self._array.shape[0]) > 0.5:
-            return self.rng.permutation(self._array.shape[0])[:self.minibatch_size]
-        else:
-            # This is much faster than above, and for N >> minibatch,
-            # it doesn't make much difference. This actually
-            # becomes the limit when N is around 10**6, which isn't
-            # uncommon when using SVI.
-            return self.rng.randint(self._array.shape[0], size=self.minibatch_size)
-
-    def update_feed_dict(self, key_dict, feed_dict):
-        feed_dict[key_dict[self]] = self._array[self.generate_index()]
-
+from .minibatch import MinibatchData
 
 class SVGP(GPModel):
     """
@@ -83,7 +55,7 @@ def __init__(self, X, Y, kern, likelihood, Z, mean_function=Zero(),
         """
         # sort out the X, Y into MiniBatch objects.
         if minibatch_size is None:
-            minibatch_size = X.shape[0]
+            minibatch_size = X.shape[0]     
         self.num_data = X.shape[0]
         X = MinibatchData(X, minibatch_size, np.random.RandomState(0))
         Y = MinibatchData(Y, minibatch_size, np.random.RandomState(0))


Test patch:
diff --git a/testing/test_methods.py b/testing/test_methods.py
--- a/testing/test_methods.py
+++ b/testing/test_methods.py
@@ -13,11 +13,11 @@
 # limitations under the License.from __future__ import print_function
 
 import GPflow
+from GPflow.minibatch import SequenceIndices
 import numpy as np
 import unittest
 import tensorflow as tf
 
-
 class TestMethods(unittest.TestCase):
     def setUp(self):
         tf.reset_default_graph()
@@ -154,6 +154,107 @@ def test_q_sqrt_fixing(self):
         m1.q_sqrt.fixed = True
         m1._compile()
 
+class TestStochasticGradients(unittest.TestCase):
+    """
+    In response to bug #281, we need to make sure stochastic update
+    happens correctly in tf optimizer mode.
+    To do this compare stochastic updates with deterministic updates
+    that should be equivalent.
+    
+    Data term in svgp likelihood is 
+    \sum_{i=1^N}E_{q(i)}[\log p(y_i | f_i )
+    
+    This sum is then approximated with an unbiased minibatch estimate.
+    In this test we substitute a deterministic analogue of the batchs
+    sampler for which we can predict the effects of different updates.
+    """
+    def setUp(self):        
+        self.XAB = np.atleast_2d(np.array([0.,1.])).T
+        self.YAB = np.atleast_2d(np.array([-1.,3.])).T
+        self.sharedZ = np.atleast_2d(np.array([0.5]) )
+        self.indexA = 0
+        self.indexB = 1
+        
+    def getIndexedData(self,baseX,baseY,indeces):
+        newX = baseX[indeces]
+        newY = baseY[indeces]
+        return newX, newY
+            
+    def getModel(self,X,Y,Z,minibatch_size):
+        model = GPflow.svgp.SVGP(X,
+                                 Y,
+                                 kern = GPflow.kernels.RBF(1),
+                                 likelihood = GPflow.likelihoods.Gaussian(),
+                                 Z = Z,
+                                 minibatch_size=minibatch_size)
+        #This step changes the batch indeces to cycle.
+        model.X.index_manager = SequenceIndices(minibatch_size,X.shape[0])
+        model.Y.index_manager = SequenceIndices(minibatch_size,X.shape[0])
+        return model        
+                
+    def getTfOptimizer(self):
+        learning_rate = .1
+        opt = tf.train.GradientDescentOptimizer(learning_rate, 
+                                                use_locking=True)
+        return opt
+                
+    def getIndexedModel(self,X,Y,Z,minibatch_size,indeces):
+        Xindeces,Yindeces = self.getIndexedData(X,Y,indeces)
+        indexedModel = self.getModel(Xindeces,Yindeces,Z,minibatch_size)
+        return indexedModel
+        
+    def checkModelsClose(self,modelA,modelB,tolerance=1e-2):
+        modelA_dict = modelA.get_parameter_dict()
+        modelB_dict = modelB.get_parameter_dict()
+        if sorted(modelA_dict.keys())!=sorted(modelB_dict.keys()):
+            return False
+        for key in modelA_dict:
+            if ((modelA_dict[key] - modelB_dict[key])>tolerance).any():
+                return False
+        return True
+    
+    def compareTwoModels(self,indecesOne,indecesTwo,
+                              batchOne,batchTwo,
+                              maxiter,
+                              checkSame=True):
+        modelOne = self.getIndexedModel(self.XAB,
+                                        self.YAB,
+                                        self.sharedZ,
+                                        batchOne,
+                                        indecesOne)
+        modelTwo = self.getIndexedModel(self.XAB,
+                                        self.YAB,
+                                        self.sharedZ,
+                                        batchTwo,
+                                        indecesTwo)
+        modelOne.optimize(method=self.getTfOptimizer(),maxiter=maxiter)
+        modelTwo.optimize(method=self.getTfOptimizer(),maxiter=maxiter)
+        if checkSame:
+            self.assertTrue(self.checkModelsClose(modelOne,modelTwo))        
+        else:
+            self.assertFalse(self.checkModelsClose(modelOne,modelTwo))          
+        
+    def testOne(self):  
+        self.compareTwoModels([self.indexA,self.indexB],
+                              [self.indexB,self.indexA],
+                              2,
+                              2,
+                              3)
+                              
+    def testTwo(self):
+        self.compareTwoModels([self.indexA,self.indexB],
+                              [self.indexA,self.indexA],
+                              1,
+                              2,
+                              1)        
+                                          
+    def testThree(self):
+        self.compareTwoModels([self.indexA,self.indexA],
+                              [self.indexA,self.indexB],
+                              1,
+                              1,
+                              2,
+                              False)                  
 
 class TestSparseMCMC(unittest.TestCase):
     """
diff --git a/testing/test_minibatch_data.py b/testing/test_minibatch_data.py
new file mode 100644
--- /dev/null
+++ b/testing/test_minibatch_data.py
@@ -0,0 +1,138 @@
+# Copyright 2016 alexggmatthews
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import tensorflow as tf
+import numpy as np
+import unittest
+from GPflow.minibatch import SequenceIndices, MinibatchData
+from GPflow.minibatch import ReplacementSampling, NoReplacementSampling
+
+class TestSequentialManager(unittest.TestCase):
+    def setUp(self):
+        tf.reset_default_graph()
+     
+    def testA(self):
+        minibatch_size = 3
+        total_points = 5
+        sequenceManager = SequenceIndices(minibatch_size,total_points)
+        
+        indecesA = sequenceManager.nextIndices()
+        self.assertTrue((indecesA==np.arange(0,minibatch_size)).all())
+
+        indecesB = sequenceManager.nextIndices()
+        self.assertTrue((indecesB==np.array([3,4,0 ])).all())
+    
+    def testB(self):
+        minibatch_size = 5
+        total_points = 2
+        sequenceManager = SequenceIndices(minibatch_size,total_points)
+        
+        indecesA = sequenceManager.nextIndices()
+        targetIndicesA = np.array([0,1,0,1,0])
+            
+        self.assertTrue((indecesA==targetIndicesA).all())
+
+        indecesB = sequenceManager.nextIndices()
+        targetIndicesB = np.array([1,0,1,0,1])
+            
+        self.assertTrue((indecesB==targetIndicesB).all())
+
+class TestRandomIndexManagers(unittest.TestCase):
+    def setUp(self):
+        tf.reset_default_graph()
+    
+    def checkUniformDist(self, indeces, nChoices):
+        fTotalPoints = float(len(indeces))
+        tolerance = 1e-2
+        for possibleIndex in range(nChoices):
+            empirical = indeces.count(possibleIndex) / fTotalPoints
+            error = np.abs(empirical - 1./nChoices)        
+            if error > tolerance:
+                return False
+        return True
+        
+    def testReplacement(self):
+        minibatch_size = 1000
+        data_size = 3
+        tolerance = 1e-2
+        rs = ReplacementSampling(minibatch_size,data_size)
+        indeces = rs.nextIndices().tolist()
+        self.assertTrue(self.checkUniformDist(indeces,data_size))
+                    
+    def testNoReplacement(self):
+        mini_size_err = 5
+        data_size_err = 3
+        constructor = lambda : NoReplacementSampling(mini_size_err,
+                                                     data_size_err)
+        self.assertRaises(AssertionError,constructor)
+        
+        mini_size = 3
+        data_size = 3
+        nrs = NoReplacementSampling(mini_size,data_size)
+        indeces = np.sort(nrs.nextIndices()).tolist()
+        self.assertEqual(indeces,[0,1,2])
+
+        one = 1     
+        nrsb = NoReplacementSampling(one,data_size)
+        
+        indecesOverall = []
+        for repeatIndex in range(3000):
+            indeces = nrsb.nextIndices().tolist()
+            indecesOverall = indecesOverall + indeces
+        self.assertTrue(self.checkUniformDist(indecesOverall,data_size))
+            
+class TestMinibatchData(unittest.TestCase):
+    def setUp(self):
+        tf.reset_default_graph()
+        self.nDataPoints = 10
+        self.minibatch_size = 4
+        self.dummyArray = np.atleast_2d(np.arange(self.nDataPoints)).T
+                
+    def testA(self):
+        constructor = lambda : MinibatchData(self.dummyArray, 
+                                         self.minibatch_size, 
+                                         rng=None, 
+                                         batch_manager=[])
+        self.assertRaises(NotImplementedError,constructor)
+
+    def testB(self):
+        sm = SequenceIndices(self.minibatch_size, self.nDataPoints)
+        md = MinibatchData(self.dummyArray, 
+                           self.minibatch_size, 
+                           rng=None, 
+                           batch_manager=sm)
+        output_string = 'test_out'
+        key_dict = {md: output_string}
+        feed_dict = {} 
+        md.update_feed_dict( key_dict, feed_dict )
+        test_out_array = np.atleast_2d(np.arange(self.minibatch_size)).T
+        test_dict = {output_string : test_out_array }                     
+        self.assertEqual(feed_dict.keys(),test_dict.keys())
+        self.assertTrue((feed_dict[output_string]==test_out_array).all())
+
+    def testC(self):
+        #Test the manager selection logic.
+        md = MinibatchData(self.dummyArray, 
+                           self.minibatch_size, 
+                           rng=None)        
+        self.assertEqual(md.index_manager.__class__, ReplacementSampling)                  
+
+
+        md = MinibatchData(self.dummyArray, 
+                           self.minibatch_size*2, 
+                           rng=None)        
+        self.assertEqual(md.index_manager.__class__, NoReplacementSampling)          
+ 
+if __name__ == "__main__":
+    unittest.main()