Repository: lisa-lab/pylearn2

Issue:
Use minimal value for the denominator in AdaGrad
See #1496 for a corner case where the accumulated gradient can be 0.



Discussion:

Resolving patch:
diff --git a/pylearn2/training_algorithms/learning_rule.py b/pylearn2/training_algorithms/learning_rule.py
--- a/pylearn2/training_algorithms/learning_rule.py
+++ b/pylearn2/training_algorithms/learning_rule.py
@@ -330,7 +330,17 @@ class AdaGrad(LearningRule):
     Implements the AdaGrad learning rule as described in:
     "Adaptive subgradient methods for online learning and
     stochastic optimization", Duchi J, Hazan E, Singer Y.
+
+    Parameters
+    ----------
+    max_scaling: float, optional
+        Restrict the gradient scaling coefficient to values
+        below `max_scaling`. This prevents corner cases (like all-zero weights)
+        to generate NaNs (see #1496).
     """
+    def __init__(self, max_scaling=1e5):
+        assert max_scaling > 0
+        self.eps = 1. / max_scaling
 
     def get_updates(self, learning_rate, grads, lr_scalers=None):
         """
@@ -363,8 +373,8 @@ def get_updates(self, learning_rate, grads, lr_scalers=None):
 
             # Compute update
             epsilon = lr_scalers.get(param, 1.) * learning_rate
-            delta_x_t = (- epsilon / T.sqrt(new_sum_squared_grad)
-                         * grads[param])
+            scale = T.maximum(self.eps, T.sqrt(new_sum_squared_grad))
+            delta_x_t = (-epsilon / scale * grads[param])
 
             # Apply update
             updates[sum_square_grad] = new_sum_squared_grad


Test patch:
diff --git a/pylearn2/training_algorithms/tests/test_learning_rule.py b/pylearn2/training_algorithms/tests/test_learning_rule.py
--- a/pylearn2/training_algorithms/tests/test_learning_rule.py
+++ b/pylearn2/training_algorithms/tests/test_learning_rule.py
@@ -3,6 +3,7 @@
 from theano.compat.six.moves import zip as izip
 
 from pylearn2.costs.cost import SumOfCosts
+from pylearn2.datasets.dense_design_matrix import DenseDesignMatrix
 from pylearn2.testing.cost import SumOfOneHalfParamsSquared
 from pylearn2.testing.cost import SumOfParams
 from pylearn2.testing.datasets import ArangeDataset
@@ -143,7 +144,6 @@ def test_adadelta():
         state[param]['dx2'] = np.zeros(param_shape)
 
     def adadelta_manual(model, state):
-        inc = []
         rval = []
         for scale, param in izip(scales, model.get_params()):
             pstate = state[param]
@@ -170,23 +170,32 @@ def adadelta_manual(model, state):
                izip(manual, model.get_params()))
 
 
-def test_adagrad():
+def prepare_adagrad_test(dataset_type='arange', model_type='random'):
     """
-    Make sure that learning_rule.AdaGrad obtains the same parameter values as
-    with a hand-crafted AdaGrad implementation, given a dummy model and
-    learning rate scaler for each parameter.
-
-    Reference:
-    "Adaptive subgradient methods for online learning and
-    stochastic optimization", Duchi J, Hazan E, Singer Y.
+    Factor out common code for AdaGrad tests.
+    Parameters
+    ----------
+    dataset_type : string, optional
+        Can use either `arange` to use an ArangeDataset instance or
+        `zeros` to create an all-zeros DenseDesignMatrix.
+    model_type : string, optional
+        How to initialize the model; `random` will initialize parameters
+        to random values, `zeros` to zero.
     """
-
     # We include a cost other than SumOfParams so that data is actually
     # queried from the training set, and the expected number of updates
     # are applied.
     cost = SumOfCosts([SumOfOneHalfParamsSquared(), (0., DummyCost())])
-    model = DummyModel(shapes, lr_scalers=scales)
-    dataset = ArangeDataset(1)
+    model = DummyModel(shapes, lr_scalers=scales, init_type=model_type)
+    if dataset_type == 'arange':
+        dataset = ArangeDataset(1)
+    elif dataset_type == 'zeros':
+        X = np.zeros((1, 1))
+        X[:, 0] = np.arange(1)
+        dataset = DenseDesignMatrix(X)
+    else:
+        raise ValueError('Unknown value for dataset_type: %s',
+                         dataset_type)
 
     sgd = SGD(cost=cost,
               learning_rate=learning_rate,
@@ -201,6 +210,21 @@ def test_adagrad():
         state[param] = {}
         state[param]['sg2'] = np.zeros(param_shape)
 
+    return (cost, model, dataset, sgd, state)
+
+
+def test_adagrad():
+    """
+    Make sure that learning_rule.AdaGrad obtains the same parameter values as
+    with a hand-crafted AdaGrad implementation, given a dummy model and
+    learning rate scaler for each parameter.
+    Reference:
+    "Adaptive subgradient methods for online learning and
+    stochastic optimization", Duchi J, Hazan E, Singer Y.
+    """
+
+    cost, model, dataset, sgd, state = prepare_adagrad_test()
+
     def adagrad_manual(model, state):
         rval = []
         for scale, param in izip(scales, model.get_params()):
@@ -227,6 +251,36 @@ def adagrad_manual(model, state):
                izip(manual, model.get_params()))
 
 
+def test_adagard_max_scaling():
+    """
+    Tests the max_scaling argument to the constructor.
+    One corner case that prompted the insertion of this parameter is when
+    the weights are all-zeros (see #1496).
+    """
+
+    # the value must be positive
+    try:
+        AdaGrad(-1.0)
+        allows_null = True
+    except AssertionError:
+        allows_null = False
+    assert not allows_null
+
+    # should not generate RuntimeError and/or generate NaNs
+    def one_case(dataset_type, model_type):
+        # prepare a test bed
+        cost, model, dataset, sgd, state = prepare_adagrad_test(dataset_type,
+                                                                model_type)
+        # SGD will also throw RuntimeError if it finds NaNs
+        sgd.train(dataset=dataset)
+        for param in model.get_params():
+            assert not np.any(np.isnan(param.get_value()))
+    one_case('zeros', 'zeros')
+    one_case('arange', 'zeros')
+    one_case('zeros', 'random')
+    one_case('arange', 'random')
+
+
 def test_rmsprop():
     """
     Make sure that learning_rule.RMSProp obtains the same parameter values as
diff --git a/pylearn2/training_algorithms/tests/test_sgd.py b/pylearn2/training_algorithms/tests/test_sgd.py
--- a/pylearn2/training_algorithms/tests/test_sgd.py
+++ b/pylearn2/training_algorithms/tests/test_sgd.py
@@ -50,10 +50,28 @@ def expr(self, model, data):
 
 
 class DummyModel(Model):
-
-    def __init__(self, shapes, lr_scalers=None):
+    """
+    A dummy model used for testing.
+
+    Parameters
+    ----------
+    shapes : list
+        List of shapes for each parameter.
+    lr_scalers : list, optional
+        Scalers to use for each parameter.
+    init_type : string, optional
+        How to fill initial values in parameters: `random` - generate random
+        values; `zeros` - set all to zeros.
+    """
+    def __init__(self, shapes, lr_scalers=None, init_type='random'):
         super(DummyModel, self).__init__()
-        self._params = [sharedX(np.random.random(shape)) for shape in shapes]
+        if init_type == 'random':
+            self._params = [sharedX(np.random.random(shp)) for shp in shapes]
+        elif init_type == 'zeros':
+            self._params = [sharedX(np.zeros(shp)) for shp in shapes]
+        else:
+            raise ValueError('Unknown value for init_type: %s',
+                             init_type)
         self.input_space = VectorSpace(1)
         self.lr_scalers = lr_scalers