Repository: keras-team/keras

Issue:
Learning rate and momentum as shared scalars
I'd like to ask if we could make the optimizer's learning rate (`lr`) and momentum (`momentum`) into shared scalars. This way we could change their values during training with `.set_value` using custom rules.

I could work on the PR as long as you guys don't have a reason for not to.



Discussion:
I would have issues with changing the definition of the learning rate. But if the PR is just about switching `lr` and `momentum` to shared scalars (hence modifiable after compilation) instead of scalars, without touching the logic, that's fine by me.

This could be a very useful feature. Actually, I can wait to have it:)

So right now, the only way to change learning rate during training is 1) Save weights of the model. 2) Compile a new model with same architecture and different learning rate. 3) Load weights and continue training??

I think you can also create a shared scalar and pass it to the optimizer's initializer.
I tried something like this for changing momentum from a callback.
https://github.com/wuaalb/keras_extensions/blob/master/keras_extensions/callbacks.py

The problem is, since you don't recompile the graph it won't use your new learning rate or momentum, even if you change it with a callback.

Something like this doesn't work, or isn't what you're looking for?

``` python
model = ...
optimizer = SGD(lr=shared_scalar(0.01))
model.compile(optimizer, loss)
...
optimizer.lr.set_value(0.02) # e.g. from a callback during fit()
```

Anyways, making them shared scalars by default sounds good..

That is exactly what I'm doing! What we were discussing was, should we make `lr` a `shared_scalar` by default or not? It seems that we don't have anything to lose with that. I'm also writing a callback that gets a dict of `epoch x lr` pairs and switch values on the go.

> What we were discussing was, should we make lr a shared_scalar by default or not? It seems that we don't have anything to lose with that.

Indeed. You would pass a float to the constructor and internally this float would be used as the initial value of a shared scalar. The logic of the optimizer itself would be unchanged.


Resolving patch:
diff --git a/keras/callbacks.py b/keras/callbacks.py
--- a/keras/callbacks.py
+++ b/keras/callbacks.py
@@ -1,13 +1,10 @@
 from __future__ import absolute_import
 from __future__ import print_function
 
-import theano
-import theano.tensor as T
 import numpy as np
-
 import time, json, warnings
-from collections import deque
 
+from collections import deque
 from .utils.generic_utils import Progbar
 
 
@@ -262,3 +259,16 @@ def on_epoch_end(self, epoch, logs={}):
             r = requests.post(self.root + '/publish/epoch/end/', {'data': json.dumps(send)})
         except:
             print('Warning: could not reach RemoteMonitor root server at ' + str(self.root))
+
+
+class LearningRateScheduler(Callback):
+    '''LearningRateScheduler
+    func is a function that gets an epoch number as input and returns a new
+    learning rate as output.
+    '''
+    def __init__(self, func):
+        super(LearningRateScheduler, self).__init__()
+        self.func = func
+
+    def on_epoch_begin(self, epoch, logs={}):
+        model.lr.set_value(self.func(epoch))
diff --git a/keras/optimizers.py b/keras/optimizers.py
--- a/keras/optimizers.py
+++ b/keras/optimizers.py
@@ -1,7 +1,6 @@
 from __future__ import absolute_import
 import theano
 import theano.tensor as T
-import numpy as np
 
 from .utils.theano_utils import shared_zeros, shared_scalar, floatX
 from six.moves import zip
@@ -49,10 +48,13 @@ def get_config(self):
 
 class SGD(Optimizer):
 
-    def __init__(self, lr=0.01, momentum=0., decay=0., nesterov=False, *args, **kwargs):
+    def __init__(self, lr=0.01, momentum=0., decay=0., nesterov=False, *args,
+                 **kwargs):
         super(SGD, self).__init__(**kwargs)
         self.__dict__.update(locals())
         self.iterations = shared_scalar(0)
+        self.lr = shared_scalar(lr)
+        self.momentum = shared_scalar(momentum)
 
     def get_updates(self, params, constraints, loss):
         grads = self.get_gradients(loss, params)
@@ -84,6 +86,8 @@ class RMSprop(Optimizer):
     def __init__(self, lr=0.001, rho=0.9, epsilon=1e-6, *args, **kwargs):
         super(RMSprop, self).__init__(**kwargs)
         self.__dict__.update(locals())
+        self.lr = shared_scalar(lr)
+        self.rho = shared_scalar(rho)
 
     def get_updates(self, params, constraints, loss):
         grads = self.get_gradients(loss, params)
@@ -109,6 +113,7 @@ class Adagrad(Optimizer):
     def __init__(self, lr=0.01, epsilon=1e-6, *args, **kwargs):
         super(Adagrad, self).__init__(**kwargs)
         self.__dict__.update(locals())
+        self.lr = shared_scalar(lr)
 
     def get_updates(self, params, constraints, loss):
         grads = self.get_gradients(loss, params)
@@ -135,6 +140,7 @@ class Adadelta(Optimizer):
     def __init__(self, lr=1.0, rho=0.95, epsilon=1e-6, *args, **kwargs):
         super(Adadelta, self).__init__(**kwargs)
         self.__dict__.update(locals())
+        self.lr = shared_scalar(lr)
 
     def get_updates(self, params, constraints, loss):
         grads = self.get_gradients(loss, params)
@@ -142,12 +148,14 @@ def get_updates(self, params, constraints, loss):
         delta_accumulators = [shared_zeros(p.get_value().shape) for p in params]
         self.updates = []
 
-        for p, g, a, d_a, c in zip(params, grads, accumulators, delta_accumulators, constraints):
+        for p, g, a, d_a, c in zip(params, grads, accumulators,
+                                   delta_accumulators, constraints):
             new_a = self.rho * a + (1 - self.rho) * g ** 2  # update accumulator
             self.updates.append((a, new_a))
 
             # use the new accumulator and the *old* delta_accumulator
-            update = g * T.sqrt(d_a + self.epsilon) / T.sqrt(new_a + self.epsilon)
+            update = g * T.sqrt(d_a + self.epsilon) / T.sqrt(new_a +
+                                                             self.epsilon)
 
             new_p = p - self.lr * update
             self.updates.append((p, c(new_p)))  # apply constraints
@@ -170,10 +178,12 @@ class Adam(Optimizer):
 
         Default parameters follow those provided in the original paper.
     '''
-    def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-8, *args, **kwargs):
+    def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-8, *args,
+                 **kwargs):
         super(Adam, self).__init__(**kwargs)
         self.__dict__.update(locals())
         self.iterations = shared_scalar(0)
+        self.lr = shared_scalar(lr)
 
     def get_updates(self, params, constraints, loss):
         grads = self.get_gradients(loss, params)
@@ -210,5 +220,8 @@ def get_config(self):
 adam = Adam
 
 from .utils.generic_utils import get_from_module
+
+
 def get(identifier, kwargs=None):
-    return get_from_module(identifier, globals(), 'optimizer', instantiate=True, kwargs=kwargs)
+    return get_from_module(identifier, globals(), 'optimizer', instantiate=True,
+                           kwargs=kwargs)


Test patch: