Repository: tensorflow/addons

Issue:
`Lookahead` optimizer is not compatible with `ReduceLROnPlateau`
**System information**
- OS Platform and Distribution (Linux Ubuntu 16.04):
- TensorFlow version and how it was installed (pip tensorflow-gpu==2.0.0):
- TensorFlow-Addons version and how it was installed (pip tensorflow-addon==0.6.0):
- Python version: 3.7
- Is GPU used? (yes/no): yes

**Describe the bug**

Lookahead is not compatible with callbacks. The attribute 'lr' or 'learning_rate' is missing.

**Code to reproduce the issue**

c.f. title

**Other info / logs**

Error:
```bash
  File ".../lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training.py", line 728, in fit
    use_multiprocessing=use_multiprocessing)
  File ".../lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_v2.py", line 372, in fit
    prefix='val_')
  File ".../lib/python3.7/contextlib.py", line 119, in __exit__
    next(self.gen)
  File ".../lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_v2.py", line 685, in on_epoch
    self.callbacks.on_epoch_end(epoch, epoch_logs)
  File ".../lib/python3.7/site-packages/tensorflow_core/python/keras/callbacks.py", line 298, in on_epoch_end
    callback.on_epoch_end(epoch, logs)
  File .../lib/python3.7/site-packages/tensorflow_core/python/keras/callbacks.py", line 1819, in on_epoch_end
    logs['lr'] = K.get_value(self.model.optimizer.lr)
  File ".../lib/python3.7/site-packages/tensorflow_core/python/keras/optimizer_v2/optimizer_v2.py", line 551, in __getattribute__
    raise e
  File .../ib/python3.7/site-packages/tensorflow_core/python/keras/optimizer_v2/optimizer_v2.py", line 541, in __getattribute__
    return super(OptimizerV2, self).__getattribute__(name)
AttributeError: 'Lookahead' object has no attribute 'lr'
```

@CyberZHG:  Possible quick fix from the source repository: https://github.com/CyberZHG/keras-lookahead/blob/master/keras_lookahead/optimizers.py

```python
    @property
    def lr(self):
        return self._optimizer.learning_rate

    @lr.setter
    def lr(self, lr):
        self._optimizer.learning_rate = lr

    @property
    def learning_rate(self):
        return self._optimizer.learning_rate

    @learning_rate.setter
    def learning_rate(self, learning_rate):
        self._optimizer.learning_rate = learning_rate
```



Discussion:
Note that the original code available at  https://github.com/CyberZHG/keras-lookahead/blob/master/keras_lookahead/optimizers.py works very well with `ReduceLROnPlateau`.
Here's a quick user level patch. Waiting for an official one.

```python
from tensorflow_addons.optimizers import Lookahead


def lr_getter(self):
    return self._optimizer.learning_rate


def lr_setter(self, lr):
    self._optimizer.learning_rate = lr


def learning_rate_getter(self):
    return self._optimizer.learning_rate


def learning_rate_setter(self, learning_rate):
    self._optimizer.learning_rate = learning_rate


def patch_me(cls):
    cls.lr = property(lr_getter, lr_setter)
    cls.learning_rate = property(learning_rate_getter, learning_rate_setter)


patch_me(Lookahead)
```
@CyberZHG Any reason these were removed or is it okay to merge this change?

Resolving patch:
diff --git a/tensorflow_addons/optimizers/lookahead.py b/tensorflow_addons/optimizers/lookahead.py
--- a/tensorflow_addons/optimizers/lookahead.py
+++ b/tensorflow_addons/optimizers/lookahead.py
@@ -162,6 +162,22 @@ def get_config(self):
         base_config = super(Lookahead, self).get_config()
         return dict(list(base_config.items()) + list(config.items()))
 
+    @property
+    def lr(self):
+        return self._optimizer._get_hyper('learning_rate')
+
+    @lr.setter
+    def lr(self, lr):
+        self._optimizer._set_hyper('learning_rate', lr)
+
+    @property
+    def learning_rate(self):
+        return self._optimizer._get_hyper('learning_rate')
+
+    @learning_rate.setter
+    def learning_rate(self, learning_rate):
+        self._optimizer._set_hyper('learning_rate', learning_rate)
+
     @classmethod
     def from_config(cls, config, custom_objects=None):
         optimizer = tf.keras.optimizers.deserialize(
diff --git a/tensorflow_addons/optimizers/moving_average.py b/tensorflow_addons/optimizers/moving_average.py
--- a/tensorflow_addons/optimizers/moving_average.py
+++ b/tensorflow_addons/optimizers/moving_average.py
@@ -163,3 +163,19 @@ def _resource_apply_sparse_duplicate_indices(self, grad, var, indices):
 
     def _resource_apply_sparse(self, grad, var, indices):
         return self._optimizer._resource_apply_sparse(grad, var, indices)  # pylint: disable=protected-access
+
+    @property
+    def lr(self):
+        return self._optimizer._get_hyper('learning_rate')
+
+    @lr.setter
+    def lr(self, lr):
+        self._optimizer._set_hyper('learning_rate', lr)
+
+    @property
+    def learning_rate(self):
+        return self._optimizer._get_hyper('learning_rate')
+
+    @learning_rate.setter
+    def learning_rate(self, learning_rate):
+        self._optimizer._set_hyper('learning_rate', learning_rate)


Test patch:
diff --git a/tensorflow_addons/optimizers/lookahead_test.py b/tensorflow_addons/optimizers/lookahead_test.py
--- a/tensorflow_addons/optimizers/lookahead_test.py
+++ b/tensorflow_addons/optimizers/lookahead_test.py
@@ -127,6 +127,26 @@ def test_fit_simple_linear_model(self):
         max_abs_diff = np.max(np.abs(predicted - y))
         self.assertLess(max_abs_diff, 1e-4)
 
+    def test_model_dynamic_lr(self):
+        grad = tf.Variable([[0.1]])
+        model = tf.keras.Sequential([
+            tf.keras.layers.Dense(
+                1,
+                kernel_initializer=tf.keras.initializers.Constant([[1.0]]),
+                use_bias=False)
+        ])
+        model.build(input_shape=[1, 1])
+
+        opt = Lookahead('adam', sync_period=10, slow_step_size=0.4)
+        update = opt.apply_gradients(list(zip([grad], model.variables)))
+
+        self.evaluate(tf.compat.v1.global_variables_initializer())
+        self.evaluate(update)
+        self.assertAllClose(opt.lr.read_value(), 1e-3)
+
+        opt.lr = 1e-4
+        self.assertAllClose(opt.lr.read_value(), 1e-4)
+
     def test_get_config(self):
         self.skipTest('Wait #33614 to be fixed')
         opt = Lookahead('adam', sync_period=10, slow_step_size=0.4)
diff --git a/tensorflow_addons/optimizers/moving_average_test.py b/tensorflow_addons/optimizers/moving_average_test.py
--- a/tensorflow_addons/optimizers/moving_average_test.py
+++ b/tensorflow_addons/optimizers/moving_average_test.py
@@ -107,6 +107,26 @@ def test_model_weights_update(self):
         self.evaluate(mean_update)
         self.assertAllClose(model.variables[0].read_value(), [[0.9]])
 
+    def test_model_dynamic_lr(self):
+        grad = tf.Variable([[0.1]])
+        model = tf.keras.Sequential([
+            tf.keras.layers.Dense(
+                1,
+                kernel_initializer=tf.keras.initializers.Constant([[1.0]]),
+                use_bias=False)
+        ])
+        model.build(input_shape=[1, 1])
+
+        opt = MovingAverage(tf.keras.optimizers.SGD(lr=1e-3), 0.5)
+        update = opt.apply_gradients(list(zip([grad], model.variables)))
+
+        self.evaluate(tf.compat.v1.global_variables_initializer())
+        self.evaluate(update)
+        self.assertAllClose(opt.lr.read_value(), 1e-3)
+
+        opt.lr = 1e-4
+        self.assertAllClose(opt.lr.read_value(), 1e-4)
+
     def test_optimizer_string(self):
         _ = MovingAverage('adam')