Repository: chainer/chainer

Issue:
resuming issue of ExponentialShift
`ExponentialShift` doesn't work well after the trainer resumed.
Here is an example.
```
import chainer
from chainer import iterators
from chainer import optimizers
from chainer import training
from chainer.training import extensions
from chainer import serializers


class DummyModel(chainer.Chain):

    def __call__(self, x):
        return x


def setup_trainer(iteration):
    model = DummyModel()
    optimizer = optimizers.SGD()
    optimizer.setup(model)

    iterator = iterators.SerialIterator([1, 2, 3], 1)

    updater = training.StandardUpdater(iterator, optimizer)
    trainer = training.Trainer(updater, (iteration, 'iteration'), out='.')

    trainer.extend(extensions.LogReport(trigger=(1, 'iteration')))
    trainer.extend(extensions.observe_lr(), trigger=(1, 'iteration'))
    trainer.extend(
        extensions.PrintReport(['iteration', 'lr']),
        trigger=(1, 'iteration'))

    # after 15 itrerations, lr should be 0.001
    trainer.extend(
        extensions.ExponentialShift('lr', 0.1, init=0.01), trigger=(15, 'iteration'))

    return trainer


trainer = setup_trainer(10)
trainer.run()
# iteration   lr
# 1           0.01
# 2           0.01
# 3           0.01
# 4           0.01
# 5           0.01
# 6           0.01
# 7           0.01
# 8           0.01
# 9           0.01
# 10          0.01
serializers.save_npz('tmp', trainer)

resumed_trainer = setup_trainer(20)
serializers.load_npz('tmp', resumed_trainer)
resumed_trainer.run()
# iteration   lr
# 1           0.01
# 2           0.01
# 3           0.01
# 4           0.01
# 5           0.01
# 6           0.01
# 7           0.01
# 8           0.01
# 9           0.01
# 10          0.01
# 11          0.001       (lr is changed unexpectedly)
# 12          0.001
# 13          0.001
# 14          0.001
# 15          0.001
# 16          0.0001
# 17          0.0001
# 18          0.0001
# 19          0.0001
# 20          0.0001
```


Discussion:

Resolving patch:
diff --git a/chainer/training/extensions/exponential_shift.py b/chainer/training/extensions/exponential_shift.py
--- a/chainer/training/extensions/exponential_shift.py
+++ b/chainer/training/extensions/exponential_shift.py
@@ -37,13 +37,21 @@ def __init__(self, attr, rate, init=None, target=None, optimizer=None):
         self._init = init
         self._target = target
         self._optimizer = optimizer
-        self._t = 0
+        self._t = 1
+        self._before_training = True
 
     def __call__(self, trainer):
         optimizer = self._optimizer or trainer.updater.get_optimizer('main')
-        if self._init is None:
-            self._init = getattr(optimizer, self._attr)
-        value = self._init * (self._rate ** self._t)
+
+        if self._before_training:
+            if self._init is None:
+                self._init = getattr(optimizer, self._attr)
+            value = self._init * (self._rate ** (self._t - 1))
+            self._before_training = False
+        else:
+            value = self._init * (self._rate ** self._t)
+            self._t += 1
+
         if self._target is not None:
             if self._rate > 1:
                 # almost same as value = min(value, self._target), but this
@@ -55,7 +63,6 @@ def __call__(self, trainer):
                 if value / self._target < 1:
                     value = self._target
         setattr(optimizer, self._attr, value)
-        self._t += 1
 
     def serialize(self, serializer):
         self._t = serializer('_t', self._t)


Test patch:
diff --git a/tests/chainer_tests/training_tests/extensions_tests/test_exponential_shift.py b/tests/chainer_tests/training_tests/extensions_tests/test_exponential_shift.py
--- a/tests/chainer_tests/training_tests/extensions_tests/test_exponential_shift.py
+++ b/tests/chainer_tests/training_tests/extensions_tests/test_exponential_shift.py
@@ -1,9 +1,58 @@
+import numpy
 import unittest
 
 import mock
 
+from chainer import serializer
 from chainer import testing
 from chainer.training import extensions
+from chainer.training.util import get_trigger
+
+
+class DummySerializer(serializer.Serializer):
+
+    def __init__(self, target):
+        super(DummySerializer, self).__init__()
+        self.target = target
+
+    def __getitem__(self, key):
+        raise NotImplementedError
+
+    def __call__(self, key, value):
+        self.target[key] = value
+        return self.target[key]
+
+
+class DummyDeserializer(serializer.Deserializer):
+
+    def __init__(self, target):
+        super(DummyDeserializer, self).__init__()
+        self.target = target
+
+    def __getitem__(self, key):
+        raise NotImplementedError
+
+    def __call__(self, key, value):
+        if value is None:
+            value = self.target[key]
+        elif isinstance(value, numpy.ndarray):
+            numpy.copyto(value, self.target[key])
+        else:
+            value = type(value)(numpy.asarray(self.target[key]))
+        return value
+
+
+def _get_mocked_trainer():
+    trainer = mock.Mock()
+
+    def update():
+        trainer.updater.iteration += 1
+    trainer.updater.iteration = 0
+    trainer.updater.update = update
+
+    trainer.updater.get_optimizer = lambda _: trainer.updater.optimizer
+
+    return trainer
 
 
 @testing.parameterize(
@@ -17,15 +66,72 @@
 class TestExponentialShift(unittest.TestCase):
 
     def setUp(self):
-        self.optimizer = mock.MagicMock()
-        self.trainer = mock.MagicMock()
-        self.extension = extensions.ExponentialShift(
-            'x', self.rate, self.init, self.target, self.optimizer)
-
-    def test_call(self):
-        for e in self.expect:
-            self.extension(self.trainer)
-            self.assertEqual(self.optimizer.x, e)
+        self.trainer = _get_mocked_trainer()
+
+        self.interval = 4
+        self.expect = [e for e in self.expect for _ in range(self.interval)]
+        self.trigger = get_trigger((self.interval, 'iteration'))
+
+    def _run_trainer(self, extension, expect, optimizer=None):
+        if optimizer is None:
+            optimizer = self.trainer.updater.optimizer
+
+        if extension.invoke_before_training:
+            extension(self.trainer)
+
+        for e in expect:
+            self.trainer.updater.update()
+            self.assertEqual(optimizer.x, e)
+            if self.trigger(self.trainer):
+                extension(self.trainer)
+
+    def test_basic(self):
+        self.trainer.updater.optimizer.x = 0
+        extension = extensions.ExponentialShift(
+            'x', self.rate, init=self.init, target=self.target)
+        self._run_trainer(extension, self.expect)
+
+    def test_without_init(self):
+        self.trainer.updater.optimizer.x = self.init
+        extension = extensions.ExponentialShift(
+            'x', self.rate, target=self.target)
+        self._run_trainer(extension, self.expect)
+
+    def test_with_optimizer(self):
+        optimizer = mock.Mock()
+        optimizer.x = 0
+        extension = extensions.ExponentialShift(
+            'x', self.rate, init=self.init, target=self.target,
+            optimizer=optimizer)
+        self._run_trainer(extension, self.expect, optimizer)
+
+    def test_serialize(self):
+        self.trainer.updater.optimizer.x = 0
+        extension = extensions.ExponentialShift(
+            'x', self.rate, init=self.init, target=self.target)
+        self._run_trainer(extension, self.expect[:len(self.expect) // 2])
+        target = dict()
+        extension.serialize(DummySerializer(target))
+
+        self.trainer.updater.optimizer.x = 0
+        extension = extensions.ExponentialShift(
+            'x', self.rate, init=self.init, target=self.target)
+        extension.serialize(DummyDeserializer(target))
+        self._run_trainer(extension, self.expect[len(self.expect) // 2:])
+
+    def test_serialize_before_first_interval(self):
+        self.trainer.updater.optimizer.x = 0
+        extension = extensions.ExponentialShift(
+            'x', self.rate, init=self.init, target=self.target)
+        self._run_trainer(extension, self.expect[:self.interval - 1])
+        target = dict()
+        extension.serialize(DummySerializer(target))
+
+        self.trainer.updater.optimizer.x = 0
+        extension = extensions.ExponentialShift(
+            'x', self.rate, init=self.init, target=self.target)
+        extension.serialize(DummyDeserializer(target))
+        self._run_trainer(extension, self.expect[self.interval - 1:])
 
 
 class TestExponentialShiftInvalidArgument(unittest.TestCase):