Repository: chainer/chainer

Issue:
Add unit test for Optimizer
Related to #5

As discussed in #63, we open new issues about implementation of unit tests separately for each remaining core modules.

This issue is to add unittest for `Optimizer`, especially,
- `Optimizer.setup`
- `Optimizer.init_state`
- `Optimizer.zero_grads`
- `Optimizer.compute_grads_norm`
- `Optimizer.clip_grads`
- `Optimizer.weight_decay`
- `Optimizer.accumulate_grads`
- `Optimizer.update`



Discussion:

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -19,5 +19,6 @@
                       'numpy',
                       'protobuf',
                       'six>=1.9.0'],
-    tests_require=['nose'],
+    tests_require=['mock',
+                   'nose'],
 )


Test patch:
diff --git a/tests/test_optimizer.py b/tests/test_optimizer.py
--- a/tests/test_optimizer.py
+++ b/tests/test_optimizer.py
@@ -1,7 +1,9 @@
 import unittest
 
+import mock
 import numpy as np
 
+import chainer
 from chainer import cuda
 from chainer import gradient_check
 from chainer import optimizer
@@ -38,7 +40,7 @@ def test_sqnorm_scalar_gpu(self):
         self.assertAlmostEqual(optimizer._sqnorm(a), 4)
 
 
-class TestOptimizer(unittest.TestCase):
+class TestOptimizerWeightDecay(unittest.TestCase):
 
     def setUp(self):
         self.w = np.arange(6, dtype=np.float32).reshape(2, 3)
@@ -63,4 +65,132 @@ def test_weight_decay_gpu(self):
         self.check_weight_decay(cuda.to_gpu(self.w), cuda.to_gpu(self.g))
 
 
+class TestOptimizer(unittest.TestCase):
+
+    def _suffix(self, gpu):
+        if gpu:
+            return 'gpu'
+        else:
+            return 'cpu'
+
+    def _get_method(self, prefix, gpu):
+        return getattr(self.optimizer, prefix + '_' + self._suffix(gpu))
+
+    def setUp(self):
+        opt = chainer.Optimizer()
+        opt.init_state_cpu = mock.MagicMock(return_value=1)
+        opt.init_state_gpu = mock.MagicMock(return_value=1)
+        opt.update_one_cpu = mock.MagicMock()
+        opt.update_one_gpu = mock.MagicMock()
+        self.optimizer = opt
+
+        self.params = [np.arange(3).astype(np.float32)]
+        self.grads = [np.arange(3).astype(np.float32)]
+
+    def setup_cpu(self):
+        self.optimizer.setup((self.params, self.grads))
+
+    def setup_gpu(self):
+        self.params = list(map(cuda.to_gpu, self.params))
+        self.grads = list(map(cuda.to_gpu, self.grads))
+        self.optimizer.setup((self.params, self.grads))
+
+    def check_init_state(self, param, grad, gpu):
+        state = self.optimizer.init_state(param, grad)
+
+        self.assertEqual(state, 1)
+        self._get_method('init_state', gpu).assert_called_once_with(
+            param, grad)
+        self.assertEqual(self._get_method('init_state', not gpu).call_count, 0)
+
+    def test_init_state_cpu(self):
+        param = np.arange(3)
+        grad = np.arange(3)
+        self.check_init_state(param, grad, False)
+
+    @attr.gpu
+    def test_init_state_gpu(self):
+        param = cuda.to_gpu(np.arange(3))
+        grad = cuda.to_gpu(np.arange(3))
+        self.check_init_state(param, grad, True)
+
+    def check_update(self, gpu):
+        self.assertEqual(self.optimizer.t, 0)
+
+        self.optimizer.update()
+        self.assertEqual(self.optimizer.t, 1)
+
+        self._get_method('update_one', gpu).assert_called_once_with(
+            self.params[0], self.grads[0], 1)
+        self.assertEqual(self._get_method('update_one', not gpu).call_count, 0)
+
+        self.optimizer.zero_grads()
+        self.assertTrue((cuda.to_cpu(self.grads[0]) == 0).all())
+
+    def test_update_cpu(self):
+        self.setup_cpu()
+        self.check_update(False)
+
+    @attr.gpu
+    def test_update_gpu(self):
+        self.setup_gpu()
+        self.check_update(True)
+
+    def check_accumulate_grads(self):
+        self.optimizer.accumulate_grads([np.arange(3)])
+        self.assertTrue((cuda.to_cpu(self.grads[0]) == np.arange(3) * 2).all())
+
+    def test_accumulate_grads_cpu(self):
+        self.setup_cpu()
+        self.check_accumulate_grads()
+
+    @attr.gpu
+    def test_accumulate_grads_gpu(self):
+        self.setup_gpu()
+        self.check_accumulate_grads()
+
+    def check_compute_grads_norm(self):
+        norm = self.optimizer.compute_grads_norm()
+        self.assertAlmostEqual(norm, np.sqrt(5))
+
+    def test_compute_grads_norm_cpu(self):
+        self.setup_cpu()
+        self.check_compute_grads_norm()
+
+    @attr.gpu
+    def test_compute_grads_norm_gpu(self):
+        self.setup_gpu()
+        self.check_compute_grads_norm()
+
+    def check_weight_decay(self):
+        self.optimizer.weight_decay(0.1)
+        g = cuda.to_cpu(self.grads[0])
+        expect = np.array([0.0, 1.1, 2.2], dtype=np.float32)
+        gradient_check.assert_allclose(g, expect)
+
+    def test_weight_decay_cpu(self):
+        self.setup_cpu()
+        self.check_weight_decay()
+
+    @attr.gpu
+    def test_weight_decay_gpu(self):
+        self.setup_gpu()
+        self.check_weight_decay()
+
+    def check_clip_grads(self):
+        self.optimizer.clip_grads(1.0)
+        g = cuda.to_cpu(self.grads[0])
+        sqnorm = g.dot(g)
+        self.assertAlmostEqual(sqnorm, 1.0, delta=1.0e-5)
+
+    def test_clip_grads_cpu(self):
+        self.setup_cpu()
+        self.check_clip_grads()
+
+    @attr.gpu
+    def test_clip_grads_gpu(self):
+        self.setup_gpu()
+        self.check_clip_grads()
+
+
 testing.run_module(__name__, __file__)