Repository: chainer/chainer

Issue:
Use CUDNN_BN_MIN_EPSILON
`cudnn.h` defines `CUDNN_BN_MIN_EPSILON`.
Remove magic number.
https://github.com/pfnet/chainer/blob/ef14cb9c3c59ca0bf8eb253f79357769e2c290c1/chainer/functions/normalization/batch_normalization.py#L44

Rel #1955.


Discussion:
This issue has been automatically marked as stale because it has not had recent activity. It will be closed after 30 days if no further activity occurs. Thank you for your contributions.

This issue is closed as announced. Feel free to re-open it if needed.

This issue has been automatically marked as stale because it has not had recent activity. It will be closed after 30 days if no further activity occurs. Thank you for your contributions.

We have to fix CuPy.

Resolving patch:
diff --git a/chainer/functions/normalization/batch_normalization.py b/chainer/functions/normalization/batch_normalization.py
--- a/chainer/functions/normalization/batch_normalization.py
+++ b/chainer/functions/normalization/batch_normalization.py
@@ -24,15 +24,16 @@ def __init__(self, eps=2e-5, mean=None, var=None, decay=0.9):
         self.running_mean = mean
         self.running_var = var
 
-        # Note: cuDNN v5 requires that eps be greater than 1e-5. Otherwise, an
-        # error will occur.
+        # Note: cuDNN requires that eps be greater than or equals to
+        # CUDNN_BN_MIN_EPSILON. Otherwise, an error will occur.
         # See CUDNN_BN_MIN_EPSILON value in cudnn.h to verify minimum allowable
         # value.
         self.eps = eps
         if chainer.should_use_cudnn('>=auto'):
-            if eps < 1e-5:
-                msg = 'cuDNN does not allow an eps value less than 1e-5.'
-                raise RuntimeError(msg)
+            if eps < libcudnn.CUDNN_BN_MIN_EPSILON:
+                raise RuntimeError(
+                    'cuDNN does not allow an eps value '
+                    'less than {}.'.format(libcudnn.CUDNN_BN_MIN_EPSILON))
         self.decay = decay
 
     def check_type_forward(self, in_types):


Test patch:
diff --git a/tests/chainer_tests/functions_tests/normalization_tests/test_batch_normalization.py b/tests/chainer_tests/functions_tests/normalization_tests/test_batch_normalization.py
--- a/tests/chainer_tests/functions_tests/normalization_tests/test_batch_normalization.py
+++ b/tests/chainer_tests/functions_tests/normalization_tests/test_batch_normalization.py
@@ -343,4 +343,24 @@ def test_call_cudnn_backward(self):
                 self.assertEqual(func.called, self.expect)
 
 
+@attr.cudnn
+class TestBatchNormalizationCudnnEps(unittest.TestCase):
+    def setUp(self):
+        ndim = 0
+        param_shape = (3,)
+        dtype = numpy.float32
+        gamma = cuda.cupy.random.uniform(.5, 1, param_shape).astype(dtype)
+        beta = cuda.cupy.random.uniform(-1, 1, param_shape).astype(dtype)
+        shape = (7,) + param_shape + (2,) * ndim
+        x = cuda.cupy.random.uniform(-1, 1, shape).astype(dtype)
+        self.args = [x, gamma, beta]
+
+    def test_valid(self):
+        functions.batch_normalization(*self.args, eps=1e-5)
+
+    def test_invalid(self):
+        with self.assertRaises(RuntimeError):
+            functions.batch_normalization(*self.args, eps=2e-6)
+
+
 testing.run_module(__name__, __file__)