Repository: chainer/chainer

Issue:
ELU activation function
Djork-Arné Clevert, Thomas Unterthiner and Sepp Hochreiter propose a new activation function which seems to work well:

Fast and Accurate Deep Network Learning by Exponential Linear Units (ELUs)
http://arxiv.org/abs/1511.07289

The ELU (eq. 16) and its gradient (eq. 17) should be very easy to implement.



Discussion:
somebody already did the work: https://github.com/muupan/chainer-elu
I agree that it should be included.

I did the work, but not yet examined its performance. It would be great if anyone can reproduce some of the results in the paper. I'll willingly make a PR if it should be included.


Resolving patch:
diff --git a/chainer/functions/__init__.py b/chainer/functions/__init__.py
--- a/chainer/functions/__init__.py
+++ b/chainer/functions/__init__.py
@@ -1,6 +1,7 @@
 """Collection of :class:`~chainer.Function` implementations."""
 
 from chainer.functions.activation import clipped_relu
+from chainer.functions.activation import elu
 from chainer.functions.activation import leaky_relu
 from chainer.functions.activation import lstm
 from chainer.functions.activation import prelu
@@ -66,6 +67,8 @@
 ConnectionistTemporalClassification = ctc.ConnectionistTemporalClassification
 connectionist_temporal_classification \
     = ctc.connectionist_temporal_classification
+ELU = elu.ELU
+elu = elu.elu
 LeakyReLU = leaky_relu.LeakyReLU
 leaky_relu = leaky_relu.leaky_relu
 LSTM = lstm.LSTM
diff --git a/chainer/functions/activation/elu.py b/chainer/functions/activation/elu.py
new file mode 100644
--- /dev/null
+++ b/chainer/functions/activation/elu.py
@@ -0,0 +1,72 @@
+import numpy
+
+from chainer import cuda
+from chainer import function
+from chainer.utils import type_check
+
+
+class ELU(function.Function):
+
+    """Exponential Linear Unit."""
+
+    def __init__(self, alpha=1.0):
+        self.alpha = numpy.float32(alpha)
+
+    def check_type_forward(self, in_types):
+        type_check.expect(in_types.size() == 1)
+        x_type, = in_types
+
+        type_check.expect(
+            x_type.dtype == numpy.float32,
+        )
+
+    def forward_cpu(self, x):
+        y = x[0].copy()
+        neg_indices = x[0] < 0
+        y[neg_indices] = self.alpha * (numpy.exp(y[neg_indices]) - 1)
+        return y,
+
+    def forward_gpu(self, x):
+        y = cuda.elementwise(
+            'T x, T alpha', 'T y',
+            'y = x >= 0 ? x : alpha * (exp(x) - 1)', 'elu_fwd')(
+                x[0], self.alpha)
+        return y,
+
+    def backward_cpu(self, x, gy):
+        gx = gy[0].copy()
+        neg_indices = x[0] < 0
+        gx[neg_indices] *= self.alpha * numpy.exp(x[0][neg_indices])
+        return gx,
+
+    def backward_gpu(self, x, gy):
+        gx = cuda.elementwise(
+            'T x, T gy, T alpha', 'T gx',
+            'gx = x >= 0 ? gy : gy * alpha * exp(x)', 'elu_bwd')(
+                x[0], gy[0], self.alpha)
+        return gx,
+
+
+def elu(x, alpha=1.0):
+    """Exponential Linear Unit function.
+
+    This function is expressed as
+
+    .. math::
+        f(x) = \\left \\{ \\begin{array}{ll}
+        x & {\\rm if}~ x \\ge 0 \\\\
+        \\alpha (\\exp(x) - 1) & {\\rm if}~ x < 0,
+        \\end{array} \\right.
+
+    where :math:`\\alpha` is a parameter.
+    See: http://arxiv.org/abs/1511.07289
+
+    Args:
+        x (~chainer.Variable): Input variable.
+        alpha (float): Parameter :math:`\\alpha`.
+
+    Returns:
+        ~chainer.Variable: Output variable.
+
+    """
+    return ELU(alpha=alpha)(x)


Test patch:
diff --git a/tests/chainer_tests/functions_tests/activation_tests/test_elu.py b/tests/chainer_tests/functions_tests/activation_tests/test_elu.py
new file mode 100644
--- /dev/null
+++ b/tests/chainer_tests/functions_tests/activation_tests/test_elu.py
@@ -0,0 +1,69 @@
+import random
+import unittest
+
+import numpy
+
+import chainer
+from chainer import cuda
+from chainer import functions
+from chainer import gradient_check
+from chainer import testing
+from chainer.testing import attr
+from chainer.testing import condition
+
+
+class TestELU(unittest.TestCase):
+
+    def setUp(self):
+        # Avoid unstability of numeraical grad
+        self.x = numpy.random.uniform(-1, 1, (5, 4)).astype(numpy.float32)
+        for i in range(self.x.size):
+            if -0.01 < self.x.flat[i] < 0.01:
+                self.x.flat[i] = 0.5
+        self.gy = numpy.random.uniform(-1, 1, (5, 4)).astype(numpy.float32)
+        self.alpha = random.random()
+
+    def check_forward(self, x_data):
+        x = chainer.Variable(x_data)
+        y = functions.elu(x, alpha=self.alpha)
+        self.assertEqual(y.data.dtype, numpy.float32)
+
+        expected = self.x.copy()
+        for i in numpy.ndindex(self.x.shape):
+            if self.x[i] < 0:
+                expected[i] = self.alpha * (numpy.exp(expected[i]) - 1)
+
+        gradient_check.assert_allclose(expected, y.data)
+
+    @condition.retry(3)
+    def test_forward_cpu(self):
+        self.check_forward(self.x)
+
+    @attr.gpu
+    @condition.retry(3)
+    def test_forward_gpu(self):
+        self.check_forward(cuda.to_gpu(self.x))
+
+    def check_backward(self, x_data, y_grad):
+        x = chainer.Variable(x_data)
+        y = functions.elu(x, alpha=self.alpha)
+        y.grad = y_grad
+        y.backward()
+
+        func = y.creator
+        f = lambda: func.forward((x.data,))
+        gx, = gradient_check.numerical_grad(f, (x.data,), (y.grad,))
+
+        gradient_check.assert_allclose(gx, x.grad)
+
+    @condition.retry(3)
+    def test_backward_cpu(self):
+        self.check_backward(self.x, self.gy)
+
+    @attr.gpu
+    @condition.retry(3)
+    def test_backward_gpu(self):
+        self.check_backward(cuda.to_gpu(self.x), cuda.to_gpu(self.gy))
+
+
+testing.run_module(__name__, __file__)