Repository: chainer/chainer

Issue:
Implement Variational AutoEncoder
Reference : http://arxiv.org/abs/1312.6114



Discussion:
- [x] softplus: #180 
- [x] gaussian noise: #184 


Resolving patch:
diff --git a/chainer/functions/vae.py b/chainer/functions/vae.py
new file mode 100644
--- /dev/null
+++ b/chainer/functions/vae.py
@@ -0,0 +1,110 @@
+import math
+
+import chainer.functions as F
+from chainer import variable
+
+
+def gaussian_kl_divergence(mean, ln_var):
+    """Calculate KL-divergence between given gaussian and the standard one.
+
+    Given two variable ``mean`` representing :math:`\\mu` and ``ln_var``
+    representing :math:`\\log(\\sigma^2)`, this function returns a variable
+    representing KL-divergence between given multi-dimensional gaussian
+    :math:`N(\\mu, S)` and the standard Gaussian :math:`N(0, I)`
+
+    .. math::
+
+       D_{\\mathbf{KL}}(N(\\mu, S) \\| N(0, I)),
+
+    where :math:`S` is a diagonal matrix such that :math:`S_{ii} = \\sigma_i^2`
+    and :math:`I` is an identity matrix.
+
+    Args:
+        mean (~chainer.Variable): A variable representing mean of given
+            gaussian distribution, :math:`\\mu`.
+        ln_var (~chainer.Variable): A variable representing logarithm of
+            variance of given gaussian distribution, :math:`\\log(\\sigma^2)`.
+
+    Returns:
+        ~chainer.Variable: A variable representing KL-divergence between
+            given gaussian distribution and the standard gaussian.
+
+    """
+    assert isinstance(mean, variable.Variable)
+    assert isinstance(ln_var, variable.Variable)
+
+    J = mean.data.size
+    var = F.exp(ln_var)
+    return (F.sum(mean * mean) + F.sum(var) - F.sum(ln_var) - J) * 0.5
+
+
+def bernoulli_nll(x, y):
+    """Calculate negative log-likelihood of Bernoulli distribution.
+
+    This function calculates negative log-likelihood on a Bernoulli
+    distribution.
+
+    .. math::
+
+        -B(x; p) = -\\sum_i {x_i \\log(p_i) + (1 - x_i)\\log(1 - p_i)},
+
+    where :math:`p = \\sigma(y)`, and :math:`\\sigma(\\cdot)` is a sigmoid
+    funciton.
+
+    .. note::
+
+       As this funtion uses a sigmoid function, you can pass a result of
+       fully-connected layer (that means :class:`Linear`) to this function
+       directly.
+
+    Args:
+        x (~chainer.Variable): Input variable.
+        y (~chainer.Variable): A variable representing the parameter of
+            Bernoulli distribution.
+
+    Returns:
+        ~chainer.Variable: A variable representing negative log-likelihood.
+
+    """
+    assert isinstance(x, variable.Variable)
+    assert isinstance(y, variable.Variable)
+
+    return F.sum(F.softplus(-y)) + F.sum(y) - F.sum(y * x)
+
+
+def gaussian_nll(x, mean, ln_var):
+    """Calculate negative log-likelihood of Gaussian distribution.
+
+    Given two variable ``mean`` representing :math:`\\mu` and ``ln_var``
+    representing :math:`\\log(\\sigma^2)`, this function returns negative
+    log-likelihood of :math:`x` on a Gaussian distribution :math:`N(\\mu, S)`,
+
+    .. math::
+
+        -\\log N(x; \\mu, \\sigma^2) =
+        \\log\\left(\\sqrt{(2\\pi)^D |S|}\\right) +
+        \\frac{1}{2}(x - \\mu)^\\top S^{-1}(x - \\mu),
+
+    where :math:`D` is a dimention of :math:`x` and :math:`S` is a diagonal
+    matrix where :math:`S_{ii} = \\sigma_i^2`.
+
+    Args:
+        x (~chainer.Variable): Input variable.
+        mean (~chainer.Variable): A variable representing mean of a gaussian
+            distribution, :math:`\\mu`.
+        ln_var (~chainer.Variable): A variable representing logarithm of
+            variance of a gaussian distribution, :math:`\\log(\\sigma^2)`.
+
+    Returns:
+        ~chainer.Variable: A variable representing negative log-likelihood.
+
+    """
+    assert isinstance(x, variable.Variable)
+    assert isinstance(mean, variable.Variable)
+    assert isinstance(ln_var, variable.Variable)
+
+    D = x.data.size
+    x_prec = F.exp(-ln_var)
+    x_diff = x - mean
+    x_power = (x_diff * x_diff) * x_prec * -0.5
+    return (F.sum(ln_var) + D * math.log(2 * math.pi)) / 2 - F.sum(x_power)


Test patch:
diff --git a/tests/functions_tests/test_vae.py b/tests/functions_tests/test_vae.py
new file mode 100644
--- /dev/null
+++ b/tests/functions_tests/test_vae.py
@@ -0,0 +1,113 @@
+import unittest
+
+import numpy
+
+import chainer
+from chainer import cuda
+from chainer.functions import vae
+from chainer import gradient_check
+from chainer import testing
+from chainer.testing import attr
+from chainer.testing import condition
+
+
+if cuda.available:
+    cuda.init()
+
+
+class TestGaussianKLDivergence(unittest.TestCase):
+
+    def setUp(self):
+        self.mean = numpy.random.uniform(-1, 1, (3,)).astype(numpy.float32)
+        self.ln_var = numpy.random.uniform(-1, 1, (3,)).astype(numpy.float32)
+
+        # Refer to Appendix B in the original paper
+        # Auto-Encoding Variational Bayes (http://arxiv.org/abs/1312.6114)
+        J = self.mean.size
+        self.expect = -(J + numpy.sum(self.ln_var)
+                        - numpy.sum(self.mean * self.mean)
+                        - numpy.sum(numpy.exp(self.ln_var))) * 0.5
+
+    def check_gaussian_kl_divergence(self, mean, ln_var):
+        m = chainer.Variable(mean)
+        v = chainer.Variable(ln_var)
+        actual = cuda.to_cpu(vae.gaussian_kl_divergence(m, v).data)
+        gradient_check.assert_allclose(self.expect, actual)
+
+    @condition.retry(3)
+    def test_gaussian_kl_divergence_cpu(self):
+        self.check_gaussian_kl_divergence(self.mean, self.ln_var)
+
+    @attr.gpu
+    @condition.retry(3)
+    def test_gaussian_kl_divergence_gpu(self):
+        self.check_gaussian_kl_divergence(cuda.to_gpu(self.mean),
+                                          cuda.to_gpu(self.ln_var))
+
+
+class TestBernoulliNLL(unittest.TestCase):
+
+    def setUp(self):
+        self.x = numpy.random.uniform(-1, 1, (3,)).astype(numpy.float32)
+        self.y = numpy.random.uniform(-1, 1, (3,)).astype(numpy.float32)
+
+        # Refer to Appendix C.1 in the original paper
+        # Auto-Encoding Variational Bayes (http://arxiv.org/abs/1312.6114)
+        p = 1 / (1 + numpy.exp(-self.y))
+        self.expect = - (numpy.sum(self.x * numpy.log(p))
+                         + numpy.sum((1 - self.x) * numpy.log(1 - p)))
+
+    def check_bernoulli_nll(self, x_data, y_data):
+        x = chainer.Variable(x_data)
+        y = chainer.Variable(y_data)
+        actual = cuda.to_cpu(vae.bernoulli_nll(x, y).data)
+        gradient_check.assert_allclose(self.expect, actual)
+
+    @condition.retry(3)
+    def test_bernoulli_nll_cpu(self):
+        self.check_bernoulli_nll(self.x, self.y)
+
+    @attr.gpu
+    @condition.retry(3)
+    def test_bernoulli_nll_gpu(self):
+        self.check_bernoulli_nll(cuda.to_gpu(self.x),
+                                 cuda.to_gpu(self.y))
+
+
+class TestGaussianNLL(unittest.TestCase):
+
+    def setUp(self):
+        self.x = numpy.random.uniform(-1, 1, (3,)).astype(numpy.float32)
+        self.mean = numpy.random.uniform(-1, 1, (3,)).astype(numpy.float32)
+        self.ln_var = numpy.random.uniform(-1, 1, (3,)).astype(numpy.float32)
+
+        # Refer to Appendix C.2 in the original paper
+        # Auto-Encoding Variational Bayes (http://arxiv.org/abs/1312.6114)
+        D = self.x.size
+        x_d = self.x - self.mean
+        var = numpy.exp(self.ln_var)
+
+        self.expect = (0.5 * D * numpy.log(2 * numpy.pi)
+                       + 0.5 * numpy.sum(self.ln_var)
+                       + numpy.sum(x_d * x_d / var) * 0.5)
+
+    def check_gaussian_nll(self, x_data, mean_data, ln_var_data):
+        x = chainer.Variable(x_data)
+        mean = chainer.Variable(mean_data)
+        ln_var = chainer.Variable(ln_var_data)
+        actual = cuda.to_cpu(vae.gaussian_nll(x, mean, ln_var).data)
+        gradient_check.assert_allclose(self.expect, actual)
+
+    @condition.retry(3)
+    def test_gaussian_nll_cpu(self):
+        self.check_gaussian_nll(self.x, self.mean, self.ln_var)
+
+    @attr.gpu
+    @condition.retry(3)
+    def test_gaussian_nll_gpu(self):
+        self.check_gaussian_nll(cuda.to_gpu(self.x),
+                                cuda.to_gpu(self.mean),
+                                cuda.to_gpu(self.ln_var))
+
+
+testing.run_module(__name__, __file__)