Repository: GPflow/GPflow

Issue:
multivariate normal density cannot handle vectors as it claims
It seems that if `x` and `mu` are vectors, the call to `tf.matrix_triangular_solve` would fail (as `d = x - mu` is rank 1). Worse, if `x` and `mu` are of different dimensionality (e.g. one a row, one a column vector) broadcasting happens and fails silently.


Discussion:

Resolving patch:
diff --git a/gpflow/densities.py b/gpflow/densities.py
--- a/gpflow/densities.py
+++ b/gpflow/densities.py
@@ -71,18 +71,25 @@ def laplace(mu, sigma, y):
 
 def multivariate_normal(x, mu, L):
     """
-    L is the Cholesky decomposition of the covariance.
-
-    x and mu are either vectors (ndim=1) or matrices. In the matrix case, we
-    assume independence over the *columns*: the number of rows must match the
-    size of L.
+    Computes the log-density of a multivariate normal.
+    :param x  : D or DxN sample(s) for which we want the density
+    :param mu : D or DxN mean(s) of the normal distribution
+    :param L  : DxD Cholesky decomposition of the covariance matrix
+    :return p : N vector of log densities for each of the N x's and/or mu's
+
+    x and mu are either vectors or matrices. If both are vectors ((N,) or (N,1)):
+    p[0] = log pdf(x) where x ~ N(mu, LL^T)
+    If at least one is a matrix, we assume independence over the *columns*:
+    the number of rows must match the size of L. Broadcasting behaviour:
+    p[n] = log pdf of:
+    x[n] ~ N(mu, LL^T) or x ~ N(mu[n], LL^T) or x[n] ~ N(mu[n], LL^T)
     """
+    x  = tf.cond(tf.rank(x)  < 2, lambda: x[:, None],  lambda: x)
+    mu = tf.cond(tf.rank(mu) < 2, lambda: mu[:, None], lambda: mu)
     d = x - mu
     alpha = tf.matrix_triangular_solve(L, d, lower=True)
-    num_col = 1 if tf.rank(x) == 1 else tf.shape(x)[1]
-    num_col = tf.cast(num_col, settings.float_type)
-    num_dims = tf.cast(tf.shape(x)[0], settings.float_type)
-    ret = - 0.5 * num_dims * num_col * np.log(2 * np.pi)
-    ret += - num_col * tf.reduce_sum(tf.log(tf.matrix_diag_part(L)))
-    ret += - 0.5 * tf.reduce_sum(tf.square(alpha))
-    return ret
+    num_dims = tf.cast(tf.shape(d)[0], L.dtype)
+    p = - 0.5 * tf.reduce_sum(tf.square(alpha), 0)
+    p -= 0.5 * num_dims * np.log(2 * np.pi)
+    p -= tf.reduce_sum(tf.log(tf.matrix_diag_part(L)))
+    return p
diff --git a/gpflow/models/gpr.py b/gpflow/models/gpr.py
--- a/gpflow/models/gpr.py
+++ b/gpflow/models/gpr.py
@@ -63,8 +63,9 @@ def _build_likelihood(self):
         K = self.kern.K(self.X) + tf.eye(tf.shape(self.X)[0], dtype=settings.float_type) * self.likelihood.variance
         L = tf.cholesky(K)
         m = self.mean_function(self.X)
+        logpdf = multivariate_normal(self.Y, m, L)  # (R,) log-likelihoods for each independent dimension of Y
 
-        return multivariate_normal(self.Y, m, L)
+        return tf.reduce_sum(logpdf)
 
     @name_scope('predict')
     @params_as_tensors


Test patch:
diff --git a/tests/test_densities.py b/tests/test_densities.py
new file mode 100644
--- /dev/null
+++ b/tests/test_densities.py
@@ -0,0 +1,44 @@
+# Copyright 2018 the GPflow authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.from __future__ import print_function
+
+import numpy as np
+from numpy.random import randn
+import tensorflow as tf
+import pytest
+import gpflow
+from gpflow import densities
+from gpflow.test_util import session_tf
+from scipy.stats import multivariate_normal as mvn
+from numpy.testing import assert_allclose
+
+
+rng = np.random.RandomState(1)
+
+
+@pytest.mark.parametrize("x", [randn(4,10), randn(4,1), randn(4)])
+@pytest.mark.parametrize("mu", [randn(4,10), randn(4,1), randn(4)])
+@pytest.mark.parametrize("cov", [randn(4,4), np.eye(4)])
+def test_multivariate_normal(session_tf, x, mu, cov):
+    cov = np.dot(cov, cov.T)
+    L = np.linalg.cholesky(cov)
+    gp_result = densities.multivariate_normal(x, mu, L)
+    gp_result = session_tf.run(gp_result)
+    if len(mu.shape) > 1 and mu.shape[1] > 1:
+        if len(x.shape) > 1 and x.shape[1] > 1:
+            sp_result = [mvn.logpdf(x[:,i], mu[:,i], cov) for i in range(mu.shape[1])]
+        else:
+            sp_result = [mvn.logpdf(x.ravel(), mu[:, i], cov) for i in range(mu.shape[1])]
+    else:
+        sp_result = mvn.logpdf(x.T, mu.ravel(), cov)
+    assert_allclose(gp_result, sp_result)