Repository: GPflow/GPflow

Issue:
SyntaxError when importing GPflow
### Please make sure this is a bug, performance or build issue.

### Please fill out the template below.
___

System information:

* Python version:  3.5
* TensorFlow installed from (source or binary):  binary (pip3)
* TensorFlow version (use command below):  1.14
* GPflow installed from (source or binary):  source
* GPflow version:  1.15

> You can collect some of this information using our environment capture script You can also obtain the TensorFlow version with python -c "import tensorflow as tf; print(tf.GIT_VERSION, tf.VERSION)"

... _Describe the current behavior_
Import gpflow gives an error and exits the program.

... _Describe the expected behavior_
No error expected.

... _Code to reproduce the issue Provide a reproducible test case that is the bare minimum necessary to generate the problem._
    import gpflow

.. _Other info / logs, include any logs or source code that would be helpful to diagnose the problem. If including tracebacks, please include the full traceback. Large logs and files should be attached._

    File "/home/miquel/git/pybpslam/gaussian_process.py", line 1, in <module>
        import gpflow
    File "/usr/local/lib/python3.5/dist-packages/gpflow-1.5.0-py3.5.egg/gpflow/__init__.py", line 36, in <module>
        from . import training as train
    File "/usr/local/lib/python3.5/dist-packages/gpflow-1.5.0-py3.5.egg/gpflow/training/__init__.py", line 3, in <module>
        from .scipy_optimizer import ScipyOptimizer
    File "/usr/local/lib/python3.5/dist-packages/gpflow-1.5.0-py3.5.egg/gpflow/training/scipy_optimizer.py", line 117
        msg = f"Original error message: \n\t{error}\nOptimiser message: {msg}."
                                                                              ^
    SyntaxError: invalid syntax



Discussion:

Resolving patch:
diff --git a/gpflow/kernels.py b/gpflow/kernels.py
--- a/gpflow/kernels.py
+++ b/gpflow/kernels.py
@@ -526,7 +526,7 @@ class Matern32(Stationary):
     The Matern 3/2 kernel. Functions drawn from a GP with this kernel are once
     differentiable. The kernel equation is
 
-    k(r) =  σ² (1 + √3r) exp{-√3 r}
+    k(r) = σ² (1 + √3r) exp{-√3 r}
 
     where:
     r  is the Euclidean distance between the input points, scaled by the lengthscale parameter ℓ,
@@ -544,7 +544,7 @@ class Matern52(Stationary):
     The Matern 5/2 kernel. Functions drawn from a GP with this kernel are twice
     differentiable. The kernel equation is
 
-    k(r) =  σ² (1 + √5r + 5/3r²) exp{-√5 r}
+    k(r) = σ² (1 + √5r + 5/3r²) exp{-√5 r}
 
     where:
     r  is the Euclidean distance between the input points, scaled by the lengthscale parameter ℓ,
@@ -683,42 +683,72 @@ def Kdiag(self, X, presliced=False):
 
 class Periodic(Kernel):
     """
-    The periodic kernel. Defined in  Equation (47) of
+    The periodic family of kernels. Can be used to wrap any Stationary kernel
+    to transform it into a periodic version. The canonical form (based on the
+    SquaredExponential kernel) can be found in Equation (47) of
 
     D.J.C.MacKay. Introduction to Gaussian processes. In C.M.Bishop, editor,
     Neural Networks and Machine Learning, pages 133--165. Springer, 1998.
 
-    Derived using an RBF kernel once mapped the original inputs through
-    the mapping u=(cos(x), sin(x)).
+    The derivation can be achieved by mapping the original inputs through the
+    transformation u = (cos(x), sin(x)).
 
-    The resulting kernel can be expressed as:
-        k(r) =  σ² exp{ -0.5 sin²(π r / γ) / ℓ²}
+    For the SquaredExponential base kernel, the result can be expressed as:
+        k(r) = σ² exp{ -0.5 sin²(π r / γ) / ℓ² }
 
     where:
-    r  is the Euclidean distance between the input points
+    r is the Euclidean distance between the input points,
     ℓ is the lengthscale parameter,
     σ² is the variance parameter,
     γ is the period parameter.
 
-    (note that usually we have a factor of 4 instead of 0.5 in front but this is absorbed into lengthscale
-    hyperparameter).
+    (note that usually we have a factor of 4 instead of 0.5 in front but this
+    is absorbed into lengthscale hyperparameter).
     """
+    def __init__(self, input_dim=None, period=1.0, variance=1.0,
+                 lengthscales=1.0, base=None, active_dims=None, name=None):
+        """
+        The Periodic kernel supports the specification of any Stationary kernel
+        as the base through the "base" parameter. If this is provided then
+        input_dim, variance, lengthscales and active_dims parameters are
+        taken from there.
+
+        If a "base" kernel is not provided, then a SquaredExponential is
+        constructed implicitly using the input_dim, variance, lengthscales
+        and active_dims parameters.
+        """
+        if (base is not None) and (input_dim is not None):
+            raise ValueError("input_dim should be defined through the base kernel, not explicitly")
+
+        if base is None:
+            warnings.warn(
+                'Implicit specification of the base kernel for Periodic is '
+                'deprecated, use Periodic(base=SquaredExponential({i}, '
+                'variance={v}, lengthscales={l}, active_dims={a}), period={p}) '
+                'instead.'.format(i=input_dim, v=variance, l=lengthscales,
+                                  a=active_dims, p=period),
+                DeprecationWarning)
+            base = SquaredExponential(
+                input_dim,
+                variance=variance,
+                lengthscales=lengthscales,
+                active_dims=active_dims)
+
+        if not isinstance(base, Stationary):
+            raise TypeError("Periodic requires a Stationary kernel as the `base`")
+
+        super().__init__(base.input_dim, base.active_dims, name=name)
+        self.base = base
+        self.period = Parameter(  # No ARD support for period yet
+            period, transform=transforms.positive, dtype=settings.float_type)
 
-    def __init__(self, input_dim, period=1.0, variance=1.0,
-                 lengthscales=1.0, active_dims=None, name=None):
-        # No ARD support for lengthscale or period yet
-        super().__init__(input_dim, active_dims, name=name)
-        self.variance = Parameter(variance, transform=transforms.positive,
-                                  dtype=settings.float_type)
-        self.lengthscales = Parameter(lengthscales, transform=transforms.positive,
-                                      dtype=settings.float_type)
-        self.ARD = False
-        self.period = Parameter(period, transform=transforms.positive,
-                                dtype=settings.float_type)
+    @property
+    def ARD(self):
+        return self.base.ARD
 
     @params_as_tensors
     def Kdiag(self, X, presliced=False):
-        return tf.fill(tf.shape(X)[:-1], tf.squeeze(self.variance))
+        return self.base.Kdiag(X)
 
     @params_as_tensors
     def K(self, X, X2=None, presliced=False):
@@ -732,9 +762,14 @@ def K(self, X, X2=None, presliced=False):
         f2 = tf.expand_dims(X2, -3)  # ... x 1 x M x D
 
         r = np.pi * (f - f2) / self.period
-        r = tf.reduce_sum(tf.square(tf.sin(r) / self.lengthscales), -1)
-
-        return self.variance * tf.exp(-0.5 * r)
+        scaled_sine = tf.sin(r) / self.base.lengthscales
+        try:
+            sine_r = tf.reduce_sum(tf.abs(scaled_sine), -1)
+            K = self.base.K_r(sine_r)
+        except NotImplementedError:
+            sine_r2 = tf.reduce_sum(tf.square(scaled_sine), -1)
+            K = self.base.K_r2(sine_r2)
+        return K
 
 
 class Coregion(Kernel):
diff --git a/gpflow/training/monitor.py b/gpflow/training/monitor.py
--- a/gpflow/training/monitor.py
+++ b/gpflow/training/monitor.py
@@ -1050,21 +1050,21 @@ def __init__(self, file_writer: LogdirWriter, func: Callable[[], Figure], func_n
 
         super().__init__(file_writer)
         self.func = func
-        self.placeholder = tf.placeholder(tf.float64, [1, None, None, None])
-        self._summary = tf.summary.image(func_name, self.placeholder)
 
-    def run(self, context: MonitorContext, *args, **kwargs) -> None:
+        self.buf_placeholder = tf.placeholder(tf.string)
+        # Create TF image and load its content from the buffer:
+        image_tensor = tf.image.decode_png(self.buf_placeholder)[None, ...]
+        # needs to have shape [batch_size=1, height, width, channels]
+        self._summary = tf.summary.image(func_name, image_tensor)
 
-        # Get the image and write it into a buffer in the PNG format.
+    def run(self, context: MonitorContext, *args, **kwargs) -> None:
+        # Get the image and write it into a buffer in the PNG format:
         fig = self.func()
         buf = io.BytesIO()
         fig.savefig(buf, format='png', bbox_inches='tight')
         plt.close(fig)
 
-        # Create TF image and load its content from the buffer.
+        # Turn buffer into TF image:
         buf.seek(0)
-        image = tf.image.decode_png(buf.getvalue())
-        # Add the image number as a new dimension
-        image = context.session.run(tf.expand_dims(image, 0))
-
-        self._eval_summary(context, {self.placeholder: image})
+        feed_dict = {self.buf_placeholder: buf.getvalue()}
+        self._eval_summary(context, feed_dict)
diff --git a/gpflow/training/scipy_optimizer.py b/gpflow/training/scipy_optimizer.py
--- a/gpflow/training/scipy_optimizer.py
+++ b/gpflow/training/scipy_optimizer.py
@@ -112,13 +112,19 @@ def minimize(self,
                 self._optimizer.optimizer_kwargs.update(options)
                 self._optimizer.optimize(session=session, feed_dict=feed_dict, step_callback=step_callback)
             except InvalidArgumentError as error:
-                msg = ("This error might occur because the internal state (for example, variables shape or dtype) of the model is changed. "
+                msg = ("This error might occur because the internal state "
+                       "(for example, variables shape or dtype) of the model "
+                       "is changed. "
                        "In this case, you have to use a new optimiser")
-                msg = f"Original error message: \n\t{error}\nOptimiser message: {msg}."
+                msg = "Original error message: \n\t{error}\nOptimiser message: {msg}.".format(
+                    error=error, msg=msg)
                 raise RuntimeError(msg)
             except Exception as error:
-                msg = "Unknown error has occured at reusage of the scipy optimizer. Make sure that you use the same session"
-                msg = f"Original error message: \n\t{error}\nOptimiser message: {msg}."
+                msg = ("Unknown error has occured at reusage of the "
+                       "ScipyOptimizer. Make sure that you use the same "
+                       "TensorFlow session.")
+                msg = "Original error message: \n\t{error}\nOptimiser message: {msg}.".format(
+                    error=error, msg=msg)
                 raise RuntimeError(msg)
         else:
             self._optimizer.minimize(session=session, feed_dict=feed_dict, step_callback=step_callback, **kwargs)


Test patch:
diff --git a/tests/reference.py b/tests/reference.py
--- a/tests/reference.py
+++ b/tests/reference.py
@@ -1,5 +1,6 @@
 import numpy as np
 
+
 def referenceRbfKernel(X, lengthScale, signalVariance):
     nDataPoints, _ = X.shape
     kernel = np.zeros((nDataPoints, nDataPoints))
@@ -13,7 +14,7 @@ def referenceRbfKernel(X, lengthScale, signalVariance):
     return kernel
 
 
-def referenceArcCosineKernel( X, order, weightVariances, biasVariance, signalVariance ):
+def referenceArcCosineKernel(X, order, weightVariances, biasVariance, signalVariance):
     num_points = X.shape[0]
     kernel = np.empty((num_points, num_points))
     for row in range(num_points):
@@ -42,8 +43,21 @@ def referenceArcCosineKernel( X, order, weightVariances, biasVariance, signalVar
     return kernel
 
 
-def referencePeriodicKernel( X, lengthScale, signalVariance, period ):
+def referencePeriodicKernel(X, lengthScale, signalVariance, period, baseClassName="RBF"):
     # Based on the GPy implementation of standard_period kernel
+    lengthScale = np.array(lengthScale)
     base = np.pi * (X[:, None, :] - X[None, :, :]) / period
-    exp_dist = np.exp( -0.5* np.sum( np.square(  np.sin( base ) / lengthScale ), axis = -1 ) )
-    return signalVariance * exp_dist
+    sine_base = np.sin(base) / lengthScale[None, None, :]
+    if baseClassName in {"RBF", "SquaredExponential"}:
+        dist = 0.5 * np.sum(np.square(sine_base), axis=-1)
+        K = np.exp(-dist)
+    elif baseClassName == "Matern12":
+        dist = np.sum(np.abs(sine_base), axis=-1)
+        K = np.exp(-dist)
+    elif baseClassName == "Matern32":
+        dist = np.sqrt(3) * np.sum(np.abs(sine_base), axis=-1)
+        K = (1 + dist) * np.exp(-dist)
+    elif baseClassName == "Matern52":
+        dist = np.sqrt(5) * np.sum(np.abs(sine_base), axis=-1)
+        K = (1 + dist + dist ** 2 / 3) * np.exp(-dist)
+    return signalVariance * K
diff --git a/tests/test_kerns.py b/tests/test_kerns.py
--- a/tests/test_kerns.py
+++ b/tests/test_kerns.py
@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from functools import partial
 import tensorflow as tf
 
 import numpy as np
@@ -170,12 +171,11 @@ def setUp(self):
 
     def evalKernelError(self, D, lengthscale, variance, period, X_data):
         with self.test_context() as session:
-            kernel = gpflow.kernels.Periodic(
-                D, period=period, variance=variance, lengthscales=lengthscale)
-
+            kernel = self._init_kernel(D, lengthscale, variance, period)
             X = tf.placeholder(gpflow.settings.float_type)
             reference_gram_matrix = referencePeriodicKernel(
-                X_data, lengthscale, variance, period)
+                X_data, lengthscale, variance, period,
+                baseClassName=kernel.base.__class__.__name__)
             kernel.compile()
             gram_matrix = session.run(kernel.K(X), feed_dict={X: X_data})
             assert_allclose(gram_matrix, reference_gram_matrix)
@@ -183,23 +183,75 @@ def evalKernelError(self, D, lengthscale, variance, period, X_data):
     def test_1d(self):
         with self.test_context():
             D = 1
-            lengthScale = 2.0
+            lengthscale = [2.0]
             variance = 2.3
             period = 2.
             rng = np.random.RandomState(1)
             X_data = rng.randn(3, 1)
-            self.evalKernelError(D, lengthScale, variance, period, X_data)
+            self.evalKernelError(D, lengthscale, variance, period, X_data)
 
     def test_2d(self):
         with self.test_context():
             D = 2
             N = 5
-            lengthScale = 11.5
+            lengthscale = [11.5]
+            variance = 1.3
+            period = 20.
+            rng = np.random.RandomState(1)
+            X_data = rng.multivariate_normal(np.zeros(D), np.eye(D), N)
+            self.evalKernelError(D, lengthscale, variance, period, X_data)
+
+    def test_2d_with_ARD(self):
+        with self.test_context():
+            D = 2
+            N = 5
+            lengthscale = [11.5, 5.2]
             variance = 1.3
             period = 20.
             rng = np.random.RandomState(1)
             X_data = rng.multivariate_normal(np.zeros(D), np.eye(D), N)
-            self.evalKernelError(D, lengthScale, variance, period, X_data)
+            self.evalKernelError(D, lengthscale, variance, period, X_data)
+
+    def test_init_with_non_stationary_base(self):
+        msg = "Periodic requires a Stationary kernel as the `base`"
+        with self.assertRaisesWithLiteralMatch(TypeError, msg):
+            gpflow.kernels.Periodic(base=gpflow.kernels.Linear(1))
+
+    def test_init_with_unnecessary_input_dim(self):
+        msg = "input_dim should be defined through the base kernel, not explicitly"
+        with self.assertRaisesWithLiteralMatch(ValueError, msg):
+            gpflow.kernels.Periodic(1, base=gpflow.kernels.SquaredExponential(1))
+
+    def test_ard(self):
+        k1 = gpflow.kernels.Periodic(base=gpflow.kernels.SquaredExponential(1, lengthscales=1.))
+        k2 = gpflow.kernels.Periodic(base=gpflow.kernels.SquaredExponential(2, lengthscales=1.))
+        k3 = gpflow.kernels.Periodic(base=gpflow.kernels.SquaredExponential(2, lengthscales=[1., 1.]))
+        self.assertFalse(k1.ARD)
+        self.assertFalse(k2.ARD)
+        self.assertTrue(k3.ARD)
+
+    def _init_kernel(self, D, lengthscale, variance, period):
+        # Ensure backward compatibility
+        return gpflow.kernels.Periodic(
+            D, period=period, variance=variance, lengthscales=lengthscale)
+
+
+class TestPeriodicMatern12(TestPeriodic):
+    def _init_kernel(self, D, lengthscale, variance, period):
+        base = gpflow.kernels.Matern12(D, variance=variance, lengthscales=lengthscale)
+        return gpflow.kernels.Periodic(base=base, period=period)
+
+
+class TestPeriodicMatern32(TestPeriodic):
+    def _init_kernel(self, D, lengthscale, variance, period):
+        base = gpflow.kernels.Matern32(D, variance=variance, lengthscales=lengthscale)
+        return gpflow.kernels.Periodic(base=base, period=period)
+
+
+class TestPeriodicMatern52(TestPeriodic):
+    def _init_kernel(self, D, lengthscale, variance, period):
+        base = gpflow.kernels.Matern52(D, variance=variance, lengthscales=lengthscale)
+        return gpflow.kernels.Periodic(base=base, period=period)
 
 
 class TestCoregion(GPflowTestCase):
diff --git a/tests/test_likelihoods.py b/tests/test_likelihoods.py
--- a/tests/test_likelihoods.py
+++ b/tests/test_likelihoods.py
@@ -26,8 +26,12 @@
 class LikelihoodSetup(object):
     def __init__(self, likelihood, Y, tolerance):
         self.likelihood, self.Y, self.tolerance = likelihood, Y, tolerance
-        self.is_analytic = likelihood.predict_density is not \
-                           gpflow.likelihoods.Likelihood.predict_density
+
+    def is_analytic(self, method_name):
+        actual_method = getattr(self.likelihood, method_name)
+        quadrature_fallback = getattr(gpflow.likelihoods.Likelihood, method_name)
+        assert not isinstance(self.likelihood, gpflow.likelihoods.MonteCarloLikelihood)
+        return actual_method is not quadrature_fallback
 
 
 def getLikelihoodSetups(includeMultiClass=True, addNonStandardLinks=False):
@@ -148,7 +152,7 @@ def test_var_exp(self):
         for test_setup in self.test_setups:
             with self.test_context() as session:
                 # get all the likelihoods where variational expectations has been overwritten
-                if not test_setup.is_analytic:
+                if not test_setup.is_analytic("variational_expectations"):
                     continue
                 l = test_setup.likelihood
                 y = test_setup.Y
@@ -166,7 +170,7 @@ def test_pred_density(self):
         # get all the likelihoods where predict_density has been overwritten.
         for test_setup in self.test_setups:
             with self.test_context() as session:
-                if not test_setup.is_analytic:
+                if not test_setup.is_analytic("predict_density"):
                     continue
                 l = test_setup.likelihood
                 y = test_setup.Y
@@ -183,7 +187,7 @@ def test_pred_mean_and_var(self):
         # get all the likelihoods where predict_density has been overwritten.
         for test_setup in self.test_setups:
             with self.test_context() as session:
-                if not test_setup.is_analytic:
+                if not test_setup.is_analytic("predict_mean_and_var"):
                     continue
                 l = test_setup.likelihood
                 l.compile()