Repository: cornellius-gp/gpytorch
Pull request #925 (2019-11-06 13:53:41)

Refactor of has_lengthscale
Following the discussion on #922, I would appreciate feedback on this PR. The proposed changes are basically as follows:

1. Make `has_lengthscale` a class attribute
2. Introduce an `is_stationary` property
3. Re-order import in accordance with PEP8 convention of builtins followed by third-party followed by internal

Not sure I have correctly captured exactly which kernels are stationary or not...also would need to add more test coverage, hence PR is in draft.


Patch:
diff --git a/.isort.cfg b/.isort.cfg
index 92b18f67..3e8d8aaf 100644
--- a/.isort.cfg
+++ b/.isort.cfg
@@ -1,2 +1,2 @@
 [settings]
-known_third_party = matplotlib,numpy,pyro,setuptools,sphinx_rtd_theme,torch
+known_third_party = gpytorch,matplotlib,numpy,pyro,setuptools,sphinx_rtd_theme,torch
diff --git a/gpytorch/kernels/additive_structure_kernel.py b/gpytorch/kernels/additive_structure_kernel.py
index f14694ad..c5b8c4d1 100644
--- a/gpytorch/kernels/additive_structure_kernel.py
+++ b/gpytorch/kernels/additive_structure_kernel.py
@@ -32,6 +32,13 @@ class AdditiveStructureKernel(Kernel):
             Passed down to the `base_kernel`.
     """
 
+    @property
+    def is_stationary(self) -> bool:
+        """
+        Kernel is stationary if the base kernel is stationary.
+        """
+        return self.base_kernel.is_stationary
+
     def __init__(self, base_kernel, num_dims, active_dims=None):
         super(AdditiveStructureKernel, self).__init__(active_dims=active_dims)
         self.base_kernel = base_kernel
diff --git a/gpytorch/kernels/cosine_kernel.py b/gpytorch/kernels/cosine_kernel.py
index 9b0f877f..e81566f8 100644
--- a/gpytorch/kernels/cosine_kernel.py
+++ b/gpytorch/kernels/cosine_kernel.py
@@ -54,6 +54,8 @@ class CosineKernel(Kernel):
         >>> covar = covar_module(x)  # Output: LazyVariable of size (2 x 10 x 10)
     """
 
+    is_stationary = True
+
     def __init__(self, period_length_prior=None, period_length_constraint=None, **kwargs):
         super(CosineKernel, self).__init__(**kwargs)
 
diff --git a/gpytorch/kernels/cylindrical_kernel.py b/gpytorch/kernels/cylindrical_kernel.py
index b82893b0..93c33aa1 100644
--- a/gpytorch/kernels/cylindrical_kernel.py
+++ b/gpytorch/kernels/cylindrical_kernel.py
@@ -36,7 +36,6 @@ class CylindricalKernel(Kernel):
         :attr:`inv_param_transform` (function, optional):
             Set this to allow setting parameters directly in transformed space and sampling from priors.
             Automatically inferred for common transformations such as torch.exp or torch.nn.functional.softplus.
-
     """
 
     def __init__(
@@ -61,7 +60,7 @@ class CylindricalKernel(Kernel):
         if beta_constraint is None:
             beta_constraint = Positive()
 
-        super().__init__(has_lengthscale=False, **kwargs)
+        super().__init__(**kwargs)
         self.num_angular_weights = num_angular_weights
         self.radial_base_kernel = radial_base_kernel
         self.eps = eps
diff --git a/gpytorch/kernels/grid_interpolation_kernel.py b/gpytorch/kernels/grid_interpolation_kernel.py
index 841eecfd..d7a9f3f2 100644
--- a/gpytorch/kernels/grid_interpolation_kernel.py
+++ b/gpytorch/kernels/grid_interpolation_kernel.py
@@ -4,10 +4,9 @@ from typing import List, Optional, Tuple, Union
 
 import torch
 
-from gpytorch.utils.grid import create_grid
-
 from ..lazy import InterpolatedLazyTensor
 from ..models.exact_prediction_strategies import InterpolatedPredictionStrategy
+from ..utils.grid import create_grid
 from ..utils.interpolation import Interpolation
 from .grid_kernel import GridKernel
 from .kernel import Kernel
diff --git a/gpytorch/kernels/grid_kernel.py b/gpytorch/kernels/grid_kernel.py
index 5733d92c..d725642f 100644
--- a/gpytorch/kernels/grid_kernel.py
+++ b/gpytorch/kernels/grid_kernel.py
@@ -5,11 +5,9 @@ from typing import List
 import torch
 from torch import Tensor
 
-import gpytorch
-from gpytorch.utils.grid import convert_legacy_grid, create_data_from_grid
-
 from .. import settings
-from ..lazy import KroneckerProductLazyTensor, ToeplitzLazyTensor, delazify
+from ..lazy import KroneckerProductLazyTensor, ToeplitzLazyTensor, cat, delazify
+from ..utils.grid import convert_legacy_grid, create_data_from_grid
 from .kernel import Kernel
 
 
@@ -45,9 +43,14 @@ class GridKernel(Kernel):
         http://www.cs.cmu.edu/~andrewgw/manet.pdf
     """
 
+    is_stationary = True
+
     def __init__(
         self, base_kernel: Kernel, grid: List[Tensor], interpolation_mode: bool = False, active_dims: bool = None
     ):
+        if not base_kernel.is_stationary:
+            raise RuntimeError("The base_kernel for GridKernel must be stationary.")
+
         super(GridKernel, self).__init__(active_dims=active_dims)
         if torch.is_tensor(grid):
             grid = convert_legacy_grid(grid)
@@ -136,7 +139,7 @@ class GridKernel(Kernel):
                 ]  # Each entry i contains a grid_size[i] x grid_size[i] covariance matrix
                 if last_dim_is_batch:
                     # Note that this requires all the dimensions to have the same number of grid points
-                    covar = gpytorch.cat([c.unsqueeze(-3) for c in covars], dim=-3)
+                    covar = cat([c.unsqueeze(-3) for c in covars], dim=-3)
                 else:
                     covar = KroneckerProductLazyTensor(*covars[::-1])
 
diff --git a/gpytorch/kernels/keops/keops_kernel.py b/gpytorch/kernels/keops/keops_kernel.py
index f635e12b..f4e502ba 100644
--- a/gpytorch/kernels/keops/keops_kernel.py
+++ b/gpytorch/kernels/keops/keops_kernel.py
@@ -8,9 +8,6 @@ try:
     from pykeops.torch import LazyTensor as KEOLazyTensor
 
     class KeOpsKernel(Kernel):
-        def __init__(self, *args, **kwargs):
-            super().__init__(*args, **kwargs)
-
         @abstractmethod
         def covar_func(self, x1: torch.Tensor, x2: torch.Tensor) -> KEOLazyTensor:
             raise NotImplementedError("KeOpsKernels must define a covar_func method")
diff --git a/gpytorch/kernels/keops/matern_kernel.py b/gpytorch/kernels/keops/matern_kernel.py
index b94ccf3b..d96f5247 100644
--- a/gpytorch/kernels/keops/matern_kernel.py
+++ b/gpytorch/kernels/keops/matern_kernel.py
@@ -3,8 +3,7 @@ import math
 
 import torch
 
-from gpytorch.lazy import KeOpsLazyTensor
-
+from ...lazy import KeOpsLazyTensor
 from .keops_kernel import KeOpsKernel
 
 try:
@@ -19,10 +18,12 @@ try:
         most other features like ARD will work.
         """
 
+        has_lengthscale = True
+
         def __init__(self, nu=2.5, **kwargs):
             if nu not in {0.5, 1.5, 2.5}:
                 raise RuntimeError("nu expected to be 0.5, 1.5, or 2.5")
-            super(MaternKernel, self).__init__(has_lengthscale=True, **kwargs)
+            super(MaternKernel, self).__init__(**kwargs)
             self.nu = nu
 
         def covar_func(self, x1, x2, diag=False):
diff --git a/gpytorch/kernels/keops/rbf_kernel.py b/gpytorch/kernels/keops/rbf_kernel.py
index b55a38ab..4121412c 100644
--- a/gpytorch/kernels/keops/rbf_kernel.py
+++ b/gpytorch/kernels/keops/rbf_kernel.py
@@ -1,7 +1,6 @@
 import torch
 
-from gpytorch.lazy import KeOpsLazyTensor
-
+from ...lazy import KeOpsLazyTensor
 from ..rbf_kernel import postprocess_rbf
 from .keops_kernel import KeOpsKernel
 
@@ -17,8 +16,7 @@ try:
         most other features like ARD will work.
         """
 
-        def __init__(self, **kwargs):
-            super(RBFKernel, self).__init__(has_lengthscale=True, **kwargs)
+        has_lengthscale = True
 
         def covar_func(self, x1, x2, diag=False):
             # TODO: x1 / x2 size checks are a work around for a very minor bug in KeOps.
diff --git a/gpytorch/kernels/kernel.py b/gpytorch/kernels/kernel.py
index c89366db..50cd12a0 100644
--- a/gpytorch/kernels/kernel.py
+++ b/gpytorch/kernels/kernel.py
@@ -57,7 +57,7 @@ class Distance(torch.nn.Module):
 
 
 class Kernel(Module, _ClassWithDeprecatedBatchSize):
-    """
+    r"""
     Kernels in GPyTorch are implemented as a :class:`gpytorch.Module` that, when called on two :obj:`torch.tensor`
     objects `x1` and `x2` returns either a :obj:`torch.tensor` or a :obj:`gpytorch.lazy.LazyTensor` that represents
     the covariance matrix between `x1` and `x2`.
@@ -83,11 +83,11 @@ class Kernel(Module, _ClassWithDeprecatedBatchSize):
 
     * Single lengthscale: One lengthscale can be applied to all input dimensions/batches
       (i.e. :math:`\Theta` is a constant diagonal matrix).
-      This is controlled by setting `has_lengthscale=True`.
+      This is controlled by setting the attribute `has_lengthscale=True`.
 
     * ARD: Each input dimension gets its own separate lengthscale
       (i.e. :math:`\Theta` is a non-constant diagonal matrix).
-      This is controlled by the `ard_num_dims` keyword argument (as well has `has_lengthscale=True`).
+      This is controlled by the `ard_num_dims` keyword argument (as well as `has_lengthscale=True`).
 
     In batch-mode (i.e. when :math:`x_1` and :math:`x_2` are batches of input matrices), each
     batch of data can have its own lengthscale parameter by setting the `batch_shape`
@@ -99,8 +99,6 @@ class Kernel(Module, _ClassWithDeprecatedBatchSize):
         You can set a prior on this parameter using the :attr:`lengthscale_prior` argument.
 
     Base Args:
-        :attr:`has_lengthscale` (bool):
-            Set this if the kernel has a lengthscale. Default: `False`.
         :attr:`ard_num_dims` (int, optional):
             Set this if you want a separate lengthscale for each input
             dimension. It should be `d` if :attr:`x1` is a `n x d` matrix.  Default: `None`
@@ -129,9 +127,17 @@ class Kernel(Module, _ClassWithDeprecatedBatchSize):
         >>> tensor_covar_matrix = lazy_covar_matrix.evaluate() # Gets the actual tensor for this kernel matrix
     """
 
+    has_lengthscale = False
+
+    @property
+    def is_stationary(self) -> bool:
+        """
+        Property to indicate whether kernel is stationary or not.
+        """
+        return self.has_lengthscale
+
     def __init__(
         self,
-        has_lengthscale=False,
         ard_num_dims=None,
         batch_shape=torch.Size([]),
         active_dims=None,
@@ -152,7 +158,6 @@ class Kernel(Module, _ClassWithDeprecatedBatchSize):
             kwargs, "batch_size", "batch_shape", batch_shape, lambda n: torch.Size([n])
         )
 
-        self.__has_lengthscale = has_lengthscale
         self.eps = eps
 
         param_transform = kwargs.get("param_transform")
@@ -166,7 +171,7 @@ class Kernel(Module, _ClassWithDeprecatedBatchSize):
                 "transformation, specify a different 'lengthscale_constraint' instead."
             )
 
-        if has_lengthscale:
+        if self.has_lengthscale:
             lengthscale_num_dims = 1 if ard_num_dims is None else ard_num_dims
             self.register_parameter(
                 name="raw_lengthscale",
@@ -209,7 +214,6 @@ class Kernel(Module, _ClassWithDeprecatedBatchSize):
                 * `diag`: `n` or `b x n`
                 * `diag` with `last_dim_is_batch=True`: `k x n` or `b x k x n`
         """
-
         raise NotImplementedError()
 
     @property
@@ -221,10 +225,6 @@ class Kernel(Module, _ClassWithDeprecatedBatchSize):
                 return param.dtype
             return torch.get_default_dtype()
 
-    @property
-    def has_lengthscale(self):
-        return self.__has_lengthscale
-
     @property
     def lengthscale(self):
         if self.has_lengthscale:
@@ -334,9 +334,6 @@ class Kernel(Module, _ClassWithDeprecatedBatchSize):
     def prediction_strategy(self, train_inputs, train_prior_dist, train_labels, likelihood):
         return DefaultPredictionStrategy(train_inputs, train_prior_dist, train_labels, likelihood)
 
-    def __add__(self, other):
-        return AdditiveKernel(self, other)
-
     def __call__(self, x1, x2=None, diag=False, last_dim_is_batch=False, **params):
         x1_, x2_ = x1, x2
 
@@ -387,6 +384,9 @@ class Kernel(Module, _ClassWithDeprecatedBatchSize):
         self.distance_module = None
         return self.__dict__
 
+    def __add__(self, other):
+        return AdditiveKernel(self, other)
+
     def __mul__(self, other):
         return ProductKernel(self, other)
 
@@ -424,6 +424,13 @@ class AdditiveKernel(Kernel):
         >>> additive_kernel_matrix = covar_module(x1)
     """
 
+    @property
+    def is_stationary(self) -> bool:
+        """
+        Kernel is stationary if all components are stationary.
+        """
+        return all(k.is_stationary for k in self.kernels)
+
     def __init__(self, *kernels):
         super(AdditiveKernel, self).__init__()
         self.kernels = ModuleList(kernels)
@@ -463,6 +470,13 @@ class ProductKernel(Kernel):
         >>> kernel_matrix = covar_module(x1) # The RBF Kernel already decomposes multiplicatively, so this is foolish!
     """
 
+    @property
+    def is_stationary(self) -> bool:
+        """
+        Kernel is stationary if all components are stationary.
+        """
+        return all(k.is_stationary for k in self.kernels)
+
     def __init__(self, *kernels):
         super(ProductKernel, self).__init__()
         self.kernels = ModuleList(kernels)
diff --git a/gpytorch/kernels/matern_kernel.py b/gpytorch/kernels/matern_kernel.py
index 43a17d6b..aa23e127 100644
--- a/gpytorch/kernels/matern_kernel.py
+++ b/gpytorch/kernels/matern_kernel.py
@@ -77,10 +77,12 @@ class MaternKernel(Kernel):
         >>> covar = covar_module(x)  # Output: LazyVariable of size (2 x 10 x 10)
     """
 
+    has_lengthscale = True
+
     def __init__(self, nu=2.5, **kwargs):
         if nu not in {0.5, 1.5, 2.5}:
             raise RuntimeError("nu expected to be 0.5, 1.5, or 2.5")
-        super(MaternKernel, self).__init__(has_lengthscale=True, **kwargs)
+        super(MaternKernel, self).__init__(**kwargs)
         self.nu = nu
 
     def forward(self, x1, x2, diag=False, **params):
diff --git a/gpytorch/kernels/newton_girard_additive_kernel.py b/gpytorch/kernels/newton_girard_additive_kernel.py
index 3049fff8..87cd44a1 100644
--- a/gpytorch/kernels/newton_girard_additive_kernel.py
+++ b/gpytorch/kernels/newton_girard_additive_kernel.py
@@ -13,7 +13,7 @@ class NewtonGirardAdditiveKernel(Kernel):
         :param active_dims:
         :param kwargs:
         """
-        super(NewtonGirardAdditiveKernel, self).__init__(has_lengthscale=False, active_dims=active_dims, **kwargs)
+        super(NewtonGirardAdditiveKernel, self).__init__(active_dims=active_dims, **kwargs)
 
         self.base_kernel = base_kernel
         self.num_dims = num_dims
diff --git a/gpytorch/kernels/periodic_kernel.py b/gpytorch/kernels/periodic_kernel.py
index b4759d05..e362e48f 100644
--- a/gpytorch/kernels/periodic_kernel.py
+++ b/gpytorch/kernels/periodic_kernel.py
@@ -72,8 +72,10 @@ class PeriodicKernel(Kernel):
         >>> covar = covar_module(x)  # Output: LazyVariable of size (2 x 10 x 10)
     """
 
+    has_lengthscale = True
+
     def __init__(self, period_length_prior=None, period_length_constraint=None, **kwargs):
-        super(PeriodicKernel, self).__init__(has_lengthscale=True, **kwargs)
+        super(PeriodicKernel, self).__init__(**kwargs)
         if period_length_constraint is None:
             period_length_constraint = Positive()
 
diff --git a/gpytorch/kernels/product_structure_kernel.py b/gpytorch/kernels/product_structure_kernel.py
index 1e73481f..99e85550 100644
--- a/gpytorch/kernels/product_structure_kernel.py
+++ b/gpytorch/kernels/product_structure_kernel.py
@@ -38,6 +38,13 @@ class ProductStructureKernel(Kernel):
         https://arxiv.org/pdf/1802.08903
     """
 
+    @property
+    def is_stationary(self) -> bool:
+        """
+        Kernel is stationary if the base kernel is stationary.
+        """
+        return self.base_kernel.is_stationary
+
     def __init__(self, base_kernel, num_dims, active_dims=None):
         super(ProductStructureKernel, self).__init__(active_dims=active_dims)
         self.base_kernel = base_kernel
diff --git a/gpytorch/kernels/rbf_kernel.py b/gpytorch/kernels/rbf_kernel.py
index bc39804c..323d47eb 100644
--- a/gpytorch/kernels/rbf_kernel.py
+++ b/gpytorch/kernels/rbf_kernel.py
@@ -66,8 +66,7 @@ class RBFKernel(Kernel):
         >>> covar = covar_module(x)  # Output: LazyTensor of size (2 x 10 x 10)
     """
 
-    def __init__(self, **kwargs):
-        super(RBFKernel, self).__init__(has_lengthscale=True, **kwargs)
+    has_lengthscale = True
 
     def forward(self, x1, x2, diag=False, **params):
         if x1.requires_grad or x2.requires_grad or (self.ard_num_dims is not None and self.ard_num_dims > 1) or diag:
diff --git a/gpytorch/kernels/rq_kernel.py b/gpytorch/kernels/rq_kernel.py
index 8d869df4..e752301f 100644
--- a/gpytorch/kernels/rq_kernel.py
+++ b/gpytorch/kernels/rq_kernel.py
@@ -55,8 +55,10 @@ class RQKernel(Kernel):
             on the :attr:`batch_shape` argument
     """
 
+    has_lengthscale = True
+
     def __init__(self, alpha_constraint=None, **kwargs):
-        super(RQKernel, self).__init__(has_lengthscale=True, **kwargs)
+        super(RQKernel, self).__init__(**kwargs)
         self.register_parameter(name="raw_alpha", parameter=torch.nn.Parameter(torch.zeros(*self.batch_shape, 1)))
         if alpha_constraint is None:
             alpha_constraint = Positive()
diff --git a/gpytorch/kernels/scale_kernel.py b/gpytorch/kernels/scale_kernel.py
index 401eda10..9f8e9eae 100644
--- a/gpytorch/kernels/scale_kernel.py
+++ b/gpytorch/kernels/scale_kernel.py
@@ -51,8 +51,15 @@ class ScaleKernel(Kernel):
         >>> covar = scaled_covar_module(x)  # Output: LazyTensor of size (10 x 10)
     """
 
+    @property
+    def is_stationary(self) -> bool:
+        """
+        Kernel is stationary if base kernel is stationary.
+        """
+        return self.base_kernel.is_stationary
+
     def __init__(self, base_kernel, outputscale_prior=None, outputscale_constraint=None, **kwargs):
-        super(ScaleKernel, self).__init__(has_lengthscale=False, **kwargs)
+        super(ScaleKernel, self).__init__(**kwargs)
         if outputscale_constraint is None:
             outputscale_constraint = Positive()
 
diff --git a/gpytorch/kernels/spectral_mixture_kernel.py b/gpytorch/kernels/spectral_mixture_kernel.py
index 1bdd9210..ba23ecb9 100644
--- a/gpytorch/kernels/spectral_mixture_kernel.py
+++ b/gpytorch/kernels/spectral_mixture_kernel.py
@@ -65,6 +65,8 @@ class SpectralMixtureKernel(Kernel):
         https://arxiv.org/pdf/1302.4245.pdf
     """
 
+    is_stationary = True  # kernel is stationary even though it does not have a lengthscale
+
     def __init__(
         self,
         num_mixtures=None,


Test patch:
diff --git a/test/kernels/test_additive_kernel.py b/test/kernels/test_additive_kernel.py
index 4d02ddf1..bbabdcbe 100644
--- a/test/kernels/test_additive_kernel.py
+++ b/test/kernels/test_additive_kernel.py
@@ -5,7 +5,7 @@ import unittest
 
 import torch
 
-from gpytorch.kernels import AdditiveKernel, ProductKernel, RBFKernel
+from gpytorch.kernels import AdditiveKernel, LinearKernel, ProductKernel, RBFKernel
 
 
 class TestAdditiveKernel(unittest.TestCase):
@@ -196,6 +196,16 @@ class TestAdditiveKernel(unittest.TestCase):
         )
         self.assertLess(torch.norm(res - actual_param_grad), 2e-5)
 
+    def test_is_stationary(self):
+        kernel_1 = RBFKernel().initialize(lengthscale=1)
+        kernel_2 = RBFKernel().initialize(lengthscale=2)
+        kernel_3 = LinearKernel().initialize()
+
+        self.assertTrue((kernel_1 + kernel_2).is_stationary)
+        self.assertTrue((kernel_1 * kernel_2).is_stationary)
+        self.assertFalse((kernel_1 + kernel_3).is_stationary)
+        self.assertFalse((kernel_1 * kernel_3).is_stationary)
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/test/kernels/test_grid_kernel.py b/test/kernels/test_grid_kernel.py
index 111c337f..a2523875 100644
--- a/test/kernels/test_grid_kernel.py
+++ b/test/kernels/test_grid_kernel.py
@@ -4,7 +4,7 @@ import unittest
 
 import torch
 
-from gpytorch.kernels import GridKernel, RBFKernel
+from gpytorch.kernels import GridKernel, LinearKernel, RBFKernel
 from gpytorch.lazy import KroneckerProductLazyTensor
 from gpytorch.utils.grid import create_data_from_grid
 
@@ -39,6 +39,11 @@ class TestGridKernel(unittest.TestCase):
         actual_eval = base_kernel(data, data).evaluate()
         self.assertLess(torch.norm(grid_eval - actual_eval), 1e-5)
 
+    def test_non_stationary_base(self):
+        base_kernel = LinearKernel()
+        with self.assertRaisesRegex(RuntimeError, "The base_kernel for GridKernel must be stationary."):
+            GridKernel(base_kernel, grid)
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/test/kernels/test_scale_kernel.py b/test/kernels/test_scale_kernel.py
index e85cd361..23efc40e 100644
--- a/test/kernels/test_scale_kernel.py
+++ b/test/kernels/test_scale_kernel.py
@@ -4,7 +4,7 @@ import unittest
 
 import torch
 
-from gpytorch.kernels import RBFKernel, ScaleKernel
+from gpytorch.kernels import LinearKernel, RBFKernel, ScaleKernel
 from gpytorch.test.base_kernel_test_case import BaseKernelTestCase
 
 
@@ -104,6 +104,14 @@ class TestScaleKernel(BaseKernelTestCase, unittest.TestCase):
         actual_value = ls_init.view_as(kernel.outputscale)
         self.assertLess(torch.norm(kernel.outputscale - actual_value), 1e-5)
 
+    def test_stationary(self):
+        kernel = ScaleKernel(RBFKernel())
+        self.assertTrue(kernel.is_stationary)
+
+    def test_non_stationary(self):
+        kernel = ScaleKernel(LinearKernel())
+        self.assertFalse(kernel.is_stationary)
+
 
 if __name__ == "__main__":
     unittest.main()