Repository: cornellius-gp/gpytorch

Issue:
How are additive kernels represented
I am trying to learn how to add and multiply different kernels together, and am doing this a bit arbitrarily at this point. 
The thing I find confusing is, how is the internal representations of combined kernels being made? 

In the screenshot below you can see that things like `kernels.0.kernels.0` are being set. I would have have thought that if I added three kernels then I might get 'kernels.0`, 'kernels.1`, 'kernels.2` etc.  

Am I doing something wrong? or is there some documentation on how the parameters get represented? I am asking because I want to initialize the different parameters, and it is quite confusing to figure out which kernel is getting inherited inside what.

![Screen Shot 2021-02-11 at 11 05 19 PM](https://user-images.githubusercontent.com/18236610/107739424-b23ea200-6cbd-11eb-996a-77c1e523fe9f.png)



Discussion:
That's a good point. I'll put up a quick PR to make this read better.
In the future, it's better to post actual code rather than screenshots so we can reproduce the examples easier.

Resolving patch:
diff --git a/gpytorch/kernels/kernel.py b/gpytorch/kernels/kernel.py
--- a/gpytorch/kernels/kernel.py
+++ b/gpytorch/kernels/kernel.py
@@ -402,10 +402,16 @@ def __getstate__(self):
         return self.__dict__
 
     def __add__(self, other):
-        return AdditiveKernel(self, other)
+        kernels = []
+        kernels += self.kernels if isinstance(self, AdditiveKernel) else [self]
+        kernels += other.kernels if isinstance(other, AdditiveKernel) else [other]
+        return AdditiveKernel(*kernels)
 
     def __mul__(self, other):
-        return ProductKernel(self, other)
+        kernels = []
+        kernels += self.kernels if isinstance(self, ProductKernel) else [self]
+        kernels += other.kernels if isinstance(other, ProductKernel) else [other]
+        return ProductKernel(*kernels)
 
     def __setstate__(self, d):
         self.__dict__ = d


Test patch:
diff --git a/test/kernels/test_additive_kernel.py b/test/kernels/test_additive_and_product_kernels.py
similarity index 69%
rename from test/kernels/test_additive_kernel.py
rename to test/kernels/test_additive_and_product_kernels.py
--- a/test/kernels/test_additive_kernel.py
+++ b/test/kernels/test_additive_and_product_kernels.py
@@ -5,10 +5,10 @@
 
 import torch
 
-from gpytorch.kernels import AdditiveKernel, LinearKernel, ProductKernel, RBFKernel
+from gpytorch.kernels import LinearKernel, RBFKernel
 
 
-class TestAdditiveKernel(unittest.TestCase):
+class TestAdditiveAndProductKernel(unittest.TestCase):
     def test_computes_product_of_radial_basis_function(self):
         a = torch.tensor([4, 2, 8], dtype=torch.float).view(3, 1)
         b = torch.tensor([0, 2], dtype=torch.float).view(2, 1)
@@ -118,7 +118,10 @@ def test_computes_product_of_three_radial_basis_function(self):
         kernel_1 = RBFKernel().initialize(lengthscale=lengthscale)
         kernel_2 = RBFKernel().initialize(lengthscale=lengthscale)
         kernel_3 = RBFKernel().initialize(lengthscale=lengthscale)
-        kernel = ProductKernel(kernel_1, kernel_2, kernel_3)
+        kernel = (kernel_1 * kernel_2) * kernel_3
+        self.assertEqual(len(kernel.kernels), 3)
+        for sub_kernel in kernel.kernels:
+            self.assertIsInstance(sub_kernel, RBFKernel)
 
         actual = torch.tensor([[16, 4], [4, 0], [64, 36]], dtype=torch.float)
         actual = actual.mul_(-0.5).div_(lengthscale ** 2).exp() ** 3
@@ -127,6 +130,61 @@ def test_computes_product_of_three_radial_basis_function(self):
         res = kernel(a, b).evaluate()
         self.assertLess(torch.norm(res - actual), 2e-5)
 
+        kernel_1 = RBFKernel().initialize(lengthscale=lengthscale)
+        kernel_2 = RBFKernel().initialize(lengthscale=lengthscale)
+        kernel_3 = RBFKernel().initialize(lengthscale=lengthscale)
+        kernel = (kernel_1 * kernel_2) * kernel_3
+        self.assertEqual(len(kernel.kernels), 3)
+        for sub_kernel in kernel.kernels:
+            self.assertIsInstance(sub_kernel, RBFKernel)
+
+        kernel.eval()
+        res = kernel(a, b).evaluate()
+        self.assertLess(torch.norm(res - actual), 2e-5)
+
+    def test_computes_product_of_four_radial_basis_function(self):
+        a = torch.tensor([4, 2, 8], dtype=torch.float).view(3, 1)
+        b = torch.tensor([0, 2], dtype=torch.float).view(2, 1)
+        lengthscale = 2
+
+        kernel_1 = RBFKernel().initialize(lengthscale=lengthscale)
+        kernel_2 = RBFKernel().initialize(lengthscale=lengthscale)
+        kernel_3 = RBFKernel().initialize(lengthscale=lengthscale)
+        kernel_4 = RBFKernel().initialize(lengthscale=lengthscale)
+        kernel = kernel_1 * kernel_2 * kernel_3 * kernel_4
+        self.assertEqual(len(kernel.kernels), 4)
+        for sub_kernel in kernel.kernels:
+            self.assertIsInstance(sub_kernel, RBFKernel)
+
+        actual = torch.tensor([[16, 4], [4, 0], [64, 36]], dtype=torch.float)
+        actual = actual.mul_(-0.5).div_(lengthscale ** 2).exp() ** 4
+
+        kernel.eval()
+        res = kernel(a, b).evaluate()
+        self.assertLess(torch.norm(res - actual), 2e-5)
+
+    def test_computes_sum_of_four_radial_basis_function(self):
+        a = torch.tensor([4, 2, 8], dtype=torch.float).view(3, 1)
+        b = torch.tensor([0, 2], dtype=torch.float).view(2, 1)
+        lengthscale = 2
+
+        kernel_1 = RBFKernel().initialize(lengthscale=lengthscale)
+        kernel_2 = RBFKernel().initialize(lengthscale=lengthscale)
+        kernel_3 = RBFKernel().initialize(lengthscale=lengthscale)
+        kernel_4 = RBFKernel().initialize(lengthscale=lengthscale)
+        kernel = (kernel_1 + kernel_2) + (kernel_3 + kernel_4)
+        self.assertEqual(len(kernel.kernels), 4)
+        for sub_kernel in kernel.kernels:
+            self.assertIsInstance(sub_kernel, RBFKernel)
+
+        actual = (
+            torch.tensor([[16, 4], [4, 0], [64, 36]], dtype=torch.float).mul_(-0.5).div_(lengthscale ** 2).exp() * 4
+        )
+
+        kernel.eval()
+        res = kernel(a, b).evaluate()
+        self.assertLess(torch.norm(res - actual), 2e-5)
+
     def test_computes_sum_of_three_radial_basis_function(self):
         a = torch.tensor([4, 2, 8], dtype=torch.float).view(3, 1)
         b = torch.tensor([0, 2], dtype=torch.float).view(2, 1)
@@ -135,7 +193,10 @@ def test_computes_sum_of_three_radial_basis_function(self):
         kernel_1 = RBFKernel().initialize(lengthscale=lengthscale)
         kernel_2 = RBFKernel().initialize(lengthscale=lengthscale)
         kernel_3 = RBFKernel().initialize(lengthscale=lengthscale)
-        kernel = AdditiveKernel(kernel_1, kernel_2, kernel_3)
+        kernel = (kernel_1 + kernel_2) + kernel_3
+        self.assertEqual(len(kernel.kernels), 3)
+        for sub_kernel in kernel.kernels:
+            self.assertIsInstance(sub_kernel, RBFKernel)
 
         actual = (
             torch.tensor([[16, 4], [4, 0], [64, 36]], dtype=torch.float).mul_(-0.5).div_(lengthscale ** 2).exp() * 3
@@ -145,6 +206,18 @@ def test_computes_sum_of_three_radial_basis_function(self):
         res = kernel(a, b).evaluate()
         self.assertLess(torch.norm(res - actual), 2e-5)
 
+        kernel_1 = RBFKernel().initialize(lengthscale=lengthscale)
+        kernel_2 = RBFKernel().initialize(lengthscale=lengthscale)
+        kernel_3 = RBFKernel().initialize(lengthscale=lengthscale)
+        kernel = kernel_1 + (kernel_2 + kernel_3)
+        self.assertEqual(len(kernel.kernels), 3)
+        for sub_kernel in kernel.kernels:
+            self.assertIsInstance(sub_kernel, RBFKernel)
+
+        kernel.eval()
+        res = kernel(a, b).evaluate()
+        self.assertLess(torch.norm(res - actual), 2e-5)
+
     def test_computes_sum_radial_basis_function_gradient(self):
         softplus = torch.nn.functional.softplus
         a = torch.tensor([4, 2, 8], dtype=torch.float).view(3, 1)
@@ -184,7 +257,7 @@ def test_computes_sum_three_radial_basis_function_gradient(self):
         kernel_1 = RBFKernel().initialize(lengthscale=lengthscale)
         kernel_2 = RBFKernel().initialize(lengthscale=lengthscale)
         kernel_3 = RBFKernel().initialize(lengthscale=lengthscale)
-        kernel = AdditiveKernel(kernel_1, kernel_2, kernel_3)
+        kernel = kernel_1 + kernel_2 + kernel_3
         kernel.eval()
 
         output = kernel(a, b).evaluate()