Repository: cornellius-gp/gpytorch

Issue:
linear kernel breaks when given certain types of active_dims
Hey,

So the linear kernel will break with some types of given active_dims arguments. I'll lay those out here.

```
import numpy as np
import gpytorch
import torch
torch.set_default_tensor_type('torch.DoubleTensor')

X = np.array([range(100*10)]).reshape(-1,10)
X = torch.from_numpy(X).to(torch.float64)
y = torch.from_numpy(np.random.normal(size=100)).to(torch.float64)
```

Below, I've commented out some active_dims which either work or don't work. 

```
class GP(gpytorch.models.ExactGP):
    def __init__(self, train_x, train_y, likelihood):
        super(GP, self).__init__(train_x, train_y, likelihood)
        self.mean_module = gpytorch.means.ConstantMean()
#         active_dims = list(range(1,10)) # Doesn't work!
        active_dims = list(range(3,10)) # Doesn't work!
#         active_dims = [0] + list(range(3,10)) # Doesn't work!
#         active_dims = [0] + list(range(2,10)) # Doesn't work!
#         active_dims = list(range(1)) # Works
#         active_dims = list(range(2)) # Works
#         active_dims = list(range(10)) # Works
        self.linear_kernel = gpytorch.kernels.LinearKernel(num_dimensions=len(active_dims),
                                                           active_dims=active_dims)
            
    def forward(self, x, requested_comp=None):
        
        mean_x = self.mean_module(x)
        linear_kernel_x = self.linear_kernel(x)
        covar_x = linear_kernel_x
        
        return gpytorch.distributions.MultivariateNormal(mean_x, covar_x)
```

Before I get to the error, we set up a few things:

```
likelihood = gpytorch.likelihoods.GaussianLikelihood()
likelihood.initialize(noise=8e-4) 
model = GP(X, y, likelihood)

learning_rate = .1
model.train()
likelihood.train()

optimizer = torch.optim.Adam([
    {'params': list(model.parameters())[2:]},
], lr=learning_rate)

mll = gpytorch.mlls.ExactMarginalLogLikelihood(likelihood, model)
```
When we get into the optimization, it breaks:

```
for i in range(100):
    optimizer.zero_grad()
    output = model(X)
    ### Breaks on the next line! ###
    loss = -mll(output, y)
    loss.backward()
    if not i % 30:
        print('i: {0}, Loss: {1}'.format(i, loss.item()))
    optimizer.step()
```

With this error:

```
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-67-cb234b5fe124> in <module>
     15     output = model(X)
     16     # Calc loss and backprop gradients
---> 17     loss = -mll(output, y)
     18     loss.backward()
     19     if not i % 30:

~/.virtualenvs/GenRS/lib/python3.7/site-packages/gpytorch/module.py in __call__(self, *inputs, **kwargs)
     18 
     19     def __call__(self, *inputs, **kwargs):
---> 20         outputs = self.forward(*inputs, **kwargs)
     21         if isinstance(outputs, list):
     22             return [_validate_module_outputs(output) for output in outputs]

~/.virtualenvs/GenRS/lib/python3.7/site-packages/gpytorch/mlls/exact_marginal_log_likelihood.py in forward(self, output, target, *params)
     26         # Get the log prob of the marginal distribution
     27         output = self.likelihood(output, *params)
---> 28         res = output.log_prob(target)
     29 
     30         # Add terms for SGPR / when inducing points are learned

~/.virtualenvs/GenRS/lib/python3.7/site-packages/gpytorch/distributions/multivariate_normal.py in log_prob(self, value)
    123 
    124         # Get log determininat and first part of quadratic form
--> 125         inv_quad, logdet = covar.inv_quad_logdet(inv_quad_rhs=diff.unsqueeze(-1), logdet=True)
    126 
    127         res = -0.5 * sum([inv_quad, logdet, diff.size(-1) * math.log(2 * math.pi)])

~/.virtualenvs/GenRS/lib/python3.7/site-packages/gpytorch/lazy/lazy_tensor.py in inv_quad_logdet(self, inv_quad_rhs, logdet, reduce_inv_quad)
    745                 )
    746 
--> 747         args = self.representation()
    748         if inv_quad_rhs is not None:
    749             args = [inv_quad_rhs] + list(args)

~/.virtualenvs/GenRS/lib/python3.7/site-packages/gpytorch/lazy/lazy_tensor.py in representation(self)
    980                 representation.append(arg)
    981             elif hasattr(arg, "representation") and callable(arg.representation):  # Is it a LazyTensor?
--> 982                 representation += list(arg.representation())
    983             else:
    984                 raise RuntimeError("Representation of a LazyTensor should consist only of Tensors")

~/.virtualenvs/GenRS/lib/python3.7/site-packages/gpytorch/lazy/lazy_evaluated_kernel_tensor.py in representation(self)
    306         # representation
    307         else:
--> 308             return self.evaluate_kernel().representation()
    309 
    310     def representation_tree(self):

~/.virtualenvs/GenRS/lib/python3.7/site-packages/gpytorch/utils/memoize.py in g(self, *args, **kwargs)
     15         cache_name = name if name is not None else method
     16         if cache_name not in self._memoize_cache:
---> 17             self._memoize_cache[cache_name] = method(self, *args, **kwargs)
     18         return self._memoize_cache[cache_name]
     19 

~/.virtualenvs/GenRS/lib/python3.7/site-packages/gpytorch/lazy/lazy_evaluated_kernel_tensor.py in evaluate_kernel(self)
    260         with settings.lazily_evaluate_kernels(False):
    261             res = self.kernel(
--> 262                 x1, x2, diag=False, batch_dims=self.batch_dims, **self.params
    263             )
    264         if self.squeeze_row:

~/.virtualenvs/GenRS/lib/python3.7/site-packages/gpytorch/kernels/kernel.py in __call__(self, x1, x2, diag, batch_dims, **params)
    313         # Select the active dimensions
    314         if self.active_dims is not None:
--> 315             x1_ = x1_.index_select(-1, self.active_dims)
    316             if x2_ is not None:
    317                 x2_ = x2_.index_select(-1, self.active_dims)

RuntimeError: invalid argument 3: out of range at /Users/soumith/b101/2019_02_04/wheel_build_dirs/wheel_3.7/pytorch/aten/src/TH/generic/THTensor.cpp:350
```




Discussion:
@Duane321 Have you pulled recently (specifically, since Thursdsay)? I very recently fixed a bug in `LinearKernel` where we weren't transforming the variance parameter to be positive, which could cause learning to make it not psd anymore.

If this is happening on like today's master let me know and I'll take a look.
Right before I did this, I uninstalled and re-installed the latest (unstable) version. So I think it might be a fresh issue. Unless, should I not be using the unstable version?

Resolving patch:
diff --git a/gpytorch/lazy/lazy_evaluated_kernel_tensor.py b/gpytorch/lazy/lazy_evaluated_kernel_tensor.py
--- a/gpytorch/lazy/lazy_evaluated_kernel_tensor.py
+++ b/gpytorch/lazy/lazy_evaluated_kernel_tensor.py
@@ -258,9 +258,12 @@ def evaluate_kernel(self):
             x2 = self.x2
 
         with settings.lazily_evaluate_kernels(False):
+            temp_active_dims = self.kernel.active_dims
+            self.kernel.active_dims = None
             res = self.kernel(
                 x1, x2, diag=False, batch_dims=self.batch_dims, **self.params
             )
+            self.kernel.active_dims = temp_active_dims
         if self.squeeze_row:
             res.squeeze_(-2)
         if self.squeeze_col:


Test patch:
diff --git a/test/kernels/_base_kernel_test_case.py b/test/kernels/_base_kernel_test_case.py
new file mode 100644
--- /dev/null
+++ b/test/kernels/_base_kernel_test_case.py
@@ -0,0 +1,29 @@
+#!/usr/bin/env python3
+
+from abc import abstractmethod
+import torch
+
+
+class BaseKernelTestCase(object):
+    @abstractmethod
+    def create_kernel_no_ard(self, **kwargs):
+        raise NotImplementedError()
+
+    def test_active_dims_list(self):
+        kernel = self.create_kernel_no_ard(active_dims=[0, 2, 4, 6])
+        x = torch.randn(50, 10)
+        covar_mat = kernel(x).evaluate_kernel().evaluate()
+        kernel_basic = self.create_kernel_no_ard()
+        covar_mat_actual = kernel_basic(x[:, [0, 2, 4, 6]]).evaluate_kernel().evaluate()
+
+        self.assertLess(torch.norm(covar_mat - covar_mat_actual), 1e-5)
+
+    def test_active_dims_range(self):
+        active_dims = list(range(3, 9))
+        kernel = self.create_kernel_no_ard(active_dims=active_dims)
+        x = torch.randn(50, 10)
+        covar_mat = kernel(x).evaluate_kernel().evaluate()
+        kernel_basic = self.create_kernel_no_ard()
+        covar_mat_actual = kernel_basic(x[:, active_dims]).evaluate_kernel().evaluate()
+
+        self.assertLess(torch.norm(covar_mat - covar_mat_actual), 1e-5)
diff --git a/test/kernels/test_linear_kernel.py b/test/kernels/test_linear_kernel.py
--- a/test/kernels/test_linear_kernel.py
+++ b/test/kernels/test_linear_kernel.py
@@ -3,9 +3,13 @@
 import torch
 import unittest
 from gpytorch.kernels import LinearKernel
+from test.kernels._base_kernel_test_case import BaseKernelTestCase
 
 
-class TestLinearKernel(unittest.TestCase):
+class TestLinearKernel(unittest.TestCase, BaseKernelTestCase):
+    def create_kernel_no_ard(self, **kwargs):
+        return LinearKernel(num_dimensions=10, **kwargs)
+
     def test_computes_linear_function_rectangular(self):
         a = torch.tensor([4, 2, 8], dtype=torch.float).view(3, 1)
         b = torch.tensor([0, 2, 1], dtype=torch.float).view(3, 1)
diff --git a/test/kernels/test_matern_kernel.py b/test/kernels/test_matern_kernel.py
--- a/test/kernels/test_matern_kernel.py
+++ b/test/kernels/test_matern_kernel.py
@@ -4,9 +4,13 @@
 import torch
 import unittest
 from gpytorch.kernels import MaternKernel
+from test.kernels._base_kernel_test_case import BaseKernelTestCase
 
 
-class TestMaternKernel(unittest.TestCase):
+class TestMaternKernel(unittest.TestCase, BaseKernelTestCase):
+    def create_kernel_no_ard(self, **kwargs):
+        return MaternKernel(nu=1.5, **kwargs)
+
     def test_forward_nu_1_over_2(self):
         a = torch.tensor([4, 2, 8], dtype=torch.float).view(3, 1)
         b = torch.tensor([0, 2], dtype=torch.float).view(2, 1)
diff --git a/test/kernels/test_rbf_kernel.py b/test/kernels/test_rbf_kernel.py
--- a/test/kernels/test_rbf_kernel.py
+++ b/test/kernels/test_rbf_kernel.py
@@ -4,9 +4,13 @@
 import torch
 import unittest
 from gpytorch.kernels import RBFKernel
+from test.kernels._base_kernel_test_case import BaseKernelTestCase
 
 
-class TestRBFKernel(unittest.TestCase):
+class TestRBFKernel(unittest.TestCase, BaseKernelTestCase):
+    def create_kernel_no_ard(self, **kwargs):
+        return RBFKernel(**kwargs)
+
     def test_ard(self):
         a = torch.tensor([[[1, 2], [2, 4]]], dtype=torch.float)
         b = torch.tensor([[[1, 3], [0, 4]]], dtype=torch.float)