Repository: cornellius-gp/gpytorch

Issue:
Exact GP regression Lapack Error syev
Training a simple GP regression model produces following runtime error randomly at some seeds: 
`RuntimeError: Lapack Error syev : 3 off-diagonal elements didn't converge to zero at /pytorch/aten/src/TH/generic/THTensorLapack.c:388`.
I have been able to reproduce the error with the following code:

```
import math
import torch
import gpytorch
from matplotlib import pyplot as plt

from torch import optim
from gpytorch.kernels import RBFKernel
from gpytorch.means import ConstantMean
from gpytorch.likelihoods import GaussianLikelihood
from gpytorch.random_variables import GaussianRandomVariable

torch.manual_seed(1)
train_x = torch.FloatTensor([[ 6,  6],
       [ 7,  3],
       [ 7,  8],
       [ 8,  6],
       [10,  0]])

train_y = torch.FloatTensor([2.38922077, 0.35414244, 2.24837906, 1.40895557, 0.68164634])

class ExactGPModel(gpytorch.models.ExactGP):
    def __init__(self, train_x, train_y, likelihood):
        super(ExactGPModel, self).__init__(train_x, train_y, likelihood)
        self.mean_module = ConstantMean(constant_bounds=(-10, 10))
        self.covar_module = RBFKernel(log_lengthscale_bounds=(-5, 5))
    
    def forward(self, x):
        mean_x = self.mean_module(x)
        covar_x = self.covar_module(x)
        return GaussianRandomVariable(mean_x, covar_x)

likelihood = GaussianLikelihood(log_noise_bounds=(-5, 5))
model = ExactGPModel(train_x.data, train_y.data, likelihood)

model.train()
likelihood.train()

optimizer = torch.optim.Adam([
    {'params': model.parameters()},  
], lr=0.1)

mll = gpytorch.mlls.ExactMarginalLogLikelihood(likelihood, model)

training_iter = 50
for i in range(training_iter):
    optimizer.zero_grad()
    output = model(train_x)
    loss = -mll(output, train_y)
    loss.backward()
    print('Iter %d/%d - Loss: %.3f   log_lengthscale: %.3f   log_noise: %.3f' % (
        i + 1, training_iter, loss.data[0],
        model.covar_module.log_lengthscale.data[0, 0],
        model.likelihood.log_noise.data[0]
    ))
    optimizer.step()


```
I am using the latest version of Gpytorch and Pytorch (0.4.0). The above code is identical to the simple gp regression example provided with this repo with changes in train_x and train_y only. Also, the code works fine for seed = 0. 



Discussion:
I'll take a look!
Okay, I've figured out what the problem is here. Basically, with so little data, the solve and log determinant we get with the preconditioner is exact to machine precision, so actually running CG / Stochastic LQ runs in to trouble. I'm working on a solution.

Resolving patch:
diff --git a/gpytorch/utils/linear_cg.py b/gpytorch/utils/linear_cg.py
--- a/gpytorch/utils/linear_cg.py
+++ b/gpytorch/utils/linear_cg.py
@@ -113,6 +113,8 @@ def linear_cg(
         prev_alpha_reciprocal = alpha.new(alpha_reciprocal.size())
         prev_beta = alpha.new(alpha_reciprocal.size())
 
+    update_tridiag = True
+    last_tridiag_iter = 0
     # Start the iteration
     for k in range(n_iter):
         # Get next alpha
@@ -134,7 +136,7 @@ def linear_cg(
         # If residual are sufficiently small, then exit loop
         # Alternatively, exit if this is our last iteration
         torch.norm(residual, 2, dim=-2, out=residual_norm)
-        if (residual_norm < tolerance).all() and not n_tridiag:
+        if (residual_norm < tolerance).all() and not (n_tridiag and k < n_tridiag_iter):
             break
 
         # Update precond_residual
@@ -153,7 +155,7 @@ def linear_cg(
         curr_conjugate_vec.mul_(beta).add_(precond_residual)
 
         # Update tridiagonal matrices, if applicable
-        if n_tridiag and k < n_tridiag_iter:
+        if n_tridiag and k < n_tridiag_iter and update_tridiag:
             alpha_tridiag = alpha.squeeze_(-2).narrow(-1, 0, n_tridiag)
             beta_tridiag = beta.squeeze_(-2).narrow(-1, 0, n_tridiag)
             torch.reciprocal(alpha_tridiag, out=alpha_reciprocal)
@@ -165,6 +167,11 @@ def linear_cg(
                 torch.mul(prev_beta.sqrt_(), prev_alpha_reciprocal, out=t_mat[k, k - 1])
                 t_mat[k - 1, k].copy_(t_mat[k, k - 1])
 
+                if t_mat[k - 1, k].max() < 1e-6:
+                    update_tridiag = False
+
+            last_tridiag_iter = k
+
             prev_alpha_reciprocal.copy_(alpha_reciprocal)
             prev_beta.copy_(beta_tridiag)
 
@@ -172,6 +179,7 @@ def linear_cg(
         result = result.squeeze(-1)
 
     if n_tridiag:
+        t_mat = t_mat[:last_tridiag_iter + 1, :last_tridiag_iter + 1]
         if rhs.ndimension() == 3:
             return result, t_mat.permute(3, 2, 0, 1).contiguous()
         else:


Test patch: