Repository: cornellius-gp/gpytorch
Pull request #115 (2018-05-03 22:09:12)

Change definition of RBF Kernel
Previous: `k(x, y) = exp(-|x-y|^2 / l)`
Now: `k(x,y) = exp(-0.5 * |(x-y)/l|^2)`

This doesn't really change any functionality whatsoever, but makes the definition consistent with some of the literature (e.g. Rasmussen & Williams) and the implementations in other packages (e.g. scipy and GPy)

Also, making baby steps towards getting rid of variables #110 


Patch:
diff --git a/gpytorch/kernels/rbf_kernel.py b/gpytorch/kernels/rbf_kernel.py
index b1de6f71..86db70a3 100644
--- a/gpytorch/kernels/rbf_kernel.py
+++ b/gpytorch/kernels/rbf_kernel.py
@@ -25,5 +25,5 @@ class RBFKernel(Kernel):
 
     def forward(self, x1, x2):
         lengthscales = self.log_lengthscale.exp() + self.eps
-        diff = (x1.unsqueeze(2) - x2.unsqueeze(1)).div_(lengthscales.sqrt())
-        return diff.pow_(2).sum(-1).mul_(-1).exp_()
+        diff = (x1.unsqueeze(2) - x2.unsqueeze(1)).div_(lengthscales)
+        return diff.pow_(2).sum(-1).mul_(-0.5).exp_()
diff --git a/gpytorch/utils/__init__.py b/gpytorch/utils/__init__.py
index 4519dcc5..ce004239 100644
--- a/gpytorch/utils/__init__.py
+++ b/gpytorch/utils/__init__.py
@@ -47,11 +47,12 @@ def approx_equal(self, other, epsilon=1e-4):
         - bool
     """
     if self.size() != other.size():
-        raise RuntimeError('Size mismatch between self (%s) and other (%s)' % (str(self.size()), str(other.size())))
-    if isinstance(self, Variable):
-        self = self.data
-    if isinstance(other, Variable):
-        other = other.data
+        raise RuntimeError(
+            'Size mismatch between self ({self}) and other ({other})'.format(
+                self=self.size(),
+                other=other.size(),
+            )
+        )
     return torch.max((self - other).abs()) <= epsilon
 
 
@@ -65,8 +66,11 @@ def bdsmm(sparse, dense):
         indices = sparse._indices()[1:].clone()
         indices[0].add_(n_rows, batch_assignment)
         indices[1].add_(n_cols, batch_assignment)
-        sparse_2d = sparse.new(indices, sparse._values(),
-                               torch.Size((batch_size * n_rows, batch_size * n_cols)))
+        sparse_2d = sparse.new(
+            indices,
+            sparse._values(),
+            torch.Size((batch_size * n_rows, batch_size * n_cols)),
+        )
 
         if dense.size(0) == 1:
             dense = dense.repeat(batch_size, 1, 1)
@@ -273,8 +277,10 @@ def sparse_repeat(sparse, *repeat_sizes):
     orig_nvalues = sparse._indices().size(1)
 
     # Expand the number of dimensions to match repeat_sizes
-    indices = torch.cat([sparse._indices().new().resize_(new_ndim - orig_ndim, orig_nvalues).zero_(),
-                         sparse._indices()])
+    indices = torch.cat([
+        sparse._indices().new().resize_(new_ndim - orig_ndim, orig_nvalues).zero_(),
+        sparse._indices(),
+    ])
     values = sparse._values()
     size = [1] * (new_ndim - orig_ndim) + list(sparse.size())
 


Test patch:
diff --git a/test/examples/test_kissgp_kronecker_product_classification.py b/test/examples/test_kissgp_kronecker_product_classification.py
index 1bb72bc1..79512cbc 100644
--- a/test/examples/test_kissgp_kronecker_product_classification.py
+++ b/test/examples/test_kissgp_kronecker_product_classification.py
@@ -33,7 +33,7 @@ class GPClassificationModel(gpytorch.models.GridInducingVariationalGP):
             grid_bounds=[(0, 3), (0, 3)],
         )
         self.mean_module = ConstantMean(constant_bounds=[-1e-5, 1e-5])
-        self.covar_module = RBFKernel(log_lengthscale_bounds=(-5, 6))
+        self.covar_module = RBFKernel(log_lengthscale_bounds=(-2.5, 3))
         self.register_parameter(
             'log_outputscale',
             nn.Parameter(torch.Tensor([0])),
@@ -79,7 +79,7 @@ class TestKissGPKroneckerProductClassification(unittest.TestCase):
 
         test_preds = model(train_x).mean().ge(0.5).float().mul(2).sub(1).squeeze()
         mean_abs_error = torch.mean(torch.abs(train_y - test_preds) / 2)
-        self.assertLess(mean_abs_error.data.squeeze().item(), 1e-5)
+        self.assertLess(mean_abs_error.squeeze().item(), 1e-5)
 
 
 if __name__ == '__main__':
diff --git a/test/kernels/test_additive_kernel.py b/test/kernels/test_additive_kernel.py
index 41af1f43..057789e1 100644
--- a/test/kernels/test_additive_kernel.py
+++ b/test/kernels/test_additive_kernel.py
@@ -6,7 +6,6 @@ from __future__ import unicode_literals
 import math
 import torch
 import unittest
-from torch.autograd import Variable
 from gpytorch.kernels import RBFKernel
 
 
@@ -20,15 +19,16 @@ class TestAdditiveKernel(unittest.TestCase):
         kernel_1 = RBFKernel().initialize(log_lengthscale=math.log(lengthscale))
         kernel_2 = RBFKernel().initialize(log_lengthscale=math.log(lengthscale))
         kernel = kernel_1 * kernel_2
-        kernel.eval()
+
         actual = torch.Tensor([
             [16, 4],
             [4, 0],
             [64, 36],
-        ]).mul_(-1).div_(lengthscale).exp() ** 2
+        ]).mul_(-0.5).div_(lengthscale**2).exp() ** 2
 
-        res = kernel(Variable(a), Variable(b)).data
-        self.assertLess(torch.norm(res - actual), 1e-5)
+        kernel.eval()
+        res = kernel(a, b)
+        self.assertLess(torch.norm(res - actual), 2e-5)
 
     def test_computes_sum_of_radial_basis_function(self):
         a = torch.Tensor([4, 2, 8]).view(3, 1)
@@ -38,39 +38,41 @@ class TestAdditiveKernel(unittest.TestCase):
         kernel_1 = RBFKernel().initialize(log_lengthscale=math.log(lengthscale))
         kernel_2 = RBFKernel().initialize(log_lengthscale=math.log(lengthscale))
         kernel = kernel_1 + kernel_2
-        kernel.eval()
+
         actual = torch.Tensor([
             [16, 4],
             [4, 0],
             [64, 36],
-        ]).mul_(-1).div_(lengthscale).exp() * 2
+        ]).mul_(-0.5).div_(lengthscale**2).exp() * 2
 
-        res = kernel(Variable(a), Variable(b)).data
-        self.assertLess(torch.norm(res - actual), 1e-5)
+        kernel.eval()
+        res = kernel(a, b)
+        self.assertLess(torch.norm(res - actual), 2e-5)
 
     def test_computes_sum_radial_basis_function_gradient(self):
         a = torch.Tensor([4, 2, 8]).view(3, 1)
         b = torch.Tensor([0, 2, 2]).view(3, 1)
         lengthscale = 2
 
+        param = math.log(lengthscale) * torch.ones(3, 3)
+        param.requires_grad_()
+        diffs = a.expand(3, 3) - b.expand(3, 3).transpose(0, 1)
+        actual_output = (-0.5 * (diffs / param.exp()) ** 2).exp()
+        actual_output.backward(torch.eye(3))
+        actual_param_grad = param.grad.sum() * 2
+
         kernel_1 = RBFKernel().initialize(log_lengthscale=math.log(lengthscale))
         kernel_2 = RBFKernel().initialize(log_lengthscale=math.log(lengthscale))
         kernel = kernel_1 + kernel_2
         kernel.eval()
-        param = Variable(
-            torch.Tensor(3, 3).fill_(math.log(lengthscale)),
-            requires_grad=True,
-        )
-        diffs = Variable(a.expand(3, 3) - b.expand(3, 3).transpose(0, 1))
-        actual_output = (-(diffs ** 2) / (param.exp())).exp()
-        actual_output.backward(torch.eye(3))
-        actual_param_grad = param.grad.data.sum() * 2
 
-        output = kernel(Variable(a), Variable(b))
+        output = kernel(a, b)
         output.backward(gradient=torch.eye(3))
-        res = kernel.kernel_1.log_lengthscale.grad.data
-        res += kernel.kernel_2.log_lengthscale.grad.data
-        self.assertLess(torch.norm(res - actual_param_grad), 1e-5)
+        res = (
+            kernel.kernel_1.log_lengthscale.grad +
+            kernel.kernel_2.log_lengthscale.grad
+        )
+        self.assertLess(torch.norm(res - actual_param_grad), 2e-5)
 
 
 if __name__ == '__main__':
diff --git a/test/kernels/test_rbf_kernel.py b/test/kernels/test_rbf_kernel.py
index 4b66220a..365ab944 100644
--- a/test/kernels/test_rbf_kernel.py
+++ b/test/kernels/test_rbf_kernel.py
@@ -6,7 +6,6 @@ from __future__ import unicode_literals
 import math
 import torch
 import unittest
-from torch.autograd import Variable
 from gpytorch.kernels import RBFKernel
 
 
@@ -20,9 +19,9 @@ class TestRBFKernel(unittest.TestCase):
         kernel = RBFKernel(ard_num_dims=2)
         kernel.initialize(log_lengthscale=lengthscales.log())
         kernel.eval()
-        actual = (a - b).pow(2).div_(lengthscales).sum(dim=-1).mul_(-1).exp()
 
-        res = kernel(Variable(a), Variable(b)).data
+        actual = (a - b).div_(lengthscales).pow(2).sum(dim=-1).mul_(-0.5).exp()
+        res = kernel(a, b)
         self.assertLess(torch.norm(res - actual.unsqueeze(-1)), 1e-5)
 
     def test_subset_active_compute_radial_basis_function(self):
@@ -35,13 +34,13 @@ class TestRBFKernel(unittest.TestCase):
         kernel = RBFKernel(active_dims=[0])
         kernel.initialize(log_lengthscale=math.log(lengthscale))
         kernel.eval()
+
         actual = torch.Tensor([
             [16, 4],
             [4, 0],
             [64, 36],
-        ]).mul_(-1).div_(lengthscale).exp()
-
-        res = kernel(Variable(a), Variable(b)).data
+        ]).mul_(-0.5).div_(lengthscale**2).exp()
+        res = kernel(a, b)
         self.assertLess(torch.norm(res - actual), 1e-5)
 
     def test_computes_radial_basis_function(self):
@@ -51,13 +50,13 @@ class TestRBFKernel(unittest.TestCase):
 
         kernel = RBFKernel().initialize(log_lengthscale=math.log(lengthscale))
         kernel.eval()
+
         actual = torch.Tensor([
             [16, 4],
             [4, 0],
             [64, 36],
-        ]).mul_(-1).div_(lengthscale).exp()
-
-        res = kernel(Variable(a), Variable(b)).data
+        ]).mul_(-0.5).div_(lengthscale**2).exp()
+        res = kernel(a, b)
         self.assertLess(torch.norm(res - actual), 1e-5)
 
     def test_computes_radial_basis_function_gradient(self):
@@ -67,18 +66,18 @@ class TestRBFKernel(unittest.TestCase):
 
         kernel = RBFKernel().initialize(log_lengthscale=math.log(lengthscale))
         kernel.eval()
-        param = Variable(
-            torch.Tensor(3, 3).fill_(math.log(lengthscale)),
-            requires_grad=True,
-        )
-        diffs = Variable(a.expand(3, 3) - b.expand(3, 3).transpose(0, 1))
-        actual_output = (-(diffs ** 2) / (param.exp())).exp()
-        actual_output.backward(torch.eye(3))
-        actual_param_grad = param.grad.data.sum()
 
-        output = kernel(Variable(a), Variable(b))
+        param = math.log(lengthscale) * torch.ones(3, 3)
+        param.requires_grad_()
+        diffs = a.expand(3, 3) - b.expand(3, 3).transpose(0, 1)
+        actual_output = (-0.5 * (diffs / param.exp()) ** 2).exp()
+        actual_output.backward(gradient=torch.eye(3))
+        actual_param_grad = param.grad.sum()
+
+        output = kernel(a, b)
         output.backward(gradient=torch.eye(3))
-        res = kernel.log_lengthscale.grad.data
+        res = kernel.log_lengthscale.grad
+
         self.assertLess(torch.norm(res - actual_param_grad), 1e-5)
 
     def test_subset_active_computes_radial_basis_function_gradient(self):
@@ -88,21 +87,19 @@ class TestRBFKernel(unittest.TestCase):
         b = torch.Tensor([0, 2, 2]).view(3, 1)
         lengthscale = 2
 
+        param = math.log(lengthscale) * torch.ones(3, 3)
+        param.requires_grad_()
+        diffs = a_1.expand(3, 3) - b.expand(3, 3).transpose(0, 1)
+        actual_output = (-0.5 * (diffs / param.exp()) ** 2).exp()
+        actual_output.backward(torch.eye(3))
+        actual_param_grad = param.grad.sum()
+
         kernel = RBFKernel(active_dims=[0])
         kernel.initialize(log_lengthscale=math.log(lengthscale))
         kernel.eval()
-        param = Variable(
-            torch.Tensor(3, 3).fill_(math.log(lengthscale)),
-            requires_grad=True,
-        )
-        output = kernel(Variable(a), Variable(b))
+        output = kernel(a, b)
         output.backward(gradient=torch.eye(3))
-        res = kernel.log_lengthscale.grad.data
-
-        diffs = Variable(a_1.expand(3, 3) - b.expand(3, 3).transpose(0, 1))
-        actual_output = (-(diffs ** 2) / (param.exp())).exp()
-        actual_output.backward(torch.eye(3))
-        actual_param_grad = param.grad.data.sum()
+        res = kernel.log_lengthscale.grad
 
         self.assertLess(torch.norm(res - actual_param_grad), 1e-5)
 
diff --git a/test/util/test_pivoted_cholesky.py b/test/util/test_pivoted_cholesky.py
index e4b0f59f..a5e53c45 100644
--- a/test/util/test_pivoted_cholesky.py
+++ b/test/util/test_pivoted_cholesky.py
@@ -5,7 +5,6 @@ from __future__ import unicode_literals
 
 import torch
 import unittest
-from torch.autograd import Variable
 from gpytorch.utils import pivoted_cholesky, approx_equal
 from gpytorch.kernels import RBFKernel
 
@@ -14,62 +13,66 @@ class TestPivotedCholesky(unittest.TestCase):
 
     def test_pivoted_cholesky(self):
         size = 100
-        train_x = Variable(torch.linspace(0, 1, size))
-        covar_matrix = RBFKernel()(train_x, train_x).data
+        train_x = torch.linspace(0, 1, size)
+        covar_matrix = RBFKernel()(train_x, train_x)
         piv_chol = pivoted_cholesky.pivoted_cholesky(covar_matrix, 10)
         covar_approx = piv_chol.t().matmul(piv_chol)
-
-        self.assertTrue(approx_equal(covar_approx, covar_matrix))
+        self.assertTrue(approx_equal(covar_approx, covar_matrix, 2e-4))
 
     def test_solve_vector(self):
         size = 100
-        train_x = Variable(torch.linspace(0, 1, size))
-        covar_matrix = RBFKernel()(train_x, train_x).data
+        train_x = torch.linspace(0, 1, size)
+        covar_matrix = RBFKernel()(train_x, train_x)
         piv_chol = pivoted_cholesky.pivoted_cholesky(covar_matrix, 10)
         woodbury_factor = pivoted_cholesky.woodbury_factor(piv_chol, 1)
 
         rhs_vector = torch.randn(100)
         shifted_covar_matrix = covar_matrix + torch.eye(size)
         real_solve = shifted_covar_matrix.inverse().matmul(rhs_vector)
-        approx_solve = pivoted_cholesky.woodbury_solve(rhs_vector, piv_chol, woodbury_factor, 1)
+        approx_solve = pivoted_cholesky.woodbury_solve(
+            rhs_vector, piv_chol, woodbury_factor, 1,
+        )
 
-        self.assertTrue(approx_equal(approx_solve, real_solve))
+        self.assertTrue(approx_equal(approx_solve, real_solve, 2e-4))
 
     def test_solve(self):
         size = 100
-        train_x = Variable(torch.linspace(0, 1, size))
-        covar_matrix = RBFKernel()(train_x, train_x).data
+        train_x = torch.linspace(0, 1, size)
+        covar_matrix = RBFKernel()(train_x, train_x)
         piv_chol = pivoted_cholesky.pivoted_cholesky(covar_matrix, 10)
         woodbury_factor = pivoted_cholesky.woodbury_factor(piv_chol, 1)
 
         rhs_vector = torch.randn(100, 50)
         shifted_covar_matrix = covar_matrix + torch.eye(size)
         real_solve = shifted_covar_matrix.inverse().matmul(rhs_vector)
-        approx_solve = pivoted_cholesky.woodbury_solve(rhs_vector, piv_chol, woodbury_factor, 1)
+        approx_solve = pivoted_cholesky.woodbury_solve(
+            rhs_vector, piv_chol, woodbury_factor, 1,
+        )
 
-        self.assertTrue(approx_equal(approx_solve, real_solve))
+        self.assertTrue(approx_equal(approx_solve, real_solve, 2e-4))
 
 
 class TestPivotedCholeskyBatch(unittest.TestCase):
+
     def test_pivoted_cholesky(self):
         size = 100
-        train_x = Variable(torch.cat([
+        train_x = torch.cat([
             torch.linspace(0, 1, size).unsqueeze(0),
             torch.linspace(0, 0.5, size).unsqueeze(0),
-        ], 0)).unsqueeze(-1)
-        covar_matrix = RBFKernel()(train_x, train_x).data
+        ], 0).unsqueeze(-1)
+        covar_matrix = RBFKernel()(train_x, train_x)
         piv_chol = pivoted_cholesky.pivoted_cholesky(covar_matrix, 10)
         covar_approx = piv_chol.transpose(1, 2).matmul(piv_chol)
 
-        self.assertTrue(approx_equal(covar_approx, covar_matrix))
+        self.assertTrue(approx_equal(covar_approx, covar_matrix, 2e-4))
 
     def test_solve(self):
         size = 100
-        train_x = Variable(torch.cat([
+        train_x = torch.cat([
             torch.linspace(0, 1, size).unsqueeze(0),
             torch.linspace(0, 0.5, size).unsqueeze(0),
-        ], 0)).unsqueeze(-1)
-        covar_matrix = RBFKernel()(train_x, train_x).data
+        ], 0).unsqueeze(-1)
+        covar_matrix = RBFKernel()(train_x, train_x)
         piv_chol = pivoted_cholesky.pivoted_cholesky(covar_matrix, 10)
         woodbury_factor = pivoted_cholesky.woodbury_factor(piv_chol, 1)
 
@@ -79,9 +82,11 @@ class TestPivotedCholeskyBatch(unittest.TestCase):
             shifted_covar_matrix[0].inverse().matmul(rhs_vector[0]).unsqueeze(0),
             shifted_covar_matrix[1].inverse().matmul(rhs_vector[1]).unsqueeze(0),
         ], 0)
-        approx_solve = pivoted_cholesky.woodbury_solve(rhs_vector, piv_chol, woodbury_factor, 1)
+        approx_solve = pivoted_cholesky.woodbury_solve(
+            rhs_vector, piv_chol, woodbury_factor, 1,
+        )
 
-        self.assertTrue(approx_equal(approx_solve, real_solve))
+        self.assertTrue(approx_equal(approx_solve, real_solve, 2e-4))
 
 
 if __name__ == '__main__':