Repository: cornellius-gp/gpytorch

Issue:
Deprecate log space parameters in favor of softplus
I’m getting very good feedback from a bunch of people using softplus in place of exp to enforce positivity constraints on some of the hyperparameters. It appears to require very little tweaking of optimization hyperparameters, and significantly improves model robustness and performance.

#383 lets us gracefully deprecate `log_*` parameters in favor of this in a way that we should be able to preserve model loading, calls to `model.covar_module.log_lengthscale = ...` and others. Given these two factors (strong empirical benefits, relatively easy code transition), I am now in favor of putting this in for 0.1.0.

@gpleiss are you on board for a 0.1.0 release given that we can gracefully deprecate `log` parameters now, or would you still prefer 0.2.0 for the switch? 

Discussion:

Patch:
diff --git a/docs/source/conf.py b/docs/source/conf.py
index 620e1fb7..41c57de9 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -67,6 +67,7 @@ MOCK_MODULES = [
     "torch",
     "torch.autograd",
     "torch.nn",
+    "torch.nn.functional",
     "torch.optim",
     "torch.utils",
     "torch.utils.data",
diff --git a/gpytorch/kernels/cosine_kernel.py b/gpytorch/kernels/cosine_kernel.py
index bc7eaa4b..ba34165f 100644
--- a/gpytorch/kernels/cosine_kernel.py
+++ b/gpytorch/kernels/cosine_kernel.py
@@ -4,6 +4,7 @@ import math
 import torch
 from .kernel import Kernel
 from ..utils.deprecation import _deprecate_kwarg
+from torch.nn.functional import softplus
 
 
 class CosineKernel(Kernel):
@@ -33,7 +34,7 @@ class CosineKernel(Kernel):
             The minimum value that the lengthscale/period length can take
             (prevents divide by zero errors). Default: `1e-6`.
         :attr:`param_transform` (function, optional):
-            Set this if you want to use something other than torch.exp to ensure positiveness of parameters.
+            Set this if you want to use something other than softplus to ensure positiveness of parameters.
         :attr:`inv_param_transform` (function, optional):
             Set this to allow setting parameters directly in transformed space and sampling from priors.
             Automatically inferred for common transformations such as torch.exp or torch.nn.functional.softplus.
@@ -61,7 +62,7 @@ class CosineKernel(Kernel):
         batch_size=1,
         period_length_prior=None,
         eps=1e-6,
-        param_transform=torch.exp,
+        param_transform=softplus,
         inv_param_transform=None,
         **kwargs
     ):
@@ -72,7 +73,7 @@ class CosineKernel(Kernel):
             active_dims=active_dims, param_transform=param_transform, inv_param_transform=inv_param_transform
         )
         self.eps = eps
-        self.register_parameter(name="log_period_length", parameter=torch.nn.Parameter(torch.zeros(batch_size, 1, 1)))
+        self.register_parameter(name="raw_period_length", parameter=torch.nn.Parameter(torch.zeros(batch_size, 1, 1)))
         if period_length_prior is not None:
             self.register_prior(
                 "period_length_prior",
@@ -83,7 +84,7 @@ class CosineKernel(Kernel):
 
     @property
     def period_length(self):
-        return self._param_transform(self.log_period_length).clamp(self.eps, 1e5)
+        return self._param_transform(self.raw_period_length).clamp(self.eps, 1e5)
 
     @period_length.setter
     def period_length(self, value):
diff --git a/gpytorch/kernels/index_kernel.py b/gpytorch/kernels/index_kernel.py
index cc73d3fe..afacbd54 100644
--- a/gpytorch/kernels/index_kernel.py
+++ b/gpytorch/kernels/index_kernel.py
@@ -3,6 +3,7 @@
 import torch
 from .kernel import Kernel
 from ..lazy import DiagLazyTensor, InterpolatedLazyTensor, PsdSumLazyTensor, RootLazyTensor
+from torch.nn.functional import softplus
 
 
 class IndexKernel(Kernel):
@@ -29,7 +30,7 @@ class IndexKernel(Kernel):
         :attr:`prior` (:obj:`gpytorch.priors.Prior`):
             Prior for :math:`B` matrix.
         :attr:`param_transform` (function, optional):
-            Set this if you want to use something other than torch.exp to ensure positiveness of parameters.
+            Set this if you want to use something other than softplus to ensure positiveness of parameters.
         :attr:`inv_param_transform` (function, optional):
             Set this to allow setting parameters directly in transformed space and sampling from priors.
             Automatically inferred for common transformations such as torch.exp or torch.nn.functional.softplus.
@@ -42,7 +43,7 @@ class IndexKernel(Kernel):
     """
 
     def __init__(
-        self, num_tasks, rank=1, batch_size=1, prior=None, param_transform=torch.exp, inv_param_transform=None
+        self, num_tasks, rank=1, batch_size=1, prior=None, param_transform=softplus, inv_param_transform=None
     ):
         if rank > num_tasks:
             raise RuntimeError("Cannot create a task covariance matrix larger than the number of tasks")
@@ -50,13 +51,13 @@ class IndexKernel(Kernel):
         self.register_parameter(
             name="covar_factor", parameter=torch.nn.Parameter(torch.randn(batch_size, num_tasks, rank))
         )
-        self.register_parameter(name="log_var", parameter=torch.nn.Parameter(torch.randn(batch_size, num_tasks)))
+        self.register_parameter(name="raw_var", parameter=torch.nn.Parameter(torch.randn(batch_size, num_tasks)))
         if prior is not None:
             self.register_prior("IndexKernelPrior", prior, self._eval_covar_matrix)
 
     @property
     def var(self):
-        return self._param_transform(self.log_var)
+        return self._param_transform(self.raw_var)
 
     @var.setter
     def var(self, value):
diff --git a/gpytorch/kernels/kernel.py b/gpytorch/kernels/kernel.py
index adc46ae3..9de5aa0d 100644
--- a/gpytorch/kernels/kernel.py
+++ b/gpytorch/kernels/kernel.py
@@ -8,6 +8,7 @@ from ..module import Module
 from .. import settings
 from ..utils.deprecation import _deprecate_kwarg
 from ..utils.transforms import _get_inv_param_transform
+from torch.nn.functional import softplus
 
 
 class Kernel(Module):
@@ -67,7 +68,7 @@ class Kernel(Module):
         :attr:`lengthscale_prior` (Prior, optional):
             Set this if you want to apply a prior to the lengthscale parameter.  Default: `None`
         :attr:`param_transform` (function, optional):
-            Set this if you want to use something other than torch.exp to ensure positiveness of parameters.
+            Set this if you want to use something other than softplus to ensure positiveness of parameters.
         :attr:`inv_param_transform` (function, optional):
             Set this to allow setting parameters directly in transformed space and sampling from priors.
             Automatically inferred for common transformations such as torch.exp or torch.nn.functional.softplus.
@@ -93,7 +94,7 @@ class Kernel(Module):
         batch_size=1,
         active_dims=None,
         lengthscale_prior=None,
-        param_transform=torch.exp,
+        param_transform=softplus,
         inv_param_transform=None,
         eps=1e-6,
         **kwargs
@@ -112,7 +113,7 @@ class Kernel(Module):
             self.eps = eps
             lengthscale_num_dims = 1 if ard_num_dims is None else ard_num_dims
             self.register_parameter(
-                name="log_lengthscale", parameter=torch.nn.Parameter(torch.zeros(batch_size, 1, lengthscale_num_dims))
+                name="raw_lengthscale", parameter=torch.nn.Parameter(torch.zeros(batch_size, 1, lengthscale_num_dims))
             )
             if lengthscale_prior is not None:
                 self.register_prior(
@@ -126,7 +127,7 @@ class Kernel(Module):
     @property
     def lengthscale(self):
         if self.has_lengthscale:
-            return self._param_transform(self.log_lengthscale).clamp(self.eps, 1e5)
+            return self._param_transform(self.raw_lengthscale).clamp(self.eps, 1e5)
         else:
             return None
 
diff --git a/gpytorch/kernels/matern_kernel.py b/gpytorch/kernels/matern_kernel.py
index 839e4c13..56e4cf97 100644
--- a/gpytorch/kernels/matern_kernel.py
+++ b/gpytorch/kernels/matern_kernel.py
@@ -4,6 +4,7 @@ import math
 import torch
 from .kernel import Kernel
 from ..utils.deprecation import _deprecate_kwarg
+from torch.nn.functional import softplus
 
 
 class MaternKernel(Kernel):
@@ -51,7 +52,7 @@ class MaternKernel(Kernel):
             Set this if you want
             to apply a prior to the lengthscale parameter.  Default: `None`
         :attr:`param_transform` (function, optional):
-            Set this if you want to use something other than torch.exp to ensure positiveness of parameters.
+            Set this if you want to use something other than softplus to ensure positiveness of parameters.
         :attr:`inv_param_transform` (function, optional):
             Set this to allow setting parameters directly in transformed space and sampling from priors.
             Automatically inferred for common transformations such as torch.exp or torch.nn.functional.softplus.
@@ -86,7 +87,7 @@ class MaternKernel(Kernel):
         batch_size=1,
         active_dims=None,
         lengthscale_prior=None,
-        param_transform=torch.exp,
+        param_transform=softplus,
         inv_param_transform=None,
         eps=1e-6,
         **kwargs
diff --git a/gpytorch/kernels/periodic_kernel.py b/gpytorch/kernels/periodic_kernel.py
index 0daa6796..b7a1a2cf 100644
--- a/gpytorch/kernels/periodic_kernel.py
+++ b/gpytorch/kernels/periodic_kernel.py
@@ -4,6 +4,7 @@ import math
 import torch
 from .kernel import Kernel
 from ..utils.deprecation import _deprecate_kwarg
+from torch.nn.functional import softplus
 
 
 class PeriodicKernel(Kernel):
@@ -44,7 +45,7 @@ class PeriodicKernel(Kernel):
         :attr:`lengthscale_prior` (Prior, optional):
             Set this if you want to apply a prior to the lengthscale parameter.  Default: `None`.
         :attr:`param_transform` (function, optional):
-            Set this if you want to use something other than torch.exp to ensure positiveness of parameters.
+            Set this if you want to use something other than softplus to ensure positiveness of parameters.
         :attr:`inv_param_transform` (function, optional):
             Set this to allow setting parameters directly in transformed space and sampling from priors.
             Automatically inferred for common transformations such as torch.exp or torch.nn.functional.softplus.
@@ -77,7 +78,7 @@ class PeriodicKernel(Kernel):
         batch_size=1,
         lengthscale_prior=None,
         period_length_prior=None,
-        param_transform=torch.exp,
+        param_transform=softplus,
         inv_param_transform=None,
         eps=1e-6,
         **kwargs
@@ -95,7 +96,7 @@ class PeriodicKernel(Kernel):
             inv_param_transform=inv_param_transform,
             eps=eps,
         )
-        self.register_parameter(name="log_period_length", parameter=torch.nn.Parameter(torch.zeros(batch_size, 1, 1)))
+        self.register_parameter(name="raw_period_length", parameter=torch.nn.Parameter(torch.zeros(batch_size, 1, 1)))
         if period_length_prior is not None:
             self.register_prior(
                 "period_length_prior",
@@ -106,14 +107,14 @@ class PeriodicKernel(Kernel):
 
     @property
     def period_length(self):
-        return self._param_transform(self.log_period_length).clamp(self.eps, 1e5)
+        return self._param_transform(self.raw_period_length).clamp(self.eps, 1e5)
 
     @period_length.setter
     def period_length(self, value):
         self._set_period_length(value)
 
     def _set_period_length(self, value):
-        self.initialize(log_period_length=self._inv_param_transform(value))
+        self.initialize(raw_period_length=self._inv_param_transform(value))
 
     def forward(self, x1, x2, **params):
         x1_ = x1.div(self.period_length)
diff --git a/gpytorch/kernels/rbf_kernel.py b/gpytorch/kernels/rbf_kernel.py
index 92e2e278..5df3ab0b 100644
--- a/gpytorch/kernels/rbf_kernel.py
+++ b/gpytorch/kernels/rbf_kernel.py
@@ -1,8 +1,8 @@
 #!/usr/bin/env python3
 
 from .kernel import Kernel
-import torch
 from ..utils.deprecation import _deprecate_kwarg
+from torch.nn.functional import softplus
 
 
 class RBFKernel(Kernel):
@@ -38,7 +38,7 @@ class RBFKernel(Kernel):
         :attr:`lengthscale_prior` (Prior, optional):
             Set this if you want to apply a prior to the lengthscale parameter.  Default: `None`.
         :attr:`param_transform` (function, optional):
-            Set this if you want to use something other than torch.exp to ensure positiveness of parameters.
+            Set this if you want to use something other than softplus to ensure positiveness of parameters.
         :attr:`inv_param_transform` (function, optional):
             Set this to allow setting parameters directly in transformed space and sampling from priors.
             Automatically inferred for common transformations such as torch.exp or torch.nn.functional.softplus.
@@ -72,7 +72,7 @@ class RBFKernel(Kernel):
         batch_size=1,
         active_dims=None,
         lengthscale_prior=None,
-        param_transform=torch.exp,
+        param_transform=softplus,
         inv_param_transform=None,
         eps=1e-6,
         **kwargs
diff --git a/gpytorch/kernels/scale_kernel.py b/gpytorch/kernels/scale_kernel.py
index b480488c..69fd4a66 100644
--- a/gpytorch/kernels/scale_kernel.py
+++ b/gpytorch/kernels/scale_kernel.py
@@ -4,6 +4,7 @@ import torch
 from .kernel import Kernel
 from ..utils.deprecation import _deprecate_kwarg
 from ..utils.transforms import _get_inv_param_transform
+from torch.nn.functional import softplus
 
 
 class ScaleKernel(Kernel):
@@ -35,7 +36,7 @@ class ScaleKernel(Kernel):
         :attr:`outputscale_prior` (Prior, optional): Set this if you want to apply a prior to the outputscale
             parameter.  Default: `None`
         :attr:`param_transform` (function, optional):
-            Set this if you want to use something other than torch.exp to ensure positiveness of parameters.
+            Set this if you want to use something other than softplus to ensure positiveness of parameters.
         :attr:`inv_param_transform` (function, optional):
             Set this to allow setting parameters directly in transformed space and sampling from priors.
             Automatically inferred for common transformations such as torch.exp or torch.nn.functional.softplus.
@@ -58,7 +59,7 @@ class ScaleKernel(Kernel):
         base_kernel,
         batch_size=1,
         outputscale_prior=None,
-        param_transform=torch.exp,
+        param_transform=softplus,
         inv_param_transform=None,
         **kwargs
     ):
@@ -67,7 +68,7 @@ class ScaleKernel(Kernel):
         self.base_kernel = base_kernel
         self._param_transform = param_transform
         self._inv_param_transform = _get_inv_param_transform(param_transform, inv_param_transform)
-        self.register_parameter(name="log_outputscale", parameter=torch.nn.Parameter(torch.zeros(batch_size)))
+        self.register_parameter(name="raw_outputscale", parameter=torch.nn.Parameter(torch.zeros(batch_size)))
         if outputscale_prior is not None:
             self.register_prior(
                 "outputscale_prior", outputscale_prior, lambda: self.outputscale, lambda v: self._set_outputscale(v)
@@ -75,7 +76,7 @@ class ScaleKernel(Kernel):
 
     @property
     def outputscale(self):
-        return self._param_transform(self.log_outputscale)
+        return self._param_transform(self.raw_outputscale)
 
     @outputscale.setter
     def outputscale(self, value):
diff --git a/gpytorch/kernels/spectral_mixture_kernel.py b/gpytorch/kernels/spectral_mixture_kernel.py
index a258fb97..9dcb43cf 100644
--- a/gpytorch/kernels/spectral_mixture_kernel.py
+++ b/gpytorch/kernels/spectral_mixture_kernel.py
@@ -5,6 +5,7 @@ import math
 import torch
 from .kernel import Kernel
 from ..utils.deprecation import _deprecate_kwarg
+from torch.nn.functional import softplus
 
 logger = logging.getLogger()
 
@@ -35,7 +36,7 @@ class SpectralMixtureKernel(Kernel):
             Set this if you want to compute the covariance of only a few input dimensions. The ints
             corresponds to the indices of the dimensions. Default: `None`.
         :attr:`param_transform` (function, optional):
-            Set this if you want to use something other than torch.exp to ensure positiveness of parameters.
+            Set this if you want to use something other than softplus to ensure positiveness of parameters.
         :attr:`inv_param_transform` (function, optional):
             Set this to allow setting parameters directly in transformed space and sampling from priors.
             Automatically inferred for common transformations such as torch.exp or torch.nn.functional.softplus.
@@ -78,7 +79,7 @@ class SpectralMixtureKernel(Kernel):
         mixture_scales_prior=None,
         mixture_means_prior=None,
         mixture_weights_prior=None,
-        param_transform=torch.exp,
+        param_transform=softplus,
         inv_param_transform=None,
         **kwargs
     ):
@@ -107,23 +108,23 @@ class SpectralMixtureKernel(Kernel):
         self.eps = eps
 
         self.register_parameter(
-            name="log_mixture_weights", parameter=torch.nn.Parameter(torch.zeros(self.batch_size, self.num_mixtures))
+            name="raw_mixture_weights", parameter=torch.nn.Parameter(torch.zeros(self.batch_size, self.num_mixtures))
         )
         ms_shape = torch.Size([self.batch_size, self.num_mixtures, 1, self.ard_num_dims])
-        self.register_parameter(name="log_mixture_means", parameter=torch.nn.Parameter(torch.zeros(ms_shape)))
-        self.register_parameter(name="log_mixture_scales", parameter=torch.nn.Parameter(torch.zeros(ms_shape)))
+        self.register_parameter(name="raw_mixture_means", parameter=torch.nn.Parameter(torch.zeros(ms_shape)))
+        self.register_parameter(name="raw_mixture_scales", parameter=torch.nn.Parameter(torch.zeros(ms_shape)))
 
     @property
     def mixture_scales(self):
-        return self._param_transform(self.log_mixture_scales).clamp(self.eps, 1e5)
+        return self._param_transform(self.raw_mixture_scales).clamp(self.eps, 1e5)
 
     @property
     def mixture_means(self):
-        return self._param_transform(self.log_mixture_means).clamp(self.eps, 1e5)
+        return self._param_transform(self.raw_mixture_means).clamp(self.eps, 1e5)
 
     @property
     def mixture_weights(self):
-        return self._param_transform(self.log_mixture_weights).clamp(self.eps, 1e5)
+        return self._param_transform(self.raw_mixture_weights).clamp(self.eps, 1e5)
 
     def initialize_from_data(self, train_x, train_y, **kwargs):
         if not torch.is_tensor(train_x) or not torch.is_tensor(train_y):
@@ -141,11 +142,14 @@ class SpectralMixtureKernel(Kernel):
             min_dist[:, ind] = min_dist_sort[(torch.nonzero(min_dist_sort[:, ind]))[0], ind]
 
         # Inverse of lengthscales should be drawn from truncated Gaussian | N(0, max_dist^2) |
-        self.log_mixture_scales.data.normal_().mul_(max_dist).abs_().pow_(-1).log_()
+        self.raw_mixture_scales.data.normal_().mul_(max_dist).abs_().pow_(-1)
+        self.raw_mixture_scales.data = self._inv_param_transform(self.raw_mixture_scales.data)
         # Draw means from Unif(0, 0.5 / minimum distance between two points)
-        self.log_mixture_means.data.uniform_().mul_(0.5).div_(min_dist).log_()
+        self.raw_mixture_means.data.uniform_().mul_(0.5).div_(min_dist)
+        self.raw_mixture_means.data = self._inv_param_transform(self.raw_mixture_means.data)
         # Mixture weights should be roughly the stdv of the y values divided by the number of mixtures
-        self.log_mixture_weights.data.fill_(train_y.std() / self.num_mixtures).log_()
+        self.raw_mixture_weights.data.fill_(train_y.std() / self.num_mixtures)
+        self.raw_mixture_weights.data = self._inv_param_transform(self.raw_mixture_weights.data)
 
     def forward(self, x1, x2, **params):
         batch_size, n, num_dims = x1.size()
diff --git a/gpytorch/likelihoods/gaussian_likelihood.py b/gpytorch/likelihoods/gaussian_likelihood.py
index 10d2235d..ff60f3dd 100644
--- a/gpytorch/likelihoods/gaussian_likelihood.py
+++ b/gpytorch/likelihoods/gaussian_likelihood.py
@@ -9,31 +9,32 @@ from ..lazy import DiagLazyTensor
 from .. import settings
 from ..utils.deprecation import _deprecate_kwarg
 from ..utils.transforms import _get_inv_param_transform
+from torch.nn.functional import softplus
 
 
 class GaussianLikelihood(Likelihood):
     r"""
     """
 
-    def __init__(self, noise_prior=None, batch_size=1, param_transform=torch.exp, inv_param_transform=None, **kwargs):
+    def __init__(self, noise_prior=None, batch_size=1, param_transform=softplus, inv_param_transform=None, **kwargs):
         noise_prior = _deprecate_kwarg(kwargs, "log_noise_prior", "noise_prior", noise_prior)
         super(GaussianLikelihood, self).__init__()
         self._param_transform = param_transform
         self._inv_param_transform = _get_inv_param_transform(param_transform, inv_param_transform)
-        self.register_parameter(name="log_noise", parameter=torch.nn.Parameter(torch.zeros(batch_size, 1)))
+        self.register_parameter(name="raw_noise", parameter=torch.nn.Parameter(torch.zeros(batch_size, 1)))
         if noise_prior is not None:
             self.register_prior("noise_prior", noise_prior, lambda: self.noise, lambda v: self._set_noise(v))
 
     @property
     def noise(self):
-        return self._param_transform(self.log_noise)
+        return self._param_transform(self.raw_noise)
 
     @noise.setter
     def noise(self, value):
         self._set_noise(value)
 
     def _set_noise(self, value):
-        self.initialize(log_noise=self._inv_param_transform(value))
+        self.initialize(raw_noise=self._inv_param_transform(value))
 
     def forward(self, input):
         if not isinstance(input, MultivariateNormal):
diff --git a/gpytorch/likelihoods/multitask_gaussian_likelihood.py b/gpytorch/likelihoods/multitask_gaussian_likelihood.py
index 3f182975..afb3d424 100644
--- a/gpytorch/likelihoods/multitask_gaussian_likelihood.py
+++ b/gpytorch/likelihoods/multitask_gaussian_likelihood.py
@@ -6,6 +6,7 @@ from ..lazy import DiagLazyTensor, KroneckerProductLazyTensor, RootLazyTensor
 from ..likelihoods import GaussianLikelihood
 from .. import settings
 from ..utils.deprecation import _deprecate_kwarg
+from torch.nn.functional import softplus
 
 
 class MultitaskGaussianLikelihood(GaussianLikelihood):
@@ -25,7 +26,7 @@ class MultitaskGaussianLikelihood(GaussianLikelihood):
         task_prior=None,
         batch_size=1,
         noise_prior=None,
-        param_transform=torch.exp,
+        param_transform=softplus,
         inv_param_transform=None,
         **kwargs
     ):
@@ -50,7 +51,7 @@ class MultitaskGaussianLikelihood(GaussianLikelihood):
 
         if rank == 0:
             self.register_parameter(
-                name="log_task_noises", parameter=torch.nn.Parameter(torch.zeros(batch_size, num_tasks))
+                name="raw_task_noises", parameter=torch.nn.Parameter(torch.zeros(batch_size, num_tasks))
             )
             if task_prior is not None:
                 raise RuntimeError("Cannot set a `task_prior` if rank=0")
@@ -62,9 +63,9 @@ class MultitaskGaussianLikelihood(GaussianLikelihood):
                 self.register_prior("MultitaskErrorCovariancePrior", task_prior, self._eval_covar_matrix)
         self.num_tasks = num_tasks
 
-    def _eval_covar_matrix(self, task_noise_covar_factor, log_noise):
+    def _eval_covar_matrix(self, task_noise_covar_factor, raw_noise):
         num_tasks = task_noise_covar_factor.size(0)
-        noise = self._param_transform(log_noise)
+        noise = self._param_transform(raw_noise)
         D = noise * torch.eye(num_tasks, dtype=noise.dtype, device=noise.device)
         return task_noise_covar_factor.matmul(task_noise_covar_factor.transpose(-1, -2)) + D
 
@@ -79,7 +80,7 @@ class MultitaskGaussianLikelihood(GaussianLikelihood):
         an identity matrix with size equal to the data and a (not necessarily diagonal) matrix containing the task
         noises :math:`D_{t}`.
 
-        We also incorporate a shared `log_noise` parameter from the base
+        We also incorporate a shared `raw_noise` parameter from the base
         :class:`gpytorch.likelihoods.GaussianLikelihood` that we extend.
 
         The final covariance matrix after this method is then :math:`K + D_{t} \otimes I_{n} + \sigma^{2}I_{nt}`.
@@ -94,8 +95,8 @@ class MultitaskGaussianLikelihood(GaussianLikelihood):
         """
         mean, covar = input.mean, input.lazy_covariance_matrix
 
-        if hasattr(self, "log_task_noises"):
-            noises = self._param_transform(self.log_task_noises)
+        if hasattr(self, "raw_task_noises"):
+            noises = self._param_transform(self.raw_task_noises)
             if covar.ndimension() == 2:
                 if settings.debug.on() and noises.size(0) > 1:
                     raise RuntimeError(
diff --git a/gpytorch/module.py b/gpytorch/module.py
index 9c676a67..f603e319 100644
--- a/gpytorch/module.py
+++ b/gpytorch/module.py
@@ -5,9 +5,10 @@ from collections import OrderedDict
 import torch
 from torch import nn
 from torch.distributions import Distribution
-
+import itertools
 from .lazy import LazyTensor
 from .utils.deprecation import DeprecationError
+import warnings
 
 
 class Module(nn.Module):
@@ -61,18 +62,53 @@ class Module(nn.Module):
             yield param
 
     def initialize(self, **kwargs):
+        # TODO: Change to initialize actual parameter (e.g. lengthscale) rather than untransformed parameter.
         """
         Set a value for a parameter
 
         kwargs: (param_name, value) - parameter to initialize
         Value can take the form of a tensor, a float, or an int
         """
+        from .kernels import (
+            CosineKernel,
+            IndexKernel,
+            MaternKernel,
+            PeriodicKernel,
+            RBFKernel,
+            ScaleKernel,
+            SpectralMixtureKernel,
+        )
+
+        from .likelihoods import GaussianLikelihood, MultitaskGaussianLikelihood
+
+        modules_with_log_params = [
+            CosineKernel,
+            IndexKernel,
+            MaternKernel,
+            PeriodicKernel,
+            RBFKernel,
+            ScaleKernel,
+            SpectralMixtureKernel,
+            GaussianLikelihood,
+            MultitaskGaussianLikelihood,
+        ]
+
         for name, val in kwargs.items():
+            if isinstance(val, int):
+                val = float(val)
+            if any([isinstance(self, mod_type) for mod_type in modules_with_log_params]) and 'log_' in name:
+                base_name = name.split('log_')[1]
+                name = 'raw_' + base_name
+                if not torch.is_tensor(val):
+                    val = self._inv_param_transform(torch.tensor(val).exp()).item()
+                else:
+                    val = self._inv_param_transform(val.exp())
+
             if name not in self._parameters:
                 raise AttributeError("Unknown parameter {p} for {c}".format(p=name, c=self.__class__.__name__))
             if torch.is_tensor(val):
                 self.__getattr__(name).data.copy_(val)
-            elif isinstance(val, float) or isinstance(val, int):
+            elif isinstance(val, float):
                 self.__getattr__(name).data.fill_(val)
             else:
                 raise AttributeError("Type {t} not valid to initialize parameter {p}".format(t=type(val), p=name))
@@ -213,6 +249,109 @@ class Module(nn.Module):
         for _, param in self.named_variational_parameters():
             yield param
 
+    def _load_from_state_dict(
+        self, state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs
+    ):
+        from .kernels import (
+            CosineKernel,
+            IndexKernel,
+            MaternKernel,
+            PeriodicKernel,
+            RBFKernel,
+            ScaleKernel,
+            SpectralMixtureKernel,
+        )
+
+        from .likelihoods import GaussianLikelihood, MultitaskGaussianLikelihood
+
+        local_name_params = itertools.chain(self._parameters.items(), self._buffers.items())
+        local_state = {k: v.data for k, v in local_name_params if v is not None}
+
+        modules_with_log_params = [
+            CosineKernel,
+            IndexKernel,
+            MaternKernel,
+            PeriodicKernel,
+            RBFKernel,
+            ScaleKernel,
+            SpectralMixtureKernel,
+            GaussianLikelihood,
+            MultitaskGaussianLikelihood,
+        ]
+
+        if not any([isinstance(self, mod_type) for mod_type in modules_with_log_params]):
+            return super(Module, self)._load_from_state_dict(
+                state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs
+            )
+
+        super(Module, self)._load_from_state_dict(
+            state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs
+        )
+
+        # Load log space parameters and throw deprecation warnings.
+        for name, param in local_state.items():
+            if 'raw_' in name:
+                base_name = name.split('raw_')[1]
+                log_name = "log_" + base_name
+                log_key = prefix + log_name
+                if log_key in state_dict and log_key not in local_state:
+                    warnings.warn(
+                        "The '{log_name}' parameter is deprecated in favor of '{name}' because we no longer ensure "
+                        "positiveness with torch.exp for improved stability reasons and will be removed in a future "
+                        "release. To solve this issue, just save this model "
+                        "again.".format(log_name=log_name, name=name),
+                        DeprecationWarning,
+                    )
+                    input_param = state_dict[log_key]
+                    if isinstance(input_param, nn.Parameter):
+                        input_param = input_param.data
+
+                    real_input_param = self._inv_param_transform(input_param.exp())
+                    param.copy_(real_input_param)
+
+                    if prefix + name in missing_keys:
+                        missing_keys.remove(prefix + name)
+                    if prefix + name in unexpected_keys:
+                        unexpected_keys.remove(prefix + log_name)
+
+    def __getattr__(self, name):
+        from .kernels import (
+            CosineKernel,
+            IndexKernel,
+            MaternKernel,
+            PeriodicKernel,
+            RBFKernel,
+            ScaleKernel,
+            SpectralMixtureKernel,
+        )
+
+        from .likelihoods import GaussianLikelihood, MultitaskGaussianLikelihood
+
+        modules_with_log_params = [
+            CosineKernel,
+            IndexKernel,
+            MaternKernel,
+            PeriodicKernel,
+            RBFKernel,
+            ScaleKernel,
+            SpectralMixtureKernel,
+            GaussianLikelihood,
+            MultitaskGaussianLikelihood,
+        ]
+
+        if not any([isinstance(self, mod_type) for mod_type in modules_with_log_params]) or 'log_' not in name:
+            return super(Module, self).__getattr__(name)
+        else:
+            base_name = name.split('log_')[1]  # e.g. log_lengthscale -> lengthscale
+            raw_name = 'raw_' + base_name
+            warnings.warn(
+                "The '{log_name}' parameter is deprecated in favor of '{name}'  because we no longer ensure "
+                "positiveness with torch.exp for improved stability reasons and will be removed in a future "
+                "release.".format(log_name=name, name=raw_name),
+                DeprecationWarning,
+            )
+            return super(Module, self).__getattribute__(base_name).log()  # Get real param value and transform to log
+
 
 def _extract_named_added_loss_terms(module, memo=None, prefix=""):
     if memo is None:


Test patch:
diff --git a/test/examples/test_batch_gp_regression.py b/test/examples/test_batch_gp_regression.py
index 60bf7680..80477359 100644
--- a/test/examples/test_batch_gp_regression.py
+++ b/test/examples/test_batch_gp_regression.py
@@ -69,21 +69,16 @@ class TestBatchGPRegression(unittest.TestCase):
 
     def test_train_on_single_set_test_on_batch(self):
         # We're manually going to set the hyperparameters to something they shouldn't be
-        likelihood = GaussianLikelihood(
-            noise_prior=gpytorch.priors.NormalPrior(loc=torch.zeros(1), scale=torch.ones(1))
-        )
+        likelihood = GaussianLikelihood()
         gp_model = ExactGPModel(train_x1, train_y1, likelihood)
         mll = gpytorch.ExactMarginalLogLikelihood(likelihood, gp_model)
 
         # Find optimal model hyperparameters
         gp_model.train()
         likelihood.train()
-        optimizer = optim.Adam(list(gp_model.parameters()) + list(likelihood.parameters()), lr=0.1)
-        optimizer.n_iter = 0
-        gp_model.train()
-        likelihood.train()
         optimizer = optim.Adam(gp_model.parameters(), lr=0.1)
-        for _ in range(50):
+        optimizer.n_iter = 0
+        for _ in range(75):
             optimizer.zero_grad()
             output = gp_model(train_x1)
             loss = -mll(output, train_y1).sum()
diff --git a/test/examples/test_grid_gp_regression.py b/test/examples/test_grid_gp_regression.py
index 95437e39..ca8e9c9e 100644
--- a/test/examples/test_grid_gp_regression.py
+++ b/test/examples/test_grid_gp_regression.py
@@ -94,7 +94,7 @@ class TestGridGPRegression(unittest.TestCase):
             train_preds = likelihood(gp_model(train_x)).mean
             mean_abs_error = torch.mean(torch.abs(train_y - train_preds))
 
-        self.assertLess(mean_abs_error.squeeze().item(), 0.15)
+        self.assertLess(mean_abs_error.squeeze().item(), 0.3)
 
 
 if __name__ == "__main__":
diff --git a/test/examples/test_kissgp_dkl_regression.py b/test/examples/test_kissgp_dkl_regression.py
index f79ba729..eb5e3107 100644
--- a/test/examples/test_kissgp_dkl_regression.py
+++ b/test/examples/test_kissgp_dkl_regression.py
@@ -145,10 +145,10 @@ class TestDKLRegression(unittest.TestCase):
 
             # Now bump up the likelihood to something huge
             # This will make it easy to calculate the variance
-            likelihood.log_noise.data.fill_(3)
+            likelihood.raw_noise.data.fill_(3)
             test_function_predictions = likelihood(gp_model(train_x))
 
-            noise = likelihood.log_noise.exp()
+            noise = likelihood.noise
             var_diff = (test_function_predictions.variance - noise).abs()
             self.assertLess(torch.max(var_diff / noise), 0.15)
 
diff --git a/test/examples/test_kissgp_gp_regression.py b/test/examples/test_kissgp_gp_regression.py
index 9ffe5c46..6eed15ad 100644
--- a/test/examples/test_kissgp_gp_regression.py
+++ b/test/examples/test_kissgp_gp_regression.py
@@ -129,10 +129,10 @@ class TestKISSGPRegression(unittest.TestCase):
 
             # Now bump up the likelihood to something huge
             # This will make it easy to calculate the variance
-            likelihood.log_noise.data.fill_(3)
+            likelihood.raw_noise.data.fill_(3)
             test_function_predictions = likelihood(gp_model(train_x))
 
-            noise = likelihood.log_noise.exp()
+            noise = likelihood.noise
             var_diff = (test_function_predictions.variance - noise).abs()
             self.assertLess(torch.max(var_diff / noise), 0.05)
 
diff --git a/test/examples/test_kissgp_white_noise_regression.py b/test/examples/test_kissgp_white_noise_regression.py
index a232ef81..a15b460a 100644
--- a/test/examples/test_kissgp_white_noise_regression.py
+++ b/test/examples/test_kissgp_white_noise_regression.py
@@ -131,10 +131,10 @@ class TestKISSGPWhiteNoiseRegression(unittest.TestCase):
 
             # Now bump up the likelihood to something huge
             # This will make it easy to calculate the variance
-            likelihood.log_noise.data.fill_(3)
+            likelihood.raw_noise.data.fill_(3)
             test_function_predictions = likelihood(gp_model(train_x))
 
-            noise = likelihood.log_noise.exp()
+            noise = likelihood.noise
             var_diff = (test_function_predictions.variance - noise).abs()
             self.assertLess(torch.max(var_diff / noise), 0.05)
 
diff --git a/test/examples/test_simple_gp_regression.py b/test/examples/test_simple_gp_regression.py
index b13db77d..8e278417 100644
--- a/test/examples/test_simple_gp_regression.py
+++ b/test/examples/test_simple_gp_regression.py
@@ -54,7 +54,7 @@ class TestSimpleGPRegression(unittest.TestCase):
         gp_model = ExactGPModel(None, None, likelihood)
         # Update lengthscale prior to accommodate extreme parameters
         gp_model.covar_module.base_kernel.register_prior(
-            "log_lengthscale_prior", SmoothedBoxPrior(exp(-10), exp(10), sigma=0.5), "log_lengthscale"
+            "log_lengthscale_prior", SmoothedBoxPrior(exp(-10), exp(10), sigma=0.5), "raw_lengthscale"
         )
         gp_model.mean_module.initialize(constant=1.5)
         gp_model.covar_module.base_kernel.initialize(log_lengthscale=0)
@@ -66,20 +66,17 @@ class TestSimpleGPRegression(unittest.TestCase):
 
         # The model should predict in prior mode
         function_predictions = likelihood(gp_model(train_x))
+        correct_variance = gp_model.covar_module.outputscale + likelihood.noise
 
         self.assertLess(torch.norm(function_predictions.mean - 1.5), 1e-3)
-        self.assertLess(torch.norm(function_predictions.variance - 2), 1e-3)
+        self.assertLess(torch.norm(function_predictions.variance - correct_variance), 1e-3)
 
     def test_posterior_latent_gp_and_likelihood_without_optimization(self):
         # We're manually going to set the hyperparameters to be ridiculous
-        likelihood = GaussianLikelihood(noise_prior=SmoothedBoxPrior(exp(-3), exp(3), sigma=0.1))
+        likelihood = GaussianLikelihood()
         gp_model = ExactGPModel(train_x, train_y, likelihood)
-        # Update lengthscale prior to accommodate extreme parameters
-        gp_model.covar_module.base_kernel.register_prior(
-            "log_lengthscale_prior", SmoothedBoxPrior(exp(-10), exp(10), sigma=0.5), "log_lengthscale"
-        )
-        gp_model.covar_module.base_kernel.initialize(log_lengthscale=-10)
-        likelihood.initialize(log_noise=-10)
+        gp_model.covar_module.base_kernel.initialize(raw_lengthscale=-15)
+        likelihood.initialize(raw_noise=-15)
 
         # Compute posterior distribution
         gp_model.eval()
@@ -97,7 +94,7 @@ class TestSimpleGPRegression(unittest.TestCase):
         test_function_predictions = gp_model(torch.tensor([1.1]))
 
         self.assertLess(torch.norm(test_function_predictions.mean - 0), 1e-4)
-        self.assertLess(torch.norm(test_function_predictions.variance - 1), 1e-4)
+        self.assertLess(torch.norm(test_function_predictions.variance - gp_model.covar_module.outputscale), 1e-4)
 
     def test_posterior_latent_gp_and_likelihood_with_optimization(self):
         # We're manually going to set the hyperparameters to something they shouldn't be
@@ -111,7 +108,7 @@ class TestSimpleGPRegression(unittest.TestCase):
         # Find optimal model hyperparameters
         gp_model.train()
         likelihood.train()
-        optimizer = optim.Adam(list(gp_model.parameters()) + list(likelihood.parameters()), lr=0.1)
+        optimizer = optim.Adam(list(gp_model.parameters()) + list(likelihood.parameters()), lr=0.15)
         optimizer.n_iter = 0
         for _ in range(50):
             optimizer.zero_grad()
@@ -178,10 +175,10 @@ class TestSimpleGPRegression(unittest.TestCase):
 
             # Now bump up the likelihood to something huge
             # This will make it easy to calculate the variance
-            likelihood.log_noise.data.fill_(3)
+            likelihood.raw_noise.data.fill_(3)
             test_function_predictions = likelihood(gp_model(train_x))
 
-            noise = likelihood.log_noise.exp()
+            noise = likelihood.noise
             var_diff = (test_function_predictions.variance - noise).abs()
 
             self.assertLess(torch.max(var_diff / noise), 0.05)
diff --git a/test/examples/test_white_noise_regression.py b/test/examples/test_white_noise_regression.py
index 7c91c1da..4421a7af 100644
--- a/test/examples/test_white_noise_regression.py
+++ b/test/examples/test_white_noise_regression.py
@@ -57,7 +57,7 @@ class TestWhiteNoiseGPRegression(unittest.TestCase):
             gp_model = ExactGPModel(train_x, train_y, likelihood)
             # Update lengthscale prior to accommodate extreme parameters
             gp_model.rbf_covar_module.register_prior(
-                "log_lengthscale_prior", SmoothedBoxPrior(exp(-10), exp(10), sigma=0.5), "log_lengthscale"
+                "log_lengthscale_prior", SmoothedBoxPrior(exp(-10), exp(10), sigma=0.5), "raw_lengthscale"
             )
             gp_model.rbf_covar_module.initialize(log_lengthscale=-10)
             gp_model.mean_module.initialize(constant=0)
@@ -78,7 +78,7 @@ class TestWhiteNoiseGPRegression(unittest.TestCase):
             test_function_predictions = gp_model(torch.tensor([1.1], dtype=torch.float))
 
             self.assertLess(torch.norm(test_function_predictions.mean - 0), 1e-4)
-            self.assertLess(torch.norm(test_function_predictions.variance - 1), 1e-4)
+            self.assertLess(torch.norm(test_function_predictions.variance - gp_model.covar_module.outputscale), 1e-4)
 
     def test_posterior_latent_gp_and_likelihood_with_optimization(self):
         # We're manually going to set the hyperparameters to something they shouldn't be
@@ -159,10 +159,10 @@ class TestWhiteNoiseGPRegression(unittest.TestCase):
 
             # Now bump up the likelihood to something huge
             # This will make it easy to calculate the variance
-            likelihood.log_noise.data.fill_(3)
+            likelihood.raw_noise.data.fill_(3)
             test_function_predictions = likelihood(gp_model(train_x))
 
-            noise = likelihood.log_noise.exp()
+            noise = likelihood.noise
             var_diff = (test_function_predictions.variance - noise).abs()
 
             self.assertLess(torch.max(var_diff / noise), 0.05)
diff --git a/test/kernels/test_additive_kernel.py b/test/kernels/test_additive_kernel.py
index 11c2edb7..d0727d20 100644
--- a/test/kernels/test_additive_kernel.py
+++ b/test/kernels/test_additive_kernel.py
@@ -75,14 +75,15 @@ class TestAdditiveKernel(unittest.TestCase):
         self.assertLess(torch.norm(res - actual), 2e-5)
 
     def test_computes_sum_radial_basis_function_gradient(self):
+        softplus = torch.nn.functional.softplus
         a = torch.tensor([4, 2, 8], dtype=torch.float).view(3, 1)
         b = torch.tensor([0, 2, 2], dtype=torch.float).view(3, 1)
         lengthscale = 2
 
-        param = math.log(lengthscale) * torch.ones(3, 3)
+        param = math.log(math.exp(lengthscale) - 1) * torch.ones(3, 3)
         param.requires_grad_()
         diffs = a.expand(3, 3) - b.expand(3, 3).transpose(0, 1)
-        actual_output = (-0.5 * (diffs / param.exp()) ** 2).exp()
+        actual_output = (-0.5 * (diffs / softplus(param)) ** 2).exp()
         actual_output.backward(torch.eye(3))
         actual_param_grad = param.grad.sum() * 2
 
@@ -93,18 +94,19 @@ class TestAdditiveKernel(unittest.TestCase):
 
         output = kernel(a, b).evaluate()
         output.backward(gradient=torch.eye(3))
-        res = kernel.kernels[0].log_lengthscale.grad + kernel.kernels[1].log_lengthscale.grad
+        res = kernel.kernels[0].raw_lengthscale.grad + kernel.kernels[1].raw_lengthscale.grad
         self.assertLess(torch.norm(res - actual_param_grad), 2e-5)
 
     def test_computes_sum_three_radial_basis_function_gradient(self):
+        softplus = torch.nn.functional.softplus
         a = torch.tensor([4, 2, 8], dtype=torch.float).view(3, 1)
         b = torch.tensor([0, 2, 2], dtype=torch.float).view(3, 1)
         lengthscale = 2
 
-        param = math.log(lengthscale) * torch.ones(3, 3)
+        param = math.log(math.exp(lengthscale) - 1) * torch.ones(3, 3)
         param.requires_grad_()
         diffs = a.expand(3, 3) - b.expand(3, 3).transpose(0, 1)
-        actual_output = (-0.5 * (diffs / param.exp()) ** 2).exp()
+        actual_output = (-0.5 * (diffs / softplus(param)) ** 2).exp()
         actual_output.backward(torch.eye(3))
         actual_param_grad = param.grad.sum() * 3
 
@@ -117,9 +119,9 @@ class TestAdditiveKernel(unittest.TestCase):
         output = kernel(a, b).evaluate()
         output.backward(gradient=torch.eye(3))
         res = (
-            kernel.kernels[0].log_lengthscale.grad
-            + kernel.kernels[1].log_lengthscale.grad
-            + kernel.kernels[2].log_lengthscale.grad
+            kernel.kernels[0].raw_lengthscale.grad
+            + kernel.kernels[1].raw_lengthscale.grad
+            + kernel.kernels[2].raw_lengthscale.grad
         )
         self.assertLess(torch.norm(res - actual_param_grad), 2e-5)
 
diff --git a/test/kernels/test_rbf_kernel.py b/test/kernels/test_rbf_kernel.py
index 29339757..878fd4f2 100644
--- a/test/kernels/test_rbf_kernel.py
+++ b/test/kernels/test_rbf_kernel.py
@@ -129,6 +129,7 @@ class TestRBFKernel(unittest.TestCase):
         self.assertLess(torch.norm(res - actual), 1e-5)
 
     def test_computes_radial_basis_function_gradient(self):
+        softplus = torch.nn.functional.softplus
         a = torch.tensor([4, 2, 8], dtype=torch.float).view(3, 1)
         b = torch.tensor([0, 2, 2], dtype=torch.float).view(3, 1)
         lengthscale = 2
@@ -136,30 +137,31 @@ class TestRBFKernel(unittest.TestCase):
         kernel = RBFKernel().initialize(log_lengthscale=math.log(lengthscale))
         kernel.eval()
 
-        param = math.log(lengthscale) * torch.ones(3, 3)
+        param = math.log(math.exp(lengthscale) - 1) * torch.ones(3, 3)
         param.requires_grad_()
         diffs = a.expand(3, 3) - b.expand(3, 3).transpose(0, 1)
-        actual_output = (-0.5 * (diffs / param.exp()) ** 2).exp()
+        actual_output = (-0.5 * (diffs / softplus(param)) ** 2).exp()
         actual_output.backward(gradient=torch.eye(3))
         actual_param_grad = param.grad.sum()
 
         output = kernel(a, b).evaluate()
         output.backward(gradient=torch.eye(3))
-        res = kernel.log_lengthscale.grad
+        res = kernel.raw_lengthscale.grad
 
         self.assertLess(torch.norm(res - actual_param_grad), 1e-5)
 
     def test_subset_active_computes_radial_basis_function_gradient(self):
+        softplus = torch.nn.functional.softplus
         a_1 = torch.tensor([4, 2, 8], dtype=torch.float).view(3, 1)
         a_p = torch.tensor([1, 2, 3], dtype=torch.float).view(3, 1)
         a = torch.cat((a_1, a_p), 1)
         b = torch.tensor([0, 2, 2], dtype=torch.float).view(3, 1)
         lengthscale = 2
 
-        param = math.log(lengthscale) * torch.ones(3, 3)
+        param = math.log(math.exp(lengthscale) - 1) * torch.ones(3, 3)
         param.requires_grad_()
         diffs = a_1.expand(3, 3) - b.expand(3, 3).transpose(0, 1)
-        actual_output = (-0.5 * (diffs / param.exp()) ** 2).exp()
+        actual_output = (-0.5 * (diffs / softplus(param)) ** 2).exp()
         actual_output.backward(torch.eye(3))
         actual_param_grad = param.grad.sum()
 
@@ -168,7 +170,7 @@ class TestRBFKernel(unittest.TestCase):
         kernel.eval()
         output = kernel(a, b).evaluate()
         output.backward(gradient=torch.eye(3))
-        res = kernel.log_lengthscale.grad
+        res = kernel.raw_lengthscale.grad
 
         self.assertLess(torch.norm(res - actual_param_grad), 1e-5)