Repository: cornellius-gp/gpytorch
Pull request #376 (2018-11-14 23:09:13)

Revamp priors
As discussed in #374.

This 
1. registers parameter transforms with the parameters
2. makes sure priors are always evaluated on the transformed parameters
3. unifies `parameter_prior` and `derived_prior` into simply `prior` (addressing #181)
4. allows general transforms also for the priors. For instance, by passing in `torch.log` to the `transform` constructor of the `NormalPrior`, you get a log-normal prior.


Patch:
diff --git a/examples/04_Scalable_GP_Regression_1D/KISSGP_Regression_1D_With_Stochastic_Variational_Inference.ipynb b/examples/04_Scalable_GP_Regression_1D/KISSGP_Regression_1D_With_Stochastic_Variational_Inference.ipynb
index 7224c916..18542369 100644
--- a/examples/04_Scalable_GP_Regression_1D/KISSGP_Regression_1D_With_Stochastic_Variational_Inference.ipynb
+++ b/examples/04_Scalable_GP_Regression_1D/KISSGP_Regression_1D_With_Stochastic_Variational_Inference.ipynb
@@ -99,7 +99,7 @@
     "        self.mean_module = gpytorch.means.ConstantMean()\n",
     "        self.covar_module = gpytorch.kernels.ScaleKernel(\n",
     "            gpytorch.kernels.RBFKernel(\n",
-    "                log_lengthscale_prior=gpytorch.priors.SmoothedBoxPrior(\n",
+    "                lengthscale_prior=gpytorch.priors.SmoothedBoxPrior(\n",
     "                    math.exp(-3), math.exp(6), sigma=0.1, log_transform=True\n",
     "                )\n",
     "            )\n",
diff --git a/examples/07_Scalable_GP_Classification_Multidimensional/KISSGP_Kronecker_Classification.ipynb b/examples/07_Scalable_GP_Classification_Multidimensional/KISSGP_Kronecker_Classification.ipynb
index b8ef1256..c314b9cf 100644
--- a/examples/07_Scalable_GP_Classification_Multidimensional/KISSGP_Kronecker_Classification.ipynb
+++ b/examples/07_Scalable_GP_Classification_Multidimensional/KISSGP_Kronecker_Classification.ipynb
@@ -79,7 +79,7 @@
     "        self.mean_module = gpytorch.means.ConstantMean()\n",
     "        self.covar_module = gpytorch.kernels.ScaleKernel(\n",
     "            gpytorch.kernels.RBFKernel(\n",
-    "                log_lengthscale_prior=gpytorch.priors.SmoothedBoxPrior(\n",
+    "                lengthscale_prior=gpytorch.priors.SmoothedBoxPrior(\n",
     "                    exp(0), exp(3), sigma=0.1, log_transform=True\n",
     "                )\n",
     "            )\n",
diff --git a/examples/08_Deep_Kernel_Learning/Deep_Kernel_Learning_DenseNet_CIFAR_Tutorial.ipynb b/examples/08_Deep_Kernel_Learning/Deep_Kernel_Learning_DenseNet_CIFAR_Tutorial.ipynb
index 9bdb4999..c9007e21 100644
--- a/examples/08_Deep_Kernel_Learning/Deep_Kernel_Learning_DenseNet_CIFAR_Tutorial.ipynb
+++ b/examples/08_Deep_Kernel_Learning/Deep_Kernel_Learning_DenseNet_CIFAR_Tutorial.ipynb
@@ -148,7 +148,7 @@
     "                                                   num_dim=num_dim, mixing_params=False, sum_output=False)\n",
     "        self.covar_module = gpytorch.kernels.ScaleKernel(\n",
     "            gpytorch.kernels.RBFKernel(\n",
-    "                log_lengthscale_prior=gpytorch.priors.SmoothedBoxPrior(\n",
+    "                lengthscale_prior=gpytorch.priors.SmoothedBoxPrior(\n",
     "                    math.exp(-1), math.exp(1), sigma=0.1, log_transform=True\n",
     "                )\n",
     "            )\n",
diff --git a/examples/09_Pyro_Integration/Pyro_SVDKL_GridInterp.ipynb b/examples/09_Pyro_Integration/Pyro_SVDKL_GridInterp.ipynb
index 67f7efdb..b65e0f46 100644
--- a/examples/09_Pyro_Integration/Pyro_SVDKL_GridInterp.ipynb
+++ b/examples/09_Pyro_Integration/Pyro_SVDKL_GridInterp.ipynb
@@ -124,7 +124,7 @@
     "                                                       name_prefix=name_prefix)\n",
     "        self.mean_module = gpytorch.means.ConstantMean()\n",
     "        self.covar_module = gpytorch.kernels.ScaleKernel(gpytorch.kernels.RBFKernel(\n",
-    "            log_lengthscale_prior=gpytorch.priors.SmoothedBoxPrior(0.001, 1., sigma=0.1, log_transform=True)\n",
+    "            lengthscale_prior=gpytorch.priors.SmoothedBoxPrior(0.001, 1., sigma=0.1, log_transform=True)\n",
     "        ))\n",
     "\n",
     "    def forward(self, x):\n",
diff --git a/gpytorch/kernels/cosine_kernel.py b/gpytorch/kernels/cosine_kernel.py
index a8902258..33e6086d 100644
--- a/gpytorch/kernels/cosine_kernel.py
+++ b/gpytorch/kernels/cosine_kernel.py
@@ -6,6 +6,7 @@ from __future__ import unicode_literals
 import math
 import torch
 from .kernel import Kernel
+from ..utils.deprecation import _deprecate_kwarg
 
 
 class CosineKernel(Kernel):
@@ -55,21 +56,21 @@ class CosineKernel(Kernel):
     """
 
     def __init__(
-        self, active_dims=None, batch_size=1, log_period_length_prior=None, eps=1e-6, param_transform=torch.exp
+        self, active_dims=None, batch_size=1, period_length_prior=None, eps=1e-6, param_transform=torch.exp, **kwargs
     ):
-        super(CosineKernel, self).__init__(
-            has_lengthscale=False, active_dims=active_dims, param_transform=param_transform
+        period_length_prior = _deprecate_kwarg(
+            kwargs, "log_period_length_prior", "period_length_prior", period_length_prior
         )
+        super(CosineKernel, self).__init__(has_lengthscale=False, active_dims=active_dims)
         self.eps = eps
-        self.register_parameter(
-            name="log_period_length",
-            parameter=torch.nn.Parameter(torch.zeros(batch_size, 1, 1)),
-            prior=log_period_length_prior,
-        )
+        self._param_transform = param_transform
+        self.register_parameter(name="log_period_length", parameter=torch.nn.Parameter(torch.zeros(batch_size, 1, 1)))
+        if period_length_prior is not None:
+            self.register_prior("period_length_prior", period_length_prior, lambda: self.period_length)
 
     @property
     def period_length(self):
-        return self.param_transform(self.log_period_length).clamp(self.eps, 1e5)
+        return self._param_transform(self.log_period_length).clamp(self.eps, 1e5)
 
     def forward(self, x1, x2, **params):
         x1_ = x1.div(self.period_length)
diff --git a/gpytorch/kernels/index_kernel.py b/gpytorch/kernels/index_kernel.py
index 53416807..10584a76 100644
--- a/gpytorch/kernels/index_kernel.py
+++ b/gpytorch/kernels/index_kernel.py
@@ -50,24 +50,25 @@ class IndexKernel(Kernel):
         )
         self.register_parameter(name="log_var", parameter=torch.nn.Parameter(torch.randn(batch_size, num_tasks)))
         if prior is not None:
-            self.register_derived_prior(
-                name="IndexKernelPrior",
-                prior=prior,
-                parameter_names=("covar_factor", "log_var"),
-                transform=self._eval_covar_matrix,
-            )
+            self.register_prior("IndexKernelPrior", prior, self._eval_covar_matrix)
 
-    def _eval_covar_matrix(self, covar_factor, log_var):
-        var = self.param_transform(log_var)
-        return covar_factor.matmul(covar_factor.transpose(-1, -2)) + var.diag()
+    @property
+    def var(self):
+        return self._param_transform(self.log_var)
+
+    def _eval_covar_matrix(self):
+        covar_factor, variance = self.covar_factor, self.var
+        return covar_factor.matmul(covar_factor.transpose(-1, -2)) + variance.diag()
+        # D = var * torch.eye(var.shape[-1], dtype=var.dytpe, device=var.device)
+        # return self.covar_factor.matmul(self.covar_factor.transpose(-1, -2)) + D
 
     @property
     def covar_matrix(self):
-        var = self.param_transform(self.log_var)
+        var = self.var
         res = PsdSumLazyTensor(RootLazyTensor(self.covar_factor), DiagLazyTensor(var))
         return res
 
     def forward(self, i1, i2, **params):
-        covar_matrix = self._eval_covar_matrix(self.covar_factor, self.log_var)
+        covar_matrix = self._eval_covar_matrix()
         res = InterpolatedLazyTensor(base_lazy_tensor=covar_matrix, left_interp_indices=i1, right_interp_indices=i2)
         return res
diff --git a/gpytorch/kernels/kernel.py b/gpytorch/kernels/kernel.py
index 2a507e95..53d002ae 100644
--- a/gpytorch/kernels/kernel.py
+++ b/gpytorch/kernels/kernel.py
@@ -9,6 +9,7 @@ from torch.nn import ModuleList
 from ..lazy import LazyEvaluatedKernelTensor, ZeroLazyTensor
 from ..module import Module
 from .. import settings
+from ..utils.deprecation import _deprecate_kwarg
 
 
 class Kernel(Module):
@@ -51,8 +52,7 @@ class Kernel(Module):
     .. note::
 
         The :attr:`lengthscale` parameter is parameterized on a log scale to constrain it to be positive.
-        You can set a prior on this parameter using the :attr:`log_lengthscale_prior` argument, but
-        be sure that the prior has :attr:`log_transform=True` set.
+        You can set a prior on this parameter using the :attr:`lengthscale_prior` argument.
 
     Base Args:
         :attr:`has_lengthscale` (bool):
@@ -66,7 +66,7 @@ class Kernel(Module):
         :attr:`active_dims` (tuple of ints, optional):
             Set this if you want to compute the covariance of only a few input dimensions. The ints
             corresponds to the indices of the dimensions. Default: `None`.
-        :attr:`log_lengthscale_prior` (Prior, optional):
+        :attr:`lengthscale_prior` (Prior, optional):
             Set this if you want to apply a prior to the lengthscale parameter.  Default: `None`
         :attr:`param_transform` (function, optional):
             Set this if you want to use something other than torch.exp to ensure positiveness of parameters.
@@ -91,10 +91,12 @@ class Kernel(Module):
         ard_num_dims=None,
         batch_size=1,
         active_dims=None,
-        log_lengthscale_prior=None,
+        lengthscale_prior=None,
         param_transform=torch.exp,
         eps=1e-6,
+        **kwargs
     ):
+        lengthscale_prior = _deprecate_kwarg(kwargs, "log_lengthscale_prior", "lengthscale_prior", lengthscale_prior)
         super(Kernel, self).__init__()
         if active_dims is not None and not torch.is_tensor(active_dims):
             active_dims = torch.tensor(active_dims, dtype=torch.long)
@@ -102,15 +104,15 @@ class Kernel(Module):
         self.ard_num_dims = ard_num_dims
         self.batch_size = batch_size
         self.__has_lengthscale = has_lengthscale
-        self.param_transform = param_transform
+        self._param_transform = param_transform
         if has_lengthscale:
             self.eps = eps
             lengthscale_num_dims = 1 if ard_num_dims is None else ard_num_dims
             self.register_parameter(
-                name="log_lengthscale",
-                parameter=torch.nn.Parameter(torch.zeros(batch_size, 1, lengthscale_num_dims)),
-                prior=log_lengthscale_prior,
+                name="log_lengthscale", parameter=torch.nn.Parameter(torch.zeros(batch_size, 1, lengthscale_num_dims))
             )
+            if lengthscale_prior is not None:
+                self.register_prior("lengthscale_prior", lengthscale_prior, lambda: self.lengthscale)
 
     @property
     def has_lengthscale(self):
@@ -119,7 +121,7 @@ class Kernel(Module):
     @property
     def lengthscale(self):
         if self.has_lengthscale:
-            return self.param_transform(self.log_lengthscale).clamp(self.eps, 1e5)
+            return self._param_transform(self.log_lengthscale).clamp(self.eps, 1e5)
         else:
             return None
 
diff --git a/gpytorch/kernels/linear_kernel.py b/gpytorch/kernels/linear_kernel.py
index 44f6e99d..02ecbe8f 100644
--- a/gpytorch/kernels/linear_kernel.py
+++ b/gpytorch/kernels/linear_kernel.py
@@ -49,10 +49,12 @@ class LinearKernel(Kernel):
 
     def __init__(self, num_dimensions, variance_prior=None, offset_prior=None, active_dims=None):
         super(LinearKernel, self).__init__(active_dims=active_dims)
-        self.register_parameter(name="variance", parameter=torch.nn.Parameter(torch.zeros(1)), prior=variance_prior)
-        self.register_parameter(
-            name="offset", parameter=torch.nn.Parameter(torch.zeros(1, 1, num_dimensions)), prior=offset_prior
-        )
+        self.register_parameter(name="variance", parameter=torch.nn.Parameter(torch.zeros(1)))
+        self.register_parameter(name="offset", parameter=torch.nn.Parameter(torch.zeros(1, 1, num_dimensions)))
+        if variance_prior is not None:
+            self.register_prior("variance_prior", variance_prior, "variance")
+        if offset_prior is not None:
+            self.register_prior("offset_prior", offset_prior, "offset")
 
     def forward(self, x1, x2, batch_dims=None, **params):
         x1_ = x1 - self.offset
diff --git a/gpytorch/kernels/matern_kernel.py b/gpytorch/kernels/matern_kernel.py
index 689c605c..729ba155 100644
--- a/gpytorch/kernels/matern_kernel.py
+++ b/gpytorch/kernels/matern_kernel.py
@@ -6,6 +6,7 @@ from __future__ import unicode_literals
 import math
 import torch
 from .kernel import Kernel
+from ..utils.deprecation import _deprecate_kwarg
 
 
 class MaternKernel(Kernel):
@@ -85,10 +86,12 @@ class MaternKernel(Kernel):
         ard_num_dims=None,
         batch_size=1,
         active_dims=None,
-        log_lengthscale_prior=None,
+        lengthscale_prior=None,
         eps=1e-6,
         param_transform=torch.exp,
+        **kwargs
     ):
+        _deprecate_kwarg(kwargs, "log_lengthscale_prior", "lengthscale_prior", lengthscale_prior)
         if nu not in {0.5, 1.5, 2.5}:
             raise RuntimeError("nu expected to be 0.5, 1.5, or 2.5")
         super(MaternKernel, self).__init__(
@@ -96,7 +99,7 @@ class MaternKernel(Kernel):
             ard_num_dims=ard_num_dims,
             batch_size=batch_size,
             active_dims=active_dims,
-            log_lengthscale_prior=log_lengthscale_prior,
+            lengthscale_prior=lengthscale_prior,
             param_transform=param_transform,
             eps=eps,
         )
diff --git a/gpytorch/kernels/multitask_kernel.py b/gpytorch/kernels/multitask_kernel.py
index d8615d3f..2d875969 100644
--- a/gpytorch/kernels/multitask_kernel.py
+++ b/gpytorch/kernels/multitask_kernel.py
@@ -46,18 +46,13 @@ class MultitaskKernel(Kernel):
     def forward(self, x1, x2, diag=False, batch_dims=None, **params):
         if batch_dims == (0, 2):
             raise RuntimeError("AdditiveGridInterpolationKernel does not accept the batch_dims argument.")
-
         covar_i = self.task_covar_module.covar_matrix
         covar_i = covar_i.repeat(x1.size(0), 1, 1)
         covar_x = self.data_covar_module.forward(x1, x2, **params)
         if not isinstance(covar_x, LazyTensor):
             covar_x = NonLazyTensor(covar_x)
         res = KroneckerProductLazyTensor(covar_i, covar_x)
-
-        if diag:
-            return res.diag()
-        else:
-            return res
+        return res.diag() if diag else res
 
     def size(self, x1, x2):
         """
diff --git a/gpytorch/kernels/periodic_kernel.py b/gpytorch/kernels/periodic_kernel.py
index 2cb30a9e..08cc7bfa 100644
--- a/gpytorch/kernels/periodic_kernel.py
+++ b/gpytorch/kernels/periodic_kernel.py
@@ -6,6 +6,7 @@ from __future__ import unicode_literals
 import math
 import torch
 from .kernel import Kernel
+from ..utils.deprecation import _deprecate_kwarg
 
 
 class PeriodicKernel(Kernel):
@@ -42,7 +43,7 @@ class PeriodicKernel(Kernel):
             Set this if you want to
             compute the covariance of only a few input dimensions. The ints
             corresponds to the indices of the dimensions. Default: `None`.
-        :attr:`log_period_length_prior` (Prior, optional):
+        :attr:`period_length_prior` (Prior, optional):
             Set this if you want
             to apply a prior to the period length parameter.  Default: `None`
         :attr:`log_lengthscale_prior` (Prior, optional):
@@ -76,27 +77,30 @@ class PeriodicKernel(Kernel):
         active_dims=None,
         batch_size=1,
         eps=1e-6,
-        log_lengthscale_prior=None,
-        log_period_length_prior=None,
+        lengthscale_prior=None,
+        period_length_prior=None,
         param_transform=torch.exp,
+        **kwargs
     ):
+        lengthscale_prior = _deprecate_kwarg(kwargs, "log_lengthscale_prior", "lengthscale_prior", lengthscale_prior)
+        period_length_prior = _deprecate_kwarg(
+            kwargs, "log_period_length_prior", "period_length_prior", period_length_prior
+        )
         super(PeriodicKernel, self).__init__(
             has_lengthscale=True,
             active_dims=active_dims,
             batch_size=batch_size,
-            log_lengthscale_prior=log_lengthscale_prior,
+            lengthscale_prior=lengthscale_prior,
             param_transform=param_transform,
             eps=eps,
         )
-        self.register_parameter(
-            name="log_period_length",
-            parameter=torch.nn.Parameter(torch.zeros(batch_size, 1, 1)),
-            prior=log_period_length_prior,
-        )
+        self.register_parameter(name="log_period_length", parameter=torch.nn.Parameter(torch.zeros(batch_size, 1, 1)))
+        if period_length_prior is not None:
+            self.register_prior("period_length_prior", period_length_prior, lambda: self.period_length)
 
     @property
     def period_length(self):
-        return self.param_transform(self.log_period_length).clamp(self.eps, 1e5)
+        return self._param_transform(self.log_period_length).clamp(self.eps, 1e5)
 
     def forward(self, x1, x2, **params):
         x1_ = x1.div(self.period_length)
diff --git a/gpytorch/kernels/rbf_kernel.py b/gpytorch/kernels/rbf_kernel.py
index f936d2d1..6b41362c 100644
--- a/gpytorch/kernels/rbf_kernel.py
+++ b/gpytorch/kernels/rbf_kernel.py
@@ -5,6 +5,7 @@ from __future__ import unicode_literals
 
 from .kernel import Kernel
 import torch
+from ..utils.deprecation import _deprecate_kwarg
 
 
 class RBFKernel(Kernel):
@@ -38,7 +39,7 @@ class RBFKernel(Kernel):
             Set this if you want to
             compute the covariance of only a few input dimensions. The ints
             corresponds to the indices of the dimensions. Default: `None`.
-        :attr:`log_lengthscale_prior` (Prior, optional):
+        :attr:`lengthscale_prior` (Prior, optional):
             Set this if you want
             to apply a prior to the lengthscale parameter.  Default: `None`
         :attr:`eps` (float):
@@ -71,18 +72,20 @@ class RBFKernel(Kernel):
     def __init__(
         self,
         ard_num_dims=None,
-        log_lengthscale_prior=None,
+        lengthscale_prior=None,
         eps=1e-6,
         active_dims=None,
         batch_size=1,
         param_transform=torch.exp,
+        **kwargs
     ):
+        _deprecate_kwarg(kwargs, "log_lengthscale_prior", "lengthscale_prior", lengthscale_prior)
         super(RBFKernel, self).__init__(
             has_lengthscale=True,
             ard_num_dims=ard_num_dims,
             batch_size=batch_size,
             active_dims=active_dims,
-            log_lengthscale_prior=log_lengthscale_prior,
+            lengthscale_prior=lengthscale_prior,
             param_transform=param_transform,
             eps=eps,
         )
diff --git a/gpytorch/kernels/scale_kernel.py b/gpytorch/kernels/scale_kernel.py
index d5e7caf4..15311918 100644
--- a/gpytorch/kernels/scale_kernel.py
+++ b/gpytorch/kernels/scale_kernel.py
@@ -5,6 +5,7 @@ from __future__ import unicode_literals
 
 import torch
 from .kernel import Kernel
+from ..utils.deprecation import _deprecate_kwarg
 
 
 class ScaleKernel(Kernel):
@@ -25,13 +26,12 @@ class ScaleKernel(Kernel):
 
     .. note::
         The :attr:`outputscale` parameter is parameterized on a log scale to constrain it to be positive.
-        You can set a prior on this parameter using the :attr:`log_outputscale_prior` argument, but
-        be sure that the prior has :attr:`log_transform=True` set.
+        You can set a prior on this parameter using the :attr:`outputscale_prior` argument.
 
     Args:
         - :attr:`batch_size` (int, optional): Set this if you want a separate outputscale for each
             batch of input data. It should be `b` if :attr:`x1` is a `b x n x d` tensor. Default: `1`
-        - :attr:`log_outputscale_prior` (Prior, optional): Set this if you want
+        - :attr:`outputscale_prior` (Prior, optional): Set this if you want
             to apply a prior to the outputscale parameter.  Default: `None`
         - :attr:`param_transform` (function, optional):
             Set this if you want to use something other than torch.exp to ensure positiveness of parameters.
@@ -48,16 +48,18 @@ class ScaleKernel(Kernel):
         >>> covar = scaled_covar_module(x)  # Output: LazyTensor of size (10 x 10)
     """
 
-    def __init__(self, base_kernel, batch_size=1, log_outputscale_prior=None, param_transform=torch.exp):
-        super(ScaleKernel, self).__init__(has_lengthscale=False, batch_size=batch_size, param_transform=param_transform)
+    def __init__(self, base_kernel, batch_size=1, outputscale_prior=None, param_transform=torch.exp, **kwargs):
+        outputscale_prior = _deprecate_kwarg(kwargs, "log_outputscale_prior", "outputscale_prior", outputscale_prior)
+        super(ScaleKernel, self).__init__(has_lengthscale=False, batch_size=batch_size)
         self.base_kernel = base_kernel
-        self.register_parameter(
-            name="log_outputscale", parameter=torch.nn.Parameter(torch.zeros(batch_size)), prior=log_outputscale_prior
-        )
+        self._param_transform = param_transform
+        self.register_parameter(name="log_outputscale", parameter=torch.nn.Parameter(torch.zeros(batch_size)))
+        if outputscale_prior is not None:
+            self.register_prior("outputscale_prior", outputscale_prior, lambda: self.outputscale)
 
     @property
     def outputscale(self):
-        return self.param_transform(self.log_outputscale)
+        return self._param_transform(self.log_outputscale)
 
     def forward(self, x1, x2, batch_dims=None, **params):
         outputscales = self.outputscale
diff --git a/gpytorch/kernels/spectral_mixture_kernel.py b/gpytorch/kernels/spectral_mixture_kernel.py
index 4a9915a9..e9db032f 100644
--- a/gpytorch/kernels/spectral_mixture_kernel.py
+++ b/gpytorch/kernels/spectral_mixture_kernel.py
@@ -7,6 +7,7 @@ import logging
 import math
 import torch
 from .kernel import Kernel
+from ..utils.deprecation import _deprecate_kwarg
 
 logger = logging.getLogger()
 
@@ -76,22 +77,29 @@ class SpectralMixtureKernel(Kernel):
         batch_size=1,
         active_dims=None,
         eps=1e-6,
-        log_mixture_scales_prior=None,
-        log_mixture_means_prior=None,
-        log_mixture_weights_prior=None,
+        mixture_scales_prior=None,
+        mixture_means_prior=None,
+        mixture_weights_prior=None,
         param_transform=torch.exp,
+        **kwargs
     ):
+        mixture_scales_prior = _deprecate_kwarg(
+            kwargs, "log_mixture_scales_prior", "mixture_scales_prior", mixture_scales_prior
+        )
+        mixture_means_prior = _deprecate_kwarg(
+            kwargs, "log_mixture_means_prior", "mixture_means_prior", mixture_means_prior
+        )
+        mixture_weights_prior = _deprecate_kwarg(
+            kwargs, "log_mixture_weights_prior", "mixture_weights_prior", mixture_weights_prior
+        )
+
         if num_mixtures is None:
             raise RuntimeError("num_mixtures is a required argument")
-        if (
-            log_mixture_means_prior is not None
-            or log_mixture_scales_prior is not None
-            or log_mixture_weights_prior is not None
-        ):
+        if mixture_means_prior is not None or mixture_scales_prior is not None or mixture_weights_prior is not None:
             logger.warning("Priors not implemented for SpectralMixtureKernel")
 
         # This kernel does not use the default lengthscale
-        super(SpectralMixtureKernel, self).__init__(active_dims=active_dims, param_transform=param_transform)
+        super(SpectralMixtureKernel, self).__init__(active_dims=active_dims)
         self.num_mixtures = num_mixtures
         self.batch_size = batch_size
         self.ard_num_dims = ard_num_dims
@@ -100,26 +108,21 @@ class SpectralMixtureKernel(Kernel):
         self.register_parameter(
             name="log_mixture_weights", parameter=torch.nn.Parameter(torch.zeros(self.batch_size, self.num_mixtures))
         )
-        self.register_parameter(
-            name="log_mixture_means",
-            parameter=torch.nn.Parameter(torch.zeros(self.batch_size, self.num_mixtures, 1, self.ard_num_dims)),
-        )
-        self.register_parameter(
-            name="log_mixture_scales",
-            parameter=torch.nn.Parameter(torch.zeros(self.batch_size, self.num_mixtures, 1, self.ard_num_dims)),
-        )
+        ms_shape = torch.Size([self.batch_size, self.num_mixtures, 1, self.ard_num_dims])
+        self.register_parameter(name="log_mixture_means", parameter=torch.nn.Parameter(torch.zeros(ms_shape)))
+        self.register_parameter(name="log_mixture_scales", parameter=torch.nn.Parameter(torch.zeros(ms_shape)))
 
     @property
     def mixture_scales(self):
-        return self.param_transform(self.log_mixture_scales).clamp(self.eps, 1e5)
+        return self._param_transform(self.log_mixture_scales).clamp(self.eps, 1e5)
 
     @property
     def mixture_means(self):
-        return self.param_transform(self.log_mixture_means).clamp(self.eps, 1e5)
+        return self._param_transform(self.log_mixture_means).clamp(self.eps, 1e5)
 
     @property
     def mixture_weights(self):
-        return self.param_transform(self.log_mixture_weights).clamp(self.eps, 1e5)
+        return self._param_transform(self.log_mixture_weights).clamp(self.eps, 1e5)
 
     def initialize_from_data(self, train_x, train_y, **kwargs):
         if not torch.is_tensor(train_x) or not torch.is_tensor(train_y):
diff --git a/gpytorch/lazy/batch_repeat_lazy_tensor.py b/gpytorch/lazy/batch_repeat_lazy_tensor.py
index 6d391925..3b2b82ab 100644
--- a/gpytorch/lazy/batch_repeat_lazy_tensor.py
+++ b/gpytorch/lazy/batch_repeat_lazy_tensor.py
@@ -171,10 +171,11 @@ class BatchRepeatLazyTensor(LazyTensor):
         return RootLazyTensor(self.base_lazy_tensor.root_decomposition().root.repeat(*self.batch_repeat, 1, 1))
 
     def root_inv_decomposition(self, initial_vectors=None, test_vectors=None):
-        return RootLazyTensor(self.base_lazy_tensor.root_inv_decomposition(
-            initial_vectors=initial_vectors,
-            test_vectors=test_vectors,
-        ).root.repeat(*self.batch_repeat, 1, 1))
+        return RootLazyTensor(
+            self.base_lazy_tensor.root_inv_decomposition(
+                initial_vectors=initial_vectors, test_vectors=test_vectors
+            ).root.repeat(*self.batch_repeat, 1, 1)
+        )
 
     def __getitem__(self, index):
         """
diff --git a/gpytorch/likelihoods/gaussian_likelihood.py b/gpytorch/likelihoods/gaussian_likelihood.py
index 32979e07..9b8910d9 100644
--- a/gpytorch/likelihoods/gaussian_likelihood.py
+++ b/gpytorch/likelihoods/gaussian_likelihood.py
@@ -7,22 +7,24 @@ from ..functions import add_diag
 from ..likelihoods import Likelihood
 from ..lazy import DiagLazyTensor
 from .. import settings
+from ..utils.deprecation import _deprecate_kwarg
 
 
 class GaussianLikelihood(Likelihood):
     r"""
     """
 
-    def __init__(self, log_noise_prior=None, batch_size=1, param_transform=torch.exp):
+    def __init__(self, noise_prior=None, batch_size=1, param_transform=torch.exp, **kwargs):
+        noise_prior = _deprecate_kwarg(kwargs, "log_noise_prior", "noise_prior", noise_prior)
         super(GaussianLikelihood, self).__init__()
-        self.param_transform = param_transform
-        self.register_parameter(
-            name="log_noise", parameter=torch.nn.Parameter(torch.zeros(batch_size, 1)), prior=log_noise_prior
-        )
+        self._param_transform = param_transform
+        self.register_parameter(name="log_noise", parameter=torch.nn.Parameter(torch.zeros(batch_size, 1)))
+        if noise_prior is not None:
+            self.register_prior("noise_prior", noise_prior, lambda: self.noise)
 
     @property
     def noise(self):
-        return self.param_transform(self.log_noise)
+        return self._param_transform(self.log_noise)
 
     def forward(self, input):
         if not isinstance(input, MultivariateNormal):
@@ -45,7 +47,7 @@ class GaussianLikelihood(Likelihood):
                 raise RuntimeError("With batch_size > 1, expected a batched MultivariateNormal distribution.")
             log_noise = log_noise.squeeze(0)
 
-        res = -0.5 * ((target - mean) ** 2 + variance) / self.param_transform(log_noise)
+        res = -0.5 * ((target - mean) ** 2 + variance) / self.noise
         res += -0.5 * log_noise - 0.5 * math.log(2 * math.pi)
         return res.sum(-1)
 
diff --git a/gpytorch/likelihoods/multitask_gaussian_likelihood.py b/gpytorch/likelihoods/multitask_gaussian_likelihood.py
index e5ac3fab..0c2997dd 100644
--- a/gpytorch/likelihoods/multitask_gaussian_likelihood.py
+++ b/gpytorch/likelihoods/multitask_gaussian_likelihood.py
@@ -5,6 +5,7 @@ from ..functions import add_diag
 from ..lazy import DiagLazyTensor, KroneckerProductLazyTensor, RootLazyTensor
 from ..likelihoods import GaussianLikelihood
 from .. import settings
+from ..utils.deprecation import _deprecate_kwarg
 
 
 class MultitaskGaussianLikelihood(GaussianLikelihood):
@@ -18,7 +19,7 @@ class MultitaskGaussianLikelihood(GaussianLikelihood):
     """
 
     def __init__(
-        self, num_tasks, rank=0, task_prior=None, batch_size=1, log_noise_prior=None, param_transform=torch.exp
+        self, num_tasks, rank=0, task_prior=None, batch_size=1, noise_prior=None, param_transform=torch.exp, **kwargs
     ):
         """
         Args:
@@ -31,32 +32,31 @@ class MultitaskGaussianLikelihood(GaussianLikelihood):
             `rank` > 0, or a prior over the log of just the diagonal elements, if `rank` == 0.
 
         """
-        super(MultitaskGaussianLikelihood, self).__init__(batch_size=batch_size, log_noise_prior=log_noise_prior)
+        noise_prior = _deprecate_kwarg(kwargs, "log_noise_prior", "noise_prior", noise_prior)
+        super(MultitaskGaussianLikelihood, self).__init__(
+            batch_size=batch_size, noise_prior=noise_prior, param_transform=param_transform
+        )
 
         if rank == 0:
             self.register_parameter(
                 name="log_task_noises",
                 parameter=torch.nn.Parameter(torch.zeros(batch_size, num_tasks)),
-                prior=task_prior,
             )
+            if task_prior is not None:
+                raise RuntimeError("Cannot set a `task_prior` if rank=0")
         else:
             self.register_parameter(
                 name="task_noise_covar_factor", parameter=torch.nn.Parameter(torch.randn(batch_size, num_tasks, rank))
             )
             if task_prior is not None:
-                self.register_derived_prior(
-                    name="MultitaskErrorCovariancePrior",
-                    prior=task_prior,
-                    parameter_names=("task_noise_covar_factor", "log_noise"),
-                    transform=self._eval_covar_matrix,
-                )
+                self.register_prior("MultitaskErrorCovariancePrior", task_prior, self._eval_covar_matrix)
         self.num_tasks = num_tasks
-        self.param_transform = param_transform
 
     def _eval_covar_matrix(self, task_noise_covar_factor, log_noise):
         num_tasks = task_noise_covar_factor.size(0)
-        noise = self.param_transform(log_noise)
-        return task_noise_covar_factor.matmul(task_noise_covar_factor.transpose(-1, -2)) + noise * torch.eye(num_tasks)
+        noise = self._param_transform(log_noise)
+        D = noise * torch.eye(num_tasks, dtype=noise.dtype, device=noise.device)
+        return task_noise_covar_factor.matmul(task_noise_covar_factor.transpose(-1, -2)) + D
 
     def forward(self, input):
         """
@@ -85,7 +85,7 @@ class MultitaskGaussianLikelihood(GaussianLikelihood):
         mean, covar = input.mean, input.lazy_covariance_matrix
 
         if hasattr(self, "log_task_noises"):
-            noises = self.param_transform(self.log_task_noises)
+            noises = self._param_transform(self.log_task_noises)
             if covar.ndimension() == 2:
                 if settings.debug.on() and noises.size(0) > 1:
                     raise RuntimeError(
diff --git a/gpytorch/likelihoods/softmax_likelihood.py b/gpytorch/likelihoods/softmax_likelihood.py
index 8c665136..407de04d 100755
--- a/gpytorch/likelihoods/softmax_likelihood.py
+++ b/gpytorch/likelihoods/softmax_likelihood.py
@@ -20,8 +20,9 @@ class SoftmaxLikelihood(Likelihood):
         self.register_parameter(
             name="mixing_weights",
             parameter=torch.nn.Parameter(torch.ones(n_classes, num_features).fill_(1.0 / num_features)),
-            prior=mixing_weights_prior,
         )
+        if mixing_weights_prior is not None:
+            self.register_prior("mixing_weights_prior", mixing_weights_prior, "mixing_weights")
 
     def forward(self, latent_func):
         if not isinstance(latent_func, MultivariateNormal):
diff --git a/gpytorch/means/constant_mean.py b/gpytorch/means/constant_mean.py
index 684846db..07d0eefe 100644
--- a/gpytorch/means/constant_mean.py
+++ b/gpytorch/means/constant_mean.py
@@ -11,9 +11,9 @@ class ConstantMean(Mean):
     def __init__(self, prior=None, batch_size=None):
         super(ConstantMean, self).__init__()
         self.batch_size = batch_size
-        self.register_parameter(
-            name="constant", parameter=torch.nn.Parameter(torch.zeros(batch_size or 1, 1)), prior=prior
-        )
+        self.register_parameter(name="constant", parameter=torch.nn.Parameter(torch.zeros(batch_size or 1, 1)))
+        if prior is not None:
+            self.register_prior("mean_prior", prior, "constant")
 
     def forward(self, input):
         return self.constant.expand(input.size(0), input.size(1))
diff --git a/gpytorch/mlls/exact_marginal_log_likelihood.py b/gpytorch/mlls/exact_marginal_log_likelihood.py
index da77c18d..162c0611 100644
--- a/gpytorch/mlls/exact_marginal_log_likelihood.py
+++ b/gpytorch/mlls/exact_marginal_log_likelihood.py
@@ -37,11 +37,9 @@ class ExactMarginalLogLikelihood(MarginalLogLikelihood):
 
         res = res.add(0.5, added_loss)
 
-        # Add log probs of priors on the parameters
-        for _, param, prior in self.named_parameter_priors():
-            res.add_(prior.log_prob(param).sum())
-        for _, prior, params, transform in self.named_derived_priors():
-            res.add_(prior.log_prob(transform(*params)).sum())
+        # Add log probs of priors on the (functions of) parameters
+        for _, prior, closure in self.named_priors():
+            res.add_(prior.log_prob(closure()).sum())
 
         # Scale by the amount of data we have
         num_data = target.size(-1)
diff --git a/gpytorch/mlls/variational_elbo.py b/gpytorch/mlls/variational_elbo.py
index 0746937b..cf0a10b4 100644
--- a/gpytorch/mlls/variational_elbo.py
+++ b/gpytorch/mlls/variational_elbo.py
@@ -1,94 +1,88 @@
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
-
-import torch
-from .marginal_log_likelihood import MarginalLogLikelihood
-from .. import settings
-
-
-class VariationalELBO(MarginalLogLikelihood):
-    def __init__(self, likelihood, model, num_data, combine_terms=True):
-        """
-        A special MLL designed for variational inference
-
-        Args:
-        - likelihood: (Likelihood) - the likelihood for the model
-        - model: (Module) - the variational GP model
-        - num_data: (int) - the total number of training data points (necessary for SGD)
-        - combine_terms: (bool) - whether or not to sum the expected NLL with the KL terms (default True)
-        """
-        super(VariationalELBO, self).__init__(likelihood, model)
-        self.combine_terms = combine_terms
-        self.num_data = num_data
-
-    def forward(self, variational_dist_f, target, **kwargs):
-        num_batch = variational_dist_f.event_shape.numel()
-        variational_dist_u = self.model.variational_strategy.variational_distribution.variational_distribution
-        prior_dist = self.model.variational_strategy.prior_distribution
-
-        log_likelihood = self.likelihood.variational_log_probability(variational_dist_f, target, **kwargs).div(
-            num_batch
-        )
-
-        kl_divergence = torch.distributions.kl.kl_divergence(variational_dist_u, prior_dist)
-
-        if kl_divergence.dim() > log_likelihood.dim():
-            kl_divergence = kl_divergence.sum(-1)
-
-        if log_likelihood.numel() == 1:
-            kl_divergence = kl_divergence.sum()
-
-        kl_divergence = kl_divergence.div(self.num_data)
-
-        if self.combine_terms:
-            res = log_likelihood - kl_divergence
-            for _, param, prior in self.named_parameter_priors():
-                res.add_(prior.log_prob(param).sum().div(self.num_data))
-            for _, params, transform, prior in self.named_derived_priors():
-                res.add_(prior.log_prob(transform(*params)).sum().div(self.num_data))
-            return res
-        else:
-            log_prior = torch.zeros_like(log_likelihood)
-            for _, param, prior in self.named_parameter_priors():
-                log_prior.add_(prior.log_prob(param).sum())
-            for _, params, transform, prior in self.named_derived_priors():
-                log_prior.add_(prior.log_prob(transform(*params)).sum())
-            return log_likelihood, kl_divergence, log_prior.div(self.num_data)
-
-
-class VariationalELBOEmpirical(VariationalELBO):
-    def __init__(self, likelihood, model, num_data):
-        """
-        A special MLL designed for variational inference.
-        This computes an empirical (rather than exact) estimate of the KL divergence
-
-        Args:
-        - likelihood: (Likelihood) - the likelihood for the model
-        - model: (Module) - the variational GP model
-        - num_data: (int) - the total number of training data points (necessary for SGD)
-        """
-        super(VariationalELBOEmpirical, self).__init__(likelihood, model, num_data, combine_terms=True)
-
-    def forward(self, variational_dist_f, target, **kwargs):
-        num_batch = variational_dist_f.event_shape[0]
-        variational_dist_u = self.model.variational_strategy.variational_distribution.variational_distribution
-        prior_dist = self.model.variational_strategy.prior_distribution
-
-        log_likelihood = self.likelihood.variational_log_probability(variational_dist_f, target, **kwargs)
-        log_likelihood = log_likelihood.div(num_batch)
-
-        num_samples = settings.num_likelihood_samples.value()
-        variational_samples = variational_dist_u.rsample(torch.Size([num_samples]))
-        kl_divergence = (
-            variational_dist_u.log_prob(variational_samples) - prior_dist.log_prob(variational_samples)
-        ).mean(0)
-        kl_divergence = kl_divergence.div(self.num_data)
-
-        res = log_likelihood - kl_divergence
-        for _, param, prior in self.named_parameter_priors():
-            res.add_(prior.log_prob(param).sum().div(self.num_data))
-        for _, params, transform, prior in self.named_derived_priors():
-            res.add_(prior.log_prob(transform(*params)).sum().div(self.num_data))
-        return res
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import torch
+from .marginal_log_likelihood import MarginalLogLikelihood
+from .. import settings
+
+
+class VariationalELBO(MarginalLogLikelihood):
+    def __init__(self, likelihood, model, num_data, combine_terms=True):
+        """
+        A special MLL designed for variational inference
+
+        Args:
+        - likelihood: (Likelihood) - the likelihood for the model
+        - model: (Module) - the variational GP model
+        - num_data: (int) - the total number of training data points (necessary for SGD)
+        - combine_terms: (bool) - whether or not to sum the expected NLL with the KL terms (default True)
+        """
+        super(VariationalELBO, self).__init__(likelihood, model)
+        self.combine_terms = combine_terms
+        self.num_data = num_data
+
+    def forward(self, variational_dist_f, target, **kwargs):
+        num_batch = variational_dist_f.event_shape.numel()
+        variational_dist_u = self.model.variational_strategy.variational_distribution.variational_distribution
+        prior_dist = self.model.variational_strategy.prior_distribution
+
+        log_likelihood = self.likelihood.variational_log_probability(variational_dist_f, target, **kwargs).div(
+            num_batch
+        )
+
+        kl_divergence = torch.distributions.kl.kl_divergence(variational_dist_u, prior_dist)
+
+        if kl_divergence.dim() > log_likelihood.dim():
+            kl_divergence = kl_divergence.sum(-1)
+
+        if log_likelihood.numel() == 1:
+            kl_divergence = kl_divergence.sum()
+
+        kl_divergence = kl_divergence.div(self.num_data)
+
+        if self.combine_terms:
+            res = log_likelihood - kl_divergence
+            for _, prior, closure in self.named_priors():
+                res.add_(prior.log_prob(closure()).sum().div(self.num_data))
+            return res
+        else:
+            log_prior = torch.zeros_like(log_likelihood)
+            for _, prior, closure in self.named_priors():
+                log_prior.add_(prior.log_prob(closure()).sum())
+            return log_likelihood, kl_divergence, log_prior.div(self.num_data)
+
+
+class VariationalELBOEmpirical(VariationalELBO):
+    def __init__(self, likelihood, model, num_data):
+        """
+        A special MLL designed for variational inference.
+        This computes an empirical (rather than exact) estimate of the KL divergence
+
+        Args:
+        - likelihood: (Likelihood) - the likelihood for the model
+        - model: (Module) - the variational GP model
+        - num_data: (int) - the total number of training data points (necessary for SGD)
+        """
+        super(VariationalELBOEmpirical, self).__init__(likelihood, model, num_data, combine_terms=True)
+
+    def forward(self, variational_dist_f, target, **kwargs):
+        num_batch = variational_dist_f.event_shape[0]
+        variational_dist_u = self.model.variational_strategy.variational_distribution.variational_distribution
+        prior_dist = self.model.variational_strategy.prior_distribution
+
+        log_likelihood = self.likelihood.variational_log_probability(variational_dist_f, target, **kwargs)
+        log_likelihood = log_likelihood.div(num_batch)
+
+        num_samples = settings.num_likelihood_samples.value()
+        variational_samples = variational_dist_u.rsample(torch.Size([num_samples]))
+        kl_divergence = (
+            variational_dist_u.log_prob(variational_samples) - prior_dist.log_prob(variational_samples)
+        ).mean(0)
+        kl_divergence = kl_divergence.div(self.num_data)
+
+        res = log_likelihood - kl_divergence
+        for _, prior, closure in self.named_priors():
+            res.add_(prior.log_prob(closure()).sum().div(self.num_data))
+        return res
diff --git a/gpytorch/models/abstract_variational_gp.py b/gpytorch/models/abstract_variational_gp.py
index c8b35cc5..da147629 100644
--- a/gpytorch/models/abstract_variational_gp.py
+++ b/gpytorch/models/abstract_variational_gp.py
@@ -1,22 +1,22 @@
-from __future__ import absolute_import, division, print_function, unicode_literals
-
-from .gp import GP
-
-
-class AbstractVariationalGP(GP):
-    def __init__(self, variational_strategy):
-        super(AbstractVariationalGP, self).__init__()
-        self.variational_strategy = variational_strategy
-
-    def forward(self, x):
-        """
-        As in the exact GP setting, the user-defined forward method should return the GP prior mean and covariance
-        evaluated at input locations x.
-        """
-        raise NotImplementedError
-
-    def __call__(self, inputs, **kwargs):
-        if inputs.dim() == 1:
-            inputs = inputs.unsqueeze(-1)
-
-        return self.variational_strategy(inputs)
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from .gp import GP
+
+
+class AbstractVariationalGP(GP):
+    def __init__(self, variational_strategy):
+        super(AbstractVariationalGP, self).__init__()
+        self.variational_strategy = variational_strategy
+
+    def forward(self, x):
+        """
+        As in the exact GP setting, the user-defined forward method should return the GP prior mean and covariance
+        evaluated at input locations x.
+        """
+        raise NotImplementedError
+
+    def __call__(self, inputs, **kwargs):
+        if inputs.dim() == 1:
+            inputs = inputs.unsqueeze(-1)
+
+        return self.variational_strategy(inputs)
diff --git a/gpytorch/module.py b/gpytorch/module.py
index 87631b87..dd660c93 100644
--- a/gpytorch/module.py
+++ b/gpytorch/module.py
@@ -7,14 +7,37 @@ from torch import nn
 from torch.distributions import Distribution
 
 from .lazy import LazyTensor
+from .utils.deprecation import DeprecationError
 
 
 class Module(nn.Module):
     def __init__(self):
         super(Module, self).__init__()
-        self._priors = OrderedDict()
-        self._derived_priors = OrderedDict()
         self._added_loss_terms = OrderedDict()
+        self._priors = OrderedDict()
+
+    def __call__(self, *inputs, **kwargs):
+        outputs = self.forward(*inputs, **kwargs)
+
+        if isinstance(outputs, tuple):
+            if not all(
+                torch.is_tensor(output) or isinstance(output, Distribution) or isinstance(output, LazyTensor)
+                for output in outputs
+            ):
+                raise RuntimeError(
+                    "All outputs must be a Distribution, torch.Tensor, or LazyTensor. "
+                    "Got {}".format([output.__class__.__name__ for output in outputs])
+                )
+            if len(outputs) == 1:
+                outputs = outputs[0]
+            return outputs
+
+        elif torch.is_tensor(outputs) or isinstance(outputs, Distribution) or isinstance(outputs, LazyTensor):
+            return outputs
+        else:
+            raise RuntimeError(
+                "Output must be a Distribution, torch.Tensor, or LazyTensor. Got {}".format(outputs.__class__.__name__)
+            )
 
     def _get_module_and_name(self, parameter_name):
         """Get module and name from full parameter name."""
@@ -26,23 +49,17 @@ class Module(nn.Module):
                 "Invalid parameter name {}. {} has no module {}".format(parameter_name, type(self).__name__, module)
             )
 
+    def added_loss_terms(self):
+        for _, strategy in self.named_added_loss_terms():
+            yield strategy
+
     def forward(self, *inputs, **kwargs):
         raise NotImplementedError
 
     def hyperparameters(self):
-        for name, param in self.named_hyperparameters():
+        for _, param in self.named_hyperparameters():
             yield param
 
-    def named_hyperparameters(self):
-        for name, param in self.named_parameters():
-            if "variational_" not in name:
-                yield name, param
-
-    def named_variational_parameters(self):
-        for name, param in self.named_parameters():
-            if "variational_" in name:
-                yield name, param
-
     def initialize(self, **kwargs):
         """
         Set a value for a parameter
@@ -61,39 +78,15 @@ class Module(nn.Module):
                 raise AttributeError("Type {t} not valid to initialize parameter {p}".format(t=type(val), p=name))
 
             # Ensure value is contained in support of prior (if present)
-            prior = self._priors.get(name)
-            if prior is not None:
-                param = self._parameters[name]
+            prior_name = "_".join([name, "prior"])
+            if prior_name in self._priors:
+                prior, closure = self._priors[prior_name]
                 try:
-                    prior._validate_sample(param)
+                    prior._validate_sample(closure())
                 except ValueError as e:
-                    raise ValueError(
-                        "Value of parameter {p} not valid for specified prior. Original exception:\n{e}".format(
-                            p=param, e=e
-                        )
-                    )
-        return self
-
-    def named_parameter_priors(self):
-        """
-        Returns an iterator over module parameter priors, yielding the name of
-        the parameter, the parameter itself, as well as the associated prior
-        (excludes parameters for which no prior has been registered)
-        """
-        return _extract_named_parameter_priors(module=self, memo=None, prefix="")
-
-    def named_derived_priors(self, memo=None, prefix=""):
-        """Returns an iterator over module derived priors, yielding both the
-        name of the prior as well as the prior, the associated parameters, and
-        the transformation callable.
-
-        Yields:
-            (string, Prior, tuple(string), callable): Tuple containing the name
-                of the prior, the prior itself, its parameters, and the transform
-                to be called on the parameters.
+                    raise ValueError("Invalid input value for prior {}. Error:\n{}".format(prior_name, e))
 
-        """
-        return _extract_named_derived_priors(module=self, memo=None, prefix="")
+        return self
 
     def named_added_loss_terms(self):
         """Returns an iterator over module variational strategies, yielding both
@@ -106,68 +99,83 @@ class Module(nn.Module):
         """
         return _extract_named_added_loss_terms(module=self, memo=None, prefix="")
 
-    def register_parameter(self, name, parameter, prior=None):
-        """
-        Adds a parameter to the module.
-        The parameter can be accessed as an attribute using given name.
+    def named_hyperparameters(self):
+        for name, param in self.named_parameters():
+            if "variational_" not in name:
+                yield name, param
 
-        name (str): name of parameter
-        param (torch.nn.Parameter): parameter
-        prior (Prior): prior for parameter (default: None)
-        """
-        if "_parameters" not in self.__dict__:
-            raise AttributeError("Cannot assign parameter before Module.__init__() call")
-        super(Module, self).register_parameter(name, parameter)
-        if prior is not None:
-            self.set_parameter_priors(**{name: prior})
+    def named_priors(self, memo=None, prefix=""):
+        """Returns an iterator over the module's priors, yielding the name of the prior,
+        the prior, the associated parameter names, and the transformation callable.
 
-    def register_derived_prior(self, name, prior, parameter_names, transform):
+        Yields:
+            (string, Prior, tuple((Parameter, callable)), callable): Tuple containing:
+                - the name of the prior
+                - the prior
+                - a tuple of tuples (param, transform), one for each of the parameters associated with the prior
+                - the prior's transform to be called on the parameters
         """
-        Adds a derived prior to the module.
-        The prior can be accessed as an attribute using the given name.
+        return _extract_named_priors(module=self, memo=None, prefix="")
 
-        Args:
-            - name (str): name of the derived prior
-            - prior (Prior): the prior object
-            - parameter_names (tuple(str)): The parameters the transform operaters on,
-                in the same order as expected by the transform callable.
-            - transform (Callable): The function called on the specified parameters. The
-                log-pdf of the prior will be evaluating on the output of this transform.
-
-        A derived prior operates on a transform of one or multiple parameters.
-        This can be used, for instance, to put a prior over the ICM Kernel
-        covariance matrix generated from covar_factor and log_var parameters.
-        """
-        self.add_module(name, prior)
-        self._derived_priors[name] = (prior, tuple(parameter_names), transform)
+    def named_variational_parameters(self):
+        for name, param in self.named_parameters():
+            if "variational_" in name:
+                yield name, param
 
     def register_added_loss_term(self, name):
         self._added_loss_terms[name] = None
 
-    def set_parameter_priors(self, **kwargs):
+    def register_parameter(self, name, parameter, prior=None):
+        r"""
+        Adds a parameter to the module. The parameter can be accessed as an attribute using the given name.
+
+        Args:
+            :attr:`name` (str):
+                The name of the parameter
+            :attr:`parameter` (torch.nn.Parameter):
+                The parameter
+        """
+        if prior is not None:
+            raise DeprecationError(
+                "Setting a prior upon registering a parameter is deprecated. Please use "
+                ".register_prior('{name}_prior', prior, '{name}') instead.".format(name=name)
+            )
+        if "_parameters" not in self.__dict__:
+            raise AttributeError("Cannot assign parameter before Module.__init__() call")
+        super(Module, self).register_parameter(name, parameter)
+
+    def register_prior(self, name, prior, arg):
         """
-        Set prior for a parameter.
-        The prior can be accessed as an attribute using <PARAMETER_NAME>_prior.
+        Adds a prior to the module. The prior can be accessed as an attribute using the given name.
 
-        kwargs: (param_name, prior) - parameter to initialize
-        prior must be a gpytorch Prior
+        Args:
+            :attr:`name` (str):
+                The name of the prior
+            :attr:`prior` (Prior):
+                The prior to be registered`
+            :attr:`arg` (string or callable):
+                Either the name of the parameter, or a closure (which upon calling evalutes a function on
+                one or more parameters):
+                single parameter without a transform: `.register_prior("foo_prior", foo_prior, "foo_param")`
+                transform a single parameter (e.g. put a log-Normal prior on it):
+                `.register_prior("foo_prior", NormalPrior(0, 1), lambda: torch.log(self.foo_param))`
+                function of multiple parameters:
+                `.register_prior("foo2_prior", foo2_prior, lambda: f(self.param1, self.param2)))`
         """
-        for name, prior in kwargs.items():
-            if name not in self._parameters:
+        if isinstance(arg, str):
+            if arg not in self._parameters:
                 raise AttributeError(
-                    "Unknown parameter {name} for {module}".format(name=name, module=self.__class__.__name__)
+                    "Unknown parameter {name} for {module}".format(name=arg, module=self.__class__.__name__)
+                    + "Make sure the parameter is registered before registering a prior."
                 )
-            self.add_module("_".join([name, "prior"]), prior)
-            self._priors[name] = prior
-        return self
 
-    def variational_parameters(self):
-        for name, param in self.named_variational_parameters():
-            yield param
+            def closure():
+                return self._parameters[arg]
 
-    def added_loss_terms(self):
-        for _, strategy in self.named_added_loss_terms():
-            yield strategy
+        else:
+            closure = arg
+        self.add_module(name, prior)
+        self._priors[name] = (prior, closure)
 
     def update_added_loss_term(self, name, added_loss_term):
         from .mlls import AddedLossTerm
@@ -178,45 +186,9 @@ class Module(nn.Module):
             raise RuntimeError("added_loss_term {} not registered".format(name))
         self._added_loss_terms[name] = added_loss_term
 
-    def __call__(self, *inputs, **kwargs):
-        outputs = self.forward(*inputs, **kwargs)
-
-        if isinstance(outputs, tuple):
-            if not all(
-                torch.is_tensor(output) or isinstance(output, Distribution) or isinstance(output, LazyTensor)
-                for output in outputs
-            ):
-                raise RuntimeError(
-                    "All outputs must be a Distribution, torch.Tensor, or LazyTensor. "
-                    "Got {}".format([output.__class__.__name__ for output in outputs])
-                )
-            if len(outputs) == 1:
-                outputs = outputs[0]
-            return outputs
-
-        elif torch.is_tensor(outputs) or isinstance(outputs, Distribution) or isinstance(outputs, LazyTensor):
-            return outputs
-        else:
-            raise RuntimeError(
-                "Output must be a Distribution, torch.Tensor, or LazyTensor. Got {}".format(outputs.__class__.__name__)
-            )
-
-
-def _extract_named_parameter_priors(module, memo=None, prefix=""):
-    if memo is None:
-        memo = set()
-    if hasattr(module, "_priors"):
-        for name, parameter in module._parameters.items():
-            if name in module._priors and module._priors[name] not in memo:
-                prior = module._priors[name]
-                memo.add(prior)
-                yield prefix + ("." if prefix else "") + name, parameter, prior
-    for mname, module_ in module.named_children():
-        submodule_prefix = prefix + ("." if prefix else "") + mname
-        for name, parameter, prior in _extract_named_parameter_priors(
-            module=module_, memo=memo, prefix=submodule_prefix
-        ):
-            yield name, parameter, prior
+    def variational_parameters(self):
+        for _, param in self.named_variational_parameters():
+            yield param
 
 
 def _extract_named_added_loss_terms(module, memo=None, prefix=""):
@@ -233,16 +205,16 @@ def _extract_named_added_loss_terms(module, memo=None, prefix=""):
             yield name, strategy
 
 
-def _extract_named_derived_priors(module, memo=None, prefix=""):
+def _extract_named_priors(module, memo=None, prefix=""):
     if memo is None:
         memo = set()
-    if hasattr(module, "_derived_priors"):
-        for name, (prior, pnames, tf) in module._derived_priors.items():
+    if hasattr(module, "_priors"):
+        for name, (prior, closure) in module._priors.items():
             if prior is not None and prior not in memo:
                 memo.add(prior)
-                parameters = tuple(getattr(module, pname) for pname in pnames)
-                yield prefix + ("." if prefix else "") + name, prior, parameters, tf
+                full_name = ("." if prefix else "").join([prefix, name])
+                yield full_name, prior, closure
     for mname, module_ in module.named_children():
         submodule_prefix = prefix + ("." if prefix else "") + mname
-        for name, prior, parameters, tf in _extract_named_derived_priors(module_, memo=memo, prefix=submodule_prefix):
-            yield name, prior, parameters, tf
+        for name, prior, closure in _extract_named_priors(module_, memo=memo, prefix=submodule_prefix):
+            yield name, prior, closure
diff --git a/gpytorch/priors/lkj_prior.py b/gpytorch/priors/lkj_prior.py
index c8d805d2..0dc17a36 100644
--- a/gpytorch/priors/lkj_prior.py
+++ b/gpytorch/priors/lkj_prior.py
@@ -50,20 +50,15 @@ class LKJPrior(Prior):
         del self.eta, self.C
         self.register_buffer("eta", eta)
         self.register_buffer("C", C)
-        self._log_transform = False
 
-    def log_prob(self, parameter):
-        if self.log_transform:
-            raise RuntimeError("log-transform not supported for LKJPrior")
-        if any(s != self.n for s in parameter.shape[-2:]):
+    def log_prob(self, X):
+        if any(s != self.n for s in X.shape[-2:]):
             raise ValueError("Correlation matrix is not of size n={}".format(self.n.item()))
-        if not _is_valid_correlation_matrix(parameter):
+        if not _is_valid_correlation_matrix(X):
             raise ValueError("Input is not a valid correlation matrix")
         # TODO: Replace this loop with batch Cholesky decomposition when available
         # https://github.com/pytorch/pytorch/pull/11796
-        log_diag_sum = torch.stack(
-            [p.cholesky(upper=True).diag().log().sum() for p in parameter.view(-1, *parameter.shape[-2:])]
-        )
+        log_diag_sum = torch.stack([p.cholesky(upper=True).diag().log().sum() for p in X.view(-1, *X.shape[-2:])])
         return self.C + (self.eta - 1) * 2 * log_diag_sum
 
 
@@ -86,14 +81,12 @@ class LKJCholeskyFactorPrior(LKJPrior):
 
     support = constraints.lower_cholesky
 
-    def log_prob(self, parameter):
-        if self.log_transform:
-            raise RuntimeError("log-transform not supported for LKJCholeskyFactorPrior")
-        if any(s != self.n for s in parameter.shape[-2:]):
+    def log_prob(self, X):
+        if any(s != self.n for s in X.shape[-2:]):
             raise ValueError("Cholesky factor is not of size n={}".format(self.n.item()))
-        if not _is_valid_correlation_matrix_cholesky_factor(parameter):
+        if not _is_valid_correlation_matrix_cholesky_factor(X):
             raise ValueError("Input is not a Cholesky factor of a valid correlation matrix")
-        log_diag_sum = torch.diagonal(parameter, dim1=-2, dim2=-1).log().sum(-1)
+        log_diag_sum = torch.diagonal(X, dim1=-2, dim2=-1).log().sum(-1)
         return self.C + (self.eta - 1) * 2 * log_diag_sum
 
 
@@ -125,17 +118,14 @@ class LKJCovariancePrior(LKJPrior):
         )
         self.correlation_prior = correlation_prior
         self.sd_prior = sd_prior
-        self._log_transform = False
 
-    def log_prob(self, parameter):
-        if self.log_transform:
-            raise RuntimeError("log-transform not supported for LKJCovariancePrior")
-        marginal_var = torch.diagonal(parameter, dim1=-2, dim2=-1)
+    def log_prob(self, X):
+        marginal_var = torch.diagonal(X, dim1=-2, dim2=-1)
         if not torch.all(marginal_var >= 0):
             raise ValueError("Variance(s) cannot be negative")
         marginal_sd = marginal_var.sqrt()
         sd_diag_mat = _batch_form_diag(1 / marginal_sd)
-        correlations = torch.matmul(torch.matmul(sd_diag_mat, parameter), sd_diag_mat)
+        correlations = torch.matmul(torch.matmul(sd_diag_mat, X), sd_diag_mat)
         log_prob_corr = self.correlation_prior.log_prob(correlations)
         log_prob_sd = self.sd_prior.log_prob(marginal_sd)
         return log_prob_corr + log_prob_sd
diff --git a/gpytorch/priors/prior.py b/gpytorch/priors/prior.py
index 3bebd66d..aced82eb 100644
--- a/gpytorch/priors/prior.py
+++ b/gpytorch/priors/prior.py
@@ -15,10 +15,9 @@ class Prior(Distribution, Module, ABC):
     respective prior in computing the Marginal Log-Likelihood.
     """
 
-    @property
-    def log_transform(self):
-        return self._log_transform
+    def transform(self, x):
+        return self._transform(x) if self._transform is not None else x
 
-    def log_prob(self, parameter):
+    def log_prob(self, x):
         """Returns the log-probability of the parameter value under the prior."""
-        return super(Prior, self).log_prob(parameter.exp() if self.log_transform else parameter)
+        return super(Prior, self).log_prob(self.transform(x))
diff --git a/gpytorch/priors/smoothed_box_prior.py b/gpytorch/priors/smoothed_box_prior.py
index 88b665f5..e35281a3 100644
--- a/gpytorch/priors/smoothed_box_prior.py
+++ b/gpytorch/priors/smoothed_box_prior.py
@@ -31,7 +31,7 @@ class SmoothedBoxPrior(Prior):
     support = constraints.real
     _validate_args = True
 
-    def __init__(self, a, b, sigma=0.01, log_transform=False, validate_args=False):
+    def __init__(self, a, b, sigma=0.01, validate_args=False, transform=None):
         TModule.__init__(self)
         _a = torch.tensor(float(a)) if isinstance(a, Number) else a
         _a = _a.view(-1) if _a.dim() < 1 else _a
@@ -49,7 +49,7 @@ class SmoothedBoxPrior(Prior):
         self.register_buffer("b", _b)
         self.register_buffer("sigma", _sigma)
         self.tails = NormalPrior(torch.zeros_like(_a), _sigma, validate_args=validate_args)
-        self._log_transform = log_transform
+        self._transform = transform
 
     @property
     def _c(self):
@@ -64,10 +64,10 @@ class SmoothedBoxPrior(Prior):
         # normalization factor to make this a probability distribution
         return torch.log(1 + (self.b - self.a) / (math.sqrt(2 * math.pi) * self.sigma))
 
-    def log_prob(self, parameter):
-        return self._log_prob(parameter.exp() if self.log_transform else parameter)
+    def log_prob(self, x):
+        return self._log_prob(self.transform(x))
 
-    def _log_prob(self, parameter):
+    def _log_prob(self, x):
         # x = "distances from box`"
-        X = ((parameter - self._c).abs_() - self._r).clamp(min=0)
+        X = ((x - self._c).abs_() - self._r).clamp(min=0)
         return (self.tails.log_prob(X) - self._M).sum(-1)
diff --git a/gpytorch/priors/torch_priors.py b/gpytorch/priors/torch_priors.py
index 5a07f358..ef32a38d 100644
--- a/gpytorch/priors/torch_priors.py
+++ b/gpytorch/priors/torch_priors.py
@@ -1,10 +1,10 @@
 from __future__ import absolute_import, division, print_function, unicode_literals
 
 from torch.distributions import Gamma, MultivariateNormal, Normal
-from torch.nn import Module as TModule
 
 from .prior import Prior
 from .utils import _bufferize_attributes, _del_attributes
+from torch.nn import Module as TModule
 
 MVN_LAZY_PROPERTIES = ("covariance_matrix", "scale_tril", "precision_matrix")
 
@@ -17,11 +17,11 @@ class NormalPrior(Prior, Normal):
     where mu is the mean and sigma^2 is the variance.
     """
 
-    def __init__(self, loc, scale, log_transform=False, validate_args=False):
+    def __init__(self, loc, scale, validate_args=False, transform=None):
         TModule.__init__(self)
         Normal.__init__(self, loc=loc, scale=scale, validate_args=validate_args)
         _bufferize_attributes(self, ("loc", "scale"))
-        self._log_transform = log_transform
+        self._transform = transform
 
 
 class GammaPrior(Prior, Gamma):
@@ -32,11 +32,11 @@ class GammaPrior(Prior, Gamma):
     were alpha > 0 and beta > 0 are the concentration and rate parameters, respectively.
     """
 
-    def __init__(self, concentration, rate, log_transform=False, validate_args=False):
+    def __init__(self, concentration, rate, validate_args=False, transform=None):
         TModule.__init__(self)
         Gamma.__init__(self, concentration=concentration, rate=rate, validate_args=validate_args)
         _bufferize_attributes(self, ("concentration", "rate"))
-        self._log_transform = log_transform
+        self._transform = transform
 
 
 class MultivariateNormalPrior(Prior, MultivariateNormal):
@@ -48,13 +48,7 @@ class MultivariateNormalPrior(Prior, MultivariateNormal):
     """
 
     def __init__(
-        self,
-        loc,
-        covariance_matrix=None,
-        precision_matrix=None,
-        scale_tril=None,
-        log_transform=False,
-        validate_args=False,
+        self, loc, covariance_matrix=None, precision_matrix=None, scale_tril=None, validate_args=False, transform=None
     ):
         TModule.__init__(self)
         MultivariateNormal.__init__(
@@ -66,7 +60,7 @@ class MultivariateNormalPrior(Prior, MultivariateNormal):
             validate_args=validate_args,
         )
         _bufferize_attributes(self, ("loc", "_unbroadcasted_scale_tril"))
-        self._log_transform = log_transform
+        self._transform = transform
 
     def cuda(self, device=None):
         """Applies module-level cuda() call and resets all lazy properties"""
diff --git a/gpytorch/priors/wishart_prior.py b/gpytorch/priors/wishart_prior.py
index d301e053..249b0070 100644
--- a/gpytorch/priors/wishart_prior.py
+++ b/gpytorch/priors/wishart_prior.py
@@ -53,12 +53,11 @@ class WishartPrior(Prior):
         self.register_buffer("nu", nu)
         self.register_buffer("K_inv", K_inv)
         self.register_buffer("C", C)
-        self._log_transform = False
 
-    def log_prob(self, parameter):
+    def log_prob(self, X):
         # I'm sure this could be done more elegantly
-        logdetp = torch.logdet(parameter) if parameter.dim() == 2 else torch.stack([torch.logdet(p) for p in parameter])
-        Kinvp = torch.matmul(self.K_inv, parameter)
+        logdetp = torch.logdet(X) if X.dim() == 2 else torch.stack([torch.logdet(p) for p in X])
+        Kinvp = torch.matmul(self.K_inv, X)
         trKinvp = torch.diagonal(Kinvp, dim1=-2, dim2=-1).sum(-1)
         return self.C + 0.5 * (self.nu - self.n - 1) * logdetp - trKinvp
 
@@ -104,10 +103,9 @@ class InverseWishartPrior(Prior):
         self.register_buffer("nu", nu)
         self.register_buffer("K", K)
         self.register_buffer("C", C)
-        self._log_transform = False
 
-    def log_prob(self, parameter):
-        logdetp = torch.logdet(parameter) if parameter.dim() == 2 else torch.stack([torch.logdet(p) for p in parameter])
-        pinvK = torch.gesv(self.K, parameter)[0]
+    def log_prob(self, X):
+        logdetp = torch.logdet(X) if X.dim() == 2 else torch.stack([torch.logdet(p) for p in X])
+        pinvK = torch.gesv(self.K, X)[0]
         trpinvK = torch.diagonal(pinvK, dim1=-2, dim2=-1).sum(-1)  # trace in batch mode
         return self.C - 0.5 * ((self.nu + 2 * self.n) * logdetp + trpinvK)
diff --git a/gpytorch/utils/deprecation.py b/gpytorch/utils/deprecation.py
new file mode 100644
index 00000000..e9604116
--- /dev/null
+++ b/gpytorch/utils/deprecation.py
@@ -0,0 +1,20 @@
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import warnings
+
+
+class DeprecationError(Exception):
+    pass
+
+
+def _deprecate_kwarg(kwargs, old_kw, new_kw, new_kw_value):
+    old_kwarg = kwargs.get(old_kw)
+    if old_kwarg is not None:
+        warnings.warn("The `{}` argument is deprecated. Use `{}` instead.".format(old_kw, new_kw), DeprecationWarning)
+        if new_kw_value is not None:
+            raise ValueError("Cannot set both `{}` and `{}`".format(old_kw, new_kw))
+        return old_kwarg
+    return new_kw_value
diff --git a/gpytorch/variational/cholesky_variational_distribution.py b/gpytorch/variational/cholesky_variational_distribution.py
index c1ce169f..ae1d8bc3 100644
--- a/gpytorch/variational/cholesky_variational_distribution.py
+++ b/gpytorch/variational/cholesky_variational_distribution.py
@@ -1,83 +1,83 @@
-from __future__ import absolute_import, division, print_function, unicode_literals
-
-import torch
-from ..lazy import CholLazyTensor
-from ..distributions import MultivariateNormal
-from .variational_distribution import VariationalDistribution
-from ..utils.cholesky import batch_potrf
-
-
-class CholeskyVariationalDistribution(VariationalDistribution):
-    """
-    VariationalDistribution objects represent the variational distribution q(u) over a set of inducing points for GPs.
-
-    The most common way this distribution is defined is to parameterize it in terms of a mean vector and a covariance
-    matrix. In order to ensure that the covariance matrix remains positive definite, we only consider the lower triangle
-    and we manually ensure that the diagonal remains positive.
-    """
-
-    def __init__(self, num_inducing_points, batch_size=None):
-        """
-        Args:
-            num_inducing_points (int): Size of the variational distribution. This implies that the variational mean
-                should be this size, and the variational covariance matrix should have this many rows and columns.
-            batch_size (int, optional): Specifies an optional batch size for the variational parameters. This is useful
-                for example when doing additive variational inference.
-        """
-        super(VariationalDistribution, self).__init__()
-        mean_init = torch.zeros(num_inducing_points)
-        covar_init = torch.eye(num_inducing_points, num_inducing_points)
-        if batch_size is not None:
-            mean_init = mean_init.repeat(batch_size, 1)
-            covar_init = covar_init.repeat(batch_size, 1, 1)
-
-        self.register_parameter(name="variational_mean", parameter=torch.nn.Parameter(mean_init))
-        self.register_parameter(name="chol_variational_covar", parameter=torch.nn.Parameter(covar_init))
-
-    def initialize_variational_distribution(self, prior_dist):
-        self.variational_mean.data.copy_(prior_dist.mean)
-        if prior_dist.batch_shape.numel():
-            self.chol_variational_covar.data.copy_(batch_potrf(prior_dist.covariance_matrix))
-        else:
-            self.chol_variational_covar.data.copy_(prior_dist.covariance_matrix.cholesky(upper=True))
-
-    @property
-    def variational_distribution(self):
-        """
-        Return the variational distribution q(u) that this module represents.
-
-        In this simplest case, this involves directly returning the variational mean. For the variational covariance
-        matrix, we consider the lower triangle of the registered variational covariance parameter, while also ensuring
-        that the diagonal remains positive.
-        """
-        chol_variational_covar = self.chol_variational_covar
-
-        # Negate each row with a negative diagonal (the Cholesky decomposition
-        # of a matrix requires that the diagonal elements be positive).
-        if chol_variational_covar.ndimension() == 2:
-            chol_variational_covar = chol_variational_covar.triu()
-            inside = chol_variational_covar.diag().sign().unsqueeze(1).expand_as(chol_variational_covar).triu()
-        elif chol_variational_covar.ndimension() == 3:
-            batch_size, diag_size, _ = chol_variational_covar.size()
-
-            # Batch mode
-            chol_variational_covar_size = list(chol_variational_covar.size())[-2:]
-            mask = torch.ones(
-                *chol_variational_covar_size, dtype=chol_variational_covar.dtype, device=chol_variational_covar.device
-            ).triu_()
-            mask = mask.unsqueeze(0).expand(*([chol_variational_covar.size(0)] + chol_variational_covar_size))
-
-            batch_index = torch.arange(0, batch_size, dtype=torch.long, device=mask.device)
-            batch_index = batch_index.unsqueeze(1).repeat(1, diag_size).view(-1)
-            diag_index = torch.arange(0, diag_size, dtype=torch.long, device=mask.device)
-            diag_index = diag_index.unsqueeze(1).repeat(batch_size, 1).view(-1)
-            diag = chol_variational_covar[batch_index, diag_index, diag_index].view(batch_size, diag_size)
-
-            chol_variational_covar = chol_variational_covar.mul(mask)
-            inside = diag.sign().unsqueeze(-1).expand_as(chol_variational_covar).mul(mask)
-        else:
-            raise RuntimeError("Invalid number of variational covar dimensions")
-
-        chol_variational_covar = inside.mul(chol_variational_covar)
-        variational_covar = CholLazyTensor(chol_variational_covar.transpose(-1, -2))
-        return MultivariateNormal(self.variational_mean, variational_covar)
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import torch
+from ..lazy import CholLazyTensor
+from ..distributions import MultivariateNormal
+from .variational_distribution import VariationalDistribution
+from ..utils.cholesky import batch_potrf
+
+
+class CholeskyVariationalDistribution(VariationalDistribution):
+    """
+    VariationalDistribution objects represent the variational distribution q(u) over a set of inducing points for GPs.
+
+    The most common way this distribution is defined is to parameterize it in terms of a mean vector and a covariance
+    matrix. In order to ensure that the covariance matrix remains positive definite, we only consider the lower triangle
+    and we manually ensure that the diagonal remains positive.
+    """
+
+    def __init__(self, num_inducing_points, batch_size=None):
+        """
+        Args:
+            num_inducing_points (int): Size of the variational distribution. This implies that the variational mean
+                should be this size, and the variational covariance matrix should have this many rows and columns.
+            batch_size (int, optional): Specifies an optional batch size for the variational parameters. This is useful
+                for example when doing additive variational inference.
+        """
+        super(VariationalDistribution, self).__init__()
+        mean_init = torch.zeros(num_inducing_points)
+        covar_init = torch.eye(num_inducing_points, num_inducing_points)
+        if batch_size is not None:
+            mean_init = mean_init.repeat(batch_size, 1)
+            covar_init = covar_init.repeat(batch_size, 1, 1)
+
+        self.register_parameter(name="variational_mean", parameter=torch.nn.Parameter(mean_init))
+        self.register_parameter(name="chol_variational_covar", parameter=torch.nn.Parameter(covar_init))
+
+    def initialize_variational_distribution(self, prior_dist):
+        self.variational_mean.data.copy_(prior_dist.mean)
+        if prior_dist.batch_shape.numel():
+            self.chol_variational_covar.data.copy_(batch_potrf(prior_dist.covariance_matrix))
+        else:
+            self.chol_variational_covar.data.copy_(prior_dist.covariance_matrix.cholesky(upper=True))
+
+    @property
+    def variational_distribution(self):
+        """
+        Return the variational distribution q(u) that this module represents.
+
+        In this simplest case, this involves directly returning the variational mean. For the variational covariance
+        matrix, we consider the lower triangle of the registered variational covariance parameter, while also ensuring
+        that the diagonal remains positive.
+        """
+        chol_variational_covar = self.chol_variational_covar
+
+        # Negate each row with a negative diagonal (the Cholesky decomposition
+        # of a matrix requires that the diagonal elements be positive).
+        if chol_variational_covar.ndimension() == 2:
+            chol_variational_covar = chol_variational_covar.triu()
+            inside = chol_variational_covar.diag().sign().unsqueeze(1).expand_as(chol_variational_covar).triu()
+        elif chol_variational_covar.ndimension() == 3:
+            batch_size, diag_size, _ = chol_variational_covar.size()
+
+            # Batch mode
+            chol_variational_covar_size = list(chol_variational_covar.size())[-2:]
+            mask = torch.ones(
+                *chol_variational_covar_size, dtype=chol_variational_covar.dtype, device=chol_variational_covar.device
+            ).triu_()
+            mask = mask.unsqueeze(0).expand(*([chol_variational_covar.size(0)] + chol_variational_covar_size))
+
+            batch_index = torch.arange(0, batch_size, dtype=torch.long, device=mask.device)
+            batch_index = batch_index.unsqueeze(1).repeat(1, diag_size).view(-1)
+            diag_index = torch.arange(0, diag_size, dtype=torch.long, device=mask.device)
+            diag_index = diag_index.unsqueeze(1).repeat(batch_size, 1).view(-1)
+            diag = chol_variational_covar[batch_index, diag_index, diag_index].view(batch_size, diag_size)
+
+            chol_variational_covar = chol_variational_covar.mul(mask)
+            inside = diag.sign().unsqueeze(-1).expand_as(chol_variational_covar).mul(mask)
+        else:
+            raise RuntimeError("Invalid number of variational covar dimensions")
+
+        chol_variational_covar = inside.mul(chol_variational_covar)
+        variational_covar = CholLazyTensor(chol_variational_covar.transpose(-1, -2))
+        return MultivariateNormal(self.variational_mean, variational_covar)
diff --git a/gpytorch/variational/variational_distribution.py b/gpytorch/variational/variational_distribution.py
index 89efa9af..5230f117 100644
--- a/gpytorch/variational/variational_distribution.py
+++ b/gpytorch/variational/variational_distribution.py
@@ -1,39 +1,39 @@
-from __future__ import absolute_import, division, print_function, unicode_literals
-
-from ..module import Module
-
-
-class VariationalDistribution(Module):
-    """
-    VariationalDistribution objects represent the variational distribution q(u) over a set of inducing points for GPs.
-
-    The most common way this distribution is defined is to parameterize it in terms of a mean vector and a covariance
-    matrix. In order to ensure that the covariance matrix remains positive definite, we only consider the lower triangle
-    and we manually ensure that the diagonal remains positive.
-    """
-
-    def __init__(self, num_inducing_points, batch_size=None):
-        """
-        Args:
-            num_inducing_points (int): Size of the variational distribution. This implies that the variational mean
-                should be this size, and the variational covariance matrix should have this many rows and columns.
-            batch_size (int, optional): Specifies an optional batch size for the variational parameters. This is useful
-                for example when doing additive variational inference.
-        """
-        super(VariationalDistribution, self).__init__()
-
-    @property
-    def variational_distribution(self):
-        raise NotImplementedError
-
-    def forward(self, *args, **kwargs):
-        raise RuntimeError("VariationalDistribution is not intended to be called!")
-
-    def initialize_variational_distribution(self, prior_dist):
-        """
-        Method for initializing the variational distribution, based on the prior distribution.
-
-        Args:
-            - `prior_dist` (gpytorch.distribution.Distribution): the prior distribution
-        """
-        raise NotImplementedError
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from ..module import Module
+
+
+class VariationalDistribution(Module):
+    """
+    VariationalDistribution objects represent the variational distribution q(u) over a set of inducing points for GPs.
+
+    The most common way this distribution is defined is to parameterize it in terms of a mean vector and a covariance
+    matrix. In order to ensure that the covariance matrix remains positive definite, we only consider the lower triangle
+    and we manually ensure that the diagonal remains positive.
+    """
+
+    def __init__(self, num_inducing_points, batch_size=None):
+        """
+        Args:
+            num_inducing_points (int): Size of the variational distribution. This implies that the variational mean
+                should be this size, and the variational covariance matrix should have this many rows and columns.
+            batch_size (int, optional): Specifies an optional batch size for the variational parameters. This is useful
+                for example when doing additive variational inference.
+        """
+        super(VariationalDistribution, self).__init__()
+
+    @property
+    def variational_distribution(self):
+        raise NotImplementedError
+
+    def forward(self, *args, **kwargs):
+        raise RuntimeError("VariationalDistribution is not intended to be called!")
+
+    def initialize_variational_distribution(self, prior_dist):
+        """
+        Method for initializing the variational distribution, based on the prior distribution.
+
+        Args:
+            - `prior_dist` (gpytorch.distribution.Distribution): the prior distribution
+        """
+        raise NotImplementedError
diff --git a/gpytorch/variational/variational_strategy.py b/gpytorch/variational/variational_strategy.py
index 2f988c58..4f2610ce 100644
--- a/gpytorch/variational/variational_strategy.py
+++ b/gpytorch/variational/variational_strategy.py
@@ -112,8 +112,7 @@ class VariationalStrategy(Module):
             inv_product = induc_induc_covar.inv_matmul(induc_data_covar)
             factor = variational_dist.lazy_covariance_matrix.root_decomposition().root.matmul(inv_product)
             predictive_mean = torch.add(
-                test_mean,
-                inv_product.transpose(-1, -2).matmul((var_dist_mean - induc_mean).unsqueeze(-1)).squeeze(-1)
+                test_mean, inv_product.transpose(-1, -2).matmul((var_dist_mean - induc_mean).unsqueeze(-1)).squeeze(-1)
             )
             predictive_covar = RootLazyTensor(factor.transpose(-2, -1))
 


Test patch:
diff --git a/test/examples/test_batch_gp_regression.py b/test/examples/test_batch_gp_regression.py
index 06ecac26..55b3351c 100644
--- a/test/examples/test_batch_gp_regression.py
+++ b/test/examples/test_batch_gp_regression.py
@@ -43,12 +43,12 @@ class ExactGPModel(gpytorch.models.ExactGP):
         self.covar_module = ScaleKernel(
             RBFKernel(
                 batch_size=batch_size,
-                log_lengthscale_prior=gpytorch.priors.NormalPrior(
-                    loc=torch.zeros(batch_size, 1, 1), scale=torch.ones(batch_size, 1, 1), log_transform=True
+                lengthscale_prior=gpytorch.priors.NormalPrior(
+                    loc=torch.zeros(batch_size, 1, 1), scale=torch.ones(batch_size, 1, 1)
                 ),
             ),
             batch_size=batch_size,
-            log_outputscale_prior=gpytorch.priors.SmoothedBoxPrior(-2, 2, log_transform=True),
+            outputscale_prior=gpytorch.priors.SmoothedBoxPrior(-2, 2),
         )
 
     def forward(self, x):
@@ -73,7 +73,7 @@ class TestBatchGPRegression(unittest.TestCase):
     def test_train_on_single_set_test_on_batch(self):
         # We're manually going to set the hyperparameters to something they shouldn't be
         likelihood = GaussianLikelihood(
-            log_noise_prior=gpytorch.priors.NormalPrior(loc=torch.zeros(1), scale=torch.ones(1), log_transform=True)
+            noise_prior=gpytorch.priors.NormalPrior(loc=torch.zeros(1), scale=torch.ones(1))
         )
         gp_model = ExactGPModel(train_x1, train_y1, likelihood)
         mll = gpytorch.ExactMarginalLogLikelihood(likelihood, gp_model)
@@ -116,8 +116,7 @@ class TestBatchGPRegression(unittest.TestCase):
     def test_train_on_batch_test_on_batch(self):
         # We're manually going to set the hyperparameters to something they shouldn't be
         likelihood = GaussianLikelihood(
-            log_noise_prior=gpytorch.priors.NormalPrior(loc=torch.zeros(2), scale=torch.ones(2), log_transform=True),
-            batch_size=2,
+            noise_prior=gpytorch.priors.NormalPrior(loc=torch.zeros(2), scale=torch.ones(2)), batch_size=2
         )
         gp_model = ExactGPModel(train_x12, train_y12, likelihood, batch_size=2)
         mll = gpytorch.ExactMarginalLogLikelihood(likelihood, gp_model)
@@ -156,8 +155,7 @@ class TestBatchGPRegression(unittest.TestCase):
     def test_train_on_batch_test_on_batch_shared_hypers_over_batch(self):
         # We're manually going to set the hyperparameters to something they shouldn't be
         likelihood = GaussianLikelihood(
-            log_noise_prior=gpytorch.priors.NormalPrior(loc=torch.zeros(2), scale=torch.ones(2), log_transform=True),
-            batch_size=1,
+            noise_prior=gpytorch.priors.NormalPrior(loc=torch.zeros(2), scale=torch.ones(2)), batch_size=1
         )
         gp_model = ExactGPModel(train_x12, train_y12, likelihood, batch_size=1)
         mll = gpytorch.ExactMarginalLogLikelihood(likelihood, gp_model)
diff --git a/test/examples/test_batch_multitask_gp_regression.py b/test/examples/test_batch_multitask_gp_regression.py
index 68c695f1..a535c078 100644
--- a/test/examples/test_batch_multitask_gp_regression.py
+++ b/test/examples/test_batch_multitask_gp_regression.py
@@ -43,8 +43,8 @@ class ExactGPModel(gpytorch.models.ExactGP):
         self.covar_module = MultitaskKernel(
             RBFKernel(
                 batch_size=batch_size,
-                log_lengthscale_prior=gpytorch.priors.NormalPrior(
-                    loc=torch.zeros(batch_size, 1, 1), scale=torch.ones(batch_size, 1, 1), log_transform=True
+                lengthscale_prior=gpytorch.priors.NormalPrior(
+                    loc=torch.zeros(batch_size, 1, 1), scale=torch.ones(batch_size, 1, 1)
                 ),
             ),
             num_tasks=2,
@@ -73,8 +73,7 @@ class TestBatchGPRegression(unittest.TestCase):
     def test_train_on_single_set_test_on_batch(self):
         # We're manually going to set the hyperparameters to something they shouldn't be
         likelihood = MultitaskGaussianLikelihood(
-            log_noise_prior=gpytorch.priors.NormalPrior(loc=torch.zeros(1), scale=torch.ones(1), log_transform=True),
-            num_tasks=2,
+            noise_prior=gpytorch.priors.NormalPrior(loc=torch.zeros(1), scale=torch.ones(1)), num_tasks=2
         )
         gp_model = ExactGPModel(train_x1, train_y1, likelihood)
         mll = gpytorch.ExactMarginalLogLikelihood(likelihood, gp_model)
@@ -117,7 +116,7 @@ class TestBatchGPRegression(unittest.TestCase):
     def test_train_on_batch_test_on_batch(self):
         # We're manually going to set the hyperparameters to something they shouldn't be
         likelihood = MultitaskGaussianLikelihood(
-            log_noise_prior=gpytorch.priors.NormalPrior(loc=torch.zeros(2), scale=torch.ones(2), log_transform=True),
+            noise_prior=gpytorch.priors.NormalPrior(loc=torch.zeros(2), scale=torch.ones(2)),
             batch_size=2,
             num_tasks=2,
         )
@@ -158,7 +157,7 @@ class TestBatchGPRegression(unittest.TestCase):
     def test_train_on_batch_test_on_batch_shared_hypers_over_batch(self):
         # We're manually going to set the hyperparameters to something they shouldn't be
         likelihood = MultitaskGaussianLikelihood(
-            log_noise_prior=gpytorch.priors.NormalPrior(loc=torch.zeros(2), scale=torch.ones(2), log_transform=True),
+            noise_prior=gpytorch.priors.NormalPrior(loc=torch.zeros(2), scale=torch.ones(2)),
             batch_size=1,
             num_tasks=2,
         )
diff --git a/test/examples/test_batch_svgp_gp_regression.py b/test/examples/test_batch_svgp_gp_regression.py
index 4f8322ba..fd1459b5 100644
--- a/test/examples/test_batch_svgp_gp_regression.py
+++ b/test/examples/test_batch_svgp_gp_regression.py
@@ -40,9 +40,7 @@ class SVGPRegressionModel(AbstractVariationalGP):
         super(SVGPRegressionModel, self).__init__(variational_strategy)
         self.mean_module = gpytorch.means.ConstantMean()
         self.covar_module = gpytorch.kernels.ScaleKernel(
-            gpytorch.kernels.RBFKernel(
-                log_lengthscale_prior=gpytorch.priors.SmoothedBoxPrior(0.001, 1.0, sigma=0.1, log_transform=True)
-            )
+            gpytorch.kernels.RBFKernel(lengthscale_prior=gpytorch.priors.SmoothedBoxPrior(0.001, 1.0, sigma=0.1))
         )
 
     def forward(self, x):
@@ -75,10 +73,7 @@ class TestSVGPRegression(unittest.TestCase):
         # Find optimal model hyperparameters
         model.train()
         likelihood.train()
-        optimizer = optim.Adam([
-            {'params': model.parameters()},
-            {'params': likelihood.parameters()},
-        ], lr=0.01)
+        optimizer = optim.Adam([{"params": model.parameters()}, {"params": likelihood.parameters()}], lr=0.01)
         for _ in range(150):
             optimizer.zero_grad()
             output = model(train_x)
@@ -114,10 +109,7 @@ class TestSVGPRegression(unittest.TestCase):
             # Find optimal model hyperparameters
             model.train()
             likelihood.train()
-            optimizer = optim.Adam([
-                {'params': model.parameters()},
-                {'params': likelihood.parameters()},
-            ], lr=0.01)
+            optimizer = optim.Adam([{"params": model.parameters()}, {"params": likelihood.parameters()}], lr=0.01)
             for _ in range(150):
                 optimizer.zero_grad()
                 output = model(train_x)
diff --git a/test/examples/test_grid_gp_regression.py b/test/examples/test_grid_gp_regression.py
index 8f2ee48a..f3c39fe3 100644
--- a/test/examples/test_grid_gp_regression.py
+++ b/test/examples/test_grid_gp_regression.py
@@ -1,99 +1,99 @@
-import gpytorch
-import torch
-import math
-import unittest
-import os
-import random
-from torch import optim
-
-
-def make_data(grid, cuda=False):
-    train_x = gpytorch.utils.grid.create_data_from_grid(grid)
-    train_y = torch.sin((train_x[:, 0] + train_x[:, 1]) * (2 * math.pi)) + torch.randn_like(train_x[:, 0]).mul(0.01)
-    n = 20
-    test_x = torch.zeros(int(pow(n, 2)), 2)
-    for i in range(n):
-        for j in range(n):
-            test_x[i * n + j][0] = float(i) / (n - 1)
-            test_x[i * n + j][1] = float(j) / (n - 1)
-    test_y = torch.sin(((test_x[:, 0] + test_x[:, 1]) * (2 * math.pi)))
-    if cuda:
-        train_x = train_x.cuda()
-        train_y = train_y.cuda()
-        test_x = test_x.cuda()
-        test_y = test_y.cuda()
-    return train_x, train_y, test_x, test_y
-
-
-class GridGPRegressionModel(gpytorch.models.ExactGP):
-    def __init__(self, grid, train_x, train_y, likelihood):
-        super(GridGPRegressionModel, self).__init__(train_x, train_y, likelihood)
-        self.mean_module = gpytorch.means.ConstantMean()
-        self.covar_module = gpytorch.kernels.GridKernel(gpytorch.kernels.RBFKernel(), grid=grid)
-
-    def forward(self, x):
-        mean_x = self.mean_module(x)
-        covar_x = self.covar_module(x)
-        return gpytorch.distributions.MultivariateNormal(mean_x, covar_x)
-
-
-class TestGridGPRegression(unittest.TestCase):
-    def setUp(self):
-        if os.getenv("UNLOCK_SEED") is None or os.getenv("UNLOCK_SEED").lower() == "false":
-            self.rng_state = torch.get_rng_state()
-            torch.manual_seed(0)
-            if torch.cuda.is_available():
-                torch.cuda.manual_seed_all(0)
-            random.seed(0)
-
-    def tearDown(self):
-        if hasattr(self, "rng_state"):
-            torch.set_rng_state(self.rng_state)
-
-    def test_grid_gp_mean_abs_error(self):
-        grid_bounds = [(0, 1), (0, 2)]
-        grid_size = 25
-        grid = torch.zeros(grid_size, len(grid_bounds))
-        for i in range(len(grid_bounds)):
-            grid_diff = float(grid_bounds[i][1] - grid_bounds[i][0]) / (grid_size - 2)
-            grid[:, i] = torch.linspace(grid_bounds[i][0] - grid_diff, grid_bounds[i][1] + grid_diff, grid_size)
-
-        train_x, train_y, test_x, test_y = make_data(grid)
-        likelihood = gpytorch.likelihoods.GaussianLikelihood()
-        gp_model = GridGPRegressionModel(grid, train_x, train_y, likelihood)
-        mll = gpytorch.mlls.ExactMarginalLogLikelihood(likelihood, gp_model)
-
-        # Optimize the model
-        gp_model.train()
-        likelihood.train()
-
-        optimizer = optim.Adam(list(gp_model.parameters()) + list(likelihood.parameters()), lr=0.1)
-        optimizer.n_iter = 0
-        with gpytorch.settings.debug(False):
-            for _ in range(25):
-                optimizer.zero_grad()
-                output = gp_model(train_x)
-                loss = -mll(output, train_y)
-                loss.backward()
-                optimizer.n_iter += 1
-                optimizer.step()
-
-            for param in gp_model.parameters():
-                self.assertTrue(param.grad is not None)
-                self.assertGreater(param.grad.norm().item(), 0)
-            for param in likelihood.parameters():
-                self.assertTrue(param.grad is not None)
-                self.assertGreater(param.grad.norm().item(), 0)
-
-            # Test the model
-            gp_model.eval()
-            likelihood.eval()
-
-            train_preds = likelihood(gp_model(train_x)).mean
-            mean_abs_error = torch.mean(torch.abs(train_y - train_preds))
-
-        self.assertLess(mean_abs_error.squeeze().item(), 0.15)
-
-
-if __name__ == "__main__":
-    unittest.main()
+import gpytorch
+import torch
+import math
+import unittest
+import os
+import random
+from torch import optim
+
+
+def make_data(grid, cuda=False):
+    train_x = gpytorch.utils.grid.create_data_from_grid(grid)
+    train_y = torch.sin((train_x[:, 0] + train_x[:, 1]) * (2 * math.pi)) + torch.randn_like(train_x[:, 0]).mul(0.01)
+    n = 20
+    test_x = torch.zeros(int(pow(n, 2)), 2)
+    for i in range(n):
+        for j in range(n):
+            test_x[i * n + j][0] = float(i) / (n - 1)
+            test_x[i * n + j][1] = float(j) / (n - 1)
+    test_y = torch.sin(((test_x[:, 0] + test_x[:, 1]) * (2 * math.pi)))
+    if cuda:
+        train_x = train_x.cuda()
+        train_y = train_y.cuda()
+        test_x = test_x.cuda()
+        test_y = test_y.cuda()
+    return train_x, train_y, test_x, test_y
+
+
+class GridGPRegressionModel(gpytorch.models.ExactGP):
+    def __init__(self, grid, train_x, train_y, likelihood):
+        super(GridGPRegressionModel, self).__init__(train_x, train_y, likelihood)
+        self.mean_module = gpytorch.means.ConstantMean()
+        self.covar_module = gpytorch.kernels.GridKernel(gpytorch.kernels.RBFKernel(), grid=grid)
+
+    def forward(self, x):
+        mean_x = self.mean_module(x)
+        covar_x = self.covar_module(x)
+        return gpytorch.distributions.MultivariateNormal(mean_x, covar_x)
+
+
+class TestGridGPRegression(unittest.TestCase):
+    def setUp(self):
+        if os.getenv("UNLOCK_SEED") is None or os.getenv("UNLOCK_SEED").lower() == "false":
+            self.rng_state = torch.get_rng_state()
+            torch.manual_seed(0)
+            if torch.cuda.is_available():
+                torch.cuda.manual_seed_all(0)
+            random.seed(0)
+
+    def tearDown(self):
+        if hasattr(self, "rng_state"):
+            torch.set_rng_state(self.rng_state)
+
+    def test_grid_gp_mean_abs_error(self):
+        grid_bounds = [(0, 1), (0, 2)]
+        grid_size = 25
+        grid = torch.zeros(grid_size, len(grid_bounds))
+        for i in range(len(grid_bounds)):
+            grid_diff = float(grid_bounds[i][1] - grid_bounds[i][0]) / (grid_size - 2)
+            grid[:, i] = torch.linspace(grid_bounds[i][0] - grid_diff, grid_bounds[i][1] + grid_diff, grid_size)
+
+        train_x, train_y, test_x, test_y = make_data(grid)
+        likelihood = gpytorch.likelihoods.GaussianLikelihood()
+        gp_model = GridGPRegressionModel(grid, train_x, train_y, likelihood)
+        mll = gpytorch.mlls.ExactMarginalLogLikelihood(likelihood, gp_model)
+
+        # Optimize the model
+        gp_model.train()
+        likelihood.train()
+
+        optimizer = optim.Adam(list(gp_model.parameters()) + list(likelihood.parameters()), lr=0.1)
+        optimizer.n_iter = 0
+        with gpytorch.settings.debug(False):
+            for _ in range(25):
+                optimizer.zero_grad()
+                output = gp_model(train_x)
+                loss = -mll(output, train_y)
+                loss.backward()
+                optimizer.n_iter += 1
+                optimizer.step()
+
+            for param in gp_model.parameters():
+                self.assertTrue(param.grad is not None)
+                self.assertGreater(param.grad.norm().item(), 0)
+            for param in likelihood.parameters():
+                self.assertTrue(param.grad is not None)
+                self.assertGreater(param.grad.norm().item(), 0)
+
+            # Test the model
+            gp_model.eval()
+            likelihood.eval()
+
+            train_preds = likelihood(gp_model(train_x)).mean
+            mean_abs_error = torch.mean(torch.abs(train_y - train_preds))
+
+        self.assertLess(mean_abs_error.squeeze().item(), 0.15)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/test/examples/test_hadamard_multitask_gp_regression.py b/test/examples/test_hadamard_multitask_gp_regression.py
index 1716901e..a9c0f8c6 100644
--- a/test/examples/test_hadamard_multitask_gp_regression.py
+++ b/test/examples/test_hadamard_multitask_gp_regression.py
@@ -37,7 +37,7 @@ class HadamardMultitaskGPModel(gpytorch.models.ExactGP):
         self.covar_module = RBFKernel()
         # We learn an IndexKernel for 2 tasks
         # (so we'll actually learn 2x2=4 tasks with correlations)
-        sd_prior = SmoothedBoxPrior(exp(-4), exp(4), log_transform=True)
+        sd_prior = SmoothedBoxPrior(exp(-4), exp(4))
         cov_prior = LKJCovariancePrior(n=2, eta=1, sd_prior=sd_prior)
         self.task_covar_module = IndexKernel(num_tasks=2, rank=1, prior=cov_prior)
 
@@ -66,7 +66,7 @@ class TestHadamardMultitaskGPRegression(unittest.TestCase):
             torch.set_rng_state(self.rng_state)
 
     def test_multitask_gp_mean_abs_error(self):
-        likelihood = GaussianLikelihood(log_noise_prior=SmoothedBoxPrior(-6, 6))
+        likelihood = GaussianLikelihood(noise_prior=SmoothedBoxPrior(-6, 6))
         gp_model = HadamardMultitaskGPModel(
             (torch.cat([train_x, train_x]), torch.cat([y1_inds, y2_inds])), torch.cat([train_y1, train_y2]), likelihood
         )
diff --git a/test/examples/test_kissgp_additive_classification.py b/test/examples/test_kissgp_additive_classification.py
index a298d40b..a92c3da7 100644
--- a/test/examples/test_kissgp_additive_classification.py
+++ b/test/examples/test_kissgp_additive_classification.py
@@ -42,9 +42,9 @@ class GPClassificationModel(AbstractVariationalGP):
         self.mean_module = ConstantMean(prior=SmoothedBoxPrior(-1e-5, 1e-5))
         self.covar_module = ScaleKernel(
             RBFKernel(
-                ard_num_dims=1, log_lengthscale_prior=SmoothedBoxPrior(exp(-5), exp(6), sigma=0.1, log_transform=True)
+                ard_num_dims=1, lengthscale_prior=SmoothedBoxPrior(exp(-5), exp(6), sigma=0.1)
             ),
-            log_outputscale_prior=SmoothedBoxPrior(exp(-5), exp(6), sigma=0.1, log_transform=True),
+            outputscale_prior=SmoothedBoxPrior(exp(-5), exp(6), sigma=0.1),
         )
 
     def forward(self, x):
diff --git a/test/examples/test_kissgp_additive_regression.py b/test/examples/test_kissgp_additive_regression.py
index 47f5e175..184e2313 100644
--- a/test/examples/test_kissgp_additive_regression.py
+++ b/test/examples/test_kissgp_additive_regression.py
@@ -42,7 +42,7 @@ class GPRegressionModel(gpytorch.models.ExactGP):
         self.mean_module = ZeroMean()
         self.base_covar_module = ScaleKernel(
             RBFKernel(
-                ard_num_dims=2, log_lengthscale_prior=SmoothedBoxPrior(exp(-3), exp(3), sigma=0.1, log_transform=True)
+                ard_num_dims=2, lengthscale_prior=SmoothedBoxPrior(exp(-3), exp(3), sigma=0.1)
             )
         )
         self.covar_module = AdditiveStructureKernel(
diff --git a/test/examples/test_kissgp_dkl_regression.py b/test/examples/test_kissgp_dkl_regression.py
index 53d29673..61a4b77e 100644
--- a/test/examples/test_kissgp_dkl_regression.py
+++ b/test/examples/test_kissgp_dkl_regression.py
@@ -52,7 +52,7 @@ class GPRegressionModel(gpytorch.models.ExactGP):
         super(GPRegressionModel, self).__init__(train_x, train_y, likelihood)
         self.mean_module = ConstantMean(prior=SmoothedBoxPrior(-1e-5, 1e-5))
         self.base_covar_module = ScaleKernel(
-            RBFKernel(log_lengthscale_prior=SmoothedBoxPrior(exp(-5), exp(6), sigma=0.1, log_transform=True))
+            RBFKernel(lengthscale_prior=SmoothedBoxPrior(exp(-5), exp(6), sigma=0.1))
         )
         self.covar_module = GridInterpolationKernel(self.base_covar_module, grid_size=50, num_dims=1)
         self.feature_extractor = feature_extractor
diff --git a/test/examples/test_kissgp_gp_classification.py b/test/examples/test_kissgp_gp_classification.py
index 04e002e8..b03cde3e 100644
--- a/test/examples/test_kissgp_gp_classification.py
+++ b/test/examples/test_kissgp_gp_classification.py
@@ -27,8 +27,8 @@ class GPClassificationModel(gpytorch.models.GridInducingVariationalGP):
         super(GPClassificationModel, self).__init__(grid_size=32, grid_bounds=[(0, 1)])
         self.mean_module = ConstantMean(prior=SmoothedBoxPrior(-5, 5))
         self.covar_module = ScaleKernel(
-            RBFKernel(log_lengthscale_prior=SmoothedBoxPrior(exp(-5), exp(6), sigma=0.1, log_transform=True)),
-            log_outputscale_prior=SmoothedBoxPrior(exp(-5), exp(6), sigma=0.1, log_transform=True),
+            RBFKernel(lengthscale_prior=SmoothedBoxPrior(exp(-5), exp(6), sigma=0.1)),
+            outputscale_prior=SmoothedBoxPrior(exp(-5), exp(6), sigma=0.1),
         )
 
     def forward(self, x):
diff --git a/test/examples/test_kissgp_gp_regression.py b/test/examples/test_kissgp_gp_regression.py
index f3a379dd..14f96441 100644
--- a/test/examples/test_kissgp_gp_regression.py
+++ b/test/examples/test_kissgp_gp_regression.py
@@ -38,7 +38,7 @@ class GPRegressionModel(gpytorch.models.ExactGP):
         super(GPRegressionModel, self).__init__(train_x, train_y, likelihood)
         self.mean_module = ConstantMean(prior=SmoothedBoxPrior(-1e-5, 1e-5))
         self.base_covar_module = ScaleKernel(
-            RBFKernel(log_lengthscale_prior=SmoothedBoxPrior(exp(-5), exp(6), sigma=0.1, log_transform=True))
+            RBFKernel(lengthscale_prior=SmoothedBoxPrior(exp(-5), exp(6), sigma=0.1))
         )
         self.covar_module = GridInterpolationKernel(self.base_covar_module, grid_size=50, num_dims=1)
 
diff --git a/test/examples/test_kissgp_kronecker_product_classification.py b/test/examples/test_kissgp_kronecker_product_classification.py
index ce463cae..0b84f1a9 100644
--- a/test/examples/test_kissgp_kronecker_product_classification.py
+++ b/test/examples/test_kissgp_kronecker_product_classification.py
@@ -35,7 +35,7 @@ class GPClassificationModel(gpytorch.models.GridInducingVariationalGP):
         self.mean_module = ConstantMean(prior=SmoothedBoxPrior(-1e-5, 1e-5))
         self.covar_module = ScaleKernel(
             RBFKernel(
-                ard_num_dims=2, log_lengthscale_prior=SmoothedBoxPrior(exp(-2.5), exp(3), sigma=0.1, log_transform=True)
+                ard_num_dims=2, lengthscale_prior=SmoothedBoxPrior(exp(-2.5), exp(3), sigma=0.1)
             )
         )
 
diff --git a/test/examples/test_kissgp_kronecker_product_regression.py b/test/examples/test_kissgp_kronecker_product_regression.py
index 61ba0b42..0353685c 100644
--- a/test/examples/test_kissgp_kronecker_product_regression.py
+++ b/test/examples/test_kissgp_kronecker_product_regression.py
@@ -48,7 +48,7 @@ class GPRegressionModel(gpytorch.models.ExactGP):
         self.mean_module = ConstantMean(prior=SmoothedBoxPrior(-1, 1))
         self.base_covar_module = ScaleKernel(
             RBFKernel(
-                ard_num_dims=2, log_lengthscale_prior=SmoothedBoxPrior(exp(-3), exp(3), sigma=0.1, log_transform=True)
+                ard_num_dims=2, lengthscale_prior=SmoothedBoxPrior(exp(-3), exp(3), sigma=0.1)
             )
         )
         self.covar_module = GridInterpolationKernel(self.base_covar_module, grid_size=64, num_dims=2)
diff --git a/test/examples/test_kissgp_multiplicative_regression.py b/test/examples/test_kissgp_multiplicative_regression.py
index 27687557..1b2b9434 100644
--- a/test/examples/test_kissgp_multiplicative_regression.py
+++ b/test/examples/test_kissgp_multiplicative_regression.py
@@ -44,7 +44,7 @@ class GPRegressionModel(gpytorch.models.ExactGP):
         super(GPRegressionModel, self).__init__(train_x, train_y, likelihood)
         self.mean_module = ConstantMean(prior=SmoothedBoxPrior(-1, 1))
         self.base_covar_module = ScaleKernel(
-            RBFKernel(log_lengthscale_prior=SmoothedBoxPrior(exp(-3), exp(3), sigma=0.1, log_transform=True))
+            RBFKernel(lengthscale_prior=SmoothedBoxPrior(exp(-3), exp(3), sigma=0.1))
         )
         self.covar_module = ProductStructureKernel(
             GridInterpolationKernel(self.base_covar_module, grid_size=100, num_dims=2), num_dims=2
diff --git a/test/examples/test_kissgp_variational_regression.py b/test/examples/test_kissgp_variational_regression.py
index f2b29a76..054e4bcb 100644
--- a/test/examples/test_kissgp_variational_regression.py
+++ b/test/examples/test_kissgp_variational_regression.py
@@ -39,7 +39,7 @@ class GPRegressionModel(gpytorch.models.AbstractVariationalGP):
         super(GPRegressionModel, self).__init__(variational_strategy)
         self.mean_module = ConstantMean(prior=SmoothedBoxPrior(-10, 10))
         self.covar_module = ScaleKernel(
-            RBFKernel(log_lengthscale_prior=SmoothedBoxPrior(exp(-3), exp(6), sigma=0.1, log_transform=True))
+            RBFKernel(lengthscale_prior=SmoothedBoxPrior(exp(-3), exp(6), sigma=0.1))
         )
 
     def forward(self, x):
diff --git a/test/examples/test_kissgp_white_noise_regression.py b/test/examples/test_kissgp_white_noise_regression.py
index 6f5a509b..fe442eab 100644
--- a/test/examples/test_kissgp_white_noise_regression.py
+++ b/test/examples/test_kissgp_white_noise_regression.py
@@ -38,7 +38,7 @@ class GPRegressionModel(gpytorch.models.ExactGP):
         super(GPRegressionModel, self).__init__(train_x, train_y, likelihood)
         self.mean_module = ConstantMean(prior=SmoothedBoxPrior(-1e-5, 1e-5))
         self.base_covar_module = ScaleKernel(
-            RBFKernel(log_lengthscale_prior=SmoothedBoxPrior(exp(-5), exp(6), sigma=0.1, log_transform=True))
+            RBFKernel(lengthscale_prior=SmoothedBoxPrior(exp(-5), exp(6), sigma=0.1))
         )
         self.grid_covar_module = GridInterpolationKernel(self.base_covar_module, grid_size=50, num_dims=1)
         self.noise_covar_module = WhiteNoiseKernel(variances=torch.ones(100) * 0.001)
diff --git a/test/examples/test_sgpr_regression.py b/test/examples/test_sgpr_regression.py
index f84949fd..7d1940a7 100644
--- a/test/examples/test_sgpr_regression.py
+++ b/test/examples/test_sgpr_regression.py
@@ -39,7 +39,7 @@ class GPRegressionModel(gpytorch.models.ExactGP):
         super(GPRegressionModel, self).__init__(train_x, train_y, likelihood)
         self.mean_module = ConstantMean(prior=SmoothedBoxPrior(-1e-5, 1e-5))
         self.base_covar_module = ScaleKernel(
-            RBFKernel(log_lengthscale_prior=SmoothedBoxPrior(exp(-5), exp(6), sigma=0.1, log_transform=True))
+            RBFKernel(lengthscale_prior=SmoothedBoxPrior(exp(-5), exp(6), sigma=0.1))
         )
         self.covar_module = InducingPointKernel(
             self.base_covar_module, inducing_points=torch.linspace(0, 1, 32), likelihood=likelihood
diff --git a/test/examples/test_simple_gp_regression.py b/test/examples/test_simple_gp_regression.py
index 63ce09a2..5617ee2a 100644
--- a/test/examples/test_simple_gp_regression.py
+++ b/test/examples/test_simple_gp_regression.py
@@ -30,9 +30,7 @@ class ExactGPModel(gpytorch.models.ExactGP):
     def __init__(self, train_inputs, train_targets, likelihood):
         super(ExactGPModel, self).__init__(train_inputs, train_targets, likelihood)
         self.mean_module = ConstantMean(prior=SmoothedBoxPrior(-1, 1))
-        self.covar_module = ScaleKernel(
-            RBFKernel(log_lengthscale_prior=SmoothedBoxPrior(exp(-3), exp(3), sigma=0.1, log_transform=True))
-        )
+        self.covar_module = ScaleKernel(RBFKernel(lengthscale_prior=SmoothedBoxPrior(exp(-3), exp(3), sigma=0.1)))
 
     def forward(self, x):
         mean_x = self.mean_module(x)
@@ -55,13 +53,11 @@ class TestSimpleGPRegression(unittest.TestCase):
 
     def test_prior(self):
         # We're manually going to set the hyperparameters to be ridiculous
-        likelihood = GaussianLikelihood(
-            log_noise_prior=SmoothedBoxPrior(exp(-3), exp(3), sigma=0.1, log_transform=True)
-        )
+        likelihood = GaussianLikelihood(noise_prior=SmoothedBoxPrior(exp(-3), exp(3), sigma=0.1))
         gp_model = ExactGPModel(None, None, likelihood)
         # Update lengthscale prior to accommodate extreme parameters
-        gp_model.covar_module.base_kernel.set_parameter_priors(
-            log_lengthscale=SmoothedBoxPrior(exp(-10), exp(10), sigma=0.5, log_transform=True)
+        gp_model.covar_module.base_kernel.register_prior(
+            "log_lengthscale_prior", SmoothedBoxPrior(exp(-10), exp(10), sigma=0.5), "log_lengthscale"
         )
         gp_model.mean_module.initialize(constant=1.5)
         gp_model.covar_module.base_kernel.initialize(log_lengthscale=0)
@@ -79,13 +75,11 @@ class TestSimpleGPRegression(unittest.TestCase):
 
     def test_posterior_latent_gp_and_likelihood_without_optimization(self):
         # We're manually going to set the hyperparameters to be ridiculous
-        likelihood = GaussianLikelihood(
-            log_noise_prior=SmoothedBoxPrior(exp(-3), exp(3), sigma=0.1, log_transform=True)
-        )
+        likelihood = GaussianLikelihood(noise_prior=SmoothedBoxPrior(exp(-3), exp(3), sigma=0.1))
         gp_model = ExactGPModel(train_x, train_y, likelihood)
         # Update lengthscale prior to accommodate extreme parameters
-        gp_model.covar_module.base_kernel.set_parameter_priors(
-            log_lengthscale=SmoothedBoxPrior(exp(-10), exp(10), sigma=0.5, log_transform=True)
+        gp_model.covar_module.base_kernel.register_prior(
+            "log_lengthscale_prior", SmoothedBoxPrior(exp(-10), exp(10), sigma=0.5), "log_lengthscale"
         )
         gp_model.covar_module.base_kernel.initialize(log_lengthscale=-10)
         likelihood.initialize(log_noise=-10)
@@ -110,9 +104,7 @@ class TestSimpleGPRegression(unittest.TestCase):
 
     def test_posterior_latent_gp_and_likelihood_with_optimization(self):
         # We're manually going to set the hyperparameters to something they shouldn't be
-        likelihood = GaussianLikelihood(
-            log_noise_prior=SmoothedBoxPrior(exp(-3), exp(3), sigma=0.1, log_transform=True)
-        )
+        likelihood = GaussianLikelihood(noise_prior=SmoothedBoxPrior(exp(-3), exp(3), sigma=0.1))
         gp_model = ExactGPModel(train_x, train_y, likelihood)
         mll = gpytorch.ExactMarginalLogLikelihood(likelihood, gp_model)
         gp_model.covar_module.base_kernel.initialize(log_lengthscale=1)
@@ -153,9 +145,7 @@ class TestSimpleGPRegression(unittest.TestCase):
         with gpytorch.fast_pred_var(), gpytorch.settings.debug(False):
             # We're manually going to set the hyperparameters to
             # something they shouldn't be
-            likelihood = GaussianLikelihood(
-                log_noise_prior=SmoothedBoxPrior(exp(-3), exp(3), sigma=0.1, log_transform=True)
-            )
+            likelihood = GaussianLikelihood(noise_prior=SmoothedBoxPrior(exp(-3), exp(3), sigma=0.1))
             gp_model = ExactGPModel(train_x, train_y, likelihood)
             mll = gpytorch.mlls.ExactMarginalLogLikelihood(likelihood, gp_model)
             gp_model.covar_module.base_kernel.initialize(log_lengthscale=1)
@@ -203,9 +193,7 @@ class TestSimpleGPRegression(unittest.TestCase):
         if torch.cuda.is_available():
             # We're manually going to set the hyperparameters to
             # something they shouldn't be
-            likelihood = GaussianLikelihood(
-                log_noise_prior=SmoothedBoxPrior(exp(-3), exp(3), sigma=0.1, log_transform=True)
-            ).cuda()
+            likelihood = GaussianLikelihood(noise_prior=SmoothedBoxPrior(exp(-3), exp(3), sigma=0.1)).cuda()
             gp_model = ExactGPModel(train_x.cuda(), train_y.cuda(), likelihood).cuda()
             mll = gpytorch.mlls.ExactMarginalLogLikelihood(likelihood, gp_model)
             gp_model.covar_module.base_kernel.initialize(log_lengthscale=1)
diff --git a/test/examples/test_spectral_mixture_gp_regression.py b/test/examples/test_spectral_mixture_gp_regression.py
index b65cf2bb..25b55bbf 100644
--- a/test/examples/test_spectral_mixture_gp_regression.py
+++ b/test/examples/test_spectral_mixture_gp_regression.py
@@ -73,7 +73,7 @@ class TestSpectralMixtureGPRegression(unittest.TestCase):
 
     def test_spectral_mixture_gp_mean_abs_error(self):
         likelihood = GaussianLikelihood(
-            log_noise_prior=SmoothedBoxPrior(exp(-5), exp(3), sigma=0.1, log_transform=True)
+            noise_prior=SmoothedBoxPrior(exp(-5), exp(3), sigma=0.1)
         )
         gp_model = SpectralMixtureGPModel(train_x, train_y, likelihood)
         mll = gpytorch.mlls.ExactMarginalLogLikelihood(likelihood, gp_model)
diff --git a/test/examples/test_svgp_gp_regression.py b/test/examples/test_svgp_gp_regression.py
index 863f9292..b8305afa 100644
--- a/test/examples/test_svgp_gp_regression.py
+++ b/test/examples/test_svgp_gp_regression.py
@@ -36,7 +36,7 @@ class SVGPRegressionModel(AbstractVariationalGP):
         self.mean_module = gpytorch.means.ConstantMean()
         self.covar_module = gpytorch.kernels.ScaleKernel(
             gpytorch.kernels.RBFKernel(
-                log_lengthscale_prior=gpytorch.priors.SmoothedBoxPrior(0.001, 1.0, sigma=0.1, log_transform=True)
+                lengthscale_prior=gpytorch.priors.SmoothedBoxPrior(0.001, 1.0, sigma=0.1)
             )
         )
 
diff --git a/test/examples/test_white_noise_regression.py b/test/examples/test_white_noise_regression.py
index 1c63aca5..527a74dc 100644
--- a/test/examples/test_white_noise_regression.py
+++ b/test/examples/test_white_noise_regression.py
@@ -30,9 +30,7 @@ class ExactGPModel(gpytorch.models.ExactGP):
     def __init__(self, train_inputs, train_targets, likelihood):
         super(ExactGPModel, self).__init__(train_inputs, train_targets, likelihood)
         self.mean_module = ConstantMean(prior=SmoothedBoxPrior(-1, 1))
-        self.rbf_covar_module = RBFKernel(
-            log_lengthscale_prior=SmoothedBoxPrior(exp(-3), exp(3), sigma=0.1, log_transform=True)
-        )
+        self.rbf_covar_module = RBFKernel(lengthscale_prior=SmoothedBoxPrior(exp(-3), exp(3), sigma=0.1))
         self.noise_covar_module = WhiteNoiseKernel(variances=torch.ones(11) * 0.001)
         self.covar_module = ScaleKernel(self.rbf_covar_module + self.noise_covar_module)
 
@@ -58,13 +56,11 @@ class TestSimpleGPRegression(unittest.TestCase):
     def test_posterior_latent_gp_and_likelihood_without_optimization(self):
         with gpytorch.settings.debug(False):
             # We're manually going to set the hyperparameters to be ridiculous
-            likelihood = GaussianLikelihood(
-                log_noise_prior=SmoothedBoxPrior(exp(-10), exp(10), sigma=0.25, log_transform=True)
-            )
+            likelihood = GaussianLikelihood(noise_prior=SmoothedBoxPrior(exp(-10), exp(10), sigma=0.25))
             gp_model = ExactGPModel(train_x, train_y, likelihood)
             # Update lengthscale prior to accommodate extreme parameters
-            gp_model.rbf_covar_module.set_parameter_priors(
-                log_lengthscale=SmoothedBoxPrior(exp(-10), exp(10), sigma=0.5, log_transform=True)
+            gp_model.rbf_covar_module.register_prior(
+                "log_lengthscale_prior", SmoothedBoxPrior(exp(-10), exp(10), sigma=0.5), "log_lengthscale"
             )
             gp_model.rbf_covar_module.initialize(log_lengthscale=-10)
             gp_model.mean_module.initialize(constant=0)
@@ -89,9 +85,7 @@ class TestSimpleGPRegression(unittest.TestCase):
 
     def test_posterior_latent_gp_and_likelihood_with_optimization(self):
         # We're manually going to set the hyperparameters to something they shouldn't be
-        likelihood = GaussianLikelihood(
-            log_noise_prior=SmoothedBoxPrior(exp(-3), exp(3), sigma=0.1, log_transform=True)
-        )
+        likelihood = GaussianLikelihood(noise_prior=SmoothedBoxPrior(exp(-3), exp(3), sigma=0.1))
         gp_model = ExactGPModel(train_x, train_y, likelihood)
         mll = gpytorch.ExactMarginalLogLikelihood(likelihood, gp_model)
         gp_model.rbf_covar_module.initialize(log_lengthscale=1)
@@ -132,9 +126,7 @@ class TestSimpleGPRegression(unittest.TestCase):
         with gpytorch.fast_pred_var(), gpytorch.settings.debug(False):
             # We're manually going to set the hyperparameters to
             # something they shouldn't be
-            likelihood = GaussianLikelihood(
-                log_noise_prior=SmoothedBoxPrior(exp(-3), exp(3), sigma=0.1, log_transform=True)
-            )
+            likelihood = GaussianLikelihood(noise_prior=SmoothedBoxPrior(exp(-3), exp(3), sigma=0.1))
             gp_model = ExactGPModel(train_x, train_y, likelihood)
             mll = gpytorch.mlls.ExactMarginalLogLikelihood(likelihood, gp_model)
             gp_model.rbf_covar_module.initialize(log_lengthscale=1)
@@ -182,9 +174,7 @@ class TestSimpleGPRegression(unittest.TestCase):
         if torch.cuda.is_available():
             # We're manually going to set the hyperparameters to
             # something they shouldn't be
-            likelihood = GaussianLikelihood(
-                log_noise_prior=SmoothedBoxPrior(exp(-3), exp(3), sigma=0.1, log_transform=True)
-            ).cuda()
+            likelihood = GaussianLikelihood(noise_prior=SmoothedBoxPrior(exp(-3), exp(3), sigma=0.1)).cuda()
             gp_model = ExactGPModel(train_x.cuda(), train_y.cuda(), likelihood).cuda()
             mll = gpytorch.mlls.ExactMarginalLogLikelihood(likelihood, gp_model)
             gp_model.rbf_covar_module.initialize(log_lengthscale=1)
diff --git a/test/priors/test_gamma_prior.py b/test/priors/test_gamma_prior.py
index 3d050441..20f1aadc 100644
--- a/test/priors/test_gamma_prior.py
+++ b/test/priors/test_gamma_prior.py
@@ -27,7 +27,6 @@ class TestGammaPrior(unittest.TestCase):
         prior = GammaPrior(concentration, rate)
         dist = Gamma(concentration, rate)
 
-        self.assertFalse(prior.log_transform)
         t = torch.tensor(1.0, device=device)
         self.assertTrue(torch.equal(prior.log_prob(t), dist.log_prob(t)))
         t = torch.tensor([1.5, 0.5], device=device)
@@ -43,10 +42,9 @@ class TestGammaPrior(unittest.TestCase):
         device = torch.device("cuda") if cuda else torch.device("cpu")
         concentration = torch.tensor(1.0, device=device)
         rate = torch.tensor(1.0, device=device)
-        prior = GammaPrior(concentration, rate, log_transform=True)
+        prior = GammaPrior(concentration, rate, transform=torch.exp)
         dist = Gamma(concentration, rate)
 
-        self.assertTrue(prior.log_transform)
         t = torch.tensor(0.0, device=device)
         self.assertTrue(torch.equal(prior.log_prob(t), dist.log_prob(t.exp())))
         t = torch.tensor([-1, 0.5], device=device)
@@ -56,7 +54,7 @@ class TestGammaPrior(unittest.TestCase):
 
     def test_gamma_prior_log_prob_log_transform_cuda(self):
         if torch.cuda.is_available():
-            return self.test_gamma_prior_log_prob(cuda=True)
+            return self.test_gamma_prior_log_prob_log_transform(cuda=True)
 
     def test_gamma_prior_batch_log_prob(self, cuda=False):
         device = torch.device("cuda") if cuda else torch.device("cpu")
diff --git a/test/priors/test_lkj_prior.py b/test/priors/test_lkj_prior.py
index 2fd86be6..23ca2356 100644
--- a/test/priors/test_lkj_prior.py
+++ b/test/priors/test_lkj_prior.py
@@ -26,7 +26,6 @@ class TestLKJPrior(unittest.TestCase):
         device = torch.device("cuda") if cuda else torch.device("cpu")
         prior = LKJPrior(2, torch.tensor(0.5, device=device))
 
-        self.assertFalse(prior.log_transform)
         S = torch.eye(2, device=device)
         self.assertAlmostEqual(prior.log_prob(S).item(), -1.86942, places=4)
         S = torch.stack([S, torch.tensor([[1.0, 0.5], [0.5, 1]], device=S.device)])
@@ -46,7 +45,6 @@ class TestLKJPrior(unittest.TestCase):
         device = torch.device("cuda") if cuda else torch.device("cpu")
         prior = LKJPrior(2, torch.tensor([0.5, 1.5], device=device))
 
-        self.assertFalse(prior.log_transform)
         S = torch.eye(2, device=device)
         self.assertTrue(approx_equal(prior.log_prob(S), torch.tensor([-1.86942, -0.483129], device=S.device)))
         S = torch.stack([S, torch.tensor([[1.0, 0.5], [0.5, 1]], device=S.device)])
@@ -76,7 +74,6 @@ class TestLKJCholeskyFactorPrior(unittest.TestCase):
     def test_lkj_cholesky_factor_prior_log_prob(self, cuda=False):
         device = torch.device("cuda") if cuda else torch.device("cpu")
         prior = LKJCholeskyFactorPrior(2, torch.tensor(0.5, device=device))
-        self.assertFalse(prior.log_transform)
         S = torch.eye(2, device=device)
         S_chol = torch.cholesky(S)
         self.assertAlmostEqual(prior.log_prob(S_chol).item(), -1.86942, places=4)
@@ -98,7 +95,6 @@ class TestLKJCholeskyFactorPrior(unittest.TestCase):
         device = torch.device("cuda") if cuda else torch.device("cpu")
         prior = LKJCholeskyFactorPrior(2, torch.tensor([0.5, 1.5], device=device))
 
-        self.assertFalse(prior.log_transform)
         S = torch.eye(2, device=device)
         S_chol = torch.cholesky(S)
         self.assertTrue(approx_equal(prior.log_prob(S_chol), torch.tensor([-1.86942, -0.483129], device=S_chol.device)))
@@ -116,14 +112,14 @@ class TestLKJCholeskyFactorPrior(unittest.TestCase):
 class TestLKJCovariancePrior(unittest.TestCase):
     def test_lkj_covariance_prior_to_gpu(self):
         if torch.cuda.is_available():
-            sd_prior = SmoothedBoxPrior(exp(-1), exp(1), log_transform=True)
+            sd_prior = SmoothedBoxPrior(exp(-1), exp(1))
             prior = LKJCovariancePrior(2, 1.0, sd_prior).cuda()
             self.assertEqual(prior.correlation_prior.eta.device.type, "cuda")
             self.assertEqual(prior.correlation_prior.C.device.type, "cuda")
             self.assertEqual(prior.sd_prior.a.device.type, "cuda")
 
     def test_lkj_covariance_prior_validate_args(self):
-        sd_prior = SmoothedBoxPrior(exp(-1), exp(1), log_transform=True, validate_args=True)
+        sd_prior = SmoothedBoxPrior(exp(-1), exp(1), validate_args=True)
         LKJCovariancePrior(2, 1.0, sd_prior)
         with self.assertRaises(ValueError):
             LKJCovariancePrior(1.5, 1.0, sd_prior, validate_args=True)
@@ -132,11 +128,10 @@ class TestLKJCovariancePrior(unittest.TestCase):
 
     def test_lkj_covariance_prior_log_prob(self, cuda=False):
         device = torch.device("cuda") if cuda else torch.device("cpu")
-        sd_prior = SmoothedBoxPrior(exp(-1), exp(1), log_transform=True)
+        sd_prior = SmoothedBoxPrior(exp(-1), exp(1))
         if cuda:
             sd_prior = sd_prior.cuda()
         prior = LKJCovariancePrior(2, torch.tensor(0.5, device=device), sd_prior)
-        self.assertFalse(prior.log_transform)
         S = torch.eye(2, device=device)
         self.assertAlmostEqual(prior.log_prob(S).item(), -3.59981, places=4)
         S = torch.stack([S, torch.tensor([[1.0, 0.5], [0.5, 1]], device=S.device)])
@@ -158,9 +153,8 @@ class TestLKJCovariancePrior(unittest.TestCase):
         device = torch.device("cuda") if cuda else torch.device("cpu")
         a = torch.tensor([exp(-1), exp(-2)], device=device)
         b = torch.tensor([exp(1), exp(2)], device=device)
-        sd_prior = SmoothedBoxPrior(a, b, log_transform=True)
+        sd_prior = SmoothedBoxPrior(a, b)
         prior = LKJCovariancePrior(2, torch.tensor(0.5, device=device), sd_prior)
-        self.assertFalse(prior.log_transform)
         S = torch.eye(2, device=device)
         self.assertAlmostEqual(prior.log_prob(S).item(), -4.71958, places=4)
         S = torch.stack([S, torch.tensor([[1.0, 0.5], [0.5, 1]], device=S.device)])
@@ -181,10 +175,9 @@ class TestLKJCovariancePrior(unittest.TestCase):
     def test_lkj_covariance_prior_batch_log_prob(self, cuda=False):
         device = torch.device("cuda") if cuda else torch.device("cpu")
         v = torch.ones(2, 1, device=device)
-        sd_prior = SmoothedBoxPrior(exp(-1) * v, exp(1) * v, log_transform=True)
+        sd_prior = SmoothedBoxPrior(exp(-1) * v, exp(1) * v)
         prior = LKJCovariancePrior(2, torch.tensor([0.5, 1.5], device=device), sd_prior)
 
-        self.assertFalse(prior.log_transform)
         S = torch.eye(2, device=device)
         self.assertTrue(approx_equal(prior.log_prob(S), torch.tensor([-3.59981, -2.21351], device=S.device)))
         S = torch.stack([S, torch.tensor([[1.0, 0.5], [0.5, 1]], device=S.device)])
diff --git a/test/priors/test_multivariate_normal_prior.py b/test/priors/test_multivariate_normal_prior.py
index 7932341b..76826568 100644
--- a/test/priors/test_multivariate_normal_prior.py
+++ b/test/priors/test_multivariate_normal_prior.py
@@ -16,15 +16,11 @@ class TestMultivariateNormalPrior(unittest.TestCase):
             self.assertEqual(prior.scale_tril.device.type, "cuda")
             self.assertEqual(prior.precision_matrix.device.type, "cuda")
 
-    # def test_multivariate_normal_prior_validate_args(self):
-    #     # TODO: It seems like the error raised here changed in a recent build of PyTorch.
-    #     # We can turn this error back on once that stabilizies.
-
-    #     # This should be a ValueError in pytorch, see pytorch Github issue #11997
-    #     with self.assertRaises(RuntimeError):
-    #         mean = torch.tensor([0.0, 1.0])
-    #         cov = torch.tensor([[1.0, 2.0], [2.0, 0.5]])
-    #         MultivariateNormalPrior(mean, covariance_matrix=cov, validate_args=True)
+    def test_multivariate_normal_prior_validate_args(self):
+        with self.assertRaises(ValueError):
+            mean = torch.tensor([0.0, 1.0])
+            cov = torch.tensor([[1.0, 2.0], [2.0, 0.5]])
+            MultivariateNormalPrior(mean, covariance_matrix=cov, validate_args=True)
 
     def test_multivariate_normal_prior_log_prob(self, cuda=False):
         device = torch.device("cuda") if cuda else torch.device("cpu")
@@ -33,7 +29,6 @@ class TestMultivariateNormalPrior(unittest.TestCase):
         prior = MultivariateNormalPrior(mean, covariance_matrix=cov)
         dist = MultivariateNormal(mean, covariance_matrix=cov)
 
-        self.assertFalse(prior.log_transform)
         t = torch.tensor([-1, 0.5], device=device)
         self.assertTrue(torch.equal(prior.log_prob(t), dist.log_prob(t)))
         t = torch.tensor([[-1, 0.5], [1.5, -2.0]], device=device)
@@ -49,10 +44,9 @@ class TestMultivariateNormalPrior(unittest.TestCase):
         device = torch.device("cuda") if cuda else torch.device("cpu")
         mean = torch.tensor([0.0, 1.0], device=device)
         cov = torch.eye(2, device=device)
-        prior = MultivariateNormalPrior(mean, covariance_matrix=cov, log_transform=True)
+        prior = MultivariateNormalPrior(mean, covariance_matrix=cov, transform=torch.exp)
         dist = MultivariateNormal(mean, covariance_matrix=cov)
 
-        self.assertTrue(prior.log_transform)
         t = torch.tensor([-1, 0.5], device=device)
         self.assertTrue(torch.equal(prior.log_prob(t), dist.log_prob(t.exp())))
         t = torch.tensor([[-1, 0.5], [1.5, -2.0]], device=device)
@@ -62,7 +56,7 @@ class TestMultivariateNormalPrior(unittest.TestCase):
 
     def test_multivariate_normal_prior_log_prob_log_transform_cuda(self):
         if torch.cuda.is_available():
-            return self.test_multivariate_normal_prior_log_prob(cuda=True)
+            return self.test_multivariate_normal_prior_log_prob_log_transform(cuda=True)
 
     def test_multivariate_normal_prior_batch_log_prob(self, cuda=False):
         device = torch.device("cuda") if cuda else torch.device("cpu")
@@ -72,7 +66,6 @@ class TestMultivariateNormalPrior(unittest.TestCase):
         prior = MultivariateNormalPrior(mean, covariance_matrix=cov)
         dist = MultivariateNormal(mean, covariance_matrix=cov)
 
-        self.assertFalse(prior.log_transform)
         t = torch.tensor([-1, 0.5], device=device)
         self.assertTrue(torch.equal(prior.log_prob(t), dist.log_prob(t)))
         t = torch.tensor([[-1, 0.5], [1.5, -2.0]], device=device)
diff --git a/test/priors/test_normal_prior.py b/test/priors/test_normal_prior.py
index bd1934aa..3af4cd1f 100644
--- a/test/priors/test_normal_prior.py
+++ b/test/priors/test_normal_prior.py
@@ -25,7 +25,6 @@ class TestNormalPrior(unittest.TestCase):
         prior = NormalPrior(mean, variance)
         dist = Normal(mean, variance)
 
-        self.assertFalse(prior.log_transform)
         t = torch.tensor(0.0, device=device)
         self.assertTrue(torch.equal(prior.log_prob(t), dist.log_prob(t)))
         t = torch.tensor([-1, 0.5], device=device)
@@ -41,10 +40,9 @@ class TestNormalPrior(unittest.TestCase):
         device = torch.device("cuda") if cuda else torch.device("cpu")
         mean = torch.tensor(0.0, device=device)
         variance = torch.tensor(1.0, device=device)
-        prior = NormalPrior(mean, variance, log_transform=True)
+        prior = NormalPrior(mean, variance, transform=torch.exp)
         dist = Normal(mean, variance)
 
-        self.assertTrue(prior.log_transform)
         t = torch.tensor(0.0, device=device)
         self.assertTrue(torch.equal(prior.log_prob(t), dist.log_prob(t.exp())))
         t = torch.tensor([-1, 0.5], device=device)
@@ -54,7 +52,7 @@ class TestNormalPrior(unittest.TestCase):
 
     def test_normal_prior_log_prob_log_transform_cuda(self):
         if torch.cuda.is_available():
-            return self.test_normal_prior_log_prob(cuda=True)
+            return self.test_normal_prior_log_prob_log_transform(cuda=True)
 
     def test_normal_prior_batch_log_prob(self, cuda=False):
         device = torch.device("cuda") if cuda else torch.device("cpu")
diff --git a/test/priors/test_smoothed_box_prior.py b/test/priors/test_smoothed_box_prior.py
index 3cc80e15..02ca0bc0 100644
--- a/test/priors/test_smoothed_box_prior.py
+++ b/test/priors/test_smoothed_box_prior.py
@@ -30,7 +30,6 @@ class TestSmoothedBoxPrior(unittest.TestCase):
         sigma = 0.1
         prior = SmoothedBoxPrior(a, b, sigma)
 
-        self.assertFalse(prior.log_transform)
         self.assertTrue(torch.equal(prior.a, a))
         self.assertTrue(torch.equal(prior.b, b))
         self.assertTrue(torch.equal(prior.sigma, torch.full_like(prior.a, sigma)))
@@ -52,7 +51,7 @@ class TestSmoothedBoxPrior(unittest.TestCase):
         device = torch.device("cuda") if cuda else torch.device("cpu")
         a, b = torch.zeros(2, device=device), torch.ones(2, device=device)
         sigma = 0.1
-        prior = SmoothedBoxPrior(a, b, sigma, log_transform=True)
+        prior = SmoothedBoxPrior(a, b, sigma, transform=torch.exp)
 
         t = torch.tensor([0.5, 1.1], device=device).log()
         self.assertAlmostEqual(prior.log_prob(t).item(), -0.9473, places=4)
@@ -64,7 +63,7 @@ class TestSmoothedBoxPrior(unittest.TestCase):
 
     def test_smoothed_box_prior_log_prob_log_transform_cuda(self):
         if torch.cuda.is_available():
-            return self.test_smoothed_box_prior_log_prob(cuda=True)
+            return self.test_smoothed_box_prior_log_prob_log_transform(cuda=True)
 
     def test_smoothed_box_prior_batch_log_prob(self, cuda=False):
         # TODO: Implement test for batch mode