Repository: cornellius-gp/gpytorch

Issue:
Logit transform and fixed noise gaussian likelihood
Dear gpytorch developers,

This a probably basic question rather than a documentation/examples question. I removed the template in order to acomodate my question better. I hope it is fine

 I have an issue using logit transformed data and a fixed noise gaussian likelihood:

**My setting**

I have a set of training observations `train_x` and `train_y` , and I have known noise for the `train_y`.  I know the observation noise associated to to the training inputs, and the noise is a bit heterocedastic. Thus, I am using a FixedNoiseGaussianLikelihood with a vector  `my_noise` specifing the input noise. I simply implement this as follows:

```
likelihood = gpytorch.likelihoods.FixedNoiseGaussianLikelihood(noise=my_noise)
model = ExactGPMode(train_x,train_y,likelihood)
```

The rest of the model is pretty much the same as the Spectral mixture kernel notebook

**The problem**

My `train_y` data lies in the interval (0,0.5], in and has a lot of probability density close to 0 and 0.5. In order to account for this, I logit transform `train_y` before to feeding it to the gpytorch engine. However, due to the non-linear transformation of the logit, I cannot directly pass the noise vector as:
'
$logit(y+noise) != logit(y) + logit(noise)$

Is there a way to deal with this inside the gpytorch engine?


Having said that, thanks a lot for this great library. Makes implementing gaussian processes very easy to people!

Best,

Inigo


Discussion:

Patch:
diff --git a/gpytorch/likelihoods/__init__.py b/gpytorch/likelihoods/__init__.py
index ccee934d..5f364474 100644
--- a/gpytorch/likelihoods/__init__.py
+++ b/gpytorch/likelihoods/__init__.py
@@ -1,7 +1,9 @@
 #!/usr/bin/env python3
 
 from .bernoulli_likelihood import BernoulliLikelihood
+from .beta_likelihood import BetaLikelihood
 from .gaussian_likelihood import FixedNoiseGaussianLikelihood, GaussianLikelihood, _GaussianLikelihoodBase
+from .laplace_likelihood import LaplaceLikelihood
 from .likelihood import Likelihood, _OneDimensionalLikelihood
 from .likelihood_list import LikelihoodList
 from .multitask_gaussian_likelihood import (
@@ -11,18 +13,22 @@ from .multitask_gaussian_likelihood import (
 )
 from .noise_models import HeteroskedasticNoise
 from .softmax_likelihood import SoftmaxLikelihood
+from .student_t_likelihood import StudentTLikelihood
 
 __all__ = [
     "_GaussianLikelihoodBase",
     "_OneDimensionalLikelihood",
     "_MultitaskGaussianLikelihoodBase",
     "BernoulliLikelihood",
+    "BetaLikelihood",
     "FixedNoiseGaussianLikelihood",
     "GaussianLikelihood",
     "HeteroskedasticNoise",
+    "LaplaceLikelihood",
     "Likelihood",
     "LikelihoodList",
     "MultitaskGaussianLikelihood",
     "MultitaskGaussianLikelihoodKronecker",
     "SoftmaxLikelihood",
+    "StudentTLikelihood",
 ]
diff --git a/gpytorch/likelihoods/beta_likelihood.py b/gpytorch/likelihoods/beta_likelihood.py
new file mode 100644
index 00000000..e3a97483
--- /dev/null
+++ b/gpytorch/likelihoods/beta_likelihood.py
@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+
+import torch
+
+from ..constraints import Positive
+from ..distributions import base_distributions
+from .likelihood import _OneDimensionalLikelihood
+
+
+class BetaLikelihood(_OneDimensionalLikelihood):
+    r"""
+    A Beta likelihood for regressing over percentages.
+
+    The Beta distribution is parameterized by :math:`\alpha > 0` and :math:`\beta > 0` parameters
+    which roughly correspond to the number of prior positive and negative observations.
+    We instead parameterize it through a mixture :math:`m \in [0, 1]` and scale :math:`s > 0` parameter.
+
+    .. math::
+        \begin{equation*}
+            \alpha = ms, \quad \beta = (1-m)s
+        \end{equation*}
+
+    The mixture parameter is the output of the GP passed through a logit function.
+    The scale parameter is learned.
+
+    :var torch.Tensor scale: :math:`s` parameter (scale)
+    """
+
+    def __init__(self, batch_shape=torch.Size([]), scale_prior=None, scale_constraint=None):
+        super().__init__()
+
+        if scale_constraint is None:
+            scale_constraint = Positive()
+
+        self.raw_scale = torch.nn.Parameter(torch.ones(*batch_shape, 1))
+        if scale_prior is not None:
+            self.register_prior("scale_prior", scale_prior, lambda: self.scale, lambda v: self._set_scale(v))
+
+        self.register_constraint("raw_scale", scale_constraint)
+
+    @property
+    def scale(self):
+        return self.raw_scale_constraint.transform(self.raw_scale)
+
+    @scale.setter
+    def scale(self, value):
+        self._set_scale(value)
+
+    def _set_scale(self, value):
+        if not torch.is_tensor(value):
+            value = torch.as_tensor(value).to(self.raw_scale)
+        self.initialize(raw_scale=self.raw_scale_constraint.inverse_transform(value))
+
+    def forward(self, function_samples, **kwargs):
+        mixture = torch.sigmoid(function_samples)
+        scale = self.scale
+        alpha = mixture * scale + 1
+        beta = scale - alpha + 2
+        return base_distributions.Beta(concentration1=alpha, concentration0=beta)
diff --git a/gpytorch/likelihoods/laplace_likelihood.py b/gpytorch/likelihoods/laplace_likelihood.py
new file mode 100644
index 00000000..3a7736cd
--- /dev/null
+++ b/gpytorch/likelihoods/laplace_likelihood.py
@@ -0,0 +1,45 @@
+#!/usr/bin/env python3
+
+import torch
+
+from ..constraints import Positive
+from ..distributions import base_distributions
+from .likelihood import _OneDimensionalLikelihood
+
+
+class LaplaceLikelihood(_OneDimensionalLikelihood):
+    r"""
+    A Laplace likelihood/noise model for GP regression.
+    It has one learnable parameter: :math:`\sigma^2` - the noise
+
+    :var torch.Tensor noise: :math:`\sigma^2` parameter (noise)
+    """
+
+    def __init__(self, batch_shape=torch.Size([]), noise_prior=None, noise_constraint=None):
+        super().__init__()
+
+        if noise_constraint is None:
+            noise_constraint = Positive()
+
+        self.raw_noise = torch.nn.Parameter(torch.zeros(*batch_shape, 1))
+
+        if noise_prior is not None:
+            self.register_prior("noise_prior", noise_prior, lambda: self.noise, lambda v: self._set_noise(v))
+
+        self.register_constraint("raw_noise", noise_constraint)
+
+    @property
+    def noise(self):
+        return self.raw_noise_constraint.transform(self.raw_noise)
+
+    @noise.setter
+    def noise(self, value):
+        self._set_noise(value)
+
+    def _set_noise(self, value):
+        if not torch.is_tensor(value):
+            value = torch.as_tensor(value).to(self.raw_noise)
+        self.initialize(raw_noise=self.raw_noise_constraint.inverse_transform(value))
+
+    def forward(self, function_samples, **kwargs):
+        return base_distributions.Laplace(loc=function_samples, scale=self.noise.sqrt())
diff --git a/gpytorch/likelihoods/student_t_likelihood.py b/gpytorch/likelihoods/student_t_likelihood.py
new file mode 100644
index 00000000..2b402141
--- /dev/null
+++ b/gpytorch/likelihoods/student_t_likelihood.py
@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+
+import torch
+
+from ..constraints import GreaterThan, Positive
+from ..distributions import base_distributions
+from .likelihood import _OneDimensionalLikelihood
+
+
+class StudentTLikelihood(_OneDimensionalLikelihood):
+    r"""
+    A Student T likelihood/noise model for GP regression.
+    It has two learnable parameters: :math:`\nu` - the degrees of freedom, and
+    :math:`\sigma^2` - the noise
+
+    :var torch.Tensor deg_free: :math:`\nu` parameter (degrees of freedom)
+    :var torch.Tensor noise: :math:`\sigma^2` parameter (noise)
+    """
+
+    def __init__(
+        self,
+        batch_shape: torch.Size = torch.Size([]),
+        deg_free_prior=None,
+        deg_free_constraint=None,
+        noise_prior=None,
+        noise_constraint=None,
+    ):
+        super().__init__()
+
+        if deg_free_constraint is None:
+            deg_free_constraint = GreaterThan(2)
+
+        if noise_constraint is None:
+            noise_constraint = Positive()
+
+        self.raw_deg_free = torch.nn.Parameter(torch.zeros(*batch_shape, 1))
+        self.raw_noise = torch.nn.Parameter(torch.zeros(*batch_shape, 1))
+
+        if noise_prior is not None:
+            self.register_prior("noise_prior", noise_prior, lambda: self.noise, lambda v: self._set_noise(v))
+
+        self.register_constraint("raw_noise", noise_constraint)
+
+        if deg_free_prior is not None:
+            self.register_prior(
+                "deg_free_prior", deg_free_prior, lambda: self.deg_free, lambda v: self._set_deg_free(v)
+            )
+
+        self.register_constraint("raw_deg_free", deg_free_constraint)
+
+        # Rough initialization
+        self.initialize(deg_free=7)
+
+    @property
+    def deg_free(self):
+        return self.raw_deg_free_constraint.transform(self.raw_deg_free)
+
+    @deg_free.setter
+    def deg_free(self, value):
+        self._set_deg_free(value)
+
+    def _set_deg_free(self, value):
+        if not torch.is_tensor(value):
+            value = torch.as_tensor(value).to(self.raw_deg_free)
+        self.initialize(raw_deg_free=self.raw_deg_free_constraint.inverse_transform(value))
+
+    @property
+    def noise(self):
+        return self.raw_noise_constraint.transform(self.raw_noise)
+
+    @noise.setter
+    def noise(self, value):
+        self._set_noise(value)
+
+    def _set_noise(self, value):
+        if not torch.is_tensor(value):
+            value = torch.as_tensor(value).to(self.raw_noise)
+        self.initialize(raw_noise=self.raw_noise_constraint.inverse_transform(value))
+
+    def forward(self, function_samples, **kwargs):
+        return base_distributions.StudentT(df=self.deg_free, loc=function_samples, scale=self.noise.sqrt())


Test patch:
diff --git a/test/likelihoods/test_beta_likelihood.py b/test/likelihoods/test_beta_likelihood.py
new file mode 100644
index 00000000..cae4a93e
--- /dev/null
+++ b/test/likelihoods/test_beta_likelihood.py
@@ -0,0 +1,45 @@
+#!/usr/bin/env python3
+
+import unittest
+
+import torch
+
+from gpytorch.likelihoods import BetaLikelihood, _OneDimensionalLikelihood
+from gpytorch.test.base_likelihood_test_case import BaseLikelihoodTestCase
+
+
+class TestBetaLikelihood(BaseLikelihoodTestCase, unittest.TestCase):
+    seed = 1
+
+    def _create_targets(self, batch_shape=torch.Size([])):
+        res = torch.sigmoid(torch.randn(*batch_shape, 5)).float()
+        return res
+
+    def create_likelihood(self, **kwargs):
+        return BetaLikelihood(**kwargs)
+
+    def _test_log_marginal(self, batch_shape):
+        likelihood = self.create_likelihood()
+        input = self._create_marginal_input(batch_shape)
+        target = self._create_targets(batch_shape)
+        output = likelihood.log_marginal(target, input)
+
+        self.assertTrue(torch.is_tensor(output))
+        self.assertEqual(output.shape, batch_shape + torch.Size([5]))
+        default_log_prob = _OneDimensionalLikelihood.log_marginal(likelihood, target, input)
+        self.assertAllClose(output.sum(-1), default_log_prob.sum(-1), rtol=0.25, atol=0.1)
+
+    def _test_log_prob(self, batch_shape):
+        likelihood = self.create_likelihood()
+        input = self._create_marginal_input(batch_shape)
+        target = self._create_targets(batch_shape)
+        output = likelihood.expected_log_prob(target, input)
+
+        self.assertTrue(torch.is_tensor(output))
+        self.assertEqual(output.shape, batch_shape + torch.Size([5]))
+        default_log_prob = _OneDimensionalLikelihood.expected_log_prob(likelihood, target, input)
+        self.assertAllClose(output.sum(-1), default_log_prob.sum(-1), rtol=0.25, atol=0.1)
+
+    def _test_marginal(self, batch_shape):
+        # Likelihood uses the default marginal behavior, no point testing a set of samples against a set of samples.
+        pass
diff --git a/test/likelihoods/test_laplace_likelihood.py b/test/likelihoods/test_laplace_likelihood.py
new file mode 100644
index 00000000..ff8974ce
--- /dev/null
+++ b/test/likelihoods/test_laplace_likelihood.py
@@ -0,0 +1,45 @@
+#!/usr/bin/env python3
+
+import unittest
+
+import torch
+
+from gpytorch.likelihoods import LaplaceLikelihood, _OneDimensionalLikelihood
+from gpytorch.test.base_likelihood_test_case import BaseLikelihoodTestCase
+
+
+class TestLaplaceLikelihood(BaseLikelihoodTestCase, unittest.TestCase):
+    seed = 1
+
+    def _create_targets(self, batch_shape=torch.Size([])):
+        res = torch.sigmoid(torch.randn(*batch_shape, 5)).float()
+        return res
+
+    def create_likelihood(self, **kwargs):
+        return LaplaceLikelihood(**kwargs)
+
+    def _test_log_marginal(self, batch_shape):
+        likelihood = self.create_likelihood()
+        input = self._create_marginal_input(batch_shape)
+        target = self._create_targets(batch_shape)
+        output = likelihood.log_marginal(target, input)
+
+        self.assertTrue(torch.is_tensor(output))
+        self.assertEqual(output.shape, batch_shape + torch.Size([5]))
+        default_log_prob = _OneDimensionalLikelihood.log_marginal(likelihood, target, input)
+        self.assertAllClose(output.sum(-1), default_log_prob.sum(-1), rtol=0.25, atol=0.1)
+
+    def _test_log_prob(self, batch_shape):
+        likelihood = self.create_likelihood()
+        input = self._create_marginal_input(batch_shape)
+        target = self._create_targets(batch_shape)
+        output = likelihood.expected_log_prob(target, input)
+
+        self.assertTrue(torch.is_tensor(output))
+        self.assertEqual(output.shape, batch_shape + torch.Size([5]))
+        default_log_prob = _OneDimensionalLikelihood.expected_log_prob(likelihood, target, input)
+        self.assertAllClose(output.sum(-1), default_log_prob.sum(-1), rtol=0.25, atol=0.1)
+
+    def _test_marginal(self, batch_shape):
+        # Likelihood uses the default marginal behavior, no point testing a set of samples against a set of samples.
+        pass
diff --git a/test/likelihoods/test_student_t_likelihood.py b/test/likelihoods/test_student_t_likelihood.py
new file mode 100644
index 00000000..41e5929e
--- /dev/null
+++ b/test/likelihoods/test_student_t_likelihood.py
@@ -0,0 +1,45 @@
+#!/usr/bin/env python3
+
+import unittest
+
+import torch
+
+from gpytorch.likelihoods import StudentTLikelihood, _OneDimensionalLikelihood
+from gpytorch.test.base_likelihood_test_case import BaseLikelihoodTestCase
+
+
+class TestStudentTLikelihood(BaseLikelihoodTestCase, unittest.TestCase):
+    seed = 1
+
+    def _create_targets(self, batch_shape=torch.Size([])):
+        res = torch.sigmoid(torch.randn(*batch_shape, 5)).float()
+        return res
+
+    def create_likelihood(self, **kwargs):
+        return StudentTLikelihood(**kwargs)
+
+    def _test_log_marginal(self, batch_shape):
+        likelihood = self.create_likelihood()
+        input = self._create_marginal_input(batch_shape)
+        target = self._create_targets(batch_shape)
+        output = likelihood.log_marginal(target, input)
+
+        self.assertTrue(torch.is_tensor(output))
+        self.assertEqual(output.shape, batch_shape + torch.Size([5]))
+        default_log_prob = _OneDimensionalLikelihood.log_marginal(likelihood, target, input)
+        self.assertAllClose(output.sum(-1), default_log_prob.sum(-1), rtol=0.25, atol=0.1)
+
+    def _test_log_prob(self, batch_shape):
+        likelihood = self.create_likelihood()
+        input = self._create_marginal_input(batch_shape)
+        target = self._create_targets(batch_shape)
+        output = likelihood.expected_log_prob(target, input)
+
+        self.assertTrue(torch.is_tensor(output))
+        self.assertEqual(output.shape, batch_shape + torch.Size([5]))
+        default_log_prob = _OneDimensionalLikelihood.expected_log_prob(likelihood, target, input)
+        self.assertAllClose(output.sum(-1), default_log_prob.sum(-1), rtol=0.25, atol=0.1)
+
+    def _test_marginal(self, batch_shape):
+        # Likelihood uses the default marginal behavior, no point testing a set of samples against a set of samples.
+        pass