Repository: cornellius-gp/gpytorch

Issue:
[Bug] Cast constraints to floating point
# 🐛 Bug

Using integers to construct constraints throws an overflow exception.

## To reproduce

```python
import torch
from gpytorch.constraints import Interval
from gpytorch.distributions import MultivariateNormal
from gpytorch.kernels import RBFKernel
from gpytorch.likelihoods import GaussianLikelihood
from gpytorch.means import ConstantMean
from gpytorch.mlls import ExactMarginalLogLikelihood
from gpytorch.models import ExactGP

X = torch.randn(100, 2)
y = torch.randn(100)


class GP(ExactGP):
    def __init__(self, X, y, likelihood, ls_constraint):
        super().__init__(X, y, likelihood)
        self.likelihood = likelihood
        self.mean_module = ConstantMean()
        self.covar_module = RBFKernel(lengthscale_constraint=ls_constraint)

    def forward(self, x):
        mean = self.mean_module(x)
        covar = self.covar_module(x)
        return MultivariateNormal(mean, covar)


likelihood = GaussianLikelihood()

ls_constraint = Interval(1, 2)

model = GP(X, y, likelihood, ls_constraint)
optim = torch.optim.Adam(model.parameters())
mll = ExactMarginalLogLikelihood(likelihood, model)

loss = mll(model(X), y)
```

** Stack trace/error message **
```
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-1-068518c8dae8> in <module>
     33 mll = ExactMarginalLogLikelihood(likelihood, model)
     34
---> 35 loss = mll(model(X), y)

~/github/gpytorch/gpytorch/module.py in __call__(self, *inputs, **kwargs)
     26
     27     def __call__(self, *inputs, **kwargs):
---> 28         outputs = self.forward(*inputs, **kwargs)
     29         if isinstance(outputs, list):
     30             return [_validate_module_outputs(output) for output in outputs]

~/github/gpytorch/gpytorch/mlls/exact_marginal_log_likelihood.py in forward(self, function_dist, target, *params)
     49         # Get the log prob of the marginal distribution
     50         output = self.likelihood(function_dist, *params)
---> 51         res = output.log_prob(target)
     52
     53         # Add additional terms (SGPR / learned inducing points, heteroskedastic likelihood models)

~/github/gpytorch/gpytorch/distributions/multivariate_normal.py in log_prob(self, value)
    133
    134         # Get log determininat and first part of quadratic form
--> 135         inv_quad, logdet = covar.inv_quad_logdet(inv_quad_rhs=diff.unsqueeze(-1), logdet=True)
    136
    137         res = -0.5 * sum([inv_quad, logdet, diff.size(-1) * math.log(2 * math.pi)])

~/github/gpytorch/gpytorch/lazy/lazy_tensor.py in inv_quad_logdet(self, inv_quad_rhs, logdet, reduce_inv_quad)
   1000             from .chol_lazy_tensor import CholLazyTensor
   1001
-> 1002             cholesky = CholLazyTensor(self.cholesky())
   1003             return cholesky.inv_quad_logdet(inv_quad_rhs=inv_quad_rhs, logdet=logdet, reduce_inv_quad=reduce_inv_quad)
   1004

~/github/gpytorch/gpytorch/lazy/lazy_tensor.py in cholesky(self, upper)
    737             (LazyTensor) Cholesky factor (lower triangular)
    738         """
--> 739         res = self._cholesky()
    740         if upper:
    741             res = res.transpose(-1, -2)

~/github/gpytorch/gpytorch/utils/memoize.py in g(self, *args, **kwargs)
     32         cache_name = name if name is not None else method
     33         if not is_in_cache(self, cache_name):
---> 34             add_to_cache(self, cache_name, method(self, *args, **kwargs))
     35         return get_from_cache(self, cache_name)
     36

~/github/gpytorch/gpytorch/lazy/lazy_tensor.py in _cholesky(self)
    400         from .keops_lazy_tensor import KeOpsLazyTensor
    401
--> 402         evaluated_kern_mat = self.evaluate_kernel()
    403
    404         if any(isinstance(sub_mat, KeOpsLazyTensor) for sub_mat in evaluated_kern_mat._args):

~/github/gpytorch/gpytorch/lazy/lazy_tensor.py in evaluate_kernel(self)
    883         all lazily evaluated kernels actually evaluated.
    884         """
--> 885         return self.representation_tree()(*self.representation())
    886
    887     def inv_matmul(self, right_tensor, left_tensor=None):

~/github/gpytorch/gpytorch/lazy/lazy_tensor.py in representation_tree(self)
   1273         including all subobjects. This is used internally.
   1274         """
-> 1275         return LazyTensorRepresentationTree(self)
   1276
   1277     @property

~/github/gpytorch/gpytorch/lazy/lazy_tensor_representation_tree.py in __init__(self, lazy_tsr)
     11         for arg in lazy_tsr._args:
     12             if hasattr(arg, "representation") and callable(arg.representation):  # Is it a lazy tensor?
---> 13                 representation_size = len(arg.representation())
     14                 self.children.append((slice(counter, counter + representation_size, None), arg.representation_tree()))
     15                 counter += representation_size

~/github/gpytorch/gpytorch/lazy/lazy_evaluated_kernel_tensor.py in representation(self)
    311         # representation
    312         else:
--> 313             return self.evaluate_kernel().representation()
    314
    315     def representation_tree(self):

~/github/gpytorch/gpytorch/utils/memoize.py in g(self, *args, **kwargs)
     32         cache_name = name if name is not None else method
     33         if not is_in_cache(self, cache_name):
---> 34             add_to_cache(self, cache_name, method(self, *args, **kwargs))
     35         return get_from_cache(self, cache_name)
     36

~/github/gpytorch/gpytorch/lazy/lazy_evaluated_kernel_tensor.py in evaluate_kernel(self)
    278             temp_active_dims = self.kernel.active_dims
    279             self.kernel.active_dims = None
--> 280             res = self.kernel(x1, x2, diag=False, last_dim_is_batch=self.last_dim_is_batch, **self.params)
    281             self.kernel.active_dims = temp_active_dims
    282

~/github/gpytorch/gpytorch/kernels/kernel.py in __call__(self, x1, x2, diag, last_dim_is_batch, **params)
    394                 res = LazyEvaluatedKernelTensor(x1_, x2_, kernel=self, last_dim_is_batch=last_dim_is_batch, **params)
    395             else:
--> 396                 res = lazify(super(Kernel, self).__call__(x1_, x2_, last_dim_is_batch=last_dim_is_batch, **params))
    397             return res
    398

~/github/gpytorch/gpytorch/module.py in __call__(self, *inputs, **kwargs)
     26
     27     def __call__(self, *inputs, **kwargs):
---> 28         outputs = self.forward(*inputs, **kwargs)
     29         if isinstance(outputs, list):
     30             return [_validate_module_outputs(output) for output in outputs]

~/github/gpytorch/gpytorch/kernels/rbf_kernel.py in forward(self, x1, x2, diag, **params)
     87             x1,
     88             x2,
---> 89             self.lengthscale,
     90             lambda x1, x2: self.covar_dist(
     91                 x1, x2, square_dist=True, diag=False, dist_postprocess_func=postprocess_rbf, postprocess=False, **params

~/github/gpytorch/gpytorch/kernels/kernel.py in lengthscale(self)
    237     def lengthscale(self):
    238         if self.has_lengthscale:
--> 239             return self.raw_lengthscale_constraint.transform(self.raw_lengthscale)
    240         else:
    241             return None

~/github/gpytorch/gpytorch/constraints/constraints.py in transform(self, tensor)
     92             min_bound = torch.min(self.lower_bound)
     93
---> 94             if max_bound == math.inf or min_bound == -math.inf:
     95                 raise RuntimeError(
     96                     "Cannot make an Interval directly with non-finite bounds. Use a derived class like "

~/anaconda3/lib/python3.7/site-packages/torch/tensor.py in wrapped(*args, **kwargs)
     26     def wrapped(*args, **kwargs):
     27         try:
---> 28             return f(*args, **kwargs)
     29         except TypeError:
     30             return NotImplemented

RuntimeError: value cannot be converted to type int64_t without overflow: inf
```

## Expected Behavior

While the bounds in constraints are strictly speaking not ints, it feels unnecessarily picky to error because the user used ```Interval(1, 2)``` rather than ```Interval(1., 2.)```. I've wasted time looking for genuine numerical problems with the inference, when this was in fact the only problem. Could we just silently cast bounds to floats when constraints are initialised?

## System information

**Please complete the following information:**
- GPyTorch Version 1.1.1
- PyTorch Version 1.6.0
- OSX Catalina


Discussion:
Yes, I agree this can easily just be fixed, and can see how this would lead to pointlessly difficult debugging. We should never have Parameters that are ints, so I agree that the obvious solution here is to just cast the bounds to floats upon initialization.

Resolving patch:
diff --git a/gpytorch/constraints/constraints.py b/gpytorch/constraints/constraints.py
--- a/gpytorch/constraints/constraints.py
+++ b/gpytorch/constraints/constraints.py
@@ -21,16 +21,16 @@ def __init__(self, lower_bound, upper_bound, transform=sigmoid, inv_transform=in
             lower_bound (float or torch.Tensor): The lower bound on the parameter.
             upper_bound (float or torch.Tensor): The upper bound on the parameter.
         """
-        lower_bound = torch.as_tensor(lower_bound)
-        upper_bound = torch.as_tensor(upper_bound)
+        lower_bound = torch.as_tensor(lower_bound).float()
+        upper_bound = torch.as_tensor(upper_bound).float()
 
         if torch.any(torch.ge(lower_bound, upper_bound)):
             raise RuntimeError("Got parameter bounds with empty intervals.")
 
         super().__init__()
 
-        self.lower_bound = lower_bound
-        self.upper_bound = upper_bound
+        self.register_buffer("lower_bound", lower_bound)
+        self.register_buffer("upper_bound", upper_bound)
 
         self._transform = transform
         self._inv_transform = inv_transform
@@ -44,6 +44,22 @@ def _apply(self, fn):
         self.upper_bound = fn(self.upper_bound)
         return super()._apply(fn)
 
+    def _load_from_state_dict(
+        self, state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs
+    ):
+        result = super()._load_from_state_dict(
+            state_dict=state_dict,
+            prefix=prefix,
+            local_metadata=local_metadata,
+            strict=False,
+            missing_keys=missing_keys,
+            unexpected_keys=unexpected_keys,
+            error_msgs=error_msgs,
+        )
+        # The lower_bound and upper_bound buffers are new, and so may not be present in older state dicts
+        # Because of this, we won't have strict-mode on when loading this module
+        return result
+
     @property
     def enforced(self):
         return self._transform is not None


Test patch: