Repository: cornellius-gp/gpytorch

Issue:
[Docs] Settings defaults mismatched in source code and documentation
# 📚 Documentation/Examples
Separating out from #1526 

The defaults shown in the settings don't consistently match the source code.

### For example:  
settings.memory_efficient is [True in the documentation](https://docs.gpytorch.ai/en/stable/settings.html#gpytorch.settings.memory_efficient), but [False in the corresponding source code](https://docs.gpytorch.ai/en/stable/_modules/gpytorch/settings.html#memory_efficient).



Discussion:

Resolving patch:
diff --git a/gpytorch/settings.py b/gpytorch/settings.py
--- a/gpytorch/settings.py
+++ b/gpytorch/settings.py
@@ -183,6 +183,8 @@ class skip_posterior_variances(_feature_flag):
     forward pass. If this is on, the returned gpytorch MultivariateNormal will have a
     ZeroLazyTensor as its covariance matrix. This allows gpytorch to not compute
     the covariance matrix when it is not needed, speeding up computations.
+
+    (Default: False)
     """
 
     _default = False
@@ -194,6 +196,8 @@ class detach_test_caches(_feature_flag):
     as this will speed up derivative computations of the predictions with respect to test inputs. However,
     if you also need derivatives with respect to training inputs (e.g., because you have fantasy observations),
     then you must disable this.
+
+    (Default: True)
     """
 
     _default = True
@@ -207,6 +211,8 @@ class deterministic_probes(_feature_flag):
 
     NOTE: Currently, probe vectors are cached in a global scope. Therefore, this setting cannot be used
     if multiple independent GP models are being trained in the same context (i.e., it works fine with a single GP model)
+
+    (Default: False)
     """
 
     probe_vectors = None
@@ -223,6 +229,8 @@ class debug(_feature_flag):
     (For example, that the correct training data is supplied in Exact GP training mode)
     Pros: fewer data checks, fewer warning messages
     Cons: possibility of supplying incorrect data, model accidentially in wrong mode
+
+    (Default: True)
     """
 
     _default = True
@@ -240,6 +248,8 @@ class fast_pred_var(_feature_flag):
     See also: :class:`gpytorch.settings.max_root_decomposition_size` (to control the
     size of the low rank decomposition used for variance estimates).
 
+    (Default: False)
+
     .. _`Constant-Time Predictive Distributions for Gaussian Processes`:
         https://arxiv.org/pdf/1803.06058.pdf
     """
@@ -280,6 +290,8 @@ class fast_pred_samples(_feature_flag):
     See also: :class:`gpytorch.settings.max_root_decomposition_size` (to control the
     size of the low rank decomposition used for samples).
 
+    (Default: False)
+
     .. _`Constant-Time Predictive Distributions for Gaussian Processes`:
         https://arxiv.org/pdf/1803.06058.pdf
     """
@@ -373,6 +385,8 @@ class lazily_evaluate_kernels(_feature_flag):
 
     If set to False, gpytorch will always compute the entire covariance matrix between
     training and test data.
+
+    (Default: True)
     """
 
     _default = True
@@ -382,7 +396,8 @@ class max_eager_kernel_size(_value_context):
     """
     If the joint train/test covariance matrix is less than this size, then we will avoid as
     much lazy evaluation of the kernel as possible.
-    Default: 512
+
+    (Default: 512)
     """
 
     _global_value = 512
@@ -392,7 +407,8 @@ class max_cg_iterations(_value_context):
     """
     The maximum number of conjugate gradient iterations to perform (when computing
     matrix solves). A higher value rarely results in more accurate solves -- instead, lower the CG tolerance.
-    Default: 1000
+
+    (Default: 1000)
     """
 
     _global_value = 1000
@@ -400,8 +416,12 @@ class max_cg_iterations(_value_context):
 
 class min_variance(_dtype_value_context):
     """
-    The minimum variance that can be returned from :obj:`~gpytorch.distributions.MultivariateNormal#variance`
-    (default 1e-6). If variances are smaller than this, they are rounded up and a warning is raised.
+    The minimum variance that can be returned from :obj:`~gpytorch.distributions.MultivariateNormal#variance`.
+    If variances are smaller than this, they are rounded up and a warning is raised.
+
+    - Default for `float`: 1e-6
+    - Default for `double`: 1e-10
+    - Default for `half`: 1e-3
     """
 
     _global_float_value = 1e-6
@@ -412,7 +432,9 @@ class min_variance(_dtype_value_context):
 class cholesky_jitter(_dtype_value_context):
     """
     The jitter value passed to `psd_safe_cholesky` when using cholesky solves.
-    Default: None
+
+    - Default for `float`: 1e-6
+    - Default for `double`: 1e-8
     """
 
     _global_float_value = 1e-6
@@ -434,7 +456,7 @@ class cg_tolerance(_value_context):
     """
     Relative residual tolerance to use for terminating CG.
 
-    Default: 1
+    (Default: 1)
     """
 
     _global_value = 1
@@ -442,7 +464,18 @@ class cg_tolerance(_value_context):
 
 class ciq_samples(_feature_flag):
     """
-    Whether to draw samples using CIQ or not
+    Whether to draw samples using Contour Integral Quadrature or not.
+    This may be slower than standard sampling methods for `N < 5000`.
+    However, it should be faster with larger matrices.
+
+    As described in the paper:
+
+    `Fast Matrix Square Roots with Applications to Gaussian Processes and Bayesian Optimization`_.
+
+    (Default: False)
+
+    .. _`Fast Matrix Square Roots with Applications to Gaussian Processes and Bayesian Optimization`:
+        https://arxiv.org/abs/2006.11267
     """
 
     _default = False
@@ -452,7 +485,7 @@ class preconditioner_tolerance(_value_context):
     """
     Diagonal trace tolerance to use for checking preconditioner convergence.
 
-    Default: 1e-3
+    (Default: 1e-3)
     """
 
     _global_value = 1e-3
@@ -462,7 +495,7 @@ class eval_cg_tolerance(_value_context):
     """
     Relative residual tolerance to use for terminating CG when making predictions.
 
-    Default: 0.01
+    (Default: 1e-2)
     """
 
     _global_value = 0.01
@@ -476,7 +509,8 @@ class max_cholesky_size(_value_context):
     """
     If the size of of a LazyTensor is less than `max_cholesky_size`,
     then `root_decomposition` and `inv_matmul` of LazyTensor will use Cholesky rather than Lanczos/CG.
-    Default: 800
+
+    (Default: 800)
     """
 
     _global_value = 800
@@ -488,7 +522,8 @@ class max_root_decomposition_size(_value_context):
     This is used when 1) computing variance estiamtes 2) when drawing from MVNs,
     or 3) for kernel multiplication
     More values results in higher accuracy
-    Default: 100
+
+    (Default: 100)
     """
 
     _global_value = 100
@@ -498,7 +533,8 @@ class max_preconditioner_size(_value_context):
     """
     The maximum size of preconditioner to use. 0 corresponds to turning
     preconditioning off. When enabled, usually a value of around ~10 works fairly well.
-    Default: 15
+
+    (Default: 15)
     """
 
     _global_value = 15
@@ -509,7 +545,10 @@ class max_lanczos_quadrature_iterations(_value_context):
     The maximum number of Lanczos iterations to perform when doing stochastic
     Lanczos quadrature. This is ONLY used for log determinant calculations and
     computing Tr(K^{-1}dK/d\theta)
+
+    (Default: 20)
     """
+
     _global_value = 20
 
 
@@ -518,6 +557,8 @@ class memory_efficient(_feature_flag):
     Whether or not to use Toeplitz math with gridded data, grid inducing point modules
     Pros: memory efficient, faster on CPU
     Cons: slower on GPUs with < 10000 inducing points
+
+    (Default: False)
     """
 
     _default = False
@@ -528,7 +569,7 @@ class min_preconditioning_size(_value_context):
     If the size of of a LazyTensor is less than `min_preconditioning_size`,
     then we won't use pivoted Cholesky based preconditioning.
 
-    Default: 2000
+    (Default: 2000)
     """
 
     _global_value = 2000
@@ -538,7 +579,7 @@ class minres_tolerance(_value_context):
     """
     Relative update term tolerance to use for terminating MINRES.
 
-    Default: 1e-4
+    (Default: 1e-4)
     """
 
     _global_value = 1e-4
@@ -547,7 +588,8 @@ class minres_tolerance(_value_context):
 class num_contour_quadrature(_value_context):
     """
     The number of quadrature points to compute CIQ.
-    Default: 15
+
+    (Default: 15)
     """
 
     _global_value = 15
@@ -557,7 +599,8 @@ class num_likelihood_samples(_value_context):
     """
     The number of samples to draw from a latent GP when computing a likelihood
     This is used in variational inference and training
-    Default: 10
+
+    (Default: 10)
     """
 
     _global_value = 10
@@ -567,7 +610,8 @@ class num_gauss_hermite_locs(_value_context):
     """
     The number of samples to draw from a latent GP when computing a likelihood
     This is used in variational inference and training
-    Default: 10
+
+    (Default: 20)
     """
 
     _global_value = 20
@@ -578,7 +622,8 @@ class num_trace_samples(_value_context):
     The number of samples to draw when stochastically computing the trace of a matrix
     More values results in more accurate trace estimations
     If the value is set to 0, then the trace will be deterministically computed
-    Default: 10
+
+    (Default: 10)
     """
 
     _global_value = 10
@@ -588,6 +633,8 @@ class prior_mode(_feature_flag):
     """
     If set to true, GP models will be evaluated in prior mode.
     This allows evaluating any Exact GP model in prior mode, even it if has training data / targets.
+
+    (Default: False)
     """
 
     _default = False
@@ -614,6 +661,8 @@ class skip_logdet_forward(_feature_flag):
     If you're using SGD (or a varient) to optimize parameters, you probably
     don't need an accurate MLL estimate; you only need accurate gradients. So
     this setting may give your model a performance boost.
+
+    (Default: False)
     """
 
     _default = False
@@ -622,6 +671,8 @@ class skip_logdet_forward(_feature_flag):
 class terminate_cg_by_size(_feature_flag):
     """
     If set to true, cg will terminate after n iterations for an n x n matrix.
+
+    (Default: False)
     """
 
     _default = False
@@ -638,6 +689,8 @@ class trace_mode(_feature_flag):
 
     Our hope is that this flag will not be necessary long term, once https://github.com/pytorch/pytorch/issues/22329
     is fixed.
+
+    (Default: False)
     """
 
     _default = False
@@ -649,6 +702,8 @@ class tridiagonal_jitter(_value_context):
     eigendecomposing. root_decomposition becomes slightly more stable with this, as we need
     to take the square root of the eigenvalues. Any eigenvalues still negative after adding jitter
     will be zeroed out.
+
+    (Default: 1e-6)
     """
 
     _global_value = 1e-6
@@ -659,6 +714,8 @@ class use_toeplitz(_feature_flag):
     Whether or not to use Toeplitz math with gridded data, grid inducing point modules
     Pros: memory efficient, faster on CPU
     Cons: slower on GPUs with < 10000 inducing points
+
+    (Default: True)
     """
 
     _default = True
@@ -667,6 +724,8 @@ class use_toeplitz(_feature_flag):
 class verbose_linalg(_feature_flag):
     """
     Print out information whenever running an expesnive linear algebra routine (e.g. Cholesky, CG, Lanczos, CIQ, etc.)
+
+    (Default: False)
     """
 
     _default = False


Test patch: