Repository: arviz-devs/arviz

Issue:
Improve effective sample size docs
The [docs for `effective_sample_size`](https://arviz-devs.github.io/arviz/generated/arviz.effective_sample_size.html#arviz.effective_sample_size) have some typos and unexplained notation. 

This issue is a remainder to correct them once #623 is merged.



Discussion:

Resolving patch:
diff --git a/arviz/stats/diagnostics.py b/arviz/stats/diagnostics.py
--- a/arviz/stats/diagnostics.py
+++ b/arviz/stats/diagnostics.py
@@ -42,6 +42,17 @@ def bfmi(data):
     z : array
         The Bayesian fraction of missing information of the model and trace. One element per
         chain in the trace.
+
+    Examples
+    --------
+    Compute the BFMI of an InferenceData object
+
+    .. ipython::
+
+        In [1]: import arviz as az
+           ...: data = az.load_arviz_data('radon')
+           ...: az.bfmi(data)
+
     """
     if isinstance(data, np.ndarray):
         return _bfmi(data)
@@ -67,7 +78,8 @@ def effective_sample_size(data, *, var_names=None, method="bulk", relative=False
     var_names : list
         Names of variables to include in the effective_sample_size_mean report
     method : str
-        Select ess method. Valid methods are
+        Select ess method. Valid methods are:
+
         - "bulk"
         - "tail"     # prob, optional
         - "quantile" # prob
@@ -78,6 +90,7 @@ def effective_sample_size(data, *, var_names=None, method="bulk", relative=False
         - "z_scale"
         - "folded"
         - "identity"
+
     relative : bool
         Return relative ess
         `ress = ess / N`
@@ -105,9 +118,10 @@ def effective_sample_size(data, *, var_names=None, method="bulk", relative=False
 
     References
     ----------
-    Vehtari et al. (2019) see https://arxiv.org/abs/1903.08008
-    https://mc-stan.org/docs/2_18/reference-manual/effective-sample-size-section.html Section 15.4.2
-    Gelman et al. BDA (2014) Formula 11.8
+    * Vehtari et al. (2019) see https://arxiv.org/abs/1903.08008
+    * https://mc-stan.org/docs/2_18/reference-manual/effective-sample-size-section.html
+      Section 15.4.2
+    * Gelman et al. BDA (2014) Formula 11.8
     """
     warnings.warn(
         "Function `arviz.effective_sample_size` is deprecated. Use `arviz.ess`", DeprecationWarning
@@ -128,7 +142,8 @@ def ess(data, *, var_names=None, method="bulk", relative=False, prob=None):
     var_names : list
         Names of variables to include in the effective_sample_size_mean report
     method : str
-        Select ess method. Valid methods are
+        Select ess method. Valid methods are:
+
         - "bulk"
         - "tail"     # prob, optional
         - "quantile" # prob
@@ -139,36 +154,56 @@ def ess(data, *, var_names=None, method="bulk", relative=False, prob=None):
         - "z_scale"
         - "folded"
         - "identity"
+
     relative : bool
         Return relative ess
-        `ress = ess / N`
+        `ress = ess / n`
     prob : float, optional
         probability value for "tail" and "quantile" ess functions.
 
     Returns
     -------
     xarray.Dataset
-        Return the effective sample size for mean, :math:`\hat{N}_{eff}`
+        Return the effective sample size, :math:`\hat{N}_{eff}`
 
     Notes
     -----
     The basic ess diagnostic is computed by:
 
     .. math:: \hat{N}_{eff} = \frac{MN}{\hat{\tau}}
-    .. math:: \hat{\tau} = -1 + 2 \sum_{t'=0}^K \hat{P}_t'
+    .. math:: \hat{\tau} = -1 + 2 \sum_{t'=0}^K \hat{P}_{t'}
 
-    where :math:`\hat{\rho}_t` is the estimated _autocorrelation at lag t, and T
-    is the first odd positive integer for which the sum
-    :math:`\hat{\rho}_{T+1} + \hat{\rho}_{T+1}` is negative.
+    where :math:`M` is the number of chains, :math:`N` the number of draws,
+    :math:`\hat{\rho}_t` is the estimated _autocorrelation at lag :math:`t`, and
+    :math:`K` is the last integer for which :math:`\hat{P}_{K} = \hat{\rho}_{2K} +
+    \hat{\rho}_{2K+1}` is still positive.
 
     The current implementation is similar to Stan, which uses Geyer's initial monotone sequence
     criterion (Geyer, 1992; Geyer, 2011).
 
     References
     ----------
-    Vehtari et al. (2019) see https://arxiv.org/abs/1903.08008
-    https://mc-stan.org/docs/2_18/reference-manual/effective-sample-size-section.html Section 15.4.2
-    Gelman et al. BDA (2014) Formula 11.8
+    * Vehtari et al. (2019) see https://arxiv.org/abs/1903.08008
+    * https://mc-stan.org/docs/2_18/reference-manual/effective-sample-size-section.html
+      Section 15.4.2
+    * Gelman et al. BDA (2014) Formula 11.8
+
+    Examples
+    --------
+    Calculate the effective_sample_size using the default arguments:
+
+    .. ipython::
+
+        In [1]: import arviz as az
+           ...: data = az.load_arviz_data('non_centered_eight')
+           ...: az.ess(data)
+
+    Calculate the ress of some of the variables
+
+    .. ipython::
+
+        In [1]: az.ess(data, relative=True, var_names=["mu", "theta_t"])
+
     """
     methods = {
         "bulk": _ess_bulk,
@@ -239,7 +274,8 @@ def rhat(data, *, var_names=None, method="rank"):
     var_names : list
         Names of variables to include in the rhat report
     method : str
-        Select R-hat method. Valid methods are
+        Select R-hat method. Valid methods are:
+
         - "rank"        # recommended by Vehtari et al. (2019)
         - "split"
         - "folded"
@@ -267,10 +303,27 @@ def rhat(data, *, var_names=None, method="rank"):
 
     References
     ----------
-    Vehtari et al. (2019) see https://arxiv.org/abs/1903.08008
-    Gelman et al. BDA (2014)
-    Brooks and Gelman (1998)
-    Gelman and Rubin (1992)
+    * Vehtari et al. (2019) see https://arxiv.org/abs/1903.08008
+    * Gelman et al. BDA (2014)
+    * Brooks and Gelman (1998)
+    * Gelman and Rubin (1992)
+
+    Examples
+    --------
+    Calculate the R-hat using the default arguments:
+
+    .. ipython::
+
+        In [1]: import arviz as az
+           ...: data = az.load_arviz_data("non_centered_eight")
+           ...: az.rhat(data)
+
+    Calculate the R-hat of some variables using the folded method:
+
+    .. ipython::
+
+        In [1]: az.rhat(data, var_names=["mu", "theta_t"], method="folded")
+
     """
     methods = {
         "rank": _rhat_rank,
@@ -311,7 +364,7 @@ def rhat(data, *, var_names=None, method="rank"):
 
 
 def mcse(data, *, var_names=None, method="mean", prob=None):
-    """Calculate Markov Chain Standard Error for statistic.
+    """Calculate Markov Chain Standard Error statistic.
 
     Parameters
     ----------
@@ -323,10 +376,12 @@ def mcse(data, *, var_names=None, method="mean", prob=None):
     var_names : list
         Names of variables to include in the rhat report
     method : str
-        Select mcse method. Valid methods are
+        Select mcse method. Valid methods are:
+
         - "mean"
         - "sd"
         - "quantile"
+
     prob : float
         Quantile information.
 
@@ -334,6 +389,23 @@ def mcse(data, *, var_names=None, method="mean", prob=None):
     -------
     xarray.Dataset
         Return the msce dataset
+
+    Examples
+    --------
+    Calculate the Markov Chain Standard Error using the default arguments:
+
+    .. ipython::
+
+        In [1]: import arviz as az
+           ...: data = az.load_arviz_data("non_centered_eight")
+           ...: az.mcse(data)
+
+    Calculate the Markov Chain Standard Error using the quantile method:
+
+    .. ipython::
+
+        In [1]: az.mcse(data, method="quantile", prob=.7)
+
     """
     methods = {"mean": _mcse_mean, "sd": _mcse_sd, "quantile": _mcse_quantile}
     if method not in methods:
@@ -410,7 +482,7 @@ def geweke(ary, first=0.1, last=0.5, intervals=20):
 
     References
     ----------
-    Geweke (1992)
+    * Geweke (1992)
     """
     # Filter out invalid intervals
     for interval in (first, last):
diff --git a/arviz/stats/stats.py b/arviz/stats/stats.py
--- a/arviz/stats/stats.py
+++ b/arviz/stats/stats.py
@@ -29,9 +29,9 @@ def compare(
     seed=None,
     scale="deviance",
 ):
-    r"""Compare models based on WAIC or LOO cross validation.
+    r"""Compare models based on WAIC or LOO cross-validation.
 
-    WAIC is Widely applicable information criterion, and LOO is leave-one-out
+    WAIC is the widely applicable information criterion, and LOO is leave-one-out
     (LOO) cross-validation. Read more theory here - in a paper by some of the
     leading authorities on model selection - dx.doi.org/10.1111/1467-9868.00353
 
@@ -46,9 +46,9 @@ def compare(
 
         - 'stacking' : stacking of predictive distributions.
         - 'BB-pseudo-BMA' : (default) pseudo-Bayesian Model averaging using Akaike-type
-           weighting. The weights are stabilized using the Bayesian bootstrap
+          weighting. The weights are stabilized using the Bayesian bootstrap.
         - 'pseudo-BMA': pseudo-Bayesian Model averaging using Akaike-type
-           weighting, without Bootstrap stabilization (not recommended)
+          weighting, without Bootstrap stabilization (not recommended).
 
         For more information read https://arxiv.org/abs/1704.02030
     b_samples: int
@@ -71,14 +71,13 @@ def compare(
 
     Returns
     -------
-    A DataFrame, ordered from lowest to highest IC. The index reflects the order in which the
+    A DataFrame, ordered from lowest to highest IC. The index reflects the key with which the
     models are passed to this function. The columns are:
     IC : Information Criteria (WAIC or LOO).
         Smaller IC indicates higher out-of-sample predictive fit ("better" model). Default WAIC.
         If `scale == log` higher IC indicates higher out-of-sample predictive fit ("better" model).
     pIC : Estimated effective number of parameters.
-    dIC : Relative difference between each IC (WAIC or LOO)
-    and the lowest IC (WAIC or LOO).
+    dIC : Relative difference between each IC (WAIC or LOO) and the lowest IC (WAIC or LOO).
         It's always 0 for the top-ranked model.
     weight: Relative weight for each model.
         This can be loosely interpreted as the probability of each model (among the compared model)
@@ -86,12 +85,32 @@ def compare(
         Bayesian bootstrap.
     SE : Standard error of the IC estimate.
         If method = BB-pseudo-BMA these values are estimated using Bayesian bootstrap.
-    dSE : Standard error of the difference in IC between each model and
-    the top-ranked model.
+    dSE : Standard error of the difference in IC between each model and the top-ranked model.
         It's always 0 for the top-ranked model.
-    warning : A value of 1 indicates that the computation of the IC may not be reliable. This could
-        be indication of WAIC/LOO starting to fail see http://arxiv.org/abs/1507.04544 for details.
+    warning : A value of 1 indicates that the computation of the IC may not be reliable.
+        This could be indication of WAIC/LOO starting to fail see
+        http://arxiv.org/abs/1507.04544 for details.
     scale : Scale used for the IC.
+
+    Examples
+    --------
+    Compare the centered and non centered models of the eight school problem:
+
+    .. ipython::
+
+        In [1]: import arviz as az
+           ...: data1 = az.load_arviz_data("non_centered_eight")
+           ...: data2 = az.load_arviz_data("centered_eight")
+           ...: compare_dict = {"non centered": data1, "centered": data2}
+           ...: az.compare(compare_dict)
+
+    Compare the models using LOO-CV, returning the IC in log scale and calculating the
+    weights using the stacking method.
+
+    .. ipython::
+
+        In [1]: az.compare(compare_dict, ic="loo", method="stacking", scale="log")
+
     """
     names = list(dataset_dict.keys())
     scale = scale.lower()
@@ -105,6 +124,7 @@ def compare(
             scale_value = -2
         ascending = True
 
+    ic = ic.lower()
     if ic == "waic":
         ic_func = waic
         df_comp = pd.DataFrame(
@@ -264,6 +284,17 @@ def hpd(ary, credible_interval=0.94, circular=False):
     -------
     np.ndarray
         lower and upper value of the interval.
+
+    Examples
+    --------
+    Calculate the hpd of a Normal random variable:
+
+    .. ipython::
+
+        In [1]: import arviz as az
+           ...: import numpy as np
+           ...: data = np.random.normal(size=2000)
+           ...: az.hpd(data, credible_interval=.68)
     """
     if ary.ndim > 1:
         hpd_array = np.array(
@@ -312,8 +343,10 @@ def loo(data, pointwise=False, reff=None, scale="deviance"):
 
     Parameters
     ----------
-    data : result of MCMC run
-    pointwise: bool, optional
+    data : obj
+        Any object that can be converted to an az.InferenceData object. Refer to documentation
+        of az.convert_to_inference_data for details
+    pointwise : bool, optional
         if True the pointwise predictive accuracy will be returned. Defaults to False
     reff : float, optional
         Relative MCMC efficiency, `ess / n` i.e. number of effective samples divided by
@@ -328,14 +361,26 @@ def loo(data, pointwise=False, reff=None, scale="deviance"):
     Returns
     -------
     pandas.Series with the following columns:
-    loo: approximated Leave-one-out cross-validation
-    loo_se: standard error of loo
-    p_loo: effective number of parameters
-    shape_warn: 1 if the estimated shape parameter of
+    loo : approximated Leave-one-out cross-validation
+    loo_se : standard error of loo
+    p_loo : effective number of parameters
+    shape_warn : bool
+        True if the estimated shape parameter of
         Pareto distribution is greater than 0.7 for one or more samples
-    loo_i: array of pointwise predictive accuracy, only if pointwise True
-    pareto_k: array of Pareto shape values, only if pointwise True
-    loo_scale: scale of the loo results
+    loo_i : array of pointwise predictive accuracy, only if pointwise True
+    pareto_k : array of Pareto shape values, only if pointwise True
+    loo_scale : scale of the loo results
+
+    Examples
+    --------
+    Calculate the LOO-CV of a model:
+
+    .. ipython::
+
+        In [1]: import arviz as az
+           ...: data = az.load_arviz_data("centered_eight")
+           ...: az.loo(data, pointwise=True)
+
     """
     inference_data = convert_to_inference_data(data)
     for group in ("posterior", "sample_stats"):
@@ -374,16 +419,16 @@ def loo(data, pointwise=False, reff=None, scale="deviance"):
     log_weights, pareto_shape = psislw(-log_likelihood, reff)
     log_weights += log_likelihood
 
-    warn_mg = 0
+    warn_mg = False
     if np.any(pareto_shape > 0.7):
         warnings.warn(
-            """Estimated shape parameter of Pareto distribution is greater than 0.7 for
-        one or more samples. You should consider using a more robust model, this is because
-        importance sampling is less likely to work well if the marginal posterior and LOO posterior
-        are very different. This is more likely to happen with a non-robust model and highly
-        influential observations."""
+            "Estimated shape parameter of Pareto distribution is greater than 0.7 for "
+            "one or more samples. You should consider using a more robust model, this is because "
+            "importance sampling is less likely to work well if the marginal posterior and "
+            "LOO posterior are very different. This is more likely to happen with a non-robust "
+            "model and highly influential observations."
         )
-        warn_mg = 1
+        warn_mg = True
 
     loo_lppd_i = scale_value * _logsumexp(log_weights, axis=0)
     loo_lppd = loo_lppd_i.sum()
@@ -564,7 +609,7 @@ def r2_score(y_true, y_pred):
 
     Parameters
     ----------
-    y_true: : array-like of shape = (n_samples) or (n_samples, n_outputs)
+    y_true : array-like of shape = (n_samples) or (n_samples, n_outputs)
         Ground truth (correct) target values.
     y_pred : array-like of shape = (n_samples) or (n_samples, n_outputs)
         Estimated target values.
@@ -643,29 +688,37 @@ def summary(
 
     Examples
     --------
-    .. code:: ipython
+    .. ipython::
 
-        >>> az.summary(trace, ['mu'])
-               mean    sd  hpd_3  hpd_97  ess_bulk  ess_tail   r_hat
-        mu[0]  0.10  0.06  -0.02    0.23     487.0              1.00
-        mu[1] -0.04  0.06   0.00   -0.17     379.0              1.00
+        In [1]: import arviz as az
+           ...: data = az.load_arviz_data("centered_eight")
+           ...: az.summary(data, var_names=["mu", "tau"])
 
     Other statistics can be calculated by passing a list of functions
     or a dictionary with key, function pairs.
 
-    .. code:: ipython
-
-        >>> import pandas as pd
-        >>> def trace_sd(x):
-        ...     return pd.Series(np.std(x, 0), name='sd')
-        ...
-        >>> def trace_quantiles(x):
-        ...     return pd.DataFrame(pd.quantiles(x, [5, 50, 95]))
-        ...
-        >>> az.summary(trace, ['mu'], stat_funcs=[trace_sd, trace_quantiles], extend=False)
-                 sd     5    50    95
-        mu[0]  0.06  0.00  0.10  0.21
-        mu[1]  0.07 -0.16 -0.04  0.06
+    .. ipython::
+
+        In [1]: import numpy as np
+           ...: def median_sd(x):
+           ...:     median = np.percentile(x, 50)
+           ...:     sd = np.sqrt(np.mean((x-median)**2))
+           ...:     return sd
+           ...:
+           ...: func_dict = {
+           ...:     "std": np.std,
+           ...:     "median_std": median_sd,
+           ...:     "5%": lambda x: np.percentile(x, 5),
+           ...:     "median": lambda x: np.percentile(x, 50),
+           ...:     "95%": lambda x: np.percentile(x, 95),
+           ...: }
+           ...: az.summary(
+           ...:     data,
+           ...:     var_names=["mu", "tau"],
+           ...:     stat_funcs=func_dict,
+           ...:     extend=False
+           ...: )
+
     """
     posterior = convert_to_dataset(data, group="posterior")
     var_names = _var_names(var_names, posterior)
@@ -865,14 +918,14 @@ def waic(data, pointwise=False, scale="deviance"):
     Also calculates the WAIC's standard error and the effective number of
     parameters of the samples in trace from model. Read more theory here - in
     a paper by some of the leading authorities on model selection
-    dx.doi.org/10.1111/1467-9868.00353
+    <dx.doi.org/10.1111/1467-9868.00353>
 
     Parameters
     ----------
     data : obj
         Any object that can be converted to an az.InferenceData object
-        Refer to documentation of az.convert_to_dataset for details
-    pointwise: bool
+        Refer to documentation of az.convert_to_inference_data for details
+    pointwise : bool
         if True the pointwise predictive accuracy will be returned.
         Default False
     scale : str
@@ -885,13 +938,25 @@ def waic(data, pointwise=False, scale="deviance"):
     Returns
     -------
     DataFrame with the following columns:
-    waic: widely available information criterion
-    waic_se: standard error of waic
-    p_waic: effective number parameters
-    var_warn: 1 if posterior variance of the log predictive
-         densities exceeds 0.4
-    waic_i: and array of the pointwise predictive accuracy, only if pointwise True
-    waic_scale: scale of the waic results
+    waic : widely available information criterion
+    waic_se : standard error of waic
+    p_waic : effective number parameters
+    var_warn : bool
+        True if posterior variance of the log predictive
+        densities exceeds 0.4
+    waic_i : and array of the pointwise predictive accuracy, only if pointwise True
+    waic_scale : scale of the waic results
+
+    Examples
+    --------
+    Calculate the LOO-CV of a model:
+
+    .. ipython::
+
+        In [1]: import arviz as az
+           ...: data = az.load_arviz_data("centered_eight")
+           ...: az.waic(data, pointwise=True)
+
     """
     inference_data = convert_to_inference_data(data)
     for group in ("sample_stats",):
@@ -919,7 +984,7 @@ def waic(data, pointwise=False, scale="deviance"):
     lppd_i = _logsumexp(log_likelihood, axis=0, b_inv=log_likelihood.shape[0])
 
     vars_lpd = np.var(log_likelihood, axis=0)
-    warn_mg = 0
+    warn_mg = False
     if np.any(vars_lpd > 0.4):
         warnings.warn(
             """For one or more samples the posterior variance of the log predictive
@@ -927,7 +992,7 @@ def waic(data, pointwise=False, scale="deviance"):
         http://arxiv.org/abs/1507.04544 for details
         """
         )
-        warn_mg = 1
+        warn_mg = True
 
     waic_i = scale_value * (lppd_i - vars_lpd)
     waic_se = (len(waic_i) * np.var(waic_i)) ** 0.5
diff --git a/doc/conf.py b/doc/conf.py
--- a/doc/conf.py
+++ b/doc/conf.py
@@ -48,6 +48,7 @@
     "matplotlib.sphinxext.plot_directive",
     "numpydoc",
     "nbsphinx",
+    "IPython.sphinxext.ipython_directive",
     "IPython.sphinxext.ipython_console_highlighting",
     "gallery_generator",
 ]


Test patch: