Repository: scikit-hep/pyhf

Issue:
clean up name/signature of `runOnePoint`
# Description

The signauture of `runOnePoint` is too complicated for newcomers. mostly people are only interested in CLs obs/exp. 

### Describe the solution you'd like

We could use the appraoch used in scikit of a signature that is *iteratively discoverable*

sucht that 

```
clsobs = pyhf.utils.cls(...)
clsobs, clsexp = pyhf.utils.cls(..., return_exp = True) #returns only two numbers
clsobs, clsexpset = pyhf.utils.cls(..., return_expset = True) #returns full 'brazil-band'
clsobs, clsexpset, qvals = pyhf.utils.cls(..., return_expset = True, return_teststatistics = True) #returns full 'brazil-band'  + test statistic values  
```



Discussion:
I very much like this idea.

Resolving patch:
diff --git a/pyhf/commandline.py b/pyhf/commandline.py
--- a/pyhf/commandline.py
+++ b/pyhf/commandline.py
@@ -8,7 +8,7 @@
 
 from . import readxml
 from . import writexml
-from .utils import runOnePoint
+from .utils import hypotest
 from .pdf import Model
 from .version import __version__
 
@@ -111,9 +111,9 @@ def cls(workspace, output_file, measurement, qualify_names, patch):
             qualify_names=qualify_names,
         )
         observed = sum((d['data'][c] for c in p.config.channels), []) + p.config.auxdata
-        result = runOnePoint(1.0, observed, p)
+        result = hypotest(1.0, observed, p, return_expected_set=True)
         result = {
-            'CLs_obs': result[-2].tolist()[0],
+            'CLs_obs': result[0].tolist()[0],
             'CLs_exp': result[-1].ravel().tolist(),
         }
         if output_file is None:
diff --git a/pyhf/utils.py b/pyhf/utils.py
--- a/pyhf/utils.py
+++ b/pyhf/utils.py
@@ -67,10 +67,13 @@ def qmu(mu, data, pdf, init_pars, par_bounds):
     Args:
         mu (Number or Tensor): The signal strength parameter
         data (Tensor): The data to be considered
-        pdf (Tensor): The model used in the likelihood ratio calculation
+        pdf (|pyhf.pdf.Model|_): The HistFactory statistical model used in the likelihood ratio calculation
         init_pars (Tensor): The initial parameters
         par_bounds(Tensor): The bounds on the paramter values
 
+    .. |pyhf.pdf.Model| replace:: ``pyhf.pdf.Model``
+    .. _pyhf.pdf.Model: https://diana-hep.org/pyhf/_generated/pyhf.pdf.Model.html
+
     Returns:
         Float: The calculated test statistic, :math:`q_{\mu}`
     """
@@ -96,8 +99,7 @@ def generate_asimov_data(asimov_mu, data, pdf, init_pars, par_bounds):
 
 def pvals_from_teststat(sqrtqmu_v, sqrtqmuA_v):
     r"""
-    The :math:`p`-values for signal strength :math:`\mu` and Asimov strength :math:`\mu'`
-    as defined in Equations (59) and (57) of `arXiv:1007.1727`_
+    The :math:`p`-values for signal strength :math:`\mu` and Asimov strength :math:`\mu'` as defined in Equations (59) and (57) of `arXiv:1007.1727`_
 
     .. _`arXiv:1007.1727`: https://arxiv.org/abs/1007.1727
 
@@ -127,27 +129,74 @@ def pvals_from_teststat(sqrtqmu_v, sqrtqmuA_v):
     return CLsb, CLb, CLs
 
 
-def runOnePoint(muTest, data, pdf, init_pars=None, par_bounds=None):
+def hypotest(poi_test, data, pdf, init_pars=None, par_bounds=None, **kwargs):
     r"""
-    Computes test statistics (and expected statistics) for a single value
-    of the parameter of interest
+    Computes :math:`p`-values and test statistics for a single value of the parameter of interest
 
     Args:
-        muTest (Number or Tensor): The value of the parameter of interest (POI)
+        poi_test (Number or Tensor): The value of the parameter of interest (POI)
         data (Number or Tensor): The root of the calculated test statistic given the Asimov data, :math:`\sqrt{q_{\mu,A}}`
-        init_pars (Array or Tensor): the initial parameter values to be used for minimization
-        par_bounds (Array or Tensor): the parameter value bounds to be used for minimization
+        pdf (|pyhf.pdf.Model|_): The HistFactory statistical model
+        init_pars (Array or Tensor): The initial parameter values to be used for minimization
+        par_bounds (Array or Tensor): The parameter value bounds to be used for minimization
+
+    .. |pyhf.pdf.Model| replace:: ``pyhf.pdf.Model``
+    .. _pyhf.pdf.Model: https://diana-hep.org/pyhf/_generated/pyhf.pdf.Model.html
+
+    Keyword Args:
+        return_tail_probs (bool): Bool for returning :math:`\textrm{CL}_{s+b}` and :math:`\textrm{CL}_{b}`
+        return_expected (bool): Bool for returning :math:`\textrm{CL}_{\textrm{exp}}`
+        return_expected_set (bool): Bool for returning the :math:`(-2,-1,0,1,2)\sigma` :math:`\textrm{CL}_{\textrm{exp}}` --- the "Brazil band"
+        return_test_statistics (bool): Bool for returning :math:`q_{\mu}` and :math:`q_{\mu,A}`
 
     Returns:
-        Tuple of Floats: a tuple containing (qmu, qmu_A, CLsb, CLb, CLs, CLs_exp)
-                         where qmu and qmu_A are the test statistics for the
-                         observed and Asimov datasets respectively.
-                         CLsb, CLb are the signal + background and background-only p-values
-                         CLs is the modified p-value
-                         CLs_exp is a 5-tuple of expected CLs values at percentiles
-                         of the background-only test-statistics corresponding to
-                         percentiles of the normal distribution for
-                         (-2,-1,0,1,2) :math:`\sigma`
+        Tuple of Floats and lists of Floats:
+
+            - :math:`\textrm{CL}_{s}`: The :math:`p`-value compared to the given threshold :math:`\alpha`, typically taken to be :math:`0.05`, defined in `arXiv:1007.1727`_ as
+
+            .. _`arXiv:1007.1727`: https://arxiv.org/abs/1007.1727
+
+            .. math::
+
+                \textrm{CL}_{s} = \frac{\textrm{CL}_{s+b}}{\textrm{CL}_{b}} = \frac{p_{s+b}}{1-p_{b}}
+
+            to protect against excluding signal models in which there is little sensitivity. In the case that :math:`\textrm{CL}_{s} \leq \alpha` the given signal model is excluded.
+
+            - :math:`\left[\textrm{CL}_{s+b}, \textrm{CL}_{b}\right]`: The signal + background :math:`p`-value and 1 minus the background only :math:`p`-value as defined in Equations (75) and (76) of `arXiv:1007.1727`_
+
+            .. math::
+
+                \textrm{CL}_{s+b} = p_{s+b} = \int\limits_{q_{\textrm{obs}}}^{\infty} f\left(q\,\middle|s+b\right)\,dq = 1 - \Phi\left(\frac{q_{\textrm{obs}} + 1/\sigma_{s+b}^{2}}{2/\sigma_{s+b}}\right)
+
+            .. math::
+
+                \textrm{CL}_{b} = 1- p_{b} = 1 - \int\limits_{-\infty}^{q_{\textrm{obs}}} f\left(q\,\middle|b\right)\,dq = 1 - \Phi\left(\frac{q_{\textrm{obs}} - 1/\sigma_{b}^{2}}{2/\sigma_{b}}\right)
+
+            with Equations (73) and (74) for the mean
+
+            .. math::
+
+                E\left[q\right] = \frac{1 - 2\mu}{\sigma^{2}}
+
+            and variance
+
+            .. math::
+
+                V\left[q\right] = \frac{4}{\sigma^{2}}
+
+            of the test statistic :math:`q` under the background only and and signal + background hypotheses. Only returned when ``return_tail_probs`` is ``True``.
+
+            - :math:`\textrm{CL}_{s,\textrm{exp}}`: The expected :math:`\textrm{CL}_{s}` value corresponding to the test statistic under the background only hypothesis :math:`\left(\mu=0\right)`. Only returned when ``return_expected`` is ``True``.
+
+            - :math:`\textrm{CL}_{s,\textrm{exp}}` band: The set of expected :math:`\textrm{CL}_{s}` values corresponding to the median significance of variations of the signal strength from the background only hypothesis :math:`\left(\mu=0\right)` at :math:`(-2,-1,0,1,2)\sigma`. That is, the :math:`p`-values that satisfy Equation (89) of `arXiv:1007.1727`_
+
+            .. math::
+
+                \textrm{band}_{N\sigma} = \mu' + \sigma\,\Phi^{-1}\left(1-\alpha\right) \pm N\sigma
+
+            for :math:`\mu'=0` and :math:`N \in \left\{-2, -1, 0, 1, 2\right\}`. These values define the boundaries of an uncertainty band sometimes referred to as the "Brazil band". Only returned when ``return_expected_set`` is ``True``.
+
+            - :math:`\left[q_{\mu}, q_{\mu,A}\right]`: The test statistics for the observed and Asimov datasets respectively. Only returned when ``return_test_statistics`` is ``True``.
     """
 
     init_pars = init_pars or pdf.config.suggested_init()
@@ -157,19 +206,32 @@ def runOnePoint(muTest, data, pdf, init_pars=None, par_bounds=None):
     asimov_mu = 0.0
     asimov_data = generate_asimov_data(asimov_mu, data, pdf, init_pars, par_bounds)
 
-    qmu_v = tensorlib.clip(qmu(muTest, data, pdf, init_pars, par_bounds), 0, max=None)
+    qmu_v = tensorlib.clip(qmu(poi_test, data, pdf, init_pars, par_bounds), 0, max=None)
     sqrtqmu_v = tensorlib.sqrt(qmu_v)
 
     qmuA_v = tensorlib.clip(
-        qmu(muTest, asimov_data, pdf, init_pars, par_bounds), 0, max=None
+        qmu(poi_test, asimov_data, pdf, init_pars, par_bounds), 0, max=None
     )
     sqrtqmuA_v = tensorlib.sqrt(qmuA_v)
 
     CLsb, CLb, CLs = pvals_from_teststat(sqrtqmu_v, sqrtqmuA_v)
 
-    CLs_exp = []
-    for nsigma in [-2, -1, 0, 1, 2]:
-        sqrtqmu_v_sigma = sqrtqmuA_v - nsigma
-        CLs_exp.append(pvals_from_teststat(sqrtqmu_v_sigma, sqrtqmuA_v)[-1])
-    CLs_exp = tensorlib.astensor(CLs_exp)
-    return qmu_v, qmuA_v, CLsb, CLb, CLs, CLs_exp
+    _returns = [CLs]
+    if kwargs.get('return_tail_probs'):
+        _returns.append([CLsb, CLb])
+    if kwargs.get('return_expected_set'):
+        CLs_exp = []
+        for n_sigma in [-2, -1, 0, 1, 2]:
+            sqrtqmu_v_sigma = sqrtqmuA_v - n_sigma
+            CLs_exp.append(pvals_from_teststat(sqrtqmu_v_sigma, sqrtqmuA_v)[-1])
+        CLs_exp = tensorlib.astensor(CLs_exp)
+        if kwargs.get('return_expected'):
+            _returns.append(CLs_exp[2])
+        _returns.append(CLs_exp)
+    elif kwargs.get('return_expected'):
+        _returns.append(pvals_from_teststat(sqrtqmuA_v, sqrtqmuA_v)[-1])
+    if kwargs.get('return_test_statistics'):
+        _returns.append([qmu_v, qmuA_v])
+
+    # Enforce a consistent return type of the observed CLs
+    return tuple(_returns) if len(_returns) > 1 else _returns[0]


Test patch:
diff --git a/tests/benchmarks/test_benchmark.py b/tests/benchmarks/test_benchmark.py
--- a/tests/benchmarks/test_benchmark.py
+++ b/tests/benchmarks/test_benchmark.py
@@ -52,9 +52,17 @@ def generate_source_poisson(n_bins):
     return source
 
 
-def runOnePoint(pdf, data):
-    return pyhf.utils.runOnePoint(
-        1.0, data, pdf, pdf.config.suggested_init(), pdf.config.suggested_bounds()
+def hypotest(pdf, data):
+    return pyhf.utils.hypotest(
+        1.0,
+        data,
+        pdf,
+        pdf.config.suggested_init(),
+        pdf.config.suggested_bounds(),
+        return_tail_probs=True,
+        return_expected=True,
+        return_expected_set=True,
+        return_test_statistics=True,
     )
 
 
@@ -65,9 +73,9 @@ def runOnePoint(pdf, data):
 
 @pytest.mark.parametrize('n_bins', bins, ids=bin_ids)
 @pytest.mark.skip_mxnet
-def test_runOnePoint(benchmark, backend, n_bins):
+def test_hypotest(benchmark, backend, n_bins):
     """
-    Benchmark the performance of pyhf.runOnePoint()
+    Benchmark the performance of pyhf.utils.hypotest()
     for various numbers of bins and different backends
 
     Args:
@@ -83,4 +91,4 @@ def test_runOnePoint(benchmark, backend, n_bins):
         source['bindata']['sig'], source['bindata']['bkg'], source['bindata']['bkgerr']
     )
     data = source['bindata']['data'] + pdf.config.auxdata
-    assert benchmark(runOnePoint, pdf, data)
+    assert benchmark(hypotest, pdf, data)
diff --git a/tests/test_backend_consistency.py b/tests/test_backend_consistency.py
--- a/tests/test_backend_consistency.py
+++ b/tests/test_backend_consistency.py
@@ -59,7 +59,7 @@ def generate_source_poisson(n_bins):
 
 @pytest.mark.parametrize('n_bins', bins, ids=bin_ids)
 @pytest.mark.parametrize('invert_order', [False, True], ids=['normal', 'inverted'])
-def test_runOnePoint_q_mu(
+def test_hypotest_q_mu(
     n_bins, invert_order, tolerance={'numpy': 1e-02, 'tensors': 5e-03}
 ):
     """
@@ -118,9 +118,14 @@ def test_runOnePoint_q_mu(
             backend.session = tf.Session()
         pyhf.set_backend(backend)
 
-        q_mu = pyhf.utils.runOnePoint(
-            1.0, data, pdf, pdf.config.suggested_init(), pdf.config.suggested_bounds()
-        )[0]
+        q_mu = pyhf.utils.hypotest(
+            1.0,
+            data,
+            pdf,
+            pdf.config.suggested_init(),
+            pdf.config.suggested_bounds(),
+            return_test_statistics=True,
+        )[-1][0]
         test_statistic.append(pyhf.tensorlib.tolist(q_mu))
 
     # compare to NumPy/SciPy
diff --git a/tests/test_utils.py b/tests/test_utils.py
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -1,6 +1,9 @@
-import pyhf
-import pytest
 import os
+import pytest
+
+import pyhf
+import pyhf.simplemodels
+import pyhf.utils
 
 
 def test_get_default_schema():
@@ -20,3 +23,114 @@ def test_load_custom_schema(tmpdir):
     temp = tmpdir.join("custom_schema.json")
     temp.write('{"foo": "bar"}')
     assert pyhf.utils.load_schema(temp.strpath)
+
+
+@pytest.fixture(scope='module')
+def hypotest_args():
+    pdf = pyhf.simplemodels.hepdata_like(
+        signal_data=[12.0, 11.0], bkg_data=[50.0, 52.0], bkg_uncerts=[3.0, 7.0]
+    )
+    mu_test = 1.0
+    data = [51, 48] + pdf.config.auxdata
+    return mu_test, data, pdf
+
+
+def check_uniform_type(in_list):
+    return all(
+        [isinstance(item, type(pyhf.tensorlib.astensor(item))) for item in in_list]
+    )
+
+
+def test_hypotest_default(tmpdir, hypotest_args):
+    """
+    Check that the default return structure of pyhf.utils.hypotest is as expected
+    """
+    tb = pyhf.tensorlib
+
+    kwargs = {}
+    result = pyhf.utils.hypotest(*hypotest_args, **kwargs)
+    # CLs_obs
+    assert len(list(result)) == 1
+    assert isinstance(result, type(tb.astensor(result)))
+
+
+def test_hypotest_return_tail_probs(tmpdir, hypotest_args):
+    """
+    Check that the return structure of pyhf.utils.hypotest with the
+    return_tail_probs keyword arg is as expected
+    """
+    tb = pyhf.tensorlib
+
+    kwargs = {'return_tail_probs': True}
+    result = pyhf.utils.hypotest(*hypotest_args, **kwargs)
+    # CLs_obs, [CL_sb, CL_b]
+    assert len(list(result)) == 2
+    assert isinstance(result[0], type(tb.astensor(result[0])))
+    assert len(result[1]) == 2
+    assert check_uniform_type(result[1])
+
+
+def test_hypotest_return_expected(tmpdir, hypotest_args):
+    """
+    Check that the return structure of pyhf.utils.hypotest with the
+    additon of the return_expected keyword arg is as expected
+    """
+    tb = pyhf.tensorlib
+
+    kwargs = {'return_tail_probs': True, 'return_expected': True}
+    result = pyhf.utils.hypotest(*hypotest_args, **kwargs)
+    # CLs_obs, [CLsb, CLb], CLs_exp
+    assert len(list(result)) == 3
+    assert isinstance(result[0], type(tb.astensor(result[0])))
+    assert len(result[1]) == 2
+    assert check_uniform_type(result[1])
+    assert isinstance(result[2], type(tb.astensor(result[2])))
+
+
+def test_hypotest_return_expected_set(tmpdir, hypotest_args):
+    """
+    Check that the return structure of pyhf.utils.hypotest with the
+    additon of the return_expected_set keyword arg is as expected
+    """
+    tb = pyhf.tensorlib
+
+    kwargs = {
+        'return_tail_probs': True,
+        'return_expected': True,
+        'return_expected_set': True,
+    }
+    result = pyhf.utils.hypotest(*hypotest_args, **kwargs)
+    # CLs_obs, [CLsb, CLb], CLs_exp, CLs_exp @[-2, -1, 0, +1, +2]sigma
+    assert len(list(result)) == 4
+    assert isinstance(result[0], type(tb.astensor(result[0])))
+    assert len(result[1]) == 2
+    assert check_uniform_type(result[1])
+    assert isinstance(result[2], type(tb.astensor(result[2])))
+    assert len(result[3]) == 5
+    assert check_uniform_type(result[3])
+
+
+def test_hypotest_return_test_statistics(tmpdir, hypotest_args):
+    """
+    Check that the return structure of pyhf.utils.hypotest with the
+    additon of the return_test_statistics keyword arg is as expected
+    """
+    tb = pyhf.tensorlib
+
+    kwargs = {
+        'return_tail_probs': True,
+        'return_expected': True,
+        'return_expected_set': True,
+        'return_test_statistics': True,
+    }
+    result = pyhf.utils.hypotest(*hypotest_args, **kwargs)
+    # CLs_obs, [CLsb, CLb], CLs_exp, CLs_exp @[-2, -1, 0, +1, +2]sigma, [q_mu, q_mu_Asimov]
+    assert len(list(result)) == 5
+    assert isinstance(result[0], type(tb.astensor(result[0])))
+    assert len(result[1]) == 2
+    assert check_uniform_type(result[1])
+    assert isinstance(result[2], type(tb.astensor(result[2])))
+    assert len(result[3]) == 5
+    assert check_uniform_type(result[3])
+    assert len(result[4]) == 2
+    assert check_uniform_type(result[4])
diff --git a/tests/test_validation.py b/tests/test_validation.py
--- a/tests/test_validation.py
+++ b/tests/test_validation.py
@@ -618,16 +618,16 @@ def setup_2bin_2channel_coupledshapefactor(
     }
 
 
-def validate_runOnePoint(pdf, data, mu_test, expected_result, tolerance=1e-6):
+def validate_hypotest(pdf, data, mu_test, expected_result, tolerance=1e-6):
     init_pars = pdf.config.suggested_init()
     par_bounds = pdf.config.suggested_bounds()
 
-    CLs_obs, CLs_exp = pyhf.utils.runOnePoint(
-        mu_test, data, pdf, init_pars, par_bounds
-    )[-2:]
+    CLs_obs, CLs_exp_set = pyhf.utils.hypotest(
+        mu_test, data, pdf, init_pars, par_bounds, return_expected_set=True
+    )
 
     assert abs(CLs_obs - expected_result['obs']) / expected_result['obs'] < tolerance
-    for result, expected_result in zip(CLs_exp, expected_result['exp']):
+    for result, expected_result in zip(CLs_exp_set, expected_result['exp']):
         assert abs(result - expected_result) / expected_result < tolerance
 
 
@@ -672,6 +672,6 @@ def test_validation(setup_and_tolerance):
         len(pdf.config.suggested_bounds()) == setup['expected']['config']['par_bounds']
     )
 
-    validate_runOnePoint(
+    validate_hypotest(
         pdf, data, setup['mu'], setup['expected']['result'], tolerance=tolerance
     )