Repository: scikit-hep/pyhf

Issue:
figure out why we do `CLb / CLsb`
# Description

we have this snippet:

```python
from scipy.stats import norm
def pvals_from_teststat(sqrtqmu_v, sqrtqmuA_v):
    CLsb = 1 - norm.cdf(sqrtqmu_v)
    CLb = norm.cdf(sqrtqmuA_v - sqrtqmu_v)
    CLs = CLb / CLsb
    return CLsb, CLb, CLs
```

as @matthewfeickert correctly observed this seems off at first sight. But I seem to remember there was a specific reason I did it this way (perhaps while keeping around wrong variable names) 

i think it has something to do with this line in ROOT

```c++
 virtual Double_t CLb() const { return !fBackgroundIsAlt ? NullPValue() : AlternatePValue(); }
```

https://root.cern.ch/doc/v606/HypoTestResult_8h_source.html#l00138


Discussion:
I suspect the variable names are backwards --- I remember the `cdf` functions confused me based on which direction the value is reporting. I think `cdf(x)` is from -infinity to x.

https://cds.cern.ch/record/1099994, LHC Statistics for Pedestrians

<img width="698" alt="screenshot 2018-04-20 07 17 25" src="https://user-images.githubusercontent.com/761483/39050435-f0aef9a0-446a-11e8-88dc-5250f5e03220.png">

I think the issue is that in ROOT, it's setup genericall as a Hypothesis test between a Null Hypothesis and an Alternate Hypothesis, one will be "background-only" while the other one will not, and this assignment is not fixed. It's a choice set by the HypoTestInverter, and relates to whether we do an exclusion or a discovery fit

https://root.cern.ch/doc/v608/HypoTestInverter_8cxx_source.html#l00500

e.g. see this

https://github.com/dguest/HistFitter/blob/master/src/StatTools.cxx#L843



From PR #141:
> Arguably we should just return CLs instead of 1/CLs, but I didn't want to change all the other pieces that might rely on the output of runOnePoint

@lukasheinrich Are there any objections to in another PR going through and checking all of this and moving to the CLs return structure?
no, not at all. I think that's what we should do
Unless someone beats me to it I'll work on this once I catch up on sleep.

Resolving patch:
diff --git a/pyhf/__init__.py b/pyhf/__init__.py
--- a/pyhf/__init__.py
+++ b/pyhf/__init__.py
@@ -10,6 +10,7 @@
 optimizer = optimize.scipy_optimizer()
 default_optimizer = optimizer
 
+
 def get_backend():
     """
     Get the current backend and the associated optimizer
@@ -62,6 +63,7 @@ def set_backend(backend):
     else:
         optimizer = optimize.scipy_optimizer()
 
+
 class modelconfig(object):
     @classmethod
     def from_spec(cls,spec,poiname = 'mu'):
@@ -311,9 +313,11 @@ def loglambdav(pars, data, pdf):
 
 def qmu(mu, data, pdf, init_pars, par_bounds):
     r"""
-    The test statistic, q_mu, for establishing an upper
-    limit on the strength parameter, mu, as defiend in
-    Equation (14) in arXiv:1007.1727
+    The test statistic, :math:`q_{\mu}`, for establishing an upper
+    limit on the strength parameter, :math:`\mu`, as defiend in
+    Equation (14) in `arXiv:1007.1727`_ .
+
+    .. _`arXiv:1007.1727`: https://arxiv.org/abs/1007.1727
 
     .. math::
        :nowrap:
@@ -334,7 +338,7 @@ def qmu(mu, data, pdf, init_pars, par_bounds):
         par_bounds(Tensor): The bounds on the paramter values
 
     Returns:
-        Float: The calculated test statistic, q_mu
+        Float: The calculated test statistic, :math:`q_{\mu}`
     """
     mubhathat = optimizer.constrained_bestfit(
         loglambdav, mu, data, pdf, init_pars, par_bounds)
@@ -346,13 +350,35 @@ def qmu(mu, data, pdf, init_pars, par_bounds):
 
 
 def pvals_from_teststat(sqrtqmu_v, sqrtqmuA_v):
-    # these pvals are from formula
-    # (59) in arxiv:1007.1727 p_mu = 1-F(q_mu|mu') = 1- \Phi(q_mu - (mu-mu')/sigma)
-    # and  (mu-mu')/sigma = sqrt(Lambda)= sqrt(q_mu_A)
+    r"""
+    The :math:`p`-values for signal strength :math:`\mu` and Asimov strength :math:`\mu'`
+    as defined in Equations (59) and (57) of `arXiv:1007.1727`_
+
+    .. _`arXiv:1007.1727`: https://arxiv.org/abs/1007.1727
+
+    .. math::
+
+        p_{\mu} = 1-F\left(q_{\mu}\middle|\mu'\right) = 1- \Phi\left(q_{\mu} - \frac{\left(\mu-\mu'\right)}{\sigma}\right)
+
+    with Equation (29)
+
+    .. math::
+
+        \frac{(\mu-\mu')}{\sigma} = \sqrt{\Lambda}= \sqrt{q_{\mu,A}}
+
+    given the observed test statistics :math:`q_{\mu}` and :math:`q_{\mu,A}`.
+
+    Args:
+        sqrtqmu_v (Number or Tensor): The root of the calculated test statistic, :math:`\sqrt{q_{\mu}}`
+        sqrtqmuA_v (Number or Tensor): The root of the calculated test statistic given the Asimov data, :math:`\sqrt{q_{\mu,A}}`
+
+    Returns:
+        Tuple of Floats: The :math:`p`-values for the signal + background, background only, and signal only hypotheses respectivley
+    """
     CLsb = 1 - tensorlib.normal_cdf(sqrtqmu_v)
     CLb = 1 - tensorlib.normal_cdf(sqrtqmu_v - sqrtqmuA_v)
-    oneOverCLs = CLb / CLsb
-    return CLsb, CLb, oneOverCLs
+    CLs = CLsb / CLb
+    return CLsb, CLb, CLs
 
 
 def runOnePoint(muTest, data, pdf, init_pars, par_bounds):
@@ -368,12 +394,12 @@ def runOnePoint(muTest, data, pdf, init_pars, par_bounds):
         qmu(muTest, asimov_data, pdf, init_pars, par_bounds), 0, max=None)
     sqrtqmuA_v = tensorlib.sqrt(qmuA_v)
 
-    CLsb, CLb, oneOverCLs = pvals_from_teststat(sqrtqmu_v, sqrtqmuA_v)
+    CLsb, CLb, CLs = pvals_from_teststat(sqrtqmu_v, sqrtqmuA_v)
 
-    oneOverCLs_exp = []
+    CLs_exp = []
     for nsigma in [-2, -1, 0, 1, 2]:
         sqrtqmu_v_sigma = sqrtqmuA_v - nsigma
-        oneOverCLs_exp.append(
+        CLs_exp.append(
             pvals_from_teststat(sqrtqmu_v_sigma, sqrtqmuA_v)[-1])
-    oneOverCLs_exp = tensorlib.astensor(oneOverCLs_exp)
-    return qmu_v, qmuA_v, CLsb, CLb, oneOverCLs, oneOverCLs_exp
+    CLs_exp = tensorlib.astensor(CLs_exp)
+    return qmu_v, qmuA_v, CLsb, CLb, CLs, CLs_exp


Test patch:
diff --git a/tests/test_validation.py b/tests/test_validation.py
--- a/tests/test_validation.py
+++ b/tests/test_validation.py
@@ -661,8 +661,6 @@ def validate_runOnePoint(pdf, data, mu_test, expected_result, tolerance=1e-5):
 
     CLs_obs, CLs_exp = pyhf.runOnePoint(
         mu_test, data, pdf, init_pars, par_bounds)[-2:]
-    CLs_obs = 1. / CLs_obs
-    CLs_exp = [1. / x for x in CLs_exp]
     assert (CLs_obs - expected_result['obs']) / \
         expected_result['obs'] < tolerance
     for result, expected_result in zip(CLs_exp, expected_result['exp']):