Repository: scikit-hep/pyhf

Issue:
Parameter bounds affect fits with fixed parameters
# Description

See this example:
```python
import pyhf

spec = {
    "channels": [
        {
            "name": "SR",
            "samples": [
                {
                    "data": [10.0],
                    "modifiers": [
                        {"data": None, "name": "Signal_norm", "type": "normfactor"}
                    ],
                    "name": "Signal",
                },
                {"data": [50.0], "modifiers": [], "name": "Background"},
            ],
        }
    ],
    "measurements": [
        {
            "config": {
                "parameters": [
                    {"bounds": [[0, 5]], "inits": [2.0], "name": "Signal_norm"}
                ],
                "poi": "Signal_norm",
            },
            "name": "minimal_example",
        }
    ],
    "observations": [{"data": [65.0], "name": "SR"}],
    "version": "1.0.0",
}

ws = pyhf.Workspace(spec)
model = ws.model()
data = ws.data(model)

# pyhf.infer.hypotest(6.0, data, model, qtilde=True)  # this fails

pyhf.set_backend("numpy", pyhf.optimize.minuit_optimizer(verbose=False))
# the next two work, but result in identical values
print(pyhf.infer.hypotest(6.0, data, model, qtilde=True))
print(pyhf.infer.hypotest(7.0, data, model, qtilde=True))
```

The CLs value reported in the last two lines is identical. This seems to be related to the parameter bounds `[0,5]`. When increasing the range, the CLs values are different (as expected).

When uncommenting the first `hypotest` (using `scipy`), the minimization fails:
```
pyhf.exceptions.FailedMinimization: Positive directional derivative for linesearch
```

# Expected Behavior

All fits succeed, and different CLs for any POI value. If the bounds are somehow still respected, a warning about parameters being outside the bounds (and scaled back to the bound) would be helpful.

# Actual Behavior

Same CLs for POI values above bound, and `scipy` backend crashes.

# Steps to Reproduce

See above, same behavior with 0.5.2 and master.

# Checklist

- [X] Run `git fetch` to get the most up to date version of `master`
- [X] Searched through existing Issues to confirm this is not a duplicate issue
- [X] Filled out the Description, Expected Behavior, Actual Behavior, and Steps to Reproduce sections above or have edited/removed them in a way that fully describes the issue



Discussion:
Thanks for reporting this @alexander-held.

@lukasheinrich @kratsg I'm tempted to call this a bug given that if the user has set the parameter bounds and are then attempting to evaluate with the POI _outside_ the bounds this is almost certainly not what the user intended to do.

Likewise, if you didn't set the bounds but then are scanning above the POI ([as we've already seen some people attempt to do](https://stackoverflow.com/questions/60089405/fit-convergence-failure-in-pyhf-for-small-signal-model)) then you should also get a warning or an error.
maybe it's something in the state of iminuit? I would hope that the fact that you did a hypotest before another one doesn't affect the 2nd hypotest. But this seems like what @alexander-held is reporting. I know minuit is very stateful but I was hoping the fact that we always create a minuit object from scratch insulates us from that
ah I read more carefully now:

scipy fails hard when exceeding the boundary while minuit fails somewhat silently. And what we should do is produce helpful error messages if the best fit val is at a boundary?
Just to add a demo of the explicit output of the above on `master` (so basically a release candidate of `v0.5.3`)

```python
# issue_1123.py
import pyhf


def build_spec(meas_bounds):
    spec = {
        "channels": [
            {
                "name": "SR",
                "samples": [
                    {
                        "data": [10.0],
                        "modifiers": [
                            {"data": None, "name": "Signal_norm", "type": "normfactor"}
                        ],
                        "name": "Signal",
                    },
                    {"data": [50.0], "modifiers": [], "name": "Background"},
                ],
            }
        ],
        "measurements": [
            {
                "config": {
                    "parameters": [
                        {"bounds": [meas_bounds], "inits": [2.0], "name": "Signal_norm"}
                    ],
                    "poi": "Signal_norm",
                },
                "name": "minimal_example",
            }
        ],
        "observations": [{"data": [65.0], "name": "SR"}],
        "version": "1.0.0",
    }
    return spec


def main(meas_bounds, use_minuit=False):
    print(f"\n---\nbounds on POI: {meas_bounds}")
    spec = build_spec(meas_bounds)

    ws = pyhf.Workspace(spec)
    model = ws.model()
    data = ws.data(model)

    if use_minuit:
        optimizer_name = "MINUIT"
        pyhf.set_backend("numpy", pyhf.optimize.minuit_optimizer(verbose=False))
    else:
        optimizer_name = "SciPy"
        pyhf.set_backend("numpy")

    print(f"Optimizer: {optimizer_name}")

    for test_mu in [6.0, 7.0]:
        print(
            f"test_mu: {test_mu}, CLs_obs: {pyhf.infer.hypotest(test_mu, data, model, qtilde=True)}"
        )


if __name__ == "__main__":
    main([0, 10])
    main([0, 10], use_minuit=True)
    main([0, 5], use_minuit=True)
    main([0, 5])
```

and running gives

```
$ python issue_1123.py

---
bounds on POI: [0, 10]
Optimizer: SciPy
test_mu: 6.0, CLs_obs: 1.7395702736810923e-06
test_mu: 7.0, CLs_obs: 1.9334865580514044e-08

---
bounds on POI: [0, 10]
Optimizer: MINUIT
test_mu: 6.0, CLs_obs: 1.739570574901934e-06
test_mu: 7.0, CLs_obs: 1.933486871767672e-08

---
bounds on POI: [0, 5]
Optimizer: MINUIT
test_mu: 6.0, CLs_obs: 9.491730291075429e-05
test_mu: 7.0, CLs_obs: 9.491730291075429e-05

---
bounds on POI: [0, 5]
Optimizer: SciPy
     fun: 20.01304932662174
     jac: array([7.])
 message: 'Positive directional derivative for linesearch'
    nfev: 68
     nit: 5
    njev: 6
  status: 8
 success: False
       x: array([5.])
Traceback (most recent call last):
  File "/home/feickert/Code/GitHub/pyhf/src/pyhf/optimize/mixins.py", line 49, in _internal_minimize
    assert result.success
AssertionError

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "issue_1123.py", line 65, in <module>
    main([0, 5])
  File "issue_1123.py", line 57, in main
    f"test_mu: {test_mu}, CLs_obs: {pyhf.infer.hypotest(test_mu, data, model, qtilde=True)}"
  File "/home/feickert/Code/GitHub/pyhf/src/pyhf/infer/__init__.py", line 133, in hypotest
    teststat = calc.teststatistic(poi_test)
  File "/home/feickert/Code/GitHub/pyhf/src/pyhf/infer/calculators.py", line 195, in teststatistic
    self.fixed_params,
  File "/home/feickert/Code/GitHub/pyhf/src/pyhf/infer/test_statistics.py", line 177, in qmu_tilde
    return _qmu_like(mu, data, pdf, init_pars, par_bounds, fixed_params)
  File "/home/feickert/Code/GitHub/pyhf/src/pyhf/infer/test_statistics.py", line 20, in _qmu_like
    mu, data, pdf, init_pars, par_bounds, fixed_params, return_fitted_pars=True
  File "/home/feickert/Code/GitHub/pyhf/src/pyhf/infer/test_statistics.py", line 39, in _tmu_like
    mu, data, pdf, init_pars, par_bounds, fixed_params, return_fitted_val=True
  File "/home/feickert/Code/GitHub/pyhf/src/pyhf/infer/mle.py", line 177, in fixed_poi_fit
    return fit(data, pdf, init_pars, par_bounds, fixed_params, **kwargs)
  File "/home/feickert/Code/GitHub/pyhf/src/pyhf/infer/mle.py", line 110, in fit
    twice_nll, data, pdf, init_pars, par_bounds, fixed_vals, **kwargs
  File "/home/feickert/Code/GitHub/pyhf/src/pyhf/optimize/mixins.py", line 136, in minimize
    result = self._internal_minimize(**minimizer_kwargs, options=kwargs)
  File "/home/feickert/Code/GitHub/pyhf/src/pyhf/optimize/mixins.py", line 52, in _internal_minimize
    raise exceptions.FailedMinimization(result)
pyhf.exceptions.FailedMinimization: Positive directional derivative for linesearch
```

> scipy fails hard when exceeding the boundary while minuit fails somewhat silently. And what we should do is produce helpful error messages if the best fit val is at a boundary?

I think yes. I would suggest error messages regardless of the optimizer being used as in any case that a user is asking for a test POI that is outside the bounds for the POI then they _shouldn't_ be and are probably not meaning to. The SciPy error is good, but I think we should be more explicit and tell people "This isn't going to make physical sense given the choices you made. You don't want to do this, so please go change them and try again."

Resolving patch:
diff --git a/src/pyhf/infer/mle.py b/src/pyhf/infer/mle.py
--- a/src/pyhf/infer/mle.py
+++ b/src/pyhf/infer/mle.py
@@ -47,6 +47,17 @@ def twice_nll(pars, data, pdf):
     return -2 * pdf.logpdf(pars, data)
 
 
+def _validate_fit_inputs(init_pars, par_bounds, fixed_params):
+    for par_idx, (value, bound) in enumerate(zip(init_pars, par_bounds)):
+        if not (bound[0] <= value <= bound[1]):
+            raise ValueError(
+                f"fit initialization parameter (index: {par_idx}, value: {value}) lies outside of its bounds: {bound}"
+                + "\nTo correct this adjust the initialization parameter values in the model spec or those given"
+                + "\nas arguments to pyhf.infer.fit. If this value is intended, adjust the range of the parameter"
+                + "\nbounds."
+            )
+
+
 def fit(data, pdf, init_pars=None, par_bounds=None, fixed_params=None, **kwargs):
     r"""
     Run a maximum likelihood fit.
@@ -99,6 +110,8 @@ def fit(data, pdf, init_pars=None, par_bounds=None, fixed_params=None, **kwargs)
     par_bounds = par_bounds or pdf.config.suggested_bounds()
     fixed_params = fixed_params or pdf.config.suggested_fixed()
 
+    _validate_fit_inputs(init_pars, par_bounds, fixed_params)
+
     # get fixed vals from the model
     fixed_vals = [
         (index, init)


Test patch:
diff --git a/tests/test_infer.py b/tests/test_infer.py
--- a/tests/test_infer.py
+++ b/tests/test_infer.py
@@ -77,6 +77,22 @@ def test_hypotest_default(tmpdir, hypotest_args):
     assert isinstance(result, type(tb.astensor(result)))
 
 
+def test_hypotest_poi_outofbounds(tmpdir, hypotest_args):
+    """
+    Check that the fit errors for POI outside of parameter bounds
+    """
+    pdf = pyhf.simplemodels.hepdata_like(
+        signal_data=[12.0, 11.0], bkg_data=[50.0, 52.0], bkg_uncerts=[3.0, 7.0]
+    )
+    data = [51, 48] + pdf.config.auxdata
+
+    with pytest.raises(ValueError):
+        pyhf.infer.hypotest(-1.0, data, pdf)
+
+    with pytest.raises(ValueError):
+        pyhf.infer.hypotest(10.1, data, pdf)
+
+
 def test_hypotest_return_tail_probs(tmpdir, hypotest_args):
     """
     Check that the return structure of pyhf.infer.hypotest with the