Repository: scikit-hep/pyhf

Issue:
iminuit v2.4.0 breaks test_optim
# Description

[`iminuit` `v2.4.0`](https://github.com/scikit-hep/iminuit/releases/tag/v2.4.0) (released today 2021-02-10) is breaking the tests for 32b and 64b minuit in the `test_minimize` tests

https://github.com/scikit-hep/pyhf/blob/bc32a695aaea0b79b3f0b40676446452d115cf8e/tests/test_optim.py#L49

```pytb
_______________ test_minimize[do_grad-minuit-jax-64b-do_stitch] ________________

tensorlib = <class 'pyhf.tensor.jax_backend.jax_backend'>, precision = '64b'
optimizer = <class 'pyhf.optimize.minuit_optimizer'>, do_grad = True
do_stitch = True

    @pytest.mark.parametrize('do_stitch', [False, True], ids=['no_stitch', 'do_stitch'])
    @pytest.mark.parametrize('precision', ['32b', '64b'], ids=['32b', '64b'])
    @pytest.mark.parametrize(
        'tensorlib',
        [
            pyhf.tensor.numpy_backend,
            pyhf.tensor.pytorch_backend,
            pyhf.tensor.tensorflow_backend,
            pyhf.tensor.jax_backend,
        ],
        ids=['numpy', 'pytorch', 'tensorflow', 'jax'],
    )
    @pytest.mark.parametrize(
        'optimizer',
        [pyhf.optimize.scipy_optimizer, pyhf.optimize.minuit_optimizer],
        ids=['scipy', 'minuit'],
    )
    @pytest.mark.parametrize('do_grad', [False, True], ids=['no_grad', 'do_grad'])
    def test_minimize(tensorlib, precision, optimizer, do_grad, do_stitch):
        pyhf.set_backend(tensorlib(precision=precision), optimizer())
        m = pyhf.simplemodels.hepdata_like([50.0], [100.0], [10.0])
        data = pyhf.tensorlib.astensor([125.0] + m.config.auxdata)
        # numpy does not support grad
        if pyhf.tensorlib.name == 'numpy' and do_grad:
            with pytest.raises(pyhf.exceptions.Unsupported):
                pyhf.infer.mle.fit(data, m, do_grad=do_grad)
        else:
            identifier = f'{"do_grad" if do_grad else "no_grad"}-{pyhf.optimizer.name}-{pyhf.tensorlib.name}-{pyhf.tensorlib.precision}'
            expected = {
                # numpy does not do grad
                'do_grad-scipy-numpy-32b': None,
                'do_grad-scipy-numpy-64b': None,
                'do_grad-minuit-numpy-32b': None,
                'do_grad-minuit-numpy-64b': None,
                # no grad, scipy, 32b - never works
                'no_grad-scipy-numpy-32b': [1.0, 1.0],
                'no_grad-scipy-pytorch-32b': [1.0, 1.0],
                'no_grad-scipy-tensorflow-32b': [1.0, 1.0],
                'no_grad-scipy-jax-32b': [1.0, 1.0],
                # no grad, scipy, 64b
                'no_grad-scipy-numpy-64b': [0.49998815367220306, 0.9999696999038924],
                'no_grad-scipy-pytorch-64b': [0.49998815367220306, 0.9999696999038924],
                'no_grad-scipy-tensorflow-64b': [0.49998865164653106, 0.9999696533705097],
                'no_grad-scipy-jax-64b': [0.4999880886490433, 0.9999696971774877],
                # do grad, scipy, 32b
                'do_grad-scipy-pytorch-32b': [0.49993881583213806, 1.0001085996627808],
                'do_grad-scipy-tensorflow-32b': [0.4999384582042694, 1.0001084804534912],
                'do_grad-scipy-jax-32b': [0.4999389052391052, 1.0001085996627808],
                # do grad, scipy, 64b
                'do_grad-scipy-pytorch-64b': [0.49998837853531425, 0.9999696648069287],
                'do_grad-scipy-tensorflow-64b': [0.4999883785353142, 0.9999696648069278],
                'do_grad-scipy-jax-64b': [0.49998837853531414, 0.9999696648069285],
                # no grad, minuit, 32b - not very consistent for pytorch
                'no_grad-minuit-numpy-32b': [0.49622172117233276, 1.0007264614105225],
                #    nb: macos gives different numerics than CI
                # 'no_grad-minuit-pytorch-32b': [0.7465415000915527, 0.8796938061714172],
                'no_grad-minuit-pytorch-32b': [0.9684963226318359, 0.9171305894851685],
                'no_grad-minuit-tensorflow-32b': [0.5284154415130615, 0.9911751747131348],
                # 'no_grad-minuit-jax-32b': [0.5144518613815308, 0.9927923679351807],
                'no_grad-minuit-jax-32b': [0.49620240926742554, 1.0018986463546753],
                # no grad, minuit, 64b - quite consistent
                'no_grad-minuit-numpy-64b': [0.5000493563629738, 1.0000043833598724],
                'no_grad-minuit-pytorch-64b': [0.5000493563758468, 1.0000043833508256],
                'no_grad-minuit-tensorflow-64b': [0.5000493563645547, 1.0000043833598657],
                'no_grad-minuit-jax-64b': [0.5000493563528641, 1.0000043833614634],
                # do grad, minuit, 32b
                'do_grad-minuit-pytorch-32b': [0.5017611384391785, 0.9997190237045288],
                'do_grad-minuit-tensorflow-32b': [0.5012885928153992, 1.0000673532485962],
                # 'do_grad-minuit-jax-32b': [0.5029529333114624, 0.9991086721420288],
                'do_grad-minuit-jax-32b': [0.5007095336914062, 0.9999282360076904],
                # do grad, minuit, 64b
                'do_grad-minuit-pytorch-64b': [0.500273961181471, 0.9996310135736226],
                'do_grad-minuit-tensorflow-64b': [0.500273961167223, 0.9996310135864218],
                'do_grad-minuit-jax-64b': [0.5002739611532436, 0.9996310135970794],
            }[identifier]
    
            result = pyhf.infer.mle.fit(data, m, do_grad=do_grad, do_stitch=do_stitch)
    
            rtol = 2e-06
            # handle cases where macos and ubuntu provide very different results numerical
            if 'no_grad-minuit-tensorflow-32b' in identifier:
                # not a very large difference, so we bump the relative difference down
                rtol = 3e-02
            if 'no_grad-minuit-pytorch-32b' in identifier:
                # quite a large difference
                rtol = 3e-01
            if 'do_grad-minuit-pytorch-32b' in identifier:
                # a small difference
                rtol = 7e-05
            if 'no_grad-minuit-jax-32b' in identifier:
                rtol = 4e-02
            if 'do_grad-minuit-jax-32b' in identifier:
                rtol = 5e-03
    
            # check fitted parameters
>           assert pytest.approx(expected, rel=rtol) == pyhf.tensorlib.tolist(
                result
            ), f"{identifier} = {pyhf.tensorlib.tolist(result)}"
E           AssertionError: do_grad-minuit-jax-64b = [0.500049321731032, 1.0000044174002167]
E           assert approx([0.5002739611532436 ± 1.0e-06, 0.9996310135970794 ± 2.0e-06]) == [0.500049321731032, 1.0000044174002167]
E            +  where approx([0.5002739611532436 ± 1.0e-06, 0.9996310135970794 ± 2.0e-06]) = <function approx at 0x7fb30c6b6e50>([0.5002739611532436, 0.9996310135970794], rel=2e-06)
E            +    where <function approx at 0x7fb30c6b6e50> = pytest.approx
E            +  and   [0.500049321731032, 1.0000044174002167] = <bound method jax_backend.tolist of <pyhf.tensor.jax_backend.jax_backend object at 0x7fb210064b00>>(DeviceArray([0.50004932, 1.00000442], dtype=float64))
E            +    where <bound method jax_backend.tolist of <pyhf.tensor.jax_backend.jax_backend object at 0x7fb210064b00>> = <pyhf.tensor.jax_backend.jax_backend object at 0x7fb210064b00>.tolist
E            +      where <pyhf.tensor.jax_backend.jax_backend object at 0x7fb210064b00> = pyhf.tensorlib

tests/test_optim.py:126: AssertionError
```

`test_minuit_strategy_do_grad`

https://github.com/scikit-hep/pyhf/blob/bc32a695aaea0b79b3f0b40676446452d115cf8e/tests/test_optim.py#L182

as well as in `test_minuit_strategy_global` tests

https://github.com/scikit-hep/pyhf/blob/bc32a695aaea0b79b3f0b40676446452d115cf8e/tests/test_optim.py#L210

```pytb
__________________ test_minuit_strategy_global[tensorflow-1] ___________________

self = <pyhf.optimize.minuit_optimizer object at 0x7fb2107be700>
func = <function wrap_objective.<locals>.func at 0x7fb228255a60>
x0 = [1.0, 1.0], do_grad = True, bounds = [(0, 10), (1e-10, 10.0)]
fixed_vals = [], options = {}

    def _internal_minimize(
        self, func, x0, do_grad=False, bounds=None, fixed_vals=None, options={}
    ):
    
        minimizer = self._get_minimizer(
            func, x0, bounds, fixed_vals=fixed_vals, do_grad=do_grad
        )
        result = self._minimize(
            minimizer,
            func,
            x0,
            do_grad=do_grad,
            bounds=bounds,
            fixed_vals=fixed_vals,
            options=options,
        )
    
        try:
>           assert result.success
E           AssertionError

src/pyhf/optimize/mixins.py:49: AssertionError

During handling of the above exception, another exception occurred:

mocker = <pytest_mock.plugin.MockerFixture object at 0x7fb1e3d59370>
backend = (<pyhf.tensor.tensorflow_backend.tensorflow_backend object at 0x7fb2433ed080>, None)
strategy = 1

    @pytest.mark.parametrize('strategy', [0, 1])
    def test_minuit_strategy_global(mocker, backend, strategy):
        pyhf.set_backend(pyhf.tensorlib, pyhf.optimize.minuit_optimizer(strategy=strategy))
        spy = mocker.spy(pyhf.optimize.minuit_optimizer, '_minimize')
        m = pyhf.simplemodels.hepdata_like([50.0], [100.0], [10.0])
        data = pyhf.tensorlib.astensor([125.0] + m.config.auxdata)
    
        do_grad = pyhf.tensorlib.default_do_grad
>       pyhf.infer.mle.fit(data, m)

tests/test_optim.py:217: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
src/pyhf/infer/mle.py:122: in fit
    return opt.minimize(
src/pyhf/optimize/mixins.py:157: in minimize
    result = self._internal_minimize(**minimizer_kwargs, options=kwargs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <pyhf.optimize.minuit_optimizer object at 0x7fb2107be700>
func = <function wrap_objective.<locals>.func at 0x7fb228255a60>
x0 = [1.0, 1.0], do_grad = True, bounds = [(0, 10), (1e-10, 10.0)]
fixed_vals = [], options = {}

    def _internal_minimize(
        self, func, x0, do_grad=False, bounds=None, fixed_vals=None, options={}
    ):
    
        minimizer = self._get_minimizer(
            func, x0, bounds, fixed_vals=fixed_vals, do_grad=do_grad
        )
        result = self._minimize(
            minimizer,
            func,
            x0,
            do_grad=do_grad,
            bounds=bounds,
            fixed_vals=fixed_vals,
            options=options,
        )
    
        try:
            assert result.success
        except AssertionError:
            log.error(result, exc_info=True)
>           raise exceptions.FailedMinimization(result)
E           pyhf.exceptions.FailedMinimization: Optimization failed. Estimated distance to minimum too large.
```

We will need to investigate what's up and perhaps loosen tolerances for `iminuit`.


Discussion:
@HDembinski :wave: while we don't need you to do anything (probably) I thought I'd just give you a heads up that we might have some questions in the next day or so, given that I'm not sure what in [the `CHANGELOG`](https://iminuit.readthedocs.io/en/stable/changelog.html#february-10-2021) could be causing this beyond the

> Upgrade Minuit2 C++ code in ROOT to latest master with following improvements

:+1: 

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -12,7 +12,7 @@
         'uproot3>=3.14.1',
         'uproot~=4.0',
     ],  # uproot3 required until writing to ROOT supported in uproot4
-    'minuit': ['iminuit~=2.1,<2.4'],  # iminuit v2.4.0 behavior needs to be understood
+    'minuit': ['iminuit>=2.4'],
 }
 extras_require['backends'] = sorted(
     set(


Test patch:
diff --git a/tests/test_optim.py b/tests/test_optim.py
--- a/tests/test_optim.py
+++ b/tests/test_optim.py
@@ -83,10 +83,8 @@ def test_minimize(tensorlib, precision, optimizer, do_grad, do_stitch):
             # no grad, minuit, 32b - not very consistent for pytorch
             'no_grad-minuit-numpy-32b': [0.7465415000915527, 0.8796938061714172],
             #    nb: macos gives different numerics than CI
-            # 'no_grad-minuit-pytorch-32b': [0.7465415000915527, 0.8796938061714172],
             'no_grad-minuit-pytorch-32b': [0.9684963226318359, 0.9171305894851685],
             'no_grad-minuit-tensorflow-32b': [0.5284154415130615, 0.9911751747131348],
-            # 'no_grad-minuit-jax-32b': [0.5144518613815308, 0.9927923679351807],
             'no_grad-minuit-jax-32b': [0.49620240926742554, 1.0018986463546753],
             # no grad, minuit, 64b - quite consistent
             'no_grad-minuit-numpy-64b': [0.5000493563629738, 1.0000043833598724],
@@ -94,39 +92,47 @@ def test_minimize(tensorlib, precision, optimizer, do_grad, do_stitch):
             'no_grad-minuit-tensorflow-64b': [0.5000493563645547, 1.0000043833598657],
             'no_grad-minuit-jax-64b': [0.5000493563528641, 1.0000043833614634],
             # do grad, minuit, 32b
-            'do_grad-minuit-pytorch-32b': [0.5017611384391785, 0.9997190237045288],
-            'do_grad-minuit-tensorflow-32b': [0.5012885928153992, 1.0000673532485962],
-            # 'do_grad-minuit-jax-32b': [0.5029529333114624, 0.9991086721420288],
+            # large divergence by tensorflow and pytorch
+            'do_grad-minuit-pytorch-32b': [0.9731879234313965, 0.9999999403953552],
+            'do_grad-minuit-tensorflow-32b': [0.9366918206214905, 0.9126002788543701],
             'do_grad-minuit-jax-32b': [0.5007095336914062, 0.9999282360076904],
             # do grad, minuit, 64b
-            'do_grad-minuit-pytorch-64b': [0.500273961181471, 0.9996310135736226],
-            'do_grad-minuit-tensorflow-64b': [0.500273961167223, 0.9996310135864218],
-            'do_grad-minuit-jax-64b': [0.5002739611532436, 0.9996310135970794],
+            'do_grad-minuit-pytorch-64b': [0.500049321728735, 1.00000441739846],
+            'do_grad-minuit-tensorflow-64b': [0.5000492930412292, 1.0000044107437134],
+            'do_grad-minuit-jax-64b': [0.500049321731032, 1.0000044174002167],
         }[identifier]
 
         result = pyhf.infer.mle.fit(data, m, do_grad=do_grad, do_stitch=do_stitch)
 
-        rtol = 2e-06
+        rel_tol = 1e-6
+        # Fluctuations beyond precision shouldn't matter
+        abs_tol = 1e-5 if "32b" in identifier else 1e-8
+
         # handle cases where macos and ubuntu provide very different results numerical
-        if 'no_grad-minuit-tensorflow-32b' in identifier:
-            # not a very large difference, so we bump the relative difference down
-            rtol = 3e-02
-        if 'no_grad-minuit-pytorch-32b' in identifier:
-            # quite a large difference
-            rtol = 3e-01
-        if 'do_grad-minuit-pytorch-32b' in identifier:
-            # a small difference
-            rtol = 7e-05
-        if 'no_grad-minuit-jax-32b' in identifier:
-            rtol = 4e-02
-        # NB: ubuntu and macos give different results for 32b
-        if "do_grad-scipy-jax-32b" in identifier:
-            rtol = 5e-03
-        if "do_grad-minuit-jax-32b" in identifier:
-            rtol = 5e-03
+        if "no_grad" in identifier:
+            rel_tol = 1e-5
+            if "minuit-pytorch-32b" in identifier:
+                # large difference between local and CI
+                rel_tol = 3e-1
+            if "minuit-tensorflow-32b" in identifier:
+                # not a very large difference, so we bump the relative difference down
+                rel_tol = 3e-2
+            if "minuit-jax-32b" in identifier:
+                rel_tol = 4e-2
+        elif all(part in identifier for part in ["do_grad", "32b"]):
+            if "scipy-jax" in identifier:
+                rel_tol = 1e-2
+            # NB: ubuntu and macos give different results for 32b
+            if "minuit-tensorflow" in identifier:
+                # large difference between local and CI
+                rel_tol = 1e-1
+            if "minuit-jax" in identifier:
+                rel_tol = 1e-2
 
         # check fitted parameters
-        assert pytest.approx(expected, rel=rtol) == pyhf.tensorlib.tolist(
+        assert pytest.approx(
+            expected, rel=rel_tol, abs=abs_tol
+        ) == pyhf.tensorlib.tolist(
             result
         ), f"{identifier} = {pyhf.tensorlib.tolist(result)}"
 
@@ -190,7 +196,7 @@ def test_minuit_strategy_do_grad(mocker, backend):
     the minuit strategy=0. When there is no user-provided gradient, check that
     one automatically sets the minuit strategy=1.
     """
-    pyhf.set_backend(pyhf.tensorlib, 'minuit')
+    pyhf.set_backend(pyhf.tensorlib, pyhf.optimize.minuit_optimizer(tolerance=0.2))
     spy = mocker.spy(pyhf.optimize.minuit_optimizer, '_minimize')
     m = pyhf.simplemodels.hepdata_like([50.0], [100.0], [10.0])
     data = pyhf.tensorlib.astensor([125.0] + m.config.auxdata)
@@ -211,7 +217,9 @@ def test_minuit_strategy_do_grad(mocker, backend):
 
 @pytest.mark.parametrize('strategy', [0, 1])
 def test_minuit_strategy_global(mocker, backend, strategy):
-    pyhf.set_backend(pyhf.tensorlib, pyhf.optimize.minuit_optimizer(strategy=strategy))
+    pyhf.set_backend(
+        pyhf.tensorlib, pyhf.optimize.minuit_optimizer(strategy=strategy, tolerance=0.2)
+    )
     spy = mocker.spy(pyhf.optimize.minuit_optimizer, '_minimize')
     m = pyhf.simplemodels.hepdata_like([50.0], [100.0], [10.0])
     data = pyhf.tensorlib.astensor([125.0] + m.config.auxdata)