Repository: scikit-hep/pyhf

Issue:
test_backend_consistency not resetting to default backend if test fails unexpectedly
# Description

A cascading error is observed when test_backend_consistency fails, which keeps the backend as tensorflow and causes all the other tests to erroneously fail.

    
<img width="1550" alt="screenshot 2018-04-15 20 45 50" src="https://user-images.githubusercontent.com/761483/38786764-92380ebc-40ef-11e8-921c-fc20a2d96578.png">

Easy to reproduce, run `pytest` and see `test_pdf.py` fail. Run `pytest tests/test_pdf.py` and see that it's fine (as in screenshot).


Discussion:
@kratsg Does `test_backend_consistency.py` fail when run by itself?
> @kratsg Does test_backend_consistency.py fail when run by itself?

yes, for me. When I explicitly reset the backend, it stops failing.

Resolving patch:
diff --git a/pyhf/__init__.py b/pyhf/__init__.py
--- a/pyhf/__init__.py
+++ b/pyhf/__init__.py
@@ -5,7 +5,9 @@
 
 log = logging.getLogger(__name__)
 tensorlib = tensor.numpy_backend()
+default_backend = tensorlib
 optimizer = optimize.scipy_optimizer()
+default_optimizer = optimizer
 
 def set_backend(backend):
     """


Test patch:
diff --git a/tests/benchmarks/test_benchmark.py b/tests/benchmarks/test_benchmark.py
--- a/tests/benchmarks/test_benchmark.py
+++ b/tests/benchmarks/test_benchmark.py
@@ -106,7 +106,6 @@ def test_runOnePoint(benchmark, backend, n_bins):
     Returns:
         None
     """
-    default_backend = pyhf.tensorlib
     pyhf.set_backend(backend)
 
     source = generate_source_static(n_bins)
@@ -118,8 +117,6 @@ def test_runOnePoint(benchmark, backend, n_bins):
         assert benchmark(runOnePoint, pdf, data) is not None
     except AssertionError:
         print('benchmarking has failed for n_bins = {}'.formant(n_bins))
-        pyhf.set_backend(default_backend)
         assert False
 
     # Reset backend
-    pyhf.set_backend(default_backend)
diff --git a/tests/conftest.py b/tests/conftest.py
new file mode 100644
--- /dev/null
+++ b/tests/conftest.py
@@ -0,0 +1,7 @@
+import pytest
+import pyhf
+
+@pytest.fixture(scope='function', autouse=True)
+def reset_backend():
+  yield reset_backend
+  pyhf.set_backend(pyhf.default_backend)
diff --git a/tests/test_backend_consistency.py b/tests/test_backend_consistency.py
--- a/tests/test_backend_consistency.py
+++ b/tests/test_backend_consistency.py
@@ -4,7 +4,6 @@
 import numpy as np
 import pytest
 
-
 def generate_source_static(n_bins):
     """
     Create the source structure for the given number of bins.
@@ -86,7 +85,6 @@ def test_runOnePoint_q_mu(n_bins,
     Returns:
         None
     """
-    default_backend = pyhf.tensorlib
 
     source = generate_source_static(n_bins)
     pdf = hepdata_like(source['bindata']['sig'],
@@ -128,15 +126,10 @@ def test_runOnePoint_q_mu(n_bins,
     except AssertionError:
         print('Ratio to NumPy+SciPy exceeded tolerance of {}: {}'.format(
             tolerance['numpy'], numpy_ratio_delta_unity.tolist()))
-        pyhf.set_backend(default_backend)
         assert False
     try:
         assert (tensors_ratio_delta_unity < tolerance['tensors']).all()
     except AssertionError:
         print('Ratio between tensor backends exceeded tolerance of {}: {}'.format(
             tolerance['tensors'], tensors_ratio_delta_unity.tolist()))
-        pyhf.set_backend(default_backend)
         assert False
-
-    # Reset backend
-    pyhf.set_backend(default_backend)
diff --git a/tests/test_optim.py b/tests/test_optim.py
--- a/tests/test_optim.py
+++ b/tests/test_optim.py
@@ -42,7 +42,6 @@ def test_optim_numpy():
     init_pars = pdf.config.suggested_init()
     par_bounds = pdf.config.suggested_bounds()
 
-    oldlib = pyhf.tensorlib
     pyhf.set_backend(pyhf.tensor.numpy_backend(poisson_from_normal=True))
     optim = pyhf.optimizer
 
@@ -53,8 +52,6 @@ def test_optim_numpy():
     result = optim.constrained_bestfit(pyhf.loglambdav, 1.0, data, pdf, init_pars, par_bounds)
     assert pyhf.tensorlib.tolist(result)
 
-    pyhf.set_backend(oldlib)
-
 
 def test_optim_pytorch():
     source = {
@@ -96,8 +93,6 @@ def test_optim_pytorch():
     init_pars = pdf.config.suggested_init()
     par_bounds = pdf.config.suggested_bounds()
 
-    oldlib = pyhf.tensorlib
-
     pyhf.set_backend(pyhf.tensor.pytorch_backend(poisson_from_normal=True))
     optim = pyhf.optimizer
 
@@ -107,8 +102,6 @@ def test_optim_pytorch():
     result = optim.constrained_bestfit(pyhf.loglambdav, 1.0, data, pdf, init_pars, par_bounds)
     assert pyhf.tensorlib.tolist(result)
 
-    pyhf.set_backend(oldlib)
-
 
 def test_optim_tflow():
     source = {
@@ -150,8 +143,6 @@ def test_optim_tflow():
     init_pars = pdf.config.suggested_init()
     par_bounds = pdf.config.suggested_bounds()
 
-    oldlib = pyhf.tensorlib
-
     pyhf.set_backend(pyhf.tensor.tensorflow_backend())
     pyhf.tensorlib.session = tf.Session()
     optim = pyhf.optimizer
@@ -161,5 +152,3 @@ def test_optim_tflow():
 
     result = optim.constrained_bestfit(pyhf.loglambdav, 1.0, data, pdf, init_pars, par_bounds)
     assert pyhf.tensorlib.tolist(result)
-
-    pyhf.set_backend(oldlib)
diff --git a/tests/test_tensor.py b/tests/test_tensor.py
--- a/tests/test_tensor.py
+++ b/tests/test_tensor.py
@@ -41,8 +41,6 @@ def test_common_tensor_backends():
 
 
 def test_pdf_eval():
-    oldlib = pyhf.tensorlib
-
     tf_sess = tf.Session()
     backends = [numpy_backend(poisson_from_normal=True),
                 pytorch_backend(),
@@ -92,12 +90,8 @@ def test_pdf_eval():
 
     assert np.std(values) < 1e-6
 
-    pyhf.set_backend(oldlib)
-
 
 def test_pdf_eval_2():
-    oldlib = pyhf.tensorlib
-
     tf_sess = tf.Session()
     backends = [numpy_backend(poisson_from_normal=True),
                 pytorch_backend(),
@@ -126,5 +120,3 @@ def test_pdf_eval_2():
         values.append(pyhf.tensorlib.tolist(v1)[0])
 
     assert np.std(values) < 1e-6
-
-    pyhf.set_backend(oldlib)