Repository: scikit-hep/pyhf

Issue:
Fix Fixture use in pytest
# Description

In pytest `v4.0.0` the [direct call of a fixture results in an error](https://travis-ci.org/diana-hep/pyhf/jobs/455364238#L661-L669). 

```
==================================== ERRORS ====================================
__________________ ERROR collecting tests/test_validation.py ___________________
tests/test_validation.py:13: in <module>
    def spec_1bin_shapesys(source=source_1bin_example1()):
E   _pytest.warning_types.RemovedInPytest4Warning: Fixture "source_1bin_example1" called directly. Fixtures are not meant to be called directly, are created automatically when test functions request them as parameters. See https://docs.pytest.org/en/latest/fixture.html for more information.
__________________ ERROR collecting tests/test_validation.py ___________________
tests/test_validation.py:13: in <module>
    def spec_1bin_shapesys(source=source_1bin_example1()):
E   _pytest.warning_types.RemovedInPytest4Warning: Fixture "source_1bin_example1" called directly. Fixtures are not meant to be called directly, are created automatically when test functions request them as parameters. See https://docs.pytest.org/en/latest/fixture.html for more information.
```

This requires changing the way that pytest is used a bit.

This was noticed in preparation of PR #369 

# Checklist

- [x] Run `git fetch` to get the most up to date version of `master`
- [x] Searched through existing Issues to confirm this is not a duplicate issue
- [x] Filled out the Description, Expected Behavior, Actual Behavior, and Steps to Reproduce sections above or have edited/removed them in a way that fully describes the issue



Discussion:
Also seen in #352 .

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -29,7 +29,7 @@
         + extras_require['contrib']
         + extras_require['shellcomplete']
         + [
-            'pytest~=3.5',
+            'pytest~=6.0',
             'pytest-cov>=2.5.1',
             'pytest-mock',
             'pytest-benchmark[histogram]',


Test patch:
diff --git a/tests/conftest.py b/tests/conftest.py
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -98,14 +98,18 @@ def backend(request):
     func_name = request._pyfuncitem.name
 
     # skip backends if specified
-    skip_backend = request.node.get_marker('skip_{param}'.format(param=param_id))
+    skip_backend = request.node.get_closest_marker(
+        'skip_{param}'.format(param=param_id)
+    )
     # allow the specific backend to fail if specified
-    fail_backend = request.node.get_marker('fail_{param}'.format(param=param_id))
+    fail_backend = request.node.get_closest_marker(
+        'fail_{param}'.format(param=param_id)
+    )
     # only look at the specific backends
     only_backends = [
         pid
         for pid in param_ids
-        if request.node.get_marker('only_{param}'.format(param=pid))
+        if request.node.get_closest_marker('only_{param}'.format(param=pid))
     ]
 
     if skip_backend and (param_id in only_backends):
diff --git a/tests/test_teststats.py b/tests/test_teststats.py
--- a/tests/test_teststats.py
+++ b/tests/test_teststats.py
@@ -16,7 +16,7 @@ def test_qmu(caplog):
         pyhf.infer.test_statistics.qmu(
             mu, data, model, init_pars, par_bounds, fixed_params
         )
-        assert "WARNING  qmu test statistic used for fit" in caplog.text
+        assert "qmu test statistic used for fit" in caplog.text
         caplog.clear()
 
 
@@ -33,7 +33,7 @@ def test_qmu_tilde(caplog):
         pyhf.infer.test_statistics.qmu_tilde(
             mu, data, model, init_pars, par_bounds, fixed_params
         )
-        assert "WARNING  qmu_tilde test statistic used for fit" in caplog.text
+        assert "qmu_tilde test statistic used for fit" in caplog.text
         caplog.clear()
 
 
@@ -49,7 +49,7 @@ def test_tmu(caplog):
         pyhf.infer.test_statistics.tmu(
             mu, data, model, init_pars, par_bounds, fixed_params
         )
-        assert "WARNING  tmu test statistic used for fit" in caplog.text
+        assert "tmu test statistic used for fit" in caplog.text
         caplog.clear()
 
 
@@ -66,7 +66,7 @@ def test_tmu_tilde(caplog):
         pyhf.infer.test_statistics.tmu_tilde(
             mu, data, model, init_pars, par_bounds, fixed_params
         )
-        assert "WARNING  tmu_tilde test statistic used for fit" in caplog.text
+        assert "tmu_tilde test statistic used for fit" in caplog.text
         caplog.clear()
 
 
diff --git a/tests/test_validation.py b/tests/test_validation.py
--- a/tests/test_validation.py
+++ b/tests/test_validation.py
@@ -7,14 +7,17 @@
 import numpy as np
 
 
-@pytest.fixture(scope='module')
-def source_1bin_example1():
+def get_source_1bin_example1():
     with open('validation/data/1bin_example1.json') as read_json:
         return json.load(read_json)
 
 
 @pytest.fixture(scope='module')
-def spec_1bin_shapesys(source=source_1bin_example1()):
+def source_1bin_example1():
+    return get_source_1bin_example1()
+
+
+def get_spec_1bin_shapesys(source=get_source_1bin_example1()):
     spec = {
         'channels': [
             {
@@ -46,7 +49,11 @@ def spec_1bin_shapesys(source=source_1bin_example1()):
 
 
 @pytest.fixture(scope='module')
-def expected_result_1bin_shapesys(mu=1.0):
+def spec_1bin_shapesys():
+    return get_spec_1bin_shapesys()
+
+
+def get_expected_result_1bin_shapesys(mu=1.0):
     if mu == 1:
         expected_result = {
             "exp": [
@@ -62,11 +69,15 @@ def expected_result_1bin_shapesys(mu=1.0):
 
 
 @pytest.fixture(scope='module')
-def setup_1bin_shapesys(
-    source=source_1bin_example1(),
-    spec=spec_1bin_shapesys(source_1bin_example1()),
+def expected_result_1bin_shapesys():
+    return get_expected_result_1bin_shapesys()
+
+
+def get_setup_1bin_shapesys(
+    source=get_source_1bin_example1(),
+    spec=get_spec_1bin_shapesys(get_source_1bin_example1()),
     mu=1,
-    expected_result=expected_result_1bin_shapesys(1.0),
+    expected_result=get_expected_result_1bin_shapesys(1.0),
     config={'init_pars': 2, 'par_bounds': 2},
 ):
     return {
@@ -78,7 +89,11 @@ def setup_1bin_shapesys(
 
 
 @pytest.fixture(scope='module')
-def spec_1bin_lumi():
+def setup_1bin_shapesys():
+    return get_setup_1bin_shapesys()
+
+
+def get_spec_1bin_lumi():
     spec = {
         "channels": [
             {
@@ -118,7 +133,11 @@ def spec_1bin_lumi():
 
 
 @pytest.fixture(scope='module')
-def expected_result_1bin_lumi(mu=1.0):
+def spec_1bin_lumi():
+    return get_spec_1bin_lumi()
+
+
+def get_expected_result_1bin_lumi(mu=1.0):
     if mu == 1:
         expected_result = {
             "exp": [
@@ -134,11 +153,15 @@ def expected_result_1bin_lumi(mu=1.0):
 
 
 @pytest.fixture(scope='module')
-def setup_1bin_lumi(
-    source=source_1bin_example1(),
-    spec=spec_1bin_lumi(),
+def expected_result_1bin_lumi():
+    return get_expected_result_1bin_lumi()
+
+
+def get_setup_1bin_lumi(
+    source=get_source_1bin_example1(),
+    spec=get_spec_1bin_lumi(),
     mu=1,
-    expected_result=expected_result_1bin_lumi(1.0),
+    expected_result=get_expected_result_1bin_lumi(1.0),
     config={'init_pars': 2, 'par_bounds': 2},
 ):
     return {
@@ -150,7 +173,11 @@ def setup_1bin_lumi(
 
 
 @pytest.fixture(scope='module')
-def source_1bin_normsys():
+def setup_1bin_lumi():
+    return get_setup_1bin_lumi()
+
+
+def get_source_1bin_normsys():
     source = {
         'binning': [2, -0.5, 1.5],
         'bindata': {'data': [120.0, 180.0], 'bkg': [100.0, 150.0], 'sig': [30.0, 95.0]},
@@ -159,7 +186,11 @@ def source_1bin_normsys():
 
 
 @pytest.fixture(scope='module')
-def spec_1bin_normsys(source=source_1bin_normsys()):
+def source_1bin_normsys():
+    return get_source_1bin_normsys()
+
+
+def get_spec_1bin_normsys(source=get_source_1bin_normsys()):
     spec = {
         'channels': [
             {
@@ -191,7 +222,11 @@ def spec_1bin_normsys(source=source_1bin_normsys()):
 
 
 @pytest.fixture(scope='module')
-def expected_result_1bin_normsys(mu=1.0):
+def spec_1bin_normsys():
+    return get_spec_1bin_normsys()
+
+
+def get_expected_result_1bin_normsys(mu=1.0):
     if mu == 1:
         expected_result = {
             "exp": [
@@ -207,11 +242,15 @@ def expected_result_1bin_normsys(mu=1.0):
 
 
 @pytest.fixture(scope='module')
-def setup_1bin_normsys(
-    source=source_1bin_normsys(),
-    spec=spec_1bin_normsys(source_1bin_normsys()),
+def expected_result_1bin_normsys():
+    return get_expected_result_1bin_normsys()
+
+
+def get_setup_1bin_normsys(
+    source=get_source_1bin_normsys(),
+    spec=get_spec_1bin_normsys(get_source_1bin_normsys()),
     mu=1,
-    expected_result=expected_result_1bin_normsys(1.0),
+    expected_result=get_expected_result_1bin_normsys(1.0),
     config={'init_pars': 2, 'par_bounds': 2},
 ):
     return {
@@ -223,13 +262,21 @@ def setup_1bin_normsys(
 
 
 @pytest.fixture(scope='module')
-def source_2bin_histosys_example2():
+def setup_1bin_normsys():
+    return get_setup_1bin_normsys()
+
+
+def get_source_2bin_histosys_example2():
     with open('validation/data/2bin_histosys_example2.json') as read_json:
         return json.load(read_json)
 
 
 @pytest.fixture(scope='module')
-def spec_2bin_histosys(source=source_2bin_histosys_example2()):
+def source_2bin_histosys_example2():
+    return get_source_2bin_histosys_example2()
+
+
+def get_spec_2bin_histosys(source=get_source_2bin_histosys_example2()):
     spec = {
         'channels': [
             {
@@ -264,7 +311,11 @@ def spec_2bin_histosys(source=source_2bin_histosys_example2()):
 
 
 @pytest.fixture(scope='module')
-def expected_result_2bin_histosys(mu=1):
+def spec_2bin_histosys():
+    return get_spec_2bin_histosys()
+
+
+def get_expected_result_2bin_histosys(mu=1):
     if mu == 1:
         expected_result = {
             "exp": [
@@ -280,11 +331,15 @@ def expected_result_2bin_histosys(mu=1):
 
 
 @pytest.fixture(scope='module')
-def setup_2bin_histosys(
-    source=source_2bin_histosys_example2(),
-    spec=spec_2bin_histosys(source_2bin_histosys_example2()),
+def expected_result_2bin_histosys():
+    return get_expected_result_2bin_histosys()
+
+
+def get_setup_2bin_histosys(
+    source=get_source_2bin_histosys_example2(),
+    spec=get_spec_2bin_histosys(get_source_2bin_histosys_example2()),
     mu=1,
-    expected_result=expected_result_2bin_histosys(1.0),
+    expected_result=get_expected_result_2bin_histosys(1.0),
     config={'init_pars': 2, 'par_bounds': 2},
 ):
     return {
@@ -296,13 +351,21 @@ def setup_2bin_histosys(
 
 
 @pytest.fixture(scope='module')
-def source_2bin_2channel_example1():
+def setup_2bin_histosys():
+    return get_setup_2bin_histosys()
+
+
+def get_source_2bin_2channel_example1():
     with open('validation/data/2bin_2channel_example1.json') as read_json:
         return json.load(read_json)
 
 
 @pytest.fixture(scope='module')
-def spec_2bin_2channel(source=source_2bin_2channel_example1()):
+def source_2bin_2channel_example1():
+    return get_source_2bin_2channel_example1()
+
+
+def get_spec_2bin_2channel(source=get_source_2bin_2channel_example1()):
     spec = {
         'channels': [
             {
@@ -354,7 +417,11 @@ def spec_2bin_2channel(source=source_2bin_2channel_example1()):
 
 
 @pytest.fixture(scope='module')
-def expected_result_2bin_2channel(mu=1.0):
+def spec_2bin_2channel():
+    return get_spec_2bin_2channel()
+
+
+def get_expected_result_2bin_2channel(mu=1.0):
     if mu == 1:
         expected_result = {
             "exp": [
@@ -370,11 +437,15 @@ def expected_result_2bin_2channel(mu=1.0):
 
 
 @pytest.fixture(scope='module')
-def setup_2bin_2channel(
-    source=source_2bin_2channel_example1(),
-    spec=spec_2bin_2channel(source_2bin_2channel_example1()),
+def expected_result_2bin_2channel():
+    return get_expected_result_2bin_2channel()
+
+
+def get_setup_2bin_2channel(
+    source=get_source_2bin_2channel_example1(),
+    spec=get_spec_2bin_2channel(get_source_2bin_2channel_example1()),
     mu=1,
-    expected_result=expected_result_2bin_2channel(1.0),
+    expected_result=get_expected_result_2bin_2channel(1.0),
     config={'init_pars': 5, 'par_bounds': 5},
 ):
     # 1 mu + 2 gammas for 2 channels each
@@ -387,13 +458,21 @@ def setup_2bin_2channel(
 
 
 @pytest.fixture(scope='module')
-def source_2bin_2channel_couplednorm():
+def setup_2bin_2channel():
+    return get_setup_2bin_2channel()
+
+
+def get_source_2bin_2channel_couplednorm():
     with open('validation/data/2bin_2channel_couplednorm.json') as read_json:
         return json.load(read_json)
 
 
 @pytest.fixture(scope='module')
-def spec_2bin_2channel_couplednorm(source=source_2bin_2channel_couplednorm()):
+def source_2bin_2channel_couplednorm():
+    return get_source_2bin_2channel_couplednorm()
+
+
+def get_spec_2bin_2channel_couplednorm(source=get_source_2bin_2channel_couplednorm()):
     spec = {
         'channels': [
             {
@@ -452,7 +531,11 @@ def spec_2bin_2channel_couplednorm(source=source_2bin_2channel_couplednorm()):
 
 
 @pytest.fixture(scope='module')
-def expected_result_2bin_2channel_couplednorm(mu=1.0):
+def spec_2bin_2channel_couplednorm():
+    return get_spec_2bin_2channel_couplednorm()
+
+
+def get_expected_result_2bin_2channel_couplednorm(mu=1.0):
     # NB: mac/linux differ for exp[0]
     # Mac:   0.055222676184648795
     # Linux: 0.05522273289103311
@@ -472,11 +555,15 @@ def expected_result_2bin_2channel_couplednorm(mu=1.0):
 
 
 @pytest.fixture(scope='module')
-def setup_2bin_2channel_couplednorm(
-    source=source_2bin_2channel_couplednorm(),
-    spec=spec_2bin_2channel_couplednorm(source_2bin_2channel_couplednorm()),
+def expected_result_2bin_2channel_couplednorm():
+    return get_expected_result_2bin_2channel_couplednorm()
+
+
+def get_setup_2bin_2channel_couplednorm(
+    source=get_source_2bin_2channel_couplednorm(),
+    spec=get_spec_2bin_2channel_couplednorm(get_source_2bin_2channel_couplednorm()),
     mu=1,
-    expected_result=expected_result_2bin_2channel_couplednorm(1.0),
+    expected_result=get_expected_result_2bin_2channel_couplednorm(1.0),
     config={'init_pars': 2, 'par_bounds': 2},
 ):
     # 1 mu + 1 alpha
@@ -489,13 +576,23 @@ def setup_2bin_2channel_couplednorm(
 
 
 @pytest.fixture(scope='module')
-def source_2bin_2channel_coupledhisto():
+def setup_2bin_2channel_couplednorm():
+    return get_setup_2bin_2channel_couplednorm()
+
+
+def get_source_2bin_2channel_coupledhisto():
     with open('validation/data/2bin_2channel_coupledhisto.json') as read_json:
         return json.load(read_json)
 
 
 @pytest.fixture(scope='module')
-def spec_2bin_2channel_coupledhistosys(source=source_2bin_2channel_coupledhisto()):
+def source_2bin_2channel_coupledhisto():
+    return get_source_2bin_2channel_coupledhisto()
+
+
+def get_spec_2bin_2channel_coupledhistosys(
+    source=get_source_2bin_2channel_coupledhisto(),
+):
     spec = {
         'channels': [
             {
@@ -575,7 +672,11 @@ def spec_2bin_2channel_coupledhistosys(source=source_2bin_2channel_coupledhisto(
 
 
 @pytest.fixture(scope='module')
-def expected_result_2bin_2channel_coupledhistosys(mu=1.0):
+def spec_2bin_2channel_coupledhistosys():
+    return get_spec_2bin_2channel_coupledhistosys()
+
+
+def get_expected_result_2bin_2channel_coupledhistosys(mu=1.0):
     if mu == 1:
         expected_result = {
             "exp": [
@@ -591,11 +692,17 @@ def expected_result_2bin_2channel_coupledhistosys(mu=1.0):
 
 
 @pytest.fixture(scope='module')
-def setup_2bin_2channel_coupledhistosys(
-    source=source_2bin_2channel_coupledhisto(),
-    spec=spec_2bin_2channel_coupledhistosys(source_2bin_2channel_coupledhisto()),
+def expected_result_2bin_2channel_coupledhistosys():
+    return get_expected_result_2bin_2channel_coupledhistosys()
+
+
+def get_setup_2bin_2channel_coupledhistosys(
+    source=get_source_2bin_2channel_coupledhisto(),
+    spec=get_spec_2bin_2channel_coupledhistosys(
+        get_source_2bin_2channel_coupledhisto()
+    ),
     mu=1,
-    expected_result=expected_result_2bin_2channel_coupledhistosys(1.0),
+    expected_result=get_expected_result_2bin_2channel_coupledhistosys(1.0),
     config={'auxdata': 1, 'init_pars': 2, 'par_bounds': 2},
 ):
     # 1 mu 1 shared histosys
@@ -608,14 +715,22 @@ def setup_2bin_2channel_coupledhistosys(
 
 
 @pytest.fixture(scope='module')
-def source_2bin_2channel_coupledshapefactor():
+def setup_2bin_2channel_coupledhistosys():
+    return get_setup_2bin_2channel_coupledhistosys()
+
+
+def get_source_2bin_2channel_coupledshapefactor():
     with open('validation/data/2bin_2channel_coupledshapefactor.json') as read_json:
         return json.load(read_json)
 
 
 @pytest.fixture(scope='module')
-def spec_2bin_2channel_coupledshapefactor(
-    source=source_2bin_2channel_coupledshapefactor(),
+def source_2bin_2channel_coupledshapefactor():
+    return get_source_2bin_2channel_coupledshapefactor()
+
+
+def get_spec_2bin_2channel_coupledshapefactor(
+    source=get_source_2bin_2channel_coupledshapefactor(),
 ):
     spec = {
         'channels': [
@@ -664,7 +779,11 @@ def spec_2bin_2channel_coupledshapefactor(
 
 
 @pytest.fixture(scope='module')
-def expected_result_2bin_2channel_coupledshapefactor(mu=1.0):
+def spec_2bin_2channel_coupledshapefactor():
+    return get_spec_2bin_2channel_coupledshapefactor()
+
+
+def get_expected_result_2bin_2channel_coupledshapefactor(mu=1.0):
     if mu == 1:
         expected_result = {
             'obs': 0.5421679124909312,
@@ -680,13 +799,17 @@ def expected_result_2bin_2channel_coupledshapefactor(mu=1.0):
 
 
 @pytest.fixture(scope='module')
-def setup_2bin_2channel_coupledshapefactor(
-    source=source_2bin_2channel_coupledshapefactor(),
-    spec=spec_2bin_2channel_coupledshapefactor(
-        source_2bin_2channel_coupledshapefactor()
+def expected_result_2bin_2channel_coupledshapefactor():
+    return get_expected_result_2bin_2channel_coupledshapefactor()
+
+
+def get_setup_2bin_2channel_coupledshapefactor(
+    source=get_source_2bin_2channel_coupledshapefactor(),
+    spec=get_spec_2bin_2channel_coupledshapefactor(
+        get_source_2bin_2channel_coupledshapefactor()
     ),
     mu=1,
-    expected_result=expected_result_2bin_2channel_coupledshapefactor(1.0),
+    expected_result=get_expected_result_2bin_2channel_coupledshapefactor(1.0),
     config={'auxdata': 0, 'init_pars': 3, 'par_bounds': 3},
 ):
     # 1 mu 2 shared shapefactors
@@ -698,6 +821,11 @@ def setup_2bin_2channel_coupledshapefactor(
     }
 
 
+@pytest.fixture(scope='module')
+def setup_2bin_2channel_coupledshapefactor():
+    return get_setup_2bin_2channel_coupledshapefactor()
+
+
 def validate_hypotest(pdf, data, mu_test, expected_result, tolerance=1e-6):
     init_pars = pdf.config.suggested_init()
     par_bounds = pdf.config.suggested_bounds()
@@ -716,17 +844,16 @@ def validate_hypotest(pdf, data, mu_test, expected_result, tolerance=1e-6):
         assert abs(result - expected) / expected < tolerance, result
 
 
-@pytest.mark.parametrize(
-    'setup_and_tolerance',
-    [
-        (setup_1bin_shapesys(), 1e-6),
-        (setup_1bin_lumi(), 4e-6),
-        (setup_1bin_normsys(), 2e-9),
-        (setup_2bin_histosys(), 8e-5),
-        (setup_2bin_2channel(), 1e-6),
-        (setup_2bin_2channel_couplednorm(), 1e-6),
-        (setup_2bin_2channel_coupledhistosys(), 1e-6),
-        (setup_2bin_2channel_coupledshapefactor(), 2.5e-6),
+@pytest.fixture(
+    params=[
+        ('setup_1bin_shapesys', 1e-6),
+        ('setup_1bin_lumi', 4e-6),
+        ('setup_1bin_normsys', 2e-9),
+        ('setup_2bin_histosys', 8e-5),
+        ('setup_2bin_2channel', 1e-6),
+        ('setup_2bin_2channel_couplednorm', 1e-6),
+        ('setup_2bin_2channel_coupledhistosys', 1e-6),
+        ('setup_2bin_2channel_coupledshapefactor', 2.5e-6),
     ],
     ids=[
         '1bin_shapesys_mu1',
@@ -739,6 +866,10 @@ def validate_hypotest(pdf, data, mu_test, expected_result, tolerance=1e-6):
         '2bin_2channel_coupledshapefactor_mu1',
     ],
 )
+def setup_and_tolerance(request):
+    return (request.getfixturevalue(request.param[0]), request.param[1])
+
+
 def test_validation(setup_and_tolerance):
     setup, tolerance = setup_and_tolerance
     source = setup['source']