Repository: scikit-hep/pyhf

Issue:
create proper test fixtures
Many of the tests have large blocks of setup code that could be shared as proper `@pytest.fixtures` 

https://docs.pytest.org/en/latest/fixture.html


Discussion:

Resolving patch:
diff --git a/pyhf/__init__.py b/pyhf/__init__.py
--- a/pyhf/__init__.py
+++ b/pyhf/__init__.py
@@ -24,9 +24,10 @@ def get_backend():
     return tensorlib, optimizer
 
 # modifiers need access to tensorlib
-#   make sure import is below get_backend()
+# make sure import is below get_backend()
 from . import modifiers
 
+
 def set_backend(backend):
     """
     Set the backend and the associated optimizer
@@ -39,7 +40,9 @@ def set_backend(backend):
         None
 
     Example:
-        pyhf.set_backend(tensorflow_backend(session=tf.Session()))
+        import pyhf.tensor as tensor
+        import tensorflow as tf
+        pyhf.set_backend(tensor.tensorflow_backend(session=tf.Session()))
     """
     global tensorlib
     global optimizer
@@ -47,10 +50,10 @@ def set_backend(backend):
     tensorlib = backend
     if isinstance(tensorlib, tensor.tensorflow_backend):
         optimizer = optimize.tflow_optimizer(tensorlib)
-    elif isinstance(tensorlib,tensor.pytorch_backend):
+    elif isinstance(tensorlib, tensor.pytorch_backend):
         optimizer = optimize.pytorch_optimizer(tensorlib=tensorlib)
     # TODO: Add support for mxnet_optimizer()
-    # elif isinstance(tensorlib, mxnet_backend):
+    # elif isinstance(tensorlib, tensor.mxnet_backend):
     #     optimizer = mxnet_optimizer()
     else:
         optimizer = optimize.scipy_optimizer()


Test patch:
diff --git a/tests/benchmarks/test_benchmark.py b/tests/benchmarks/test_benchmark.py
--- a/tests/benchmarks/test_benchmark.py
+++ b/tests/benchmarks/test_benchmark.py
@@ -85,7 +85,7 @@ def runOnePoint(pdf, data):
                              pyhf.tensor.numpy_backend(poisson_from_normal=True),
                              pyhf.tensor.tensorflow_backend(session=tf.Session()),
                              pyhf.tensor.pytorch_backend(),
-                             # mxnet_backend(),
+                             # pyhf.tensor.mxnet_backend(),
                          ],
                          ids=[
                              'numpy',
@@ -113,10 +113,4 @@ def test_runOnePoint(benchmark, backend, n_bins):
                        source['bindata']['bkg'],
                        source['bindata']['bkgerr'])
     data = source['bindata']['data'] + pdf.config.auxdata
-    try:
-        assert benchmark(runOnePoint, pdf, data) is not None
-    except AssertionError:
-        print('benchmarking has failed for n_bins = {}'.formant(n_bins))
-        assert False
-
-    # Reset backend
+    assert benchmark(runOnePoint, pdf, data)
diff --git a/tests/test_optim.py b/tests/test_optim.py
--- a/tests/test_optim.py
+++ b/tests/test_optim.py
@@ -1,119 +1,25 @@
 import pyhf
 import tensorflow as tf
+import pytest
 
 
-def test_optim_numpy():
+@pytest.fixture(scope='module')
+def source():
     source = {
-      "binning": [2,-0.5,1.5],
-      "bindata": {
-        "data":    [120.0, 180.0],
-        "bkg":     [100.0, 150.0],
-        "bkgsys_up":  [102, 190],
-        "bkgsys_dn":  [98, 100],
-        "sig":     [30.0, 95.0]
-      }
+        'binning': [2, -0.5, 1.5],
+        'bindata': {
+            'data': [120.0, 180.0],
+            'bkg': [100.0, 150.0],
+            'bkgsys_up': [102, 190],
+            'bkgsys_dn': [98, 100],
+            'sig': [30.0, 95.0]
+        }
     }
-    spec = {
-        'channels': [
-            {
-                'name': 'singlechannel',
-                'samples': [
-                    {
-                        'name': 'signal',
-                        'data': source['bindata']['sig'],
-                        'modifiers': [
-                            {'name': 'mu', 'type': 'normfactor', 'data': None}
-                        ]
-                    },
-                    {
-                        'name': 'background',
-                        'data': source['bindata']['bkg'],
-                        'modifiers': [
-                            {'name': 'bkg_norm', 'type': 'histosys', 'data': {'lo_data': source['bindata']['bkgsys_dn'], 'hi_data': source['bindata']['bkgsys_up']}}
-                        ]
-                    }
-                ]
-            }
-        ]
-    }
-    pdf = pyhf.hfpdf(spec)
-    data = source['bindata']['data'] + pdf.config.auxdata
-
-    init_pars = pdf.config.suggested_init()
-    par_bounds = pdf.config.suggested_bounds()
-
-    pyhf.set_backend(pyhf.tensor.numpy_backend(poisson_from_normal=True))
-    optim = pyhf.optimizer
-
-    v1 = pdf.logpdf(init_pars, data)
-    result = optim.unconstrained_bestfit(pyhf.loglambdav, data, pdf, init_pars, par_bounds)
-    assert pyhf.tensorlib.tolist(result)
-
-    result = optim.constrained_bestfit(pyhf.loglambdav, 1.0, data, pdf, init_pars, par_bounds)
-    assert pyhf.tensorlib.tolist(result)
-
-
-def test_optim_pytorch():
-    source = {
-      "binning": [2,-0.5,1.5],
-      "bindata": {
-        "data":    [120.0, 180.0],
-        "bkg":     [100.0, 150.0],
-        "bkgsys_up":  [102, 190],
-        "bkgsys_dn":  [98, 100],
-        "sig":     [30.0, 95.0]
-      }
-    }
-    spec = {
-        'channels': [
-            {
-                'name': 'singlechannel',
-                'samples': [
-                    {
-                        'name': 'signal',
-                        'data': source['bindata']['sig'],
-                        'modifiers': [
-                            {'name': 'mu', 'type': 'normfactor', 'data': None}
-                        ]
-                    },
-                    {
-                        'name': 'background',
-                        'data': source['bindata']['bkg'],
-                        'modifiers': [
-                            {'name': 'bkg_norm', 'type': 'histosys', 'data': {'lo_data': source['bindata']['bkgsys_dn'], 'hi_data': source['bindata']['bkgsys_up']}}
-                        ]
-                    }
-                ]
-            }
-        ]
-    }
-    pdf = pyhf.hfpdf(spec)
-    data = source['bindata']['data'] + pdf.config.auxdata
+    return source
 
-    init_pars = pdf.config.suggested_init()
-    par_bounds = pdf.config.suggested_bounds()
 
-    pyhf.set_backend(pyhf.tensor.pytorch_backend(poisson_from_normal=True))
-    optim = pyhf.optimizer
-
-    result = optim.unconstrained_bestfit(pyhf.loglambdav, data, pdf, init_pars, par_bounds)
-    assert pyhf.tensorlib.tolist(result)
-
-    result = optim.constrained_bestfit(pyhf.loglambdav, 1.0, data, pdf, init_pars, par_bounds)
-    assert pyhf.tensorlib.tolist(result)
-
-
-def test_optim_tflow():
-    source = {
-      "binning": [2,-0.5,1.5],
-      "bindata": {
-        "data":    [120.0, 180.0],
-        "bkg":     [100.0, 150.0],
-        "bkgsys_up":  [102, 190],
-        "bkgsys_dn":  [98, 100],
-        "sig":     [30.0, 95.0]
-      }
-    }
+@pytest.fixture(scope='module')
+def spec(source):
     spec = {
         'channels': [
             {
@@ -123,32 +29,71 @@ def test_optim_tflow():
                         'name': 'signal',
                         'data': source['bindata']['sig'],
                         'modifiers': [
-                            {'name': 'mu', 'type': 'normfactor', 'data': None}
+                            {
+                                'name': 'mu',
+                                'type': 'normfactor',
+                                'data': None
+                            }
                         ]
                     },
                     {
                         'name': 'background',
                         'data': source['bindata']['bkg'],
                         'modifiers': [
-                            {'name': 'bkg_norm', 'type': 'histosys', 'data': {'lo_data': source['bindata']['bkgsys_dn'], 'hi_data': source['bindata']['bkgsys_up']}}
+                            {
+                                'name': 'bkg_norm',
+                                'type': 'histosys',
+                                'data': {
+                                    'lo_data': source['bindata']['bkgsys_dn'],
+                                    'hi_data': source['bindata']['bkgsys_up']
+                                }
+                            }
                         ]
                     }
                 ]
             }
         ]
     }
+    return spec
+
+
+@pytest.mark.parametrize('mu',
+                         [
+                             1.,
+                         ],
+                         ids=[
+                             'mu=1',
+                         ])
+@pytest.mark.parametrize('backend',
+                         [
+                             pyhf.tensor.numpy_backend(poisson_from_normal=True),
+                             pyhf.tensor.tensorflow_backend(session=tf.Session()),
+                             pyhf.tensor.pytorch_backend(poisson_from_normal=True),
+                             # pyhf.tensor.mxnet_backend(),
+                         ],
+                         ids=[
+                             'numpy',
+                             'tensorflow',
+                             'pytorch',
+                             # 'mxnet',
+                         ])
+def test_optim(source, spec, mu, backend):
     pdf = pyhf.hfpdf(spec)
     data = source['bindata']['data'] + pdf.config.auxdata
 
     init_pars = pdf.config.suggested_init()
     par_bounds = pdf.config.suggested_bounds()
 
-    pyhf.set_backend(pyhf.tensor.tensorflow_backend())
-    pyhf.tensorlib.session = tf.Session()
+    pyhf.set_backend(backend)
     optim = pyhf.optimizer
+    if isinstance(pyhf.tensorlib, pyhf.tensor.tensorflow_backend):
+        tf.reset_default_graph()
+        pyhf.tensorlib.session = tf.Session()
 
-    result = optim.unconstrained_bestfit(pyhf.loglambdav, data, pdf, init_pars, par_bounds)
+    result = optim.unconstrained_bestfit(
+        pyhf.loglambdav, data, pdf, init_pars, par_bounds)
     assert pyhf.tensorlib.tolist(result)
 
-    result = optim.constrained_bestfit(pyhf.loglambdav, 1.0, data, pdf, init_pars, par_bounds)
+    result = optim.constrained_bestfit(
+        pyhf.loglambdav, mu, data, pdf, init_pars, par_bounds)
     assert pyhf.tensorlib.tolist(result)
diff --git a/tests/test_validation.py b/tests/test_validation.py
--- a/tests/test_validation.py
+++ b/tests/test_validation.py
@@ -1,61 +1,17 @@
 import pyhf
-import pyhf.simplemodels
-import numpy as np
-import json
 
-VALIDATION_TOLERANCE = 1e-5
-
-def test_validation_1bin_shapesys():
-    expected_result = {
-        'obs': 0.4541865416107029,
-        'exp': [
-            0.06371799398864626,
-            0.15096503398048894,
-            0.3279606950533305,
-            0.6046087303039118,
-            0.8662627605298466
-        ]
-    }
+import json
+import jsonschema
+import pytest
 
 
-    source = json.load(open('validation/data/1bin_example1.json'))
-    pdf  = pyhf.simplemodels.hepdata_like(source['bindata']['sig'], source['bindata']['bkg'], source['bindata']['bkgerr'])
+@pytest.fixture(scope='module')
+def source_1bin_example1():
+    return json.load(open('validation/data/1bin_example1.json'))
 
-    data = source['bindata']['data'] + pdf.config.auxdata
-    muTest = 1.0
 
-    assert len(pdf.config.suggested_init()) == 2
-    assert len(pdf.config.suggested_bounds()) == 2
-    init_pars  = pdf.config.suggested_init()
-    par_bounds = pdf.config.suggested_bounds()
-
-    clsobs, cls_exp = pyhf.runOnePoint(muTest, data,pdf,init_pars,par_bounds)[-2:]
-    cls_obs = 1./clsobs
-    cls_exp = [1./x for x in cls_exp]
-    assert (cls_obs - expected_result['obs'])/expected_result['obs'] < VALIDATION_TOLERANCE
-    for result,expected_result in zip(cls_exp, expected_result['exp']):
-        assert (result-expected_result)/expected_result < VALIDATION_TOLERANCE
-
-
-def test_validation_1bin_normsys():
-    expected_result = {
-        'obs': 0.0007930094233140433,
-        'exp': [
-            1.2529050370718884e-09,
-            8.932001833559302e-08,
-            5.3294967286010575e-06,
-            0.00022773982308763686,
-            0.0054897420571466075
-        ]
-    }
-    source = {
-      "binning": [2,-0.5,1.5],
-      "bindata": {
-        "data":    [120.0, 180.0],
-        "bkg":     [100.0, 150.0],
-        "sig":     [30.0, 95.0]
-      }
-    }
+@pytest.fixture(scope='module')
+def spec_1bin_shapesys(source=source_1bin_example1()):
     spec = {
         'channels': [
             {
@@ -65,52 +21,81 @@ def test_validation_1bin_normsys():
                         'name': 'signal',
                         'data': source['bindata']['sig'],
                         'modifiers': [
-                            {'name': 'mu', 'type': 'normfactor', 'data': None}
+                            {
+                                'name': 'mu',
+                                'type': 'normfactor',
+                                'data': None
+                            }
                         ]
                     },
                     {
                         'name': 'background',
                         'data': source['bindata']['bkg'],
                         'modifiers': [
-                            {'name': 'bkg_norm', 'type': 'normsys', 'data': {'lo': 0.90, 'hi': 1.10}}
+                            {
+                                'name': 'uncorr_bkguncrt',
+                                'type': 'shapesys',
+                                'data': source['bindata']['bkgerr']
+                            }
                         ]
                     }
                 ]
             }
         ]
     }
-    pdf  = pyhf.hfpdf(spec)
-
-    data = source['bindata']['data'] + pdf.config.auxdata
-
-    muTest = 1.0
-
-    assert len(pdf.config.suggested_init()) == 2
-    assert len(pdf.config.suggested_bounds()) == 2
-    init_pars  = pdf.config.suggested_init()
-    par_bounds = pdf.config.suggested_bounds()
+    schema = json.load(open('validation/spec.json'))
+    jsonschema.validate(spec, schema)
+    return spec
+
+
+@pytest.fixture(scope='module')
+def expected_result_1bin_shapesys(mu=1.):
+    if mu == 1:
+        expected_result = {
+            'obs': 0.4541865416107029,
+            'exp': [
+                0.06371799398864626,
+                0.15096503398048894,
+                0.3279606950533305,
+                0.6046087303039118,
+                0.8662627605298466
+            ]
+        }
+    return expected_result
+
+
+@pytest.fixture(scope='module')
+def setup_1bin_shapesys(source=source_1bin_example1(),
+                        spec=spec_1bin_shapesys(source_1bin_example1()),
+                        mu=1,
+                        expected_result=expected_result_1bin_shapesys(1.),
+                        config={'init_pars': 2, 'par_bounds': 2}):
+    return {
+        'source': source,
+        'spec': spec,
+        'mu': mu,
+        'expected': {
+            'result': expected_result,
+            'config': config
+        }
+    }
 
 
-    clsobs, cls_exp = pyhf.runOnePoint(muTest, data,pdf,init_pars,par_bounds)[-2:]
-    cls_obs = 1./clsobs
-    cls_exp = [1./x for x in cls_exp]
-    assert (cls_obs - expected_result['obs'])/expected_result['obs'] < VALIDATION_TOLERANCE
-    for result,expected_result in zip(cls_exp, expected_result['exp']):
-        assert (result-expected_result)/expected_result < VALIDATION_TOLERANCE
+@pytest.fixture(scope='module')
+def source_1bin_normsys():
+    source = {
+        'binning': [2, -0.5, 1.5],
+        'bindata': {
+            'data': [120.0, 180.0],
+            'bkg': [100.0, 150.0],
+            'sig': [30.0, 95.0]
+        }
+    }
+    return source
 
 
-def test_validation_2bin_histosys():
-    expected_result = {
-        'obs': 0.10014623469489856,
-        'exp': [
-            8.131143652258812e-06,
-            0.0001396307700293439,
-            0.0020437905684851376,
-            0.022094931468776054,
-            0.14246926685789288,
-        ]
-    }
-    source = json.load(open('validation/data/2bin_histosys_example2.json'))
+@pytest.fixture(scope='module')
+def spec_1bin_normsys(source=source_1bin_normsys()):
     spec = {
         'channels': [
             {
@@ -120,54 +105,154 @@ def test_validation_2bin_histosys():
                         'name': 'signal',
                         'data': source['bindata']['sig'],
                         'modifiers': [
-                            {'name': 'mu', 'type': 'normfactor', 'data': None}
+                            {
+                                'name': 'mu',
+                                'type': 'normfactor',
+                                'data': None
+                            }
                         ]
                     },
                     {
                         'name': 'background',
                         'data': source['bindata']['bkg'],
                         'modifiers': [
-                            {'name': 'bkg_norm', 'type': 'histosys', 'data': {'lo_data': source['bindata']['bkgsys_dn'], 'hi_data': source['bindata']['bkgsys_up']}}
+                            {
+                                'name': 'bkg_norm',
+                                'type': 'normsys',
+                                'data': {'lo': 0.90, 'hi': 1.10}
+                            }
                         ]
                     }
                 ]
             }
         ]
     }
-    pdf  = pyhf.hfpdf(spec)
+    schema = json.load(open('validation/spec.json'))
+    jsonschema.validate(spec, schema)
+    return spec
+
+
+@pytest.fixture(scope='module')
+def expected_result_1bin_normsys(mu=1.):
+    if mu == 1:
+        expected_result = {
+            'obs': 0.0007930094233140433,
+            'exp': [
+                1.2529050370718884e-09,
+                8.932001833559302e-08,
+                5.3294967286010575e-06,
+                0.00022773982308763686,
+                0.0054897420571466075
+            ]
+        }
+    return expected_result
+
+
+@pytest.fixture(scope='module')
+def setup_1bin_normsys(source=source_1bin_normsys(),
+                       spec=spec_1bin_normsys(source_1bin_normsys()),
+                       mu=1,
+                       expected_result=expected_result_1bin_normsys(1.),
+                       config={'init_pars': 2, 'par_bounds': 2}):
+    return {
+        'source': source,
+        'spec': spec,
+        'mu': mu,
+        'expected': {
+            'result': expected_result,
+            'config': config
+        }
+    }
 
-    data = source['bindata']['data'] + pdf.config.auxdata
 
-    muTest = 1.0
+@pytest.fixture(scope='module')
+def source_2bin_histosys_example2():
+    return json.load(open('validation/data/2bin_histosys_example2.json'))
 
-    assert len(pdf.config.suggested_init()) == 2
-    assert len(pdf.config.suggested_bounds()) == 2
-    init_pars  = pdf.config.suggested_init()
-    par_bounds = pdf.config.suggested_bounds()
 
+@pytest.fixture(scope='module')
+def spec_2bin_histosys(source=source_2bin_histosys_example2()):
+    spec = {
+        'channels': [
+            {
+                'name': 'singlechannel',
+                'samples': [
+                    {
+                        'name': 'signal',
+                        'data': source['bindata']['sig'],
+                        'modifiers': [
+                            {
+                                'name': 'mu',
+                                'type': 'normfactor',
+                                'data': None
+                            }
+                        ]
+                    },
+                    {
+                        'name': 'background',
+                        'data': source['bindata']['bkg'],
+                        'modifiers': [
+                            {
+                                'name': 'bkg_norm',
+                                'type': 'histosys',
+                                'data': {
+                                    'lo_data': source['bindata']['bkgsys_dn'],
+                                    'hi_data': source['bindata']['bkgsys_up']
+                                }
+                            }
+                        ]
+                    }
+                ]
+            }
+        ]
+    }
+    schema = json.load(open('validation/spec.json'))
+    jsonschema.validate(spec, schema)
+    return spec
+
+
+@pytest.fixture(scope='module')
+def expected_result_2bin_histosys(mu=1):
+    if mu == 1:
+        expected_result = {
+            'obs': 0.10014623469489856,
+            'exp': [
+                8.131143652258812e-06,
+                0.0001396307700293439,
+                0.0020437905684851376,
+                0.022094931468776054,
+                0.14246926685789288,
+            ]
+        }
+    return expected_result
+
+
+@pytest.fixture(scope='module')
+def setup_2bin_histosys(source=source_2bin_histosys_example2(),
+                        spec=spec_2bin_histosys(
+                            source_2bin_histosys_example2()),
+                        mu=1,
+                        expected_result=expected_result_2bin_histosys(1.),
+                        config={'init_pars': 2, 'par_bounds': 2}):
+    return {
+        'source': source,
+        'spec': spec,
+        'mu': mu,
+        'expected': {
+            'result': expected_result,
+            'config': config
+        }
+    }
 
-    clsobs, cls_exp = pyhf.runOnePoint(muTest, data,pdf,init_pars,par_bounds)[-2:]
-    cls_obs = 1./clsobs
-    cls_exp = [1./x for x in cls_exp]
-    assert (cls_obs - expected_result['obs'])/expected_result['obs'] < VALIDATION_TOLERANCE
-    for result,expected_result in zip(cls_exp, expected_result['exp']):
-        assert (result-expected_result)/expected_result < VALIDATION_TOLERANCE
 
+@pytest.fixture(scope='module')
+def source_2bin_2channel_example1():
+    return json.load(open('validation/data/2bin_2channel_example1.json'))
 
 
-def test_validation_2bin_2channel():
-    expected_result = {
-        'obs': 0.05691881515460979,
-        'exp': [
-            0.0004448774256747925,
-            0.0034839534635069816,
-            0.023684793938725246,
-            0.12294326553585197,
-            0.4058143629613449
-        ]
-    }
-    source = json.load(open('validation/data/2bin_2channel_example1.json'))
-    spec =  {
+@pytest.fixture(scope='module')
+def spec_2bin_2channel(source=source_2bin_2channel_example1()):
+    spec = {
         'channels': [
             {
                 'name': 'signal',
@@ -176,14 +261,22 @@ def test_validation_2bin_2channel():
                         'name': 'signal',
                         'data': source['channels']['signal']['bindata']['sig'],
                         'modifiers': [
-                            {'name': 'mu', 'type': 'normfactor', 'data': None}
+                            {
+                                'name': 'mu',
+                                'type': 'normfactor',
+                                'data': None
+                            }
                         ]
                     },
                     {
                         'name': 'background',
                         'data': source['channels']['signal']['bindata']['bkg'],
                         'modifiers': [
-                            {'name': 'uncorr_bkguncrt_signal', 'type': 'shapesys', 'data': source['channels']['signal']['bindata']['bkgerr']}
+                            {
+                                'name': 'uncorr_bkguncrt_signal',
+                                'type': 'shapesys',
+                                'data': source['channels']['signal']['bindata']['bkgerr']
+                            }
                         ]
                     }
                 ]
@@ -195,47 +288,65 @@ def test_validation_2bin_2channel():
                         'name': 'background',
                         'data': source['channels']['control']['bindata']['bkg'],
                         'modifiers': [
-                            {'name': 'uncorr_bkguncrt_control', 'type': 'shapesys', 'data': source['channels']['control']['bindata']['bkgerr']}
+                            {
+                                'name': 'uncorr_bkguncrt_control',
+                                'type': 'shapesys',
+                                'data': source['channels']['control']['bindata']['bkgerr']
+                            }
                         ]
                     }
                 ]
             }
         ]
     }
-    pdf  = pyhf.hfpdf(spec)
-    data = []
-    for c in pdf.spec['channels']:
-        data += source['channels'][c['name']]['bindata']['data']
-    data = data + pdf.config.auxdata
-
-    muTest = 1.0
-
-    assert len(pdf.config.suggested_init())   == 5 # 1 mu + 2 gammas for 2 channels each
-    assert len(pdf.config.suggested_bounds()) == 5
-    init_pars  = pdf.config.suggested_init()
-    par_bounds = pdf.config.suggested_bounds()
+    schema = json.load(open('validation/spec.json'))
+    jsonschema.validate(spec, schema)
+    return spec
+
+
+@pytest.fixture(scope='module')
+def expected_result_2bin_2channel(mu=1.):
+    if mu == 1:
+        expected_result = {
+            'obs': 0.05691881515460979,
+            'exp': [
+                0.0004448774256747925,
+                0.0034839534635069816,
+                0.023684793938725246,
+                0.12294326553585197,
+                0.4058143629613449
+            ]
+        }
+    return expected_result
+
+
+@pytest.fixture(scope='module')
+def setup_2bin_2channel(source=source_2bin_2channel_example1(),
+                        spec=spec_2bin_2channel(
+                            source_2bin_2channel_example1()),
+                        mu=1,
+                        expected_result=expected_result_2bin_2channel(1.),
+                        config={'init_pars': 5, 'par_bounds': 5}):
+    # 1 mu + 2 gammas for 2 channels each
+    return {
+        'source': source,
+        'spec': spec,
+        'mu': mu,
+        'expected': {
+            'result': expected_result,
+            'config': config
+        }
+    }
 
-    clsobs, cls_exp = pyhf.runOnePoint(muTest, data,pdf,init_pars,par_bounds)[-2:]
-    cls_obs = 1./clsobs
-    cls_exp = [1./x for x in cls_exp]
-    assert (cls_obs - expected_result['obs'])/expected_result['obs'] < VALIDATION_TOLERANCE
-    for result,expected_result in zip(cls_exp, expected_result['exp']):
-        assert (result-expected_result)/expected_result < VALIDATION_TOLERANCE
 
+@pytest.fixture(scope='module')
+def source_2bin_2channel_couplednorm():
+    return json.load(open('validation/data/2bin_2channel_couplednorm.json'))
 
 
-def test_validation_2bin_2channel_couplednorm():
-    expected_result = {
-        'obs': 0.5999662863185762,
-        'exp': [0.06596134134354742,
-          0.15477912571478988,
-          0.33323967895587736,
-          0.6096429330789306,
-          0.8688213053042003
-        ]
-    }
-    source = json.load(open('validation/data/2bin_2channel_couplednorm.json'))
-    spec =  {
+@pytest.fixture(scope='module')
+def spec_2bin_2channel_couplednorm(source=source_2bin_2channel_couplednorm()):
+    spec = {
         'channels': [
             {
                 'name': 'signal',
@@ -244,21 +355,33 @@ def test_validation_2bin_2channel_couplednorm():
                         'name': 'signal',
                         'data': source['channels']['signal']['bindata']['sig'],
                         'modifiers': [
-                            {'name': 'mu', 'type': 'normfactor', 'data': None}
+                            {
+                                'name': 'mu',
+                                'type': 'normfactor',
+                                'data': None
+                            }
                         ]
                     },
                     {
                         'name': 'bkg1',
                         'data': source['channels']['signal']['bindata']['bkg1'],
                         'modifiers': [
-                            {'name': 'coupled_normsys', 'type': 'normsys', 'data':  {'lo': 0.9, 'hi': 1.1}}
+                            {
+                                'name': 'coupled_normsys',
+                                'type': 'normsys',
+                                'data':  {'lo': 0.9, 'hi': 1.1}
+                            }
                         ]
                     },
                     {
                         'name': 'bkg2',
                         'data': source['channels']['signal']['bindata']['bkg2'],
                         'modifiers': [
-                            {'name': 'coupled_normsys', 'type': 'normsys', 'data':  {'lo': 0.5, 'hi': 1.5}}
+                            {
+                                'name': 'coupled_normsys',
+                                'type': 'normsys',
+                                'data':  {'lo': 0.5, 'hi': 1.5}
+                            }
                         ]
                     }
                 ]
@@ -270,48 +393,66 @@ def test_validation_2bin_2channel_couplednorm():
                         'name': 'background',
                         'data': source['channels']['control']['bindata']['bkg1'],
                         'modifiers': [
-                            {'name': 'coupled_normsys', 'type': 'normsys', 'data': {'lo': 0.9, 'hi': 1.1}}
+                            {
+                                'name': 'coupled_normsys',
+                                'type': 'normsys',
+                                'data': {'lo': 0.9, 'hi': 1.1}
+                            }
                         ]
                     }
                 ]
             }
         ]
     }
-    pdf  = pyhf.hfpdf(spec)
-    data = []
-    for c in pdf.spec['channels']:
-        data += source['channels'][c['name']]['bindata']['data']
-    data = data + pdf.config.auxdata
+    schema = json.load(open('validation/spec.json'))
+    jsonschema.validate(spec, schema)
+    return spec
+
+
+@pytest.fixture(scope='module')
+def expected_result_2bin_2channel_couplednorm(mu=1.):
+    if mu == 1:
+        expected_result = {
+            'obs': 0.5999662863185762,
+            'exp': [
+                0.06596134134354742,
+                0.15477912571478988,
+                0.33323967895587736,
+                0.6096429330789306,
+                0.8688213053042003
+            ]
+        }
+    return expected_result
+
+
+@pytest.fixture(scope='module')
+def setup_2bin_2channel_couplednorm(
+        source=source_2bin_2channel_couplednorm(),
+        spec=spec_2bin_2channel_couplednorm(
+            source_2bin_2channel_couplednorm()),
+        mu=1,
+        expected_result=expected_result_2bin_2channel_couplednorm(1.),
+        config={'init_pars': 2, 'par_bounds': 2}):
+    # 1 mu + 1 alpha
+    return {
+        'source': source,
+        'spec': spec,
+        'mu': mu,
+        'expected': {
+            'result': expected_result,
+            'config': config
+        }
+    }
 
 
-    muTest = 1.0
-    assert len(pdf.config.suggested_init())   == 2 # 1 mu + 1 alpha
-    assert len(pdf.config.suggested_bounds()) == 2 # 1 mu + 1 alpha
-    init_pars  = pdf.config.suggested_init()
-    par_bounds = pdf.config.suggested_bounds()
+@pytest.fixture(scope='module')
+def source_2bin_2channel_coupledhisto():
+    return json.load(open('validation/data/2bin_2channel_coupledhisto.json'))
 
-    clsobs, cls_exp = pyhf.runOnePoint(muTest, data,pdf,init_pars,par_bounds)[-2:]
-    cls_obs = 1./clsobs
-    cls_exp = [1./x for x in cls_exp]
-    assert (cls_obs - expected_result['obs'])/expected_result['obs'] < VALIDATION_TOLERANCE
-    for result,expected_result in zip(cls_exp, expected_result['exp']):
-        assert (result-expected_result)/expected_result < VALIDATION_TOLERANCE
-
-
-
-def test_validation_2bin_2channel_coupledhistosys():
-    expected_result = {
-    'obs': 0.0796739833305826,
-     'exp': [
-        1.765372502072074e-05,
-        0.00026265618793683054,
-        0.003340033567379219,
-        0.03152233566143051,
-        0.17907736639946248
-    ]
-    }
-    source = json.load(open('validation/data/2bin_2channel_coupledhisto.json'))
-    spec   =  {
+
+@pytest.fixture(scope='module')
+def spec_2bin_2channel_coupledhistosys(source=source_2bin_2channel_coupledhisto()):
+    spec = {
         'channels': [
             {
                 'name': 'signal',
@@ -320,21 +461,39 @@ def test_validation_2bin_2channel_coupledhistosys():
                         'name': 'signal',
                         'data': source['channels']['signal']['bindata']['sig'],
                         'modifiers': [
-                            {'name': 'mu', 'type': 'normfactor', 'data': None}
+                            {
+                                'name': 'mu',
+                                'type': 'normfactor',
+                                'data': None
+                            }
                         ]
                     },
                     {
                         'name': 'bkg1',
                         'data': source['channels']['signal']['bindata']['bkg1'],
                         'modifiers': [
-                            {'name': 'coupled_histosys','type': 'histosys', 'data': {'lo_data': source['channels']['signal']['bindata']['bkg1_dn'], 'hi_data': source['channels']['signal']['bindata']['bkg1_up']}}
+                            {
+                                'name': 'coupled_histosys',
+                                'type': 'histosys',
+                                'data': {
+                                    'lo_data': source['channels']['signal']['bindata']['bkg1_dn'],
+                                    'hi_data': source['channels']['signal']['bindata']['bkg1_up']
+                                }
+                            }
                         ]
                     },
                     {
                         'name': 'bkg2',
                         'data': source['channels']['signal']['bindata']['bkg2'],
                         'modifiers': [
-                            {'name': 'coupled_histosys', 'type': 'histosys', 'data': {'lo_data': source['channels']['signal']['bindata']['bkg2_dn'], 'hi_data': source['channels']['signal']['bindata']['bkg2_up']}}
+                            {
+                                'name': 'coupled_histosys',
+                                'type': 'histosys',
+                                'data': {
+                                    'lo_data': source['channels']['signal']['bindata']['bkg2_dn'],
+                                    'hi_data': source['channels']['signal']['bindata']['bkg2_up']
+                                }
+                            }
                         ]
                     }
                 ]
@@ -346,48 +505,69 @@ def test_validation_2bin_2channel_coupledhistosys():
                         'name': 'background',
                         'data': source['channels']['control']['bindata']['bkg1'],
                         'modifiers': [
-                            {'name': 'coupled_histosys', 'type': 'histosys', 'data': {'lo_data': source['channels']['control']['bindata']['bkg1_dn'], 'hi_data': source['channels']['control']['bindata']['bkg1_up']}}
+                            {
+                                'name': 'coupled_histosys',
+                                'type': 'histosys',
+                                'data': {
+                                    'lo_data': source['channels']['control']['bindata']['bkg1_dn'],
+                                    'hi_data': source['channels']['control']['bindata']['bkg1_up']
+                                }
+                            }
                         ]
                     }
                 ]
             }
         ]
     }
-    pdf  = pyhf.hfpdf(spec)
-    data = []
-    for c in pdf.spec['channels']:
-        data += source['channels'][c['name']]['bindata']['data']
-    data = data + pdf.config.auxdata
+    schema = json.load(open('validation/spec.json'))
+    jsonschema.validate(spec, schema)
+    return spec
+
+
+@pytest.fixture(scope='module')
+def expected_result_2bin_2channel_coupledhistosys(mu=1.):
+    if mu == 1:
+        expected_result = {
+            'obs': 0.0796739833305826,
+            'exp': [
+                1.765372502072074e-05,
+                0.00026265618793683054,
+                0.003340033567379219,
+                0.03152233566143051,
+                0.17907736639946248
+            ]
+        }
+    return expected_result
+
+
+@pytest.fixture(scope='module')
+def setup_2bin_2channel_coupledhistosys(
+        source=source_2bin_2channel_coupledhisto(),
+        spec=spec_2bin_2channel_coupledhistosys(
+            source_2bin_2channel_coupledhisto()),
+        mu=1,
+        expected_result=expected_result_2bin_2channel_coupledhistosys(1.),
+        config={'auxdata': 1, 'init_pars': 2, 'par_bounds': 2}):
+    # 1 mu 1 shared histosys
+    return {
+        'source': source,
+        'spec': spec,
+        'mu': mu,
+        'expected': {
+            'result': expected_result,
+            'config': config
+        }
+    }
 
-    init_pars = pdf.config.suggested_init()
-    par_bounds = pdf.config.suggested_bounds()
 
-    assert len(pdf.config.auxdata) == 1
-    assert len(init_pars)  == 2 #1 mu 1 shared histosys
-    assert len(par_bounds) == 2
-
-    muTest = 1.0
-    clsobs, cls_exp = pyhf.runOnePoint(muTest, data,pdf,init_pars,par_bounds)[-2:]
-    cls_obs = 1./clsobs
-    cls_exp = [1./x for x in cls_exp]
-    assert (cls_obs - expected_result['obs'])/expected_result['obs'] < VALIDATION_TOLERANCE
-    for result,expected_result in zip(cls_exp, expected_result['exp']):
-        assert (result-expected_result)/expected_result < VALIDATION_TOLERANCE
-
-
-def test_validation_2bin_2channel_coupledshapefactor():
-    expected_result = {
-    'obs': 0.5421679124909312,
-     'exp': [
-        0.013753299929451691,
-        0.048887400056355966,
-        0.15555296253957684,
-        0.4007561343326305,
-        0.7357169630955912
-        ]
-    }
-    source = json.load(open('validation/data/2bin_2channel_coupledshapefactor.json'))
-    spec =  {
+@pytest.fixture(scope='module')
+def source_2bin_2channel_coupledshapefactor():
+    return json.load(open('validation/data/2bin_2channel_coupledshapefactor.json'))
+
+
+@pytest.fixture(scope='module')
+def spec_2bin_2channel_coupledshapefactor(source=source_2bin_2channel_coupledshapefactor()):
+    spec = {
         'channels': [
             {
                 'name': 'signal',
@@ -396,14 +576,22 @@ def test_validation_2bin_2channel_coupledshapefactor():
                         'name': 'signal',
                         'data': source['channels']['signal']['bindata']['sig'],
                         'modifiers': [
-                            {'name': 'mu', 'type': 'normfactor', 'data': None}
+                            {
+                                'name': 'mu',
+                                'type': 'normfactor',
+                                'data': None
+                            }
                         ]
                     },
                     {
                         'name': 'bkg1',
                         'data': source['channels']['signal']['bindata']['bkg1'],
                         'modifiers': [
-                            {'name': 'coupled_shapefactor', 'type': 'shapefactor', 'data': None}
+                            {
+                                'name': 'coupled_shapefactor',
+                                'type': 'shapefactor',
+                                'data': None
+                            }
                         ]
                     }
                 ]
@@ -415,30 +603,109 @@ def test_validation_2bin_2channel_coupledshapefactor():
                         'name': 'background',
                         'data': source['channels']['control']['bindata']['bkg1'],
                         'modifiers': [
-                            {'name': 'coupled_shapefactor', 'type': 'shapefactor', 'data': None}
+                            {
+                                'name': 'coupled_shapefactor',
+                                'type': 'shapefactor',
+                                'data': None
+                            }
                         ]
                     }
                 ]
             }
         ]
     }
-    pdf  = pyhf.hfpdf(spec)
-    data = []
-    for c in pdf.spec['channels']:
-        data += source['channels'][c['name']]['bindata']['data']
-    data = data + pdf.config.auxdata
+    schema = json.load(open('validation/spec.json'))
+    jsonschema.validate(spec, schema)
+    return spec
+
+
+@pytest.fixture(scope='module')
+def expected_result_2bin_2channel_coupledshapefactor(mu=1.):
+    if mu == 1:
+        expected_result = {
+            'obs': 0.5421679124909312,
+            'exp': [
+                0.013753299929451691,
+                0.048887400056355966,
+                0.15555296253957684,
+                0.4007561343326305,
+                0.7357169630955912
+            ]
+        }
+    return expected_result
+
+
+@pytest.fixture(scope='module')
+def setup_2bin_2channel_coupledshapefactor(
+        source=source_2bin_2channel_coupledshapefactor(),
+        spec=spec_2bin_2channel_coupledshapefactor(
+            source_2bin_2channel_coupledshapefactor()),
+        mu=1,
+        expected_result=expected_result_2bin_2channel_coupledshapefactor(1.),
+        config={'auxdata': 0, 'init_pars': 3, 'par_bounds': 3}):
+    # 1 mu 2 shared shapefactors
+    return {
+        'source': source,
+        'spec': spec,
+        'mu': mu,
+        'expected': {
+            'result': expected_result,
+            'config': config
+        }
+    }
+
 
+def validate_runOnePoint(pdf, data, mu_test, expected_result, tolerance=1e-5):
     init_pars = pdf.config.suggested_init()
     par_bounds = pdf.config.suggested_bounds()
 
-    assert len(pdf.config.auxdata) == 0
-    assert len(init_pars)  == 3 #1 mu 2 shared shapefactors
-    assert len(par_bounds) == 3
-
-    muTest = 1.0
-    clsobs, cls_exp = pyhf.runOnePoint(muTest, data,pdf,init_pars,par_bounds)[-2:]
-    cls_obs = 1./clsobs
-    cls_exp = [1./x for x in cls_exp]
-    assert (cls_obs - expected_result['obs'])/expected_result['obs'] < VALIDATION_TOLERANCE
-    for result,expected_result in zip(cls_exp, expected_result['exp']):
-        assert (result-expected_result)/expected_result < VALIDATION_TOLERANCE
+    CLs_obs, CLs_exp = pyhf.runOnePoint(
+        mu_test, data, pdf, init_pars, par_bounds)[-2:]
+    CLs_obs = 1. / CLs_obs
+    CLs_exp = [1. / x for x in CLs_exp]
+    assert (CLs_obs - expected_result['obs']) / \
+        expected_result['obs'] < tolerance
+    for result, expected_result in zip(CLs_exp, expected_result['exp']):
+        assert (result - expected_result) / \
+            expected_result < tolerance
+
+
+@pytest.mark.parametrize('setup', [
+    setup_1bin_shapesys(),
+    setup_1bin_normsys(),
+    setup_2bin_histosys(),
+    setup_2bin_2channel(),
+    setup_2bin_2channel_couplednorm(),
+    setup_2bin_2channel_coupledhistosys(),
+    setup_2bin_2channel_coupledshapefactor()
+],
+    ids=[
+    '1bin_shapesys_mu1',
+    '1bin_normsys_mu1',
+    '2bin_histosys_mu1',
+    '2bin_2channel_mu1',
+    '2bin_2channel_couplednorm_mu1',
+    '2bin_2channel_coupledhistosys_mu1',
+    '2bin_2channel_coupledshapefactor_mu1'
+])
+def test_validation(setup):
+    source = setup['source']
+    pdf = pyhf.hfpdf(setup['spec'])
+
+    if 'channels' in source:
+        data = []
+        for c in pdf.spec['channels']:
+            data += source['channels'][c['name']]['bindata']['data']
+        data = data + pdf.config.auxdata
+    else:
+        data = source['bindata']['data'] + pdf.config.auxdata
+
+    if 'auxdata' in setup['expected']['config']:
+        assert len(pdf.config.auxdata) == \
+            setup['expected']['config']['auxdata']
+    assert len(pdf.config.suggested_init()) == \
+        setup['expected']['config']['init_pars']
+    assert len(pdf.config.suggested_bounds()) == \
+        setup['expected']['config']['par_bounds']
+
+    validate_runOnePoint(pdf, data, setup['mu'], setup['expected']['result'])