Repository: scikit-hep/pyhf

Issue:
Safeguard invalid modifiers definition for stat errors
# Description

right now the spec reading allows the `data` field in staterrors to be empty which is invalid, since we need to be able to compute the stat. uncertainty for it.

This can probably be caught in the JSON schema validation. The only modifiers who do not need additional data are the factor modifiers (the unconstrained ones) i.e. `normfactor` and `shapefactor`


this showed up in #231  when processing mbj

```json
{"data":[0.20280006527900696],"modifiers":[{"data":[],"name":"staterror_SR0L_Lnj_Imeff_cuts","type":"staterror"},{"data":{"hi":2,"lo":0.01},"name":"QCDHundred","type":"normsys"}],"name":"QCD"}
{"data":[0.4424484670162201],"modifiers":[{"data":[],"name":"staterror_SR0L_Inj_Imeff_cuts","type":"staterror"},{"data":{"hi":2,"lo":0.01},"name":"QCDHundred","type":"normsys"}],"name":"QCD"}
{"data":[0.0742708146572113],"modifiers":[{"data":[],"name":"staterror_SR0L_Hnj_Imeff_cuts","type":"staterror"},{"data":{"hi":2,"lo":0.01},"name":"QCDHundred","type":"normsys"}],"name":"QCD"}
{"data":[0.047470349818468094],"modifiers":[{"data":[],"name":"staterror_SR0L_Hnj_Lmeff_cuts","type":"staterror"},{"data":{"hi":2,"lo":0.01},"name":"QCDHundred","type":"normsys"}],"name":"QCD"}
{"data":[0.3601486086845398],"modifiers":[{"data":[],"name":"staterror_SR0L_Lnj_Hmeff_cuts","type":"staterror"},{"data":{"hi":2,"lo":0.01},"name":"QCDHundred","type":"normsys"}],"name":"QCD"}
{"data":[0.03753824904561043],"modifiers":[{"data":[],"name":"staterror_SR0L_Lnj_Lmeff_cuts","type":"staterror"},{"data":{"hi":2,"lo":0.01},"name":"QCDHundred","type":"normsys"}],"name":"QCD"}
{"data":[0.052878595888614655],"modifiers":[{"data":[],"name":"staterror_SR0L_Hnj_Hmeff_cuts","type":"staterror"},{"data":{"hi":2,"lo":0.01},"name":"QCDHundred","type":"normsys"}],"name":"QCD"}
{"data":[0.028282178565859795],"modifiers":[{"data":[],"name":"staterror_SR0L_IStR_cuts","type":"staterror"},{"data":{"hi":2,"lo":0.01},"name":"QCDHundred","type":"normsys"}],"name":"QCD"}
```


Discussion:

Resolving patch:
diff --git a/pyhf/exceptions/__init__.py b/pyhf/exceptions/__init__.py
--- a/pyhf/exceptions/__init__.py
+++ b/pyhf/exceptions/__init__.py
@@ -1,8 +1,8 @@
 import sys
 
 class InvalidNameReuse(Exception):
-    pass 
-       
+    pass
+
 class InvalidSpecification(Exception):
     """
     InvalidSpecification is raised when a specification does not validate against the given schema.
@@ -22,6 +22,13 @@ def __init__(self, ValidationError):
         # Call the base class constructor with the parameters it needs
         super(InvalidSpecification, self).__init__(message)
 
+class InvalidModel(Exception):
+    """
+    InvalidModel is raised when a given model does not have the right configuration, even though it validates correctly against the schema.
+
+    This can occur, for example, when the provided parameter of interest to fit against does not get declared in the specification provided.
+    """
+    pass
 
 class InvalidModifier(Exception):
     """
diff --git a/pyhf/pdf.py b/pyhf/pdf.py
--- a/pyhf/pdf.py
+++ b/pyhf/pdf.py
@@ -30,8 +30,11 @@ def from_spec(cls,spec,poiname = 'mu', qualify_names = False):
                     modifier = instance.add_or_get_modifier(channel, sample, modifier_def)
                     modifier.add_sample(channel, sample, modifier_def)
                     modifiers.append(modifier_def['name'])
+        instance.channels = list(set(channels))
+        instance.samples = list(set(samples))
+        instance.modifiers = list(set(modifiers))
         instance.set_poi(poiname)
-        return (instance, (list(set(channels)), list(set(samples)), list(set(modifiers))))
+        return instance
 
     def __init__(self):
         # set up all other bookkeeping variables
@@ -61,6 +64,8 @@ def modifier(self, name):
         return self.par_map[name]['modifier']
 
     def set_poi(self,name):
+        if name not in self.modifiers:
+            raise exceptions.InvalidModel("The paramter of interest '{0:s}' cannot be fit as it is not declared in the model specification.".format(name))
         s = self.par_slice(name)
         assert s.stop-s.start == 1
         self.poi_index = s.start
@@ -119,7 +124,7 @@ def __init__(self, spec, **config_kwargs):
         log.info("Validating spec against schema: {0:s}".format(self.schema))
         utils.validate(self.spec, self.schema)
         # build up our representation of the specification
-        self.config, (self.channels, self.samples, self.modifiers) = _ModelConfig.from_spec(self.spec,**config_kwargs)
+        self.config = _ModelConfig.from_spec(self.spec,**config_kwargs)
 
     def expected_sample(self, channel, sample, pars):
         """


Test patch:
diff --git a/tests/test_schema.py b/tests/test_schema.py
--- a/tests/test_schema.py
+++ b/tests/test_schema.py
@@ -1,7 +1,37 @@
 import pyhf
 import pytest
 
-def test_missing_sample_name():
+def test_no_samples():
+    spec = {
+        'channels': [
+            {
+                'name': 'channel',
+                'samples': []
+            },
+        ]
+    }
+    with pytest.raises(pyhf.exceptions.InvalidSpecification):
+        pyhf.Model(spec)
+
+def test_sample_missing_data():
+    spec = {
+        'channels': [
+            {
+                'name': 'channel',
+                'samples': [
+                    {
+                        'name': 'sample',
+                        'data': [],
+                        'modifiers': []
+                    }
+                ]
+            },
+        ]
+    }
+    with pytest.raises(pyhf.exceptions.InvalidSpecification):
+        pyhf.Model(spec)
+
+def test_sample_missing_name():
     spec = {
         'channels': [
             {
@@ -18,6 +48,47 @@ def test_missing_sample_name():
     with pytest.raises(pyhf.exceptions.InvalidSpecification):
         pyhf.Model(spec)
 
+def test_sample_missing_all_modifiers():
+    spec = {
+        'channels': [
+            {
+                'name': 'channel',
+                'samples': [
+                    {
+                        'name': 'sample',
+                        'data': [10.],
+                        'modifiers': []
+                    }
+                ]
+            },
+        ]
+    }
+    with pytest.raises(pyhf.exceptions.InvalidModel):
+        pyhf.Model(spec)
+
+def test_one_sample_missing_modifiers():
+    spec = {
+        'channels': [
+            {
+                'name': 'channel',
+                'samples': [
+                    {
+                        'name': 'sample',
+                        'data': [10.],
+                        'modifiers': []
+                    },
+                    {
+                        'name': 'another_sample',
+                        'data': [5.],
+                        'modifiers': [{'name': 'mypoi', 'type': 'normfactor', 'data': None}]
+                    }
+                ]
+            },
+        ]
+    }
+    pyhf.Model(spec, poiname='mypoi')
+
+
 def test_add_unknown_modifier():
     spec = {
         'channels': [
@@ -37,3 +108,63 @@ def test_add_unknown_modifier():
     }
     with pytest.raises(pyhf.exceptions.InvalidSpecification):
         pyhf.Model(spec)
+
+def test_empty_staterror():
+    spec = {
+        'channels': [
+            {
+                'name': 'channel',
+                'samples': [
+                    {
+                        'name': 'sample',
+                        'data': [10.],
+                        'modifiers': [
+                            {'name': 'staterror_channel', 'type': 'staterror', 'data': []}
+                        ]
+                    }
+                ]
+            },
+        ]
+    }
+    with pytest.raises(pyhf.exceptions.InvalidSpecification):
+        pyhf.Model(spec)
+
+def test_empty_shapesys():
+    spec = {
+        'channels': [
+            {
+                'name': 'channel',
+                'samples': [
+                    {
+                        'name': 'sample',
+                        'data': [10.],
+                        'modifiers': [
+                            {'name': 'sample_norm', 'type': 'shapesys','data': []}
+                        ]
+                    }
+                ]
+            },
+        ]
+    }
+    with pytest.raises(pyhf.exceptions.InvalidSpecification):
+        pyhf.Model(spec)
+
+def test_empty_histosys():
+    spec = {
+        'channels': [
+            {
+                'name': 'channel',
+                'samples': [
+                    {
+                        'name': 'sample',
+                        'data': [10.],
+                        'modifiers': [
+                            {'name': 'modifier', 'type': 'histosys', 'data': {'lo_data': [], 'hi_data': []}}
+                        ]
+                    }
+                ]
+            },
+        ]
+    }
+    with pytest.raises(pyhf.exceptions.InvalidSpecification):
+        pyhf.Model(spec)