Repository: scikit-hep/pyhf

Issue:
better error message when systematic has wrong number of bins for the channel
# Description

It's good that we can catch samples with wrong number of bins for a channel relatively early. We should catch this earlier too for systematics on the samples. Example here has a systematic with 3 bins for a channel with 4 bins:

```json
{
    "channels": [
        {
            "name": "SR",
            "samples": [
                {
                    "data": [
                        1,
                        2,
                        3,
                        4
                    ],
                    "modifiers": [
                        {
                            "data": null,
                            "name": "SigXsecOverSM",
                            "type": "normfactor"
                        }
                    ],
                    "name": "signal"
                },
                {
                    "data": [
                        2,
                        4,
                        6,
                        8
                    ],
                    "modifiers": [
                        {
                            "data": {
                                "hi_data": [
                                    3,
                                    6,
                                    9
                                ],
                                "lo_data": [
                                    1,
                                    2,
                                    3
                                ]
                            },
                            "name": "syst_wrongbins",
                            "type": "histosys"
                        }
                    ],
                    "name": "WtZ"
                }
            ]
        }
    ],
    "measurements": [
        {
            "config": {
                "parameters": [
                    {
                        "bounds": [
                            [
                                -10.0,
                                10.0
                            ]
                        ],
                        "fixed": true,
                        "inits": [
                            1.0
                        ],
                        "name": "SigXsecOverSM"
                    }
                ],
                "poi": "SigXsecOverSM"
            },
            "name": "NormalMeasurement"
        }
    ],
    "observations": [
        {
            "data": [
                2.0,
                2.0,
                2.0,
                2.0
            ],
            "name": "SR"
        }
    ],
    "version": "1.0.0"
}
```

when running `pyhf cls` on this (or just loading in the workspace), the stacktrace is a bit long and crashes in the combined histosys modifier code (a bit further than we'd like)

```
Traceback (most recent call last):
  File "/Users/kratsg/.pyenv/versions/pyhf-dev/bin/pyhf", line 33, in <module>
    sys.exit(load_entry_point('pyhf', 'console_scripts', 'pyhf')())
  File "/Users/kratsg/.pyenv/versions/3.8.6/envs/pyhf-dev/lib/python3.8/site-packages/click/core.py", line 829, in __call__
    return self.main(*args, **kwargs)
  File "/Users/kratsg/.pyenv/versions/3.8.6/envs/pyhf-dev/lib/python3.8/site-packages/click/core.py", line 782, in main
    rv = self.invoke(ctx)
  File "/Users/kratsg/.pyenv/versions/3.8.6/envs/pyhf-dev/lib/python3.8/site-packages/click/core.py", line 1259, in invoke
    return _process_result(sub_ctx.command.invoke(sub_ctx))
  File "/Users/kratsg/.pyenv/versions/3.8.6/envs/pyhf-dev/lib/python3.8/site-packages/click/core.py", line 1066, in invoke
    return ctx.invoke(self.callback, **ctx.params)
  File "/Users/kratsg/.pyenv/versions/3.8.6/envs/pyhf-dev/lib/python3.8/site-packages/click/core.py", line 610, in invoke
    return callback(*args, **kwargs)
  File "/Users/kratsg/pyhf/src/pyhf/cli/infer.py", line 198, in cls
    model = ws.model(
  File "/Users/kratsg/pyhf/src/pyhf/workspace.py", line 412, in model
    return Model(modelspec, poi_name=measurement['config']['poi'], **config_kwargs)
  File "/Users/kratsg/pyhf/src/pyhf/pdf.py", line 561, in __init__
    self.main_model = _MainModel(
  File "/Users/kratsg/pyhf/src/pyhf/pdf.py", line 417, in __init__
    self.modifiers_appliers = {
  File "/Users/kratsg/pyhf/src/pyhf/pdf.py", line 418, in <dictcomp>
    k: c(
  File "/Users/kratsg/pyhf/src/pyhf/modifiers/histosys.py", line 63, in __init__
    if histosys_mods:
  File "/Users/kratsg/pyhf/src/pyhf/interpolators/code4p.py", line 22, in __init__
    self._histogramssets = default_backend.astensor(histogramssets)
  File "/Users/kratsg/pyhf/src/pyhf/tensor/numpy_backend.py", line 214, in astensor
    return np.asarray(tensor_in, dtype=dtype)
  File "/Users/kratsg/.pyenv/versions/3.8.6/envs/pyhf-dev/lib/python3.8/site-packages/numpy/core/_asarray.py", line 83, in asarray
    return array(a, dtype, copy=False, order=order)
ValueError: setting an array element with a sequence.
```

The best thing to do is probably say something like "modifier <name> in <sample> has incompatible structure for <channel>" or something similar.


Discussion:
I guess we probably want to check all the modifier data entries length just after model spec creation in 

https://github.com/scikit-hep/pyhf/blob/f25b8ddc6ab34d089954eb4898d54220f7f2defa/src/pyhf/workspace.py#L416-L419

Thinking more on this, I guess this only needs to be applied to bin-wise modifiers, so we shouldn't check after model spec creation, but in each relevant modifier itself.

Resolving patch:
diff --git a/src/pyhf/modifiers/histosys.py b/src/pyhf/modifiers/histosys.py
--- a/src/pyhf/modifiers/histosys.py
+++ b/src/pyhf/modifiers/histosys.py
@@ -1,10 +1,10 @@
 import logging
 
 import pyhf
-from pyhf import events
-from pyhf.tensor.manager import get_backend
-from pyhf import interpolators
+from pyhf import events, interpolators
+from pyhf.exceptions import InvalidModifier
 from pyhf.parameters import ParamViewer
+from pyhf.tensor.manager import get_backend
 
 log = logging.getLogger(__name__)
 
@@ -61,8 +61,8 @@ def append(self, key, channel, sample, thismod, defined_samp):
     def finalize(self):
         default_backend = pyhf.default_backend
 
-        for modifier in self.builder_data.values():
-            for sample in modifier.values():
+        for modifier_name, modifier in self.builder_data.items():
+            for sample_name, sample in modifier.items():
                 sample["data"]["mask"] = default_backend.concatenate(
                     sample["data"]["mask"]
                 )
@@ -75,6 +75,21 @@ def finalize(self):
                 sample["data"]["nom_data"] = default_backend.concatenate(
                     sample["data"]["nom_data"]
                 )
+                if (
+                    not len(sample["data"]["nom_data"])
+                    == len(sample["data"]["lo_data"])
+                    == len(sample["data"]["hi_data"])
+                ):
+                    _modifier_type, _modifier_name = modifier_name.split("/")
+                    _sample_data_len = len(sample["data"]["nom_data"])
+                    _lo_data_len = len(sample["data"]["lo_data"])
+                    _hi_data_len = len(sample["data"]["hi_data"])
+                    raise InvalidModifier(
+                        f"The '{sample_name}' sample {_modifier_type} modifier"
+                        + f" '{_modifier_name}' has data shape inconsistent with the sample.\n"
+                        + f"{sample_name} has 'data' of length {_sample_data_len} but {_modifier_name}"
+                        + f" has 'lo_data' of length {_lo_data_len} and 'hi_data' of length {_hi_data_len}."
+                    )
         return self.builder_data
 
 
diff --git a/src/pyhf/modifiers/shapesys.py b/src/pyhf/modifiers/shapesys.py
--- a/src/pyhf/modifiers/shapesys.py
+++ b/src/pyhf/modifiers/shapesys.py
@@ -2,8 +2,9 @@
 
 import pyhf
 from pyhf import events
-from pyhf.tensor.manager import get_backend
+from pyhf.exceptions import InvalidModifier
 from pyhf.parameters import ParamViewer
+from pyhf.tensor.manager import get_backend
 
 log = logging.getLogger(__name__)
 
@@ -70,8 +71,8 @@ def append(self, key, channel, sample, thismod, defined_samp):
     def finalize(self):
         default_backend = pyhf.default_backend
 
-        for modifier in self.builder_data.values():
-            for sample in modifier.values():
+        for modifier_name, modifier in self.builder_data.items():
+            for sample_name, sample in modifier.items():
                 sample["data"]["mask"] = default_backend.concatenate(
                     sample["data"]["mask"]
                 )
@@ -81,6 +82,16 @@ def finalize(self):
                 sample["data"]["nom_data"] = default_backend.concatenate(
                     sample["data"]["nom_data"]
                 )
+                if len(sample["data"]["nom_data"]) != len(sample["data"]["uncrt"]):
+                    _modifier_type, _modifier_name = modifier_name.split("/")
+                    _sample_data_len = len(sample["data"]["nom_data"])
+                    _uncrt_len = len(sample["data"]["uncrt"])
+                    raise InvalidModifier(
+                        f"The '{sample_name}' sample {_modifier_type} modifier"
+                        + f" '{_modifier_name}' has data shape inconsistent with the sample.\n"
+                        + f"{sample_name} has 'data' of length {_sample_data_len} but {_modifier_name}"
+                        + f" has 'data' of length {_uncrt_len}."
+                    )
         return self.builder_data
 
 
diff --git a/src/pyhf/modifiers/staterror.py b/src/pyhf/modifiers/staterror.py
--- a/src/pyhf/modifiers/staterror.py
+++ b/src/pyhf/modifiers/staterror.py
@@ -3,8 +3,9 @@
 
 import pyhf
 from pyhf import events
-from pyhf.tensor.manager import get_backend
+from pyhf.exceptions import InvalidModifier
 from pyhf.parameters import ParamViewer
+from pyhf.tensor.manager import get_backend
 
 log = logging.getLogger(__name__)
 
@@ -54,8 +55,8 @@ def append(self, key, channel, sample, thismod, defined_samp):
     def finalize(self):
         default_backend = pyhf.default_backend
 
-        for modifier in self.builder_data.values():
-            for sample in modifier.values():
+        for modifier_name, modifier in self.builder_data.items():
+            for sample_name, sample in modifier.items():
                 sample["data"]["mask"] = default_backend.concatenate(
                     sample["data"]["mask"]
                 )
@@ -65,6 +66,16 @@ def finalize(self):
                 sample["data"]["nom_data"] = default_backend.concatenate(
                     sample["data"]["nom_data"]
                 )
+                if len(sample["data"]["nom_data"]) != len(sample["data"]["uncrt"]):
+                    _modifier_type, _modifier_name = modifier_name.split("/")
+                    _sample_data_len = len(sample["data"]["nom_data"])
+                    _uncrt_len = len(sample["data"]["uncrt"])
+                    raise InvalidModifier(
+                        f"The '{sample_name}' sample {_modifier_type} modifier"
+                        + f" '{_modifier_name}' has data shape inconsistent with the sample.\n"
+                        + f"{sample_name} has 'data' of length {_sample_data_len} but {_modifier_name}"
+                        + f" has 'data' of length {_uncrt_len}."
+                    )
 
         for modname in self.builder_data.keys():
             parname = modname.split('/')[1]


Test patch:
diff --git a/tests/test_modifiers.py b/tests/test_modifiers.py
--- a/tests/test_modifiers.py
+++ b/tests/test_modifiers.py
@@ -1,15 +1,10 @@
-import pyhf
+import json
+
 import numpy
+import pytest
+from jsonpatch import JsonPatch
 
-modifiers_to_test = [
-    "histosys",
-    "normfactor",
-    "normsys",
-    "shapefactor",
-    "shapesys",
-    "staterror",
-]
-modifier_pdf_types = ["normal", None, "normal", None, "poisson", "normal"]
+import pyhf
 
 
 def test_shapefactor_build():
@@ -174,3 +169,27 @@ def test_shapesys_holes():
         True,
         False,
     ]
+
+
+@pytest.mark.parametrize(
+    "patch_file",
+    [
+        "bad_histosys_modifier_patch.json",
+        "bad_shapesys_modifier_patch.json",
+        "bad_staterror_modifier_patch.json",
+    ],
+)
+def test_invalid_bin_wise_modifier(datadir, patch_file):
+    """
+    Test that bin-wise modifiers will raise an exception if their data shape
+    differs from their sample's.
+    """
+    spec = json.load(open(datadir.join("spec.json")))
+
+    assert pyhf.Model(spec)
+
+    patch = JsonPatch.from_string(open(datadir.join(patch_file)).read())
+    bad_spec = patch.apply(spec)
+
+    with pytest.raises(pyhf.exceptions.InvalidModifier):
+        pyhf.Model(bad_spec)
diff --git a/tests/test_modifiers/bad_histosys_modifier_patch.json b/tests/test_modifiers/bad_histosys_modifier_patch.json
new file mode 100644
--- /dev/null
+++ b/tests/test_modifiers/bad_histosys_modifier_patch.json
@@ -0,0 +1,24 @@
+[
+    {
+        "op": "add",
+        "path": "/channels/0/samples/1/modifiers",
+        "value": [
+            {
+                "name": "histosys_bad",
+                "type": "histosys",
+                "data": {
+                    "hi_data": [
+                        3,
+                        6,
+                        9
+                    ],
+                    "lo_data": [
+                        1,
+                        2,
+                        3
+                    ]
+                }
+            }
+        ]
+    }
+]
diff --git a/tests/test_modifiers/bad_shapesys_modifier_patch.json b/tests/test_modifiers/bad_shapesys_modifier_patch.json
new file mode 100644
--- /dev/null
+++ b/tests/test_modifiers/bad_shapesys_modifier_patch.json
@@ -0,0 +1,17 @@
+[
+    {
+        "op": "add",
+        "path": "/channels/0/samples/1/modifiers",
+        "value": [
+            {
+                "name": "shapesys_bad",
+                "type": "shapesys",
+                "data": [
+                    1,
+                    2,
+                    3
+                ]
+            }
+        ]
+    }
+]
diff --git a/tests/test_modifiers/bad_staterror_modifier_patch.json b/tests/test_modifiers/bad_staterror_modifier_patch.json
new file mode 100644
--- /dev/null
+++ b/tests/test_modifiers/bad_staterror_modifier_patch.json
@@ -0,0 +1,17 @@
+[
+    {
+        "op": "add",
+        "path": "/channels/0/samples/1/modifiers",
+        "value": [
+            {
+                "name": "staterror_bad",
+                "type": "staterror",
+                "data": [
+                    1,
+                    2,
+                    3
+                ]
+            }
+        ]
+    }
+]
diff --git a/tests/test_modifiers/spec.json b/tests/test_modifiers/spec.json
new file mode 100644
--- /dev/null
+++ b/tests/test_modifiers/spec.json
@@ -0,0 +1,35 @@
+{
+    "channels": [
+        {
+            "name": "channel_1",
+            "samples": [
+                {
+                    "name": "sample_1",
+                    "data": [
+                        1,
+                        2,
+                        3,
+                        4
+                    ],
+                    "modifiers": [
+                        {
+                            "name": "mu",
+                            "type": "normfactor",
+                            "data": null
+                        }
+                    ]
+                },
+                {
+                    "name": "sample_2",
+                    "data": [
+                        2,
+                        4,
+                        6,
+                        8
+                    ],
+                    "modifiers": []
+                }
+            ]
+        }
+    ]
+}