Repository: scikit-hep/pyhf

Issue:
Fix ShapeSys masking
# Description

https://github.com/diana-hep/pyhf/blob/49f30cd52bd543e67b1464d0ce916e15014a6f5b/pyhf/modifiers/shapesys.py#L106-L119

is a bit hacky right now by letting `==0` mean that the bin does not participate. Should be masking instead.

# Expected Behavior

Expect to do something more like

```
factors = nom_sq / unc_sq
```

and then mask `factors` after the fact.

# Actual Behavior

No masking is happening.

# Steps to Reproduce

See the code.

# Checklist

- [x] Run `git fetch` to get the most up to date version of `master`
- [x] Searched through existing Issues to confirm this is not a duplicate issue
- [x] Filled out the Description, Expected Behavior, Actual Behavior, and Steps to Reproduce sections above or have edited/removed them in a way that fully describes the issue



Discussion:
the issue here is not really the masking but two separate things

```
            unc_nom = default_backend.astensor(
                [x for x in uncert_this_mod[:, :, :] if any(x[0][x[0] > 0])]
            )
            unc = unc_nom[0, 0]
            nom = unc_nom[0, 1]
```

this code assumes that len(unc_nom) == 1  since shapesys cannot be shared, we are sure that unc_norm is not >1 but it can be `len(unc_nom)==0` since some samples might have a declared uncertainty of "0"

then later we have the issue here:

```
            numerator = default_backend.where(
                unc_sq > 0, nom_sq, default_backend.zeros(unc_sq.shape)
            )
            denominator = default_backend.where(
                unc_sq > 0, unc_sq, default_backend.ones(unc_sq.shape)
            )

            factors = numerator / denominator
            factors = factors[factors > 0]
```

where this becomes an issue when the nominal data for the sample is zero..

both cases typically happen at the same time when 


```
                {
                    "data": [
                        0.0
                    ],
                    "modifiers": [
                        {
                            "data": [
                                0.0
                            ],
                            "name": "shape_fakes_stat_fakes_SRee_eMLLc_hghmet_obs_cuts",
                            "type": "shapesys"
                        }
                    ],
                    "name": "fakes"
                },
```


just dumping this in for now

```
$ python
Python 3.7.3 (default, Mar 27 2019, 09:23:32) 
[Clang 9.0.0 (clang-900.0.39.2)] on darwin
Type "help", "copyright", "credits" or "license" for more information.
>>> setup = """
... import json, numpy as np
... data = np.array(json.load(open('uncert_this_mod.json')))
... """
>>> timeit.timeit('[x for x in data if any(x[0][x[0]>=0])]', setup=setup, number=10000)
0.19244455399999083
>>> timeit.timeit('data[np.sum(data[:,0]>=0, axis=1)==1,:,:]', setup=setup, number=10000)
0.13176023799999825
```

Resolving patch:
diff --git a/src/pyhf/modifiers/shapesys.py b/src/pyhf/modifiers/shapesys.py
--- a/src/pyhf/modifiers/shapesys.py
+++ b/src/pyhf/modifiers/shapesys.py
@@ -49,12 +49,13 @@ def __init__(self, shapesys_mods, pdfconfig, mega_mods, batch_size=None):
         self._shapesys_mask = [
             [[mega_mods[m][s]['data']['mask']] for s in pdfconfig.samples] for m in keys
         ]
-        self.__shapesys_uncrt = default_backend.astensor(
+        self.__shapesys_info = default_backend.astensor(
             [
                 [
                     [
-                        mega_mods[m][s]['data']['uncrt'],
+                        mega_mods[m][s]['data']['mask'],
                         mega_mods[m][s]['data']['nom_data'],
+                        mega_mods[m][s]['data']['uncrt'],
                     ]
                     for s in pdfconfig.samples
                 ]
@@ -84,18 +85,21 @@ def _reindex_access_field(self, pdfconfig):
             if not pdfconfig.param_set(self._shapesys_mods[syst_index]).n_parameters:
                 self._access_field[syst_index] = 0
                 continue
+
+            singular_sample_index = [
+                idx
+                for idx, syst in enumerate(
+                    default_backend.astensor(self._shapesys_mask)[syst_index, :, 0]
+                )
+                if any(syst)
+            ][-1]
+
             for batch_index, batch_access in enumerate(syst_access):
                 selection = self.param_viewer.index_selection[syst_index][batch_index]
                 access_field_for_syst_and_batch = default_backend.zeros(
                     len(batch_access)
                 )
-                singular_sample_index = [
-                    idx
-                    for idx, syst in enumerate(
-                        default_backend.astensor(self._shapesys_mask)[syst_index, :, 0]
-                    )
-                    if any(syst)
-                ][-1]
+
                 sample_mask = self._shapesys_mask[syst_index][singular_sample_index][0]
                 access_field_for_syst_and_batch[sample_mask] = selection
                 self._access_field[
@@ -115,30 +119,36 @@ def _precompute(self):
         self.shapesys_default = tensorlib.ones(tensorlib.shape(self.shapesys_mask))
 
     def finalize(self, pdfconfig):
-        for uncert_this_mod, pname in zip(self.__shapesys_uncrt, self._shapesys_mods):
+        # self.__shapesys_info: (parameter, sample, [mask, nominal rate, uncertainty], bin)
+        for mod_uncert_info, pname in zip(self.__shapesys_info, self._shapesys_mods):
+            # skip cases where given shapesys modifier affects zero samples
             if not pdfconfig.param_set(pname).n_parameters:
                 continue
-            unc_nom = default_backend.astensor(
-                [x for x in uncert_this_mod[:, :, :] if any(x[0][x[0] > 0])]
-            )
-            unc = unc_nom[0, 0]
-            nom = unc_nom[0, 1]
-            unc_sq = default_backend.power(unc, 2)
-            nom_sq = default_backend.power(nom, 2)
-
-            # the below tries to filter cases in which
-            # this modifier is not used by checking non
-            # zeroness.. shoudl probably use mask
-            numerator = default_backend.where(
-                unc_sq > 0, nom_sq, default_backend.zeros(unc_sq.shape)
-            )
-            denominator = default_backend.where(
-                unc_sq > 0, unc_sq, default_backend.ones(unc_sq.shape)
-            )
-
-            factors = numerator / denominator
-            factors = factors[factors > 0]
+
+            # identify the information for the sample that the given parameter
+            # affects. shapesys is not shared, so there should only ever be at
+            # most one sample
+            # sample_uncert_info: ([mask, nominal rate, uncertainty], bin)
+            sample_uncert_info = mod_uncert_info[
+                default_backend.astensor(
+                    default_backend.sum(mod_uncert_info[:, 0] > 0, axis=1), dtype='bool'
+                )
+            ][0]
+
+            # bin_mask: ([mask], bin)
+            bin_mask = default_backend.astensor(sample_uncert_info[0], dtype='bool')
+            # nom_unc: ([nominal, uncertainty], bin)
+            nom_unc = sample_uncert_info[1:]
+
+            # compute gamma**2 and sigma**2
+            nom_unc_sq = default_backend.power(nom_unc, 2)
+            # when the nominal rate = 0 OR uncertainty = 0, set = 1
+            nom_unc_sq[nom_unc_sq == 0] = 1
+            # divide (gamma**2 / sigma**2) and mask to set factors for only the
+            # parameters we have allocated
+            factors = (nom_unc_sq[0] / nom_unc_sq[1])[bin_mask]
             assert len(factors) == pdfconfig.param_set(pname).n_parameters
+
             pdfconfig.param_set(pname).factors = default_backend.tolist(factors)
             pdfconfig.param_set(pname).auxdata = default_backend.tolist(factors)
 
diff --git a/src/pyhf/pdf.py b/src/pyhf/pdf.py
--- a/src/pyhf/pdf.py
+++ b/src/pyhf/pdf.py
@@ -195,12 +195,7 @@ def _nominal_and_modifiers_from_spec(config, spec):
                     mega_mods[key][s]['data']['mask'] += maskval
                     mega_mods[key][s]['data']['uncrt'] += uncrt
                     mega_mods[key][s]['data']['nom_data'] += nom
-                else:
-                    raise RuntimeError(
-                        'not sure how to combine {mtype} into the mega-channel'.format(
-                            mtype=mtype
-                        )
-                    )
+
         sample_dict = {'name': 'mega_{}'.format(s), 'nom': mega_nom}
         mega_samples[s] = sample_dict
 


Test patch: