Repository: European-XFEL/EXtra-data

Issue:
Can't instantiate multi-module detector access if any module has no data
This happens a lot with LPD data. It looks like the root cause is a problem with `.data_counts()`.

```python
run = open_run(2729, 44)
LPD1M(run.select_trains(np.s_[:100])
```

```
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-12-fb113b52e6e9> in <module>
----> 1 LPD1M(run_trains, parallel_gain=True)

~/Code/EXtra-data/extra_data/components.py in __init__(self, data, detector_name, modules, min_modules, parallel_gain)
   1210     def __init__(self, data: DataCollection, detector_name=None, modules=None,
   1211                  *, min_modules=1, parallel_gain=False):
-> 1212         super().__init__(data, detector_name, modules, min_modules=min_modules)
   1213 
   1214         self.parallel_gain = parallel_gain

~/Code/EXtra-data/extra_data/components.py in __init__(self, data, detector_name, modules, min_modules)
    393     def __init__(self, data: DataCollection, detector_name=None, modules=None,
    394                  *, min_modules=1):
--> 395         super().__init__(data, detector_name, modules, min_modules=min_modules)
    396 
    397     @staticmethod

~/Code/EXtra-data/extra_data/components.py in __init__(self, data, detector_name, modules, min_modules)
    116         mod_data_counts = pd.DataFrame({
    117             src: data.get_data_counts(src, self._main_data_key)
--> 118             for src in source_to_modno
    119         }).fillna(0).astype(np.uint64)
    120 

~/Code/EXtra-data/extra_data/components.py in <dictcomp>(.0)
    116         mod_data_counts = pd.DataFrame({
    117             src: data.get_data_counts(src, self._main_data_key)
--> 118             for src in source_to_modno
    119         }).fillna(0).astype(np.uint64)
    120 

~/Code/EXtra-data/extra_data/reader.py in get_data_counts(self, source, key)
    442             Key of parameter within that device, e.g. "image.data".
    443         """
--> 444         return self._get_key_data(source, key).data_counts()
    445 
    446     def get_series(self, source, key):

~/Code/EXtra-data/extra_data/keydata.py in data_counts(self, labelled)
    149         IDs. Otherwise, returns a NumPy array of counts to match ``.train_ids``.
    150         """
--> 151         train_ids = np.concatenate([c.train_ids for c in self._data_chunks])
    152         counts = np.concatenate([c.counts for c in self._data_chunks])
    153 

<__array_function__ internals> in concatenate(*args, **kwargs)

ValueError: need at least one array to concatenate
```

Discussion:

Patch:
diff --git a/extra_data/keydata.py b/extra_data/keydata.py
index 86fc141..167b173 100644
--- a/extra_data/keydata.py
+++ b/extra_data/keydata.py
@@ -53,11 +53,14 @@ class KeyData:
         """An ordered list of chunks containing data"""
         if self._cached_chunks is None:
             self._cached_chunks = sorted(
-                [c for c in self._find_chunks() if c.total_count],
-                key=lambda c: c.train_ids[0]
+                self._find_chunks(), key=lambda c: c.train_ids[0]
             )
         return self._cached_chunks
 
+    @property
+    def _data_chunks_nonempty(self) -> List[DataChunk]:
+        return [c for c in self._data_chunks if c.total_count]
+
     def __repr__(self):
         return f"<extra_data.KeyData source={self.source!r} key={self.key!r} " \
                f"for {len(self.train_ids)} trains>"
@@ -148,8 +151,11 @@ class KeyData:
         If *labelled* is True, returns a pandas series with an index of train
         IDs. Otherwise, returns a NumPy array of counts to match ``.train_ids``.
         """
-        train_ids = np.concatenate([c.train_ids for c in self._data_chunks])
-        counts = np.concatenate([c.counts for c in self._data_chunks])
+        if self._data_chunks:
+            train_ids = np.concatenate([c.train_ids for c in self._data_chunks])
+            counts = np.concatenate([c.counts for c in self._data_chunks])
+        else:
+            train_ids = counts = np.zeros(0, dtype=np.uint64)
 
         if labelled:
             import pandas as pd
@@ -182,7 +188,7 @@ class KeyData:
 
         # Read the data from each chunk into the result array
         dest_cursor = 0
-        for chunk in self._data_chunks:
+        for chunk in self._data_chunks_nonempty:
             dest_chunk_end = dest_cursor + chunk.total_count
 
             slices = (chunk.slice,) + roi
@@ -195,6 +201,8 @@ class KeyData:
 
     def _trainid_index(self):
         """A 1D array of train IDs, corresponding to self.shape[0]"""
+        if not self._data_chunks:
+            return np.zeros(0, dtype=np.uint64)
         chunks_trainids = [
             np.repeat(chunk.train_ids, chunk.counts.astype(np.intp))
             for chunk in self._data_chunks
@@ -234,9 +242,7 @@ class KeyData:
         dims = ['trainId'] + extra_dims
 
         # Train ID index
-        coords = {}
-        if self.shape[0]:
-            coords = {'trainId': self._trainid_index()}
+        coords = {'trainId': self._trainid_index()}
 
         if name is None:
             name = f'{self.source}.{self.key}'
@@ -287,7 +293,7 @@ class KeyData:
 
         chunks_darrs = []
 
-        for chunk in self._data_chunks:
+        for chunk in self._data_chunks_nonempty:
             chunk_dim0 = chunk.total_count
             chunk_shape = (chunk_dim0,) + chunk.dataset.shape[1:]
             itemsize = chunk.dataset.dtype.itemsize
@@ -310,7 +316,11 @@ class KeyData:
                 )[chunk.slice]
             )
 
-        dask_arr = da.concatenate(chunks_darrs, axis=0)
+        if chunks_darrs:
+            dask_arr = da.concatenate(chunks_darrs, axis=0)
+        else:
+            shape = (0,) + self.entry_shape
+            dask_arr = da.zeros(shape=shape, dtype=self.dtype, chunks=shape)
 
         if labelled:
             # Dimension labels
@@ -369,7 +379,7 @@ class KeyData:
 
         Yields pairs of (train ID, array). Skips trains where data is missing.
         """
-        for chunk in self._data_chunks:
+        for chunk in self._data_chunks_nonempty:
             start = chunk.first
             ds = chunk.dataset
             for tid, count in zip(chunk.train_ids, chunk.counts):


Test patch:
diff --git a/extra_data/tests/test_keydata.py b/extra_data/tests/test_keydata.py
index cb7406f..abdf9e8 100644
--- a/extra_data/tests/test_keydata.py
+++ b/extra_data/tests/test_keydata.py
@@ -68,6 +68,10 @@ def test_nodata(mock_fxe_raw_run):
     assert arr.shape == (0, 255, 1024)
     assert arr.dtype == np.dtype('u2')
 
+    dask_arr = cam_pix.dask_array(labelled=True)
+    assert dask_arr.shape == (0, 255, 1024)
+    assert dask_arr.dtype == np.dtype('u2')
+
     assert list(cam_pix.trains()) == []
     tid, data = cam_pix.train_from_id(10010)
     assert tid == 10010
@@ -121,6 +125,24 @@ def test_data_counts(mock_reduced_spb_proc_run):
     assert count.values.sum() == mod.shape[0]
 
 
+def test_data_counts_empty(mock_fxe_raw_run):
+    run = RunDirectory(mock_fxe_raw_run)
+    cam_nodata = run['FXE_XAD_GEC/CAM/CAMERA_NODATA:daqOutput', 'data.image.pixels']
+
+    count_ser = cam_nodata.data_counts(labelled=True)
+    assert len(count_ser) == 480
+    assert count_ser.sum() == 0
+
+    count_arr = cam_nodata.data_counts(labelled=False)
+    assert len(count_arr) == 480
+    assert count_arr.sum() == 0
+
+    count_none_ser = cam_nodata.drop_empty_trains().data_counts(labelled=True)
+    assert len(count_none_ser) == 0
+
+    count_none_arr = cam_nodata.drop_empty_trains().data_counts(labelled=False)
+    assert len(count_none_arr) == 0
+
 def test_select_by(mock_spb_raw_run):
     run = RunDirectory(mock_spb_raw_run)
     am0 = run['SPB_DET_AGIPD1M-1/DET/0CH0:xtdf', 'image.data']