Repository: European-XFEL/EXtra-data

Issue:
Virtual overview file missing instrument sources with no data
E.g. `MID_EXP_AGIPD1M1/REDU/LITFRM:output` is

- present in `/gpfs/exfel/exp/MID/202230/p900249/raw/r0198/RAW-R0198-DA05-S00000.h5`
- missing from `/gpfs/exfel/exp/MID/202230/p900249/usr/.extra_data/RAW-R0198-OVERVIEW.h5`

We haven't figured out why yet.

Discussion:

Patch:
diff --git a/extra_data/writer.py b/extra_data/writer.py
index 89e540e..1d1678d 100644
--- a/extra_data/writer.py
+++ b/extra_data/writer.py
@@ -199,14 +199,10 @@ class VirtualFileWriter(FileWriter):
 
         super().__init__(path,  data)
 
-    def _assemble_data(self, source, key):
+    def _assemble_data(self, keydata):
         """Assemble chunks of data into a virtual layout"""
         # First, get a list of all non-empty data chunks
-        chunks = [c for c in self.data._find_data_chunks(source, key)
-                  if (c.counts > 0).any()]
-        chunks.sort(key = lambda c: c.train_ids[0])
-        if not chunks:
-            return None, None
+        chunks = keydata._data_chunks
 
         # Create the layout, which will describe what data is where
         n_total = np.sum([c.counts.sum() for c in chunks])
@@ -225,11 +221,7 @@ class VirtualFileWriter(FileWriter):
 
         assert output_cursor == n_total
 
-        # Make an array of which train ID each data entry is for:
-        train_ids = np.concatenate([
-            np.repeat(c.train_ids, c.counts.astype(np.intp)) for c in chunks
-        ])
-        return layout, train_ids
+        return layout
 
     # In big detector data, these fields are like extra indexes.
     # So we'll copy them to the output file for fast access, rather than
@@ -257,20 +249,21 @@ class VirtualFileWriter(FileWriter):
         self._make_index(source, key, a.coords['trainId'].values)
 
     def add_dataset(self, source, key):
-        layout, train_ids = self._assemble_data(source, key)
-        if not layout:
-            return  # No data
+        keydata = self.data[source, key]
 
-        path = f"{self._section(source)}/{source}/{key.replace('.', '/')}"
-        self.file.create_virtual_dataset(path, layout)
+        if keydata.shape[0] == 0:  # No data
+            self.file.create_dataset(keydata.hdf5_data_path, shape=keydata.shape, dtype=keydata.dtype)
+        else:
+            layout = self._assemble_data(keydata)
+            self.file.create_virtual_dataset(keydata.hdf5_data_path, layout)
 
-        self._make_index(source, key, train_ids)
+        self._make_index(source, key, keydata.train_id_coordinates())
         if source in self.data.instrument_sources:
             self.data_sources.add(f"INSTRUMENT/{source}/{key.partition('.')[0]}")
         else:
             self.data_sources.add(f"CONTROL/{source}")
 
-        return path
+        return keydata.hdf5_data_path
 
     def copy_source(self, source):
         pass  # Override base class copying data


Test patch:
diff --git a/extra_data/tests/test_writer.py b/extra_data/tests/test_writer.py
index acc6ff5..b0b9bdd 100644
--- a/extra_data/tests/test_writer.py
+++ b/extra_data/tests/test_writer.py
@@ -59,3 +59,6 @@ def test_write_virtual(mock_fxe_raw_run):
 
             r = f.get_run_value('SPB_XTD9_XGM/DOOCS/MAIN', 'beamPosition.ixPos.value')
             assert isinstance(r, np.float32)
+
+            cam_nodata = f['FXE_XAD_GEC/CAM/CAMERA_NODATA:daqOutput', 'data.image.pixels']
+            assert cam_nodata.shape == (0, 255, 1024)