Repository: cta-observatory/ctapipe (python)

Issue:
Remove ctapipe-dump-triggers
This just writes timestamps of events into a table that is not compatible with the rest of ctapipe. 

If there is a need for such a tool, we should adapt the `ctapipe-process` tool to just write out the trigger information in the datawriter without doing any actual processing, but I doubt anyone really needs this.

Pull request:
Remove dump-triggers, enable ctapipe-process to store only trigger and simulation data
fixes #2372 

Running `ctapipe-process -i <input> -o <output> --no-write-parameters` will result in an output file only containing trigger and simulation information, no any actual processing will happen.

Patch:
diff --git a/ctapipe/io/datawriter.py b/ctapipe/io/datawriter.py
index 0b0631d5..3a5cc8f0 100644
--- a/ctapipe/io/datawriter.py
+++ b/ctapipe/io/datawriter.py
@@ -27,7 +27,7 @@ from .datalevels import DataLevel
 from .eventsource import EventSource
 from .hdf5tableio import HDF5TableWriter
 from .simteleventsource import SimTelEventSource
-from .tableio import FixedPointColumnTransform, TableWriter, TelListToMaskTransform
+from .tableio import FixedPointColumnTransform, TelListToMaskTransform
 
 __all__ = ["DataWriter", "DATA_MODEL_VERSION", "write_reference_metadata_headers"]
 
@@ -298,15 +298,11 @@ class DataWriter(Component):
         Write a single event to the output file.
         """
         self._at_least_one_event = True
+        self.log.debug("WRITING EVENT %s", event.index)
 
-        # Write subarray event data
-        self._write_subarray_pointing(event, writer=self._writer)
+        self._write_subarray_pointing(event)
+        self._write_trigger(event)
 
-        self.log.debug("WRITING EVENT %s", event.index)
-        self._writer.write(
-            table_name="dl1/event/subarray/trigger",
-            containers=[event.index, event.trigger],
-        )
         if event.simulation is not None and event.simulation.shower is not None:
             self._writer.write(
                 table_name="simulation/event/subarray/shower",
@@ -322,21 +318,21 @@ class DataWriter(Component):
                 )
 
         if self.write_waveforms:
-            self._write_r1_telescope_events(self._writer, event)
+            self._write_r1_telescope_events(event)
 
         if self.write_raw_waveforms:
-            self._write_r0_telescope_events(self._writer, event)
+            self._write_r0_telescope_events(event)
 
         # write telescope event data
-        self._write_dl1_telescope_events(self._writer, event)
+        self._write_dl1_telescope_events(event)
 
         # write DL2 info if requested
         if self.write_showers:
-            self._write_dl2_telescope_events(self._writer, event)
-            self._write_dl2_stereo_event(self._writer, event)
+            self._write_dl2_telescope_events(event)
+            self._write_dl2_stereo_event(event)
 
         if self.write_muon_parameters:
-            self._write_muon_telescope_events(self._writer, event)
+            self._write_muon_telescope_events(event)
 
     def finish(self):
         """called after all events are done"""
@@ -358,7 +354,6 @@ class DataWriter(Component):
         )
 
         self._write_context_metadata_headers()
-
         self._writer.close()
 
     @property
@@ -415,8 +410,9 @@ class DataWriter(Component):
             self.write_muon_parameters,
         ]
         if not any(writable_things):
-            raise ToolConfigurationError(
-                "DataWriter configured to write no information"
+            self.log.warning(
+                "No processing results were selected for writing"
+                ", only writing trigger and simulation information"
             )
 
     def _setup_writer(self):
@@ -507,13 +503,13 @@ class DataWriter(Component):
         self._writer = writer
         self.log.debug("Writer initialized: %s", self._writer)
 
-    def _write_subarray_pointing(self, event: ArrayEventContainer, writer: TableWriter):
+    def _write_subarray_pointing(self, event: ArrayEventContainer):
         """store subarray pointing info in a monitoring table"""
         pnt = event.pointing
         current_pointing = (pnt.array_azimuth, pnt.array_altitude)
         if current_pointing != self._last_pointing:
             pnt.prefix = ""
-            writer.write("dl1/monitoring/subarray/pointing", [event.trigger, pnt])
+            self._writer.write("dl1/monitoring/subarray/pointing", [event.trigger, pnt])
             self._last_pointing = current_pointing
 
     def _write_scheduling_and_observation_blocks(self):
@@ -616,49 +612,50 @@ class DataWriter(Component):
         """construct dataset table names depending on chosen split method"""
         return f"tel_{tel_id:03d}"
 
-    def _write_r1_telescope_events(
-        self, writer: TableWriter, event: ArrayEventContainer
-    ):
+    def _write_trigger(self, event: ArrayEventContainer):
+        """
+        Write trigger information
+        """
+        self._writer.write(
+            table_name="dl1/event/subarray/trigger",
+            containers=[event.index, event.trigger],
+        )
+
+        for tel_id, trigger in event.trigger.tel.items():
+            self._writer.write(
+                "dl1/event/telescope/trigger", (_get_tel_index(event, tel_id), trigger)
+            )
+
+    def _write_r1_telescope_events(self, event: ArrayEventContainer):
         for tel_id, r1_tel in event.r1.tel.items():
 
             tel_index = _get_tel_index(event, tel_id)
             table_name = self.table_name(tel_id)
 
             r1_tel.prefix = ""
-            writer.write(f"r1/event/telescope/{table_name}", [tel_index, r1_tel])
+            self._writer.write(f"r1/event/telescope/{table_name}", [tel_index, r1_tel])
 
-    def _write_r0_telescope_events(
-        self, writer: TableWriter, event: ArrayEventContainer
-    ):
+    def _write_r0_telescope_events(self, event: ArrayEventContainer):
         for tel_id, r0_tel in event.r0.tel.items():
 
             tel_index = _get_tel_index(event, tel_id)
             table_name = self.table_name(tel_id)
 
             r0_tel.prefix = ""
-            writer.write(f"r0/event/telescope/{table_name}", [tel_index, r0_tel])
+            self._writer.write(f"r0/event/telescope/{table_name}", [tel_index, r0_tel])
 
-    def _write_dl1_telescope_events(
-        self, writer: TableWriter, event: ArrayEventContainer
-    ):
+    def _write_dl1_telescope_events(self, event: ArrayEventContainer):
         """
         add entries to the event/telescope tables for each telescope in a single
         event
         """
 
-        # write the telescope tables
-        # trigger info
-        for tel_id, trigger in event.trigger.tel.items():
-            writer.write(
-                "dl1/event/telescope/trigger", [_get_tel_index(event, tel_id), trigger]
-            )
-
         # pointing info
         for tel_id, pnt in event.pointing.tel.items():
             current_pointing = (pnt.azimuth, pnt.altitude)
             if current_pointing != self._last_pointing_tel[tel_id]:
                 pnt.prefix = ""
-                writer.write(
+                self._writer.write(
                     f"dl1/monitoring/telescope/pointing/tel_{tel_id:03d}",
                     [event.trigger.tel[tel_id], pnt],
                 )
@@ -674,7 +671,7 @@ class DataWriter(Component):
             table_name = self.table_name(tel_id)
 
             if self.write_parameters:
-                writer.write(
+                self._writer.write(
                     table_name=f"dl1/event/telescope/parameters/{table_name}",
                     containers=[tel_index, *dl1_camera.parameters.values()],
                 )
@@ -685,14 +682,14 @@ class DataWriter(Component):
                         "DataWriter.write_images is True but event does not contain image"
                     )
 
-                writer.write(
+                self._writer.write(
                     table_name=f"dl1/event/telescope/images/{table_name}",
                     containers=[tel_index, dl1_camera],
                 )
 
             if self._is_simulation:
                 # always write this, so that at least the sum is included
-                writer.write(
+                self._writer.write(
                     f"simulation/event/telescope/images/{table_name}",
                     [tel_index, event.simulation.tel[tel_id]],
                 )
@@ -705,7 +702,7 @@ class DataWriter(Component):
                     true_parameters = event.simulation.tel[tel_id].true_parameters
                     # only write the available containers, no peak time related
                     # features for true image available.
-                    writer.write(
+                    self._writer.write(
                         f"simulation/event/telescope/parameters/{table_name}",
                         [
                             tel_index,
@@ -717,21 +714,17 @@ class DataWriter(Component):
                         ],
                     )
 
-    def _write_muon_telescope_events(
-        self, writer: TableWriter, event: ArrayEventContainer
-    ):
+    def _write_muon_telescope_events(self, event: ArrayEventContainer):
 
         for tel_id, muon in event.muon.tel.items():
             table_name = self.table_name(tel_id)
             tel_index = _get_tel_index(event, tel_id)
-            writer.write(
+            self._writer.write(
                 f"dl1/event/telescope/muon/{table_name}",
                 [tel_index, muon.ring, muon.parameters, muon.efficiency],
             )
 
-    def _write_dl2_telescope_events(
-        self, writer: TableWriter, event: ArrayEventContainer
-    ):
+    def _write_dl2_telescope_events(self, event: ArrayEventContainer):
         """
         write per-telescope DL2 shower information.
 
@@ -749,9 +742,11 @@ class DataWriter(Component):
                         f"dl2/event/telescope/{container_name}/{algorithm}/{table_name}"
                     )
 
-                    writer.write(table_name=name, containers=[tel_index, container])
+                    self._writer.write(
+                        table_name=name, containers=[tel_index, container]
+                    )
 
-    def _write_dl2_stereo_event(self, writer: TableWriter, event: ArrayEventContainer):
+    def _write_dl2_stereo_event(self, event: ArrayEventContainer):
         """
         write per-telescope DL2 shower information to e.g.
         `/dl2/event/stereo/{geometry,energy,classification}/<algorithm_name>`
@@ -762,7 +757,7 @@ class DataWriter(Component):
                 # note this will only write info if the particular algorithm
                 # generated it (otherwise the algorithm map is empty, and no
                 # data will be written)
-                writer.write(
+                self._writer.write(
                     table_name=f"dl2/event/subarray/{container_name}/{algorithm}",
                     containers=[event.index, container],
                 )
diff --git a/ctapipe/tools/dump_triggers.py b/ctapipe/tools/dump_triggers.py
deleted file mode 100644
index af03cdd8..00000000
--- a/ctapipe/tools/dump_triggers.py
+++ /dev/null
@@ -1,142 +0,0 @@
-"""
-dump a table of the event times and trigger patterns from a
-simtelarray input file.
-"""
-
-import numpy as np
-from astropy import units as u
-from astropy.table import Table
-
-from ..core import Provenance, Tool
-from ..core.traits import Dict, Path, Unicode
-from ..io import EventSource
-
-MAX_TELS = 1000
-
-
-class DumpTriggersTool(Tool):
-    description = Unicode(__doc__)
-    name = "ctapipe-dump-triggers"
-
-    # =============================================
-    # configuration parameters:
-    # =============================================
-
-    input_path = Path(
-        exists=True, directory_ok=False, help="input simtelarray file", allow_none=False
-    ).tag(config=True)
-
-    output_path = Path(
-        default_value="triggers.fits",
-        directory_ok=False,
-        help="output filename (*.fits, *.h5)",
-    ).tag(config=True)
-
-    # =============================================
-    # map low-level options to high-level command-line options
-    # =============================================
-
-    aliases = Dict(
-        {
-            "input": "DumpTriggersTool.input_path",
-            "output": "DumpTriggersTool.output_path",
-        }
-    )
-
-    examples = (
-        "ctapipe-dump-triggers --input gamma.simtel.gz "
-        "--output trig.fits --overwrite"
-        "\n\n"
-        "If you want to see more output, use --log_level=DEBUG"
-    )
-
-    # =============================================
-    # The methods of the Tool (initialize, start, finish):
-    # =============================================
-
-    def add_event_to_table(self, event):
-        """
-        add the current hessio event to a row in the `self.events` table
-        """
-        time = event.trigger.time
-
-        if self._prev_time is None:
-            self._prev_time = time
-
-        if self._current_starttime is None:
-            self._current_starttime = time
-
-        relative_time = time - self._current_starttime
-        delta_t = time - self._prev_time
-        self._prev_time = time
-
-        # build the trigger pattern as a fixed-length array
-        # (better for storage in FITS format)
-        # trigtels = event.get_telescope_with_data_list()
-        trigtels = event.dl0.tel.keys()
-        self._current_trigpattern[:] = 0  # zero the trigger pattern
-        self._current_trigpattern[list(trigtels)] = 1  # set the triggered tels
-        # to 1
-
-        # insert the row into the table
-        self.events.add_row(
-            (
-                event.index.event_id,
-                relative_time.sec,
-                delta_t.sec,
-                len(trigtels),
-                self._current_trigpattern,
-            )
-        )
-
-    def setup(self):
-        """setup function, called before `start()`"""
-
-        self.check_output(self.output_path)
-        self.events = Table(
-            names=["EVENT_ID", "T_REL", "DELTA_T", "N_TRIG", "TRIGGERED_TELS"],
-            dtype=[np.int64, np.float64, np.float64, np.int32, np.uint8],
-        )
-
-        self.events["TRIGGERED_TELS"].shape = (0, MAX_TELS)
-        self.events["T_REL"].unit = u.s
-        self.events["T_REL"].description = "Time relative to first event"
-        self.events["DELTA_T"].unit = u.s
-        self.events.meta["INPUT"] = str(self.input_path)
-
-        self._current_trigpattern = np.zeros(MAX_TELS)
-        self._current_starttime = None
-        self._prev_time = None
-
-    def start(self):
-        """main event loop"""
-        with EventSource(self.input_path) as source:
-            for event in source:
-                self.add_event_to_table(event)
-
-    def finish(self):
-        """
-        finish up and write out results (called automatically after
-        `start()`)
-        """
-        # write out the final table
-        try:
-            if ".fits" in self.output_path.suffixes:
-                self.events.write(self.output_path, overwrite=self.overwrite)
-            elif self.output_path.suffix in (".hdf5", ".h5", ".hdf"):
-                self.events.write(
-                    self.output_path, path="/events", overwrite=self.overwrite
-                )
-            else:
-                self.events.write(self.output_path)
-
-            Provenance().add_output_file(self.output_path)
-        except IOError as err:
-            self.log.warning("Couldn't write output (%s)", err)
-
-        self.log.info("\n %s", self.events)
-
-
-def main():
-    tool = DumpTriggersTool()
-    tool.run()
diff --git a/ctapipe/tools/process.py b/ctapipe/tools/process.py
index b7dc27de..32848532 100644
--- a/ctapipe/tools/process.py
+++ b/ctapipe/tools/process.py
@@ -282,6 +282,7 @@ class ProcessorTool(Tool):
         """
         Process events
         """
+        self.log.info("applying calibration: %s", self.should_calibrate)
         self.log.info("(re)compute DL1: %s", self.should_compute_dl1)
         self.log.info("(re)compute DL2: %s", self.should_compute_dl2)
         self.log.info(
diff --git a/docs/changes/2375.api.rst b/docs/changes/2375.api.rst
new file mode 100644
index 00000000..36ab27b3
--- /dev/null
+++ b/docs/changes/2375.api.rst
@@ -0,0 +1,5 @@
+The ``ctapipe-dump-triggers`` tool was removed, since it wrote a custom data format
+not compatble with e.g. the output of the ``DataWriter`` and ``ctapipe-process``.
+If you only want to store trigger and simulation information from simulated / DL0
+input files into the ctapipe format HDF5 files, you can now use
+``ctapipe-process -i <input> -o <output> --no-write-parameters``.
diff --git a/setup.cfg b/setup.cfg
index 5092c8e6..6386aa80 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -88,7 +88,6 @@ exclude =
 [options.entry_points]
 console_scripts =
     ctapipe-info = ctapipe.tools.info:main
-    ctapipe-dump-triggers = ctapipe.tools.dump_triggers:main
     ctapipe-dump-instrument=ctapipe.tools.dump_instrument:main
     ctapipe-display-dl1 = ctapipe.tools.display_dl1:main
     ctapipe-process = ctapipe.tools.process:main


Test patch:
diff --git a/ctapipe/tools/tests/test_process.py b/ctapipe/tools/tests/test_process.py
index 04a87079..01039649 100644
--- a/ctapipe/tools/tests/test_process.py
+++ b/ctapipe/tools/tests/test_process.py
@@ -478,3 +478,25 @@ def test_plugin_help(capsys):
     assert (
         "PluginReconstructor.foo" in captured.out
     ), "Tool help is missing plugin classes, did you run `pip install -e ./test_plugin`?"
+
+
+def test_only_trigger_and_simulation(tmp_path):
+    output = tmp_path / "only_trigger_and_simulation.h5"
+
+    run_tool(
+        ProcessorTool(),
+        argv=[
+            "--input=dataset://gamma_prod5.simtel.zst",
+            f"--output={output}",
+            "--no-write-parameters",
+            "--overwrite",
+        ],
+        cwd=tmp_path,
+        raises=True,
+    )
+
+    with TableLoader(output, load_simulated=True) as loader:
+        events = loader.read_subarray_events()
+        assert len(events) == 7
+        assert "tels_with_trigger" in events.colnames
+        assert "true_energy" in events.colnames
diff --git a/ctapipe/tools/tests/test_tools.py b/ctapipe/tools/tests/test_tools.py
index c31c7736..c7e2dd46 100644
--- a/ctapipe/tools/tests/test_tools.py
+++ b/ctapipe/tools/tests/test_tools.py
@@ -81,19 +81,6 @@ def test_fileinfo(tmp_path, dl1_image_file):
     assert "CTA ACTIVITY ID" in header[str(dl1_image_file)]
 
 
-def test_dump_triggers(tmp_path):
-    from ctapipe.tools.dump_triggers import DumpTriggersTool
-
-    sys.argv = ["dump_triggers"]
-    output_path = tmp_path / "triggers.fits"
-    tool = DumpTriggersTool(input_path=PROD5B_PATH, output_path=str(output_path))
-
-    assert run_tool(tool, cwd=tmp_path) == 0
-
-    assert output_path.exists()
-    assert run_tool(tool, ["--help-all"]) == 0
-
-
 def test_dump_instrument(tmp_path):
     from ctapipe.tools.dump_instrument import DumpInstrumentTool