Repository: cta-observatory/ctapipe
Pull request #2520 (2024-03-04 14:14:07)

Refactor/rename datawriter options (redux)
This replaces #1920 and is based on latest main branch




Patch:
diff --git a/docs/changes/2520.api.rst b/docs/changes/2520.api.rst
new file mode 100644
index 00000000..bd20f325
--- /dev/null
+++ b/docs/changes/2520.api.rst
@@ -0,0 +1,18 @@
+Unified the options for DataWriter and the data level names:
+
++-------------------------+--------------------------+
+| Old                     | New                      |
++=========================+==========================+
+| ``write_raw_waveforms`` | ``write_r0_waveforms``   |
++-------------------------+--------------------------+
+| ``write_waveforms``     | ``write_r1_waveforms``   |
++-------------------------+--------------------------+
+| ``write_images``        | ``write_dl1_images``     |
++-------------------------+--------------------------+
+| ``write_parameters``    | ``write_dl1_parameters`` |
++-------------------------+--------------------------+
+| ``write_showers``       | ``write_dl2``            |
++-------------------------+--------------------------+
+
+This changes requires that existing configuration files are updated
+if they use these parameters
diff --git a/examples/tutorials/ctapipe_overview.py b/examples/tutorials/ctapipe_overview.py
index ced2606c..38ed0ac3 100644
--- a/examples/tutorials/ctapipe_overview.py
+++ b/examples/tutorials/ctapipe_overview.py
@@ -369,9 +369,7 @@ horizon_frame = AltAz()
 
 f = tempfile.NamedTemporaryFile(suffix=".hdf5")
 
-with DataWriter(
-    source, output_path=f.name, overwrite=True, write_showers=True
-) as writer:
+with DataWriter(source, output_path=f.name, overwrite=True, write_dl2=True) as writer:
     for event in source:
         energy = event.simulation.shower.energy
         n_telescopes_r1 = len(event.r1.tel)
diff --git a/examples/tutorials/tableloader_and_eventsources.py b/examples/tutorials/tableloader_and_eventsources.py
index 65db252c..815654a4 100644
--- a/examples/tutorials/tableloader_and_eventsources.py
+++ b/examples/tutorials/tableloader_and_eventsources.py
@@ -138,9 +138,9 @@ calibrate = CameraCalibrator(subarray=source.subarray)
 with DataWriter(
     event_source=source,
     output_path="events.dl1.h5",
-    write_parameters=False,
+    write_dl1_parameters=False,
     overwrite=True,
-    write_images=True,
+    write_dl1_images=True,
 ) as write_data:
     for event in source:
         calibrate(event)
diff --git a/src/ctapipe/io/datawriter.py b/src/ctapipe/io/datawriter.py
index 5408fbbb..637177a4 100644
--- a/src/ctapipe/io/datawriter.py
+++ b/src/ctapipe/io/datawriter.py
@@ -183,23 +183,23 @@ class DataWriter(Component):
         help="output filename", default_value=pathlib.Path("events.dl1.h5")
     ).tag(config=True)
 
-    write_raw_waveforms = Bool(
+    write_r0_waveforms = Bool(
         help="Store R0 waveforms if available", default_value=False
     ).tag(config=True)
 
-    write_waveforms = Bool(
+    write_r1_waveforms = Bool(
         help="Store R1 waveforms if available", default_value=False
     ).tag(config=True)
 
-    write_images = Bool(help="Store DL1 Images if available", default_value=False).tag(
-        config=True
-    )
+    write_dl1_images = Bool(
+        help="Store DL1 Images if available", default_value=False
+    ).tag(config=True)
 
-    write_parameters = Bool(
+    write_dl1_parameters = Bool(
         help="Store DL1 image parameters if available", default_value=True
     ).tag(config=True)
 
-    write_showers = Bool(
+    write_dl2 = Bool(
         help="Store DL2 stereo shower parameters if available", default_value=False
     ).tag(config=True)
 
@@ -322,17 +322,17 @@ class DataWriter(Component):
                     [tel_index, sim.impact],
                 )
 
-        if self.write_waveforms:
+        if self.write_r1_waveforms:
             self._write_r1_telescope_events(event)
 
-        if self.write_raw_waveforms:
+        if self.write_r0_waveforms:
             self._write_r0_telescope_events(event)
 
         # write telescope event data
         self._write_dl1_telescope_events(event)
 
         # write DL2 info if requested
-        if self.write_showers:
+        if self.write_dl2:
             self._write_dl2_telescope_events(event)
             self._write_dl2_stereo_event(event)
 
@@ -392,17 +392,17 @@ class DataWriter(Component):
     def datalevels(self):
         """returns a list of data levels requested"""
         data_levels = []
-        if self.write_images:
+        if self.write_dl1_images:
             data_levels.append(DataLevel.DL1_IMAGES)
-        if self.write_parameters:
+        if self.write_dl1_parameters:
             data_levels.append(DataLevel.DL1_PARAMETERS)
         if self.write_muon_parameters:
             data_levels.append(DataLevel.DL1_MUON)
-        if self.write_showers:
+        if self.write_dl2:
             data_levels.append(DataLevel.DL2)
-        if self.write_raw_waveforms:
+        if self.write_r0_waveforms:
             data_levels.append(DataLevel.R0)
-        if self.write_waveforms:
+        if self.write_r1_waveforms:
             data_levels.append(DataLevel.R1)
         return data_levels
 
@@ -435,10 +435,10 @@ class DataWriter(Component):
 
         # check that options make sense
         writable_things = [
-            self.write_parameters,
-            self.write_images,
-            self.write_showers,
-            self.write_waveforms,
+            self.write_dl1_parameters,
+            self.write_dl1_images,
+            self.write_dl2,
+            self.write_r1_waveforms,
             self.write_muon_parameters,
         ]
         if not any(writable_things):
@@ -472,10 +472,10 @@ class DataWriter(Component):
         writer.exclude("/dl1/event/telescope/images/.*", "parameters")
         writer.exclude("/simulation/event/telescope/images/.*", "true_parameters")
 
-        if not self.write_images:
+        if not self.write_dl1_images:
             writer.exclude("/simulation/event/telescope/images/.*", "true_image")
 
-        if not self.write_parameters:
+        if not self.write_dl1_parameters:
             writer.exclude("/dl1/event/telescope/images/.*", "image_mask")
 
         # Set up transforms
@@ -671,16 +671,16 @@ class DataWriter(Component):
 
             table_name = self.table_name(tel_id)
 
-            if self.write_parameters:
+            if self.write_dl1_parameters:
                 self._writer.write(
                     table_name=f"dl1/event/telescope/parameters/{table_name}",
                     containers=[tel_index, *dl1_camera.parameters.values()],
                 )
 
-            if self.write_images:
+            if self.write_dl1_images:
                 if dl1_camera.image is None:
                     raise ValueError(
-                        "DataWriter.write_images is True but event does not contain image"
+                        "DataWriter.write_dl1_images is True but event does not contain image"
                     )
 
                 self._writer.write(
@@ -699,7 +699,7 @@ class DataWriter(Component):
                     tel_id in event.simulation.tel
                     and event.simulation.tel[tel_id].true_image is not None
                 )
-                if self.write_parameters and has_sim_image:
+                if self.write_dl1_parameters and has_sim_image:
                     true_parameters = event.simulation.tel[tel_id].true_parameters
                     # only write the available containers, no peak time related
                     # features for true image available.
@@ -783,7 +783,7 @@ class DataWriter(Component):
     def _generate_indices(self):
         """generate PyTables index tables for common columns"""
         self.log.debug("Writing index tables")
-        if self.write_images:
+        if self.write_dl1_images:
             self._generate_table_indices(
                 self._writer.h5file, "/dl1/event/telescope/images"
             )
@@ -791,7 +791,7 @@ class DataWriter(Component):
                 self._generate_table_indices(
                     self._writer.h5file, "/simulation/event/telescope/images"
                 )
-        if self.write_parameters:
+        if self.write_dl1_parameters:
             self._generate_table_indices(
                 self._writer.h5file, "/dl1/event/telescope/parameters"
             )
diff --git a/src/ctapipe/resources/base_config.yaml b/src/ctapipe/resources/base_config.yaml
index 13fa1a20..d6d570fa 100644
--- a/src/ctapipe/resources/base_config.yaml
+++ b/src/ctapipe/resources/base_config.yaml
@@ -21,11 +21,11 @@ DataWriter:
   # write nothing (can be overridden on the command-line or in subsequent config
   # files)
   overwrite: false # do not overwrite existing files
-  write_images: false # store DL1 images
-  write_parameters: false # store DL1 parameters
-  write_showers: false # store DL2 stereo geometry
-  write_raw_waveforms: false # write R0 waveforms
-  write_waveforms: false # write R1 waveforms
+  write_dl1_images: false # store DL1 images
+  write_dl1_parameters: false # store DL1 parameters
+  write_dl2: false # store DL2 stereo geometry
+  write_r0_waveforms: false # write R0 waveforms
+  write_r1_waveforms: false # write R1 waveforms
   write_muon_parameters: false # analyze and write muon parameters
 
 # The CameraCalibrator takes data from R1 or DL0 to DL1a level, applying finer
@@ -95,7 +95,7 @@ ImageProcessor:
 
 
 # The ShowerProcessor performs the DL1 to DL2a (reconstructed shower geometry)
-# transition. It is run only if the parameter DataWriter.write_showers=True.
+# transition. It is run only if the parameter DataWriter.write_dl2=True.
 ShowerProcessor:
   # choose between HillasReconstructor and HillasIntersection or both (two
   # implementations of the standard stereo line-intersection method)
diff --git a/src/ctapipe/resources/image_modification_config.json b/src/ctapipe/resources/image_modification_config.json
index 90e92c4c..253e10be 100644
--- a/src/ctapipe/resources/image_modification_config.json
+++ b/src/ctapipe/resources/image_modification_config.json
@@ -6,9 +6,9 @@
             "organization": "CTA Consortium"
         },
         "overwrite": false,
-        "write_images": true,
-        "write_parameters": true,
-        "write_showers": false,
+        "write_dl1_images": true,
+        "write_dl1_parameters": true,
+        "write_dl2": false,
         "transform_image": true,
         "transform_peak_time": true
     },
diff --git a/src/ctapipe/resources/ml_preprocessing_config.json b/src/ctapipe/resources/ml_preprocessing_config.json
index 04109454..1fecee85 100644
--- a/src/ctapipe/resources/ml_preprocessing_config.json
+++ b/src/ctapipe/resources/ml_preprocessing_config.json
@@ -6,9 +6,9 @@
             "organization": "YOUR-ORGANIZATION"
         },
         "overwrite": false,
-        "write_images": false,
-        "write_parameters": true,
-        "write_showers": true,
+        "write_dl1_images": false,
+        "write_dl1_parameters": true,
+        "write_dl2": true,
         "transform_image": true,
         "transform_peak_time": true
     },
diff --git a/src/ctapipe/resources/ml_preprocessing_config.yaml b/src/ctapipe/resources/ml_preprocessing_config.yaml
index 28f8644d..eeeef143 100644
--- a/src/ctapipe/resources/ml_preprocessing_config.yaml
+++ b/src/ctapipe/resources/ml_preprocessing_config.yaml
@@ -5,13 +5,13 @@
 # Perform Stage 1 and Stage 2a processing to generate features needed for
 # training machine learning algorithms.
 #
-# Note that here write_images is disabled by default (to avoid very large
+# Note that here write_dl1_images is disabled by default (to avoid very large
 # files), however for training deep learning algorithms, it should be turned on.
 # ======================================================================
 #
 #  Make sure you first include `--config base_config.yaml` before including this file
 
 DataWriter:
-  write_images: false
-  write_parameters: true
-  write_showers: true
+  write_dl1_images: false
+  write_dl1_parameters: true
+  write_dl2: true
diff --git a/src/ctapipe/resources/stage1_config.json b/src/ctapipe/resources/stage1_config.json
index c414f5d3..1dcb765a 100644
--- a/src/ctapipe/resources/stage1_config.json
+++ b/src/ctapipe/resources/stage1_config.json
@@ -6,9 +6,9 @@
             "organization": "YOUR-ORGANIZATION"
         },
         "overwrite": false,
-        "write_images": true,
-        "write_parameters": true,
-        "write_showers": false,
+        "write_dl1_images": true,
+        "write_dl1_parameters": true,
+        "write_dl2": false,
         "transform_image": true,
         "transform_peak_time": true
     },
diff --git a/src/ctapipe/resources/stage1_config.toml b/src/ctapipe/resources/stage1_config.toml
index 296798bc..776293a5 100644
--- a/src/ctapipe/resources/stage1_config.toml
+++ b/src/ctapipe/resources/stage1_config.toml
@@ -8,9 +8,9 @@ organization = "YOUR-ORGANIZATION"
 [DataWriter]
 # options that control what is stored in the output file
 overwrite = false
-write_images = true
-write_parameters = true
-write_showers = false
+write_dl1_images = true
+write_dl1_parameters = true
+write_dl2 = false
 transform_image = true
 transform_peak_time = true
 
diff --git a/src/ctapipe/resources/stage1_config.yaml b/src/ctapipe/resources/stage1_config.yaml
index d0413061..9f5cb100 100644
--- a/src/ctapipe/resources/stage1_config.yaml
+++ b/src/ctapipe/resources/stage1_config.yaml
@@ -10,6 +10,6 @@
 #  Make sure you first include `--config base_config.yaml` before including this file
 
 DataWriter:
-  write_images: true
-  write_parameters: true
-  write_showers: false
+  write_dl1_images: true
+  write_dl1_parameters: true
+  write_dl2: false
diff --git a/src/ctapipe/resources/stage2_config.json b/src/ctapipe/resources/stage2_config.json
index 76c05fff..3fe35426 100644
--- a/src/ctapipe/resources/stage2_config.json
+++ b/src/ctapipe/resources/stage2_config.json
@@ -6,9 +6,9 @@
             "organization": "YOUR-ORGANIZATION"
         },
         "overwrite": false,
-        "write_images": false,
-        "write_parameters": false,
-        "write_showers": true,
+        "write_dl1_images": false,
+        "write_dl1_parameters": false,
+        "write_dl2": true,
         "transform_image": true,
         "transform_peak_time": true
     },
diff --git a/src/ctapipe/resources/stage2_config.yaml b/src/ctapipe/resources/stage2_config.yaml
index 992fbe52..e0000242 100644
--- a/src/ctapipe/resources/stage2_config.yaml
+++ b/src/ctapipe/resources/stage2_config.yaml
@@ -11,6 +11,6 @@
 #  Make sure you first include `--config base_config.yaml` before including this file
 
 DataWriter:
-  write_images: false
-  write_parameters: false
-  write_showers: true
+  write_dl1_images: false
+  write_dl1_parameters: false
+  write_dl2: true
diff --git a/src/ctapipe/tools/process.py b/src/ctapipe/tools/process.py
index 06e739cf..dc0123c5 100644
--- a/src/ctapipe/tools/process.py
+++ b/src/ctapipe/tools/process.py
@@ -43,7 +43,7 @@ class ProcessorTool(Tool):
 
     Note that the muon analysis and shower reconstruction both depend on
     parametrized images and therefore compute image parameters even if
-    DataWriter.write_parameters=False in case these are not already present
+    DataWriter.write_dl1_parameters=False in case these are not already present
     in the input file.
     """
 
@@ -110,19 +110,19 @@ class ProcessorTool(Tool):
         ),
         **flag(
             "write-images",
-            "DataWriter.write_images",
+            "DataWriter.write_dl1_images",
             "store DL1/Event/Telescope images in output",
             "don't store DL1/Event/Telescope images in output",
         ),
         **flag(
             "write-parameters",
-            "DataWriter.write_parameters",
+            "DataWriter.write_dl1_parameters",
             "store DL1/Event/Telescope parameters in output",
             "don't store DL1/Event/Telescope parameters in output",
         ),
         **flag(
             "write-showers",
-            "DataWriter.write_showers",
+            "DataWriter.write_dl2",
             "store DL2/Event parameters in output",
             "don't DL2/Event parameters in output",
         ),
@@ -209,7 +209,7 @@ class ProcessorTool(Tool):
         if self.force_recompute_dl2:
             return True
 
-        return self.write.write_showers
+        return self.write.write_dl2
 
     @property
     def should_compute_dl1(self):
@@ -221,7 +221,7 @@ class ProcessorTool(Tool):
             return False
 
         return (
-            self.write.write_parameters
+            self.write.write_dl1_parameters
             or self.should_compute_dl2
             or self.should_compute_muon_parameters
         )
@@ -233,7 +233,7 @@ class ProcessorTool(Tool):
             return True
 
         if (
-            self.write.write_images
+            self.write.write_dl1_images
             and DataLevel.DL1_IMAGES not in self.event_source.datalevels
         ):
             return True


Test patch:
diff --git a/src/ctapipe/conftest.py b/src/ctapipe/conftest.py
index 7efaa992..b521d82c 100644
--- a/src/ctapipe/conftest.py
+++ b/src/ctapipe/conftest.py
@@ -415,7 +415,7 @@ def dl1_image_file(dl1_tmp_path, prod5_gamma_simtel_path):
             f"--input={prod5_gamma_simtel_path}",
             f"--output={output}",
             "--write-images",
-            "--DataWriter.write_parameters=False",
+            "--DataWriter.write_dl1_parameters=False",
             "--max-events=20",
             "--DataWriter.Contact.name=αℓℓ the äüöß",
         ]
diff --git a/src/ctapipe/io/tests/conftest.py b/src/ctapipe/io/tests/conftest.py
index 758ee7d1..9f4e6e21 100644
--- a/src/ctapipe/io/tests/conftest.py
+++ b/src/ctapipe/io/tests/conftest.py
@@ -20,11 +20,11 @@ def r1_hdf5_file(prod5_proton_simtel_path, r1_path):
     writer = DataWriter(
         event_source=source,
         output_path=path,
-        write_parameters=False,
-        write_images=False,
-        write_showers=False,
-        write_raw_waveforms=False,
-        write_waveforms=True,
+        write_dl1_parameters=False,
+        write_dl1_images=False,
+        write_dl2=False,
+        write_r0_waveforms=False,
+        write_r1_waveforms=True,
     )
 
     for e in source:
diff --git a/src/ctapipe/io/tests/test_datawriter.py b/src/ctapipe/io/tests/test_datawriter.py
index a39ad4c4..72450ea1 100644
--- a/src/ctapipe/io/tests/test_datawriter.py
+++ b/src/ctapipe/io/tests/test_datawriter.py
@@ -80,11 +80,11 @@ def test_write(tmpdir: Path):
     with DataWriter(
         event_source=source,
         output_path=output_path,
-        write_parameters=False,
-        write_images=True,
-        write_showers=True,
-        write_raw_waveforms=True,
-        write_waveforms=True,
+        write_dl1_parameters=False,
+        write_dl1_images=True,
+        write_dl2=True,
+        write_r0_waveforms=True,
+        write_r1_waveforms=True,
     ) as writer:
         writer.log.level = logging.DEBUG
         for event in source:
@@ -158,15 +158,15 @@ def test_roundtrip(tmpdir: Path):
     with DataWriter(
         event_source=source,
         output_path=output_path,
-        write_parameters=False,
-        write_images=True,
+        write_dl1_parameters=False,
+        write_dl1_images=True,
         transform_image=True,
         image_dtype="int32",
         image_scale=10,
         transform_peak_time=True,
         peak_time_dtype="int16",
         peak_time_scale=100,
-        write_showers=True,
+        write_dl2=True,
     ) as write:
         write.log.level = logging.DEBUG
         for event in source:
@@ -229,8 +229,8 @@ def test_dl1writer_no_events(tmpdir: Path):
     with DataWriter(
         event_source=source,
         output_path=output_path,
-        write_parameters=True,
-        write_images=True,
+        write_dl1_parameters=True,
+        write_dl1_images=True,
     ) as writer:
         writer.log.level = logging.DEBUG
         writer.write_simulation_histograms(source)
@@ -269,8 +269,8 @@ def test_metadata(tmpdir: Path):
         with DataWriter(
             event_source=source,
             output_path=output_path,
-            write_parameters=True,
-            write_images=True,
+            write_dl1_parameters=True,
+            write_dl1_images=True,
             config=config,
         ):
             pass
diff --git a/src/ctapipe/tools/tests/test_process.py b/src/ctapipe/tools/tests/test_process.py
index 95237160..769574c7 100644
--- a/src/ctapipe/tools/tests/test_process.py
+++ b/src/ctapipe/tools/tests/test_process.py
@@ -62,10 +62,7 @@ def test_multiple_configs(dl1_image_file):
     tool.setup()
 
     # ensure the overwriting works (base config has this option disabled)
-    assert (
-        tool.get_current_config()["ProcessorTool"]["DataWriter"]["write_showers"]
-        is True
-    )
+    assert tool.get_current_config()["ProcessorTool"]["DataWriter"]["write_dl2"] is True
 
 
 def test_stage_1_dl1(tmp_path, dl1_image_file, dl1_parameters_file):