Repository: cta-observatory/ctapipe
Pull request #1866 (2022-03-30 17:03:57)

Fix traitlets deprecation warning
Fixes "Explicit usage of default_value=Undefined is deprecated"


Patch:
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 09769dc0..73ef4689 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -18,9 +18,31 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-latest, macos-latest]
-        python-version: ["3.8", ]
-        install-method: ["pip", "conda"]
+        include:
+          - os: ubuntu-latest
+            python-version: "3.8"
+            install-method: conda
+
+          - os: ubuntu-latest
+            python-version: "3.9"
+            install-method: conda
+
+          - os: ubuntu-latest
+            python-version: "3.10"
+            install-method: conda
+
+          - os: ubuntu-latest
+            python-version: "3.10"
+            install-method: pip
+
+          - os: macos-latest
+            python-version: "3.10"
+            install-method: conda
+
+          - os: macos-latest
+            python-version: "3.8"
+            install-method: pip
+
     defaults:
       run:
         shell: bash -l {0}
@@ -59,6 +81,8 @@ jobs:
         if: matrix.install-method == 'pip'
         uses: conda-incubator/setup-miniconda@v2
         with:
+          mamba-version: "*"
+          channels: conda-forge,default
           python-version: ${{ matrix.python-version }}
 
       - name: Update pip
@@ -82,7 +106,12 @@ jobs:
         # need to use a login shell for the conda setup to work
         shell: bash -leo pipefail {0}
         run: |
-          pytest -n auto --dist loadscope --cov --cov-report=xml --doctest-modules --doctest-glob='*.rst' ctapipe docs
+          pytest -n auto --dist loadscope \
+            --cov --cov-report=xml \
+            --doctest-modules --doctest-glob='*.rst' \
+            --ignore=docs/conf.py \
+            ctapipe docs
+
           ctapipe-info --version
 
       - uses: codecov/codecov-action@v1
diff --git a/ctapipe/instrument/camera/geometry.py b/ctapipe/instrument/camera/geometry.py
index bf834624..48fbec3e 100644
--- a/ctapipe/instrument/camera/geometry.py
+++ b/ctapipe/instrument/camera/geometry.py
@@ -407,6 +407,27 @@ class CameraGeometry:
         """
         return ~np.any(~np.isclose(self.pix_area.value, self.pix_area[0].value), axis=0)
 
+    def image_index_to_cartesian_index(self, pixel_index):
+        '''
+        Convert pixel index in the 1d image representation to row and col
+        '''
+        rows, cols = self._pixel_positions_2d
+        return rows[pixel_index], cols[pixel_index]
+
+    def cartesian_index_to_image_index(self, row, col):
+        '''
+        Convert cartesian index (row, col) to pixel index in 1d representation.
+        '''
+        return self._pixel_indices_cartesian[row, col]
+
+    @lazyproperty
+    def _pixel_indices_cartesian(self):
+        img = np.arange(self.n_pixels)
+        img2d = self.image_to_cartesian_representation(img)
+        invalid = np.iinfo(np.int32).min
+        img2d = np.nan_to_num(img2d, nan=invalid).astype(np.int32)
+        return img2d
+
     @lazyproperty
     def _pixel_positions_2d(self):
         """
@@ -456,12 +477,21 @@ class CameraGeometry:
             pixel_row = rows_1d
             pixel_column = cols_1d
 
+            # flip image so that imshow looks like original camera display
+            pixel_row = pixel_row.max() - pixel_row
+            pixel_column = pixel_column.max() - pixel_column
+
         elif self.pix_type is PixelShape.SQUARE:
             pixel_row = get_orthogonal_grid_indices(self.pix_y, np.sqrt(self.pix_area))
             pixel_column = get_orthogonal_grid_indices(
                 self.pix_x, np.sqrt(self.pix_area)
             )
 
+            # flip image so that imshow looks like original camera display
+            pixel_row = pixel_row.max() - pixel_row
+        else:
+            raise ValueError(f"Unsupported pixel shape {self.pix_type}")
+
         return pixel_row, pixel_column
 
     def image_to_cartesian_representation(self, image):
@@ -484,12 +514,10 @@ class CameraGeometry:
         """
         rows, cols = self._pixel_positions_2d
         image = np.atleast_2d(image)  # this allows for multiple images at once
+
         image_2d = np.full((image.shape[0], rows.max() + 1, cols.max() + 1), np.nan)
         image_2d[:, rows, cols] = image
-        if self.pix_type == PixelShape.SQUARE:
-            image_2d = np.flip(image_2d, axis=1)
-        else:
-            image_2d = np.flip(image_2d)
+
         return np.squeeze(image_2d)  # removes the extra dimension for single images
 
     def image_from_cartesian_representation(self, image_2d):
@@ -515,10 +543,7 @@ class CameraGeometry:
         # to a different shape compared to a multi image array
         if image_2d.ndim == 2:
             image_2d = image_2d[np.newaxis, :]
-        if self.pix_type == PixelShape.SQUARE:
-            image_2d = np.flip(image_2d, axis=1)
-        else:
-            image_2d = np.flip(image_2d)
+
         image_flat = np.zeros((image_2d.shape[0], rows.shape[0]), dtype=image_2d.dtype)
         image_flat[:] = image_2d[:, rows, cols]
         image_1d = image_flat
diff --git a/ctapipe/instrument/camera/image_conversion.py b/ctapipe/instrument/camera/image_conversion.py
index 19b0e1e7..c4131a77 100644
--- a/ctapipe/instrument/camera/image_conversion.py
+++ b/ctapipe/instrument/camera/image_conversion.py
@@ -272,8 +272,8 @@ def get_orthogonal_grid_edges(pix_x, pix_y, scale_aspect=True):
 
     # finding the size of the square patches
 
-    d_x = 99
-    d_y = 99
+    d_x = np.inf
+    d_y = np.inf
     x_base = pix_x[0]
     y_base = pix_y[0]
     for x, y in zip(pix_x, pix_y):
diff --git a/ctapipe/instrument/subarray.py b/ctapipe/instrument/subarray.py
index fa3f315f..557040b4 100644
--- a/ctapipe/instrument/subarray.py
+++ b/ctapipe/instrument/subarray.py
@@ -2,7 +2,7 @@
 Description of Arrays or Subarrays of telescopes
 """
 from typing import Dict, List, Union
-from pathlib import Path
+from contextlib import ExitStack
 import warnings
 
 import numpy as np
@@ -449,61 +449,69 @@ class SubarrayDescription:
                 return False
         return True
 
-    def to_hdf(self, output_path, overwrite=False):
+    def to_hdf(self, h5file, overwrite=False, mode="a"):
         """write the SubarrayDescription
 
         Parameters
         ----------
-        subarray : ctapipe.instrument.SubarrayDescription
-            subarray description
+        h5file : str, bytes, path or tables.File
+            Path or already opened tables.File with write permission
+        overwrite : False
+            If the output path already contains a subarray, by default
+            an error will be raised. Set ``overwrite=True`` to overwrite an
+            existing subarray. This does not affect other content of the file.
+            Use ``mode="w"`` to completely overwrite the output path.
+        mode : str
+            If h5file is not an already opened file, the output file will
+            be opened with the given mode. Must be a mode that enables writing.
         """
         # here to prevent circular import
         from ..io import write_table
 
-        output_path = Path(output_path)
-        if output_path.suffix not in (".h5", ".hdf", ".hdf5"):
-            raise ValueError(
-                f"This function can only write to hdf files, got {output_path.suffix}"
-            )
-
-        with tables.open_file(output_path, mode="a") as f:
+        with ExitStack() as stack:
+            if not isinstance(h5file, tables.File):
+                h5file = stack.enter_context(tables.open_file(h5file, mode=mode))
 
-            if "/configuration/instrument/subarray" in f.root:
+            if "/configuration/instrument/subarray" in h5file.root:
                 if overwrite is False:
                     raise IOError(
                         "File already contains a SubarrayDescription and overwrite=False"
                     )
 
-                f.remove_node("/configuration/instrument/", "subarray", recursive=True)
-                f.remove_node("/configuration/instrument/", "telescope", recursive=True)
+                h5file.remove_node(
+                    "/configuration/instrument/", "subarray", recursive=True
+                )
+                h5file.remove_node(
+                    "/configuration/instrument/", "telescope", recursive=True
+                )
 
             write_table(
                 self.to_table(),
-                f,
+                h5file,
                 path="/configuration/instrument/subarray/layout",
                 mode="a",
             )
             write_table(
                 self.to_table(kind="optics"),
-                f,
+                h5file,
                 path="/configuration/instrument/telescope/optics",
                 mode="a",
             )
             for i, camera in enumerate(self.camera_types):
                 write_table(
                     camera.geometry.to_table(),
-                    f,
+                    h5file,
                     path=f"/configuration/instrument/telescope/camera/geometry_{i}",
                     mode="a",
                 )
                 write_table(
                     camera.readout.to_table(),
-                    f,
+                    h5file,
                     path=f"/configuration/instrument/telescope/camera/readout_{i}",
                     mode="a",
                 )
 
-            f.root.configuration.instrument.subarray._v_attrs.name = self.name
+            h5file.root.configuration.instrument.subarray._v_attrs.name = self.name
 
     @classmethod
     def from_hdf(cls, path):
@@ -580,12 +588,14 @@ class SubarrayDescription:
 
         positions = np.column_stack([layout[f"pos_{c}"] for c in "xyz"])
 
-        with tables.open_file(path, mode="r") as f:
-            attrs = f.root.configuration.instrument.subarray._v_attrs
+        name = "Unknown"
+        with ExitStack() as stack:
+            if not isinstance(path, tables.File):
+                path = stack.enter_context(tables.open_file(path, mode="r"))
+
+            attrs = path.root.configuration.instrument.subarray._v_attrs
             if "name" in attrs:
                 name = str(attrs.name)
-            else:
-                name = "Unknown"
 
         return cls(
             name=name,
diff --git a/ctapipe/io/datawriter.py b/ctapipe/io/datawriter.py
index 55d60c9b..4ff92efd 100644
--- a/ctapipe/io/datawriter.py
+++ b/ctapipe/io/datawriter.py
@@ -6,7 +6,6 @@ Class to write DL1 (a,b) and DL2 (a) data from an event stream
 
 import pathlib
 from collections import defaultdict
-from typing import DefaultDict, Tuple
 from traitlets import Instance
 
 import numpy as np
@@ -232,20 +231,22 @@ class DataWriter(Component):
         self._subarray: SubarrayDescription = event_source.subarray
 
         self._hdf5_filters = None
-        self._last_pointing_tel: DefaultDict[Tuple] = None
-        self._last_pointing: Tuple = None
         self._writer: HDF5TableWriter = None
 
         self._setup_output_path()
-        self._subarray.to_hdf(self.output_path)  # must be first (uses astropy io)
         self._setup_compression()
         self._setup_writer()
-        if self._is_simulation:
-            self._write_simulation_configuration()
+        self._setup_outputfile()
 
         # store last pointing to only write unique poitings
+        self._last_pointing = None
         self._last_pointing_tel = defaultdict(lambda: (np.nan * u.deg, np.nan * u.deg))
 
+    def _setup_outputfile(self):
+        self._subarray.to_hdf(self._writer.h5file)
+        if self._is_simulation:
+            self._write_simulation_configuration()
+
     def __enter__(self):
         return self
 
@@ -459,7 +460,6 @@ class DataWriter(Component):
         )
 
         # final initialization
-
         self._writer = writer
         self.log.debug("Writer initialized: %s", self._writer)
 
diff --git a/ctapipe/io/eventsource.py b/ctapipe/io/eventsource.py
index 6325ff68..4b46f33c 100644
--- a/ctapipe/io/eventsource.py
+++ b/ctapipe/io/eventsource.py
@@ -385,3 +385,10 @@ class EventSource(Component):
         """
         input_url = cls._find_input_url_in_config(config=config, parent=parent)
         return cls.from_url(input_url, config=config, parent=parent, **kwargs)
+
+    def close(self):
+        """Close this event source.
+
+        No-op by default. Should be overriden by sources needing a cleanup-step
+        """
+        pass
diff --git a/ctapipe/tools/dl1_merge.py b/ctapipe/tools/merge.py
similarity index 75%
rename from ctapipe/tools/dl1_merge.py
rename to ctapipe/tools/merge.py
index 5ab71b9e..e1e64cb8 100644
--- a/ctapipe/tools/dl1_merge.py
+++ b/ctapipe/tools/merge.py
@@ -17,30 +17,18 @@ from ..core import Provenance, Tool, traits
 from ..core.traits import Bool, Set, Unicode, flag, CInt
 from ..instrument import SubarrayDescription
 
-import warnings
-
-warnings.filterwarnings("ignore", category=tables.NaturalNameWarning)
 
 PROV = Provenance()
 
 VERSION_KEY = "CTA PRODUCT DATA MODEL VERSION"
 IMAGE_STATISTICS_PATH = "/dl1/service/image_statistics"
 
-all_nodes = {
-    "/dl1/monitoring/subarray/pointing",
-    "/dl1/monitoring/telescope/pointing",
-    "/dl1/service/image_statistics",
-    "/dl1/service/image_statistics.__table_column_meta__",
+required_nodes = {
     "/dl1/event/subarray/trigger",
     "/dl1/event/telescope/trigger",
-    "/dl1/event/telescope/parameters",
-    "/dl1/event/telescope/images",
-    "/configuration/simulation/run",
-    "/simulation/event/subarray/shower",
-    "/simulation/event/telescope/parameters",
-    "/simulation/event/telescope/images",
-    "/simulation/service/shower_distribution",
+    "/dl1/monitoring/subarray/pointing",
 }
+
 optional_nodes = {
     "/simulation/service/shower_distribution",
     "/simulation/event/telescope/images",
@@ -49,8 +37,10 @@ optional_nodes = {
     "/dl1/event/telescope/images",
     "/dl1/service/image_statistics",
     "/dl1/service/image_statistics.__table_column_meta__",
+    "/dl2/event/subarray/geometry",
 }
-simu_nodes = {
+
+simulation_nodes = {
     "/simulation/event/subarray/shower",
     "/simulation/event/telescope/parameters",
     "/simulation/event/telescope/images",
@@ -73,7 +63,23 @@ parameter_nodes = {
     "/simulation/event/telescope/parameters",
     "/dl1/event/telescope/parameters",
 }
-simu_images = {"/simulation/event/telescope/images"}
+
+simulation_images = {"/simulation/event/telescope/images"}
+
+dl2_subarray_nodes = {"/dl2/event/subarray/geometry"}
+
+
+all_nodes = (
+    required_nodes
+    | optional_nodes
+    | simulation_nodes
+    | service_nodes
+    | nodes_with_tels
+    | image_nodes
+    | parameter_nodes
+    | simulation_images
+    | dl2_subarray_nodes
+)
 
 
 class MergeTool(Tool):
@@ -226,29 +232,49 @@ class MergeTool(Tool):
             )
             sys.exit(1)
 
+        # setup required nodes
+        self.usable_nodes = all_nodes.copy()
+
+        if self.skip_simu_images is True:
+            self.usable_nodes -= simulation_images
+
+        if self.skip_images is True:
+            self.usable_nodes -= image_nodes
+
+        if self.skip_parameters is True:
+            self.usable_nodes -= parameter_nodes
+
+        # Use first file as reference to setup nodes to merge
+        with tables.open_file(self.input_files[0], "r") as h5file:
+            self.data_model_version = h5file.root._v_attrs[VERSION_KEY]
+
+            # Check if first file is simulation
+            if "/simulation" not in h5file.root:
+                self.log.info("Merging observed data")
+                self.usable_nodes -= simulation_nodes
+                self.is_simulation = False
+            else:
+                self.log.info("Merging simulated data")
+                self.is_simulation = True
+
+            # do not try to merge optional nodes not present in first file
+            for node in filter(lambda n: n not in h5file, optional_nodes):
+                self.log.info(f"First file does not contain {node}, ignoring")
+                self.usable_nodes.remove(node)
+
         # create output file with subarray from first file
         self.first_subarray = SubarrayDescription.from_hdf(self.input_files[0])
         if self.allowed_tels:
             self.first_subarray = self.first_subarray.select_subarray(
                 tel_ids=self.allowed_tels
             )
-            self.allowed_tel_names = {"tel_%03d" % i for i in self.allowed_tels}
+            self.allowed_tel_names = {
+                f"tel_{tel_id:03d}" for tel_id in self.allowed_tels
+            }
 
         self.first_subarray.to_hdf(self.output_path)
         self.output_file = tables.open_file(self.output_path, mode="a")
 
-        # setup required nodes
-        self.usable_nodes = all_nodes
-
-        if self.skip_simu_images is True:
-            self.usable_nodes = self.usable_nodes - simu_images
-
-        if self.skip_images is True:
-            self.usable_nodes = self.usable_nodes - image_nodes
-
-        if self.skip_parameters is True:
-            self.usable_nodes = self.usable_nodes - parameter_nodes
-
     def check_file_broken(self, file):
         # Check that the file is not broken or any node is missing
         file_path = file.root._v_file.filename
@@ -315,56 +341,62 @@ class MergeTool(Tool):
             )
 
             for node in service_nodes:
-                file.copy_node(node, newparent=target_group)
+                if node in file:
+                    file.copy_node(node, newparent=target_group)
 
-    def merge_tables(self, file):
-        # Loop over all nodes listed in usable_nodes. Appends table to output_file
-        # if it already exists, otherwise creates group and copies node.
-        for node in self.usable_nodes:
-            if node in service_nodes:
-                continue
+    def _merge_tel_group(self, file, input_node):
+        """Add a group that has one child table per telescope (type) to outputfile"""
+        if not isinstance(input_node, tables.Group):
+            raise TypeError(f"node must be a `tables.Group`, got {input_node}")
 
-            if node in file:
-                if node in nodes_with_tels:
-                    if node not in self.output_file:
-                        self._create_group(node)
+        node_path = input_node._v_pathname
 
-                    if self.allowed_tels:
-                        for tel_name in self.allowed_tel_names:
-                            if tel_name in file.root[node]:
-                                self._copy_or_append_tel_table(file, node, tel_name)
+        if input_node not in self.output_file:
+            self._create_group(node_path)
 
-                        continue
+        for tel_name, table in input_node._v_children.items():
+            if not self.allowed_tels or tel_name in self.allowed_tel_names:
+                self._merge_table(file, table)
 
-                    for tel in file.root[node]:
-                        self._copy_or_append_tel_table(file, node, tel.name)
+    def _merge_table(self, file, input_node):
+        if not isinstance(input_node, tables.Table):
+            raise TypeError(f"node must be a `tables.Table`, got {input_node}")
 
-                    continue
+        node_path = input_node._v_pathname
+        if node_path in self.output_file:
+            output_table = self.output_file.get_node(node_path)
+            output_table.append(input_node[:].astype(output_table.dtype))
+        else:
+            group_path, _ = os.path.split(node_path)
+            if group_path not in self.output_file:
+                self._create_group(group_path)
 
-                if node in self.output_file:
-                    data = file.root[node][:]
-                    output_node = self.output_file.get_node(node)
-                    output_node.append(data)
-                else:
-                    group_path, _ = os.path.split(node)
-                    if group_path not in self.output_file:
-                        self._create_group(group_path)
+            target_group = self.output_file.root[group_path]
+            file.copy_node(node_path, newparent=target_group)
 
-                    target_group = self.output_file.root[group_path]
-                    file.copy_node(node, newparent=target_group)
+    def merge_tables(self, file):
+        """Go over all mergeable nodes and append to outputfile"""
+        for node_path in self.usable_nodes:
 
-    def _copy_or_append_tel_table(self, file, node, tel_name):
-        tel_node_path = node + "/" + tel_name
-        if tel_node_path in self.output_file:
-            output_node = self.output_file.get_node(tel_node_path)
-            input_node = file.root[tel_node_path]
+            # skip service nodes that should only be included from the first file
+            if node_path in service_nodes:
+                continue
 
-            # cast needed for some image parameters that are sometimes
-            # float32 and sometimes float64
-            output_node.append(input_node[:].astype(output_node.dtype))
-        else:
-            target_group = self.output_file.root[node]
-            file.copy_node(tel_node_path, newparent=target_group)
+            if node_path in file:
+                node = file.root[node_path]
+
+                # nodes with child tables per telescope (type)
+                if node_path in nodes_with_tels:
+                    self._merge_tel_group(file, node)
+
+                # groups of tables (e.g. dl2)
+                elif isinstance(node, tables.Group):
+                    for table in node._v_children.values():
+                        self._merge_table(file, table)
+
+                # single table
+                else:
+                    self._merge_table(file, node)
 
     def _create_group(self, node):
         head, tail = os.path.split(node)
@@ -373,50 +405,33 @@ class MergeTool(Tool):
     def start(self):
         merged_files_counter = 0
 
-        for i, current_file in enumerate(
-            tqdm(
-                self.input_files,
-                desc="Merging",
-                unit="Files",
-                disable=not self.progress_bar,
-            )
+        for input_path in tqdm(
+            self.input_files,
+            desc="Merging",
+            unit="Files",
+            disable=not self.progress_bar,
         ):
-
-            if not HDF5EventSource.is_compatible(current_file):
-                self.log.critical(
-                    f"input file {current_file} is not a supported DL1 file"
-                )
+            if not HDF5EventSource.is_compatible(input_path):
+                self.log.critical(f"input file {input_path} is not a supported file")
                 if self.skip_broken_files:
                     continue
                 else:
                     sys.exit(1)
 
-            with tables.open_file(current_file, mode="r") as file:
-                if i == 0:
-                    self.data_model_version = file.root._v_attrs[VERSION_KEY]
-
-                    # Check if first file is simulation
-
-                    if "/simulation" not in file.root:
-                        self.usable_nodes = self.usable_nodes - simu_nodes
-                        self.log.info("Merging real data")
-                        self.is_simulation = False
-                    else:
-                        self.log.info("Merging simulation-files")
-                        self.is_simulation = True
+            with tables.open_file(input_path, mode="r") as h5file:
 
-                if self.check_file_broken(file) is True:
+                if self.check_file_broken(h5file) is True:
                     if self.skip_broken_files is True:
                         continue
                     else:
                         self.log.critical("Broken file detected.")
                         sys.exit(1)
 
-                self.merge_tables(file)
-                if IMAGE_STATISTICS_PATH in file:
-                    self.add_image_statistics(file)
+                self.merge_tables(h5file)
+                if IMAGE_STATISTICS_PATH in h5file:
+                    self.add_image_statistics(h5file)
 
-            PROV.add_input_file(str(current_file))
+            PROV.add_input_file(str(input_path))
             merged_files_counter += 1
 
         self.log.info(
diff --git a/ctapipe/tools/process.py b/ctapipe/tools/process.py
index 24846ac6..c19e16dd 100644
--- a/ctapipe/tools/process.py
+++ b/ctapipe/tools/process.py
@@ -10,7 +10,7 @@ from ..core import QualityQuery, Tool
 from ..core.traits import Bool, classes_with_traits, flag
 from ..image import ImageCleaner, ImageProcessor
 from ..image.extractor import ImageExtractor
-from ..io import DataLevel, DataWriter, EventSource, SimTelEventSource
+from ..io import DataLevel, DataWriter, EventSource, SimTelEventSource, write_table
 from ..io.datawriter import DATA_MODEL_VERSION
 from ..reco import ShowerProcessor
 from ..utils import EventTypeFilter
@@ -221,20 +221,20 @@ class ProcessorTool(Tool):
 
         if self.should_compute_dl1:
             image_stats = self.process_images.check_image.to_table(functions=True)
-            image_stats.write(
+            write_table(
+                image_stats,
                 self.write.output_path,
                 path="/dl1/service/image_statistics",
                 append=True,
-                serialize_meta=True,
             )
 
         if self.should_compute_dl2:
             shower_stats = self.process_shower.check_shower.to_table(functions=True)
-            shower_stats.write(
+            write_table(
+                shower_stats,
                 self.write.output_path,
-                path="/dl2/service/image_statistics",
+                "/dl2/service/image_statistics",
                 append=True,
-                serialize_meta=True,
             )
 
     def start(self):
@@ -275,6 +275,7 @@ class ProcessorTool(Tool):
         """
         self.write.write_simulation_histograms(self.event_source)
         self.write.finish()
+        self.event_source.close()
         self._write_processing_statistics()
 
 
diff --git a/ctapipe/tools/quickstart.py b/ctapipe/tools/quickstart.py
index 72190dd6..678c5978 100644
--- a/ctapipe/tools/quickstart.py
+++ b/ctapipe/tools/quickstart.py
@@ -132,7 +132,7 @@ class QuickStartTool(Tool):
     def start(self):
 
         for filename in CONFIGS_TO_WRITE:
-            config = files("ctapipe.tools.tests.resources").joinpath(filename)
+            config = files("ctapipe.tools.tests").joinpath("resources", filename)
             destination = self.workdir / filename
 
             if destination.exists():
diff --git a/setup.py b/setup.py
index 7bc179fe..298ed9d5 100755
--- a/setup.py
+++ b/setup.py
@@ -16,7 +16,7 @@ entry_points["console_scripts"] = [
     "ctapipe-reconstruct-muons = ctapipe.tools.muon_reconstruction:main",
     "ctapipe-display-dl1 = ctapipe.tools.display_dl1:main",
     "ctapipe-process = ctapipe.tools.process:main",
-    "ctapipe-merge = ctapipe.tools.dl1_merge:main",
+    "ctapipe-merge = ctapipe.tools.merge:main",
     "ctapipe-fileinfo = ctapipe.tools.fileinfo:main",
     "ctapipe-quickstart = ctapipe.tools.quickstart:main",
 ]


Test patch:
diff --git a/ctapipe/conftest.py b/ctapipe/conftest.py
index b3a7c0cb..cec3dcce 100644
--- a/ctapipe/conftest.py
+++ b/ctapipe/conftest.py
@@ -173,6 +173,32 @@ def dl2_shower_geometry_file(dl2_tmp_path, prod5_gamma_simtel_path):
         return output
 
 
+@pytest.fixture(scope="session")
+def dl2_proton_geometry_file(dl2_tmp_path, prod5_proton_simtel_path):
+    """
+    File containing both parameters and shower geometry from a gamma simulation set.
+    """
+    from ctapipe.core import run_tool
+    from ctapipe.tools.process import ProcessorTool
+
+    output = dl2_tmp_path / "proton.training.h5"
+
+    # prevent running process multiple times in case of parallel tests
+    with FileLock(output.with_suffix(output.suffix + ".lock")):
+        if output.is_file():
+            return output
+
+        argv = [
+            f"--input={prod5_proton_simtel_path}",
+            f"--output={output}",
+            "--write-images",
+            "--write-stereo-shower",
+            "--max-events=20",
+        ]
+        assert run_tool(ProcessorTool(), argv=argv, cwd=dl2_tmp_path) == 0
+        return output
+
+
 @pytest.fixture(scope="session")
 def dl2_shower_geometry_file_type(dl2_tmp_path, prod5_gamma_simtel_path):
     """
diff --git a/ctapipe/core/tests/test_traits.py b/ctapipe/core/tests/test_traits.py
index ab7353fd..f3dd7bbf 100644
--- a/ctapipe/core/tests/test_traits.py
+++ b/ctapipe/core/tests/test_traits.py
@@ -41,7 +41,7 @@ def mock_subarray():
 
 def test_path_allow_none_false():
     class C(Component):
-        path = Path(allow_none=False)
+        path = Path(default_value=None, allow_none=False)
 
     c = C()
 
diff --git a/ctapipe/instrument/camera/tests/test_image_conversion.py b/ctapipe/instrument/camera/tests/test_image_conversion.py
index ebfca03c..dfdb440a 100644
--- a/ctapipe/instrument/camera/tests/test_image_conversion.py
+++ b/ctapipe/instrument/camera/tests/test_image_conversion.py
@@ -2,6 +2,7 @@ import numpy as np
 from numpy.testing import assert_allclose
 import astropy.units as u
 from ctapipe.image.toymodel import Gaussian
+import pytest
 
 
 def create_mock_image(geom, psi=25 * u.deg):
@@ -50,3 +51,9 @@ def test_multiple_images(camera_geometry):
     # in general this introduces extra pixels in the 2d array, which are set to nan
     assert np.nansum(images) == np.nansum(images_2d)
     assert_allclose(images, images_1d)
+
+
+@pytest.mark.parametrize("pixel_id", [0, 1, 100])
+def test_pixel_coordinates_roundtrip(pixel_id, camera_geometry):
+    row, col = camera_geometry.image_index_to_cartesian_index(pixel_id)
+    assert camera_geometry.cartesian_index_to_image_index(row, col) == pixel_id
diff --git a/ctapipe/instrument/tests/test_subarray.py b/ctapipe/instrument/tests/test_subarray.py
index 62ca7afa..333957cf 100644
--- a/ctapipe/instrument/tests/test_subarray.py
+++ b/ctapipe/instrument/tests/test_subarray.py
@@ -140,6 +140,13 @@ def test_hdf(example_subarray, tmp_path):
     no_name = SubarrayDescription.from_hdf(path)
     assert no_name.name == "Unknown"
 
+    # Test we can also write and read to an already opened h5file
+    with tables.open_file(path, "w") as h5file:
+        example_subarray.to_hdf(h5file)
+
+    with tables.open_file(path, "r") as h5file:
+        assert SubarrayDescription.from_hdf(h5file) == example_subarray
+
 
 def test_hdf_same_camera(tmp_path):
     """Test writing / reading subarray to hdf5 with a subarray that has two
diff --git a/ctapipe/tools/tests/test_merge.py b/ctapipe/tools/tests/test_merge.py
index 249b3385..c1d24080 100644
--- a/ctapipe/tools/tests/test_merge.py
+++ b/ctapipe/tools/tests/test_merge.py
@@ -4,6 +4,10 @@ import shutil
 
 from ctapipe.core import run_tool
 from pathlib import Path
+from ctapipe.io.astropy_helpers import read_table
+from astropy.table import vstack
+from astropy.utils.diff import report_diff_values
+from io import StringIO
 
 from ctapipe.tools.process import ProcessorTool
 
@@ -14,7 +18,7 @@ except ImportError:
 
 
 def run_stage1(input_path, cwd, output_path=None):
-    config = files("ctapipe.tools.tests.resources").joinpath("stage1_config.json")
+    config = files("ctapipe.tools.tests").joinpath("resources", "stage1_config.json")
 
     if output_path is None:
         output_path = Path(
@@ -38,7 +42,7 @@ def run_stage1(input_path, cwd, output_path=None):
 
 
 def test_simple(tmp_path, dl1_file, dl1_proton_file):
-    from ctapipe.tools.dl1_merge import MergeTool
+    from ctapipe.tools.merge import MergeTool
 
     output = tmp_path / "merged_simple.dl1.h5"
     ret = run_tool(
@@ -51,7 +55,7 @@ def test_simple(tmp_path, dl1_file, dl1_proton_file):
 
 
 def test_pattern(tmp_path: Path, dl1_file, dl1_proton_file):
-    from ctapipe.tools.dl1_merge import MergeTool
+    from ctapipe.tools.merge import MergeTool
 
     # touch a random file to test that the pattern does not use it
     open(dl1_file.parent / "foo.h5", "w").close()
@@ -78,7 +82,7 @@ def test_pattern(tmp_path: Path, dl1_file, dl1_proton_file):
 
 
 def test_skip_images(tmp_path, dl1_file, dl1_proton_file):
-    from ctapipe.tools.dl1_merge import MergeTool
+    from ctapipe.tools.merge import MergeTool
 
     # create a second file so we can test the patterns
     output = tmp_path / "merged_no_images.dl1.h5"
@@ -102,24 +106,59 @@ def test_skip_images(tmp_path, dl1_file, dl1_proton_file):
 
 
 def test_allowed_tels(tmp_path, dl1_file, dl1_proton_file):
-    from ctapipe.tools.dl1_merge import MergeTool
+    from ctapipe.tools.merge import MergeTool
     from ctapipe.instrument import SubarrayDescription
 
     # create file to test 'allowed-tels' option
     output = tmp_path / "merged_allowed_tels.dl1.h5"
+
+    allowed_tels = {25, 125}
+
+    argv = [str(dl1_file), str(dl1_proton_file), f"--output={output}", "--overwrite"]
+    for tel_id in allowed_tels:
+        argv.append(f"--allowed-tels={tel_id}")
+
+    ret = run_tool(MergeTool(), argv=argv, cwd=tmp_path)
+    assert ret == 0
+
+    s = SubarrayDescription.from_hdf(output)
+    assert s.tel.keys() == allowed_tels
+
+    tel_keys = {f"tel_{tel_id:03d}" for tel_id in allowed_tels}
+    with tables.open_file(output) as f:
+        assert set(f.root.dl1.event.telescope.parameters._v_children).issubset(tel_keys)
+        assert set(f.root.dl1.event.telescope.images._v_children).issubset(tel_keys)
+        assert set(f.root.dl1.monitoring.telescope.pointing._v_children).issubset(
+            tel_keys
+        )
+
+
+def test_dl2(tmp_path, dl2_shower_geometry_file, dl2_proton_geometry_file):
+    from ctapipe.tools.merge import MergeTool
+
+    output = tmp_path / "merged.dl2.h5"
     ret = run_tool(
         MergeTool(),
         argv=[
-            str(dl1_file),
-            str(dl1_proton_file),
             f"--output={output}",
-            "--allowed-tels=1",
-            "--allowed-tels=2",
-            "--overwrite",
+            str(dl2_shower_geometry_file),
+            str(dl2_proton_geometry_file),
         ],
-        cwd=tmp_path,
     )
-    assert ret == 0
+    assert ret == 0, f"Running merge for dl2 files failed with exit code {ret}"
 
-    s = SubarrayDescription.from_hdf(output)
-    assert s.tel.keys() == {1, 2}
+    table1 = read_table(
+        dl2_shower_geometry_file, "/dl2/event/subarray/geometry/HillasReconstructor"
+    )
+    table2 = read_table(
+        dl2_proton_geometry_file, "/dl2/event/subarray/geometry/HillasReconstructor"
+    )
+    table_merged = read_table(
+        output, "/dl2/event/subarray/geometry/HillasReconstructor"
+    )
+
+    diff = StringIO()
+    identical = report_diff_values(vstack([table1, table2]), table_merged, fileobj=diff)
+    assert (
+        identical
+    ), f"Merged table not equal to individual tables. Diff:\n {diff.getvalue()}"
diff --git a/ctapipe/tools/tests/test_process.py b/ctapipe/tools/tests/test_process.py
index 38529f59..822f3720 100644
--- a/ctapipe/tools/tests/test_process.py
+++ b/ctapipe/tools/tests/test_process.py
@@ -21,10 +21,14 @@ except ImportError:
 GAMMA_TEST_LARGE = get_dataset_path("gamma_test_large.simtel.gz")
 
 
+def resource_file(filename):
+    return files("ctapipe.tools.tests").joinpath("resources", filename)
+
+
 def test_stage_1_dl1(tmp_path, dl1_image_file, dl1_parameters_file):
     """  check simtel to DL1 conversion """
 
-    config = files("ctapipe.tools.tests.resources").joinpath("stage1_config.json")
+    config = resource_file("stage1_config.json")
     # DL1A file as input
     dl1b_from_dl1a_file = tmp_path / "dl1b_fromdl1a.dl1.h5"
     assert (
@@ -131,7 +135,7 @@ def test_stage1_datalevels(tmp_path):
         infile.write(b"dummy")
         infile.flush()
 
-    config = files("ctapipe.tools.tests.resources").joinpath("stage1_config.json")
+    config = resource_file("stage1_config.json")
     tool = ProcessorTool()
 
     assert (
@@ -154,7 +158,7 @@ def test_stage1_datalevels(tmp_path):
 
 def test_stage_2_from_simtel(tmp_path):
     """ check we can go to DL2 geometry from simtel file """
-    config = files("ctapipe.tools.tests.resources").joinpath("stage2_config.json")
+    config = resource_file("stage2_config.json")
     output = tmp_path / "test_stage2_from_simtel.DL2.h5"
 
     assert (
@@ -179,7 +183,7 @@ def test_stage_2_from_simtel(tmp_path):
 
 def test_stage_2_from_dl1_images(tmp_path, dl1_image_file):
     """ check we can go to DL2 geometry from DL1 images """
-    config = files("ctapipe.tools.tests.resources").joinpath("stage2_config.json")
+    config = resource_file("stage2_config.json")
     output = tmp_path / "test_stage2_from_dl1image.DL2.h5"
 
     assert (
@@ -204,7 +208,7 @@ def test_stage_2_from_dl1_images(tmp_path, dl1_image_file):
 def test_stage_2_from_dl1_params(tmp_path, dl1_parameters_file):
     """ check we can go to DL2 geometry from DL1 parameters """
 
-    config = files("ctapipe.tools.tests.resources").joinpath("stage2_config.json")
+    config = resource_file("stage2_config.json")
     output = tmp_path / "test_stage2_from_dl1param.DL2.h5"
 
     assert (
@@ -229,7 +233,7 @@ def test_stage_2_from_dl1_params(tmp_path, dl1_parameters_file):
 def test_training_from_simtel(tmp_path):
     """ check we can write both dl1 and dl2 info (e.g. for training input) """
 
-    config = files("ctapipe.tools.tests.resources").joinpath("training_config.json")
+    config = resource_file("training_config.json")
     output = tmp_path / "test_training.DL1DL2.h5"
 
     assert (
@@ -256,7 +260,7 @@ def test_training_from_simtel(tmp_path):
 @pytest.mark.parametrize("filename", CONFIGS_TO_WRITE)
 def test_quickstart_templates(filename):
     """ ensure template configs have an appropriate placeholder for the contact info """
-    config = files("ctapipe.tools.tests.resources").joinpath(filename)
+    config = resource_file(filename)
     text = config.read_text()
 
     assert "YOUR-NAME-HERE" in text, "Missing expected name placeholder"