Task: Implement Output.cc for Carpet/CarpetIOHDF5 thorn

thorn_name: Carpet/CarpetIOHDF5
src_filename: Output.cc
interface: # Interface definition for thorn CarpetIOHDF5

IMPLEMENTS: IOHDF5

includes header: CarpetIOHDF5.hh in CarpetIOHDF5.hh

uses include header: Timer.hh

uses include header: carpet.hh

uses include header: defs.hh

uses include header: bbox.hh
uses include header: vect.hh

uses include header: data.hh
uses include header: gdata.hh

uses include header: ggf.hh
uses include header: gh.hh

uses include header: typecase.hh
uses include header: typeprops.hh



CCTK_INT next_output_iteration TYPE=scalar
CCTK_REAL next_output_time TYPE=scalar
CCTK_INT this_iteration TYPE=scalar

CCTK_INT last_output_iteration_slice[4] TYPE=scalar
CCTK_REAL last_output_time_slice[4] TYPE=scalar
CCTK_INT this_iteration_slice[4] TYPE=scalar


CCTK_INT FUNCTION Coord_GroupSystem    \
  (CCTK_POINTER_TO_CONST IN GH,        \
   CCTK_STRING IN groupname)
USES FUNCTION Coord_GroupSystem

# Specify whether a map uses a Cartesian or a curvilinear coordinate system
CCTK_INT FUNCTION             \
    MultiPatch_MapIsCartesian \
        (CCTK_INT IN map)
USES FUNCTION MultiPatch_MapIsCartesian


# Determine which boundaries are symmetry boundaries
CCTK_INT FUNCTION \
    SymmetryTableHandleForGrid (CCTK_POINTER_TO_CONST IN cctkGH)
REQUIRES FUNCTION SymmetryTableHandleForGrid

CCTK_INT FUNCTION \
    SymmetryTableHandleForGI (CCTK_POINTER_TO_CONST IN cctkGH, \
                              CCTK_INT IN group_index)
REQUIRES FUNCTION SymmetryTableHandleForGI


# Check whether existing output files should be truncated or not
CCTK_INT FUNCTION IO_TruncateOutputFiles \
  (CCTK_POINTER_TO_CONST IN GH)

REQUIRES FUNCTION IO_TruncateOutputFiles



# Checkpoint only a restricted set of variables
# (Set ngroups=-1 to checkpoint all variables.)
CCTK_INT \
FUNCTION IO_SetCheckpointGroups(CCTK_INT ARRAY IN groups, \
                                CCTK_INT       IN ngroups)
PROVIDES FUNCTION IO_SetCheckpointGroups \
    WITH CarpetIOHDF5_SetCheckpointGroups \
    LANGUAGE C



# Return a pointer to an unmodifiable C string
# which contains a unique ID for this configuration
CCTK_POINTER_TO_CONST \
FUNCTION UniqueConfigID (CCTK_POINTER_TO_CONST IN cctkGH)
USES FUNCTION UniqueConfigID

# Return a pointer to an unmodifiable C string
# which contains a unique ID for this build
CCTK_POINTER_TO_CONST \
FUNCTION UniqueBuildID (CCTK_POINTER_TO_CONST IN cctkGH)
USES FUNCTION UniqueBuildID

# Return a pointer to an unmodifiable C string
# which contains a unique ID for this simulation
CCTK_POINTER_TO_CONST \
FUNCTION UniqueSimulationID (CCTK_POINTER_TO_CONST IN cctkGH)
USES FUNCTION UniqueSimulationID

# Return a pointer to an unmodifiable C string
# which contains a unique ID for this run
CCTK_POINTER_TO_CONST \
FUNCTION UniqueRunID (CCTK_POINTER_TO_CONST IN cctkGH)
USES FUNCTION UniqueRunID

schedule: # Schedule definitions for thorn CarpetIOHDF5

storage: next_output_iteration next_output_time this_iteration
storage: last_output_iteration_slice last_output_time_slice this_iteration_slice

schedule CarpetIOHDF5_Startup at STARTUP after IOUtil_Startup
{
  LANG: C
} "Startup routine"

schedule CarpetIOHDF5_Init at INITIAL
{
  LANG: C
  OPTIONS: global
} "Initialisation routine"

schedule CarpetIOHDF5_InitCheckpointingIntervals at POST_RECOVER_VARIABLES
{
  LANG: C
  OPTIONS: global
} "Initialisation of checkpointing intervals after recovery"

schedule CarpetIOHDF5_InitialDataCheckpoint at CPINITIAL
{
  LANG: C
  OPTIONS: meta
} "Initial data checkpoint routine"

schedule CarpetIOHDF5_EvolutionCheckpoint at CHECKPOINT
{
  LANG: C
  OPTIONS: meta
} "Evolution checkpoint routine"

schedule CarpetIOHDF5_TerminationCheckpoint at TERMINATE
{
  LANG: C
  OPTIONS: meta
} "Termination checkpoint routine"

schedule CarpetIOHDF5_CloseFiles at POSTINITIAL
{
  LANG: C
  OPTIONS: global
} "Close all filereader input files"

if (! CCTK_Equals (recover, "no") && *recover_file)
{
  schedule CarpetIOHDF5_RecoverParameters at RECOVER_PARAMETERS
  {
    LANG:C
    OPTIONS: meta
  } "Parameter recovery routine"

  if (use_reflevels_from_checkpoint)
  {
    schedule CarpetIOHDF5_SetNumRefinementLevels at STARTUP before CarpetIOHDF5_Startup
    {
      LANG:C
      OPTIONS: meta
    } "Overwrite 'CarpetRegrid::refinement_levels' with the number of levels found in the checkpoint file"
  }

  schedule CarpetIOHDF5_CloseFiles at POST_RECOVER_VARIABLES
  {
    LANG: C
    OPTIONS: meta
  } "Close all initial data checkpoint files after recovery"
}

param: # Parameter definitions for thorn CarpetIOHDF5



shares: IO

USES STRING  out_dir       AS io_out_dir
USES KEYWORD out_criterion AS io_out_criterion
USES INT     out_every     AS io_out_every
USES REAL    out_dt        AS io_out_dt
USES BOOLEAN out_unchunked AS io_out_unchunked

USES KEYWORD verbose
USES BOOLEAN out_single_precision
USES KEYWORD out_mode 
USES INT     out_timesteps_per_file
USES KEYWORD out_save_parameters 
USES BOOLEAN abort_on_io_errors 
USES STRING  out_group_separator

USES INT     checkpoint_every
USES REAL    checkpoint_every_walltime_hours
USES INT     checkpoint_keep
USES BOOLEAN checkpoint_ID
USES BOOLEAN recover_and_remove
USES BOOLEAN checkpoint_on_terminate
USES BOOLEAN strict_io_parameter_check
USES KEYWORD recover
USES STRING  checkpoint_dir
USES STRING  checkpoint_ID_file
USES STRING  checkpoint_file
USES STRING  recover_dir
USES STRING  recover_file
USES STRING  filereader_ID_dir

# 1D slice output
USES CCTK_INT out_xline_yi
USES CCTK_INT out_xline_zi
USES CCTK_INT out_yline_xi
USES CCTK_INT out_yline_zi
USES CCTK_INT out_zline_xi
USES CCTK_INT out_zline_yi

USES CCTK_REAL out_xline_y
USES CCTK_REAL out_xline_z
USES CCTK_REAL out_yline_x
USES CCTK_REAL out_yline_z
USES CCTK_REAL out_zline_x
USES CCTK_REAL out_zline_y

# 2D slice output
USES CCTK_INT out_xyplane_zi
USES CCTK_INT out_xzplane_yi
USES CCTK_INT out_yzplane_xi

USES CCTK_REAL out_xyplane_z
USES CCTK_REAL out_xzplane_y
USES CCTK_REAL out_yzplane_x


shares: Cactus

USES REAL cctk_initial_time



private:

STRING out_dir "Name of CarpetIOHDF5 output directory, overrides 'IO::out_dir'" STEERABLE = ALWAYS
{
  "^$" :: "Empty: use IO::out_dir"
  ".+" :: "Not empty: directory name"
} ""

CCTK_STRING out0D_dir "Name of 0D HDF5 slice output directory, overrides IO::out_dir" STEERABLE = ALWAYS
{
  "^$" :: "Empty: use IO::out_dir"
  ".+" :: "Not empty: directory name"
} ""

CCTK_STRING out1D_dir "Name of 1D HDF5 slice output directory, overrides IO::out_dir" STEERABLE = ALWAYS
{
  "^$" :: "Empty: use IO::out_dir"
  ".+" :: "Not empty: directory name"
} ""

CCTK_STRING out2D_dir "Name of 2D HDF5 slice output directory, overrides IO::out_dir" STEERABLE = ALWAYS
{
  "^$" :: "Empty: use IO::out_dir"
  ".+" :: "Not empty: directory name"
} ""

CCTK_STRING out3D_dir "Name of 3D HDF5 slice output directory, overrides IO::out_dir" STEERABLE = ALWAYS
{
  "^$" :: "Empty: use IO::out_dir"
  ".+" :: "Not empty: directory name"
} ""

STRING out_vars "Variables to output in CarpetIOHDF5 file format" STEERABLE = ALWAYS
{
  "" :: "List of group or variable names"
} ""

CCTK_STRING out0D_vars "Variables to output in 0D HDF5 file format" STEERABLE = ALWAYS
{
  "" :: "List of group or variable names"
} ""

CCTK_STRING out1D_vars "Variables to output in 1D HDF5 file format" STEERABLE = ALWAYS
{
  "" :: "List of group or variable names"
} ""

CCTK_STRING out2D_vars "Variables to output in 2D HDF5 file format" STEERABLE = ALWAYS
{
  "" :: "List of group or variable names"
} ""

CCTK_STRING out3D_vars "Variables to output in 3D HDF5 file format" STEERABLE = ALWAYS
{
  "" :: "List of group or variable names"
} ""

STRING out_extension "File extension to use for CarpetIOHDF5 output" STEERABLE = ALWAYS
{
  "" :: "File extension (including a leading dot, if desired)"
} ".h5"

KEYWORD out_criterion "Criterion to select CarpetIOHDF5 output intervals, overrides out_every" STEERABLE = ALWAYS
{
  "default"   :: "Use 'IO::out_criterion'"
  "never"     :: "Never output"
  "iteration" :: "Output every so many iterations"
  "divisor"   :: "Output if (iteration % out_every) == 0."
  "time"      :: "Output every that much coordinate time"
} "default"

KEYWORD out0D_criterion "Criterion to select 0D HDF5 slice output intervals, overrides out_every" STEERABLE = ALWAYS
{
  "default"   :: "Use IO::out_criterion"
  "never"     :: "Never output"
  "iteration" :: "Output every so many iterations"
  "divisor"   :: "Output if iteration mod divisor == 0."
  "time"      :: "Output every that much coordinate time"
} "default"

KEYWORD out1D_criterion "Criterion to select 1D HDF5 slice output intervals, overrides out_every" STEERABLE = ALWAYS
{
  "default"   :: "Use IO::out_criterion"
  "never"     :: "Never output"
  "iteration" :: "Output every so many iterations"
  "divisor"   :: "Output if (iteration % out_every) == 0."
  "time"      :: "Output every that much coordinate time"
} "default"

KEYWORD out2D_criterion "Criterion to select 2D HDF5 slice output intervals, overrides out_every" STEERABLE = ALWAYS
{
  "default"   :: "Use IO::out_criterion"
  "never"     :: "Never output"
  "iteration" :: "Output every so many iterations"
  "divisor"   :: "Output if (iteration % out_every) == 0."
  "time"      :: "Output every that much coordinate time"
} "default"

KEYWORD out3D_criterion "Criterion to select 3D HDF5 slice output intervals, overrides out_every" STEERABLE = ALWAYS
{
  "default"   :: "Use IO::out_criterion"
  "never"     :: "Never output"
  "iteration" :: "Output every so many iterations"
  "divisor"   :: "Output if (iteration % out_every) == 0."
  "time"      :: "Output every that much coordinate time"
} "default"

INT out_every "How often to do CarpetIOHDF5 output, overrides out_every" STEERABLE = ALWAYS
{
  1:*  :: "Output every so many time steps"
  -1:0 :: "No output"
  -2   :: "Use 'IO::out_every'"
} -2

CCTK_INT out0D_every "How often to do 0D HDF5 slice output, overrides out_every" STEERABLE = ALWAYS
{
  1:*  :: "Output every so many time steps"
  -1:0 :: "No output"
  -2   :: "Use IO::out_every"
} -2

CCTK_INT out1D_every "How often to do 1D HDF5 slice output, overrides out_every" STEERABLE = ALWAYS
{
  1:*  :: "Output every so many time steps"
  -1:0 :: "No output"
  -2   :: "Use IO::out_every"
} -2

CCTK_INT out2D_every "How often to do 2D HDF5 slice output, overrides out_every" STEERABLE = ALWAYS
{
  1:*  :: "Output every so many time steps"
  -1:0 :: "No output"
  -2   :: "Use IO::out_every"
} -2

CCTK_INT out3D_every "How often to do 3D HDF5 slice output, overrides out_every" STEERABLE = ALWAYS
{
  1:*  :: "Output every so many time steps"
  -1:0 :: "No output"
  -2   :: "Use IO::out_every"
} -2

REAL out_dt "How often to do CarpetIOHDF5 output, overrides 'IO::out_dt'" STEERABLE = ALWAYS
{
  (0:* :: "In intervals of that much coordinate time"
   0   :: "As often as possible"
  -1   :: "Disable output"
  -2   :: "Default to 'IO::out_dt'"
} -2

REAL out0D_dt "How often to do 0D HDF5 slice output, overrides IO::out_dt" STEERABLE = ALWAYS
{
  (0:* :: "In intervals of that much coordinate time"
   0   :: "As often as possible"
  -1   :: "Disable output"
  -2   :: "Default to IO::out_dt"
} -2

REAL out1D_dt "How often to do 1D HDF5 slice output, overrides IO::out_dt" STEERABLE = ALWAYS
{
  (0:* :: "In intervals of that much coordinate time"
   0   :: "As often as possible"
  -1   :: "Disable output"
  -2   :: "Default to IO::out_dt"
} -2

REAL out2D_dt "How often to do 2D HDF5 slice output, overrides IO::out_dt" STEERABLE = ALWAYS
{
  (0:* :: "In intervals of that much coordinate time"
   0   :: "As often as possible"
  -1   :: "Disable output"
  -2   :: "Default to IO::out_dt"
} -2

REAL out3D_dt "How often to do 3D HDF5 slice output, overrides IO::out_dt" STEERABLE = ALWAYS
{
  (0:* :: "In intervals of that much coordinate time"
   0   :: "As often as possible"
  -1   :: "Disable output"
  -2   :: "Default to IO::out_dt"
} -2


############################################################################
# 0D slice output
############################################################################
CCTK_INT out0D_point_xi "x-index (counting from 0) for 0D points" STEERABLE = ALWAYS
{
  0:* :: ""
} 0
CCTK_INT out0D_point_yi "y-index (counting from 0) for 0D points" STEERABLE = ALWAYS
{
  0:* :: ""
} 0
CCTK_INT out0D_point_zi "z-index (counting from 0) for 0D points" STEERABLE = ALWAYS
{
  0:* :: ""
} 0

CCTK_REAL out0D_point_x "x coordinate for 0D points" STEERABLE = ALWAYS
{
  *:* :: ""
} 0
CCTK_REAL out0D_point_y "y coordinate for 0D points" STEERABLE = ALWAYS
{
  *:* :: ""
} 0
CCTK_REAL out0D_point_z "z coordinate for 0D points" STEERABLE = ALWAYS
{
  *:* :: ""
} 0



############################################################################
# 1D slice output
############################################################################
BOOLEAN out1D_x "Do 1D HDF5 slice output in the x-direction" STEERABLE = ALWAYS
{
} "yes"
BOOLEAN out1D_y "Do 1D HDF5 slice output in the y-direction" STEERABLE = ALWAYS
{
} "yes"
BOOLEAN out1D_z "Do 1D HDF5 slice output in the z-direction" STEERABLE = ALWAYS
{
} "yes"

CCTK_INT out1D_xline_yi "y-index (counting from 0) for 1D lines in x-direction" STEERABLE = ALWAYS
{
  0:* :: ""
} 0
CCTK_INT out1D_xline_zi "z-index (counting from 0) for 1D lines in x-direction" STEERABLE = ALWAYS
{
  0:* :: ""
} 0

CCTK_INT out1D_yline_xi "x-index (counting from 0) for 1D lines in y-direction" STEERABLE = ALWAYS
{
  0:* :: ""
} 0
CCTK_INT out1D_yline_zi "z-index (counting from 0) for 1D lines in y-direction" STEERABLE = ALWAYS
{
  0:* :: ""
} 0

CCTK_INT out1D_zline_xi "x-index (counting from 0) for 1D lines in z-direction" STEERABLE = ALWAYS
{
  0:* :: ""
} 0
CCTK_INT out1D_zline_yi "y-index (counting from 0) for 1D lines in z-direction" STEERABLE = ALWAYS
{
  0:* :: ""
} 0

CCTK_REAL out1D_xline_y "y coordinate for 1D lines in x-direction" STEERABLE = ALWAYS
{
  *:* :: ""
} 0
CCTK_REAL out1D_xline_z "z coordinate for 1D lines in x-direction" STEERABLE = ALWAYS
{
  *:* :: ""
} 0

CCTK_REAL out1D_yline_x "x coordinate for 1D lines in y-direction" STEERABLE = ALWAYS
{
  *:* :: ""
} 0
CCTK_REAL out1D_yline_z "z coordinate for 1D lines in y-direction" STEERABLE = ALWAYS
{
  *:* :: ""
} 0

CCTK_REAL out1D_zline_x "x coordinate for 1D lines in z-direction" STEERABLE = ALWAYS
{
  *:* :: ""
} 0
CCTK_REAL out1D_zline_y "y coordinate for 1D lines in z-direction" STEERABLE = ALWAYS
{
  *:* :: ""
} 0


############################################################################
# 2D slice output
############################################################################
BOOLEAN out2D_xy "Do 2D HDF5 slice output in the xy-direction" STEERABLE = ALWAYS
{
} "yes"
BOOLEAN out2D_xz "Do 2D HDF5 slice output in the xz-direction" STEERABLE = ALWAYS
{
} "yes"
BOOLEAN out2D_yz "Do 2D HDF5 slice output in the yz-direction" STEERABLE = ALWAYS
{
} "yes"

CCTK_INT out2D_xyplane_zi "z-index (counting from 0) for 2D planes in xy-direction" STEERABLE = ALWAYS
{
  0:* :: ""
} 0

CCTK_INT out2D_xzplane_yi "y-index (counting from 0) for 2D planes in xz-direction" STEERABLE = ALWAYS
{
  0:* :: ""
} 0

CCTK_INT out2D_yzplane_xi "x-index (counting from 0) for 2D planes in yz-direction" STEERABLE = ALWAYS
{
  0:* :: ""
} 0

CCTK_REAL out2D_xyplane_z "z coordinate for 2D planes in xy-direction" STEERABLE = ALWAYS
{
  *:* :: ""
} 0

CCTK_REAL out2D_xzplane_y "y coordinate for 2D planes in xz-direction" STEERABLE = ALWAYS
{
  *:* :: ""
} 0

CCTK_REAL out2D_yzplane_x "x coordinate for 2D planes in yz-direction" STEERABLE = ALWAYS
{
  *:* :: ""
} 0

#######################################################################
# general slice output parameters
#######################################################################
BOOLEAN output_all_timelevels "Output all timelevels instead of only the current" STEERABLE = ALWAYS
{
} "no"

BOOLEAN output_symmetry_points "Output symmetry points (assuming that there are nghostzones symmetry points)" STEERABLE = ALWAYS
{
} "yes"

BOOLEAN output_ghost_points "Output ghost points" STEERABLE = ALWAYS
{
} "yes"

BOOLEAN output_boundary_points "Output outer boundary points (assuming that there are nghostzones boundary points)" STEERABLE = ALWAYS
{
} "yes"

BOOLEAN output_buffer_points "Output refinement buffer points" STEERABLE = ALWAYS
{
} "yes"

BOOLEAN out3D_ghosts "Output ghost zones (DEPRECATED)" STEERABLE = ALWAYS
{
} "yes"

BOOLEAN out3D_outer_ghosts "Output outer boundary zones (assuming that there are nghostzones boundary points) (DEPRECATED)" STEERABLE = ALWAYS
{
} "yes"

# These parameters are here for historic reasons only.
# They might go away in the future.  Do not use them.

BOOLEAN out1D_d "Do output along the diagonal" STEERABLE = ALWAYS
{
} "yes"

#######################################################################
# checkpoint/recovery
#######################################################################
BOOLEAN checkpoint "Do checkpointing with CarpetIOHDF5 ?" STEERABLE = ALWAYS
{
} "no"

BOOLEAN checkpoint_next "Checkpoint at next iteration ?" STEERABLE = ALWAYS
{
} "no"

INT checkpoint_every_divisor "Checkpoint if (iteration % out_every) == 0" STEERABLE = ALWAYS
{
   1:* :: "Every so many iterations"
  -1:0 :: "Disable periodic checkpointing"
} -1

BOOLEAN use_reflevels_from_checkpoint "Use 'CarpetRegrid::refinement_levels' from the checkpoint file rather than from the parameter file ?" STEERABLE = RECOVER
{
} "no"

BOOLEAN use_grid_structure_from_checkpoint "Use the grid structure stored in the checkpoint file" STEERABLE = RECOVER
{
} "yes"

BOOLEAN one_file_per_group "Write one file per group instead of per variable" STEERABLE = ALWAYS
{
} "no"

BOOLEAN open_one_input_file_at_a_time "Open only one HDF5 file at a time when reading data from multiple chunked checkpoint/data files" STEERABLE = ALWAYS
{
  "no"  :: "Open all input files first, then import data (most efficient)"
  "yes" :: "Process input files one after another (reduces memory requirements)"
} "no"

STRING skip_recover_variables "Skip these variables while recovering" STEERABLE = RECOVER
{
  "" :: ""
} ""

INT compression_level "Compression level to use for writing HDF5 data" STEERABLE = ALWAYS
{
  0:9 :: "Higher numbers compress better, a value of zero disables compression"
} 0

INT minimum_size_for_compression "Only compress datasets larger than this many bytes" STEERABLE = ALWAYS
{
  0:* :: "This should to be large enough so that compression gains outweigh the overhead"
} 32768

BOOLEAN use_checksums "Use checksums for the HDF5 data" STEERABLE = ALWAYS
{
} "no"

BOOLEAN output_index "Output an index file for each output file" STEERABLE = ALWAYS
{
} "no"

configuration: # Configuration definitions for thorn CarpetIOHDF5

REQUIRES Carpet CarpetLib IOUtil HDF5

OPTIONAL MPI
{
}

PROVIDES CarpetIOHDF5
{
}

context: make.configuration.defn:
```
# add the Carpet HDF5-to-ASCII converter/slicer/recombiner
ALL_UTILS += hdf5toascii_slicer hdf5tobinary_slicer hdf5_slicer hdf5_recombiner

```

CarpetIOHDF5.hh:
```
#ifndef CARPETIOHDF5_HH
#define CARPETIOHDF5_HH

#define H5_USE_16_API 1
#include <hdf5.h>

#include <vector>

#include "cctk_Arguments.h"
#include "CactusBase/IOUtil/src/ioutil_Utils.h"
#include "carpet.hh"

// some macros for HDF5 group names
#define METADATA_GROUP "Parameters and Global Attributes"
#define ALL_PARAMETERS "All Parameters"
#define GRID_STRUCTURE "Grid Structure v5"

// atomic HDF5 datatypes for the generic CCTK datatypes
// (the one for CCTK_COMPLEX is created at startup as a compound HDF5 datatype)
#define HDF5_CHAR H5T_NATIVE_CHAR

#ifdef CCTK_REAL_PRECISION_16
#define HDF5_REAL H5T_NATIVE_LDOUBLE
#elif CCTK_REAL_PRECISION_8
#define HDF5_REAL H5T_NATIVE_DOUBLE
#elif CCTK_REAL_PRECISION_4
#define HDF5_REAL H5T_NATIVE_FLOAT
#endif

#ifdef CCTK_INTEGER_PRECISION_8
#define HDF5_INT H5T_NATIVE_LLONG
#elif CCTK_INTEGER_PRECISION_4
#define HDF5_INT H5T_NATIVE_INT
#elif CCTK_INTEGER_PRECISION_2
#define HDF5_INT H5T_NATIVE_SHORT
#elif CCTK_INTEGER_PRECISION_1
#define HDF5_INT H5T_NATIVE_CHAR
#endif

// check return code of HDF5 call and print a warning in case of an error
#define HDF5_ERROR(fn_call)                                                    \
  do {                                                                         \
    int _error_code = fn_call;                                                 \
                                                                               \
    if (_error_code < 0) {                                                     \
      CCTK_VWarn(1, __LINE__, __FILE__, CCTK_THORNSTRING,                      \
                 "HDF5 call '%s' returned error code %d", #fn_call,            \
                 _error_code);                                                 \
      error_count++;                                                           \
    }                                                                          \
  } while (0)

// datatype of the start[] parameter in a call to H5Sselect_hyperslab()
// (the HDF5 API has changed in this respect after version 1.6.3)
#if (H5_VERS_MAJOR == 1 &&                                                     \
     (H5_VERS_MINOR < 6 || (H5_VERS_MINOR == 6 && H5_VERS_RELEASE < 4)))
#define slice_start_size_t hssize_t
#else
#define slice_start_size_t hsize_t
#endif

// CarpetIOHDF5 GH extension structure
typedef struct {
  // default number of times to output
  int out_every_default;

  // list of variables to output
  char *out_vars;

  // stop on I/O parameter parsing errors ?
  int stop_on_parse_errors;

  // I/O request description list (for all variables)
  vector<ioRequest *> requests;

  // directory in which to output
  char *out_dir;

  // ring buffer for list of successfully created cp files
  int checkpoint_keep;
  int cp_filename_index;
  char **cp_filename_list;

  // list of recovery files to remove
  int recovery_num_filenames;
  char **recovery_filename_list;

  // iteration number of the last checkpoint
  int last_checkpoint_iteration;

  // hdf5 datatype for complex variables; to be set at run time
  hid_t HDF5_COMPLEX, HDF5_COMPLEX8, HDF5_COMPLEX16, HDF5_COMPLEX32;

} CarpetIOHDF5GH;

namespace CarpetIOHDF5 {
// callback routine registered for recovery/filereader
int Recover(cGH *cctkGH, const char *basefilename, int called_from);

// worker routines to write a single variable
int WriteVarUnchunked(const cGH *const cctkGH, hid_t file, CCTK_REAL &io_bytes,
                      const ioRequest *const request,
                      bool called_from_checkpoint);
int WriteVarChunkedSequential(const cGH *const cctkGH, hid_t file,
                              CCTK_REAL &io_bytes,
                              const ioRequest *const request,
                              bool called_from_checkpoint, hid_t index = -1);
int WriteVarChunkedParallel(const cGH *const cctkGH, hid_t file,
                            CCTK_REAL &io_bytes, const ioRequest *const request,
                            bool called_from_checkpoint, hid_t index = -1);

int WriteMetadata(const cGH *const cctkGH, int const nioprocs,
                  int const firstvar, int const numvars,
                  bool const called_from_checkpoint, hid_t const file);

int AddSliceAttributes(const cGH *const cctkGH, const char *const fullname,
                       const int refinementlevel, const int multigridlevel,
                       const int map, const int timelevel,
                       const vector<double> &origin,
                       const vector<double> &delta, const vector<int> &iorigin,
                       const vector<int> &ioffset,
                       const vector<int> &ioffsetdenom, const vector<int> &bbox,
                       const vector<int> &nghostzones, const string &active,
                       hid_t &dataset, const vector<hsize_t> &shape,
                       const bool is_index);

int WriteAttribute(hid_t const group, char const *const name, int const ivalue);
int WriteAttribute(hid_t const group, char const *const name,
                   double const dvalue);
int WriteAttribute(hid_t const group, char const *const name,
                   char const *const svalue);
int WriteAttribute(hid_t const group, char const *const name,
                   int const *const ivalues, int const nvalues);
int WriteAttribute(hid_t const group, char const *const name,
                   double const *const dvalues, int const nvalues);
int WriteAttribute(hid_t const group, char const *const name,
                   char const *const *const svalues, int const nvalues);
int WriteAttribute(hid_t const group, char const *const name,
                   hsize_t const *const svalues, int const nvalues);
int WriteLargeAttribute(hid_t const group, char const *const name,
                        char const *const svalue);

// returns an HDF5 datatype corresponding to the given CCTK datatype
hid_t CCTKtoHDF5_Datatype(const cGH *const cctkGH, int cctk_type,
                          bool single_precision);

// Everything is a class template, so that it can easily be
// instantiated for all output dimensions

// computes the active region the way dh::regrid does
void GetAllActive(const dh *dd, const gh *hh, int ml, int rl, ibset &allactive);

template <int outdim> struct IOHDF5 {

  // name of the output directory
  static char *my_out_slice_dir;

  // list of variables to output
  static char *my_out_slice_vars;

  // I/O request description list (for all variables)
  static vector<ioRequest *> slice_requests;

  // number of I/O processors to use, which processor does I/O for me
  static int nioprocs;
  static int ioproc;
  static int ioproc_every;

  // Scheduled functions
  static int Startup();

  // Registered functions
  static void *SetupGH(tFleshConfig *fc, int convLevel, cGH *cctkGH);

  static int OutputGH(const cGH *cctkGH);
  static int OutputVarAs(const cGH *cctkGH, const char *varname,
                         const char *alias);
  static int TimeToOutput(const cGH *cctkGH, int vindex);
  static int TriggerOutput(const cGH *cctkGH, int vindex);

  // Other functions
  static void CheckSteerableParameters(const cGH *cctkGH);

  static bool DidOutput(const cGH *cctkGH, int vindex, string basefilename,
                        bool &is_new_file, bool &truncate_file);

  static bool DirectionIsRequested(const vect<int, outdim> &dirs);

  static void OutputDirection(const cGH *cctkGH, int vindex, string alias,
                              string basefilename,
                              const vect<int, outdim> &dirs, bool is_new_file,
                              bool truncate_file);

  static int OpenFile(const cGH *cctkGH, int m, int vindex, int numvars,
                      string alias, string basefilename,
                      const vect<int, outdim> &dirs, bool is_new_file,
                      bool truncate_file, hid_t &file, hid_t &index_file);

  static int WriteHDF5(const cGH *cctkGH, hid_t &file, hid_t &index_file,
                       vector<gdata *> const gfdatas,
                       const bbox<int, dim> &gfext, const int vi,
                       const vect<int, dim> &org, const vect<int, outdim> &dirs,
                       const int rl, const int ml, const int m, const int c,
                       const int output_component, const int tl,
                       const CCTK_REAL coord_time,
                       const vect<CCTK_REAL, dim> &coord_lower,
                       const vect<CCTK_REAL, dim> &coord_upper);

  static int CloseFile(const cGH *cctkGH, hid_t &file, hid_t &index_file);

  static ivect GetOutputOffset(const cGH *cctkGH, int m,
                               const vect<int, outdim> &dirs);
  static int IOProcForProc(int proc);

}; // struct IOHDF5

// scheduled and aliased routines (must be declared as C according
// to schedule.ccl)
extern "C" {

int CarpetIOHDF5_RecoverParameters(void);
int CarpetIOHDF5_SetNumRefinementLevels(void);
int CarpetIOHDF5_Startup(void);
void CarpetIOHDF5_Init(CCTK_ARGUMENTS);
void CarpetIOHDF5_InitCheckpointingIntervals(CCTK_ARGUMENTS);
void CarpetIOHDF5_RecoverGridStructure(CCTK_ARGUMENTS);
void CarpetIOHDF5_CloseFiles(CCTK_ARGUMENTS);
void CarpetIOHDF5_InitialDataCheckpoint(CCTK_ARGUMENTS);
void CarpetIOHDF5_EvolutionCheckpoint(CCTK_ARGUMENTS);
void CarpetIOHDF5_TerminationCheckpoint(CCTK_ARGUMENTS);

CCTK_INT CarpetIOHDF5_SetCheckpointGroups(CCTK_INT const *groups,
                                          CCTK_INT ngroups);

} // extern "C"

// Which groups should be checkpointed. If empty, all variables
// should be checkpointed (default).
extern vector<bool> groups_to_checkpoint;

} // namespace CarpetIOHDF5

#endif // !defined(CARPETIOHDF5_HH)

```

make.configuration.deps:
```
CARPETIOHDF5_BUILD_DIR = $(BUILD_DIR)/CarpetIOHDF5
CARPETIOHDF5_SRC_DIR   = $(PACKAGE_DIR)/Carpet/CarpetIOHDF5/src/util

CARPETIOHDF5_CXXFLAGS  = -DCCODE $(CXXFLAGS)

# Extend CXXFLAGS if HDF5 library was built with LFS support
ifneq ($(strip $(HDF5_LFS_FLAGS)),)
  CARPETIOHDF5_CXXFLAGS += $(HDF5_LFS_FLAGS)
endif

# Add includes
CARPETIOHDF5_CXXFLAGS += -I$(CONFIG) -I$(BINDINGS_DIR)/include -I$(FLESH_DIR)/include -I$(PACKAGE_DIR) $(HDF5_INC_DIRS:%=-I%) $(MPI_INC_DIRS:%=-I%)

CARPETIOHDF5_LDFLAGS = $(CARPETIOHDF5_CXXFLAGS) $(DEBUG_LD) $(LDFLAGS) $(EXTRAFLAGS) $(GENERAL_LIBRARIES)



# general linking rule
$(UTIL_DIR)/%: $(CARPETIOHDF5_BUILD_DIR)/%.o
	@echo Creating $* in $(UTIL_DIR) from $<
	-$(MKDIR) $(MKDIRFLAGS) $(UTIL_DIR)
	$(LD) $< $(CARPETIOHDF5_LDFLAGS) -o $@

# general compile rule
$(CARPETIOHDF5_BUILD_DIR)/%.o: $(CARPETIOHDF5_SRC_DIR)/%.cc
	@echo Compiling $<
	-$(MKDIR) $(MKDIRFLAGS) $(CARPETIOHDF5_BUILD_DIR)
	$(CXX) $< $(CARPETIOHDF5_CXXFLAGS) -DTHORN_IS_CarpetIOHDF5 -c -o $@

```

CarpetIOHDF5.cc:
```
#include <algorithm>
#include <cassert>
#include <cstddef>
#include <cstdlib>
#include <cstring>
#include <iomanip>
#include <map>
#include <sstream>
#include <string>
#include <vector>

#include <cctk.h>
#include <cctk_Arguments.h>
#include <cctk_Parameters.h>
#include <cctk_Version.h>
#include <util_Table.h>

#include "CactusBase/IOUtil/src/ioGH.h"
#include "CactusBase/IOUtil/src/ioutil_CheckpointRecovery.h"

#include <Timer.hh>

#include "CarpetIOHDF5.hh"

#include "defs.hh"
#include "gh.hh"

namespace CarpetIOHDF5 {

using namespace std;
using namespace Carpet;

// Variable definitions

// when was the last checkpoint written ?
static int last_checkpoint_iteration = -1;
static CCTK_REAL last_checkpoint_walltime;

// registered GH extension setup routine
static void *SetupGH(tFleshConfig *const fleshconfig, const int convLevel,
                     cGH *const cctkGH);

// callbacks for CarpetIOHDF5's I/O method
static int OutputGH(const cGH *const cctkGH);
static int OutputVarAs(const cGH *const cctkGH, const char *const varname,
                       const char *const alias);
static int TimeToOutput(const cGH *const cctkGH, const int vindex);
static int TriggerOutput(const cGH *const cctkGH, const int vindex);

// general checkpoint routine
static void Checkpoint(const cGH *const cctkGH, int called_from);

// callback for I/O parameter parsing routine
static void GetVarIndex(int vindex, const char *optstring, void *arg);

static void CheckSteerableParameters(const cGH *const cctkGH,
                                     CarpetIOHDF5GH *myGH);

//////////////////////////////////////////////////////////////////////////////
// public routines
//////////////////////////////////////////////////////////////////////////////

int CarpetIOHDF5_Startup(void) {
  CCTK_RegisterBanner("AMR HDF5 I/O provided by CarpetIOHDF5");

  const int GHExtension = CCTK_RegisterGHExtension(CCTK_THORNSTRING);
  CCTK_RegisterGHExtensionSetupGH(GHExtension, SetupGH);

  IOHDF5<0>::Startup();
  IOHDF5<1>::Startup();
  IOHDF5<2>::Startup();
  IOHDF5<3>::Startup();

  return (0);
}

// Called at basegrid during regular startup
void CarpetIOHDF5_Init(CCTK_ARGUMENTS) {
  DECLARE_CCTK_ARGUMENTS;

  *this_iteration = -1;
  *next_output_iteration = 0;
  *next_output_time = cctk_time;

  for (int d = 0; d < 4; ++d) {
    this_iteration_slice[d] = 0;
    last_output_iteration_slice[d] = 0;
    last_output_time_slice[d] = cctk_time;
  }

  last_checkpoint_iteration = cctk_iteration;
  last_checkpoint_walltime = CCTK_RunTime() / 3600.0;
}

// Called after recovering
void CarpetIOHDF5_InitCheckpointingIntervals(CCTK_ARGUMENTS) {
  DECLARE_CCTK_ARGUMENTS;

  last_checkpoint_iteration = cctk_iteration;
  last_checkpoint_walltime = CCTK_RunTime() / 3600.0;
}

void CarpetIOHDF5_InitialDataCheckpoint(CCTK_ARGUMENTS) {
  DECLARE_CCTK_ARGUMENTS;
  DECLARE_CCTK_PARAMETERS;

  if (checkpoint and checkpoint_ID) {
    if (not CCTK_Equals(verbose, "none")) {
      CCTK_INFO("---------------------------------------------------------");
      CCTK_VInfo(CCTK_THORNSTRING, "Dumping initial checkpoint at "
                                   "iteration %d, simulation time %g",
                 cctk_iteration, double(cctk_time));
      CCTK_INFO("---------------------------------------------------------");
    }
    Checkpoint(cctkGH, CP_INITIAL_DATA);
  }
}

void CarpetIOHDF5_EvolutionCheckpoint(CCTK_ARGUMENTS) {
  DECLARE_CCTK_ARGUMENTS;
  DECLARE_CCTK_PARAMETERS;

  CCTK_INT const iteration = cctk_iteration;
  CCTK_REAL const walltime = CCTK_RunTime() / 3600.0;

  bool const checkpoint_by_iteration =
      checkpoint_every > 0 and
      iteration >= last_checkpoint_iteration + checkpoint_every;
  bool const checkpoint_by_iteration_divisor =
      checkpoint_every_divisor > 0 and
      iteration % checkpoint_every_divisor == 0;
  bool const checkpoint_by_walltime =
      checkpoint_every_walltime_hours > 0 and
      walltime >= last_checkpoint_walltime + checkpoint_every_walltime_hours;

  int do_checkpoint =
      checkpoint and
      (checkpoint_by_iteration or checkpoint_by_iteration_divisor or
       checkpoint_by_walltime or checkpoint_next);
  if (checkpoint_every_walltime_hours > 0) {
    // broadcast the decision since comparing wall times may differ on
    // different processors
    MPI_Bcast(&do_checkpoint, 1, MPI_INT, 0, dist::comm());
  }

  if (do_checkpoint) {
    if (not CCTK_Equals(verbose, "none")) {
      CCTK_INFO("---------------------------------------------------------");
      CCTK_VInfo(CCTK_THORNSTRING, "Dumping periodic checkpoint at "
                                   "iteration %d, simulation time %g",
                 cctk_iteration, double(cctk_time));
      CCTK_INFO("---------------------------------------------------------");
    }

    Checkpoint(cctkGH, CP_EVOLUTION_DATA);
  }
}

void CarpetIOHDF5_TerminationCheckpoint(CCTK_ARGUMENTS) {
  DECLARE_CCTK_ARGUMENTS;
  DECLARE_CCTK_PARAMETERS;

  if (checkpoint and checkpoint_on_terminate) {

    if (last_checkpoint_iteration < cctk_iteration or
        (cctk_iteration == 0 and not checkpoint_ID)) {
      if (not CCTK_Equals(verbose, "none")) {
        CCTK_INFO("---------------------------------------------------------");
        CCTK_VInfo(CCTK_THORNSTRING, "Dumping termination checkpoint at "
                                     "iteration %d, simulation time %g",
                   cctk_iteration, double(cctk_time));
        CCTK_INFO("---------------------------------------------------------");
      }
      Checkpoint(cctkGH, CP_EVOLUTION_DATA);

    } else {
      if (not CCTK_Equals(verbose, "none")) {
        CCTK_INFO("---------------------------------------------------------");
        CCTK_VInfo(CCTK_THORNSTRING,
                   "Termination checkpoint already dumped "
                   "as last evolution checkpoint at iteration %d",
                   last_checkpoint_iteration);
        CCTK_INFO("---------------------------------------------------------");
      }
    }
  }
}

CCTK_INT CarpetIOHDF5_SetCheckpointGroups(CCTK_INT const *const groups,
                                          CCTK_INT const ngroups) {
  if (ngroups == -1) {
    // Checkpoint all groups
    groups_to_checkpoint.clear();
  } else {
    assert(ngroups >= 0);
    groups_to_checkpoint.resize(CCTK_NumGroups());
    for (int n = 0; n < CCTK_NumGroups(); ++n) {
      groups_to_checkpoint.at(n) = false;
    }
    for (int n = 0; n < ngroups; ++n) {
      groups_to_checkpoint.at(groups[n]) = true;
    }
  }
  return 0;
}

vector<bool> groups_to_checkpoint;

hid_t CCTKtoHDF5_Datatype(const cGH *const cctkGH, int cctk_type,
                          bool single_precision) {
  hid_t retval;

  const CarpetIOHDF5GH *myGH =
      (CarpetIOHDF5GH *)CCTK_GHExtension(cctkGH, CCTK_THORNSTRING);

  if (single_precision) {
    switch (cctk_type) {
    case CCTK_VARIABLE_REAL:
      cctk_type = CCTK_VARIABLE_REAL4;
      break;
    case CCTK_VARIABLE_COMPLEX:
      cctk_type = CCTK_VARIABLE_COMPLEX8;
      break;
    default:
      break; // do nothing
    }
  }

  // Normalize the type
  switch (cctk_type) {
  case CCTK_VARIABLE_INT:
#ifdef CCTK_INTEGER_PRECISION_1
    cctk_type = CCTK_VARIABLE_INT1;
#endif
#ifdef CCTK_INTEGER_PRECISION_2
    cctk_type = CCTK_VARIABLE_INT2;
#endif
#ifdef CCTK_INTEGER_PRECISION_4
    cctk_type = CCTK_VARIABLE_INT4;
#endif
#ifdef CCTK_INTEGER_PRECISION_8
    cctk_type = CCTK_VARIABLE_INT8;
#endif
#ifdef CCTK_INTEGER_PRECISION_16
    cctk_type = CCTK_VARIABLE_INT16;
#endif
    break;
  case CCTK_VARIABLE_REAL:
#ifdef CCTK_REAL_PRECISION_4
    cctk_type = CCTK_VARIABLE_REAL4;
#endif
#ifdef CCTK_REAL_PRECISION_8
    cctk_type = CCTK_VARIABLE_REAL8;
#endif
#ifdef CCTK_REAL_PRECISION_16
    cctk_type = CCTK_VARIABLE_REAL16;
#endif
    break;
  case CCTK_VARIABLE_COMPLEX:
#ifdef CCTK_REAL_PRECISION_4
    cctk_type = CCTK_VARIABLE_COMPLEX8;
#endif
#ifdef CCTK_REAL_PRECISION_8
    cctk_type = CCTK_VARIABLE_COMPLEX16;
#endif
#ifdef CCTK_REAL_PRECISION_16
    cctk_type = CCTK_VARIABLE_COMPLEX32;
#endif
    break;
  default:
    // do nothing
    break;
  }

  switch (cctk_type) {

  case CCTK_VARIABLE_CHAR:
    retval = HDF5_CHAR;
    break;

  case CCTK_VARIABLE_INT1:
    retval = H5T_NATIVE_INT8;
    break;
  case CCTK_VARIABLE_INT2:
    retval = H5T_NATIVE_INT16;
    break;
  case CCTK_VARIABLE_INT4:
    retval = H5T_NATIVE_INT32;
    break;
  case CCTK_VARIABLE_INT8:
    retval = H5T_NATIVE_INT64;
    break;
  // case CCTK_VARIABLE_INT16:     retval = H5T_NATIVE_INT128; break;

  case CCTK_VARIABLE_REAL4:
    retval = H5T_NATIVE_FLOAT;
    break;
  case CCTK_VARIABLE_REAL8:
    retval = H5T_NATIVE_DOUBLE;
    break;
  case CCTK_VARIABLE_REAL16:
    retval = H5T_NATIVE_LDOUBLE;
    break;

  case CCTK_VARIABLE_COMPLEX8:
    retval = myGH->HDF5_COMPLEX8;
    break;
  case CCTK_VARIABLE_COMPLEX16:
    retval = myGH->HDF5_COMPLEX16;
    break;
  case CCTK_VARIABLE_COMPLEX32:
    retval = myGH->HDF5_COMPLEX32;
    break;

  default:
    CCTK_VWarn(0, __LINE__, __FILE__, CCTK_THORNSTRING,
               "Unsupported CCTK variable datatype %d", cctk_type);
    retval = -1;
  }

  return (retval);
}

// add attributes to an HDF5 slice dataset
int AddSliceAttributes(const cGH *const cctkGH, const char *const fullname,
                       const int refinementlevel, const int multigridlevel,
                       const int map, const int timelevel,
                       const vector<double> &origin,
                       const vector<double> &delta, const vector<int> &iorigin,
                       const vector<int> &ioffset,
                       const vector<int> &ioffsetdenom, const vector<int> &bbox,
                       const vector<int> &nghostzones, const string &active,
                       hid_t &dataset, const vector<hsize_t> &shape,
                       const bool is_index) {
  int error_count = 0;

  error_count += WriteAttribute(dataset, "time", cctkGH->cctk_time);
  error_count += WriteAttribute(dataset, "timestep", cctkGH->cctk_iteration);
  error_count += WriteAttribute(dataset, "name", fullname);
  error_count += WriteAttribute(dataset, "level", refinementlevel);
  error_count += WriteAttribute(dataset, "carpet_mglevel", multigridlevel);
  error_count += WriteAttribute(dataset, "group_timelevel", timelevel);
  error_count += WriteAttribute(dataset, "origin", &origin[0], origin.size());
  error_count += WriteAttribute(dataset, "delta", &delta[0], delta.size());
  error_count +=
      WriteAttribute(dataset, "iorigin", &iorigin[0], iorigin.size());
  error_count +=
      WriteAttribute(dataset, "ioffset", &ioffset[0], ioffset.size());
  error_count += WriteAttribute(dataset, "ioffsetdenom", &ioffsetdenom[0],
                                ioffsetdenom.size());
  // TODO: Add "active" only for grid functions since it's trivial otherwise
  error_count += WriteAttribute(dataset, "active", active.c_str());
  // bbox and nghostzones are only used for grid functions and grid arrays
  if (bbox.size() > 0) {
    error_count += WriteAttribute(dataset, "cctk_bbox", &bbox[0], bbox.size());
  }
  if (nghostzones.size() > 0) {
    error_count += WriteAttribute(dataset, "cctk_nghostzones", &nghostzones[0],
                                  nghostzones.size());
  }
  // Specify whether the coordinate system is Cartesian or not
  if (CCTK_IsFunctionAliased("MultiPatch_MapIsCartesian")) {
    int const map_is_cartesian = MultiPatch_MapIsCartesian(map);
    error_count += WriteAttribute(dataset, "MapIsCartesian", map_is_cartesian);
  }
  if (is_index) {
    error_count += WriteAttribute(dataset, "h5shape", &shape[0], shape.size());
  }

  return error_count;
}

// Write an int attribute
int WriteAttribute(hid_t const group, char const *const name,
                   int const ivalue) {
  hid_t dataspace, attribute;
  int error_count = 0;

  HDF5_ERROR(dataspace = H5Screate(H5S_SCALAR));
  HDF5_ERROR(attribute = H5Acreate(group, name, H5T_NATIVE_INT, dataspace,
                                   H5P_DEFAULT));
  HDF5_ERROR(H5Awrite(attribute, H5T_NATIVE_INT, &ivalue));
  HDF5_ERROR(H5Aclose(attribute));
  HDF5_ERROR(H5Sclose(dataspace));

  return error_count;
}

// Write a double attribute
int WriteAttribute(hid_t const group, char const *const name,
                   double const dvalue) {
  hid_t dataspace, attribute;
  int error_count = 0;

  HDF5_ERROR(dataspace = H5Screate(H5S_SCALAR));
  HDF5_ERROR(attribute = H5Acreate(group, name, H5T_NATIVE_DOUBLE, dataspace,
                                   H5P_DEFAULT));
  HDF5_ERROR(H5Awrite(attribute, H5T_NATIVE_DOUBLE, &dvalue));
  HDF5_ERROR(H5Aclose(attribute));
  HDF5_ERROR(H5Sclose(dataspace));

  return error_count;
}

// Write a string attribute
int WriteAttribute(hid_t const group, char const *const name,
                   char const *const svalue) {
  hid_t datatype, dataspace, attribute;
  int error_count = 0;

  HDF5_ERROR(datatype = H5Tcopy(H5T_C_S1));
  HDF5_ERROR(H5Tset_size(datatype, strlen(svalue) + 1));
  HDF5_ERROR(dataspace = H5Screate(H5S_SCALAR));
  HDF5_ERROR(attribute =
                 H5Acreate(group, name, datatype, dataspace, H5P_DEFAULT));
  HDF5_ERROR(H5Awrite(attribute, datatype, svalue));
  HDF5_ERROR(H5Aclose(attribute));
  HDF5_ERROR(H5Sclose(dataspace));
  HDF5_ERROR(H5Tclose(datatype));

  return error_count;
}

// Write an array of int attributes
int WriteAttribute(hid_t const group, char const *const name,
                   int const *const ivalues, int const nvalues) {
  hid_t dataspace, attribute;
  int error_count = 0;

  hsize_t const size = nvalues;
  HDF5_ERROR(dataspace = H5Screate_simple(1, &size, NULL));
  HDF5_ERROR(attribute = H5Acreate(group, name, H5T_NATIVE_INT, dataspace,
                                   H5P_DEFAULT));
  HDF5_ERROR(H5Awrite(attribute, H5T_NATIVE_INT, ivalues));
  HDF5_ERROR(H5Aclose(attribute));
  HDF5_ERROR(H5Sclose(dataspace));

  return error_count;
}

// Write an array of double attributes
int WriteAttribute(hid_t const group, char const *const name,
                   double const *const dvalues, int const nvalues) {
  hid_t dataspace, attribute;
  int error_count = 0;

  hsize_t const size = nvalues;
  HDF5_ERROR(dataspace = H5Screate_simple(1, &size, NULL));
  HDF5_ERROR(attribute = H5Acreate(group, name, H5T_NATIVE_DOUBLE, dataspace,
                                   H5P_DEFAULT));
  HDF5_ERROR(H5Awrite(attribute, H5T_NATIVE_DOUBLE, dvalues));
  HDF5_ERROR(H5Aclose(attribute));
  HDF5_ERROR(H5Sclose(dataspace));

  return error_count;
}

// Write an array of hsize_t attributes
int WriteAttribute(hid_t const group, char const *const name,
                   hsize_t const *const svalues, int const nvalues) {
  hid_t dataspace, attribute;
  int error_count = 0;

  hsize_t const size = nvalues;
  HDF5_ERROR(dataspace = H5Screate_simple(1, &size, NULL));
  HDF5_ERROR(attribute = H5Acreate(group, name, H5T_NATIVE_HSIZE, dataspace,
                                   H5P_DEFAULT));
  HDF5_ERROR(H5Awrite(attribute, H5T_NATIVE_HSIZE, svalues));
  HDF5_ERROR(H5Aclose(attribute));
  HDF5_ERROR(H5Sclose(dataspace));

  return error_count;
}

// Write an array of string attributes
int WriteAttribute(hid_t const group, char const *const name,
                   char const *const *const svalues, int const nvalues) {
  hid_t datatype, dataspace, attribute;
  int error_count = 0;

  size_t maxstrlen = 0;
  for (int i = 0; i < nvalues; ++i) {
    maxstrlen = max(maxstrlen, strlen(svalues[i]));
  }
  vector<char> svalue(nvalues * (maxstrlen + 1));
  for (int i = 0; i < nvalues; ++i) {
    strncpy(&svalue.at(i * maxstrlen), svalues[i], maxstrlen + 1);
  }

  HDF5_ERROR(datatype = H5Tcopy(H5T_C_S1));
  HDF5_ERROR(H5Tset_size(datatype, maxstrlen + 1));
  hsize_t const size = nvalues;
  HDF5_ERROR(dataspace = H5Screate_simple(1, &size, NULL));
  HDF5_ERROR(attribute =
                 H5Acreate(group, name, datatype, dataspace, H5P_DEFAULT));
  HDF5_ERROR(H5Awrite(attribute, datatype, &svalue.front()));
  HDF5_ERROR(H5Aclose(attribute));
  HDF5_ERROR(H5Sclose(dataspace));
  HDF5_ERROR(H5Tclose(datatype));

  return error_count;
}

// Write a large string attribute
int WriteLargeAttribute(hid_t const group, char const *const name,
                        char const *const svalue) {
  hid_t dataspace, dataset, datatype;
  int error_count = 0;

  // Create a dataset, since the data may not fit into an attribute
  hsize_t const size = strlen(svalue) + 1;
  HDF5_ERROR(datatype = H5Tcopy(H5T_C_S1));
  HDF5_ERROR(H5Tset_size(datatype, size));

  HDF5_ERROR(dataspace = H5Screate(H5S_SCALAR));
  HDF5_ERROR(dataset =
                 H5Dcreate(group, name, datatype, dataspace, H5P_DEFAULT));
  HDF5_ERROR(
      H5Dwrite(dataset, datatype, H5S_ALL, H5S_ALL, H5P_DEFAULT, svalue));
  HDF5_ERROR(H5Dclose(dataset));
  HDF5_ERROR(H5Sclose(dataspace));
  HDF5_ERROR(H5Tclose(datatype));

  return error_count;
}

//////////////////////////////////////////////////////////////////////////////
// private routines
//////////////////////////////////////////////////////////////////////////////
static void *SetupGH(tFleshConfig *const fleshconfig, const int convLevel,
                     cGH *const cctkGH) {
  DECLARE_CCTK_PARAMETERS;

  int error_count = 0;

  // register CarpetIOHDF5's routines as a new I/O method
  const int IOMethod = CCTK_RegisterIOMethod("IOHDF5");
  CCTK_RegisterIOMethodOutputGH(IOMethod, OutputGH);
  CCTK_RegisterIOMethodOutputVarAs(IOMethod, OutputVarAs);
  CCTK_RegisterIOMethodTimeToOutput(IOMethod, TimeToOutput);
  CCTK_RegisterIOMethodTriggerOutput(IOMethod, TriggerOutput);

  if (not CCTK_Equals(verbose, "none")) {
    CCTK_INFO("I/O Method 'IOHDF5' registered: AMR output of grid variables "
              "to HDF5 files");
  }

  // register CarpetIOHDF5's recovery routine with IOUtil
  if (IOUtil_RegisterRecover("CarpetIOHDF5 recovery", Recover) < 0) {
    CCTK_WARN(1, "Failed to register " CCTK_THORNSTRING " recovery routine");
  }

  const int numvars = CCTK_NumVars();

  // allocate a new GH extension structure
  CarpetIOHDF5GH *myGH = new CarpetIOHDF5GH;

  myGH->requests.resize(numvars);
  myGH->cp_filename_index = 0;
  myGH->checkpoint_keep = abs(checkpoint_keep);
  myGH->cp_filename_list =
      (char **)calloc(myGH->checkpoint_keep, sizeof(char *));
  myGH->recovery_num_filenames = 0;
  myGH->recovery_filename_list = NULL;
  myGH->out_vars = strdup("");
  myGH->out_every_default = out_every - 1;

  // initial I/O parameter check
  myGH->out_dir = 0;
  myGH->stop_on_parse_errors = strict_io_parameter_check;
  CheckSteerableParameters(cctkGH, myGH);
  myGH->stop_on_parse_errors = 0;

  // check parallel I/O parameters for chunked output
  if (not(CCTK_EQUALS(out_mode, "onefile") or CCTK_EQUALS(out_mode, "proc"))) {
    CCTK_VWarn(CCTK_WARN_COMPLAIN, __LINE__, __FILE__, CCTK_THORNSTRING,
               "IO::out_mode = '%s' is not implemented in %s. "
               "Defaulting to one output file per processor...",
               out_mode, CCTK_THORNSTRING);
  }
  if (CCTK_EQUALS(out_mode, "proc") and io_out_unchunked) {
    CCTK_WARN(CCTK_WARN_COMPLAIN,
              "IO::out_unchunked = 'yes' is incompatible with IO::out_mode = "
              "'proc'. Ignoring setting for IO::out_unchunked...");
  }

  if (output_index && CCTK_EQUALS(out_mode, "onefile"))
    CCTK_VWarn(CCTK_WARN_COMPLAIN, __LINE__, __FILE__, CCTK_THORNSTRING,
               "CarpetIOHDF5::output_index with IO::out_mode = '%s' is not "
               "implemented in %s. "
               "Index will not be output",
               out_mode, CCTK_THORNSTRING);

  // Now set hdf5 complex datatypes
  HDF5_ERROR(myGH->HDF5_COMPLEX =
                 H5Tcreate(H5T_COMPOUND, sizeof(CCTK_COMPLEX)));
  HDF5_ERROR(H5Tinsert(myGH->HDF5_COMPLEX, "real", 0, HDF5_REAL));
  HDF5_ERROR(
      H5Tinsert(myGH->HDF5_COMPLEX, "imag", sizeof(CCTK_REAL), HDF5_REAL));
#ifdef HAVE_CCTK_REAL4
  HDF5_ERROR(myGH->HDF5_COMPLEX8 =
                 H5Tcreate(H5T_COMPOUND, sizeof(CCTK_COMPLEX8)));
  HDF5_ERROR(H5Tinsert(myGH->HDF5_COMPLEX8, "real", 0, H5T_NATIVE_FLOAT));
  HDF5_ERROR(H5Tinsert(myGH->HDF5_COMPLEX8, "imag", sizeof(CCTK_REAL4),
                       H5T_NATIVE_FLOAT));
#endif
#ifdef HAVE_CCTK_REAL8
  HDF5_ERROR(myGH->HDF5_COMPLEX16 =
                 H5Tcreate(H5T_COMPOUND, sizeof(CCTK_COMPLEX16)));
  HDF5_ERROR(H5Tinsert(myGH->HDF5_COMPLEX16, "real", 0, H5T_NATIVE_DOUBLE));
  HDF5_ERROR(H5Tinsert(myGH->HDF5_COMPLEX16, "imag", sizeof(CCTK_REAL8),
                       H5T_NATIVE_DOUBLE));
#endif
#ifdef HAVE_CCTK_REAL16
  HDF5_ERROR(myGH->HDF5_COMPLEX32 =
                 H5Tcreate(H5T_COMPOUND, sizeof(CCTK_COMPLEX32)));
  HDF5_ERROR(H5Tinsert(myGH->HDF5_COMPLEX32, "real", 0, H5T_NATIVE_LDOUBLE));
  HDF5_ERROR(H5Tinsert(myGH->HDF5_COMPLEX32, "imag", sizeof(CCTK_REAL16),
                       H5T_NATIVE_LDOUBLE));
#endif

  return (myGH);
}

static void CheckSteerableParameters(const cGH *const cctkGH,
                                     CarpetIOHDF5GH *myGH) {
  DECLARE_CCTK_PARAMETERS;

  // re-parse the 'IOHDF5::out_dir' parameter if it has changed
  const char *my_out_dir = *out_dir ? out_dir : io_out_dir;
  char *the_out_dir;
  if (strcmp(my_out_dir, ".")) {
    int i = strlen(my_out_dir);
    if (CCTK_Equals(out_mode, "onefile") or not strstr(my_out_dir, "%u")) {
      the_out_dir = (char *)malloc(i + 2);
      strcpy(the_out_dir, my_out_dir);
      the_out_dir[i] = '/';
      the_out_dir[i + 1] = 0;
    } else {
      // TODO: ensure that there is exactly one "%u" and no other "%"
      // substrings, except possibly "%%".
      the_out_dir = (char *)malloc(i + 20);
      snprintf(the_out_dir, i + 19, my_out_dir, dist::rank());
      strcat(the_out_dir, "/");
    }
  } else {
    the_out_dir = strdup("");
  }

  if (not myGH->out_dir or strcmp(the_out_dir, myGH->out_dir)) {
    free(myGH->out_dir);
    myGH->out_dir = the_out_dir;

    // create the output directory
    // const ioGH* const ioUtilGH = (const ioGH*) CCTK_GHExtension (cctkGH,
    // "IO");
    int result = IOUtil_CreateDirectory(cctkGH, myGH->out_dir,
                                        not CCTK_Equals(out_mode, "onefile"),
                                        dist::rank());
    if (result < 0) {
      CCTK_VWarn(1, __LINE__, __FILE__, CCTK_THORNSTRING,
                 "Problem creating HDF5 output directory '%s'", myGH->out_dir);
    } else if (result > 0 and CCTK_Equals(verbose, "full")) {
      CCTK_VInfo(CCTK_THORNSTRING, "HDF5 output directory '%s' already exists",
                 myGH->out_dir);
    }
  } else {
    free(the_out_dir);
  }

  // re-parse the 'IOHDF5::out_vars' parameter if it has changed
  if (strcmp(out_vars, myGH->out_vars)) {
#ifdef IOUTIL_PARSER_HAS_OUT_DT
    IOUtil_ParseVarsForOutput(cctkGH, CCTK_THORNSTRING, "IOHDF5::out_vars",
                              myGH->stop_on_parse_errors, out_vars, -1, -1.0,
                              &myGH->requests[0]);
#else
    IOUtil_ParseVarsForOutput(cctkGH, CCTK_THORNSTRING, "IOHDF5::out_vars",
                              myGH->stop_on_parse_errors, out_vars, -1,
                              &myGH->requests[0]);
#endif

    // notify the user about the new setting
    if (not CCTK_Equals(verbose, "none")) {
      int count = 0;
      ostringstream msg;
      msg << "Periodic AMR output requested for:";
      for (int vi = 0; vi < CCTK_NumVars(); ++vi) {
        if (myGH->requests[vi]) {
          ++count;
          char *const fullname = CCTK_FullName(vi);
          msg << eol << "   " << fullname;
          free(fullname);
        }
      }
      if (count > 0) {
        CCTK_INFO(msg.str().c_str());
      }
    }

    // save the last setting of 'IOHDF5::out_vars' parameter
    free(myGH->out_vars);
    myGH->out_vars = strdup(out_vars);
  }
}

static int OutputGH(const cGH *const cctkGH) {
  static Timers::Timer timer("OutputGH");
  timer.start();

  CarpetIOHDF5GH *myGH =
      (CarpetIOHDF5GH *)CCTK_GHExtension(cctkGH, CCTK_THORNSTRING);
  CheckSteerableParameters(cctkGH, myGH);

  if (strcmp(myGH->out_vars, "")) {
    for (int vindex = CCTK_NumVars() - 1; vindex >= 0; vindex--) {
      if (TimeToOutput(cctkGH, vindex)) {
        TriggerOutput(cctkGH, vindex);
      }
    }
  }
  timer.stop();

  return (0);
}

static int TimeToOutput(const cGH *const cctkGH, const int vindex) {
  DECLARE_CCTK_ARGUMENTS;
  DECLARE_CCTK_PARAMETERS;

  const int numvars = CCTK_NumVars();
  assert(vindex >= 0 and vindex < numvars);

  if (CCTK_GroupTypeFromVarI(vindex) != CCTK_GF and not do_global_mode) {
    return 0;
  }

  CarpetIOHDF5GH *myGH =
      (CarpetIOHDF5GH *)CCTK_GHExtension(cctkGH, CCTK_THORNSTRING);

  // check if output for this variable was requested
  if (not myGH->requests[vindex]) {
    return (0);
  }

  // check whether this refinement level should be output
  if (not(myGH->requests[vindex]->refinement_levels & (1 << reflevel))) {
    return (0);
  }

  // check if output for this variable was requested individually
  // by a "<varname>{ out_every = <number> }" option string
  // this will overwrite the output criterion setting
  const char *myoutcriterion =
      CCTK_EQUALS(out_criterion, "default") ? io_out_criterion : out_criterion;
  if (myGH->requests[vindex]->out_every >= 0) {
    myoutcriterion = "divisor";
  }

  if (CCTK_EQUALS(myoutcriterion, "never")) {
    return (0);
  }

  // check whether to output at this iteration
  bool output_this_iteration = false;

  if (CCTK_EQUALS(myoutcriterion, "iteration")) {
    int myoutevery = out_every == -2 ? io_out_every : out_every;
    if (myoutevery > 0) {
      if (*this_iteration == cctk_iteration) {
        // we already decided to output this iteration
        output_this_iteration = true;
      } else if (cctk_iteration >= *next_output_iteration) {
        // it is time for the next output
        output_this_iteration = true;
        *this_iteration = cctk_iteration;
        *next_output_iteration = cctk_iteration + myoutevery;
      }
    }
  } else if (CCTK_EQUALS(myoutcriterion, "divisor")) {
    int myoutevery = out_every == -2 ? io_out_every : out_every;
    if (myGH->requests[vindex]->out_every >= 0) {
      myoutevery = myGH->requests[vindex]->out_every;
    }
    if (myoutevery > 0 and (cctk_iteration % myoutevery) == 0) {
      // we already decided to output this iteration
      output_this_iteration = true;
    }
  } else if (CCTK_EQUALS(myoutcriterion, "time")) {
    CCTK_REAL myoutdt = out_dt == -2 ? io_out_dt : out_dt;
    if (myoutdt == 0 or *this_iteration == cctk_iteration) {
      output_this_iteration = true;
    } else if (myoutdt > 0) {
      int do_output = (cctk_time / cctk_delta_time >=
                       *next_output_time / cctk_delta_time - 1.0e-12);
      MPI_Bcast(&do_output, 1, MPI_INT, 0, dist::comm());
      if (do_output) {
        // it is time for the next output
        output_this_iteration = true;
        *this_iteration = cctk_iteration;
        *next_output_time = cctk_time + myoutdt;
      }
    }
  }

  return output_this_iteration ? 1 : 0;
}

static int TriggerOutput(const cGH *const cctkGH, const int vindex) {
  DECLARE_CCTK_PARAMETERS;
  int retval;

  char *const fullname = CCTK_FullName(vindex);
  if (one_file_per_group) {
    const int gindex = CCTK_GroupIndexFromVarI(vindex);
    char *const groupname_c = CCTK_GroupName(gindex);
    string groupname(groupname_c);
    free(groupname_c);
    transform(groupname.begin(), groupname.end(), groupname.begin(), ::tolower);
    string const oldsep("::");
    size_t const oldseppos = groupname.find(oldsep);
    assert(oldseppos != string::npos);
    groupname.replace(oldseppos, oldsep.size(), out_group_separator);
    retval = OutputVarAs(cctkGH, fullname, groupname.c_str());
  } else {
    const char *varname = CCTK_VarName(vindex);
    retval = OutputVarAs(cctkGH, fullname, varname);
  }
  free(fullname);

  return (retval);
}

static void GetVarIndex(int vindex, const char *optstring, void *arg) {
  if (optstring) {
    char *fullname = CCTK_FullName(vindex);
    CCTK_VWarn(2, __LINE__, __FILE__, CCTK_THORNSTRING,
               "Option string '%s' will be ignored for HDF5 output of "
               "variable '%s'",
               optstring, fullname);
    free(fullname);
  }

  *((int *)arg) = vindex;
}

static int OutputVarAs(const cGH *const cctkGH, const char *const fullname,
                       const char *const alias) {
  DECLARE_CCTK_ARGUMENTS;
  DECLARE_CCTK_PARAMETERS;

  int error_count = 0;
  int vindex = -1;

  if (CCTK_TraverseString(fullname, GetVarIndex, &vindex, CCTK_VAR) < 0) {
    CCTK_VWarn(1, __LINE__, __FILE__, CCTK_THORNSTRING,
               "error while parsing variable name '%s' (alias name '%s')",
               fullname, alias);
    return (-1);
  }

  if (vindex < 0) {
    return (-1);
  }

  const int group = CCTK_GroupIndexFromVarI(vindex);
  assert(group >= 0);
  cGroup groupdata;
  CCTK_GroupData(group, &groupdata);
  if (groupdata.grouptype == CCTK_SCALAR or groupdata.grouptype == CCTK_ARRAY) {
    assert(do_global_mode);
  }

  // get the default I/O request for this variable
  const CarpetIOHDF5GH *myGH =
      (CarpetIOHDF5GH *)CCTK_GHExtension(cctkGH, CCTK_THORNSTRING);
  ioRequest *request = myGH->requests[vindex];
  if (not request) {
#ifdef IOUTIL_PARSER_HAS_OUT_DT
    request = IOUtil_DefaultIORequest(cctkGH, vindex, 1, -1.0);
#else
    request = IOUtil_DefaultIORequest(cctkGH, vindex, 1);
#endif
  }

  // Get grid hierarchy extentsion from IOUtil
  const ioGH *const iogh = (const ioGH *)CCTK_GHExtension(cctkGH, "IO");
  assert(iogh);

  // Invent a file name
  int ioproc = 0, nioprocs = 1;
  string filename;
  filename.append(myGH->out_dir);
  filename.append(alias);
  if (out_timesteps_per_file > 0) {
    // Round down to nearest multiple of out_timesteps_per_file
    int const iter =
        cctk_iteration / out_timesteps_per_file * out_timesteps_per_file;
    char buffer[32];
    snprintf(buffer, sizeof(buffer), ".iter_%d", iter);
    filename.append(buffer);
  }
  if (not(CCTK_EQUALS(out_mode, "onefile") or request->out_unchunked or
          groupdata.disttype == CCTK_DISTRIB_CONSTANT or dist::size() == 1)) {
    char buffer[32];
    ioproc = dist::rank();
    nioprocs = dist::size();
    snprintf(buffer, sizeof(buffer), ".file_%d", ioproc);
    filename.append(buffer);
  }

  string base_filename(filename);
  filename.append(out_extension);

  string index_filename(base_filename + ".idx" + out_extension);

  const char *const c_filename = filename.c_str();

  // check if the file has been created already
  typedef std::map<string, vector<vector<vector<int> > > > filelist;
  static filelist created_files;
  filelist::iterator thisfile = created_files.find(filename);
  bool is_new_file = thisfile == created_files.end();
  if (is_new_file) {
    int const numvars = CCTK_NumVars();
    vector<vector<vector<int> > > last_outputs; // [ml][rl][var]
    last_outputs.resize(mglevels);
    for (int ml = 0; ml < mglevels; ++ml) {
      last_outputs[ml].resize(maxreflevels);
      for (int rl = 0; rl < maxreflevels; ++rl) {
        last_outputs[ml][rl].resize(numvars, cctk_iteration - 1);
      }
    }
    thisfile = created_files.insert(
        thisfile, filelist::value_type(filename, last_outputs));
    assert(thisfile != created_files.end());
  }

  const int firstvar = one_file_per_group ? CCTK_FirstVarIndexI(group) : vindex;
  const int numvars = one_file_per_group ? CCTK_NumVarsInGroupI(group) : 1;

  // check if this variable has been output already during this iteration
  int &last_output = thisfile->second.at(mglevel).at(reflevel).at(vindex);
  if (last_output == cctk_iteration) {
    // Has already been output during this iteration
    if (not one_file_per_group or vindex == firstvar + numvars - 1) {
      char *varname = CCTK_FullName(vindex);
      CCTK_VWarn(5, __LINE__, __FILE__, CCTK_THORNSTRING,
                 "Skipping output for variable \"%s\", because this variable "
                 "has already been output during the current iteration -- "
                 "probably via a trigger during the analysis stage",
                 varname);
      free(varname);
    }
    return (0);
  }
  assert(last_output < cctk_iteration);
  last_output = cctk_iteration;

  // Check for storage
  if (not CCTK_QueryGroupStorageI(cctkGH, group)) {
    CCTK_VWarn(1, __LINE__, __FILE__, CCTK_THORNSTRING,
               "Cannot output variable '%s' because it has no storage",
               fullname);
    return (0);
  }

  // Open the output file if this is a designated I/O processor
  hid_t file = -1;
  hid_t index_file = -1;
  CCTK_REAL io_files = 0;
  CCTK_REAL io_bytes = 0;
  BeginTimingIO(cctkGH);
  if (dist::rank() == ioproc) {

    if (is_new_file and not IO_TruncateOutputFiles(cctkGH)) {
      H5E_BEGIN_TRY { is_new_file = H5Fis_hdf5(c_filename) <= 0; }
      H5E_END_TRY;
    }

    if (is_new_file) {
      hid_t fapl_id;
      HDF5_ERROR(fapl_id = H5Pcreate(H5P_FILE_ACCESS));
      HDF5_ERROR(H5Pset_fclose_degree(fapl_id, H5F_CLOSE_STRONG));
      HDF5_ERROR(
          file = H5Fcreate(c_filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id));
      if (output_index) {
        HDF5_ERROR(index_file = H5Fcreate(index_filename.c_str(), H5F_ACC_TRUNC,
                                          H5P_DEFAULT, fapl_id));
      }
      HDF5_ERROR(H5Pclose(fapl_id));
      // write metadata information
      error_count +=
          WriteMetadata(cctkGH, nioprocs, firstvar, numvars, false, file);

      if (output_index) {
        error_count += WriteMetadata(cctkGH, nioprocs, firstvar, numvars, false,
                                     index_file);
      }
    } else {
      hid_t fapl_id;
      HDF5_ERROR(fapl_id = H5Pcreate(H5P_FILE_ACCESS));
      HDF5_ERROR(H5Pset_fclose_degree(fapl_id, H5F_CLOSE_STRONG));
      HDF5_ERROR(file = H5Fopen(c_filename, H5F_ACC_RDWR, fapl_id));
      if (output_index)
        HDF5_ERROR(index_file =
                       H5Fopen(index_filename.c_str(), H5F_ACC_RDWR, fapl_id));
      HDF5_ERROR(H5Pclose(fapl_id));
    }
    io_files += 1;
  }

  if (CCTK_Equals(verbose, "full")) {
    CCTK_VInfo(CCTK_THORNSTRING,
               "Writing variable '%s' on mglevel %d reflevel %d", fullname,
               mglevel, reflevel);
  }
  for (int var = firstvar; var < firstvar + numvars; var++) {
    ioRequest *r = myGH->requests[var];
    if (not r) {
#ifdef IOUTIL_PARSER_HAS_OUT_DT
      r = IOUtil_DefaultIORequest(cctkGH, var, 1, -1.0);
#else
      r = IOUtil_DefaultIORequest(cctkGH, var, 1);
#endif
    }
    if ((CCTK_EQUALS(out_mode, "onefile") and io_out_unchunked) or
        r->out_unchunked or groupdata.disttype == CCTK_DISTRIB_CONSTANT) {
      error_count += WriteVarUnchunked(cctkGH, file, io_bytes, r, false);
    } else if (CCTK_EQUALS(out_mode, "onefile")) {
      error_count += WriteVarChunkedSequential(cctkGH, file, io_bytes, r, false,
                                               index_file);
    } else {
      error_count +=
          WriteVarChunkedParallel(cctkGH, file, io_bytes, r, false, index_file);
    }
    if (r != myGH->requests[var])
      IOUtil_FreeIORequest(&r);

    // mark this variable to have been output at this iteration
    thisfile->second.at(mglevel).at(reflevel).at(var) = cctk_iteration;
  }

  // free I/O request structure
  if (request != myGH->requests[vindex]) {
    IOUtil_FreeIORequest(&request);
  }

  // Close the file
  if (file >= 0) {
    HDF5_ERROR(H5Fclose(file));
    if (output_index)
      HDF5_ERROR(H5Fclose(index_file));
  }
  HDF5_ERROR(H5garbage_collect());
  {
    CCTK_REAL local[2], global[2];
    local[0] = io_files;
    local[1] = io_bytes;
    MPI_Allreduce(local, global, 2, dist::mpi_datatype(local[0]), MPI_SUM,
                  dist::comm());
    io_files = global[0];
    io_bytes = global[1];
  }
  EndTimingIO(cctkGH, io_files, io_bytes, true);

  if (error_count > 0 and abort_on_io_errors) {
    CCTK_WARN(0, "Aborting simulation due to previous I/O errors");
  }

  return (0);
}

static void Checkpoint(const cGH *const cctkGH, int called_from) {
  int error_count = 0;
  DECLARE_CCTK_PARAMETERS;

  /* get the filenames for both the temporary and real checkpoint file */
  int ioproc = 0, nioprocs = 1;
  int parallel_io = 0;
  if (not(CCTK_EQUALS(out_mode, "onefile") or dist::size() == 1)) {
    ioproc = dist::rank();
    nioprocs = dist::size();
    parallel_io = 1;
  }
  char *filename = IOUtil_AssembleFilename(cctkGH, NULL, "", ".h5", called_from,
                                           ioproc, not parallel_io);
  char *tempname = IOUtil_AssembleFilename(
      cctkGH, NULL, ".tmp", ".h5", called_from, ioproc, not parallel_io);

  char *index_tempname = IOUtil_AssembleFilename(
      cctkGH, NULL, ".tmp", ".idx.h5", called_from, ioproc, not parallel_io);
  char *index_filename = IOUtil_AssembleFilename(
      cctkGH, NULL, "", ".idx.h5", called_from, ioproc, not parallel_io);

  hid_t file = -1;
  hid_t index_file = -1;
  if (dist::rank() == ioproc) {
    if (CCTK_Equals(verbose, "full")) {
      CCTK_VInfo(CCTK_THORNSTRING, "Creating temporary checkpoint file '%s'",
                 tempname);
    }

    hid_t fapl_id;
    HDF5_ERROR(fapl_id = H5Pcreate(H5P_FILE_ACCESS));
    HDF5_ERROR(H5Pset_fclose_degree(fapl_id, H5F_CLOSE_STRONG));
    HDF5_ERROR(file = H5Fcreate(tempname, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id));

    // write metadata information
    error_count += WriteMetadata(cctkGH, nioprocs, -1, -1, true, file);

    if (output_index) {

      HDF5_ERROR(index_file = H5Fcreate(index_tempname, H5F_ACC_TRUNC,
                                        H5P_DEFAULT, fapl_id));
      error_count += WriteMetadata(cctkGH, nioprocs, -1, -1, true, index_file);
    }
    HDF5_ERROR(H5Pclose(fapl_id));
  }

  // remember the current wall time
  CCTK_REAL const walltime = CCTK_RunTime() / 3600.0;

  // now dump the grid variables on all mglevels, reflevels, maps and components
  BEGIN_MGLEVEL_LOOP(cctkGH) {

    CCTK_REAL io_files = 1;
    CCTK_REAL io_bytes = 0;
    BeginTimingIO(cctkGH);

    BEGIN_REFLEVEL_LOOP(cctkGH) {

      if (CCTK_Equals(verbose, "full")) {
        CCTK_VInfo(CCTK_THORNSTRING,
                   "Dumping grid variables on mglevel %d reflevel %d ...",
                   mglevel, reflevel);
      }

      for (int group = CCTK_NumGroups() - 1; group >= 0; group--) {
        /* skip variables which have been disabled for checkpointing */
        if (not groups_to_checkpoint.empty() and
            not groups_to_checkpoint.at(group)) {
          continue;
        }
        /* only dump groups which have storage assigned */
        if (CCTK_QueryGroupStorageI(cctkGH, group) <= 0 or
            CCTK_NumVarsInGroupI(group) == 0) {
          continue;
        }

        cGroup gdata;
        CCTK_GroupData(group, &gdata);
        assert(gdata.grouptype == CCTK_ARRAY or gdata.grouptype == CCTK_GF or
               gdata.grouptype == CCTK_SCALAR);

        // scalars and grid arrays only have one reflevel
        if (gdata.grouptype != CCTK_GF and reflevel > 0) {
          continue;
        }

        int const len =
            Util_TableGetString(gdata.tagstable, 0, NULL, "checkpoint");
        if (len > 0) {
          vector<char> value_buf(len + 1);
          char *value = &value_buf[0];
          Util_TableGetString(gdata.tagstable, len + 1, value, "checkpoint");
          if (len == sizeof("no") - 1 and CCTK_Equals(value, "no")) {
            continue;
          } else if (not CCTK_Equals(value, "yes")) {
            char *groupname = CCTK_GroupName(group);
            CCTK_VWarn(1, __LINE__, __FILE__, CCTK_THORNSTRING,
                       "Ignoring unknown checkpoint tag '%s' for group '%s'",
                       value, groupname);
            free(groupname);
          }
        }

        /* get the number of active timelevels */
        gdata.numtimelevels = CCTK_ActiveTimeLevelsGI(cctkGH, group);

        int first_vindex = CCTK_FirstVarIndexI(group);

/* get the default I/O request for this group */
#ifdef IOUTIL_PARSER_HAS_OUT_DT
        ioRequest *request =
            IOUtil_DefaultIORequest(cctkGH, first_vindex, 1, -1.0);
#else
        ioRequest *request = IOUtil_DefaultIORequest(cctkGH, first_vindex, 1);
#endif

        /* disable checking for old data objects, disable datatype conversion
           and downsampling */
        request->check_exist = 0;
        request->hdatatype = gdata.vartype;
        for (request->hdim = 0; request->hdim < request->vdim;
             request->hdim++) {
          request->downsample[request->hdim] = 1;
        }

        /* loop over all variables in this group */
        for (request->vindex = first_vindex;
             request->vindex < first_vindex + gdata.numvars;
             request->vindex++) {
          char *fullname = CCTK_FullName(request->vindex);
          assert(fullname);

          /* loop over all timelevels of this variable */
          for (request->timelevel = 0; request->timelevel < gdata.numtimelevels;
               request->timelevel++) {
            if (CCTK_Equals(verbose, "full")) {
              CCTK_VInfo(CCTK_THORNSTRING, "  %s (timelevel %d)", fullname,
                         request->timelevel);
            }

            // write the var
            error_count +=
                parallel_io
                    ? WriteVarChunkedParallel(cctkGH, file, io_bytes, request,
                                              true, index_file)
                    : WriteVarChunkedSequential(cctkGH, file, io_bytes, request,
                                                true, index_file);
          }
          free(fullname);

        } /* end of loop over all variables */

        // free I/O request structure
        IOUtil_FreeIORequest(&request);

      } /* end of loop over all groups */
    }
    END_REFLEVEL_LOOP;

    {
      CCTK_REAL local[2], global[2];
      local[0] = io_files;
      local[1] = io_bytes;
      MPI_Allreduce(local, global, 2, dist::mpi_datatype(local[0]), MPI_SUM,
                    dist::comm());
      io_files = global[0];
      io_bytes = global[1];
    }
    EndTimingIO(cctkGH, io_files, io_bytes, true);
  }
  END_MGLEVEL_LOOP;

  // Close the file
  if (file >= 0) {
    HDF5_ERROR(H5Fclose(file));
  }
  if (index_file >= 0) {
    HDF5_ERROR(H5Fclose(index_file));
  }
  HDF5_ERROR(H5garbage_collect());

  // get global error count
  int temp = error_count;
  MPI_Allreduce(&temp, &error_count, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);

  if (error_count == 0) {
    if (file >= 0) {
      if (rename(tempname, filename)) {
        CCTK_VWarn(1, __LINE__, __FILE__, CCTK_THORNSTRING,
                   "Could not rename temporary checkpoint file '%s' to '%s'",
                   tempname, filename);
        error_count = -1;
      } else if (called_from == CP_EVOLUTION_DATA and checkpoint_keep > 0) {
        CarpetIOHDF5GH *myGH =
            (CarpetIOHDF5GH *)CCTK_GHExtension(cctkGH, CCTK_THORNSTRING);

        // should an older checkpoint file be removed ?
        if (myGH->cp_filename_list[myGH->cp_filename_index]) {
          // check whether the recovery checkpoint (which can be a list of
          // several chunked files) or a checkpoint file should be removed
          if (myGH->recovery_filename_list) {
            for (int i = 0; i < myGH->recovery_num_filenames; i++) {
              if (myGH->recovery_filename_list[i]) {
                // remove possible index file
                string old_index_filename = myGH->recovery_filename_list[i];
                size_t const basenamelen =
                    old_index_filename.rfind(out_extension);
                if (basenamelen != string::npos) {
                  old_index_filename.insert(basenamelen, ".idx");
                  remove(old_index_filename.c_str());
                }

                remove(myGH->recovery_filename_list[i]);
                free(myGH->recovery_filename_list[i]);
              }
            }
            free(myGH->recovery_filename_list);
            myGH->recovery_filename_list = NULL;
          } else {
            // remove possible index file
            string old_index_filename =
                myGH->cp_filename_list[myGH->cp_filename_index];
            size_t const basenamelen = old_index_filename.rfind(out_extension);
            if (basenamelen != string::npos) {
              old_index_filename.insert(basenamelen, ".idx");
              remove(old_index_filename.c_str());
            }
            remove(myGH->cp_filename_list[myGH->cp_filename_index]);
            free(myGH->cp_filename_list[myGH->cp_filename_index]);
          }
        }

        // add this checkpoint to the checkpoint filename ring buffer
        myGH->cp_filename_list[myGH->cp_filename_index] = strdup(filename);
        myGH->cp_filename_index =
            (myGH->cp_filename_index + 1) % checkpoint_keep;

        // since the 'checkpoint_keep' parameter is steerable,
        // we may need to resize the ring buffer
        if (myGH->checkpoint_keep != checkpoint_keep) {
          char **cp_filename_list =
              (char **)calloc(checkpoint_keep, sizeof(char *));
          int min = myGH->checkpoint_keep < checkpoint_keep
                        ? myGH->checkpoint_keep
                        : checkpoint_keep;
          while (min-- > 0) {
            cp_filename_list[min] = myGH->cp_filename_list[min];
          }
          free(myGH->cp_filename_list);
          myGH->cp_filename_list = cp_filename_list;
          myGH->checkpoint_keep = checkpoint_keep;
        }
      }
    }
    if (index_file >= 0) {
      if (rename(index_tempname, index_filename)) {
        CCTK_VWarn(1, __LINE__, __FILE__, CCTK_THORNSTRING,
                   "Could not rename temporary index file '%s' to '%s'",
                   index_tempname, index_filename);
      }
    }
  } else {
    CCTK_VWarn(1, __LINE__, __FILE__, CCTK_THORNSTRING,
               "Failed to create checkpoint at iteration %d",
               cctkGH->cctk_iteration);
  }

  // save the iteration number of this checkpoint
  last_checkpoint_iteration = cctkGH->cctk_iteration;
  last_checkpoint_walltime = walltime;
  if (checkpoint_next) {
    CCTK_ParameterSet("checkpoint_next", CCTK_THORNSTRING, "no");
  }

  // free allocated resources
  free(index_tempname);
  free(index_filename);
  free(tempname);
  free(filename);

  if (error_count > 0 and abort_on_io_errors) {
    CCTK_WARN(0, "Aborting simulation due to previous I/O errors");
  }

} // Checkpoint

int WriteMetadata(const cGH *const cctkGH, int const nioprocs,
                  int const firstvar, int const numvars,
                  bool const called_from_checkpoint, hid_t const file) {
  DECLARE_CCTK_PARAMETERS;
  hid_t group;
  int error_count = 0;

  if (CCTK_Equals(verbose, "full")) {
    CCTK_INFO("Writing simulation metadata...");
  }

  // const ioGH *ioUtilGH = (const ioGH *) CCTK_GHExtension (cctkGH, "IO");
  HDF5_ERROR(group = H5Gcreate(file, METADATA_GROUP, 0));

  error_count += WriteAttribute(group, "main loop index", CCTK_MainLoopIndex());
  error_count += WriteAttribute(group, "GH$iteration", cctkGH->cctk_iteration);
  error_count += WriteAttribute(group, "nioprocs", nioprocs);
  error_count += WriteAttribute(group, "carpet_reflevels", reflevels);
  error_count += WriteAttribute(group, "carpet_global_time", global_time);
  error_count += WriteAttribute(group, "carpet_delta_time", delta_time);
  error_count += WriteAttribute(group, "Cactus version", CCTK_FullVersion());

  // all times on all refinement levels
  // error_count += WriteAttribute (group,
  //                                "numberofmgtimes", mglevels);
  // for (int i = 0; i < mglevels; i++) {
  //   char name[100];
  //   snprintf (name, sizeof (name), "mgleveltimes %d", i);
  //   error_count += WriteAttribute
  //     (group, name, &leveltimes.at(i).front(), leveltimes.at(i).size());
  // }

  // unique configuration identifier
  if (CCTK_IsFunctionAliased("UniqueConfigID")) {
    error_count += WriteAttribute(
        group, "config id", static_cast<char const *>(UniqueConfigID(cctkGH)));
  }

  // unique build identifier
  if (CCTK_IsFunctionAliased("UniqueBuildID")) {
    error_count += WriteAttribute(
        group, "build id", static_cast<char const *>(UniqueBuildID(cctkGH)));
  }

  // unique simulation identifier
  if (CCTK_IsFunctionAliased("UniqueSimulationID")) {
    error_count +=
        WriteAttribute(group, "simulation id",
                       static_cast<char const *>(UniqueSimulationID(cctkGH)));
  }

  // unique run identifier
  if (CCTK_IsFunctionAliased("UniqueRunID")) {
    error_count += WriteAttribute(
        group, "run id", static_cast<char const *>(UniqueRunID(cctkGH)));
  }

  // list all datasets in this file
  {
    ostringstream buf;
    if (firstvar >= 0) {
      // list the selected variables
      assert(firstvar >= 0 and numvars >= 0);
      for (int vi = 0; vi < numvars; ++vi) {
        char *const fullname = CCTK_FullName(firstvar + vi);
        buf << fullname << "\n";
        free(fullname);
      }
    } else {
      // list all variables with storage and without checkpoint="no"
      // TODO: check this only once, then pass the list of variables around
      ENTER_GLOBAL_MODE(cctkGH, 0) {
        for (int gi = 0; gi < CCTK_NumGroups(); ++gi) {
          int const numvars_group = CCTK_NumVarsInGroupI(gi);
          if (numvars_group > 0) {
            if (CCTK_QueryGroupStorageI(cctkGH, gi) > 0) {
              cGroup gdata;
              CCTK_GroupData(gi, &gdata);
              bool do_checkpoint = true;
              int const len =
                  Util_TableGetString(gdata.tagstable, 0, NULL, "checkpoint");
              if (len >= 0) {
                char value[len + 1];
                Util_TableGetString(gdata.tagstable, len + 1, value,
                                    "checkpoint");
                if (CCTK_Equals(value, "no")) {
                  do_checkpoint = false;
                } else if (CCTK_Equals(value, "yes")) {
                  do_checkpoint = true;
                } else {
                  char *const groupname = CCTK_GroupName(gi);
                  CCTK_VWarn(
                      1, __LINE__, __FILE__, CCTK_THORNSTRING,
                      "Ignoring unknown checkpoint tag '%s' for group '%s'",
                      value, groupname);
                  free(groupname);
                }
              }
              if (do_checkpoint) {
                int const firstvar_group = CCTK_FirstVarIndexI(gi);
                for (int vi = 0; vi < numvars_group; ++vi) {
                  char *const fullname = CCTK_FullName(firstvar_group + vi);
                  buf << fullname << "\n";
                  free(fullname);
                }
              }
            }
          }
        } // for vi
      }
      LEAVE_GLOBAL_MODE;
    } // if firstvar < 0
    string const str = buf.str();
    error_count += WriteLargeAttribute(group, "Datasets", str.c_str());
  }

  // save parameters in a separate dataset (may be too big for an attribute)
  if (called_from_checkpoint or not CCTK_Equals(out_save_parameters, "no")) {
    const int get_all =
        called_from_checkpoint or CCTK_Equals(out_save_parameters, "all");
    char *parameters = IOUtil_GetAllParameters(cctkGH, get_all);
    assert(parameters);
    error_count += WriteLargeAttribute(group, ALL_PARAMETERS, parameters);
    free(parameters);
  }

  // Save grid structure
  if (called_from_checkpoint or not CCTK_Equals(out_save_parameters, "no")) {
    vector<vector<vector<region_t> > > grid_superstructure(maps);
    vector<vector<vector<region_t> > > grid_structure(maps);
    vector<vector<i2vect> > grid_ghosts(maps);
    vector<vector<i2vect> > grid_buffers(maps);
    vector<vector<int> > grid_prolongation_orders(maps);
    for (int m = 0; m < maps; ++m) {
      grid_superstructure.at(m) = vhh.at(m)->superregions;
      grid_structure.at(m) = vhh.at(m)->regions.at(0);
      grid_ghosts.at(m) = vdd.at(m)->ghost_widths;
      grid_buffers.at(m) = vdd.at(m)->buffer_widths;
      grid_prolongation_orders.at(m) = vdd.at(m)->prolongation_orders_space;
    }
    vector<vector<vector<CCTK_REAL> > > grid_times(mglevels);
    for (int ml = 0; ml < mglevels; ++ml) {
      grid_times.at(ml).resize(vhh.at(0)->reflevels());
      for (int rl = 0; rl < vhh.at(0)->reflevels(); ++rl) {
        grid_times.at(ml).at(rl).resize(tt->timelevels);
        for (int tl = 0; tl < tt->timelevels; ++tl) {
          grid_times.at(ml).at(rl).at(tl) = tt->get_time(ml, rl, tl);
        }
      }
    }
    vector<vector<CCTK_REAL> > grid_delta_times(mglevels);
    for (int ml = 0; ml < mglevels; ++ml) {
      grid_delta_times.at(ml).resize(vhh.at(0)->reflevels());
      for (int rl = 0; rl < vhh.at(0)->reflevels(); ++rl) {
        grid_delta_times.at(ml).at(rl) = tt->get_delta(ml, rl);
      }
    }
    ostringstream gs_buf;
    gs_buf << setprecision(17);
    // We could write this information only into one of the checkpoint
    // files (to save space), or write it into a separate metadata
    // file
    gs_buf << "grid_superstructure:" << grid_superstructure << ",";
    // We could omit the grid structure (to save space), or write it
    // only into one of the checkpoint files
    gs_buf << "grid_structure:" << grid_structure << ",";
    gs_buf << "grid_times:" << grid_times << ",";
    gs_buf << "grid_delta_times:" << grid_delta_times << ",";
    // gs_buf << "grid_leveltimes:" << leveltimes << ",";
    gs_buf << "grid_ghosts:" << grid_ghosts << ",";
    gs_buf << "grid_buffers:" << grid_buffers << ",";
    gs_buf << "grid_prolongation_orders:" << grid_prolongation_orders << ".";
    string const gs_str = gs_buf.str();
    error_count += WriteLargeAttribute(group, GRID_STRUCTURE, gs_str.c_str());
  }

  HDF5_ERROR(H5Gclose(group));

  return (error_count);
}

} // namespace CarpetIOHDF5

```

Input.cc:
```
#include <cassert>
#include <cstdlib>
#include <cstring>
#include <list>
#include <sstream>
#include <string>
#include <vector>
#include <map>
#include <algorithm>

#include "util_Table.h"
#include "cctk.h"
#include "cctk_Parameters.h"

#include "CarpetIOHDF5.hh"
#include "CactusBase/IOUtil/src/ioGH.h"
#include "CactusBase/IOUtil/src/ioutil_CheckpointRecovery.h"

#include "defs.hh"

namespace CarpetIOHDF5 {

using namespace std;
using namespace Carpet;

// structure describing a patch as a single dataset of an HDF5 file to read from
typedef struct {
  string patchname;
  int vindex;
  int map;
  int mglevel;
  int reflevel;
  int timestep;
  int timelevel;
  int component;
  int rank;
  ivect iorigin;
  ivect ioffset;
  ivect ioffsetdenom;
  vector<hsize_t> shape;
} patch_t;

// structure describing the contents of an HDF5 file to read from
typedef struct {
  char *filename, *indexfilename;
  hid_t file, indexfile;
  list<patch_t> patches;
} file_t;

// structure describing a set of HDF5 files
typedef struct {
  string setname;
  string basefilename;
  int first_ioproc;
  bool has_index;
  vector<file_t> files; // [nioprocs]

  int nioprocs;
  int num_mglevels;
  int num_reflevels;
  int cctk_iteration;
  int main_loop_index;
  double global_time;
  double delta_time;
  // vector<double> mgleveltimes;  // [num_mglevels*num_reflevels]

  vector<vector<vector<region_t> > >
      grid_superstructure; // [map][reflevel][component]
  vector<vector<vector<region_t> > >
      grid_structure; // [map][reflevel][component]
  vector<vector<vector<CCTK_REAL> > >
      grid_times;                              // [mglevel][reflevel][timelevel]
  vector<vector<CCTK_REAL> > grid_delta_times; // [mglevel][reflevel]
  // vector<vector<CCTK_REAL> > leveltimes;             // [mglevel][reflevel]
  vector<vector<i2vect> > grid_ghosts;           // [map]
  vector<vector<i2vect> > grid_buffers;          // [map]
  vector<vector<int> > grid_prolongation_orders; // [map]

} fileset_t;

// list of checkpoint/filereader files
static list<fileset_t> filesets;

// number of reflevels in the checkpoint
static int num_reflevels = -1;

static list<fileset_t>::iterator OpenFileSet(const cGH *const cctkGH,
                                             const string setname,
                                             const char *basefilename,
                                             int called_from);
static void ReadMetadata(fileset_t &fileset, const file_t &file);
static herr_t BrowseDatasets(hid_t group, const char *objectname, void *arg);
static int ReadVar(const cGH *const cctkGH, file_t &file, CCTK_REAL &io_bytes,
                   list<patch_t>::const_iterator patch,
                   vector<ibset> &bboxes_read, bool in_recovery);

//////////////////////////////////////////////////////////////////////////////
// Register with the Cactus Recovery Interface
//////////////////////////////////////////////////////////////////////////////
int CarpetIOHDF5_RecoverParameters() {
  return IOUtil_RecoverParameters(Recover, ".h5", "HDF5");
}

//////////////////////////////////////////////////////////////////////////////
// Recover the grid structure
// TODO: Move this routine into CarpetLib
//////////////////////////////////////////////////////////////////////////////
void CarpetIOHDF5_RecoverGridStructure(CCTK_ARGUMENTS) {
  DECLARE_CCTK_PARAMETERS;
  DECLARE_CCTK_ARGUMENTS;

  fileset_t &fileset = *filesets.begin();

  if (not CCTK_Equals(verbose, "none")) {
    CCTK_INFO("recovering grid structure");
  }

  // Abort with an error if there is no grid structure in the
  // checkpoint file, or if the number of maps is wrong
  if (fileset.grid_structure.empty()) {
    CCTK_WARN(0, "No grid structure information found in checkpoint file !");
  } else if (int(fileset.grid_structure.size()) != maps) {
    CCTK_VWarn(0, __LINE__, __FILE__, CCTK_THORNSTRING,
               "Number of maps in the checkpoint's grid structure information "
               "(%d) doesn't match current number of maps (%d) !",
               int(fileset.grid_structure.size()), maps);
  }

  vector<vector<vector<region_t> > > superregsss = fileset.grid_superstructure;
  vector<vector<vector<vector<region_t> > > > regssss(maps);

  int type;
  void const *const ptr =
      CCTK_ParameterGet("regrid_in_level_mode", "Carpet", &type);
  assert(ptr != 0);
  assert(type == PARAMETER_BOOLEAN);
  CCTK_INT const regrid_in_level_mode = *static_cast<CCTK_INT const *>(ptr);

  if (not regrid_in_level_mode) {
    // Distribute each map independently

    for (int m = 0; m < maps; ++m) {
      vector<vector<region_t> > &superregss = superregsss.at(m);

      // Make multiprocessor aware
      vector<vector<region_t> > regss(superregss.size());
      for (size_t rl = 0; rl < superregss.size(); ++rl) {
        Carpet::SplitRegions(cctkGH, superregss.at(rl), regss.at(rl));
      } // for rl

      // Make multigrid aware
      Carpet::MakeMultigridBoxes(cctkGH, m, regss, regssss.at(m));

    } // for m

  } else { // if regrid_in_level_mode
    // Distribute all maps at the same time

    vector<vector<vector<region_t> > > regsss(maps);

    // Count levels
    vector<int> rls(maps);
    for (int m = 0; m < maps; ++m) {
      rls.at(m) = superregsss.at(m).size();
    }
    int maxrl = 0;
    for (int m = 0; m < maps; ++m) {
      maxrl = max(maxrl, rls.at(m));
    }
    // All maps must have the same number of levels
    for (int m = 0; m < maps; ++m) {
      superregsss.at(m).resize(maxrl);
      regsss.at(m).resize(maxrl);
    }

    // Make multiprocessor aware
    for (int rl = 0; rl < maxrl; ++rl) {
      vector<vector<region_t> > superregss(maps);
      for (int m = 0; m < maps; ++m) {
        superregss.at(m) = superregsss.at(m).at(rl);
      }
      vector<vector<region_t> > regss(maps);
      Carpet::SplitRegionsMaps(cctkGH, superregss, regss);
      for (int m = 0; m < maps; ++m) {
        superregsss.at(m).at(rl) = superregss.at(m);
        regsss.at(m).at(rl) = regss.at(m);
      }
    } // for rl

    // Make multigrid aware
    Carpet::MakeMultigridBoxesMaps(cctkGH, regsss, regssss);

  } // if regrid_in_level_mode

  for (int m = 0; m < maps; ++m) {
    // Regrid
    RegridMap(cctkGH, m, superregsss.at(m), regssss.at(m), false);
  } // for m

  // Set time hierarchy correctly after RegridMap created it
  for (int ml = 0; ml < vhh.at(0)->mglevels(); ++ml) {
    for (int rl = 0; rl < vhh.at(0)->reflevels(); ++rl) {
      for (int tl = 0; tl < tt->timelevels; ++tl) {
        tt->set_time(ml, rl, tl, fileset.grid_times.at(ml).at(rl).at(tl));
      }
      tt->set_delta(ml, rl, fileset.grid_delta_times.at(ml).at(rl));
    }
  }
  // We are in level mode here (we probably shouldn't be, but that's
  // how Cactus I/O was designed), so we need to fix up certain
  // things. See Carpet's function leave_level_mode to see what
  // happens when we leave level mode.
  {
    // TODO: Switch to global mode instead of fixing up things
    // manually
    assert(reflevel != -1);
    int const tl = 0;
    delta_time = fileset.delta_time;
    cctkGH->cctk_delta_time = delta_time;
    global_time = fileset.global_time;
    cctkGH->cctk_time = tt->get_time(mglevel, reflevel, tl);
  }

  PostRegrid(cctkGH);

  // Set level times correctly after PostRegrid created them
  // leveltimes = fileset.leveltimes;

  for (int rl = 0; rl < reflevels; ++rl) {
    Recompose(cctkGH, rl, false);
  }

  RegridFree(cctkGH, false);

  // Check ghost and buffer widths and prolongation orders
  // TODO: Instead of only checking them, set them during regridding.
  // (This requires setting them during regridding instead of during setup.)
  for (int m = 0; m < maps; ++m) {
    assert(fileset.grid_ghosts.at(m).size() == vdd.at(m)->ghost_widths.size());
    for (size_t rl = 0; rl < fileset.grid_ghosts.at(m).size(); ++rl) {
      assert(all(all(fileset.grid_ghosts.at(m).at(rl) ==
                     vdd.at(m)->ghost_widths.at(rl))));
    }
    assert(fileset.grid_buffers.at(m).size() ==
           vdd.at(m)->buffer_widths.size());
    for (size_t rl = 0; rl < fileset.grid_buffers.at(m).size(); ++rl) {
      assert(all(all(fileset.grid_buffers.at(m).at(rl) ==
                     vdd.at(m)->buffer_widths.at(rl))));
    }
    assert(fileset.grid_prolongation_orders.at(m).size() ==
           vdd.at(m)->prolongation_orders_space.size());
    for (size_t rl = 0; rl < fileset.grid_prolongation_orders.at(m).size();
         ++rl) {
      assert(fileset.grid_prolongation_orders.at(m).at(rl) ==
             vdd.at(m)->prolongation_orders_space.at(rl));
    }
  }
}

//////////////////////////////////////////////////////////////////////////////
// Overwrite the "CarpetRegrid::refinement_levels"
// with the number of levels given in the checkpoint file
//
// Note that this has to be done after parameter recovery in order to have
// any effect of steering "CarpetRegrid::refinement_levels".
//////////////////////////////////////////////////////////////////////////////
int CarpetIOHDF5_SetNumRefinementLevels() {
  DECLARE_CCTK_PARAMETERS;

  if (num_reflevels > 0) {
    if (not CCTK_Equals(verbose, "none")) {
      char *buffer =
          CCTK_ParameterValString("refinement_levels", "CarpetRegrid");
      assert(buffer);
      CCTK_VInfo(CCTK_THORNSTRING, "Using %i reflevels from checkpoint file. "
                                   "Ignoring value '%s' in parameter file.",
                 num_reflevels, buffer);
      free(buffer);
    }

    char buffer[32];
    snprintf(buffer, sizeof(buffer), "%d", num_reflevels);
    int const retval =
        CCTK_ParameterSet("refinement_levels", "CarpetRegrid", buffer);
    assert(retval == 0);
  }

  return 0;
}

//////////////////////////////////////////////////////////////////////////////
// close all open checkpoint/filereader files after recovering grid variables
//////////////////////////////////////////////////////////////////////////////
void CarpetIOHDF5_CloseFiles(CCTK_ARGUMENTS) {
  DECLARE_CCTK_ARGUMENTS;
  DECLARE_CCTK_PARAMETERS;
  int error_count = 0;

  for (list<fileset_t>::const_iterator set = filesets.begin();
       set != filesets.end(); set++) {
    for (unsigned int i = 0; i < set->files.size(); i++) {
      if (set->files[i].file >= 0 or set->files[i].indexfile >= 0) {
        if (CCTK_Equals(verbose, "full")) {
          CCTK_VInfo(CCTK_THORNSTRING, "closing file '%s' after recovery",
                     set->files[i].filename);
        }

        if (set->files[i].file >= 0) {
          HDF5_ERROR(H5Fclose(set->files[i].file));
        }
        if (set->files[i].indexfile >= 0) {
          HDF5_ERROR(H5Fclose(set->files[i].indexfile));
        }
      }
      if (set->files[i].filename != NULL) {
        free(set->files[i].filename);
      }
      if (set->files[i].indexfilename) {
        free(set->files[i].indexfilename);
      }
    }
  }
  HDF5_ERROR(H5garbage_collect());

  filesets.clear();
}

//////////////////////////////////////////////////////////////////////////////
// Top-level recovery/filereader routine
// Opens the checkpoint/data file on the first time through,
// recovers parameters and variables
//////////////////////////////////////////////////////////////////////////////
int Recover(cGH *cctkGH, const char *basefilename, int called_from) {
  int error_count = 0;
  DECLARE_CCTK_PARAMETERS;

  assert(called_from == CP_RECOVER_PARAMETERS or
         called_from == CP_RECOVER_DATA or called_from == FILEREADER_DATA);
  const bool in_recovery =
      called_from == CP_RECOVER_PARAMETERS or called_from == CP_RECOVER_DATA;

  const char *const dir = in_recovery ? recover_dir : filereader_ID_dir;
  string setname(dir);
  if (called_from == FILEREADER_DATA) {
    setname.append(basefilename);
  }

  list<fileset_t>::iterator fileset = filesets.begin();
  while (fileset != filesets.end()) {
    if (fileset->setname == setname) {
      break;
    }
    fileset++;
  }
  if (fileset == filesets.end()) {
    assert(called_from == CP_RECOVER_PARAMETERS or
           called_from == FILEREADER_DATA);

    fileset = OpenFileSet(cctkGH, setname, basefilename, called_from);
    if (fileset == filesets.end()) {
      return (-1);
    }

    if (called_from == CP_RECOVER_PARAMETERS) {
      // return here if only parameters should be recovered
      // (come back later with called_from == CP_RECOVER_DATA)
      return (1);
    }
  }

  // set global Cactus/Carpet variables
  if (in_recovery) {

    // global_time = fileset->global_time;
    // delta_time = fileset->delta_time;
    CCTK_SetMainLoopIndex(fileset->main_loop_index);

    cctkGH->cctk_iteration = fileset->cctk_iteration;
    // int const idx = mglevel*fileset->num_reflevels + reflevel;
    // cctkGH->cctk_time = fileset->mgleveltimes.at(idx);
    // cctkGH->cctk_time = global_time;

    if (use_grid_structure_from_checkpoint) {
      // recover the grid structure only once
      static bool have_grid_structure = false;
      if (not have_grid_structure) {
        have_grid_structure = true;
        CarpetIOHDF5_RecoverGridStructure(cctkGH);
      }
    }
  }

  if (not CCTK_Equals(verbose, "none")) {
    CCTK_VInfo(CCTK_THORNSTRING,
               "reading grid variables on mglevel %d reflevel %d", mglevel,
               reflevel);
  }

  CCTK_REAL io_files = 0;
  CCTK_REAL io_bytes = 0;
  BeginTimingIO(cctkGH);

  // create a bbox set for each active timelevel of all variables
  // to mark how much has been read already
  const int numvars = CCTK_NumVars();
  vector<vector<bool> > warned_about_multiple_files(numvars);
  vector<vector<bool> > read_completely(numvars);
  vector<vector<vector<ibset> > > bboxes_read(numvars);
  for (unsigned int vindex = 0; vindex < bboxes_read.size(); vindex++) {
    const int timelevels = CCTK_ActiveTimeLevelsVI(cctkGH, vindex);
    read_completely[vindex].resize(timelevels, false);
    warned_about_multiple_files[vindex].resize(timelevels, false);
    bboxes_read[vindex].resize(timelevels);
    for (int tl = 0; tl < timelevels; tl++) {
      bboxes_read[vindex][tl].resize(maps);
    }
  }

  // Mark certain variable as not-to-be read. We do this by marking
  // them as "read completely", so that they will be skipped later.
  if (in_recovery and not CCTK_EQUALS(skip_recover_variables, "")) {
    struct callback_t {
      vector<vector<bool> > *read_completely;
      static void function(int const idx, const char *const optstring,
                           void *const callback_arg) {
        callback_t &callback = *(callback_t *)callback_arg;
        char *const fullname = CCTK_FullName(idx);
        CCTK_VInfo(CCTK_THORNSTRING, "   skipping %s", fullname);
        free(fullname);
        for (size_t tl = 0; tl < callback.read_completely->at(idx).size();
             ++tl) {
          callback.read_completely->at(idx).at(tl) = true;
        }
      }
    } callback;
    callback.read_completely = &read_completely;
    CCTK_VInfo(CCTK_THORNSTRING,
               "Skipping the following variables while recovering:");
    int const iret =
        CCTK_TraverseString(skip_recover_variables, &callback_t::function,
                            &callback, CCTK_GROUP_OR_VAR);
    if (iret < 0) {
      CCTK_WARN(CCTK_WARN_ALERT,
                "Error in parameter CarpetIOHDF5::skip_recover_variables");
    }
  }

  // query for groups which have the 'CHECKPOINT = "no"' option set
  // Such groups are not checked against being read completely.
  vector<bool> not_checkpointed(numvars);
  if (in_recovery) {
    for (unsigned int vindex = 0; vindex < not_checkpointed.size(); vindex++) {
      int gindex = CCTK_GroupIndexFromVarI(vindex);
      int tagstable = CCTK_GroupTagsTableI(gindex);
      int const len = Util_TableGetString(tagstable, 0, NULL, "checkpoint");
      if (len > 0) {
        vector<char> value_buf(len + 1);
        char *value = &value_buf[0];
        Util_TableGetString(tagstable, len + 1, value, "checkpoint");
        if (len == sizeof("no") - 1 and CCTK_Equals(value, "no")) {
          not_checkpointed[vindex] = true;
        }
      }
    }
  }

  // create a bbox set for each group to list how much needs to be read
  const int numgroups = CCTK_NumGroups();
  vector<vector<ibset> > group_bboxes(numgroups);
  for (unsigned int gindex = 0; gindex < group_bboxes.size(); gindex++) {
    group_bboxes[gindex].resize(maps);
    const int grouptype = CCTK_GroupTypeI(gindex);
    if (grouptype == CCTK_GF) {
      for (int m = 0; m < maps; ++m) {
        struct arrdesc &data = arrdata.at(gindex).at(m);
        for (int lc = 0; lc < data.hh->local_components(reflevel); ++lc) {
          int const c = data.hh->get_component(reflevel, lc);
          group_bboxes[gindex][m] |=
              data.dd->light_boxes.at(mglevel).at(reflevel).at(c).exterior;
        }
      }
    } else {
      if (mglevel == 0 and reflevel == 0) {
        int const m = 0;
        struct arrdesc &data = arrdata.at(gindex).at(m);
        int const c = dist::rank();
        group_bboxes[gindex][m] |=
            data.dd->light_boxes.at(mglevel).at(reflevel).at(c).exterior;
      }
    }
  }

  // mark variables in groups with no grid points (size 0) as already done
  for (int group = 0; group < (int)group_bboxes.size(); group++) {
    bool is_empty = true;
    for (int m = 0; m < (int)group_bboxes[group].size(); m++) {
      is_empty &= group_bboxes[group][m].empty();
    }
    if (is_empty) {
      int vindex = CCTK_FirstVarIndexI(group);
      const int last = vindex + CCTK_NumVarsInGroupI(group);
      while (vindex < last) {
        read_completely[vindex].assign(read_completely[vindex].size(), true);
        vindex++;
      }
    }
  }

  // in filereader mode, mark all variables that are not requested as being
  // read completely, mark all timelevels > 0 as being read completely
  const ioGH *ioUtilGH = (const ioGH *)CCTK_GHExtension(cctkGH, "IO");
  if (not in_recovery and ioUtilGH->do_inVars) {
    for (unsigned int vindex = 0; vindex < read_completely.size(); vindex++) {
      const int timestep = ioUtilGH->do_inVars[vindex];
      for (unsigned int tl = 0; tl < read_completely[vindex].size(); ++tl) {
        read_completely[vindex][tl] = tl > 0 or not timestep;
      }
    }
  }

  // construct list of aliases for variable names. This is a global list even
  // though the user gives a per-variable list.
  std::map<string, unsigned int> alias;
#ifdef IOUTIL_IOGH_HAS_ALIAS
  for (unsigned int vindex = 0; vindex < read_completely.size(); vindex++) {
    if (ioUtilGH->alias && ioUtilGH->alias[vindex]) {
      string key(ioUtilGH->alias[vindex]);
      // the explicit cast avoids ambiguity in the overloaded function pointer
      transform(key.begin(), key.end(), key.begin(), (int (*)(int))toupper);
      alias[key] = vindex;
    }
  }
#endif

  CarpetIOHDF5GH *myGH =
      (CarpetIOHDF5GH *)CCTK_GHExtension(cctkGH, CCTK_THORNSTRING);
  // allocate list of recovery filenames
  // if the recovery checkpoint should be removed eventually
  if (in_recovery and recover_and_remove and checkpoint_keep > 0) {
    if (not myGH->recovery_filename_list) {
      myGH->recovery_num_filenames = fileset->files.size();
      myGH->recovery_filename_list =
          (char **)calloc(fileset->files.size(), sizeof(char **));
      assert(myGH->cp_filename_index == 0);

      // add a dummy entry in the checkpoint filename ring buffer
      static char bla[] = "bla";
      myGH->cp_filename_list[myGH->cp_filename_index] = bla;
      myGH->cp_filename_index = (myGH->cp_filename_index + 1) % checkpoint_keep;
    }
  }

  // loop over all input files of this set
  for (unsigned int i = 0; i < fileset->files.size(); i++) {

    const int file_idx = (i + fileset->first_ioproc) % fileset->nioprocs;
    file_t &file = fileset->files[file_idx];

    // open the file (if it hasn't been already) and read its metadata
    // if we already browsed this file once we defer opening it again until we
    // have to
    if (not file.filename) {
      assert(not file.indexfilename);
      file.indexfilename =
          IOUtil_AssembleFilename(NULL, fileset->basefilename.c_str(), "",
                                  ".idx.h5", called_from, file_idx, 0);
      assert(file.indexfilename);

      file.filename =
          IOUtil_AssembleFilename(NULL, fileset->basefilename.c_str(), "",
                                  ".h5", called_from, file_idx, 0);
      assert(file.filename);
    }
    if (file.patches.size() == 0) {
      // we defer opening the actual data file until we have to
      // and open only the index file if it exists

      io_files += 1;

      hid_t fapl_id;
      HDF5_ERROR(fapl_id = H5Pcreate(H5P_FILE_ACCESS));
      HDF5_ERROR(H5Pset_fclose_degree(fapl_id, H5F_CLOSE_STRONG));
      if (file.indexfile < 0 and fileset->has_index) {
        HDF5_ERROR(file.indexfile =
                       H5Fopen(file.indexfilename, H5F_ACC_RDONLY, fapl_id));
        if (CCTK_Equals(verbose, "full")) {
          CCTK_VInfo(CCTK_THORNSTRING, "opening index file '%s'",
                     file.indexfilename);
        }
      } else {
        HDF5_ERROR(file.file = H5Fopen(file.filename, H5F_ACC_RDONLY, fapl_id));
        if (CCTK_Equals(verbose, "full")) {
          CCTK_VInfo(CCTK_THORNSTRING, "opening %s file '%s'",
                     in_recovery ? "checkpoint" : "input", file.filename);
        }
      }
      HDF5_ERROR(H5Pclose(fapl_id));

      // browse through all datasets contained in this file
      void *args[2] = {&file, &alias};
      HDF5_ERROR(H5Giterate(file.indexfile >= 0 ? file.indexfile : file.file,
                            "/", NULL, BrowseDatasets, args));
    }
    assert(file.patches.size() > 0);
    if (myGH->recovery_filename_list and not myGH->recovery_filename_list[i]) {
      myGH->recovery_filename_list[i] = strdup(file.filename);
    }

    // some optimisation for the case when all processors recover
    // from a single chunked checkpoint file:
    // reorder the list so that processor-local components come first
    if (fileset->nioprocs == 1) {
      list<patch_t>::iterator patch = file.patches.begin();
      while (patch != file.patches.end()) {
        if (patch->component == dist::rank()) {
          file.patches.push_front(*patch);
          patch = file.patches.erase(patch);
        } else {
          patch++;
        }
      }
    }

    // now loop over all patches contained in this file
    for (list<patch_t>::const_iterator patch = file.patches.begin();
         patch != file.patches.end(); patch++) {

      // skip all variables which aren't expected to be recovered
      // this should only happen for variables whose checkpoint tag was changed
      // between the code version used to write the checkpoint and the current
      // one
      if (not_checkpointed[patch->vindex]) {
        continue;
      }

      // only recover grid variables for the current mglevel/reflevel
      if (patch->mglevel != mglevel or patch->reflevel != reflevel) {
        continue;
      }

      // When called by the filereader, IOUtil will set the 'do_inVars' vector
      // if the parameter 'IO::filereader_ID_vars' has been set.
      // In this parameter the user can specify individual variables to be read
      // as well as (optionally) a specific timestep.
      //
      // If 'IO::filereader_ID_vars' is left empty, 'do_inVars' will be NULL
      // which implicitely selects all variables for input.
      // If 'do_inVars[vindex]' is 0, variable 'vindex' was not selected.
      // If 'do_inVars[vindex]' is positive, a specific timestep of 'vindex'
      // was selected.
      // If 'do_inVars[vindex]' is negative, 'vindex' was selected with no
      // specific timestep; all timesteps will be read, the last one wins.
      if (ioUtilGH->do_inVars and ioUtilGH->do_inVars[patch->vindex] >= 0 and
          ioUtilGH->do_inVars[patch->vindex] != patch->timestep + 1) {
        continue;
      }

      // check the timelevel
      if ((unsigned int)patch->timelevel >=
          read_completely.at(patch->vindex).size()) {
        CCTK_VWarn(3, __LINE__, __FILE__, CCTK_THORNSTRING,
                   "Ignoring dataset '%s' (invalid timelevel %d)",
                   patch->patchname.c_str(), patch->timelevel);
        continue;
      }

      // actually read the patch
      if (not read_completely.at(patch->vindex).at(patch->timelevel)) {
        error_count += ReadVar(
            cctkGH, file, io_bytes, patch,
            bboxes_read.at(patch->vindex).at(patch->timelevel), in_recovery);

        // update the read_completely entry
        const int gindex = CCTK_GroupIndexFromVarI(patch->vindex);
        read_completely.at(patch->vindex).at(patch->timelevel) =
            bboxes_read.at(patch->vindex).at(patch->timelevel) ==
            group_bboxes.at(gindex);
      }
    }

    // check if all variables have been read completely already
    bool all_done = true;
    for (unsigned int vindex = 0; vindex < read_completely.size(); vindex++) {

      // skip all variables which aren't expected to be recovered
      if (not_checkpointed[vindex] or
          (CCTK_GroupTypeFromVarI(vindex) != CCTK_GF and reflevel > 0)) {
        continue;
      }

      for (unsigned int tl = 0; tl < read_completely[vindex].size(); tl++) {
        all_done &= read_completely[vindex][tl];
        if (not read_completely[vindex][tl] and
            not warned_about_multiple_files[vindex][tl]) {
          static bool warned_once_already = false;
          const int warnlevel =
              warned_once_already ? CCTK_WARN_DEBUG : CCTK_WARN_COMPLAIN;
          char *const fullname = CCTK_FullName(vindex);
          CCTK_VWarn(
              warnlevel, __LINE__, __FILE__, CCTK_THORNSTRING,
              "Variable %s on rl %d and tl %d not read completely.  Will have "
              "to look for it in other files.%s",
              fullname, reflevel, tl,
              warned_once_already ? "" : " Further warnings will be level 4.");
          free(fullname);
          warned_about_multiple_files[vindex][tl] = true;
          warned_once_already = true;
        }
      }
    }
    if (all_done) {
      break;
    }

    // keep the file open if not requested otherwise by the user
    if (open_one_input_file_at_a_time) {
      if (file.file >= 0) {
        HDF5_ERROR(H5Fclose(file.file));
        file.file = -1;
      }
      if (file.filename) {
        free(file.filename);
        file.filename = NULL;
      }
      if (file.indexfile >= 0) {
        HDF5_ERROR(H5Fclose(file.indexfile));
        file.indexfile = -1;
      }
      if (file.indexfilename) {
        free(file.indexfilename);
        file.indexfilename = NULL;
      }

      HDF5_ERROR(H5garbage_collect());
    }
  }

  // check that all variables have been read completely on this mglevel/reflevel
  int num_incomplete = 0;
  for (unsigned int vindex = 0; vindex < read_completely.size(); vindex++) {

    if (CCTK_GroupTypeFromVarI(vindex) != CCTK_GF and reflevel > 0) {
      continue;
    }

    for (unsigned int tl = 0; tl < read_completely[vindex].size(); tl++) {
      if (not read_completely[vindex][tl]) {
        // check if the variable has been read partially
        size_t size = 0;
        for (int m = 0; m < maps; m++) {
          size += bboxes_read[vindex][tl][m].size();
        }
        char *fullname = CCTK_FullName(vindex);
        if (size == 0) {
          if (not_checkpointed[vindex]) {
            CCTK_VWarn(4, __LINE__, __FILE__, CCTK_THORNSTRING,
                       "variable '%s' timelevel %d has not been read "
                       "(variable has option tag \"CHECKPOINT = 'no'\")",
                       fullname, tl);
          } else {
            // TODO: With the file reader, the files are read
            // sequentially, and each file contains only some of the
            // variables.  The warning below is emitted for all files,
            // even those which don't contain the variable.  This
            // makes this warning useless, so we omit it when called
            // for the file reader.
            if (called_from != FILEREADER_DATA) {
              CCTK_VWarn(2, __LINE__, __FILE__, CCTK_THORNSTRING,
                         "variable '%s' timelevel %d has not been read",
                         fullname, tl);
            }
          }
        } else {
          CCTK_VWarn(1, __LINE__, __FILE__, CCTK_THORNSTRING,
                     "variable '%s' timelevel %d has been read only partially",
                     fullname, tl);
          // for (int m = 0; m < maps; m++) {
          //   const int gindex = CCTK_GroupIndexFromVarI (vindex);
          //   cout << "Need: " << group_bboxes[gindex][m] << endl;
          //   cout << "Read: " << bboxes_read[vindex][tl][m] << endl;
          // }
          num_incomplete++;
        }
        free(fullname);
      }
    }
  }
  if (num_incomplete) {
    // cout.flush();
    // cerr.flush();
    CCTK_VWarn(0, __LINE__, __FILE__, CCTK_THORNSTRING,
               "%d variables on mglevel %d reflevel %d have been read "
               "only partially",
               num_incomplete, mglevel, reflevel);
  }

  {
    CCTK_REAL local[2], global[2];
    local[0] = io_files;
    local[1] = io_bytes;
    MPI_Allreduce(local, global, 2, dist::mpi_datatype(local[0]), MPI_SUM,
                  dist::comm());
    io_files = global[0];
    io_bytes = global[1];
  }
  EndTimingIO(cctkGH, io_files, io_bytes, true);

  // Synchronise all variables which have been read
  {
    vector<bool> dosync(CCTK_NumGroups(), false);
    for (size_t vindex = 0; vindex < read_completely.size(); vindex++) {
      if (CCTK_GroupTypeFromVarI(vindex) == CCTK_GF or reflevel == 0) {
        for (size_t tl = 0; tl < read_completely[vindex].size(); tl++) {
          // Do not synchronize variables that the user request not to be
          // read (only possible in FILEREADER mode)
          if (not(called_from == FILEREADER_DATA and
                  (ioUtilGH->do_inVars and not ioUtilGH->do_inVars[vindex]))) {
            if (read_completely[vindex][tl]) {
              int const gindex = CCTK_GroupIndexFromVarI(vindex);
              dosync.at(gindex) = true;
            }
          }
        }
      }
    }
    for (comm_state state; not state.done(); state.step()) {
      for (size_t group = 0; group < dosync.size(); ++group) {
        if (dosync.at(group)) {
          for (size_t m = 0; m < arrdata.at(group).size(); ++m) {
            arrdesc &ad = arrdata.at(group).at(m);
            for (int ml = 0; ml < ad.hh->mglevels(); ++ml) {
              for (int rl = 0; rl < ad.hh->reflevels(); ++rl) {
                for (size_t v = 0; v < ad.data.size(); ++v) {
                  ggf *const gf = ad.data.at(v);
                  for (int tl = 0; tl < gf->timelevels(ml, rl); ++tl) {
                    gf->sync_all(state, tl, rl, ml);
                  }
                }
              }
            }
          }
        }
      }
    }
  }
  if (error_count and abort_on_io_errors) {
    CCTK_WARN(
        0, "Found errors while trying to restart from checkpoint, aborting.");
  }

  if (in_recovery and not CCTK_Equals(verbose, "none")) {
    CCTK_VInfo(
        CCTK_THORNSTRING, "restarting simulation on mglevel %d reflevel %d "
                          "at iteration %d (simulation time %g)",
        mglevel, reflevel, cctkGH->cctk_iteration, (double)cctkGH->cctk_time);
  }

  return (0);
}

//////////////////////////////////////////////////////////////////////////////
// Open a set of checkpoint/filereader files
//////////////////////////////////////////////////////////////////////////////
static list<fileset_t>::iterator OpenFileSet(const cGH *const cctkGH,
                                             const string setname,
                                             const char *basefilename,
                                             int called_from) {
  file_t file;
  hid_t fapl_id;
  fileset_t fileset;
  int error_count = 0;
  list<string> filenames;
  DECLARE_CCTK_PARAMETERS;

  // first try to open a chunked file written on this processor
  // (note that dist::rank() cannot be called yet during RECOVER_PARAMETERS)
  fileset.first_ioproc = CCTK_MyProc(cctkGH);
  file.filename = IOUtil_AssembleFilename(NULL, basefilename, "", ".h5",
                                          called_from, fileset.first_ioproc, 0);
  assert(file.filename);
  file.indexfilename = IOUtil_AssembleFilename(
      NULL, basefilename, "", ".idx.h5", called_from, fileset.first_ioproc, 0);
  assert(file.indexfilename);

  // close all dangling HDF5 objects when file is closed
  HDF5_ERROR(fapl_id = H5Pcreate(H5P_FILE_ACCESS));
  HDF5_ERROR(H5Pset_fclose_degree(fapl_id, H5F_CLOSE_STRONG));

  // try to open the file (prevent HDF5 error messages if it fails)
  H5E_BEGIN_TRY {
    filenames.push_back(string(file.filename));
    file.file = H5Fopen(file.filename, H5F_ACC_RDONLY, fapl_id);
  }
  H5E_END_TRY;

  // if that failed, try a chunked file written on processor 0
  // (which always is an I/O proc)
  if (file.file < 0) {
    free(file.indexfilename);
    free(file.filename);
    fileset.first_ioproc = 0;
    file.filename = IOUtil_AssembleFilename(
        NULL, basefilename, "", ".h5", called_from, fileset.first_ioproc, 0);
    assert(file.filename);
    file.indexfilename =
        IOUtil_AssembleFilename(NULL, basefilename, "", ".idx.h5", called_from,
                                fileset.first_ioproc, 0);
    assert(file.indexfilename);
    H5E_BEGIN_TRY {
      filenames.push_back(string(file.filename));
      file.file = H5Fopen(file.filename, H5F_ACC_RDONLY, fapl_id);
    }
    H5E_END_TRY;
  }

  // if that still failed, try an unchunked file
  // (which is always written on processor 0)
  if (file.file < 0) {
    free(file.indexfilename);
    free(file.filename);
    file.filename = IOUtil_AssembleFilename(
        NULL, basefilename, "", ".h5", called_from, fileset.first_ioproc, 1);
    assert(file.filename);
    file.indexfilename =
        IOUtil_AssembleFilename(NULL, basefilename, "", ".idx.h5", called_from,
                                fileset.first_ioproc, 1);
    assert(file.indexfilename);
    H5E_BEGIN_TRY {
      filenames.push_back(string(file.filename));
      file.file = H5Fopen(file.filename, H5F_ACC_RDONLY, fapl_id);
    }
    H5E_END_TRY;
  }

  // return if no valid checkpoint could be found
  if (file.file < 0) {
    CCTK_VWarn(1, __LINE__, __FILE__, CCTK_THORNSTRING,
               "No valid HDF5 file with basename \"%s\" found", basefilename);
    free(file.filename);
    free(file.indexfilename);
    file.filename = NULL;
    file.indexfilename = NULL;
    file.file = -1;
    file.indexfile = -1; // to avoid trying to close it later
    HDF5_ERROR(H5Pclose(fapl_id));

    for (list<string>::const_iterator lsi = filenames.begin();
         lsi != filenames.end(); ++lsi) {
      CCTK_VWarn(2, __LINE__, __FILE__, CCTK_THORNSTRING,
                 "Tried filename \"%s\"", lsi->c_str());
    }
    return (filesets.end());
  }

  // open index file if it exists
  H5E_BEGIN_TRY {
    file.indexfile = H5Fopen(file.indexfilename, H5F_ACC_RDONLY, fapl_id);
  }
  H5E_END_TRY;
  fileset.has_index = file.indexfile >= 0;

  HDF5_ERROR(H5Pclose(fapl_id));

  if (CCTK_Equals(verbose, "full")) {
    CCTK_VInfo(CCTK_THORNSTRING, "opening %s file '%s'",
               called_from == CP_RECOVER_PARAMETERS ? "checkpoint" : "input",
               file.filename);
  }

  // read all the metadata information
  ReadMetadata(fileset, file);

  // recover parameters
  if (called_from == CP_RECOVER_PARAMETERS) {
    if (not CCTK_Equals(verbose, "none")) {
      CCTK_VInfo(CCTK_THORNSTRING, "Recovering parameters from checkpoint "
                                   "file '%s'",
                 file.filename);
    }
    hid_t dataset, datatype, memtype = -1;
    size_t len;
    htri_t old_data;

    HDF5_ERROR(dataset = H5Dopen(file.file, METADATA_GROUP "/" ALL_PARAMETERS));
    HDF5_ERROR(datatype = H5Dget_type(dataset));
    HDF5_ERROR(old_data = H5T_STRING != H5Tget_class(datatype));
    if (old_data) {
      hid_t dataspace;
      HDF5_ERROR(dataspace = H5Dget_space(dataset));
      /* old data that stored this as an array of chars */
      CCTK_WARN(CCTK_WARN_ALERT, "Old-style checkpoint data found.");
      HDF5_ERROR(len = H5Sget_simple_extent_npoints(dataspace) + 1);
      HDF5_ERROR(H5Sclose(dataspace));
      HDF5_ERROR(memtype = H5Tcopy(H5T_NATIVE_CHAR));
    } else {
      HDF5_ERROR(len = H5Tget_size(datatype));
      HDF5_ERROR(memtype = H5Tcopy(H5T_C_S1));
      HDF5_ERROR(H5Tset_size(memtype, len));
    }
    vector<char> parameter_buf(len);
    char *parameters = &parameter_buf[0];

    HDF5_ERROR(
        H5Dread(dataset, memtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, parameters));
    HDF5_ERROR(H5Dclose(dataset));
    HDF5_ERROR(H5Tclose(memtype));
    HDF5_ERROR(H5Tclose(datatype));
    IOUtil_SetAllParameters(parameters);

    num_reflevels = fileset.num_reflevels;
  }

  // allocate and initialise the list of input files for this set
  fileset.files.resize(fileset.nioprocs);
  for (int i = 0; i < fileset.nioprocs; i++) {
    fileset.files[i].file = -1;
    fileset.files[i].indexfile = -1;
    fileset.files[i].filename = NULL;
    fileset.files[i].indexfilename = NULL;
  }
  fileset.files[fileset.first_ioproc] = file;
  fileset.setname = setname;
  fileset.basefilename = basefilename;

  // add the new fileset to the list of sets and return an iterator to it
  filesets.push_front(fileset);
  return (filesets.begin());
}

//////////////////////////////////////////////////////////////////////////////
// Read the metadata for a set of input files
//////////////////////////////////////////////////////////////////////////////
static void ReadMetadata(fileset_t &fileset, const file_t &file) {
  int error_count = 0;
  DECLARE_CCTK_PARAMETERS;

  fileset.nioprocs = 1;
  hid_t metadata, attr; //, dataspace;
  H5E_BEGIN_TRY { metadata = H5Gopen(file.file, METADATA_GROUP); }
  H5E_END_TRY;
  if (metadata < 0) {
    // no metadata at all - this must be old-fashioned data output file
    return;
  }

  // in old times (before June 2005) the metadata attributes were attached
  // to the parameters dataset which was stored in checkpoint files only
  H5E_BEGIN_TRY { attr = H5Aopen_name(metadata, "nioprocs"); }
  H5E_END_TRY;
  const bool is_old_fashioned_file = attr < 0;
  if (is_old_fashioned_file) {
    HDF5_ERROR(H5Gclose(metadata));
    HDF5_ERROR(metadata =
                   H5Dopen(file.file, METADATA_GROUP "/" ALL_PARAMETERS));
  } else {
    HDF5_ERROR(H5Aread(attr, H5T_NATIVE_INT, &fileset.nioprocs));
    HDF5_ERROR(H5Aclose(attr));
  }
  HDF5_ERROR(attr = H5Aopen_name(metadata, "carpet_reflevels"));
  HDF5_ERROR(H5Aread(attr, H5T_NATIVE_INT, &fileset.num_reflevels));
  HDF5_ERROR(H5Aclose(attr));
  //   HDF5_ERROR (attr = H5Aopen_name (metadata, "numberofmgtimes"));
  //   HDF5_ERROR (H5Aread (attr, H5T_NATIVE_INT, &fileset.num_mglevels));
  //   HDF5_ERROR (H5Aclose (attr));
  HDF5_ERROR(attr = H5Aopen_name(metadata, "GH$iteration"));
  HDF5_ERROR(H5Aread(attr, H5T_NATIVE_INT, &fileset.cctk_iteration));
  HDF5_ERROR(H5Aclose(attr));
  HDF5_ERROR(attr = H5Aopen_name(metadata, "main loop index"));
  HDF5_ERROR(H5Aread(attr, H5T_NATIVE_INT, &fileset.main_loop_index));
  HDF5_ERROR(H5Aclose(attr));
  HDF5_ERROR(attr = H5Aopen_name(metadata, "carpet_global_time"));
  HDF5_ERROR(H5Aread(attr, H5T_NATIVE_DOUBLE, &fileset.global_time));
  HDF5_ERROR(H5Aclose(attr));
  HDF5_ERROR(attr = H5Aopen_name(metadata, "carpet_delta_time"));
  HDF5_ERROR(H5Aread(attr, H5T_NATIVE_DOUBLE, &fileset.delta_time));
  HDF5_ERROR(H5Aclose(attr));

  // Read grid structure if it is present
  hid_t dataset;
  H5E_BEGIN_TRY { dataset = H5Dopen(metadata, GRID_STRUCTURE); }
  H5E_END_TRY;
  if (dataset >= 0) {
    hid_t datatype, memtype = -1;
    htri_t old_data;
    size_t len;

    HDF5_ERROR(datatype = H5Dget_type(dataset));
    HDF5_ERROR(old_data = H5T_STRING != H5Tget_class(datatype));
    if (old_data) {
      hid_t dataspace;
      HDF5_ERROR(dataspace = H5Dget_space(dataset));
      /* old data that stored this as an array of chars */
      CCTK_WARN(CCTK_WARN_ALERT, "Old-style checkpoint data found.");
      HDF5_ERROR(len = H5Sget_simple_extent_npoints(dataspace) + 1);
      HDF5_ERROR(H5Sclose(dataspace));
      HDF5_ERROR(memtype = H5Tcopy(H5T_NATIVE_CHAR));
    } else {
      HDF5_ERROR(len = H5Tget_size(datatype));
      HDF5_ERROR(memtype = H5Tcopy(H5T_C_S1));
      HDF5_ERROR(H5Tset_size(memtype, len));
    }
    vector<char> gs_str_buf(len);
    char *gs_str = &gs_str_buf[0];

    HDF5_ERROR(
        H5Dread(dataset, memtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, gs_str));
    HDF5_ERROR(H5Dclose(dataset));
    HDF5_ERROR(H5Tclose(memtype));
    HDF5_ERROR(H5Tclose(datatype));
    istringstream gs_buf(gs_str);

    skipws(gs_buf);
    consume(gs_buf, "grid_superstructure:");
    skipws(gs_buf);
    gs_buf >> fileset.grid_superstructure;
    skipws(gs_buf);
    consume(gs_buf, ",");

    skipws(gs_buf);
    consume(gs_buf, "grid_structure:");
    skipws(gs_buf);
    gs_buf >> fileset.grid_structure;
    skipws(gs_buf);
    consume(gs_buf, ",");

    skipws(gs_buf);
    consume(gs_buf, "grid_times:");
    skipws(gs_buf);
    gs_buf >> fileset.grid_times;
    skipws(gs_buf);
    consume(gs_buf, ",");

    skipws(gs_buf);
    consume(gs_buf, "grid_delta_times:");
    skipws(gs_buf);
    gs_buf >> fileset.grid_delta_times;
    skipws(gs_buf);
    consume(gs_buf, ",");

    // skipws (gs_buf);
    // consume (gs_buf, "grid_leveltimes:");
    // skipws (gs_buf);
    // gs_buf >> fileset.leveltimes;
    // skipws (gs_buf);
    // consume (gs_buf, ",");

    skipws(gs_buf);
    consume(gs_buf, "grid_ghosts:");
    skipws(gs_buf);
    gs_buf >> fileset.grid_ghosts;
    skipws(gs_buf);
    consume(gs_buf, ",");

    skipws(gs_buf);
    consume(gs_buf, "grid_buffers:");
    skipws(gs_buf);
    gs_buf >> fileset.grid_buffers;
    skipws(gs_buf);
    consume(gs_buf, ",");

    skipws(gs_buf);
    consume(gs_buf, "grid_prolongation_orders:");
    skipws(gs_buf);
    gs_buf >> fileset.grid_prolongation_orders;
    skipws(gs_buf);
    consume(gs_buf, ".");
  }

  //   fileset.mgleveltimes.resize (fileset.num_mglevels *
  //   fileset.num_reflevels);
  //   for (int i = 0; i < fileset.num_mglevels; i++) {
  //     char buffer[32];

  //     snprintf (buffer, sizeof (buffer), "mgleveltimes %d", i);
  //     HDF5_ERROR (attr = H5Aopen_name (metadata, buffer));
  //     HDF5_ERROR (dataspace = H5Aget_space (attr));
  //     assert (H5Sget_simple_extent_npoints (dataspace) ==
  //     fileset.num_reflevels);
  //     HDF5_ERROR (H5Sclose (dataspace));
  //     HDF5_ERROR (H5Aread (attr, H5T_NATIVE_DOUBLE,
  //                          &fileset.mgleveltimes[i *
  //                          fileset.num_reflevels]));
  //     HDF5_ERROR (H5Aclose (attr));
  //   }

  if (is_old_fashioned_file) {
    HDF5_ERROR(H5Dclose(metadata));
  } else {
    HDF5_ERROR(H5Gclose(metadata));
  }
}

//////////////////////////////////////////////////////////////////////////////
// Browses through all the datasets in a checkpoint/data file
// and stores this information in a list of patch_t objects
//////////////////////////////////////////////////////////////////////////////
static herr_t BrowseDatasets(hid_t group, const char *objectname, void *args) {
  int error_count = 0;
  file_t *file = ((file_t **)args)[0];
  std::map<string, unsigned int> &alias =
      *((std::map<string, unsigned int> **)args)[1];
  patch_t patch;
  hid_t dataset, dataspace, attr, attrtype;
  H5G_stat_t object_info;

  // skip everything in the metadata group
  if (strcmp(objectname, METADATA_GROUP) == 0) {
    return (0);
  }
  // we are interested only in datasets
  HDF5_ERROR(H5Gget_objinfo(group, objectname, 0, &object_info));
  if (object_info.type != H5G_DATASET) {
    return (0);
  }
  HDF5_ERROR(dataset = H5Dopen(group, objectname));
  HDF5_ERROR(dataspace = H5Dget_space(dataset));
  patch.rank = H5Sget_simple_extent_ndims(dataspace);
  assert(patch.rank > 0 and patch.rank <= ::dim);
  patch.shape.resize(patch.rank);
  if (file->indexfile >= 0) { // index files store shape of master file in attr
    HDF5_ERROR(attr = H5Aopen_name(dataset, "h5shape"));
    HDF5_ERROR(H5Aread(attr, H5T_NATIVE_HSIZE, &patch.shape[0]));
    HDF5_ERROR(H5Aclose(attr));
  } else {
    HDF5_ERROR(H5Sget_simple_extent_dims(dataspace, &patch.shape[0], NULL));
  }
  HDF5_ERROR(H5Sclose(dataspace));
  HDF5_ERROR(attr = H5Aopen_name(dataset, "name"));
  HDF5_ERROR(attrtype = H5Aget_type(attr));
  size_t length = H5Tget_size(attrtype);
  assert(length > 0);
  // before carpet-6-init-413-g68ba001 we did not store the terminating NUL
  // in the attribute and therefore have to provide it here
  vector<char> varname(length + 1);
  HDF5_ERROR(H5Aread(attr, attrtype, &varname[0]));
  HDF5_ERROR(H5Tclose(attrtype));
  HDF5_ERROR(H5Aclose(attr));
  string key(&varname[0]);
  transform(key.begin(), key.end(), key.begin(), (int (*)(int))toupper);
  if (alias.count(key)) {
    patch.vindex = alias[key];
  } else {
    patch.vindex = CCTK_VarIndex(&varname[0]);
  }
  HDF5_ERROR(attr = H5Aopen_name(dataset, "carpet_mglevel"));
  HDF5_ERROR(H5Aread(attr, H5T_NATIVE_INT, &patch.mglevel));
  HDF5_ERROR(H5Aclose(attr));
  HDF5_ERROR(attr = H5Aopen_name(dataset, "level"));
  HDF5_ERROR(H5Aread(attr, H5T_NATIVE_INT, &patch.reflevel));
  HDF5_ERROR(H5Aclose(attr));
  HDF5_ERROR(attr = H5Aopen_name(dataset, "timestep"));
  HDF5_ERROR(H5Aread(attr, H5T_NATIVE_INT, &patch.timestep));
  HDF5_ERROR(H5Aclose(attr));
  HDF5_ERROR(attr = H5Aopen_name(dataset, "group_timelevel"));
  HDF5_ERROR(H5Aread(attr, H5T_NATIVE_INT, &patch.timelevel));
  HDF5_ERROR(H5Aclose(attr));
  HDF5_ERROR(attr = H5Aopen_name(dataset, "iorigin"));
  HDF5_ERROR(dataspace = H5Aget_space(attr));
  assert(H5Sget_simple_extent_npoints(dataspace) == patch.rank);
  HDF5_ERROR(H5Sclose(dataspace));
  patch.iorigin = 0;
  HDF5_ERROR(H5Aread(attr, H5T_NATIVE_INT, &patch.iorigin[0]));
  HDF5_ERROR(H5Aclose(attr));
  patch.ioffset = 0;
  H5E_BEGIN_TRY { attr = H5Aopen_name(dataset, "ioffset"); }
  H5E_END_TRY;
  // ioffset and ioffsetdenom may not be present; if so, use a default
  if (attr >= 0) {
    HDF5_ERROR(dataspace = H5Aget_space(attr));
    assert(H5Sget_simple_extent_npoints(dataspace) == patch.rank);
    HDF5_ERROR(H5Sclose(dataspace));
    HDF5_ERROR(H5Aread(attr, H5T_NATIVE_INT, &patch.ioffset[0]));
    HDF5_ERROR(H5Aclose(attr));
  }
  patch.ioffsetdenom = 1;
  H5E_BEGIN_TRY { attr = H5Aopen_name(dataset, "ioffsetdenom"); }
  H5E_END_TRY;
  if (attr >= 0) {
    HDF5_ERROR(dataspace = H5Aget_space(attr));
    assert(H5Sget_simple_extent_npoints(dataspace) == patch.rank);
    HDF5_ERROR(H5Sclose(dataspace));
    HDF5_ERROR(H5Aread(attr, H5T_NATIVE_INT, &patch.ioffsetdenom[0]));
    HDF5_ERROR(H5Aclose(attr));
  }
  HDF5_ERROR(H5Dclose(dataset));

  // try to obtain the map and component number from the patch's name
  // (cleaner way would be to store attributes with the dataset)
  patch.map = 0;
  const char *map_string = strstr(objectname, " m=");
  if (map_string) {
    sscanf(map_string, " m=%d", &patch.map);
  }
  patch.component = -1;
  const char *component_string = strstr(objectname, " c=");
  if (component_string) {
    sscanf(component_string, " c=%d", &patch.component);
  }

  // add this patch to our list
  if (patch.vindex >= 0 and patch.vindex < CCTK_NumVars()) {
    patch.patchname = objectname;
    file->patches.push_back(patch);
  } else {
    CCTK_VWarn(3, __LINE__, __FILE__, CCTK_THORNSTRING,
               "Ignoring dataset '%s' (invalid variable name)", objectname);
  }

  return (0);
}

//////////////////////////////////////////////////////////////////////////////
// Reads a single grid variable from a checkpoint/data file
//////////////////////////////////////////////////////////////////////////////
static int ReadVar(const cGH *const cctkGH, file_t &file, CCTK_REAL &io_bytes,
                   list<patch_t>::const_iterator patch,
                   vector<ibset> &bboxes_read, bool in_recovery) {
  int error_count = 0;
  DECLARE_CCTK_PARAMETERS;

  const int gindex = CCTK_GroupIndexFromVarI(patch->vindex);
  assert(gindex >= 0 and (unsigned int)gindex < Carpet::arrdata.size());
  cGroup group;
  CCTK_GroupData(gindex, &group);

  // grid arrays and scalars are read once on the coarsest reflevel
  assert(group.grouptype == CCTK_GF or (mglevel == 0 and reflevel == 0));

  // Check for storage
  if (not CCTK_QueryGroupStorageI(cctkGH, gindex)) {
    char *fullname = CCTK_FullName(patch->vindex);
    CCTK_VWarn(1, __LINE__, __FILE__, CCTK_THORNSTRING,
               "Cannot input variable '%s' (no storage)", fullname);
    free(fullname);
    return 1;
  }

  // filereader reads the current timelevel
  int timelevel = in_recovery ? patch->timelevel : 0;

  // get dataset dimensions
  assert(group.dim ==
         (group.grouptype == CCTK_SCALAR ? patch->rank - 1 : patch->rank));
  ivect shape(1);
  int elems = 1;
  for (int i = 0; i < patch->rank; i++) {
    shape[i] = patch->shape[patch->rank - i - 1];
    elems *= shape[i];
  }
  const hid_t datatype = CCTKtoHDF5_Datatype(cctkGH, group.vartype, 0);

  // Traverse all local components on all maps
  hid_t filespace = -1, dataset = -1;
  hid_t xfer = H5P_DEFAULT;
  H5Z_EDC_t checksums = H5Z_NO_EDC;

  BEGIN_LOCAL_MAP_LOOP(cctkGH, group.grouptype) {

    // skip this dataset if it belongs to another map
    if (group.grouptype == CCTK_GF and patch->map != Carpet::map) {
      continue;
    }

    // map patch into simulation grid, needs existing map
    const ivect stride = group.grouptype == CCTK_GF
                             ? arrdata.AT(gindex)
                                   .AT(patch->map)
                                   .hh->baseextent(mglevel, reflevel)
                                   .stride()
                             : 1;
    assert(all(stride % patch->ioffsetdenom == 0));
    ivect lower =
        patch->iorigin * stride + patch->ioffset * stride / patch->ioffsetdenom;
    ivect upper = lower + (shape - 1) * stride;

    struct arrdesc &data = arrdata.at(gindex).at(Carpet::map);

    BEGIN_LOCAL_COMPONENT_LOOP(cctkGH, group.grouptype) {

      // reset bounds for DISTRIB=CONST variables
      // these are handled by extending an extra dimension to nprocs elements.
      // All processes write the data of process zero into their checkpoint.
      // Here we ensure that any copy is acceptable to be read in again by
      // shifting the data that pretends to come from component 0 into out own
      // component.
      // We use the current array sizes so that the array size may change
      // during recovery.
      if (group.disttype == CCTK_DISTRIB_CONSTANT) {
        assert(group.grouptype == CCTK_SCALAR or group.grouptype == CCTK_ARRAY);
        const int dir = min(dim - 1, patch->rank == 0 ? 1 : patch->rank);
        const ibbox &exterior_membox = data.dd->light_boxes.at(mglevel)
                                           .at(reflevel)
                                           .at(component)
                                           .exterior;
        const ibbox &exterior0_membox =
            data.dd->light_boxes.at(mglevel).at(reflevel).at(0).exterior;
        const int newlower = lower[dir] + (exterior_membox.lower()[dir] -
                                           exterior0_membox.lower()[dir]);
        const int newupper = upper[dir] + (exterior_membox.lower()[dir] -
                                           exterior0_membox.lower()[dir]);
        lower[dir] = newlower;
        upper[dir] = newupper;
      }
      const ibbox filebox(lower, upper, stride);
// cout << "Found in file: " << filebox << endl;

#if 0
      const ibbox& interior_membox =
        data.dd->boxes.at(mglevel).at(reflevel).at(component).interior;

      // skip this dataset if it doesn't overlap with this component's interior
      const ibbox interior_overlap = interior_membox & filebox;
      if (interior_overlap.empty()) {
        continue;
      }
#endif
      const ibbox &exterior_membox =
          data.dd->light_boxes.at(mglevel).at(reflevel).at(component).exterior;
      int const var0 = CCTK_FirstVarIndexI(gindex);
      assert(var0 >= 0 and var0 < CCTK_NumVars());
      int const var = patch->vindex - var0;
      assert(var >= 0 and var < CCTK_NumVarsInGroupI(gindex));
      const ggf *const ff = data.data.AT(var);
      const gdata *const processor_component =
          ff->data_pointer(timelevel, reflevel, local_component, mglevel);

      // skip this dataset if it doesn't overlap with this component's
      // exterior, or if it only reads what has already been read
      const ibbox exterior_overlap = exterior_membox & filebox;
      const ibset exterior_overlaps =
          exterior_overlap - bboxes_read.at(Carpet::map);
      if (exterior_overlaps.empty()) {
        // cout << "Skipping this bbox" << endl;
        continue;
      }

      if (CCTK_Equals(verbose, "full")) {
        char *fullname = CCTK_FullName(patch->vindex);
        CCTK_VInfo(CCTK_THORNSTRING, "  reading '%s' from dataset '%s'",
                   fullname, patch->patchname.c_str());
        free(fullname);
      }

      // get the overlap with this component's exterior
      const ibbox &membox =
          data.dd->light_boxes.at(mglevel).at(reflevel).at(component).exterior;
      const ibbox overlap = membox & filebox;
      // cout << "Overlap: " << overlap << endl;
      bboxes_read.at(Carpet::map) |= overlap;
      // cout << "New read: " << bboxes_read.at(Carpet::map) << endl;

      // calculate hyperslab selection parameters

      // before HDF5-1.6.4 the H5Sselect_hyperslab() function expected
      // the 'start' argument to be of type 'hssize_t'
      slice_start_size_t memorigin[dim], fileorigin[dim];
      hsize_t memdims[dim], count[dim];
      for (int i = 0; i < patch->rank; i++) {
        assert(processor_component->shape()[i] ==
               (membox.upper() - membox.lower())[i] / stride[i] + 1);
        memdims[patch->rank - i - 1] = processor_component->padded_shape()[i];
        count[patch->rank - i - 1] =
            (overlap.upper() - overlap.lower())[i] / stride[i] + 1;
        memorigin[patch->rank - i - 1] =
            (overlap.lower() - membox.lower())[i] / stride[i];
        fileorigin[patch->rank - i - 1] =
            group.disttype == CCTK_DISTRIB_CONSTANT
                ? 0
                : (overlap.lower() - lower)[i] / stride[i];
      }

      // open file if opening was defered due to presence of index file
      if (file.file < 0) {
        hid_t fapl_id;
        HDF5_ERROR(fapl_id = H5Pcreate(H5P_FILE_ACCESS));
        HDF5_ERROR(H5Pset_fclose_degree(fapl_id, H5F_CLOSE_STRONG));
        HDF5_ERROR(file.file = H5Fopen(file.filename, H5F_ACC_RDONLY, fapl_id));
        HDF5_ERROR(H5Pclose(fapl_id));
        if (CCTK_Equals(verbose, "full")) {
          CCTK_VInfo(CCTK_THORNSTRING, "opening %s file '%s'",
                     in_recovery ? "checkpoint" : "input", file.filename);
        }
      }

      // open the dataset on the first time through
      if (dataset < 0) {
        HDF5_ERROR(dataset = H5Dopen(file.file, patch->patchname.c_str()));
        HDF5_ERROR(filespace = H5Dget_space(dataset));
        xfer = H5Pcreate(H5P_DATASET_XFER);
        checksums = H5Pget_edc_check(xfer);
        if (use_checksums && (checksums == H5Z_DISABLE_EDC))
          CCTK_WARN(1, "Checksums not enabled in HDF5 library, but "
                       "requested in parameter, reading data without "
                       "tests on checksums.");
        if (!use_checksums && (checksums == H5Z_ENABLE_EDC))
          H5Pset_edc_check(xfer, H5Z_DISABLE_EDC);
      }

      // read the hyperslab
      hid_t memspace;
      HDF5_ERROR(memspace = H5Screate_simple(patch->rank, memdims, NULL));
      HDF5_ERROR(H5Sselect_hyperslab(filespace, H5S_SELECT_SET, fileorigin,
                                     NULL, count, NULL));
      HDF5_ERROR(H5Sselect_hyperslab(memspace, H5S_SELECT_SET, memorigin, NULL,
                                     count, NULL));
      HDF5_ERROR(H5Dread(dataset, datatype, memspace, filespace, xfer,
                         cctkGH->data[patch->vindex][timelevel]));
      hid_t iodatatype;
      HDF5_ERROR(iodatatype = H5Dget_type(dataset));
      io_bytes += H5Sget_select_npoints(filespace) * H5Tget_size(iodatatype);
      HDF5_ERROR(H5Tclose(iodatatype));
      HDF5_ERROR(H5Sclose(memspace));
    }
    END_LOCAL_COMPONENT_LOOP;

    if (in_recovery) {
      if (group.grouptype != CCTK_GF) {
        for (int tl = 0; tl < data.tt->timelevels; ++tl) {
          data.tt->set_time(mglevel, reflevel, tl,
                            cctkGH->cctk_time - tl * cctkGH->cctk_delta_time);
        }
      }
    }
  }
  END_LOCAL_MAP_LOOP;

  if (dataset >= 0) {
    HDF5_ERROR(H5Dclose(dataset));
    HDF5_ERROR(H5Sclose(filespace));
    HDF5_ERROR(H5Pclose(xfer));
  }

  return error_count;
}

} // namespace CarpetIOHDF5

```

OutputSlice.cc:
```
#include <cassert>
#include <cctype>
#include <climits>
#include <cmath>
#include <cstdlib>
#include <cstring>

#include <map>
#include <string>

#include <cctk.h>
#include <util_Table.h>

#include "CactusBase/IOUtil/src/ioGH.h"

#include <Timer.hh>

#include "typeprops.hh"

#include "CarpetIOHDF5.hh"

// That's a hack
namespace Carpet {
void UnsupportedVarType(const int vindex);
}

#define GetParameter(parameter)                                                \
  outdim == 0 ? out0D_##parameter : outdim == 1                                \
                                        ? out1D_##parameter                    \
                                        : outdim == 2 ? out2D_##parameter      \
                                                      : out3D_##parameter

namespace CarpetIOHDF5 {

using namespace std;
using namespace Carpet;

// routines which are independent of the output dimension
static ibset GetOutputBBoxes(const cGH *cctkGH, int group, int rl, int m, int c,
                             const ibbox &ext, const ibset &allactive);

static void GetCoordinates(const cGH *cctkGH, int m, const cGroup &groupdata,
                           const ibset &exts, CCTK_REAL &coord_time,
                           vector<rvect> &coord_lower,
                           vector<rvect> &coord_upper);

static int GetGridOffset(const cGH *cctkGH, int m, int dir, const char *itempl,
                         const char *iglobal, const char *ctempl,
                         const char *cglobal, CCTK_REAL cfallback);
static int CoordToOffset(const cGH *cctkGH, int m, int dir, CCTK_REAL coord,
                         int ifallback);

// IO processor
template <int outdim> int IOHDF5<outdim>::ioproc;
template <int outdim> int IOHDF5<outdim>::nioprocs;
template <int outdim> int IOHDF5<outdim>::ioproc_every;

// Global configuration parameters
bool stop_on_parse_errors = false;

// Definition of static members
template <int outdim> char *IOHDF5<outdim>::my_out_slice_dir;
template <int outdim> char *IOHDF5<outdim>::my_out_slice_vars;
template <int outdim> vector<ioRequest *> IOHDF5<outdim>::slice_requests;

template <int outdim> int IOHDF5<outdim>::Startup() {
  ostringstream msg;
  msg << "AMR " << outdim << "D HDF5 I/O provided by CarpetIOHDF5";
  CCTK_RegisterBanner(msg.str().c_str());

  ostringstream extension_name;
  extension_name << "CarpetIOHDF5_" << outdim << "D";
  const int GHExtension =
      CCTK_RegisterGHExtension(extension_name.str().c_str());
  CCTK_RegisterGHExtensionSetupGH(GHExtension, SetupGH);

  ostringstream method_name;
  method_name << "IOHDF5_" << outdim << "D";
  const int IOMethod = CCTK_RegisterIOMethod(method_name.str().c_str());
  CCTK_RegisterIOMethodOutputGH(IOMethod, OutputGH);
  CCTK_RegisterIOMethodOutputVarAs(IOMethod, OutputVarAs);
  CCTK_RegisterIOMethodTimeToOutput(IOMethod, TimeToOutput);
  CCTK_RegisterIOMethodTriggerOutput(IOMethod, TriggerOutput);

  return 0;
}

template <int outdim>
void *IOHDF5<outdim>::SetupGH(tFleshConfig *const fc, const int convLevel,
                              cGH *const cctkGH) {
  DECLARE_CCTK_PARAMETERS;
  const void *dummy;

  dummy = &fc;
  dummy = &convLevel;
  dummy = &cctkGH;
  dummy = &dummy;

  if (not CCTK_Equals(verbose, "none")) {
    CCTK_VWarn(CCTK_WARN_ALERT, __LINE__, __FILE__, CCTK_THORNSTRING,
               "I/O Method 'IOHDF5_%dD' registered: "
               "%dD AMR output of grid variables to HDF5 files",
               outdim, outdim);
  }

  const int numvars = CCTK_NumVars();
  slice_requests.resize(numvars);

  // initial I/O parameter check
  my_out_slice_dir = 0;
  my_out_slice_vars = strdup("");
  stop_on_parse_errors = strict_io_parameter_check != 0;
  CheckSteerableParameters(cctkGH);
  stop_on_parse_errors = false;

  // We register only once, ergo we get only one handle.  We store
  // that statically, so there is no need to pass anything to
  // Cactus.
  return NULL;
}

template <int outdim>
void IOHDF5<outdim>::CheckSteerableParameters(const cGH *const cctkGH) {
  DECLARE_CCTK_PARAMETERS;

  // re-parse the 'IOHDF5::out%dD_dir' parameter if it has changed
  const char *the_out_dir = GetParameter(dir);
  if (CCTK_EQUALS(the_out_dir, "")) {
    the_out_dir = io_out_dir;
  }

  if (not my_out_slice_dir or strcmp(the_out_dir, my_out_slice_dir)) {
    free(my_out_slice_dir);
    my_out_slice_dir = strdup(the_out_dir);

    // create the output directory
    const int result = IOUtil_CreateDirectory(cctkGH, my_out_slice_dir, 0, 0);
    if (result < 0) {
      CCTK_VWarn(1, __LINE__, __FILE__, CCTK_THORNSTRING,
                 "Problem creating %dD-output directory '%s'", outdim,
                 my_out_slice_dir);
    } else if (result > 0 and CCTK_Equals(verbose, "full")) {
      CCTK_VInfo(CCTK_THORNSTRING, "%dD-output directory '%s' already exists",
                 outdim, my_out_slice_dir);
    }
  }

  // re-parse the 'IOHDF5::out%d_vars' parameter if it has changed
  const char *const out_slice_vars = GetParameter(vars);
  if (strcmp(out_slice_vars, my_out_slice_vars)) {
    ostringstream parameter_name;
    parameter_name << "IOHDF5::out" << outdim << "D_vars";
#ifdef IOUTIL_PARSER_HAS_OUT_DT
    IOUtil_ParseVarsForOutput(
        cctkGH, CCTK_THORNSTRING, parameter_name.str().c_str(),
        stop_on_parse_errors, out_slice_vars, -1, -1.0, &slice_requests[0]);
#else
    IOUtil_ParseVarsForOutput(
        cctkGH, CCTK_THORNSTRING, parameter_name.str().c_str(),
        stop_on_parse_errors, out_slice_vars, -1, &slice_requests[0]);
#endif

    // notify the user about the new setting
    if (not CCTK_Equals(verbose, "none")) {
      int count = 0;
      ostringstream msg;
      msg << "Periodic " << outdim << "D AMR output requested for:";
      for (int vi = 0; vi < CCTK_NumVars(); ++vi) {
        if (slice_requests.at(vi)) {
          ++count;
          char *const fullname = CCTK_FullName(vi);
          msg << "\n"
              << "   " << fullname;
          free(fullname);
        }
      }
      if (count > 0) {
        CCTK_INFO(msg.str().c_str());
      }
    }

    // save the last setting of 'IOHDF5::out%d_vars' parameter
    free(my_out_slice_vars);
    my_out_slice_vars = strdup(out_slice_vars);
  }

  // copy ioprocs and ioproc ot
  if (outdim == 3) { // only 3D output splits files
    const ioGH *IO = (ioGH *)CCTK_GHExtension(cctkGH, "IO");
    assert(IO);
    nioprocs = IO->nioprocs;
    ioproc = IO->ioproc;
    ioproc_every = IO->ioproc_every;
    // cout << "nioprocs: " << nioprocs << " ioproc: " << ioproc << endl;
  } else {
    nioprocs = 1;
    ioproc = 0;
    ioproc_every = dist::size();
  }
}

template <int outdim> int IOHDF5<outdim>::OutputGH(const cGH *const cctkGH) {
  ostringstream timer_name;
  timer_name << "OutputGH<" << outdim << ">";

  Timers::Timer timer(timer_name.str());

  timer.start();

  CheckSteerableParameters(cctkGH);
  if (strcmp(my_out_slice_vars, "")) {
    for (int vi = 0; vi < CCTK_NumVars(); ++vi) {
      if (TimeToOutput(cctkGH, vi)) {
        TriggerOutput(cctkGH, vi);
      }
    }
  }
  timer.stop();

  return 0;
}

template <int outdim>
int IOHDF5<outdim>::TimeToOutput(const cGH *const cctkGH, const int vindex) {
  DECLARE_CCTK_ARGUMENTS;
  DECLARE_CCTK_PARAMETERS;

  assert(vindex >= 0 and vindex < CCTK_NumVars());

  if (CCTK_GroupTypeFromVarI(vindex) != CCTK_GF and not do_global_mode) {
    return 0;
  }

  // check if output for this variable was requested
  if (not slice_requests.at(vindex)) {
    return 0;
  }

  // check whether this refinement level should be output
  if (not(slice_requests.at(vindex)->refinement_levels & (1 << reflevel))) {
    return 0;
  }

  // check if output for this variable was requested individually by
  // a "<varname>{ out_every = <number> }" option string
  // this will overwrite the output criterion setting
  const char *myoutcriterion = GetParameter(criterion);
  if (CCTK_EQUALS(myoutcriterion, "default")) {
    myoutcriterion = io_out_criterion;
  }
  if (slice_requests.at(vindex)->out_every >= 0) {
    myoutcriterion = "divisor";
  }

  if (CCTK_EQUALS(myoutcriterion, "never")) {
    return 0;
  }

  // check whether to output at this iteration
  bool output_this_iteration = false;

  if (CCTK_EQUALS(myoutcriterion, "iteration")) {
    int myoutevery = GetParameter(every);
    if (myoutevery == -2) {
      myoutevery = io_out_every;
    }
    if (myoutevery > 0) {
      if (cctk_iteration == this_iteration_slice[outdim]) {
        // we already decided to output this iteration
        output_this_iteration = true;
      } else if (cctk_iteration >=
                 last_output_iteration_slice[outdim] + myoutevery) {
        // it is time for the next output
        output_this_iteration = true;
        last_output_iteration_slice[outdim] = cctk_iteration;
        this_iteration_slice[outdim] = cctk_iteration;
      }
    }
  } else if (CCTK_EQUALS(myoutcriterion, "divisor")) {
    int myoutevery = GetParameter(every);
    if (myoutevery == -2) {
      myoutevery = io_out_every;
    }
    if (slice_requests.at(vindex)->out_every >= 0) {
      myoutevery = slice_requests.at(vindex)->out_every;
    }
    if (myoutevery > 0 and (cctk_iteration % myoutevery) == 0) {
      // we already decided to output this iteration
      output_this_iteration = true;
    }
  } else if (CCTK_EQUALS(myoutcriterion, "time")) {
    CCTK_REAL myoutdt = GetParameter(dt);
    if (myoutdt == -2) {
      myoutdt = io_out_dt;
    }
    if (myoutdt == 0 or cctk_iteration == this_iteration_slice[outdim]) {
      output_this_iteration = true;
    } else if (myoutdt > 0) {
      int do_output =
          cctk_time / cctk_delta_time >=
          (last_output_time_slice[outdim] + myoutdt) / cctk_delta_time -
              1.0e-12;
      MPI_Bcast(&do_output, 1, MPI_INT, 0, dist::comm());
      if (do_output) {
        // it is time for the next output
        output_this_iteration = true;
        last_output_time_slice[outdim] = cctk_time;
        this_iteration_slice[outdim] = cctk_iteration;
      }
    }
  } // select output criterion

  return output_this_iteration ? 1 : 0;
}

template <int outdim>
int IOHDF5<outdim>::TriggerOutput(const cGH *const cctkGH, const int vindex) {
  DECLARE_CCTK_PARAMETERS;

  assert(vindex >= 0 and vindex < CCTK_NumVars());

  char *const fullname = CCTK_FullName(vindex);

  int retval;
  if (one_file_per_group) {
    char *const alias_c = CCTK_GroupNameFromVarI(vindex);
    string alias(alias_c);
    free(alias_c);
    transform(alias.begin(), alias.end(), alias.begin(), ::tolower);
    string const oldsep("::");
    size_t const oldseppos = alias.find(oldsep);
    assert(oldseppos != string::npos);
    alias.replace(oldseppos, oldsep.size(), out_group_separator);
    retval = OutputVarAs(cctkGH, fullname, alias.c_str());
  } else {
    const char *const alias = CCTK_VarName(vindex);
    retval = OutputVarAs(cctkGH, fullname, alias);
  }

  free(fullname);

  return retval;
}

static void GetVarIndex(const int vindex, const char *const optstring,
                        void *const arg) {
  if (optstring) {
    char *const fullname = CCTK_FullName(vindex);
    CCTK_VWarn(2, __LINE__, __FILE__, CCTK_THORNSTRING,
               "Option string '%s' will be ignored for HDF5 output of "
               "variable '%s'",
               optstring, fullname);
    free(fullname);
  }

  *static_cast<int *>(arg) = vindex;
}

template <int outdim>
int IOHDF5<outdim>::OutputVarAs(const cGH *const cctkGH,
                                const char *const varname,
                                const char *const alias) {
  DECLARE_CCTK_PARAMETERS;

  int vindex = -1;

  if (CCTK_TraverseString(varname, GetVarIndex, &vindex, CCTK_VAR) < 0) {
    CCTK_VWarn(1, __LINE__, __FILE__, CCTK_THORNSTRING,
               "error while parsing variable name '%s' (alias name '%s')",
               varname, alias);
    return -1;
  }

  if (vindex < 0) {
    return -1;
  }

  if (not(is_level_mode() or (is_singlemap_mode() and maps == 1) or
          (is_local_mode() and maps == 1 and
           vhh.at(Carpet::map)->local_components(reflevel) == 1))) {
    CCTK_WARN(1, "OutputVarAs must be called in level mode");
    return -1;
  }

  BEGIN_LEVEL_MODE(cctkGH) {

    // Get information
    const int group = CCTK_GroupIndexFromVarI(vindex);
    assert(group >= 0);
    cGroup groupdata;
    {
      int const ierr = CCTK_GroupData(group, &groupdata);
      assert(not ierr);
    }

    // Check information
    if (groupdata.grouptype != CCTK_GF) {
      assert(do_global_mode);
    }

    if (outdim > groupdata.dim) {
      CCTK_VWarn(
          1, __LINE__, __FILE__, CCTK_THORNSTRING,
          "Cannot produce %dD slice HDF5 output file '%s' for variable '%s' "
          "because it has only %d dimensions",
          outdim, alias, varname, groupdata.dim);
      return -1;
    }

    // Check for storage
    if (not CCTK_QueryGroupStorageI(cctkGH, group)) {
      // This may be okay if storage is e.g. scheduled only in the
      // analysis bin
      CCTK_VWarn(4, __LINE__, __FILE__, CCTK_THORNSTRING,
                 "Cannot output variable '%s' because it has no storage",
                 varname);
      return 0;
    }

    ostringstream basefilenamebuf;
    basefilenamebuf << my_out_slice_dir << "/" << alias;
    const string basefilename = basefilenamebuf.str();

    // Check if the file has been created already
    bool is_new_file, truncate_file;
    const bool did_output =
        DidOutput(cctkGH, vindex, basefilename, is_new_file, truncate_file);
    if (did_output) {
      return 0;
    }

    // Loop over all direction combinations
    vect<int, outdim> dirs(0);
    bool done;
    do {

      // Output each combination only once
      bool ascending = true;
      for (int d1 = 0; d1 < outdim; ++d1) {
        for (int d2 = d1 + 1; d2 < outdim; ++d2) {
          ascending = ascending and dirs[d1] < dirs[d2];
        }
      }

      // Skip output if the dimensions are not ascending
      if (ascending) {

        // Skip output if not requested
        if (DirectionIsRequested(dirs)) {
          OutputDirection(cctkGH, vindex, alias, basefilename, dirs,
                          is_new_file, truncate_file);
        }

      } // if ascending

      // Next direction combination
      done = true;
      for (int d = 0; d < outdim; ++d) {
        ++dirs[d];
        if (dirs[d] < groupdata.dim + (outdim == 1 ? 1 : 0)) {
          done = false;
          break;
        }
        dirs[d] = 0;
      }

    } while (not done); // output all directions
  }
  END_LEVEL_MODE;

  return 0;
}

// Traverse all maps and components on this refinement and multigrid
// level
template <int outdim>
void IOHDF5<outdim>::OutputDirection(const cGH *const cctkGH, const int vindex,
                                     const string alias,
                                     const string basefilename,
                                     const vect<int, outdim> &dirs,
                                     const bool is_new_file,
                                     const bool truncate_file) {
  DECLARE_CCTK_PARAMETERS;

  // Get information
  const int group = CCTK_GroupIndexFromVarI(vindex);
  assert(group >= 0);
  const int vindex0 = CCTK_FirstVarIndexI(group);
  assert(vindex0 >= 0 and vindex >= vindex0);
  const int var = vindex - vindex0;
  cGroup groupdata;
  {
    int const ierr = CCTK_GroupData(group, &groupdata);
    assert(not ierr);
  }

  const int ml = groupdata.grouptype == CCTK_GF ? mglevel : 0;
  const int rl = groupdata.grouptype == CCTK_GF ? reflevel : 0;

  const int num_tl = CCTK_MaxActiveTimeLevelsVI(cctkGH, vindex);
  assert(num_tl >= 1);

  const int numvars = one_file_per_group ? CCTK_NumVarsInGroupI(group) : 1;

  // Loop over all maps
  const int m_min = 0;
  const int m_max = groupdata.grouptype == CCTK_GF ? Carpet::maps : 1;
  for (int m = m_min; m < m_max; ++m) {

    hid_t file = -1, index_file = -1;
    int error_count = 0;
    error_count += OpenFile(cctkGH, m, vindex, numvars, alias, basefilename,
                            dirs, is_new_file, truncate_file, file, index_file);

    // Find the output offset
    const ivect offset =
        groupdata.grouptype == CCTK_GF ? GetOutputOffset(cctkGH, m, dirs) : 0;

    const gh *const hh = arrdata.at(group).at(m).hh;
    const dh *const dd = arrdata.at(group).at(m).dd;

    // re-compute the active (non-buffered) region
    ibset allactive;
    GetAllActive(dd, hh, ml, rl, allactive);

    // Traverse all components on this multigrid level, refinement
    // level, and map
    const int c_min = 0;
    const int c_max = groupdata.grouptype == CCTK_GF
                          ? vhh.at(m)->components(reflevel)
                          : groupdata.disttype != CCTK_DISTRIB_CONSTANT
                                ? CCTK_nProcs(cctkGH)
                                : 1;
    for (int c = c_min, c_base = c_min; c < c_max; ++c) {
      int const lc = hh->get_local_component(rl, c);
      int const proc = hh->processor(rl, c);
      const ibbox &data_ext = dd->light_boxes.at(ml).at(rl).at(c).exterior;
      const ibset exts =
          GetOutputBBoxes(cctkGH, group, rl, m, c, data_ext, allactive);
      // we have to take part in this output if we either own data to be
      // output or are the ioproc in the same group of processors as the
      // dataholder
      if (dist::rank() == proc or dist::rank() == IOProcForProc(proc)) {

        CCTK_REAL coord_time;
        vector<rvect> coord_lower, coord_upper;
        GetCoordinates(cctkGH, m, groupdata, exts, coord_time, coord_lower,
                       coord_upper);

        // Apply offset
        vector<ivect> offsets1;
        offsets1.reserve(exts.setsize());
        for (ibset::const_iterator ext = exts.begin(); ext != exts.end();
             ++ext) {
          ivect offset1;
          if (groupdata.grouptype == CCTK_GF) {
            const ibbox &baseext = hh->baseextents.at(ml).at(rl);
            offset1 = baseext.lower() + offset * ext->stride();
          } else {
            offset1 = offset * ext->stride();
          }
          for (int d = 0; d < outdim; ++d) {
            if (dirs[d] < 3) {
              offset1[dirs[d]] = ext->lower()[dirs[d]];
            }
          }
          offsets1.push_back(offset1);
        }

        const int tl_min = 0;
        const int tl_max = output_all_timelevels ? num_tl : 1;
        for (int tl = tl_min; tl < tl_max; ++tl) {

          mempool pool;

          const int n_min = one_file_per_group ? 0 : var;
          const int n_max =
              one_file_per_group ? CCTK_NumVarsInGroupI(group) : var + 1;
          vector<const gdata *> datas(n_max - n_min);
          for (size_t n = 0; n < datas.size(); ++n) {
            if (dist::rank() == proc) {
              const ggf *const ff = arrdata.at(group).at(m).data.at(n + n_min);
              datas.at(n) = ff->data_pointer(tl, rl, lc, ml);
            } else {
              datas.at(n) = NULL;
            }
          }

          vector<gdata *> tmpdatas(datas.size());

          // tranfer data through the interconnect to ioproc if necessary
          if (proc != ioproc) {

            for (size_t n = 0; n < datas.size(); ++n) {
              const ggf *const ff = arrdata.at(group).at(m).data.at(n + n_min);
              tmpdatas.at(n) = ff->new_typed_data();
              size_t const memsize =
                  tmpdatas.at(n)->allocsize(data_ext, ioproc);
              void *const memptr = pool.alloc(memsize);
              tmpdatas.at(n)->allocate(data_ext, ioproc, memptr, memsize);
            } // for n

            for (comm_state state; not state.done(); state.step()) {
              for (size_t n = 0; n < datas.size(); ++n) {
                tmpdatas.at(n)->copy_from(state, datas.at(n), data_ext,
                                          data_ext, NULL, ioproc, proc);
              }
            }

          } else {

            for (size_t n = 0; n < datas.size(); ++n) {
              tmpdatas.at(n) = const_cast<gdata *>(datas.at(n));
            }
          }

          if (dist::rank() == IOProcForProc(proc)) {
            int c_offset = 0;
            for (ibset::const_iterator ext = exts.begin(); ext != exts.end();
                 ++ext, ++c_offset) {
              error_count += WriteHDF5(
                  cctkGH, file, index_file, tmpdatas, *ext, vindex,
                  offsets1[c_offset], dirs, rl, ml, m, c, c_base + c_offset, tl,
                  coord_time, coord_lower[c_offset], coord_upper[c_offset]);
            }
          }

          if (proc != ioproc) {
            for (size_t n = 0; n < tmpdatas.size(); ++n) {
              delete tmpdatas.at(n);
            }
          }

        } // for tl
      }

      c_base += exts.setsize();
    } // for c

    error_count += CloseFile(cctkGH, file, index_file);
    if (error_count > 0 and abort_on_io_errors) {
      CCTK_ERROR("Aborting simulation due to previous I/O errors");
    }

  } // for m
}

template <int outdim>
bool IOHDF5<outdim>::DidOutput(const cGH *const cctkGH, const int vindex,
                               const string basefilename, bool &is_new_file,
                               bool &truncate_file) {
  DECLARE_CCTK_PARAMETERS;

  typedef std::map<string, vector<vector<vector<int> > > > filelist;
  static filelist created_files;

  filelist::iterator thisfile = created_files.find(basefilename);
  is_new_file = thisfile == created_files.end();
  truncate_file = is_new_file and IO_TruncateOutputFiles(cctkGH);

  if (is_new_file) {
    const int numelems = one_file_per_group ? CCTK_NumGroups() : CCTK_NumVars();
    vector<vector<vector<int> > > last_outputs; // [ml][rl][var]
    last_outputs.resize(mglevels);
    for (int ml = 0; ml < mglevels; ++ml) {
      last_outputs.at(ml).resize(maxreflevels);
      for (int rl = 0; rl < maxreflevels; ++rl) {
        last_outputs.at(ml).at(rl).resize(numelems, cctkGH->cctk_iteration - 1);
      }
    }
    // TODO: this makes a copy of last_outputs, which is expensive;
    // change this to use a reference instead
    thisfile = created_files.insert(
        thisfile, filelist::value_type(basefilename, last_outputs));
    assert(thisfile != created_files.end());
  }

  // Check if this variable has been output already during this
  // iteration
  const int elem =
      one_file_per_group ? CCTK_GroupIndexFromVarI(vindex) : vindex;
  int &last_output = thisfile->second.at(mglevel).at(reflevel).at(elem);
  if (last_output == cctkGH->cctk_iteration) {
    // has already been output during this iteration
    char *const fullname = CCTK_FullName(vindex);
    CCTK_VWarn(5, __LINE__, __FILE__, CCTK_THORNSTRING,
               "Skipping output for variable '%s', because this variable "
               "has already been output during the current iteration -- "
               "probably via a trigger during the analysis stage",
               fullname);
    free(fullname);
    return true;
  }
  assert(last_output < cctkGH->cctk_iteration);
  last_output = cctkGH->cctk_iteration;

  return false;
}

CCTK_REAL io_files;
CCTK_REAL io_bytes;

template <int outdim>
int IOHDF5<outdim>::OpenFile(const cGH *const cctkGH, const int m,
                             const int vindex, const int numvars,
                             const string alias, const string basefilename,
                             const vect<int, outdim> &dirs,
                             const bool is_new_file, const bool truncate_file,
                             hid_t &file, hid_t &index_file) {
  DECLARE_CCTK_PARAMETERS;

  int error_count = 0;

  BeginTimingIO(cctkGH);
  io_files = 0;
  io_bytes = 0;

  if (dist::rank() == ioproc) {

    const int grouptype = CCTK_GroupTypeFromVarI(vindex);
    assert(grouptype >= 0);

    // Invent a file name
    ostringstream filenamebuf;
    filenamebuf << basefilename;
    if (maps > 1 and grouptype == CCTK_GF) {
      filenamebuf << "." << m;
    }
    // historically 3d output files do not carry a label so the files created
    // here do not conflict with the old-style output files
    filenamebuf << ".";
    for (int d = 0; d < outdim; ++d) {
      const char *const coords = "xyzd";
      filenamebuf << coords[dirs[d]];
    }
    if (nioprocs > 1) {
      filenamebuf << ".file_" << dist::rank();
    }
    string index_filename(filenamebuf.str() + ".idx" + out_extension);
    filenamebuf << out_extension;

    // we need a persistent temporary here
    const string filenamestr = filenamebuf.str();
    const char *const filename = filenamestr.c_str();

    // Open the file
    bool file_exists = false;
    if (not truncate_file) {
      H5E_BEGIN_TRY { file_exists = H5Fis_hdf5(filename) > 0; }
      H5E_END_TRY;
    }

    if (truncate_file or not file_exists) {
      hid_t fapl_id;
      HDF5_ERROR(fapl_id = H5Pcreate(H5P_FILE_ACCESS));
      HDF5_ERROR(H5Pset_fclose_degree(fapl_id, H5F_CLOSE_STRONG));
      HDF5_ERROR(file =
                     H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id));
      if (output_index) {
        HDF5_ERROR(index_file = H5Fcreate(index_filename.c_str(), H5F_ACC_TRUNC,
                                          H5P_DEFAULT, fapl_id));
      }
      HDF5_ERROR(H5Pclose(fapl_id));
      // write metadata information
      error_count +=
          WriteMetadata(cctkGH, nioprocs, vindex, numvars, false, file);

      if (output_index) {
        error_count +=
            WriteMetadata(cctkGH, nioprocs, vindex, numvars, false, index_file);
      }
    } else {
      hid_t fapl_id;
      HDF5_ERROR(fapl_id = H5Pcreate(H5P_FILE_ACCESS));
      HDF5_ERROR(H5Pset_fclose_degree(fapl_id, H5F_CLOSE_STRONG));
      HDF5_ERROR(file = H5Fopen(filename, H5F_ACC_RDWR, fapl_id));
      if (output_index)
        HDF5_ERROR(index_file =
                       H5Fopen(index_filename.c_str(), H5F_ACC_RDWR, fapl_id));
      HDF5_ERROR(H5Pclose(fapl_id));
    }
    io_files += 1;

  } // if on the I/O processor

  return error_count;
}

template <int outdim>
int IOHDF5<outdim>::CloseFile(const cGH *const cctkGH, hid_t &file,
                              hid_t &index_file) {
  DECLARE_CCTK_PARAMETERS;

  int error_count = 0;

  if (dist::rank() == ioproc) {
    if (file >= 0) {
      HDF5_ERROR(H5Fclose(file));
    }
    if (output_index and index_file >= 0) {
      HDF5_ERROR(H5Fclose(index_file));
    }
    HDF5_ERROR(H5garbage_collect());
  }
  if (nioprocs > 1) {
    CCTK_REAL local[2], global[2];
    local[0] = io_files;
    local[1] = io_bytes;
    MPI_Allreduce(local, global, 2, dist::mpi_datatype(local[0]), MPI_SUM,
                  dist::comm());
    io_files = global[0];
    io_bytes = global[1];
  }

  EndTimingIO(cctkGH, io_files, io_bytes, true);

  return error_count;
}

// Check whether this output direction has been requested
template <int outdim>
bool IOHDF5<outdim>::DirectionIsRequested(const vect<int, outdim> &dirs) {
  DECLARE_CCTK_PARAMETERS;

  switch (outdim) {

  case 0:
    // Output is always requested (if switched on)
    return true;

  case 1:
    switch (dirs[0]) {
    case 0:
      return out1D_x;
    case 1:
      return out1D_y;
    case 2:
      return out1D_z;
    case 3:
      return out1D_d;
    default:
      assert(0);
    }

  case 2:
    if (dirs[0] == 0 and dirs[1] == 1)
      return out2D_xy;
    if (dirs[0] == 0 and dirs[1] == 2)
      return out2D_xz;
    if (dirs[0] == 1 and dirs[1] == 2)
      return out2D_yz;
    assert(0);

  case 3:
    // Output is always requested (if switched on)
    return true;

  default:
    assert(0);
    // Prevent compiler warning about missing return statement
    return false;
  }
}

// Get the region that should be output, in terms of grid points;
// this is the offset perpendicular to the output hyperslab
template <int outdim>
ivect IOHDF5<outdim>::GetOutputOffset(const cGH *const cctkGH, const int m,
                                      const vect<int, outdim> &dirs) {
  DECLARE_CCTK_PARAMETERS;

  // Default is zero
  ivect offset(0);

  switch (outdim) {

  case 0:
    // 0D output
    offset[0] =
        GetGridOffset(cctkGH, m, 1, "out0D_point_xi", /*"out_point_xi"*/ NULL,
                      "out0D_point_x", /*"out_point_x"*/ NULL,
                      /*out_point_x*/ 0.0);
    offset[1] =
        GetGridOffset(cctkGH, m, 2, "out0D_point_yi", /*"out_point_yi"*/ NULL,
                      "out0D_point_y", /*"out_point_y"*/ NULL,
                      /*out_point_y*/ 0.0);
    offset[2] =
        GetGridOffset(cctkGH, m, 3, "out0D_point_zi", /*"out_point_zi"*/ NULL,
                      "out0D_point_z", /*"out_point_z"*/ NULL,
                      /*out_point_z*/ 0.0);
    break;

  case 1:
    // 1D output
    switch (dirs[0]) {
    case 0:
      offset[1] = GetGridOffset(cctkGH, m, 2, "out1D_xline_yi", "out_xline_yi",
                                "out1D_xline_y", "out_xline_y", out_xline_y);
      offset[2] = GetGridOffset(cctkGH, m, 3, "out1D_xline_zi", "out_xline_zi",
                                "out1D_xline_z", "out_xline_z", out_xline_z);
      break;
    case 1:
      offset[0] = GetGridOffset(cctkGH, m, 1, "out1D_yline_xi", "out_yline_xi",
                                "out1D_yline_x", "out_yline_x", out_yline_x);
      offset[2] = GetGridOffset(cctkGH, m, 3, "out1D_yline_zi", "out_yline_zi",
                                "out1D_yline_z", "out_yline_z", out_yline_z);
      break;
    case 2:
      offset[0] = GetGridOffset(cctkGH, m, 1, "out1D_zline_xi", "out_zline_xi",
                                "out1D_zline_x", "out_zline_x", out_zline_x);
      offset[1] = GetGridOffset(cctkGH, m, 2, "out1D_zline_yi", "out_zline_yi",
                                "out1D_zline_y", "out_zline_y", out_zline_y);
      break;
    case 3:
      // the diagonal: we don't care about the offset
      break;
    default:
      assert(0);
    }
    break;

  case 2:
    // 2D output
    if (dirs[0] == 0 and dirs[1] == 1) {
      offset[2] =
          GetGridOffset(cctkGH, m, 3, "out2D_xyplane_zi", "out_xyplane_zi",
                        "out2D_xyplane_z", "out_xyplane_z", out_xyplane_z);
    } else if (dirs[0] == 0 and dirs[1] == 2) {
      offset[1] =
          GetGridOffset(cctkGH, m, 2, "out2D_xzplane_yi", "out_xzplane_yi",
                        "out2D_xzplane_y", "out_xzplane_y", out_xzplane_y);
    } else if (dirs[0] == 1 and dirs[1] == 2) {
      offset[0] =
          GetGridOffset(cctkGH, m, 1, "out2D_yzplane_xi", "out_yzplane_xi",
                        "out2D_yzplane_x", "out_yzplane_x", out_yzplane_x);
    } else {
      assert(0);
    }
    break;

  case 3:
    // 3D output: the offset does not matter
    break;

  default:
    assert(0);
  }

  return offset;
}

// Omit symmetry, ghost and buffer zones if requested
ibset GetOutputBBoxes(const cGH *const cctkGH, const int group, const int rl,
                      const int m, const int c, const ibbox &ext,
                      const ibset &allactive) {
  DECLARE_CCTK_PARAMETERS;

  const int groupdim = CCTK_GroupDimI(group);
  assert(groupdim >= 0);
  const int grouptype = CCTK_GroupTypeI(group);
  assert(grouptype >= 0);

  // TODO: This is a bit ad hoc
  CCTK_INT symtable;
  if (grouptype == CCTK_GF and groupdim == cctkGH->cctk_dim) {
    symtable = SymmetryTableHandleForGrid(cctkGH);
    if (symtable < 0)
      CCTK_WARN(0, "internal error");
  } else {
    symtable = SymmetryTableHandleForGI(cctkGH, group);
    if (symtable < 0)
      CCTK_WARN(0, "internal error");
  }

  CCTK_INT symbnd[2 * dim];
  int const ierr =
      Util_TableGetIntArray(symtable, 2 * groupdim, symbnd, "symmetry_handle");
  if (ierr != 2 * groupdim)
    CCTK_WARN(0, "internal error");

  bool is_symbnd[2 * dim];
  for (int d = 0; d < 2 * groupdim; ++d) {
    is_symbnd[d] = symbnd[d] >= 0;
  }

  ivect lo = ext.lower();
  ivect hi = ext.upper();
  const ivect str = ext.stride();

  const b2vect obnds = vhh.at(m)->outer_boundaries(rl, c);
  const i2vect ghost_width = arrdata.at(group).at(m).dd->ghost_widths.AT(rl);

  for (int d = 0; d < groupdim; ++d) {
    bool const output_lower_ghosts =
        obnds[0][d] ? (is_symbnd[2 * d]
                           ? output_symmetry_points
                           : (output_boundary_points and out3D_outer_ghosts))
                    : (output_ghost_points and out3D_ghosts);
    bool const output_upper_ghosts =
        obnds[1][d] ? (is_symbnd[2 * d + 1]
                           ? output_symmetry_points
                           : (output_boundary_points and out3D_outer_ghosts))
                    : (output_ghost_points and out3D_ghosts);

    if (not output_lower_ghosts) {
      lo[d] += ghost_width[0][d] * str[d];
    }
    if (not output_upper_ghosts) {
      hi[d] -= ghost_width[1][d] * str[d];
    }
  }

  ibset exts(ibbox(lo, hi, str));
  // do grid arrays have buffer zones?
  if (not output_buffer_points) {
    ivect loghosts(0);
    ivect highosts(0);
    // possibly re-add ghost points and symmetry points
    // TODO: come up with a nicer way to do this
    for (int d = 0; d < groupdim; ++d) {
      bool const output_lower_ghosts =
          obnds[0][d] ? (is_symbnd[2 * d]
                             ? output_symmetry_points
                             : (output_boundary_points and out3D_outer_ghosts))
                      : (output_ghost_points and out3D_ghosts);
      bool const output_upper_ghosts =
          obnds[1][d] ? (is_symbnd[2 * d + 1]
                             ? output_symmetry_points
                             : (output_boundary_points and out3D_outer_ghosts))
                      : (output_ghost_points and out3D_ghosts);

      if (output_lower_ghosts) {
        loghosts[d] = ghost_width[0][d];
      }
      if (output_upper_ghosts) {
        highosts[d] = ghost_width[1][d];
      }
    }
    exts = exts & allactive.expand(loghosts, highosts);
  }

  return exts;
}

// Determine coordinates
void GetCoordinates(const cGH *const cctkGH, const int m,
                    const cGroup &groupdata, const ibset &exts,
                    CCTK_REAL &coord_time, vector<rvect> &coord_lower,
                    vector<rvect> &coord_upper) {
  coord_time = cctkGH->cctk_time;

  rvect global_lower;
  rvect coord_delta;
  if (groupdata.grouptype == CCTK_GF) {
    rvect const cctk_origin_space = origin_space.at(m).at(mglevel);
    rvect const cctk_delta_space = delta_space.at(m) * rvect(mglevelfact);
    for (int d = 0; d < dim; ++d) {
      // lower boundary of Carpet's integer indexing
      global_lower[d] = cctk_origin_space[d];
      // grid spacing of Carpet's integer indexing
      coord_delta[d] = (cctk_delta_space[d] /
                        vhh.at(m)->baseextents.at(0).at(0).stride()[d]);
    }
  } else {
    for (int d = 0; d < dim; ++d) {
      global_lower[d] = 0.0;
      coord_delta[d] = 1.0;
    }
  }

  coord_lower.reserve(exts.setsize());
  coord_upper.reserve(exts.setsize());

  for (ibset::const_iterator ext = exts.begin(); ext != exts.end(); ++ext) {
    coord_lower.push_back(global_lower + coord_delta * rvect(ext->lower()));
    coord_upper.push_back(global_lower + coord_delta * rvect(ext->upper()));
  }
}

int GetGridOffset(const cGH *const cctkGH, const int m, const int dir,
                  const char *const iparam, const char *const iglobal,
                  const char *const cparam, const char *const cglobal,
                  const CCTK_REAL cfallback) {
  // First choice: explicit coordinate
  const int ncparam = CCTK_ParameterQueryTimesSet(cparam, CCTK_THORNSTRING);
  assert(ncparam >= 0);
  if (ncparam > 0) {
    int ptype;
    const CCTK_REAL *const pcoord = ((const CCTK_REAL *)CCTK_ParameterGet(
        cparam, CCTK_THORNSTRING, &ptype));
    assert(pcoord);
    const CCTK_REAL coord = *pcoord;
    assert(ptype == PARAMETER_REAL);
    return CoordToOffset(cctkGH, m, dir, coord, 0);
  }

  // Second choice: explicit index
  const int niparam = CCTK_ParameterQueryTimesSet(iparam, CCTK_THORNSTRING);
  assert(niparam >= 0);
  if (niparam > 0) {
    int ptype;
    const int *const pindex =
        (const int *)CCTK_ParameterGet(iparam, CCTK_THORNSTRING, &ptype);
    assert(pindex);
    const int index = *pindex;
    assert(ptype == PARAMETER_INT);
    return index;
  }

  // Third choice: explicit global coordinate
  const char *iothorn = CCTK_ImplementationThorn("IO");
  assert(iothorn);
  if (cglobal) {
    const int ncglobal = CCTK_ParameterQueryTimesSet(cglobal, iothorn);
    assert(ncglobal >= 0);
    if (ncglobal > 0) {
      int ptype;
      const CCTK_REAL *const pcoord =
          (const CCTK_REAL *)CCTK_ParameterGet(cglobal, iothorn, &ptype);
      assert(pcoord);
      const CCTK_REAL coord = *pcoord;
      assert(ptype == PARAMETER_REAL);
      return CoordToOffset(cctkGH, m, dir, coord, 0);
    }
  }

  // Fourth choice: explicit global index
  if (iglobal) {
    const int niglobal = CCTK_ParameterQueryTimesSet(iglobal, iothorn);
    assert(niglobal >= 0);
    if (niglobal > 0) {
      int ptype;
      const int *const pindex =
          (const int *)CCTK_ParameterGet(iglobal, iothorn, &ptype);
      assert(pindex);
      const int index = *pindex;
      assert(ptype == PARAMETER_INT);
      return index;
    }
  }

  // Fifth choice: default coordinate
  return CoordToOffset(cctkGH, m, dir, cfallback, 0);
}

int CoordToOffset(const cGH *cctkGH, const int m, const int dir,
                  const CCTK_REAL coord, const int ifallback) {
  assert(m >= 0 and m < Carpet::maps and dir >= 1 and dir <= dim);

  assert(mglevel != -1 and reflevel != -1 and Carpet::map == -1);

  rvect const cctk_origin_space = origin_space.at(m).at(mglevel);
  rvect const cctk_delta_space = delta_space.at(m) * rvect(mglevelfact);
  ivect const cctk_levfac = spacereffacts.at(reflevel);
  ibbox const &coarseext = vhh.at(m)->baseextents.at(mglevel).at(0);
  ibbox const &baseext = vhh.at(m)->baseextents.at(mglevel).at(reflevel);
  ivect const cctk_levoff = baseext.lower() - coarseext.lower();
  ivect const cctk_levoffdenom = baseext.stride();

  const CCTK_REAL delta = cctk_delta_space[dir - 1] / cctk_levfac[dir - 1];
  const CCTK_REAL lower = cctk_origin_space[dir - 1] +
                          cctk_delta_space[dir - 1] / cctk_levfac[dir - 1] *
                              cctk_levoff[dir - 1] / cctk_levoffdenom[dir - 1];

  const CCTK_REAL rindex = (coord - lower) / delta;
  int cindex = (int)floor(rindex + 0.75);

  return cindex;
}

// Output
template <int outdim>
int IOHDF5<outdim>::WriteHDF5(const cGH *cctkGH, hid_t &file, hid_t &indexfile,
                              vector<gdata *> const gfdatas,
                              const bbox<int, dim> &gfext, const int vi,
                              const vect<int, dim> &org,
                              const vect<int, outdim> &dirs, const int rl,
                              const int ml, const int m, const int c,
                              const int output_component, const int tl,
                              const CCTK_REAL coord_time,
                              const vect<CCTK_REAL, dim> &coord_lower,
                              const vect<CCTK_REAL, dim> &coord_upper) {
  DECLARE_CCTK_PARAMETERS;

  assert(outdim <= dim);

  cGroup groupdata;
  int const gi = CCTK_GroupIndexFromVarI(vi);
  assert(gi >= 0);
  int const ierr = CCTK_GroupData(gi, &groupdata);
  assert(not ierr);

  // boolean that says if we are doing 1D-diagonal output
  // This is not beautiful, but works for the moment
  bool const diagonal_output = outdim == 1 and dirs[0] == 3;

  // Check whether the output bbox overlaps
  // with the extent of the data to be output
  // FIXME: move this check up in the call stack
  bool output_bbox_overlaps_data_extent;
  if (not diagonal_output) {

    const vect<int, outdim> lo = gfext.lower()[dirs];
    const vect<int, outdim> up = gfext.upper()[dirs];
    const vect<int, outdim> str = gfext.stride()[dirs];
    const bbox<int, outdim> ext(lo, up, str);

    // Check whether the output origin is contained in the extent
    // of the data that should be output
    ivect org1(org);
    for (int d = 0; d < outdim; ++d)
      org1[dirs[d]] = ext.lower()[d];
    output_bbox_overlaps_data_extent = gfext.contains(org1);

  } else {

    gh const &hh = *vhh.at(m);
    ibbox const &base = hh.baseextents.at(mglevel).at(reflevel);

    assert(base.stride()[0] == base.stride()[1] and
           base.stride()[0] == base.stride()[2]);

    // Check if any point on the diagonal is in our gf's extent
    output_bbox_overlaps_data_extent = false;
    for (int i = maxval(base.lower()); i <= minval(base.upper());
         i += base.stride()[0]) {

      ivect const pos = ivect(i, i, i);
      output_bbox_overlaps_data_extent |= gfext.contains(pos);
    }
  }
  // Shortcut if there is nothing to output
  if (not output_bbox_overlaps_data_extent) {
    return 0;
  }

  int error_count = 0;

  ostringstream datasetname_suffix;
  datasetname_suffix << " it=" << cctkGH->cctk_iteration << " tl=" << tl;
  if (mglevels > 1)
    datasetname_suffix << " ml=" << ml;
  if (groupdata.grouptype == CCTK_GF) {
    if (maps > 1)
      datasetname_suffix << " m=" << m;
    datasetname_suffix << " rl=" << rl;
  }
  if (arrdata.at(gi).at(m).dd->light_boxes.at(ml).at(rl).size() > 1 and
      groupdata.disttype != CCTK_DISTRIB_CONSTANT) {
    datasetname_suffix << " c=" << output_component;
  }

  // enable compression and checksums if requested
  hid_t plist;
  HDF5_ERROR(plist = H5Pcreate(H5P_DATASET_CREATE));
  // enable checksums if requested
  if (use_checksums) {
    HDF5_ERROR(H5Pset_filter(plist, H5Z_FILTER_FLETCHER32, 0, 0, 0));
  }

  // enable datatype conversion if requested
  const hid_t mem_type = CCTKtoHDF5_Datatype(cctkGH, groupdata.vartype, false);
  const hid_t slice_type =
      CCTKtoHDF5_Datatype(cctkGH, groupdata.vartype, out_single_precision);

  if (not diagonal_output) { // not outputting the diagonal

    // there are currently three "extent" variables in use:
    // data_ext - the extent of the component in memory
    // gfext    - the bounding box of data to be written
    // ext      - gfext in the output dimension(s)
    // data_ext and gfext are used to construct the hyperslab location and
    // size, ext is just a shorthand.
    // TODO: maybe transfer only ext instead of data_ext in copy_from
    const vect<int, outdim> lo = gfext.lower()[dirs];
    const vect<int, outdim> up = gfext.upper()[dirs];
    const vect<int, outdim> str = gfext.stride()[dirs];
    const bbox<int, outdim> ext(lo, up, str);
    const dh *const dd = arrdata.at(gi).at(m).dd;
    const ibbox &data_ext = dd->light_boxes.at(ml).at(rl).at(c).exterior;

    // Check whether the output origin is contained in the extent of
    // the data that should be output
    ivect org1(org);
    for (int d = 0; d < outdim; ++d)
      org1[dirs[d]] = ext.lower()[d];
    assert(gfext.contains(org1));

    // HDF5 wants ranks to be >= 1
    const int rank = outdim > 0 ? outdim : 1;
    vector<hsize_t> mem_shape(dim);
    vector<hsize_t> slice_shape(rank, 1);
    hsize_t num_elems = 1;
    for (int d = 0; d < dim; d++) {
      mem_shape[dim - 1 - d] = data_ext.shape()[d] / data_ext.stride()[d];
      if (d < outdim) {
        slice_shape[outdim - 1 - d] = ext.shape()[d] / ext.stride()[d];
        num_elems *= slice_shape[outdim - 1 - d];
      }
    }

    ivect slice_lower(org - data_ext.lower());
    for (int d = 0; d < outdim; d++) {
      slice_lower[dirs[d]] = ext.lower()[d] - data_ext.lower()[dirs[d]];
    }
    ivect slice_upper(slice_lower);
    for (int d = 0; d < outdim; d++) {
      slice_upper[dirs[d]] = ext.upper()[d] - data_ext.lower()[dirs[d]];
    }
    slice_lower /= gfext.stride();
    slice_upper /= gfext.stride();

    slice_start_size_t slice_start[dim];
    hsize_t slice_count[dim];
    for (int d = 0; d < dim; d++) {
      slice_start[dim - 1 - d] = slice_lower[d];
      slice_count[dim - 1 - d] = slice_upper[d] - slice_lower[d] + 1;
    }
    const hsize_t size = num_elems * H5Tget_size(slice_type);
    if (compression_level and size > hsize_t(minimum_size_for_compression)) {
      HDF5_ERROR(H5Pset_shuffle(plist));
      HDF5_ERROR(H5Pset_deflate(plist, compression_level));
    }
    if (compression_level or use_checksums) {
      HDF5_ERROR(H5Pset_chunk(plist, slice_shape.size(), &slice_shape[0]));
    }
    hid_t slice_space, mem_space;
    HDF5_ERROR(slice_space =
                   H5Screate_simple(slice_shape.size(), &slice_shape[0], NULL));
    HDF5_ERROR(mem_space =
                   H5Screate_simple(mem_shape.size(), &mem_shape[0], NULL));
    HDF5_ERROR(H5Sselect_hyperslab(mem_space, H5S_SELECT_SET, slice_start, NULL,
                                   slice_count, NULL));

    vector<int> iorigin(rank, 0);
    vector<int> ioffset(rank, 0);
    vector<int> ioffsetdenom(rank, 1);
    vector<double> delta(rank, 0), origin(rank, 0);
    vector<int> bbox(2 * rank, 0), nghostzones(rank, 0);
    for (int d = 0; d < outdim; d++) {
      assert(gfext.upper()[dirs[d]] - gfext.lower()[dirs[d]] >= 0);
      iorigin[d] = ext.lower()[d];
      delta[d] = 0;
      origin[d] = coord_lower[dirs[d]];
      if (gfext.upper()[dirs[d]] - gfext.lower()[dirs[d]] > 0) {
        delta[d] = (coord_upper[dirs[d]] - coord_lower[dirs[d]]) /
                   (gfext.upper()[dirs[d]] - gfext.lower()[dirs[d]]) *
                   gfext.stride()[dirs[d]];
        origin[d] += (org1[dirs[d]] - gfext.lower()[dirs[d]]) * delta[d];
        ioffsetdenom[d] = gfext.stride()[dirs[d]];
        ioffset[d] =
            (iorigin[d] % gfext.stride()[dirs[d]] + gfext.stride()[dirs[d]]) %
            gfext.stride()[dirs[d]];
        assert((iorigin[d] - ioffset[d]) % gfext.stride()[dirs[d]] == 0);
        iorigin[d] = (iorigin[d] - ioffset[d]) / gfext.stride()[dirs[d]];
      }
    }
    string active;
    {
      // Determine extent of hyperslab that is output
      ivect lo = gfext.lower();
      ivect up = gfext.upper();
      ivect str = gfext.stride();
      for (int d = 0; d < dim; ++d) {
        bool isoutdir = false;
        for (int e = 0; e < outdim; ++e)
          isoutdir |= d == dirs[e];
        if (!isoutdir) {
          lo[d] = org[d];
          up[d] = org[d];
        }
      }
      const ibbox outputslab(lo, up, str);
      // Intersect active region with this hyperslab
      const int lc = vhh.at(m)->get_local_component(rl, c);
      const ibset &active0 = vdd.at(m)->level_boxes.at(ml).at(rl).active;
      const ibset active1 = active0 & outputslab;
      // Reduce dimensionality of active region
      bboxset<int, outdim> active2;
      for (ibset::const_iterator bi = active1.begin(), be = active1.end();
           bi != be; ++bi) {
        const ibbox &box0 = *bi;
        const vect<int, outdim> lo = box0.lower()[dirs];
        const vect<int, outdim> up = box0.upper()[dirs];
        const vect<int, outdim> str = box0.stride()[dirs];
        const ::bbox<int, outdim> box(lo, up, str);
        active2 += box;
      }
      ostringstream buf;
      buf << active2;
      active = buf.str();
    }

    // store cctk_bbox and cctk_nghostzones (for grid arrays only)
    if (groupdata.grouptype != CCTK_SCALAR) {
      const b2vect obnds = vhh.at(m)->outer_boundaries(rl, c);
      const i2vect ghost_width = arrdata.at(gi).at(m).dd->ghost_widths.AT(rl);
      for (int d = 0; d < outdim; d++) {
        nghostzones[d] = output_ghost_points ? ghost_width[0][dirs[d]] : 0;
        assert(all(ghost_width[0] == ghost_width[1]));

        bbox[2 * d] = obnds[0][dirs[d]];
        bbox[2 * d + 1] = obnds[1][dirs[d]];
      }
    }

    // now loop over all variables
    for (size_t n = 0; n < gfdatas.size(); n++) {

      // create a unique name for this variable's dataset
      char *fullname = CCTK_FullName(vi + n);
      string datasetname(fullname);
      datasetname.append(datasetname_suffix.str());

      // remove an already existing dataset of the same name
      ioRequest *request = slice_requests.at(vi + n);
      if (not request) {
#ifdef IOUTIL_PARSER_HAS_OUT_DT
        request = IOUtil_DefaultIORequest(cctkGH, vi + n, 1, -1.0);
#else
        request = IOUtil_DefaultIORequest(cctkGH, vi + n, 1);
#endif
      }
      if (request->check_exist) {
        H5E_BEGIN_TRY {
          H5Gunlink(file, datasetname.c_str());
          if (indexfile != -1)
            H5Gunlink(indexfile, datasetname.c_str());
        }
        H5E_END_TRY;
      }
      // free I/O request structure
      if (request != slice_requests.at(vi + n)) {
        IOUtil_FreeIORequest(&request);
      }

      // write the dataset
      hid_t dataset, index_dataset;
      HDF5_ERROR(dataset = H5Dcreate(file, datasetname.c_str(), slice_type,
                                     slice_space, plist));
      if (indexfile != -1) {
        HDF5_ERROR(index_dataset = H5Dcreate(indexfile, datasetname.c_str(),
                                             slice_type, slice_space, plist));
      }

      HDF5_ERROR(H5Dwrite(dataset, mem_type, mem_space, H5S_ALL, H5P_DEFAULT,
                          gfdatas[n]->storage()));
      error_count += AddSliceAttributes(
          cctkGH, fullname, rl, ml, m, tl, origin, delta, iorigin, ioffset,
          ioffsetdenom, bbox, nghostzones, active, dataset, slice_shape, false);
      HDF5_ERROR(H5Dclose(dataset));

      if (indexfile != -1) {
        error_count += AddSliceAttributes(
            cctkGH, fullname, rl, ml, m, tl, origin, delta, iorigin, ioffset,
            ioffsetdenom, bbox, nghostzones, active, index_dataset, slice_shape,
            true);
        HDF5_ERROR(H5Dclose(index_dataset));
      }
      free(fullname);

      io_bytes +=
          H5Sget_simple_extent_npoints(slice_space) * H5Tget_size(slice_type);
    } // for n

    HDF5_ERROR(H5Sclose(mem_space));
    HDF5_ERROR(H5Sclose(slice_space));

  } else { // taking care of the diagonal

    const ivect lo = gfext.lower();
    const ivect up = gfext.upper();
    const ivect str = gfext.stride();
    const ibbox ext(lo, up, str);

    gh const &hh = *vhh.at(m);
    ibbox const &base = hh.baseextents.at(mglevel).at(reflevel);

    assert(base.stride()[0] == base.stride()[1] and
           base.stride()[0] == base.stride()[2]);

    // count the number of points on the diagonal
    hsize_t npoints = 0;
    for (int i = maxval(base.lower()); i <= minval(base.upper());
         i += base.stride()[0]) {
      if (gfext.contains(i)) {
        ++npoints;
      }
    }
    assert(npoints > 0);

    // allocate a contiguous buffer for the diagonal points
    vector<char> buffer(CCTK_VarTypeSize(groupdata.vartype) * npoints *
                        gfdatas.size());

    // copy diagonal points into contiguous buffer
    hsize_t offset = 0;
    for (int i = maxval(base.lower()); i <= minval(base.upper());
         i += base.stride()[0]) {
      ivect const pos = ivect(i, i, i);
      if (gfext.contains(pos)) {
        for (size_t n = 0; n < gfdatas.size(); n++) {
          switch (specific_cactus_type(groupdata.vartype)) {
#define TYPECASE(N, T)                                                         \
  case N: {                                                                    \
    T *typed_buffer = (T *)&buffer.front();                                    \
    typed_buffer[offset + n * npoints] =                                       \
        (*(const data<T> *)gfdatas.at(n))[pos];                                \
    break;                                                                     \
  }
#include "typecase.hh"
#undef TYPECASE
          }
        }
        ++offset;
      }
    }
    assert(offset == npoints);

    const hsize_t size = npoints * H5Tget_size(slice_type);
    if (compression_level and size > hsize_t(minimum_size_for_compression)) {
      HDF5_ERROR(H5Pset_shuffle(plist));
      HDF5_ERROR(H5Pset_deflate(plist, compression_level));
    }
    if (compression_level or use_checksums) {
      HDF5_ERROR(H5Pset_chunk(plist, 1, &npoints));
    }
    hid_t slice_space;
    HDF5_ERROR(slice_space = H5Screate_simple(1, &npoints, NULL));

    // loop over all variables and write out diagonals
    for (size_t n = 0; n < gfdatas.size(); n++) {

      // create a unique name for this variable's dataset
      char *fullname = CCTK_FullName(vi + n);
      string datasetname(fullname);
      free(fullname);
      datasetname.append(datasetname_suffix.str());

      // remove an already existing dataset of the same name
      ioRequest *request = slice_requests.at(vi + n);
      if (not request) {
#ifdef IOUTIL_PARSER_HAS_OUT_DT
        request = IOUtil_DefaultIORequest(cctkGH, vi + n, 1, -1.0);
#else
        request = IOUtil_DefaultIORequest(cctkGH, vi + n, 1);
#endif
      }
      if (request->check_exist) {
        H5E_BEGIN_TRY { H5Gunlink(file, datasetname.c_str()); }
        H5E_END_TRY;
      }
      // free I/O request structure
      if (request != slice_requests.at(vi + n)) {
        IOUtil_FreeIORequest(&request);
      }

      // write the dataset
      hid_t dataset;
      HDF5_ERROR(dataset = H5Dcreate(file, datasetname.c_str(), slice_type,
                                     slice_space, plist));
      HDF5_ERROR(H5Dwrite(dataset, mem_type, H5S_ALL, H5S_ALL, H5P_DEFAULT,
                          &buffer.front() + n * npoints * gfdatas.size()));
      HDF5_ERROR(H5Dclose(dataset));

      io_bytes +=
          H5Sget_simple_extent_npoints(slice_space) * H5Tget_size(slice_type);
    }

    HDF5_ERROR(H5Sclose(slice_space));

  } // if(not diagonal_output)

  HDF5_ERROR(H5Pclose(plist));

  return error_count;
}

// which processor serves as IO processor for this group
// TODO: see if IOUtil offers and official way to get this information
template <int outdim> int IOHDF5<outdim>::IOProcForProc(int proc) {
  // according to IOUtil::SetupGH a proc with rank % nioprocs == 0 is an
  // ioproc
  return (proc / ioproc_every) * ioproc_every;
}

// Explicit instantiation for all slice output dimensions
template class IOHDF5<0>;
template class IOHDF5<1>;
template class IOHDF5<2>;
template class IOHDF5<3>;

} // namespace CarpetIOHDF5

```

make.code.defn:
```
# Main make.code.defn file for thorn CarpetIOHDF5

# Source files in this directory
SRCS = CarpetIOHDF5.cc Input.cc Output.cc OutputSlice.cc GetAllActive.cc

# Extend CXXFLAGS if HDF5 library was built with LFS support
ifneq ($(strip $(HDF5_LFS_FLAGS)),)
  CXXFLAGS += $(HDF5_LFS_FLAGS)
endif

```

GetAllActive.cc:
```
/*@@
  @file      GetAllActive.cc
  @date      Sun Sep 19 20:34:27 EDT 2010
  @author    Erik Schnetter, Roland Haas
  @desc
             a piece of dh::regrid to compute the active region
  @enddesc
@@*/

#include <cassert>

#include <vector>

#include "cctk.h"

#include "CarpetIOHDF5.hh"

namespace CarpetIOHDF5 {

/********************************************************************
 *********************     Internal Routines     ********************
 ********************************************************************/

static void assert_error(char const *restrict const checkstring,
                         char const *restrict const file, int const line,
                         int const ml, int const rl,
                         char const *restrict const message);
static void assert_error(char const *restrict const checkstring,
                         char const *restrict const file, int const line,
                         int const ml, int const rl, int const c,
                         char const *restrict const message);
static void assert_error(char const *restrict const checkstring,
                         char const *restrict const file, int const line,
                         int const ml, int const rl, int const c, int const cc,
                         char const *restrict const message);

/********************************************************************
 *********************     Local Data   *****************************
 ********************************************************************/

// accumulates any consistency errors
static bool there_was_an_error = false;

/*@@
  @routine
  @date       Sun Sep 19 20:46:35 EDT 2010
  @author     Erik Schnetter, Roland Haas
  @desc
              Output warning for consistency checks for a refinement level.

  @enddesc
  @history
  @endhistory
  @var        checkstring
  @vdesc      the check that failed
  @vtype      char const * restrict const
  @vio        in
  @endvar
  @var        file
  @vdesc      name of source file where error occured
  @vtype      char const * restrict const
  @vio        in
  @endvar
  @var        line
  @vdesc      source line in file where error occured
  @vtype      int const
  @vio        in
  @endvar
  @var        ml
  @vdesc      meta level where error occured
  @vtype      int const
  @vio        in
  @endvar
  @var        rl
  @vdesc      refinement level where error occured
  @vtype      int const
  @vio        in
  @endvar
  @var        message
  @vdesc      error message to display
  @vtype      char const * restrict const
  @vio        in
  @endvar
  @returntype void
@@*/
static void assert_error(char const *restrict const checkstring,
                         char const *restrict const file, int const line,
                         int const ml, int const rl,
                         char const *restrict const message) {
  CCTK_VWarn(CCTK_WARN_ALERT, __LINE__, __FILE__, CCTK_THORNSTRING,
             "\n%s:%d:\n   [ml=%d rl=%d] The following grid structure "
             "consistency check failed:\n   %s\n   %s",
             file, line, ml, rl, message, checkstring);
  there_was_an_error = true;
}

/*@@
  @routine
  @date       Sun Sep 19 20:46:35 EDT 2010
  @author     Erik Schnetter, Roland Haas
  @desc
              Output warning for consistency checks for a single component.

  @enddesc
  @history
  @endhistory
  @var        checkstring
  @vdesc      the check that failed
  @vtype      char const * restrict const
  @vio        in
  @endvar
  @var        file
  @vdesc      name of source file where error occured
  @vtype      char const * restrict const
  @vio        in
  @endvar
  @var        line
  @vdesc      source line in file where error occured
  @vtype      int const
  @vio        in
  @endvar
  @var        ml
  @vdesc      meta level where error occured
  @vtype      int const
  @vio        in
  @endvar
  @var        rl
  @vdesc      refinement level where error occured
  @vtype      int const
  @vio        in
  @endvar
  @var        c
  @vdesc      component where error occured
  @vtype      int const
  @vio        in
  @endvar
  @var        message
  @vdesc      error message to display
  @vtype      char const * restrict const
  @vio        in
  @endvar
  @returntype void
@@*/
static void assert_error(char const *restrict const checkstring,
                         char const *restrict const file, int const line,
                         int const ml, int const rl, int const c,
                         char const *restrict const message) {
  CCTK_VWarn(CCTK_WARN_ALERT, __LINE__, __FILE__, CCTK_THORNSTRING,
             "\n%s:%d:\n   [ml=%d rl=%d c=%d] The following grid structure "
             "consistency check failed:\n   %s\n   %s",
             file, line, ml, rl, c, message, checkstring);
  there_was_an_error = true;
}

/*@@
  @routine
  @date       Sun Sep 19 20:46:35 EDT 2010
  @author     Erik Schnetter, Roland Haas
  @desc
              Output warning for consistency checks between two components.

  @enddesc
  @history
  @endhistory
  @var        checkstring
  @vdesc      the check that failed
  @vtype      char const * restrict const
  @vio        in
  @endvar
  @var        file
  @vdesc      name of source file where error occured
  @vtype      char const * restrict const
  @vio        in
  @endvar
  @var        line
  @vdesc      source line in file where error occured
  @vtype      int const
  @vio        in
  @endvar
  @var        ml
  @vdesc      meta level where error occured
  @vtype      int const
  @vio        in
  @endvar
  @var        rl
  @vdesc      refinement level where error occured
  @vtype      int const
  @vio        in
  @endvar
  @var        c
  @vdesc      first component where error occured
  @vtype      int const
  @vio        in
  @endvar
  @var        cc
  @vdesc      second component where error occured
  @vtype      int const
  @vio        in
  @endvar
  @var        message
  @vdesc      error message to display
  @vtype      char const * restrict const
  @vio        in
  @endvar
  @returntype void
@@*/
static void assert_error(char const *restrict const checkstring,
                         char const *restrict const file, int const line,
                         int const ml, int const rl, int const c, int const cc,
                         char const *restrict const message) {
  CCTK_VWarn(CCTK_WARN_ALERT, __LINE__, __FILE__, CCTK_THORNSTRING,
             "\n%s:%d:\n   [ml=%d rl=%d c=%d cc=%d] The following grid "
             "structure consistency check failed:\n   %s\n   %s",
             file, line, ml, rl, c, cc, message, checkstring);
  there_was_an_error = true;
}

#ifdef CARPET_OPTIMISE

// For highest efficiency, omit all self-checks
#define ASSERT_rl(check, message)
#define ASSERT_c(check, message)
#define ASSERT_cc(check, message)

#else

#define ASSERT_rl(check, message)                                              \
  do {                                                                         \
    if (not(check)) {                                                          \
      assert_error(#check, __FILE__, __LINE__, ml, rl, message);               \
    }                                                                          \
  } while (false)

#define ASSERT_c(check, message)                                               \
  do {                                                                         \
    if (not(check)) {                                                          \
      assert_error(#check, __FILE__, __LINE__, ml, rl, c, message);            \
    }                                                                          \
  } while (false)

#define ASSERT_cc(check, message)                                              \
  do {                                                                         \
    if (not(check)) {                                                          \
      assert_error(#check, __FILE__, __LINE__, ml, rl, c, cc, message);        \
    }                                                                          \
  } while (false)

#endif

/*@@
  @routine
  @date       Sun Sep 19 20:46:35 EDT 2010
  @author     Erik Schnetter, Roland Haas
  @desc
              Recompute the union of all active regions. This routine is a
              stripped down copy of dh::regrid.
  @enddesc
  @history
  @endhistory
  @var        dh
  @vdesc      data hirarchy for this group
  @vtype      const dh *
  @vio        in
  @endvar
  @var        hh
  @vdesc      grid hirarchy
  @vtype      const gh *
  @vio        in
  @endvar
  @var        ml
  @vdesc      meta level to compute active region on
  @vtype      int
  @vio        in
  @endvar
  @var        rl
  @vdesc      refinement level to compute active region on
  @vtype      int
  @vio        in
  @endvar
  @var        allactive
  @vdesc      receives the list of active boxes
  @vtype      ibset &
  @vio        out
  @endvar
  @returntype void
@@*/
void GetAllActive(const dh *dd, const gh *hh, int ml, int rl,
                  ibset &allactive) {
  DECLARE_CCTK_PARAMETERS;

  // All owned regions
  ibset allowned;

  // Domain:

  ibbox const &domain_exterior = hh->baseextent(ml, rl);
  // Variables may have size zero
  // ASSERT_rl (not domain_exterior.empty(),
  //            "The exterior of the domain must not be empty");

  i2vect const &buffer_width = dd->buffer_widths.AT(rl);
  i2vect const &boundary_width = hh->boundary_width;
  ASSERT_rl(all(all(boundary_width >= 0)),
            "The gh boundary widths must not be negative");

  ibbox const domain_active = domain_exterior.expand(-boundary_width);
  // Variables may have size zero
  // ASSERT_rl (not domain_active.empty(),
  //            "The active part of the domain must not be empty");
  ASSERT_rl(domain_active <= domain_exterior, "The active part of the domain "
                                              "must be contained in the "
                                              "exterior part of the domain");

  for (int c = 0; c < hh->components(rl); ++c) {

    // Interior:

    ibbox intr;
    intr = ibbox::poison();

    // The interior of the grid has the extent as specified by the
    // regridding thorn
    intr = hh->extent(ml, rl, c);

    // (The interior must not be empty)
    // Variables may have size zero
    // ASSERT_c (not intr.empty(),
    //           "The interior must not be empty");

    // The interior must be contained in the domain
    ASSERT_c(intr <= hh->baseextent(ml, rl),
             "The interior must be contained in the domain");

    // Outer boundary faces:

    b2vect is_outer_boundary;

    // The outer boundary faces are where the interior extends up
    // to the outer boundary of the domain.  It is not possible to
    // check whether it extends past the active part of the
    // domain, since this would be wrong when the outer boundary
    // width is zero.
    is_outer_boundary[0] = intr.lower() == domain_exterior.lower();
    is_outer_boundary[1] = intr.upper() == domain_exterior.upper();

    // Owned region:

    ibbox owned;
    owned = ibbox::poison();

    owned = intr.expand(i2vect(is_outer_boundary) * (-boundary_width));

    // (The owned region must not be empty)
    // Variables may have size zero
    // ASSERT_c (not owned.empty(),
    //           "The owned region must not be empty");

    // The owned region must be contained in the active part of
    // the domain
    ASSERT_c(
        owned <= domain_active,
        "The owned region must be contained in the active part of the domain");

    allowned |= owned;
    ASSERT_rl(
        allowned <= domain_active,
        "The owned regions must be contained in the active part of the domain");

  } // for c

  // Enlarge active part of domain
  i2vect const safedist = i2vect(0);
  ibbox const domain_enlarged = domain_active.expand(safedist);

  // All not-owned regions
  ibset const notowned = domain_enlarged - allowned;

  // All not-active points
  ibset const notactive = notowned.expand(buffer_width);

  // All active points
  allactive = allowned - notactive;

  // Fail if any of the consistency checks failed
  assert(there_was_an_error == false);
}

} // namespace CarpetIOHDF5

```

util/SubstituteDeprecatedParameters.pl:
```
#! /usr/bin/perl -w
#/*@@
#  @file      SubstituteDeprecatedParameters.pl
#  @date      Fri 3 December 2004
#  @author    Thomas Radke
#  @desc
#             Perl script to automatically substitute deprecated parameter
#             names in a parfile
#  @enddesc
#@@*/

# Parse the command line
$parfile = shift;
if (! $parfile || shift)
{
  printf STDERR "This perl script automatically substitutes deprecated parameter names in a parfile.\n\n";
  printf STDERR "Usage: $0 <parameter file>\n\n";
  exit;
}

# build the hash table for renamed parameters
$renamed_parameters{"\UIOHDF5::out3D_dir\E"} = 'IOHDF5::out_dir';
$renamed_parameters{"\UIOHDF5::out3D_vars\E"} = 'IOHDF5::out_vars';
$renamed_parameters{"\UIOHDF5::out3D_extension\E"} = 'IOHDF5::out_extension';
$renamed_parameters{"\UIOHDF5::out3D_criterion\E"} = 'IOHDF5::out_criterion';
$renamed_parameters{"\UIOHDF5::out3D_every\E"} = 'IOHDF5::out_every';
$renamed_parameters{"\UIOHDF5::out3D_dt\E"} = 'IOHDF5::out_dt';
$renamed_parameters{"\UIOHDF5::in3D_dir\E"} = 'IOHDF5::in_dir';
$renamed_parameters{"\UIOHDF5::in3D_vars\E"} = 'IOHDF5::in_vars';
$renamed_parameters{"\UIOHDF5::in3D_extension\E"} = 'IOHDF5::in_extension';


open (PARFILE, $parfile) || die "Cannot open parameter file '$parfile' !";
print STDERR "Processing parameter file '$parfile'\n";
while (<PARFILE>)
{
  if (($pre, $parameter, $rest) = /^(\s*)(\w+::\w+)(.*)$/)
  {
    # rename old parameters if found
    if (defined $renamed_parameters{"\U$parameter\E"})
    {
      $_ = $pre . $renamed_parameters{"\U$parameter\E"} . $rest . "\n";
      print STDERR "  renaming parameter '$parameter' to '" . $renamed_parameters{"\U$parameter\E"} . "'\n";
    }
  }
  print;
}
close (PARFILE);

```

util/hdf5toascii_slicer.cc:
```
/*@@
  @file      hdf5toascii_slicer.cc
  @date      Sun 30 July 2007
  @author    Thomas Radke
  @desc
             Utility program to extract a 1D line or 2D slice from 3D datasets
             in datafiles generated by CarpetIOHDF5.
             The extracted slab is written to stdout in CarpetIOASCII format.
  @enddesc
@@*/

#include <algorithm>
#include <iostream>
#include <iomanip>
#include <string>
#include <vector>
#include <cmath>
#include <cstring>
#include <regex.h>

#define H5_USE_16_API 1
#include <hdf5.h>

using namespace std;

/*****************************************************************************
 *************************     Macro Definitions   ***************************
 *****************************************************************************/

// uncomment the following line to get some debug output
// #define DEBUG 1

// value for an unset parameter
#define PARAMETER_UNSET -424242.0

// fuzzy factor for comparing dataset timesteps with user-specified value
#define FUZZY_FACTOR 1e-6

// the datatype for the 'start' argument in H5Sselect_hyperslab()
#if (H5_VERS_MAJOR == 1 &&                                                     \
     (H5_VERS_MINOR < 6 || (H5_VERS_MINOR == 6 && H5_VERS_RELEASE < 4)))
#define h5size_t hssize_t
#else
#define h5size_t hsize_t
#endif

// macro to check the return code of calls to the HDF5 library
#define CHECK_HDF5(fn_call)                                                    \
  {                                                                            \
    const int _retval = fn_call;                                               \
    if (_retval < 0) {                                                         \
      cerr << "HDF5 call " << #fn_call << " in file " << __FILE__ << " line "  \
           << __LINE__ << " returned error code " << _retval << endl;          \
    }                                                                          \
  }

/*****************************************************************************
 **************************     Type Definitions   ***************************
 *****************************************************************************/

typedef struct {
  hid_t file;
  string name;
  bool is_real;
  hsize_t dims[3];

  int map, mglevel, component, iteration, timelevel, rflevel;
  double time;
  int iorigin[3];
  double origin[3], delta[3];
} patch_t;

/*****************************************************************************
 *************************       Global Data         *************************
 *****************************************************************************/

// the slab coordinate as selected by the user
static double slab_coord[3] = {PARAMETER_UNSET, PARAMETER_UNSET,
                               PARAMETER_UNSET};

// flag for outputting data in full 3D
static bool out3D = false;

// the specific timestep selected by the user
static double timestep = PARAMETER_UNSET;

// the regular expression to match against dataset names
static const char *regex = NULL;
static regex_t preg;

// the list of all patches
static vector<patch_t> patchlist;

// maximum refinement level across all patches
static int max_rflevel = -1;

// whether to print omitted patches
static bool verbose = false;

/*****************************************************************************
 *************************     Function Prototypes   *************************
 *****************************************************************************/
static herr_t FindPatches(hid_t group, const char *name, void *_file);
static void ReadPatch(const patch_t &patch, int last_iteration);
static bool ComparePatches(const patch_t &a, const patch_t &b);

/*@@
  @routine    main
  @date       Sun 30 July 2006
  @author     Thomas Radke
  @desc
              Evaluates command line options and opens the input files.
  @enddesc

  @var        argc
  @vdesc      number of command line parameters
  @vtype      int
  @vio        in
  @endvar
  @var        argv
  @vdesc      array of command line parameters
  @vtype      char* const []
  @vio        in
  @endvar
@@*/
int main(int argc, char *const argv[]) {
  int i;
  bool help = false;
#if 0
  int iorigin[3] = {-1, -1, -1};
#endif
  int num_slab_options = 0;

  // evaluate command line parameters
  for (i = 1; i < argc; i++) {
    if (strcmp(argv[i], "--help") == 0) {
      help = true;
      break;
    } else if (strcmp(argv[i], "--verbose") == 0) {
      verbose = true;
    } else if (strcmp(argv[i], "--out3d-cube") == 0) {
      out3D = true;
    } else if (strcmp(argv[i], "--timestep") == 0 and i + 1 < argc) {
      timestep = atof(argv[++i]);
    } else if (strcmp(argv[i], "--match") == 0 and i + 1 < argc) {
      regex = argv[++i];
      if (regcomp(&preg, regex, REG_EXTENDED)) {
        cerr << "Error: invalid regular expression '" << regex << "' given"
             << endl
             << endl;
        return (-1);
      }
    } else if (strcmp(argv[i], "--out-precision") == 0 and i + 1 < argc) {
      cout << setprecision(atoi(argv[++i]));
    } else if (strcmp(argv[i], "--out0d-point") == 0 and i + 3 < argc) {
      slab_coord[0] = atof(argv[++i]);
      slab_coord[1] = atof(argv[++i]);
      slab_coord[2] = atof(argv[++i]);
      num_slab_options++;
    } else if (strcmp(argv[i], "--out1d-xline-yz") == 0 and i + 2 < argc) {
      slab_coord[1] = atof(argv[++i]);
      slab_coord[2] = atof(argv[++i]);
      num_slab_options++;
    } else if (strcmp(argv[i], "--out1d-yline-xz") == 0 and i + 2 < argc) {
      slab_coord[0] = atof(argv[++i]);
      slab_coord[2] = atof(argv[++i]);
      num_slab_options++;
    } else if (strcmp(argv[i], "--out1d-zline-xy") == 0 and i + 2 < argc) {
      slab_coord[0] = atof(argv[++i]);
      slab_coord[1] = atof(argv[++i]);
      num_slab_options++;
    } else if (strcmp(argv[i], "--out2d-yzplane-x") == 0 and i + 1 < argc) {
      slab_coord[0] = atof(argv[++i]);
      num_slab_options++;
    } else if (strcmp(argv[i], "--out2d-xzplane-y") == 0 and i + 1 < argc) {
      slab_coord[1] = atof(argv[++i]);
      num_slab_options++;
    } else if (strcmp(argv[i], "--out2d-xyplane-z") == 0 and i + 1 < argc) {
      slab_coord[2] = atof(argv[++i]);
      num_slab_options++;
#if 0
    } else if (strcmp (argv[i], "--out2d-yzplane-xi") == 0 and i+1 < argc) {
      iorigin[0] = atoi (argv[++i]); num_slab_options++;
    } else if (strcmp (argv[i], "--out2d-xzplane-yi") == 0 and i+1 < argc) {
      iorigin[1] = atoi (argv[++i]); num_slab_options++;
    } else if (strcmp (argv[i], "--out2d-xyplane-zi") == 0 and i+1 < argc) {
      iorigin[2] = atoi (argv[++i]); num_slab_options++;
#endif
    } else {
      break;
    }
  }

  /* give some help if called with incorrect number of parameters */
  if (help or i >= argc or num_slab_options != (out3D ? 0 : 1)) {
    const string indent(strlen(argv[0]) + 1, ' ');
    cerr << endl
         << "   ---------------------------" << endl
         << "   Carpet HDF5 to ASCII Slicer" << endl
         << "   ---------------------------" << endl
         << endl
         << "Usage: " << endl
         << argv[0] << " [--help]" << endl
         << indent << "[--match <regex string>]" << endl
         << indent << "[--out-precision <digits>]" << endl
         << indent << "[--timestep <cctk_time value>]" << endl
         << indent << "[--verbose]" << endl
         << indent << "<--out0d-point value value value> | <--out1d-line value "
                      "value> | <--out2d-plane value> | <out3d-cube>"
         << endl
         << indent << "<hdf5_infiles>" << endl
         << endl
         << "  where" << endl
         << "    [--help]                         prints this help" << endl
         << "    [--match <regex string>]         selects HDF5 datasets by "
            "their names"
         << endl
         << "                                     matching a regex string "
            "using POSIX"
         << endl
         << "                                     Extended Regular Expression "
            "syntax"
         << endl
         << "    [--out-precision <digits>]       sets the output precision"
         << endl
         << "                                     for floating-point numbers"
         << endl
         << "    [--timestep <cctk_time value>]   selects all HDF5 datasets "
            "which"
         << endl
         << "                                     (fuzzily) match the "
            "specified time"
         << endl
         << "    [--verbose]                      lists skipped HDF5 datasets "
            "on stderr"
         << endl
         << endl
         << "  and either --out0d-point value value value> or <--out1d-line "
            "value value> or <--out2d-plane value> or <--out3d-cube>"
         << endl
         << "  must be specified as in the following:" << endl
         << "    --out0d-point     <origin_x> <origin_y> <origin_z>" << endl
         << endl
         << "    --out1d-xline-yz  <origin_y> <origin_z>" << endl
         << "    --out1d-yline-xz  <origin_x> <origin_z>" << endl
         << "    --out1d-zline-xy  <origin_x> <origin_y>" << endl
         << endl
         << "    --out2d-yzplane-x  <origin_x>" << endl
         << "    --out2d-xzplane-y  <origin_y>" << endl
         << "    --out2d-xyplane-z  <origin_z>" << endl
         << endl
         << "    --out3d-cube" << endl
#if 0
         << "    --out2d-yzplane-xi <origin_xi>" << endl
         << "    --out2d-xzplane-yi <origin_yi>" << endl
         << "    --out2d-xyplane-zi <origin_zi>" << endl
#endif
         << endl
         << "  eg, to extract a 2D xy-plane at z = 0:" << endl
         << "    " << argv[0] << " --out2d-xyplane-z 0 alp.file_*.h5" << endl
         << endl
         << "  or the same plane but only for datasets of refinement level 0:"
         << endl
         << "    " << argv[0] << " --match 'ADMBASE::alp it=[0-9]+ tl=0 rl=0' "
                                 "--out2d-xyplane-z 0 alp.file_*.h5"
         << endl;
    return (0);
  }

  vector<hid_t> filelist;
  for (; i < argc; i++) {
    hid_t file;

    H5E_BEGIN_TRY { file = H5Fopen(argv[i], H5F_ACC_RDONLY, H5P_DEFAULT); }
    H5E_END_TRY;

    if (file < 0) {
      cerr << "Could not open Carpet HDF5 input file '" << argv[i] << "'"
           << endl
           << endl;
      continue;
    }

    CHECK_HDF5(H5Giterate(file, "/", NULL, FindPatches, &file));

    filelist.push_back(file);
  }

  if (patchlist.size() > 0) {
    /* now sort the patches by their time attribute */
    sort(patchlist.begin(), patchlist.end(), ComparePatches);

    cout << "# 1D ASCII output created by";
    for (int j = 0; j < argc; j++)
      cout << " " << argv[j];
    cout << endl << "#" << endl;

    int last_iteration = patchlist[0].iteration - 1;
    for (size_t j = 0; j < patchlist.size(); j++) {
      ReadPatch(patchlist[j], last_iteration);
      last_iteration = patchlist[j].iteration;
    }
  } else {
    cerr << "No valid datasets found" << endl << endl;
  }

  // close all input files
  for (size_t j = 0; j < filelist.size(); j++) {
    if (filelist[j] >= 0) {
      CHECK_HDF5(H5Fclose(filelist[j]));
    }
  }

  return (0);
}

/*@@
  @routine    FindPatches
  @date       Tue 8 August 2006
  @author     Thomas Radke
  @desc
              The worker routine which is called by H5Giterate().
              It checks whether the current HDF5 object is a patch matching
              the user's slab criteria, and adds it to the patches list.
  @enddesc

  @var        group
  @vdesc      HDF5 object to start the iteration
  @vtype      hid_t
  @vio        in
  @endvar
  @var        name
  @vdesc      name of the object at the current iteration
  @vtype      const char *
  @vio        in
  @endvar
  @var        _file
  @vdesc      pointer to the descriptor of the currently opened file
  @vtype      void *
  @vio        in
  @endvar
@@*/
static herr_t FindPatches(hid_t group, const char *name, void *_file) {
  int ndims;
  hid_t dataset, datatype, attr;
  H5G_stat_t object_info;
  H5T_class_t typeclass;
  patch_t patch;

  /* we are interested in datasets only - skip anything else */
  CHECK_HDF5(H5Gget_objinfo(group, name, 0, &object_info));
  if (object_info.type != H5G_DATASET) {
    return (0);
  }

  /* check the dataset's datatype - make sure it is either integer or real */
  CHECK_HDF5(dataset = H5Dopen(group, name));
  patch.name = name;

  CHECK_HDF5(datatype = H5Dget_type(dataset));
  CHECK_HDF5(typeclass = H5Tget_class(datatype));
  CHECK_HDF5(H5Tclose(datatype));

  // read the timestep
  CHECK_HDF5(attr = H5Aopen_name(dataset, "time"));
  CHECK_HDF5(H5Aread(attr, H5T_NATIVE_DOUBLE, &patch.time));
  CHECK_HDF5(H5Aclose(attr));

  // read the dimensions
  hid_t dataspace = -1;
  CHECK_HDF5(dataspace = H5Dget_space(dataset));
  ndims = H5Sget_simple_extent_ndims(dataspace);

  bool is_okay = false;
  if (typeclass != H5T_FLOAT and typeclass != H5T_INTEGER) {
    if (verbose) {
      cerr << "skipping dataset '" << name << "':" << endl
           << "  is not of integer or floating-point datatype" << endl;
    }
  } else if (ndims != 3) {
    if (verbose) {
      cerr << "skipping dataset '" << name << "':" << endl
           << "  dataset has " << ndims << " dimensions" << endl;
    }
  } else if (regex && regexec(&preg, name, 0, NULL, 0)) {
    if (verbose) {
      cerr << "skipping dataset '" << name << "':" << endl
           << "  name doesn't match regex" << endl;
    }
  } else if (timestep != PARAMETER_UNSET and
             fabs(timestep - patch.time) > FUZZY_FACTOR) {
    if (verbose) {
      cerr << "skipping dataset '" << name << "':" << endl
           << "  timestep (" << patch.time << ") doesn't match" << endl;
    }
  } else {
    CHECK_HDF5(attr = H5Aopen_name(dataset, "origin"));
    CHECK_HDF5(H5Aread(attr, H5T_NATIVE_DOUBLE, patch.origin));
    CHECK_HDF5(H5Aclose(attr));
    CHECK_HDF5(attr = H5Aopen_name(dataset, "delta"));
    CHECK_HDF5(H5Aread(attr, H5T_NATIVE_DOUBLE, patch.delta));
    CHECK_HDF5(H5Aclose(attr));
    CHECK_HDF5(H5Sget_simple_extent_dims(dataspace, patch.dims, NULL));

    int i;
    is_okay = out3D;
    if (not out3D) {
      for (i = 0; i < 3; i++) {
        if (slab_coord[i] != PARAMETER_UNSET) {
          is_okay =
              slab_coord[i] >= patch.origin[i] and
              slab_coord[i] <=
                  patch.origin[i] + (patch.dims[2 - i] - 1) * patch.delta[i];
          if (not is_okay)
            break;
        }
      }
    }
    if (not is_okay) {
      if (verbose) {
        cerr << "skipping dataset '" << name << "':" << endl
             << "  slab " << slab_coord[i] << " is out of dataset range ["
             << patch.origin[i] << ", "
             << patch.origin[i] + (patch.dims[2 - i] - 1) * patch.delta[i]
             << "]" << endl;
      }
    }
  }
  CHECK_HDF5(H5Sclose(dataspace));

  if (not is_okay) {
    CHECK_HDF5(H5Dclose(dataset));
    return (0);
  }

  patch.file = *(hid_t *)_file;
  patch.is_real = typeclass == H5T_FLOAT;

  /* read attributes */
  CHECK_HDF5(attr = H5Aopen_name(dataset, "timestep"));
  CHECK_HDF5(H5Aread(attr, H5T_NATIVE_INT, &patch.iteration));
  CHECK_HDF5(H5Aclose(attr));
  CHECK_HDF5(attr = H5Aopen_name(dataset, "level"));
  CHECK_HDF5(H5Aread(attr, H5T_NATIVE_INT, &patch.rflevel));
  CHECK_HDF5(H5Aclose(attr));
  CHECK_HDF5(attr = H5Aopen_name(dataset, "group_timelevel"));
  CHECK_HDF5(H5Aread(attr, H5T_NATIVE_INT, &patch.timelevel));
  CHECK_HDF5(H5Aclose(attr));
  CHECK_HDF5(attr = H5Aopen_name(dataset, "carpet_mglevel"));
  CHECK_HDF5(H5Aread(attr, H5T_NATIVE_INT, &patch.mglevel));
  CHECK_HDF5(H5Aclose(attr));
  CHECK_HDF5(attr = H5Aopen_name(dataset, "iorigin"));
  CHECK_HDF5(H5Aread(attr, H5T_NATIVE_INT, patch.iorigin));
  CHECK_HDF5(H5Aclose(attr));

  CHECK_HDF5(H5Dclose(dataset));

  // the component and map info are not contained in the HDF5 dataset
  // and thus have to be parsed from the dataset's name
  patch.map = patch.component = 0;
  string::size_type loc = patch.name.find(" m=", 0);
  if (loc != string::npos)
    patch.map = atoi(patch.name.c_str() + loc + 3);
  loc = patch.name.find(" c=", 0);
  if (loc != string::npos)
    patch.component = atoi(patch.name.c_str() + loc + 3);

  if (max_rflevel < patch.rflevel)
    max_rflevel = patch.rflevel;

  patchlist.push_back(patch);

  return (0);
}

/*@@
  @routine    ReadPatch
  @date       Tue 8 August 2006
  @author     Thomas Radke
  @desc
              Reads a slab from the given patch
              and outputs it in CarpetIOASCII format.
  @enddesc

  @var        patch
  @vdesc      patch to output
  @vtype      const patch_t&
  @vio        in
  @endvar
  @var        last_iteration
  @vdesc      iteration number of the last patch that was output
  @vtype      int
  @vio        in
  @endvar
@@*/
static void ReadPatch(const patch_t &patch, int last_iteration) {
  if (patch.iteration != last_iteration)
    cout << endl;

  cout << "# iteration " << patch.iteration << endl
       << "# refinement level " << patch.rflevel << "   multigrid level "
       << patch.mglevel << "   map " << patch.map << "   component "
       << patch.component << "   time level " << patch.timelevel << endl
       << "# column format: 1:it\t2:tl 3:rl 4:c 5:ml\t6:ix 7:iy "
          "8:iz\t9:time\t10:x 11:y 12:z\t13:data"
       << endl;

  h5size_t slabstart[3] = {0, 0, 0};
  hsize_t slabcount[3] = {patch.dims[0], patch.dims[1], patch.dims[2]};
  for (int i = 0; i < 3; i++) {
    if (slab_coord[i] != PARAMETER_UNSET) {
      slabstart[2 - i] =
          (h5size_t)((slab_coord[i] - patch.origin[i]) / patch.delta[i] + 0.5);
      slabcount[2 - i] = 1;
    }
  }

  hid_t dataset, filespace, slabspace;
  CHECK_HDF5(dataset = H5Dopen(patch.file, patch.name.c_str()));
  CHECK_HDF5(filespace = H5Dget_space(dataset));
  CHECK_HDF5(slabspace = H5Screate_simple(3, slabcount, NULL));
  CHECK_HDF5(H5Sselect_hyperslab(filespace, H5S_SELECT_SET, slabstart, NULL,
                                 slabcount, NULL));
  const hssize_t npoints = H5Sget_select_npoints(filespace);
  vector<int> int_data;
  vector<double> real_data;
  // make sure the vector allocates at least one element
  if (patch.is_real) {
    real_data.resize(npoints + 1);
  } else {
    int_data.resize(npoints + 1);
  }
  CHECK_HDF5(H5Dread(dataset,
                     patch.is_real ? H5T_NATIVE_DOUBLE : H5T_NATIVE_INT,
                     slabspace, filespace, H5P_DEFAULT,
                     patch.is_real ? static_cast<void *>(&real_data[0])
                                   : static_cast<void *>(&int_data[0])));
  CHECK_HDF5(H5Sclose(slabspace));
  CHECK_HDF5(H5Sclose(filespace));
  CHECK_HDF5(H5Dclose(dataset));

  int pos = 0;
  for (h5size_t k = slabstart[0]; k < slabstart[0] + (h5size_t)slabcount[0];
       k++) {
    for (h5size_t j = slabstart[1]; j < slabstart[1] + (h5size_t)slabcount[1];
         j++) {
      for (h5size_t i = slabstart[2]; i < slabstart[2] + (h5size_t)slabcount[2];
           i++, pos++) {
        cout << patch.iteration << "\t" << patch.timelevel << " "
             << patch.rflevel << " " << patch.component << " " << patch.mglevel
             << "\t"
             << (patch.iorigin[0] + i) * (1 << (max_rflevel - patch.rflevel))
             << " "
             << (patch.iorigin[1] + j) * (1 << (max_rflevel - patch.rflevel))
             << " "
             << (patch.iorigin[2] + k) * (1 << (max_rflevel - patch.rflevel))
             << "\t" << setiosflags(ios_base::fixed) << patch.time << "\t"
             << patch.origin[0] + i * patch.delta[0] << " "
             << patch.origin[1] + j * patch.delta[1] << " "
             << patch.origin[2] + k * patch.delta[2] << "\t"
             << resetiosflags(ios_base::fixed);
        if (patch.is_real) {
          cout << setiosflags(ios_base::scientific) << real_data.at(pos)
               << resetiosflags(ios_base::scientific);
        } else {
          cout << int_data.at(pos);
        }
        cout << endl;
      }
      if (slab_coord[0] == PARAMETER_UNSET)
        cout << endl;
    }
    if (slab_coord[1] == PARAMETER_UNSET)
      cout << endl;
  }
  if (slab_coord[2] == PARAMETER_UNSET)
    cout << endl;
}

// comparison function to sort patches by iteration number, refinement level,
// multigrid level, map, and component
static bool ComparePatches(const patch_t &a, const patch_t &b) {
  if (a.iteration != b.iteration)
    return (a.iteration < b.iteration);
  if (a.rflevel != b.rflevel)
    return (a.rflevel < b.rflevel);
  if (a.mglevel != b.mglevel)
    return (a.mglevel < b.mglevel);
  if (a.map != b.map)
    return (a.map < b.map);
  if (a.component != b.component)
    return (a.component < b.component);
  return (a.timelevel < b.timelevel);
}

```

util/hdf5_slicer.cc:
```
/*@@
  @file      hdf5_slicer.cc
  @date      Fri 17 October 2008
  @author    Thomas Radke
  @desc
             Utility program to extract a 1D line or 2D slice from 3D datasets
             in datafiles generated by CarpetIOHDF5.
  @enddesc
@@*/

#include <algorithm>
#include <cassert>
#include <iostream>
#include <iomanip>
#include <string>
#include <vector>
#include <cmath>
#include <cstring>
#include <regex.h>

#define H5_USE_16_API 1
#include <hdf5.h>

using namespace std;

/*****************************************************************************
 *************************     Macro Definitions   ***************************
 *****************************************************************************/

// value for an unset parameter
#define PARAMETER_UNSET -424242.0

// fuzzy factor for comparing dataset timesteps with user-specified value
#define FUZZY_FACTOR 1e-6

// the datatype for the 'start' argument in H5Sselect_hyperslab()
#if (H5_VERS_MAJOR == 1 &&                                                     \
     (H5_VERS_MINOR < 6 || (H5_VERS_MINOR == 6 && H5_VERS_RELEASE < 4)))
#define h5size_t hssize_t
#else
#define h5size_t hsize_t
#endif

// macro to check the return code of calls to the HDF5 library
#define CHECK_HDF5(fn_call)                                                    \
  {                                                                            \
    const int _retval = fn_call;                                               \
    if (_retval < 0) {                                                         \
      cerr << "HDF5 call " << #fn_call << " in file " << __FILE__ << " line "  \
           << __LINE__ << " returned error code " << _retval << endl;          \
    }                                                                          \
  }

// the name of the group containing file metadata
#define METADATA_GROUP "Parameters and Global Attributes"

/*****************************************************************************
 *************************       Global Data         *************************
 *****************************************************************************/

// the slab coordinate as selected by the user
static double slab_coord[3] = {PARAMETER_UNSET, PARAMETER_UNSET,
                               PARAMETER_UNSET};

// flag for outputting data in full 3D
static bool out3D = false;

// the specific timestep selected by the user
static double timestep = PARAMETER_UNSET;

// the regular expression to match against dataset names
static const char *regex = NULL;
static regex_t preg;

// the total number of datasets sliced
static unsigned int slices_extracted = 0;

// whether to print omitted datasets
static bool verbose = false;

// rank of the output slices
static hsize_t outrank = 0;

// output file id
static hid_t outfile = -1;

// compression level
static int compression_level = 0;

/*****************************************************************************
 *************************     Function Prototypes   *************************
 *****************************************************************************/
static herr_t ProcessDataset(hid_t group, const char *name, void *_file);

/*@@
  @routine    main
  @date       Fri 17 October 2008
  @author     Thomas Radke
  @desc
              Evaluates command line options and opens the input files.
  @enddesc

  @var        argc
  @vdesc      number of command line parameters
  @vtype      int
  @vio        in
  @endvar
  @var        argv
  @vdesc      array of command line parameters
  @vtype      char* const []
  @vio        in
  @endvar
@@*/
int main(int argc, char *const argv[]) {
  int i;
  bool help = false;
  int num_slab_options = 0;

  // evaluate command line parameters
  for (i = 1; i < argc; i++) {
    if (strcmp(argv[i], "--help") == 0) {
      help = true;
      break;
    } else if (strcmp(argv[i], "--compression-level") == 0) {
      compression_level = atoi(argv[++i]);
      if (compression_level < 0 or compression_level > 9) {
        cerr << "Error: invalid compression level'" << argv[i] << "' given"
             << endl
             << endl;
        return (-1);
      }
    } else if (strcmp(argv[i], "--verbose") == 0) {
      verbose = true;
    } else if (strcmp(argv[i], "--out3d-cube") == 0) {
      outrank = 3;
      out3D = true;
    } else if (strcmp(argv[i], "--timestep") == 0 and i + 1 < argc) {
      timestep = atof(argv[++i]);
    } else if (strcmp(argv[i], "--match") == 0 and i + 1 < argc) {
      regex = argv[++i];
      if (regcomp(&preg, regex, REG_EXTENDED)) {
        cerr << "Error: invalid regular expression '" << regex << "' given"
             << endl
             << endl;
        return (-1);
      }
    } else if (strcmp(argv[i], "--out1d-xline-yz") == 0 and i + 2 < argc) {
      outrank = 1;
      slab_coord[1] = atof(argv[++i]);
      slab_coord[2] = atof(argv[++i]);
      num_slab_options++;
    } else if (strcmp(argv[i], "--out1d-yline-xz") == 0 and i + 2 < argc) {
      outrank = 1;
      slab_coord[0] = atof(argv[++i]);
      slab_coord[2] = atof(argv[++i]);
      num_slab_options++;
    } else if (strcmp(argv[i], "--out1d-zline-xy") == 0 and i + 2 < argc) {
      outrank = 1;
      slab_coord[0] = atof(argv[++i]);
      slab_coord[1] = atof(argv[++i]);
      num_slab_options++;
    } else if (strcmp(argv[i], "--out2d-yzplane-x") == 0 and i + 1 < argc) {
      outrank = 2;
      slab_coord[0] = atof(argv[++i]);
      num_slab_options++;
    } else if (strcmp(argv[i], "--out2d-xzplane-y") == 0 and i + 1 < argc) {
      outrank = 2;
      slab_coord[1] = atof(argv[++i]);
      num_slab_options++;
    } else if (strcmp(argv[i], "--out2d-xyplane-z") == 0 and i + 1 < argc) {
      outrank = 2;
      slab_coord[2] = atof(argv[++i]);
      num_slab_options++;
    } else {
      break;
    }
  }

  /* give some help if called with incorrect number of parameters */
  if (help or i >= argc - 1 or num_slab_options != (out3D ? 0 : 1)) {
    const string indent(strlen(argv[0]) + 1, ' ');
    cerr << endl
         << "   ------------------" << endl
         << "   Carpet HDF5 Slicer" << endl
         << "   ------------------" << endl
         << endl
         << "Usage: " << endl
         << argv[0] << " [--help]" << endl
         << indent << "[--match <regex string>]" << endl
         << indent << "[--timestep <cctk_time value>]" << endl
         << indent << "[--verbose]" << endl
         << indent << "[--compression-level <integer between 0 and 9 inclusive>"
         << endl
         << indent
         << "<--out1d-line value value> | <--out2d-plane value> | <out3d-cube>"
         << endl
         << indent << "<hdf5_infiles> <hdf5_outfile>" << endl
         << endl
         << "  where" << endl
         << "    [--help]                         prints this help" << endl
         << "    [--match <regex string>]         selects HDF5 datasets by "
            "their names"
         << endl
         << "                                     matching a regex string "
            "using POSIX"
         << endl
         << "                                     Extended Regular Expression "
            "syntax"
         << endl
         << "    [--timestep <cctk_time value>]   selects all HDF5 datasets "
            "which"
         << endl
         << "                                     (fuzzily) match the "
            "specified time"
         << endl
         << "    [--verbose]                      lists skipped HDF5 datasets "
            "on stderr"
         << endl
         << endl
         << "  and either <--out1d-line value value> or <--out2d-plane value> "
            "or <--out3d-cube>"
         << endl
         << "  must be specified as in the following:" << endl
         << "    --out1d-xline-yz  <origin_y> <origin_z>" << endl
         << "    --out1d-yline-xz  <origin_x> <origin_z>" << endl
         << "    --out1d-zline-xy  <origin_x> <origin_y>" << endl
         << endl
         << "    --out2d-yzplane-x  <origin_x>" << endl
         << "    --out2d-xzplane-y  <origin_y>" << endl
         << "    --out2d-xyplane-z  <origin_z>" << endl
         << endl
         << "    --out3d-cube" << endl
#if 0
         << "    --out2d-yzplane-xi <origin_xi>" << endl
         << "    --out2d-xzplane-yi <origin_yi>" << endl
         << "    --out2d-xyplane-zi <origin_zi>" << endl
#endif
         << endl
         << "  eg, to extract a 2D xy-plane at z = 0:" << endl
         << "    " << argv[0] << " --out2d-xyplane-z 0 alp.file_*.h5 alp.z=0.h5"
         << endl
         << endl
         << "  or the same plane but only for datasets of refinement level 0:"
         << endl
         << "    " << argv[0] << " --match 'ADMBASE::alp it=[0-9]+ tl=0 rl=0' "
                                 "--out2d-xyplane-z 0 alp.file_*.h5 alp.z=0.h5"
         << endl;
    return (0);
  }

  // check that the output doesn't exist already
  const char *const outfilename = argv[argc - 1];
  H5E_BEGIN_TRY {
    if (H5Fis_hdf5(outfilename) >= 0) {
      cerr << "The designated output file '" << outfilename
           << "' must not exist beforehand." << endl
           << "Please remove it before continuing !" << endl
           << endl;
      return (-1);
    }
  }
  H5E_END_TRY;

  // create the output file
  cout << endl << "  creating output file '" << outfilename << "'" << endl;
  H5E_BEGIN_TRY {
    outfile = H5Fcreate(outfilename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
  }
  H5E_END_TRY;
  if (outfile < 0) {
    cerr << "Could not create output file '" << outfilename << "'" << endl;
    return (-1);
  }

  // browse though input file(s)
  for (; i < argc - 1; i++) {
    hid_t file;

    H5E_BEGIN_TRY { file = H5Fopen(argv[i], H5F_ACC_RDONLY, H5P_DEFAULT); }
    H5E_END_TRY;

    if (file < 0) {
      cerr << "Could not open input file '" << argv[i] << "'" << endl << endl;
      continue;
    }

    cout << "  iterating through input file '" << argv[i] << "'..." << endl;
    CHECK_HDF5(H5Giterate(file, "/", NULL, ProcessDataset, &file));

    // close file
    if (file >= 0) {
      CHECK_HDF5(H5Fclose(file));
    }
  }

  if (slices_extracted == 0) {
    cerr << endl
         << "No matching datasets were found for the selected "
            "slice parameters."
         << endl
         << endl;
  }

  // close output file
  CHECK_HDF5(H5Fclose(outfile));

  cout << endl << "Done." << endl << endl;

  return (0);
}

/*@@
  @routine    ProcessDataset
  @date       Fri 17 October 2008
  @author     Thomas Radke
  @desc
              The worker routine which is called by H5Giterate().
              It checks whether the current HDF5 object is a dataset matching
              the user's slab criteria.
  @enddesc

  @var        group
  @vdesc      HDF5 object to start the iteration
  @vtype      hid_t
  @vio        in
  @endvar
  @var        datasetname
  @vdesc      name of the object at the current iteration
  @vtype      const char *
  @vio        in
  @endvar
  @var        _file
  @vdesc      pointer to the descriptor of the currently opened file
  @vtype      void *
  @vio        in
  @endvar
@@*/
static herr_t ProcessDataset(hid_t group, const char *datasetname,
                             void *_file) {
  // we are interested in datasets only - skip anything else
  H5G_stat_t object_info;
  CHECK_HDF5(H5Gget_objinfo(group, datasetname, 0, &object_info));
  if (object_info.type != H5G_DATASET) {
    return (0);
  }

  hid_t dataset, datatype, attr;
  CHECK_HDF5(dataset = H5Dopen(group, datasetname));

  // check the dataset's datatype - make sure it is either integer or real
  CHECK_HDF5(datatype = H5Dget_type(dataset));
  H5T_class_t typeclass;
  CHECK_HDF5(typeclass = H5Tget_class(datatype));

  // read the timestep and variable name
  CHECK_HDF5(attr = H5Aopen_name(dataset, "time"));
  double time;
  CHECK_HDF5(H5Aread(attr, H5T_NATIVE_DOUBLE, &time));
  CHECK_HDF5(H5Aclose(attr));
  CHECK_HDF5(attr = H5Aopen_name(dataset, "name"));
  hid_t stringdatatype;
  CHECK_HDF5(stringdatatype = H5Aget_type(attr));
  size_t length = H5Tget_size(stringdatatype);
  assert(length > 0);
  // before carpet-6-init-413-g68ba001 we did not store the terminating NUL
  // in the attribute and therefore have to provide it here
  vector<char> varname(length + 1);
  CHECK_HDF5(H5Aread(attr, stringdatatype, &varname[0]));
  CHECK_HDF5(H5Aclose(attr));

  // read the dimensions
  hid_t dataspace = -1;
  CHECK_HDF5(dataspace = H5Dget_space(dataset));
  int ndims;
  ndims = H5Sget_simple_extent_ndims(dataspace);
  hsize_t dims[3];
  int iorigin[3], cctk_nghostzones[3];
  double origin[3], delta[3];
  int itimestep = 0, level = 0;

  bool is_okay = false;
  if (typeclass != H5T_FLOAT and typeclass != H5T_INTEGER) {
    if (verbose) {
      cerr << "skipping dataset '" << datasetname << "':" << endl
           << "  is not of integer or floating-point datatype" << endl;
    }
  } else if (ndims != 3) {
    if (verbose) {
      cerr << "skipping dataset '" << datasetname << "':" << endl
           << "  dataset has " << ndims << " dimensions" << endl;
    }
  } else if (regex && regexec(&preg, datasetname, 0, NULL, 0)) {
    if (verbose) {
      cerr << "skipping dataset '" << datasetname << "':" << endl
           << "  name doesn't match regex" << endl;
    }
  } else if (timestep != PARAMETER_UNSET and
             fabs(timestep - time) > FUZZY_FACTOR) {
    if (verbose) {
      cerr << "skipping dataset '" << datasetname << "':" << endl
           << "  timestep (" << time << ") doesn't match" << endl;
    }
  } else {
    CHECK_HDF5(attr = H5Aopen_name(dataset, "iorigin"));
    CHECK_HDF5(H5Aread(attr, H5T_NATIVE_INT, iorigin));
    CHECK_HDF5(H5Aclose(attr));
    CHECK_HDF5(attr = H5Aopen_name(dataset, "origin"));
    CHECK_HDF5(H5Aread(attr, H5T_NATIVE_DOUBLE, origin));
    CHECK_HDF5(H5Aclose(attr));
    CHECK_HDF5(attr = H5Aopen_name(dataset, "delta"));
    CHECK_HDF5(H5Aread(attr, H5T_NATIVE_DOUBLE, delta));
    CHECK_HDF5(H5Aclose(attr));
    CHECK_HDF5(attr = H5Aopen_name(dataset, "timestep"));
    CHECK_HDF5(H5Aread(attr, H5T_NATIVE_INT, &itimestep));
    CHECK_HDF5(H5Aclose(attr));
    CHECK_HDF5(attr = H5Aopen_name(dataset, "level"));
    CHECK_HDF5(H5Aread(attr, H5T_NATIVE_INT, &level));
    CHECK_HDF5(H5Aclose(attr));
    CHECK_HDF5(attr = H5Aopen_name(dataset, "cctk_nghostzones"));
    CHECK_HDF5(H5Aread(attr, H5T_NATIVE_INT, cctk_nghostzones));
    CHECK_HDF5(H5Aclose(attr));
    CHECK_HDF5(H5Sget_simple_extent_dims(dataspace, dims, NULL));

    int i;
    is_okay = out3D;
    if (not out3D) {
      for (i = 0; i < 3; i++) {
        if (slab_coord[i] != PARAMETER_UNSET) {
          is_okay = slab_coord[i] >= origin[i] and
                    slab_coord[i] <= origin[i] + (dims[2 - i] - 1) * delta[i];
          if (not is_okay)
            break;
        }
      }
    }
    if (not is_okay) {
      if (verbose) {
        cerr << "skipping dataset '" << datasetname << "':" << endl
             << "  slab " << slab_coord[i] << " is out of dataset range ["
             << origin[i] << ", " << origin[i] + (dims[2 - i] - 1) * delta[i]
             << "]" << endl;
      }
    }
  }

  if (not is_okay) {
    CHECK_HDF5(H5Tclose(stringdatatype));
    CHECK_HDF5(H5Tclose(datatype));
    CHECK_HDF5(H5Dclose(dataset));
    return (0);
  }

  slices_extracted++;

  h5size_t slabstart[3] = {0, 0, 0};
  hsize_t slabcount[3] = {dims[0], dims[1], dims[2]};
  hsize_t outslabcount[3];
  double slice_origin[3], slice_delta[3];
  int slice_iorigin[3], slice_cctk_nghostzones[3];
  hsize_t j = 0;
  for (int i = 0; i < 3; i++) {
    if (slab_coord[i] != PARAMETER_UNSET) {
      slabstart[2 - i] =
          (h5size_t)((slab_coord[i] - origin[i]) / delta[i] + 0.5);
      slabcount[2 - i] = 1;
    } else {
      outslabcount[outrank - j - 1] = dims[2 - i];
      slice_origin[j] = origin[i];
      slice_delta[j] = delta[i];
      slice_iorigin[j] = iorigin[i];
      slice_cctk_nghostzones[j] = cctk_nghostzones[i];
      j++;
    }
  }
  assert(j == outrank);

  hid_t slabspace;
  CHECK_HDF5(slabspace = H5Screate_simple(3, slabcount, NULL));
  CHECK_HDF5(H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, slabstart, NULL,
                                 slabcount, NULL));
  const hssize_t npoints = H5Sget_select_npoints(dataspace);
  // make sure the vector allocates at least one element
  char *data = new char[(npoints + 1) * H5Tget_size(datatype)];
  CHECK_HDF5(
      H5Dread(dataset, datatype, slabspace, dataspace, H5P_DEFAULT, data));
  CHECK_HDF5(H5Dclose(dataset));
  CHECK_HDF5(H5Sclose(slabspace));
  CHECK_HDF5(H5Sclose(dataspace));

  // enable compression if requested
  hid_t plist;
  CHECK_HDF5(plist = H5Pcreate(H5P_DATASET_CREATE));
  const int compression_lvl = compression_level;

  if (compression_lvl) {
    CHECK_HDF5(H5Pset_chunk(plist, outrank, outslabcount));
    CHECK_HDF5(H5Pset_shuffle(plist));
    CHECK_HDF5(H5Pset_deflate(plist, compression_lvl));
  }

  CHECK_HDF5(dataspace = H5Screate_simple(outrank, outslabcount, NULL));
  CHECK_HDF5(dataset =
                 H5Dcreate(outfile, datasetname, datatype, dataspace, plist));
  CHECK_HDF5(H5Dwrite(dataset, datatype, H5S_ALL, H5S_ALL, H5P_DEFAULT, data));
  delete[] data;
  CHECK_HDF5(H5Pclose(plist));
  CHECK_HDF5(H5Tclose(datatype));
  CHECK_HDF5(H5Sclose(dataspace));

  // write global attributes (only on the first time through)
  static bool first_time_through = true;
  if (first_time_through) {
    hid_t metadata_group;
    CHECK_HDF5(metadata_group = H5Gcreate(outfile, METADATA_GROUP, 0));
    CHECK_HDF5(dataspace = H5Screate(H5S_SCALAR));
    CHECK_HDF5(attr = H5Acreate(metadata_group, "nioprocs", H5T_NATIVE_INT,
                                dataspace, H5P_DEFAULT));
    const int nioprocs = 1;
    CHECK_HDF5(H5Awrite(attr, H5T_NATIVE_INT, &nioprocs));
    CHECK_HDF5(H5Aclose(attr));
    CHECK_HDF5(H5Sclose(dataspace));
    CHECK_HDF5(H5Gclose(metadata_group));
    first_time_through = false;
  }

  // write basic dataset attributes
  CHECK_HDF5(dataspace = H5Screate_simple(1, &outrank, NULL));
  CHECK_HDF5(attr = H5Acreate(dataset, "origin", H5T_NATIVE_DOUBLE, dataspace,
                              H5P_DEFAULT));
  CHECK_HDF5(H5Awrite(attr, H5T_NATIVE_DOUBLE, slice_origin));
  CHECK_HDF5(H5Aclose(attr));
  CHECK_HDF5(attr = H5Acreate(dataset, "delta", H5T_NATIVE_DOUBLE, dataspace,
                              H5P_DEFAULT));
  CHECK_HDF5(H5Awrite(attr, H5T_NATIVE_DOUBLE, slice_delta));
  CHECK_HDF5(H5Aclose(attr));
  CHECK_HDF5(attr = H5Acreate(dataset, "iorigin", H5T_NATIVE_INT, dataspace,
                              H5P_DEFAULT));
  CHECK_HDF5(H5Awrite(attr, H5T_NATIVE_INT, slice_iorigin));
  CHECK_HDF5(H5Aclose(attr));
  CHECK_HDF5(attr = H5Acreate(dataset, "cctk_nghostzones", H5T_NATIVE_INT,
                              dataspace, H5P_DEFAULT));
  CHECK_HDF5(H5Awrite(attr, H5T_NATIVE_INT, slice_cctk_nghostzones));
  CHECK_HDF5(H5Aclose(attr));
  CHECK_HDF5(H5Sclose(dataspace));
  CHECK_HDF5(dataspace = H5Screate(H5S_SCALAR));
  CHECK_HDF5(attr = H5Acreate(dataset, "time", H5T_NATIVE_DOUBLE, dataspace,
                              H5P_DEFAULT));
  CHECK_HDF5(H5Awrite(attr, H5T_NATIVE_DOUBLE, &time));
  CHECK_HDF5(H5Aclose(attr));
  CHECK_HDF5(attr = H5Acreate(dataset, "timestep", H5T_NATIVE_INT, dataspace,
                              H5P_DEFAULT));
  CHECK_HDF5(H5Awrite(attr, H5T_NATIVE_INT, &itimestep));
  CHECK_HDF5(H5Aclose(attr));
  CHECK_HDF5(attr = H5Acreate(dataset, "level", H5T_NATIVE_INT, dataspace,
                              H5P_DEFAULT));
  CHECK_HDF5(H5Awrite(attr, H5T_NATIVE_INT, &level));
  CHECK_HDF5(H5Aclose(attr));
  CHECK_HDF5(attr = H5Acreate(dataset, "name", stringdatatype, dataspace,
                              H5P_DEFAULT));
  CHECK_HDF5(H5Awrite(attr, stringdatatype, &varname[0]));
  CHECK_HDF5(H5Tclose(stringdatatype));
  CHECK_HDF5(H5Sclose(dataspace));
  CHECK_HDF5(H5Dclose(dataset));

  return 0;
}

```

util/hdf5_recombiner.cc:
```
#include <cassert>
#include <cstdio>
#include <cstring>
#include <iostream>
#include <list>
#include <string>
#include <vector>

#define H5_USE_16_API 1
#include <cctk_Config.h>

#include <hdf5.h>

using namespace std;

// Global constants
int const dim = 3;

// Files and datasets
int const num_input_files = 0;
char const *const input_file_pattern = "interptoarray::parrays3d.file_%d.h5";
int const iteration_divisor = 1024;
char const *const output_file_name = "parrays3d-float.h5";
char const *const output_dataset_name = "phi";
hsize_t const output_dims[] = {0, 1024, 1024, 1024}; // [t,z,y,x]
hsize_t const output_maxdims[] = {H5S_UNLIMITED, 1024, 1024, 1024};
typedef float output_t;
hid_t const output_type = H5T_NATIVE_FLOAT;

// Technical details
hsize_t const chunk_size[] = {1, 128, 128, 128}; // 8 MB for float
int const compression_level = 1;
bool const write_checksum = true;

// Check a return value
#define check(_expr)                                                           \
  do {                                                                         \
    bool const _val = (_expr);                                                 \
    assert(_val);                                                              \
  } while (0)

static herr_t add_name(hid_t group, const char *name, H5L_info_t const *info,
                       void *op_data);

int main(int argc, char **argv) {
  cout << "hdf5_recombiner" << endl
       << "Copyright 2009 Erik Schnetter <schnetter@cct.lsu.edu>" << endl
       << endl;

  cout << "Opening output file \"" << output_file_name << "\"" << endl;

  htri_t is_hdf5;
  H5E_BEGIN_TRY { is_hdf5 = H5Fis_hdf5(output_file_name); }
  H5E_END_TRY;
  bool const file_exists = is_hdf5 > 0;
  hid_t const output_file =
      file_exists
          ? H5Fopen(output_file_name, H5F_ACC_RDWR, H5P_DEFAULT)
          : H5Fcreate(output_file_name, H5F_ACC_EXCL, H5P_DEFAULT, H5P_DEFAULT);

  cout << "Opening output dataset \"" << output_dataset_name << "\"" << endl;

  hid_t const output_datatype = output_type;

  hid_t const output_dataspace =
      H5Screate_simple(dim + 1, output_dims, output_maxdims);
  assert(output_dataspace >= 0);

  hid_t const output_properties = H5Pcreate(H5P_DATASET_CREATE);
  assert(output_properties >= 0);
  check(not H5Pset_chunk(output_properties, dim + 1, chunk_size));
  if (compression_level > 0) {
    check(not H5Pset_shuffle(output_properties));
    check(not H5Pset_deflate(output_properties, compression_level));
  }
  if (write_checksum) {
    check(not H5Pset_fletcher32(output_properties));
  }

  hid_t const output_dataset =
      file_exists ? H5Dopen(output_file, output_dataset_name)
                  : H5Dcreate(output_file, output_dataset_name, output_datatype,
                              output_dataspace, output_properties);
  assert(output_dataset >= 0);

  list<string> input_file_names;
  for (int input_file_num = 0; input_file_num < num_input_files;
       ++input_file_num) {
    char input_file_name[10000];
    snprintf(input_file_name, sizeof input_file_name, input_file_pattern,
             input_file_num);
    input_file_names.push_back(input_file_name);
  }
  for (int n = 1; n < argc; ++n) {
    input_file_names.push_back(argv[n]);
  }

  for (list<string>::const_iterator iinput_file_name = input_file_names.begin();
       iinput_file_name != input_file_names.end(); ++iinput_file_name) {
    string const &input_file_name = *iinput_file_name;
    cout << "Opening input file \"" << input_file_name << "\"" << endl;
    hid_t const input_file =
        H5Fopen(input_file_name.c_str(), H5F_ACC_RDWR, H5P_DEFAULT);
    assert(input_file >= 0);

    typedef list<string> names_t;
    names_t names;
    hsize_t idx = 0;
    check(not H5Literate(input_file, H5_INDEX_NAME, H5_ITER_NATIVE, &idx,
                         add_name, &names));
    for (names_t::const_iterator iname = names.begin(); iname != names.end();
         ++iname) {
      char const *const input_dataset_name = iname->c_str();
      cout << "Reading input dataset \"" << input_dataset_name << "\"" << endl;

      char varname[10000];
      int it, tl, c;
      int const icnt = sscanf(input_dataset_name, "%s it=%d tl=%d c=%d",
                              varname, &it, &tl, &c);
      assert(icnt == 4);
      assert(it % iteration_divisor == 0);
      int const iteration = it / iteration_divisor;

      hid_t const input_dataset = H5Dopen(input_file, input_dataset_name);
      assert(input_dataset >= 0);

      int iorigin[dim];
      hid_t const iorigin_attr = H5Aopen(input_dataset, "iorigin", H5P_DEFAULT);
      assert(iorigin_attr >= 0);
      check(not H5Aread(iorigin_attr, H5T_NATIVE_INT, iorigin));
      check(not H5Aclose(iorigin_attr));

      hid_t const input_dataspace = H5Dget_space(input_dataset);
      assert(input_dataspace >= 0);
      assert(H5Sis_simple(input_dataspace) > 0);
      assert(H5Sget_simple_extent_ndims(input_dataspace) == dim);
      hsize_t input_dims[dim];
      check(H5Sget_simple_extent_dims(input_dataspace, input_dims, NULL) ==
            dim);

      hid_t const input_memory_dataspace =
          H5Screate_simple(dim, input_dims, NULL);
      assert(input_memory_dataspace);
      hssize_t const input_memory_npoints =
          H5Sget_simple_extent_npoints(input_memory_dataspace);
      vector<output_t> data(input_memory_npoints);

      check(not H5Dread(input_dataset, output_type, input_memory_dataspace,
                        input_dataspace, H5P_DEFAULT, &data.front()));

      check(not H5Sclose(input_memory_dataspace));
      check(not H5Sclose(input_dataspace));
      check(not H5Dclose(input_dataset));

      cout << "Writing output dataset" << endl;

      hsize_t output_extent[dim + 1];
      output_extent[0] = iteration + 1;
      for (int d = 0; d < dim; ++d) {
        output_extent[d + 1] = output_dims[d + 1];
      }
      check(not H5Dextend(output_dataset, output_extent));

      hid_t const output_dataspace =
          H5Screate_simple(dim + 1, output_extent, output_maxdims);
      assert(output_dataspace >= 0);

      hsize_t output_offset[dim + 1];
      hsize_t output_dims[dim + 1];
      output_offset[0] = iteration;
      output_dims[0] = 1;
      for (int d = 0; d < dim; ++d) {
        output_offset[d + 1] = iorigin[dim - 1 - d];
        output_dims[d + 1] = input_dims[d];
      }
      check(not H5Sselect_hyperslab(output_dataspace, H5S_SELECT_SET,
                                    output_offset, NULL, output_dims, NULL));
      cout << "   extent [" << output_offset[3] << "," << output_offset[2]
           << "," << output_offset[1] << "," << output_offset[0] << "] - ["
           << output_offset[3] + output_dims[3] << ","
           << output_offset[2] + output_dims[2] << ","
           << output_offset[1] + output_dims[1] << ","
           << output_offset[0] + output_dims[0] << "] "
           << "(" << data.size() << " points)" << endl;

      hid_t const output_memory_dataspace =
          H5Screate_simple(dim + 1, output_dims, NULL);
      assert(output_memory_dataspace);
      hssize_t const output_memory_npoints =
          H5Sget_simple_extent_npoints(output_memory_dataspace);
      assert(output_memory_npoints == input_memory_npoints);

      check(not H5Dwrite(output_dataset, output_type, output_memory_dataspace,
                         output_dataspace, H5P_DEFAULT, &data.front()));

      H5Sclose(output_memory_dataspace);
      H5Sclose(output_dataspace);

    } // for names

    check(not H5Fclose(input_file));

  } // for input_file_num

  check(not H5Dclose(output_dataset));
  check(not H5Pclose(output_properties));
  check(not H5Sclose(output_dataspace));
  check(not H5Fclose(output_file));

  cout << "Done." << endl;
  return 0;
}

static herr_t add_name(hid_t const group, const char *const name,
                       H5L_info_t const *const info, void *const op_data) {
  typedef list<string> names_t;
  names_t &names = *static_cast<names_t *>(op_data);
  if (strcmp(name, "Parameters and Global Attributes") != 0) {
    names.push_back(name);
  }
  return 0;
}

```

util/hdf5tobinary_slicer.cc:
```
/*@@
  @file      hdf5tobinary_slicer.cc
  @date      April 23, 2009
  @author    Erik Schnetter, based on hdf5toascii_slicer.cc by Thomas Radke
  @desc
             Utility program to extract a 1D line or 2D slice from 3D datasets
             in datafiles generated by CarpetIOHDF5.
             The extracted slab is written to a binary file.
  @enddesc
@@*/

#include <algorithm>
#include <iostream>
#include <iomanip>
#include <limits>
#include <sstream>
#include <string>
#include <vector>
#include <cassert>
#include <cmath>
#include <cstdio>
#include <cstring>
#include <regex.h>

#define H5_USE_16_API 1
#include <hdf5.h>

using namespace std;

/*****************************************************************************
 *************************     Macro Definitions   ***************************
 *****************************************************************************/

// uncomment the following line to get some debug output
// #define DEBUG 1

// value for an unset parameter
#define PARAMETER_UNSET -424242.0

// fuzzy factor for comparing dataset timesteps with user-specified value
#define FUZZY_FACTOR 1e-6

// the datatype for the 'start' argument in H5Sselect_hyperslab()
#if (H5_VERS_MAJOR == 1 &&                                                     \
     (H5_VERS_MINOR < 6 || (H5_VERS_MINOR == 6 && H5_VERS_RELEASE < 4)))
#define h5size_t hssize_t
#else
#define h5size_t hsize_t
#endif

// macro to check the return code of calls to the HDF5 library
#define CHECK_HDF5(fn_call)                                                    \
  {                                                                            \
    const int _retval = fn_call;                                               \
    if (_retval < 0) {                                                         \
      cerr << "HDF5 call " << #fn_call << " in file " << __FILE__ << " line "  \
           << __LINE__ << " returned error code " << _retval << endl;          \
    }                                                                          \
  }

/*****************************************************************************
 **************************     Type Definitions   ***************************
 *****************************************************************************/

typedef struct {
  hid_t file;
  string name;
  bool is_real;
  hsize_t dims[3]; // C order

  int map, mglevel, component, iteration, timelevel, rflevel;
  double time;
  int iorigin[3];             // Fortran order
  double origin[3], delta[3]; // Fortran order
} patch_t;

/*****************************************************************************
 *************************       Global Data         *************************
 *****************************************************************************/

// the slab coordinate as selected by the user (Fortan order)
static double slab_coord[3] = {PARAMETER_UNSET, PARAMETER_UNSET,
                               PARAMETER_UNSET};

// flag for outputting data in full 3D
static bool out3D = false;

// the specific timestep selected by the user
static double timestep = PARAMETER_UNSET;

// the regular expression to match against dataset names
static const char *regex = NULL;
static regex_t preg;

// the dataset name, if given explicitly
static const char *datasetname = NULL;

// the list of all patches
static vector<patch_t> patchlist;

// maximum refinement level across all patches
static int max_rflevel = -1;

// whether to print omitted patches
static bool verbose = false;

// pointer to array that is filled with the data (Fortran order)
static int datadims[3] = {-1, -1, -1};
static float *dataptr = NULL;
static unsigned char *bdataptr = NULL;

// output file name
static char *basename1 = NULL;

// output format, either raw binary or HDF5
static bool out_hdf5 = false;
static int out_hdf5_4d_index = -1;

// output type (float or byte)
static bool out_float = false;
static float data_min = -1.0;
static float data_max = +1.0;

/*****************************************************************************
 *************************     Function Prototypes   *************************
 *****************************************************************************/
static herr_t FindPatches(hid_t group, const char *name, void *_file);
static void ReadPatch(const patch_t &patch, int last_iteration);
static bool ComparePatches(const patch_t &a, const patch_t &b);

/*@@
  @routine    main
  @date       Sun 30 July 2006
  @author     Thomas Radke
  @desc
              Evaluates command line options and opens the input files.
  @enddesc

  @var        argc
  @vdesc      number of command line parameters
  @vtype      int
  @vio        in
  @endvar
  @var        argv
  @vdesc      array of command line parameters
  @vtype      char* const []
  @vio        in
  @endvar
@@*/
int main(int argc, char *const argv[]) {
  int i;
  bool help = false;
#if 0
  int iorigin[3] = {-1, -1, -1};
#endif
  int num_slab_options = 0;

  // evaluate command line parameters
  for (i = 1; i < argc; i++) {
    if (strcmp(argv[i], "--help") == 0) {
      help = true;
      break;
    } else if (strcmp(argv[i], "--verbose") == 0) {
      verbose = true;
    } else if (strcmp(argv[i], "--out3d-cube") == 0) {
      out3D = true;
    } else if (strcmp(argv[i], "--timestep") == 0 and i + 1 < argc) {
      timestep = atof(argv[++i]);
    } else if (strcmp(argv[i], "--match") == 0 and i + 1 < argc) {
      regex = argv[++i];
      if (regcomp(&preg, regex, REG_EXTENDED)) {
        cerr << "Error: invalid regular expression '" << regex << "' given"
             << endl
             << endl;
        return (-1);
      }
    } else if (strcmp(argv[i], "--dataset") == 0 and i + 1 < argc) {
      datasetname = argv[++i];
    } else if (strcmp(argv[i], "--out-precision") == 0 and i + 1 < argc) {
      cout << setprecision(atoi(argv[++i]));
    } else if (strcmp(argv[i], "--out0d-point") == 0 and i + 3 < argc) {
      slab_coord[0] = atof(argv[++i]);
      slab_coord[1] = atof(argv[++i]);
      slab_coord[2] = atof(argv[++i]);
      num_slab_options++;
    } else if (strcmp(argv[i], "--out1d-xline-yz") == 0 and i + 2 < argc) {
      slab_coord[1] = atof(argv[++i]);
      slab_coord[2] = atof(argv[++i]);
      num_slab_options++;
    } else if (strcmp(argv[i], "--out1d-yline-xz") == 0 and i + 2 < argc) {
      slab_coord[0] = atof(argv[++i]);
      slab_coord[2] = atof(argv[++i]);
      num_slab_options++;
    } else if (strcmp(argv[i], "--out1d-zline-xy") == 0 and i + 2 < argc) {
      slab_coord[0] = atof(argv[++i]);
      slab_coord[1] = atof(argv[++i]);
      num_slab_options++;
    } else if (strcmp(argv[i], "--out2d-yzplane-x") == 0 and i + 1 < argc) {
      slab_coord[0] = atof(argv[++i]);
      num_slab_options++;
    } else if (strcmp(argv[i], "--out2d-xzplane-y") == 0 and i + 1 < argc) {
      slab_coord[1] = atof(argv[++i]);
      num_slab_options++;
    } else if (strcmp(argv[i], "--out2d-xyplane-z") == 0 and i + 1 < argc) {
      slab_coord[2] = atof(argv[++i]);
      num_slab_options++;
#if 0
    } else if (strcmp (argv[i], "--out2d-yzplane-xi") == 0 and i+1 < argc) {
      iorigin[0] = atoi (argv[++i]); num_slab_options++;
    } else if (strcmp (argv[i], "--out2d-xzplane-yi") == 0 and i+1 < argc) {
      iorigin[1] = atoi (argv[++i]); num_slab_options++;
    } else if (strcmp (argv[i], "--out2d-xyplane-zi") == 0 and i+1 < argc) {
      iorigin[2] = atoi (argv[++i]); num_slab_options++;
#endif
    } else if (strcmp(argv[i], "--size") == 0 and i + 3 < argc) {
      datadims[0] = atof(argv[++i]);
      datadims[1] = atof(argv[++i]);
      datadims[2] = atof(argv[++i]);
    } else if (strcmp(argv[i], "--basename") == 0 and i + 1 < argc) {
      basename1 = argv[++i];
    } else if (strcmp(argv[i], "--out-hdf5") == 0) {
      out_hdf5 = true;
    } else if (strcmp(argv[i], "--out-float") == 0) {
      out_float = true;
    } else if (strcmp(argv[i], "--data-min") == 0 and i + 1 < argc) {
      data_min = atof(argv[++i]);
    } else if (strcmp(argv[i], "--data-max") == 0 and i + 1 < argc) {
      data_max = atof(argv[++i]);
    } else if (strcmp(argv[i], "--out-hdf5-4d-index") == 0 and i + 1 < argc) {
      out_hdf5_4d_index = atof(argv[++i]);
    } else {
      break;
    }
  }

  /* give some help if called with incorrect number of parameters */
  if (help or i >= argc or num_slab_options != (out3D ? 0 : 1)) {
    const string indent(strlen(argv[0]) + 1, ' ');
    cerr << endl
         << "   ---------------------------" << endl
         << "   Carpet HDF5 to ASCII Slicer" << endl
         << "   ---------------------------" << endl
         << endl
         << "Usage: " << endl
         << argv[0] << " [--help]" << endl
         << indent << "[--match <regex string>]" << endl
         << indent << "[--dataset <string (can contain %p)>]" << endl
         << indent << "[--out-precision <digits>]" << endl
         << indent << "[--timestep <cctk_time value>]" << endl
         << indent << "[--verbose]" << endl
         << indent << "<--out0d-point value value value> | <--out1d-line value "
                      "value> | <--out2d-plane value> | <out3d-cube>"
         << endl
         << indent << "--size <ni> <nj> <nk>" << endl
         << indent << "--basename <filename>" << endl
         << indent << "[--out-hdf5]" << endl
         << indent << "[--out-float]" << endl
         << indent << "[--out-hdf5-4d-index <t index>]" << endl
         << indent << "<hdf5_infiles>" << endl
         << endl
         << "  where" << endl
         << "    [--help]                         prints this help" << endl
         << "    [--match <regex string>]         selects HDF5 datasets by "
            "their names"
         << endl
         << "                                     matching a regex string "
            "using POSIX"
         << endl
         << "                                     Extended Regular Expression "
            "syntax"
         << endl
         << "    [--out-precision <digits>]       sets the output precision"
         << endl
         << "                                     for floating-point numbers"
         << endl
         << "    [--timestep <cctk_time value>]   selects all HDF5 datasets "
            "which"
         << endl
         << "                                     (fuzzily) match the "
            "specified time"
         << endl
         << "    [--verbose]                      lists skipped HDF5 datasets "
            "on stderr"
         << endl
         << endl
         << "  and either --out0d-point value value value> or <--out1d-line "
            "value value> or <--out2d-plane value> or <--out3d-cube>"
         << endl
         << "  must be specified as in the following:" << endl
         << "    --out0d-point     <origin_x> <origin_y> <origin_z>" << endl
         << endl
         << "    --out1d-xline-yz  <origin_y> <origin_z>" << endl
         << "    --out1d-yline-xz  <origin_x> <origin_z>" << endl
         << "    --out1d-zline-xy  <origin_x> <origin_y>" << endl
         << endl
         << "    --out2d-yzplane-x  <origin_x>" << endl
         << "    --out2d-xzplane-y  <origin_y>" << endl
         << "    --out2d-xyplane-z  <origin_z>" << endl
         << endl
         << "    --out3d-cube" << endl
#if 0
         << "    --out2d-yzplane-xi <origin_xi>" << endl
         << "    --out2d-xzplane-yi <origin_yi>" << endl
         << "    --out2d-xyplane-zi <origin_zi>" << endl
#endif
         << endl
         << "  eg, to extract a 2D xy-plane at z = 0:" << endl
         << "    " << argv[0] << " --out2d-xyplane-z 0 alp.file_*.h5" << endl
         << endl
         << "  or the same plane but only for datasets of refinement level 0:"
         << endl
         << "    " << argv[0] << " --match 'ADMBASE::alp it=[0-9]+ tl=0 rl=0' "
                                 "--out2d-xyplane-z 0 alp.file_*.h5"
         << endl;
    return (0);
  }

  if (not basename1) {
    cerr << "Parameter --basename is not set" << endl;
    return 1;
  }

  if (regex and datasetname) {
    cerr << "Parameters --match and --dataset cannot be used together" << endl;
    return 1;
  }

  {
    if (datadims[0] < 0 || datadims[1] < 0 || datadims[2] < 0) {
      cerr << "Parameter --size is not set" << endl;
      return 1;
    }
    size_t const npoints = (size_t)datadims[0] * datadims[1] * datadims[2];
    dataptr = new float[npoints];
    assert(dataptr);
    bdataptr = new unsigned char[npoints];
    assert(bdataptr);
  }

  vector<hid_t> filelist;
  for (; i < argc; i++) {
    hid_t file;

    H5E_BEGIN_TRY { file = H5Fopen(argv[i], H5F_ACC_RDONLY, H5P_DEFAULT); }
    H5E_END_TRY;

    if (file < 0) {
      cerr << "Could not open Carpet HDF5 input file '" << argv[i] << "'"
           << endl
           << endl;
      continue;
    }

    if (not datasetname) {
      cout << "File " << argv[i] << ":" << endl;
      size_t const before = patchlist.size();
      CHECK_HDF5(H5Giterate(file, "/", NULL, FindPatches, &file));
      size_t const after = patchlist.size();
      size_t const count = after - before;
      cout << "   will read " << count << " "
           << (count == 1 ? "patch" : "patches") << endl;
    } else {
      const string ppmarker("%p");
      const string filemarker(".file_");
      string name(datasetname);
      const size_t pppos = name.find(ppmarker);
      if (pppos != string::npos) {
        const string filename(argv[i]);
        const size_t procpos = filename.find(filemarker);
        assert(procpos != string::npos);
        const int proc = atoi(filename.c_str() + procpos + filemarker.length());
        ostringstream namebuf;
        namebuf << name.substr(0, pppos) << proc
                << name.substr(pppos + ppmarker.length());
        name = namebuf.str();
      }
      cout << "   Examining dataset " << name << endl;
      hid_t group;
      CHECK_HDF5(group = H5Dopen(file, name.c_str()));
      FindPatches(group, name.c_str(), &file);
      CHECK_HDF5(H5Dclose(group));
    }

    filelist.push_back(file);
  }

  if (!patchlist.empty()) {
    /* now sort the patches by their time attribute */
    sort(patchlist.begin(), patchlist.end(), ComparePatches);

    cout << "# 1D ASCII output created by";
    for (int j = 0; j < argc; j++)
      cout << " " << argv[j];
    cout << endl << "#" << endl;

    int last_iteration = patchlist[0].iteration - 1;
    for (size_t j = 0; j < patchlist.size(); j++) {
      ReadPatch(patchlist[j], last_iteration);
      last_iteration = patchlist[j].iteration;
    }
  } else {
    cerr << "No valid datasets found" << endl << endl;
  }

  // find minimum and maximum
  {
    float const float_max = numeric_limits<float>::max();
    float data_min1 = +float_max;
    float data_max1 = -float_max;
    double data_sum = 0.0;
    double data_sum2 = 0.0;
    double data_count = 0.0;
    size_t const npoints = (size_t)datadims[0] * datadims[1] * datadims[2];
    for (size_t n = 0; n < npoints; ++n) {
      float const data = dataptr[n];
      if (data > -sqrt(float_max) and data < sqrt(float_max)) {
        data_min1 = min(data_min1, data);
        data_max1 = max(data_max1, data);
        data_sum += (double)data;
        data_sum2 += pow((double)data, 2.0);
        ++data_count;
      }
    }
    double const data_avg = data_sum / data_count;
    double const data_stddev =
        sqrt(max(0.0, data_sum2 / data_count - pow(data_avg, 2)));

    cerr << "Minimum:            " << data_min1 << endl
         << "Maximum:            " << data_max1 << endl
         << "Average:            " << data_avg << endl
         << "Standard deviation: " << data_stddev << endl
         << "Valid points:       " << data_count << endl;
  }

  // convert from float to byte
  if (not out_float) {
    unsigned char const byte_min = 0;
    unsigned char const byte_max = 255;
    float const scale_factor =
        ((float)byte_max - (float)byte_min + 1) / (data_max - data_min);
    size_t const npoints = (size_t)datadims[0] * datadims[1] * datadims[2];
    for (size_t n = 0; n < npoints; ++n) {
      float const data = dataptr[n];
      float const data_scaled =
          (float)byte_min + floor((data - data_min) * scale_factor);
      float const data_clamped =
          max((float)byte_min, min((float)byte_max, data_scaled));
      unsigned char const bdata = (unsigned char)data_clamped;
      bdataptr[n] = bdata;
    }
  }

  hid_t const out_type = out_float ? H5T_NATIVE_FLOAT : H5T_NATIVE_UCHAR;
  size_t const out_size = out_float ? sizeof *dataptr : sizeof *bdataptr;
  void const *const out_data = out_float ? static_cast<void const *>(dataptr)
                                         : static_cast<void const *>(bdataptr);

  // write output files
  if (out_hdf5) {
    // HDF5 output
    if (out_hdf5_4d_index == -1) {
      // 3D output
      char filename[1000];
      snprintf(filename, sizeof filename, "%s.h5", basename1);
      hid_t const outfile =
          H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
      assert(outfile >= 0);
      hsize_t const dims[3] = {(hsize_t)datadims[2], (hsize_t)datadims[1],
                               (hsize_t)datadims[0]};
      hid_t const dataspace = H5Screate_simple(3, dims, NULL);
      assert(dataspace >= 0);
      hid_t const dataset_properties = H5Pcreate(H5P_DATASET_CREATE);
      assert(dataset_properties >= 0);
      hid_t const dataset =
          H5Dcreate(outfile, "data", out_type, dataspace, dataset_properties);
      assert(dataset >= 0);
      hid_t const transfer_properties = H5Pcreate(H5P_DATASET_XFER);
      assert(transfer_properties >= 0);
      {
        herr_t const herr = H5Dwrite(dataset, out_type, H5S_ALL, H5S_ALL,
                                     transfer_properties, out_data);
        assert(herr >= 0);
      }
      CHECK_HDF5(H5Dclose(dataset));
      CHECK_HDF5(H5Sclose(dataspace));
      CHECK_HDF5(H5Fclose(outfile));
    } else {
      // 4d output
      char filename[1000];
      snprintf(filename, sizeof filename, "%s.h5", basename1);
      htri_t is_hdf5;
      H5E_BEGIN_TRY { is_hdf5 = H5Fis_hdf5(filename); }
      H5E_END_TRY;
      bool const file_is_new = not(is_hdf5 > 0);
      hid_t const outfile =
          file_is_new
              ? H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)
              : H5Fopen(filename, H5F_ACC_RDWR, H5P_DEFAULT);
      assert(outfile >= 0);

      hsize_t const dims[4] = {(hsize_t)0, (hsize_t)datadims[2],
                               (hsize_t)datadims[1], (hsize_t)datadims[0]};
      hsize_t const maxdims[4] = {(hsize_t)H5S_UNLIMITED, (hsize_t)datadims[2],
                                  (hsize_t)datadims[1], (hsize_t)datadims[0]};
      hid_t const dataspace = H5Screate_simple(4, dims, maxdims);
      assert(dataspace >= 0);
      hid_t const dataset_properties = H5Pcreate(H5P_DATASET_CREATE);
      assert(dataset_properties >= 0);
      // hsize_t const chunk_dims[4] = {1, 32, 32, 32 };
      // Chunks must be < 4 GByte
      // hsize_t const chunk_dims[4] =
      //   {1, datadims[2], datadims[1], datadims[0] };
      hsize_t const chunk_dims[4] = {(hsize_t)1, (hsize_t)min(128, datadims[2]),
                                     (hsize_t)min(128, datadims[1]),
                                     (hsize_t)min(128, datadims[0])};
      CHECK_HDF5(H5Pset_chunk(dataset_properties, 4, chunk_dims));
      int const compression_level = 1;
      CHECK_HDF5(H5Pset_shuffle(dataset_properties));
      CHECK_HDF5(H5Pset_deflate(dataset_properties, compression_level));
      hid_t const dataset = file_is_new
                                ? H5Dcreate(outfile, "data", out_type,
                                            dataspace, dataset_properties)
                                : H5Dopen(outfile, "data");
      assert(dataset >= 0);
      // TODO: assert that the dataset's dataspace has the correct
      // dimensions

      hsize_t const extended_dims[4] = {
          (hsize_t)out_hdf5_4d_index + 1, (hsize_t)datadims[2],
          (hsize_t)datadims[1], (hsize_t)datadims[0]};
      CHECK_HDF5(H5Dextend(dataset, extended_dims));

      // for debugging only
      // CHECK_HDF5 (H5Fflush (outfile, H5F_SCOPE_GLOBAL));

      hsize_t const memory_dims[4] = {(hsize_t)1, (hsize_t)datadims[2],
                                      (hsize_t)datadims[1],
                                      (hsize_t)datadims[0]};
      hid_t const memory_dataspace = H5Screate_simple(4, memory_dims, NULL);
      assert(memory_dataspace >= 0);

      hid_t const file_dataspace = H5Screate_simple(4, extended_dims, NULL);
      assert(file_dataspace >= 0);
      // hssize_t const file_offset[4] =
      //   { out_hdf5_4d_index, 0, 0, 0 };
      // CHECK_HDF5 (H5Soffset_simple (file_dataspace, file_offset));
      hsize_t const file_offset[4] = {(hsize_t)out_hdf5_4d_index, (hsize_t)0,
                                      (hsize_t)0, (hsize_t)0};
      CHECK_HDF5(H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET,
                                     file_offset, NULL, memory_dims, NULL));

      hid_t const transfer_properties = H5Pcreate(H5P_DATASET_XFER);
      assert(transfer_properties >= 0);
      CHECK_HDF5(H5Dwrite(dataset, out_type, memory_dataspace, file_dataspace,
                          transfer_properties, out_data));

      CHECK_HDF5(H5Sclose(file_dataspace));
      CHECK_HDF5(H5Sclose(memory_dataspace));
      CHECK_HDF5(H5Dclose(dataset));
      CHECK_HDF5(H5Sclose(dataspace));
      CHECK_HDF5(H5Fclose(outfile));
    }
  } else {
    // raw binary output
    {
      char filename[1000];
      snprintf(filename, sizeof filename, "%s.bin", basename1);
      FILE *const outfile = fopen(filename, "w");
      assert(outfile);
      size_t const npoints = (size_t)datadims[0] * datadims[1] * datadims[2];
      size_t const icnt = fwrite(out_data, out_size, npoints, outfile);
      assert(icnt == npoints);
      int const ierr = fclose(outfile);
      assert(not ierr);
    }
  }

  {
    delete[] dataptr;
    dataptr = NULL;
    delete[] bdataptr;
    bdataptr = NULL;
  }

  // close all input files
  for (size_t j = 0; j < filelist.size(); j++) {
    if (filelist[j] >= 0) {
      CHECK_HDF5(H5Fclose(filelist[j]));
    }
  }

  return (0);
}

/*@@
  @routine    FindPatches
  @date       Tue 8 August 2006
  @author     Thomas Radke
  @desc
              The worker routine which is called by H5Giterate().
              It checks whether the current HDF5 object is a patch matching
              the user's slab criteria, and adds it to the patches list.
  @enddesc

  @var        group
  @vdesc      HDF5 object to start the iteration
  @vtype      hid_t
  @vio        in
  @endvar
  @var        name
  @vdesc      name of the object at the current iteration
  @vtype      const char *
  @vio        in
  @endvar
  @var        _file
  @vdesc      pointer to the descriptor of the currently opened file
  @vtype      void *
  @vio        in
  @endvar
@@*/
static herr_t FindPatches(hid_t group, const char *name, void *_file) {
  int ndims;
  hid_t dataset, datatype, attr;
  H5G_stat_t object_info;
  H5T_class_t typeclass;
  patch_t patch;

  /* we are interested in datasets only - skip anything else */
  CHECK_HDF5(H5Gget_objinfo(group, name, 0, &object_info));
  if (object_info.type != H5G_DATASET) {
    return (0);
  }

  /* check the dataset's datatype - make sure it is either integer or real */
  CHECK_HDF5(dataset = H5Dopen(group, name));
  patch.name = name;

  CHECK_HDF5(datatype = H5Dget_type(dataset));
  CHECK_HDF5(typeclass = H5Tget_class(datatype));
  CHECK_HDF5(H5Tclose(datatype));

  // read the timestep
  CHECK_HDF5(attr = H5Aopen_name(dataset, "time"));
  CHECK_HDF5(H5Aread(attr, H5T_NATIVE_DOUBLE, &patch.time));
  CHECK_HDF5(H5Aclose(attr));

  // read the dimensions
  hid_t dataspace = -1;
  CHECK_HDF5(dataspace = H5Dget_space(dataset));
  ndims = H5Sget_simple_extent_ndims(dataspace);

  bool is_okay = false;
  if (typeclass != H5T_FLOAT and typeclass != H5T_INTEGER) {
    if (verbose) {
      cerr << "skipping dataset '" << name << "':" << endl
           << "  is not of integer or floating-point datatype" << endl;
    }
  } else if (ndims != 3) {
    if (verbose) {
      cerr << "skipping dataset '" << name << "':" << endl
           << "  dataset has " << ndims << " dimensions" << endl;
    }
  } else if (regex && regexec(&preg, name, 0, NULL, 0)) {
    if (verbose) {
      cerr << "skipping dataset '" << name << "':" << endl
           << "  name doesn't match regex" << endl;
    }
  } else if (timestep != PARAMETER_UNSET and
             fabs(timestep - patch.time) > FUZZY_FACTOR) {
    if (verbose) {
      cerr << "skipping dataset '" << name << "':" << endl
           << "  timestep (" << patch.time << ") doesn't match" << endl;
    }
  } else {
    if (not out3D) {
      CHECK_HDF5(attr = H5Aopen_name(dataset, "origin"));
      CHECK_HDF5(H5Aread(attr, H5T_NATIVE_DOUBLE, patch.origin));
      CHECK_HDF5(H5Aclose(attr));
      CHECK_HDF5(attr = H5Aopen_name(dataset, "delta"));
      CHECK_HDF5(H5Aread(attr, H5T_NATIVE_DOUBLE, patch.delta));
      CHECK_HDF5(H5Aclose(attr));
    }
    CHECK_HDF5(H5Sget_simple_extent_dims(dataspace, patch.dims, NULL));

    int i;
    is_okay = out3D;
    if (not out3D) {
      for (i = 0; i < 3; i++) {
        if (slab_coord[i] != PARAMETER_UNSET) {
          is_okay =
              slab_coord[i] >= patch.origin[i] and
              slab_coord[i] <=
                  patch.origin[i] + (patch.dims[2 - i] - 1) * patch.delta[i];
          if (not is_okay)
            break;
        }
      }
    }
    if (not is_okay) {
      if (verbose) {
        cerr << "skipping dataset '" << name << "':" << endl
             << "  slab " << slab_coord[i] << " is out of dataset range ["
             << patch.origin[i] << ", "
             << patch.origin[i] + (patch.dims[2 - i] - 1) * patch.delta[i]
             << "]" << endl;
      }
    }
  }
  CHECK_HDF5(H5Sclose(dataspace));

  if (not is_okay) {
    CHECK_HDF5(H5Dclose(dataset));
    return (0);
  }

  patch.file = *(hid_t *)_file;
  patch.is_real = typeclass == H5T_FLOAT;

  /* read attributes */
  CHECK_HDF5(attr = H5Aopen_name(dataset, "timestep"));
  CHECK_HDF5(H5Aread(attr, H5T_NATIVE_INT, &patch.iteration));
  CHECK_HDF5(H5Aclose(attr));
  CHECK_HDF5(attr = H5Aopen_name(dataset, "level"));
  CHECK_HDF5(H5Aread(attr, H5T_NATIVE_INT, &patch.rflevel));
  CHECK_HDF5(H5Aclose(attr));
  CHECK_HDF5(attr = H5Aopen_name(dataset, "group_timelevel"));
  CHECK_HDF5(H5Aread(attr, H5T_NATIVE_INT, &patch.timelevel));
  CHECK_HDF5(H5Aclose(attr));
  CHECK_HDF5(attr = H5Aopen_name(dataset, "carpet_mglevel"));
  CHECK_HDF5(H5Aread(attr, H5T_NATIVE_INT, &patch.mglevel));
  CHECK_HDF5(H5Aclose(attr));
  CHECK_HDF5(attr = H5Aopen_name(dataset, "iorigin"));
  CHECK_HDF5(H5Aread(attr, H5T_NATIVE_INT, patch.iorigin));
  CHECK_HDF5(H5Aclose(attr));

  CHECK_HDF5(H5Dclose(dataset));

  // the component and map info are not contained in the HDF5 dataset
  // and thus have to be parsed from the dataset's name
  patch.map = patch.component = 0;
  string::size_type loc = patch.name.find(" m=", 0);
  if (loc != string::npos)
    patch.map = atoi(patch.name.c_str() + loc + 3);
  loc = patch.name.find(" c=", 0);
  if (loc != string::npos)
    patch.component = atoi(patch.name.c_str() + loc + 3);

  if (max_rflevel < patch.rflevel)
    max_rflevel = patch.rflevel;

  patchlist.push_back(patch);

  return (0);
}

/*@@
  @routine    ReadPatch
  @date       Tue 8 August 2006
  @author     Thomas Radke
  @desc
              Reads a slab from the given patch
              and outputs it in CarpetIOASCII format.
  @enddesc

  @var        patch
  @vdesc      patch to output
  @vtype      const patch_t&
  @vio        in
  @endvar
  @var        last_iteration
  @vdesc      iteration number of the last patch that was output
  @vtype      int
  @vio        in
  @endvar
@@*/
static void ReadPatch(const patch_t &patch, int last_iteration) {
  if (patch.iteration != last_iteration)
    cout << endl;

  cout << "# iteration " << patch.iteration << endl
       << "# refinement level " << patch.rflevel << "   multigrid level "
       << patch.mglevel << "   map " << patch.map << "   component "
       << patch.component << "   time level " << patch.timelevel << endl;

  // C order
  h5size_t slabstart[3] = {(hsize_t)patch.iorigin[2], (hsize_t)patch.iorigin[1],
                           (hsize_t)patch.iorigin[0]};
  hsize_t slabcount[3] = {(hsize_t)patch.dims[0], (hsize_t)patch.dims[1],
                          (hsize_t)patch.dims[2]};
  hsize_t slabsize[3] = {(hsize_t)datadims[2], (hsize_t)datadims[1],
                         (hsize_t)datadims[0]};
  for (int d = 0; d < 3; ++d) {
    if (slab_coord[d] != PARAMETER_UNSET) {
      slabstart[2 - d] =
          (h5size_t)((slab_coord[d] - patch.origin[d]) / patch.delta[d] + 0.5);
      slabcount[2 - d] = 1;
    }
  }
  for (int d = 0; d < 3; ++d) {
    cout << "slab[" << d << "]: lbnd=" << slabstart[2 - d]
         << " lsh=" << slabcount[2 - d] << " gsh=" << slabsize[2 - d] << endl;
  }
  for (int d = 0; d < 3; ++d) {
    assert(slabstart[2 - d] >= 0);
    assert(slabcount[2 - d] >= 0);
    assert(slabstart[2 - d] + slabcount[2 - d] <= slabsize[2 - d]);
  }

  hid_t dataset, filespace, slabspace;
  CHECK_HDF5(dataset = H5Dopen(patch.file, patch.name.c_str()));
  CHECK_HDF5(filespace = H5Dget_space(dataset));
#if 0
  CHECK_HDF5 (slabspace = H5Screate_simple (3, slabcount, slabsize));
  CHECK_HDF5 (H5Soffset_simple (slabspace, (hssize_t*)slabstart));
//   CHECK_HDF5 (H5Sselect_hyperslab (filespace, H5S_SELECT_SET,
//                                    slabstart, NULL, slabcount, NULL));
#endif
  CHECK_HDF5(slabspace = H5Screate_simple(3, slabsize, NULL));
  CHECK_HDF5(H5Sselect_hyperslab(slabspace, H5S_SELECT_SET, slabstart, NULL,
                                 slabcount, NULL));
  assert(patch.is_real);
  CHECK_HDF5(H5Dread(dataset, H5T_NATIVE_FLOAT, slabspace, filespace,
                     H5P_DEFAULT, dataptr));
  CHECK_HDF5(H5Sclose(slabspace));
  CHECK_HDF5(H5Sclose(filespace));
  CHECK_HDF5(H5Dclose(dataset));
}

// comparison function to sort patches by iteration number, refinement level,
// multigrid level, map, and component
static bool ComparePatches(const patch_t &a, const patch_t &b) {
  if (a.iteration != b.iteration)
    return (a.iteration < b.iteration);
  if (a.rflevel != b.rflevel)
    return (a.rflevel < b.rflevel);
  if (a.mglevel != b.mglevel)
    return (a.mglevel < b.mglevel);
  if (a.map != b.map)
    return (a.map < b.map);
  if (a.component != b.component)
    return (a.component < b.component);
  return (a.timelevel < b.timelevel);
}

```


combined_doc_context: ## README (README):
```
Cactus Code Thorn CarpetIOHDF5
Author(s)    : Erik Schnetter <schnetter@cct.lsu.edu>
Maintainer(s): Erik Schnetter <schnetter@cct.lsu.edu>
Licence      : GPLv2+
--------------------------------------------------------------------------

1. Purpose

This thorn provides HDF5 based file I/O for Carpet.  See
http://www.hdfgroup.org/HDF5/ for information about HDF5.

```

## Documentation (.tex files):
Files: documentation.tex

```tex
=== documentation.tex ===
\documentclass{article}

% Use the Cactus ThornGuide style file
% (Automatically used from Cactus distribution, if you have a 
%  thorn without the Cactus Flesh download this from the Cactus
%  homepage at www.cactuscode.org)
\usepackage{../../../../doc/latex/cactus}

\begin{document}

% The author of the documentation
\author{Erik Schnetter \textless schnetter@uni-tuebingen.de\textgreater\\
        Christian D. Ott \textless cott@aei.mpg.de\textgreater\\
        Thomas Radke \textless tradke@aei.mpg.de\textgreater}

% The title of the document (not necessarily the name of the Thorn)
\title{CarpetIOHDF5}

% the date your document was last changed, if your document is in CVS, 
% please use:
\date{1 December 2004}

\maketitle

% Do not delete next line
% START CACTUS THORNGUIDE

\ifx\ThisThorn\undefined
\newcommand{\ThisThorn}{{\it CarpetIOHDF5}}
\else
\renewcommand{\ThisThorn}{{\it CarpetIOHDF5}}
\fi


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{abstract}
Thorn \ThisThorn\ provides HDF5-based output to the {\em Carpet} mesh
refinement driver in {\em Cactus}.
This document explains \ThisThorn's usage and contains a specification
of the HDF5 file format that was adapted from John Shalf's FlexIO library.
\end{abstract}


\tableofcontents

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Introduction}

Having encountered various problems with the Carpet I/O thorn
{\bf CarpetIOFlexIO} and the underlying FlexIO library,
Erik Schnetter decided to write this thorn \ThisThorn\ which bypasses
any intermediate binary I/O layer and outputs in HDF5\footnote{Hierarchical
Data Format version 5, see {\tt http://hdf.ncsa.uiuc.edu/whatishdf5.html}
for details} file format directly.

\ThisThorn\ provides output for the {\em Carpet} Mesh Refinement driver
within the Cactus Code. Christian D. Ott added  a file reader (analogous to
Erik Schnetter's implementation present in {\bf CarpetIOFlexIO}) 
as well as checkpoint/recovery functionality to \ThisThorn.
Thomas Radke has taken over maintainence of this I/O thorn and is continuously
working on fixing known bugs and improving the code functionality and
efficiency.

The \ThisThorn\ I/O method can output any type of CCTK grid variables
(grid scalars, grid functions, and grid arrays of arbitrary dimension);
data is written into separate files named {\tt "<varname>.h5"}.
It implements both serial and full parallel I/O --
data files can be written/read either by processor 0 only or by all processors.
Such datafiles can be used for further postprocessing (eg. visualization with
OpenDX or DataVault%%%
\footnote{%%%
         See our visualization page at
	 \url{http://www.cactuscode.org/Visualization/}
	 for details.
	 }%%%
) or fed back into Cactus via the filereader capabilities of thorn
{\bf IOUtil}.

This document aims at giving the user a first handle on how to use
\ThisThorn. It also documents the HDF5 file layout used.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{\ThisThorn\ Parameters}

Parameters to control the \ThisThorn\ I/O method are:

\begin{itemize}
  \item {\tt IOHDF5::out\_every} (steerable)\\
        How often to do periodic \ThisThorn\ output. If this parameter is set
        in the parameter file, it will override the setting of the shared
        {\tt IO::out\_every} parameter. The output frequency can also be set
        for individual variables using the {\tt out\_every} option in an option
        string appended to the {\tt IOHDF5::out\_vars} parameter.

  \item {\tt IOHDF5::out\_dt} (steerable)\\
        output in intervals of that much coordinate time (overwrites {\tt IO::out\_dt})

  \item {\tt IOHDF5::out\_criterion} (steerable)\\
        criterion to select output intervals (overwrites {\tt IO::out\_criterion})

  \item {\tt IOHDF5::out\_vars} (steerable)\\
        The list of variables to output using the \ThisThorn\ I/O method.
        The variables must be given by their fully qualified variable or group
        name. The special keyword {\it all} requests \ThisThorn\ output for
        all variables. Multiple names must be separated by whitespaces.

        Each group/variable name can have an option string attached in which you
        can specify a different output frequency for that individual variable,
	a set of individual refinement levels to be output, the compression
        level, or an individual output mode, e.g.
\begin{verbatim}
  IOHDF5::out_vars = "wavetoy::phi{ out_every = 4 refinement_levels = { 1 2 } }"
\end{verbatim}
        Option strings currently supported by \ThisThorn\ are: {\tt out\_every,
        out\_unchunked}, {\tt refinement\_levels}, and {\tt compression\_level}.

  \item {\tt IOHDF5::out\_dir}\\
        The directory in which to place the \ThisThorn\ output files.
        If the directory doesn't exist at startup it will be created.
        If parallel output is enabled and the directory name contains the
        substring {\tt "\%u"} it will be substituted by the processor ID.
        By this means each processor can have its own output directory.\\
        If this parameter is set to an empty string \ThisThorn\ output will go
        to the standard output directory as specified in {\tt IO::out\_dir}.

  \item {\tt IOHDF5::compression\_level (steerable)}\\
        Compression level to use for writing HDF5 datasets.
        Automatic {\tt gzip} dataset compression can be enabled by setting
        this integer parameter to values between 1 and 9 (inclusive),
        with increasing values requesting higher compression rates (at the cost
        of additional runtime for outputting HDF5 data); a value of zero
        (which is the default setting for this parameter) disables compression.
        The output compression level can also be set
        for individual variables using the {\tt compression\_level} option in
        an option string appended to the {\tt IOHDF5::out\_vars} parameter.

  \item {\tt IO::out\_single\_precision (steerable)}\\
        whether to output double-precision data in single precision

\end{itemize}


\section{Serial versus Parallel Output}

According to the ouptput mode parameter settings of ({\tt IO::out\_mode},
{\tt IO::out\_unchunked},\newline{\tt IO::out\_proc\_every}) of thorn
{\bf IOUtil}, thorn \ThisThorn\ will output distributed grid variables either

\begin{itemize}
  \item in serial from processor 0 into a single unchunked file
\begin{verbatim}
  IO::out_mode      = "onefile"
  IO::out_unchunked = "yes"
\end{verbatim}

  \item in serial from processor 0 into a single chunked file
\begin{verbatim}
  IO::out_mode      = "onefile"
  IO::out_unchunked = "no"
\end{verbatim}

  \item in parallel, that is, into separate chunked files (one per processor)
        containing the individual processors' patches of the
        distributed grid variable
\begin{verbatim}
  IO::out_mode      = "proc"
\end{verbatim}
\end{itemize}

{\em Unchunked} means that an entire Cactus grid array (gathered across all
processors) is stored in a single HDF5 dataset whereas {\em chunked} means
that all the processor-local patches of this array are stored as separate
HDF5 datasets (called {\it chunks}).
Consequently, for unchunked data all interprocessor ghostzones are excluded
from the output.  In contrast, for chunked data the interprocessor ghostzones
\emph{are} included in the output.

When visualising chunked datasets, they probably need to be recombined
for a global view on the data. This needs to be done within the visualisation
tool (see also below), Cactus itself does not provide its own recombiner
utility program for \ThisThorn's output files.

The default is to output distributed grid variables in parallel, each processor
writing a file
{\tt \textless varname\textgreater.file\_\textless processor ID\textgreater.h5}.
The chunked/unchunked mode can also be set individually in a key/value
option string (with the key {\tt out\_unchunked} and possible string values
{\tt "true|false|yes|no"}) appended to a group/variable name in the
{\tt out\_vars} parameter, eg.
\begin{verbatim}
  IOHDF5::out_vars = "wavetoy::phi{out_unchunked = 'true'}  grid::coordinates"
\end{verbatim}
will cause the variable {\tt phi} to be output into a single unchunked file
whereas other variables will still be output into separate chunked files
(assuming the output mode is left to its default).
Grid scalars
and {\tt DISTRIB = CONST} grid arrays are always output as unchunked data
on processor 0 only.

Parallel output in a parallel simulation will ensure maximum I/O
performance. Note that changing the output mode to serial I/O might only be
necessary if the data analysis and visualisation tools cannot deal with
chunked output files. Cactus itself, as well as many of the tools to
visualise Carpet HDF5 data
(see \url{http://www.cactuscode.org/Visualization}),
can process both chunked and unchunked data. For instance, to visualise parallel
output datafiles with DataVault, you would just send all the individual files
to the DV server: {\tt hdf5todv phi.file\_*.h5}. In OpenDX the {\tt
ImportCarpetIOHDF5} module can be given any filename from the set of parallel
chunked files; the module will determine the total number of files in the set
automatically and read them all.


\section{Using the flesh I/O API to produce HDF5 output}

Periodic output of grid variables is usually specified via I/O parameters
in the parameter file and then automatically triggered by the flesh scheduler
at each iteration step after analysis. If output should also be triggered
at a different time, one can do that from within an application thorn by
invoking one of the {\tt CCTK\_OutputVar*()} I/O routines provided
by the flesh I/O API (see chapter B8.2 ``IO'' in the Cactus Users Guide).
In this case, the application thorn routine which calls {\tt CCTK\_OutputVar*()}
must be scheduled in level mode.

It should be noted here that -- due to a restriction in the naming scheme of
objects in an HDF5 data file -- \ThisThorn\ can output a given grid variable
with given refinement level only once per timestep. Attempts of application
thorns to trigger the output of the same variable multiple times during an iteration
will result in a runtime warning and have no further effect.
If output for a variable is required also for intermediate timesteps
this can be achieved by calling {\tt CCTK\_OutputVarAs*()} with a different
{\tt alias} name; output for the same variable is then written into
different HDF5 files based on the {\tt alias} argument.


\section{Checkpointing \& Recovery and Importing Data}

Thorn \ThisThorn\ can also be used to create HDF5 checkpoint files and
to recover from such files later on. In addition it can read HDF5 datafiles
back in using the generic filereader interface described in the thorn
documentation of {\bf IOUtil}.

Checkpoint routines are scheduled at several timebins so that you can save
the current state of your simulation after the initial data phase,
during evolution, or at termination. Checkpointing for thorn \ThisThorn\ 
is enabled by setting the parameter {\tt IOHDF5::checkpoint = "yes"}.

A recovery routine is registered with thorn {\bf IOUtil} in order to restart
a new simulation from a given HDF5 checkpoint.
The very same recovery mechanism is used to implement a filereader
functionality to feed back data into Cactus.

Checkpointing and recovery are controlled by corresponding checkpoint/recovery
parameters of thorn {\bf IOUtil} (for a description of these parameters please
refer to this thorn's documentation).


\section{CarpetIOHDF5 Utility Programs}

\subsection{{\tt hdf5toascii\_slicer}}

This utility program extracts 1D lines and 2D slices from 3D HDF5
datasets produced by \ThisThorn\ and outputs them in {\tt CarpetIOASCII} format
(suitable to be further processed by gnuplot).

The {\tt hdf5toascii\_slicer} program is contained in the {\tt src/utils/}
subdirectory of thorn \ThisThorn. It is built with
\begin{verbatim}
  make <configuration>-utils
\end{verbatim}
where the executable ends up in the subdirectory {\tt exe/<configuration>/}.

For details on how to use the {\tt hdf5toascii\_slicer} program,
run it with no command-line options (or with the {\tt --help} option).

{\tt hdf5toascii\_slicer} can be used on either chunked or unchunked
data:
\begin{itemize}
\item	If the HDF5 data is unchunked, then {\tt hdf5toascii\_slicer}
	will output unchunked ASCII data.
\item	If the HDF5 data is chunked, then {\tt hdf5toascii\_slicer}
	will output chunked ASCII data reflecting whatever subset of
	the HDF5 files are provided.  That is, for example, the command
	\begin{verbatim}
	hdf5toascii_slicer my_variable.file_2.h5 my_variable.file_4.h5
	\end{verbatim}
	will output ASCII data only for that part of the Carpet grid
	which lived on processors~2 and~4.  It's probably more useful
	to use the command
	\begin{verbatim}
	hdf5toascii_slicer my_variable.file_*.h5
	\end{verbatim}
	which will output ASCII data for the entire Carpet grid.
\end{itemize}


\subsection{{\tt hdf5\_extract}}

This utility program extracts selected datasets from any given HDF5 output file
which may be useful when only certain parts (eg. a specific
timestep) of large files are required (eg. for copying to some other
location for further processing).

The {\tt hdf5\_extract} program is contained in the {\tt src/utils/}
subdirectory of thorn {\tt CactusPUGHIO/IOHDF5}. It is built with
\begin{verbatim}
  make <configuration>-utils
\end{verbatim}
where the executable ends up in the subdirectory {\tt exe/<configuration>/}.


\section{Example Parameter File Excerpts}

\subsection{Serial (unchunked) Output of Grid Variables}

\begin{verbatim}
  # how often to output and where output files should go
  IO::out_every = 2
  IO::out_dir   = "wavetoy-data"

  # request output for wavetoy::psi at every other iteration for timelevel 0,
  #                for wavetoy::phi every 4th iteration with timelevels 1 and 2
  IOHDF5::out_vars = "wavetoy::phi{ out_every = 4 refinement_levels = { 1 2 } }
                      wavetoy::psi"

  # we want unchunked output
  # (because the visualisation tool cannot deal with chunked data files)
  IO::out_mode      = "onefile"
  IO::out_unchunked = 1
\end{verbatim}

\subsection{Parallel (chunked) Output of Grid Variables}

\begin{verbatim}
  # how often to output
  IO::out_every = 2

  # each processor writes to its own output directory
  IOHDF5::out_dir = "wavetoy-data-proc%u"

  # request output for wavetoy::psi at every other iteration for timelevel 0,
  #                for wavetoy::phi every 4th iteration with timelevels 1 and 2
  IOHDF5::out_vars = "wavetoy::phi{ out_every = 4 refinement_levels = { 1 2 } }
                      wavetoy::psi"

  # we want parallel chunked output (note that this already is the default)
  IO::out_mode = "proc"
\end{verbatim}

\subsection{Checkpointing \& Recovery}

\begin{verbatim}
  # say how often we want to checkpoint, how many checkpoints should be kept,
  # how the checkpoints should be named, and they should be written to
  IO::checkpoint_every = 100
  IO::checkpoint_keep  = 2
  IO::checkpoint_file  = "wavetoy"
  IO::checkpoint_dir   = "wavetoy-checkpoints"

  # enable checkpointing for CarpetIOHDF5
  IOHDF5::checkpoint = "yes"

  #######################################################

  # recover from the latest checkpoint found
  IO::recover_file = "wavetoy"
  IO::recover_dir  = "wavetoy-checkpoints"
  IO::recover      = "auto"
\end{verbatim}

\subsection{Importing Grid Variables via Filereader}

\begin{verbatim}
  # which data files to import and where to find them
  IO::filereader_ID_files = "phi psi"
  IO::filereader_ID_dir   = "wavetoy-data"

  # what variables and which timestep to read
  # (if this parameter is left empty, all variables and timesteps found
  #  in the data files will be read)
  IO::filereader_ID_vars  = "WaveToyMoL::phi{ cctk_iteration = 0 }
                             WaveToyMoL::psi"
\end{verbatim}


\iffalse
\section{CarpetIOHDF5's HDF5 file layout}

The HDF5 file layout of {\bf CarpetIOHDF5} is quite simple.
There are no groups besides the standard HDF5 root data object group:

Each dataset is named according to this template:

\begin{verbatim}
  <group::varname> it=<cctk_iteration> tl=<timelevel> [ml=<mglevel>] [m=<map>]
  [rl=<reflevel>] [c=<component>]}
\end{verbatim}

where optional parts only contribute to the name if they vary (if there is
more than one multigrid level, map, refinement level, component respectively).

Each HDF5 dataset has the following attributes associated with it:

\begin{itemize}
  \item {\tt level} : Carpet::reflevel
  \item {\tt origin} : 1-D array of length vdim. \\
        origin[d] = CCTK\_ORIGIN\_SPACE(d) + cctk\_lbnd[d] * delta[d]
  \item {\tt delta} : 1-D array of length vdim. \\
        delta[d] = CCTK\_DELTA\_SPACE(d)
  \item {\tt time} : cctk\_time
  \item {\tt timestep} : cctk\_iteration
  \item {\tt iorigin} : 1-D array of length vdim. \\ iorigin[d] = (Carpet::ext.lower() / Carpet::ext.stride())[d]
  \item {\tt name} : CCTK\_FullName(variable index)
  \item {\tt cctk\_bbox} : 1-D array of length 2*Carpet::dim. cctk\_box
  \item {\tt cctk\_nghostzones} : 1-D array of length Carpet::dim. cctk\_nghostzones
  \item {\tt carpet\_mglevel} : Carpet::mglevel
  \item {\tt carpet\_reflevel} : Carpet::reflevel
\end{itemize}


\subsection{Attributes needed by the file reader}

The number of attributes needed by the CarpetIOHDF5 file reader is much smaller then the total
number of attributes attached to each dataset:

\begin{itemize}
  \item {\tt name}
  \item {\tt level}
  \item {\tt iorigin}
\end{itemize}

\fi


% Do not delete next line
% END CACTUS THORNGUIDE

\end{document}

```



Reference solution:
src_code: #include <cassert>
#include <cstdlib>
#include <cstring>
#include <sstream>

#include "cctk.h"
#include "cctk_Arguments.h"
#include "cctk_Parameters.h"
#include "util_Table.h"

#include "operators.hh"
#include "typeprops.hh"
#include "CarpetIOHDF5.hh"
#include "CactusBase/IOUtil/src/ioGH.h"

namespace CarpetIOHDF5 {

using namespace std;
using namespace Carpet;

// add attributes to an HDF5 dataset
static int AddAttributes(const cGH *const cctkGH, const char *fullname,
                         int vdim, int refinementlevel,
                         const ioRequest *const request, const ibbox &bbox,
                         hid_t dataset, bool is_index = false);

int WriteVarUnchunked(const cGH *const cctkGH, hid_t outfile,
                      CCTK_REAL &io_bytes, const ioRequest *const request,
                      bool called_from_checkpoint) {
  DECLARE_CCTK_PARAMETERS;

  int error_count = 0;
  char *fullname = CCTK_FullName(request->vindex);
  const int gindex = CCTK_GroupIndexFromVarI(request->vindex);
  assert(gindex >= 0 and gindex < (int)Carpet::arrdata.size());
  const int var = request->vindex - CCTK_FirstVarIndexI(gindex);
  assert(var >= 0 and var < CCTK_NumVars());
  cGroup group;
  CCTK_GroupData(gindex, &group);

  // Scalars and arrays have only one refinement level 0,
  // regardless of what the current refinement level is.
  // Output for them must be called in global mode.
  int refinementlevel = reflevel;
  if (group.grouptype == CCTK_SCALAR or group.grouptype == CCTK_ARRAY) {
    assert(do_global_mode);
    refinementlevel = 0;
  }

  // HDF5 doesn't like 0-dimensional arrays
  if (group.grouptype == CCTK_SCALAR)
    group.dim = 1;

  // If the user requested so, select single precision data output
  hid_t memdatatype, filedatatype;
  HDF5_ERROR(memdatatype = CCTKtoHDF5_Datatype(cctkGH, group.vartype, 0));
  HDF5_ERROR(filedatatype = CCTKtoHDF5_Datatype(
                 cctkGH, group.vartype,
                 out_single_precision and not called_from_checkpoint));

  // create a file access property list to use the CORE virtual file driver
  hid_t plist;
  HDF5_ERROR(plist = H5Pcreate(H5P_FILE_ACCESS));
  HDF5_ERROR(H5Pset_fapl_core(plist, 0, 0));
  HDF5_ERROR(H5Pset_fclose_degree(plist, H5F_CLOSE_STRONG));

  // Traverse all maps
  BEGIN_MAP_LOOP(cctkGH, group.grouptype) {
    // Collect the set of all components' bboxes
    ibset bboxes;
    for (int c = 0;
         c < arrdata.at(gindex).at(Carpet::map).hh->components(refinementlevel);
         ++c) {
// Using "interior" removes ghost zones and refinement boundaries.
#if 0
      bboxes += arrdata.at(gindex).at(Carpet::map).dd->
                light_boxes.at(mglevel).at(refinementlevel).at(c).interior;
#endif
      bboxes += arrdata.at(gindex)
                    .at(Carpet::map)
                    .dd->light_boxes.at(mglevel)
                    .at(refinementlevel)
                    .at(c)
                    .exterior;
    }

#if 0
    // According to Cactus conventions, DISTRIB=CONSTANT arrays
    // (including grid scalars) are assumed to be the same on all
    // processors and are therefore stored only by processor 0.
    if (group.disttype != CCTK_DISTRIB_CONSTANT);
#endif

    // Loop over all components in the bbox set
    int bbox_id = 0;
    for (ibset::const_iterator bbox = bboxes.begin(); bbox != bboxes.end();
         bbox++, bbox_id++) {
      // Get the shape of the HDF5 dataset (in Fortran index order)
      hsize_t shape[dim];
      hsize_t num_elems = 1;
      for (int d = 0; d < group.dim; ++d) {
        assert(group.dim - 1 - d >= 0 and group.dim - 1 - d < dim);
        shape[group.dim - 1 - d] = (bbox->shape() / bbox->stride())[d];
        num_elems *= shape[group.dim - 1 - d];
      }

      // Don't create zero-sized components
      if (num_elems == 0)
        continue;

      // create the dataset on the I/O processor
      // skip DISTRIB=CONSTANT components from processors other than 0
      hid_t memfile = -1, memdataset = -1;
      hid_t dataspace = -1, dataset = -1;
      if (dist::rank() == 0 and
          (bbox_id == 0 or group.disttype != CCTK_DISTRIB_CONSTANT)) {
        // Construct a file-wide unique HDF5 dataset name
        // (only add parts with varying metadata)
        ostringstream datasetname;
        datasetname << fullname << " it=" << cctkGH->cctk_iteration
                    << " tl=" << request->timelevel;
        if (mglevels > 1)
          datasetname << " ml=" << mglevel;
        if (group.grouptype == CCTK_GF) {
          if (maps > 1)
            datasetname << " m=" << Carpet::map;
          datasetname << " rl=" << refinementlevel;
        }
        if (bboxes.setsize() > 1 and group.disttype != CCTK_DISTRIB_CONSTANT) {
          datasetname << " c=" << bbox_id;
        }

        // We create a temporary HDF5 file in memory (using the core VFD)
        // in order to do the recombination of individual processor components
        // for a single dataset.
        // Although this requires some more memory, it should be much faster
        // than recombining an HDF5 dataset on a disk file.
        HDF5_ERROR(memfile =
                       H5Fcreate("tempfile", H5F_ACC_EXCL, H5P_DEFAULT, plist));
        HDF5_ERROR(dataspace = H5Screate_simple(group.dim, shape, NULL));
        HDF5_ERROR(memdataset =
                       H5Dcreate(memfile, datasetname.str().c_str(),
                                 filedatatype, dataspace, H5P_DEFAULT));

        // remove an already existing dataset of the same name
        if (request->check_exist) {
          H5E_BEGIN_TRY { H5Gunlink(outfile, datasetname.str().c_str()); }
          H5E_END_TRY;
        }
        // enable compression if requested
        hid_t plist_dataset;
        HDF5_ERROR(plist_dataset = H5Pcreate(H5P_DATASET_CREATE));
        const int compression_lvl = request->compression_level >= 0
                                        ? request->compression_level
                                        : compression_level;
        const hsize_t size = num_elems * H5Tget_size(filedatatype);
        if (compression_lvl and size > hsize_t(minimum_size_for_compression)) {
          HDF5_ERROR(H5Pset_chunk(plist_dataset, group.dim, shape));
          HDF5_ERROR(H5Pset_shuffle(plist_dataset));
          HDF5_ERROR(H5Pset_deflate(plist_dataset, compression_lvl));
        }
        // enable checksums if requested
        if (use_checksums) {
          HDF5_ERROR(H5Pset_chunk(plist_dataset, group.dim, shape));
          HDF5_ERROR(
              H5Pset_filter(plist_dataset, H5Z_FILTER_FLETCHER32, 0, 0, NULL));
        }
        HDF5_ERROR(dataset = H5Dcreate(outfile, datasetname.str().c_str(),
                                       filedatatype, dataspace, plist_dataset));
        HDF5_ERROR(H5Pclose(plist_dataset));
      }

      // Loop over all components
      bool first_time = true;
      BEGIN_COMPONENT_LOOP(cctkGH, group.grouptype) {
        // Continue if this processor is not involved
        if (local_component == -1 and dist::rank() != 0)
          continue;

        // Get the intersection of the current component with this combination
        // (use either the interior or exterior here, as we did above)
        gh const *const hh = arrdata.at(gindex).at(Carpet::map).hh;
        dh const *const dd = arrdata.at(gindex).at(Carpet::map).dd;
#if 0
        ibbox const overlap = *bbox &
          dd->light_boxes.at(mglevel).at(refinementlevel).at(component).
          interior;
#endif
        ibbox const overlap = *bbox &
                              dd->light_boxes.at(mglevel)
                                  .at(refinementlevel)
                                  .at(component)
                                  .exterior;

        // Continue if this component is not part of this combination
        if (overlap.empty())
          continue;

        // Copy the overlap to the local processor
        const ggf *ff = arrdata.at(gindex).at(Carpet::map).data.at(var);
        const gdata *const data =
            local_component >= 0
                ? ff->data_pointer(request->timelevel, refinementlevel,
                                   local_component, mglevel)
                : NULL;
#if 0
        // TODO: This does not work; data may be NULL
        gdata* const processor_component =
          data->make_typed (request->vindex, error_centered, op_sync);
#else
        gdata *const processor_component = ff->new_typed_data();
#endif

        processor_component->allocate(overlap, 0);
        for (comm_state state; not state.done(); state.step()) {
          int const p = hh->processor(refinementlevel, component);
          processor_component->copy_from(state, data, overlap, overlap, NULL, 0,
                                         p);
        }

        // Write data
        if (dist::rank() == 0) {
          const void *data = (const void *)processor_component->storage();

          // As per Cactus convention, DISTRIB=CONSTANT arrays
          // (including grid scalars) are assumed to be the same on
          // all processors and are therefore stored only by processor
          // 0.
          //
          // Warn the user if this convention is violated.
          if (bbox_id > 0 and group.disttype == CCTK_DISTRIB_CONSTANT) {
            const void *proc0 =
                CCTK_VarDataPtrI(cctkGH, request->timelevel, request->vindex);

            if (memcmp(proc0, data,
                       num_elems * CCTK_VarTypeSize(group.vartype))) {
              CCTK_VWarn(1, __LINE__, __FILE__, CCTK_THORNSTRING,
                         "values for DISTRIB=CONSTANT grid variable '%s' "
                         "(timelevel %d) differ between processors 0 and %d; "
                         "only the array from processor 0 will be stored",
                         fullname, request->timelevel, bbox_id);
            }
          } else {
            hsize_t overlapshape[dim];

            // before HDF5-1.6.4 the H5Sselect_hyperslab() function expected
            // the 'start' argument to be of type 'hssize_t'
            slice_start_size_t overlaporigin[dim];
            hsize_t hyperslab_start[dim], hyperslab_count[dim];
            for (int d = 0; d < group.dim; ++d) {
              assert(group.dim - 1 - d >= 0 and group.dim - 1 - d < dim);
              overlaporigin[group.dim - 1 - d] =
                  ((overlap.lower() - bbox->lower()) / overlap.stride())[d];
              overlapshape[group.dim - 1 - d] = processor_component->shape()[d];
              assert(all(processor_component->shape() ==
                         overlap.shape() / overlap.stride()));
              hyperslab_start[group.dim - 1 - d] = 0;
              hyperslab_count[group.dim - 1 - d] =
                  processor_component->shape()[d];
            }

            // Write the processor component as a hyperslab into the recombined
            // dataset.
            hid_t overlap_dataspace;
            HDF5_ERROR(overlap_dataspace =
                           H5Screate_simple(group.dim, overlapshape, NULL));
            HDF5_ERROR(H5Sselect_hyperslab(overlap_dataspace, H5S_SELECT_SET,
                                           hyperslab_start, NULL,
                                           hyperslab_count, NULL));
            HDF5_ERROR(H5Sselect_hyperslab(dataspace, H5S_SELECT_SET,
                                           overlaporigin, NULL, overlapshape,
                                           NULL));
            HDF5_ERROR(H5Dwrite(memdataset, memdatatype, overlap_dataspace,
                                dataspace, H5P_DEFAULT, data));
            io_bytes += H5Sget_simple_extent_npoints(overlap_dataspace) *
                        H5Tget_size(filedatatype);
            HDF5_ERROR(H5Sclose(overlap_dataspace));

            // Add metadata information on the first time through
            // (have to do it inside of the COMPONENT_LOOP so that we have
            //  access to the cGH elements)
            if (first_time) {
              error_count +=
                  AddAttributes(cctkGH, fullname, group.dim, refinementlevel,
                                request, *bbox, dataset);
              first_time = false;
            }
          }
        }

        // Delete temporary copy of this component
        delete processor_component;
      }
      END_COMPONENT_LOOP;

      // Finally create the recombined dataset in the real HDF5 file on disk
      // (skip DISTRIB=CONSTANT components from processors other than 0)
      if (dist::rank() == 0 and
          (bbox_id == 0 or group.disttype != CCTK_DISTRIB_CONSTANT)) {
        void *data = malloc(H5Dget_storage_size(memdataset));
        HDF5_ERROR(H5Dread(memdataset, filedatatype, H5S_ALL, H5S_ALL,
                           H5P_DEFAULT, data));
        HDF5_ERROR(H5Dclose(memdataset));
        HDF5_ERROR(H5Fclose(memfile));
        HDF5_ERROR(H5Dwrite(dataset, filedatatype, H5S_ALL, H5S_ALL,
                            H5P_DEFAULT, data));
        free(data);
        HDF5_ERROR(H5Sclose(dataspace));
        HDF5_ERROR(H5Dclose(dataset));
      }

    } // for bboxes
  }
  END_MAP_LOOP;

  HDF5_ERROR(H5Pclose(plist));

  free(fullname);

  // return the number of errors that occured during this output
  return error_count;
}

int WriteVarChunkedSequential(const cGH *const cctkGH, hid_t outfile,
                              CCTK_REAL &io_bytes,
                              const ioRequest *const request,
                              bool called_from_checkpoint, hid_t indexfile) {
  DECLARE_CCTK_PARAMETERS;

  int error_count = 0;
  char *fullname = CCTK_FullName(request->vindex);
  const int gindex = CCTK_GroupIndexFromVarI(request->vindex);
  assert(gindex >= 0 and gindex < (int)Carpet::arrdata.size());
  const int var = request->vindex - CCTK_FirstVarIndexI(gindex);
  assert(var >= 0 and var < CCTK_NumVars());
  cGroup group;
  CCTK_GroupData(gindex, &group);

  // Scalars and arrays have only one refinement level 0,
  // regardless of what the current refinement level is.
  // Output for them must be called in global mode.
  int refinementlevel = reflevel;
  if (group.grouptype == CCTK_SCALAR or group.grouptype == CCTK_ARRAY) {
    assert(do_global_mode);
    refinementlevel = 0;
  }

  // HDF5 doesn't like 0-dimensional arrays
  if (group.grouptype == CCTK_SCALAR)
    group.dim = 1;

  // If the user requested so, select single precision data output
  hid_t memdatatype, filedatatype;
  HDF5_ERROR(memdatatype = CCTKtoHDF5_Datatype(cctkGH, group.vartype, 0));
  HDF5_ERROR(filedatatype = CCTKtoHDF5_Datatype(
                 cctkGH, group.vartype,
                 out_single_precision and not called_from_checkpoint));

  // Traverse all maps
  BEGIN_MAP_LOOP(cctkGH, group.grouptype) {
    BEGIN_COMPONENT_LOOP(cctkGH, group.grouptype) {
      // Continue if this processor is not involved
      if (local_component == -1 and dist::rank() != 0)
        continue;

      // Using "exterior" includes ghost zones and refinement boundaries.
      gh const *const hh = arrdata.at(gindex).at(Carpet::map).hh;
      dh const *const dd = arrdata.at(gindex).at(Carpet::map).dd;
      ibbox const &bbox = dd->light_boxes.at(mglevel)
                              .at(refinementlevel)
                              .at(component)
                              .exterior;

      // Get the shape of the HDF5 dataset (in Fortran index order)
      hsize_t shape[dim];
      hsize_t num_elems = 1;
      for (int d = 0; d < group.dim; ++d) {
        assert(group.dim - 1 - d >= 0 and group.dim - 1 - d < dim);
        shape[group.dim - 1 - d] = (bbox.shape() / bbox.stride())[d];
        num_elems *= shape[group.dim - 1 - d];
      }

      // Don't create zero-sized components
      if (num_elems == 0)
        continue;

      // Copy the overlap to the local processor
      const ggf *ff = arrdata.at(gindex).at(Carpet::map).data.at(var);
      const gdata *const data =
          local_component >= 0
              ? ff->data_pointer(request->timelevel, refinementlevel,
                                 local_component, mglevel)
              : NULL;
#if 0
      // TODO: This does not work; data may be NULL
      gdata* const processor_component =
        data->make_typed (request->vindex, error_centered, op_sync);
#else
      gdata *const processor_component = ff->new_typed_data();
#endif

      processor_component->allocate(bbox, 0);
      for (comm_state state; not state.done(); state.step()) {
        int const p = hh->processor(refinementlevel, component);
        processor_component->copy_from(state, data, bbox, bbox, NULL, 0, p);
      }

      // Write data on I/O processor 0
      if (dist::rank() == 0) {
        const void *data = (const void *)processor_component->storage();

        // As per Cactus convention, DISTRIB=CONSTANT arrays
        // (including grid scalars) are assumed to be the same on
        // all processors and are therefore stored only by processor 0.
        //
        // Warn the user if this convention is violated.
        if (component > 0 and group.disttype == CCTK_DISTRIB_CONSTANT) {
          const void *proc0 =
              CCTK_VarDataPtrI(cctkGH, request->timelevel, request->vindex);

          if (memcmp(proc0, data,
                     num_elems * CCTK_VarTypeSize(group.vartype))) {
            CCTK_VWarn(1, __LINE__, __FILE__, CCTK_THORNSTRING,
                       "values for DISTRIB=CONSTANT grid variable '%s' "
                       "(timelevel %d) differ between processors 0 and %d; "
                       "only the array from processor 0 will be stored",
                       fullname, request->timelevel, component);
          }
        } else {
          // Construct a file-wide unique HDF5 dataset name
          // (only add parts with varying metadata)
          ostringstream datasetname;
          datasetname << fullname << " it=" << cctkGH->cctk_iteration
                      << " tl=" << request->timelevel;
          if (mglevels > 1)
            datasetname << " ml=" << mglevel;
          if (group.grouptype == CCTK_GF) {
            if (maps > 1)
              datasetname << " m=" << Carpet::map;
            datasetname << " rl=" << refinementlevel;
          }
          if (arrdata.at(gindex)
                      .at(Carpet::map)
                      .dd->light_boxes.at(mglevel)
                      .at(refinementlevel)
                      .size() > 1 and
              group.disttype != CCTK_DISTRIB_CONSTANT) {
            datasetname << " c=" << component;
          }

          // remove an already existing dataset of the same name
          if (request->check_exist) {
            H5E_BEGIN_TRY {
              H5Gunlink(outfile, datasetname.str().c_str());
              if (indexfile != -1)
                H5Gunlink(indexfile, datasetname.str().c_str());
            }
            H5E_END_TRY;
          }

          hsize_t shape[dim];
          hsize_t num_elems = 1;
          hsize_t hyperslab_start[dim], hyperslab_count[dim];
          for (int d = 0; d < group.dim; ++d) {
            assert(group.dim - 1 - d >= 0 and group.dim - 1 - d < dim);
            shape[group.dim - 1 - d] = processor_component->padded_shape()[d];
            assert(all(processor_component->shape() ==
                       bbox.shape() / bbox.stride()));
            hyperslab_start[group.dim - 1 - d] = 0;
            hyperslab_count[group.dim - 1 - d] =
                processor_component->shape()[d];
            num_elems *= hyperslab_count[group.dim - 1 - d];
          }

          // Write the component as an individual dataset
          hid_t plist, dataspace, dataset, index_dataset = -1;
          HDF5_ERROR(plist = H5Pcreate(H5P_DATASET_CREATE));
          // enable compression if requested
          const int compression_lvl = request->compression_level >= 0
                                          ? request->compression_level
                                          : compression_level;
          const hsize_t size = num_elems * H5Tget_size(filedatatype);
          if (compression_lvl and
              size > hsize_t(minimum_size_for_compression)) {
            HDF5_ERROR(H5Pset_chunk(plist, group.dim, shape));
            HDF5_ERROR(H5Pset_shuffle(plist));
            HDF5_ERROR(H5Pset_deflate(plist, compression_lvl));
          }
          // enable checksums if requested
          if (use_checksums) {
            HDF5_ERROR(H5Pset_chunk(plist, group.dim, shape));
            HDF5_ERROR(H5Pset_filter(plist, H5Z_FILTER_FLETCHER32, 0, 0, NULL));
          }
          HDF5_ERROR(dataspace = H5Screate_simple(group.dim, shape, NULL));
          HDF5_ERROR(H5Sselect_hyperslab(dataspace, H5S_SELECT_SET,
                                         hyperslab_start, NULL, hyperslab_count,
                                         NULL));
          HDF5_ERROR(dataset = H5Dcreate(outfile, datasetname.str().c_str(),
                                         filedatatype, dataspace, plist));

          if (indexfile != -1) {
            HDF5_ERROR(index_dataset =
                           H5Dcreate(indexfile, datasetname.str().c_str(),
                                     filedatatype, dataspace, H5P_DEFAULT));
          }

          io_bytes += H5Sget_simple_extent_npoints(dataspace) *
                      H5Tget_size(filedatatype);
          HDF5_ERROR(H5Pclose(plist));
          HDF5_ERROR(H5Sclose(dataspace));
          HDF5_ERROR(H5Dwrite(dataset, memdatatype, H5S_ALL, H5S_ALL,
                              H5P_DEFAULT, data));
          error_count += AddAttributes(cctkGH, fullname, group.dim,
                                       refinementlevel, request, bbox, dataset);
          HDF5_ERROR(H5Dclose(dataset));

          if (indexfile != -1) {
            error_count +=
                AddAttributes(cctkGH, fullname, group.dim, refinementlevel,
                              request, bbox, index_dataset, true);
            HDF5_ERROR(H5Dclose(index_dataset));
          }
        }

      } // if dist::rank() == 0

      // Delete temporary copy of this component
      delete processor_component;
    }
    END_COMPONENT_LOOP;
  }
  END_MAP_LOOP;

  free(fullname);

  // return the number of errors that occured during this output
  return error_count;
}

int WriteVarChunkedParallel(const cGH *const cctkGH, hid_t outfile,
                            CCTK_REAL &io_bytes, const ioRequest *const request,
                            bool called_from_checkpoint, hid_t indexfile) {
  DECLARE_CCTK_PARAMETERS;

  int error_count = 0;
  char *fullname = CCTK_FullName(request->vindex);
  const int gindex = CCTK_GroupIndexFromVarI(request->vindex);
  assert(gindex >= 0 and gindex < (int)Carpet::arrdata.size());
  const int var = request->vindex - CCTK_FirstVarIndexI(gindex);
  assert(var >= 0 and var < CCTK_NumVars());
  cGroup group;
  CCTK_GroupData(gindex, &group);

  // Scalars and arrays have only one refinement level 0,
  // regardless of what the current refinement level is.
  // Output for them must be called in global mode.
  int refinementlevel = reflevel;
  if (group.grouptype == CCTK_SCALAR or group.grouptype == CCTK_ARRAY) {
    assert(do_global_mode);
    refinementlevel = 0;
  }

  // HDF5 doesn't like 0-dimensional arrays
  if (group.grouptype == CCTK_SCALAR)
    group.dim = 1;

  // If the user requested so, select single precision data output
  hid_t memdatatype, filedatatype;
  HDF5_ERROR(memdatatype = CCTKtoHDF5_Datatype(cctkGH, group.vartype, 0));
  HDF5_ERROR(filedatatype = CCTKtoHDF5_Datatype(
                 cctkGH, group.vartype,
                 out_single_precision and not called_from_checkpoint));

  // Traverse all maps
  BEGIN_LOCAL_MAP_LOOP(cctkGH, group.grouptype) {
    BEGIN_LOCAL_COMPONENT_LOOP(cctkGH, group.grouptype) {

      // const ggf* ff = arrdata.at(gindex).at(Carpet::map).data.at(var);
      // const ibbox& bbox = (*ff) (request->timelevel, refinementlevel,
      //                            group.disttype == CCTK_DISTRIB_CONSTANT ?
      //                            0 : component, mglevel)->extent();
      const dh *const dd = arrdata.AT(gindex).AT(Carpet::map).dd;
      const ibbox &bbox =
          dd->light_boxes.AT(mglevel)
              .AT(refinementlevel)
              .AT(group.disttype == CCTK_DISTRIB_CONSTANT ? 0 : component)
              .exterior;
      const ggf *const ff = arrdata.AT(gindex).AT(Carpet::map).data.AT(var);
      const gdata *const processor_component = ff->data_pointer(
          request->timelevel, refinementlevel,
          group.disttype == CCTK_DISTRIB_CONSTANT ? 0 : local_component,
          mglevel);

      // Don't create zero-sized components
      if (bbox.empty())
        continue;

      // As per Cactus convention, DISTRIB=CONSTANT arrays
      // (including grid scalars) are assumed to be the same on
      // all processors and are therefore stored only by processor 0.
      void *data = cctkGH->data[request->vindex][request->timelevel];
      const void *mydata = data;
      if (group.disttype == CCTK_DISTRIB_CONSTANT) {

        MPI_Datatype datatype;
        switch (specific_cactus_type(group.vartype)) {
#define TYPECASE(N, T)                                                         \
  case N: {                                                                    \
    T dummy;                                                                   \
    datatype = dist::mpi_datatype(dummy);                                      \
  } break;
#include "typecase.hh"
#undef TYPECASE
        default: {
          CCTK_WARN(CCTK_WARN_ABORT, "invalid datatype");
          abort();
        }
        }
        if (datatype == MPI_DATATYPE_NULL) {
          CCTK_VWarn(CCTK_WARN_ABORT, __LINE__, __FILE__, CCTK_THORNSTRING,
                     "MPI datatype for Cactus datatype %d is not defined",
                     group.vartype);
        }

        const size_t size = bbox.size() * CCTK_VarTypeSize(group.vartype);
        if (dist::rank() > 0) {
          data = malloc(size);
        }
        MPI_Bcast(data, bbox.size(), datatype, 0, MPI_COMM_WORLD);

        if (memcmp(mydata, data, size)) {
          CCTK_VWarn(2, __LINE__, __FILE__, CCTK_THORNSTRING,
                     "values for DISTRIB=CONSTANT grid variable '%s' "
                     "(timelevel %d) differ between processors 0 and %d; "
                     "only the array from processor 0 will be stored",
                     fullname, request->timelevel, component);
        }
      }

      // Construct a file-wide unique HDF5 dataset name
      // (only add parts with varying metadata)
      ostringstream datasetname;
      datasetname << fullname << " it=" << cctkGH->cctk_iteration
                  << " tl=" << request->timelevel;
      if (mglevels > 1)
        datasetname << " ml=" << mglevel;
      if (group.grouptype == CCTK_GF) {
        if (maps > 1)
          datasetname << " m=" << Carpet::map;
        datasetname << " rl=" << refinementlevel;
      }
      if (arrdata.at(gindex)
                  .at(Carpet::map)
                  .dd->light_boxes.at(mglevel)
                  .at(refinementlevel)
                  .size() > 1 and
          group.disttype != CCTK_DISTRIB_CONSTANT) {
        datasetname << " c=" << component;
      }

      // remove an already existing dataset of the same name
      if (request->check_exist) {
        H5E_BEGIN_TRY {
          H5Gunlink(outfile, datasetname.str().c_str());
          if (indexfile != -1)
            H5Gunlink(indexfile, datasetname.str().c_str());
        }
        H5E_END_TRY;
      }

      // Get the shape of the HDF5 dataset (in Fortran index order)
      hsize_t shape[dim];
      hsize_t num_elems = 1;
      hsize_t hyperslab_start[dim], hyperslab_count[dim];
      for (int d = 0; d < group.dim; ++d) {
        assert(group.dim - 1 - d >= 0 and group.dim - 1 - d < dim);
        shape[group.dim - 1 - d] = processor_component->padded_shape()[d];
        assert(
            all(processor_component->shape() == bbox.shape() / bbox.stride()));
        hyperslab_start[group.dim - 1 - d] = 0;
        hyperslab_count[group.dim - 1 - d] = processor_component->shape()[d];
        num_elems *= hyperslab_count[group.dim - 1 - d];
      }

      // Write the component as an individual dataset
      hid_t plist, dataspace, dataset, index_dataset = -1;
      HDF5_ERROR(plist = H5Pcreate(H5P_DATASET_CREATE));
      // enable compression if requested
      const int compression_lvl = request->compression_level >= 0
                                      ? request->compression_level
                                      : compression_level;
      const hsize_t size = num_elems * H5Tget_size(filedatatype);
      if (compression_lvl and size > hsize_t(minimum_size_for_compression)) {
        HDF5_ERROR(H5Pset_chunk(plist, group.dim, shape));
        HDF5_ERROR(H5Pset_shuffle(plist));
        HDF5_ERROR(H5Pset_deflate(plist, compression_lvl));
      }
      // enable checksums if requested
      if (use_checksums) {
        HDF5_ERROR(H5Pset_chunk(plist, group.dim, shape));
        HDF5_ERROR(H5Pset_filter(plist, H5Z_FILTER_FLETCHER32, 0, 0, NULL));
      }
      HDF5_ERROR(dataspace = H5Screate_simple(group.dim, shape, NULL));
      HDF5_ERROR(H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, hyperslab_start,
                                     NULL, hyperslab_count, NULL));
      HDF5_ERROR(dataset = H5Dcreate(outfile, datasetname.str().c_str(),
                                     filedatatype, dataspace, plist));

      if (indexfile != -1) {
        HDF5_ERROR(index_dataset =
                       H5Dcreate(indexfile, datasetname.str().c_str(),
                                 filedatatype, dataspace, H5P_DEFAULT));
      }

      io_bytes +=
          H5Sget_simple_extent_npoints(dataspace) * H5Tget_size(filedatatype);
      HDF5_ERROR(H5Pclose(plist));
      HDF5_ERROR(H5Sclose(dataspace));
      HDF5_ERROR(
          H5Dwrite(dataset, memdatatype, H5S_ALL, H5S_ALL, H5P_DEFAULT, data));
      error_count += AddAttributes(cctkGH, fullname, group.dim, refinementlevel,
                                   request, bbox, dataset);
      HDF5_ERROR(H5Dclose(dataset));

      if (indexfile != -1) {
        error_count +=
            AddAttributes(cctkGH, fullname, group.dim, refinementlevel, request,
                          bbox, index_dataset, true);
        HDF5_ERROR(H5Dclose(index_dataset));
      }

      if (data != mydata)
        free(data);
    }
    END_LOCAL_COMPONENT_LOOP;
  }
  END_LOCAL_MAP_LOOP;

  free(fullname);

  // return the number of errors that occured during this output
  return error_count;
}

// add attributes to an HDF5 dataset
static int AddAttributes(const cGH *const cctkGH, const char *fullname,
                         int vdim, int refinementlevel,
                         const ioRequest *request, const ibbox &bbox,
                         hid_t dataset, bool is_index) {
  assert(vdim >= 0 and vdim <= dim);
  int error_count = 0;

  bool is_scalar = CCTK_GroupTypeFromVarI(request->vindex) == CCTK_SCALAR;
  bool is_gf = CCTK_GroupTypeFromVarI(request->vindex) == CCTK_GF;

  // Legacy arguments
  hid_t attr, dataspace, datatype;
  HDF5_ERROR(dataspace = H5Screate(H5S_SCALAR));
  HDF5_ERROR(attr = H5Acreate(dataset, "level", H5T_NATIVE_INT, dataspace,
                              H5P_DEFAULT));
  HDF5_ERROR(H5Awrite(attr, H5T_NATIVE_INT, &refinementlevel));
  HDF5_ERROR(H5Aclose(attr));

  HDF5_ERROR(attr = H5Acreate(dataset, "carpet_mglevel", H5T_NATIVE_INT,
                              dataspace, H5P_DEFAULT));
  HDF5_ERROR(H5Awrite(attr, H5T_NATIVE_INT, &mglevel));
  HDF5_ERROR(H5Aclose(attr));

  HDF5_ERROR(attr = H5Acreate(dataset, "timestep", H5T_NATIVE_INT, dataspace,
                              H5P_DEFAULT));
  HDF5_ERROR(H5Awrite(attr, H5T_NATIVE_INT, &cctkGH->cctk_iteration));
  HDF5_ERROR(H5Aclose(attr));

  HDF5_ERROR(attr = H5Acreate(dataset, "group_timelevel", H5T_NATIVE_INT,
                              dataspace, H5P_DEFAULT));
  HDF5_ERROR(H5Awrite(attr, H5T_NATIVE_INT, &request->timelevel));
  HDF5_ERROR(H5Aclose(attr));

  HDF5_ERROR(attr =
                 H5Acreate(dataset, "time", HDF5_REAL, dataspace, H5P_DEFAULT));
  HDF5_ERROR(H5Awrite(attr, HDF5_REAL, &cctkGH->cctk_time));
  HDF5_ERROR(H5Aclose(attr));

  HDF5_ERROR(datatype = H5Tcopy(H5T_C_S1));
  HDF5_ERROR(H5Tset_size(datatype, strlen(fullname) + 1));
  HDF5_ERROR(attr =
                 H5Acreate(dataset, "name", datatype, dataspace, H5P_DEFAULT));
  HDF5_ERROR(H5Awrite(attr, datatype, fullname));
  HDF5_ERROR(H5Aclose(attr));
  HDF5_ERROR(H5Tclose(datatype));

  // Specify whether the coordinate system is Cartesian or not
  if (CCTK_IsFunctionAliased("MultiPatch_MapIsCartesian")) {
    int const map_is_cartesian = MultiPatch_MapIsCartesian(Carpet::map);
    HDF5_ERROR(attr = H5Acreate(dataset, "MapIsCartesian", H5T_NATIVE_INT,
                                dataspace, H5P_DEFAULT));
    HDF5_ERROR(H5Awrite(attr, H5T_NATIVE_INT, &map_is_cartesian));
    HDF5_ERROR(H5Aclose(attr));
  }

  HDF5_ERROR(H5Sclose(dataspace));

  // store cctk_bbox and cctk_nghostzones (for grid arrays only)
  if (!is_scalar) {
    vector<int> cctk_bbox(2 * vdim);
    hsize_t size = cctk_bbox.size();
    HDF5_ERROR(dataspace = H5Screate_simple(1, &size, NULL));
    CCTK_GroupbboxVI(cctkGH, size, &cctk_bbox[0], request->vindex);
    HDF5_ERROR(attr = H5Acreate(dataset, "cctk_bbox", H5T_NATIVE_INT, dataspace,
                                H5P_DEFAULT));
    HDF5_ERROR(H5Awrite(attr, H5T_NATIVE_INT, &cctk_bbox[0]));
    HDF5_ERROR(H5Aclose(attr));
    HDF5_ERROR(H5Sclose(dataspace));

    ivect cctk_nghostzones;
    size = vdim;
    HDF5_ERROR(dataspace = H5Screate_simple(1, &size, NULL));
    CCTK_GroupnghostzonesVI(cctkGH, size, &cctk_nghostzones[0],
                            request->vindex);
    HDF5_ERROR(attr = H5Acreate(dataset, "cctk_nghostzones", H5T_NATIVE_INT,
                                dataspace, H5P_DEFAULT));
    HDF5_ERROR(H5Awrite(attr, H5T_NATIVE_INT, &cctk_nghostzones[0]));
    HDF5_ERROR(H5Aclose(attr));
    HDF5_ERROR(H5Sclose(dataspace));
  }

  // write bbox attributes if we have coordinate system info
  CCTK_REAL origin[dim], delta[dim];
  int coord_system_handle = -1;
  if (CCTK_IsFunctionAliased("Coord_GroupSystem")) {
    char *groupname = CCTK_GroupNameFromVarI(request->vindex);
    coord_system_handle = Coord_GroupSystem(cctkGH, groupname);
    free(groupname);
  }

  hsize_t size = vdim;
  HDF5_ERROR(dataspace = H5Screate_simple(1, &size, NULL));

  CCTK_INT coord_handles[dim];
  if (coord_system_handle >= 0 and
      Util_TableGetIntArray(coord_system_handle, vdim, coord_handles,
                            "COORDINATES") >= 0) {
#if 0 // dh::dbases
    const ibbox& baseext =
      vdd.at(Carpet::map)->bases.at(mglevel).at(refinementlevel).exterior;
#endif
    const ibbox &baseext =
        vhh.at(Carpet::map)->baseextents.at(mglevel).at(refinementlevel);

    const ivect pos = (bbox.lower() - baseext.lower()) / bbox.stride();

    rvect global_lower;
    rvect coord_delta;
    const int m = Carpet::map;
    if (CCTK_GroupTypeFromVarI(request->vindex) == CCTK_GF) {
      rvect const cctk_origin_space = origin_space.at(m).at(mglevel);
      rvect const cctk_delta_space = delta_space.at(m) * rvect(mglevelfact);
      for (int d = 0; d < dim; ++d) {
        // lower boundary of Carpet's integer indexing
        global_lower[d] = cctk_origin_space[d];
        // grid spacing of Carpet's integer indexing
        coord_delta[d] = cctk_delta_space[d] / cctkGH->cctk_levfac[d];
      }
    } else {
      for (int d = 0; d < dim; ++d) {
        global_lower[d] = 0.0;
        coord_delta[d] = 1.0;
      }
    }

    for (int d = 0; d < dim; ++d) {
      origin[d] =
          (global_lower[d] +
           coord_delta[d] *
               (cctkGH->cctk_levoff[d] / cctkGH->cctk_levoffdenom[d] + pos[d]));
      delta[d] = coord_delta[d];
    }

    HDF5_ERROR(
        attr = H5Acreate(dataset, "origin", HDF5_REAL, dataspace, H5P_DEFAULT));
    HDF5_ERROR(H5Awrite(attr, HDF5_REAL, origin));
    HDF5_ERROR(H5Aclose(attr));
    HDF5_ERROR(
        attr = H5Acreate(dataset, "delta", HDF5_REAL, dataspace, H5P_DEFAULT));
    HDF5_ERROR(H5Awrite(attr, HDF5_REAL, delta));
    HDF5_ERROR(H5Aclose(attr));
  }

  ivect const ioffset =
      ((bbox.lower() % bbox.stride()) + bbox.stride()) % bbox.stride();
  ivect const iorigin = (bbox.lower() - ioffset) / bbox.stride();
  HDF5_ERROR(attr = H5Acreate(dataset, "iorigin", H5T_NATIVE_INT, dataspace,
                              H5P_DEFAULT));
  HDF5_ERROR(H5Awrite(attr, H5T_NATIVE_INT, &iorigin[0]));
  HDF5_ERROR(H5Aclose(attr));

  if (is_gf) {
    ivect const ioffsetdenom = bbox.stride();
    HDF5_ERROR(attr = H5Acreate(dataset, "ioffsetdenom", H5T_NATIVE_INT,
                                dataspace, H5P_DEFAULT));
    HDF5_ERROR(H5Awrite(attr, H5T_NATIVE_INT, &ioffsetdenom[0]));
    HDF5_ERROR(H5Aclose(attr));
    HDF5_ERROR(attr = H5Acreate(dataset, "ioffset", H5T_NATIVE_INT, dataspace,
                                H5P_DEFAULT));
    HDF5_ERROR(H5Awrite(attr, H5T_NATIVE_INT, &ioffset[0]));
    HDF5_ERROR(H5Aclose(attr));

    ostringstream buf;
    buf << (vdd.at(Carpet::map)
                ->local_boxes.at(mglevel)
                .at(refinementlevel)
                .at(local_component)
                .active);
    WriteAttribute(dataset, "active", buf.str().c_str());
  }

  if (is_index) {
    hsize_t shape[vdim];
    for (int d = 0; d < vdim; ++d) {
      assert(vdim - 1 - d >= 0 and vdim - 1 - d < vdim);
      shape[vdim - 1 - d] = (bbox.shape() / bbox.stride())[d];
    }

    HDF5_ERROR(attr = H5Acreate(dataset, "h5shape", H5T_NATIVE_HSIZE, dataspace,
                                H5P_DEFAULT));
    HDF5_ERROR(H5Awrite(attr, H5T_NATIVE_HSIZE, &shape[0]));
    HDF5_ERROR(H5Aclose(attr));
  }

  HDF5_ERROR(H5Sclose(dataspace));

  // return the number of errors that occured during this output
  return error_count;
}

} // namespace CarpetIOHDF5