Task: Implement LoadBalanceReal/carpet_boxtypes.F90 for Carpet/Carpet thorn

thorn_name: Carpet/Carpet
src_filename: LoadBalanceReal/carpet_boxtypes.F90
interface: # Interface definition for thorn Carpet

IMPLEMENTS: Driver

include header: carpet_public.hh in carpet.hh
include header: carpet_public.h in carpet.h

uses include header: Requirements.hh

uses include header: nompi.h

uses include header: loopcontrol.h

uses include header: CactusTimerSet.hh
uses include header: Timer.hh
uses include header: TimerTree.hh

uses include header: mpi_string.hh

uses include header: cacheinfo.hh
uses include header: defs.hh
uses include header: dist.hh
uses include header: typecase.hh
uses include header: typeprops.hh

uses include header: balance.hh
uses include header: bbox.hh
uses include header: bboxset.hh
uses include header: fulltree.hh
uses include header: region.hh
uses include header: vect.hh

uses include header: data.hh
uses include header: gdata.hh

uses include header: dh.hh
uses include header: gf.hh
uses include header: ggf.hh
uses include header: gh.hh
uses include header: th.hh

uses include header: operators.hh



# Get access to communicators
CCTK_POINTER_TO_CONST \
FUNCTION GetMPICommUniverse (CCTK_POINTER_TO_CONST IN cctkGH)
PROVIDES FUNCTION GetMPICommUniverse WITH Carpet_GetMPICommUniverse LANGUAGE C

CCTK_POINTER_TO_CONST \
FUNCTION GetMPICommWorld (CCTK_POINTER_TO_CONST IN cctkGH)
PROVIDES FUNCTION GetMPICommWorld WITH Carpet_GetMPICommWorld LANGUAGE C



# Get access to process->host mapping
CCTK_INT FUNCTION CCTK_MyHost(CCTK_POINTER_TO_CONST IN cctkGH)
CCTK_INT FUNCTION CCTK_nHosts(CCTK_POINTER_TO_CONST IN cctkGH)
CCTK_INT FUNCTION CCTK_nProcsOnHost(CCTK_POINTER_TO_CONST IN cctkGH, CCTK_INT IN host)
CCTK_INT FUNCTION CCTK_ProcsOnHost(CCTK_POINTER_TO_CONST IN cctkGH, CCTK_INT IN host, CCTK_INT ARRAY OUT procs, CCTK_INT IN nprocs)
PROVIDES FUNCTION CCTK_MyHost       WITH Carpet_MyHost       LANGUAGE C
PROVIDES FUNCTION CCTK_nHosts       WITH Carpet_nHosts       LANGUAGE C
PROVIDES FUNCTION CCTK_nProcsOnHost WITH Carpet_nProcsOnHost LANGUAGE C
PROVIDES FUNCTION CCTK_ProcsOnHost  WITH Carpet_ProcsOnHost  LANGUAGE C



# Register and unregister routines which are called before and after
# every scheduled routine
CCTK_INT \
FUNCTION RegisterScheduleWrapper \
  (CCTK_INT IN CCTK_FPOINTER func_before (CCTK_POINTER_TO_CONST IN cctkGH,    \
                                          CCTK_POINTER          IN function,  \
                                          CCTK_POINTER          IN attribute, \
                                          CCTK_POINTER          IN data),     \
   CCTK_INT IN CCTK_FPOINTER func_after  (CCTK_POINTER_TO_CONST IN cctkGH,    \
                                          CCTK_POINTER          IN function,  \
                                          CCTK_POINTER          IN attribute, \
                                          CCTK_POINTER          IN data))
CCTK_INT \
FUNCTION UnRegisterScheduleWrapper \
  (CCTK_INT IN CCTK_FPOINTER func_before (CCTK_POINTER_TO_CONST IN cctkGH,    \
                                          CCTK_POINTER          IN function,  \
                                          CCTK_POINTER          IN attribute, \
                                          CCTK_POINTER          IN data),     \
   CCTK_INT IN CCTK_FPOINTER func_after  (CCTK_POINTER_TO_CONST IN cctkGH,    \
                                          CCTK_POINTER          IN function,  \
                                          CCTK_POINTER          IN attribute, \
                                          CCTK_POINTER          IN data))
PROVIDES FUNCTION RegisterScheduleWrapper \
  WITH Carpet_RegisterScheduleWrapper \
  LANGUAGE C
PROVIDES FUNCTION UnRegisterScheduleWrapper \
  WITH Carpet_UnRegisterScheduleWrapper \
  LANGUAGE C



# Provide the function to en-/disable prolongation
CCTK_INT FUNCTION EnableProlongating(CCTK_INT IN flag)
PROVIDES FUNCTION EnableProlongating WITH CarpetEnableProlongating LANGUAGE C

CCTK_INT FUNCTION QueryProlongating()
PROVIDES FUNCTION QueryProlongating WITH CarpetQueryProlongating LANGUAGE C



# The number of time integrator substeps
CCTK_INT FUNCTION MoLNumIntegratorSubsteps ()
USES FUNCTION MoLNumIntegratorSubsteps




# The symmetry boundaries
CCTK_INT FUNCTION GetSymmetryBoundaries \
  (CCTK_POINTER_TO_CONST IN cctkGH, \
   CCTK_INT IN size, \
   CCTK_INT OUT ARRAY symbnd)
USES FUNCTION GetSymmetryBoundaries



# The location of the boundary points
CCTK_INT FUNCTION GetBoundarySpecification \
  (CCTK_INT IN size, \
   CCTK_INT OUT ARRAY nboundaryzones, \
   CCTK_INT OUT ARRAY is_internal, \
   CCTK_INT OUT ARRAY is_staggered, \
   CCTK_INT OUT ARRAY shiftout)
USES FUNCTION GetBoundarySpecification

# The overall size of the domain
CCTK_INT FUNCTION GetDomainSpecification \
  (CCTK_INT IN size, \
   CCTK_REAL OUT ARRAY physical_min, \
   CCTK_REAL OUT ARRAY physical_max, \
   CCTK_REAL OUT ARRAY interior_min, \
   CCTK_REAL OUT ARRAY interior_max, \
   CCTK_REAL OUT ARRAY exterior_min, \
   CCTK_REAL OUT ARRAY exterior_max, \
   CCTK_REAL OUT ARRAY spacing)
USES FUNCTION GetDomainSpecification

# Conversion between boundary types
CCTK_INT FUNCTION ConvertFromPhysicalBoundary \
  (CCTK_INT IN size, \
   CCTK_REAL IN  ARRAY physical_min, \
   CCTK_REAL IN  ARRAY physical_max, \
   CCTK_REAL OUT ARRAY interior_min, \
   CCTK_REAL OUT ARRAY interior_max, \
   CCTK_REAL OUT ARRAY exterior_min, \
   CCTK_REAL OUT ARRAY exterior_max, \
   CCTK_REAL IN  ARRAY spacing)
USES FUNCTION ConvertFromPhysicalBoundary

CCTK_INT FUNCTION ConvertFromExteriorBoundary \
  (CCTK_INT IN size, \
   CCTK_REAL OUT ARRAY physical_min, \
   CCTK_REAL OUT ARRAY physical_max, \
   CCTK_REAL OUT ARRAY interior_min, \
   CCTK_REAL OUT ARRAY interior_max, \
   CCTK_REAL IN  ARRAY exterior_min, \
   CCTK_REAL IN  ARRAY exterior_max, \
   CCTK_REAL IN  ARRAY spacing)
USES FUNCTION ConvertFromExteriorBoundary



# The setup of the system
CCTK_INT FUNCTION MultiPatch_GetSystemSpecification \
  (CCTK_INT OUT maps)
USES FUNCTION MultiPatch_GetSystemSpecification

# The location of the boundary points
CCTK_INT FUNCTION MultiPatch_GetBoundarySpecification \
  (CCTK_INT IN map, \
   CCTK_INT IN size, \
   CCTK_INT OUT ARRAY nboundaryzones, \
   CCTK_INT OUT ARRAY is_internal, \
   CCTK_INT OUT ARRAY is_staggered, \
   CCTK_INT OUT ARRAY shiftout)
USES FUNCTION MultiPatch_GetBoundarySpecification

# The overall size of the domain
CCTK_INT FUNCTION MultiPatch_GetDomainSpecification \
  (CCTK_INT IN map, \
   CCTK_INT IN size, \
   CCTK_REAL OUT ARRAY physical_min, \
   CCTK_REAL OUT ARRAY physical_max, \
   CCTK_REAL OUT ARRAY interior_min, \
   CCTK_REAL OUT ARRAY interior_max, \
   CCTK_REAL OUT ARRAY exterior_min, \
   CCTK_REAL OUT ARRAY exterior_max, \
   CCTK_REAL OUT ARRAY spacing)
USES FUNCTION MultiPatch_GetDomainSpecification

# Conversion between boundary types
CCTK_INT FUNCTION MultiPatch_ConvertFromPhysicalBoundary \
  (CCTK_INT IN map, \
   CCTK_INT IN size, \
   CCTK_REAL IN  ARRAY physical_min, \
   CCTK_REAL IN  ARRAY physical_max, \
   CCTK_REAL OUT ARRAY interior_min, \
   CCTK_REAL OUT ARRAY interior_max, \
   CCTK_REAL OUT ARRAY exterior_min, \
   CCTK_REAL OUT ARRAY exterior_max, \
   CCTK_REAL IN  ARRAY spacing)
USES FUNCTION MultiPatch_ConvertFromPhysicalBoundary



CCTK_INT FUNCTION Accelerator_Cycle             \
  (CCTK_POINTER_TO_CONST IN cctkGH)
USES FUNCTION Accelerator_Cycle

CCTK_INT FUNCTION Accelerator_PreCallFunction   \
  (CCTK_POINTER_TO_CONST IN cctkGH,             \
   CCTK_POINTER_TO_CONST IN attribute)
USES FUNCTION Accelerator_PreCallFunction

CCTK_INT FUNCTION Accelerator_PostCallFunction  \
  (CCTK_POINTER_TO_CONST IN cctkGH,             \
   CCTK_POINTER_TO_CONST IN attribute)
USES FUNCTION Accelerator_PostCallFunction

CCTK_INT FUNCTION Accelerator_PreSync           \
  (CCTK_POINTER_TO_CONST IN cctkGH,             \
   CCTK_INT ARRAY        IN groups,             \
   CCTK_INT              IN ngroups)
USES FUNCTION Accelerator_PreSync

CCTK_INT FUNCTION Accelerator_PostSync          \
  (CCTK_POINTER_TO_CONST IN cctkGH,             \
   CCTK_INT ARRAY        IN groups,             \
   CCTK_INT              IN ngroups)
USES FUNCTION Accelerator_PostSync

CCTK_INT FUNCTION Accelerator_NotifyDataModified        \
  (CCTK_POINTER_TO_CONST IN cctkGH,                     \
   CCTK_INT ARRAY        IN variables,                  \
   CCTK_INT ARRAY        IN rls,                        \
   CCTK_INT ARRAY        IN tls,                        \
   CCTK_INT              IN nvariables,                 \
   CCTK_INT              IN on_device)
USES FUNCTION Accelerator_NotifyDataModified

CCTK_INT FUNCTION Accelerator_RequireValidData          \
  (CCTK_POINTER_TO_CONST IN cctkGH,                     \
   CCTK_INT ARRAY        IN variables,                  \
   CCTK_INT ARRAY        IN rls,                        \
   CCTK_INT ARRAY        IN tls,                        \
   CCTK_INT              IN nvariables,                 \
   CCTK_INT              IN on_device)
USES FUNCTION Accelerator_RequireValidData



# Access coordinate information (on the coarse level)
CCTK_INT FUNCTION GetCoordRange        \
    (CCTK_POINTER_TO_CONST IN  cctkGH, \
     CCTK_INT              IN  m,      \
     CCTK_INT              IN  ml,     \
     CCTK_INT              IN  size,   \
     CCTK_INT        ARRAY OUT gsh,    \
     CCTK_REAL       ARRAY OUT lower,  \
     CCTK_REAL       ARRAY OUT upper,  \
     CCTK_REAL       ARRAY OUT delta)
PROVIDES FUNCTION GetCoordRange WITH Carpet_GetCoordRange LANGUAGE C



# Get current refinement level and number of refinement levels
CCTK_INT FUNCTION                         \
    GetRegriddingEpoch                    \
        (CCTK_POINTER_TO_CONST IN cctkGH)
PROVIDES FUNCTION GetRegriddingEpoch \
    WITH Carpet_GetRegriddingEpoch   \
    LANGUAGE C

CCTK_INT FUNCTION                         \
    GetRegriddingEpochs                   \
        (CCTK_POINTER_TO_CONST IN cctkGH, \
         CCTK_INT IN size,                \
         CCTK_INT ARRAY OUT epochs)
PROVIDES FUNCTION GetRegriddingEpochs \
    WITH Carpet_GetRegriddingEpochs   \
    LANGUAGE C

CCTK_INT FUNCTION                         \
    GetRefinementLevel                    \
        (CCTK_POINTER_TO_CONST IN cctkGH)
PROVIDES FUNCTION GetRefinementLevel \
    WITH Carpet_GetRefinementLevel   \
    LANGUAGE C

CCTK_INT FUNCTION                         \
    GetRefinementLevels                   \
        (CCTK_POINTER_TO_CONST IN cctkGH)
PROVIDES FUNCTION GetRefinementLevels \
    WITH Carpet_GetRefinementLevels   \
    LANGUAGE C

CCTK_INT FUNCTION                         \
    GetMaxRefinementLevels                \
        (CCTK_POINTER_TO_CONST IN cctkGH)
PROVIDES FUNCTION GetMaxRefinementLevels \
    WITH Carpet_GetMaxRefinementLevels   \
    LANGUAGE C

CCTK_INT FUNCTION                         \
    GetLocalComponent                     \
        (CCTK_POINTER_TO_CONST IN cctkGH)
PROVIDES FUNCTION GetLocalComponent \
    WITH Carpet_GetLocalComponent   \
    LANGUAGE C

CCTK_INT FUNCTION                         \
    GetLocalComponents                    \
        (CCTK_POINTER_TO_CONST IN cctkGH)
PROVIDES FUNCTION GetLocalComponents \
    WITH Carpet_GetLocalComponents   \
    LANGUAGE C

CCTK_INT FUNCTION                         \
    GetMap                                \
        (CCTK_POINTER_TO_CONST IN cctkGH)
PROVIDES FUNCTION GetMap \
    WITH Carpet_GetMap   \
    LANGUAGE C

CCTK_INT FUNCTION                         \
    GetMaps                               \
        (CCTK_POINTER_TO_CONST IN cctkGH)
PROVIDES FUNCTION GetMaps \
    WITH Carpet_GetMaps   \
    LANGUAGE C

CCTK_INT FUNCTION                         \
    GetTimeLevel                          \
        (CCTK_POINTER_TO_CONST IN cctkGH)
PROVIDES FUNCTION GetTimeLevel \
    WITH Carpet_GetTimeLevel   \
    LANGUAGE C

CCTK_INT FUNCTION                         \
    GetTimeLevelOffset                    \
        (CCTK_POINTER_TO_CONST IN cctkGH)
PROVIDES FUNCTION GetTimeLevelOffset \
    WITH Carpet_GetTimeLevelOffset   \
    LANGUAGE C



# Get pointer to grid variable for a specific map and refinement level
CCTK_POINTER FUNCTION                     \
    VarDataPtrI                           \
        (CCTK_POINTER_TO_CONST IN cctkGH, \
         CCTK_INT IN map,                 \
         CCTK_INT IN reflevel,            \
         CCTK_INT IN component,           \
         CCTK_INT IN timelevel,           \
         CCTK_INT IN varindex)
PROVIDES FUNCTION VarDataPtrI \
    WITH Carpet_VarDataPtrI   \
    LANGUAGE C



# The true prototype of the routine below:
#   int Carpet_Regrid (const cGH * cctkGH,
#                      gh::rregs * superregss,
#                      gh::mregs * regsss,
#                      int         force);
CCTK_INT FUNCTION Carpet_Regrid (CCTK_POINTER_TO_CONST IN cctkGH, \
                                 CCTK_POINTER IN superregss,      \
                                 CCTK_POINTER IN regsss,          \
                                 CCTK_INT IN force)
USES FUNCTION Carpet_Regrid

# The true prototype of the routine below:
#   int Carpet_Regrid (const cGH * cctkGH,
#                      vector<gh::rregs> * superregsss,
#                      vector<gh::mregs> * regssss,
#                      int                  force);
CCTK_INT FUNCTION Carpet_RegridMaps (CCTK_POINTER_TO_CONST IN cctkGH, \
                                     CCTK_POINTER IN superregsss,     \
                                     CCTK_POINTER IN regssss,         \
                                     CCTK_INT IN force)
USES FUNCTION Carpet_RegridMaps



# function to check whether existing output files should be truncated or not
CCTK_INT FUNCTION IO_TruncateOutputFiles \
  (CCTK_POINTER_TO_CONST IN GH)

REQUIRES FUNCTION IO_TruncateOutputFiles

CCTK_INT FUNCTION Boundary_SelectedGVs(CCTK_POINTER_TO_CONST IN GH, \
  CCTK_INT IN array_size, CCTK_INT ARRAY OUT var_indicies, \
  CCTK_INT ARRAY OUT faces, CCTK_INT ARRAY OUT boundary_widths, \
  CCTK_INT ARRAY OUT table_handles, CCTK_STRING IN bc_name)
USES FUNCTION Boundary_SelectedGVs


# TODO: make this somehow public, e.g. by moving it into its own thorn

CCTK_INT point_classification TYPE=gf TAGS='checkpoint="no" prolongation="none"'
{
  point_class
  # negative: needs to be set explicitly (e.g. boundary)
  # zero:     unused (e.g. ghost)
  # positive: needs to be evolved
  # -1:      boundary point (needs to be set explicitly)
  #  0:      unused (e.g. ghost point, or restriction target)
  #  n=1..N: evolved, used for integrator substeps i<=n
  #          (i=N..1, counting backwards; see MoL documentation) 
  #          i.e.: n=1: used between time steps (i.e., should be visualised)
  #                n>1: used only while time stepping (e.g. buffer zones)
} "Grid point classification"

CCTK_INT carpet_unusedpoints_mask TYPE=GF tags='Prolongation="none"' "mask which is set !=0 for points which do not influence future evolution, assuming appropriate regridding"



CCTK_REAL timing TAGS='checkpoint="no"'
{
  physical_time_per_hour
  current_physical_time_per_hour
  
  time_total time_evolution time_computing time_communicating time_io
  
  evolution_steps_count
  
  local_grid_points_per_second total_grid_points_per_second
  local_grid_point_updates_count total_grid_point_updates_count
  local_interior_points_per_second total_interior_points_per_second
  local_interior_point_updates_count total_interior_point_updates_count
  
  io_per_second
  io_bytes_per_second io_bytes_ascii_per_second io_bytes_binary_per_second
  io_count
  io_bytes_count io_bytes_ascii_count io_bytes_binary_count
  
  comm_per_second
  comm_bytes_per_second
  comm_count
  comm_bytes_count
  
  time_levels
} "Physical timing information"

CCTK_REAL timing_procs TYPE=array DIM=1 SIZE=1 DISTRIB=constant TAGS='checkpoint="no"'
{
  grid_points_per_second
  grid_point_updates_count
  interior_points_per_second
  interior_point_updates_count
} "Per-processor timing information"

CCTK_REAL timing_levels TYPE=array DIM=1 SIZE=max_refinement_levels DISTRIB=constant TAGS='checkpoint="no"'
{
  time_level time_level_count
} "Per-level timing information"

CCTK_REAL memory_procs TYPE=array DIM=1 SIZE=1 DISTRIB=CONSTANT TAGS='checkpoint="no"'
{
  metadata
  grid_structure
  grid_arrays
  grid_functions
} "Memory usage information"

schedule: # Schedule definitions for thorn Carpet

# This is disabled because its implementation is not complete
#storage: point_classification
storage: timing timing_procs timing_levels
storage: memory_procs

schedule CarpetMultiModelStartup at STARTUP as MultiModel_Startup before Driver_Startup
{
  LANG: C
} "Multi-model Startup routine"

schedule CarpetStartup at STARTUP as Driver_Startup
{
  LANG: C
} "Startup routine"

schedule CarpetParamCheck at PARAMCHECK
{
  LANG: C
} "Parameter checking routine"



# Correct time step for finer levels when there are non-trivial
# time refinement factors

if (refine_timestep)
{
  SCHEDULE CarpetRefineTimeStep AT basegrid AFTER Time_Simple
  {
    LANG: C
    OPTIONS: singlemap
  } "Correct time step size for spacing on finer grids"
}

if (use_unusedpoints_mask)
{
  storage: carpet_unusedpoints_mask

  schedule CarpetUnusedMask AT BASEGRID
  {
    LANG: C
  } "Set mask of unused points"

  schedule CarpetUnusedMask AT POSTREGRID
  {
    LANG: C
  } "Set mask of unused points"
}

param: # Parameter definitions for thorn Carpet

shares: Cactus

USES BOOLEAN terminate_next
USES KEYWORD terminate
USES CCTK_INT cctk_itlast
USES CCTK_REAL cctk_initial_time
USES CCTK_REAL cctk_final_time
USES CCTK_REAL max_runtime 


shares: InitBase

USES KEYWORD initial_data_setup_method


shares: IO

USES STRING out_dir


shares: CarpetLib

USES BOOLEAN use_higher_order_restriction
USES INT restriction_order_space


private:

BOOLEAN domain_from_coordbase "Use the domain description from CoordBase"
{
} "no"

BOOLEAN domain_from_multipatch "Use the domain description from MultiPatch"
{
} "no"



CCTK_INT global_nx "Grid size in x direction" STEERABLE=recover
{
  0:* :: "must be nonnegative"
} 10

CCTK_INT global_ny "Grid size in y direction" STEERABLE=recover
{
  0:* :: "must be nonnegative"
} 10

CCTK_INT global_nz "Grid size in z direction" STEERABLE=recover
{
  0:* :: "must be nonnegative"
} 10

CCTK_INT global_nsize "Grid size in each spatial direction" STEERABLE=recover
{
  0:* :: "must be nonnegative"
  -1  :: "use the per-dimension parameters"
} -1



CCTK_INT ghost_size_x "Ghost zones in x direction"
{
  0:* :: "must be nonnegative"
} 1

CCTK_INT ghost_size_y "Ghost zones in y direction"
{
  0:* :: "must be nonnegative"
} 1

CCTK_INT ghost_size_z "Ghost zones in z direction"
{
  0:* :: "must be nonnegative"
} 1

CCTK_INT ghost_size "Ghost zones in each spatial direction"
{
  0:* :: "must be nonnegative"
  -1  :: "use the per-dimension parameters"
} -1

STRING ghost_sizes "Number of ghost zones for each refinement level"
{
  "^$" :: "Use the value of ghost_size or ghost_size_[xyz]"
  "^[[:space:]]*\[[[:space:]]*[[:digit:]]+[[:space:]]*(,[[:space:]]*[[:digit:]]+[[:space:]]*)*\][[:space:]]*$" :: "[ <ghost_size>, ... ]"
} ""



restricted:

# Carpet ignores all periodic* parameters and aborts in ParamCheck if any is
# changed to "yes"
BOOLEAN periodic "do not use this parameter"
{
} "no"

BOOLEAN periodic_x "do not use this parameter"
{
} "yes"

BOOLEAN periodic_y "do not use this parameter"
{
} "yes"

BOOLEAN periodic_z "do not use this parameter"
{
} "yes"



private:

KEYWORD refinement_centering "Centering"
{
  "vertex" :: "use a vertex centred grid structure"
  "cell"   :: "use a cell centred grid structure"
} "vertex"

KEYWORD eno_interpolation_type "What is represented by values in cells DEPRECATED"
{
  "samples"  :: "grid values a sample values of the solution"
} "samples"

CCTK_INT max_refinement_levels "Maximum number of refinement levels (including the base level)"
{
  1:* :: "must be positive"
} 1

CCTK_INT max_timelevels "Maximum number of time levels (including the current time level)"
{
  -1 :: "Set automatically to prolonation_order_time+1"
  1:* :: "Set this explicitly"
} -1

CCTK_INT refinement_factor "Refinement factor"
{
  1:* :: "must be positive"
} 2

STRING space_refinement_factors "Spatial refinement factors over the coarsest level"
{
  "^$" :: "Use the value of refinement_factor"
  # V = [SDS(,SDS)*]
  # L = [SVS(,SVS)*]
  #   = [S[SDS(,SDS)*]S(,S[SDS(,SDS)*]S)*]
  "^[[:space:]]*\[[[:space:]]*\[[[:space:]]*[[:digit:]]+[[:space:]]*(,[[:space:]]*[[:digit:]]+[[:space:]]*)*\][[:space:]]*(,[[:space:]]*\[[[:space:]]*[[:digit:]]+[[:space:]]*(,[[:space:]]*[[:digit:]]+[[:space:]]*)*\][[:space:]]*)*\][[:space:]]*$" :: "[ [<ifact>,<jfact>,<kfact>], ... ]"
} ""

STRING time_refinement_factors "Temporal refinement factors over the coarsest level"
{
  "^$" :: "Use the value of refinement_factor"
  # L = [SDS(,SDS)*]
  "^[[:space:]]*\[[[:space:]]*[[:digit:]]+[[:space:]]*(,[[:space:]]*[[:digit:]]+[[:space:]]*)*\][[:space:]]*$" :: "[ <tfact>, ... ]"
} ""

BOOLEAN refine_timestep "Correct Time::dtfac for spacings on finer grids"
{
} "no"



CCTK_INT convergence_level "Convergence level"
{
  *:* :: "negative for finer, positive for coarser resolutions"
} 0

CCTK_INT num_convergence_levels "Number of convergence levels (including the base level)"
{
  1:* :: "must be positive"
} 1

CCTK_INT convergence_factor "Multigrid factor"
{
  1:* :: "must be positive"
} 2



CCTK_INT num_maps "Number of maps"
{
  1:* :: ""
} 1



STRING model "Model name for multi-model simulations -- the model name is used to distribute the processors onto the models"
{
  ".+" :: ""
} "world"



CCTK_INT prolongation_order_space "Order of prolongation operator in space" STEERABLE=recover
{
  0:* :: "vertex centred orders must be odd"
} 1

STRING prolongation_orders_space "Order of prolongation operator in space for each refinement level" STEERABLE=recover
{
  "^$" :: "Use the value of prolongation_order_space"
  "^[[:space:]]*\[[[:space:]]*[[:digit:]]+[[:space:]]*(,[[:space:]]*[[:digit:]]+[[:space:]]*)*\][[:space:]]*$" :: "[ <order>, ... ]"
} ""

CCTK_INT prolongation_order_time "Order of prolongation operator in time" STEERABLE=recover
{
  0:* :: ""
} 1



BOOLEAN use_buffer_zones "Use buffer zones"
{
} "no"

CCTK_INT additional_buffer_zones "Additional buffer zones"
{
  *:* :: ""
} 0

BOOLEAN use_overlap_zones "Use overlap zones"
{
} "no"

CCTK_INT additional_overlap_zones "Additional overlap zones"
{
  *:* :: ""
} 0

BOOLEAN use_tapered_grids "Use tapered grids, avoiding time interpolation during evolution"
{
} "no"

CCTK_INT num_integrator_substeps "Number of substeps of the time integrator"
{
  -1: :: "Call MoLNumIntegratorSubsteps"
  0:* :: ""
} -1



BOOLEAN sync_during_time_integration "Synchronise during time integration, even when prolongation is switched off"
{
} "yes"



CCTK_STRING base_extents "Extents of base grid components, in grid point units of the finest level"
{
  "^$" :: "leave empty for one grid component covering the whole region (default)"

# We want the string to contain a list of bboxes.  Each bbox contains
# three vectors specifying the lower bound, upper bound, and stride.
# (The upper bound is inclusive.  All values are nonnegative integers.)
# The syntax for vectors, bboxes, and lists is described below.
# All spaces are optional.

# The () parentheses and [] brackets are literals.
# The {} braces are used for grouping.
# If you put everything together, you get the tapeworm below.
# There should be a way to define abbreviations in regexps!
# Having #defines in param.ccl files would help a lot.
# VECT := " [ 0 , 0 , 0 ]"
# BBOX := " ( VECT : VECT : VECT )"
# LIST := " [ {{ BBOX }* BBOX}? ]"

# Almost human readable explanation follows.
# The specification is hierarchical.

# The following definitions are generic:
# VECT3(x) := "[ x, x, x ]"
#    three x, separated by commas, enclosed in square brackets
# BBOX3(x) := "( x : x : x )"
#    three x, separated by colons, enclosed in round parentheses
# LIST(x) := "[ {{ x, }* x}? ]"
#    zero or more x, separated by commas, enclosed in square brackets

# A domain is built up as follows:
# REGION   := BBOX3(VECT3(double))
# DOMAIN   := LIST(REGION)

  "^[[:space:]]*\[(([[:space:]]*\([[:space:]]*\[[[:space:]]*[[:digit:]]+[[:space:]]*,[[:space:]]*[[:digit:]]+[[:space:]]*,[[:space:]]*[[:digit:]]+[[:space:]]*\][[:space:]]*:[[:space:]]*\[[[:space:]]*[[:digit:]]+[[:space:]]*,[[:space:]]*[[:digit:]]+[[:space:]]*,[[:space:]]*[[:digit:]]+[[:space:]]*\][[:space:]]*:[[:space:]]*\[[[:space:]]*[[:digit:]]+[[:space:]]*,[[:space:]]*[[:digit:]]+[[:space:]]*,[[:space:]]*[[:digit:]]+[[:space:]]*\][[:space:]]*\)[[:space:]]*,)*[[:space:]]*\([[:space:]]*\[[[:space:]]*[[:digit:]]+[[:space:]]*,[[:space:]]*[[:digit:]]+[[:space:]]*,[[:space:]]*[[:digit:]]+[[:space:]]*\][[:space:]]*:[[:space:]]*\[[[:space:]]*[[:digit:]]+[[:space:]]*,[[:space:]]*[[:digit:]]+[[:space:]]*,[[:space:]]*[[:digit:]]+[[:space:]]*\][[:space:]]*:[[:space:]]*\[[[:space:]]*[[:digit:]]+[[:space:]]*,[[:space:]]*[[:digit:]]+[[:space:]]*,[[:space:]]*[[:digit:]]+[[:space:]]*\][[:space:]]*\))?[[:space:]]*\][[:space:]]*$" :: "[ ([<imin>,<jmin>,<kmin>]:[<imax>,<jmax>,<kmax>]:[<istride>,<jstride>,<kstride>]), ... ]"
} ""

CCTK_STRING base_outerbounds "Outer boundaries of base grid components"
{
  "^$" :: "leave empty for using the default, which depends on cctk_gsh"

# See above for an explanation of this syntax, and of the tapeworm below.
# Each vector element is 0 or 1,
# where 0 is handled by synchronisation or prolongation,
# and 1 stands for a user-supplied ("outer") boundary condition.

# BND := " [ 0 , 0 ]"
# VECT := " [ BND , BND , BND ]"
# LIST := " [{{VECT ,}*VECT}? ]"

# Almost human readable explanation follows.
# The specification is hierarchical.

# The following definitions are generic:
# VECT2(x) := "[ x, x ]"
#    two x, separated by commas, enclosed in square brackets
# VECT3(x) := "[ x, x, x ]"
#    three x, separated by commas, enclosed in square brackets
# LIST(x) := "[ {{ x, }* x}? ]"
#    zero or more x, separated by commas, enclosed in square brackets

# A domain is built up as follows:
# REGION   := VECT3(VECT2(double))
# DOMAIN   := LIST(REGION)

  "^[[:space:]]*\[(([[:space:]]*\[[[:space:]]*\[[[:space:]]*[[:digit:]]+[[:space:]]*,[[:space:]]*[[:digit:]]+[[:space:]]*\][[:space:]]*,[[:space:]]*\[[[:space:]]*[[:digit:]]+[[:space:]]*,[[:space:]]*[[:digit:]]+[[:space:]]*\][[:space:]]*,[[:space:]]*\[[[:space:]]*[[:digit:]]+[[:space:]]*,[[:space:]]*[[:digit:]]+[[:space:]]*\][[:space:]]*\][[:space:]]*,)*[[:space:]]*\[[[:space:]]*\[[[:space:]]*[[:digit:]]+[[:space:]]*,[[:space:]]*[[:digit:]]+[[:space:]]*\][[:space:]]*,[[:space:]]*\[[[:space:]]*[[:digit:]]+[[:space:]]*,[[:space:]]*[[:digit:]]+[[:space:]]*\][[:space:]]*,[[:space:]]*\[[[:space:]]*[[:digit:]]+[[:space:]]*,[[:space:]]*[[:digit:]]+[[:space:]]*\][[:space:]]*\])?[[:space:]]*\][[:space:]]*$" :: "[ [ [<ilower>,<iupper>], [<jlower>,<jupper>], [<klower>,<kupper>] ], ... ]"
} ""



BOOLEAN enable_all_storage "Enable storage for all grid functions" STEERABLE=recover
{
} "no"

BOOLEAN enable_no_storage "Exit before beginning to enable storage for grid functions" STEERABLE=recover
{
} "no"



BOOLEAN poison_new_timelevels "Try to catch uninitialised grid elements by setting new timelevels to values that will catch your attention" STEERABLE=always
{
} "yes"

BOOLEAN check_for_poison "Explicitely check for the poison value after every time step" STEERABLE=always
{
} "no"

CCTK_INT poison_value "UNUSED; use CarpetLib::poison_value instead" STEERABLE=always
{
  * :: ""
} 0

CCTK_INT max_poison_locations "Maximum number of poison locations that are printed to the screen" STEERABLE=always
{
  -1  :: "print all locations"
  0:* :: "print only that many locations"
} 10



BOOLEAN checksum_timelevels "Try to catch unintentionally changed timelevels by taking checksums and comparing against these" STEERABLE=always
{
} "no"



BOOLEAN suppress_restriction "Suppress the restriction operations.  This makes the coarser refinement levels independent of the finer ones." STEERABLE=always
{
} "no"



BOOLEAN verbose "Display more info on the screen" STEERABLE=always
{
} "no"

BOOLEAN veryverbose "Display a lot of info on the screen" STEERABLE=always
{
} "no"

BOOLEAN storage_verbose "Display verbose storage information if veryverbose" STEERABLE=ALWAYS
{
} "no"

BOOLEAN barriers "Insert barriers at strategic places for debugging purposes (slows down execution)" STEERABLE=always
{
} "no"

BOOLEAN schedule_barriers "Insert barriers between scheduled items, so that timer statistics become more reliable (slows down execution)" STEERABLE=always
{
} "no"

BOOLEAN sync_barriers "Insert barriers before and after syncs, so that the sync timer is more reliable (slows down execution)" STEERABLE=always
{
} "no"

BOOLEAN output_internal_data "Periodically print internal data to the screen for debugging purposes" STEERABLE=always
{
} "no"

REAL timing_average_window_minutes "Time interval (in wall time minutes) for calculating the current physics time per hour" STEERABLE=always
{
  (0.0:* :: ""
} 10.0

INT print_timestats_every "Print interesting timing statistics periodically" STEERABLE=always
{
  -1  :: "don't report"
  0   :: "don't report"
  1:* :: "report every so many iterations"
} 0

BOOLEAN print_grid_info "Print information about the grids on regridding" STEERABLE=always
{
} yes

INT output_timers_every "Print detailed statistics periodically" STEERABLE=always
{
  -1  :: "don't report"
  0   :: "don't report"
  1:* :: "report every so many iterations"
} 0

STRING timer_file "File name in which detailed timing statistics are collected" STEERABLE=recover
{
  "^$"   :: "empty filename: no file output"
  "^.+$" :: "file name"
} "carpet-timing-statistics"

BOOLEAN output_initialise_timer_tree "Output timing information in tree form to standard output for Initialise"
{
} "no"

INT output_timer_tree_every "Output timing information in tree form to standard output for Evolve every so many iterations" STEERABLE=always
{
  0   :: "don't report"
  1:* :: "report every so many iterations"
} 0

BOOLEAN output_xml_timer_tree "Output timing information in tree form as XML" STEERABLE=always
{
} "no"

BOOLEAN include_maps_in_mode_timer_tree  "Include a breakdown of time spent on different multipatch maps in the mode timer tree" STEERABLE=always
{
} "no"

BOOLEAN include_local_mode_in_mode_timer_tree  "Include a node for the time spent in local mode in the mode timer tree" STEERABLE=always
{
} "no"



BOOLEAN recompose_verbose "Output debug information during recomposing" STEERABLE=ALWAYS
{
} "no"

KEYWORD processor_topology "How to determine the processor topology" STEERABLE=recover
{
  "manual"    :: "Specified by processor_topology_*"
  "along-z"   :: "Split the region along the z direction only"
  "along-dir" :: "Split the region along one direction only"
  "automatic" :: "Choose the topology automatically"
  "recursive" :: "Choose the topology automatically, using a different algorithm that may lead to better load balancing"
  "balanced"  :: "Choose the topology automatically, ensuring a maximum load imbalance"
} "automatic"

CCTK_INT processor_topology_3d_x "Number of processors in x-direction" STEERABLE=recover
{
  1:* :: "must be positive"
} 1

CCTK_INT processor_topology_3d_y "Number of processors in y-direction" STEERABLE=recover
{
  1:* :: "must be positive"
} 1

CCTK_INT processor_topology_3d_z "Number of processors in z-direction" STEERABLE=recover
{
  1:* :: "must be positive"
} 1

CCTK_INT split_direction "Direction in which the domain should be split (for processor_topology=along-dir)" STEERABLE=recover
{
  0:* :: "0 for x, 1 for y, 2 for z, etc."
} 2

CCTK_INT no_split_direction "Direction in which the domain must not be split (for processor_topology=automatic)" STEERABLE=recover
{
  -1  :: "split in all directions"
  0:* :: "0 for x, 1 for y, 2 for z, etc."
} -1

BOOLEAN constant_load_per_processor "Keep the load per processor constant -- this is meant for benchmarks" STEERABLE=recover
{
} "no"

CCTK_REAL aspect_ratio_x "Desired aspect ratio for each processor's domain" STEERABLE=always
{
  (0:* :: ""
} 1.0

CCTK_REAL aspect_ratio_y "Desired aspect ratio for each processor's domain" STEERABLE=always
{
  (0:* :: ""
} 1.0

CCTK_REAL aspect_ratio_z "Desired aspect ratio for each processor's domain" STEERABLE=always
{
  (0:* :: ""
} 1.0

CCTK_INT min_points_per_proc "Minimum number of grid points per processor" STEERABLE=always
{
  0:* :: "that many"
} 0

BOOLEAN split_components "Split components onto processes; without this, one needs many components and few processes" STEERABLE=always
{
} "yes"

CCTK_INT granularity "When splitting components, create sizes that are multiples of this granularity" STEERABLE=always
{
  1:* :: "TODO: query CoordBase or related thorns for this information"
} 1

CCTK_INT granularity_boundary "When splitting components, assume this many boundary points that don't count towards the granularity" STEERABLE=always
{
  0:* :: "TODO: use CoordBase's number of boundary points for this"
} 0

CCTK_REAL ghost_zone_cost "Relative cost of ghost zones for 'recursive' load balancing" STEERABLE=always
{
  0:* :: ""
} 0.025

CCTK_REAL maximum_imbalance "Maximum load imbalance" STEERABLE=always
{
  (0.0:* :: ""
} 0.1

BOOLEAN same_number_of_components_on_each_process "Ensure that each process has the same number of components, adding empty dummy components if necessary" STEERABLE=always
{
} "yes"



CCTK_INT num_threads "Number of threads per process" STEERABLE=recover
{
  -1  :: "use system default, probably influenced by OMP_NUM_THREADS"
  1:* :: "use this many threads"
} -1

BOOLEAN set_cpu_affinity "Set the process CPU affinity, overwriting the respective system setting" STEERABLE=recover
{
} "no"



STRING grid_structure_filename "File name to output grid structure to (empty = no output)" STEERABLE=recover
{
  ".*" :: "must be a legal file name"
} ""

STRING grid_coordinates_filename "File name to output grid coordinates to (empty = no output)" STEERABLE=recover
{
  ".*" :: "must be a legal file name"
} ""



private:

BOOLEAN init_each_timelevel "Call initial data routines once for each timelevel" STEERABLE=always
{
} "no"

BOOLEAN init_fill_timelevels "Fill past time levels from current time level after calling initial data routines" STEERABLE=always
{
} "no"

BOOLEAN prolongate_initial_data "Prolongate the refined regions during initial data generation" STEERABLE=always
{
} "no"

BOOLEAN regrid_during_initialisation "Regrid while initialising" STEERABLE=recover
{
} "no"

BOOLEAN regrid_during_recovery "Regrid while recovering" STEERABLE=recover
{
} "no"

BOOLEAN regrid_in_level_mode "Regrid in level mode (instead of singlemap mode), enabling more efficient processor distributions when there are multiple maps" STEERABLE=always
{
} "yes"

BOOLEAN time_interpolation_during_regridding "Interpolate finer levels in time during regridding" STEERABLE=recover
{
} "yes"

BOOLEAN output_after_regridding "Call OutputGH after regridding" STEERABLE=always
{
} "no"



BOOLEAN init_3_timelevels "Set up 3 timelevels of initial data" STEERABLE=always
{
} "no"



BOOLEAN adaptive_stepsize "Allow adaptive timestep sizes"
{
} "no"

BOOLEAN use_unusedpoints_mask "Turn on storage and usage of 'unusedpoints_mask'" STEERABLE=recover
{
} "no"

configuration: # Configuration definitions for thorn Carpet

PROVIDES Carpet
{
}

REQUIRES CarpetLib IOUtil MPI Timers

OPTIONAL LoopControl
{
}

context: UnusedMask.cc:
```
#include <cctk.h>
#include <cctk_Arguments.h>
#include <cctk_Parameters.h>

#include <carpet.hh>

#include <loopcontrol.h>

namespace Carpet {

using namespace std;

void CarpetUnusedMask(CCTK_ARGUMENTS) {
  DECLARE_CCTK_ARGUMENTS;
  DECLARE_CCTK_PARAMETERS;

  dh const &dd = *vdd.AT(map);
  ibbox const &ext =
      dd.light_boxes.AT(mglevel).AT(reflevel).AT(component).exterior;
  ibset const &unused_region =
      dd.local_boxes.AT(mglevel).AT(reflevel).AT(local_component).unused_region;

  assert(dim == 3);
  // Zero out
  LOOP_OVER_BSET(cctkGH, ext, box, imin, imax) {
#pragma omp parallel
    CCTK_LOOP3(unused_mask_zero, i, j, k, imin[0], imin[1], imin[2], imax[0],
               imax[1], imax[2], cctk_ash[0], cctk_ash[1], cctk_ash[2]) {
      CCTK_INT i3D = CCTK_GFINDEX3D(cctkGH, i, j, k);
      carpet_unusedpoints_mask[i3D] = 0;
    }
    CCTK_ENDLOOP3(unused_mask_zero);
  }
  END_LOOP_OVER_BSET;

  // Set it where unused
  LOOP_OVER_BSET(cctkGH, unused_region, box, imin, imax) {
#pragma omp parallel
    CCTK_LOOP3(unused_mask_set, i, j, k, imin[0], imin[1], imin[2], imax[0],
               imax[1], imax[2], cctk_ash[0], cctk_ash[1], cctk_ash[2]) {
      CCTK_INT i3D = CCTK_GFINDEX3D(cctkGH, i, j, k);
      carpet_unusedpoints_mask[i3D] = 1;
    }
    CCTK_ENDLOOP3(unused_mask_set);
  }
  END_LOOP_OVER_BSET;

} // CarpetUnusedMask
}

```

Restrict.cc:
```
#include <cassert>
#include <cmath>
#include <cstdlib>

#include <cctk.h>
#include <cctk_Parameters.h>

#include <Requirements.hh>

#include <Timer.hh>

#include <ggf.hh>
#include <gh.hh>

#include <carpet.hh>

namespace Carpet {

using namespace std;

// restricts a set of groups
static void RestrictGroups(const cGH *cctkGH, const vector<int> &groups);

void Restrict(const cGH *cctkGH) {
  DECLARE_CCTK_PARAMETERS;

  assert(is_level_mode());

  if (suppress_restriction) {
    Checkpoint("Restriction suppressed");
    return;
  }

  Checkpoint("Restrict");

  // all but the finest level are restricted
  if (reflevel == reflevels - 1) {
    return;
  }

  // remove all groups that are non-GFs, empty, or have no storage assigned
  vector<int> groups;
  groups.reserve(CCTK_NumGroups());

  for (int group = 0; group < CCTK_NumGroups(); ++group) {
    operator_type const op = groupdata.AT(group).transport_operator;
    bool const do_restrict = op != op_none and op != op_sync;
    if (do_restrict and CCTK_NumVarsInGroupI(group) > 0 and
        CCTK_QueryGroupStorageI(cctkGH, group)) {
      groups.push_back(group);
    }
  }
  if (groups.empty())
    return;

#ifdef REQUIREMENTS_HH
  Requirements::Restrict(groups, cctkGH->cctk_iteration, reflevel);
#endif

  // Restrict
  {
    static Timers::Timer timer("Restrict");
    timer.start();
    RestrictGroups(cctkGH, groups);
    timer.stop();
  }

  // Note: Carpet used to call SyncGroups here. This is now done in
  // Evolve.cc.
}

// restrict a set of groups
static void RestrictGroups(const cGH *cctkGH, const vector<int> &groups) {
  DECLARE_CCTK_PARAMETERS;

  // TODO: Transmit only the regions that are actually needed
  if (CCTK_IsFunctionAliased("Accelerator_RequireValidData")) {
    vector<CCTK_INT> vis, rls, tls;
    for (int group = 0; group < (int)groups.size(); ++group) {
      const int g = groups.AT(group);
      const int active_tl = CCTK_ActiveTimeLevelsGI(cctkGH, g);
      assert(active_tl >= 0);
      const int tl = active_tl > 1 ? timelevel : 0;
      for (int m = 0; m < (int)arrdata.AT(g).size(); ++m) {
        const int var0 = CCTK_FirstVarIndexI(g);
        const int varn = CCTK_NumVarsInGroupI(g);
        for (int vi = var0; vi < var0 + varn; ++vi) {
          vis.push_back(vi);
          rls.push_back(reflevel);
          tls.push_back(tl);

          vis.push_back(vi);
          rls.push_back(reflevel + 1);
          tls.push_back(tl);
        }
      }
    }
    const CCTK_INT on_device = 0;
    Accelerator_RequireValidData(cctkGH, &vis.front(), &rls.front(),
                                 &tls.front(), vis.size(), on_device);
  }

  static vector<Timers::Timer *> timers;
  if (timers.empty()) {
    timers.push_back(new Timers::Timer("comm_state[0].create"));
    for (astate state = static_cast<astate>(0); state != state_done;
         state = static_cast<astate>(static_cast<int>(state) + 1)) {
      ostringstream name1;
      name1 << "comm_state[" << timers.size() << "]"
            << "." << tostring(state) << ".user";
      timers.push_back(new Timers::Timer(name1.str()));
      ostringstream name2;
      name2 << "comm_state[" << timers.size() << "]"
            << "." << tostring(state) << ".step";
      timers.push_back(new Timers::Timer(name2.str()));
    }
  }

  vector<Timers::Timer *>::iterator ti = timers.begin();
  (*ti)->start();
  for (comm_state state; not state.done(); state.step()) {
    (*ti)->stop();
    ++ti;
    (*ti)->start();
    for (int group = 0; group < (int)groups.size(); ++group) {
      const int g = groups.AT(group);
      const int active_tl = CCTK_ActiveTimeLevelsGI(cctkGH, g);
      assert(active_tl >= 0);
      const int tl = active_tl > 1 ? timelevel : 0;
      for (int m = 0; m < (int)arrdata.AT(g).size(); ++m) {
        for (int v = 0; v < (int)arrdata.AT(g).AT(m).data.size(); ++v) {
          ggf *const gv = arrdata.AT(g).AT(m).data.AT(v);
          gv->ref_restrict_all(state, tl, reflevel, mglevel);
        }
      }
    } // loop over groups
    (*ti)->stop();
    ++ti;
    (*ti)->start();
  } // for state
  (*ti)->stop();
  ++ti;
  assert(ti == timers.end());

  // TODO: Transmit only the regions that are actually needed
  if (CCTK_IsFunctionAliased("Accelerator_NotifyDataModified")) {
    vector<CCTK_INT> vis, rls, tls;
    for (int group = 0; group < (int)groups.size(); ++group) {
      const int g = groups.AT(group);
      const int active_tl = CCTK_ActiveTimeLevelsGI(cctkGH, g);
      assert(active_tl >= 0);
      const int tl = active_tl > 1 ? timelevel : 0;
      for (int m = 0; m < (int)arrdata.AT(g).size(); ++m) {
        const int var0 = CCTK_FirstVarIndexI(g);
        const int varn = CCTK_NumVarsInGroupI(g);
        for (int vi = var0; vi < var0 + varn; ++vi) {
          vis.push_back(vi);
          rls.push_back(reflevel);
          tls.push_back(tl);
        }
      }
    }
    const CCTK_INT on_device = 0;
    Accelerator_NotifyDataModified(cctkGH, &vis.front(), &rls.front(),
                                   &tls.front(), vis.size(), on_device);
  }
}

} // namespace Carpet

```

Doxygen.hh:
```
/**
   \file

   \mainpage

   \author Erik Schnetter
   \date August, 2010

   http://www.carpetcode.org

   %Carpet is a parallel AMR (adaptive mesh refinement) driver with MPI
   for Cactus.  This documentation is for developers of %Carpet (users
   of %Carpet should instead see the User Documentation obtained by
   typing "make Carpet-ThornDoc" in the Cactus directory).

   Important source files:

     Carpet/src/Initialise.cc

     Carpet/src/Evolve.cc

     Carpet/src/Recompose.cc

   Important classes:

     vect
     bbox
     bboxset
     Carpet::bbvect
     region_t

*/

```

Initialise.cc:
```
#include <cassert>
#include <cstdlib>
#include <cstring>
#include <iostream>
#include <map>
#include <string>
#include <sstream>
#include <fstream>

#include <cctk.h>
#include <cctk_Parameters.h>
#include <cctki_GHExtensions.h>
#include <cctki_ScheduleBindings.h>
#include <cctki_WarnLevel.h>

#include <Requirements.hh>

#include <CactusTimerSet.hh>
#include <Timer.hh>

#include <carpet.hh>

namespace Carpet {

using namespace std;

static void CallSetup(cGH *cctkGH);
static void CallRecoverVariables(cGH *cctkGH);
#if 0
  static void CallRegridRecoverMeta (cGH * cctkGH);
#endif
static void CallRegridRecoverLevel(cGH *cctkGH);
static void CallRegridInitialMeta(cGH *cctkGH);
static void CallRegridInitialLevel(cGH *cctkGH);
static void CallPostRecoverVariables(cGH *cctkGH);
static void CallInitial(cGH *cctkGH);
static void CallRestrict(cGH *cctkGH);
static void CallPostInitial(cGH *cctkGH);
static void CallAnalysis(cGH *cctkGH, bool did_recover);

static void Initialise3tl(cGH *cctkGH);

static void print_internal_data();

static void ScheduleTraverse(char const *where, char const *name, cGH *cctkGH);
static void OutputGH(char const *where, cGH *cctkGH);

int Initialise(tFleshConfig *const fc) {
  DECLARE_CCTK_PARAMETERS;

  int const convlev = 0;
  cGH *const cctkGH = CCTK_SetupGH(fc, convlev);
  CCTKi_AddGH(fc, convlev, cctkGH);

  do_global_mode = true;
  do_early_global_mode = true;
  do_late_global_mode = true;
  do_meta_mode = true;
  do_early_meta_mode = true;
  do_late_meta_mode = true;
  global_time = cctk_initial_time;
  delta_time = 1.0;
  for (int ml = 0; ml < mglevels; ++ml) {
    // assert (leveltimes.AT(ml).size() == 1);
    // leveltimes.AT(ml).AT(0) = global_time;
    for (int rl = 0; rl < reflevels; ++rl) {
      CCTK_REAL const dt = delta_time / timereffacts.AT(rl);
      for (int tl = 0; tl < tt->timelevels; ++tl) {
        tt->set_time(ml, rl, tl, global_time - tl * dt);
      }
    }
  }

  cctkGH->cctk_iteration = 0;
  cctkGH->cctk_time = global_time;
  cctkGH->cctk_delta_time = delta_time;

  static Timers::Timer timer("Initialise");
  timer.start();

  // Delay checkpoint until MPI has been initialised
  Waypoint("Starting initialisation");

  CCTKi_ScheduleGHInit(cctkGH); // Enable storage and communication
  GroupsStorageCheck(cctkGH);
  do_warn_about_storage = true;

  CCTKi_InitGHExtensions(cctkGH);

  // Write grid structure to file
  for (int m = 0; m < maps; ++m) {
    OutputGridStructure(cctkGH, m, vhh.AT(m)->regions);
    OutputGridCoordinates(cctkGH, m, vhh.AT(m)->regions);
  } // for m

  CallSetup(cctkGH);

  if (enable_no_storage) {
    CCTK_INFO("Carpet completed its internal setup, and would now normally go "
              "on to allocate memory.  Since the parameter "
              "Carpet::enable_no_storage has been set, Carpet will exit "
              "instead.");
    CCTK_Exit(const_cast<cGH *>(cctkGH), 0);
  }

  if (fc->recovered) {
    // Read data from a checkpoint file

    CallRecoverVariables(cctkGH);
    CallPostRecoverVariables(cctkGH);
    // TODO: We should probably restrict here.
    // CallRestrict (cctkGH);
    // TODO: Should we also execute another bin after this?
    // CallPostRestrictRecover (cctkGH);
    print_internal_data();

  } else {
    // Calculate initial data

    CallInitial(cctkGH);
    CallRestrict(cctkGH);
    CallPostInitial(cctkGH);
    print_internal_data();

    if (init_3_timelevels) {
      Initialise3tl(cctkGH);
    }
  }

  // Analyse initial data
  CallAnalysis(cctkGH, fc->recovered);
  print_internal_data();

  timer.stop();
  if (output_timers_every > 0) {
    Timers::CactusTimerSet::writeData(cctkGH, timer_file);
  }

  if (output_initialise_timer_tree) {
    Timers::Timer::outputTree("Initialise");
  }

  Waypoint("Done with initialisation");

  return 0;
}

//////////////////////////////////////////////////////////////////////////////
//////////////////////////////////////////////////////////////////////////////
//////////////////////////////////////////////////////////////////////////////

void CallSetup(cGH *const cctkGH) {
  char const *const where = "CallSetup";
  static Timers::Timer timer(where);
  timer.start();

  BEGIN_MGLEVEL_LOOP(cctkGH) {
    do_early_global_mode = true;
    do_late_global_mode = true;
    do_global_mode = true;
    do_early_meta_mode = mglevel == mglevels - 1;
    do_late_meta_mode = mglevel == 0;
    do_meta_mode = do_early_meta_mode; // on first iteration, coarsest grid

    // Checking
    {
      int const rl = 0;
      ENTER_LEVEL_MODE(cctkGH, rl) {
        BeginTimingLevel(cctkGH);
        Poison(cctkGH, alltimes, CCTK_ARRAY);
        EndTimingLevel(cctkGH);
      }
      LEAVE_LEVEL_MODE;
    }

    // Register coordinates
    ScheduleTraverse(where, "CCTK_WRAGH", cctkGH);

    // Check parameters
    ScheduleTraverse(where, "CCTK_PARAMCHECK", cctkGH);
  }
  END_MGLEVEL_LOOP;

#ifdef REQUIREMENTS_HH
// TODO: Why is this disabled?
// Requirements::CheckRequirements (cctkGH);
#endif

  CCTKi_FinaliseParamWarn();

  timer.stop();
}

void CallRecoverVariables(cGH *const cctkGH) {
  char const *const where = "CallRecoverVariables";
  static Timers::Timer timer(where);
  timer.start();

  DECLARE_CCTK_PARAMETERS;

  for (int rl = 0; rl < reflevels; ++rl) {
    BEGIN_MGLEVEL_LOOP(cctkGH) {
      ENTER_LEVEL_MODE(cctkGH, rl) {
        BeginTimingLevel(cctkGH);

        do_early_global_mode = reflevel == 0;
        do_late_global_mode = reflevel == reflevels - 1;
        do_early_meta_mode = do_early_global_mode and mglevel == mglevels - 1;
        do_late_meta_mode = do_late_global_mode and mglevel == 0;
        do_global_mode =
            do_early_global_mode;          // on first iteration, coarsest grid
        do_meta_mode = do_early_meta_mode; // on first iteration, coarsest grid

        // cctkGH->cctk_time = global_time;

        Waypoint("Recover I at iteration %d time %g%s%s",
                 cctkGH->cctk_iteration, (double)cctkGH->cctk_time,
                 (do_global_mode ? " (global)" : ""),
                 (do_meta_mode ? " (meta)" : ""));

        // Checking
        Poison(cctkGH, alltimes, CCTK_GF);

        // Set up the grids
        ScheduleTraverse(where, "CCTK_BASEGRID", cctkGH);

        // Recover
        ScheduleTraverse(where, "CCTK_RECOVER_VARIABLES", cctkGH);

        // Timing statistics
        // (do this here, after cctk_time has been recovered)
        if (do_early_global_mode) {
          InitTimingStats(cctkGH);
        }

        if (regrid_during_recovery) {
          EndTimingLevel(cctkGH);
          CallRegridRecoverLevel(cctkGH);
          BeginTimingLevel(cctkGH);
        }

        EndTimingLevel(cctkGH);
      }
      LEAVE_LEVEL_MODE;
    }
    END_MGLEVEL_LOOP;
  } // for rl

#if 0
    // TODO: Maybe do not checkpoint (nor read back) ghost and buffer
    // zones
    CallRegridRecoverMeta (cctkGH);
#endif

  timer.stop();
}

void CallPostRecoverVariables(cGH *const cctkGH) {
  DECLARE_CCTK_PARAMETERS;

  char const *const where = "CallPostRecoverVariables";
  static Timers::Timer timer(where);
  timer.start();

  for (int rl = 0; rl < reflevels; ++rl) {
    BEGIN_MGLEVEL_LOOP(cctkGH) {
      ENTER_LEVEL_MODE(cctkGH, rl) {
        BeginTimingLevel(cctkGH);

        do_early_global_mode = reflevel == 0;
        do_late_global_mode = reflevel == reflevels - 1;
        do_early_meta_mode = do_early_global_mode and mglevel == mglevels - 1;
        do_late_meta_mode = do_late_global_mode and mglevel == 0;
        do_global_mode =
            do_early_global_mode;          // on first iteration, coarsest grid
        do_meta_mode = do_early_meta_mode; // on first iteration, coarsest grid

// This has been activated and deactivated a number of times by now,
// so I'll put down the reasoning behind deactivating it. Maybe the
// other party in this tug of war will be able to respond. Most of
// this is from b4429f4006e5:
//
// Traverse post_recover_variables in only for current timelevel (not
// for all timelevels). Variables on past timelevels cannot have
// their boundary conditions applied consistently, because time
// interpolation for these may requires even older timelevels that
// are not available.  The problem is not that prolongation in time
// on the older timelevels will fail. Prolongation will succeed and
// is not an extrapolation in time, so Carpet does not abort. However
// the time slices used as sources for the interpolation differ from
// the ones that were originally used during the evolution. This
// means that data on the old slices differs from what was present
// during the evolution ie. checkpointing and recovery change data
// on the grid.
//
// Current wisdom (ie. Erik says) is that all grid functions with
// multiple time levels should to be checkpointed because of this. If
// you do not checkpoint them, then you must be able to recompute
// them exactly using a pointwise, local, algebraic method that does
// not rely on present values of the re-computed data on the grid.
// This exlcudes the Hydro primitives (since con2prim requires an
// initial guess), but would in principle (I think) work with the ADM
// variables.
//
// In particular one cannot call MoL_PostStep on all levels since it
// involves prolongation.
//
// Roland Haas, June 21st 2012

#if 0
          BEGIN_TIMELEVEL_LOOP(cctkGH) {
            
            Waypoint ("Recovering II at iteration %d time %g timelevel %d%s%s",
                      cctkGH->cctk_iteration,
                      (double)cctkGH->cctk_time,
                      timelevel,
                      (do_global_mode ? " (global)" : ""),
                      (do_meta_mode ? " (meta)" : ""));
            
            // Post recover variables
            ScheduleTraverse (where, "CCTK_POST_RECOVER_VARIABLES", cctkGH);
            
          } END_TIMELEVEL_LOOP;
#else
        Waypoint("Recovering II at iteration %d time %g%s%s",
                 cctkGH->cctk_iteration, (double)cctkGH->cctk_time,
                 (do_global_mode ? " (global)" : ""),
                 (do_meta_mode ? " (meta)" : ""));

        // Post recover variables
        ScheduleTraverse(where, "CCTK_POST_RECOVER_VARIABLES", cctkGH);
#endif

        // Checking
        PoisonCheck(cctkGH, currenttime);

        CheckChecksums(cctkGH, allbutcurrenttime);

        EndTimingLevel(cctkGH);
      }
      LEAVE_LEVEL_MODE;
    }
    END_MGLEVEL_LOOP;
  } // for rl

  timer.stop();
}

void CallInitial(cGH *const cctkGH) {
  DECLARE_CCTK_PARAMETERS;

  char const *const where = "CallInitial";
  static Timers::Timer timer(where);
  timer.start();

  if (not regrid_during_initialisation) {
    // Regrid once in the beginning
    CallRegridInitialMeta(cctkGH);
  }

  // Poison early, since grid functions may be initialised in global
  // loop-local mode, ane we must not overwrite them accidentally
  for (int rl = 0; rl < reflevels; ++rl) {
    BEGIN_MGLEVEL_LOOP(cctkGH) {
      ENTER_LEVEL_MODE(cctkGH, rl) {
        BeginTimingLevel(cctkGH);

        // Checking
        Poison(cctkGH, alltimes, CCTK_GF);

        EndTimingLevel(cctkGH);
      }
      LEAVE_LEVEL_MODE;
    }
    END_MGLEVEL_LOOP;
  } // for rl

  for (int rl = 0; rl < reflevels; ++rl) {
    BEGIN_MGLEVEL_LOOP(cctkGH) {
      ENTER_LEVEL_MODE(cctkGH, rl) {
        BeginTimingLevel(cctkGH);

        do_early_global_mode = reflevel == 0;
        do_late_global_mode = reflevel == reflevels - 1;
        do_early_meta_mode = do_early_global_mode and mglevel == mglevels - 1;
        do_late_meta_mode = do_late_global_mode and mglevel == 0;
        do_global_mode =
            do_early_global_mode;          // on first iteration, coarsest grid
        do_meta_mode = do_early_meta_mode; // on first iteration, coarsest grid

        // cctkGH->cctk_time = global_time;

        Waypoint("Initialisation I at iteration %d time %g%s%s",
                 cctkGH->cctk_iteration, (double)cctkGH->cctk_time,
                 (do_global_mode ? " (global)" : ""),
                 (do_meta_mode ? " (meta)" : ""));

        // Timing statistics
        if (do_early_global_mode) {
          InitTimingStats(cctkGH);
        }

        // Set up the grids
        ScheduleTraverse(where, "CCTK_BASEGRID", cctkGH);

        if (init_each_timelevel) {

          BEGIN_TIMELEVEL_LOOP(cctkGH) {

            // Set up the initial data
            ScheduleTraverse(where, "CCTK_INITIAL", cctkGH);
            ScheduleTraverse(where, "CCTK_POSTINITIAL", cctkGH);
          }
          END_TIMELEVEL_LOOP;

        } else { // not init_each_timelevel

          assert(do_allow_past_timelevels);
          do_allow_past_timelevels =
              not CCTK_EQUALS(initial_data_setup_method, "init_single_level");

          // Set up the initial data
          ScheduleTraverse(where, "CCTK_INITIAL", cctkGH);
          ScheduleTraverse(where, "CCTK_POSTINITIAL", cctkGH);

          do_allow_past_timelevels = true;

        } // not init_each_timelevel

        if (init_fill_timelevels) {
          FillTimeLevels(cctkGH);
        }

        // Checking
        PoisonCheck(cctkGH, currenttime);

        if (regrid_during_initialisation and mglevel == 0) {
          // Regrid after initialising each level
          EndTimingLevel(cctkGH);
          CallRegridInitialLevel(cctkGH);
          BeginTimingLevel(cctkGH);
        }

        EndTimingLevel(cctkGH);
      }
      LEAVE_LEVEL_MODE;
    }
    END_MGLEVEL_LOOP;
  } // for rl

  timer.stop();
}

void CallRestrict(cGH *const cctkGH) {
  DECLARE_CCTK_PARAMETERS;

  char const *const where = "Initialise::CallRestrict";
  static Timers::Timer timer("CallRestrict");
  timer.start();

  for (int ml = mglevels - 1; ml >= 0; --ml) {

    for (int rl = reflevels - 2; rl >= 0; --rl) {
      ENTER_GLOBAL_MODE(cctkGH, ml) {
        ENTER_LEVEL_MODE(cctkGH, rl) {
          BeginTimingLevel(cctkGH);

          Waypoint("Initialisation/Restrict at iteration %d time %g",
                   cctkGH->cctk_iteration, (double)cctkGH->cctk_time);

          Restrict(cctkGH);

          if (use_higher_order_restriction) {
            do_early_global_mode = false;
            do_late_global_mode = false;
            do_early_meta_mode = false;
            do_late_meta_mode = false;
            do_global_mode = false;
            do_meta_mode = false;

            Waypoint("Initialisation/PostRestrict (intermediate) at iteration "
                     "%d time %g",
                     cctkGH->cctk_iteration, (double)cctkGH->cctk_time);

            ScheduleTraverse(where, "CCTK_POSTRESTRICTINITIAL", cctkGH);

            if (init_fill_timelevels) {
              FillTimeLevels(cctkGH);
            }
          }

          EndTimingLevel(cctkGH);
        }
        LEAVE_LEVEL_MODE;
      }
      LEAVE_GLOBAL_MODE;
    } // for rl

    if (reflevels > 1) {

      bool have_done_global_mode = false;
      bool have_done_early_global_mode = false;
      bool have_done_late_global_mode = false;
      bool have_done_anything = false;

      for (int rl = 0; rl < reflevels; ++rl) {
        ENTER_GLOBAL_MODE(cctkGH, ml) {
          ENTER_LEVEL_MODE(cctkGH, rl) {
            BeginTimingLevel(cctkGH);

            do_early_global_mode = not have_done_early_global_mode;
            do_late_global_mode = reflevel == reflevels - 1;
            do_early_meta_mode =
                do_early_global_mode and mglevel == mglevels - 1;
            do_late_meta_mode = do_late_global_mode and mglevel == 0;
            do_global_mode = do_late_global_mode;
            do_meta_mode = do_global_mode and do_late_meta_mode;
            assert(not(have_done_global_mode and do_global_mode));
            assert(not(have_done_early_global_mode and do_early_global_mode));
            assert(not(have_done_late_global_mode and do_late_global_mode));
            have_done_global_mode |= do_global_mode;
            have_done_early_global_mode |= do_early_global_mode;
            have_done_late_global_mode |= do_late_global_mode;
            have_done_anything = true;

            Waypoint("Initialisation/PostRestrict at iteration %d time %g",
                     cctkGH->cctk_iteration, (double)cctkGH->cctk_time);

            ScheduleTraverse(where, "CCTK_POSTRESTRICTINITIAL", cctkGH);

            if (init_fill_timelevels) {
              FillTimeLevels(cctkGH);
            }

            EndTimingLevel(cctkGH);
          }
          LEAVE_LEVEL_MODE;
        }
        LEAVE_GLOBAL_MODE;
      } // for rl

      if (have_done_anything)
        assert(have_done_global_mode);
      if (have_done_anything)
        assert(have_done_early_global_mode);
      if (have_done_anything)
        assert(have_done_late_global_mode);
    }

  } // for ml

  timer.stop();
}

void CallPostInitial(cGH *const cctkGH) {
  char const *const where = "CallPostInitial";
  static Timers::Timer timer(where);
  timer.start();

  for (int rl = 0; rl < reflevels; ++rl) {
    BEGIN_MGLEVEL_LOOP(cctkGH) {
      ENTER_LEVEL_MODE(cctkGH, rl) {
        BeginTimingLevel(cctkGH);

        do_early_global_mode = reflevel == 0;
        do_late_global_mode = reflevel == reflevels - 1;
        do_early_meta_mode = do_early_global_mode and mglevel == mglevels - 1;
        do_late_meta_mode = do_late_global_mode and mglevel == 0;
        do_global_mode = do_late_global_mode; // on last iteration, finest grid
        do_meta_mode = do_late_meta_mode;     // on last iteration, finest grid

        Waypoint("Initialisation II at iteration %d time %g%s%s",
                 cctkGH->cctk_iteration, (double)cctkGH->cctk_time,
                 (do_global_mode ? " (global)" : ""),
                 (do_meta_mode ? " (meta)" : ""));

#if 0
          if (reflevel < reflevels-1) {
            ScheduleTraverse (where, "CCTK_POSTRESTRICTINITIAL", cctkGH);
          }
#endif

        ScheduleTraverse(where, "CCTK_POSTPOSTINITIAL", cctkGH);
        ScheduleTraverse(where, "CCTK_POSTSTEP", cctkGH);

        PoisonCheck(cctkGH, alltimes);
        CheckChecksums(cctkGH, allbutcurrenttime);

        EndTimingLevel(cctkGH);
      }
      LEAVE_LEVEL_MODE;
    }
    END_MGLEVEL_LOOP;
  } // for rl

  timer.stop();
}

void CallAnalysis(cGH *const cctkGH, bool const did_recover) {
  char const *const where = "CallAnalysis";
  static Timers::Timer timer(where);
  timer.start();

  for (int rl = 0; rl < reflevels; ++rl) {
    BEGIN_MGLEVEL_LOOP(cctkGH) {
      ENTER_LEVEL_MODE(cctkGH, rl) {
        BeginTimingLevel(cctkGH);

        do_early_global_mode = reflevel == 0;
        do_late_global_mode = reflevel == reflevels - 1;
        do_early_meta_mode = do_early_global_mode and mglevel == mglevels - 1;
        do_late_meta_mode = do_late_global_mode and mglevel == 0;
        do_global_mode = do_late_global_mode; // on last iteration, finest grid
        do_meta_mode = do_late_meta_mode;     // on last iteration, finest grid

        Waypoint("Initialisation III at iteration %d time %g%s%s",
                 cctkGH->cctk_iteration, (double)cctkGH->cctk_time,
                 (do_global_mode ? " (global)" : ""),
                 (do_meta_mode ? " (meta)" : ""));

        int const do_every =
            ipow(mgfact, mglevel) * (maxtimereflevelfact / timereffacts.AT(rl));
        if (cctkGH->cctk_iteration % do_every == 0) {

          if (not did_recover) {
            // Checkpoint, but only if we did not recover
            ScheduleTraverse(where, "CCTK_CPINITIAL", cctkGH);
          }

          // Analysis
          in_analysis_bin = true;
          ScheduleTraverse(where, "CCTK_ANALYSIS", cctkGH);
          in_analysis_bin = false;

          if (do_late_global_mode) {
            // Timing statistics
            UpdateTimingStats(cctkGH);
          }

          // Output
          OutputGH(where, cctkGH);

          // Checking
          PoisonCheck(cctkGH, alltimes);
          CheckChecksums(cctkGH, allbutcurrenttime);
        } // if do_every

        EndTimingLevel(cctkGH);
      }
      LEAVE_LEVEL_MODE;
    }
    END_MGLEVEL_LOOP;
  } // for rl

  timer.stop();
}

#if 0
  void
  CallRegrid (cGH * const cctkGH,
              bool const callpreregrid,
              bool const callregrid,
              bool const callpostregrid,
              bool const regridinitial)
  {
    DECLARE_CCTK_PARAMETERS;
    
    assert (is_level_mode());
    
    bool const old_do_global_mode = do_global_mode;
    bool const old_do_meta_mode = do_meta_mode;
    do_global_mode = true;
    do_meta_mode = true;
    
    // Preregrid
    if (callpreregrid) {
      if (regridinitial) {
        Waypoint ("Preregridinitial at iteration %d time %g%s%s",
                  cctkGH->cctk_iteration, (double)cctkGH->cctk_time,
                  (do_global_mode ? " (global)" : ""),
                  (do_meta_mode ? " (meta)" : ""));
        ScheduleTraverse (where, "CCTK_PREREGRIDINITIAL", cctkGH);
      } else {
        Waypoint ("Preregrid at iteration %d time %g%s%s",
                  cctkGH->cctk_iteration, (double)cctkGH->cctk_time,
                  (do_global_mode ? " (global)" : ""),
                  (do_meta_mode ? " (meta)" : ""));
        ScheduleTraverse (where, "CCTK_PREREGRID", cctkGH);
      }
    }
    
    // Regrid
    bool did_regrid = false;
    if (callregrid) {
      Checkpoint ("Regrid");
      did_regrid = Regrid (cctkGH, true);
    }
    
    if (did_regrid or not callregrid) {
      BEGIN_META_MODE (cctkGH) {
        for (int rl=0; rl<reflevels; ++rl) {
          
          bool did_recompose = false;
          if (did_regrid) {
            did_recompose = Recompose (cctkGH, rl, prolongate_initial_data);
          }
          
          // Call postregridinitial only if initial data have already
          // been set up
          if (callpostregrid and (did_recompose or not callregrid)) {
            BEGIN_MGLEVEL_LOOP (cctkGH) {
              ENTER_LEVEL_MODE (cctkGH, rl) {
                BeginTimingLevel (cctkGH);
                
                do_global_mode = reflevel == reflevels - 1;
                do_meta_mode = do_global_mode and mglevel==mglevels-1;
                
                if (regridinitial) {
                  Waypoint ("Postregridinitial at iteration %d time %g%s%s",
                            cctkGH->cctk_iteration, (double)cctkGH->cctk_time,
                            (do_global_mode ? " (global)" : ""),
                            (do_meta_mode ? " (meta)" : ""));
                } else {
                  Waypoint ("Postregrid at iteration %d time %g%s%s",
                            cctkGH->cctk_iteration, (double)cctkGH->cctk_time,
                            (do_global_mode ? " (global)" : ""),
                            (do_meta_mode ? " (meta)" : ""));
                }
                
                int const num_tl =
                  regridinitial
                  ? (init_each_timelevel ? maxtimelevels : 1)
                  : maxtimelevels;
                
                bool const old_do_allow_past_timelevels =
                  do_allow_past_timelevels;
                do_allow_past_timelevels = false;
                
                // Rewind times
                for (int m=0; m<maps; ++m) {
                  CCTK_REAL const old_delta =
                    vtt.AT(m)->get_delta (reflevel, mglevel);
                  vtt.AT(m)->set_delta (reflevel, mglevel, - old_delta);
                }
                FlipTimeLevels (cctkGH);
                for (int tl=0; tl<num_tl; ++tl) {
                  for (int m=0; m<maps; ++m) {
                    vtt.AT(m)->advance_time (reflevel, mglevel);
                  }
                  CycleTimeLevels (cctkGH);
                }
                for (int m=0; m<maps; ++m) {
                  CCTK_REAL const old_delta =
                    vtt.AT(m)->get_delta (reflevel, mglevel);
                  vtt.AT(m)->set_delta (reflevel, mglevel, - old_delta);
                }
                FlipTimeLevels (cctkGH);
                CCTK_REAL const old_cctk_time = cctkGH->cctk_time;
                cctkGH->cctk_time -=
                  num_tl * (cctkGH->cctk_delta_time / cctkGH->cctk_timefac);
                
                for (int tl=0; tl<num_tl; ++tl) {
                  
                  // Advance times
                  for (int m=0; m<maps; ++m) {
                    vtt.AT(m)->advance_time (reflevel, mglevel);
                  }
                  CycleTimeLevels (cctkGH);
                  cctkGH->cctk_time +=
                    cctkGH->cctk_delta_time / cctkGH->cctk_timefac;
                  
                  // Postregrid
                  if (regridinitial) {
                    ScheduleTraverse (where, "CCTK_POSTREGRIDINITIAL", cctkGH);
                  } else {
                    ScheduleTraverse (where, "CCTK_POSTREGRID", cctkGH);
                  }
                  
                } // for tl
                cctkGH->cctk_time = old_cctk_time;
                
                do_allow_past_timelevels = old_do_allow_past_timelevels;
                
                EndTimingLevel (cctkGH);
              } LEAVE_LEVEL_MODE;
            } END_MGLEVEL_LOOP;
          } // if did_recompose
          
        } // for rl
      } END_META_MODE;
    } // if did_regrid
    
    if (callregrid) {
      RegridFree (cctkGH);
    }
    
    do_global_mode = old_do_global_mode;
    do_meta_mode = old_do_meta_mode;
  }
#endif

#if 0
  void
  CallRegridRecoverMeta (cGH * const cctkGH)
  {
    DECLARE_CCTK_PARAMETERS;
    
    char const * const where = "CallRegridRecoverMeta";
    static Timers::Timer timer (where);
    timer.start();
    
    assert (is_meta_mode());
    
    bool const old_do_global_mode = do_global_mode;
    bool const old_do_meta_mode = do_meta_mode;
    do_global_mode = true;
    do_meta_mode = true;
    
    for (int rl=0; rl<reflevels; ++rl) {
      
      BEGIN_MGLEVEL_LOOP (cctkGH) {
        ENTER_LEVEL_MODE (cctkGH, rl) {
          BeginTimingLevel (cctkGH);
          
          do_global_mode = reflevel == reflevels - 1;
          do_meta_mode = do_global_mode and mglevel==mglevels-1;
          
          Waypoint ("Postregrid at iteration %d time %g%s%s",
                    cctkGH->cctk_iteration, (double)cctkGH->cctk_time,
                    (do_global_mode ? " (global)" : ""),
                    (do_meta_mode ? " (meta)" : ""));
          
          int const num_tl = maxtimelevels;
          
          bool const old_do_allow_past_timelevels = do_allow_past_timelevels;
          do_allow_past_timelevels = false;
          
          // Rewind times
          for (int m=0; m<maps; ++m) {
            CCTK_REAL const old_delta =
              vtt.AT(m)->get_delta (reflevel, mglevel);
            vtt.AT(m)->set_delta (reflevel, mglevel, - old_delta);
          }
          FlipTimeLevels (cctkGH);
          for (int tl=0; tl<num_tl; ++tl) {
            for (int m=0; m<maps; ++m) {
              vtt.AT(m)->advance_time (reflevel, mglevel);
            }
            CycleTimeLevels (cctkGH);
          }
          for (int m=0; m<maps; ++m) {
            CCTK_REAL const old_delta =
              vtt.AT(m)->get_delta (reflevel, mglevel);
            vtt.AT(m)->set_delta (reflevel, mglevel, - old_delta);
          }
          FlipTimeLevels (cctkGH);
          CCTK_REAL const old_cctk_time = cctkGH->cctk_time;
          cctkGH->cctk_time -=
            num_tl * (cctkGH->cctk_delta_time / cctkGH->cctk_timefac);
          
          for (int tl=0; tl<num_tl; ++tl) {
            
            // Advance times
            for (int m=0; m<maps; ++m) {
              vtt.AT(m)->advance_time (reflevel, mglevel);
            }
            CycleTimeLevels (cctkGH);
            cctkGH->cctk_time +=
              cctkGH->cctk_delta_time / cctkGH->cctk_timefac;
            
            // Postregrid
            ScheduleTraverse (where, "CCTK_POSTREGRID", cctkGH);
              
          } // for tl
          cctkGH->cctk_time = old_cctk_time;
          
          do_allow_past_timelevels = old_do_allow_past_timelevels;
          
          EndTimingLevel (cctkGH);
        } LEAVE_LEVEL_MODE;
      } END_MGLEVEL_LOOP;
    } // for rl
    
    do_global_mode = old_do_global_mode;
    do_meta_mode = old_do_meta_mode;
    
    timer.stop();
  }
#endif

void CallRegridRecoverLevel(cGH *const cctkGH) {
  DECLARE_CCTK_PARAMETERS;

  char const *const where = "CallRegridRecoverLevel";
  static Timers::Timer timer(where);
  timer.start();

  CCTK_WARN(CCTK_WARN_ALERT,
            "Regridding in level mode after recovering is discouraged");

  assert(is_level_mode());

  bool const old_do_global_mode = do_global_mode;
  bool const old_do_early_global_mode = do_early_global_mode;
  bool const old_do_late_global_mode = do_late_global_mode;
  bool const old_do_meta_mode = do_meta_mode;
  bool const old_do_early_meta_mode = do_early_meta_mode;
  bool const old_do_late_meta_mode = do_late_meta_mode;
  do_global_mode = true;
  do_early_global_mode = true;
  do_late_global_mode = true;
  do_meta_mode = true;
  do_early_meta_mode = true;
  do_late_meta_mode = true;

  // Preregrid
  Waypoint("Preregrid at iteration %d time %g%s%s", cctkGH->cctk_iteration,
           (double)cctkGH->cctk_time, (do_global_mode ? " (global)" : ""),
           (do_meta_mode ? " (meta)" : ""));
  ScheduleTraverse(where, "CCTK_PREREGRID", cctkGH);

  // Regrid
  Checkpoint("Regrid");
  int const oldreflevels = reflevels;
  bool const did_regrid = Regrid(cctkGH, false, true);
  bool const did_remove_level = reflevels < oldreflevels;
  assert(not did_remove_level or did_regrid);

  if (did_regrid) {
#ifdef REQUIREMENTS_HH
    Requirements::Regrid(reflevels);
#endif
    bool did_any_recompose = false;
    BEGIN_META_MODE(cctkGH) {

      bool have_done_global_mode = false;
      bool have_done_early_global_mode = false;
      bool have_done_late_global_mode = false;
      bool have_done_anything = false;

      for (int rl = 0; rl < reflevels; ++rl) {

        bool const did_recompose =
          Recompose(cctkGH, rl, true);
        did_any_recompose = did_any_recompose or did_recompose;
#ifdef REQUIREMENTS_HH
        Requirements::Recompose(cctkGH->cctk_iteration, rl,
                                not did_recompose
                                    ? Requirements::valid::everywhere
                                    : Requirements::valid::interior);
#endif

        // Carpet assumes that a regridding operation always changes
        // "level N and all finer levels" so we should call POSTREGRID
        // on all finer levels
        if (did_any_recompose or (did_remove_level and rl == reflevels - 1)) {
          BEGIN_MGLEVEL_LOOP(cctkGH) {
            ENTER_LEVEL_MODE(cctkGH, rl) {
              BeginTimingLevel(cctkGH);

              do_early_global_mode = not have_done_early_global_mode;
              do_late_global_mode = reflevel == reflevels - 1;
              do_early_meta_mode =
                  do_early_global_mode and mglevel == mglevels - 1;
              do_late_meta_mode = do_late_global_mode and mglevel == 0;
              do_global_mode = do_late_global_mode;
              do_meta_mode = do_late_meta_mode;
              assert(not(have_done_global_mode and do_global_mode));
              assert(not(have_done_early_global_mode and do_early_global_mode));
              assert(not(have_done_late_global_mode and do_late_global_mode));
              have_done_global_mode |= do_global_mode;
              have_done_early_global_mode |= do_early_global_mode;
              have_done_late_global_mode |= do_late_global_mode;
              have_done_anything = true;

              BEGIN_TIMELEVEL_LOOP(cctkGH) {

                Waypoint("Postregrid at iteration %d time %g timelevel %d%s%s",
                         cctkGH->cctk_iteration, (double)cctkGH->cctk_time,
                         timelevel, (do_global_mode ? " (global)" : ""),
                         (do_meta_mode ? " (meta)" : ""));

                // Postregrid
                ScheduleTraverse(where, "CCTK_POSTREGRID", cctkGH);
              }
              END_TIMELEVEL_LOOP;

              if (output_after_regridding) {
                // Output
                OutputGH(where, cctkGH);
              }

              EndTimingLevel(cctkGH);
            }
            LEAVE_LEVEL_MODE;
          }
          END_MGLEVEL_LOOP;
        } // if did_recompose

      } // for rl

      if (have_done_anything)
        assert(have_done_global_mode);
      if (have_done_anything)
        assert(have_done_early_global_mode);
      if (have_done_anything)
        assert(have_done_late_global_mode);
    }
    END_META_MODE;
#ifdef REQUIREMENTS_HH
    Requirements::RegridFree();
#endif
  } // if did_regrid

  RegridFree(cctkGH, prolongate_initial_data);

  do_global_mode = old_do_global_mode;
  do_early_global_mode = old_do_early_global_mode;
  do_late_global_mode = old_do_late_global_mode;
  do_meta_mode = old_do_meta_mode;
  do_early_meta_mode = old_do_early_meta_mode;
  do_late_meta_mode = old_do_late_meta_mode;

  timer.stop();
}

void CallRegridInitialMeta(cGH *const cctkGH) {
  DECLARE_CCTK_PARAMETERS;

  char const *const where = "CallRegridInitialMeta";
  static Timers::Timer timer(where);
  timer.start();

  assert(is_meta_mode());

  bool const old_do_global_mode = do_global_mode;
  bool const old_do_early_global_mode = do_early_global_mode;
  bool const old_do_late_global_mode = do_late_global_mode;
  bool const old_do_meta_mode = do_meta_mode;
  bool const old_do_early_meta_mode = do_early_meta_mode;
  bool const old_do_late_meta_mode = do_late_meta_mode;
  do_global_mode = true;
  do_early_global_mode = true;
  do_late_global_mode = true;
  do_meta_mode = true;
  do_early_meta_mode = true;
  do_late_meta_mode = true;

  ENTER_GLOBAL_MODE(cctkGH, 0) {
    ENTER_LEVEL_MODE(cctkGH, 0) {
      BeginTimingLevel(cctkGH);

      // Preregrid
      Waypoint("Preregridinitial at iteration %d time %g%s%s",
               cctkGH->cctk_iteration, (double)cctkGH->cctk_time,
               (do_global_mode ? " (global)" : ""),
               (do_meta_mode ? " (meta)" : ""));
      ScheduleTraverse(where, "CCTK_PREREGRIDINITIAL", cctkGH);

      // Regrid
      Checkpoint("Regrid");
      bool const did_regrid = Regrid(cctkGH, true, prolongate_initial_data);

      if (did_regrid) {
#ifdef REQUIREMENTS_HH
        Requirements::Regrid(reflevels);
#endif
        for (int rl = 0; rl < reflevels; ++rl) {
          Recompose(cctkGH, rl, prolongate_initial_data);
#ifdef REQUIREMENTS_HH
          Requirements::Recompose(cctkGH->cctk_iteration, rl,
                                  prolongate_initial_data
                                      ? Requirements::valid::interior
                                      : Requirements::valid::nowhere);
#endif
        } // for rl
#ifdef REQUIREMENTS_HH
        Requirements::RegridFree();
#endif
      } // if did_regrid

      RegridFree(cctkGH, prolongate_initial_data);

      EndTimingLevel(cctkGH);
    }
    LEAVE_LEVEL_MODE;
  }
  LEAVE_GLOBAL_MODE;

  do_global_mode = old_do_global_mode;
  do_early_global_mode = old_do_early_global_mode;
  do_late_global_mode = old_do_late_global_mode;
  do_meta_mode = old_do_meta_mode;
  do_early_meta_mode = old_do_early_meta_mode;
  do_late_meta_mode = old_do_late_meta_mode;

  timer.stop();
}

void CallRegridInitialLevel(cGH *const cctkGH) {
  DECLARE_CCTK_PARAMETERS;

  char const *const where = "CallRegridInitialLevel";
  static Timers::Timer timer(where);
  timer.start();

  CCTK_WARN(CCTK_WARN_ALERT,
            "Regridding in level mode while initialising is discouraged");

  assert(is_level_mode());

  bool const old_do_global_mode = do_global_mode;
  bool const old_do_early_global_mode = do_early_global_mode;
  bool const old_do_late_global_mode = do_late_global_mode;
  bool const old_do_meta_mode = do_meta_mode;
  bool const old_do_early_meta_mode = do_early_meta_mode;
  bool const old_do_late_meta_mode = do_late_meta_mode;
  do_global_mode = true;
  do_early_global_mode = true;
  do_late_global_mode = true;
  do_meta_mode = true;
  do_early_meta_mode = true;
  do_late_meta_mode = true;

  // Preregrid
  Waypoint("Preregridinitial at iteration %d time %g%s%s",
           cctkGH->cctk_iteration, (double)cctkGH->cctk_time,
           (do_global_mode ? " (global)" : ""),
           (do_meta_mode ? " (meta)" : ""));
  ScheduleTraverse(where, "CCTK_PREREGRIDINITIAL", cctkGH);

  // Regrid
  Checkpoint("Regrid");
  int const oldreflevels = reflevels;
  bool const did_regrid = Regrid(cctkGH, true, prolongate_initial_data);
  bool const did_remove_level = reflevels < oldreflevels;
  assert(not did_remove_level or did_regrid);

  if (did_regrid) {
#ifdef REQUIREMENTS_HH
    Requirements::Regrid(reflevels);
#endif
    bool did_any_recompose = false;
    BEGIN_META_MODE(cctkGH) {

      bool have_done_global_mode = false;
      bool have_done_early_global_mode = false;
      bool have_done_late_global_mode = false;
      bool have_done_anything = false;

      for (int rl = 0; rl < reflevels; ++rl) {

        bool did_recompose = Recompose(cctkGH, rl, prolongate_initial_data);
        did_any_recompose = did_any_recompose or did_recompose;
#ifdef REQUIREMENTS_HH
        Requirements::Recompose(cctkGH->cctk_iteration, rl,
                                not did_recompose
                                    ? Requirements::valid::everywhere
                                    : prolongate_initial_data
                                          ? Requirements::valid::interior
                                          : Requirements::valid::nowhere);
#endif

        // Carpet assumes that a regridding operation always changes
        // "level N and all finer levels" so we should call
        // POSTREGRIDINITIAL on all finer levels
        if (did_any_recompose or (did_remove_level and rl == reflevels - 1)) {
          BEGIN_MGLEVEL_LOOP(cctkGH) {
            ENTER_LEVEL_MODE(cctkGH, rl) {
              BeginTimingLevel(cctkGH);

              do_early_global_mode = not have_done_early_global_mode;
              do_late_global_mode = reflevel == reflevels - 1;
              do_early_meta_mode =
                  do_early_global_mode and mglevel == mglevels - 1;
              do_late_meta_mode = do_late_global_mode and mglevel == 0;
              do_global_mode = do_late_global_mode;
              do_meta_mode = do_late_meta_mode;
              assert(not(have_done_global_mode and do_global_mode));
              assert(not(have_done_early_global_mode and do_early_global_mode));
              assert(not(have_done_late_global_mode and do_late_global_mode));
              have_done_global_mode |= do_global_mode;
              have_done_early_global_mode |= do_early_global_mode;
              have_done_late_global_mode |= do_late_global_mode;
              have_done_anything = true;

              Waypoint("Postregridinitial at iteration %d time %g%s%s",
                       cctkGH->cctk_iteration, (double)cctkGH->cctk_time,
                       (do_global_mode ? " (global)" : ""),
                       (do_meta_mode ? " (meta)" : ""));

              if (init_each_timelevel) {

                BEGIN_TIMELEVEL_LOOP(cctkGH) {

                  // Postregridinitial
                  ScheduleTraverse(where, "CCTK_POSTREGRIDINITIAL", cctkGH);
                }
                END_TIMELEVEL_LOOP;

              } else { // not init_each_timelevel

                assert(do_allow_past_timelevels);
                do_allow_past_timelevels = false;

                // Postregridinitial
                ScheduleTraverse(where, "CCTK_POSTREGRIDINITIAL", cctkGH);

                do_allow_past_timelevels = true;

                if (init_fill_timelevels) {
                  FillTimeLevels(cctkGH);
                }

              } // not init_each_timelevel

              EndTimingLevel(cctkGH);
            }
            LEAVE_LEVEL_MODE;
          }
          END_MGLEVEL_LOOP;
        } // if did_recompose

      } // for rl

      if (have_done_anything)
        assert(have_done_global_mode);
      if (have_done_anything)
        assert(have_done_early_global_mode);
      if (have_done_anything)
        assert(have_done_late_global_mode);
    }
    END_META_MODE;
#ifdef REQUIREMENTS_HH
    Requirements::RegridFree();
#endif
  } // if did_regrid

  RegridFree(cctkGH, prolongate_initial_data);

  do_global_mode = old_do_global_mode;
  do_early_global_mode = old_do_early_global_mode;
  do_late_global_mode = old_do_late_global_mode;
  do_meta_mode = old_do_meta_mode;
  do_early_meta_mode = old_do_early_meta_mode;
  do_late_meta_mode = old_do_late_meta_mode;

  timer.stop();
}

//////////////////////////////////////////////////////////////////////////////
//////////////////////////////////////////////////////////////////////////////
//////////////////////////////////////////////////////////////////////////////

static void initialise_3tl_flip_timelevels(cGH *const cctkGH);
static void initialise_3tl_evolve(cGH *const cctkGH);
static void initialise_3tl_recycle(cGH *const cctkGH);

void Initialise3tl(cGH *const cctkGH) {
  DECLARE_CCTK_PARAMETERS;

  Waypoint("Initialising three timelevels:");

  char const *const where = "Initialise3TL";
  static Timers::Timer timer(where);
  timer.start();

#if 0
    initialise_3tl_flip_timelevels (cctkGH);
    initialise_3tl_evolve (cctkGH);
    // TODO: May want to restrict here if possible (i.e. if the time
    // refinement factor is one)
    initialise_3tl_recycle (cctkGH);
    initialise_3tl_flip_timelevels (cctkGH);
#endif

  initialise_3tl_flip_timelevels(cctkGH);
  initialise_3tl_evolve(cctkGH);
  initialise_3tl_evolve(cctkGH);
  // TODO: May want to restrict where possible (i.e. if the time
  // refinement factor is one)
  initialise_3tl_recycle(cctkGH);
  initialise_3tl_recycle(cctkGH);
  initialise_3tl_flip_timelevels(cctkGH);

  timer.stop();

  Waypoint("Finished initialising three timelevels");
}

void initialise_3tl_flip_timelevels(cGH *const cctkGH) {
  Waypoint("Initialise3TL::Flip");

  delta_time *= -1;

  BEGIN_MGLEVEL_LOOP(cctkGH) {
    BEGIN_REFLEVEL_LOOP(cctkGH) { FlipTimeLevels(cctkGH); }
    END_REFLEVEL_LOOP;
  }
  END_MGLEVEL_LOOP;
}

void initialise_3tl_evolve(cGH *const cctkGH) {
  char const *const where = "Evolve";
  static Timers::Timer timer(where);
  timer.start();

  BEGIN_MGLEVEL_LOOP(cctkGH) {
    BEGIN_REFLEVEL_LOOP(cctkGH) {
      BeginTimingLevel(cctkGH);

      do_early_global_mode = reflevel == 0;
      do_late_global_mode = reflevel == reflevels - 1;
      do_early_meta_mode = do_early_global_mode and mglevel == mglevels - 1;
      do_late_meta_mode = do_late_global_mode and mglevel == 0;
      do_global_mode = do_early_global_mode;
      do_meta_mode = do_early_meta_mode;

      Waypoint("Initialisation 3TL evolution",
               (do_global_mode ? " (global)" : ""),
               (do_meta_mode ? " (meta)" : ""));

      CycleTimeLevels(cctkGH);

      CalculateChecksums(cctkGH, allbutcurrenttime);
      Poison(cctkGH, currenttimebutnotifonly);

      // Evolve
      ScheduleTraverse(where, "CCTK_PRESTEP", cctkGH);
      ScheduleTraverse(where, "CCTK_EVOL", cctkGH);

      PoisonCheck(cctkGH, currenttime);

      EndTimingLevel(cctkGH);
    }
    END_REFLEVEL_LOOP;
  }
  END_MGLEVEL_LOOP;

  timer.stop();
}

void initialise_3tl_recycle(cGH *const cctkGH) {
  char const *const where = "Recycle";
  static Timers::Timer timer(where);
  timer.start();

  BEGIN_MGLEVEL_LOOP(cctkGH) {
    BEGIN_REFLEVEL_LOOP(cctkGH) {
      BeginTimingLevel(cctkGH);

      Waypoint("Initialisation 3TL recycling");

      UncycleTimeLevels(cctkGH);

      EndTimingLevel(cctkGH);
    }
    END_REFLEVEL_LOOP;
  }
  END_MGLEVEL_LOOP;

  timer.stop();
}

//////////////////////////////////////////////////////////////////////////////
//////////////////////////////////////////////////////////////////////////////
//////////////////////////////////////////////////////////////////////////////

void print_internal_data() {
  DECLARE_CCTK_PARAMETERS;

  if (output_internal_data) {
    CCTK_INFO("Internal data dump:");
    streamsize const oldprecision = cout.precision();
    cout.precision(17);
    cout << "   global_time: " << global_time << endl
         // << "   leveltimes: " << leveltimes << endl
         << "   delta_time: " << delta_time << endl;
    cout.precision(oldprecision);
  }
}

void ScheduleTraverse(char const *const where, char const *const name,
                      cGH *const cctkGH) {
  Timers::Timer timer(name);

  timer.start();
  ostringstream infobuf;
  infobuf << "Scheduling " << name;
  string const info = infobuf.str();
  Checkpoint(info.c_str());
  CCTK_ScheduleTraverse(name, cctkGH, CallFunction);
  timer.stop();
}

void OutputGH(char const *const where, cGH *const cctkGH) {
  static Timers::Timer timer("OutputGH");
  timer.start();
  CCTK_OutputGH(cctkGH);
  timer.stop();
}

} // namespace Carpet

```

carpet_public.hh:
```
#ifndef CARPET_PUBLIC_HH
#define CARPET_PUBLIC_HH

// Stuff with C linkage
#include "carpet_public.h"

// Other declarations
#include "functions.hh"
#include "modes.hh"
#include "variables.hh"

#endif // !defined(CARPET_PUBLIC_HH)

```

CarpetParamCheck.cc:
```
#include <cassert>
#include <cstdlib>

#include <cctk.h>
#include <cctk_Arguments.h>
#include <cctk_Parameters.h>

#include <carpet.hh>

namespace Carpet {

using namespace std;

/** Ensure that the parameters have legal values.
 *
 * Note that this checking happens only after most of Carpet has
 * already been set up.
 */
void CarpetParamCheck(CCTK_ARGUMENTS) {
  DECLARE_CCTK_ARGUMENTS;
  DECLARE_CCTK_PARAMETERS;

  if ((CCTK_ParameterQueryTimesSet("periodic", "Carpet") and periodic) or
      (CCTK_ParameterQueryTimesSet("periodic_x", "Carpet") and periodic_x) or
      (CCTK_ParameterQueryTimesSet("periodic_y", "Carpet") and periodic_y) or
      (CCTK_ParameterQueryTimesSet("periodic_z", "Carpet") and periodic_z)) {
    CCTK_PARAMWARN("Some of the parameters \"Carpet::periodic*\" have been set "
                   "to \"yes\".  These parameters are there for compatibility "
                   "reasons only and must not be used.");
  }

  if (adaptive_stepsize and max_refinement_levels > 1) {
    CCTK_PARAMWARN("Adaptive time step sizes do not work with mesh refinement "
                   "yet.  Please use only a single level, and set "
                   "max_refinement_levels=1.");
  }

  if (max_refinement_levels > 1) {

    // InitBase

    enum setup_method_t {
      init_error,
      init_some_levels,
      init_single_level,
      init_two_levels,
      init_all_levels,
    };
    setup_method_t setup_method = init_error;

    if (CCTK_EQUALS(initial_data_setup_method, "init_some_levels")) {
      setup_method = init_some_levels;
    } else if (CCTK_EQUALS(initial_data_setup_method, "init_single_level")) {
      setup_method = init_single_level;
    } else if (CCTK_EQUALS(initial_data_setup_method, "init_two_levels")) {
      setup_method = init_two_levels;
    } else if (CCTK_EQUALS(initial_data_setup_method, "init_all_levels")) {
      setup_method = init_all_levels;
    } else {
      CCTK_PARAMWARN(
          "Unknown value of parameter InitBase::initial_data_setup_method");
    }

    // Carpet

    enum init_method_t {
      error_timelevel,
      each_timelevel,   // Initialise each time level
      fill_timelevels,  // Copy the current to all past timelevels
      three_timelevels, // Carpet's 3 timelevel scheme
      all_timelevels,   // Initial data thorn initialises all timelevels
    };
    init_method_t init_method = error_timelevel;

    if (init_each_timelevel) {
      if (init_fill_timelevels) {
        CCTK_PARAMWARN("Cannot set parameters init_each_timelevel and "
                       "init_fill_timelevels at the same time");
      }
      if (init_3_timelevels) {
        CCTK_PARAMWARN("Cannot set parameters init_each_timelevel and "
                       "init_3_timelevels at the same time");
      }
      init_method = each_timelevel;
    } else if (init_fill_timelevels) {
      if (init_3_timelevels) {
        CCTK_PARAMWARN("Cannot set parameters init_fill_timelevels and "
                       "init_3_timelevels at the same time");
      }
      init_method = fill_timelevels;
    } else if (init_3_timelevels) {
      init_method = three_timelevels;
    } else {
      init_method = all_timelevels;
    }

    switch (init_method) {
    case each_timelevel:
      if (setup_method != init_single_level) {
        CCTK_PARAMWARN(
            "When you set Carpet::init_each_timelevel=yes, then you must also "
            "use InitBase::initial_data_setup_method=\"init_single_level\"");
      }
      break;
    case fill_timelevels:
      // Everything is allowed
      break;
    case three_timelevels:
      // Everything is allowed
      break;
    case all_timelevels:
      if (setup_method != init_all_levels) {
        CCTK_PARAMWARN("When you set neither Carpet::init_each_timelevel=yes "
                       "nor Carpet::init_fill_timelevels=yes nor "
                       "Carpet::init_3_timelevels=yes, then you must also use "
                       "InitBase::initial_data_setup_method=\"init_all_"
                       "levels\"");
      }
      break;
    default:
      assert(0);
    }

  } // if max_refinement_levels > 0
}

} // namespace Carpet

```

helpers.cc:
```
#include <cctk.h>
#include <cctk_FortranString.h>
#include <cctk_Parameters.h>
#include <util_Table.h>

#include <algorithm>
#include <cassert>
#include <cstdarg>
#include <cstdio>
#include <cstdlib>
#include <cstring>

#ifdef CCTK_MPI
#include <mpi.h>
#else
#include "nompi.h"
#endif

#include <defs.hh>
#include <dist.hh>
#include <ggf.hh>
#include <typeprops.hh>

#include <carpet.hh>

namespace Carpet {

using namespace std;

// Get Carpet's GH extension
CarpetGH const *GetCarpetGH(const cGH *const cgh) {
  assert(cgh);
  return &carpetGH;
}

// Get current regridding epoch
extern "C" CCTK_INT
Carpet_GetRegriddingEpoch(CCTK_POINTER_TO_CONST const cctkGH) {
  return regridding_epoch;
}

// Get current regridding epochs
extern "C" CCTK_INT
Carpet_GetRegriddingEpochs(CCTK_POINTER_TO_CONST const cctkGH,
                           CCTK_INT const size,
                           CCTK_INT *restrict const epochs) {
  int const maxrl = min(size, CCTK_INT(level_regridding_epochs.size()));
  for (int rl = 0; rl < maxrl; ++rl) {
    epochs[rl] = level_regridding_epochs.AT(rl);
  }
  return CCTK_INT(level_regridding_epochs.size());
}

// Get current refinement level
extern "C" CCTK_INT
Carpet_GetRefinementLevel(CCTK_POINTER_TO_CONST const cctkGH) {
  return reflevel;
}

// Get number of refinement levels
extern "C" CCTK_INT
Carpet_GetRefinementLevels(CCTK_POINTER_TO_CONST const cctkGH) {
  return reflevels;
}

// Get maximum number of refinement levels
extern "C" CCTK_INT
Carpet_GetMaxRefinementLevels(CCTK_POINTER_TO_CONST const cctkGH) {
  return maxreflevels;
}

// Get current local component
extern "C" CCTK_INT
Carpet_GetLocalComponent(CCTK_POINTER_TO_CONST const cctkGH) {
  assert(reflevel >= 0);
  assert(map >= 0);
  return local_component;
}

// Get number of local components
extern "C" CCTK_INT
Carpet_GetLocalComponents(CCTK_POINTER_TO_CONST const cctkGH) {
  assert(reflevel >= 0);
  assert(map >= 0);
  return vhh.AT(map)->local_components(reflevel);
}

// Get current map level
extern "C" CCTK_INT Carpet_GetMap(CCTK_POINTER_TO_CONST const cctkGH) {
  return map;
}

// Get number of maps
extern "C" CCTK_INT Carpet_GetMaps(CCTK_POINTER_TO_CONST const cctkGH) {
  return maps;
}

// Get current map level
extern "C" CCTK_INT Carpet_GetTimeLevel(CCTK_POINTER_TO_CONST const cctkGH) {
  return timelevel;
}

// Get current map level
extern "C" CCTK_INT
Carpet_GetTimeLevelOffset(CCTK_POINTER_TO_CONST const cctkGH) {
  return timelevel_offset;
}

// Enable or disable prolongating
CCTK_INT CarpetEnableProlongating(const CCTK_INT flag) {
  assert(flag == 0 or flag == 1);
  do_prolongate = flag;
  if (do_prolongate) {
    Checkpoint("Prolongating enabled");
  } else {
    Checkpoint("Prolongating disabled");
  }
  return 0;
}

CCTK_INT CarpetQueryProlongating() { return do_prolongate; }

// Get pointer to grid variable for a specific map and refinement level
CCTK_POINTER
Carpet_VarDataPtrI(CCTK_POINTER_TO_CONST const cctkGH_, CCTK_INT const m,
                   CCTK_INT const rl, CCTK_INT const c, CCTK_INT const tl,
                   CCTK_INT const varindex) {
  assert(cctkGH_);
  cGH const *const cctkGH = static_cast<cGH const *>(cctkGH_);
  assert(varindex >= 0 and varindex < CCTK_NumVars());
  int const groupindex = CCTK_GroupIndexFromVarI(varindex);
  assert(groupindex >= 0);
  int const grouptype = CCTK_GroupTypeI(groupindex);
  assert(mglevel >= 0);
  if (grouptype == CCTK_GF) {
    assert(m >= 0 and m < maps);
    assert(rl >= 0 and rl < reflevels);
    assert(c >= 0 and
           c < arrdata.AT(groupindex).AT(m).hh->components(reflevel));
    assert(arrdata.AT(groupindex).AT(m).hh->is_local(rl, c));
  } else {
    assert(m == 0);
    assert(rl == 0);
    assert(c == CCTK_MyProc(NULL));
  }
  int const maxtls = CCTK_MaxActiveTimeLevelsGI(cctkGH, groupindex);
  assert(tl >= 0 and tl < maxtls);

  int const activetimelevels =
      groupdata.AT(groupindex).activetimelevels.AT(mglevel).AT(rl);
  if (tl < activetimelevels) {
    int const var = varindex - CCTK_FirstVarIndexI(groupindex);
    ggf *const ff = arrdata.AT(groupindex).AT(m).data.AT(var);
    int const lc = arrdata.AT(groupindex).AT(m).hh->get_local_component(rl, c);
    assert(lc >= 0 and
           lc < arrdata.AT(groupindex).AT(m).hh->local_components(rl));
    gdata *const data = ff->data_pointer(tl, rl, lc, mglevel);
    return data->storage();
  } else {
    return NULL;
  }
}

// Multi-Model
CCTK_POINTER_TO_CONST
Carpet_GetMPICommUniverse(CCTK_POINTER_TO_CONST const cctkGH) {
  assert(comm_universe != MPI_COMM_NULL);
  return &comm_universe;
}

CCTK_POINTER_TO_CONST
Carpet_GetMPICommWorld(CCTK_POINTER_TO_CONST const cctkGH) {
  assert(comm_world != MPI_COMM_NULL);
  return &comm_world;
}

// Hosts
extern "C" CCTK_INT Carpet_MyHost(CCTK_POINTER_TO_CONST const cctkGH_) {
  return HostId(dist::rank());
}

extern "C" CCTK_INT Carpet_nHosts(CCTK_POINTER_TO_CONST const cctkGH_) {
  return HostNames().size();
}

extern "C" CCTK_INT Carpet_nProcsOnHost(CCTK_POINTER_TO_CONST const cctkGH_,
                                        CCTK_INT const host) {
  return HostProcs(host).size();
}

extern "C" CCTK_INT Carpet_ProcsOnHost(CCTK_POINTER_TO_CONST const cctkGH_,
                                       CCTK_INT const host, CCTK_INT procs[],
                                       CCTK_INT const nprocs) {
  int const nprocs1 = HostProcs(host).size();
  for (int p = 0; p < nprocs1; ++p) {
    if (p >= nprocs)
      break;
    procs[p] = HostProcs(host).AT(p);
  }
  return nprocs1;
}

// Coordinates

CCTK_INT
Carpet_GetCoordRange(CCTK_POINTER_TO_CONST const cctkGH_, CCTK_INT const m,
                     CCTK_INT const ml, CCTK_INT const size,
                     CCTK_INT *const gsh, CCTK_REAL *const lower,
                     CCTK_REAL *const upper, CCTK_REAL *const delta) {
  cGH const *const cctkGH = static_cast<cGH const *>(cctkGH_);
  assert(cctkGH);
  assert(m >= 0 and m < maps);
  assert(ml >= 0 and ml < mglevels);
  assert(size >= 0);
  assert(not size or gsh);
  assert(not size or lower);
  assert(not size or upper);
  assert(not size or delta);

  assert(size == dim);

  ibbox const &baseext = vhh.AT(m)->baseextents.AT(ml).AT(0);
  ivect const igsh = baseext.shape() / baseext.stride();

  for (int d = 0; d < dim; ++d) {
    gsh[d] = igsh[d];
    lower[d] = origin_space.AT(m).AT(ml)[d];
    delta[d] = delta_space.AT(m)[d] * mglevelfact;
    upper[d] = lower[d] + delta[d] * (gsh[d] - 1);
  }

  return 0;
}

// Communication

int Barrier(const cGH *cgh) {
  const void *dummy = &dummy;
  dummy = &cgh;

  MPI_Barrier(dist::comm());
  return 0;
}

int NamedBarrier(const cGH *const cgh, const unsigned int id,
                 const char *name) {
  const void *dummy = &dummy;
  dummy = &cgh;

  // Prevent recursive calls
  static bool inside = false;
  if (inside)
    return 0;
  inside = true;

  Checkpoint("About to Barrier %ud \"%s\"", id, name);
  dist::barrier(dist::comm(), id, name);
  Checkpoint("Finished Barrier");

  inside = false;
  return 0;
}

int Exit(const cGH *cgh, int retval) {
  NamedBarrier(cgh, 792686462, "Carpet::Exit");
  dist::finalize();
  exit(retval);
  return -1;
}

int Abort(const cGH *cgh, int retval) {
  void *dummy = &dummy;
  dummy = &cgh;

  assert(0);

  MPI_Abort(MPI_COMM_WORLD, retval);
  exit(retval);
  return -1;
}

int MyProc(const cGH *cgh) {
  // This may be called very early, before dist:comm() is valid
  int rank;
  MPI_Comm_rank(dist::goodcomm(), &rank);
  return rank;
}

int nProcs(const cGH *cgh) {
  // This may be called very early, before dist:comm() is valid
  int size;
  MPI_Comm_size(dist::goodcomm(), &size);
  return size;
}

MPI_Comm CarpetMPIComm() { return dist::comm(); }

// Datatypes

MPI_Datatype CarpetMPIDatatype(const int vartype) {
  switch (specific_cactus_type(vartype)) {
#define TYPECASE(N, T)                                                         \
  case N: {                                                                    \
    T dummy;                                                                   \
    return dist::mpi_datatype(dummy);                                          \
  }
#include "typecase.hh"
#undef TYPECASE
  }
  CCTK_VWarn(CCTK_WARN_ABORT, __LINE__, __FILE__, CCTK_THORNSTRING,
             "Carpet does not support the variable type %d.", vartype);
  // notreached
  return MPI_CHAR;
}

MPI_Datatype CarpetSimpleMPIDatatype(const int vartype) {
  switch (vartype) {
  case CCTK_VARIABLE_COMPLEX:
    return CarpetMPIDatatype(CCTK_VARIABLE_REAL);
#ifdef HAVE_CCTK_COMPLEX8
  case CCTK_VARIABLE_COMPLEX8:
    return CarpetMPIDatatype(CCTK_VARIABLE_REAL4);
#endif
#ifdef HAVE_CCTK_COMPLEX16
  case CCTK_VARIABLE_COMPLEX16:
    return CarpetMPIDatatype(CCTK_VARIABLE_REAL8);
#endif
#ifdef HAVE_CCTK_COMPLEX32
  case CCTK_VARIABLE_COMPLEX32:
    return CarpetMPIDatatype(CCTK_VARIABLE_REAL16);
#endif
  }
  // default
  return CarpetMPIDatatype(vartype);
}

int CarpetSimpleMPIDatatypeLength(const int vartype) {
  switch (vartype) {
  case CCTK_VARIABLE_COMPLEX:
#ifdef HAVE_CCTK_COMPLEX8
  case CCTK_VARIABLE_COMPLEX8:
#endif
#ifdef HAVE_CCTK_COMPLEX16
  case CCTK_VARIABLE_COMPLEX16:
#endif
#ifdef HAVE_CCTK_COMPLEX32
  case CCTK_VARIABLE_COMPLEX32:
#endif
    return 2;
  }
  // default
  return 1;
}

// Timelevels

int min_timelevel(const checktimes where, const int num_tl,
                  const bool persistent) {
  assert(num_tl > 0);
  switch (where) {
  case currenttime:
    return 0;
  case currenttimebutnotifonly:
    // don't include current time if there is only one (persistent)
    // time level
    return (not persistent or num_tl > 1) ? 0 : 1;
  case previoustime:
    return 1;
  case allbutlasttime:
    // do include current time if there is only one time level
    return 0;
  case allbutcurrenttime:
    return 1;
  case alltimes:
    return 0;
  default:
    assert(0);
  }
  return -999;
}

int max_timelevel(const checktimes where, const int num_tl,
                  const bool persistent) {
  assert(num_tl > 0);
  switch (where) {
  case currenttime:
    return 0;
  case currenttimebutnotifonly:
    return 0;
  case previoustime:
    return num_tl > 1 ? 1 : 0;
  case allbutlasttime:
    return num_tl - 2;
  case allbutcurrenttime:
    return num_tl - 1;
  case alltimes:
    return num_tl - 1;
  default:
    assert(0);
  }
  return -999;
}

// Diagnostic output

static void prepend_id(char *const msg, size_t const msglen) {
  bool needspace = false;
  if (mglevel != -1) {
    snprintf(msg + strlen(msg), msglen - strlen(msg), "[ml=%d]", mglevel);
    needspace = true;
  }
  if (reflevel != -1) {
    snprintf(msg + strlen(msg), msglen - strlen(msg), "[rl=%d]", reflevel);
    needspace = true;
  }
  if (map != -1) {
    snprintf(msg + strlen(msg), msglen - strlen(msg), "[m=%d]", map);
    needspace = true;
  }
  if (component != -1) {
    snprintf(msg + strlen(msg), msglen - strlen(msg), "[c=%d,lc=%d]", component,
             local_component);
    needspace = true;
  }
  if (timelevel != -1) {
    snprintf(msg + strlen(msg), msglen - strlen(msg), "[tl=%d]", timelevel);
    needspace = true;
  }
  if (needspace) {
    snprintf(msg + strlen(msg), msglen - strlen(msg), " ");
  }
}

void Output(const char *fmt, ...) {
  DECLARE_CCTK_PARAMETERS;
  va_list args;
  char msg[1000];
  snprintf(msg, sizeof msg, "%s", "");
  prepend_id(msg + strlen(msg), sizeof msg - strlen(msg));
  va_start(args, fmt);
  vsnprintf(msg + strlen(msg), sizeof msg - strlen(msg), fmt, args);
  va_end(args);
  CCTK_INFO(msg);
  if (barriers) {
    NamedBarrier(NULL, 988121033, "Carpet::Output");
  }
}

void Waypoint(const char *fmt, ...) {
  DECLARE_CCTK_PARAMETERS;
  if (verbose or veryverbose) {
    va_list args;
    char msg[1000];
    snprintf(msg, sizeof msg, "%s", "");
    prepend_id(msg + strlen(msg), sizeof msg - strlen(msg));
    va_start(args, fmt);
    vsnprintf(msg + strlen(msg), sizeof msg - strlen(msg), fmt, args);
    va_end(args);
    CCTK_INFO(msg);
  }
  if (barriers) {
    NamedBarrier(NULL, 471561432, "Carpet::Waypoint");
  }
}

void Checkpoint(const char *fmt, ...) {
  DECLARE_CCTK_PARAMETERS;
  if (veryverbose) {
    va_list args;
    char msg[1000];
    snprintf(msg, sizeof msg, "%s", "");
    prepend_id(msg + strlen(msg), sizeof msg - strlen(msg));
    va_start(args, fmt);
    vsnprintf(msg + strlen(msg), sizeof msg - strlen(msg), fmt, args);
    va_end(args);
    CCTK_INFO(msg);
  }
  if (barriers) {
    NamedBarrier(NULL, 572893143, "Carpet::Checkpoint");
  }
}

void UnsupportedVarType(const int vindex) {
  assert(vindex >= 0 and vindex < CCTK_NumVars());
  CCTK_VError(__LINE__, __FILE__, CCTK_THORNSTRING,
              "Carpet does not support the type of the variable \"%s\".\n"
              "Either enable support for this type, "
              "or change the type of this variable.",
              CCTK_FullName(vindex));
}

bool IsMap0Group(const int gindex) {
  assert(gindex >= 0 and gindex < CCTK_NumGroups());
  const int table = CCTK_GroupTagsTableI(gindex);
  assert(table >= 0);
  CCTK_INT map0group;
  int status = Util_TableGetInt(table, &map0group, "map0group");
  if (status == UTIL_ERROR_TABLE_NO_SUCH_KEY) {
    map0group = false;
    status = 1;
  }
  if (status != 1) {
    CCTK_ERROR("illegal table value");
  }
  return map0group;
}

} // namespace Carpet

```

OutputGH.cc:
```
#include <cassert>
#include <cstdio>
#include <cstdlib>
#include <sstream>

#include <cctk.h>
#include <cctk_Parameters.h>

#include <Timer.hh>

#include <dist.hh>

#include <carpet.hh>

namespace Carpet {

using namespace std;

int OutputGH(cGH const *const cctkGH) {
  DECLARE_CCTK_PARAMETERS;

  static Timers::Timer timer("OutputGH");
  timer.start();

  Checkpoint("OutputGH");

  int const num_methods = CCTK_NumIOMethods();
  if (num_methods == 0) {
    timer.stop();
    return -1;
  }

  static vector<Timers::Timer *> timers;
  timers.resize(num_methods, NULL);

  int num_vars = 0;
  for (int handle = 0; handle < num_methods; ++handle) {

    IOMethod const *const method = CCTK_IOMethod(handle);
    assert(method);

    if (not timers.AT(handle)) {
      ostringstream buf;
      buf << method->implementation << "::" << method->name << " [" << handle
          << "]";
      timers.AT(handle) = new Timers::Timer(buf.str().c_str());
    }

    timers.AT(handle)->start();
    num_vars += method->OutputGH(cctkGH);
    timers.AT(handle)->stop();

  } // for handle

  timer.stop();

  return num_vars;
}

} // namespace Carpet

```

SetupGH.cc:
```
#define _GNU_SOURCE                                                            \
  1 // needed for sched_getaffinity, best at the top to avoid inconsistent
    // includes

#include <algorithm>
#include <cassert>
#include <climits>
#include <cmath>
#include <cstdlib>
#include <cstring>
#include <iomanip>
#include <iostream>
#include <limits>
#include <sstream>
#include <stack>
#include <string>
#include <vector>

#include <cctk.h>
#include <cctk_Parameters.h>
#include <util_ErrorCodes.h>
#include <util_Network.h>
#include <util_Table.h>

#ifdef HAVE_SCHED_H
#include <sched.h>
#endif
#include <unistd.h>

#include <Requirements.hh>

#include <Timer.hh>

#include <bbox.hh>
#include <defs.hh>
#include <dist.hh>
#include <ggf.hh>
#include <gh.hh>
#include <mpi_string.hh>
#include <region.hh>
#include <vect.hh>

#include <carpet.hh>

namespace Carpet {

using namespace std;

static void setup_model_information(cGH *cctkGH);
static void setup_multigrid_information(cGH *cctkGH);
static void setup_refinement_information();
static void setup_map_information();
static void setup_time_information();
static void setup_domain_extents(cGH const *cctkGH);
static void allocate_grid_hierarchy(cGH const *cctkGH, int m);
static void allocate_data_hierarchy(cGH const *cctkGH, int m);
static void allocate_time_hierarchy(cGH const *cctkGH);
static void setup_grid_hierarchy(cGH const *cctkGH);
static void set_base_extent(int m, vector<vector<region_t> > &regss);

static void allocate_group_data(cGH const *cctkGH);
static void allocate_group_hierarchies(int group, ivect const &sizes,
                                       vector<i2vect> const &ghosts);
static void setup_group_grid_hierarchy(cGH const *cctkGH, int group,
                                       cGroup const &gdata,
                                       ivect const &convpowers,
                                       ivect const &convoffsets);
static void initialise_group_info(cGH const *cctkGH, int group,
                                  cGroup const &gdata);
static void set_state(cGH *cctkGH);
static void enable_storage_for_all_groups(cGH const *cctkGH);

static vector<i2vect> get_ghostzones();
static vector<int> get_prolongation_orders_space();
static ivect get_npoints();
static void get_boundary_specification(cGH const *cctkGH, int m,
                                       vector<i2vect> const &ghosts,
                                       i2vect &nboundaryzones,
                                       b2vect &is_internal,
                                       b2vect &is_staggered, i2vect &shiftout);
static void get_domain_specification(cGH const *cctkGH, int m,
                                     ivect const &npoints, rvect &physical_min,
                                     rvect &physical_max, rvect &base_spacing);
static void adapt_domain_specification(int m, rvect const &physical_min,
                                       rvect const &physical_max,
                                       rvect const &base_spacing,
                                       rvect &exterior_min, rvect &exterior_max,
                                       rvect &spacing);
static void calculate_grid_points(int m, vector<i2vect> const &ghosts,
                                  rvect const &exterior_min,
                                  rvect const &exterior_max,
                                  rvect const &spacing, ivect &npoints);
static void
calculate_base_extents(ibbox const &baseextent, centering refcentering,
                       centering mgcentering, i2vect const &nboundaryzones,
                       b2vect const &is_internal, b2vect const &is_staggered,
                       i2vect const &shiftout,
                       vector<vector<ibbox> > &baseextents);
static void find_processor_decomposition(
    cGH const *cctkGH, vector<vector<vector<region_t> > > &superregsss,
    vector<vector<vector<vector<region_t> > > > &regssss);

static void get_group_size(int group, cGroup const &gdata, ivect &sizes,
                           vector<i2vect> &ghosts);
static void adapt_group_size_mglevel(int group, cGroup const &gdata,
                                     ivect &sizes, ivect &convpowers,
                                     ivect &convoffsets);
static void get_convergence_options(int group, cGroup const &gdata,
                                    ivect &convpowers, ivect &convoffsets);
static void adapt_group_size_disttype(cGH const *cctkGH, int group,
                                      cGroup const &gdata, ivect &sizes,
                                      vector<i2vect> const &ghosts);
static void output_group_statistics(cGH const *cctkGH);
static operator_type get_transport_operator(cGH const *cctkGH, int group,
                                            cGroup const &gdata);
static bool can_transfer_variable_type(cGH const *cctkGH, int group,
                                       cGroup const &gdata);

static void ensure_CartGrid3D_type();
#if 0
  static void
  ensure_CartGrid3D_domain ();  // UNUSED
#endif
static void ensure_CartGrid3D_avoid_origin();
static void ensure_ReflectionSymmetry_avoid_origin(centering refcentering);
static void ensure_ghostzones(int m, vector<i2vect> const &ghosts);
static void ensure_group_options(int group, cGroup const &gdata);

void *SetupGH(tFleshConfig *const fc, int const convLevel, cGH *const cctkGH) {
  DECLARE_CCTK_PARAMETERS;

  // Some statistics
  {
    int const nprocs = dist::size();
    int const myproc = dist::rank();
    dist::set_num_threads(num_threads);
    int const mynthreads = dist::num_threads();
    int const nthreads_total = dist::total_num_threads();
    // Can't store this in a variable because the value is different
    // for each thread
    // int const mythreadnum = dist::thread_num();
    char const *const CACTUS_NUM_PROCS = getenv("CACTUS_NUM_PROCS");
    int const cactus_num_procs = CACTUS_NUM_PROCS ? atoi(CACTUS_NUM_PROCS) : 0;
#ifdef CCTK_MPI
    CCTK_VInfo(CCTK_THORNSTRING, "MPI is enabled");
    CCTK_VInfo(CCTK_THORNSTRING, "Carpet is running on %d processes", nprocs);
    if (not CACTUS_NUM_PROCS) {
      CCTK_VWarn(CCTK_WARN_COMPLAIN, __LINE__, __FILE__, CCTK_THORNSTRING,
                 "Although MPI is enabled, the environment variable "
                 "CACTUS_NUM_PROCS is not set.");
    } else {
      if (cactus_num_procs != nprocs) {
        CCTK_VWarn(CCTK_WARN_ABORT, __LINE__, __FILE__, CCTK_THORNSTRING,
                   "The environment variable CACTUS_NUM_PROCS is set to %d, "
                   "but there are %d MPI processes. This may indicate a severe "
                   "problem with the MPI startup mechanism.",
                   cactus_num_procs, nprocs);
      }
    }
    CCTK_VInfo(CCTK_THORNSTRING, "This is process %d", myproc);
#else
    CCTK_VInfo(CCTK_THORNSTRING, "MPI is disabled");
    if (CACTUS_NUM_PROCS and cactus_num_procs != nprocs) {
      CCTK_VWarn(CCTK_WARN_ABORT, __LINE__, __FILE__, CCTK_THORNSTRING,
                 "Although MPI is disabled, the environment variable "
                 "CACTUS_NUM_PROCS is set to %d. This may indicate a severe "
                 "problem with the Cactus startup mechanism.",
                 cactus_num_procs);
    }
#endif
    char const *const OMP_NUM_THREADS = getenv("OMP_NUM_THREADS");
    char const *const CACTUS_NUM_THREADS = getenv("CACTUS_NUM_THREADS");
    int const cactus_num_threads =
        CACTUS_NUM_THREADS ? atoi(CACTUS_NUM_THREADS) : 0;
#ifdef _OPENMP
    CCTK_VInfo(CCTK_THORNSTRING, "OpenMP is enabled");
    if (not OMP_NUM_THREADS and num_threads == -1) {
      CCTK_VWarn(CCTK_WARN_COMPLAIN, __LINE__, __FILE__, CCTK_THORNSTRING,
                 "Although OpenMP is enabled, neither the environment variable "
                 "OMP_NUM_THREADS nor the parameter Carpet::num_threads are "
                 "set. A system-specific default value is used instead.");
    }
    CCTK_VInfo(CCTK_THORNSTRING,
               "This process contains %d threads, this is thread %d",
               mynthreads, dist::thread_num());
    if (not CACTUS_NUM_THREADS) {
      CCTK_VWarn(CCTK_WARN_COMPLAIN, __LINE__, __FILE__, CCTK_THORNSTRING,
                 "Although OpenMP is enabled, the environment variable "
                 "CACTUS_NUM_THREADS is not set.");
    } else {
      if (cactus_num_threads != mynthreads) {
        CCTK_VWarn(CCTK_WARN_ABORT, __LINE__, __FILE__, CCTK_THORNSTRING,
                   "The environment variable CACTUS_NUM_THREADS is set to %d, "
                   "but there are %d threads on this process. This may "
                   "indicate a severe problem with the OpenMP startup "
                   "mechanism.",
                   cactus_num_threads, mynthreads);
      }
    }
    CCTK_VInfo(CCTK_THORNSTRING, "There are %d threads in total",
               nthreads_total);
#ifdef CCTK_MPI
    CCTK_VInfo(CCTK_THORNSTRING, "There are %g threads per process",
               1.0 * nthreads_total / nprocs);
#endif
#else
    CCTK_VInfo(CCTK_THORNSTRING, "OpenMP is disabled");
    int const omp_num_threads = OMP_NUM_THREADS ? atoi(OMP_NUM_THREADS) : 0;
    if (omp_num_threads > 0) {
      CCTK_VWarn(CCTK_WARN_ALERT, __LINE__, __FILE__, CCTK_THORNSTRING,
                 "Although OpenMP is disabled, the environment variable "
                 "OMP_NUM_THREADS is set to %d. It will be ignored.",
                 omp_num_threads);
    }
    if (num_threads > 0) {
      CCTK_VWarn(CCTK_WARN_ALERT, __LINE__, __FILE__, CCTK_THORNSTRING,
                 "Although OpenMP is disabled, the parameter "
                 "Carpet::num_threads is set to %d. It will be ignored.",
                 num_threads);
    }
    if (CACTUS_NUM_THREADS and cactus_num_threads != mynthreads) {
      CCTK_VWarn(CCTK_WARN_ABORT, __LINE__, __FILE__, CCTK_THORNSTRING,
                 "Although OpenMP is disabled, the environment variable "
                 "CACTUS_NUM_THREADS is set to %d. This may indicate a severe "
                 "problem with the Cactus startup mechanism.",
                 cactus_num_threads);
    }
#endif

    char hostnamebuf[MPI_MAX_PROCESSOR_NAME];
    int hostnamelen;
    MPI_Get_processor_name(hostnamebuf, &hostnamelen);
    string const hostname(hostnamebuf);
    DetermineHosts(hostname, verbose);
#if HAVE_GETPID
    int const mypid = static_cast<int>(getpid());
#else
    int const mypid = -1;
#endif
    // Output
    CCTK_VInfo(CCTK_THORNSTRING, "This process runs on host %s, pid=%d",
               hostname.c_str(), mypid);
    if (verbose or veryverbose) {
      // Collect process ids
      vector<int> pids(nprocs);
      MPI_Allgather(const_cast<int *>(&mypid), 1, MPI_INT, &pids.front(), 1,
                    MPI_INT, dist::comm());
      // Collect number of threads
      vector<int> nthreads(nprocs);
      MPI_Allgather(const_cast<int *>(&mynthreads), 1, MPI_INT,
                    &nthreads.front(), 1, MPI_INT, dist::comm());
      // Output
      CCTK_VInfo(CCTK_THORNSTRING, "Running on the following hosts:");
      for (int n = 0; n < nprocs; ++n) {
        int const host_id = HostId(n);
        CCTK_VInfo(CCTK_THORNSTRING,
                   "   %6d: hid=%d (%s), pid=%d, num_threads=%d", n, host_id,
                   HostName(host_id).c_str(), pids.AT(n), nthreads.AT(n));
      }
    }

    if (set_cpu_affinity) {
#ifdef HAVE_SCHED_GETAFFINITY
      // Cores available to this process
      assert(mynthreads <= CPU_SETSIZE);
      vector<bool> mask(CPU_SETSIZE, false);
#pragma omp parallel
      {
        cpu_set_t cpumask;
        int const ierr = sched_getaffinity(0, sizeof cpumask, &cpumask);
        assert(not ierr);
        for (int n = 0; n < CPU_SETSIZE; ++n) {
          if (CPU_ISSET(n, &cpumask)) {
#pragma omp critical
            mask.at(n) = true;
          }
        }
      }
      int const host_id = HostId(myproc);
      vector<int> const host_procs = HostProcs(host_id);
      int const num_host_procs = host_procs.size();
      assert(num_host_procs > 0);
#if 0
        // Collect information from all processes on this host
        vector<char> sendbuf(num_host_procs * CPU_SETSIZE, 0);
        vector<char> recvbuf(num_host_procs * CPU_SETSIZE);
        for (int i=0; i<num_host_procs; ++i) {
          for (int n=0; n<CPU_SETSIZE; ++n) {
            sendbuf.at(i * CPU_SETSIZE + n) = mask.at(n);
          }
        }
        vector<int> sendcount(nprocs, 0);
        vector<int> recvcount(nprocs, 0);
        for (int i=0; i<num_host_procs; ++i) {
          int const p = host_procs.at(i);
          sendcount.at(p) = CPU_SETSIZE;
          recvcount.at(p) = CPU_SETSIZE;
        }
        vector<int> senddispls(nprocs);
        vector<int> recvdispls(nprocs);
        senddispls.at(0) = 0;
        recvdispls.at(0) = 0;
        for (int p=1; p<nprocs; ++p) {
          senddispls.at(p) = senddispls.at(p-1) + sendcount.at(p-1);
          recvdispls.at(p) = recvdispls.at(p-1) + recvcount.at(p-1);
        }
        MPI_Alltoallv(&sendbuf[0], &sendcounts[0], &senddispls[0], MPI_CHAR,
                      &recvbuf[0], &recvcounts[0], &recvdispls[0], MPI_CHAR,
                      MPI_COMM_WORLD);
        // Cores available on this host
        vector<bool> othermasks(CPU_SETSIZE, false);
        for (int i=0; i<num_host_procs; ++i) {
          int const p = host_procs.at(i);
          if (p != myproc) {
            for (int n=0; n<CPU_SETSIZE; ++n) {
              othermasks.at(n) =
                othermasks.at(n) or recvbuf.at(i * CPU_SETSIZE + n);
            }
          }
        }
#endif
      int num_cores = 0;
      for (int n = 0; n < CPU_SETSIZE; ++n) {
        if (mask.at(n))
          ++num_cores;
      }
      if (num_cores == 0) {
        CCTK_WARN(CCTK_WARN_ALERT, "Cannot select core set for this process "
                                   "(no cores seem available to this process)");
      } else {
        bool const other_procs_need_cores =
            num_cores >= mynthreads * num_host_procs;
        int my_host_index = 0;
        if (other_procs_need_cores) {
          // It seems that all cores are available to all processes
          // on this host. Split the cores evenly across the
          // processes.
          for (int i = 0; i < num_host_procs; ++i) {
            int const p = host_procs.at(i);
            if (p == myproc) {
              my_host_index = i;
              break;
            }
          }
          if (num_cores % num_host_procs != 0) {
            CCTK_WARN(CCTK_WARN_ALERT, "The number of processes on this host "
                                       "does not evenly divide the number of "
                                       "cores on this host -- leaving some "
                                       "cores unassigned");
          }
          num_cores /= num_host_procs;
        }
        if (num_cores % mynthreads != 0) {
          CCTK_WARN(CCTK_WARN_ALERT, "The number of threads in this process "
                                     "does not evenly divide the number of "
                                     "cores for this process -- leaving some "
                                     "cores unassigned");
        }
        int const num_cores_per_thread = num_cores / mynthreads;
        // Choose cores for this process
        int const first_core_index =
            my_host_index * mynthreads * num_cores_per_thread;
        vector<int> available_cores;
        for (int n = 0; n < CPU_SETSIZE; ++n) {
          if (mask.at(n))
            available_cores.push_back(n);
        }
        int const n0 = available_cores.at(first_core_index);
        CCTK_VInfo(CCTK_THORNSTRING,
                   "Selecting cores %d (and following) for this process", n0);
#pragma omp parallel
        {
          cpu_set_t cpumask;
          CPU_ZERO(&cpumask);
          int const i0 =
              first_core_index + dist::thread_num() * num_cores_per_thread;
          for (int i = 0; i < num_cores_per_thread; ++i) {
            CPU_SET(available_cores[i0 + i], &cpumask);
          }
          int const ierr = sched_setaffinity(0, sizeof(cpumask), &cpumask);
          assert(not ierr);
        }
      }
#else
      CCTK_WARN(CCTK_WARN_ALERT, "Cannot select core set for this process; "
                                 "sched_getaffinity is not available on this "
                                 "system");
#endif
    }

#ifdef HAVE_SCHED_GETAFFINITY
    {
      vector<bool> mask(CPU_SETSIZE, false);
#pragma omp parallel
      {
        cpu_set_t cpumask;
        int const ierr = sched_getaffinity(0, sizeof cpumask, &cpumask);
        assert(not ierr);
        for (int n = 0; n < CPU_SETSIZE; ++n) {
          if (CPU_ISSET(n, &cpumask)) {
#pragma omp critical
            mask.at(n) = true;
          }
        }
      }

      ostringstream buf;
      int num_cores = 0;
      bool isfirst = true;
      int first_active = -1;
      for (int n = 0; n < CPU_SETSIZE; ++n) {
        if (mask.at(n))
          ++num_cores;
        if (first_active == -1 and mask.at(n)) {
          if (not isfirst)
            buf << ", ";
          isfirst = false;
          buf << n;
          first_active = n;
        } else if (first_active >= 0 and not mask.at(n)) {
          if (n - 1 > first_active)
            buf << "-" << n - 1;
          first_active = -1;
        }
      }
      CCTK_VInfo(CCTK_THORNSTRING, "This process runs on %d core%s: %s",
                 num_cores, num_cores == 1 ? "" : "s", buf.str().c_str());
      if (mynthreads > num_cores) {
        CCTK_WARN(CCTK_WARN_ALERT, "The number of threads for this process is "
                                   "larger its number of cores. This may "
                                   "indicate a performance problem.");
      }
    }

    for (int thread = 0; thread < mynthreads; ++thread) {
      vector<bool> mask(CPU_SETSIZE, false);
      cpu_set_t cpumask;
#pragma omp parallel
      if (thread == dist::thread_num()) {
        int const ierr = sched_getaffinity(0, sizeof cpumask, &cpumask);
        assert(not ierr);
      }
      for (int n = 0; n < CPU_SETSIZE; ++n) {
        if (CPU_ISSET(n, &cpumask)) {
          mask.at(n) = true;
        }
      }

      ostringstream buf;
      int num_cores = 0;
      bool isfirst = true;
      int first_active = -1;
      for (int n = 0; n < CPU_SETSIZE; ++n) {
        if (mask.at(n))
          ++num_cores;
        if (first_active == -1 and mask.at(n)) {
          if (not isfirst)
            buf << ", ";
          isfirst = false;
          buf << n;
          first_active = n;
        } else if (first_active >= 0 and not mask.at(n)) {
          if (n - 1 > first_active)
            buf << "-" << n - 1;
          first_active = -1;
        }
      }
      CCTK_VInfo(CCTK_THORNSTRING, "Thread %d runs on %d core%s: %s", thread,
                 num_cores, num_cores == 1 ? "" : "s", buf.str().c_str());
    }
#else
    CCTK_INFO("Cannot determine core affinity");
#endif
  }

  // Initialise current position (must be the very first thing,
  // before the first output)
  mglevel = -1;
  reflevel = -1;
  mc_grouptype = -1;
  map = -1;
  component = -1;
  local_component = -1;
  timelevel = -1;
  timelevel_offset = -1;
#ifdef HAVE_CGH_CCTK_MODE
  cctkGH->cctk_mode = CCTK_MODE_META;
#endif

  // Say hello

  Timers::Timer timer("CarpetStartup");
  timer.start();

  Waypoint("Setting up the grid hierarchy");

  // Check arguments:
  CCTK_VInfo(CCTK_THORNSTRING, "This simulation is running in %d dimensions",
             cctkGH->cctk_dim);
  // Only a specific number of dimensions is supported
  assert(cctkGH->cctk_dim == dim);
  // Not sure what to do with that
  assert(convLevel == 0);

  // Set up descriptions from user parameters
  setup_model_information(cctkGH);
  setup_multigrid_information(cctkGH);
  setup_refinement_information();
  setup_map_information();
  setup_time_information();

  // Calculate domain extents for each map
  setup_domain_extents(cctkGH);

  // Set up grid hierarchy
  setup_grid_hierarchy(cctkGH);

  // Allocate space for group descriptions
  allocate_group_data(cctkGH);

  // Set times, origin, and spacings, and go to meta mode
  set_state(cctkGH);

  // Enable prolongating
  do_allow_past_timelevels = true;
  do_prolongate = true;
  do_taper = false;
  do_warn_about_storage = false; // This is enabled later
  in_analysis_bin = false;

  if (enable_all_storage) {
    if (not enable_no_storage) {
      enable_storage_for_all_groups(cctkGH);
    }
  }

  Waypoint("Done with setting up the grid hierarchy");
  timer.stop();

  return &carpetGH;
}

//////////////////////////////////////////////////////////////////////////////
// Routines which perform actions ////////////////////////////////////////////
//////////////////////////////////////////////////////////////////////////////

void setup_model_information(cGH *const cctkGH) {
  DECLARE_CCTK_PARAMETERS;

  cctkGH->identity = strdup(model);
}

void setup_multigrid_information(cGH *const cctkGH) {
  DECLARE_CCTK_PARAMETERS;

  basemglevel = convergence_level;
  mglevels = num_convergence_levels;
  mgfact = convergence_factor;
  maxmglevelfact = ipow(mgfact, mglevels - 1);
  cctkGH->cctk_convfac = mgfact;
}

void setup_refinement_information() {
  DECLARE_CCTK_PARAMETERS;

  // Set maximum number of refinement levels
  maxreflevels = max_refinement_levels;

  // Set the per-level spatial refinement factors
  if (CCTK_EQUALS(space_refinement_factors, "")) {
    // Calculate them from the default refinement factor
    spacereffacts.resize(maxreflevels);
    assert(ipow(2, 0) == 1);
    assert(ipow(3, 1) == 3);
    assert(ipow(4, 2) == 16);
    for (int rl = 0; rl < maxreflevels; ++rl) {
      spacereffacts.AT(rl) = ivect(ipow(int(refinement_factor), rl));
    }
  } else {
    // Read them from the parameter
    try {
      istringstream srf(space_refinement_factors);
      srf >> spacereffacts;
    } catch (input_error) {
      CCTK_ERROR("Could not parse parameter \"space_refinement_factors\"");
    }
  }
  // TODO: turn these into real error messages
  assert(int(spacereffacts.size()) >= maxreflevels);
  assert(all(spacereffacts.AT(0) == 1));
  for (int rl = 1; rl < maxreflevels; ++rl) {
    assert(all(spacereffacts.AT(rl) >= spacereffacts.AT(rl - 1)));
    assert(all(spacereffacts.AT(rl) % spacereffacts.AT(rl - 1) == 0));
  }
  spacereffacts.resize(maxreflevels);

  // Set the per-level temporal refinement factors
  if (CCTK_EQUALS(time_refinement_factors, "")) {
    // Calculate them from the default refinement factor
    timereffacts.resize(maxreflevels);
    for (int rl = 0; rl < maxreflevels; ++rl) {
      timereffacts.AT(rl) = ipow(int(refinement_factor), rl);
    }
  } else {
    // Read them from the parameter
    try {
      istringstream trf(time_refinement_factors);
      trf >> timereffacts;
    } catch (input_error) {
      CCTK_ERROR("Could not parse parameter \"time_refinement_factors\"");
    }
  }
  // TODO: turn these into real error messages
  assert(int(timereffacts.size()) >= maxreflevels);
  assert(timereffacts.AT(0) == 1);
  for (int rl = 1; rl < maxreflevels; ++rl) {
    assert(timereffacts.AT(rl) >= timereffacts.AT(rl - 1));
    assert(timereffacts.AT(rl) % timereffacts.AT(rl - 1) == 0);
  }
  timereffacts.resize(maxreflevels);

  // Calculate the maximum refinement factors
  maxtimereflevelfact = timereffacts.AT(maxreflevels - 1);
  maxspacereflevelfact = spacereffacts.AT(maxreflevels - 1);
}

void setup_map_information() {
  DECLARE_CCTK_PARAMETERS;

  if (domain_from_multipatch) {
    assert(num_maps == 1); // must be the default to avoid confusion
    assert(CCTK_IsFunctionAliased("MultiPatch_GetSystemSpecification"));
    CCTK_INT maps1;
    check(not MultiPatch_GetSystemSpecification(&maps1));
    maps = maps1;
  } else {
    maps = num_maps;
  }
  carpetGH.maps = maps;
#ifdef REQUIREMENTS_HH
  Requirements::Setup(maps);
#endif
}

void setup_time_information() {
  DECLARE_CCTK_PARAMETERS;

  // Set maximum number of time levels
  if (max_timelevels < 0) {
    // Set automatically (backward compatibility)
    maxtimelevels = prolongation_order_time + 1;
  } else {
    maxtimelevels = max_timelevels;
  }
  if (maxtimelevels < prolongation_order_time + 1) {
    CCTK_VError(__LINE__, __FILE__, CCTK_THORNSTRING,
                "There are not enough time levels for this time prolongation "
                "order: max_timelevels=%d, prolongation_order_time=%d",
                int(max_timelevels), int(prolongation_order_time));
  }
}

void setup_domain_extents(cGH const *const cctkGH) {
  DECLARE_CCTK_PARAMETERS;

  for (int m = 0; m < maps; ++m) {

    // Allocate hierarchies
    allocate_grid_hierarchy(cctkGH, m);
    allocate_data_hierarchy(cctkGH, m);

  } // for m

  allocate_time_hierarchy(cctkGH);
}

void allocate_grid_hierarchy(cGH const *const cctkGH, int const m) {
  DECLARE_CCTK_PARAMETERS;

  // Centering
  centering refcentering;
  int reffactdenom;
  if (CCTK_EQUALS(refinement_centering, "vertex")) {
    refcentering = vertex_centered;
    reffactdenom = 1;
  } else if (CCTK_EQUALS(refinement_centering, "cell")) {
    refcentering = cell_centered;
    reffactdenom = 2;
  } else {
    assert(0);
  }

  // Number of grid points
  ivect npoints = get_npoints();

  // Number of ghost zones
  vector<i2vect> const ghosts = get_ghostzones();

  // Boundary description
  i2vect nboundaryzones;
  b2vect is_internal;
  b2vect is_staggered;
  i2vect shiftout;
  get_boundary_specification(cctkGH, m, ghosts, nboundaryzones, is_internal,
                             is_staggered, shiftout);

  // Grid size
  rvect physical_min, physical_max;
  rvect base_spacing;
  get_domain_specification(cctkGH, m, npoints, physical_min, physical_max,
                           base_spacing);

  if (maxreflevels > 1) {
    // Ensure that ReflectionSymmetry::avoid_origin is set correctly
    ensure_ReflectionSymmetry_avoid_origin(refcentering);
  }

  // Adapt domain specification for convergence level
  rvect exterior_min, exterior_max;
  rvect spacing;
  adapt_domain_specification(m, physical_min, physical_max, base_spacing,
                             exterior_min, exterior_max, spacing);

  // Calculate global number of grid points
  calculate_grid_points(m, ghosts, exterior_min, exterior_max, spacing,
                        npoints);

  centering const mgcentering = refcentering;

  // Base grid extent
  ivect const str(maxspacereflevelfact * reffactdenom);
  ivect const lb(0);
  ivect const ub((npoints - 1) * str);
  ibbox const baseext(lb, ub, str);

  vector<vector<ibbox> > baseexts;
  calculate_base_extents(baseext, refcentering, mgcentering, nboundaryzones,
                         is_internal, is_staggered, shiftout, baseexts);

  // Allocate grid hierarchy

  Timers::Timer timer("AllocateGridHierarchy");
  timer.start();
  vhh.resize(maps);
  vhh.AT(m) = new gh(spacereffacts, refcentering, convergence_factor,
                     mgcentering, baseexts, nboundaryzones);
  timer.stop();
}

void allocate_data_hierarchy(cGH const *const cctkGH, int const m) {
  DECLARE_CCTK_PARAMETERS;

  // Number of ghost zones
  vector<i2vect> const ghosts = get_ghostzones();

  int buffer_factor;
  if (use_buffer_zones) {
    if (num_integrator_substeps == -1) {
      assert(CCTK_IsFunctionAliased("MoLNumIntegratorSubsteps"));
      buffer_factor = MoLNumIntegratorSubsteps();
    } else {
      buffer_factor = num_integrator_substeps;
    }
  } else {
    buffer_factor = 1;
  }
  assert(buffer_factor > 0);

  int const taper_factor = use_tapered_grids ? refinement_factor : 1;
  for (int rl = 0; rl < maxreflevels; ++rl) {
    assert(all(all(
        buffer_factor * ghosts.AT(rl) + int(additional_buffer_zones) >= 0)));
  }

  const streamsize oldprecision = cout.precision();
  const ios_base::fmtflags oldflags = cout.flags();
  cout.setf(ios::fixed);
  CCTK_INFO("Buffer zone counts (excluding ghosts):");
  vector<i2vect> buffers(maxreflevels);
  for (int rl = 0; rl < maxreflevels; ++rl) {
    buffers.AT(rl) =
        rl == 0 ? i2vect(0) : taper_factor * (buffer_factor * ghosts.AT(rl) +
                                              int(additional_buffer_zones)) -
                                  ghosts.AT(rl);
    cout << "   [" << rl << "]: " << buffers.AT(rl) << "\n";
    assert(all(all(buffers.AT(rl) >= 0)));
  }
  CCTK_INFO("Overlap zone counts:");
  vector<i2vect> overlaps(maxreflevels);
  for (int rl = 0; rl < maxreflevels; ++rl) {
    gh const &hh = *vhh.AT(m);
    overlaps.AT(rl) =
        rl == 0 ? i2vect(0)
                : (use_overlap_zones
                       ? hh.reffacts.AT(rl) / hh.reffacts.AT(rl - 1) *
                             (ghosts.AT(rl) + int(additional_overlap_zones))
                       : i2vect(0));
    cout << "   [" << rl << "]: " << overlaps.AT(rl) << "\n";
    assert(all(all(overlaps.AT(rl) >= 0)));
  }
  cout.precision(oldprecision);
  cout.setf(oldflags);

  vector<int> const my_prolongation_orders_space =
      get_prolongation_orders_space();

  vdd.resize(maps);
  vdd.AT(m) = new dh(*vhh.AT(m), ghosts, buffers, overlaps,
                     my_prolongation_orders_space);

  if (maxreflevels > 1) {
    ensure_ghostzones(m, ghosts);
  }
}

void allocate_time_hierarchy(cGH const *const cctkGH) {
  DECLARE_CCTK_PARAMETERS;

  // We are using the gh of the first map.  This works because all
  // maps have the same number of refinement levels.
  tt = new th(*vhh.AT(0), maxtimelevels, timereffacts,
              time_interpolation_during_regridding);
}

void setup_grid_hierarchy(cGH const *const cctkGH) {
  DECLARE_CCTK_PARAMETERS;

  vector<vector<vector<region_t> > > superregsss(maps);
  for (int m = 0; m < maps; ++m) {
    set_base_extent(m, superregsss.AT(m));
  }

  vector<vector<vector<vector<region_t> > > > regssss;
  find_processor_decomposition(cctkGH, superregsss, regssss);

  for (int m = 0; m < maps; ++m) {

    // Check the regions
    CheckRegions(regssss.AT(m));

    // Recompose grid hierarchy
    vhh.AT(m)->regrid(superregsss.AT(m), regssss.AT(m), false);
    int const rl = 0;
    vhh.AT(m)->recompose(rl, false);
    vhh.AT(m)->regrid_free(false);

    // Output grid structure to screen but not to files since
    // IO_TruncateFiles might not be present yet
    OutputGrids(cctkGH, m, *vhh.AT(m), *vdd.AT(m));

  } // for m

  regridding_epoch = 0;
  level_regridding_epochs.resize(1);
  level_regridding_epochs.AT(0) = 0;

  if (verbose or veryverbose) {
    CCTK_INFO("Grid structure (grid points):");
    for (int ml = 0; ml < mglevels; ++ml) {
      int const rl = 0;
      for (int m = 0; m < maps; ++m) {
        for (int c = 0; c < vhh.AT(m)->components(rl); ++c) {
          ibbox const ext = vhh.AT(m)->extent(ml, rl, c);
          ivect const lower = ext.lower();
          ivect const upper = ext.upper();
          int const convfact = ipow(mgfact, ml);
          ibbox const &base = vhh.AT(m)->baseextent(ml, 0);
          ivect const &bstride = base.stride();
          assert(all(lower % bstride == 0));
          assert(all(upper % bstride == 0));
          assert(all(((upper - lower) / bstride) % convfact == 0));
          cout << "   [" << ml << "][" << rl << "][" << m << "][" << c << "]"
               << "   exterior: "
               << "proc " << vhh.AT(m)->processor(rl, c) << "   "
               << lower / bstride << " : " << upper / bstride << "   ("
               << (upper - lower) / bstride / convfact + 1 << ") "
               << prod((upper - lower) / bstride / convfact + 1) << endl;
        }
      }
    }
  }

  // Assert that all maps have one refinement level
  reflevels = 1;
  for (int m = 0; m < maps; ++m) {
    assert(vhh.AT(m)->reflevels() == reflevels);
  }

  timelevels = maxtimelevels;
}

void set_base_extent(int const m, vector<vector<region_t> > &regss) {
  DECLARE_CCTK_PARAMETERS;

  // Create one refinement level
  int const rl = 0;
  regss.resize(1);
  vector<region_t> &regs = regss.AT(rl);

  if (CCTK_EQUALS(base_extents, "")) {

    // Default: one grid component covering everything
    region_t reg;
    reg.extent = vhh.AT(m)->baseextents.AT(0).AT(0);
    reg.outer_boundaries = b2vect(bvect(true));
    reg.map = m;
    regs.push_back(reg);

  } else {

    // Read explicit grid components
    // TODO: invent something for the other convergence levels
    vector<ibbox> exts;
    istringstream ext_str(base_extents);
    try {
      ext_str >> exts;
    } catch (input_error) {
      CCTK_ERROR("Could not parse parameter \"base_extents\"");
    }
    CCTK_VInfo(CCTK_THORNSTRING, "Using %d grid components", int(exts.size()));
    if (exts.size() == 0) {
      CCTK_ERROR("Cannot evolve with zero grid components");
    }

    vector<bbvect> obs;
    istringstream ob_str(base_outerbounds);
    try {
      ob_str >> obs;
    } catch (input_error) {
      CCTK_ERROR("Could not parse parameter \"base_outerbounds\"");
    }
    assert(obs.size() == exts.size());

    for (size_t n = 0; n < exts.size(); ++n) {
      region_t reg;
      reg.extent = exts.AT(n);
      reg.outer_boundaries = xpose(obs.AT(n));
      reg.map = m;
      regs.push_back(reg);
    }
  }
}

void allocate_group_data(cGH const *const cctkGH) {
  groupdata.resize(CCTK_NumGroups());
  arrdata.resize(CCTK_NumGroups());

  for (int group = 0; group < CCTK_NumGroups(); ++group) {

    cGroup gdata;
    check(not CCTK_GroupData(group, &gdata));

    // Check for compact, contiguous, and staggered groups
    ensure_group_options(group, gdata);

    switch (gdata.grouptype) {

    // Grid functions
    case CCTK_GF: {

      // All grid function groups must have the standard rank
      assert(gdata.dim == dim);

      // Set up one refinement level
      groupdata.AT(group).activetimelevels.resize(mglevels);
      for (int ml = 0; ml < mglevels; ++ml) {
        groupdata.AT(group).activetimelevels.AT(ml).resize(1);
      }

      // Initial maximum number of timelevels from interface.ccl
      groupdata.AT(group).info.maxtimelevels = gdata.numtimelevels;

      // Grid function groups use the global grid descriptors
      arrdata.AT(group).resize(maps);
      for (int m = 0; m < maps; ++m) {
        arrdata.AT(group).AT(m).hh = vhh.AT(m);
        arrdata.AT(group).AT(m).dd = vdd.AT(m);
        arrdata.AT(group).AT(m).tt = tt;
      }

      break;
    }

    // Grid scalars and grid arrays are treated in the same way
    case CCTK_SCALAR:
    case CCTK_ARRAY: {

      // All grid variables must have at most the standard rank
      assert(gdata.dim >= 0 and gdata.dim <= dim);

      // Use only one refinement level for grid arrays
      groupdata.AT(group).activetimelevels.resize(mglevels);
      for (int ml = 0; ml < mglevels; ++ml) {
        groupdata.AT(group).activetimelevels.AT(ml).resize(1);
      }

      // Initial maximum number of timelevels from interface.ccl
      groupdata.AT(group).info.maxtimelevels = gdata.numtimelevels;

      // Use only one map for grid arrays
      arrdata.AT(group).resize(1);

      ivect sizes;
      vector<i2vect> ghosts;
      get_group_size(group, gdata, sizes, ghosts);

      // Adapt group sizes for convergence level
      ivect convpowers, convoffsets;
      adapt_group_size_mglevel(group, gdata, sizes, convpowers, convoffsets);
      // Adapt group sizes for disttype
      adapt_group_size_disttype(cctkGH, group, gdata, sizes, ghosts);

      allocate_group_hierarchies(group, sizes, ghosts);

      setup_group_grid_hierarchy(cctkGH, group, gdata, convpowers, convoffsets);

      break;
    } // case scalar or array

    default:
      assert(0);
    } // switch grouptype

    initialise_group_info(cctkGH, group, gdata);

  } // for groups

  output_group_statistics(cctkGH);
}

void allocate_group_hierarchies(int const group, ivect const &sizes,
                                vector<i2vect> const &ghosts) {
  DECLARE_CCTK_PARAMETERS;

  // Calculate base extent
  ivect const str(1);
  ivect const lb(0);
  ivect const ub((sizes - 1) * str);
  ibbox const baseext(lb, ub, str);
  vector<vector<ibbox> > baseexts(1);
  baseexts.AT(0).resize(1);
  baseexts.AT(0).AT(0) = baseext;
  i2vect const nboundaryzones(0);

  // One refinement level
  vector<int> grouptimereffacts(1);
  grouptimereffacts.AT(0) = 1;
  vector<ivect> groupspacereffacts(1);
  groupspacereffacts.AT(0) = ivect(1);

  // There is only one map
  int const m = 0;

  arrdata.AT(group).AT(m).hh =
      new gh(groupspacereffacts, vertex_centered, convergence_factor,
             vertex_centered, baseexts, nboundaryzones);

  vector<i2vect> const buffers(1, i2vect(0));
  vector<i2vect> const overlaps(1, i2vect(0));
  vector<int> const my_prolongation_orders_space(1, 0);
  arrdata.AT(group).AT(m).dd =
      new dh(*arrdata.AT(group).AT(m).hh, ghosts, buffers, overlaps,
             my_prolongation_orders_space);

  arrdata.AT(group).AT(m).tt =
      new th(*arrdata.AT(group).AT(m).hh, maxtimelevels, grouptimereffacts,
             time_interpolation_during_regridding);
}

void setup_group_grid_hierarchy(cGH const *const cctkGH, int const group,
                                cGroup const &gdata, ivect const &convpowers,
                                ivect const &convoffsets) {
#if 0
    // Do not set up anything for groups that have zero variables
    // TODO: To do this, need to change modes.cc to provide illegal
    // entries for GroupDynamicData
    int const nvars = CCTK_NumVarsInGroupI (group);
    assert (nvars >= 0);
    if (nvars == 0) return;
#endif

  // Set refinement structure for scalars and arrays
  vector<region_t> superregs(1);
  int const m = 0;
  int const rl = 0;
  {
    int const c = 0;
    superregs.AT(c).extent =
        arrdata.AT(group).AT(m).hh->baseextents.AT(rl).AT(c);
    superregs.AT(c).outer_boundaries = b2vect(true);
    superregs.AT(c).map = m;
  }
  vector<region_t> regs;

  // Split it into components, one for each processor
  switch (gdata.disttype) {
  case CCTK_DISTRIB_DEFAULT: {
    CCTK_INT no_split_directions[dim];
    bvect no_split_dims(false);
    int const nvals = Util_TableGetIntArray(
        gdata.tagstable, dim, no_split_directions, "no_split_directions");
    assert((0 <= nvals && nvals <= dim) ||
           nvals == UTIL_ERROR_TABLE_NO_SUCH_KEY);
    for (int i = 0; i < nvals; ++i) {
      assert(no_split_directions[i] < dim);
      no_split_dims[no_split_directions[i]] = true;
    }
    SplitRegions_Automatic(cctkGH, superregs, regs, no_split_dims);
    break;
  }
  case CCTK_DISTRIB_CONSTANT: {
    int const d = min(dim - 1, gdata.dim == 0 ? 1 : gdata.dim);
    SplitRegions_AlongDir(cctkGH, superregs, regs, d);
    break;
  }
  default:
    assert(0);
  }

  // Add empty regions if there are fewer regions than processors
  {
    int const nprocs = CCTK_nProcs(cctkGH);
    int const oldsize = regs.size();
    if (oldsize < nprocs) {
      // Ensure that there are at least nprocs components

      // Construct an empty bbox that is just to the right of the
      // last bbox, and which has the correct stride
      assert(not regs.empty());
      ibbox const &lbox = (*regs.rbegin()).extent;
      ibbox const ebox(lbox.upper() + lbox.stride(), lbox.upper(),
                       lbox.stride());

      regs.resize(nprocs);
      for (int c = oldsize; c < nprocs; ++c) {
        region_t empty;
        empty.extent = ebox;
        empty.processor = c;
        regs.AT(c) = empty;
      }
    }
  }

  // Check processor distribution
  {
    int const nprocs = CCTK_nProcs(cctkGH);
    vector<bool> used(nprocs, false);
    for (int c = 0; c < nprocs; ++c) {
      int const p = regs.AT(c).processor;
      assert(p >= 0 and p < nprocs);
      assert(not used.AT(p));
      used.AT(p) = true;
    }
    for (int c = 0; c < nprocs; ++c) {
      assert(used.AT(c));
    }
  }

  // Only one refinement level
  vector<vector<region_t> > superregss(1);
  superregss.AT(rl) = superregs;
  vector<vector<region_t> > regss(1);
  regss.AT(rl) = regs;

  // Create all multigrid levels
  vector<vector<vector<region_t> > > regsss(mglevels);
  ivect mgfact1;
  i2vect offset;
  for (int d = 0; d < dim; ++d) {
    mgfact1[d] = ipow(mgfact, convpowers[d]);
    offset[0][d] = 0;
    offset[1][d] = convoffsets[d];
  }
  regsss.AT(0) = regss;

  for (int ml = 1; ml < mglevels; ++ml) {
    for (int c = 0; c < int(regss.AT(rl).size()); ++c) {
      // this base
      ivect const baselo = ivect(0);
      ivect const baselo1 = baselo;
      // next finer grid
      ivect const flo = regsss.AT(ml - 1).AT(rl).AT(c).extent.lower();
      ivect const fhi = regsss.AT(ml - 1).AT(rl).AT(c).extent.upper();
      ivect const fstr = regsss.AT(ml - 1).AT(rl).AT(c).extent.stride();
      // this grid
      ivect const str = fstr * mgfact1;
      ivect const lo =
          flo + either(regsss.AT(ml - 1).AT(rl).AT(c).outer_boundaries[0],
                       +(offset[0] - mgfact1 * offset[0]) * fstr, ivect(0));
      ivect const hi =
          fhi + either(regsss.AT(ml - 1).AT(rl).AT(c).outer_boundaries[1],
                       -(offset[1] - mgfact1 * offset[1]) * fstr, ivect(0));
      ivect const lo1 = baselo1 + (lo - baselo1 + str - 1) / str * str;
      ivect const hi1 = lo1 + (hi - lo1) / str * str;
      regsss.AT(ml).AT(rl).AT(c) = regsss.AT(ml - 1).AT(rl).AT(c);
      regsss.AT(ml).AT(rl).AT(c).extent = ibbox(lo1, hi1, str);
    }
  }

  // Recompose for this map
  {
    char *const groupname = CCTK_GroupName(group);
    assert(groupname);
    Checkpoint("Recomposing grid array group \"%s\"...", groupname);
    arrdata.AT(group).AT(0).hh->regrid(superregss, regsss, false);
    arrdata.AT(group).AT(0).hh->recompose(0, false);
    arrdata.AT(group).AT(0).hh->regrid_free(false);
    Checkpoint("Done recomposing grid array group \"%s\".", groupname);
    free(groupname);
  }
}

void initialise_group_info(cGH const *const cctkGH, int const group,
                           cGroup const &gdata) {
  DECLARE_CCTK_PARAMETERS;

  // Initialise group information
  groupdata.AT(group).info.dim = gdata.dim;
  groupdata.AT(group).info.gsh = new int[dim];
  groupdata.AT(group).info.lsh = new int[dim];
  groupdata.AT(group).info.ash = new int[dim];
  groupdata.AT(group).info.lbnd = new int[dim];
  groupdata.AT(group).info.ubnd = new int[dim];
  groupdata.AT(group).info.bbox = new int[2 * dim];
  groupdata.AT(group).info.nghostzones = new int[dim];

  groupdata.AT(group).transport_operator =
      get_transport_operator(cctkGH, group, gdata);

  groupdata.AT(group).info.activetimelevels = 0;

  // Initialise group variables
  for (size_t m = 0; m < arrdata.AT(group).size(); ++m) {

    arrdata.AT(group).AT(m).data.resize(CCTK_NumVarsInGroupI(group));
    for (size_t var = 0; var < arrdata.AT(group).AT(m).data.size(); ++var) {
      arrdata.AT(group).AT(m).data.AT(var) = 0;
    }
  }
}

void set_state(cGH *const cctkGH) {
  DECLARE_CCTK_PARAMETERS;

  // // Allocate level times
  // leveltimes.resize (mglevels);
  // for (int ml=0; ml<mglevels; ++ml) {
  //   leveltimes.AT(ml).resize (1);
  // }

  // Allocate orgin and spacings
  origin_space.resize(maps);
  delta_space.resize(maps);
  for (int m = 0; m < maps; ++m) {
    origin_space.AT(m).resize(mglevels);
  }

  // Current state
  mglevelfact = 1;
  cctkGH->cctk_time = 0.0;
  cctkGH->cctk_delta_time = 1.0;
  for (int d = 0; d < dim; ++d) {
    cctkGH->cctk_origin_space[d] = 0.0;
    cctkGH->cctk_delta_space[d] = 1.0;
  }

  // Set up things as if in local mode
  mglevel = 0;
  reflevel = 0;
  mc_grouptype = CCTK_GF;
  map = 0;
  component = 0;
  local_component = -1;
  timelevel = 0;
  timelevel_offset = 0;

  // Leave everything, so that everything is set up correctly
  Timers::Timer("meta mode", 1).start();
  Timers::Timer("global mode", 1).start();
  Timers::Timer("level(0)", 1).start();
  if (include_maps_in_mode_timer_tree) {
    Timers::Timer("map(0)", 1).start();
  }
  if (include_local_mode_in_mode_timer_tree) {
    Timers::Timer("local", 1).start();
  }
  leave_local_mode(cctkGH);
  leave_singlemap_mode(cctkGH);
  leave_level_mode(cctkGH);
  leave_global_mode(cctkGH);
}

void enable_storage_for_all_groups(cGH const *const cctkGH) {
  BEGIN_MGLEVEL_LOOP(cctkGH) {
    BEGIN_REFLEVEL_LOOP(cctkGH) {

      for (int group = 0; group < CCTK_NumGroups(); ++group) {
        char *const groupname = CCTK_GroupName(group);
        EnableGroupStorage(cctkGH, groupname);
        free(groupname);
      }
    }
    END_REFLEVEL_LOOP;
  }
  END_MGLEVEL_LOOP;
}

//////////////////////////////////////////////////////////////////////////////
// Routines which do not change state ////////////////////////////////////////
//////////////////////////////////////////////////////////////////////////////

vector<i2vect> get_ghostzones() {
  DECLARE_CCTK_PARAMETERS;

  vector<i2vect> ghostzones;
  // Decide which parameters to use
  if (CCTK_EQUALS(ghost_sizes, "")) {
    i2vect ghostzones1;
    if (ghost_size == -1) {
      ghostzones1 = i2vect(ivect(ghost_size_x, ghost_size_y, ghost_size_z),
                           ivect(ghost_size_x, ghost_size_y, ghost_size_z));
    } else {
      ghostzones1 = i2vect(ivect(ghost_size, ghost_size, ghost_size),
                           ivect(ghost_size, ghost_size, ghost_size));
    }
    ghostzones.resize(maxreflevels, ghostzones1);
  } else {
    // Read them from the parameter
    vector<int> ghostzones1;
    try {
      istringstream gs(ghost_sizes);
      gs >> ghostzones1;
    } catch (input_error) {
      CCTK_WARN(CCTK_WARN_ABORT, "Could not parse parameter \"ghost_sizes\"");
    }
    assert(int(ghostzones1.size()) >= maxreflevels);
    for (int rl = 0; rl < maxreflevels; ++rl) {
      ghostzones.AT(rl) = i2vect(ghostzones1.AT(rl));
    }
  }
  for (int rl = 0; rl < maxreflevels; ++rl) {
    assert(all(all(ghostzones.AT(rl) >= 0)));
  }
  return ghostzones;
}

vector<int> get_prolongation_orders_space() {
  DECLARE_CCTK_PARAMETERS;

  vector<int> orders;
  // Decide which parameters to use
  if (CCTK_EQUALS(prolongation_orders_space, "")) {
    orders.resize(maxreflevels, prolongation_order_space);
  } else {
    // Read them from the parameter
    try {
      istringstream pos(prolongation_orders_space);
      pos >> orders;
    } catch (input_error) {
      CCTK_WARN(CCTK_WARN_ABORT,
                "Could not parse parameter \"prolongation_orders_space\"");
    }
    assert(int(orders.size()) >= maxreflevels);
    for (int rl = 0; rl < maxreflevels; ++rl) {
      assert(orders.AT(rl) >= 0);
    }
  }
  return orders;
}

ivect get_npoints() {
  DECLARE_CCTK_PARAMETERS;

  // Decide which parameters to use
  ivect npoints;
  if (global_nsize == -1) {
    npoints = ivect(global_nx, global_ny, global_nz);
  } else {
    npoints = ivect(global_nsize, global_nsize, global_nsize);
  }

  // Modify npoints for benchmarks
  if (constant_load_per_processor) {
    if (CCTK_EQUALS(processor_topology, "manual")) {
      // Enlarge the domain so that each processor has the specified
      // number of grid points, using the specified processor
      // topology
      assert(processor_topology_3d_x >= 1);
      assert(processor_topology_3d_y >= 1);
      assert(processor_topology_3d_z >= 1);
      ivect const topo = ivect(processor_topology_3d_x, processor_topology_3d_y,
                               processor_topology_3d_z);
      npoints *= topo;
    } else if (CCTK_EQUALS(processor_topology, "automatic")) {
      // Enlarge the domain in a smart way so that each processor
      // has the specified number of grid points
      int const nprocs = dist::total_num_threads();
      // Factorise the number of processors, placing the smallest
      // factors in the tail
      stack<int> factors;
      for (int procsleft = nprocs; procsleft > 1;) {
        for (int divisor = 2; divisor <= procsleft; ++divisor) {
          while (procsleft % divisor == 0) {
            factors.push(divisor);
            procsleft /= divisor;
          }
        }
      }
      // Distribute the factors greedily onto the directions,
      // starting with the largest factor, and preferring to enlarge
      // the x direction
      while (not factors.empty()) {
        int const mindir = minloc(npoints);
        assert(mindir >= 0 and mindir < dim);
        int const factor = factors.top();
        factors.pop();
        npoints[mindir] *= factor;
      }
    } else {
      // TODO: handle processor_topology values "along-z" and "along-dir"
      CCTK_ERROR("Unsupported value of parameter processor_topology");
    }
  } // if constant_load_per_processor
  return npoints;
}

void get_boundary_specification(cGH const *const cctkGH, int const m,
                                vector<i2vect> const &ghosts,
                                i2vect &nboundaryzones, b2vect &is_internal,
                                b2vect &is_staggered, i2vect &shiftout) {
  DECLARE_CCTK_PARAMETERS;

  if (domain_from_multipatch or domain_from_coordbase) {

    jjvect nboundaryzones_, is_internal_, is_staggered_, shiftout_;
    if (CCTK_IsFunctionAliased("MultiPatch_GetBoundarySpecification")) {
      check(not MultiPatch_GetBoundarySpecification(
          m, 2 * dim, &nboundaryzones_[0][0], &is_internal_[0][0],
          &is_staggered_[0][0], &shiftout_[0][0]));
    } else {
      check(not GetBoundarySpecification(
          2 * dim, &nboundaryzones_[0][0], &is_internal_[0][0],
          &is_staggered_[0][0], &shiftout_[0][0]));
    }
    nboundaryzones = xpose(nboundaryzones_);
    is_internal = xpose(is_internal_);
    is_staggered = xpose(is_staggered_);
    shiftout = xpose(shiftout_);

  } else {
    // Legacy code

    // Assume that there are 0 boundary points at outer boundaries
    // and nghostzones boundary points at symmetry boundaries

    // Ensure that all levels have the same number of ghost zones
    for (size_t rl = 1; rl < ghosts.size(); ++rl) {
      assert(all(all(ghosts.AT(rl) == ghosts.AT(0))));
    }

#if 0
      // We cannot call GetSymmetryBoundaries boundaries here, since
      // the symmetry boundaries have not yet been registered.
      jjvect symbnd_;
      check (not GetSymmetryBoundaries (cctkGH, 2*dim, &symbnd_[0][0]));
      b2vect const symbnd = xpose (symbnd_);
#else
    b2vect const symbnd = b2vect(true);
#endif

    for (int f = 0; f < 2; ++f) {
      for (int d = 0; d < dim; ++d) {
        if (symbnd[f][d]) {
          nboundaryzones[f][d] = ghosts.AT(0)[f][d];
          is_internal[f][d] = false;
          // TODO: Look at what CartGrid3D's avoid_origin
          // TODO: Take cell centring into account
          is_staggered[f][d] = false;
          shiftout[f][d] = 1;
        } else {
          // TODO: This is wrong
          nboundaryzones[f][d] = 0;
          is_internal[f][d] = false;
          is_staggered[f][d] = false;
          shiftout[f][d] = 0;
        }
      }
    }
  }

  ostringstream buf;
  buf << "Boundary specification for map " << m << ":" << endl
      << "   nboundaryzones: " << nboundaryzones << endl
      << "   is_internal   : " << is_internal << endl
      << "   is_staggered  : " << is_staggered << endl
      << "   shiftout      : " << shiftout;
  Output(buf.str().c_str());

  if (max_refinement_levels > 1) {
    // Ensure that the boundary is not staggered
    for (int d = 0; d < dim; ++d) {
      for (int f = 0; f < 2; ++f) {
        if (CCTK_EQUALS(refinement_centering, "vertex")) {
          if (is_staggered[f][d]) {
            CCTK_ERROR("The parameters CoordBase::boundary_staggered specify a "
                       "staggered boundary.  Carpet does not support staggered "
                       "boundaries when Carpet::max_refinement_levels > 1 with "
                       "Carpet::centering = \"vertex\"");
          }
        } else if (CCTK_EQUALS(refinement_centering, "cell")) {
          if (not is_staggered[f][d]) {
            CCTK_ERROR("The parameters CoordBase::boundary_staggered specify a "
                       "non-staggered boundary.  Carpet does not support "
                       "non-staggered boundaries when "
                       "Carpet::max_refinement_levels > 1 with "
                       "Carpet::centering = \"cell\"");
          }
        } else {
          assert(0);
        }
      }
    }
  }
}

void get_domain_specification(cGH const *cctkGH, int const m,
                              ivect const &npoints, rvect &physical_min,
                              rvect &physical_max, rvect &base_spacing) {
  DECLARE_CCTK_PARAMETERS;

  rvect interior_min, interior_max;
  rvect exterior_min, exterior_max;

  if (domain_from_multipatch) {
    assert(not domain_from_coordbase);

    // TODO: handle CartGrid3D: either add parameter
    // type=multipatch, and make it handle map numbers, or ignore it
    // altogether, maybe creating a new thorn

    assert(CCTK_IsFunctionAliased("MultiPatch_GetDomainSpecification"));
    check(not MultiPatch_GetDomainSpecification(
        m, dim, &physical_min[0], &physical_max[0], &interior_min[0],
        &interior_max[0], &exterior_min[0], &exterior_max[0],
        &base_spacing[0]));

  } else if (domain_from_coordbase) {

    assert(not CCTK_IsFunctionAliased("MultiPatch_GetDomainSpecification"));

    // Ensure that CartGrid3D::type = "coordbase"
    ensure_CartGrid3D_type();

    check(not GetDomainSpecification(dim, &physical_min[0], &physical_max[0],
                                     &interior_min[0], &interior_max[0],
                                     &exterior_min[0], &exterior_max[0],
                                     &base_spacing[0]));

  } else {
    // Legacy code

    assert(not CCTK_IsFunctionAliased("MultiPatch_GetDomainSpecification"));

    if (max_refinement_levels > 1) {
      // Ensure that CartGrid3D::avoid_origin = no
      ensure_CartGrid3D_avoid_origin();
    } // if max_refinement_levels > 1

    ostringstream buf;
    buf << "Standard grid specification for map " << m << ":" << endl
        << "   number of grid points: " << npoints;
    Output(buf.str().c_str());

    // Reduce to physical domain
    // TODO: This is not the true domain specification.  However, it
    // is later written to the domainspec, and it is used by Carpet
    // for screen output.
    exterior_min = 0.0;
    exterior_max = rvect(npoints - 1);
    base_spacing = 1.0;
    check(not ConvertFromExteriorBoundary(dim, &physical_min[0],
                                          &physical_max[0], &interior_min[0],
                                          &interior_max[0], &exterior_min[0],
                                          &exterior_max[0], &base_spacing[0]));

  } // if legacy domain specification

  ostringstream buf;
  buf << "CoordBase domain specification for map " << m << ":" << endl
      << "   physical extent: " << physical_min << " : " << physical_max
      << "   (" << physical_max - physical_min << ")" << endl
      << "   interior extent: " << interior_min << " : " << interior_max
      << "   (" << interior_max - interior_min << ")" << endl
      << "   exterior extent: " << exterior_min << " : " << exterior_max
      << "   (" << exterior_max - exterior_min << ")" << endl
      << "   base_spacing   : " << base_spacing;
  Output(buf.str().c_str());
}

void adapt_domain_specification(int const m, rvect const &physical_min,
                                rvect const &physical_max,
                                rvect const &base_spacing, rvect &exterior_min,
                                rvect &exterior_max, rvect &spacing) {
  DECLARE_CCTK_PARAMETERS;

  CCTK_REAL const baseconvfact =
      ipow(static_cast<CCTK_REAL>(convergence_factor), basemglevel);
  spacing = base_spacing * baseconvfact;

  rvect interior_min, interior_max;

  if (domain_from_multipatch and
      CCTK_IsFunctionAliased("MultiPatch_ConvertFromPhysicalBoundary")) {
    assert(not domain_from_coordbase);
    check(not MultiPatch_ConvertFromPhysicalBoundary(
        m, dim, &physical_min[0], &physical_max[0], &interior_min[0],
        &interior_max[0], &exterior_min[0], &exterior_max[0], &spacing[0]));
  } else {
    check(not ConvertFromPhysicalBoundary(
        dim, &physical_min[0], &physical_max[0], &interior_min[0],
        &interior_max[0], &exterior_min[0], &exterior_max[0], &spacing[0]));
  }

  ostringstream buf;
  buf << "Adapted domain specification for map " << m << ":" << endl
      << "   convergence factor: " << convergence_factor << endl
      << "   convergence level : " << basemglevel << endl
      << "   physical extent   : " << physical_min << " : " << physical_max
      << "   (" << physical_max - physical_min << ")" << endl
      << "   interior extent   : " << interior_min << " : " << interior_max
      << "   (" << interior_max - interior_min << ")" << endl
      << "   exterior extent   : " << exterior_min << " : " << exterior_max
      << "   (" << exterior_max - exterior_min << ")" << endl
      << "   spacing           : " << spacing;
  Output(buf.str().c_str());
}

void calculate_grid_points(int const m, vector<i2vect> const &ghosts,
                           rvect const &exterior_min, rvect const &exterior_max,
                           rvect const &spacing, ivect &npoints) {
  DECLARE_CCTK_PARAMETERS;

  rvect const real_npoints = either(
      spacing, (exterior_max - exterior_min) / spacing + rvect(1), rvect(1));

  ostringstream buf;
  buf << "Base grid specification for map " << m << ":" << endl
      << "   number of grid points             : " << real_npoints << endl
      << "   number of coarse grid ghost points: " << ghosts.AT(0);
  Output(buf.str().c_str());

  npoints = floor(real_npoints + static_cast<CCTK_REAL>(0.5));

  // Check domain size
  if (any(fabs(rvect(npoints) - real_npoints) >
          static_cast<CCTK_REAL>(0.001))) {
    CCTK_VError(__LINE__, __FILE__, CCTK_THORNSTRING,
                "The domain size for map %d scaled for convergence level %d "
                "with convergence factor %d is not integer",
                m, int(basemglevel), int(convergence_factor));
  }

  // Sanity check
  assert(all(npoints <= INT_MAX));
#if 0
    int max = INT_MAX;
    for (int d=0; d<dim; ++d) {
      assert (npoints[d] <= max);
      max /= npoints[d];
    }
#endif
  {
    CCTK_REAL const total_npoints = prod(rvect(npoints));
    CCTK_REAL const size_max = numeric_limits<size_type>::max();
    if (total_npoints > size_max) {
      CCTK_VError(__LINE__, __FILE__, CCTK_THORNSTRING,
                  "The domain for map %d contains %g grid points.  This number "
                  "is larger than the maximum number supported by Carpet (%g).",
                  m, double(total_npoints), double(size_max));
    }
    CCTK_REAL const int_max = numeric_limits<int>::max();
    if (total_npoints > int_max) {
      if (dist::rank() == 0) {
        CCTK_VWarn(1, __LINE__, __FILE__, CCTK_THORNSTRING,
                   "The domain for map %d contains %g grid points.  This "
                   "number is larger than the maximum number that can be "
                   "represented as an integer (%g).  This can lead to strange "
                   "problems in thorns which try to calculate the total number "
                   "of grid points.",
                   m, double(total_npoints), double(int_max));
      }
    }
  }

  // Save domain specification
  domainspecs.resize(maps);
  domainspecs.AT(m).exterior_min = exterior_min;
  domainspecs.AT(m).exterior_max = exterior_max;
  domainspecs.AT(m).npoints = npoints;
}

void calculate_base_extents(ibbox const &baseextent,
                            centering const refcentering,
                            centering const mgcentering,
                            i2vect const &nboundaryzones,
                            b2vect const &is_internal,
                            b2vect const &is_staggered, i2vect const &shiftout,
                            vector<vector<ibbox> > &baseextents) {
  DECLARE_CCTK_PARAMETERS;

  assert(baseextents.empty());
  baseextents.resize(num_convergence_levels);
  for (int ml = 0; ml < num_convergence_levels; ++ml) {
    baseextents.AT(ml).resize(maxreflevels);
    for (int rl = 0; rl < maxreflevels; ++rl) {
      if (ml == 0) {
        if (rl == 0) {
          // Use base extent
          baseextents.AT(ml).AT(rl) = baseextent;
        } else {
          // Refine next coarser refinement level
          switch (refcentering) {
          case vertex_centered: {
            ibbox const &cbox = baseextents.AT(ml).AT(rl - 1);
            assert(not any(any(is_staggered)));
            i2vect const bnd_shift =
                (nboundaryzones - 1) * i2vect(not is_internal) + shiftout;
            ibbox const cbox_phys = cbox.expand(-bnd_shift);
            assert(all(baseextent.stride() % spacereffacts.AT(rl - 1) == 0));
            ivect const fstride = baseextent.stride() / spacereffacts.AT(rl);
            ibbox const fbox_phys =
                ibbox(cbox_phys.lower(), cbox_phys.upper(), fstride);
            ibbox const fbox = fbox_phys.expand(bnd_shift);
            baseextents.AT(ml).AT(rl) = fbox;
            break;
          }
          case cell_centered: {
            ibbox const &cbox = baseextents.AT(ml).AT(rl - 1);
            assert(all(all(is_staggered)));
            ivect const cstride = cbox.stride();
            assert(all(cstride % 2 == 0));
            i2vect const bnd_shift_cstride =
                +(cstride / 2 - i2vect(is_internal) * cstride) +
                (nboundaryzones - 1) * i2vect(not is_internal) * cstride +
                shiftout * cstride;
            ibbox const cbox_phys(cbox.lower() - (-bnd_shift_cstride[0]),
                                  cbox.upper() + (-bnd_shift_cstride[1]),
                                  cbox.stride());
            assert(all(baseextent.stride() % spacereffacts.AT(rl - 1) == 0));
            ivect const fstride = baseextent.stride() / spacereffacts.AT(rl);
            ibbox const fbox_phys =
                ibbox(cbox_phys.lower(), cbox_phys.upper(), fstride);
            assert(all(cstride % fstride == 0));
            assert(all(all(bnd_shift_cstride % (cstride / fstride) == 0)));
            i2vect const bnd_shift_fstride =
                bnd_shift_cstride / (cstride / fstride);
            ibbox const fbox(fbox_phys.lower() - bnd_shift_fstride[0],
                             fbox_phys.upper() + bnd_shift_fstride[1],
                             fbox_phys.stride());
            baseextents.AT(ml).AT(rl) = fbox;
            break;
          }
          default:
            assert(0);
          }
        }
      } else {
        // Coarsen next finer convergence level
        assert(mgcentering == vertex_centered);
        assert(not any(any(is_staggered)));
        i2vect const bnd_shift =
            (nboundaryzones - 1) * i2vect(not is_internal) + shiftout;
        ibbox const &fbox = baseextents.AT(ml - 1).AT(rl);
        ibbox const fbox_phys = fbox.expand(-bnd_shift);
        ivect const cstride = fbox.stride() * int(convergence_factor);
        ibbox const cbox_phys =
            ibbox(fbox_phys.lower(), fbox_phys.upper(), cstride);
        ibbox const cbox = cbox_phys.expand(bnd_shift);
        baseextents.AT(ml).AT(rl) = cbox;
      }
    }
  }
}

void find_processor_decomposition(
    cGH const *const cctkGH, vector<vector<vector<region_t> > > &superregsss,
    vector<vector<vector<vector<region_t> > > > &regssss) {
  DECLARE_CCTK_PARAMETERS;

  assert(regssss.empty());
  regssss.resize(maps);

  if (not regrid_in_level_mode) {
    // Distribute each map independently

    for (int m = 0; m < maps; ++m) {
      int const rl = 0;

      vector<vector<region_t> > regss(1);

      // Distribute onto the processors
      SplitRegions(cctkGH, superregsss.AT(m).AT(rl), regss.AT(rl));

      // Create all multigrid levels
      MakeMultigridBoxes(cctkGH, m, regss, regssss.AT(m));
    } // for m

  } else {
    // Distribute all maps at the same time

    int const rl = 0;

    vector<vector<region_t> > superregss(maps);
    for (int m = 0; m < maps; ++m) {
      superregss.AT(m) = superregsss.AT(m).AT(rl);
    }

    vector<vector<region_t> > regss(maps);
    SplitRegionsMaps(cctkGH, superregss, regss);

    vector<vector<vector<region_t> > > regsss(maps);
    for (int m = 0; m < maps; ++m) {
      superregsss.AT(m).AT(rl) = superregss.AT(m);
      regsss.AT(m).resize(1);
      regsss.AT(m).AT(rl) = regss.AT(m);
    }

    // Create all multigrid levels
    MakeMultigridBoxesMaps(cctkGH, regsss, regssss);

  } // if
}

void get_group_size(int const group, cGroup const &gdata, ivect &sizes,
                    vector<i2vect> &ghosts) {
  // Default values
  sizes = 1;
  ghosts.resize(1, i2vect(ivect(0)));

  switch (gdata.grouptype) {

  case CCTK_SCALAR:
    // treat scalars as DIM=0, DISTRIB=const arrays
    assert(gdata.dim == 0);
    assert(gdata.disttype == CCTK_DISTRIB_CONSTANT);
    break;

  case CCTK_ARRAY: {
    assert(gdata.dim >= 1 or gdata.dim <= dim);
    CCTK_INT const *const *const sz = CCTK_GroupSizesI(group);
    CCTK_INT const *const *const gsz = CCTK_GroupGhostsizesI(group);
    // Decode group sizes
    for (int d = 0; d < gdata.dim; ++d) {
      if (sz) {
        sizes[d] = *sz[d];
      }
      if (gsz) {
        ghosts.AT(0)[0][d] = *gsz[d];
        ghosts.AT(0)[1][d] = *gsz[d];
      }
    }
    break;
  }

  default:
    assert(0);
  } // switch grouptype
}

void adapt_group_size_mglevel(int const group, cGroup const &gdata,
                              ivect &sizes, ivect &convpowers,
                              ivect &convoffsets) {
  DECLARE_CCTK_PARAMETERS;

  // Adapt array sizes for convergence level
  get_convergence_options(group, gdata, convpowers, convoffsets);

  ivect baseconvpowers = convpowers * int(basemglevel);
  rvect real_sizes = (((rvect(sizes) - rvect(convoffsets)) /
                       ipow(rvect(convergence_factor), baseconvpowers)) +
                      rvect(convoffsets));
  // Do not modify extra dimensions
  for (int d = gdata.dim; d < dim; ++d) {
    real_sizes[d] = sizes[d];
  }

  // Round group sizes
  sizes = floor(real_sizes + static_cast<CCTK_REAL>(0.5));

  if (any(sizes < 0)) {
    char *const groupname = CCTK_GroupName(group);
    CCTK_VError(__LINE__, __FILE__, CCTK_THORNSTRING,
                "The shape of group \"%s\" scaled for convergence level %d "
                "with convergence factor %d is negative",
                groupname, int(basemglevel), int(convergence_factor));
    free(groupname);
  }

  if (any(fabs(rvect(sizes) - real_sizes) > static_cast<CCTK_REAL>(1.0e-8))) {
    char *const groupname = CCTK_GroupName(group);
    CCTK_VError(__LINE__, __FILE__, CCTK_THORNSTRING,
                "The shape of group \"%s\" scaled for convergence level %d "
                "with convergence factor %d is not integer",
                groupname, int(basemglevel), int(convergence_factor));
    free(groupname);
  }
}

void get_convergence_options(int const group, cGroup const &gdata,
                             ivect &convpowers, ivect &convoffsets) {
  if (gdata.tagstable >= 0) {
    {
      jvect convpowers1;
      int const status = Util_TableGetIntArray(
          gdata.tagstable, gdata.dim, &convpowers1[0], "convergence_power");
      if (status == UTIL_ERROR_TABLE_NO_SUCH_KEY) {
        // use default: independent of convergence level
        convpowers = 0;
      } else if (status == 1) {
        // a scalar was given
        convpowers = convpowers1[0];
      } else if (status == gdata.dim) {
        convpowers = convpowers1;
      } else {
        char *const groupname = CCTK_GroupName(group);
        CCTK_VError(__LINE__, __FILE__, CCTK_THORNSTRING,
                    "The key \"convergence_power\" in the tags table of group "
                    "\"%s\" is wrong",
                    groupname);
        free(groupname);
      }
      assert(all(convpowers >= 0));
    }

    {
      jvect convoffsets1;
      int const status = Util_TableGetIntArray(
          gdata.tagstable, gdata.dim, &convoffsets1[0], "convergence_offset");
      if (status == UTIL_ERROR_TABLE_NO_SUCH_KEY) {
        // use default: offset is 0
        convoffsets = 0;
      } else if (status == 1) {
        // a scalar was given

      } else if (status == gdata.dim) {
        convoffsets = convoffsets1;
      } else {
        char *const groupname = CCTK_GroupName(group);
        CCTK_VError(__LINE__, __FILE__, CCTK_THORNSTRING,
                    "The key \"convergence_offset\" in the tags table of group "
                    "\"%s\" is wrong",
                    groupname);
        free(groupname);
      }
    }
  }
}

void adapt_group_size_disttype(cGH const *const cctkGH, int const group,
                               cGroup const &gdata, ivect &sizes,
                               vector<i2vect> const &ghosts) {
  switch (gdata.disttype) {

  case CCTK_DISTRIB_DEFAULT: {
    // do nothing
    break;
  }

  case CCTK_DISTRIB_CONSTANT: {
    if (not all(all(ghosts.AT(0) == 0))) {
      char *const groupname = CCTK_GroupName(group);
      CCTK_VError(__LINE__, __FILE__, CCTK_THORNSTRING,
                  "The group \"%s\" has DISTRIB=constant, but its "
                  "ghostsize is not 0",
                  groupname);
      free(groupname);
    }
    assert(all(all(ghosts.AT(0) == 0)));

    int const nprocs = CCTK_nProcs(cctkGH);

    // Find dimension which should be extended
    int const d = min(dim - 1, gdata.dim == 0 ? 1 : gdata.dim);
    // Extend group sizes
    assert(sizes[d] < numeric_limits<int>::max() / nprocs);
    sizes[d] *= nprocs;
    assert(sizes[d] >= 0);
    break;
  }

  default:
    assert(0);
  } // switch disttype
}

void output_group_statistics(cGH const *const cctkGH) {
  int num_gf_groups = 0;
  int num_gf_vars = 0;
  vector<int> num_array_groups(dim + 1), num_array_vars(dim + 1);
  for (int d = 0; d <= dim; ++d) {
    num_array_groups.AT(d) = 0;
    num_array_vars.AT(d) = 0;
  }

  for (int group = 0; group < CCTK_NumGroups(); ++group) {

    cGroup gdata;
    check(not CCTK_GroupData(group, &gdata));

    switch (gdata.grouptype) {
    case CCTK_GF:
      num_gf_groups += 1;
      num_gf_vars += gdata.numvars * gdata.numtimelevels;
      break;
    case CCTK_SCALAR:
    case CCTK_ARRAY:
      assert(gdata.dim <= dim);
      num_array_groups.AT(gdata.dim) += 1;
      num_array_vars.AT(gdata.dim) += gdata.numvars * gdata.numtimelevels;
      break;
    default:
      assert(0);
    }
  } // for group

  CCTK_INFO("Group and variable statistics:");
  CCTK_VInfo(CCTK_THORNSTRING, "   There are %d grid functions in %d groups",
             num_gf_vars, num_gf_groups);
  CCTK_VInfo(CCTK_THORNSTRING, "   There are %d grid scalars in %d groups",
             num_array_vars.AT(0), num_array_groups.AT(0));
  for (int d = 1; d <= dim; ++d) {
    CCTK_VInfo(CCTK_THORNSTRING,
               "   There are %d %d-dimensional grid arrays in %d groups",
               num_array_vars.AT(d), d, num_array_groups.AT(d));
  }
  CCTK_VInfo(CCTK_THORNSTRING,
             "   (The number of variables counts all time levels)");
}

operator_type get_transport_operator(cGH const *const cctkGH, int const group,
                                     cGroup const &gdata) {
  assert(group >= 0 and group < CCTK_NumGroups());

  if (gdata.grouptype != CCTK_GF) {
    // Ignore everything but true grid functions
    return op_sync; // was: op_copy -- why?
  }

  bool const can_transfer = can_transfer_variable_type(cctkGH, group, gdata);

  // Catch a common error (using the tag "prolongate" instead of
  // "prolongation")
  {
    int const prolong_length =
        Util_TableGetString(gdata.tagstable, 0, NULL, "Prolongate");
    if (prolong_length >= 0) {
      char *const groupname = CCTK_GroupName(group);
      CCTK_VError(__LINE__, __FILE__, CCTK_THORNSTRING,
                  "Group \"%s\" contains the illegal tag \"Prolongate\".  (Did "
                  "you mean \"Prolongation\" instead?)",
                  groupname);
      free(groupname);
    } else if (prolong_length == UTIL_ERROR_TABLE_NO_SUCH_KEY) {
      // good -- do nothing
    } else {
      assert(0);
    }
  }

  // Get prolongation method
  char prolong_string[1000];
  bool have_prolong_string = false;
  {
    int const prolong_length = Util_TableGetString(
        gdata.tagstable, sizeof prolong_string, prolong_string, "Prolongation");
    if (prolong_length >= 0) {
      have_prolong_string = true;
    } else if (prolong_length == UTIL_ERROR_TABLE_NO_SUCH_KEY) {
      // do nothing
    } else {
      assert(0);
    }
  }

  // Get prolongation parameter name
  char prolong_param_string[1000];
  bool have_prolong_param_string = false;
  {
    int const prolong_param_length =
        Util_TableGetString(gdata.tagstable, sizeof prolong_param_string,
                            prolong_param_string, "ProlongationParameter");
    if (prolong_param_length >= 0) {
      have_prolong_param_string = true;
    } else if (prolong_param_length == UTIL_ERROR_TABLE_NO_SUCH_KEY) {
      // do nothing
    } else {
      assert(0);
    }
  }

  // Complain if both are given
  if (have_prolong_string and have_prolong_param_string) {
    char *const groupname = CCTK_GroupName(group);
    CCTK_VError(__LINE__, __FILE__, CCTK_THORNSTRING,
                "Group \"%s\" has both the tags \"Prolongation\" and "
                "\"ProlongationParameter\".  This is not possible.",
                groupname);
    free(groupname);
  }

  // Map the parameter name
  if (have_prolong_param_string) {
    char *thorn;
    char *name;
    int const ierr = CCTK_DecomposeName(prolong_param_string, &thorn, &name);
    if (ierr < 0) {
      char *const groupname = CCTK_GroupName(group);
      CCTK_VError(__LINE__, __FILE__, CCTK_THORNSTRING,
                  "Group \"%s\" has the \"ProlongationParameter\" tag \"%s\".  "
                  "This is not a valid parameter name.",
                  groupname, prolong_param_string);
      free(groupname);
    }
    int type;
    char const *const *const value = (static_cast<char const *const *>(
        CCTK_ParameterGet(name, thorn, &type)));
    if (not value or not*value) {
      char *const groupname = CCTK_GroupName(group);
      CCTK_VError(__LINE__, __FILE__, CCTK_THORNSTRING,
                  "Group \"%s\" has the \"ProlongationParameter\" tag \"%s\".  "
                  "This parameter does not exist.",
                  groupname, prolong_param_string);
      free(groupname);
    }
    if (type != PARAMETER_KEYWORD and type != PARAMETER_STRING) {
      char *const groupname = CCTK_GroupName(group);
      CCTK_VError(__LINE__, __FILE__, CCTK_THORNSTRING,
                  "Group \"%s\" has the \"ProlongationParameter\" tag \"%s\".  "
                  "This parameter has the wrong type; it must be either "
                  "KEYWORD or STRING.",
                  groupname, prolong_param_string);
      free(groupname);
    }
    free(thorn);
    free(name);
    assert(strlen(*value) < sizeof prolong_string);
    strcpy(prolong_string, *value);
    have_prolong_string = true;
  }

  // Select a default, if necessary
  if (not have_prolong_string) {
    if (can_transfer) {
// Use the default
#if 0
        if (gdata.numtimelevels == 1) {
          // Only one time level:
          char * const groupname = CCTK_GroupName (group);
          CCTK_VWarn (2, __LINE__, __FILE__, CCTK_THORNSTRING,
                      "Group \"%s\" has only one time level; therefore it will not be prolongated or restricted.",
                      groupname);
          free (groupname);
          return op_sync;
        } else {
          // Several time levels: use the default
          return op_Lagrange;
        }
#endif
      return op_Lagrange;
    } else {
      if (gdata.grouptype == CCTK_GF) {
        char *const groupname = CCTK_GroupName(group);
        CCTK_VWarn(2, __LINE__, __FILE__, CCTK_THORNSTRING,
                   "Group \"%s\" has the variable type \"%s\" which cannot be "
                   "prolongated or restricted.",
                   groupname, CCTK_VarTypeName(gdata.vartype));
        free(groupname);
        return op_sync;
      } else {
        return op_error;
      }
    }
  }

  // Select the prolongation method
  assert(have_prolong_string);
  if (CCTK_Equals(prolong_string, "none")) {
    // This would surprise too many people
    // return op_none;
    return op_sync;
  } else if (CCTK_Equals(prolong_string, "sync")) {
    return op_sync;
  } else if (CCTK_Equals(prolong_string, "restrict")) {
    return op_restrict;
  } else if (CCTK_Equals(prolong_string, "copy")) {
    return op_copy;
  } else if (CCTK_Equals(prolong_string, "Lagrange")) {
    return op_Lagrange;
  } else if (CCTK_Equals(prolong_string, "ENO")) {
    return op_ENO;
  } else if (CCTK_Equals(prolong_string, "WENO")) {
    return op_WENO;
  } else if (CCTK_Equals(prolong_string, "TVD")) {
    return op_TVD;
  } else if (CCTK_Equals(prolong_string, "Lagrange_monotone")) {
    return op_Lagrange_monotone;
  } else if (CCTK_Equals(prolong_string, "STAGGER011")) {
    return op_STAGGER011;
  } else if (CCTK_Equals(prolong_string, "STAGGER101")) {
    return op_STAGGER101;
  } else if (CCTK_Equals(prolong_string, "STAGGER110")) {
    return op_STAGGER110;
  } else if (CCTK_Equals(prolong_string, "STAGGER111")) {
    return op_STAGGER111;
  } else {
    char *const groupname = CCTK_GroupName(group);
    CCTK_VError(__LINE__, __FILE__, CCTK_THORNSTRING,
                "Group \"%s\" has the unknown prolongation method \"%s\".",
                groupname, prolong_string);
    free(groupname);
    return op_error;
  }
  return op_error;
}

bool can_transfer_variable_type(cGH const *const cctkGH, int const group,
                                cGroup const &gdata) {
// Find out which types correspond to the default types
#if CCTK_INTEGER_PRECISION_1
#define CCTK_DEFAULT_INTEGER_TYPE CCTK_VARIABLE_INT1
#elif CCTK_INTEGER_PRECISION_2
#define CCTK_DEFAULT_INTEGER_TYPE CCTK_VARIABLE_INT2
#elif CCTK_INTEGER_PRECISION_4
#define CCTK_DEFAULT_INTEGER_TYPE CCTK_VARIABLE_INT4
#elif CCTK_INTEGER_PRECISION_8
#define CCTK_DEFAULT_INTEGER_TYPE CCTK_VARIABLE_INT8
#elif CCTK_INTEGER_PRECISION_16
#define CCTK_DEFAULT_INTEGER_TYPE CCTK_VARIABLE_INT16
#else
#error "Unsupported default integer type"
#endif

#if CCTK_REAL_PRECISION_4
#define CCTK_DEFAULT_REAL_TYPE CCTK_VARIABLE_REAL4
#define CCTK_DEFAULT_COMPLEX_TYPE CCTK_VARIABLE_COMPLEX8
#elif CCTK_REAL_PRECISION_8
#define CCTK_DEFAULT_REAL_TYPE CCTK_VARIABLE_REAL8
#define CCTK_DEFAULT_COMPLEX_TYPE CCTK_VARIABLE_COMPLEX16
#elif CCTK_REAL_PRECISION_16
#define CCTK_DEFAULT_REAL_TYPE CCTK_VARIABLE_REAL16
#define CCTK_DEFAULT_COMPLEX_TYPE CCTK_VARIABLE_COMPLEX32
#else
#error "Unsupported default real type"
#endif

  int const type0 = gdata.vartype;
  int type1;

  switch (type0) {
  case CCTK_VARIABLE_INT:
    type1 = CCTK_DEFAULT_INTEGER_TYPE;
    break;
  case CCTK_VARIABLE_REAL:
    type1 = CCTK_DEFAULT_REAL_TYPE;
    break;
  case CCTK_VARIABLE_COMPLEX:
    type1 = CCTK_DEFAULT_COMPLEX_TYPE;
    break;
  default:
    type1 = type0;
  }
  switch (type1) {

#ifdef HAVE_CCTK_REAL8
  case CCTK_VARIABLE_REAL8:
    // This type is supported.
    return true;
#endif

#ifdef HAVE_CCTK_REAL4
  case CCTK_VARIABLE_REAL4:
#endif
#ifdef HAVE_CCTK_REAL16
  case CCTK_VARIABLE_REAL16:
#endif
#ifdef HAVE_CCTK_COMPLEX8
  case CCTK_VARIABLE_COMPLEX8:
#endif
#ifdef HAVE_CCTK_COMPLEX16
  case CCTK_VARIABLE_COMPLEX16:
#endif
#ifdef HAVE_CCTK_COMPLEX32
  case CCTK_VARIABLE_COMPLEX32:
#endif
    // This type is not supported, but could be.
    return false;

  case CCTK_VARIABLE_BYTE:
#ifdef HAVE_CCTK_INT1
  case CCTK_VARIABLE_INT1:
#endif
#ifdef HAVE_CCTK_INT2
  case CCTK_VARIABLE_INT2:
#endif
#ifdef HAVE_CCTK_INT4
  case CCTK_VARIABLE_INT4:
#endif
#ifdef HAVE_CCTK_INT8
  case CCTK_VARIABLE_INT8:
#endif
#ifdef HAVE_CCTK_INT16
  case CCTK_VARIABLE_INT16:
#endif
    // This type is not supported, and cannot be.
    return false;

  default: {
    CCTK_VError(
        __LINE__, __FILE__, CCTK_THORNSTRING,
        "Internal error: encountered variable type %d (%s) for group %d (%s)",
        type1, CCTK_VarTypeName(type1), group, CCTK_GroupName(group));
  }
  }

  // not reached
  abort();
  return false;
}

//////////////////////////////////////////////////////////////////////////////
// Parameter and consistency checking ////////////////////////////////////////
//////////////////////////////////////////////////////////////////////////////

// Ensure that CartGrid3D::type = "coordbase"
void ensure_CartGrid3D_type() {
  if (CCTK_IsThornActive("CartGrid3D")) {
    int type;
    void const *const ptr = CCTK_ParameterGet("type", "CartGrid3D", &type);
    assert(ptr != 0);
    assert(type == PARAMETER_KEYWORD);
    char const *const coordtype = *static_cast<char const *const *>(ptr);
    if (not CCTK_EQUALS(coordtype, "coordbase")) {
      CCTK_ERROR("When Carpet::domain_from_coordbase = yes, and when thorn "
                 "CartGrid3D is active, then you also have to set "
                 "CartGrid3D::type = \"coordbase\"");
    }
  }
}

#if 0
  // UNUSED:
  // Ensure that CartGrid3D doesn't apply symmetries
  void
  ensure_CartGrid3D_domain ()
  {
    if (CCTK_IsThornActive ("CartGrid3D")) {
      int type;
      void const * ptr;
      
      ptr = CCTK_ParameterGet ("domain", "CartGrid3D", & type);
      assert (ptr != 0);
      assert (type == PARAMETER_KEYWORD);
      char const * const domain
        = * static_cast<char const * const *> (ptr);
      if (not CCTK_EQUALS (domain, "full")) {
        CCTK_ERROR ("When Carpet::domain_from_coordbase = no, and when Carpet::max_refinement_levels > 1, then thorn CartGrid3D cannot provide symmetry boundaries");
      }
    }
  }
#endif

// Ensure that CartGrid3D::avoid_origin = no
void ensure_CartGrid3D_avoid_origin() {
  if (CCTK_IsThornActive("CartGrid3D")) {
    int type;
    void const *ptr;

    ptr = CCTK_ParameterGet("no_origin", "CartGrid3D", &type);
    assert(ptr != 0);
    assert(type == PARAMETER_BOOLEAN);
    CCTK_INT const no_origin = *static_cast<CCTK_INT const *>(ptr);

    ptr = CCTK_ParameterGet("no_originx", "CartGrid3D", &type);
    assert(ptr != 0);
    assert(type == PARAMETER_BOOLEAN);
    CCTK_INT const no_originx = *static_cast<CCTK_INT const *>(ptr);

    ptr = CCTK_ParameterGet("no_originy", "CartGrid3D", &type);
    assert(ptr != 0);
    assert(type == PARAMETER_BOOLEAN);
    CCTK_INT const no_originy = *static_cast<CCTK_INT const *>(ptr);

    ptr = CCTK_ParameterGet("no_originz", "CartGrid3D", &type);
    assert(ptr != 0);
    assert(type == PARAMETER_BOOLEAN);
    CCTK_INT const no_originz = *static_cast<CCTK_INT const *>(ptr);

    ptr = CCTK_ParameterGet("avoid_origin", "CartGrid3D", &type);
    assert(ptr != 0);
    assert(type == PARAMETER_BOOLEAN);
    CCTK_INT const avoid_origin = *static_cast<CCTK_INT const *>(ptr);

    ptr = CCTK_ParameterGet("avoid_originx", "CartGrid3D", &type);
    assert(ptr != 0);
    assert(type == PARAMETER_BOOLEAN);
    CCTK_INT const avoid_originx = *static_cast<CCTK_INT const *>(ptr);

    ptr = CCTK_ParameterGet("avoid_originy", "CartGrid3D", &type);
    assert(ptr != 0);
    assert(type == PARAMETER_BOOLEAN);
    CCTK_INT const avoid_originy = *static_cast<CCTK_INT const *>(ptr);

    ptr = CCTK_ParameterGet("avoid_originz", "CartGrid3D", &type);
    assert(ptr != 0);
    assert(type == PARAMETER_BOOLEAN);
    CCTK_INT const avoid_originz = *static_cast<CCTK_INT const *>(ptr);

    ptr = CCTK_ParameterGet("domain", "CartGrid3D", &type);
    assert(ptr != 0);
    assert(type == PARAMETER_KEYWORD);
    char const *const domain = *static_cast<char const *const *>(ptr);

    bvect stag;
    stag[0] = no_origin and no_originx and avoid_origin and avoid_originx;
    stag[1] = no_origin and no_originy and avoid_origin and avoid_originy;
    stag[2] = no_origin and no_originz and avoid_origin and avoid_originz;

    // TODO: Check only if there is actually a symmetry boundary
    // TODO: Take cell centring into account
    if (not CCTK_EQUALS(domain, "full") and any(stag)) {
      CCTK_ERROR("When Carpet::domain_from_coordbase = no, when "
                 "Carpet::max_refinement_levels > 1, and when thorn CartGrid3D "
                 "provides symmetry boundaries, then you have to set "
                 "CartGrid3D::avoid_origin = no");
    }
  }
}

// Ensure that ReflectionSymmetry::avoid_origin = no (if vertex
// centred)
void ensure_ReflectionSymmetry_avoid_origin(centering const refcentering) {
  if (CCTK_IsThornActive("ReflectionSymmetry")) {
    int type;
    void const *ptr;

    ptr = CCTK_ParameterGet("avoid_origin_x", "ReflectionSymmetry", &type);
    assert(ptr != 0);
    assert(type == PARAMETER_BOOLEAN);
    CCTK_INT const avoid_origin_x = *static_cast<CCTK_INT const *>(ptr);

    ptr = CCTK_ParameterGet("avoid_origin_y", "ReflectionSymmetry", &type);
    assert(ptr != 0);
    assert(type == PARAMETER_BOOLEAN);
    CCTK_INT const avoid_origin_y = *static_cast<CCTK_INT const *>(ptr);

    ptr = CCTK_ParameterGet("avoid_origin_z", "ReflectionSymmetry", &type);
    assert(ptr != 0);
    assert(type == PARAMETER_BOOLEAN);
    CCTK_INT const avoid_origin_z = *static_cast<CCTK_INT const *>(ptr);

    ptr = CCTK_ParameterGet("reflection_x", "ReflectionSymmetry", &type);
    assert(ptr != 0);
    assert(type == PARAMETER_BOOLEAN);
    CCTK_INT const reflection_x = *static_cast<CCTK_INT const *>(ptr);

    ptr = CCTK_ParameterGet("reflection_y", "ReflectionSymmetry", &type);
    assert(ptr != 0);
    assert(type == PARAMETER_BOOLEAN);
    CCTK_INT const reflection_y = *static_cast<CCTK_INT const *>(ptr);

    ptr = CCTK_ParameterGet("reflection_z", "ReflectionSymmetry", &type);
    assert(ptr != 0);
    assert(type == PARAMETER_BOOLEAN);
    CCTK_INT const reflection_z = *static_cast<CCTK_INT const *>(ptr);

    if (refcentering == vertex_centered) {
      if ((reflection_x and avoid_origin_x) or
          (reflection_y and avoid_origin_y) or
          (reflection_z and avoid_origin_z)) {
        CCTK_ERROR("When Carpet::max_refinement_levels > 1, and when "
                   "ReflectionSymmetry::symmetry_[xyz] = yes, then you also "
                   "have to set ReflectionSymmetry::avoid_origin_[xyz] = no");
      }
    } else if (refcentering == cell_centered) {
      if ((reflection_x and not avoid_origin_x) or
          (reflection_y and not avoid_origin_y) or
          (reflection_z and not avoid_origin_z)) {
        CCTK_ERROR("When Carpet::max_refinement_levels > 1, and when "
                   "ReflectionSymmetry::symmetry_[xyz] = yes, then you have to "
                   "set ReflectionSymmetry::avoid_origin_[xyz] = yes");
      }
    } else {
      assert(0);
    }
  }
}

void ensure_ghostzones(int const m, vector<i2vect> const &ghosts) {
  DECLARE_CCTK_PARAMETERS;

  int const rls = vhh.AT(m)->reffacts.size();
  for (int rl = 0; rl < rls; ++rl) {
    int const my_prolongation_order_space =
        vdd.AT(m)->prolongation_orders_space.AT(rl);
    int const prolongation_stencil_size =
        vdd.AT(m)->prolongation_stencil_size(rl);
    int const min_nghosts =
        ((prolongation_stencil_size + refinement_factor - 1) /
         (refinement_factor - 1));
    int const min_nghosts_restrict = restriction_order_space / 2;
    if (any(any(ghosts.AT(rl) < i2vect(min_nghosts)))) {
      CCTK_VError(__LINE__, __FILE__, CCTK_THORNSTRING,
                  "There are not enough ghost zones for the desired spatial "
                  "prolongation order on map %d, refinement level %d.  With a "
                  "spatial prolongation order of %d, you need at least %d "
                  "ghost zones.",
                  m, rl, my_prolongation_order_space, min_nghosts);
    }
    if (use_higher_order_restriction and
        any(any(ghosts.AT(rl) < i2vect(min_nghosts_restrict)))) {
      CCTK_VError(__LINE__, __FILE__, CCTK_THORNSTRING,
                  "There are not enough ghost zones for the desired "
                  "restriction order on map %d, refinement level %d.  With a "
                  "restriction order of %d, you need at least %d ghost zones.",
                  m, rl, int(restriction_order_space), min_nghosts_restrict);
    }
  }
}

void ensure_group_options(int const group, cGroup const &gdata) {
#ifdef CCTK_HAVE_COMPACT_GROUPS
  if (gdata.compact) {
    char *const groupname = CCTK_GroupName(group);
    CCTK_VError(
        __LINE__, __FILE__, CCTK_THORNSTRING,
        "The group \"%s\" has COMPACT=1.  Compact groups are not yet supported",
        groupname);
    free(groupname);
  }
#endif

#ifdef CCTK_HAVE_CONTIGUOUS_GROUPS
  if (gdata.contiguous) {
    char *const groupname = CCTK_GroupName(group);
    CCTK_VError(__LINE__, __FILE__, CCTK_THORNSTRING,
                "The group \"%s\" has CONTIGUOUS=1.  Contiguous groups are not "
                "yet supported",
                groupname);
    free(groupname);
  }
#endif
}

} // namespace Carpet

```

Timing.cc:
```
#include <algorithm>
#include <cassert>
#include <cmath>
#include <cstdlib>

#include <cctk.h>
#include <cctk_Arguments.h>
#include <cctk_Parameters.h>

// IRIX wants this before <time.h>
#if HAVE_SYS_TYPES_H
#include <sys/types.h>
#endif

#if TIME_WITH_SYS_TIME
#include <sys/time.h>
#include <time.h>
#else
#if HAVE_SYS_TIME_H
#include <sys/time.h>
#elif HAVE_TIME_H
#include <time.h>
#endif
#endif

#if HAVE_UNISTD_H
#include <unistd.h>
#endif

#include <defs.hh>
#include <dist.hh>

#include <carpet.hh>

namespace Carpet {

using namespace std;

// Small number to avoid division by zero
CCTK_REAL const eps = 1.0e-15;

// Return the current wall time
static CCTK_REAL get_walltime() {
#ifdef HAVE_TIME_GETTIMEOFDAY
  // get the current time
  struct timeval tv;
  gettimeofday(&tv, 0);
  return tv.tv_sec + tv.tv_usec / CCTK_REAL(1.0e6);
#else
  return CCTK_REAL(0.0);
#endif
}

// Calculate the number of updates for the current level
static void current_level_updates(cGH const *const cctkGH,
                                  CCTK_REAL &local_grid_updates,
                                  CCTK_REAL &global_grid_updates,
                                  CCTK_REAL &local_interior_updates,
                                  CCTK_REAL &global_interior_updates) {
  DECLARE_CCTK_PARAMETERS;

  // Count the weighted number of grid points
  // (int is not good enough for this calculation)
  CCTK_REAL local_num_grid_points = 0;
  CCTK_REAL global_num_grid_points = 0;
  CCTK_REAL local_num_interior_points = 0;
  CCTK_REAL global_num_interior_points = 0;
  for (int m = 0; m < maps; ++m) {
    assert(reflevel >= 0);
    int const rl = reflevel;
    for (int c = 0; c < vhh.AT(m)->components(rl); ++c) {
      assert(mglevel >= 0);
      int const ml = mglevel;

      // Regions
      dh::light_dboxes const &light_box =
          vdd.AT(m)->light_boxes.AT(ml).AT(rl).AT(c);
      ibbox const &interior = light_box.interior; // with outer boundary
      ibbox const &owned = light_box.owned;       // without outer boundary

      // Count the grid points
      CCTK_REAL const interior_size = interior.size();
      CCTK_REAL const owned_size = owned.size();

      if (vhh.AT(m)->is_local(rl, c)) {
        local_num_grid_points += interior_size;
        local_num_interior_points += owned_size;
      }
      global_num_grid_points += interior_size;
      global_num_interior_points += owned_size;

    } // for c
  }   // for m

  // Take number of RHS evaluations per time step into account
  static int int_steps = -1;
  if (int_steps == -1) {
    if (num_integrator_substeps != -1) {
      // if the user parameter is set, use it
      int_steps = num_integrator_substeps;
    } else if (CCTK_IsFunctionAliased("MoLNumIntegratorSubsteps")) {
      // if there is an aliased function, use it
      int_steps = MoLNumIntegratorSubsteps();
    } else {
      // use a sensible default, even if it is wrong -- it is better
      // to have timing information which is wrong by a constant
      // factor than to abort the code
      int_steps = 1;
    }
  }
  local_grid_updates = local_num_grid_points * int_steps;
  global_grid_updates = global_num_grid_points * int_steps;
  local_interior_updates = local_num_interior_points * int_steps;
  global_interior_updates = global_num_interior_points * int_steps;
}

// Time at which the simulation started
CCTK_REAL startup_walltime; // in seconds

// Time at which the evolution started
bool in_evolution = false;
CCTK_REAL initial_walltime; // in seconds
CCTK_REAL initial_phystime;

// Last starting time
enum timing_state_t { state_computing, state_communicating, state_io };
timing_state_t timing_state = state_computing;
CCTK_REAL time_start;

// Last starting time for this level
int timing_level = -1;
CCTK_REAL time_level_start;

// Initialise the timing variables (to be called before basegrid)
void InitTimingStats(cGH const *const cctkGH) {
  DECLARE_CCTK_ARGUMENTS;
  DECLARE_CCTK_PARAMETERS;

  startup_walltime = get_walltime();

  *physical_time_per_hour = 0.0;
  *current_physical_time_per_hour = 0.0;

  *time_total = 0.0;
  *time_evolution = 0.0;
  *time_computing = 0.0;
  *time_communicating = 0.0;
  *time_io = 0.0;

  *evolution_steps_count = 0.0;

  *local_grid_points_per_second = 0.0;
  *total_grid_points_per_second = 0.0;
  *local_grid_point_updates_count = 0.0;
  *total_grid_point_updates_count = 0.0;
  *local_interior_points_per_second = 0.0;
  *total_interior_points_per_second = 0.0;
  *local_interior_point_updates_count = 0.0;
  *total_interior_point_updates_count = 0.0;

  *io_per_second = 0.0;
  *io_bytes_per_second = 0.0;
  *io_bytes_ascii_per_second = 0.0;
  *io_bytes_binary_per_second = 0.0;
  *io_count = 0.0;
  *io_bytes_count = 0.0;
  *io_bytes_ascii_count = 0.0;
  *io_bytes_binary_count = 0.0;

  *comm_per_second = 0.0;
  *comm_bytes_per_second = 0.0;
  *comm_count = 0.0;
  *comm_bytes_count = 0.0;

  *time_levels = 0.0;

  *grid_points_per_second = 0.0;
  *grid_point_updates_count = 0.0;

  for (int rl = 0; rl < max_refinement_levels; ++rl) {
    time_level[rl] = 0.0;
    time_level_count[rl] = 0.0;
  }
}

// Begin timing (to be called after initialisation, just before the
// main evolution begins)
void BeginTimingEvolution(cGH const *const cctkGH) {
  DECLARE_CCTK_ARGUMENTS;

  in_evolution = true;
  initial_walltime = get_walltime();
  initial_phystime = cctkGH->cctk_time;
}

// Take a step on the current refinement and multigrid level (to be
// called when EVOL is scheduled)
void StepTimingEvolution(cGH const *const cctkGH) {
  DECLARE_CCTK_ARGUMENTS;

  assert(in_evolution);
  assert(timing_state == state_computing);

  CCTK_REAL local_grid_updates, global_grid_updates;
  CCTK_REAL local_interior_updates, global_interior_updates;
  current_level_updates(cctkGH, local_grid_updates, global_grid_updates,
                        local_interior_updates, global_interior_updates);

  ++*evolution_steps_count;

  *local_grid_point_updates_count += local_grid_updates;
  *total_grid_point_updates_count += global_grid_updates;
  *local_interior_point_updates_count += local_interior_updates;
  *total_interior_point_updates_count += global_interior_updates;

  *grid_point_updates_count = *local_grid_point_updates_count;
  *interior_point_updates_count = *local_interior_point_updates_count;
}

// Count time spent on individual levels (to be called from Carpet's
// initialisation, evolution, and shutdown drivers)
void BeginTimingLevel(cGH const *const cctkGH) {
  assert(reflevel != -1);
  assert(timing_level == -1);
  timing_level = reflevel;
  time_level_start = get_walltime();
}

void EndTimingLevel(cGH const *const cctkGH) {
  DECLARE_CCTK_ARGUMENTS;

  assert(reflevel != -1);
  assert(timing_level == reflevel);
  timing_level = -1;
  CCTK_REAL const time_level_end = get_walltime();

  time_level[reflevel] += time_level_end - time_level_start;
  ++time_level_count[reflevel];
  *time_levels += time_level_end - time_level_start;
}

// Count some I/O (to be called from the I/O routine)
void BeginTimingIO(cGH const *const cctkGH) {
  assert(timing_state == state_computing);
  timing_state = state_io;
  time_start = get_walltime();
}

void EndTimingIO(cGH const *const cctkGH, CCTK_REAL const files,
                 CCTK_REAL const bytes, bool const is_binary) {
  DECLARE_CCTK_ARGUMENTS;

  assert(timing_state == state_io);
  timing_state = state_computing;
  CCTK_REAL const time_end = get_walltime();

  *time_io += time_end - time_start;

  *io_count += files;
  *io_bytes_count += bytes;
  *(is_binary ? io_bytes_binary_count : io_bytes_ascii_count) += bytes;
}

// Count some communication (to be called from the communication routine)
void BeginTimingCommunication(cGH const *const cctkGH) {
  assert(timing_state == state_computing);
  timing_state = state_communicating;
  time_start = get_walltime();
}

void EndTimingCommunication(cGH const *const cctkGH, CCTK_REAL const messages,
                            CCTK_REAL const bytes) {
  DECLARE_CCTK_ARGUMENTS;

  assert(timing_state == state_communicating);
  timing_state = state_computing;
  CCTK_REAL const time_end = get_walltime();

  *time_communicating += time_end - time_start;

  *comm_count += messages;
  *comm_bytes_count += bytes;
}

static void UpdateTimes(cGH const *const cctkGH) {
  DECLARE_CCTK_ARGUMENTS;

  assert(timing_state == state_computing);

  // Measure the elapsed time
  double const walltime = get_walltime();
  *time_total = walltime - startup_walltime;
  *time_evolution = in_evolution ? walltime - initial_walltime : 0.0;

  *time_computing = *time_total - (*time_communicating + *time_io);
}

static void UpdateUpdatesPerSecond(cGH const *const cctkGH) {
  DECLARE_CCTK_ARGUMENTS;

  // Calculate updates per second
  *local_grid_points_per_second =
      *local_grid_point_updates_count / max(*time_computing, eps);
  *total_grid_points_per_second =
      *total_grid_point_updates_count / max(*time_computing, eps);
  *local_interior_points_per_second =
      *local_interior_point_updates_count / max(*time_computing, eps);
  *total_interior_points_per_second =
      *total_interior_point_updates_count / max(*time_computing, eps);

  *grid_points_per_second = *local_grid_points_per_second;
  *interior_points_per_second = *local_interior_points_per_second;
}

static void UpdateIOStats(cGH const *const cctkGH) {
  DECLARE_CCTK_ARGUMENTS;

  *io_per_second = *io_count / max(*time_io, eps);
  *io_bytes_per_second = *io_bytes_count / max(*time_io, eps);
  *io_bytes_ascii_per_second = *io_bytes_ascii_count / max(*time_io, eps);
  *io_bytes_binary_per_second = *io_bytes_binary_count / max(*time_io, eps);
}

static void UpdateCommunicationStats(cGH const *const cctkGH) {
  DECLARE_CCTK_ARGUMENTS;

  *comm_per_second = *comm_count / max(*time_communicating, eps);
  *comm_bytes_per_second = *comm_bytes_count / max(*time_communicating, eps);
}

static void UpdatePhysicalTimePerHour(cGH const *const cctkGH) {
  DECLARE_CCTK_ARGUMENTS;
  DECLARE_CCTK_PARAMETERS;

  if (not in_evolution) {
    *physical_time_per_hour = 0.0;
    *current_physical_time_per_hour = 0.0;
    return;
  }

  static int last_iteration = -1;
  static size_t num_samples = 0;
  // static CCTK_REAL last_physical_time;
  static CCTK_REAL last_time_evolution;
  assert(cctk_iteration > last_iteration); // expect progress

  // Calculate elapsed physical time
  CCTK_REAL const physical_time = cctkGH->cctk_time - initial_phystime;

  // Calculate average physical time per hour
  *physical_time_per_hour = 3600.0 * physical_time / max(*time_evolution, eps);

  // Calculate current physical time per hour as moving average
  if (last_iteration < 0) {
    // No past data are available
    *current_physical_time_per_hour = *physical_time_per_hour;
  } else if (num_samples < 3 or
             *time_evolution < 0.01 * timing_average_window_minutes * 60.0) {
    // Less than three previous samples are available, or less than
    // one percent of a window of past data are available
    *current_physical_time_per_hour = *physical_time_per_hour;
  } else {
    CCTK_REAL const window =
        min(*time_evolution, (CCTK_REAL)(timing_average_window_minutes * 60.0));
    CCTK_REAL const alpha =
        exp(-(*time_evolution - last_time_evolution) / window);
    *current_physical_time_per_hour = (1.0 - alpha) * *physical_time_per_hour +
                                      (alpha) * *current_physical_time_per_hour;
  }

  // Remember last iteration
  last_iteration = cctk_iteration;
  ++num_samples;
  // last_physical_time  = physical_time;
  last_time_evolution = *time_evolution;
}

static void UpdateMemoryStats(cGH const *const cctkGH) {
  DECLARE_CCTK_ARGUMENTS;

  // Various metadata, excluding the small change
  *metadata = 0.0 +
              // memoryof(main_timer_tree) +
              // memoryof(mode_timer_tree) +
              memoryof(origin_space) + memoryof(delta_space) +
              // memoryof(domainspecs) +
              memoryof(vhh) + memoryof(vdd) + memoryof(level_regridding_epochs);
  // memoryof(groupdata) +
  // memoryof(arrdata);

  // TODO: Add TimerNode.root(?), TimerSet.timers

  // Grid structure
  *grid_structure = 0.0;
  // Storage for grid arrays
  *grid_arrays = 0.0;
  // Storage for grid functions
  *grid_functions = 0.0;

  for (size_t m = 0; m < vhh.size(); ++m) {
    *grid_structure += memoryof(*vhh.AT(m));
  }
  for (size_t m = 0; m < vdd.size(); ++m) {
    *grid_structure += memoryof(*vdd.AT(m));
  }
  *grid_structure += memoryof(*tt);

  const int myproc = CCTK_MyProc(0);
  for (size_t gi = 0; gi < arrdata.size(); ++gi) {
    if (CCTK_GroupTypeI(gi) == CCTK_GF) {
      for (size_t m = 0; m < arrdata.AT(gi).size(); ++m) {
        const gh &hh = *vhh.AT(m);
        for (size_t vi = 0; vi < arrdata.AT(gi).AT(m).data.size(); ++vi) {
          const ggf *f = arrdata.AT(gi).AT(m).data.AT(vi);
          if (f) {
            *grid_structure += memoryof(*f);
            for (int ml = 0; ml < hh.mglevels(); ++ml) {
              for (int rl = 0; rl < hh.reflevels(); ++rl) {
                const int tls = f->timelevels(ml, rl);
                for (int lc = 0; lc < hh.local_components(rl); ++lc) {
                  for (int tl = 0; tl < tls; ++tl) {
                    const gdata *d = f->data_pointer(tl, rl, lc, ml);
                    *grid_structure += memoryof(*d);
                    // TODO: add memoryof(d->_memory);
                    if (d->has_storage()) {
                      assert(d->proc() == myproc);
                      *grid_functions += d->size() * d->elementsize();
                    }
                  }
                }
              }
            }
          }
        }
      }
    } else { // not CCTK_GF
      const int m = 0;
      *grid_structure += 0.0 + memoryof(*arrdata.AT(gi).AT(m).hh) +
                         memoryof(*arrdata.AT(gi).AT(m).dd) +
                         memoryof(*arrdata.AT(gi).AT(m).tt);
      for (size_t vi = 0; vi < arrdata.AT(gi).AT(m).data.size(); ++vi) {
        const ggf *f = arrdata.AT(gi).AT(m).data.AT(vi);
        if (f) {
          *grid_structure += memoryof(*f);
          const int ml = 0;
          const int rl = 0;
          const int tls = f->timelevels(ml, rl);
          const int lc = 0;
          for (int tl = 0; tl < tls; ++tl) {
            const gdata *d = f->data_pointer(tl, rl, lc, ml);
            *grid_arrays += memoryof(*d);
            // TODO: add memoryof(d->_memory);
            if (d->has_storage()) {
              assert(d->proc() == myproc);
              *grid_functions += d->size() * d->elementsize();
            }
          }
        }
      }
    }
  }
}

// Calculate timing statistics (to be called before output)
void UpdateTimingStats(cGH const *const cctkGH) {
  UpdateTimes(cctkGH);
  UpdateUpdatesPerSecond(cctkGH);
  UpdateIOStats(cctkGH);
  UpdateCommunicationStats(cctkGH);
  UpdatePhysicalTimePerHour(cctkGH);
  UpdateMemoryStats(cctkGH);
}

static void PrintTimes(cGH const *const cctkGH) {
  DECLARE_CCTK_ARGUMENTS;

  CCTK_VInfo(CCTK_THORNSTRING, "Total run       time: %g h   (iteration %d)",
             double(*time_total / 3600.0), cctk_iteration);
  CCTK_VInfo(CCTK_THORNSTRING, "Total evolution time: %g h   (%d steps)",
             double(*time_evolution / 3600.0), int(*evolution_steps_count));
  CCTK_VInfo(CCTK_THORNSTRING,
             "(Comp, Comm, I/O) fractions = (%3.1f%%, %3.1f%%, %3.1f%%)",
             double(100.0 * *time_computing / max(*time_total, eps)),
             double(100.0 * *time_communicating / max(*time_total, eps)),
             double(100.0 * *time_io / max(*time_total, eps)));
}

static void PrintUpdatesPerSecond(cGH const *const cctkGH) {
  DECLARE_CCTK_ARGUMENTS;
  DECLARE_CCTK_PARAMETERS;

  CCTK_VInfo(CCTK_THORNSTRING,
             "This processor's grid point updates per second (local)    : %g",
             double(*local_grid_points_per_second));
  CCTK_VInfo(CCTK_THORNSTRING,
             "Overall grid point updates per second (total)             : %g",
             double(*total_grid_points_per_second));
  CCTK_VInfo(CCTK_THORNSTRING,
             "This processor's interior point updates per second (local): %g",
             double(*local_interior_points_per_second));
  CCTK_VInfo(CCTK_THORNSTRING,
             "Overall interior point updates per second (total)         : %g",
             double(*total_interior_points_per_second));

#if 0
    CCTK_REAL const updates_per_second_2 = ipow (updates_per_second, 2);
    
    struct {
      CCTK_REAL ups, ups2;
    } local, global;
    local.ups  = updates_per_second;
    local.ups2 = updates_per_second_2;
    MPI_Allreduce (& local, & global, 2,
                   dist::datatype (global.ups), MPI_SUM, dist::comm());
    
    int const count = dist::size();
    CCTK_REAL const avg = global.ups / count;
    CCTK_REAL const stddev = sqrt (fabs (global.ups2 - ipow (avg,2)) / count);
    
    CCTK_VInfo (CCTK_THORNSTRING,
                "Local updates per second:   %g",
                double (updates_per_second));
    CCTK_VInfo (CCTK_THORNSTRING,
                "Global updates per second:  %g", double (global.ups));
    
    if (verbose) {
      CCTK_VInfo (CCTK_THORNSTRING,
                  "Average updates per second: %g", double (avg));
      CCTK_VInfo (CCTK_THORNSTRING,
                  "Standard deviation:         %g", double (stddev));
    }
#endif
}

static void PrintIOStats(cGH const *const cctkGH) {
  DECLARE_CCTK_ARGUMENTS;

  CCTK_VInfo(CCTK_THORNSTRING, "I/O operations per second:     %g",
             double(*io_per_second));
  CCTK_VInfo(CCTK_THORNSTRING, "I/O bytes per second:          %g",
             double(*io_bytes_per_second));
  CCTK_VInfo(CCTK_THORNSTRING, "I/O bytes per second (ASCII):  %g",
             double(*io_bytes_ascii_per_second));
  CCTK_VInfo(CCTK_THORNSTRING, "I/O bytes per second (binary): %g",
             double(*io_bytes_binary_per_second));
}

static void PrintCommunicationStats(cGH const *const cctkGH) {
  DECLARE_CCTK_ARGUMENTS;

  CCTK_VInfo(CCTK_THORNSTRING, "Communication operations per second: %g",
             double(*comm_per_second));
  CCTK_VInfo(CCTK_THORNSTRING, "Communicated bytes per second:       %g",
             double(*comm_bytes_per_second));
}

static void PrintPhysicalTimePerHour(cGH const *const cctkGH) {
  DECLARE_CCTK_ARGUMENTS;

  CCTK_VInfo(CCTK_THORNSTRING, "Current physical time per hour: %g",
             double(*current_physical_time_per_hour));
  CCTK_VInfo(CCTK_THORNSTRING, "Average physical time per hour: %g",
             double(*physical_time_per_hour));
}

#define PRINTMEM(x) (memoryof(x) / 1.0e+6) << " MB"

static void PrintMemoryStats(cGH const *const cctkGH) {
  DECLARE_CCTK_ARGUMENTS;

  cout << eol << "Memory statistics:" << eol << "   Grid hierarchy:" << eol;
  for (int m = 0; m < Carpet::maps; ++m) {
    cout << "   gh[" << m << "]: " << PRINTMEM(*vhh.AT(m)) << eol << "   dh["
         << m << "]: " << PRINTMEM(*vdd.AT(m)) << eol;
  }
  cout << "   th: " << PRINTMEM(*tt) << eol;
#if 0
    for (int g = 0; g < (int)arrdata.size(); ++ g) {
      if (CCTK_GroupTypeI(g) != CCTK_GF) {
        char * const groupname = CCTK_GroupName(g);
        for (int m = 0; m < (int)arrdata.AT(g).size(); ++ m) {
          cout << "   Group " << groupname << ":" << eol
               << "   gh[" << m << "]: " << PRINTMEM(*arrdata.AT(g).AT(m).hh) << eol
               << "   dh[" << m << "]: " << PRINTMEM(*arrdata.AT(g).AT(m).dd) << eol
               << "   th[" << m << "]: " << PRINTMEM(*arrdata.AT(g).AT(m).tt) << eol;
          for (int v = 0; v < (int)arrdata.AT(g).AT(m).data.size(); ++ v) {
            char * const fullname = CCTK_FullName(CCTK_FirstVarIndexI(g)+v);
            cout << "   Variable " << fullname << ":" << eol
                 << "   ggf[" << m << "]: ";
            if (arrdata.AT(g).AT(m).data.AT(v)) {
              cout << PRINTMEM(*arrdata.AT(g).AT(m).data.AT(v));
            } else {
              cout << "<null>";
            }
            cout << eol;
            free (fullname);
          }
        }
        free (groupname);
      }
    }
#endif
  cout << endl;
}

// Calculate and print some timing statistics (to be called at any
// time)
void PrintTimingStats(cGH const *const cctkGH) {
  DECLARE_CCTK_PARAMETERS;

  if (print_timestats_every > 0 and
      cctkGH->cctk_iteration % print_timestats_every == 0) {
    PrintTimes(cctkGH);
    PrintUpdatesPerSecond(cctkGH);
    PrintIOStats(cctkGH);
    PrintCommunicationStats(cctkGH);
    PrintPhysicalTimePerHour(cctkGH);
    PrintMemoryStats(cctkGH);
  }
}

} // namespace Carpet

```

Storage.cc:
```
#include <algorithm>
#include <cassert>
#include <climits>
#include <cstdlib>

#include <cctk.h>
#include <cctk_Parameters.h>

#include <Requirements.hh>

#include <defs.hh>
#include <dh.hh>
#include <gf.hh>
#include <operators.hh>
#include <typeprops.hh>

#include <carpet.hh>

namespace Carpet {

using namespace std;

static int GroupStorageCrease(const cGH *cctkGH, int n_groups,
                              const int *groups, const int *tls, int *status,
                              const bool inc);

static void GroupStorageCheck(cGH const *const cctkGH, int const group,
                              int const ml, int const rl);

int GroupStorageCrease(const cGH *cctkGH, int n_groups, const int *groups,
                       const int *requested_tls, int *status, const bool inc) {
  DECLARE_CCTK_PARAMETERS;

  assert(cctkGH);
  assert(n_groups >= 0);
  assert(groups);
  assert(requested_tls);
  for (int n = 0; n < n_groups; ++n) {
    if (groups[n] < 0 or groups[n] >= CCTK_NumGroups()) {
      CCTK_VWarn(1, __LINE__, __FILE__, CCTK_THORNSTRING,
                 "Group index %d is illegal", groups[n]);
      return -1;
    }
    assert(groups[n] >= 0 and groups[n] < CCTK_NumGroups());
    assert(requested_tls[n] >= 0 or requested_tls[n] == -1);
  }

  // sanitize list of requested timelevels
  std::vector<int> tls(n_groups);
  for (int n = 0; n < n_groups; ++n) {
    int ntls = requested_tls[n];
    int const declared_tls = CCTK_DeclaredTimeLevelsGI(groups[n]);
    if (inc and declared_tls < 2 and ntls > declared_tls) {
      char *groupname = CCTK_GroupName(groups[n]);
      CCTK_VWarn(CCTK_WARN_ALERT, __LINE__, __FILE__, CCTK_THORNSTRING,
                 "Attempting to activate %d timelevels for group '%s' which "
                 "only has a single timelevel declared in interface.ccl. "
                 "Please declared at least 2 timelevels in interface.ccl to "
                 "allow more timelevels to be created at runtime.",
                 ntls, groupname);
      free(groupname);
      ntls = declared_tls;
    }
    if (ntls == -1) {
      ntls = declared_tls;
    }
    tls.at(n) = ntls;
  }

#ifdef REQUIREMENTS_HH
  vector<int> vgroups, vtimelevels;
  vgroups.reserve(n_groups);
  vtimelevels.reserve(n_groups);
  for (int i = 0; i < n_groups; ++i) {
    int const gi = groups[i];
    int const ml = 0;
    int const grouptype = CCTK_GroupTypeI(gi);
    bool const is_array = grouptype != CCTK_GF;
    bool const all_rl = is_meta_mode() or is_global_mode();
    int const min_rl = is_array ? 0 : all_rl ? 0 : reflevel;
    int const max_rl = is_array ? 1 : all_rl ? reflevels : reflevel + 1;
    int const ntls = tls[i];
    bool do_change = false;
    for (int rl = min_rl; rl < max_rl; ++rl) {
      int const ctls = groupdata.AT(gi).activetimelevels.AT(ml).AT(rl);
      do_change =
          do_change or (inc and ctls < ntls) or (not inc and ctls > ntls);
    }
    if (do_change) {
      vgroups.push_back(gi);
      vtimelevels.push_back(ntls);
    }
  }
  if (not vgroups.empty()) {
    Requirements::ChangeStorage(vgroups, vtimelevels, reflevel);
  }
#endif

  bool const can_do = is_meta_mode() or is_global_mode() or is_level_mode();
  bool const all_ml = is_meta_mode();
  int const min_ml = all_ml ? 0 : mglevel;
  int const max_ml = all_ml ? mglevels : mglevel + 1;

  int min_num_timelevels = INT_MAX;

  for (int n = 0; n < n_groups; ++n) {
    int const group = groups[n];

    if (storage_verbose) {
      char *const groupname = CCTK_GroupName(group);
      assert(groupname);
      Checkpoint("  %s: %screase to %d", groupname, inc ? "in" : "de", tls[n]);
      free(groupname);
    }

    cGroup gp;
    check(not CCTK_GroupData(group, &gp));

    bool const all_rl = is_meta_mode() or is_global_mode();
    bool const is_array = gp.grouptype != CCTK_GF;
    int const min_rl = is_array ? 0 : all_rl ? 0 : reflevel;
    int const max_rl = is_array ? 1 : all_rl ? reflevels : reflevel + 1;

    int const firstvarindex = CCTK_FirstVarIndexI(group);
    assert(gp.numvars == 0 or
           (firstvarindex >= 0 and firstvarindex < CCTK_NumVars()));

    // Check an assumption
    if (not gp.vectorgroup)
      assert(gp.vectorlength == 1);

    // Allocate the time levels
    for (int ml = min_ml; ml < max_ml; ++ml) {
      for (int rl = min_rl; rl < max_rl; ++rl) {

        // Record previous number of allocated time levels
        if (status) {
          // Note: This remembers only the last level
          status[n] = groupdata.AT(group).activetimelevels.AT(ml).AT(rl);
        }

        // Only do something if the number of time levels actually
        // needs to be changed -- do nothing otherwise

        const bool do_increase =
            inc and tls[n] > groupdata.AT(group).activetimelevels.AT(ml).AT(rl);
        const bool do_decrease =
            not inc and
            tls[n] < groupdata.AT(group).activetimelevels.AT(ml).AT(rl);
        if (do_increase or do_decrease) {

          if (not can_do) {
            char *const groupname = CCTK_GroupName(group);
            char const *const modestring =
                (is_meta_mode() ? "meta" : is_global_mode()
                                               ? "global"
                                               : is_level_mode()
                                                     ? "level"
                                                     : is_singlemap_mode()
                                                           ? "singlemap"
                                                           : is_local_mode()
                                                                 ? "local"
                                                                 : NULL);
            CCTK_VWarn(0, __LINE__, __FILE__, CCTK_THORNSTRING,
                       "Cannot change storage for group \"%s\" in %s mode",
                       groupname, modestring);
            free(groupname);
          }
          assert(can_do);

          // No storage change in local mode
          // TODO: This this seems superfluous, given the test above
          if (gp.grouptype == CCTK_GF) {
            assert((map == -1 or maps == 1) and
                   (component == -1 or
                    vhh.AT(0)->local_components(reflevel) == 1));
          }

          // Set the new number of active time levels
          groupdata.AT(group).activetimelevels.AT(ml).AT(rl) = tls[n];

          // Allocate storage only on map 0?
          const bool ismap0group = IsMap0Group(group);

          for (int m = 0; m < (int)arrdata.AT(group).size(); ++m) {
            if (m > 0 and ismap0group)
              continue;
            for (int var = 0; var < gp.numvars; ++var) {
#ifdef CCTK_HAVE_CONTIGUOUS_GROUPS
              bool const contiguous = gp.contiguous;
#else
              bool const contiguous = false;
#endif
              const int vectorindex =
                  (contiguous ? var : gp.vectorgroup ? var % gp.vectorlength
                                                     : 0);
              const int vectorlength =
                  (contiguous ? gp.numvars : gp.vectorgroup ? gp.vectorlength
                                                            : 1);
              assert(vectorindex >= 0 and vectorindex < gp.numvars);
              assert(vectorlength > 0 and vectorlength <= gp.numvars);
              ggf *const vectorleader =
                  (vectorindex > 0
                       ? arrdata.AT(group).AT(m).data.AT(var - vectorindex)
                       : NULL);
              const int varindex = firstvarindex + var;
              // TODO: allocate these in SetupGH, and after recomposing
              if (not arrdata.AT(group).AT(m).data.AT(var)) {
                switch (specific_cactus_type(gp.vartype)) {
#define TYPECASE(N, T)                                                         \
  case N:                                                                      \
    arrdata.AT(group).AT(m).data.AT(var) =                                     \
        new gf<T>(varindex, groupdata.AT(group).transport_operator,            \
                  *arrdata.AT(group).AT(m).tt, *arrdata.AT(group).AT(m).dd,    \
                  prolongation_order_time, vectorlength, vectorindex,          \
                  (gf<T> *)vectorleader);                                      \
    break;
#include "typecase.hh"
#undef TYPECASE
                default:
                  UnsupportedVarType(varindex);
                } // switch gp.vartype
              }   // if not allocated

              arrdata.AT(group).AT(m).data.AT(var)->set_timelevels(ml, rl,
                                                                   tls[n]);

              // Make room in CCTK grid hierarchy if we increase the number of
              // timelevels beyond what was in interface.ccl
              if (tls[n] > groupdata.AT(group).info.maxtimelevels) {
                // this needs to be realloc since we inherit the layout from
                // Cactus'
                // default SetupGH routine
                void **const temp = static_cast<void **>(
                    realloc(cctkGH->data[varindex], tls[n] * sizeof(void *)));
                assert(temp or tls[n] == 0);
                for (int tl = groupdata.AT(group).info.maxtimelevels;
                     tl < tls[n]; ++tl)
                  temp[tl] = NULL;
                cctkGH->data[varindex] = temp;
                groupdata.AT(group).info.maxtimelevels = tls[n];
              }

              // Set the data pointers for grid arrays
              if (gp.grouptype != CCTK_GF) {
                assert(rl == 0 and m == 0);
                for (int tl = 0; tl < groupdata.AT(group).info.maxtimelevels;
                     ++tl) {
                  if (tl < groupdata.AT(group).activetimelevels.AT(ml).AT(rl)) {
                    ggf *const ff = arrdata.AT(group).AT(0).data.AT(var);
                    void *const ptr = ff->data_pointer(tl, 0, 0, 0)->storage();
                    cctkGH->data[varindex][tl] = ptr;
                  } else {
                    cctkGH->data[varindex][tl] = NULL;
                  }
                }
              } // if grouptype != GF

            } // for var
          }   // for m

        } // if really change the number of active time levels

        // Complain if there are not enough active time levels
        GroupStorageCheck(cctkGH, group, ml, rl);

        // Record (minimum of) current number of time levels
        min_num_timelevels =
            min(min_num_timelevels,
                groupdata.AT(group).activetimelevels.AT(ml).AT(rl));

      } // for rl
    }   // for ml

  } // for n
  if (min_num_timelevels == INT_MAX) {
    min_num_timelevels = 0;
  }

  return do_allow_past_timelevels ? min_num_timelevels
                                  : min(1, min_num_timelevels);
}

int QueryMaxTimeLevels(const cGH *, int n_groups, const int *groups,
                       int *status) {
  assert(status || n_groups == 0);
  assert(groups || n_groups == 0);

  for (int g = 0; g < n_groups; g++) {
    if (groups[g] >= 0) {
      status[g] = groupdata.AT(groups[g]).info.maxtimelevels;
    } else {
      CCTK_VWarn(CCTK_WARN_ALERT, __LINE__, __FILE__, CCTK_THORNSTRING,
                 "QueryMaxTimeLevels: Invalid group index %d given", groups[g]);
      status[g] = -1;
    }
  }

  return 0;
}

int GroupStorageIncrease(const cGH *cctkGH, int n_groups, const int *groups,
                         const int *tls, int *status) {
  DECLARE_CCTK_PARAMETERS

  if (storage_verbose) {
    Checkpoint("GroupStorageIncrease");
  }
  return GroupStorageCrease(cctkGH, n_groups, groups, tls, status, true);
}

int GroupStorageDecrease(const cGH *cctkGH, int n_groups, const int *groups,
                         const int *tls, int *status) {
  DECLARE_CCTK_PARAMETERS

  if (storage_verbose) {
    Checkpoint("GroupStorageDecrease");
  }
  return GroupStorageCrease(cctkGH, n_groups, groups, tls, status, false);
}

int EnableGroupStorage(const cGH *cctkGH, const char *groupname) {
  const int group = CCTK_GroupIndex(groupname);
  assert(group >= 0 and group < CCTK_NumGroups());
  // TODO: decide whether to use CCTK_MaxActiveTimeLevelsGI
  const int tls = CCTK_DeclaredTimeLevelsGI(group);
  int status;
  GroupStorageIncrease(cctkGH, 1, &group, &tls, &status);
  // Return whether storage was allocated previously
  return status;
}

int DisableGroupStorage(const cGH *cctkGH, const char *groupname) {
  const int group = CCTK_GroupIndex(groupname);
  assert(group >= 0 and group < CCTK_NumGroups());
  const int tls = 0;
  int status;
  GroupStorageDecrease(cctkGH, 1, &group, &tls, &status);
  // Return whether storage was allocated previously
  return status;
}

int QueryGroupStorageB(const cGH *cctkGH, int group, const char *groupname) {
  DECLARE_CCTK_PARAMETERS;
  if (groupname) {
    group = CCTK_GroupIndex(groupname);
  }
  if (group < 0 or group >= CCTK_NumGroups()) {
    CCTK_WARN(1, "QueryGroupStorage: illegal group specified");
    return -1;
  }
  int const grouptype = CCTK_GroupTypeI(group);
  if (is_meta_mode() or is_global_mode()) {
    if (grouptype == CCTK_GF)
      return -2;
  }
  if (groupdata.size() == 0)
    return -3;
  int const rl = grouptype == CCTK_GF ? reflevel : 0;
  // Return whether storage is allocated
  CCTK_INT const deadbeef = get_deadbeef();
  assert(groupdata.AT(group).activetimelevels.AT(mglevel).AT(rl) != deadbeef);
  return groupdata.AT(group).activetimelevels.AT(mglevel).AT(rl) > 0;
}

const int *ArrayGroupSizeB(const cGH *cctkGH, int dir, int group,
                           const char *groupname) {
  static const int zero = 0;
  static const int error = 0;

  if (groupname) {
    group = CCTK_GroupIndex(groupname);
  }
  assert(group >= 0 and group < CCTK_NumGroups());

  if (mglevel == -1) {
    return &error; // meta mode
  }

  const int gptype = CCTK_GroupTypeI(group);
  if (gptype == CCTK_GF and map == -1) {
    return &error; // global or level mode for a GF
  }

  const int gpdim = groupdata.AT(group).info.dim;
  assert(dir >= 0 and dir < gpdim);

  if (CCTK_QueryGroupStorageI(cctkGH, group)) {

    return &groupdata.AT(group).info.ash[dir];

  } else {

    // no storage
    return &zero;
  }
}

int GroupDynamicData(const cGH *cctkGH, int group, cGroupDynamicData *data) {
  // Return values:
  //  0 for success
  // -1 if given pointer to data structure is NULL
  // -3 if given GH pointer is invalid
  // (-77 if group has zero variables)
  // -78 if group does not exist
  if (not cctkGH)
    return -3;
  if (not(group >= 0 and group < CCTK_NumGroups()))
    return -78;
  if (not data)
    return -1;
  assert(cctkGH);
  assert(group >= 0 and group < CCTK_NumGroups());
  assert(data);
  *data = groupdata.AT(group).info;
  return 0;
}

void GroupStorageCheck(cGH const *const cctkGH, int const group, int const ml,
                       int const rl) {
  DECLARE_CCTK_PARAMETERS;

  if (not do_warn_about_storage)
    return;
  if (max_refinement_levels == 1)
    return;

  cGroup gp;
  check(not CCTK_GroupData(group, &gp));

  if (gp.grouptype == CCTK_GF) {
    operator_type const op = groupdata.AT(group).transport_operator;
    if (op != op_none and op != op_sync and op != op_restrict and
        op != op_copy) {
      int const tls = groupdata.AT(group).activetimelevels.AT(ml).AT(rl);
      if (tls != 0 and tls < prolongation_order_time + 1) {
        static vector<bool> didwarn;
        didwarn.resize(CCTK_NumGroups(), false);
        if (not didwarn.AT(group)) {
          // Warn only once per group
          didwarn.AT(group) = true;
          char *const groupname = CCTK_GroupName(group);
          CCTK_VInfo(CCTK_THORNSTRING,
                     "There are not enough time levels for the desired "
                     "temporal prolongation order in the grid function group "
                     "\"%s\".  With Carpet::prolongation_order_time=%d, you "
                     "need at least %d time levels.",
                     groupname, (int)prolongation_order_time,
                     (int)(prolongation_order_time + 1));
          free(groupname);
        }
      }
    }
  }
}

void GroupsStorageCheck(cGH const *const cctkGH) {
  for (int group = 0; group < CCTK_NumGroups(); ++group) {
    for (int ml = 0; ml < mglevels; ++ml) {
      for (int rl = 0; rl < reflevels; ++rl) {
        GroupStorageCheck(cctkGH, group, ml, rl);
      }
    }
  }
}

} // namespace Carpet

```

variables.cc:
```
#include <variables.hh>

namespace Carpet {

using namespace std;

// Handle from CCTK_RegisterGHExtension
int GHExtension;

// Maximum number of refinement levels
int maxreflevels;

// Maximum number of time levels
int maxtimelevels;

// Timelevels
int timelevels;

// Refinement levels
int reflevels;

// Temporal refinement factors over the coarsest grid
vector<int> timereffacts;

// Spatial refinement factors over the coarsest grid
vector<vect<int, dim> > spacereffacts;

// Maximum refinement factors on finest possible grid
int maxtimereflevelfact;
vect<int, dim> maxspacereflevelfact;

// Base multigrid level
int basemglevel;

// Multigrid levels
int mglevels;

// Multigrid factor
int mgfact;

// Multigrid factor on coarsest grid
int maxmglevelfact;

// Maps
int maps;

// Current position on the grid hierarchy
int reflevel;
int mglevel;
int mc_grouptype;
int map;
int component;
int local_component;
int timelevel, timelevel_offset;

// Current refinement factors
int timereflevelfact;
vect<int, dim> spacereflevelfact;

// Current multigrid factor
int mglevelfact;

// Carpet's GH
CarpetGH carpetGH;

// Times and spaces on the refinement levels
CCTK_REAL global_time;
// vector<vector<CCTK_REAL> > leveltimes; // [mglevel][reflevel]
CCTK_REAL delta_time;

vector<vector<vect<CCTK_REAL, dim> > > origin_space; // [map][mglevel]
vector<vect<CCTK_REAL, dim> > delta_space;           // [map]

vector<domainspec> domainspecs; // [map]

// Is this the time for a global mode call?
bool do_meta_mode;
bool do_early_meta_mode;
bool do_late_meta_mode;
bool do_global_mode;
bool do_early_global_mode;
bool do_late_global_mode;

// Can past time levels be accessed?
bool do_allow_past_timelevels;

// Is prolongation enabled?
// (This flag disables prolongation during MoL integration
// substeps.)
bool do_prolongate;

// Is tapering enabled?
// (This flag disables prolongation while the current refinement
// level is not aligned with the parent.)
bool do_taper;

// Should we warn about groups with insufficiently many time levels?
bool do_warn_about_storage;

// Are we in the analysis bin?
bool in_analysis_bin;

// Data for grid functions

// The grid hierarchy
vector<gh *> vhh; // [map]
vector<dh *> vdd; // [map]
th *tt;
int regridding_epoch;
vector<int> level_regridding_epochs;

// Data for the groups
vector<groupdesc> groupdata; // [group]

// Data for everything
vector<vector<arrdesc> > arrdata; // [group][map]

// MPI Communicators
MPI_Comm comm_universe = MPI_COMM_NULL;
MPI_Comm comm_world = MPI_COMM_NULL;

} // namespace Carpet

```

make.code.defn:
```
# Main make.code.defn file for thorn Carpet   -*-Makefile-*-

# Source files in this directory
SRCS =	CallFunction.cc				\
	CarpetBasegrid.cc			\
	CarpetParamCheck.cc			\
	CarpetStartup.cc			\
	Checksum.cc				\
	Comm.cc					\
	Cycle.cc				\
	Evolve.cc				\
	Hosts.cc				\
	Initialise.cc				\
	MultiModel.cc				\
	OutputGH.cc				\
	Poison.cc				\
	Recompose.cc				\
	Restrict.cc				\
	ScheduleWrapper.cc			\
	SetupGH.cc				\
	Shutdown.cc				\
	Storage.cc				\
	Timing.cc				\
	UnusedMask.cc				\
	adler32.cc				\
	helpers.cc				\
	modes.cc				\
	variables.cc

# Subdirectories containing source files
SUBDIRS = LoadBalanceReal


```

CarpetBasegrid.cc:
```
#include <limits>

#include <cctk.h>
#include <cctk_Arguments.h>
#include <cctk_Parameters.h>

#include <carpet.hh>

namespace Carpet {

using namespace std;

void CarpetRefineTimeStep(CCTK_ARGUMENTS) {
  DECLARE_CCTK_ARGUMENTS;
  DECLARE_CCTK_PARAMETERS;

  // Find the smallest CFL factor for all refinement levels
  CCTK_REAL min_cfl = numeric_limits<CCTK_REAL>::max();
  for (int rl = 0; rl < reflevels; ++rl) {
    CCTK_REAL const level_cfl =
        timereffacts.AT(rl) / maxval(spacereffacts.AT(rl));
    min_cfl = min(min_cfl, level_cfl);
  }

  // Reduce the time step correspondingly
  cctkGH->cctk_delta_time *= min_cfl;
  CCTK_VInfo(CCTK_THORNSTRING, "Timestep reduced by a factor of %g to %g",
             double(1 / min_cfl),
             double(cctkGH->cctk_delta_time / cctkGH->cctk_timefac));
}

} // namespace Carpet

```

ScheduleWrapper.cc:
```
#include <cassert>
#include <list>

#include <cctk.h>
#include <cctk_Arguments.h>

#include <carpet.hh>

namespace Carpet {

using namespace std;

typedef CCTK_INT (*func)(CCTK_POINTER_TO_CONST cctkGH, CCTK_POINTER function,
                         CCTK_POINTER attribute, CCTK_POINTER data);
typedef list<func> flist;

static flist func_befores, func_afters;

extern "C" CCTK_INT Carpet_RegisterScheduleWrapper(func const func_before,
                                                   func const func_after) {
  // Add functions
  if (func_before)
    func_befores.push_back(func_before);
  if (func_after)
    func_afters.push_front(func_after);
  return 0;
}

extern "C" CCTK_INT Carpet_UnRegisterScheduleWrapper(func const func_before,
                                                     func const func_after) {
  // Remove functions
  if (func_before)
    func_befores.remove(func_before);
  if (func_after)
    func_afters.remove(func_after);
  return 0;
}

int CallBeforeRoutines(cGH const *const cctkGH, void *const function,
                       cFunctionData *const attribute, void *const data) {
  int skip = 0;
  for (flist::const_iterator fli = func_befores.begin();
       fli != func_befores.end(); ++fli) {
    skip |= (*fli)(cctkGH, function, attribute, data);
  }
  return skip;
}

int CallAfterRoutines(cGH const *const cctkGH, void *const function,
                      cFunctionData *const attribute, void *const data) {
  int res = 0;
  for (flist::const_iterator fli = func_afters.begin();
       fli != func_afters.end(); ++fli) {
    res |= (*fli)(cctkGH, function, attribute, data);
  }
  return res;
}

} // namespace Carpet

```

CarpetStartup.cc:
```
#include <cassert>
#include <cstdlib>
#include <iomanip>
#include <limits>

#include <cctk.h>
#include <cctk_Parameters.h>

#include <dist.hh>

#include <carpet.hh>

namespace Carpet {

using namespace std;

int CarpetMultiModelStartup() {
  DECLARE_CCTK_PARAMETERS;

  // Increase the default output precision, so that all relevant
  // digits are displayed. (The C++ output streams are mostly used
  // for debug messages.)
  int const precision = numeric_limits<CCTK_REAL>::digits10;
  cout << setprecision(precision);
  cerr << setprecision(precision);

  comm_universe = MPI_COMM_WORLD;
  SplitUniverse(comm_universe, model, comm_world, true);
  dist::pseudoinit(comm_world);

  return 0;
}

int CarpetStartup() {
  CCTK_RegisterBanner("AMR driver provided by Carpet");

  GHExtension = CCTK_RegisterGHExtension("Carpet");
  CCTK_RegisterGHExtensionSetupGH(GHExtension, SetupGH);

  CCTK_OverloadInitialise(Initialise);
  CCTK_OverloadEvolve(Evolve);
  CCTK_OverloadShutdown(Shutdown);

  CCTK_OverloadOutputGH(OutputGH);

  CCTK_OverloadSyncGroupsByDirI(SyncGroupsByDirI);
  CCTK_OverloadEnableGroupStorage(EnableGroupStorage);
  CCTK_OverloadDisableGroupStorage(DisableGroupStorage);
  CCTK_OverloadGroupStorageIncrease(GroupStorageIncrease);
  CCTK_OverloadGroupStorageDecrease(GroupStorageDecrease);
  CCTK_OverloadQueryMaxTimeLevels(QueryMaxTimeLevels);
  CCTK_OverloadEnableGroupComm(EnableGroupComm);
  CCTK_OverloadDisableGroupComm(DisableGroupComm);
  CCTK_OverloadBarrier(Barrier);
  // CCTK_OverloadNamedBarrier (NamedBarrier);
  CCTK_OverloadExit((int (*)(cGH *, int))Exit);
  CCTK_OverloadAbort((int (*)(cGH *, int))Abort);
  CCTK_OverloadMyProc(MyProc);
  CCTK_OverloadnProcs(nProcs);
  CCTK_OverloadArrayGroupSizeB(ArrayGroupSizeB);
  CCTK_OverloadQueryGroupStorageB(QueryGroupStorageB);
  CCTK_OverloadGroupDynamicData(GroupDynamicData);

  return 0;
}

} // namespace Carpet

```

modes.cc:
```
#include <algorithm>
#include <cassert>
#include <cstdarg>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <functional>

#include <cctk.h>
#include <cctk_Parameters.h>

#include <Timer.hh>

#include <cacheinfo.hh>
#include <defs.hh>
#include <gdata.hh>
#include <ggf.hh>

#include <carpet.hh>

namespace Carpet {

using namespace std;

// Local routine:
// Access a const int* as ivect reference (without the const)
static inline ivect &ivect_ref(const int *const iptr) {
  return ivect::ref(const_cast<int *>(iptr));
}

//
// Mode indicators
//

bool is_meta_mode() {
  if (mglevel == -1)
    assert(reflevel == -1 and map == -1 and component == -1);
  return mglevel == -1;
}

bool is_global_mode() {
  if (mglevel == -1 and reflevel != -1)
    assert(map == -1 and component == -1);
  return mglevel != -1 and reflevel == -1;
}

bool is_level_mode() {
  if (mglevel != -1 and reflevel != -1 and map == -1)
    assert(component == -1);
  return mglevel != -1 and reflevel != -1 and map == -1;
}

bool is_singlemap_mode() {
  return mglevel != -1 and reflevel != -1 and map != -1 and component == -1;
}

bool is_local_mode() {
  return mglevel != -1 and reflevel != -1 and map != -1 and component != -1;
  //       assert (mglevel>=0 and mglevel<mglevels);
  //       assert (reflevel>=0 and reflevel<reflevels);
  //       assert (map>=0 and map<maps);
  //       assert (vhh.AT(map)->local_components(reflevel)==1 or component==-1);
}

//
// Mode setting
//

// Set global mode

void enter_global_mode(cGH *const cctkGH, int const ml) {
  DECLARE_CCTK_PARAMETERS;

  assert(is_meta_mode());
  assert(ml >= 0 and ml < mglevels);
  Checkpoint("Entering global mode");

  static Timers::Timer ftimer("enter_global_mode", 1);
  ftimer.start();

  mglevel = ml;
  mglevelfact = ipow(mgfact, mglevel);
  // TODO: this could also just be "mglevel" instead
  cctkGH->cctk_convlevel = basemglevel + mglevel;

// Set mode
#ifdef HAVE_CGH_CCTK_MODE
  cctkGH->cctk_mode = CCTK_MODE_GLOBAL;
#endif

  // Set time delta
  if (not adaptive_stepsize) {
    cctkGH->cctk_delta_time = delta_time * mglevelfact;
  } else {
    delta_time = cctkGH->cctk_delta_time;
  }
  if (maps == 1) {
    // Set space delta
    for (int d = 0; d < dim; ++d) {
      cctkGH->cctk_origin_space[d] = origin_space.AT(0).AT(mglevel)[d];
      cctkGH->cctk_delta_space[d] = delta_space.AT(0)[d] * mglevelfact;
    }
  }

  // Set array information
  for (int group = 0; group < CCTK_NumGroups(); ++group) {
    cGroup gp;
    check(not CCTK_GroupData(group, &gp));
    if (gp.grouptype != CCTK_GF) {

      const int rl = 0;
      const int m = 0;
      const int c = dist::rank();

      const ibbox &baseext =
          arrdata.AT(group).AT(m).hh->baseextents.AT(ml).AT(rl);
      const ibbox &ext =
          arrdata.AT(group).AT(m).dd->light_boxes.AT(ml).AT(rl).AT(c).exterior;
      const b2vect &obnds = arrdata.AT(group).AT(m).hh->outer_boundaries(rl, c);

      cGroupDynamicData &info = groupdata.AT(group).info;

      ivect_ref(info.nghostzones) =
          arrdata.AT(group).AT(m).dd->ghost_widths.AT(0)[0];
      assert(all(ivect_ref(info.nghostzones) ==
                 arrdata.AT(group).AT(m).dd->ghost_widths.AT(0)[1]));
      ivect_ref(info.gsh) = baseext.shape() / baseext.stride();
      ivect_ref(info.lbnd) = (ext.lower() - baseext.lower()) / ext.stride();
      ivect_ref(info.ubnd) = (ext.upper() - baseext.lower()) / ext.stride();
      ivect_ref(info.lsh) = ext.shape() / ext.stride();
      ivect_ref(info.ash) = pad_shape(ext);
      // CCTK_DISTRIB_CONSTANT groups are handled by extending an extra
      // dimension to nprocs elements. Here we undo this so that the user
      // code does not see this trick.
      if (gp.disttype == CCTK_DISTRIB_CONSTANT) {
        int const dir = min(dim - 1, gp.dim == 0 ? 1 : gp.dim);
        ivect &gsh = ivect_ref(info.gsh);
        ivect &lbnd = ivect_ref(info.lbnd);
        ivect &ubnd = ivect_ref(info.ubnd);
        ivect const &lsh = ivect_ref(info.lsh);
        gsh[dir] = lsh[dir];
        lbnd[dir] = 0;
        ubnd[dir] = lsh[dir] - 1;
      }
      for (int d = 0; d < dim; ++d) {
        const_cast<int *>(info.bbox)[2 * d] = obnds[0][d];
        const_cast<int *>(info.bbox)[2 * d + 1] = obnds[1][d];
      }
      info.activetimelevels =
          groupdata.AT(group).activetimelevels.AT(mglevel).AT(0);

      for (int d = 0; d < dim; ++d) {
        assert(info.lsh[d] >= 0);
        assert(info.lsh[d] <= info.gsh[d]);
        assert(info.lbnd[d] >= 0);
        assert(info.lbnd[d] <= info.ubnd[d] + 1);
        assert(info.ubnd[d] < info.gsh[d]);
        assert(info.lbnd[d] + info.lsh[d] - 1 == info.ubnd[d]);
        assert(info.lbnd[d] <= info.ubnd[d] + 1);
        assert(info.lsh[d] <= info.ash[d]);
      }

      const int numvars = CCTK_NumVarsInGroupI(group);
      if (numvars > 0) {
        const int firstvar = CCTK_FirstVarIndexI(group);
        assert(firstvar >= 0);
        // TODO: modify flesh to allow changing max_tl
        const int active_tl = info.activetimelevels;
        assert(active_tl >= 0);

        assert(arrdata.AT(group).AT(m).hh->is_local(rl, c));

        for (int var = 0; var < numvars; ++var) {
          assert(firstvar + var < CCTK_NumVars());
          ggf *const ff = arrdata.AT(group).AT(m).data.AT(var);
          for (int tl = 0; tl < info.maxtimelevels; ++tl) {
            if (ff and tl < active_tl) {
              int const lc = 0;
              gdata *const data = ff->data_pointer(tl, rl, lc, ml);
              assert(data);
              cctkGH->data[firstvar + var][tl] = data->storage();
            } else {
              cctkGH->data[firstvar + var][tl] = NULL;
            }
          }
        }
      }

    } // if grouptype
  }   // for group

  ftimer.stop();

  Timers::Timer timer("global mode", 1);
  timer.start();

  assert(is_global_mode());
}

void leave_global_mode(cGH *const cctkGH) {
  DECLARE_CCTK_PARAMETERS;

  assert(is_global_mode() or is_meta_mode());

  if (mglevel == -1)
    return; // early return

  Checkpoint("Leaving global mode");

  Timers::Timer timer("global mode", 1);
  timer.stop();

  Timers::Timer ftimer("leave_global_mode", 1);
  ftimer.start();

  // Unset time delta
  if (not adaptive_stepsize) {
    cctkGH->cctk_delta_time = 0.0;
  }
  if (maps == 1) {
    // Save and unset space delta
    for (int d = 0; d < dim; ++d) {
      origin_space.AT(0).AT(mglevel)[d] = cctkGH->cctk_origin_space[d];
      delta_space.AT(mglevel)[d] = cctkGH->cctk_delta_space[d] / mglevelfact;
      cctkGH->cctk_origin_space[d] = -424242.0;
      cctkGH->cctk_delta_space[d] = -424242.0;
    }
  }

  CCTK_INT const deadbeef = get_deadbeef();

  // Set array information
  for (int group = 0; group < CCTK_NumGroups(); ++group) {
    if (CCTK_GroupTypeI(group) != CCTK_GF) {

      const int m = 0;

      cGroupDynamicData &info = groupdata.AT(group).info;

      // ivect_ref(info.nghostzones) = deadbeef;
      ivect_ref(info.nghostzones) =
          arrdata.AT(group).AT(m).dd->ghost_widths.AT(0)[0];
      ivect_ref(info.gsh) = deadbeef;
      ivect_ref(info.lbnd) = -deadbeef;
      ivect_ref(info.ubnd) = deadbeef;
      ivect_ref(info.lsh) = deadbeef;
      for (int d = 0; d < dim; ++d) {
        const_cast<int *>(info.bbox)[2 * d] = deadbeef;
        const_cast<int *>(info.bbox)[2 * d + 1] = deadbeef;
      }
      ivect_ref(info.ash) = deadbeef;
      info.activetimelevels = deadbeef;

      const int numvars = CCTK_NumVarsInGroupI(group);
      if (numvars > 0) {
        const int firstvar = CCTK_FirstVarIndexI(group);
        assert(firstvar >= 0);
        const int max_tl = groupdata.AT(group).info.maxtimelevels;
        assert(max_tl >= 0);

        assert(group < (int)arrdata.size());
        for (int var = 0; var < numvars; ++var) {
          assert(firstvar + var < CCTK_NumVars());
          for (int tl = 0; tl < max_tl; ++tl) {
            cctkGH->data[firstvar + var][tl] = NULL;
          }
        }
      }

    } // if grouptype
  }   // for group

// Set mode
#ifdef HAVE_CGH_CCTK_MODE
  cctkGH->cctk_mode = CCTK_MODE_META;
#endif

  mglevel = -1;
  mglevelfact = -deadbeef;
  cctkGH->cctk_convlevel = -deadbeef;

  ftimer.stop();

  assert(is_meta_mode());
}

// Set level mode

void enter_level_mode(cGH *const cctkGH, int const rl) {
  DECLARE_CCTK_PARAMETERS;

  assert(is_global_mode());
  assert(rl >= 0 and rl < reflevels);
  Checkpoint("Entering level mode");

  static Timers::Timer ftimer("enter_level_mode", 1);
  ftimer.start();

// Set mode
#ifdef HAVE_CGH_CCTK_MODE
  cctkGH->cctk_mode = CCTK_MODE_LEVEL;
#endif

  reflevel = rl;
  timereflevelfact = timereffacts.AT(reflevel);
  spacereflevelfact = spacereffacts.AT(reflevel);
  ivect_ref(cctkGH->cctk_levfac) = spacereflevelfact;
  cctkGH->cctk_timefac = timereflevelfact;

  // Set number of time levels
  for (int group = 0; group < CCTK_NumGroups(); ++group) {
    if (CCTK_GroupTypeI(group) == CCTK_GF) {
      cGroupDynamicData &info = groupdata.AT(group).info;
      info.activetimelevels =
          groupdata.AT(group).activetimelevels.AT(mglevel).AT(reflevel);
    }
  }

  // Set current time
  // assert (mglevel>=0 and mglevel<(int)leveltimes.size());
  // assert (reflevel>=0 and reflevel<(int)leveltimes.AT(mglevel).size());
  // if (not adaptive_stepsize) {
  //   cctkGH->cctk_time = leveltimes.AT(mglevel).AT(reflevel);
  // } else {
  //   leveltimes.AT(mglevel).AT(reflevel) = cctkGH->cctk_time;
  // }
  if (not adaptive_stepsize) {
    cctkGH->cctk_time = tt->get_time(mglevel, reflevel, timelevel);
  } else {
    tt->set_time(mglevel, reflevel, timelevel, cctkGH->cctk_time);
  }

  ftimer.stop();

  ostringstream mode_s;
  mode_s << "level(" << reflevel << ")";
  Timers::Timer timer(mode_s.str(), 1);
  timer.start();

  assert(is_level_mode());
}

void leave_level_mode(cGH *const cctkGH) {
  DECLARE_CCTK_PARAMETERS;

  assert(is_level_mode() or is_global_mode());

  if (reflevel == -1)
    return; // early return

  Checkpoint("Leaving level mode");

  ostringstream mode_s;
  mode_s << "level(" << reflevel << ")";
  Timers::Timer timer(mode_s.str(), 1);
  timer.stop();

  static Timers::Timer ftimer("leave_level_mode", 1);
  ftimer.start();

  CCTK_INT const deadbeef = get_deadbeef();

  // Save and unset current time
  // assert (mglevel>=0 and mglevel<(int)leveltimes.size());
  // assert (reflevel>=0 and reflevel<(int)leveltimes.AT(mglevel).size());
  // leveltimes.AT(mglevel).AT(reflevel) = cctkGH->cctk_time;
  // if (not adaptive_stepsize) {
  //   cctkGH->cctk_time = global_time;
  // } else {
  //   global_time = cctkGH->cctk_time;
  // }
  tt->set_time(mglevel, reflevel, timelevel, cctkGH->cctk_time);
  if (not adaptive_stepsize) {
    cctkGH->cctk_time = global_time;
  } else {
    global_time = cctkGH->cctk_time;
  }

  // Unset number of time levels
  for (int group = 0; group < CCTK_NumGroups(); ++group) {
    if (CCTK_GroupTypeI(group) == CCTK_GF) {
      cGroupDynamicData &info = groupdata.AT(group).info;
      info.activetimelevels = deadbeef;
    }
  }

// Set mode
#ifdef HAVE_CGH_CCTK_MODE
  cctkGH->cctk_mode = CCTK_MODE_GLOBAL;
#endif

  reflevel = -1;
  timereflevelfact = timereffacts.AT(reflevels - 1);
  // TODO: use spacereffacts.AT (reflevel - 1) instead?
  spacereflevelfact = ivect(-deadbeef);
  ivect_ref(cctkGH->cctk_levfac) = spacereflevelfact;
  cctkGH->cctk_timefac = timereflevelfact;

  ftimer.stop();

  assert(is_global_mode());
}

// Set singlemap mode

void enter_singlemap_mode(cGH *const cctkGH, int const m, int const grouptype) {
  DECLARE_CCTK_PARAMETERS

  assert(is_level_mode());
  assert(mc_grouptype == -1);
  assert(m >= 0 and m < maps);
  assert(grouptype == CCTK_SCALAR or grouptype == CCTK_ARRAY or
         grouptype == CCTK_GF);
  Checkpoint("Entering singlemap mode");

  static Timers::Timer ftimer("enter_singlemap_mode", 1);
  ftimer.start();

// Set mode
#ifdef HAVE_CGH_CCTK_MODE
  cctkGH->cctk_mode = CCTK_MODE_SINGLEMAP;
#endif

  assert(mc_grouptype == -1);
  mc_grouptype = grouptype;
  carpetGH.map = map = m;

  if (mc_grouptype == CCTK_GF) {

    if (maps > 1) {
      // Set space delta
      for (int d = 0; d < dim; ++d) {
        cctkGH->cctk_origin_space[d] = origin_space.AT(map).AT(mglevel)[d];
        cctkGH->cctk_delta_space[d] = delta_space.AT(map)[d] * mglevelfact;
      }
    }

    // Set grid shape
    const ibbox &coarseext = vhh.AT(map)->baseextents.AT(mglevel).AT(0);
    const ibbox &baseext = vhh.AT(map)->baseextents.AT(mglevel).AT(reflevel);
    //       assert (all (baseext.lower() % baseext.stride() == 0));

    ivect_ref(cctkGH->cctk_levoff) = baseext.lower() - coarseext.lower();
    ivect_ref(cctkGH->cctk_levoffdenom) = baseext.stride();
    ivect_ref(cctkGH->cctk_gsh) = baseext.shape() / baseext.stride();
    assert(all(vdd.AT(map)->ghost_widths.AT(reflevel)[0] ==
               vdd.AT(map)->ghost_widths.AT(reflevel)[1]));
    ivect_ref(cctkGH->cctk_nghostzones) =
        vdd.AT(map)->ghost_widths.AT(reflevel)[0];

    for (int group = 0; group < CCTK_NumGroups(); ++group) {
      if (CCTK_GroupTypeI(group) == CCTK_GF) {
        cGroupDynamicData &info = groupdata.AT(group).info;
        ivect_ref(info.gsh) = ivect_ref(cctkGH->cctk_gsh);
        ivect_ref(info.nghostzones) = ivect_ref(cctkGH->cctk_nghostzones);
      }
    }

  } // if mc_grouptype

  ftimer.stop();

  if (include_maps_in_mode_timer_tree) {
    ostringstream mode_s;
    mode_s << "map(" << map << ")";
    Timers::Timer timer(mode_s.str(), 1);
    timer.start();
  }

  assert(is_singlemap_mode());
}

void leave_singlemap_mode(cGH *const cctkGH) {
  DECLARE_CCTK_PARAMETERS;

  assert(is_singlemap_mode() or is_level_mode());

  if (map == -1)
    return; // early return

  Checkpoint("Leaving singlemap mode");

  if (include_maps_in_mode_timer_tree) {
    ostringstream mode_s;
    mode_s << "map(" << map << ")";
    Timers::Timer timer(mode_s.str(), 1);
    timer.stop();
  }

  static Timers::Timer ftimer("leave_singlemap_mode", 1);
  ftimer.start();

  assert(mc_grouptype == CCTK_SCALAR or mc_grouptype == CCTK_ARRAY or
         mc_grouptype == CCTK_GF);

  if (mc_grouptype == CCTK_GF) {

    CCTK_INT const deadbeef = get_deadbeef();

    // Save space delta
    // (Do this early and often, so that interpolation has access to
    // the correct values right away.)
    for (int d = 0; d < dim; ++d) {
      origin_space.AT(map).AT(mglevel)[d] = cctkGH->cctk_origin_space[d];
      delta_space.AT(map)[d] = cctkGH->cctk_delta_space[d] / mglevelfact;
    }
    if (maps > 1) {
      // Unset space delta
      for (int d = 0; d < dim; ++d) {
        cctkGH->cctk_origin_space[d] = -424242.0;
        cctkGH->cctk_delta_space[d] = -424242.0;
      }
    }

    // Unset grid shape
    ivect_ref(cctkGH->cctk_levoff) = deadbeef;
    ivect_ref(cctkGH->cctk_levoffdenom) = 0;
    ivect_ref(cctkGH->cctk_gsh) = deadbeef;
    ivect_ref(cctkGH->cctk_nghostzones) = deadbeef;

    for (int group = 0; group < CCTK_NumGroups(); ++group) {
      if (CCTK_GroupTypeI(group) == CCTK_GF) {
        cGroupDynamicData &info = groupdata.AT(group).info;
        ivect_ref(info.gsh) = ivect_ref(cctkGH->cctk_gsh);
        ivect_ref(info.nghostzones) = ivect_ref(cctkGH->cctk_nghostzones);
      }
    }

  } // if mc_grouptype

// Set mode
#ifdef HAVE_CGH_CCTK_MODE
  cctkGH->cctk_mode = CCTK_MODE_LEVEL;
#endif

  mc_grouptype = -1;
  carpetGH.map = map = -1;

  ftimer.stop();

  assert(is_level_mode());
}

// Set local mode

void enter_local_mode(cGH *const cctkGH, int const c, int const lc,
                      int const grouptype) {
  DECLARE_CCTK_PARAMETERS

  assert(is_singlemap_mode());
  if (mc_grouptype == CCTK_GF) {
    assert(c >= 0 and c < vhh.AT(map)->components(reflevel));
    assert(lc == -1 or
           (lc >= 0 and lc < vhh.AT(map)->local_components(reflevel)));
  } else {
    assert(c >= 0 and c < dist::size());
    assert(lc == -1 or lc == 0);
  }
  Checkpoint("Entering local mode");

  static Timers::Timer ftimer("enter_local_mode", 1);
  ftimer.start();

// Set mode
#ifdef HAVE_CGH_CCTK_MODE
  cctkGH->cctk_mode = CCTK_MODE_LOCAL;
#endif

  assert(grouptype == mc_grouptype);
  component = c;
  local_component = lc;

  if (mc_grouptype == CCTK_GF) {

    // Set cGH fields
    const ibbox &baseext = vhh.AT(map)->baseextents.AT(mglevel).AT(reflevel);
    const ibbox &ext = vdd.AT(map)
                           ->light_boxes.AT(mglevel)
                           .AT(reflevel)
                           .AT(component)
                           .exterior;
    const b2vect &obnds = vhh.AT(map)->outer_boundaries(reflevel, component);

    ivect_ref(cctkGH->cctk_lbnd) =
        (ext.lower() - baseext.lower()) / ext.stride();
    ivect_ref(cctkGH->cctk_ubnd) =
        (ext.upper() - baseext.lower()) / ext.stride();
    ivect_ref(cctkGH->cctk_lsh) = ext.shape() / ext.stride();
    ivect_ref(cctkGH->cctk_ash) = pad_shape(ext);

    for (int d = 0; d < dim; ++d) {
      cctkGH->cctk_bbox[2 * d] = obnds[0][d];
      cctkGH->cctk_bbox[2 * d + 1] = obnds[1][d];
    }

    for (int d = 0; d < dim; ++d) {
      cctkGH->cctk_from[d] = 0;
      cctkGH->cctk_to[d] = cctkGH->cctk_lsh[d];
    }

    for (int d = 0; d < dim; ++d) {
      assert(cctkGH->cctk_lsh[d] >= 0);
      assert(cctkGH->cctk_lsh[d] <= cctkGH->cctk_gsh[d]);
      assert(cctkGH->cctk_lbnd[d] >= 0);
      assert(cctkGH->cctk_lbnd[d] <= cctkGH->cctk_ubnd[d] + 1);
      assert(cctkGH->cctk_ubnd[d] < cctkGH->cctk_gsh[d]);
      assert(cctkGH->cctk_lbnd[d] + cctkGH->cctk_lsh[d] - 1 ==
             cctkGH->cctk_ubnd[d]);
      assert(cctkGH->cctk_lbnd[d] <= cctkGH->cctk_ubnd[d] + 1);
      assert(cctkGH->cctk_lsh[d] <= cctkGH->cctk_ash[d]);
      assert(cctkGH->cctk_from[d] >= 0);
      assert(cctkGH->cctk_from[d] <= cctkGH->cctk_to[d]);
      assert(cctkGH->cctk_to[d] <= cctkGH->cctk_lsh[d]);
    }

    for (int group = 0; group < CCTK_NumGroups(); ++group) {
      if (CCTK_GroupTypeI(group) == CCTK_GF) {

        cGroupDynamicData &info = groupdata.AT(group).info;

        ivect_ref(info.lbnd) = ivect_ref(cctkGH->cctk_lbnd);
        ivect_ref(info.ubnd) = ivect_ref(cctkGH->cctk_ubnd);
        ivect_ref(info.lsh) = ivect_ref(cctkGH->cctk_lsh);
        ivect_ref(info.ash) = ivect_ref(cctkGH->cctk_ash);

        for (int d = 0; d < dim; ++d) {
          const_cast<int *>(info.bbox)[2 * d] = cctkGH->cctk_bbox[2 * d];
          const_cast<int *>(info.bbox)[2 * d + 1] =
              cctkGH->cctk_bbox[2 * d + 1];
        }

        if (local_component != -1) {
          const int numvars = CCTK_NumVarsInGroupI(group);
          if (numvars > 0) {
            const int firstvar = CCTK_FirstVarIndexI(group);
            assert(firstvar >= 0);
            // TODO: Modify flesh so the max_tl can be updated
            const int active_tl =
                groupdata.AT(group).activetimelevels.AT(mglevel).AT(reflevel);
            assert(active_tl >= 0);
            int available_tl;
            int tl_offset;
            if (active_tl == 0) {
              // group has no storage
              available_tl = active_tl;
              tl_offset = 0;
            } else if (do_allow_past_timelevels) {
              // regular case; all timelevels are accessible
              available_tl = active_tl;
              tl_offset = 0;
            } else {
              // only one timelevel is accessible
              available_tl = 1;
              if (timelevel < active_tl) {
                // timelevel "timelevel" exists
                tl_offset = timelevel;
              } else {
                // timelevel "timelevel" does not exist
                tl_offset = active_tl - 1;
              }
            }

            // assert (vhh.AT(map)->is_local(reflevel,component));

            assert(group < (int)arrdata.size());
            for (int var = 0; var < numvars; ++var) {
              assert(firstvar + var < CCTK_NumVars());
              ggf *const ff = arrdata.AT(group).AT(map).data.AT(var);
              for (int tl = 0; tl < info.maxtimelevels; ++tl) {
                if (ff and tl < available_tl) {
                  gdata *const data = ff->data_pointer(
                      tl_offset + tl, reflevel, local_component, mglevel);
                  assert(data);
                  cctkGH->data[firstvar + var][tl] = data->storage();
                } else {
                  cctkGH->data[firstvar + var][tl] = NULL;
                }
              }
            }
          }
        }

      } // if grouptype
    }   // for group

  } // if mc_grouptype

  ftimer.stop();

  if (include_local_mode_in_mode_timer_tree) {
    Timers::Timer timer("local", 1);
    timer.start();
  }

  assert(is_local_mode());
}

void leave_local_mode(cGH *const cctkGH) {
  DECLARE_CCTK_PARAMETERS;

  assert(is_local_mode() or is_singlemap_mode());

  if (component == -1)
    return; // early return

  Checkpoint("Leaving local mode");

  if (include_local_mode_in_mode_timer_tree) {
    Timers::Timer timer("local", 1);
    timer.stop();
  }

  static Timers::Timer ftimer("leave_local_mode", 1);
  ftimer.start();

  if (mc_grouptype == CCTK_GF) {

    CCTK_INT const deadbeef = get_deadbeef();

    // Unset cGH fields
    ivect_ref(cctkGH->cctk_lbnd) = -deadbeef;
    ivect_ref(cctkGH->cctk_ubnd) = deadbeef;
    ivect_ref(cctkGH->cctk_from) = -deadbeef;
    ivect_ref(cctkGH->cctk_to) = deadbeef;
    ivect_ref(cctkGH->cctk_lsh) = deadbeef;
    ivect_ref(cctkGH->cctk_ash) = deadbeef;

    for (int d = 0; d < dim; ++d) {
      cctkGH->cctk_bbox[2 * d] = deadbeef;
      cctkGH->cctk_bbox[2 * d + 1] = deadbeef;
    }

    for (int group = 0; group < CCTK_NumGroups(); ++group) {
      if (CCTK_GroupTypeI(group) == CCTK_GF) {

        cGroupDynamicData &info = groupdata.AT(group).info;

        ivect_ref(info.lbnd) = ivect_ref(cctkGH->cctk_lbnd);
        ivect_ref(info.ubnd) = ivect_ref(cctkGH->cctk_ubnd);
        ivect_ref(info.lsh) = ivect_ref(cctkGH->cctk_lsh);
        ivect_ref(info.ash) = ivect_ref(cctkGH->cctk_ash);

        for (int d = 0; d < dim; ++d) {
          const_cast<int *>(info.bbox)[2 * d] = cctkGH->cctk_bbox[2 * d];
          const_cast<int *>(info.bbox)[2 * d + 1] =
              cctkGH->cctk_bbox[2 * d + 1];
        }

        if (local_component != -1) {
          const int numvars = CCTK_NumVarsInGroupI(group);
          if (numvars > 0) {
            const int firstvar = CCTK_FirstVarIndexI(group);
            assert(firstvar >= 0);
            const int active_tl =
                groupdata.AT(group).activetimelevels.AT(mglevel).AT(reflevel);
            assert(active_tl >= 0);

            assert(group < (int)arrdata.size());
            for (int var = 0; var < numvars; ++var) {
              assert(firstvar + var < CCTK_NumVars());
              for (int tl = 0; tl < active_tl; ++tl) {
                cctkGH->data[firstvar + var][tl] = NULL;
              }
            }
          }
        }

      } // if grouptype
    }   // for group

  } // if mc_grouptype

// Set mode
#ifdef HAVE_CGH_CCTK_MODE
  cctkGH->cctk_mode = CCTK_MODE_SINGLEMAP;
#endif

  component = -1;
  local_component = -1;

  ftimer.stop();

  assert(is_singlemap_mode());
}

//
// Mode iterators
//

// mglevel iterator

mglevel_iterator::mglevel_iterator(cGH const *const cctkGH_)
    : cctkGH(const_cast<cGH *>(cctkGH_)), ml(mglevels - 1) {
  enter_global_mode(cctkGH, ml);
}

mglevel_iterator::~mglevel_iterator() { leave_global_mode(cctkGH); }

bool mglevel_iterator::done() const { return ml < 0; }

void mglevel_iterator::step() {
  --ml;
  if (not done()) {
    leave_global_mode(cctkGH);
    enter_global_mode(cctkGH, ml);
  }
}

// reverse mglevel iterator

reverse_mglevel_iterator::reverse_mglevel_iterator(cGH const *const cctkGH_)
    : cctkGH(const_cast<cGH *>(cctkGH_)), ml(0) {
  enter_global_mode(cctkGH, ml);
}

reverse_mglevel_iterator::~reverse_mglevel_iterator() {
  leave_global_mode(cctkGH);
}

bool reverse_mglevel_iterator::done() const { return ml >= mglevels; }

void reverse_mglevel_iterator::step() {
  ++ml;
  if (not done()) {
    leave_global_mode(cctkGH);
    enter_global_mode(cctkGH, ml);
  }
}

// reflevel iterator

reflevel_iterator::reflevel_iterator(cGH const *const cctkGH_)
    : cctkGH(const_cast<cGH *>(cctkGH_)), rl(0) {
  enter_level_mode(cctkGH, rl);
}

reflevel_iterator::~reflevel_iterator() { leave_level_mode(cctkGH); }

bool reflevel_iterator::done() const { return rl >= reflevels; }

void reflevel_iterator::step() {
  ++rl;
  if (not done()) {
    leave_level_mode(cctkGH);
    enter_level_mode(cctkGH, rl);
  }
}

// reverse reflevel iterator

reverse_reflevel_iterator::reverse_reflevel_iterator(cGH const *const cctkGH_)
    : cctkGH(const_cast<cGH *>(cctkGH_)), rl(reflevels - 1) {
  enter_level_mode(cctkGH, rl);
}

reverse_reflevel_iterator::~reverse_reflevel_iterator() {
  leave_level_mode(cctkGH);
}

bool reverse_reflevel_iterator::done() const { return rl < 0; }

void reverse_reflevel_iterator::step() {
  --rl;
  if (not done()) {
    leave_level_mode(cctkGH);
    enter_level_mode(cctkGH, rl);
  }
}

// map iterator

map_iterator::map_iterator(cGH const *const cctkGH_, int const grouptype_)
    : cctkGH(const_cast<cGH *>(cctkGH_)), grouptype(grouptype_), m(0) {
  assert(grouptype == CCTK_GF or grouptype == CCTK_ARRAY or
         grouptype == CCTK_SCALAR);
  enter_singlemap_mode(cctkGH, m, grouptype);
}

map_iterator::~map_iterator() { leave_singlemap_mode(cctkGH); }

bool map_iterator::done() const {
  int const maxm = grouptype == CCTK_GF ? maps : 1;
  return m >= maxm;
}

void map_iterator::step() {
  ++m;
  if (not done()) {
    leave_singlemap_mode(cctkGH);
    enter_singlemap_mode(cctkGH, m, grouptype);
  }
}

// processor local map iterator

local_map_iterator::local_map_iterator(cGH const *const cctkGH_,
                                       int const grouptype_)
    : cctkGH(const_cast<cGH *>(cctkGH_)), grouptype(grouptype_), m(0) {
  assert(grouptype == CCTK_GF or grouptype == CCTK_ARRAY or
         grouptype == CCTK_SCALAR);
  enter_singlemap_mode(cctkGH, m, grouptype);
}

local_map_iterator::~local_map_iterator() { leave_singlemap_mode(cctkGH); }

bool local_map_iterator::done() const {
  int const maxm = grouptype == CCTK_GF ? maps : 1;
  return m >= maxm;
}

void local_map_iterator::step() {
  while (true) {
    ++m;
    if (done())
      break;
    int const maxlc =
        grouptype == CCTK_GF ? vhh.AT(m)->local_components(reflevel) : 1;
    if (maxlc > 0)
      break;
  }
  if (not done()) {
    leave_singlemap_mode(cctkGH);
    enter_singlemap_mode(cctkGH, m, grouptype);
  }
}

// local mode iterator

component_iterator::component_iterator(cGH const *const cctkGH_,
                                       int const grouptype_)
    : cctkGH(const_cast<cGH *>(cctkGH_)), grouptype(grouptype_), c(-1) {
  assert(grouptype == CCTK_GF or grouptype == CCTK_ARRAY or
         grouptype == CCTK_SCALAR);
  step();
}

component_iterator::~component_iterator() { leave_local_mode(cctkGH); }

bool component_iterator::done() const {
  int const maxc =
      (grouptype == CCTK_GF ? vhh.AT(map)->components(reflevel) : dist::size());
  return c >= maxc;
}

void component_iterator::step() {
  ++c;
  if (not done()) {
    leave_local_mode(cctkGH);
    int const lc =
        (grouptype == CCTK_GF ? vhh.AT(map)->get_local_component(reflevel, c)
                              : (c == dist::rank() ? 0 : -1));
    enter_local_mode(cctkGH, c, lc, grouptype);
  }
}

// processor local mode iterator

local_component_iterator::local_component_iterator(cGH const *const cctkGH_,
                                                   int const grouptype_)
    : cctkGH(const_cast<cGH *>(cctkGH_)), grouptype(grouptype_), lc(-1) {
  assert(grouptype == CCTK_GF or grouptype == CCTK_ARRAY or
         grouptype == CCTK_SCALAR);
  assert(is_singlemap_mode());
  step();
}

local_component_iterator::~local_component_iterator() {
  leave_local_mode(cctkGH);
}

bool local_component_iterator::done() const {
  int const maxlc =
      (grouptype == CCTK_GF ? vhh.AT(map)->local_components(reflevel) : 1);
  return lc >= maxlc;
}

void local_component_iterator::step() {
  ++lc;
  if (not done()) {
    int const c =
        (grouptype == CCTK_GF ? vhh.AT(map)->get_component(reflevel, lc)
                              : dist::rank());
    leave_local_mode(cctkGH);
    enter_local_mode(cctkGH, c, lc, grouptype);
  }
}

//
// Mode escapes
//

// Singlemap escape

singlemap_escape::singlemap_escape(cGH const *const cctkGH_)
    : cctkGH(const_cast<cGH *>(cctkGH_)), c(component), lc(local_component) {
  assert(not is_meta_mode());
  assert(not is_global_mode());
  assert(not is_level_mode());
  if (not is_singlemap_mode()) {
    leave_local_mode(cctkGH);
  }
}

singlemap_escape::~singlemap_escape() {
  assert(is_singlemap_mode());
  if (c != -1) {
    enter_local_mode(cctkGH, c, lc, mc_grouptype);
  }
}

// Level escape

level_escape::level_escape(cGH const *const cctkGH_)
    : cctkGH(const_cast<cGH *>(cctkGH_)), grouptype(mc_grouptype), m(map),
      c(component), lc(local_component) {
  assert(not is_meta_mode());
  assert(not is_global_mode());
  if (not is_level_mode()) {
    if (not is_singlemap_mode()) {
      leave_local_mode(cctkGH);
    }
    leave_singlemap_mode(cctkGH);
  }
}

level_escape::~level_escape() {
  assert(is_level_mode());
  if (m != -1) {
    enter_singlemap_mode(cctkGH, m, grouptype);
    if (c != -1) {
      enter_local_mode(cctkGH, c, lc, grouptype);
    }
  }
}

// Global escape

global_escape::global_escape(cGH const *const cctkGH_)
    : cctkGH(const_cast<cGH *>(cctkGH_)), rl(reflevel), grouptype(mc_grouptype),
      m(map), c(component), lc(local_component) {
  assert(not is_meta_mode());
  if (not is_global_mode()) {
    if (not is_level_mode()) {
      if (not is_singlemap_mode()) {
        leave_local_mode(cctkGH);
      }
      leave_singlemap_mode(cctkGH);
    }
    leave_level_mode(cctkGH);
  }
}

global_escape::~global_escape() {
  assert(is_global_mode());
  if (rl != -1) {
    enter_level_mode(cctkGH, rl);
    if (m != -1) {
      enter_singlemap_mode(cctkGH, m, grouptype);
      if (c != -1) {
        enter_local_mode(cctkGH, c, lc, grouptype);
      }
    }
  }
}

// Meta escape

meta_escape::meta_escape(cGH const *const cctkGH_)
    : cctkGH(const_cast<cGH *>(cctkGH_)), ml(mglevel), rl(reflevel),
      grouptype(mc_grouptype), m(map), c(component), lc(local_component) {
  if (not is_meta_mode()) {
    if (not is_global_mode()) {
      if (not is_level_mode()) {
        if (not is_singlemap_mode()) {
          leave_local_mode(cctkGH);
        }
        leave_singlemap_mode(cctkGH);
      }
      leave_level_mode(cctkGH);
    }
    leave_global_mode(cctkGH);
  }
}

meta_escape::~meta_escape() {
  assert(is_meta_mode());
  if (ml != -1) {
    enter_global_mode(cctkGH, ml);
    if (rl != -1) {
      enter_level_mode(cctkGH, rl);
      if (m != -1) {
        enter_singlemap_mode(cctkGH, m, grouptype);
        if (c != -1) {
          enter_local_mode(cctkGH, c, lc, grouptype);
        }
      }
    }
  }
}

//
// Call functions in specific modes
//

int CallLocalFunction(cGH *const cctkGH,
                      void (*const function)(cGH *const cctkGH)) {
  if (is_meta_mode()) {
    BEGIN_MGLEVEL_LOOP(cctkGH) {
      BEGIN_REFLEVEL_LOOP(cctkGH) {
        BEGIN_LOCAL_MAP_LOOP(cctkGH, CCTK_GF) {
          BEGIN_LOCAL_COMPONENT_LOOP(cctkGH, CCTK_GF) { function(cctkGH); }
          END_LOCAL_COMPONENT_LOOP;
        }
        END_LOCAL_MAP_LOOP;
      }
      END_REFLEVEL_LOOP;
    }
    END_MGLEVEL_LOOP;
  } else if (is_global_mode()) {
    BEGIN_REFLEVEL_LOOP(cctkGH) {
      BEGIN_LOCAL_MAP_LOOP(cctkGH, CCTK_GF) {
        BEGIN_LOCAL_COMPONENT_LOOP(cctkGH, CCTK_GF) { function(cctkGH); }
        END_LOCAL_COMPONENT_LOOP;
      }
      END_LOCAL_MAP_LOOP;
    }
    END_REFLEVEL_LOOP;
  } else if (is_level_mode()) {
    BEGIN_LOCAL_MAP_LOOP(cctkGH, CCTK_GF) {
      BEGIN_LOCAL_COMPONENT_LOOP(cctkGH, CCTK_GF) { function(cctkGH); }
      END_LOCAL_COMPONENT_LOOP;
    }
    END_LOCAL_MAP_LOOP;
  } else if (is_singlemap_mode()) {
    BEGIN_LOCAL_COMPONENT_LOOP(cctkGH, CCTK_GF) { function(cctkGH); }
    END_LOCAL_COMPONENT_LOOP;
  } else {
    function(cctkGH);
  }
  return 0;
}

int CallSinglemapFunction(cGH *const cctkGH,
                          void (*const function)(cGH *const cctkGH)) {
  if (is_meta_mode()) {
    BEGIN_MGLEVEL_LOOP(cctkGH) {
      BEGIN_REFLEVEL_LOOP(cctkGH) {
        BEGIN_MAP_LOOP(cctkGH, CCTK_GF) { function(cctkGH); }
        END_MAP_LOOP;
      }
      END_REFLEVEL_LOOP;
    }
    END_MGLEVEL_LOOP;
  } else if (is_global_mode()) {
    BEGIN_REFLEVEL_LOOP(cctkGH) {
      BEGIN_MAP_LOOP(cctkGH, CCTK_GF) { function(cctkGH); }
      END_MAP_LOOP;
    }
    END_REFLEVEL_LOOP;
  } else if (is_level_mode()) {
    BEGIN_MAP_LOOP(cctkGH, CCTK_GF) { function(cctkGH); }
    END_MAP_LOOP;
  } else {
    BEGIN_SINGLEMAP_MODE(cctkGH) { function(cctkGH); }
    END_SINGLEMAP_MODE;
  }
  return 0;
}

int CallLevelFunction(cGH *const cctkGH,
                      void (*const function)(cGH *const cctkGH)) {
  if (is_meta_mode()) {
    BEGIN_MGLEVEL_LOOP(cctkGH) {
      BEGIN_REFLEVEL_LOOP(cctkGH) { function(cctkGH); }
      END_REFLEVEL_LOOP;
    }
    END_MGLEVEL_LOOP;
  } else if (is_global_mode()) {
    BEGIN_REFLEVEL_LOOP(cctkGH) { function(cctkGH); }
    END_REFLEVEL_LOOP;
  } else {
    BEGIN_LEVEL_MODE(cctkGH) { function(cctkGH); }
    END_LEVEL_MODE;
  }
  return 0;
}

int CallGlobalFunction(cGH *const cctkGH,
                       void (*const function)(cGH *const cctkGH)) {
  if (is_meta_mode()) {
    BEGIN_MGLEVEL_LOOP(cctkGH) { function(cctkGH); }
    END_MGLEVEL_LOOP;
  } else {
    BEGIN_GLOBAL_MODE(cctkGH) { function(cctkGH); }
    END_GLOBAL_MODE;
  }
  return 0;
}

int CallMetaFunction(cGH *const cctkGH,
                     void (*const function)(cGH *const cctkGH)) {
  BEGIN_META_MODE(cctkGH) { function(cctkGH); }
  END_META_MODE;
  return 0;
}

//
// Mode setters
//

// mglevel setter

mglevel_setter::mglevel_setter(cGH const *const cctkGH_, int const ml)
    : cctkGH(const_cast<cGH *>(cctkGH_)) {
  assert(is_meta_mode());
  enter_global_mode(cctkGH, ml);
}

mglevel_setter::~mglevel_setter() { leave_global_mode(cctkGH); }

// reflevel setter

reflevel_setter::reflevel_setter(cGH const *const cctkGH_, int const rl)
    : cctkGH(const_cast<cGH *>(cctkGH_)) {
  assert(is_global_mode());
  enter_level_mode(cctkGH, rl);
}

reflevel_setter::~reflevel_setter() { leave_level_mode(cctkGH); }

// map setter

map_setter::map_setter(cGH const *const cctkGH_, int const m,
                       int const grouptype)
    : cctkGH(const_cast<cGH *>(cctkGH_)) {
  assert(is_level_mode());
  enter_singlemap_mode(cctkGH, m, grouptype);
}

map_setter::~map_setter() { leave_singlemap_mode(cctkGH); }

// component setter

component_setter::component_setter(cGH const *const cctkGH_, int const c,
                                   int const grouptype)
    : cctkGH(const_cast<cGH *>(cctkGH_)) {
  assert(is_singlemap_mode());
  int const lc = (c != -1 ? (grouptype == CCTK_GF
                                 ? vhh.AT(map)->get_local_component(reflevel, c)
                                 : (c == dist::rank() ? 0 : -1))
                          : -1);
  enter_local_mode(cctkGH, c, lc, grouptype);
}

component_setter::~component_setter() { leave_local_mode(cctkGH); }

//
// Loop over a bounding box
//

void ibbox2iminimax(ibbox const &ext, // component extent
                    ibbox const &box, // this bbox
                    ivect &imin, ivect &imax) {
  assert(all((box.lower() - ext.lower()) >= 0));
  assert(all((box.upper() - ext.lower() + ext.stride()) >= 0));
  assert(all((box.lower() - ext.lower()) % ext.stride() == 0));
  assert(all((box.upper() - ext.lower() + ext.stride()) % ext.stride() == 0));

  imin = (box.lower() - ext.lower()) / ext.stride();
  imax = (box.upper() - ext.lower() + ext.stride()) / ext.stride();

  assert(all(0 <= imin));
  assert(box.empty() xor all(imin <= imax));
}

//
// Call a scheduling group
//

int CallScheduleGroup(cGH *const cctkGH, const char *const group) {
  CCTK_ScheduleTraverse(group, cctkGH, CallFunction);
  return 0;
}

extern "C" void CCTK_FCALL CCTK_FNAME(CallScheduleGroup)(
    int *const ierr, cGH *const *const cctkGH, ONE_FORTSTRING_ARG) {
  ONE_FORTSTRING_CREATE(group);
  *ierr = CallScheduleGroup(*cctkGH, group);
  free(group);
}

} // namespace Carpet

```

Poison.cc:
```
#include <cassert>
#include <cstdlib>
#include <cstring>

#include <cctk.h>
#include <cctk_Parameters.h>
#include <util_ErrorCodes.h>
#include <util_Table.h>

#include <Timer.hh>

#include <defs.hh>
#include <typeprops.hh>

#include <carpet.hh>

namespace Carpet {

using namespace std;

// The parameter where specifies which time levels should be
// poisoned.  what specifies what kind of grid variables should be
// poisoned.
void Poison(cGH const *const cctkGH, checktimes const where, int const what) {
  DECLARE_CCTK_PARAMETERS;

  assert(what == 0 or what == CCTK_GF or what == CCTK_ARRAY);

  if (not poison_new_timelevels)
    return;

  Timers::Timer timer("Poison");
  timer.start();

  for (int group = 0; group < CCTK_NumGroups(); ++group) {
    if (CCTK_QueryGroupStorageI(cctkGH, group)) {
      int const grouptype = CCTK_GroupTypeI(group);
      if (what == 0 or (what == CCTK_GF and grouptype == CCTK_GF) or
          (what == CCTK_ARRAY and
           (grouptype == CCTK_ARRAY or grouptype == CCTK_SCALAR))) {
        PoisonGroup(cctkGH, group, where);
      }
    } // if has storage
  }   // for group
  timer.stop();
}

void PoisonGroup(cGH const *const cctkGH, int const group,
                 checktimes const where) {
  DECLARE_CCTK_PARAMETERS;

  assert(group >= 0 and group < CCTK_NumGroups());

  if (not poison_new_timelevels)
    return;

  if (not CCTK_QueryGroupStorageI(cctkGH, group)) {
    char *const groupname = CCTK_GroupName(group);
    CCTK_VWarn(2, __LINE__, __FILE__, CCTK_THORNSTRING,
               "Cannot poison group \"%s\" because it has no storage",
               groupname);
    free(groupname);
    return;
  }

  int const nvar = CCTK_NumVarsInGroupI(group);
  if (nvar == 0)
    return;
  int const n0 = CCTK_FirstVarIndexI(group);
  assert(n0 >= 0);
  int const sz = CCTK_VarTypeSize(CCTK_VarTypeI(n0));
  assert(sz > 0);

  int const table = CCTK_GroupTagsTableI(group);
  assert(table >= 0);
  bool persistent;
  char buf[100];
  int const ilen = Util_TableGetString(table, sizeof buf, buf, "Persistent");
  if (ilen > 0) {
    if (CCTK_EQUALS(buf, "yes")) {
      persistent = true;
    } else if (CCTK_EQUALS(buf, "no")) {
      persistent = false;
    } else {
      assert(0);
    }
  } else if (ilen == UTIL_ERROR_TABLE_NO_SUCH_KEY) {
    // default
    persistent = true;
  } else {
    assert(0);
  }

  int const num_tl = CCTK_ActiveTimeLevelsVI(cctkGH, n0);
  assert(num_tl > 0);
  int const min_tl = min_timelevel(where, num_tl, persistent);
  int const max_tl = max_timelevel(where, num_tl, persistent);

  if (min_tl <= max_tl) {

    {
      char *const groupname = CCTK_GroupName(group);
      Checkpoint("PoisonGroup \"%s\"", groupname);
      free(groupname);
    }

    CCTK_INT const poison_value = get_poison_value();

    int const grouptype = CCTK_GroupTypeI(group);

    BEGIN_LOCAL_MAP_LOOP(cctkGH, grouptype) {
      BEGIN_LOCAL_COMPONENT_LOOP(cctkGH, grouptype) {

        ivect size(1);
        int const gpdim = groupdata.AT(group).info.dim;
        for (int d = 0; d < gpdim; ++d) {
          size[d] = groupdata.AT(group).info.ash[d];
        }
        int const np = prod(size);

        for (int var = 0; var < nvar; ++var) {
          int const n = n0 + var;
          for (int tl = min_tl; tl <= max_tl; ++tl) {
            memset(cctkGH->data[n][tl], poison_value, size_t(np) * sz);
          } // for tl
        }   // for var
      }
      END_LOCAL_COMPONENT_LOOP;
    }
    END_LOCAL_MAP_LOOP;

  } // if tl
}

void PoisonCheck(cGH const *const cctkGH, checktimes const where) {
  DECLARE_CCTK_PARAMETERS;

  if (not check_for_poison)
    return;

  Checkpoint("PoisonCheck");
  Timers::Timer timer("PoisonCheck");
  timer.start();

  for (int group = 0; group < CCTK_NumGroups(); ++group) {
    int const nvar = CCTK_NumVarsInGroupI(group);
    if (nvar > 0 and CCTK_QueryGroupStorageI(cctkGH, group)) {

      int const grouptype = CCTK_GroupTypeI(group);
      int const n0 = CCTK_FirstVarIndexI(group);
      assert(n0 >= 0);
      int const tp = CCTK_VarTypeI(n0);
      int const gpdim = groupdata.AT(group).info.dim;

      int const table = CCTK_GroupTagsTableI(group);
      assert(table >= 0);
      bool persistent;
      char buf[100];
      int const ilen =
          Util_TableGetString(table, sizeof buf, buf, "Persistent");
      if (ilen > 0) {
        if (CCTK_EQUALS(buf, "yes")) {
          persistent = true;
        } else if (CCTK_EQUALS(buf, "no")) {
          persistent = false;
        } else {
          assert(0);
        }
      } else if (ilen == UTIL_ERROR_TABLE_NO_SUCH_KEY) {
        // default
        persistent = true;
      } else {
        assert(0);
      }

      CCTK_INT const poison_value = get_poison_value();

      int const num_tl = CCTK_ActiveTimeLevelsVI(cctkGH, n0);
      assert(num_tl > 0);
      int const min_tl = min_timelevel(where, num_tl, persistent);
      int const max_tl = max_timelevel(where, num_tl, persistent);

      BEGIN_LOCAL_MAP_LOOP(cctkGH, grouptype) {
        BEGIN_LOCAL_COMPONENT_LOOP(cctkGH, grouptype) {

          ivect size(1), asize(1);
          for (int d = 0; d < gpdim; ++d) {
            size[d] = groupdata.AT(group).info.lsh[d];
            asize[d] = groupdata.AT(group).info.ash[d];
          }
          int const np = prod(size);

          for (int var = 0; var < nvar; ++var) {
            int const n = n0 + var;

            for (int tl = min_tl; tl <= max_tl; ++tl) {

              const void *const data = cctkGH->data[n][tl];
              int numpoison = 0;
              for (int k = 0; k < size[2]; ++k) {
                for (int j = 0; j < size[1]; ++j) {
                  for (int i = 0; i < size[0]; ++i) {
                    int const idx = i + asize[0] * (j + asize[1] * k);
                    bool poisoned = false;
                    switch (specific_cactus_type(tp)) {
#define TYPECASE(N, T)                                                         \
  case N: {                                                                    \
    T worm;                                                                    \
    memset(&worm, poison_value, sizeof worm);                                  \
    const T &val = ((const T *)data)[idx];                                     \
    poisoned = memcmp(&worm, &val, sizeof worm) == 0;                          \
    break;                                                                     \
  }
#include "typecase.hh"
#undef TYPECASE
                    default:
                      UnsupportedVarType(n);
                    }
                    if (poisoned) {
                      ++numpoison;
                      if (max_poison_locations == -1 or
                          numpoison <= max_poison_locations) {
                        char *fullname = CCTK_FullName(n);
                        CCTK_VWarn(1, __LINE__, __FILE__, CCTK_THORNSTRING,
                                   "At iteration %d: timelevel %d, component "
                                   "%d, map %d, refinement level %d of the "
                                   "variable \"%s\" contains poison at "
                                   "[%d,%d,%d]",
                                   cctkGH->cctk_iteration, tl, component, map,
                                   reflevel, fullname, i, j, k);
                        free(fullname);
                      }
                    } // if poisoned
                  }   // for i
                }     // for j
              }       // for k
              if (max_poison_locations != -1 and
                  numpoison > max_poison_locations) {
                char *fullname = CCTK_FullName(n);
                CCTK_VWarn(1, __LINE__, __FILE__, CCTK_THORNSTRING,
                           "At iteration %d: timelevel %d, component %d, map "
                           "%d, refinement level %d of the variable \"%s\" "
                           "contains poison at %d of %d locations; not all "
                           "locations were printed",
                           cctkGH->cctk_iteration, tl, component, map, reflevel,
                           fullname, numpoison, np);
                free(fullname);
              } else if (numpoison > 0) {
                CCTK_VWarn(1, __LINE__, __FILE__, CCTK_THORNSTRING,
                           "Found poison at %d of %d locations", numpoison, np);
              }

            } // for tl
          }   // for var
        }
        END_LOCAL_COMPONENT_LOOP;
      }
      END_LOCAL_MAP_LOOP;

    } // if has storage
  }   // for group
  timer.stop();
}

} // namespace Carpet

```

carpet.hh:
```
#ifndef CARPET_HH
#define CARPET_HH

#include <vector>

#include <cctk.h>
#include <cctk_Arguments.h>
#include <cctk_Functions.h>
#include <cctk_Schedule.h>

#include <gh.hh>

#include "carpet_public.hh"

namespace Carpet {

using namespace std;

// Scheduled functions
extern "C" {
int CarpetStartup(void);
int CarpetMultiModelStartup(void);
void CarpetParamCheck(CCTK_ARGUMENTS);
void CarpetRefineTimeStep(CCTK_ARGUMENTS);
void CarpetUnusedMask(CCTK_ARGUMENTS);
}

// Registered functions
void *SetupGH(tFleshConfig *fc, int convLevel, cGH *cgh);

int Initialise(tFleshConfig *config);
int Evolve(tFleshConfig *config);
int Shutdown(tFleshConfig *config);
int OutputGH(const cGH *cgh);

int CallFunction(void *function, cFunctionData *attribute, void *data);

// Other functions
bool Regrid(cGH const *cctkGH, bool force_recompose, bool do_init);

void CycleTimeLevels(cGH *cgh);
void UncycleTimeLevels(cGH *cgh);
void FlipTimeLevels(cGH *cgh);
void FillTimeLevels(const cGH *cgh);
void SyncGroups(const cGH *cgh, const vector<int> &groups);
int SyncProlongateGroups(const cGH *cgh, const vector<int> &groups,
                         cFunctionData const *function_data = NULL);

// Sanity checks
enum checktimes {
  currenttime,
  currenttimebutnotifonly,
  previoustime,
  allbutlasttime,
  allbutcurrenttime,
  alltimes
};

int min_timelevel(checktimes where, int num_tl,
                  bool persistent) CCTK_ATTRIBUTE_CONST;
int max_timelevel(checktimes where, int num_tl,
                  bool persistent) CCTK_ATTRIBUTE_CONST;

void Poison(const cGH *cgh, checktimes where, int what = 0);
void PoisonGroup(const cGH *cgh, int group, checktimes where);
void PoisonCheck(const cGH *cgh, checktimes where);

void CalculateChecksums(const cGH *cgh, checktimes where);
void CheckChecksums(const cGH *cgh, checktimes where);

// Schedule
int CallBeforeRoutines(cGH const *cctkGH, void *const function,
                       cFunctionData *const attribute, void *const data);
int CallAfterRoutines(cGH const *cctkGH, void *const function,
                      cFunctionData *const attribute, void *const data);

// Debugging output
void Output(const char *fmt, ...);
void Waypoint(const char *fmt, ...);
void Checkpoint(const char *fmt, ...);

// Error output
void UnsupportedVarType(int vindex);

// Check for a map0group
bool IsMap0Group(int gindex);

} // namespace Carpet

#endif // !defined(CARPET_HH)

```

Shutdown.cc:
```
#include <cstdio>
#include <cstdlib>

#include <cctk.h>
#include <cctk_Parameters.h>

#include <CactusTimerSet.hh>
#include <Timer.hh>

#include <dist.hh>

#include <carpet.hh>

namespace Carpet {

using namespace std;

int Shutdown(tFleshConfig *fc) {
  DECLARE_CCTK_PARAMETERS;

  Waypoint("Starting shutdown");

  const int convlev = 0;
  cGH *cctkGH = fc->GH[convlev];

  static Timers::Timer timer("Shutdown");
  timer.start();
  for (int rl = reflevels - 1; rl >= 0; --rl) {
    BEGIN_REVERSE_MGLEVEL_LOOP(cctkGH) {
      ENTER_LEVEL_MODE(cctkGH, rl) {
        BeginTimingLevel(cctkGH);

        do_early_global_mode = reflevel == reflevels - 1;
        do_late_global_mode = reflevel == 0;
        do_early_meta_mode = do_early_global_mode and mglevel == 0;
        do_late_meta_mode = do_late_global_mode and mglevel == mglevels - 1;
        do_global_mode = do_late_global_mode;
        do_meta_mode = do_late_meta_mode;

        Checkpoint("Shutdown at iteration %d time %g%s%s",
                   cctkGH->cctk_iteration, (double)cctkGH->cctk_time,
                   (do_global_mode ? " (global)" : ""),
                   (do_meta_mode ? " (meta)" : ""));

        // Terminate
        Checkpoint("Scheduling TERMINATE");
        CCTK_ScheduleTraverse("CCTK_TERMINATE", cctkGH, CallFunction);

        EndTimingLevel(cctkGH);
      }
      LEAVE_LEVEL_MODE;
    }
    END_REVERSE_MGLEVEL_LOOP;
  } // for rl

  // Stop all timers before shutdown, since timers may rely on data
  // structures which are destroyed during shutdown
  int const ierr = CCTK_TimerStop("CCTK total time");
  assert(not ierr);
  timer.stop();
  if (output_timers_every > 0) {
    Timers::CactusTimerSet::writeData(cctkGH, timer_file);
  }

  if (output_timer_tree_every > 0) {
    Timers::Timer::outputTree("Evolve");
  }

  if (output_xml_timer_tree) {
    Timers::Timer::outputTreeXML();
  }

  BEGIN_REVERSE_MGLEVEL_LOOP(cctkGH) {
    do_early_global_mode = true;
    do_late_global_mode = true;
    do_early_meta_mode = do_early_global_mode and mglevel == 0;
    do_late_meta_mode = do_late_global_mode and mglevel == mglevels - 1;
    do_global_mode = do_late_global_mode;
    do_meta_mode = do_late_meta_mode;

    // Shutdown
    Checkpoint("Scheduling SHUTDOWN");
    CCTK_ScheduleTraverse("CCTK_SHUTDOWN", cctkGH, CallFunction);
  }
  END_REVERSE_MGLEVEL_LOOP;

  // Free all memory, call all destructors
  for (size_t gi = 0; gi < arrdata.size(); ++gi) {
    for (size_t m = 0; m < arrdata.AT(gi).size(); ++m) {
      for (size_t vi = 0; vi < arrdata.AT(gi).AT(m).data.size(); ++vi) {
        ggf *&f = arrdata.AT(gi).AT(m).data.AT(vi);
        if (f) {
          delete f;
          f = 0;
        }
      }
    }
  }
  // for (int gi=0; gi<CCTK_NumGroups(); ++gi) {
  //   const int tls = 0;
  //   CCTK_GroupStorageDecrease(cctkGH, 1, &gi, &tls, 0);
  // }

  for (size_t gi = 0; gi < arrdata.size(); ++gi) {
    if (CCTK_GroupTypeI(gi) == CCTK_GF) {
      for (size_t m = 0; m < arrdata.AT(gi).size(); ++m) {
        arrdata.AT(gi).AT(m).tt = 0;
        arrdata.AT(gi).AT(m).dd = 0;
        arrdata.AT(gi).AT(m).hh = 0;
      }
    } else {
      for (size_t m = 0; m < arrdata.AT(gi).size(); ++m) {
        delete arrdata.AT(gi).AT(m).tt;
        arrdata.AT(gi).AT(m).tt = 0;
        delete arrdata.AT(gi).AT(m).dd;
        arrdata.AT(gi).AT(m).dd = 0;
        delete arrdata.AT(gi).AT(m).hh;
        arrdata.AT(gi).AT(m).hh = 0;
      }
    }
  }

  delete tt;
  tt = 0;
  for (size_t m = 0; m < vdd.size(); ++m) {
    delete vdd.AT(m);
    vdd.AT(m) = 0;
  }
  for (size_t m = 0; m < vhh.size(); ++m) {
    delete vhh.AT(m);
    vhh.AT(m) = 0;
  }

  // earlier checkpoint before finalising MPI
  Waypoint("Done with shutdown");

  return 0;
}

} // namespace Carpet

```

MultiModel.cc:
```
#include <cctk.h>

#include <cassert>
#include <cstring>
#include <iostream>
#include <map>
#include <string>
#include <vector>

#ifdef CCTK_MPI
#include <mpi.h>
#else
#include "nompi.h"
#endif

#include <functions.hh>
#include <mpi_string.hh>

namespace Carpet {

using namespace std;
using namespace CarpetLib;

vector<string> model_names;       // Model id to model name
std::map<string, int> model_map;  // Model name to model id
vector<int> model_ids;            // Process to model id
vector<vector<int> > model_procs; // Model id to processes

vector<string> const &ModelNames() { return model_names; }
std::map<string, int> const &ModelMap() { return model_map; }
vector<int> const &ModelIds() { return model_ids; }
vector<vector<int> > const &ModelProcs() { return model_procs; }

string ModelName(int const id) { return model_names.AT(id); }

int ModelId(string const name) {
  if (model_map.find(name) != model_map.end()) {
    return model_map[name];
  } else {
    return -1;
  }
}

int ModelId(int const proc) { return model_ids.AT(proc); }

vector<int> const &ModelProcs(int const id) { return model_procs.AT(id); }

void SplitUniverse(MPI_Comm const world, string const model, MPI_Comm &comm,
                   bool const verbose) {
  // Get the total number of processes
  int num_procs;
  MPI_Comm_size(world, &num_procs);
  int my_proc;
  MPI_Comm_rank(world, &my_proc);

  // Gather all model names
  vector<string> const models(allgather_string(world, model));

  // Map model strings to small integers
  int num_models = 0;
  model_ids.resize(num_procs);
  model_map.clear();
  for (int n = 0; n < num_procs; ++n) {
    if (model_map.find(models.AT(n)) != model_map.end()) {
      model_ids.AT(n) = model_map[models.AT(n)];
    } else {
      model_map[models.AT(n)] = num_models;
      model_ids.AT(n) = num_models;
      ++num_models;
    }
  }

  // Determine processes per model
  vector<int> num_model_procs(num_models, 0);
  for (int n = 0; n < num_procs; ++n) {
    ++num_model_procs.AT(model_ids.AT(n));
  }

  model_names.resize(num_models);
  model_procs.resize(num_models);
  for (int m = 0; m < num_models; ++m) {
    model_procs.AT(m).reserve(num_model_procs.AT(m));
  }
  for (int n = 0; n < num_procs; ++n) {
    model_names.AT(model_ids.AT(n)) = models.AT(n);
    model_procs.AT(model_ids.AT(n)).push_back(n);
  }
  for (int m = 0; m < num_models; ++m) {
    assert(static_cast<int>(model_procs.AT(m).size()) == num_model_procs.AT(m));
  }

  // Create a new communicator for each model
  MPI_Comm_split(world, model_ids.AT(my_proc), my_proc, &comm);

  if (verbose) {
    CCTK_INFO("Multi-Model listing:");
    for (int m = 0; m < num_models; ++m) {
      cout << "   model " << m << ": \"" << model_names.AT(m) << "\"" << endl;
    }
    CCTK_INFO("Multi-Model process distribution:");
    for (int n = 0; n < num_procs; ++n) {
      int const m = model_ids.AT(n);
      bool const same_model_as_prev = n - 1 >= 0 and model_ids.AT(n - 1) == m;
      bool const same_model_as_next =
          n + 1 < num_procs and model_ids.AT(n + 1) == m;
      if (same_model_as_next) {
        if (same_model_as_prev) {
          // Output nothing
        } else {
          // This process has the same model as the next one:
          // output only a partial line
          cout << "   processes " << n << "-";
        }
      } else {
        if (same_model_as_prev) {
          // This process has the same model as the previous one:
          // finish a partial line
          cout << n << ": "
               << "model " << m << " \"" << model_names.AT(m) << "\"" << endl;
        } else {
          cout << "   process " << n << ": "
               << "model " << m << " \"" << model_names.AT(m) << "\"" << endl;
        }
      }
    }
    int const my_model = model_ids.AT(my_proc);
    CCTK_VInfo(CCTK_THORNSTRING,
               "Multi-Model: This is process %d, model %d \"%s\"", my_proc,
               my_model, model.c_str());
  }
}

} // namespace Carpet

```

adler32.hh:
```
// ADLER-32 checksum algorithm

#ifndef ADLER32_HH
#define ADLER32_HH

#include <cstdlib>

namespace Carpet {

unsigned long adler32(char const *const data, // location of the data
                      std::size_t const len); // length of the data in bytes
}

#endif // #ifndef ADLER32_HH

```

functions.hh:
```
#ifndef FUNCTIONS_HH
#define FUNCTIONS_HH

#include <cctk.h>
#include <cctk_Schedule.h>

#include <map>
#include <string>
#include <vector>

#ifdef CCTK_MPI
#include <mpi.h>
#else
#include "nompi.h"
#endif

#include <bbox.hh>
#include <dh.hh>
#include <gh.hh>
#include <vect.hh>

namespace Carpet {

using namespace std;

int SyncGroupsByDirI(const cGH *cctkGH, int num_groups, const int *groups,
                     const int *directions);
int EnableGroupComm(const cGH *cgh, const char *groupname);
int DisableGroupComm(const cGH *cgh, const char *groupname);
int EnableGroupStorage(const cGH *cgh, const char *groupname);
int DisableGroupStorage(const cGH *cgh, const char *groupname);
int GroupStorageIncrease(const cGH *cgh, int n_groups, const int *groups,
                         const int *timelevels, int *status);
int GroupStorageDecrease(const cGH *cgh, int n_groups, const int *groups,
                         const int *timelevels, int *status);
int QueryMaxTimeLevels(const cGH *cgh, int n_groups, const int *groups,
                       int *status);
int Barrier(const cGH *cgh);
int NamedBarrier(const cGH *cgh, unsigned int id, const char *name);
int Exit(const cGH *cgh, int retval);
int Abort(const cGH *cgh, int retval);
int MyProc(const cGH *cgh) CCTK_ATTRIBUTE_PURE;
int nProcs(const cGH *cgh) CCTK_ATTRIBUTE_PURE;
const int *ArrayGroupSizeB(const cGH *cgh, int dir, int group,
                           const char *groupname) CCTK_ATTRIBUTE_PURE;
int QueryGroupStorageB(const cGH *cgh, int group,
                       const char *groupname) CCTK_ATTRIBUTE_PURE;
int GroupDynamicData(const cGH *cgh, int group, cGroupDynamicData *data);

void Restrict(const cGH *cgh);

// Multi-Model
void SplitUniverse(MPI_Comm const world, string const model, MPI_Comm &comm,
                   bool verbose);

// Model id to model name
vector<string> const &ModelNames();
string ModelName(int id);

// Model name to model id
std::map<string, int> const &ModelMap();
int ModelId(string name);

// Processor to model id
vector<int> const &ModelIds();
int ModelId(int proc);

// Model id to processes
vector<vector<int> > const &ModelProcs();
vector<int> const &ModelProcs(int id);

// Host mapping
void DetermineHosts(string host, bool verbose);

// Host id to host name
vector<string> const &HostNames();
string HostName(int id);

// Host name to host id
std::map<string, int> const &HostMap();
int HostId(string name);

// Process to host id
vector<int> const &HostIds();
int HostId(int proc);

// Host id to processes
vector<vector<int> > const &HostProcs();
vector<int> const &HostProcs(int id);

extern "C" {
CCTK_POINTER_TO_CONST
Carpet_GetMPICommUniverse(CCTK_POINTER_TO_CONST cctkGH);
CCTK_POINTER_TO_CONST
Carpet_GetMPICommWorld(CCTK_POINTER_TO_CONST cctkGH);
CCTK_INT
Carpet_GetCoordRange(CCTK_POINTER_TO_CONST const cctkGH_, CCTK_INT const m,
                     CCTK_INT const ml, CCTK_INT const size,
                     CCTK_INT *const gsh, CCTK_REAL *const lower,
                     CCTK_REAL *const upper, CCTK_REAL *const delta);
}

// Helpers for storage
void GroupsStorageCheck(cGH const *const cctkGH);

// Helpers for recomposing the grid hierarchy
void RegridMap(cGH const *cctkGH, int m, gh::rregs const &supeerregss,
               gh::mregs const &regsss, bool do_init);
void PostRegrid(cGH const *cctkGH);
bool Recompose(cGH const *cctkGH, int rl, bool do_init);
void RegridFree(cGH const *cctkGH, bool do_init);

void CheckRegions(gh::mregs const &regsss);

void OutputSuperregions(cGH const *cctkGH, int m, gh const &hh,
                        gh::rregs const &superregss);
void OutputGrids(cGH const *cctkGH, int const m, gh const &hh, dh const &dd);

void OutputGridStructure(cGH const *cctkGH, int const m,
                         gh::mregs const &regsss);

void OutputGridCoordinates(cGH const *cctkGH, int const m,
                           gh::mregs const &regsss);

void OutputGridStatistics(cGH const *cctkGH);

// Functions for recomposing the grid hierarchy
void SplitRegions(cGH const *cctkGH, vector<region_t> &superregs,
                  vector<region_t> &regs);
void SplitRegions_AlongZ(cGH const *cctkGH, vector<region_t> &superregs,
                         vector<region_t> &regs);
void SplitRegions_AlongDir(cGH const *cctkGH, vector<region_t> &superregs,
                           vector<region_t> &regs, int dir);
void SplitRegions_Automatic(cGH const *cctkGH, vector<region_t> &superregs,
                            vector<region_t> &regs,
                            bvect const &no_split_dims = false);
void SplitRegionsMaps_Recursively(cGH const *cctkGH,
                                  vector<vector<region_t> > &superregss,
                                  vector<vector<region_t> > &regss);

void SplitRegionsMaps(cGH const *cctkGH, vector<vector<region_t> > &superregss,
                      vector<vector<region_t> > &regss);
void SplitRegionsMaps_Automatic(cGH const *cctkGH,
                                vector<vector<region_t> > &superregss,
                                vector<vector<region_t> > &regss,
                                bvect const &no_split_dims = false);
void SplitRegionsMaps_Recursively(cGH const *cctkGH,
                                  vector<vector<region_t> > &superregss,
                                  vector<vector<region_t> > &regss);
void SplitRegionsMaps_Balanced(cGH const *cctkGH,
                               vector<vector<region_t> > &superregss,
                               vector<vector<region_t> > &regss);

void MakeMultigridBoxes(cGH const *cctkGH, int m, gh::rregs const &regss,
                        gh::mregs &regsss);

void MakeMultigridBoxesMaps(cGH const *cctkGH, vector<gh::rregs> const &regsss,
                            vector<gh::mregs> &regssss);

// Timing statistics functions
void InitTimingStats(cGH const *cctkGH);
void BeginTimingEvolution(cGH const *cctkGH);
void StepTimingEvolution(cGH const *cctkGH);
void BeginTimingLevel(cGH const *cctkGH);
void EndTimingLevel(cGH const *cctkGH);
void BeginTimingIO(cGH const *cctkGH);
void EndTimingIO(cGH const *cctkGH, CCTK_REAL files, CCTK_REAL bytes,
                 bool is_binary);
void BeginTimingCommunication(cGH const *cctkGH);
void EndTimingCommunication(cGH const *cctkGH, CCTK_REAL messages,
                            CCTK_REAL bytes);
void UpdateTimingStats(cGH const *cctkGH);
void PrintTimingStats(cGH const *cctkGH);

} // namespace Carpet

#endif // !defined(FUNCTIONS_HH)

```

Hosts.cc:
```
#include <cctk.h>

#include <cassert>
#include <cstring>
#include <iostream>
#include <map>
#include <string>
#include <vector>

#ifdef CCTK_MPI
#include <mpi.h>
#else
#include "nompi.h"
#endif

#include <dist.hh>
#include <functions.hh>
#include <mpi_string.hh>

namespace Carpet {

using namespace std;
using namespace CarpetLib;

vector<string> host_names;       // Host id to host name
std::map<string, int> host_map;  // Host name to host id
vector<int> host_ids;            // Process to host id
vector<vector<int> > host_procs; // Host id to processes

vector<string> const &HostNames() { return host_names; }
std::map<string, int> const &HostMap() { return host_map; }
vector<int> const &HostIds() { return host_ids; }
vector<vector<int> > const &HostProcs() { return host_procs; }

string HostName(int const id) { return host_names.AT(id); }

int HostId(string const name) {
  if (host_map.find(name) != host_map.end()) {
    return host_map[name];
  } else {
    return -1;
  }
}

int HostId(int const proc) { return host_ids.AT(proc); }

vector<int> const &HostProcs(int const id) { return host_procs.AT(id); }

void DetermineHosts(string const host, bool const verbose) {
  // Get the total number of processes
  int num_procs;
  MPI_Comm_size(dist::comm(), &num_procs);
  int my_proc;
  MPI_Comm_rank(dist::comm(), &my_proc);

  // Gather all host names
  vector<string> const hosts(allgather_string(dist::comm(), host));

  // Map host strings to small integers
  int num_hosts = 0;
  host_ids.resize(num_procs);
  host_map.clear();
  for (int n = 0; n < num_procs; ++n) {
    if (host_map.find(hosts.AT(n)) != host_map.end()) {
      host_ids.AT(n) = host_map[hosts.AT(n)];
    } else {
      host_map[hosts.AT(n)] = num_hosts;
      host_ids.AT(n) = num_hosts;
      ++num_hosts;
    }
  }

  // Determine processes per host
  vector<int> num_host_procs(num_hosts, 0);
  for (int n = 0; n < num_procs; ++n) {
    ++num_host_procs.AT(host_ids.AT(n));
  }

  host_names.resize(num_hosts);
  host_procs.resize(num_hosts);
  for (int m = 0; m < num_hosts; ++m) {
    host_procs.AT(m).reserve(num_host_procs.AT(m));
  }
  for (int n = 0; n < num_procs; ++n) {
    host_names.AT(host_ids.AT(n)) = hosts.AT(n);
    host_procs.AT(host_ids.AT(n)).push_back(n);
  }
  for (int m = 0; m < num_hosts; ++m) {
    assert(static_cast<int>(host_procs.AT(m).size()) == num_host_procs.AT(m));
  }

  if (verbose) {
    CCTK_INFO("Host listing:");
    for (int m = 0; m < num_hosts; ++m) {
      cout << "   host " << m << ": \"" << host_names.AT(m) << "\"" << endl;
    }
    CCTK_INFO("Host/process mapping:");
    for (int n = 0; n < num_procs; ++n) {
      int const m = host_ids.AT(n);
      bool const same_host_as_prev = n - 1 >= 0 and host_ids.AT(n - 1) == m;
      bool const same_host_as_next =
          n + 1 < num_procs and host_ids.AT(n + 1) == m;
      if (same_host_as_next) {
        if (same_host_as_prev) {
          // Output nothing
        } else {
          // This process has the same host as the next one:
          // output only a partial line
          cout << "   processes " << n << "-";
        }
      } else {
        if (same_host_as_prev) {
          // This process has the same host as the previous one:
          // finish a partial line
          cout << n << ": "
               << "host " << m << " \"" << host_names.AT(m) << "\"" << endl;
        } else {
          cout << "   process " << n << ": "
               << "host " << m << " \"" << host_names.AT(m) << "\"" << endl;
        }
      }
    }
    int const my_host = host_ids.AT(my_proc);
    CCTK_VInfo(CCTK_THORNSTRING,
               "Host mapping: This is process %d, host %d \"%s\"", my_proc,
               my_host, host.c_str());
  }
}

} // namespace Carpet

```

Recompose.cc:
```
#include <cctk.h>
#include <cctk_Parameters.h>

#include <algorithm>
#include <cassert>
#include <cmath>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <fstream>
#include <iomanip>
#include <iostream>
#include <limits>
#include <list>
#include <sstream>
#include <string>
#include <vector>

#include <sys/stat.h>
#include <sys/types.h>

#ifdef CCTK_MPI
#include <mpi.h>
#else
#include "nompi.h"
#endif

#include <Timer.hh>

#include <loopcontrol.h>

#include <balance.hh>
#include <bbox.hh>
#include <bboxset.hh>
#include <defs.hh>
#include <dh.hh>
#include <gh.hh>
#include <region.hh>
#include <vect.hh>

#include <carpet.hh>
#include <modes.hh>
#include <variables.hh>

namespace Carpet {

using namespace std;

// Helper routines for spliting regions automatically

// The cost for a region, assuming a cost of 1 per interior point
static rvect cost(region_t const &reg) {
  DECLARE_CCTK_PARAMETERS;
  static rvect costfactor;
  static bool initialised = false;
  if (not initialised) {
    costfactor = rvect(1.0);
    if (dim > 0)
      costfactor[0] = 1.0 / aspect_ratio_x;
    if (dim > 1)
      costfactor[1] = 1.0 / aspect_ratio_y;
    if (dim > 2)
      costfactor[2] = 1.0 / aspect_ratio_z;
  }
  if (reg.extent.empty())
    return rvect(0);
  return rvect(reg.extent.shape() / reg.extent.stride()) * costfactor;
}

static void ClassifyPoints(cGH const *cctkGH, int rl);

static void SplitRegionsMaps_Automatic_Recursively(bvect const &dims,
                                                   int const firstproc,
                                                   int const nprocs,
                                                   region_t &superreg,
                                                   vector<region_t> &newregs);
static void SplitRegionsMaps_AsSpecified(cGH const *cctkGH,
                                         vector<vector<region_t> > &superregss,
                                         vector<vector<region_t> > &regss);
static void SplitRegions_AsSpecified(cGH const *cctkGH,
                                     vector<region_t> &superregs,
                                     vector<region_t> &regs);

void CheckRegions(gh::mregs const &regsss) {
  char const *const where = "CheckRegions";
  static Timers::Timer timer(where);
  timer.start();

  // At least one multigrid level
  if (regsss.size() == 0) {
    CCTK_WARN(0,
              "I cannot set up a grid hierarchy with zero multigrid levels.");
  }
  assert(regsss.size() > 0);
  // At least one refinement level
  if (regsss.AT(0).size() == 0) {
    CCTK_WARN(0,
              "I cannot set up a grid hierarchy with zero refinement levels.");
  }
  assert(regsss.AT(0).size() > 0);
  // At most maxreflevels levels
  if ((int)regsss.AT(0).size() > maxreflevels) {
    CCTK_VWarn(0, __LINE__, __FILE__, CCTK_THORNSTRING,
               "I cannot set up a grid hierarchy with more than "
               "Carpet::max_refinement_levels refinement levels.  I found "
               "Carpet::max_refinement_levels=%d, while %d levels were "
               "requested.",
               (int)maxreflevels, (int)regsss.AT(0).size());
  }
  assert((int)regsss.AT(0).size() <= maxreflevels);
  for (int ml = 0; ml < (int)regsss.size(); ++ml) {
    int num_regions = 0;
    for (int rl = 0; rl < (int)regsss.AT(0).size(); ++rl) {
      // No empty levels
      // (but allow some empty maps)
      // assert (regsss.AT(ml).AT(rl).size() > 0);
      num_regions += regsss.AT(ml).AT(rl).size();
      for (int c = 0; c < (int)regsss.AT(ml).AT(rl).size(); ++c) {
        // Check sizes
        // (but allow processors with zero grid points)
        // assert (all(regsss.AT(rl).AT(c).AT(ml).extent.lower() <=
        //             regsss.AT(rl).AT(c).AT(ml).extent.upper()));
        // Check strides
        ivect const str =
            (maxspacereflevelfact / spacereffacts.AT(rl) * ipow(mgfact, ml));
        assert(all(regsss.AT(ml).AT(rl).AT(c).extent.stride() % str == 0));
        // Check alignments
        assert(all(regsss.AT(ml).AT(rl).AT(c).extent.lower() % str == 0));
        assert(all(regsss.AT(ml).AT(rl).AT(c).extent.upper() % str == 0));
      }
    }
    // No empty levels
    assert(num_regions > 0);
  }

  timer.stop();
}

bool Regrid(cGH const *const cctkGH, bool const force_recompose,
            bool const do_init) {
  DECLARE_CCTK_PARAMETERS;

  char const *const where = "Regrid";
  static Timers::Timer timer(where);
  timer.start();

  Checkpoint("Regridding level %d...", reflevel);

  assert(is_level_mode());

  bool const have_regrid = CCTK_IsFunctionAliased("Carpet_Regrid");
  bool const have_regridmaps = CCTK_IsFunctionAliased("Carpet_RegridMaps");
  bool const use_regridmaps = regrid_in_level_mode and have_regridmaps;

  if (not use_regridmaps and not have_regrid) {
    static bool didtell = false;
    if (maxreflevels > 1 and not didtell) {
      CCTK_WARN(2, "The regridding routine Carpet_Regrid has not been "
                   "provided.  There will be no regridding.  Maybe you forgot "
                   "to activate a regridding thorn?");
      didtell = true;
    }
    timer.stop();
    return false;
  }

  if (regrid_in_level_mode and not have_regridmaps) {
    static bool didtell = false;
    if (maps > 1 and maxreflevels > 1 and not didtell) {
      CCTK_WARN(2, "The regridding routine Carpet_RegridMaps has not been "
                   "provided.  Regridding will be performed in singlemap mode "
                   "instead of level mode.");
      didtell = true;
    }
  }

  bool did_change = false;

  if (not use_regridmaps) {

    BEGIN_MAP_LOOP(cctkGH, CCTK_GF) {

      gh::rregs superregss = vhh.AT(map)->superregions;
      gh::mregs regsss;

      // Check whether to recompose
      CCTK_INT const do_recompose =
          Carpet_Regrid(cctkGH, &superregss, &regsss, force_recompose);
      assert(do_recompose >= 0);
      did_change = did_change or do_recompose;

      if (do_recompose) {
        RegridMap(cctkGH, map, superregss, regsss, do_init);
      }
    }
    END_MAP_LOOP;

  } else {

    vector<gh::rregs> superregsss(maps);
    vector<gh::mregs> regssss(maps);
    BEGIN_MAP_LOOP(cctkGH, CCTK_GF) {
      superregsss.AT(map) = vhh.AT(map)->superregions;
    }
    END_MAP_LOOP;

    // Check whether to recompose
    CCTK_INT const do_recompose =
        Carpet_RegridMaps(cctkGH, &superregsss, &regssss, force_recompose);
    assert(do_recompose >= 0);
// TODO
#if 1 // #ifdef CARPET_DEBUG
    {
      int ival = do_recompose;
      MPI_Bcast(&ival, 1, MPI_INT, 0, dist::comm());
      assert(ival == do_recompose);
    }
#endif
    did_change = did_change or do_recompose;

    if (do_recompose) {
      BEGIN_MAP_LOOP(cctkGH, CCTK_GF) {
        gh::rregs const &superregss = superregsss.AT(map);
        gh::mregs const &regsss = regssss.AT(map);
        RegridMap(cctkGH, map, superregss, regsss, do_init);
      }
      END_MAP_LOOP;
    }

  } // if regrid in level mode

  if (did_change) {

    PostRegrid(cctkGH);

  } // if did change

  timer.stop();

  return did_change;
}

void RegridMap(cGH const *const cctkGH, int const m,
               gh::rregs const &superregss, gh::mregs const &regsss,
               bool const do_init) {
  DECLARE_CCTK_PARAMETERS;

  char const *const where = "RegridMap";
  static Timers::Timer timer(where);
  timer.start();

  Waypoint("Regridding map %d...", m);

// TODO: keep levels fixed here
#if 0
    //
    // Keep this level fixed if it is not evolved
    //
    if (regrid_only_evolved_levels) {
      int type;
      bool const use_tapered_grids =
        * static_cast<CCTK_INT const *>
        (CCTK_ParameterGet ("use_tapered_grids", "Carpet", &type));
      assert (type == PARAMETER_BOOLEAN);
      
      int const do_every =
        use_tapered_grids ?
        maxtimereflevelfact / timereffacts.AT(max(0,rl-1)):
        maxtimereflevelfact / timereffacts.AT(      rl   );
      
      bool const regrid_this_level =
        (cctkGH->cctk_iteration - 1) % do_every == 0;
      
      if (not regrid_this_level) {
        // Set regions from current grid structure
        regions.AT(rl) = ...;
      }
    }
#endif

  // Check the regions
  CheckRegions(regsss);
  // TODO: check also that the current and all coarser levels did
  // not change

  // Regrid
  vhh.AT(m)->regrid(superregss, regsss, do_init);

  if (print_grid_info) {
    // Output grid structure to screen
    OutputSuperregions(cctkGH, m, *vhh.AT(m), superregss);
    OutputGrids(cctkGH, m, *vhh.AT(m), *vdd.AT(m));
  }

  // Write grid structure to file
  OutputGridStructure(cctkGH, m, regsss);
  OutputGridCoordinates(cctkGH, m, regsss);

  Waypoint("Done regridding map %d.", m);
  timer.stop();
}

void PostRegrid(cGH const *const cctkGH) {
  DECLARE_CCTK_PARAMETERS;

  char const *const where = "PostRegrid";
  static Timers::Timer timer(where);
  timer.start();

  // Calculate new number of levels
  // int const oldreflevels = reflevels;
  reflevels = vhh.AT(0)->reflevels();
  for (int m = 0; m < maps; ++m) {
    assert(vhh.AT(m)->reflevels() == reflevels);
  }

// TODO
#if 1 // #ifdef CARPET_DEBUG
  {
    // All processes must use the same number of levels
    int ival = reflevels;
    MPI_Bcast(&ival, 1, MPI_INT, 0, dist::comm());
    assert(ival == reflevels);
  }
#endif

  // One cannot switch off the current level
  assert(reflevels > reflevel);

  // Set new number of active time levels
  for (int n = 0; n < CCTK_NumGroups(); ++n) {
    int const grouptype = CCTK_GroupTypeI(n);
    if (grouptype == CCTK_GF) {
      for (int ml = 0; ml < mglevels; ++ml) {
        groupdata.AT(n).activetimelevels.AT(ml).resize(
            reflevels, groupdata.AT(n).activetimelevels.AT(ml).AT(0));
      }
    }
  }

  // Set new level times
  // for (int ml=0; ml<mglevels; ++ml) {
  //   leveltimes.AT(ml).resize
  //     (reflevels, leveltimes.AT(ml).AT(oldreflevels-1));
  // }

  ++regridding_epoch;
  // Remove levels that vanished
  if (int(level_regridding_epochs.size()) > reflevels) {
    level_regridding_epochs.resize(reflevels);
  }
  // Insert entries for new levels
  if (int(level_regridding_epochs.size()) < reflevels) {
    level_regridding_epochs.resize(reflevels, 0);
  }

  OutputGridStatistics(cctkGH);

  timer.stop();
}

bool Recompose(cGH const *const cctkGH, int const rl, bool const do_init) {
  char const *const where = "Recompose";
  static Timers::Timer timer(where);
  timer.start();

  bool did_recompose = false;

  for (int m = 0; m < maps; ++m) {
    Waypoint("Recomposing the grid hierarchy for map %d level %d...", m, rl);

    assert(rl >= 0 and rl < vhh.AT(m)->reflevels());
    did_recompose |= vhh.AT(m)->recompose(rl, do_init);

    Waypoint("Done recomposing the grid hierarchy for map %d level %d.", m, rl);
  }

  ClassifyPoints(cctkGH, rl);

  if (did_recompose) {
    ++level_regridding_epochs.at(rl);
  }

  timer.stop();
  return did_recompose;
}

void RegridFree(cGH const *const cctkGH, bool const do_init) {
  char const *const where = "RegridFree";
  static Timers::Timer timer(where);
  timer.start();

  Checkpoint("Freeing after regridding level %d...", reflevel);

  assert(is_level_mode());

  // Free old grid structure
  for (int m = 0; m < maps; ++m) {
    vhh.AT(m)->regrid_free(do_init);
  }

  timer.stop();
}

void OutputSuperregions(cGH const *const cctkGH, int const m, gh const &hh,
                        gh::rregs const &superregss) {
  CCTK_INFO("Grid structure (superregions, grid points):");
  for (int rl = 0; rl < (int)superregss.size(); ++rl) {
    assert(rl < hh.reflevels());
    for (int c = 0; c < (int)superregss.AT(rl).size(); ++c) {
      const ivect &levfact = spacereffacts.AT(rl);
      const ibbox &ext = superregss.AT(rl).AT(c).extent;
      const ivect &lower = ext.lower();
      const ivect &upper = ext.upper();
      const ivect &stride = ext.stride();
      assert(all(lower * levfact % maxspacereflevelfact == 0));
      assert(all(upper * levfact % maxspacereflevelfact == 0));
      cout << "   [" << rl << "][" << m << "][" << c << "]"
           << "   exterior: " << lower / stride << " : " << upper / stride
           << "   (" << (upper - lower) / stride + 1 << " + PADDING) "
           << prod((upper - lower) / stride + 1) << eol;
    }
  }

  CCTK_INFO("Grid structure (superregions, coordinates):");
  for (int rl = 0; rl < (int)superregss.size(); ++rl) {
    assert(rl < hh.reflevels());
    for (int c = 0; c < (int)superregss.AT(rl).size(); ++c) {
      const rvect origin = domainspecs.AT(m).exterior_min;
      const rvect delta =
          (domainspecs.AT(m).exterior_max - domainspecs.AT(m).exterior_min) /
          rvect(domainspecs.AT(m).npoints - 1);
      const ibbox &ext = superregss.AT(rl).AT(c).extent;
      const ivect &ilower = ext.lower();
      const ivect &iupper = ext.upper();
      const ivect &levfact = spacereffacts.AT(rl);
      const ibbox &base = hh.baseextent(0, 0);
      const ivect &bstride = base.stride();
      cout << "   [" << rl << "][" << m << "][" << c << "]"
           << "   exterior: " << origin + delta * rvect(ilower) / rvect(bstride)
           << " : " << origin + delta * rvect(iupper) / rvect(bstride) << " : "
           << delta / rvect(levfact) << eol;
    }
  }

  fflush(stdout);
}

void OutputGrids(cGH const *const cctkGH, int const m, gh const &hh,
                 dh const &dd) {
  DECLARE_CCTK_PARAMETERS;

  if (verbose or veryverbose) {

    CCTK_INFO("Grid structure (grid points):");
    for (int ml = 0; ml < hh.mglevels(); ++ml) {
      for (int rl = 0; rl < hh.reflevels(); ++rl) {
        for (int c = 0; c < hh.components(rl); ++c) {
          const int convfact = ipow(mgfact, ml);
          const ivect levfact = spacereffacts.AT(rl);
          const ibbox ext = hh.extent(ml, rl, c);
          const ivect &lower = ext.lower();
          const ivect &upper = ext.upper();
          const ivect &stride = ext.stride();
          assert(all(lower * levfact % maxspacereflevelfact == 0));
          assert(all(upper * levfact % maxspacereflevelfact == 0));
          assert(all(((upper - lower) * levfact / maxspacereflevelfact) %
                         convfact ==
                     0));
          cout << "   [" << ml << "][" << rl << "][" << m << "][" << c << "]"
               << "   exterior: "
               << "proc " << hh.processor(rl, c) << "   " << lower / stride
               << " : " << upper / stride << "   ("
               << (upper - lower) / stride + 1 << ") "
               << prod((upper - lower) / stride + 1) << eol;
        }
      }
    }

    CCTK_INFO("Grid structure (boundaries):");
    for (int rl = 0; rl < hh.reflevels(); ++rl) {
      for (int c = 0; c < hh.components(rl); ++c) {
        cout << "   [" << rl << "][" << m << "][" << c << "]"
             << "   bbox: " << hh.outer_boundaries(rl, c) << eol;
      }
    }

    CCTK_INFO("Grid structure (coordinates):");
    for (int ml = 0; ml < hh.mglevels(); ++ml) {
      for (int rl = 0; rl < hh.reflevels(); ++rl) {
        for (int c = 0; c < hh.components(rl); ++c) {
          const rvect origin = domainspecs.AT(m).exterior_min;
          const rvect delta = (domainspecs.AT(m).exterior_max -
                               domainspecs.AT(m).exterior_min) /
                              rvect(domainspecs.AT(m).npoints - 1);
          const ibbox ext = hh.extent(ml, rl, c);
          const ivect &ilower = ext.lower();
          const ivect &iupper = ext.upper();
          const int convfact = ipow(mgfact, ml);
          const ivect levfact = spacereffacts.AT(rl);
          const ibbox &base = hh.baseextent(ml, 0);
          const ivect &bstride = base.stride();
          cout << "   [" << ml << "][" << rl << "][" << m << "][" << c << "]"
               << "   exterior: "
               << origin + delta * rvect(ilower) / rvect(bstride) << " : "
               << origin + delta * rvect(iupper) / rvect(bstride) << " : "
               << delta * rvect(convfact) / rvect(levfact) << eol;
        }
      }
    }

    CCTK_INFO("Grid structure (coordinates, including ghosts):");
    for (int ml = 0; ml < hh.mglevels(); ++ml) {
      for (int rl = 0; rl < hh.reflevels(); ++rl) {
        for (int c = 0; c < hh.components(rl); ++c) {
          const rvect origin = domainspecs.AT(m).exterior_min;
          const rvect delta = (domainspecs.AT(m).exterior_max -
                               domainspecs.AT(m).exterior_min) /
                              rvect(domainspecs.AT(m).npoints - 1);
          const ivect lower =
              dd.light_boxes.AT(ml).AT(rl).AT(c).exterior.lower();
          const ivect upper =
              dd.light_boxes.AT(ml).AT(rl).AT(c).exterior.upper();
          const int convfact = ipow(mgfact, ml);
          const ivect levfact = spacereffacts.AT(rl);
          const ibbox &base = hh.baseextent(ml, 0);
          const ivect &bstride = base.stride();
          cout << "   [" << ml << "][" << rl << "][" << m << "][" << c << "]"
               << "   exterior: "
               << origin + delta * rvect(lower) / rvect(bstride) << " : "
               << origin + delta * rvect(upper) / rvect(bstride) << " : "
               << delta * rvect(convfact) / rvect(levfact) << eol;
        }
      }
    }

    CCTK_INFO("Grid statistics:");
    const streamsize oldprecision = cout.precision();
    const ios_base::fmtflags oldflags = cout.flags();
    cout.setf(ios::fixed);
    for (int ml = 0; ml < hh.mglevels(); ++ml) {
      CCTK_REAL CCTK_ATTRIBUTE_UNUSED coarsevolume = 0;
      for (int rl = 0; rl < hh.reflevels(); ++rl) {

        const CCTK_REAL basevolume = hh.baseextents.AT(0).AT(0).size();
        CCTK_REAL countvolume = 0;
        CCTK_REAL totalvolume = 0;
        CCTK_REAL totalvolume2 = 0;

        for (int c = 0; c < hh.components(rl); ++c) {
          const CCTK_REAL volume = hh.extent(ml, rl, c).size();
          ++countvolume;
          totalvolume += volume;
          totalvolume2 += ipow(volume, 2);
        }

        const CCTK_REAL avgvolume = totalvolume / countvolume;
        const CCTK_REAL stddevvolume = sqrt(
            max(CCTK_REAL(0), totalvolume2 / countvolume - ipow(avgvolume, 2)));

        for (int c = 0; c < hh.components(rl); ++c) {
          const CCTK_REAL volume = hh.extent(ml, rl, c).size();
          cout << "   [" << ml << "][" << rl << "][" << m << "][" << c << "]"
               << "   volume: " << setprecision(0) << volume
               << "   of parent: " << setprecision(1)
               << 100 * volume / totalvolume << "%"
               << "   of domain: " << setprecision(1)
               << 100 * volume / basevolume << "%" << eol;
        }

        cout << "   [" << ml << "][" << rl << "][" << m << "]"
             << "   average volume: " << setprecision(0) << avgvolume
             << "   of parent: " << setprecision(1)
             << 100 * avgvolume / totalvolume << "%"
             << "   of domain: " << setprecision(1)
             << 100 * avgvolume / basevolume << "%" << eol;
        cout << "   [" << ml << "][" << rl << "][" << m << "]"
             << "   standard deviation: " << setprecision(0) << stddevvolume
             << "   of parent: " << setprecision(1)
             << 100 * stddevvolume / totalvolume << "%"
             << "   of domain: " << setprecision(1)
             << 100 * stddevvolume / basevolume << "%" << eol;

        // TODO: ghost points vs. interior points (and boundary
        // points)

        CCTK_REAL countquadrupole = 0;
        CCTK_REAL minquadrupole = 1;
        CCTK_REAL totalquadrupole = 0;
        CCTK_REAL totalquadrupole2 = 0;

        for (int c = 0; c < hh.components(rl); ++c) {
          const ibbox ext = hh.extent(ml, rl, c);
          const ivect shape = ext.shape();
          const ivect stride = ext.stride();
          const CCTK_REAL minlength = minval(rvect(shape) / rvect(stride));
          const CCTK_REAL maxlength = maxval(rvect(shape) / rvect(stride));
          const CCTK_REAL quadrupole = minlength / maxlength;
          ++countquadrupole;
          minquadrupole = min(minquadrupole, quadrupole);
          totalquadrupole += quadrupole;
          totalquadrupole2 += ipow(quadrupole, 2);
          cout << "   [" << ml << "][" << rl << "][" << m << "][" << c << "]"
               << "   length ratio: " << setprecision(3) << quadrupole << eol;
        }

        const CCTK_REAL avgquadrupole = totalquadrupole / countquadrupole;
        const CCTK_REAL stddevquadrupole =
            sqrt(max(CCTK_REAL(0), (totalquadrupole2 / countquadrupole -
                                    ipow(avgquadrupole, 2))));

        cout << "   [" << ml << "][" << rl << "][" << m << "]"
             << "   average length ratio: " << setprecision(3) << avgquadrupole
             << "   standard deviation: " << setprecision(3) << stddevquadrupole
             << eol;

        // TODO: processor distribution, average load, std deviation

        coarsevolume = totalvolume * prod(rvect(spacereflevelfact));
        // TODO: output coarsevolume
      }
    }
    cout.precision(oldprecision);
    cout.setf(oldflags);

    fflush(stdout);
  }
}

void OutputGridStructure(cGH const *const cctkGH, int const m,
                         gh::mregs const &regsss) {
  DECLARE_CCTK_PARAMETERS;

  // Output only on the root processor
  if (CCTK_MyProc(cctkGH) != 0)
    return;

  // Output only if output is desired
  if (CCTK_EQUALS(grid_structure_filename, ""))
    return;

  // Create the output directory
  CCTK_CreateDirectory(0755, out_dir);

  ostringstream filenamebuf;
  filenamebuf << out_dir << "/" << grid_structure_filename;
  // we need a persistent temporary here
  string filenamestr = filenamebuf.str();
  const char *filename = filenamestr.c_str();

  ofstream file;

  static bool do_truncate = true;

  if (do_truncate) {
    do_truncate = false;
    struct stat fileinfo;
    if (IO_TruncateOutputFiles(cctkGH) or stat(filename, &fileinfo) != 0) {
      file.open(filename, ios::out | ios::trunc);
      assert(file.good());
      file << "# grid structure" << eol
           << "# format: map reflevel component mglevel   processor "
              "bounding-box is-outer-boundary"
           << eol;
      assert(file.good());
    }
  }
  if (not file.is_open()) {
    file.open(filename, ios::app);
    assert(file.good());
  }

  if (m == 0) {
    // Output a separator
    file << eol;
    file << "iteration " << cctkGH->cctk_iteration << eol;
    file << "maps " << maps << eol;
  }
  file << m << " mglevels " << regsss.size() << eol;
  for (int ml = 0; ml < (int)regsss.size(); ++ml) {
    file << m << " " << ml << " reflevels " << regsss.AT(ml).size() << eol;
    for (int rl = 0; rl < (int)regsss.AT(ml).size(); ++rl) {
      file << m << " " << ml << " " << rl << " components "
           << regsss.AT(ml).AT(rl).size() << eol;
      for (int c = 0; c < (int)regsss.AT(ml).AT(rl).size(); ++c) {
        file << m << " " << ml << " " << rl << " " << c << "   "
             << regsss.AT(ml).AT(rl).AT(c).processor << " "
             << regsss.AT(ml).AT(rl).AT(c).extent << " "
             << regsss.AT(ml).AT(rl).AT(c).outer_boundaries << eol;
      }
    }
  }

  file.close();
  assert(file.good());
}

void OutputGridCoordinates(cGH const *const cctkGH, int const m,
                           gh::mregs const &regsss) {
  DECLARE_CCTK_PARAMETERS;

  // Output only on the root processor
  if (CCTK_MyProc(cctkGH) != 0)
    return;

  // Output only if output is desired
  if (CCTK_EQUALS(grid_coordinates_filename, ""))
    return;

  // Create the output directory
  CCTK_CreateDirectory(0755, out_dir);

  ostringstream filenamebuf;
  filenamebuf << out_dir << "/" << grid_coordinates_filename;
  // we need a persistent temporary here
  string filenamestr = filenamebuf.str();
  const char *filename = filenamestr.c_str();

  ofstream file;

  static bool do_truncate = true;

  if (do_truncate) {
    do_truncate = false;
    struct stat fileinfo;
    if (IO_TruncateOutputFiles(cctkGH) or stat(filename, &fileinfo) != 0) {
      file.open(filename, ios::out | ios::trunc);
      assert(file.good());
      file << "# grid coordinates" << eol
           << "# format: map reflevel region mglevel   bounding-box" << eol;
      assert(file.good());
    }
  }
  if (not file.is_open()) {
    file.open(filename, ios::app);
    assert(file.good());
  }

  // Affine transformation between index space and coordinate space
  rvect const origin = domainspecs.AT(m).exterior_min;
  rvect const spacing =
      (domainspecs.AT(m).exterior_max - domainspecs.AT(m).exterior_min) /
      rvect(domainspecs.AT(m).npoints - 1);
  rvect const scale =
      rvect(vhh.AT(m)->baseextents.AT(0).AT(0).stride()) / spacing;

  file << "iteration " << cctkGH->cctk_iteration << eol;
  file << "maps " << maps << eol;
  file << m << " mglevels " << regsss.size() << eol;
  for (int ml = 0; ml < (int)regsss.size(); ++ml) {
    file << m << " " << ml << " reflevels " << regsss.AT(ml).size() << eol;
    for (int rl = 0; rl < (int)regsss.AT(ml).size(); ++rl) {
      ibset const extents(regsss.AT(ml).AT(rl), &region_t::extent);
      file << m << " " << ml << " " << rl << " regions " << extents.setsize()
           << eol;
      int c = 0;
      for (ibset::const_iterator bi = extents.begin(); bi != extents.end();
           ++bi) {
#if 0
          ibbox const & ext       = * bi;
          ibbox const & baseext   = vhh.AT(m)->baseextents.AT(ml).AT(rl);
          ibbox const & coarseext = vhh.AT(m)->baseextents.AT(ml).AT(0 );
          
          // This is nice, but is wrong since CartGrid3D has not yet
          // initialised the coordinates
          ivect const cctk_levfac       = spacereffacts.AT(rl);
          ivect const cctk_levoff       = baseext.lower() - coarseext.lower();
          ivect const cctk_levoffdenom  = baseext.stride();
          
          ivect const cctk_lbnd =
            (ext.lower() - baseext.lower()) / ext.stride();
          ivect const cctk_ubnd =
            (ext.upper() - baseext.lower()) / ext.stride();
          
          rvect const cctk_origin_space =
            origin_space.AT(m).AT(ml);
          rvect const cctk_delta_space  =
            delta_space.AT(m) * rvect (ipow (mgfact, ml));
          
          rvect const CCTK_ORIGIN_SPACE =
            cctk_origin_space +
            cctk_delta_space / rvect (cctk_levfac) *
            rvect (cctk_levoff) / rvect (cctk_levoffdenom);
          rvect const CCTK_DELTA_SPACE =
            cctk_delta_space / rvect (cctk_levfac);
          
          rvect const rlower =
            CCTK_ORIGIN_SPACE + rvect (cctk_lbnd) * CCTK_DELTA_SPACE;
          rvect const rupper =
            CCTK_ORIGIN_SPACE + rvect (cctk_ubnd) * CCTK_DELTA_SPACE;
          rvect const rdelta =
            CCTK_DELTA_SPACE;
#endif

        ibbox const &ext = *bi;

        rvect const rlower = origin + rvect(ext.lower()) / scale;
        rvect const rupper = origin + rvect(ext.upper()) / scale;
        rvect const rdelta = rvect(ext.stride()) / scale;

        rbbox const rb(rlower, rupper, rdelta);
        const std::streamsize oldprecision = file.precision(19);
        file << m << " " << ml << " " << rl << " " << c << "   " << rb << eol;
        file.precision(oldprecision);
        ++c;
      }
    }
  }
  file << eol;

  file.close();
  assert(file.good());
}

// TODO: this routine should go into CarpetRegrid (except maybe
// SplitRegions_AlongZ for grid arrays)
void SplitRegions(cGH const *const cctkGH, vector<region_t> &superregs,
                  vector<region_t> &regs) {
  DECLARE_CCTK_PARAMETERS;

  assert(regs.empty());
  if (CCTK_EQUALS(processor_topology, "along-z")) {
    SplitRegions_AlongZ(cctkGH, superregs, regs);
  } else if (CCTK_EQUALS(processor_topology, "along-dir")) {
    SplitRegions_AlongDir(cctkGH, superregs, regs, split_direction);
  } else if (CCTK_EQUALS(processor_topology, "automatic")) {
    SplitRegions_Automatic(cctkGH, superregs, regs);
  } else if (CCTK_EQUALS(processor_topology, "manual")) {
    SplitRegions_AsSpecified(cctkGH, superregs, regs);
  } else {
    assert(0);
  }
}

void OutputGridStatistics(cGH const *const cctkGH) {
  // Grid array statistics
  int num_arrays = 0;
  int num_gfs = 0;
  int size_gfs = 0;
  CCTK_REAL num_active_array_points = 0;
  CCTK_REAL num_total_array_points = 0;
  CCTK_REAL size_total_array_points = 0;
  for (int g = 0; g < CCTK_NumGroups(); ++g) {
    cGroup gdata;
    check(not CCTK_GroupData(g, &gdata));
    int const num_tl = CCTK_ActiveTimeLevelsGI(cctkGH, g);
    int const num_vars = gdata.numvars;
    int const size_vars = gdata.numvars * CCTK_VarTypeSize(gdata.vartype);
    switch (gdata.grouptype) {
    case CCTK_SCALAR:
    case CCTK_ARRAY: {
      num_arrays += num_tl * num_vars;
      gh const *const hh = arrdata.AT(g).AT(0).hh;
      dh const *const dd = arrdata.AT(g).AT(0).dd;
      for (int c = 0; c < hh->components(0); ++c) {
        dh::light_dboxes const &b = dd->light_boxes.AT(0).AT(0).AT(c);
        num_active_array_points += num_tl * num_vars * b.active_size;
        num_total_array_points += num_tl * num_vars * b.exterior_size;
        size_total_array_points += num_tl * size_vars * b.exterior_size;
      }
      break;
    }
    case CCTK_GF:
      num_gfs += num_tl * num_vars;
      size_gfs += num_tl * size_vars;
      break;
    default:
      assert(0);
    }
  }

  // Grid function statistics
  int num_comps = 0;
  int num_steps = 0;
  CCTK_REAL num_active_mem_points = 0;
  CCTK_REAL num_owned_mem_points = 0;
  CCTK_REAL num_total_mem_points = 0;
  CCTK_REAL size_total_mem_points = 0;
  CCTK_REAL num_active_cpu_points = 0;
  CCTK_REAL num_owned_cpu_points = 0;
  CCTK_REAL num_total_cpu_points = 0;
  for (int m = 0; m < maps; ++m) {
    gh const *const hh = vhh.AT(m);
    dh const *const dd = vdd.AT(m);
    for (int ml = 0; ml < mglevels; ++ml) {
      for (int rl = 0; rl < reflevels; ++rl) {
        int const trf = timereffacts.AT(rl);
        if (m == 0 and ml == 0) {
          num_steps += trf;
        }
        for (int c = 0; c < hh->components(rl); ++c) {
          ++num_comps;
          dh::light_dboxes const &b = dd->light_boxes.AT(ml).AT(rl).AT(c);
          num_active_mem_points += num_gfs * b.active_size;
          num_owned_mem_points += num_gfs * b.owned_size;
          num_total_mem_points += num_gfs * b.exterior_size;
          size_total_mem_points += size_gfs * b.exterior_size;
          num_active_cpu_points += trf * b.active_size;
          num_owned_cpu_points += trf * b.owned_size;
          num_total_cpu_points += trf * b.exterior_size;
        }
      }
    }
  }
  num_active_cpu_points /= num_steps * delta_time;
  num_owned_cpu_points /= num_steps * delta_time;
  num_total_cpu_points /= num_steps * delta_time;

  // Load balance statistics
  // We count owned points, since these are evolved. Owned points
  // include active points, overlap points, and buffer points.
  CCTK_REAL const rzero = 0;
  CCTK_REAL const rmax = numeric_limits<CCTK_REAL>::max();
  vector<CCTK_REAL> min_owned(reflevels, rmax);
  vector<CCTK_REAL> max_owned(reflevels, 0);
  vector<CCTK_REAL> avg_owned(reflevels, 0);
  vector<CCTK_REAL> sdv_owned(reflevels, 0);
  for (int rl = 0; rl < reflevels; ++rl) {
    vector<CCTK_REAL> num_owned_per_proc(dist::size(), 0);
    for (int m = 0; m < maps; ++m) {
      gh const *const hh = vhh.AT(m);
      dh const *const dd = vdd.AT(m);
      for (int ml = 0; ml < mglevels; ++ml) {
        for (int c = 0; c < hh->components(rl); ++c) {
          int const p = hh->processor(rl, c);
          dh::light_dboxes const &b = dd->light_boxes.AT(ml).AT(rl).AT(c);
          num_owned_per_proc.AT(p) += num_gfs * b.owned_size;
        }
      }
    }
    for (int p = 0; p < dist::size(); ++p) {
      min_owned.AT(rl) = min(min_owned.AT(rl), num_owned_per_proc.AT(p));
      max_owned.AT(rl) = max(max_owned.AT(rl), num_owned_per_proc.AT(p));
      avg_owned.AT(rl) += num_owned_per_proc.AT(p);
      sdv_owned.AT(rl) += ipow(num_owned_per_proc.AT(p), 2);
    }
    avg_owned.AT(rl) /= dist::size();
    sdv_owned.AT(rl) /= dist::size();
    sdv_owned.AT(rl) =
        sqrt(max(rzero, sdv_owned.AT(rl) - ipow(avg_owned.AT(rl), 2)));
  } // for rl

  // Output
  CCTK_VInfo(CCTK_THORNSTRING, "Global grid structure statistics:");
  CCTK_VInfo(
      CCTK_THORNSTRING, "GF: rhs: %.0fk active, %.0fk owned (+%.0f%%), %.0fk "
                        "total (+%.0f%%), %.3g steps/time",
      double(num_active_cpu_points / 1e+3), double(num_owned_cpu_points / 1e+3),
      double(num_active_cpu_points == 0
                 ? 0
                 : num_owned_cpu_points / num_active_cpu_points * 100 - 100),
      double(num_total_cpu_points / 1e+3),
      double(num_owned_cpu_points == 0
                 ? 0
                 : num_total_cpu_points / num_owned_cpu_points * 100 - 100),
      double(num_steps / delta_time));
  CCTK_VInfo(
      CCTK_THORNSTRING, "GF: vars: %d, pts: %.0fM active, %.0fM owned "
                        "(+%.0f%%), %.0fM total (+%.0f%%), %.1f comp/proc",
      num_gfs, double(num_active_mem_points / 1e+6),
      double(num_owned_mem_points / 1e+6),
      double(num_active_mem_points == 0
                 ? 0
                 : num_owned_mem_points / num_active_mem_points * 100 - 100),
      double(num_total_mem_points / 1e+6),
      double(num_owned_mem_points == 0
                 ? 0
                 : num_total_mem_points / num_owned_mem_points * 100 - 100),
      double(num_comps / (reflevels * CCTK_nProcs(cctkGH))));
  CCTK_VInfo(
      CCTK_THORNSTRING,
      "GA: vars: %d, pts: %.0fM active, %.0fM total (+%.0f%%)", num_arrays,
      double(num_active_array_points / 1e+6),
      double(num_total_array_points / 1e+6),
      double(num_active_array_points == 0
                 ? 0
                 : num_total_array_points / num_active_array_points * 100 -
                       100));
  CCTK_VInfo(
      CCTK_THORNSTRING,
      "Total required memory: %.3f GByte (for GAs and currently active GFs)",
      double((size_total_array_points + size_total_mem_points) / 1e+9));
  CCTK_VInfo(CCTK_THORNSTRING,
             "Load balance:  min     avg     max     sdv     max/avg-1");
  for (int rl = 0; rl < reflevels; ++rl) {
    CCTK_VInfo(
        CCTK_THORNSTRING,
        "Level %2d:    %4.0fM   %4.0fM   %4.0fM   %4.0fM owned   %4.0f%%", rl,
        double(min_owned.AT(rl) / 1e+6), double(avg_owned.AT(rl) / 1e+6),
        double(max_owned.AT(rl) / 1e+6), double(sdv_owned.AT(rl) / 1e+6),
        double(100 * (max_owned.AT(rl) / avg_owned.AT(rl) - 1)));
  }

  // After this, we will begin to allocate memory for the grid
  // structure.  If we run out of memory, ensure that this output
  // still makes it to disk.
  fflush(stdout);
  fflush(stderr);
}

void SplitRegions_AlongZ(cGH const *const cctkGH, vector<region_t> &superregs,
                         vector<region_t> &regs) {
  assert(regs.empty());
  SplitRegions_AlongDir(cctkGH, superregs, regs, dim - 1);
}

void SplitRegions_AlongDir(cGH const *const cctkGH, vector<region_t> &superregs,
                           vector<region_t> &regs, int const dir) {
  assert(regs.empty());

  // Something to do?
  if (superregs.size() == 0) {
    return;
  }

  const int nprocs = CCTK_nProcs(cctkGH);

  assert(superregs.size() == 1);
  regs = superregs;

  if (nprocs == 1) {
    regs.AT(0).processor = 0;
    pseudoregion_t const preg(regs.AT(0).extent, regs.AT(0).processor);
    assert(superregs.AT(0).processors == NULL);
    superregs.AT(0).processors = new ipfulltree(preg);
    return;
  }

  assert(dir >= 0 and dir < dim);

  region_t const &reg0 = regs.AT(0);
  const ivect rstr0 = reg0.extent.stride();
  const ivect rlb0 = reg0.extent.lower();
  const ivect rub0 = reg0.extent.upper() + rstr0;
  const b2vect obnd0 = reg0.outer_boundaries;

  regs.resize(nprocs);
  vector<int> bounds(nprocs + 1);
  vector<ipfulltree *> subtrees(nprocs);
  for (int c = 0; c < nprocs; ++c) {
    ivect cstr = rstr0;
    ivect clb = rlb0;
    ivect cub = rub0;
    const int glonpz = (rub0[dir] - rlb0[dir]) / cstr[dir];
    const int locnpz = (glonpz + nprocs - 1) / nprocs;
    const int zstep = locnpz * cstr[dir];
    clb[dir] = rlb0[dir] + zstep * c;
    cub[dir] = rlb0[dir] + zstep * (c + 1);
    if (clb[dir] > rub0[dir])
      clb[dir] = rub0[dir];
    if (cub[dir] > rub0[dir])
      cub[dir] = rub0[dir];
    assert(clb[dir] <= cub[dir]);
    assert(cub[dir] <= rub0[dir]);
    region_t &reg = regs.AT(c);
    ibbox &ext = reg.extent;
    b2vect &obnd = reg.outer_boundaries;
    int &proc = reg.processor;
    ext = ibbox(clb, cub - cstr, cstr);
    obnd = obnd0;
    obnd[0][dir] &= clb[dir] == rlb0[dir];
    obnd[1][dir] &= cub[dir] == rub0[dir];
    proc = c;
    pseudoregion_t const preg(reg.extent, c);
    bounds.AT(c) = reg.extent.lower()[dir];
    subtrees.AT(c) = new ipfulltree(preg);
  }
  bounds.AT(nprocs) = rub0[dir] + rstr0[dir];

  assert(superregs.AT(0).processors == NULL);
  superregs.AT(0).processors = new ipfulltree(dir, bounds, subtrees);

  for (int c = 0; c < (int)regs.size(); ++c) {
    assert(regs.AT(c).processor == c);
  }
}

void SplitRegions_Automatic(cGH const *const cctkGH,
                            vector<region_t> &superregs, vector<region_t> &regs,
                            bvect const &no_split_dims) {
  assert(regs.empty());
  vector<vector<region_t> > superregss(1, superregs);
  vector<vector<region_t> > regss(1);
  SplitRegionsMaps_Automatic(cctkGH, superregss, regss, no_split_dims);
  assert(superregss.size() == 1);
  superregs = superregss.AT(0);
  assert(regss.size() == 1);
  regs = regss.AT(0);
}

static void SplitRegions_AsSpecified(cGH const *const cctkGH,
                                     vector<region_t> &superregs,
                                     vector<region_t> &regs) {
  DECLARE_CCTK_PARAMETERS;

  assert(regs.empty());

  // Something to do?
  if (superregs.size() == 0) {
    return;
  }

  const int nprocs = CCTK_nProcs(cctkGH);

  assert(superregs.size() == 1);

  region_t const &reg0 = superregs.AT(0);
  const ivect rstr0 = reg0.extent.stride();
  const ivect rlb0 = reg0.extent.lower();
  const ivect rub0 = reg0.extent.upper() + rstr0;
  const b2vect obnd0 = reg0.outer_boundaries;

  const ivect nprocs_dir(processor_topology_3d_x, processor_topology_3d_y,
                         processor_topology_3d_z);
  assert(all(nprocs_dir > 0));
  if (prod(nprocs_dir) != nprocs) {
    CCTK_VWarn(0, __LINE__, __FILE__, CCTK_THORNSTRING,
               "The specified processor topology [%d,%d,%d] requires %d "
               "processors, but there are %d processors",
               nprocs_dir[0], nprocs_dir[1], nprocs_dir[2], prod(nprocs_dir),
               nprocs);
  }
  assert(prod(nprocs_dir) == nprocs);

  regs.resize(nprocs);
  const ivect cstr = rstr0;
  const ivect glonp = (rub0 - rlb0) / cstr;
  //     const ivect locnp = (glonp + nprocs_dir - 1) / nprocs_dir;
  const ivect locnp = glonp / nprocs_dir;
  const ivect rem = glonp % nprocs_dir;
  const ivect step = locnp * cstr;
  assert(dim == 3);

  vector<int> boundsz(nprocs_dir[2] + 1);
  vector<ipfulltree *> subtreesz(nprocs_dir[2]);
  for (int k = 0; k < nprocs_dir[2]; ++k) {
    vector<int> boundsy(nprocs_dir[1] + 1);
    vector<ipfulltree *> subtreesy(nprocs_dir[1]);
    for (int j = 0; j < nprocs_dir[1]; ++j) {
      vector<int> boundsx(nprocs_dir[0] + 1);
      vector<ipfulltree *> subtreesx(nprocs_dir[0]);
      for (int i = 0; i < nprocs_dir[0]; ++i) {

        const int c = i + nprocs_dir[0] * (j + nprocs_dir[1] * k);
        const ivect ipos(i, j, k);
        ivect clb = rlb0 + step * ipos;
        ivect cub = rlb0 + step * (ipos + 1);
        // 	  clb = min (clb, rub);
        // 	  cub = min (cub, rub);
        for (int d = 0; d < dim; ++d) {
          if (ipos[d] < rem[d]) {
            clb[d] += cstr[d] * ipos[d];
            cub[d] += cstr[d] * (ipos[d] + 1);
          } else {
            clb[d] += cstr[d] * rem[d];
            cub[d] += cstr[d] * rem[d];
          }
        }
        assert(all(clb >= 0));
        assert(all(clb <= cub));
        assert(all(cub <= rub0));
        assert(all(not(ipos == 0) or clb == rlb0));
        assert(all(not(ipos == nprocs_dir - 1) or cub == rub0));
        region_t &reg = regs.AT(c);
        ibbox &ext = reg.extent;
        b2vect &obnd = reg.outer_boundaries;
        int &proc = reg.processor;
        ext = ibbox(clb, cub - cstr, cstr);
        obnd = obnd0;
        obnd[0] &= clb == rlb0;
        obnd[1] &= cub == rub0;
        proc = c;

        pseudoregion_t preg(reg.extent, c);
        subtreesx.AT(i) = new ipfulltree(preg);
      }
      boundsx.AT(nprocs_dir[0]) = rub0[0] + rstr0[0];
      subtreesy.AT(j) = new ipfulltree(0, boundsx, subtreesx);
    }
    boundsy.AT(nprocs_dir[1]) = rub0[1] + rstr0[1];
    subtreesz.AT(k) = new ipfulltree(1, boundsy, subtreesy);
  }
  boundsz.AT(nprocs_dir[2]) = rub0[2] + rstr0[2];

  assert(superregs.AT(0).processors == NULL);
  superregs.AT(0).processors = new ipfulltree(2, boundsz, subtreesz);

  for (int c = 0; c < (int)regs.size(); ++c) {
    assert(regs.AT(c).processor == c);
  }
}

// TODO: this routine should go into CarpetRegrid (except maybe
// SplitRegions_AlongZ for grid arrays)
void SplitRegionsMaps(cGH const *const cctkGH,
                      vector<vector<region_t> > &superregss,
                      vector<vector<region_t> > &regss) {
  DECLARE_CCTK_PARAMETERS;

  assert(regss.size() == superregss.size());
  for (size_t m = 0; m < regss.size(); ++m) {
    assert(regss.AT(m).empty());
  }
  if (CCTK_EQUALS(processor_topology, "along-z")) {
    assert(0);
    //       SplitRegionsMaps_AlongZ (cctkGH, superregss, regss);
  } else if (CCTK_EQUALS(processor_topology, "along-dir")) {
    assert(0);
    //       SplitRegionsMaps_AlongDir (cctkGH, superregss, regss,
    //       split_direction);
  } else if (CCTK_EQUALS(processor_topology, "automatic")) {
    SplitRegionsMaps_Automatic(cctkGH, superregss, regss);
  } else if (CCTK_EQUALS(processor_topology, "recursive")) {
    SplitRegionsMaps_Recursively(cctkGH, superregss, regss);
  } else if (CCTK_EQUALS(processor_topology, "balanced")) {
    SplitRegionsMaps_Balanced(cctkGH, superregss, regss);
  } else if (CCTK_EQUALS(processor_topology, "manual")) {
    SplitRegionsMaps_AsSpecified(cctkGH, superregss, regss);
  } else {
    assert(0);
  }

  for (size_t m = 0; m < superregss.size(); ++m) {
    for (size_t r = 0; r < superregss.AT(m).size(); ++r) {
      bool const good_superregs = superregss.AT(m).AT(r).check_region(true);
      if (not good_superregs) {
        cout << "superregs[" << m << "][" << r << "]:\n"
             << superregss.AT(m).AT(r) << "\n";
        cout << "all superregions:\n" << superregss << "\n";
        CCTK_VWarn(CCTK_WARN_ALERT, __LINE__, __FILE__, CCTK_THORNSTRING,
                   "Superregions failed self-check for map %d superregion %d",
                   int(m), int(r));
      }
    }
  }
  assert(regss.size() == superregss.size());
  for (size_t m = 0; m < regss.size(); ++m) {
    for (size_t r = 0; r < regss.AT(m).size(); ++r) {
      bool const good_regs = regss.AT(m).AT(r).check_region(false);
      if (not good_regs) {
        cout << "regs[" << m << "][" << r << "]:\n" << regss.AT(m).AT(r)
             << "\n";
        cout << "all superregions:\n" << superregss << "\n";
        cout << "all regions:\n" << regss << "\n";
        CCTK_VWarn(CCTK_WARN_ALERT, __LINE__, __FILE__, CCTK_THORNSTRING,
                   "Regions failed self-check for map %d superregion %d",
                   int(m), int(r));
      }
    }
  }
}

static void SplitRegionsMaps_Automatic_Recursively(bvect const &dims,
                                                   int const firstproc,
                                                   int const nprocs,
                                                   region_t &superreg,
                                                   vector<region_t> &newregs) {
  DECLARE_CCTK_PARAMETERS;

  if (recompose_verbose)
    cout << "SRMAR enter" << endl;
  // Check preconditions
  assert(nprocs >= 1);

  // Remember old size
  size_t const oldsize = newregs.size();

  // Are we done?
  if (all(dims)) {
    if (recompose_verbose)
      cout << "SRMAR bottom" << endl;

    // Check precondition
    assert(nprocs == 1);

    // Return argument
    region_t newreg = superreg;
    newreg.processor = firstproc;
    newregs.push_back(newreg);
    pseudoregion_t const preg(newreg.extent, newreg.processor);
    superreg.processors = new ipfulltree(preg);

    // Check postcondition
    assert(newregs.size() == oldsize + nprocs);

    if (recompose_verbose)
      cout << "SRMAR exit" << endl;
    return;
  }

  // Special case
  if (superreg.extent.empty()) {
    if (recompose_verbose)
      cout << "SRMAR empty" << endl;

    // Choose a direction
    int mydim = -1;
    for (int d = 0; d < dim; ++d) {
      if (not dims[d]) {
        mydim = d;
        break;
      }
    }
    assert(mydim >= 0 and mydim < dim);

    // Create a new region
    region_t newreg(superreg);
    newreg.outer_boundaries = b2vect(false);
    if (recompose_verbose)
      cout << "SRMAR newreg " << newreg << endl;

#if 0
      // Store
      for (int p=0; p<nprocs; ++p) {
        newreg.processor = firstproc + p;
        newregs.push_back (newreg);
      }
      assert (superreg.processors == NULL);
      superreg.processors = new ipfulltree ();
#endif

    // Store
    vector<int> bounds(nprocs + 1);
    vector<ipfulltree *> subtrees(nprocs);
    for (int p = 0; p < nprocs; ++p) {
      newreg.processor = firstproc + p;
      newregs.push_back(newreg);
      bounds.AT(p) = newreg.extent.lower()[mydim];
      pseudoregion_t const preg(newreg.extent, newreg.processor);
      subtrees.AT(p) = new ipfulltree(preg);
    }
    bounds.AT(nprocs) = newreg.extent.lower()[mydim];
    assert(superreg.processors == NULL);
    superreg.processors = new ipfulltree(mydim, bounds, subtrees);

    // Check postcondition
    assert(newregs.size() == oldsize + nprocs);

    if (recompose_verbose)
      cout << "SRMAR exit" << endl;
    return;
  }

  // Calculate cost factors
  rvect rcost = cost(superreg);
  CCTK_REAL const rfact = pow(nprocs / prod(rcost), CCTK_REAL(1) / dim);
  rcost *= rfact;
  assert(fabs(prod(rcost) - nprocs) <= 1.0e-6);
  if (recompose_verbose)
    cout << "SRMA shapes " << rcost << endl;

  // Choose a direction
  int mydim = -1;
  int nslices = -1;
  if (no_split_direction != -1 and not dims[no_split_direction]) {
    mydim = no_split_direction;
    nslices = 1;
  } else {
    int alldims = 0;
    CCTK_REAL mycost = 0;
    CCTK_REAL totalcost = 1;
    for (int d = 0; d < dim; ++d) {
      if (not dims[d]) {
        ++alldims;
        CCTK_REAL const thiscost = rcost[d];
        // Prefer to split in the z direction
        // (Choose this dimension even if it is slightly worse than
        // a previous dimension)
        if (thiscost >= 0.999999 * mycost) {
          mydim = d;
          mycost = thiscost;
        }
        totalcost *= thiscost;
      }
    }
    assert(mydim >= 0 and mydim < dim);
    assert(mycost >= 0);
    if (recompose_verbose)
      cout << "SRMAR mydim " << mydim << endl;
    if (recompose_verbose)
      cout << "SRMAR mycost " << mycost << endl;

    // Choose a number of slices for this direction
    CCTK_REAL const mycost1 =
        mycost * pow(nprocs / totalcost, CCTK_REAL(1) / alldims);
    // Round up, so that we use more slices in this direction than
    // in the left-over directions
    nslices = min(nprocs, int(lrint(ceil(mycost1))));
  }
  assert(nslices <= nprocs);
  if (recompose_verbose)
    cout << "SRMAR " << mydim << " nprocs " << nprocs << endl;
  if (recompose_verbose)
    cout << "SRMAR " << mydim << " nslices " << nslices << endl;

  // Mark this direction as done
  assert(not dims[mydim]);
  bvect const newdims = dims.replace(mydim, true);

  // Split the remaining processors
  vector<int> mynprocs(nslices);
  int const mynprocs_base = nprocs / nslices;
  int const mynprocs_left = nprocs - nslices * mynprocs_base;
  int sum_mynprocs = 0;
  for (int n = 0; n < nslices; ++n) {
    mynprocs.AT(n) = mynprocs_base + int(n < mynprocs_left);
    sum_mynprocs += mynprocs.AT(n);
  }
  assert(sum_mynprocs == nprocs);
  if (recompose_verbose)
    cout << "SRMAR " << mydim << " mynprocs " << mynprocs << endl;

  // Split the region
  vector<int> mynpoints(nslices);
  int const npoints =
      (superreg.extent.shape() / superreg.extent.stride())[mydim];
  int const npoints_bnd_lo =
      superreg.outer_boundaries[0][mydim] ? granularity_boundary : 0;
  int const npoints_bnd_hi =
      superreg.outer_boundaries[1][mydim] ? granularity_boundary : 0;

  // Keep track of how many points and processors we have left to
  // distribute
  int npoints_left = npoints;
  int nprocs_left = nprocs;
  for (int n = 0; n < nslices; ++n) {
    assert(nprocs_left > 0);
    CCTK_REAL const npoints1 =
        CCTK_REAL(1) * npoints_left * mynprocs.AT(n) / nprocs_left;
    mynpoints.AT(n) =
        int(floor(npoints1 / granularity + CCTK_REAL(0.5))) * granularity;
    if (n == 0)
      mynpoints.AT(n) += npoints_bnd_lo;
    if (npoints_left - mynpoints.AT(n) <= npoints_bnd_hi)
      mynpoints.AT(n) += npoints_bnd_hi;
    mynpoints.AT(n) = min(npoints_left, mynpoints.AT(n));
    assert(mynpoints.AT(n) >= 0 and mynpoints.AT(n) <= npoints_left);
    npoints_left -= mynpoints.AT(n);
    nprocs_left -= mynprocs.AT(n);
    assert(npoints_left >= 0);
    assert(nprocs_left >= 0);
  }
  assert(npoints_left == 0);
  assert(nprocs_left == 0);
  if (recompose_verbose)
    cout << "SRMAR " << mydim << " mynpoints " << mynpoints << endl;

  // Create the regions and recurse
  if (recompose_verbose)
    cout << "SRMAR " << mydim << ": create bboxes" << endl;
  ivect lo = superreg.extent.lower();
  ivect up = superreg.extent.upper();
  ivect const str = superreg.extent.stride();
  int p = firstproc;
  vector<int> bounds(nslices + 1);
  vector<ipfulltree *> subtrees(nslices);
  for (int n = 0; n < nslices; ++n) {
    if (recompose_verbose)
      cout << "SRMAR " << mydim << " n " << n << endl;

    // Create a new region
    up[mydim] = lo[mydim] + (mynpoints.AT(n) - 1) * str[mydim];
    b2vect newob(superreg.outer_boundaries);
    if (n > 0) {
      newob[0][mydim] = false;
    }
    if (n < nslices - 1) {
      up[mydim] = lo[mydim] + (mynpoints.AT(n) - 1) * str[mydim];
      newob[1][mydim] = false;
    }
    region_t newreg(superreg);
    newreg.extent = ibbox(lo, up, str);
    newreg.outer_boundaries = newob;
    if (recompose_verbose)
      cout << "SRMAR " << mydim << " newreg " << newreg << endl;

    // Recurse
    bounds.AT(n) = lo[mydim];
    SplitRegionsMaps_Automatic_Recursively(newdims, p, mynprocs.AT(n), newreg,
                                           newregs);
    if (recompose_verbose)
      cout << "SRMAR " << mydim << " newregs " << newregs << endl;

    assert(newreg.processors != NULL);
    subtrees.AT(n) = newreg.processors;
    newreg.processors = NULL;
    newreg.processor = p;

    // Next slice
    lo[mydim] = up[mydim] + str[mydim];
    p += mynprocs.AT(n);
  }
  assert(up[mydim] == superreg.extent.upper()[mydim]);
  assert(p == firstproc + nprocs);
  bounds.AT(nslices) = up[mydim] + str[mydim];

  // Create tree
  assert(superreg.processors == NULL);
  superreg.processors = new ipfulltree(mydim, bounds, subtrees);

  // Check postcondition
  assert(newregs.size() == oldsize + nprocs);

  if (recompose_verbose)
    cout << "SRMAR exit" << endl;
}

void SplitRegionsMaps_Automatic(cGH const *const cctkGH,
                                vector<vector<region_t> > &superregss,
                                vector<vector<region_t> > &regss,
                                bvect const &no_split_dims) {
  DECLARE_CCTK_PARAMETERS;

  if (recompose_verbose)
    cout << "SRMA enter" << endl;

  int const nmaps = superregss.size();
  int minmap = 1000000000;
  for (int m = 0; m < nmaps; ++m) {
    for (int r = 0; r < int(superregss.AT(m).size()); ++r) {
      minmap = min(minmap, superregss.AT(m).AT(r).map);
    }
  }
  int nregs = 0;
  for (int m = 0; m < nmaps; ++m) {
    nregs += superregss.AT(m).size();
  }
  if (recompose_verbose)
    cout << "SRMA nregs " << nregs << endl;

  // Something to do?
  if (nregs == 0) {
    return;
  }

  // Collect slices
  vector<region_t> superregs;
  {
    for (int m = 0; m < nmaps; ++m) {
      combine_regions(superregss.AT(m), superregs);
    }
    nregs = superregs.size();

    // If the last region was removed, add a new empty region again.
    // A set of regions (corresponding to a refinement level or a
    // grid array) cannot be empty.
    if (nregs == 0) {
      assert(nmaps == 1); // we should only be here for grid
                          // arrays
      region_t reg;
      reg.extent = ibbox(ivect(0), ivect(-1), ivect(1));
      reg.outer_boundaries = b2vect(bvect(true), bvect(true));
      reg.map = 0;
      superregs.push_back(reg);
      nregs = superregs.size();
    }
  }

  int const real_nprocs = CCTK_nProcs(cctkGH);
  if (recompose_verbose)
    cout << "SRMA real_nprocs " << real_nprocs << endl;

  // Deactivate some processors if there are too many
  int nprocs;
  if (min_points_per_proc == 0) {
    nprocs = real_nprocs;
  } else {
    CCTK_REAL mycost = 0;
    for (int r = 0; r < nregs; ++r) {
      mycost += prod(cost(superregs.AT(r)));
    }
    int const goodnprocs = int(floor(mycost / min_points_per_proc));
    nprocs = max(1, min(real_nprocs, goodnprocs));
  }
  if (recompose_verbose)
    cout << "SRMA nprocs " << nprocs << endl;

  // ncomps: number of components per processor
  int const ncomps = (nregs + nprocs - 1) / nprocs;
  if (recompose_verbose)
    cout << "SRMA ncomps " << ncomps << endl;
  assert(ncomps > 0);

  int const newnregs = nprocs * ncomps;
  if (recompose_verbose)
    cout << "SRMA newnregs " << newnregs << endl;

  if (recompose_verbose)
    cout << "SRMA: distributing processors to regions" << endl;
  vector<CCTK_REAL> mycosts(nregs);
  for (int r = 0; r < nregs; ++r) {
    mycosts.AT(r) = prod(cost(superregs.AT(r)));
  }
  int nregs_left = newnregs;
  vector<int> mynprocs(nregs);
  for (int r = 0; r < nregs; ++r) {
    mynprocs.AT(r) = 1;
    --nregs_left;
  }
  // TODO: split regions if necessary
  while (nregs_left > 0) {
    if (recompose_verbose)
      cout << "SRMA nregs_left " << nregs_left << endl;
    int maxr = -1;
    CCTK_REAL maxratio = -1;
    for (int r = 0; r < nregs; ++r) {
      CCTK_REAL const ratio = mycosts.AT(r) / mynprocs.AT(r);
      if (ratio > maxratio) {
        maxr = r;
        maxratio = ratio;
      }
    }
    assert(maxr >= 0 and maxr < nregs);
    ++mynprocs.AT(maxr);
    if (recompose_verbose)
      cout << "SRMA maxr " << maxr << endl;
    if (recompose_verbose)
      cout << "SRMA mynprocs[maxr] " << mynprocs.AT(maxr) << endl;
    --nregs_left;
  }
  assert(nregs_left == 0);
  int sum_nprocs = 0;
  for (int r = 0; r < nregs; ++r) {
    sum_nprocs += mynprocs.AT(r);
  }
  assert(sum_nprocs == newnregs);
  if (recompose_verbose)
    cout << "SRMA mynprocs " << mynprocs << endl;

  if (recompose_verbose)
    cout << "SRMA: splitting work units" << endl;
  // TODO: rename newregs to regs
  // TODO: rename nregs to nsuperregs
  // TODO: rename newnregs to nregs
  vector<region_t> newregs;
  newregs.reserve(newnregs);
  for (int r = 0, p = 0; r < nregs; p += mynprocs.AT(r), ++r) {
    if (recompose_verbose)
      cout << "SRMA superreg[" << r << "] " << superregs.AT(r) << endl;
    // bvect const dims = false;
    bvect const dims = not split_components || no_split_dims;
    SplitRegionsMaps_Automatic_Recursively(dims, p, mynprocs.AT(r),
                                           superregs.AT(r), newregs);
  } // for r
  if (recompose_verbose)
    cout << "SRMA newregs " << newregs << endl;
  assert(int(newregs.size()) == newnregs);

  // Count components per map
  vector<int> myncomps(nmaps, 0);
#if 0
    vector<int> empty_comps(nmaps, 0);
#endif
  for (int r = 0; r < newnregs; ++r) {
    int const m = newregs.AT(r).map - minmap;
    assert(m >= 0 and m < nmaps);
#if 0
      if (not newregs.AT(r).extent.empty()) {
        // Ignore empty regions, which may come from empty grid arrays
        ++ myncomps.AT(m);
      } else {
        ++ empty_comps.AT(m);
      }
#endif
    ++myncomps.AT(m);
  }
  vector<int> mynregs(nmaps, 0);
#if 0
    vector<int> empty_regs(nmaps, 0);
#endif
  for (int r = 0; r < nregs; ++r) {
    int const m = superregs.AT(r).map - minmap;
    assert(m >= 0 and m < nmaps);
    ++mynregs.AT(m);
#if 0
      if (not superregs.AT(r).extent.empty()) {
        // Ignore empty regions, which may come from empty grid arrays
        ++ mynregs.AT(m);
      } else {
        ++ empty_regs.AT(m);
      }
#endif
  }

  // Convert virtual to real processors
  for (int r = 0; r < newnregs; ++r) {
    newregs.AT(r).processor /= ncomps;
    assert(newregs.AT(r).processor >= 0 and newregs.AT(r).processor < nprocs);
  }
  {
    vector<int> tmpncomps(nmaps, 0);
    for (int r = 0; r < nregs; ++r) {
      int const m = superregs.AT(r).map - minmap;
      ipfulltree *const regf = superregs.AT(r).processors;
      assert(regf != NULL);
      for (ipfulltree::iterator fti(*regf); not fti.done(); ++fti) {
        pseudoregion_t &preg = (*fti).payload();
        preg.component = tmpncomps.AT(m)++;
      }
    }
    for (int m = 0; m < nmaps; ++m) {
      if (not(tmpncomps.AT(m) == myncomps.AT(m))) {
        cout << "Recompose.cc" << endl
             << "superregss=" << superregss << endl
             << "regss=" << regss << endl
             << "nregs=" << nregs << endl
             << "newregs=" << newregs << endl
             << "newnregs=" << newnregs << endl
             << "m=" << m << endl
             << "tmpncomps.AT(m)=" << tmpncomps.AT(m) << endl
             << "myncomps.AT(m)=" << myncomps.AT(m) << endl;
        cout << "newregs:" << endl;
        for (int r = 0; r < newnregs; ++r) {
          int const m = newregs.AT(r).map - minmap;
          assert(m >= 0 and m < nmaps);
          cout << "r=" << r << endl
               << "newregs=" << newregs.at(r) << endl
               << "newregs.extent.size()=" << newregs.at(r).extent.size()
               << endl
               << "newregs.extent.empty()=" << newregs.at(r).extent.empty()
               << endl;
        }
        cout << "*regf:" << endl;
        for (int r = 0; r < nregs; ++r) {
          int const m = superregs.AT(r).map - minmap;
          ipfulltree *const regf = superregs.AT(r).processors;
          cout << "r=" << r << endl << "m=" << m << endl;
          cout << "*regf=" << *regf << endl;
        }
        cout.flush();
      }
      assert(tmpncomps.AT(m) == myncomps.AT(m));
    }
  }
  // Count componets per process
  {
    vector<int> ncompsperproc(real_nprocs, 0);
    for (int r = 0; r < newnregs; ++r) {
      int const p = newregs.AT(r).processor;
      ++ncompsperproc.at(p);
    }
    if (same_number_of_components_on_each_process) {
      for (int p = 0; p < real_nprocs; ++p) {
        assert(ncompsperproc.at(p) == ncomps);
      }
    }
  }

  // Distribute regions
  // Allocate regions
  assert((int)regss.size() == nmaps);
  for (int m = 0; m < nmaps; ++m) {
    assert(regss.AT(m).empty());
    regss.AT(m).reserve(myncomps.AT(m));
    superregss.AT(m).clear();
    superregss.AT(m).reserve(mynregs.AT(m));
  }
  // Assign regions
  for (int r = 0; r < newnregs; ++r) {
    int const m = newregs.AT(r).map - minmap;
    assert(m >= 0 and m < nmaps);
    regss.AT(m).push_back(newregs.AT(r));
  }
  for (int r = 0; r < nregs; ++r) {
    int const m = superregs.AT(r).map - minmap;
    assert(m >= 0 and m < nmaps);
    superregss.AT(m).push_back(superregs.AT(r));
  }
  // Output regions
  if (recompose_verbose) {
    cout << "SRMA superregss " << superregss << endl;
    cout << "SRMA regss " << regss << endl;
  }
  // Check sizes
  for (int m = 0; m < nmaps; ++m) {
#if 0
      assert (int(regss.AT(m).size()) == myncomps.AT(m) + empty_comps.AT(m));
#endif
    assert(int(regss.AT(m).size()) == myncomps.AT(m));
    assert(int(superregss.AT(m).size()) == mynregs.AT(m));
  }

  if (recompose_verbose)
    cout << "SRMA exit" << endl;
}

void SplitRegionsMaps_Balanced(cGH const *const cctkGH,
                               vector<vector<region_t> > &superregss,
                               vector<vector<region_t> > &regss) {
  DECLARE_CCTK_PARAMETERS;

  // Find map numbers
  int const nmaps = superregss.size();
  vector<int> map_index(maps, -1);
  for (int m = 0; m < nmaps; ++m) {
    assert(not superregss.AT(m).empty());
    if (not superregss.AT(m).empty()) {
      assert(map_index.AT(superregss.AT(m).AT(0).map) == -1);
      map_index.AT(superregss.AT(m).AT(0).map) = m;
    }
  }

  // Combine regions from all maps
  vector<region_t> regs;
  for (int m = 0; m < nmaps; ++m) {
    combine_regions(superregss.AT(m), regs);
  }

  // Create superregion tree structure
  {
    int const ncomps = regs.size();
    for (int c = 0; c < ncomps; ++c) {
      assert(regs.AT(c).processor == -1);
      assert(not regs.AT(c).processors);
      pseudoregion_t const preg(regs.AT(c).extent, -1);
      regs.AT(c).processors = new ipfulltree(preg);
    }
  }

  // Set up new superregions
  // Note: The tree structure is now reachable from both the
  // superregions and the regions
  for (int m = 0; m < nmaps; ++m) {
    superregss.AT(m).clear();
  }
  {
    int const ncomps = regs.size();
    for (int c = 0; c < ncomps; ++c) {
      int const m = map_index.AT(regs.AT(c).map);
      superregss.AT(m).push_back(regs.AT(c));
    }
  }

  // Split onto processes
  vector<vector<region_t> > split_regss;
  int const nprocs = CCTK_nProcs(cctkGH);
  int const nworkers = nprocs;
  balance(regs, split_regss, nworkers, maximum_imbalance,
          same_number_of_components_on_each_process);

  // Assign process numbers, and make the tree structure
  // inaccessible from the regions
  for (int p = 0; p < nprocs; ++p) {
    int const ncomps = split_regss.AT(p).size();
    for (int c = 0; c < ncomps; ++c) {
      // Set process number in tree structure
      assert(split_regss.AT(p).AT(c).processors);
      assert(split_regss.AT(p).AT(c).processors->is_leaf());
      assert(split_regss.AT(p).AT(c).processors->payload().component == -1);
      split_regss.AT(p).AT(c).processors->payload().component = p;
      // Set process number in region
      split_regss.AT(p).AT(c).processors = NULL;
      assert(split_regss.AT(p).AT(c).processor == -1);
      split_regss.AT(p).AT(c).processor = p;
    }
  }

  // Distribute by map, implicitly assigning component numbers
  for (int p = 0; p < nprocs; ++p) {
    int const ncomps = split_regss.AT(p).size();
    for (int c = 0; c < ncomps; ++c) {
      int const m = map_index.AT(split_regss.AT(p).AT(c).map);
      regss.AT(m).push_back(split_regss.AT(p).AT(c));
    }
  }
}

static void SplitRegionsMaps_AsSpecified(cGH const *cctkGH,
                                         vector<vector<region_t> > &superregss,
                                         vector<vector<region_t> > &regss) {
  DECLARE_CCTK_PARAMETERS;

  int const nmaps = superregss.size();
  assert(nmaps == 1); // for now
  assert((int)regss.size() == nmaps);

  SplitRegions_AsSpecified(cctkGH, superregss.AT(0), regss.AT(0));
}

//////////////////////////////////////////////////////////////////////////////

static void MakeMultigridBoxes(cGH const *const cctkGH, int const m,
                               ibbox const &base, region_t const &reg,
                               vector<region_t> &regs) {
  DECLARE_CCTK_PARAMETERS;

  regs.resize(mglevels, reg);
  if (mglevels > 1) {
    // boundary offsets
    jjvect nboundaryzones, is_internal, is_staggered, shiftout;
    if (domain_from_multipatch and
        CCTK_IsFunctionAliased("MultiPatch_GetBoundarySpecification")) {
      check(not MultiPatch_GetBoundarySpecification(
          m, 2 * dim, &nboundaryzones[0][0], &is_internal[0][0],
          &is_staggered[0][0], &shiftout[0][0]));
    } else {
      check(not GetBoundarySpecification(2 * dim, &nboundaryzones[0][0],
                                         &is_internal[0][0],
                                         &is_staggered[0][0], &shiftout[0][0]));
    }
    // (distance in grid points between the exterior and the physical boundary)
    iivect offset;
    for (int d = 0; d < dim; ++d) {
      for (int f = 0; f < 2; ++f) {
        assert(not is_staggered[d][f]);
        offset[d][f] = (+(is_internal[d][f] ? 0 : nboundaryzones[d][f] - 1) +
                        shiftout[d][f]);
      }
    }
    vector<ibbox> bases(mglevels);
    bases.AT(0) = base;
    for (int ml = 1; ml < mglevels; ++ml) {
      // next finer base
      ivect const fbaselo = bases.AT(ml - 1).lower();
      ivect const fbasehi = bases.AT(ml - 1).upper();
      ivect const fbasestr = bases.AT(ml - 1).stride();
      // this base
      ivect const basestr = fbasestr * mgfact;
      ivect const baselo =
          fbaselo +
          (xpose(offset)[0] - ivect(mgfact) * xpose(offset)[0]) * fbasestr;
      ivect const basehi =
          fbasehi +
          (xpose(offset)[1] - ivect(mgfact) * xpose(offset)[1]) * fbasestr;
      ivect const baselo1 = baselo;
      ivect const basehi1 = baselo1 + (basehi - baselo1) / basestr * basestr;
      bases.AT(ml) = ibbox(baselo1, basehi1, basestr);
      // next finer grid
      ivect const flo = regs.AT(ml - 1).extent.lower();
      ivect const fhi = regs.AT(ml - 1).extent.upper();
      ivect const fstr = regs.AT(ml - 1).extent.stride();
      // this grid
      ivect const str = fstr * mgfact;
      ivect const lo =
          flo +
          either(reg.outer_boundaries[0],
                 (xpose(offset)[0] - ivect(mgfact) * xpose(offset)[0]) * fstr,
                 ivect(0));
      ivect const hi =
          fhi +
          either(reg.outer_boundaries[1],
                 -(xpose(offset)[1] - ivect(mgfact) * xpose(offset)[1]) * fstr,
                 ivect(0));
      ivect const lo1 = baselo1 + (lo - baselo1 + str - 1) / str * str;
      ivect const hi1 = lo1 + (hi - lo1) / str * str;
      regs.AT(ml).extent = ibbox(lo1, hi1, str);
    }
  }
}

void MakeMultigridBoxes(cGH const *const cctkGH, int const m,
                        vector<vector<region_t> > const &regss,
                        vector<vector<vector<region_t> > > &regsss) {
  regsss.resize(mglevels);
  for (int ml = 0; ml < mglevels; ++ml) {
    regsss.AT(ml).resize(regss.size());
    for (int rl = 0; rl < (int)regss.size(); ++rl) {
      regsss.AT(ml).AT(rl).resize(regss.AT(rl).size());
    }
  }

  for (int rl = 0; rl < (int)regss.size(); ++rl) {

    ibbox base;
    for (int c = 0; c < (int)regss.AT(rl).size(); ++c) {
      base = base.expanded_containing(regss.AT(rl).AT(c).extent);
    }

    for (int c = 0; c < (int)regss.AT(rl).size(); ++c) {
      vector<region_t> mg_regs;
      MakeMultigridBoxes(cctkGH, m, base, regss.AT(rl).AT(c), mg_regs);
      assert((int)mg_regs.size() == mglevels);
      for (int ml = 0; ml < mglevels; ++ml) {
        regsss.AT(ml).AT(rl).AT(c) = mg_regs.AT(ml);
      }

    } // for c

  } // for rl
}

void MakeMultigridBoxesMaps(
    cGH const *const cctkGH, vector<vector<vector<region_t> > > const &regsss,
    vector<vector<vector<vector<region_t> > > > &regssss) {
  regssss.resize(maps);
  for (int m = 0; m < maps; ++m) {
    MakeMultigridBoxes(cctkGH, m, regsss.AT(m), regssss.AT(m));
  } // for m
}

static void ClassifyPoints(cGH const *const cctkGH, int const rl) {
  // negative: needs to be set explicitly (e.g. boundary)
  // zero:     unused (e.g. ghost)
  // positive: needs to be evolved
  // -1:      boundary point (needs to be set explicitly)
  //  0:      unused (e.g. ghost point, or restriction target)
  //  n=1..N: evolved, used for integrator substeps i<=n
  //          (i=N..1, counting backwards; see MoL documentation)
  //          i.e.: n=1: used between time steps (i.e., should be visualised)
  //                n>1: used only while time stepping (e.g. buffer zones)

  BEGIN_META_MODE(cctkGH) {
    BEGIN_MGLEVEL_LOOP(cctkGH) {
      ENTER_LEVEL_MODE(cctkGH, rl) {
        BEGIN_LOCAL_MAP_LOOP(cctkGH, CCTK_GF) {
          BEGIN_LOCAL_COMPONENT_LOOP(cctkGH, CCTK_GF) {
            DECLARE_CCTK_ARGUMENTS;
            if (point_class) {
#pragma omp parallel
              CCTK_LOOP3_ALL(CarpetClassifyPoints, cctkGH, i, j, k) {
                int const ind = CCTK_GFINDEX3D(cctkGH, i, j, k);
                point_class[ind] = 1;
              }
              CCTK_ENDLOOP3_ALL(CarpetClassifyPoints);
            }
          }
          END_LOCAL_COMPONENT_LOOP;
        }
        END_LOCAL_MAP_LOOP;
      }
      LEAVE_LEVEL_MODE;
    }
    END_MGLEVEL_LOOP;
  }
  END_META_MODE;
}

} // namespace Carpet

```

CallFunction.cc:
```
#include <algorithm>
#include <cassert>
#include <cstdlib>
#include <cstring>
#include <map>
#include <string>
#include <sstream>

#include <cctk.h>
#include <cctk_Parameters.h>
#include <cctki_GHExtensions.h>

#include <Requirements.hh>

#include <Timer.hh>

#include <gh.hh>

#include <carpet.hh>

#include "adler32.hh"

namespace Carpet {

using namespace std;

static void CallScheduledFunction(char const *restrict time_and_mode,
                                  void *function, cFunctionData *attribute,
                                  void *data, Timers::Timer &user_timer);

static void SyncGroupsInScheduleBlock(cFunctionData *attribute, cGH *cctkGH,
                                      vector<int> const &sync_groups,
                                      Timers::Timer &sync_timer);

// check if scheduled function overwrote any of the poison surrounding
// allocated memory
static void CheckFence(cGH const *const cctkGH, cFunctionData *attribute);

/// Traverse one function on all components of one refinement level
/// of one multigrid level.
int CallFunction(void *function,           ///< the function to call
                 cFunctionData *attribute, ///< attributes of the function
                 void *data) ///< private data for CCTK_CallFunction
{
  DECLARE_CCTK_PARAMETERS;

  static Timers::Timer total_timer("CallFunction");
  static Timers::Timer user_timer("thorns");
  static Timers::Timer sync_timer("syncs");

  total_timer.start();

  cGH *cctkGH = static_cast<cGH *>(data);

  assert(
      int(not not attribute->meta) + int(not not attribute->meta_early) +
          int(not not attribute->meta_late) + int(not not attribute->global) +
          int(not not attribute->global_early) +
          int(not not attribute->global_late) + int(not not attribute->level) +
          int(not not attribute->singlemap) + int(not not attribute->local) <=
      1);

  assert(not not attribute->loop_global + not not attribute->loop_level +
             not not attribute->loop_singlemap +
             not not attribute->loop_local <=
         1);

  // Create list of all groups that need to be synchronised
  vector<int> sync_groups;
  sync_groups.reserve(attribute->n_SyncGroups);
  for (int g = 0; g < attribute->n_SyncGroups; g++) {
    const int group = attribute->SyncGroups[g];
    if (CCTK_NumVarsInGroupI(group) > 0) {
      // don't add empty groups from the list
      sync_groups.push_back(group);
    }
  }

  if (attribute->meta or attribute->meta_early or attribute->meta_late or
      is_meta_mode()) {
    // Convtest operation

    if ((attribute->meta and do_meta_mode) or
        (attribute->meta_early and do_early_meta_mode) or
        (attribute->meta_late and do_late_meta_mode) or is_meta_mode()) {
      if (attribute->loop_local) {
        BEGIN_META_MODE(cctkGH) {
          BEGIN_MGLEVEL_LOOP(cctkGH) {
            BEGIN_REFLEVEL_LOOP(cctkGH) {
              BEGIN_LOCAL_MAP_LOOP(cctkGH, CCTK_GF) {
                BEGIN_LOCAL_COMPONENT_LOOP(cctkGH, CCTK_GF) {
                  CallScheduledFunction("Meta time local mode", function,
                                        attribute, data, user_timer);
                }
                END_LOCAL_COMPONENT_LOOP;
              }
              END_LOCAL_MAP_LOOP;
              if (not sync_groups.empty()) {
                SyncGroupsInScheduleBlock(attribute, cctkGH, sync_groups,
                                          sync_timer);
              }
            }
            END_REFLEVEL_LOOP;
          }
          END_MGLEVEL_LOOP;
        }
        END_META_MODE;
      } else if (attribute->loop_singlemap) {
        BEGIN_META_MODE(cctkGH) {
          BEGIN_MGLEVEL_LOOP(cctkGH) {
            BEGIN_REFLEVEL_LOOP(cctkGH) {
              BEGIN_MAP_LOOP(cctkGH, CCTK_GF) {
                CallScheduledFunction("Meta time singlemap mode", function,
                                      attribute, data, user_timer);
              }
              END_MAP_LOOP;
              if (not sync_groups.empty()) {
                SyncGroupsInScheduleBlock(attribute, cctkGH, sync_groups,
                                          sync_timer);
              }
            }
            END_REFLEVEL_LOOP;
          }
          END_MGLEVEL_LOOP;
        }
        END_META_MODE;
      } else if (attribute->loop_level) {
        BEGIN_META_MODE(cctkGH) {
          BEGIN_MGLEVEL_LOOP(cctkGH) {
            BEGIN_REFLEVEL_LOOP(cctkGH) {
              CallScheduledFunction("Meta time level mode", function, attribute,
                                    data, user_timer);
              if (not sync_groups.empty()) {
                SyncGroupsInScheduleBlock(attribute, cctkGH, sync_groups,
                                          sync_timer);
              }
            }
            END_REFLEVEL_LOOP;
          }
          END_MGLEVEL_LOOP;
        }
        END_META_MODE;
      } else if (attribute->loop_global) {
        BEGIN_META_MODE(cctkGH) {
          BEGIN_MGLEVEL_LOOP(cctkGH) {
            CallScheduledFunction("Meta time global mode", function, attribute,
                                  data, user_timer);
            if (not sync_groups.empty()) {
              BEGIN_REFLEVEL_LOOP(cctkGH) {
                SyncGroupsInScheduleBlock(attribute, cctkGH, sync_groups,
                                          sync_timer);
              }
              END_REFLEVEL_LOOP;
            }
          }
          END_MGLEVEL_LOOP;
        }
        END_META_MODE;
      } else {
        BEGIN_META_MODE(cctkGH) {
          CallScheduledFunction("Meta mode", function, attribute, data,
                                user_timer);
          if (not sync_groups.empty()) {
            BEGIN_MGLEVEL_LOOP(cctkGH) {
              BEGIN_REFLEVEL_LOOP(cctkGH) {
                SyncGroupsInScheduleBlock(attribute, cctkGH, sync_groups,
                                          sync_timer);
              }
              END_REFLEVEL_LOOP;
            }
            END_MGLEVEL_LOOP;
          }
        }
        END_META_MODE;
      }
    }

  } else if (attribute->global or attribute->global_early or
             attribute->global_late or is_global_mode()) {
    // Global operation: call once

    if ((attribute->global and do_global_mode) or
        (attribute->global_early and do_early_global_mode) or
        (attribute->global_late and do_late_global_mode) or is_global_mode()) {
      if (attribute->loop_local) {
        BEGIN_GLOBAL_MODE(cctkGH) {
          BEGIN_REFLEVEL_LOOP(cctkGH) {
            BEGIN_LOCAL_MAP_LOOP(cctkGH, CCTK_GF) {
              BEGIN_LOCAL_COMPONENT_LOOP(cctkGH, CCTK_GF) {
                CallScheduledFunction("Global time local mode", function,
                                      attribute, data, user_timer);
              }
              END_LOCAL_COMPONENT_LOOP;
            }
            END_LOCAL_MAP_LOOP;
            if (not sync_groups.empty()) {
              SyncGroupsInScheduleBlock(attribute, cctkGH, sync_groups,
                                        sync_timer);
            }
          }
          END_REFLEVEL_LOOP;
        }
        END_GLOBAL_MODE;
      } else if (attribute->loop_singlemap) {
        BEGIN_GLOBAL_MODE(cctkGH) {
          BEGIN_REFLEVEL_LOOP(cctkGH) {
            BEGIN_MAP_LOOP(cctkGH, CCTK_GF) {
              CallScheduledFunction("Global time singlemap mode", function,
                                    attribute, data, user_timer);
            }
            END_MAP_LOOP;
            if (not sync_groups.empty()) {
              SyncGroupsInScheduleBlock(attribute, cctkGH, sync_groups,
                                        sync_timer);
            }
          }
          END_REFLEVEL_LOOP;
        }
        END_GLOBAL_MODE;
      } else if (attribute->loop_level) {
        BEGIN_GLOBAL_MODE(cctkGH) {
          BEGIN_REFLEVEL_LOOP(cctkGH) {
            CallScheduledFunction("Global time level mode", function, attribute,
                                  data, user_timer);
            if (not sync_groups.empty()) {
              SyncGroupsInScheduleBlock(attribute, cctkGH, sync_groups,
                                        sync_timer);
            }
          }
          END_REFLEVEL_LOOP;
        }
        END_GLOBAL_MODE;
      } else {
        BEGIN_GLOBAL_MODE(cctkGH) {
          CallScheduledFunction("Global mode", function, attribute, data,
                                user_timer);
          if (not sync_groups.empty()) {
            BEGIN_REFLEVEL_LOOP(cctkGH) {
              SyncGroupsInScheduleBlock(attribute, cctkGH, sync_groups,
                                        sync_timer);
            }
            END_REFLEVEL_LOOP;
          }
        }
        END_GLOBAL_MODE;
      }
    }

  } else if (attribute->level) {
    // Level operation: call once per refinement level

    if (attribute->loop_local) {
      BEGIN_LOCAL_MAP_LOOP(cctkGH, CCTK_GF) {
        BEGIN_LOCAL_COMPONENT_LOOP(cctkGH, CCTK_GF) {
          CallScheduledFunction("Level time local mode", function, attribute,
                                data, user_timer);
        }
        END_LOCAL_COMPONENT_LOOP;
      }
      END_LOCAL_MAP_LOOP;
    } else if (attribute->loop_singlemap) {
      BEGIN_MAP_LOOP(cctkGH, CCTK_GF) {
        CallScheduledFunction("Level time singlemap mode", function, attribute,
                              data, user_timer);
      }
      END_MAP_LOOP;
    } else {
      CallScheduledFunction("Level mode", function, attribute, data,
                            user_timer);
    }
    if (not sync_groups.empty()) {
      SyncGroupsInScheduleBlock(attribute, cctkGH, sync_groups, sync_timer);
    }

  } else if (attribute->singlemap) {
    // Single map operation: call once per refinement level and map

    if (attribute->loop_local) {
      BEGIN_LOCAL_MAP_LOOP(cctkGH, CCTK_GF) {
        BEGIN_LOCAL_COMPONENT_LOOP(cctkGH, CCTK_GF) {
          CallScheduledFunction("Singlemap time local mode", function,
                                attribute, data, user_timer);
        }
        END_LOCAL_COMPONENT_LOOP;
      }
      END_LOCAL_MAP_LOOP;
    } else {
      BEGIN_MAP_LOOP(cctkGH, CCTK_GF) {
        CallScheduledFunction("Singlemap mode", function, attribute, data,
                              user_timer);
      }
      END_MAP_LOOP;
    }
    if (not sync_groups.empty()) {
      SyncGroupsInScheduleBlock(attribute, cctkGH, sync_groups, sync_timer);
    }

  } else {
    // Local operation: call once per component

    BEGIN_LOCAL_MAP_LOOP(cctkGH, CCTK_GF) {
      BEGIN_LOCAL_COMPONENT_LOOP(cctkGH, CCTK_GF) {
        CallScheduledFunction("Local mode", function, attribute, data,
                              user_timer);
      }
      END_LOCAL_COMPONENT_LOOP;
    }
    END_LOCAL_MAP_LOOP;
    if (not sync_groups.empty()) {
      SyncGroupsInScheduleBlock(attribute, cctkGH, sync_groups, sync_timer);
    }
  }

  if (schedule_barriers) {
    // Create an ID that is almost unique for this scheduled
    // function call
    stringstream buf;
    buf << cctkGH->cctk_iteration << "\n";
    buf << attribute->meta << attribute->meta_early << attribute->meta_late
        << attribute->global << attribute->global_early
        << attribute->global_late << attribute->level << attribute->singlemap
        << attribute->local << "\n";
    buf << attribute->where << "\n";
    buf << attribute->thorn << "\n";
    buf << attribute->routine << "\n";
    string const str = buf.str();
    int const id = adler32(str.c_str(), str.length());
    static Timers::Timer barrier_timer("barrier");
    barrier_timer.start();
    Carpet::NamedBarrier(NULL, id, "Carpet::CallFunction");
    barrier_timer.stop();
  }

  total_timer.stop();

  // The return value indicates whether the grid functions have been
  // synchronised.
  // 0: let the flesh do the synchronisation
  // 1: we did the synchronisation
  return 1;
}

void CallScheduledFunction(char const *restrict const time_and_mode,
                           void *const function, cFunctionData *const attribute,
                           void *const data, Timers::Timer &user_timer) {
  cGH const *const cctkGH = static_cast<cGH const *>(data);
  Checkpoint("%s call at %s to %s::%s", time_and_mode, attribute->where,
             attribute->thorn, attribute->routine);
  int const skip = CallBeforeRoutines(cctkGH, function, attribute, data);
  if (not skip) {
    Timers::Timer timer(attribute->routine);

    // Save the time step size
    CCTK_REAL const saved_cctk_delta_time = cctkGH->cctk_delta_time;

    user_timer.start();
#ifdef REQUIREMENTS_HH
    Requirements::BeforeRoutine(attribute, cctkGH->cctk_iteration, reflevel,
                                map, timelevel, timelevel_offset);
#endif
    timer.start();
    if (CCTK_IsFunctionAliased("Accelerator_PreCallFunction")) {
      Timers::Timer pre_timer("PreCall");
      pre_timer.start();
      Accelerator_PreCallFunction(cctkGH, attribute);
      pre_timer.stop();
    }
    int const res = CCTK_CallFunction(function, attribute, data);
    assert(res == 0);
    if (CCTK_IsFunctionAliased("Accelerator_PostCallFunction")) {
      Timers::Timer post_timer("PostCall");
      post_timer.start();
      Accelerator_PostCallFunction(cctkGH, attribute);
      post_timer.stop();
    }
    timer.stop();
    CheckFence(cctkGH, attribute);
#ifdef REQUIREMENTS_HH
    Requirements::AfterRoutine(attribute, cctkGH->cctk_iteration, reflevel, map,
                               timelevel, timelevel_offset);
#endif
    user_timer.stop();

    // Manage the time step size. If the time step size changes
    // during initialisation, assume it is thorn Time, and update
    // the time hierarchy. If it changes during evolution, assume it
    // is MoL, and do nothing.
    if (cctkGH->cctk_iteration == 0 and
        cctkGH->cctk_delta_time != saved_cctk_delta_time) {
      // The user changed cctk_delta_time during initialisation --
      // update our internals and the time hierarchy
      bool const is_global = attribute->meta or attribute->meta_early or
                             attribute->meta_late or attribute->global or
                             attribute->global_early or attribute->global_late;
      delta_time = cctkGH->cctk_delta_time / mglevelfact *
                   (is_global ? 1.0 : timereflevelfact);
      for (int ml = 0; ml < mglevels; ++ml) {
        for (int rl = 0; rl < reflevels; ++rl) {
          // Update the time delta
          CCTK_REAL const dt =
              delta_time / timereffacts.AT(rl) * ipow(mgfact, ml);
          tt->set_delta(ml, rl, dt);
          CCTK_REAL const t0 = tt->get_time(ml, rl, 0);
          // Update the times of the past timelevels
          for (int tl = 1; tl < timelevels; ++tl) {
            CCTK_REAL const t = t0 - tl * dt;
            tt->set_time(ml, rl, tl, t);
          }
        }
      }
    }
  }
  CallAfterRoutines(cctkGH, function, attribute, data);
}

void SyncGroupsInScheduleBlock(cFunctionData *attribute, cGH *cctkGH,
                               vector<int> const &sync_groups,
                               Timers::Timer &sync_timer) {
  DECLARE_CCTK_PARAMETERS;

  if (sync_barriers) {
    // Create an ID that is almost unique for this scheduled
    // function call
    // stringstream buf;
    // buf << cctkGH->cctk_iteration << "\n";
    // buf << attribute->meta
    //     << attribute->meta_early
    //     << attribute->meta_late
    //     << attribute->global
    //     << attribute->global_early
    //     << attribute->global_late
    //     << attribute->level
    //     << attribute->singlemap
    //     << attribute->local << "\n";
    // buf << attribute->where << "\n";
    // buf << attribute->thorn << "\n";
    // buf << attribute->routine << " sync\n";
    // string const str = buf.str();
    // int const id = adler32(str.c_str(), str.length());
    int const id = 101;
    static Timers::Timer barrier_timer("pre_sync_barrier");
    barrier_timer.start();
    Carpet::NamedBarrier(NULL, id, "Carpet::Sync");
    barrier_timer.stop();
  }

  sync_timer.start();
  SyncProlongateGroups(cctkGH, sync_groups, attribute);
  sync_timer.stop();

  if (sync_barriers) {
    // Create an ID that is almost unique for this scheduled
    // function call
    // stringstream buf;
    // buf << cctkGH->cctk_iteration << "\n";
    // buf << attribute->meta
    //     << attribute->meta_early
    //     << attribute->meta_late
    //     << attribute->global
    //     << attribute->global_early
    //     << attribute->global_late
    //     << attribute->level
    //     << attribute->singlemap
    //     << attribute->local << "\n";
    // buf << attribute->where << "\n";
    // buf << attribute->thorn << "\n";
    // buf << attribute->routine << " sync\n";
    // string const str = buf.str();
    // int const id = adler32(str.c_str(), str.length());
    int const id = 102;
    static Timers::Timer barrier_timer("post_sync_barrier");
    barrier_timer.start();
    Carpet::NamedBarrier(NULL, id, "Carpet::Sync");
    barrier_timer.stop();
  }
}

void CheckFence(cGH const *const cctkGH, cFunctionData *attribute) {
  DECLARE_CCTK_PARAMETERS;

  if (is_meta_mode())
    return; // meta mode has no accessible variables
  // enumerating the grid variables is expensive
  if (not gdata::fence_is_energized())
    return;

  Timers::Timer timer("FenceCheck");
  timer.start();

  for (int group = 0; group < CCTK_NumGroups(); ++group) {
    if (CCTK_QueryGroupStorageI(cctkGH, group)) {
      int const nvar = CCTK_NumVarsInGroupI(group);
      if (nvar == 0)
        continue;
      int const n0 = CCTK_FirstVarIndexI(group);
      assert(n0 >= 0);

      int const num_tl = CCTK_ActiveTimeLevelsVI(cctkGH, n0);
      if (num_tl == 0)
        continue;
      int const min_tl = 0;
      int const max_tl = num_tl - 1;

      int const grouptype = CCTK_GroupTypeI(group);
      if (grouptype == CCTK_GF && not is_local_mode())
        continue;
      assert(not is_meta_mode());

      // FIXME: query is_XXX() functions
      const bool is_array = grouptype == CCTK_ARRAY or grouptype == CCTK_SCALAR;
      const int m = is_array ? 0 : map;
      const int rl = is_array ? 0 : reflevel;
      const int ml = is_array ? 0 : mglevel;
      const int lc = is_array ? 0 : local_component;

      {
        char *const groupname = CCTK_GroupName(group);
        Checkpoint("FenceCheck \"%s\"", groupname);
        free(groupname);
      }

      for (int var = 0; var < nvar; ++var) {
        assert(n0 + var < CCTK_NumVars());
        for (int tl = min_tl; tl <= max_tl; ++tl) {
          // TODO: turns this into a call of ggf taken a callback routine
          ggf *const ff = arrdata.AT(group).AT(m).data.AT(var);
          assert(ff);
          gdata *const data = ff->data_pointer(tl, rl, lc, ml);
          for (int f = 0; f < 2; ++f) {
            if (not data->check_fence(f)) {
              char *fullname = CCTK_FullName(n0 + var);
              CCTK_VError(__LINE__, __FILE__, CCTK_THORNSTRING,
                          "At iteration %d: timelevel %d, component %d, map "
                          "%d, refinement level %d of the variable \"%s\" "
                          "contains was written to beyond to %s bound of the "
                          "allocated memory by %s::%s at %s.",
                          cctkGH->cctk_iteration, tl, component, m, rl,
                          fullname, f ? "lower" : "upper", attribute->thorn,
                          attribute->routine, attribute->where);
              free(fullname);
            }
          } // for f
        }   // for tl
      }     // for var
    }
  }
  timer.stop();
}

} // namespace Carpet

```

Cycle.cc:
```
#include <cassert>
#include <cstdlib>

#include <cctk.h>
#include <cctk_Parameters.h>

#include <Requirements.hh>

#include <Timer.hh>

#include <ggf.hh>
#include <gh.hh>

#include <carpet.hh>

namespace Carpet {

using namespace std;

void CycleTimeLevels(cGH *const cctkGH) {
  DECLARE_CCTK_PARAMETERS;

  Timers::Timer timer("CycleTimeLevels");
  timer.start();

  Checkpoint("CycleTimeLevels");
  assert(is_level_mode());

  assert(timelevel == 0);
  tt->advance_time(mglevel, reflevel);
  if (not adaptive_stepsize) {
    cctkGH->cctk_time = tt->get_time(mglevel, reflevel, timelevel);
  }

#ifdef REQUIREMENTS_HH
  Requirements::Cycle(reflevel);
#endif

  int errors = 0;

  for (int group = 0; group < CCTK_NumGroups(); ++group) {
    if (CCTK_QueryGroupStorageI(cctkGH, group)) {

      int const activetimelevels = CCTK_ActiveTimeLevelsGI(cctkGH, group);

      if (activetimelevels > 1) {
        switch (groupdata.AT(group).transport_operator) {
        case op_Lagrange:
        case op_ENO:
        case op_WENO:
        case op_Lagrange_monotone:
          if (activetimelevels > 1) {
            if (activetimelevels < prolongation_order_time + 1) {
              char *const groupname = CCTK_GroupName(group);
              CCTK_VWarn(CCTK_WARN_ALERT, __LINE__, __FILE__, CCTK_THORNSTRING,
                         "Group \"%s\" has %d only active time levels.  Groups "
                         "with more than one active time level need at least "
                         "%d active time levels for prolongation_order_time=%d",
                         groupname, activetimelevels,
                         int(prolongation_order_time + 1),
                         int(prolongation_order_time));
              free(groupname);
              ++errors;
            }
          }
          break;
        default:; // do nothing
        }

        switch (CCTK_GroupTypeI(group)) {

        case CCTK_GF:
          assert(reflevel >= 0 and reflevel < reflevels);
          for (int m = 0; m < (int)arrdata.AT(group).size(); ++m) {
            for (int var = 0; var < CCTK_NumVarsInGroupI(group); ++var) {
              arrdata.AT(group).AT(m).data.AT(var)->cycle_all(reflevel,
                                                              mglevel);
            }
          }
          break;

        case CCTK_SCALAR:
        case CCTK_ARRAY:
          if (do_global_mode) {
            int const numtimelevels = CCTK_MaxActiveTimeLevelsVI(cctkGH, group);
            int const firstvarindex = CCTK_FirstVarIndexI(group);
            for (int var = 0; var < CCTK_NumVarsInGroupI(group); ++var) {
              arrdata.AT(group).AT(0).data.AT(var)->cycle_all(0, mglevel);
              {
                int const varindex = firstvarindex + var;
                for (int tl = 0; tl < numtimelevels; ++tl) {
                  if (tl < groupdata.AT(group).info.activetimelevels) {
                    ggf *const ff = arrdata.AT(group).AT(0).data.AT(var);
                    void *const ptr = ff->data_pointer(tl, 0, 0, 0)->storage();
                    cctkGH->data[varindex][tl] = ptr;
                  } else {
                    cctkGH->data[varindex][tl] = NULL;
                  }
                }
              }
            }
          }
          break;

        default:
          assert(0);
        } // switch grouptype
      }   // if activetimelevels > 1
    }     // if storage
  }       // for group

  if (CCTK_IsFunctionAliased("Accelerator_Cycle")) {
    Accelerator_Cycle(cctkGH);
  }

  if (errors > 0) {
    CCTK_VWarn(CCTK_WARN_ABORT, __LINE__, __FILE__, CCTK_THORNSTRING,
               "Errors in %d groups detected; aborting", errors);
  }
  timer.stop();
}

void UncycleTimeLevels(cGH *const cctkGH) {
  DECLARE_CCTK_PARAMETERS;

  Checkpoint("UncycleTimeLevels");
  assert(is_level_mode());

  assert(timelevel == 0);
  tt->retreat_time(mglevel, reflevel);
  assert(not adaptive_stepsize);
  cctkGH->cctk_time = tt->get_time(mglevel, reflevel, timelevel);

  for (int group = 0; group < CCTK_NumGroups(); ++group) {
    if (CCTK_QueryGroupStorageI(cctkGH, group)) {

      switch (CCTK_GroupTypeI(group)) {

      case CCTK_GF:
        assert(reflevel >= 0 and reflevel < reflevels);
        for (int m = 0; m < (int)arrdata.AT(group).size(); ++m) {
          for (int var = 0; var < CCTK_NumVarsInGroupI(group); ++var) {
            arrdata.AT(group).AT(m).data.AT(var)->uncycle_all(reflevel,
                                                              mglevel);
          }
        }
        break;

      case CCTK_SCALAR:
      case CCTK_ARRAY:
        if (do_global_mode) {
          int const numtimelevels = CCTK_MaxActiveTimeLevelsVI(cctkGH, group);
          int const firstvarindex = CCTK_FirstVarIndexI(group);
          for (int var = 0; var < CCTK_NumVarsInGroupI(group); ++var) {
            arrdata.AT(group).AT(0).data.AT(var)->uncycle_all(0, mglevel);
            {
              int const varindex = firstvarindex + var;
              for (int tl = 0; tl < numtimelevels; ++tl) {
                if (tl < groupdata.AT(group).info.activetimelevels) {
                  ggf *const ff = arrdata.AT(group).AT(0).data.AT(var);
                  void *const ptr = ff->data_pointer(tl, 0, 0, 0)->storage();
                  cctkGH->data[varindex][tl] = ptr;
                } else {
                  cctkGH->data[varindex][tl] = NULL;
                }
              }
            }
          }
        }
        break;

      default:
        assert(0);
      } // switch grouptype
    }   // if storage
  }     // for group
}

void FlipTimeLevels(cGH *const cctkGH) {
  DECLARE_CCTK_PARAMETERS;

  Checkpoint("FlipTimeLevels");
  assert(is_level_mode());

  assert(timelevel == 0);
  tt->flip_timelevels(mglevel, reflevel);
  assert(not adaptive_stepsize);
  cctkGH->cctk_time = tt->get_time(mglevel, reflevel, timelevel);
  cctkGH->cctk_delta_time *= -1;

  for (int group = 0; group < CCTK_NumGroups(); ++group) {
    if (CCTK_QueryGroupStorageI(cctkGH, group)) {

      switch (CCTK_GroupTypeI(group)) {

      case CCTK_GF:
        assert(reflevel >= 0 and reflevel < reflevels);
        for (int m = 0; m < (int)arrdata.AT(group).size(); ++m) {
          for (int var = 0; var < CCTK_NumVarsInGroupI(group); ++var) {
            arrdata.AT(group).AT(m).data.AT(var)->flip_all(reflevel, mglevel);
          }
        }
        break;

      case CCTK_SCALAR:
      case CCTK_ARRAY:
        if (do_global_mode) {
          int const numtimelevels = CCTK_MaxActiveTimeLevelsVI(cctkGH, group);
          int const firstvarindex = CCTK_FirstVarIndexI(group);
          for (int var = 0; var < CCTK_NumVarsInGroupI(group); ++var) {
            arrdata.AT(group).AT(0).data.AT(var)->flip_all(0, mglevel);
            {
              int const varindex = firstvarindex + var;
              for (int tl = 0; tl < numtimelevels; ++tl) {
                if (tl < groupdata.AT(group).info.activetimelevels) {
                  ggf *const ff = arrdata.AT(group).AT(0).data.AT(var);
                  void *const ptr = ff->data_pointer(tl, 0, 0, 0)->storage();
                  cctkGH->data[varindex][tl] = ptr;
                } else {
                  cctkGH->data[varindex][tl] = NULL;
                }
              }
            }
          }
        }
        break;

      default:
        assert(0);
      } // switch grouptype
    }   // if storage
  }     // for group
}

void FillTimeLevels(const cGH *const cctkGH) {
  Checkpoint("FillTimeLevels");
  assert(is_level_mode());

  for (int group = 0; group < CCTK_NumGroups(); ++group) {
    if (CCTK_QueryGroupStorageI(cctkGH, group)) {
      switch (CCTK_GroupTypeI(group)) {

      case CCTK_GF:
        assert(reflevel >= 0 and reflevel < reflevels);
        for (int m = 0; m < (int)arrdata.AT(group).size(); ++m) {
          const bool have_accel =
              CCTK_IsFunctionAliased("Accelerator_NotifyDataModified");
          vector<CCTK_INT> vis, rls, tls;
          const int varn = CCTK_NumVarsInGroupI(group);
          for (int var = 0; var < varn; ++var) {
            arrdata.AT(group).AT(m).data.AT(var)->fill_all(reflevel, mglevel);
            if (have_accel) {
              const int var0 = CCTK_FirstVarIndexI(group);
              const int num_tl =
                  arrdata.AT(group).AT(m).data.AT(var)->timelevels(mglevel,
                                                                   reflevel);
              for (int tl = 1; tl < num_tl; tl++) {
                vis.push_back(var0 + var);
                rls.push_back(reflevel);
                tls.push_back(tl);
              }
            }
          }

          if (have_accel) {
            const CCTK_INT on_device = 0;
            Accelerator_NotifyDataModified(cctkGH, &vis.front(), &rls.front(),
                                           &tls.front(), vis.size(), on_device);
          }
        }
        break;

      case CCTK_SCALAR:
      case CCTK_ARRAY:
        if (do_global_mode) {
          for (int var = 0; var < CCTK_NumVarsInGroupI(group); ++var) {
            arrdata.AT(group).AT(0).data.AT(var)->fill_all(0, mglevel);
          }
        }
        break;

      default:
        assert(0);
      } // switch grouptype
    }   // if storage
  }     // for group
}

} // namespace Carpet

```

Evolve.cc:
```
#include <algorithm>
#include <cassert>
#include <cmath>
#include <cstdio>
#include <cstdlib>
#include <iostream>
#include <limits>
#include <map>
#include <string>
#include <sstream>

#include <cctk.h>
#include <cctk_Parameters.h>
#include <cctk_Termination.h>
#include <util_String.h>

#include <Requirements.hh>

#include <CactusTimerSet.hh>
#include <Timer.hh>

#include <dist.hh>
#include <th.hh>

#include <carpet.hh>

namespace Carpet {

using namespace std;

static bool do_terminate(const cGH *cctkGH);

static void AdvanceTime(cGH *cctkGH);
static void CallRegrid(cGH *cctkGH);
static void CallEvol(cGH *cctkGH);
static void CallRestrict(cGH *cctkGH);
static void CallAnalysis(cGH *cctkGH);

static void print_internal_data();

static void ScheduleTraverse(char const *where, char const *name, cGH *cctkGH);
static void OutputGH(char const *where, cGH *cctkGH);

int Evolve(tFleshConfig *const fc) {
  DECLARE_CCTK_PARAMETERS;

  Waypoint("Starting evolution loop");

  int const convlev = 0;
  cGH *cctkGH = fc->GH[convlev];

  // Main loop
  BeginTimingEvolution(cctkGH);
  static Timers::Timer timer("Evolve");
  timer.start();
  while (not do_terminate(cctkGH)) {

    AdvanceTime(cctkGH);
    {
      int const do_every = maxtimereflevelfact / timereffacts.AT(reflevels - 1);
      if ((cctkGH->cctk_iteration - 1) % do_every == 0) {
        ENTER_GLOBAL_MODE(cctkGH, 0) {
          BEGIN_REFLEVEL_LOOP(cctkGH) { CallRegrid(cctkGH); }
          END_REFLEVEL_LOOP;
        }
        LEAVE_GLOBAL_MODE;
      }
    }
    CallEvol(cctkGH);
    CallRestrict(cctkGH);
    CallAnalysis(cctkGH);
    print_internal_data();

    // Print timer values
    {
      Timers::Timer timer("PrintTimers");
      timer.start();
      int const do_every = maxtimereflevelfact / timereffacts.AT(reflevels - 1);
      if (output_timers_every > 0 and
          cctkGH->cctk_iteration % output_timers_every == 0 and
          cctkGH->cctk_iteration % do_every == 0) {
        Timers::CactusTimerSet::writeData(cctkGH, timer_file);
      }

      if (output_timer_tree_every > 0 and
          cctkGH->cctk_iteration % output_timer_tree_every == 0 and
          cctkGH->cctk_iteration % do_every == 0) {
        Timers::Timer::outputTree("Evolve");
      }
      timer.stop();
    }

    // Ensure that all levels have consistent times
    {
      Timers::Timer timer("CheckLevelTimes");
      timer.start();
      CCTK_REAL const eps =
          pow(numeric_limits<CCTK_REAL>::epsilon(), CCTK_REAL(0.75));
      assert(fabs(cctkGH->cctk_time - global_time) <= eps * global_time);
      for (int ml = 0; ml < mglevels; ++ml) {
        for (int rl = 0; rl < reflevels; ++rl) {
          int const do_every =
              ipow(mgfact, ml) * (maxtimereflevelfact / timereffacts.AT(rl));
          if (cctkGH->cctk_iteration % do_every == 0) {
            // assert (fabs (leveltimes.AT(ml).AT(rl) - global_time) <=
            //         eps * global_time);
            assert(fabs(tt->get_time(ml, rl, 0) - global_time) <=
                   eps * global_time);
          }
        }
      }
      timer.stop();
    }

  } // end main loop
  timer.stop();

  Waypoint("Done with evolution loop");

  return 0;
}

bool do_terminate(const cGH *cctkGH) {
  DECLARE_CCTK_PARAMETERS;

  static Timers::Timer timer("DoTerminate");
  timer.start();

  bool term;

  // Do not test on non-active reflevels to save the call to
  // MPI_Allreduce below
  int const do_every = maxtimereflevelfact / timereffacts.AT(reflevels - 1);
  if (cctkGH->cctk_iteration % do_every != 0) {

    term = false;

  } else {

    if (terminate_next or CCTK_TerminationReached(cctkGH)) {

      // Terminate if someone or something said so
      term = true;

    } else {

      // Test the various conditions
      bool const term_iter =
          cctk_itlast >= 0 and cctkGH->cctk_iteration >= cctk_itlast;
      bool const term_time =
          (delta_time > 0.0
               ? (cctkGH->cctk_time >=
                  cctk_final_time - 1.0e-8 * cctkGH->cctk_delta_time)
               : (cctkGH->cctk_time <=
                  cctk_final_time + 1.0e-8 * cctkGH->cctk_delta_time));
      bool const term_runtime =
          max_runtime > 0.0 and CCTK_RunTime() >= 60.0 * max_runtime;

      if (CCTK_Equals(terminate, "never")) {
        term = false;
      } else if (CCTK_Equals(terminate, "iteration")) {
        term = term_iter;
      } else if (CCTK_Equals(terminate, "time")) {
        term = term_time;
        if (term) {
          CCTK_VInfo(CCTK_THORNSTRING,
                     "Terminating due to cctk_final_time at t = %f",
                     cctkGH->cctk_time);
        }

      } else if (CCTK_Equals(terminate, "runtime")) {
        term = term_runtime;
      } else if (CCTK_Equals(terminate, "any")) {
        term = term_iter or term_time or term_runtime;
      } else if (CCTK_Equals(terminate, "all")) {
        term = term_iter and term_time and term_runtime;
      } else if (CCTK_Equals(terminate, "either")) {
        term = term_iter or term_time;
      } else if (CCTK_Equals(terminate, "both")) {
        term = term_iter and term_time;
      } else if (CCTK_Equals(terminate, "immediately")) {
        term = true;
      } else {
        CCTK_WARN(0, "Unsupported termination condition");
        abort(); // keep the compiler happy
      }
    }

    // Reduce termination condition
    int local, global;
    local = term;
    MPI_Allreduce(&local, &global, 1, MPI_INT, MPI_LOR, dist::comm());
    term = global;
  }

  timer.stop();
  return term;
}

//////////////////////////////////////////////////////////////////////////////
//////////////////////////////////////////////////////////////////////////////
//////////////////////////////////////////////////////////////////////////////

void AdvanceTime(cGH *const cctkGH) {
  DECLARE_CCTK_PARAMETERS;

  static Timers::Timer timer("AdvanceTime");
  timer.start();

  Checkpoint("AdvanceTime");

  ++cctkGH->cctk_iteration;

  if (not adaptive_stepsize) {
    // Avoid accumulation of errors
    global_time = cctk_initial_time +
                  cctkGH->cctk_iteration * delta_time / maxtimereflevelfact;
    cctkGH->cctk_time = global_time;
  } else {
    // Take varying step sizes into account
    cctkGH->cctk_time += cctkGH->cctk_delta_time;
    delta_time = cctkGH->cctk_delta_time;
    global_time = cctkGH->cctk_time;
  }

  if ((cctkGH->cctk_iteration - 1) %
          (maxtimereflevelfact / timereffacts.AT(reflevels - 1)) ==
      0) {
    Waypoint("Evolving iteration %d at t=%g", cctkGH->cctk_iteration,
             (double)cctkGH->cctk_time);
  }

  timer.stop();
}

void CallRegrid(cGH *const cctkGH) {
  DECLARE_CCTK_PARAMETERS;

  char const *const where = "CallRegrid";
  static Timers::Timer timer(where);
  timer.start();

  assert(is_level_mode());

  bool const old_do_global_mode = do_global_mode;
  bool const old_do_early_global_mode = do_early_global_mode;
  bool const old_do_late_global_mode = do_late_global_mode;
  bool const old_do_meta_mode = do_meta_mode;
  bool const old_do_early_meta_mode = do_early_meta_mode;
  bool const old_do_late_meta_mode = do_late_meta_mode;
  do_global_mode = true;
  do_early_global_mode = true;
  do_late_global_mode = true;
  do_meta_mode = true;
  do_early_meta_mode = true;
  do_late_meta_mode = true;

  // Preregrid
  Waypoint("Preregrid at iteration %d time %g%s%s", cctkGH->cctk_iteration,
           (double)cctkGH->cctk_time, (do_global_mode ? " (global)" : ""),
           (do_meta_mode ? " (meta)" : ""));
  ScheduleTraverse(where, "CCTK_PREREGRID", cctkGH);

  // Regrid
  Checkpoint("Regrid");
  int const oldreflevels = reflevels;
  bool const did_regrid = Regrid(cctkGH, false, true);
  bool const did_remove_level = reflevels < oldreflevels;
  assert(not did_remove_level or did_regrid);

  if (did_regrid) {
#ifdef REQUIREMENTS_HH
    Requirements::Regrid(reflevels);
#endif
    bool did_any_recompose = false;
    BEGIN_META_MODE(cctkGH) {

      bool have_done_global_mode = false;
      bool have_done_early_global_mode = false;
      bool have_done_late_global_mode = false;
      bool have_done_anything = false;

      for (int rl = 0; rl < reflevels; ++rl) {

        bool const did_recompose = Recompose(cctkGH, rl, true);
        did_any_recompose = did_any_recompose or did_recompose;
#ifdef REQUIREMENTS_HH
        Requirements::Recompose(cctkGH->cctk_iteration, rl,
                                not did_recompose
                                    ? Requirements::valid::everywhere
                                    : Requirements::valid::interior);
#endif

        // Carpet assumes that a regridding operation always changes
        // "level N and all finer levels" so we should call POSTREGRID
        // on all finer levels
        if (did_any_recompose or (did_remove_level and rl == reflevels - 1)) {
          BEGIN_MGLEVEL_LOOP(cctkGH) {
            ENTER_LEVEL_MODE(cctkGH, rl) {
              BeginTimingLevel(cctkGH);

              do_early_global_mode = not have_done_early_global_mode;
              do_late_global_mode = reflevel == reflevels - 1;
              do_early_meta_mode =
                  do_early_global_mode and mglevel == mglevels - 1;
              do_late_meta_mode = do_late_global_mode and mglevel == 0;
              do_global_mode = do_late_global_mode;
              do_meta_mode = do_late_meta_mode;
              assert(not(have_done_global_mode and do_global_mode));
              assert(not(have_done_early_global_mode and do_early_global_mode));
              assert(not(have_done_late_global_mode and do_late_global_mode));
              have_done_global_mode |= do_global_mode;
              have_done_early_global_mode |= do_early_global_mode;
              have_done_late_global_mode |= do_late_global_mode;
              have_done_anything = true;

              BEGIN_TIMELEVEL_LOOP(cctkGH) {

                Waypoint("Postregrid at iteration %d time %g timelevel %d%s%s",
                         cctkGH->cctk_iteration, (double)cctkGH->cctk_time,
                         timelevel, (do_global_mode ? " (global)" : ""),
                         (do_meta_mode ? " (meta)" : ""));

                // Postregrid
                ScheduleTraverse(where, "CCTK_POSTREGRID", cctkGH);
              }
              END_TIMELEVEL_LOOP;

              if (output_after_regridding) {
                // Output
                OutputGH(where, cctkGH);
              }

              EndTimingLevel(cctkGH);
            }
            LEAVE_LEVEL_MODE;
          }
          END_MGLEVEL_LOOP;

        } // if did_recompose

      } // for rl

      if (have_done_anything)
        assert(have_done_global_mode);
      if (have_done_anything)
        assert(have_done_early_global_mode);
      if (have_done_anything)
        assert(have_done_late_global_mode);
    }
    END_META_MODE;
#ifdef REQUIREMENTS_HH
    Requirements::RegridFree();
#endif
  } // if did_regrid

  RegridFree(cctkGH, true);

  do_global_mode = old_do_global_mode;
  do_early_global_mode = old_do_early_global_mode;
  do_late_global_mode = old_do_late_global_mode;
  do_meta_mode = old_do_meta_mode;
  do_early_meta_mode = old_do_early_meta_mode;
  do_late_meta_mode = old_do_late_meta_mode;

  timer.stop();
}

void CallEvol(cGH *const cctkGH) {
  DECLARE_CCTK_PARAMETERS;

  char const *const where = "CallEvol";
  static Timers::Timer timer(where);
  timer.start();

  for (int ml = mglevels - 1; ml >= 0; --ml) {

    bool have_done_global_mode = false;
    bool have_done_anything = false;

    for (int rl = 0; rl < reflevels; ++rl) {
      int const do_every =
          ipow(mgfact, ml) * (maxtimereflevelfact / timereffacts.AT(rl));
      if ((cctkGH->cctk_iteration - 1) % do_every == 0) {
        ENTER_GLOBAL_MODE(cctkGH, ml) {
          ENTER_LEVEL_MODE(cctkGH, rl) {
            BeginTimingLevel(cctkGH);

            do_early_global_mode = not have_done_global_mode;
            do_late_global_mode = reflevel == reflevels - 1;
            do_early_meta_mode =
                do_early_global_mode and mglevel == mglevels - 1;
            do_late_meta_mode = do_late_global_mode and mglevel == 0;
            do_global_mode = do_early_global_mode;
            do_meta_mode = do_early_meta_mode;
            assert(not(have_done_global_mode and do_global_mode));
            have_done_global_mode |= do_global_mode;
            have_done_anything = true;

            if (use_tapered_grids and reflevel > 0) {
              int const parent_do_every =
                  ipow(mgfact, mglevel) *
                  (maxtimereflevelfact / timereffacts.AT(reflevel - 1));
              bool const parent_is_active =
                  (cctkGH->cctk_iteration - 1) % parent_do_every == 0;
              do_taper = not parent_is_active;
            }

            // Advance times
            CycleTimeLevels(cctkGH);
            if (not adaptive_stepsize) {
              cctkGH->cctk_time =
                  (global_time - delta_time / maxtimereflevelfact +
                   delta_time * mglevelfact / timereflevelfact);
            }
            tt->set_time(mglevel, reflevel, timelevel, cctkGH->cctk_time);

            Waypoint("Evolution I at iteration %d time %g%s%s%s",
                     cctkGH->cctk_iteration, (double)cctkGH->cctk_time,
                     (do_global_mode ? " (global)" : ""),
                     (do_meta_mode ? " (meta)" : ""),
                     (do_taper ? " (tapering)" : ""));

            // Checking
            CalculateChecksums(cctkGH, allbutcurrenttime);
            Poison(cctkGH, currenttimebutnotifonly);

            // Evolve
            ScheduleTraverse(where, "CCTK_PRESTEP", cctkGH);
            ScheduleTraverse(where, "CCTK_EVOL", cctkGH);

            // Checking
            PoisonCheck(cctkGH, currenttime);

            // Timing statistics
            StepTimingEvolution(cctkGH);

            do_taper = false;

            EndTimingLevel(cctkGH);
          }
          LEAVE_LEVEL_MODE;
        }
        LEAVE_GLOBAL_MODE;
      } // if do_every
    }   // for rl

    if (have_done_anything)
      assert(have_done_global_mode);

  } // for ml

  timer.stop();
}

void CallRestrict(cGH *const cctkGH) {
  DECLARE_CCTK_PARAMETERS;

  char const *const where = "Evolve::CallRestrict";
  static Timers::Timer timer("CallRestrict");
  timer.start();

  for (int ml = mglevels - 1; ml >= 0; --ml) {

    bool did_restrict = false;

    for (int rl = reflevels - 2; rl >= 0; --rl) {
      int const do_every =
          ipow(mgfact, ml) * (maxtimereflevelfact / timereffacts.AT(rl));
      if (cctkGH->cctk_iteration % do_every == 0) {
        ENTER_GLOBAL_MODE(cctkGH, ml) {
          ENTER_LEVEL_MODE(cctkGH, rl) {
            BeginTimingLevel(cctkGH);

            Waypoint("Evolution/Restrict at iteration %d time %g",
                     cctkGH->cctk_iteration, (double)cctkGH->cctk_time);

            Restrict(cctkGH);
            did_restrict = true;

            if (use_higher_order_restriction) {
              do_early_global_mode = false;
              do_late_global_mode = false;
              do_early_meta_mode = false;
              do_late_meta_mode = false;
              do_global_mode = false;
              do_meta_mode = false;

              if (use_tapered_grids and reflevel > 0) {
                int const parent_do_every =
                    ipow(mgfact, mglevel) *
                    (maxtimereflevelfact / timereffacts.AT(reflevel - 1));
                bool const parent_is_active =
                    (cctkGH->cctk_iteration - 1) % parent_do_every == 0;
                do_taper = not parent_is_active;
              }

              // TODO: disable prolongation (but not
              // synchronization)
              Waypoint("Evolution/PostRestrict (intermediate) at iteration %d "
                       "time %g",
                       cctkGH->cctk_iteration, (double)cctkGH->cctk_time);

              ScheduleTraverse(where, "CCTK_POSTRESTRICT", cctkGH);

              do_taper = false;
            }

            EndTimingLevel(cctkGH);
          }
          LEAVE_LEVEL_MODE;
        }
        LEAVE_GLOBAL_MODE;
      } // if do_every
    }   // for rl

    if (did_restrict) {

      bool have_done_global_mode = false;
      bool have_done_early_global_mode = false;
      bool have_done_late_global_mode = false;
      bool have_done_anything = false;

      for (int rl = 0; rl < reflevels; ++rl) {
        int const do_every =
            ipow(mgfact, ml) * (maxtimereflevelfact / timereffacts.AT(rl));
        if (cctkGH->cctk_iteration % do_every == 0) {
          ENTER_GLOBAL_MODE(cctkGH, ml) {
            ENTER_LEVEL_MODE(cctkGH, rl) {
              BeginTimingLevel(cctkGH);

              do_early_global_mode = not have_done_early_global_mode;
              do_late_global_mode = reflevel == reflevels - 1;
              do_early_meta_mode =
                  do_early_global_mode and mglevel == mglevels - 1;
              do_late_meta_mode = do_late_global_mode and mglevel == 0;
              do_global_mode = do_late_global_mode;
              do_meta_mode = do_global_mode and do_late_meta_mode;
              assert(not(have_done_global_mode and do_global_mode));
              assert(not(have_done_early_global_mode and do_early_global_mode));
              assert(not(have_done_late_global_mode and do_late_global_mode));
              have_done_global_mode |= do_global_mode;
              have_done_early_global_mode |= do_early_global_mode;
              have_done_late_global_mode |= do_late_global_mode;
              have_done_anything = true;

              if (use_tapered_grids and reflevel > 0) {
                int const parent_do_every =
                    ipow(mgfact, mglevel) *
                    (maxtimereflevelfact / timereffacts.AT(reflevel - 1));
                bool const parent_is_active =
                    (cctkGH->cctk_iteration - 1) % parent_do_every == 0;
                do_taper = not parent_is_active;
              }

              Waypoint("Evolution/PostRestrict at iteration %d time %g",
                       cctkGH->cctk_iteration, (double)cctkGH->cctk_time);

              ScheduleTraverse(where, "CCTK_POSTRESTRICT", cctkGH);

              do_taper = false;

              EndTimingLevel(cctkGH);
            }
            LEAVE_LEVEL_MODE;
          }
          LEAVE_GLOBAL_MODE;
        } // if do_every
      }   // for rl

      if (have_done_anything)
        assert(have_done_global_mode);
      if (have_done_anything)
        assert(have_done_early_global_mode);
      if (have_done_anything)
        assert(have_done_late_global_mode);

    } // if did_restrict

  } // for ml

  timer.stop();
}

void CallAnalysis(cGH *const cctkGH) {
  DECLARE_CCTK_PARAMETERS;

  char const *const where = "CallAnalysis";
  static Timers::Timer timer(where);
  timer.start();

  for (int ml = mglevels - 1; ml >= 0; --ml) {

    bool have_done_global_mode = false;
    bool have_done_early_global_mode = false;
    bool have_done_late_global_mode = false;
    bool have_done_anything = false;

    for (int rl = 0; rl < reflevels; ++rl) {
      int const do_every =
          ipow(mgfact, ml) * (maxtimereflevelfact / timereffacts.AT(rl));
      if (cctkGH->cctk_iteration % do_every == 0) {
        ENTER_GLOBAL_MODE(cctkGH, ml) {
          ENTER_LEVEL_MODE(cctkGH, rl) {
            BeginTimingLevel(cctkGH);

            do_early_global_mode = not have_done_early_global_mode;
            do_late_global_mode = reflevel == reflevels - 1;
            do_early_meta_mode =
                do_early_global_mode and mglevel == mglevels - 1;
            do_late_meta_mode = do_late_global_mode and mglevel == 0;
            do_global_mode = do_late_global_mode;
            do_meta_mode = do_global_mode and do_late_meta_mode;
            assert(not(have_done_global_mode and do_global_mode));
            assert(not(have_done_early_global_mode and do_early_global_mode));
            assert(not(have_done_late_global_mode and do_late_global_mode));
            have_done_global_mode |= do_global_mode;
            have_done_early_global_mode |= do_early_global_mode;
            have_done_late_global_mode |= do_late_global_mode;
            have_done_anything = true;

            if (use_tapered_grids and reflevel > 0) {
              int const parent_do_every =
                  ipow(mgfact, mglevel) *
                  (maxtimereflevelfact / timereffacts.AT(reflevel - 1));
              bool const parent_is_active =
                  (cctkGH->cctk_iteration - 1) % parent_do_every == 0;
              do_taper = not parent_is_active;
            }

            Waypoint("Evolution II at iteration %d time %g%s%s%s",
                     cctkGH->cctk_iteration, (double)cctkGH->cctk_time,
                     (do_global_mode ? " (global)" : ""),
                     (do_meta_mode ? " (meta)" : ""),
                     (do_taper ? " (tapering)" : ""));

#if 0
              if (reflevel < reflevels-1) {
                ScheduleTraverse (where, "CCTK_POSTRESTRICT", cctkGH);
              }
#endif

            // Poststep
            ScheduleTraverse(where, "CCTK_POSTSTEP", cctkGH);

            // Checking
            PoisonCheck(cctkGH, currenttime);
            CalculateChecksums(cctkGH, currenttime);

            // Checkpoint
            ScheduleTraverse(where, "CCTK_CHECKPOINT", cctkGH);

            // Analysis
            in_analysis_bin = true;
            ScheduleTraverse(where, "CCTK_ANALYSIS", cctkGH);
            in_analysis_bin = false;

            if (do_late_global_mode) {
              // Timing statistics
              UpdateTimingStats(cctkGH);
            }

            // Output
            OutputGH(where, cctkGH);

            // Checking
            CheckChecksums(cctkGH, alltimes);

            if (do_late_global_mode) {
              // Timing statistics
              PrintTimingStats(cctkGH);
            }

            do_taper = false;

            EndTimingLevel(cctkGH);
          }
          LEAVE_LEVEL_MODE;
        }
        LEAVE_GLOBAL_MODE;
      } // if do_every
    }   // for rl

    if (have_done_anything)
      assert(have_done_global_mode);
    if (have_done_anything)
      assert(have_done_early_global_mode);
    if (have_done_anything)
      assert(have_done_late_global_mode);

  } // for ml

  timer.stop();
}

//////////////////////////////////////////////////////////////////////////////
//////////////////////////////////////////////////////////////////////////////
//////////////////////////////////////////////////////////////////////////////

void print_internal_data() {
  DECLARE_CCTK_PARAMETERS;

  if (output_internal_data) {
    CCTK_INFO("Internal data dump:");
    streamsize const oldprecision = cout.precision();
    cout.precision(17);
    cout << "   global_time: " << global_time << endl
         // << "   leveltimes: " << leveltimes << endl
         << "   delta_time: " << delta_time << endl;
    cout.precision(oldprecision);
  }
}

void ScheduleTraverse(char const *const where, char const *const name,
                      cGH *const cctkGH) {
  Timers::Timer timer(name);
  timer.start();
  ostringstream infobuf;
  infobuf << "Scheduling " << name;
  string const info = infobuf.str();
  Checkpoint(info.c_str());
  CCTK_ScheduleTraverse(name, cctkGH, CallFunction);
  timer.stop();
}

void OutputGH(char const *const where, cGH *const cctkGH) {
  static Timers::Timer timer("OutputGH");
  timer.start();
  CCTK_OutputGH(cctkGH);
  timer.stop();
}

} // namespace Carpet

```

modes.hh:
```
#ifndef MODES_HH
#define MODES_HH

#include <cctk.h>

namespace Carpet {

using namespace std;

//
// These are the modes:
//
// meta mode:
// global mode:    mglevel ("convergence level") is defined
// level mode:     reflevel is defined
// singlemap mode: map ("map index") is defined
// local mode:     component ("patch index")
//
// maybe missing:
// convtest mode:  [rl, map, c]
//

// Mode indicators
bool is_meta_mode() CCTK_ATTRIBUTE_PURE;
bool is_global_mode() CCTK_ATTRIBUTE_PURE;
bool is_level_mode() CCTK_ATTRIBUTE_PURE;
bool is_singlemap_mode() CCTK_ATTRIBUTE_PURE;
bool is_local_mode() CCTK_ATTRIBUTE_PURE;

// Mode setting

void enter_global_mode(cGH *cctkGH, int ml);
void leave_global_mode(cGH *cctkGH);

void enter_level_mode(cGH *cctkGH, int rl);
void leave_level_mode(cGH *cctkGH);

void enter_singlemap_mode(cGH *cctkGH, int m, int grouptype);
void leave_singlemap_mode(cGH *cctkGH);

void enter_local_mode(cGH *cctkGH, int c, int lc, int grouptype);
void leave_local_mode(cGH *cctkGH);

// Mode iterators

class mglevel_iterator {
  cGH *cctkGH;
  int ml;

public:
  mglevel_iterator(cGH const *cctkGH);
  ~mglevel_iterator();
  bool done() const;
  void step();
};

class reverse_mglevel_iterator {
  cGH *cctkGH;
  int ml;

public:
  reverse_mglevel_iterator(cGH const *cctkGH);
  ~reverse_mglevel_iterator();
  bool done() const;
  void step();
};

class reflevel_iterator {
  cGH *cctkGH;
  int rl;

public:
  reflevel_iterator(cGH const *cctkGH);
  ~reflevel_iterator();
  bool done() const;
  void step();
};

class reverse_reflevel_iterator {
  cGH *cctkGH;
  int rl;

public:
  reverse_reflevel_iterator(cGH const *cctkGH);
  ~reverse_reflevel_iterator();
  bool done() const;
  void step();
};

// Loop over all maps.  If grouptype is CCTK_GF, then loop over grid
// function maps.  If grouptype is CCTK_ARRAY (or CCTK_SCALAR), then
// loop over grid array (or grid scalar) maps.  In the latter case,
// map denotes the current grid array map, i.e. it cannot be used to
// access grid functions.
class map_iterator {
  cGH *cctkGH;
  int grouptype;
  int m;

public:
  map_iterator(cGH const *cctkGH, int grouptype);
  ~map_iterator();
  bool done() const;
  void step();
};

class local_map_iterator {
  cGH *cctkGH;
  int grouptype;
  int m;

public:
  local_map_iterator(cGH const *cctkGH, int grouptype);
  ~local_map_iterator();
  bool done() const;
  void step();
};

// Loop over all components.  If grouptype is CCTK_GF, then loop
// over grid function components.  If grouptype is CCTK_ARRAY (or
// CCTK_SCALAR), then loop over grid array (or grid scalar)
// components.  In the latter case, component denotes the current
// grid array component, i.e. it cannot be used to index grid
// functions.
class component_iterator {
  cGH *cctkGH;
  int grouptype;
  int c;

public:
  component_iterator(cGH const *cctkGH, int grouptype);
  ~component_iterator();
  bool done() const;
  void step();
};

class local_component_iterator {
  cGH *cctkGH;
  int grouptype;
  int lc;

public:
  local_component_iterator(cGH const *cctkGH, int grouptype);
  ~local_component_iterator();
  bool done() const;
  void step();
};

// Compatibility defines for the mode iterators

#define BEGIN_MGLEVEL_LOOP(cctkGH)                                             \
  do {                                                                         \
    bool mglevel_loop_ = true;                                                 \
    for (Carpet::mglevel_iterator mg_iter_(cctkGH); not mg_iter_.done();       \
         mg_iter_.step()) {
#define END_MGLEVEL_LOOP                                                       \
  }                                                                            \
  assert(mglevel_loop_);                                                       \
  mglevel_loop_ = false;                                                       \
  }                                                                            \
  while (false)

#define BEGIN_REVERSE_MGLEVEL_LOOP(cctkGH)                                     \
  do {                                                                         \
    bool reverse_mglevel_loop_ = true;                                         \
    for (Carpet::reverse_mglevel_iterator mg_iter_(cctkGH);                    \
         not mg_iter_.done(); mg_iter_.step()) {
#define END_REVERSE_MGLEVEL_LOOP                                               \
  }                                                                            \
  assert(reverse_mglevel_loop_);                                               \
  reverse_mglevel_loop_ = false;                                               \
  }                                                                            \
  while (false)

#define BEGIN_REFLEVEL_LOOP(cctkGH)                                            \
  do {                                                                         \
    bool reflevel_loop_ = true;                                                \
    for (Carpet::reflevel_iterator ref_iter_(cctkGH); not ref_iter_.done();    \
         ref_iter_.step()) {
#define END_REFLEVEL_LOOP                                                      \
  }                                                                            \
  assert(reflevel_loop_);                                                      \
  reflevel_loop_ = false;                                                      \
  }                                                                            \
  while (false)

#define BEGIN_REVERSE_REFLEVEL_LOOP(cctkGH)                                    \
  do {                                                                         \
    bool reverse_reflevel_loop_ = true;                                        \
    for (Carpet::reverse_reflevel_iterator ref_iter_(cctkGH);                  \
         not ref_iter_.done(); ref_iter_.step()) {
#define END_REVERSE_REFLEVEL_LOOP                                              \
  }                                                                            \
  assert(reverse_reflevel_loop_);                                              \
  reverse_reflevel_loop_ = false;                                              \
  }                                                                            \
  while (false)

#define BEGIN_MAP_LOOP(cctkGH, grouptype)                                      \
  do {                                                                         \
    bool map_loop_ = true;                                                     \
    for (Carpet::map_iterator map_iter_(cctkGH, grouptype);                    \
         not map_iter_.done(); map_iter_.step()) {
#define END_MAP_LOOP                                                           \
  }                                                                            \
  assert(map_loop_);                                                           \
  map_loop_ = false;                                                           \
  }                                                                            \
  while (false)

#define BEGIN_LOCAL_MAP_LOOP(cctkGH, grouptype)                                \
  do {                                                                         \
    bool local_map_loop_ = true;                                               \
    for (Carpet::local_map_iterator local_map_iter_(cctkGH, grouptype);        \
         not local_map_iter_.done(); local_map_iter_.step()) {
#define END_LOCAL_MAP_LOOP                                                     \
  }                                                                            \
  assert(local_map_loop_);                                                     \
  local_map_loop_ = false;                                                     \
  }                                                                            \
  while (false)

#define BEGIN_COMPONENT_LOOP(cctkGH, grouptype)                                \
  do {                                                                         \
    bool component_loop_ = true;                                               \
    for (Carpet::component_iterator comp_iter_(cctkGH, grouptype);             \
         not comp_iter_.done(); comp_iter_.step()) {
#define END_COMPONENT_LOOP                                                     \
  }                                                                            \
  assert(component_loop_);                                                     \
  component_loop_ = false;                                                     \
  }                                                                            \
  while (false)

#define BEGIN_LOCAL_COMPONENT_LOOP(cctkGH, grouptype)                          \
  do {                                                                         \
    bool local_component_loop_ = true;                                         \
    for (Carpet::local_component_iterator comp_iter_(cctkGH, grouptype);       \
         not comp_iter_.done(); comp_iter_.step()) {
#define END_LOCAL_COMPONENT_LOOP                                               \
  }                                                                            \
  assert(local_component_loop_);                                               \
  local_component_loop_ = false;                                               \
  }                                                                            \
  while (false)

#define BEGIN_TIMELEVEL_LOOP(cctkGH)                                           \
  do {                                                                         \
    bool timelevel_loop_ = true;                                               \
    assert(do_allow_past_timelevels);                                          \
    do_allow_past_timelevels = false;                                          \
    assert(timelevel == 0);                                                    \
    assert(timelevel_offset == 0);                                             \
    assert(not adaptive_stepsize);                                             \
    for (timelevel = timelevels - 1; timelevel >= 0; --timelevel) {            \
      timelevel_offset = timelevel;                                            \
      cctkGH->cctk_time = tt->get_time(mglevel, reflevel, timelevel);          \
      {
#define END_TIMELEVEL_LOOP                                                     \
  }                                                                            \
  }                                                                            \
  assert(timelevel_loop_);                                                     \
  timelevel_loop_ = false;                                                     \
  timelevel = 0;                                                               \
  timelevel_offset = 0;                                                        \
  /* cctkGH->cctk_time = tt->get_time (mglevel, reflevel, timelevel); */       \
  do_allow_past_timelevels = true;                                             \
  }                                                                            \
  while (false)

#if 0
// TODO: Introduce such maybe-loops for the other loops as well

#define BEGIN_MAYBE_TIMELEVEL_LOOP(cctkGH)                                     \
  do {                                                                         \
    bool timelevel_maybe_loop_ = true;                                         \
    assert(do_allow_past_timelevels);                                          \
    int const min_tl = timelevel == -1 ? 0 : timelevel;                        \
    int const max_tl = timelevel == -1 ? timelevels : timelevel + 1;           \
    int const old_timelevel = timelevel;                                       \
    bool const old_do_allow_past_timelevels = do_allow_past_timelevels;        \
    do_allow_past_timelevels = timelevel != -1;                                \
    for (timelevel = min_tl; timelevel < max_tl; ++timelevel) {                \
      if (mglevel != -1 and reflevel != -1) {                                  \
        cctkGH->cctk_time = tt->get_time(mglevel, reflevel, timelevel);        \
      }                                                                        \
      {
#define END_MAYBE_TIMELEVEL_LOOP                                               \
  }                                                                            \
  }                                                                            \
  assert(timelevel_maybe_loop_);                                               \
  timelevel_maybe_loop_ = false;                                               \
  timelevel = old_timelevel;                                                   \
  if (mglevel != -1 and reflevel != -1) {                                      \
    int const tl = timelevel == -1 ? 0 : timelevel;                            \
    cctkGH->cctk_time = tt->get_time(mglevel, reflevel, tl);                   \
  }                                                                            \
  do_allow_past_timelevels = old_do_allow_past_timelevels;                     \
  }                                                                            \
  while (false)
#endif

// Mode escapes

class singlemap_escape {
  cGH *cctkGH;
  int c;
  int lc;

public:
  singlemap_escape(cGH const *cctkGH);
  ~singlemap_escape();
};

class level_escape {
  cGH *cctkGH;
  int grouptype;
  int m;
  int c;
  int lc;

public:
  level_escape(cGH const *cctkGH);
  ~level_escape();
};

class global_escape {
  cGH *cctkGH;
  int rl;
  int grouptype;
  int m;
  int c;
  int lc;

public:
  global_escape(cGH const *cctkGH);
  ~global_escape();
};

class meta_escape {
  cGH *cctkGH;
  int ml;
  int rl;
  int grouptype;
  int m;
  int c;
  int lc;

public:
  meta_escape(cGH const *cctkGH);
  ~meta_escape();
};

// Compatibility defines for the mode escapes

#define BEGIN_SINGLEMAP_MODE(cctkGH)                                           \
  do {                                                                         \
    bool singlemap_mode_ = true;                                               \
    Carpet::singlemap_escape esc_(cctkGH);                                     \
    {
#define END_SINGLEMAP_MODE                                                     \
  }                                                                            \
  assert(singlemap_mode_);                                                     \
  singlemap_mode_ = false;                                                     \
  }                                                                            \
  while (false)

#define BEGIN_LEVEL_MODE(cctkGH)                                               \
  do {                                                                         \
    bool level_mode_ = true;                                                   \
    Carpet::level_escape esc_(cctkGH);                                         \
    {
#define END_LEVEL_MODE                                                         \
  }                                                                            \
  assert(level_mode_);                                                         \
  level_mode_ = false;                                                         \
  }                                                                            \
  while (false)

#define BEGIN_GLOBAL_MODE(cctkGH)                                              \
  do {                                                                         \
    bool global_mode_ = true;                                                  \
    Carpet::global_escape esc_(cctkGH);                                        \
    {
#define END_GLOBAL_MODE                                                        \
  }                                                                            \
  assert(global_mode_);                                                        \
  global_mode_ = false;                                                        \
  }                                                                            \
  while (false)

#define BEGIN_META_MODE(cctkGH)                                                \
  do {                                                                         \
    bool meta_mode_ = true;                                                    \
    Carpet::meta_escape esc_(cctkGH);                                          \
    {
#define END_META_MODE                                                          \
  }                                                                            \
  assert(meta_mode_);                                                          \
  meta_mode_ = false;                                                          \
  }                                                                            \
  while (false)

// Mode setters

class mglevel_setter {
  cGH *cctkGH;

public:
  mglevel_setter(cGH const *cctkGH, int ml);
  ~mglevel_setter();
};

class reflevel_setter {
  cGH *cctkGH;

public:
  reflevel_setter(cGH const *cctkGH, int rl);
  ~reflevel_setter();
};

class map_setter {
  cGH *cctkGH;

public:
  map_setter(cGH const *cctkGH, int m, int grouptype);
  ~map_setter();
};

class component_setter {
  cGH *cctkGH;

public:
  component_setter(cGH const *cctkGH, int c, int grouptype);
  ~component_setter();
};

// Compatibility defines for the mode setters

#define ENTER_GLOBAL_MODE(cctkGH, ml)                                          \
  do {                                                                         \
    Carpet::mglevel_setter mg_setter_(cctkGH, ml);                             \
    {
#define LEAVE_GLOBAL_MODE                                                      \
  }                                                                            \
  }                                                                            \
  while (false)

#define ENTER_LEVEL_MODE(cctkGH, rl)                                           \
  do {                                                                         \
    Carpet::reflevel_setter ref_setter_(cctkGH, rl);                           \
    {
#define LEAVE_LEVEL_MODE                                                       \
  }                                                                            \
  }                                                                            \
  while (false)

#define ENTER_SINGLEMAP_MODE(cctkGH, m, grouptype)                             \
  do {                                                                         \
    Carpet::map_setter m_setter_(cctkGH, m, grouptype);                        \
    {
#define LEAVE_SINGLEMAP_MODE                                                   \
  }                                                                            \
  }                                                                            \
  while (false)

#define ENTER_LOCAL_MODE(cctkGH, c, grouptype)                                 \
  do {                                                                         \
    Carpet::component_setter c_setter_(cctkGH, c, grouptype);                  \
    {
#define LEAVE_LOCAL_MODE                                                       \
  }                                                                            \
  }                                                                            \
  while (false)

// Loop over a bounding box

void ibbox2iminimax(ibbox const &ext, // component extent
                    ibbox const &box, // this bbox
                    ivect &imin, ivect &imax);

#define LOOP_OVER_BBOX(cctkGH, box_, box, imin, imax)                          \
  do {                                                                         \
    bool bbox_loop_ = true;                                                    \
    assert(Carpet::is_local_mode());                                           \
    dh::light_mboxes const &light_boxes_ =                                     \
        Carpet::vdd.AT(Carpet::map)->light_boxes;                              \
    ibbox const &ext_ =                                                        \
        light_boxes_.AT(mglevel).AT(reflevel).AT(component).exterior;          \
    ibbox const box = (box_)&ext_;                                             \
    if (not box.empty()) {                                                     \
      ivect imin;                                                              \
      ivect imax;                                                              \
      Carpet::ibbox2iminimax(ext_, box, imin, imax);                           \
      {
#define END_LOOP_OVER_BBOX                                                     \
  }                                                                            \
  } /* if not empty */                                                         \
  assert(bbox_loop_);                                                          \
  bbox_loop_ = false;                                                          \
  }                                                                            \
  while (false)

#define LOOP_OVER_BSET(cctkGH, set_, box, imin, imax)                          \
  do {                                                                         \
    bool bset_loop_ = true;                                                    \
    ibset const &set1_(set_);                                                  \
    for (ibset::const_iterator bi = set1_.begin(); bi != set1_.end(); ++bi) {  \
      LOOP_OVER_BBOX(cctkGH, *bi, box, imin, imax) {
#define END_LOOP_OVER_BSET                                                     \
  }                                                                            \
  END_LOOP_OVER_BBOX;                                                          \
  } /* for */                                                                  \
  assert(bset_loop_);                                                          \
  bset_loop_ = false;                                                          \
  }                                                                            \
  while (false)

} // namespace Carpet

#endif // #ifndef MODES_HH

```

adler32.cc:
```
// ADLER-32 checksum algorithm, taken from
// <http://en.wikipedia.org/wiki/Adler-32>

#include "adler32.hh"

namespace Carpet {

unsigned long const mod_adler = 65521;

unsigned long adler32(char const *const data, // location of the data
                      std::size_t const len)  // length of the data in bytes
{
  unsigned long a = 1, b = 0;

  // Process each byte of the data in order
  for (std::size_t i = 0; i < len; ++i) {
    a = (a + (unsigned char)data[i]) % mod_adler;
    b = (b + a) % mod_adler;
  }

  return (b << 16) | a;
}
}

```

Comm.cc:
```
#include <algorithm>
#include <cassert>
#include <cmath>
#include <cstdlib>
#include <iostream>
#include <vector>

#include <cctk.h>
#include <cctk_Parameters.h>

#include <Requirements.hh>

#include <Timer.hh>

#include <ggf.hh>
#include <gh.hh>

#include <carpet.hh>

namespace Carpet {

using namespace std;

static void ProlongateGroupBoundaries(const cGH *cctkGH,
                                      const vector<int> &groups);

// Carpet's overload function for CCTK_SyncGroupsByDirI()
// which synchronises a set of groups given by their indices.
// Synchronisation of individual directions is not (yet?) implemented.
//
// returns the number of groups successfully synchronised
//        -1 if a group in the set doesn't have storage assigned
int SyncGroupsByDirI(const cGH *cctkGH, int num_groups, const int *groups,
                     const int *directions) {
  int group, retval = 0;
  vector<int> groups_set;
  groups_set.reserve(num_groups);

  // individual directions aren't supported (yet?)
  if (directions != NULL) {
    CCTK_WARN(0, "Carpet doesn't support synchronisation of individual "
                 "directions");
  }

  for (group = 0; group < num_groups; group++) {
    if (CCTK_NumVarsInGroupI(groups[group]) > 0) {
      groups_set.push_back(groups[group]);
    }
  }

  if (groups_set.size() > 0) {
    retval = SyncProlongateGroups(cctkGH, groups_set);
    if (retval == 0) {
      retval = groups_set.size();
    }
  }

  return retval;
}

// synchronises ghostzones and prolongates boundaries of a set of groups
//
// returns 0 for success and -1 if the set contains a group with no storage
int SyncProlongateGroups(const cGH *cctkGH, const vector<int> &groups,
                         cFunctionData const *function_data) {
  int retval = 0;
  DECLARE_CCTK_PARAMETERS;

  assert(groups.size() > 0);

  // check consistency of all groups:
  // create a new set with empty and no-storage groups removed
  vector<int> goodgroups;
  goodgroups.reserve(groups.size());
  for (size_t group = 0; group < groups.size(); group++) {
    const int g = groups.AT(group);
    const int grouptype = CCTK_GroupTypeI(g);
    char *const groupname = CCTK_GroupName(g);
    Checkpoint("SyncGroup \"%s\" iteration=%d time=%g", groupname,
               cctkGH->cctk_iteration, (double)cctkGH->cctk_time);

    if (grouptype == CCTK_GF) {
      if (reflevel == -1) {
        CCTK_VWarn(0, __LINE__, __FILE__, CCTK_THORNSTRING,
                   "Cannot synchronise in global mode "
                   "(Tried to synchronise group \"%s\")",
                   groupname);
      }
      if (map != -1 and component == -1) {
        if (maps == 1) {
          CCTK_VWarn(2, __LINE__, __FILE__, CCTK_THORNSTRING,
                     "Synchronising group \"%s\" in singlemap mode", groupname);
        } else {
          CCTK_VWarn(0, __LINE__, __FILE__, CCTK_THORNSTRING,
                     "Cannot synchronise in singlemap mode "
                     "(Tried to synchronise group \"%s\")",
                     groupname);
        }
      }
      if (component != -1) {
        if (maps == 1 and vhh.AT(map)->local_components(reflevel) == 1) {
          CCTK_VWarn(2, __LINE__, __FILE__, CCTK_THORNSTRING,
                     "Synchronising group \"%s\" in local mode", groupname);
        } else {
          CCTK_VWarn(0, __LINE__, __FILE__, CCTK_THORNSTRING,
                     "Cannot synchronise in local mode "
                     "(Tried to synchronise group \"%s\")",
                     groupname);
        }
      }
    }

    if (not CCTK_QueryGroupStorageI(cctkGH, g)) {
      CCTK_VWarn(4, __LINE__, __FILE__, CCTK_THORNSTRING,
                 "Cannot synchronise group \"%s\" because it has no storage",
                 groupname);
      retval = -1;
    } else if (CCTK_NumVarsInGroupI(g) > 0) {
      goodgroups.push_back(g);
    }

    free(groupname);
  } // for g

  if (goodgroups.size() > 0) {

#ifdef REQUIREMENTS_HH
    Requirements::Sync(function_data, goodgroups, cctkGH->cctk_iteration,
                       reflevel, timelevel);
#endif

    // prolongate boundaries
    bool const local_do_prolongate = do_prolongate and not do_taper;
    if (local_do_prolongate) {
      if (sync_barriers) {
        static Timers::Timer barrier_timer("PreProlongateBarrier");
        barrier_timer.start();
        CCTK_Barrier(cctkGH);
        barrier_timer.stop();
      }

      static Timers::Timer timer("Prolongate");
      timer.start();
      ProlongateGroupBoundaries(cctkGH, goodgroups);
      timer.stop();

      if (sync_barriers) {
        static Timers::Timer barrier_timer("PostProlongateBarrier");
        barrier_timer.start();
        CCTK_Barrier(cctkGH);
        barrier_timer.stop();
      }
    }

// This was found to be necessary on Hopper, otherwise memory
// seems to be overwritten while prolongating/syncronizing. It
// looks liks this might be an MPI implementation issue, but
// this is not clear. A barrier at this point seems to be a
// sufficient workaround, and is now used on Hopper. For more
// information about this ask Frank Loeffler
// <knarf@cct.lsu.edu>.
#ifdef CARPET_MPI_BARRIER_PROLONGATE_SYNC
    Carpet::NamedBarrier(cctkGH, 8472211063,
                         "CARPET_MPI_BARRIER_PROLONGATE_SYNC");
#endif

    // synchronise ghostzones
    if (sync_during_time_integration or local_do_prolongate) {
      if (sync_barriers) {
        static Timers::Timer barrier_timer("PreSyncBarrier");
        barrier_timer.start();
        CCTK_Barrier(cctkGH);
        barrier_timer.stop();
      }

      static Timers::Timer timer("Sync");
      timer.start();
      SyncGroups(cctkGH, goodgroups);
      timer.stop();

      if (sync_barriers) {
        static Timers::Timer barrier_timer("PostSyncBarrier");
        barrier_timer.start();
        CCTK_Barrier(cctkGH);
        barrier_timer.stop();
      }
    }
  }

  return retval;
}

// Prolongate the boundaries of all CCTK_GF groups in the given set
static void ProlongateGroupBoundaries(const cGH *cctkGH,
                                      const vector<int> &groups) {
  DECLARE_CCTK_PARAMETERS;

  if (reflevel == 0)
    return;

  Checkpoint("ProlongateGroups");

  assert(groups.size() > 0);

  // TODO: Transmit only the regions that are actually needed
  if (CCTK_IsFunctionAliased("Accelerator_RequireValidData")) {
    // TODO: Require only as many time levels as needed
    // TODO: Handle time levels correctly
    vector<CCTK_INT> vis, rls, tls;
    for (int group = 0; group < (int)groups.size(); ++group) {
      const int g = groups.AT(group);
      const int grouptype = CCTK_GroupTypeI(g);
      if (grouptype != CCTK_GF) {
        continue;
      }

      const int num_tl = prolongation_order_time + 1;
      const int var0 = CCTK_FirstVarIndexI(g);
      const int varn = CCTK_NumVarsInGroupI(g);
      for (int vi = var0; vi < var0 + varn; ++vi) {
        vis.push_back(vi);
        rls.push_back(reflevel);
        tls.push_back(timelevel);
        for (int tl = 0; tl < num_tl; ++tl) {
          vis.push_back(vi);
          rls.push_back(reflevel - 1);
          tls.push_back(tl);
        }
      }
    }
    const CCTK_INT on_device = 0;
    Accelerator_RequireValidData(cctkGH, &vis.front(), &rls.front(),
                                 &tls.front(), vis.size(), on_device);
  }

  // use the current time here (which may be modified by the user)
  const CCTK_REAL time = cctkGH->cctk_time;

  static vector<Timers::Timer *> timers;
  if (timers.empty()) {
    timers.push_back(new Timers::Timer("comm_state[0].create"));
    for (astate state = static_cast<astate>(0); state != state_done;
         state = static_cast<astate>(static_cast<int>(state) + 1)) {
      ostringstream name1;
      name1 << "comm_state[" << timers.size() << "]"
            << "." << tostring(state) << ".user";
      timers.push_back(new Timers::Timer(name1.str()));
      ostringstream name2;
      name2 << "comm_state[" << timers.size() << "]"
            << "." << tostring(state) << ".step";
      timers.push_back(new Timers::Timer(name2.str()));
    }
  }

  vector<Timers::Timer *>::iterator ti = timers.begin();
  (*ti)->start();
  for (comm_state state; not state.done(); state.step()) {
    (*ti)->stop();
    ++ti;
    (*ti)->start();
    for (int group = 0; group < (int)groups.size(); ++group) {
      const int g = groups.AT(group);
      const int grouptype = CCTK_GroupTypeI(g);
      if (grouptype != CCTK_GF) {
        continue;
      }
      assert(reflevel >= 0 and reflevel < reflevels);
      const int active_tl =
          groupdata.AT(g).activetimelevels.AT(mglevel).AT(reflevel);
      assert(active_tl >= 0);
      const int tl = active_tl > 1 ? timelevel : 0;

      for (int m = 0; m < (int)arrdata.AT(g).size(); ++m) {
        for (int v = 0; v < (int)arrdata.AT(g).AT(m).data.size(); ++v) {
          ggf *const gv = arrdata.AT(g).AT(m).data.AT(v);
          gv->ref_bnd_prolongate_all(state, tl, reflevel, mglevel, time);
        }
      }
    }
    (*ti)->stop();
    ++ti;
    (*ti)->start();
  }
  (*ti)->stop();
  ++ti;
  assert(ti == timers.end());

  // TODO: Transmit only the regions that were actually written
  // (i.e. the buffer zones)
  if (CCTK_IsFunctionAliased("Accelerator_NotifyDataModified")) {
    // TODO: Handle time levels correctly
    vector<CCTK_INT> vis, rls, tls;
    for (int group = 0; group < (int)groups.size(); ++group) {
      const int g = groups.AT(group);
      const int grouptype = CCTK_GroupTypeI(g);
      if (grouptype != CCTK_GF) {
        continue;
      }
      const int var0 = CCTK_FirstVarIndexI(g);
      const int varn = CCTK_NumVarsInGroupI(g);
      for (int vi = var0; vi < var0 + varn; ++vi) {
        vis.push_back(vi);
        rls.push_back(reflevel);
        tls.push_back(timelevel);
      }
    }
    const CCTK_INT on_device = 0;
    Accelerator_NotifyDataModified(cctkGH, &vis.front(), &rls.front(),
                                   &tls.front(), vis.size(), on_device);
  }
}

// synchronises a set of groups
void SyncGroups(const cGH *cctkGH, const vector<int> &groups) {
  DECLARE_CCTK_PARAMETERS;

  Checkpoint("SyncGroups");

  assert(groups.size() > 0);

  if (CCTK_IsFunctionAliased("Accelerator_PreSync")) {
    vector<CCTK_INT> groups_(groups.size());
    for (size_t i = 0; i < groups.size(); ++i)
      groups_[i] = groups[i];
    Accelerator_PreSync(cctkGH, &groups_.front(), groups_.size());
  }

  static vector<Timers::Timer *> timers;
  if (timers.empty()) {
    timers.push_back(new Timers::Timer("comm_state[0].create"));
    for (astate state = static_cast<astate>(0); state != state_done;
         state = static_cast<astate>(static_cast<int>(state) + 1)) {
      ostringstream name1;
      name1 << "comm_state[" << timers.size() << "]"
            << "." << tostring(state) << ".user";
      timers.push_back(new Timers::Timer(name1.str()));
      ostringstream name2;
      name2 << "comm_state[" << timers.size() << "]"
            << "." << tostring(state) << ".step";
      timers.push_back(new Timers::Timer(name2.str()));
    }
  }

  vector<Timers::Timer *>::iterator ti = timers.begin();
  (*ti)->start();
  for (comm_state state; not state.done(); state.step()) {
    (*ti)->stop();
    ++ti;
    (*ti)->start();
    for (int group = 0; group < (int)groups.size(); ++group) {
      const int g = groups.AT(group);
      const int grouptype = CCTK_GroupTypeI(g);
      const int ml = grouptype == CCTK_GF ? mglevel : 0;
      const int rl = grouptype == CCTK_GF ? reflevel : 0;
      const int active_tl = groupdata.AT(g).activetimelevels.AT(ml).AT(rl);
      assert(active_tl >= 0);
      const int tl = active_tl > 1 ? timelevel : 0;
      for (int m = 0; m < (int)arrdata.AT(g).size(); ++m) {
        for (int v = 0; v < (int)arrdata.AT(g).AT(m).data.size(); ++v) {
          arrdesc &array = arrdata.AT(g).AT(m);
          array.data.AT(v)->sync_all(state, tl, rl, ml);
        }
      }
    }
    (*ti)->stop();
    ++ti;
    (*ti)->start();
  }
  (*ti)->stop();
  ++ti;
  assert(ti == timers.end());

  if (CCTK_IsFunctionAliased("Accelerator_PostSync")) {
    vector<CCTK_INT> groups_(groups.size());
    for (size_t i = 0; i < groups.size(); ++i)
      groups_[i] = groups[i];
    Accelerator_PostSync(cctkGH, &groups_.front(), groups_.size());
  }
}

int EnableGroupComm(const cGH *cctkGH, const char *groupname) {
  // Communication is always enabled
  return 0;
}

int DisableGroupComm(const cGH *cctkGH, const char *groupname) {
  // Communication is always enabled
  return -1;
}

} // namespace Carpet

```

Checksum.cc:
```
#include <cassert>
#include <cstdlib>
#include <vector>

#include <cctk.h>
#include <cctk_Parameters.h>
#include <util_ErrorCodes.h>
#include <util_Table.h>

#include <Timer.hh>

#include <gh.hh>

#include <carpet.hh>

namespace Carpet {

using namespace std;

// Checksum information
struct ckdesc {
  bool valid;
  unsigned long sum;
};

// Helper class
struct ckdesc4 {
  vector<vector<vector<vector<ckdesc> > > > a; // [m][c][var][tl]
};

// Checksum information
vector<vector<vector<ckdesc4> > > checksums; // [rl][ml][group]

// Calculate the internet checksum (see RFC 1071)
static unsigned short internet_checksum(void const *restrict const addr,
                                        size_t const len) {
  unsigned long chk = 0;
#pragma omp parallel for reduction(+ : chk)
  for (ptrdiff_t i = 0; i < (ptrdiff_t)len; i += 2) {
    unsigned long const lb = ((unsigned char const *)addr)[i];
    unsigned long const ub = ((unsigned char const *)addr)[i + 1];
    chk += lb + (ub << 8);
  }
  if (len % 1) {
    unsigned long const lb = ((unsigned char const *)addr)[len - 1];
    chk += lb;
  }
  while (chk >> 16) {
    chk = (chk & 0xffffUL) + (chk >> 16);
  }
  return ~chk;
}

// The parameter where specifies which time levels should be
// poisoned.  what specifies what kind of grid variables should be
// poisoned.
void CalculateChecksums(const cGH *cgh, const checktimes where) {
  DECLARE_CCTK_PARAMETERS;

  if (!checksum_timelevels)
    return;

  Timers::Timer timer("CalculateChecksums");
  timer.start();

  Checkpoint("CalculateChecksums");

  checksums.resize(maxreflevels);
  checksums.at(reflevel).resize(mglevels);
  checksums.at(reflevel).at(mglevel).resize(CCTK_NumGroups());
  for (int group = 0; group < CCTK_NumGroups(); ++group) {
    if (CCTK_QueryGroupStorageI(cgh, group)) {
      const int grouptype = CCTK_GroupTypeI(group);
      if (reflevel == 0 or grouptype == CCTK_GF) {
        checksums.at(reflevel).at(mglevel).at(group).a.resize(
            arrdata.at(group).size());
        BEGIN_LOCAL_MAP_LOOP(cgh, grouptype) {
          checksums.at(reflevel).at(mglevel).at(group).a.at(map).resize(
              arrdata.at(group).at(map).hh->local_components(reflevel));
          BEGIN_LOCAL_COMPONENT_LOOP(cgh, grouptype) {
            const int nvars = CCTK_NumVarsInGroupI(group);
            checksums.at(reflevel)
                .at(mglevel)
                .at(group)
                .a.at(map)
                .at(local_component)
                .resize(nvars);
            if (nvars > 0) {

              const int n0 = CCTK_FirstVarIndexI(group);
              const int sz = CCTK_VarTypeSize(CCTK_VarTypeI(n0));
              assert(sz > 0);

              ivect size(1);
              const int gpdim = groupdata.at(group).info.dim;
              for (int d = 0; d < gpdim; ++d) {
                size[d] = groupdata.at(group).info.lsh[d];
              }
              const int np = prod(size);

              int const table = CCTK_GroupTagsTableI(group);
              assert(table >= 0);
              bool persistent;
              char buf[100];
              int const ilen =
                  Util_TableGetString(table, sizeof buf, buf, "Persistent");
              if (ilen > 0) {
                if (CCTK_EQUALS(buf, "yes")) {
                  persistent = true;
                } else if (CCTK_EQUALS(buf, "no")) {
                  persistent = false;
                } else {
                  assert(0);
                }
              } else if (ilen == UTIL_ERROR_TABLE_NO_SUCH_KEY) {
                // default
                persistent = true;
              } else {
                assert(0);
              }

              const int num_tl = CCTK_MaxActiveTimeLevelsVI(cgh, n0);
              assert(num_tl > 0);
              const int min_tl = min_timelevel(where, num_tl, persistent);
              const int max_tl = max_timelevel(where, num_tl, persistent);

              for (int var = 0; var < nvars; ++var) {
                checksums.at(reflevel)
                    .at(mglevel)
                    .at(group)
                    .a.at(map)
                    .at(local_component)
                    .at(var)
                    .resize(num_tl);
                for (int tl = min_tl; tl <= max_tl; ++tl) {

                  const int n = n0 + var;
                  const void *data = cgh->data[n][tl];
                  unsigned long const chk = internet_checksum(data, np * sz);

                  checksums.at(reflevel)
                      .at(mglevel)
                      .at(group)
                      .a.at(map)
                      .at(local_component)
                      .at(var)
                      .at(tl)
                      .sum = chk;
                  checksums.at(reflevel)
                      .at(mglevel)
                      .at(group)
                      .a.at(map)
                      .at(local_component)
                      .at(var)
                      .at(tl)
                      .valid = true;

                } // for tl
              }   // for var
            }     // if group has vars
          }
          END_LOCAL_COMPONENT_LOOP;
        }
        END_LOCAL_MAP_LOOP;
      } // if grouptype fits
    }   // if storage
  }     // for group

  timer.stop();
}

void CheckChecksums(const cGH *cgh, const checktimes where) {
  DECLARE_CCTK_PARAMETERS;

  if (!checksum_timelevels)
    return;

  Checkpoint("CheckChecksums");

  assert((int)checksums.size() == maxreflevels);
  assert((int)checksums.at(reflevel).size() == mglevels);
  assert((int)checksums.at(reflevel).at(mglevel).size() == CCTK_NumGroups());
  for (int group = 0; group < CCTK_NumGroups(); ++group) {
    if (CCTK_QueryGroupStorageI(cgh, group)) {
      const int grouptype = CCTK_GroupTypeI(group);
      if (reflevel == 0 or grouptype == CCTK_GF) {
        assert(checksums.at(reflevel).at(mglevel).at(group).a.size() ==
               arrdata.at(group).size());
        BEGIN_LOCAL_MAP_LOOP(cgh, grouptype) {
          assert((int)checksums.at(reflevel)
                     .at(mglevel)
                     .at(group)
                     .a.at(map)
                     .size() ==
                 arrdata.at(group).at(map).hh->local_components(reflevel));
          BEGIN_LOCAL_COMPONENT_LOOP(cgh, grouptype) {
            const int nvars = CCTK_NumVarsInGroupI(group);
            assert((int)checksums.at(reflevel)
                       .at(mglevel)
                       .at(group)
                       .a.at(map)
                       .at(local_component)
                       .size() == nvars);
            if (nvars > 0) {

              const int n0 = CCTK_FirstVarIndexI(group);
              const int sz = CCTK_VarTypeSize(CCTK_VarTypeI(n0));
              assert(sz > 0);

              ivect size(1);
              const int gpdim = groupdata.at(group).info.dim;
              for (int d = 0; d < gpdim; ++d) {
                size[d] = groupdata.at(group).info.lsh[d];
              }
              const int np = prod(size);

              int const table = CCTK_GroupTagsTableI(group);
              assert(table >= 0);
              bool persistent;
              char buf[100];
              int const ilen =
                  Util_TableGetString(table, sizeof buf, buf, "Persistent");
              if (ilen > 0) {
                if (CCTK_EQUALS(buf, "yes")) {
                  persistent = true;
                } else if (CCTK_EQUALS(buf, "no")) {
                  persistent = false;
                } else {
                  assert(0);
                }
              } else if (ilen == UTIL_ERROR_TABLE_NO_SUCH_KEY) {
                // default
                persistent = true;
              } else {
                assert(0);
              }

              const int num_tl = CCTK_MaxActiveTimeLevelsVI(cgh, n0);
              assert(num_tl > 0);
              const int min_tl = min_timelevel(where, num_tl, persistent);
              const int max_tl = max_timelevel(where, num_tl, persistent);

              for (int var = 0; var < nvars; ++var) {
                assert((int)checksums.at(reflevel)
                           .at(mglevel)
                           .at(group)
                           .a.at(map)
                           .at(local_component)
                           .at(var)
                           .size() == num_tl);
                for (int tl = min_tl; tl <= max_tl; ++tl) {
                  if (checksums.at(reflevel)
                          .at(mglevel)
                          .at(group)
                          .a.at(map)
                          .at(local_component)
                          .at(var)
                          .at(tl)
                          .valid) {

                    const int n = n0 + var;
                    const void *data = cgh->data[n][tl];
                    unsigned long const chk = internet_checksum(data, np * sz);
                    const bool unexpected_change = chk !=
                                                   checksums.at(reflevel)
                                                       .at(mglevel)
                                                       .at(group)
                                                       .a.at(map)
                                                       .at(local_component)
                                                       .at(var)
                                                       .at(tl)
                                                       .sum;

                    if (unexpected_change) {
                      char *fullname = CCTK_FullName(n);
                      CCTK_VWarn(1, __LINE__, __FILE__, CCTK_THORNSTRING,
                                 "Timelevel %d, component %d, refinement level "
                                 "%d of the variable \"%s\" has changed "
                                 "unexpectedly",
                                 tl, component, reflevel, fullname);
                      free(fullname);
                    }

                  } // if valid
                }   // for tl
              }     // for var
            }       // if group has vars
          }
          END_LOCAL_COMPONENT_LOOP;
        }
        END_LOCAL_MAP_LOOP;
      } // if grouptype fits
    }   // if storage
  }     // for group
}

} // namespace Carpet

```

variables.hh:
```
// It is assumed that each group has at least one map.  All arrays
// have exactly one map.  All maps have the same number of refinement
// levels.

// It is assumed that each group has at least one component.

// It is assumed that the number of components of all arrays is equal
// to the number of components of the grid functions, and that their
// distribution onto the processors is the same, and that all
// processors own the same number of components.

#ifndef VARIABLES_HH
#define VARIABLES_HH

#include <cctk.h>

#include <vector>

#ifdef CCTK_MPI
#include <mpi.h>
#else
#include "nompi.h"
#endif

#include <bbox.hh>
#include <data.hh>
#include <dh.hh>
#include <ggf.hh>
#include <gh.hh>
#include <operators.hh>
#include <th.hh>
#include <vect.hh>

#include "carpet_public.h"

namespace Carpet {

using namespace std;

// Handle from CCTK_RegisterGHExtension
extern int GHExtension;

// Maximum number of refinement levels
extern int maxreflevels;

// Maximum number of time levels
extern int maxtimelevels;

// Timelevels
extern int timelevels;

// Refinement levels
extern int reflevels;

#define CARPET_NEW_REFFACT
// Temporal refinement factors over the coarsest grid
extern vector<int> timereffacts;

// Spatial refinement factors over the coarsest grid
extern vector<vect<int, dim> > spacereffacts;

// Maximum refinement factors on finest possible grid
extern int maxtimereflevelfact;
extern vect<int, dim> maxspacereflevelfact;

// Base multigrid level
extern int basemglevel;

// Multigrid levels
extern int mglevels;

// Multigrid factor
extern int mgfact;

// Multigrid factor on coarsest grid
extern int maxmglevelfact;

// Maps
extern int maps;

// Current position on the grid hierarchy
extern int reflevel;
extern int mglevel;
extern int mc_grouptype; // -1, CCTK_SCALAR/CCTK_ARRAY, CCTK_GF
extern int map;
extern int component;
extern int local_component; // -1 for non-local
extern int timelevel, timelevel_offset;

// Current refinement factors
extern int timereflevelfact;
extern vect<int, dim> spacereflevelfact;

// Current multigrid factor
extern int mglevelfact;

// Carpet's GH
extern CarpetGH carpetGH;

// Times and spaces on the refinement levels
extern CCTK_REAL global_time;
// extern vector<vector<CCTK_REAL> > leveltimes; // [mglevel][reflevel]
extern CCTK_REAL delta_time;

extern vector<vector<vect<CCTK_REAL, dim> > > origin_space; // [map][mglevel]
extern vector<vect<CCTK_REAL, dim> > delta_space;           // [map]

// Domain extent, as used for the original grid setup
// TODO: Unify this with origin_space and delta_space,
//       possibly asserting constency at run time
struct domainspec {
  vect<CCTK_REAL, dim> exterior_min, exterior_max;
  vect<int, dim> npoints;
};
extern vector<domainspec> domainspecs; // [map]

// Is this the time for a global mode call?
extern bool do_meta_mode;
extern bool do_early_meta_mode;
extern bool do_late_meta_mode;
extern bool do_global_mode;
extern bool do_early_global_mode;
extern bool do_late_global_mode;

// Can past time levels be accessed?
extern bool do_allow_past_timelevels;

// Is prolongation enabled?
// (This flag disables prolongation during MoL integration
// substeps.)
extern bool do_prolongate;

// Is tapering enabled?
// (This flag disables prolongation while the current refinement
// level is not aligned with the parent.)
extern bool do_taper;

// Should we warn about groups with insufficiently many time levels?
extern bool do_warn_about_storage;

// Are we in the analysis bin?
extern bool in_analysis_bin;

// Data for grid functions

// The grid hierarchy
extern vector<gh *> vhh; // [map]
extern vector<dh *> vdd; // [map]
extern th *tt;
extern int regridding_epoch; // increases with each regridding
extern vector<int> level_regridding_epochs;

// Data for the groups
struct groupdesc {
  cGroupDynamicData info;
  operator_type transport_operator;      // prolongation and restriction
  vector<vector<int> > activetimelevels; // [mglevel][reflevel]
};
extern vector<groupdesc> groupdata; // [group]

// Data for everything
struct arrdesc {
  // points to hh etc. for GF, and is unique for SCALAR and ARRAY
  gh *hh;
  dh *dd;
  th *tt;
  vector<ggf *> data; // [var]
};
extern vector<vector<arrdesc> > arrdata; // [group][map]

// MPI Communicators
extern MPI_Comm comm_universe;
extern MPI_Comm comm_world;

} // namespace Carpet

#endif // !defined(VARIABLES_HH)

```

carpet_public.h:
```
#ifndef CARPET_PUBLIC_H
#define CARPET_PUBLIC_H

#include <cctk.h>

#ifdef CCTK_MPI
#include <mpi.h>
#else
#include "nompi.h"
#endif

/* Tell thorns that the Carpet routines exist */
#define HAVE_CARPET

#ifdef __cplusplus
namespace Carpet {
extern "C" {
#endif

/* Carpet's GH extension */
struct CarpetGH {

#if 0
      /* Maximum number of refinement levels */
      int maxreflevels;
      
      /* Refinement levels */
      int reflevels;
      
      /* Refinement factor */
      int reffact;
      
      /* Refinement factor on finest possible grid */
      int maxreflevelfact;
      
      /* Base multigrid level */
      int basemglevel;
      
      /* Multigrid levels */
      int mglevels;
      
      /* Multigrid factor */
      int mgfact;
      
      /* Multigrid factor on coarsest grid */
      int maxmglevelfact;
#endif

  /* Maps */
  int maps;

#if 0
      /* Current position on the grid hierarchy */
      int reflevel;
      int mglevel;
#endif
  int map;
#if 0
      int component;
      
      /* Current refinement factor */
      int reflevelfact;
      
      /* Current multigrid factor */
      int mglevelfact;
#endif

#if 0
      /* Number of buffer zones */
      int const * nbufferzones;
#endif
};

struct CarpetGH const *GetCarpetGH(const cGH *const cgh) CCTK_ATTRIBUTE_PURE;

/* Prolongation management */
CCTK_INT CarpetEnableProlongating(const CCTK_INT flag);
CCTK_INT CarpetQueryProlongating(void);

/* Grid function access */
CCTK_POINTER Carpet_VarDataPtrI(CCTK_POINTER_TO_CONST const cctkGH,
                                CCTK_INT const m, CCTK_INT const rl,
                                CCTK_INT const c, CCTK_INT const tl,
                                CCTK_INT const varindex);

/* Call a schedule group */
int CallScheduleGroup(cGH *const cgh, const char *const group);

/* Call a local function */
int CallLocalFunction(cGH *const cgh, void (*const function)(cGH *const cgh));
int CallSinglemapFunction(cGH *const cgh,
                          void (*const function)(cGH *const cgh));
int CallLevelFunction(cGH *const cgh, void (*const function)(cGH *const cgh));
int CallGlobalFunction(cGH *const cgh, void (*const function)(cGH *const cgh));
int CallMetaFunction(cGH *const cgh, void (*const function)(cGH *const cgh));

/* Helper functions */
MPI_Comm CarpetMPIComm(void) CCTK_ATTRIBUTE_PURE;
MPI_Datatype CarpetMPIDatatype(int vartype) CCTK_ATTRIBUTE_PURE;
MPI_Datatype CarpetSimpleMPIDatatype(int vartype) CCTK_ATTRIBUTE_PURE;
int CarpetSimpleMPIDatatypeLength(int vartype) CCTK_ATTRIBUTE_PURE;

#ifdef __cplusplus
} /* extern "C" */
} /* namespace Carpet */
#endif

#endif /* !defined(CARPET_PUBLIC_H) */

```

util/plot-bboxes:
```
splot "< awk 'BEGIN { FS=\"[][,]\"; } { print $2,$3,$4; print $6,$3,$4; print $6,$7,$4; print $2,$7,$4; print $2,$3,$4; print \"\"; print $2,$3,$8; print $6,$3,$8; print $6,$7,$8; print $2,$7,$8; print $2,$3,$8; print \"\"; print $2,$3,$4; print $2,$3,$8; print \"\"; print $6,$3,$4; print $6,$3,$8; print \"\"; print $2,$7,$4; print $2,$7,$8; print \"\"; print $6,$7,$4; print $6,$7,$8; print \"\"; }' < /dev/tty" w l

```

LoadBalanceReal/test.F90:
```
program test

  use carpet_boxtypes
  implicit none

  type(bbox), dimension(100) :: inboxes
  integer, dimension(3,2) :: obound_init =  reshape (source = (/ 1, 1, 1, 1, 1, 1 /), shape = (/ 3, 2 /))
  type(boundary) :: outbound
  integer :: nprocs, i, j, nregs, &
             mode
  type(ptr), dimension(:), allocatable :: sregions
  integer, dimension(:), allocatable :: proc_load
  real(wp) :: totcost, idealcost

  namelist /input/ nregs
  namelist /superregions/ inboxes, nprocs, ghostsize, alpha, limit_size

  open(unit=1,file='boxes.par',status='old',action='read')
  read(1, nml=input)
  close(1)

  open(unit=1,file='boxes.par',status='old',action='read')
  read(1, nml=superregions)
  close(1)

  mode = 2

! In mode 1 the processor decomposition is obtained for all integer values
! from 1 to nprocs and only the processor number and load imbalance is printed.
  if ( mode == 1) then 
    do j = 1, nprocs

      allocate ( sregions(nregs) )
      outbound%obound = obound_init

      do i = 1, nregs
        call create_sregion ( inboxes(i), outbound, i-1, sregions(i)%point )
      end do

      call SplitSuperRegions ( sregions, j )

      allocate ( proc_load(j) )

      proc_load = 0

      call calc_proc_load ( sregions, proc_load )
      totcost = sum ( proc_load )
      idealcost = totcost / j

      do i = 1, nregs
        call destroy_sregion ( sregions(i)%point )
      end do

      print*, j, 100*(maxval(proc_load) - idealcost)/idealcost

      deallocate ( sregions, proc_load )

    end do

! In mode 2 the processor decomposition is done for nprocs only but more datail
! including the full tree information is printed.
  else

    allocate ( sregions(nregs) )
    outbound%obound = obound_init

    do i = 1, nregs
      call create_sregion ( inboxes(i), outbound, i-1, sregions(i)%point )
    end do

    call SplitSuperRegions ( sregions, nprocs )

    do i = 1, nregs
      call print_tree ( sregions(i)%point )
    end do

    allocate ( proc_load(nprocs) )

    proc_load = 0

    call calc_proc_load ( sregions, proc_load )

    print*,'proc_load = ', proc_load 

    totcost = sum ( proc_load )
    print*,'total load = ', totcost

    idealcost = totcost / nprocs
    print*,'ideal load = ', idealcost
    print*,'max load = ', maxval ( proc_load )
    print*,'load_imbalance (%) = ', 100*(maxval(proc_load) - idealcost)/idealcost

    do i = 1, nregs
      call destroy_sregion ( sregions(i)%point )
    end do

    deallocate ( sregions, proc_load )

  end if
  
end program test

```

LoadBalanceReal/splitregions_recursively.F90:
```
#include "cctk.h"

subroutine splitregions_recursively ( &
     cxx_superregs, nsuperregs, &
     cxx_regs, &
     nprocs, &
     ghostsize_, alpha_, limit_size_, &
     procid_)
  use carpet_boxtypes
  implicit none
  
  integer,      intent(in) :: nsuperregs
  CCTK_POINTER, intent(in) :: cxx_superregs
  CCTK_POINTER, intent(in) :: cxx_regs
  integer,      intent(in) :: nprocs
  integer,      intent(in) :: ghostsize_
  CCTK_REAL,    intent(in) :: alpha_
  integer,      intent(in) :: limit_size_
  integer,      intent(in) :: procid_
  
  type(ptr), allocatable :: sregions(:)
  type(boundary)         :: outbound
  
  CCTK_POINTER :: cxx_superreg
  CCTK_POINTER :: cxx_tree
  type(bbox)   :: box
  integer      :: i
  
  
  
  ! Callback functions that are implemented in C++
  interface
     
     subroutine carpet_get_region (cxx_superregs, i, cxx_superreg)
       use carpet_boxtypes
       implicit none
       CCTK_POINTER, intent(in)  :: cxx_superregs
       integer,      intent(in)  :: i
       CCTK_POINTER, intent(out) :: cxx_superreg
     end subroutine carpet_get_region
     
     subroutine carpet_get_bbox (cxx_superreg, box, obound)
       use carpet_boxtypes
       implicit none
       CCTK_POINTER,   intent(in)  :: cxx_superreg
       type(bbox),     intent(out) :: box
       type(boundary), intent(out) :: obound
     end subroutine carpet_get_bbox
     
     subroutine carpet_insert_region (cxx_regs, reg)
       use carpet_boxtypes
       implicit none
       CCTK_POINTER,           intent(in) :: cxx_regs
       type(superregion2slim), intent(in) :: reg
     end subroutine carpet_insert_region
     
     subroutine carpet_create_tree_branch &
          (nch, dir, bounds, cxx_subtrees, cxx_tree)
       use carpet_boxtypes
       implicit none
       integer,      intent(in) :: nch
       integer,      intent(in) :: dir
       integer,      intent(in) :: bounds(nch+1)
       CCTK_POINTER, intent(in) :: cxx_subtrees(nch)
       CCTK_POINTER, intent(in) :: cxx_tree
     end subroutine carpet_create_tree_branch
     
     subroutine carpet_create_tree_leaf (sreg, cxx_tree)
       use carpet_boxtypes
       implicit none
       type(superregion2slim), intent(in)  :: sreg
       CCTK_POINTER,           intent(out) :: cxx_tree
     end subroutine carpet_create_tree_leaf
     
  end interface
  
  
  
  ! Set global parameters
  ghostsize  = ghostsize_
  alpha      = alpha_
  limit_size = limit_size_ /= 0
  procid     = procid_
  
  
  
  allocate (sregions(nsuperregs))
  do i=1, nsuperregs
     call carpet_get_region (cxx_superregs, i-1, cxx_superreg)
     call carpet_get_bbox (cxx_superreg, box, outbound)
     call create_sregion (box, outbound, i-1, sregions(i)%point)
  end do
  
  call SplitSuperRegions (sregions, nprocs)
  
  do i=1, nsuperregs
     call carpet_get_region (cxx_superregs, i-1, cxx_superreg)
     call insert_region (sregions(i)%point, cxx_tree, cxx_regs)
     call carpet_set_tree (cxx_superreg, cxx_tree)
  end do
  
  do i=1, nsuperregs
     call destroy_sregion (sregions(i)%point)
  end do
  deallocate (sregions)
  
contains
  
  recursive subroutine insert_region (sreg, cxx_tree, cxx_regs)
!   The intent has been removed to make it compile with gfortran 4.1.
!    type(superregion2), pointer, intent(in)  :: sreg
    type(superregion2), pointer  :: sreg
    CCTK_POINTER,                intent(in)  :: cxx_regs
    CCTK_POINTER,                intent(out) :: cxx_tree
    
    integer                   :: nch, ich
    integer                   :: dir
    integer                   :: mydir
    integer,      allocatable :: bounds(:)
    CCTK_POINTER, allocatable :: cxx_subtrees(:)
    type(superregion2slim)    :: sregslim
    
    ! TODO: insert tree dependencies into superregs
    
    if (associated(sreg%children)) then
       ! The region has children: traverse them recursively
       nch = size(sreg%children)
       allocate (bounds(nch+1))
       allocate (cxx_subtrees(nch))
       if (nch /= 2) then
          call CCTK_WARN (CCTK_WARN_ABORT, "number of children is not 2")
       end if
       mydir = -1
       do dir=1, 3
          if (sreg%children(1)%point%extent%dim(dir)%upper + &
               sreg%children(2)%point%extent%dim(dir)%stride == &
               sreg%children(2)%point%extent%dim(dir)%lower) then
             ! Found direction
             if (mydir > 0) then
                call CCTK_WARN (CCTK_WARN_ABORT, "could not determine direction")
             end if
             mydir = dir
          else
             if (sreg%children(1)%point%extent%dim(dir)%lower /= &
                  sreg%children(2)%point%extent%dim(dir)%lower .or. &
                  sreg%children(1)%point%extent%dim(dir)%upper /= &
                  sreg%children(2)%point%extent%dim(dir)%upper) then
                call CCTK_WARN (CCTK_WARN_ABORT, "children differ in unexpected ways")
             end if
          end if
       end do
       if (mydir < 0) then
          call CCTK_WARN (CCTK_WARN_ABORT, "could not determine direction")
       end if
       bounds(1) = sreg%children(1)%point%extent%dim(mydir)%lower
       bounds(2) = sreg%children(2)%point%extent%dim(mydir)%lower
       bounds(3) = sreg%children(2)%point%extent%dim(mydir)%upper + &
            sreg%children(2)%point%extent%dim(mydir)%stride
       do ich=1, nch
          call insert_region &
               (sreg%children(ich)%point, cxx_subtrees(ich), cxx_regs)
       end do
       call carpet_create_tree_branch &
            (nch, mydir-1, bounds, cxx_subtrees, cxx_tree)
    else
       ! The region is a leaf: insert it
       sregslim%extent           = sreg%extent
       sregslim%outer_boundaries = sreg%outer_boundaries
       sregslim%map              = sreg%map
       sregslim%processor        = sreg%processor
       call carpet_create_tree_leaf (sregslim, cxx_tree)
       call carpet_insert_region (cxx_regs, sregslim)
    end if
  end subroutine insert_region
  
end subroutine splitregions_recursively

```

LoadBalanceReal/splitregions_recursively.cc:
```
#include <cctk.h>
#include <cctk_Parameters.h>

#include <algorithm>
#include <cassert>
#include <cmath>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <fstream>
#include <iomanip>
#include <iostream>
#include <limits>
#include <list>
#include <sstream>
#include <string>
#include <vector>

#include <sys/stat.h>
#include <sys/types.h>

#ifdef CCTK_MPI
#include <mpi.h>
#else
#include "nompi.h"
#endif

#include <Timer.hh>

#include <loopcontrol.h>

#include <bbox.hh>
#include <bboxset.hh>
#include <defs.hh>
#include <dh.hh>
#include <gh.hh>
#include <region.hh>
#include <vect.hh>

#include <carpet.hh>
#include <modes.hh>
#include <variables.hh>

namespace Carpet {

using namespace std;

// Helper routines for spliting regions automatically

// The cost for a region, assuming a cost of 1 per interior point
static rvect cost(region_t const &reg) {
  DECLARE_CCTK_PARAMETERS;
  static rvect costfactor;
  static bool initialised = false;
  if (not initialised) {
    costfactor = rvect(1.0);
    if (dim > 0)
      costfactor[0] = 1.0 / aspect_ratio_x;
    if (dim > 1)
      costfactor[1] = 1.0 / aspect_ratio_y;
    if (dim > 2)
      costfactor[2] = 1.0 / aspect_ratio_z;
  }
  if (reg.extent.empty())
    return rvect(0);
  return rvect(reg.extent.shape() / reg.extent.stride()) * costfactor;
}

struct f_range {
  int lower, upper, stride;
};

struct f_bbox {
  f_range dim[3];

  f_bbox() {}
  f_bbox(ibbox const &box) {
    assert(::dim == 3);
    for (int d = 0; d < 3; ++d) {
      dim[d].lower = box.lower()[d];
      dim[d].upper = box.upper()[d];
      dim[d].stride = box.stride()[d];
    }
  }
  /*explicit*/ operator ibbox() const {
    ivect lower, upper, stride;
    assert(::dim == 3);
    for (int d = 0; d < 3; ++d) {
      lower[d] = dim[d].lower;
      upper[d] = dim[d].upper;
      stride[d] = dim[d].stride;
    }
    return ibbox(lower, upper, stride);
  }
};

struct f_boundary {
  int obound[2][3];

  f_boundary() {}
  f_boundary(b2vect const &ob) {
    assert(::dim == 3);
    for (int d = 0; d < 3; ++d) {
      for (int f = 0; f < 2; ++f) {
        obound[f][d] = ob[f][d];
      }
    }
  }
  /*explicit*/ operator b2vect() const {
    b2vect ob;
    assert(::dim == 3);
    for (int d = 0; d < 3; ++d) {
      for (int f = 0; f < 2; ++f) {
        ob[f][d] = obound[f][d];
      }
    }
    return ob;
  }
};

struct f_superregion2slim {
  f_bbox extent;
  f_boundary outer_boundaries;
  int map;
  int processor;

  /*explicit*/ operator region_t() const {
    region_t reg;
    reg.extent = ibbox(extent);
    reg.outer_boundaries = b2vect(outer_boundaries);
    reg.map = map;
    reg.processor = processor;
    reg.processors = NULL;
    return reg;
  }

  /*explicit*/ operator pseudoregion_t() const {
    pseudoregion_t preg;
    preg.extent = ibbox(extent);
    preg.component = processor;
    return preg;
  }
};

extern "C" CCTK_FCALL void CCTK_FNAME(splitregions_recursively)(
    CCTK_POINTER const &cxx_superregs, int const &nsuperregs,
    CCTK_POINTER const &cxx_regs, int const &nprocs, int const &ghostsize,
    CCTK_REAL const &alpha, int const &limit_size, int const &procid);

void SplitRegionsMaps_Recursively(cGH const *const cctkGH,
                                  vector<vector<region_t> > &superregss,
                                  vector<vector<region_t> > &regss) {
  DECLARE_CCTK_PARAMETERS;

  if (recompose_verbose)
    cout << "SRMR enter" << endl;

  int const nmaps = superregss.size();
  assert(int(regss.size()) == nmaps);
  int map_offset = 1000000000;
  int max_map = 0;
  vector<bool> have_map(maps, false);
  for (int m = 0; m < nmaps; ++m) {
    for (int r = 0; r < int(superregss.AT(m).size()); ++r) {
      map_offset = min(map_offset, superregss.AT(m).AT(r).map);
      max_map = max(max_map, superregss.AT(m).AT(r).map);
      have_map.AT(superregss.AT(m).AT(r).map) = true;
    }
  }
  // Apparently this is not always the case:
  // assert (max_map - map_offset == nmaps - 1);

  int nsuperregs = 0;
  for (int m = 0; m < nmaps; ++m) {
    nsuperregs += superregss.AT(m).size();
  }
  if (recompose_verbose)
    cout << "SRMR nsuperregs " << nsuperregs << endl;

  // Something to do?
  if (nsuperregs == 0)
    return;

  // Collect slices
  vector<region_t> superregs;
  {
    for (int m = 0; m < nmaps; ++m) {
      combine_regions(superregss.AT(m), superregs);
    }
    nsuperregs = superregs.size();

    // If the last region was removed, add a new empty region again.
    // A set of regions (corresponding to a refinement level or a
    // grid array) cannot be empty.
    if (nsuperregs == 0) {
      assert(nmaps == 1); // we should only be here for grid
                          // arrays
      region_t reg;
      reg.extent = ibbox(ivect(0), ivect(-1), ivect(1));
      reg.outer_boundaries = b2vect(bvect(true), bvect(true));
      reg.map = 0;
      superregs.push_back(reg);
      nsuperregs = superregs.size();
    }
  }

  // Create a mapping from this list of regions to maps, and set the
  // map to the superregion index instead
  vector<int> superreg_maps(nsuperregs);
  for (int r = 0; r < nsuperregs; ++r) {
    superreg_maps.AT(r) = superregs.AT(r).map;
    superregs.AT(r).map = r;
  }

  // Handle granularity: artificially reduce the size of the
  // superregions
  for (int r = 0; r < nsuperregs; ++r) {
    region_t &superreg = superregs.AT(r);
    ibbox &ext = superregs.AT(r).extent;
    ivect lo = ext.lower();
    ivect hi = ext.upper() + ext.stride();
    ivect str = ext.stride();
    // cut off outer boundaries
    lo +=
        (ivect(superreg.outer_boundaries[0]) * int(granularity_boundary) * str);
    hi -=
        (ivect(superreg.outer_boundaries[1]) * int(granularity_boundary) * str);
    // multiply stride by granularity
    str *= granularity;
    assert(all((hi - lo) % str == 0));
    ext = ibbox(lo, hi - str, str);
  }

  int const real_nprocs = CCTK_nProcs(cctkGH);
  if (recompose_verbose)
    cout << "SRMR real_nprocs " << real_nprocs << endl;

  // Deactivate some processors if there are too many
  int nprocs;
  if (min_points_per_proc == 0) {
    nprocs = real_nprocs;
  } else {
    CCTK_REAL mycost = 0;
    for (int r = 0; r < nsuperregs; ++r) {
      mycost += prod(cost(superregs.AT(r)));
    }
    int const goodnprocs = int(floor(mycost / min_points_per_proc));
    nprocs = max(1, min(real_nprocs, goodnprocs));
  }
  if (recompose_verbose)
    cout << "SRMR nprocs " << nprocs << endl;

  // Distribute load
  CCTK_POINTER const cxx_superregs = &superregs;
  assert((int)superregs.size() == nsuperregs);
  vector<region_t> regs;
  // regs.reserve (...);
  CCTK_POINTER const cxx_regs = &regs;
  int const ghostsize = vdd.AT(0)->ghost_widths.AT(0)[0][0];
  for (int m = 0; m < maps; ++m) {
    for (int rl = 0; rl < reflevels; ++rl) {
      for (int f = 0; f < 2; ++f) {
        for (int d = 0; d < dim; ++d) {
          assert(vdd.AT(m)->ghost_widths.AT(rl)[f][d] == ghostsize);
        }
      }
    }
  }
  CCTK_REAL const alpha = ghost_zone_cost;
  int const limit_size = true;
  int const procid = CCTK_MyProc(cctkGH);
  CCTK_FNAME(splitregions_recursively)
  (cxx_superregs, nsuperregs, cxx_regs, nprocs, ghostsize, alpha, limit_size,
   procid);
  int nregs = regs.size();

  // Handle granularity: increase the size of the superregions and
  // regions again
  for (int r = 0; r < nsuperregs; ++r) {
    region_t &superreg = superregs.AT(r);
    ibbox &ext = superregs.AT(r).extent;
    ivect lo = ext.lower();
    ivect hi = ext.upper() + ext.stride();
    ivect str = ext.stride();
    // divide stride by granularity
    assert(all(str % int(granularity) == 0));
    str /= granularity;
    // add outer boundaries again
    lo -=
        (ivect(superreg.outer_boundaries[0]) * int(granularity_boundary) * str);
    hi +=
        (ivect(superreg.outer_boundaries[1]) * int(granularity_boundary) * str);
    ext = ibbox(lo, hi - str, str);
  }
  for (int r = 0; r < nregs; ++r) {
    region_t &reg = regs.AT(r);
    ibbox &ext = regs.AT(r).extent;
    ivect lo = ext.lower();
    ivect hi = ext.upper() + ext.stride();
    ivect str = ext.stride();
    // divide stride by granularity
    assert(all(str % int(granularity) == 0));
    str /= int(granularity);
    // add outer boundaries again
    lo -= ivect(reg.outer_boundaries[0]) * int(granularity_boundary) * str;
    hi += ivect(reg.outer_boundaries[1]) * int(granularity_boundary) * str;
    ext = ibbox(lo, hi - str, str);
  }

  if (same_number_of_components_on_each_process) {
    // Ensure all processes have the same number of components
    vector<int> ncomps(nprocs, 0);
    for (int r = 0; r < nregs; ++r) {
      int const p = regs.AT(r).processor;
      assert(p >= 0);
      ++ncomps.AT(p);
    }
    int maxncomps = 0;
    int sumncomps = 0;
    for (int p = 0; p < nprocs; ++p) {
      maxncomps = max(maxncomps, ncomps.AT(p));
      sumncomps += ncomps.AT(p);
    }
    int const missingcomps = maxncomps * nprocs - sumncomps;
    if (missingcomps > 0) {
      // Invent a dummy component
      ibbox const &ext = superregss.AT(0).AT(0).extent;
      region_t dummy;
      dummy.extent =
          ibbox(ext.lower(), ext.lower() - ext.stride(), ext.stride());
      assert(dummy.extent.empty());
      dummy.outer_boundaries = b2vect(true);
      dummy.map = nmaps - 1; // arbitrary choice
      // Insert dummy regions at the end
      regs.resize(nregs + missingcomps, dummy);
      for (int p = 0; p < nprocs; ++p) {
        for (int i = ncomps.AT(p); i < maxncomps; ++i) {
          regs.AT(nregs++).processor = p;
        }
      }
      assert(nregs == int(regs.size()));
      // Insert a superregion
      // TODO: Do we need this? Should we skip this?
      pseudoregion_t sample;
      sample.extent = dummy.extent;
      // sample.component remains unset (component will be set later)
      if (missingcomps == 1) {
        dummy.processors = new ipfulltree(sample);
      } else {
        int const dir = 0; // arbitrary choice
        vector<int> bounds(missingcomps + 1);
        for (int n = 0; n < missingcomps + 1; ++n) {
          bounds.AT(n) = dummy.extent.lower()[dir];
        }
        vector<ipfulltree *> subtrees(missingcomps);
        for (int n = 0; n < missingcomps; ++n) {
          subtrees.AT(n) = new ipfulltree(sample);
        }
        dummy.processors = new ipfulltree(dir, bounds, subtrees);
      }
      superregs.push_back(dummy);
    }
  }

  // Allocate regions, saving the old regions for debugging or
  // self-checking
  vector<vector<region_t> > old_superregss;
  swap(superregss, old_superregss);
  superregss.resize(old_superregss.size());
  assert((int)regss.size() == nmaps);
  for (int m = 0; m < nmaps; ++m) {
    assert(regss.AT(m).empty());
    // regss.AT(m).reserve (...);
    // superregss.AT(m).clear();
    assert(superregss.AT(m).empty());
    // superregss.AT(m).reserve (...);
  }
  // Assign regions
  for (int r = 0; r < nsuperregs; ++r) {
    superregs.AT(r).map = superreg_maps.AT(r); // correct map
    int const m = superregs.AT(r).map - map_offset;
    assert(m >= 0 and m < nmaps);
    superregss.AT(m).push_back(superregs.AT(r));
  }
  // Renumber components
  for (int m = 0; m < nmaps; ++m) {
    int c = 0;
    for (size_t r = 0; r < superregss.AT(m).size(); ++r) {
      ipfulltree *const procs = superregss.AT(m).AT(r).processors;
      assert(procs);
      for (ipfulltree::iterator it(*procs); not it.done(); ++it) {
        (*it).payload().component = c++;
      }
    }
  }
  for (int r = 0; r < nregs; ++r) {
    int const s = regs.AT(r).map;
    assert(s >= 0 and s < nsuperregs);
    regs.AT(r).map = superreg_maps.AT(s); // correct map
    int const m = regs.AT(r).map - map_offset;
    assert(m >= 0 and m < nmaps);
    regss.AT(m).push_back(regs.AT(r));
  }
  // Output regions
  if (recompose_verbose) {
    region_t::full_output = true;
    cout << "SRMR superregss " << superregss << endl;
    cout << "SRMR regss " << regss << endl;
    region_t::full_output = false;
  }

  // Consistency check
  bool has_error = false;
  vector<ibset> all_old_superregss(maps);
  for (size_t m = 0; m < superregss.size(); ++m) {
    for (size_t r = 0; r < old_superregss.AT(m).size(); ++r) {
      region_t const &reg = old_superregss.AT(m).AT(r);
      if (all_old_superregss.AT(reg.map).intersects(reg.extent)) {
        has_error = true;
        cout << "SRMR: old_superregss:\n"
             << "m=" << m << " r=" << r << " reg=" << reg << "\n";
      }
      all_old_superregss.AT(reg.map) += reg.extent;
    }
  }
  vector<ibset> all_superregss(maps);
  for (size_t m = 0; m < superregss.size(); ++m) {
    for (size_t r = 0; r < superregss.AT(m).size(); ++r) {
      region_t const &reg = superregss.AT(m).AT(r);
      if (all_superregss.AT(reg.map).intersects(reg.extent)) {
        has_error = true;
        cout << "SRMR: all_superregss:\n"
             << "m=" << m << " r=" << r << " reg=" << reg << "\n";
      }
      all_superregss.AT(reg.map) += reg.extent;
    }
  }
  for (int m = 0; m < maps; ++m) {
    if (not(all_superregss.AT(m) == all_old_superregss.AT(m))) {
      has_error = true;
      cout << "SRMR: all_superregss m=" << m << "\n";
    }
  }
  vector<ibset> all_regss(maps);
  for (size_t m = 0; m < regss.size(); ++m) {
    for (size_t r = 0; r < regss.AT(m).size(); ++r) {
      region_t const &reg = regss.AT(m).AT(r);
      if (all_regss.AT(reg.map).intersects(reg.extent)) {
        has_error = true;
        cout << "SRMR: all_regss:\n"
             << "m=" << m << " r=" << r << " reg=" << reg << "\n";
      }
      all_regss.AT(reg.map) += reg.extent;
    }
  }
  for (int m = 0; m < maps; ++m) {
    if (not(all_regss.AT(m) == all_old_superregss.AT(m))) {
      has_error = true;
      cout << "SRMR: all_regss m=" << m << "\n";
    }
  }
  if (has_error) {
    region_t::full_output = true;
    cout << "SRMR: all_old_superregss=" << all_old_superregss << "\n"
         << "SRMR: all_superregss=" << all_superregss << "\n"
         << "SRMR: all_regss=" << all_regss << "\n";
    region_t::full_output = false;
    CCTK_WARN(CCTK_WARN_ABORT, "Internal error");
  }

  if (recompose_verbose)
    cout << "SRMR exit" << endl;
}

extern "C" CCTK_FCALL void
    CCTK_FNAME(carpet_get_region)(CCTK_POINTER &cxx_superregs, int const &i,
                                  CCTK_POINTER &cxx_superreg) {
  vector<region_t> &superregs = *static_cast<vector<region_t> *>(cxx_superregs);
  region_t &superreg = superregs.AT(i);
  cxx_superreg = &superreg;
}

extern "C" CCTK_FCALL void
    CCTK_FNAME(carpet_get_bbox)(CCTK_POINTER &cxx_superreg, f_bbox &box,
                                f_boundary &obound) {
  region_t &superreg = *static_cast<region_t *>(cxx_superreg);
  box = f_bbox(superreg.extent);
  obound = f_boundary(superreg.outer_boundaries);
}

extern "C" CCTK_FCALL void
    CCTK_FNAME(carpet_insert_region)(CCTK_POINTER &cxx_regs,
                                     f_superregion2slim const &reg) {
  vector<region_t> &regs = *static_cast<vector<region_t> *>(cxx_regs);
  regs.push_back(region_t(reg));
}

extern "C" CCTK_FCALL void CCTK_FNAME(carpet_create_tree_branch)(
    int const &nch, int const &dir, int const fbounds[],
    CCTK_POINTER cxx_subtrees[], CCTK_POINTER &cxx_tree) {
  vector<int> bounds(nch + 1);
  vector<ipfulltree *> subtrees(nch);
  for (int i = 0; i < nch + 1; ++i) {
    bounds.AT(i) = fbounds[i];
  }
  for (int i = 0; i < nch; ++i) {
    ipfulltree *const tree = static_cast<ipfulltree *>(cxx_subtrees[i]);
    assert(tree->invariant());
    subtrees.AT(i) = tree;
  }
  cxx_tree = new ipfulltree(dir, bounds, subtrees);
}

extern "C" CCTK_FCALL void
    CCTK_FNAME(carpet_create_tree_leaf)(f_superregion2slim const &sreg,
                                        CCTK_POINTER &cxx_tree) {
  cxx_tree = new ipfulltree(pseudoregion_t(sreg));
}

extern "C" CCTK_FCALL void
    CCTK_FNAME(carpet_set_tree)(CCTK_POINTER &cxx_superreg,
                                CCTK_POINTER &cxx_tree) {
  region_t &superreg = *static_cast<region_t *>(cxx_superreg);
  ipfulltree *tree = static_cast<ipfulltree *>(cxx_tree);
  assert(not superreg.processors);
  superreg.processors = tree;
}

} // namespace Carpet

```

LoadBalanceReal/make.code.defn:
```
# Main make.code.defn file for thorn Carpet   -*-Makefile-*-

# Source files in this directory
SRCS = carpet_boxtypes.F90 carpet_region.F90 splitregions_recursively.F90 splitregions_recursively.cc
```

LoadBalanceReal/carpet_region.F90:
```
module carpet_region 
  
  implicit none

  integer, parameter :: wp = selected_real_kind(12,99)
  
  ! These empty arrays are used to initialize variables to either the
  ! min or max possible number of kind wp or integer.
  real(wp), dimension(2:1) :: empty
  integer, dimension(2:1) :: iempty
  
  
  
  ! Note that using intent on pointer arguments requires fortran 2003.
  ! It is not allowed in fortran 90 or 95.
  
  ! The basic range structure with lower and upper bounds and the stride
  type range
     integer :: lower, upper, stride
  end type range
  
  ! The basic box structure: a length 3 vector of ranges.
  type bbox
     type(range), dimension(3) :: dim
  end type bbox
  
  ! The outer boundary information structure.
  type boundary
     integer, dimension(3,2) :: obound
  end type boundary
  
! A super region structure similar to the Carpet tree structure.
  type ptr
    type(superregion2), pointer :: point
  end type ptr
  type superregion2
    type(bbox) :: extent
    type(boundary) :: outer_boundaries
    integer :: map
    integer :: processor
    real(wp) :: frac
    type(ptr), pointer :: children(:)
  end type superregion2
  type superregion2slim
    type(bbox) :: extent
    type(boundary) :: outer_boundaries
    integer :: map
    integer :: processor
  end type superregion2slim
 
  contains

!   Routine to allocate memory for a super region. Input is the bbox
!   structure, outer boundary information structure and map number.
!   The processor id is initialized to -1 and the pointer to the children
!   is nullified.
    subroutine create_sregion ( box, outerbound, map, sregion )
      type(bbox), intent(in) :: box
      type(boundary), intent(in) :: outerbound
      integer, intent(in) :: map
!     The intent has been removed to make it compile with gfortran 4.1.
!      type(superregion2), pointer, intent(out) :: sregion
      type(superregion2), pointer :: sregion
!      gfortran dies with an internal compiler error on this initialization.
!      type(superregion2), pointer, intent(out) :: sregion => null()

      allocate ( sregion )
      sregion%extent = box
      sregion%outer_boundaries = outerbound
      sregion%map = map
      sregion%processor = -1
      sregion%frac = 0.0_wp
      nullify(sregion%children)

    end subroutine create_sregion
 

!   Routine to nullify the pointers of the children. This has to be used
!   before destroying a temporary super region and assumes another super
!   region has children with pointers to these super regions as well.
!   The use of intent(in) on the sregion just ensures that sregion itself
!   is not modified. Its childrens pointer can be nullified.
    subroutine disassociate ( sregion )
!     The intent has been removed to make it compile with gfortran 4.1.
!      type(superregion2), pointer, intent(in) :: sregion
      type(superregion2), pointer :: sregion
      integer :: n, i

!     Only do something if the children are associated.
      if ( associated(sregion%children) ) then
        n = size(sregion%children)
        do i = 1, n
          nullify ( sregion%children(i)%point )
        end do
      end if
    end subroutine disassociate


!   Routine to assign the pointers of the children of sregion1 to the same
!   super regions as sregion2.
    subroutine point_to_children ( sregion1, sregion2 )
!     The intent has been removed to make it compile with gfortran 4.1.
!      type(superregion2), pointer, intent(inout) :: sregion1
!      type(superregion2), pointer, intent(in) :: sregion2
      type(superregion2), pointer :: sregion1
      type(superregion2), pointer :: sregion2
      integer :: n1, n2, i

!     Only do something if sregion2 actually has children.
      if ( associated(sregion2%children) ) then
        n2 = size(sregion2%children)

!       If sregion1 has children as well make sure to deallocate their
!       sub tree (this probably shouldn't happen.
        if ( associated(sregion1%children) ) then
          n1 = size(sregion1%children)
          do i = 1, n1
            call destroy_sregion ( sregion1%children(i)%point )
          end do
        end if

!       Allocate the children and assign the pointers.
        allocate ( sregion1%children(n2) )
        do i = 1, n2
          sregion1%children(i)%point => sregion2%children(i)%point
        end do

!     Otherwise print a warning. This should not happen.
      else
        print*, 'Warning sregion2 has no children'
      end if  
    end subroutine point_to_children
      

!   Routine to recursively deallocate memory for a super region.
    recursive subroutine destroy_sregion ( sregion )
!     The intent has been removed to make it compile with gfortran 4.1.
!      type(superregion2), pointer, intent(inout) :: sregion
      type(superregion2), pointer :: sregion
      integer :: n, i

!     If the super region has children.
      if ( associated(sregion%children) ) then

!       Find out how many children.
        n = size(sregion%children)

!       Loop over all children.
        do i = 1, n

!         If the children has children call recursively.
          if ( associated ( sregion%children(i)%point ) ) then
            call destroy_sregion ( sregion%children(i)%point )
          end if
        end do

!       Then deallocate the storage for the children pointers.
        if ( n > 0 ) then
          deallocate ( sregion%children )
        end if
      end if

!     Finally deallocate storage for the super region itself.
      deallocate (sregion)

    end subroutine destroy_sregion


!   Routine to recursively print all information about a super region tree
!   structure to stdout. 
    recursive subroutine print_tree ( sregion )
!     The intent has been removed to make it compile with gfortran 4.1.
!      type(superregion2), pointer, intent(in) :: sregion
      type(superregion2), pointer :: sregion
      integer :: n, i
      
!     Only do something if the super region pointer is associated with a
!     valid target.
      if ( associated(sregion) ) then
        print*, 'bbox = ', sregion%extent
        print*, 'outer_boundaries = ', sregion%outer_boundaries
        print*, 'map = ', sregion%map
        print*, 'processor = ', sregion%processor
        print*, 'fraction = ', sregion%frac

!       If the super region has no children...
        if ( .not. associated(sregion%children) ) then
          print*, 'No children '
          print*

!       Otherwise call recursively for all the children.
        else
          n = size(sregion%children)
          print*, 'Number of children = ', n
          print*
          print*
          do i = 1, n
            call print_tree ( sregion%children(i)%point )
            print*
          end do
        end if
      else
        print*, "Pointer to super region not associated"
        print*
      end if

    end subroutine print_tree


!   Split a super region into an arbitrary number of pieces in direction dir.
!   The number of pieces is size(frac)+1. The integer array frac contains the
!   upper boundary of the n first regions to split and has to be ordered with
!   the largest element smaller than the size of the super region in direction
!   dir. There is currently no check that this is indeed the case.
    subroutine split_sregion ( sregion, dir, frac )
!     The intent has been removed to make it compile with gfortran 4.1.
!      type(superregion2), pointer, intent(in) :: sregion
      type(superregion2), pointer :: sregion
      integer, intent(in) :: dir
      integer, dimension(:), intent(in) :: frac
      type(bbox), dimension(size(frac)+1) :: newboxes
      type(boundary), dimension(size(frac)+1) :: newboundaries
      integer :: n, i, lo, up, str

!     Determine the size of frac.
      n = size(frac)

!     Only do something if n is larger than 0.
      if (n>0) then

!       Allocate storage for the the children pointers.
        allocate(sregion%children(n+1))

!       Determine the low and high boundaries and the stride of the super
!       region.
        lo = sregion%extent%dim(dir)%lower
        up = sregion%extent%dim(dir)%upper
        str = sregion%extent%dim(dir)%stride

!       copy the bbox and boundary information from the super region.
        do i = 1, n+1
          newboxes(i) = sregion%extent
          newboundaries(i) = sregion%outer_boundaries
        end do

!       Update the lower and upper boundaries of the children regions
!       according to the number of points passed in through frac.
!       Also set the outer boundary information to 0.
        do i = 1, n
          newboxes(i)%dim(dir)%upper = lo + ( frac(i) - 1 ) * str
          newboxes(i+1)%dim(dir)%lower = newboxes(i)%dim(dir)%upper + str
          newboundaries(i)%obound(dir,2) = 0
          newboundaries(i+1)%obound(dir,1) = 0
        end do

!       Allocate memory for the children and copy the bbox and outer boundary
!       information in. nullify the pointers of the children to ensure that
!       they are correctly disassociated.
        do i = 1, n+1
          allocate ( sregion%children(i)%point )
          sregion%children(i)%point%extent = newboxes(i)
          sregion%children(i)%point%outer_boundaries = newboundaries(i)
          sregion%children(i)%point%map = sregion%map
          sregion%children(i)%point%processor = sregion%processor
          sregion%children(i)%point%frac = 0.0_wp
          nullify(sregion%children(i)%point%children)
        end do
      end if

    end subroutine split_sregion
          
end module carpet_region

```

LoadBalanceReal/boxes.par:
```
&input nregs=7 /
&superregions inboxes=0 44 1 0 44 1 0 1993 1
                      0 44 1 0 44 1 0 1993 1
                      0 44 1 0 44 1 0 1993 1
                      0 44 1 0 44 1 0 1993 1
                      0 44 1 0 44 1 0 1993 1
                      0 44 1 0 44 1 0 1993 1
                      0 194 1 0 194 1 0 194 1,
              nprocs=10, ghostsize=5, alpha=0.025, limit_size=.T. /

```


combined_doc_context: ## README (README):
```
Cactus Code Thorn Carpet
Author(s)    : Erik Schnetter <schnetter@cct.lsu.edu>
Maintainer(s): Erik Schnetter <schnetter@cct.lsu.edu>
Licence      : GPLv2+
--------------------------------------------------------------------------

1. Purpose

This thorn provides a parallel AMR (adaptive mesh refinement) driver
with MPI.

```

## Documentation (.tex files):
Files: threelev_initdata.tex

```tex
=== threelev_initdata.tex ===
\documentclass{article}

\usepackage{epsf}
\topmargin -0.5in
\textheight 9.9in
\oddsidemargin -0.3in
\leftmargin -1in
\textwidth 7in


\begin{document}

\title{Getting Three Timelevels of FMR Initial Data}
\author{Scott Hawley}
\date{June 6, 2002}
\maketitle

For parabolic interpolation in time, one needs three timelevels worth
of data on the coarse grid in order to provide boundary data on the 
finer grid.  Here we describe one scheme for obtaining these extra 
timelevels of data at the initial time.  Essentially, we evolve 
each grid backwards in time by two steps, but we do so rather carefully.

The following diagram shows the scheme.  We start with initial
data for all grids being specified at the current time (Step 1, not 
shown).  We then evolve the coarse grid forward one step (Step 2) by
the timestep $\Delta t$;
in so doing we first move the coarse grid storage from the current
timelevel (denoted by `` '' on the right of the first pane in the
diagram, because variables at the current time receive no suffix 
in Cactus) to the previous timelevel (denoted by ``\_p'') before
evolving.  We then ``flip'' all the timelevels for the coarse grid (l=0), 
meaning
that we exchange the data on the `` '' and \_p\_p timelevels, 
thus turning the picture upside down.
We then evolve the coarse grid ``backward'' --- which is really
just forward but with a timestep of $-\Delta t$.  Then we move on
to the finer grid, evolving and flipping...

Take a look at the diagram.  (We use $\tau$ instead of $t$ to denote
the time, simply because it was easier to do that in the
drawing program.) After the diagram we give a
listing of pseudo-code which implements this scheme.
\begin{center}
   \epsfxsize=5.5in
    \epsffile{threelev_initdata.eps}
\end{center}

\begin{center}
   \epsfxsize=5.5in
    \epsffile{threelev_initdata_2.eps}
\end{center}

Diagram finished.

\pagebreak

Here's some pseudo-code:
\begin{verbatim}

  int time_dir = 1; //Positive = forward (+t), Negative = backward (-t)
  int lev;

  // At this point we assume that we have initial data given on
  // one timelevel, and we want to get data on the other timelevels

  do lev = 0 to MaxLevels-1
     // Evolve "forward" (which may be backward for lev=1,3,5,7...)
     Evolve(lev, time_dir*dt(lev))

     flip_timelevels_on_lev_and_coarser(lev)
     //    Keep track of which direction (in time) we're integrating
     time_dir = -time_dir

     // Evolve in the opposite time-direction
     Evolve(lev, time_dir*dt(lev))
  end do

  // Make sure we're pointed backwards, in order to get 2 "previous"
  //   timelevels.  We could change the if statement to 
  //   "if (mod(MaxLevels,2) == 0)", but I prefer to check time_dir 
  //   explicitly, because it's easier to follow and I don't have to
  //   worry about having made a mistake
  if (time_dir > 0) then
     flip_timelevels_on_lev_and_coarser(MaxLevels-1)
     time_dir = -time_dir
  end if

  // Evolve each level backwards one more timestep
  do lev = MaxLevels-1 to 0
     Evolve(lev,-dt(lev))
     if (lev>0) then
       Restrict(lev -> lev-1)
     end if
  end do

  // Here's where a user can add in extra evolution code if they want 
  // to be extra-careful ("anal"), but remember that you'll be 
  // overwriting the t=0 data so you'll need to re-load it if you do
  // this.   

  // Finally, one last flip to point us forward again
  flip_all_timelevels()
  time_dir = -time_dir
\end{verbatim}

\end{document}

```



Reference answer:
src_code: #include "cctk.h"

module carpet_boxtypes

  use carpet_region
  implicit none

  integer :: ghostsize
  real(wp) :: alpha
  logical :: limit_size
  integer :: procid

  contains

!   The basic calculation of the cost in each of the 3 directions.
    subroutine cost_box ( reg, cost )
      type(bbox), intent(in) :: reg
      integer, dimension(3), intent(out) :: cost
      integer :: i
    
      do i = 1, 3
        cost(i) = (reg%dim(i)%upper -  reg%dim(i)%lower)/reg%dim(i)%stride + 1
      end do
    end subroutine cost_box


!   Find a sorted list of the best options for splitting npoints in 2 chunks
!   among nprocs processors. The fractional remainder is passed in through 
!   fract. The maximum allowed number of options to return is nmax, and the
!   actual number of returned options is nact. The result is returned in
!   np (number of procs for the first chunk) and nc (a measure of the
!   imabalance of the split.
    subroutine choosenprocs ( npoints, nprocs, fract, nmax, nact, np, nc )
      integer, intent(in) :: npoints, nmax
      real(wp), intent(in) :: nprocs, fract
      integer, intent(out) :: nact
      integer, dimension(:), intent(out) :: np
      real(wp), dimension(:), intent(out) :: nc
      integer :: idealcost
      integer :: quot, i, j, round, ind
      real(wp) :: c1, c2, currentimbal
      logical :: exchange

!     The idealcost should be npoints/nprocs but I rescale with nprocs for
!     The purpose of avoiding some problems with floating point comparisons.
      idealcost = npoints

!     Initialize the cost to the maximum possible floating point number.
      nc = minval( empty )

!     initialize the actual returned number of options to 0.
      nact = 0

!     Initialize an integer index to zero. Used to keep track where the current
!     option should be inserted in the result vector.
      ind = 1

!     Calculate the starting number of processors to start the search with
!     (quot) as the largest integer smaller than nprocs.
      quot = floor(nprocs)
!     If the fractional part we have to leave unsplit is larger than 1
!     decrement quot by 1.
      if ( fract > 1.0_wp ) quot = quot-1
!     If nprocs has zero fractional part, we can start at quot/2 due to
!     symmetry.
      if ( .not. ( nprocs>quot ) ) then
        quot = quot / 2
      end if

!     Try all options until reaching a single processor.
      options: do i=quot, 1, -1

!       Calculate the number of points that will yield the smallest imbalance.
        round = nint(real(npoints,wp)/nprocs*i)

!       Calculate the imbalance for the current choice of processors.
!       This has been scaled by nprocs (like idealcost has been above).
        c1 = abs ( (round*nprocs)/i - idealcost )
        c2 = abs ( (npoints-round)/(nprocs-i)*nprocs - idealcost )
        currentimbal = max ( c1, c2 )

!       Initialize the boolean exchange that keeps track of whether the current
!       choice needs to replace an entry in the return vectors.
        exchange = .false.

!       Start from the current number of entries in the return vector and check
!       if the current choice is better than any of them. If so indicate that
!       an exchange has to happen and record the index.
        do j = nact, 1, -1
          if ( currentimbal < nc(j) ) then
            exchange = .true.
            ind = j
          end if
        end do

!       If an exchange has to happen, shift all elements with indices higher
!       than or equal to ind to the right.
        if (exchange) then
          np(ind:nmax) = eoshift(np(ind:nmax),-1)
          nc(ind:nmax) = eoshift(nc(ind:nmax),-1)
        end if

!       If we don't need to do an exchange, and if nact<nmax add the current
!       case after the previous entries.
        if (nact<nmax .and. .not.exchange) then
          nact = nact+1
          np(nact) = i
          nc(nact) = currentimbal
        end if

!       If the return vectors are empty or an exhange has to be done, insert
!       the current choice at the appropriate position (ind) and increment the
!       number of elements in the return vector (capped at nmax)
        if (nact == 0 .or. exchange) then
          nact = min(nact+1,nmax)
          np(ind) = i
          nc(ind) = currentimbal
        end if
      end do options

!     This is commented out for now. Used to return only the elements that are
!     as good as the best case.
!      nbad = nact + 1
!      do i = 2, nact
!        if ( abs(nc(i)-nc(i-1))>1.0d-10 ) nbad = i
!      end do
!      nact = nbad - 1
    end subroutine choosenprocs


!   Calculate a measure of the load imbalance of a super region.
!   The measure is a weighted sum taking into account the imbalance in the
!   number of points (computation) and the imbalance in the number of ghost
!   zones (communication). Ghost zones are added with a multiplication factor
!   of alpha.
    subroutine calc_imbalance ( sregion, imbalance )
!     The intent has been removed to make it compile with gfortran 4.1.
!      type(superregion2), pointer, intent(in) :: sregion
      type(superregion2), pointer :: sregion
      real(wp), intent(out) :: imbalance
      integer, dimension(3) :: cost
      real(wp) :: maxcost, maxgcost, n
      real(wp) :: idealcost, idealgcost

!     Initialize counter and accumulation variables
      n = 0.0_wp
      maxcost = 0.0_wp
      idealcost = 0.0_wp
      maxgcost = 0.0_wp
      idealgcost = 0.0_wp

!     Traverse the tree structure and add values for all leaves.
      call traverse_tree ( sregion )

!     Calculate the ideal cost
      idealcost = idealcost/n
      idealgcost = idealgcost/n

!     Calculate the imbalance
      imbalance = ( maxcost/idealcost - 1.0_wp ) &
                  + alpha * ( maxgcost/idealgcost - 1.0_wp )

      contains

!       Routine to traverse the tree and accumulate the cost of the leaves.
        recursive subroutine traverse_tree ( sreg )
!         The intent has been removed to make it compile with gfortran 4.1.
!          type(superregion2), pointer, intent(in) :: sreg
          type(superregion2), pointer :: sreg
          integer :: ich, nch, tcost, gcost

!         If the region has children recursively traverse the children.
          if ( associated(sreg%children) ) then
            nch = size(sreg%children)
            do ich = 1, nch
              call traverse_tree ( sreg%children(ich)%point )
            end do

!         Otherwise it's a leaf and we accumulate the cost and update the
!         maximum cost.
          else
            call cost_box ( sreg%extent, cost )
            tcost = product ( cost, mask = cost /= 0 )
            gcost = product ( cost+2*ghostsize, mask = cost /= 0 ) - tcost
            tcost = tcost
            gcost = gcost
            maxcost = max ( maxcost, tcost / sreg%frac )
            maxgcost = max ( maxgcost, gcost / sreg%frac )
            idealcost = idealcost + tcost
            idealgcost = idealgcost + gcost
            n = n + sreg%frac
          end if

        end subroutine traverse_tree

    end subroutine calc_imbalance


!   Assign processor numbers to the leaves in a tree starting from start_proc.
    subroutine AssignProcs ( minproc, maxproc, frac1, frac2, sregion )
      integer, intent(in) :: minproc, maxproc
      real(wp ), intent(in) :: frac1, frac2
      logical :: lower_is_outer, upper_is_outer
      integer :: boundarysize, minsize
!     The intent has been removed to make it compile with gfortran 4.1.
!      type(superregion2), pointer, intent(inout) :: sregion
      type(superregion2), pointer :: sregion
      character :: msg*200

!     Initialize.
      if ( frac1 == 0 ) then
        procid = minproc
      else
        procid = minproc + 1
      end if

!     Traverse the tree.
      call traverse_treep ( sregion )
      
      contains

!       Routine to traverse the tree and assign processor ids to the leaves.
        recursive subroutine traverse_treep ( sreg )
!          The intent has been removed to make it compile with gfortran 4.1.
!          type(superregion2), pointer, intent(inout) :: sreg
          type(superregion2), pointer :: sreg
          integer :: ich, nch, maxcost, np
          integer, dimension(3) :: cost
          integer, dimension(1) :: mydim

!         If the region has children recursively traverse the children.
          if ( associated(sreg%children) ) then
            nch = size(sreg%children)
            do ich = 1, nch
              call traverse_treep ( sreg%children(ich)%point )
            end do

!         Otherwise it's a leaf and we assign the processor id and increment it.
          else
            if ( sreg%frac == 1.0_wp ) then
              sreg%processor = procid
              procid = procid + 1
            else
              call cost_box ( sreg%extent, cost )
              maxcost = maxval(cost)
              mydim = maxloc (cost)
              lower_is_outer = sreg%outer_boundaries%obound(mydim(1),1)/=0
              upper_is_outer = sreg%outer_boundaries%obound(mydim(1),2)/=0
              if (frac1 == 0.0_wp) then
                np = 0
              else if (frac2 == 0.0_wp) then
                np = maxcost
              else
                np = nint(frac1/(frac1+frac2)*maxcost)
              end if
              if (limit_size) then
                boundarysize = ghostsize
                minsize = boundarysize+ghostsize
                if (lower_is_outer .and. np>0 .and. np<minsize) then
                  if (np<minsize/2) then
                    np = 0
                  else
                    np = minsize
                  end if
                end if
                if (upper_is_outer .and. &
                    np<maxcost .and. np>maxcost-minsize) then
                  if (np>maxcost-minsize/2) then
                    np = maxcost
                  else
                    np = maxcost-minsize
                  end if
                end if
              end if
              if ((lower_is_outer .and. np>0 .and. np<minsize) .or. &
                  (upper_is_outer .and. &
                   np<maxcost .and. np>maxcost-minsize)) then
                write (msg, '("Not enough grid points: np=",i4," gs=",i4, "maxcost=",i4)') &
                       np, boundarysize, maxcost
                call CCTK_WARN(CCTK_WARN_ABORT, msg)
              end if
              if (np>0 .and. np<maxcost) then
                call split_sregion ( sreg, mydim(1), (/ np /) )
                sreg%processor = -1
                sreg%children(1)%point%processor = minproc
                sreg%children(1)%point%frac = frac1
                sreg%children(2)%point%processor = maxproc
                sreg%children(2)%point%frac = frac2
              else if (np==0) then
                sreg%processor = maxproc
              else if (np==maxcost) then
                sreg%processor = minproc
              end if
              
            end if
          end if

        end subroutine traverse_treep

      end subroutine AssignProcs


!   Routine to sum up the load of an array of super regions. Return the result
!   in the pre-allocated array of length nprocs. The array should be
!   initialized to zero.
    subroutine calc_proc_load ( sregions, proc_load )
      type(ptr), dimension(:), intent(in) :: sregions
      integer, dimension(:), intent(inout) :: proc_load
      integer :: i

!     For each of the super regions in the array.
      do i = 1, size(sregions)

!       Traverse the tree and 
        call traverse_treel ( sregions(i)%point )
      end do

      contains

!       Routine to traverse the tree and add the load of each leaf to the
!       assigned processor.
        recursive subroutine traverse_treel ( sreg )
!         The intent has been removed to make it compile with gfortran 4.1.
!          type(superregion2), pointer, intent(in) :: sreg
          type(superregion2), pointer :: sreg
          integer, dimension(3) :: cost
          integer :: ich, nch

!         If the region has children recursively traverse the children.
          if ( associated(sreg%children) ) then
            nch = size(sreg%children)
            do ich = 1, nch
              call traverse_treel ( sreg%children(ich)%point )
            end do

!         Otherwise it's a leaf and we calculate the cost and accumulate it
!         into the correct spot in the proc_load array.

          else
            call cost_box ( sreg%extent, cost )
            proc_load(sreg%processor+1) = proc_load(sreg%processor+1) + &
                                          product ( cost )
          end if

        end subroutine traverse_treel

    end subroutine calc_proc_load


!   Split a superregion (newreg) recursively into smaller boxes while trying
!   to minimize the load imbalance. Here we allow for fractional processor
!   numbers, nprocs. The fraction that should be left unsplit is given by
!   fract (0.0<=fract<2). The inputs maxchoices and maxfactor are used to
!   determine how many options to examine at a given recursion level. The
!   variable clevel should be initialized to 0 and is used to keep track of
!   the recursion level. The number of options testes is counter and returned
!   in ncount.
    recursive subroutine SplitRegionsRecursive ( nprocs, fract, maxchoices, &
                                                 maxfactor, clevel, &
                                                 newreg, ncount )
      real(wp), intent(in) :: nprocs, fract
      integer, intent(in) :: maxchoices, maxfactor
      integer, intent(inout) :: clevel
!     The intent has been removed to make it compile with gfortran 4.1.
!      type(superregion2), pointer, intent(inout) :: newreg
      type(superregion2), pointer :: newreg
      integer, intent(inout) :: ncount
      type(bbox) :: reg
      integer, dimension(3) :: cost
      integer :: maxcost, ngood, i, np1, lo1, up1, str
      real(wp) :: p1, p2
      integer, dimension(1) :: mydim
      integer, dimension(maxchoices) :: npr
      real(wp), dimension(maxchoices) :: ncr
      type(ptr), dimension(maxchoices) :: newregarr
      real(wp), dimension(maxchoices) :: imbalancearr
      logical :: lower_is_outer, upper_is_outer
      integer :: boundarysize, minsize
      type(bbox) :: reg1, reg2
      integer, dimension(1) :: minimbalanceloc
      integer :: nlocal
      character :: msg*200

!     Extract the bounding box information
      reg = newreg%extent

!     Make sure the array with the test super regions are nullified.
      do i = 1, maxchoices
        nullify ( newregarr(i)%point )
      end do

!     Initialize the imbalance array to max floating point value.
      imbalancearr(:) = minval(empty)

!     Increment the recursion level variable.
      clevel = clevel + 1

!     Calculate the number of choices to examine at the current recursion
!     level based on the value of maxchoices and maxfactor.
      nlocal = max ( maxchoices - ( clevel - 1)  / maxfactor, 1 )

!     If we are called with nprocs==max(1,fract) mark the region to be a leaf
!     in the tree, increment the choice counter and decrement the recursion
!     level counter and return.
      if (nprocs <= max(1.0_wp,fract+0.000001_wp) ) then
        ncount = ncount + 1
        clevel = clevel - 1
        newreg%processor = 1
        newreg%frac = nprocs
        return
      end if

!     Calculate the cost of the current region.
      call cost_box ( reg, cost )

!     Find the dimension with the most points.
      maxcost = maxval(cost)
      mydim = maxloc (cost)

!     Get the list of chunk options to investigate.
      call choosenprocs ( cost(mydim(1)), nprocs, fract, nlocal, ngood, &
                                                                 npr, ncr )

!     Loop over the chunk options to investigate.
      chunks: do i=1, ngood

!       Initialize the super region.
        allocate ( newregarr(i)%point )
        newregarr(i)%point = newreg

!       Find the processor numbers of the two chunks (p1 always an integer).
        p1 = npr(i)
        p2 = nprocs - npr(i)

!       Find the number of points in each of the 2 chunks while making
!       sure the chunks are not too small compared to the ghostsize if
!       limit_size is true.
!       At the outer boundary, take also the boundary size into account,
!       assuming that the boundary size is equal to the ghost size.
!       (Boundary points cannot be split, and near a boundary, the
!        minimum number of interior points is the number of ghost points.)
        np1 = nint((real(maxcost,wp)*p1)/nprocs)
        if (limit_size) then
          lower_is_outer = &
               newregarr(i)%point%outer_boundaries%obound(mydim(1),1)/=0
          upper_is_outer = &
               newregarr(i)%point%outer_boundaries%obound(mydim(1),2)/=0
          boundarysize = ghostsize
          minsize = boundarysize+ghostsize
          if (lower_is_outer .and. np1>0 .and. np1<minsize) then
            np1 = minsize
          end if
          if (upper_is_outer .and. np1<maxcost .and. np1>maxcost-minsize) then
            np1 = maxcost-minsize
          end if
          if ((lower_is_outer .and. np1>0 .and. np1<minsize) .or. &
              (upper_is_outer .and. np1<maxcost .and. np1>maxcost-minsize)) then
            write (msg, '("Not enough grid points: np=",i4," gs=",i4, "maxcost=",i4)') &
                   np1, boundarysize, maxcost
            call CCTK_WARN(CCTK_WARN_ABORT, msg)
          end if
        end if

!       Split the region (reg) into two regions (reg1, reg2) taking the
!       stride into account.
        reg1 = reg
        reg2 = reg
        lo1 = reg%dim(mydim(1))%lower
        up1 = reg%dim(mydim(1))%upper
        str = reg%dim(mydim(1))%stride
        reg1%dim(mydim(1))%upper = lo1 + (np1 - 1)*str
        reg2%dim(mydim(1))%lower = reg1%dim(mydim(1))%upper + str

!       Add the new leaves to the tree.
        call split_sregion ( newregarr(i)%point, mydim(1), (/ np1 /) )

!       Split region reg1 among p1 processors and region reg2 among p2
!       processors adding the new regions to the super region.
        call SplitRegionsRecursive ( p1, fract, maxchoices, maxfactor, &
                                     clevel, &
                                     newregarr(i)%point%children(1)%point, &
                                     ncount )
        call SplitRegionsRecursive ( p2, fract, maxchoices, maxfactor, &
                                     clevel, &
                                     newregarr(i)%point%children(2)%point, &
                                     ncount )

!       Calculate the load imbalance of the resulting super region.
        call calc_imbalance ( newregarr(i)%point, imbalancearr(i) )

!       If the balance is perfect exit the do loop and don't do any further
!       tests.
        if ( imbalancearr(i) == 0.0_wp ) then
          ngood = i
          exit
        end if
      end do chunks

!     Find the location of the minimum load imbalance.
      minimbalanceloc = minloc ( imbalancearr(1:ngood) )      

!     Insert the best choice into the tree structure      
      call point_to_children ( newreg,  newregarr(minimbalanceloc(1))%point )

!     Cut the connection between the best choice and its children.
      call disassociate ( newregarr(minimbalanceloc(1))%point )

!     Deallocate all the tested options.
      do i = 1, ngood
        call destroy_sregion ( newregarr(i)%point )
      end do

!     Decrement the recursion level counter.
      clevel = clevel - 1

    end subroutine SplitRegionsRecursive


!   Routine to return the union of 2 pre-sorted arrays of integers.
!   only n1 elements of list1 and n2 elements of list2 are actually used.
!   The number of elements in the union is n. The union is returned in list.
    subroutine union (n1, list1, n2, list2, n, list )
      integer, intent(in) :: n1, n2
      integer, dimension(:), intent(in) :: list1, list2
      integer, intent(out) :: n
      integer, dimension(:), intent(out) :: list
      integer :: j, k, newl

!     Initialize n and list to 0. The integers j and k are used to march through
!     the 2 input arrays and are initialized to 1 if the lists have some
!     valid elements and 0 otherwise.
      n = 0; list = 0; j = min(1,n1); k = min(1,n2)

!     Initialize the output array to 0
      list = 0

!     If both input arrays are 0, so are the output array and we are done.
      if (n1==0 .and. n2==0) then
        return
      end if

!     Repeat until j>n1 and k>n2.
      loop: do

!       If there are still valid elements in both list1 and list2...
        lists: if ( (j>0 .and. j<=n1) .and. (k>0 .and. k<=n2) ) then

!         The next element to add to list is the minimum of the current values
!         in list1 and list2
          newl = min ( list1(j), list2(k) )

!         If the element is not already in the output list....
          both: if ( n > 0 ) then
            if ( newl > list(n) ) then

!             Add the element
              n = n + 1
              list(n) = newl

!             If the added element comes from list1 increment j
              if ( list1(j) == list(n) ) j = j+1

!             If the added element comes from list2 increment j
              if ( list2(k) == list(n) ) k = k+1

            end if

!         Or if the output list is empty
          else both

!           Add the element
            n = n + 1
            list(n) = newl

!           If the added element comes from list1 increment j
            if ( list1(j) == list(n) ) j = j+1

!           If the added element comes from list2 increment j
            if ( list2(k) == list(n) ) k = k+1

          end if both

!       If there there is only valid elements left in list1
        else if (j>0 .and. j<=n1) then lists

!         Add the current element of list1 to list if it is not already there
!         or list is empty and increment j.
          newl = list1(j)
          list_one: if ( n > 0 ) then
            if ( newl > list(n) ) then
              n = n + 1
              list(n) = newl
              j = j+1
            end if
          else list_one
            n = n + 1
            list(n) = newl
            j = j+1
          end if list_one

!       If there there is only valid elements left in list2
        else if (k>0 .and. k<=n2) then lists

!         Add the current element of list2 to list if it is not already there
!         or list is empty and increment k.
          newl = list2(k)
          list_two: if ( n > 0 ) then
            if ( newl > list(n) ) then
              n = n + 1
              list(n) = newl
              k = k+1
            end if
          else list_two
            n = n + 1
            list(n) = newl
            k = k+1
          end if list_two
        end if lists

!       If both list1 and list2 have been fully traversed then exit.
        if ( j > n1 .and. k > n2 ) exit

      end do loop

    end subroutine union


!   Routine to figure out how many processors to assign to a list of super
!   regions and if necessary split super regions into regions to maintain load
!   balance across all regions. Returns a lits of potentially split super
!   regions and 2 arrays containing the minimum and maximum processor id
!   assigned to all leaf regions in the list of super region tree structures.
    subroutine SplitSuperRegions ( sregion, nprocs )
      type(ptr), dimension(:), intent(inout) :: sregion
      integer, intent(in) :: nprocs
      integer :: i, nregs, totcost, &
                 clevel, ncount, maxcost, np
      logical :: lower_is_outer, upper_is_outer
      integer :: boundarysize, minsize
      integer, dimension(1) :: mydir
      integer, dimension(:), allocatable :: cost
      integer, dimension(3) :: mycost
      integer, dimension(:,:), allocatable :: lcost
      real(wp) :: idealcost, nrprocs
      real(wp), dimension(:), allocatable :: nsplit, accumsplit, frac1, frac2
      real(wp) :: fract
      integer, dimension(:), allocatable :: accumcost, minmap, maxmap, &
                                            splitnum
      character :: msg*200

!     Set nregs to the number of super regions in the input list.
      nregs = size(sregion)

!     Allocate helper arrays. 
      allocate ( cost(nregs), accumcost(nregs+1), minmap(nregs), &
                 maxmap(nregs), nsplit(nregs), accumsplit(nregs+1), &
                 frac1(nregs), frac2(nregs), lcost(nregs,3), splitnum(nregs) )

!     Set cost to the total cost of each super region and accumcost to the
!     accumulated cost.
      accumcost(1) = 0
      do i = 1, nregs
        call cost_box ( sregion(i)%point%extent, lcost(i,:) )
        cost(i) = product ( lcost(i,:) )
        accumcost(i+1) = accumcost(i) + cost(i)
      end do

!     Calculate the total cost (of all super regions) as well as the idealcost.
!     Note that totcost is an integer, while idealcost is real.
      totcost = sum ( cost )
      idealcost = real(totcost,wp) / nprocs

!     Calculate the ideal processor location for the beginning and end of each
!     super region, the fractional number of processors to be assigned to this
!     region as well as the accumulated fractional processor count.
      accumsplit(1) = 0.0_wp
      do i = 1, nregs
        minmap(i) = floor(accumcost(i)/idealcost) 
        maxmap(i) = ceiling(accumcost(i+1)/idealcost) - 1 
        nsplit(i) = cost(i)/idealcost
        accumsplit(i+1) = accumsplit(i)+nsplit(i)
      end do

!     Calculate the size of the fractional regions at the beginning and the
!     end of each super region.
      do i = 1, nregs
        if ( maxmap(i) > minmap(i) ) then
          frac1(i) = ceiling(accumsplit(i))-accumsplit(i)
          frac2(i) = accumsplit(i+1)-floor(accumsplit(i+1))
        else
          frac1(i) = 0.0_wp
          frac2(i) = 0.0_wp
        end if
      end do
 
!     Loop over the super regions.
      do i = 1, nregs
        nrprocs = nsplit(i)
!       If the region begins and ends on the same processor, don't split
!       but just assign the region to that processor.
        if ( maxmap(i) == minmap(i) ) then
          sregion(i)%point%processor = minmap(i)
          sregion(i)%point%frac = nrprocs
!       Otherwise split the region.
        else
!         Initialize counter variables used in the recursive routine.
          clevel = 0
          ncount = 0
!         Calculate the fractional part that has to be left unsplit. This is
!         the sum of the fractional leftovers at either end.
          fract = frac1(i)+frac2(i)
!         If nrprocs > fract (fuzzy logic) then split recursively on nrprocs
!         processors with fract left unsplit (0.0<fract<2.0) and then
!         assign processors to the resulting regions.
          if ( abs ( nrprocs -  fract ) > 0.000001_wp ) then
            call SplitRegionsRecursive ( nrprocs, fract, 4, 3, &
                    clevel, sregion(i)%point, ncount )
            call AssignProcs ( minmap(i), maxmap(i), frac1(i), frac2(i), &
                               sregion(i)%point )
!         Otherwise we just need to split the superregion into 2 regions
!         of size fract1 and fract2.
          else
!           Find the cost of the region.
            call cost_box ( sregion(i)%point%extent, mycost )
!           Find the direction of maximal cost.
            maxcost = maxval(mycost)
            mydir = maxloc (mycost)
!           Find the number of gridpoints in direction mydir of the first
!           split region.
            np = nint(frac1(i)/fract*mycost(mydir(1)))
            if (limit_size) then
              lower_is_outer = &
                   sregion(i)%point%outer_boundaries%obound(mydir(1),1)/=0
              upper_is_outer = &
                   sregion(i)%point%outer_boundaries%obound(mydir(1),2)/=0
              boundarysize = ghostsize
              minsize = boundarysize+ghostsize
              if (lower_is_outer .and. np>0 .and. np<minsize) then
                np = minsize
              end if
              if (upper_is_outer .and. np<maxcost .and. np>maxcost-minsize) then
                np = maxcost-minsize
              end if
              if ((lower_is_outer .and. np>0 .and. np<minsize) .or. &
                  (upper_is_outer .and. &
                   np<maxcost .and. np>maxcost-minsize)) then
                write (msg, '("Not enough grid points: np=",i4," gs=",i4, "maxcost=",i4)') &
                       np, boundarysize, maxcost
                call CCTK_WARN(CCTK_WARN_ABORT, msg)
              end if
            end if
!           Split the region.
            call split_sregion ( sregion(i)%point, mydir(1), (/ np /) )
!           Assign the processors to the resulting regions.
            sregion(i)%point%processor = -1
            sregion(i)%point%children(1)%point%processor = minmap(i)
            sregion(i)%point%children(1)%point%frac = frac1(i)
            sregion(i)%point%children(2)%point%processor = maxmap(i)
            sregion(i)%point%children(2)%point%frac = frac2(i)
          end if
        end if
      end do

!     Deallocate local helper arrays.
      deallocate ( cost, accumcost, minmap, &
                 maxmap, nsplit, accumsplit, &
                 frac1, frac2, lcost, splitnum )
 
    end subroutine SplitSuperRegions

end module carpet_boxtypes