Task: Implement regrid.cc for Carpet/CarpetRegrid2 thorn

thorn_name: Carpet/CarpetRegrid2
src_filename: regrid.cc
interface: # Interface definition for thorn CarpetRegrid2

IMPLEMENTS: CarpetRegrid2

USES INCLUDE HEADER: Timer.hh

USES INCLUDE HEADER: bbox.hh
USES INCLUDE HEADER: bboxset.hh
USES INCLUDE HEADER: defs.hh
USES INCLUDE HEADER: dh.hh
USES INCLUDE HEADER: gh.hh
USES INCLUDE HEADER: mpi_string.hh
USES INCLUDE HEADER: region.hh
USES INCLUDE HEADER: vect.hh

USES INCLUDE HEADER: carpet.hh



# The location of the boundary points
CCTK_INT FUNCTION GetBoundarySpecification \
    (CCTK_INT IN        size,              \
     CCTK_INT OUT ARRAY nboundaryzones,    \
     CCTK_INT OUT ARRAY is_internal,       \
     CCTK_INT OUT ARRAY is_staggered,      \
     CCTK_INT OUT ARRAY shiftout)
USES FUNCTION GetBoundarySpecification

# The overall size of the domain
CCTK_INT FUNCTION GetDomainSpecification \
    (CCTK_INT  IN        size,           \
     CCTK_REAL OUT ARRAY physical_min,   \
     CCTK_REAL OUT ARRAY physical_max,   \
     CCTK_REAL OUT ARRAY interior_min,   \
     CCTK_REAL OUT ARRAY interior_max,   \
     CCTK_REAL OUT ARRAY exterior_min,   \
     CCTK_REAL OUT ARRAY exterior_max,   \
     CCTK_REAL OUT ARRAY spacing)
USES FUNCTION GetDomainSpecification

# Conversion between boundary types
CCTK_INT FUNCTION ConvertFromPhysicalBoundary \
    (CCTK_INT  IN        size,                \
     CCTK_REAL IN  ARRAY physical_min,        \
     CCTK_REAL IN  ARRAY physical_max,        \
     CCTK_REAL OUT ARRAY interior_min,        \
     CCTK_REAL OUT ARRAY interior_max,        \
     CCTK_REAL OUT ARRAY exterior_min,        \
     CCTK_REAL OUT ARRAY exterior_max,        \
     CCTK_REAL IN  ARRAY spacing)
USES FUNCTION ConvertFromPhysicalBoundary

# The location of the boundary points
CCTK_INT FUNCTION MultiPatch_GetBoundarySpecification \
    (CCTK_INT IN        map,                          \
     CCTK_INT IN        size,                         \
     CCTK_INT OUT ARRAY nboundaryzones,               \
     CCTK_INT OUT ARRAY is_internal,                  \
     CCTK_INT OUT ARRAY is_staggered,                 \
     CCTK_INT OUT ARRAY shiftout)
USES FUNCTION MultiPatch_GetBoundarySpecification

# The overall size of the domain
CCTK_INT FUNCTION MultiPatch_GetDomainSpecification \
    (CCTK_INT  IN        map,                       \
     CCTK_INT  IN        size,                      \
     CCTK_REAL OUT ARRAY physical_min,              \
     CCTK_REAL OUT ARRAY physical_max,              \
     CCTK_REAL OUT ARRAY interior_min,              \
     CCTK_REAL OUT ARRAY interior_max,              \
     CCTK_REAL OUT ARRAY exterior_min,              \
     CCTK_REAL OUT ARRAY exterior_max,              \
     CCTK_REAL OUT ARRAY spacing)
USES FUNCTION MultiPatch_GetDomainSpecification

# Conversion between boundary types
CCTK_INT FUNCTION MultiPatch_ConvertFromPhysicalBoundary \
    (CCTK_INT  IN        map,                            \
     CCTK_INT  IN        size,                           \
     CCTK_REAL IN  ARRAY physical_min,                   \
     CCTK_REAL IN  ARRAY physical_max,                   \
     CCTK_REAL OUT ARRAY interior_min,                   \
     CCTK_REAL OUT ARRAY interior_max,                   \
     CCTK_REAL OUT ARRAY exterior_min,                   \
     CCTK_REAL OUT ARRAY exterior_max,                   \
     CCTK_REAL IN  ARRAY spacing)
USES FUNCTION MultiPatch_ConvertFromPhysicalBoundary



# The true prototype of the routine below:
#   int Carpet_Regrid (cGH const * cctkGH,
#                      gh::rregs * superregss,
#                      gh::mregs * regsss,
#                      int         force);
CCTK_INT FUNCTION Carpet_Regrid           \
    (CCTK_POINTER_TO_CONST IN cctkGH,     \
     CCTK_POINTER          IN superregss, \
     CCTK_POINTER          IN regsss,     \
     CCTK_INT              IN force)
PROVIDES FUNCTION Carpet_Regrid WITH CarpetRegrid2_Regrid LANGUAGE C

# The true prototype of the routine below:
#   int Carpet_Regrid (cGH const          * cctkGH,
#                      vector <gh::rregs> * superregsss,
#                      vector <gh::mregs> * regssss,
#                      int                  force)
CCTK_INT FUNCTION Carpet_RegridMaps        \
    (CCTK_POINTER_TO_CONST IN cctkGH,      \
     CCTK_POINTER          IN superregsss, \
     CCTK_POINTER          IN regssss,     \
     CCTK_INT              IN force)
PROVIDES FUNCTION Carpet_RegridMaps WITH CarpetRegrid2_RegridMaps LANGUAGE C



CCTK_INT last_iteration "Last iteration at which there was a regridding"
CCTK_INT last_map "Last map at which there was a regridding at last_iteration"



PUBLIC:

CCTK_INT active[10] TYPE=scalar "Whether this centre is active"

CCTK_INT num_levels[10] TYPE=scalar "Number of refinement levels"

CCTK_REAL positions[10] TYPE=scalar
{
  position_x position_y position_z
} "Positions of refined regions"

CCTK_REAL radii[10] TYPE=array DIM=1 SIZE=30 DISTRIB=constant
{
  radius
} "Radii of refined regions for each level"

CCTK_REAL radiixyz[10] TYPE=array DIM=1 SIZE=30 DISTRIB=constant
{
  radius_x radius_y radius_z
} "Radii of refined regions for each level"

CCTK_REAL level_mask TYPE=gf TAGS='checkpoint="no" prolongation="none"' "Requested refinement level for this grid point"

PRIVATE:

CCTK_INT old_active[10] TYPE=scalar "Old whether this centre is active"

CCTK_INT old_num_levels[10] TYPE=scalar "Old number of refinement levels"

CCTK_REAL old_positions[10] TYPE=scalar
{
  old_position_x old_position_y old_position_z
} "Old positions of refined regions"

CCTK_REAL old_radiixyz[10] TYPE=array DIM=1 SIZE=30 DISTRIB=constant
{
  old_radius_x old_radius_y old_radius_z
} "Old radii of refined regions for each level"

schedule: # Schedule definitions for thorn CarpetRegrid2

STORAGE: last_iteration last_map
STORAGE: active num_levels positions radii radiixyz
STORAGE: old_active old_positions old_num_levels old_radiixyz
if (adaptive_refinement)
{
  STORAGE: level_mask
}



SCHEDULE CarpetRegrid2_ParamCheck AT paramcheck
{
  LANG: C
} "Check parameters"

SCHEDULE CarpetRegrid2_Initialise AT wragh
{
  LANG: C
  OPTIONS: global
} "Initialise locations of refined regions"

param: # Parameter definitions for thorn CarpetRegrid2

BOOLEAN verbose "Display regridding information on the terminal" STEERABLE=always
{
} "no"

BOOLEAN veryverbose "Display much regridding information on the terminal" STEERABLE=always
{
} "no"



CCTK_INT min_distance "Minimum distance (in grid points) between coarse and fine grid boundaries" STEERABLE=always
{
  0:* :: ""
} 4

BOOLEAN ensure_proper_nesting "Ensure proper nesting automatically" STEERABLE=always
{
} "yes"

BOOLEAN freeze_unaligned_levels "Do not change refinement levels that do not exist at this time" STEERABLE=always
{
} "no"

BOOLEAN freeze_unaligned_parent_levels "Do not change refinement levels where the parent does not exist at this time" STEERABLE=always
{
} "no"



CCTK_REAL min_fraction "Minimum fraction of required refined points that need to be present in a refined region" STEERABLE=always
{
  0:* :: ""
} 0.9

BOOLEAN snap_to_coarse "Ensure that the fine grid extent coincides with coarse grid points" STEERABLE=always
{
} "no"

CCTK_INT boundary_shiftout "Number of grid points added to the refinement boundary radius" STEERABLE=always
{
  *:* :: ""
} 0



CCTK_INT regrid_every "Regrid every n time steps" STEERABLE=always
{
  -1   :: "regrid never"
   0   :: "regrid during initial data calculation only"
   1:* :: "regrid every n time steps"
} 0



BOOLEAN symmetry_rotating90 "Ensure a 90 degree rotating symmetry about the z axis"
{
} no

BOOLEAN symmetry_rotating180 "Ensure a 180 degree rotating symmetry about the z axis"
{
} no

BOOLEAN symmetry_parity "parity "
{
} no


BOOLEAN symmetry_periodic_x "Ensure a periodicity symmetry in the x direction"
{
} no

BOOLEAN symmetry_periodic_y "Ensure a periodicity symmetry in the y direction"
{
} no

BOOLEAN symmetry_periodic_z "Ensure a periodicity symmetry in the z direction"
{
} no



BOOLEAN expect_symmetric_grids "Expect a grid structure that is symmetric about the origin, and abort if it is not"
{
} "no"



BOOLEAN adaptive_refinement "Use level_mask for adaptive refinement" STEERABLE=always
{
} "no"

CCTK_INT adaptive_block_size "Block size for adaptive refinement" STEERABLE=always
{
  1:* :: ""
} 8

CCTK_INT adaptive_block_size_x "Block size in x direction for adaptive refinement" STEERABLE=always
{
  -1 :: "use adaptive_block_size"
  1:* :: ""
} -1
CCTK_INT adaptive_block_size_y "Block size in y direction for adaptive refinement" STEERABLE=always
{
  -1 :: "use adaptive_block_size"
  1:* :: ""
} -1
CCTK_INT adaptive_block_size_z "Block size in z direction for adaptive refinement" STEERABLE=always
{
  -1 :: "use adaptive_block_size"
  1:* :: ""
} -1



CCTK_INT num_centres "Number of refinement centres"
{
  0:10 :: ""
} 0



################################################################################

BOOLEAN add_levels_automatically "Automatically add a new refinement level at each regrid" STEERABLE=always
{
} "no"



################################################################################

CCTK_INT num_levels_1 "Number of refinement levels for this centre"
{
  1:30 :: ""
} 1

BOOLEAN active_1 "Is this region active?"
{
} "yes"



CCTK_REAL position_x_1 "Position of this centre"
{
  *:* :: ""
} 0.0

CCTK_REAL position_y_1 "Position of this centre"
{
  *:* :: ""
} 0.0

CCTK_REAL position_z_1 "Position of this centre"
{
  *:* :: ""
} 0.0



CCTK_REAL radius_1[30] "Radius of refined region for this level of this centre"
{
  -1 :: "illegal"
  0:* :: ""
} -1.0

CCTK_REAL radius_x_1[30] "x-radius of refined region for this level of this centre"
{
  -1  :: "ignore this radius"
  0:* :: ""
} -1.0

CCTK_REAL radius_y_1[30] "y-radius of refined region for this level of this centre"
{
  -1  :: "ignore this radius"
  0:* :: ""
} -1.0

CCTK_REAL radius_z_1[30] "z-radius of refined region for this level of this centre"
{
  -1  :: "ignore this radius"
  0:* :: ""
} -1.0



CCTK_REAL movement_threshold_1 "Minimum movement to trigger a regridding" STEERABLE=always
{
  0:* :: ""
} 0.0

CCTK_REAL radius_rel_change_threshold_1 "Minimum RELATIVE change in radius to trigger a regridding" STEERABLE=always
{
  0.0:* :: ""
} 0.0



################################################################################

CCTK_INT num_levels_2 "Number of refinement levels for this centre"
{
  1:30 :: ""
} 1

BOOLEAN active_2 "Is this region active?"
{
} "yes"



CCTK_REAL position_x_2 "Position of this centre"
{
  *:* :: ""
} 0.0

CCTK_REAL position_y_2 "Position of this centre"
{
  *:* :: ""
} 0.0

CCTK_REAL position_z_2 "Position of this centre"
{
  *:* :: ""
} 0.0



CCTK_REAL radius_2[30] "Radius of refined region for this level of this centre"
{
  -1 :: "illegal"
  0:* :: ""
} -1.0

CCTK_REAL radius_x_2[30] "x-radius of refined region for this level of this centre"
{
  -1  :: "ignore this radius"
  0:* :: ""
} -1.0

CCTK_REAL radius_y_2[30] "y-radius of refined region for this level of this centre"
{
  -1  :: "ignore this radius"
  0:* :: ""
} -1.0

CCTK_REAL radius_z_2[30] "z-radius of refined region for this level of this centre"
{
  -1  :: "ignore this radius"
  0:* :: ""
} -1.0



CCTK_REAL movement_threshold_2 "Minimum movement to trigger a regridding" STEERABLE=always
{
  0:* :: ""
} 0.0

CCTK_REAL radius_rel_change_threshold_2 "Minimum change in radius to trigger a regridding" STEERABLE=always
{
  0.0:* :: ""
} 0.0



################################################################################

CCTK_INT num_levels_3 "Number of refinement levels for this centre"
{
  1:30 :: ""
} 1

BOOLEAN active_3 "Is this region active?"
{
} "yes"



CCTK_REAL position_x_3 "Position of this centre"
{
  *:* :: ""
} 0.0

CCTK_REAL position_y_3 "Position of this centre"
{
  *:* :: ""
} 0.0

CCTK_REAL position_z_3 "Position of this centre"
{
  *:* :: ""
} 0.0



CCTK_REAL radius_3[30] "Radius of refined region for this level of this centre"
{
  -1 :: "illegal"
  0:* :: ""
} -1.0

CCTK_REAL radius_x_3[30] "x-radius of refined region for this level of this centre"
{
  -1  :: "ignore this radius"
  0:* :: ""
} -1.0

CCTK_REAL radius_y_3[30] "y-radius of refined region for this level of this centre"
{
  -1  :: "ignore this radius"
  0:* :: ""
} -1.0

CCTK_REAL radius_z_3[30] "z-radius of refined region for this level of this centre"
{
  -1  :: "ignore this radius"
  0:* :: ""
} -1.0



CCTK_REAL movement_threshold_3 "Minimum movement to trigger a regridding" STEERABLE=always
{
  0:* :: ""
} 0.0

CCTK_REAL radius_rel_change_threshold_3 "Minimum change in radius to trigger a regridding" STEERABLE=always
{
  0.0:* :: ""
} 0.0



################################################################################

CCTK_INT num_levels_4 "Number of refinement levels for this centre"
{
  1:30 :: ""
} 1

BOOLEAN active_4 "Is this region active?"
{
} "yes"



CCTK_REAL position_x_4 "Position of this centre"
{
  *:* :: ""
} 0.0

CCTK_REAL position_y_4 "Position of this centre"
{
  *:* :: ""
} 0.0

CCTK_REAL position_z_4 "Position of this centre"
{
  *:* :: ""
} 0.0



CCTK_REAL radius_4[30] "Radius of refined region for this level of this centre"
{
  -1 :: "illegal"
  0:* :: ""
} -1.0

CCTK_REAL radius_x_4[30] "x-radius of refined region for this level of this centre"
{
  -1  :: "ignore this radius"
  0:* :: ""
} -1.0

CCTK_REAL radius_y_4[30] "y-radius of refined region for this level of this centre"
{
  -1  :: "ignore this radius"
  0:* :: ""
} -1.0

CCTK_REAL radius_z_4[30] "z-radius of refined region for this level of this centre"
{
  -1  :: "ignore this radius"
  0:* :: ""
} -1.0



CCTK_REAL movement_threshold_4 "Minimum movement to trigger a regridding" STEERABLE=always
{
  0:* :: ""
} 0.0

CCTK_REAL radius_rel_change_threshold_4 "Minimum change in radius to trigger a regridding" STEERABLE=always
{
  0.0:* :: ""
} 0.0



################################################################################

CCTK_INT num_levels_5 "Number of refinement levels for this centre"
{
  1:30 :: ""
} 1

BOOLEAN active_5 "Is this region active?"
{
} "yes"



CCTK_REAL position_x_5 "Position of this centre"
{
  *:* :: ""
} 0.0

CCTK_REAL position_y_5 "Position of this centre"
{
  *:* :: ""
} 0.0

CCTK_REAL position_z_5 "Position of this centre"
{
  *:* :: ""
} 0.0



CCTK_REAL radius_5[30] "Radius of refined region for this level of this centre"
{
  -1 :: "illegal"
  0:* :: ""
} -1.0

CCTK_REAL radius_x_5[30] "x-radius of refined region for this level of this centre"
{
  -1  :: "ignore this radius"
  0:* :: ""
} -1.0

CCTK_REAL radius_y_5[30] "y-radius of refined region for this level of this centre"
{
  -1  :: "ignore this radius"
  0:* :: ""
} -1.0

CCTK_REAL radius_z_5[30] "z-radius of refined region for this level of this centre"
{
  -1  :: "ignore this radius"
  0:* :: ""
} -1.0



CCTK_REAL movement_threshold_5 "Minimum movement to trigger a regridding" STEERABLE=always
{
  0:* :: ""
} 0.0

CCTK_REAL radius_rel_change_threshold_5 "Minimum change in radius to trigger a regridding" STEERABLE=always
{
  0.0:* :: ""
} 0.0



################################################################################

CCTK_INT num_levels_6 "Number of refinement levels for this centre"
{
  1:30 :: ""
} 1

BOOLEAN active_6 "Is this region active?"
{
} "yes"



CCTK_REAL position_x_6 "Position of this centre"
{
  *:* :: ""
} 0.0

CCTK_REAL position_y_6 "Position of this centre"
{
  *:* :: ""
} 0.0

CCTK_REAL position_z_6 "Position of this centre"
{
  *:* :: ""
} 0.0



CCTK_REAL radius_6[30] "Radius of refined region for this level of this centre"
{
  -1 :: "illegal"
  0:* :: ""
} -1.0

CCTK_REAL radius_x_6[30] "x-radius of refined region for this level of this centre"
{
  -1  :: "ignore this radius"
  0:* :: ""
} -1.0

CCTK_REAL radius_y_6[30] "y-radius of refined region for this level of this centre"
{
  -1  :: "ignore this radius"
  0:* :: ""
} -1.0

CCTK_REAL radius_z_6[30] "z-radius of refined region for this level of this centre"
{
  -1  :: "ignore this radius"
  0:* :: ""
} -1.0



CCTK_REAL movement_threshold_6 "Minimum movement to trigger a regridding" STEERABLE=always
{
  0:* :: ""
} 0.0

CCTK_REAL radius_rel_change_threshold_6 "Minimum change in radius to trigger a regridding" STEERABLE=always
{
  0.0:* :: ""
} 0.0



################################################################################

CCTK_INT num_levels_7 "Number of refinement levels for this centre"
{
  1:30 :: ""
} 1

BOOLEAN active_7 "Is this region active?"
{
} "yes"



CCTK_REAL position_x_7 "Position of this centre"
{
  *:* :: ""
} 0.0

CCTK_REAL position_y_7 "Position of this centre"
{
  *:* :: ""
} 0.0

CCTK_REAL position_z_7 "Position of this centre"
{
  *:* :: ""
} 0.0



CCTK_REAL radius_7[30] "Radius of refined region for this level of this centre"
{
  -1 :: "illegal"
  0:* :: ""
} -1.0

CCTK_REAL radius_x_7[30] "x-radius of refined region for this level of this centre"
{
  -1  :: "ignore this radius"
  0:* :: ""
} -1.0

CCTK_REAL radius_y_7[30] "y-radius of refined region for this level of this centre"
{
  -1  :: "ignore this radius"
  0:* :: ""
} -1.0

CCTK_REAL radius_z_7[30] "z-radius of refined region for this level of this centre"
{
  -1  :: "ignore this radius"
  0:* :: ""
} -1.0



CCTK_REAL movement_threshold_7 "Minimum movement to trigger a regridding" STEERABLE=always
{
  0:* :: ""
} 0.0

CCTK_REAL radius_rel_change_threshold_7 "Minimum change in radius to trigger a regridding" STEERABLE=always
{
  0.0:* :: ""
} 0.0



################################################################################

CCTK_INT num_levels_8 "Number of refinement levels for this centre"
{
  1:30 :: ""
} 1

BOOLEAN active_8 "Is this region active?"
{
} "yes"



CCTK_REAL position_x_8 "Position of this centre"
{
  *:* :: ""
} 0.0

CCTK_REAL position_y_8 "Position of this centre"
{
  *:* :: ""
} 0.0

CCTK_REAL position_z_8 "Position of this centre"
{
  *:* :: ""
} 0.0



CCTK_REAL radius_8[30] "Radius of refined region for this level of this centre"
{
  -1 :: "illegal"
  0:* :: ""
} -1.0

CCTK_REAL radius_x_8[30] "x-radius of refined region for this level of this centre"
{
  -1  :: "ignore this radius"
  0:* :: ""
} -1.0

CCTK_REAL radius_y_8[30] "y-radius of refined region for this level of this centre"
{
  -1  :: "ignore this radius"
  0:* :: ""
} -1.0

CCTK_REAL radius_z_8[30] "z-radius of refined region for this level of this centre"
{
  -1  :: "ignore this radius"
  0:* :: ""
} -1.0



CCTK_REAL movement_threshold_8 "Minimum movement to trigger a regridding" STEERABLE=always
{
  0:* :: ""
} 0.0

CCTK_REAL radius_rel_change_threshold_8 "Minimum change in radius to trigger a regridding" STEERABLE=always
{
  0.0:* :: ""
} 0.0



################################################################################

CCTK_INT num_levels_9 "Number of refinement levels for this centre"
{
  1:30 :: ""
} 1

BOOLEAN active_9 "Is this region active?"
{
} "yes"



CCTK_REAL position_x_9 "Position of this centre"
{
  *:* :: ""
} 0.0

CCTK_REAL position_y_9 "Position of this centre"
{
  *:* :: ""
} 0.0

CCTK_REAL position_z_9 "Position of this centre"
{
  *:* :: ""
} 0.0



CCTK_REAL radius_9[30] "Radius of refined region for this level of this centre"
{
  -1 :: "illegal"
  0:* :: ""
} -1.0

CCTK_REAL radius_x_9[30] "x-radius of refined region for this level of this centre"
{
  -1  :: "ignore this radius"
  0:* :: ""
} -1.0

CCTK_REAL radius_y_9[30] "y-radius of refined region for this level of this centre"
{
  -1  :: "ignore this radius"
  0:* :: ""
} -1.0

CCTK_REAL radius_z_9[30] "z-radius of refined region for this level of this centre"
{
  -1  :: "ignore this radius"
  0:* :: ""
} -1.0



CCTK_REAL movement_threshold_9 "Minimum movement to trigger a regridding" STEERABLE=always
{
  0:* :: ""
} 0.0

CCTK_REAL radius_rel_change_threshold_9 "Minimum change in radius to trigger a regridding" STEERABLE=always
{
  0.0:* :: ""
} 0.0



################################################################################

CCTK_INT num_levels_10 "Number of refinement levels for this centre"
{
  1:30 :: ""
} 1

BOOLEAN active_10 "Is this region active?"
{
} "yes"



CCTK_REAL position_x_10 "Position of this centre"
{
  *:* :: ""
} 0.0

CCTK_REAL position_y_10 "Position of this centre"
{
  *:* :: ""
} 0.0

CCTK_REAL position_z_10 "Position of this centre"
{
  *:* :: ""
} 0.0



CCTK_REAL radius_10[30] "Radius of refined region for this level of this centre"
{
  -1 :: "illegal"
  0:* :: ""
} -1.0

CCTK_REAL radius_x_10[30] "x-radius of refined region for this level of this centre"
{
  -1  :: "ignore this radius"
  0:* :: ""
} -1.0

CCTK_REAL radius_y_10[30] "y-radius of refined region for this level of this centre"
{
  -1  :: "ignore this radius"
  0:* :: ""
} -1.0

CCTK_REAL radius_z_10[30] "z-radius of refined region for this level of this centre"
{
  -1  :: "ignore this radius"
  0:* :: ""
} -1.0



CCTK_REAL movement_threshold_10 "Minimum movement to trigger a regridding" STEERABLE=always
{
  0:* :: ""
} 0.0

CCTK_REAL radius_rel_change_threshold_10 "Minimum change in radius to trigger a regridding" STEERABLE=always
{
  0.0:* :: ""
} 0.0

configuration: # Configuration definitions for thorn CarpetRegrid2

REQUIRES Carpet CarpetLib

context: amr.hh:
```
#ifndef AMR_HH
#define AMR_HH

#include <carpet.hh>

namespace CarpetRegrid2 {

void evaluate_level_mask(cGH const *restrict cctkGH, vector<ibset> &regions,
                         int rl);

} // namespace CarpetRegrid2

#endif // #ifndef AMR_HH

```

indexing.cc:
```
#include <cassert>

#include <cctk.h>

#include "indexing.hh"

namespace CarpetRegrid2 {

// Get indexing information for a vector grid array
void getvectorindex2(cGH const *const cctkGH, char const *const groupname,
                     int *const lsh) {
  assert(groupname);
  assert(lsh);

  int const gi = CCTK_GroupIndex(groupname);
  assert(gi >= 0);

  {
    int const ierr = CCTK_GrouplshGI(cctkGH, 1, lsh, gi);
    assert(not ierr);
  }

  cGroup groupdata;
  {
    int const ierr = CCTK_GroupData(gi, &groupdata);
    assert(not ierr);
  }
  assert(groupdata.vectorgroup);
  assert(groupdata.vectorlength >= 0);
  lsh[1] = groupdata.vectorlength;
}

} // namespace CarpetRegrid2

```

initialise.cc:
```
#include <cassert>

#include <cctk.h>
#include <cctk_Arguments.h>
#include <cctk_Parameters.h>

#include "indexing.hh"

namespace CarpetRegrid2 {

extern "C" {
void CarpetRegrid2_Initialise(CCTK_ARGUMENTS);
}

void CarpetRegrid2_Initialise(CCTK_ARGUMENTS) {
  DECLARE_CCTK_ARGUMENTS;
  DECLARE_CCTK_PARAMETERS;

  // Initialise meta-information
  *last_iteration = -1;
  *last_map = -1;

  // Initialise refinement information
  for (int n = 0; n < 10; ++n) {
    num_levels[n] = 0;
  }

  int lsh[2];
  getvectorindex2(cctkGH, "CarpetRegrid2::radii", lsh);

#define INIT_CENTRE(N)                                                         \
  do {                                                                         \
    if (num_centres >= N) {                                                    \
      num_levels[N - 1] = num_levels_##N;                                      \
      active[N - 1] = active_##N;                                              \
      position_x[N - 1] = position_x_##N;                                      \
      position_y[N - 1] = position_y_##N;                                      \
      position_z[N - 1] = position_z_##N;                                      \
      for (int rl = 0; rl < 30; ++rl) {                                        \
        int const ind = index2(lsh, rl, N - 1);                                \
        radius[ind] = radius_##N[rl];                                          \
        radius_x[ind] = radius_x_##N[rl];                                      \
        radius_y[ind] = radius_y_##N[rl];                                      \
        radius_z[ind] = radius_z_##N[rl];                                      \
      }                                                                        \
      old_position_x[N - 1] = position_x[N - 1];                               \
      old_position_y[N - 1] = position_y[N - 1];                               \
      old_position_z[N - 1] = position_z[N - 1];                               \
    }                                                                          \
  } while (0)

  INIT_CENTRE(1);
  INIT_CENTRE(2);
  INIT_CENTRE(3);
  INIT_CENTRE(4);
  INIT_CENTRE(5);
  INIT_CENTRE(6);
  INIT_CENTRE(7);
  INIT_CENTRE(8);
  INIT_CENTRE(9);
  INIT_CENTRE(10);

#undef INIT_CENTRE

  if (verbose or veryverbose) {
    for (int n = 0; n < num_centres; ++n) {
      CCTK_VInfo(CCTK_THORNSTRING,
                 "Initialising position of centre %d to [%g,%g,%g]", n,
                 static_cast<double>(position_x[n]),
                 static_cast<double>(position_y[n]),
                 static_cast<double>(position_z[n]));
    }
  }
}

} // namespace CarpetRegrid2

```

amr.cc:
```
#include <cctk.h>
#include <cctk_Arguments.h>
#include <cctk_Parameters.h>

#include <cmath>

#include <carpet.hh>
#include <mpi_string.hh>

#include "boundary.hh"

namespace CarpetRegrid2 {

using namespace std;
using namespace Carpet;

void evaluate_level_mask(cGH const *restrict const cctkGH,
                         vector<ibset> &regions, int const rl) {
  DECLARE_CCTK_PARAMETERS;

  assert(is_singlemap_mode());
  int const m = Carpet::map;
  assert(rl > 0);
  BEGIN_GLOBAL_MODE(cctkGH) {
    ENTER_LEVEL_MODE(cctkGH, rl - 1) {
      ENTER_SINGLEMAP_MODE(cctkGH, m, CCTK_GF) {
        if (verbose or veryverbose) {
          cout << "Refinement level " << reflevel << ":\n";
        }

        gh const &hh = *vhh.AT(Carpet::map);

        ivect const gsh = ivect::ref(cctkGH->cctk_gsh);
        ivect const nghostzones = ivect::ref(cctkGH->cctk_nghostzones);
        if (veryverbose) {
          cout << "   gsh: " << gsh << "\n"
               << "   nghostzones: " << nghostzones << "\n";
        }

        // Determine the block size
        ivect block_size = adaptive_block_size;
        if (adaptive_block_size_x > 0)
          block_size[0] = adaptive_block_size_x;
        if (adaptive_block_size_y > 0)
          block_size[1] = adaptive_block_size_y;
        if (adaptive_block_size_z > 0)
          block_size[2] = adaptive_block_size_z;
        assert(all(block_size > 0));

        // Determine the block offset, i.e. the grid point index of
        // the coordinate origin. We don't start blocks at index
        // zero, so that one can change the location of the outer
        // boundary without influencing the refinement mechanism.
        // Instead, we align the block such that one of the blocks
        // begins at the coordinate origin.
        ivect origin;
        rvect rorigin;
        {
          DECLARE_CCTK_ARGUMENTS;
          for (int d = 0; d < dim; ++d) {
            // We round up, because the origin is located between
            // grid points for cell-centred grids, and we want the
            // next upper grid point in this case. We subtract 0.25
            // just for proper rounding.
            origin[d] =
                ceil(-CCTK_ORIGIN_SPACE(d) / CCTK_DELTA_SPACE(d) - 0.25);
            rorigin[d] = CCTK_ORIGIN_SPACE(d) + origin[d] * CCTK_DELTA_SPACE(d);
          }
        }
        // block_index := ([0...gsh] + block_offset) / block_size
        // [0...gsh] = block_index * block_size - block_offset
        ivect const block_offset =
            (block_size - origin % block_size) % block_size;
        ivect const num_blocks =
            (gsh + block_offset + block_size - 1) / block_size;
        if (veryverbose) {
          cout << "   origin: " << origin << "\n"
               << "   coordinates of origin: " << rorigin << "\n"
               << "   offset: " << block_offset << "\n"
               << "   num blocks: " << num_blocks << "\n";
        }
        assert(all(num_blocks > 0));

        // Mask containing all refined block indices (in arbitrary
        // order, and potentially even duplicated). This is really a
        // set, but we need to communicate this data structure, and
        // therefore a vector is a better choice.
        vector<ivect> mask;

        // For vertex centred grids, the blocks need to overlap by
        // one grid point, so that e.g. an 8^3 block is turned into
        // a 9^3 block. This will ensure symmetry of refined regions
        // in the simulation domain.
        int const overlap = hh.refcent == vertex_centered ? 1 : 0;

        BEGIN_LOCAL_COMPONENT_LOOP(cctkGH, CCTK_GF) {
          DECLARE_CCTK_ARGUMENTS;

          if (veryverbose) {
            cout << "   component " << component << ":\n";
          }

          ivect const lbnd = ivect::ref(cctk_lbnd);
          ivect const lsh = ivect::ref(cctk_lsh);
          ivect const bboxlo(cctk_bbox[0], cctk_bbox[2], cctk_bbox[4]);
          ivect const bboxhi(cctk_bbox[1], cctk_bbox[3], cctk_bbox[5]);

          ivect const imin = 0 + either(bboxlo, 0, nghostzones);
          ivect const imax = lsh - either(bboxhi, 0, nghostzones);

          ivect const bmin =
              max(0, ((lbnd + imin + block_offset + block_size - overlap) /
                      block_size) -
                         1);
          ivect const bmax = (lbnd + imax + block_offset) / block_size;

          // Loop over all blocks
          int nblocks = 0;
          int nrefined = 0;
          for (int bk = bmin[2]; bk < bmax[2]; ++bk) {
            for (int bj = bmin[1]; bj < bmax[1]; ++bj) {
              for (int bi = bmin[0]; bi < bmax[0]; ++bi) {
                ivect const bind(bi, bj, bk);

                ivect const bimin =
                    max(imin, (bind)*block_size - block_offset - lbnd);
                ivect const bimax =
                    min(imax, (bind + 1) * block_size - block_offset - lbnd +
                                  overlap);

                bool refine = false;
                bool have_nan = false;

                // Loop over all points in this block
                for (int k = bimin[2]; k < bimax[2]; ++k) {
                  for (int j = bimin[1]; j < bimax[1]; ++j) {
                    for (int i = bimin[0]; i < bimax[0]; ++i) {
                      int const ind3d = CCTK_GFINDEX3D(cctkGH, i, j, k);
                      if (isfinite(level_mask[ind3d])) {
                        refine = refine or level_mask[ind3d] >= rl;
                      } else {
                        have_nan = true;
                      }
                    }
                  }
                }

                // Refine this block if any point in this block requires
                // refinement
                if (have_nan) {
                  cout << "      *** found nan in block " << bind << " ***\n";
                }
                if (refine) {
                  if (veryverbose) {
                    cout << "      refining block " << bind << "\n";
                  }
                  mask.push_back(bind);
                  ++nrefined;
                }
                ++nblocks;
              }
            }
          }

          if (veryverbose) {
            cout << "   refining " << nrefined << " "
                 << "of " << nblocks << " blocks on this component\n";
          }
        }
        END_LOCAL_COMPONENT_LOOP;

        // Combine this mask from all processes, and to all processes
        vector<ivect> const fullmask = allgatherv1(dist::comm(), mask);

        // Convert vector into ibset
        const ibbox &cbaseext = hh.baseextent(mglevel, rl - 1);
        const ibbox &baseext = hh.baseextent(mglevel, rl);

        ibset &region = regions.at(rl);
        for (vector<ivect>::const_iterator it = fullmask.begin();
             it != fullmask.end(); ++it) {
          ivect const &bind = *it;
          ivect const cstr = cbaseext.stride();
          ivect const clo =
              cbaseext.lower() + (bind * block_size - block_offset) * cstr;
          ivect const cup = clo + (block_size + overlap - 1) * cstr;
          ibbox const cblock(clo, cup, cstr);
          ibbox const block = cblock.expanded_for(baseext);
          region |= block;
        }

        if (verbose or veryverbose) {
          if (veryverbose) {
            cout << "   refined region is " << region << "\n"
                 << "   domain is " << baseext << "\n"
                 << "   next coarser level is " << regions.at(rl - 1) << "\n";
          }
          size_type const nrefined = region.size();
          size_type const npoints = baseext.size();
          size_type const ncoarse =
              regions.at(rl - 1).expanded_for(baseext).size();
          cout << "   refining " << (100.0 * nrefined / ncoarse)
               << "% of the next coarser level\n"
               << "   refining " << (100.0 * nrefined / npoints)
               << "% of the domain\n";
        }
      }
      LEAVE_SINGLEMAP_MODE;
    }
    LEAVE_LEVEL_MODE;
  }
  END_GLOBAL_MODE;
}

} // namespace CarpetRegrid2

```

paramcheck.cc:
```
#include <cctk.h>
#include <cctk_Arguments.h>
#include <cctk_Parameters.h>

#include <carpet.hh>

namespace CarpetRegrid2 {

using namespace Carpet;

extern "C" {
void CarpetRegrid2_ParamCheck(CCTK_ARGUMENTS);
}

void CarpetRegrid2_ParamCheck(CCTK_ARGUMENTS) {
  DECLARE_CCTK_ARGUMENTS;
  DECLARE_CCTK_PARAMETERS;

  enum sym_t { sym_unknown, sym_90, sym_180, sym_parity };

  int num_params = 0;
  sym_t params = sym_unknown;
  char const *param = "";
  if (symmetry_rotating90) {
    ++num_params;
    params = sym_90;
    param = "symmetry_rotating90";
  }
  if (symmetry_rotating180) {
    ++num_params;
    params = sym_180;
    param = "symmetry_rotating180";
  }
  if (symmetry_parity) {
    ++num_params;
    params = sym_parity;
    param = "symmetry_parity";
  }

  int num_thorns = 0;
  sym_t thorns = sym_unknown;
  char const *thorn = "";
  if (CCTK_IsThornActive("RotatingSymmetry90")) {
    ++num_thorns;
    thorns = sym_90;
    thorn = "RotatingSymmetry90";
  }
  if (CCTK_IsThornActive("RotatingSymmetry90r")) {
    ++num_thorns;
    thorns = sym_90;
    thorn = "RotatingSymmetry90r";
  }
  if (CCTK_IsThornActive("RotatingSymmetry180")) {
    ++num_thorns;
    thorns = sym_180;
    thorn = "RotatingSymmetry180";
  }
  if (CCTK_IsThornActive("ParitySymmetry")) {
    ++num_thorns;
    thorns = sym_parity;
    thorn = "ParitySymmetry";
  }

  if (num_params > 1) {
    CCTK_PARAMWARN("Too many of the symmetry parameters at least two of "
                   "symmetry_rotating90, symmetry_rotating180, and "
                   "parity_symmetry are specified.  (At most one of these can "
                   "be specified.)");
  }

  if (num_thorns > 1) {
    CCTK_PARAMWARN("Too many of the symmetry thorns RotatingSymmetry90, "
                   "RotatingSymmetry90r, RotatingSymmetry180, and "
                   "ParitySymmetry are active.  (At most one of these can be "
                   "active.)");
  }

  if (params != sym_unknown and thorns != sym_unknown and params != thorns) {
    CCTK_VParamWarn(CCTK_THORNSTRING, "The symmetry parameters and the active "
                                      "thorns do not agree.  The symmetry "
                                      "parameter \"%s\" and the active thorn "
                                      "\"%s\" cannot be used together.",
                    param, thorn);
  }

  if ((num_centres >= 1 and num_levels_1 > maxreflevels) or
      (num_centres >= 2 and num_levels_2 > maxreflevels) or
      (num_centres >= 3 and num_levels_3 > maxreflevels) or
      (num_centres >= 4 and num_levels_4 > maxreflevels) or
      (num_centres >= 5 and num_levels_5 > maxreflevels) or
      (num_centres >= 6 and num_levels_6 > maxreflevels) or
      (num_centres >= 7 and num_levels_7 > maxreflevels) or
      (num_centres >= 8 and num_levels_8 > maxreflevels) or
      (num_centres >= 9 and num_levels_9 > maxreflevels) or
      (num_centres >= 10 and num_levels_10 > maxreflevels)) {
    CCTK_PARAMWARN("The number of requested refinement levels is larger than "
                   "the maximum number of levels specified by "
                   "Carpet::max_refinement_levels");
  }
}

} // namespace CarpetRegrid2

```

indexing.hh:
```
#ifndef INDEXING_HH
#define INDEXING_HH

#include <cassert>

#include <cctk.h>

namespace CarpetRegrid2 {

// Get indexing information for a vector grid array
void getvectorindex2(cGH const *cctkGH, char const *groupname, int *lsh);

static inline int index2(int const *const lsh, int const i, int const j) {
  assert(lsh);
  assert(lsh[0] >= 0);
  assert(lsh[1] >= 0);
  assert(i >= 0 and i < lsh[0]);
  assert(j >= 0 and j < lsh[1]);
  return i + lsh[0] * j;
}

} // namespace CarpetRegrid2

#endif // #ifndef INDEXING_HH

```

property.hh:
```
#ifndef PROPERTY_HH
#define PROPERTY_HH

// Consistency properties for the grid structure

#include <vector>

#include <bboxset.hh>
#include <defs.hh>
#include <dh.hh>
#include <gh.hh>

namespace CarpetRegrid2 {

// Each property consists of a test, which returns true or false
// depending on whether the property is satisfied, and an action
// that enforces the property.
class property {
protected:
  virtual bool test_impl(gh const &hh, dh const &dd, level_boundary const &bnd,
                         vector<ibset> const &regions, int rl) = 0;
  virtual void enforce_impl(gh const &hh, dh const &dd,
                            level_boundary const &bnd, vector<ibset> &regions,
                            int rl) = 0;

public:
  virtual ~property() {}
  bool test(gh const &hh, dh const &dd, level_boundary const &bnd,
            vector<ibset> const &regions, int rl);
  void enforce(gh const &hh, dh const &dd, level_boundary const &bnd,
               vector<ibset> &regions, int rl);
};

// Ensure that this grid contains the next finer grid
class proper_nesting : public property {
  ibset enlarged_fine_grid(gh const &hh, dh const &dd,
                           level_boundary const &bnd,
                           vector<ibset> const &regions, int rl);
  bool test_impl(gh const &hh, dh const &dd, level_boundary const &bnd,
                 vector<ibset> const &regions, int rl);
  void enforce_impl(gh const &hh, dh const &dd, level_boundary const &bnd,
                    vector<ibset> &regions, int rl);
};

// Add buffer zones (do this only once)
class add_buffers : public property {
  ibset buffered_regions(gh const &hh, dh const &dd, level_boundary const &bnd,
                         vector<ibset> const &regions, int rl);
  bool test_impl(gh const &hh, dh const &dd, level_boundary const &bnd,
                 vector<ibset> const &regions, int rl);
  void enforce_impl(gh const &hh, dh const &dd, level_boundary const &bnd,
                    vector<ibset> &regions, int rl);
};

// Combine all regions into a single region, if this is worthwhile
class combine_regions : public property {
  ibbox combined_regions(gh const &hh, dh const &dd, level_boundary const &bnd,
                         vector<ibset> const &regions, int rl);
  bool test_impl(gh const &hh, dh const &dd, level_boundary const &bnd,
                 vector<ibset> const &regions, int rl);
  void enforce_impl(gh const &hh, dh const &dd, level_boundary const &bnd,
                    vector<ibset> &regions, int rl);
};

// Align the boxes with the next coarser grid
class snap_coarse : public property {
  ibset snapped_regions(gh const &hh, dh const &dd, level_boundary const &bnd,
                        vector<ibset> const &regions, int rl);
  bool test_impl(gh const &hh, dh const &dd, level_boundary const &bnd,
                 vector<ibset> const &regions, int rl);
  void enforce_impl(gh const &hh, dh const &dd, level_boundary const &bnd,
                    vector<ibset> &regions, int rl);
};

// Make the boxes rotating-90 symmetric
class rotsym90 : public property {
  ibset symmetrised_regions(gh const &hh, dh const &dd,
                            level_boundary const &bnd,
                            vector<ibset> const &regions, int rl);
  bool test_impl(gh const &hh, dh const &dd, level_boundary const &bnd,
                 vector<ibset> const &regions, int rl);
  void enforce_impl(gh const &hh, dh const &dd, level_boundary const &bnd,
                    vector<ibset> &regions, int rl);
};

// Make the boxes parity symmetric
class parsym : public property {
  ibset symmetrised_regions(gh const &hh, dh const &dd,
                            level_boundary const &bnd,
                            vector<ibset> const &regions, int rl);
  bool test_impl(gh const &hh, dh const &dd, level_boundary const &bnd,
                 vector<ibset> const &regions, int rl);
  void enforce_impl(gh const &hh, dh const &dd, level_boundary const &bnd,
                    vector<ibset> &regions, int rl);
};

// Make the boxes rotating-180 symmetric
class rotsym180 : public property {
  ibset symmetrised_regions(gh const &hh, dh const &dd,
                            level_boundary const &bnd,
                            vector<ibset> const &regions, int rl);
  bool test_impl(gh const &hh, dh const &dd, level_boundary const &bnd,
                 vector<ibset> const &regions, int rl);
  void enforce_impl(gh const &hh, dh const &dd, level_boundary const &bnd,
                    vector<ibset> &regions, int rl);
};

// Make the boxes periodic in one direction
template <int dir> class periodic : public property {
  ibset symmetrised_regions(gh const &hh, dh const &dd,
                            level_boundary const &bnd,
                            vector<ibset> const &regions, int rl);
  bool test_impl(gh const &hh, dh const &dd, level_boundary const &bnd,
                 vector<ibset> const &regions, int rl);
  void enforce_impl(gh const &hh, dh const &dd, level_boundary const &bnd,
                    vector<ibset> &regions, int rl);
};

// Clip at the outer boundary
class boundary_clip : public property {
  ibset clipped_regions(gh const &hh, dh const &dd, level_boundary const &bnd,
                        vector<ibset> const &regions, int rl);
  bool test_impl(gh const &hh, dh const &dd, level_boundary const &bnd,
                 vector<ibset> const &regions, int rl);
  void enforce_impl(gh const &hh, dh const &dd, level_boundary const &bnd,
                    vector<ibset> &regions, int rl);
};

// Ensure that this grid is contained in the domain
class in_domain : public property {
  bool test_impl(gh const &hh, dh const &dd, level_boundary const &bnd,
                 vector<ibset> const &regions, int rl);
  void enforce_impl(gh const &hh, dh const &dd, level_boundary const &bnd,
                    vector<ibset> &regions, int rl);
};

// Ensure that this grid is symmetric, if desired
class is_symmetric : public property {
  ibset symmetrised_regions(gh const &hh, dh const &dd,
                            level_boundary const &bnd,
                            vector<ibset> const &regions, int rl);
  bool test_impl(gh const &hh, dh const &dd, level_boundary const &bnd,
                 vector<ibset> const &regions, int rl);
  void enforce_impl(gh const &hh, dh const &dd, level_boundary const &bnd,
                    vector<ibset> &regions, int rl);
};

} // namespace CarpetRegrid2

#endif // #ifndef PROPERTY_HH

```

make.code.defn:
```
# Main make.code.defn file for thorn CarpetRegrid2

# Source files in this directory
SRCS =	amr.cc					\
	boundary.cc				\
	indexing.cc				\
	initialise.cc				\
	paramcheck.cc				\
	property.cc				\
	regrid.cc

# Subdirectories containing source files
SUBDIRS = 

```

boundary.hh:
```
#ifndef BOUNDARY_HH
#define BOUNDARY_HH

#include <defs.hh>
#include <gh.hh>
#include <vect.hh>

#include <ostream>

namespace CarpetRegrid2 {

// Convert a coordinate location to an index location.  For cell
// centring, shift upwards.
ivect rpos2ipos(rvect const &rpos, rvect const &origin, rvect const &scale,
                gh const &hh, int const rl);

// Convert a coordinate location to an index location, rounding in
// the opposite manner as rpos2ipos.  For cell centring, shift
// downwards instead of upwards.
ivect rpos2ipos1(rvect const &rpos, rvect const &origin, rvect const &scale,
                 gh const &hh, int const rl);

// Convert an index location to a coordinate location
rvect ipos2rpos(ivect const &ipos, rvect const &origin, rvect const &scale,
                gh const &hh, int const rl);

// Convert an index bbox to a coordinate bbox
rbbox ibbox2rbbox(ibbox const &ib, rvect const &origin, rvect const &scale,
                  gh const &hh, int const rl);

// Snap (enlarge) a bbox to the next coarser level, if desired
ibbox snap_ibbox(ibbox const &ib, gh const &hh, int const rl);

void get_boundary_specification(jjvect &nboundaryzones, jjvect &is_internal,
                                jjvect &is_staggered, jjvect &shiftout);

void get_physical_boundary(rvect &physical_lower, rvect &physical_upper,
                           rvect &spacing);

void calculate_exterior_boundary(rvect const &physical_lower,
                                 rvect const &physical_upper,
                                 rvect &exterior_lower, rvect &exterior_upper,
                                 rvect const &spacing);

// Location and description of the outer boundary
struct domain_boundary {
  jjvect nboundaryzones, is_internal;
  jjvect is_staggered, shiftout;

  b2vect boundary_staggering_mismatch;

  rvect physical_lower, physical_upper;
  rvect spacing;

  rvect exterior_lower, exterior_upper;

  // The physical boundary
  rvect origin;
  rvect scale;

  // The location of the outermost grid points. For cell centring,
  // these are 1/2 grid spacing inside of the boundary.
  ivect physical_ilower, physical_iupper;

  domain_boundary(gh const &hh, dh const &dd);
};

struct level_boundary : public domain_boundary {
  rvect level_physical_lower;
  rvect level_physical_upper;
  rvect level_spacing;

  rvect level_exterior_lower, level_exterior_upper;

  // The location of the outermost grid points. For cell centring,
  // these are 1/2 grid spacing inside of the boundary.
  ivect level_physical_ilower;
  ivect level_physical_iupper;

  ivect level_exterior_ilower;
  ivect level_exterior_iupper;

  // The minimum necessary distance away from the outer boundary due
  // to buffer and ghost zones. This is e.g. the distance that the
  // lower boundary of a bbox has to have from the lower boundary.
  // This is in terms of grid points.
  i2vect min_bnd_dist_away;
  // The minimum necessary distance from the outer boundary due to
  // buffer and ghost zones. This is e.g. the distance that the
  // upper boundary of a bbox has to have from the lower boundary.
  // This is in terms of grid points.
  i2vect min_bnd_dist_incl;

  level_boundary(gh const &hh, dh const &dd, int rl);
};

} // namespace CarpetRegrid2

ostream &operator<<(ostream &os, CarpetRegrid2::domain_boundary const &bnd);
ostream &operator<<(ostream &os, CarpetRegrid2::level_boundary const &bnd);

#endif // #ifndef BOUNDARY_HH

```

boundary.cc:
```
#include <cctk.h>
#include <cctk_Parameters.h>

#include <carpet.hh>

#include "boundary.hh"

namespace CarpetRegrid2 {

using namespace Carpet;

// Convert a coordinate location to an index location. For cell
// centring, shift upwards.
ivect rpos2ipos(rvect const &rpos, rvect const &origin, rvect const &scale,
                gh const &hh, int const rl) {
  ivect const istride = hh.baseextent(0, rl).stride();
  ivect const bistride = hh.baseextent(0, 0).stride();

  if (hh.refcent == cell_centered) {
    assert(all(istride % 2 == 0));
  }

  CCTK_REAL const eps = 1.0e-10; // prevent rounding errors

#if 1
  ivect const ipos =
      hh.refcent == vertex_centered
          ? ivect(floor(((rpos - origin) * scale) / rvect(istride) +
                        rvect(0.5) + rvect(eps))) *
                istride
          : ivect(floor(((rpos - origin) * scale - rvect(bistride / 2)) /
                            rvect(istride) +
                        rvect(eps))) *
                    istride +
                istride / 2 + bistride / 2;
#else
  ivect const ipos =
      hh.refcent == vertex_centered
          ? ivect(floor(((rpos - origin) * scale) / rvect(istride) +
                        rvect(0.5))) *
                istride
          : ivect(floor(((rpos - origin) * scale - rvect(bistride / 2)) /
                            rvect(istride) +
                        rvect(0.5))) *
                    istride +
                istride / 2 + bistride / 2;
#endif

  return ipos;
}

// Convert a coordinate location to an index location, rounding in
// the opposite manner as rpos2ipos. For cell centring, shift
// downwards instead of upwards.
ivect rpos2ipos1(rvect const &rpos, rvect const &origin, rvect const &scale,
                 gh const &hh, int const rl) {
  ivect const istride = hh.baseextent(0, rl).stride();
  ivect const bistride = hh.baseextent(0, 0).stride();

  if (hh.refcent == cell_centered) {
    assert(all(istride % 2 == 0));
  }

  CCTK_REAL const eps = 1.0e-10; // prevent rounding errors

#if 1
  ivect const ipos =
      hh.refcent == vertex_centered
          ? ivect(ceil(((rpos - origin) * scale) / rvect(istride) - rvect(0.5) -
                       rvect(eps))) *
                istride
          : ivect(ceil(((rpos - origin) * scale - rvect(bistride / 2)) /
                           rvect(istride) -
                       rvect(eps))) *
                    istride -
                istride / 2 + bistride / 2;
#else
  ivect const ipos =
      hh.refcent == vertex_centered
          ? ivect(
                ceil(((rpos - origin) * scale) / rvect(istride) - rvect(0.5))) *
                istride
          : ivect(ceil(((rpos - origin) * scale - rvect(bistride / 2)) /
                           rvect(istride) -
                       rvect(0.5))) *
                    istride -
                istride / 2 + bistride / 2;
#endif

  return ipos;
}

// Convert an index location to a coordinate location
rvect ipos2rpos(ivect const &ipos, rvect const &origin, rvect const &scale,
                gh const &hh, int const rl) {
  return rvect(ipos) / scale + origin;
}

// Convert an index bbox to a coordinate bbox
rbbox ibbox2rbbox(ibbox const &ib, rvect const &origin, rvect const &scale,
                  gh const &hh, int const rl) {
  rvect const zero(0);
  return rbbox(ipos2rpos(ib.lower(), origin, scale, hh, rl),
               ipos2rpos(ib.upper(), origin, scale, hh, rl),
               ipos2rpos(ib.stride(), zero, scale, hh, rl));
}

void get_boundary_specification(jjvect &nboundaryzones, jjvect &is_internal,
                                jjvect &is_staggered, jjvect &shiftout) {
  if (CCTK_IsFunctionAliased("MultiPatch_GetBoundarySpecification")) {
    assert(Carpet::map >= 0);
    CCTK_INT const ierr = MultiPatch_GetBoundarySpecification(
        Carpet::map, 2 * dim, &nboundaryzones[0][0], &is_internal[0][0],
        &is_staggered[0][0], &shiftout[0][0]);
    assert(not ierr);
  } else if (CCTK_IsFunctionAliased("GetBoundarySpecification")) {
    CCTK_INT const ierr = GetBoundarySpecification(
        2 * dim, &nboundaryzones[0][0], &is_internal[0][0], &is_staggered[0][0],
        &shiftout[0][0]);
    assert(not ierr);
  } else {
    assert(0);
  }
}

void get_physical_boundary(rvect &physical_lower, rvect &physical_upper,
                           rvect &spacing) {
  rvect interior_lower, interior_upper;
  rvect exterior_lower, exterior_upper;
  if (CCTK_IsFunctionAliased("MultiPatch_GetDomainSpecification")) {
    assert(Carpet::map >= 0);
    CCTK_INT const ierr = MultiPatch_GetDomainSpecification(
        Carpet::map, dim, &physical_lower[0], &physical_upper[0],
        &interior_lower[0], &interior_upper[0], &exterior_lower[0],
        &exterior_upper[0], &spacing[0]);
    assert(not ierr);
  } else if (CCTK_IsFunctionAliased("GetDomainSpecification")) {
    CCTK_INT const ierr = GetDomainSpecification(
        dim, &physical_lower[0], &physical_upper[0], &interior_lower[0],
        &interior_upper[0], &exterior_lower[0], &exterior_upper[0],
        &spacing[0]);
    assert(not ierr);
  } else {
    assert(0);
  }
}

void calculate_exterior_boundary(rvect const &physical_lower,
                                 rvect const &physical_upper,
                                 rvect &exterior_lower, rvect &exterior_upper,
                                 rvect const &spacing) {
  rvect interior_lower, interior_upper;
  if (CCTK_IsFunctionAliased("MultiPatch_ConvertFromPhysicalBoundary")) {
    assert(Carpet::map >= 0);
    CCTK_INT const ierr = MultiPatch_ConvertFromPhysicalBoundary(
        Carpet::map, dim, &physical_lower[0], &physical_upper[0],
        &interior_lower[0], &interior_upper[0], &exterior_lower[0],
        &exterior_upper[0], &spacing[0]);
    assert(not ierr);
  } else if (CCTK_IsFunctionAliased("ConvertFromPhysicalBoundary")) {
    CCTK_INT const ierr = ConvertFromPhysicalBoundary(
        dim, &physical_lower[0], &physical_upper[0], &interior_lower[0],
        &interior_upper[0], &exterior_lower[0], &exterior_upper[0],
        &spacing[0]);
    assert(not ierr);
  } else {
    assert(0);
  }
}

// Location and description of the outer boundary
domain_boundary::domain_boundary(gh const &hh, dh const &dd) {
  DECLARE_CCTK_PARAMETERS;

  if (veryverbose) {
    cout << "Determining domain outer boundary...\n";
  }

  get_boundary_specification(nboundaryzones, is_internal, is_staggered,
                             shiftout);

  boundary_staggering_mismatch =
      xpose((hh.refcent == vertex_centered) != (iivect(is_staggered) == 0));
  // TODO: This is too strict
  assert(all(all(not boundary_staggering_mismatch)));

  get_physical_boundary(physical_lower, physical_upper, spacing);

  // Adapt spacing for convergence level
  spacing *= ipow(CCTK_REAL(mgfact), basemglevel);

  calculate_exterior_boundary(physical_lower, physical_upper, exterior_lower,
                              exterior_upper, spacing);

  // The physical boundary
  origin = exterior_lower;
  scale = rvect(hh.baseextent(0, 0).stride()) / spacing;

  // The location of the outermost grid points. For cell centring,
  // these are 1/2 grid spacing inside of the boundary.
  physical_ilower = rpos2ipos(physical_lower, origin, scale, hh, 0);
  physical_iupper = rpos2ipos1(physical_upper, origin, scale, hh, 0);
}

level_boundary::level_boundary(gh const &hh, dh const &dd, int const rl)
    : domain_boundary(hh, dd) {
  DECLARE_CCTK_PARAMETERS;

  if (veryverbose) {
    cout << "Refinement level " << rl << ": determining outer boundary...\n";
  }

  level_physical_lower = physical_lower;
  level_physical_upper = physical_upper;
  level_spacing = spacing / rvect(hh.reffacts.at(rl));
  if (veryverbose) {
    cout << "Refinement level " << rl << ": physical coordinate boundary is at "
         << r2vect(level_physical_lower, level_physical_upper) << "\n";
    cout << "Refinement level " << rl << ": spacing is " << level_spacing
         << "\n";
  }

  calculate_exterior_boundary(level_physical_lower, level_physical_upper,
                              level_exterior_lower, level_exterior_upper,
                              level_spacing);
  if (veryverbose) {
    cout << "Refinement level " << rl << ": exterior coordinate boundary is at "
         << r2vect(level_exterior_lower, level_exterior_upper) << "\n";
  }

  ibbox const &baseextent = hh.baseextent(0, rl);
  ivect const istride = baseextent.stride();
  if (veryverbose) {
    cout << "Refinement level " << rl << ": stride is " << istride << "\n";
  }

  // This is the location of the outermost grid points. For cell
  // centring, these are 1/2 grid spacing inside of the boundary.
  level_physical_ilower = rpos2ipos(physical_lower, origin, scale, hh, rl);
  level_physical_iupper = rpos2ipos1(physical_upper, origin, scale, hh, rl);
  if (veryverbose) {
    cout << "Refinement level " << rl << ": physical boundary is at "
         << i2vect(level_physical_ilower, level_physical_iupper) << "\n";
    cout << "Refinement level " << rl
         << ": reconstructed physical coordinate boundary is at "
         << r2vect(
                ipos2rpos(level_physical_ilower -
                              (hh.refcent == cell_centered ? istride / 2 : 0),
                          origin, scale, hh, rl),
                ipos2rpos(level_physical_iupper +
                              (hh.refcent == cell_centered ? istride / 2 : 0),
                          origin, scale, hh, rl))
         << "\n";
  }

  level_exterior_ilower =
      rpos2ipos(level_exterior_lower, origin, scale, hh, rl);
  level_exterior_iupper =
      rpos2ipos1(level_exterior_upper, origin, scale, hh, rl);
  assert(all(level_exterior_ilower >= baseextent.lower()));
  assert(all(level_exterior_iupper <= baseextent.upper()));
  if (veryverbose) {
    cout << "Refinement level " << rl << ": exterior boundary is at "
         << i2vect(level_exterior_ilower, level_exterior_iupper) << "\n";
    cout << "Refinement level " << rl
         << ": reconstructed exterior coordinate boundary is at "
         << r2vect(ipos2rpos(level_exterior_ilower, origin, scale, hh, rl),
                   ipos2rpos(level_exterior_iupper, origin, scale, hh, rl))
         << "\n";
  }

  // Find the minimum necessary distance away from the outer
  // boundary due to buffer and ghost zones. This is e.g. the
  // distance that the lower boundary of a bbox has to have from the
  // lower boundary. This is in terms of grid points.
  min_bnd_dist_away = dd.ghost_widths.at(rl);
  // Find the minimum necessary distance from the outer boundary due
  // to buffer and ghost zones. This is e.g. the distance that the
  // upper boundary of a bbox has to have from the lower boundary.
  // This is in terms of grid points.
  min_bnd_dist_incl = dd.ghost_widths.at(rl);
  // TODO: The above is required only near symmetry boundaries.
}

} // namespace CarpetRegrid2

ostream &operator<<(ostream &os, CarpetRegrid2::domain_boundary const &bnd) {
  return os << "domain_boundary:{"
            << "nboundaryzones=" << bnd.nboundaryzones << ","
            << "is_internal=" << bnd.is_internal << ","
            << "is_staggered=" << bnd.is_staggered << ","
            << "shiftout=" << bnd.shiftout << ","
            << "boundary_staggering_mismatch="
            << bnd.boundary_staggering_mismatch << ","
            << "physical_lower=" << bnd.physical_lower << ","
            << "physical_upper=" << bnd.physical_upper << ","
            << "spacing=" << bnd.spacing << ","
            << "exterior_lower=" << bnd.exterior_lower << ","
            << "exterior_upper=" << bnd.exterior_upper << ","
            << "origin=" << bnd.origin << ","
            << "scale=" << bnd.scale << ","
            << "physical_ilower=" << bnd.physical_ilower << ","
            << "physical_iupper=" << bnd.physical_iupper << "}";
}

ostream &operator<<(ostream &os, CarpetRegrid2::level_boundary const &bnd) {
  return os << "level_boundary:{"
            << *static_cast<CarpetRegrid2::domain_boundary const *>(&bnd) << ","
            << "level_physical_lower=" << bnd.level_physical_lower << ","
            << "level_physical_upper=" << bnd.level_physical_upper << ","
            << "level_spacing=" << bnd.level_spacing << ","
            << "level_exterior_lower=" << bnd.level_exterior_lower << ","
            << "level_exterior_upper=" << bnd.level_exterior_upper << ","
            << "level_physical_ilower=" << bnd.level_physical_ilower << ","
            << "level_physical_iupper=" << bnd.level_physical_iupper << ","
            << "level_exterior_ilower=" << bnd.level_exterior_ilower << ","
            << "level_exterior_iupper=" << bnd.level_exterior_iupper << ","
            << "min_bnd_dist_away=" << bnd.min_bnd_dist_away << ","
            << "min_bnd_dist_incl=" << bnd.min_bnd_dist_incl << "}";
}

```

property.cc:
```
#include <cctk.h>
#include <cctk_Parameters.h>

#include <carpet.hh>

#include "boundary.hh"
#include "property.hh"

#include <typeinfo>

// Consistency properties for the grid structure

namespace CarpetRegrid2 {

using namespace std;
using namespace Carpet;

// Each property consists of a test, which returns true or false
// depending on whether the property is satisfied, and an action
// that enforces the property.

bool property::test(gh const &hh, dh const &dd, level_boundary const &bnd,
                    vector<ibset> const &regions, int const rl) {
  assert(rl >= 0 and rl < int(regions.size()));
  return test_impl(hh, dd, bnd, regions, rl);
}

void property::enforce(gh const &hh, dh const &dd, level_boundary const &bnd,
                       vector<ibset> &regions, int const rl) {
  DECLARE_CCTK_PARAMETERS;
  enforce_impl(hh, dd, bnd, regions, rl);
  if (not test(hh, dd, bnd, regions, rl)) {
    cout << "Property " << typeid(*this).name() << "\n";
    CCTK_WARN(CCTK_WARN_ABORT, "Property does not hold after being enforced");
  }
}

//////////////////////////////////////////////////////////////////////////////
// Ensure that this grid contains the next finer grid
//////////////////////////////////////////////////////////////////////////////

ibset proper_nesting::enlarged_fine_grid(gh const &hh, dh const &dd,
                                         level_boundary const &bnd,
                                         vector<ibset> const &regions,
                                         int const rl) {
  DECLARE_CCTK_PARAMETERS;

  assert(rl + 1 < int(regions.size()));

  // The minimum amount of space required between the boundaries of
  // this and the next finer grid. We need a certain amount of space
  // on the coarse and a certain amount on the fine grid.
  i2vect const fdistance = dd.ghost_widths.at(rl + 1);
  i2vect const cdistance =
      i2vect(min_distance + dd.prolongation_stencil_size(rl));

  ibset enlarged;

  // Loop over all bboxes that make up the next finer level
  for (ibset::const_iterator ibb = regions.at(rl + 1).begin();
       ibb != regions.at(rl + 1).end(); ++ibb) {
    ibbox const &fbb = *ibb;

    // Find out which faces are on a boundary
    bvect const lower_is_outer = fbb.lower() <= bnd.level_physical_ilower;
    bvect const upper_is_outer = fbb.upper() >= bnd.level_physical_iupper;
    b2vect const ob(lower_is_outer, upper_is_outer);

    ibbox const domext = hh.baseextent(0, rl);

    // Enlarge the bbox, first on the fine grid, then transfer it to
    // the coarse grid, then enlarge it again
    ibbox const ebb = fbb.expand(i2vect(not ob) * fdistance);
    ibbox const cbb = ebb.expanded_for(domext);
    ibbox const ecbb = cbb.expand(i2vect(not ob) * cdistance);

    // Add it
    enlarged |= ecbb;
  }

  return enlarged;
}

bool proper_nesting::test_impl(gh const &hh, const dh &dd,
                               level_boundary const &bnd,
                               vector<ibset> const &regions, int const rl) {
  // This should not be tested because it has to be applied
  // unconditionally and only once
  return true;
}

void proper_nesting::enforce_impl(gh const &hh, dh const &dd,
                                  level_boundary const &bnd,
                                  vector<ibset> &regions, int const rl) {
  DECLARE_CCTK_PARAMETERS;

  if (not ensure_proper_nesting)
    return;
  if (rl == int(regions.size()) - 1)
    return;

  if (veryverbose) {
    cout << "Refinement level " << rl << ": ensuring proper nesting...\n";
  }

  // Enlarge the level
  regions.AT(rl) |= enlarged_fine_grid(hh, dd, bnd, regions, rl);

  if (veryverbose) {
    cout << "   New regions are " << regions.at(rl) << "\n";
  }
}

//////////////////////////////////////////////////////////////////////////////
// Add buffer zones (do this only once)
//////////////////////////////////////////////////////////////////////////////

ibset add_buffers::buffered_regions(gh const &hh, dh const &dd,
                                    level_boundary const &bnd,
                                    vector<ibset> const &regions,
                                    int const rl) {
  return regions.at(rl)
      .expand(dd.buffer_widths.at(rl) + dd.overlap_widths.at(rl));
}

bool add_buffers::test_impl(gh const &hh, dh const &dd,
                            level_boundary const &bnd,
                            vector<ibset> const &regions, int const rl) {
  // This should not be tested because it has to be applied
  // unconditionally and only once
  return true;
}

void add_buffers::enforce_impl(gh const &hh, dh const &dd,
                               level_boundary const &bnd,
                               vector<ibset> &regions, int const rl) {
  DECLARE_CCTK_PARAMETERS;

  if (veryverbose) {
    cout << "Refinement level " << rl << ": adding buffer zones...\n";
  }

  regions.at(rl) = buffered_regions(hh, dd, bnd, regions, rl);

  if (veryverbose) {
    cout << "   New regions are " << regions.at(rl) << "\n";
  }
}

//////////////////////////////////////////////////////////////////////////////
// Combine all regions into a single region, if this is worthwhile
//////////////////////////////////////////////////////////////////////////////

ibbox combine_regions::combined_regions(gh const &hh, dh const &dd,
                                        level_boundary const &bnd,
                                        vector<ibset> const &regions,
                                        int const rl) {
  return regions.at(rl).container();
}

bool combine_regions::test_impl(gh const &hh, dh const &dd,
                                level_boundary const &bnd,
                                vector<ibset> const &regions, int const rl) {
  DECLARE_CCTK_PARAMETERS;

  if (min_fraction == 1.0)
    return true;

  ibbox const combined = combined_regions(hh, dd, bnd, regions, rl);

  CCTK_REAL const regions_size = static_cast<CCTK_REAL>(regions.at(rl).size());
  CCTK_REAL const combined_size = static_cast<CCTK_REAL>(combined.size());

  // Is the current setup "simple enough"? (It is "simple enough" if
  // either it already consists of a single box, or if using a
  // single bbox would be too inefficient.)
  // TODO: Check this also for pairs of regions
  return (regions.at(rl).setsize() == 1 or
          min_fraction * combined_size > regions_size);
}

void combine_regions::enforce_impl(gh const &hh, dh const &dd,
                                   level_boundary const &bnd,
                                   vector<ibset> &regions, int const rl) {
  DECLARE_CCTK_PARAMETERS;

  if (veryverbose) {
    cout << "Refinement level " << rl << ": combining regions...\n";
  }

  regions.at(rl) = combined_regions(hh, dd, bnd, regions, rl);

  if (veryverbose) {
    cout << "   New regions are " << regions.at(rl) << "\n";
  }
}

//////////////////////////////////////////////////////////////////////////////
// Align the boxes with the next coarser grid
//////////////////////////////////////////////////////////////////////////////

ibset snap_coarse::snapped_regions(gh const &hh, dh const &dd,
                                   level_boundary const &bnd,
                                   vector<ibset> const &regions, int const rl) {
  assert(rl > 0);

  ibbox const &base = hh.baseextent(0, rl);
  ibbox const &cbase = hh.baseextent(0, rl - 1);
  assert(all(cbase.stride() % base.stride() == 0));
  ivect const reffact = cbase.stride() / base.stride();
  i2vect const &buffers = dd.buffer_widths.at(rl);

  ibset snapped;

  for (ibset::const_iterator ibb = regions.at(rl).begin();
       ibb != regions.at(rl).end(); ++ibb) {
    ibbox const &bb = *ibb;

    // We want to align the current level (without its buffer zones)
    // with the next coarser level. Conceptually, we therefore
    // subtract the buffer zones, align, and add the buffer zones
    // again.

    // In detail, we first expand by reffact + reffact-1 - N points,
    // then contract to the next coarser grid, then expand back to
    // the current grid, and expand by N - reffact points again.
    // This sequence is correct for both vertex and cell centred
    // grids, and N is determined by the number of buffer zones.

    // N is the number of buffer zones modulo the refinement factor.
    // We cannot shrink the domain (since we cannot shrink
    // bboxsets). For alignment, only operations modulo the
    // refinement factor are relevant.

    // The additional constant reffact is necessary to ensure that
    // the coarsened boxes, which may be smaller than the fine
    // boxes, cannot become empty.

    ibbox aa = bb;
    assert(not aa.empty());
    aa = aa.expand(reffact + reffact - 1 - buffers % reffact);
    assert(not aa.empty());
    aa = aa.contracted_for(cbase);
    assert(not aa.empty());
    aa = aa.expanded_for(base);
    assert(not aa.empty());
    aa = aa.expand(buffers % reffact - reffact);
    assert(not aa.empty());

    snapped |= aa;
  }

  // We don't want to remove any points
  ibset const &original = regions.at(rl);
  // return snapped | original;
  if (not(snapped >= original)) {
    cout << "snapped=" << snapped << "\n"
         << "original=" << original << "\n"
         << "original-snapped=" << (original - snapped) << "\n";
  }
  assert(snapped >= original);

  return snapped;
}

bool snap_coarse::test_impl(gh const &hh, dh const &dd,
                            level_boundary const &bnd,
                            vector<ibset> const &regions, int const rl) {
  DECLARE_CCTK_PARAMETERS;

  if (not snap_to_coarse)
    return true;

  ibset const snapped = snapped_regions(hh, dd, bnd, regions, rl);

  // We cannot test for equality, since the difference may be
  // outside of the domain (and hence irrelevant)
  // return regions.AT(rl) == snapped;

  // Test whether any part of the difference (i.e. that part of the
  // level that would be added by snapping) is inside the domain. If
  // the difference is outside, we can safely ignore it.
  ibbox const &baseextent = hh.baseextent(0, rl);
  ibset const difference = snapped - regions.AT(rl);
  return (difference & baseextent).empty();
}

void snap_coarse::enforce_impl(gh const &hh, dh const &dd,
                               level_boundary const &bnd,
                               vector<ibset> &regions, int const rl) {
  DECLARE_CCTK_PARAMETERS;

  assert(snap_to_coarse);

  if (veryverbose) {
    cout << "Refinement level " << rl
         << ": aligning regions with next coarser grid...\n";
  }

  regions.AT(rl) = snapped_regions(hh, dd, bnd, regions, rl);

  if (veryverbose) {
    cout << "   New regions are " << regions.at(rl) << "\n";
  }
}

//////////////////////////////////////////////////////////////////////////////
// Make the boxes rotating-90 symmetric
//////////////////////////////////////////////////////////////////////////////

ibset rotsym90::symmetrised_regions(gh const &hh, dh const &dd,
                                    level_boundary const &bnd,
                                    vector<ibset> const &regions,
                                    int const rl) {
  // ibbox const& baseextent = hh.baseextent(0,rl);

  ibset symmetrised = regions.at(rl);
  for (ibset::const_iterator ibb = regions.at(rl).begin();
       ibb != regions.at(rl).end(); ++ibb) {
    ibbox const &bb = *ibb;

    bvect const lower_is_outside_lower =
        bb.lower() - bnd.min_bnd_dist_away[0] * bb.stride() <=
        bnd.level_physical_ilower;

    // Treat both x and y directions
    for (int dir = 0; dir <= 1; ++dir) {
      if (lower_is_outside_lower[dir]) {
        ivect const ilo = bb.lower();
        ivect const iup = bb.upper();
        ivect const istr = bb.stride();

        // Origin
        rvect const axis(bnd.physical_lower[0], bnd.physical_lower[1],
                         bnd.physical_lower[2]); // z component is unused
        ivect const iaxis0 = rpos2ipos(axis, bnd.origin, bnd.scale, hh, rl);
        assert(all(iaxis0 % istr == 0));
        ivect const iaxis1 = rpos2ipos1(axis, bnd.origin, bnd.scale, hh, rl);
        assert(all(iaxis1 % istr == 0));
        ivect const offset = iaxis1 - iaxis0;
        assert(all(offset % istr == 0));
        assert(all(offset >= 0 and offset < 2 * istr));
        assert(all((iaxis0 + iaxis1 - offset) % (2 * istr) == 0));
        ivect const iaxis = (iaxis0 + iaxis1 - offset) / 2;
        // negated (reflected) domain boundaries
        ivect const neg_ilo = (2 * iaxis + offset) - ilo;
        ivect const neg_iup = (2 * iaxis + offset) - iup;
        // offset to add when permuting directions
        ivect const permute01(-iaxis[0] + iaxis[1], -iaxis[1] + iaxis[0], 0);

        // Rotate 90 degrees about z axis
        ivect new_ilo, new_iup;
        if (dir == 0) {
          // rotate clockwise
          new_ilo = ivect(ilo[1], neg_iup[0], ilo[2]) + permute01;
          new_iup = ivect(iup[1], neg_ilo[0], iup[2]) + permute01;
        }
        if (dir == 1) {
          // rotate counterclockwise
          new_ilo = ivect(neg_iup[1], ilo[0], ilo[2]) + permute01;
          new_iup = ivect(neg_ilo[1], iup[0], iup[2]) + permute01;
        }
        ivect const new_istr(istr);

        ibbox const new_bb(new_ilo, new_iup, new_istr);
        // Will be clipped later
        // assert (new_bb.is_contained_in (baseextent));

        // symmetrised |= new_bb & baseextent;
        symmetrised |= new_bb;
      }
    }
  }

  return symmetrised;
}

bool rotsym90::test_impl(gh const &hh, dh const &dd, level_boundary const &bnd,
                         vector<ibset> const &regions, int const rl) {
  DECLARE_CCTK_PARAMETERS;

  if (not symmetry_rotating90)
    return true;

  ibset const symmetrised = symmetrised_regions(hh, dd, bnd, regions, rl);

  // We cannot test for equality, since the difference may be
  // outside of the domain (and hence irrelevant)
  // return regions.AT(rl) == symmetrised;

  // Test whether any part of the difference (i.e. that part of the
  // level that would be added by symmetrising) is inside the
  // domain. If the difference is outside, we can safely ignore it.
  ibbox const &baseextent = hh.baseextent(0, rl);
  ibset const difference = symmetrised - regions.AT(rl);
  return (difference & baseextent).empty();
}

void rotsym90::enforce_impl(gh const &hh, dh const &dd,
                            level_boundary const &bnd, vector<ibset> &regions,
                            int const rl) {
  DECLARE_CCTK_PARAMETERS;

  assert(symmetry_rotating90);

  if (veryverbose) {
    cout << "Refinement level " << rl
         << ": making regions rotating-90 symmetric...\n";
  }

  regions.AT(rl) = symmetrised_regions(hh, dd, bnd, regions, rl);

  if (veryverbose) {
    cout << "   New regions are " << regions.at(rl) << "\n";
  }
}

//////////////////////////////////////////////////////////////////////////////
// Make the boxes rotating-180 symmetric
//////////////////////////////////////////////////////////////////////////////

ibset rotsym180::symmetrised_regions(gh const &hh, dh const &dd,
                                     level_boundary const &bnd,
                                     vector<ibset> const &regions,
                                     int const rl) {
  ibbox const &baseextent = hh.baseextent(0, rl);

  ibset symmetrised = regions.at(rl);
  for (ibset::const_iterator ibb = regions.at(rl).begin();
       ibb != regions.at(rl).end(); ++ibb) {
    ibbox const &bb = *ibb;

    bvect const lower_is_outside_lower =
        bb.lower() - bnd.min_bnd_dist_away[0] * bb.stride() <=
        bnd.level_physical_ilower;

    // Treat x direction
    int const dir = 0;
    if (lower_is_outside_lower[dir]) {
      ivect const ilo = bb.lower();
      ivect const iup = bb.upper();
      ivect const istr = bb.stride();
      assert(istr[0] == istr[1]);

      // Origin
      assert(hh.refcent == vertex_centered or all(istr % 2 == 0));
      rvect const axis(bnd.physical_lower[0],
                       (bnd.physical_lower[1] + bnd.physical_upper[1]) / 2,
                       bnd.physical_lower[2]); // z component is unused
      ivect const iaxis0 = rpos2ipos(axis, bnd.origin, bnd.scale, hh, rl);
      assert(all((iaxis0 - baseextent.lower()) % istr == 0));
      ivect const iaxis1 = rpos2ipos1(axis, bnd.origin, bnd.scale, hh, rl);
      assert(all((iaxis1 - baseextent.lower()) % istr == 0));
      ivect const offset = iaxis1 - iaxis0;
      assert(all(offset % istr == 0));
      if (hh.refcent == vertex_centered) {
        assert(all(offset >= 0 and offset < 2 * istr));
        assert(all((iaxis0 + iaxis1 - offset) % (2 * istr) == 0));
      } else {
        // The offset may be negative because both boundaries are
        // shifted inwards by 1/2 grid spacing, and therefore iaxis0
        // < iaxis1 + istr
        assert(all(offset >= -istr and offset < istr));
        assert(all((iaxis0 + iaxis1 - offset) % (2 * istr) == istr));
        assert(all(istr % 2 == 0));
      }
      ivect const iaxis = (iaxis0 + iaxis1 - offset) / 2;
      ivect const neg_ilo = (2 * iaxis + offset) - ilo;
      ivect const neg_iup = (2 * iaxis + offset) - iup;

      // Rotate 180 degrees about z axis
      ivect const new_ilo(neg_iup[0], neg_iup[1], ilo[2]);
      ivect const new_iup(neg_ilo[0], neg_ilo[1], iup[2]);
      ivect const new_istr(istr);

      ibbox const new_bb(new_ilo, new_iup, new_istr);
      // Will be clipped later
      // assert (new_bb.is_contained_in (baseextent));

      // symmetrised |= new_bb & baseextent;
      symmetrised |= new_bb;
    }
  }

  return symmetrised;
}

bool rotsym180::test_impl(gh const &hh, dh const &dd, level_boundary const &bnd,
                          vector<ibset> const &regions, int const rl) {
  DECLARE_CCTK_PARAMETERS;

  if (not symmetry_rotating180)
    return true;

  ibset const symmetrised = symmetrised_regions(hh, dd, bnd, regions, rl);

  // We cannot test for equality, since the difference may be
  // outside of the domain (and hence irrelevant)
  // return regions.AT(rl) == symmetrised;

  // Test whether any part of the difference (i.e. that part of the
  // level that would be added by symmetrising) is inside the
  // domain. If the difference is outside, we can safely ignore it.
  ibbox const &baseextent = hh.baseextent(0, rl);
  ibset const difference = symmetrised - regions.AT(rl);
  return (difference & baseextent).empty();
}

void rotsym180::enforce_impl(gh const &hh, dh const &dd,
                             level_boundary const &bnd, vector<ibset> &regions,
                             int const rl) {
  DECLARE_CCTK_PARAMETERS;

  assert(symmetry_rotating180);

  if (veryverbose) {
    cout << "Refinement level " << rl
         << ": making regions rotating-180 symmetric...\n";
  }

  regions.AT(rl) = symmetrised_regions(hh, dd, bnd, regions, rl);

  if (veryverbose) {
    cout << "   New regions are " << regions.at(rl) << "\n";
  }
}

//////////////////////////////////////////////////////////////////////////////
// Make the boxes  parity symmetric
//////////////////////////////////////////////////////////////////////////////

ibset parsym::symmetrised_regions(gh const &hh, dh const &dd,
                                  level_boundary const &bnd,
                                  vector<ibset> const &regions, int const rl) {
  ibbox const &baseextent = hh.baseextent(0, rl);

  ibset symmetrised = regions.at(rl);
  for (ibset::const_iterator ibb = regions.at(rl).begin();
       ibb != regions.at(rl).end(); ++ibb) {
    ibbox const &bb = *ibb;

    bvect const lower_is_outside_lower =
        bb.lower() - bnd.min_bnd_dist_away[0] * bb.stride() <=
        bnd.level_physical_ilower;

    // Treat z direction
    int const dir = 2;
    if (lower_is_outside_lower[dir]) {
      ivect const ilo = bb.lower();
      ivect const iup = bb.upper();
      ivect const istr = bb.stride();
      assert(istr[0] == istr[1]);

      // Origin
      assert(hh.refcent == vertex_centered or all(istr % 2 == 0));
      rvect const axis((bnd.physical_lower[0] + bnd.physical_upper[0]) / 2,
                       (bnd.physical_lower[1] + bnd.physical_upper[1]) / 2,
                       bnd.physical_lower[2]);
      ivect const iaxis0 = rpos2ipos(axis, bnd.origin, bnd.scale, hh, rl);
      assert(all((iaxis0 - baseextent.lower()) % istr == 0));
      ivect const iaxis1 = rpos2ipos1(axis, bnd.origin, bnd.scale, hh, rl);
      assert(all((iaxis1 - baseextent.lower()) % istr == 0));
      ivect const offset = iaxis1 - iaxis0;
      assert(all(offset % istr == 0));
      if (hh.refcent == vertex_centered) {
        assert(all(offset >= 0 and offset < 2 * istr));
        assert(all((iaxis0 + iaxis1 - offset) % (2 * istr) == 0));
      } else {
        // The offset may be negative because both boundaries are
        // shifted inwards by 1/2 grid spacing, and therefore iaxis0
        // < iaxis1 + istr
        assert(all(offset >= -istr and offset < istr));
        assert(all((iaxis0 + iaxis1 - offset) % (2 * istr) == istr));
        assert(all(istr % 2 == 0));
      }
      ivect const iaxis = (iaxis0 + iaxis1 - offset) / 2;
      ivect const neg_ilo = (2 * iaxis + offset) - ilo;
      ivect const neg_iup = (2 * iaxis + offset) - iup;

      // Rotate 180 degrees about z axis
      ivect const new_ilo(neg_iup[0], neg_iup[1], neg_iup[2]);
      ivect const new_iup(neg_ilo[0], neg_ilo[1], neg_ilo[2]);
      ivect const new_istr(istr);

      ibbox const new_bb(new_ilo, new_iup, new_istr);
      // Will be clipped later
      // assert (new_bb.is_contained_in (baseextent));

      // symmetrised |= new_bb & baseextent;
      symmetrised |= new_bb;
    }
  }

  return symmetrised;
}

bool parsym::test_impl(gh const &hh, dh const &dd, level_boundary const &bnd,
                       vector<ibset> const &regions, int const rl) {
  DECLARE_CCTK_PARAMETERS;

  if (not symmetry_parity)
    return true;

  ibset const symmetrised = symmetrised_regions(hh, dd, bnd, regions, rl);

  // We cannot test for equality, since the difference may be
  // outside of the domain (and hence irrelevant)
  // return regions.AT(rl) == symmetrised;

  // Test whether any part of the difference (i.e. that part of the
  // level that would be added by symmetrising) is inside the
  // domain. If the difference is outside, we can safely ignore it.
  ibbox const &baseextent = hh.baseextent(0, rl);
  ibset const difference = symmetrised - regions.AT(rl);
  return (difference & baseextent).empty();
}

void parsym::enforce_impl(gh const &hh, dh const &dd, level_boundary const &bnd,
                          vector<ibset> &regions, int const rl) {
  DECLARE_CCTK_PARAMETERS;

  assert(symmetry_parity);

  if (veryverbose) {
    cout << "Refinement level " << rl
         << ": making regions parity symmetric...\n";
  }

  regions.AT(rl) = symmetrised_regions(hh, dd, bnd, regions, rl);

  if (veryverbose) {
    cout << "   New regions are " << regions.at(rl) << "\n";
  }
}

//////////////////////////////////////////////////////////////////////////////
// Make the boxes periodic in one direction
//////////////////////////////////////////////////////////////////////////////

template <int dir>
ibset periodic<dir>::symmetrised_regions(gh const &hh, dh const &dd,
                                         level_boundary const &bnd,
                                         vector<ibset> const &regions,
                                         int const rl) {
  ibbox const &baseextent = hh.baseextent(0, rl);

  // We are not using level_physical_ilower and
  // level_physical_iupper here, because these are rounded in
  // opposite directions for cell centring, so that their difference
  // is smaller than the domain size
  ivect const ilower =
      rpos2ipos(bnd.physical_lower, bnd.origin, bnd.scale, hh, rl);
  ivect const iupper =
      rpos2ipos(bnd.physical_upper, bnd.origin, bnd.scale, hh, rl);
  ivect const ioffset = ivect::dir(dir) * (iupper - ilower);
  assert(all(ioffset % baseextent.stride() == 0));

  ibset symmetrised = regions.at(rl);
  for (ibset::const_iterator ibb = regions.at(rl).begin();
       ibb != regions.at(rl).end(); ++ibb) {
    ibbox const &bb = *ibb;

    // Shift boxes upwards and downwards by one period
    symmetrised |= bb.shift(+ioffset / bb.stride());
    symmetrised |= bb.shift(-ioffset / bb.stride());
  }

  return symmetrised;
}

template <int dir>
bool periodic<dir>::test_impl(gh const &hh, dh const &dd,
                              level_boundary const &bnd,
                              vector<ibset> const &regions, int const rl) {
  DECLARE_CCTK_PARAMETERS;

  ivect const symmetry_periodic(symmetry_periodic_x, symmetry_periodic_y,
                                symmetry_periodic_z);
  if (not symmetry_periodic[dir])
    return true;

  ibset const symmetrised = symmetrised_regions(hh, dd, bnd, regions, rl);

  // We cannot test for equality, since the difference may be
  // outside of the domain (and hence irrelevant)
  // return regions.AT(rl) == symmetrised;

  // Test whether any part of the difference (i.e. that part of the
  // level that would be added by symmetrising) is inside the
  // domain. If the difference is outside, we can safely ignore it.
  ibbox const &baseextent = hh.baseextent(0, rl);
  ibset const difference = symmetrised - regions.AT(rl);
  return (difference & baseextent).empty();
}

template <int dir>
void periodic<dir>::enforce_impl(gh const &hh, dh const &dd,
                                 level_boundary const &bnd,
                                 vector<ibset> &regions, int const rl) {
  DECLARE_CCTK_PARAMETERS;

  ivect const symmetry_periodic(symmetry_periodic_x, symmetry_periodic_y,
                                symmetry_periodic_z);
  assert(symmetry_periodic[dir]);

  if (veryverbose) {
    cout << "Refinement level " << rl << ": making regions periodic in the "
         << "xyz"[dir] << " direction...\n";
  }

  regions.AT(rl) = symmetrised_regions(hh, dd, bnd, regions, rl);

  if (veryverbose) {
    cout << "   New regions are " << regions.at(rl) << "\n";
  }
}

template class periodic<0>;
template class periodic<1>;
template class periodic<2>;

//////////////////////////////////////////////////////////////////////////////
// Clip at the outer boundary
//////////////////////////////////////////////////////////////////////////////

ibset boundary_clip::clipped_regions(gh const &hh, dh const &dd,
                                     level_boundary const &bnd,
                                     vector<ibset> const &regions,
                                     int const rl) {
  ibbox const &baseextent = hh.baseextent(0, rl);

  ibset clipped;
  for (ibset::const_iterator ibb = regions.at(rl).begin();
       ibb != regions.at(rl).end(); ++ibb) {
    ibbox const &bb = *ibb;

    // Clip boxes that extend outside the boundary. Enlarge boxes
    // that are inside but too close to the outer boundary.
    bvect const lower_is_outside_lower =
        bb.lower() - bnd.min_bnd_dist_away[0] * bb.stride() <=
        bnd.level_physical_ilower;
    // Remove bboxes that are completely outside.
    bvect const upper_is_outside_lower = bb.upper() < bnd.level_physical_ilower;
    // Enlarge bboxes that extend not far enough inwards.
    bvect const upper_is_almost_outside_lower =
        bb.upper() <
        bnd.level_physical_ilower + bnd.min_bnd_dist_incl[0] * bb.stride();

    // Ditto for the upper boundary.
    bvect const upper_is_outside_upper =
        bb.upper() + bnd.min_bnd_dist_away[1] * bb.stride() >=
        bnd.level_physical_iupper;
    bvect const lower_is_outside_upper = bb.lower() > bnd.level_physical_iupper;
    bvect const lower_is_almost_outside_upper =
        bb.lower() >
        bnd.level_physical_iupper - bnd.min_bnd_dist_incl[1] * bb.stride();

    // Don't assert for trivial dimensions ( e.g., 2D: (x,y,1) )
    bvect const domain_is_trivial =
        bnd.level_physical_ilower == bnd.level_physical_iupper;

    assert(not any(lower_is_almost_outside_upper and lower_is_outside_lower and
                   not domain_is_trivial));
    assert(not any(upper_is_almost_outside_lower and upper_is_outside_upper and
                   not domain_is_trivial));

    if (any(upper_is_outside_lower or lower_is_outside_upper)) {
      // The box is completely outside. Ignore it.
      continue;
    }

    if (any((lower_is_outside_lower and bnd.boundary_staggering_mismatch[0]) or
            (upper_is_outside_upper and bnd.boundary_staggering_mismatch[1]))) {
      ostringstream msg;
      msg << "Level " << rl << " of the refinement hierarchy has inconsistent "
                               "bountary staggering."
          << "  The refined region extends up to the boundary, but the "
             "staggering of the boundary is different from the staggering of "
             "the mesh refinement."
          << "  lower_is_outside_lower=" << lower_is_outside_lower
          << "  upper_is_outside_upper=" << upper_is_outside_upper
          << "  boundary_staggering_mismatch="
          << bnd.boundary_staggering_mismatch
          << "  level_physical_ilower=" << bnd.level_physical_ilower
          << "  level_physical_iupper=" << bnd.level_physical_iupper
          << "  baseextent=" << baseextent;
      CCTK_WARN(CCTK_WARN_ABORT, msg.str().c_str());
    }

    ibbox const clipped_bb(
        either(lower_is_outside_lower, bnd.level_exterior_ilower,
               either(lower_is_almost_outside_upper,
                      (bnd.level_physical_iupper -
                       bnd.min_bnd_dist_incl[1] * bb.stride()),
                      bb.lower())),
        either(upper_is_outside_upper, bnd.level_exterior_iupper,
               either(upper_is_almost_outside_lower,
                      (bnd.level_physical_ilower +
                       bnd.min_bnd_dist_incl[0] * bb.stride()),
                      bb.upper())),
        bb.stride());
    if (not clipped_bb.is_contained_in(baseextent)) {
      ostringstream msg;
      msg << "Level " << rl << " of the refinement hierarchy is not contained "
                               "in the simulation domain."
          << "  (There may be too many ghost or buffer zones.)"
          << "  One bbox is " << clipped_bb << "."
          << "  lower_is_outside_lower=" << lower_is_outside_lower
          << "  upper_is_outside_upper=" << upper_is_outside_upper
          << "  lower_is_almost_outside_upper=" << lower_is_almost_outside_upper
          << "  upper_is_almost_outside_lower=" << upper_is_almost_outside_lower
          << "  level_exterior_ilower=" << bnd.level_exterior_ilower
          << "  level_exterior_iupper=" << bnd.level_exterior_iupper
          << "  level_physical_ilower=" << bnd.level_physical_ilower
          << "  level_physical_iupper=" << bnd.level_physical_iupper
          << "  baseextent=" << baseextent;
      CCTK_WARN(CCTK_WARN_ABORT, msg.str().c_str());
    }
    assert(clipped_bb.is_contained_in(baseextent));

    clipped |= clipped_bb;
  }

  return clipped;
}

bool boundary_clip::test_impl(gh const &hh, dh const &dd,
                              level_boundary const &bnd,
                              vector<ibset> const &regions, int const rl) {
  DECLARE_CCTK_PARAMETERS;

  ibset const clipped = clipped_regions(hh, dd, bnd, regions, rl);
  return regions.AT(rl) == clipped;
}

void boundary_clip::enforce_impl(gh const &hh, dh const &dd,
                                 level_boundary const &bnd,
                                 vector<ibset> &regions, int const rl) {
  DECLARE_CCTK_PARAMETERS;

  if (veryverbose) {
    cout << "Refinement level " << rl << ": clipping at outer boundary...\n";
  }

  regions.AT(rl) = clipped_regions(hh, dd, bnd, regions, rl);

  if (veryverbose) {
    cout << "   New regions are " << regions.at(rl) << "\n";
  }
}

//////////////////////////////////////////////////////////////////////////////
// Ensure that this grid is contained in the domain
//////////////////////////////////////////////////////////////////////////////

bool in_domain::test_impl(gh const &hh, dh const &dd, level_boundary const &bnd,
                          vector<ibset> const &regions, int const rl) {
  // TODO: baseextent is the exterior of the domain (i.e. it
  // includes the boundary); we need to use the interior of the
  // domain here (i.e. baseextent minus the boundary). Such a
  // calculation is already performed in the beginning of dh.cc's
  // regrid.
  return regions.at(rl) <= hh.baseextent(0, rl);
}

void in_domain::enforce_impl(gh const &hh, dh const &dd,
                             level_boundary const &bnd, vector<ibset> &regions,
                             int const rl) {
  // There is nothing we can do here, since we can't enlarge the
  // domain
  CCTK_WARN(CCTK_WARN_ABORT, "internal error");
}

//////////////////////////////////////////////////////////////////////////////
// Ensure that this grid is in the domain, if desired
//////////////////////////////////////////////////////////////////////////////

ibset is_symmetric::symmetrised_regions(gh const &hh, dh const &dd,
                                        level_boundary const &bnd,
                                        vector<ibset> const &regions,
                                        int const rl) {
  ibset symmetrised = regions.at(rl);
  for (ibset::const_iterator ibb = regions.at(rl).begin();
       ibb != regions.at(rl).end(); ++ibb) {
    ibbox const &bb = *ibb;

    ivect const ilo = bb.lower();
    ivect const iup = bb.upper();
    ivect const istr = bb.stride();

    // Origin
    rvect const axis(bnd.physical_lower[0], bnd.physical_lower[1],
                     bnd.physical_lower[2]);
    ivect const iaxis0 = rpos2ipos(axis, bnd.origin, bnd.scale, hh, rl);
    assert(all(iaxis0 % istr == 0));
    ivect const iaxis1 = rpos2ipos1(axis, bnd.origin, bnd.scale, hh, rl);
    assert(all(iaxis1 % istr == 0));
    ivect const offset = iaxis1 - iaxis0;
    assert(all(offset % istr == 0));
    assert(all(offset >= 0 and offset < 2 * istr));
    assert(all((iaxis0 + iaxis1 - offset) % (2 * istr) == 0));
    ivect const iaxis = (iaxis0 + iaxis1 - offset) / 2;
    // negated (reflected) domain boundaries
    ivect const neg_ilo = (2 * iaxis + offset) - ilo;
    ivect const neg_iup = (2 * iaxis + offset) - iup;

    // Mirror
    ivect const new_ilo = neg_iup;
    ivect const new_iup = neg_ilo;
    ivect const new_istr(istr);

    ibbox const new_bb(new_ilo, new_iup, new_istr);

    symmetrised |= new_bb;
  }

  return symmetrised;
}

bool is_symmetric::test_impl(gh const &hh, dh const &dd,
                             level_boundary const &bnd,
                             vector<ibset> const &regions, int const rl) {
  DECLARE_CCTK_PARAMETERS;

  if (not expect_symmetric_grids)
    return true;

  ibset const symmetrised = symmetrised_regions(hh, dd, bnd, regions, rl);
  return regions.AT(rl) == symmetrised;
}

void is_symmetric::enforce_impl(gh const &hh, dh const &dd,
                                level_boundary const &bnd,
                                vector<ibset> &regions, int const rl) {
  // There is nothing we want to do here
  CCTK_WARN(CCTK_WARN_ABORT, "internal error");
}

} // namespace CarpetRegrid2

```


combined_doc_context: ## README (README):
```
Cactus Code Thorn CarpetRegrid2
Author(s)    : Erik Schnetter <schnetter@cct.lsu.edu>
Maintainer(s): Erik Schnetter <schnetter@cct.lsu.edu>
Licence      : GPLv2+
--------------------------------------------------------------------------

1. Purpose

Set up refined regions by specifying a set of centres and radii about
them.  The refined regions are then the conjunction of these regions.

```

## Documentation (.tex files):
Files: documentation.tex

```tex
=== documentation.tex ===
% *======================================================================*
%  Cactus Thorn template for ThornGuide documentation
%  Author: Ian Kelley
%  Date: Sun Jun 02, 2002
%  $Header: /cactusdevcvs/Cactus/doc/ThornGuide/template.tex,v 1.12 2004/01/07 20:12:39 rideout Exp $
%
%  Thorn documentation in the latex file doc/documentation.tex
%  will be included in ThornGuides built with the Cactus make system.
%  The scripts employed by the make system automatically include
%  pages about variables, parameters and scheduling parsed from the
%  relevant thorn CCL files.
%
%  This template contains guidelines which help to assure that your
%  documentation will be correctly added to ThornGuides. More
%  information is available in the Cactus UsersGuide.
%
%  Guidelines:
%   - Do not change anything before the line
%       % START CACTUS THORNGUIDE",
%     except for filling in the title, author, date, etc. fields.
%        - Each of these fields should only be on ONE line.
%        - Author names should be separated with a \\ or a comma.
%   - You can define your own macros, but they must appear after
%     the START CACTUS THORNGUIDE line, and must not redefine standard
%     latex commands.
%   - To avoid name clashes with other thorns, 'labels', 'citations',
%     'references', and 'image' names should conform to the following
%     convention:
%       ARRANGEMENT_THORN_LABEL
%     For example, an image wave.eps in the arrangement CactusWave and
%     thorn WaveToyC should be renamed to CactusWave_WaveToyC_wave.eps
%   - Graphics should only be included using the graphicx package.
%     More specifically, with the "\includegraphics" command.  Do
%     not specify any graphic file extensions in your .tex file. This
%     will allow us to create a PDF version of the ThornGuide
%     via pdflatex.
%   - References should be included with the latex "\bibitem" command.
%   - Use \begin{abstract}...\end{abstract} instead of \abstract{...}
%   - Do not use \appendix, instead include any appendices you need as
%     standard sections.
%   - For the benefit of our Perl scripts, and for future extensions,
%     please use simple latex.
%
% *======================================================================*
%
% Example of including a graphic image:
%    \begin{figure}[ht]
% 	\begin{center}
%    	   \includegraphics[width=6cm]{MyArrangement_MyThorn_MyFigure}
% 	\end{center}
% 	\caption{Illustration of this and that}
% 	\label{MyArrangement_MyThorn_MyLabel}
%    \end{figure}
%
% Example of using a label:
%   \label{MyArrangement_MyThorn_MyLabel}
%
% Example of a citation:
%    \cite{MyArrangement_MyThorn_Author99}
%
% Example of including a reference
%   \bibitem{MyArrangement_MyThorn_Author99}
%   {J. Author, {\em The Title of the Book, Journal, or periodical}, 1 (1999),
%   1--16. {\tt http://www.nowhere.com/}}
%
% *======================================================================*

% If you are using CVS use this line to give version information
% $Header: /cactusdevcvs/Cactus/doc/ThornGuide/template.tex,v 1.12 2004/01/07 20:12:39 rideout Exp $

\documentclass{article}

% Use the Cactus ThornGuide style file
% (Automatically used from Cactus distribution, if you have a
%  thorn without the Cactus Flesh download this from the Cactus
%  homepage at www.cactuscode.org)
\usepackage{../../../../doc/latex/cactus}

\begin{document}

% The author of the documentation
\author{Erik Schnetter \textless schnetter@cct.lsu.edu\textgreater}

% The title of the document (not necessarily the name of the Thorn)
\title{CarpetRegrid2}

% the date your document was last changed, if your document is in CVS,
% please use:
%    \date{$ $Date: 2004/01/07 20:12:39 $ $}
\date{February 15, 2006}

\maketitle

% Do not delete next line
% START CACTUS THORNGUIDE

% Add all definitions used in this documentation here
%   \def\mydef etc

% Add an abstract for this thorn's documentation
\begin{abstract}

\end{abstract}

% The following sections are suggestive only.
% Remove them or add your own.

\section{Introduction}

\section{Physical System}

\section{Numerical Implementation}

\section{Using This Thorn}

\subsection{Obtaining This Thorn}

\subsection{Basic Usage}

\subsection{Special Behaviour}

\subsection{Interaction With Other Thorns}

\subsection{Examples}

\subsection{Support and Feedback}

\section{History}

\subsection{Thorn Source Code}

\subsection{Thorn Documentation}

\subsection{Acknowledgements}


\begin{thebibliography}{9}

\end{thebibliography}

% Do not delete next line
% END CACTUS THORNGUIDE

\end{document}

```



Reference solution:
src_code: #include <algorithm>
#include <cassert>
#include <cmath>
#include <cstdlib>
#include <iostream>
#include <sstream>
#include <typeinfo>
#include <vector>

#include <cctk.h>
#include <cctk_Arguments.h>
#include <cctk_Parameters.h>

#include <Timer.hh>

#include <bbox.hh>
#include <bboxset.hh>
#include <defs.hh>
#include <dh.hh>
#include <gh.hh>
#include <region.hh>
#include <vect.hh>

#include <carpet.hh>

#include "amr.hh"
#include "boundary.hh"
#include "indexing.hh"
#include "property.hh"

namespace CarpetRegrid2 {

using namespace std;
using namespace Carpet;

struct centre_description {
  int _num_levels;
  int _active;
  rvect _position;
  vector<rvect> _radius;

  centre_description(cGH const *cctkGH, int n);
};

centre_description::centre_description(cGH const *const cctkGH, int const n) {
  DECLARE_CCTK_ARGUMENTS;
  DECLARE_CCTK_PARAMETERS;

  assert(n >= 0 and n < num_centres);

  bool found_error = false;

  int lsh[2];
  getvectorindex2(cctkGH, "CarpetRegrid2::radii", lsh);

  this->_num_levels = num_levels[n];
  this->_active = active[n];
  this->_position = rvect(position_x[n], position_y[n], position_z[n]);
  if (any(not isfinite(this->_position))) {
    CCTK_VWarn(CCTK_WARN_ABORT, __LINE__, __FILE__, CCTK_THORNSTRING,
               "The position of region %d is [%g,%g,%g], which is not finite",
               n + 1, double(this->_position[0]), double(this->_position[1]),
               double(this->_position[2]));
    found_error = true;
  }
  this->_radius.resize(this->_num_levels);
  this->_radius.at(0) = rvect(-1.0, -1.0, -1.0); // unused
  for (int rl = 1; rl < this->_num_levels; ++rl) {
    int const ind = index2(lsh, rl, n);
    CCTK_REAL const rx = radius_x[ind] < 0.0 ? radius[ind] : radius_x[ind];
    CCTK_REAL const ry = radius_y[ind] < 0.0 ? radius[ind] : radius_y[ind];
    CCTK_REAL const rz = radius_z[ind] < 0.0 ? radius[ind] : radius_z[ind];
    rvect const rad(rx, ry, rz);
    if (any(not isfinite(rad))) {
      CCTK_VWarn(CCTK_WARN_ABORT, __LINE__, __FILE__, CCTK_THORNSTRING,
                 "The radius of refinement level %d of region %d is "
                 "[%g,%g,%g], which is not finite",
                 rl, n + 1, double(rad[0]), double(rad[1]), double(rad[2]));
      found_error = true;
    }
    if (any(rad < CCTK_REAL(0))) {
      CCTK_VWarn(CCTK_WARN_ABORT, __LINE__, __FILE__, CCTK_THORNSTRING,
                 "The radius of refinement level %d of region %d is "
                 "[%g,%g,%g], which is not non-negative",
                 rl, n + 1, double(rad[0]), double(rad[1]), double(rad[2]));
      found_error = true;
    }
    this->_radius.at(rl) = rad;
  }

  if (this->_num_levels > maxreflevels) {
    CCTK_VWarn(CCTK_WARN_ABORT, __LINE__, __FILE__, CCTK_THORNSTRING,
               "Region %d has %d levels active, which is larger than the "
               "maximum number of refinement levels %d",
               n + 1, this->_num_levels, maxreflevels);
    found_error = true;
  }

  if (found_error) {
    CCTK_WARN(CCTK_WARN_ABORT, "Errors found in grid structure specification");
  }
}

extern "C" {
CCTK_INT
CarpetRegrid2_Regrid(CCTK_POINTER_TO_CONST const cctkGH_,
                     CCTK_POINTER const superregss_, CCTK_POINTER const regsss_,
                     CCTK_INT const force);

CCTK_INT
CarpetRegrid2_RegridMaps(CCTK_POINTER_TO_CONST const cctkGH_,
                         CCTK_POINTER const superregsss_,
                         CCTK_POINTER const regssss_, CCTK_INT const force);
}

void Regrid(cGH const *const cctkGH, gh::rregs &regss) {
  DECLARE_CCTK_PARAMETERS;

  if (verbose or veryverbose) {
    CCTK_VInfo(CCTK_THORNSTRING,
               "Regridding level %d map %d at iteration %d time %g", reflevel,
               Carpet::map, cctkGH->cctk_iteration, cctkGH->cctk_time);
  }

  assert(is_singlemap_mode());
  gh const &hh = *vhh.at(Carpet::map);
  dh const &dd = *vdd.at(Carpet::map);

  //
  // Find extent of domain
  //

  // This requires that CoordBase is used
  // TODO: check this (for Carpet, and maybe also for CartGrid3D)
  // TODO: (the check that these two are consistent should be in
  // Carpet)

  jjvect nboundaryzones, is_internal;
  jjvect is_staggered, shiftout;
  get_boundary_specification(nboundaryzones, is_internal, is_staggered,
                             shiftout);

  rvect physical_lower, physical_upper;
  rvect spacing;
  get_physical_boundary(physical_lower, physical_upper, spacing);

  // Adapt spacing for convergence level
  spacing *= ipow((CCTK_REAL)mgfact, basemglevel);

  rvect exterior_lower, exterior_upper;
  calculate_exterior_boundary(physical_lower, physical_upper, exterior_lower,
                              exterior_upper, spacing);

  //
  // Determine which refinement levels may be changed
  //
  int min_rl = 1; // we cannot change the coarsest level
  if (freeze_unaligned_levels or freeze_unaligned_parent_levels) {
    while (min_rl < int(regss.size())) {
// Increase min_rl until we find a level that can be changed
#if 0
        // TODO: think about this a bit more
        // TODO: use this taper-checking also in Comm.cc
        bool in_sync = true;
        if (freeze_unaligned_parent_levels) {
          int const parent_do_every =
            ipow(mgfact, mglevel) *
            (maxtimereflevelfact / timereffacts.at(min_rl-1));
          in_sync =
            cctkGH->cctk_iteration == 0 or
            (cctkGH->cctk_iteration-1) % parent_do_every == 0;
        }
#else
      bool in_sync = true;
      if (freeze_unaligned_parent_levels) {
        // Assume that non-existing levels are always aligned
        if (min_rl < reflevels) {
          CCTK_REAL const mytime = tt->get_time(mglevel, min_rl, 0);
          CCTK_REAL const parenttime = tt->get_time(mglevel, min_rl - 1, 0);
          CCTK_REAL const eps = 1.0e-12;
          in_sync = abs(mytime - parenttime) <= eps * abs(delta_time);
        }
      }
#endif
      int const do_every = ipow(mgfact, mglevel) *
                           (maxtimereflevelfact / timereffacts.at(min_rl));
      bool const is_active = cctkGH->cctk_iteration == 0 or
                             (cctkGH->cctk_iteration - 1) % do_every == 0;
      if (is_active and in_sync)
        break;
      ++min_rl;
    }
    if (verbose or veryverbose) {
      CCTK_VInfo(CCTK_THORNSTRING, "Regridding levels %d and up", min_rl);
    }
  }

  //
  // Calculate the union of the bounding boxes for all levels
  //

  // The set of refined regions
  vector<ibset> regions(min_rl);

  // Set up coarse levels
  for (int rl = 0; rl < min_rl; ++rl) {
    if (veryverbose) {
      cout << "Refinement level " << rl << ": will not be changed" << endl;
    }
    for (size_t c = 0; c < regss.at(rl).size(); ++c) {
      regions.at(rl) += regss.at(rl).at(c).extent;
    }
    if (veryverbose) {
      cout << "Refinement level " << rl << ": regions are " << regions.at(rl)
           << endl;
    }
  }

  // Refine only patch 0
  if (Carpet::map == 0) {

    // Loop over all centres
    for (int n = 0; n < num_centres; ++n) {
      centre_description centre(cctkGH, n);
      if (centre._active) {

        // Loop over all levels for this centre
        for (int rl = min_rl; rl < centre._num_levels; ++rl) {

          if (veryverbose) {
            cout << "Refinement level " << rl
                 << ": importing refined region settings..." << endl;
          }

          level_boundary const bnd(hh, dd, rl);

          // Calculate a bbox for this region
          rvect const rmin = centre._position - centre._radius.at(rl);
          rvect const rmax = centre._position + centre._radius.at(rl);

          if (veryverbose) {
            cout << "Centre " << n + 1 << " refinement level " << rl
                 << ": coordinate region is (" << rmin << ":" << rmax << ")\n";
          }

          // Convert to an integer bbox
          ivect const istride = hh.baseextent(0, rl).stride();
          ivect const imin = rpos2ipos(rmin, bnd.origin, bnd.scale, hh, rl) -
                             int(boundary_shiftout) * istride;
          ivect const imax = rpos2ipos1(rmax, bnd.origin, bnd.scale, hh, rl) +
                             int(boundary_shiftout) * istride;

          if (veryverbose) {
            cout << "Centre " << n + 1 << " refinement level " << rl
                 << ": integer region is (" << imin << ":" << imax << ")\n";
          }

          ibbox const region(imin, imax, istride);

          // Add this region to the list of regions
          if (static_cast<int>(regions.size()) < rl + 1) {
            regions.resize(rl + 1);
          }
          regions.at(rl) |= region;

          if (veryverbose) {
            cout << "Refinement level " << rl << ": preliminary regions are "
                 << regions.at(rl) << endl;
          }

        } // for rl

      } // if centre is active
    }   // for n

    if (adaptive_refinement) {
      // Loop over all levels
      for (int rl = min_rl; rl < min(maxreflevels, reflevels + 1); ++rl) {
        if (static_cast<int>(regions.size()) < rl + 1) {
          regions.resize(rl + 1);
        }
        evaluate_level_mask(cctkGH, regions, rl);
        if (regions.at(rl).empty()) {
          // If there are no refined regions on this level, truncate
          // the refinement hierarchy, and stop
          assert(static_cast<int>(regions.size()) == rl + 1);
          regions.resize(rl);
          break;
        }
      }
    }

  } // if map==0

  // We need to check and/or enforce certain properties of the grid
  // structure, such as e.g. proper nesting. Unfortunately,
  // enforcing one of the properties may invalidate another. We
  // therefore abstract the concept of a property into a class
  // "property", and enforce them round-robin until the grid
  // structure does not change any more. The order in which the
  // properties are enforced is relevant (i.e. it influence the
  // final grid structure).

  // Properties to be applied (without testing), only once, in the
  // beginning
  vector<property *> once_properties;
  once_properties.push_back(new proper_nesting());
  once_properties.push_back(new add_buffers());

  // Properties to be enforced "until all is well"
  vector<property *> properties;
  properties.push_back(new combine_regions());
  properties.push_back(new snap_coarse());
  properties.push_back(new rotsym90());
  properties.push_back(new rotsym180());
  properties.push_back(new parsym());
  properties.push_back(new periodic<0>());
  properties.push_back(new periodic<1>());
  properties.push_back(new periodic<2>());
  properties.push_back(new boundary_clip());

  // Properties to be tested (and not enforced) in the end
  vector<property *> final_properties;
  final_properties.push_back(new in_domain());
  final_properties.push_back(new is_symmetric());

  // TODO: Check that the coarse grid contains all finer grids. To
  // do this, apply proper_nesting to the coarse grid, and then
  // check whether it grew -- if so, this is an error.

  regss.resize(regions.size());

  // Loop over all levels from finest to coarsest
  assert(regions.size() > 0);
  for (int rl = regions.size() - 1; rl >= min_rl; --rl) {

    // Sanity check
    assert(not regions.at(rl).empty());

    // Determine boundary locations of this level
    level_boundary const bnd(hh, dd, rl);

    if (veryverbose) {
      cout << "Refinement level " << rl << ":\n"
           << "   Original regions are " << regions.at(rl) << "\n";
    }

    // Apply "once" properties unconditionally
    for (vector<property *>::iterator pi = once_properties.begin();
         pi != once_properties.end(); ++pi) {
      (*pi)->enforce(hh, dd, bnd, regions, rl);
    }

    // Enforce properties on this level
    for (int count = 0;; ++count) {
      CCTK_VInfo(CCTK_THORNSTRING,
                 "Enforcing grid structure properties, iteration %d", count);
      bool done_enforcing = true;
      ibset const old_regions = regions.at(rl);
      for (vector<property *>::iterator pi = properties.begin();
           pi != properties.end(); ++pi) {
        bool const test_satisfied = (*pi)->test(hh, dd, bnd, regions, rl);
        if (not test_satisfied) {
          (*pi)->enforce(hh, dd, bnd, regions, rl);
        }
        done_enforcing = done_enforcing and test_satisfied;
      }

      if (done_enforcing)
        break;

      if (regions.at(rl) == old_regions) {
        CCTK_WARN(CCTK_WARN_ABORT, "Could not enforce grid structure "
                                   "properties (not making any progress); "
                                   "giving up");
      }
      if (count == 10) {
        CCTK_VWarn(CCTK_WARN_ABORT, __LINE__, __FILE__, CCTK_THORNSTRING,
                   "Could not enforce grid structure properties after %d "
                   "iterations; giving up",
                   count);
      }
      if (count != 0) {
        // This may not be true. However, the previous version of
        // the code assumed this, so we want to know whether this
        // fails.
        CCTK_WARN(CCTK_WARN_ALERT, "Could not enforce grid structure "
                                   "properties in this round, starting another "
                                   "round");
      }
    }

    if (veryverbose) {
      cout << "Refinement level " << rl << ":\n"
           << "   Final regions are " << regions.at(rl) << "\n";
    }

    // Test final properties
    bool all_is_well = true;
    for (vector<property *>::const_iterator pi = final_properties.begin();
         pi != final_properties.end(); ++pi) {
      bool const test_satisfied = (*pi)->test(hh, dd, bnd, regions, rl);
      if (not test_satisfied) {
        cout << "Necessary property " << typeid(*pi).name()
             << " does not hold\n";
      }
      all_is_well = all_is_well and test_satisfied;
    }
    if (not all_is_well) {
      CCTK_WARN(CCTK_WARN_ABORT,
                "Not all necessary grid structure properties are holding");
    }

    // Create a vector of bboxes for this level
    gh::cregs regs;
    for (ibset::const_iterator ibb = regions.AT(rl).begin();
         ibb != regions.AT(rl).end(); ++ibb) {
      ibbox const &bb = *ibb;
      assert(bb.is_contained_in(hh.baseextent(0, rl)));

      bvect const lower_is_outer = bb.lower() <= bnd.level_physical_ilower;
      bvect const upper_is_outer = bb.upper() >= bnd.level_physical_iupper;

      b2vect const ob(lower_is_outer, upper_is_outer);

      region_t reg;
      reg.extent = bb;
      reg.map = Carpet::map;
      reg.outer_boundaries = ob;
      regs.push_back(reg);
    }

    regss.AT(rl) = regs;

  } // for rl

  // Delete properties
  for (vector<property *>::iterator pi = once_properties.begin();
       pi != once_properties.end(); ++pi) {
    delete *pi;
  }
  for (vector<property *>::iterator pi = properties.begin();
       pi != properties.end(); ++pi) {
    delete *pi;
  }
  for (vector<property *>::iterator pi = final_properties.begin();
       pi != final_properties.end(); ++pi) {
    delete *pi;
  }
}

CCTK_INT
CarpetRegrid2_Regrid(CCTK_POINTER_TO_CONST const cctkGH_,
                     CCTK_POINTER const superregss_, CCTK_POINTER const regsss_,
                     CCTK_INT const force) {
  cGH const *const cctkGH = static_cast<cGH const *>(cctkGH_);

  DECLARE_CCTK_ARGUMENTS;
  DECLARE_CCTK_PARAMETERS;

  char const *const where = "CarpetRegrid2::Regrid";
  static Timers::Timer timer(where);
  timer.start();

  assert(is_singlemap_mode());

  // Decide whether to change the grid hierarchy
  bool do_recompose;
  if (force) {
    do_recompose = true;
  } else {
    if (regrid_every == -1) {
      do_recompose = false;
    } else if (regrid_every == 0) {
      do_recompose = cctk_iteration == 0;
    } else {
      // Regrid at most once per iteration
      do_recompose =
          (cctk_iteration == 0 or
           (cctk_iteration > 0 and (cctk_iteration - 1) % regrid_every == 0 and
            (cctk_iteration > *last_iteration or
             (cctk_iteration == *last_iteration and Carpet::map > *last_map))));
    }
  }

  if (verbose or veryverbose) {
    if (do_recompose) {
      for (int n = 0; n < num_centres; ++n) {
        if (active[n]) {
          CCTK_VInfo(CCTK_THORNSTRING,
                     "Centre %d is at position [%g,%g,%g] with %d levels",
                     n + 1, static_cast<double>(position_x[n]),
                     static_cast<double>(position_y[n]),
                     static_cast<double>(position_z[n]),
                     static_cast<int>(num_levels[n]));
        } else {
          CCTK_VInfo(CCTK_THORNSTRING, "Centre %d is not active", n + 1);
        }
      }
    }
  }

  if (not force and do_recompose and *last_iteration != -1) {

    int lsh[2];
    getvectorindex2(cctkGH, "CarpetRegrid2::radii", lsh);

    // Regrid only if the regions have changed sufficiently
    do_recompose = false;
    if (adaptive_refinement)
      do_recompose = true;
    if (not do_recompose) {
      for (int n = 0; n < num_centres; ++n) {

        // Regrid if a region became active or inactive
        do_recompose = active[n] != old_active[n];
        if (do_recompose)
          break;

        // Check only active regions
        if (not active[n])
          continue;

        // Regrid if the number of levels changed
        do_recompose = num_levels[n] != old_num_levels[n];
        if (do_recompose)
          break;

        // Regrid if the positions have changed sufficiently
        CCTK_REAL const dist2 = pow(position_x[n] - old_position_x[n], 2) +
                                pow(position_y[n] - old_position_y[n], 2) +
                                pow(position_z[n] - old_position_z[n], 2);
        CCTK_REAL mindist;
        switch (n) {
        case 0:
          mindist = movement_threshold_1;
          break;
        case 1:
          mindist = movement_threshold_2;
          break;
        case 2:
          mindist = movement_threshold_3;
          break;
        case 3:
          mindist = movement_threshold_4;
          break;
        case 4:
          mindist = movement_threshold_5;
          break;
        case 5:
          mindist = movement_threshold_6;
          break;
        case 6:
          mindist = movement_threshold_7;
          break;
        case 7:
          mindist = movement_threshold_8;
          break;
        case 8:
          mindist = movement_threshold_9;
          break;
        case 9:
          mindist = movement_threshold_10;
          break;
        default:
          assert(0);
        }
        do_recompose = dist2 > pow(mindist, 2);
        if (do_recompose)
          break;

        // Regrid if the radii have changed sufficiently
        for (int rl = 1; rl < num_levels[n]; ++rl) {
          int const ind = index2(lsh, rl, n);
          CCTK_REAL const rx = radius_x[ind] < 0 ? radius[ind] : radius_x[ind];
          CCTK_REAL const ry = radius_y[ind] < 0 ? radius[ind] : radius_y[ind];
          CCTK_REAL const rz = radius_z[ind] < 0 ? radius[ind] : radius_z[ind];
          rvect const rad(rx, ry, rz);
          rvect const oldrad(old_radius_x[ind], old_radius_y[ind],
                             old_radius_z[ind]);
          CCTK_REAL const drfac =
              sqrt(sum(ipow(rad - oldrad, 2))) / sqrt(sum(ipow(oldrad, 2)));
          CCTK_REAL mindrfac;
          switch (n) {
          case 0:
            mindrfac = radius_rel_change_threshold_1;
            break;
          case 1:
            mindrfac = radius_rel_change_threshold_2;
            break;
          case 2:
            mindrfac = radius_rel_change_threshold_3;
            break;
          case 3:
            mindrfac = radius_rel_change_threshold_4;
            break;
          case 4:
            mindrfac = radius_rel_change_threshold_5;
            break;
          case 5:
            mindrfac = radius_rel_change_threshold_6;
            break;
          case 6:
            mindrfac = radius_rel_change_threshold_7;
            break;
          case 7:
            mindrfac = radius_rel_change_threshold_8;
            break;
          case 8:
            mindrfac = radius_rel_change_threshold_9;
            break;
          case 9:
            mindrfac = radius_rel_change_threshold_10;
            break;
          default:
            assert(0);
          }
          do_recompose = drfac > mindrfac;
          if (do_recompose)
            break;
        } // for rl
        if (do_recompose)
          break;

      } // for n
      if (verbose or veryverbose) {
        if (not do_recompose) {
          CCTK_INFO("Refined regions have not changed sufficiently; skipping "
                    "regridding");
        }
      }
    }
  }

  if (do_recompose) {
    *last_iteration = cctk_iteration;
    *last_map = Carpet::map;
  }

  if (do_recompose) {

    gh::rregs &superregss = *static_cast<gh::rregs *>(superregss_);
    gh::mregs &regsss = *static_cast<gh::mregs *>(regsss_);

    Regrid(cctkGH, superregss);

    // Make multiprocessor aware
    vector<vector<region_t> > regss(superregss.size());
    for (size_t rl = 0; rl < regss.size(); ++rl) {
      SplitRegions(cctkGH, superregss.at(rl), regss.at(rl));
    } // for rl

    // Make multigrid aware
    MakeMultigridBoxes(cctkGH, Carpet::map, regss, regsss);

    // Remember current positions
    for (int n = 0; n < num_centres; ++n) {
      old_active[n] = active[n];

      old_num_levels[n] = num_levels[n];

      old_position_x[n] = position_x[n];
      old_position_y[n] = position_y[n];
      old_position_z[n] = position_z[n];

      old_radius_x[n] = radius_x[n] < 0 ? radius[n] : radius_x[n];
      old_radius_y[n] = radius_y[n] < 0 ? radius[n] : radius_y[n];
      old_radius_z[n] = radius_z[n] < 0 ? radius[n] : radius_z[n];
    }

  } // if do_recompose

  timer.stop();

  return do_recompose;
}

CCTK_INT
CarpetRegrid2_RegridMaps(CCTK_POINTER_TO_CONST const cctkGH_,
                         CCTK_POINTER const superregsss_,
                         CCTK_POINTER const regssss_, CCTK_INT const force) {
  cGH const *const cctkGH = static_cast<cGH const *>(cctkGH_);

  DECLARE_CCTK_ARGUMENTS;
  DECLARE_CCTK_PARAMETERS;

  char const *const where = "CarpetRegrid2::RegridMaps";
  static Timers::Timer timer(where);
  timer.start();

  assert(is_level_mode());

  // Decide whether to change the grid hierarchy
  assert(*last_map == -1); // ensure this remains unused
  bool do_recompose;
  if (force) {
    do_recompose = true;
  } else {
    if (regrid_every == -1) {
      do_recompose = false;
    } else if (regrid_every == 0) {
      do_recompose = cctk_iteration == 0;
    } else {
      // Regrid at most once per iteration
      do_recompose =
          (cctk_iteration == 0 or
           (cctk_iteration > 0 and (cctk_iteration - 1) % regrid_every == 0 and
            cctk_iteration > *last_iteration));
    }
  }

  if (verbose or veryverbose) {
    if (do_recompose) {
      for (int n = 0; n < num_centres; ++n) {
        if (active[n]) {
          CCTK_VInfo(CCTK_THORNSTRING,
                     "Centre %d is at position [%g,%g,%g] with %d levels",
                     n + 1, static_cast<double>(position_x[n]),
                     static_cast<double>(position_y[n]),
                     static_cast<double>(position_z[n]),
                     static_cast<int>(num_levels[n]));
        } else {
          CCTK_VInfo(CCTK_THORNSTRING, "Centre %d is not active", n + 1);
        }
      }
    }
  }

  if (not force and do_recompose and *last_iteration != -1) {

    int lsh[2];
    getvectorindex2(cctkGH, "CarpetRegrid2::radii", lsh);

    // Regrid only if the regions have changed sufficiently
    do_recompose = false;
    if (adaptive_refinement)
      do_recompose = true;
    if (not do_recompose) {
      for (int n = 0; n < num_centres; ++n) {

        // When debugging, sneakily add a new level, but skip the
        // initial regrid, and the regrid before the first time step
        if (add_levels_automatically and cctk_iteration > 1) {
          num_levels[n] = min(int(num_levels[n] + 1), maxreflevels);
          CCTK_VInfo(CCTK_THORNSTRING,
                     "Increasing number of levels of centre %d to %d (it=%d)",
                     n + 1, static_cast<int>(num_levels[n]), cctk_iteration);
        }

        // Regrid if a region became active or inactive
        do_recompose = active[n] != old_active[n];
        if (do_recompose)
          break;

        // Check only active regions
        if (not active[n])
          continue;

        // Regrid if the number of levels changed
        do_recompose = num_levels[n] != old_num_levels[n];
        if (do_recompose)
          break;

        // Regrid if the positions have changed sufficiently
        CCTK_REAL const dist2 = pow(position_x[n] - old_position_x[n], 2) +
                                pow(position_y[n] - old_position_y[n], 2) +
                                pow(position_z[n] - old_position_z[n], 2);
        CCTK_REAL mindist;
        switch (n) {
        case 0:
          mindist = movement_threshold_1;
          break;
        case 1:
          mindist = movement_threshold_2;
          break;
        case 2:
          mindist = movement_threshold_3;
          break;
        case 3:
          mindist = movement_threshold_4;
          break;
        case 4:
          mindist = movement_threshold_5;
          break;
        case 5:
          mindist = movement_threshold_6;
          break;
        case 6:
          mindist = movement_threshold_7;
          break;
        case 7:
          mindist = movement_threshold_8;
          break;
        case 8:
          mindist = movement_threshold_9;
          break;
        case 9:
          mindist = movement_threshold_10;
          break;
        default:
          assert(0);
        }
        do_recompose = dist2 > pow(mindist, 2);
        if (do_recompose)
          break;

        // Regrid if the radii have changed sufficiently
        for (int rl = 1; rl < num_levels[n]; ++rl) {
          int const ind = index2(lsh, rl, n);
          CCTK_REAL const rx = radius_x[ind] < 0 ? radius[ind] : radius_x[ind];
          CCTK_REAL const ry = radius_y[ind] < 0 ? radius[ind] : radius_y[ind];
          CCTK_REAL const rz = radius_z[ind] < 0 ? radius[ind] : radius_z[ind];
          rvect const rad(rx, ry, rz);
          rvect const oldrad(old_radius_x[ind], old_radius_y[ind],
                             old_radius_z[ind]);
          CCTK_REAL const drfac =
              sqrt(sum(ipow(rad - oldrad, 2))) / sqrt(sum(ipow(oldrad, 2)));
          CCTK_REAL mindrfac;
          switch (n) {
          case 0:
            mindrfac = radius_rel_change_threshold_1;
            break;
          case 1:
            mindrfac = radius_rel_change_threshold_2;
            break;
          case 2:
            mindrfac = radius_rel_change_threshold_3;
            break;
          case 3:
            mindrfac = radius_rel_change_threshold_4;
            break;
          case 4:
            mindrfac = radius_rel_change_threshold_5;
            break;
          case 5:
            mindrfac = radius_rel_change_threshold_6;
            break;
          case 6:
            mindrfac = radius_rel_change_threshold_7;
            break;
          case 7:
            mindrfac = radius_rel_change_threshold_8;
            break;
          case 8:
            mindrfac = radius_rel_change_threshold_9;
            break;
          case 9:
            mindrfac = radius_rel_change_threshold_10;
            break;
          default:
            assert(0);
          }
          do_recompose = drfac > mindrfac;
          if (do_recompose)
            break;
        } // for rl
        if (do_recompose)
          break;

      } // for n
      if (verbose or veryverbose) {
        if (not do_recompose) {
          CCTK_INFO("Refined regions have not changed sufficiently; skipping "
                    "regridding");
        }
      }
    }
  }

  if (do_recompose) {
    *last_iteration = cctk_iteration;
  }

  if (do_recompose) {

    vector<gh::rregs> &superregsss =
        *static_cast<vector<gh::rregs> *>(superregsss_);
    vector<gh::mregs> &regssss = *static_cast<vector<gh::mregs> *>(regssss_);

    vector<gh::rregs> old_superregsss;
    vector<gh::mregs> old_regssss;
    old_superregsss.swap(superregsss);
    old_regssss.swap(regssss);

    superregsss = old_superregsss; // only needed for output
    BEGIN_MAP_LOOP(cctkGH, CCTK_GF) {
      Regrid(cctkGH, superregsss.at(Carpet::map));
    }
    END_MAP_LOOP;

    vector<vector<vector<region_t> > > regsss(maps);

    // Count levels
    vector<int> rls(maps);
    for (int m = 0; m < maps; ++m) {
      rls.at(m) = superregsss.at(m).size();
    }
    int maxrl = 0;
    for (int m = 0; m < maps; ++m) {
      maxrl = max(maxrl, rls.at(m));
    }
    // All maps must have the same number of levels
    for (int m = 0; m < maps; ++m) {
      superregsss.at(m).resize(maxrl);
      regsss.at(m).resize(maxrl);
    }

    // Make multiprocessor aware
    bool any_level_did_change = false;
    for (int rl = 0; rl < maxrl; ++rl) {
      bool level_did_change = false;
      for (int m = 0; m < maps; ++m) {
        level_did_change =
            level_did_change or int(old_superregsss.at(m).size()) <= rl or
            superregsss.at(m).at(rl) != old_superregsss.at(m).at(rl);
      }
      any_level_did_change = any_level_did_change or level_did_change;

      if (level_did_change) {
        // The level changed: perform domain decomposition

        vector<vector<region_t> > superregss(maps);
        for (int m = 0; m < maps; ++m) {
          superregss.at(m) = superregsss.at(m).at(rl);
        }
        vector<vector<region_t> > regss(maps);
        SplitRegionsMaps(cctkGH, superregss, regss);

        for (int m = 0; m < maps; ++m) {
          superregsss.at(m).at(rl) = superregss.at(m);
          regsss.at(m).at(rl) = regss.at(m);
        }

      } else {
        // The level did not actually change: re-use the old domain
        // decomposition

        for (int m = 0; m < maps; ++m) {
          superregsss.at(m).at(rl).swap(old_superregsss.at(m).at(rl));
          int const ml = 0;
          regsss.at(m).at(rl).swap(old_regssss.at(m).at(ml).at(rl));
        }

      } // if level did change
    }   // for rl

    // Make multigrid aware
    MakeMultigridBoxesMaps(cctkGH, regsss, regssss);

    int lsh[2];
    getvectorindex2(cctkGH, "CarpetRegrid2::radii", lsh);

    // Remember current positions
    for (int n = 0; n < num_centres; ++n) {
      old_active[n] = active[n];

      old_num_levels[n] = num_levels[n];

      old_position_x[n] = position_x[n];
      old_position_y[n] = position_y[n];
      old_position_z[n] = position_z[n];

      for (int rl = 1; rl < num_levels[n]; ++rl) {
        int const ind = index2(lsh, rl, n);
        old_radius_x[ind] = radius_x[ind] < 0 ? radius[ind] : radius_x[ind];
        old_radius_y[ind] = radius_y[ind] < 0 ? radius[ind] : radius_y[ind];
        old_radius_z[ind] = radius_z[ind] < 0 ? radius[ind] : radius_z[ind];
      }
    }

    do_recompose = do_recompose and any_level_did_change;

  } // if do_recompose

  timer.stop();

  return do_recompose;
}

} // namespace CarpetRegrid2