Task: Implement periodic.cc for Carpet/PeriodicCarpet thorn

thorn_name: Carpet/PeriodicCarpet
src_filename: periodic.cc
interface: # Interface definition for thorn PeriodicCarpet

IMPLEMENTS: PeriodicCarpet

INHERITS: grid

USES INCLUDE: carpet.hh
USES INCLUDE: mpi_string.hh
USES INCLUDE: dist.hh



CCTK_INT FUNCTION SymmetryRegister (CCTK_STRING IN sym_name)
REQUIRES FUNCTION SymmetryRegister

CCTK_INT FUNCTION                               \
    SymmetryRegisterGrid                        \
        (CCTK_POINTER IN cctkGH,                \
         CCTK_INT IN sym_handle,                \
         CCTK_INT IN ARRAY which_faces,         \
         CCTK_INT IN ARRAY symmetry_zone_width)
REQUIRES FUNCTION SymmetryRegisterGrid

CCTK_INT FUNCTION                                           \
    SymmetryRegisterGridInterpolator                        \
        (CCTK_POINTER IN cctkGH,                            \
         CCTK_INT IN sym_handle,                            \
         CCTK_INT CCTK_FPOINTER IN symmetry_interpolate     \
             (CCTK_POINTER_TO_CONST IN cctkGH,              \
              CCTK_INT IN N_dims,                           \
              CCTK_INT IN local_interp_handle,              \
              CCTK_INT IN param_table_handle,               \
              CCTK_INT IN coord_system_handle,              \
              CCTK_INT IN N_interp_points,                  \
              CCTK_INT IN interp_coords_type,               \
              CCTK_POINTER_TO_CONST ARRAY IN interp_coords, \
              CCTK_INT IN N_input_arrays,                   \
              CCTK_INT ARRAY IN input_array_indices,        \
              CCTK_INT IN N_output_arrays,                  \
              CCTK_INT ARRAY IN output_array_types,         \
              CCTK_POINTER ARRAY IN output_arrays,          \
              CCTK_INT IN faces))
REQUIRES FUNCTION SymmetryRegisterGridInterpolator

CCTK_INT FUNCTION                                           \
    SymmetryInterpolateFaces                                \
        (CCTK_POINTER_TO_CONST IN cctkGH,                   \
         CCTK_INT IN N_dims,                                \
         CCTK_INT IN local_interp_handle,                   \
         CCTK_INT IN param_table_handle,                    \
         CCTK_INT IN coord_system_handle,                   \
         CCTK_INT IN N_interp_points,                       \
         CCTK_INT IN interp_coords_type,                    \
         CCTK_POINTER_TO_CONST ARRAY IN interp_coords,      \
         CCTK_INT IN N_input_arrays,                        \
         CCTK_INT ARRAY IN input_array_indices,             \
         CCTK_INT IN N_output_arrays,                       \
         CCTK_INT ARRAY IN output_array_types,              \
         CCTK_POINTER ARRAY IN output_arrays,               \
         CCTK_INT IN faces)
REQUIRES FUNCTION SymmetryInterpolateFaces

CCTK_INT FUNCTION                               \
    Boundary_SelectedGVs                        \
        (CCTK_POINTER_TO_CONST IN GH,           \
         CCTK_INT IN array_size,                \
         CCTK_INT ARRAY OUT var_indicies,       \
         CCTK_INT ARRAY OUT faces,              \
         CCTK_INT ARRAY OUT boundary_widths,    \
         CCTK_INT ARRAY OUT table_handles,      \
         CCTK_STRING IN bc_name)
REQUIRES FUNCTION Boundary_SelectedGVs

CCTK_INT FUNCTION                               \
    GetBoundarySpecification                    \
        (CCTK_INT IN size,                      \
         CCTK_INT OUT ARRAY nboundaryzones,     \
         CCTK_INT OUT ARRAY is_internal,        \
         CCTK_INT OUT ARRAY is_staggered,       \
         CCTK_INT OUT ARRAY shiftout)
REQUIRES FUNCTION GetBoundarySpecification

CCTK_INT FUNCTION                               \
    GetDomainSpecification                      \
        (CCTK_INT IN size,                      \
        CCTK_REAL OUT ARRAY physical_min,       \
        CCTK_REAL OUT ARRAY physical_max,       \
        CCTK_REAL OUT ARRAY interior_min,       \
        CCTK_REAL OUT ARRAY interior_max,       \
        CCTK_REAL OUT ARRAY exterior_min,       \
        CCTK_REAL OUT ARRAY exterior_max,       \
        CCTK_REAL OUT ARRAY spacing)
REQUIRES FUNCTION GetDomainSpecification



CCTK_INT FUNCTION                               \
    Periodic_ApplyVI                            \
        (CCTK_POINTER_TO_CONST IN cctkGH,       \
         CCTK_INT IN size,                      \
         CCTK_INT IN ARRAY stencil,             \
         CCTK_INT IN ARRAY do_periodic,         \
         CCTK_INT IN var_index)
PROVIDES FUNCTION Periodic_ApplyVI WITH BndPeriodicCarpetVI LANGUAGE C

CCTK_INT FUNCTION                               \
    Periodic_ApplyVN                            \
        (CCTK_POINTER_TO_CONST IN cctkGH,       \
         CCTK_INT IN size,                      \
         CCTK_INT IN ARRAY stencil,             \
         CCTK_INT IN ARRAY do_periodic,         \
         CCTK_STRING IN var_name)
PROVIDES FUNCTION Periodic_ApplyVN WITH BndPeriodicCarpetVN LANGUAGE C

CCTK_INT FUNCTION                               \
    Periodic_ApplyGI                            \
        (CCTK_POINTER_TO_CONST IN cctkGH,       \
         CCTK_INT IN size,                      \
         CCTK_INT IN ARRAY stencil,             \
         CCTK_INT IN ARRAY do_periodic,         \
         CCTK_INT IN group_index)
PROVIDES FUNCTION Periodic_ApplyGI WITH BndPeriodicCarpetGI LANGUAGE C

CCTK_INT FUNCTION                               \
    Periodic_ApplyGN                            \
        (CCTK_POINTER_TO_CONST IN cctkGH,       \
         CCTK_INT IN size,                      \
         CCTK_INT IN ARRAY stencil,             \
         CCTK_INT IN ARRAY do_periodic,         \
         CCTK_STRING IN group_name)
PROVIDES FUNCTION Periodic_ApplyGN WITH BndPeriodicCarpetGN LANGUAGE C

schedule: # Schedule definitions for thorn PeriodicCarpet

SCHEDULE PeriodicCarpet_ParamCheck AT paramcheck
{
  LANG: C
  OPTIONS: level
} "Register periodic boundary conditions"

SCHEDULE PeriodicCarpet_RegisterBC IN SymmetryRegister
{
  LANG: C
} "Register periodic boundary conditions"

SCHEDULE PeriodicCarpet_ApplyBC IN BoundaryConditions
{
  LANG: C
} "Apply periodic boundary conditions"

param: # Parameter definitions for thorn PeriodicCarpet

BOOLEAN verbose "Produce screen output while applying boundary conditions"
{
} "no"

BOOLEAN periodic "Periodic boundary conditions in all directions"
{
} "no"

BOOLEAN periodic_x "Periodic boundary conditions in x-direction"
{
} "no"

BOOLEAN periodic_y "Periodic boundary conditions in y-direction"
{
} "no"

BOOLEAN periodic_z "Periodic boundary conditions in z-direction"
{
} "no"

configuration: # Configuration definitions for thorn PeriodicCarpet

REQUIRES Carpet CarpetLib

context: interpolate.c:
```
#include <assert.h>
#include <math.h>
#include <stdlib.h>
#include <string.h>

#include "cctk.h"
#include "cctk_Parameters.h"

#include "util_ErrorCodes.h"
#include "util_Table.h"

#include "interpolate.h"



CCTK_INT
PeriodicCarpet_Interpolate (CCTK_POINTER_TO_CONST restrict const cctkGH,
                            CCTK_INT const N_dims,
                            CCTK_INT const local_interp_handle,
                            CCTK_INT const param_table_handle,
                            CCTK_INT const coord_system_handle,
                            CCTK_INT const N_interp_points,
                            CCTK_INT const interp_coords_type,
                            CCTK_POINTER_TO_CONST const interp_coords[],
                            CCTK_INT const N_input_arrays,
                            CCTK_INT const input_array_indices[],
                            CCTK_INT const N_output_arrays,
                            CCTK_INT const output_array_types[],
                            CCTK_POINTER const output_arrays[],
                            CCTK_INT const faces)
{
  DECLARE_CCTK_PARAMETERS;

  int dim=3;
  
  int do_periodic[dim];
  
  CCTK_POINTER_TO_CONST restrict new_interp_coords[dim];
  CCTK_INT newfaces;
  
  int dir;
  
  int iret;
  
  int m;
  int n;
  int d;
  
  int ierr;
  
  CCTK_REAL physical_min[dim];
  CCTK_REAL physical_max[dim];
  CCTK_REAL tmp[dim];
  ierr = GetDomainSpecification(dim, 
                                physical_min, physical_max,
                                tmp, tmp, tmp, tmp, tmp);
  assert(!ierr);
  
  CCTK_REAL periodic_bbox[6] = { physical_min[0], physical_max[0],
                                 physical_min[1], physical_max[1],
                                 physical_min[2], physical_max[2] };



  /* Get symmetry information */
  do_periodic[0] = periodic || periodic_x;
  do_periodic[1] = periodic || periodic_y;
  do_periodic[2] = periodic || periodic_z;
  
  newfaces = faces;
  for (d=0; d<dim; ++d) {
    if (do_periodic[d]) { 
      /* Lower face */
      CCTK_INT dl = 2*d;
      assert (newfaces & (1 << dl));
      newfaces &= ~ (1 << dl);

      /* Upper face */
      CCTK_INT du = 2*d+1;
      assert (newfaces & (1 << du));
      newfaces &= ~ (1 << du);
    }
  }
  
  
  
  /* Fold coordinates */
  assert (interp_coords_type == CCTK_VARIABLE_REAL);
  for (dir=0; dir<dim; ++dir) {
    if (do_periodic[dir]) {
      CCTK_REAL * restrict coords_in_dir
        = malloc (N_interp_points * sizeof (CCTK_REAL));
      assert (N_interp_points == 0 || coords_in_dir);
      
      for (n=0; n<N_interp_points; ++n) {
        CCTK_REAL const pos = ((CCTK_REAL const *)interp_coords[dir])[n];
        CCTK_REAL newpos = pos;
        while (newpos < periodic_bbox[2*dir]) {
          newpos = newpos + periodic_bbox[2*dir+1] - periodic_bbox[2*dir];
        }
        while (newpos > periodic_bbox[2*dir+1]) {
          newpos = newpos - periodic_bbox[2*dir+1] + periodic_bbox[2*dir];
        }
        coords_in_dir[n] = newpos;
      }
      new_interp_coords[dir] = coords_in_dir;
    } else {
      new_interp_coords[dir] = interp_coords[dir];
    }
  }
  

  
  
  /* Recursive call */
  iret = SymmetryInterpolateFaces
    (cctkGH, 
     N_dims, local_interp_handle, param_table_handle, coord_system_handle,
     N_interp_points, interp_coords_type, (CCTK_POINTER_TO_CONST)new_interp_coords,
     N_input_arrays, input_array_indices,
     N_output_arrays, output_array_types, (CCTK_POINTER_TO_CONST)output_arrays,
     newfaces);
  
  
  
  /* Free coordinates */
  for (dir=0; dir<dim; ++dir) {
    if (do_periodic[dir]) {
      free ((void*)new_interp_coords[dir]);
    }
  }
  
  
  /* Return */
  return iret;
}

```

paramcheck.cc:
```
#include <cctk.h>
#include <cctk_Arguments.h>
#include <cctk_Parameters.h>

extern "C" void PeriodicCarpet_ParamCheck(CCTK_ARGUMENTS) {
  DECLARE_CCTK_ARGUMENTS;
  DECLARE_CCTK_PARAMETERS;

  if (periodic and (periodic_x or periodic_y or periodic_z)) {
    CCTK_PARAMWARN("When PeriodicCarpet::periodic is set, all boundaries are "
                   "periodic. It does not make sense to also set any of "
                   "PeriodicCarpet::periodic_[xyz]");
  }
}

```

interpolate.h:
```
#ifndef PERIODICCARPET_H
#define PERIODICCARPET_H

#include "cctk.h"
#include "cctk_Arguments.h"

CCTK_INT
PeriodicCarpet_Interpolate (CCTK_POINTER_TO_CONST restrict const cctkGH,
                            CCTK_INT const N_dims,
                            CCTK_INT const local_interp_handle,
                            CCTK_INT const param_table_handle,
                            CCTK_INT const coord_system_handle,
                            CCTK_INT const N_interp_points,
                            CCTK_INT const interp_coords_type,
                            CCTK_POINTER_TO_CONST const interp_coords[],
                            CCTK_INT const N_input_arrays,
                            CCTK_INT const input_array_indices[],
                            CCTK_INT const N_output_arrays,
                            CCTK_INT const output_array_types[],
                            CCTK_POINTER const output_arrays[],
                            CCTK_INT const faces);

#endif /* ! defined PERIODICCARPET_H */

```

make.code.defn:
```
# Main make.code.defn file for thorn PeriodicCarpet

# Source files in this directory
SRCS = paramcheck.cc periodic.cc interpolate.c

# Subdirectories containing source files
SUBDIRS = 

```


combined_doc_context: ## README (README):
```
Cactus Code Thorn PeriodicCarpet
Author(s)    : Erik Schnetter <eschnetter@perimeterinstitute.ca>
Maintainer(s): Erik Schnetter <eschnetter@perimeterinstitute.ca>
Licence      : GPL
--------------------------------------------------------------------------

1. Purpose

Periodic boundary conditions using Carpet's communication routines.

```

## Documentation (.tex files):
Files: documentation.tex

```tex
=== documentation.tex ===
% *======================================================================*
%  Cactus Thorn template for ThornGuide documentation
%  Author: Ian Kelley
%  Date: Sun Jun 02, 2002
%  $Header$
%
%  Thorn documentation in the latex file doc/documentation.tex
%  will be included in ThornGuides built with the Cactus make system.
%  The scripts employed by the make system automatically include
%  pages about variables, parameters and scheduling parsed from the
%  relevant thorn CCL files.
%
%  This template contains guidelines which help to assure that your
%  documentation will be correctly added to ThornGuides. More
%  information is available in the Cactus UsersGuide.
%
%  Guidelines:
%   - Do not change anything before the line
%       % START CACTUS THORNGUIDE",
%     except for filling in the title, author, date, etc. fields.
%        - Each of these fields should only be on ONE line.
%        - Author names should be separated with a \\ or a comma.
%   - You can define your own macros, but they must appear after
%     the START CACTUS THORNGUIDE line, and must not redefine standard
%     latex commands.
%   - To avoid name clashes with other thorns, 'labels', 'citations',
%     'references', and 'image' names should conform to the following
%     convention:
%       ARRANGEMENT_THORN_LABEL
%     For example, an image wave.eps in the arrangement CactusWave and
%     thorn WaveToyC should be renamed to CactusWave_WaveToyC_wave.eps
%   - Graphics should only be included using the graphicx package.
%     More specifically, with the "\includegraphics" command.  Do
%     not specify any graphic file extensions in your .tex file. This
%     will allow us to create a PDF version of the ThornGuide
%     via pdflatex.
%   - References should be included with the latex "\bibitem" command.
%   - Use \begin{abstract}...\end{abstract} instead of \abstract{...}
%   - Do not use \appendix, instead include any appendices you need as
%     standard sections.
%   - For the benefit of our Perl scripts, and for future extensions,
%     please use simple latex.
%
% *======================================================================*
%
% Example of including a graphic image:
%    \begin{figure}[ht]
% 	\begin{center}
%    	   \includegraphics[width=6cm]{MyArrangement_MyThorn_MyFigure}
% 	\end{center}
% 	\caption{Illustration of this and that}
% 	\label{MyArrangement_MyThorn_MyLabel}
%    \end{figure}
%
% Example of using a label:
%   \label{MyArrangement_MyThorn_MyLabel}
%
% Example of a citation:
%    \cite{MyArrangement_MyThorn_Author99}
%
% Example of including a reference
%   \bibitem{MyArrangement_MyThorn_Author99}
%   {J. Author, {\em The Title of the Book, Journal, or periodical}, 1 (1999),
%   1--16. {\tt http://www.nowhere.com/}}
%
% *======================================================================*

% If you are using CVS use this line to give version information
% $Header$

\documentclass{article}

% Use the Cactus ThornGuide style file
% (Automatically used from Cactus distribution, if you have a
%  thorn without the Cactus Flesh download this from the Cactus
%  homepage at www.cactuscode.org)
\usepackage{../../../../doc/latex/cactus}

\begin{document}

% The author of the documentation
\author{Erik Schnetter \textless eschnetter@perimeterinstitute.ca\textgreater}

% The title of the document (not necessarily the name of the Thorn)
\title{PeriodicCarpet}

% the date your document was last changed, if your document is in CVS,
% please use:
%    \date{$ $Date: 2004-01-07 14:12:39 -0600 (Wed, 07 Jan 2004) $ $}
\date{January 09 2012}

\maketitle

% Do not delete next line
% START CACTUS THORNGUIDE

% Add all definitions used in this documentation here
%   \def\mydef etc

% Add an abstract for this thorn's documentation
\begin{abstract}

\end{abstract}

% The following sections are suggestive only.
% Remove them or add your own.

\section{Introduction}
{\tt PeriodicCarpet} applies periodic boundary conditions to Carpet
grids. In particular, it provides correct behaviour for the case
where a refinement level overlaps the periodic boundaries, and 
may therefore be made up of more than one component on the same
process. Thorn {\tt Periodic}, by constrast, will not handle this case
properly. 

%\section{Numerical Implementation}

\section{Using This Thorn}
The usage is rather simple and resembles that of {\tt Periodic}, 
with boolean parameters {\tt periodic}, {\tt periodic\_x}, {\tt 
periodic\_y} and {\tt periodic\_z}. An important point to notice is 
that if {\tt CarpetRegrid2} is used, its parameters {\tt symmetry\_periodic\_x}, 
{\tt  symmetry\_periodic\_y} and {\tt symmetry\_periodic\_z} have to be
set accordingly.

%\subsection{Obtaining This Thorn}

%\subsection{Basic Usage}

%\subsection{Special Behaviour}

%\subsection{Interaction With Other Thorns}

%\subsection{Examples}

%\subsection{Support and Feedback}

%\section{History}

%\subsection{Thorn Source Code}

%\subsection{Thorn Documentation}

%\subsection{Acknowledgements}


%\begin{thebibliography}{9}

%\end{thebibliography}

% Do not delete next line
% END CACTUS THORNGUIDE

\end{document}

```



Reference solution:
src_code: #include <cassert>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <vector>

#include <cctk.h>
#include <cctk_Arguments.h>
#include <cctk_Parameters.h>

#include <carpet.hh>
#include <mpi_string.hh>
#include <dist.hh>

extern "C" {
#include "interpolate.h"
}

using namespace std;
using namespace Carpet;

// Divide rounding downwards (towards negative infinity)
static int divdn(int const a, int const b) {
  assert(b > 0);
  // Note: C++ division rounds towards zero
  return a >= 0 ? a / b : (a - (b - 1)) / b;
}

// Round downwards towards next multiple
static int rounddn(int const a, int const b) {
  assert(b > 0);
  return divdn(a, b) * b;
}

struct xferinfo_t {
  int m;                            // map
  sendrecv_pseudoregion_t sendrecv; // regions and components
  islab slab;                       // slabbing offset
};

MPI_Datatype mpi_datatype(xferinfo_t const &);
namespace dist {
template <> inline MPI_Datatype mpi_datatype<xferinfo_t>() {
  xferinfo_t dummy;
  return mpi_datatype(dummy);
}
}

MPI_Datatype mpi_datatype(xferinfo_t const &) {
  static bool initialised = false;
  static MPI_Datatype newtype;
  if (not initialised) {
    static xferinfo_t s;
#define ENTRY(type, name)                                                      \
  {                                                                            \
    sizeof s.name / sizeof(type),         /* count elements */                 \
        (char *) & s.name - (char *) & s, /* offsetof doesn't work (why?) */   \
        dist::mpi_datatype<type>(),       /* find MPI datatype */              \
        STRINGIFY(name),                  /* field name */                     \
        STRINGIFY(type),                  /* type name */                      \
  }
    dist::mpi_struct_descr_t const descr[] = {
        ENTRY(int, m),
        ENTRY(sendrecv_pseudoregion_t, sendrecv),
        ENTRY(islab, slab),
        {1, sizeof s, MPI_UB, "MPI_UB", "MPI_UB"}};
#undef ENTRY
    newtype = dist::create_mpi_datatype(sizeof descr / sizeof descr[0], descr,
                                        "xferinfo_t", sizeof s);
    initialised = true;
  }
  return newtype;
}

struct levelinfo_t {
  int regridding_epoch;
  vector<xferinfo_t> xferinfos;
  levelinfo_t() : regridding_epoch(-1) {}
};

static vector<levelinfo_t> levelinfos;

static void periodic_carpet(cGH const *restrict const cctkGH, int const size,
                            CCTK_INT const *restrict const stencil,
                            CCTK_INT const do_periodic[3],
                            CCTK_INT const *restrict const vars,
                            int const nvars) {
  DECLARE_CCTK_PARAMETERS;

  // Check arguments
  assert(cctkGH);
  assert(stencil);
  assert(vars);

  // Early return if possible, because the code below requires at
  // least one variable
  if (nvars == 0)
    return;

  // Get and check group info
  cGroup group;
  cGroupDynamicData data;
  for (int i = 0; i < nvars; ++i) {
    int const vi = vars[i];
    assert(vi >= 0 and vi < CCTK_NumVars());
    int const gi = CCTK_GroupIndexFromVarI(vi);
    assert(gi >= 0 and gi < CCTK_NumGroups());

    int ierr = CCTK_GroupData(gi, &group);
    assert(not ierr);
    assert(group.grouptype == CCTK_GF);
    assert(group.disttype == CCTK_DISTRIB_DEFAULT);
    int const vartypesize = CCTK_VarTypeSize(group.vartype);
    assert(vartypesize > 0);

    if (group.dim != size) {
      CCTK_VWarn(CCTK_WARN_ABORT, __LINE__, __FILE__, CCTK_THORNSTRING,
                 "The group \"%s\" has dimension %d, but the given stencil has "
                 "size %d.",
                 CCTK_GroupNameFromVarI(vi), group.dim, size);
    }

    ierr = CCTK_GroupDynamicData(cctkGH, gi, &data);
    assert(not ierr);
  }

  // Ensure we are in level mode (and not global mode)
  assert(reflevel >= 0);

  // The cached data structures, containing the communication schedule
  // for each refinement level
  int const ml = mglevel;
  int const rl = reflevel;
  int const tl = 0;

  if (reflevels >= (int)levelinfos.size()) {
    levelinfos.resize(reflevels);
  }
  levelinfo_t &levelinfo = levelinfos.at(reflevel);
  vector<xferinfo_t> &xferinfos = levelinfo.xferinfos;

  // Do we need a new levelinfo? (We need a new levelinfo after each
  // regridding.)
  if (levelinfo.regridding_epoch != level_regridding_epochs.at(reflevel)) {
    levelinfo.regridding_epoch = level_regridding_epochs.at(reflevel);
    xferinfos.clear();

    // Loop over all components and find out how to fill their
    // periodic boundaries

      gh const &hh = *vhh.at(Carpet::map);
      ibbox const &domain_exterior = hh.baseextent(mglevel, reflevel);
      i2vect const &boundary_width = hh.boundary_width;
      ibbox const domain_active = domain_exterior.expand(-boundary_width);
      dh const &dd = *vdd.at(Carpet::map);
      dh::light_cboxes const &light_level =
          dd.light_boxes.AT(mglevel).AT(reflevel);

      // Interior of the domain with respect to periodic boundaries,
      // i.e. periodic boundaries cut off, but all other boundaries
      // still included
      ibbox domain_perint;
      {
        ivect lo, hi, str;
        for (int d = 0; d < dim; ++d) {
          ibbox const &dom = do_periodic[d] ? domain_active : domain_exterior;
          lo[d] = dom.lower()[d];
          hi[d] = dom.upper()[d];
          str[d] = dom.stride()[d];
        }
        domain_perint = ibbox(lo, hi, str);
      }

      // CCTK_INT width[2*dim];
      // CCTK_INT is_internal[2*dim];
      // CCTK_INT is_staggered[2*dim];
      // CCTK_INT shiftout[2*dim];
      // int ierr = GetBoundarySpecification
      //   (2*dim, width, is_internal, is_staggered, shiftout);
      // assert(not ierr);

      // // Domain size
      // CCTK_REAL physical_min[dim];
      // CCTK_REAL physical_max[dim];
      // CCTK_REAL interior_min[dim];
      // CCTK_REAL interior_max[dim];
      // CCTK_REAL exterior_min[dim];
      // CCTK_REAL exterior_max[dim];
      // CCTK_REAL spacing[dim];
      // ierr = GetDomainSpecification(dim,
      //                               physical_min, physical_max,
      //                               interior_min, interior_max,
      //                               exterior_min, exterior_max,
      //                               spacing);
      // assert(not ierr);

      // Domain size on this level, in terms of grid points
      // ivect npoints;
      // for (int d=0; d<dim; ++d) {
      //   CCTK_REAL const rnpoints =
      //     (exterior_max[d] - exterior_min[d]) / spacing[d]
      //     + 1 - (width[2*d] + width[2*d+1]);
      //   npoints[d] = floor(rnpoints + 0.01);
      //   // Ensure the domain size is integer
      //   assert(fabs(npoints[d] - rnpoints) < 0.01);
      // }

        DECLARE_CCTK_ARGUMENTS;

        // Exterior of this component
        ibbox const &ext = light_level.at(component).exterior;

        // Collect all periodic boundaries for this component
        ibset dst_bset;
        for (int d = 0; d < dim; ++d) {
          for (int f = 0; f < 2; ++f) {
            if (cctk_bbox[2 * d + f] and do_periodic[d]) {
              // This is a periodic boundary
              ivect lo = ext.lower();
              ivect up = ext.upper();
              ivect const str = ext.stride();
              int const bnd = boundary_width[f][d];
              if (f == 0) {
                up[d] = lo[d] + (bnd - 1) * str[d];
              } else {
                lo[d] = up[d] - (bnd - 1) * str[d];
              }
              ibbox const dst_bbox(lo, up, str);
              assert(dst_bbox.shape()[d] / dst_bbox.stride()[d] == bnd);
              dst_bset |= dst_bbox;
            }
          }
        }

        // Determine the source bboxes
        while (not dst_bset.empty()) {

          // Pick one (arbitrary) dst_bbox
          ibbox const &dst_bbox1 = *dst_bset.begin();

          // Find the slabbing offset (a multiple of the domain size)
          // that moves dst_bbox1 into the domain
          islab slab;
          for (int d = 0; d < dim; ++d) {
            if (not do_periodic[d]) {
              slab.offset[d] = 0;
            } else {
              assert((dst_bbox1.lower()[d] - domain_perint.lower()[d]) %
                         domain_perint.stride()[d] ==
                     0);
              slab.offset[d] =
                  -rounddn(dst_bbox1.lower()[d] - domain_perint.lower()[d],
                           domain_perint.shape()[d]) /
                  domain_perint.stride()[d];
            }
          }
          assert(not all(slab.offset == 0)); // this would be trivial

          // Determine corresponding source bbox
          ibbox const src_bbox1 = dst_bbox1.shift(slab.offset);

          // Ensure we copy from within the domain, not from boundary
          // points
          ibbox const src_bbox = src_bbox1 & domain_perint;
          assert(not src_bbox.empty());
          ibbox const dst_bbox = src_bbox.shift(-slab.offset);
          assert(dst_bbox.is_contained_in(dst_bbox1));
          dst_bset -= dst_bbox;

          // Find source components for all points
          ibset src_bset(src_bbox);
          for (int oc = 0; oc < hh.components(reflevel); ++oc) {
            // We cannot copy only from the interior (the "good"
            // points), because this excludes the ghost zones, and we
            // may need to copy from prolongation boundaries. (This
            // assumes that synchronisation and prolongation occurs
            // before periodic boundaries are applied!)
            ibset const intersection = src_bset & light_level.at(oc).exterior;
            if (not intersection.empty()) {
              src_bset -= intersection;

              // Loop over the intersection and insert respective
              // xferinfos
              ibset::const_iterator const ib = intersection.begin();
              ibset::const_iterator const ie = intersection.end();
              for (ibset::const_iterator ireg = ib; ireg != ie; ++ireg) {
                ibbox const src_reg = *ireg;
                ibbox const dst_reg = src_reg.shift(-slab.offset);
                xferinfo_t xferinfo;
                xferinfo.m = Carpet::map;
                xferinfo.sendrecv.send.extent = src_reg;
                xferinfo.sendrecv.send.component = oc;
                xferinfo.sendrecv.recv.extent = dst_reg;
                xferinfo.sendrecv.recv.component = component;
                xferinfo.slab = slab;
                xferinfos.push_back(xferinfo);
              }
            }
          } // for oc
          // Ensure we know how to handle all points
          assert(src_bset.empty());

        } // while dst_bset

    // Tell the source processes what they have to send
    {
      int const p = dist::rank();
      vector<vector<xferinfo_t> > sends(dist::size());
      vector<xferinfo_t>::const_iterator const ib = xferinfos.begin();
      vector<xferinfo_t>::const_iterator const ie = xferinfos.end();
      for (vector<xferinfo_t>::const_iterator ixferinfo = ib; ixferinfo != ie;
           ++ixferinfo) {
        xferinfo_t const &xferinfo = *ixferinfo;
        gh const &hh = *vhh.at(xferinfo.m);
        int const oc = xferinfo.sendrecv.send.component;
        int const op = hh.processor(rl, oc);
        if (op != p) {
          sends.at(op).push_back(xferinfo);
        }
      }

      vector<xferinfo_t> const recv = alltoallv1(dist::comm(), sends);

      xferinfos.insert(xferinfos.end(), recv.begin(), recv.end());
    }

  } // if need new levelinfo

  // Transfer: Loop over all communication phases, all variables, and
  // all regions
  for (comm_state state; not state.done(); state.step()) {
    for (int n = 0; n < nvars; ++n) {
      int const vi = vars[n];
      int const gi = CCTK_GroupIndexFromVarI(vi);
      int const v0 = CCTK_FirstVarIndexI(gi);

      vector<xferinfo_t>::const_iterator const ib = xferinfos.begin();
      vector<xferinfo_t>::const_iterator const ie = xferinfos.end();
      for (vector<xferinfo_t>::const_iterator ixferinfo = ib; ixferinfo != ie;
           ++ixferinfo) {
        xferinfo_t const &xferinfo = *ixferinfo;
        gh const &hh = *vhh.at(xferinfo.m);
        ggf &ff = *arrdata.at(gi).at(xferinfo.m).data.at(vi - v0);

        // Determine components, local components, processes
        int const oc = xferinfo.sendrecv.send.component;
        int const c = xferinfo.sendrecv.recv.component;
        int const olc = hh.get_local_component(rl, oc);
        int const lc = hh.get_local_component(rl, c);
        int const op = hh.processor(rl, oc);
        int const p = hh.processor(rl, c);

        // Get pointers to the variable's data
        gdata *const src =
            hh.is_local(rl, oc) ? ff.data_pointer(tl, rl, olc, ml) : NULL;
        gdata *const dst =
            hh.is_local(rl, c) ? ff.data_pointer(tl, rl, lc, ml) : NULL;

        // Copy
        ibbox const &dstbox = xferinfo.sendrecv.recv.extent;
        ibbox const &srcbox = xferinfo.sendrecv.send.extent;
        islab const &slab = xferinfo.slab;
        dst->copy_from(state, src, dstbox, srcbox, &slab, p, op);
      }
    }
  }
}

extern "C" CCTK_INT BndPeriodicCarpetVI(CCTK_POINTER_TO_CONST const cctkGH_,
                                        CCTK_INT const size,
                                        CCTK_INT const *restrict const stencil,
                                        CCTK_INT const do_periodic[3],
                                        CCTK_INT const vi) {
  cGH const *restrict const cctkGH = static_cast<cGH const *>(cctkGH_);
  CCTK_INT const vi1 = vi;
  periodic_carpet(cctkGH, size, stencil, do_periodic, &vi1, 1);
  return 0;
}

extern "C" CCTK_INT BndPeriodicCarpetVN(CCTK_POINTER_TO_CONST const cctkGH_,
                                        CCTK_INT const size,
                                        CCTK_INT const *restrict const stencil,
                                        CCTK_INT const do_periodic[3],
                                        char const *restrict const vn) {
  CCTK_INT const vi = CCTK_VarIndex(vn);
  assert(vi >= 0 and vi < CCTK_NumVars());
  BndPeriodicCarpetVI(cctkGH_, size, stencil, do_periodic, vi);
  return 0;
}

extern "C" CCTK_INT BndPeriodicCarpetGI(CCTK_POINTER_TO_CONST const cctkGH_,
                                        CCTK_INT const size,
                                        CCTK_INT const *restrict const stencil,
                                        CCTK_INT const do_periodic[3],
                                        CCTK_INT const gi) {
  cGH const *restrict const cctkGH = static_cast<cGH const *>(cctkGH_);
  int const nvars = CCTK_NumVarsInGroupI(gi);
  vector<CCTK_INT> vis(nvars);
  int const v0 = CCTK_FirstVarIndexI(gi);
  for (int vi = 0; vi < nvars; ++vi)
    vis.at(vi) = v0 + vi;
  periodic_carpet(cctkGH, size, stencil, do_periodic, &vis.front(), nvars);
  return 0;
}

extern "C" CCTK_INT BndPeriodicCarpetGN(CCTK_POINTER_TO_CONST const cctkGH_,
                                        CCTK_INT const size,
                                        CCTK_INT const *restrict const stencil,
                                        CCTK_INT const do_periodic[3],
                                        char const *restrict const gn) {
  int const gi = CCTK_GroupIndex(gn);
  assert(gi >= 0 and gi < CCTK_NumGroups());
  BndPeriodicCarpetGI(cctkGH_, size, stencil, do_periodic, gi);
  return 0;
}

extern "C" void PeriodicCarpet_RegisterBC(CCTK_ARGUMENTS) {
  DECLARE_CCTK_ARGUMENTS;
  DECLARE_CCTK_PARAMETERS;

  CCTK_INT faces[6];
  faces[0] = faces[1] = periodic or periodic_x;
  faces[2] = faces[3] = periodic or periodic_y;
  faces[4] = faces[5] = periodic or periodic_z;

  CCTK_INT width[6];
  CCTK_INT is_internal[6];
  CCTK_INT is_staggered[6];
  CCTK_INT shiftout[6];
  int ierr =
      GetBoundarySpecification(6, width, is_internal, is_staggered, shiftout);
  if (ierr < 0) {
    CCTK_WARN(CCTK_WARN_ABORT, "Could not get the boundary specification");
  }

  CCTK_INT const handle = SymmetryRegister("periodic");
  if (handle < 0) {
    CCTK_WARN(CCTK_WARN_ABORT,
              "Could not register periodicity boundary condition");
  }

  ierr = SymmetryRegisterGrid(cctkGH, handle, faces, width);
  if (ierr < 0) {
    CCTK_WARN(CCTK_WARN_ABORT, "Could not register the periodic boundaries -- "
                               "probably some other thorn has already "
                               "registered the same boundary faces for a "
                               "different symmetry");
  }

  ierr = SymmetryRegisterGridInterpolator(cctkGH, handle, PeriodicCarpet_Interpolate);
  if (ierr < 0)
  {
    CCTK_WARN (0, "Could not register the symmetry interpolator");
  }
}

extern "C" void PeriodicCarpet_ApplyBC(CCTK_ARGUMENTS) {
  DECLARE_CCTK_ARGUMENTS;
  DECLARE_CCTK_PARAMETERS;

  assert(cctkGH);

  CCTK_INT do_periodic[dim];
  assert(dim == 3);
  do_periodic[0] = periodic or periodic_x;
  do_periodic[1] = periodic or periodic_y;
  do_periodic[2] = periodic or periodic_z;

  CCTK_INT const nvars = Boundary_SelectedGVs(cctkGH, 0, 0, 0, 0, 0, 0);
  assert(nvars >= 0);
  if (nvars == 0)
    return;

  vector<CCTK_INT> indices(nvars);
  vector<CCTK_INT> faces(nvars);
  vector<CCTK_INT> widths(nvars);
  vector<CCTK_INT> tables(nvars);
  int const iret =
      Boundary_SelectedGVs(cctkGH, nvars, &indices.front(), &faces.front(),
                           &widths.front(), &tables.front(), 0);
  assert(iret == nvars);

  CCTK_INT width[2 * dim];
  CCTK_INT is_internal[2 * dim];
  CCTK_INT is_staggered[2 * dim];
  CCTK_INT shiftout[2 * dim];
  int ierr = GetBoundarySpecification(2 * dim, width, is_internal, is_staggered,
                                      shiftout);
  if (ierr < 0) {
    CCTK_WARN(CCTK_WARN_ABORT, "Could not get the boundary specification");
  }

  CCTK_INT stencil[dim];
  for (int d = 0; d < dim; ++d) {
    assert(width[d] == width[d + 1]);
    stencil[d] = width[2 * d];
  }

  for (int n = 0; n < nvars; ++n) {
    int const vi = indices.at(n);
    assert(vi >= 0 and vi < CCTK_NumVars());

    if (verbose) {
      char *const fullname = CCTK_FullName(vi);
      assert(fullname);
      CCTK_VInfo(CCTK_THORNSTRING,
                 "Applying periodicity boundary conditions to \"%s\"",
                 fullname);
      free(fullname);
    }
  } // for n

  periodic_carpet(cctkGH, dim, stencil, do_periodic, &indices.front(), nvars);
}

namespace CarpetLib {
template vector<xferinfo_t> alltoallv1(MPI_Comm comm,
                                       vector<vector<xferinfo_t> > const &data);
}