Task: Implement evolution_mask_enforce.c for Carpet/CarpetEvolutionMask thorn

thorn_name: Carpet/CarpetEvolutionMask
src_filename: evolution_mask_enforce.c
interface: # Interface definition for thorn CarpetEvolutionMask

IMPLEMENTS: CarpetEvolutionMask



CCTK_INT FUNCTION MoLQueryEvolvedRHS (CCTK_INT IN EvolvedIndex)
REQUIRES FUNCTION MoLQueryEvolvedRHS



USES INCLUDE HEADER: dist.hh
USES INCLUDE HEADER: vect.hh

USES INCLUDE HEADER: carpet.hh



PUBLIC:

CCTK_REAL evolution_mask TYPE=gf TAGS='prolongation="none" InterpNumTimelevels=1' "mask function"

# value key:
# 0 - grid point is not in buffer region
# 1 - grid point is in buffer region
CCTK_INT buffer_mask TYPE=gf TAGS='prolongation="none" InterpNumTimelevels=1' "buffer mask function"

schedule: # Schedule definitions for thorn CarpetEvolutionMask

STORAGE: evolution_mask

if (provide_buffer_mask) {
  STORAGE: buffer_mask
}


SCHEDULE GROUP EvolutionMaskBase_SetupEvolutionMask AT basegrid
{
} "Set up the mask function"

SCHEDULE GROUP EvolutionMaskBase_SetupEvolutionMask AT postregridinitial
{
} "Set up the mask function"

SCHEDULE GROUP EvolutionMaskBase_SetupEvolutionMask AT postregrid
{
} "Set up the mask function"



SCHEDULE EvolutionMaskBase_InitEvolutionMask IN EvolutionMaskBase_SetupEvolutionMask
{
  LANG: C
  OPTIONS: global loop-local
} "Initialise the mask function"

SCHEDULE GROUP SetupEvolutionMask IN EvolutionMaskBase_SetupEvolutionMask AFTER EvolutionMaskBase_InitEvolutionMask
{
} "Set up the weight function (schedule other routines in here)"



SCHEDULE CarpetEvolutionMaskSetup IN SetupEvolutionMask
{
  LANG: C
  OPTIONS: global loop-singlemap
} "Set up the mask function for the restriction regions"

if (enforce_mask)
{
  SCHEDULE enforce_evolution_mask IN MoL_PostRHS
  {
    LANG: C
  } "Enforce Evolution Mask"
}

param: # Parameter definitions for thorn CarpetEvolutionMask



BOOLEAN verbose "Do verbose debugging output" STEERABLE=always
{
} "no"

BOOLEAN enforce_mask "Set RHS to zero where mask value is 0?" STEERABLE=recover
{
} "no"



BOOLEAN writeNaNs "Write NaNs into masked regions" STEERABLE=always
{
} "no"

STRING enforce_vars "Enforce mask in MoL automatically on the following variables" STEERABLE=always
{
  "" :: "a list of variables or groups"
} ""

INT enlarge_evolved_region_by "enlarge or shrink evolved region" STEERABLE=always
{
 0:*  :: "positive values enlarge the evolved region"
 *:0) :: "negative values shrink the evolved region. Up to buffer_width + nghostzones points can be removed"
} 0

BOOLEAN provide_buffer_mask "Compute mask indicating which points need only be computed for some MoL substeps"
{
} "no"

configuration: # Configuration definition for thorn CarpetEvolutionMask

REQUIRES Carpet

context: make.code.defn:
```
# Main make.code.defn file for thorn CarpetEvolutionMask

# Source files in this directory
SRCS = evolution_mask.cc evolution_mask_init.c evolution_mask_enforce.c

# Subdirectories containing source files
SUBDIRS = 

```

evolution_mask.hh:
```
#include "cctk.h"
#include "cctk_Arguments.h"

namespace CarpetEvolutionMask {

extern "C" {
void CarpetEvolutionMaskSetup(CCTK_ARGUMENTS);
}

} // namespace CarpetEovlutionMask

```

evolution_mask_init.c:
```
#include "cctk.h"
#include "cctk_Arguments.h"
#include "cctk_Parameters.h"

#include "util_Table.h"

void EvolutionMaskBase_InitEvolutionMask(CCTK_ARGUMENTS) {
  DECLARE_CCTK_ARGUMENTS;
  DECLARE_CCTK_PARAMETERS;

  int i, j, k;

  /* Initialise the mask to 1 everywhere */
  if (verbose) {
    CCTK_INFO("Initialising the mask to 1");
  }

  for (k = 0; k < cctk_lsh[2]; ++k) {
    for (j = 0; j < cctk_lsh[1]; ++j) {
      for (i = 0; i < cctk_lsh[0]; ++i) {

        int const ind = CCTK_GFINDEX3D(cctkGH, i, j, k);
        evolution_mask[ind] = 1.0;
        if (provide_buffer_mask) {
          buffer_mask[ind] = 0;
        }
      }
    }
  }
}

```

evolution_mask.cc:
```
#include <cassert>
#include <sstream>

#include "cctk.h"
#include "cctk_Arguments.h"
#include "cctk_Parameters.h"

#include "carpet.hh"

#include "evolution_mask.hh"

namespace CarpetEvolutionMask {

using namespace std;
using namespace Carpet;

void CarpetEvolutionMaskSetup(CCTK_ARGUMENTS) {
  DECLARE_CCTK_PARAMETERS;

  if (!is_singlemap_mode()) {
    CCTK_WARN(0, "This routine may only be called in singlemap mode");
  }

  if (reflevel > 0) {

    ivect const izero = ivect(0);
    ivect const ione = ivect(1);

    gh const &hh = *vhh.at(Carpet::map);
    dh const &dd = *vdd.at(Carpet::map);

    ibbox const base = hh.baseextents.at(mglevel).at(reflevel).expand(100, 100);
    ibbox const coarsebase =
        hh.baseextents.at(mglevel).at(reflevel - 1).expand(100, 100);

    ivect const reffact =
        spacereffacts.at(reflevel) / spacereffacts.at(reflevel - 1);
    assert(all(reffact == 2));

    const i2vect buffer_widths = dd.buffer_widths.at(reflevel);
    // const i2vect overlap_widths = dd.overlap_widths.at(reflevel);
    // overlap_widths do not need to be taken into account since coincide with
    // the prolongation stencil
    const i2vect ghost_widths = dd.ghost_widths.at(reflevel);

    // cout << "base: " << base << endl;
    // cout << "coarsebase: " << coarsebase << endl;

    // Calculate the union of all refined regions
    // TODO: we really want allrestricted but that is not stored and would have
    //       to be copied out of dh.cc/regrid()
    ibset refined;
    for (int c = 0; c < hh.components(reflevel); ++c) {
      ibbox refcomp = hh.extent(mglevel, reflevel, c);
      b2vect outer_boundary = hh.outer_boundaries(reflevel, c);
      ivect expand_right, expand_left;
      for (int d = 0; d < dim; d++) {
        if (outer_boundary[0][d]) {
          expand_left[d] = 50;
        } else {
          expand_left[d] = 0;
        }
        if (outer_boundary[1][d]) {
          expand_right[d] = 50;
        } else {
          expand_right[d] = 0;
        }
      }
      //	cout << "rl,c,expand_left,expand_right" << " " << reflevel << "
      //" << c
      //	     << " " << expand_left << " " << expand_right << endl;
      refcomp = refcomp.expand(expand_left, expand_right);
      refined |= refcomp;
    }

    //      cout << "refined: " << refined << endl;

    // calculate inverse set of refined regions on the current level
    ibset antirefined = base - refined;

    //      cout << "antirefined: " << antirefined << endl;

    // now make antibase larger
    // make refined regions SMALLER
    // TODO: this is wrong at outer boundaries (which are unlikely to exist on
    //     rl>0) and symmetry boundaries (which are likely to also exist on
    //     rl>0)
    i2vect antishrinkby;
    for (int f = 0; f < 2; f++) {
      for (int d = 0; d < dim; d++) {
        antishrinkby[f][d] = ghost_widths[f][d] + buffer_widths[f][d] +
                             enlarge_evolved_region_by;
        if (hh.refcent == vertex_centered) {
          antishrinkby[f][d] += 1; // remove the weight = 1/2 points so that
                                   // reduction operation are unaffected
        }
      }
    }
    ibset antishrunk(antirefined.expand(antishrinkby).expanded_for(coarsebase));
    // TODO: look up how this is done in regrid.cc
    ibset buffers(antirefined.expand(antishrinkby) & base);
    buffers -= antirefined;
    buffers &= base;

    //      cout << "antishrunk1: " << antishrunk << endl;

    // now cut away dangling edges
    antishrunk &= coarsebase;

    //      cout << "antishrunk2: " << antishrunk << endl;

    // cut holes into coarsebase
    ibset const shrunk = coarsebase - antishrunk;

    //      cout << "shrunk: " << shrunk << endl;

    // Calculate the union of all coarse regions
    ibset parent;
    for (int c = 0; c < hh.components(reflevel - 1); ++c) {
      parent |= hh.extent(mglevel, reflevel - 1, c);
    }

    //      cout << "parent: " << parent << endl;

    // Subtract the refined region
    ibset const notrefined = parent - shrunk;

    //      cout << "notrefined: " << notrefined << endl;

    i2vect enlargeby;
    int stencil_size = dd.prolongation_stencil_size(reflevel);
    for (int f = 0; f < 2; f++) {
      for (int d = 0; d < dim; d++) {
        enlargeby[f][d] =
            cctkGH->cctk_nghostzones[d] + buffer_widths[f][d] + stencil_size;
      }
    }
    ibset const enlarged(notrefined.expand(enlargeby));

    //      cout << "enlarged: " << enlarged << endl;

    // Intersect with the original union
    ibset const evolveon = parent & enlarged;

    ibset const notevolveon = parent - evolveon;

    //      cout << "notevolveon: " << notevolveon << endl;

    // Set not evolved region on next coarser level
    {
      int const oldreflevel = reflevel;
      int const oldgrouptype = mc_grouptype;
      int const oldmap = Carpet::map;
      leave_singlemap_mode(cctkGH);
      leave_level_mode(cctkGH);
      enter_level_mode(cctkGH, oldreflevel - 1);
      enter_singlemap_mode(cctkGH, oldmap, oldgrouptype);

      BEGIN_LOCAL_COMPONENT_LOOP(cctkGH, CCTK_GF) {

        DECLARE_CCTK_ARGUMENTS;

        ibbox const &ext =
            dd.light_boxes.at(mglevel).at(reflevel).at(component).exterior;

        for (ibset::const_iterator bi = notevolveon.begin();
             bi != notevolveon.end(); ++bi) {

          ibbox const &box = (*bi) & ext;
          if (!box.empty()) {

            assert(all((box.lower() - ext.lower()) >= 0));
            assert(all((box.upper() - ext.lower() + ext.stride()) >= 0));
            assert(all((box.lower() - ext.lower()) % ext.stride() == 0));
            assert(
                all((box.upper() - ext.lower() + ext.stride()) % ext.stride() ==
                    0));
            ivect const imin = (box.lower() - ext.lower()) / ext.stride();
            ivect const imax =
                (box.upper() - ext.lower() + ext.stride()) / ext.stride();
            assert(all(izero <= imin));
            assert(box.empty() || all(imin <= imax));
            assert(all(imax <= ivect::ref(cctk_lsh)));

            if (verbose) {
              ostringstream buf;
              buf << "Setting restricted region on level " << reflevel
                  << " to weight 0: " << imin << ":" << imax - ione;
              CCTK_INFO(buf.str().c_str());
            }

            // Set mask in the restricted region to 0
            assert(dim == 3);
            for (int k = imin[2]; k < imax[2]; ++k) {
              for (int j = imin[1]; j < imax[1]; ++j) {
                for (int i = imin[0]; i < imax[0]; ++i) {
                  int const ind = CCTK_GFINDEX3D(cctkGH, i, j, k);
                  evolution_mask[ind] = 0;
                }
              }
            }

          } // if box not empty

        } // for box
      }
      END_LOCAL_COMPONENT_LOOP;

      leave_singlemap_mode(cctkGH);
      leave_level_mode(cctkGH);
      enter_level_mode(cctkGH, oldreflevel);
      enter_singlemap_mode(cctkGH, oldmap, oldgrouptype);
    }

    // Indicate which points are in the buffer region on current level
    if (provide_buffer_mask) {
      BEGIN_LOCAL_COMPONENT_LOOP(cctkGH, CCTK_GF) {

        DECLARE_CCTK_ARGUMENTS;

        ibbox const &ext =
            dd.light_boxes.at(mglevel).at(reflevel).at(component).exterior;

        for (ibset::const_iterator bi = buffers.begin(); bi != buffers.end();
             ++bi) {

          ibbox const &box = (*bi) & ext;
          if (!box.empty()) {

            assert(all((box.lower() - ext.lower()) >= 0));
            assert(all((box.upper() - ext.lower() + ext.stride()) >= 0));
            assert(all((box.lower() - ext.lower()) % ext.stride() == 0));
            assert(
                all((box.upper() - ext.lower() + ext.stride()) % ext.stride() ==
                    0));
            ivect const imin = (box.lower() - ext.lower()) / ext.stride();
            ivect const imax =
                (box.upper() - ext.lower() + ext.stride()) / ext.stride();
            assert(all(izero <= imin));
            assert(box.empty() || all(imin <= imax));
            assert(all(imax <= ivect::ref(cctk_lsh)));

            if (verbose) {
              ostringstream buf;
              buf << "Setting buffer region on level " << reflevel << ": "
                  << imin << ":" << imax - ione;
              CCTK_INFO(buf.str().c_str());
            }

            // Set mask in the buffer region to 1
            assert(dim == 3);
            for (int k = imin[2]; k < imax[2]; ++k) {
              for (int j = imin[1]; j < imax[1]; ++j) {
                for (int i = imin[0]; i < imax[0]; ++i) {
                  int const ind = CCTK_GFINDEX3D(cctkGH, i, j, k);
                  buffer_mask[ind] = 1;
                }
              }
            }

          } // if box not empty

        } // for box
      }
      END_LOCAL_COMPONENT_LOOP;
    }

  } // if reflevel>0
}

} // namespace CarpetEvolutionMask

```


combined_doc_context: ## README (README):
```
Cactus Code Thorn CarpetEvolutionMask
Author(s)    : Christian D. Ott <cott@aei.mpg.de>
               Erik Schnetter <schnetter@cct.lsu.edu>
Maintainer(s): Christian D. Ott <cott@aei.mpg.de>
               Erik Schnetter <schnetter@cct.lsu.edu>
Licence      : GPLv2+
--------------------------------------------------------------------------

1. Purpose

Provide a grid function "evolution_mask" which is zero for all grid
points that do not need to be evolved because they are going to be
restricted from a finer grid.

```

## Documentation (.tex files):
Files: documentation.tex

```tex
=== documentation.tex ===
% *======================================================================*
%  Cactus Thorn template for ThornGuide documentation
%  Author: Ian Kelley
%  Date: Sun Jun 02, 2002
%  $Header: /cactusdevcvs/Cactus/doc/ThornGuide/template.tex,v 1.12 2004/01/07 20:12:39 rideout Exp $
%
%  Thorn documentation in the latex file doc/documentation.tex
%  will be included in ThornGuides built with the Cactus make system.
%  The scripts employed by the make system automatically include
%  pages about variables, parameters and scheduling parsed from the
%  relevant thorn CCL files.
%
%  This template contains guidelines which help to assure that your
%  documentation will be correctly added to ThornGuides. More
%  information is available in the Cactus UsersGuide.
%
%  Guidelines:
%   - Do not change anything before the line
%       % START CACTUS THORNGUIDE",
%     except for filling in the title, author, date, etc. fields.
%        - Each of these fields should only be on ONE line.
%        - Author names should be separated with a \\ or a comma.
%   - You can define your own macros, but they must appear after
%     the START CACTUS THORNGUIDE line, and must not redefine standard
%     latex commands.
%   - To avoid name clashes with other thorns, 'labels', 'citations',
%     'references', and 'image' names should conform to the following
%     convention:
%       ARRANGEMENT_THORN_LABEL
%     For example, an image wave.eps in the arrangement CactusWave and
%     thorn WaveToyC should be renamed to CactusWave_WaveToyC_wave.eps
%   - Graphics should only be included using the graphicx package.
%     More specifically, with the "\includegraphics" command.  Do
%     not specify any graphic file extensions in your .tex file. This
%     will allow us to create a PDF version of the ThornGuide
%     via pdflatex.
%   - References should be included with the latex "\bibitem" command.
%   - Use \begin{abstract}...\end{abstract} instead of \abstract{...}
%   - Do not use \appendix, instead include any appendices you need as
%     standard sections.
%   - For the benefit of our Perl scripts, and for future extensions,
%     please use simple latex.
%
% *======================================================================*
%
% Example of including a graphic image:
%    \begin{figure}[ht]
% 	\begin{center}
%    	   \includegraphics[width=6cm]{MyArrangement_MyThorn_MyFigure}
% 	\end{center}
% 	\caption{Illustration of this and that}
% 	\label{MyArrangement_MyThorn_MyLabel}
%    \end{figure}
%
% Example of using a label:
%   \label{MyArrangement_MyThorn_MyLabel}
%
% Example of a citation:
%    \cite{MyArrangement_MyThorn_Author99}
%
% Example of including a reference
%   \bibitem{MyArrangement_MyThorn_Author99}
%   {J. Author, {\em The Title of the Book, Journal, or periodical}, 1 (1999),
%   1--16. {\tt http://www.nowhere.com/}}
%
% *======================================================================*

% If you are using CVS use this line to give version information
% $Header: /cactusdevcvs/Cactus/doc/ThornGuide/template.tex,v 1.12 2004/01/07 20:12:39 rideout Exp $

\documentclass{article}

% Use the Cactus ThornGuide style file
% (Automatically used from Cactus distribution, if you have a
%  thorn without the Cactus Flesh download this from the Cactus
%  homepage at www.cactuscode.org)
\usepackage{../../../../doc/latex/cactus}

\begin{document}

% The author of the documentation
\author{Christian D. Ott, Erik Schnetter \textless cott@aei.mpg.de\textgreater}

% The title of the document (not necessarily the name of the Thorn)
\title{CarpetEvolutionMask}

% the date your document was last changed, if your document is in CVS,
% please use:
\date{August 01 2005}

\maketitle

% Do not delete next line
% START CACTUS THORNGUIDE

% Add all definitions used in this documentation here
%   \def\mydef etc

% Add an abstract for this thorn's documentation
\begin{abstract}

\end{abstract}

% The following sections are suggestive only.
% Remove them or add your own.

\section{Introduction}

\section{Physical System}

\section{Numerical Implementation}

\section{Using This Thorn}

\subsection{Obtaining This Thorn}

\subsection{Basic Usage}

\subsection{Special Behaviour}

\subsection{Interaction With Other Thorns}

\subsection{Examples}

\subsection{Support and Feedback}

\section{History}

\subsection{Thorn Source Code}

\subsection{Thorn Documentation}

\subsection{Acknowledgements}


\begin{thebibliography}{9}

\end{thebibliography}

% Do not delete next line
% END CACTUS THORNGUIDE

\end{document}

```



Reference answer:
src_code: #include <assert.h>
#include <stdlib.h>
#include <stdio.h>

#include "cctk.h"
#include "cctk_Arguments.h"
#include "cctk_Parameters.h"

static void enforce(int const varindex, char const *const optstring,
                    void *const arg);

void enforce_evolution_mask(CCTK_ARGUMENTS) {
  DECLARE_CCTK_ARGUMENTS;
  DECLARE_CCTK_PARAMETERS;

  CCTK_TraverseString(enforce_vars, enforce, cctkGH, CCTK_GROUP_OR_VAR);
}

void enforce(int const varindex, char const *const optstring, void *const arg) {

  /* enforce the rhs of a given var to zero where the evolution mask
     says so */

  cGH const *const cctkGH = (cGH const *)arg;
  DECLARE_CCTK_ARGUMENTS;
  DECLARE_CCTK_PARAMETERS;
  int rhsindex;
  int vargroup, rhsgroup;
  cGroup vardata, rhsdata;
  CCTK_REAL *rhsptr;

  int index;
  int i, j, k;
  int ierr;

  int nx = cctkGH->cctk_lsh[0];
  int ny = cctkGH->cctk_lsh[1];
  int nz = cctkGH->cctk_lsh[2];

  assert(varindex >= 0);

  /* make sure we do have a rhs for this var */

  rhsindex = MoLQueryEvolvedRHS(varindex);
  if (rhsindex < 0) {
    char *const fullvarname = CCTK_FullName(varindex);
    assert(fullvarname);
    CCTK_VWarn(0, __LINE__, __FILE__, CCTK_THORNSTRING,
               "There is no RHS variable registered with MoL for the evolved "
               "variable \"%s\"",
               fullvarname);
    free(fullvarname);
  }

  if (verbose) {
    char *const fullvarname = CCTK_FullName(varindex);
    char *const fullrhsname = CCTK_FullName(rhsindex);
    assert(fullvarname);
    assert(fullrhsname);
    CCTK_VInfo(CCTK_THORNSTRING,
               "Enforcing evolution mask on \"%s\" (RHS \"%s\")", fullvarname,
               fullrhsname);
    free(fullvarname);
    free(fullrhsname);
  }

  assert(rhsindex >= 0);

  vargroup = CCTK_GroupIndexFromVarI(varindex);
  assert(vargroup >= 0);
  rhsgroup = CCTK_GroupIndexFromVarI(rhsindex);
  assert(rhsgroup >= 0);

  ierr = CCTK_GroupData(vargroup, &vardata);
  assert(!ierr);
  ierr = CCTK_GroupData(rhsgroup, &rhsdata);
  assert(!ierr);

  assert(vardata.grouptype == CCTK_GF);
  assert(vardata.vartype == CCTK_VARIABLE_REAL);
  assert(vardata.dim == cctk_dim);
  assert(rhsdata.grouptype == CCTK_GF);
  assert(rhsdata.vartype == CCTK_VARIABLE_REAL);
  assert(rhsdata.dim == cctk_dim);

  rhsptr = CCTK_VarDataPtrI(cctkGH, 0, rhsindex);
  assert(rhsptr);

  if (!writeNaNs) {

    for (k = 0; k < nz; k++)
      for (j = 0; j < ny; j++)
        for (i = 0; i < nx; i++) {

          index = CCTK_GFINDEX3D(cctkGH, i, j, k);
          rhsptr[index] = evolution_mask[index] * rhsptr[index];
        }

  } else {

    for (k = 0; k < nz; k++)
      for (j = 0; j < ny; j++)
        for (i = 0; i < nx; i++) {

          index = CCTK_GFINDEX3D(cctkGH, i, j, k);
          if (evolution_mask[index] < 1e-10) {
            rhsptr[index] = 1.0e+12;
          }
        }
  }
}