Task: Implement make.code.defn for CactusExamples/Poisson thorn

thorn_name: CactusExamples/Poisson
src_filename: make.code.defn
interface: # Interface definition for thorn Poisson

IMPLEMENTS: Poisson

INHERITS: boundary grid

USES INCLUDE HEADER: carpet.h
USES INCLUDE HEADER: TATelliptic.h



CCTK_INT FUNCTION Boundary_SelectGroupForBC \
    (CCTK_POINTER_TO_CONST IN cctkGH,       \
     CCTK_INT IN faces,                     \
     CCTK_INT IN boundary_width,            \
     CCTK_INT IN table_handle,              \
     CCTK_STRING IN group_name,             \
     CCTK_STRING IN bc_name)
REQUIRES FUNCTION Boundary_SelectGroupForBC



CCTK_REAL potential TYPE=gf
{
  phi
} "Potential for elliptic equation"

CCTK_REAL residual TYPE=gf
{
  res
} "Residual for elliptic equation"

schedule: # Schedule definitions for thorn Poisson

STORAGE: potential residual

SCHEDULE Poisson_prepare AT initial
{
  LANG: C
} "Set up initial guess for initial data"

SCHEDULE Poisson_solve AT postinitial
{
  LANG: C
  # OPTIONS: global
} "Calculate uniform charge initial data"

SCHEDULE GROUP Poisson_boundaries
{
} "Apply boundary conditions to initial data"

SCHEDULE Poisson_boundaries_select IN Poisson_boundaries
{
  LANG: C
  OPTIONS: level
  SYNC: potential
} "Select boundary conditions for initial data"

SCHEDULE GROUP ApplyBCs AS Poisson_boundaries_apply IN Poisson_boundaries AFTER Poisson_boundaries_select
{
} "Apply boundary conditions to initial data"

param: # Parameter definitions for thorn Poisson

STRING solver "Name of TATelliptic solver that should be used"
{
  .* :: "must be an activated TATelliptic solver"
} "TATJacobi"

STRING options "Options for the solver"
{
  .* :: "no restriction"
} ""



REAL radius "Radius of uniformly charged sphere"
{
  0:* :: ""
} 1.0

REAL charge "Charge of uniformly charged sphere"
{
  *:* :: ""
} 1.0

configuration: # Configuration definition for thorn Poisson

REQUIRES Boundary Carpet CartGrid3D TATelliptic

context: uniform_charge.c:
```
#include <assert.h>
#include <math.h>
#include <stdio.h>
#include <stdlib.h>

#include <cctk.h>
#include <cctk_Arguments.h>
#include <cctk_Parameters.h>

#include <util_ErrorCodes.h>
#include <util_Table.h>

#include <carpet.h>

#include <TATelliptic.h>



static int calc_residual (const cGH * const cctkGH,
                          int const options_table,
                          void * const userdata);
static int apply_bounds (const cGH * const cctkGH,
                         int const options_table,
                         void * const userdata);
static void apply_bounds_level (CCTK_ARGUMENTS);



void Poisson_prepare (CCTK_ARGUMENTS)
{
  DECLARE_CCTK_ARGUMENTS;
  DECLARE_CCTK_PARAMETERS;
  
  /* Initial data for the solver */
#pragma omp parallel for collapse(3)
  for (int k=0; k<cctk_lsh[2]; ++k) {
    for (int j=0; j<cctk_lsh[1]; ++j) {
      for (int i=0; i<cctk_lsh[0]; ++i) {
	int ipos = CCTK_GFINDEX3D(cctkGH,i,j,k);
	phi[ipos] = 0;
      }
    }
  }
}



void Poisson_solve (CCTK_ARGUMENTS)
{
  DECLARE_CCTK_ARGUMENTS;
  DECLARE_CCTK_PARAMETERS;
  
  int ierr;
  
  /* Grid variables for the solver */
#define NVAR 1                  /* number of equations to solve */
  int var_ind[NVAR];		/* index of variable */
  int res_ind[NVAR];		/* index of residual */
  var_ind[0] = CCTK_VarIndex ("Poisson::phi");
  res_ind[0] = CCTK_VarIndex ("Poisson::res");
  
  /* Options for the solver */
  int options_table = Util_TableCreateFromString (options);
  assert (options_table>=0);
  
  /* Call solver */
  ierr = TATelliptic_CallSolver (cctkGH, var_ind, res_ind, NVAR,
				 options_table,
				 calc_residual, apply_bounds, 0,
				 solver);
  if (ierr!=0) {
    CCTK_WARN (CCTK_WARN_ALERT, "Failed to solve elliptic equation");
  }
  
  ierr = Util_TableDestroy (options_table);
  assert (!ierr);
}



/* Caculate the residual */
int calc_residual (const cGH * const cctkGH,
                   int const options_table,
                   void * const userdata)
{
  DECLARE_CCTK_ARGUMENTS;
  DECLARE_CCTK_PARAMETERS;
  
  /* charge density */
  CCTK_REAL rho = charge / (4.0/3.0 * M_PI * pow(radius,3));
  
  /* offsets for 3D array layout */
  int di = CCTK_GFINDEX3D(cctkGH,1,0,0) - CCTK_GFINDEX3D(cctkGH,0,0,0);
  int dj = CCTK_GFINDEX3D(cctkGH,0,1,0) - CCTK_GFINDEX3D(cctkGH,0,0,0);
  int dk = CCTK_GFINDEX3D(cctkGH,0,0,1) - CCTK_GFINDEX3D(cctkGH,0,0,0);
  
  CCTK_REAL idx2[3];		/* inverse squared grid spacing */
  for (int d=0; d<3; ++d) {
    idx2[d] = 1.0 / pow(CCTK_DELTA_SPACE(d), 2);
  }
  
  for (int d=0; d<3; ++d) {
    assert (cctk_nghostzones[d] >= 1);
  }

  /* Initialize residual to zero:
     We only do this to ensure the boundaries of the residual are
     defined, and are not nan. That in turn is only needed to make
     output look nicer. */
#pragma omp parallel for collapse(3)
  for (int k=0; k<cctk_lsh[2]; ++k) {
    for (int j=0; j<cctk_lsh[1]; ++j) {
      for (int i=0; i<cctk_lsh[0]; ++i) {
	int ipos = CCTK_GFINDEX3D(cctkGH,i,j,k);
        res[ipos] = 0.0;
      }
    }
  }
  
  /* Calculate residual:
     res = Laplace phi + 4 pi rho */
#pragma omp parallel for collapse(3)
  for (int k=cctk_nghostzones[2]; k<cctk_lsh[2]-cctk_nghostzones[2]; ++k) {
    for (int j=cctk_nghostzones[1]; j<cctk_lsh[1]-cctk_nghostzones[1]; ++j) {
      for (int i=cctk_nghostzones[0]; i<cctk_lsh[0]-cctk_nghostzones[0]; ++i) {
	int ipos = CCTK_GFINDEX3D(cctkGH,i,j,k);
	res[ipos] = 
	  + (phi[ipos-di] - 2*phi[ipos] + phi[ipos+di]) * idx2[0]
	  + (phi[ipos-dj] - 2*phi[ipos] + phi[ipos+dj]) * idx2[1]
	  + (phi[ipos-dk] - 2*phi[ipos] + phi[ipos+dk]) * idx2[2]
	  + (r[ipos]<radius ? 4 * M_PI * rho : 0);
      }
    }
  }
  
  /* Success */
  return 0;
}



/* Apply the boundary conditions to the solution */
int apply_bounds (const cGH * const cctkGH,
                  int const options_table,
                  void * const userdata)
{
  DECLARE_CCTK_ARGUMENTS;
  DECLARE_CCTK_PARAMETERS;
  
  int ierr;
  
  /* Switch to level mode */
  ierr = CallLevelFunction ((cGH *)cctkGH, apply_bounds_level);
  assert (!ierr);
  
  /* Success */
  return 0;
}

void apply_bounds_level (CCTK_ARGUMENTS)
{
  DECLARE_CCTK_ARGUMENTS;
  DECLARE_CCTK_PARAMETERS;
  
  int ierr;
  
  /* Call boundary condition schedule group */
  ierr = CallScheduleGroup (cctkGH, "Poisson_boundaries");
  assert (!ierr);
}

void Poisson_boundaries_select (CCTK_ARGUMENTS)
{
  DECLARE_CCTK_ARGUMENTS;
  DECLARE_CCTK_PARAMETERS;
  
  int ierr;
  
  /* Select Direchlet boundary conditions */
  ierr = Boundary_SelectGroupForBC
    (cctkGH, CCTK_ALL_FACES, 1, -1, "Poisson::potential", "scalar");
  assert (!ierr);
}

```


combined_doc_context: ## README (README):
```
Cactus Code Thorn Poisson
Author(s)    : Erik Schnetter <schnetter@gmail.com>
Maintainer(s): Erik Schnetter <schnetter@gmail.com>
Licence      : GPL
--------------------------------------------------------------------------

1. Purpose

Solve the Poisson equation with the TATelliptic framework.

```

## Documentation (.tex files):
Files: documentation.tex

```tex
=== documentation.tex ===
% *======================================================================*
%  Cactus Thorn template for ThornGuide documentation
%  Author: Ian Kelley
%  Date: Sun Jun 02, 2002
%  $Header$
%
%  Thorn documentation in the latex file doc/documentation.tex
%  will be included in ThornGuides built with the Cactus make system.
%  The scripts employed by the make system automatically include
%  pages about variables, parameters and scheduling parsed from the
%  relevant thorn CCL files.
%
%  This template contains guidelines which help to assure that your
%  documentation will be correctly added to ThornGuides. More
%  information is available in the Cactus UsersGuide.
%
%  Guidelines:
%   - Do not change anything before the line
%       % START CACTUS THORNGUIDE",
%     except for filling in the title, author, date, etc. fields.
%        - Each of these fields should only be on ONE line.
%        - Author names should be separated with a \\ or a comma.
%   - You can define your own macros, but they must appear after
%     the START CACTUS THORNGUIDE line, and must not redefine standard
%     latex commands.
%   - To avoid name clashes with other thorns, 'labels', 'citations',
%     'references', and 'image' names should conform to the following
%     convention:
%       ARRANGEMENT_THORN_LABEL
%     For example, an image wave.eps in the arrangement CactusWave and
%     thorn WaveToyC should be renamed to CactusWave_WaveToyC_wave.eps
%   - Graphics should only be included using the graphicx package.
%     More specifically, with the "\includegraphics" command.  Do
%     not specify any graphic file extensions in your .tex file. This
%     will allow us to create a PDF version of the ThornGuide
%     via pdflatex.
%   - References should be included with the latex "\bibitem" command.
%   - Use \begin{abstract}...\end{abstract} instead of \abstract{...}
%   - Do not use \appendix, instead include any appendices you need as
%     standard sections.
%   - For the benefit of our Perl scripts, and for future extensions,
%     please use simple latex.
%
% *======================================================================*
%
% Example of including a graphic image:
%    \begin{figure}[ht]
% 	\begin{center}
%    	   \includegraphics[width=6cm]{MyArrangement_MyThorn_MyFigure}
% 	\end{center}
% 	\caption{Illustration of this and that}
% 	\label{MyArrangement_MyThorn_MyLabel}
%    \end{figure}
%
% Example of using a label:
%   \label{MyArrangement_MyThorn_MyLabel}
%
% Example of a citation:
%    \cite{MyArrangement_MyThorn_Author99}
%
% Example of including a reference
%   \bibitem{MyArrangement_MyThorn_Author99}
%   {J. Author, {\em The Title of the Book, Journal, or periodical}, 1 (1999),
%   1--16. {\tt http://www.nowhere.com/}}
%
% *======================================================================*

% If you are using CVS use this line to give version information
% $Header$

\documentclass{article}

% Use the Cactus ThornGuide style file
% (Automatically used from Cactus distribution, if you have a
%  thorn without the Cactus Flesh download this from the Cactus
%  homepage at www.cactuscode.org)
\usepackage{../../../../doc/latex/cactus}

\begin{document}

% The author of the documentation
\author{Erik Schnetter \textless schnetter@gmail.com\textgreater}

% The title of the document (not necessarily the name of the Thorn)
\title{Poisson}

% the date your document was last changed, if your document is in CVS,
% please use:
%    \date{$ $Date$ $}
\date{November 25, 2014}

\maketitle

% Do not delete next line
% START CACTUS THORNGUIDE

% Add all definitions used in this documentation here
%   \def\mydef etc

% Add an abstract for this thorn's documentation
\begin{abstract}
  This is an example thorn describing how to use the TATPETSc
  interface to PETSc. It solves the Poisson equation for a spherical
  charge distribution on a uniform grid.
\end{abstract}

% The following sections are suggestive only.
% Remove them or add your own.

\section{Introduction}

PETSc is a well-known library for solving elliptic equations. TATPETSc
is a Cactus thorn that provides a wrapper for calling PETSc to solve
elliptic equations on uniform grids. (TATPETSc currently supports
neither mesh refinement nor multi-block systems.) TATPETSc can solve
both linear and non-linear systems.

\section{Physical System}

Here we solve the Poisson equation
\begin{eqnarray}
  \Delta\Phi(x) &=& \rho(x)
\end{eqnarray}
where the right hand side $\rho$ is given by
\begin{eqnarray}
  \rho(r) & = & \left\{
  \begin{array}{ll}
    Q/V & r\le R \\
    0 & r>R
  \end{array}
  \right.
\end{eqnarray}
for the charge $Q$ and the radius $R$, with $V=4\pi R^3/3$. We use
Dirichlet boundary conditions $\Phi(x)=0$.

\section{Numerical Implementation}

PETSc supports a large number of options to choose solvers. Here we
use PETSc's default settings.

\section{Using This Thorn}

In the example parameter file, we set the parameter
\texttt{TATPETSc::options} to select the following PETSc options:
\begin{itemize}
\item \verb+-snes_atol 1e-8+: set absolute tolerance for residual
\item \verb+-snes_stol 1e-8+: set relative tolerance for residual
\item \verb+-snes_monitor+: output progress information at each
  iteration of the non-linear solver
\item \verb+-ksp_monitor+: output progress information at each
  iteration of the linear (Krylov subspace) solver
\end{itemize}

\subsection{Examples}

The solution (the potential $\Phi(x)$) is stored in the grid function
\texttt{potential}, the residual (a measure for the error) in the grid
function \texttt{residual}.

% Do not delete next line
% END CACTUS THORNGUIDE

\end{document}

```



Reference solution:
src_code: # Main make.code.defn file for thorn Poisson

# Source files in this directory
SRCS = uniform_charge.c

# Subdirectories containing source files
SUBDIRS =