Task: Implement patch/fd_grid.cc for EinsteinAnalysis/AHFinderDirect thorn

thorn_name: EinsteinAnalysis/AHFinderDirect
src_filename: patch/fd_grid.cc
interface: # Interface definition for thorn AHFinderDirect
# $Header$

implements: AHFinderDirect
inherits: Grid ADMBase StaticConformal SpaceMask SphericalSurface IO

# include file provided by SpaceMask
USES INCLUDE: SpaceMask.h

################################################################################

# we use a mask in order to find out whether we interpolate from the excised
# region.  (we need 3 time levels because Carpet might need to interpolate
# in time.)

REAL ahmask TYPE=gf TIMELEVELS=3 TAGS='tensortypealias="Scalar" Prolongation="None"'

################################################################################

# we use an aliased function maybe provided by DriftCorrect
SUBROUTINE SetDriftCorrectPosition					\
   (CCTK_POINTER_TO_CONST IN GH,					\
    CCTK_REAL IN x, CCTK_REAL IN y, CCTK_REAL IN z)
USES FUNCTION SetDriftCorrectPosition

################################################################################

#
# This thorn provides an aliased-function interface which other thorns
# can use to find out about the horizons.  Each function returns a status
# code which is >= 0 for ok, or < 0 for an error.
#

########################################

#
# This function returns the local coordinate origin for a given horizon.
#
CCTK_INT FUNCTION HorizonLocalCoordinateOrigin				\
   (CCTK_INT IN horizon_number,						\
    CCTK_REAL OUT origin_x, CCTK_REAL OUT origin_y, CCTK_REAL OUT origin_z)

#
# The following function queries whether or not the specified horizon
# was found the most recent time AHFinderDirect searched for it.
# The return value is:
#  1 if the horizon was found
#  0 if the horizon was not found
#  negative for an error
#
CCTK_INT FUNCTION HorizonWasFound(CCTK_INT IN horizon_number)

#
# The following function queries whether or not the specified horizon
# was found the most recent time AHFinderDirect searched for it, and
# if it was, returns the horizon centroid.  The returned value is the
# same as that of  HorizonWasFound()  (above), i.e.
#  1 if the horizon was found; in this case  centroid_[xyz]  are set
#    to the centroid position
#  0 if the horizon was not found; in this case  centroid_[xyz]  are
#    set to zeros
#  negative for an error
#
CCTK_INT FUNCTION HorizonCentroid					\
  (CCTK_INT IN horizon_number,						\
   CCTK_REAL OUT centroid_x, CCTK_REAL OUT centroid_y, CCTK_REAL OUT centroid_z)

#
# The following function computes the horizon radius in the direction
# of each (x,y,z) point, or -1.0 if this horizon wasn't found the most
# recent time AHFinderDirect searched for it.  More precisely, for each
# (x,y,z), consider the ray from the local coordinate origin through
# (x,y,z).  This function computes the Euclidean distance between the
# local coordinate origin and this ray's intersection with the horizon,
# or -1.0 if this horizon wasn't found the most recent time AHFinderDirect
# searched for it.  
#
CCTK_INT FUNCTION HorizonRadiusInDirection				\
   (CCTK_INT IN horizon_number,						\
    CCTK_INT IN N_points,						\
    CCTK_REAL IN ARRAY x, CCTK_REAL IN ARRAY y, CCTK_REAL IN ARRAY z,	\
    CCTK_REAL OUT ARRAY radius)

########################################

# all these functions are in src/driver/aliased_functions.cc
PROVIDES FUNCTION HorizonLocalCoordinateOrigin				\
	WITH AHFinderDirect_local_coordinate_origin			\
	LANGUAGE C
PROVIDES FUNCTION HorizonWasFound					\
	WITH AHFinderDirect_horizon_was_found				\
	LANGUAGE C
PROVIDES FUNCTION HorizonCentroid					\
	WITH AHFinderDirect_horizon_centroid				\
	LANGUAGE C
PROVIDES FUNCTION HorizonRadiusInDirection				\
	WITH AHFinderDirect_radius_in_direction				\
	LANGUAGE C

################################################################################

# Grid arrays to store the horizon shape, for easy output and checkpointing.
# These are similar and equivalent to SphericalSurface.
# Note that the horizons are numbered [0...N_horizon-1].

CCTK_REAL ah_radius[N_horizons] TYPE=array DIM=3 SIZE=max_N_zones_per_right_angle+1,max_N_zones_per_right_angle+1,6 DISTRIB=constant

CCTK_REAL ah_origin[N_horizons] TYPE=scalar
{
  ah_origin_x ah_origin_y ah_origin_z
}

CCTK_REAL ah_centroid[N_horizons] TYPE=scalar
{
  ah_centroid_x ah_centroid_y ah_centroid_z ah_centroid_t
  ah_centroid_x_p ah_centroid_y_p ah_centroid_z_p ah_centroid_t_p
}

CCTK_INT ah_flags[N_horizons] TYPE=scalar
{
  ah_initial_find_flag
  ah_really_initial_find_flag
  ah_search_flag
  ah_found_flag
  ah_centroid_valid
  ah_centroid_valid_p
  ah_centroid_iteration
  ah_centroid_iteration_p
}

################################################################################

# This IOUtil function is used to determine whether the diagnostics file
# should be truncated or appended (as in a recovery run )

CCTK_INT FUNCTION IO_TruncateOutputFiles \
  (CCTK_POINTER_TO_CONST IN GH)
USES FUNCTION IO_TruncateOutputFiles

################################################################################

# translate SphericalSurface name into ID
CCTK_INT \
FUNCTION sf_IdFromName (CCTK_INT IN id, CCTK_POINTER_TO_CONST IN name)
USES FUNCTION sf_IdFromName

################################################################################

schedule: # Schedule definitions for thorn AHFinderDirect
# $Header$

storage: ah_radius ah_origin ah_centroid ah_flags

if (metric_timelevels == 1)
{
  storage: ahmask[1]
}
else if (metric_timelevels == 2)
{
  storage: ahmask[2]
}
else if (metric_timelevels == 3)
{
  storage: ahmask[3]
}

#
# setup
#
schedule AHFinderDirect_setup at CCTK_BASEGRID        \
   after SpatialCoordinates
   {
   lang: C
   options: global
   } "setup data structures"

#
# checkpoint/recover support
#
schedule AHFinderDirect_recover at CCTK_POST_RECOVER_VARIABLES
   {
   lang: C
   options: global
   } "import horizon data from Cactus variables"

#
# find horizons
#
if (run_at_CCTK_ANALYSIS != 0)
  {
  schedule AHFinderDirect_find_horizons at CCTK_ANALYSIS
     {
     lang: C
     options: global-early
     } "find apparent horizon(s) after this time step"
  }
if (run_at_CCTK_POSTSTEP != 0)
  {
  schedule AHFinderDirect_find_horizons at CCTK_POSTSTEP
     {
     lang: C
     options: global-early
     } "find apparent horizon(s) after this time step"
  }
if (run_at_CCTK_POSTINITIAL != 0)
  {
  schedule AHFinderDirect_find_horizons at CCTK_POSTINITIAL
     {
     lang: C
     options: global-early
     } "find apparent horizon(s) after this time step"
  }
if (run_at_CCTK_POSTPOSTINITIAL != 0)
  {
  schedule AHFinderDirect_find_horizons at CCTK_POSTPOSTINITIAL
     {
     lang: C
     options: global-early
     } "find apparent horizon(s) after this time step"
  }
if (run_at_CCTK_POST_RECOVER_VARIABLES != 0)
  {
  schedule AHFinderDirect_find_horizons at CCTK_POST_RECOVER_VARIABLES \
           after MoL_PostStep
     {
     lang: C
     options: global-early
     } "find apparent horizon(s) after this time step"
  }

#
# find horizons, set excision mask, maybe announce centroid to other thorns
#
# FIXME: the name  AHFinderDirect_maybe_do_masks  is archaic,
#        AHFinderDirect_do_masks  would be more apt
#
# We use two Cactus Groups here:
#  group_for_mask_stuff
#    group_where_mask_is_set
#      AHFinderDirect_maybe_do_masks
# this way other thorns can schedule routines which need to run immediately
# before/after we set the mask in group_for_mask_stuff before/after
# group_where_mask_is_set, without having to know in what higher-level
# Cactus schedule bins/groups this thorn runs.
#
if (run_at_CCTK_ANALYSIS != 0)
  {
  schedule AHFinderDirect_import_mask at CCTK_ANALYSIS \
     before AHFinderDirect_find_horizons
     {
     lang: C
     options: global-early loop-local
     } "import the excision mask"

  schedule AHFinderDirect_store at CCTK_ANALYSIS \
     after AHFinderDirect_find_horizons \
     before SphericalSurface_HasBeenSet
     {
     lang: C
     options: global-early
     } "store apparent horizon(s) into spherical surface(s)"

  schedule AHFinderDirect_save at CCTK_ANALYSIS \
     after AHFinderDirect_find_horizons \
     {
     lang: C
     options: global-early
     } "save apparent horizon(s) into Cactus variables"

  if (which_horizon_to_announce_centroid != 0)
    {
    schedule AHFinderDirect_announce at CCTK_ANALYSIS \
       before DriftCorrect \
       after AHFinderDirect_find_horizons
       {
       lang: C
       options: global-early
       } "announce horizon position(s) to other thorns"
    }

  schedule AHFinderDirect_maybe_do_masks at CCTK_ANALYSIS \
     after AHFinderDirect_find_horizons
     {
     lang: C
     } "set mask(s) based on apparent horizon position(s)"
  }
if (run_at_CCTK_POSTSTEP != 0)
  {
  schedule AHFinderDirect_import_mask at CCTK_POSTSTEP \
     before AHFinderDirect_find_horizons
     {
     lang: C
     options: global-early loop-local
     } "import the excision mask"

  schedule AHFinderDirect_store at CCTK_POSTSTEP \
     after AHFinderDirect_find_horizons \
     before SphericalSurface_HasBeenSet
     {
     lang: C
     options: global-early
     } "store apparent horizon(s) into spherical surface(s)"

  schedule AHFinderDirect_save at CCTK_POSTSTEP \
     after AHFinderDirect_find_horizons \
     {
     lang: C
     options: global-early
     } "save apparent horizon(s) into Cactus variables"

  if (which_horizon_to_announce_centroid != 0)
    {
    schedule AHFinderDirect_announce at CCTK_POSTSTEP \
       before DriftCorrect \
       after AHFinderDirect_find_horizons
       {
       lang: C
       options: global-early
       } "announce horizon position(s) to other thorns"
    }

  schedule AHFinderDirect_maybe_do_masks at CCTK_POSTSTEP \
     after AHFinderDirect_find_horizons
     {
     lang: C
     } "set mask(s) based on apparent horizon position(s)"
  }
if (run_at_CCTK_POSTINITIAL != 0)
  {
  schedule AHFinderDirect_import_mask at CCTK_POSTINITIAL \
     before AHFinderDirect_find_horizons
     {
     lang: C
     options: global-early loop-local
     } "import the excision mask"

  schedule AHFinderDirect_store at CCTK_POSTINITIAL \
     after AHFinderDirect_find_horizons \
     before SphericalSurface_HasBeenSet
     {
     lang: C
     options: global-early
     } "store apparent horizon(s) into spherical surface(s)"

  schedule AHFinderDirect_save at CCTK_POSTINITIAL \
     after AHFinderDirect_find_horizons \
     {
     lang: C
     options: global-early
     } "save apparent horizon(s) into Cactus variables"

  if (which_horizon_to_announce_centroid != 0)
    {
    schedule AHFinderDirect_announce at CCTK_POSTINITIAL \
       before DriftCorrect \
       after AHFinderDirect_find_horizons
       {
       lang: C
       options: global-early
       } "announce horizon position(s) to other thorns"
    }

  schedule AHFinderDirect_maybe_do_masks at CCTK_POSTINITIAL \
     after AHFinderDirect_find_horizons
     {
     lang: C
     } "set mask(s) based on apparent horizon position(s)"
  }
if (run_at_CCTK_POSTPOSTINITIAL != 0)
  {
  schedule AHFinderDirect_import_mask at CCTK_POSTPOSTINITIAL \
     before AHFinderDirect_find_horizons
     {
     lang: C
     options: global-early loop-local
     } "import the excision mask"

  schedule AHFinderDirect_store at CCTK_POSTPOSTINITIAL \
     after AHFinderDirect_find_horizons
     {
     lang: C
     options: global-early
     } "store apparent horizon(s) into spherical surface(s)"

  schedule AHFinderDirect_save at CCTK_POSTPOSTINITIAL \
     after AHFinderDirect_find_horizons \
     {
     lang: C
     options: global-early
     } "save apparent horizon(s) into Cactus variables"

  if (which_horizon_to_announce_centroid != 0)
    {
    schedule AHFinderDirect_announce at CCTK_POSTPOSTINITIAL \
       before DriftCorrect \
       after AHFinderDirect_find_horizons
       {
       lang: C
       options: global-early
       } "announce horizon position(s) to other thorns"
    }

  schedule AHFinderDirect_maybe_do_masks at CCTK_POSTPOSTINITIAL \
     after AHFinderDirect_find_horizons
     {
     lang: C
     } "set mask(s) based on apparent horizon position(s)"
  }
if (run_at_CCTK_POST_RECOVER_VARIABLES != 0)
  {
  schedule AHFinderDirect_import_mask at CCTK_POST_RECOVER_VARIABLES \
     before AHFinderDirect_find_horizons
     {
     lang: C
     options: global-early loop-local
     } "import the excision mask"

  schedule AHFinderDirect_store at CCTK_POST_RECOVER_VARIABLES \
     after AHFinderDirect_find_horizons \
     before SphericalSurface_HasBeenSet
     {
     lang: C
     options: global-early
     } "store apparent horizon(s) into spherical surface(s)"

  schedule AHFinderDirect_save at CCTK_POST_RECOVER_VARIABLES \
     after AHFinderDirect_find_horizons
     {
     lang: C
     options: global-early
     } "save apparent horizon(s) into Cactus variables"

  if (which_horizon_to_announce_centroid != 0)
    {
    schedule AHFinderDirect_announce at CCTK_POST_RECOVER_VARIABLES \
       before DriftCorrect \
       after AHFinderDirect_find_horizons
       {
       lang: C
       options: global-early
       } "announce horizon position(s) to other thorns"
    }

  schedule AHFinderDirect_maybe_do_masks at CCTK_POST_RECOVER_VARIABLES \
     after AHFinderDirect_find_horizons
     {
     lang: C
     } "set mask(s) based on apparent horizon position(s)"
  }

# TODO: Don't call this, since no horizon has been found yet
schedule AHFinderDirect_maybe_do_masks at CCTK_POSTREGRIDINITIAL \
   after (MaskOne MaskZero)
   {
   lang: C
   } "set mask(s) based on apparent horizon position(s)"

schedule AHFinderDirect_maybe_do_masks at CCTK_POSTREGRID \
   after (MaskOne MaskZero)
   {
   lang: C
   } "set mask(s) based on apparent horizon position(s)"

########################################

##
## uncomment this if you want to use the test driver
##  src/util/test_patch_system.cc
##
##schedule test_patch_system at CCTK_POSTINITIAL
##{
##LANG: C
##OPTIONS: global
##} "test driver to verify that src/util/ and src/jtutil/ code works properly"

param: # Parameter definitions for thorn AHFinderDirect
# $Header$

################################################################################

#
# ***** interface to the rest of Cactus *****
#

# we may need to look at some grid:: parameters to choose our patch system type
# note our access is read-only
shares: grid
USES KEYWORD domain
USES KEYWORD bitant_plane
USES KEYWORD quadrant_direction
USES KEYWORD rotation_axis

# we need to look at ADMBase::metric_type and StaticConformal::conformal_state
# to check physical/conformal metric
# note our access is read-only
shares: ADMBase
USES KEYWORD metric_type
USES CCTK_INT metric_timelevels

# we need to look at SpaceMask::use_mask in order to find out about the
# excision region
shares: SpaceMask
USES BOOLEAN use_mask

# we need to look at IO::out_dir to set the default for our own
# output directory parameters
shares: IO
USES STRING out_dir

# we want to know how many surfaces there are, so that we don't access
# a non-existing one
shares: SphericalSurface
USES INT nsurfaces
USES INT maxntheta
USES INT maxnphi

# all remaining parameters are private to this thorn
private:

################################################################################

#
# ***** overall parameters *****
#

# set this to (try to) find apparent horizons at every time step
int find_every "how often should we try to find apparent horizons?" STEERABLE=always
{
0   :: "don't find AHs at all (this thorn is a no-op)"
1:* :: "any integer >= 1"
} 1

# set this to (try to) find individual apparent horizons at every time step
int find_every_individual[101] "how often should we try to find individual apparent horizons?  (overrides find_every)" STEERABLE=always
{
-1  :: "use the value of find_every"
0   :: "don't find this AH at all"
1:* :: "any integer >= 1"
} -1

# set this to (try to) find individual apparent horizons after a time step
int find_after_individual[101] "when should we start to find individual apparent horizons?" STEERABLE=always
{
0:* :: "after this iteration (inclusively)"
} 0

int dont_find_after_individual[101] "when should we stop finding individual apparent horizons?" STEERABLE=always
{
0:* :: "after this iteration (exclusively)"
-1  :: "do not use this parameter"
} -1

# set this to (try to) find individual apparent horizons after a certain time
real find_after_individual_time[101] "when should we start to find individual apparent horizons?" STEERABLE=always
{
*:* :: "after this time (inclusively)"
} 0.0

real dont_find_after_individual_time[101] "when should we stop finding individual apparent horizons?" STEERABLE=always
{
*:* :: "after this time (exclusively); ignore this value if it is less than or equal to find_after_individual_time"
} 0.0

# set this to disable finding individual horizons
boolean disable_horizon[101] "should this horizon be disabled?" STEERABLE=always
{
} "false"

#
# These parameters control which schedule bin (or even bins, if you
# want) AHFinderDirect runs in.  Historically, Erik's branch of
# AHFinderDirect has run at CCTK_POSTSTEP.  However, for some purposes
# it's more convenient to run at CCTK_ANALYSIS and/or CCTK_POSTINITIAL.
#
# In general we need to run at CCTK_POST_RECOVER_VARIABLES, since
# (a) parameters may have been steered at recovery, so we may need
#     to find a new horizon or horizons, and
# (b) we need to set the mask again to make sure it's correct
#     right away (since our next regular horizon-finding may not
#     be until some time steps later)
# Therefore the  run_at_CCTK_POST_RECOVER_VARIABLES  parameter should
# probably be left at its default setting of  true .
#
Boolean run_at_CCTK_ANALYSIS "should we run at CCTK_ANALYSIS?"
{
} false
Boolean run_at_CCTK_POSTSTEP "should we run at CCTK_POSTSTEP?"
{
} true
Boolean run_at_CCTK_POSTINITIAL "should we run at CCTK_POSTINITIAL?"
{
} false
Boolean run_at_CCTK_POSTPOSTINITIAL "should we run at CCTK_POSTPOSTINITIAL?"
{
} false
Boolean run_at_CCTK_POST_RECOVER_VARIABLES				\
  "should we run at CCTK_POST_RECOVER_VARIABLES?"
{
} false

keyword method "what should this thorn do for each apparent horizon?" \
  STEERABLE=recover
{
# these options are mostly for testing/debugging
# ... in a multiprocessor Cactus run, the horizons are done sequentually
#     on processor #0; the other processors do dummy computations
"evaluate expansions"	  :: "evaluate the LHS function Theta(h)"

# ... in a multiprocessor Cactus run, the Jacobian is computed on
#     processor #0; the other processors do dummy computations
"test expansion Jacobians" :: \
  "compute/print horizon 1's J[Theta(h)] Jacobian matrix (possibly in \
   multiple ways, depending on the  test_all_Jacobian_methods  parameter)"

# this is for normal apparent horizon finding
# ... in a multiprocessor Cactus run, the horizons are done in parallel
#     across processors; see src/driver/README.parallel for details
"find horizons"		  :: "find the apparent horizon"
} "find horizons"

#
# We support searching for up to N_horizons distinct apparent horizons
# (some of which may be nested inside others) in a slice.
# ***** IMPORTANT *****: The horizons are numbered 1 to N_horizons.
# Since Cactus arrays are 0-origin, we make the arrays be of size
# N_horizons+1, and don't use the [0] array elements.
#
# To change the N_horizons limit, just change the upper limit for
# N_horizons itself, change all the [N_horizons+1] array sizes in this
# paramter file, change the upper limit for which_horizon_to_announce_centroid,
# and recompile your configuration.  No changes are needed to the source code.
#
int N_horizons "number of apparent horizons to search for" \
  STEERABLE=recover
{
0     :: "turn this thorn into a fancy no-op :)"
1:100 :: "search for this many apparent horizons"
} 1

#
# This parameter controls whether or not (assuming we have found a
# horizon or horizons), we announce a selected horizon centroid to
# some other thorn.  In practice the other thorn will be DriftCorrect,
# which will use this information to adjust the coordinates to try to
# keep the horizon centroid at a constant coordinate position.
#
int which_horizon_to_announce_centroid \
  "for which horizon should we announce the centroid?" \
  STEERABLE=recover
{
0     :: "don't announce any centroid(s)"
1:100 :: "announce this horizon's centroid each time we find it"
} 0

#
# This parameter controls whether and (if) into which surface the horizon
# information will be stored.
#
int which_surface_to_store_info[101] \
  "into which surface should we store the info?" \
  STEERABLE=recover
{
-1  :: "don't store info"
0:* :: "store info into the corresponding surface"
} -1

string which_surface_to_store_info_by_name[101] \
  "into which surface should we store the info?" \
  STEERABLE=recover
{
""   :: "use which_surface_to_store_info"
".*" :: "any string"
} ""

#
# This parameter controls how verbose this thorn is in printing
# informational (non-error) messages describing what it's doing.
#
keyword verbose_level \
  "controls which (how many) messages to print describing AH finding" \
  STEERABLE=recover
{
# 1 line each time step giving number of horizons found and their masses
# ... this doesn't work yet :(
"physics highlights"   :: "just a few physics messages"

# 1 line for each horizon giving position/mass/area, + a summary line or two
"physics details"      :: "more detailed physics messages"

# 1 line giving Theta(h) norms at each Newton iteration
"algorithm highlights" :: \
  "physics details + a few messages about the AH-finding algorithm"

# lots of details tracing what the code is doing
"algorithm details"    :: \
  "physics details + lots of messages about the AH-finding algorithm"

# even more details tracing what the code is doing :)
"algorithm debug"    :: \
  "physics details + lots and lots of messages about the AH-finding algorithm"
} "physics details"

# n.b. printing timing stats is independent of  verbose_level
boolean print_timing_stats \
  "should we print timing stats for the whole apparent-horizon-finding process?" \
  STEERABLE=recover
{
} "false"

# Do we want the gradients of the surface expansion?
# (Increases run time)
boolean want_expansion_gradients "should we print the gradients of the expansions?" \
  STEERABLE=recover
{
} "false"

# By default, we find apparent horizons or CE surfaces.
# This parameter allows the equation to be change so that other surfaces
# are found instead.
keyword surface_definition[101] "search for what kind of surface?" STEERABLE=recover
{
"expansion"         :: "Theta_(l) (apparent horizons etc.)"
"inner expansion"   :: "Theta_(n) (expansion of ingoing null normal)"
"mean curvature"    :: "for CMC surfaces"
"expansion product" :: "Theta_(l) Theta_(n) (product of the expansions)"
} "expansion"

# By default, we find surfaces as defined above.
# This parameter allows the definition to be modified.
keyword surface_modification[101] "how to modify the surface definition" STEERABLE=recover
{
"none"         :: "no modification"
"radius"       :: "multiply with the coordinate radius"
"radius^2"     :: "multiply with the square of the coordinate radius"
"mean radius"  :: "multiply with the mean coordinate radius"
"areal radius" :: "multiply with the areal radius (does not converge nicely, because the Jacobian is only approximate)"
} "none"

# By default, we find apparent horizons by looking for a specified expansion.
# This parameter allows looking for a specified areal radius etc. instead.
keyword surface_selection[101] "search for a surface with this areal radius" STEERABLE=recover
{
"definition"                             :: "look for a surface as defined by 'surface_definition'"
"mean coordinate radius"                 :: "look for a surface with a certain mean coordinate radius (not covariant, but fast)"
"areal radius"                           :: "look for a surface with a certain areal radius"
"expansion times mean coordinate radius" :: "look for a surface with a certain product of expansion and mean coordiante radius"
"expansion times areal radius"           :: "look for a surface with a certain product of expansion and areal radius"
} "definition"

# By default, we find apparent horizons by solving the equation
#  Theta(h) = 0
# This parameter allows the RHS to be set to any specified constant,
# to find a surface of constant expansion.
real desired_value[101] "search for a surface with this (constant) value" STEERABLE=recover
{
*:* :: "any real number"
} 0.0



# By default, we just find apparent horizons.  This parameter uses
# pretracking to search for outermost apparent horizons instead.
boolean use_pretracking[101] "search for an outermost apparent horizon" STEERABLE=recover
{
} "no"

int pretracking_max_iterations[101] "maximum number of pretracking iterations" STEERABLE=recover
{
  0:* :: ""
} 100

real pretracking_value[101] "initial pretracking value" STEERABLE=recover
{
  *:* :: ""
} 1.0

real pretracking_minimum_value[101] "minimum pretracking value" STEERABLE=recover
{
  *:* :: ""
} 0.0

real pretracking_maximum_value[101] "maximum pretracking value (should be near the outer boundary)" STEERABLE=recover
{
  *:* :: ""
} 10.0

real pretracking_delta[101] "step size for value" STEERABLE=recover
{
  (0.0:* :: ""
} 1.0

real pretracking_minimum_delta[101] "minimum step size for value" STEERABLE=recover
{
  (0.0:* :: ""
} 1.0e-4

real pretracking_maximum_delta[101] "maximum step size for value" STEERABLE=recover
{
  (0.0:* :: ""
} 1.0

# Horizons can depend on other (usually pretracked) horizons
int depends_on[101] "the other horizon that this horizon depends on" STEERABLE=recover
{
  0     :: "the horizon is independent"
  1:100 :: "horizon index"
} 0

real desired_value_factor[101] "factor between this horizon's and the other horizon's desired value, applied before the offset" STEERABLE=recover
{
  *:* :: ""
} 1.0

real desired_value_offset[101] "difference between this horizon's and the other horizon's desired value, applied after the factor" STEERABLE=recover
{
  *:* :: ""
} 0.0



# By default, we keep the surface from the last time (if so desired).
# We can also modify it before each finding to try make it find inner
# or outer horizons.
real shiftout_factor[101] "enlarge the surface initial guess before finding" STEERABLE=recover
{
  (0:* :: "choose 1 for doing nothing, larger for enlarging, smaller for shrinking"
} 1.0

real smoothing_factor[101] "smoothen (remove higher multipole moments) the surface initial guess before finding" STEERABLE=recover
{
  *:* :: "choose 0 for no smoothing, 1 for complete smoothing, larger for over-smoothing, negative for roughening"
} 0.0

################################################################################

#
# ***** parameters for the initial guess for each apparent horizon shape *****
#

#
# (Note that if at any time we fail to find the (an) apparent horizon,
# then we reset our trial horizon surface to this initial guess before
# next attempting to find this horizon.)
#

keyword initial_guess_method[101] \
  "method used to set up initial guess for apparent horizon shape" \
  STEERABLE=recover
{
"read from named file"	:: "read from explicitly-named input file"
"read from h file"	:: \
  "read from input file named the same as the (later) h output file"
"Kerr/Kerr"		:: \
  "set to the (analytical) horizon of Kerr spacetime in Kerr coordinates \
   (n.b. Schwarzschild/EF is the special case spin=0 of this)"
"Kerr/Kerr-Schild"	:: \
  "set to the (analytical) horizon of Kerr spacetime in Kerr-Schild coordinates"
"coordinate sphere"	:: "set to a coordinate sphere"
"coordinate ellipsoid"	:: "set to a coordinate ellipsoid"
} "coordinate sphere"

# if a horizon could not be found, do we want to reset the horizon shape,
# or do we want to continue with the last shape?
boolean reset_horizon_after_not_finding[101] \
  "reset the horizon shape to the initial data if a horizon was not found" \
  STEERABLE=recover
{
} "yes"

# parameters for initial_guess_method = "read from named file"
string initial_guess__read_from_named_file__file_name[101] \
  "file name to read initial guess from" \
  STEERABLE=recover
{
".+" :: "file name to read initial guess from"
} "h.gp"

# parameters for initial_guess_method = "Kerr/Kerr"
real initial_guess__Kerr_Kerr__x_posn[101] "x coordinate of Kerr BH" STEERABLE=recover
{
*:* :: "any real number"
} 0.0
real initial_guess__Kerr_Kerr__y_posn[101] "y coordinate of Kerr BH" STEERABLE=recover
{
*:* :: "any real number"
} 0.0
real initial_guess__Kerr_Kerr__z_posn[101] "z coordinate of Kerr BH" STEERABLE=recover
{
*:* :: "any real number"
} 0.0
real initial_guess__Kerr_Kerr__mass[101] "mass of Kerr BH" STEERABLE=recover
{
(0.0:*		:: "BH mass = any real number > 0"
} 1.0
# n.b. my convention is that a=J/m^2 is dimensionless,
#      while MTW take a=J/m=m * (my a)
real initial_guess__Kerr_Kerr__spin[101] "dimensionless spin a=J/m^2 of Kerr BH" STEERABLE=recover
{
(-1.0:1.0)	:: \
  "dimensionless BH spin = J/m^2 = any real number with absolute value < 1"
} 0.6

# parameters for initial_guess_method = "Kerr/Kerr-Schild"
real initial_guess__Kerr_KerrSchild__x_posn[101] "x coordinate of Kerr BH" STEERABLE=recover
{
*:* :: "any real number"
} 0.0
real initial_guess__Kerr_KerrSchild__y_posn[101] "y coordinate of Kerr BH" STEERABLE=recover
{
*:* :: "any real number"
} 0.0
real initial_guess__Kerr_KerrSchild__z_posn[101] "z coordinate of Kerr BH" STEERABLE=recover
{
*:* :: "any real number"
} 0.0
real initial_guess__Kerr_KerrSchild__mass[101] "mass of Kerr BH" STEERABLE=recover
{
(0.0:*		:: "BH mass = any real number > 0"
} 1.0
# n.b. my convention is that a=J/m^2 is dimensionless,
#      while MTW take a=J/m=m * (my a)
real initial_guess__Kerr_KerrSchild__spin[101] "dimensionless spin a=J/m^2 of Kerr BH" STEERABLE=recover
{
(-1.0:1.0)	:: \
  "dimensionless BH spin = J/m^2 = any real number with absolute value < 1"
} 0.6

# parameters for initial_guess_method = "sphere"
real initial_guess__coord_sphere__x_center[101] "x coordinate of sphere center" STEERABLE=recover
{
*:* :: "any real number"
} 0.0
real initial_guess__coord_sphere__y_center[101] "y coordinate of sphere center" STEERABLE=recover
{
*:* :: "any real number"
} 0.0
real initial_guess__coord_sphere__z_center[101] "z coordinate of sphere center" STEERABLE=recover
{
*:* :: "any real number"
} 0.0
real initial_guess__coord_sphere__radius[101] "radius of sphere" STEERABLE=recover
{
(0.0:* :: "any real number > 0.0"
} 2.0

# parameters for initial_guess_method = "ellipsoid"
real initial_guess__coord_ellipsoid__x_center[101] \
  "x coordinate of ellipsoid center" STEERABLE=recover
{
*:* :: "any real number"
} 0.0
real initial_guess__coord_ellipsoid__y_center[101] \
  "y coordinate of ellipsoid center" STEERABLE=recover
{
*:* :: "any real number"
} 0.0
real initial_guess__coord_ellipsoid__z_center[101] \
  "z coordinate of ellipsoid center" STEERABLE=recover
{
*:* :: "any real number"
} 0.0
real initial_guess__coord_ellipsoid__x_radius[101] "x radius of ellipsoid" STEERABLE=recover
{
(0.0:* :: "any real number > 0.0"
} 2.0
real initial_guess__coord_ellipsoid__y_radius[101] "y radius of ellipsoid" STEERABLE=recover
{
(0.0:* :: "any real number > 0.0"
} 2.0
real initial_guess__coord_ellipsoid__z_radius[101] "z radius of ellipsoid" STEERABLE=recover
{
(0.0:* :: "any real number > 0.0"
} 2.0

################################################################################

#
# ***** I/O parameters *****
#

########################################

#
# parameters for BH diagnostics
#

#
# The file format is currently hard-wired to a simple ASCII format:
# - there is one file per AH
# - after some header comments (starting with '#'),
#   there is one line per successful-AH-finding
#   (there is *no* line if we fail to find an AH)
# - each line contains various tab-separated fields, as described by
#   the header comment
#
boolean output_BH_diagnostics \
  "should we output BH diagnostics to a data file for each AH found?" \
  STEERABLE=always
{
} "true"

#
# These file names are actually just "base" file names, with the full
# file names being given by a printf() format "%s/%s.ah%d.%s",
# where
# - the first %s is the BH diagnostics directory as set by the
#   BH_diagnostics_directory  parameter; this is automagically created
#   if it doesn't already exist
# - the second %s is the base file name,
# - the %d is the global Cactus time iteration number cctk_iteration,
# - the third %s is the file name extension as set by the
#   BH_diagnostics_file_name_extension  parameter
#
string BH_diagnostics_directory "directory for BH diagnostics output file(s)" \
  STEERABLE=always
{
".+" :: "any nonempty string"
"^$" :: "an empty string to default to IO::out_dir"
} ""
string BH_diagnostics_base_file_name \
  "base file name for BH diagnostics output file(s)" STEERABLE=always
{
".+" :: "any nonempty string"
} "BH_diagnostics"

string BH_diagnostics_file_name_extension \
  "extension for BH diagnostics data files" STEERABLE=always
{
".+" :: "any nonempty string"
} "gp"

########################################

#
# parameters for horizon-shape and other similar data files
#

# this parameter controls how often we write apparent-horizon-shape
# output files (format controlled by  horizon_file_format ); we still
# *find* apparent horizons at each time step, and compute BH diagnostics
# and (if desired) write them to the one-per-horizon BH diagnostics
# output files, but this parameter may be used to reduce the number
# of one-file-per-horizon-per-time-step apparent-horizon-shape output
# files
int output_h_every \
  "how often (in Cactus time steps) should we output h (0 to disable)?" \
  STEERABLE=always
{
0   :: "don't output h at all"
1:* :: "any integer >= 1"
} 1

# setting this > 0 is probably only of interest if the Newton iteration
# fails to converge, or if you're debugging AHFinderDirect internals
int output_Theta_every \
  "how often (in Cactus time steps) should we output the Theta(h) functions?" \
  STEERABLE=always
{
0   :: "don't output Theta(h) at all"
1:* :: "any integer >= 1"
} 0

int output_mean_curvature_every \
  "how often (in Cactus time steps) should we output the mean_curvature(h) functions?" \
  STEERABLE=always
{
0   :: "don't output mean_curvature(h) at all"
1:* :: "any integer >= 1"
} 0

boolean output_ASCII_files "output h and Theta(h) as ASCII files" STEERABLE=always
{
} "yes"

boolean output_HDF5_files "output h and Theta(h) as HDF5 files" STEERABLE=always
{
} "no"

# n.b. this refers to the patch system (angular) interpatch ghost zones,
#      *not* the Cactus interprocessor ghost zones
boolean output_ghost_zones_for_h \
  "should we include the patch system (angular) interpatch ghost zones \
   in h data files?" STEERABLE=always
{
} "false"

# n.b. making this "gnuplot" probably isn't a good idea,
#      since it could create confusion with gnuplot command files
string ASCII_gnuplot_file_name_extension \
  "extension for ASCII (gnuplot) data files" STEERABLE=always
{
".+" :: "any nonempty string"
} "gp"
string HDF5_file_name_extension "extension for HDF5 data files" STEERABLE=always
{
".+" :: "any nonempty string"
} "h5"

#
# These file names are actually just "base" file names, with the full
# file names being given by a printf() format "%s/%s.t%d.ah%d[.it%d].%s",
# where
# - the first %s is the output directory as set by the  h_directory
#   parameter (this same directory is used for all of these files);
#   this is automagically created if it doesn't already exist
# - the second %s is the base file name,
# - the first %d is the global Cactus time iteration number cctk_iteration,
# - the second %d is the apparent horizon number,
# - the optional third %d is the horizon finder iteration number
# - the third %s is the file name extension
#   {ASCII,HDF5}_data_file_name_extension
#
string h_directory \
  "directory for horizon shape and other similar output (or input) file(s)" \
   STEERABLE=always
{
".+" :: "any nonempty string"
"^$" :: "an empty string to default to IO::out_dir"
} ""

string h_base_file_name \
  "base file name for horizon shape h input/output file(s)" \
  STEERABLE=always
{
".+" :: "any nonempty string"
} "h"

string Theta_base_file_name "base file name for Theta(h) output file(s)" \
  STEERABLE=always
{
".+" :: "any nonempty string"
} "Theta"

string mean_curvature_base_file_name "base file name for mean_curvature(h) output file(s)" \
  STEERABLE=always
{
".+" :: "any nonempty string"
} "mean_curvature"

string Delta_h_base_file_name \
  "base file name for horizon-shape-update Delta_h output file(s)" \
  STEERABLE=always
{
".+" :: "any nonempty string"
} "Delta_h"

int h_min_digits \
  "minimum number of digits for the iteration number in the file name" STEERABLE=always
{
0:* :: ""
} 0

#
# Thomas Radke has written a set of OpenDX macros to read AHFinderDirect
# ASCII (gnuplot) format files.  These macros require a "control file"
# for each patch to specify some of the patch metadata.
#
# If we write them, the control file names are given by a printf() format
# "%s/%s.ah%d.%s", where
# - the first %s is the output directory as set by the  h_directory
#   parameter (this same directory is used for all of these files);
#   this is automagically created if it doesn't already exist
# - the second %s is h_base_file_name
# - the %d is the apparent horizon number
# - the third %s is the file name extension OpenDX_control_file_name_extension
#
boolean output_OpenDX_control_files \
  "should we output OpenDX control files to allow reading of AHFinderDirect \
   'ASCII (gnuplot)' format data files?" STEERABLE=always
{
} "true"
string OpenDX_control_file_name_extension \
  "file name extension for OpenDX control files" STEERABLE=always
{
".+" :: "any nonempty string"
} "dx"

########################################

#
# parameters mainly for debugging
#

# this is mainly useful for debugging purposes
boolean output_initial_guess \
  "should we output the initial guess back to the h data file?" \
  STEERABLE=always
{
} "false"

# for debugging convergence failures, we can optionally output
# h, Theta, and delta_h at each Newton iteration
# (the file names are the usual ones with ".it%d" appended)
boolean debugging_output_at_each_Newton_iteration \
  "should we output {h, Theta, delta_h} at each Newton iteration?" \
  STEERABLE=recover
{
} "false"

string Jacobian_base_file_name "base file name for Jacobian output file(s)" \
  STEERABLE=always
{
".+" :: "any valid file name"
} "Jacobian.dat"

################################################################################

#
# ***** parameters for setting a mask grid function ***
# ***** based on each apparent horizon shape *****
#

boolean set_mask_for_all_horizons \
  "should we set a mask grid function (or functions) for all horizons?" \
  STEERABLE=recover
{
} "false"

boolean set_mask_for_individual_horizon[101] \
  "should we set a mask grid function (or functions) for *this* horizon?" \
  STEERABLE=recover
{
} "false"

#
# If we set a mask, we do so based on partitioning the Cactus grid into
# 3 regions, an "inside", a "buffer", and an "outside" (typically the inside
# region is excised).  For a grid point a distance r[i] from horizon i's
# local coordinate origin, with horizon i's radius in this same direction
# (again, measured from its local coordinate origin) being r_horizon[i],
# the regions are defined as follows:
#	r[i] <= r_inner[i] for some i				==> inside
#	r[i] >  r_inner[i] for all i
#		and r[i] <= r_outer[i] for some i		==> buffer
#	r[i] >  r_outer[i] for all i				==> outside
# where
#	r_inner = mask_radius_multiplier * r_horizon
#		  +  mask_radius_offset * Delta_x
#	r_outer = r_innter  +  mask_buffer_thickness * Delta_x
# and where  Delta_x  is the Cactus base grid spacing (more precisely,
# the geometric mean of the base grid's x, y, and z Cactus grid spacings)
# (The "base grid" part here only matters if you're doing mesh refinement.)
#
real mask_radius_multiplier \
  "radius multiplier to define the 'inside' mask region" \
  STEERABLE=recover
{
(0:*) :: \
  "any positive real number; typically this will be slightly less than 1.0"
} 0.8
real mask_radius_offset \
  "radius offset (in Cactus base grid spacings) to define the 'inside' mask region" \
  STEERABLE=recover
{
*:* :: "any real number; typically this will be negative"
} -5.0
real mask_buffer_thickness \
  "thickness (in Cactus base grid spacings) of the 'buffer' mask region" \
  STEERABLE=recover
{
*:* :: "any real number; typically this will be positive"
} 5.0

#
# If this parameter is set to true, then we prevent the inside & buffer
#    regions from ever shrinking, i.e. if a point is ever set to the inside
#    value then we leave at that value forever, and if it's ever set to
#    the buffer value then we leave at that value or the inside value forever.
# If this parameter is set to false, then we set the inside/buffer/outside
#    values independently at each time step, so all three regions can grow
#    or shrink depending on the horizon positions/shapes.
#
boolean mask_is_noshrink \
  "should we prevent the inside & buffer regions from ever shrinking?" \
  STEERABLE=recover
{
} "true"

#
# If we're using mesh refinement (eg Carpet) it's possible to find a
# horizon on a fine grid, and then set a mask on a coarser grid where
# the masked-off region is only a few grid points across.  This then
# confuses various other thorns (eg LegoExcision) which require excised
# regions to be "not too small".
#
# So...  This parameter may be used to specify that the mask not be set
# for a given horizon (just as if that horizon wasn't found) on any grid
# where the inner mask radius  r_inner  is < this many grid spacings.
# More precisely, we skip masking for any horizon where
#	r_inner,min < min_horizon_radius_points_for_mask * Delta_x_current,max
# where
#	r_inner,min = the minimum over all angles of the inner mask radius
#		      r_inner
#	Delta_x_current,max = the maximum of the x,y,and z grid spacings
#			      in the *current* Cactus grid
#
# Note that since r_inner can be negative (if  mask_radius_offset < 0 ),
# the default setting for this parameter is effectively -infinity, to
# excise any horizon.
#
real min_horizon_radius_points_for_mask \
  "only set mask if min r_inner >= this number of local grid spacings" \
  STEERABLE=recover
{
*:* :: "any real number"
} -1.0e10

#
# This thorn supports two types of mask grid functions:
# * the "old" style is that defined by CactusEinstein/AHFinder:
#   the mask is a CCTK_REAL grid function, which is set to specified
#   (CCTK_REAL) values inside/outside the (adjusted) horizon
# * the "new" style is that defined by CactusEinstein/SpaceMask:
#   the mask is a CCTK_INT grid function, with a specified bit field
#   set to specified values inside/outside the (adjusted) horizon
#	[N.b. AHFinderDirect doesn't set up the bit fields --
#	you must arrange for some other thorn to do this.  At
#	present AHFinderDirect converts the names specified here
#	to integer bit masks only once, in its setup routine;
#	this may change to do this conversion each time AHFinderDirect
#	does horizon finding.]
# You can choose to set either (or even both!) of these.
#
boolean set_old_style_mask \
  "if we're setting a mask, should we set an old-style (CCTK_REAL) mask gridfn?" STEERABLE=recover
{
} "true"
boolean set_new_style_mask \
  "if we're setting a mask, should we set an new-style (CCTK_INT) mask gridfn?" STEERABLE=recover
{
} "false"

#
# mask info for  mask_type = "old-style (CCTK_REAL)"
# defaults match CactusEinstein/AHFinder
#
string old_style_mask_gridfn_name "name of the old-style mask grid function" STEERABLE=recover
{
".+" :: "any valid Cactus grid function name"
} "SpaceMask::emask"
real old_style_mask_inside_value \
  "set the old-style mask to this value in the 'inside' region" STEERABLE=recover
{
*:* :: "any real number"
} 0.0
real old_style_mask_buffer_value \
  "set the old-style mask to this value in the 'buffer' region" STEERABLE=recover
{
*:* :: "any real number"
} 0.5
real old_style_mask_outside_value \
  "set the old-style mask to this value in the 'outside' region" STEERABLE=recover
{
*:* :: "any real number"
} 1.0

#
# mask info for  mask_type = "new-style (CCTK_INT)"
#
string new_style_mask_gridfn_name "name of the new-style mask grid function" STEERABLE=recover
{
".+" :: "any valid Cactus grid function name"
} "SpaceMask::space_mask"
string new_style_mask_bitfield_name \
  "name of the new-style mask bit field ('type') as registered with SpaceMask" STEERABLE=recover
{
".+" :: "any valid bit field ('type') name registered with the SpaceMask thorn"
} "mask"
string new_style_mask_inside_value \
  "set the specified bit field of the new-style mask to this named state \
   at grid points in the 'inside' region" STEERABLE=recover
{
".+" :: "any state name registered with the SpaceMask thorn"
} "inside"
string new_style_mask_buffer_value \
  "set the specified bit field of the new-style mask to this named state \
   at grid points in the 'buffer' region" STEERABLE=recover
{
".+" :: "any state name registered with the SpaceMask thorn"
} "buffer"
string new_style_mask_outside_value \
  "set the specified bit field of the new-style mask to this named state \
   at grid points in the 'outside' region" STEERABLE=recover
{
".+" :: "any state name registered with the SpaceMask thorn"
} "outside"

################################################################################

#
# ***** parameters controlling how we handle errors/warnings *****
#

#
# We currently "handle" the following errors/warnings by CCTK_VWarn();
# these parameters give the warning levels (in the sense of the -W flag
# to Cactus, and the first argument to CCTK_VWarn()) for various conditions:
#

# point outside (or too close to boundary of) Cactus grid,
# i.e. geometry interpolator returns CCTK_ERROR_INTERP_POINT_OUTSIDE
# ... warning level if error occurs on first Newton iteration,
#     i.e. when evaluating expansion/Jacobian for initial guess
# ... this may mean that  driver::ghost_size  is too small, especially
#     if any horizon crosses a symmetry boundary
#     ==> visible message by default
int warn_level__point_outside__initial \
  "warning level for point outside (or too close to boundary of) Cactus grid \
   (error occurs on first Newton iteration)" STEERABLE=recover
{
-1:* :: "any valid Cactus warning level"
} 1
# ... warning level if error occurs on a subsequent Newton iteration
# ... this probably "just" means there's no horizon in the grid
#     ==> no visible message by default
int warn_level__point_outside__subsequent \
  "warning level for point outside (or too close to boundary of) Cactus grid \
   (error occurs on subsequent Newton iteration)" STEERABLE=recover
{
-1:* :: "any valid Cactus warning level"
} 2

# the Cactus configure process didn't find a finite() function,
# so we're skipping the "check that the geometry is finite" tests
# even though the user set  check_that_geometry_is_finite = "true"
int warn_level__skipping_finite_check \
  "warning level if the user sets  check_that_geometry_is_finite \
   but the Cactus configure process doesn't find a  finite()  function \
   so we have to skip the finite-geometry check" STEERABLE=recover
{
-1:* :: "any valid Cactus warning level"
} 3

# infinity and/or NaN in interpolated {g_ij, partial_k g_ij, K_ij}
int warn_level__nonfinite_geometry \
  "warning level if we find infinity and/or NaN in the interpolated geometry \
   values {g_ij, partial_k g_ij, K_ij}" STEERABLE=recover
{
-1:* :: "any valid Cactus warning level"
} 1

# interpolated g_{ij} isn't positive definite
# (usually this means we're too close to a singularity)
# ... warning level if error occurs on first Newton iteration,
#     i.e. when evaluating expansion/Jacobian for initial guess
int warn_level__gij_not_positive_definite__initial \
  "warning level if the interpolated g_{ij} isn't positive definite \
   (usually this means we're too close to a singularity) \
   (error occurs on first Newton iteration)" STEERABLE=recover
{
-1:* :: "any valid Cactus warning level"
} 2
# ... warning level if error occurs on a subsequent Newton iteration
int warn_level__gij_not_positive_definite__subsequent \
  "warning level if the interpolated g_{ij} isn't positive definite \
   (usually this means we're too close to a singularity) \
   (error occurs on subsequent Newton iteration)" STEERABLE=recover
{
-1:* :: "any valid Cactus warning level"
} 2

################################################################################

#
# ***** parameters for the Newton's-method solution of Theta(h) = 0 *****
#

#
# The first time we (try to) find a given horizon, our initial guess
# is likely to be rather inaccurate, so we may need a larger number of
# iterations.  But if we've found this horizon before, then we have its
# previous position as an initial guess, so (assuming that we've been
# scheduled at each time step) we shouldn't need as many iterations.
#
int max_Newton_iterations__initial \
  "maximum number of Newton iterations before giving up \
   when initially finding a given horizon" STEERABLE=recover
{
(0:* :: "any positive integer"
} 20
int max_Newton_iterations__subsequent \
  "maximum number of Newton iterations before giving up \
   when re-finding a given horizon after finding it before" STEERABLE=recover
{
(0:* :: "any positive integer"
} 10

#
# To avoid the Newton iteration diverging if there is no horizon present
# or if the initial guess is bad, we limit the distance the horizon may
# move in any single Newton iteration (i.e. the infinity-norm of Delta_h)
# to <= this fraction of the mean horizon radius
#
real max_allowable_Delta_h_over_h \
  "don't let horizon move > this fraction of mean radius in a Newton iteration" STEERABLE=recover
{
(0.0:* :: "any positive real number"
} 0.1

#
# To avoid the Newton iteration diverging (or more accurately, to detect
# this divergence faster) if there is no horizon present or if the initial
# guess is bad, we also deem any horizon with a mean radius (measured around
# its local coordinate origin) larger than this to be "not found".  The
# default is effectively "no limit"; AHFinderDirect will be more efficient
# if you (re)set this (for each horizon) to the coordinate radius of a
# sphere a bit larger than you expect the horizon to be (eg a sphere with
# areal radius 4m or so, where m is the ADM mass of the slice).
#
real max_allowable_horizon_radius[101] \
  "max mean-coordinate-radius allowed for any trial surface \
   before we give up and say we can't find this horizon" STEERABLE=recover
{
(0.0:* :: "any positive real number"
} 1.0e10

#
# convergence tolerance for the Newton iteration
#
real Theta_norm_for_convergence \
  "we declare the horizon to be found if ||Theta||_infinity <= this" STEERABLE=recover
{
(0.0:* :: "any positive real number"
} 1.0e-8

#
# To detect some divergence cases faster, we also deem any horizon where
# the infinity-norm of Theta exceeds this parameter, to be "not found".
# The default is effectively "no limit".
#
real max_allowable_Theta \
  "max ||Theta||_infinity allowed for any trial surface \
   before we give up and say we can't find this horizon" STEERABLE=recover
{
(0.0:* :: "any positive real number"
} 1.0e10

#
# To detect some divergence cases faster, we also do not allow the Theta
# norm to grow for more than a certain number of iterations.
#
int max_allowable_Theta_growth_iterations \
  "max number of consecutive iterations during which ||Theta||_infinity \
   is allowed to grow before we give up and say we can't find this horizon" STEERABLE=recover
{
0   :: "infinitly many"
1:* :: "that many"
} 0

#
# To detect some divergence cases faster, we also do not allow the Theta
# norm to oscillate without shrinking for more than a certain number of
# iterations.
#
int max_allowable_Theta_nonshrink_iterations \
  "max number of consecutive iterations during which ||Theta||_infinity \
   is allowed to oscillate without shrinking before we give up and say \
   we can't find this horizon" STEERABLE=recover
{
0   :: "infinitly many"
1:* :: "that many"
} 0

################################################################################

#
# ***** parameters to define the patch system(s) *****
#

#
# For each apparent horizon, you need to set these parameters to the
# Cactus xyz coordinates of a "local origin point" inside the horizon,
# which will serve as the origin for the apparent horizon finder's
# local angular coordinate system.
#
# The apparent horizon surface (and in fact all the trial surfaces the
# apparent horizon finder generates while iteratively solving the apparent
# horizon equation) is restricted to being a Strahlkoerper ("star-shaped
# region") about the origin point.  That is, each surface must be of
# the form r = h(angle) with h a single-valued function.
#
# If the origin point is too far from the actual horizon centroid, i.e.
# if it's too close to the horizon surface itself, then the apparent
# horizon finder's Newton iteration won't converge as quickly or robustly.
# However, in practice the tolerances on this are quite loose -- 1/4
# of the horizon radius is no problem, and even 1/2 the horizon radius
# only slows the convergence by an extra iteration or two.
#
real origin_x[101] "global x coordinate of patch system origin" STEERABLE=recover
{
*:* :: "any real number"
} 0.0
real origin_y[101] "global y coordinate of patch system origin" STEERABLE=recover
{
*:* :: "any real number"
} 0.0
real origin_z[101] "global z coordinate of patch system origin" STEERABLE=recover
{
*:* :: "any real number"
} 0.0

boolean move_origins "move the origins with the horizons" STEERABLE=recover
{
} "no"

boolean reshape_while_moving "reshape the horizons when moving them" STEERABLE=recover
{
} "no"

boolean predict_origin_movement "predict origin movement when moving the origins" STEERABLE=recover
{
} "no"

boolean track_origin_from_grid_scalar[101] "track horizon origin from given grid scalars" STEERABLE=recover
{
} "no"
string track_origin_source_x[101] "grid scalar containing the x component of the origin estimate" STEERABLE=recover
{
  "" :: "don't use this feature"
  "[a-zA-Z_][a-zA-Z0-9_]*[:][:][a-zA-Z_][a-zA-Z0-9_]*(\[0-9+\])" :: "name of a grid scalar"
} ""
string track_origin_source_y[101] "grid scalar containing the x component of the origin estimate" STEERABLE=recover
{
  "" :: "don't use this feature"
  "[a-zA-Z_][a-zA-Z0-9_]*[:][:][a-zA-Z_][a-zA-Z0-9_]*(\[0-9+\])" :: "name of a grid scalar"
} ""
string track_origin_source_z[101] "grid scalar containing the x component of the origin estimate" STEERABLE=recover
{
  "" :: "don't use this feature"
  "[a-zA-Z_][a-zA-Z0-9_]*[:][:][a-zA-Z_][a-zA-Z0-9_]*(\[0-9+\])" :: "name of a grid scalar"
} ""

#
# The "(rotating)" patch system types are ok for evaluating Theta(h),
# but alas they don't work yet for apparent horizon finding
# (the Jacobian computation doesn't yet grok the nonlocal rotation BCs).
#
keyword patch_system_type[101] "what type of patch system should we use?"
{
# choose this for normal use
"match Cactus grid symmetry" :: \
  "choose automagically based on grid symmetries and the patch system's origin"

"full sphere"	:: "full sphere, no symmetries"
"+z hemisphere"	:: "mirror symmetry across z=0 plane"

"+xy quadrant (rotating)" :: "90 degree periodic rotation symmetry about z axis"
"+xy quadrant (mirrored)" :: "mirror symmetry across x=0 and y=0 planes"

"+xz quadrant (mirrored)" :: "mirror symmetry across x=0 and z=0 planes"
"+xz quadrant (rotating)" :: \
  "180 degree periodic rotation symmetry about z axis \
   and mirror symmetry across z=0 plane"

"+xyz octant (rotating)"  :: \
  "90 degree periodic rotation symmetry about z axis \
   and mirror symmetry across z=0 plane"
"+xyz octant (mirrored)"  :: "mirror symmetry across x=0 and y=0 and z=0 planes"
} "match Cactus grid symmetry"

#
# This parameter sets the angular resolution of each patch systems:
# the angular grid spacing in degrees is 90.0/N_zones_per_right_angle.
#
# In practice the error in the horizon position is usually dominated
# by the errors from interpolating the Cactus gij and Kij to the horizon
# position, not by the angular finite differencing or interpatch interpolation
# errors.  Thus this parameter can be made fairly small (low resolution)
# for better performance, without seriously affecting the accuracy
# with which we can locate the horizon.
#
# *** IMPORTANT
# *** For any patch system type other than "full sphere", there are
# *** patches with 45 degree widths, so this parameter must be even.
#
# Normally we use Simpson's rule (in a variant which doesn't require the
# number of zones to be even) for angular integrations over the patch
# system.  However, if the number of angular zones in a patch is very
# small and odd (i.e. 3 or 5), then we have to use the trapezoid rule
# instead, so the integrations are less accurate.  This occurs for
# N_zones_per_right_angle = 3 or 5 for a full sphere patch system,
# or N_zones_per_right_angle = 6 or 10 for any other patch system type.
#
# For large values of this parameter, this thorn takes CPU time which
# scales roughly as the ...
# ... 6th power of this parameter
#     if Jacobian_store_solve_method = "dense matrix/LAPACK"
# ... 3rd power of this parameter
#     if Jacobian_store_solve_method = "row-oriented sparse matrix/ILUCG"
#
#int N_zones_per_right_angle[101] "sets angular resolution of patch systems" ACCUMULATOR-BASE max_N_zones_per_right_angle
int N_zones_per_right_angle[101] "sets angular resolution of patch systems"
{
1:* :: "any integer >= 1; must be even for patch systems other than full-sphere"
} 18

# Default parameter values do not trigger accumulator expressions.
# Therefore the default values here must be large enough for the default
# horizon shape.
#int max_N_zones_per_right_angle "the maximum of all N_zones_per_right_angle -- calculated automatically; do not set this parameter directly" ACCUMULATOR=((x>y)*x+!(x>y)*y)
int max_N_zones_per_right_angle "the maximum of all N_zones_per_right_angle"
{
1:* :: "must be at least the maximum of all N_zones_per_right_angle"
} 18

#
# This parameter sets the width of the interpatch ghost zones in the
# patch system.  Note that this thorn uses the terminology "ghost zone"
# for any of what Cactus in general now calls a "boundary zone" or a
# "symmetry zone" or a "patch zone".
#
# This parameter must be at least
# ... 2 if FINITE_DIFF_ORDER is set to 4 in "src/include/config.hh"
# ... 1 if FINITE_DIFF_ORDER is set to 2 in "src/include/config.hh"
# The code checks for this being too small, and reports a fatal error if so.
#
int ghost_zone_width "number of ghost zones on each side of a patch"
{
0:* :: "any integer >= 0"
} 2

#
# Our code that computes surface integrals over patches (used for
# computing BH diagnostics like centroids, areas, masses, etc) silently
# assumes that this parameter is == 1, so you should probably leave
# it at that setting.
#
int patch_overlap_width \
  "number of grid points that nominally-just-touching patches should overlap"
{
1:*:2 :: "any integer >= 0; current implementation requires that it be odd"
} 1

################################################################################

#
# ***** parameters for the Jacobian matrix *****
#

keyword Jacobian_compute_method "how do we compute the Jacobian matrix?" STEERABLE=recover
{
# for debugging only"
"numerical perturbation"    :: "n.b. this is *very* slow"

# use this for normal apparent horizon finding
"symbolic differentiation with finite diff d/dr" :: \
  "fast, tricky programming, uses only gij, dx gij, Kij"

# alas, this isn't implemented yet :(
"symbolic differentiation"  :: \
  "fast, tricky programming, uses gij, dx gij, dxx gij, Kij, dx Kij"
} "symbolic differentiation with finite diff d/dr"

########################################

#
# This parameter lists all known methods for storing the Jacobian and
# solving linear systems with the Jacobian as the LHS matrix.  See
# "src/include/config.hh" for which of these are actually compiled in at
# the moment.  N.b. each compiled-in method requires linking with the
# corresponding linear-solver library; see "src/make.configuration.defn"
# for details on these libraries.
#
keyword Jacobian_store_solve_method \
  "how do we store/linear-solve the Jacobian matrix?" STEERABLE=recover
{
# good for debugging, but very slow for large  N_zones_per_right_angle
"dense matrix/LAPACK"	:: \
  "store as (Fortran) dense matrix, solve with LAPACK routines"
# use this for normal runs
"row-oriented sparse matrix/ILUCG" :: \
  "store as sparse matrix (row-oriented storage format), \
   solve with ILUCG (incomplete LU decomposition - conjugate gradient) method"
# this will probably replace ILUCG, but it's not ready yet :(
"row-oriented sparse matrix/UMFPACK" :: \
  "store as sparse matrix (row-oriented storage format), \
   solve with UMFPACK (sparse LU decomposition) method"
} "row-oriented sparse matrix/UMFPACK"

#
# Extra parameters for individual linear solvers
#

# extra parameters for
#   Jacobian_store_solve_method == "row-oriented sparse matrix/ILUCG"
real ILUCG__error_tolerance "error tolerance for conjugate gradient iteration" STEERABLE=recover
{
(*:0.0)	:: \
  "negative ==> scale the absolute value by the floating point roundoff \
                threshold, e.g. -256.0 means to allow the last 8 bits of \
                the solution to be in error"
(0.0:*)	:: \
  "positive ==> error tolerance"
} 1.0e-10
boolean ILUCG__limit_CG_iterations \
  "should we limit the maximum number of conjugate gradient iterations allowed?" STEERABLE=recover
{
"false"	:: "no limit on CG iterations"
"true"	:: "limit to Neqns CG iterations"
} "true"

# extra parameters for
#   Jacobian_store_solve_method == "row-oriented sparse matrix/UMFPACK"
int UMFPACK__N_II_iterations \
  "number of iterative-improvement iterations to do inside UMFPACK \
   after the sparse LU decompose/solve, each time we solve a linear system" STEERABLE=recover
{
-1	:: "use the UMFPACK default"
0	:: "don't do iterative improvement"
1:*	:: "any positive integer \
	    (in practice a few iterations give almost all the benefit)"
} 0

########################################

#
# This parameter controls two different sorts of one-sided finite
# differencing:
# - numerical-perturbation Jacobian computations
# - the finite differencing part of the "symbolic differentiation
#   with finite diff d/dr" Jacobian computation
#
# Notes on this parameter:
# - don't set it too small or roundoff errors will become large
# - don't set it too large or finite differencing errors will become large
# In practice the default value should be fine
#
real Jacobian_perturbation_amplitude \
  "perturbation amplitude for 1-sided finite differencing for Jacobians" STEERABLE=recover
{
(0.0:* :: "any real number > 0"
} 1.0e-6

# if AHFinderDirect::method = "test Jacobian", should we test all
# known methods for computing the Jacobian, or just the numerical perturbation
# method (the latter may be useful of some other methods are broken)
boolean test_all_Jacobian_compute_methods \
  "should we test all Jacobian computation methods, or just NP?" STEERABLE=recover
{
} "true"

################################################################################

#
# ***** parameters for how we compute the slice's geometry *****
# ***** (gij, Kij, partial_k gij) *****
#

########################################

#
# parameters for geometry interpolation
#
# This 3D interpolator is used to interpolate gij and Kij from the
# Cactus grid to the position of each trial horizon surface, giving
# gij, Kij, and partial_x gij as outputs.  This interpolator must have
# the following properties:
# - It must support taking at least 1st derivatives as part of the
#   interpolation.
# - It should give at least $C^1$ interpolants for smooth data, otherwise
#   the Theta(h) function will have "spikes" and the Newton iteration may
#   fail to converge all the way down to tight error tolerances.  $C^2$
#   would be even better, but in practice a ($C^1$) Hermite interpolant
#   works well.
# In practice the default values for these parameters should work fine
# (with the AEIThorns/AEILocalInterp local interpolator).
#

string coordinate_system_name \
  "name under which the coordinate system is registered in Cactus" STEERABLE=recover
{
".+" :: "any nonempty string"
} "cart3d"

string geometry_interpolator_name \
  "name under which the geometry interpolation operator is registered in Cactus" STEERABLE=recover
{
".+" :: "any nonempty string"
} "Hermite polynomial interpolation"

#
# The defaults here are set up for 2nd order Hermite interpolation,
# which requires driver::ghost_size >= 2.
#
string geometry_interpolator_pars \
  "parameters for the geometry interpolator" STEERABLE=recover
{
".*" :: \
  "any string acceptable to Util_TableSetFromString() and to the interpolator"
} "order=2 \
   boundary_off_centering_tolerance={1.0e-10 1.0e-10 1.0e-10 1.0e-10 1.0e-10 1.0e-10} \
   boundary_extrapolation_tolerance={0.0 0.0 0.0 0.0 0.0 0.0}"

########################################

#
# for testing/debugging purposes, instead of interpolating the geometry
# from the Cactus grid, we can instead hardwire it to an Eddington-Finkelstein
# slice of Schwarzschild spacetime
#
boolean hardwire_Schwarzschild_EF_geometry \
  "should we hard-wire the geometry to Schwarzschild/EF instead of \
   interpolating from the Cactus grid?"
{
} "false"

real geometry__Schwarzschild_EF__mass "mass of Schwarzschild BH"
{
(0.0:*		:: "BH mass = any real number > 0"
} 1.0

real geometry__Schwarzschild_EF__x_posn "x coordinate of Schwarzschild BH"
{
*:* :: "any real number"
} 0.0
real geometry__Schwarzschild_EF__y_posn "y coordinate of Schwarzschild BH"
{
*:* :: "any real number"
} 0.0
real geometry__Schwarzschild_EF__z_posn "z coordinate of Schwarzschild BH"
{
*:* :: "any real number"
} 0.0

# some of the Schwarzschild/EF formulas have 0/0 limits on the z axis;
# this parameter controls where we switch from the generic formulas to
# the L'Hopital's-rule z axis limits
# - don't set this parameter too small or roundoff errors will be excessive
# - don't set this parameter too large or finite differencing errors will
#   be excessive
# in practice the default value should be fine
# n.b. this is used for centered finite differencing, unlike the Jacobian
real geometry__Schwarzschild_EF__epsilon \
  "threshold for sin^2 theta = (x^2+y^2)/r^2 below which we use z axis limits"
{
(0.0:*	:: "this should be somewhat above the floating-point roundoff level"
} 1.0e-9

# we compute partial_k g_ij by numerical finite differencing of the exact
# analytical g_ij values; this parameter sets the "grid spacing" for this
# - don't set this parameter too small or roundoff errors will be excessive
# - don't set this parameter too large or finite differencing errors will
#   be excessive
# in practice the default value should be fine
# ... n.b. this finite differencing is *centered*, unlike that in the
#          Jacobian computation
real geometry__Schwarzschild_EF__Delta_xyz \
  "finite diff pseuo-grid spacing for computing partial_k g_ij"
{
(0.0:* :: "any real number > 0"
} 1.0e-6

########################################

#
# These parameters control whether we check that various angular gridfns
# are finite (neither NaN nor infinity) at various points in evaluating
# the Theta(h) function.  These are pretty cheap tests, and they're quite
# useful in catching assorted wierdness, so it's probably worth leaving
# them enabled unless you're trying to squeeze every last nanosecond...
#
boolean check_that_h_is_finite \
  "should we check that horizon shape function h is finite?" \
  STEERABLE=recover
{
} "true"
boolean check_that_geometry_is_finite \
  "should we check the interpolated geometry variables are finite?" \
  STEERABLE=recover
{
} "true"

################################################################################

#
# ***** parameters for the interpatch interpolation *****
#

#
# This 1D local interpolator is used to interpolate the h function between
# angular patches.  Because any given patch boundary only interpolates
# from a single neighboring patch (this is to simplify the bookkeeping),
# near the patch corners this interpolator will have to be used off-centered.
# Thus it's desirable to use an interpolator which retains reasonable
# accuracy right up to the edge of the data range.  In practice a Lagrange
# polynomial interpolant works well.
#

string interpatch_interpolator_name \
  "name under which the interpatch interpolation operator is registered in Cactus" STEERABLE=recover
{
".+" :: "any string (in practice it should be nonempty)"
} "Lagrange polynomial interpolation"

string interpatch_interpolator_pars \
  "parameters for the interpatch interpolator" STEERABLE=recover
{
".*" :: \
  "any string acceptable to Util_TableSetFromString() and to the interpolator"
} "order=5"

################################################################################

#
# ***** parameters for the surface interpolation *****
#

#
# This 2D local interpolator is used to interpolate the horizon radius
# arbitrary angular coordinate, to find out whether a given xyz point is
# inside or outside the horizon.
#
# At present this interpolator is used only if  set_horizon_mask  is true.
# Otherwise, these parameters may both be set to empty strings.
#

string surface_interpolator_name \
  "name under which the surface interpolation operator is registered in Cactus" STEERABLE=recover
{
"^$" :: "the empty string if this interpolator isn't going to be used"
".+" :: "any string (in practice it should be nonempty)"
} "Lagrange polynomial interpolation"

string surface_interpolator_pars \
  "parameters for the surface interpolator" STEERABLE=recover
{
".*" :: \
  "any string acceptable to Util_TableSetFromString() and to the interpolator"
} "order=2 \
   boundary_off_centering_tolerance={1.0e-10 1.0e-10 1.0e-10 1.0e-10} \
   boundary_extrapolation_tolerance={0.0 0.0 0.0 0.0}"

################################################################################

#
# ***** parameters for computing line/surface integrals over the horizon *****
#

#
# In the descriptions, N is the number of grid zones in a patch
# in either the rho or the sigma direction, i.e. it's 45 or 90 degrees
# divided by delta_drho_dsigma.
#
keyword integral_method "how do we compute integrals over the horizon?" STEERABLE=recover
{
"trapezoid"		:: "alternate name for trapezoid rule"
"trapezoid rule"	:: "trapezoid rule (2nd order for smooth functions)"
"Simpson"		:: "alternate name for Simpson's rule"
"Simpson's rule"	:: \
  "Simpson's rule (4th order for smooth fns, requires N to be even)"
"Simpson (variant)"	:: "alternate name for Simpson's rule variant"
"Simpson's rule (variant)":: \
  "Simpson's rule variant (4th order for smooth fns, requires N >= 7)"

# choose this for normal use (assuming FINITE_DIFF_ORDER is set to 4
# in "src/include/config.hh")
"automatic choice"	:: \
  "choose Simpson's rule or variant if applicable, otherwise trapezoid rule"
} "automatic choice"

################################################################################

###
### ***** parameters for the test driver "src/patch/test_patch_system.cc" *****
###
### By default this test driver isn't compiled into the cactus executable,
### and these parameters are ignored.  To compile this test driver into
### the cactus executable (and have these parameters used),
### - edit the list of "source files in this directory" in
###   "src/patch/make.code.defn" to add  test_patch_system.cc
### - comment out the list of "subdirectories containing source files" in
###   "src/make.code.defn", and uncomment the alternate list which is
###   normally commented out there (this omits the higher-level directories
###   src/elliptic/, src/gr/, and src/driver/)
###
##keyword which_test "which test should we do?"
##{
##"gridfn"	:: "set up test fn(x,y,z), print it"
##"read gridfn"	:: "read in ghosted test fn(x,y,z), print it"
##"synchronize"	:: "set up test fn(x,y,z), synchronize it, print errors"
##"ghost zone Jacobian":: \
##  "set up test fn(x,y,z), compute Jacobian of gz.synchronize(), compare with NP"
##"derivatives"	:: "set up test fn(rho,sigma), take derivs, print errors"
##} "gridfn"
##
##int which_derivs "bit flags to specify which derivatives to test"
##{
##0:63 :: "any set of bit flags"
##} 63
##
### true ==> gives a more thorough test of the Jacobian,
###          but makes the test run much slower
### false ==> gives a slightly less thorough test, but runs faster
##boolean NP_Jacobian__perturb_all_y_patch_points \
##  "should we perturb at *all* points in the y patch, or just those with the \
##   iperp which is (supposedly) involved in the interpatch interpolation?"
##{
##} "true"
##
##################################################################################

configuration: REQUIRES THORNS: SpaceMask

OPTIONAL HDF5
{
}

context: driver/mask.cc:
```
// mask.cc -- set a mask gridfn based on each horizon shape
// $Header$
//
// <<<access to persistent data>>>
/// <<<data structures local to this file>>>
/// <<<prototypes for functions local to this file>>>
// AHFinderDirect_do_masks - top-level driver for all mask stuff
/// setup_mask_grid_info - setup mask grid origin/delta etc
/// setup_mask_dataptrs_and_bitfields - map gridfn/bitfield names to ptr/bitmask
/// set_mask_gridfn - set mask gridfn(s) based on each horizon's shape
/// set_mask_gridfn_to_outside_value - ... "outside" value
/// set_mask_gridfn_to_inside_and_buffer_values - "inside"/"buffer" values
/// inner_mask_radius - compute inner mask radius r_inner given r_horizon
/// outer_mask_radius - compute outer mask radius r_outer given r_inner
//

#include <stdio.h>
#include <assert.h>
#include <math.h>

#include "util_Table.h"
#include "cctk.h"
#include "cctk_Arguments.h"
#include "cctk_Parameters.h"

#include "SpaceMask.h"			// from thorn SpaceMask

#include "config.h"
#include "stdc.h"
#include "../jtutil/util.hh"
#include "../jtutil/array.hh"
#include "../jtutil/cpm_map.hh"
#include "../jtutil/linear_map.hh"
using jtutil::error_exit;

#include "../patch/coords.hh"
#include "../patch/grid.hh"
#include "../patch/fd_grid.hh"
#include "../patch/patch.hh"
#include "../patch/patch_edge.hh"
#include "../patch/patch_interp.hh"
#include "../patch/ghost_zone.hh"
#include "../patch/patch_system.hh"

#include "../elliptic/Jacobian.hh"

#include "../gr/gfns.hh"
#include "../gr/gr.hh"

#include "horizon_sequence.hh"
#include "BH_diagnostics.hh"
#include "driver.hh"

// define this to get extra debugging on the mask grid origin/delta etc
#undef DEBUG_MASK_GRID

// all the code in this file is inside this namespace
namespace AHFinderDirect
	  {

//******************************************************************************

//
// ***** access to persistent data *****
//
extern struct state state;

//******************************************************************************

//
// ***** data structures local to this file *****
//

//
// This structure holds all the information we need about the (Cactus)
// grid where the mask gridfn(s) is/are stored.
//
namespace {
struct	mask_grid_info
	{
	const cGH *GH;			// --> Cactus grid hierarchy

	// Cactus coordinate system
	// (for the *current* grid if we are doing mesh refinement)
	fp proc_coord_origin[N_GRID_DIMS];	// global (x,y,z) of
						// *this processor's*
						// (i,j,k) = (0,0,0)
						// grid point
	fp coord_delta[N_GRID_DIMS];		// (x,y,z) grid spacing

	// maximum of x,y,z grid spacings
	fp max_coord_delta;

	// geometric mean of x,y,z grid spacings,
	// on the *base* grid if we are doing mesh refinement
	// (we need the base-grid semantics to make excision consistent
	//  across refinement levels)
	fp base_grid_mean_coord_delta;

	// dimensions of gridfn data on this processor, viewed as a 3-D array
	// n.b. storage ordering is Fortran,
	//	i.e. i is contiguous, j has stride Ni, k has stride Ni*Nj
	CCTK_INT proc_gridfn_dims[N_GRID_DIMS];


	//
	// coordinate conversion functions
	//
	// here ijk = gridfn array indices on this processor
	//      xyz = floating-point global coordinates
	//

	// convert integer ijk --> floating-point global xyz coordinates
	fp global_xyz_of_ijk(int axis, int ijk) const
		{ return proc_coord_origin[axis] + ijk*coord_delta[axis]; }
	// convert floating-point global xyz --> integer ijk coordinates
	// ... but as a floating-point number
	fp fp_ijk_of_global_xyz(int axis, fp xyz) const
		{ return (xyz - proc_coord_origin[axis]) / coord_delta[axis]; }
	// convert floating-point global_xyz --> integer ijk coordinates
	// ... rounding down/up to the next integer (= to the next grid point)
	int ijk_floor_of_global_xyz(int axis, fp xyz) const
		{ return jtutil::ifloor(fp_ijk_of_global_xyz(axis, xyz)); }
	int ijk_ceil_of_global_xyz (int axis, fp xyz) const
		{ return jtutil::iceil(fp_ijk_of_global_xyz(axis, xyz)); }
	};
	  }

//******************************************************************************

//
// ***** prototypes for functions local to this file *****
//

namespace {
void setup_mask_grid_info(CCTK_ARGUMENTS, struct mask_grid_info& mgi);
void setup_mask_dataptrs_and_bitfields(const cGH *GH,
				       struct mask_info& mask_info);
void set_mask_gridfn(int N_horizons,
		     const struct AH_data* const AH_data_array[],
		     const struct mask_grid_info& mgi,
		     const struct mask_info& mask_info,
		     const struct verbose_info& verbose_info);

void set_mask_gridfn_to_outside_value(const struct mask_grid_info& mgi,
				      const struct mask_info& mask_info,
				      const struct verbose_info& verbose_info);
void set_mask_gridfn_to_inside_and_buffer_values
	(const struct mask_grid_info& mgi,
	 int use_min_i, int use_max_i,
	 int use_min_j, int use_max_j,
	 int use_min_k, int use_max_k,
	 const struct mask_info& mask_info,
	 const patch_system& ps,
	 const struct verbose_info& verbose_info);
fp inner_mask_radius(const struct mask_grid_info& mgi,
		     const struct mask_info& mask_info,
		     fp r_horizon);
fp outer_mask_radius(const struct mask_grid_info& mgi,
		     const struct mask_info& mask_info,
		     fp r_inner);
	  }

//******************************************************************************

//
// This function is called by the Cactus scheduler after we find any
// apparent horizons, to do all this thorn's mask processing.
//
extern "C"
  void AHFinderDirect_maybe_do_masks(CCTK_ARGUMENTS)
{
DECLARE_CCTK_ARGUMENTS
DECLARE_CCTK_PARAMETERS

const struct verbose_info& verbose_info = state.verbose_info;
      struct    mask_info&    mask_info = state.mask_info;

// optionally set the mask gridfn based on each horizon's shape
if (mask_info.set_mask_for_any_horizon)
   then {
	//
	// this setup has to be done each time we're called, and
	// the  mask_grid_info  structure can't live in  struct state ,
	// because if mesh refinement is in effect we'll be called
	// separately for each separate locally-uniform grid patch
	//
	mask_grid_info mgi;
	setup_mask_grid_info(CCTK_PASS_CTOC, mgi);

	setup_mask_dataptrs_and_bitfields(cctkGH, mask_info);
	set_mask_gridfn(state.N_horizons, state.AH_data_array,
			mgi, mask_info,
			verbose_info);
	}
}

//******************************************************************************

//
// This function sets up the  mask_grid_info  structure to describe
// the (current) Cactus grid where the mask gridfn(s) live.
//
namespace {
void setup_mask_grid_info(CCTK_ARGUMENTS, struct mask_grid_info& mgi)
{
DECLARE_CCTK_ARGUMENTS

mgi.GH = cctkGH;

// Cactus grid spacing on *this* grid
mgi.coord_delta[X_AXIS] = CCTK_DELTA_SPACE(X_AXIS);
mgi.coord_delta[Y_AXIS] = CCTK_DELTA_SPACE(Y_AXIS);
mgi.coord_delta[Z_AXIS] = CCTK_DELTA_SPACE(Z_AXIS);
mgi.max_coord_delta = jtutil::max(mgi.coord_delta[X_AXIS],
		      jtutil::max(mgi.coord_delta[Y_AXIS],
				  mgi.coord_delta[Z_AXIS]));

// Cactus grid spacings on the *base* grid
const fp base_grid_delta_product =   cctk_delta_space[X_AXIS]
				   * cctk_delta_space[Y_AXIS]
				   * cctk_delta_space[Z_AXIS];
mgi.base_grid_mean_coord_delta = pow(base_grid_delta_product, 1.0/3.0);

// get global/local Cactus grid origin
// KLUDGE -- is this the right way to get this??
mgi.proc_coord_origin[X_AXIS] = CCTK_ORIGIN_SPACE(X_AXIS)
				 + cctk_lbnd[X_AXIS] * mgi.coord_delta[X_AXIS];
mgi.proc_coord_origin[Y_AXIS] = CCTK_ORIGIN_SPACE(Y_AXIS)
				 + cctk_lbnd[Y_AXIS] * mgi.coord_delta[Y_AXIS];
mgi.proc_coord_origin[Z_AXIS] = CCTK_ORIGIN_SPACE(Z_AXIS)
				 + cctk_lbnd[Z_AXIS] * mgi.coord_delta[Z_AXIS];
mgi.proc_gridfn_dims[X_AXIS] = cctk_lsh[X_AXIS];
mgi.proc_gridfn_dims[Y_AXIS] = cctk_lsh[Y_AXIS];
mgi.proc_gridfn_dims[Z_AXIS] = cctk_lsh[Z_AXIS];

#ifdef DEBUG_MASK_GRID
printf("mask.cc:: cctk_lsh[] = [%d,%d,%d]\n",
	int(cctk_lsh[X_AXIS]),
	int(cctk_lsh[Y_AXIS]),
	int(cctk_lsh[Z_AXIS]));
printf("mask.cc:: mgi.coord_delta[] = [%g,%g,%g]\n",
	double(mgi.coord_delta[X_AXIS]),
	double(mgi.coord_delta[Y_AXIS]),
	double(mgi.coord_delta[Z_AXIS]));
printf("mask.cc:: mgi.proc_coord_origin[] = [%g,%g,%g]\n",
	double(mgi.proc_coord_origin[X_AXIS]),
	double(mgi.proc_coord_origin[Y_AXIS]),
	double(mgi.proc_coord_origin[Z_AXIS]));
#endif
}
	  }

//******************************************************************************

//
// This function maps the character-string names of the mask gridfn(s)
// and/or bitfield(s) into internal data pointers and/or bit masks/values:
//
//	mask_info.old_style_mask_info
//		 .gridfn_varindex --> .gridfn_dataptr
//	mask_info.new_style_mask_info
//		 .gridfn_varindex --> .gridfn_dataptr
//		 .bitfield_name   --> .bitfield_bitmask
//		 .inside_value    --> .inside_bitvalue
//		 .buffer_value    --> .buffer_bitvalue
//		 .outside_value   --> .outside_bitvalue
//
namespace {
void setup_mask_dataptrs_and_bitfields(const cGH *GH,
				       struct mask_info& mask_info)
{
const int mask_time_level = 0;
const int SPACEMASK_ERROR = -1;

struct mask_info::old_style_mask_info& osmi = mask_info.old_style_mask_info;
if (mask_info.set_old_style_mask)
   then {
	osmi.gridfn_dataptr = static_cast<CCTK_REAL*>(
				CCTK_VarDataPtrI(GH, mask_time_level,
						 osmi.gridfn_varindex)
						     );
	if (osmi.gridfn_dataptr == NULL)
	   then CCTK_VWarn(FATAL_ERROR, __LINE__, __FILE__, CCTK_THORNSTRING,
"\n"
"setup_mask_dataptrs_and_bitfields():\n"
"        couldn't get data pointer for old-style mask \"%s\"!\n"
			   ,
			   osmi.gridfn_name);			/*NOTREACHED*/
	}

if (mask_info.set_new_style_mask)
   then {
	struct mask_info::new_style_mask_info& nsmi
		= mask_info.new_style_mask_info;
	nsmi.gridfn_dataptr = static_cast<CCTK_INT*>(
				CCTK_VarDataPtrI(GH, mask_time_level,
						 nsmi.gridfn_varindex)
						    );
	if (nsmi.gridfn_dataptr == NULL)
	   then CCTK_VWarn(FATAL_ERROR, __LINE__, __FILE__, CCTK_THORNSTRING,
"\n"
"setup_mask_dataptrs_and_bitfields():\n"
"        couldn't get data pointer for new-style mask \"%s\"!\n"
			   ,
			   nsmi.gridfn_name);			/*NOTREACHED*/

	nsmi.bitfield_bitmask = SpaceMask_GetTypeBits(nsmi.bitfield_name);
	if (nsmi.bitfield_bitmask == SPACEMASK_ERROR)
	   then CCTK_VWarn(FATAL_ERROR, __LINE__, __FILE__, CCTK_THORNSTRING,
"\n"
"setup_mask_dataptrs_and_bitfields():\n"
"        couldn't get bit mask for new-style mask bit field \"%s\"!\n"
"        (this almost certainly means this bit field was never registered\n"
"         with SpaceMask; remember that AHFinderDirect doesn't do this\n"
"         registration, rather you need to arrange for some other thorn(s)\n"
"         to do it)\n"
		   ,
		   nsmi.bitfield_name);				/*NOTREACHED*/

	nsmi.inside_bitvalue  = SpaceMask_GetStateBits(nsmi.bitfield_name,
						       nsmi.inside_value);
	nsmi.buffer_bitvalue  = SpaceMask_GetStateBits(nsmi.bitfield_name,
						       nsmi.buffer_value);
	nsmi.outside_bitvalue = SpaceMask_GetStateBits(nsmi.bitfield_name,
						       nsmi.outside_value);
	if (    (nsmi.inside_bitvalue == SPACEMASK_ERROR)
	     || (nsmi.buffer_bitvalue == SPACEMASK_ERROR)
	     || (nsmi.outside_bitvalue == SPACEMASK_ERROR)    )
	   then CCTK_VWarn(FATAL_ERROR, __LINE__, __FILE__, CCTK_THORNSTRING,
"\n"
"setup_mask_dataptrs_and_bitfields():\n"
"        couldn't get bit value(s) for one or more of the new-style mask\n"
"        inside/buffer/outside values \"%s\"/\"%s\"/\"%s\"!\n"
"        (this almost certainly means one or more of the values was/were\n"
"         never registered with SpaceMask; remember that AHFinderDirect\n"
"         doesn't do this registration, rather you need to arrange for\n"
"         some other thorn(s) to do it)\n"
			   ,
			   nsmi.inside_value,
			   nsmi.buffer_value,
			   nsmi.outside_value);			/*NOTREACHED*/
	}
}
	  }

//******************************************************************************

//
// This function sets the mask grid function specified by  mask_info
// based on each horizon's shape.
//
namespace {
void set_mask_gridfn(int N_horizons,
		     const struct AH_data* const AH_data_array[],
		     const struct mask_grid_info& mgi,
		     const struct mask_info& mask_info,
		     const struct verbose_info& verbose_info)
{
const bool set_old_style_mask = mask_info.set_old_style_mask;
const bool set_new_style_mask = mask_info.set_new_style_mask;
const struct mask_info::old_style_mask_info& osmi = mask_info.old_style_mask_info;
const struct mask_info::new_style_mask_info& nsmi = mask_info.new_style_mask_info;

if (verbose_info.print_algorithm_highlights)
   then {
	if (set_old_style_mask)
	   then CCTK_VInfo(CCTK_THORNSTRING,
			   "setting old-style (CCTK_REAL) mask grid function %s",
			   osmi.gridfn_name);
	if (set_new_style_mask)
	   then {
		CCTK_VInfo(CCTK_THORNSTRING,
			   "setting new-style (CCTK_INT) mask grid function %s",
			   nsmi.gridfn_name);
		CCTK_VInfo(CCTK_THORNSTRING,
			   "                                  bit field %s",
			   nsmi.bitfield_name);
		}
	}

if (verbose_info.print_algorithm_debug)
   then {
	CCTK_VInfo(CCTK_THORNSTRING,
		   "      grid on this processor has x=[%g,%g]",
		   double(mgi.global_xyz_of_ijk(X_AXIS, 0)),
		   double(mgi.global_xyz_of_ijk(X_AXIS, mgi.proc_gridfn_dims[X_AXIS]-1)));
	CCTK_VInfo(CCTK_THORNSTRING,
		   "                                 y=[%g,%g]",
		   double(mgi.global_xyz_of_ijk(Y_AXIS, 0)),
		   double(mgi.global_xyz_of_ijk(Y_AXIS, mgi.proc_gridfn_dims[Y_AXIS]-1)));
	CCTK_VInfo(CCTK_THORNSTRING,
		   "                                 z=[%g,%g]",
		   double(mgi.global_xyz_of_ijk(Z_AXIS, 0)),
		   double(mgi.global_xyz_of_ijk(Z_AXIS, mgi.proc_gridfn_dims[Z_AXIS]-1)));
	}


//
// set the mask to the outside value everywhere in
// (this processor's chunk of) the grid
//
set_mask_gridfn_to_outside_value(mgi,
				 mask_info,
				 verbose_info);


//
// loop over each horizon's xyz bounding box
// (intersected with (this processor's chunk of) the grid)
// to set the mask accurately (or skip this horizon if it's too small)
//
	for (int hn = 1 ; hn <= N_horizons ; ++hn)
	{
	if (! mask_info.set_mask_for_this_horizon[hn])
	   then continue;				// *** LOOP CONTROL ***

	const struct AH_data& AH_data = *AH_data_array[hn];
	if (! AH_data.found_flag)
	   then continue;				// *** LOOP CONTROL ***

	const patch_system& ps                      = *AH_data.ps_ptr;
	const struct BH_diagnostics& BH_diagnostics =  AH_data.BH_diagnostics;

	//
	// skip this horizon if it's too small
	//
	// ... The condition is documented as using the minimum over angle
	//     of r_inner(r_horizon(angle)).  But this is trivially equivalent
	//     to r_inner(the minimum over angle of r_horizon), i.e. to
	//     r_inner(BH_diagnostics.min_radius); this latter form is what
	//     we actually compute.
	//
	const fp r_inner_min
		= inner_mask_radius(mgi, mask_info, BH_diagnostics.min_radius);
	if (r_inner_min
	    < mask_info.min_horizon_radius_points_for_mask*mgi.max_coord_delta)
	   then {
		if (verbose_info.print_algorithm_details)
		   then CCTK_VInfo(CCTK_THORNSTRING,
   "   r_inner_min=%g < %g grid points ==> skipping mask for horizon %d",
				   r_inner_min,
				   mask_info.min_horizon_radius_points_for_mask,
				   hn);
		continue;				// *** LOOP CONTROL ***
		}

	//
	// get to here ==> normal mask processing for this horizon
	//

	if (verbose_info.print_algorithm_details)
	   then CCTK_VInfo(CCTK_THORNSTRING,
   "   setting mask grid function to \"buffer\"/\"inside\" for horizon %d",
			   hn);


	// horizon bounding box, rounded "out" to the next grid point
	const int AH_min_i = mgi.ijk_floor_of_global_xyz(X_AXIS, BH_diagnostics.min_x);
	const int AH_max_i = mgi.ijk_ceil_of_global_xyz (X_AXIS, BH_diagnostics.max_x);
	const int AH_min_j = mgi.ijk_floor_of_global_xyz(Y_AXIS, BH_diagnostics.min_y);
	const int AH_max_j = mgi.ijk_ceil_of_global_xyz (Y_AXIS, BH_diagnostics.max_y);
	const int AH_min_k = mgi.ijk_floor_of_global_xyz(Z_AXIS, BH_diagnostics.min_z);
	const int AH_max_k = mgi.ijk_ceil_of_global_xyz (Z_AXIS, BH_diagnostics.max_z);
	if (verbose_info.print_algorithm_debug)
	   then {
		CCTK_VInfo(CCTK_THORNSTRING,
			   "      horizon bounding box is x=[%g,%g]",
			   double(BH_diagnostics.min_x),
			   double(BH_diagnostics.max_x));
		CCTK_VInfo(CCTK_THORNSTRING,
			   "                              y=[%g,%g]",
			   double(BH_diagnostics.min_y),
			   double(BH_diagnostics.max_y));
		CCTK_VInfo(CCTK_THORNSTRING,
			   "                              z=[%g,%g]",
			   double(BH_diagnostics.min_z),
			   double(BH_diagnostics.max_z));
		}

	//
	// compute intersection of horizon bounding box with
	// (this processor's chunk of) the grid
	//
	// as can be seen from the following picture, the intersection
	// has the max/min of the min/max coordinate as its min/max:
	//
	//	           +--------------------+
	//	           |                    |
	//	+----------+--------------+     |
	//	|          |XXXXXXXXXXXXXX|     |
	//	|          |XXXXXXXXXXXXXX|     |
	//	|          |XXXXXXXXXXXXXX|     |
	//	|          +--------------+-----+
	//	|                         |
	//	|                         |
	//	+-------------------------+
	//
	const int use_min_i = jtutil::max(AH_min_i, 0);
	const int use_max_i = jtutil::min(AH_max_i, int(mgi.proc_gridfn_dims[X_AXIS]-1));
	const int use_min_j = jtutil::max(AH_min_j, 0);
	const int use_max_j = jtutil::min(AH_max_j, int(mgi.proc_gridfn_dims[Y_AXIS]-1));
	const int use_min_k = jtutil::max(AH_min_k, 0);
	const int use_max_k = jtutil::min(AH_max_k, int(mgi.proc_gridfn_dims[Z_AXIS]-1));

	if (verbose_info.print_algorithm_debug)
	   then {
		CCTK_VInfo(CCTK_THORNSTRING,
			   "      use bounding box is x=[%g,%g]",
			   double(mgi.global_xyz_of_ijk(X_AXIS, use_min_i)),
			   double(mgi.global_xyz_of_ijk(X_AXIS, use_max_i)));
		CCTK_VInfo(CCTK_THORNSTRING,
			   "                          y=[%g,%g]",
			   double(mgi.global_xyz_of_ijk(Y_AXIS, use_min_j)),
			   double(mgi.global_xyz_of_ijk(Y_AXIS, use_max_j)));
		CCTK_VInfo(CCTK_THORNSTRING,
			   "                          z=[%g,%g]",
			   double(mgi.global_xyz_of_ijk(Z_AXIS, use_min_k)),
			   double(mgi.global_xyz_of_ijk(Z_AXIS, use_max_k)));
		}

	set_mask_gridfn_to_inside_and_buffer_values(mgi,
						    use_min_i, use_max_i,
						    use_min_j, use_max_j,
						    use_min_k, use_max_k,
						    mask_info,
						    ps,
						    verbose_info);
	}
}
	  }

//******************************************************************************

//
// For each Cactus grid point in (this processor's chunk of) the Cactus
// grid, this function sets the mask grid function specified by  mask_info
// to the appropriate value for the "outside" region (possibly leaving
// the mask unchanged at its previous value if the  mask_is_noshrink
// option is set).
//
namespace {
void set_mask_gridfn_to_outside_value(const struct mask_grid_info& mgi,
				      const struct mask_info& mask_info,
				      const struct verbose_info& verbose_info)
{
const bool set_old_style_mask = mask_info.set_old_style_mask;
const bool set_new_style_mask = mask_info.set_new_style_mask;
const struct mask_info::old_style_mask_info& osmi = mask_info.old_style_mask_info;
const struct mask_info::new_style_mask_info& nsmi = mask_info.new_style_mask_info;


if (verbose_info.print_algorithm_details)
   then CCTK_VInfo(CCTK_THORNSTRING,
		   "   setting mask grid function to \"outside\"");

	for (int k = 0 ; k < mgi.proc_gridfn_dims[Z_AXIS] ; ++k)
	{
	for (int j = 0 ; j < mgi.proc_gridfn_dims[Y_AXIS] ; ++j)
	{
	for (int i = 0 ; i < mgi.proc_gridfn_dims[X_AXIS] ; ++i)
	{
	const int posn = CCTK_GFINDEX3D(mgi.GH, i,j,k);
	if (set_old_style_mask)
	   then {
		const CCTK_REAL old_value = osmi.gridfn_dataptr[posn];
		if ( mask_info.mask_is_noshrink
		     && (    (old_value == osmi.inside_value)
			  || (old_value == osmi.buffer_value)    ) )
		   then {
			// we're in "noshrink mode"
			// and this point was previously "inside" or "buffer"
			// ==> no-op here
			}
		   else osmi.gridfn_dataptr[posn] = osmi.outside_value;
		}
	if (set_new_style_mask)
	   then {
		if ( mask_info.mask_is_noshrink
		     && (    SpaceMask_CheckStateBits(nsmi.gridfn_dataptr, posn,
						      nsmi.bitfield_bitmask,
						      nsmi.inside_bitvalue)
			  || SpaceMask_CheckStateBits(nsmi.gridfn_dataptr, posn,
						      nsmi.bitfield_bitmask,
						      nsmi.buffer_bitvalue)    ) )
		   then {
			// we're in "noshrink mode"
			// and this point was previously "inside" or "buffer"
			// ==> no-op here
			}
		   else SpaceMask_SetStateBits(nsmi.gridfn_dataptr, posn,
					       nsmi.bitfield_bitmask,
					       nsmi.outside_bitvalue);
		}
	}
	}
	}
}
	  }

//******************************************************************************

//
// For each Cactus grid point in a specified region of the Cactus grid,
// and for a specified (single) horizon, this function either sets the
// mask grid function specified by  mask_info to the appropriate value
// if the point is in the "inside" or "buffer" region for a specified
// horizon (possibly leaving the mask unchanged at its previous value
// if the  mask_is_noshrink  option is set), or leaves the mask grid
// function unchanged if the point is actually outside this horizon.
//
// Arguments:
// ps = The patch system for this horizon.
// use_{min,max}_{i,j,k} = Specifies the region of the Cactus grid.
//
// FIXME:
// The current implementation is quite inefficient: it does a full 2-D
// interpolation (to find the horizon radius) for each xyz grid point
// in the specified region.  It would be more efficient to batch the
// interpolations, and maybe also use r_min/r_max tests for early-out.
//
namespace {
void set_mask_gridfn_to_inside_and_buffer_values
	(const struct mask_grid_info& mgi,
	 int use_min_i, int use_max_i,
	 int use_min_j, int use_max_j,
	 int use_min_k, int use_max_k,
	 const struct mask_info& mask_info,
	 const patch_system& ps,
	 const struct verbose_info& verbose_info)
{
const bool set_old_style_mask = mask_info.set_old_style_mask;
const bool set_new_style_mask = mask_info.set_new_style_mask;
const struct mask_info::old_style_mask_info& osmi = mask_info.old_style_mask_info;
const struct mask_info::new_style_mask_info& nsmi = mask_info.new_style_mask_info;

long inside_count = 0;
long buffer_count = 0;

	for (int k = use_min_k ; k <= use_max_k ; ++k)
	{
	for (int j = use_min_j ; j <= use_max_j ; ++j)
	{
	for (int i = use_min_i ; i <= use_max_i ; ++i)
	{
	const int posn = CCTK_GFINDEX3D(mgi.GH, i,j,k);

	const fp global_x = mgi.global_xyz_of_ijk(X_AXIS, i);
	const fp global_y = mgi.global_xyz_of_ijk(Y_AXIS, j);
	const fp global_z = mgi.global_xyz_of_ijk(Z_AXIS, k);
	const fp local_x = global_x - ps.origin_x();
	const fp local_y = global_y - ps.origin_y();
	const fp local_z = global_z - ps.origin_z();

	const fp r = jtutil::hypot3(local_x, local_y, local_z);

	// interpolate to find r_horizon in this direction
	//
	// FIXME:
	// it would be more efficient here to compute the
	// radii of a whole batch of points at once
	const fp r_horizon = ps.radius_in_local_xyz_direction
				   (gfns::gfn__h,
				    local_x, local_y, local_z);

	const fp r_inner = inner_mask_radius(mgi, mask_info, r_horizon);
	const fp r_outer = outer_mask_radius(mgi, mask_info, r_inner);

	if	(r <= r_inner)
	   then {
		// set the mask to the "inside" value
		++inside_count;
		if (set_old_style_mask)
		   then osmi.gridfn_dataptr[posn] = osmi.inside_value;
		if (set_new_style_mask)
		   then SpaceMask_SetStateBits(nsmi.gridfn_dataptr, posn,
					       nsmi.bitfield_bitmask,
					       nsmi.inside_bitvalue);
		}
	else if (r <= r_outer)
	   then {
		// set the mask to the "buffer" value
		// ... except that if the  mask_is_noshrink  option is set
		//     and the mask is already set to the "inside" value
		//     then we need to leave it unchanged
		//     (at the "inside" value)
		++buffer_count;
		if (set_old_style_mask)
		   then {
			if ( mask_info.mask_is_noshrink
			     && (osmi.gridfn_dataptr[posn] == osmi.inside_value) )
			   then {
				// we're in "noshrink mode"
				// and this point was previously "inside"
				// ==> no-op here
				}
			   else osmi.gridfn_dataptr[posn] = osmi.buffer_value;
			}
		if (set_new_style_mask)
		   then {
			if ( mask_info.mask_is_noshrink
			     && SpaceMask_CheckStateBits(nsmi.gridfn_dataptr, posn,
							 nsmi.bitfield_bitmask,
							 nsmi.inside_bitvalue) )
			   then {
				// we're in "noshrink mode"
				// and this point was previously "inside"
				// ==> no-op here
				}
			   else SpaceMask_SetStateBits(nsmi.gridfn_dataptr, posn,
						       nsmi.bitfield_bitmask,
						       nsmi.buffer_bitvalue);
			}
		}
	else	{
		// it's "outside" ==> no-op here
		}
	}
	}
	}

if (verbose_info.print_algorithm_details)
   then CCTK_VInfo(CCTK_THORNSTRING,
		   "      %ld \"inside\" points, %ld \"buffer\" points on this processor",
		   inside_count, buffer_count);
}
	  }

//******************************************************************************

//
// This function computes the inner mask radius  r_inner  for a given
// horizon radius  r_horizon .
//
namespace {
fp inner_mask_radius(const struct mask_grid_info& mgi,
		     const struct mask_info& mask_info,
		     fp r_horizon)
{
const fp Cactus_dx = mgi.base_grid_mean_coord_delta;
return mask_info.radius_multiplier*r_horizon
       + mask_info.radius_offset*Cactus_dx;
}
	  }

//******************************************************************************

//
// This function computes the outer mask radius  r_outer  for a given
// inner mask radius  r_inner .
//
namespace {
fp outer_mask_radius(const struct mask_grid_info& mgi,
		     const struct mask_info& mask_info,
		     fp r_inner)
{
const fp Cactus_dx = mgi.base_grid_mean_coord_delta;
return r_inner + mask_info.buffer_thickness*Cactus_dx;
}
	  }

//******************************************************************************

	  }	// namespace AHFinderDirect

```

include/config.h:
```
/* config.h -- compile-time configuration for AHFinderDirect */
/* $Header$ */

#ifndef AHFINDERDIRECT__CONFIG_H
#define AHFINDERDIRECT__CONFIG_H

/*
 * This header file is #included by C++ and C files, and also by
 * Fortran 77 files which are run through a C-like preprocessor.  Alas,
 * due to a Cactus bug on some systems, this means that all preprocessor
 * commands must start in column 1. :( :( :(
 */

/*
 * prerequisites:
 *	"cctk.h" or "fake_cctk.h"
 */

/******************************************************************************/
/******************************************************************************/
/******************************************************************************/

/*
 * definitions for C/C++ only
 */

#ifdef CCODE
typedef CCTK_REAL fp;

/* Fortran 'integer' type */
typedef int integer;
#endif	/* CCODE */

/******************************************************************************/
/******************************************************************************/
/******************************************************************************/

/*
 * definitions for C++, C, and Fortran
 */

/* CCTK_VWarn() "warning level" for fatal errors (terminate execution) */
/* note low-level software uses error_exit() instead of CCTK_VWarn(), */
/* with exit codes defined in "stdc.h" */
#define FATAL_ERROR	(-1)

/******************************************************************************/

/*
 * ***** definitions for integer and floating point datatypes *****
 */

/*
 * FIXME: this assumes fp == C 'double'
 * CCTK_REAL_PRECISION_{4,8,16} are helpful, but not quite enough)
 */
#include <cctk.h>
#undef  FP_IS_FLOAT
#undef  FP_IS_DOUBLE
#ifdef CCTK_REAL_PRECISION_4
#define FP_IS_FLOAT
#endif
#ifdef CCTK_REAL_PRECISION_8
#define FP_IS_DOUBLE
#endif

#if   defined(FP_IS_FLOAT)
#define FP_SCANF_FORMAT "%f"
#define FP_EPSILON	FLT_EPSILON	/* from <float.h> */
#elif defined(FP_IS_DOUBLE)
#define FP_SCANF_FORMAT "%lf"
#define FP_EPSILON	DBL_EPSILON	/* from <float.h> */
#else
#error "don't know fp datatype!"
#endif

/*
 * FIXME: this should be set somehow at configure time
 */
#define FORTRAN_INTEGER_IS_INT
#undef  FORTRAN_INTEGER_IS_LONG

/******************************************************************************/

/*
 * The angular finite differencing in our multipatch system can be
 * either 2nd order or 4th order, configurable here.
 */
#ifdef FINITE_DIFF_ORDER
/* someone (probably a Makefile) has already defined this */
/* ==> leave it alone */
#else
#define FINITE_DIFF_ORDER	4
#endif

/******************************************************************************/

/*
 * What types of Jacobian matrix storage and linear solvers do we want to
 * compile in support for?  Note that each of these requires linking with
 * the corresponding linear-solver library; see "../elliptic/Jacobian.hh"
 * for details on the storage formats, or "../make.configuration.defn"
 * (if it exists) for details on the libraries.
 */

/* store as (Fortran) dense matrix, solve with LAPACK */
#undef  HAVE_DENSE_JACOBIAN__LAPACK

/* store as row-oriented sparse matrix, solve with ILUCG */
#define HAVE_ROW_SPARSE_JACOBIAN__ILUCG

/* store as row-oriented sparse matrix, solve with UMFPACK */
#define HAVE_ROW_SPARSE_JACOBIAN__UMFPACK

/******************************************************************************/
/******************************************************************************/
/******************************************************************************/

/*
 * The following #ifdefs and #defines shouldn't need changing; they
 * decode the previous set of Jacobian storage format and linear solver
 * #defines to determine which storage formats we are using.
 */
#ifdef HAVE_DENSE_JACOBIAN__LAPACK
#define HAVE_DENSE_JACOBIAN
#endif

#ifdef HAVE_ROW_SPARSE_JACOBIAN__ILUCG
#define HAVE_ROW_SPARSE_JACOBIAN
#endif

#ifdef HAVE_ROW_SPARSE_JACOBIAN__UMFPACK
#define HAVE_ROW_SPARSE_JACOBIAN
#endif

#endif	/* AHFINDERDIRECT__CONFIG_H */

```

include/README:
```
This directory holds header files common to all of this thorn.

fake_cctk.h
	is a fake version of "cctk.h" for non-Cactus standalone test
	compilations, containing typedefs for CCTK_INT, CCTK_REAL, etc

stdc.h
	is a low-level file defining Jonathan Thornburg's "dialect" of C,
	including the "then" pseudo-keyword you'll see all through the
	source code

config.hh
	defines compile-time configuration for this thorn, eg the "fp"
	typedef (= CCTK_REAL), the choice of 2nd vs 4th order finite
	differencing, etc etc

```

include/stdc.h:
```
/* stdc.h -- JT standard C/C++ header file, cut-down for modern code only */
/* $Header$ */

#ifndef AHFINDERDIRECT__STDC_H
#define AHFINDERDIRECT__STDC_H

/*
 * ***** THIS VERSION IS FOR ANSI/ISO C AND C++ *****
 */

/******************************************************************************/

/*
 * I use this macro in all if statements -- I like the symmetry between
 * the two branches of an if statement.
 *
 *	if (foo)
 *	   then bar;
 *
 *	if (foo)
 *	   then bar;
 *	   else baz;
 */
#define then	/* empty */

/******************************************************************************/

/*
 * misc stuff
 */

#ifdef M_PI		/* usually defined in <math.h> */
  #define PI	M_PI
#endif

#define STRING_EQUAL(s_,t_)	(strcmp(s_,t_) == 0)

/* C library uses abs() for integer absolute value; I prefer iabs() */
#define iabs(x_)	abs(x_)

/******************************************************************************/

/*
 * misc sentinal values
 */

/* n.b. this value is usually 10...0 binary for 2's-complement arithmetic */
#define INT_SENTINAL_VALUE	(~ INT_MAX)	/* from <limits.h> */

#define FLOAT_SENTINAL_VALUE	(- FLT_MAX)	/* from <float.h> */
#define DOUBLE_SENTINAL_VALUE	(- DBL_MAX)	/* from <float.h> */

/******************************************************************************/

#ifdef __cplusplus
namespace jtutil
	  {
#endif

/*
 * Low-level code in this thorn is done with  error_exit() .  In this
 * this thorn it's a wrapper around  CCTK_VWarn(msg_level, ...) .  It's
 * declared to return  int  so it may be easily used in conditional
 * expressions like
 *	foo = bar_ok ? baz(bar) : error_exit(...);
 */
#ifdef __cplusplus
  extern "C"
#endif
int error_exit(int msg_level, const char *format, ...)
  #ifdef __GNUC__
  __attribute__ ((noreturn))
  __attribute__ ((format(printf,2,3)))
  #endif
;

/*
 * error_exit() uses the following  msg_level  codes:
 *	ERROR_EXIT	==> something bad has happened
 *			    (eg inconsistent user input)
 *	PANIC_EXIT	==> diasaster (eg internal consistency check failed)
 *			==> we'd like to force (eg) a core dump or stack trace
 *			    (but at present this isn't implemented; we just
 *			     do a  CCTK_VWarn()  just like for  ERROR_EXIT)
 */
#define ERROR_EXIT	(-1)
#define PANIC_EXIT	(-2)

#ifdef __cplusplus
	  }	/* namespace jtutil */
#endif

/******************************************************************************/

#endif	/* AHFINDERDIRECT__STDC_H */

```

include/fake_cctk.h:
```
/* fake_cctk.h -- fake "cctk.h" for non-Cactus standalone compilations */
/* $Header$ */

#ifndef AHFINDERDIRECT__FAKE_CCTK_H
#define AHFINDERDIRECT__FAKE_CCTK_H

#define CCODE
typedef int    CCTK_INT;
typedef double CCTK_REAL;

#endif	/* AHFINDERDIRECT__FAKE_CCTK_H */

```

maple/setup.maple:
```
# setup.maple -- top-level Maple input file to set up PDE compiler system
# $Header$

#
# Note not all files here actually *use* the preprocessor, but for
# convenience we preprocess everything (this way we don't have to keep
# track of which files should be used as .maple and which as .mm).
#

#
# Note paths are evaluated relative to where read command gets executed,
# not relative to this directory, so we use ../maple/ to try and avoid
# ambiguities in case this file gets read from another directory.
#

read "../maple/util.mm";

read "../maple/coords.mm";
setup_coords();

read "../maple/Diff.mm";

read "../maple/gfa.mm";

read "../maple/coeffs.mm";
setup_coeff_gfas();

read "../maple/codegen2.mm";

```

maple/Makefile:
```
# Makefile for maple code
# $Header: /usr/local/svn/cvs-repositories/numrelcvs/AEIThorns/AHFinderDirect/src/maple/Makefile,v 1.4.2.1 2004-04-06 21:04:22 schnetter Exp $

#
# Environment Variables:
#	MAPLE_VERSION	used via @ifdef for version control in Maple code;
#			typically set to something like  MAPLE_V_RELEASE_4
#			(not presently used, but may be needed in the future)
#
# Targets:
#	mm		==> preprocess all *.maple files to produce *.mm files
#	clean		==> delete *.mm (mpp Maple preprocessor output)
#	run		==> load all the code into Maple
#

ifneq ($(MAPLE_VERSION),)
MPP_FLAGS := -D$(MAPLE_VERSION)
endif

###############################################################################

.PHONY	: mm
mm	: $(patsubst %.maple, %.mm, $(wildcard *.maple))

%.mm	: %.maple $(wildcard *.minc $(gfa_dir)/*.minc $(gfa_dir)/*.maple)
	../misc/mpp $(MPP_FLAGS) <$< >$@

.PHONY	: clean
clean	:
	-rm *.mm

.PHONY	: run
run	: mm
	maple <setup.mm >maple.log

```

maple/util.maple:
```
# util.maple -- Maple utility functions
# $Header$

#
# msum - multiple-index sum()
# arctan_xy - 2-argument arctan() function
# ssqrt - symbolic sqrt()
#
# indices_in_order - list of indices of table/array, in lexicographic order
# lexorder_integer_list - lexorder() for lists
# sort_var_list - sort a list of variables into a canonical order
# lexorder_vars - lexorder() for "interesting" variables
#
# gensym - generate a "new" symbol name
# gensym/init - reinitialize the gensym() global naming counter
#
# saveit - optionally save variables for debugging
#

################################################################################
################################################################################
################################################################################

#
# This function is identical to the standard Maple library function  sum()
# except that it allows multiple summation indices.  It's usage is best
# illustrated by a series of examples:
#
#	msum('f', 'k'=m..n)
#		= sum('f', 'k'=m..n)
#
#	msum('f', 'k1'=m1..n1, 'k2'=m2..n2)
#		= sum( 'sum('f', 'k1'=m1..n1)' , 'k2'=m2..n2 )
#
#	msum('f', 'k1'=m1..n1, 'k2'=m2..n2, 'k3'=m3..n3)
#		= sum(
#			 'sum( 'sum('f', 'k1'=m1..n1)' , 'k2'=m2..n2 )'
#		       ,
#			 'k3'=m3..n3
#		     )
#
# Note, in particular, that the summation indices are nested with the
# innermost summation appearing first.  This is the same nesting convention
# as Fortran I/O statement's implicit do loops use.
#
# Bugs:
# - The quoting in the code is pretty obscure...
#
msum :=
proc(fn::algebraic)			# varargs
local expr, sum_index;

if (nargs < 2)
   then ERROR("must have two or more arguments")
end if;

expr := fn;

	for sum_index in [args[2..nargs]]
	do
	# loop invariant:
	# msum(fn, <args processed so far>) = sum(expr, sum_index)
	expr; sum_index; expr:= 'sum'(''%%'', ''%'');
	end do;

return eval(expr);
end proc;

################################################################################

#
# This function computes the 4-quadrant arctangent of  arctan(y/x) .
# It's "just" a wrapper around the Maple 2-argument arctan function,
# providing the what-I-find-natural (x,y) argument
#
# Arguments:
# (x,y) = (In) The x,y coordinates.
#
arctan_xy :=
proc(x::algebraic, y::algebraic)
arctan(y,x);		# !!! reversedargument order !!!
end proc;

################################################################################

#
# This function computes the square root of its argument with the
# symbolic option ==> fewer branch-cut problems.
#
ssqrt :=
proc(x::algebraic)
sqrt(x, 'symbolic');
end proc;

################################################################################
################################################################################
################################################################################

#
# This function is identical to Maple's  indices()  built-in function,
# except that (a) it returns a list, not an expression sequence, and
# (b) the indices are returned in lexicographic order rather than in
# Maple's internal hash ordering.
#
# Arguments:
# T = (in) A table or array with (only) integer-sequence indices.
#
# Results:
# The lexicographic-ordered expression sequence of indices is returned
# as the function result.
#
# Bugs:
# - The function fails if the indices aren't a list of numbers.
#
indices_in_order :=
proc(T::table)
return sort([indices(T)], lexorder_integer_list);
end;

###############################################################################

#
# This function lexicographically orders lists of integers.
#
# Arguments:
# list1 = list of integers to be sorted
# list2 = list of integers to be sorted
#
# Results:
# The function returns  true  iff  list1 < list2  lexicographically,
# with the [1] list elements being most significant.
#
lexorder_integer_list :=
proc(list1::list(numeric), list2::list(numeric))
local len1, len2, k;

len1 := nops(list1);
len2 := nops(list2);

	for k from 1 to min(len1,len2)
	do
	if   (list1[k] < list2[k])
	   then return true;
	elif (list1[k] > list2[k])
	   then return false;
	fi;
	end do;

# get to here ==> the shorter list is an exact prefix of the longer one
#             ==> order the shorter one < the longer one
return evalb(len1 < len2);
end;

################################################################################

#
# This function sorts a list of names (in practice, a list of coordinates)
# into a canonical order.
#
sort_var_list :=
proc(var_list::list(name))
option remember;		# performance optimization
global lexorder_vars;

# only get to here the first time we see a given variable list
# (i.e. if it's not in the remember table)
return sort(var_list, lexorder_vars);
end;

################################################################################

#
# This function defines a lexical ordering on variable names.
#
# Arguments:
# x, y = Two names to be compared.
#
# Results:
# This function returns true iff a < b, false otherwise.
#
lexorder_vars :=
proc(x::name, y::name)
option remember;		# performance optimization
local xposn, yposn;
global
  @include "../maple/coords.minc";

if (member(x, xy_all_list, 'xposn') and member(y, xy_all_list, 'yposn'))
   then return evalb(xposn < yposn);
   else return lexorder(x, y);
fi;

end;

################################################################################
################################################################################
################################################################################

#
# This procedure generates a (presumably) new symbol name of the
# form
#	cat(base_name,count)
# where  base_name  is passed as an (optional, with a default value
# supplied if it's omitted) argument, and  count  is the current
# value of the global variable
#	`gensym/counter`
# which is postincremented after being used.  If the counter is
# unassigned on entry to this function,  `gensym/init`()  is first
# called to initialize it.
#
# `gensym/init`() may be used to (re)initialize the counter as
# desired, or `gensym/counter` may be examined or altered directly
# to otherwise modify the name sequence generated by this function.
#
# Arguments:
# base_name = (in) (optional) The base name for the name generation.
#			      This argument defaults to `temp_` ifo
#			      omitted.
#
# Bugs:
# No checks are made for whether the names in question are indeed
# unassigned.
#
gensym :=
proc(opt_base_name::string)
global `gensym/counter`;		# in out
local base_name, tn;

if (nargs >= 1)
   then base_name := opt_base_name;
   else base_name := '`temp_`';
end if;

# force an initialization if it hasn't been done before
if (not assigned(`gensym/counter`))
   then `gensym/init`();
end if;

tn := cat(base_name, `gensym/counter`);
`gensym/counter` := `gensym/counter` + 1;

tn; return '%';
end proc;

################################################################################

#
# This procedure (re)initializes the gensym() global naming counter.
#
# Arguments:
# initial_counter = (in) (optional)
#			 The naming counter value to (re)initialize with.
#			 This argument defaults to 1 if omitted.
#
`gensym/init` :=
proc(opt_initial_counter::integer)
global `gensym/counter`;		# in out
local initial_counter;

if (nargs >= 1)
   then initial_counter := opt_initial_counter;
   else initial_counter := 1;
end if;

`gensym/counter` := initial_counter;
NULL;
end proc;

################################################################################
################################################################################
################################################################################

#
# This function optionally saves a local variable to a global variable
# for later examination.  This is useful for debugging purposes, since
# Maple doesn't have a proper debugger :-( .
#
# Arguments (explicit):
# n = (in) An integer controlling how important this saveit() call is;
#	   the saving will only be done if `saveit/level` >= n, i.e.
#	   smaller values of n make the saving more likely.
# fn = (in) The calling function or subsystem's name.
# label = (in) A string identifying this saveit() call among all those
#	       within the calling function.
# expr = (in) The temporary result to save.
#
# Arguments (implicit, as global variables)
# `saveit/level` = (in) (optional) If this name is assigned (as determined
#				  by the  assigned()  function), this
#				  function does the saving.  Otherwise,
#				  this function is a nop.
#
saveit :=
proc(n::integer, fn::{procedure,string}, label::string, expr::anything)
global `saveit/level`;
local save_name;

if ( assigned(`saveit/level`)
     and type(`saveit/level`, integer)
     and (`saveit/level` >= n) )
   then
	save_name := cat(convert(fn,string),"/",label);
	printf("      --> `%s`\n", save_name);
	assign(  convert(eval(save_name,1),name) = expr  );
end if;

NULL;
end proc;

```

maple/gfa.maple:
```
# gfa.maple -- low-level gridfn array "database"
# $Header$
#
# ***** gridfn arrays *****
# ***** functional dependences *****
# make_gfa - construct a new gridfn array
# assert_fnd_exists - assert (verify) that a given gridfn array fnd form exists
# Maple_name - Maple name of functional-dependence form
#
# index/symmetric3_23 - Maple indexing fn for rank 3 array sym on axes 23
#
# print_symmetric3_23 - prettyprint a symmetric3_23 array
#

################################################################################

#
# ***** gridfn arrays *****
#

#
# The basic data objects for the Maple code are grid function arrays,
# a.k.a. gridfn arrays, such as the metric g_dd, the Christoffel symbols
# Gamma_udd, etc etc.
#
# By convention, the name of a nonscalar geometric object ends with an 
# underscore ("_") followed by a sequence of "d" or "u" specifying whether
# the indices are down (covariant) or up (contravariant).  Thus, for
# example, g_dd is the covariant metric, while g_uu is the inverse metric.
#
# Scalar geometric objects are represented by Maple scalars, while
# nonscalar objects (often but not necessarily tensors) are represented
# by Maple arrays, using Maple indexing functions for any symmetries
# or sparsity.  For example, the metric  g_dd  is represented by an
# array
#	array(1..N, 1..N, symmetric)
# By convention, for a symmetric array of this type, we only compute
# the upper triangle.
#

################################################################################

#
# ***** functional dependences *****
#

#
# Within the Maple code, we distinguish between two distinct forms
# that a gridfn array may take:
# - The "inert" form of a gridfn array represents (only) the symmetries,
#   zero (or nonzero but constant) elements, and indexing semantics of
#   the gridfn array.
# - The "functional dependence" form of a gridfn array additionally
#   represents how that gridfn array is computed from other gridfn
#   arrays.
#
# For example, consider the inverse metric  g_uu .  It's inert form
# is a 3*3 symmetric array, with all elements being unassigned symbols
# (i.e. evaluating to themselves, as is usual in Maple).  It's functional
# dependence form, in contrast, is also a 3*3 symmetric array, but its
# elements are expressions giving the inverse-metric elements in terms
# of components of the metric and/or the metric determinant.
#
# Any given gridfn array may exist in either or (the usual case) both
# an inert form and a functional dependence forms.  There may also be
# multiple functional dependence forms, corresponding to alternative
# formulas for its computation.)
#
# The inert form of a gridfn array is represented by a Maple variable
# or array (see below) of the same name (eg  g_uu ), unassigned except
# for any symmetries and/or zero components the gridfn array has.
#
# The functional dependence form of a gridfn array is represented by
# a Maple array (see below) with a name formed by appending "__fnd"
# to the gridfn array's name (eg  g_uu__fnd ); this array has the same
# symmetries as the corresponding inert Maple array, but only those
# elements needed will be assigned.  If there are multiple functional
# dependence forms, by convention each uses a name beginning in "__"
# and ending in "fnd", eg  H__ps_fnd .
#
# The PDE compiler normally generates C code referring to the (inert)
# gridfn array names, so these should be valid C variable names.
#

#
# To allow some run-time error checking, we keep a global table
# of all the gridfn array functional-dependence forms:
#	`gfa/fnd_table`
#	... table index is  gfa__fnd_name
#	... table value is  true
# assert_fnd_exists() uses this table to verify that a given gridfn
# array functional-dependence form does indeed exist.
#

#
# Functional-dependence forms may be multilevel, as in (eg)
#  g_dd__Kerr_fnd__ps_fnd .  cat() may be used to combine names in
# this manner.
#

################################################################################

#
# This function makes the Maple representation of a single gridfn array.
#
# Arguments:
# gfa_name = (out*) The Maple variable to be created for the gridfn
#		    array.  This argument should be passed as an
#		    unevaluated (quoted) name.
# fnd_name_set = (in) A set of functional dependence types the gridfn
#		      array is to have.  The set's members should be
#		      drawn from from {inert, fnd, ...} as appropriate,
#		      where the special value  inert  refers to the
#		      inert form.
# index_bounds = (in) The list of index bounds for the gridfn array,
# index_fn = (in) The name of the array indexing function, if any, or
#		  'none' if the gridfn array has no symmetries and thus
#		  should be represented by a Maple array with the standard
#		  "ordinary array" indexing semantics.
#
# Thus, for example, the calls
#	make_gfa('g', {inert, fnd}, [], none);
#	make_gfa('g_dd', {inert}, [1..N, 1..N], symmetric);
#	make_gfa('R_dd', {inert, fnd}, [1..N, 1..N], symmetric);
#	make_gfa('Gamma_udd', {inert, fnd}, [1..N, 1..N, 1..N], symmetric3_23);
# would set up the Maple representations of the metric determinant,
# the metric, the Ricci tensor, and the Christoffel symbols respectively.
#
make_gfa :=
proc(gfa_name::name,
     fnd_name_set::set(name),
     index_bounds::list(integer..integer),
     index_fn::{identical(none), name})	# note "name" here, not "procedure",
					# since we'll get the name of the
					# indexing fn, which may not have
					# been assigned its procedure yet
global N, `gfa/fnd_table`;
local fnd_name, var_name, index_fn_seq;

# firewall checks
if (not type(N, integer))
   then ERROR("must set up coordinates first!");
end if;
if ((index_bounds = []) and (index_fn <> 'none'))
   then ERROR("not meaningful to specify a symmetry function for a scalar!",
	      "   gfa_name=",gfa_name);
end if;

# create the Maple variable(s)
	for fnd_name in fnd_name_set
	do
	var_name := Maple_name(gfa_name, fnd_name);
	if (assigned(`gfa/fnd_table`[eval(var_name,1)]))
	   then ERROR("duplicate gfa/fnd definition!",
		      "   gfa_name=", gfa_name,
		      "   fnd_name_set=", fnd_name_set);
	end if;
	`gfa/fnd_table`[eval(var_name,1)] := true;
	if (index_bounds = [])
	   then # scalar
		unassign(  eval(var_name,1)  );
	   else # array
		if (index_fn = 'none')
		   then index_fn_seq := NULL;
		   else index_fn_seq := index_fn;
		end if;
		assign(
		    eval(var_name,1) = array(index_fn_seq, op(index_bounds))
		      );
	end if;
	end do;

NULL;
end proc;

################################################################################

#
# This function checks that a given gridfn array functional-dependence
# form exists.  If so, this function is a no-op; if not, this function
# calls ERROR().  Hence after calling this function, the caller can assume
# that the given functional-dependence form does indeed exist.
#
# This function may be called with either 1 or 2 arguments (the two
# types of calls are equivalent).
#
# Arguments (1-argument form):
# gfa_name = The name of the gridfn array itself.  In this case the
#	     fnd name defaults to `fnd`.
#
# Arguments (2-argument form):
# gfa_name = The name of the gridfn array itself, eg `g_dd`.
# fnd_name = The name of the fnd itself, eg `fnd`.
#
assert_fnd_exists :=
proc(gfa_name::name, fnd_name::name)
global `gfa/fnd_table`;
local var_name;

var_name := Maple_name(gfa_name, args[2..nargs]);

if (not assigned(`gfa/fnd_table`[eval(var_name,1)]))
   then ERROR("functional-dependence form doesn't exist!",
	      "var_name=",var_name);
end if;
end proc;

################################################################################

#
# This function computes the Maple name corresponding to a given
# gridfn array and functional dependence form.
#
# Arguments:
# gfa_name = The name of the gridfn array itself, eg `g_dd`.
# fnd_name = (optional)
#	     The name of the functional-dependence form, eg `fnd`,
#	     or the special name 'inert'.  If this argument is omitted
#	     it defaults to 'inert'.
#
Maple_name :=
proc(gfa_name::name, fnd_name::name)
if ((nargs = 1) or (fnd_name = 'inert'))
   then return gfa_name;
   else return cat(gfa_name, "__", fnd_name);
end if;
end proc;

################################################################################
################################################################################
################################################################################

#
# This function is a Maple indexing function for a rank 3 table/array
# which is symmetric in the 2nd and 3rd indices.  This function is
# called automagically by Maple whenever a component of such an array
# is referenced.
#
# The ordering is done using the Maple  sort()  function, which sorts
# numbers numerically, strings lexicographically, and "other things"
# by machine address.
#
# Arguments:
# ilist = (in) The indices to be used.  There must be exactly 3 of these.
# tab = (in out) The table/array to be indexed.  This uses any remaining
#		 indexing methods.
# vlist = (optional)
#	  If present, this is a list of the values to be assigned to
#	  the (lvalue) table entry.
#
`index/symmetric3_23` :=
proc(ilist::list, tab::table, vlist::list)
local k, i, j, index_seq;

if (not (nops(ilist) = 3))
   then ERROR(`must have exactly 3 indices!`);
end if;

# ... note we need eval() here to get actual integers,
#     rather than variable names whose values are integers
k := eval(ilist[1]);
i := eval(ilist[2]);
j := eval(ilist[3]);

index_seq := k, op(sort([i,j]));

if (nargs = 2)
   then return tab[index_seq];		# rvalue
   else tab[index_seq] := op(vlist);	# lvalue
end if;
end proc;

################################################################################
################################################################################
################################################################################

#
# This function prettyprints a symmetric3_23 array
#
# Arguments:
# A = (in) The array to be prettyprinted
#
# Results:
# There is no explicit result; the array is prettyprinted as a side effect.
#
print_symmetric3_23 :=
proc(A::array)
local bounds, k, i, j, M23;

if (op(1, eval(A)) <> 'symmetric3_23')
   then ERROR(`can only print symmetric3_23 arrays`);
end if;

bounds := op(2, eval(A));

M23 := array(bounds[2..3], symmetric);

	for k in $(bounds[1]) do

		for i in $(bounds[2]) do
		for j in $(bounds[3]) do
		if (i <= j)		# only need upper triangle in (i,j)
		   then M23[i,j] := A[k,i,j];
		end if;
		end do;
		end do;

	printf("[%d] = \n", k);
	print(M23);
	end do;

NULL;
end proc;

```

maple/gfa.minc:
```
# gfa.minc -- global variables for gridfn array system and other misc stuff
# $Header$

# arguments to make_gfa
inert, none,
fnd,

# indexing functions
symmetric3_23,

# coordinate-transformation coefficient gfas
X_ud, X_ud__fnd,
X_udd, X_udd__fnd
		# no comma

```

maple/README:
```
This directory holds Maple code for manipulating tensor differential
operators and converting them into C/C++ code.  This code doesn't itself
know any physics, but it does have a few bits of knowledge of our
coordinate systems:
coords.*	define all the coordinates
gfa.minc	defines the X_ud and X_udd coordinate-transformation
		coefficients

```

maple/coords.maple:
```
# coords.maple -- set up and covert between coordinates, gridfn arrays, etc
# $Header$

#
# setup_coords - set up coordinates
#

################################################################################

#
# This function sets up the coordinates.
#
setup_coords :=
proc()
global
  @include "../maple/coords.minc";

N := 3;					# number of spatial coordinates,
					# i.e. spatial coordinate indices
					#      are 1..N
N_ang := 2;				# number of angular spatial coordinates

# Kronecker delta
delta := array(1..N, 1..N, identity);

# (x,y,z) coordinates
# n.b. individual coordinates are (xx,yy,zz) to avoid confusion with
#      local variables x, y, etc
x_xyz := array(1..N);
x_xyz[1] := xx;
x_xyz[2] := yy;
x_xyz[3] := zz;
x_xyz_list := [xx, yy, zz];
x_xyz_set := {xx, yy, zz};

# r coordinate
r__fnd := ssqrt(xx^2 + yy^2 + zz^2);

# (rho,sigma) coordinates
y_rs := array(1..N_ang);
y_rs[1] := rho;
y_rs[2] := sigma;
y_rs_list := [rho, sigma];
y_rs_set := {rho, sigma};

# list of all coordinates (for sorting things into "nice" order)
xy_all_list := [op(x_xyz_list), op(y_rs_list)];
xy_all_set := {op(xy_all_list)};

NULL;
end proc;

```

maple/coeffs.maple:
```
# coeffs.maple - set up coordinate coefficients
# $Header$
#
# setup_coeff_gfas - create the coordinate-transformation coefficient gfas
#

################################################################################

#
# This function sets up the coordinate-transformation gfas.
# At present these are only inert -- Maple doesn't worry about
# their fnd expressions (ugly trig functions).
#
setup_coeff_gfas :=
proc()
global
  @include "../maple/coords.minc",
  @include "../maple/gfa.minc";

# 1st ($X^u_i$) and 2nd ($X^u_{ij}$) partial derivs
# of (r,rho,sigma) wrt (xx,yy,zz)
make_gfa('X_ud', {inert}, [1..N_ang, 1..N], none);
make_gfa('X_udd', {inert}, [1..N_ang, 1..N, 1..N], symmetric3_23);

NULL;
end proc;

```

maple/coords.minc:
```
# coords.minc -- global variables for coordinates
# $Header$

# Kronecker delta
delta,

# number of coordinates
N,				# number of spatial coordinates
N_ang,				# number of angular spatial coordinates

# coordinates
xx, yy, zz,			# Cartesian coordinates
x_xyz, x_xyz_list, x_xyz_set,	# ... array/list/set of these
r, r__fnd,			# radial coordinate,
				# ... inert and as fn(xx,yy,zz) coords
rho, sigma,			# generic patch coordinates
y_rs, y_rs_list, y_rs_set,	# array and list of generic patch coordinates
xy_all_list, xy_all_set		# union of Cartesian and generic patch coords
			# no comma

```

maple/codegen2.maple:
```
# codegen2.maple -- generate C code from Maple expressions
# $Header$
#
# codegen2 - generate C code from Maple expressions
#
# cvt_to_eqnlist - convert an expression to an equation list
# fix_Diff - convert Diff() calls to PARTIAL_*() calls
#   fix_Diff/remap_table - remapping table for fix_Diff()
# temps_in_eqnlist - find temporaries used by an equation list
# is_result - is a name equal to a "result" or an array/table component of it?
# deindex_names - remove indices from indexed names
# unindex_names - convert indexed names to scalars
# fix_rationals - convert numbers to RATIONAL() calls
# print_name_list_dcl - print a C declaration for a list of names
#

################################################################################
################################################################################
################################################################################

#
# This function is a high-level driver to generate C code for a Maple
# expression.  It does the following:
# - convert the input into a list of equations of the form  name = expression
# - rewrite Diff(...) calls to DIFF_RHO_RHO(...) etc calls
# - generate an "optimized" form of the computation sequence with
#   codegen[optimize, tryhard]
# - determine which temporary variables have been introduced by the
#   codegen[optimize] library
# - "unindex" all array accesses, eg R_dd[2,3] becomes R_dd_23.
# - convert rational numbers to RATIONAL(p,q) calls
# - print C declarations for the temporary variables
# - print C code for the the optimized computation sequence
#
# Note that "codegen" is a Maple package; this function is called "codegen2".
#
# Sample usage:
#	codegen2([R_dd__fnd, R__fnd], ['R_dd', 'R'], "Ricci.c");
#
# Arguments:
# expr = (in) The expression or list of expressions for which code is to
#	      be generated.  This will typically be the name of a gridfn
#	      array functional-dependence form, or a list of such names,
#	      but this isn't required.
# lhs_name = (in) The name or list of names to be used for the results in
#		  the generated C code, eg R_dd.
# output_file_name = (in) The name of the file to which the generated
#			  code is to be written.
#
# Arguments (as global variables)
# `saveit/level`
#	= (in) (optional)
#	       If this global variable is assigned, intermediate results
#	       are saved for debugging purposes.  If it's assigned an
#	       integral value, making this value larger may increase the
#	       level of debugging output.  (10 is a good number for typical
#	       debugging.)
#
codegen2 :=
proc(expr_in::{algebraic, list(algebraic)},
     lhs_name::{name, list(name)},
     output_file_name::string)
global
  @include "../maple/coords.minc",
  @include "../maple/gfa.minc";
local expr, expr_temps, input_set, output_set, expr_cost;

printf("codegen2(%a) --> \"%s\"\n", lhs_name, output_file_name);

expr := expr_in;
saveit(10, procname, "input", expr);

printf("   convert --> equation list\n");
expr := cvt_to_eqnlist(expr, lhs_name);
saveit(10, procname, "eqnlist", expr);

printf("   optimizing computation sequence\n");
expr := [codegen[optimize](expr)];
##expr := [codegen[optimize](expr, tryhard)];
saveit(10, procname, "optimize", expr);

printf("   find temporary variables\n");
expr_temps := temps_in_eqnlist(expr, lhs_name);
saveit(10, procname, "temps", expr_temps);

printf("   convert Diff(expr,rho,sigma) --> PARTIAL_RHO_SIGMA(expr) etc\n");
expr := fix_Diff(expr);
saveit(10, procname, "fix_Diff", expr);

input_set := deindex_names( indets(map(rhs,expr),name)
			    minus {op(expr_temps)}
			    minus xy_all_set );
output_set := deindex_names( {op(map(lhs,expr))} minus {op(expr_temps)} );
printf("   convert R_dd[2,3] --> R_dd_23 etc\n");
expr := unindex_names(expr);
saveit(10, procname, "unindex", expr);

expr_cost := codegen[cost](expr);

printf("   convert p/q --> RATIONAL(p/q)\n");
expr := fix_rationals(expr);
saveit(10, procname, "fix_rationals", expr);

#
# write the C code
#
printf("   writing C code\n");
ftruncate(output_file_name);
fprintf(output_file_name, "/*\n");
fprintf(output_file_name, " * inputs = %a\n", input_set);
fprintf(output_file_name, " * outputs = %a\n", output_set);
fprintf(output_file_name, " * cost = %a\n", expr_cost);
fprintf(output_file_name, " */\n");
print_name_list_dcl(expr_temps, "fp", output_file_name);
codegen[C](expr, filename=output_file_name);

NULL;
end proc;

################################################################################
################################################################################
################################################################################

#
# This function converts an expression or list of expressions into an
# equation list.  That is, given an expression  expr , this function
# computes an equation list of the form
#
# if type(expr, algebraic)
#	[ lhs_name = expr ]
#
# if type(expr, array)			# illustrated here for a rank-1 array
#	[
#	lhs_name[1] = expr[1],		# note equations are in lexicographic
#	lhs_name[2] = expr[2],		# order of the indices, and include
#	lhs_name[3] = expr[3],		# only those array elements that are
#	    ...				# explicitly stored (as reported by
#	lhs_name[N] = expr[N]		#  indices() )
#	]
#
# if type(expr, list)
#	then concatenate the equations lists from expr's elements
#
# Arguments:
# expr = (in) The expression to be converted.
# lhs_name = (in) The unevaluated name or list of names to use for the
#		  left hand side(s) in the equation list.
#
# Results:
# The equation list is returned as the function result.
#
cvt_to_eqnlist :=
proc(expr::{algebraic, array, list({algebraic, array})},
     lhs_name::{name, list(name)})

# ... test for array first since otherwise expr itself is a "name",
#     which would match type "algebraic" as well
if (type(expr, array) and type(lhs_name, name))
   then return map(
		      proc(ii)
		      return lhs_name[op(ii)] = expr[op(ii)];
		      end
		    ,
		      indices_in_order(expr)
		  );
fi;

if (type(expr, algebraic) and type(lhs_name, name))
   then return [lhs_name = expr];
fi;

if (type(expr, list({algebraic, array})) and type(lhs_name, list(name)))
   then return zip(op @ cvt_to_eqnlist, expr, lhs_name);
fi;

error "unknown type for expression!\n"
      "   expr=%1\n"
      "   whattype(expr)=%2\n"
      ,
      expr, whattype(expr);
end;

################################################################################

#
# This function converts  Diff()  calls into  PARTIAL_*()  calls, eg
# Diff(src, rho, sigma) --> PARTIAL_RHO_SIGMA(src).
#
fix_Diff :=
proc(expr::{algebraic, name = algebraic, list({algebraic, name = algebraic})})
local nn, k, base, power, fn, fn_args_list, Darg, Dvars;
global `fix_Diff/remap_table`;

# recurse over lists
if (type(expr, list))
   then return map(fix_Diff, expr);
fi;

# recurse over equation right hand sides
if (type(expr, name = algebraic))
   then return lhs(expr) = fix_Diff(rhs(expr));
fi;

nn := nops(expr);

# recurse over sums
if (type(expr, `+`))
   then return sum('fix_Diff(op(k,expr))', 'k'=1..nn);
fi;

# recurse over products
if (type(expr, `*`))
   then return product('fix_Diff(op(k,expr))', 'k'=1..nn);
fi;

# recurse over powers
if (type(expr, `^`))
   then 
	base := op(1, expr);
	power := op(2, expr);

	return fix_Diff(base) ^ power;
	fi;

# recurse over non-Diff functions
if type(expr, function) and (op(0, expr) <> 'Diff')
   then 
	fn := op(0, expr);
	fn_args_list := [op(expr)];
	
	fn; return '%'( op(map(fix_Diff, fn_args_list)) );
fi;

# remap derivatives
if type(expr, function) and (op(0, expr) = 'Diff')
   then
	Darg := op(1, expr);
	Dvars := [op(2..nn, expr)];
	if (assigned(`fix_Diff/remap_table`[op(Dvars)]))
	   then `fix_Diff/remap_table`[op(Dvars)]; return '%'(Darg);
	   else error "don't know how to remap Diff() call!\n"
		      "   Darg = %1\n"
		      "   Dvars = %2\n"
		      ,
		      Darg, Dvars;
	fi;
fi;

# otherwise, the identity function
return expr;
end;

########################################

#
# this table defines the remapping of Diff() calls for  fix_Diff()  (above)
# n.b. Diff() should already have canonicalized the order of variables
#
`fix_Diff/remap_table`[rho  ] := 'PARTIAL_RHO';
`fix_Diff/remap_table`[sigma] := 'PARTIAL_SIGMA';
`fix_Diff/remap_table`[rho  , rho  ] := 'PARTIAL_RHO_RHO';
`fix_Diff/remap_table`[rho  , sigma] := 'PARTIAL_RHO_SIGMA';
`fix_Diff/remap_table`[sigma, sigma] := 'PARTIAL_SIGMA_SIGMA';

`fix_Diff/remap_table`[xx] := 'PARTIAL_X';
`fix_Diff/remap_table`[yy] := 'PARTIAL_Y';
`fix_Diff/remap_table`[zz] := 'PARTIAL_Z';
`fix_Diff/remap_table`[xx,xx] := 'PARTIAL_XX';
`fix_Diff/remap_table`[xx,yy] := 'PARTIAL_XY';
`fix_Diff/remap_table`[xx,zz] := 'PARTIAL_XZ';
`fix_Diff/remap_table`[yy,yy] := 'PARTIAL_YY';
`fix_Diff/remap_table`[yy,zz] := 'PARTIAL_YZ';
`fix_Diff/remap_table`[zz,zz] := 'PARTIAL_ZZ';

################################################################################

#
# Given an equation list, this function finds all the temporaries
# assigned by it.  A "temporary" is defined here to be a name on the
# left hand side of an equation, which isn't the result or a component
# of it.
#
# Arguments:
# expr = (in) The equation list to operate on.
# result_name = (in) The result name or list/set of result names.
#
# Results:
# The function returns the list of temporaries assigned.
#
temps_in_eqnlist :=
proc(expr::list(name = algebraic),
     result_name::{name, list(name), set(name)})

# "temporary" = lhs name which isn't a result
return remove(is_result, map(lhs,expr), result_name);
end;

################################################################################

#
# This function tests whether or not a name is a "result" name or
# an array/table component of it.  Either a single result name, or a
# list/set of these, may be specified; in the latter case the function
# tests whether or not a name matches *any* of the result names.
#
# Arguments:
# try_name = (in) The name to test.
# result_name = (in) The name or list/set of names of the result to test
#		     against.
#
# Results:
# The function returns  true  if the name is equal to the result or an
# array/table component of it,  false  otherwise.
#
is_result :=
proc(try_name::name,
     result_name_in::{name, list(name), set(name)})
local result_name, rn;

if type(result_name_in, name)
   then result_name := { result_name_in };
   else result_name := result_name_in;
fi;

	for rn in result_name
	do
	if (try_name = rn)
	   then return true;
	elif (type(try_name, indexed) and (op(0,try_name) = rn))
	   then return true;
	fi;
	end do;

return false;
end;

################################################################################

#
# This function removes all indices from indexed names, eg
#	A[1,2,3] --> A .
#
# Arguments:
# expr = (in) The expression to be converted.
#
# Results:
# The converted expression is returned as the function result.
#
deindex_names :=
proc(expr::{name, function, list({name,function}), set({name,function})})
local fn, fn_args_list;

# recurse over lists and sets
if (type(expr, {list, set}))
   then return map(deindex_names, expr);
fi;

# recurse over function calls
if (type(expr, function))
   then
	fn := op(0, expr);
	fn_args_list := [op(expr)];
	fn; return '%'(op(map(deindex_names, fn_args_list)));
fi;

# convert indexed names
if (type(expr, indexed))
   then return op(0, expr);
fi;

# return non-indexed names and numbers unchanged
if (type(expr, {name, numeric}))
   then return expr;
fi;

# unknown type
error "expr has unknown type!\n"
      "whattype(expr)=%1\n"
      ,
      whattype(expr);
end;

################################################################################

#
# This function converts all occurence of indexed names in an expression
# to new non-indexed "scalar names" of the form
#	A[1,2,3] --> A_123 .
#
# Arguments:
# expr = (in) The expression to be converted.
#
# Results:
# The converted expression is returned as the function result.
#
unindex_names :=
proc(expr::{
	   algebraic, name = algebraic,
	   list({algebraic, name = algebraic}),
	   set({algebraic, name = algebraic})
	   })
local nn, k,
      base, power,
      fn, fn_args_list,
      base_name, index_seq;

# recurse over lists and sets
if (type(expr, {list, set}))
   then return map(unindex_names, expr);
fi;

# recurse over equations (both lhs and rhs)
if (type(expr, `=`))
   then return unindex_names(lhs(expr))  =  unindex_names(rhs(expr));
fi;

nn := nops(expr);

# recurse over sums
if (type(expr, `+`))
   then return sum('unindex_names(op(k,expr))', 'k'=1..nn);
fi;

# recurse over products
if (type(expr, `*`))
   then return product('unindex_names(op(k,expr))', 'k'=1..nn);
fi;

# recurse over powers
if (type(expr, `^`))
   then
	base := op(1, expr);
	power := op(2, expr);
	return unindex_names(base) ^ power;
fi;

# recurse over function calls
if (type(expr, function))
   then
	fn := op(0, expr);
	fn_args_list := [op(expr)];
	fn; return '%'(op(map(unindex_names, fn_args_list)));
fi;

# convert indexed names
if (type(expr, indexed))
   then
	base_name := op(0, expr);
	index_seq := op(expr);
	return cat(base_name,"_",index_seq);
fi;

# return numbers and non-indexed names
if (type(expr, {numeric, name}))
   then return expr;
fi;

# unknown type
error "expr has unknown type!\n"
      "whattype(expr)=%1\n"
      ,
      whattype(expr);
end;

################################################################################

#
# This function converts all integer or rational subexpressions of its
# input except integer exponents and integer factors in products, into
# function calls  RATIONAL(num,den)  with  num  and  den  integers.
#
# This is useful in conjunction with the  C() library function, since
#
#	C( (1/3) * foo * bar )
#		t0 = foo*bar/3;
#
# generates a (slow) division (and runs the risk of mixed-mode-arithmetic
# problems).  In contrast, with this function
#
#	fix_rationals((1/3) * foo * bar);
#	     RATIONAL(1,3) foo bar
#	codegen[C](%);
#	     t0 = RATIONAL(1.0,3.0)*foo*bar;
#
# which a C preprocessor macro can easily convert to the desired
#
#	     t0 = (1.0/3.0)*foo*bar;
#
# Arguments:
# expr = (in) The expression to be converted.
#
fix_rationals :=
proc(expr::{algebraic, name = algebraic, list({algebraic, name = algebraic})})
local nn, k,
      expr_sign, expr_abs,
      base, power, fbase, fpower,
      fn, fn_args_list,
      int_factors, nonint_factors,
      num, den, mult;

# recurse over lists
if (type(expr, list))
   then return map(fix_rationals, expr);
fi;

# recurse over equation right hand sides
if (type(expr, name = algebraic))
   then return lhs(expr) = fix_rationals(rhs(expr));
fi;

# recurse over functions other than  RATIONAL()
if (type(expr, function))
   then
	fn := op(0, expr);
	if (fn <> 'RATIONAL')
	   then
		fn_args_list := [op(expr)];
		fn; return '%'(op(map(fix_rationals, fn_args_list)));
	fi;
fi;

nn := nops(expr);

# recurse over sums
if (type(expr, `+`))
   then return sum('fix_rationals(op(k,expr))', 'k'=1..nn);
fi;

# recurse over products
# ... leaving integer factors intact
if (type(expr, `*`))
   then
	# compute lists of all integer/non-integer factors
	int_factors,nonint_factors := selectremove(type, expr, integer);

	if (nops(int_factors) > 0)
	   then return op(1,int_factors)
		       * product('fix_rationals(op(k,nonint_factors))',
				 'k'=1..nops(nonint_factors));
	   else return product('fix_rationals(op(k,expr))', 'k'=1..nn);
	fi;
fi;

# recurse over powers
# ... leaving integer exponents intact
if (type(expr, `^`))
   then
	base := op(1, expr);
	power := op(2, expr);

	fbase := fix_rationals(base);
	if (type(power, integer))
	   then fpower := power;
	   else fpower := fix_rationals(power);
	fi;
	return fbase ^ fpower;;
fi;

# fix integers and fractions
if (type(expr, integer))
   then return 'RATIONAL'(expr, 1);
fi;
if (type(expr, fraction))
   then
	num := op(1, expr);
	den := op(2, expr);
	return 'RATIONAL'(num, den);
fi;

# turn Maple floating-point into integer fraction, then recursively fix that
if (type(expr, float))
   then
	mult := op(1, expr);
	power := op(2, expr);
	return fix_rationals(mult * 10^power);
fi;

# identity op on names
if (type(expr, name))
   then return expr;
fi;

# unknown type
error "expr has unknown type!\n"
      "whattype(expr)=%1\n"
      "expr=%2\n"
      ,
      whattype(expr), expr;
end;

################################################################################

#
# This function prints C declarations for a list of names.
#
# Argument:
# name_list = A list of the names.
# name_type = The C type of the names, eg. "double".
# file_name = The file name to write (append) the declaration to.
#
print_name_list_dcl :=
proc( name_list::list({name,string}),
      name_type::string,
      file_name::string )
local nn;

nn := nops(name_list);

# print up to 10 declarations on one line
if (nn <= 10)
   then
	map(convert, name_list, string);
	ListTools[Join](%, ", ");
	cat(op(%));
	fprintf(file_name,
		"%s %s;\n",
		name_type, %);
	NULL;
	return;
fi;

# recurse for larger numbers of declarations
print_name_list_dcl([op(1..10, name_list)], name_type, file_name);
print_name_list_dcl([op(11..nn, name_list)], name_type, file_name);
end proc;

```

maple/Diff.maple:
```
# Diff.maple -- Diff() and friends
# $Header$

#
# simplify/Diff - simplify(...) function for expressions containing Diff()
# Diff - "inert" derivative function (low-level simplifications)
# Diff/gridfn - simplifications that know about gridfns
#

################################################################################
################################################################################
################################################################################

#
# This function is called automatically by  simplify()  whenever
#  simplify()'s  argument contains a  Diff()  call.  This interface
# to  simplify()  is described in
#	Michael B. Monagan
#	"Tips for Maple Users"
#	The Maple Technical Newsletter
#	Issue 10 (Fall 1993), p.10-18, especially p.17-18
# but not in the Maple library manual or the online help, which both
# imply that  simplify(..., Diff)  is needed for this procedure to be
# invoked).
#
# This function recurses over the expression structure.  For each
#  Diff() call, it calls  Diff()  itself to do "basic" derivative
# simplifications, then calls `Diff/gridfn`() to do further simplifications
# based on gridfn properties.
#
# expr = (in) An expression to be simplified.
#
# Results:
# The function returns the "simplified" expression.
#
`simplify/Diff` :=
proc(expr)
option remember;		# performance optimization
local temp;

# recurse on tables, equations, lists, sets, sums, products, powers
if type(expr, {table, `=`, list, set, `+`, `*`, `^`})
   then return map(simplify, expr);
end if;

# return unchanged on numbers, names, procedures
if type(expr, {numeric, name, procedure})
   then return expr;
end if;

# function call ==> simplify  Diff()  calls, recurse on others
if type(expr, function)
   then
	if (op(0, expr) = 'Diff')
	   then return Diff(op(expr));	# "basic" derivative simplifications
	   else
		temp := map(simplify, [op(expr)]);
		op(0,expr); return '%'(op(temp));
	end if;
end if;

# unknown type
ERROR(`expr has unknown type`, `whattype(expr)=`, whattype(expr));

end proc;

################################################################################

#
# This function implements some of the more useful simplification
# rules of algebra and calculus for Maple's "inert" differentiation
# function  Diff() .  It then calls  `Diff/gridfn`()  to do any
# further simplifications based on gridfn properties.
#
# In particular:
# - It distributes over sums.
# - It distributes over leading numeric factors.  (Maple's built-in
#   simplifier will always make numeric factors leading.)
# - It knows that derivatives of numeric constants are zero.
# - It knows that the derivative of a name with respect to itself is one.
# - It knows the product rule
#	Diff(f*g,x) = Diff(f,x)*g + f*Diff(g,x)
# - It knows the power rule
#	Diff(f^n,x) = n * f^(n-1) * Diff(f,x)
# - It flattens multi-level "Diff() trees", i.e. it makes the transformation
#	Diff(Diff(f,x),y) = Diff(f,x,y);
# - It canonicalizes the order of the things we're differentiating with
#   respect to, so that  simplify()  will recognize that derivatives
#   commute, i.e. that
#	Diff(f,x,y) = Diff(f,y,x)
#
# Arguments:
# operand = (in) The thing to be differentiated.
# var_seq = (in) (varargs) An expression sequence of the variables to
#			   differentiate with respect to.
#
unprotect('Diff');
Diff :=
proc(operand)			# varargs
option remember;		# performance optimization
local var_list,
      nn, nderiv,
      op_cdr,
      f, g, x, x_car, x_cdr, temp,
      n,
      inner_operand, inner_var_list, k,
      sorted_var_list,
      operand2, var_seq2;

var_list := [args[2..nargs]];

nn := nops(operand);
nderiv := nops(var_list);

##print(`Diff():  nn=`,nn,`  nderiv=`,nderiv,
##      `  operand=`,operand,`  var_list=`,var_list);

# distribute (recurse) over sums
if (type(operand, `+`))
   then return simplify(
		  sum('Diff(op(k, operand), op(var_list))', 'k'=1..nn)
		       );
end if;

# distribute (recurse) over leading numeric factors
if (type(operand, `*`) and (nn >= 1) and type(op(1,operand), numeric))
   then
	op_cdr := product('op(k,operand)', 'k'=2..nn);
	return simplify(  op(1,operand) * Diff(op_cdr, op(var_list))  );
end if;

# derivatives of numbers are zero
if (type(operand, numeric))
   then return 0;
end if;

# derivative of a name with respect to itself is one
if (type(operand, name) and (var_list = [operand]))
   then return 1;
end if;

# implement the product rule
# ... we actually implement it only for a 1st derivative of the
#     product of two factors; for fancier cases we recurse
if (type(operand, `*`))
   then
	if (nn = 0)
	   then # empty product = 1
		return 0;		# ==> Diff(1) = 0

	elif (nn = 1)
	   then # singleton product is equal to the single factor
		return simplify(  Diff(op(1,operand), op(var_list))  );

	elif (nn >= 2)
	   then # nontrivial product rule case

		# note that if we have more than 2 factors, g will
		# be the product of the cdr of the factor list, so
		# our later  Diff(g,...)  calls will recurse as
		# necessary to handle this case

		f := op(1,operand);
		g := product('op(k,operand)', 'k'=2..nn);

		if (nderiv = 1)
		   then # product rule for 1st derivatives:
			#   Diff(f*g,x) = Diff(f,x)*g + f*Diff(g,x)
			x := var_list[1];

			return simplify(  Diff(f,x)*g  +  f*Diff(g,x)  );

		   else # product rule for 2nd (and higher) derivatives
			# ==> recurse on derivatives
			x_car := var_list[1];
			x_cdr := var_list[2..nderiv];
			temp := simplify(  Diff(f*g, x_car)  );
			return simplify(  Diff(temp, op(x_cdr))  );
		end if;
	else ERROR(`impossible value of nn in product rule!`,
		   `(this should never happen!)`,
		   `   operand=`,operand,`   nn=`,nn);
	end if;
end if;

# implement the power rule
# ... we actually implement it only for 1st derivatives; for higher
#     derivatives we recurse
if (type(operand, `^`))
   then
	f := op(1,operand);
	n := op(2,operand);

	if (nderiv = 1)
	   then # power rule for 1st derivatives:
		#   Diff(f^n,x) = n * f^(n-1) * Diff(f,x)
		x := var_list[1];

		return simplify(  n * f^(n-1) * Diff(f, x)  );

	   else # power rule for 2nd (and higher) derivatives:
		# ==> recurse on derivatives
		x_car := var_list[1];
		x_cdr := var_list[2..nderiv];
		temp := simplify(  Diff(f^n, x_car)  );

		return simplify(  Diff(temp, op(x_cdr))  );
	end if;
end if;

# flatten (recurse on) multi-level "Diff() trees"
if (type(operand, function) and (op(0, operand) = 'Diff'))
   then
	inner_operand := op(1, operand);
	inner_var_list := [op(2..nn, operand)];

	return simplify(
		  Diff(inner_operand, op(inner_var_list), op(var_list))
		       );
end if;

# canonicalize ordering of derivatives
sorted_var_list := sort_var_list(var_list);

# simplifications based on gridfn properties known here...
temp := `Diff/gridfn`(operand, op(sorted_var_list));

# more simplifications based on gridfn properties known in other directories
if ( type(`Diff/gridfn2`, procedure)
     and type(temp, function) and (op(0,temp) = 'Diff') )
   then operand2 := op(1,temp);
	var_seq2 := op(2..nops(temp), temp);
	temp := `Diff/gridfn2`(operand2, var_seq2);
fi;

return temp;
end proc;

################################################################################

#
# This function implements further simplification rules for  Diff()
# based on gridfn properties (or at least those known here).
#
# It currently knows about the following simplifications:
# - Diff(X_ud[u,i], x_xyz[j]) --> X_udd[u,i,j]
#
# Anything else is returned unchanged.  (To avoid infinite recursion,
# such a return is *unevaluated*.)
#
# Arguments:
# operand = (in) The thing to be differentiated.
# var_seq = (in) (varargs) An expression sequence of the variables to
#			   differentiate with respect to.
#
`Diff/gridfn` :=
proc(operand)			# varargs
option remember;		# performance optimization
global
  @include "../maple/coords.minc",
  @include "../maple/gfa.minc",
  @include "../gr/gr_gfas.minc";
local var_list, posn;

var_list := [args[2..nargs]];

if ( type(operand, indexed) and (op(0,operand) = 'X_ud')
     and (nops(var_list) = 1) and member(var_list[1],x_xyz_list,'posn') )
   then return X_udd[op(operand), posn];
end if;

# unevaluated return to avoid infinite recursion
return 'Diff'(operand, op(var_list));
end proc;

```

patch/patch_info.cc:
```
// patch_info.cc -- POD struct of minimal info varying from one patch to another
// $Header$

//
// patch_info::grid_array_pars
// patch_info::grid_pars
/// patch_info::verify_grid_spacing_ok
//

#include <stdio.h>
#include <math.h>
#include <assert.h>

#include "cctk.h"

#include "config.h"
#include "stdc.h"
#include "../jtutil/util.hh"
#include "../jtutil/array.hh"
#include "../jtutil/cpm_map.hh"
#include "../jtutil/linear_map.hh"
using jtutil::error_exit;

#include "coords.hh"
#include "grid.hh"
#include "patch_info.hh"

// all the code in this file is inside this namespace
namespace AHFinderDirect
	  {

//******************************************************************************
//******************************************************************************
//******************************************************************************

//
// This function computes, and returns a reference to, a
//  struct grid_arrays::grid_array_pars  from the info in a
//  struct patch_info  and the additional information in the arguments.
//
// The result refers to an internal static buffer in this function; the
// usual caveats about lifetimes/overwriting apply.
//
// Arguments:
// ghost_zone_width = Width in grid points of all ghost zones.
// patch_extend_width = Number of grid points to extend each patch past
//		     "just touching" so as to overlap neighboring patches.
//		     Thus patches overlap by
//			patch_overlap_width = 2*patch_extend_width + 1
//		     grid points.  For example, with patch_extend_width == 2,
//		     here are the grid points of two neighboring patches:
//			x   x   x   x   x   X   X
//                                      |
//			        O   O   o   o   o   o   o
//		     Here | marks the "just touching" boundary,
//		     x and o the grid points before this extension,
//		     and X and O the extra grid points added by this
//		     extension.
// N_zones_per_right_angle = This sets the grid spacing (same in both
//			     directions) to 90.0 / N_zones_per_right_angle.
//			     It's a fatal error (error_exit()) if this
//			     doesn't evenly divide the grid sizes in both
//			     directions.
//
const grid_arrays::grid_array_pars&
  patch_info::grid_array_pars(int ghost_zone_width, int patch_extend_width,
			      int N_zones_per_right_angle)
	const
{
static
  struct grid_arrays::grid_array_pars grid_array_pars_buffer;

//
// the values of min_(irho,isigma) are actually arbitrary, but for
// debugging convenience it's handy to have (irho,isigma) ranges map
// one-to-one with (rho,sigma) ranges across all patches; the assignments
// here have this property
//
const fp delta_drho_dsigma = 90.0 / fp(N_zones_per_right_angle);
grid_array_pars_buffer.min_irho
	= jtutil::round<fp>::to_integer(min_drho  /delta_drho_dsigma);
grid_array_pars_buffer.min_isigma
	= jtutil::round<fp>::to_integer(min_dsigma/delta_drho_dsigma);

verify_grid_spacing_ok(N_zones_per_right_angle);
const int N_irho_zones
	= jtutil::round<fp>::to_integer(
		   fp(N_zones_per_right_angle) * (max_drho  -min_drho  ) / 90.0
				       );
const int N_isigma_zones
	= jtutil::round<fp>::to_integer(
		   fp(N_zones_per_right_angle) * (max_dsigma-min_dsigma) / 90.0
				       );

grid_array_pars_buffer.max_irho
	= grid_array_pars_buffer.min_irho   + N_irho_zones;
grid_array_pars_buffer.max_isigma
	= grid_array_pars_buffer.min_isigma + N_isigma_zones;

grid_array_pars_buffer.min_irho   -= patch_extend_width;
grid_array_pars_buffer.min_isigma -= patch_extend_width;
grid_array_pars_buffer.max_irho   += patch_extend_width;
grid_array_pars_buffer.max_isigma += patch_extend_width;

grid_array_pars_buffer.min_rho_ghost_zone_width = ghost_zone_width;
grid_array_pars_buffer.max_rho_ghost_zone_width = ghost_zone_width;
grid_array_pars_buffer.min_sigma_ghost_zone_width = ghost_zone_width;
grid_array_pars_buffer.max_sigma_ghost_zone_width = ghost_zone_width;

return grid_array_pars_buffer;
}

//******************************************************************************
//
//
// This function computes, and returns a reference to, a
//  struct grid_arrays::grid_pars  from the info in a  struct patch_info
// and the additional information in the arguments.
//
// The result refers to an internal static buffer in this function; the
// usual caveats about lifetimes/overwriting apply.
//
// Arguments:
// patch_extend_width = Number of grid points to extend each patch past
//		     "just touching" so as to overlap neighboring patches.
//		     Thus patches overlap by  2*patch_extend_width + 1  grid
//		     points.  For example, with patch_extend_width == 2, here
//		     are the grid points of two neighboring patches:
//			x   x   x   x   x   X   X
//                                      |
//			        O   O   o   o   o   o   o
//		     Here | marks the "just touching" boundary,
//		     x and o the grid points before this extension,
//		     and X and O the extra grid points added by this
//		     extension.
// N_zones_per_right_angle = This sets the grid spacing (same in both
//			     directions) to 90.0 / N_zones_per_right_angle.
//			     It's a fatal error (error_exit()) if this
//			     doesn't evenly divide the grid sizes in both
//			     directions.
//
const grid::grid_pars& patch_info::grid_pars(int patch_extend_width,
					     int N_zones_per_right_angle)
	const
{
static
  struct grid::grid_pars grid_pars_buffer;

verify_grid_spacing_ok(N_zones_per_right_angle);
const fp delta_drho_dsigma = 90.0 / fp(N_zones_per_right_angle);
const fp extend_drho_dsigma = fp(patch_extend_width) * delta_drho_dsigma;

grid_pars_buffer.  min_drho   = min_drho   - extend_drho_dsigma;
grid_pars_buffer.delta_drho   = delta_drho_dsigma;
grid_pars_buffer.  max_drho   = max_drho   + extend_drho_dsigma;
grid_pars_buffer.  min_dsigma = min_dsigma - extend_drho_dsigma;
grid_pars_buffer.delta_dsigma = delta_drho_dsigma;
grid_pars_buffer.  max_dsigma = max_dsigma + extend_drho_dsigma;

return grid_pars_buffer;
}

//******************************************************************************

//
// This function verifies that the grid spacing evenly divides the
// grid sizes in both directions, and does an  error_exit()  if not.
//
// Arguments:
// N_zones_per_right_angle = This sets the grid spacing (same in both
//			     directions) to 90.0 / N_zones_per_right_angle.
//
void patch_info::verify_grid_spacing_ok(int N_zones_per_right_angle)
	const
{
const fp N_irho_zones_fp
	= fp(N_zones_per_right_angle) * (max_drho  -min_drho  ) / 90.0;
const fp N_isigma_zones_fp
	= fp(N_zones_per_right_angle) * (max_dsigma-min_dsigma) / 90.0;

if (! (    jtutil::fuzzy<fp>::is_integer(N_irho_zones_fp)
	&& jtutil::fuzzy<fp>::is_integer(N_isigma_zones_fp)    ) )
   then error_exit(ERROR_EXIT,
"***** patch_info::verify_grid_spacing_ok():\n"
"        N_zones_per_right_angle=%d gives grid spacing which\n"
"        doesn't evenly divide grid sizes!\n"
"        [min,max]_drho=[%g,%g] [min,max]_dsigma=[%g,%g]\n"
"        ==> N_irho_zones_fp=%g N_isigma_zones_fp=%g\n"
		   ,
		   N_zones_per_right_angle,
		   double(min_drho), double(max_drho),
		   double(min_dsigma), double(max_dsigma),
		   double(N_irho_zones_fp), double(N_isigma_zones_fp));
								/*NOTREACHED*/
}

//******************************************************************************
//******************************************************************************
//******************************************************************************

	  }	// namespace AHFinderDirect

```

patch/test_patch_system.maple:
```
# test_patch_system.maple -- test function for  test_fd_grid
# $Header$

# test function
fn := exp(sin(1.38*rho)) * tanh(0.17+0.83*sin(sigma)^2);

# 1st derivatives
diff(fn, rho);			codegen[C](%);
diff(fn, sigma);		codegen[C](%);

# 2nd derivatives
diff(fn, rho,rho);		codegen[C](%);
diff(fn, rho,sigma);		codegen[C](%);
diff(fn, sigma,sigma);		codegen[C](%);

```

patch/ghost_zone.hh:
```
// ghost_zone.hh -- fill in gridfn data in patch ghost zones
// $Header$
//
// ***** design notes for ghost zones *****
// ghost_zone - abstract base class to describe ghost zone of patch
// symmetry_ghost_zone - ... derived class for spacetime-symmetry ghost zone
// interpatch_ghost_zone - ... derived class for interpatch ghost zone
//

//
// prerequisites:
//	<stdio.h>
//	<assert.h>
//	<math.h>
//	"stdc.h"
//	"config.hh"
//	"../jtutil/util.hh"
//	"../jtutil/array.hh"
//	"../jtutil/cpm_map.hh"
//	"../jtutil/linear_map.hh"
//	"coords.hh"
//	"grid.hh"
//	"fd_grid.hh"
//	"patch.hh"
//	"patch_edge.hh"
//	"patch_interp.hh"
//

// everything in this file is inside this namespace
namespace AHFinderDirect
	  {

//*****************************************************************************

//
// ***** design notes for ghost zones *****
//

//
// A  ghost_zone  object describes a patch's ghost zone, and knows how
// to compute gridfns there (we usually speak of "synchronizing" the
// ghost zone or zones) based on either the patch system's symmetry
// or interpolation from a neighboring patch.  ghost_zone is an abstract
// base class, from which we derive two concrete classes:
// * A  symmetry_ghost_zone  object describes a ghost zone which is a
//   (discrete) symmetry of spacetime, either mirror-image or periodic.
//   Such an object knows how to fill in ghost-zone gridfn data from
//   the "other side" of the symmetry.
// * An  interpatch_ghost_zone  object describes a ghost zone which
//   overlaps another patch.  Such an object knows how to get ghost
//   zone gridfn data from the other patch.  More accurately, it gets
//   the data by asking (calling) the appropriate one of the other
//   patch's  patch_interp  objects.
// Every patch has (points to) 4  ghost_zone  objects, one for each of
// the patch's sides.  See the comments in "patch.hh" for a "big picture"
// discussion of patches, patch edges, ghost zones, and patch interpolators.
//

//
// There are some unobvious complications involved in synchronizing
// the ghost zone "corners", i.e. in ghost zone points that are outside
// the nominal grid in *both* coordinates.  There are 3 basic cases here:
// * A corner between two symmetry ghost zones, for example the -x/-y
//   corner in the example below.  In this case it takes *two* sequential
//   symmetry operations to get gridfn data in the corner from the
//   nominal grid.  Symmetry operations commute, so at each point we'll
//   always get the same results independently of in which order we do
//   the symmetry operations.  Computationally, we actually do the operations
//   in both orders, one order's results overwriting the other's, but
//   this doesn't matter (because the results are the same).
// * A corner between two interpatch ghost zones, for example the +x/+y
//   corner in the example below.  In this case we could get the gridfn
//   data by either of two distinct interpolation operations (presumably
//   from two distinct patches), which would in general give slightly
//   different results.  In some ideal world we might do a centered
//   interpolation using data from both patches, but this would be
//   complicated:
//   - it would require a 2-D interpolation
//   - it would require bookkeeping for interpolating from multiple
//     patches within the same ghost zone, indeed for the same ghost
//     zone point
//   At present, we follow a simpler approach: we split the corner down
//   its diagonal,
//	[for the points on the diagonal we make an arbitrary choice;
//	at present this is that they belong to (and get their data via)
//	the rho ghost zone.]
//   and off-center the interpolation as necessary so each ghost-zone
//   point gets data solely from the neighboring patch on its own side.
// * A corner between a symmetry and an interpatch ghost zone, for
//   example the +x/-y or -x/+y corners in the example below.  In this
//   case we first do a symmetry operation in the neighboring patch,
//   then a fully centered interpolation (using the data just obtained
//   from a symmetry operation) to get data in the non-corner part of
//   the interpatch ghost zone.  After the interpatch interpolation,
//   we do a final symmetry operation to get gridfn data in the corner.
//
// In general, then, a ghost zone is rhomboid-shaped: iperp lies in a
// fixed interval, while ipar lies in an interval which may depend on
// iperp.  In general, this shape depends on the type (symmetry vs interpatch)
// of the adjacent ghost zones.
//

//
// To properly handle all the symmetry/interpatch cases described above,
// we use a 3-phase algorithm to synchronize ghost zones:
// Phase 1: Fill in gridfn data at all the non-corner points of symmetry
//	    ghost zones, by using the symmetries to get this data from
//	    its "home patch" nominal grids.
// Phase 2: Fill in gridfn data in all the interpatch ghost zones, by
//	    interpatch interpolating from neighboring patches as described
//	    above.
// Phase 3: Fill in gridfn data at all the corner points of symmetry
//	    ghost zones, by using the symmetries to get this data from
//	    its "home patch" nominal grids or ghost zones.
// Here a given ghost zone corner may be either a full rectangle (so any
// given point is a member of both adjacent corners), or split down its
// diagonal (so any given point is a member of only one corner).  This
// 3-phase algorithm is actually implemented by
//    patch_system::synchronize()
// which in turn calls
//    symmetry_ghost_zone::synchronize()
//    interpatch_ghost_zone::synchronize()
//

//
// For example, consider the +z patch in an octant patch system, with
// the ghost zones being 2 points wide.  The following illustration is
// looking down the z axis, and uses (x,y) for the patch coordinates
// for simplicity:
//
//                    #                                                   //
//                   i+y    i+y    i+y    i+y    i+y    i+y    i+y      //
//   (-2,7) (-1,7)  (0,7)  (1,7)  (2,7)  (3,7)  (4,7)  (5,7)  (6,7)  (7,7)
//    <s-x>  <s-x>    #                                              /i+x
//                    #                                            //
//                   i+y    i+y    i+y    i+y    i+y    i+y      //
//   (-2,6) (-1,6)  (0,6)  (1,6)  (2,6)  (3,6)  (4,6)  (5,6)  (6,6)  (7,6)
//    <s-x>  <s-x>    #                                       /i+x    i+x
//                    #                                     //
//                    #                                   //
//   (-2,5) (-1,5)   2,5)--(1,5)--(2,5)--(3,5)--(4,5)--(5,5)  (6,5)  (7,5)
//     s-x    s-x     #                                  |     i+x    i+x
//                    #                                  |
//                    #                                  |
//   (-2,4) (-1,4)  (0,4)  (1,4)  (2,4)  (3,4)  (4,4)  (5,4)  (6,4)  (7,4)
//     s-x    s-x     #                                  |     i+x    i+x
//                    #                                  |
//                    #                                  |
//   (-2,3) (-1,3)  (0,3)  (1,3)  (2,3)  (3,3)  (4,3)  (5,3)  (6,3)  (7,3)
//     s-x    s-x     #                                  |     i+x    i+x
//                    #                                  |
//                    #                                  |
//   (-2,2) (-1,2)  (0,2)  (1,2)  (2,2)  (3,2)  (4,2)  (5,2)  (6,2)  (7,2)
//     s-x    s-x     #                                  |     i+x    i+x
//                    #                                  |
//                    #                                  |
//   (-2,1) (-1,1)  (0,1)  (1,1)  (2,1)  (3,1)  (4,1)  (5,1)  (6,1)  (7,1)
//     s-x    s-x     #                                  |     i+x    i+x
//                    #                                  |
//                    #                                  |
//  #(-2,0)#(-1,0)##(0,0)##(1,0)##(2,0)##(3,0)##(4,0)##(5,0)##(6,0)##(7,0)
//     s-x    s-x     #                                        i+x    i+x
//                    #
//    <s-y>  <s-y>   s-y    s-y    s-y    s-y    s-y    s-y   <s-y>  <s-y>
//   (-2,-1)(-1,-1) (0,-1) (1,-1) (2,-1) (3,-1) (4,-1) (5,-1) (6,-1) (7,-1)
//    <s-x>  <s-x>    #
//                    #
//    <s-y>  <s-y>   s-y    s-y    s-y    s-y    s-y    s-y   <s-y>  <s-y>
//   (-2,-2)(-1,-2) (0,-2) (1,-2) (2,-2) (3,-2) (4,-2) (5,-2) (6,-2) (7,-2)
//    <s-x>  <s-x>    #
//                    #
//
// For this example,
// * The xz plane and yz plane are marked with ### lines
// * The +z patch's nominal grid is ([0,5],[0,5]), i.e. 0 <= x,y <= 5;
//   its boundary lines are shown with single lines --- and | .
// * The diagonal where we've split corners are marked with // lines.
// * The +z patch's ghost zones are
//	-x: (-1,[-1,7]), (-2,[-2,7]) 
//	+x: (6,[-2,6]), (7,[-2,7])
//	-y: ([-2, 7],[-2,-1])
//	+y: ([-2,5],6), ([-2,6],7)
// * The regions where we will interpolate data from the +z patch are
//	+x: ([ 3,4],[-2,7])
//	+y: ([-2,7],[ 3,4])
//   Note that in both cases the interpolation region includes the points
//   computed by symmetry (in phase 1 of our 3-phase algorithm) on the
//   adjacent edges! There are no interpolation regions inside the -x or
//   -y boundaries, since no interpolation is needed across those boundaries
//   of this patch.
// The diagonal *** line shows the boundary between the +x and +y ghost
// zones.
//
// Our 3-phase algorithm described above thus becomes:
// Phase 1: Fill in gridfn values at points marked with "s-x" below or
//	    "s-y" above via symmetry mirroring across the -x boundary
//	    (yz plane) or -y boundary (xz plane), as described by the
//	    +z patch's -x or -y  symmetry_ghost_zone  object respectively.
// Phase 2: Fill in gridfn values at points marked with "i+x" below or
//	    "i+y" above via interpatch interpolation from the neighboring
//	    patch across the +z patch's +x or +y boundary, as described
//	    by the +z patch's +x or +y  interpatch_ghost_zone  object
//	    respectively.
// Phase 3: Fill in gridfn values at points marked with "<s-x>" below or
//	    "<s-y>" above via symmetry mirroring across the -x boundary
//	    (yz plane) or -y boundary (xz plane), as described by the
//	    +z patch's -x or -y  symmetry_ghost_zone  object respectively.
//

//*****************************************************************************

//
// ghost_zone - abstract base class to describe ghost zone of patch
//
// This is an abstract base class describing a generic patch ghost zone.
// This might represent either of:
// - a discrete symmetry of spacetime (derived class symmetry_ghost_zone)
// - an overlap with another patch (derived class interpatch_ghost_zone)
//

//
// N.b. const qualifiers in ghost_zone and its derived classes refer to
//      the underlying gridfn data.
// 

// forward declarations
class symmetry_ghost_zone;
class interpatch_ghost_zone;
class patch_system;

class	ghost_zone
	{
public:
	//
	// ***** main high-level client interface *****
	//
	// "synchronize" a ghost zone, i.e. update the ghost-zone values
	// of the specified gridfns via the appropriate sequence of
	// symmetry operations and interpatch interpolations
	// (flags specify which part(s) of the ghost zone we want)
	//
	virtual void synchronize(int ghosted_min_gfn, int ghosted_max_gfn,
				 bool want_corners = true,
				 bool want_noncorner = true)
		= 0;


public:
	//
	// ***** Jacobian of synchronize() *****
	//
	// This function computes the Jacobian of the  synchronize()
	// operation into internal buffers; the following functions
	// provide access to that Jacobian.
	//
	// FIXME: should these be moved out into a separate Jacobian
	//        object/class?
	//
	// Note that this function just computes the Jacobian of this
	// ghost zone's  synchronize()  operation -- it does *NOT* take
	// into account the 3-phase synchronization algorithm described
	// in the header comments for this file.  (That's done by
	//  patch_system::synchronize_Jacobian()  and its subfunctions.)
	//
	// n.b. terminology is
	//	partial gridfn at x
	//	-------------------
	//	partial gridfn at y
	//
	virtual void compute_Jacobian(int ghosted_min_gfn, int ghosted_max_gfn,
				      bool want_corners = true,
				      bool want_noncorner = true)
		const
		= 0;

	//
	// The API in the remaining functions implicitly assumes that
	// the Jacobian is independent of  ghosted_gfn , and also that
	// the structure of the Jacobian is such that the set of y points
	// on which a single ghost-zone point depends,
	// - has a single yiperp value (depending on our iperp, of course)
	// - have a contiguous interval of yipar (depending on our iperp
	//   and ipar, of course), whose size is
	//	[or can be taken to be without an unreasonable
	//	amount of zero-padding]
	//   independent of our iperp and ipar; we parameterize this
	//   interval as  yipar = posn+m  where  posn  is determined by
	//   our iperp and ipar, and  m  has a fixed range independent
	//   of our iperp and ipar
	//

	// what is the [min,max] range of m for this ghost zone?
	virtual int Jacobian_min_y_ipar_m() const = 0;
	virtual int Jacobian_max_y_ipar_m() const = 0;

	// what is the iperp of the Jacobian y points in their (y) patch?
	virtual int Jacobian_y_iperp(int x_iperp) const = 0;

	// what is the  posn  value of the y points in this Jacobian row?
	virtual int Jacobian_y_ipar_posn(int x_iperp, int x_ipar) const = 0;

	// what is the Jacobian
	//	partial synchronize() px.gridfn(ghosted_gfn, x_iperp, x_ipar)
	//	-------------------------------------------------------------
	//	   partial py.gridfn(ghosted_gfn, y_iperp, y_posn+y_ipar_m)
	// where
	//	y_iperp = Jacobian_y_iperp(x_iperp)
	//	y_posn = Jacobian_y_ipar_posn(x_iperp, x_ipar)
	virtual fp Jacobian(int x_iperp, int x_ipar, int y_ipar_m) const = 0;


public:
	//
	// ***** low-level client interface *****
	//

	// to which patch/edge do we belong?
	patch&            my_patch() const { return my_patch_; }
	const patch_edge& my_edge()  const { return my_edge_; }

	// from which patch/edge do we get data?
	patch&            other_patch() const { return other_patch_; }
	const patch_edge& other_edge()  const { return other_edge_; }

	// what type of ghost zone are we?
	bool is_interpatch() const { return  is_interpatch_; }
	bool is_symmetry()   const { return !is_interpatch_; }

	// convenience forwarding functions down to patch_edge::
	bool is_min() const { return my_edge().is_min(); }
	bool is_rho() const { return my_edge().is_rho(); }

	// min/max iperp of the ghost zone
	int min_iperp() const
		{
		return my_patch()
		       .minmax_ang_ghost_zone__min_iperp(is_min(), is_rho());
		}
	int max_iperp() const
		{
		return my_patch()
		       .minmax_ang_ghost_zone__max_iperp(is_min(), is_rho());
		}

	// inner/outer iperp of the ghost zone wrt our patch
	int inner_iperp() const { return is_min() ? max_iperp() : min_iperp(); }
	int outer_iperp() const { return is_min() ? min_iperp() : max_iperp(); }

	// extreme min/max ipar that might possibly be part of this ghost zone
	// (derived classes may actually use a subset of this)
	int extreme_min_ipar() const
		{ return my_edge().min_ipar_with_corners(); }
	int extreme_max_ipar() const
		{ return my_edge().max_ipar_with_corners(); }

	// actual min/max ipar in the ghost zone at a particular iperp
	// (may depend on type of the adjacent ghost zones)
	virtual int min_ipar(int iperp) const = 0;
	virtual int max_ipar(int iperp) const = 0;

	// point membership predicate
	bool is_in_ghost_zone(int iperp, int ipar)
		const
		{
		// n.b. don't test ipar until we're sure iperp is in range!
		return (iperp >= min_iperp()) && (iperp <= max_iperp())
		       && (ipar >= min_ipar(iperp))
		       && (ipar <= max_ipar(iperp));
		}

	// adjacent ghost zones to our min/max corners
	const ghost_zone& min_par_adjacent_ghost_zone() const
		{
		return my_patch()
		       .ghost_zone_on_edge( my_edge().min_par_adjacent_edge() );
		}
	const ghost_zone& max_par_adjacent_ghost_zone() const
		{
		return my_patch()
		       .ghost_zone_on_edge( my_edge().max_par_adjacent_edge() );
		}

	//
	// ***** safely cast to derived classes *****
	//

	// assert that gz is of specified type,
	// then static_cast to derive type
	const symmetry_ghost_zone& cast_to_symmetry_ghost_zone() const;
	      symmetry_ghost_zone& cast_to_symmetry_ghost_zone();
	const interpatch_ghost_zone& cast_to_interpatch_ghost_zone() const;
	      interpatch_ghost_zone& cast_to_interpatch_ghost_zone();

	//
	// ***** constructor, finish setup, destructor *****
	//
protected:
	// ... values for  is_interpatch_in  constructor argument
	//     FIXME: these should really be bool, but then we couldn't
	//            use the "enum hack" for in-class constants
	enum {
	     ghost_zone_is_symmetry = false,
	     ghost_zone_is_interpatch = true // no comma
	     };

	// constructor
	// ... only used in implementing our derived classes;
	//     the rest of the world constructs our derived classes instead
	ghost_zone(const patch_edge& my_edge_in,
		   const patch_edge& other_edge_in,
		   bool is_interpatch_in)
		: my_patch_(my_edge_in.my_patch()),
		  my_edge_(my_edge_in),
		  other_patch_(other_edge_in.my_patch()),
		  other_edge_(other_edge_in),
		  is_interpatch_(is_interpatch_in)
		{ }
public:
	// assert() that ghost zone is fully setup:
	// defined here ==> no-op
	// symmetry ghost zone ==> unchanged ==> no-op
	// interpatch ghost zone ==> check consistency of this and the
	//			     other patch's ghost zones and
	//			     patch_interp objects
	virtual void assert_fully_setup() const { }

	// destructor must be virtual to allow destruction
	// of derived classes via ptr/ref to this class
	virtual ~ghost_zone() { }

private:
	// we forbid copying and passing by value
	// by declaring the copy constructor and assignment operator
	// private, but never defining them (either here or in derived classes)
	ghost_zone(const ghost_zone& rhs);
	ghost_zone& operator=(const ghost_zone& rhs);

private:
	patch& my_patch_;
	const patch_edge& my_edge_;
	patch& other_patch_;
	const patch_edge& other_edge_;
	const bool is_interpatch_;
	};

//*****************************************************************************

//
// symmetry_ghost_zone - derived class for spacetime-symmetry ghost zone
//
// In practice, there are two types of spacetime symmetry ghost zone:
// mirror symmetry and periodic symmetry.  However, it turns out that the
// code needed to handle periodic BCs is basically a superset of that
// needed to handle mirror symmetries, so this class represents a generic
// symmetry ghost zone which may be of either type, and once constructed
// doesn't distinguish between the two.
//
// In general, a symmetry ghost zone implies that there's a 1-1 mapping
// between ghost zone points of this patch, and (a subset of the) interior
// points of this or another patch.  If tensors are involved (this isn't
// used at present in the horizon finder), there's also a corresponding
// 1-1 mapping between (angular) tensor components.
//
// A mirror-symmetry ghost zone is specified by (the constructor arguments)
// - a patch edge
// - the (fp) perp coordinate of the mirror plane
// The mapping of ghost zone points is thus "just" the mirror imaging of
// iperp across the symmetry plane within this same patch.  (The mapping
// leaves ipar invariant.)
//
// A periodic-symmetry ghost zone is specified by (the constructor arguments)
// - a patch edge (specifies the ghost zone)
// - the patch edge to which the ghost zone is to be mapped
// - a pair of ipar coordinates, one on this edge and one on the other edge,
//   which map into each other
// - the sign of the ipar mapping (does increasing ipar on this edge map to
//   increasing or decreasing ipar on the other edge?)
// The mapping of ghost zone points is the periodic mapping; this may map
// the ghost zone points to interior points of either this same patch or a
// different one.
//
// In general, the symmetry mapping of ghost zone points is of the form
//	(iperp, ipar) --> (const +/- iperp, const +/- ipar)
// The iperp mapping is always in the direction
//	outside the patch --> inside the patch
// while the ipar mapping might have either sign.
// If there are tensors, the corresponding mapping of tensor components is
//	(index_perp, index_par) --> (+/-) (+/-) (index_perp, index_par)
// (that is, the two +/- signs are multiplied).

//
// Since all the member functions are  const , a  symmetry_ghost_zone
// object is effectively always  const .
//
class	symmetry_ghost_zone
	: public ghost_zone
	{
public:
	//
	// ***** main high-level client interface *****
	//
	// "synchronize" a ghost zone, i.e. update the ghost-zone values
	// of the specified gridfns via the appropriate symmetry operations
	// (flags specify which part(s) of the ghost zone we want)
	//
	void synchronize(int ghosted_min_gfn, int ghosted_max_gfn,
			 bool want_corners = true,
			 bool want_noncorner = true);

	//
	// ***** Jacobian of synchronize() *****
	//
	// n.b. terminology is
	//      partial gridfn at x
	//      -------------------
	//      partial gridfn at y
	//

	// allocate internal buffers, compute Jacobian
	// ... this function is a no-op in this class
	void compute_Jacobian(int ghosted_min_gfn, int ghosted_max_gfn,
			      bool want_corners = true,
			      bool want_noncorner = true)
		const
		{ }

	// what is the [min,max] range of m for this ghost zone?
	int Jacobian_min_y_ipar_m() const { return 0; }
	int Jacobian_max_y_ipar_m() const { return 0; }

	// what is the oiperp of the Jacobian points (= iperp in their patch)?
	virtual int Jacobian_y_iperp(int x_iperp) const
		{ return iperp_map_of_iperp(x_iperp); }

	// what is the  posn  value of the points in this Jacobian row?
	int Jacobian_y_ipar_posn(int x_iperp, int x_ipar) const
		{ return ipar_map_of_ipar(x_ipar); }

	// what is the Jacobian
	//	partial synchronize() px.gridfn(ghosted_gfn, x_iperp, x_ipar)
	//	-------------------------------------------------------------
	//	   partial py.gridfn(ghosted_gfn, y_iperp, y_posn+y_ipar_m)
	// where
	//	y_iperp = Jacobian_y_iperp(x_iperp)
	//	y_posn = Jacobian_y_ipar_posn(x_iperp, x_ipar)
	fp Jacobian(int x_iperp, int x_ipar, int y_ipar_m) const
		{ return (y_ipar_m == 0) ? 1.0 : 0.0; }


	//
	// ***** low-level client interface *****
	//

	// symmetry-map coordinates
	int iperp_map_of_iperp(int iperp) const
		{ return iperp_map_->map(iperp); }
	int ipar_map_of_ipar(int ipar) const
		{ return ipar_map_->map(ipar); }
	fp fp_sign_of_iperp_map() const
		{ return iperp_map_->fp_sign(); }
	fp fp_sign_of_ipar_map() const
		{ return ipar_map_->fp_sign(); }

	// min/max ipar of the ghost zone
	// ... we always include the corners
	//     (cf. the example at the start of this file)
	int min_ipar(int iperp) const { return extreme_min_ipar(); }
	int max_ipar(int iperp) const { return extreme_max_ipar(); }

	//
	// ***** constructors, destructor *****
	//
public:
	// constructor for mirror-symmetry ghost zone
	symmetry_ghost_zone(const patch_edge& my_edge_in);

	// constructor for periodic-symmetry ghost zone
	// ... ipar mapping specified by giving sample point and mapping sign
	symmetry_ghost_zone
	    (const patch_edge& my_edge_in, const patch_edge& other_edge_in,
	     int my_edge_sample_ipar,      int other_edge_sample_ipar,
	     bool ipar_map_is_plus);

	~symmetry_ghost_zone();

private:
        // we forbid copying and passing by value
        // by declaring the copy constructor and assignment operator
        // private, but never defining them
	symmetry_ghost_zone(const symmetry_ghost_zone& rhs);
	symmetry_ghost_zone& operator=(const symmetry_ghost_zone& rhs);

private:
	// symmetry mappings for (iperp,ipar)
	// ... we own these objects
	const jtutil::cpm_map<fp>* iperp_map_;
	const jtutil::cpm_map<fp>* ipar_map_;
	};

//*****************************************************************************

//
// interpatch_ghost_zone - derived class for interpatch ghost zone of a patch
//
// A ghost_zone object maps (my_iperp,my_ipar) coordinates to the other
// patch's (other_iperp,other_par) coordinates, then calls the other patch's
// patch_interp object to interpolate the other patch's data to those
// coordinates.
//
// Note that as described in the "design notes for ghost zones"
// comments above,  interpatch_ghost_zone  objects are constructed in
// the 2nd and 3rd phase of the overall construction process described
// at the comments at the start of "patch.hh"
// [done by our constructor]
// - set up the object itslf and its links to/from the patches and
//   their edges
// [done by  finish_setup()]
// - set up the interpatch mapping information, data pointers, and
//   interpolation result buffer
// - construct the  patch_interp  object to interpolate from the other
//   patch, and save a pointer to it
//

class patch_interp;

class	interpatch_ghost_zone
	: public ghost_zone
	{
public:
	//
	// ***** main high-level client interface *****
	//
	// "synchronize" a ghost zone, i.e. update the ghost-zone
	// values of the specified gridfns via the appropriate
	// interpatch interpolations
	// (flags specify which part(s) of the ghost zone we want)
	//
	// ... the present implementation only supports the case where
	//     both flags are set
	//
	void synchronize(int ghosted_min_gfn, int ghosted_max_gfn,
			 bool want_corners = true,
			 bool want_noncorner = true);

	//
	// ***** Jacobian of synchronize() *****
	//
	// n.b. terminology is
	//      partial gridfn at x
	//      -------------------
	//      partial gridfn at y
	//

	// allocate internal buffers, compute Jacobian
	//
	// ... the present implementation only supports the case where
	//     both flags are set
	//
	void compute_Jacobian(int ghosted_min_gfn, int ghosted_max_gfn,
			      bool want_corners = true,
			      bool want_noncorner = true)
		const;

	// what is the [min,max] range of m for this ghost zone?
	int Jacobian_min_y_ipar_m() const { return Jacobian_min_y_ipar_m_; }
	int Jacobian_max_y_ipar_m() const { return Jacobian_max_y_ipar_m_; }

	// what is the iperp of the Jacobian y points in their (y) patch?
	// ... the ipar row of grid points is actually the same, so
	//     we just have to translate x_iperp to the y patch's coordinates
	int Jacobian_y_iperp(int x_iperp) const { return other_iperp(x_iperp); }

	// what is the  posn  value of the y points in this Jacobian row?
	int Jacobian_y_ipar_posn(int x_iperp, int x_ipar) const
		{
		assert(Jacobian_y_ipar_posn_ != NULL);
		const int y_iperp = Jacobian_y_iperp(x_iperp);
		return (*Jacobian_y_ipar_posn_)(y_iperp, x_ipar);
		}

	// what is the Jacobian
	//	partial synchronize() px.gridfn(ghosted_gfn, x_iperp, x_ipar)
	//	-------------------------------------------------------------
	//	   partial py.gridfn(ghosted_gfn, y_iperp, y_posn+y_ipar_m)
	// where
	//	y_iperp = Jacobian_y_iperp(x_iperp)
	//	y_posn = Jacobian_y_ipar_posn(x_iperp, x_ipar)
	fp Jacobian(int x_iperp, int x_ipar, int y_ipar_m) const
		{
		assert(Jacobian_buffer_ != NULL);
		assert(y_ipar_m >= Jacobian_min_y_ipar_m_);
		assert(y_ipar_m <= Jacobian_max_y_ipar_m_);
		const int y_iperp = Jacobian_y_iperp(x_iperp);
		return (*Jacobian_buffer_)(y_iperp, x_ipar, y_ipar_m);
		}


	//
	// ***** low-level client interface *****
	//

public:
	// check consistency of this and the other patch's ghost zones
	// and patch_interp objects
	void assert_fully_setup() const;

	// min/max ipar of the ghost zone for specified iperp
	// with possibly "triangular" corners depending on the type
	// (symmetry vs interpatch) of the adjacent ghost zones
	// (cf. comments & example at the start of this file)
	int min_ipar(int iperp) const;
	int max_ipar(int iperp) const;

	// convert our iperp --> other patch's iperp
	int other_iperp(int iperp) const
		{
		assert(other_iperp_ != NULL);
		return other_iperp_->map(iperp);
		}

	//
	// ***** constructor, finish setup, destructor *****
	//
public:
	interpatch_ghost_zone(const patch_edge& my_edge_in,
			      const patch_edge& other_edge_in,
			      int patch_overlap_width);

	// finish setup (requires adjacent-side ghost_zone objects
	// to exist, though not to have finish_setup() called):
	// - setup par coordinate mapping information
	// - setup interpatch interpolator data pointers & result buffer
	// - create patch_interp object to interpolate from *other* patch
	void finish_setup(int interp_handle, int interp_par_table_handle);

	~interpatch_ghost_zone();

private:
        // we forbid copying and passing by value
        // by declaring the copy constructor and assignment operator
        // private, but never defining them
	interpatch_ghost_zone(const interpatch_ghost_zone& rhs);
	interpatch_ghost_zone& operator=(const interpatch_ghost_zone& rhs);

private:
	//
	// all the remaining pointers are initialized to NULL pointers
	// in our constructor, then finally allocated and set up by
	// finish_setup() or compute_Jacobian() as appropriate
	//
	// FIXME: should these be moved out into a separate object/class
	//        for the interp stuff and/or another one for the Jacobian?
	//

	// see comment in "patch_interp.hh" for why this is "const"
	const patch_interp* other_patch_interp_;

	// other patch's iperp coordinates of our ghost zone points
	// ... maps my_iperp --> other_iperp
	jtutil::cpm_map<fp>* other_iperp_;

	// min/max values of other patch's iperp coordinates
	// of our ghost zone points
	int min_other_iperp_, max_other_iperp_;

	// [min,max]_ipar used at each other_iperp
	// ... we will pass these arrays by reference
	//     to the other patch's patch_interp object
	// ... index is (other_iperp)
	jtutil::array1d<int>* min_ipar_used_;
	jtutil::array1d<int>* max_ipar_used_;

	// other patch's (fp) parallel coordinates of our ghost zone points
	// ... we will pass this array by reference
	//     to the other patch's patch_interp object
	//     using my_ipar as the patch_interp's parindex
	// ... subscripts are (other_iperp, my_ipar)
	jtutil::array2d<fp>* other_par_;

	// buffer into which the other patch's patch_interp object
	// will store the interpolated gridfn values
	// ... we will pass this array by reference
	//     to the other patch's patch_interp object
	//     using my_ipar as the patch_interp's parindex
	// ... subscripts are (gfn, other_iperp,my_ipar)
	jtutil::array3d<fp>* interp_result_buffer_;

	//
	// stuff computed by  compute_Jacobian()
	//
	// n.b. terminology is
	//      partial gridfn at x
	//      -------------------
	//      partial gridfn at y
	//
        mutable int Jacobian_min_y_ipar_m_;
	mutable int Jacobian_max_y_ipar_m_;

	// other patch's y ipar posn for a Jacobian row
	// ... subscripts are (oiperp, ipar)
	mutable jtutil::array2d<CCTK_INT>* Jacobian_y_ipar_posn_;

	// Jacobian values
	// ... subscripts are (y_iperp, x_ipar, y_ipar_m)
	mutable jtutil::array3d<fp>* Jacobian_buffer_;
	};

//******************************************************************************

	  }	// namespace AHFinderDirect

```

patch/patch_interp.cc:
```
// patch_interp.cc -- interpolation from a patch
// $Header$

//
// patch_interp::patch_interp
// patch_interp::~patch_interp
// patch_interp::interpolate
//
// patch_interp::verify_Jacobian_sparsity_pattern_ok
// patch_interp::molecule_minmax_ipar_m
// patch_interp::molecule_posn
// patch_interp::Jacobian
// patch_interp::query_interpolator
//

#include <stdio.h>
#include <assert.h>
#include <math.h>

#include "util_Table.h"
#include "cctk.h"

#include "config.h"
#include "stdc.h"
#include "../jtutil/util.hh"
#include "../jtutil/array.hh"
#include "../jtutil/cpm_map.hh"
#include "../jtutil/linear_map.hh"
using jtutil::error_exit;

#include "coords.hh"
#include "grid.hh"
#include "fd_grid.hh"
#include "patch.hh"
#include "patch_edge.hh"
#include "patch_interp.hh"
#include "ghost_zone.hh"

// all the code in this file is inside this namespace
namespace AHFinderDirect
	  {

//*****************************************************************************
//*****************************************************************************
//*****************************************************************************

//
// This function constructs an  patch_interp  object.
//
// Note this requires that the adjacent-side  ghost_zone  objects already
// exist, since they're used in determining the ipar range over which the
// interpolation will be done.
//
// It also requires that the patch's gridfns exist, since we size various
// arrays based on the patch's min/max ghosted gfn.
//
patch_interp::patch_interp(const patch_edge& my_edge_in,
			   int min_iperp_in, int max_iperp_in,
			   const jtutil::array1d<int>& min_parindex_array_in,
			   const jtutil::array1d<int>& max_parindex_array_in,
			   const jtutil::array2d<fp>& interp_par_in,
			   bool ok_to_use_min_par_ghost_zone,
			   bool ok_to_use_max_par_ghost_zone,
			   int interp_handle_in, int interp_par_table_handle_in)
	: my_patch_(my_edge_in.my_patch()),
	  my_edge_(my_edge_in),
	  min_gfn_(my_patch().ghosted_min_gfn()),
	  max_gfn_(my_patch().ghosted_max_gfn()),
	  ok_to_use_min_par_ghost_zone_(ok_to_use_min_par_ghost_zone),
	  ok_to_use_max_par_ghost_zone_(ok_to_use_max_par_ghost_zone),
	  min_iperp_(min_iperp_in), max_iperp_(max_iperp_in),
	  min_ipar_(ok_to_use_min_par_ghost_zone
		    ? my_edge_in.min_ipar_with_corners()
		    : my_edge_in.min_ipar_without_corners()),
	  max_ipar_(ok_to_use_max_par_ghost_zone
		    ? my_edge_in.max_ipar_with_corners()
		    : my_edge_in.max_ipar_without_corners()),
	  min_parindex_array_(min_parindex_array_in),
	  max_parindex_array_(max_parindex_array_in),
	  interp_par_(interp_par_in),
	  interp_handle_(interp_handle_in),
	  interp_par_table_handle_(Util_TableClone(interp_par_table_handle_in)),
	  // note interpolator coordinate origin = min_ipar_
	  // cf. comments in "patch_interp.hh" on gridfn array subscripting
	  gridfn_coord_origin_(my_edge().par_map().fp_of_int(min_ipar_)),
	  gridfn_coord_delta_ (my_edge().par_map().delta_fp()),
	  gridfn_type_codes_      (min_gfn_, max_gfn_),
	  gridfn_data_ptrs_       (min_gfn_, max_gfn_),
	  interp_data_buffer_ptrs_(min_gfn_, max_gfn_) // no comma
{
int status;

// did interpolator-parameter-table cloning work ok?
status = interp_par_table_handle_;
if (status < 0)
   then error_exit(ERROR_EXIT,
"***** patch_interp::patch_interp():\n"
"        can't clone interpolator parmameter table (handle=%d)!\n"
"        error status=%d\n"
,
		   interp_par_table_handle_in,
		   status);					/*NOTREACHED*/

//
// set up the gridfn array stride in the parameter table
// so the interpolator can access the gridfn data arrays
//

const int N_dims = 1;

// stride
const CCTK_INT stride = my_edge().ghosted_par_stride();
status = Util_TableSetIntArray(interp_par_table_handle_,
			       N_dims, &stride,
			       "input_array_strides");
if (status < 0)
   then error_exit(ERROR_EXIT,
"***** patch_interp::patch_interp():\n"
"        can't set gridfn stride in interpolator parmameter table!\n"
"        error status=%d\n"
,
		   status);					/*NOTREACHED*/

//
// set up the gridfn type codes for the interpolator
//
	for (int gfn = min_gfn_ ; gfn <= max_gfn_ ; ++gfn)
	{
	gridfn_type_codes_(gfn) = CCTK_VARIABLE_REAL;	// fp == CCTK_REAL
	}
}

//*****************************************************************************

//
// This function destroys an  patch_interp  object
//
patch_interp::~patch_interp()
{
Util_TableDestroy(interp_par_table_handle_);
}

//*****************************************************************************

//
// This function interpolates the specified range of ghosted gridfns
// at all the coordinates specified when we were constructed.  (It does
// a separate interpolator call for each iperp.)  This function stores
// the interpolated data in  data_buffer(gfn, iperp,parindex) .
//
void patch_interp::interpolate(int ghosted_min_gfn_to_interp,
			       int ghosted_max_gfn_to_interp,
			       jtutil::array3d<fp>& data_buffer)
	const
{
int status;

const int N_dims = 1;
const int N_gridfns = jtutil::how_many_in_range(ghosted_min_gfn_to_interp,
						ghosted_max_gfn_to_interp);
const CCTK_INT N_gridfn_data_points
	= jtutil::how_many_in_range(min_ipar(), max_ipar());
const int interp_coords_type_code = CCTK_VARIABLE_REAL;
const CCTK_INT *gridfn_type_codes_ptr
	= & gridfn_type_codes_(ghosted_min_gfn_to_interp);

//
// do the interpolations at each iperp
//
	for (int iperp = min_iperp() ; iperp <= max_iperp() ; ++iperp)
	{
	//
	// interpolation-point coordinates
	//
	const int min_parindex = min_parindex_array_(iperp);
	const int max_parindex = max_parindex_array_(iperp);
	const CCTK_INT N_interp_points
		= jtutil::how_many_in_range(min_parindex, max_parindex);
	const fp* const interp_coords_ptr = & interp_par_(iperp, min_parindex);
	const void* const interp_coords[N_dims]
		= { static_cast<const void *>(interp_coords_ptr) };

	//
	// pointers to gridfn data to interpolate, and to result buffer
	//
		for (int ghosted_gfn = ghosted_min_gfn_to_interp ;
		     ghosted_gfn <= ghosted_max_gfn_to_interp ;
		     ++ghosted_gfn)
		{
		// set up data pointer to --> (iperp,min_ipar) gridfn
		const int start_irho
			= my_edge().  irho_of_iperp_ipar(iperp, min_ipar());
		const int start_isigma
			= my_edge().isigma_of_iperp_ipar(iperp, min_ipar());
		gridfn_data_ptrs_(ghosted_gfn)
			= static_cast<const void *>(
				& my_patch()
				  .ghosted_gridfn(ghosted_gfn,
						  start_irho, start_isigma)
						   );
		interp_data_buffer_ptrs_(ghosted_gfn)
			= static_cast<void *>(
				& data_buffer(ghosted_gfn, iperp,min_parindex)
					     );
		}
	const void* const* const gridfn_data
		= & gridfn_data_ptrs_(ghosted_min_gfn_to_interp);
	void* const* const interp_buffer
		= & interp_data_buffer_ptrs_(ghosted_min_gfn_to_interp);

	//
	// interpolate for all the parindex values at this iperp
	//
	status = CCTK_InterpLocalUniform(N_dims,
					 interp_handle_,
					 interp_par_table_handle_,
					 &gridfn_coord_origin_,
					 &gridfn_coord_delta_,
					 N_interp_points,
					    interp_coords_type_code,
					    interp_coords,
					 N_gridfns,
					    &N_gridfn_data_points,
					    gridfn_type_codes_ptr,
					    gridfn_data,
					 N_gridfns,
					    gridfn_type_codes_ptr,
					    interp_buffer);

	if (status < 0)
	   then error_exit(ERROR_EXIT,
"***** patch_interp::interpolate():\n"
"        error return %d from interpolator at iperp=%d of [%d,%d]!\n"
"        my_patch()=\"%s\" my_edge()=\"%s\"\n"
,
			   status, iperp, min_iperp(), max_iperp(),
			   my_patch().name(), my_edge().name()); /*NOTREACHED*/
	}
}

//******************************************************************************
//******************************************************************************
//******************************************************************************

//
// This function verifies that our interpolator has a Jacobian sparsity
// pattern which we grok: at present this means that the molecules are
// fixed-sized hypercubes, with size/shape independent of the interpolation
// coordinates and the floating-point values in the input arrays.
// (N.b. we don't take derivatives in the interpatch interpolation,
// so it doesn't matter of the molecule size/shape depends on any
// operation_codes[] values.)
//
// FIXME: we should also query/check the molecule family!
//
// If the verification is successful, this function is a no-op.  If not,
// this function does an error_exit() and does not return to its caller.
//
void patch_interp::verify_Jacobian_sparsity_pattern_ok()
	const
{
int status1, status2, status3;

//
// query the interpolator
// ... we don't need to set up anything in the parameter table: the
//     Jacobian metainfo is returned by the interpolator on every call,
//     even without any explicit request
//
query_interpolator("verify_Jacobian_sparsity_pattern_ok");

//
// get Jacobian-sparsity query results from parameter table
//
CCTK_INT MSS_is_fn_of_interp_coords, MSS_is_fn_of_input_array_values;
CCTK_INT Jacobian_is_fn_of_input_array_values;
status1 = Util_TableGetInt(interp_par_table_handle_,
			   &MSS_is_fn_of_interp_coords,
			   "MSS_is_fn_of_interp_coords");
status2 = Util_TableGetInt(interp_par_table_handle_,
			   &MSS_is_fn_of_input_array_values,
			   "MSS_is_fn_of_input_array_values");
status3 = Util_TableGetInt(interp_par_table_handle_,
			   &Jacobian_is_fn_of_input_array_values,
			   "Jacobian_is_fn_of_input_array_values");
if ((status1 < 0) || (status2 < 0) || (status3 < 0))
   then error_exit(ERROR_EXIT,
"***** patch_interp::verify_Jacobian_sparsity_pattern_ok():\n"
"        can't get Jacobian sparsity info\n"
"        from interpolator parmameter table!\n"
"        error status1=%d status2=%d status3=%d\n"
,
		   status1, status2, status3);			/*NOTREACHED*/

//
// verify that we grok the Jacobian sparsity pattern
//
if ( MSS_is_fn_of_interp_coords || MSS_is_fn_of_input_array_values
     || Jacobian_is_fn_of_input_array_values )
   then error_exit(ERROR_EXIT,
"***** patch_interp::verify_Jacobian_sparsity_pattern_ok():\n"
"        implementation restriction: we only grok Jacobians with\n"
"        fixed-sized hypercube-shaped molecules, independent of\n"
"        the interpolation coordinates and the floating-point values!\n"
"        MSS_is_fn_of_interp_coords=(int)%d (we only grok 0)\n"
"        MSS_is_fn_of_input_array_values=(int)%d (we only grok 0)\n"
"        Jacobian_is_fn_of_input_array_values=(int)%d (we only grok 0)\n"
,
		   int(MSS_is_fn_of_interp_coords),
		   int(MSS_is_fn_of_input_array_values),
		   int(Jacobian_is_fn_of_input_array_values));	/*NOTREACHED*/
}

//******************************************************************************

//
// This function queries the interpolator to get the [min,max] ipar m
// coordinates of the interpolation molecules.
//
// (This API implicitly assumes that the Jacobian sparsity is one which
// is "ok" as verified by  verify_Jacobian_sparsity_pattern_ok() .)
//
void patch_interp::molecule_minmax_ipar_m(int& min_ipar_m, int& max_ipar_m)
	const
{
const int N_dims = 1;
int status1, status2;

//
// set molecule min/max m query entries in parameter table
//
status1 = Util_TableSetInt(interp_par_table_handle_,
			   0, "molecule_min_m");
status2 = Util_TableSetInt(interp_par_table_handle_,
			   0, "molecule_max_m");
if ((status1 < 0) || (status2 < 0))
   then error_exit(ERROR_EXIT,
"***** patch_interp::molecule_minmax_ipar_m():\n"
"        can't set molecule min/max m queries\n"
"        in interpolator parmameter table!\n"
"        error status1=%d status2=%d\n"
,
		   status1, status2);				/*NOTREACHED*/

//
// query the interpolator
//
query_interpolator("molecule_minmax_ipar_m");

//
// get molecule min/max m query results from parameter table
// ... must go through temp variables for CCTK_INT to int type conversion
//
CCTK_INT molecule_min_m, molecule_max_m;
status1 = Util_TableGetIntArray(interp_par_table_handle_,
				N_dims, &molecule_min_m,
				"molecule_min_m");
status2 = Util_TableGetIntArray(interp_par_table_handle_,
				N_dims, &molecule_max_m,
				"molecule_max_m");
if ((status1 < 0) || (status2 < 0))
   then error_exit(ERROR_EXIT,
"***** patch_interp::molecule_maxmax_ipar_m():\n"
"        can't get molecule min/max m query results\n"
"        from interpolator parmameter table!\n"
"        error status1=%d status2=%d\n"
,
		   status1, status2);				/*NOTREACHED*/
min_ipar_m = molecule_min_m;
max_ipar_m = molecule_max_m;

//
// delete Jacobian-sparsity-pattern query entries from the parameter table
// (so future interpolator calls don't mistakenly redo this query)
//
status1 = Util_TableDeleteKey(interp_par_table_handle_,
			      "molecule_min_m");
status2 = Util_TableDeleteKey(interp_par_table_handle_,
			      "molecule_max_m");
if ((status1 < 0) || (status2 < 0))
   then error_exit(ERROR_EXIT,
"***** patch_interp::verify_Jacobian_sparsity_pattern_ok():\n"
"        can't delete molecule min/max m queries\n"
"        from interpolator parmameter table!\n"
"        error status1=%d status2=%d\n"
,
		   status1, status2);				/*NOTREACHED*/
}

//******************************************************************************

//
// This function queries the interpolator at each iperp to find out the
// molecule ipar positions (which we implicitly assume to be independent
// of ghosted_gfn), and stores these in  posn_buffer(iperp, parindex) .
//
// (This API implicitly assumes that the Jacobian sparsity is one which
// is "ok" as verified by  verify_Jacobian_sparsity_pattern_ok() .)
//
void patch_interp::molecule_posn(jtutil::array2d<CCTK_INT>& posn_buffer)
	const
{
const int N_dims = 1;
int status;

	for (int iperp = min_iperp() ; iperp <= max_iperp() ; ++iperp)
	{
	const int min_parindex = min_parindex_array_(iperp);
	const int max_parindex = max_parindex_array_(iperp);

	// set up the molecule-position query in the parameter table
	CCTK_POINTER molecule_posn_ptrs[N_dims]
	    = { static_cast<CCTK_POINTER>(& posn_buffer(iperp, min_parindex)) };
	status = Util_TableSetPointerArray(interp_par_table_handle_,
					   N_dims, molecule_posn_ptrs,
					   "molecule_positions");
	if (status < 0)
	   then error_exit(ERROR_EXIT,
"***** patch_interp::molecule_posn():\n"
"        can't set molecule position query\n"
"        in interpolator parmameter table at iperp=%d of [%d,%d]!\n"
"        error status=%d\n"
,
			   iperp, min_iperp(), max_iperp(),
			   status);				/*NOTREACHED*/

	// query the interpolator to get the molecule positions
	// for all parindex at this iperp; the interpolator will
	// store the  parindex-min_parindex  values (cf comments
	// on array subscripting at the start of "patch_interp.hh")
	// directly in the  posn_buffer(iperp,*)  array)
	query_interpolator("molecule_positions", iperp);

	// convert the molecule positions from  parindex-min_ipar
	// to  parindex  values (again, cf comments on array subscripting
	// at the start of "patch_interp.hh")
		for (int parindex = min_parindex ;
		     parindex <= max_parindex ;
		     ++parindex)
		{
		posn_buffer(iperp, parindex) += min_ipar();
		}
	}

//
// if we actually did any queries,
// delete molecule-position query entry from the parameter table
// (so future interpolator calls don't mistakenly redo this query)
//
if (jtutil::how_many_in_range(min_iperp(), max_iperp()) > 0)
   then {
	status = Util_TableDeleteKey(interp_par_table_handle_,
				     "molecule_positions");
	if (status < 0)
	   then error_exit(ERROR_EXIT,
"***** patch_interp::molecule_posn():\n"
"        can't delete molecule position query"
"        from interpolator parmameter table!\n"
"        error status=%d\n"
,
			   status);				/*NOTREACHED*/
	}
}

//*****************************************************************************

//
// This function queries the interpolator at each iperp to get the
// Jacobian of the interpolated data with respect to this patch's
// ghosted gridfns,
//	partial interpolate() data_buffer(ghosted_gfn, iperp, parindex)
//	---------------------------------------------------------------
//	    partial ghosted_gridfn(ghosted_gfn, iperp, posn+ipar_m)
//
// This function stores the Jacobian in
//	Jacobian_buffer(iperp, parindex, ipar_m)
// where we implicitly assume the Jacobian to be independent of
// ghosted_gfn[*], and where
//	posn = posn_buffer(iperp, parindex)
// is the molecule position as returned by  molecule_posn() .
//
// [*] We actually do the queries for  min_gfn_  (as computed in our
//     constructor).
//
// This function calls  error_exit() if any interpolation point is out
// of range or any other error is detected.
//
// (This API implicitly assumes that the Jacobian sparsity is one which
// is "ok" as verified by  verify_Jacobian_sparsity_pattern_ok() .)
//
void patch_interp::Jacobian(jtutil::array3d<fp>& Jacobian_buffer)
	const
{
const int N_dims = 1;
const int N_gridfns = 1;

int status, status1, status2;


//
// set Jacobian stride info in parameter table
//
const int Jacobian_interp_point_stride
	= Jacobian_buffer.subscript_stride_j();
status1 = Util_TableSetInt(interp_par_table_handle_,
			   Jacobian_interp_point_stride,
			   "Jacobian_interp_point_stride");
const CCTK_INT Jacobian_m_strides[N_dims]
	= { Jacobian_buffer.subscript_stride_k() };	// = { 1 } in practice
status2 = Util_TableSetIntArray(interp_par_table_handle_,
				N_dims, Jacobian_m_strides,
				"Jacobian_m_strides");
if ((status1 < 0) || (status2 < 0))
   then error_exit(ERROR_EXIT,
"***** patch_interp::Jacobian():\n"
"        can't set Jacobian stride info in interpolator parmameter table!\n"
"        error status1=%d status2=%d\n"
,
		   status1, status2);			/*NOTREACHED*/


//
// query the Jacobians at each iperp
//
	for (int iperp = min_iperp() ; iperp <= max_iperp() ; ++iperp)
	{
	const int min_parindex = min_parindex_array_(iperp);

	//
	// set up the Jacobian query in the parameter table
	//
	CCTK_POINTER const Jacobian_ptrs[N_gridfns]
		= { static_cast<CCTK_POINTER>(
			& Jacobian_buffer(iperp, min_parindex, 0)
					     ) };
	status = Util_TableSetPointerArray(interp_par_table_handle_,
					   N_gridfns, Jacobian_ptrs,
					   "Jacobian_pointer");
	if (status < 0)
	   then error_exit(ERROR_EXIT,
"***** patch_interp::Jacobian():\n"
"        can't set Jacobian pointer in interpolator parmameter table!\n"
"        error status=%d at iperp=%d of [%d,%d]!\n"
,
			   status, iperp, min_iperp(), max_iperp());
								/*NOTREACHED*/

	//
	// query the interpolator to get the Jacobian for this iperp
	//
	query_interpolator("Jacobian", iperp);
	}

//
// delete the query entries from the parameter table
// (so future interpolator calls don't mistakenly redo this query)
//
status = (jtutil::how_many_in_range(min_iperp(), max_iperp()) > 0)
	 ? Util_TableDeleteKey(interp_par_table_handle_,
			       "Jacobian_pointer")
	 : 0;
status1 = Util_TableDeleteKey(interp_par_table_handle_,
			      "Jacobian_interp_point_stride");
status2 = Util_TableDeleteKey(interp_par_table_handle_,
			      "Jacobian_m_strides");
if ((status < 0) || (status1 < 0) || (status2 < 0))
   then error_exit(ERROR_EXIT,
"***** patch_interp::interpolate():\n"
"        can't delete Jacobian query entries\n"
"        from interpolator parmameter table!\n"
"        error status=%d status1=%d status2=%d\n"
,
		   status, status1, status2);			/*NOTREACHED*/
}

//******************************************************************************

//
// This function queries the interpolator with whatever arguments are
// in the parameter table.  It specifies arguments such that no actual
// interpolation is done.
//
// In particular, the following interpolator arguments are set up properly
// based on  iperp :
//	N_dims
//	operator_handle, param_table_handle,
//	coord_origin, coord_delta,
//	N_interp_points, interp_coords_type_code, interp_coords,
//	input_array_dims	# specifies the correct par size of the
//				# patch interpolation region at this iperp
// The following arguments are set to specify a single input array, but
// with a NULL data pointer so no actual data is used:
//	N_input_arrays
//	input_array_type_codes, input_arrays
// The following arguments are set to specify a single output array, but
// with a NULL data pointer so no actual data is stored:
//	N_output_arrays
//	output_array_type_codes, output_arrays
//
// Arguments:
// iperp = (in) Specifies where in the patch interpolation region the
//		interpolator query should be done.
//
// Results:
// If the interpolator returns an error status, this function does an
//  error_exit()  (and does not return to its caller).  Otherwise, this
// function returns the interpolator status code.
//
int patch_interp::query_interpolator(const char function_name[], int iperp)
	const
{
int status;
const int N_dims = 1;

const int min_parindex = min_parindex_array_(iperp);
const int max_parindex = max_parindex_array_(iperp);
const int N_interp_points = jtutil::how_many_in_range(min_parindex,
						      max_parindex);
const int interp_coords_type_code = CCTK_VARIABLE_REAL;
const void* const interp_coords[N_dims]
	= { static_cast<const void *>(& interp_par_(iperp, min_parindex)) };

const int N_input_arrays = 1;
const CCTK_INT input_array_dims[N_dims]
	= { jtutil::how_many_in_range(min_ipar(), max_ipar()) };
const CCTK_INT input_array_type_codes[N_input_arrays] = { CCTK_VARIABLE_REAL };
const void* const input_arrays[N_input_arrays] = { NULL };

const int N_output_arrays = 1;
const CCTK_INT output_array_type_codes[N_output_arrays]
	= { CCTK_VARIABLE_REAL };
void* const output_arrays[N_output_arrays] = { NULL };

status = CCTK_InterpLocalUniform(N_dims,
				 interp_handle_, interp_par_table_handle_,
				 &gridfn_coord_origin_, &gridfn_coord_delta_,
				 N_interp_points,
				    interp_coords_type_code,
				    interp_coords,
				 N_input_arrays,
				    input_array_dims,
				    input_array_type_codes,
				    input_arrays,
				 N_output_arrays,
				    output_array_type_codes,
				    output_arrays);
if (status < 0)
   then error_exit(ERROR_EXIT,
"***** patch_interp::query_interpolator():\n"
"        on behalf of patch_interp::%s()\n"
"        error return %d from interpolator query at iperp=%d of [%d,%d]!\n"
,
		   function_name,
		   status, iperp, min_iperp(), max_iperp());	/*NOTREACHED*/

return status;
}

//******************************************************************************
//******************************************************************************
//******************************************************************************

	  }	// namespace AHFinderDirect

```

patch/test_patch_system.cc:
```
// test_patch_system.cc -- test driver for patch_system::
// $Header$

//
// <<<misc global data>>>
// <<<prototypes>>>
//
// driver - Cactus interface
/// test_ghost_zone_Jacobians - test Jacobian of ghost zone synchronization ops
//
/// verify_gpn - verify gpn <--> (patch,irho,isigma) conversions
///
/// setup_sym_fn_xyz - set up symmetrized test function fn(global_[xyz])
/// setup_fn_rho_sigma - set up test function fn(rho,sigma)
/// finite_diff - compute linear combination of finite differences
/// analytic_derivs - compute linear combination of analytic derivatives
///
/// sym_fn_xyz - symmetrized test function fn(x,y,z) + fn(-y,x,z) + ...
/// fn_xyz - test function fn(x,y,z)
///
/// gridfn_minus - compute gridfn x - gridfn y --> gridfn z
/// ghosted_gridfn_minus - compute [ghosted] gridfn x - gridfn y --> gridfn z
///
/// fn_rho_sigma - test function fn(rho,sigma)
/// finite_diff_fn - finite differences of fn(rho,sigma)
/// analytic_deriv_fn - analytical derivs of fn(rho,sigma) (via Maple)
//

#include <stdio.h>
#include <assert.h>
#include <math.h>

#include "util_Table.h"
#include "cctk.h"
#include "cctk_Arguments.h"
#include "cctk_Parameters.h"

#include "config.h"
#include "stdc.h"
#include "../jtutil/util.hh"
#include "../jtutil/array.hh"
#include "../jtutil/cpm_map.hh"
#include "../jtutil/linear_map.hh"
using jtutil::error_exit;

#include "coords.hh"
#include "grid.hh"
#include "fd_grid.hh"
#include "patch.hh"
#include "patch_edge.hh"
#include "patch_interp.hh"
#include "ghost_zone.hh"
#include "patch_system.hh"

using namespace AHFinderDirect;

//******************************************************************************

//
// misc global data
//

// which test are we going to do?
static const int which_deriv_fn          = 0x1;
static const int which_deriv_rho         = 0x2;
static const int which_deriv_sigma       = 0x4;
static const int which_deriv_rho_rho     = 0x8;
static const int which_deriv_rho_sigma   = 0x10;
static const int which_deriv_sigma_sigma = 0x20;

// when testing multiple derivatives, we combine them together
// with these "quasi-random" weights (chosen from digits of pi)
static const fp deriv_weight_fn          = 3.14,
		deriv_weight_rho         = 1.59,
		deriv_weight_sigma       = 2.65,
		deriv_weight_rho_rho     = 3.58,
		deriv_weight_rho_sigma   = 9.79,
		deriv_weight_sigma_sigma = 3.23;

// n.b. nominal gfns must all be < 0, ghosted > 0
// nominal gridfns
static const int FD_derivs_gfn = -1;
static const int analytic_derivs_gfn = -2;
static const int nominal_error_gfn = -3;
static const int nominal_min_gfn = -3;
static const int nominal_max_gfn = -1;

// ghosted gridfns
static const int test_fn_gfn = 1;
static const int test_fn_copy_gfn = 2;
static const int ghosted_error_gfn = 3;
static const int ghosted_min_gfn = 1;
static const int ghosted_max_gfn = 3;

//******************************************************************************

//
// ***** prototypes *****
//

extern "C"
  void test_patch_system(CCTK_ARGUMENTS);

namespace {
void test_ghost_zone_Jacobians(patch_system& ps,
				int test_gfn, int NP_test_gfn,
				fp perturbation_amplitude,
				bool perturb_all_y_patch_points,
				const char Jacobian_file_name[]);

void verify_gpn(const patch_system& ps);

void setup_sym_fn_xyz(patch_system& ps, int ghosted_gfn, bool want_ghost_zones);
void setup_fn_rho_sigma(patch_system& ps, int ghosted_gfn);
void finite_diff(patch_system& ps,
		 int ghosted_gfn_src, int gfn_dst,
		 int which_derivs);
void analytic_derivs(patch_system& ps, int gfn_dst, int which_derivs);

void gridfn_minus(patch_system& ps, 
		  int gfn_x, int gfn_y, int gfn_dst);
void ghosted_gridfn_minus(patch_system& ps, 
			  int ghosted_gfn_x, int ghosted_gfn_y,
			  int ghosted_gfn_dst);

fp sym_fn_xyz(enum patch_system::patch_system_type type, fp x, fp y, fp z);
fp fn_xyz(fp x, fp y, fp z);

fp fn_rho_sigma(fp rho, fp sigma);
fp finite_diff_fn(const patch& p,
		  int ghosted_gfn_src, int which_derivs,
		  int irho, int isigma);
fp analytic_deriv_fn(fp rho, fp sigma, int which_derivs);
	  }

//******************************************************************************
//******************************************************************************
//******************************************************************************

//
// This function is the Cactus interface for the test driver.
//
extern "C"
  void test_patch_system(CCTK_ARGUMENTS)
{
DECLARE_CCTK_ARGUMENTS
DECLARE_CCTK_PARAMETERS

//
// set up the interpatch interpolator
//
CCTK_VInfo(CCTK_THORNSTRING, "setting up interpatch interpolator");
const int interp_handle = CCTK_InterpHandle(interpatch_interpolator_name);
if (interp_handle < 0)
   then CCTK_VWarn(FATAL_ERROR, __LINE__, __FILE__, CCTK_THORNSTRING,
		   "couldn't find interpolator \"%s\"!",
		   interpatch_interpolator_name);		/*NOTREACHED*/
const int interp_par_table_handle
	= Util_TableCreateFromString(interpatch_interpolator_pars);
if (interp_par_table_handle < 0)
   then CCTK_VWarn(FATAL_ERROR, __LINE__, __FILE__, CCTK_THORNSTRING,
		   "bad interpatch-interpolator parameter(s) \"%s\"!",
		   interpatch_interpolator_pars);		/*NOTREACHED*/

//
// create the patch system
//
CCTK_VInfo(CCTK_THORNSTRING, "about to create patch system");
patch_system ps(origin_x, origin_y, origin_z,
		patch_system::type_of_name(patch_system_type),
		ghost_zone_width, patch_overlap_width, N_zones_per_right_angle,
		nominal_min_gfn, nominal_max_gfn,
		ghosted_min_gfn, ghosted_max_gfn,
		interp_handle, interp_par_table_handle);
CCTK_VInfo(CCTK_THORNSTRING, "patch system created ok");

//
// do the actual tests
//
if      (STRING_EQUAL(which_test, "gridfn"))
   then {
	verify_gpn(ps);
	setup_sym_fn_xyz(ps, test_fn_gfn, true);
	ps.print_ghosted_gridfn(test_fn_gfn, "test_fn.dat");
	}

else if (STRING_EQUAL(which_test, "read gridfn"))
   then {
	verify_gpn(ps);
	ps.read_ghosted_gridfn(test_fn_gfn, "test_fn.dat");
	ps.print_ghosted_gridfn(test_fn_gfn, "test_fn2.dat");
	}

else if (STRING_EQUAL(which_test, "synchronize"))
   then {
	verify_gpn(ps);
	setup_sym_fn_xyz(ps, test_fn_gfn, false);
	ps.print_ghosted_gridfn(test_fn_gfn, "test_fn_init.dat");

	setup_sym_fn_xyz(ps, test_fn_copy_gfn, true);
	ps.print_ghosted_gridfn(test_fn_copy_gfn, "test_fn_copy.dat");

	ps.synchronize(test_fn_gfn, test_fn_gfn);
	ps.print_ghosted_gridfn(test_fn_gfn, "test_fn_sync.dat");

	ghosted_gridfn_minus(ps,
			     test_fn_gfn, test_fn_copy_gfn,
			     ghosted_error_gfn);
	ps.print_ghosted_gridfn(ghosted_error_gfn, "ghosted_error.dat");
	}

else if (STRING_EQUAL(which_test, "ghost zone Jacobian"))
   then {
	verify_gpn(ps);
	test_ghost_zone_Jacobians(ps,
				   test_fn_gfn, test_fn_copy_gfn,
				   NP_Jacobian__perturbation_amplitude,
				 (NP_Jacobian__perturb_all_y_patch_points != 0),
				   Jacobian_file_name);
	}

else if (STRING_EQUAL(which_test, "derivatives"))
   then {
	verify_gpn(ps);
	setup_fn_rho_sigma(ps, test_fn_gfn);
	ps.print_ghosted_gridfn(test_fn_gfn, "test_fn.dat");
	finite_diff(ps, test_fn_gfn, FD_derivs_gfn, which_derivs);
	ps.print_gridfn(FD_derivs_gfn, "FD_derivs.dat");
	analytic_derivs(ps, analytic_derivs_gfn, which_derivs);
	ps.print_gridfn(analytic_derivs_gfn, "analytic_derivs.dat");
	gridfn_minus(ps,
		     FD_derivs_gfn, analytic_derivs_gfn,
		     nominal_error_gfn);
	ps.print_gridfn(nominal_error_gfn, "nominal_error.dat");
	}

else	CCTK_VWarn(FATAL_ERROR, __LINE__, __FILE__, CCTK_THORNSTRING,
		   "unknown which_test=\"%s\"!",
		   which_test);					/*NOTREACHED*/

CCTK_VInfo(CCTK_THORNSTRING, "destroying patch system");
}

//******************************************************************************
//******************************************************************************
//******************************************************************************

//
// This function tests the computation of the Jacobian of the
// ghost_zone::synchronize() operation.  In outline, it does the following:
//
//	set up a test function in test_gfn on the nominal grid 
//	synchronize test_gfn
//	print ths synchronized function
//	set up the same test function in NP_test_gfn on the nominal grid
//		for each patch xp and ghost zone xgz
//		{
//		compute the synchronize() Jacobian for this ghost zone
//		   (via  xgz.compute_Jacobian()  et al)
//			for each point x in (p,xgz)
//			{
//				for each point y in xgz.Jacobian_y_patch()
//				{
//				if ( !perturb_all_y_patch_points
//				     && (y iperp != xgz Jacobian y iperp) )
//				   then continue;	// *** LOOP CONTROL ***
//				const fp save_y_gridfn = NP_test_gfn at y
//				NP_test_gfn at y += perturbation_amplitude
//				synchronize NP_test_gfn
//				NP_Jacobian = (NP_test_gfn at x - test_gfn at x)
//					      / perturbation_amplitude
//				NP_test_gfn at y = save_y_gridfn
//				Jacobian = xgz.Jacobian(...)
//				print Jacobians to output file
//				if (the Jacobians differ)
//				   then {
//					print lots of debugging info to stdout
//					goto done;
//					}
//				}
//			}
//		}
//	done:
//
// Note that this test can be *very* slow to run!  Eg. on a 731 MHz PIII
// laptop, a full-sphere patch system with 5 degree resolution, a ghost zone
// iperp width of 2 points, and perturb_all_y_patch_points set to true,
// took around 25 minutes of cpu time, and produced an 20 MB Jacobian
// output file.  Even with perturb_all_y_patch_points set to false, it
// took 1 minute 15 seconds cpu time, and produced a 1 MB Jacobian output
// file.
//
// Arguments:
// ps = (in out) THe patch system in/on which to do the computations.
// test_gfn, NP_test_gfn = The gfns of two ghosted test gridfns.
// perturbation_amplitude = The perturbation amplitude for the NP Jacobian.
// perturb_all_y_patch_points
//	= true  ==> When computing the NP Jacobian, try perturbing at
//		    every point in the y patch.  This gives 
//	  false ==> Only try perturbing at y_iper == Jacobian_y_iperp.
// Jacobian_file_name = The name of the output file to which both Jacobians
//			should be written.
//
namespace {
void test_ghost_zone_Jacobians(patch_system& ps,
				int test_gfn, int NP_test_gfn,
				fp perturbation_amplitude,
				bool perturb_all_y_patch_points,
				const char Jacobian_file_name[])
{
CCTK_VInfo(CCTK_THORNSTRING, "testing ghost zone synchronize() Jacobian...");

setup_sym_fn_xyz(ps, test_gfn, false);
ps.synchronize(test_fn_gfn, test_fn_gfn);
ps.print_ghosted_gridfn(test_fn_gfn, "test_fn.dat");

setup_sym_fn_xyz(ps, NP_test_gfn, false);

FILE *fileptr = fopen(Jacobian_file_name, "w");
if (fileptr == NULL)
   then CCTK_VWarn(FATAL_ERROR, __LINE__, __FILE__, CCTK_THORNSTRING,
"test_ghost_zone_Jacobians(): can't open output plot file \"%s\"!",
		   Jacobian_file_name);				/*NOTREACHED*/
fprintf(fileptr, "# column  1 = x patch number\n");
fprintf(fileptr, "# column  2 = x ghost_zone is_min()\n");
fprintf(fileptr, "# column  3 = x ghost_zone is_rho()\n");
fprintf(fileptr, "# column  4 = x_iperp\n");
fprintf(fileptr, "# column  5 = x_ipar\n");
fprintf(fileptr, "# column  6 = x_irho\n");
fprintf(fileptr, "# column  7 = x_isigma\n");
fprintf(fileptr, "# column  8 = y patch number\n");
fprintf(fileptr, "# column  9 = y ghost_zone is_min()\n");
fprintf(fileptr, "# column 10 = y ghost_zone is_rho()\n");
fprintf(fileptr, "# column 11 = y_iperp\n");
fprintf(fileptr, "# column 12 = y_ipar\n");
fprintf(fileptr, "# column 13 = y_irho\n");
fprintf(fileptr, "# column 14 = y_isigma\n");
fprintf(fileptr, "# column 15 = Jacobian\n");
fprintf(fileptr, "# column 16 = NP_Jacobian\n");
fprintf(fileptr, "# column 17 = Jacobian error\n");

    //*** for each patch p and ghost zone xgz
    for (int xpn = 0 ; xpn < ps.N_patches() ; ++xpn)
    {
    patch& xp = ps.ith_patch(xpn);

    // n.b. these loops must use _int_ variables for the loop
    //      to terminate!
    for (int want_min = false ; want_min <= true ; ++want_min)
    {
    for (int want_rho = false ; want_rho <= true ; ++want_rho)
    {
    const patch_edge& xe = xp.minmax_ang_patch_edge(want_min, want_rho);
    ghost_zone& xgz      = xp.minmax_ang_ghost_zone(want_min, want_rho);

    patch&            yp = xgz.Jacobian_y_patch();
    const patch_edge& ye = xgz.Jacobian_y_edge();

    CCTK_VInfo(CCTK_THORNSTRING,
	       "   testing x patch %s, edge %s",
	       xp.name(), xe.name());
    CCTK_VInfo(CCTK_THORNSTRING,
	       "           y patch %s, edge %s",
	       yp.name(), ye.name());

    xgz.compute_Jacobian(test_gfn, test_gfn);
    const int Jacobian_min_y_ipar_m = xgz.Jacobian_min_y_ipar_m();
    const int Jacobian_max_y_ipar_m = xgz.Jacobian_max_y_ipar_m();

	//*** for each point x in (p,xgz)
	for (int x_iperp = xgz.min_iperp() ;
	     x_iperp <= xgz.max_iperp() ;
	     ++x_iperp)
	{
	for (int x_ipar = xgz.min_ipar(x_iperp) ;
	     x_ipar <= xgz.max_ipar(x_iperp) ;
	     ++x_ipar)
	{
	const int x_irho   = xe.  irho_of_iperp_ipar(x_iperp, x_ipar);
	const int x_isigma = xe.isigma_of_iperp_ipar(x_iperp, x_ipar);

	const int Jacobian_y_iperp = xgz.Jacobian_y_iperp(x_iperp);
	const int Jacobian_y_ipar_posn
		= xgz.Jacobian_y_ipar_posn(x_iperp, x_ipar);

	    //*** for each point y in gz.Jacobian_patch()
	    for (int y_irho = yp.min_irho() ;
		     y_irho <= yp.max_irho() ;
		     ++y_irho)
	    {
	    for (int y_isigma = yp.min_isigma() ;
		 y_isigma <= yp.max_isigma() ;
		 ++y_isigma)
	    {
	    const int y_iperp = ye.iperp_of_irho_isigma(y_irho,y_isigma);
	    const int y_ipar  = ye. ipar_of_irho_isigma(y_irho,y_isigma);

	    if ( !perturb_all_y_patch_points && (y_iperp != Jacobian_y_iperp) )
	       then continue;				// *** LOOP CONTROL ***

	    // compute the NP Jacobian
	    const fp save_y_gridfn
		= yp.ghosted_gridfn(NP_test_gfn, y_irho,y_isigma);
	    yp.ghosted_gridfn(NP_test_gfn, y_irho,y_isigma)
		+= perturbation_amplitude;

	    ps.synchronize(NP_test_gfn, NP_test_gfn);
	    const fp NP_Jacobian
		= (   xp.ghosted_gridfn(NP_test_gfn, x_irho,x_isigma)
		    - xp.ghosted_gridfn(   test_gfn, x_irho,x_isigma) )
		  / perturbation_amplitude;
	    yp.ghosted_gridfn(NP_test_gfn, y_irho,y_isigma) = save_y_gridfn;

	    // compute the query Jacobian
	    const int y_ipar_m = y_ipar - Jacobian_y_ipar_posn;
	    const bool m_in_molecule
		= (y_iperp == Jacobian_y_iperp)
		  && (y_ipar_m >= Jacobian_min_y_ipar_m)
		  && (y_ipar_m <= Jacobian_max_y_ipar_m);
	    const fp Jacobian = m_in_molecule
				? xgz.Jacobian(x_iperp, x_ipar, y_ipar_m)
				: 0.0;

	    // print the results
	    const fp error = Jacobian - NP_Jacobian;
	    fprintf(fileptr,
"%d %d %d\t%d %d\t%d %d\t%d %d %d\t%d %d\t%d %d\t%.10g\t%.10g\t%e\n",
		    xp.patch_number(), xe.is_min(), xe.is_rho(),
		    x_iperp, x_ipar, x_irho, x_isigma,
		    yp.patch_number(), ye.is_min(), ye.is_rho(),
		    y_iperp, y_ipar, y_irho, y_isigma,
		    double(Jacobian), double(NP_Jacobian),
		    double(error));

	    // debugging code in case the Jacobian is wrong :(
	    if (jtutil::abs(error) > 1.0e-6)
	       then {
		    printf("### large Jacobian error!\n");

		    printf("x: p patch %s, edge %s\n", xp.name(), xe.name());
		    printf("y: q patch %s, edge %s\n", yp.name(), ye.name());

		    const fp x_rho   = xp.rho_of_irho    (x_irho);
		    const fp x_sigma = xp.sigma_of_isigma(x_isigma);
		    const fp y_rho   = yp.rho_of_irho    (y_irho);
		    const fp y_sigma = yp.sigma_of_isigma(y_isigma);

		    const fp x_drho   = jtutil::degrees_of_radians(x_rho);
		    const fp x_dsigma = jtutil::degrees_of_radians(x_sigma);
		    const fp y_drho   = jtutil::degrees_of_radians(y_rho);
		    const fp y_dsigma = jtutil::degrees_of_radians(y_sigma);

		    printf(
"x iperp=%d ipar=%d   irho=%d isigma=%d   drho=%g dsigma=%g\n",
			   x_iperp, x_ipar, x_irho, x_isigma, x_drho, x_dsigma);
		    printf(
"y iperp=%d ipar=%d   irho=%d isigma=%d   drho=%g dsigma=%g\n",
			   y_iperp, y_ipar, y_irho, y_isigma, y_drho, y_dsigma);

		    const fp x_mu  = xp.mu_of_rho_sigma (x_rho, x_sigma);
		    const fp x_nu  = xp.nu_of_rho_sigma (x_rho, x_sigma);
		    const fp x_phi = xp.phi_of_rho_sigma(x_rho, x_sigma);
		    const fp y_mu  = yp.mu_of_rho_sigma (y_rho, y_sigma);
		    const fp y_nu  = yp.nu_of_rho_sigma (y_rho, y_sigma);
		    const fp y_phi = yp.phi_of_rho_sigma(y_rho, y_sigma);

		    const fp x_dmu  = jtutil::degrees_of_radians(x_mu);
		    const fp x_dnu  = jtutil::degrees_of_radians(x_nu);
		    const fp x_dphi = jtutil::degrees_of_radians(x_phi);
		    const fp y_dmu  = jtutil::degrees_of_radians(y_mu);
		    const fp y_dnu  = jtutil::degrees_of_radians(y_nu);
		    const fp y_dphi = jtutil::degrees_of_radians(y_phi);
		
		    printf("x dmu=%g dnu=%g dphi=%g\n", x_dmu, x_dnu, x_dphi);
		    printf("y dmu=%g dnu=%g dphi=%g\n", y_dmu, y_dnu, y_dphi);

		    printf("Jacobian=%.10g\tNP_Jacobian=%.10g\terror=%e\n",
			   double(Jacobian), double(NP_Jacobian),
			   double(error));

		    printf("Jacobian_y_[min,max]_ipar_m=[%d,%d]\n",
			   Jacobian_min_y_ipar_m, Jacobian_max_y_ipar_m);
		    printf("Jacobian_y_iperp=%d Jacobian_y_ipar_posn=%d\n",
			   Jacobian_y_iperp, Jacobian_y_ipar_posn);
		    printf("y_ipar_m=%d\n", y_ipar_m);

		    printf("###\n");
		    goto done;
		    }
	    }
	    }
	}
	}
    }
    }

    }

done:
fclose(fileptr);
}
	  }

//******************************************************************************
//******************************************************************************
//******************************************************************************

//
// This function verifies the nominal- and ghosted-grid
// gpn <--> (patch,irho,isigma) conversions.
//
namespace {
void verify_gpn(const patch_system& ps)
{
CCTK_VInfo(CCTK_THORNSTRING,
	   "testing gpn <--> (patch,irho,isigma) conversions");
int gpn = 0;
int ghosted_gpn = 0;

	for (int pn = 0 ; pn < ps.N_patches() ; ++pn)
	{
	const patch& p = ps.ith_patch(pn);

	// nominal grid
		  {
		for (int irho = p.min_irho() ; irho <= p.max_irho() ; ++irho)
		{
		for (int isigma = p.min_isigma() ;
		     isigma <= p.max_isigma() ;
		     ++isigma)
		{
		assert( ps.gpn_of_patch_irho_isigma(p,irho,isigma) == gpn );
		int gpn_irho, gpn_isigma;
		const patch& gpn_p
		   = ps.patch_irho_isigma_of_gpn(gpn, gpn_irho, gpn_isigma);
		assert( gpn_p      == p      );
		assert( gpn_irho   == irho   );
		assert( gpn_isigma == isigma );

		++gpn;
		}
		}
		  }

	// ghosted grid
		  {
		for (int irho = p.ghosted_min_irho() ;
		     irho <= p.ghosted_max_irho() ;
		     ++irho)
		{
		for (int isigma = p.ghosted_min_isigma() ;
		     isigma <= p.ghosted_max_isigma() ;
		     ++isigma)
		{
		assert( ps.ghosted_gpn_of_patch_irho_isigma(p,irho,isigma)
			== ghosted_gpn );
		int gpn_irho, gpn_isigma;
		const patch& gpn_p
		    = ps.ghosted_patch_irho_isigma_of_gpn(ghosted_gpn,
							  gpn_irho, gpn_isigma);
		assert( gpn_p      == p      );
		assert( gpn_irho   == irho   );
		assert( gpn_isigma == isigma );

		++ghosted_gpn;
		}
		}
		  }
	}

assert(         gpn == ps.        N_grid_points() );
assert( ghosted_gpn == ps.ghosted_N_grid_points() );
}
	  }

//******************************************************************************
//******************************************************************************
//******************************************************************************

//
// This function sets up the ghosted test function for the gridfn and
// synchronize tests, symmetrizing the test function to match the
// symmetry of the patch system.
//
// Arguments:
// ps = The patch system.
// ghosted_gfn = Specifies the gridfn to set up.
// want_ghost_zones = true ==> Set up on ghosted part of ghosted grid
//		      false ==> Set up on nominal part of ghosted grid
//
namespace {
void setup_sym_fn_xyz(patch_system& ps, int ghosted_gfn, bool want_ghost_zones)
{
CCTK_VInfo(CCTK_THORNSTRING, "setting up ghosted test fn(x,y,z)");
CCTK_VInfo(CCTK_THORNSTRING, "   on %s part of ghosted grid",
	   want_ghost_zones ? "ghosted" : "nominal");

	for (int pn = 0 ; pn < ps.N_patches() ; ++pn)
	{
	patch& p = ps.ith_patch(pn);

		for (int irho = p.effective_min_irho(want_ghost_zones) ;
		     irho <= p.effective_max_irho(want_ghost_zones) ;
		     ++irho)
		{
		for (int isigma = p.effective_min_isigma(want_ghost_zones) ;
		     isigma <= p.effective_max_isigma(want_ghost_zones) ;
		     ++isigma)
		{
		const fp rho = p.rho_of_irho(irho);
		const fp sigma = p.sigma_of_isigma(isigma);

		fp local_x, local_y, local_z;
		p.xyz_of_r_rho_sigma(1.0, rho, sigma,
				     local_x, local_y, local_z);

		p.ghosted_gridfn(ghosted_gfn, irho,isigma)
			= sym_fn_xyz(ps.type(), local_x, local_y, local_z);
		}
		}
	}
}
	  }

//******************************************************************************
//******************************************************************************
//******************************************************************************

//
// This function sets up the test function for the finite differencing
// tests.
//
// Arguments:
// ps = The patch system.
// ghosted_gfn = Specifies the gridfn to set up.
//
namespace {
void setup_fn_rho_sigma(patch_system& ps, int ghosted_gfn)
{
CCTK_VInfo(CCTK_THORNSTRING, "setting up ghosted test fn(rho,sigma)");

	for (int pn = 0 ; pn < ps.N_patches() ; ++pn)
	{
	patch& p = ps.ith_patch(pn);

		for (int irho = p.ghosted_min_irho() ;
		     irho <= p.ghosted_max_irho() ;
		     ++irho)
		{
		for (int isigma = p.ghosted_min_isigma() ;
		     isigma <= p.ghosted_max_isigma() ;
		     ++isigma)
		{
		const fp rho = p.rho_of_irho(irho);
		const fp sigma = p.sigma_of_isigma(isigma);

		p.ghosted_gridfn(ghosted_gfn, irho,isigma)
			= fn_rho_sigma(rho, sigma);
		}
		}
	}
}
	  }

//******************************************************************************

//
// This function computes (on the nominal grid only) the specified
// linear combination of finite derivatives of a test function.
//
// Arguments:
// ps = The patch system.
// gfn_src = Specifies the gridfn to finite difference.
// gfn_dst = Specifies the gridfn in which to store the result.
// which = Specifies which finite derivatives to include in the test.
//
namespace {
void finite_diff(patch_system& ps,
		 int ghosted_gfn_src, int gfn_dst,
		 int which_derivs)
{
CCTK_VInfo(CCTK_THORNSTRING,
	   "finite differencing ghosted_gfn=%d --> gfn=%d (which_derivs=%d)",
	   ghosted_gfn_src, gfn_dst, which_derivs);

	for (int pn = 0 ; pn < ps.N_patches() ; ++pn)
	{
	patch& p = ps.ith_patch(pn);

	for (int irho = p.min_irho() ; irho <= p.max_irho() ; ++irho)
	{
	for (int isigma = p.min_isigma() ; isigma <= p.max_isigma() ; ++isigma)
	{
	p.gridfn(gfn_dst, irho,isigma)
		= finite_diff_fn(p,
				 ghosted_gfn_src, which_derivs,
				 irho,isigma);
	}
	}

	}
}
	  }

//******************************************************************************

//
// This function computes the specified linear combination of
// derivatives of the test function, everywhere on the nominal grid.
//
namespace {
void analytic_derivs(patch_system& ps,
		     int gfn_dst,
		     int which_derivs)
{
CCTK_VInfo(CCTK_THORNSTRING,
	   "computing analytic derivatives (which_derivs=%d)",
	   which_derivs);

	for (int pn = 0 ; pn < ps.N_patches() ; ++pn)
	{
	patch& p = ps.ith_patch(pn);

	for (int irho = p.min_irho() ; irho <= p.max_irho() ; ++irho)
	{
	for (int isigma = p.min_isigma() ; isigma <= p.max_isigma() ; ++isigma)
	{
	const fp rho = p.rho_of_irho(irho);
	const fp sigma = p.sigma_of_isigma(isigma);
	p.gridfn(gfn_dst, irho,isigma)
		= analytic_deriv_fn(rho,sigma, which_derivs);
	}
	}

	}
}
	  }

//******************************************************************************
//******************************************************************************
//******************************************************************************

//
// This function computes [nominal]  gridfn_x - gridfn_y --> gridfn_z .
//
namespace {
void gridfn_minus(patch_system& ps, 
		  int gfn_x, int gfn_y, int gfn_dst)
{
CCTK_VInfo(CCTK_THORNSTRING,
	   "[nominal] gfn=%d - gfn=%d --> gfn=%d",
	   gfn_x, gfn_y, gfn_dst);

	for (int pn = 0 ; pn < ps.N_patches() ; ++pn)
	{
	patch& p = ps.ith_patch(pn);

	for (int irho = p.min_irho() ; irho <= p.max_irho() ; ++irho)
	{
	for (int isigma = p.min_isigma() ; isigma <= p.max_isigma() ; ++isigma)
	{
	p.gridfn(gfn_dst, irho,isigma)
		=   p.gridfn(gfn_x, irho,isigma)
		  - p.gridfn(gfn_y, irho,isigma);
	}
	}

	}
}
	  }

//******************************************************************************

//
// This function computes [ghosted]  gridfn_x - gridfn_y --> gridfn_z .
//
namespace {
void ghosted_gridfn_minus(patch_system& ps, 
			  int ghosted_gfn_x, int ghosted_gfn_y,
			  int ghosted_gfn_dst)
{
CCTK_VInfo(CCTK_THORNSTRING,
	   "[ghosted] gfn=%d - gfn=%d --> gfn=%d",
	   ghosted_gfn_x, ghosted_gfn_y, ghosted_gfn_dst);

	for (int pn = 0 ; pn < ps.N_patches() ; ++pn)
	{
	patch& p = ps.ith_patch(pn);

		for (int irho = p.ghosted_min_irho() ;
		     irho <= p.ghosted_max_irho() ;
		     ++irho)
		{
		for (int isigma = p.ghosted_min_isigma() ;
		     isigma <= p.ghosted_max_isigma() ;
		     ++isigma)
		{
		p.ghosted_gridfn(ghosted_gfn_dst, irho,isigma)
			=   p.ghosted_gridfn(ghosted_gfn_x, irho,isigma)
			  - p.ghosted_gridfn(ghosted_gfn_y, irho,isigma);
		}
		}

	}
}
	  }

//******************************************************************************
//******************************************************************************
//******************************************************************************

//
// This function symmetrizes  fn_xyz()  (about the origin) to match
// the patch system's symmetries.
//
// To rotate f(x,y) by 90, 180, or 270 degrees:
//
//	      (-y,x) |
//	             |
//	             |
//	             |
//	             |        (x,y)
//	             |
//	-------------+-------------
//	             |
//	(-x,-y)      |
//	             |
//	             |
//	             |
//	             | (y,-x)
//
namespace {
fp sym_fn_xyz(enum patch_system::patch_system_type type, fp x, fp y, fp z)
{
switch	(type)
	{
case patch_system::full_sphere_patch_system:
	return fn_xyz(x,y,z);
	break;
case patch_system::plus_z_hemisphere_patch_system:
	return fn_xyz(x,y,+z) + fn_xyz(x,y,-z);
	break;
case patch_system::plus_xy_quadrant_patch_system:
	return   fn_xyz(+x,+y,z)
	       + fn_xyz(-y,+x,z)
	       + fn_xyz(-x,-y,z)
	       + fn_xyz(+y,-x,z);
	break;
case patch_system::plus_xz_quadrant_patch_system:
	return   fn_xyz(+x,+y,+z) + fn_xyz(-x,-y,+z)
	       + fn_xyz(+x,+y,-z) + fn_xyz(-x,-y,-z);
	break;
case patch_system::plus_xyz_octant_patch_system:
	return   fn_xyz(+x,+y,+z) + fn_xyz(+x,+y,-z)
	       + fn_xyz(-y,+x,+z) + fn_xyz(-y,+x,-z)
	       + fn_xyz(-x,-y,+z) + fn_xyz(-x,-y,-z)
	       + fn_xyz(+y,-x,+z) + fn_xyz(+y,-x,-z);
	break;
default:
	error_exit(PANIC_EXIT,
"***** sym_fn_xyz(): impossible type=(int)%d!\n",
		   int(type));					/*NOTREACHED*/
			}
}
	  }

//******************************************************************************

//
// This is the underlying test function for our function and ghost-zone tests.
//
namespace {
fp fn_xyz(fp x, fp y, fp z)
{
return (x*(x+0.238) + 2.417*y*(y-0.917) + 1.38*z*(z-0.472))
       * tanh(jtutil::pow3(cos(z)));
}
	  }

//******************************************************************************
//******************************************************************************
//******************************************************************************

//
// This is the underlying test function for our finite differencing
// tests.
//
namespace {
fp fn_rho_sigma(fp rho, fp sigma)
{
return exp(sin(1.38*rho)) * tanh(0.17+0.83*jtutil::pow2(sin(sigma)));
}
	  }

//******************************************************************************

//
// This function computes the sum of our linear combination of various
// finite difference approximations to derivatives of fn_rho_sigma().
//
// Arguments:
// ghosted_gfn_src = Specifies the gridfn to finite difference.
//
namespace {
fp finite_diff_fn(const patch& p,
		  int ghosted_gfn_src, int which_derivs,
		  int irho, int isigma)
{
fp sum = 0.0;

if (which_derivs & which_deriv_fn)
   then sum += deriv_weight_fn
	       * p.ghosted_gridfn(ghosted_gfn_src, irho,isigma);

if (which_derivs & which_deriv_rho)
   then sum += deriv_weight_rho
	       * p.partial_rho(ghosted_gfn_src, irho,isigma);
if (which_derivs & which_deriv_sigma)
   then sum += deriv_weight_sigma
	       * p.partial_sigma(ghosted_gfn_src, irho,isigma);

if (which_derivs & which_deriv_rho_rho)
   then sum += deriv_weight_rho_rho
	       * p.partial_rho_rho(ghosted_gfn_src, irho,isigma);
if (which_derivs & which_deriv_rho_sigma)
   then sum += deriv_weight_rho_sigma
	       * p.partial_rho_sigma(ghosted_gfn_src, irho,isigma);
if (which_derivs & which_deriv_sigma_sigma)
   then sum += deriv_weight_sigma_sigma
	       * p.partial_sigma_sigma(ghosted_gfn_src, irho,isigma);

return sum;
}
	  }

//******************************************************************************

//
// This function computes the sum of a specified linear combination of
// various (analytical) derivatives of fn_rho_sigma().
//
// The derivatives were machine-generated via Maple's codegen[C]() function:
// "deriv_patch_system.maple" is the Maple input
// "deriv_patch_system.out" is the Maple input; code here is cut-n-pasted
//			   from the Maple codegen[C]() output there
//
namespace {
fp analytic_deriv_fn(fp rho, fp sigma, int which_derivs)
{
fp sum = 0.0;

if (which_derivs & which_deriv_fn)
   then sum += deriv_weight_fn
	       * fn_rho_sigma(rho,sigma);

if (which_derivs & which_deriv_rho)
   then sum += deriv_weight_rho
	       * 0.138E1*cos(0.138E1*rho)*exp(sin(0.138E1*rho))
		 *tanh(0.17+0.83*pow(sin(sigma),2.0));
if (which_derivs & which_deriv_sigma)
   then sum += deriv_weight_sigma
	       * 0.166E1*exp(sin(0.138E1*rho))
		 *(1.0-pow(tanh(0.17+0.83*pow(sin(sigma),2.0)),2.0))
		 *sin(sigma)*cos(sigma);

if (which_derivs & which_deriv_rho_rho)
   then sum += deriv_weight_rho_rho
	       * (
		 -0.19044E1*sin(0.138E1*rho)*exp(sin(0.138E1*rho))
		  *tanh(0.17+0.83*pow(sin(sigma),2.0))
		 +0.19044E1*pow(cos(0.138E1*rho),2.0)*exp(sin(0.138E1*rho))
		  *tanh(0.17+0.83*pow(sin(sigma),2.0))
		 );
if (which_derivs & which_deriv_rho_sigma)
   then sum += deriv_weight_rho_sigma
	       * 0.22908E1*cos(0.138E1*rho)*exp(sin(0.138E1*rho))
		 *(1.0-pow(tanh(0.17+0.83*pow(sin(sigma),2.0)),2.0))
		 *sin(sigma)*cos(sigma);
if (which_derivs & which_deriv_sigma_sigma)
   then sum += deriv_weight_sigma_sigma
	       * (
		 -0.55112E1*exp(sin(0.138E1*rho))
		  *tanh(0.17+0.83*pow(sin(sigma),2.0))
		  *(1.0-pow(tanh(0.17+0.83*pow(sin(sigma),2.0)),2.0))
		  *pow(sin(sigma),2.0)*pow(cos(sigma),2.0)
		 +0.166E1*exp(sin(0.138E1*rho))
		  *(1.0-pow(tanh(0.17+0.83*pow(sin(sigma),2.0)),2.0))
		  *pow(cos(sigma),2.0)
		 -0.166E1*exp(sin(0.138E1*rho))
		  *(1.0-pow(tanh(0.17+0.83*pow(sin(sigma),2.0)),2.0))
		  *pow(sin(sigma),2.0)
		 );

return sum;
}
	  }

```

patch/coord_derivs.maple:
```
# coord_derivs.maple -- compute derivatives of (mu,nu,phi) wrt (x,y,z)
# $Header$

mu  := arctan(y/z);
nu  := arctan(x/z);
phi := arctan(y/x);

########################################

simplify(diff(mu,x));
simplify(diff(mu,y));
simplify(diff(mu,z));

simplify(diff(nu,x));
simplify(diff(nu,y));
simplify(diff(nu,z));

simplify(diff(phi,x));
simplify(diff(phi,y));
simplify(diff(phi,z));

########################################

simplify(diff(mu,x,x));
simplify(diff(mu,x,y));
simplify(diff(mu,x,z));
simplify(diff(mu,y,y));
simplify(diff(mu,y,z));
simplify(diff(mu,z,z));

simplify(diff(nu,x,x));
simplify(diff(nu,x,y));
simplify(diff(nu,x,z));
simplify(diff(nu,y,y));
simplify(diff(nu,y,z));
simplify(diff(nu,z,z));

simplify(diff(phi,x,x));
simplify(diff(phi,x,y));
simplify(diff(phi,x,z));
simplify(diff(phi,y,y));
simplify(diff(phi,y,z));
simplify(diff(phi,z,z));

```

patch/patch_interp.hh:
```
// patch_interp.hh -- interpolation from a patch
// $Header$

//
// prerequisites:
//	<stdio.h>
//	<assert.h>
//	<math.h>
//
//	"cctk.h"
//
//	"stdc.h"
//	"config.hh"
//	"../jtutil/util.hh"
//	"../jtutil/array.hh"
//	"../jtutil/linear_map.hh"
//	"coords.hh"
//	"grid.hh"
//	"fd_grid.hh"
//	"patch.hh"
//

// everything in this file is inside this namespace
namespace AHFinderDirect
	  {

//*****************************************************************************

//
// patch_interp - interpolation from a patch
//

//
// A patch_interp object is responsible for interpolating gridfn data
// from its owning patch for use by another patch's ghost_zone object
// (in setting up the gridfn in the other ghost zone).  A patch_interp
// object deals only in its own patch's coordinates; other code elsewhere
// (in practice in interpatch_ghost_zone::) is responsible for translating
// other patch's coordinates into our coordinates.
//

//
// A patch_interp defines a "patch interpolation region", the region of
// its owning patch from which this interpolation will use gridfn data.
//

//
// The way the patch coordnates are constructed, any two adjacent patches
// share a common (perpendicular) coordinate.  Thus we only have to do
// 1-dimensional interpolation here (in the parallel direction).  In
// other words, for each iperp we interpolate in par.
//
// In general we interpolate each gridfn at a number of distinct par
// for each iperp; the integer "parindex" indexes these points.  We
// attach no particular semantics to parindex, and it need not be
// 0-origin or have the same range for each iperp.  [In practice,
// parindex will be the other patch's ipar coordinate.]  However,
// we assume that the range of parindex is roughly similar for each
// iperp, so it's ok to use (iperp,parindex) as a 2-D rectangular
// index space.
//
// For example, we might interpolate at the points
//            ipar ipar ipar ipar ipar ipar ipar ipar ipar
//              1    2    3    4    5    6    7    8    9
// iperp=10           (2a)   (3b)   (4c)
// iperp=11          (2d)   (3e)  (4f)   (5g)
// where the (2a)-(5g) are the interpolation points, with 2-5 being the
// parindex values and a-g being unique identifiers used in our description
// below.  In terms of our member data, this interpolation region would
// be described by
//	[min,max]_iperp_=[10,11]
//	[min,max]_ipar_=[1,9]
//	[min,max]_parindex_array_(10)=[2,5]
//	[min,max]_parindex_array_(11)=[2,6]
//	interp_par_(10,2) = x[a]
//	interp_par_(10,3) = x[b]
//	interp_par_(10,4) = x[c]
//	interp_par_(11,2) = x[d]
//	interp_par_(11,3) = x[e]
//	interp_par_(11,4) = x[f]
//	interp_par_(11,5) = x[g]
//

//
// We use the Cactus local interpolator CCTK_InterpLocalUniform()
// to do the interpolation.  To minimize interpolator overheads, we
// interpolate all the gridfns at each iperp in a single interpolator
// call.  [Different iperp values involve different sets of (1-D)
// gridfn data, and so inherently require distinct interpolator calls.]
//
// Setting up the array subscripting for the interpolator to access
// the gridfn data is a bit tricky:  The interpolator accesses the
// gridfn data using the generic (1-D) subscripting expression
//	data[offset + i*stride]
// where  i  is the data array index.  However, we'd rather not use
//  offset , because it has to be supplied in the parameter table as
// an array subscripted by  gfn , and so would require changing the
// parameter table for each call on  interpolate()  (with potentially
// different numbers of gridfns being interpolated).  Instead, at each
//  iperp  we use  i = ipar-min_ipar , so the default  offset=0  makes
// the subscripting expression zero for  ipar = min_ipar .  This also
// makes the interpolator's  min_i = 0  and  max_i  be  dims-1  (both
// the defaults), so those also don't have to be set in the parameter
// table either.  We set the interpolator's data coordinate origin to
// the  par  coordinate for  min_ipar , so it correctly maps  i --> par .
// With this strategy we can share the interpolator parameter table
// across all the  iperp  values, and we don't need to modify the
// parameter table at all after the initial setup in our constructor.
// However, we do have to adjust the molecule positions in
//  patch_interp::molecule_posn() , since the interpolator will return
//  i  values, while  molecule_posn()  needs  ipar  values.
//

class	patch_interp
	{
public:
	// to which patch/edge do we belong?
	const patch& my_patch() const { return my_patch_; }
	const patch_edge& my_edge() const { return my_edge_; }


public:
	//
	// ***** main client interface *****
	//
	// interpolate specified range of ghosted gridfns
	// at all the coordinates specified when we were constructed,
	// store interpolated data in
	//	data_buffer(ghosted_gfn, iperp, parindex)
	void interpolate(int ghosted_min_gfn_to_interp,
			 int ghosted_max_gfn_to_interp,
			 jtutil::array3d<fp>& data_buffer)
		const;

public:
	//
	// ***** Jacobian of interpolate() *****
	//

	// verify (no-op if ok, error_exit() if not) that interpolator
	// has a Jacobian sparsity pattern which we grok: at present this
	// means molecules are fixed-sized hypercubes, with size/shape
	// independent of interpolation coordinates and the floating-point
	// values in the input arrays
	void verify_Jacobian_sparsity_pattern_ok() const;

	//
	// The API for the remaining Jacobian functions implicitly
	// assumes that the Jacobian sparsity pattern is "ok" as
	// verified by  verify_Jacobian_sparsity_pattern_ok() ,
	// and in particular that  [min,max]_ipar_m  are independent
	// of iperp and parindex.
	//

	// get [min,max] ipar m coordinates of interpolation molecules
	void molecule_minmax_ipar_m(int& min_ipar_m, int& max_ipar_m) const;

	// get interpolation molecule ipar positions in
	//  molecule_posn_buffer(iperp, parindex)
	// ... array type is CCTK_INT so we can pass by reference
	//     to interpolator
	void molecule_posn(jtutil::array2d<CCTK_INT>& posn_buffer) const;

	// get Jacobian of interpolated data with respect to this patch's
	// ghosted gridfns,
	//	partial interpolate() data_buffer(ghosted_gfn, iperp, parindex)
	//	---------------------------------------------------------------
	//	    partial ghosted_gridfn(ghosted_gfn, iperp, posn+ipar_m)
	// store Jacobian in
	//	Jacobian_buffer(iperp, parindex, ipar_m)
	// where we implicitly assume the Jacobian to be independent of
	// ghosted_gfn, and where
	//	posn = posn_buffer(iperp, parindex)
	// as returned by  molecule_posn()
	void Jacobian(jtutil::array3d<fp>& Jacobian_buffer) const;

	//
	// ***** internal functions *****
	//
private:
	// [min,max] iperp for interpolation and gridfn data
	int min_iperp() const { return min_iperp_; }
	int max_iperp() const { return max_iperp_; }

	// min/max (iperp,ipar) of the gridfn data to use for interpolation
	int min_ipar() const { return min_ipar_; }
	int max_ipar() const { return max_ipar_; }

	// query the interpolator via the parameter table
	// on behalf of the specified function (name used for error msgs only)
	// ... error_exit() if error return from interpolator,
	//     otherwise return interpolator status code
	int query_interpolator(const char function_name[], int iperp)
		const;
	// ... use default iperp for queries that don't care
	int query_interpolator(const char function_name[])
		const
		{ return query_interpolator(function_name, min_iperp_); }


	//
	// ***** constructor, destructor, et al *****
	//
public:
	//
	// Constructor arguments:
	// my_edge_in = Identifies the patch/edge to which this
	//		interpolation region is to belong.
	// [min,max]_iperp_in = The range of iperp for this interpolation
	//			region
	// [min,max]_parindex_array_in(iperp)
	//	= [min,max] range of parindex actually used at each iperp.
	//	  We keep references to these arrays, so they should have
	//	  lifetimes at last as long as that of this object.
	// interp_par_in(iperp,parindex)
	//	= Gives the par coordinates at which we will interpolate;
	//	  array entries outside the range [min,max]_parindex_in
	//	  are unused.  We keep a reference to this array, so it
	//	  should have a lifetime at last as long as that of this
	//	  object.
	// ok_to_use_[min,max]_par_ghost_zone
	//	= Boolean flags saying whether or not we should use gridfn
	//	  data from the [min,max]_par ghost zones in the interpolation.
	// interp_handle_in = Cactus handle to the interpatch interpolation
	//		      operator.
	// interp_par_table_handle_in
	//	= Cactus handle to a Cactus key/value table giving
	//	  parameters (eg order) for the interpatch interpolation
	//	  operator.  This class internally clones this table and
	//	  modifies the clone, so the original table is not modified
	//	  by any actions of this class.
	//
	// This constructor requires that this patch's gridfns already
	// exist, since we size various arrays based on the patch's min/max
	// ghosted gfn.
	//
	patch_interp(const patch_edge& my_edge_in,
		     int min_iperp_in, int max_iperp_in,
		     const jtutil::array1d<int>& min_parindex_array_in,
		     const jtutil::array1d<int>& max_parindex_array_in,
		     const jtutil::array2d<fp>& interp_par_in,
		     bool ok_to_use_min_par_ghost_zone,
		     bool ok_to_use_max_par_ghost_zone,
		     int interp_handle_in, int interp_par_table_handle_in);
	~patch_interp();

private:
        // we forbid copying and passing by value
        // by declaring the copy constructor and assignment operator
        // private, but never defining them
	patch_interp(const patch_interp& rhs);
	patch_interp& operator=(const patch_interp& rhs);


	//
	// ***** data members *****
	//
private:
	const patch& my_patch_;
	const patch_edge& my_edge_;

	// range of gfn we can handle
	// (any given interpolate() call may specify a subrange)
	const int min_gfn_, max_gfn_;

	// these are strictly speaking redundant
	// but we keep them for use in debugging
	bool ok_to_use_min_par_ghost_zone_, ok_to_use_max_par_ghost_zone_;

	// patch interpolation region,
	// i.e. range of (iperp,ipar) in this patch from which
	// we will use gridfn data in interpolation
	const int min_iperp_, max_iperp_;
	const int min_ipar_, max_ipar_;

	// [min,max] parindex at each iperp
	// ... these are references to arrays passed in to our constructor
	//     ==> we do *not* own them!
	// ... indices are (iperp)
	const jtutil::array1d<int>& min_parindex_array_;
	const jtutil::array1d<int>& max_parindex_array_;

	// interp_par_(iperp,parindex)
	//	= Gives the par coordinates at which we will interpolate;
	//	  array entries outside the range [min,max]_parindex_in
	//	  are unused (n.b. this interface implicitly takes the
	//	  par coordinates to be independent of ghosted_gfn).
	// ... this is a reference to an array passed in to our constructor
	//     ==> we do *not* own this!
	const jtutil::array2d<fp>& interp_par_;	// indices (iperp,parindex)

	// Cactus handle to the interpolation operator
	int interp_handle_;

	// Cactus handle to our private Cactus key/value table
	// giving parameters for the interpolation operator
	// ... this starts out as a copy of the passed-in table,
	//     then gets extra stuff added to it specific to this
	//     interpolation region; it's shared across all iperp
	// ... we own this table
	const int interp_par_table_handle_;

	// (par) origin and delta values of the gridfn data
	const fp gridfn_coord_origin_, gridfn_coord_delta_;

	// array of gridfn type codes for interpolator
	// ... must be CCTK_INT so we can pass by reference to interpolator
	// ... values set in ctor body, const thereafter
	// ... index is (gfn)
	mutable jtutil::array1d<CCTK_INT> gridfn_type_codes_;

	// --> start of gridfn data to use for interpolation
	//     (reset for each iperp)
	// ... we do *not* own the pointed-to data!
	// ... index is (gfn)
	mutable jtutil::array1d<const void*> gridfn_data_ptrs_;

	// --> start of interpolation data buffer for each gridfn
	//     (reset for each iperp)
	// ... we do *not* own the pointed-to data!
	// ... index is (gfn)
	mutable jtutil::array1d<void*> interp_data_buffer_ptrs_;
	};

//******************************************************************************

	  }	// namespace AHFinderDirect

```

patch/patch_system_info.hh:
```
// patch_system_info.hh -- static info describing various types of patch systems
// $Header$

//
// prerequisites:
//    <stdio.h>
//    <assert.h>
//    <math.h>
//    "../jtutil/util.hh"
//    "../jtutil//array.hh"
//    "../jtutil/linear_map.hh"
//    "coords.hh"
//    "grid.hh"
//    "patch_info.hh"
//

// everything in this file is inside this namespace
namespace AHFinderDirect
	  {

//******************************************************************************

//
// This namespace contains static data describing the patch sizes and
// shapes for each type of patch system.  Since this data only describes
// the patch sizes/shapes, we don't distinguish between the different
// boundary conditions.
//

namespace patch_system_info
{
//
// full-sphere patch system
// ... covers all 4pi steradians
//
namespace full_sphere
    {
    enum {
	 patch_number__pz = 0,
	 patch_number__px,
	 patch_number__py,
	 patch_number__mx,
	 patch_number__my,
	 patch_number__mz,
	 N_patches // no comma
	 };
    static const struct patch_info patch_info_array[N_patches]
      = {
	// +z patch (90 x 90 degrees): dmu [ -45,    45], dnu  [ -45,    45]
	 {"+z", patch::patch_is_plus,  'z',  -45.0,  45.0,       -45.0,  45.0},

	// +x patch (90 x 90 degrees): dnu [  45,   135], dphi [ -45,    45]
	 {"+x", patch::patch_is_plus,  'x',   45.0, 135.0,       -45.0,  45.0},

	// +y patch (90 x 90 degrees): dmu [  45,   135], dphi [  45,   135]
	 {"+y", patch::patch_is_plus,  'y',   45.0, 135.0,        45.0, 135.0},

	// -x patch (90 x 90 degrees): dnu [-135,   -45], dphi [ 135,   225]
	 {"-x", patch::patch_is_minus, 'x', -135.0, -45.0,       135.0, 225.0},

	// -y patch (90 x 90 degrees): dmu [-135,   -45], dphi [-135,   -45]
	 {"-y", patch::patch_is_minus, 'y', -135.0, -45.0,      -135.0, -45.0},

	// -z patch (90 x 90 degrees): dmu [ 135,   225], dnu  [ 135,   225]
	 {"-z", patch::patch_is_minus, 'z',  135.0, 225.0,       135.0, 225.0},
	};
    }	// namespace patch_system_info::full_sphere

//
// +z hemisphere (half) patch system
// ... mirror symmetry across z=0 plane
//
namespace plus_z_hemisphere
    {
    enum {
	 patch_number__pz = 0,
	 patch_number__px,
	 patch_number__py,
	 patch_number__mx,
	 patch_number__my,
	 N_patches // no comma
	 };
    static const struct patch_info patch_info_array[N_patches]
      = {
	// +z patch (90 x 90 degrees): dmu [ -45,    45], dnu  [ -45,    45]
	 {"+z", patch::patch_is_plus,  'z',  -45.0,  45.0,       -45.0,  45.0},

	// +x patch (45 x 90 degrees): dnu [  45,    90], dphi [ -45,    45]
	 {"+x", patch::patch_is_plus,  'x',   45.0,  90.0,       -45.0,  45.0},

	// +y patch (45 x 90 degrees): dmu [  45,    90], dphi [  45,   135]
	 {"+y", patch::patch_is_plus,  'y',   45.0,  90.0,        45.0, 135.0},

	// -x patch (45 x 90 degrees): dnu [ -90,   -45], dphi [ 135,   225]
	 {"-x", patch::patch_is_minus, 'x',  -90.0, -45.0,       135.0, 225.0},

	// -y patch (45 x 90 degrees): dmu [ -90,   -45], dphi [-135,   -45]
	 {"-y", patch::patch_is_minus, 'y',  -90.0, -45.0,      -135.0, -45.0},
	};
    }	// namespace patch_system_info::plus_z_hemisphere

//
// +[xy] "vertical" quarter-grid (quadrant) patch system
// two types of boundary conditions:
// ... mirror symmetry across x=0 and y=0 planes
// ... 90 degree periodic rotation symmetry about z axis
//
namespace plus_xy_quadrant
    {
    enum {
	 patch_number__pz = 0,
	 patch_number__px,
	 patch_number__py,
	 patch_number__mz,
	 N_patches // no comma
	 };
    static const struct patch_info patch_info_array[N_patches]
      = {
	// +z patch (45 x 45 degrees): dmu [   0,    45], dnu  [   0,    45]
	 {"+z", patch::patch_is_plus,  'z',  0.0,    45.0,         0.0,  45.0},

	// +x patch (90 x 45 degrees): dnu [  45,   135], dphi [   0,    45]
	 {"+x", patch::patch_is_plus,  'x',  45.0, 135.0,         0.0,  45.0},

	// +y patch (90 x 45 degrees): dmu [  45,   135], dphi [  45,    90]
	 {"+y", patch::patch_is_plus,  'y',  45.0, 135.0,        45.0,  90.0},

	// -z patch (45 x 45 degrees): dmu [ 135,   180], dnu  [ 135,   180]
	 {"-z", patch::patch_is_minus,  'z', 135.0, 180.0,       135.0, 180.0},
	};
    }	// namespace patch_system_info::plus_xy_quadrant

//
// +[xz] "horizontal" quarter-grid (quadrant) patch system
// two types of boundary conditions
// ... mirror symmetry across x=0 plane, z=0 plane
// ... 180 degree periodic rotation symmetry about z axis,
//     mirror symmetry across z=0 plane
//
namespace plus_xz_quadrant
    {
    enum {
	 patch_number__pz = 0,
	 patch_number__px,
	 patch_number__py,
	 patch_number__my,
	 N_patches // no comma
	 };
    static const struct patch_info patch_info_array[N_patches]
      = {
	// +z patch (90 x 45 degrees): dmu [ -45,    45], dnu  [   0,    45]
	 {"+z", patch::patch_is_plus,  'z',  -45.0,  45.0,         0.0,  45.0},

	// +x patch (45 x 90 degrees): dnu [  45,    90], dphi [ -45,    45]
	 {"+x", patch::patch_is_plus,  'x',   45.0,  90.0,       -45.0,  45.0},

	// +y patch (45 x 45 degrees): dmu [  45,    90], dphi [  45,    90]
	 {"+y", patch::patch_is_plus,  'y',   45.0,  90.0,        45.0,  90.0},

	// -y patch (45 x 45 degrees): dmu [ -90,   -45], dphi [ -90,   -45]
	 {"-y", patch::patch_is_minus, 'y',  -90.0, -45.0,       -90.0, -45.0},
	};
    }	// namespace patch_system_info::plus_xz_quadrant_rotating

//
// +[xyz] (octant) patch system
// two types of boundary conditions:
// ... mirror symmetry across x=0 plane, y=0 plane, z=0 plane
// ... 90 degree periodic rotation symmetry about z axis,
//     mirror symmetry across z=0 plane
//
namespace plus_xyz_octant
    {
    enum {
	 patch_number__pz = 0,
	 patch_number__px,
	 patch_number__py,
	 N_patches // no comma
	 };
    static const struct patch_info patch_info_array[N_patches]
      = {
	// +z patch (45 x 45 degrees): dmu [   0,    45], dnu  [   0,    45]
	 {"+z", patch::patch_is_plus,  'z',    0.0,  45.0,         0.0,  45.0},

	// +x patch (45 x 45 degrees): dnu [  45,    90], dphi [   0,    45]
	 {"+x", patch::patch_is_plus,  'x',   45.0,  90.0,         0.0,  45.0},

	// +y patch (45 x 45 degrees): dmu [  45,    90], dphi [  45,    90]
	 {"+y", patch::patch_is_plus,  'y',   45.0,  90.0,        45.0,  90.0},
	};
    }	// namespace patch_system_info::octant_mirrored

	  }	// namespace patch_system_info::

//******************************************************************************

	  }	// namespace AHFinderDirect

```

patch/FIXME:
```
patch_interp::interp_molecule_minmax_mipar() should also query/check
- molecule family
- MSS_is_fn_of_gridfn_values and other new queries


```

patch/test_coords.cc:
```
// test_coords.cc -- test driver for coordinate systems/conversions
// $Header$

#include <stdio.h>
#include <assert.h>
#include <math.h>
#include <string.h>

#ifdef STANDALONE_TEST
  #include "fake_cctk.h"
#else
  #include "cctk.h"
#endif

#include "config.h"
#include "stdc.h"
#include "../jtutil/util.hh"
using jtutil::error_exit;
using jtutil::radians_of_degrees;
using jtutil::degrees_of_radians;

#include "coords.hh"

using namespace AHFinderDirect;
using namespace local_coords;

//******************************************************************************

//
// This program is a test driver for the local_coords:: coordinate-conversion
// functions.  See the help message below for details.
//
int main(int argc, const char* argv[])
{
const char* help_msg =
"Usage:\n"
"n.b. all angles are input/output in degrees!\n"
"   test_coords  [  r=number ]   [ theta=number ]\n"
"                [ mu=number ]   [    nu=number ]   [ phi=number ]\n"
"                [  x=number ]   [     y=number ]   [   z=number ]\n"
;

//
// ***** command line arguments *****
//

if (    (argc == 1)
     || ((argc == 2) && STRING_EQUAL(argv[1], "--help"))    )
   then {
	printf("%s", help_msg);
	return 0;						/*NOTREACHED*/
	}

fp r, theta;
fp mu, nu, phi;
fp x, y, z;
fp dtheta;
fp dmu, dnu, dphi;
bool r_flag = false, theta_flag = false;
bool mu_flag = false, nu_flag = false, phi_flag = false;
bool x_flag = false, y_flag = false, z_flag = false;

	for (int ap = 1 ; ap < argc ; ++ap)
	{
	if	(sscanf(argv[ap], "r=" FP_SCANF_FORMAT, &r) == 1)
	   then r_flag = true;
	else if	(sscanf(argv[ap], "theta=" FP_SCANF_FORMAT, &dtheta) == 1)
	   then {
		theta = radians_of_degrees(dtheta);
		theta_flag = true;
		}

	else if	(sscanf(argv[ap], "mu=" FP_SCANF_FORMAT, &dmu) == 1)
	   then {
		mu = radians_of_degrees(dmu);
		mu_flag = true;
		}
	else if	(sscanf(argv[ap], "nu=" FP_SCANF_FORMAT, &dnu) == 1)
	   then {
		nu = radians_of_degrees(dnu);
		nu_flag = true;
		}
	else if	(sscanf(argv[ap], "phi=" FP_SCANF_FORMAT, &dphi) == 1)
	   then {
		phi = radians_of_degrees(dphi);
		phi_flag = true;
		}

	else if	(sscanf(argv[ap], "x=" FP_SCANF_FORMAT, &x) == 1)
	   then x_flag = true;
	else if	(sscanf(argv[ap], "y=" FP_SCANF_FORMAT, &y) == 1)
	   then y_flag = true;
	else if	(sscanf(argv[ap], "z=" FP_SCANF_FORMAT, &z) == 1)
	   then z_flag = true;

	else	error_exit(ERROR_EXIT, "%s", help_msg);		/*NOTREACHED*/
	}


//
// ***** input coordinates *****
//

printf("input:");
if (r_flag) then printf(" r=%g", r);
if (theta_flag) then printf(" theta=%g", dtheta);
if (mu_flag) then printf(" mu=%g", dmu);
if (nu_flag) then printf(" nu=%g", dnu);
if (phi_flag) then printf(" phi=%g", dphi);
if (x_flag) then printf(" x=%g", x);
if (y_flag) then printf(" y=%g", y);
if (z_flag) then printf(" z=%g", z);
printf("\n");


//
// ***** coordinate conversions *****
//

// ((mu,nu,phi)) --> the 3rd
if (mu_flag && nu_flag)
   then printf("phi_of_mu_nu() ==> phi=%g\n",
	       degrees_of_radians(phi_of_mu_nu(mu,nu)));
if (mu_flag && phi_flag)
   then printf("nu_of_mu_phi() ==> nu=%g\n",
	       degrees_of_radians(nu_of_mu_phi(mu,phi)));
if (nu_flag && phi_flag)
   then printf("mu_of_nu_phi() ==> mu=%g\n",
	       degrees_of_radians(mu_of_nu_phi(nu,phi)));

// (r,(mu,nu,phi)) --> (x,y,z)
if (r_flag && mu_flag && nu_flag)
   then {
	fp x2, y2, z2;
	xyz_of_r_mu_nu(r,mu,nu, x2,y2,z2);
	printf("xyz_of_r_mu_nu() ==> x=%g y=%g z=%g\n", x2,y2,z2);
	}
if (r_flag && mu_flag && phi_flag)
   then {
	fp x2, y2, z2;
	xyz_of_r_mu_phi(r,mu,phi, x2,y2,z2);
	printf("xyz_of_r_mu_phi() ==> x=%g y=%g z=%g\n", x2,y2,z2);
	}
if (r_flag && nu_flag && phi_flag)
   then {
	fp x2, y2, z2;
	xyz_of_r_nu_phi(r,nu,phi, x2,y2,z2);
	printf("xyz_of_r_nu_phi() ==> x=%g y=%g z=%g\n", x2,y2,z2);
	}

// (x,y,z) --> (r,(mu,nu,phi))
if (x_flag && y_flag && z_flag)
   then {
	fp r2 = r_of_xyz(x,y,z);
	fp mu2 = mu_of_yz(y,z);
	fp nu2 = nu_of_xz(x,z);
	fp phi2 = phi_of_xy(x,y);
	printf("*_of_xyz() ==> r=%g mu=%g nu=%g phi=%g\n",
	       r2,
	       degrees_of_radians(mu2),
	       degrees_of_radians(nu2),
	       degrees_of_radians(phi2));
	}

// usual polar spherical (r,theta,phi) <--> (x,y,z)
if (r_flag && theta_flag && phi_flag)
   then {
	fp x2, y2, z2;
	xyz_of_r_theta_phi(r,theta,phi, x2,y2,z2);
	printf("xyz_of_r_theta_phi() ==> x=%g y=%g z=%g\n", x2,y2,z2);
	}
if (x_flag && y_flag && z_flag)
   then {
	fp r2 = r_of_xyz(x,y,z);
	fp theta2 = theta_of_xyz(x,y,z);
	fp phi2 = phi_of_xy(x,y);
	printf("*_of_xyz() ==> r=%g theta=%g phi=%g\n",
	       r2,
	       degrees_of_radians(theta2),
	       degrees_of_radians(phi2));
	}

// ((mu,nu,phi)) --> usual polar spherical (theta,phi)
// ... note phi is the same coordinate in both systems
if (mu_flag && nu_flag)
   then {
	fp ps_theta, ps_phi;
	theta_phi_of_mu_nu(mu,nu, ps_theta,ps_phi);
	printf("theta_phi_of_mu_nu() ==> ps_theta=%g ps_phi=%g\n",
	       degrees_of_radians(ps_theta),degrees_of_radians(ps_phi));
	}
if (mu_flag && phi_flag)
   then {
	fp ps_theta, ps_phi;
	theta_phi_of_mu_phi(mu,phi, ps_theta,ps_phi);
	printf("theta_phi_of_mu_phi() ==> ps_theta=%g ps_phi=%g\n",
	       degrees_of_radians(ps_theta),degrees_of_radians(ps_phi));
	}
if (nu_flag && phi_flag)
   then {
	fp ps_theta, ps_phi;
	theta_phi_of_nu_phi(nu,phi, ps_theta,ps_phi);
	printf("theta_phi_of_nu_phi() ==> ps_theta=%g ps_phi=%g\n",
	       degrees_of_radians(ps_theta),degrees_of_radians(ps_phi));
	}

// usual polar spherical (theta,phi) --> ((mu,nu,phi))
// ... note phi is the same coordinate in both systems
if (theta_flag && phi_flag)
   then {
	fp mu2, nu2;
	mu_nu_of_theta_phi(theta,phi, mu2,nu2);
	printf("mu_nu_of_theta_phi() ==> mu=%g nu=%g\n",
	       degrees_of_radians(mu2),degrees_of_radians(nu2));
	}
if (theta_flag && phi_flag)
   then {
	fp mu2, mnp_phi;
	mu_phi_of_theta_phi(theta,phi, mu2,mnp_phi);
	printf("mu_phi_of_theta_phi() ==> mu=%g mnp_phi=%g\n",
	       degrees_of_radians(mu2),degrees_of_radians(mnp_phi));
	}
if (theta_flag && phi_flag)
   then {
	fp nu2, mnp_phi;
	nu_phi_of_theta_phi(theta,phi, nu2,mnp_phi);
	printf("nu_phi_of_theta_phi() ==> nu=%g mnp_phi=%g\n",
	       degrees_of_radians(nu2),degrees_of_radians(mnp_phi));
	}

return 0;
}

```

patch/test_patch_system.out:
```
    |\^/|     Maple 7 (DEC ALPHA UNIX)
._|\|   |/|_. Copyright (c) 2001 by Waterloo Maple Inc.
 \  MAPLE  /  All rights reserved. Maple is a registered trademark of
 <____ ____>  Waterloo Maple Inc.
      |       Type ? for help.
# test_patch_system.maple -- test function for  test_fd_grid
# $Id: test_patch_system.maple,v 1.4 2001/09/15 18:06:33 jthorn Exp $
> 
# test function
> fn := exp(sin(1.38*rho)) * tanh(0.17+0.83*sin(sigma)^2);
                                                                2
              fn := exp(sin(1.38 rho)) tanh(.17 + .83 sin(sigma) )

> 
# 1st derivatives
> diff(fn, rho);			codegen[C](%);
                                                                      2
       1.38 cos(1.38 rho) exp(sin(1.38 rho)) tanh(.17 + .83 sin(sigma) )

      t0 = 0.138E1*cos(0.138E1*rho)*exp(sin(0.138E1*rho))*tanh(0.17+0.83*pow(
sin(sigma),2.0));
> diff(fn, sigma);		codegen[C](%);
1.66

                                                     2 2
    exp(sin(1.38 rho)) (1 - tanh(.17 + .83 sin(sigma) ) ) sin(sigma) cos(sigma)

      t0 = 0.166E1*exp(sin(0.138E1*rho))*(1.0-pow(tanh(0.17+0.83*pow(sin(sigma)
,2.0)),2.0))*sin(sigma)*cos(sigma);
> 
# 2nd derivatives
> diff(fn, rho,rho);		codegen[C](%);
                                                                  2
-1.9044 sin(1.38 rho) exp(sin(1.38 rho)) tanh(.17 + .83 sin(sigma) )

                           2                                             2
     + 1.9044 cos(1.38 rho)  exp(sin(1.38 rho)) tanh(.17 + .83 sin(sigma) )

      t0 = -0.19044E1*sin(0.138E1*rho)*exp(sin(0.138E1*rho))*tanh(0.17+0.83*pow
(sin(sigma),2.0))+0.19044E1*pow(cos(0.138E1*rho),2.0)*exp(sin(0.138E1*rho))*
tanh(0.17+0.83*pow(sin(sigma),2.0));
> diff(fn, rho,sigma);		codegen[C](%);
                                                                      2 2
2.2908 cos(1.38 rho) exp(sin(1.38 rho)) (1 - tanh(.17 + .83 sin(sigma) ) )

    sin(sigma) cos(sigma)

      t0 = 0.22908E1*cos(0.138E1*rho)*exp(sin(0.138E1*rho))*(1.0-pow(tanh(0.17+
0.83*pow(sin(sigma),2.0)),2.0))*sin(sigma)*cos(sigma);
> diff(fn, sigma,sigma);		codegen[C](%);
                                                    2
-5.5112 exp(sin(1.38 rho)) tanh(.17 + .83 sin(sigma) )

                                  2 2            2           2
    (1 - tanh(.17 + .83 sin(sigma) ) ) sin(sigma)  cos(sigma)

                                                             2 2            2
     + 1.66 exp(sin(1.38 rho)) (1 - tanh(.17 + .83 sin(sigma) ) ) cos(sigma)

                                                             2 2            2
     - 1.66 exp(sin(1.38 rho)) (1 - tanh(.17 + .83 sin(sigma) ) ) sin(sigma)

      t0 = -0.55112E1*exp(sin(0.138E1*rho))*tanh(0.17+0.83*pow(sin(sigma),2.0))
*(1.0-pow(tanh(0.17+0.83*pow(sin(sigma),2.0)),2.0))*pow(sin(sigma),2.0)*pow(cos
(sigma),2.0)+0.166E1*exp(sin(0.138E1*rho))*(1.0-pow(tanh(0.17+0.83*pow(sin(
sigma),2.0)),2.0))*pow(cos(sigma),2.0)-0.166E1*exp(sin(0.138E1*rho))*(1.0-pow(
tanh(0.17+0.83*pow(sin(sigma),2.0)),2.0))*pow(sin(sigma),2.0);
> quit
bytes used=1170560, alloc=1179432, time=0.10

```

patch/patch_info.hh:
```
// patch_info.hh -- POD struct of minimal info varying from one patch to another
// $Header$

//
// prerequisites:
//    <stdio.h>
//    <assert.h>
//    <math.h>
//    "../jtutil/util.hh"
//    "../jtutil//array.hh"
//    "../jtutil/linear_map.hh"
//    "coords.hh"
//    "grid.hh"
//

// everything in this file is inside this namespace
namespace AHFinderDirect
	  {

//*****************************************************************************

//
// This (POD, and hence static-initializable) struct gives a minimal
// set of information which varies from one patch to another.
//
// The member functions allow computing all the grid:: constructor
// arguments; with these in hand it's fairly easy to construct the
// patch itself.  This scheme doesn't allow the most general possible
// type of patch (eg it constrains all ghost zones to have the same width,
// and it requires the grid spacing to evenly divide 90 degrees), but
// it does cover all the cases that seem to come up in practice.
//
// Arguments for member functions:
// ghost_zone_width = Width in grid points of all ghost zones.
// patch_extend_width = Number of grid points to extend each patch past
//		     "just touching" so as to overlap neighboring patches.
//		     Thus patches overlap by
//			patch_overlap_width = 2*patch_extend_width + 1
//		     grid points.  For example, with patch_extend_width == 2,
//		     here are the grid points of two neighboring patches:
//			x   x   x   x   x   X   X
//                                      |
//			        O   O   o   o   o   o   o
//		     Here | marks the "just touching" boundary,
//		     x and o the grid points before this extension,
//		     and X and O the extra grid points added by this
//		     extension.
// N_zones_per_right_angle = This sets the grid spacing (same in both
//			     directions) to 90.0 / N_zones_per_right_angle.
//			     It's a fatal error (error_exit()) if this
//			     doesn't evenly divide the grid sizes in both
//			     directions.
//
struct	patch_info
	{
	const char* name;
	bool is_plus;
	char ctype;
	fp min_drho, max_drho;
	fp min_dsigma, max_dsigma;

	// compute and return reference to  struct grid_arrays::grid_array_pars
	// ... result refers to internal static buffer;
	//     the usual caveats about lifetimes/overwriting apply
	const grid_arrays::grid_array_pars&
	  grid_array_pars(int ghost_zone_width, int patch_extend_width,
			  int N_zones_per_right_angle)
		const;

	// compute and return reference to  struct grid::grid_pars
	// ... result refers to internal static buffer;
	//     the usual caveats about lifetimes/overwriting apply
	const grid::grid_pars& grid_pars(int patch_extend_width,
					 int N_zones_per_right_angle)
		const;

private:
	// verify that grid spacing evenly divides grid sizes
	// in both directions; no-op if ok, error_exit() if not ok
	void verify_grid_spacing_ok(int N_zones_per_right_angle)
		const;
	};

//******************************************************************************

	  }	// namespace AHFinderDirect

```

patch/ghost_zone.cc:
```
// ghost_zone.cc -- fill in gridfn data in patch ghost zones
// $Header$

//
// ghost_zone::cast_to_symmetry_ghost_zone
// ghost_zone::cast_to_interpatch_ghost_zone
//
// symmetry_ghost_zone::symmetry_ghost_zone (mirror symmetry)
// symmetry_ghost_zone::symmetry_ghost_zone (periodic BC)
// symmetry_ghost_zone::~symmetry_ghost_zone
// symmetry_ghost_zone::synchronize
//
// interpatch_ghost_zone::interpatch_ghost_zone
// interpatch_ghost_zone::~interpatch_ghost_zone
// interpatch_ghost_zone::[min,max]_ipar
// interpatch_ghost_zone::finish_setup
// interpatch_ghost_zone::assert_fully_setup
// interpatch_ghost_zone::synchronize
// interpatch_ghost_zone::compute_Jacobian
//

#include <stdio.h>
#include <assert.h>
#include <stdlib.h>
#include <limits.h>
#include <math.h>

#include "cctk.h"

#include "config.h"
#include "stdc.h"

#include "../jtutil/util.hh"
#include "../jtutil/array.hh"
#include "../jtutil/cpm_map.hh"
#include "../jtutil/linear_map.hh"
using jtutil::error_exit;

#include "coords.hh"
#include "grid.hh"
#include "fd_grid.hh"
#include "patch.hh"
#include "patch_edge.hh"
#include "patch_interp.hh"
#include "ghost_zone.hh"

// all the code in this file is inside this namespace
namespace AHFinderDirect
	  {

//******************************************************************************
//******************************************************************************
//******************************************************************************

//
// These functions verify (assert()) that a ghost zone is indeed of
// the specified type, then static_cast to the appropriate derived class.
//

const symmetry_ghost_zone& ghost_zone::cast_to_symmetry_ghost_zone()
	const
{
assert( is_symmetry() );
return static_cast<const symmetry_ghost_zone &>(*this);
}

symmetry_ghost_zone& ghost_zone::cast_to_symmetry_ghost_zone()
{
assert( is_symmetry() );
return static_cast<symmetry_ghost_zone &>(*this);
}

//**************************************

const interpatch_ghost_zone& ghost_zone::cast_to_interpatch_ghost_zone()
	const
{
assert( is_interpatch() );
return static_cast<const interpatch_ghost_zone &>(*this);
}

interpatch_ghost_zone& ghost_zone::cast_to_interpatch_ghost_zone()
{
assert( is_interpatch() );
return static_cast<interpatch_ghost_zone &>(*this);
}

//******************************************************************************
//******************************************************************************
//******************************************************************************

//
// This function constructs a mirror-symmetry ghost zone object
//
symmetry_ghost_zone::symmetry_ghost_zone(const patch_edge& my_edge_in)
	: ghost_zone(my_edge_in,
		     my_edge_in,	// other edge == my edge
		     ghost_zone_is_symmetry)
{
// iperp_map: i --> (i of ghost zone) - i
iperp_map_ = new jtutil::cpm_map<fp>(min_iperp(), max_iperp(),
				     my_edge_in.fp_grid_outer_iperp());

// ipar_map_: identity map
ipar_map_ = new jtutil::cpm_map<fp>(extreme_min_ipar(), extreme_max_ipar());
}

//******************************************************************************

//
// This function constructs a periodic-symmetry ghost zone object.
//
symmetry_ghost_zone::symmetry_ghost_zone
	(const patch_edge& my_edge_in, const patch_edge& other_edge_in,
	 int my_edge_sample_ipar,      int other_edge_sample_ipar,
	 bool ipar_map_is_plus)
	: ghost_zone(my_edge_in,
		     other_edge_in,
		     ghost_zone_is_symmetry)
{
//
// perpendicular map
//
const fp fp_my_period_plane_iperp    = my_edge()   .fp_grid_outer_iperp();
const fp fp_other_period_plane_iperp = other_edge().fp_grid_outer_iperp();

// iperp mapping must be outside --> inside
// i.e. if both edges have iperp as the same min/max "direction",
//	then the mapping is  iperp increasing --> iperp decreasing
//      (i.e. the map's sign is -1)
const bool is_iperp_map_plus
	= ! (my_edge().is_min() == other_edge().is_min());
iperp_map_ = new jtutil::cpm_map<fp>(min_iperp(), max_iperp(),
				     fp_my_period_plane_iperp,
				     fp_other_period_plane_iperp,
				     is_iperp_map_plus);

//
// parallel map
//
ipar_map_ = new jtutil::cpm_map<fp>(extreme_min_ipar(), extreme_max_ipar(),
				    my_edge_sample_ipar, other_edge_sample_ipar,
				    ipar_map_is_plus);
}

//******************************************************************************

//
// This function destroys a  symmetry_ghost_zone  object.
//
symmetry_ghost_zone::~symmetry_ghost_zone()
{
delete ipar_map_;
delete iperp_map_;
}

//******************************************************************************

//
// This function "synchronizes" a ghost zone, i.e. it updates the
// ghost-zone values of the specified gridfns via the appropriate
// symmetry operations.The flags specify which part(s) of the ghost zone
// we want.
//
void symmetry_ghost_zone::synchronize(int ghosted_min_gfn, int ghosted_max_gfn,
				      bool want_corners /* = true */,
				      bool want_noncorner /* = true */)
{
	for (int gfn = ghosted_min_gfn ; gfn <= ghosted_max_gfn ; ++gfn)
	{
	for (int iperp = min_iperp() ; iperp <= max_iperp() ; ++iperp)
	{
	for (int ipar = min_ipar(iperp) ; ipar <= max_ipar(iperp) ; ++ipar)
	{
	// do we want to do this point?
	if (! my_edge().ipar_is_in_selected_part(want_corners, want_noncorner,
						 ipar) )
	   then continue;				// *** LOOP CONTROL ***

	const int sym_iperp = iperp_map_of_iperp(iperp);
	const int sym_ipar  = ipar_map_of_ipar  (ipar );
	const int sym_irho = other_edge()
			     .irho_of_iperp_ipar  (sym_iperp,sym_ipar);
	const int sym_isigma = other_edge()
			       .isigma_of_iperp_ipar(sym_iperp,sym_ipar);
	const fp sym_gridfn = other_patch()
			      .ghosted_gridfn(gfn, sym_irho,sym_isigma);

	const int irho   = my_edge().  irho_of_iperp_ipar(iperp,ipar);
	const int isigma = my_edge().isigma_of_iperp_ipar(iperp,ipar);
	my_patch().ghosted_gridfn(gfn, irho,isigma) = sym_gridfn;
	}
	}
	}
}

//******************************************************************************
//******************************************************************************
//******************************************************************************

//
// This function constructs an  interpatch_ghost_zone  object.
//
interpatch_ghost_zone::interpatch_ghost_zone(const patch_edge& my_edge_in,
					     const patch_edge& other_edge_in,
					     int patch_overlap_width)
	: ghost_zone(my_edge_in,
		     other_edge_in,
		     ghost_zone_is_interpatch),
	  // remaining pointers are all set up properly by finish_setup()
	  other_patch_interp_(NULL),
	  other_iperp_(NULL),
          min_ipar_used_(NULL),	max_ipar_used_(NULL),
	  other_par_(NULL),
	  interp_result_buffer_(NULL),
	  Jacobian_y_ipar_posn_(NULL), Jacobian_buffer_(NULL) // no comma
{
//
// verify that we have the expected relationships between
// this and the other patch's (mu,nu,phi) coordinates:
//

// perp coordinate is common to us and the other patch, so
// ghost zone must be min in one patch, max in the other
if (my_edge().is_min() == other_edge().is_min())
   then error_exit(ERROR_EXIT,
"***** interpatch_ghost_zone::interpatch_ghost_zone:\n"
"        my_patch().name()=\"%s\" my_edge().name()=%s\n"
"        other_patch().name()=\"%s\" other_edge().name()=%s\n"
"        ghost zone must be min in one patch, max in the other!\n"
,
		   my_patch().name(), my_edge().name(),
		   other_patch().name(), other_edge().name());	/*NOTREACHED*/

// coord in common between the two patches must be perp coord in both patches
// and this patch's tau coordinate must be other edge's parallel coordinate
const local_coords::coords_set common_coords_set
	= local_coords::coords_set_not(my_patch().coords_set_rho_sigma()
				       ^
				       other_patch().coords_set_rho_sigma());
if (! (    (common_coords_set == my_edge().coords_set_perp())
	&& (common_coords_set == other_edge().coords_set_perp())
	&& (my_patch().coords_set_tau() == other_edge().coords_set_par())    ) )
   then error_exit(PANIC_EXIT,
"***** interpatch_ghost_zone::interpatch_ghost_zone:\n"
"        (rho,sigma,tau) coordinates don't match up properly\n"
"        between this patch/edge and the other patch/edge!\n"
"        my_patch().name()=\"%s\" my_edge().name()=%s\n"
"        other_patch().name()=\"%s\" other_edge().name()=%s\n"
"        my_patch().coords_set_{rho,sigma,tau}={%s,%s,%s}\n"
"        my_edge().coords_set_{perp,par}={%s,%s}\n"
"        other_patch().coords_set_{rho,sigma,tau}={%s,%s,%s}\n"
"        other_edge().coords_set_{perp,par}={%s,%s}\n"
,
	my_patch().name(), my_edge().name(),
	other_patch().name(), other_edge().name(),
	local_coords::name_of_coords_set(my_patch().coords_set_rho()),
	local_coords::name_of_coords_set(my_patch().coords_set_sigma()),
	local_coords::name_of_coords_set(my_patch().coords_set_tau()),
	local_coords::name_of_coords_set(my_edge().coords_set_perp()),
	local_coords::name_of_coords_set(my_edge().coords_set_par()),
	local_coords::name_of_coords_set(other_patch().coords_set_rho()),
	local_coords::name_of_coords_set(other_patch().coords_set_sigma()),
	local_coords::name_of_coords_set(other_patch().coords_set_tau()),
	local_coords::name_of_coords_set(other_edge().coords_set_perp()),
	local_coords::name_of_coords_set(other_edge().coords_set_par()));
								/*NOTREACHED*/

// perp coordinate must match (mod 2*pi) across the two patches
// after taking into account any overlap
// ... eg patch_overlap_width = 3 would be
//	p   p   p   p   p
//		q   q   q   q   q
//     so the overlap would be (patch_overlap_width-1) * delta
const fp other_overlap
	= (patch_overlap_width-1) * other_edge().perp_map().delta_fp();
const fp other_outer_perp_minus_overlap	// move back inwards into other patch
					// by overlap distance, to get a value
					// that should match our own
					// grid_outer_perp() value
	= other_edge().grid_outer_perp()
	  + (other_edge().is_min() ? + other_overlap : - other_overlap);
if (! local_coords::fuzzy_EQ_ang(my_edge().grid_outer_perp(),
				 other_outer_perp_minus_overlap))
   then error_exit(ERROR_EXIT,
"***** interpatch_ghost_zone::interpatch_ghost_zone:\n"
"        my_patch().name()=\"%s\" my_edge().name()=%s\n"
"        other_patch().name()=\"%s\" other_edge().name()=%s\n"
"        perp coordinate doesn't match (mod 2*pi) across the two patches!\n"
"        my_edge().grid_outer_perp()=%g   <--(compare this)\n"
"        patch_overlap_width=%d other_overlap=%g\n"
"        other_edge.grid_outer_perp()=%g\n"
"        other_outer_perp_minus_overlap=%g   <--(against this)\n"
,
		   my_patch().name(), my_edge().name(),
		   other_patch().name(), other_edge().name(),
		   double(my_edge().grid_outer_perp()),
		   patch_overlap_width, double(other_overlap),
		   double(other_edge().grid_outer_perp()),
		   double(other_outer_perp_minus_overlap));	/*NOTREACHED*/


//
// set up the iperp interpatch coordinate mapping
// (gives other patch's iperp coordinate for interpolation)
//

// compute the iperp --> other_iperp mapping for a sample point;
// ... if the ghost zone is empty, then the sample point will necessarily
//     be out-of-range in the ghost zone, so we use the *unchecked*
//     conversions to avoid errors in this case
// ... we do the computation using the fact that  perp  is the same
//     coordinate in both patches (modulo 2*pi radians = 360 degrees)
const int sample_iperp = outer_iperp();
const fp sample_perp = my_edge().perp_map()
				.fp_of_int_unchecked(sample_iperp);
						// unchecked conversion here!
const fp other_sample_perp = other_patch()
			     .modulo_reduce_ang(other_edge().perp_is_rho(),
						sample_perp);
const fp fp_other_sample_iperp = other_edge()
				 .fp_iperp_of_perp(other_sample_perp);

// verify that this is fuzzily a grid point
if (! jtutil::fuzzy<fp>::is_integer(fp_other_sample_iperp))
   then error_exit(ERROR_EXIT,
"***** interpatch_ghost_zone::interpatch_ghost_zone:\n"
"        my_patch().name()=\"%s\" my_edge().name()=%s\n"
"        other_patch().name()=\"%s\" other_edge().name()=%s\n"
"        sample_iperp=%d sample_perp=%g\n"
"        other_sample_perp=%g fp_other_sample_iperp=%g\n"
"        ==> fp_other_sample_iperp isn't fuzzily an integer!\n"
"        ==> patches aren't commensurate in the perpendicular coordinate!\n"
,
		   my_patch().name(), my_edge().name(),
		   other_patch().name(), other_edge().name(),
		   sample_iperp, double(sample_perp),
		   double(other_sample_perp),
		   double(fp_other_sample_iperp));		/*NOTREACHED*/
const int other_sample_iperp
	= jtutil::round<fp>::to_integer(fp_other_sample_iperp);

// compute the +/- sign (direction) of the iperp --> other_iperp mapping
//
// Since perp is the same in both patches (mod 2*pi radians = 360 degrees),
// the overall +/- sign is just the product of the signs of the two individual
// iperp <--> perp mappings.
//
// ... signs encoded as (floating-point) +/- 1.0
const double iperp_map_sign_pm1
	=   jtutil::signum(    my_edge().perp_map().delta_fp() )
	  * jtutil::signum( other_edge().perp_map().delta_fp() );
// ... signs encoded as is_plus bool flag
const bool is_iperp_map_plus = (iperp_map_sign_pm1 > 0.0);

// now we finally know enough to set up the other_iperp(iperp)
// coordinate mapping
other_iperp_ = new jtutil::cpm_map<fp>(min_iperp(), max_iperp(),
				       sample_iperp, other_sample_iperp,
				       is_iperp_map_plus);
}

//******************************************************************************

//
// this function destroys an  interpatch_ghost_zone  object.
//
interpatch_ghost_zone::~interpatch_ghost_zone()
{
delete Jacobian_buffer_;
delete Jacobian_y_ipar_posn_;
delete interp_result_buffer_;
delete other_par_;
delete max_ipar_used_;
delete min_ipar_used_;
delete other_iperp_;
delete other_patch_interp_;
}

//******************************************************************************

//
// These functions compute the [min,max] ipar of the ghost zone for
// a given iperp, taking into account how we treat the corners
// (cf. the example in the header comments in "ghost_zone.hh"):
//
// If an adjacent ghost zone is symmetry,
//    we do not include that corner;
// If an adjacent ghost zone is interpatch,
//    we include up to the diagonal line, and if we are a rho ghost zone,
//    then also the diagonal line itself.  E.g. For the example in the
//    header comments "ghost_zone.hh", the +x ghost zone includes (6,6),
//    (7,6), and (7,7), while the +y ghost zone includes (6,7)
//
// ... in the following 2 functions,
//     the  iabs()  term includes the diagonal,
//     so we must remove the diagonal for !is_rho,
//     i.e. add 1 to min_ipar and subtract 1 from max_ipar
//
int interpatch_ghost_zone::min_ipar(int iperp) const
{
return min_par_adjacent_ghost_zone().is_symmetry()
       ? my_edge().min_ipar_without_corners()
       : my_edge().min_ipar_without_corners()
	 - iabs(iperp - my_edge().nominal_grid_outer_iperp())
	 + (is_rho() ? 0 : 1);
}

int interpatch_ghost_zone::max_ipar(int iperp) const
{
return max_par_adjacent_ghost_zone().is_symmetry()
       ? my_edge().max_ipar_without_corners()
       : my_edge().max_ipar_without_corners()
	 + iabs(iperp - my_edge().nominal_grid_outer_iperp())
	 - (is_rho() ? 0 : 1);
}

//******************************************************************************

//
// This function finishes the construction/setup of an  interpatch_ghost_zone
// object.  It
// - sets up the par coordinate mapping information
// - sets up the interpatch interpolator data pointer and result arrays
// - constructs the patch_interp object to interpolate from the *other* patch
//
// We use our ipar as the patch_interp's parindex.
//
void interpatch_ghost_zone::finish_setup(int interp_handle,
					 int interp_par_table_handle)
{
min_other_iperp_ = jtutil::min(other_iperp(min_iperp()),
			       other_iperp(max_iperp()));
max_other_iperp_ = jtutil::max(other_iperp(min_iperp()),
			       other_iperp(max_iperp()));


//
// set up arrays giving actual [min,max] ipar that we'll use
// at each other_iperp (later on we will pass these arrays to the
// other patch's  patch_interp  object, with ipar being parindex there
//
min_ipar_used_ = new jtutil::array1d<int>(min_other_iperp_, max_other_iperp_);
max_ipar_used_ = new jtutil::array1d<int>(min_other_iperp_, max_other_iperp_);
	  {
	for (int iperp = min_iperp() ; iperp <= max_iperp() ; ++iperp)
	{
	(*min_ipar_used_)(other_iperp(iperp)) = min_ipar(iperp);
	(*max_ipar_used_)(other_iperp(iperp)) = max_ipar(iperp);
	}
	  }


//
// set up array giving other patch's par coordinate for interpolation
//

other_par_ = new jtutil::array2d<fp>(min_other_iperp_, max_other_iperp_,
				     extreme_min_ipar(), extreme_max_ipar());

	  {
	for (int iperp = min_iperp() ; iperp <= max_iperp() ; ++iperp)
	{
	for (int ipar = min_ipar(iperp); ipar <= max_ipar(iperp) ; ++ipar)
	{
	// compute the  other_par corresponding to  (iperp,ipar)
	// ... here we use the fact (which we verified in our constructor)
	//     that other edge's parallel coordinate == our tau coordinate
	//     (at least modulo 2*pi radians = 360 degrees)
	const fp perp  = my_edge().perp_of_iperp(iperp);
	const fp par   = my_edge().par_of_ipar(ipar);

	const fp rho   = my_edge().  rho_of_perp_par(perp, par);
	const fp sigma = my_edge().sigma_of_perp_par(perp, par);

	const fp tau   = my_patch().tau_of_rho_sigma(rho, sigma);
	const fp other_par = other_patch()
			     .modulo_reduce_ang(other_edge().par_is_rho(), tau);

	(*other_par_)(other_iperp(iperp),ipar) = other_par;
	}
	}
	  }


//
// set up interpolation result buffer
//
interp_result_buffer_
	= new jtutil::array3d<fp>(my_patch().ghosted_min_gfn(),
					  my_patch().ghosted_max_gfn(),
				  min_other_iperp_, max_other_iperp_,
				  extreme_min_ipar(), extreme_max_ipar());

//
// construct the patch_interp object to interpolate from the *other* patch
// ... the patch_interp should use gridfn data from it's (the other patch's)
//     min/max par ghost zones if those (adjacent) adjacent ghost zones
//     are symmetry, but not if they're interpatch,
//     cf the header comments in "ghost_zone.hh"
//
const ghost_zone& other_ghost_zone = other_patch()
				     .ghost_zone_on_edge(other_edge());
const bool ok_to_use_min_par_ghost_zone
	= other_ghost_zone.min_par_adjacent_ghost_zone()
			  .is_symmetry()
	  ? true
	  : false;
const bool ok_to_use_max_par_ghost_zone
	= other_ghost_zone.max_par_adjacent_ghost_zone()
			  .is_symmetry()
	  ? true
	  : false;
other_patch_interp_ = new patch_interp(other_edge(),
				       min_other_iperp_, max_other_iperp_,
				       *min_ipar_used_, *max_ipar_used_,
				       *other_par_,
				       ok_to_use_min_par_ghost_zone,
				       ok_to_use_max_par_ghost_zone,
				       interp_handle, interp_par_table_handle);
}


//******************************************************************************

//
// This function asserts() that
// - we have a patch_interp object
// - our and the patch_interp object's notions of the "other patch" agree
// - the other patch has an interpatch ghost zone on this edge
// - the other patch's interpatch ghost zone on this edge,
//   points back to our patch
//
void interpatch_ghost_zone::assert_fully_setup() const
{
assert(other_patch_interp_ != NULL);
assert(other_patch() == other_patch_interp_->my_patch());
assert( other_patch()
	.ghost_zone_on_edge(other_edge())
	.is_interpatch() );
assert( my_patch() == other_patch()
		      .ghost_zone_on_edge(other_edge())
		      .other_patch() );
}

//******************************************************************************

//
// This function "synchronizes" a ghost zone, i.e. it updates the
// ghost-zone values of the specified gridfns via the appropriate
// interpatch interpolations.
//
// The flags specify which part(s) of the ghost zone we want, but
// the present implementation only supports the case where all the
// flags are  true , i.e. we want the entire ghost zone.
//
void interpatch_ghost_zone::synchronize
	(int ghosted_min_gfn, int ghosted_max_gfn,
	 bool want_corners /* = true */,
	 bool want_noncorner /* = true */)
{
// make sure the caller wants the entire ghost zone
if (! (want_corners && want_noncorner))
   then error_exit(ERROR_EXIT,
"***** interpatch_ghost_zone::synchronize():\n"
"        we only support operating on the *entire* ghost zone,\n"
"        but we were passed flags specifying a proper subset!\n"
"        want_corners=(int)%d want_noncorner=(int)%d\n"
,
		   want_corners, want_noncorner);		/*NOTREACHED*/

// do the interpolation into our result buffer
other_patch_interp_->interpolate(ghosted_min_gfn, ghosted_max_gfn,
				 *interp_result_buffer_);

// store the results back into our gridfns
    for (int gfn = ghosted_min_gfn ; gfn <= ghosted_max_gfn ; ++gfn)
    {
	for (int iperp = min_iperp() ; iperp <= max_iperp() ; ++iperp)
	{
	const int oiperp = other_iperp(iperp);

	for (int ipar = min_ipar(iperp) ; ipar <= max_ipar(iperp) ; ++ipar)
	{
	int irho   = my_edge().  irho_of_iperp_ipar(iperp,ipar);
	int isigma = my_edge().isigma_of_iperp_ipar(iperp,ipar);
	my_patch().ghosted_gridfn(gfn, irho,isigma)
		= (*interp_result_buffer_)(gfn, oiperp,ipar);
	}
	}
    }
}

//******************************************************************************

//
// This function allocates the internal buffers for the Jacobian, and
// computes that Jacobian
//	    partial synchronize gridfn(ghosted_gfn, iperp, ipar)
//	------------------------------------------------------------
//	partial other patch gridfn(ghosted_gfn, oiperp, posn+ipar_m)
// where
//	oiperp = Jacobian_oiperp(iperp)
//	posn = Jacobian_oipar_posn(iperp, ipar)
// into the internal buffers.
//
void interpatch_ghost_zone::compute_Jacobian
	(int ghosted_min_gfn, int ghosted_max_gfn,
	 bool want_corners /* = true */,
	 bool want_noncorner /* = true */)
	const
{
// make sure the caller wants the entire ghost zone
if (! (want_corners && want_noncorner))
   then error_exit(ERROR_EXIT,
"***** interpatch_ghost_zone::compute_Jacobian():\n"
"        we only support operating on the *entire* ghost zone,\n"
"        but we were passed flags specifying a proper subset!\n"
"        want_corners=(int)%d want_noncorner=(int)%d\n"
,
		   want_corners, want_noncorner);		/*NOTREACHED*/

assert(other_patch_interp_ != NULL);
other_patch_interp_->verify_Jacobian_sparsity_pattern_ok();

other_patch_interp_->molecule_minmax_ipar_m(Jacobian_min_y_ipar_m_,
					    Jacobian_max_y_ipar_m_);

if (Jacobian_y_ipar_posn_ == NULL)
   then Jacobian_y_ipar_posn_ = new jtutil::array2d<CCTK_INT>
				     (min_other_iperp_, max_other_iperp_,
				      extreme_min_ipar(), extreme_max_ipar());
other_patch_interp_->molecule_posn(*Jacobian_y_ipar_posn_);

if (Jacobian_buffer_ == NULL)
   then Jacobian_buffer_
		= new jtutil::array3d<fp>
			 (min_other_iperp_, max_other_iperp_,
			  extreme_min_ipar(), extreme_max_ipar(),
			  Jacobian_min_y_ipar_m_, Jacobian_max_y_ipar_m_);
other_patch_interp_->Jacobian(*Jacobian_buffer_);
}

//******************************************************************************
//******************************************************************************
//******************************************************************************

	  }	// namespace AHFinderDirect

```

patch/README:
```
This directory contains C++ code for the horizon finder's basic
multi-patch infrastructure.  This code doesn't know about relativity.

The main classes defined in this directory are as follows:
patch_system
	is the top-level class representing a multipatch system.
	It stores the actual grid function data, defines the actual
	finite differencing operations (both 2nd and 4th order are
	provided, presently selected by #ifdef at compile time),
	and keeps pointers to all the other objects making up the
	patch system.
x_patch, y_patch, z_patch
	represent individual patches near the +/- x, +/- y, and +/- z
	axes respectively.
patch
	is an virtual base class representing a grid patch.
patch_edge
	represents the geometry of a single edge of a patch, ie it
	represents the conversions between (perpendicular,parallel)
	and (rho,sigma) coordinates.
ghost_zone
	is a virtual base class representing the ghost zone
	along a single edge of a patch.
interpatch_ghost_zone
	represents a ghost zone where we get data by interpolating
	it from a neighboring patch.
symmetry_ghost_zone
	represents a ghost zone where we get data by a discrete
	symmetry (eg a reflection or periodic boundary condition).
patch_interp
	is our interface to the interpatch interpolation.

Class  patch  is in term implemented using the following helper classes:
grid
	This class is derived from class  grid_arrays , and adds
	the floating-point grid-coordinate metadata.
grid_arrays
	This class stores the basic integer-array-subscripts metadata
	for a 2-D grid.

The implementation also uses the low-level support routines and classes
in namespace jtutil::, which are all defined in  ../jtutil/ .

All the code makes heavy use of the type  fp , defined in
../include/config.h .  This is the floating-point type, i.e. CCTK_REAL.


coord_derivs.maple
coord_derivs.out
	are maple input and output to compute derivatives of the
	coordinate-transformation functions for the
	local_coords::partial*_wrt_*() and local_coords::partial2*_wrt_*()
	functions


There are also a number of test drivers in  test_*.cc :
test_coords.cc
test_coords2.cc
	These are standalone test drivers to test various coordinate
	conversions; there are targets in ./makefile to build them.

test_patch_system.cc
	This is a Cactus driver to do various tests of the patch system.
	To use it you need to uncomment its line in ./make.code.defn .

```

patch/patch_edge.hh:
```
// patch_edge.hh -- perpendicular/parallel geometry of one side of a patch
// $Header$
//

//
// prerequisites:
//	<stdio.h>
//	<assert.h>
//	<math.h>
//	"stdc.h"
//	"config.hh"
//	"../jtutil/util.hh"
//	"../jtutil/array.hh"
//	"../jtutil/linear_map.hh"
//	"coords.hh"
//	"grid.hh"
//	"fd_grid.hh"
//	"patch.hh"
//

// everything in this file is inside this namespace
namespace AHFinderDirect
	  {

//*****************************************************************************

//
// patch_edge -- perpendicular/parallel geometry of one side of a patch
//
// A  patch_edge  object is a very light-weight object which represents
// the basic geometry of a min/max rho/sigma side of a patch, i.e. it
// provides which-side-am-I predicates, coordinate conversions between
// (perp,par) and (rho,sigma), etc.  Every patch has (points to) 4  patch_edge
//  objects, one for each of the patch's sides.  See the comments in
// "patch.hh" for a "big picture" discussion of patches, patch edges,
// ghost zones, and patch interpolation regions.
//
// Note that since  patch_edge  has only  const  member functions
// (and members!), a  patch_edge  object is effectively always  const .
// This means there's no harm in always declaring  patch_edge  objects
// to be  const .
//

class	patch_edge
	{
public:
	//
	// ***** meta-info *****
	//

	// meta-info about patch
	patch& my_patch() const { return my_patch_; }

	// meta-info about edge
	bool is_rho() const { return is_rho_; }
	bool is_min() const { return is_min_; }
	bool perp_is_rho() const { return is_rho(); }
	bool par_is_rho() const { return ! is_rho(); }

	// human-readable {min,max}_{rho,sigma} name (for debugging etc)
	const char* name() const
		{
		return is_min()
		       ? (is_rho() ? "min_rho" : "min_sigma")
		       : (is_rho() ? "max_rho" : "max_sigma");
		}

	// are two edges really the same edge?
	bool operator==(const patch_edge& other_edge) const
		{
		return    ( my_patch() == other_edge.my_patch() )
		       && (   is_rho() == other_edge.  is_rho() )
		       && (   is_min() == other_edge.  is_min() );
		}
	bool operator!=(const patch_edge& other_edge) const
		{ return ! operator==(other_edge); }


	//
	// ***** adjacent edges *****
	//

	// get adjacent edges to our min/max par corners
	const patch_edge& min_par_adjacent_edge() const
		{
		return my_patch()
		       .minmax_ang_patch_edge(grid::side_is_min, par_is_rho());
		}
	const patch_edge& max_par_adjacent_edge() const
		{
		return my_patch()
		       .minmax_ang_patch_edge(grid::side_is_max, par_is_rho());
		}
	const patch_edge& minmax_par_adjacent_edge(bool want_min) const
		{
		return want_min ? min_par_adjacent_edge()
				: max_par_adjacent_edge();
		}


	//
	// ***** gridfn subscripting and coordinate maps *****
	//

	// gridfn strides perpendicular/parallel to the edge
	int perp_stride() const
		{ return my_patch().iang_stride(perp_is_rho()); }
	int par_stride() const
		{ return my_patch().iang_stride(par_is_rho()); }
	int ghosted_perp_stride() const
		{ return my_patch().ghosted_iang_stride(perp_is_rho()); }
	int ghosted_par_stride() const
		{ return my_patch().ghosted_iang_stride(par_is_rho()); }

	// coordinate maps perpendicular/parallel to the edge
	// ... range is that of the grid *including* ghost zones
	const jtutil::linear_map<fp>& perp_map() const
		{ return my_patch().ang_map(perp_is_rho()); }
	const jtutil::linear_map<fp>& par_map() const
		{ return my_patch().ang_map(par_is_rho()); }

	// meta-info about perp/par coordinates
	// ... as (mu,nu,phi) tensor indices
	local_coords::coords_set coords_set_perp() const
		{
		return perp_is_rho() ? my_patch().coords_set_rho()
				     : my_patch().coords_set_sigma();
		}
	local_coords::coords_set coords_set_par() const
		{
		return par_is_rho() ? my_patch().coords_set_rho()
				    : my_patch().coords_set_sigma();
		}


	//
	// ***** coordinate conversions *****
	//

	// coordinate conversions based on ghost zone direction
	// ... (iperp,ipar) <--> (perp,par)
	fp perp_of_iperp(int iperp) const
		{ return my_patch().ang_of_iang(perp_is_rho(), iperp); }
	fp par_of_ipar(int ipar) const
		{ return my_patch().ang_of_iang(par_is_rho(), ipar); }
	fp fp_iperp_of_perp(fp perp) const
		{ return my_patch().fp_iang_of_ang(perp_is_rho(), perp); }
	fp fp_ipar_of_par(fp par) const
		{ return my_patch().fp_iang_of_ang(par_is_rho(), par); }
	int iperp_of_perp(fp perp, jtutil::linear_map<fp>::noninteger_action
				   nia = jtutil::linear_map<fp>::nia_error)
		{ return my_patch().iang_of_ang(perp_is_rho(), perp, nia); }
	int ipar_of_par(fp par, jtutil::linear_map<fp>::noninteger_action
				nia = jtutil::linear_map<fp>::nia_error)
		{ return my_patch().iang_of_ang(par_is_rho(), par, nia); }

	// ... (perp,par) --> (rho,sigma)
	int irho_of_iperp_ipar(int iperp, int ipar) const
		{ return perp_is_rho() ? iperp : ipar; }
	int isigma_of_iperp_ipar(int iperp, int ipar) const
		{ return perp_is_rho() ? ipar : iperp; }
	fp rho_of_perp_par(fp perp, fp par) const
		{ return perp_is_rho() ? perp : par; }
	fp sigma_of_perp_par(fp perp, fp par) const
		{ return perp_is_rho() ? par : perp; }
	// ... (rho,sigma) --> (perp,par)
	int iperp_of_irho_isigma(int irho, int isigma) const
		{ return perp_is_rho() ? irho : isigma; }
	int ipar_of_irho_isigma(int irho, int isigma) const
		{ return par_is_rho() ? irho : isigma; }
	fp perp_of_rho_sigma(fp rho, fp sigma) const
		{ return perp_is_rho() ? rho : sigma; }
	fp par_of_rho_sigma(fp rho, fp sigma) const
		{ return par_is_rho() ? rho : sigma; }

	// outer perp of nominal grid on this edge
	// ... this is outermost *grid point*
	fp grid_outer_iperp() const
		{ return my_patch().minmax_iang(is_min(), is_rho()); }
	// ... this is actual outer edge of grid
	//     (might be halfway between two grid points)
	fp grid_outer_perp() const
		{ return my_patch().minmax_ang(is_min(), is_rho()); }
	// ... this is grid_outer_perp() converted back to the iperp
	//     coordinate, but still returned as floating-point;
	//     it will be either integer or half-integer
	fp fp_grid_outer_iperp() const
		{ return fp_iperp_of_perp(grid_outer_perp()); }



	//
	// ***** min/max/outer coordinates of edge *****
	//

	// min/max/size ipar of the edge
	// (these are exteme limits for any iperp, a given ghost zone
	//  or interpolation region may have tighter and/or iperp-dependent
	// limits)
	// ... not including corners
	int min_ipar_without_corners() const
		{ return my_patch().min_iang(par_is_rho()); }
	int max_ipar_without_corners() const
		{ return my_patch().max_iang(par_is_rho()); }
	// ... including corners
	int min_ipar_with_corners() const
		{ return my_patch().ghosted_min_iang(par_is_rho()); }
	int max_ipar_with_corners() const
		{ return my_patch().ghosted_max_iang(par_is_rho()); }
	// ... of the corners themselves
	int min_ipar_corner__min_ipar() const
		{ return min_ipar_with_corners(); }
	int min_ipar_corner__max_ipar() const
		{ return min_ipar_without_corners() - 1; }
	int max_ipar_corner__min_ipar() const
		{ return max_ipar_without_corners() + 1; }
	int max_ipar_corner__max_ipar() const
		{ return max_ipar_with_corners(); }

	// membership predicates for ipar corners, non-corners
	bool ipar_is_in_min_ipar_corner(int ipar) const
		{
		return    (ipar >= min_ipar_corner__min_ipar())
		       && (ipar <= min_ipar_corner__max_ipar());
		}
	bool ipar_is_in_max_ipar_corner(int ipar) const
		{
		return    (ipar >= max_ipar_corner__min_ipar())
		       && (ipar <= max_ipar_corner__max_ipar());
		}
	bool ipar_is_in_corner(int ipar) const
		{
		return    ipar_is_in_min_ipar_corner(ipar)
		       || ipar_is_in_max_ipar_corner(ipar);
		}
	bool ipar_is_in_noncorner(int ipar) const
		{
		return    (ipar >= min_ipar_without_corners())
		       && (ipar <= max_ipar_without_corners());
		}

	// convenience function selecting amongst the above
	// membership predicates
	bool ipar_is_in_selected_part(bool want_corners,
				      bool want_noncorner,
				      int ipar)
		const
		{
		return    (want_corners   && ipar_is_in_corner   (ipar))
		       || (want_noncorner && ipar_is_in_noncorner(ipar));
		}

	// outer (farthest from patch center) iperp of nominal grid
	int nominal_grid_outer_iperp() const
		{
		return my_patch()
		       .minmax_iang(is_min(), is_rho());
		}


	//
	// ***** constructor, destructor *****
	//

	patch_edge(patch& my_patch_in,
		   bool is_min_in, bool is_rho_in)
		: my_patch_(my_patch_in),
		  is_min_(is_min_in), is_rho_(is_rho_in)
		{ }
	// compiler-synthesized (no-op) destructor is fine

private:
	// we forbid copying and passing by value
	// by declaring the copy constructor and assignment operator
	// private, but never defining them
	patch_edge(const patch_edge& rhs);
	patch_edge& operator=(const patch_edge& rhs);

private:
	patch& my_patch_;
	const bool is_min_, is_rho_;
	};

//******************************************************************************

	  }	// namespace AHFinderDirect

```

patch/Makefile.standalone:
```
# Makefile for standalone test drivers in this directory
# $Header$
#
#	CC, CXX = C and C++ compilers.  Defaults are gcc and g++ if
#		  variables aren't set from command line or environment.
#	CC_FLAGS, CXX_FLAGS = C++ compiler flags.  Defaults are
#				$(STD_GCC_FLAGS) -I.. -g
#			      and
#				$(STD_GXX_FLAGS) -I.. -g
#			      if variables aren't set from command line or
#			      environment.
#
# Targets:
# test		==> build test programs
# clean		==> delete object files, test drivers
# superclean	==> delete object files, test drivers
#
# Bugs:
# - The makefile in ../jtutil/ should be executed first, since this
#   makefile assumes all the ../jtutil/*.o files already exist
# - Dependencies on *.hh are omitted.
#

CXX      := g++
CFLAGS   := $(STD_GCC_FLAGS) -I../include -DSTANDALONE_TEST -g
CXXFLAGS := $(STD_GXX_FLAGS) -I../include -DSTANDALONE_TEST -g

JTUTIL_OBJ	:= $(filter-out ../jtutil/test%, $(wildcard ../jtutil/*.o))

################################################################################

test		: test_coords test_coords2

test_coords	: test_coords.cc coords.cc $(JTUTIL_OBJ) -lm
test_coords2	: test_coords2.cc coords.cc $(JTUTIL_OBJ) -lm

.PHONY		: coord.derivs
coord.derivs	:
	maple <coord_derivs.maple >coord_derivs.out

.PHONY		: clean
clean		:
	-rm *.o test_coords test_coords2

```

patch/coords.hh:
```
// coords.hh -- coordinate systems and conversions
// $Header$
//
// ***** coordinate systems *****
// ***** conversions between different local coordinate systems *****
// ***** bit masks for coordinates ****
// global_coords - conversions between global and local coordinates
//

//
// prerequisites:
//    <math.h>
//    "cctk.h" or "fake_cctk.h"
//    "stdc.h"
//    "config.hh"
//    "../jtutil/util.hh"
//

// everything in this file is inside this namespace
namespace AHFinderDirect
	  {

//******************************************************************************

//
// ***** coordinate systems *****
//

//
// We use the following terminology for coordinates:
//	{a,b,c} == any one of (a,b,c) == a | b | c
//	((a,b,c)) == any two of (a,b,c) == (a,b) | (a,c) | (b,c)
//	(r,(a,b,c)) == r and any two of (a,b,c) == (r,a,b) | (r,a,c) | (r,b,c)
//

//
// We use the following coordinates and coordinate systems:
//	global_(x,y,z)	global Cartesian x,y,z coordinates
//			(these are the coordinates Cactus gives us)
//	(x,y,z)		local Cartesian x,y,z coordinates relative to a
//			local origin point (stored in this object);
//			This origin should be chosen to be somewhere
//			at least vaguely in the "middle" of our
//			apparent horizon
//	(r,theta,phi)	polar spherical coordinates about the local
//			origin point
//	(r,(mu,nu,phi))	patch coordinates about the local origin point
//		mu	arctan(y/z) = rotation about local x axis
//		nu	arctan(x/z) = rotation about local y axis
//		phi	arctan(y/x) = rotation about local z axis
//				    = usual polar spherical phi
//	(rho,sigma)	generic patch angular coordinates, these will
//			be whichever two of (mu,nu,phi) are nonsingular
//			throughout the patch
//	tau		the third member of (mu,nu,phi) (the one which
//			is neither rho nor sigma)
//	(perp,par)	(rho,sigma) coordinates (perpendicular,parallel)
//			to a patch boundary; these are defined by class
//			patch_edge and used by a lot of the boundary code
//

//
// We also use direction cosines (xcos,ycos,zcos):
// xcos = x/r = cos(angle between vector from origin to (x,y,z), and x axis)
// ycos = y/r = cos(angle between vector from origin to (x,y,z), and y axis)
// zcos = z/r = cos(angle between vector from origin to (x,y,z), and z axis)
//

//
// We also use prefixes on coordinates, eg.
//	irho		integer grid coordinate
//	 rho		floating point angular coordinate, measured in radians
//	drho		floating point angular coordinate, measured in degrees
// I/O uses the degree coordinates in preference to radian ones.
//

//
// We also define plotting coordinates (px,py) (in both fp and degrees-fp
// variants), based on the conventional "looking down the z axis" picture
// of a patch system.  Unfortunately there's no good place to stick a -z
// patch in this scheme, so we arbitrarily put it just beyond the +x patch:
//
//	           py axis
//	              |
//	          +-------+
//	          |       |
//	          |  +y   |
//	          |       |
//	  +-------+-------+-------+-------+
//	  |       |       |       |       |
//	--|  -x   |  +z   |  +x   |  -z   |-- px axis
//	  |       |       |       |       |
//	  +-------+-------+-------+-------+
//	          |       |
//	          |  -y   |
//	          |       |
//	          +-------+
//	              |
//
// The main (in fact only) use of these coordinates is in their degrees
// forms (dpx,dpy), for I/O and constructor arguments.
//

//*****************************************************************************

//
// ***** stuff for treating angles modulo 2*pi radians (360 degrees) *****
//

namespace local_coords
	  {

// compare if two angles are fuzzily equal mod 2*pi radians (360 degrees)
bool fuzzy_EQ_ang(fp ang1, fp ang2);	// radians
bool fuzzy_EQ_dang(fp dang1, fp dang2);	// degrees

// modulo-reduce  {ang,dang}  to be (fuzzily) within the range
// [min,max]_{ang,dang}, or error_exit() if no such value exists
fp modulo_reduce_ang(fp ang, fp min_ang, fp max_ang);
fp modulo_reduce_dang(fp dang, fp min_dang, fp max_dang);

	  }	// close namespace local_coords::

//*****************************************************************************

//
// ***** conversions between different local coordinate systems *****
//
// Bugs:
//	These functions aren't optimally efficient: they do lots
//	of could-be-optimized away arithmetic and trig calls.  But
//	In practice this doesn't matter, since we don't call these
//	functions inside inner loops (instead, we cache their values
//	at grid points).
//

namespace local_coords
	  {
// (r,(mu,nu,phi)) <--> (x,y,z)
void xyz_of_r_mu_nu (fp r, fp mu, fp nu ,   fp& x, fp& y, fp& z);
void xyz_of_r_mu_phi(fp r, fp mu, fp phi,   fp& x, fp& y, fp& z);
void xyz_of_r_nu_phi(fp r, fp nu, fp phi,   fp& x, fp& y, fp& z);
fp r_of_xyz(fp x, fp y, fp z);
fp mu_of_yz(fp y, fp z);
fp nu_of_xz(fp x, fp z);
fp phi_of_xy(fp x, fp y);

// ((mu,nu,phi)) --> the 3rd
fp phi_of_mu_nu(fp mu, fp nu );
fp nu_of_mu_phi(fp mu, fp phi);
fp mu_of_nu_phi(fp nu, fp phi);

// partial {x,y,z} / partial {mu,nu,phi}
void partial_xyz_wrt_r_mu_nu
	(fp r, fp mu, fp nu,
	 fp& partial_x_wrt_r, fp& partial_x_wrt_mu, fp& partial_x_wrt_nu,
	 fp& partial_y_wrt_r, fp& partial_y_wrt_mu, fp& partial_y_wrt_nu,
	 fp& partial_z_wrt_r, fp& partial_z_wrt_mu, fp& partial_z_wrt_nu);
void partial_xyz_wrt_r_mu_phi
	(fp r, fp mu, fp phi,
	 fp& partial_x_wrt_r, fp& partial_x_wrt_mu, fp& partial_x_wrt_phi,
	 fp& partial_y_wrt_r, fp& partial_y_wrt_mu, fp& partial_y_wrt_phi,
	 fp& partial_z_wrt_r, fp& partial_z_wrt_mu, fp& partial_z_wrt_phi);
void partial_xyz_wrt_r_nu_phi
	(fp r, fp nu, fp phi,
	 fp& partial_x_wrt_r, fp& partial_x_wrt_nu, fp& partial_x_wrt_phi,
	 fp& partial_y_wrt_r, fp& partial_y_wrt_nu, fp& partial_y_wrt_phi,
	 fp& partial_z_wrt_r, fp& partial_z_wrt_nu, fp& partial_z_wrt_phi);

// partial {mu,nu,phi} / partial {x,y,z}
fp partial_mu_wrt_y(fp y, fp z);
fp partial_mu_wrt_z(fp y, fp z);
fp partial_nu_wrt_x(fp x, fp z);
fp partial_nu_wrt_z(fp x, fp z);
fp partial_phi_wrt_x(fp x, fp y);
fp partial_phi_wrt_y(fp x, fp y);

// partial^2 {mu,nu,phi} / partial {x,y,z}{x,y,z}
fp partial2_mu_wrt_yy(fp y, fp z);
fp partial2_mu_wrt_yz(fp y, fp z);
fp partial2_mu_wrt_zz(fp y, fp z);
fp partial2_nu_wrt_xx(fp x, fp z);
fp partial2_nu_wrt_xz(fp x, fp z);
fp partial2_nu_wrt_zz(fp x, fp z);
fp partial2_phi_wrt_xx(fp x, fp y);
fp partial2_phi_wrt_xy(fp x, fp y);
fp partial2_phi_wrt_yy(fp x, fp y);

// usual polar spherical (r,theta,phi) <--> (x,y,z)
void xyz_of_r_theta_phi(fp r, fp theta, fp phi,   fp& x, fp& y, fp& z);
void r_theta_phi_of_xyz(fp x, fp y, fp z,   fp& r, fp& theta, fp& phi);
// ... already have r_of_xyz()
// ... already have phi_of_xy()
fp theta_of_xyz(fp x, fp y, fp z);

// ((mu,nu,phi)) <--> usual polar spherical (theta,phi)
// ... note phi is the same coordinate in both systems
void theta_phi_of_mu_nu (fp mu, fp nu ,   fp& ps_theta, fp& ps_phi);
void theta_phi_of_mu_phi(fp mu, fp phi,   fp& ps_theta, fp& ps_phi);
void theta_phi_of_nu_phi(fp nu, fp phi,   fp& ps_theta, fp& ps_phi);
void  mu_nu_of_theta_phi(fp ps_theta, fp ps_phi,   fp& mu, fp& nu );
void mu_phi_of_theta_phi(fp ps_theta, fp ps_phi,   fp& mu, fp& phi);
void nu_phi_of_theta_phi(fp ps_theta, fp ps_phi,   fp& nu, fp& phi);

// ((mu,nu,phi)) --> direction cosines (xcos,ycos,zcos)
void xyzcos_of_mu_nu (fp mu, fp nu , fp& xcos, fp& ycos, fp& zcos);
void xyzcos_of_mu_phi(fp mu, fp phi, fp& xcos, fp& ycos, fp& zcos);
void xyzcos_of_nu_phi(fp nu, fp phi, fp& xcos, fp& ycos, fp& zcos);
	  }	// close namespace local_coords::

//*****************************************************************************

//
// ***** bit masks for coordinates ****
//

//
// We need to manipulate coordinates to do calculations like "which
// coordinate do these two patches have in common".  We do these by
// Boolean operations on integers using the following bit masks:
//

namespace local_coords
	  {

typedef int coords_set;

enum {
     coords_set_mu  = 0x1,
     coords_set_nu  = 0x2,
     coords_set_phi = 0x4,

     coords_set_empty = 0x0,
     coords_set_all = coords_set_mu | coords_set_nu | coords_set_phi // no comma
     };

// human-readable coordinate names for debugging etc
const char* name_of_coords_set(coords_set S);

// set complement of coordinates
inline
  coords_set coords_set_not(coords_set S)
	{ return coords_set_all & ~S; }

	  }	// close namespace local_coords::

//******************************************************************************

//
// This class stores the origin point of our local coordinates, and
// provides conversions between local and global coordinates.
//
class	global_coords
	{
public:
	#ifdef NOT_USED
	// global (x,y,z) --> local (x,y,z)
	fp local_x_of_global_x(fp global_x) const
		{ return global_x - origin_x_; }
	fp local_y_of_global_y(fp global_y) const
		{ return global_y - origin_y_; }
	fp local_z_of_global_z(fp global_z) const
		{ return global_z - origin_z_; }
	#endif	/* NOT_USED */

	#ifdef NOT_USED
	// local (x,y,z) --> global (x,y,z)
	fp global_x_of_local_x(fp local_x) const
		{ return origin_x_ + local_x; }
	fp global_y_of_local_y(fp local_y) const
		{ return origin_y_ + local_y; }
	fp global_z_of_local_z(fp local_z) const
		{ return origin_z_ + local_z; }
	#endif	/* NOT_USED */

	// get global (x,y,z) coordinates of local origin point
	fp origin_x() const { return origin_x_; }
	fp origin_y() const { return origin_y_; }
	fp origin_z() const { return origin_z_; }

	// set global (x,y,z) coordinates of local origin point
	void origin_x(const fp x) { origin_x_=x; }
	void origin_y(const fp y) { origin_y_=y; }
	void origin_z(const fp z) { origin_z_=z; }

	// radius of given (x,y,z) with respect to local origin point
	#ifdef NOT_USED
	fp radius_of_local_xyz(fp local_x, fp local_y, fp local_z) const
		{ return jtutil::hypot3(local_x, local_y, local_z); }
	fp radius_of_global_xyz(fp global_x, fp global_y, fp global_z)
		const
		{
		return radius_of_local_xyz(local_x_of_global_x(global_x),
					   local_y_of_global_y(global_y),
					   local_z_of_global_z(global_z));
		}
	#endif	/* NOT_USED */

	// constructor: specify global (x,y,z) coordinates of local origin point
	global_coords(fp origin_x_in, fp origin_y_in, fp origin_z_in)
		: origin_x_(origin_x_in),
		  origin_y_(origin_y_in),
		  origin_z_(origin_z_in)
		{ }
	// destructor: compiler-generated no-op is ok

private:
	// we forbid copying and passing by value
	// by declaring the copy constructor and assignment operator
	// private, but never defining them
	global_coords(const global_coords& rhs);
	global_coords& operator=(const global_coords& rhs);

private:
	// global (x,y,z) coordinates of local origin point
	fp origin_x_, origin_y_, origin_z_;
	};

//******************************************************************************

	  }	// namespace AHFinderDirect

```

patch/patch_system.hh:
```
// patch_system.hh -- describes the (an) entire system of interlinked patches
// $Header$

//
// patch_system - describes a system of interlinked patches
//

//
// prerequisites:
//	<stdio.h>
//	<assert.h>
//	<math.h>
//	"stdc.h"
//	"config.hh"
//	"../jtutil/util.hh"
//	"../jtutil/array.hh"
//	"../jtutil/linear_map.hh"
//	"coords.hh"
//	"grid.hh"
//	"fd_grid.hh"
//	"patch.hh"
//	"patch_edge.hh"
//	"ghost_zone.hh"
//	"patch_interp.hh"
//	"patch_info.hh"
//

// everything in this file is inside this namespace
namespace AHFinderDirect
	  {

//******************************************************************************

//
// A  patch_system  object describes a system of interlinked patches.
//
// Its  const  qualifiers refer (only) to the gridfn data.  Notably, this
// means that  synchronize()  is a non-const function (it modifies gridfn
// data), while  synchronize_Jacobian()  et al are const functions (they
// don't modify gridfn data) even though they may update other internal
// state in the  patch_system  object and its subobjects.
//

class	patch_system
	{
	//
	// ***** static data & functions describing patch systems *****
	//
public:
	// what patch-system type are supported?
	// (see "patch_system_info.hh" for detailed descriptions of these)
	enum patch_system_type {
			       patch_system__full_sphere,
			       patch_system__plus_z_hemisphere,
			       patch_system__plus_xy_quadrant_mirrored,
			       patch_system__plus_xy_quadrant_rotating,
			       patch_system__plus_xz_quadrant_mirrored,
			       patch_system__plus_xz_quadrant_rotating,
			       patch_system__plus_xyz_octant_mirrored,
			       patch_system__plus_xyz_octant_rotating
			       };

	// maximum number of patches in any patch-system type
	static
	  const int max_N_patches = 6;

	// decode patch system type into N_patches
	static
	  int N_patches_of_type(enum patch_system_type type_in);

	// patch system type <--> human-readable character-string name
	static
	  const char* name_of_type(enum patch_system_type type_in);
	static
	  enum patch_system_type type_of_name(const char* name_in);


	//
	// ***** coordinates *****
	//
public:
	#ifdef NOT_USED
	// global (x,y,z) --> local (x,y,z)
	fp local_x_of_global_x(fp global_x) const
		{ return global_coords_.local_x_of_global_x(global_x); }
	fp local_y_of_global_y(fp global_y) const
		{ return global_coords_.local_y_of_global_y(global_y); }
	fp local_z_of_global_z(fp global_z) const
		{ return global_coords_.local_z_of_global_z(global_z); }
	#endif	/* NOT_USED */

	#ifdef NOT_USED
	// local (x,y,z) --> global (x,y,z)
	fp global_x_of_local_x(fp local_x) const
		{ return global_coords_.global_x_of_local_x(local_x); }
	fp global_y_of_local_y(fp local_y) const
		{ return global_coords_.global_y_of_local_y(local_y); }
	fp global_z_of_local_z(fp local_z) const
		{ return global_coords_.global_z_of_local_z(local_z); }
	#endif	/* NOT_USED */

	// get global (x,y,z) coordinates of local origin point
	fp origin_x() const { return global_coords_.origin_x(); }
	fp origin_y() const { return global_coords_.origin_y(); }
	fp origin_z() const { return global_coords_.origin_z(); }

	// set global (x,y,z) coordinates of local origin point
        void origin_x(const fp x) { global_coords_.origin_x(x); }
	void origin_y(const fp y) { global_coords_.origin_y(y); }
	void origin_z(const fp z) { global_coords_.origin_z(z); }

 
	//
	// ***** meta-info about the entire patch system *****
	//
public:
	// patch-system type
	enum patch_system_type type() const { return type_; }

	// total number of patches
	int N_patches() const { return N_patches_; }

	// get patches by patch number
	const patch& ith_patch(int pn) const
		{ return * all_patches_[pn]; }
	patch& ith_patch(int pn)
		{ return * all_patches_[pn]; }

	// find a patch by +/- xyz "ctype"
	// FIXME: the present implementation of this function is quite slow
	const patch& plus_or_minus_xyz_patch(bool is_plus, char ctype)
		const;

	// find a patch by name, return patch number; error_exit() if not found
	int patch_number_of_name(const char* name) const;

	// total number of grid points
	int N_grid_points() const { return N_grid_points_; }
	int ghosted_N_grid_points() const { return ghosted_N_grid_points_; }
	int max_N_additional_points() const { return max_N_additional_points_; }
	int N_additional_points() const { return N_additional_points_; }
        int N_additional_points(const int n) { return N_additional_points_=n; }


	//
	// ***** meta-info about gridfns *****
	//
public:
	int min_gfn() const { return ith_patch(0).min_gfn(); }
	int max_gfn() const { return ith_patch(0).max_gfn(); }
	int N_gridfns() const { return ith_patch(0).N_gridfns(); }
	bool is_valid_gfn(int gfn) const
		{ return ith_patch(0).is_valid_gfn(gfn); }
	int ghosted_min_gfn() const { return ith_patch(0).ghosted_min_gfn(); }
	int ghosted_max_gfn() const { return ith_patch(0).ghosted_max_gfn(); }
	int ghosted_N_gridfns() const
		{ return ith_patch(0).ghosted_N_gridfns(); }
	bool is_valid_ghosted_gfn(int ghosted_gfn) const
		{ return ith_patch(0).is_valid_ghosted_gfn(ghosted_gfn); }


	//
	// ***** synchronize() and its Jacobian *****
	//
public:
	// "synchronize" all ghost zones of all patches,
	// i.e. update the ghost-zone values of the specified gridfns
	// via the appropriate sequence of symmetry operations
	// and interpatch interpolations
	void synchronize(int ghosted_min_gfn_to_sync,
			 int ghosted_max_gfn_to_sync);

	// ... do this for all ghosted gridfns
	void synchronize()
		{
		synchronize(ghosted_min_gfn(),
			    ghosted_max_gfn());
		}


	//
	// do any precomputation necessary to compute Jacobian of
	//  synchronize() , taking into account synchronize()'s
	// full 3-phase algorithm
	//
	void compute_synchronize_Jacobian(int ghosted_min_gfn_to_sync,
					  int ghosted_max_gfn_to_sync)
		const;

	// ... do this for all ghosted gridfns
	void compute_synchronize_Jacobian()
		const
		{
		compute_synchronize_Jacobian(ghosted_min_gfn(),
					     ghosted_max_gfn());
		}

	//
	// The following functions access the Jacobian computed by
	//  compute_synchronize_Jacobian() .  Note this API is rather
	// different than that of ghost_zone::comute_Jacobian()  et al:
	// here we must take into account synchronize()'s full 3-phase
	// algorithm, and this may lead to a more general Jacobian
	// structure.
	//
	// This API still implicitly assumes that the Jacobian is
	// independent of  ghosted_gfn , and that the set of y points
	// (with nonzero Jacobian values) in a single row of the Jacobian
	// matrix (i.e. the set of points on which a single ghost-zone
	// point depends),
	// - lies entirely within a single y patch
	// - has a single yiperp value
	// - have a contiguous interval of yipar; we parameterize this
	//   interval as  yipar = posn+m
	//

	// what are the global min/max  m  over all ghost zone points?
	// (this is useful for sizing the buffer for synchronize_Jacobian())
	void synchronize_Jacobian_global_minmax_ym(int& min_ym, int& max_ym)
		const;

	// compute a single row of the Jacobian:
	// - return value is edge to which y point belongs
	//   (caller can get patch from this edge)
	// - store y_iperp and y_posn and min/max ym in named arguments
	// - stores the Jacobian elements
	//	partial synchronize() gridfn(ghosted_gfn, px, x_iperp, x_ipar)
	//	-------------------------------------------------------------
	//	     partial gridfn(ghosted_gfn, py, y_iperp, y_posn+ym)
	//   (taking into account synchronize()'s full 3-phase algorithm)
	//   in the caller-supplied buffer
	//	Jacobian_buffer(ym)
	//   for each  ym  in the min/max ym range
	const patch_edge&
	  synchronize_Jacobian(const ghost_zone& xgz,
			       int x_iperp, int x_ipar,
			       int& y_iperp,
			       int& y_posn, int& min_ym, int& max_ym,
			       jtutil::array1d<fp>& Jacobian_buffer)
		const;

	// helper functions for synchronize_Jacobian():
private:
	// "fold" (part of) a Jacobian row
	// to take a symmetry operation into acount
	// e_Jac = edge which the Jacobian lies along
	// e_fold = edge about which to fold
	// [min,max]_m = range of m in the Jacobian
	// [min,max]_fold_m = range of m to fold
	//		      (must be a subrange of {min,max}_m)
	void fold_Jacobian(const patch_edge& e_Jac, const patch_edge& e_fold,
			   int iperp,
			   int posn, int min_m, int max_m,
			   int min_fold_m, int max_fold_m,
			   jtutil::array1d<fp>& Jacobian_buffer)
		const;

	// compute the Jacobian of ghost zone's synchronize()
	// *without* taking into account 3-phase algorithm
	const patch_edge&
	  ghost_zone_Jacobian(const ghost_zone& xgz,
			      int x_iperp, int x_ipar,
			      int& y_iperp,
			      int& y_posn, int& min_ym, int& max_ym,
			      jtutil::array1d<fp>& Jacobian_buffer)
		const;


	//
	// ***** gridfn operations *****
	//
public:
	// dst = a
	void         set_gridfn_to_constant(fp a, int         dst_gfn);
	void set_ghosted_gridfn_to_constant(fp a, int ghosted_dst_gfn);

	// dst = src
	void         gridfn_copy(int         src_gfn, int         dst_gfn);
	void ghosted_gridfn_copy(int ghosted_src_gfn, int ghosted_dst_gfn);
	void ghosted_gridfn_copy_to_nominal(int ghosted_src_gfn, int dst_gfn);

	// dst += delta
	void         add_to_gridfn(fp delta, int         dst_gfn);
	void add_to_ghosted_gridfn(fp delta, int ghosted_dst_gfn);

	// dst *= factor
	void scale_ghosted_gridfn(fp factor, int ghosted_dst_gfn);

	// compute norms of gridfn (only over nominal grid)
	void         gridfn_norms(int         src_gfn, jtutil::norm<fp>& norms)
		const;
	void ghosted_gridfn_norms(int ghosted_src_gfn, jtutil::norm<fp>& norms)
		const;


	//
	// ***** testing (x,y,z) point position versus a surface *****
	//

	// find patch containing (ray from origin to) given local (x,y,z)
	// ... if there are multiple patches containing the position,
	//     we return the one which would still contain it if patches
	//     didn't overlap; if multiple patches satisfy this criterion
	//     then it's arbitrary which one we return
	// ... if no patch contains the position (for a non--full-sphere
	//     patch system), or the position is at the origin, then
	//     we return a NULL pointer
	const patch* patch_containing_local_xyz(fp x, fp y, fp z)
		const;

	// radius of surface in direction of an (x,y,z) point,
	// taking into account any patch-system symmetries;
	// or dummy value 1.0 if point is identical to local origin
	fp radius_in_local_xyz_direction(int ghosted_radius_gfn,
					 fp x, fp y, fp z)
		const;

	void radii_in_local_xyz_directions(int ghosted_radius_gfn,
                                           int npoints,
                                           fp const * xp,
                                           fp const * yp,
                                           fp const * zp,
                                           fp * radii)
		const;


	//
	// ***** line/surface operations *****
	//

	// compute the circumference of a surface in the {xy, xz, yz} plane
	// ... note this is the full circumference all around the sphere,
	//     even if the patch system only covers a proper subset of this
	// ... the implementation assumes adjacent patches are butt-joined
	// ... plane must be one of "xy", "xz", or "yz"
	fp circumference(const char plane[],
			 int ghosted_radius_gfn,
			 int g_xx_gfn, int g_xy_gfn, int g_xz_gfn,
				       int g_yy_gfn, int g_yz_gfn,
						     int g_zz_gfn,
			 enum patch::integration_method method)
		const;

	// compute the surface integral of a gridfn over the 2-sphere
	//	$\int f(\rho,\sigma) \, dA$
	//		= \int f(\rho,\sigma) \sqrt{|J|} \, d\rho \, d\sigma$
	// where $J$ is the Jacobian of $(x,y,z)$ with respect to $(rho,sigma)
	// ... integration method selected by  method  argument
	// ... src gridfn may be either nominal-grid or ghosted-grid
	// ... Boolean flags  src_gfn_is_even_across_{xy,xz,yz}_planes
	//     specify whether the gridfn to be integrated is even (true)
	//     or odd (false) across the corresponding planes.  Only the
	//     flags corresponding to boundaries of the patch system are
	//     used.  For example, for a  plus_z_hemisphere  patch system,
	//     only the  src_gfn_is_even_across_xy_plane  flag is used.
	// ... note integral is over the full 2-sphere,
	//     even if the patch system only covers a proper subset of this
	// ... the implementation assumes adjacent patches are butt-joined
	fp integrate_gridfn(int unknown_src_gfn,
			    bool src_gfn_is_even_across_xy_plane,
			    bool src_gfn_is_even_across_xz_plane,
			    bool src_gfn_is_even_across_yz_plane,
			    int ghosted_radius_gfn,
			    int g_xx_gfn, int g_xy_gfn, int g_xz_gfn,
					  int g_yy_gfn, int g_yz_gfn,
							int g_zz_gfn,
			    enum patch::integration_method method)
		const;

	fp integrate_gridpoint(int unknown_src_gfn,
                               int ghosted_radius_gfn,
                               int g_xx_gfn, int g_xy_gfn, int g_xz_gfn,
                                             int g_yy_gfn, int g_yz_gfn,
                                                           int g_zz_gfn,
                               enum patch::integration_method method,
                               int pn, int irho, int isigma)
		const;

	fp integrate_correction(bool src_gfn_is_even_across_xy_plane,
                                bool src_gfn_is_even_across_xz_plane,
                                bool src_gfn_is_even_across_yz_plane)
		const;



	//
	// ***** I/O *****
	//
public:
	// print to a named file (newly (re)created)
	// output format is
	//	dpx	dpy	gridfn
	void print_gridfn(int gfn, const char output_file_name[]) const
		{
		print_unknown_gridfn(false, gfn,
				     false, false, 0,
				     output_file_name, false);
		}
	void print_ghosted_gridfn(int ghosted_gfn,
				  const char output_file_name[],
				  bool want_ghost_zones = true)
		const
		{
		print_unknown_gridfn(true, ghosted_gfn,
				     false, false, 0,
				     output_file_name, want_ghost_zones);
		}

	// print to a named file (newly (re)created)
	// output format is
	//	dpx	dpy	gridfn   global_x   global_y   global_z
	// where global_[xyz} are derived from the angular position
	// and a specified (unknown-grid) radius gridfn
	void print_gridfn_with_xyz
		(int gfn,
		 bool radius_is_ghosted_flag, int unknown_radius_gfn,
		 const char output_file_name[])
		const
		{
		print_unknown_gridfn(false, gfn,
				     true, radius_is_ghosted_flag,
					   unknown_radius_gfn,
				     output_file_name, false);
		}
	void print_ghosted_gridfn_with_xyz
		(int ghosted_gfn,
		 bool radius_is_ghosted_flag, int unknown_radius_gfn,
		 const char output_file_name[],
		 bool want_ghost_zones = true)
		const
		{
		print_unknown_gridfn(true, ghosted_gfn,
				     true, radius_is_ghosted_flag,
					   unknown_radius_gfn,
				     output_file_name, want_ghost_zones);
		}

public:
	// read from a named file
	void read_gridfn(int gfn, const char input_file_name[])
		{ read_unknown_gridfn(false, gfn, input_file_name, false); }
	void read_ghosted_gridfn(int ghosted_gfn,
				 const char input_file_name[],
				 bool want_ghost_zones = true)
		{
		read_unknown_gridfn(true, ghosted_gfn,
				    input_file_name, want_ghost_zones);
		}

private:
	// ... internal worker functions
	void print_unknown_gridfn
		(bool ghosted_flag, int unknown_gfn,
		 bool print_xyz_flag, bool radius_is_ghosted_flag,
				      int unknown_radius_gfn,
		 const char output_file_name[], bool want_ghost_zones)
		const;
	void read_unknown_gridfn(bool ghosted_flag, int unknown_gfn,
				 const char input_file_name[],
				 bool want_ghost_zones);

public:
	// output to a named file
	// output format is HDF5
        void output_gridfn(int gfn,
                           const char gfn_name[], const cGH *cctkGH,
                           const char output_file_name[]) const
		{
		output_unknown_gridfn(false, gfn,
                                      gfn_name, cctkGH,
                                      false, false, 0,
                                      output_file_name, false);
		}
	void output_ghosted_gridfn(int ghosted_gfn,
                                   const char gfn_name[], const cGH *cctkGH,
                                   const char output_file_name[],
                                   bool want_ghost_zones = true)
		const
		{
		output_unknown_gridfn(true, ghosted_gfn,
                                      gfn_name, cctkGH,
                                      false, false, 0,
                                      output_file_name, want_ghost_zones);
		}

	// output to a named file (newly (re)created)
	// output format is
	//	dpx	dpy	gridfn   global_x   global_y   global_z
	// where global_[xyz} are derived from the angular position
	// and a specified (unknown-grid) radius gridfn
	void output_gridfn_with_xyz
		(int gfn,
                 const char gfn_name[], const cGH *cctkGH,
		 bool radius_is_ghosted_flag, int unknown_radius_gfn,
		 const char output_file_name[])
		const
		{
		output_unknown_gridfn(false, gfn,
                                      gfn_name, cctkGH,
                                      true, radius_is_ghosted_flag,
                                            unknown_radius_gfn,
                                      output_file_name, false);
		}
	void output_ghosted_gridfn_with_xyz
		(int ghosted_gfn,
                 const char gfn_name[], const cGH *cctkGH,
		 bool radius_is_ghosted_flag, int unknown_radius_gfn,
		 const char output_file_name[],
		 bool want_ghost_zones = true)
		const
		{
		output_unknown_gridfn(true, ghosted_gfn,
                                      gfn_name, cctkGH,
                                      true, radius_is_ghosted_flag,
                                            unknown_radius_gfn,
                                      output_file_name, want_ghost_zones);
		}

private:
	// ... internal worker functions
	void output_unknown_gridfn
		(bool ghosted_flag, int unknown_gfn,
                 const char gfn_name[], const cGH *cctkGH,
		 bool print_xyz_flag, bool radius_is_ghosted_flag,
				      int unknown_radius_gfn,
		 const char output_file_name[], bool want_ghost_zones)
		const;



	//
	// ***** access to gridfns as 1-D arrays *****
	//
	// ... n.b. this interface implicitly assumes that gridfn data
	//     arrays are contiguous across patches; this is ensured by
	//     setup_gridfn_storage() (called by our constructor)
	//
public:
	// convert (patch,irho,isigma) <--> 1-D 0-origin grid point number (gpn)
	int gpn_of_patch_irho_isigma(const patch& p, int irho, int isigma)
		const
		{
		return starting_gpn_[p.patch_number()]
		       + p.gpn_of_irho_isigma(irho,isigma);
		}
	int ghosted_gpn_of_patch_irho_isigma(const patch& p,
					     int irho, int isigma)
		const
		{
		return ghosted_starting_gpn_[p.patch_number()]
		       + p.ghosted_gpn_of_irho_isigma(irho,isigma);
		}
	// ... n.b. we return patch as a reference via the function result;
	//     an alternative would be to have a patch*& argument
	const patch&
	  patch_irho_isigma_of_gpn(int gpn, int& irho, int& isigma)
		const;
	const patch&
	  ghosted_patch_irho_isigma_of_gpn(int gpn, int& irho, int& isigma)
		const;

	// access actual gridfn data arrays
	// (low-level, dangerous, use with caution)
	const fp* gridfn_data(int gfn) const
		{ return ith_patch(0).gridfn_data_array(gfn); }
	      fp* gridfn_data(int gfn)
		{ return ith_patch(0).gridfn_data_array(gfn); }
	const fp* ghosted_gridfn_data(int ghosted_gfn) const
		{ return ith_patch(0).ghosted_gridfn_data_array(ghosted_gfn); }
	      fp* ghosted_gridfn_data(int ghosted_gfn)
		{ return ith_patch(0).ghosted_gridfn_data_array(ghosted_gfn); }


	//
	// ***** constructor, destructor *****
	//
	// This constructor doesn't support the full generality of the
	// patch data structures (which would, eg, allow ghost_zone_width
	// and patch_extend_width and the interpolator parameters to vary
	// from ghost zone to ghost zone, and the grid spacings to vary
	// from patch to patch.  But in practice we'd probably never
	// use that generality...
	//
public:
	patch_system(fp origin_x_in, fp origin_y_in, fp origin_z_in,
		     enum patch_system_type type_in,
		     int ghost_zone_width, int patch_overlap_width,
		     int N_zones_per_right_angle,
		     int min_gfn_in, int max_gfn_in,
		     int ghosted_min_gfn_in, int ghosted_max_gfn_in,
		     int ip_interp_handle_in, int ip_interp_par_table_handle_in,
		     int surface_interp_handle_in,
		     int surface_interp_par_table_handle_in,
		     bool print_summary_msg_flag, bool print_detailed_msg_flag);
	~patch_system();


	//
	// ***** helper functions for constructor *****
	//
private:
	// construct patches as described by patch_info[] array,
	// and link them into the patch system
	// does *NOT* create ghost zones
	// does *NOT* set up gridfns
	void create_patches(const struct patch_info patch_info_in[],
			    int ghost_zone_width, int patch_extend_width,
			    int N_zones_per_right_angle,
			    bool print_msg_flag);

	// setup all gridfns with contiguous-across-patches storage
	void setup_gridfn_storage
		(int min_gfn_in, int max_gfn_in,
		 int ghosted_min_gfn_in, int ghosted_max_gfn_in,
		 bool print_msg_flag);

	// setup (create/interlink) all ghost zones
	void setup_ghost_zones__full_sphere
		(int patch_overlap_width,
		 int ip_interp_handle, int ip_interp_par_table_handle,
		 bool print_msg_flag);
	void setup_ghost_zones__plus_z_hemisphere
		(int patch_overlap_width,
		 int ip_interp_handle, int ip_interp_par_table_handle,
		 bool print_msg_flag);
	void setup_ghost_zones__plus_xy_quadrant_mirrored
		(int patch_overlap_width,
		 int ip_interp_handle, int ip_interp_par_table_handle,
		 bool print_msg_flag);
	void setup_ghost_zones__plus_xy_quadrant_rotating
		(int patch_overlap_width,
		 int ip_interp_handle, int ip_interp_par_table_handle,
		 bool print_msg_flag);
	void setup_ghost_zones__plus_xz_quadrant_mirrored
		(int patch_overlap_width,
		 int ip_interp_handle, int ip_interp_par_table_handle,
		 bool print_msg_flag);
	void setup_ghost_zones__plus_xz_quadrant_rotating
		(int patch_overlap_width,
		 int ip_interp_handle, int ip_interp_par_table_handle,
		 bool print_msg_flag);
	void setup_ghost_zones__plus_xyz_octant_mirrored
		(int patch_overlap_width,
		 int ip_interp_handle, int ip_interp_par_table_handle,
		 bool print_msg_flag);
	void setup_ghost_zones__plus_xyz_octant_rotating
		(int patch_overlap_width,
		 int ip_interp_handle, int ip_interp_par_table_handle,
		 bool print_msg_flag);

	// create/interlink a pair of periodic-symmetry ghost zones
	static
	  void create_periodic_symmetry_ghost_zones
		(const patch_edge& ex, const patch_edge& ey,
		 bool ipar_map_is_plus);

	// construct a pair of interpatch ghost zones
	// ... automagically figures out which edges are adjacent
	static
	  void create_interpatch_ghost_zones(patch &px, patch &py,
					     int patch_overlap_width);

	// finish setup of a pair of interpatch ghost zones
	// ... automagically figures out which edges are adjacent
	static
	  void finish_interpatch_setup
		(patch &px, patch &py,
		 int patch_overlap_width,
		 int ip_interp_handle, int ip_interp_par_table_handle);

	// assert() that all ghost zones of all patches are fully setup
	void assert_all_ghost_zones_fully_setup() const;


private:
        // we forbid copying and passing by value
        // by declaring the copy constructor and assignment operator
        // private, but never defining them
	patch_system(const patch_system &rhs);
	patch_system& operator=(const patch_system &rhs);

private:
	// local <--> global coordinate mapping
	global_coords global_coords_;

	// meta-info about patch system
	enum patch_system_type type_;
	int N_patches_;
	int N_grid_points_, ghosted_N_grid_points_;
        int max_N_additional_points_;
        int N_additional_points_;

	// [pn] = --> individual patches
	// *** constructor initialization list ordering:
	// *** this must be declared after  N_patches_
	patch** all_patches_;

	// [pn] = starting grid point number of individual patches
	// ... arrays are actually of size N_patches_+1, the [N_patches_]
	//     entries are == N_grid_points_ and ghosted_N_grid_points_
	// *** constructor initialization list ordering:
	// *** these must be declared after  N_patches_
	int* starting_gpn_;
	int* ghosted_starting_gpn_;

	// pointers to storage blocks for all gridfns
	// ... patches point into these, but we own the storage blocks
	fp* gridfn_storage_;
	fp* ghosted_gridfn_storage_;

	// min/max m over all ghost zone points
        mutable int global_min_ym_;
	mutable int global_max_ym_;

	// info about the surface interpolator
	// ... used only by radius_in_local_xyz_direction()
	int surface_interp_handle_, surface_interp_par_table_handle_;
	};

//******************************************************************************

	  }	// namespace AHFinderDirect

```

patch/test_coords2.cc:
```
// test_coords2.cc -- test driver #2 for coordinate systems/conversions
// $Header$

#include <stdio.h>
#include <assert.h>
#include <math.h>
#include <string.h>

#ifdef STANDALONE_TEST
  #include "fake_cctk.h"
#else
  #include "cctk.h"
#endif

#include "config.h"
#include "stdc.h"
#include "../jtutil/util.hh"

#include "coords.hh"

using jtutil::fuzzy;
using jtutil::error_exit;
using jtutil::radians_of_degrees;
using jtutil::degrees_of_radians;

using namespace AHFinderDirect;
using namespace local_coords;

// prototypes
namespace {
void test_r_mu_nu_phi(fp x, fp y, fp z);
void test_r_theta_phi(fp x, fp y, fp z);
	  }

//******************************************************************************

//
// This program is a test driver for the local_coords:: coordinate-conversion
// functions.  It tries a large number of coordinate conversions, and checks
// them all for mutual consistency.
//
int main(int argc, const char* argv[])
{
bool verbose_flag = (argc == 2) && STRING_EQUAL(argv[1], "--verbose");
const fp xyz_minmax = 1.0;
const fp xyz_delta = 0.1;

	// these loops go *down* so we start in (+,+,+) quadrant
	// ==> get some successful tests before hard stuff
	// ==> help test the tests themselves
	for (fp x = xyz_minmax ; fuzzy<fp>::GE(x,-xyz_minmax) ; x -= xyz_delta)
	{
	for (fp y = xyz_minmax ; fuzzy<fp>::GE(y,-xyz_minmax) ; y -= xyz_delta)
	{
	for (fp z = xyz_minmax ; fuzzy<fp>::GE(z,-xyz_minmax) ; z -= xyz_delta)
	{
	// avoid places where angular coords might not be defined
	if (    fuzzy<fp>::EQ(x,0.0)
	     || fuzzy<fp>::EQ(y,0.0)
	     || fuzzy<fp>::EQ(z,0.0)    )
	   then continue;

	if (verbose_flag)
	   then printf("testing x=%g y=%g z=%g\n", x,y,z);

	test_r_mu_nu_phi(x, y, z);
	test_r_theta_phi(x, y, z);
	}
	}
	}

printf("all ok!\n");
return 0;
}

//******************************************************************************

namespace {
void test_r_mu_nu_phi(fp x, fp y, fp z)
{
const fp r = r_of_xyz(x, y, z);
const fp mu  =  mu_of_yz(y, z);
const fp nu  =  nu_of_xz(x, z);
const fp phi = phi_of_xy(x, y);

const fp  mu2 = mu_of_nu_phi(nu,phi);
const fp  nu2 = nu_of_mu_phi(mu,phi);
const fp phi2 = phi_of_mu_nu(mu,nu );
assert( fuzzy_EQ_ang(mu , mu2 ) );
assert( fuzzy_EQ_ang(nu , nu2 ) );
assert( fuzzy_EQ_ang(phi, phi2) );

fp x2, y2, z2;
xyz_of_r_mu_nu(r,mu,nu, x2,y2,z2);
assert( fuzzy<fp>::EQ(x, x2) );
assert( fuzzy<fp>::EQ(y, y2) );
assert( fuzzy<fp>::EQ(z, z2) );

fp x3, y3, z3;
xyz_of_r_mu_phi(r,mu,phi, x3,y3,z3);
assert( fuzzy<fp>::EQ(x, x3) );
assert( fuzzy<fp>::EQ(y, y3) );
assert( fuzzy<fp>::EQ(z, z3) );

fp x4, y4, z4;
xyz_of_r_nu_phi(r,nu,phi, x4,y4,z4);
assert( fuzzy<fp>::EQ(x, x4) );
assert( fuzzy<fp>::EQ(y, y4) );
assert( fuzzy<fp>::EQ(z, z4) );

}
	  }

//******************************************************************************

namespace {
void test_r_theta_phi(fp x, fp y, fp z)
{
const fp r        =     r_of_xyz(x, y, z);
const fp ps_theta = theta_of_xyz(x, y, z);
const fp ps_phi   =   phi_of_xy (x, y);

fp x2, y2, z2;
xyz_of_r_theta_phi(r,ps_theta,ps_phi, x2,y2,z2);
assert( fuzzy<fp>::EQ(x, x2) );
assert( fuzzy<fp>::EQ(y, y2) );
assert( fuzzy<fp>::EQ(z, z2) );

fp mu3, nu3;
mu_nu_of_theta_phi(ps_theta,ps_phi, mu3,nu3);
fp ps_theta3, ps_phi3;
theta_phi_of_mu_nu(mu3,nu3, ps_theta3,ps_phi3);
assert( fuzzy_EQ_ang(ps_theta3, ps_theta) );
assert( fuzzy_EQ_ang(ps_phi3  , ps_phi  ) );

fp mu4, phi4;
mu_phi_of_theta_phi(ps_theta,ps_phi, mu4,phi4);
fp ps_theta4, ps_phi4;
theta_phi_of_mu_phi(mu4,phi4, ps_theta4,ps_phi4);
assert( fuzzy_EQ_ang(ps_theta4, ps_theta) );
assert( fuzzy_EQ_ang(ps_phi4  , ps_phi  ) );

fp nu5, phi5;
nu_phi_of_theta_phi(ps_theta,ps_phi, nu5,phi5);
fp ps_theta5, ps_phi5;
theta_phi_of_nu_phi(nu5,phi5, ps_theta5,ps_phi5);
assert( fuzzy_EQ_ang(ps_theta5, ps_theta) );
assert( fuzzy_EQ_ang(ps_phi5  , ps_phi  ) );
}
	  }

```

patch/make.code.defn:
```
# Cactus specification of things to be compiled in this directory
# $Header$

# Source files in this directory
# ... uncomment test_patch_system.cc to test the patch system
SRCS = coords.cc \
       grid.cc \
       fd_grid.cc \
       patch.cc \
       ghost_zone.cc \
       patch_interp.cc \
       patch_info.cc \
       patch_system.cc \
       # test_patch_system.cc

# Subdirectories containing source files
SUBDIRS =

```

patch/grid.hh:
```
// grid.hh -- classes for a 2D uniform tensor-product grid
// $Header$
//
// grid_arrays - data arrays for a 2D tensor-product grid
// grid - uniform 2D tensor-product grid
//

//
// prerequisites:
//    <stdio.h>
//    <assert.h>
//    <math.h>
//    "cctk.h" or "fake_cctk.h"
//    "stdc.h"
//    "config.hh"
//    "../jtutil/util.hh"
//    "../jtutil/array.hh"
//    "../jtutil/linear_map.hh"
//    "coords.hh"
//

// everything in this file is inside this namespace
namespace AHFinderDirect
	  {

//*****************************************************************************

//
// grid_arrays - data arrays for a 2D tensor-product grid
//
// This is a helper class for class grid (below).  This class stores
// most of the actual grid function (gridfn) data arrays for a uniform
// tensor-product 2D grid.
//
// The integer grid coordinates are (irho,isigma).  This class deals
// with the grid solely at the level of arrays with integer subscripts;
// the derived class  grid  deals with the floating-point coordinates
// related to those subscripts.
//
// The grid has a nominal extent, surrounded by "ghost zones" on each
// side for finite differencing purposes.
//
// There are separate sets of nominal-grid and ghosted-grid gridfns.
// We identify a gridfn by a small-integer "grid function number", a.k.a.
// "gfn".  There are separate gfns for nominal and ghosted gridfns.
// In a very few places we refer to "unknown-grid" gridfns; these might
// be either nominal-grid or ghosted-grid.
//
// For our application (apparent horizon finding), it's useful for the
// storage for a single gridfn to be contiguous *across all patches*.
// (Note this means that the set of all our gridfns is *not* contiguous!)
// To accomplish this, we don't allocate the gridfns when we're created,
// but rather later, with a separate call  setup_gridfn_storage() .
// This way higher-level code can first create all patches, then count
// the total amount of storage used, allocate it, then finally call each
// patch again to set up its gridfns appropriately.
//

class	grid_arrays
	{
public:
	//
	// ***** {min,max}_{rho,sigma} "sides" of grid *****
	//

	//
	// A grid has 4 (angular) "sides", which we identify as
	// {min,max}_{rho,sigma}.  Given a side, we define coordinates
	// (perpendicular,parallel) to it, normally abbreviated to
	// (perp,par).
	//
	// As well as functions directly referring to a specific side,
	// we also support referring to one of these chosen at run-time,
	// via Boolean flags:
	//
	//	// generic (irho,isigma) coordinate
	//	iang = want_rho ? irho : isigma
	//
	//	// opposite (irho,isigma) coordinate
	//	ixang = want_rho ? isigma : irho
	//
	//	// generic (min,max) direction
	//	minmax = want_min ? min : max
	//
	// FIXME: This system of Boolean flags works ok, but it requires
	//	  a lot of repetitive code conditional-expression functions
	//	  in this class.  Is there a cleaner solution?

	// there are precisely this many possible sides
	enum { N_sides = 4 };

	// we specify {min,max} with a Boolean  want_min
	// ... values for want_min
	//     FIXME: these should really be bool, but then we couldn't
	//            use the "enum hack" for in-class constants
	enum { side_is_min = true, side_is_max = false };

	// we specify {rho,sigma} with a Boolean  want_rho
	// ... values for wanr_rho
	//     FIXME: these should really be bool, but then we couldn't
	//            use the "enum hack" for in-class constants
	enum { side_is_rho = true, side_is_sigma = false };

	// human-readable names for the sides (for debugging)
	static const char* minmax_name(bool minmax)
		{ return minmax ? "min" : "max"; }
	static const char* iang_name(bool want_rho)
		{ return want_rho ? "irho" : "isigma"; }


	//
	// ***** array info *****
	//
public:

	// nominal-grid min/max/sizes
	int min_irho() const { return min_irho_; }
	int max_irho() const { return max_irho_; }
	int min_isigma() const { return min_isigma_; }
	int max_isigma() const { return max_isigma_; }
	int min_iang(bool want_rho) const
		{ return want_rho ? min_irho() : min_isigma(); }
	int max_iang(bool want_rho) const
		{ return want_rho ? max_irho() : max_isigma(); }
	int minmax_iang(bool want_min, bool want_rho) const
		{ return want_min ? min_iang(want_rho) : max_iang(want_rho); }
	int N_irho() const
		{ return jtutil::how_many_in_range(min_irho(), max_irho()); }
	int N_isigma() const
		{
		return jtutil::how_many_in_range(min_isigma(), max_isigma());
		}
	int N_grid_points() const
		{ return N_irho() * N_isigma(); }

	// ghosted-grid min/max/sizes
	int ghosted_min_irho() const { return ghosted_min_irho_; }
	int ghosted_max_irho() const { return ghosted_max_irho_; }
	int ghosted_min_isigma() const
		{ return ghosted_min_isigma_; }
	int ghosted_max_isigma() const
		{ return ghosted_max_isigma_; }
	int ghosted_min_iang(bool want_rho) const
		{
		return want_rho ? ghosted_min_irho()
				: ghosted_min_isigma();
		}
	int ghosted_max_iang(bool want_rho) const
		{
		return want_rho ? ghosted_max_irho()
				: ghosted_max_isigma();
		}
	int ghosted_minmax_iang(bool want_min, bool want_rho) const
		{
		return want_min ? ghosted_min_iang(want_rho)
				: ghosted_max_iang(want_rho);
		}
	int ghosted_N_irho() const
		{
		return jtutil::how_many_in_range(ghosted_min_irho(),
						 ghosted_max_irho());
		}
	int ghosted_N_isigma() const
		{
		return jtutil::how_many_in_range(ghosted_min_isigma(),
						 ghosted_max_isigma());
		}
	int ghosted_N_grid_points() const
		{ return ghosted_N_irho() * ghosted_N_isigma(); }

	// "effective" grid min/max/sizes
	// (= dynamic select between nominal and full grids)
	int effective_min_irho(bool want_ghost_zones) const
		{
		return want_ghost_zones ? ghosted_min_irho() : min_irho();
		}
	int effective_max_irho(bool want_ghost_zones) const
		{
		return want_ghost_zones ? ghosted_max_irho() : max_irho();
		}
	int effective_min_isigma(bool want_ghost_zones) const
		{
		return want_ghost_zones ? ghosted_min_isigma() : min_isigma();
		}
	int effective_max_isigma(bool want_ghost_zones) const
		{
		return want_ghost_zones ? ghosted_max_isigma() : max_isigma();
		}
	int effective_N_irho(bool want_ghost_zones) const
		{ return want_ghost_zones ? ghosted_N_irho() : N_irho(); }
	int effective_N_isigma(bool want_ghost_zones) const
		{ return want_ghost_zones ? ghosted_N_isigma() : N_isigma(); }


	//
	// ***** ghost zones *****
	//
public:

	// ghost zone min/max perpendicular coordinates
	int min_rho_ghost_zone__min_iperp() const
		{ return ghosted_min_irho(); }
	int min_rho_ghost_zone__max_iperp() const
		{ return min_irho() - 1; }
	int max_rho_ghost_zone__min_iperp() const
		{ return max_irho() + 1; }
	int max_rho_ghost_zone__max_iperp() const
		{ return ghosted_max_irho(); }
	int min_sigma_ghost_zone__min_iperp() const
		{ return ghosted_min_isigma(); }
	int min_sigma_ghost_zone__max_iperp() const
		{ return min_isigma() - 1; }
	int max_sigma_ghost_zone__min_iperp() const
		{ return max_isigma() + 1; }
	int max_sigma_ghost_zone__max_iperp() const
		{ return ghosted_max_isigma(); }
	int minmax_ang_ghost_zone__min_iperp(bool want_min, bool want_rho) const
		{
		return want_min
		       ? (want_rho ? min_rho_ghost_zone__min_iperp()
				   : min_sigma_ghost_zone__min_iperp())
		       : (want_rho ? max_rho_ghost_zone__min_iperp()
				   : max_sigma_ghost_zone__min_iperp());
		}
	int minmax_ang_ghost_zone__max_iperp(bool want_min, bool want_rho) const
		{
		return want_min
		       ? (want_rho ? min_rho_ghost_zone__max_iperp()
				   : min_sigma_ghost_zone__max_iperp())
		       : (want_rho ? max_rho_ghost_zone__max_iperp()
				   : max_sigma_ghost_zone__max_iperp());
		}

	// ghost zone min/max parallel coordinates
	// ... not including corners
	int rho_ghost_zone_without_corners__min_ipar() const
		{ return min_isigma(); }
	int rho_ghost_zone_without_corners__max_ipar() const
		{ return max_isigma(); }
	int sigma_ghost_zone_without_corners__min_ipar() const
		{ return min_irho(); }
	int sigma_ghost_zone_without_corners__max_ipar() const
		{ return max_irho(); }
	int ang_ghost_zone_without_corners__min_ipar(bool want_rho) const
		{
		return want_rho ?   rho_ghost_zone_without_corners__min_ipar()
				: sigma_ghost_zone_without_corners__min_ipar();
		}
	int ang_ghost_zone_without_corners__max_ipar(bool want_rho) const
		{
		return want_rho ?   rho_ghost_zone_without_corners__max_ipar()
				: sigma_ghost_zone_without_corners__max_ipar();
		}
	// ... including corners
	int rho_ghost_zone_with_corners__min_ipar() const
		{ return ghosted_min_isigma(); }
	int rho_ghost_zone_with_corners__max_ipar() const
		{ return ghosted_max_isigma(); }
	int sigma_ghost_zone_with_corners__min_ipar() const
		{ return ghosted_min_irho(); }
	int sigma_ghost_zone_with_corners__max_ipar() const
		{ return ghosted_max_irho(); }
	int ang_ghost_zone_with_corners__min_ipar(bool want_rho) const
		{
		return want_rho ?   rho_ghost_zone_with_corners__min_ipar()
				: sigma_ghost_zone_with_corners__min_ipar();
		}
	int ang_ghost_zone_with_corners__max_ipar(bool want_rho) const
		{
		return want_rho ?   rho_ghost_zone_with_corners__max_ipar()
				: sigma_ghost_zone_with_corners__max_ipar();
		}


	//
	// ***** grid-point validity and membership predicates *****
	//
public:
	bool is_valid_irho(int irho) const
		{ return (irho >= min_irho()) && (irho <= max_irho()); }
	bool is_valid_isigma(int isigma) const
		{ return (isigma >= min_isigma()) && (isigma <= max_isigma()); }
	bool is_in_nominal_grid(int irho, int isigma) const
		{ return is_valid_irho(irho) && is_valid_isigma(isigma); }

	bool is_valid_ghosted_irho(int irho) const
		{
		return    (irho >= ghosted_min_irho())
		       && (irho <= ghosted_max_irho());
		}
	bool is_valid_ghosted_isigma(int isigma) const
		{
		return    (isigma >= ghosted_min_isigma())
		       && (isigma <= ghosted_max_isigma());
		}
	bool is_in_ghosted_grid(int irho, int isigma) const
		{
		return    is_valid_ghosted_irho(irho)
		       && is_valid_ghosted_isigma(isigma);
		}

	bool is_in_ghost_zone(int irho, int isigma) const
		{
		return     is_in_ghosted_grid(irho, isigma)
		       && !is_in_nominal_grid(irho, isigma);
		}


	//
	// ***** gfn ranges and validity predicates *****
	//
public:
	// gfn ranges
	int min_gfn() const
		{
		assert(gridfn_data_ != NULL);
		return (*gridfn_data_).min_i();
		}
	int max_gfn() const
		{
		assert(gridfn_data_ != NULL);
		return (*gridfn_data_).max_i();
		}
	int N_gridfns() const
		{ return jtutil::how_many_in_range(min_gfn(), max_gfn()); }
	int ghosted_min_gfn() const
		{
		assert(ghosted_gridfn_data_ != NULL);
		return (*ghosted_gridfn_data_).min_i();
		}
	int ghosted_max_gfn() const
		{
		assert(ghosted_gridfn_data_ != NULL);
		return (*ghosted_gridfn_data_).max_i();
		}
	int ghosted_N_gridfns() const
		{
		return jtutil::how_many_in_range(ghosted_min_gfn(),
						 ghosted_max_gfn());
		}

	// gfn validity predicates
	bool is_valid_gfn(int gfn) const
		{ return (gfn >= min_gfn()) && (gfn <= max_gfn()); }
	bool is_valid_ghosted_gfn(int gfn) const
		{
		return (gfn >= ghosted_min_gfn()) && (gfn <= ghosted_max_gfn());
		}


	//
	// ***** gridfns *****
	//
	// n.b. access to rvalue gridfn data must be via references
	//	in order to allow using  gridfn(...)  as the operand
	//	of a unary & (address-of) operator
	//
public:
	// access to nominal-grid gridfn data
	// ... rvalue
	const fp& gridfn(int gfn,   int irho, int isigma) const
		{
		assert(gridfn_data_ != NULL);
		return (*gridfn_data_)(gfn, irho, isigma);
		}
	// ... lvalue
	      fp& gridfn(int gfn,   int irho, int isigma)
		{
		assert(gridfn_data_ != NULL);
		return (*gridfn_data_)(gfn, irho, isigma);
		}

	// access to ghosted-grid gridfn data
	// ... rvalue
	const fp& ghosted_gridfn(int gfn,   int irho, int isigma) const
		{
		assert(gridfn_data_ != NULL);
		return (*ghosted_gridfn_data_)(gfn, irho, isigma);
		}
	// ... lvalue
	      fp& ghosted_gridfn(int gfn,   int irho, int isigma)
		{
		assert(gridfn_data_ != NULL);
		return (*ghosted_gridfn_data_)(gfn, irho, isigma);
		}

	// access to unknown-grid gridfn data
	// (either nominal or ghosted, depending on Boolean flag)
	// ... rvalue
	const fp& unknown_gridfn(bool ghosted_flag,
				 int unknown_gfn,   int irho, int isigma)
		const
		{
		return ghosted_flag ? ghosted_gridfn(unknown_gfn, irho,isigma)
				    :         gridfn(unknown_gfn, irho,isigma);
		}
	// ... lvalue
	      fp& unknown_gridfn(bool ghosted_flag,
				 int unknown_gfn,   int irho, int isigma)
		{
		return ghosted_flag ? ghosted_gridfn(unknown_gfn, irho,isigma)
				    :         gridfn(unknown_gfn, irho,isigma);
		}

	// subscripting info
	int gfn_stride() const
		{
		assert(gridfn_data_ != NULL);
		return gridfn_data_->subscript_stride_i();
		}
	int irho_stride() const
		{
		assert(gridfn_data_ != NULL);
		return gridfn_data_->subscript_stride_j();
		}
	int isigma_stride() const
		{
		assert(gridfn_data_ != NULL);
		return gridfn_data_->subscript_stride_k();
		}
	int iang_stride(bool want_rho) const
		{ return want_rho ? irho_stride() : isigma_stride(); }
	int ghosted_gfn_stride() const
		{
		assert(ghosted_gridfn_data_ != NULL);
		return ghosted_gridfn_data_->subscript_stride_i();
		}
	int ghosted_irho_stride() const
		{
		assert(ghosted_gridfn_data_ != NULL);
		return ghosted_gridfn_data_->subscript_stride_j();
		}
	int ghosted_isigma_stride() const
		{
		assert(ghosted_gridfn_data_ != NULL);
		return ghosted_gridfn_data_->subscript_stride_k();
		}
	int ghosted_iang_stride(bool want_rho) const
		{
		return want_rho ? ghosted_irho_stride()
				: ghosted_isigma_stride();
		}

	// validity predicates for 1-D 0-origin grid point number (gpn)
	bool is_valid_gpn(int gpn) const
		{ return (gpn >= 0) && (gpn < N_grid_points()); }
	bool is_valid_ghosted_gpn(int gpn) const
		{ return (gpn >= 0) && (gpn < ghosted_N_grid_points()); }

	// convert (irho,isigma) <--> 1-D 0-origin grid point number (gpn)
	int gpn_of_irho_isigma(int irho, int isigma) const
		{
		assert( is_valid_irho(irho) );
		assert( is_valid_isigma(isigma) );
		return   (irho   - min_irho()  ) * irho_stride()
		       + (isigma - min_isigma()) * isigma_stride();
		}
	int ghosted_gpn_of_irho_isigma(int irho, int isigma) const
		{
		assert( is_valid_ghosted_irho(irho) );
		assert( is_valid_ghosted_isigma(isigma) );
		return
		     (irho   - ghosted_min_irho()  ) * ghosted_irho_stride()
		   + (isigma - ghosted_min_isigma()) * ghosted_isigma_stride();
		}
	// ... current implementation assumes (& verifies) isigma is contiguous
	void irho_isigma_of_gpn(int gpn, int& irho, int& isigma) const
		{
		assert( is_valid_gpn(gpn) );
		assert( isigma_stride() == 1 );	// implementation restriction
		irho   = min_irho()   + gpn / N_isigma();
		isigma = min_isigma() + gpn % N_isigma();
		assert( is_valid_irho(irho) );
		assert( is_valid_isigma(isigma) );
		}
	// ... current implementation assumes (& verifies) isigma is contiguous
	void ghosted_irho_isigma_of_gpn(int gpn, int& irho, int& isigma) const
		{
		assert( is_valid_ghosted_gpn(gpn) );
		assert( ghosted_isigma_stride() == 1 );	// implementation
							// restriction
		irho   = ghosted_min_irho()   + gpn / ghosted_N_isigma();
		isigma = ghosted_min_isigma() + gpn % ghosted_N_isigma();
		assert( is_valid_ghosted_irho(irho) );
		assert( is_valid_ghosted_isigma(isigma) );
		}

	// low-level access to data arrays (!!dangerous!!)
	const fp* gridfn_data_array(int gfn) const
		{ return & gridfn(gfn, min_irho(),min_isigma()); }
	      fp* gridfn_data_array(int gfn)
		{ return & gridfn(gfn, min_irho(),min_isigma()); }
	const fp* ghosted_gridfn_data_array(int ghosted_gfn) const
		{
		return & ghosted_gridfn(ghosted_gfn, ghosted_min_irho(),
						     ghosted_min_isigma());
		}
	      fp* ghosted_gridfn_data_array(int ghosted_gfn)
		{
		return & ghosted_gridfn(ghosted_gfn, ghosted_min_irho(),
						     ghosted_min_isigma());
		}


	//
	// ***** argument structures for constructor et al *****
	//
public:
	// these structures bundle related arguments together so we don't
	// have 20+ (!) separate arguments to our top-level constructors
	struct	grid_array_pars
		{
		int min_irho,   max_irho;
		int min_isigma, max_isigma;
		int min_rho_ghost_zone_width,   max_rho_ghost_zone_width;
		int min_sigma_ghost_zone_width, max_sigma_ghost_zone_width;
		};
	struct	gridfn_pars
		{
		int min_gfn, max_gfn;

		// gridfn storage will be automatically allocated
		// if pointer is NULL; any 0 strides are automatically
		// set to C-style row-major subscripting
		fp* storage_array;
		int gfn_stride, irho_stride, isigma_stride;
		};

	//
	// ***** constructor, gridfn setup, destructor *****
	//
public:
	// construct with no gridfns
	grid_arrays(const grid_array_pars& grid_array_pars_in);

	// set up storage for gridfns
	void setup_gridfn_storage(const gridfn_pars& gridfn_pars_in,
				  const gridfn_pars& ghosted_gridfn_pars_in);

	~grid_arrays();

private:
	// we forbid copying and passing by value
	// by declaring the copy constructor and assignment operator
	// private, but never defining them
	grid_arrays(const grid_arrays& rhs);
	grid_arrays& operator=(const grid_arrays& rhs);

private:
	//
	// ***** the actual gridfn storage arrays *****
	//
	// n.b. these pointers are *first* data member in this class
	// ==> possibly slightly faster access (0 offset from pointer)
	// ... indices are (gfn, irho, isigma)
	jtutil::array3d<fp>* gridfn_data_;
	jtutil::array3d<fp>* ghosted_gridfn_data_;

	// gfn bounds
	const int min_gfn_,         max_gfn_;
	const int ghosted_min_gfn_, ghosted_max_gfn_;

	// nominal grid min/max bounds
	const int min_irho_,   max_irho_;
	const int min_isigma_, max_isigma_;

	// full grid min/max bounds
	const int ghosted_min_irho_,   ghosted_max_irho_;
	const int ghosted_min_isigma_, ghosted_max_isigma_;
	};

//******************************************************************************

//
// grid - uniform 2D tensor-product grid
//
// The grid is uniform in the floating point grid coordinates (rho,sigma).
// There is also some (limited) support for expressing these coordinates
// in degrees (drho,dsigma); this is useful for humans trying to specify
// things in parameter files.
//
// The nominal (not including the ghost zones) angular grid boundaries
// may coincide with grid points, or they may be at "half-integer" grid
// coordinates.  That is, suppose we have a unit grid spacing, and a boundary
// at an angular coordinate of 0; then the grid may be either 0, 1, 2, ...,
// or 0.5, 1.5, 2.5, ... .
//

class	grid
	: public grid_arrays
	{
	//
	// ***** low-level access to coordinate maps *****
	//
public:
	// direct (read-only) access to the underlying linear_map objects
	// ... useful for (eg) passing to interpolators
	const jtutil::linear_map<fp>& rho_map() const { return rho_map_; }
	const jtutil::linear_map<fp>& sigma_map() const { return sigma_map_; }
	const jtutil::linear_map<fp>& ang_map(bool want_rho) const
		{ return want_rho ? rho_map() : sigma_map(); }


	//
	// ***** single-axis coordinate conversions *****
	//
public:
	// ... angles in radians
	fp rho_of_irho(int irho) const { return rho_map().fp_of_int(irho); }
	fp sigma_of_isigma(int isigma) const
		{ return sigma_map().fp_of_int(isigma); }
	fp ang_of_iang(bool want_rho, int iang) const
		{
		return want_rho ? rho_of_irho(iang)
				: sigma_of_isigma(iang);
		}

	fp fp_irho_of_rho(fp rho) const
		{ return rho_map().fp_int_of_fp(rho); }
	int irho_of_rho(fp rho, jtutil::linear_map<fp>::noninteger_action
				nia = jtutil::linear_map<fp>::nia_error)
		const
		{ return rho_map().int_of_fp(rho, nia); }
	fp fp_isigma_of_sigma(fp sigma) const
		{ return sigma_map().fp_int_of_fp(sigma); }
	int isigma_of_sigma(fp sigma, jtutil::linear_map<fp>::noninteger_action
				      nia = jtutil::linear_map<fp>::nia_error)
		const
		{ return sigma_map().int_of_fp(sigma, nia); }
	fp fp_iang_of_ang(bool want_rho, fp ang)
		const
		{
		return want_rho ? fp_irho_of_rho(ang)
				: fp_isigma_of_sigma(ang);
		}
	int iang_of_ang(bool want_rho,
			fp ang, jtutil::linear_map<fp>::noninteger_action
				nia = jtutil::linear_map<fp>::nia_error)
		const
		{
		return want_rho ? irho_of_rho(ang, nia)
				: isigma_of_sigma(ang, nia);
		}

	// ... angles in degrees
	fp rho_of_drho(fp drho) const
		{ return jtutil::radians_of_degrees(drho); }
	fp sigma_of_dsigma(fp dsigma) const
		{ return jtutil::radians_of_degrees(dsigma); }
	fp drho_of_rho(fp rho) const
		{ return jtutil::degrees_of_radians(rho); }
	fp dsigma_of_sigma(fp sigma) const
		{ return jtutil::degrees_of_radians(sigma); }
	fp drho_of_irho(int irho) const
		{ return jtutil::degrees_of_radians(rho_of_irho(irho)); }
	fp dsigma_of_isigma(int isigma) const
		{ return jtutil::degrees_of_radians(sigma_of_isigma(isigma)); }

	int irho_of_drho(fp drho, jtutil::linear_map<fp>::noninteger_action
				  nia = jtutil::linear_map<fp>::nia_error)
		const
		{ return irho_of_rho(jtutil::radians_of_degrees(drho), nia); }
	int isigma_of_dsigma(fp dsigma,
			     jtutil::linear_map<fp>::noninteger_action
			        nia = jtutil::linear_map<fp>::nia_error)
		const
		{
		return isigma_of_sigma(jtutil::radians_of_degrees(dsigma), nia);
		}


	//
	// ***** grid info *****
	//
public:
	// grid spacings
	fp delta_rho() const { return rho_map().delta_fp(); }
	fp delta_sigma() const { return sigma_map().delta_fp(); }
	fp delta_drho() const
		{ return jtutil::degrees_of_radians(delta_rho()); }
	fp delta_dsigma() const
		{ return jtutil::degrees_of_radians(delta_sigma()); }
	fp delta_ang(bool want_rho) const
		{ return want_rho ? delta_rho() : delta_sigma(); }
	fp delta_dang(bool want_rho) const
		{ return want_rho ? delta_drho() : delta_dsigma(); }

	// inverse grid spacings
	fp inverse_delta_rho() const { return rho_map().inverse_delta_fp(); }
	fp inverse_delta_sigma() const
		{ return sigma_map().inverse_delta_fp(); }

	// nominal grid min/max
	fp min_rho() const { return min_rho_; }
	fp max_rho() const { return max_rho_; }
	fp min_sigma() const { return min_sigma_; }
	fp max_sigma() const { return max_sigma_; }
	fp minmax_ang(bool want_min, bool want_rho) const
		{
		return want_min ? (want_rho ? min_rho() : min_sigma())
				: (want_rho ? max_rho() : max_sigma());
		}
	fp min_drho() const { return jtutil::degrees_of_radians(min_rho()); }
	fp max_drho() const { return jtutil::degrees_of_radians(max_rho()); }
	fp min_dsigma() const
		{ return jtutil::degrees_of_radians(min_sigma()); }
	fp max_dsigma() const
		{ return jtutil::degrees_of_radians(max_sigma()); }
	fp min_dang(bool want_rho) const
		{ return want_rho ? min_drho() : min_dsigma(); }
	fp max_dang(bool want_rho) const
		{ return want_rho ? max_drho() : max_dsigma(); }

	// ghosted-grid min/max
	fp ghosted_min_rho() const
		{ return rho_of_irho(ghosted_min_irho()); }
	fp ghosted_max_rho() const
		{ return rho_of_irho(ghosted_max_irho()); }
	fp ghosted_min_sigma() const
		{ return sigma_of_isigma(ghosted_min_isigma()); }
	fp ghosted_max_sigma() const
		{ return sigma_of_isigma(ghosted_max_isigma()); }

	// is a given (drho,dsigma) within the grid?
	bool is_valid_drho(fp drho) const
		{
		return    jtutil::fuzzy<fp>::GE(drho, min_drho())
		       && jtutil::fuzzy<fp>::LE(drho, max_drho());
		}
	bool is_valid_dsigma(fp dsigma) const
		{
		return    jtutil::fuzzy<fp>::GE(dsigma, min_dsigma())
		       && jtutil::fuzzy<fp>::LE(dsigma, max_dsigma());
		}

	// reduce a rho/sigma coordinate modulo 2*pi radians (360 degrees)
	// to be within the ghosted grid,
	// or error_exit() if no such value exists
	fp modulo_reduce_rho(fp rho_in) const
		{
		return local_coords
		       ::modulo_reduce_ang(rho_in, ghosted_min_rho(),
						   ghosted_max_rho());
		}
	fp modulo_reduce_sigma(fp sigma_in) const
		{
		return local_coords
		       ::modulo_reduce_ang(sigma_in, ghosted_min_sigma(),
						     ghosted_max_sigma());
		}
	fp modulo_reduce_ang(bool want_rho, fp ang_in) const
		{
		return want_rho ? modulo_reduce_rho(ang_in)
				: modulo_reduce_sigma(ang_in);
		}

	//
	// ***** misc stuff *****
	//
public:
	// human-readable names for the sides (for debugging)
	static const char* ang_name(bool want_rho)
		{ return want_rho ?  "rho" :  "sigma"; }
	static const char* dang_name(bool want_rho)
		{ return want_rho ? "drho" : "dsigma"; }


	//
	// ***** argument structure for constructor *****
	//

	// this structure bundles related arguments together so we don't
	// have 20+ (!) separate arguments to our top-level constructors
	struct	grid_pars		// *** note angles in degrees ***
		{
		fp min_drho,   delta_drho,   max_drho;
		fp min_dsigma, delta_dsigma, max_dsigma;
		};


	//
	// ***** constructor, destructor *****
	//
	grid(const grid_array_pars& grid_array_pars_in,
	     const grid_pars& grid_pars_in);
	// compiler-generated default destructor is ok

private:
	// we forbid copying and passing by value
	// by declaring the copy constructor and assignment operator
	// private, but never defining them
	grid(const grid& rhs);
	grid& operator=(const grid& rhs);

private:
	// range of these is the full grid (including ghost zones)
	const jtutil::linear_map<fp> rho_map_, sigma_map_;

	// angular boundaries of nominal grid
	const fp min_rho_,   max_rho_;
	const fp min_sigma_, max_sigma_;
	};

//******************************************************************************

	  }	// namespace AHFinderDirect

```

patch/fd_grid.hh:
```
// fd_grid.hh -- grid with finite differencing operations
// $Header$
//
// *** Implementation Notes -- Overview ***
// *** Implementation Notes -- Techniques using C++ Templates ***
// *** Implementation Notes -- Techniques using the C/C++ Preprocessor ***
// *** Implementation Notes -- Run-Time Choice of Molecules ***
// *** finite difference molecules ***
// ***** fd_grid - grid with finite differencing operations *****
//

//
// prerequisites:
//    <stdio.h>
//    <assert.h>
//    <math.h>
//    "stdc.h"
//    "config.hh"
//    "../jtutil/util.hh"
//    "../jtutil/array.hh"
//    "../jtutil/linear_map.hh"
//    "coords.hh"
//    "grid.hh"
//

// everything in this file is inside this namespace
namespace AHFinderDirect
	  {

//******************************************************************************

//
// *** Implementation Notes -- Overview ***
//

//
// The key design problem for our finite differencing is how to
// implement an entire family of 5(9) finite difference operations in
// 2D(3D)
//
//	partial_rho		partial_sigma
//	partial_{rho,rho}	partial_{rho,sigma}
//				partial_{sigma,sigma}
//
//	partial_x		partial_y		partial_z
//	partial_xx		partial_xy		partial_xz
//				partial_yy		partial_yz
//							partial_zz
//
// without having to write out the finite differencing molecules multiple
// times, and while still preserving maximum inline-function efficiency.
// In particular, mixed 2nd-order derivative operations like partial_xy
// should be automatically composed from the two individual 1st derivative
// operations (partial_x and partial_y).
//

//
// Our basic approach is to define each finite difference molecule in
// a generic 1-dimensional form using an abstract "data(m)" interface.
// Here we use the terminology that a finite difference molecule is
// defined as
//	out[k] = sum(m) c[m] * in[k+m]
// where c[] is the vector/matrix of molecule coefficients, and m is
// the (integer) relative grid coordinate within a molecule.
//
// That is, for example, we define the usual 2nd order centered 1st
// derivative operator as
//	diff = 0.5*inv_delta_x*(data(+1) - data(-1))
// leaving unspecified just what the data source is.  We then use this
// with an appropriate data source (indexing along that gridfn array axis)
// for each directional derivative operation, and we compose two of
// these, using the first along x as the data source for the second
// along y, for the mixed 2nd-order derivative operation.
//

//******************************************************************************

//
// *** Implementation Notes -- Techniques using C++ Templates ***
// 

//
// There are two plausible ways to use C++ templates
//	[C++ templates are described in detail in chapter 13 of
//	Stroustrup's "The C++ Programming Language" (3rd Edition),
//	hereinafter "C++PL", and chapter 15 of Stroustrup's
//	"The Design and Evolution of C++", hereinafter "D&EC++".]
// to write the sort of generic-at-compile-time code we want:
// - Template specializations for each axis, as discussed in D&EC++
//   section 15.10.3.
// - Overloaded functions for each axis, with an argument type
//   (possibly that of an extra unused argument) selecting the
//   appropriate axis and hence the appropriate function.  This
//   technique is discussed in D&EC++ section 15.6.3.1.
//
// Quoting from D&EC++ (section 15.6.3.1),
//
//	The fundamental observation is that every property
//	of a type or an algorithm can be represented by a
//	type (possibly defined specificaly to do exactly
//	that).  That done, such a type can be used to guide
//	the overload resolution to select a function that
//	depends on the desired property.  [...]
//
//	Please note that thanks to inlining this resolution
//	is done at compile-time, so the appropriate [...]
//	function will be called directly without any run-time
//	overhead.
//
// Quoting from C++PL3 (section 13.4),
//
//	Passing [...] operations as a template parameter has two
//	significant benefits compared to alternatives such as
//	passing pointers to functions.  Several operations can
//	be passed as a single argument with no run-time cost.
//	In addition, the [...] operators [passed this way] are
//	trivial to inline, whereas inlininkg a call through a
//	pointer to function requires exceptional attention from
//	a compiler.
//

//
// In my opinion the template-specialization design is cleaner, and it
// clearly has no run-time cost (whereas the overloaded-function design
// may have a run-time cost for constructing and passing unused objects),
// so we use it here.
//
// There are, however, two (non-fatal) problema with this approach:
// - Unfortunately, it appears C++ (or at least gcc 2.95.1) forbids
//   template specialization within a class, so some of the functions
//   which whould logically be class members, must instead be defined
//   outside any class.  We use the namespace  fd_stuff::  to hide
//   these from the outside world.
// - C++PL3, section C.13.3, states that
//	Only class templates can be template arguments.
//   so we have to use dummy classes around some of our template
//   functions.  To avoid extra constructor/destructor overhead, we
//   make these template functions static.
//

//******************************************************************************

//
// *** Implementation Notes -- Techniques using the C/C++ Preprocessor ***
//

//
// The fundamental problem with the template approaches is portability:
// Although the C++ standard describes powerful template facilities, not
// all C++ compilers yet fully support these.  As an alternative, we can
// use the C/C++ preprocessor.  This is ugly and dangerous (global names!),
// but is probably simpler than any of the template approaches.  It can
// provide the same finite differencing functionality and efficiency as
// the template-based approaches.
//
// Because of its greater portability, we use the preprocessor-based
// approach here.
//

//******************************************************************************


//
// *** Implementation Notes -- Run-Time Choice of Molecules ***
//
// *If* we want to allow the finite differencing scheme to be changed
// at run-time (e.g. from a parameter file), there are three plausible
// ways to do this:
// - Using  switch(molecule_type) , as is standard in C.  This is
//   simple, and for this particular application quite well-structured
//   and maintainable (there are only a few different molecule types,
//   all centralized in this file).
// - Using virtual functions, with  molecule  a virtual base class
//   and individual molecules derived from it.  This is elegant, but
//   may have some performance problems (below).  It also requires some
//   sort of switch-based "object factory" to interface with with the
//   molecule-choice parameters.
// - Write all the finite differencing code multiple times, once for
//   each finite differencing scheme.
//
// The typical use of these functions will be from within a loop over
// a whole grid.  In both cases we can expect excellent accuracy from
// modern hardware branch prediction (and thus minimal performance loss
// from the branching).  It's reasonable to expect a compiler to fully
// inline the switch-based code, exposing all the gridfn array subscriptings
// to strength reduction etc, but this is much trickier for the
// virtual-function--based code.  For this reason, the switch-based
// design seems superior to the virtual-function--based one.
//
// However, at present we don't implement any run-time selection: we
// "just" fix the finite differencing scheme at compile time via the
// preprocessor.
//

//******************************************************************************

//
// *** finite difference molecules ***
//

//**************************************

//
// define the actual molecules
//
// In the following macros, we first define all the distinct floating-
// -point numbers appearing in a molecules as "K" constants (all > 0),
// then define the actual derivative and its molecule coefficients
// using +/- the "K" constants, with multiplies by 1.0 elided and 0
// terms skipped in computing the derivative.  This (hopefully) gives
// maximum efficiency by avoiding the generated code loading the same
// constants multiple times.
// 

//
// The molecule macros all take the following arguments:
// inv_delta_x_ = inverse of grid spacing in the finite differencing
//		  direction
// data_= a data-fetching function or macro: data_(ghosted_gfn, irho, isigma)
//	  is the data to be finite differenced
// irho_plus_m_ = a function or macro: irho_plus_m_(irho,m) returns the
//		  rho coordinate to be passed to data_() for the [m]
//		  molecule coefficient
// isigma_plus_m_ = same thing, for the sigma coordinate
//
// n.b. We grab the variables ghosted_gfn, irho, and isigma from the calling
//      environment, and we define assorted local variables as needed!
//

//**************************************

//
// 2nd order
//

#define FD_GRID__ORDER2__MOL_RADIUS	1
#define FD_GRID__ORDER2__MOL_DIAMETER	3

#define FD_GRID__ORDER2__DX__KPM1	0.5
#define FD_GRID__ORDER2__DX(inv_delta_x_, data_,			\
			    irho_plus_m_, isigma_plus_m_)		\
	const fp data_p1 = data_(ghosted_gfn,				\
				   irho_plus_m_(irho  ,+1),		\
				 isigma_plus_m_(isigma,+1));		\
	const fp data_m1 = data_(ghosted_gfn,				\
				   irho_plus_m_(irho  ,-1),		\
				 isigma_plus_m_(isigma,-1));		\
	const fp sum = FD_GRID__ORDER2__DX__KPM1 * (data_p1 - data_m1);	\
	return inv_delta_x_ * sum;				/* end macro */
#define FD_GRID__ORDER2__DX__COEFF_M1	(-FD_GRID__ORDER2__DX__KPM1)
#define FD_GRID__ORDER2__DX__COEFF_0	0.0
#define FD_GRID__ORDER2__DX__COEFF_P1	(+FD_GRID__ORDER2__DX__KPM1)

#define FD_GRID__ORDER2__DXX__K0		2.0
#define FD_GRID__ORDER2__DXX(inv_delta_x_, data_,			\
			     irho_plus_m_, isigma_plus_m_)		\
	const fp data_p1 = data_(ghosted_gfn,				\
				   irho_plus_m_(irho  ,+1),		\
				 isigma_plus_m_(isigma,+1));		\
	const fp data_0  = data_(ghosted_gfn,				\
				   irho_plus_m_(irho  , 0),		\
				 isigma_plus_m_(isigma, 0));		\
	const fp data_m1 = data_(ghosted_gfn,				\
				   irho_plus_m_(irho  ,-1),		\
				 isigma_plus_m_(isigma,-1));		\
	const fp sum							\
		= data_m1 - FD_GRID__ORDER2__DXX__K0*data_0 + data_p1;	\
	return jtutil::pow2(inv_delta_x_) * sum;		/* end macro */
#define FD_GRID__ORDER2__DXX__COEFF_M1	1.0
#define FD_GRID__ORDER2__DXX__COEFF_0	(-FD_GRID__ORDER2__DXX__K0)
#define FD_GRID__ORDER2__DXX__COEFF_P1	1.0

//**************************************

//
// 4th order
//

#define FD_GRID__ORDER4__MOL_RADIUS	2
#define FD_GRID__ORDER4__MOL_DIAMETER	5

#define FD_GRID__ORDER4__DX__KPM2	(1.0/12.0)
#define FD_GRID__ORDER4__DX__KPM1	(8.0/12.0)
#define FD_GRID__ORDER4__DX(inv_delta_x_, data_,			\
			    irho_plus_m_, isigma_plus_m_)		\
	const fp data_p2 = data_(ghosted_gfn,				\
				   irho_plus_m_(irho  ,+2),		\
				 isigma_plus_m_(isigma,+2));		\
	const fp data_p1 = data_(ghosted_gfn,				\
				   irho_plus_m_(irho  ,+1),		\
				 isigma_plus_m_(isigma,+1));		\
	const fp data_m1 = data_(ghosted_gfn,				\
				   irho_plus_m_(irho  ,-1),		\
				 isigma_plus_m_(isigma,-1));		\
	const fp data_m2 = data_(ghosted_gfn,				\
				   irho_plus_m_(irho  ,-2),		\
				 isigma_plus_m_(isigma,-2));		\
	const fp sum							\
		=   FD_GRID__ORDER4__DX__KPM1 * (data_p1 - data_m1)	\
		  + FD_GRID__ORDER4__DX__KPM2 * (data_m2 - data_p2);	\
	return inv_delta_x_ * sum;				/* end macro */
#define FD_GRID__ORDER4__DX__COEFF_M2	(+FD_GRID__ORDER4__DX__KPM2)
#define FD_GRID__ORDER4__DX__COEFF_M1	(-FD_GRID__ORDER4__DX__KPM1)
#define FD_GRID__ORDER4__DX__COEFF_0	0.0
#define FD_GRID__ORDER4__DX__COEFF_P1	(+FD_GRID__ORDER4__DX__KPM1)
#define FD_GRID__ORDER4__DX__COEFF_P2	(-FD_GRID__ORDER4__DX__KPM2)

//**************************************

#define FD_GRID__ORDER4__DXX__KPM2	( 1.0/12.0)
#define FD_GRID__ORDER4__DXX__KPM1	(16.0/12.0)
#define FD_GRID__ORDER4__DXX__K0	(30.0/12.0)
#define FD_GRID__ORDER4__DXX(inv_delta_x_, data_,			\
			     irho_plus_m_, isigma_plus_m_)		\
	const fp data_p2 = data_(ghosted_gfn,				\
				   irho_plus_m_(irho  ,+2),		\
				 isigma_plus_m_(isigma,+2));		\
	const fp data_p1 = data_(ghosted_gfn,				\
				   irho_plus_m_(irho  ,+1),		\
				 isigma_plus_m_(isigma,+1));		\
	const fp data_0  = data_(ghosted_gfn,				\
				   irho_plus_m_(irho  , 0),		\
				 isigma_plus_m_(isigma, 0));		\
	const fp data_m1 = data_(ghosted_gfn,				\
				   irho_plus_m_(irho  ,-1),		\
				 isigma_plus_m_(isigma,-1));		\
	const fp data_m2 = data_(ghosted_gfn,				\
				   irho_plus_m_(irho  ,-2),		\
				 isigma_plus_m_(isigma,-2));		\
	const fp sum							\
		= - FD_GRID__ORDER4__DXX__K0 * data_0			\
		  + FD_GRID__ORDER4__DXX__KPM1 * (data_m1 + data_p1)	\
		  - FD_GRID__ORDER4__DXX__KPM2 * (data_m2 + data_p2);	\
	return jtutil::pow2(inv_delta_x_) * sum;		/* end macro */
#define FD_GRID__ORDER4__DXX__COEFF_M2	(-FD_GRID__ORDER4__DXX__KPM2)
#define FD_GRID__ORDER4__DXX__COEFF_M1	(+FD_GRID__ORDER4__DXX__KPM1)
#define FD_GRID__ORDER4__DXX__COEFF_0	(-FD_GRID__ORDER4__DXX__K0  )
#define FD_GRID__ORDER4__DXX__COEFF_P1	(+FD_GRID__ORDER4__DXX__KPM1)
#define FD_GRID__ORDER4__DXX__COEFF_P2	(-FD_GRID__ORDER4__DXX__KPM2)

//******************************************************************************

//
// choose finite differencing order via preprocessor symbol FINITE_DIFF_ORDER
//
#ifndef FINITE_DIFF_ORDER
  #error "must define FINITE_DIFF_ORDER!"
#endif

#if   (FINITE_DIFF_ORDER == 2)
  #define FD_GRID__MOL_RADIUS	FD_GRID__ORDER2__MOL_RADIUS
  #define FD_GRID__MOL_DIAMETER	FD_GRID__ORDER2__MOL_DIAMETER
  #define FD_GRID__DX		FD_GRID__ORDER2__DX
  #define FD_GRID__DXX		FD_GRID__ORDER2__DXX
#elif (FINITE_DIFF_ORDER == 4)
  #define FD_GRID__MOL_RADIUS	FD_GRID__ORDER4__MOL_RADIUS
  #define FD_GRID__MOL_DIAMETER	FD_GRID__ORDER4__MOL_DIAMETER
  #define FD_GRID__DX		FD_GRID__ORDER4__DX
  #define FD_GRID__DXX		FD_GRID__ORDER4__DXX
#else
  #error "unknown value " FINITE_DIFF_ORDER " for FINITE_DIFF_ORDER!"
#endif

#define FD_GRID__MOL_AREA	(FD_GRID__MOL_DIAMETER * FD_GRID__MOL_DIAMETER)

//******************************************************************************

//
// ***** fd_grid - grid with finite differencing operations *****
//
// An  fd_grid  is identical to a  grid  except that it also defines
// (rho,sigma)-coordinate finite differencing operations on gridfns.
//

class	fd_grid
	: public grid
	{
	//
	// molecule sizes
	//
public:
	// n.b. this interface implicitly assumes that all molecules
	//      are centered and are the same order and size
	static
	  int finite_diff_order() { return FINITE_DIFF_ORDER; }
	static
	  int molecule_radius()   { return FD_GRID__MOL_RADIUS; }
	static
	  int molecule_diameter() { return FD_GRID__MOL_DIAMETER; }
	static
	  int molecule_min_m() { return -FD_GRID__MOL_RADIUS; }
	static
	  int molecule_max_m() { return  FD_GRID__MOL_RADIUS; }

	//
	// helper functions to compute (irho,isigma) + [m]
	// along each axis
	//
private:
	static
	  int     rho_axis__irho_plus_m(int irho  , int m) { return irho  +m; }
	static
	  int   rho_axis__isigma_plus_m(int isigma, int m) { return isigma  ; }
	static
	  int   sigma_axis__irho_plus_m(int irho  , int m) { return irho    ; }
	static
	  int sigma_axis__isigma_plus_m(int isigma, int m) { return isigma+m; }


	//
	// ***** finite differencing *****
	//
public:

	// 1st derivatives
	fp partial_rho(int ghosted_gfn,  int irho, int isigma)
		const
		{
		FD_GRID__DX(inverse_delta_rho(),
			    ghosted_gridfn,
			    rho_axis__irho_plus_m,
			    rho_axis__isigma_plus_m);
		}
	fp partial_sigma(int ghosted_gfn,  int irho, int isigma)
		const
		{
		FD_GRID__DX(inverse_delta_sigma(),
			    ghosted_gridfn,
			    sigma_axis__irho_plus_m,
			    sigma_axis__isigma_plus_m);
		}

	// "pure" 2nd derivatives
	fp partial_rho_rho(int ghosted_gfn,  int irho, int isigma)
		const
		{
		FD_GRID__DXX(inverse_delta_rho(),
			     ghosted_gridfn,
			     rho_axis__irho_plus_m,
			     rho_axis__isigma_plus_m);
		}
	fp partial_sigma_sigma(int ghosted_gfn,  int irho, int isigma)
		const
		{
		FD_GRID__DXX(inverse_delta_sigma(),
			     ghosted_gridfn,
			     sigma_axis__irho_plus_m,
			     sigma_axis__isigma_plus_m);
		}

	// mixed 2nd partial derivative
	fp partial_rho_sigma(int ghosted_gfn,   int irho, int isigma)
		const
		{
		FD_GRID__DX(inverse_delta_rho(),
			    partial_sigma,
			    rho_axis__irho_plus_m,
			    rho_axis__isigma_plus_m);
		}


	//
	// ***** molecule coefficients *****
	//
public:
	// molecule coefficients
	// n.b. this interface implicitly assumes that all molecules
	//      are position-independent
	fp partial_rho_coeff        (int m) const
		{ return inverse_delta_rho()   * dx_coeff(m); }
	fp partial_sigma_coeff      (int m) const
		{ return inverse_delta_sigma() * dx_coeff(m); }
	fp partial_rho_rho_coeff    (int m) const
		{ return jtutil::pow2(inverse_delta_rho()) * dxx_coeff(m); }
	fp partial_sigma_sigma_coeff(int m) const
		{ return jtutil::pow2(inverse_delta_sigma()) * dxx_coeff(m); }
	fp partial_rho_sigma_coeff(int m_rho, int m_sigma) const
		{
		return partial_rho_coeff(m_rho) * partial_sigma_coeff(m_sigma);
		}

	// worker functions: molecule coefficients for unit grid spacing
private:
	static
	  fp dx_coeff(int m);
	static
	  fp dxx_coeff(int m);


	//
	// ***** constructor, destructor *****
	//
public:

	// constructor: pass through to grid:: constructor
	fd_grid(const grid_array_pars& grid_array_pars_in,
		const grid_pars& grid_pars_in)
		: grid(grid_array_pars_in, grid_pars_in)
		{ }
	// compiler-generated default destructor is ok


private:
        // we forbid copying and passing by value
        // by declaring the copy constructor and assignment operator
        // private, but never defining them
	fd_grid(const fd_grid& rhs);
	fd_grid& operator=(const fd_grid& rhs);
	};

//******************************************************************************

	  }	// namespace AHFinderDirect

```

patch/patch.hh:
```
// patch.hh -- describes a coordinate/grid patch
// $Header$
//
// ***** how patch boundaries are handled *****
// patch - abstract base class to describe a coordinate/grid patch
//
// z_patch - derived class for a +/- z patch
// x_patch - derived class for a +/- x patch
// y_patch - derived class for a +/- y patch
//

//
// prerequisites:
//	<stdio.h>
//	<assert.h>
//	<math.h>
//	"stdc.h"
//	"config.hh"
//	"../jtutil/util.hh"
//	"../jtutil/array.hh"
//	"../jtutil/linear_map.hh"
//	"coords.hh"
//	"grid.hh"
//	"fd_grid.hh"
//

// everything in this file is inside this namespace
namespace AHFinderDirect
	  {

//*****************************************************************************
//*****************************************************************************
//*****************************************************************************

//
// ***** how patch boundaries are handled *****
//

//
// Basically, we handle patch boundaries using the usual "ghost zone"
// technique, interpolating values from neighboring patches as necessary.
//
// In more detail, we use the following interrelated types of objects
// to handle patch boundaries:
//
// A  patch_edge  object represents the basic geometry of a min/max
// rho/sigma side of a patch, i.e. it provides which-side-am-I predicates,
// coordinate conversions between (perp,par) and (rho,sigma), etc.
// Every patch has (points to) 4  patch_edge  objects, one for each of
// the patch's sides.
//
// A  ghost_zone  object describes a patch's ghost zone, and knows how
// to fill in gridfns there based on either the patch system's symmetry
// or interpolation from a neighboring patch.  ghost_zone is an abstract
// base class, from which we derive two classes:
// * A  symmetry_ghost_zone  object describes a ghost zone which is a
//   (discrete) symmetry of spacetime, either mirror-image or periodic.
//   Such an object knows how to fill in ghost-zone gridfn data from
//   the "other side" of the symmetry.
// * An  interpatch_ghost_zone  object describes a ghost zone which
//   overlaps another patch.  Such an object knows how to get ghost
//   zone gridfn data from the other patch.  More accurately, it gets
//   the data by asking (calling) the appropriate one of the other
//   patch's  patch_interp  objects.
// Every patch has (points to) 4  ghost_zone  objects, one for each of
// the patch's sides.
//
// A  patch_interp  object does the actual interpolation of data from
// within a patch (for filling in data in another patch's ghost zone).
// A  patch_interp  object points to the patch and patch_edge where it
// will be interpolating.
//
// For example, suppose we have two patches p and q with a common
// angular boundary.  Then the desired network of pointers looks like
// this (omitting the  patch_edge  objects for simplicity):
//
// +-----+                                                       +-----+
// |     | <--> p.interpatch_ghost_zone ---> q.patch_interp ---> |     |
// |  p  |                                                       |  q  |
// |     | <--- p.patch_interp <--- q.interpatch_ghost_zone <--> |     |
// +-----+                                                       +-----+
//
// Because of the mutual pointers, we can't easily construct (say) p's
// interpatch_ghost_zone until after q itself has been constructed, and
// vice versa.  Moreover, the  patch_interp::  constructor needs the
// adjacent-side  ghost_zone  objects to already exist, and it needs to
// know the iperp range of the interpolation region, which can only be
// computed from the adjacent-patch  interpatch_ghost_zone  object.
//
// The solution adopted here is to use a 3-phase algorithm, ultimately
// driven by the patch_system constructor:
// * The patch constructors themselves construct the  patch_edge  objects
//   and links them to/from the patches.
// * The patch_system constructor calls the appropriate functions
//	patch::create_mirror_symmetry_ghost_zone()
//	patch::create_periodic_symmetry_ghost_zone()
//	patch::create_interpatch_ghost_zone()
//   to construct the  ghost_zone  objects and link them to/from the
//   patches.
// * The patch_system constructor calls the functions
//	interpatch_ghost_zone::finish_setup()
//   to finish setting up the  interpatch_ghost_zone  objects, construct
//   the other patch's  patch_interp  objects, and finish linking the
//    interpatch_ghost_zone  objects to the  patch_interp  objects.
//

//*****************************************************************************
//*****************************************************************************
//*****************************************************************************

//
// patch - abstract base class to describe a generic coordinate/grid patch
//

//
// There are 3 types of patches, z, x, and y.  Each type uses two of
// (mu,nu,phi) as its angular coordinates (rho,sigma); the remaining
// "unused" one of (mu,nu,phi) is tau.
//
//	z patch ==> (rho,sigma) = (mu,nu)    tau = phi
//	x patch ==> (rho,sigma) = (nu,phi)   tau = mu
//	y patch ==> (rho,sigma) = (mu,phi)   tau = nu
//

// forward declarations
class patch_edge;
class ghost_zone;
class symmetry_ghost_zone;
class interpatch_ghost_zone;
class patch_interp;
class patch_system;

//
// const qualifiers refer to the gridfn values
//
class	patch
	: public fd_grid
	{
	//
	// ***** patch system, type, and coordinate metadata *****
	//
public:

	// to which patch system do we belong?
	patch_system& my_patch_system() const
		{ return my_patch_system_; }

	// each patch has a unique 0-origin small-integer patch number,
	// usually denoted  pn
	int patch_number() const { return patch_number_; }

	// each patch has a unique human-readable patch name for debugging etc
	const char* name() const { return name_; }	// typically "+z" etc

	// are we a +[xyz] or -[xyz] patch?
	bool is_plus() const { return is_plus_; }

	// ... values for the  is_plus_in  constructor argument
	//     FIXME: these should really be bool, but then we couldn't
	//            use the "enum hack" for in-class constants
	enum { patch_is_plus = true, patch_is_minus = false };

	// are we a (+/-) x or y or z patch?
	// ... n.b. type is `char' because this is handy for both
	//	    switch() and human-readable printing
	char ctype() const { return ctype_; }		// 'z' or 'x' or 'y'

	// are two patches really the same patch?
	// n.b. this does *not* compare any of the gridfn data!
	bool operator==(const patch& other_patch) const
		{ return this == &other_patch; }
	bool operator!=(const patch& other_patch) const
		{ return ! operator==(other_patch); }

	// (rho,sigma,tau) coordinates as singleton coordinate sets
	local_coords::coords_set coords_set_rho() const
		{ return coords_set_rho_; } 
	local_coords::coords_set coords_set_sigma() const
		{ return coords_set_sigma_; }
	local_coords::coords_set coords_set_tau() const
		{ return coords_set_tau_; }

	// {rho,sigma} coordinate set
	local_coords::coords_set coords_set_rho_sigma() const
		{ return coords_set_rho() | coords_set_sigma(); }

	// (rho,sigma) coordinates as human-readable character strings
	// (for labelling output files etc)
	virtual const char* name_of_rho() const = 0;
	virtual const char* name_of_sigma() const = 0;


	//
	// ***** (rho,sigma,tau) coordinates *****
	//
public:

	// convert (rho,sigma) --> tau
	virtual fp tau_of_rho_sigma(fp rho, fp sigma) const = 0;

	// convert (rho,sigma) --> (mu,nu,phi)
	virtual fp  mu_of_rho_sigma(fp rho, fp sigma) const = 0;
	virtual fp  nu_of_rho_sigma(fp rho, fp sigma) const = 0;
	virtual fp phi_of_rho_sigma(fp rho, fp sigma) const = 0;

	// convert (rho,sigma) <--> usual polar spherical (theta,phi)
	virtual void theta_phi_of_rho_sigma(fp rho, fp sigma,
					    fp& ps_theta, fp& ps_phi)
		const = 0;
	virtual void rho_sigma_of_theta_phi(fp ps_theta, fp ps_phi,
					    fp& rho, fp& sigma)
		const = 0;

	// convert (r,rho,sigma) <--> local (x,y,z)
	virtual void xyz_of_r_rho_sigma(fp r, fp rho, fp sigma,
					fp& x, fp& y, fp& z)
		const = 0;
	virtual fp   rho_of_xyz(fp x, fp y, fp z) const = 0;
	virtual fp sigma_of_xyz(fp x, fp y, fp z) const = 0;

	// convert (rho,sigma) --> direction cosines (xcos,ycos,zcos)
	//                         with respect to the local coordinate system
	virtual void xyzcos_of_rho_sigma(fp rho, fp sigma,
					 fp& xcos, fp& ycos, fp& zcos)
		const = 0;

	// partial (x,y,z) / partial (rho,sigma)
	virtual void partial_xyz_wrt_r_rho_sigma
	   (fp r, fp rho, fp sigma,
	    fp& partial_x_wrt_r, fp& partial_x_wrt_rho, fp& partial_x_wrt_sigma,
	    fp& partial_y_wrt_r, fp& partial_y_wrt_rho, fp& partial_y_wrt_sigma,
	    fp& partial_z_wrt_r, fp& partial_z_wrt_rho, fp& partial_z_wrt_sigma)
		const = 0;

	// partial (rho,sigma) / partial (x,y,z)
	virtual fp partial_rho_wrt_x(fp x, fp y, fp z) const = 0;
	virtual fp partial_rho_wrt_y(fp x, fp y, fp z) const = 0;
	virtual fp partial_rho_wrt_z(fp x, fp y, fp z) const = 0;
	virtual fp partial_sigma_wrt_x(fp x, fp y, fp z) const = 0;
	virtual fp partial_sigma_wrt_y(fp x, fp y, fp z) const = 0;
	virtual fp partial_sigma_wrt_z(fp x, fp y, fp z) const = 0;

	// partial^2 (rho,sigma) / partial (xx,xy,xz,yy,yz)
	virtual fp partial2_rho_wrt_xx(fp x, fp y, fp z) const = 0;
	virtual fp partial2_rho_wrt_xy(fp x, fp y, fp z) const = 0;
	virtual fp partial2_rho_wrt_xz(fp x, fp y, fp z) const = 0;
	virtual fp partial2_rho_wrt_yy(fp x, fp y, fp z) const = 0;
	virtual fp partial2_rho_wrt_yz(fp x, fp y, fp z) const = 0;
	virtual fp partial2_rho_wrt_zz(fp x, fp y, fp z) const = 0;
	virtual fp partial2_sigma_wrt_xx(fp x, fp y, fp z) const = 0;
	virtual fp partial2_sigma_wrt_xy(fp x, fp y, fp z) const = 0;
	virtual fp partial2_sigma_wrt_xz(fp x, fp y, fp z) const = 0;
	virtual fp partial2_sigma_wrt_yy(fp x, fp y, fp z) const = 0;
	virtual fp partial2_sigma_wrt_yz(fp x, fp y, fp z) const = 0;
	virtual fp partial2_sigma_wrt_zz(fp x, fp y, fp z) const = 0;

	// compute (rho,sigma) 2-D induced metric from 3-D xyz metric
	// as per p.33 of my apparent horizon finding notes
	// ... returns Jacobian of (rho,sigma) 2-D induced metric
	fp rho_sigma_metric(fp r, fp rho, fp sigma,
			    fp partial_surface_r_wrt_rho,
			    fp partial_surface_r_wrt_sigma,
			    fp g_xx, fp g_xy, fp g_xz,
				     fp g_yy, fp g_yz,
					      fp g_zz,
			    fp& g_rho_rho, fp& g_rho_sigma,
					   fp& g_sigma_sigma)
		const;

	// plotting coordinates (dpx,dpy)
	// ... character string describing how (dpx,dpy) are
	//     defined in terms of (mu,nu,phi), eg "90 - drho = 90 - dphi"
	//     (used for labelling output files)
	virtual const char* name_of_dpx() const = 0;
	virtual const char* name_of_dpy() const = 0;
	// ... (irho,isimga) --> (px,py)
	virtual fp dpx_of_rho_sigma(fp rho, fp sigma) const = 0;
	virtual fp dpy_of_rho_sigma(fp rho, fp sigma) const = 0;


	//
	// ***** line/surface integrals *****
	//
public:

	//
	// The following enum describes the integration methods supported
	// by  integrate_gridfn() .
	//
	// For convenience of exposition we describe the methods as if for
	// 1-D integration, but  integrate_gridfn()  actually does 2-D
	// (surface) integration over the patch.
	//
	// Suppose we're computing $\int_{x_0}^{x^N} f(x) \, dx$, using the
	// equally spaced integration points $f_0$, $f_1$, \dots, $f_N$,
	// spaced $\Delta x$ apart.  Then the integration methods are as
	// follows, with the convention that $\langle X \rangle$ denotes
	// indefinite repetition of the "X" terms, depending on N:
	//
	enum	integration_method
		{
		// Trapezoid rule
		// ... character-string name "trapezoid" or "trapezoid rule"
		// ... 2nd order accurate for smooth functions
		// ... requires N >= 1
		// $$
		// \Delta x \left[
		//            \half f_0
		//          + \langle
		//            f_k
		//            \rangle
		//          + \half f_N
		//          \right]
		// $$
		integration_method__trapezoid,

		// Simpson's rule
		// ... character-string name "Simpson" or "Simpson's rule"
		// ... 4th order accurate for smooth functions
		// ... requires N >= 2 and N even
		// $$
		// \Delta x \left[
		//            \frac{1}{3} f_0
		//          + \frac{4}{3} f_1
		//          + \langle
		//            \frac{2}{3} f_{2k} + \frac{4}{3} f_{2k+1}
		//            \rangle
		//          + \frac{1}{3} f_N
		//          \right]
		// $$
		integration_method__Simpson,

		// Simpson's rule, variant form
		// ... characgter-string name "Simpson (variant)"
		//     or "Simpson's rule (variant)"
		// ... described in Numerical Recipes 1st edition (4.1.14)
		// ... 4th order accurate for smooth functions
		// ... requires N >= 7
		// $$
		// \Delta x \left[
		//            \frac{17}{48} f_0
		//          + \frac{59}{48} f_1
		//          + \frac{43}{48} f_2
		//          + \frac{49}{48} f_3
		//          + \langle
		//            f_k
		//            \rangle
		//          + \frac{49}{48} f_{N-3}
		//          + \frac{43}{48} f_{N-2}
		//          + \frac{59}{48} f_{N-1}
		//          + \frac{17}{48} f_N
		//          \right]
		// $$
		integration_method__Simpson_variant,

		// automatic choice of the "best" one of the above methods:
		// ... i.e. choose Simpson's rule or variant if applicable,
		//     otherwise trapezoid rule
		// N == 2	Simpson's rule
		// N == 3	trapezoid rule
		// N == 4	Simpson's rule
		// N == 5	trapezoid rule
		// N == 6	Simpson's rule
		// N >= 7	Simpson's rule, variant form
		integration_method__automatic_choice // no comma here!
		};

	// decode character string name into internal enum
	static
	  enum integration_method
	    decode_integration_method(const char method_string[]);

	// compute the arc length of a surface in the specified plane
	// (must be one of "xy", "xz", or "yz") over the patch's nominal bounds
	// ... error_exit() if  plane  is invalid and/or
	//     the patch doesn't contain that coordinate plane
	virtual fp plane_arc_length(const char plane[],
				    int ghosted_radius_gfn,
				    int g_xx_gfn, int g_xy_gfn, int g_xz_gfn,
						  int g_yy_gfn, int g_yz_gfn,
								int g_zz_gfn,
				    enum integration_method method)
		const = 0;

	// ... along the rho direction (i.e. in a dsigma=constant plane
	//     where dsigma is a multiple of 90 degrees)
	fp rho_arc_length(int ghosted_radius_gfn,
			  int g_xx_gfn, int g_xy_gfn, int g_xz_gfn,
					int g_yy_gfn, int g_yz_gfn,
						      int g_zz_gfn,
			  enum integration_method method)
		const;
	// ... along the sigma direction (i.e. in a drho=constant plane
	//     where drho is a multiple of 90 degrees)
	fp sigma_arc_length(int ghosted_radius_gfn,
			    int g_xx_gfn, int g_xy_gfn, int g_xz_gfn,
					  int g_yy_gfn, int g_yz_gfn,
							int g_zz_gfn,
			    enum integration_method method)
		const;

	// compute the surface integral of a gridfn over the patch's
	// nominal area,
	//	$\int f(\rho,\sigma) \, dA$
	//		= \int f(\rho,\sigma) \sqrt{|J|} \, d\rho \, d\sigma$
	// where $J$ is the Jacobian of $(x,y,z)$ with respect to $(rho,sigma)
	// ... integration method selected by  method  argument
	// ... src gridfn may be either nominal-grid or ghosted-grid
	//     (n.b. in the latter case the integral is still done
	//           only over the patch's nominal area)
	fp integrate_gridfn(int unknown_src_gfn,
			    int ghosted_radius_gfn,
			    int g_xx_gfn, int g_xy_gfn, int g_xz_gfn,
					  int g_yy_gfn, int g_yz_gfn,
							int g_zz_gfn,
			    enum integration_method method)
		const;

	fp integrate_gridpoint(int unknown_src_gfn,
                               int ghosted_radius_gfn,
                               int g_xx_gfn, int g_xy_gfn, int g_xz_gfn,
                                             int g_yy_gfn, int g_yz_gfn,
                                                           int g_zz_gfn,
                               enum integration_method method,
                               int irho, int isigma)
		const;

	// compute integration coefficient $c_i$ where
	// $\int_{x_0}^{x_N} f(x) \, dx
	//	\approx \Delta x \, \sum_{i=0}^N c_i f(x_i)$
private:
	static
	  fp integration_coeff(enum integration_method method, int N, int i);


	//
	// ***** patch edges ****
	//
public:
	const patch_edge& min_rho_patch_edge() const
		{ return min_rho_patch_edge_; }
	const patch_edge& max_rho_patch_edge() const
		{ return max_rho_patch_edge_; }
	const patch_edge& min_sigma_patch_edge() const
		{ return min_sigma_patch_edge_; }
	const patch_edge& max_sigma_patch_edge() const
		{ return max_sigma_patch_edge_; }
	const patch_edge& minmax_ang_patch_edge(bool want_min, bool want_rho)
		const
		{
		return want_min ? (want_rho ? min_rho_patch_edge()
					    : min_sigma_patch_edge())
				: (want_rho ? max_rho_patch_edge()
					    : max_sigma_patch_edge());
		}

	// find which patch edge is adjacent to neighboring patch q,
	// or error_exit() if it's not actually a neighboring patch
	// ... computation done using only (rho,sigma) coordinate sets
	//     and min/max dang bounds ==> ok to use in setting up ghost zones
	// ... patch_overlap_width = number of grid points (grid spacings
	//     in the perpendicular direction) these patches' nominal grids
	//     overlap,
	//     ... if this is nonzero, then these patches must have
	//         the *same* grid spacing in the perpendicular direction
	//     ... e.g. delta_dang = 5, this patch max_dang = 50,
	//         other patch min_dang = 40 ==> patch_overlap_width = 3
	//		p   p   p   p   p
	//			q   q   q   q   q
	const patch_edge& edge_adjacent_to_patch(const patch& q,
						 int patch_overlap_width = 0)
		const;


	//
	// ***** ghost zones *****
	//
public:
	ghost_zone& min_rho_ghost_zone() const
		{
		assert(min_rho_ghost_zone_ != NULL);
		return *min_rho_ghost_zone_;
		}
	ghost_zone& max_rho_ghost_zone() const
		{
		assert(max_rho_ghost_zone_ != NULL);
		return *max_rho_ghost_zone_;
		}
	ghost_zone& min_sigma_ghost_zone() const
		{
		assert(min_sigma_ghost_zone_ != NULL);
		return *min_sigma_ghost_zone_;
		}
	ghost_zone& max_sigma_ghost_zone() const
		{
		assert(max_sigma_ghost_zone_ != NULL);
		return *max_sigma_ghost_zone_;
		}
	ghost_zone& minmax_rho_ghost_zone(bool want_min)
		const
		{
		return want_min ? min_rho_ghost_zone()
				: max_rho_ghost_zone();
		}
	ghost_zone& minmax_sigma_ghost_zone(bool want_min)
		const
		{
		return want_min ? min_sigma_ghost_zone()
				: max_sigma_ghost_zone();
		}

	ghost_zone& minmax_ang_ghost_zone(bool want_min, bool want_rho)
		const
		{
		return want_rho ? minmax_rho_ghost_zone(want_min)
				: minmax_sigma_ghost_zone(want_min);
		}

	ghost_zone& ghost_zone_on_edge(const patch_edge &e) const;

	// which of the two ghost zones at a specified corner,
	// contains a specified point?
	ghost_zone& corner_ghost_zone_containing_point
		(bool rho_is_min, bool sigma_is_min,	// specifies corner
		 int irho, int isigma)			// specifies point
		const;

	// which ghost zone contains a specified noncorner point?
	ghost_zone& ghost_zone_containing_noncorner_point(int irho, int isigma)
		const;


	//
	// ***** set up ghost zones
	//
public:

	// assert() that this ghost zone hasn't been set up yet,
	// then set it up as mirror-symmetry
	void create_mirror_symmetry_ghost_zone(const patch_edge& edge);

	// assert() that this ghost zone hasn't been set up yet,
	// then set it up as periodic-symmetry
	void create_periodic_symmetry_ghost_zone
		(const patch_edge& my_edge, const patch_edge& other_edge,
		 bool ipar_map_is_plus);

	// assert() that this ghost zone hasn't been set up yet,
	// then set it up as interpatch
	// ... this only sets up ghost zone in skeletal form; use
	//     interpatch_ghost_zone::finish_setup()  to complete
	//     the setup process
	void create_interpatch_ghost_zone
		(const patch_edge& my_edge, const patch_edge& other_edge,
		 int patch_overlap_width);

	// assert() that all ghost zones
	// are fully setup
	void assert_all_ghost_zones_fully_setup() const;

private:
	// helper function for setup_*_ghost_zone():
	// assert() that ghost zone pointer on specified edge is NULL
	// (i.e. that we haven't already setup this ghost zone),
	// then assign new value to it
	void set_ghost_zone(const patch_edge& edge, ghost_zone* gzp);


	//
	// ***** constructor, destructor, et al *****
	//
protected:
	// ... used only from derived classes
	// ... doesn't set up ghost zone info, since this depends on
	//     knowing our neighbouring patches, which might not exist yet
	// ... saves a pointer to name_in[], so this should have a
	//     lifetime at least as long as that of this object
	patch(patch_system &my_patch_system_in, int patch_number_in,
	      const char name_in[], bool is_plus_in, char ctype_in,
	      local_coords::coords_set coords_set_rho_in,
	      local_coords::coords_set coords_set_sigma_in,
	      local_coords::coords_set coords_set_tau_in,
	      const grid_arrays::grid_array_pars& grid_array_pars_in,
	      const grid::grid_pars& grid_pars_in);
public:
	// destructor must be virtual to allow destruction
	// of derived classes via ptr/ref to this class
	virtual ~patch();

private:
	// we forbid copying and passing by value
	// by declaring the copy constructor and assignment operator
	// private, but never defining them
	patch(const patch& rhs);
	patch& operator=(const patch& rhs);


	//
	// ***** data members *****
	//
private:

	// type/coordinate metadata
	patch_system &my_patch_system_;
	const int patch_number_;
	const char* name_;
	const bool is_plus_;
	const char ctype_;
	const local_coords::coords_set coords_set_rho_,
				       coords_set_sigma_,
				       coords_set_tau_;

	// edges
	const patch_edge& min_rho_patch_edge_;
	const patch_edge& max_rho_patch_edge_;
	const patch_edge& min_sigma_patch_edge_;
	const patch_edge& max_sigma_patch_edge_;

	// ghost zones
	// ... pointers are set to NULL by ctor,
	//     reset to non-NULL by set_ghost_zone(), which is called by
	//	  create_mirror_symmetry_ghost_zone()
	//	  create_periodic_symmetry_ghost_zone()
	//	  create_interpatch_ghost_zone()
	ghost_zone* min_rho_ghost_zone_;
	ghost_zone* max_rho_ghost_zone_;
	ghost_zone* min_sigma_ghost_zone_;
	ghost_zone* max_sigma_ghost_zone_;
	};

//*****************************************************************************
//*****************************************************************************
//*****************************************************************************

//
// This class describes a +/- z patch.  It doesn't define any new
// functions not already present in  class patch ; it "just" defines
// non-virtual versions of all the pure virtual functions defined there.
//
//	z patch ==> (rho,sigma) = (mu,nu)    tau = phi
//
class	z_patch
	: public patch
	{
public:
	// human-readable names of (rho,sigma)
	const char* name_of_rho() const { return "mu"; }
	const char* name_of_sigma() const { return "nu"; }

	// convert (rho,sigma) --> tau
	fp tau_of_rho_sigma(fp rho, fp sigma) const
		{ return local_coords::phi_of_mu_nu(rho,sigma); }

	// convert (rho,sigma) --> (mu,nu,phi)
	fp mu_of_rho_sigma(fp rho, fp sigma) const { return rho; }
	fp nu_of_rho_sigma(fp rho, fp sigma) const { return sigma; }
	fp phi_of_rho_sigma(fp rho, fp sigma) const
		{ return local_coords::phi_of_mu_nu(rho,sigma); }

	// convert (rho,sigma) <--> usual polar spherical (theta,phi)
	void theta_phi_of_rho_sigma(fp rho, fp sigma, fp& ps_theta, fp& ps_phi)
		const
		{
		local_coords::theta_phi_of_mu_nu(rho,sigma, ps_theta,ps_phi);
		}
	void rho_sigma_of_theta_phi(fp ps_theta, fp ps_phi, fp& rho, fp& sigma)
		const
		{
		local_coords::mu_nu_of_theta_phi(ps_theta,ps_phi, rho,sigma);
		}

	// convert (r,rho,sigma) <--> (x,y,z)
	void xyz_of_r_rho_sigma(fp r, fp rho, fp sigma, fp& x, fp& y, fp& z)
		const
		{ local_coords::xyz_of_r_mu_nu(r,rho,sigma, x,y,z); }
	fp rho_of_xyz(fp x, fp y, fp z) const
		{ return modulo_reduce_rho(local_coords::mu_of_yz(y,z)); }
	fp sigma_of_xyz(fp x, fp y, fp z) const
		{ return modulo_reduce_sigma(local_coords::nu_of_xz(x,z)); }

	// convert (rho,sigma) --> direction cosines (xcos,ycos,zcos)
	//                         with respect to the local coordinate system
	void xyzcos_of_rho_sigma(fp rho, fp sigma,
				 fp& xcos, fp& ycos, fp& zcos)
		const
		{ local_coords::xyzcos_of_mu_nu(rho,sigma, xcos,ycos,zcos); }

	// partial (x,y,z) / partial (rho,sigma)
	void partial_xyz_wrt_r_rho_sigma
	   (fp r, fp rho, fp sigma,
	    fp& partial_x_wrt_r, fp& partial_x_wrt_rho, fp& partial_x_wrt_sigma,
	    fp& partial_y_wrt_r, fp& partial_y_wrt_rho, fp& partial_y_wrt_sigma,
	    fp& partial_z_wrt_r, fp& partial_z_wrt_rho, fp& partial_z_wrt_sigma)
		const
		{
		local_coords::partial_xyz_wrt_r_mu_nu
		   (r, rho, sigma,
		    partial_x_wrt_r, partial_x_wrt_rho, partial_x_wrt_sigma,
		    partial_y_wrt_r, partial_y_wrt_rho, partial_y_wrt_sigma,
		    partial_z_wrt_r, partial_z_wrt_rho, partial_z_wrt_sigma);
		}

	// partial (rho,sigma) / partial (x,y,z)
	fp partial_rho_wrt_x(fp x, fp y, fp z) const { return 0.0; }
	fp partial_rho_wrt_y(fp x, fp y, fp z) const
		{ return local_coords::partial_mu_wrt_y(y,z); }
	fp partial_rho_wrt_z(fp x, fp y, fp z) const
		{ return local_coords::partial_mu_wrt_z(y,z); }
	fp partial_sigma_wrt_x(fp x, fp y, fp z) const
		{ return local_coords::partial_nu_wrt_x(x,z); }
	fp partial_sigma_wrt_y(fp x, fp y, fp z) const { return 0.0; }
	fp partial_sigma_wrt_z(fp x, fp y, fp z) const
		{ return local_coords::partial_nu_wrt_z(x,z); }

	// partial^2 (rho,sigma) / partial (xx,xy,xz,yy,yz)
	fp partial2_rho_wrt_xx(fp x, fp y, fp z) const { return 0.0; }
	fp partial2_rho_wrt_xy(fp x, fp y, fp z) const { return 0.0; }
	fp partial2_rho_wrt_xz(fp x, fp y, fp z) const { return 0.0; }
	fp partial2_rho_wrt_yy(fp x, fp y, fp z) const
		{ return local_coords::partial2_mu_wrt_yy(y,z); }
	fp partial2_rho_wrt_yz(fp x, fp y, fp z) const
		{ return local_coords::partial2_mu_wrt_yz(y,z); }
	fp partial2_rho_wrt_zz(fp x, fp y, fp z) const
		{ return local_coords::partial2_mu_wrt_zz(y,z); }
	fp partial2_sigma_wrt_xx(fp x, fp y, fp z) const
		{ return local_coords::partial2_nu_wrt_xx(x,z); }
	fp partial2_sigma_wrt_xy(fp x, fp y, fp z) const { return 0.0; }
	fp partial2_sigma_wrt_xz(fp x, fp y, fp z) const
		{ return local_coords::partial2_nu_wrt_xz(x,z); }
	fp partial2_sigma_wrt_yy(fp x, fp y, fp z) const { return 0.0; }
	fp partial2_sigma_wrt_yz(fp x, fp y, fp z) const { return 0.0; }
	fp partial2_sigma_wrt_zz(fp x, fp y, fp z) const
		{ return local_coords::partial2_nu_wrt_zz(x,z); }

	// plotting coordinates (px,py)
	// ... character string describing how (dpx,dpy) are
	//     defined in terms of (mu,nu,phi), eg "90 - drho = 90 - dphi"
	//     (used for labelling output files)
	const char* name_of_dpx() const
		{ return "dsigma = dnu"; }
	const char* name_of_dpy() const
		{ return is_plus() ? "drho = dmu" : "180 - drho = 180 - dmu"; }
	// ... (irho,isimga) --> (px,py)
	fp dpx_of_rho_sigma(fp rho, fp sigma) const
		{ return jtutil::degrees_of_radians(sigma); }
	fp dpy_of_rho_sigma(fp rho, fp sigma) const
		{
		const fp drho = jtutil::degrees_of_radians(rho);
		return is_plus() ? drho : 180.0 - drho;
		}

	// compute the arc length of a surface in the specified plane
	// (must be one of "xz" or "yz") over the patch's nominal bounds
	// ... error_exit() if  plane  is invalid
	fp plane_arc_length(const char plane[],
			    int ghosted_radius_gfn,
			    int g_xx_gfn, int g_xy_gfn, int g_xz_gfn,
					  int g_yy_gfn, int g_yz_gfn,
							int g_zz_gfn,
			    enum integration_method method)
		const;

	// constructor, destructor
	z_patch(patch_system &my_patch_system_in, int patch_number_in,
		const char* name_in, bool is_plus_in,
		const grid_arrays::grid_array_pars& grid_array_pars_in,
		const grid::grid_pars& grid_pars_in);
	~z_patch() { }

private:
        // we forbid copying and passing by value
        // by declaring the copy constructor and assignment operator
        // private, but never defining them
	z_patch(const z_patch& rhs);
	z_patch& operator=(const z_patch& rhs);
	};

//*****************************************************************************

//
// This class describes a +/- x patch.  It doesn't define any new
// functions not already present in  class patch ; it "just" defines
// non-virtual versions of all the pure virtual functions defined there.
//
//	x patch ==> (rho,sigma) = (nu,phi)   tau = mu
//
class	x_patch
	: public patch
	{
public:
	// human-readable names of (rho,sigma)
	const char* name_of_rho() const { return "nu"; }
	const char* name_of_sigma() const { return "phi"; }

	// convert (rho,sigma) --> tau
	fp tau_of_rho_sigma(fp rho, fp sigma) const
		{ return local_coords::mu_of_nu_phi(rho, sigma); }

	// convert (rho,sigma) --> (mu,nu,phi)
	fp nu_of_rho_sigma(fp rho, fp sigma) const { return rho; }
	fp phi_of_rho_sigma(fp rho, fp sigma) const { return sigma; }
	fp mu_of_rho_sigma(fp rho, fp sigma) const
		{ return local_coords::mu_of_nu_phi(rho, sigma); }

	// convert (rho,sigma) <--> usual polar spherical (theta,phi)
	void theta_phi_of_rho_sigma(fp rho, fp sigma, fp& ps_theta, fp& ps_phi)
		const
		{
		local_coords::theta_phi_of_nu_phi(rho, sigma, ps_theta, ps_phi);
		}
	void rho_sigma_of_theta_phi(fp ps_theta, fp ps_phi, fp& rho, fp& sigma)
		const
		{
		local_coords::nu_phi_of_theta_phi(ps_theta, ps_phi, rho, sigma);
		}

	// convert (r,rho,sigma) <--> (x,y,z)
	void xyz_of_r_rho_sigma(fp r, fp rho, fp sigma, fp& x, fp& y, fp& z)
		const
		{ local_coords::xyz_of_r_nu_phi(r, rho, sigma, x, y, z); }
	fp rho_of_xyz(fp x, fp y, fp z) const
		{ return modulo_reduce_rho(local_coords::nu_of_xz(x, z)); }
	fp sigma_of_xyz(fp x, fp y, fp z) const
		{ return modulo_reduce_sigma(local_coords::phi_of_xy(x, y)); }

	// convert (rho,sigma) --> direction cosines (xcos,ycos,zcos)
	//                         with respect to the local coordinate system
	void xyzcos_of_rho_sigma(fp rho, fp sigma,
				 fp& xcos, fp& ycos, fp& zcos)
		const
		{ local_coords::xyzcos_of_nu_phi(rho,sigma, xcos,ycos,zcos); }

	// partial (x,y,z) / partial (rho,sigma)
	void partial_xyz_wrt_r_rho_sigma
	   (fp r, fp rho, fp sigma,
	    fp& partial_x_wrt_r, fp& partial_x_wrt_rho, fp& partial_x_wrt_sigma,
	    fp& partial_y_wrt_r, fp& partial_y_wrt_rho, fp& partial_y_wrt_sigma,
	    fp& partial_z_wrt_r, fp& partial_z_wrt_rho, fp& partial_z_wrt_sigma)
		const
		{
		local_coords::partial_xyz_wrt_r_nu_phi
		   (r, rho, sigma,
		    partial_x_wrt_r, partial_x_wrt_rho, partial_x_wrt_sigma,
		    partial_y_wrt_r, partial_y_wrt_rho, partial_y_wrt_sigma,
		    partial_z_wrt_r, partial_z_wrt_rho, partial_z_wrt_sigma);
		}

	// partial (rho,sigma) / partial (x,y,z)
	fp partial_rho_wrt_x(fp x, fp y, fp z) const
		{ return local_coords::partial_nu_wrt_x(x,z); }
	fp partial_rho_wrt_y(fp x, fp y, fp z) const { return 0.0; }
	fp partial_rho_wrt_z(fp x, fp y, fp z) const
		{ return local_coords::partial_nu_wrt_z(x,z); }
	fp partial_sigma_wrt_x(fp x, fp y, fp z) const
		{ return local_coords::partial_phi_wrt_x(x,y); }
	fp partial_sigma_wrt_y(fp x, fp y, fp z) const
		{ return local_coords::partial_phi_wrt_y(x,y); }
	fp partial_sigma_wrt_z(fp x, fp y, fp z) const { return 0.0; }

	// partial^2 (rho,sigma) / partial (xx,xy,xz,yy,yz)
	fp partial2_rho_wrt_xx(fp x, fp y, fp z) const
		{ return local_coords::partial2_nu_wrt_xx(x,z); }
	fp partial2_rho_wrt_xy(fp x, fp y, fp z) const { return 0.0; }
	fp partial2_rho_wrt_xz(fp x, fp y, fp z) const
		{ return local_coords::partial2_nu_wrt_xz(x,z); }
	fp partial2_rho_wrt_yy(fp x, fp y, fp z) const { return 0.0; }
	fp partial2_rho_wrt_yz(fp x, fp y, fp z) const { return 0.0; }
	fp partial2_rho_wrt_zz(fp x, fp y, fp z) const
		{ return local_coords::partial2_nu_wrt_zz(x,z); }
	fp partial2_sigma_wrt_xx(fp x, fp y, fp z) const
		{ return local_coords::partial2_phi_wrt_xx(x,y); }
	fp partial2_sigma_wrt_xy(fp x, fp y, fp z) const
		{ return local_coords::partial2_phi_wrt_xy(x,y); }
	fp partial2_sigma_wrt_xz(fp x, fp y, fp z) const { return 0.0; }
	fp partial2_sigma_wrt_yy(fp x, fp y, fp z) const
		{ return local_coords::partial2_phi_wrt_yy(x,y); }
	fp partial2_sigma_wrt_yz(fp x, fp y, fp z) const { return 0.0; }
	fp partial2_sigma_wrt_zz(fp x, fp y, fp z) const { return 0.0; }

	// plotting coordinates (px,py)
	// ... character string describing how (dpx,dpy) are
	//     defined in terms of (mu,nu,phi), eg "90 - drho = 90 - dphi"
	//     (used for labelling output files)
	const char* name_of_dpx() const { return "drho = dnu"; }
	const char* name_of_dpy() const
		{
		return is_plus() ? "dsigma = dphi"
				 : "180 - dsigma = 180 - dphi";
		}
	// ... (irho,isimga) --> (px,py)
	fp dpx_of_rho_sigma(fp rho, fp sigma) const
		{ return jtutil::degrees_of_radians(rho); }
	fp dpy_of_rho_sigma(fp rho, fp sigma) const
		{
		const fp dsigma = jtutil::degrees_of_radians(sigma);
		return is_plus() ? dsigma : 180.0 - dsigma;
		}

	// compute the arc length of a surface in the specified plane
	// (must be one of "xy" or "xz") over the patch's nominal bounds
	// ... error_exit() if  plane  is invalid
	fp plane_arc_length(const char plane[],
			    int ghosted_radius_gfn,
			    int g_xx_gfn, int g_xy_gfn, int g_xz_gfn,
					  int g_yy_gfn, int g_yz_gfn,
							int g_zz_gfn,
			    enum integration_method method)
		const;

	// constructor, destructor
	x_patch(patch_system &my_patch_system_in, int patch_number_in,
		const char* name_in, bool is_plus_in,
		const grid_arrays::grid_array_pars& grid_array_pars_in,
		const grid::grid_pars& grid_pars_in);
	~x_patch() { }

private:
        // we forbid copying and passing by value
        // by declaring the copy constructor and assignment operator
        // private, but never defining them
	x_patch(const x_patch& rhs);
	x_patch& operator=(const x_patch& rhs);
	};

//*****************************************************************************

//
// This class describes a +/- y patch.  It doesn't define any new
// functions not already present in  class patch ; it "just" defines
// non-virtual versions of all the pure virtual functions defined there.
//
//	y patch ==> (rho,sigma) = (mu,phi)   tau = nu
//
class	y_patch
	: public patch
	{
public:
	// human-readable names of (rho,sigma)
	const char* name_of_rho() const { return "mu"; }
	const char* name_of_sigma() const { return "phi"; }

	// convert (rho,sigma) --> tau
	fp tau_of_rho_sigma(fp rho, fp sigma) const
		{ return local_coords::nu_of_mu_phi(rho, sigma); }

	// convert (rho,sigma) --> (mu,nu,phi)
	fp mu_of_rho_sigma(fp rho, fp sigma) const { return rho; }
	fp phi_of_rho_sigma(fp rho, fp sigma) const { return sigma; }
	fp nu_of_rho_sigma(fp rho, fp sigma) const
		{ return local_coords::nu_of_mu_phi(rho, sigma); }

	// convert (rho,sigma) <--> usual polar spherical (theta,phi)
	void theta_phi_of_rho_sigma(fp rho, fp sigma, fp& ps_theta, fp& ps_phi)
		const
		{
		local_coords::theta_phi_of_mu_phi(rho, sigma, ps_theta, ps_phi);
		}
	void rho_sigma_of_theta_phi(fp ps_theta, fp ps_phi, fp& rho, fp& sigma)
		const
		{
		local_coords::mu_phi_of_theta_phi(ps_theta, ps_phi, rho, sigma);
		}

	// convert (r,rho,sigma) <--> (x,y,z)
	void xyz_of_r_rho_sigma(fp r, fp rho, fp sigma, fp& x, fp& y, fp& z)
		const
		{ local_coords::xyz_of_r_mu_phi(r, rho, sigma, x, y, z); }
	fp rho_of_xyz(fp x, fp y, fp z) const
		{ return modulo_reduce_rho(local_coords::mu_of_yz(y, z)); }
	fp sigma_of_xyz(fp x, fp y, fp z) const
		{ return modulo_reduce_sigma(local_coords::phi_of_xy(x, y)); }

	// convert (rho,sigma) --> direction cosines (xcos,ycos,zcos)
	//                         with respect to the local coordinate system
	void xyzcos_of_rho_sigma(fp rho, fp sigma,
				 fp& xcos, fp& ycos, fp& zcos)
		const
		{ local_coords::xyzcos_of_mu_phi(rho,sigma, xcos,ycos,zcos); }

	// partial (x,y,z) / partial (rho,sigma)
	void partial_xyz_wrt_r_rho_sigma
	   (fp r, fp rho, fp sigma,
	    fp& partial_x_wrt_r, fp& partial_x_wrt_rho, fp& partial_x_wrt_sigma,
	    fp& partial_y_wrt_r, fp& partial_y_wrt_rho, fp& partial_y_wrt_sigma,
	    fp& partial_z_wrt_r, fp& partial_z_wrt_rho, fp& partial_z_wrt_sigma)
		const
		{
		local_coords::partial_xyz_wrt_r_mu_phi
		   (r, rho, sigma,
		    partial_x_wrt_r, partial_x_wrt_rho, partial_x_wrt_sigma,
		    partial_y_wrt_r, partial_y_wrt_rho, partial_y_wrt_sigma,
		    partial_z_wrt_r, partial_z_wrt_rho, partial_z_wrt_sigma);
		}

	// partial (rho,sigma) / partial (x,y,z)
	fp partial_rho_wrt_x(fp x, fp y, fp z) const { return 0.0; }
	fp partial_rho_wrt_y(fp x, fp y, fp z) const
		{ return local_coords::partial_mu_wrt_y(y,z); }
	fp partial_rho_wrt_z(fp x, fp y, fp z) const
		{ return local_coords::partial_mu_wrt_z(y,z); }
	fp partial_sigma_wrt_x(fp x, fp y, fp z) const
		{ return local_coords::partial_phi_wrt_x(x,y); }
	fp partial_sigma_wrt_y(fp x, fp y, fp z) const
		{ return local_coords::partial_phi_wrt_y(x,y); }
	fp partial_sigma_wrt_z(fp x, fp y, fp z) const { return 0.0; }

	// partial^2 (rho,sigma) / partial (xx,xy,xz,yy,yz)
	fp partial2_rho_wrt_xx(fp x, fp y, fp z) const { return 0.0; }
	fp partial2_rho_wrt_xy(fp x, fp y, fp z) const { return 0.0; }
	fp partial2_rho_wrt_xz(fp x, fp y, fp z) const { return 0.0; }
	fp partial2_rho_wrt_yy(fp x, fp y, fp z) const
		{ return local_coords::partial2_mu_wrt_yy(y,z); }
	fp partial2_rho_wrt_yz(fp x, fp y, fp z) const
		{ return local_coords::partial2_mu_wrt_yz(y,z); }
	fp partial2_rho_wrt_zz(fp x, fp y, fp z) const
		{ return local_coords::partial2_mu_wrt_zz(y,z); }
	fp partial2_sigma_wrt_xx(fp x, fp y, fp z) const
		{ return local_coords::partial2_phi_wrt_xx(x,y); }
	fp partial2_sigma_wrt_xy(fp x, fp y, fp z) const
		{ return local_coords::partial2_phi_wrt_xy(x,y); }
	fp partial2_sigma_wrt_xz(fp x, fp y, fp z) const { return 0.0; }
	fp partial2_sigma_wrt_yy(fp x, fp y, fp z) const
		{ return local_coords::partial2_phi_wrt_yy(x,y); }
	fp partial2_sigma_wrt_yz(fp x, fp y, fp z) const { return 0.0; }
	fp partial2_sigma_wrt_zz(fp x, fp y, fp z) const { return 0.0; }

	// plotting coordinates (px,py)
	// ... character string describing how (dpx,dpy) are
	//     defined in terms of (mu,nu,phi), eg "90 - drho = 90 - dphi"
	//     (used for labelling output files)
	const char* name_of_dpx() const
		{
		return is_plus() ? "90 - dsigma = 90 - dphi"
				 : "90 + dsigma = 90 + dphi";
		}
	const char* name_of_dpy() const { return "drho = dmu"; }
	// ... (rho,simga) --> (px,py)
	fp dpx_of_rho_sigma(fp rho, fp sigma) const
		{
		const fp dsigma = jtutil::degrees_of_radians(sigma);
		return is_plus() ? 90.0 - dsigma : 90.0 + dsigma;
		}
	fp dpy_of_rho_sigma(fp rho, fp sigma) const
		{ return jtutil::degrees_of_radians(rho); }

	// compute the arc length of a surface in the specified plane
	// (must be one of "xy" or "yz") over the patch's nominal bounds
	// ... error_exit() if  plane  is invalid
	fp plane_arc_length(const char plane[],
			    int ghosted_radius_gfn,
			    int g_xx_gfn, int g_xy_gfn, int g_xz_gfn,
					  int g_yy_gfn, int g_yz_gfn,
							int g_zz_gfn,
			    enum integration_method method)
		const;

	// constructor, destructor
	y_patch(patch_system &my_patch_system_in, int patch_number_in,
		const char* name_in, bool is_plus_in,
		const grid_arrays::grid_array_pars& grid_array_pars_in,
		const grid::grid_pars& grid_pars_in);
	~y_patch() { }

private:
        // we forbid copying and passing by value
        // by declaring the copy constructor and assignment operator
        // private, but never defining them
	y_patch(const y_patch& rhs);
	y_patch& operator=(const y_patch& rhs);
	};

//******************************************************************************

	  }	// namespace AHFinderDirect

```

patch/grid.cc:
```
// grid.cc -- classes for a 2D uniform tensor-product grid
// $Header$
//
// grid_arrays::grid_arrays
// grid_arrays::setup_gridfn_storage
// grid_arrays::~grid_arrays
//
// grid::grid
//

#include <stdio.h>
#include <assert.h>
#include <math.h>

#ifdef STANDALONE_TEST
  #include "fake_cctk.h"
#else
  #include "cctk.h"
#endif

#include "config.h"
#include "stdc.h"
#include "../jtutil/util.hh"
#include "../jtutil/array.hh"
#include "../jtutil/linear_map.hh"

#include "coords.hh"
#include "grid.hh"

// all the code in this file is inside this namespace
namespace AHFinderDirect
	  {

//*****************************************************************************
//*****************************************************************************
//*****************************************************************************

//
// This function constructs a  grid_arrays  object.
//
grid_arrays::grid_arrays(const grid_array_pars& grid_array_pars_in)

	: gridfn_data_(NULL),
	  ghosted_gridfn_data_(NULL),

	  // these are all set properly by setup_gridfn_storage()
	  min_gfn_(0), max_gfn_(0),
	  ghosted_min_gfn_(0), ghosted_max_gfn_(0),

	  min_irho_(grid_array_pars_in.min_irho),
	  max_irho_(grid_array_pars_in.max_irho),
	  min_isigma_(grid_array_pars_in.min_isigma),
	  max_isigma_(grid_array_pars_in.max_isigma),

	  ghosted_min_irho_(grid_array_pars_in.min_irho
			    - grid_array_pars_in.min_rho_ghost_zone_width),
	  ghosted_max_irho_(grid_array_pars_in.max_irho
			    + grid_array_pars_in.max_rho_ghost_zone_width),
	  ghosted_min_isigma_(grid_array_pars_in.min_isigma
			      - grid_array_pars_in.min_sigma_ghost_zone_width),
	  ghosted_max_isigma_(grid_array_pars_in.max_isigma
			      + grid_array_pars_in.max_sigma_ghost_zone_width)
								     // no comma
{ }

//*****************************************************************************

//
// This function sets up the gridfn storage arrays in a  grid_arrays  object.
//
void grid_arrays::setup_gridfn_storage
	(const gridfn_pars& gridfn_pars_in,
	 const gridfn_pars& ghosted_gridfn_pars_in)
{
assert(gridfn_data_ == NULL);
gridfn_data_ = new jtutil::array3d<fp>(gridfn_pars_in.min_gfn,
				       gridfn_pars_in.max_gfn,
				       min_irho(), max_irho(),
				       min_isigma(), max_isigma(),
				       gridfn_pars_in.storage_array,
				       gridfn_pars_in.gfn_stride,
				       gridfn_pars_in.irho_stride,
				       gridfn_pars_in.isigma_stride);

assert(ghosted_gridfn_data_ == NULL);
ghosted_gridfn_data_
	= new jtutil::array3d<fp>
			(ghosted_gridfn_pars_in.min_gfn,
			 ghosted_gridfn_pars_in.max_gfn,
			 ghosted_min_irho(), ghosted_max_irho(),
			 ghosted_min_isigma(), ghosted_max_isigma(),
			 ghosted_gridfn_pars_in.storage_array,
			 ghosted_gridfn_pars_in.gfn_stride,
			 ghosted_gridfn_pars_in.irho_stride,
			 ghosted_gridfn_pars_in.isigma_stride);
}

//******************************************************************************

//
// This function destroys a  grid_arrays  object.
//
grid_arrays::~grid_arrays()
{
delete ghosted_gridfn_data_;
delete gridfn_data_;
}

//*****************************************************************************
//*****************************************************************************
//*****************************************************************************

//
// This function constructs a  grid  object.
//
grid::grid(const grid_array_pars& grid_array_pars_in,
	   const grid_pars& grid_pars_in)

	: grid_arrays(grid_array_pars_in),

	  rho_map_(grid_array_pars_in.min_irho
		    - grid_array_pars_in.min_rho_ghost_zone_width,
		   grid_array_pars_in.max_irho
		    + grid_array_pars_in.max_rho_ghost_zone_width,
		   jtutil::radians_of_degrees(
		      grid_pars_in.min_drho
		       - grid_array_pars_in.min_rho_ghost_zone_width
			 * grid_pars_in.delta_drho
					     ),
		   jtutil::radians_of_degrees(grid_pars_in.delta_drho),
		   jtutil::radians_of_degrees(
		      grid_pars_in.max_drho
		       + grid_array_pars_in.max_rho_ghost_zone_width
			 * grid_pars_in.delta_drho
					     )),

	  sigma_map_(grid_array_pars_in.min_isigma
		      - grid_array_pars_in.min_sigma_ghost_zone_width,
		     grid_array_pars_in.max_isigma
		      + grid_array_pars_in.max_sigma_ghost_zone_width,
		     jtutil::radians_of_degrees(
			grid_pars_in.min_dsigma
			 - grid_array_pars_in.min_sigma_ghost_zone_width
			   * grid_pars_in.delta_dsigma
					       ),
		     jtutil::radians_of_degrees(grid_pars_in.delta_dsigma),
		     jtutil::radians_of_degrees(
			grid_pars_in.max_dsigma
			 + grid_array_pars_in.max_sigma_ghost_zone_width
			   * grid_pars_in.delta_dsigma
					       )),

	  min_rho_(jtutil::radians_of_degrees(grid_pars_in.min_drho)),
	  max_rho_(jtutil::radians_of_degrees(grid_pars_in.max_drho)),
	  min_sigma_(jtutil::radians_of_degrees(grid_pars_in.min_dsigma)),
	  max_sigma_(jtutil::radians_of_degrees(grid_pars_in.max_dsigma))
								     // no comma
{ }

//******************************************************************************
//******************************************************************************
//******************************************************************************

	  }	// namespace AHFinderDirect

```

patch/patch.cc:
```
// patch.cc -- describes a coordinate/grid patch
// $Header$

//
// patch::patch
// patch::~patch
// z_patch::z_patch
// x_patch::x_patch
// y_patch::y_patch
//
// patch::rho_sigma_metric
//
// patch::decode_integration_method
// patch::rho_arc_length
// patch::sigma_arc_length
// z_patch::plane_arc_length
// x_patch::plane_arc_length
// y_patch::plane_arc_length
// patch::integrate_gridfn
/// patch::integration_coeff
//
// patch::ghost_zone_on_edge
// patch::corner_ghost_zone_containing_point
// patch::ghost_zone_containing_noncorner_point
// patch::create_mirror_symmetry_ghost_zone
// patch::create_periodic_symmetry_ghost_zone
// patch::create_interpatch_ghost_zone
/// patch::set_ghost_zone
// patch::edge_adjacent_to_patch
// patch::assert_all_ghost_zones_fully_setup
//

#include <stdio.h>
#include <math.h>
#include <string.h>
#include <assert.h>

#include "cctk.h"

#include "config.h"
#include "stdc.h"
#include "../jtutil/util.hh"
#include "../jtutil/array.hh"
#include "../jtutil/cpm_map.hh"
#include "../jtutil/linear_map.hh"
using jtutil::error_exit;

#include "coords.hh"
#include "grid.hh"
#include "fd_grid.hh"
#include "patch.hh"
#include "patch_edge.hh"
#include "patch_interp.hh"
#include "ghost_zone.hh"

// all the code in this file is inside this namespace
namespace AHFinderDirect
	  {

//******************************************************************************
//******************************************************************************
//******************************************************************************

//
// This function constructs a  patch  object.
//
patch::patch(patch_system &my_patch_system_in, int patch_number_in,
	     const char name_in[], bool is_plus_in, char ctype_in,
	     local_coords::coords_set coords_set_rho_in,
	     local_coords::coords_set coords_set_sigma_in,
	     local_coords::coords_set coords_set_tau_in,
	     const grid_arrays::grid_array_pars& grid_array_pars_in,
	     const grid::grid_pars& grid_pars_in)

	: fd_grid(grid_array_pars_in, grid_pars_in),

	  my_patch_system_(my_patch_system_in),
	  patch_number_(patch_number_in),
	  name_(name_in),
	  is_plus_(is_plus_in), ctype_(ctype_in),

	  coords_set_rho_  (coords_set_rho_in  ),
	  coords_set_sigma_(coords_set_sigma_in),
	  coords_set_tau_  (coords_set_tau_in  ),

	  min_rho_patch_edge_(*new patch_edge(*this, side_is_min, side_is_rho)),
	  max_rho_patch_edge_(*new patch_edge(*this, side_is_max, side_is_rho)),
	  min_sigma_patch_edge_
	  	(*new patch_edge(*this, side_is_min, side_is_sigma)),
	  max_sigma_patch_edge_
		(*new patch_edge(*this, side_is_max, side_is_sigma)),

	  min_rho_ghost_zone_(NULL),
	  max_rho_ghost_zone_(NULL),
	  min_sigma_ghost_zone_(NULL),
	  max_sigma_ghost_zone_(NULL) // no comma

{ }

//******************************************************************************

//
// This function destroys a  patch  object.
//
patch::~patch()
{
// no need to check for null pointers, since  delete NULL  is a silent no-op

delete max_sigma_ghost_zone_;
delete min_sigma_ghost_zone_;
delete max_rho_ghost_zone_;
delete min_rho_ghost_zone_;

delete & max_sigma_patch_edge_;
delete & min_sigma_patch_edge_;
delete & max_rho_patch_edge_;
delete & min_rho_patch_edge_;
}

//******************************************************************************

//
// This function constructs a  z_patch  object.
//
z_patch::z_patch(patch_system &my_patch_system_in, int patch_number_in,
		 const char* name_in, bool is_plus_in,
		 const grid_arrays::grid_array_pars& grid_array_pars_in,
		 const grid::grid_pars& grid_pars_in)
	: patch(my_patch_system_in, patch_number_in,
		name_in, is_plus_in, 'z',
		local_coords::coords_set_mu, local_coords::coords_set_nu,
		local_coords::coords_set_phi,
		grid_array_pars_in, grid_pars_in)
{ }

//******************************************************************************

//
// This function constructs an  x_patch  object.
//
x_patch::x_patch(patch_system &my_patch_system_in, int patch_number_in,
		 const char* name_in, bool is_plus_in,
		 const grid_arrays::grid_array_pars& grid_array_pars_in,
		 const grid::grid_pars& grid_pars_in)
	: patch(my_patch_system_in, patch_number_in,
		name_in, is_plus_in, 'x',
		local_coords::coords_set_nu, local_coords::coords_set_phi,
		local_coords::coords_set_mu,
		grid_array_pars_in, grid_pars_in)
{ }

//******************************************************************************

//
// This function constructs a  y_patch  object.
//
y_patch::y_patch(patch_system &my_patch_system_in, int patch_number_in,
		 const char* name_in, bool is_plus_in,
		 const grid_arrays::grid_array_pars& grid_array_pars_in,
		 const grid::grid_pars& grid_pars_in)
	: patch(my_patch_system_in, patch_number_in,
		name_in, is_plus_in, 'y',
		local_coords::coords_set_mu, local_coords::coords_set_phi,
		local_coords::coords_set_nu,
		grid_array_pars_in, grid_pars_in)
{ }

//******************************************************************************
//******************************************************************************
//******************************************************************************

//
// This function computes the (rho,sigma) induced 2-D metric from the
// 3-D (x,y,z) metric of the space containing the patch, as per p.33 of
// my apparent horizon finding notes.
//
// Arguments:
// (r,rho,sigma) = The coordinates where the Jacobian is wanted.
// partial_surface_r_wrt_(rho,sigma)
//	= The partial derivatives of the surface radius with respect to
//	  the (rho,sigma) coordinates.
// g_{xx,xy,xz,yy,yz,zz} = The xyz 3-metric components $g_{ij}$.
// g_{rho_rho,rho_sigma,sigma_sigma} = The (rho,sigma) induced 2-D metric.
//
// Results:
// This function returns the Jacobian of the (rho,sigma) induced 2-D metric.
//
fp patch::rho_sigma_metric(fp r, fp rho, fp sigma,
			   fp partial_surface_r_wrt_rho,
			   fp partial_surface_r_wrt_sigma,
			   fp g_xx, fp g_xy, fp g_xz,
				    fp g_yy, fp g_yz,
					     fp g_zz,
			   fp& g_rho_rho, fp& g_rho_sigma,
					  fp& g_sigma_sigma)
	const
{
fp partial_x_wrt_r, partial_x_wrt_rho, partial_x_wrt_sigma;
fp partial_y_wrt_r, partial_y_wrt_rho, partial_y_wrt_sigma;
fp partial_z_wrt_r, partial_z_wrt_rho, partial_z_wrt_sigma;
partial_xyz_wrt_r_rho_sigma
	(r, rho, sigma,
	 partial_x_wrt_r, partial_x_wrt_rho, partial_x_wrt_sigma,
	 partial_y_wrt_r, partial_y_wrt_rho, partial_y_wrt_sigma,
	 partial_z_wrt_r, partial_z_wrt_rho, partial_z_wrt_sigma);

const fp dx_wrt_rho   = partial_x_wrt_rho
			+ partial_x_wrt_r*partial_surface_r_wrt_rho;
const fp dx_wrt_sigma = partial_x_wrt_sigma
			+ partial_x_wrt_r*partial_surface_r_wrt_sigma;
const fp dy_wrt_rho   = partial_y_wrt_rho
			+ partial_y_wrt_r*partial_surface_r_wrt_rho;
const fp dy_wrt_sigma = partial_y_wrt_sigma
			+ partial_y_wrt_r*partial_surface_r_wrt_sigma;
const fp dz_wrt_rho   = partial_z_wrt_rho
			+ partial_z_wrt_r*partial_surface_r_wrt_rho;
const fp dz_wrt_sigma = partial_z_wrt_sigma
			+ partial_z_wrt_r*partial_surface_r_wrt_sigma;

g_rho_rho     = +     dx_wrt_rho*dx_wrt_rho*g_xx
		+ 2.0*dx_wrt_rho*dy_wrt_rho*g_xy
		+ 2.0*dx_wrt_rho*dz_wrt_rho*g_xz
		+     dy_wrt_rho*dy_wrt_rho*g_yy
		+ 2.0*dy_wrt_rho*dz_wrt_rho*g_yz
		+     dz_wrt_rho*dz_wrt_rho*g_zz;
g_rho_sigma =   +  dx_wrt_rho*dx_wrt_sigma                           *g_xx
		+ (dx_wrt_rho*dy_wrt_sigma + dy_wrt_rho*dx_wrt_sigma)*g_xy
		+ (dx_wrt_rho*dz_wrt_sigma + dz_wrt_rho*dx_wrt_sigma)*g_xz
		+  dy_wrt_rho*dy_wrt_sigma                           *g_yy
		+ (dy_wrt_rho*dz_wrt_sigma + dz_wrt_rho*dy_wrt_sigma)*g_yz
		+  dz_wrt_rho*dz_wrt_sigma                           *g_zz;
g_sigma_sigma = + dx_wrt_sigma*dx_wrt_sigma*g_xx
		+ 2.0*dx_wrt_sigma*dy_wrt_sigma*g_xy
		+ 2.0*dx_wrt_sigma*dz_wrt_sigma*g_xz
		+     dy_wrt_sigma*dy_wrt_sigma*g_yy
		+ 2.0*dy_wrt_sigma*dz_wrt_sigma*g_yz
		+     dz_wrt_sigma*dz_wrt_sigma*g_zz;

return g_rho_rho*g_sigma_sigma - jtutil::pow2(g_rho_sigma);
}

//******************************************************************************
//******************************************************************************
//******************************************************************************

//
// This function decodes the character-string name of an integration method
// into an  enum integration_method .  See the comments in "patch.hh" on the
// declaration of  enum integration_method  for details on the methods and
// their character-string names.
//
//static
  enum patch::integration_method
    patch::decode_integration_method(const char method_string[])
{
if	(    STRING_EQUAL(method_string, "trapezoid")
	  || STRING_EQUAL(method_string, "trapezoid rule")    )
   then return integration_method__trapezoid;
else if (    STRING_EQUAL(method_string, "Simpson")
	  || STRING_EQUAL(method_string, "Simpson's rule")    )
   then return integration_method__Simpson;
else if (    STRING_EQUAL(method_string, "Simpson (variant)")
	  || STRING_EQUAL(method_string, "Simpson's rule (variant)")    )
   then return integration_method__Simpson_variant;
else if (    STRING_EQUAL(method_string, "automatic choice")    )
   then return integration_method__automatic_choice;
else	error_exit(ERROR_EXIT,
"***** patch::decode_integration_method():\n"
"        unknown method_string=\"%s\"!\n"
,
		   method_string);				/*NOTREACHED*/
}

//******************************************************************************

//
// This function computes an approximation to the arc length of a surface
// over the patch's nominal bounds along the rho direction (i.e. in a
// dsigma=constant plane where dsigma is a multiple of 90 degrees)
//
// Arguments:
// ghosted_radius_gfn = (in) The surface radius.
// g_{xx,xy,xz,yy,yz,zz}_gfn = (in) The xyz 3-metric components.
// method = (in) Selects the integration scheme.
//
fp patch::rho_arc_length(int ghosted_radius_gfn,
			 int g_xx_gfn, int g_xy_gfn, int g_xz_gfn,
				       int g_yy_gfn, int g_yz_gfn,
						     int g_zz_gfn,
			 enum integration_method method)
	const
{
fp dsigma;
if	(is_valid_dsigma(  0.0)) then dsigma =   0.0;
else if (is_valid_dsigma( 90.0)) then dsigma =  90.0;
else if (is_valid_dsigma(180.0)) then dsigma = 180.0;
else if (is_valid_dsigma(-90.0)) then dsigma = -90.0;
else	error_exit(PANIC_EXIT,
"***** patch::rho_arc_length(): can't find valid dsigma\n"
"                               which is a multiple of 90 degrees!\n"
"                               %s patch: [min,max]_dsigma()=[%g,%g]\n"
		   ,
		   name(), double(min_dsigma()), double(max_dsigma()));
const fp sigma = sigma_of_dsigma(dsigma);
const int isigma = isigma_of_sigma(sigma);

fp sum = 0.0;

	for (int irho = min_irho() ; irho <= max_irho() ; ++irho)
	{
	const fp rho = rho_of_irho(irho);
	const fp r = ghosted_gridfn(ghosted_radius_gfn, irho,isigma);
	const fp partial_surface_r_wrt_rho
		= partial_rho  (ghosted_radius_gfn, irho,isigma);
	const fp partial_surface_r_wrt_sigma
		= partial_sigma(ghosted_radius_gfn, irho,isigma);

	const fp g_xx = gridfn(g_xx_gfn, irho,isigma);
	const fp g_xy = gridfn(g_xy_gfn, irho,isigma);
	const fp g_xz = gridfn(g_xz_gfn, irho,isigma);
	const fp g_yy = gridfn(g_yy_gfn, irho,isigma);
	const fp g_yz = gridfn(g_yz_gfn, irho,isigma);
	const fp g_zz = gridfn(g_zz_gfn, irho,isigma);

	fp g_rho_rho, g_rho_sigma, g_sigma_sigma;
	rho_sigma_metric(r, rho, sigma,
			 partial_surface_r_wrt_rho,
			 partial_surface_r_wrt_sigma,
			 g_xx, g_xy, g_xz,
			       g_yy, g_yz,
				     g_zz,
			 g_rho_rho, g_rho_sigma,
				    g_sigma_sigma);

	const fp coeff = integration_coeff(method,
					   max_irho()-min_irho(),
					   irho      -min_irho());

	sum += coeff * sqrt(g_rho_rho);
	}

return delta_rho() * sum;
}

//******************************************************************************

//
// This function computes an approximation to the arc length of a surface
// over the patch's nominal bounds along the sigma direction (i.e. in a
// drho=constant plane where drho is a multiple of 90 degrees)
//
// Arguments:
// ghosted_radius_gfn = (in) The surface radius.
// g_{xx,xy,xz,yy,yz,zz}_gfn = (in) The xyz 3-metric components.
// method = (in) Selects the integration scheme.
//
fp patch::sigma_arc_length(int ghosted_radius_gfn,
			   int g_xx_gfn, int g_xy_gfn, int g_xz_gfn,
					 int g_yy_gfn, int g_yz_gfn,
						       int g_zz_gfn,
			   enum integration_method method)
	const
{
fp drho;
if	(is_valid_drho(  0.0)) then drho =   0.0;
else if (is_valid_drho( 90.0)) then drho =  90.0;
else if (is_valid_drho(180.0)) then drho = 180.0;
else if (is_valid_drho(-90.0)) then drho = -90.0;
else	error_exit(PANIC_EXIT,
"***** patch::sigma_arc_length(): can't find valid drho\n"
"                                 which is a multiple of 90 degrees!\n"
"                                 %s patch: [min,max]_drho()=[%g,%g]\n"
		   ,
		   name(), double(min_drho()), double(max_drho()));
const fp rho = rho_of_drho(drho);
const int irho = irho_of_rho(rho);

fp sum = 0.0;

	for (int isigma = min_isigma() ; isigma <= max_isigma() ; ++isigma)
	{
	const fp sigma = sigma_of_isigma(isigma);
	const fp r = ghosted_gridfn(ghosted_radius_gfn, irho,isigma);
	const fp partial_surface_r_wrt_rho
		= partial_rho  (ghosted_radius_gfn, irho,isigma);
	const fp partial_surface_r_wrt_sigma
		= partial_sigma(ghosted_radius_gfn, irho,isigma);

	const fp g_xx = gridfn(g_xx_gfn, irho,isigma);
	const fp g_xy = gridfn(g_xy_gfn, irho,isigma);
	const fp g_xz = gridfn(g_xz_gfn, irho,isigma);
	const fp g_yy = gridfn(g_yy_gfn, irho,isigma);
	const fp g_yz = gridfn(g_yz_gfn, irho,isigma);
	const fp g_zz = gridfn(g_zz_gfn, irho,isigma);

	fp g_rho_rho, g_rho_sigma, g_sigma_sigma;
	rho_sigma_metric(r, rho, sigma,
			 partial_surface_r_wrt_rho,
			 partial_surface_r_wrt_sigma,
			 g_xx, g_xy, g_xz,
			       g_yy, g_yz,
				     g_zz,
			 g_rho_rho, g_rho_sigma,
				    g_sigma_sigma);

	const fp coeff = integration_coeff(method,
					   max_isigma()-min_isigma(),
					   isigma      -min_isigma());

	sum += coeff * sqrt(g_sigma_sigma);
	}

return delta_sigma() * sum;
}

//******************************************************************************

//
// This function computes the arc length of a surface in the specified
// plane ("xz" or "yz") over the patch's nominal bounds.
//
// Arguments:
// plane[] = (in) "xz" or "yz" to specify the plane.
// ghosted_radius_gfn = (in) The surface radius.
// g_{xx,xy,xz,yy,yz,zz}_gfn = (in) The xyz 3-metric components.
// method = (in) Selects the integration scheme.
//
fp z_patch::plane_arc_length(const char plane[],
			     int ghosted_radius_gfn,
			     int g_xx_gfn, int g_xy_gfn, int g_xz_gfn,
					   int g_yy_gfn, int g_yz_gfn,
							 int g_zz_gfn,
			     enum integration_method method)
	const
{
if	((plane[0] == 'x') && (plane[1] == 'z'))
   then // xz-plane = rotation about y = nu arc = sigma sigma
	return sigma_arc_length(ghosted_radius_gfn,
				g_xx_gfn, g_xy_gfn, g_xz_gfn,
					  g_yy_gfn, g_yz_gfn,
						    g_zz_gfn,
				method);
else if ((plane[0] == 'y') && (plane[1] == 'z'))
   then // yz-plane = rotation about x = mu arc = rho arc
	return rho_arc_length(ghosted_radius_gfn,
			      g_xx_gfn, g_xy_gfn, g_xz_gfn,
					g_yy_gfn, g_yz_gfn,
						  g_zz_gfn,
			      method);
else	error_exit(ERROR_EXIT,
"***** z_patch::plane_arc_length(): %s patch, plane=\"%s\", but\n"
"                                   this patch doesn't contain that plane!\n"
,
		   name(), plane);				/*NOTREACHED*/
}

//******************************************************************************

//
// This function computes the arc length of a surface in the specified
// plane ("xy" or "xz") over the patch's nominal bounds.
//
// Arguments:
// plane[] = (in) "xy" or "xz" to specify the plane.
// ghosted_radius_gfn = (in) The surface radius.
// g_{xx,xy,xz,yy,yz,zz}_gfn = (in) The xyz 3-metric components.
// method = (in) Selects the integration scheme.
//
fp x_patch::plane_arc_length(const char plane[],
			     int ghosted_radius_gfn,
			     int g_xx_gfn, int g_xy_gfn, int g_xz_gfn,
					   int g_yy_gfn, int g_yz_gfn,
							 int g_zz_gfn,
			     enum integration_method method)
	const
{
if	((plane[0] == 'x') && (plane[1] == 'y'))
   then // xy-plane = rotation about z = phi arc = sigma arc
	return sigma_arc_length(ghosted_radius_gfn,
				g_xx_gfn, g_xy_gfn, g_xz_gfn,
					  g_yy_gfn, g_yz_gfn,
						    g_zz_gfn,
				method);
else if ((plane[0] == 'x') && (plane[1] == 'z'))
   then // xz-plane = rotation about y = nu arc = rho arc
	return rho_arc_length(ghosted_radius_gfn,
			      g_xx_gfn, g_xy_gfn, g_xz_gfn,
					g_yy_gfn, g_yz_gfn,
						  g_zz_gfn,
			      method);
else	error_exit(ERROR_EXIT,
"***** x_patch::plane_arc_length(): %s patch, plane=\"%s\", but\n"
"                                   this patch doesn't contain that plane!\n"
,
		   name(), plane);				/*NOTREACHED*/
}

//******************************************************************************

//
// This function computes the arc length of a surface in the specified
// plane ("xy" or "yz") over the patch's nominal bounds.
//
// Arguments:
// plane[] = (in) "xy" or "yz" to specify the plane.
// ghosted_radius_gfn = (in) The surface radius.
// g_{xx,xy,xz,yy,yz,zz}_gfn = (in) The xyz 3-metric components.
// method = (in) Selects the integration scheme.
//
fp y_patch::plane_arc_length(const char plane[],
			     int ghosted_radius_gfn,
			     int g_xx_gfn, int g_xy_gfn, int g_xz_gfn,
					   int g_yy_gfn, int g_yz_gfn,
							 int g_zz_gfn,
			     enum integration_method method)
	const
{
if	((plane[0] == 'x') && (plane[1] == 'y'))
   then // xy-plane = rotation about z = phi arc = sigma arc
	return sigma_arc_length(ghosted_radius_gfn,
				g_xx_gfn, g_xy_gfn, g_xz_gfn,
					  g_yy_gfn, g_yz_gfn,
						    g_zz_gfn,
				method);
else if ((plane[0] == 'y') && (plane[1] == 'z'))
   then // yz-plane = rotation about x = mu arc = rho arc
	return rho_arc_length(ghosted_radius_gfn,
			      g_xx_gfn, g_xy_gfn, g_xz_gfn,
					g_yy_gfn, g_yz_gfn,
						  g_zz_gfn,
			      method);
else	error_exit(ERROR_EXIT,
"***** y_patch::plane_arc_length(): %s patch, plane=\"%s\", but\n"
"                                   this patch doesn't contain that plane!\n"
,
		   name(), plane);				/*NOTREACHED*/
}

//******************************************************************************

//
// This function computes an approximation to the (surface) integral of
// a gridfn over the patch's nominal area,
//	$\int f(\rho,\sigma) \, dA$
//		= \int f(\rho,\sigma) \sqrt{|J|} \, d\rho \, d\sigma$
// where $J$ is the Jacobian of $(x,y,z)$ with respect to $(rho,sigma).
//
// Arguments:
// unknown_src_gfn = (in) The gridfn to be integrated.  This may be
//			  either nominal-grid or ghosted-grid; n.b. in
//			  the latter case the integral is still done only
//			  over the patch's nominal area.
// ghosted_radius_gfn = (in) The surface radius.
// g_{xx,xy,xz,yy,yz,zz}_gfn = (in) The xyz 3-metric components.
// method = (in) Selects the integration scheme.
//
fp patch::integrate_gridfn(int unknown_src_gfn,
			   int ghosted_radius_gfn,
			   int g_xx_gfn, int g_xy_gfn, int g_xz_gfn,
					 int g_yy_gfn, int g_yz_gfn,
						       int g_zz_gfn,
			   enum integration_method method)
	const
{
const bool src_is_ghosted = is_valid_ghosted_gfn(unknown_src_gfn);

fp sum = 0.0;
	for (int irho = min_irho() ; irho <= max_irho() ; ++irho)
	{
	for (int isigma = min_isigma() ; isigma <= max_isigma() ; ++isigma)
	{
	const fp fn = unknown_gridfn(src_is_ghosted,
				     unknown_src_gfn, irho,isigma);

	const fp rho   = rho_of_irho    (irho);
	const fp sigma = sigma_of_isigma(isigma);
	const fp r = ghosted_gridfn(ghosted_radius_gfn, irho,isigma);
	const fp partial_surface_r_wrt_rho
		= partial_rho  (ghosted_radius_gfn, irho,isigma);
	const fp partial_surface_r_wrt_sigma
		= partial_sigma(ghosted_radius_gfn, irho,isigma);

	const fp g_xx = gridfn(g_xx_gfn, irho,isigma);
	const fp g_xy = gridfn(g_xy_gfn, irho,isigma);
	const fp g_xz = gridfn(g_xz_gfn, irho,isigma);
	const fp g_yy = gridfn(g_yy_gfn, irho,isigma);
	const fp g_yz = gridfn(g_yz_gfn, irho,isigma);
	const fp g_zz = gridfn(g_zz_gfn, irho,isigma);

	fp g_rho_rho, g_rho_sigma, g_sigma_sigma;
	const fp Jac = rho_sigma_metric(r, rho, sigma,
					partial_surface_r_wrt_rho,
					partial_surface_r_wrt_sigma,
					g_xx, g_xy, g_xz,
					      g_yy, g_yz,
						    g_zz,
					g_rho_rho, g_rho_sigma,
						   g_sigma_sigma);

	const fp coeff_rho   = integration_coeff(method,
						 max_irho()-min_irho(),
						 irho      -min_irho());
	const fp coeff_sigma = integration_coeff(method,
						 max_isigma()-min_isigma(),
						 isigma      -min_isigma());

	sum += coeff_rho*coeff_sigma * fn * sqrt(jtutil::abs(Jac));
	}
	}

return delta_rho() * delta_sigma() * sum;
}

fp patch::integrate_gridpoint(int unknown_src_gfn,
                              int ghosted_radius_gfn,
                              int g_xx_gfn, int g_xy_gfn, int g_xz_gfn,
					    int g_yy_gfn, int g_yz_gfn,
                                                          int g_zz_gfn,
                              enum integration_method method,
                              int irho, int isigma)
	const
{
const bool src_is_ghosted = is_valid_ghosted_gfn(unknown_src_gfn);

const fp fn = unknown_gridfn(src_is_ghosted,
			     unknown_src_gfn, irho,isigma);

const fp rho   = rho_of_irho    (irho);
const fp sigma = sigma_of_isigma(isigma);
const fp r = ghosted_gridfn(ghosted_radius_gfn, irho,isigma);
const fp partial_surface_r_wrt_rho
	= partial_rho  (ghosted_radius_gfn, irho,isigma);
const fp partial_surface_r_wrt_sigma
	= partial_sigma(ghosted_radius_gfn, irho,isigma);

const fp g_xx = gridfn(g_xx_gfn, irho,isigma);
const fp g_xy = gridfn(g_xy_gfn, irho,isigma);
const fp g_xz = gridfn(g_xz_gfn, irho,isigma);
const fp g_yy = gridfn(g_yy_gfn, irho,isigma);
const fp g_yz = gridfn(g_yz_gfn, irho,isigma);
const fp g_zz = gridfn(g_zz_gfn, irho,isigma);

fp g_rho_rho, g_rho_sigma, g_sigma_sigma;
const fp Jac = rho_sigma_metric(r, rho, sigma,
				partial_surface_r_wrt_rho,
				partial_surface_r_wrt_sigma,
				g_xx, g_xy, g_xz,
				      g_yy, g_yz,
					    g_zz,
				g_rho_rho, g_rho_sigma,
					   g_sigma_sigma);

const fp coeff_rho   = integration_coeff(method,
					 max_irho()-min_irho(),
					 irho      -min_irho());
const fp coeff_sigma = integration_coeff(method,
					 max_isigma()-min_isigma(),
					 isigma      -min_isigma());

const fp val = coeff_rho*coeff_sigma * fn * sqrt(jtutil::abs(Jac));

return delta_rho() * delta_sigma() * val;
}

//******************************************************************************

//
// This function computes the integration coefficients for
//  integrate_gridfn() .  That is, if we write
//	$\int_{x_0}^{x_N} f(x) \, dx
//		\approx \Delta x \, \sum_{i=0}^N c_i f(x_i)$
// then this function computes $c_i$.
//
// For method == integration_method__automatic_choice the choices are
//	N=1		trapezoid
//	N=2		Simpson
//	N=3		trapezoid
//	N=4		Simpson
//	N=5		trapezoid
//	N=6		Simpson
//	N=7 and up	Simpson variant
//
// Arguments:
// method = Specifies the integration method.
// N = The number of integration *intervals*.  (The number of integration
//     *points* is N+1.)
// i = Specifies the point at which the coefficient is desired.
//
//static
  fp patch::integration_coeff(enum integration_method method, int N, int i)
{
assert(i >= 0);
assert(i <= N);

if (method == integration_method__automatic_choice)
   then {
	if	(N >= 7)
	   then method = integration_method__Simpson_variant;
	else if ((N % 2) == 0)
	   then method = integration_method__Simpson;
	else	method = integration_method__trapezoid;
	}

switch	(method)
	{
case integration_method__trapezoid:
	if ((i == 0) || (i == N))
	   then return 0.5;
	   else return 1.0;

case integration_method__Simpson:
	if ((N % 2) != 0)
	   then error_exit(ERROR_EXIT,
"***** patch::integration_coeff():\n"
"        Simpson's rule requires N to be even, but N=%d!\n",
			   N);					/*NOTREACHED*/
	if	((i == 0) || (i == N))
	   then return 1.0/3.0;
	else if ((i % 2) == 0)
	   then return 2.0/3.0;
	else	return 4.0/3.0;

case integration_method__Simpson_variant:
	if (N < 7)
	   then error_exit(ERROR_EXIT,
"***** patch::integration_coeff():\n"
"        Simpson's rule (variant) requires N >= 7, but N=%d!\n",
			   N);					/*NOTREACHED*/
	if	((i == 0) || (i == N))
	   then return 17.0/48.0;
	else if ((i == 1) || (i == N-1))
	   then return 59.0/48.0;
	else if ((i == 2) || (i == N-2))
	   then return 43.0/48.0;
	else if ((i == 3) || (i == N-3))
	   then return 49.0/48.0;
	else	return 1.0;

default:
	error_exit(ERROR_EXIT,
"***** patch::integration_coeff(): unknown method=(int)%d!\n"
"                                  (this should never happen!)\n"
,
		   int(method));				/*NOTREACHED*/
	}
}

//******************************************************************************
//******************************************************************************
//******************************************************************************

//
// This function returns a reference to the ghost zone on a specified
// edge, after first assert()ing that the edge belongs to this patch.
//
// N.b. This function can't be inline in "patch.hh" because it needs
//	member functions of class patch_edge, which comes after class patch
//	in our #include order.
//
ghost_zone& patch::ghost_zone_on_edge(const patch_edge& e)
	const
{
assert(e.my_patch() == *this);
return minmax_ang_ghost_zone(e.is_min(), e.is_rho());
}

//******************************************************************************

//
// This function determines which of the two adjacent ghost zones meeting
// at a specified corner, contains a specified point.  If the point isn't
// in either ghost zone, an error_exit() is done.  If the point is in both
// ghost zones, it's arbitrary which one will be chosen.
//
// Arguments:
// {rho,sigma}_is_min = Specify the corner (and implicitly the ghost zones).
// irho,isigma = Specify the point.
//
// Results:
// This function returns (a reference to) the desired ghost zone.
ghost_zone& patch::corner_ghost_zone_containing_point
	(bool rho_is_min, bool sigma_is_min,
	 int irho, int isigma)
	const
{
ghost_zone&   rho_gz =   minmax_rho_ghost_zone(  rho_is_min);
ghost_zone& sigma_gz = minmax_sigma_ghost_zone(sigma_is_min);

const patch_edge&   rho_edge =   rho_gz.my_edge();
const patch_edge& sigma_edge = sigma_gz.my_edge();

const int   rho_iperp =   rho_edge.iperp_of_irho_isigma(irho, isigma);
const int   rho_ipar  =   rho_edge. ipar_of_irho_isigma(irho, isigma);
const int sigma_iperp = sigma_edge.iperp_of_irho_isigma(irho, isigma);
const int sigma_ipar  = sigma_edge. ipar_of_irho_isigma(irho, isigma);

const bool is_in_rho_ghost_zone
	=   rho_gz.is_in_ghost_zone(  rho_iperp,   rho_ipar);
const bool is_in_sigma_ghost_zone
	= sigma_gz.is_in_ghost_zone(sigma_iperp, sigma_ipar);

// check that point is in at least one ghost zone
if (!is_in_rho_ghost_zone && !is_in_sigma_ghost_zone)
   then error_exit(ERROR_EXIT,
"***** patch::corner_ghost_zone_containing_point():\n"
"        neither ghost zone contains point (this should never happen)!\n"
"        patch=%s rho_is_min=(int)%d sigma_is_min=(int)%d\n"
"        irho=%d isigma=%d\n"
,
	   name(), int(rho_is_min), int(sigma_is_min),
	   irho, isigma);					/*NOTREACHED*/

return is_in_rho_ghost_zone ? rho_gz : sigma_gz;
}

//******************************************************************************

//
// This function determines which ghost zone contains a specified
// noncorner point.
//
// If the point isn't in any ghost zone of this patch, or if the point
// is in the corner of a ghost zone, an error_exit() is done.
//
// Arguments:
// irho,isigma = Specify the point.
//
// Results:
// This function returns (a reference to) the desired ghost zone.
ghost_zone& patch::ghost_zone_containing_noncorner_point(int irho, int isigma)
	const
{
	// n.b. these loops must use _int_ variables for the loop
	//      to terminate!
	for (int want_min = false ; want_min <= true ; ++want_min)
	{
	for (int want_rho = false ; want_rho <= true ; ++want_rho)
	{
	const patch_edge& e = minmax_ang_patch_edge(want_min, want_rho);
	const int iperp = e.iperp_of_irho_isigma(irho, isigma);
	const int ipar  = e.ipar_of_irho_isigma (irho, isigma);

	ghost_zone& gz = minmax_ang_ghost_zone(want_min, want_rho);
	if ( gz.is_in_ghost_zone(iperp, ipar)
	     && gz.my_edge().ipar_is_in_noncorner(ipar) )
	   then return gz;
	}
	}

error_exit(ERROR_EXIT,
"***** patch::ghost_zone_containing_noncorner_point():\n"
"        no ghost zone contains point (this should never happen)!\n"
"        patch=%s irho=%d isigma=%d\n"
,
	   name(), irho, isigma);				/*NOTREACHED*/
}

//******************************************************************************

//
// This function assert()s that a specified ghost zone of this patch
// hasn't already been set up, then constructs it as a mirror-symmetry
// ghost zone and properly links this to/from the patch.
//
void patch::create_mirror_symmetry_ghost_zone(const patch_edge& my_edge)
{
// make sure we belong to the right patch
assert(my_edge.my_patch() == *this);

symmetry_ghost_zone *temp = new symmetry_ghost_zone(my_edge);
set_ghost_zone(my_edge, temp);
}

//******************************************************************************

//
// This function assert()s that a specified ghost zone of this patch
// hasn't already been set up, then creates it as a periodic-symmetry
// ghost zone and properly links this to/from the patch.
//
void patch::create_periodic_symmetry_ghost_zone
	(const patch_edge& my_edge, const patch_edge& other_edge,
	 bool is_ipar_map_plus)
{
// make sure we belong to the right patch
assert(my_edge.my_patch() == *this);

int my_sample_ipar = my_edge.min_ipar_without_corners();
int other_sample_ipar = is_ipar_map_plus
			? other_edge.min_ipar_without_corners()
			: other_edge.max_ipar_without_corners();

symmetry_ghost_zone *temp
	= new symmetry_ghost_zone(my_edge,        other_edge,
				  my_sample_ipar, other_sample_ipar,
				  is_ipar_map_plus);
set_ghost_zone(my_edge, temp);
}

//******************************************************************************

//
// This function assert()s that a specified ghost zone of this patch
// hasn't already been set up, then creates it as an interpatch ghost
// zone (with lots of NULL pointers for info we can't compute yet)
// and properly links this to/from the patch.
//
void patch::create_interpatch_ghost_zone
	(const patch_edge& my_edge, const patch_edge& other_edge,
	 int patch_overlap_width)
{
// make sure we belong to the right patch
assert(my_edge.my_patch() == *this);

interpatch_ghost_zone *temp
	= new interpatch_ghost_zone(my_edge, other_edge,
				    patch_overlap_width);
set_ghost_zone(my_edge, temp);
}

//******************************************************************************

//
// This is a helper function for  setup_*_ghost_zone().  This function
// assert()s that one of the ghost zone pointers (which one is selected
// by  edge ) is NULL, then stores a value in it.
//
void patch::set_ghost_zone(const patch_edge& edge, ghost_zone* gzp)
{
ghost_zone*& ghost_zone_ptr_to_set
	= edge.is_min()
	  ? (edge.is_rho() ? min_rho_ghost_zone_ : min_sigma_ghost_zone_)
	  : (edge.is_rho() ? max_rho_ghost_zone_ : max_sigma_ghost_zone_);

assert(ghost_zone_ptr_to_set == NULL);
ghost_zone_ptr_to_set = gzp;
}

//******************************************************************************

//
// This function finds which patch edge is adjacent to a neighboring
// patch q, or does an error_exit() if q isn't actually a neighboring patch.
// The computation is done using only (rho,sigma) coordinate sets and
// min/max dang bounds ==> it's ok to use this function in setting up
// interpatch ghost zones.
//
// Arguments:
// q = The (supposedly) neighboring patch.
// patch_overlap_width = The number of grid points these patches overlap.
//		      If this is nonzero, then these patches must have the
//		      same grid spacing in the perpendicular direction.
//
const patch_edge& patch::edge_adjacent_to_patch(const patch& q,
						int patch_overlap_width /* = 0 */)
	const
{
const patch& p = *this;

// which (rho,sigma) coordinate do the patches have in common?
// ... this is the perp coordinate for the border
const local_coords::coords_set common_coord_set
	= p.coords_set_rho_sigma() & q.coords_set_rho_sigma();

// is this coordinate rho or sigma in each patch?
const bool common_is_p_rho   = (common_coord_set == p.coords_set_rho  ());
const bool common_is_p_sigma = (common_coord_set == p.coords_set_sigma());
if ((common_is_p_rho ^ common_is_p_sigma) != 0x1)
   then error_exit(ERROR_EXIT,
"***** patch::edge_adjacent_to_patch():\n"
"        common coordinate isn't exactly one of p.{rho,sigma}!\n"
"        p.name()=\"%s\" q.name()=\"%s\"\n"
"        common_coord_set=%s\n"
"        common_is_p_rho=%d common_is_p_sigma=%d\n"
,
		   p.name(), q.name(),
		   local_coords::name_of_coords_set(common_coord_set),
		   int(common_is_p_rho), int(common_is_p_sigma));
								/*NOTREACHED*/
const bool common_is_q_rho   = (common_coord_set == q.coords_set_rho  ());
const bool common_is_q_sigma = (common_coord_set == q.coords_set_sigma());
if ((common_is_q_rho ^ common_is_q_sigma) != 0x1)
   then error_exit(ERROR_EXIT,
"***** patch::edge_adjacent_to_patch():\n"
"        common coordinate isn't exactly one of q.{rho,sigma}!\n"
"        p.name()=\"%s\" q.name()=\"%s\"\n"
"        common_coord_set=%s\n"
"        common_is_q_rho=%d common_is_q_sigma=%d\n"
,
		   p.name(), q.name(),
		   local_coords::name_of_coords_set(common_coord_set),
		   int(common_is_q_rho), int(common_is_q_sigma));
								/*NOTREACHED*/

// how much do the patches overlap?
// ... eg patch_overlap_width = 3 would be
//	p   p   p   p   p
//		q   q   q   q   q
//     so the overlap would be (patch_overlap_width-1) * delta = 2 * delta
if ( (patch_overlap_width-1 != 0)
     && jtutil::fuzzy<fp>::NE(p.delta_dang(common_is_p_rho),
			      q.delta_dang(common_is_q_rho)) )
   then error_exit(ERROR_EXIT,
"***** patch::edge_adjacent_to_patch():\n"
"        patch_overlap_width != 0 must have same perp grid spacing in both patches!\n"
"        p.name()=\"%s\" q.name()=\"%s\"\n"
"        common_coord_set=%s\n"
"        common_is_p_rho=%d common_is_q_rho=%d\n"
"        p.delta_dang(common_is_p_rho)=%g\n"
"        q.delta_dang(common_is_q_rho)=%g\n"
,
		   p.name(), q.name(),
		   local_coords::name_of_coords_set(common_coord_set),
		   int(common_is_p_rho), int(common_is_q_rho),
		   double(p.delta_dang(common_is_p_rho)),
		   double(q.delta_dang(common_is_q_rho)));	/*NOTREACHED*/


const fp doverlap = fp(patch_overlap_width-1) * p.delta_dang(common_is_p_rho);

// where is the common boundary relative to the min/max sides of each patch?
const bool common_is_p_min_q_max
    = local_coords::fuzzy_EQ_dang(p.min_dang(common_is_p_rho),
				  q.max_dang(common_is_q_rho) - doverlap);
const bool common_is_p_max_q_min
    = local_coords::fuzzy_EQ_dang(p.max_dang(common_is_p_rho),
				  q.min_dang(common_is_q_rho) + doverlap);
if ((common_is_p_min_q_max ^ common_is_p_max_q_min) != 0x1)
   then error_exit(ERROR_EXIT,
"***** patch::edge_adjacent_to_patch():\n"
"        common coordinate isn't exactly one of {pmax/qmin, pmin/qmax}!\n"
"        p.name()=\"%s\" q.name()=\"%s\"\n"
"        common_coord_set=%s\n"
"        common_is_p_rho=%d common_is_q_rho=%d\n"
"        p.delta_dang(common_is_p_rho)=%g\n"
"        q.delta_dang(common_is_q_rho)=%g\n"
"        patch_overlap_width=%d doverlap=%g\n"
"        common_is_p_min_q_max=%d common_is_p_max_q_min=%d\n"
,
		   p.name(), q.name(),
		   local_coords::name_of_coords_set(common_coord_set),
		   int(common_is_p_rho), int(common_is_q_rho),
		   double(p.delta_dang(common_is_p_rho)),
		   double(q.delta_dang(common_is_q_rho)),
		   patch_overlap_width, double(doverlap),
		   int(common_is_p_min_q_max), int(common_is_p_max_q_min));
								/*NOTREACHED*/

return p.minmax_ang_patch_edge(common_is_p_min_q_max, common_is_p_rho);
}

//******************************************************************************

//
// This function verifies (via assert()) that all ghost zones of this
// patch have been fully set up.
//
void patch::assert_all_ghost_zones_fully_setup() const
{
assert(min_rho_ghost_zone_ != NULL);
assert(max_rho_ghost_zone_ != NULL);
assert(min_sigma_ghost_zone_ != NULL);
assert(max_sigma_ghost_zone_ != NULL);

// these calls are no-ops for non-interpatch ghost zones
min_rho_ghost_zone().assert_fully_setup();
max_rho_ghost_zone().assert_fully_setup();
min_sigma_ghost_zone().assert_fully_setup();
max_sigma_ghost_zone().assert_fully_setup();
}

//******************************************************************************
//******************************************************************************
//******************************************************************************

	  }	// namespace AHFinderDirect

```

patch/coords.cc:
```
// coords.cc - coordinate conversions etc
// $Header$

//
// local_coords::fuzzy_EQ_{ang,dang}
// local_coords::modulo_reduce_{ang,dang}
//
// (r,(mu,nu,phi)) <--> (x,y,z)
// ((mu,nu,phi)) --> the 3rd
//
// partial_{x,y,z}_wrt_{mu,nu,phi}
// partial_{mu,nu,phi}_wrt_{x,y,z}
// partial2_{mu,nu,phi}_wrt_{xx,xy,xz,yy,yz,zz}
//
// usual polar spherical (r,theta,phi) <--> (x,y,z)
// ((mu,nu,phi)) <--> usual polar spherical (theta,phi)
// ((mu,nu,phi)) --> direction cosines (xcos,ycos,zcos)
//
// local_coords::mu_nu_phi::name_of_coords_set
//

#include <math.h>
#include <float.h>
#include <assert.h>
#include <limits.h>

#ifdef STANDALONE_TEST
  #include "fake_cctk.h"
#else
  #include "cctk.h"
#endif

#include "config.h"
#include "stdc.h"
#include "../jtutil/util.hh"
using jtutil::error_exit;
using jtutil::arctan_xy;
using jtutil::signum;
using jtutil::pow2;
using jtutil::hypot3;

#include "coords.hh"

// all the code in this file is inside this namespace
namespace AHFinderDirect
	  {

//******************************************************************************
//******************************************************************************
//******************************************************************************

//
// these functions test if two angles are fuzzily equal mod 2*pi radians
// (360 degrees)
//

namespace local_coords
	  {

bool fuzzy_EQ_ang(fp ang1, fp ang2)
{
return jtutil::fuzzy<fp>::is_integer( (ang2 - ang1)/(2.0*PI) );
}

bool fuzzy_EQ_dang(fp dang1, fp dang2)
{
return jtutil::fuzzy<fp>::is_integer( (dang2 - dang1)/360.0 );
}

	  }

//******************************************************************************

//
// modulo-reduce  {ang,dang}  to be (fuzzily) within the range
// [min,max]_{ang,dang}, or error_exit() if no such value exists
//

namespace local_coords
	  {

fp modulo_reduce_ang(fp ang, fp min_ang, fp max_ang)
{
return jtutil::modulo_reduce(ang, 2.0*PI, min_ang, max_ang);
}

fp modulo_reduce_dang(fp dang, fp min_dang, fp max_dang)
{
return jtutil::modulo_reduce(dang, 360.0, min_dang, max_dang);
}

	  }

//******************************************************************************
//******************************************************************************
//******************************************************************************

//
// these functions convert ((mu,nu,phi)) <--> (x,y,z)
//

//**************************************

// valid in +/- z patch
// not valid in xy plane (z == 0, i.e. (cos(mu) == 0 || cos(nu) == 0))
// unless r == 0 (x == y == z == 0)
// c.f. page 9.2 of my mpe notes
namespace local_coords
	  {
void xyz_of_r_mu_nu(fp r, fp mu, fp nu,   fp& x, fp& y, fp& z)
{
const fp sign_y        = signum(sin(mu));
const fp sign_z_via_mu = signum(cos(mu));
assert( jtutil::fuzzy<fp>::NE(cos(mu), 0.0) );
const fp y_over_z = tan(mu);

const fp sign_x        = signum(sin(nu));
const fp sign_z_via_nu = signum(cos(nu));
assert( jtutil::fuzzy<fp>::NE(cos(nu), 0.0) );
const fp x_over_z = tan(nu);

// failure of next assert() ==> inconsistent input (mu,nu)
assert(sign_z_via_mu == sign_z_via_nu);
const fp sign_z = sign_z_via_mu;

const fp temp = 1.0 / sqrt(1.0 + pow2(y_over_z) + pow2(x_over_z));

z = sign_z * r * temp;
x = x_over_z * z;
y = y_over_z * z;

// if (jtutil::fuzzy<fp>::NE(r, 0.0))
//    then {
// 	if (jtutil::fuzzy<fp>::NE(x, 0.0))
//  	   then assert( signum(x) == sign_x );
// 	if (jtutil::fuzzy<fp>::NE(y, 0.0))
// 	   then assert( signum(y) == sign_y );
// 	}
}
	  }

//**************************************

// valid in +/- y patch
// not valid in xz plane (y == 0, i.e. (sin(mu) == 0 || sin(phi) == 0)
// unless r == 0 (x == y == z == 0)
// c.f. page 9.3 of my mpe notes
namespace local_coords
	  {
void xyz_of_r_mu_phi(fp r, fp mu, fp phi,   fp& x, fp& y, fp& z)
{
const fp  mu_bar = 0.5*PI - mu ;
const fp phi_bar = 0.5*PI - phi;

const fp sign_z            = signum(sin(mu_bar));
const fp sign_y_via_mu_bar = signum(cos(mu_bar));
assert( jtutil::fuzzy<fp>::NE(cos(mu_bar), 0.0) );
const fp z_over_y = tan(mu_bar);

const fp sign_x             = signum(sin(phi_bar));
const fp sign_y_via_phi_bar = signum(cos(phi_bar));
assert( jtutil::fuzzy<fp>::NE(cos(phi_bar), 0.0) );
const fp x_over_y = tan(phi_bar);

// failure of next assert() ==> inconsistent input (mu,phi)
assert(sign_y_via_mu_bar == sign_y_via_phi_bar);
const fp sign_y = sign_y_via_mu_bar;

const fp temp = 1.0 / sqrt(1.0 + pow2(z_over_y) + pow2(x_over_y));

y = sign_y * r * temp;
z = z_over_y * y;
x = x_over_y * y;

// if (jtutil::fuzzy<fp>::NE(r, 0.0))
//    then {
// 	if (jtutil::fuzzy<fp>::NE(z, 0.0))
// 	   then assert( signum(z) == sign_z );
// 	if (jtutil::fuzzy<fp>::NE(x, 0.0))
// 	   then assert( signum(x) == sign_x );
// 	}
}
	  }

//**************************************

// valid in +/- x patch
// not valid in yz plane (x == 0, i.e. (sin(nu) == 0 || cos(phi) == 0))
// unless r == 0 (x == y == z == 0)
// c.f. page 9.4 of my mpe notes
namespace local_coords
	  {
void xyz_of_r_nu_phi(fp r, fp nu, fp phi,   fp& x, fp& y, fp& z)
{
const fp nu_bar = 0.5*PI - nu ;

const fp sign_z            = signum(sin(nu_bar));
const fp sign_x_via_nu_bar = signum(cos(nu_bar));
assert( jtutil::fuzzy<fp>::NE(cos(nu_bar), 0.0) );
const fp z_over_x = tan(nu_bar);

const fp sign_y         = signum(sin(phi));
const fp sign_x_via_phi = signum(cos(phi));
assert( jtutil::fuzzy<fp>::NE(cos(phi), 0.0) );
const fp y_over_x = tan(phi   );

// failure of next assert() ==> inconsistent input (nu,phi)
assert(sign_x_via_nu_bar == sign_x_via_phi);
const fp sign_x = sign_x_via_nu_bar;

const fp temp = 1.0 / sqrt(1.0 + pow2(z_over_x) + pow2(y_over_x));

x = sign_x * r * temp;
z = z_over_x * x;
y = y_over_x * x;

// if (jtutil::fuzzy<fp>::NE(r, 0.0))
//    then {
// 	if (jtutil::fuzzy<fp>::NE(z, 0.0))
// 	   then assert( signum(z) == sign_z );
// 	if (jtutil::fuzzy<fp>::NE(y, 0.0))
// 	   then assert( signum(y) == sign_y );
// 	}
}
	  }

//******************************************************************************

//
// these functions give any one of {mu,nu,phi} in terms of the other two
//

//**************************************

// ill-conditioned near z axis
// not valid in xy plane (cos(mu) == 0 || cos(nu) == 0)
namespace local_coords
	  {
fp phi_of_mu_nu(fp mu, fp nu)
{
fp x, y, z;
xyz_of_r_mu_nu(1.0,mu,nu, x,y,z);
return phi_of_xy(x, y);
}
	  }

//**************************************

// ill-conditioned near y axis
// not valid in xz plane (sin(mu) == 0 || sin(phi == 0)
namespace local_coords
	  {
fp nu_of_mu_phi(fp mu, fp phi)
{
fp x, y, z;
xyz_of_r_mu_phi(1.0,mu,phi, x,y,z);
return nu_of_xz(x, z);
}
	  }

//**************************************

// ill-conditioned near x axis
// not valid in yz plane (sin(nu) == 0 || cos(phi) == 0)
namespace local_coords
	  {
fp mu_of_nu_phi(fp nu, fp phi)
{
fp x, y, z;
xyz_of_r_nu_phi(1.0,nu,phi, x,y,z);
return mu_of_yz(y, z);
}
	  }


//******************************************************************************

namespace local_coords
	  {
fp r_of_xyz(fp x, fp y, fp z) { return hypot3(x, y, z); }
fp mu_of_yz(fp y, fp z)  { return arctan_xy(z,y); }
fp nu_of_xz(fp x, fp z)  { return arctan_xy(z,x); }
fp phi_of_xy(fp x, fp y) { return arctan_xy(x,y); }
	  }

//******************************************************************************
//******************************************************************************
//******************************************************************************

//
// these functions compute the partial derivatives
//	partial {x,y,z} / partial {mu,nu,phi}
//
// Bugs: they're slow :(
//

namespace local_coords
	  {
void partial_xyz_wrt_r_mu_nu
	(fp r, fp mu, fp nu,
	 fp& partial_x_wrt_r, fp& partial_x_wrt_mu, fp& partial_x_wrt_nu,
	 fp& partial_y_wrt_r, fp& partial_y_wrt_mu, fp& partial_y_wrt_nu,
	 fp& partial_z_wrt_r, fp& partial_z_wrt_mu, fp& partial_z_wrt_nu)
{
const fp tan_mu = tan(mu);
const fp tan_nu = tan(nu);
const fp tan2_mu = pow2(tan_mu);
const fp tan2_nu = pow2(tan_nu);

fp x, y, z;
xyz_of_r_mu_nu(r,mu,nu, x,y,z);

// Comment this out, accept a nan instead
// assert( jtutil::fuzzy<fp>::NE(r, 0.0) );
const fp rinv = 1.0/r;
partial_x_wrt_r = x*rinv;
partial_y_wrt_r = y*rinv;
partial_z_wrt_r = z*rinv;

const fp t = 1 + tan2_mu + tan2_nu;	// = $r^2/z^2$
const fp partial_t_wrt_mu = 2.0 * tan_mu * (1.0+tan2_mu);
const fp partial_t_wrt_nu = 2.0 * tan_nu * (1.0+tan2_nu);

const fp r2_over_zt2 = (r*r) / (z*t*t);
partial_z_wrt_mu = -0.5 * r2_over_zt2 * partial_t_wrt_mu;
partial_z_wrt_nu = -0.5 * r2_over_zt2 * partial_t_wrt_nu;

partial_x_wrt_mu = tan_nu*partial_z_wrt_mu;
partial_x_wrt_nu = tan_nu*partial_z_wrt_nu + z*(1.0+tan2_nu);
partial_y_wrt_mu = tan_mu*partial_z_wrt_mu + z*(1.0+tan2_mu);
partial_y_wrt_nu = tan_mu*partial_z_wrt_nu;
}
	  }

//**************************************

namespace local_coords
	  {
void partial_xyz_wrt_r_mu_phi
	(fp r, fp mu, fp phi,
	 fp& partial_x_wrt_r, fp& partial_x_wrt_mu, fp& partial_x_wrt_phi,
	 fp& partial_y_wrt_r, fp& partial_y_wrt_mu, fp& partial_y_wrt_phi,
	 fp& partial_z_wrt_r, fp& partial_z_wrt_mu, fp& partial_z_wrt_phi)
{
const fp  mu_bar = 0.5*PI - mu ;
const fp phi_bar = 0.5*PI - phi;

const fp tan_mu_bar  = tan(mu_bar);
const fp tan_phi_bar = tan(phi_bar);
const fp tan2_mu_bar  = pow2(tan_mu_bar);
const fp tan2_phi_bar = pow2(tan_phi_bar);

fp x, y, z;
xyz_of_r_mu_phi(r,mu,phi, x,y,z);

// Comment this out, accept a nan instead
// assert( jtutil::fuzzy<fp>::NE(r, 0.0) );
const fp rinv = 1.0/r;
partial_x_wrt_r = x*rinv;
partial_y_wrt_r = y*rinv;
partial_z_wrt_r = z*rinv;

const fp t = 1 + tan2_mu_bar + tan2_phi_bar;	// = $r^2/y^2$
const fp partial_t_wrt_mu_bar  = 2.0 * tan_mu_bar  * (1.0+tan2_mu_bar );
const fp partial_t_wrt_phi_bar = 2.0 * tan_phi_bar * (1.0+tan2_phi_bar);

const fp r2_over_yt2 = (r*r) / (y*t*t);
partial_y_wrt_mu  = 0.5 * r2_over_yt2 * partial_t_wrt_mu_bar;
partial_y_wrt_phi = 0.5 * r2_over_yt2 * partial_t_wrt_phi_bar;

partial_x_wrt_mu  = tan_phi_bar*partial_y_wrt_mu;
partial_x_wrt_phi = tan_phi_bar*partial_y_wrt_phi - y*(1.0+tan2_phi_bar);
partial_z_wrt_mu  = tan_mu_bar *partial_y_wrt_mu  - y*(1.0+tan2_mu_bar );
partial_z_wrt_phi = tan_mu_bar *partial_y_wrt_phi;
}
	  }

//**************************************

namespace local_coords
	  {
void partial_xyz_wrt_r_nu_phi
	(fp r, fp nu, fp phi,
	 fp& partial_x_wrt_r, fp& partial_x_wrt_nu, fp& partial_x_wrt_phi,
	 fp& partial_y_wrt_r, fp& partial_y_wrt_nu, fp& partial_y_wrt_phi,
	 fp& partial_z_wrt_r, fp& partial_z_wrt_nu, fp& partial_z_wrt_phi)
{
const fp  nu_bar = 0.5*PI - nu ;

const fp tan_nu_bar = tan(nu_bar);
const fp tan_phi    = tan(phi);
const fp tan2_nu_bar = pow2(tan_nu_bar);
const fp tan2_phi    = pow2(tan_phi);

fp x, y, z;
xyz_of_r_nu_phi(r,nu,phi, x,y,z);

// Comment this out, accept a nan instead
// assert( jtutil::fuzzy<fp>::NE(r, 0.0) );
const fp rinv = 1.0/r;
partial_x_wrt_r = x*rinv;
partial_y_wrt_r = y*rinv;
partial_z_wrt_r = z*rinv;

const fp t = 1 + tan2_nu_bar + tan2_phi;	// = $r^2/x^2$
const fp partial_t_wrt_nu_bar  = 2.0 * tan_nu_bar * (1.0+tan2_nu_bar);
const fp partial_t_wrt_phi     = 2.0 * tan_phi    * (1.0+tan2_phi   );

const fp r2_over_xt2 = (r*r) / (x*t*t);
partial_x_wrt_nu  =  0.5 * r2_over_xt2 * partial_t_wrt_nu_bar;
partial_x_wrt_phi = -0.5 * r2_over_xt2 * partial_t_wrt_phi;

partial_y_wrt_nu  = tan_phi   *partial_x_wrt_nu;
partial_y_wrt_phi = tan_phi   *partial_x_wrt_phi + x*(1.0+tan2_phi   );
partial_z_wrt_nu  = tan_nu_bar*partial_x_wrt_nu  - x*(1.0+tan2_nu_bar);
partial_z_wrt_phi = tan_nu_bar*partial_x_wrt_phi;
}
	  }

//******************************************************************************

//
// these functions compute the partial derivatives
//	partial {mu,nu,phi} / partial {x,y,z}
// as computed by the maple file "coord_derivs.{maple,out}" in this directory
//
namespace local_coords
	  {
fp partial_mu_wrt_y(fp y, fp z) { return  z / (y*y + z*z); }
fp partial_mu_wrt_z(fp y, fp z) { return -y / (y*y + z*z); }

fp partial_nu_wrt_x(fp x, fp z) { return  z / (x*x + z*z); }
fp partial_nu_wrt_z(fp x, fp z) { return -x / (x*x + z*z); }

fp partial_phi_wrt_x(fp x, fp y) { return -y / (x*x + y*y); }
fp partial_phi_wrt_y(fp x, fp y) { return  x / (x*x + y*y); }
	  }

//******************************************************************************

//
// these functions compute the 2nd partial derivatives
//	partial {mu,nu,phi} / partial {xx,xy,xz,yy,yz,zz}
// as computed by the maple file "coord_derivs.{maple,out}" in this directory
//
namespace local_coords
	  {
fp partial2_mu_wrt_yy(fp y, fp z) {return   -2.0*y*z  / pow2(y*y + z*z);}
fp partial2_mu_wrt_yz(fp y, fp z) {return (y*y - z*z) / pow2(y*y + z*z);}
fp partial2_mu_wrt_zz(fp y, fp z) {return    2.0*y*z  / pow2(y*y + z*z);}

fp partial2_nu_wrt_xx(fp x, fp z) {return   -2.0*x*z  / pow2(x*x + z*z);}
fp partial2_nu_wrt_xz(fp x, fp z) {return (x*x - z*z) / pow2(x*x + z*z);}
fp partial2_nu_wrt_zz(fp x, fp z) {return    2.0*x*z  / pow2(x*x + z*z);}

fp partial2_phi_wrt_xx(fp x, fp y) {return    2.0*x*y  / pow2(x*x + y*y);}
fp partial2_phi_wrt_xy(fp x, fp y) {return (y*y - x*x) / pow2(x*x + y*y);}
fp partial2_phi_wrt_yy(fp x, fp y) {return   -2.0*x*y  / pow2(x*x + y*y);}
	  }

//******************************************************************************
//******************************************************************************
//******************************************************************************

//
// these functions convert the usual polar spherical (theta,phi) <--> (x,y,z)
//

//**************************************

namespace local_coords
	  {
void xyz_of_r_theta_phi(fp r, fp theta, fp phi,   fp& x, fp& y, fp& z)
{
z = r * cos(theta);
x = r * sin(theta) * cos(phi);
y = r * sin(theta) * sin(phi);
}
	  }

//**************************************

namespace local_coords
	  {
void r_theta_phi_of_xyz(fp x, fp y, fp z,   fp& r, fp& theta, fp& phi)
{
r = r_of_xyz(x, y, z);
theta = theta_of_xyz(x, y, z);
phi = phi_of_xy(x, y);
}
	  }

//**************************************

namespace local_coords
	  {
fp theta_of_xyz(fp x, fp y, fp z)
{
return arctan_xy(z , hypot(x,y));
}
	  }

//******************************************************************************

//
// these functions convert ((mu,nu,phi)) <--> usual polar spherical (theta,phi)
// ... note phi is the same coordinate in both systems
//

namespace local_coords
	  {
void theta_phi_of_mu_nu(fp mu, fp nu,   fp& ps_theta, fp& ps_phi)
{
fp x, y, z;
xyz_of_r_mu_nu(1.0,mu,nu, x,y,z);

ps_theta = theta_of_xyz(x, y, z);
ps_phi = phi_of_xy(x, y);
}
	  }

//**************************************

// Bugs: computes ps_phi via trig, even though it's trivially == phi
namespace local_coords
	  {
void theta_phi_of_mu_phi(fp mu, fp phi,   fp& ps_theta, fp& ps_phi)
{
fp x, y, z;
xyz_of_r_mu_phi(1.0,mu,phi, x,y,z);

ps_theta = theta_of_xyz(x, y, z);
ps_phi = phi_of_xy(x, y);
assert( fuzzy_EQ_ang(ps_phi, phi) );
}
	  }

//**************************************

// Bugs: computes ps_phi via trig, even though it's trivially == phi
namespace local_coords
	  {
void theta_phi_of_nu_phi(fp nu, fp phi,   fp& ps_theta, fp& ps_phi)
{
fp x, y, z;
xyz_of_r_nu_phi(1.0,nu,phi, x,y,z);

ps_theta = theta_of_xyz(x, y, z);
ps_phi = phi_of_xy(x, y);
assert( fuzzy_EQ_ang(ps_phi, phi) );
}
	  }

//******************************************************************************

namespace local_coords
	  {
void mu_nu_of_theta_phi(fp ps_theta, fp ps_phi,   fp& mu, fp& nu)
{
fp x, y, z;
xyz_of_r_theta_phi(1.0,ps_theta,ps_phi, x,y,z);

mu = mu_of_yz(y, z);
nu = nu_of_xz(x, z);
}
	  }

//**************************************

// Bugs: computes phi via trig, even though it's trivially == ps_phi
namespace local_coords
	  {
void mu_phi_of_theta_phi(fp ps_theta, fp ps_phi,   fp& mu, fp& phi)
{
fp x, y, z;
xyz_of_r_theta_phi(1.0,ps_theta,ps_phi, x,y,z);

mu  =  mu_of_yz(y, z);
phi = phi_of_xy(x, y);
assert( fuzzy_EQ_ang(phi, ps_phi) );
}
	  }

//**************************************

// Bugs: computes phi via trig, even though it's trivially == ps_phi
namespace local_coords
	  {
void nu_phi_of_theta_phi(fp ps_theta, fp ps_phi,   fp& nu, fp& phi)
{
fp x, y, z;
xyz_of_r_theta_phi(1.0,ps_theta,ps_phi, x,y,z);

nu  =  nu_of_xz(x, z);
phi = phi_of_xy(x, y);
assert( fuzzy_EQ_ang(phi, ps_phi) );
}
	  }

//******************************************************************************

//
// these functions convert ((mu,nu,phi)) to the direction cosines
// (xcos,ycos,zcos)
//

namespace local_coords
	  {
void xyzcos_of_mu_nu (fp mu, fp nu , fp& xcos, fp& ycos, fp& zcos)
{
xyz_of_r_mu_nu(1.0,mu,nu, xcos,ycos,zcos);
}
	  }

namespace local_coords
	  {
void xyzcos_of_mu_phi(fp mu, fp phi, fp& xcos, fp& ycos, fp& zcos)
{
xyz_of_r_mu_phi(1.0,mu,phi, xcos,ycos,zcos);
}
	  }

namespace local_coords
	  {
void xyzcos_of_nu_phi(fp nu, fp phi, fp& xcos, fp& ycos, fp& zcos)
{
xyz_of_r_nu_phi(1.0,nu,phi, xcos,ycos,zcos);
}
	  }

//******************************************************************************
//******************************************************************************
//******************************************************************************

//
// This function computes a human-readable name from a (mu,nu,phi)
// coordinates set.
//
const char* local_coords::name_of_coords_set(coords_set S)
{
//
// we have to use an if-else chain because the  local_coords::set_*
// constants aren't compile-time constants and hence aren't eligible
// to be switch case labels
//
if	(S == coords_set_empty)
   then return "{}";
else if (S == coords_set_mu)
   then return "mu";
else if (S == coords_set_nu)
   then return "nu";
else if (S == coords_set_phi)
   then return "phi";
else if (S == (coords_set_mu|coords_set_nu))
   then return "{mu,nu}";
else if (S == (coords_set_mu|coords_set_phi))
   then return "{mu,phi}";
else if (S == (coords_set_nu|coords_set_phi))
   then return "{nu,phi}";
else if (S == (coords_set_mu|coords_set_nu|coords_set_phi))
   then return "{mu,nu,phi}";
else	error_exit(PANIC_EXIT,
"***** local_coords::mu_nu_phi::name_of_coords_set:\n"
"        S=0x%x isn't a valid  coords_set  bit vector!\n"
,
		   int(S));					/*NOTREACHED*/
}

//******************************************************************************
//******************************************************************************
//******************************************************************************

	  }	// namespace AHFinderDirect

```

patch/patch_system.cc:
```
// patch_system.cc -- describes the (an) entire system of patches
// $Header$

//
// patch_system::patch_system
// patch_system::~patch_system
// patch_system::create_patches
// patch_system::setup_gridfn_storage
// patch_system::setup_ghost_zones__full_sphere
// patch_system::setup_ghost_zones__plus_z_hemisphere
// patch_system::setup_ghost_zones__plus_xy_quadrant_mirrored
// patch_system::setup_ghost_zones__plus_xy_quadrant_rotating
// patch_system::setup_ghost_zones__plus_xz_quadrant_mirrored
// patch_system::setup_ghost_zones__plus_xz_quadrant_rotating
// patch_system::setup_ghost_zones__plus_xyz_octant_mirrored
// patch_system::setup_ghost_zones__plus_xyz_octant_rotating
// patch_system::create_periodic_symmetry_ghost_zones
// patch_system::create_interpatch_ghost_zones
// patch_system::finish_interpatch_setup
// patch_system::assert_all_ghost_zones_fully_setup
//
// patch_system::N_patches_of_type
// patch_system::name_of_type
// patch_system::type_of_name
// patch_system::plus_or_minus_xyz_patch
// patch_system::patch_number_of_name
//
// patch_system::patch_irho_isigma_of_gpn
// patch_system::ghosted_patch_irho_isigma_of_gpn
//
// patch_system::set_gridfn_to_constant
// patch_system::gridfn_copy
// patch_system::ghosted_gridfn_copy
// patch_system::add_to_gridfn
// patch_system::add_to_ghosted_gridfn
// patch_system::gridfn_norms
// patch_system::ghosted_gridfn_norms
//
// patch_system::circumference
// patch_system::integrate_gridfn
// 
// patch_system::patch_containing_local_xyz
// patch_system::radius_in_local_xyz_direction
// patch_system::radii_in_local_xyz_directions
//
// patch_system::print_unknown_gridfn
// patch_system::read_unknown_gridfn
// patch_system::output_unknown_gridfn
//
// patch_system::synchronize
// patch_system::compute_synchronize_Jacobian
// patch_system::synchronize_Jacobian_global_minmax_ym
// patch_system::synchronize_Jacobian
/// patch_system::fold_Jacobian
/// patch_system::ghost_zone_Jacobian
//

#include <assert.h>
#include <stdio.h>
#include <math.h>
#include <string.h>
#include <assert.h>
#include <limits.h>
#include <sys/stat.h>
#include <sys/types.h>

#include <vector>
#include <sstream>
#include <string>

#include "cctk.h"
#include "CactusBase/IOUtil/src/ioGH.h"

#include "config.h"
#include "stdc.h"
#include "../gr/gfns.hh"
#include "../jtutil/util.hh"
#include "../jtutil/array.hh"
#include "../jtutil/cpm_map.hh"
#include "../jtutil/linear_map.hh"
using jtutil::error_exit;

#ifdef HAVE_CAPABILITY_HDF5
// Some macros to fix compatibility issues as long as 1.8.0 is in beta
// phase
#  define H5_USE_16_API 1
#  include <hdf5.h>
#endif

#include "coords.hh"
#include "grid.hh"
#include "fd_grid.hh"
#include "patch.hh"
#include "patch_edge.hh"
#include "patch_interp.hh"
#include "ghost_zone.hh"
#include "patch_info.hh"
#include "patch_system.hh"
#include "patch_system_info.hh"

// all the code in this file is inside this namespace
namespace AHFinderDirect
	  {

//******************************************************************************
//******************************************************************************
//******************************************************************************

//
// This function constructs a  patch_system  object.
//
// Constructor arguments:
// ghost_zone_width = Width in grid points of all ghost zones.
// patch_overlap_width = Number of grid points that adjacent
//		      nominally-just-touching patches should overlap.
//		      For example, with patch_overlap_width == 3, here
//		      are the grid points of two neighboring patches:
//			x   x   x   x   x   X
//                                      |
//			            O   o   o   o   o   o
//		      Here | marks the "just touching" boundary,
//		      x and o the grid points before this extension,
//		      and X and O the extra grid points added by this
//		      extension.  For this example, the patch_extend_width
//		      parameter used by some other functions would
//		      be 1; in general
//			patch_overlap_width = 2*patch_extend_width + 1
// N_zones_per_right_angle = This sets the grid spacing (same in both
//			     directions) to 90.0 / N_zones_per_right_angle.
//			     It's a fatal error (error_exit()) if this
//			     doesn't evenly divide the grid sizes in both
//			     directions.
// ip_interp_handle = Cactus handle to the interpatch interpolation operator;
//		      this must be a 1-D interpolator
// ip_interp_par_table_handle = Cactus handle to the parameter table for the
//				interpatch interpolation operator
// surface_interp_handle = Cactus handle to the surface interpolation
//			   operator; this is optional, and is only used by
//			     radius_in_{local,global}_xyz_direction()
//			   If this is used, it must be a 2-D interpolator
// surface_interp_par_table_handle = Cactus handle to the parameter table
//				     for the surface interpolation operator;
//				     this is optional, and is only used by
//				       radius_in_local_xyz_direction()
// print_summary_msg_flag = true to print 2 lines of CCTK_VInfo messages
//				 giving the patch system type and origin
//			    false to skip this
// print_detailed_msg_flag = true to print extensive messages tracing the
//				  creation and initialization of various
//				  data structures
//			     false to skip this
//
patch_system::patch_system(fp origin_x_in, fp origin_y_in, fp origin_z_in,
			   enum patch_system_type type_in,
			   int ghost_zone_width, int patch_overlap_width,
			   int N_zones_per_right_angle,
			   int min_gfn_in, int max_gfn_in,
			   int ghosted_min_gfn_in, int ghosted_max_gfn_in,
			   int ip_interp_handle, int ip_interp_par_table_handle,
			   int surface_interp_handle_in,
			   int surface_interp_par_table_handle_in,
			   bool print_summary_msg_flag,
			   bool print_detailed_msg_flag)

	: global_coords_(origin_x_in, origin_y_in, origin_z_in),
	  type_(type_in),
	  N_patches_(N_patches_of_type(type_in)),
	  all_patches_(new patch*[N_patches_]),
	  starting_gpn_(new int[N_patches_+1]),
	  ghosted_starting_gpn_(new int[N_patches_+1]),
	  gridfn_storage_(NULL),		// set in setup_gridfn_storage()
	  ghosted_gridfn_storage_(NULL),	// set in setup_gridfn_storage()
	  global_min_ym_(0), global_max_ym_(0),
					// set in compute_synchronize_Jacobian()
	  surface_interp_handle_          (surface_interp_handle_in),
	  surface_interp_par_table_handle_(surface_interp_par_table_handle_in)
{
if (! jtutil::is_odd(patch_overlap_width))
   then error_exit(ERROR_EXIT,
"***** patch_system::patch_system(): implementation restriction:\n"
"        patch_overlap_width=%d, but we only support odd values!\n"
,
		   patch_overlap_width);			/*NOTREACHED*/
const int patch_extend_width = patch_overlap_width >> 1;

if (ghost_zone_width < fd_grid::molecule_radius())
   then error_exit(ERROR_EXIT,
"***** patch_system::patch_system():\n"
"        must have ghost_zone_width >= fd_grid::molecule_radius()\n"
"        but got ghost_zone_width=%d fd_grid::molecule_radius()=%d!\n"
"        FINITE_DIFF_ORDER=%d (see #define in src/include/config.hh)\n"
,
		   ghost_zone_width, fd_grid::molecule_radius(),
		   FINITE_DIFF_ORDER);				/*NOTREACHED*/

if (print_summary_msg_flag)
   then {
	CCTK_VInfo(CCTK_THORNSTRING,
		   "      constructing %s patch system",
		   name_of_type(type()));
	CCTK_VInfo(CCTK_THORNSTRING,
		   "                   at origin (%g,%g,%g)",
		   double(origin_x()), double(origin_y()), double(origin_z()));
	CCTK_VInfo(CCTK_THORNSTRING,
		   "                   with %d angular zones per right angle",
		   N_zones_per_right_angle);
	}

// construct/interlink the patches and ghost zones
switch	(type_in)
	{
case patch_system__full_sphere:
	create_patches(patch_system_info::full_sphere::patch_info_array,
		       ghost_zone_width, patch_extend_width,
		       N_zones_per_right_angle,
		       print_detailed_msg_flag);
	setup_gridfn_storage(min_gfn_in, max_gfn_in,
			     ghosted_min_gfn_in, ghosted_max_gfn_in,
			     print_detailed_msg_flag);
	setup_ghost_zones__full_sphere(patch_overlap_width,
				       ip_interp_handle,
				       ip_interp_par_table_handle,
				       print_detailed_msg_flag);
	break;

case patch_system__plus_z_hemisphere:
	create_patches(patch_system_info::plus_z_hemisphere::patch_info_array,
		       ghost_zone_width, patch_extend_width,
		       N_zones_per_right_angle,
		       print_detailed_msg_flag);
	setup_gridfn_storage(min_gfn_in, max_gfn_in,
			     ghosted_min_gfn_in, ghosted_max_gfn_in,
			     print_detailed_msg_flag);
	setup_ghost_zones__plus_z_hemisphere(patch_overlap_width,
					     ip_interp_handle,
					     ip_interp_par_table_handle,
					     print_detailed_msg_flag);
	break;

case patch_system__plus_xy_quadrant_mirrored:
	create_patches(patch_system_info::plus_xy_quadrant::patch_info_array,
		       ghost_zone_width, patch_extend_width,
		       N_zones_per_right_angle,
		       print_detailed_msg_flag);
	setup_gridfn_storage(min_gfn_in, max_gfn_in,
			     ghosted_min_gfn_in, ghosted_max_gfn_in,
			     print_detailed_msg_flag);
	setup_ghost_zones__plus_xy_quadrant_mirrored(patch_overlap_width,
						     ip_interp_handle,
						     ip_interp_par_table_handle,
						     print_detailed_msg_flag);
	break;

case patch_system__plus_xy_quadrant_rotating:
	create_patches(patch_system_info::plus_xy_quadrant::patch_info_array,
		       ghost_zone_width, patch_extend_width,
		       N_zones_per_right_angle,
		       print_detailed_msg_flag);
	setup_gridfn_storage(min_gfn_in, max_gfn_in,
			     ghosted_min_gfn_in, ghosted_max_gfn_in,
			     print_detailed_msg_flag);
	setup_ghost_zones__plus_xy_quadrant_rotating(patch_overlap_width,
						     ip_interp_handle,
						     ip_interp_par_table_handle,
						     print_detailed_msg_flag);
	break;

case patch_system__plus_xz_quadrant_mirrored:
	create_patches(patch_system_info::plus_xz_quadrant::patch_info_array,
		       ghost_zone_width, patch_extend_width,
		       N_zones_per_right_angle,
		       print_detailed_msg_flag);
	setup_gridfn_storage(min_gfn_in, max_gfn_in,
			     ghosted_min_gfn_in, ghosted_max_gfn_in,
			     print_detailed_msg_flag);
	setup_ghost_zones__plus_xz_quadrant_mirrored(patch_overlap_width,
						     ip_interp_handle,
						     ip_interp_par_table_handle,
						     print_detailed_msg_flag);
	break;

case patch_system__plus_xz_quadrant_rotating:
	create_patches(patch_system_info::plus_xz_quadrant::patch_info_array,
		       ghost_zone_width, patch_extend_width,
		       N_zones_per_right_angle,
		       print_detailed_msg_flag);
	setup_gridfn_storage(min_gfn_in, max_gfn_in,
			     ghosted_min_gfn_in, ghosted_max_gfn_in,
			     print_detailed_msg_flag);
	setup_ghost_zones__plus_xz_quadrant_rotating(patch_overlap_width,
						     ip_interp_handle,
						     ip_interp_par_table_handle,
						     print_detailed_msg_flag);
	break;

case patch_system__plus_xyz_octant_mirrored:
	create_patches(patch_system_info::plus_xyz_octant::patch_info_array,
		       ghost_zone_width, patch_extend_width,
		       N_zones_per_right_angle,
		       print_detailed_msg_flag);
	setup_gridfn_storage(min_gfn_in, max_gfn_in,
			     ghosted_min_gfn_in, ghosted_max_gfn_in,
			     print_detailed_msg_flag);
	setup_ghost_zones__plus_xyz_octant_mirrored(patch_overlap_width,
						    ip_interp_handle,
						    ip_interp_par_table_handle,
						    print_detailed_msg_flag);
	break;

case patch_system__plus_xyz_octant_rotating:
	create_patches(patch_system_info::plus_xyz_octant::patch_info_array,
		       ghost_zone_width, patch_extend_width,
		       N_zones_per_right_angle,
		       print_detailed_msg_flag);
	setup_gridfn_storage(min_gfn_in, max_gfn_in,
			     ghosted_min_gfn_in, ghosted_max_gfn_in,
			     print_detailed_msg_flag);
	setup_ghost_zones__plus_xyz_octant_rotating(patch_overlap_width,
						    ip_interp_handle,
						    ip_interp_par_table_handle,
						    print_detailed_msg_flag);
	break;

default:
	error_exit(ERROR_EXIT,
"***** patch_system::patch_system(): bad type_in=(int)%d!\n"
,
		   int(type_in));				/*NOTREACHED*/
	}

if (print_summary_msg_flag)
   then CCTK_VInfo(CCTK_THORNSTRING,
		   "      ==> %d nominal, %d ghosted angular grid points",
		   N_grid_points(), ghosted_N_grid_points());
}

//******************************************************************************

//
// This function destroys a  patch_system  object.
//
patch_system::~patch_system()
{
delete[] ghosted_gridfn_storage_;
delete[] gridfn_storage_;
delete[] ghosted_starting_gpn_;
delete[] starting_gpn_;

	for (int pn = N_patches()-1 ; pn >= 0 ; --pn)
	{
	delete &ith_patch(pn);
	}

delete[] all_patches_;
}

//******************************************************************************

//
// This function is called from the patch_system:: constructor to
// construct a set of patches as described by an array of patch_info
// structures and associated arguments, and make these patches members
// of this patch system.  This function also correctly sets
//	N_grid_points_
//	N_ghosted_grid_points_
//	max_N_additional_points_
//	N_additional_points_
//	all_patches_[]
//	starting_gpn_[]
//	ghosted_starting_gpn_[]
// This function does *NOT* create any of the ghost zones, and does
// *NOT* set up any gridfns.
//
void patch_system::create_patches(const struct patch_info patch_info_in[],
				  int ghost_zone_width, int patch_extend_width,
				  int N_zones_per_right_angle,
				  bool print_msg_flag)
{
max_N_additional_points_ = 1;
// N_additional_points_ = 0;
N_additional_points_ = 1;
N_grid_points_ = 0;
ghosted_N_grid_points_ = 0;
	for (int pn = 0 ; pn < N_patches() ; ++pn)
	{
	const struct patch_info& pi = patch_info_in[pn];
	const struct grid::grid_array_pars& grid_array_pars
		= pi.grid_array_pars(ghost_zone_width,
				     patch_extend_width,
				     N_zones_per_right_angle);
	const struct grid::grid_pars& grid_pars
		= pi.grid_pars(patch_extend_width,
			       N_zones_per_right_angle);

	if (print_msg_flag)
	   then CCTK_VInfo(CCTK_THORNSTRING,
			   "         constructing %s patch (%d x %d grid points)",
			   pi.name,
			   jtutil::how_many_in_range(grid_array_pars.min_irho,
						     grid_array_pars.max_irho),
			   jtutil::how_many_in_range(grid_array_pars.min_isigma,
						     grid_array_pars.max_isigma));

	struct patch *p;
	switch	(pi.ctype)
		{
	case 'z':
		p = new z_patch(*this, pn,
				pi.name, pi.is_plus,
				grid_array_pars, grid_pars);
		break;
	case 'x':
		p = new x_patch(*this, pn,
				pi.name, pi.is_plus,
				grid_array_pars, grid_pars);
		break;
	case 'y':
		p = new y_patch(*this, pn,
				pi.name, pi.is_plus,
				grid_array_pars, grid_pars);
		break;
	default:
		error_exit(ERROR_EXIT,
"***** patch_system::create_patches():\n"
"        unknown patch_info_in[pn=%d].ctype=0x%02d='%c'!\n"
,
			   pn, pi.ctype, pi.ctype);		/*NOTREACHED*/
		}

	// these record number of grid points in *previous* patches,
	// i.e. they do *not* include the number of grid points in this patch
	starting_gpn_[pn] = N_grid_points_;
	ghosted_starting_gpn_[pn] = ghosted_N_grid_points_;

	N_grid_points_+= p->N_grid_points();
	ghosted_N_grid_points_ += p->ghosted_N_grid_points();

	all_patches_[pn] = p;
	}

starting_gpn_        [N_patches_] = N_grid_points_;
ghosted_starting_gpn_[N_patches_] = ghosted_N_grid_points_;
}

//******************************************************************************

//
// This function is called from the patch_system:: constructor to set
// up the storage for all gridfns in all patches, giving each gridfn a
// contiguous-across-all-patches storage array.  This function also makes
// a number of self-consistency checks to ensure that the gridfn storage
// subscripting is set up properly.
//
// This function assumes that all the patches have already been constructed
// before it is called.
//
// For example, given the patches {x,y,z}, the ghosted gridfns {H,J},
// and the nominal gridfns {a,b,c}, we might picture the storage like
// this:
//
//	xa xa xa ya ya za za za za
//	xb xb xb yb yb zb zb zb zb
//	xc xc xc yc yc zc zc zc zc
//
//	xH xH xH xH yH yH yH zH zH zH zH zH
//	xJ xJ xJ xJ yJ yJ yJ zJ zJ zJ zJ zJ
//
// Here the upper/lower blocks are for nominal/ghosted  gridfns.
// The storage is taken as being contiguous within each row (in fact
// within each block).  Thus the storage for all the nominal gridfns
// (or all the ghosted gridfns) in a single patch is *non*-contiguous.
//
// The creation of patches is done in several phases: first the patches
// are constructed with no gridfn storage, then we are called to set up
// the gridfn storage (taking into account the sizes of the other patches),
// then finally ghost zones are constructed and interlinked.
//
// FIXME: We should pad the gridfn storage as necessary to avoid cache
//        conflicts, but we don't do this at present.
//
void patch_system::setup_gridfn_storage
	(int min_gfn_in, int max_gfn_in,
	 int ghosted_min_gfn_in, int ghosted_max_gfn_in,
	 bool print_msg_flag)
{
const int N_gridfns_in = jtutil::how_many_in_range(min_gfn_in, max_gfn_in);
const int ghosted_N_gridfns_in
	= jtutil::how_many_in_range(ghosted_min_gfn_in, ghosted_max_gfn_in);

const int gfn_stride = N_grid_points() + max_N_additional_points();
const int ghosted_gfn_stride = ghosted_N_grid_points() + max_N_additional_points();

const int N_storage = gfn_stride * N_gridfns_in;
const int ghosted_N_storage = ghosted_gfn_stride * ghosted_N_gridfns_in;

if (print_msg_flag)
   then {
	CCTK_VInfo(CCTK_THORNSTRING,
		   "      setting up gridfn storage");
	CCTK_VInfo(CCTK_THORNSTRING,
		   "         gfn=[%d,%d] ghosted_gfn=[%d,%d]",
		   min_gfn_in, max_gfn_in,
		   ghosted_min_gfn_in, ghosted_max_gfn_in);
	CCTK_VInfo(CCTK_THORNSTRING,
		   "         N_grid_points()=%d ghosted_N_grid_points()=%d",
		   N_grid_points(), ghosted_N_grid_points());
	}

// storage arrays for all gridfns
gridfn_storage_         = new fp[N_storage];
ghosted_gridfn_storage_ = new fp[ghosted_N_storage];

// divide up the storage array among the patches
// and set up the storage in the individual patches themselves
	  {
	for (int pn = 0 ; pn < N_patches() ; ++pn)
	{
	const int         posn =         starting_gpn_[pn];
	const int ghosted_posn = ghosted_starting_gpn_[pn];
	const struct grid_arrays::gridfn_pars gridfn_pars
		= {
		  min_gfn_in, max_gfn_in,
		  & gridfn_storage_[posn],
		  gfn_stride, 0, 0
		  };
	const struct grid_arrays::gridfn_pars ghosted_gridfn_pars
		= {
		  ghosted_min_gfn_in, ghosted_max_gfn_in,
		  & ghosted_gridfn_storage_[ghosted_posn],
		  ghosted_gfn_stride, 0, 0
		  };

	patch& p = ith_patch(pn);
	p.setup_gridfn_storage(gridfn_pars, ghosted_gridfn_pars);
	}
	  }

if (print_msg_flag)
   then {
	CCTK_VInfo(CCTK_THORNSTRING,
		   "         checking that storage is partitioned properly");
	}

// check to make sure storage for distinct gridfns
// forms a partition of the overall storage array
const patch& pfirst = ith_patch(0);
const patch& plast  = ith_patch(N_patches()-1);
	  {
	for (int gfn = min_gfn() ; gfn+1 < max_gfn() ; ++gfn)
	{
	// range of storage occupied by gridfns:
	//	gfn   --> [gfn_first, gfn_last]
	//	gfn+1 --> [gfn1_first, gfn1_last]
	const fp* const gfn_last_ptr
		= & plast.gridfn(gfn, plast.max_irho(),
				      plast.max_isigma());
	const fp* const gfn1_first_ptr
		= & pfirst.gridfn(gfn+1, pfirst.min_irho(),
					 pfirst.min_isigma());
	if (! (gfn1_first_ptr == gfn_last_ptr+max_N_additional_points()+1))
	   then error_exit(PANIC_EXIT,
"***** patch_system::setup_gridfn_storage():\n"
"        nominal-grid gridfns don't partition overall storage array!"
"        (this should never happen!)\n"
"        gfn=%d last point at gfn_last_ptr=%p\n"
"	 gfn+1=%d first point at gfn1_first_ptr=%p\n"
"        should have gfn1_first_ptr == gfn_last_ptr+1\n"
,
			   gfn, (const void *) gfn_last_ptr,
			   gfn+1, (const void *) gfn1_first_ptr); /*NOTREACHED*/
	}
	  }

	  {
	for (int ghosted_gfn = ghosted_min_gfn() ;
	     ghosted_gfn+1 < ghosted_max_gfn() ;
	     ++ghosted_gfn)
	{
	// range of storage occupied by ghosted gridfns:
	//	ghosted_gfn   --> [gfn_first, gfn_last]
	//	ghosted_gfn+1 --> [gfn1_first, gfn1_last]
	const fp* const ghosted_gfn_last_ptr
		= & plast.ghosted_gridfn(ghosted_gfn,
					 plast.ghosted_max_irho(),
					 plast.ghosted_max_isigma());
	const fp* const ghosted_gfn1_first_ptr
		= & pfirst.ghosted_gridfn(ghosted_gfn+1,
					  pfirst.ghosted_min_irho(),
					  pfirst.ghosted_min_isigma());
	if (! (ghosted_gfn1_first_ptr == ghosted_gfn_last_ptr+max_N_additional_points()+1))
	   then error_exit(PANIC_EXIT,
"***** patch_system::setup_gridfn_storage():\n"
"        ghosted-grid gridfns don't partition overall storage array!"
"        (this should never happen!)\n"
"        ghosted_gfn=%d last point at ghosted_gfn_last_ptr=%p\n"
"	 ghosted_gfn+1=%d first point at ghosted_gfn1_first_ptr=%p\n"
"        should have ghosted_gfn1_first_ptr == ghosted_gfn_last_ptr+1\n"
,
			   ghosted_gfn, (const void *) ghosted_gfn_last_ptr,
			   ghosted_gfn+1,
			      (const void *) ghosted_gfn1_first_ptr);
								/*NOTREACHED*/
	}
	  }

// check to make sure storage for distinct patches
// forms a partition of the storage for each gridfn
	  {
	for (int gfn = min_gfn() ; gfn < max_gfn() ; ++gfn)
	{
		for (int pn = 0 ; pn+1 < N_patches() ; ++pn)
		{
		const patch& p = ith_patch(pn);
		const patch& p1 = ith_patch(pn+1);

		// range of storage occupied by gridfn:
		//	p  --> [p_first, p_last]
		//	p1 --> [p1_first, p1_last]
		const fp* const p_last_ptr
			= & p.gridfn(gfn, p.max_irho(), p.max_isigma());
		const fp* const p1_first_ptr
			= & p1.gridfn(gfn, p1.min_irho(), p1.min_isigma());
		if (! (p1_first_ptr == p_last_ptr+1))
		   then error_exit(PANIC_EXIT,
"***** patch_system::setup_gridfn_storage():\n"
"        nominal-grid patches gridfns don't partition storage for gfn=%d!\n"
"        (this should never happen!)\n"
"        gfn=%d %s patch last point at p_last_ptr=%p\n"
"	 gfn=%d %s patch first point at p1_first_ptr=%p\n"
"        should have p1_first_ptr == p_last_ptr+1\n"
,
			   gfn,
			   gfn, p.name(), (const void *) p_last_ptr,
			   gfn+1, p1.name(), (const void *) p1_first_ptr);
								 /*NOTREACHED*/
		}
	}
	  }

	  {
	for (int ghosted_gfn = ghosted_min_gfn() ;
	     ghosted_gfn < ghosted_max_gfn() ;
	     ++ghosted_gfn)
	{
		for (int pn = 0 ; pn+1 < N_patches() ; ++pn)
		{
		const patch& p = ith_patch(pn);
		const patch& p1 = ith_patch(pn+1);

		// range of storage occupied by ghosted gridfn:
		//	p  --> [p_first, p_last]
		//	p1 --> [p1_first, p1_last]
		const fp* const p_last_ptr
			= & p.ghosted_gridfn(ghosted_gfn,
					     p.ghosted_max_irho(),
					     p.ghosted_max_isigma());
		const fp* const p1_first_ptr
			= & p1.ghosted_gridfn(ghosted_gfn,
					      p1.ghosted_min_irho(),
					      p1.ghosted_min_isigma());
		if (! (p1_first_ptr == p_last_ptr+1))
		   then error_exit(PANIC_EXIT,
"***** patch_system::setup_gridfn_storage():\n"
"        ghosted-grid patches gridfns don't partition storage for gfn=%d!\n"
"        (this should never happen!)\n"
"        %s patch (pn=%d) last point at p_last_ptr=%p\n"
"	 %s patch (pn=%d) first point at p1_first_ptr=%p\n"
"        should have p1_first_ptr == p_last_ptr+1\n"
,
			   ghosted_gfn,
			   p.name(), pn, (const void *) p_last_ptr,
			   p1.name(), pn+1, (const void *) p1_first_ptr);
								 /*NOTREACHED*/
		}
	}
	  }
}

//******************************************************************************

//
// This function sets up (constructs and interlinks) the ghost zones
// for a full-sphere patch system.
//
void patch_system::setup_ghost_zones__full_sphere
	(int patch_overlap_width,
	 int ip_interp_handle, int ip_interp_par_table_handle,
	 bool print_msg_flag)
{
if (print_msg_flag)
   then CCTK_VInfo(CCTK_THORNSTRING,
		   "      seting up full sphere ghost zones");

patch& pz = ith_patch(patch_system_info::full_sphere::patch_number__pz);
patch& px = ith_patch(patch_system_info::full_sphere::patch_number__px);
patch& py = ith_patch(patch_system_info::full_sphere::patch_number__py);
patch& mx = ith_patch(patch_system_info::full_sphere::patch_number__mx);
patch& my = ith_patch(patch_system_info::full_sphere::patch_number__my);
patch& mz = ith_patch(patch_system_info::full_sphere::patch_number__mz);

// create the ghost zones
if (print_msg_flag)
   then CCTK_VInfo(CCTK_THORNSTRING,
		   "         creating ghost zones");
create_interpatch_ghost_zones(pz, px, patch_overlap_width);
create_interpatch_ghost_zones(pz, py, patch_overlap_width);
create_interpatch_ghost_zones(pz, mx, patch_overlap_width);
create_interpatch_ghost_zones(pz, my, patch_overlap_width);
create_interpatch_ghost_zones(px, py, patch_overlap_width);
create_interpatch_ghost_zones(py, mx, patch_overlap_width);
create_interpatch_ghost_zones(mx, my, patch_overlap_width);
create_interpatch_ghost_zones(my, px, patch_overlap_width);
create_interpatch_ghost_zones(mz, px, patch_overlap_width);
create_interpatch_ghost_zones(mz, py, patch_overlap_width);
create_interpatch_ghost_zones(mz, mx, patch_overlap_width);
create_interpatch_ghost_zones(mz, my, patch_overlap_width);

// finish setting up the interpatch ghost zones
if (print_msg_flag)
   then CCTK_VInfo(CCTK_THORNSTRING,
		   "         finishing interpatch setup");
finish_interpatch_setup(pz, px,
			patch_overlap_width,
			ip_interp_handle, ip_interp_par_table_handle);
finish_interpatch_setup(pz, py,
			patch_overlap_width,
			ip_interp_handle, ip_interp_par_table_handle);
finish_interpatch_setup(pz, mx,
			patch_overlap_width,
			ip_interp_handle, ip_interp_par_table_handle);
finish_interpatch_setup(pz, my,
			patch_overlap_width,
			ip_interp_handle, ip_interp_par_table_handle);
finish_interpatch_setup(px, py,
			patch_overlap_width,
			ip_interp_handle, ip_interp_par_table_handle);
finish_interpatch_setup(py, mx,
			patch_overlap_width,
			ip_interp_handle, ip_interp_par_table_handle);
finish_interpatch_setup(mx, my,
			patch_overlap_width,
			ip_interp_handle, ip_interp_par_table_handle);
finish_interpatch_setup(my, px,
			patch_overlap_width,
			ip_interp_handle, ip_interp_par_table_handle);
finish_interpatch_setup(mz, px,
			patch_overlap_width,
			ip_interp_handle, ip_interp_par_table_handle);
finish_interpatch_setup(mz, py,
			patch_overlap_width,
			ip_interp_handle, ip_interp_par_table_handle);
finish_interpatch_setup(mz, mx,
			patch_overlap_width,
			ip_interp_handle, ip_interp_par_table_handle);
finish_interpatch_setup(mz, my,
			patch_overlap_width,
			ip_interp_handle, ip_interp_par_table_handle);

assert_all_ghost_zones_fully_setup();
}

//******************************************************************************

//
// This function sets up (constructs and interlinks) the ghost zones
// for a +z hemisphere patch system.
//
void patch_system::setup_ghost_zones__plus_z_hemisphere
	(int patch_overlap_width,
	 int ip_interp_handle, int ip_interp_par_table_handle,
	 bool print_msg_flag)
{
if (print_msg_flag)
   then CCTK_VInfo(CCTK_THORNSTRING,
		   "      setting up +z hemisphere ghost zones");

patch& pz = ith_patch(patch_system_info::plus_z_hemisphere::patch_number__pz);
patch& px = ith_patch(patch_system_info::plus_z_hemisphere::patch_number__px);
patch& py = ith_patch(patch_system_info::plus_z_hemisphere::patch_number__py);
patch& mx = ith_patch(patch_system_info::plus_z_hemisphere::patch_number__mx);
patch& my = ith_patch(patch_system_info::plus_z_hemisphere::patch_number__my);

// create the ghost zones
if (print_msg_flag)
   then CCTK_VInfo(CCTK_THORNSTRING,
		   "         creating ghost zones");
create_interpatch_ghost_zones(pz, px, patch_overlap_width);
create_interpatch_ghost_zones(pz, py, patch_overlap_width);
create_interpatch_ghost_zones(pz, mx, patch_overlap_width);
create_interpatch_ghost_zones(pz, my, patch_overlap_width);
create_interpatch_ghost_zones(px, py, patch_overlap_width);
create_interpatch_ghost_zones(py, mx, patch_overlap_width);
create_interpatch_ghost_zones(mx, my, patch_overlap_width);
create_interpatch_ghost_zones(my, px, patch_overlap_width);
px.create_mirror_symmetry_ghost_zone(px.max_rho_patch_edge());
py.create_mirror_symmetry_ghost_zone(py.max_rho_patch_edge());
mx.create_mirror_symmetry_ghost_zone(mx.min_rho_patch_edge());
my.create_mirror_symmetry_ghost_zone(my.min_rho_patch_edge());

// finish setting up the interpatch ghost zones
if (print_msg_flag)
   then CCTK_VInfo(CCTK_THORNSTRING,
		   "         finishing interpatch setup");
finish_interpatch_setup(pz, px,
			patch_overlap_width,
			ip_interp_handle, ip_interp_par_table_handle);
finish_interpatch_setup(pz, py,
			patch_overlap_width,
			ip_interp_handle, ip_interp_par_table_handle);
finish_interpatch_setup(pz, mx,
			patch_overlap_width,
			ip_interp_handle, ip_interp_par_table_handle);
finish_interpatch_setup(pz, my,
			patch_overlap_width,
			ip_interp_handle, ip_interp_par_table_handle);
finish_interpatch_setup(px, py,
			patch_overlap_width,
			ip_interp_handle, ip_interp_par_table_handle);
finish_interpatch_setup(py, mx,
			patch_overlap_width,
			ip_interp_handle, ip_interp_par_table_handle);
finish_interpatch_setup(mx, my,
			patch_overlap_width,
			ip_interp_handle, ip_interp_par_table_handle);
finish_interpatch_setup(my, px,
			patch_overlap_width,
			ip_interp_handle, ip_interp_par_table_handle);

assert_all_ghost_zones_fully_setup();
}

//******************************************************************************

//
// This function sets up (constructs and interlinks) the ghost zones
// for a +xy quadrant (mirrored) patch system.
//
void patch_system::setup_ghost_zones__plus_xy_quadrant_mirrored
	(int patch_overlap_width,
	 int ip_interp_handle, int ip_interp_par_table_handle,
	 bool print_msg_flag)
{
if (print_msg_flag)
   then CCTK_VInfo(CCTK_THORNSTRING,
		   "      setting up +xy quadrant (mirrored) ghost zones");

patch& pz = ith_patch(patch_system_info::plus_xy_quadrant::patch_number__pz);
patch& px = ith_patch(patch_system_info::plus_xy_quadrant::patch_number__px);
patch& py = ith_patch(patch_system_info::plus_xy_quadrant::patch_number__py);
patch& mz = ith_patch(patch_system_info::plus_xy_quadrant::patch_number__mz);

// create the ghost zones
if (print_msg_flag)
   then CCTK_VInfo(CCTK_THORNSTRING,
		   "         creating ghost zones");
create_interpatch_ghost_zones(pz, px, patch_overlap_width);
create_interpatch_ghost_zones(pz, py, patch_overlap_width);
create_interpatch_ghost_zones(px, py, patch_overlap_width);
create_interpatch_ghost_zones(mz, px, patch_overlap_width);
create_interpatch_ghost_zones(mz, py, patch_overlap_width);
pz.create_mirror_symmetry_ghost_zone(pz.min_rho_patch_edge());
pz.create_mirror_symmetry_ghost_zone(pz.min_sigma_patch_edge());
px.create_mirror_symmetry_ghost_zone(px.min_sigma_patch_edge());
py.create_mirror_symmetry_ghost_zone(py.max_sigma_patch_edge());
mz.create_mirror_symmetry_ghost_zone(mz.max_rho_patch_edge());
mz.create_mirror_symmetry_ghost_zone(mz.max_sigma_patch_edge());

// finish setting up the interpatch ghost zones
if (print_msg_flag)
   then CCTK_VInfo(CCTK_THORNSTRING,
		   "         finishing interpatch setup");
finish_interpatch_setup(pz, px,
			patch_overlap_width,
			ip_interp_handle, ip_interp_par_table_handle);
finish_interpatch_setup(pz, py,
			patch_overlap_width,
			ip_interp_handle, ip_interp_par_table_handle);
finish_interpatch_setup(px, py,
			patch_overlap_width,
			ip_interp_handle, ip_interp_par_table_handle);
finish_interpatch_setup(mz, px,
			patch_overlap_width,
			ip_interp_handle, ip_interp_par_table_handle);
finish_interpatch_setup(mz, py,
			patch_overlap_width,
			ip_interp_handle, ip_interp_par_table_handle);

assert_all_ghost_zones_fully_setup();
}

//******************************************************************************

//
// This function sets up (constructs and interlinks) the ghost zones
// for a +xy quadrant (rotating) patch system.
//
void patch_system::setup_ghost_zones__plus_xy_quadrant_rotating
	(int patch_overlap_width,
	 int ip_interp_handle, int ip_interp_par_table_handle,
	 bool print_msg_flag)
{
if (print_msg_flag)
   then CCTK_VInfo(CCTK_THORNSTRING,
		   "      setting up +xy quadrant (rotating) ghost zones");

patch& pz = ith_patch(patch_system_info::plus_xy_quadrant::patch_number__pz);
patch& px = ith_patch(patch_system_info::plus_xy_quadrant::patch_number__px);
patch& py = ith_patch(patch_system_info::plus_xy_quadrant::patch_number__py);
patch& mz = ith_patch(patch_system_info::plus_xy_quadrant::patch_number__mz);

// create the ghost zones
if (print_msg_flag)
   then CCTK_VInfo(CCTK_THORNSTRING,
		   "         creating ghost zones");
create_interpatch_ghost_zones(pz, px, patch_overlap_width);
create_interpatch_ghost_zones(pz, py, patch_overlap_width);
create_interpatch_ghost_zones(px, py, patch_overlap_width);
create_interpatch_ghost_zones(mz, px, patch_overlap_width);
create_interpatch_ghost_zones(mz, py, patch_overlap_width);
create_periodic_symmetry_ghost_zones(pz.min_rho_patch_edge(),
				     pz.min_sigma_patch_edge(),
				     true);
create_periodic_symmetry_ghost_zones(px.min_sigma_patch_edge(),
				     py.max_sigma_patch_edge(),
				     true);
create_periodic_symmetry_ghost_zones(mz.max_rho_patch_edge(),
				     mz.max_sigma_patch_edge(),
				     true);

// finish setting up the interpatch ghost zones
if (print_msg_flag)
   then CCTK_VInfo(CCTK_THORNSTRING,
		   "         finishing interpatch setup");
finish_interpatch_setup(pz, px,
			patch_overlap_width,
			ip_interp_handle, ip_interp_par_table_handle);
finish_interpatch_setup(pz, py,
			patch_overlap_width,
			ip_interp_handle, ip_interp_par_table_handle);
finish_interpatch_setup(px, py,
			patch_overlap_width,
			ip_interp_handle, ip_interp_par_table_handle);
finish_interpatch_setup(mz, px,
			patch_overlap_width,
			ip_interp_handle, ip_interp_par_table_handle);
finish_interpatch_setup(mz, py,
			patch_overlap_width,
			ip_interp_handle, ip_interp_par_table_handle);

assert_all_ghost_zones_fully_setup();
}

//******************************************************************************

//
// This function sets up (constructs and interlinks) the ghost zones
// for a +xz quadrant (mirrored) patch system.
//
void patch_system::setup_ghost_zones__plus_xz_quadrant_mirrored
	(int patch_overlap_width,
	 int ip_interp_handle, int ip_interp_par_table_handle,
	 bool print_msg_flag)
{
if (print_msg_flag)
   then CCTK_VInfo(CCTK_THORNSTRING,
		   "      setting up +xz quadrant (mirrored) ghost zones");

patch& pz = ith_patch(patch_system_info::plus_xz_quadrant::patch_number__pz);
patch& px = ith_patch(patch_system_info::plus_xz_quadrant::patch_number__px);
patch& py = ith_patch(patch_system_info::plus_xz_quadrant::patch_number__py);
patch& my = ith_patch(patch_system_info::plus_xz_quadrant::patch_number__my);

// create the ghost zones
if (print_msg_flag)
   then CCTK_VInfo(CCTK_THORNSTRING,
		   "         creating ghost zones");
create_interpatch_ghost_zones(pz, px, patch_overlap_width);
create_interpatch_ghost_zones(pz, py, patch_overlap_width);
create_interpatch_ghost_zones(pz, my, patch_overlap_width);
create_interpatch_ghost_zones(px, py, patch_overlap_width);
create_interpatch_ghost_zones(px, my, patch_overlap_width);
pz.create_mirror_symmetry_ghost_zone(pz.min_sigma_patch_edge());
px.create_mirror_symmetry_ghost_zone(px.max_rho_patch_edge());
py.create_mirror_symmetry_ghost_zone(py.max_rho_patch_edge());
py.create_mirror_symmetry_ghost_zone(py.max_sigma_patch_edge());
my.create_mirror_symmetry_ghost_zone(my.min_rho_patch_edge());
my.create_mirror_symmetry_ghost_zone(my.min_sigma_patch_edge());

// finish setting up the interpatch ghost zones
if (print_msg_flag)
   then CCTK_VInfo(CCTK_THORNSTRING,
		   "         finishing interpatch setup");
finish_interpatch_setup(pz, px,
			patch_overlap_width,
			ip_interp_handle, ip_interp_par_table_handle);
finish_interpatch_setup(pz, py,
			patch_overlap_width,
			ip_interp_handle, ip_interp_par_table_handle);
finish_interpatch_setup(pz, my,
			patch_overlap_width,
			ip_interp_handle, ip_interp_par_table_handle);
finish_interpatch_setup(px, py,
			patch_overlap_width,
			ip_interp_handle, ip_interp_par_table_handle);
finish_interpatch_setup(px, my,
			patch_overlap_width,
			ip_interp_handle, ip_interp_par_table_handle);

assert_all_ghost_zones_fully_setup();
}

//******************************************************************************

//
// This function sets up (constructs and interlinks) the ghost zones
// for a +xz quadrant (rotating) patch system.
//
void patch_system::setup_ghost_zones__plus_xz_quadrant_rotating
	(int patch_overlap_width,
	 int ip_interp_handle, int ip_interp_par_table_handle,
	 bool print_msg_flag)
{
if (print_msg_flag)
   then CCTK_VInfo(CCTK_THORNSTRING,
		   "      setting up +xz quadrant (rotating) ghost zones");

patch& pz = ith_patch(patch_system_info::plus_xz_quadrant::patch_number__pz);
patch& px = ith_patch(patch_system_info::plus_xz_quadrant::patch_number__px);
patch& py = ith_patch(patch_system_info::plus_xz_quadrant::patch_number__py);
patch& my = ith_patch(patch_system_info::plus_xz_quadrant::patch_number__my);

// create the ghost zones
if (print_msg_flag)
   then CCTK_VInfo(CCTK_THORNSTRING,
		   "         creating ghost zones");
create_interpatch_ghost_zones(pz, px, patch_overlap_width);
create_interpatch_ghost_zones(pz, py, patch_overlap_width);
create_interpatch_ghost_zones(pz, my, patch_overlap_width);
create_interpatch_ghost_zones(px, py, patch_overlap_width);
create_interpatch_ghost_zones(px, my, patch_overlap_width);
px.create_mirror_symmetry_ghost_zone(px.max_rho_patch_edge());
py.create_mirror_symmetry_ghost_zone(py.max_rho_patch_edge());
my.create_mirror_symmetry_ghost_zone(my.min_rho_patch_edge());
create_periodic_symmetry_ghost_zones(pz.min_sigma_patch_edge(),
				     pz.min_sigma_patch_edge(),
				     false);
create_periodic_symmetry_ghost_zones(py.max_sigma_patch_edge(),
				     my.min_sigma_patch_edge(),
				     false);

// finish setting up the interpatch ghost zones
if (print_msg_flag)
   then CCTK_VInfo(CCTK_THORNSTRING,
		   "         finishing interpatch setup");
finish_interpatch_setup(pz, px,
			patch_overlap_width,
			ip_interp_handle, ip_interp_par_table_handle);
finish_interpatch_setup(pz, py,
			patch_overlap_width,
			ip_interp_handle, ip_interp_par_table_handle);
finish_interpatch_setup(pz, my,
			patch_overlap_width,
			ip_interp_handle, ip_interp_par_table_handle);
finish_interpatch_setup(px, py,
			patch_overlap_width,
			ip_interp_handle, ip_interp_par_table_handle);
finish_interpatch_setup(px, my,
			patch_overlap_width,
			ip_interp_handle, ip_interp_par_table_handle);

assert_all_ghost_zones_fully_setup();
}

//******************************************************************************

//
// This function sets up (constructs and interlinks) the ghost zones
// for a +xyz octant (mirrored) patch system.
//
void patch_system::setup_ghost_zones__plus_xyz_octant_mirrored
	(int patch_overlap_width,
	 int ip_interp_handle, int ip_interp_par_table_handle,
	 bool print_msg_flag)
{
if (print_msg_flag)
   then CCTK_VInfo(CCTK_THORNSTRING,
		   "      setting up +xyz octant (mirrored) ghost zones");

patch& pz = ith_patch(patch_system_info::plus_xyz_octant::patch_number__pz);
patch& px = ith_patch(patch_system_info::plus_xyz_octant::patch_number__px);
patch& py = ith_patch(patch_system_info::plus_xyz_octant::patch_number__py);

// create the ghost zones
if (print_msg_flag)
   then CCTK_VInfo(CCTK_THORNSTRING,
		   "         creating ghost zones");
create_interpatch_ghost_zones(pz, px, patch_overlap_width);
create_interpatch_ghost_zones(pz, py, patch_overlap_width);
create_interpatch_ghost_zones(px, py, patch_overlap_width);
pz.create_mirror_symmetry_ghost_zone(pz.min_rho_patch_edge());
pz.create_mirror_symmetry_ghost_zone(pz.min_sigma_patch_edge());
px.create_mirror_symmetry_ghost_zone(px.max_rho_patch_edge());
px.create_mirror_symmetry_ghost_zone(px.min_sigma_patch_edge());
py.create_mirror_symmetry_ghost_zone(py.max_rho_patch_edge());
py.create_mirror_symmetry_ghost_zone(py.max_sigma_patch_edge());

// finish setting up the interpatch ghost zones
if (print_msg_flag)
   then CCTK_VInfo(CCTK_THORNSTRING,
		   "         finishing interpatch setup");
finish_interpatch_setup(pz, px,
			patch_overlap_width,
			ip_interp_handle, ip_interp_par_table_handle);
finish_interpatch_setup(pz, py,
			patch_overlap_width,
			ip_interp_handle, ip_interp_par_table_handle);
finish_interpatch_setup(px, py,
			patch_overlap_width,
			ip_interp_handle, ip_interp_par_table_handle);

assert_all_ghost_zones_fully_setup();
}

//******************************************************************************

//
// This function sets up (constructs and interlinks) the ghost zones
// for a +xyz octant (rotating) patch system.
//
void patch_system::setup_ghost_zones__plus_xyz_octant_rotating
	(int patch_overlap_width,
	 int ip_interp_handle, int ip_interp_par_table_handle,
	 bool print_msg_flag)
{
if (print_msg_flag)
   then CCTK_VInfo(CCTK_THORNSTRING,
		   "      setting up +xyz octant (rotating) ghost zones");

patch& pz = ith_patch(patch_system_info::plus_xyz_octant::patch_number__pz);
patch& px = ith_patch(patch_system_info::plus_xyz_octant::patch_number__px);
patch& py = ith_patch(patch_system_info::plus_xyz_octant::patch_number__py);

// create the ghost zones
if (print_msg_flag)
   then CCTK_VInfo(CCTK_THORNSTRING,
		   "         creating ghost zones");
create_interpatch_ghost_zones(pz, px, patch_overlap_width);
create_interpatch_ghost_zones(pz, py, patch_overlap_width);
create_interpatch_ghost_zones(px, py, patch_overlap_width);
px.create_mirror_symmetry_ghost_zone(px.max_rho_patch_edge());
py.create_mirror_symmetry_ghost_zone(py.max_rho_patch_edge());
create_periodic_symmetry_ghost_zones(pz.min_rho_patch_edge(),
				     pz.min_sigma_patch_edge(),
				     true);
create_periodic_symmetry_ghost_zones(px.min_sigma_patch_edge(),
				     py.max_sigma_patch_edge(),
				     true);

// finish setting up the interpatch ghost zones
if (print_msg_flag)
   then CCTK_VInfo(CCTK_THORNSTRING,
		   "         finishing interpatch setup");
finish_interpatch_setup(pz, px,
			patch_overlap_width,
			ip_interp_handle, ip_interp_par_table_handle);
finish_interpatch_setup(pz, py,
			patch_overlap_width,
			ip_interp_handle, ip_interp_par_table_handle);
finish_interpatch_setup(px, py,
			patch_overlap_width,
			ip_interp_handle, ip_interp_par_table_handle);

assert_all_ghost_zones_fully_setup();
}

//******************************************************************************
//******************************************************************************
//******************************************************************************

//
// This function creates a pair of periodic-symmetry ghost zones.
//
//static
  void patch_system::create_periodic_symmetry_ghost_zones
	(const patch_edge& ex, const patch_edge& ey,
	 bool ipar_map_is_plus)
{
ex.my_patch()
  .create_periodic_symmetry_ghost_zone(ex, ey, ipar_map_is_plus);

if (ex == ey)
   then {
	// ex and ey are the same edge (i.e. the symmetry maps the edge
	// back to itself), so we only want to set up the edge once
	// ==> no-op here
	}
   else ey.my_patch()
	  .create_periodic_symmetry_ghost_zone(ey, ex, ipar_map_is_plus);
}

//******************************************************************************

//
// This function automagically figures out which edges of two adjacent
// patches are adjacent, then creates both patches' ghost zones on those
// edges and interlinks them with their respective patches.
//
//static
  void patch_system::create_interpatch_ghost_zones(patch &px, patch &py,
						   int patch_overlap_width)
{
const patch_edge& ex = px.edge_adjacent_to_patch(py, patch_overlap_width);
const patch_edge& ey = py.edge_adjacent_to_patch(px, patch_overlap_width);

px.create_interpatch_ghost_zone(ex, ey, patch_overlap_width);
py.create_interpatch_ghost_zone(ey, ex, patch_overlap_width);
}

//******************************************************************************

//
// This function automagically figures out which edges of two adjacent
// patches are adjacent, then finishes setting up both ghost zones.
//
//static
  void patch_system::finish_interpatch_setup
	(patch &px, patch &py,
	 int patch_overlap_width,
	 int ip_interp_handle, int ip_interp_par_table_handle)
{
const patch_edge& ex = px.edge_adjacent_to_patch(py, patch_overlap_width);
const patch_edge& ey = py.edge_adjacent_to_patch(px, patch_overlap_width);
px.ghost_zone_on_edge(ex)
  .cast_to_interpatch_ghost_zone()
  .finish_setup(ip_interp_handle, ip_interp_par_table_handle);
py.ghost_zone_on_edge(ey)
  .cast_to_interpatch_ghost_zone()
  .finish_setup(ip_interp_handle, ip_interp_par_table_handle);
}

//******************************************************************************

//
// This function assert()s that all ghost zones of all patches have
// been fully set up.
//
void patch_system::assert_all_ghost_zones_fully_setup() const
{
	for (int pn = 0 ; pn < N_patches() ; ++pn)
	{
	ith_patch(pn).assert_all_ghost_zones_fully_setup();
	}
}

//******************************************************************************
//******************************************************************************
//******************************************************************************

//
// This function decodes a patch system's type into N_patches.
//
//static
  int patch_system::N_patches_of_type(enum patch_system_type type_in)
{
switch	(type_in)
	{
case patch_system__full_sphere:
	return patch_system_info::full_sphere::N_patches;
case patch_system__plus_z_hemisphere:
	return patch_system_info::plus_z_hemisphere::N_patches;
case patch_system__plus_xy_quadrant_mirrored:
case patch_system__plus_xy_quadrant_rotating:
	return patch_system_info::plus_xy_quadrant::N_patches;
case patch_system__plus_xz_quadrant_mirrored:
case patch_system__plus_xz_quadrant_rotating:
	return patch_system_info::plus_xz_quadrant::N_patches;
case patch_system__plus_xyz_octant_mirrored:
case patch_system__plus_xyz_octant_rotating:
	return patch_system_info::plus_xyz_octant::N_patches;
default:
	error_exit(PANIC_EXIT,
"***** patch_system::N_patches_of_type(): bad type=(int)%d!\n"
"                                         (this should never happen!)\n",
		   int(type_in));				/*NOTREACHED*/
	}
}

//******************************************************************************

//
// This function decodes a patch system's type into a human-readable
// type name (a C string).
//
//static
  const char* patch_system::name_of_type(enum patch_system_type type_in)
{
switch	(type_in)
	{
case patch_system__full_sphere:               return "full sphere";
case patch_system__plus_z_hemisphere:         return "+z hemisphere";
case patch_system__plus_xy_quadrant_mirrored: return "+xy quadrant (mirrored)";
case patch_system__plus_xy_quadrant_rotating: return "+xy quadrant (rotating)";
case patch_system__plus_xz_quadrant_mirrored: return "+xz quadrant (mirrored)";
case patch_system__plus_xz_quadrant_rotating: return "+xz quadrant (rotating)";
case patch_system__plus_xyz_octant_mirrored:  return "+xyz octant (mirrored)";
case patch_system__plus_xyz_octant_rotating:  return "+xyz octant (rotating)";
default:
	error_exit(PANIC_EXIT,
"***** patch_system::name_of_type(): bad type=(int)%d!\n"
"                                    (this should never happen!)\n",
		   int(type_in));				/*NOTREACHED*/
	}
}

//******************************************************************************

//
// This function encodes a human-readable type name (a C string) into
// a patch system's type into.
//
//static
  enum patch_system::patch_system_type
    patch_system::type_of_name(const char* name_in)
{
if	(STRING_EQUAL(name_in, "full sphere"))
   then return patch_system__full_sphere;
else if (STRING_EQUAL(name_in, "+z hemisphere"))
   then return patch_system__plus_z_hemisphere;
else if (STRING_EQUAL(name_in, "+xy quadrant (mirrored)"))
   then return patch_system__plus_xy_quadrant_mirrored;
else if (STRING_EQUAL(name_in, "+xy quadrant (rotating)"))
   then return patch_system__plus_xy_quadrant_rotating;
else if (STRING_EQUAL(name_in, "+xz quadrant (mirrored)"))
   then return patch_system__plus_xz_quadrant_mirrored;
else if (STRING_EQUAL(name_in, "+xz quadrant (rotating)"))
   then return patch_system__plus_xz_quadrant_rotating;
else if (STRING_EQUAL(name_in, "+xyz octant (mirrored)"))
   then return patch_system__plus_xyz_octant_mirrored;
else if (STRING_EQUAL(name_in, "+xyz octant (rotating)"))
   then return patch_system__plus_xyz_octant_rotating;
else	error_exit(PANIC_EXIT,
"***** patch_system::type_of_name(): unknown name=\"%s\"!",
		   name_in);					/*NOTREACHED*/
}

//******************************************************************************

//
// This function finds a (the) patch with a specified sign and xyz ctype.
// If no such patch exists, it does an  error_exit()  (and doesn't return
// to the caller).
//
// FIXME:
// - This function could be implemented to be very fast (using the
//   patch numbers in patch_system_info::), but right now it just does
//   a sequential search through all the patches, so it's pretty slow :(
//
const patch& patch_system::plus_or_minus_xyz_patch(bool is_plus, char ctype)
	const
{
	for (int pn = 0 ; pn < N_patches() ; ++pn)
	{
	const patch& p = ith_patch(pn);
	if ((p.is_plus() == is_plus) && (p.ctype() == ctype))
	   then return p;
	}

error_exit(ERROR_EXIT,
"***** patch_system::plus_or_minus_xyz_patch():\n"
"        can't find any %c%c patch!"
,
	   (is_plus ? '+' : '-'), ctype);			/*NOTREACHED*/
}

//******************************************************************************

//
// This function finds a patch from its human-readable name, and returns
// the patch number, or does an error_exit() if no patch is found with
// the specified name.
//
int patch_system::patch_number_of_name(const char* name) const
{
	for (int pn = 0 ; pn < N_patches() ; ++pn)
	{
	if (STRING_EQUAL(ith_patch(pn).name(), name))
	   then return pn;
	}

error_exit(ERROR_EXIT,
"***** patch_system::patch_number_of_name():\n"
"        no patch with name \"%s\"!\n",
	   name);						/*NOTREACHED*/
}

//******************************************************************************
//******************************************************************************
//******************************************************************************

//
// This function decodes a 0-origin grid point number into a
// (patch,irho,isigma) triple.
//
// Arguments:
// gpn = The grid point number to decode.
// (irho,isigma) = (out) The decoded patch coordinates.
//
// Results:
// This function returns a reference to the decoded patch.  (An alternative
// design would be to return this via a  patch*&  argument, but the design
// used here seems slightly cleaner to use in practice.)
//
const patch&
  patch_system::patch_irho_isigma_of_gpn(int gpn, int& irho, int& isigma)
	const
{
assert( gpn >= 0 );
assert( gpn < N_grid_points() );

	for (int pn = 0 ; pn < N_patches() ; ++pn)
	{
	// n.b. [pn+1] is ok since starting_gpn_[] has size N_patches()+1
	if ((gpn >= starting_gpn_[pn])  && (gpn < starting_gpn_[pn+1]))
	   then {
		const patch& p = ith_patch(pn);
		const int gpn_in_patch = gpn - starting_gpn_[pn];
		assert( gpn_in_patch >= 0 );
		assert( gpn_in_patch < p.N_grid_points() );
		p.irho_isigma_of_gpn(gpn_in_patch, irho,isigma);
		return p;
		}
	}

error_exit(PANIC_EXIT,
"***** patch_system::patch_irho_isigma_of_gpn(gpn=%d):\n"
"        couldn't find any patch! (this should never happen!)\n"
"        N_grid_points()=%d\n"
,
	   gpn,
	   N_grid_points());					/*NOTREACHED*/
}

//******************************************************************************

//
// This function decodes a 0-origin grid point number into a *ghosted*
// (patch,irho,isigma) triple.
//
// Arguments:
// gpn = The grid point number to decode.
// (irho,isigma) = (out) The decoded patch coordinates.
//
// Results:
// This function returns a reference to the decoded patch.  (An alternative
// design would be to return this via a  patch*&  argument, but the design
// used here seems slightly cleaner to use in practice.)
//
const patch&
  patch_system::ghosted_patch_irho_isigma_of_gpn
	(int gpn, int& irho, int& isigma)
	const
{
assert( gpn >= 0 );
assert( gpn < ghosted_N_grid_points() );

	for (int pn = 0 ; pn < N_patches() ; ++pn)
	{
	// n.b. [pn+1] is ok since ghosted_starting_gpn_[]
	//      has size N_patches()+1
	if (    (gpn >= ghosted_starting_gpn_[pn])
	     && (gpn <  ghosted_starting_gpn_[pn+1]))
	   then {
		const patch& p = ith_patch(pn);
		const int gpn_in_patch = gpn - ghosted_starting_gpn_[pn];
		assert( gpn_in_patch >= 0 );
		assert( gpn_in_patch < p.ghosted_N_grid_points() );
		p.ghosted_irho_isigma_of_gpn(gpn_in_patch, irho,isigma);
		return p;
		}
	}

error_exit(PANIC_EXIT,
"***** patch_system::ghosted_patch_irho_isigma_of_gpn(gpn=%d):\n"
"        couldn't find any patch! (this should never happen!)\n"
"        ghosted_N_grid_points()=%d\n"
,
	   gpn,
	   ghosted_N_grid_points());				/*NOTREACHED*/
}

//******************************************************************************
//******************************************************************************
//******************************************************************************

//
// This function sets a (nominal-grid) gridfn to a constant value.
//
void patch_system::set_gridfn_to_constant(fp a, int dst_gfn)
{
	for (int pn = 0 ; pn < N_patches() ; ++pn)
	{
	patch& p = ith_patch(pn);
		for (int irho = p.min_irho() ; irho <= p.max_irho() ; ++irho)
		{
		for (int isigma = p.min_isigma() ;
		     isigma <= p.max_isigma() ;
		     ++isigma)
		{
		p.gridfn(dst_gfn, irho,isigma) = a;
		}
		}
	}
}

//******************************************************************************

//
// This function sets a ghosted gridfn to a constant value.
//
void patch_system::set_ghosted_gridfn_to_constant(fp a, int ghosted_dst_gfn)
{
	for (int pn = 0 ; pn < N_patches() ; ++pn)
	{
	patch& p = ith_patch(pn);
		for (int irho = p.ghosted_min_irho() ;
                     irho <= p.ghosted_max_irho() ; ++irho)
		{
		for (int isigma = p.ghosted_min_isigma() ;
		     isigma <= p.ghosted_max_isigma() ;
		     ++isigma)
		{
		p.ghosted_gridfn(ghosted_dst_gfn, irho,isigma) = a;
		}
		}
	}
}

//******************************************************************************

//
// This function copies one (nominal-grid) gridfn to another.
//
void patch_system::gridfn_copy(int src_gfn, int dst_gfn)
{
	for (int pn = 0 ; pn < N_patches() ; ++pn)
	{
	patch& p = ith_patch(pn);
		for (int irho = p.min_irho() ; irho <= p.max_irho() ; ++irho)
		{
		for (int isigma = p.min_isigma() ;
		     isigma <= p.max_isigma() ;
		     ++isigma)
		{
		p.gridfn(dst_gfn, irho,isigma) = p.gridfn(src_gfn, irho,isigma);
		}
		}
	}
}

//******************************************************************************

//
// This function copies one ghosted gridfn to another.
//
void patch_system::ghosted_gridfn_copy(int ghosted_src_gfn,
                                       int ghosted_dst_gfn)
{
	for (int pn = 0 ; pn < N_patches() ; ++pn)
	{
	patch& p = ith_patch(pn);
		for (int irho = p.ghosted_min_irho() ;
                     irho <= p.ghosted_max_irho() ; ++irho)
		{
		for (int isigma = p.ghosted_min_isigma() ;
		     isigma <= p.ghosted_max_isigma() ;
		     ++isigma)
		{
		p.ghosted_gridfn(ghosted_dst_gfn, irho,isigma)
                  = p.ghosted_gridfn(ghosted_src_gfn, irho,isigma);
		}
		}
	}
}

//******************************************************************************

//
// This function copies a ghosted gridfn to a nominal gridfn.
//
void patch_system::ghosted_gridfn_copy_to_nominal(int ghosted_src_gfn,
                                                  int         dst_gfn)
{
	for (int pn = 0 ; pn < N_patches() ; ++pn)
	{
	patch& p = ith_patch(pn);
		for (int irho = p.min_irho() ; irho <= p.max_irho() ; ++irho)
		{
		for (int isigma = p.min_isigma() ; isigma <= p.max_isigma() ;
		     ++isigma)
		{
		p.gridfn(dst_gfn, irho,isigma)
                  = p.ghosted_gridfn(ghosted_src_gfn, irho,isigma);
		}
		}
	}
}

//******************************************************************************

//
// This function adds a scalar to a (nominal-grid) gridfn.
//
void patch_system::add_to_gridfn(fp delta, int dst_gfn)
{
	for (int pn = 0 ; pn < N_patches() ; ++pn)
	{
	patch& p = ith_patch(pn);
		for (int irho = p.min_irho() ;
		     irho <= p.max_irho() ;
		     ++irho)
		{
		for (int isigma = p.min_isigma() ;
		     isigma <= p.max_isigma() ;
		     ++isigma)
		{
		p.gridfn(dst_gfn, irho,isigma) += delta;
		}
		}
	}
}

//******************************************************************************

//
// This function adds a scalar to a ghosted gridfn.
//
void patch_system::add_to_ghosted_gridfn(fp delta, int ghosted_dst_gfn)
{
	for (int pn = 0 ; pn < N_patches() ; ++pn)
	{
	patch& p = ith_patch(pn);
		for (int irho = p.ghosted_min_irho() ;
		     irho <= p.ghosted_max_irho() ;
		     ++irho)
		{
		for (int isigma = p.ghosted_min_isigma() ;
		     isigma <= p.ghosted_max_isigma() ;
		     ++isigma)
		{
		p.ghosted_gridfn(ghosted_dst_gfn, irho,isigma) += delta;
		}
		}
	}
}

//******************************************************************************

//
// This function scales a ghosted gridfn with a scalar.
//
void patch_system::scale_ghosted_gridfn(fp factor, int ghosted_dst_gfn)
{
	for (int pn = 0 ; pn < N_patches() ; ++pn)
	{
	patch& p = ith_patch(pn);
		for (int irho = p.ghosted_min_irho() ;
		     irho <= p.ghosted_max_irho() ;
		     ++irho)
		{
		for (int isigma = p.ghosted_min_isigma() ;
		     isigma <= p.ghosted_max_isigma() ;
		     ++isigma)
		{
		p.ghosted_gridfn(ghosted_dst_gfn, irho,isigma) *= factor;
		}
		}
	}
}

//******************************************************************************

//
// This function computes norms of a nominal-grid gridfn.
//
void patch_system::gridfn_norms(int src_gfn, jtutil::norm<fp>& norms)
	const
{
if (! is_valid_gfn(src_gfn))
   then error_exit(ERROR_EXIT,
"***** patch_system::gridfn_norms(): invalid src_gfn=%d!\n",
		   src_gfn);					/*NOTREACHED*/

norms.reset();

	for (int pn = 0 ; pn < N_patches() ; ++pn)
	{
	const patch& p = ith_patch(pn);
		for (int irho = p.min_irho() ; irho <= p.max_irho() ; ++irho)
		{
		for (int isigma = p.min_isigma() ;
		     isigma <= p.max_isigma() ;
		     ++isigma)
		{
		norms.data(p.gridfn(src_gfn, irho,isigma));
		}
		}
	}
}

//******************************************************************************

//
// This function computes norms of a ghosted-grid gridfn over the
// nominal grid.
//
void patch_system::ghosted_gridfn_norms(int ghosted_src_gfn,
					jtutil::norm<fp>& norms)
	const
{
if (! is_valid_ghosted_gfn(ghosted_src_gfn))
   then error_exit(ERROR_EXIT,
"***** patch_system::gridfn_norms(): invalid ghosted_src_gfn=%d!\n",
		   ghosted_src_gfn);				/*NOTREACHED*/
norms.reset();

	for (int pn = 0 ; pn < N_patches() ; ++pn)
	{
	const patch& p = ith_patch(pn);
		for (int irho = p.min_irho() ; irho <= p.max_irho() ; ++irho)
		{
		for (int isigma = p.min_isigma() ;
		     isigma <= p.max_isigma() ;
		     ++isigma)
		{
		norms.data(p.ghosted_gridfn(ghosted_src_gfn, irho,isigma));
		}
		}
	}
}

//******************************************************************************
//******************************************************************************
//******************************************************************************

//
// This function computes an approximation to the circumference of a
// surface in the xy, xz, or yz plane.  Note that we compute the full
// circumference all around the sphere, even if the patch system only
// covers a proper subset of this.
//
// We assume that adjacent patches are butt-joined, i.e. that their
// nominal boundaries just touch.
//
// Arguments:
// plane[] = (in) "xy", "xz", or "yz" to specify the integration plane.
// ghosted_radius_gfn = (in) The surface radius.
// g_{xx,xy,xz,yy,yz,zz}_gfn = (in) The xyz 3-metric components.
// method = (in) Selects the integration scheme.
//
fp patch_system::circumference(const char plane[],
			       int ghosted_radius_gfn,
			       int g_xx_gfn, int g_xy_gfn, int g_xz_gfn,
					     int g_yy_gfn, int g_yz_gfn,
							   int g_zz_gfn,
			       enum patch::integration_method method)
	const
{
//
// compute arc length around the patch system
//
fp arc_length = 0.0;
    for (int pn = 0 ; pn < N_patches() ; ++pn)
    {
    const patch& p = ith_patch(pn);
    if ((p.ctype() == plane[0]) || (p.ctype() == plane[1]))
       then arc_length += p.plane_arc_length(plane,
					     ghosted_radius_gfn,
					     g_xx_gfn, g_xy_gfn, g_xz_gfn,
						       g_yy_gfn, g_yz_gfn,
								 g_zz_gfn,
					     method);
    }

//
// correct the arc length
// for the fact that the patch system may not cover the full 2-sphere
//
switch	(type())
	{
case patch_system__full_sphere:
	break;
case patch_system__plus_z_hemisphere:
	arc_length *= ((plane[0] == 'x') && (plane[1] == 'y')) ? 1.0 : 2.0;
	break;
case patch_system__plus_xy_quadrant_mirrored:
case patch_system__plus_xy_quadrant_rotating:
	arc_length *= ((plane[0] == 'x') && (plane[1] == 'y')) ? 4.0 : 2.0;
	break;
case patch_system__plus_xz_quadrant_mirrored:
case patch_system__plus_xz_quadrant_rotating:
	arc_length *= ((plane[0] == 'x') && (plane[1] == 'z')) ? 4.0 : 2.0;
	break;
case patch_system__plus_xyz_octant_mirrored:
case patch_system__plus_xyz_octant_rotating:
	arc_length *= 4.0;
	break;
default:
	error_exit(PANIC_EXIT,
"***** patch_system::circumference(): unknown patch system type()=(int)%d!\n"
"                                     (this should never happen!)\n",
		   int(type()));				/*NOTREACHED*/
	}

return arc_length;
}

//******************************************************************************

//
// This function computes an approximation to the (surface) integral of
// a gridfn over the 2-sphere
//	$\int f(\rho,\sigma) \, dA$
//		= \int f(\rho,\sigma) \sqrt{|J|} \, d\rho \, d\sigma$
// where $J$ is the Jacobian of $(x,y,z)$ with respect to $(rho,sigma).
//
// We assume that adjacent patches are butt-joined, i.e. that their
// nominal boundaries just touch.
//
// Arguments:
// unknown_src_gfn = (in) The gridfn to be integrated.  This may be
//			  either nominal-grid or ghosted-grid.
// src_gfn_is_even_across_{xy,xz,yz}_plane
//	= (in) Boolean flags specifying whether the gridfn to be integrated
//	       is even (true) or odd (false) across the corresponding planes.
//	       Only the flags corresponding to boundaries of the patch system
//	       are used.  For example, for a  plus_z_hemisphere  patch system,
//	       only the  src_gfn_is_even_across_xy_plane  flag is used.
// ghosted_radius_gfn = (in) The surface radius.
// g_{xx,xy,xz,yy,yz,zz}_gfn = (in) The xyz 3-metric components.
// method = (in) Selects the integration scheme.
//
fp patch_system::integrate_gridfn(int unknown_src_gfn,
				  bool src_gfn_is_even_across_xy_plane,
				  bool src_gfn_is_even_across_xz_plane,
				  bool src_gfn_is_even_across_yz_plane,
				  int ghosted_radius_gfn,
				  int g_xx_gfn, int g_xy_gfn, int g_xz_gfn,
						int g_yy_gfn, int g_yz_gfn,
							      int g_zz_gfn,
				  enum patch::integration_method method)
	const
{
//
// compute integral over patch system
//
fp integral = 0.0;
	for (int pn = 0 ; pn < N_patches() ; ++pn)
	{
	const patch& p = ith_patch(pn);
	integral += p.integrate_gridfn(unknown_src_gfn,
				       ghosted_radius_gfn,
				       g_xx_gfn, g_xy_gfn, g_xz_gfn,
						 g_yy_gfn, g_yz_gfn,
							   g_zz_gfn,
				       method);
	}

//
// correct the integral
// for the fact that the patch system may not cover the full 2-sphere
//
switch	(type())
	{
case patch_system__full_sphere:
	break;
case patch_system__plus_z_hemisphere:
	integral *= src_gfn_is_even_across_xy_plane ? 2.0 : 0.0;
	break;
case patch_system__plus_xy_quadrant_mirrored:
case patch_system__plus_xy_quadrant_rotating:
	integral *= src_gfn_is_even_across_xz_plane ? 2.0 : 0.0;
	integral *= src_gfn_is_even_across_yz_plane ? 2.0 : 0.0;
	break;
case patch_system__plus_xz_quadrant_mirrored:
case patch_system__plus_xz_quadrant_rotating:
	integral *= src_gfn_is_even_across_xy_plane ? 2.0 : 0.0;
	integral *= src_gfn_is_even_across_yz_plane ? 2.0 : 0.0;
	break;
case patch_system__plus_xyz_octant_mirrored:
case patch_system__plus_xyz_octant_rotating:
	integral *= src_gfn_is_even_across_xy_plane ? 2.0 : 0.0;
	integral *= src_gfn_is_even_across_xz_plane ? 2.0 : 0.0;
	integral *= src_gfn_is_even_across_yz_plane ? 2.0 : 0.0;
	break;
default:
	error_exit(PANIC_EXIT,
"***** patch_system::integrate_gridfn(): bad patch system type()=(int)%d!\n"
"                                        (this should never happen!)\n",
		   int(type()));				/*NOTREACHED*/
	}

return integral;
}

fp patch_system::integrate_gridpoint(int unknown_src_gfn,
                                     int ghosted_radius_gfn,
                                     int g_xx_gfn, int g_xy_gfn, int g_xz_gfn,
                                                   int g_yy_gfn, int g_yz_gfn,
                                                                 int g_zz_gfn,
                                     enum patch::integration_method method,
                                     int pn, int irho, int isigma)
	const
{
const patch& p = ith_patch(pn);
const fp val = p.integrate_gridpoint(unknown_src_gfn,
                                     ghosted_radius_gfn,
                                     g_xx_gfn, g_xy_gfn, g_xz_gfn,
                                               g_yy_gfn, g_yz_gfn,
                                                         g_zz_gfn,
                                     method,
                                     irho, isigma);

return val;
}
fp patch_system::integrate_correction(bool src_gfn_is_even_across_xy_plane,
                                      bool src_gfn_is_even_across_xz_plane,
                                      bool src_gfn_is_even_across_yz_plane)
	const
{
fp integral = 1.0;
//
// correct the integral
// for the fact that the patch system may not cover the full 2-sphere
//
switch	(type())
	{
case patch_system__full_sphere:
	break;
case patch_system__plus_z_hemisphere:
	integral *= src_gfn_is_even_across_xy_plane ? 2.0 : 0.0;
	break;
case patch_system__plus_xy_quadrant_mirrored:
case patch_system__plus_xy_quadrant_rotating:
	integral *= src_gfn_is_even_across_xz_plane ? 2.0 : 0.0;
	integral *= src_gfn_is_even_across_yz_plane ? 2.0 : 0.0;
	break;
case patch_system__plus_xz_quadrant_mirrored:
case patch_system__plus_xz_quadrant_rotating:
	integral *= src_gfn_is_even_across_xy_plane ? 2.0 : 0.0;
	integral *= src_gfn_is_even_across_yz_plane ? 2.0 : 0.0;
	break;
case patch_system__plus_xyz_octant_mirrored:
case patch_system__plus_xyz_octant_rotating:
	integral *= src_gfn_is_even_across_xy_plane ? 2.0 : 0.0;
	integral *= src_gfn_is_even_across_xz_plane ? 2.0 : 0.0;
	integral *= src_gfn_is_even_across_yz_plane ? 2.0 : 0.0;
	break;
default:
	error_exit(PANIC_EXIT,
"***** patch_system::integrate_gridfn(): bad patch system type()=(int)%d!\n"
"                                        (this should never happen!)\n",
		   int(type()));				/*NOTREACHED*/
	}

return integral;
}

//******************************************************************************
//******************************************************************************
//******************************************************************************

//
// This function finds what patch contains (the ray from the origin to)
// a given local (x,y,z) position.
//
// If there are multiple patches containing the position, we return the
// one which would still contain it if patches didn't overlap; if multiple
// patches satisfy this criterion then it's arbitrary which one we return.
//
// If no patch contains the position (this can only if the point as at
// the local coordinate origin, or for a non--full-sphere patch system),
// then we return a NULL pointer.
//
// Arguments:
// (x,y,z) = The local coordinates to be converted.
//
// Results:
// This function returns a reference to the containing patch.
//
const patch* patch_system::patch_containing_local_xyz(fp x, fp y, fp z)
	const
{
if ((x == 0.0) && (y == 0.0) && (z == 0.0))
   then return NULL;

// to which axis is (x,y,z) closest?
// ... or equivalently, which of |x|, |y|, and |z| is largest?
const fp abs_x = jtutil::abs(x);
const fp abs_y = jtutil::abs(y);
const fp abs_z = jtutil::abs(z);

if	((abs_z >= abs_x) && (abs_z >= abs_y))
   then return &plus_or_minus_xyz_patch(z > 0.0, 'z');	// +/- z patch
else if ((abs_x >= abs_y) && (abs_x >= abs_z))
   then return &plus_or_minus_xyz_patch(x > 0.0, 'x');	// +/- x patch
else if ((abs_y >= abs_x) && (abs_y >= abs_z))
   then return &plus_or_minus_xyz_patch(y > 0.0, 'y');	// +/- y patch
else	error_exit(ERROR_EXIT,
"***** patch_system::patch_containing_local_xyz():\n"
"        unknown (wierd!) ordering of |x|, |y|, and |z|!\n"
"        (this should never happen!)\n"
"        [local] (x,y,z)=(%g,%g,%g)\n"
,
		   double(x), double(y), double(z));		/*NOTREACHED*/
}

//******************************************************************************

//
// This function computes the radius of a patch-system 2-surface in the
// direction of a specified local (x,y,z) point, taking into account any
// patch-system symmetries.  If the point coincides with the local origin,
// we return the dummy value 1.0.
//
// Due to the surface-interpolator overhead, repeatedly calling this
// function is rather inefficient.
//
fp patch_system::radius_in_local_xyz_direction(int ghosted_radius_gfn,
					       fp x, fp y, fp z)
	const
{
if ((x == 0.0) && (y == 0.0) && (z == 0.0))
   then return 1.0;

//
// apply symmetries to map (x,y,z) into that part of the 2-sphere
// which is covered by the patch system
//
switch	(type())
	{
case patch_system__full_sphere:
	break;
case patch_system__plus_z_hemisphere:
	z = fabs(z);
	break;
case patch_system__plus_xy_quadrant_mirrored:
case patch_system__plus_xy_quadrant_rotating:
	x = fabs(x);
	y = fabs(y);
	break;
case patch_system__plus_xz_quadrant_mirrored:
case patch_system__plus_xz_quadrant_rotating:
	x = fabs(x);
	z = fabs(z);
	break;
case patch_system__plus_xyz_octant_mirrored:
case patch_system__plus_xyz_octant_rotating:
	x = fabs(x);
	y = fabs(y);
	z = fabs(z);
	break;
default:
	error_exit(PANIC_EXIT,
"***** patch_system::radius_in_local_xyz_direction():\n"
"        unknown patch system type()=(int)%d!\n"
"        (this should never happen!)\n",
		   int(type()));				/*NOTREACHED*/
	}

const patch* p_ptr = patch_containing_local_xyz(x, y, z);
if (p_ptr == NULL)
   then error_exit(ERROR_EXIT,
"***** patch_system::radius_in_local_xyz_direction():\n"
"                    can't find containing patch!\n"
"                    (this should never happen!)\n"
"                    [local] (x,y,z)=(%g,%g,%g)\n"
		   ,
		   double(x), double(y), double(z));		/*NOTREACHED*/

const patch& p = *p_ptr;
const fp   rho = p.  rho_of_xyz(x,y,z);
const fp sigma = p.sigma_of_xyz(x,y,z);


//
// Set up the surface interpolator to interpolate the surface radius
// gridfn to the (rho,sigma) coordinates:
//
// Notes on the interpolator setup:
// * The interpolator assumes Fortran subscripting, so we take the
//   coordinates in the order (sigma,rho) to match our C subscripting
//   in the patch system.
// * To avoid having to set up min/max array subscripts in the parameter
//   table, we treat the patch as using 0-origin (integer) array subscripts
//   (irho - ghosted_min_irho(), isigma - ghosted_min_isigma()).  However,
//   we use the usual floating-point coordinates.
//

const int N_dims = 2;
const CCTK_REAL coord_origin[N_dims]
	= { p.ghosted_min_sigma(), p.ghosted_min_rho() };
const CCTK_REAL coord_delta[N_dims]
	= { p.delta_sigma(), p.delta_rho() };

const int N_interp_points = 1;
const int interp_coords_type_code = CCTK_VARIABLE_REAL;
const void* const interp_coords[N_dims]
	= { static_cast<const void*>(&sigma), static_cast<const void*>(&rho) };

const int N_input_arrays = 1;
const CCTK_INT input_array_dims[N_dims]
	= { p.ghosted_N_isigma(), p.ghosted_N_irho() };
const CCTK_INT input_array_type_codes[N_input_arrays] = { CCTK_VARIABLE_REAL };
const void* const input_arrays[N_input_arrays]
	= {
	  static_cast<const void*>(
	    p.ghosted_gridfn_data_array(ghosted_radius_gfn)
				  )
	  };

const int N_output_arrays = 1;
const CCTK_INT output_array_type_codes[N_output_arrays]
	= { CCTK_VARIABLE_REAL };
fp xyz_radius;
void* const output_arrays[N_output_arrays]
	= { static_cast<void*>(&xyz_radius) };

const int status
	= CCTK_InterpLocalUniform(N_dims,
				  surface_interp_handle_,
				  surface_interp_par_table_handle_,
				  coord_origin, coord_delta,
				  N_interp_points, interp_coords_type_code,
						   interp_coords,
				  N_input_arrays, input_array_dims,
						  input_array_type_codes,
						  input_arrays,
				  N_output_arrays, output_array_type_codes,
						   output_arrays);
if (status < 0)
   then error_exit(ERROR_EXIT,
"***** patch_system::radius_in_local_xyz_direction():\n"
"                    error return (status=%d) from surface interpolator!\n"
"                    [local] (x,y,z)=(%g,%g,%g)\n"
"                    %s patch (rho,sigma)=(%g,%g)\n"
"                             (drho,dsigma)=(%g,%g)\n"
		   ,
		   status,
		   double(x), double(y), double(z),
		   p.name(), double(rho), double(sigma),
			     double(p.drho_of_rho(rho)),
			     double(p.dsigma_of_sigma(sigma)));	/*NOTREACHED*/

return xyz_radius;
}

struct perpatch {
  std::vector<int> n;
  std::vector<fp> rho, sigma;
  std::vector<fp> radii;
  void reserve (int const npoints)
  {
    n.reserve (npoints);
    rho.reserve (npoints);
    sigma.reserve (npoints);
    radii.reserve (npoints);
  }
  void addpoint (int const n_, fp const rho_, fp const sigma_)
  {
    n    .push_back (n_);
    rho  .push_back (rho_);
    sigma.push_back (sigma_);
    radii.push_back (fp (0.0));
  }
};

void patch_system::radii_in_local_xyz_directions(int const ghosted_radius_gfn,
                                                 int const npoints,
                                                 fp const * const xp,
                                                 fp const * const yp,
                                                 fp const * const zp,
                                                 fp * const radii)
	const
{
  std::vector<perpatch> patchpoints (N_patches());
  for (int pn=0; pn<N_patches(); ++pn) {
    patchpoints.at(pn).reserve (npoints);
  }
  
  for (int n=0; n<npoints; ++n) {
    fp x = xp[n];
    fp y = yp[n];
    fp z = zp[n];
    
    if ((x == 0.0) && (y == 0.0) && (z == 0.0)) {
      radii[n] = 1.0;
    } else {
      //
      // apply symmetries to map (x,y,z) into that part of the
      // 2-sphere which is covered by the patch system
      //
      switch (type())
      {
      case patch_system__full_sphere:
	break;
      case patch_system__plus_z_hemisphere:
	z = fabs(z);
	break;
      case patch_system__plus_xy_quadrant_mirrored:
      case patch_system__plus_xy_quadrant_rotating:
	x = fabs(x);
	y = fabs(y);
	break;
      case patch_system__plus_xz_quadrant_mirrored:
      case patch_system__plus_xz_quadrant_rotating:
	x = fabs(x);
	z = fabs(z);
	break;
      case patch_system__plus_xyz_octant_mirrored:
      case patch_system__plus_xyz_octant_rotating:
	x = fabs(x);
	y = fabs(y);
	z = fabs(z);
	break;
      default:
	error_exit(PANIC_EXIT,
"***** patch_system::radii_in_local_xyz_directions():\n"
"        unknown patch system type()=(int)%d!\n"
"        (this should never happen!)\n",
		   int(type()));				/*NOTREACHED*/
      }
      
      const patch* p_ptr = patch_containing_local_xyz(x, y, z);
      if (p_ptr == NULL)
        then error_exit(ERROR_EXIT,
"***** patch_system::radius_in_local_xyz_direction():\n"
"                    can't find containing patch!\n"
"                    (this should never happen!)\n"
"                    [local] (x,y,z)=(%g,%g,%g)\n"
		   ,
		   double(x), double(y), double(z));		/*NOTREACHED*/
      
      const patch& p = *p_ptr;
      const fp   rho = p.  rho_of_xyz(x,y,z);
      const fp sigma = p.sigma_of_xyz(x,y,z);
      
      patchpoints.at(p.patch_number()).addpoint (n, rho, sigma);
    }
  } // for n
  
  for (int pn=0; pn<N_patches(); ++pn) {
    const patch& p = ith_patch(pn);
    perpatch& localpatchpoints = patchpoints.at(pn);
    
    // Set up the surface interpolator to interpolate the surface
    // radius gridfn to the (rho,sigma) coordinates:
    //
    // Notes on the interpolator setup:
    // * The interpolator assumes Fortran subscripting, so we take the
    //   coordinates in the order (sigma,rho) to match our C
    //   subscripting in the patch system.
    // * To avoid having to set up min/max array subscripts in the
    //   parameter table, we treat the patch as using 0-origin
    //   (integer) array subscripts (irho - ghosted_min_irho(), isigma
    //   - ghosted_min_isigma()).  However, we use the usual
    //   floating-point coordinates.
    //
    
    const int N_dims = 2;
    const CCTK_REAL coord_origin[N_dims]
      = { p.ghosted_min_sigma(), p.ghosted_min_rho() };
    const CCTK_REAL coord_delta[N_dims]
      = { p.delta_sigma(), p.delta_rho() };
    
    const int N_interp_points = localpatchpoints.radii.size();
    const int interp_coords_type_code = CCTK_VARIABLE_REAL;
    const void* const interp_coords[N_dims]
      = { static_cast<const void*>(& localpatchpoints.sigma.front()),
          static_cast<const void*>(& localpatchpoints.rho  .front()) };
    
    const int N_input_arrays = 1;
    const CCTK_INT input_array_dims[N_dims]
      = { p.ghosted_N_isigma(), p.ghosted_N_irho() };
    const CCTK_INT input_array_type_codes[N_input_arrays]
      = { CCTK_VARIABLE_REAL };
    const void* const input_arrays[N_input_arrays]
      = { static_cast<const void*>
          (p.ghosted_gridfn_data_array(ghosted_radius_gfn)) };
    
    const int N_output_arrays = 1;
    const CCTK_INT output_array_type_codes[N_output_arrays]
      = { CCTK_VARIABLE_REAL };
    void* const output_arrays[N_output_arrays]
      = { static_cast<void*>(& localpatchpoints.radii.front()) };
    
    const int status = CCTK_InterpLocalUniform
      (N_dims,
       surface_interp_handle_,
       surface_interp_par_table_handle_,
       coord_origin, coord_delta,
       N_interp_points,
       interp_coords_type_code, interp_coords,
       N_input_arrays,
       input_array_dims, input_array_type_codes, input_arrays,
       N_output_arrays,
       output_array_type_codes, output_arrays);
    if (status < 0)
      then error_exit(ERROR_EXIT,
"***** patch_system::radii_in_local_xyz_directions():\n"
"                    error return (status=%d) from surface interpolator!\n"
		   ,
                      status);	/*NOTREACHED*/
    
    for (size_t nn=0; nn<localpatchpoints.radii.size(); ++nn) {
      radii[localpatchpoints.n.at(nn)] = localpatchpoints.radii.at(nn);
    }
    
  } // for p
}

//******************************************************************************
//******************************************************************************
//******************************************************************************

//
// This function prints an unknown-grid gridfn in ASCII format to a
// named output file.  The output format is suitable for a gnuplot
// 'splot' command.  (Individual patches may be selected with the
//  select.patch  program (perl script).)  The output format is either
//	# print_xyz_flag == false
//	dpx	dpy	gridfn
// or
//	# print_xyz_flag == true
//	dpx	dpy	gridfn   global_x   global_y   global_z
// where global_[xyz] are derived from the angular position and a
// specified (unknown-grid) radius gridfn.
//
void patch_system::print_unknown_gridfn
	(bool ghosted_flag, int unknown_gfn,
	 bool print_xyz_flag, bool radius_is_ghosted_flag,
			      int unknown_radius_gfn,
	 const char output_file_name[], bool want_ghost_zones)
	const
{
if (want_ghost_zones && !ghosted_flag)
   then error_exit(PANIC_EXIT,
"***** patch_system::print_unknown_gridfn(unknown_gfn=%d):\n"
"        can't have want_ghost_zones && !ghosted_flag !\n"
,
		   unknown_gfn);				/*NOTREACHED*/
if (want_ghost_zones && print_xyz_flag && !radius_is_ghosted_flag)
   then error_exit(PANIC_EXIT,
"***** patch_system::print_unknown_gridfn(unknown_gfn=%d):\n"
"        can't have want_ghost_zones && print_xyz_flag\n"
"        && !radius_is_ghosted_flag!\n"
"        unknown_radius_gfn=%d\n"
,
		   unknown_gfn,
		   unknown_radius_gfn);				/*NOTREACHED*/

FILE *output_fp = fopen(output_file_name, "w");
if (output_fp == NULL)
   then error_exit(ERROR_EXIT,
"***** patch_system::print_unknown_gridfn(unknown_gfn=%d):\n"
"        can't open output file \"%s\"\n!"
,
		   unknown_gfn,
		   output_file_name);				/*NOTREACHED*/

fprintf(output_fp, "# N_patches = %d\n", N_patches());
fprintf(output_fp, "# origin = %.15g %.15g %.15g\n",
        double(origin_x()), double(origin_y()), double(origin_z()));
fprintf(output_fp, "\n");

	for (int pn = 0 ; pn < N_patches() ; ++pn)
	{
	const patch& p = ith_patch(pn);

	fprintf(output_fp, "### %s patch\n", p.name());
	fprintf(output_fp, "# N_rho = %d\n",
		p.effective_N_irho(want_ghost_zones));
	fprintf(output_fp, "# N_sigma = %d\n",
		p.effective_N_isigma(want_ghost_zones));
	fprintf(output_fp, "# %s_gfn=%d\n",
		(ghosted_flag ? "ghosted" : "nominal"), unknown_gfn);
	fprintf(output_fp, "# dpx = %s\n", p.name_of_dpx());
	fprintf(output_fp, "# dpy = %s\n", p.name_of_dpy());
	fprintf(output_fp, "#\n");
	fprintf(output_fp,
		print_xyz_flag
		? "# dpx\tdpy\tgridfn\tglobal_x\tglobal_y\tglobal_z\n"
		: "# dpx\tdpy\tgridfn\n");

		for (int irho = p.effective_min_irho(want_ghost_zones) ;
		     irho <= p.effective_max_irho(want_ghost_zones) ;
		     ++irho)
		{
		for (int isigma = p.effective_min_isigma(want_ghost_zones) ;
		     isigma <= p.effective_max_isigma(want_ghost_zones) ;
		     ++isigma)
		{
		const fp rho = p.rho_of_irho(irho);
		const fp sigma = p.sigma_of_isigma(isigma);
		const fp dpx = p.dpx_of_rho_sigma(rho, sigma);
		const fp dpy = p.dpy_of_rho_sigma(rho, sigma);
		fprintf(output_fp,
			"%g\t%g\t%#.15g",
			double(dpx), double(dpy),
                        double(p.unknown_gridfn(ghosted_flag,
                                                unknown_gfn, irho,isigma)));
		if (print_xyz_flag)
		   then {
			const fp r = p.unknown_gridfn(radius_is_ghosted_flag,
						      unknown_radius_gfn,
						      irho,isigma);
			fp local_x, local_y, local_z;
			p.xyz_of_r_rho_sigma(r,rho,sigma,
					     local_x,local_y,local_z);
			const fp global_x = origin_x() + local_x;
			const fp global_y = origin_y() + local_y;
			const fp global_z = origin_z() + local_z;
			fprintf(output_fp,
				"\t%#.10g\t%#.10g\t%#.10g",
				double(global_x), double(global_y),
                                double(global_z));
			}
		fprintf(output_fp, "\n");
		}
		fprintf(output_fp, "\n");
		}
	fprintf(output_fp, "\n");
	}

fclose(output_fp);
}

//******************************************************************************

//
// This function reads an unknown-grid gridfn in ASCII format from
// a named input file.  Comments ('#' in column 1) and blank lines
// are ignored, otherwise the input format matches that written by
// print_unknown_gridfn(): the first 3 numbers on each line are taken
// to be dpx, dpy, and the gridfn value; anything else on the line is
// ignored.
//
void patch_system::read_unknown_gridfn(bool ghosted_flag, int unknown_gfn,
				       const char input_file_name[],
				       bool want_ghost_zones)
{
if (want_ghost_zones && !ghosted_flag)
   then error_exit(PANIC_EXIT,
"***** patch_system::read_unknown_gridfn(unknown_gfn=%d):\n"
"        can't have want_ghost_zones && !ghosted_flag !\n"
,
		   unknown_gfn);				/*NOTREACHED*/

FILE *input_fp = fopen(input_file_name, "r");
if (input_fp == NULL)
   then error_exit(ERROR_EXIT,
"***** patch_system::read_unknown_gridfn(unknown_gfn=%d):\n"
"        can't open input file \"%s\"\n!"
,
		   unknown_gfn,
		   input_file_name);				/*NOTREACHED*/

int line_number = 1;
	for (int pn = 0 ; pn < N_patches() ; ++pn)
	{
	patch& p = ith_patch(pn);

	for (int irho = p.effective_min_irho(want_ghost_zones) ;
	     irho <= p.effective_max_irho(want_ghost_zones) ;
	     ++irho)
	{
	for (int isigma = p.effective_min_isigma(want_ghost_zones) ;
	     isigma <= p.effective_max_isigma(want_ghost_zones) ;
	     ++isigma)
	{
	const fp rho   = p.rho_of_irho(irho);
	const fp sigma = p.sigma_of_isigma(isigma);
	const fp dpx = p.dpx_of_rho_sigma(rho, sigma);
	const fp dpy = p.dpy_of_rho_sigma(rho, sigma);

	const int buffer_size = 250;
	char buffer[buffer_size];
	// read/discard comments and blank lines
		do
		{
		if (fgets(buffer, buffer_size, input_fp) == NULL)
		   then error_exit(ERROR_EXIT,
"***** patch::read_unknown_gridfn(%s patch, unknown_gfn=%d):\n"
"        I/O error or unexpected end-of-file on input!\n"
"        at irho=%d of [%d,%d], isigma=%d of [%d,%d]\n"
"        dpx=%g dpy=%g\n"
,
				   p.name(), unknown_gfn,
				   int(irho),
                                      p.effective_min_irho(want_ghost_zones),
				      p.effective_max_irho(want_ghost_zones),
				   int(isigma),
				      p.effective_min_isigma(want_ghost_zones),
				      p.effective_max_isigma(want_ghost_zones),
				   double(dpx), double(dpy));	/*NOTREACHED*/
		++line_number;
		} while ((buffer[0] == '#') || (buffer[0] == '\n'));

	double read_dpx, read_dpy, read_gridfn_value;
	if (sscanf(buffer, "%lf %lf %lf",
		   &read_dpx, &read_dpy, &read_gridfn_value) != 3)
	   then error_exit(ERROR_EXIT,
"***** patch::read_unknown_gridfn(%s patch, unknown_gfn=%d):\n"
"        bad input data at input line %d!\n"
,
			   p.name(), unknown_gfn,
			   line_number);			/*NOTREACHED*/
	if (! (    jtutil::fuzzy<fp>::EQ(read_dpx,dpx)
		&& jtutil::fuzzy<fp>::EQ(read_dpy,dpy)    ) )
	   then error_exit(ERROR_EXIT,
"***** patch::read_unknown_gridfn(%s patch, unknown_gfn=%d):\n"
"        wrong (dpx,dpy) at input line %d!\n"
"        expected (%g,%g)\n"
"        read     (%g,%g)\n"
,
			   p.name(), unknown_gfn,
			   line_number,
			   double(dpx), double(dpy),
			   double(read_dpx), double(read_dpy));	/*NOTREACHED*/

	p.unknown_gridfn(ghosted_flag,
			 unknown_gfn, irho,isigma) = read_gridfn_value;
	}
	}

	}

fclose(input_fp);
}

//******************************************************************************

#ifdef HAVE_CAPABILITY_HDF5

static void WriteAttribute (const hid_t dataset, const char* name, char value);
static void WriteAttribute (const hid_t dataset, const char* name, const char* values);
static void WriteAttribute (const hid_t dataset, const char* name, const char* values, int nvalues);

static void WriteAttribute (const hid_t dataset, const char* const name, const char value)
{
  WriteAttribute (dataset, name, &value, 1);
}

static void WriteAttribute (const hid_t dataset, const char* const name, const char* const values)
{
  WriteAttribute (dataset, name, values, strlen(values));
}

static void WriteAttribute (const hid_t dataset, const char* const name, const char* const values, const int nvalues)
{
  assert (dataset>=0);
  assert (name);
  assert (values);
  assert (nvalues>=0);
  
  herr_t herr;
  
  const hid_t dataspace = H5Screate (H5S_SCALAR);
  assert (dataspace>=0);
  
  const hid_t datatype = H5Tcopy (H5T_C_S1);
  assert (datatype>=0);
  herr = H5Tset_size (datatype, nvalues);
  assert (!herr);
  
  const hid_t attribute = H5Acreate (dataset, name, datatype, dataspace, H5P_DEFAULT);
  assert (attribute>=0);
  herr = H5Awrite (attribute, datatype, values);
  assert (!herr);
  herr = H5Aclose (attribute);
  assert (!herr);
  
  herr = H5Tclose (datatype);
  assert (!herr);
  
  herr = H5Sclose (dataspace);
  assert (!herr);
}

#endif

//
// This function output an unknown-grid gridfn in HDF5 format to a
// named output file.
//
void patch_system::output_unknown_gridfn
	(bool ghosted_flag, int unknown_gfn,
         const char gfn_name[], const cGH *cctkGH,
	 bool output_xyz_flag, bool radius_is_ghosted_flag,
                               int unknown_radius_gfn,
	 const char output_file_name[], bool want_ghost_zones)
	const
{
if (want_ghost_zones && !ghosted_flag)
   then error_exit(PANIC_EXIT,
"***** patch_system::output_unknown_gridfn(unknown_gfn=%d):\n"
"        can't have want_ghost_zones && !ghosted_flag !\n"
,
		   unknown_gfn);				/*NOTREACHED*/
if (want_ghost_zones && output_xyz_flag && !radius_is_ghosted_flag)
   then error_exit(PANIC_EXIT,
"***** patch_system::output_unknown_gridfn(unknown_gfn=%d):\n"
"        can't have want_ghost_zones && output_xyz_flag\n"
"        && !radius_is_ghosted_flag!\n"
"        unknown_radius_gfn=%d\n"
,
		   unknown_gfn,
		   unknown_radius_gfn);				/*NOTREACHED*/

#ifdef HAVE_CAPABILITY_HDF5
  
  using std::ostringstream;
  using std::string;
  using std::vector;
    
  herr_t herr;
  
  hid_t writer;
  
  // Get grid hierarchy extentsion from IOUtil
  const ioGH * const iogh = (const ioGH *)CCTK_GHExtension (cctkGH, "IO");
  assert (iogh);
  
  struct stat fileinfo;
  if (! iogh->recovered || stat(output_file_name, &fileinfo) != 0) {
    writer = H5Fcreate (output_file_name, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
  } else {
    writer = H5Fopen (output_file_name, H5F_ACC_RDWR, H5P_DEFAULT);
  }
  assert (writer >= 0);
  
  hid_t datatype;
  if (sizeof(fp) == sizeof(float)) {
    datatype = H5T_NATIVE_FLOAT;
  } else if (sizeof(fp) == sizeof(double)) {
    datatype = H5T_NATIVE_DOUBLE;
  } else if (sizeof(fp) == sizeof(long double)) {
    datatype = H5T_NATIVE_LDOUBLE;
  } else {
    assert (0);
  }
  assert (datatype >= 0);
  
  for (int pn=0; pn<N_patches(); ++pn) {
    const patch& p = ith_patch(pn);
    
    hsize_t shape[2];
    shape[0] = p.effective_N_isigma(want_ghost_zones);
    shape[1] = p.effective_N_irho(want_ghost_zones);
    
    hid_t dataspace = H5Screate_simple (2, shape, NULL);
    assert (dataspace >= 0);
    
    
    
    ostringstream datasetnamebuf;
    datasetnamebuf << gfn_name
                   << " it=" << cctkGH->cctk_iteration
                   << " patch=" << pn;
    string datasetnamestr = datasetnamebuf.str();
    assert (datasetnamestr.size() <= 256); // limit dataset name length
    const char * const datasetname = datasetnamestr.c_str();
    assert (datasetname);
    
    ostringstream datasetnamebufx;
    datasetnamebufx << "x"
                    << " it=" << cctkGH->cctk_iteration
                    << " patch=" << pn;
    string datasetnamestrx = datasetnamebufx.str();
    assert (datasetnamestrx.size() <= 256); // limit dataset name length
    const char * const datasetnamex = datasetnamestrx.c_str();
    assert (datasetnamex);
    
    ostringstream datasetnamebufy;
    datasetnamebufy << "y"
                    << " it=" << cctkGH->cctk_iteration
                    << " patch=" << pn;
    string datasetnamestry = datasetnamebufy.str();
    assert (datasetnamestry.size() <= 256); // limit dataset name length
    const char * const datasetnamey = datasetnamestry.c_str();
    assert (datasetnamey);
    
    ostringstream datasetnamebufz;
    datasetnamebufz << "z"
                    << " it=" << cctkGH->cctk_iteration
                    << " patch=" << pn;
    string datasetnamestrz = datasetnamebufz.str();
    assert (datasetnamestrz.size() <= 256); // limit dataset name length
    const char * const datasetnamez = datasetnamestrz.c_str();
    assert (datasetnamez);
    
    
    
    hid_t dataset = H5Dcreate (writer, datasetname, datatype, dataspace, H5P_DEFAULT);
    assert (dataset >= 0);
    
    hid_t datasetx;
    hid_t datasety;
    hid_t datasetz;
    if (output_xyz_flag) {
      
      datasetx = H5Dcreate (writer, datasetnamex, datatype, dataspace, H5P_DEFAULT);
      assert (datasetx >= 0);
      
      datasety = H5Dcreate (writer, datasetnamey, datatype, dataspace, H5P_DEFAULT);
      assert (datasety >= 0);
      
      datasetz = H5Dcreate (writer, datasetnamez, datatype, dataspace, H5P_DEFAULT);
      assert (datasetz >= 0);
      
    } // if output_xyf_flag
    
    
    
    vector<fp> tmpdata;
    vector<fp> tmpx, tmpy, tmpz;
    tmpdata.resize (shape[0] * shape[1]);
    if (output_xyz_flag) {
      tmpx.resize (shape[0] * shape[1]);
      tmpy.resize (shape[0] * shape[1]);
      tmpz.resize (shape[0] * shape[1]);
    }
    
    int elt = 0;
    for (int irho = p.effective_min_irho(want_ghost_zones);
         irho <= p.effective_max_irho(want_ghost_zones);
         ++irho)
    {
      for (int isigma = p.effective_min_isigma(want_ghost_zones);
           isigma <= p.effective_max_isigma(want_ghost_zones);
           ++isigma)
      {
        
        tmpdata.at(elt) = p.unknown_gridfn (ghosted_flag, unknown_gfn, irho,isigma);
        if (output_xyz_flag) {
          const fp r = p.unknown_gridfn(radius_is_ghosted_flag,
                                        unknown_radius_gfn,
                                        irho,isigma);
          const fp rho   = p.rho_of_irho(irho);
          const fp sigma = p.sigma_of_isigma(isigma);
          fp local_x, local_y, local_z;
          p.xyz_of_r_rho_sigma(r,rho,sigma,
                               local_x,local_y,local_z);
          const fp global_x = origin_x() + local_x;
          const fp global_y = origin_y() + local_y;
          const fp global_z = origin_z() + local_z;
          tmpx.at(elt) = global_x;
          tmpy.at(elt) = global_y;
          tmpz.at(elt) = global_z;
        }
        ++elt;
        
      }
    }
    
    
    
    herr = H5Dwrite (dataset, datatype, H5S_ALL, H5S_ALL, H5P_DEFAULT, &tmpdata.front());
    assert (! herr);
    
    WriteAttribute (dataset, "name", gfn_name);
    
    herr = H5Dclose (dataset);
    assert (! herr);
    
    
    
    if (output_xyz_flag) {
      
      herr = H5Dwrite (datasetx, datatype, H5S_ALL, H5S_ALL, H5P_DEFAULT, &tmpx.front());
      assert (! herr);
      
      WriteAttribute (datasetx, "name", "x");
      
      herr = H5Dclose (datasetx);
      assert (! herr);
      
      
      
      herr = H5Dwrite (datasety, datatype, H5S_ALL, H5S_ALL, H5P_DEFAULT, &tmpy.front());
      assert (! herr);
      
      WriteAttribute (datasety, "name", "y");
      
      herr = H5Dclose (datasety);
      assert (! herr);
      
      
      
      herr = H5Dwrite (datasetz, datatype, H5S_ALL, H5S_ALL, H5P_DEFAULT, &tmpz.front());
      assert (! herr);
      
      WriteAttribute (datasetz, "name", "z");
      
      herr = H5Dclose (datasetz);
      assert (! herr);
      
    } // if output_xyz_flag
    
    
    
    herr = H5Sclose (dataspace);
    assert (! herr);
    
  } // for pn
  
  
  
  herr = H5Fclose (writer);
  assert (! herr);
  
#else
  error_exit(ERROR_EXIT,
"***** patch_system::output_unknown_gridfn(unknown_gfn=%d):\n"
"        no HDF5 support compiled in.  Cannot write output file \"%s\"\n!"
,
             unknown_gfn,
             output_file_name);					/*NOTREACHED*/

#endif
}

//******************************************************************************
//******************************************************************************
//******************************************************************************

//
// This function "synchronizes" all ghost zones of all patches, i.e. it
// update the ghost-zone values of the specified gridfns via the appropriate
// sequence of symmetry operations and interpatch interpolations.  This
// process is described in detail in the header comments in "ghost_zone.hh".
//
void patch_system::synchronize(int ghosted_min_gfn_to_sync,
			       int ghosted_max_gfn_to_sync)
{
//
// Phase 1:
// Fill in gridfn data at all the non-corner points of symmetry ghost
// zones, using the symmetries to get this data from its "home patch"
// nominal grids.
//
	  {
	for (int pn = 0 ; pn < N_patches() ; ++pn)
	{
	patch& p = ith_patch(pn);
		// n.b. these loops must use _int_ variables for the loop
		//      to terminate!
		for (int want_min = false ; want_min <= true ; ++want_min)
		{
		for (int want_rho = false ; want_rho <= true ; ++want_rho)
		{
		ghost_zone& gz = p.minmax_ang_ghost_zone(want_min, want_rho);
		if (gz.is_symmetry())
		   then gz.synchronize(ghosted_min_gfn_to_sync,
				       ghosted_max_gfn_to_sync,
				       false,	// want corners?
				       true);	// want non-corner?
		}
		}
	}
	  }

//
// Phase 2:
// Fill in gridfn data in all the interpatch ghost zones, using interpatch
// interpolation from neighboring patches as described above.
//
	  {
	for (int pn = 0 ; pn < N_patches() ; ++pn)
	{
	patch& p = ith_patch(pn);
		// n.b. these loops must use _int_ variables for the loop
		//      to terminate!
		for (int want_min = false ; want_min <= true ; ++want_min)
		{
		for (int want_rho = false ; want_rho <= true ; ++want_rho)
		{
		ghost_zone& gz = p.minmax_ang_ghost_zone(want_min, want_rho);
		if (gz.is_interpatch())
		   then gz.synchronize(ghosted_min_gfn_to_sync,
				       ghosted_max_gfn_to_sync);
		}
		}
	}
	  }

//
// Phase 3:
// Fill in gridfn data at all the corner points of symmetry ghost zones,
// using the symmetries to get this data from its "home patch" nominal
// grids or ghost zones.
//
	  {
	for (int pn = 0 ; pn < N_patches() ; ++pn)
	{
	patch& p = ith_patch(pn);
		// n.b. these loops must use _int_ variables for the loop
		//      to terminate!
		for (int want_min = false ; want_min <= true ; ++want_min)
		{
		for (int want_rho = false ; want_rho <= true ; ++want_rho)
		{
		ghost_zone& gz = p.minmax_ang_ghost_zone(want_min, want_rho);
		if (gz.is_symmetry())
		   then gz.synchronize(ghosted_min_gfn_to_sync,
				       ghosted_max_gfn_to_sync,
				       true,	// want corners?
				       false);	// want non-corner?
		}
		}
	}
	  }
}

//******************************************************************************

//
// This function does any precomputation necessary to compute the Jacobian
// of  synchronize() , taking into account synchronize()'s full 3-phase
// algorithm.  In practice, this means it computes the individual Jacobian
// of each ghost zone, and sets  global_{min,max}_ym_ .
//
void patch_system::compute_synchronize_Jacobian(int ghosted_min_gfn_to_sync,
						int ghosted_max_gfn_to_sync)
	const
{
global_min_ym_ = +INT_MAX;
global_max_ym_ = -INT_MAX;
	for (int pn = 0 ; pn < N_patches() ; ++pn)
	{
	const patch& p = ith_patch(pn);
		// n.b. these loops must use _int_ variables for the loop
		//      to terminate!
		for (int want_min = false ; want_min <= true ; ++want_min)
		{
		for (int want_rho = false ; want_rho <= true ; ++want_rho)
		{
		ghost_zone& gz = p.minmax_ang_ghost_zone(want_min, want_rho);
		gz.compute_Jacobian(ghosted_min_gfn_to_sync,
				    ghosted_max_gfn_to_sync);

		global_min_ym_ = jtutil::min(global_min_ym_,
					     gz.Jacobian_min_y_ipar_m());
		global_max_ym_ = jtutil::max(global_max_ym_,
					     gz.Jacobian_max_y_ipar_m());
		}
		}
	}
}

//******************************************************************************

//
// Given that  compute_synchronize_Jacobian()  has been called, this
// function computes the global min/max  m  over all ghost zone points.
// This is useful for sizing the buffer for synchronize_Jacobian().
//
void patch_system::synchronize_Jacobian_global_minmax_ym
	(int& min_ym, int& max_ym)
	const
{
min_ym = global_min_ym_;
max_ym = global_max_ym_;
}

//******************************************************************************

//
// Given that  compute_synchronize_Jacobian()  has been called, this
// function computes a single row of the Jacobian, taking into account
// synchronize()'s 3-phase algorithm:
// - It returns the edge to which the y point belongs (the caller can get
//   the patch from this edge).
// - It stores y_iperp and y_posn and min/max ym in the named arguments.
// - It stores the Jacobian elements
//	partial synchronize() gridfn(ghosted_gfn, px, x_iperp, x_ipar)
//	-------------------------------------------------------------
//	     partial gridfn(ghosted_gfn, py, y_iperp, y_posn+ym)
//   in the caller-supplied buffer
//	Jacobian_buffer(ym)
//   for each  ym  in the min/max ym range.
//
// In practice, the main task of this function is taking into account
// synchronize()'s 3-phase algorithm.  There are several cases:
// - ghost zone is symmetry && x point is in non-corner
//   ==> x value was computed by a phase 1 symmetry operation,
//       using (only) nominal-grid data
//   ==> overall Jacobian(x,y) = ghost zone Jacobian(x,y)
// - ghost zone is symmetry && x point is in corner
//   --> x value was computed by a phase 3 symmetry operation,
//       from some point (call it z),
//       ==> overall Jacobian(x,y) = overall Jacobian(z,y)
//       ==> call this function recursively to get z's Jacobian
//           (z must be in the noncorner part of some ghost zone,
//	      so this won't lead to infinite recursion)
// - ghost zone is interpatch
//   ==> x value was computed by a phase 2 interpatch interpolation
//       - using (only) nominal-grid data
//         ==> overall Jacobian(x,y) = ghost zone Jacobian(x,y)
//       - using a mixture of nominal-grid data
//         and data computed by a phase 1 symmetry operation
//         ==> overall Jacobian(x,y) = "fold" ghost zone Jacobian(x,y)
//                                     to take the phase 1 symmetry
//                                     operation into account
//
const patch_edge&
  patch_system::synchronize_Jacobian(const ghost_zone& xgz,
				     int x_iperp, int x_ipar,
				     int& y_iperp,
				     int& y_posn, int& min_ym, int& max_ym,
				     jtutil::array1d<fp>& Jacobian_buffer)
	const
{
const patch_edge& xe = xgz.my_edge();

if (xgz.is_symmetry() && xe.ipar_is_in_noncorner(x_ipar))
   then {
	// ghost zone is symmetry && x point is in non-corner
	// ==> x value was computed by a phase 1 symmetry operation,
	//     using (only) nominal-grid data
	// ==> overall Jacobian(x,y) = ghost zone Jacobian(x,y)
	return ghost_zone_Jacobian(xgz,
				   x_iperp, x_ipar,
				   y_iperp,
				   y_posn, min_ym, max_ym,
				   Jacobian_buffer);
	}

else if (xgz.is_symmetry() && xe.ipar_is_in_corner(x_ipar))
   then {
	// ghost zone is symmetry && x point is in corner
	// --> x value was computed by a phase 3 symmetry operation,
	//     from some point (call it z),
	//     ==> overall Jacobian(x,y) = overall Jacobian(z,y)
	//     ==> call this function recursively to get z's Jacobian
	//         (z must be in the noncorner part of some ghost zone,
	//          so this won't lead to infinite recursion)

	const patch&      zp = xgz.other_patch();
	const patch_edge& ze = xgz.other_edge();
	const symmetry_ghost_zone& xsgz = xgz.cast_to_symmetry_ghost_zone();
	const int z_iperp = xsgz.iperp_map_of_iperp(x_iperp);
	const int z_ipar  = xsgz.ipar_map_of_ipar  (x_ipar);

	//
	// Computing z's edge/ghost zone is tricky.  For example:
	//	                        |
	//	           p1         e3|e4          p2
	//	                        |
	//	                        |   z
	//	-----------e1-----------+------------e2---------
	//	                        |   x
	//	                        |
	// Here the point x in the corner of p1's e1 ghost zone,
	// is computed by the phase 3 symmetry operation (a reflection
	// about e1) from z.  Thus zp == p1 and ze == e1.
	//
	// But we need to "turn the corner" to compute z's "true" edge
	// e3 (so we can recursively call this function to compute z's
	// Jacobian).  Thus we explicitly check which ghost zone of p1
	// (here the e3 ghost zone) contains the point z.
	//
	const int z_irho   = ze.  irho_of_iperp_ipar(z_iperp, z_ipar);
	const int z_isigma = ze.isigma_of_iperp_ipar(z_iperp, z_ipar);
	const ghost_zone& true_zgz
		= zp.ghost_zone_containing_noncorner_point(z_irho, z_isigma);
	const patch_edge& true_ze = true_zgz.my_edge();
	const int true_z_iperp = true_ze.iperp_of_irho_isigma(z_irho, z_isigma);
	const int true_z_ipar  = true_ze. ipar_of_irho_isigma(z_irho, z_isigma);

	// make sure we have the right ghost zone!
	assert( true_zgz.is_in_ghost_zone(true_z_iperp, true_z_ipar) );

	return synchronize_Jacobian(true_zgz,
				    true_z_iperp, true_z_ipar,
				    y_iperp,
				    y_posn, min_ym, max_ym,
				    Jacobian_buffer);
	}

else if (xgz.is_interpatch())
   then {
	// ghost zone is interpatch
	// ==> x value was computed by a phase 2 interpatch interpolation
	//     - using (only) nominal-grid data
	//       ==> overall Jacobian(x,y) = ghost zone Jacobian(x,y)
	//     - using a mixture of nominal-grid data
	//       and data computed by a phase 1 symmetry operation
	//       ==> overall Jacobian(x,y) = "fold" ghost zone Jacobian(x,y)
	//                                   to take the phase 1 symmetry
	//                                   operation into account
	//
	// For example,
	//	                        |
	//	           xp         xe|ye a        yp
	//	                        |   b
	//	                        |  xc
	//	----------xse-----------+---d-------yse----------
	//	                        |   e
	//	                        |
	// here point x is computed by interpatch-interpolating in the
	// par direction from the 5 y points abcde.  e is outside the
	// nominal grid, so its Jacobian must be "folded" over to c.
	// Notice that this "folding" must be done about the edge yse,
	// *not* about ye itself.

	// Jacobian of the phase 2 interpatch interpolation
	const patch_edge& ye = ghost_zone_Jacobian(xgz,
						   x_iperp, x_ipar,
						   y_iperp,
						   y_posn, min_ym, max_ym,
						   Jacobian_buffer);
	const int min_y_ipar = y_posn + min_ym;
	const int max_y_ipar = y_posn + max_ym;

	// fold any points in the Jacobian outside the nominal grid
	if (ye.ipar_is_in_min_ipar_corner(min_y_ipar))
	   then {
		fold_Jacobian(ye, ye.min_par_adjacent_edge(),
			      y_iperp,
			      y_posn, min_ym, max_ym,
			      min_ym, ye.min_ipar_corner__max_ipar() - y_posn,
			      Jacobian_buffer);
		min_ym = ye.min_ipar_without_corners() - y_posn;
		}
	if (ye.ipar_is_in_max_ipar_corner(max_y_ipar))
	   then {
		fold_Jacobian(ye, ye.max_par_adjacent_edge(),
			      y_iperp,
			      y_posn, min_ym, max_ym,
			      ye.max_ipar_corner__min_ipar() - y_posn, max_ym,
			      Jacobian_buffer);
		max_ym = ye.max_ipar_without_corners() - y_posn;
		}

	return ye;
	}

else	error_exit(PANIC_EXIT,
"***** patch_system::synchronize_Jacobian():\n"
"        don't know what to do with ghost zone (this should never happen)!\n"
"        xgz.my_patch()=\"%s\"    xe=xgz.my_edge()=\"%s\"\n"
"        xgz.other_patch()=\"%s\"    xgz.other_edge()=\"%s\"\n"
"        xgz.is_symmetry()=(int)%d xgz.is_interpatch()=(int)%d\n"
"        x_iperp=%d x_ipar=%d\n"
"        xe.ipar_is_in_{min,max}_ipar_corner(x_ipar)=(int){%d,%d}\n"
"        xe.ipar_is_in_{corner,noncorner}(x_ipar)=(int){%d,%d}\n"
,
		   xgz.my_patch().name(),    xe.name(),
		   xgz.other_patch().name(), xgz.other_edge().name(),
		   int(xgz.is_symmetry()), int(xgz.is_interpatch()),
		   x_iperp, x_ipar,
		   xe.ipar_is_in_min_ipar_corner(x_ipar),
		   xe.ipar_is_in_max_ipar_corner(x_ipar),
		   xe.ipar_is_in_corner(x_ipar),
		   xe.ipar_is_in_noncorner(x_ipar));		/*NOTREACHED*/
}

//******************************************************************************

//
// This function "folds" part of a(n interpatch) Jacobian row to take
// a symmetry operation into account.  For example:
//	         |
//	         |e_Jac
//	         |             p
//	         |   a
//	         |   b
//	         |   c=y
//	---------+---d-------e_fold-------
//	         |   e=x    sgz_fold
//	         |
// Here the Jacobian abcde is to be "folded", because e is outside the
// nominal grid (its Jacobian must be "folded" over to c).
//
// Notice that the folding (about the edge e_fold) is in the par direction
// with respect to e_Jac, but the perp direction with respect to e_fold.
// Since e_fold and e_Jac are adjacent edges, 
//	e_Jac (iperp,ipar) == e_fold (ipar,iperp)
//
// Arguments:
// e_Jac = edge which the Jacobian lies along
// e_fold = edge about which to fold; the corresponding ghost zone must be
//	    symmetry ghost zone, and at present we only support the case
//	    where this is a "local" (mirror-image) symmetry ghost zone
// iperp = iperp-wrt-e_Jac coordinate of Jacobian
// posn = ipar-wrt-e_Jac coordinate of Jacobian molecule reference point
// [min,max]_m = range of ipar-wrt-e_Jac molecule m in Jacobian
// [min,max]_fold_m = range of ipar-wrt-e_Jac molecule m which is to folded;
//		      this must be a subrange of [min,max]_m
//
void patch_system::fold_Jacobian(const patch_edge& e_Jac,
				 const patch_edge& e_fold,
				 int iperp,
				 int posn, int min_m, int max_m,
				 int min_fold_m, int max_fold_m,
				 jtutil::array1d<fp>& Jacobian_buffer)
	const
{
// check that [min,max]_fold_m is a subrange of [min,max]_m
assert( min_fold_m >= min_m );
assert( min_fold_m <= max_m );
assert( max_fold_m >= min_m );
assert( max_fold_m <= max_m );

const patch& p = e_fold.my_patch();
assert( e_Jac.my_patch() == p );

const symmetry_ghost_zone& sgz_fold = p.ghost_zone_on_edge(e_fold)
				       .cast_to_symmetry_ghost_zone();

//
// At present we only handle the case show in the comments above,
// where sgz_fold is a local (mirror-image) symmetry, i.e. where
// y is guaranteed to be within the molecule abcde.
//
if (sgz_fold.other_edge() != e_fold)
   then error_exit(ERROR_EXIT,
"***** patch_system::fold_Jacobian()\n"
"        implementation restriction: at present we only handle folding\n"
"        via \"local\" (mirror-image) symmetries!\n"
"        p=\"%s\" e_Jac=\"%s\" e_fold=\"%s\"\n"
"        but sgz_fold.other_edge()=\"%s\" != e_fold\n"
,
		   p.name(), e_Jac.name(), e_fold.name(),
		   sgz_fold.other_edge().name());		/*NOTREACHED*/

	for (int xm = min_fold_m ; xm <= max_fold_m ; ++xm)
	{
	const int x_Jac_ipar = posn + xm;	// x ipar wrt e_Jac
	const int x_fold_iperp = x_Jac_ipar;	// ... == iperp wrt e_fold

	const int y_fold_iperp = sgz_fold.iperp_map_of_iperp(x_fold_iperp);
						// y iperp wrt e_fold
	const int y_Jac_ipar = y_fold_iperp;	// ... == ipar wrt e_Jac
	const int ym = y_Jac_ipar - posn;

	// check that y is indeed within the molecule
	assert( ym >= min_m );
	assert( ym <= max_m );

	// actually "fold" the molecule
	Jacobian_buffer(ym) += Jacobian_buffer(xm);
	}
}

//******************************************************************************

//
// Given that  compute_synchronize_Jacobian()  has been called, this
// function computes a single row of the Jacobian of a given ghost zone,
// *not* taking into account synchronize()'s 3-phase algorithm:
// - It returns the edge to which the y point belongs (the caller can get
//   the patch from this edge).
// - It stores y_iperp and y_posn and min/max ym in the named arguments.
// - It stores the Jacobian elements
//	partial synchronize() gridfn(ghosted_gfn, px, x_iperp, x_ipar)
//	-------------------------------------------------------------
//	     partial gridfn(ghosted_gfn, py, y_iperp, y_posn+ym)
//   in the caller-supplied buffer
//	Jacobian_buffer(ym)
//   for each  ym  in the min/max ym range
//
const patch_edge&
  patch_system::ghost_zone_Jacobian(const ghost_zone& xgz,
				    int x_iperp, int x_ipar,
				    int& y_iperp,
				    int& y_posn, int& min_ym, int& max_ym,
				    jtutil::array1d<fp>& Jacobian_buffer)
	const
{
y_iperp = xgz.Jacobian_y_iperp(x_iperp);

y_posn = xgz.Jacobian_y_ipar_posn(x_iperp, x_ipar);
min_ym = xgz.Jacobian_min_y_ipar_m();
max_ym = xgz.Jacobian_max_y_ipar_m();

	for (int ym = min_ym ; ym <= max_ym ; ++ym)
	{
	Jacobian_buffer(ym) = xgz.Jacobian(x_iperp, x_ipar, ym);
	}

return xgz.other_edge();
}

//******************************************************************************
//******************************************************************************
//******************************************************************************

	  }	// namespace AHFinderDirect

```

patch/coord_derivs.out:
```
    |\^/|     Maple 7 (IBM INTEL LINUX)
._|\|   |/|_. Copyright (c) 2001 by Waterloo Maple Inc.
 \  MAPLE  /  All rights reserved. Maple is a registered trademark of
 <____ ____>  Waterloo Maple Inc.
      |       Type ? for help.
# coord_derivs.maple -- compute derivatives of (mu,nu,phi) wrt (x,y,z)
# $Id: coord_derivs.maple,v 1.1 2002/04/17 14:34:16 jthorn Exp $
> 
> mu  := arctan(y/z);
                               mu := arctan(y/z)

> nu  := arctan(x/z);
                               nu := arctan(x/z)

> phi := arctan(y/x);
                               phi := arctan(y/x)

> 
########################################
> 
> simplify(diff(mu,x));
                                       0

> simplify(diff(mu,y));
                                       z
                                    -------
                                     2    2
                                    z  + y

> simplify(diff(mu,z));
                                        y
                                   - -------
                                      2    2
                                     z  + y

> 
> simplify(diff(nu,x));
                                       z
                                    -------
                                     2    2
                                    z  + x

> simplify(diff(nu,y));
                                       0

> simplify(diff(nu,z));
                                        x
                                   - -------
                                      2    2
                                     z  + x

> 
> simplify(diff(phi,x));
                                        y
                                   - -------
                                      2    2
                                     x  + y

> simplify(diff(phi,y));
                                       x
                                    -------
                                     2    2
                                    x  + y

> simplify(diff(phi,z));
                                       0

> 
########################################
> 
> simplify(diff(mu,x,x));
                                       0

> simplify(diff(mu,x,y));
                                       0

> simplify(diff(mu,x,z));
                                       0

> simplify(diff(mu,y,y));
                                       z y
                                 -2 ----------
                                      2    2 2
                                    (z  + y )

> simplify(diff(mu,y,z));
                                      2    2
                                    -z  + y
                                   ----------
                                     2    2 2
                                   (z  + y )

> simplify(diff(mu,z,z));
                                       z y
                                  2 ----------
                                      2    2 2
                                    (z  + y )

> 
> simplify(diff(nu,x,x));
                                       z x
                                 -2 ----------
                                      2    2 2
                                    (z  + x )

> simplify(diff(nu,x,y));
                                       0

> simplify(diff(nu,x,z));
                                      2    2
                                     z  - x
                                  - ----------
                                      2    2 2
                                    (z  + x )

> simplify(diff(nu,y,y));
                                       0

> simplify(diff(nu,y,z));
                                       0

> simplify(diff(nu,z,z));
                                       z x
                                  2 ----------
                                      2    2 2
                                    (z  + x )

> 
> simplify(diff(phi,x,x));
                                       y x
                                  2 ----------
                                      2    2 2
                                    (x  + y )

> simplify(diff(phi,x,y));
                                      2    2
                                    -x  + y
                                   ----------
                                     2    2 2
                                   (x  + y )

> simplify(diff(phi,x,z));
                                       0

> simplify(diff(phi,y,y));
                                       y x
                                 -2 ----------
                                      2    2 2
                                    (x  + y )

> simplify(diff(phi,y,z));
                                       0

> simplify(diff(phi,z,z));
                                       0

> quit
bytes used=444120, alloc=458668, time=0.04

```

cctest/README:
```
This directory contains subdirectories with test programs for
various C++ compiler/library features:

namespace/	This directory tests how namespaces are handled for
		C standard library functions, eg <stdio.h> vs <cstdio>

vector/		This directory test how namespaces are handled for
		the STL vector class, eg <vector.h> vs <cvector>

inline-fns/	This directory tests how inline functions are handled,
		both within C++ classes and "globally"

```

cctest/inline-fns/mini-util.hh:
```
// mini-util.hh -- cut-down version of src/jtutil/util.hh for testing inline fns
// $Header$

namespace jtutil
          {
// how many integers are in the closed interval [low,high]
inline int how_many_in_range(int low, int high) { return high - low + 1; }
          }

```

cctest/inline-fns/Makefile:
```
# Makefile to test various C/C++ compiler features
# $Header: /usr/local/svn/cvs-repositories/numrelcvs/AEIThorns/AHFinderDirect/src/cctest/inline-fns/Makefile,v 1.1 2003-07-31 17:29:48 jthorn Exp $
#
# Targets:
#	default		inline-fn inline-class
#	inline-fn	test non-class inline function in .cc file
#	inline-fn2	test non-class inline function in header file
#	inline-class	test class with inline function
#	clean		remove all binaries
#
# Arguments:
# CXX		how to invoke the C++ compiler, including any desired flags
#

# default setting, may be overridden from command line
CXX := g++ -W -Wall -pedantic -ansi

.PHONY	: default
default	: inline-fn inline-fn2 inline-class

.PHONY	: inline-fn
inline-fn:
	$(CXX) -o inline-fn inline-fn.cc

.PHONY	: inline-fn2
inline-fn2:
	$(CXX) -o inline-fn2 inline-fn2.cc

.PHONY	: inline-class
inline-class:
	$(CXX) -o inline-class inline-class.cc

.PHONY	: clean
clean	:
	-rm inline-fn inline-fn2 inline-class

```

cctest/inline-fns/README:
```
$Header$

This directory contains test programs to see how (or if!) compilers
handle inline functions.

inline-fn.cc	Tests a trivial non-class inline function
		defined in the .cc file
inline-fn2.cc	Tests a trivial non-class inline function
		defined in a header file
inline-class.cc	Tests a trivial class containing an inline function
		(and an out-of-line constructor)

```

cctest/inline-fns/inline-class.cc:
```
// inline-class.cc -- test simple class with inline function
// $Header$
#include <stdio.h>

class	addx
	{
public:
	int operator()(int i) { return i+x_; }
	addx(int x);
private:
	int x_;
	};

addx::addx(int x)
	: x_(x)
{ }

int main(void)
{
addx add3(3);
printf("add3(42) = %d\n", add3(42));
return 0;
}

```

cctest/inline-fns/inline-fn.cc:
```
// inline-fn.cc -- test non-class inline function
// $Header$
#include <stdio.h>

inline int add3(int x) { return x+3; }

int main(void)
{
printf("add3(42) = %d\n", add3(42));
return 0;
}

```

cctest/inline-fns/inline-fn2.cc:
```
// inline-fn2.cc -- test non-class inline function in header file
// $Header$
#include <stdio.h>
#include "mini-util.hh"

int main(void)
{
printf("jtutil::how_many_in_range(69,105) = %d\n",
       jtutil::how_many_in_range(69,105));
return 0;
}

```

cctest/namespace/stdio_h-std--using-std-printf.cc:
```
// $Header$

#include <stdio.h>
using std::printf;

int main()
{
printf("testing <stdio.h> functions in std:: namespace:\n");
printf("==> #include <stdio.h>; using std::printf; printf() is ok\n");
return 0;
}

```

cctest/namespace/Makefile:
```
# Makefile to test various C/C++ compiler features
#
# Targets:
#	default		test namespace handling for <stdio.h> and <cstdio>
#	stdio_h		test namespace handling for <stdio.h>
#	cstdio		test namespace handling for <cstdio>
#	clean		remove all binaries
#
# Arguments:
# CXX		how to invoke the C++ compiler, including any desired flags
#

# default setting, may be overridden from command line
CXX := g++ -W -Wall -pedantic -ansi

.PHONY	: default
default	: stdio_h cstdio
	
.PHONY	: stdio_h
stdio_h	:
	-$(CXX) -o stdio_h-global \
                   stdio_h-global.cc
	        -./stdio_h-global
	-$(CXX) -o stdio_h-std--std-printf \
                   stdio_h-std--std-printf.cc
	        -./stdio_h-std--std-printf
	-$(CXX) -o stdio_h-std--using-std-printf \
                   stdio_h-std--using-std-printf.cc
	        -./stdio_h-std--using-std-printf
	-$(CXX) -o stdio_h-std--using-namespace-std \
                   stdio_h-std--using-namespace-std.cc
	        -./stdio_h-std--using-namespace-std

.PHONY	: cstdio
cstdio	:
	-$(CXX) -o cstdio-global \
                   cstdio-global.cc
	        -./cstdio-global
	-$(CXX) -o cstdio-std--std-printf \
                   cstdio-std--std-printf.cc
	        -./cstdio-std--std-printf
	-$(CXX) -o cstdio-std--using-std-printf \
                   cstdio-std--using-std-printf.cc
	        -./cstdio-std--using-std-printf
	-$(CXX) -o cstdio-std--using-namespace-std \
                   cstdio-std--using-namespace-std.cc
	        -./cstdio-std--using-namespace-std

.PHONY	: clean
clean	:
	-rm -f stdio_h-global \
               stdio_h-std--std-printf \
               stdio_h-std--using-std-printf \
               stdio_h-std--using-namespace-std
	-rm -f cstdio-global \
               cstdio-std--std-printf \
               cstdio-std--using-std-printf \
               cstdio-std--using-namespace-std

```

cctest/namespace/cstdio-std--std-printf.cc:
```
// $Header$

#include <cstdio>

int main()
{
std::printf("testing <cstdio> functions in std:: namespace:\n");
std::printf("==> #include <cstdio>; std::printf() is ok\n");
return 0;
}

```

cctest/namespace/stdio_h-std--std-printf.cc:
```
// $Header$

#include <stdio.h>

int main()
{
std::printf("testing <stdio.h> functions in std:: namespace:\n");
std::printf("==> #include <stdio.h>; std::printf() is ok\n");
return 0;
}

```

cctest/namespace/status:
```
osmium.aei.mpg.de	# OpenBSD 3.1
gcc version 2.95.3 20010125 (prerelease)
Sun Sep 29 11:32:19 EDT 2002
CXX='/usr/bin/g++ -W -Wall -ansi -pedantic'
==> #include <stdio.h>; printf() is ok
==> #include <stdio.h>; std::printf() is ok
==> #include <stdio.h>; using std::printf; printf() is ok
==> #include <stdio.h>; using namespace std; printf() is ok
==> #include <cstdio>; printf() is ok (THIS SHOULD FAIL)
==> #include <cstdio>; std::printf() is ok
==> #include <cstdio>; using std::printf; printf() is ok
==> #include <cstdio>; using namespace std; printf() is ok

xeon21.aei.mpg.de
gcc version 2.96 20000731 (Red Hat Linux 7.1 2.96-98)
Sun Sep 29 18:24:39 CEST 2002
CXX='/usr/bin/g++ -W -Wall -pedantic -ansi'
==> #include <stdio.h>; printf() is ok
==> #include <stdio.h>; std::printf() is ok
==> #include <stdio.h>; using std::printf; printf() is ok
==> #include <stdio.h>; using namespace std; printf() is ok
==> #include <cstdio>; printf() is ok (THIS SHOULD FAIL)
==> #include <cstdio>; std::printf() is ok
==> #include <cstdio>; using std::printf; printf() is ok
==> #include <cstdio>; using namespace std; printf() is ok

xeon21.aei.mpg.de
gcc version 3.2		# built by JT from release tarball
Sun Sep 29 18:27:28 CEST 2002
CXX='/scratch/jthorn/local/bin/g++ -Wall -pedantic -ansi'
==> #include <stdio.h>; printf() is ok
==> #include <stdio.h>; using namespace std; printf() is ok
==> #include <cstdio>; printf() is ok (THIS SHOULD FAIL)
==> #include <cstdio>; std::printf() is ok
==> #include <cstdio>; using std::printf; printf() is ok
==> #include <cstdio>; using namespace std; printf() is ok

xeon21.aei.mpg.de
Intel(R) C++ Compiler for 32-bit applications, Version 5.0.1   Build 010730D0
Sun Sep 29 18:28:37 CEST 2002
CXX='icc -Xc'
==> #include <stdio.h>; printf() is ok
==> #include <stdio.h>; using namespace std; printf() is ok
==> #include <cstdio>; printf() is ok (THIS SHOULD FAIL)
==> #include <cstdio>; std::printf() is ok
==> #include <cstdio>; using std::printf; printf() is ok
==> #include <cstdio>; using namespace std; printf() is ok

xeon21.aei.mpg.de
Intel(R) C++ Compiler for 32-bit applications, Version 5.0.1   Build 010730D0
Sun Sep 29 18:29:44 CEST 2002
CXX='icc -ansi -Xc'
==> #include <stdio.h>; printf() is ok
==> #include <stdio.h>; using namespace std; printf() is ok
==> #include <cstdio>; printf() is ok (THIS SHOULD FAIL)
==> #include <cstdio>; std::printf() is ok
==> #include <cstdio>; using std::printf; printf() is ok
==> #include <cstdio>; using namespace std; printf() is ok

Peter Diener's Laptop using icc version 6.0
options: -ansi -Xc
==> #include <stdio.h>; printf() is ok
==> #include <stdio.h>; using namespace std; printf() is ok
==> #include <cstdio>; printf() is ok (THIS SHOULD FAIL)
==> #include <cstdio>; std::printf() is ok
==> #include <cstdio>; using std::printf; printf() is ok
==> #include <cstdio>; using namespace std; printf() is ok

origin.aei.mpg.de
MIPSpro Compilers: Version 7.3.1.2m
Mon Oct  7 20:31:11 CEST 2002
CXX='CC -LANG:std'
==> #include <stdio.h>; printf() is ok
all others fail

Hitachi using sCC (native)
options:  -64 -stl -O3 -pvec -predicate -model=F1
also identical leaving out -64 (compare with ../vector/ results!)
==> #include <stdio.h>; printf() is ok
==> #include <stdio.h>; using namespace std; printf() is ok
==> #include <cstdio>; printf() is ok (THIS SHOULD FAIL)
==> #include <cstdio>; std::printf() is ok
==> #include <cstdio>; using namespace std; printf() is ok

Loslobos using pgCC version 3.2-3
options: -tused -O2
==> #include <stdio.h>; printf() is ok

```

cctest/namespace/README:
```
$Header$

This directory contains a set of "hello, world" programs to test how
the C++ namespaces are handled for standard library functions.

Each .cc file is a "hello, world\n" program using a particular set of
options for what header file to #include, what (if any) using declarations
or definitions to use, and whether or not to use an explicit std::
qualifier when calling printf().

According to standard C++, #include <cstdio> puts names *only* in the
std:: namespace, so the cstdio-global.cc program should fail to compile.
All the other programs should be legal in standard C++.

					what to		what namespace		how we call	status
					#include?	are we using		printf()?	in
							printf() in?				standard
								using?				C++

stdio_h-global.cc			<stdio.h>	global	--		printf()	ok
stdio_h-std:std-printf.cc		<stdio.h>	std	--		std::printf()	ok
stdio_h-std:using-std-printf.cc		<stdio.h>	std	std::printf;	printf()	ok
stdio_h-std:using-namespace-std.cc	<stdio.h>	std	namespace std;	printf()	ok

cstdio-global.cc			<cstdio>	global	--		printf()	WRONG
cstdio-std:std-printf.cc		<cstdio>	std	--		std::printf()	ok
cstdio-std:using-std-printf.cc		<cstdio>	std	std::printf;	printf()	ok
cstdio-std:using-namespace-std.cc	<cstdio>	std	namespace std;	printf()	ok

```

cctest/namespace/cstdio-std--using-std-printf.cc:
```
// $Header$

#include <cstdio>
using std::printf;

int main()
{
printf("testing <cstdio> functions in std:: namespace:\n");
printf("==> #include <cstdio>; using std::printf; printf() is ok\n");
return 0;
}

```

cctest/namespace/stdio_h-global.cc:
```
// $Header$

#include <stdio.h>

int main()
{
printf("testing <stdio.h> functions in global namespace:\n");
printf("==> #include <stdio.h>; printf() is ok\n");
return 0;
}

```

cctest/namespace/stdio_h-std--using-namespace-std.cc:
```
// $Header$

#include <stdio.h>
using namespace std;

int main()
{
printf("testing <stdio.h> functions in std:: namespace:\n");
printf("==> #include <stdio.h>; using namespace std; printf() is ok\n");
return 0;
}

```

cctest/namespace/cstdio-std--using-namespace-std.cc:
```
// $Header$

#include <cstdio>
using namespace std;

int main()
{
printf("testing <cstdio> functions in std:: namespace:\n");
printf("==> #include <cstdio>; using namespace std; printf() is ok\n");
return 0;
}

```

cctest/namespace/cstdio-global.cc:
```
// $Header$

#include <cstdio>

int main()
{
printf("testing <cstdio> functions in global namespace (THIS SHOULD FAIL):\n");
printf("==> #include <cstdio>; printf() is ok (THIS SHOULD FAIL)\n");
return 0;
}

```

cctest/vector/Makefile:
```
# Makefile to test various C/C++ compiler features
#
# Targets:
#	default		test namespace handling for <vector.h> and <vector>
#	vector_h	test namespace handling for <vector.h>
#	vector		test namespace handling for <vector>
#	clean		remove all binaries
#
# Arguments:
# CXX		how to invoke the C++ compiler, including any desired flags
#

# default setting, may be overridden from command line
CXX := g++ -W -Wall -pedantic -ansi

.PHONY	: default
default	: vector_h vector
	
.PHONY	: vector_h
vector_h	:
	-$(CXX) -o vector_h-global \
                   vector_h-global.cc
	        -./vector_h-global
	-$(CXX) -o vector_h-std--std-vector \
                   vector_h-std--std-vector.cc
	        -./vector_h-std--std-vector
	-$(CXX) -o vector_h-std--using-std-vector \
                   vector_h-std--using-std-vector.cc
	        -./vector_h-std--using-std-vector
	-$(CXX) -o vector_h-std--using-namespace-std \
                   vector_h-std--using-namespace-std.cc
	        -./vector_h-std--using-namespace-std

.PHONY	: vector
vector	:
	-$(CXX) -o vector-global \
                   vector-global.cc
	        -./vector-global
	-$(CXX) -o vector-std--std-vector \
                   vector-std--std-vector.cc
	        -./vector-std--std-vector
	-$(CXX) -o vector-std--using-std-vector \
                   vector-std--using-std-vector.cc
	        -./vector-std--using-std-vector
	-$(CXX) -o vector-std--using-namespace-std \
                   vector-std--using-namespace-std.cc
	        -./vector-std--using-namespace-std

.PHONY	: clean
clean	:
	-rm -f vector_h-global \
               vector_h-std--std-vector \
               vector_h-std--using-std-vector \
               vector_h-std--using-namespace-std
	-rm -f vector-global \
               vector-std--std-vector \
               vector-std--using-std-vector \
               vector-std--using-namespace-std

```

cctest/vector/vector-std--std-vector.cc:
```
// $Header$

#include <stdio.h>
#include <vector>

int main()
{
printf("testing <vector> functions in std:: namespace:\n");
std::vector<int> v(3);
v[0] = 42;
v[1] = 69;
v[2] = 105;
printf("%d %d %d should be 42 69 105... ", v[0], v[1], v[2]);
printf(((v[0] == 42) && (v[1] == 69) && (v[2] == 105)) ? "ok\n" : "FAIL\n");
printf("==> #include <vector>; std::vector is ok\n");
return 0;
}

```

cctest/vector/status:
```
osmium.aei.mpg.de	# OpenBSD 3.1
gcc version 2.95.3 20010125 (prerelease)
Sun Oct  6 21:15:55 EDT 2002
CXX='/usr/bin/g++ -W -Wall -ansi -pedantic'
==> #include <vector.h>; vector is ok
==> #include <vector.h>; std::vector is ok
==> #include <vector.h>; using std::vector; vector is ok
==> #include <vector.h>; using namespace std; vector is ok
==> #include <vector>; vector is ok (THIS SHOULD FAIL)
==> #include <vector>; std::vector is ok
==> #include <vector>; using std::vector; vector is ok
==> #include <vector>; using namespace std; vector is ok

xeon21.aei.mpg.de
gcc version 2.96 20000731 (Red Hat Linux 7.1 2.96-98)
Mon Oct  7 18:38:20 CEST 2002
CXX='/usr/bin/g++ -W -Wall -pedantic -ansi'
==> #include <vector.h>; vector is ok
==> #include <vector.h>; std::vector is ok
==> #include <vector.h>; using std::vector; vector is ok
==> #include <vector.h>; using namespace std; vector is ok
==> #include <vector>; vector is ok (THIS SHOULD FAIL)
==> #include <vector>; std::vector is ok
==> #include <vector>; using std::vector; vector is ok
==> #include <vector>; using namespace std; vector is ok

xeon21.aei.mpg.de
g++ (GCC) 3.2
Mon Oct  7 18:34:53 CEST 2002
CXX='/scratch/jthorn/local/bin/g++ -W -Wall -ansi -pedantic'
==> #include <vector.h>; vector is ok
==> #include <vector.h>; std::vector is ok
==> #include <vector.h>; using std::vector; vector is ok
==> #include <vector.h>; using namespace std; vector is ok
==> #include <vector>; std::vector is ok
==> #include <vector>; using std::vector; vector is ok
==> #include <vector>; using namespace std; vector is ok

xeon21.aei.mpg.de
Intel(R) C++ Compiler for 32-bit applications, Version 5.0.1   Build 010730D0
Mon Oct  7 18:36:42 CEST 2002
CXX='icc -Xc -ansi'
==> #include <vector>; std::vector is ok
==> #include <vector>; using std::vector; vector is ok
==> #include <vector>; using namespace std; vector is ok

origin.aei.mpg.de
MIPSpro Compilers: Version 7.3.1.2m
Mon Oct  7 20:31:11 CEST 2002
CXX='CC -LANG:std'
==> #include <vector.h>; vector is ok
all others fail

Hitachi using sCC (native)
options:  -64 -stl -O3 -pvec -predicate -model=F1
all tests fail (internal compiler error)

Hitachi using sCC (native)
options:  -stl -O3 -pvec -predicate -model=F1
==> #include <vector>; std::vector is ok
==> #include <vector>; using std::vector; vector is ok
==> #include <vector>; using namespace std; vector is ok

Loslobos using pgCC version 3.2-3
options: -tused -O2
==> #include <vector>; vector is ok (THIS SHOULD FAIL)
==> #include <vector>; std::vector is ok
==> #include <vector>; using std::vector; vector is ok
==> #include <vector>; using namespace std; vector is ok

```

cctest/vector/README:
```
$Header$

This directory contains a set of "hello, world" programs to test how
C++ namespaces are handled for the STL vector class.

Each .cc file is a "hello, world\n" program using a particular set of
options for what header file to #include, what (if any) using declarations
or definitions to use, and whether or not to use an explicit std::
qualifier when referring to the vector class.

According to standard C++, #include <vector> puts names *only* in the
std:: namespace, so the vector-global.cc program should fail to compile.
All the other programs should be legal in standard C++.

					what to		what namespace		how we call	status
					#include?	are we using		vector()?	in
							vector() in?				standard
								using?				C++

vector_h-global.cc			<vector.h>	global	--		vector()	ok
vector_h-std:std-vector.cc		<vector.h>	std	--		std::vector()	ok
vector_h-std:using-std-vector.cc	<vector.h>	std	std::vector;	vector()	ok
vector_h-std:using-namespace-std.cc	<vector.h>	std	namespace std;	vector()	ok

vector-global.cc			<vector>	global	--		vector()	WRONG
vector-std:std-vector.cc		<vector>	std	--		std::vector()	ok
vector-std:using-std-vector.cc		<vector>	std	std::vector;	vector()	ok
vector-std:using-namespace-std.cc	<vector>	std	namespace std;	vector()	ok

```

cctest/vector/vector-std--using-std-vector.cc:
```
// $Header$

#include <stdio.h>
#include <vector>
using std::vector;

int main()
{
printf("testing <vector> functions in std:: namespace:\n");
vector<int> v(3);
v[0] = 42;
v[1] = 69;
v[2] = 105;
printf("%d %d %d should be 42 69 105... ", v[0], v[1], v[2]);
printf(((v[0] == 42) && (v[1] == 69) && (v[2] == 105)) ? "ok\n" : "FAIL\n");
printf("==> #include <vector>; using std::vector; vector is ok\n");
return 0;
}

```

cctest/vector/vector_h-std--std-vector.cc:
```
// $Header$

#include <stdio.h>
#include <vector.h>

int main()
{
printf("testing <vector.h> functions in std:: namespace:\n");
std::vector<int> v(3);
v[0] = 42;
v[1] = 69;
v[2] = 105;
printf("%d %d %d should be 42 69 105... ", v[0], v[1], v[2]);
printf(((v[0] == 42) && (v[1] == 69) && (v[2] == 105)) ? "ok\n" : "FAIL\n");
printf("==> #include <vector.h>; std::vector is ok\n");
return 0;
}

```

cctest/vector/vector_h-std--using-namespace-std.cc:
```
// $Header$

#include <stdio.h>
#include <vector.h>
using namespace std;

int main()
{
printf("testing <vector.h> functions in std:: namespace:\n");
vector<int> v(3);
v[0] = 42;
v[1] = 69;
v[2] = 105;
printf("%d %d %d should be 42 69 105... ", v[0], v[1], v[2]);
printf(((v[0] == 42) && (v[1] == 69) && (v[2] == 105)) ? "ok\n" : "FAIL\n");
printf("==> #include <vector.h>; using namespace std; vector is ok\n");
return 0;
}

```

cctest/vector/vector_h-std--using-std-vector.cc:
```
// $Header$

#include <stdio.h>
#include <vector.h>
using std::vector;

int main()
{
printf("testing <vector.h> functions in std:: namespace:\n");
vector<int> v(3);
v[0] = 42;
v[1] = 69;
v[2] = 105;
printf("%d %d %d should be 42 69 105... ", v[0], v[1], v[2]);
printf(((v[0] == 42) && (v[1] == 69) && (v[2] == 105)) ? "ok\n" : "FAIL\n");
printf("==> #include <vector.h>; using std::vector; vector is ok\n");
return 0;
}

```

cctest/vector/vector-std--using-namespace-std.cc:
```
// $Header$

#include <stdio.h>
#include <vector>
using namespace std;

int main()
{
printf("testing <vector> functions in std:: namespace:\n");
vector<int> v(3);
v[0] = 42;
v[1] = 69;
v[2] = 105;
printf("%d %d %d should be 42 69 105... ", v[0], v[1], v[2]);
printf(((v[0] == 42) && (v[1] == 69) && (v[2] == 105)) ? "ok\n" : "FAIL\n");
printf("==> #include <vector>; using namespace std; vector is ok\n");
return 0;
}

```

cctest/vector/vector_h-global.cc:
```
// $Header$

#include <stdio.h>
#include <vector.h>

int main()
{
printf("testing <vector.h> functions in global namespace:\n");
vector<int> v(3);
v[0] = 42;
v[1] = 69;
v[2] = 105;
printf("%d %d %d should be 42 69 105... ", v[0], v[1], v[2]);
printf(((v[0] == 42) && (v[1] == 69) && (v[2] == 105)) ? "ok\n" : "FAIL\n");
printf("==> #include <vector.h>; vector is ok\n");
return 0;
}

```

cctest/vector/vector-global.cc:
```
// $Header$

#include <stdio.h>
#include <vector>

int main()
{
printf("testing <vector> functions in global namespace (THIS SHOULD FAIL):\n");
vector<int> v(3);
v[0] = 42;
v[1] = 69;
v[2] = 105;
printf("%d %d %d should be 42 69 105... ", v[0], v[1], v[2]);
printf(((v[0] == 42) && (v[1] == 69) && (v[2] == 105)) ? "ok\n" : "FAIL\n");
printf("==> #include <vector>; vector is ok (THIS SHOULD FAIL)\n");
return 0;
}

```

sparse-matrix/README:
```
This directory contains sparse matrix routines imported from elsewhere.
See the individual files or subdirectories for details on authorship,
copyright, etc.

ilucg
	ILUCG (Incomplete LU-decomposition / conjugate gradiant)
	subroutines.

umfpack
	UMFPACK sparse LU decomposition package (version 4.0 of 11.Apr.2002).

y12m
	Sparse LU decomposition subroutines from the Harwell Y12M package.
	I haven't written the AHFinderDirect support code for this, so
	the Y12M code isn't compiled yet.

```

sparse-matrix/ilucg/ilucg.h:
```
/* ilucg.h -- C/C++ prototypes for [sd]ilucg() routines */
/* $Header$ */

/*
 * prerequisites:
 *	"cctk.h"
 *	"config.h"	// for "integer" = Fortran integer
 */

#ifdef __cplusplus
extern "C"
       {
#endif

/*
 * ***** ILUCG *****
 */
void CCTK_FCALL
  CCTK_FNAME(silucg)(const integer* N,
		     const integer IA[], const integer JA[], const float A[],
		     const float B[], float X[],
		     integer ITEMP[], float RTEMP[],
		     const float* EPS, const integer* ITER,
		     integer* ISTATUS);
void CCTK_FCALL
  CCTK_FNAME(dilucg)(const integer* N,
		     const integer IA[], const integer JA[], const double A[],
		     const double B[], double X[],
		     integer ITEMP[], double RTEMP[],
		     const double* EPS, const integer* ITER,
		     integer* ISTATUS);

#ifdef __cplusplus
       }	/* extern "C" */
#endif

```

sparse-matrix/ilucg/silucg.f77:
```
      SUBROUTINE SILUCG(N,
     $                  IA,JA,A,
     $                  B,X,
     $                  ITEMP,RTEMP,
     $                  EPS,ITER,
     $                  ISTATUS,IERROR)
      IMPLICIT REAL (A-H,O-Z)
      DIMENSION IA(*),JA(*),A(*),B(*),X(*),ITEMP(*),RTEMP(*)
C
C     INCOMPLETE LU DECOMPOSITION-CONJUGATE GRADIENT
C     -          --               -         -
C WHERE:
C     |N| IS THE NUMBER OF EQUATIONS.  IF N < 0, ITEMP AND
C       RTEMP CONTAIN THE ILU FROM A PREVIOUS CALL AND
C       B AND X ARE THE NEW RHS AND INITIAL GUESS.
C     IA IS AN INTEGER ARRAY DIMENSIONED |N|+1.  IA(I) IS THE
C       INDEX INTO ARRAYS JA AND A OF THE FIRST NON-ZERO
C       ELEMENT IN ROW I.  LET MAX=IA(|N|+1)-IA(1).
C     JA IS AN INTEGER ARRAY DIMENSIONED MAX.  JA(K) GIVES
C       THE COLUMN NUMBER OF A(K).
C     A IS A REAL ARRAY DIMENSIONED MAX.  IT CONTAINS THE
C       NONZERO ELEMENTS OF THE MATRIX STORED BY ROW.
C     B CONTAINS THE RHS VECTOR.
C     X IS A REAL ARRAY DIMENSIONED |N|.  ON ENTRY, IT CONTAINS
C       AN INITIAL ESTIMATE; ON EXIT, THE SOLUTION.
C     ITEMP IS AN INTEGER SCRATCH ARRAY DIMENSIONED 3*(|N|+MAX)+2.
C     RTEMP IS AN REAL SCRATCH ARRAY DIMENSIONED 4*|N|+MAX.
C     EPS IS THE CONVERGENCE CRITERIA.  IT SPECIFIES THE RELATIVE
C       ERROR ALLOWED IN THE SOLUTION.  TO BE PRECISE, CONVERGENCE
C       IS DEEMED TO HAVE OCCURED WHEN THE INFINITY-NORM OF THE
C       CHANGE IN THE SOLUTION IN ONE ITERATION IS .LE. EPS * THE
C       INFINITY-NORM OF THE CURRENT SOLUTION.  HOWEVER, IF EPS
C       .LT. 0.0, IT IS INTERNALLY SCALED BY THE MACHINE PRECISION,
C       SO THAT, FOR EXAMPLE, EPS = -256.0 WILL ALLOW THE LAST 8 BITS
C       OF THE SOLUTION TO BE IN ERROR.       
C     MAXITER GIVES THE REQUESTED NUMBER OF ITERATIONS,
C       OR IS 0 FOR "NO LIMIT".
C     ISTATUS IS AN INTEGER VARIABLE, WHICH IS SET TO:
C       -I IF THERE IS AN ERROR IN THE MATRIX STRUCTURE IN ROW I
C          (SUCH AS A ZERO ELEMENT ON THE DIAGONAL).
C        0 IF THE ITERATION FAILED TO REACH THE CONVERGENCE CRITERION
C          IN ITER ITERATIONS.
C       +I IF THE ITERATION CONVERGED IN I ITERATIONS.
C
C MODIFICATIONS
C * RETURN LOGICAL VALUE, J. THORNBURG, 13/MAY/85.
C * ADD CONVERGENCE CRITERIA, J. THORNBURG, 17/MAY/85.
C * CHANGED CONVERGENCE CRITERIA, J. THORNBURG, 4/AUG/89.
C * AVOID SKIP RETURNS, J. THORNBURG, 10/SEP/89.
C * CHANGE FUNCTION BACK TO SUBROUTINE, J. THORNBURG, 3.JUN.2003
C * CHANGE CONVERGENCE CRITERIA TO AVOID SPURIOUSLY STOPPING TOO SOON
C   ON SMALL RHS VECTORS -- J. THORNBURG, 11.JUN.2003
C
C REFERENCE:
C     D.S.KERSHAW,"THE INCOMPLETE CHOLESKY-CONJUGATE GRADIENT
C       METHOD FOR INTERATIVE SOLUTION OF LINEAR EQUATIONS",
C       J.COMPUT.PHYS. JAN 1978 PP 43-65
C
      LOGICAL SLU0
      NP=IABS(N)
      ISTATUS=0
      IF (NP.EQ.0) GO TO 20
C CALCULATE INDICES FOR BREAKING UP TEMPORARY ARRAYS.
      N1=NP+1
      MAX=IA(N1)-IA(1)
      ILU=1
      JLU=ILU+N1
      ID=JLU+MAX
      IC=ID+NP
      JC=IC+N1
      JCI=JC+MAX
      IR=1
      IP=IR+NP
      IS1=IP+NP
      IS2=IS1+NP
      IALU=IS2+NP
      IF (N.LT.0) GO TO 10
C DO INCOMPLETE LU DECOMPOSITION
      IF (SLU0(NP,IA,JA,A,ITEMP(IC),ITEMP(JC),ITEMP(JCI),RTEMP(IALU),
     *    ITEMP(ILU),ITEMP(JLU),ITEMP(ID),RTEMP(IR),IERROR)) GOTO 20
C AND DO CONJUGATE GRADIENT ITERATIONS
C CALL MODIFIED TO ADD ADJUSTABLE CONVERGENCE CRITERIA EPS
C - J. THORNBURG, 17/MAY/85.
10    CALL SNCG0(NP,IA,JA,A,B,X,ITEMP(ILU),ITEMP(JLU),ITEMP(ID),
     *  RTEMP(IALU),RTEMP(IR),RTEMP(IP),RTEMP(IS1),RTEMP(IS2),
     *  EPS,MAXITER,ITER)
C ITER IS ACTUAL NUMBER OF ITERATIONS (NEGATIVE IF NO CONVERGENCE)
      ISTATUS = ITER
      IF (ITER .LT. 0) ISTATUS = 0
      RETURN
C ERROR RETURN FROM INCOMPLETE LU DECOMPOSITION
20    ISTATUS = -IERROR
      RETURN
      END

      LOGICAL FUNCTION SLU0(N,IA,JA,A,IC,JC,JCI,ALU,ILU,JLU,ID,V,IE)
      IMPLICIT REAL (A-H,O-Z)
      DIMENSION IA(*),JA(*),A(*),IC(*),JC(*),JCI(*),
     *  ALU(*),ILU(*),JLU(*),ID(N),V(N)
      LOGICAL NODIAG
      COMMON /ICBD00/ ICBAD
C     INCOMPLETE LU DECOMPOSITION
C WHERE:
C     N,IA,JA, AND A ARE DESCRIBED IN FUNCTION SILUCG
C     IC IS AN INTEGER ARRAY DIMENSIONED N+1, IC(J) GIVES THE
C       INDEX OF THE FIRST NONZERO ELEMENT IN COLMN J IN
C       ARRAY JC.
C     JC IS AN INTEGER ARRAY WITH THE SAME DIMENSION AS A.
C       JC(K) GIVES THE ROW NUMBER OF THE K'TH ELEMENT IN
C       THE COLUMN STRUCTURE.
C     JCI IS AN INTEGER ARRAY WITH THE SAME DIMENSION AS A.
C       JCI(K) GIVES THE INDEX INTO ARRAY A OF THE K'TH ELEMENT
C       OF THE COLUMN STRUCTURE.
C     ALU HAS THE SAME DIMENSION AS A.  ON EXIT, IT WILL
C       CONTAIN THE INCOMPLETE LU DECOMPOSITION OF A WITH THE
C       RECIPROCALS OF THE DIAGONAL ELEMENTS OF U.
C     ILU AND JLU CORRESPONDS TO IA AND JA BUT FOR ALU.
C     ID IS AN INTEGER ARRAY DIMENSIONED N.  IT CONTAINS
C       INDICES TO THE DIAGONAL ELEMENTS OF U.
C     V IS A REAL SCRATCH VECTOR OF LENGTH N.
C     IE GIVES THE ROW NUMBER IN ERROR IF AN ERROR OCCURED
C       (RETURN VALUE .TRUE.), OR IS UNUSED IF ALL IS WELL
C       (RETURN VALUE .FALSE.).
C
C     RETURN VALUE = .FALSE. IF ALL IS WELL, .TRUE. IF ERROR.
C
C NOTE: SLU0 SETS ARGUMENTS IC THROUGH V.
C
      ICBAD=0
C ZERO COUNT OF ZERO DIAGONAL ELEMENTS IN U.
C
C FIRST CHECK STRUCTURE OF A AND BUILD COLUMN STRUCTURE
      DO 10 I=1,N
        IC(I)=0
10    CONTINUE
      DO 30 I=1,N
        KS=IA(I)
        KE=IA(I+1)-1
        NODIAG=.TRUE.
        DO 20 K=KS,KE
          J=JA(K)
          IF (J.LT.1.OR.J.GT.N) GO TO 210
          IC(J)=IC(J)+1
          IF (J.EQ.I) NODIAG=.FALSE.
20      CONTINUE
        IF (NODIAG) GO TO 210
30    CONTINUE
C MAKE IC INTO INDICES
      KOLD=IC(1)
      IC(1)=1
      DO 40 I=1,N
        KNEW=IC(I+1)
        IF (KOLD.EQ.0) GO TO 210
        IC(I+1)=IC(I)+KOLD
        KOLD=KNEW
40    CONTINUE
C SET JC AND JCI FOR COLUMN STRUCTURE
      DO 60 I=1,N
        KS=IA(I)
        KE=IA(I+1)-1
        DO 50 K=KS,KE
          J=JA(K)
          L=IC(J)
          IC(J)=L+1
          JC(L)=I
          JCI(L)=K
50      CONTINUE
60    CONTINUE
C FIX UP IC
      KOLD=IC(1)
      IC(1)=1
      DO 70 I=1,N
        KNEW=IC(I+1)
        IC(I+1)=KOLD
        KOLD=KNEW
70    CONTINUE
C FIND SORTED ROW STRUCTURE FROM SORTED COLUMN STRUCTURE
      NP=N+1
      DO 80 I=1,NP
        ILU(I)=IA(I)
80    CONTINUE
C MOVE ELEMENTS, SET JLU AND ID
      DO 100 J=1,N
        KS=IC(J)
        KE=IC(J+1)-1
        DO 90 K=KS,KE
          I=JC(K)
          L=ILU(I)
          ILU(I)=L+1
          JLU(L)=J
          KK=JCI(K)
          ALU(L)=A(KK)
          IF (I.EQ.J) ID(J)=L
90      CONTINUE
100   CONTINUE
C RESET ILU (COULD JUST USE IA)
      DO 110 I=1,NP
        ILU(I)=IA(I)
110   CONTINUE
C FINISHED WITH SORTED COLUMN AND ROW STRUCTURE
C
C DO LU DECOMPOSITION USING GAUSSIAN ELIMINATION
      DO 120 I=1,N
        V(I)=0.0
120   CONTINUE
      DO 200 IROW=1,N
        I=ID(IROW)
        PIVOT=ALU(I)
        IF (PIVOT.NE.0.0) GO TO 140
C THIS CASE MAKES THE ILU LESS ACCURATE
        ICBAD=ICBAD+1
        KS=ILU(IROW)
        KE=ILU(IROW+1)-1
        DO 130 K=KS,KE
          PIVOT=PIVOT+ABS(ALU(K))
130     CONTINUE
        IF (PIVOT.EQ.0.0) GO TO 220
140     PIVOT=1.0/PIVOT
        ALU(I)=PIVOT
        KKS=I+1
        KKE=ILU(IROW+1)-1
        IF (KKS.GT.KKE) GO TO 160
        DO 150 K=KKS,KKE
          J=JLU(K)
          V(J)=ALU(K)
150     CONTINUE
C FIX L IN COLUMN IROW AND DO PARTIAL LU IN SUBMATRIX
160     KS=IC(IROW)
        KE=IC(IROW+1)-1
        DO 190 K=KS,KE
          I=JC(K)
          IF (I.LE.IROW) GO TO 190
          LS=ILU(I)
          LE=ILU(I+1)-1
          DO 180 L=LS,LE
            J=JLU(L)
            IF (J.LT.IROW) GO TO 180
            IF (J.GT.IROW) GO TO 170
            AMULT=ALU(L)*PIVOT
            ALU(L)=AMULT
            IF (AMULT.EQ.0.0) GO TO 190
            GO TO 180
170         IF (V(J).EQ.0.0) GO TO 180
            ALU(L)=ALU(L)-AMULT*V(J)
180       CONTINUE
190     CONTINUE
C RESET V
        IF (KKS.GT.KKE) GO TO 200
        DO 195 K=KKS,KKE
          J=JLU(K)
          V(J)=0.0
195     CONTINUE
200   CONTINUE
C NORMAL RETURN
      SLU0 = .FALSE.
      RETURN
C ERROR RETURNS
210   IE=I
      SLU0 = .TRUE.
      RETURN
C NORMAL RETURN
220   IE=IROW
      SLU0 = .TRUE.
      RETURN
      END

      SUBROUTINE SNCG0(N,IA,JA,A,B,X,ILU,JLU,ID,ALU,R,P,S1,S2,
     *  EPS,ITER,IE)
      IMPLICIT REAL (A-H,O-Z)
      DIMENSION IA(*),JA(*),A(*),B(N),X(N),ILU(*),JLU(*),ALU(*),ID(N),
     *  R(N),P(N),S1(N),S2(N)
C     NONSYMMETRIC CONJUGATE GRADIENT
C WHERE:
C     N,IA,JA,A,B, AND X ARE DESCRIBED IN SUBROUTINE SILUCG.
C     ILU GIVES INDEX OF FIRST NONZERO ELEMENT IN ROW OF LU.
C     JLU GIVES COLUMN NUMBER.
C     ID GIVES INDEX OF DIAGONAL ELEMENT OF U.
C     ALU HAS NONZERO ELEMENTS OF LU MATRIX STORED BY ROW
C       WITH RECIPROCALS OF DIAGONAL ELEMENTS OF U.
C     R,P,S1, AND S2 ARE VECTORS OF LENGTH N USED IN THE
C       ITERATIONS.
C FOLLOWING PARAMETER ADDED BY J. THORNBURG, 17/MAY/85.
C     EPS IS CONVERGENCE CRITERIA.  (DESCRIBED IN SUBROUTINE
C       SILUCG).
C     ITER IS MAX NUMBER OF ITERATIONS, OR 0 FOR "NO LIMIT".
C     IE GIVES ACTUAL NUMBER OF ITERATIONS, NEGATIVE IF
C       NO CONVERGENCE.
C
C R0=B-A*X0
      CALL SMUL10(N,IA,JA,A,X,R)
      DO 10 I=1,N
        R(I)=B(I)-R(I)
10    CONTINUE
C P0=(UT*U)(-1)*AT*(L*LT)(-1)*R0
C FIRST SOLVE L*LT*S1=R0
      CALL SSUBL0(N,ILU,JLU,ID,ALU,R,S1)
C TIMES TRANSPOSE OF A
      CALL SMUL20(N,IA,JA,A,S1,S2)
C THEN SOLVE UT*U*P=S2
      CALL SSUBU0(N,ILU,JLU,ID,ALU,S2,P)
      IE=0
      RDOT = SGVV(R,S1,N)
C LOOP BEGINS HERE
20    CALL SMUL30(N,ILU,JLU,ID,ALU,P,S2)
      PDOT = SGVV(P,S2,N)
C
      IF (PDOT.EQ.0.0) RETURN
C
      ALPHA=RDOT/PDOT
C EQUATION 9PA                      ALPHA=(R,LINV*R)/(P,UT*U*P)
C 1 + ||X CHANGE|| CHANGED TO ||X CHANGE|| -- J. THORNBURG, 11.JUN.2003
      XMAX=0.0
      XDIF=0.0
      DO 30 I=1,N
        AP=ALPHA*P(I)
        X(I)=X(I)+AP
C EQUATION 9PB                      X=X+ALPHA*P
        AP=ABS(AP)
        XX=ABS(X(I))
        IF (AP.GT.XDIF) XDIF=AP
        IF (XX.GT.XMAX) XMAX=XX
30    CONTINUE
      IE=IE+1
C
C CONVERGENCE TEST (CHANGED BY J. THORNBURG, 17/MAY/85, 4/AUG/89)
C
      IF ((EPS .GT. 0.0) .AND. (XDIF .LE. EPS * XMAX)) RETURN
      IF ((EPS .LT. 0.0) .AND. (XMAX + XDIF/ABS(EPS) .EQ. XMAX))
     *   RETURN
C
C EXCEEDED ITERATION LIMIT?
C
      IF ((ITER .NE. 0) .AND. (IE .GE. ITER)) GO TO 60
      CALL SMUL10(N,IA,JA,A,P,S2)
      DO 40 I=1,N
        R(I)=R(I)-ALPHA*S2(I)
C EQUATION 9PC                      R=R-ALPHA*A*P
40    CONTINUE
      CALL SSUBL0(N,ILU,JLU,ID,ALU,R,S1)
C     CALL INPROD(R,S1,EDOT,RRDOT,N)
      RRDOT = SGVV(R,S1,N)
      BETA=RRDOT/RDOT
C EQUATION 9PD                      BETA=(R+,LINV*R+)/(R,LINV*R)
      RDOT=RRDOT
      CALL SMUL20(N,IA,JA,A,S1,S2)
      CALL SSUBU0(N,ILU,JLU,ID,ALU,S2,S1)
      DO 50 I=1,N
        P(I)=S1(I)+BETA*P(I)
C EQUATION 9PE                      P=(UT*U)(-1)*AT*(L*LT)(-1)*R+BETA*P
50    CONTINUE
      GO TO 20
60    IE=-IE
      RETURN
      END

      SUBROUTINE SMUL10(N,IA,JA,A,B,X)
      IMPLICIT REAL (A-H,O-Z)
      DIMENSION IA(*),JA(*),A(*),B(N),X(N)
C MULTIPLY A TIMES B TO GET X
C WHERE:
C     N IS THE ORDER OF THE MATRIX
C     IA GIVES INDEX OF FIRST NONZERO ELEMENT IN ROW
C     JA GIVES COLUMN NUMBER
C     A CONTAINS THE NONZERO ELEMENTS OF THE NONSYMMETRIC
C       MATRIX STORED BY ROW
C     B IS THE VECTOR
C     X IS THE PRODUCT (MUST BE DIFFERENT FROM B)
C
      DO 20 I=1,N
        KS=IA(I)
        KE=IA(I+1)-1
        SUM=0.0
        DO 10 K=KS,KE
          J=JA(K)
          SUM=SUM+A(K)*B(J)
10      CONTINUE
        X(I)=SUM
20    CONTINUE
      RETURN
      END

      SUBROUTINE SMUL20(N,IA,JA,A,B,X)
      IMPLICIT REAL (A-H,O-Z)
      DIMENSION IA(*),JA(*),A(*),B(N),X(N)
C MULTIPLY TRANSPOSE OF A TIMES B TO GET X
C WHERE:
C     N IS THE ORDER OF THE MATRIX
C     IA GIVES INDEX OF FIRST NONZERO ELEMENT IN ROW
C     JA GIVES COLUMN NUMBER
C     A CONTAINS THE NONZERO ELEMENTS OF THE NONSYMMETRIC
C       MATRIX STORED BY ROW
C     B IS THE VECTOR
C     X IS THE PRODUCT (MUST BE DIFFERENT FROM B)
C
      DO 10 I=1,N
        X(I)=0.0
10    CONTINUE
      DO 30 I=1,N
        KS=IA(I)
        KE=IA(I+1)-1
        BB=B(I)
        DO 20 K=KS,KE
          J=JA(K)
          X(J)=X(J)+A(K)*BB
20      CONTINUE
30    CONTINUE
      RETURN
      END

      SUBROUTINE SMUL30(N,ILU,JLU,ID,ALU,B,X)
      IMPLICIT REAL (A-H,O-Z)
      DIMENSION ILU(*),JLU(*),ID(*),ALU(*),B(N),X(N)
C MULTIPLY TRANSPOSE OF U TIMES U TIMES B TO GET X
C WHERE:
C     N IS THE ORDER OF THE MATRIX
C     ILU GIVES INDEX OF FIRST NONZERO ELEMENT IN ROW OF LU
C     JLU GIVES COLUMN NUMBER
C     ID GIVES INDEX OF DIAGONAL ELEMENT OF U
C     ALU HAS NONZERO ELEMENTS OF LU MATRIX STORED BY ROW
C       WITH RECIPROCALS OF DIAGONAL ELEMENTS
C     B IS THE VECTOR
C     X IS THE PRODUCT UT*U*B (X MUST BE DIFFERENT FROM B)
C
      DO 10 I=1,N
        X(I)=0.0
10    CONTINUE
      DO 50 I=1,N
        KS=ID(I)+1
        KE=ILU(I+1)-1
        DIAG=1.0/ALU(KS-1)
        XX=DIAG*B(I)
        IF (KS.GT.KE) GO TO 30
        DO 20 K=KS,KE
          J=JLU(K)
          XX=XX+ALU(K)*B(J)
20      CONTINUE
30      X(I)=X(I)+DIAG*XX
        IF (KS.GT.KE) GO TO 50
        DO 40 K=KS,KE
          J=JLU(K)
          X(J)=X(J)+ALU(K)*XX
40      CONTINUE
50    CONTINUE
      RETURN
      END

      SUBROUTINE SSUBU0(N,ILU,JLU,ID,ALU,B,X)
      IMPLICIT REAL (A-H,O-Z)
      DIMENSION ILU(*),JLU(*),ID(*),ALU(*),B(N),X(N)
C DO FORWARD AND BACK SUBSTITUTION TO SOLVE UT*U*X=B
C WHERE:
C     N IS THE ORDER OF THE MATRIX
C     ILU GIVES INDEX OF FIRST NONZERO ELEMENT IN ROW OF LU
C     JLU GIVES COLUMN NUMBER
C     ID GIVES INDEX OF DIAGONAL ELEMENT OF U
C     ALU HAS NONZERO ELMENTS OF LU MATRIX STORED BY ROW
C       WITH RECIPROCALS OF DIAGONAL ELEMENTS OF U
C     B IS THE RHS VECTOR
C     X IS THE SOLUTION VECTOR
C
      NP=N+1
      DO 10 I=1,N
        X(I)=B(I)
10    CONTINUE
C FORWARD SUBSTITUTION
      DO 30 I=1,N
        KS=ID(I)+1
        KE=ILU(I+1)-1
        XX=X(I)*ALU(KS-1)
        X(I)=XX
        IF (KS.GT.KE) GO TO 30
        DO 20 K=KS,KE
          J=JLU(K)
          X(J)=X(J)-ALU(K)*XX
20      CONTINUE
30    CONTINUE
C BACK SUBSTITUTION
      DO 60 II=1,N
        I=NP-II
        KS=ID(I)+1
        KE=ILU(I+1)-1
        SUM=0.0
        IF (KS.GT.KE) GO TO 50
        DO 40 K=KS,KE
          J=JLU(K)
          SUM=SUM+ALU(K)*X(J)
40      CONTINUE
50      X(I)=(X(I)-SUM)*ALU(KS-1)
60    CONTINUE
      RETURN
      END

      SUBROUTINE SSUBL0(N,ILU,JLU,ID,ALU,B,X)
      IMPLICIT REAL (A-H,O-Z)
      DIMENSION ILU(*),JLU(*),ID(*),ALU(*),B(N),X(N)
C DO FORWARD AND BACK SUBSTITUTION TO SOLVE L*LT*X=B
C WHERE:
C     N IS THE ORDER OF THE MATRIX
C     ILU GIVES INDEX OF FIRST NONZERO ELEMENT IN ROW LU
C     JLU GIVES THE COLUMN NUMBER
C     ID GIVES INDEX OF DIAGONAL ELEMENT OF U
C     ALU HAS NONZERO ELEMENTS OF LU MATRIX STORED BY ROW
C       DIAGONAL ELEMENTS OF L ARE 1.0 AND NOT STORED
C     B IS THE RHS VECTOR
C     X IS THE SOLUTION VECTOR
C
      NP=N+1
      DO 10 I=1,N
        X(I)=B(I)
10    CONTINUE
C FORWARD SUBSTITUTION
      DO 30 I=1,N
        KS=ILU(I)
        KE=ID(I)-1
        IF (KS.GT.KE) GO TO 30
        SUM=0.0
        DO 20 K=KS,KE
          J=JLU(K)
          SUM=SUM+ALU(K)*X(J)
20      CONTINUE
        X(I)=X(I)-SUM
30    CONTINUE
C BACK SUBSTITUTION
      DO 50 II=1,N
        I=NP-II
        KS=ILU(I)
        KE=ID(I)-1
        IF (KS.GT.KE) GO TO 50
        XX=X(I)
        IF (XX.EQ.0.0) GO TO 50
        DO 40 K=KS,KE
          J=JLU(K)
          X(J)=X(J)-ALU(K)*XX
40      CONTINUE
50    CONTINUE
      RETURN
      END

      REAL FUNCTION SGVV(V,W,N)
      IMPLICIT REAL (A-H,O-Z)
      DIMENSION V(N),W(N)
C     SUBROUTINE TO COMPUTE REAL VECTOR DOT PRODUCT.
C
      SUM = 0.0
            DO 10 I = 1,N
            SUM = SUM + V(I)*W(I)
10          CONTINUE
      SGVV = SUM
      RETURN
      END

```

sparse-matrix/ilucg/README:
```
This directory contains the ILUCG (Incomplete LU-decomposition /
conjugate gradiant) sparse matrix subroutines.

These subroutines were provided to me (JT) by Tom Nicol of the University
of British Columbia Computing Center in 1985; I believe they were originally
written by P. Madderom (also of the UBC Computing Center).

I have slightly modified the convergence criteria.

```

sparse-matrix/ilucg/make.code.defn:
```
# Cactus specification of things to be compiled in this directory
# $Header$

# Source files in this directory
## uncomment the appropriate line for single vs double precision
##SRCS = silucg.f77
SRCS = dilucg.f77

# Subdirectories containing source files
SUBDIRS =

```

sparse-matrix/ilucg/dilucg.f77:
```
      SUBROUTINE DILUCG(N,
     $                  IA,JA,A,
     $                  B,X,
     $                  ITEMP,RTEMP,
     $                  EPS,MAXITER,
     $                  ISTATUS)
      IMPLICIT DOUBLE PRECISION (A-H,O-Z)
      DIMENSION IA(*),JA(*),A(*),B(*),X(*),ITEMP(*),RTEMP(*)
C
C     INCOMPLETE LU DECOMPOSITION-CONJUGATE GRADIENT
C     -          --               -         -
C WHERE:
C     |N| IS THE NUMBER OF EQUATIONS.  IF N < 0, ITEMP AND
C       RTEMP CONTAIN THE ILU FROM A PREVIOUS CALL AND
C       B AND X ARE THE NEW RHS AND INITIAL GUESS.
C     IA IS AN INTEGER ARRAY DIMENSIONED |N|+1.  IA(I) IS THE
C       INDEX INTO ARRAYS JA AND A OF THE FIRST NON-ZERO
C       ELEMENT IN ROW I.  LET MAX=IA(|N|+1)-IA(1).
C     JA IS AN INTEGER ARRAY DIMENSIONED MAX.  JA(K) GIVES
C       THE COLUMN NUMBER OF A(K).
C     A IS A DOUBLE PRECISION ARRAY DIMENSIONED MAX.  IT CONTAINS THE
C       NONZERO ELEMENTS OF THE MATRIX STORED BY ROW.
C     B CONTAINS THE RHS VECTOR.
C     X IS A DOUBLE PRECISION ARRAY DIMENSIONED |N|.  ON ENTRY, IT CONTAINS
C       AN INITIAL ESTIMATE; ON EXIT, THE SOLUTION.
C     ITEMP IS AN INTEGER SCRATCH ARRAY DIMENSIONED 3*(|N|+MAX)+2.
C     RTEMP IS A DOUBLE PRECISION SCRATCH ARRAY DIMENSIONED 4*|N|+MAX.
C     EPS IS THE CONVERGENCE CRITERIA.  IT SPECIFIES THE RELATIVE
C       ERROR ALLOWED IN THE SOLUTION.  TO BE PRECISE, CONVERGENCE
C       IS DEEMED TO HAVE OCCURED WHEN THE INFINITY-NORM OF THE
C       CHANGE IN THE SOLUTION IN ONE ITERATION IS .LE. EPS * THE
C       INFINITY-NORM OF THE CURRENT SOLUTION.  HOWEVER, IF EPS
C       .LT. 0.0D0, IT IS INTERNALLY SCALED BY THE MACHINE PRECISION,
C       SO THAT, FOR EXAMPLE, EPS = -256.0D0 WILL ALLOW THE LAST 8 BITS
C       OF THE SOLUTION TO BE IN ERROR.
C     MAXITER GIVES THE REQUESTED NUMBER OF ITERATIONS,
C       OR IS 0 FOR "NO LIMIT".
C     ISTATUS IS AN INTEGER VARIABLE, WHICH IS SET TO:
C       -I IF THERE IS AN ERROR IN THE MATRIX STRUCTURE IN ROW I
C          (SUCH AS A ZERO ELEMENT ON THE DIAGONAL).
C        0 IF THE ITERATION FAILED TO REACH THE CONVERGENCE CRITERION
C          IN ITER ITERATIONS.
C       +I IF THE ITERATION CONVERGED IN I ITERATIONS.
C
C MODIFICATIONS
C * RETURN LOGICAL VALUE, J. THORNBURG, 13/MAY/85.
C * ADD CONVERGENCE CRITERIA, J. THORNBURG, 17/MAY/85.
C * CHANGED CONVERGENCE CRITERIA, J. THORNBURG, 4/AUG/89.
C * AVOID SKIP RETURNS, J. THORNBURG, 10/SEP/89.
C * CHANGE FUNCTION BACK TO SUBROUTINE, J. THORNBURG, 3.JUN.2003
C * CHANGE CONVERGENCE CRITERIA TO AVOID SPURIOUSLY STOPPING TOO SOON
C   ON SMALL RHS VECTORS -- J. THORNBURG, 11.JUN.2003
C
C REFERENCE:
C     D.S.KERSHAW,"THE INCOMPLETE CHOLESKY-CONJUGATE GRADIENT
C       METHOD FOR INTERATIVE SOLUTION OF LINEAR EQUATIONS",
C       J.COMPUT.PHYS. JAN 1978 PP 43-65
C
      LOGICAL DLU0
      NP=IABS(N)
      ISTATUS=0
      IF (NP.EQ.0) GO TO 20
C CALCULATE INDICES FOR BREAKING UP TEMPORARY ARRAYS.
      N1=NP+1
      MAX=IA(N1)-IA(1)
      ILU=1
      JLU=ILU+N1
      ID=JLU+MAX
      IC=ID+NP
      JC=IC+N1
      JCI=JC+MAX
      IR=1
      IP=IR+NP
      IS1=IP+NP
      IS2=IS1+NP
      IALU=IS2+NP
      IF (N.LT.0) GO TO 10
C DO INCOMPLETE LU DECOMPOSITION
      IF (DLU0(NP,IA,JA,A,ITEMP(IC),ITEMP(JC),ITEMP(JCI),RTEMP(IALU),
     *    ITEMP(ILU),ITEMP(JLU),ITEMP(ID),RTEMP(IR),IERROR)) GOTO 20
C AND DO CONJUGATE GRADIENT ITERATIONS
C CALL MODIFIED TO ADD ADJUSTABLE CONVERGENCE CRITERIA EPS
C - J. THORNBURG, 17/MAY/85.
10    CALL DNCG0(NP,IA,JA,A,B,X,ITEMP(ILU),ITEMP(JLU),ITEMP(ID),
     *  RTEMP(IALU),RTEMP(IR),RTEMP(IP),RTEMP(IS1),RTEMP(IS2),
     *  EPS,MAXITER,ITER)
C ITER IS ACTUAL NUMBER OF ITERATIONS (NEGATIVE IF NO CONVERGENCE)
      ISTATUS = ITER
      IF (ITER .LT. 0) ISTATUS = 0
      RETURN
C ERROR RETURN FROM INCOMPLETE LU DECOMPOSITION
20    ISTATUS = -IERROR
      RETURN
      END

      LOGICAL FUNCTION DLU0(N,IA,JA,A,IC,JC,JCI,ALU,ILU,JLU,ID,V,IE)
      IMPLICIT DOUBLE PRECISION (A-H,O-Z)
      DIMENSION IA(*),JA(*),A(*),IC(*),JC(*),JCI(*),
     *  ALU(*),ILU(*),JLU(*),ID(N),V(N)
      LOGICAL NODIAG
      COMMON /ICBD00/ ICBAD
C     INCOMPLETE LU DECOMPOSITION
C WHERE:
C     N,IA,JA, AND A ARE DESCRIBED IN SUBROUTINE ILUCG
C     IC IS AN INTEGER ARRAY DIMENSIONED N+1, IC(J) GIVES THE
C       INDEX OF THE FIRST NONZERO ELEMENT IN COLMN J IN
C       ARRAY JC.
C     JC IS AN INTEGER ARRAY WITH THE SAME DIMENSION AS A.
C       JC(K) GIVES THE ROW NUMBER OF THE K'TH ELEMENT IN
C       THE COLUMN STRUCTURE.
C     JCI IS AN INTEGER ARRAY WITH THE SAME DIMENSION AS A.
C       JCI(K) GIVES THE INDEX INTO ARRAY A OF THE K'TH ELEMENT
C       OF THE COLUMN STRUCTURE.
C     ALU HAS THE SAME DIMENSION AS A.  ON EXIT, IT WILL
C       CONTAIN THE INCOMPLETE LU DECOMPOSITION OF A WITH THE
C       RECIPROCALS OF THE DIAGONAL ELEMENTS OF U.
C     ILU AND JLU CORRESPONDS TO IA AND JA BUT FOR ALU.
C     ID IS AN INTEGER ARRAY DIMENSIONED N.  IT CONTAINS
C       INDICES TO THE DIAGONAL ELEMENTS OF U.
C     V IS A REAL SCRATCH VECTOR OF LENGTH N.
C     IE GIVES THE ROW NUMBER IN ERROR IF AN ERROR OCCURED
C       (RETURN VALUE .TRUE.), OR IS UNUSED IF ALL IS WELL
C       (RETURN VALUE .FALSE.).
C
C     RETURN VALUE = .FALSE. IF ALL IS WELL, .TRUE. IF ERROR.
C
C NOTE: DLU0 SETS ARGUMENTS IC THROUGH V.
C
      ICBAD=0
C ZERO COUNT OF ZERO DIAGONAL ELEMENTS IN U.
C
C FIRST CHECK STRUCTURE OF A AND BUILD COLUMN STRUCTURE
      DO 10 I=1,N
        IC(I)=0
10    CONTINUE
      DO 30 I=1,N
        KS=IA(I)
        KE=IA(I+1)-1
        NODIAG=.TRUE.
        DO 20 K=KS,KE
          J=JA(K)
          IF (J.LT.1.OR.J.GT.N) GO TO 210
          IC(J)=IC(J)+1
          IF (J.EQ.I) NODIAG=.FALSE.
20      CONTINUE
        IF (NODIAG) GO TO 210
30    CONTINUE
C MAKE IC INTO INDICES
      KOLD=IC(1)
      IC(1)=1
      DO 40 I=1,N
        KNEW=IC(I+1)
        IF (KOLD.EQ.0) GO TO 210
        IC(I+1)=IC(I)+KOLD
        KOLD=KNEW
40    CONTINUE
C SET JC AND JCI FOR COLUMN STRUCTURE
      DO 60 I=1,N
        KS=IA(I)
        KE=IA(I+1)-1
        DO 50 K=KS,KE
          J=JA(K)
          L=IC(J)
          IC(J)=L+1
          JC(L)=I
          JCI(L)=K
50      CONTINUE
60    CONTINUE
C FIX UP IC
      KOLD=IC(1)
      IC(1)=1
      DO 70 I=1,N
        KNEW=IC(I+1)
        IC(I+1)=KOLD
        KOLD=KNEW
70    CONTINUE
C FIND SORTED ROW STRUCTURE FROM SORTED COLUMN STRUCTURE
      NP=N+1
      DO 80 I=1,NP
        ILU(I)=IA(I)
80    CONTINUE
C MOVE ELEMENTS, SET JLU AND ID
      DO 100 J=1,N
        KS=IC(J)
        KE=IC(J+1)-1
        DO 90 K=KS,KE
          I=JC(K)
          L=ILU(I)
          ILU(I)=L+1
          JLU(L)=J
          KK=JCI(K)
          ALU(L)=A(KK)
          IF (I.EQ.J) ID(J)=L
90      CONTINUE
100   CONTINUE
C RESET ILU (COULD JUST USE IA)
      DO 110 I=1,NP
        ILU(I)=IA(I)
110   CONTINUE
C FINISHED WITH SORTED COLUMN AND ROW STRUCTURE
C
C DO LU DECOMPOSITION USING GAUSSIAN ELIMINATION
      DO 120 I=1,N
        V(I)=0.0D0
120   CONTINUE
      DO 200 IROW=1,N
        I=ID(IROW)
        PIVOT=ALU(I)
        IF (PIVOT.NE.0.0D0) GO TO 140
C THIS CASE MAKES THE ILU LESS ACCURATE
        ICBAD=ICBAD+1
        KS=ILU(IROW)
        KE=ILU(IROW+1)-1
        DO 130 K=KS,KE
          PIVOT=PIVOT+DABS(ALU(K))
130     CONTINUE
        IF (PIVOT.EQ.0.0D0) GO TO 220
140     PIVOT=1.0D0/PIVOT
        ALU(I)=PIVOT
        KKS=I+1
        KKE=ILU(IROW+1)-1
        IF (KKS.GT.KKE) GO TO 160
        DO 150 K=KKS,KKE
          J=JLU(K)
          V(J)=ALU(K)
150     CONTINUE
C FIX L IN COLUMN IROW AND DO PARTIAL LU IN SUBMATRIX
160     KS=IC(IROW)
        KE=IC(IROW+1)-1
        DO 190 K=KS,KE
          I=JC(K)
          IF (I.LE.IROW) GO TO 190
          LS=ILU(I)
          LE=ILU(I+1)-1
          DO 180 L=LS,LE
            J=JLU(L)
            IF (J.LT.IROW) GO TO 180
            IF (J.GT.IROW) GO TO 170
            AMULT=ALU(L)*PIVOT
            ALU(L)=AMULT
            IF (AMULT.EQ.0.0) GO TO 190
            GO TO 180
170         IF (V(J).EQ.0.0D0) GO TO 180
            ALU(L)=ALU(L)-AMULT*V(J)
180       CONTINUE
190     CONTINUE
C RESET V
        IF (KKS.GT.KKE) GO TO 200
        DO 195 K=KKS,KKE
          J=JLU(K)
          V(J)=0.0D0
195     CONTINUE
200   CONTINUE
C NORMAL RETURN
      DLU0 = .FALSE.
      RETURN
C ERROR RETURNS
210   IE=I
      DLU0 = .TRUE.
      RETURN
220   IE=IROW
      DLU0 = .TRUE.
      RETURN
      END

      SUBROUTINE DNCG0(N,IA,JA,A,B,X,ILU,JLU,ID,ALU,R,P,S1,S2,
     *  EPS,ITER,IE)
      IMPLICIT DOUBLE PRECISION (A-H,O-Z)
      DIMENSION IA(*),JA(*),A(*),B(N),X(N),ILU(*),JLU(*),ALU(*),ID(N),
     *  R(N),P(N),S1(N),S2(N)
C     NONSYMMETRIC CONJUGATE GRADIENT
C WHERE:
C     N,IA,JA,A,B, AND X ARE DESCRIBED IN SUBROUTINE DILUCG.
C     ILU GIVES INDEX OF FIRST NONZERO ELEMENT IN ROW OF LU.
C     JLU GIVES COLUMN NUMBER.
C     ID GIVES INDEX OF DIAGONAL ELEMENT OF U.
C     ALU HAS NONZERO ELEMENTS OF LU MATRIX STORED BY ROW
C       WITH RECIPROCALS OF DIAGONAL ELEMENTS OF U.
C     R,P,S1, AND S2 ARE VECTORS OF LENGTH N USED IN THE
C       ITERATIONS.
C FOLLOWING PARAMETER ADDED BY J. THORNBURG, 17/MAY/85.
C     EPS IS CONVERGENCE CRITERIA.  (DESCRIBED IN SUBROUTINE
C       DILUCG).
C     ITER IS MAX NUMBER OF ITERATIONS, OR 0 FOR "NO LIMIT".
C     IE GIVES ACTUAL NUMBER OF ITERATIONS, NEGATIVE IF
C       NO CONVERGENCE.
C
C R0=B-A*X0
      CALL DMUL10(N,IA,JA,A,X,R)
      DO 10 I=1,N
        R(I)=B(I)-R(I)
10    CONTINUE
C P0=(UT*U)(-1)*AT*(L*LT)(-1)*R0
C FIRST SOLVE L*LT*S1=R0
      CALL DSUBL0(N,ILU,JLU,ID,ALU,R,S1)
C TIMES TRANSPOSE OF A
      CALL DMUL20(N,IA,JA,A,S1,S2)
C THEN SOLVE UT*U*P=S2
      CALL DSUBU0(N,ILU,JLU,ID,ALU,S2,P)
      IE=0
C INPROD IS DOT PRODUCT ROUTINE IN *LIBRARY (UBC MATRIX P 28)
C ALL CALLS ON IT COMMENTED OUT AND REPLACED WITH CALLS TO NEW
C ROUTINE DGVV(...); SOURCE CODE FOR LATTER ADDED TO END OF
C THIS FILE.  - J. THORNBURG, 10/MAY/85.
C     CALL INPROD(R,S1,EDOT,RDOT,N)
      RDOT = DGVV(R,S1,N)
C LOOP BEGINS HERE
20    CALL DMUL30(N,ILU,JLU,ID,ALU,P,S2)
C     CALL INPROD(P,S2,EDOT,PDOT,N)
      PDOT = DGVV(P,S2,N)
C
      IF (PDOT.EQ.0.0D0) RETURN
C
      ALPHA=RDOT/PDOT
C EQUATION 9PA                      ALPHA=(R,LINV*R)/(P,UT*U*P)
C 1 + ||X CHANGE|| CHANGED TO ||X CHANGE|| -- J. THORNBURG, 11.JUN.2003
      XMAX=0.0D0
      XDIF=0.0D0
      DO 30 I=1,N
        AP=ALPHA*P(I)
        X(I)=X(I)+AP
C EQUATION 9PB                      X=X+ALPHA*P
        AP=DABS(AP)
        XX=DABS(X(I))
        IF (AP.GT.XDIF) XDIF=AP
        IF (XX.GT.XMAX) XMAX=XX
30    CONTINUE
      IE=IE+1
C
C CONVERGENCE TEST (CHANGED BY J. THORNBURG, 17/MAY/85, 4/AUG/89)
C
      IF ((EPS .GT. 0.0D0) .AND. (XDIF .LE. EPS * XMAX)) RETURN
      IF ((EPS .LT. 0.0D0) .AND. (XMAX + XDIF/DABS(EPS) .EQ. XMAX))
     *   RETURN
C
C EXCEEDED ITERATION LIMIT?
C
      IF ((ITER .NE. 0) .AND. (IE .GE. ITER)) GO TO 60
      CALL DMUL10(N,IA,JA,A,P,S2)
      DO 40 I=1,N
        R(I)=R(I)-ALPHA*S2(I)
C EQUATION 9PC                      R=R-ALPHA*A*P
40    CONTINUE
      CALL DSUBL0(N,ILU,JLU,ID,ALU,R,S1)
C     CALL INPROD(R,S1,EDOT,RRDOT,N)
      RRDOT = DGVV(R,S1,N)
      BETA=RRDOT/RDOT
C EQUATION 9PD                      BETA=(R+,LINV*R+)/(R,LINV*R)
      RDOT=RRDOT
      CALL DMUL20(N,IA,JA,A,S1,S2)
      CALL DSUBU0(N,ILU,JLU,ID,ALU,S2,S1)
      DO 50 I=1,N
        P(I)=S1(I)+BETA*P(I)
C EQUATION 9PE                      P=(UT*U)(-1)*AT*(L*LT)(-1)*R+BETA*P
50    CONTINUE
      GO TO 20
60    IE=-IE
      RETURN
      END

      SUBROUTINE DMUL10(N,IA,JA,A,B,X)
      IMPLICIT DOUBLE PRECISION (A-H,O-Z)
      DIMENSION IA(*),JA(*),A(*),B(N),X(N)
C MULTIPLY A TIMES B TO GET X
C WHERE:
C     N IS THE ORDER OF THE MATRIX
C     IA GIVES INDEX OF FIRST NONZERO ELEMENT IN ROW
C     JA GIVES COLUMN NUMBER
C     A CONTAINS THE NONZERO ELEMENTS OF THE NONSYMMETRIC
C       MATRIX STORED BY ROW
C     B IS THE VECTOR
C     X IS THE PRODUCT (MUST BE DIFFERENT FROM B)
C
      DO 20 I=1,N
        KS=IA(I)
        KE=IA(I+1)-1
        SUM=0.0D0
        DO 10 K=KS,KE
          J=JA(K)
          SUM=SUM+A(K)*B(J)
10      CONTINUE
        X(I)=SUM
20    CONTINUE
      RETURN
      END

      SUBROUTINE DMUL20(N,IA,JA,A,B,X)
      IMPLICIT DOUBLE PRECISION (A-H,O-Z)
      DIMENSION IA(*),JA(*),A(*),B(N),X(N)
C MULTIPLY TRANSPOSE OF A TIMES B TO GET X
C WHERE:
C     N IS THE ORDER OF THE MATRIX
C     IA GIVES INDEX OF FIRST NONZERO ELEMENT IN ROW
C     JA GIVES COLUMN NUMBER
C     A CONTAINS THE NONZERO ELEMENTS OF THE NONSYMMETRIC
C       MATRIX STORED BY ROW
C     B IS THE VECTOR
C     X IS THE PRODUCT (MUST BE DIFFERENT FROM B)
C
      DO 10 I=1,N
        X(I)=0.0D0
10    CONTINUE
      DO 30 I=1,N
        KS=IA(I)
        KE=IA(I+1)-1
        BB=B(I)
        DO 20 K=KS,KE
          J=JA(K)
          X(J)=X(J)+A(K)*BB
20      CONTINUE
30    CONTINUE
      RETURN
      END

      SUBROUTINE DMUL30(N,ILU,JLU,ID,ALU,B,X)
      IMPLICIT DOUBLE PRECISION (A-H,O-Z)
      DIMENSION ILU(*),JLU(*),ID(*),ALU(*),B(N),X(N)
C MULTIPLY TRANSPOSE OF U TIMES U TIMES B TO GET X
C WHERE:
C     N IS THE ORDER OF THE MATRIX
C     ILU GIVES INDEX OF FIRST NONZERO ELEMENT IN ROW OF LU
C     JLU GIVES COLUMN NUMBER
C     ID GIVES INDEX OF DIAGONAL ELEMENT OF U
C     ALU HAS NONZERO ELEMENTS OF LU MATRIX STORED BY ROW
C       WITH RECIPROCALS OF DIAGONAL ELEMENTS
C     B IS THE VECTOR
C     X IS THE PRODUCT UT*U*B (X MUST BE DIFFERENT FROM B)
C
      DO 10 I=1,N
        X(I)=0.0D0
10    CONTINUE
      DO 50 I=1,N
        KS=ID(I)+1
        KE=ILU(I+1)-1
        DIAG=1.0D0/ALU(KS-1)
        XX=DIAG*B(I)
        IF (KS.GT.KE) GO TO 30
        DO 20 K=KS,KE
          J=JLU(K)
          XX=XX+ALU(K)*B(J)
20      CONTINUE
30      X(I)=X(I)+DIAG*XX
        IF (KS.GT.KE) GO TO 50
        DO 40 K=KS,KE
          J=JLU(K)
          X(J)=X(J)+ALU(K)*XX
40      CONTINUE
50    CONTINUE
      RETURN
      END

      SUBROUTINE DSUBU0(N,ILU,JLU,ID,ALU,B,X)
      IMPLICIT DOUBLE PRECISION (A-H,O-Z)
      DIMENSION ILU(*),JLU(*),ID(*),ALU(*),B(N),X(N)
C DO FORWARD AND BACK SUBSTITUTION TO SOLVE UT*U*X=B
C WHERE:
C     N IS THE ORDER OF THE MATRIX
C     ILU GIVES INDEX OF FIRST NONZERO ELEMENT IN ROW OF LU
C     JLU GIVES COLUMN NUMBER
C     ID GIVES INDEX OF DIAGONAL ELEMENT OF U
C     ALU HAS NONZERO ELMENTS OF LU MATRIX STORED BY ROW
C       WITH RECIPROCALS OF DIAGONAL ELEMENTS OF U
C     B IS THE RHS VECTOR
C     X IS THE SOLUTION VECTOR
C
      NP=N+1
      DO 10 I=1,N
        X(I)=B(I)
10    CONTINUE
C FORWARD SUBSTITUTION
      DO 30 I=1,N
        KS=ID(I)+1
        KE=ILU(I+1)-1
        XX=X(I)*ALU(KS-1)
        X(I)=XX
        IF (KS.GT.KE) GO TO 30
        DO 20 K=KS,KE
          J=JLU(K)
          X(J)=X(J)-ALU(K)*XX
20      CONTINUE
30    CONTINUE
C BACK SUBSTITUTION
      DO 60 II=1,N
        I=NP-II
        KS=ID(I)+1
        KE=ILU(I+1)-1
        SUM=0.0D0
        IF (KS.GT.KE) GO TO 50
        DO 40 K=KS,KE
          J=JLU(K)
          SUM=SUM+ALU(K)*X(J)
40      CONTINUE
50      X(I)=(X(I)-SUM)*ALU(KS-1)
60    CONTINUE
      RETURN
      END

      SUBROUTINE DSUBL0(N,ILU,JLU,ID,ALU,B,X)
      IMPLICIT DOUBLE PRECISION (A-H,O-Z)
      DIMENSION ILU(*),JLU(*),ID(*),ALU(*),B(N),X(N)
C DO FORWARD AND BACK SUBSTITUTION TO SOLVE L*LT*X=B
C WHERE:
C     N IS THE ORDER OF THE MATRIX
C     ILU GIVES INDEX OF FIRST NONZERO ELEMENT IN ROW LU
C     JLU GIVES THE COLUMN NUMBER
C     ID GIVES INDEX OF DIAGONAL ELEMENT OF U
C     ALU HAS NONZERO ELEMENTS OF LU MATRIX STORED BY ROW
C       DIAGONAL ELEMENTS OF L ARE 1.0 AND NOT STORED
C     B IS THE RHS VECTOR
C     X IS THE SOLUTION VECTOR
C
      NP=N+1
      DO 10 I=1,N
        X(I)=B(I)
10    CONTINUE
C FORWARD SUBSTITUTION
      DO 30 I=1,N
        KS=ILU(I)
        KE=ID(I)-1
        IF (KS.GT.KE) GO TO 30
        SUM=0.0D0
        DO 20 K=KS,KE
          J=JLU(K)
          SUM=SUM+ALU(K)*X(J)
20      CONTINUE
        X(I)=X(I)-SUM
30    CONTINUE
C BACK SUBSTITUTION
      DO 50 II=1,N
        I=NP-II
        KS=ILU(I)
        KE=ID(I)-1
        IF (KS.GT.KE) GO TO 50
        XX=X(I)
        IF (XX.EQ.0.0) GO TO 50
        DO 40 K=KS,KE
          J=JLU(K)
          X(J)=X(J)-ALU(K)*XX
40      CONTINUE
50    CONTINUE
      RETURN
      END

      DOUBLE PRECISION FUNCTION DGVV(V,W,N)
      IMPLICIT DOUBLE PRECISION (A-H,O-Z)
      DIMENSION V(N),W(N)
C     SUBROUTINE TO COMPUTE DOUBLE PRECISION VECTOR DOT PRODUCT.
C
      SUM = 0.0D0
            DO 10 I = 1,N
            SUM = SUM + V(I)*W(I)
10          CONTINUE
      DGVV = SUM
      RETURN
      END

```


combined_doc_context: ## README (README):
```
Cactus Code Thorn AHFinderDirect
Authors    : Jonathan Thornburg <jthorn@aei.mpg.de>
Maintainer : Jonathan Thornburg <jthorn@aei.mpg.de>
CVS info   : $Header$
--------------------------------------------------------------------------

Purpose of This Thorn
=====================

This thorn finds an apparent horizon (or more generally, a closed
2-surface with S^2 topology having any desired constant expansion)
given the 3D xyz-grid metric and extrinsic curvature (and optionally
the StaticConformal conformal factor).  It uses a direct method,
writing the apparent horizon equation as an elliptic PDE on
angular-coordinate space.  This is very fast and accurate, but
it does require an intitial guess for the apparent horizon position.


Documentation
=============

The doc/ directory contains detailed documentation for this thorn,
including a discussion of those parameters which you'll likely want
to set (or at least know about) in practice.  For full information
on all the parameters, see the comments in the  param.ccl  file.

The par/ directory contains some sample par files.

src/CODESTYLE documents the general programming conventions used in
this thorn.

src/PORTING.OUTSIDE.CACTUS has some general remarks on what it would
take to port this thorn to a non-Cactus numerical relativity code.

See below in this file for notes on compiling this thorn.


Publications
============
The algorithms used in this thorn are described in the papers
@article{
	Thornburg2003:AH-finding,
	author = "Jonathan Thornburg",
	title  = "A Fast Apparent-Horizon Finder
		  for 3-Dimensional {C}artesian Grids in Numerical Relativity",
	journal = "Classical and Quantum Gravity",
	volume = 21, number = 2,
	year   = "2004", month = "January 21",
	pages  = "743--766",
	doi    = "10.1088/0264-9381/21/2/026",
	url    = "http://stacks.iop.org/0264-9381/21/743",
	eprint = "gr-qc/0306056",
	note   = "gr-qc/0306056",
	}
and
@article{
	Thornburg-1996-horizon-finding,
	author = "Jonathan Thornburg",
	title = "Finding Apparent Horizons in Numerical Relativity",
	journal = "Physical Review D",
	pages = "4899--4918",
	volume = 54, number = 8,
	year = 1996, month = "October 15",
	eprint = "gr-qc/9508014",
	}
As a courtesy, I ask that these papers be cited in any published research
which uses this thorn, or which uses code from this thorn.


Copyright
=========

This thorn is copyright (C) 2001-2004
by Jonathan Thornburg <jthorn@aei.mpg.de>.

This thorn is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.

This thorn is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this thorn (see the file COPYING in this directory);
if not, write to the Free Software Foundation, Inc., 59 Temple
Place, Suite 330, Boston, MA  02111-1307  USA


Other Software Used (Already Contained within AHFinderDirect)
=============================================================

AHFinderDirect's  src/sparse-matrix/  directory contains various
sparse-matrix libraries, which have their own copyrights and licensing terms:

The src/sparse-matrix/umfpack/ directory contains a subset of the files
in UMFPACK version 4.0 (11.Apr.2002).  This code is copyright (C) 2002 by
Timothy A. Davis, and is subect to the UMFPACK License:

    Your use or distribution of UMFPACK or any modified version of
    UMFPACK implies that you agree to this License.

    THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY
    EXPRESSED OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.

    Permission is hereby granted to use or copy this program, provided
    that the Copyright, this License, and the Availability of the original
    version is retained on all copies.  User documentation of any code that
    uses UMFPACK or any modified version of UMFPACK code must cite the
    Copyright, this License, the Availability note, and "Used by permission."
    Permission to modify the code and to distribute modified code is granted,
    provided the Copyright, this License, and the Availability note are
    retained, and a notice that the code was modified is included.  This
    software was developed with support from the National Science Foundation,
    and is provided to you free of charge.


Other Software Required
=======================

This thorn inherits from Grid, ADMBase, StaticConformal, SpaceMask, IO
(all in the CactusBase or CactusEinstein arrangements), and
SphericalSurface (in the AEIThorns arrangement; you can get AEIThorns
by anonymous CVS checkout from cvs_anon@cvs.aei.mpg.de:/numrelcvs, CVS
path AEIThorns/SphericalSurface).

By default this thorn uses various Cactus APIs which are supplied by
other thorns:
* This thorn uses CCTK_InterpGridArrays() for interpolating grid arrays.
  This is supplied by a driver-specific thorn, such as PUGHInterp or
  CarpetInterp.
* This thorn uses CCTK_ReduceLocArrayToArray1D() for interprocessor
  communication in the multiprocessor Newton solver.
	[see src/driver/README.parallel for details]
  This is supplied by a driver-specific thorn; at present PUGHReduce
  it this for the PUGH driver.
* This thorn uses CCTK_InterpLocalUniform() for interpatch and surface
  interpolation, and it needs various interpolation options which at
  present are (only) supported by the AEIThorns/AEILocalInterp local
  interpolator.

This thorn is written in C++, so you'll need a C++ compiler -- in fact
a fairly modern one -- to compile this thorn.  See the "Compilation Notes"
section below for details.

Most of this thorn's relativity code is machine-generated using Maple
(version 7), but you don't need Maple unless you want to modify the
relativity code.

By default, this thorn doesn't use any external libraries.  However,
if HAVE_DENSE_JACOBIAN__LAPACK is defined in src/include/config.h,
then this thorn uses the LAPACK library (which in turn uses the BLAS
library), so you will need to configure your Cactus to use LAPACK.
Instructions on doing this are in the file README.library in this
directory.


Code Notes
==========

This thorn is written in C++, and requires a fairly modern C++ compiler.
In particular:
* Templates are used, but only rather simple ones for containers or other
  low-level stuff templated on the floating-point or integer datatype.
  These templates are always instantiated explicitly.
* bool, mutable, typename are used.
* The new-style casts static_cast<...> and const_cast<...> are used.
  reinterpret_cast<...> and dynamic_cast<...> are *not* used.
* The code will compile ok under either the archaic or the modern
  for-loop declaration scope rules.
* C header files are used in the pre-namespaces form (eg <stdio.h>).
  (In ISO-standard C++ this is deprecated (but still supported), but
  alas none of the more modern forms (<cstdio> and namespace std::)
  seem to be supported on as wide a range of systems as the pre-namespaces
  form.)
* <assert.h> is used fairly heavily for sanity checks.
* To avoid various portability problems, none of the C++ standard
  template library (STL) is used.


Compiler Notes
==============

This thorn should be fully portable across architectures and operating
systems, but C++ compilers may be a problem:

This thorn has been compiled and run successfully using

x86
===
* gcc version 2.95.3 20010125 (prerelease)
* g++ (GCC) 3.2 for x86
* g++ (GCC) 3.2.2 for x86
* g++ (GCC) 3.3 for x86
* Intel(R) C++ Compiler for 32-bit applications, Version 5.0.1   Build 010730D0
* Intel C++ Compiler for Linux, version 6.0
* Intel(R) C++ Compiler for 32-bit applications, Version 7.0   Build 20021021Z
  On platinum.ncsa.uiuc.edu, Denis Pollney reports success configuring Cactus
  with
    LIBS="crypt F90 CEPCF90 intrins mkl mkl_lapack guide"
    LIBDIRS=/usr/local/intel/mkl/lib/32
* Portland Group pgCC 3.3-2
  On platinum.ncsa.uiuc.edu, Denis Pollney reports success configuring Cactus
  with
    LAPACK=yes
    LAPACK_DIR=/usr/local/pgi/linux86

ia64, a.k.a. Itanium
====================
* Intel(R) C++ Itanium(R) Compiler for Itanium(R)-based applications
  Version 7.0 Beta, Build 20020911
  On titan.ncsa.uiuc.edu, Denis Pollney reports success configuring Cactus
  with
    LIBS="crypt F90 CEPCF90 intrins mkl_lapack mkl_itp guide"
    LIBDIRS=/usr/local/intel/mkl/lib/64

Alpha
=====
* g++ (GCC) 3.0
  On lemieux.psc.edu, Denis Pollney reports success configuring Cactus
  with
    LIBS="m c_r dxml for"
* Using the DEC/Compaq/HP/whatever-their-corporate-name-is-this-week
  compiler, you need to specify the  -nopt  compiler option to disable
  automatic template instantiation.

SGI
===
* On modi4.ncsa.uiuc.edu (an old SGI Origin), using
  MIPSpro Compilers: Version 7.3.1.3m
  you need to specify the  -no_auto_include  compiler option to disable
  automatic template instantiation.  In fact, you should probably specify
  this option on any SGI system...

IBM Power4 (AIX)
================
* C for AIX Compiler, Version 6
  On psi19.rzg.mpg.de, Denis Pollney reports success configuring Cactus
  with
    LIBS="f m lapack-essl essl"
    LIBDIRS="/afs/rzg/@sys/lib"

Please let me (Jonathan) know of any other compilers which are ok.

This thorn compiles, but does *NOT* work, using the folloing
systems/compilers; the cure is to switch to a newer compiler version:
x86
===
* gcc version 2.96 20000731 (Red Hat Linux 7.1 2.96-98)
  (This is the system gcc on Red Hat GNU/Linux 7.2 systems)

The code will almost certainly *not* compile using
* any "egcs" version of gcc, or more generally, any version of gcc
  earlier than 2.95
* any version of Microsoft Visual C++ up to and including mid-2002

```

## Documentation (.tex files):
Files: documentation.tex

```tex
=== documentation.tex ===
% *======================================================================*
%  Cactus Thorn template for ThornGuide documentation
%  Author: Ian Kelley
%  Date: Sun Jun 02, 2002
%  $Header$
%
%  Thorn documentation in the latex file doc/documentation.tex
%  will be included in ThornGuides built with the Cactus make system.
%  The scripts employed by the make system automatically include
%  pages about variables, parameters and scheduling parsed from the
%  relevent thorn CCL files.
%
%  This template contains guidelines which help to assure that your
%  documentation will be correctly added to ThornGuides. More
%  information is available in the Cactus UsersGuide.
%
%  Guidelines:
%   - Do not change anything before the line
%       % BEGIN CACTUS THORNGUIDE",
%     except for filling in the title, author, date etc. fields.
%   - You can define your own macros are OK, but they must appear after
%     the BEGIN CACTUS THORNGUIDE line, and do not redefine standard
%     latex commands.
%   - To avoid name clashes with other thorns, 'labels', 'citations',
%     'references', and 'image' names should conform to the following
%     convention:
%       ARRANGEMENT_THORN_LABEL
%     For example, an image wave.eps in the arrangement CactusWave and
%     thorn WaveToyC should be renamed to CactusWave_WaveToyC_wave.eps
%   - Graphics should only be included using the graphix package.
%     More specifically, with the "includegraphics" command. Do
%     not specify any graphic file extensions in your .tex file. This
%     will allow us (later) to create a PDF version of the ThornGuide
%     via pdflatex. |
%   - References should be included with the latex "bibitem" command.
%   - For the benefit of our Perl scripts, and for future extensions,
%     please use simple latex.
%
% *======================================================================*
%
% Example of including a graphic image:
%    \begin{figure}[ht]
% 	\begin{center}
%    	   \includegraphics[width=6cm]{MyArrangement_MyThorn_MyFigure}
% 	\end{center}
% 	\caption{Illustration of this and that}
% 	\label{MyArrangement_MyThorn_MyLabel}
%    \end{figure}
%
% Example of using a label:
%   \label{MyArrangement_MyThorn_MyLabel}
%
% Example of a citation:
%    \cite{MyArrangement_MyThorn_Author99}
%
% Example of including a reference
%   \bibitem{MyArrangement_MyThorn_Author99}
%   {J. Author, {\em The Title of the Book, Journal, or periodical}, 1 (1999),
%   1--16. {\tt http://www.nowhere.com/}}
%
% *======================================================================*

% If you are using CVS use this line to give version information
% $Header$

\documentclass{article}
\bibliographystyle{alpha}

% Use the Cactus ThornGuide style file
% (Automatically used from Cactus distribution, if you have a
%  thorn without the Cactus Flesh download this from the Cactus
%  homepage at www.cactuscode.org)
\usepackage{../../../../doc/latex/cactus}

\begin{document}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

% The author of the documentation
\author{Jonathan Thornburg \quad {\tt <jthorn@aei.mpg.de>}}

% The title of the document (not necessarily the name of the Thorn)
\title{{\bf AHFinderDirect} -- A Fast Apparent Horizon Finder}

% the date your document was last changed, if your document is in CVS,
% please us:
%    \date{$ $Date$ $}
\date{$ $Date$ $}

\maketitle

% Do not delete next line
% START CACTUS THORNGUIDE

% Add all definitions used in this documentation here
%   \def\mydef etc

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

% misc text stuff
\def\code#1{{\tt #1}}			% variable names, parameter values, etc
\def\defn#1{``#1''}			% definition of a term
\def\arrangement#1{{\bf #1}}		% name of an arrangement
\def\thorn#1{{\bf #1}}			% name of a thorn
\def\program#1{{\bf #1}}		% name of computer program
\def\cvsplace#1{{\bf #1}}		% name of a CVS repository/directory/tag
\def\cf{\hbox{cf.\hbox{}}}
\def\eg{\hbox{eg.\hbox{}}}
\def\ie{\hbox{i.e.\hbox{}}}
\def\eqref#1{$(\ref{#1})$}

% get size/spacing of "++" right, cf online C++ FAQ question 35.1
\def\Cplusplus{\hbox{C\raise.25ex\hbox{\footnotesize ++}}}

% misc math mode stuff
\def\assign{\leftarrow}			% for pseudocode algorithms
\def\ltsim{\lesssim}
\def\gtsim{\gtrsim}
\def\tfrac#1#2{{\textstyle    \frac{#1}{#2}}}
\def\dfrac#1#2{{\displaystyle \frac{#1}{#2}}}
\def\thalf{\tfrac{1}{2}}
\def\const{{\rm const}}
\def\ij{{ij}}
\def\uv{{uv}}
\def\del{\nabla}

\def\I{{\text{\scriptsize I}}}                  % grid-point index
\def\J{{\text{\scriptsize J}}}                  % grid-point index
\def\K{{\text{\scriptsize K}}}                  % grid-point index
\def\M{{\text{\scriptsize M}}}                  % molecule index

\def\Jac[#1]{{\bf J} \bigl[ #1 \bigr]}          % discrete Jacobian for fn #1

% Add an abstract for this thorn's documentation
\begin{abstract}
Thorn \thorn{AHFinderDirect} locates apparent horizons (or more
generally, closed 2-surfaces with $S^2$ topology having any desired
constant expansion) in a numerically computed slice using a direct
method, posing the apparent horizon equation as an elliptic PDE on
angular-coordinate space.  This is very fast and accurate, but
requires a ``reasonable'' initial guess.  This thorn guide describes
how to use the thorn.
\end{abstract}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Introduction}

A \defn{marginally trapped surface} is a closed 2-surface in a slice whose
congruence of future-pointing outgoing null geodesics has zero expansion.
There may be several such surfaces, some nested inside others; an
\defn{apparent horizon} is an outermost marginally trapped surface.
In terms of the usual $3+1$ variables, an apparent horizon satisfies
the equation
\begin{equation}
\Theta \equiv \del_i n^i + K_\ij n^i n^j - K = 0
					\label{AHFinderDirect/eqn-horizon}
\end{equation}
where $n^i$ is the outward-pointing unit normal to the apparent horizon,
and $\del_i$ is the covariant derivative operator associated with the
3-metric $g_\ij$ in the slice.  (See \cite{AHFinderDirect/York89}
for a derivation of equation~\eqref{AHFinderDirect/eqn-horizon}.)
(Optionally, you can replace the right hand side
of~\eqref{AHFinderDirect/eqn-horizon} by any specified nonzero
constant, \ie{} you can find a surface of constant
(in general nonzero) expansion.; this is dicsussed in
section~\ref{AHFinderDirect/sect-parameters/other-parameters}.)

Thorn~\thorn{AHFinderDirect} finds an apparent horizon by numerically
solving equation~\eqref{AHFinderDirect/eqn-horizon}.  It requires
as input the usual Cactus 3-metric $g_\ij$ and extrinsic curvature
$K_\ij$, (and optionally the conformal factor $\psi$ if the
\thorn{StaticConformal} metric semantics are used),
and produces as output the Cactus $(x,y,z)$ coordinates of a large
number of points on the apparent horizon, together with some auxiliary
information like the apparent horizon area and centroid position, and
the irreducable mass associated with the area.

Besides this thorn guide, the other main sources of information on
\thorn{AHFinderDirect} are the comments in the \verb|param.ccl| file,
the paper \cite{AHFinderDirect/Thornburg2003:AH-finding},
and to a lesser extent the paper \cite{AHFinderDirect/Thornburg95}.
As a courtesy, I ask that both these papers be cited in any published
research which uses this thorn, or which uses code from this thorn.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{What \thorn{AHFinderDirect} Needs}
\label{AHFinderDirect/sect-what-AHFinderDirect-needs}

There are some restrictions on the spacetime, or more precisely
on each slice where you want to find apparent horizons, which are
necessary in order for \thorn{AHFinderDirect} to work:
\begin{itemize}
\item	\thorn{AHFinderDirect} requires that the Cactus geometry
	($g_{ij}$, $K_{ij}$, and optionally $\psi$) be nonsingular
	in a neighborhood of the apparent horizon.  In particular,
	this means that it quite certainly will {\em not\/} work for
	spacetimes/slicings which have a singular geometry on the
	horizon, such as Schwarzschild/Schwarzschild and
	Kerr/Boyer-Lindquist.%%%
\footnote{%%%
	 Nonsingular slicings of those same spacetimes,
	 such as Schwarzschild/Eddington-Finkelstein,
	 Kerr/Kerr, and Kerr/Kerr-Schild, are fine, and
	 are in fact nice test cases.
	 }%%%
\item	Less obviously, this also means that if there is a
	singularity in the geometry somewhere near the apparent
	horizon, then you need to have a high enough Cactus 3-D grid
	resolution that the geometry interpolation doesn't ``see''
	the singularity.  (If \thorn{AHFinderDirect} ``sees'' the
	singularity, it may ``just'' fail to find the horizon,
	and/or it may report that the interpolated $g_{ij}$ fails
	to be positive definite or even contains NaNs.)
\item	At the moment \thorn{AHFinderDirect} and the Cactus interpolators
	don't know how to avoid an excised region, so if the apparent horizon
	(or any trial horizon surface as the algorithm is iterating
	towards the apparent horizon) gets too close to an excised
	region, you'll get garbage results as the interpolator tries
	to interpolate data from the excised region.  I plan to fix
	this sometime soon.
\item	\thorn{AHFinderDirect} requires that any apparent horizon
	it's going to (try to) find must be a \defn{Strahlk\"{o}rper}
	(literally ``ray body'', or more commonly ``star-shaped region'')
	relative to some local coordinate origin (which you must specify).
	A Strahlk\"{o}rper is defined by Minkowski
	(\cite[p.~108]{AHFinderDirect/Schroeder86})
	as
	   \begin{quote}
	   a region in $n$-dimensional Euclidean space containing the
	   origin and whose surface, as seen from the origin, exhibits
	   only one point in any direction.
	   \end{quote}
	In other words, using polar spherical coordinates relative
	to the local coordinate origin, the apparent horizon's shape
	must be parameterizable as $r = h(\text{angle})$ for some
	single-valued function $h: S^2 \to \Re^+$.  (\thorn{AHFinderDirect}
	uses precisely this parameterization.)
\end{itemize}

There are also some restrictions on your Cactus configuration and run;
here's what works and what doesn't:
\begin{itemize}
\item	I {\em strongly\/} recommend using a current-CVS checkout of
	the Cactus flesh and of all relevant thorns.  I haven't tested
	\thorn{AHFinderDirect} at all with older versions of the flesh
	or other thorns.
\item	\thorn{AHFinderDirect} works fine with the \thorn{PUGH} unigrid
	driver and with the \thorn{Carpet} mesh-refinement driver.  So
	far as I know it's never been tested with any other driver.
\item	\thorn{AHFinderDirect} works fine in single- or multi-processor
	Cactus runs.
\item	Obviously, your Cactus configuration must include
	\thorn{AHFinderDirect}, and your \code{ActiveThorns} parameter(s)
	must activate it.
\item	\thorn{AHFinderDirect} inherits from the other thorns
	(strictly speaking, implementations) listed in
	table~\ref{AEIThorns/AHFinderDirect/tab-inherits-from},
	so you'll need them (or more precisely some thorns providing them)
	in your configuration and activated, too.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{table}[tbp]
\begin{center}
\begin{tabular}{l@{\qquad}l}
\thorn{Implementation}		& Typically provided by \thorn{Thorn}	\\
\hline %---------------------------------------------------------------
\thorn{Grid}			& \thorn{CactusBase/CartGrid3d}		\\
\thorn{IO}			& \thorn{CactusBase/IOUtil}		\\
\thorn{ADMBase}			& \thorn{CactusEinstein/ADMBase}	\\
\thorn{StaticConformal}		& \thorn{CactusEinstein/StaticConformal}\\
\thorn{SpaceMask}		& \thorn{CactusEinstein/SpaceMask}	\\
\thorn{SphericalSurface}	& \thorn{AEIThorns/SphericalSurface}	%%%\\
\end{tabular}
\end{center}
\caption[Other Thorns from which \thorn{AHFinderDirect} Inherits]
	{
	This table lists all the other implementations from which
	\thorn{AHFinderDirect} inherits, and the thorns which typically
	provide these implementations.
	}
\label{AEIThorns/AHFinderDirect/tab-inherits-from}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\item	\verb|Grid::domain = "full"|, \verb|"bitant"|,
	\verb|"quadrant"|, and \verb|"octant"| are supported.
	Alas, at present rotating (or more precisely nonlocal)
	symmetry boundary conditions aren't supported.
\item	The \verb|ADMBase::metric_type| values \verb|"physical"| and
	\verb|"static conformal"| are supported; for the latter you must
	have storage turned on for at least the conformal factor
	\verb|StaticConformal::psi|.  (The Cactus 3-D grid functions
	for 1st and 2nd derivatives of \verb|psi| aren't used.)
\item	\thorn{AHFinderDirect} uses the \verb|CCTK_InterpGridArrays()|
	Cactus global (multi-processor grid array) interpolator API;
	this is provided by \thorn{PUGHInterp} or \thorn{CarpetInterp}
	(so you must have the appropriate one of these thorns compiled
	in and activated).  \verb|CCTK_InterpGridArrays()| in turn uses
	the new \verb|CCTK_InterpLocalUniform()| processor-local
	interpolator API; \thorn{AHFinderDirect} uses various options
	in this API which at present are only supported by thorn
	\thorn{AEILocalInterp} (so you must have this thorn compiled in
	and activated)
\item	\thorn{AHFinderDirect} uses various Cactus reduction APIs to
	coordinate multi-processor horizon finding, so (even if you're
	only going to run on a single processor) you must have a reduction
	thorn like \thorn{PUGHReduce} or \thorn{CarpetReduce} compiled in
	and activated.
\item	At present only a few of \thorn{AHFinderDirect}'s parameters
	are steerable.  (This is actually quite easy to fix; I just
	haven't gotten around to it yet.)
\item	I think \thorn{AHFinderDirect} will ``work'' with checkpoint/restart,
	but I haven't tested this yet.  Here ``work'' means the restart
	will be like starting a new run, in that \thorn{AHFinderDirect}
	will set the initial guess for each horizon in a start-of-a-new-run
	manner.  Alas, it will also write a new \verb|BH_diagnostics| file
	for each horizon found, overwriting any existing \verb|BH_diagnostics|
	files.  This is a bug, which I plan to fix soon (the right behavior
	is/will be to {\em append\/} to the existing \verb|BH_diagnostics|
	file).
\end{itemize}

\thorn{AHFinderDirect} can pass information to the rest of Cactus in
several ways; these are described in detail in
section~\ref{AHFinderDirect/sect-parameters/communicating-with-other-thorns}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Obtaining and Compiling \thorn{AHFinderDirect}}

You should be able to obtain the source code for this thorn via the
usual procedures for anonymous git checkout; at present it lives in
the \arrangement{EinsteinAnalysis} arrangement.

This thorn is written primarily in \Cplusplus{}, calling C and
Fortran~77 numerical libraries.%%%
\footnote{%%%
	 The (C) code for the relativity computations
	 (\ie{}~the apparent horizon equation itself)
	 is mainly machine-generated from Maple code,
	 which in turn uses a Perl preprocessor.
	 But the machine-generated C code is already
	 in CVS along with the rest of this thorn's
	 source code, so you don't need to have Maple
	 installed unless you want to modify the
	 relativity computations.%%%
	 }%%%
{}  In theory the code should be quite portable to modern \Cplusplus{}
compilers, but in practice I've had a number of portability problems
with various compilers.  See the ``Code Notes'' and ``Compiler Notes''
sections in the top-level \code{README} file for details and lists of
compilers currently known to be ok or not.

By default \thorn{AHFinderDirect} doesn't use any external libraries.
However, if \code{HAVE\_DENSE\_JACOBIAN\_\_LAPACK} is defined in
\code{src/include/config.h}, then \thorn{AHFinderDirect} uses the
LAPACK library for solving linear equations.  In this case you need
to configure Cactus with \code{LAPACK=yes}.  See the top-level
\code{README} and \code{README.library} files for details on this.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{\thorn{AHFinderDirect} Parameters}

This thorn has lots of parameters, but most of them have reasonable
default values which you probably won't need to change.  Here I describe
the parameters which you are likely to want to at least look at, and
possibly set explicitly.

Note that all of the ``\code{[}$n$\code{]}'' parameters are Cactus
array parameters, which you need to specify separately in your parameter
file for each apparent horizon.  {\bf IMPORTANT: Apparent horizons are
numbered starting at 1, not 0!}  The example in
section~\ref{AHFinderDirect/sect-examples} should make this clear.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\subsection{Overall Parameters}
\label{AHFinderDirect/sect-parameters/overall-parameters}

\begin{description}
\item[\code{find\_every}]
\mbox{}\\
	This is an integer parameter specifying how often
	\thorn{AHFinderDirect} should try to find apparent horizons:
	If $\verb|find_every| = 0$, \thorn{AHFinderDirect} is a no-op.
	If $\verb|find_every| \ne 0$, \thorn{AHFinderDirect} tries to
	find apparent horizons each \verb|find_every| time steps.%%%
\footnote{%%%
	 More precisely, if ${\tt find\_every} \ne 0$,
	 \thorn{AHFinderDirect} tries to find apparent
	 horizons at each time step where {\tt cctk\_iteration}
	 is evenly divisible by {\tt find\_every}.
	 }%%%
{}	The default value is~1, \ie{} \thorn{AHFinderDirect} tries
	to find apparent horizons at every time step.
\item[\code{N\_horizons}]
\mbox{}\\
	How many apparent horizons do you want to find in each slice?
	Typical values are 1 (the default), 2, or 3.%%%
\footnote{%%%
	 Larger values are also possible.  The present upper
	 limit is 100, but it would be trivial to raise this
	 further if desired -- see the comments in \code{param.ccl}
	 for details.
	 }%%%
{}	{\bf This thorn numbers the apparent horizons from 1 to
	\code{N\_horizons} inclusive.}
	There are a number of other parameters (described below)
	which you need to set for of these each apparent horizons.

	Note that \code{N\_horizons} sets the number of apparent horizons
	you want to find {\em in the Cactus 3-D numerical grid\/}, not in
	the whole spacetime.  For example, if you are simulating (say)
	Misner data with \verb|Grid::domain = "bitant"|, with the two
	throats at (say) roughly $z = \pm 1$, then you should set
	\verb|N_horizons = 1| to find those two apparent horizons,
	since you're only finding one apparent horizon within the
	numerical grid.  If you also want to search for a common
	apparent horizon surrounding both black holes, then you should
	set \verb|N_horizons = 2|, since you're finding at most 2
	apparent horizons within the numerical grid.

\item[\code{verbose\_level}]
\mbox{}\\
	This controls how verbose this thorn is in printing
	informational (non-error) messages describing what it's
	doing.  In order from tersest to most verbose, the allowable
	values are
	\begin{description}
	\item[\code{"no output"}]
	\mbox{}\\
		Don't print anything.
	\item[\code{"physics highlights"}]
	\mbox{}\\
		Print only a single line each time
		\thorn{AHFinderDirect} runs, giving which
		horizons were found.
	\item[\code{"physics details"}]
	\mbox{}\\
		Print two lines for each horizon found, giving
		the horizon area, centroid position, and irreducible mass.
		This is the default.
	\item[\code{"algorithm highlights"}]
	\mbox{}\\
		Also print a single line for each Newton iteration giving
		the 2-norm and $\infty$-norm of the $\Theta(h)$ function
		defined by equation~\eqref{AHFinderDirect/eqn-horizon}.
	\item[\code{"algorithm details"}]
	\mbox{}\\
		Print lots of detailed messages tracing what the code
		is doing.
	\item[\code{"algorithm debug"}]
	\mbox{}\\
		Print even more detailed messages tracing what the code
		is doing, mainly useful for debugging purposes.
	\end{description}
\end{description}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\subsection{Choosing the Local Coordinate Origin for each Apparent Horizon}
\label{AHFinderDirect/sect-parameters/local-coordinate-origin}

For each apparent horizon you want to find, you need to specify
the Cactus $(x,y,z)$ coordinates of a local coordinate system origin.
As described in
section~\ref{AHFinderDirect/sect-what-AHFinderDirect-needs},
each apparent horizon must be a Strahlk\"{o}rper with respect to its
local coordinate system origin.

You specify the local coordinate system origin for each horizon with
the (Cactus array) parameters
\begin{description}
\item[%%%
     \begin{tabular}{@{}l@{}}
     \code{origin\_x[}$n$\code{]}	\\
     \code{origin\_y[}$n$\code{]}	\\
     \code{origin\_z[}$n$\code{]}	%%%\\
     \end{tabular}
     ]
\mbox{}\\
	These all default to 0.0.
	In practice, you should set these parameters to be somewhere
	reasonably close to your best guess for the center of each apparent
	horizon.  These aren't {\em too\/} critical: being off by 1/4
	of the horizon radius is no problem, and -- assuming the algorithm
	still converges -- even 1/2 of the horizon radius only slows
	the convergence by an extra iteration or two.  But poor values
	of these parameters do make the algorithm more likely to fail
	to converge.
\end{description}

At present the local coordinate origin is fixed once you set it;
there's no provision for it to move to track a moving black hole.
I hope to add such a provision soon.%%%
\footnote{%%%
	 This would be along the lines of the
	 \thorn{AHFinder} apparent horizon finder's
	 \code{AHFinder::ahf\_wander = true}
	 option.
	 }%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\subsection{Specifying the Initial Guess}

\thorn{AHFinderDirect} requires an initial guess
for the apparent horizon's coordinate position and shape
(that is, for the $h(\text{angle})$ function defined in
section~\ref{AHFinderDirect/sect-what-AHFinderDirect-needs}),
for each apparent horizon you want to find.  Unlike some other apparent
horizon finders (\eg{}~the curvature flow method in \thorn{AHFinder}),
for \thorn{AHFinderDirect} there's no restriction on whether the initial
guess is inside, outside, or crossing the actual apparent horizon: the
only important thing is that it should be ``close''.  Just how close
the initial guess needs to be for \thorn{AHFinderDirect} to find the
(a) apparent horizon depends on the slice and the coordinates, but as
a general rule of thumb any initial guess that's within 1/3 to 1/2 of
the mean horizon radius will probably work.

The ``initial guess'' specification is used the first time we try to
find any given apparent horizon, and also any succeeding time when the
most recent attempt to find this apparent horizon failed.  If we succeed
in finding a given apparent horizon, than that apparent horizon position
is automatically reused as the initial guess the next time we try to
find the same apparent horizon; in this case the explicit ``initial guess''
specification is ignored.%%%
\footnote{%%%
	 This is similar to the \thorn{AHFinder} apparent
	 horizon finder's behavior with the
	 \code{AHFinder::ahf\_guessold = true}
	 option.
	 }%%%

There are a number of parameters for specifying the initial guess:
\begin{description}
\item[\code{initial\_guess\_method[}$n$\code{]}]
\mbox{}\\
	This sets what type of the initial guess is used for each
	apparent horizon position.
	There are several possibilities, most with their own sets
	of subparameters:%%%
\footnote{%%%
	 The naming scheme for these is similar to
	 that used in the \thorn{Exact} thorn.
	 }%%%
$^,$%%%
\footnote{%%%
	 For the Kerr parameters, note that this thorn
	 always uses the convention that the spin parameter
	 $a \equiv J/m^2$ is dimensionless.
	 }%%%
	\begin{description}
	\item[\code{"read from file"}]
	\mbox{}\\
		This reads the initial-guess $h(\text{angle})$ function
		from a data file.  The file format is currently hard-wired
		to be that written with \verb|file_format = "ASCII (gnuplot)"|
		(see below).  The subparameter
		\verb|initial_guess__read_from_named_file__file_name|
		specifies the file name.
	\item[\code{"Kerr/Kerr"}]
	\mbox{}\\
		This sets the initial guess to the analytically-known
		apparent horizon position in a Kerr spacetime
		in Kerr coordinates.  This is a coordinate sphere
		of radius $(1 + \sqrt{1 - a^2}) m$.
		There are subparameters
		\begin{description}
		\item[%%%
		     \begin{tabular}{@{}l@{}}
		     \code{initial\_guess\_\_Kerr\_Kerr\_\_x\_posn[}$n$\code{]} \\
		     \code{initial\_guess\_\_Kerr\_Kerr\_\_y\_posn[}$n$\code{]} \\
		     \code{initial\_guess\_\_Kerr\_Kerr\_\_z\_posn[}$n$\code{]} %%%\\
		     \end{tabular}%%%
		     ]
		\mbox{}\\
			to set the position of the Kerr black hole
			(note this position is in {\em global\/} Cactus
			coordinates, not relative to the local coordinate
			origin), and
		\item[%%%
		     \begin{tabular}{@{}l@{}}
		     \code{initial\_guess\_\_Kerr\_Kerr\_\_mass[}$n$\code{]} \\
		     \code{initial\_guess\_\_Kerr\_Kerr\_\_spin[}$n$\code{]} %%%\\
		     \end{tabular}%%%
		     ]
		\mbox{}\\
			to set its mass and spin.
		\end{description}
	\item[\code{"Kerr/Kerr-Schild"}]
	\mbox{}\\
		This sets the initial guess to the analytically-known
		apparent horizon position in a Kerr spacetime in
		Kerr-Schild coordinates.  This is a coordinate ellipsoid
		with radia (semi-major axes)
		\begin{equation}
		\renewcommand{\arraystretch}{2.0}
		\begin{array}{lclcl}
		r_z	& = &	(1 + \sqrt{1 - a^2}) m			\\
		r_x = r_y
			& = &	r_z \sqrt{1 + \left(\dfrac{am}{r_z}\right)^2}
			& = &	\sqrt{\dfrac{2 r_z}{m}} \, m
									%%%\\
		\end{array}
		\end{equation}
		\begin{description}
		\item[%%%
		     \begin{tabular}{@{}l@{}}
		     \code{initial\_guess\_\_Kerr\_KerrSchild\_\_x\_posn[}$n$\code{]} \\
		     \code{initial\_guess\_\_Kerr\_KerrSchild\_\_y\_posn[}$n$\code{]} \\
		     \code{initial\_guess\_\_Kerr\_KerrSchild\_\_z\_posn[}$n$\code{]} %%%\\
		     \end{tabular}%%%
		     ]
		\mbox{}\\
			(note this position is in {\em global\/} Cactus
			coordinates, not relative to the local coordinate
			origin), and
		\item[%%%
		     \begin{tabular}{@{}l@{}}
		     \code{initial\_guess\_\_Kerr\_KerrSchild\_\_mass[}$n$\code{]} \\
		     \code{initial\_guess\_\_Kerr\_KerrSchild\_\_spin[}$n$\code{]} %%%\\
		     \end{tabular}%%%
		     ]
		\mbox{}\\
			to set its mass and spin.
		\end{description}
	\item[\code{"coordinate sphere"}]
	\mbox{}\\
		This sets the initial guess to a coordinate sphere;
		there are subparameters
		\begin{description}
		\item[%%%
		     \begin{tabular}{@{}l@{}}
		     \code{initial\_guess\_\_coord\_sphere\_\_x\_center[}$n$\code{]} \\
		     \code{initial\_guess\_\_coord\_sphere\_\_y\_center[}$n$\code{]} \\
		     \code{initial\_guess\_\_coord\_sphere\_\_z\_center[}$n$\code{]} %%%\\
		     \end{tabular}%%%
		     ]
		\mbox{}\\
			(note this position is in {\em global\/} Cactus
			coordinates, not relative to the local coordinate
			origin), and
		\item[\code{initial\_guess\_\_coord\_sphere\_\_radius[}$n$\code{]}]
		\mbox{}\\
			to set the radius.
		\end{description}
	\item[\code{"coordinate ellipsoid"}]
	\mbox{}\\
		This sets the initial guess to a coordinate ellipsoid;
		there are subparameters
		\begin{description}
		\item[%%%
		     \begin{tabular}{@{}l@{}}
		     \code{initial\_guess\_\_coord\_ellipsoid\_\_x\_center[}$n$\code{]} \\
		     \code{initial\_guess\_\_coord\_ellipsoid\_\_y\_center[}$n$\code{]} \\
		     \code{initial\_guess\_\_coord\_ellipsoid\_\_z\_center[}$n$\code{]} %%%\\
		     \end{tabular}%%%
		     ]
		\mbox{}\\
			(note this position is in {\em global\/} Cactus
			coordinates, not relative to the local coordinate
			origin), and
		\item[%%%
		     \begin{tabular}{@{}l@{}}
		     \code{initial\_guess\_\_coord\_ellipsoid\_\_x\_radius[}$n$\code{]}	\\
		     \code{initial\_guess\_\_coord\_ellipsoid\_\_y\_radius[}$n$\code{]}	\\
		     \code{initial\_guess\_\_coord\_ellipsoid\_\_z\_radius[}$n$\code{]}	%%%\\
		     \end{tabular}%%%
		     ]
		\mbox{}\\
			to set the radia (semimajor axes).
		\end{description}
	\end{description}
\end{description}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\subsection{I/O Parameters for the Apparent Horizon Shape(s)}

The main output of this thorn is the computed horizon shape function
$h({\rm angle})$, and correspondingly the $(x,y,z)$ coordinate positions
of the apparent-horizon-surface (angular) grid points.  There are several
parameters controlling if, how often, and how these should be written
to data files:

\begin{description}
\item[\code{output\_h\_every}]
\mbox{}\\
	As described in
	section~\ref{AHFinderDirect/sect-parameters/overall-parameters},
	\thorn{AHFinderDirect} will try to find the apparent horizon(s)
	every \verb|find_every| time steps.  However, you can control
	how often (if at all) the apparent horizon shape(s) are written
	to data files: this is only done if \verb|output_h_every|
	is nonzero, and the Cactus time step number \verb|cctk_iteration|
	is an integral multiple of this parameter (\verb|output_h_every|).

\item[\code{file\_format}]
\mbox{}\\
	This specifies the file format for horizon-shape (and other
	angular-grid-function) data files.  Unfortunately, at the
	moment only a single format is implemented,
	\begin{description}
	\item[\code{"ASCII (gnuplot)"}]
	\mbox{}\\
		This is a simple ASCII format designed for easy
		plotting with gnuplot:
		\begin{itemize}
		\item	\thorn{AHFinderDirect} writes a separate data file
			for each Cactus time step and for each apparent
			horizon found.  By default these are all written
			in to the \verb|IOUtil::out_dir| directory;
			see \verb|h_directory| (below) to change this.
		\item	The time step number and the apparent horizon number
			are both encoded in the file name; the actual
			file name is given by a \verb|printf()| format
			\verb|"%s/%s.t%d.ah%d.%s"|, where
			\begin{itemize}
			\item	the first \verb|%s| is the directory
				set by the \verb|IO::out_dir| and/or
				\verb|h_directory| parameters (see below)
			\item	the second \verb|%s| is the base file name
				set by the \verb|h_base_file_name|
				parameter (see below)
			\item	the first \verb|%d| is the
				Cactus time step number
			\item	the second \verb|%d| is the
				apparent horizon number
			\item	the third \verb|%s| is the
				file name extension, set by the
				\verb|ASCII_gnuplot_file_name_extension|
				parameter; this defaults to \verb|"gp"|
			\end{itemize}
		\item	Comment lines begin with \verb|#|.
		\item	Patches are separated by 2 blank lines;
			rows of apparent-horizon points within a patch
			are separated by single blank lines.
		\item	Each apparent-horizon-surface point
			is described by a single line, containing
			the whitespace-separated fields
			\begin{itemize}
			\item	Two ``unwrapped'' angular coordinates
				in degrees, representing a point on $S^2$.
			\item	The $h$ value, giving the radius
				of the apparent horizon surface
				at that angle.
			\item	The corresponding Cactus $(x,y,z)$
				coordinates.
			\end{itemize}
		\end{itemize}
		See section~\ref{AHFinderDirect/sect-visualization}
		for a discussion of visualization for these and other
		\thorn{AHFinderDirect} output files.
	\end{description}

\item[\code{h\_directory}]
\mbox{}\\
	This specifies the directory in which the $h$ data files
	are to be written.  If it doesn't already exist, this directory
	is created before writing the data files.  This parameter
	defaults to the value of the \verb|IO::out_dir| parameter.

\item[\code{h\_base\_file\_name}]
\mbox{}\\
	This specifies the base file name for $h$ data files, as
	described above.  This defaults to "h".

\item[\code{ASCII\_gnuplot\_file\_name\_extension}]
\mbox{}\\
	This specifies the file name extension for $h$ data files, as
	described above.  This defaults to "gp".
\end{description}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\subsection{Parameters for the ``BH\_diagnostics'' Files}
\label{AHFinderDirect/sect-parameters/BH-diagnostics-parameters}

As well as the apparent horizon shape files, this thorn can also
write files giving time series of various diagnostics.  These are
controlled by the following parameters:

\begin{description}
\item[\code{output\_BH\_diagnostics}]
\mbox{}\\
	If this Boolean parameter is set to true,
	\thorn{AHFinderDirect} will write a ``black hole diagnostics''
	file for each distinct apparent horizon found
	(up to \verb|N_horizons| files in all).  Each such file
	contains a time series of various diagnostics for all time
	steps when the corresponding apparent horizon was found.
	The file format is again a simple ASCII format designed
	for easy plotting with gnuplot:
	\begin{itemize}
	\item	The apparent horizon number is encoded in the file name;
		the actual file name is given by a \verb|printf()| format
		\verb|"%s/%s.ah%d.%s"|, where
		\begin{itemize}
		\item	the first \verb|%s| is the directory
			set by the \verb|IO::out_dir| and/or
			\verb|BH_diagnostics_directory| parameters (see below)
		\item	the second \verb|%s| is the base file name
			set by the \verb|BH_diagnostics_base_file_name|
			parameter (see below); this defaults to
			\verb|"BH_diagnostics"|
		\item	the \verb|%d| is the apparent horizon number
		\item	the third \verb|%s| is the file name extension,
			set by the \verb|BH_diagnostics_file_name_extension|
			parameter; this defaults to \verb|"gp"|
		\end{itemize}
	\item	The file begins with a block of header comments
		(lines begining with \verb|#|) describing the data
		fields.
	\item	Each time this apparent horizon is found, a single
		line is appended to the data file, containing various
		whitespace-separated fields.  The the precise list of
		fields, see the header comments, or see the function
		\verb|output()| in \verb|src/driver/BH_diagnostics.cc|.
		As of this writing the fields are:
		\begin{itemize}
		\item	the Cactus iteration number \verb|cctk_iteration|
		\item	the Cactus time coordinate \verb|cctk_time|
		\item	the Cactus $(x,y,z)$ coordinates of the apparent
			horizon centroid
		\item	the minimum, maximum, and mean coordinate radia
			of the apparent horizon about the local
			coordinate origin
		\item	the minimum and maximum Cactus $x$, $y$, and $z$
			coordinates of the apparent horizon surface
		\item	the proper circumferences of the apparent horizon
			in the $xy$, $xz$, and $yz$ local-coordinate planes%%%
\footnote{%%%
	 Note that if the apparent horizon is not
	 symmetric across one of these coordinate planes,
	 then the circumference in this plane is not
	 very meaningful.  In particular, in this case
	 this circumference is probably different from
	 the apparent horizon's maximum ``hoop'' circumference.
	 }%%%
		\item	the $xz/xy$ and $yz/xy$ ratios of the proper
			circumferences
		\item	the proper area of the apparent horizon, $A$
		\item	the irreducible mass of the apparent horizon,
			$\sqrt{A/16\pi}$
		\item	the areal radius of the apparent horizon,
			$\sqrt{A/4\pi}$
		\end{itemize}
	\end{itemize}

\item[\code{BH\_diagnostics\_directory}]
\mbox{}\\
	This specifies the directory in which the black hole diagnostics
	data files are to be written.  If it doesn't already exist, this
	directory is created before writing the data files.  This parameter
	defaults to the value of the \verb|IO::out_dir| parameter.

\item[\code{BH\_diagnostics\_base\_file\_name}]
\mbox{}\\
	This specifies the base file name for black hole diagnostics
	data files, as described above.  This defaults to
	\verb|"BH_diagnostics"|.

\item[\code{BH\_diagnostics\_file\_name\_extension}]
\mbox{}\\
	This specifies the file name extension for black hole diagnostics
	data files, as described above.  This defaults to \verb|"gp"|.
\end{description}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\subsection{(Excision) Mask Parameters}
\label{AHFinderDirect/sect-parameters/mask-parameters}

This thorn can optionally set a mask grid function (or functions)
at each point of the Cactus grid, to indicate where that point is
with respect to the apparent horizon(s).  This is usually used for
excision.

\begin{description}
\item[%%%
     \begin{tabular}{@{}l@{}}
     \code{set\_mask\_for\_all\_horizons}			\\
     \code{set\_mask\_for\_individual\_horizon[}$n$\code{]}	%%%\\
     \end{tabular}
     ]
\mbox{}\\
	These Boolean parameters control whether \thorn{AHFinderDirect}
	should set a mask grid function(s): If the (C) expression
	\verb:set_mask_for_all_horizons || set_mask_for_individual_horizon[i]:
	is true, then \thorn{AHFinderDirect} will set a mask grid
	function(s) for apparent horizon $i$.
	All these parameters default to false (don't set a mask).

	If any of these parameters is set to true, you almost certainly
	also need to set the Boolean parameter \verb|SpaceMask::use_mask|
	to true to to turn on storage for the mask grid function(s)!
\end{description}

If it's setting a mask(s), \thorn{AHFinderDirect} partitions the
Cactus grid into 3~regions: an \defn{inside}, a \defn{buffer},
and an \defn{outside}.  Typically the inner region is excised, but
\thorn{AHFinderDirect} doesn't itself do this:  It just sets the mask(s);
you need to use some other thorn(s) to do the actual excision.

The 3~regions are defined as follows:  For a grid point a distance $r[i]$
from horizon~$i$'s local coordinate origin, with horizon~$i$'s radius
in this same direction (again, measured from its local coordinate origin)
being $r_{\text horizon}[i]$,%%%
\footnote{%%%
	 Note that these are flat-space distances
	 $[(x-x_0)^2 + (y - y_0)^2 + (z-z_0)^2]^{1/2}$
	 -- the spacetime metric is {\em not\/} used here.
	 }%%%
{} the regions are defined by
\begin{equation}
\renewcommand{\arraystretch}{1.333}
\begin{tabular}{l@{~}lll@{~}lll}
$r[i] \le r{	ext{r}}[i]$	& \text{for some $i$}
	&
	&				&
	& $\Rightarrow$	& \text{inner}					\\
$r[i] >   r_{\text{r}}[i]$	& \text{for all $i$}
	& \text{and}	&
	$r[i] \le r_{\text{r}}[i]$	& \text{for some $i$}
	& $\Rightarrow$	& \text{buffer}					\\
$r[i] >   r_{\text{r}}[i]$	& \text{for all $i$}
	&
	&				&
	& $\Rightarrow$	& \text{outer}				%%%\\
\end{tabular}
\end{equation}
where
\begin{equation}
\renewcommand{\arraystretch}{1.333}
\begin{array}{lcl}
r_{\text{r}}
	& = &	\verb|mask_radius_multiplier| \times r_{\text{n}}
		\,\,+\,\,
		\verb|mask_radius_offset| \times {\Delta x}_{\text{e}}
									\\
r_{\text{r}}
	& = &	r_{\text{r}}
		\,\,+\,\,
		\verb|mask_buffer_thickness| \times {\Delta x}_{\text{e}}
									%%%\\
\end{array}
				\label{AHFinderDirect/eqn-r-inside-outside}
\end{equation}
and where ${\Delta x}_{\text{e}}$ is the base-grid Cactus grid spacing
(more precisely, the geometric mean of the base grid's $x$, $y$,
and $z$ Cactus grid spacings)%%%
\footnote{%%%
	 The ``base grid'' part here only matters if
	 you're doing mesh refinement, in which case
	 it's necessary to keep excision consistent
	 between the different refinement levels.
	 }%%%
.

\begin{description}
\item[%%%
     \begin{tabular}{@{}l@{}}
     \code{mask\_radius\_multiplier}	\\
     \code{mask\_radius\_offset}	\\
     \code{mask\_buffer\_thickness}	%%%\\
     \end{tabular}
     ]
\mbox{}\\
	These parameters are used to define the radia $r_{\text{r}}$
	and $r_{\text{r}}$ in
	equation~\eqref{AHFinderDirect/eqn-r-inside-outside} above.

	Note that the sign convention here is that
	\verb|mask_radius_multiplier| is {\em multiplied\/}
	by the horizon radius, then \verb|mask_radius_offset|
	(scaled by the Cactus grid spacing) is {\em added\/}.
	Thus for use with excision (where the inner region --
	which will be excised -- must be somewhat {\em inside\/}
	the horizon), \verb|mask_radius_multiplier| should be a
	positive real number slightly less than~1.0, and/or
	\verb|mask_radius_offset| a negative real number.

	The default values for these parameters are
	\begin{verbatim}
	mask_radius_multiplier =  0.8
	mask_radius_offset     = -5.0
	mask_buffer_thickness  =  5.0
	\end{verbatim}

\item[\code{mask\_is\_noshrink}]
\mbox{}\\
	This Boolean parameter specifies whether the inside and
	buffer regions should be prevented from ever shrinking
	during a time evolution (this is the default), or whether
	they should be set independently from one time step to the
	next (and thus allowed to either grow or shrink).  More
	precisely, once a given grid point has been classified as
	inside, buffer, or outside, \thorn{AHFinderDirect} executes
	the following algorithm:
	\begin{description}
	\item[inside]\mbox{}\\[-\baselineskip]
		\begin{tabbing}
		$\verb|mask| \assign \verb|inside_value|$		  %%%\\
		\end{tabbing}
	\item[buffer]\mbox{}\\[-\baselineskip]
		\begin{tabbing}
		if \=(\verb|mask_is_noshrink|
		      \verb|&&|
		      ($\verb|mask| = \verb|inside_value|$))		\+\\
		   then \=\{						\+\\
			\verb|#|
			this point was previously {\bf inside}
			$\Rightarrow$ no-op here			  \\
			\}						\-\\
		   else \>$\verb|mask| \assign \verb|buffer_value|$	\-%%%\\
		\end{tabbing}
	\item[outside]\mbox{}\\[-\baselineskip]
		\begin{tabbing}
		if \=(\verb|mask_is_noshrink|
		      \verb|&&|
		      (($\verb|mask| = \verb|inside_value|$)
		       \verb.||. ($\verb|mask| = \verb|buffer_value|$))	\+\\
		   then \=\{						\+\\
			\verb|#|
			this point was previously {\bf inside}
			or {\bf buffer} $\Rightarrow$ no-op here	  \\
			\}						\-\\
		   else \>$\verb|mask| \assign \verb|outside_value|$	\-%%%\\
		\end{tabbing}
	\end{description}

\item[\code{min\_horizon\_radius\_points\_for\_mask}]
\mbox{}\\
	By default, \thorn{AHFinderDirect} sets the mask for each
	apparent horizon found.  If we're using mesh refinement, it's
	possible for an apparent horizon to be found on a coarse grid,
	and the masked region to be only a few grid points across on
	a fine grid.  This causes some other Cactus thorns
	(\eg{} \thorn{LegoExcision}) to crash. :(

	This parameter can be used to avoid this problem:
	For each apparent horizon that it finds, \thorn{AHFinderDirect}
	only sets the mask on a given grid if
	\begin{equation}
	r_{\text{inner},\min} \ge \verb|min_horizon_radius_points_for_mask|
		     * {\Delta x}_{\text{current},\max}
	\end{equation}
	where $r_{\text{inner},\min}$ is the minimum over all angles
	of $r_{\text{r}}$ as defined by
	equation~\eqref{AHFinderDirect/eqn-r-inside-outside},
	and ${\Delta x}_{\text{current},\max}$ is the maximum of the
	Cactus $x$, $y$, and~$z$ grid spacings on the current Cactus
	grid.%%%
\footnote{%%%
	 Note that this is the {\bf current\/} Cactus
	 grid spacing, not the {\bf base\/} Cactus grid
	 spacing that's used when calculating
	 $r_{\text{r}}$ and $r_{\text{r}}$ by
	 equation~\eqref{AHFinderDirect/eqn-r-inside-outside}.
	 }%%%
{}	If this condition isn't satisfied, then \thorn{AHFinderDirect}
	skips setting the mask for this apparent horizon, just as if
	this apparent horizon wasn't found.  (Note that all other
	processing for an apparent horizon being found is still done,
	including writing output files, using the apparent horizon shape
	as the initial guess for the next time step's apparent-horizon
	finding, etc.; it's only the mask processing for this horizon
	(at this time step) that's skipped.)
\end{description}

\thorn{AHFinderDirect} supports two types of mask grid functions;
the following two Boolean parameters choose which of them you want
to set; you can set either or even both of these:
\begin{description}
\item[\code{set\_old\_style\_mask}]
\mbox{}\\
	This parameter (default \verb|true|) specifies an
	old-style excision mask, one stored in a \verb|CCTK_REAL|
	Cactus grid function.  (The \thorn{AHFinder} apparent horizon
	finder uses this type of mask.)

\item[\code{set\_new\_style\_mask}]
\mbox{}\\
	This parameter (default \verb|false|) specifies a
	new-style excision mask, one stored in a specified bit field
	of a \verb|CCTK_INT| Cactus grid function.
	The bit field is specified by its name,
	as registered with the \thorn{SpaceMask} thorn.
	We plan to eventually convert all Cactus excision
	(and other uses of mask grid functions) to this scheme,
	but at the moment not much code supports it.
	Note that \thorn{AHFinderDirect} doesn't itself create/register
	any bit fields or state names with \thorn{SpaceMask} --
	you must arrange for some other thorn(s) to do this.
\end{description}

For an old-style mask, the following parameters specify the mask grid
function and how it should be set:
\begin{description}
\item[\code{old\_style\_mask\_gridfn\_name}]
\mbox{}\\
	This parameter specifies the mask grid function's name.

\item[%%%
     \begin{tabular}{@{}l@{}}
     \code{old\_style\_mask\_inside\_value}	\\
     \code{old\_style\_mask\_buffer\_value}	\\
     \code{old\_style\_mask\_outside\_value}	%%%\\
     \end{tabular}
     ]
\mbox{}\\
	If an old-style mask is to be set in the corresponding
	regions, these parameters specify the values to which
	it should be set.  These are all \verb|CCTK_REAL| values.
\end{description}

For an new-style mask, the following parameters specify the mask grid
function and how it should be set:
\begin{description}
\item[%%%
     \begin{tabular}{@{}l@{}}
     \code{new\_style\_mask\_gridfn\_name}	\\
     \code{new\_style\_mask\_bitfield\_name}	%%%\\
     \end{tabular}
     ]
\mbox{}\\
	These parameters specify the mask grid function's name
	and the bitfield name within it.

\item[%%%
     \begin{tabular}{@{}l@{}}
     \code{new\_style\_mask\_inside\_value}	\\
     \code{new\_style\_mask\_buffer\_value}	\\
     \code{new\_style\_mask\_outside\_value}	%%%\\
     \end{tabular}
     ]
\mbox{}\\
	If an new-style mask is to be set in the corresponding
	regions, these parameters specify the values to which
	it should be set.  These are all character-string state
	names, as registered with the \thorn{SpaceMask} thorn.
\end{description}
Note that \thorn{AHFinderDirect} doesn't itself register any bitfields
or states with \thorn{SpaceMask} -- you must arrange for some other thorn(s)
to do this before \thorn{AHFinderDirect} tries to find the horizon(s).

If \thorn{AHFinderDirect} sets a mask or masks, this happens in the
same schedule bin(s) as the horizon finding.  More precisely,
\thorn{AHFinderDirect} creates two schedule groups for this purpose:
\begin{itemize}
\item	The schedule group \verb|group_for_mask_stuff| is
	scheduled to run just after the horizon is found.
\item	The schedule group \verb|group_where_mask_is_set|
	is scheduled inside the schedule group \verb|group_for_mask_stuff|.
\item	The actual setting of the mask is scheduled inside
	the schedule group \verb|group_where_mask_is_set|.
\end{itemize}
Thorn \thorn{PreviousMask} uses these schedule groups to keep a
``previous'' as well as a ``current'' mask.  See that thorn's thorn
guide for further details.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\subsection{Communicating with Other Thorns}
\label{AHFinderDirect/sect-parameters/communicating-with-other-thorns}

Besides the data files it writes, \thorn{AHFinderDirect} currently
has three ways to communicate with other Cactus thorns:
\begin{itemize}
\item	\thorn{AHFinderDirect} can set a mask grid function(s) based
	on the (a) horizon's shape; other thorns may then use this for
	excision or other purposes.  \thorn{AHFinderDirect} supports
	both the old-style (\verb|CCTK_REAL|) mask
	(compatible with \thorn{AHFinder}) or
	the new-style (\verb|CCTK_INT|) mask bit-fields defined by
	\thorn{SpaceMask}; you can even use both styles simultaneously.
\item	\thorn{AHFinderDirect} can announce a selected horizon's centroid
	position to another thorn (typically \thorn{DriftCorrect}, which
	uses this to adjust its corotating shift vector).
	This uses the new function-aliasing version of \thorn{DriftCorrect},
	not the old version which worked with an auxiliary thorn
	\thorn{AHFSetDCCentroid}.
\item	\thorn{AHFinderDirect} provides a set of aliased functions
	which any other thorn(s) can call to find out the shape of
	a specified horizon.
\item	\thorn{AHFinderDirect} can store information about the
	horizon(s) it finds in the \thorn{SphericalSurface} variables
	for other thorns to use.
\end{itemize}

\thorn{AHFinderDirect}'s mask features are described in
section~\ref{AHFinderDirect/sect-parameters/mask-parameters};
the other communication mechanisms are described in the following
subsections.

%%%%%%%%%%%%%%%%%%%%

\subsubsection{Parameters for Announcing a Horizon Centroid to Other Thorns}

This thorn can optionally announce the centroid of a specified
apparent horizon to another thorn (typically \thorn{DriftCorrect})
each time that apparent horizon is found.  This is controlled by the
following parameter:

\begin{description}
\item[\code{which\_horizon\_to\_announce\_centroid}]
\mbox{}\\
	This is an integer parameter which defaults to
	0 (which means not to announce any centroid).
	If it's set to a nonzero integer, that specifies the
	horizon number to have its centroid announced.
\end{description}

%%%%%%%%%%%%%%%%%%%%

\subsubsection{Aliased Functions to Provide Horizon-Shape Information}

\thorn{AHFinderDirect} provides the following aliased functions
to allow other thorns to find out about the horizons.  Each function
returns a status code which is $\ge 0$~for ok, or negative for an error.
\begin{verbatim}
#
# This function returns the local coordinate origin for a given horizon.
#
CCTK_INT FUNCTION HorizonLocalCoordinateOrigin                          \
   (CCTK_INT IN horizon_number,                                         \
    CCTK_REAL OUT origin_x, CCTK_REAL OUT origin_y, CCTK_REAL OUT origin_z)

#
# The following function queries whether or not the specified horizon
# was found the most recent time AHFinderDirect searched for it.
# The return value is:
#  1 if the horizon was found
#  0 if the horizon was not found
#  negative for an error
#
CCTK_INT FUNCTION HorizonWasFound(CCTK_INT IN horizon_number)

#
# The following function computes the horizon radius in the direction
# of each (x,y,z) point, or -1.0 if this horizon wasn't found the most
# recent time AHFinderDirect searched for it.  More precisely, for each
# (x,y,z), consider the ray from the local coordinate origin through
# (x,y,z).  This function computes the Euclidean distance between the
# local coordinate origin and this ray's intersection with the horizon,
# or -1.0 if this horizon wasn't found the most recent time AHFinderDirect
# searched for it.  
#
# If this function is to be used in a multiprocessor run on a horizon
# which was found on some other processor, the parameter
#   AHFinderDirect::always_broadcast_horizon_shape
# must be set to true to get the correct answer.
#
CCTK_INT FUNCTION HorizonRadiusInDirection                              \
   (CCTK_INT IN horizon_number,                                         \
    CCTK_INT IN N_points,                                               \
    CCTK_REAL IN ARRAY x, CCTK_REAL IN ARRAY y, CCTK_REAL IN ARRAY z,   \
    CCTK_REAL OUT ARRAY radius)
\end{verbatim}

%%%%%%%%%%%%%%%%%%%%

\subsubsection{Storing Horizon-Shape Information in the \thorn{SphericalSurface} Variables}

\thorn{SphericalSurface} (in the \thorn{AEIThorns} arrangement)
defines a set of generic grid arrays which describe ``spherical surfaces''.
\thorn{AHFinderDirect} can optionally store information about the
horizons it finds in the \thorn{SphericalSurface} variables.
This is controlled by the following parameters:
\begin{description}
\item[\code{which\_surface\_to\_store\_info[}$n$\code{]}]
\mbox{}\\
	This parameter should be set to the \thorn{SphericalSurface}
	surface number into which information on a given \thorn{AHFinderDirect}
	horizon should be stored, or to -1 to skip storing the information.
	It defaults to -1 for each horizon.

	{\bf Note that \thorn{SphericalSurface} numbers surfaces
	starting from 0, whereas \thorn{AHFinderDirect} numbers
	horizons starting from 1!}

	At present, if multiple \thorn{AHFinderDirect} horizons
	specify the same \thorn{SphericalSurface} surface, the
	highest-numbered horizon will ``win'', \ie{} it will overwrite
	the data from any lower-numbered horizons.
\end{description}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\subsection{Other Parameters}
\label{AHFinderDirect/sect-parameters/other-parameters}

\begin{description}
\item[%%%
     \begin{tabular}{@{}l@{}}
     \code{run\_at\_CCTK\_ANALYSIS}			\\
     \code{run\_at\_CCTK\_POSTSTEP}			\\
     \code{run\_at\_CCTK\_POSTINITIAL}			\\
     \code{run\_at\_CCTK\_POST\_RECOVER\_VARIABLES}	%%%\\
     \end{tabular}
     ]
\mbox{}\\
	These parameters (which default to \verb|true|, \verb|false|,
	\verb|false|, and \verb|true| respectively) control which
	schedule bins \thorn{AHFinderDirect} runs in.  Historically,
	\thorn{AHFinderDirect} ran in CCTK\_ANALYSIS, and that's
	still the default, but these parameters allow you to change this
	so it runs in CCTK\_POSTSTEP and/or CCTK\_POSTINITIAL
	instead.  (You can even run in all three bins if you want!)

	In general we need to run at CCTK\_POST\_RECOVER\_VARIABLES, since
	\begin{itemize}
	\item	parameters may have been steered at recovery, so we may need
		to find a new horizon or horizons, and
	\item	we need to set the mask again to make sure it's correct
		right away (since our next regular horizon-finding may not
		be until some time steps later)
	\end{itemize}
	Therefore the \verb|run_at_CCTK_POST_RECOVER_VARIABLES| parameter
	should probably be left at its default setting of \verb|true|.

\item[%%%
     \begin{tabular}{@{}l@{}}
     \code{geometry\_interpolator\_name}	\\
     \code{geometry\_interpolator\_pars}	%%%\\
     \end{tabular}
     ]
\mbox{}\\
	These parameters control the (3-D) \defn{geometry interpolation}
	of the spacetime geometry ($g_{ij}$ and $K_{ij}$, or their
	\thorn{StaticConformal} equivalents) to the apparent horizon
	position.  The defaults are set to use a quadratic Hermite
	interpolator.  This works fairly well, but because of the interpolator
	molecule size you must use $\verb|driver::ghost_size| \ge 2$.

	If you want to get very high accuracy from \thorn{AHFinderDirect},
	then you should use a cubic Hermite geometry interpolator, by
	setting
\begin{verbatim}
AHFinderDirect::geometry_interpolator_pars = "
  order=3
  boundary_off_centering_tolerance={1.0e-10 1.0e-10 1.0e-10 1.0e-10 1.0e-10 1.0e-10}
  boundary_extrapolation_tolerance={0.0 0.0 0.0 0.0 0.0 0.0}
                                             "
\end{verbatim}
	Assuming perfectly accurate geometry variables in the 3-D Cactus grid,
	this will make \thorn{AHFinderDirect} (very) roughly an order
	of magnitude more accurate.  However, the larger molecule size
	will make it about a factor of 2--3~slower, and will also require
	that you set $\verb|driver::ghost_size| \ge 3$.
	The sample parameter file \verb|par/Kerr-order3.par|
	shows an example of this.

\item[\code{N\_zones\_per\_right\_angle[}$n$\code{]}]
\mbox{}\\
	This parameter sets the angular resolution used to compute
	each patch.  The units are the number of angular grid zones
	per right angle.  The default is 18, \ie{} a 5~degree angular
	resolution.  There's no problem with this parameter varying
	from one horizon to another, but for simplicity it should be
	even.%%%
\footnote{%%%
	 More precisely, it {\em must\/} be even for
	 any horizon whose patch system type is anything
	 other than {\tt "full sphere"}.  See the comments
	 in {\tt param.ccl} for further information.%%%
	 }%%%

{}	For any horizon which is close to spherical about its
	local coordinate origin, you can lower this parameter to make
	\thorn{AHFinderDirect} run faster (typical run-times scale
	roughly as the cube of this parameter); 6~is about the
	minimum reasonable value.

	For any horizon which is highly non-spherical about its
	local coordinate origin, you can raise this parameter to
	get better resolution; 30~should be enough for even a
	highly non-spherical horizon.

\item[\code{max\_allowable\_horizon\_radius[}$n$\code{]}]
\mbox{}\\
	This parameter gives the maximum mean-coordinate-radius
	which any given trial surface may have in the course of
	trying to solve the apparent horizon equation.%%%
\footnote{%%%
	 Note that this is an unweighted arithmetic
	 mean over the horizon-surface grid points,
	 the same value which is printed as \code{r\_grid}
	 during the horizon-finding iterations.
	 }%%%
{}	In particular, if any trial surface has a mean coordinate radius
	which exceeds this parameter, \thorn{AHFinderDirect} gives up
	and deems this apparent horizon to be ``not found''.

	This parameter defaults to $10^{10}$ (effectively $+\infty$)
	for each apparent horizon.  You can set it to a smaller value
	to make \thorn{AHFinderDirect} a bit more efficient, or
	(probably more important in practice) to stop
	\thorn{AHFinderDirect} from iterating off the edge of the grid
	if this causes problems with interpolation or boundary conditions.

\item[\code{max\_allowable\_Theta}]
\mbox{}\\
	This parameter gives the maximum $\|\Theta\|_\infty$
	which any given trial surface may have in the course of
	trying to solve the apparent horizon equation.
	In particular, if any trial surface has $\|\Theta\|_\infty$
	exceeding this parameter, \thorn{AHFinderDirect} gives up
	and deems this apparent horizon to be ``not found''.
	This parameter defaults to $10^{10}$ (effectively $+\infty$)
	for each apparent horizon.

\item[\code{surface\_expansion[}$n$\code{]}]
\mbox{}\\
	This parameter (which defaults to 0.0) sets the expansion
	of each surface.  With the default setting this thorn
	solves~\eqref{AHFinderDirect/eqn-horizon} to find apparent
	horizons.  With other settings of this parameter this thorn
	can be used to find \defn{surfaces of constant expansion};
	these may be useful for excision, wave extraction, or
	other purposes (\cite{AHFinderDirect/Schnetter03a}).

	To help in choosing the value(s) of the
	\verb|surface_expansion[|$n$\verb|]| parameter,
	figure~\ref{AHFinderDirect/fig-Schwarzschild-EF-Theta(r)}
	(from \cite{AHFinderDirect/Thornburg95}),
	shows the expansion of $r = \text{constant}$ surfaces
	in an Eddington-Finkelsteon slice of the unit-mass
	Schwarzschild spacetime.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{figure}[htbp]
\begin{center}
\includegraphics{AEIThorns_AHFinderDirect_Schw_EF_Theta_of_r}
\end{center}
\caption[Expansion $\Theta$ for $r = \text{constant}$ Surfaces
	 in an Eddington-Finkelstein slice
	 of the Unit-Mass Schwarzschild Spacetime]
	{
	This figure shows the expansion $\Theta$ for $r = \text{constant}$
	surfaces in an Eddington-Finkelstein slice of the unit-Mass
	Schwarzschild Spacetime.
	}
\label{AHFinderDirect/fig-Schwarzschild-EF-Theta(r)}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\end{description}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Monitoring \thorn{AHFinderDirect}'s Status}

There are two primary ways of monitoring what \thorn{AHFinderDirect}
is doing during a Cactus run: the \verb|BH_diagnostics| files and the
\verb|CCTK_INFO| messages written to the Cactus standard output:

The \verb|BH_diagnostics| files are described in detail in
section~\ref{AHFinderDirect/sect-parameters/BH-diagnostics-parameters}.
These files are written and ``flushed'' at each time step, so they're
always up-to-date.

During the apparent-horizon--finding process, \thorn{AHFinderDirect}
writes various \verb|CCTK_INFO| messages describing the convergence
of the iterative solution of the apparent horizon
equation~\ref{AHFinderDirect/eqn-horizon} on each processor.
In particular, if \verb|verbose_level| is set to
\verb|"algorithm highlights"| or a more verbose setting
(\cf{}~section~\ref{AHFinderDirect/sect-parameters/overall-parameters}),
then \thorn{AHFinderDirect} writes \verb|CCTK_INFO| messages like
these:
\begin{verbatim}
INFO (AHFinderDirect):    proc 0/horizon 1:it 1 r_grid=0.595 ||Theta||=1.1e-01
INFO (AHFinderDirect):    proc 0/horizon 1:it 2 r_grid=0.614 ||Theta||=7.2e-02
INFO (AHFinderDirect):    proc 0/horizon 1:it 3 r_grid=0.632 ||Theta||=2.9e-02
INFO (AHFinderDirect):    proc 0/horizon 1:it 4 r_grid=0.642 ||Theta||=9.9e-04
INFO (AHFinderDirect):    proc 0/horizon 1:it 5 r_grid=0.642 ||Theta||=7.9e-07
INFO (AHFinderDirect):    proc 0/horizon 1:it 6 r_grid=0.642 ||Theta||=7.2e-13
INFO (AHFinderDirect): AH 1/2: r=0.660716 at (0.000000,0.000000,1.127434)
INFO (AHFinderDirect): AH 1/2: area=338.0473838 m_irreducible=2.59330658
INFO (AHFinderDirect): writing h to "misner.h.t0.ah1.gp"
\end{verbatim}
Here \verb|r_grid| is a rough estimate of the mean radius of
the trial surface at each iteration, and \verb:||Theta||: is the
infinity-norm of $\Theta$, the left hand side of the apparent
horizon equation~\ref{AHFinderDirect/eqn-horizon} over the surface.
Once the apparent horizon has been found
(\verb:||Theta||: is sufficiently small), then \thorn{AHFinderDirect}
prints its mean radius,%%%
\footnote{%%%
	 Note that this may differ significantly from
	 the last iterations' estimated radius.  This
	 is because the mean radius printed during the
	 iterations is only a rough estimate (it's the
	 unweighted arithmetic mean of the radius at
	 all the horizon-surface grid points), while
	 the radius printed after the horizon is found
	 is a more accurate value (it's computed via
	 numerical integrals over the surface, taking
	 into account the induced metric).
	 }%%%
{} centroid position, area, and irreducible mass.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Visualization}
\label{AHFinderDirect/sect-visualization}

There are several ways to visualize \thorn{AHFinderDirect}'s output:

The simplest is to plot various quantities from the
\verb|BH_diagnostics| files (described in detail in
section~\ref{AHFinderDirect/sect-parameters/BH-diagnostics-parameters}).
For example, using gnuplot (\verb|http://www.gnuplot.info|), you can
plot a graph of the surface area of horizon \#4 as a function of
coordinate time, with the command
\begin{verbatim}
plot 'BH_diagnostics.ah4.gp' using 2:26 with points
\end{verbatim}
ygraph (\verb|http://www.aei.mpg.de/~pollney/ygraph/|) may also be
able to directly plot the \verb|BH_diagnostics| files.

Given a horizon-shape data file \verb|h.t105.h4.gp|, the gnuplot
command
\begin{verbatim}
splot 'h.t105.h4.gp' with lines
\end{verbatim}
will plot the $h(\text{angle})$ function, with the $x$ and $y$~axes
of the plot being the two ``unwrapped'' angular coordinates on $S^2$,
in degrees, and the $z$~axis being $h(\text{angle})$.  However,
in practice this usually isn't very informative.  Instead, you
probably want the gnuplot command
\begin{verbatim}
splot 'h.t105.h4.gp' using 4:5:6 with lines
\end{verbatim}
which will plot the 3-D shape of the apparent horizon surface.

The \verb|src/misc| directory of the \thorn{AHFinderDirect} source
code contains several perl scripts which are useful in visualizing
\thorn{AHFinderDirect} output.  In particular, the script \verb|select.plane|
selects a particular 2-D plane.  If you put this in your Unix path,
it can be used with a gnuplot command like
\begin{verbatim}
splot '<select.plane xy <h.t105.h4.gp' using 4:5 with lines
\end{verbatim}
to show the shape of an apparent horizon in the xy plane.%%%
\footnote{%%%
	 Note that this script isn't very smart: it doesn't
	 know about the \thorn{AHFinderDirect} local coordinate
	 origin.  That is, for example, if you select the $xy$
	 plane, you're selecting the \emph{global} $xy$ plane
	 ($z=0$).
	 }%%%

Another Visualization option is OpenDX (\verb|http://www.opendx.org/|).
Thomas Radke's has written some OpenDX macros
\program{ImportAHFinderDirectGnuplot.net}
and \program{ImportAHFinderDirectGnuplotPatch.net}
to import \thorn{AHFinderDirect} horizon-shape data files.  
These macros use a set of ``control files'' named \verb|*.dx|,
one per horizon, which \thorn{AHFinderDirect} (by default) writes
into the same directory as the main horizon--shape output files.
You can get these macros by anonymous CVS with the command
\begin{verbatim}
cvs -d :pserver:cvs_anon@cvs.aei.mpg.de:/numrelcvs \
     checkout AEIPhysics/Visualization/OpenDX
\end{verbatim}

Another Visualization option is to produce files in the standard
Xdmf~\cite{AHFinderDirect/Xdmf:web} file format which can be visualized for
example using VisIt~\cite{AHFinderDirect/VisIt:web}. Frank L\"offler has
written a python script \program{AH2xdmf.py}, which is included in this
thorn's code repository, which generates such files from
\thorn{AHFinderDirect}'s output from horizon shape files
\verb|h.t%d.ah%d.gp\|. Please see its help text for details.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Accuracy}

The apparent horizon positions are typically computed very accurately;
tests on Kerr spacetimes give typical errors of $10^{-4}m$ to $10^{-5}m$.

The various diagnostics printed to standard output and written to the
black hole diagnostics file(s), are typically computed to accuracies
on the order of a part per million or so.

Note, however, that the irreducible mass $m_{\text{e}}$
may differ considerably from the black hole's local mass or its
contribution to the slice's ADM mass.  For example, for Kerr spacetime
in Kerr-Schild coordinates,
$m_{\text{e}}/m_{\text{M}} = 0.949$, $0.894$, and $0.723$
for spin parameters $a \equiv J/m^2 = 0.6$, $0.8$, and $0.999$, respectively.
It would be better to (also) use the ``isolated horizons'' formalism of
\cite{AHFinderDirect/Dreyer-etal-2002-isolated-horizons};
at some point this thorn may be enhanced to do this.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Examples}
\label{AHFinderDirect/sect-examples}

There are a few example parameter files in the \code{par/} directory,
including Kerr initial data, Misner initial data, and Misner time-evolution
tests.  The \verb|Kerr-tiny.par| parameter file is close to a minimal
\thorn{AHFinderDirect} example:

\begin{verbatim}
# This parameter file sets up Kerr/Kerr-Schild initial data, then
# finds the apparent horizon in it.  The local coordinate system origin
# and the initial guess are both deliberately de-centered with respect
# to the black hole, to make this a non-trivial test for the apparent
# horizon finder.
#
# This parameter file is "tiny" in the sense that it sets only a
# small number of AHFinderDirect parameters.

# flesh
cactus::cctk_itlast = 0

ActiveThorns = "PUGH"
driver::ghost_size = 2
driver::global_nx = 31
driver::global_ny = 31
driver::global_nz = 19

ActiveThorns = "CoordBase CartGrid3D"
grid::domain = "bitant"
grid::avoid_origin = false
grid::type = "byspacing"
grid::dxyz = 0.2

ActiveThorns = "ADMBase ADMCoupling StaticConformal Spacemask CoordGauge Exact"
ADMBase::initial_lapse = "exact"
ADMBase::initial_shift = "exact"
ADMBase::initial_data = "exact"
ADMBase::lapse_evolution_method = "static"
ADMBase::shift_evolution_method = "static"
ADMBase::metric_type = "physical"
Exact::exact_model = "Kerr/Kerr-Schild"
Exact::Kerr_KerrSchild__mass = 1.0
Exact::Kerr_KerrSchild__spin = 0.6

########################################

ActiveThorns = "IOUtil"
IOUtil::parfile_write = "no"

########################################

ActiveThorns = "SphericalSurface"
ActiveThorns = "AEILocalInterp PUGHInterp PUGHReduce AHFinderDirect"
AHFinderDirect::h_base_file_name     = "Kerr-tiny.h"

AHFinderDirect::N_horizons = 1
AHFinderDirect::origin_x[1] = 0.5
AHFinderDirect::origin_y[1] = 0.7
AHFinderDirect::origin_z[1] = 0.0

AHFinderDirect::initial_guess_method[1] = "coordinate sphere"
AHFinderDirect::initial_guess__coord_sphere__x_center[1] = -0.2
AHFinderDirect::initial_guess__coord_sphere__y_center[1] =  0.3
AHFinderDirect::initial_guess__coord_sphere__z_center[1] =  0.0
AHFinderDirect::initial_guess__coord_sphere__radius[1] = 2.0
\end{verbatim}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Surfaces of Constant Expansion}

Surfaces of Constant Expansion (CE surfaces) are introduced in
\cite{AHFinderDirect/Schnetter03a} as a generalisation of
apparent horizons (AH).  On an AH surface, the expansion is zero
everywhere.  On a CE surfaces, the expansion is still everywhere the
same, but it need not be zero.  CE surfaces are also a generalisation
of Constant Mean Curvature surfaces (CMC surfaces); both are identical
when the extrinsic curvature vanishes.  As described in
\cite{AHFinderDirect/Schnetter03a}, it is likely that CE
surfaces foliate the spacelike hypersurface outside of some interior
region.  This interior region is inside the common apparent horizon,
if it exists.

CE surfaces can give some insight into the spacetime, because they can
be used to analyse the part of the spacelike hypersurface ``between
the horizons and infinity''.  Most notably, they can be used to look
at the region where a common horizon is about to (or believed to)
form.  Similarly, one can use them for collapsing stars where an
apparent horizon has not yet formed.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Pretracking}

Apparent horizon pretracking is introduced in
\cite{AHFinderDirect/Schnetter03a}.  This is an application
of CE surfaces.  Even when there is no common horizon, there are still
common CE surfaces surrounding multiple black holes.  Pretracking
consists of tracking in time the smallest common CE surface that can
be found.  It is reasonable to believe that this surface will evolve
into the common horizon at the time where this common horizon begins
to exist.  The expansion of this smallest CE surface is also an
indication of how close the spacelike hypersurface is to having a
common apparent horizon.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{How \thorn{AHFinderDirect} Works}
\label{AHFinderDirect/sect-how-ahfinderdirect-works}

\thorn{AHFinderDirect} uses the apparent horizon
(henceforth \defn{horizon}) finding algorithm of
\cite{AHFinderDirect/Thornburg95},
modified slightly to work with $g_\ij$ and $K_\ij$ on a Cartesian ($xyz$)
grid.  The algorithm is described in detail in
\cite{AHFinderDirect/Thornburg2003:AH-finding}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\subsection{General Description of the Algorithm}

As described above, I parameterizes the horizon shape by
$r = h(\text{angle})$ for some single-value function $h: S^2 \to \Re^+$.
The apparent horizon equation~\eqref{AHFinderDirect/eqn-horizon}
then becomes a 2-D elliptic PDE on $S^2$ for the function $h$.  I finite
difference this in angle to obtain a system of simultaneous nonlinear
algebraic equations for $h$ at the angular grid points, and solve this
system of equations by a global Newton's method (or a variant with
improved convergence).

Computationally, this algorithm has 3 main parts:
\begin{itemize}
\item	Computation of the ``horizon function'' $\Theta(h)$ given a trial
	surface defined by a trial horizon shape function $h$.  This is
	done by interpolating the Cactus geometry fields $g_\ij$ and
	$K_\ij$ (and optionally $\psi$) from the 3-D $xyz$ grid to the
	(2-D set of) trial-horizon-surface grid points (also computing
	$\partial_k g_\ij$ in the interpolation process), then doing
	all further computations with angular grid functions defined
	solely on $S^2$ (\ie{} at the horizon-surface grid points).
\item	Computation of the Jacobian matrix $\Jac[\Theta(h)]$ of $\Theta(h)$.
	This thorn incorporates the \defn{symbolic differentiation} technique
	described in \cite{AHFinderDirect/Thornburg95},
	so this computation is quite fast.  The Jacobian is a highly
	sparse matrix; \thorn{AHFinderDirect} has code to store it
	as either a dense matrix (for debugging purposes), or a sparse
	matrix (the default).  Which option is used is determined by
	a compile-time configuration in \verb|src/include/config.h|.
\item	Solving the nonlinear equations $\Theta(h) = 0$ by a global
	Newton's method or a variant.  How this is done depends on how
	the Jacobian is stored.  At present,
	\begin{itemize}
	\item	If \thorn{AHFinderDirect} is configured to store the
		Jacobian as a dense matrix, then LAPACK is used to solve
		the linear equations.
	\item	If \thorn{AHFinderDirect} is configured to store the
		Jacobian as a sparse matrix, then an
		incomplete-$\sf LU$-decomposition--conjugate-gradient
		solver is used.
	\end{itemize}
	By default only the sparse-matrix code is configured, so LAPACK
	isn't used and there's no need to link with the LAPACK library.
\end{itemize}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\subsection{The Multipatch System}
\label{AHFinderDirect/sect-multipatch-system}

Perhaps the most unusual feature of \thorn{AHFinderDirect} is the
``multipatch'' system used to cover $S^2$ without coordinate singularities.
In general there are 6~patches, one each covering a neighborhood of
the $\pm z$, $\pm x$, and $\pm y$ axes, but this may be reduced in
the presence of suitable symmetries.  For example,
figure~\ref{AHFinderDirect/fig-3patch}
on page~\pageref{AHFinderDirect/fig-3patch} shows a system
of 3~patches covering the $+xyz$~octant of $S^2$.  This would be
suitable for finding an apparent horizon with mirror symmetry about
the (local) $z=0$~plane, and either 90~degree periodic rotation symmetry
about the (local) $z$~axis, or mirror symmetry about each of the (local)
$x$~and $y$~axes.

\begin{figure}[htbp]
\begin{center}
\includegraphics{AEIThorns_AHFinderDirect_3patch}
\end{center}
\caption[Illustration of the Multipatch System]
	{
	This figure shows a multipatch system covering the
	$(+,+,+)$~octant of the unit sphere~$S^2$ with 3~patches.
	The angular resolution is 5~degrees.  Notice that the
	patches overlap by several ``ghost zone'' grid points.
	}
\label{AHFinderDirect/fig-3patch}
\end{figure}

To allow easy angular finite differencing within the patch system,
each patch is extended beyond its nominal extent by a ``ghost zone''%%%
\footnote{%%%
	 Note that this terminology differs somewhat
	 from that used by Cactus in general; Cactus
	 would call these ``patch zones'' or ``symmetry
	 zones''.
	 }%%%
{} (2~grid points wide in figure~\ref{AHFinderDirect/fig-3patch}).
Angular grid function values in the ghost zone can be obtained by
interpatch interpolation%%%
\footnote{%%%
	 Due to the way the patch coordinates are defined,
	 adjacent patches always share a common ``perpendicular''
	 angular coordinate, so only 1-D interpolation
	 is needed here.
	 }%%%
{} or by applying symmetry operations.  Once this is done, then angular
finite differencing within the nominal extent of each patch can proceed
normally, ignoring the patch boundaries.  \thorn{AHFinderDirect} can
be configured at compile time to use either 2nd~order or 4th~order
angular finite differencing (3~point or 5~point angular molecules);
the default is 4th~order (5~point).  This is configured at compile time
in \verb|src/include/config.h|.

By default \thorn{AHFinderDirect} will automagically choose a patch
system type for each apparent horizon searched for, based on the local
coordinate origin and the symmetries implicit in the Cactus grid type.
This generally works well, but if desired you can instead manually
specify the patch system type, the angular resolution, the width of
the ghost zones, etc.  See the \verb|param.ccl| file for details.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\subsection{Other Software Used}

\thorn{AHFinderDirect}'s \verb|src/sparse-matrix/| directory contains
various sparse-matrix libraries, which have their own copyrights and
licensing terms:

The \verb|src/sparse-matrix/umfpack/| directory contains a subset of the
files in UMFPACK version 4.0 (11.Apr.2002).  This code is
copyright~(\copyright)~2002 by Timothy A. Davis, and is subect to the
UMFPACK License:

\begin{verbatim}
Your use or distribution of UMFPACK or any modified version of
UMFPACK implies that you agree to this License.

THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY
EXPRESSED OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.

Permission is hereby granted to use or copy this program, provided
that the Copyright, this License, and the Availability of the original
version is retained on all copies.  User documentation of any code that
uses UMFPACK or any modified version of UMFPACK code must cite the
Copyright, this License, the Availability note, and "Used by permission."
Permission to modify the code and to distribute modified code is granted,
provided the Copyright, this License, and the Availability note are
retained, and a notice that the code was modified is included.  This
software was developed with support from the National Science Foundation,
and is provided to you free of charge.
\end{verbatim}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Other Related Thorns}

If you're interested in \thorn{AHFinderDirect}, you might also be
interested in some other related thorns:

\begin{description}
\item[\thorn{EHFinder}] (in the \arrangement{AEIDevelopment} arrangement)
	was written by Peter Diener, and finds the {\em event\/} horizon(s)
	in a numerically computed spacetime.  It's described in detail in
	the paper~\cite{AHFinderDirect/Diener03a}.
\item[\thorn{AHFinder}] (in the \arrangement{CactusEinstein} arrangement)
	was written by Miguel Alcubierre, and includes two different
	algorithms for finding apparent horizons, a minimization method
	and a ``fast flow'' method based on \cite{AHFinderDirect/Gundlach97a}.
	Unfortunately, both methods are very slow in practice.
\item[\thorn{TGRapparentHorizon2D}] (in the \arrangement{TAT} arrangement)
	was written by Erik Schnetter, and is another apparent horizon
	finder.  It uses methods very similar to this thorn, and (like
	this thorn) is very fast and accurate.  However, it's no longer
	under active development.  It's described in detail in the
	papers~\cite{AHFinderDirect/Schnetter02a}
	and~\cite{AHFinderDirect/Schnetter03a}.
\item[\thorn{AHFinderDirect} (\cvsplace{Erik} branch)]
      (in the \arrangement{AEIThorns} arrangement)\\
	Erik Schnetter has added a number of new features to
	\thorn{AHFinderDirect} on a CVS branch with the tag \cvsplace{Erik},
	including horizon pretracking (to locate places where horizons
	are about to form), and the ability to find constant-expansion
	and constant-mean-curvature surfaces specified by their areal radius.
	We hope to integrate these into the main \thorn{AHFinderDirect}
	branch during the summer of 2004.
\end{description}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Acknowledgments}

I thank Peter Diener, Ian Hawke, and Erik Schnetter for many valuable
conversations.  I think Thomas Radke for his work on the new interpolators.
I thank the whole Cactus crew for a great infrastructure!

Erik Schnetter originally implemented a number of improvements to
this thorn, notably the \thorn{SphericalSurface} interface and the
new features in the \cvsplace{Erik} branch.

I thank the Alexander von Humboldt foundation and the AEI visitors'
and postdoctoral fellowships programs for financial support.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

% make LaTeX read in AHFinderDirect.bbl produced by bibtex
\bibliography{AHFinderDirect}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

% Do not delete next line
% END CACTUS THORNGUIDE

\end{document}

```



Reference solution:
src_code: // fd_grid.cc -- grid with finite differencing operations
// $Header$
//
// fd_grid::dx_coeff
// fd_grid::dxx_coeff
//

#include <stdio.h>
#include <assert.h>
#include <math.h>

#ifdef STANDALONE_TEST
  #include "fake_cctk.h"
#else
  #include "cctk.h"
#endif

#include "config.h"
#include "stdc.h"
#include "../jtutil/util.hh"
#include "../jtutil/array.hh"
#include "../jtutil/linear_map.hh"
using jtutil::error_exit;

#include "coords.hh"
#include "grid.hh"
#include "fd_grid.hh"

// all the code in this file is inside this namespace
namespace AHFinderDirect
	  {

//*****************************************************************************

//
// This function computes a single coefficient of a 1st derivative
// molecule, for unit grid spacing.
//
//static
  fp fd_grid::dx_coeff(int m)
{
#ifndef FINITE_DIFF_ORDER
  #error "must define FINITE_DIFF_ORDER!"
#endif

switch	(m)
	{
#if   (FINITE_DIFF_ORDER == 2)
  case -1:	return FD_GRID__ORDER2__DX__COEFF_M1;
  case  0:	return FD_GRID__ORDER2__DX__COEFF_0;
  case +1:	return FD_GRID__ORDER2__DX__COEFF_P1;
#elif (FINITE_DIFF_ORDER == 4)
  case -2:	return FD_GRID__ORDER4__DX__COEFF_M2;
  case -1:	return FD_GRID__ORDER4__DX__COEFF_M1;
  case  0:	return FD_GRID__ORDER4__DX__COEFF_0;
  case +1:	return FD_GRID__ORDER4__DX__COEFF_P1;
  case +2:	return FD_GRID__ORDER4__DX__COEFF_P2;
#else
  #error "unknown value " FINITE_DIFF_ORDER " for FINITE_DIFF_ORDER!"
#endif
default:
	error_exit(ERROR_EXIT,
"***** fd_grid::dx_coeff(): m=%d is outside order=%d molecule radius=%d\n",
		   m, FINITE_DIFF_ORDER, FD_GRID__MOL_RADIUS);	/*NOTREACHED*/
	}
}

//*****************************************************************************

//
// This function computes a single coefficient of a 2nd derivative
// molecule, for unit grid spacing.
//
//static
  fp fd_grid::dxx_coeff(int m)
{
#ifndef FINITE_DIFF_ORDER
  #error "must define FINITE_DIFF_ORDER!"
#endif

switch	(m)
	{
#if   (FINITE_DIFF_ORDER == 2)
  case -1:	return FD_GRID__ORDER2__DXX__COEFF_M1;
  case  0:	return FD_GRID__ORDER2__DXX__COEFF_0;
  case +1:	return FD_GRID__ORDER2__DXX__COEFF_P1;
#elif (FINITE_DIFF_ORDER == 4)
  case -2:	return FD_GRID__ORDER4__DXX__COEFF_M2;
  case -1:	return FD_GRID__ORDER4__DXX__COEFF_M1;
  case  0:	return FD_GRID__ORDER4__DXX__COEFF_0;
  case +1:	return FD_GRID__ORDER4__DXX__COEFF_P1;
  case +2:	return FD_GRID__ORDER4__DXX__COEFF_P2;
#else
  #error "unknown value " FINITE_DIFF_ORDER " for FINITE_DIFF_ORDER!"
#endif
default:
	error_exit(ERROR_EXIT,
"***** fd_grid::dxx_coeff(): m=%d is outside order=%d molecule radius=%d\n",
		   m, FINITE_DIFF_ORDER, FD_GRID__MOL_RADIUS);	/*NOTREACHED*/
	}
}

//******************************************************************************

	  }	// namespace AHFinderDirect