Task: Implement make.code.defn for ExternalLibraries/PAPI thorn

thorn_name: ExternalLibraries/PAPI
src_filename: make.code.defn
interface: # Interface definition for thorn PAPI

IMPLEMENTS: PAPI

schedule: # Schedule definitions for thorn PAPI

SCHEDULE PAPI_init AT wragh
{
  LANG: C
  OPTIONS: meta
} "Initialise PAPI"

SCHEDULE PAPI_register_clock AT wragh AFTER PAPI_init
{
  LANG: C
  OPTIONS: meta
} "Set up Cactus clock for PAPI"

SCHEDULE PAPI_dgemm AT basegrid
{
  LANG: C
  OPTIONS: meta
} "Run a DGEMM benchmark to test PAPI"

SCHEDULE PAPI_output_stats_analysis AT analysis
{
  LANG: C
  OPTIONS: meta
} "Output PAPI statistics"

SCHEDULE PAPI_output_stats_terminate AT terminate
{
  LANG: C
  OPTIONS: meta
} "Output PAPI statistics"

param: # Parameter definitions for thorn PAPI

BOOLEAN verbose "Output debug messages?" STEERABLE=always
{
} "no"

BOOLEAN per_thread_statistics "Collect statistics per thread (instead of per process)" STEERABLE=recover
{
} "yes"

BOOLEAN define_statistics_clocks "Define multiple clocks for each PAPI counter, providing min/max/avg/sdv" STEERABLE=recover
{
} "no"

BOOLEAN use_multiplexing "Use multiplexing to gather more, but less accurate information" STEERABLE=recover
{
} "yes"

INT out_every "Output statistics every so many iterations" STEERABLE=always
{
  0   :: "after initial data and shutdown"
  1:* :: "every so many iterations"
  -1  :: "no statistics output"
} 0



STRING events_flops "PAPI events for flop/sec" STEERABLE=recover
{
  "" :: ""
} "PAPI_FP_OPS PAPI_DP_OPS PAPI_VEC_DP PAPI_TOT_CYC PAPI_REF_CYC SIMD_FP_256:PACKED_DOUBLE"

STRING events_ipc "PAPI events for instructions/cycle" STEERABLE=recover
{
  "" :: ""
} "PAPI_TOT_CYC PAPI_TOT_INS PAPI_FP_INS PAPI_INT_INS"

STRING events_icache "PAPI events for instruction cache usage" STEERABLE=recover
{
  "" :: ""
} "PAPI_L1_ICA PAPI_L1_ICM PAPI_L2_ICA PAPI_L2_ICM PAPI_L3_ICA PAPI_L3_ICM"

STRING events_dcache "PAPI events for data cache usage" STEERABLE=recover
{
  "" :: ""
} "PAPI_L1_DCA PAPI_L1_DCM PAPI_L2_DCA PAPI_L2_DCM PAPI_L3_DCA PAPI_L3_DCM"

STRING events_memory "PAPI evens for memory accesses" STEERABLE=recover
{
  "" :: ""
} "PAPI_LD_INS PAPI_SR_INS PAPI_MEM_RCY PAPI_MEM_WCY"



INT dgemm_N "Matrix size; mem = 32 N^2 bytes, cpu = 2 N^3 flop"
{
  1:* :: "N=2000 requires 128 Mbyte and 16 Gflop; N=5000 requires 800 Mbyte and 250 Gflop"
} 1000

configuration: # Configuration definitions for thorn PAPI

PROVIDES PAPI
{
  SCRIPT src/detect.sh
  LANG bash
  OPTIONS PAPI_DIR PAPI_INSTALL_DIR
}

REQUIRES PAPI

context: make.configuration.defn:
```
# make.configuration.defn file for thorn PAPI

# Define standard PAPI utilities and install those that we could build
# and/or that are present in the system
STD_PAPI_UTILS = papi_avail papi_clockres papi_command_line papi_component_avail papi_cost papi_decode papi_error_codes papi_event_chooser papi_mem_info papi_multiplex_cost papi_native_avail papi_version papi_xml_event_info
ALL_UTILS += $(filter $(STD_PAPI_UTILS), $(notdir $(wildcard $(PAPI_DIR)/bin/*)))

```

make.configuration.deps:
```
# make.configuration.deps file for thorn PAPI

# Copy standard PAPI utilities from the install directory

$(UTIL_DIR)/%: $(PAPI_DIR)/bin/%
	@echo "Copying $* from $< to $(UTIL_DIR)"
	-$(MKDIR) $(MKDIRFLAGS) $(UTIL_DIR) 2> /dev/null
	cp $< $@

```

dgemm.c:
```
#include <cctk.h>
#include <cctk_Arguments.h>
#include <cctk_Parameters.h>

#include <stdlib.h>
#include <time.h>
#include <sys/time.h>

#ifdef _OPENMP
#include <omp.h>
#else
static int omp_get_num_threads(void) { return 1; }
#endif

static double get_time(void) {
  struct timeval tv;
  gettimeofday(&tv, NULL);
  return tv.tv_sec + tv.tv_usec / 1.0e+6;
}

/* Generate a matrix with random entries between 0.0 and 1.0 */
static void dmatgen(const ptrdiff_t m, const ptrdiff_t n,
                    CCTK_REAL *restrict const A, const ptrdiff_t ldA) {
  const CCTK_REAL x = rand() / (RAND_MAX + 1.0);

  for (ptrdiff_t j = 0; j < n; ++j) {
    for (ptrdiff_t i = 0; i < m; ++i) {
      A[i + ldA * j] = x;
    }
  }
}

/* DGEMM (for transposed A and un-transposed B) */
static void dgemm(const ptrdiff_t m, const ptrdiff_t n, const ptrdiff_t l,
                  const CCTK_REAL *restrict const A, const ptrdiff_t ldA,
                  const CCTK_REAL *restrict const B, const ptrdiff_t ldB,
                  CCTK_REAL *restrict const C, const ptrdiff_t ldC) {
  for (ptrdiff_t j = 0; j < n; ++j) {
    for (ptrdiff_t i = 0; i < m; ++i) {
      CCTK_REAL s = C[i + ldC * j];
      for (ptrdiff_t k = 0; k < l; ++k) {
        s += A[k + ldA * i] * B[k + ldB * j];
      }
      C[i + ldC * j] = s;
    }
  }
}

/* "Use" a matrix */
static CCTK_REAL dmatuse(const ptrdiff_t m, const ptrdiff_t n,
                         const CCTK_REAL *restrict const A,
                         const ptrdiff_t ldA) {
  CCTK_REAL s = 0.0;

  for (ptrdiff_t j = 0; j < n; ++j) {
    for (ptrdiff_t i = 0; i < m; ++i) {
      s += A[i + ldA * j];
    }
  }

  return s;
}

static void run_dgemm(const ptrdiff_t N) {
  DECLARE_CCTK_PARAMETERS;

#pragma omp barrier
  const double t0 = get_time();

  CCTK_REAL *restrict const A = malloc(N * N * sizeof *A);
  CCTK_REAL *restrict const B = malloc(N * N * sizeof *B);
  CCTK_REAL *restrict const C = malloc(N * N * sizeof *C);

  dmatgen(N, N, A, N);
  dmatgen(N, N, B, N);
  dmatgen(N, N, C, N);

  const double t1 = get_time();
#pragma omp barrier
  dgemm(N, N, N, A, N, B, N, C, N);
#pragma omp barrier
  const double t2 = get_time();
  const int num_threads = omp_get_num_threads();
  const double Gflop = num_threads * 2.0 * N * N * N / 1.0e+9;

  const volatile CCTK_REAL s CCTK_ATTRIBUTE_UNUSED = dmatuse(N, N, C, N);

  free(A);
  free(B);
  free(C);

#pragma omp barrier
  const double t3 = get_time();

  const double Gflop_sec_pure = Gflop / (t2 - t1);
  const double Gflop_sec_overall = Gflop / (t3 - t0);

#pragma omp master
  {
    CCTK_VInfo(CCTK_THORNSTRING,
               "Floating point operations: %g Gflop over %d threads", Gflop,
               num_threads);
    CCTK_VInfo(CCTK_THORNSTRING, "Pure performance:          %g Gflop/sec",
               Gflop_sec_pure);
    CCTK_VInfo(CCTK_THORNSTRING, "Overall performance:       %g Gflop/sec",
               Gflop_sec_overall);
  }
}

void PAPI_dgemm(CCTK_ARGUMENTS) {
  DECLARE_CCTK_ARGUMENTS;
  DECLARE_CCTK_PARAMETERS;

  const ptrdiff_t N = dgemm_N;

  CCTK_VInfo(CCTK_THORNSTRING,
             "Running local single-threaded DGEMM benchmark with N=%td...", N);
  run_dgemm(N);

  CCTK_VInfo(CCTK_THORNSTRING,
             "Running local multi-threaded DGEMM benchmark with N=%td...", N);
#pragma omp parallel
  { run_dgemm(N); }

  CCTK_INFO("NOTE: These performance numbers do not have to be large, but they "
            "need to agree with what PAPI reports");
}

```

stats.c:
```
#include <cctk.h>
#include <cctk_Arguments.h>
#include <cctk_Parameters.h>

#include <assert.h>
#include <math.h>
#include <stdarg.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#ifdef _OPENMP
#include <omp.h>
#else
static int omp_get_max_threads(void) { return 1; }
static int omp_get_thread_num(void) { return 0; }
#endif

#include <papi.h>

#include "stats.h"

// For backward compatibility
#if PAPI_VERSION_MAJOR(PAPI_VERSION) <= 4

// Taken (and modified) from PAPI 5.1.0.1, file src/papi.c
static int PAPI_add_named_event(int EventSet, char *EventName) {
  int ret;
  int code;
  ret = PAPI_event_name_to_code(EventName, &code);
  if (ret != PAPI_OK)
    return ret;
  ret = PAPI_add_event(EventSet, code);
  return ret;
}
#endif

////////////////////////////////////////////////////////////////////////////////

void outinfo(const char *const function) {
  DECLARE_CCTK_PARAMETERS;
  if (verbose) {
#pragma omp master
    { printf("[%s]\n", function); }
  }
}

// Check for an error, and output an error message if there was one
void chkerr(const int ierr, const char *const function, ...) {
  if (ierr < 0) {
#pragma omp critical
    {
      va_list ap;
      va_start(ap, function);
      printf("ERROR %d [%s] in ", ierr, PAPI_strerror(ierr));
      vprintf(function, ap);
      printf("\n");
      va_end(ap);
    }
  }
}

////////////////////////////////////////////////////////////////////////////////

int num_threads;
int *restrict eventsets = NULL;
int num_events;
int *restrict events;

// The PAPI component of these eventsets
static int component = -1;

////////////////////////////////////////////////////////////////////////////////

// Remember timer values when PAPI counters were last reset
static long long *restrict last_nsec = NULL;
static long long *restrict last_cyc = NULL;

#if defined __linux__

// This is recommended for Linux, but e.g. doesn't work on OSX
#include <pthread.h>
static unsigned long thread_id(void) {
  const pthread_t tid = pthread_self();
  unsigned long ret;
  memcpy(&ret, &tid, sizeof ret);
  return ret;
}

#elif defined _OPENMP

// This works only if no threads are created/destroyed
// (if the number of threads doesn't change)
#include <omp.h>
static unsigned long thread_id(void) { return omp_get_thread_num(); }

#else

// Don't know how to obtain a thread id
static unsigned long thread_id(void) { return 0; }

#endif

void PAPI_init(CCTK_ARGUMENTS) {
  DECLARE_CCTK_ARGUMENTS;
  DECLARE_CCTK_PARAMETERS;

  int ierr;

  CCTK_INFO("Initialising PAPI");

  // OpenMP

  assert(per_thread_statistics);
  num_threads = omp_get_max_threads();

  // Initialise PAPI

  outinfo("PAPI_library_init");
  ierr = PAPI_library_init(PAPI_VER_CURRENT);
  chkerr(ierr, "PAPI_library_init");
  if (ierr != PAPI_VER_CURRENT) {
    CCTK_ERROR("PAPI library version mismatch");
  }

  if (per_thread_statistics) {
    outinfo("PAPI_thread_init");
// TODO: Investigate PAPI_INHERIT -- but this may work only with
// recent version of PAPI and/or Linux

// NOTE: Our implementation of thread_id probably cannot handle
// changes in the number of threads. That is, omp_set_num_threads
// should not be called after this point.

#if 0
    // Ensure all threads are created
    // (This is necessary for Intel's OpenMP.)
    volatile int dummy = 0;
#pragma omp parallel reduction(+ : dummy)
    {
      dummy += 1;
    }
#endif

    ierr = PAPI_thread_init(thread_id);
    chkerr(ierr, "PAPI_thread_init");
  }

  if (use_multiplexing) {
    outinfo("PAPI_multiplex_init");
    ierr = PAPI_multiplex_init();
    chkerr(ierr, "PAPI_multiplex_init");
  }

  // Get some basic information

  outinfo("PAPI_num_components");
  int num_components = ierr = PAPI_num_components();
  chkerr(ierr, "PAPI_num_components");
  if (ierr < 0)
    num_components = 0;
  if (verbose) {
    printf("There are %d PAPI components\n", num_components);
  }

  if (verbose) {
    for (int n = 0; n < num_components; ++n) {
      outinfo("PAPI_get_component_info");
      const PAPI_component_info_t *const info = PAPI_get_component_info(n);
      printf("PAPI component #%d:\n", n);
      printf("   name: %s\n", info->name);
#if PAPI_VERSION_MAJOR(PAPI_VERSION) >= 5
      printf("   short_name: %s\n", info->short_name);
      printf("   description: %s\n", info->description);
#endif
      printf("   num_cntrs: %d\n", info->num_cntrs);
      printf("   num_mpx_cntrs: %d\n", info->num_mpx_cntrs);
      printf("   num_preset_events: %d\n", info->num_preset_events);
      printf("   num_native_events: %d\n", info->num_native_events);
    }
  }

  component = 0;
  if (verbose) {
    printf("Using PAPI component %d\n", component);
  }

  outinfo("PAPI_num_counters");
  int num_counters = ierr = PAPI_num_counters();
  chkerr(ierr, "PAPI_num_counters");
  if (ierr < 0)
    num_counters = 0;
  if (verbose) {
    printf("There are %d PAPI counters\n", num_counters);
  }

  // Translate user event names to an eventset

  eventsets = malloc(num_threads * sizeof *eventsets);
  assert(eventsets);

#pragma omp parallel private(ierr)
  {
    const int thread_num = omp_get_thread_num();

    outinfo("PAPI_create_eventset");
    eventsets[thread_num] = PAPI_NULL;
    ierr = PAPI_create_eventset(&eventsets[thread_num]);
    chkerr(ierr, "PAPI_create_eventset");
    outinfo("PAPI_assign_eventset_component");
    ierr = PAPI_assign_eventset_component(eventsets[thread_num], component);
    chkerr(ierr, "PAPI_assign_eventset_component");
    if (use_multiplexing) {
      outinfo("PAPI_set_multiplex");
      ierr = PAPI_set_multiplex(eventsets[thread_num]);
      chkerr(ierr, "PAPI_set_multiplex");
    }

    // User-defined event sets
    const int num_eventsets = 5;
    const char *restrict const eventset_descriptions[] = {
        "flops", "ipc", "icache", "dcache", "memory",
    };
    assert(sizeof eventset_descriptions / sizeof *eventset_descriptions ==
           num_eventsets);
    const char *const eventset_eventnames[] = {
        events_flops, events_ipc, events_icache, events_dcache, events_memory,
    };
    assert(sizeof eventset_eventnames / sizeof *eventset_eventnames ==
           num_eventsets);

    for (int n = 0; n < num_eventsets; ++n) {
#pragma omp master
      if (verbose) {
        printf("Adding eventset %s:\n", eventset_descriptions[n]);
      }

      char *const eventnames = strdup(eventset_eventnames[n]);
      const char *const sep = ", ";
      char *lasts;
      for (char *event_name = strtok_r(eventnames, sep, &lasts); event_name;
           event_name = strtok_r(NULL, sep, &lasts)) {
#pragma omp master
        if (verbose) {
          printf("   event %s\n", event_name);
        }
        outinfo("PAPI_add_named_event");
        ierr = PAPI_add_named_event(eventsets[thread_num], event_name);
        chkerr(ierr, "PAPI_add_named_event[%s]", event_name);
      }
      free(eventnames);
    }
  }

  outinfo("PAPI_num_events");
  num_events = ierr = PAPI_num_events(eventsets[0]);
  chkerr(ierr, "PAPI_num_events");
  if (ierr < 0)
    num_events = 0;

  outinfo("PAPI_list_events");
  events = malloc(num_events * sizeof *events);
  ierr = PAPI_list_events(eventsets[0], events, &num_events);
  chkerr(ierr, "PAPI_list_events");

  last_nsec = malloc(num_threads * sizeof *last_nsec);
  assert(last_nsec);
  last_cyc = malloc(num_threads * sizeof *last_cyc);
  assert(last_cyc);

#pragma omp parallel private(ierr)
  {
    const int thread_num = omp_get_thread_num();

    outinfo("PAPI_start");
    ierr = PAPI_start(eventsets[thread_num]);
    chkerr(ierr, "PAPI_start");

    outinfo("PAPI_get_real_nsec");
    last_nsec[thread_num] = PAPI_get_real_nsec();
    outinfo("PAPI_get_real_cyc");
    last_cyc[thread_num] = PAPI_get_real_cyc();
  }
}

static void output_stats(CCTK_ARGUMENTS) {
  DECLARE_CCTK_ARGUMENTS;
  DECLARE_CCTK_PARAMETERS;

  int ierr;

  CCTK_INFO("PAPI Statistics:");

  long long elapsed_nsec = 0;
  long long elapsed_cyc = 0;
  long long values[num_events];
  for (int i = 0; i < num_events; ++i) {
    values[i] = 0;
  }
#pragma omp parallel private(ierr)
  {
    const int thread_num = omp_get_thread_num();

    outinfo("PAPI_get_real_nsec");
    const long long my_nsec = PAPI_get_real_nsec();

    outinfo("PAPI_read_ts");
    long long my_cyc;
    long long my_values[num_events];
    ierr = PAPI_read_ts(eventsets[thread_num], my_values, &my_cyc);
    chkerr(ierr, "PAPI_read_ts");

    const long long my_elapsed_nsec = my_nsec - last_nsec[thread_num];
    const long long my_elapsed_cyc = my_cyc - last_cyc[thread_num];
    last_nsec[thread_num] = my_nsec;
    last_cyc[thread_num] = my_cyc;

#pragma omp critical(PAPI_output_stats_add)
    {
      elapsed_nsec += my_elapsed_nsec;
      elapsed_cyc += my_elapsed_cyc;
      for (int i = 0; i < num_events; ++i) {
        values[i] += my_values[i];
      }
    }
  }

  printf("   CPU time          %20.9f sec\n", elapsed_nsec / 1.0e+9);
  printf("   CPU cycles        %20.9f Gcyc\n", elapsed_cyc / 1.0e+9);
  for (int i = 0; i < num_events; ++i) {
    outinfo("PAPI_event_code_to_name");
    char eventname[PAPI_MAX_STR_LEN];
    ierr = PAPI_event_code_to_name(events[i], eventname);
    chkerr(ierr, "PAPI_event_code_to_name");
    printf("   %-15s   %20.9f cyc^(-1)\n", eventname,
           1.0 * values[i] / elapsed_cyc);
  }
}

void PAPI_output_stats_analysis(CCTK_ARGUMENTS) {
  DECLARE_CCTK_ARGUMENTS;
  DECLARE_CCTK_PARAMETERS;

  if (out_every == -1)
    return;
  if (cctk_iteration > 0) {
    if (out_every == 0)
      return;
    if (cctk_iteration % out_every > 0)
      return;
  }

  output_stats(CCTK_PASS_CTOC);
}

void PAPI_output_stats_terminate(CCTK_ARGUMENTS) {
  DECLARE_CCTK_ARGUMENTS;
  DECLARE_CCTK_PARAMETERS;

  if (out_every == -1)
    return;
  if (out_every > 0)
    return;

  output_stats(CCTK_PASS_CTOC);
}

```

detect.sh:
```
#! /bin/bash

################################################################################
# Prepare
################################################################################

# Set up shell
if [ "$(echo ${VERBOSE} | tr '[:upper:]' '[:lower:]')" = 'yes' ]; then
    set -x                      # Output commands
fi
set -e                          # Abort on errors



################################################################################
# Search
################################################################################

if [ -z "${PAPI_DIR}" ]; then
    echo "BEGIN MESSAGE"
    echo "PAPI selected, but PAPI_DIR not set. Checking some places..."
    echo "END MESSAGE"
    
    DIRS="/usr /usr/local /opt/local ${HOME}"
    # look into each directory
    for dir in $DIRS; do
        # libraries might have different file extensions
        for libext in a dll dll.a dylib lib so; do
            # libraries can be in lib or lib64 (or libx32?)
            for libdir in lib64 lib; do
                # These files must exist
                FILES="include/papi.h ${libdir}/libpapi.${libext}"
                # assume this is the one and check all needed files
                PAPI_DIR="$dir"
                for file in $FILES; do
                    # discard this directory if one file was not found
                    if [ ! -r "$dir/$file" ]; then
                        unset PAPI_DIR
                        break
                    fi
                done
                # don't look further if all files have been found
                if [ -n "$PAPI_DIR" ]; then
                    break
                fi
            done
            # don't look further if all files have been found
            if [ -n "$PAPI_DIR" ]; then
                break
            fi
        done
        # don't look further if all files have been found
        if [ -n "$PAPI_DIR" ]; then
            break
        fi
    done
    
    if [ -z "$PAPI_DIR" ]; then
        echo "BEGIN MESSAGE"
        echo "PAPI not found"
        echo "END MESSAGE"
    else
        echo "BEGIN MESSAGE"
        echo "Found PAPI in ${PAPI_DIR}"
        echo "END MESSAGE"
    fi
fi

THORN=PAPI



################################################################################
# Build
################################################################################

if [ -z "${PAPI_DIR}"                                            \
     -o "$(echo "${PAPI_DIR}" | tr '[a-z]' '[A-Z]')" = 'BUILD' ]
then
    echo "BEGIN MESSAGE"
    echo "Using bundled PAPI..."
    echo "END MESSAGE"
    
    # Set locations
    NAME=papi-5.3.0
    TARNAME=papi-5.3.0
    SRCDIR="$(dirname $0)"
    BUILD_DIR=${SCRATCH_BUILD}/build/${THORN}
    if [ -z "${PAPI_INSTALL_DIR}" ]; then
        INSTALL_DIR=${SCRATCH_BUILD}/external/${THORN}
    else
        echo "BEGIN MESSAGE"
        echo "Installing PAPI into ${PAPI_INSTALL_DIR} "
        echo "END MESSAGE"
        INSTALL_DIR=${PAPI_INSTALL_DIR}
    fi
    PAPI_BUILD=1
    PAPI_DIR=${INSTALL_DIR}
else    
    PAPI_BUILD=
    DONE_FILE=${SCRATCH_BUILD}/done/${THORN}
    if [ ! -e ${DONE_FILE} ]; then
        mkdir ${SCRATCH_BUILD}/done 2> /dev/null || true
        date > ${DONE_FILE}
    fi
fi



################################################################################
# Configure Cactus
################################################################################

# Pass configuration options to build script
echo "BEGIN MAKE_DEFINITION"
echo "PAPI_BUILD       = ${PAPI_BUILD}"
echo "PAPI_INSTALL_DIR = ${PAPI_INSTALL_DIR}"
echo "END MAKE_DEFINITION"

# Set options
PAPI_INC_DIRS="${PAPI_DIR}/include"
PAPI_LIB_DIRS="${PAPI_DIR}/lib"
PAPI_LIBS="papi"

if nm ${PAPI_LIB_DIRS}/libpapi.a 2>/dev/null | grep -q pm_initialize; then
    PAPI_LIBS="${PAPI_LIBS} pmapi"
fi

PAPI_INC_DIRS="$(${CCTK_HOME}/lib/sbin/strip-incdirs.sh ${PAPI_INC_DIRS})"
PAPI_LIB_DIRS="$(${CCTK_HOME}/lib/sbin/strip-libdirs.sh ${PAPI_LIB_DIRS})"

# Pass options to Cactus
echo "BEGIN MAKE_DEFINITION"
echo "PAPI_DIR      = ${PAPI_DIR}"
echo "PAPI_INC_DIRS = ${PAPI_INC_DIRS}"
echo "PAPI_LIB_DIRS = ${PAPI_LIB_DIRS}"
echo "PAPI_LIBS     = ${PAPI_LIBS}"
echo "END MAKE_DEFINITION"

echo 'INCLUDE_DIRECTORY $(PAPI_INC_DIRS)'
echo 'LIBRARY_DIRECTORY $(PAPI_LIB_DIRS)'
echo 'LIBRARY           $(PAPI_LIBS)'

```

clock.c:
```
#include <cctk.h>
#include <cctk_Arguments.h>
#include <cctk_Parameters.h>

#include <assert.h>
#include <float.h>
#include <limits.h>
#include <math.h>
#include <stdarg.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#ifdef _OPENMP
#include <omp.h>
#else
static int omp_get_max_threads(void) { return 1; }
static int omp_get_thread_num(void) { return 0; }
#endif

#include <papi.h>

#include "stats.h"

static const char *const clock_name = "PAPI";

static int num_event_vals = -1;
static int num_clock_vals = -1;

static const int counter_nsec = 0;
static const int counter_cyc = 1;
static const int counter_count = 2;
static const int counter_begin = 3;

static int num_statistics_vals = -1;
static int num_cactus_clock_vals = -1;
static const char **counter_names = NULL; // [num_cactus_clock_vals]
static const char **counter_units = NULL; // [num_cactus_clock_vals]

typedef long long papi_counter_t;

typedef struct {
  // indexing: [num_clock_vals], laid out as [num_threads][num_event_vals]
  papi_counter_t *restrict accum;    // [num_clock_vals]
  papi_counter_t *restrict snapshot; // [num_clock_vals]
  papi_counter_t count;
  bool running;
} papi_clock_t;

static void read_papi_counters(papi_counter_t *restrict const counters) {
  int ierr;

/* for (int i=0; i<num_clock_vals; ++i) { */
/*   counters[i] = -1; */
/* } */
#pragma omp barrier
#pragma omp parallel private(ierr)
  {
    const int thread_num = omp_get_thread_num();

    papi_counter_t *restrict const my_counters =
        &counters[num_event_vals * thread_num];

    my_counters[counter_nsec] = PAPI_get_real_nsec();
    ierr = PAPI_read_ts(eventsets[thread_num], &my_counters[counter_begin],
                        &my_counters[counter_cyc]);
    // chkerr(ierr, "PAPI_read_ts");
    if (ierr < 0)
      my_counters[counter_cyc] = PAPI_get_real_cyc();
    ++my_counters[counter_count];
  }
  /* for (int i=0; i<num_clock_vals; ++i) { */
  /*   assert(counters[i] != -1); */
  /* } */
  /* for (int i=0; i<num_clock_vals; ++i) { */
  /*   assert(counters[i] >= 0); */
  /* } */
}

static void *clock_create(const int timernum) {
  papi_clock_t *restrict const data = malloc(sizeof *data);
  assert(data);
  data->accum = malloc(num_clock_vals * sizeof *data->accum);
  assert(data->accum);
  data->snapshot = malloc(num_clock_vals * sizeof *data->snapshot);
  assert(data->snapshot);
  for (int i = 0; i < num_clock_vals; ++i) {
    data->accum[i] = 0;
  }
  data->running = false;
  return data;
}

static void clock_destroy(const int timernum, void *restrict const data_) {
  papi_clock_t *restrict const data = malloc(sizeof *data);
  assert(data);
  free(data->accum);
  free(data->snapshot);
  free(data);
}

static void clock_start(const int timernum, void *restrict const data_) {
  papi_clock_t *restrict const data = data_;
  assert(!data->running);
  read_papi_counters(data->snapshot);
  data->running = true;
}

static void clock_stop(const int timernum, void *restrict const data_) {
  papi_clock_t *restrict const data = data_;
  // Apparently clocks can be created in a "running" state?
  if (!data->running)
    return;
  assert(data->running);
  papi_counter_t current[num_clock_vals];
  read_papi_counters(current);
  data->running = false;
  for (int i = 0; i < num_clock_vals; ++i) {
    data->accum[i] += current[i] - data->snapshot[i];
  }
  /* for (int i=0; i<num_clock_vals; ++i) { */
  /*   assert(data->accum[i] >= 0); */
  /* } */
}

static void clock_reset(const int timernum, void *restrict const data_) {
  papi_clock_t *restrict const data = data_;
  for (int i = 0; i < num_clock_vals; ++i) {
    data->accum[i] = 0;
  }
}

static void clock_get(const int timernum, void *restrict const data_,
                      cTimerVal *restrict const vals) {
  DECLARE_CCTK_PARAMETERS;

  const papi_clock_t *restrict const data = data_;

  int count = 0;
  for (int i = 0; i < num_event_vals; ++i) {

    double sum = 0.0, sum2 = 0.0, min = DBL_MAX, max = 0.0;
    for (int j = i; j < num_clock_vals; j += num_event_vals) {
      /* assert(data->accum[j] >= 0); */
      const double val = data->accum[j];
      sum += val;
      sum2 += val * val;
      min = fmin(min, val);
      max = fmax(max, val);
    }
    /* assert(sum >= 0.0); */

    if (!define_statistics_clocks) {

      vals[count].type = val_double;
      vals[count].heading = counter_names[count];
      vals[count].units = counter_units[count];
      vals[count].val.d = sum / 1.0e+9;
      vals[count].seconds = sum / 1.0e+9;
      vals[count].resolution = 1.0e-9;
      ++count;

    } else {

      const double avg = sum / num_threads;
      const double sdv = sqrt(fmax(0.0, sum2 / num_threads - avg * avg));

      vals[count].type = val_double;
      vals[count].heading = counter_names[count];
      vals[count].units = counter_units[count];
      vals[count].val.d = avg / 1.0e+9;
      vals[count].seconds = avg / 1.0e+9;
      vals[count].resolution = 1.0e-9;
      ++count;

      vals[count].type = val_double;
      vals[count].heading = counter_names[count];
      vals[count].units = counter_units[count];
      vals[count].val.d = sdv / 1.0e+9;
      vals[count].seconds = sdv / 1.0e+9;
      vals[count].resolution = 1.0e-9;
      ++count;

      vals[count].type = val_double;
      vals[count].heading = counter_names[count];
      vals[count].units = counter_units[count];
      vals[count].val.d = min / 1.0e+9;
      vals[count].seconds = min / 1.0e+9;
      vals[count].resolution = 1.0e-9;
      ++count;

      vals[count].type = val_double;
      vals[count].heading = counter_names[count];
      vals[count].units = counter_units[count];
      vals[count].val.d = max / 1.0e+9;
      vals[count].seconds = max / 1.0e+9;
      vals[count].resolution = 1.0e-9;
      ++count;
    }
  }
}

static void clock_set(const int timernum, void *restrict const data_,
                      cTimerVal *restrict const vals) {
  papi_clock_t *restrict const data = data_;

  for (int i = 0; i < num_event_vals; ++i) {
    assert(vals[i].type == val_double);
    const papi_counter_t val = llrint(vals[i].val.d * 1.0e+9);
    for (int j = i; j < num_clock_vals; j += num_event_vals) {
      data->accum[j] = val;
    }
  }
}

static void add_clock(const char **names, const char **units, const char *name,
                      const char *unit, int *clock, int nclocks) {
  DECLARE_CCTK_PARAMETERS;
  if (!define_statistics_clocks) {
    assert(*clock < nclocks);
    names[*clock] = strdup(name);
    units[*clock] = unit;
    ++*clock;
  } else {
    const char *const suffixes[] = {"_avg", "_sdv", "_min", "_max"};
    for (int s = 0; s < 4; ++s) {
      assert(*clock < nclocks);
      char buf[1000];
      snprintf(buf, sizeof buf, "%s%s", name, suffixes[s]);
      names[*clock] = strdup(buf);
      units[*clock] = unit;
      ++*clock;
    }
  }
}

void PAPI_register_clock(CCTK_ARGUMENTS) {
  DECLARE_CCTK_ARGUMENTS;
  DECLARE_CCTK_PARAMETERS;

  CCTK_INFO("PAPI_register_clock");

  int ierr;

  num_event_vals = counter_begin + num_events;

  // PAPI clocks

  num_clock_vals = num_threads * num_event_vals;

  // Cactus clocks

  num_statistics_vals = define_statistics_clocks ? 4 : 1;
  num_cactus_clock_vals = num_event_vals * num_statistics_vals;

  counter_names = malloc(num_cactus_clock_vals * sizeof *counter_names);
  assert(counter_names);
  counter_units = malloc(num_cactus_clock_vals * sizeof *counter_units);
  assert(counter_units);

  int clock = 0;
  add_clock(counter_names, counter_units, "P_time", "sec", &clock,
            num_cactus_clock_vals);
  add_clock(counter_names, counter_units, "P_cycles", "Gcycles", &clock,
            num_cactus_clock_vals);
  add_clock(counter_names, counter_units, "P_count", "Gevents", &clock,
            num_cactus_clock_vals);
  for (int i = 0; i < num_events; ++i) {
    char eventname[PAPI_MAX_STR_LEN + 4];
    ierr = PAPI_event_code_to_name(events[i], eventname);
    chkerr(ierr, "PAPI_event_code_to_name");
    if (strlen(eventname) >= 4) {
      // Change prefix "PAPI" to "P" to save space
      for (char *p = eventname + 1; p[2]; ++p)
        p[0] = p[3];
    }
    add_clock(counter_names, counter_units, eventname, "Gevents", &clock,
              num_cactus_clock_vals);
  }
  assert(clock == num_cactus_clock_vals);

  cClockFuncs funcs;
  funcs.name = clock_name;
  funcs.n_vals = num_cactus_clock_vals;
  funcs.create = clock_create;
  funcs.destroy = clock_destroy;
  funcs.start = clock_start;
  funcs.stop = clock_stop;
  funcs.reset = clock_reset;
  funcs.get = clock_get;
  funcs.set = clock_set;
  funcs.seconds = NULL;
  ierr = CCTK_ClockRegister(clock_name, &funcs);
  assert(ierr >= 0);
}

```

build.sh:
```
#! /bin/bash

################################################################################
# Build
################################################################################

# Set up shell
if [ "$(echo ${VERBOSE} | tr '[:upper:]' '[:lower:]')" = 'yes' ]; then
    set -x                      # Output commands
fi
set -e                          # Abort on errors



# Set locations
THORN=PAPI
NAME=papi-5.3.0
TARNAME=papi-5.3.0
SRCDIR="$(dirname $0)"
BUILD_DIR=${SCRATCH_BUILD}/build/${THORN}
if [ -z "${PAPI_INSTALL_DIR}" ]; then
    INSTALL_DIR=${SCRATCH_BUILD}/external/${THORN}
else
    echo "BEGIN MESSAGE"
    echo "Installing PAPI into ${PAPI_INSTALL_DIR} "
    echo "END MESSAGE"
    INSTALL_DIR=${PAPI_INSTALL_DIR}
fi
DONE_FILE=${SCRATCH_BUILD}/done/${THORN}
PAPI_DIR=${INSTALL_DIR}
    
# Set up environment
unset CPP
unset LIBS
export MPICC=:          # disable MPI tests
if echo '' ${ARFLAGS} | grep 64 > /dev/null 2>&1; then
    export OBJECT_MODE=64
fi

echo "PAPI: Preparing directory structure..."
cd ${SCRATCH_BUILD}
mkdir build external done 2> /dev/null || true
rm -rf ${BUILD_DIR} ${INSTALL_DIR}
mkdir ${BUILD_DIR} ${INSTALL_DIR}

echo "PAPI: Unpacking archive..."
pushd ${BUILD_DIR}
${TAR?} xzf ${SRCDIR}/../dist/${TARNAME}.tar.gz

echo "PAPI: Applying patches..."
pushd ${NAME}
# Replace <malloc.h> by <stdlib.h>
find . -type f -print | xargs perl -pi -e 's/malloc.h/stdlib.h/'
# disable Werror since new compiler warns about PAPI
find . -name config.mk -print | xargs perl -pi -e 's/-Werror//g'
popd

echo "PAPI: Configuring..."
cd ${NAME}/src
# CC          C compiler command
# CFLAGS      C compiler flags
# LDFLAGS     linker flags, e.g. -L<lib dir> if you have libraries in a
#             nonstandard directory <lib dir>
# CPPFLAGS    C/C++ preprocessor flags, e.g. -I<include dir> if you have
#             headers in a nonstandard directory <include dir>
# F77         Fortran 77 compiler command
# FFLAGS      Fortran 77 compiler flags
# CPP         C preprocessor
#unset CC
#unset CFLAGS
#unset LDFLAGS
#unset CPPFLAGS
#unset F77
#unset FFLAGS
#unset CPP
#export MPICC=

./configure --prefix=${PAPI_DIR} --with-shared-lib=no

echo "PAPI: Building..."
${MAKE}

echo "PAPI: Installing..."
${MAKE} install
popd

echo "PAPI: Cleaning up..."
rm -rf ${BUILD_DIR}

date > ${DONE_FILE}
echo "PAPI: Done."

```

make.code.deps:
```
# Main make.code.deps file for thorn PAPI

export PAPI_INSTALL_DIR

$(CCTK_TARGET) $(OBJS) $(SRCS:%=%.d): $(SCRATCH_BUILD)/done/$(THORN)

ifneq ($(PAPI_BUILD),)
$(SCRATCH_BUILD)/done/$(THORN): $(SRCDIR)/build.sh
	+$(SRCDIR)/build.sh
endif

```

stats.h:
```
#ifndef STATS_H
#define STATS_H

#include <cctk.h>
#include <cctk_Parameters.h>

#include <stdarg.h>
#include <stdio.h>

#define num_threads PAPI_Cactus_num_threads
#define eventsets PAPI_Cactus_eventsets
#define num_events PAPI_Cactus_num_events
#define events PAPI_Cactus_events

// Number of threads
extern int num_threads;
// Eventsets for each thread
extern int *restrict eventsets; // [num_threads]

// Number of events in these eventsets
extern int num_events;
// Events in these eventsets
extern int *restrict events; // [num_events]

// Output a a debug message
#define outinfo PAPI_Cactus_outinfo
void outinfo(const char *const function);

// Check for an error, and output an error message if there was one
#define chkerr PAPI_Cactus_chkerr
void chkerr(const int ierr, const char *const function, ...)
    CCTK_ATTRIBUTE_FORMAT(printf, 2, 3);

#endif // #ifndef STATS_H

```


combined_doc_context: ## README (README):
```
Cactus Code Thorn PAPI
Author(s)    : Erik Schnetter
Maintainer(s): Cactus team
Licence      : BSD
--------------------------------------------------------------------------

1. Purpose

Distribute the Performance Application Programming Interface; see
<http://icl.cs.utk.edu/papi/index.html>.



From the web site:

PAPI provides the tool designer and application engineer with a
consistent interface and methodology for use of the performance
counter hardware found in most major microprocessors. PAPI enables
software engineers to see, in near real time, the relation between
software performance and processor events.

In addition Component PAPI provides access to a collection of
components that expose performance measurement opportunites across the
hardware and software stack.

```



Reference solution:
src_code: # Main make.code.defn file for thorn PAPI

# Source files in this directory
SRCS = clock.c dgemm.c stats.c

# Subdirectories containing source files
SUBDIRS =