Repository: LLNL/Caliper (language: python)

Issue:
Leverage RADIUSS Shared CI in Caliper
Per [RADIUSS Shared CI documentation](https://radiuss-shared-ci.readthedocs.io/en/latest/sphinx/user_guide/index.html), here is a curated list of tasks to perform in order to leverage the shared CI infrastructure:

- [x] Use uberenv to build Caliper dependencies and generate a CMake Cache file:
  - [x] Setup Uberenv
  - [x] Setup Radiuss-spack-configs
  - [x] Build Caliper with Uberenv
  - [x] Duplicate Caliper Spack package in Radiuss-spack-configs and update the implementation to use CachedCMakePackage


- [x] Create a build-and-test script that takes a spack spec an input, the script must then:
  - [x] Use uberenv to build the dependencies
  - [x] Retrieve the generated CMake Cache file and use it to build Caliper and the tests
  - [x] Test Caliper

> RAJA has an example build-and-test script [here](https://github.com/LLNL/RAJA/blob/develop/scripts/gitlab/build_and_test.sh).

- [x] Setup the CI to use RADIUSS Shared CI.
  - [x] We will prefer to have the mirroring up and running at this stage. 


[Woptim] rhel 8 update
This PR updates radiuss spack configs and radiuss shared CI with changes for the rhel 8 update on ruby.

It also sets several packages as externals in the spack config files (radiuss shared CI) to speed up the CI, not rebuilding the world each time. Some of those externals also conveniently work errors found building them from source.


Discussion / hints:

Fix patch:
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 1bd3dc07d..09a09934f 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,53 +1,74 @@
-.build-quartz:
-  stage: build
-  tags: 
-    - quartz
-    - shell
-  script:
-    - echo "==== ${build_config} ===="
-    - mkdir build-${build_config} && cd build-${build_config}
-    - cmake -C ../cmake/hostconfig/gitlab-${build_config}.cmake ..
-    - make
-    - ctest --output-on-failure
+###############################################################################
+# Copyright (c) 2022, Lawrence Livermore National Security, LLC and RADIUSS
+# project contributors. See the COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (MIT)
+###############################################################################
 
-build-quartz-gcc:
-  variables:
-    build_config: "quartz-gcc"
-  extends: .build-quartz
-  artifacts:
-    paths:
-      - build-quartz-gcc/
+# DESCRIPTION:
+###############################################################################
+# General GitLab pipelines configurations for supercomputers and Linux clusters
+# at Lawrence Livermore National Laboratory (LLNL).
+# This entire pipeline is LLNL-specific
+#
+# Important note: This file is a template provided by
+# llnl/radiuss-shared-ci. It should not require any change from the project to
+# get started but could feature project-specific stages.
+#
+# Instead, each project should provide:
+# - .gitlab/custom-jobs-and-variables.yml
+# - .gitlab/subscribed-pipelines.yml
+# - .gitlab/${MACHINE}-build-and-test-extra.yml
+###############################################################################
 
-build-quartz-intel:
-  variables:
-    build_config: "quartz-intel"
-  extends: .build-quartz
+# We define the following GitLab pipeline variables:
+variables:
+# Required information about GitHub repository
+  GITHUB_PROJECT_NAME: "Caliper"
+  GITHUB_PROJECT_ORG: "LLNL"
+# Use an LLNL service user to run CI. This prevents from running pipelines as
+# an actual user.
+  LLNL_SERVICE_USER: ""
+# Use the service user workspace. Solves permission issues, stores everything
+# at the same location whoever triggers a pipeline.
+#  CUSTOM_CI_BUILDS_DIR: ""
+# Tells Gitlab to recursively update the submodules when cloning the project.
+  GIT_SUBMODULE_STRATEGY: recursive
+# We build the projects in the CI clone directory.
+# TODO: add a clean-up mechanism
+  BUILD_ROOT: ${CI_PROJECT_DIR}
+# Set the build-and-test command.
+  BUILD_AND_TEST_CMD: "./scripts/gitlab/build-and-test.sh"
+# Override the list of branch that will skip the "draft PR test".
+# Add protected branches here. Defaults to "develop main master".
+#  ALWAYS_RUN_LIST: ""
 
-build-quartz-pgi:
-  variables:
-    build_config: "quartz-pgi"
-  extends: .build-quartz
+# We organize the build-and-test stage in sub-pipelines. Each sub-pipeline
+# corresponds to a test batch on a given machine.
 
-perftest-quartz:
-  stage: test
-  tags:
-    - quartz
-    - shell
-  script:
-    - cd build-quartz-gcc
-    - ../scripts/run-annotation-perftest.sh
-  dependencies:
-    - build-quartz-gcc
+# High level stages
+stages:
+  - machine-checks
+  - build-and-test
 
-build-butte-xlc:
-  stage: build
-  tags:
-    - butte
-    - shell
-  script:
-    - echo "==== Build butte/xlc ===="
-    - module load cmake
-    - mkdir build-butte-xlc && cd build-butte-xlc
-    - cmake -C ../cmake/hostconfig/gitlab-butte-xlc.cmake ..
-    - make
-    - ctest --output-on-failure
+# Template for jobs triggering a build-and-test sub-pipelines:
+.build-and-test:
+  stage: build-and-test
+  trigger:
+    include:
+      - local: '.gitlab/custom-jobs-and-variables.yml'
+      - project: 'radiuss/radiuss-shared-ci'
+        ref: v2023.03.1
+        file: '${CI_MACHINE}-build-and-test.yml'
+      - local: '.gitlab/${CI_MACHINE}-build-and-test-extra.yml'
+    strategy: depend
+    forward:
+      pipeline_variables: true
+
+include:
+  # [Optional] checks preliminary to running the actual CI test
+  - project: 'radiuss/radiuss-shared-ci'
+    ref: v2023.03.1
+    file: 'preliminary-ignore-draft-pr.yml'
+  # pipelines subscribed by the project
+  - local: .gitlab/subscribed-pipelines.yml
diff --git a/.gitlab/custom-jobs-and-variables.yml b/.gitlab/custom-jobs-and-variables.yml
new file mode 100644
index 000000000..86360e899
--- /dev/null
+++ b/.gitlab/custom-jobs-and-variables.yml
@@ -0,0 +1,61 @@
+###############################################################################
+# Copyright (c) 2022, Lawrence Livermore National Security, LLC and RADIUSS
+# project contributors. See the COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (MIT)
+###############################################################################
+
+# We define the following GitLab pipeline variables:
+variables:
+# On LLNL's ruby, this pipeline creates only one allocation shared among jobs
+# in order to save time and resources. This allocation has to be uniquely named
+# so that we are sure to retrieve it and avoid collisions.
+  ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID}
+
+# Ruby
+# Arguments for top level allocation
+  RUBY_BUILD_AND_TEST_SHARED_ALLOC: "--exclusive --partition=pdebug --time=30 --nodes=1"
+# Arguments for job level allocation
+  RUBY_BUILD_AND_TEST_JOB_ALLOC: "--overlap --time=20 --nodes=1"
+# Project specific variants for ruby
+  PROJECT_RUBY_VARIANTS: " +tests"
+# Project specific deps for ruby
+  PROJECT_RUBY_DEPS: ""
+
+# Corona
+# Arguments for top level allocation
+  CORONA_BUILD_AND_TEST_SHARED_ALLOC: "--exclusive --time-limit=15m --nodes=1"
+# Arguments for job level allocation
+  CORONA_BUILD_AND_TEST_JOB_ALLOC: "--time-limit=10m --nodes=1 --begin-time=+5s"
+# Project specific variants for corona
+  PROJECT_CORONA_VARIANTS: " +tests"
+# Project specific deps for corona
+  PROJECT_CORONA_DEPS: ""
+
+# Tioga
+# Arguments for top level allocation
+  TIOGA_BUILD_AND_TEST_SHARED_ALLOC: "--exclusive --time-limit=20m --nodes=1"
+# Arguments for job level allocation
+  TIOGA_BUILD_AND_TEST_JOB_ALLOC: "--time-limit=10m --nodes=1 --begin-time=+5s"
+# Project specific variants for tioga
+  PROJECT_TIOGA_VARIANTS: " +tests"
+# Project specific deps for tioga
+  PROJECT_TIOGA_DEPS: ""
+
+# Lassen and Butte use a different job scheduler (spectrum lsf) that does not
+# allow pre-allocation the same way slurm does.
+# Arguments for job level allocation
+  LASSEN_BUILD_AND_TEST_JOB_ALLOC: "1 -W 10"
+# Project specific variants for lassen
+  PROJECT_LASSEN_VARIANTS: " +tests"
+# Project specific deps for lassen
+  PROJECT_LASSEN_DEPS: "^adiak ~mpi"
+
+
+# Configuration shared by build and test jobs specific to this project.
+# Not all configuration can be shared. Here projects can fine tune the
+# CI behavior.
+# See Umpire for an example (export junit test reports).
+.custom_build_and_test:
+  variables:
+    JOB_TEMPLATE_CANNOT_BE_EMPTY: "True"
diff --git a/.gitlab/subscribed-pipelines.yml b/.gitlab/subscribed-pipelines.yml
new file mode 100644
index 000000000..a946724dd
--- /dev/null
+++ b/.gitlab/subscribed-pipelines.yml
@@ -0,0 +1,92 @@
+###############################################################################
+# Copyright (c) 2022, Lawrence Livermore National Security, LLC and RADIUSS
+# project contributors. See the COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (MIT)
+###############################################################################
+
+# The template job to test whether a machine is up.
+# Expects CI_MACHINE defined to machine name.
+.machine-check:
+  stage: machine-checks
+  tags: [shell, oslic]
+  variables:
+    GIT_STRATEGY: none
+  script:
+    - |
+      if [[ $(jq '.[env.CI_MACHINE].total_nodes_up' /usr/global/tools/lorenz/data/loginnodeStatus) == 0 ]]
+      then
+        echo -e "\e[31mNo node available on ${CI_MACHINE}\e[0m"
+        curl --url "https://api.github.com/repos/${GITHUB_PROJECT_ORG}/${GITHUB_PROJECT_NAME}/statuses/${CI_COMMIT_SHA}" \
+             --header 'Content-Type: application/json' \
+             --header "authorization: Bearer ${GITHUB_TOKEN}" \
+             --data "{ \"state\": \"failure\", \"target_url\": \"${CI_PIPELINE_URL}\", \"description\": \"GitLab ${CI_MACHINE} down\", \"context\": \"ci/gitlab/${CI_MACHINE}\" }"
+        exit 1
+      fi
+
+###
+# Trigger a build-and-test pipeline for a machine.
+# Comment the jobs for machines you don’t need.
+###
+
+# RUBY
+ruby-up-check:
+  variables:
+    CI_MACHINE: "ruby"
+  extends: [.machine-check]
+
+ruby-build-and-test:
+  variables:
+    CI_MACHINE: "ruby"
+  needs: [ruby-up-check]
+  extends: [.build-and-test]
+
+# Quartz it very similar to ruby, let’s deactivate it for now
+## QUARTZ
+#quartz-up-check:
+#  variables:
+#    CI_MACHINE: "QUARTZ"
+#  extends: [.machine-check]
+#
+#quartz-build-and-test:
+#  variables:
+#    CI_MACHINE: "quartz"
+#  needs: [quartz-up-check]
+#  extends: [.build-and-test]
+
+# CORONA
+corona-up-check:
+  variables:
+    CI_MACHINE: "corona"
+  extends: [.machine-check]
+
+corona-build-and-test:
+  variables:
+    CI_MACHINE: "corona"
+  needs: [corona-up-check]
+  extends: [.build-and-test]
+
+# TIOGA
+tioga-up-check:
+  variables:
+    CI_MACHINE: "tioga"
+  extends: [.machine-check]
+
+tioga-build-and-test:
+  variables:
+    CI_MACHINE: "tioga"
+  needs: [tioga-up-check]
+  extends: [.build-and-test]
+
+# LASSEN
+lassen-up-check:
+  variables:
+    CI_MACHINE: "lassen"
+  extends: [.machine-check]
+
+lassen-build-and-test:
+  variables:
+    CI_MACHINE: "lassen"
+  needs: [lassen-up-check]
+  extends: [.build-and-test]
+
diff --git a/.gitmodules b/.gitmodules
new file mode 100644
index 000000000..03cc60566
--- /dev/null
+++ b/.gitmodules
@@ -0,0 +1,6 @@
+[submodule "scripts/uberenv"]
+	path = scripts/uberenv
+	url = https://github.com/LLNL/uberenv.git
+[submodule "scripts/radiuss-spack-configs"]
+	path = scripts/radiuss-spack-configs
+	url = https://github.com/LLNL/radiuss-spack-configs.git
diff --git a/.uberenv_config.json b/.uberenv_config.json
new file mode 100755
index 000000000..2569fd27f
--- /dev/null
+++ b/.uberenv_config.json
@@ -0,0 +1,14 @@
+{
+    "package_name" : "caliper",
+    "package_version" : "develop",
+    "package_source_dir" : "../..",
+    "package_final_phase" : "initconfig",
+    "spack_url": "https://github.com/spack/spack.git",
+    "spack_branch": "e4s-23.02",
+    "spack_activate" : {},
+    "spack_configs_path": "scripts/radiuss-spack-configs",
+    "spack_packages_path": "scripts/radiuss-spack-configs/packages",
+    "spack_concretizer": "clingo",
+    "spack_setup_clingo": false,
+    "debug": true
+    }
\ No newline at end of file
diff --git a/cmake/FindRocTX.cmake b/cmake/FindRocTX.cmake
index 86c84ef83..4bfcf0b26 100644
--- a/cmake/FindRocTX.cmake
+++ b/cmake/FindRocTX.cmake
@@ -1,17 +1,17 @@
 # Find RocTX libraries/headers
 
-find_path(ROCM_PREFIX
+find_path(ROCM_ROOT_DIR
   NAMES include/roctracer/roctx.h
 )
 
 find_library(ROCTX_LIBRARIES
   NAMES roctx64
-  HINTS ${ROCM_PREFIX}/lib
+  HINTS ${ROCM_ROOT_DIR}/lib
 )
 
 find_path(ROCTX_INCLUDE_DIRS
   NAMES roctx.h
-  HINTS ${ROCM_PREFIX}/include/roctracer
+  HINTS ${ROCM_ROOT_DIR}/include/roctracer
 )
 
 include(FindPackageHandleStandardArgs)
diff --git a/cmake/FindRoctracer.cmake b/cmake/FindRoctracer.cmake
index 35659e3b4..0cd45fddf 100644
--- a/cmake/FindRoctracer.cmake
+++ b/cmake/FindRoctracer.cmake
@@ -1,30 +1,30 @@
 # Find ROCTracer libraries/headers
 
-find_path(ROCM_PREFIX
+find_path(ROCM_ROOT_DIR
   NAMES include/roctracer/roctracer.h
 )
 
 find_library(ROCTRACER_LIBROCTRACER
   NAMES roctracer64
-  HINTS ${ROCM_PREFIX}/lib
+  HINTS ${ROCM_ROOT_DIR}/lib
 )
 find_library(ROCTRACER_LIBHSARUNTIME
   NAMES hsa-runtime64
-  HINTS ${ROCM_PREFIX}/lib
+  HINTS ${ROCM_ROOT_DIR}/lib
 )
 find_library(ROCTRACER_LIBHSAKMT
   NAMES hsakmt
-  HINTS ${ROCM_PREFIX}/lib
+  HINTS ${ROCM_ROOT_DIR}/lib
 )
 
 find_path(ROCTRACER_INCLUDE_DIR
   NAMES roctracer.h
-  HINTS ${ROCM_PREFIX}/include/roctracer
+  HINTS ${ROCM_ROOT_DIR}/include/roctracer
 )
 
 find_path(HIP_INCLUDE_DIR
   NAMES hip/hip_runtime.h
-  HINTS ${ROCM_PREFIX}/include)
+  HINTS ${ROCM_ROOT_DIR}/include)
 
 set(ROCTRACER_INCLUDE_DIRS 
   ${ROCTRACER_INCLUDE_DIR}
diff --git a/scripts/radiuss-spack-configs b/scripts/radiuss-spack-configs
new file mode 160000
index 000000000..ffde59230
--- /dev/null
+++ b/scripts/radiuss-spack-configs
@@ -0,0 +1,1 @@
+Subproject commit ffde592300392a7bbebbfe0a8d752187c1b93132
diff --git a/scripts/uberenv b/scripts/uberenv
new file mode 160000
index 000000000..0d00dc8e1
--- /dev/null
+++ b/scripts/uberenv
@@ -0,0 +1,1 @@
+Subproject commit 0d00dc8e19a889ba07ae433590b87533c4b5b3da


Test patch:
diff --git a/.gitlab/corona-build-and-test-extra.yml b/.gitlab/corona-build-and-test-extra.yml
new file mode 100644
index 000000000..c249de65a
--- /dev/null
+++ b/.gitlab/corona-build-and-test-extra.yml
@@ -0,0 +1,22 @@
+##############################################################################
+# Copyright (c) 2022, Lawrence Livermore National Security, LLC and RADIUSS
+# project contributors. See the COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (MIT)
+##############################################################################
+
+# With GitLab CI, included files cannot be empty.
+variables:
+  INCLUDED_FILE_CANNOT_BE_EMPTY: "True"
+
+# The correct way is now to use rocmcc compiler directly as done in shared spec
+### Template for extra jobs defined by the project.
+#clang14.0.6-rocm5.4.1:
+#  variables:
+#    SPEC: "%clang@14.0.6 +rocm amdgpu_target=gfx906 ^hip@5.4.1 "
+#  extends: .build_and_test_on_corona
+
+master:
+  variables:
+    SPEC: "@master +tests"
+  extends: .build_and_test_on_corona
diff --git a/.gitlab/lassen-build-and-test-extra.yml b/.gitlab/lassen-build-and-test-extra.yml
new file mode 100644
index 000000000..03b90ff54
--- /dev/null
+++ b/.gitlab/lassen-build-and-test-extra.yml
@@ -0,0 +1,16 @@
+##############################################################################
+# Copyright (c) 2022, Lawrence Livermore National Security, LLC and RADIUSS
+# project contributors. See the COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (MIT)
+##############################################################################
+
+# With GitLab CI, included files cannot be empty.
+variables:
+  INCLUDED_FILE_CANNOT_BE_EMPTY: "True"
+
+## Template for extra jobs defined by the project.
+master:
+  variables:
+    SPEC: "@master +tests"
+  extends: .build_and_test_on_lassen
diff --git a/.gitlab/ruby-build-and-test-extra.yml b/.gitlab/ruby-build-and-test-extra.yml
new file mode 100644
index 000000000..0056ae6c2
--- /dev/null
+++ b/.gitlab/ruby-build-and-test-extra.yml
@@ -0,0 +1,16 @@
+##############################################################################
+# Copyright (c) 2022, Lawrence Livermore National Security, LLC and RADIUSS
+# project contributors. See the COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (MIT)
+##############################################################################
+
+# With GitLab CI, included files cannot be empty.
+variables:
+  INCLUDED_FILE_CANNOT_BE_EMPTY: "True"
+
+## Template for extra jobs defined by the project.
+master:
+  variables:
+    SPEC: "@master +tests"
+  extends: .build_and_test_on_ruby
diff --git a/.gitlab/tioga-build-and-test-extra.yml b/.gitlab/tioga-build-and-test-extra.yml
new file mode 100644
index 000000000..c93e6a9d9
--- /dev/null
+++ b/.gitlab/tioga-build-and-test-extra.yml
@@ -0,0 +1,22 @@
+##############################################################################
+# Copyright (c) 2022, Lawrence Livermore National Security, LLC and RADIUSS
+# project contributors. See the COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (MIT)
+##############################################################################
+
+# With GitLab CI, included files cannot be empty.
+variables:
+  INCLUDED_FILE_CANNOT_BE_EMPTY: "True"
+
+# The correct way is now to use rocmcc compiler directly as done in shared spec
+### Template for extra jobs defined by the project.
+#clang13.0.1-rocm5.4.1:
+#  variables:
+#    SPEC: "%clang@13.0.1 +rocm amdgpu_target=gfx90a ^hip@5.4.1"
+#  extends: .build_and_test_on_tioga
+
+master:
+  variables:
+    SPEC: "@master +tests %cce@15.0.1"
+  extends: .build_and_test_on_tioga
diff --git a/scripts/gitlab/build-and-test.sh b/scripts/gitlab/build-and-test.sh
new file mode 100755
index 000000000..333faf597
--- /dev/null
+++ b/scripts/gitlab/build-and-test.sh
@@ -0,0 +1,173 @@
+#!/usr/bin/env bash
+
+hostname="$(hostname)"
+timestamp="$(date +"%T" | sed 's/://g')"
+project_dir="$(pwd)"
+hostname="$(hostname)"
+spec=${SPEC:-""}
+option=${1:-""}
+truehostname=${hostname//[0-9]/}
+
+prefix=""
+
+if [[ "${option}" == "--project-build" ]]
+then
+    prefix=${project_dir}"/CI-builds/${hostname}-${timestamp}"
+elif [[ -d /dev/shm ]]
+then
+    prefix="/dev/shm/${hostname}"
+    if [[ -z ${job_unique_id} ]]; then
+      job_unique_id=manual_job_$(date +%s)
+      while [[ -d ${prefix}-${job_unique_id} ]] ; do
+          sleep 1
+          job_unique_id=manual_job_$(date +%s)
+      done
+    fi
+    prefix="${prefix}-${job_unique_id}"
+fi
+
+echo "Creating directory ${prefix}"
+echo "project_dir: ${project_dir}"
+
+mkdir -p ${prefix}
+
+spack_user_cache="${prefix}/spack-user-cache"
+export SPACK_DISABLE_LOCAL_CONFIG=""
+export SPACK_USER_CACHE_PATH="${spack_user_cache}"
+mkdir -p ${spack_user_cache}
+
+if [[ -z ${spec} ]]
+    then
+        echo "SPEC is undefined, aborting..."
+        exit 1
+    fi
+
+# generate cmake cache file with uberenv and radiuss spack package
+echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
+echo "~~~~~ Building Dependencies"
+echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
+./scripts/uberenv/uberenv.py --prefix ${prefix} --spec="${spec}"
+
+# find generated cmake cache
+if [[ -z ${hostconfig} ]]
+then
+    hostconfigs=( $( ls "${project_dir}/"*.cmake ) )
+    if [[ ${#hostconfigs[@]} == 1 ]]
+    then
+        hostconfig_path=${hostconfigs[0]}
+        echo "Found host config file: ${hostconfig_path}"
+    elif [[ ${#hostconfigs[@]} == 0 ]]
+    then
+        echo "No result for: ${project_dir}/*.cmake"
+        echo "Spack generated host-config not found."
+        exit 1
+    else
+        echo "More than one result for: ${project_dir}/*.cmake"
+        echo "${hostconfigs[@]}"
+        echo "Please specify one with HOST_CONFIG variable"
+        exit 1
+    fi
+else
+    hostconfig_path = "${project_dir}/${hostconfig}"
+fi
+
+hostconfig=$(basename ${hostconfig_path})
+echo "Found hostconfig: ${hostconfig}"
+
+# build directory
+build_dir="${prefix}/build_${hostconfig//.cmake/}"
+install_dir="${prefix}/install_${hostconfig//.cmake/}"
+cmake_exe=`grep 'CMake executable' ${hostconfig_path} | cut -d ':' -f 2 | xargs`
+
+
+# Build
+if [[ "${option}" != "--deps-only" && "${option}" != "--test-only" ]]
+then
+    date
+    echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
+    echo "~~~~~ Prefix: ${prefix}"
+    echo "~~~~~ Host-config: ${hostconfig_path}"
+    echo "~~~~~ Build Dir:   ${build_dir}"
+    echo "~~~~~ Project Dir: ${project_dir}"
+    echo "~~~~~ Install Dir: ${install_dir}"
+    echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
+    echo ""
+    echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
+    echo "~~~~~ Building Caliper"
+    echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
+
+    # Map CPU core allocations
+    declare -A core_counts=(["lassen"]=40 ["ruby"]=28 ["corona"]=32 ["rzansel"]=48 ["tioga"]=32)
+
+    # If building, then delete everything first
+    # NOTE: 'cmake --build . -j core_counts' attempts to reduce individual build resources.
+    #       If core_counts does not contain hostname, then will default to '-j ', which should
+    #       use max cores.
+    rm -rf ${build_dir} 2>/dev/null
+    mkdir -p ${build_dir} && cd ${build_dir}
+
+    date
+
+    if [[ "${truehostname}" == "corona" || "${truehostname}" == "tioga" ]]
+    then
+        module unload rocm
+    fi
+    if [[ "${truehostname}" == "lassen" ]]
+    then
+        $cmake_exe \
+          -C ${hostconfig_path} \
+          -DCMAKE_INSTALL_PREFIX=${install_dir} \
+          -DRUN_MPI_TESTS=Off \
+          ${project_dir}
+    else
+        $cmake_exe \
+          -C ${hostconfig_path} \
+          -DCMAKE_INSTALL_PREFIX=${install_dir} \
+          ${project_dir}
+    fi
+    if ! $cmake_exe --build . -j ${core_counts[$truehostname]}
+    then
+        echo "ERROR: compilation failed, building with verbose output..."
+        $cmake_exe --build . --verbose -j 1
+    else
+        make install
+    fi
+
+    echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
+    echo "~~~~~ Caliper Built"
+    echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
+    date
+fi
+
+# Test
+if [[ "${option}" != "--build-only" ]]
+then
+    date
+    echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
+    echo "~~~~~ Testing Caliper"
+    echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
+
+    echo "Moving to ${build_dir}"
+    cd ${build_dir}
+
+    date
+    ctest --output-on-failure -T test 2>&1 | tee tests_output.txt
+    date
+
+
+    no_test_str="No tests were found!!!"
+    if [[ "$(tail -n 1 tests_output.txt)" == "${no_test_str}" ]]
+    then
+        echo "ERROR: No tests were found" && exit 1
+    fi
+
+    if grep -q "Errors while running CTest" ./tests_output.txt
+    then
+        echo "ERROR: failure(s) while running CTest" && exit 1
+    fi
+
+    echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
+    echo "~~~~~ Caliper Tests Complete"
+    echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
+    date
+fi