#!/bin/bash
#
# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy
# of the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "LICENSE.txt" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, express or implied. See the License for the specific
# language governing permissions and limitations under the License.

# Usage: mount_filesystem.sh head_node_ip shared_dir

error_exit_usage() {
    echo "Error executing script: $1"
    usage
    exit 1
}

error_exit() {
    echo "Error executing script: $1"
    exit 1
}

usage() {
    cat <<ENDUSAGE
This script mounts the shared dir from head node instance on the nodes in the batch job.
The shared directory will be created if it does not exist.

USAGE:
mount_nfs <head_node_ip> <shared_dir>
head_node_ip: ip address of the main node
shared_dir: directory from head node to be shared. If directory doesn't exist on compute, will be created
ENDUSAGE
}

# Check that the arguments are valid
check_arguments_valid(){
    if [ -z "${head_node_ip}" ]; then
        error_exit_usage "Head Node IP is a required argument"
    fi

    if [ -z "${shared_dir}" ]; then
        error_exit_usage "The directory to be shared via nfs is a required argument"
    fi
}

# mount nfs
mount_nfs() {

    error_message=$(rpcinfo &> /dev/null || rpcbind 2>&1)
    if [[ $? -ne 0 ]]; then
        error_exit "Failed to run rpcbind with error_message: ${error_message}"
    fi

    mkdir -p ${shared_dir}
    error_message=$(mount -t nfs -o hard,intr,noatime,_netdev "${head_node_ip}":"${shared_dir}" "${shared_dir}" 2>&1)
    if [[ $? -ne 0 ]]; then
        error_exit "Failed to mount nfs volume from ${head_node_ip}:${shared_dir} with error_message: ${error_message}"
    fi

    # Check that the filesystem is mounted as appropriate
    mount_line=$(mount | grep "${head_node_ip}:${shared_dir}")
    if [[ -z "${mount_line}" ]]; then
        error_exit "mount succeeded but nfs volume from ${head_node_ip}:${shared_dir} was not mounted as expected"
    fi
}

# main function
main() {
    head_node_ip=${1}
    shared_dir=${2}
    if [[ "${shared_dir:0:1}" != '/' ]]; then
      shared_dir="/${shared_dir}"
    fi

    check_arguments_valid
    mount_nfs
}

main "$@"


Content-Type: multipart/mixed; boundary="==BOUNDARY=="
MIME-Version: 1.0

--==BOUNDARY==
Content-Type: text/cloud-boothook; charset="us-ascii"
MIME-Version: 1.0

#!/bin/bash -x

which dnf 2>/dev/null; dnf=$?
which yum 2>/dev/null; yum=$?

if [ "${!dnf}" == "0" ]; then
  echo "proxy=${DnfProxy}" >> /etc/dnf/dnf.conf
elif [ "${!yum}" == "0" ]; then
  echo "proxy=${YumProxy}" >> /etc/yum.conf
else
  echo "Not yum system"
fi

which apt-get && echo "Acquire::http::Proxy \"${AptProxy}\";" >> /etc/apt/apt.conf || echo "Not apt system"

proxy=${ProxyServer}
if [ "${!proxy}" != "NONE" ]; then
  proxy_host=$(echo "${!proxy}" | awk -F/ '{print $3}' | cut -d: -f1)
  proxy_port=$(echo "${!proxy}" | awk -F/ '{print $3}' | cut -d: -f2)
  echo -e "[Boto]\nproxy = ${!proxy_host}\nproxy_port = ${!proxy_port}\n" >/etc/boto.cfg
  cat >> /etc/profile.d/proxy.sh <<PROXY
export http_proxy="${!proxy}"
export https_proxy="${!proxy}"
export no_proxy="localhost,127.0.0.1,169.254.169.254"
export HTTP_PROXY="${!proxy}"
export HTTPS_PROXY="${!proxy}"
export NO_PROXY="localhost,127.0.0.1,169.254.169.254"
PROXY
fi

--==BOUNDARY==
Content-Type: text/cloud-config; charset=us-ascii
MIME-Version: 1.0

bootcmd:
  # Disable multithreading using logic from https://aws.amazon.com/blogs/compute/disabling-intel-hyper-threading-technology-on-amazon-linux/
  # thread_siblings_list contains a comma (,) or dash (-) separated list of CPU hardware threads within the same core as cpu
  # e.g. 0-1 or 0,1
  # cat /sys/devices/system/cpu/cpu*/topology/thread_siblings_list
  #     | tr '-' ','       # convert hyphen (-) to comma (,), to account that some kernels and CPU architectures use a hyphen instead of a comma
  #     | cut -s -d, -f2-  # split over comma (,) and take the right part
  #     | tr ',' '\n'      # convert remaining comma (,) into new lines
  #     | sort -un         # sort and unique
  - if [ "${DisableMultiThreadingManually}" = "true" ]; then for cpunum in $(cat /sys/devices/system/cpu/cpu*/topology/thread_siblings_list | tr '-' ',' | cut -s -d, -f2- | tr ',' '\n' | sort -un); do echo 0 > /sys/devices/system/cpu/cpu$cpunum/online; done; fi

package_update: false
package_upgrade: false
repo_upgrade: none

datasource_list: [ Ec2, None ]
output:
  all: "| tee -a /var/log/cloud-init-output.log | logger -t user-data -s 2>/dev/console"
write_files:
  - path: /tmp/dna.json
    permissions: '0644'
    owner: root:root
    content: |
      {
        "cluster": {
          "cluster_name": "${ClusterName}",
          "stack_name": "${AWS::StackName}",
          "stack_arn": "${AWS::StackId}",
          "enable_efa": "${EnableEfa}",
          "raid_shared_dir": "${RAIDSharedDir}",
          "raid_type": "${RAIDType}",
          "base_os": "${BaseOS}",
          "region": "${AWS::Region}",
          "efs_fs_ids": "${EFSIds}",
          "efs_shared_dirs": "${EFSSharedDirs}",
          "efs_encryption_in_transits": "${EFSEncryptionInTransits}",
          "efs_iam_authorizations": "${EFSIamAuthorizations}",
          "fsx_fs_ids": "${FSXIds}",
          "fsx_mount_names": "${FSXMountNames}",
          "fsx_dns_names": "${FSXDNSNames}",
          "fsx_volume_junction_paths": "${FSXVolumeJunctionPaths}",
          "fsx_fs_types": "${FSXFileSystemTypes}",
          "fsx_shared_dirs": "${FSXSharedDirs}",
          "scheduler": "${Scheduler}",
          "ephemeral_dir": "${EphemeralDir}",
          "ebs_shared_dirs": "${EbsSharedDirs}",
          "proxy": "${ProxyServer}",
          "slurm_ddb_table": "${SlurmDynamoDBTable}",
          "log_group_name": "${LogGroupName}",
          "dns_domain": "${ClusterDNSDomain}",
          "hosted_zone": "${ClusterHostedZone}",
          "node_type": "ComputeFleet",
          "cluster_user": "${OSUser}",
          "enable_intel_hpc_platform": "${IntelHPCPlatform}",
          "cw_logging_enabled": "${CWLoggingEnabled}",
          "log_rotation_enabled": "${LogRotationEnabled}",
          "scheduler_queue_name": "${QueueName}",
          "scheduler_compute_resource_name": "${ComputeResourceName}",
          "enable_efa_gdr": "${EnableEfaGdr}",
          "custom_node_package": "${CustomNodePackage}",
          "custom_awsbatchcli_package": "${CustomAwsBatchCliPackage}",
          "use_private_hostname": "${UsePrivateHostname}",
          "head_node_private_ip": "${HeadNodePrivateIp}",
          "directory_service": {
            "enabled": "${DirectoryServiceEnabled}"
          }
        }
      }
  - path: /etc/chef/client.rb
    permissions: '0644'
    owner: root:root
    content: cookbook_path ['/etc/chef/cookbooks']
  - path: /tmp/extra.json
    permissions: '0644'
    owner: root:root
    content: |
      ${ExtraJson}
  - path: /tmp/bootstrap.sh
    permissions: '0744'
    owner: root:root
    content: |
      #!/bin/bash -x

      function error_exit
      {
        echo "Bootstrap failed with error: $1"
        # wait logs flush before signaling the failure
        sleep 10
        # TODO: add possibility to override this behavior and keep the instance for debugging
        shutdown -h now
        exit 1
      }
      function vendor_cookbook
      {
        mkdir /tmp/cookbooks
        cd /tmp/cookbooks
        tar -xzf /etc/chef/aws-parallelcluster-cookbook.tgz
        HOME_BAK="${!HOME}"
        export HOME="/tmp"
        for d in `ls /tmp/cookbooks`; do
          cd /tmp/cookbooks/$d
          LANG=en_US.UTF-8 /opt/cinc/embedded/bin/berks vendor /etc/chef/cookbooks --delete || error_exit 'Vendoring cookbook failed.'
        done;
        export HOME="${!HOME_BAK}"
      }

      function publish_startup_time
      {
        start_time=$1
        end_time=$2
        # Only put the startup time data if the scheduler is slurm
        slurm_node_path=/etc/parallelcluster/slurm_plugin/slurm_node_spec.json
        if test -f "$slurm_node_path"; then
            INSTANCE_ID=$(cat $slurm_node_path | jq -r .instance_id)
            INSTANCE_TYPE=$(cat $slurm_node_path | jq -r .'compute."instance-type"')
            aws cloudwatch put-metric-data --namespace "ParallelCluster" --metric-name "StartupTime" --value $(($end_time-$start_time)) --dimensions "InstanceID=${!INSTANCE_ID},InstanceType=${!INSTANCE_TYPE},ClusterName=${ClusterName}" --region "${AWS::Region}"
        fi
      }

      [ -f /etc/profile.d/proxy.sh ] && . /etc/profile.d/proxy.sh

      # Configure AWS CLI using the expected overrides, if any.
      [ -f /etc/profile.d/aws-cli-default-config.sh ] && . /etc/profile.d/aws-cli-default-config.sh

      custom_cookbook=${CustomChefCookbook}
      export _region=${AWS::Region}

      s3_url=${AWS::URLSuffix}
      if [ "${!custom_cookbook}" != "NONE" ]; then
        if [[ "${!custom_cookbook}" =~ ^s3://([^/]*)(.*) ]]; then
          # Set the socket connection timeout to 15s. Emperically, it seems like the actual
          # timeout is 8x(cli-connect-timeout). i.e. if cli-connection-timeout is set to
          # 60s, the call will timeout the connect attempt at 8m. Setting it to 15s, causes
          # each attempt to take 240s, so 2m * 3 attempts will result in a failure after 6
          # minutes.
          S3API_RESULT=$(AWS_RETRY_MODE=standard aws s3api get-bucket-location --cli-connect-timeout 15 --bucket ${!BASH_REMATCH[1]} --region ${AWS::Region} 2>&1) || error_exit "${!S3API_RESULT}"
          bucket_region=$(echo "${!S3API_RESULT}" | jq -r '.LocationConstraint')
          if [[ "${!bucket_region}" == null ]]; then
            bucket_region="us-east-1"
          fi
          cookbook_url=$(aws s3 presign "${!custom_cookbook}" --region "${!bucket_region}")
        else
          cookbook_url=${!custom_cookbook}
        fi
      fi
      export PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/opt/aws/bin
      export parallelcluster_version=aws-parallelcluster-${ParallelClusterVersion}
      export cookbook_version=${CookbookVersion}
      export chef_version=${ChefVersion}
      export berkshelf_version=${BerkshelfVersion}
      if [ -f /opt/parallelcluster/.bootstrapped ]; then
        installed_version=$(cat /opt/parallelcluster/.bootstrapped)
        if [ "${!cookbook_version}" != "${!installed_version}" ]; then
          cookbook_version_number=$(echo ${!cookbook_version} | awk -F- '{print $NF}')
          installed_version_number=$(echo ${!installed_version} | awk -F- '{print $NF}')
          error_exit "This AMI was created with ParallelCluster ${!installed_version_number}, but is trying to be used with ParallelCluster ${!cookbook_version_number}. Please either use an AMI created with ParallelCluster ${!cookbook_version_number} or change your ParallelCluster to ${!installed_version_number}"
        fi
      else
        error_exit "This AMI was not baked by ParallelCluster. Please use pcluster build-image command to create an AMI by providing your AMI as parent image."
      fi
      if [ "${!custom_cookbook}" != "NONE" ]; then
        curl --retry 3 -v -L -o /etc/chef/aws-parallelcluster-cookbook.tgz ${!cookbook_url}
        vendor_cookbook
      fi
      cd /tmp

      mkdir -p /etc/chef/ohai/hints
      touch /etc/chef/ohai/hints/ec2.json

      # measure start time
      start=$(date +%s)

      jq --argfile f1 /tmp/dna.json --argfile f2 /tmp/extra.json -n '$f1 * $f2' > /etc/chef/dna.json || ( echo "jq not installed or invalid extra_json"; cp /tmp/dna.json /etc/chef/dna.json)
      {
        pushd /etc/chef &&
        cinc-client --local-mode --config /etc/chef/client.rb --log_level info --force-formatter --no-color --chef-zero-port 8889 --json-attributes /etc/chef/dna.json --override-runlist aws-parallelcluster-entrypoints::init &&
        /opt/parallelcluster/scripts/fetch_and_run -preinstall &&
        cinc-client --local-mode --config /etc/chef/client.rb --log_level info --force-formatter --no-color --chef-zero-port 8889 --json-attributes /etc/chef/dna.json --override-runlist aws-parallelcluster-entrypoints::config &&
        /opt/parallelcluster/scripts/fetch_and_run -postinstall &&
        cinc-client --local-mode --config /etc/chef/client.rb --log_level info --force-formatter --no-color --chef-zero-port 8889 --json-attributes /etc/chef/dna.json --override-runlist aws-parallelcluster-entrypoints::finalize &&
        popd
      } || error_exit 'Failed to run bootstrap recipes. If --norollback was specified, check /var/log/cfn-init.log and /var/log/cloud-init-output.log.'

      if [ ! -f /opt/parallelcluster/.bootstrapped ]; then
        echo ${!cookbook_version} | tee /opt/parallelcluster/.bootstrapped
      fi

      # measure end time
      end=$(date +%s)

      if [ "${ComputeStartupTimeMetricEnabled}" = "True" ]; then
        publish_startup_time $start $end || echo "[WARNING] failed to push the StartupTime metric to CloudWatch."
      fi

--==BOUNDARY==
Content-Type: text/x-shellscript; charset="us-ascii"
MIME-Version: 1.0

#!/bin/bash -x

function error_exit
{
  echo "Timed-out when bootstrapping instance"
  sleep 10  # Allow logs to propagate
  shutdown -h now
  exit 1
}

if [ "${Timeout}" == "NONE" ]; then
  /tmp/bootstrap.sh
else
  timeout ${Timeout} /tmp/bootstrap.sh || error_exit
fi

# End of file
--==BOUNDARY==


Content-Type: multipart/mixed; boundary="==BOUNDARY=="
MIME-Version: 1.0

--==BOUNDARY==
Content-Type: text/cloud-boothook; charset="us-ascii"
MIME-Version: 1.0

#!/bin/bash -x

which dnf 2>/dev/null; dnf=$?
which yum 2>/dev/null; yum=$?

if [ "${!dnf}" == "0" ]; then
  echo "proxy=${DnfProxy}" >> /etc/dnf/dnf.conf
elif [ "${!yum}" == "0" ]; then
  echo "proxy=${YumProxy}" >> /etc/yum.conf
else
  echo "Not yum system"
fi

which apt-get && echo "Acquire::http::Proxy \"${AptProxy}\";" >> /etc/apt/apt.conf || echo "Not apt system"

proxy=${ProxyServer}
if [ "${!proxy}" != "NONE" ]; then
  proxy_host=$(echo "${!proxy}" | awk -F/ '{print $3}' | cut -d: -f1)
  proxy_port=$(echo "${!proxy}" | awk -F/ '{print $3}' | cut -d: -f2)
  echo -e "[Boto]\nproxy = ${!proxy_host}\nproxy_port = ${!proxy_port}\n" >/etc/boto.cfg
  cat >> /etc/profile.d/proxy.sh <<PROXY
export http_proxy="${!proxy}"
export https_proxy="${!proxy}"
export no_proxy="localhost,127.0.0.1,169.254.169.254"
export HTTP_PROXY="${!proxy}"
export HTTPS_PROXY="${!proxy}"
export NO_PROXY="localhost,127.0.0.1,169.254.169.254"
PROXY
fi

--==BOUNDARY==
Content-Type: text/cloud-config; charset=us-ascii
MIME-Version: 1.0

bootcmd:
  # Disable multithreading using logic from https://aws.amazon.com/blogs/compute/disabling-intel-hyper-threading-technology-on-amazon-linux/
  # thread_siblings_list contains a comma (,) or dash (-) separated list of CPU hardware threads within the same core as cpu
  # e.g. 0-1 or 0,1
  # cat /sys/devices/system/cpu/cpu*/topology/thread_siblings_list
  #     | tr '-' ','       # convert hyphen (-) to comma (,), to account that some kernels and CPU architectures use a hyphen instead of a comma
  #     | cut -s -d, -f2-  # split over comma (,) and take the right part
  #     | tr ',' '\n'      # convert remaining comma (,) into new lines
  #     | sort -un         # sort and unique
  - if [ "${DisableMultiThreadingManually}" = "true" ]; then for cpunum in $(cat /sys/devices/system/cpu/cpu*/topology/thread_siblings_list | tr '-' ',' | cut -s -d, -f2- | tr ',' '\n' | sort -un); do echo 0 > /sys/devices/system/cpu/cpu$cpunum/online; done; fi

package_update: false
package_upgrade: false
repo_upgrade: none

datasource_list: [ Ec2, None ]

--==BOUNDARY==
Content-Type: text/x-shellscript; charset="us-ascii"
MIME-Version: 1.0

#!/bin/bash -x

function error_exit
{
  # wait logs flush before signaling the failure
  sleep 10
  # trim the error message because there is a size limit of 4096 bytes for cfn-signal
  # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/cloudformation-limits.html
  cutoff=$(expr 4096 - $(stat --printf="%s" /tmp/wait_condition_handle.txt))
  reason=$(head --bytes=${!cutoff} /var/log/parallelcluster/bootstrap_error_msg 2>/dev/null) || reason="$1"
  cfn-signal --exit-code=1 --reason="${!reason}" "${!wait_condition_handle_presigned_url}" --region ${AWS::Region} --url ${CloudFormationUrl}
  exit 1
}
function vendor_cookbook
{
  mkdir /tmp/cookbooks
  cd /tmp/cookbooks
  tar -xzf /etc/chef/aws-parallelcluster-cookbook.tgz
  HOME_BAK="${!HOME}"
  export HOME="/tmp"
  for d in `ls /tmp/cookbooks`; do
    cd /tmp/cookbooks/$d
    LANG=en_US.UTF-8 /opt/cinc/embedded/bin/berks vendor /etc/chef/cookbooks --delete || error_exit 'Vendoring cookbook failed.'
  done;
  export HOME="${!HOME_BAK}"
}
[ -f /etc/profile.d/proxy.sh ] && . /etc/profile.d/proxy.sh

# Configure AWS CLI using the expected overrides, if any.
[ -f /etc/profile.d/aws-cli-default-config.sh ] && . /etc/profile.d/aws-cli-default-config.sh

# deploy config files
export PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/opt/aws/bin
# Load ParallelCluster environment variables
[ -f /etc/profile.d/pcluster.sh ] && . /etc/profile.d/pcluster.sh

cd /tmp
cfn-init -s ${AWS::StackName} -v -c deployFiles -r HeadNodeLaunchTemplate --region ${AWS::Region} --url ${CloudFormationUrl}
wait_condition_handle_presigned_url=$(cat /tmp/wait_condition_handle.txt)

custom_cookbook=${CustomChefCookbook}
export _region=${AWS::Region}
s3_url=${AWS::URLSuffix}
if [ "${!custom_cookbook}" != "NONE" ]; then
  if [[ "${!custom_cookbook}" =~ ^s3://([^/]*)(.*) ]]; then
    bucket_region=$(aws s3api get-bucket-location --bucket ${!BASH_REMATCH[1]} --region ${AWS::Region} | jq -r '.LocationConstraint')
    if [[ "${!bucket_region}" == null ]]; then
      bucket_region="us-east-1"
    fi
    cookbook_url=$(aws s3 presign "${!custom_cookbook}" --region "${!bucket_region}")
  else
    cookbook_url=${!custom_cookbook}
  fi
fi
export parallelcluster_version=aws-parallelcluster-${ParallelClusterVersion}
export cookbook_version=${CookbookVersion}
export chef_version=${ChefVersion}
export berkshelf_version=${BerkshelfVersion}
if [ -f /opt/parallelcluster/.bootstrapped ]; then
  installed_version=$(cat /opt/parallelcluster/.bootstrapped)
  if [ "${!cookbook_version}" != "${!installed_version}" ]; then
    cookbook_version_number=$(echo ${!cookbook_version} | awk -F- '{print $NF}')
    installed_version_number=$(echo ${!installed_version} | awk -F- '{print $NF}')
    error_exit "This AMI was created with ${!installed_version_number}, but is trying to be used with ${!cookbook_version_number}. Please either use an AMI created with ${!cookbook_version_number} or change your ParallelCluster to ${!installed_version_number}"
  fi
else
  error_exit "This AMI was not baked by ParallelCluster. Please use pcluster build-image command to create an AMI by providing your AMI as parent image."
fi
if [ "${!custom_cookbook}" != "NONE" ]; then
  curl --retry 3 -v -L -o /etc/chef/aws-parallelcluster-cookbook.tgz ${!cookbook_url}
  vendor_cookbook
fi

# Call CloudFormation
cfn-init -s ${AWS::StackName} -v -c default -r HeadNodeLaunchTemplate --region ${AWS::Region} --url ${CloudFormationUrl} || error_exit 'Failed to bootstrap the head node. Please check /var/log/cfn-init.log or /var/log/chef-client.log in the head node, or check the cfn-init.log or chef-client.log in CloudWatch logs. Please refer to https://docs.aws.amazon.com/parallelcluster/latest/ug/troubleshooting-v3.html#troubleshooting-v3-get-logs for more details on ParallelCluster logs.'
cfn-signal --exit-code=0 --reason="HeadNode setup complete" "${!wait_condition_handle_presigned_url}" --region ${AWS::Region} --url ${CloudFormationUrl}
# End of file
--==BOUNDARY==


#!/bin/bash
# Very basic first tests

set -ex
echo $PATH
which pcluster
pip check
pcluster version
pcluster --help


#!/bin/bash
set -ex

# shellcheck source=/dev/null
source /etc/parallelcluster/cfnconfig
CLUSTER_NAME="${stack_name:?}"
AWS_DEFAULT_REGION="${cfn_region:?}"

NUM_COMPUTE_NODES=$(/opt/slurm/bin/sinfo --Node --noheader | awk '{print $1,$4}' | grep -cEv '(~|#|%)$')
aws cloudwatch put-metric-data \
    --region "$AWS_DEFAULT_REGION" \
    --namespace 'ParallelCluster/AdIntegration' \
    --metric-name 'ComputeNodeCount' \
    --dimensions "ClusterName=${CLUSTER_NAME}" \
    --value "$NUM_COMPUTE_NODES"


#!/bin/bash
set -ex

# TODO: add efs
for fspath in shared efs; do
    # srun has to be used for whoami because slurm_nss plugin only send user information through srun
    date '+%Y%m%d%H%M%S' > "/$fspath/$(srun whoami)"
done

BENCHMARK_NAME=osu_barrier
OSU_BENCHMARK_VERSION=5.7.1

module load openmpi
# Run collective benchmark. The collective operations are close to what a real application looks like.
# NOTE: The test is sized for 4 compute nodes.
# -np total number of processes to run (all CPUs * 4 nodes)
mpirun \
    > /shared/"$(date '+%Y%m%d%H%M%S')-$(srun whoami)-${BENCHMARK_NAME}".out \
    /shared/openmpi/osu-micro-benchmarks-${OSU_BENCHMARK_VERSION}/mpi/collective/${BENCHMARK_NAME}


#!/bin/bash

echo "post-install script has $# arguments"
for arg in "$@"
do
    echo "arg: ${arg}"
done

case $# in
    3)
        exit 0
    ;;
    *)
        exit 1
esac


#!/bin/bash

echo "pre-install script has $# arguments"
for arg in "$@"
do
    echo "arg: ${arg}"
done

case $# in
    3)
        exit 0
    ;;
    *)
        exit 1
esac


#!/bin/bash
# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License").
# You may not use this file except in compliance with the License.
# A copy of the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "LICENSE.txt" file accompanying this file.
# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied.
# See the License for the specific language governing permissions and limitations under the License.
# add debug mode
sed -i '$s/}/,\n"agent":{"debug": true}}/' /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.d/file_amazon-cloudwatch-agent.json
# restart cw agent
/opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.d/file_amazon-cloudwatch-agent.json


#!/bin/bash
#
# Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the
# License. A copy of the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "LICENSE.txt" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions and
# limitations under the License.

# Requirements: sudo, user be able to become root without password, ethtool, netstat

# Script to generate a system information archive.
# This script needs to be run by a user in sudoers and it takes an existing directory as argument

function log() {
  local LEVEL="INFO"
  if [ "${2}" != "" ]; then
    LEVEL="${2}"
  fi
  echo "$(date) - system-analyzer - ${LEVEL} - ${1}"
}

function body() {
  # Function to print the header passed in pipe and execute the argument
  # example: cat file_with_header.csv | body sort > file_with_header_but_sorted_body.csv
  IFS= read -r HEADER
  printf '%s\n' "${HEADER}"
  "${@}"
}

function _copy_if_exists() {
  if [ -e "${1}" ]; then
    cp -r "${1}" "${2}"
  else
    log "${1} do not exists" "WARNING"
  fi
}

function signal_handler() {
  # arg 1: return code
  # arg 2: line number of the error
  # arg 3: temporary directory to remove

  if [ "${1}" != "0" ]; then
    log "Catching a signal" "ERROR"
    rm -fr "${BASE_TEMP_DIR}"
    log "Return code:${1} occurred on line ${2} - Deleted ${3}" "ERROR"
    exit "${1}"
  fi
}

function _save_installed_services_timers() {
  log "Save installed services and timer"
  local OUT_DIR=${1}

  # Save installed services
  systemctl --all --type=service --state running,active | head -n -7 | body sort -du > "${OUT_DIR}"/services_active
  systemctl list-unit-files --state enabled,generated | head -n -2 | body sort -du > "${OUT_DIR}"/services_enabled

  # Save timers
  systemctl list-timers | head -n -3 | body sort -du > "${OUT_DIR}"/timers
}

function _save_scheduled_commands() {
  log "Save scheduled commands"
  local OUT_DIR=${1}
  local OS=${2}
  local OS_VERSION=${3}

  # Save scheduled commands and scripts
  mkdir "${OUT_DIR}"/etc_cron
  mkdir "${OUT_DIR}"/spool_cron

  _copy_if_exists /etc/cron* "${OUT_DIR}"/etc_cron
  _copy_if_exists /var/spool/cron "${OUT_DIR}"/spool_cron

  _copy_if_exists /etc/anacrontab "${OUT_DIR}"/etc_cron
  _copy_if_exists /var/spool/anacron "${OUT_DIR}"/spool_cron

  _copy_if_exists /var/spool/at "${OUT_DIR}"/spool_cron

}

function _network_info() {
  log "Save network information"
  local OUT_DIR="${1}/network/"

  # Save meaningful network statistic
  mkdir "${OUT_DIR}"
  for network in $(ip link | egrep "^[0-9]+" | awk -F\: '{print $2}' | tr -d " " | grep -v "lo"); do
    ethtool -S "${network}" > "${OUT_DIR}"/eth0_stats
  done
  netstat -s > "${OUT_DIR}"/netstat_stats
  ip address > "${OUT_DIR}"/address
}

function _save_avail_mpi() {
  log "Save mpi versions"
  local OUT_DIR="${1}/mpi/"

  # Save available mpi
  mkdir "${OUT_DIR}"
  MODULES="$(echo "${MODULEPATH}" | tr : '\n')"
  set +e +o pipefail
  MPI_MODULES=$(for MODULE in $MODULES; do
    ls -1 "${MODULE}" 2>/dev/null | grep -i mpi;
    done)
  set -e -o pipefail
  for MPI_MODULE in $MPI_MODULES
  do
    module load "${MPI_MODULE}" 2>&1 1>/dev/null
    mpirun --version > "${OUT_DIR}"/"${MPI_MODULE}"
    module unload "${MPI_MODULE}" 2>&1 1>/dev/null
  done
}

function _save_imds_info() {
  log "Save IMDSv2 information"
  # Misc IMDS info
  TOKEN="$(curl -s -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600")"
  curl -s -H "X-aws-ec2-metadata-token: ${TOKEN}" http://169.254.169.254/latest/meta-data/ami-id > "${TEMP_DIR}"/ami-id
  curl -s -H "X-aws-ec2-metadata-token: ${TOKEN}" http://169.254.169.254/latest/meta-data/instance-type > "${TEMP_DIR}"/instance-type
  curl -s -H "X-aws-ec2-metadata-token: ${TOKEN}" http://169.254.169.254/latest/user-data > "${TEMP_DIR}"/user-data
}

function _is_user_root() {
  [ "${EUID:-$(id -u)}" -eq 0 ]
}

function main() {

  # Set switch to catch errors
  set -e -o pipefail

  # Check number of arguments
  if [ $# -ne 1 ]; then
    echo "usage: ${0} [directory]"
    echo "es. ${0} /tmp/"
    exit 1
  fi

  # Become root if not and re-execute the script itself
  if ! _is_user_root; then
    log "Running the script as root"
    sudo bash --login ${0} ${1}
    exit $?
  fi

  # Check if path exist
  if [ -d "${1}" ]; then
    log "Directory ${1} exists."
  else
    log "Directory ${1} does not exist - exiting"
    exit 1
  fi

  local RESULT_ARCHIVE="${1}/system-information.tar.gz"

  # temporary directory
  local BASE_TEMP_DIR="$(mktemp -d)"
  local TEMP_DIR="${BASE_TEMP_DIR}/system-information/"

  # Register signal handling to clean the temporary directory in case of kill, kill -9, ctrl+c, error in the script
  trap 'signal_handler ${?} ${LINENO} ${BASE_TEMP_DIR}' SIGINT SIGTERM SIGHUP INT ERR EXIT

  log "Create temporary directory"
  mkdir "${TEMP_DIR}"

  # Find os type and select package command
  local PACKAGE_REPORT_CMD=""
  local OS="$(grep "^ID=" /etc/os-release | cut -d"=" -f 2 | xargs)"

  case ${OS} in
    ubuntu)
      PACKAGE_REPORT_CMD="dpkg-query -l"
      ;;
    amzn | centos | rhel)
      PACKAGE_REPORT_CMD="rpm -qa"
      ;;
    *)
      echo "Unrecognized system. Found /etc/os-release ID content: ${OS}"
      exit 1
      ;;
  esac

  local OS_VERSION="$(grep "^VERSION_ID=" /etc/os-release | cut -d"=" -f 2 | xargs)"

  log "Save OS type and version"
  echo "${OS}" > "${TEMP_DIR}/os"
  echo "${OS_VERSION}" > "${TEMP_DIR}/os_version"

  # Save uname
  log "Save uname"
  uname -a > "${TEMP_DIR}/uname"

  # Save installed packages on system
  log "Save installed packages on system"
  ${PACKAGE_REPORT_CMD} | sort -du > "${TEMP_DIR}"/packages

  _save_installed_services_timers "${TEMP_DIR}"

  _save_scheduled_commands "${TEMP_DIR}" "${OS}" "${OS_VERSION}"

  _save_avail_mpi "${TEMP_DIR}"

  _network_info "${TEMP_DIR}"

  _save_imds_info "${TEMP_DIR}"

  # Save users info
  log "Save /etc/passwd content"
  cp /etc/passwd "${TEMP_DIR}/passwd"

  # Create the archive
  log "Create the archive"
  rm -fr "${RESULT_ARCHIVE}"
  cd "${TEMP_DIR}/.."
  tar -czf "${RESULT_ARCHIVE}" "system-information/" 1>/dev/null
  cd "/"
  rm -fr "${BASE_TEMP_DIR}"
  log "DONE"
}

log "START"
main "${@}"


#!/bin/bash
set -e

rm -f /shared/mpi.out
module load openmpi
mpirun --map-by ppr:1:node --timeout 20 "ring" >> /shared/mpi.out


#!/usr/bin/env bash
set -e

MPI_VERSION=${1}

OSU_BENCHMARKS_VERSION={{ osu_benchmark_version }}
OSU_BENCHMARKS_PACKAGE_NAME="osu-micro-benchmarks-${OSU_BENCHMARKS_VERSION}"
OSU_BENCHMARKS_INSTALLATION_DIR="/shared/${MPI_VERSION}/${OSU_BENCHMARKS_PACKAGE_NAME}/"

# If the compilation directory already exists, skip the compilation.
[ -d "${OSU_BENCHMARKS_INSTALLATION_DIR}" ] && exit 0

module load ${MPI_VERSION}
mkdir -p /shared/${MPI_VERSION}

#wget --no-check-certificate http://mvapich.cse.ohio-state.edu/download/mvapich/${OSU_BENCHMARKS_PACKAGE_NAME}.tgz
cp "./${OSU_BENCHMARKS_PACKAGE_NAME}.tgz" /shared/${MPI_VERSION}
cd /shared/${MPI_VERSION}
tar zxvf "./${OSU_BENCHMARKS_PACKAGE_NAME}.tgz"

# Update config.guess and config.sub files to support ARM architecture.
cd
cp "./config.guess" "${OSU_BENCHMARKS_INSTALLATION_DIR}"
cp "./config.sub" "${OSU_BENCHMARKS_INSTALLATION_DIR}"

# Compile OSU benchmarks
cd "${OSU_BENCHMARKS_INSTALLATION_DIR}"
./configure CC=$(which mpicc) CXX=$(which mpicxx)
make


#!/bin/bash
set -e

BENCHMARK_NAME={{ benchmark_name }}
OSU_BENCHMARK_VERSION={{ osu_benchmark_version }}
NUM_OF_PROCESSES={{ num_of_processes }}

module load intelmpi
export I_MPI_DEBUG=10

env

# Run collective benchmark. The collective operations are close to what a real application looks like.
# -np total number of processes to run (all vCPUs * N compute nodes), divided by 2 if multithreading is disabled
mpirun -bootstrap=slurm -np ${NUM_OF_PROCESSES} /shared/intelmpi/osu-micro-benchmarks-${OSU_BENCHMARK_VERSION}/mpi/collective/${BENCHMARK_NAME} > /shared/${BENCHMARK_NAME}.out


#!/bin/bash
set -e

BENCHMARK_NAME={{ benchmark_name }}
OSU_BENCHMARK_VERSION={{ osu_benchmark_version }}
NUM_OF_PROCESSES={{ num_of_processes }}

module load openmpi

env

# Run collective benchmark. The collective operations are close to what a real application looks like.
# -np total number of processes to run (all vCPUs * N compute nodes), divided by 2 if multithreading is disabled
mpirun -np ${NUM_OF_PROCESSES} /shared/openmpi/osu-micro-benchmarks-${OSU_BENCHMARK_VERSION}/mpi/collective/${BENCHMARK_NAME}  > /shared/${BENCHMARK_NAME}.out


#!/bin/bash
set -e
module load openmpi

BENCHMARK_NAME={{ benchmark_name }}
OSU_BENCHMARK_VERSION={{ osu_benchmark_version }}

env

# Run multiple bandwidth/message rate benchmark
# NOTE: The test is sized for two P4d compute nodes.
# -N: number of processes per node (48, 1 for each CPU)
# -n total number of processes to run (96, all CPUs from 2 nodes)
# -x FI_EFA_USE_DEVICE_RDMA=1 Enables RDMA support
mpirun --mca btl_tcp_if_exclude lo -n 96 -N 48 -x FI_EFA_USE_DEVICE_RDMA=1 /shared/openmpi/osu-micro-benchmarks-${OSU_BENCHMARK_VERSION}/mpi/pt2pt/${BENCHMARK_NAME} > /shared/${BENCHMARK_NAME}.out


#!/bin/bash
set -e

BENCHMARK_NAME={{ benchmark_name }}
OSU_BENCHMARK_VERSION={{ osu_benchmark_version }}

module load intelmpi
export I_MPI_DEBUG=10

env

mpirun -bootstrap=slurm  -np 2 --map-by ppr:1:node /shared/intelmpi/osu-micro-benchmarks-${OSU_BENCHMARK_VERSION}/mpi/pt2pt/${BENCHMARK_NAME} > /shared/${BENCHMARK_NAME}.out


#!/bin/bash
set -e

BENCHMARK_NAME={{ benchmark_name }}
OSU_BENCHMARK_VERSION={{ osu_benchmark_version }}

module load openmpi

env

mpirun -np 2 --map-by ppr:1:node /shared/openmpi/osu-micro-benchmarks-${OSU_BENCHMARK_VERSION}/mpi/pt2pt/${BENCHMARK_NAME} > /shared/${BENCHMARK_NAME}.out


#!/bin/bash
# Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License").
# You may not use this file except in compliance with the License.
# A copy of the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "LICENSE.txt" file accompanying this file.
# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied.
# See the License for the specific language governing permissions and limitations under the License.
echo "Test head node bootstrap error caused by OnNodeStart Script."
exit 1


#!/bin/sh
apt update -y
apt-get install -y nodejs


#!/bin/bash
set -x

CRASHDIR=/var/crash

# There are no crash files if the directory doesn't exist
[ -d ${CRASHDIR} ] || exit 0

# Exit nonzero if there are any files
crash_files="$(ls -A ${CRASHDIR})"
if [ -n "${crash_files}" ]; then
  echo "Found crash files in ${CRASHDIR}: ${crash_files}"
  exit 1
fi


#!/usr/bin/env bash
set -e

rm -rf /shared/${1}

module load ${1}
NCCL_BENCHMARKS_VERSION='2.10.0'
NCCL_VERSION='2.7.8-1'
OFI_NCCL_VERSION='1.1.1'
MPI_HOME=$(which mpirun | awk -F '/bin' '{print $1}')
NVCC_GENCODE="-gencode=arch=compute_80,code=sm_80" # Arch for NVIDIA A100

mkdir -p /shared/${1}

# Install and build nccl from sources
cd /shared/${1}
wget https://github.com/NVIDIA/nccl/archive/v${NCCL_VERSION}.tar.gz
tar zxvf "v${NCCL_VERSION}.tar.gz"
cd nccl-${NCCL_VERSION}
make -j src.build NVCC_GENCODE="${NVCC_GENCODE}"

# Build nccl-tests
cd /shared/${1}
wget https://github.com/NVIDIA/nccl-tests/archive/v${NCCL_BENCHMARKS_VERSION}.tar.gz
tar zxvf "v${NCCL_BENCHMARKS_VERSION}.tar.gz"
cd "nccl-tests-${NCCL_BENCHMARKS_VERSION}/"
NVCC_GENCODE="${NVCC_GENCODE}" make MPI=1 MPI_HOME=${MPI_HOME} NCCL_HOME=/shared/${1}/nccl-${NCCL_VERSION}/build/

wget https://github.com/aws/aws-ofi-nccl/archive/v${OFI_NCCL_VERSION}.tar.gz
tar xvfz v${OFI_NCCL_VERSION}.tar.gz
cd aws-ofi-nccl-${OFI_NCCL_VERSION}
./autogen.sh
./configure --with-libfabric=/opt/amazon/efa --with-cuda=/usr/local/cuda/targets/x86_64-linux/ --with-nccl=/shared/openmpi/nccl-${NCCL_VERSION}/build/ --with-mpi=${MPI_HOME} --prefix /shared/openmpi/ofi-plugin
make
make install


#!/bin/bash
#SBATCH --nodes=2
#SBATCH --exclusive
#SBATCH --ntasks-per-node=8

module load openmpi
NCCL_VERSION='2.7.8-1'
mpirun \
-x FI_PROVIDER="efa" \
-x FI_EFA_USE_DEVICE_RDMA=1 \
-x LD_LIBRARY_PATH=/shared/openmpi/nccl-${NCCL_VERSION}/build/lib/:/shared/openmpi/ofi-plugin/lib/:$LD_LIBRARY_PATH \
-x RDMAV_FORK_SAFE=1 \
-x NCCL_ALGO=ring \
-x NCCL_DEBUG=WARNING \
-x NCCL_PROTO=simple \
--mca pml ^cm --mca btl tcp,self --mca btl_tcp_if_exclude lo,docker0 --bind-to none \
/shared/openmpi/nccl-tests-2.10.0/build/all_reduce_perf -b 8 -e 1G -f 2 -g 1 -c 1 -n 100 > /shared/nccl_tests.out


#!/bin/bash
set -ex

# Installs Fabtests suite from GitHub.
# Usage: install-fabtests.sh [FABTESTS_DIR]
# Example: install-fabtests.sh /shared/fabtests

FABTESTS_DIR="$1"

FABTESTS_REPO="https://github.com/ofiwg/libfabric.git"
# TODO We should target the release 1.16.0 instead of a commit.
# We must fix this once that version will be released.
FABTESTS_COMMIT="840a4a93ccf576b950dfd15d979cbf5f06e9c64b"
FABTESTS_SOURCES_DIR="$FABTESTS_DIR/sources"
LIBFABRIC_DIR="/opt/amazon/efa"
CUDA_DIR="/usr/local/cuda"

echo "[INFO] Installing Fabtests in $FABTESTS_DIR"
rm -rf $FABTESTS_DIR
mkdir -p $FABTESTS_SOURCES_DIR
cd $FABTESTS_SOURCES_DIR
git clone $FABTESTS_REPO
cd libfabric
git reset --hard $FABTESTS_COMMIT
cd fabtests
./autogen.sh
./configure --with-libfabric=$LIBFABRIC_DIR --with-cuda=$CUDA_DIR --prefix=$FABTESTS_DIR && make -j 32 && make install
python3 -m pip install -r $FABTESTS_SOURCES_DIR/libfabric/fabtests/pytest/requirements.txt
python3 -m pip install pyyaml # TODO This is required but it is missing in the above requirements.txt
echo "[INFO] Fabtests installed in $FABTESTS_DIR"


#!/bin/bash
set -ex

# Executes Fabtests suite.
# Usage: run-fabtests.sh [FABTESTS_DIR] [PID_FILE] [LOG_FILE] [REPORT_FILE] [COMPUTE_NODE_1] [COMPUTE_NODE_2] [TEST_CASES] [TEST_OPTIONS]
# Example:
# run-fabtests.sh \
# /shared/fabtests \
# /shared/fabtests/outputs/fabtests.pid \
# /shared/fabtests/outputs/fabtests.log \
# /shared/fabtests/outputs/fabtests.report \
# q1-st-g4dn8xl-efa-1 q1-st-g4dn8xl-efa-2 \
# rdm_tagged_bw,rdm_tagged_pingpong,runt \
# enable-gdr

FABTESTS_DIR="$1"
PID_FILE="$2"
LOG_FILE="$3"
REPORT_FILE="$4"
COMPUTE_NODE_1="$5"
COMPUTE_NODE_2="$6"
TEST_CASES="$7"
TEST_OPTIONS="$8"

echo "[INFO] Starting Fabtests ($FABTESTS_DIR) using compute nodes $COMPUTE_NODE_1 and $COMPUTE_NODE_2: $TEST_CASES"
echo "[INFO] Fabtests will use the following test environment options: $TEST_OPTIONS"
echo "[INFO] Fabtests pid will be stored in $PID_FILE"
echo "[INFO] Fabtests logs will be stored in $LOG_FILE"
echo "[INFO] Fabtests report will be stored in $REPORT_FILE"

FABTESTS_BIN_DIR="$FABTESTS_DIR/bin"
FABTESTS_RUNNER="$FABTESTS_BIN_DIR/runfabtests.py"
FABTESTS_TIMEOUT="300" # Timeout for each test
COMPUTE_IP_1=$(host $COMPUTE_NODE_1 | cut -d ' ' -f 4)
COMPUTE_IP_2=$(host $COMPUTE_NODE_2 | cut -d ' ' -f 4)
TEST_EXPRESSION=${TEST_CASES//,/ or }
TEST_ENVIRONMENT_OPTION=""
[[ "$TEST_OPTIONS" == *"enable-gdr"* ]] && TEST_ENVIRONMENT_OPTION="-E \"FI_EFA_USE_DEVICE_RDMA=1\""

mkdir -p $(dirname $PID_FILE)
mkdir -p $(dirname $LOG_FILE)
mkdir -p $(dirname $REPORT_FILE)

python3 $FABTESTS_RUNNER \
  -b efa \
  -p "$FABTESTS_BIN_DIR" \
  --expression "$TEST_EXPRESSION" \
  --timeout $FABTESTS_TIMEOUT \
  --junit-xml $REPORT_FILE \
  -v $TEST_ENVIRONMENT_OPTION $COMPUTE_IP_1 $COMPUTE_IP_2 > $LOG_FILE 2>&1 &

echo $! > $PID_FILE

echo "[INFO] Fabtests launched"


#!/bin/bash

mkdir -p $HOME/mock_health_checks
sudo tee $HOME/mock_health_checks/gpu_health_check.sh > /dev/null <<EOT
#!/bin/bash
exit 1
EOT
sudo chmod +x $HOME/mock_health_checks/gpu_health_check.sh

sudo sed -i '/managed_health_check_dir =/ s|\/opt\/slurm\/etc\/pcluster\/\.slurm_plugin\/scripts\/health_checks|'$HOME'\/mock_health_checks|' /opt/slurm/etc/pcluster/.slurm_plugin/scripts/conf/health_check_manager.conf
echo "Mocked failing GPU Health Check"
cat /opt/slurm/etc/pcluster/.slurm_plugin/scripts/conf/health_check_manager.conf


#!/bin/bash

mkdir -p $HOME/mock_health_checks
sudo tee $HOME/mock_health_checks/gpu_health_check.sh > /dev/null <<EOT
#!/bin/bash
exit 0
EOT
sudo chmod +x $HOME/mock_health_checks/gpu_health_check.sh

sudo sed -i '/managed_health_check_dir =/ s|\/opt\/slurm\/etc\/pcluster\/\.slurm_plugin\/scripts\/health_checks|'$HOME'\/mock_health_checks|' /opt/slurm/etc/pcluster/.slurm_plugin/scripts/conf/health_check_manager.conf
echo "Mocked successful GPU Health Check"
cat /opt/slurm/etc/pcluster/.slurm_plugin/scripts/conf/health_check_manager.conf


#!/bin/bash

sudo sed -i '/managed_health_check_dir =/ s|'$HOME'\/mock_health_checks|\/opt\/slurm\/etc\/pcluster\/\.slurm_plugin\/scripts\/health_checks|' /opt/slurm/etc/pcluster/.slurm_plugin/scripts/conf/health_check_manager.conf

echo "Health check configuration restored"
cat /opt/slurm/etc/pcluster/.slurm_plugin/scripts/conf/health_check_manager.conf


#!/usr/bin/env bash
# These installation instructions come from the following URL:
# https://software.intel.com/content/www/us/en/develop/documentation/cluster-checker-user-guide/top/installation.html#YUMINSTALL
# To see the latest available versions:
# $ repoquery --repofrompath=reponame,https://yum.repos.intel.com/clck/2019 --repoid=reponame -a
set -e

sudo rpm --import https://yum.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB
sudo yum-config-manager --add-repo https://yum.repos.intel.com/clck/2019/setup/intel-clck-2019.repo
sudo yum-config-manager --add-repo https://yum.repos.intel.com/clck-ext/2019/setup/intel-clck-ext-2019.repo
sudo yum -y install intel-clck-2019.9-056


#!/bin/bash
set -e

source /opt/intel/clck/2019.9/bin/clckvars.sh
clck -f nodefile -F intel_hpc_platform_compat-hpc-2018.0


#!/bin/bash
echo "This script can't stop, won't stop." 1>&2
exit 1


#!/bin/bash
#SBATCH --ntasks=2 --nodes=2

set -e

module load intelmpi
srun --mpi=pmi2 /shared/intelmpi/osu-micro-benchmarks-{{ osu_benchmark_version }}/mpi/pt2pt/osu_latency
mpirun /shared/intelmpi/osu-micro-benchmarks-{{ osu_benchmark_version }}/mpi/pt2pt/osu_latency


#!/bin/bash
#SBATCH --ntasks=2 --nodes=2

set -e

module load openmpi
srun --mpi=pmix /shared/openmpi/osu-micro-benchmarks-{{ osu_benchmark_version }}/mpi/pt2pt/osu_latency
mpirun /shared/openmpi/osu-micro-benchmarks-{{ osu_benchmark_version }}/mpi/pt2pt/osu_latency


#!/bin/bash

echo "custom_action script has $# arguments"
for arg in "$@"
do
    echo "executing ${arg}"
    "${arg}" || echo "failed to execute ${arg}"
done


#!/bin/bash

echo "--- id: $(id)"
echo "--- whoami: $(whoami)"
echo "--- hostname: $(hostname)"
echo "--- pwd: $(pwd)"
echo "--- PATH: $(echo ${PATH})"
echo "--- HOME: $(echo ${HOME})"


#!/bin/bash

function _log () {
  echo "$(date +"%Y-%m-%dT%H-%M-%S") [$1] $2"
}

function log () {
  _log "INFO" "$1"
}

function log_error () {
  _log "ERROR" "$1"
}

function fail () {
  log_error "$1"
  exit 1
}

function download_asset () {
  local s3_uri="$1"
  local local_file="$2"
  local permissions="$3"

  mkdir -p $(dirname "${local_file}")
  aws s3 cp "${s3_uri}" "${local_file}"
  chmod ${permissions} "${local_file}"
}

function cron_script () {
  local cron_expression="$1"
  local script="$2"
  local log="$3"

  echo "${cron_expression} ${script} > ${log} 2>&1" >> "/var/spool/cron/$(whoami)"
}

function timestamp_millis () {
  date +"%s%3N"
}

function timestamp_datetime () {
  date +"%Y-%m-%dT%H-%M-%S"
}

function millis_to_date_time () {
  date -d @${1::-3} +"%Y-%m-%dT%H-%M-%S"
}

function date_time_to_millis () {
  date -d ${1} +"%Y-%m-%dT%H-%M-%S"
}

function get_instance_id () {
  # This function uses IMDS to retrieve the instance id, so it can be used:
  # 1. in the head node only if the function is executed by a super user or if IMDS secured is disabled
  # 2. in a compute node, always as there is no IMDS lockdown in there
  local imds_endpoint="http://169.254.169.254/latest"
  local imds_token=$(curl -X PUT "${imds_endpoint}/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600" 2>/dev/null)

  curl -H "aws-ec2-metadata-token: ${imds_token}" "${imds_endpoint}/meta-data/instance-id" 2>/dev/null
}

function wait_job_completion () {
  local job_ids=$1
  local sleep_seconds=$2
  local expected_jobs_in_queue=0

  while true; do
    local jobs_in_queue=$(/opt/slurm/bin/squeue -h -j${job_ids} | wc -l)
    log "Current jobs in queue: ${jobs_in_queue}. Expected jobs in queue: ${expected_jobs_in_queue}"
    [ ${jobs_in_queue} == ${expected_jobs_in_queue} ] && break
    log "Sleeping ${sleep_seconds} seconds"
    sleep ${sleep_seconds}
  done
}

function get_compute_fleet_running_nodes () {
  local cluster_name=$1
  local region=$2

  /usr/local/bin/pcluster describe-cluster-instances --region ${region} --cluster-name "${cluster_name}" --query 'instances[].nodeType'  | grep -o ComputeNode  | wc -l
}

function terminate_compute_fleet () {
    local cluster_name=$1
    local region=$2

    /usr/local/bin/pcluster delete-cluster-instances --region ${region} --cluster-name "${cluster_name}" --force true
}

function wait_compute_fleet () {
  local cluster_name=$1
  local region=$2
  local expected_compute_nodes=$3
  local sleep_seconds=$4

  while true; do
    local compute_nodes=$(get_compute_fleet_running_nodes ${cluster_name} ${region})
    log "Current compute nodes ${compute_nodes}. Expected compute nodes ${expected_compute_nodes}."
    [ ${compute_nodes} == ${expected_compute_nodes} ] && break
    log "Sleeping ${sleep_seconds} seconds"
    sleep ${sleep_seconds}
  done
}

function add_to_json () {
  local key=$1
  local value=$2
  local json_file=$3

  [[ ! -f ${json_file} ]] && echo "{}" > ${json_file}
  echo $(jq ".${key} = \"${value}\"" ${json_file}) > ${json_file}
}

function merge_json () {
  local json_file_1=$1
  local json_file_2=$2
  local json_file_3=$3

  echo $(jq --argfile f1 "${json_file_1}" --argfile f2 "${json_file_2}" -n '$f1 * $f2') > "${json_file_3}"
}

function get_sample_from_json () {
  local key="$1"
  local files="$2"

  jq -n "[inputs.${key}]" ${files} | jq -r -c 'map(select(length > 0)) | join(",")'
}

function get_json_with_minimum () {
  local key="$1"
  local files="$2"

  jq -n "[inputs] | sort_by(.${key}) | .[0]" ${files}
}

function get_json_with_maximum () {
  local key="$1"
  local files="$2"

  jq -n "[inputs] | sort_by(.${key}) | .[-1]" ${files}
}

function get_min () {
  python3 -c "print(min([${1}]))"
}

function get_max () {
  python3 -c "print(max([${1}]))"
}

function get_avg () {
  python3 -c "from statistics import mean;print(mean([${1}]))"
}

function get_std () {
  python3 -c "from statistics import stdev;print(stdev([${1}]))"
}

function get_med () {
  python3 -c "from statistics import median;print(median([${1}]))"
}

function get_prc () {
  python3 -c "from numpy import percentile;print(percentile([${1}], ${2}))"
}

function get_circular_array_element () {
  local position=${1}
  shift
  local array=("$@")
  local array_size=${#array[@]}
  local index=$(( ${position} % ${array_size} ))

  echo "${array[@]:${index}:1}"
}

function join_array_by {
  local d=${1-} a=${2-}

  if shift 2; then
    printf %s "$a" "${@/#/$d}"
  fi
}

function put_metric() {
  local region="$1"
  local namespace="$2"
  local dimensions="$3"
  local metric_name="$4"
  local value="$5"

  aws cloudwatch put-metric-data \
      --region "${region}" \
      --namespace "${namespace}" \
      --dimensions "${dimensions}" \
      --metric-name "${metric_name}" \
      --value "${value}"
}


#!/bin/bash
set -e

# Cluster variables
source /etc/parallelcluster/cfnconfig
SHARED_DIR="$(echo $cfn_ebs_shared_dirs | cut -d ',' -f 1)"
CLUSTER_NAME="${stack_name:?}"
AWS_DEFAULT_REGION="${cfn_region:?}"

# Load libraries
source "${SHARED_DIR}/assets/lib/functions.sh"

CW_NAMESPACE="ParallelCluster/PerformanceTests"
CW_DIMENSIONS="ClusterName=${CLUSTER_NAME}"
PCLUSTER="/usr/local/bin/pcluster"
SQUEUE="/opt/slurm/bin/squeue"

function publish_metric () {
  local metric_name="$1"
  local metric_value="$2"

  put_metric ${AWS_DEFAULT_REGION} ${CW_NAMESPACE} "${CW_DIMENSIONS}" "${metric_name}" "${metric_value}"
}

log "Publishing metrics for cluster ${CLUSTER_NAME} in region ${AWS_DEFAULT_REGION} to CloudWatch namespace ${CW_NAMESPACE}"

NUM_COMPUTE_NODES=$(${PCLUSTER} describe-cluster-instances --region ${AWS_DEFAULT_REGION} --cluster-name "$CLUSTER_NAME" --query 'instances[].nodeType'  | grep -o ComputeNode  | wc -l)
publish_metric "ComputeNodeCount" "${NUM_COMPUTE_NODES}"

NUM_JOBS_QUEUED=$(${SQUEUE} -h | wc -l)
publish_metric "JobsQueuedCount" "${NUM_JOBS_QUEUED}"

NUM_JOBS_PENDING=$(${SQUEUE} -h -t configuring,pending | wc -l)
publish_metric "JobsPendingCount" "${NUM_JOBS_PENDING}"

NUM_JOBS_RUNNING=$(${SQUEUE} -h -t running | wc -l)
publish_metric "JobsRunningCount" "${NUM_JOBS_RUNNING}"

NUM_JOBS_FAILED=$(${SQUEUE} -h -t failed | wc -l)
publish_metric "JobsFailedCount" "${NUM_JOBS_FAILED}"

log "Published metrics for cluster ${CLUSTER_NAME} in region ${AWS_DEFAULT_REGION} to CloudWatch namespace ${CW_NAMESPACE}"


#!/bin/bash
set -ex

# Cluster variables
source /etc/parallelcluster/cfnconfig
SHARED_DIR="$(echo $cfn_ebs_shared_dirs | cut -d ',' -f 1)"

# Load libraries
FUNCTIONS_SCRIPT="${SHARED_DIR}/assets/lib/functions.sh"
source "${FUNCTIONS_SCRIPT}"

# Instance Details
INSTANCE_ID=$(get_instance_id)

# Monitoring
INSTANCE_METRICS_DIR="${SHARED_DIR}/metrics/compute-nodes"
INSTANCE_METRICS_FILE="${INSTANCE_METRICS_DIR}/instance-${INSTANCE_ID}.json"
INSTANCE_METRICS_PRE_FILE="/local/parallelcluster/monitoring/instance-metrics.json"
mkdir -m 777 -p ${INSTANCE_METRICS_DIR}

# Monitoring - PreInstall Metrics
cp ${INSTANCE_METRICS_PRE_FILE} ${INSTANCE_METRICS_FILE}

# Monitoring - Start Time (milliseconds)
INSTANCE_START_TIME=$(date +"%s%3N")
add_to_json "instancePostInstallTimestamp" ${INSTANCE_START_TIME} ${INSTANCE_METRICS_FILE}

# Monitoring - Up Time (seconds)
INSTANCE_UPTIME=$(cat /proc/uptime | cut -d ' ' -f 1 | cut -d '.' -f 1)
add_to_json "instancePostInstallUpTime" ${INSTANCE_UPTIME} ${INSTANCE_METRICS_FILE}

# Monitoring - InstanceId
add_to_json "instanceId" ${INSTANCE_ID} ${INSTANCE_METRICS_FILE}

chmod 666 ${INSTANCE_METRICS_FILE}


#!/bin/bash
set -ex

# Cluster variables
source /etc/parallelcluster/cfnconfig
SHARED_DIR="$(echo $cfn_ebs_shared_dirs | cut -d ',' -f 1)"
CLUSTER_NAME="${stack_name:?}"
AWS_DEFAULT_REGION="${cfn_region:?}"

# Inputs
S3_BUCKET_URI="${1}"

# Check inputs
[ -z ${S3_BUCKET_URI} ] && echo "$(date +"%Y-%m-%dT%H-%M-%S") [ERROR] 1st argument S3_BUCKET_URI must not be empty" && exit 1

# Load libraries
FUNCTIONS_SCRIPT_S3="${S3_BUCKET_URI}/functions.sh"
FUNCTIONS_SCRIPT="${SHARED_DIR}/assets/lib/functions.sh"
mkdir -p $(dirname "${FUNCTIONS_SCRIPT}")
aws s3 cp "${FUNCTIONS_SCRIPT_S3}" "${FUNCTIONS_SCRIPT}"
chmod 755 "${FUNCTIONS_SCRIPT}"
source "${FUNCTIONS_SCRIPT}"

# Download - Scale Test Runner
SCALE_TEST_SCRIPT_S3="${S3_BUCKET_URI}/run-scale-test.sh"
SCALE_TEST_SCRIPT="${SHARED_DIR}/assets/workloads/scale-test/run-scale-test.sh"
download_asset "${SCALE_TEST_SCRIPT_S3}" "${SCALE_TEST_SCRIPT}" 755

# Download - Scale Test Job Wrapper
SCALE_TEST_JOB_WRAPPER_SCRIPT_S3="${S3_BUCKET_URI}/scale-test-job-wrapper.sh"
SCALE_TEST_JOB_WRAPPER_SCRIPT="${SHARED_DIR}/assets/workloads/scale-test/scale-test-job-wrapper.sh"
download_asset "${SCALE_TEST_JOB_WRAPPER_SCRIPT_S3}" "${SCALE_TEST_JOB_WRAPPER_SCRIPT}" 755

# Download - Script Env Info
ENV_INFO_SCRIPT_S3="${S3_BUCKET_URI}/env-info.sh"
ENV_INFO_SCRIPT="${SHARED_DIR}/assets/workloads/env-info.sh"
download_asset "${ENV_INFO_SCRIPT_S3}" "${ENV_INFO_SCRIPT}" 755

# Install AWS ParallelCluster
rm -rf aws-parallelcluster
pip3 uninstall -y aws-parallelcluster openapi-spec-validator pyrsistent
pip3 install pyrsistent==0.16.0
git clone https://github.com/aws/aws-parallelcluster.git
pip3 install aws-parallelcluster/cli/

# Install Python libraries
pip3 install numpy

# Dashboard improvements
DASHBOARD_WIDGETS_JSON_S3="${S3_BUCKET_URI}/cloudwatch-dashboard-widgets.json"
DASHBOARD_WIDGETS_JSON="${SHARED_DIR}/assets/monitoring/cloudwatch-dashboard-widgets.json"
download_asset "${DASHBOARD_WIDGETS_JSON_S3}" "${DASHBOARD_WIDGETS_JSON}" 755

TEMP_DIR=$(mktemp -d -t pcluster-post-head-XXXXXXXXXX)

CURRENT_DASHBOARD_JSON="${TEMP_DIR}/current-dashboard.json"
CURRENT_DASHBOARD=$(aws cloudwatch get-dashboard --region ${AWS_DEFAULT_REGION} --dashboard-name "${CLUSTER_NAME}-${AWS_DEFAULT_REGION}")
echo ${CURRENT_DASHBOARD} > "${CURRENT_DASHBOARD_JSON}"

NEW_DASHBOARD_WIDGETS_JSON="${TEMP_DIR}/new-dashboard-widgets.json"
NEW_DASHBOARD_WIDGETS=$(cat ${DASHBOARD_WIDGETS_JSON})
NEW_DASHBOARD_WIDGETS=${NEW_DASHBOARD_WIDGETS//INSERT_CLUSTER_NAME_HERE/${CLUSTER_NAME}}
NEW_DASHBOARD_WIDGETS=${NEW_DASHBOARD_WIDGETS//INSERT_AWS_REGION_HERE/${AWS_DEFAULT_REGION}}
echo ${NEW_DASHBOARD_WIDGETS} > ${NEW_DASHBOARD_WIDGETS_JSON}

FINAL_DASHBOARD_BODY_JSON="${TEMP_DIR}/final-dashboard-body.json"
echo $(cat ${CURRENT_DASHBOARD_JSON} | jq -r '.DashboardBody' | jq --argfile f2 ${NEW_DASHBOARD_WIDGETS_JSON} '.widgets |= . + $f2.widgets') > ${FINAL_DASHBOARD_BODY_JSON}
aws cloudwatch put-dashboard --region ${AWS_DEFAULT_REGION} --dashboard-name "${CLUSTER_NAME}-${AWS_DEFAULT_REGION}" --dashboard-body "$(cat ${FINAL_DASHBOARD_BODY_JSON})"

# TODO must complete dashboard enrichment

# Metrics publishing
METRICS_PUBLISHER_SCRIPT_S3="${S3_BUCKET_URI}/metrics-publisher.sh"
METRICS_PUBLISHER_SCRIPT="${SHARED_DIR}/assets/monitoring/metrics-publisher.sh"
download_asset "${METRICS_PUBLISHER_SCRIPT_S3}" "${METRICS_PUBLISHER_SCRIPT}" 755
cron_script "* * * * *" "${METRICS_PUBLISHER_SCRIPT}" "${METRICS_PUBLISHER_SCRIPT}.log"


#!/bin/bash
set -ex

# Monitoring
INSTANCE_METRICS_FILE="/local/parallelcluster/monitoring/instance-metrics.json"
mkdir -p $(dirname "${INSTANCE_METRICS_FILE}")
echo "{}" > ${INSTANCE_METRICS_FILE}

# Monitoring - Start Time (milliseconds)
INSTANCE_START_TIME=$(date +"%s%3N")
echo $(jq ".instancePreInstallTimestamp = \"${INSTANCE_START_TIME}\"" ${INSTANCE_METRICS_FILE}) > ${INSTANCE_METRICS_FILE}

# Monitoring - Up Time (seconds)
INSTANCE_UPTIME=$(cat /proc/uptime | cut -d ' ' -f 1 | cut -d '.' -f 1)
echo $(jq ".instancePreInstallUpTime = \"${INSTANCE_UPTIME}\"" ${INSTANCE_METRICS_FILE}) > ${INSTANCE_METRICS_FILE}


#!/bin/bash
set -e

# Usage: /shared/assets/workloads/scale-test/run-scale-test.sh [ITERATIONS] [JOBS_PER_ITERATION] [NUM_PROCESSES_PER_JOB] [JOB_USERS] [JOB_COMMAND] [OUTPUT_DIR] [OUTPUT_S3_URI]
# Usage: /shared/assets/workloads/scale-test/run-scale-test.sh 3 5 4 ec2-user "/shared/assets/workloads/env-info.sh ; sleep 30" /shared/scale-tests/scale-test-$(date +"%Y-%m-%dT%H-%M-%S") s3://aws-parallelcluster-mgiacomo/scale-test/out
# Usage: /shared/assets/workloads/scale-test/run-scale-test.sh 5 5 4 ec2-user,PclusterUser1,PclusterUser2 "sleep 120" /shared/scale-tests/scale-test-$(date +"%Y-%m-%dT%H-%M-%S") s3://aws-parallelcluster-mgiacomo/scale-test/out

# Cluster variables
source /etc/parallelcluster/cfnconfig
CLUSTER_NAME="${stack_name}"
AWS_DEFAULT_REGION="${cfn_region}"
SHARED_DIR="$(echo $cfn_ebs_shared_dirs | cut -d ',' -f 1)"

# Load libraries
FUNCTIONS_SCRIPT="${SHARED_DIR}/assets/lib/functions.sh"
source "${FUNCTIONS_SCRIPT}"

# Functions
function launch_job () {
  local job_user=$1
  local num_processes=$2
  local job_wrapper_script=$3
  local output_dir=$4
  local job_command=$5

  local sbatch_result=$(sudo -iu ${job_user} sbatch -n ${num_processes} "${job_wrapper_script}" "${output_dir}" "${job_command}")
  local submission_line=$(echo $sbatch_result | grep "Submitted batch job")

  [ -z $submission_line ] && fail "Job submission failed"

  local job_id=$(echo $submission_line | cut -d ' ' -f 4)
  echo $job_id
}

# Input
ITERATIONS=${1:-3}
JOBS_PER_ITERATION=${2:-1}
NUM_PROCESSES_PER_JOB=${3:-1}
JOB_USERS=($(echo ${4:-$(whoami)} | tr "," " "))
JOB_COMMAND="${5:-"sleep 60"}"
OUTPUT_DIR=${6:-"${SHARED_DIR}/scale-tests/${CLUSTER_NAME}-$(timestamp_datetime)"}
OUTPUT_S3=${7}

# Job Details
JOB_WRAPPER_SCRIPT="${SHARED_DIR}/assets/workloads/scale-test/scale-test-job-wrapper.sh"

# Print Info
echo "# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #"
echo "# SCALE TEST"
echo "# - Iterations: ${ITERATIONS}"
echo "# - Jobs / Iteration: ${JOBS_PER_ITERATION}"
echo "# - Processes / Job: ${NUM_PROCESSES_PER_JOB}"
echo "# - Job Users:" "${JOB_USERS[@]}"
echo "# - Job Wrapper Script: ${JOB_WRAPPER_SCRIPT}"
echo "# - Job Command: ${JOB_COMMAND}"
echo "# - Test Output: ${OUTPUT_DIR}"
echo "# - Test Output S3: ${OUTPUT_S3}"
echo "# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #"

# Init
for iteration in $(seq 1 ${ITERATIONS}); do
  log "Executing iteration ${iteration}/${ITERATIONS}"

  log "Terminating compute fleet"
  terminate_compute_fleet ${CLUSTER_NAME} ${AWS_DEFAULT_REGION}

  log "Waiting for compute fleet down to 0 nodes (can take up to the configured idle scale-down time, default 10 minutes)"
  wait_compute_fleet ${CLUSTER_NAME} ${AWS_DEFAULT_REGION} 0 120

  # Job Launch
  JOB_IDS=()
  for j in $(seq 1 ${JOBS_PER_ITERATION}); do
    JOB_USER=$(get_circular_array_element $(( ${j} - 1 )) "${JOB_USERS[@]}")
    log "Submitting job ${j}/${JOBS_PER_ITERATION} for iteration ${iteration}/${ITERATIONS} as user ${JOB_USER}"
    JOB_ID=$(launch_job "${JOB_USER}" "${NUM_PROCESSES_PER_JOB}" "${JOB_WRAPPER_SCRIPT}" "${OUTPUT_DIR}" "${JOB_COMMAND}")
    JOB_SUBMISSION_TIME_MILLIS=$(timestamp_millis)
    JOB_METRICS_FILE="${OUTPUT_DIR}/job.${JOB_ID}.sample.json"
    mkdir -m 777 -p $(dirname ${JOB_METRICS_FILE})
    echo "{}" > ${JOB_METRICS_FILE}
    chmod 666 ${JOB_METRICS_FILE}

    log "Job ${JOB_ID} submitted at $(millis_to_date_time ${JOB_SUBMISSION_TIME_MILLIS}); metrics will be collected in ${JOB_METRICS_FILE}"
    add_to_json "jobId" ${JOB_ID} ${JOB_METRICS_FILE}
    add_to_json "user" ${JOB_USER} ${JOB_METRICS_FILE}
    add_to_json "jobSubmissionTimestamp" ${JOB_SUBMISSION_TIME_MILLIS} ${JOB_METRICS_FILE}
    JOB_IDS+=(${JOB_ID})
  done

  # Waiting Jobs Completion
  log "Waiting for jobs completion:" "${JOB_IDS[@]}"
  wait_job_completion $(join_array_by "," "${JOB_IDS[@]}") 120

  # Metrics
  COMPUTE_NODES_METRICS=("instancePreInstallUpTime" "instancePostInstallUpTime")
  COMPUTE_NODES_METRICS_WITH_TIMESTAMPS=(${COMPUTE_NODES_METRICS[@]} "instancePreInstallTimestamp" "instancePostInstallTimestamp")
  COMPUTE_NODES_METRICS_WITH_TIMESTAMPS_AND_INSTANCE_ID=(${COMPUTE_NODES_METRICS_WITH_TIMESTAMPS[@]} "instanceId")
  JOB_METRICS=("jobRunTime" "jobWaitingTime" "jobWarmupLeaderNodeTime" "jobWarmupFirstNodeTime" "jobWarmupLastNodeTime")

  # Compute Nodes Sample
  # instancePreInstallTimestamp, instancePreInstallUpTime, instancePostInstallTimestamp, instancePostInstallUpTime
  COMPUTE_NODES_SAMPLE_FILE="${OUTPUT_DIR}/compute-nodes.${iteration}.sample.json"
  COMPUTE_NODES_METRICS_DIR="${SHARED_DIR}/metrics/compute-nodes"
  COMPUTE_NODES_METRICS_FILES=$(find ${COMPUTE_NODES_METRICS_DIR} -type f -name "instance-*.json")
  for metric in "${COMPUTE_NODES_METRICS_WITH_TIMESTAMPS[@]}"; do
    add_to_json "${metric}Sample" $(get_sample_from_json "${metric}" "${COMPUTE_NODES_METRICS_FILES}") ${COMPUTE_NODES_SAMPLE_FILE}
  done

  for JOB_ID in "${JOB_IDS[@]}"; do
    JOB_METRICS_FILE="${OUTPUT_DIR}/job.${JOB_ID}.sample.json"
    # Compute Nodes metrics
    # firstComputeNode timings, i.e. time metrics from the first launched compute node
    FIRST_COMPUTE_NODE_METRICS=$(get_json_with_minimum "instancePreInstallTimestamp" "${COMPUTE_NODES_METRICS_FILES}")
    for metric in "${COMPUTE_NODES_METRICS_WITH_TIMESTAMPS_AND_INSTANCE_ID[@]}"; do
      add_to_json "firstComputeNode.${metric}" "$(echo ${FIRST_COMPUTE_NODE_METRICS} | jq -r ".${metric}")" ${JOB_METRICS_FILE}
    done
    # lastComputeNode timings, i.e. time metrics from the last launched compute node
    LAST_COMPUTE_NODE_METRICS=$(get_json_with_maximum "instancePreInstallTimestamp" "${COMPUTE_NODES_METRICS_FILES}")
    for metric in "${COMPUTE_NODES_METRICS_WITH_TIMESTAMPS_AND_INSTANCE_ID[@]}"; do
      add_to_json "lastComputeNode.${metric}" "$(echo ${LAST_COMPUTE_NODE_METRICS} | jq -r ".${metric}")" ${JOB_METRICS_FILE}
    done

    # Derived Metrics: jobRunTime = jobEndTimestamp - jobStartTimestamp
    JOB_RUN_TIME_MILLIS=$(cat ${JOB_METRICS_FILE} | jq '(.jobEndTimestamp|tonumber) - (.jobStartTimestamp|tonumber)')
    add_to_json "jobRunTime" ${JOB_RUN_TIME_MILLIS} ${JOB_METRICS_FILE}

    # Derived Metrics: jobWaitingTime = jobStartTimestamp - jobSubmissionTimestamp
    JOB_WAITING_TIME_MILLIS=$(cat ${JOB_METRICS_FILE} | jq '(.jobStartTimestamp|tonumber) - (.jobSubmissionTimestamp|tonumber)')
    add_to_json "jobWaitingTime" ${JOB_WAITING_TIME_MILLIS} ${JOB_METRICS_FILE}

    # Derived Metrics: jobWarmupLeaderNodeTime = jobStartTimestamp - leaderComputeNode.instancePreInstallTimestamp
    JOB_WARMUP_LEADER_NODE_TIME_MILLIS=$(cat ${JOB_METRICS_FILE} | jq '(.jobStartTimestamp|tonumber) - (.leaderComputeNode.instancePreInstallTimestamp|tonumber)')
    add_to_json "jobWarmupLeaderNodeTime" ${JOB_WARMUP_LEADER_NODE_TIME_MILLIS} ${JOB_METRICS_FILE}

    # Derived Metrics: jobWarmupFirstNodeTime = jobStartTimestamp - firstComputeNode.instancePreInstallTimestamp
    JOB_WARMUP_FIRST_NODE_TIME_MILLIS=$(cat ${JOB_METRICS_FILE} | jq '(.jobStartTimestamp|tonumber) - (.firstComputeNode.instancePreInstallTimestamp|tonumber)')
    add_to_json "jobWarmupFirstNodeTime" ${JOB_WARMUP_FIRST_NODE_TIME_MILLIS} ${JOB_METRICS_FILE}

    # Derived Metrics: jobWarmupLastNodeTime = jobStartTimestamp - lastComputeNode.instancePreInstallTimestamp
    JOB_WARMUP_LAST_NODE_TIME_MILLIS=$(cat ${JOB_METRICS_FILE} | jq '(.jobStartTimestamp|tonumber) - (.lastComputeNode.instancePreInstallTimestamp|tonumber)')
    add_to_json "jobWarmupLastNodeTime" ${JOB_WARMUP_LAST_NODE_TIME_MILLIS} ${JOB_METRICS_FILE}
  done

  # Cleanup
  rm -rf ${COMPUTE_NODES_METRICS_FILES}

  # Finalize
  log "Iteration ${iteration}/${ITERATIONS} completed: job metrics in ${JOB_METRICS_FILE}"
  log "Waiting 60 seconds before the next iteration"
  sleep 60
  log "Terminating compute fleet"
  terminate_compute_fleet ${CLUSTER_NAME} ${AWS_DEFAULT_REGION}
done

# Samples
log "Generating samples"
SAMPLES_FILE="${OUTPUT_DIR}/samples.json"
JOBS_SAMPLE_FILES=$(find ${OUTPUT_DIR} -type f -name "job.*.sample.json")
for metric in "${JOB_METRICS[@]}"; do
  add_to_json "${metric}Sample" $(get_sample_from_json "${metric}" "${JOBS_SAMPLE_FILES}") ${SAMPLES_FILE}
done
COMPUTE_NODES_SAMPLE_FILES=$(find ${OUTPUT_DIR} -type f -name "compute-nodes.*.sample.json")
for metric in "${COMPUTE_NODES_METRICS[@]}"; do
  add_to_json "${metric}Sample" $(get_sample_from_json "${metric}Sample" "${COMPUTE_NODES_SAMPLE_FILES}") ${SAMPLES_FILE}
done

# Statistics
log "Generating statistics"
STATISTICS_FILE="${OUTPUT_DIR}/statistics.json"
STATISTICS_METRICS=(${JOB_METRICS[@]} ${COMPUTE_NODES_METRICS[@]})
for metric in "${STATISTICS_METRICS[@]}"; do
  add_to_json "${metric}.min" $(get_min "$(cat ${SAMPLES_FILE} | jq -r ".${metric}Sample")") ${STATISTICS_FILE}
  add_to_json "${metric}.max" $(get_max "$(cat ${SAMPLES_FILE} | jq -r ".${metric}Sample")") ${STATISTICS_FILE}
  add_to_json "${metric}.avg" $(get_avg "$(cat ${SAMPLES_FILE} | jq -r ".${metric}Sample")") ${STATISTICS_FILE}
  add_to_json "${metric}.std" $(get_std "$(cat ${SAMPLES_FILE} | jq -r ".${metric}Sample")") ${STATISTICS_FILE}
  add_to_json "${metric}.med" $(get_med "$(cat ${SAMPLES_FILE} | jq -r ".${metric}Sample")") ${STATISTICS_FILE}
  add_to_json "${metric}.prc25" $(get_prc "$(cat ${SAMPLES_FILE} | jq -r ".${metric}Sample")" 25) ${STATISTICS_FILE}
  add_to_json "${metric}.prc75" $(get_prc "$(cat ${SAMPLES_FILE} | jq -r ".${metric}Sample")" 75) ${STATISTICS_FILE}
done

# End
log "Scale test completed: samples in ${SAMPLES_FILE}, statistics in ${STATISTICS_FILE}"

# Upload samples and statistics on S3
if [[ -z ${OUTPUT_S3} ]]; then
  log "Skipping upload on S3 as no OUTPUT_S3 has been specified"
else
  aws s3 cp --recursive ${OUTPUT_DIR} ${OUTPUT_S3}
  log "Results published to ${OUTPUT_S3}"
fi


#!/bin/bash
set -ex

# To submit this job, run:
# sbatch -n 16 /shared/assets/workloads/scale-test/scale-test-job-wrapper.sh [OUTPUT_DIR] [JOB_COMMAND]
# sbatch -n 16 /shared/assets/workloads/scale-test/scale-test-job-wrapper.sh /shared/scale-tests/scale-test-$(date +"%Y-%m-%dT%H-%M-%S") "/shared/assets/workloads/env-info.sh ; sleep 60"
# sbatch -n 16 /shared/assets/workloads/scale-test/scale-test-job-wrapper.sh /shared/scale-tests/scale-test-$(date +"%Y-%m-%dT%H-%M-%S") "sleep 120"
#
# Note: 16 processes = 4 compute nodes c5.xlarge having 4 vCPUs each

# Cluster variables
source /etc/parallelcluster/cfnconfig
SHARED_DIR="$(echo $cfn_ebs_shared_dirs | cut -d ',' -f 1)"

# Load libraries
FUNCTIONS_SCRIPT="${SHARED_DIR}/assets/lib/functions.sh"
source "${FUNCTIONS_SCRIPT}"

# Input
OUTPUT_DIR="${1}"
JOB_COMMAND="${2:-"sleep 60"}"

# Scale Test - Directories and Files
JOB_METRICS_FILE="${OUTPUT_DIR}/job.${SLURM_JOB_ID}.sample.json"
mkdir -m 777 -p $(dirname ${JOB_METRICS_FILE})

# Scale Test - Instance Info
INSTANCE_ID=$(get_instance_id)
INSTANCE_METRICS_FILE="${SHARED_DIR}/metrics/compute-nodes/instance-${INSTANCE_ID}.json"
for metric in "instancePreInstallTimestamp" "instancePreInstallUpTime" "instancePostInstallTimestamp" "instancePostInstallUpTime" "instanceId"; do
  add_to_json "leaderComputeNode.${metric}" "$(cat ${INSTANCE_METRICS_FILE} | jq -r ".${metric}")" ${JOB_METRICS_FILE}
done

# Scale Test - Job Info
add_to_json "processes" ${SLURM_NPROCS} ${JOB_METRICS_FILE}

# Scale Test - Job Start Time
JOB_START_TIME_MILLIS=$(timestamp_millis)
add_to_json "jobStartTimestamp" ${JOB_START_TIME_MILLIS} ${JOB_METRICS_FILE}

# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# Job Command
bash -c "${JOB_COMMAND}"
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #

# Scale Test - Job End Time
JOB_END_TIME_MILLIS=$(timestamp_millis)
add_to_json "jobEndTimestamp" ${JOB_END_TIME_MILLIS} ${JOB_METRICS_FILE}


#!/bin/bash
set -x

. /etc/os-release
release="${ID}${VERSION_ID:+.${VERSION_ID}}"

if [ `echo ${release} | grep '^centos\.7'` ]; then
  sudo yum remove -y epel-release

  sudo yum repolist | grep epel

  if [ $? -eq 0 ]; then
      echo "epel installed"
      exit 1
  fi
elif [ `echo "${release}" | grep '^centos\.8'` ]; then
  sudo dnf remove -y epel-release

  sudo dnf repolist | grep epel

  if [ $? -eq 0 ]; then
      echo "epel not removed"
      exit 1
  fi
fi


#!/bin/bash
set -x

. /etc/os-release
release="${ID}${VERSION_ID:+.${VERSION_ID}}"

if [ `echo ${release} | grep '^centos\.7'` ]; then
  sudo yum repolist | grep epel

  if [ $? -ne 0 ]; then
      echo "epel not installed"
      exit 1
  fi
elif [ `echo "${release}" | grep '^centos\.8'` ]; then
  sudo dnf repolist | grep epel

  if [ $? -ne 0 ]; then
      echo "epel not installed"
      exit 1
  fi
fi


#!/bin/bash
set -e

_openmpi_module=$1
_remote_host=$2

module load ${_openmpi_module}
$(which mpirun) --host ${_remote_host} hostname


#!/bin/bash
#
# Copyright 2018      Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy
# of the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "LICENSE.txt" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, express or implied. See the License for the specific
# language governing permissions and limitations under the License.
#
#
# Script executed on clusters from parallelcluster-release-check.py.  This
# script attempts to submit two jobs which must complete in 10
# minutes, one of which (hopefully) requires scaling.
#

# Usage:
#   cluster-check.sh {submit | scaledown_check} <scheduler>
#
set -e

slurm_get_slots() {
    local -- ppn i=0
    ppn=$(scontrol show nodes -o | head -n 1 | sed -n -e 's/^.* CPUTot=\([0-9]\+\) .*$/\1/p')
    # wait 15 secs before giving up retrieving the slots per host
    while [ -z "${ppn}" -a $i -lt 15 ]; do
        sleep 1
        i=$((i+1))
        ppn=$(scontrol show nodes -o | head -n 1 | sed -n -e 's/^.* CPUTot=\([0-9]\+\) .*$/\1/p')
    done

    echo ${ppn}
}

submit_launch() {
    # in case the scheduler goes nuts, wrap ourselves in a timeout so
    # there's a bounded completion time
    if test "$CHECK_CLUSTER_SUBPROCESS" = ""; then
        export CHECK_CLUSTER_SUBPROCESS=1
        timeout -s KILL 9m /bin/bash ./cluster-check.sh "$@"
        exit $?
    fi

    scheduler="$2"

    echo "--> scheduler: $scheduler"

    submit_init ${scheduler}

    echo "$(date +%s)" > jobs_start_time

    ${scheduler}_submit
    echo "Jobs submitted successfully"
}

submit_init() {
    # we submit 3 1-node jobs, each of which are a sleep.
    # The whole thing has to run in 9 minutes (higher of the three sleep times + buffer),
    # or the kill in submit_launch will fail the job, which means that the jobs must run at the same time.
    # The initial cluster is 1 nodes, so we'll need to scale up 2 further nodes simultaneously
    # and bootstrap in less than 7 minutes and run the job that needs 105 seconds
    # in order for the test to succeed.

    # job1: 7m45s
    export _sleepjob1=465
    # job2: 1m45s
    export _sleepjob2=105
    # job3: 1m45s
    export _sleepjob3=105

    scheduler=$1
    export _ppn=$(${scheduler}_get_slots)
    if [ -z "${_ppn}" ]; then
        >&2 echo "The number of slots per instance couldn't be retrieved, no compute nodes available in ${scheduler} cluster"
        exit 1
    fi
}

slurm_submit() {
    cat > job1.sh <<EOF
#!/bin/bash
srun sleep ${_sleepjob1}
echo "\$(date +%s)" > job1.done
EOF
    cat > job2.sh <<EOF
#!/bin/bash
srun sleep ${_sleepjob2}
echo "\$(date +%s)" > job2.done
EOF

    cat > job3.sh <<EOF
#!/bin/bash
srun sleep ${_sleepjob3}
echo "\$(date +%s)" > job3.done
EOF

    chmod +x job1.sh job2.sh job3.sh
    rm -f job1.done job2.done job3.done

    sbatch -N 1 -n ${_ppn} ./job1.sh
    sbatch -N 1 -n ${_ppn} ./job2.sh
    sbatch -N 1 -n ${_ppn} ./job3.sh
}

scaledown_check_launch() {
    # bounded completion time
    if test "$CHECK_CLUSTER_SUBPROCESS" = ""; then
        export CHECK_CLUSTER_SUBPROCESS=1
        timeout -s KILL 5m /bin/bash ./cluster-check.sh "$@"
        exit $?
    fi

    scheduler="$2"

    echo "--> scheduler: $scheduler"

    scaledown_complete=0
    while test $scaledown_complete = 0 ; do
        ${scheduler}_scaledown_check
        sleep 10
    done
    echo "Scaledown successful"
}

has_zero_active_instances(){
    instances=$1
    if [[ ${instances} ]]; then
        echo "instances have not scaled down yet"
        scaledown_complete=0
    else
        echo "instances have scaled down; exiting"
        scaledown_complete=1
    fi
}

slurm_scaledown_check() {
    has_zero_active_instances $(sinfo --noheader | awk '$4 != "0"')
}

main() {
    case "$1" in
        submit)
            submit_launch "$@"
            ;;
        scaledown_check)
            scaledown_check_launch "$@"
            ;;
        *)
            echo "!! Unknown command $1 !!"
            exit 1
            ;;
    esac
}

main "$@"


#!/bin/bash
set -e

echo "ip container: $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1)"
TOKEN=$(curl --retry 3 --retry-delay 0  --fail -s -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 300")
echo "ip host: $(curl --retry 3 --retry-delay 0  --fail -s -H "X-aws-ec2-metadata-token: ${TOKEN}" "http://169.254.169.254/latest/meta-data/local-ipv4")"

# get shared dir
IFS=',' _shared_dirs=(${PCLUSTER_SHARED_DIRS})
_shared_dir=${_shared_dirs[0]}
_job_dir="${_shared_dir}/${AWS_BATCH_JOB_ID%#*}-${AWS_BATCH_JOB_ATTEMPT}"
_exit_code_file="${_job_dir}/batch-exit-code"

if [[ "${AWS_BATCH_JOB_NODE_INDEX}" -eq  "${AWS_BATCH_JOB_MAIN_NODE_INDEX}" ]]; then
    echo "Hello I'm the main node $(hostname)! I run the mpi job!"

    mkdir -p "${_job_dir}"

    echo "Writing mpi code..."
    cat > "${_shared_dir}/mpi_hello_world.c" << EOF
// Copyright 2011 www.mpitutorial.com
//
// An intro MPI hello world program that uses MPI_Init, MPI_Comm_size,
// MPI_Comm_rank, MPI_Finalize, and MPI_Get_processor_name.
//
#include <mpi.h>
#include <stdio.h>
#include <stddef.h>

int main(int argc, char** argv) {
  // Initialize the MPI environment. The two arguments to MPI Init are not
  // currently used by MPI implementations, but are there in case future
  // implementations might need the arguments.
  MPI_Init(NULL, NULL);

  // Get the number of processes
  int world_size;
  MPI_Comm_size(MPI_COMM_WORLD, &world_size);

  // Get the rank of the process
  int world_rank;
  MPI_Comm_rank(MPI_COMM_WORLD, &world_rank);

  // Get the name of the processor
  char processor_name[MPI_MAX_PROCESSOR_NAME];
  int name_len;
  MPI_Get_processor_name(processor_name, &name_len);

  // Print off a hello world message
  printf("Hello world from processor %s, rank %d out of %d processors\n",
         processor_name, world_rank, world_size);

  // Finalize the MPI environment. No more MPI calls can be made after this
  MPI_Finalize();
}
EOF

    echo "Compiling..."
    /usr/lib64/openmpi/bin/mpicc -o "${_job_dir}/mpi_hello_world" "${_shared_dir}/mpi_hello_world.c"

    echo "Running..."
    /usr/lib64/openmpi/bin/mpirun --mca btl_tcp_if_include eth0 --allow-run-as-root --machinefile "${HOME}/hostfile" "${_job_dir}/mpi_hello_world"

    # Write exit status code
    echo "0" > "${_exit_code_file}"
    # Waiting for compute nodes to terminate
    sleep 30
else
    echo "Hello I'm a compute node $(hostname)! I let the main node orchestrate the mpi execution!"
    # Since mpi orchestration happens on the main node, we need to make sure the containers representing the compute
    # nodes are not terminated. A simple trick is to wait for a file containing the status code to be created.
    # All compute nodes are terminated by Batch if the main node exits abruptly.
    while [ ! -f "${_exit_code_file}" ]; do
        sleep 2
    done
    exit $(cat "${_exit_code_file}")
fi


#!/bin/bash
set -e

echo "Executing Test Script"
env | grep TEST=test


#!/bin/bash
set -e
echo "Executing Test Script"
exit 0


#!/bin/bash
# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License").
# You may not use this file except in compliance with the License.
# A copy of the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "LICENSE.txt" file accompanying this file.
# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied.
# See the License for the specific language governing permissions and limitations under the License.

PIDS=($(pgrep supervisord$))
for PID in "${PIDS[@]}"
do
    CMD_LINE=($(ps -p ${PID} -o args --no-headers))
    # CMD_LINE is in the form
    # /opt/parallelcluster/shared/path/pyenv/versions/3.9.9/envs/some_virtualenv/bin/python3.9 /opt/parallelcluster/shared/some-path/pyenv/versions/3.9.9/envs/some_virtualenv/bin/supervisord -n -c /opt/parallelcluster/some-path/supervisord.conf
    if ${CMD_LINE[0]} $(dirname ${CMD_LINE[1]})/supervisorctl -c ${CMD_LINE[4]} status clustermgtd &>/dev/null; then
        ${CMD_LINE[0]} $(dirname ${CMD_LINE[1]})/supervisorctl -c ${CMD_LINE[4]} stop clustermgtd
    fi
done


#!/bin/bash
# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License").
# You may not use this file except in compliance with the License.
# A copy of the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "LICENSE.txt" file accompanying this file.
# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied.
# See the License for the specific language governing permissions and limitations under the License.
slurmctld_proc_id=$(ps aux | grep "slurm" | grep "sbin" | awk '{print $2}') && kill -9 ${slurmctld_proc_id}


#!/bin/bash
# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License").
# You may not use this file except in compliance with the License.
# A copy of the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "LICENSE.txt" file accompanying this file.
# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied.
# See the License for the specific language governing permissions and limitations under the License.

PIDS=($(pgrep supervisord$))
for PID in "${PIDS[@]}"
do
    CMD_LINE=($(ps -p ${PID} -o args --no-headers))
    # CMD_LINE is in the form
    # /opt/parallelcluster/shared/path/pyenv/versions/3.9.9/envs/some_virtualenv/bin/python3.9 /opt/parallelcluster/shared/path/pyenv/versions/3.9.9/envs/some_virtualenv/bin/supervisord -n -c /opt/parallelcluster/some_path/supervisord.conf
    if ${CMD_LINE[0]} $(dirname ${CMD_LINE[1]})/supervisorctl -c ${CMD_LINE[4]} status clustermgtd &>/dev/null; then
        ${CMD_LINE[0]} $(dirname ${CMD_LINE[1]})/supervisorctl -c ${CMD_LINE[4]} start clustermgtd
    fi
done


#!/bin/bash
# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License").
# You may not use this file except in compliance with the License.
# A copy of the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "LICENSE.txt" file accompanying this file.
# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied.
# See the License for the specific language governing permissions and limitations under the License.
systemctl start slurmctld


#!/bin/bash
# Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License").
# You may not use this file except in compliance with the License.
# A copy of the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "LICENSE.txt" file accompanying this file.
# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied.
# See the License for the specific language governing permissions and limitations under the License.
slurm_plugin_path=$(sudo find / -iname slurm_plugin -print0|grep -FzZ 'node_virtualenv')
cat > $slurm_plugin_path/overrides.py << EOF
from botocore.exceptions import ClientError
import boto3
import logging

logger = logging.getLogger(__name__)

def run_instances(region, boto3_config, **run_instances_kwargs):
    if "ice-compute-resource" in run_instances_kwargs.get("LaunchTemplate", {}).get("LaunchTemplateName"):
        # Forcing the Placement/tenancy override to host will trigger an InsufficientHostCapacity since we do not
        # have a fleet of reserved hosts
        run_instances_kwargs['Placement']= {'Tenancy': 'host'}
        logger.info("Updated parameters for EC2 run_instances: %s", run_instances_kwargs)
    ec2_client = boto3.client("ec2", region_name=region, config=boto3_config)
    return ec2_client.run_instances(**run_instances_kwargs)

def update_lt_instance_overrides(overrides):
    updated_overrides = []
    for ov in overrides:
        if 'InstanceType' in ov:
            # mutate t2.large into t2-large to force an Error in CreateFleet request
            mispelled_instance_type = ov['InstanceType'].replace(".", "-")
            ov['InstanceType'] = mispelled_instance_type
            updated_overrides.append(ov)
        else:
            updated_overrides.append(ov)

    return updated_overrides

def create_fleet(region, boto3_config, **create_fleet_kwargs):
    configs = create_fleet_kwargs.get("LaunchTemplateConfigs", [])
    if len(configs) >= 1 and configs[0]:
        lt_config = configs[0]

        if "ice-cr-multiple" in lt_config.get('LaunchTemplateSpecification').get("LaunchTemplateName"):
            # CreateFleet will return an Error and an empty list of instances
            lt_config['Overrides'] = update_lt_instance_overrides(lt_config['Overrides'])
            logger.info("Updated Instance Overrides for CreateFleet args: %s", create_fleet_kwargs)
        elif "exception-cr-multiple" in lt_config.get('LaunchTemplateSpecification').get("LaunchTemplateName"):
            # force CreateFleet to raise an exception since `inf*` instance types have Inferentia accelerators
            # that are manufactured by AWS, and we are also requesting Manufacturer=nvidia
            lt_config['Overrides'] = [{
                        "InstanceRequirements": {
                            "VCpuCount": { "Min": 2 },
                            "MemoryMiB": { "Min": 2048 },
                            "AllowedInstanceTypes": [  "inf*" ],
                            "AcceleratorManufacturers": [ "nvidia" ]
                        }
                    }]
            logger.info("Updated Instance Overrides for CreateFleet args: %s", create_fleet_kwargs)

        ec2_client = boto3.client("ec2", region_name=region, config=boto3_config)
        return ec2_client.create_fleet(**create_fleet_kwargs)
    else:
        raise Exception("Missing LaunchTemplateSpecification parameter in create_fleet request.")
EOF


#!/bin/bash

# This script creates a dummy systemd service that sleeps for a given time, and
# makes it a dependency for the slurmd service. This way it's possible to simulate
# a longer rebooting time of the node via scontrol reboot.

# TODO: consider disabling the network during the sleep in order to simulate the
# non-responsiveness to the EC2 status checks of the rebooted node.
#
# It doesn't seem to be possible to add a `ifconfig eth0 down` and `ifconfig eth0 up`
# in the following systemd service: the first command completely kills the instance
# and the second command doesn't seem to reactivate the network interface.

sudo mkdir /etc/systemd/system/slurmd.service.d

cat <<DELAY_SERVICE | sudo tee /etc/systemd/system/slurmd_delay.service
[Unit]
Description=Dummy Slurmd delay
Before=slurmd.service
Wants=network-online.target

[Service]
Type=simple
TimeoutSec=180s
ExecStartPre=sleep 120
ExecStart=sleep 1

[Install]
WantedBy=multi-user.target
DELAY_SERVICE

cat <<SLURMD_DROP_IN | sudo tee /etc/systemd/system/slurmd.service.d/add_delay.conf
[Unit]
After=slurmd_delay.service
Requires=slurmd_delay.service
SLURMD_DROP_IN

sudo systemctl daemon-reload


#!/bin/bash

# Reset state of the daemons after running add_sleep_to_slurmd_service_compute.sh

sudo rm /etc/systemd/system/slurmd_delay.service
sudo rm /etc/systemd/system/slurmd.service.d/add_delay.conf
sudo systemctl daemon-reload


#!/bin/bash

#SBATCH --export=ALL
#SBATCH --nodes=2
#SBATCH --ntasks-per-node=3
#SBATCH --job-name=imb2
#SBATCH --output=runscript.out

module load intelmpi
mpirun -n 6 IMB-MPI1 Alltoall -npmin 2


#!/bin/sh
echo "qsub executed successfully"


#!/bin/bash
set -ex
. "/etc/parallelcluster/cfnconfig"

if [[ $cfn_node_type == "HeadNode" ]]; then
    # Override create_fleet attributes
    cat > /opt/slurm/etc/pcluster/create_fleet_overrides.json << 'EOF'
{
    "fleet": {
        "fleet-1": {
            "TagSpecifications": [
                 {
                    "ResourceType": "instance",
                    "Tags": [
                        {
                            "Key": "overridefleet",
                            "Value": "overridefleet"
                        }
                    ]
                }
            ]
        }
    }
}
EOF

    # Override run_instances attributes
    cat > /opt/slurm/etc/pcluster/run_instances_overrides.json << 'EOF'
{
    "single": {
        "single-1": {
            "TagSpecifications": [
                 {
                    "ResourceType": "instance",
                    "Tags": [
                        {
                            "Key": "overridesingle",
                            "Value": "overridesingle"
                        }
                    ]
                }
            ]
        }
    }
}
EOF
fi


#!/bin/bash
# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License").
# You may not use this file except in compliance with the License.
# A copy of the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "LICENSE.txt" file accompanying this file.
# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied.
# See the License for the specific language governing permissions and limitations under the License.
function get_instance_type() {
  token=$(curl -s -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 300")
  instance_type_url="http://169.254.169.254/latest/meta-data/instance-type"
  instance_type=$(curl --retry 3 --retry-delay 0 --silent --fail -H "X-aws-ec2-metadata-token: ${token}" "${instance_type_url}")
}
get_instance_type
if [ "${instance_type}" == "c5.large" ]; then
  echo "Test Bootstrap error that causes instance to self terminate."
  exit 1
fi


#!/bin/bash
# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License").
# You may not use this file except in compliance with the License.
# A copy of the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "LICENSE.txt" file accompanying this file.
# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied.
# See the License for the specific language governing permissions and limitations under the License.
ssh $1 sudo pkill -f slurmd


#!/bin/bash
get_setting_value() {
  local _file=$1
  local _search=$2
  local _pattern

  _pattern=$(grep "${_search}" "${_file}")
  echo "${_pattern}" | tr -d '\n' | cut -d = -f 2 | xargs
}

# Get the root user (root is added to Slurm accounting by default during configuration)
getent passwd 0 | cut -d : -f 1
# Get the configured cluster user name from the dna.json (added to the database during node configuration)
jq -r '.cluster.cluster_user' < "/etc/chef/dna.json"
# Get the configured Slurm user from the slurm configuration (added to the database during node configuration)
get_setting_value "/opt/slurm/etc/slurm.conf" "^SlurmUser="


#!/bin/bash

set -e

CA_URL=$1
CA_NAME=$2
CA_PATH="/tmp/slurm"
CA_FILE_PATH="${CA_PATH}/${CA_NAME}"
SLURMDBD_CONFIG_FILE="/opt/slurm/etc/slurmdbd.conf"

wget ${CA_URL} -P ${CA_PATH}

echo "Clearing previous CA Setting"
sed -i "/^StorageParameters\s*=/d" "${SLURMDBD_CONFIG_FILE}"

echo "Setting 'StorageParameters' SSL_CA=${CA_FILE_PATH}"
echo "StorageParameters=SSL_CA=${CA_FILE_PATH}" >> "${SLURMDBD_CONFIG_FILE}"

echo "Restarting slurmdbd service"
systemctl restart slurmdbd


#!/bin/bash
# Test the basic functionality of the Neuron component installed in the ParallelCluster AMI

# Print available Neuron packages
OS="$(grep "^ID=" /etc/os-release | cut -d"=" -f 2 | xargs)"
case ${OS} in
  ubuntu)
    apt list --installed | grep neuron
    USER=ubuntu
    ;;
  amzn)
    rpm -qa | grep neuron
    USER=ec2-user
    ;;
  *)
    echo "Unsupported system. Found /etc/os-release ID content: ${OS}"
    exit 1
    ;;
esac

# Create python scripts required for the tests
cat <<EOF >allreduce_launch.py
import os
import torch_xla.core.xla_model as xm
import torch
import torch.distributed as dist
from torch_xla.distributed import xla_backend

#os.environ["NEURON_RT_LOG_LEVEL"] = "INFO"
#os.environ["NEURON_RT_LOG_LOCATION"] = "syslog"
#os.environ["NCCL_DEBUG"] = "ERROR"
#os.environ["NCCL_DEBUG_SUBSYS"] = "ALL"

def _mp_fn():
    dist.init_process_group('xla')
    world_size = xm.xrt_world_size()
    device = xm.xla_device()

    if world_size > 0:
        ones = torch.ones((2, 3))
        xones = ones.to(device)
        print("running all reduce")
        result = xm.all_reduce(xm.REDUCE_SUM, xones)
        result_cpu = result.cpu()
        expected = torch.ones((2,3))*world_size
        assert expected.allclose(result_cpu)
        print(result_cpu)
        print('PASS')

if __name__ == '__main__':
    print(
        'master_port:{}, master_addr:{}, rank:{}, local_rank:{}, size:{}'.format(
            os.environ['MASTER_PORT'],
            os.environ['MASTER_ADDR'],
            os.environ['RANK'],
            os.environ['LOCAL_RANK'],
            os.environ['WORLD_SIZE'],
        )
    )

    _mp_fn()
EOF

cat <<'EOF' >test_launch.py
import os
import subprocess

def test_local_launch_allreduce():
    # This test launches a two phase launcher, spawns two processes which
    # in turn launch more processes. This uses pytorch distributed.launch utility along with ptxrt bridge
    cmd = "torchrun --nproc_per_node=$PROC_PER_NODE --nnodes=1 allreduce_launch.py "

    new_env0 = os.environ.copy()
    new_env0['PROC_PER_NODE'] = '2'

    proc0 = subprocess.Popen(cmd, env=new_env0, shell=True)

    return_code = proc0.wait()
    assert return_code == 0
EOF

# Activate virtual environment created by neuron-installation.sh and run the test
# We expect to find PASSED word at the end of the output
source /home/$USER/aws_neuron_venv_pytorch/bin/activate
pytest test_launch.py -vxs | tee output-allreduce.txt


#!/bin/bash
# Test the collective functionality of the Neuron component installed in the ParallelCluster AMI
# This script can work if submitted through Slurm with the following command:
# sbatch --nodes=2 --ntasks-per-node=1 --cpus-per-task=32 neuron-ccl.sh
# or if submitted directly on a single node (SLURM_JOB_NODELIST not set)

cat <<'EOF' >submission-script.sh
#!/bin/bash
set -x

# FIXME remove this repo once packages are public available
TEMPORARY_ARTIFACTS_BUCKET_PATH=s3://aws-parallelcluster-beta/neuron/

# Print available Neuron packages
OS="$(grep "^ID=" /etc/os-release | cut -d"=" -f 2 | xargs)"
case ${OS} in
  ubuntu)
    apt list --installed | grep neuron
    ;;
  amzn)
    rpm -qa | grep neuron
    ;;
  *)
    echo "Unsupported system. Found /etc/os-release ID content: ${OS}"
    exit 1
    ;;
esac

# Identify node name to set the right START_RANK
if [ "${SLURM_NNODES:=1}" == "1" ]; then
  # SLURM_NNODES is not set (no slurm job) or script is executed in a single node
  START_RANK=0
  # retrieve first IP of the list
  HOST_IP=$(hostname -I | cut -d ' ' -f1)
else
  # SLURM_JOB_NODELIST is an array, find first and second node of the array and use number to identify first node.
  NODE_NAME_PREFIX=${SLURM_JOB_NODELIST%-[*}
  NODE1=$(echo $SLURM_JOB_NODELIST | cut -d'[' -f2 | cut -d'-' -f1)
  NODE2=$(echo $SLURM_JOB_NODELIST | cut -d'[' -f2 | cut -d'-' -f2 | tr -d ']')
  HOST_IP=$(scontrol show nodes $NODE_NAME_PREFIX-$NODE1 | grep -oe "NodeAddr=[^ ]*" | cut -d'=' -f2)

  if [ "$(hostname)" == "$NODE_NAME_PREFIX-$NODE1" ]; then
      START_RANK=0
  else
      START_RANK=32
  fi
fi
TOTAL_RANK=$(($SLURM_NNODES*32))

# Download file for simulation (64 in the file name corresponds to the number of ranks hardcoded into the file).
# To use a different number of ranks you need to generate another one:
# python3 inst-sweep/genneffs_nccl.py -n <total-number-of-ranks> --all --output <output-dir>
NEFF_FILE=test_nccl_64r_50allg_int8_393216/0/file.neff
if [[ ! -f $NEFF_FILE ]]; then
  aws s3 cp ${TEMPORARY_ARTIFACTS_BUCKET_PATH}test_nccl_64r_50allg_int8_393216_0_file.neff $NEFF_FILE
fi

# Print eth0 ip
/usr/sbin/ip -br addr show dev eth0 scope global

# Export variables required for neuron-bench
export PATH="/opt/aws/neuron/bin:$PATH"
# Export variables required for EFA
export FI_EFA_USE_DEVICE_RDMA=1
export FI_PROVIDER=efa

NCCL_DEBUG=trace NCCL_DEBUG_SUBSYS=all NCCL_DEBUG_FILE=$(pwd)/nccl_${SLURM_TASK_PID}.log \
NEURON_RT_EXEC_TIMEOUT=10 \
NEURON_RT_ROOT_COMM_ID=$HOST_IP:33666 \
LD_LIBRARY_PATH=/opt/aws/neuron/lib:/opt/amazon/efa/lib64 \
neuron-bench infer --fixed-instance-count 32 --enable-only-latency --work 1 \
  --run-as-cc-neff --minimal-tensor-io --verbose 4 --cc-skip-latencies 5\
  --cc-world-size $TOTAL_RANK --cc-rank-start $START_RANK \
  --summary-percentiles=1,50,99,100 \
  --warmup none $NEFF_FILE | tee output-ccl.txt

EOF

# srun is required to run neuron-bench process on the two nodes
srun bash submission-script.sh


#!/bin/bash

# Private Repository Access
# I manually created a TrainiumPreviewRepository secret and TrainiumPreviewPolicy on 447714826191 account to permit access to Secret below
# {
#    "Version": "2012-10-17",
#    "Statement": [
#        {
#            "Effect": "Allow",
#            "Action": [
#                "secretsmanager:GetResourcePolicy",
#                "secretsmanager:GetSecretValue",
#                "secretsmanager:DescribeSecret",
#                "secretsmanager:ListSecretVersionIds"
#            ],
#            "Resource": [
#                "arn:aws:secretsmanager:us-east-1:447714826191:secret:TrainiumPreviewRepository-<RANDOM-STRING>"
#            ]
#        },
#        {
#            "Effect": "Allow",
#            "Action": "secretsmanager:ListSecrets",
#            "Resource": "*"
#        }
#    ]
#}
REPO_USER=$(aws secretsmanager get-secret-value --secret-id arn:aws:secretsmanager:us-east-1:447714826191:secret:TrainiumPreviewRepository --region us-east-1 --query 'SecretString' --output text | jq -r '.repository_user')
REPO_SECRET=$(aws secretsmanager get-secret-value --secret-id arn:aws:secretsmanager:us-east-1:447714826191:secret:TrainiumPreviewRepository --region us-east-1 --query 'SecretString' --output text | jq -r '.repository_password')
REPO_SUFFIX=$(aws secretsmanager get-secret-value --secret-id arn:aws:secretsmanager:us-east-1:447714826191:secret:TrainiumPreviewRepository --region us-east-1 --query 'SecretString' --output text | jq -r '.repository_suffix')

TEMPORARY_ARTIFACTS_BUCKET_PATH=s3://aws-parallelcluster-beta/neuron/

_ubuntu_installation() {
  # Configure Linux for Neuron repository updates
  sudo tee /etc/apt/sources.list.d/neuron-private.list > /dev/null <<EOF
deb https://${REPO_USER}:${REPO_SECRET}@apt.${REPO_SUFFIX} focal main
EOF

  sudo apt-get update -y
  sudo apt-get install -y aws-neuronx-runtime-lib=2.* aws-neuronx-collectives=2.*

  wget -qO - https://${REPO_USER}:${REPO_SECRET}@apt.${REPO_SUFFIX}/GPG-PUB-KEY-AMAZON-AWS-NEURON.PUB | sudo apt-key add -

  # Install packages from S3 --> FIXME they should be installed from configured repository
  DEBS=$(aws secretsmanager get-secret-value --secret-id arn:aws:secretsmanager:us-east-1:447714826191:secret:TrainiumPreviewRepository --region us-east-1 --query 'SecretString' --output text | jq -r '.debs')
  for DEB in $DEBS
  do
    sudo dpkg -i $DEB
  done

  # Install Python venv and activate Python virtual environment to install Neuron pip packages.
  local OS_VERSION="$(grep "^VERSION_ID=" /etc/os-release | cut -d"=" -f 2 | xargs)"
  case ${OS_VERSION} in
      20.04)
        sudo apt-get install -y python3.8-venv g++
        python3 -m venv /home/ubuntu/aws_neuron_venv_pytorch
        ;;
      18.04)
        sudo apt-get install -y python3.7-venv g++
        python3.7 -m venv /home/ubuntu/aws_neuron_venv_pytorch
        ;;
      *)
        echo "Unrecognized VERSION_ID. Found /etc/os-release version content: ${OS_VERSION}"
        exit 1
        ;;
    esac
}

_rhel_installation() {
  sudo tee /etc/yum.repos.d/neuron-private.repo > /dev/null <<EOF
[neuron-private]
name=Neuron YUM Repository
baseurl=https://${REPO_USER}:${REPO_SECRET}@yum.${REPO_SUFFIX}
enabled=1
EOF
  sudo yum install -y aws-neuronx-runtime-lib-2.* aws-neuronx-collectives-2.*

  sudo rpm --import https://${REPO_USER}:${REPO_SECRET}@yum.${REPO_SUFFIX}/GPG-PUB-KEY-AMAZON-AWS-NEURON.PUB

  # Install packages from S3 --> FIXME they should be installed from configured repository
  RPMS=$(aws secretsmanager get-secret-value --secret-id arn:aws:secretsmanager:us-east-1:447714826191:secret:TrainiumPreviewRepository --region us-east-1 --query 'SecretString' --output text | jq -r '.rpms')
  for RPM in $RPMS
  do
    sudo rpm -i $RPM
  done

  python3 -m venv /home/ec2-user/aws_neuron_venv_pytorch
}

_dkms_ubuntu_installation() {
  sudo tee /etc/apt/sources.list.d/neuron.list > /dev/null <<EOF
deb https://apt.repos.neuron.amazonaws.com focal main
EOF
  wget -qO - https://apt.repos.neuron.amazonaws.com/GPG-PUB-KEY-AMAZON-AWS-NEURON.PUB | sudo apt-key add -

  sudo apt update
  sudo apt-get install aws-neuronx-dkms -y
}

_dkms_rhel_installation() {
  sudo tee /etc/yum.repos.d/neuron.repo > /dev/null <<EOF
[neuron]
name=Neuron YUM Repository
baseurl=https://yum.repos.neuron.amazonaws.com
enabled=1
EOF
  sudo rpm --import https://yum.repos.neuron.amazonaws.com/GPG-PUB-KEY-AMAZON-AWS-NEURON.PUB

  sudo yum install aws-neuronx-dkms -y
}

function main() {
  # Download packages from S3 --> FIXME they should be installed from configured repository
  aws s3 cp ${TEMPORARY_ARTIFACTS_BUCKET_PATH} . --recursive

  local OS="$(grep "^ID=" /etc/os-release | cut -d"=" -f 2 | xargs)"
  case ${OS} in
    ubuntu)
      _dkms_ubuntu_installation
      _ubuntu_installation
      USER=ubuntu
      ;;
    amzn)
      _dkms_rhel_installation
      _rhel_installation
      USER=ec2-user
      ;;
    *)
      echo "Unsupported system. Found /etc/os-release ID content: ${OS}"
      exit 1
      ;;
  esac

  # Install Python venv and activate Python virtual environment to install Neuron pip packages.
  source /home/$USER/aws_neuron_venv_pytorch/bin/activate
  pip3 install -U pip
  pip3 install pytest

  python3 -m pip config set global.extra-index-url "https://pip.repos.neuron.amazonaws.com"
  PIPS='torch-neuronx==1.11.0.1.* neuronx-cc==2.* transformers'
  pip3 install ${PIPS}
}

main "${@}"


#!/bin/bash
# Test the alignment of the Route 53 IP and the the host IP.

OUTPUT_FILE="output-primary-ip.txt"

# Get DNS Server
DNS_SERVER=""
grep Ubuntu /etc/issue &>/dev/null && DNS_SERVER=$(systemd-resolve --status | grep "DNS Servers" | awk '{print $3}' | sort -r | head -1)

# Determine expected entry in /etc/hosts
IP="$(host $HOSTNAME $DNS_SERVER | tail -1 | awk '{print $4}')"
DOMAIN=$(jq .cluster.dns_domain /etc/chef/dna.json | tr -d \")
EXPECTED="$IP $HOSTNAME.${DOMAIN::-1} $HOSTNAME"
echo "Expected entry in /etc/hosts: $EXPECTED" | tee $OUTPUT_FILE

# Retrieve actual entry in /etc/hosts
ACTUAL="$(grep "$HOSTNAME" /etc/hosts)"
echo "Actual entry in /etc/hosts: $ACTUAL" | tee -a $OUTPUT_FILE

# Check
if [[ "$ACTUAL" == "$EXPECTED" ]]; then
  echo "PASSED" | tee -a $OUTPUT_FILE
else
  echo "ERROR: Route53 IP does not match host IP" | tee -a $OUTPUT_FILE
fi


#!/bin/bash
# Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License").
# You may not use this file except in compliance with the License.
# A copy of the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "LICENSE.txt" file accompanying this file.
# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied.
# See the License for the specific language governing permissions and limitations under the License.
echo "starting failed_postupdate script"
exit 1


#!/bin/bash
# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License").
# You may not use this file except in compliance with the License.
# A copy of the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "LICENSE.txt" file accompanying this file.
# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied.
# See the License for the specific language governing permissions and limitations under the License.
echo "postinstall-$1" > /tmp/postinstall_out.txt
echo "postinstall run OK"


#!/bin/bash
# Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License").
# You may not use this file except in compliance with the License.
# A copy of the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "LICENSE.txt" file accompanying this file.
# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied.
# See the License for the specific language governing permissions and limitations under the License.
echo "starting postupdate script"
echo "postupdate-$1" > /tmp/postupdate_out.txt
echo "postupdate run OK"