#!/bin/sh
#sudo ./create64bit-rpm.sh xyz

# This routine packages the 64 bit rpm
odbc_version=$1
svn_rev=$2

if [ ! -d ./rpm ]
then
	mkdir -p ./rpm
fi

spec_file=rs-${odbc_version}-${svn_rev}-64.spec

# Create the stamped 64 bit spec file
echo Creating the 64 bit stamped spec file..
sed "s|Version:.*$|Version: ${odbc_version} |" < odbc_x64.spec > tt
sed "s|Name:.*$|Name: AmazonRedshiftODBC-64-bit |" < tt > tt2
sed "s|Release:.*$|Release: ${svn_rev} |" < tt2 > temp64.spec
cat temp64.spec | tr -d "\r" > $spec_file

if [ $? -ne 0 ]
then
    exit $?
fi
rm temp64.spec
rm tt
rm tt2

# Copy the files to stage for RPM creation
if [ -d "/tmp/redshiftodbcx64" ]
then
   rm -Rf /tmp/redshiftodbcx64
fi

mkdir -p /tmp/redshiftodbcx64
mkdir -p /tmp/redshiftodbcx64/samples/connect

cp ./Release/librsodbc64.so /tmp/redshiftodbcx64/
cp ./amazon.redshiftodbc.ini  /tmp/redshiftodbcx64/
cp ./root.crt /tmp/redshiftodbcx64/
cp ./odbc.ini.x64  /tmp/redshiftodbcx64/odbc.ini
cp ./odbcinst.ini.x64 /tmp/redshiftodbcx64/odbcinst.ini
cp ./odbc.csh.x64 /tmp/redshiftodbcx64/odbc.csh
cp ./odbc.sh.x64 /tmp/redshiftodbcx64/odbc.sh

cp ./samples/connect/connect.c /tmp/redshiftodbcx64/samples/connect/
cp ./samples/connect/connect.mak /tmp/redshiftodbcx64/samples/connect/connect.mak
cp ./samples/connect/connect64 /tmp/redshiftodbcx64/samples/connect/connect

cp ./rsodbcsql/Release/rsodbcsql /tmp/redshiftodbcx64/

# This is the directory used by RPMBUILD
cp -avrf /tmp/redshiftodbcx64/ /var/tmp/redshiftodbcx64/

# Set rpm_src for EL5
#rpm_src=/usr/src/rpm/RPMS/x86_64/AmazonRedshiftODBC-64-bit-${odbc_version}-${svn_rev}.x86_64.rpm

# Set rpm_src for EL7
rpm_src=$HOME/rpmbuild/RPMS/x86_64/AmazonRedshiftODBC-64-bit-${odbc_version}-${svn_rev}.x86_64.rpm
rpm_new_name=AmazonRedshiftODBC-64-bit-${odbc_version}.${svn_rev}.x86_64.rpm

# Build the 64 bit rpm
echo Running the 64 bit rpm build using this spec file: $spec_file
rpmbuild -v --target x86_64 -bb $spec_file
if [ $? -ne 0 ]
then
    exit $?
fi

rm -rf /var/tmp/redshiftodbcx64
rm -rf /tmp/redshiftodbcx64
rm rs-${odbc_version}-${svn_rev}-64.spec

# Move the 64 bit file to the current working directory
echo Moving the 64 bit rpm to the rpm folder..
rm ./rpm/$rpm_new_name
mv $rpm_src ./rpm/$rpm_new_name
if [ $? -ne 0 ]
then
    exit $?
fi

exit $?


#!/bin/sh
#./package64.sh [n.n.n n]

function checkExitCode {
  retVal=$1;

  if [ "$retVal" != 0 ]; then
    exit $retVal;
  fi
}

echo Building and Packaging 64 bit Linux Redshift ODBC Driver

# This routine packages the 64 bit rpm
odbc_version=$1
svn_rev=$2

# Build libpq & libpgport
cd ../../pgclient
./build64.sh
checkExitCode $?

# Build ODBC Driver Shared Object
cd ../odbc/rsodbc
./build64.sh $odbc_version $svn_rev
checkExitCode $?

# Build ODBC Driver Samples
cd ./samples
./build64.sh
checkExitCode $?
cd ..

# Build rsodbcsql
cd ./rsodbcsql
./build64.sh
checkExitCode $?
cd ..

# Now make the installer using RPM label provided
./create64bit-rpm.sh ${odbc_version} ${svn_rev}
checkExitCode $?

echo Please get RPM package under "./rpm" directory.

exit $?


#!/bin/sh
#./build64.sh

function checkExitCode {
  retVal=$1;

  if [ "$retVal" != 0 ]; then
    exit $retVal;
  fi
}

echo Building 64 bit Linux rsodbcsql

cd Release
make  clean
checkExitCode $?
make
checkExitCode $?
cd ..

echo Done building 64 bit Linux rsodbcsql

exit $?


#!/bin/sh
#./clean64.sh

echo Cleaning 64 bit Linux rsodbcsql Build

cd Release
make  clean
cd ..

echo Done cleaning 64 bit Linux rsodbcsql Build


#!/bin/sh
#./build64.sh

function checkExitCode {
  retVal=$1;

  if [ "$retVal" != 0 ]; then
    exit $retVal;
  fi
}

echo Building 64 bit Linux Amazon Redshift Analytic Platform ODBC Driver Samples

cd connect
./build64.sh
checkExitCode $?
cd ..

echo Done building 64 bit Linux Amazon Redshift Analytic Platform ODBC Driver Samples

exit $?


#!/bin/sh
#./clean64.sh

echo Cleaning 64 bit Linux Amazon Redshift ODBC Driver Samples Build

cd connect
./clean64.sh
cd ..

echo Done cleaning 64 bit Linux Amazon Redshift ODBC Driver Samples Build


#!/bin/sh
#./build64.sh

function checkExitCode {
  retVal=$1;

  if [ "$retVal" != 0 ]; then
    exit $retVal;
  fi
}

echo Building 64 bit Linux Amazon Redshift ODBC Driver connect Sample

rm ./connect64
make -f connect.mak clean
make -f connect.mak
checkExitCode $?
mv ./connect ./connect64
checkExitCode $?

echo Done building 64 bit Linux Amazon Redshift ODBC Driver connect Sample

exit $?


#!/bin/sh
#./clean64.sh

echo Cleaning 64 bit Linux Amazon Redshift ODBC Driver connect Sample Build

rm ./connect64
make -f connect.mak clean

echo Done cleaning 64 bit Linux Amazon Redshift ODBC Driver connect Sample Build

exit $?


#!/bin/sh
#./build64.sh

function checkExitCode {
  retVal=$1;

  if [ "$retVal" != 0 ]; then
    exit $retVal;
  fi
}

# Command-line switch to prevent removing of header files
keepFlag=on
while [ $# -gt 0 ]
do
  case "$1" in
    -k)  keepFlag=on;;
  esac
  shift
done

echo Building 64 bit Linux 9.1.2 libpq

cd src/interfaces/libpq
make clean
checkExitCode $?
cd ../../port
make clean
checkExitCode $?
cd ../..

cp src/include/pg_config.h.linux64 src/include/pg_config.h
checkExitCode $?
cp src/include/pg_config_os.h.linux64 src/include/pg_config_os.h
checkExitCode $?

cd src/interfaces/libpq
make
checkExitCode $?

if [ ! -d ./Release ]
then
	mkdir -p ./Release
fi

mv libpq.a ./Release/
checkExitCode $?

echo Done building 64 bit Linux 9.1.2 libpq

echo Building 64 bit Linux 9.1.2 libpgport

cd ../../port
make
checkExitCode $?

if [ ! -d ./Release ]
then
	mkdir -p ./Release
fi

mv libpgport.a ./Release/
checkExitCode $?

echo Done building 64 bit Linux 9.1.2 libpgport

cd ../..

if [ "$keepFlag" != "on" ];
then
  rm  src/include/pg_config.h
  rm  src/include/pg_config_os.h
  rm  src/port/pg_config_paths.h
fi

exit $?


#!/bin/sh
#./clean64.sh

echo Cleaning 64 bit Linux 9.1.2 libpq build

cd src/interfaces/libpq
make clean
cd ../../port
make clean
cd ../..

echo Done cleaning 64 bit Linux 9.1.2 libpgport build


#!/bin/bash
# Build libpq 9.1.2 with PA changes

cp ../../include/pg_config.h.linux64 ../../include/pg_config.h
cp ../../include/pg_config_os.h.linux64 ../../include/pg_config_os.h
make


## Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
## SPDX-License-Identifier: Apache-2.0

#!bin/bash

set -x

VERSION=$1
COUNTER=0
STATUS=1

echo "Looking for version $VERSION"

while [  $STATUS -ne 0 ]; do
    mvn org.apache.maven.plugins:maven-dependency-plugin:3.0.1:get \
        -Dartifact=software.amazon.encryption.s3:amazon-s3-encryption-client-java:$VERSION -U

    STATUS=$?
    if [ $STATUS -eq 0 ]; then
        echo "Found version $VERSION in Maven Central :)"
        break
    fi

    if [  $((COUNTER+=1)) -eq 15 ]; then
        echo "It has been an awfully long time, you should check Maven Central for issues"
        exit 1
    fi

    echo "Could not find version $VERSION. Trying again."
    sleep 60
done


#!/bin/bash
set -e
echo "Testing: import awsio"
python -c "import awsio; print(awsio.__version__)"
echo "import awsio succeeded"

read -p "S3 URL : " s3_url
echo Testing: checking setup by quering whether or not $s3_url is an existing file
python -c "from awsio.python.lib.io.s3.s3dataset import file_exists; print(f\"file_exists: {file_exists($s3_url)}\")"
echo Smoke test was successful.


#!/bin/bash

sudo -n true
if [ $? -eq 0 ]; then
  echo "The user has root access."
else
  echo "The user does not have root access. Everything required to run the notebook is already installed and setup. We are good to go!"
  exit 0
fi

# Do we have GPU support?
nvidia-smi > /dev/null 2>&1
if [ $? -eq 0 ]; then
  # check if we have nvidia-docker
  NVIDIA_DOCKER=`rpm -qa | grep -c nvidia-docker2`
  if [ $NVIDIA_DOCKER -eq 0 ]; then
    # Install nvidia-docker2
    #sudo pkill -SIGHUP dockerd
    sudo yum -y remove docker
    sudo yum -y install docker-17.09.1ce-1.111.amzn1

    sudo /etc/init.d/docker start

    curl -s -L https://nvidia.github.io/nvidia-docker/amzn1/nvidia-docker.repo | sudo tee /etc/yum.repos.d/nvidia-docker.repo
    sudo yum install -y nvidia-docker2-2.0.3-1.docker17.09.1.ce.amzn1
    sudo cp daemon.json /etc/docker/daemon.json
    sudo pkill -SIGHUP dockerd
    echo "installed nvidia-docker2"
  else
    echo "nvidia-docker2 already installed. We are good to go!"
  fi
fi

# This is common for both GPU and CPU instances

# check if we have docker-compose
docker-compose version >/dev/null 2>&1
if [ $? -ne 0 ]; then
  # install docker compose
  pip install docker-compose
fi

# check if we need to configure our docker interface
SAGEMAKER_NETWORK=`docker network ls | grep -c sagemaker-local`
if [ $SAGEMAKER_NETWORK -eq 0 ]; then
  docker network create --driver bridge sagemaker-local
fi

# Notebook instance Docker networking fixes
RUNNING_ON_NOTEBOOK_INSTANCE=`sudo iptables -S OUTPUT -t nat | grep -c 169.254.0.2`

# Get the Docker Network CIDR and IP for the sagemaker-local docker interface.
SAGEMAKER_INTERFACE=br-`docker network ls | grep sagemaker-local | cut -d' ' -f1`
DOCKER_NET=`ip route | grep $SAGEMAKER_INTERFACE | cut -d" " -f1`
DOCKER_IP=`ip route | grep $SAGEMAKER_INTERFACE | cut -d" " -f12`

# check if both IPTables and the Route Table are OK.
IPTABLES_PATCHED=`sudo iptables -S PREROUTING -t nat | grep -c 169.254.0.2`
ROUTE_TABLE_PATCHED=`sudo ip route show table agent | grep -c $SAGEMAKER_INTERFACE`

if [ $RUNNING_ON_NOTEBOOK_INSTANCE -gt 0 ]; then

  if [ $ROUTE_TABLE_PATCHED -eq 0 ]; then
    # fix routing
    sudo ip route add $DOCKER_NET via $DOCKER_IP dev $SAGEMAKER_INTERFACE table agent
  else
    echo "SageMaker instance route table setup is ok. We are good to go."
  fi

  if [ $IPTABLES_PATCHED -eq 0 ]; then
    sudo iptables -t nat -A PREROUTING  -i $SAGEMAKER_INTERFACE -d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j DNAT --to-destination 169.254.0.2:9081
    echo "iptables for Docker setup done"
  else
    echo "SageMaker instance routing for Docker is ok. We are good to go!"
  fi
fi


#!/bin/bash

# Get account
account=${1}

# Get the region defined in the current configuration
region=${2:-us-east-1}

# The name of our algorithm
algorithm_name=${3:-sagemaker-autogluon-inference}

uri_prefix=${4}
fullname="${uri_prefix}/${algorithm_name}:latest"

# Get the registry id
registry_id=${5}
registry_uri=${6}

# If the repository doesn't exist in ECR, create it.
aws ecr describe-repositories --region ${region} --repository-names "${algorithm_name}" > /dev/null 2>&1
if [ $? -ne 0 ]
then
    aws ecr create-repository --region ${region} --repository-name "${algorithm_name}" > /dev/null
fi

# Get the login command from ECR and execute it directly
$(aws ecr get-login --region ${region} --no-include-email)
$(aws ecr get-login --registry-ids ${registry_id} --region ${region} --no-include-email)

# Build the docker image, tag with full name and then push it to ECR
docker build -t ${algorithm_name} -f container-inference/Dockerfile.inference . --build-arg REGISTRY_URI=${registry_uri}
docker tag ${algorithm_name} ${fullname}
docker push ${fullname}


#!/bin/bash

# Get account
account=${1}

# Get the region defined in the current configuration
region=${2:-us-east-1}

# The name of our algorithm
algorithm_name=${3:-sagemaker-autogluon-training}

uri_prefix=${4}
fullname="${uri_prefix}/${algorithm_name}:latest"

# Get the registry id
registry_id=${5}
registry_uri=${6}

# If the repository doesn't exist in ECR, create it.
aws ecr describe-repositories --region ${region} --repository-names "${algorithm_name}" > /dev/null 2>&1
if [ $? -ne 0 ]
then
    aws ecr create-repository --region ${region} --repository-name "${algorithm_name}" > /dev/null
fi

# Get the login command from ECR and execute it directly
$(aws ecr get-login --region ${region} --no-include-email)
$(aws ecr get-login --registry-ids ${registry_id} --region ${region} --no-include-email)

# Build the docker image, tag with full name and then push it to ECR
docker build -t ${algorithm_name} -f container-training/Dockerfile.training . --build-arg REGISTRY_URI=${registry_uri}
docker tag ${algorithm_name} ${fullname}
docker push ${fullname}


#!/usr/bin/env bash

# This script shows how to build the Docker image and push it to ECR to be ready for use
# by SageMaker.

# The argument to this script is the image name. This will be used as the image on the local
# machine and combined with the account and region to form the repository name for ECR.
image=$1

if [ "$image" == "" ]
then
    echo "Usage: $0 <image-name>"
    exit 1
fi

chmod +x causal_nex/train
chmod +x causal_nex/serve

# Get the account number associated with the current IAM credentials
account=$(aws sts get-caller-identity --query Account --output text)

if [ $? -ne 0 ]
then
    exit 255
fi

# Get the region defined in the current configuration (default to us-west-2 if none defined)
region=$(aws configure get region)
region=${region:-us-west-2}

fullname="${account}.dkr.ecr.${region}.amazonaws.com/${image}:latest"

# If the repository doesn't exist in ECR, create it.

aws ecr describe-repositories --repository-names "${image}" > /dev/null 2>&1

if [ $? -ne 0 ]
then
    aws ecr create-repository --repository-name "${image}" > /dev/null
fi

# Get the login command from ECR and execute it directly
aws ecr get-login-password --region "${region}" | docker login --username AWS --password-stdin "${account}".dkr.ecr."${region}".amazonaws.com

# Build the docker image locally with the image name and then push it to ECR
# with the full name.

docker build  -t ${image} .
docker tag ${image} ${fullname}

docker push ${fullname}


#!/bin/bash

payload=$1
content=${2:-application/json}

curl --data-binary @${payload} -H "Content-Type: ${content}" -v http://localhost:8080/invocations


#!/bin/sh

image=$1

docker run -v $(pwd)/test_dir:/opt/ml -p 8080:8080 --rm ${image} serve


#!/bin/sh

image=$1

mkdir -p test_dir/model
mkdir -p test_dir/output

rm test_dir/model/*
rm test_dir/output/*

docker run -v $(pwd)/test_dir:/opt/ml --rm ${image} train


ACCOUNT_ID=$1
REGION=$2
REPO_NAME=$3

docker build -f ../docker/Dockerfile -t $REPO_NAME ../docker

docker tag $REPO_NAME $ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com/$REPO_NAME:latest

$(aws ecr get-login --no-include-email --registry-ids $ACCOUNT_ID)

aws ecr describe-repositories --repository-names $REPO_NAME || aws ecr create-repository --repository-name $REPO_NAME

docker push $ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com/$REPO_NAME:latest


ACCOUNT_ID=$1
REGION=$2
REPO_NAME=$3

cd ../package/ && python setup.py sdist && cp dist/custom_framework_training-1.0.0.tar.gz ../docker/code/

docker build -f ../docker/Dockerfile -t $REPO_NAME ../docker

docker tag $REPO_NAME $ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com/$REPO_NAME:latest

$(aws ecr get-login --no-include-email --registry-ids $ACCOUNT_ID)

aws ecr describe-repositories --repository-names $REPO_NAME || aws ecr create-repository --repository-name $REPO_NAME

docker push $ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com/$REPO_NAME:latest


#!/bin/bash

if [ "$#" -ne 1 ]; then
    echo "usage: $0 <s3-bucket-name>"
    exit 1
fi

S3_BUCKET=$1
S3_PREFIX="mask-rcnn/sagemaker/input"

mounted=$(df -kh | grep "$HOME/efs" | wc -l)
if [ "$mounted" -lt "1" ]; then
   echo "EFS file-system is not mounted"
   exit 1
else
   echo "EFS file-system is mounted"
   echo $(df -kh | grep "$HOME/efs")
fi

if [ -d $HOME/efs/$S3_PREFIX ]; then
    echo "$HOME/efs/$S3_PREFIX already exists"
    exit 1
fi
sudo mkdir -p $HOME/efs/$S3_PREFIX

echo "`date`: Copying files from s3://$S3_BUCKET/$S3_PREFIX [ eta 46 minutes ]"
sudo aws s3 cp --recursive s3://$S3_BUCKET/$S3_PREFIX $HOME/efs/$S3_PREFIX | awk 'BEGIN {ORS="="} {if(NR%200==0)print "="}'
echo "Done."


#!/bin/bash

set -e

if [ "$#" -ne 1 ]; then
    echo "usage: $0 <s3-bucket-name>"
    exit 1
fi

S3_BUCKET=$1
S3_PREFIX="mask-rcnn/sagemaker/input"

# Stage directory must be on EBS volume with 100 GB available space
STAGE_DIR=$HOME/SageMaker/coco-2017-$(date +"%Y-%m-%d-%H-%M-%S")

echo "Create stage directory: $STAGE_DIR"
mkdir -p $STAGE_DIR

wget -O $STAGE_DIR/train2017.zip http://images.cocodataset.org/zips/train2017.zip
echo "Extracting $STAGE_DIR/train2017.zip"
unzip -o $STAGE_DIR/train2017.zip  -d $STAGE_DIR | awk 'BEGIN {ORS="="} {if(NR%1000==0)print "="}'
echo "Done."
rm $STAGE_DIR/train2017.zip

wget -O $STAGE_DIR/val2017.zip http://images.cocodataset.org/zips/val2017.zip
echo "Extracting $STAGE_DIR/val2017.zip"
unzip -o $STAGE_DIR/val2017.zip -d $STAGE_DIR | awk 'BEGIN {ORS="="} {if(NR%1000==0)print "="}'
echo "Done."
rm $STAGE_DIR/val2017.zip

wget -O $STAGE_DIR/annotations_trainval2017.zip http://images.cocodataset.org/annotations/annotations_trainval2017.zip
unzip -o $STAGE_DIR/annotations_trainval2017.zip -d $STAGE_DIR
rm $STAGE_DIR/annotations_trainval2017.zip

mkdir $STAGE_DIR/pretrained-models
wget -O $STAGE_DIR/pretrained-models/ImageNet-R50-AlignPadding.npz http://models.tensorpack.com/FasterRCNN/ImageNet-R50-AlignPadding.npz

echo "`date`: Uploading extracted files to s3://$S3_BUCKET/$S3_PREFIX/train [ eta 12 minutes ]"
aws s3 cp --recursive $STAGE_DIR s3://$S3_BUCKET/$S3_PREFIX/train | awk 'BEGIN {ORS="="} {if(NR%100==0)print "="}'
echo "Done."

echo "Delete stage directory: $STAGE_DIR"
rm -rf $STAGE_DIR
echo "Success."


#!/bin/bash

if [ $# -lt 5 ]; then
    echo "usage: $0 <aws-region> <s3-import-path> <fsx-capacity> <subnet-id> <security-group-id>"
    exit 1
fi

# AWS Region
AWS_REGION=$1

# S3 import path
S3_IMPORT_PATH=$2

# FSx storage capacity in multiples of 3600
STORAGE_CAPACITY=$3

SUBNET_ID=$4

SG_ID=$5

DATE=`date +%s`

#Customize stack name as needed
STACK_NAME="fsx-stack-$DATE"

# cfn template name
CFN_TEMPLATE='cfn-fsx.yaml'

aws cloudformation create-stack --region $AWS_REGION  --stack-name $STACK_NAME \
--template-body file://$CFN_TEMPLATE \
--capabilities CAPABILITY_NAMED_IAM \
--parameters \
ParameterKey=S3ImportPath,ParameterValue=$S3_IMPORT_PATH \
ParameterKey=StorageCapacityGiB,ParameterValue=$STORAGE_CAPACITY \
ParameterKey=SecurityGroupId,ParameterValue=$SG_ID \
ParameterKey=SubnetId,ParameterValue=$SUBNET_ID

echo "Creating FSx Luster file-system [eta 600 seconds]"

sleep 30

progress=$(aws cloudformation list-stacks --stack-status-filter 'CREATE_IN_PROGRESS' | grep $STACK_NAME | wc -l)
while [ $progress -ne 0 ]; do
let elapsed="`date +%s` - $DATE"
echo "Stack $STACK_NAME status: CREATE_IN_PROGRESS: [ $elapsed secs elapsed ]"
sleep 30
progress=$(aws cloudformation list-stacks --stack-status-filter 'CREATE_IN_PROGRESS' | grep $STACK_NAME | wc -l)
done
sleep 5
aws cloudformation describe-stacks --stack-name $STACK_NAME


#!/bin/bash

# AWS Region; customize as needed
AWS_REGION=
[[ -z "${AWS_REGION}" ]] && echo "AWS_REGION is required" && exit 1

# Cutomize bucket name
S3_BUCKET=
[[ -z "${S3_BUCKET}" ]] && echo "S3_BUCKET is required" && exit 1

DATE=`date +%s`

#Customize stack name as needed
STACK_NAME="sm-stack-$DATE"

# customzie note book instance name as needed
NOTEBOOK_INSTANCE_NAME="sm-nb-$DATE"

# cfn template name
CFN_TEMPLATE='cfn-sm.yaml'

# Leave blank if you need to create a new EFS file system
# If you use an existing EFS file-system, it must not have any
# existing mount targets
EFS_ID=

# Notebook instance type
# ml.m5.2xlarge or ml.m4.2xlarge
NOTEBOOK_INSTANCE_TYPE='ml.m5.2xlarge'

# Code repository name
CODE_REPO_NAME="code-repo-$DATE"

# Git hub user name
GIT_USER=

# Git hub token
GIT_TOKEN=

# Git Hub repo url
GIT_URL=

# EBS volume size 100 - 500 GB
EBS_VOLUME_SIZE=100

aws cloudformation create-stack --region $AWS_REGION  --stack-name $STACK_NAME \
--template-body file://$CFN_TEMPLATE \
--capabilities CAPABILITY_NAMED_IAM \
--parameters \
ParameterKey=EFSFileSystemId,ParameterValue=$EFS_ID \
ParameterKey=NotebookInstanceType,ParameterValue=$NOTEBOOK_INSTANCE_TYPE \
ParameterKey=NotebookInstanceName,ParameterValue=$NOTEBOOK_INSTANCE_NAME \
ParameterKey=S3BucketName,ParameterValue=$S3_BUCKET \
ParameterKey=CodeRepoName,ParameterValue=$CODE_REPO_NAME \
ParameterKey=GitHubUsername,ParameterValue=$GIT_USER \
ParameterKey=GitHubToken,ParameterValue=$GIT_TOKEN \
ParameterKey=GitHubRepoUrl,ParameterValue=$GIT_URL \
ParameterKey=EbsVolumeSize,ParameterValue=$EBS_VOLUME_SIZE

echo "Creating stack [ eta 600 seconds ]"
sleep 30

progress=$(aws --region $AWS_REGION cloudformation list-stacks --stack-status-filter 'CREATE_IN_PROGRESS' | grep $STACK_NAME | wc -l)
while [ $progress -ne 0 ]; do
let elapsed="`date +%s` - $DATE"
echo "Stack $STACK_NAME status: Create in progress: [ $elapsed secs elapsed ]"
sleep 30
progress=$(aws --region $AWS_REGION  cloudformation  list-stacks --stack-status-filter 'CREATE_IN_PROGRESS' | grep $STACK_NAME | wc -l)
done
sleep 5
aws --region $AWS_REGION  cloudformation describe-stacks --stack-name $STACK_NAME


#!/usr/bin/env bash

# This script shows how to build the Docker image and push it to ECR to be ready for use
# by SageMaker.

# The argument to this script is the image name. This will be used as the image on the local
# machine and combined with the account and region to form the repository name for ECR.

DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
source $DIR/set_env.sh

# set region
region=
if [ "$#" -eq 1 ]; then
    region=$1
else
    echo "usage: $0 <aws-region>"
    exit 1
fi

image=$IMAGE_NAME
tag=$IMAGE_TAG

# Get the account number associated with the current IAM credentials
account=$(aws sts get-caller-identity --query Account --output text)

if [ $? -ne 0 ]
then
    exit 255
fi

fullname="${account}.dkr.ecr.${region}.amazonaws.com/${image}:${tag}"

# If the repository doesn't exist in ECR, create it.
aws ecr describe-repositories --region ${region} --repository-names "${image}" > /dev/null 2>&1
if [ $? -ne 0 ]; then
    aws ecr create-repository --region ${region} --repository-name "${image}" > /dev/null
fi

# Build the docker image locally with the image name and then push it to ECR
# with the full name.

aws ecr get-login-password --region us-west-2 \
    | docker login --username AWS --password-stdin 763104351884.dkr.ecr.us-west-2.amazonaws.com

docker build  -t ${image} $DIR/..
docker tag ${image} ${fullname}

aws ecr get-login-password --region ${region} \
    | docker login --username AWS --password-stdin ${account}.dkr.ecr.${region}.amazonaws.com

docker push ${fullname}
if [ $? -eq 0 ]; then
	echo "Amazon ECR URI: ${fullname}"
else
	echo "Error: Image build and push failed"
	exit 1
fi


#!/usr/bin/env bash

export IMAGE_NAME=mask-rcnn-tensorflow-sagemaker-script-mode
export IMAGE_TAG=tf1.15-99dda64


#!/usr/bin/env bash

export IMAGE_NAME=mask-rcnn-tensorpack-sagemaker-script-mode
export IMAGE_TAG=tf1.15-tpdb541e8


#!/usr/bin/env bash

# This script shows how to build the Docker image and push it to ECR to be ready for use
# by SageMaker.

# The argument to this script is the image name. This will be used as the image on the local
# machine and combined with the account and region to form the repository name for ECR.

DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
source $DIR/set_env.sh

# set region
region=
if [ "$#" -eq 1 ]; then
    region=$1
else
    echo "usage: $0 <aws-region>"
    exit 1
fi

image=$IMAGE_NAME
tag=$IMAGE_TAG

# Get the account number associated with the current IAM credentials
account=$(aws sts get-caller-identity --query Account --output text)

if [ $? -ne 0 ]
then
    exit 255
fi

fullname="${account}.dkr.ecr.${region}.amazonaws.com/${image}:${tag}"

# If the repository doesn't exist in ECR, create it.
aws ecr describe-repositories --region ${region} --repository-names "${image}" > /dev/null 2>&1
if [ $? -ne 0 ]; then
    aws ecr create-repository --region ${region} --repository-name "${image}" > /dev/null
fi

# Build the docker image locally with the image name and then push it to ECR
# with the full name.

# Get the login command from ECR and execute it directly
$(aws ecr get-login --no-include-email --region us-west-2  --registry-ids 763104351884)

docker build  -t ${image} $DIR/..
docker tag ${image} ${fullname}

# Get the login command from ECR and execute it directly
$(aws ecr get-login --region ${region} --no-include-email)
docker push ${fullname}
if [ $? -eq 0 ]; then
	echo "Amazon ECR URI: ${fullname}"
else
	echo "Error: Image build and push failed"
	exit 1
fi


#!/usr/bin/env bash

export IMAGE_NAME=mask-rcnn-tensorpack-serving-sagemaker
export IMAGE_TAG=tf1.15-tpdb541e8


#!/usr/bin/env bash

export IMAGE_NAME=mask-rcnn-tensorflow-serving-sagemaker
export IMAGE_TAG=tf1.15-99dda64


#!/bin/bash

algorithm_name=$1

chmod +x fairseq/train
chmod +x fairseq/serve

account=$(aws sts get-caller-identity --query Account --output text)

# Get the region defined in the current configuration (default to us-west-2 if none defined)
region=$(aws configure get region)
region=${region:-us-west-2}

fullname="${account}.dkr.ecr.${region}.amazonaws.com/${algorithm_name}:latest"

echo Getting from region $region and account $account

# If the repository doesn't exist in ECR, create it.
aws ecr describe-repositories --repository-names "${algorithm_name}" > /dev/null 2>&1

if [ $? -ne 0 ]
then
    aws ecr create-repository --repository-name "${algorithm_name}" > /dev/null
fi

# Get the login command from ECR and execute it directly
$(aws ecr get-login --region ${region} --no-include-email)

# Get the login command from ECR in order to pull down the SageMaker PyTorch image
$(aws ecr get-login --registry-ids 520713654638 --region ${region} --no-include-email)

docker build -t ${algorithm_name} .
docker tag ${algorithm_name} ${fullname}

docker push ${fullname}


#!/usr/bin/env bash
#
# Adapted from https://github.com/facebookresearch/MIXER/blob/master/prepareData.sh

echo 'Cloning Moses github repository (for tokenization scripts)...'
git clone https://github.com/moses-smt/mosesdecoder.git

echo 'Cloning Subword NMT repository (for BPE pre-processing)...'
git clone https://github.com/rsennrich/subword-nmt.git

SCRIPTS=mosesdecoder/scripts
TOKENIZER=$SCRIPTS/tokenizer/tokenizer.perl
LC=$SCRIPTS/tokenizer/lowercase.perl
CLEAN=$SCRIPTS/training/clean-corpus-n.perl
BPEROOT=subword-nmt
BPE_TOKENS=10000

URL="https://wit3.fbk.eu/archive/2014-01/texts/de/en/de-en.tgz"
GZ=de-en.tgz

if [ ! -d "$SCRIPTS" ]; then
    echo "Please set SCRIPTS variable correctly to point to Moses scripts."
    exit
fi

src=de
tgt=en
lang=de-en
prep=iwslt14.tokenized.de-en
tmp=$prep/tmp
orig=orig

mkdir -p $orig $tmp $prep

echo "Downloading data from ${URL}..."
cd $orig
wget "$URL"

if [ -f $GZ ]; then
    echo "Data successfully downloaded."
else
    echo "Data not successfully downloaded."
    exit
fi

tar zxvf $GZ
cd ..

echo "pre-processing train data..."
for l in $src $tgt; do
    f=train.tags.$lang.$l
    tok=train.tags.$lang.tok.$l

    cat $orig/$lang/$f | \
    grep -v '<url>' | \
    grep -v '<talkid>' | \
    grep -v '<keywords>' | \
    sed -e 's/<title>//g' | \
    sed -e 's/<\/title>//g' | \
    sed -e 's/<description>//g' | \
    sed -e 's/<\/description>//g' | \
    perl $TOKENIZER -threads 8 -l $l > $tmp/$tok
    echo ""
done
perl $CLEAN -ratio 1.5 $tmp/train.tags.$lang.tok $src $tgt $tmp/train.tags.$lang.clean 1 175
for l in $src $tgt; do
    perl $LC < $tmp/train.tags.$lang.clean.$l > $tmp/train.tags.$lang.$l
done

echo "pre-processing valid/test data..."
for l in $src $tgt; do
    for o in `ls $orig/$lang/IWSLT14.TED*.$l.xml`; do
    fname=${o##*/}
    f=$tmp/${fname%.*}
    echo $o $f
    grep '<seg id' $o | \
        sed -e 's/<seg id="[0-9]*">\s*//g' | \
        sed -e 's/\s*<\/seg>\s*//g' | \
        sed -e "s/\â€™/\'/g" | \
    perl $TOKENIZER -threads 8 -l $l | \
    perl $LC > $f
    echo ""
    done
done

echo "creating train, valid, test..."
for l in $src $tgt; do
    awk '{if (NR%23 == 0)  print $0; }' $tmp/train.tags.de-en.$l > $tmp/valid.$l
    awk '{if (NR%23 != 0)  print $0; }' $tmp/train.tags.de-en.$l > $tmp/train.$l

    cat $tmp/IWSLT14.TED.dev2010.de-en.$l \
        $tmp/IWSLT14.TEDX.dev2012.de-en.$l \
        $tmp/IWSLT14.TED.tst2010.de-en.$l \
        $tmp/IWSLT14.TED.tst2011.de-en.$l \
        $tmp/IWSLT14.TED.tst2012.de-en.$l \
        > $tmp/test.$l
done

TRAIN=$tmp/train.en-de
BPE_CODE=$prep/code
rm -f $TRAIN
for l in $src $tgt; do
    cat $tmp/train.$l >> $TRAIN
done

echo "learn_bpe.py on ${TRAIN}..."
python $BPEROOT/learn_bpe.py -s $BPE_TOKENS < $TRAIN > $BPE_CODE

for L in $src $tgt; do
    for f in train.$L valid.$L test.$L; do
        echo "apply_bpe.py to ${f}..."
        python $BPEROOT/apply_bpe.py -c $BPE_CODE < $tmp/$f > $prep/$f
    done
done


#!/bin/bash
# Adapted from https://github.com/facebookresearch/MIXER/blob/master/prepareData.sh

echo 'Cloning Moses github repository (for tokenization scripts)...'
git clone https://github.com/moses-smt/mosesdecoder.git

echo 'Cloning Subword NMT repository (for BPE pre-processing)...'
git clone https://github.com/rsennrich/subword-nmt.git

SCRIPTS=mosesdecoder/scripts
TOKENIZER=$SCRIPTS/tokenizer/tokenizer.perl
CLEAN=$SCRIPTS/training/clean-corpus-n.perl
NORM_PUNC=$SCRIPTS/tokenizer/normalize-punctuation.perl
REM_NON_PRINT_CHAR=$SCRIPTS/tokenizer/remove-non-printing-char.perl
BPEROOT=subword-nmt
BPE_TOKENS=40000

URLS=(
    "http://statmt.org/wmt13/training-parallel-europarl-v7.tgz"
    "http://statmt.org/wmt13/training-parallel-commoncrawl.tgz"
    "http://statmt.org/wmt13/training-parallel-un.tgz"
    "http://statmt.org/wmt14/training-parallel-nc-v9.tgz"
    "http://statmt.org/wmt10/training-giga-fren.tar"
    "http://statmt.org/wmt14/test-full.tgz"
)
FILES=(
    "training-parallel-europarl-v7.tgz"
    "training-parallel-commoncrawl.tgz"
    "training-parallel-un.tgz"
    "training-parallel-nc-v9.tgz"
    "training-giga-fren.tar"
    "test-full.tgz"
)
CORPORA=(
    "training/europarl-v7.fr-en"
    "commoncrawl.fr-en"
    "un/undoc.2000.fr-en"
    "training/news-commentary-v9.fr-en"
    "giga-fren.release2.fixed"
)

if [ ! -d "$SCRIPTS" ]; then
    echo "Please set SCRIPTS variable correctly to point to Moses scripts."
    exit
fi

src=en
tgt=fr
lang=en-fr
prep=wmt14_en_fr
tmp=$prep/tmp
orig=orig

mkdir -p $orig $tmp $prep

cd $orig

for ((i=0;i<${#URLS[@]};++i)); do
    file=${FILES[i]}
    if [ -f $file ]; then
        echo "$file already exists, skipping download"
    else
        url=${URLS[i]}
        wget "$url"
        if [ -f $file ]; then
            echo "$url successfully downloaded."
        else
            echo "$url not successfully downloaded."
            exit -1
        fi
        if [ ${file: -4} == ".tgz" ]; then
            tar zxvf $file
        elif [ ${file: -4} == ".tar" ]; then
            tar xvf $file
        fi
    fi
done

gunzip giga-fren.release2.fixed.*.gz
cd ..

echo "pre-processing train data..."
for l in $src $tgt; do
    rm $tmp/train.tags.$lang.tok.$l
    for f in "${CORPORA[@]}"; do
        cat $orig/$f.$l | \
            perl $NORM_PUNC $l | \
            perl $REM_NON_PRINT_CHAR | \
            perl $TOKENIZER -threads 8 -a -l $l >> $tmp/train.tags.$lang.tok.$l
    done
done

echo "pre-processing test data..."
for l in $src $tgt; do
    if [ "$l" == "$src" ]; then
        t="src"
    else
        t="ref"
    fi
    grep '<seg id' $orig/test-full/newstest2014-fren-$t.$l.sgm | \
        sed -e 's/<seg id="[0-9]*">\s*//g' | \
        sed -e 's/\s*<\/seg>\s*//g' | \
        sed -e "s/\â€™/\'/g" | \
    perl $TOKENIZER -threads 8 -a -l $l > $tmp/test.$l
    echo ""
done

echo "splitting train and valid..."
for l in $src $tgt; do
    awk '{if (NR%1333 == 0)  print $0; }' $tmp/train.tags.$lang.tok.$l > $tmp/valid.$l
    awk '{if (NR%1333 != 0)  print $0; }' $tmp/train.tags.$lang.tok.$l > $tmp/train.$l
done

TRAIN=$tmp/train.fr-en
BPE_CODE=$prep/code
rm -f $TRAIN
for l in $src $tgt; do
    cat $tmp/train.$l >> $TRAIN
done

echo "learn_bpe.py on ${TRAIN}..."
python $BPEROOT/learn_bpe.py -s $BPE_TOKENS < $TRAIN > $BPE_CODE

for L in $src $tgt; do
    for f in train.$L valid.$L test.$L; do
        echo "apply_bpe.py to ${f}..."
        python $BPEROOT/apply_bpe.py -c $BPE_CODE < $tmp/$f > $tmp/bpe.$f
    done
done

perl $CLEAN -ratio 1.5 $tmp/bpe.train $src $tgt $prep/train 1 250
perl $CLEAN -ratio 1.5 $tmp/bpe.valid $src $tgt $prep/valid 1 250

for L in $src $tgt; do
    cp $tmp/bpe.test.$L $prep/test.$L
done


#!/usr/bin/env bash

if [ $1 == 'train' ]
then
    CURRENT_HOST=$(jq .current_host  /opt/ml/input/config/resourceconfig.json)

    sed -ie "s/PLACEHOLDER_HOSTNAME/$CURRENT_HOST/g" /changehostname.c

    gcc -o /changehostname.o -c -fPIC -Wall /changehostname.c
    gcc -o /libchangehostname.so -shared -export-dynamic /changehostname.o -ldl

    LD_PRELOAD=/libchangehostname.so train
else
    serve
fi


#!/usr/bin/env bash

# This script shows how to build the Docker image and push it to ECR to be ready for use
# by SageMaker.

# The argument to this script is the image name. This will be used as the image on the local
# machine and combined with the account and region to form the repository name for ECR.
image=$1

if [ "$image" == "" ]
then
    echo "Usage: $0 <image-name>"
    exit 1
fi

# Get the account number associated with the current IAM credentials
account=$(aws sts get-caller-identity --query Account --output text)

if [ $? -ne 0 ]
then
    exit 255
fi

# Get the region defined in the current configuration (default to us-west-2 if none defined)
region=$(aws configure get region)
region=${region:-us-west-2}

fullname="${account}.dkr.ecr.${region}.amazonaws.com/${image}:latest"

# If the repository doesn't exist in ECR, create it.

aws ecr describe-repositories --repository-names "${image}" > /dev/null 2>&1

if [ $? -ne 0 ]
then
    aws ecr create-repository --repository-name "${image}" > /dev/null
fi

# Get the login command from ECR and execute it directly
aws ecr get-login-password --region $region | docker login --username AWS --password-stdin $account.dkr.ecr.$region.amazonaws.com

# Get the login command from ECR in order to pull down the SageMaker PyTorch image
aws ecr get-login-password --region $region | docker login --username AWS --password-stdin 520713654638.dkr.ecr.$region.amazonaws.com

# Build the docker image locally with the image name and then push it to ECR
# with the full name.

docker build  -t ${image} . --build-arg REGION=${region}
docker tag ${image} ${fullname}

docker push ${fullname}


#!/bin/bash

sudo -n true
if [ $? -eq 0 ]; then
  echo "The user has root access."
else
  echo "The user does not have root access. Everything required to run the notebook is already installed and setup. We are good to go!"
  exit 0
fi

# Do we have GPU support?
nvidia-smi > /dev/null 2>&1
if [ $? -eq 0 ]; then
  # check if we have nvidia-docker
  NVIDIA_DOCKER=`rpm -qa | grep -c nvidia-docker2`
  if [ $NVIDIA_DOCKER -eq 0 ]; then
    # Install nvidia-docker2
    #sudo pkill -SIGHUP dockerd
    sudo yum -y remove docker
    sudo yum -y install docker-17.09.1ce-1.111.amzn1

    sudo /etc/init.d/docker start

    curl -s -L https://nvidia.github.io/nvidia-docker/amzn1/nvidia-docker.repo | sudo tee /etc/yum.repos.d/nvidia-docker.repo
    sudo yum install -y nvidia-docker2-2.0.3-1.docker17.09.1.ce.amzn1
    sudo cp daemon.json /etc/docker/daemon.json
    sudo pkill -SIGHUP dockerd
    echo "installed nvidia-docker2"
  else
    echo "nvidia-docker2 already installed. We are good to go!"
  fi
fi

# This is common for both GPU and CPU instances

# check if we have docker-compose
docker-compose version >/dev/null 2>&1
if [ $? -ne 0 ]; then
  # install docker compose
  pip install docker-compose
fi

# check if we need to configure our docker interface
SAGEMAKER_NETWORK=`docker network ls | grep -c sagemaker-local`
if [ $SAGEMAKER_NETWORK -eq 0 ]; then
  docker network create --driver bridge sagemaker-local
fi

# Notebook instance Docker networking fixes
RUNNING_ON_NOTEBOOK_INSTANCE=`sudo iptables -S OUTPUT -t nat | grep -c 169.254.0.2`

# Get the Docker Network CIDR and IP for the sagemaker-local docker interface.
SAGEMAKER_INTERFACE=br-`docker network ls | grep sagemaker-local | cut -d' ' -f1`
DOCKER_NET=`ip route | grep $SAGEMAKER_INTERFACE | cut -d" " -f1`
DOCKER_IP=`ip route | grep $SAGEMAKER_INTERFACE | cut -d" " -f12`

# check if both IPTables and the Route Table are OK.
IPTABLES_PATCHED=`sudo iptables -S PREROUTING -t nat | grep -c 169.254.0.2`
ROUTE_TABLE_PATCHED=`sudo ip route show table agent | grep -c $SAGEMAKER_INTERFACE`

if [ $RUNNING_ON_NOTEBOOK_INSTANCE -gt 0 ]; then

  if [ $ROUTE_TABLE_PATCHED -eq 0 ]; then
    # fix routing
    sudo ip route add $DOCKER_NET via $DOCKER_IP dev $SAGEMAKER_INTERFACE table agent
  else
    echo "SageMaker instance route table setup is ok. We are good to go."
  fi

  if [ $IPTABLES_PATCHED -eq 0 ]; then
    sudo iptables -t nat -A PREROUTING  -i $SAGEMAKER_INTERFACE -d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j DNAT --to-destination 169.254.0.2:9081
    echo "iptables for Docker setup done"
  else
    echo "SageMaker instance routing for Docker is ok. We are good to go!"
  fi
fi


#!/usr/bin/env bash

# This script shows how to build the Docker image and push it to ECR to be ready for use
# by SageMaker.

# The argument to this script is the image name. This will be used as the image on the local
# machine and combined with the account and region to form the repository name for ECR.
image=$1

dockerfile=${2:-Dockerfile}

if [ "$image" == "" ]
then
    echo "Usage: $0 <image-name>"
    exit 1
fi

account=$(aws sts get-caller-identity --query Account --output text)
# Get the region defined in the current configuration (default to us-west-2 if none defined)
region=$(aws configure get region)
region=${region:-us-west-2}

fullname="${account}.dkr.ecr.${region}.amazonaws.com/${image}:latest"
echo "ECR image fullname: ${fullname}"
# If the repository doesn't exist in ECR, create it.

aws ecr describe-repositories --repository-names "${image}" > /dev/null 2>&1

if [ $? -ne 0 ]
then
    aws ecr create-repository --repository-name "${image}" > /dev/null
fi

# Get the login command from ECR and execute it directly
export _DOCKER_REPO="$(aws ecr get-authorization-token --output text  --query 'authorizationData[].proxyEndpoint')"
aws ecr get-login --no-include-email --region "${region}" | awk '{print $6}' | docker login -u AWS --password-stdin $_DOCKER_REPO

# Get the login command from ECR in order to pull down the SageMaker PyTorch image
aws ecr get-login --no-include-email --region "${region}" | awk '{print $6}' | docker login -u AWS --password-stdin "763104351884.dkr.ecr.${region}.amazonaws.com"

# Build the docker image locally with the image name and then push it to ECR
# with the full name.

docker build -f ${dockerfile} -t ${image} . --build-arg REGION=${region}
docker tag ${image} ${fullname}
docker push ${fullname}
# an alternative and simplified command that can be used instead of the code above is
# sm-docker build ${1}


#!/bin/bash

# The name of our algorithm
algorithm_name=pytorch-training-container-extension-yolov5-cpu

cd docker

account=$(aws sts get-caller-identity --query Account --output text)

# Get the region defined in the current configuration (default to us-west-2 if none defined)
region=$(aws configure get region)
region=${region:-us-east-1}

fullname="${account}.dkr.ecr.${region}.amazonaws.com/${algorithm_name}:latest"

# If the repository doesn't exist in ECR, create it.

aws ecr describe-repositories --repository-names "${algorithm_name}" > /dev/null 2>&1

if [ $? -ne 0 ]
then
    aws ecr create-repository --repository-name "${algorithm_name}" > /dev/null
fi

# Get the login command from ECR and execute it directly
$(aws ecr get-login --region ${region} --no-include-email)

# Get the login command from ECR in order to pull down the SageMaker PyTorch image
$(aws ecr get-login --registry-ids 763104351884 --region ${region} --no-include-email)

# Build the docker image locally with the image name and then push it to ECR
# with the full name.

docker build  -t ${algorithm_name} . --build-arg REGION=${region}
docker tag ${algorithm_name} ${fullname}

docker push ${fullname}


#!/usr/bin/env bash

# This script shows how to build the Docker image and push it to ECR to be ready for use
# by SageMaker.

# The argument to this script is the image name. This will be used as the image on the local
# machine and combined with the account and region to form the repository name for ECR.
image=$1

if [ "$image" == "" ]
then
    echo "Usage: $0 <image-name>"
    exit 1
fi

chmod +x decision_trees/train
chmod +x decision_trees/serve

# Get the account number associated with the current IAM credentials
account=$(aws sts get-caller-identity --query Account --output text)

if [ $? -ne 0 ]
then
    exit 255
fi

# Get the region defined in the current configuration (default to us-west-2 if none defined)
region=$(aws configure get region)
region=${region:-us-west-2}

fullname="${account}.dkr.ecr.${region}.amazonaws.com/${image}:latest"

# If the repository doesn't exist in ECR, create it.

aws ecr describe-repositories --repository-names "${image}" > /dev/null 2>&1

if [ $? -ne 0 ]
then
    aws ecr create-repository --repository-name "${image}" > /dev/null
fi

# Get the login command from ECR and execute it directly
aws ecr get-login-password --region "${region}" | docker login --username AWS --password-stdin "${account}".dkr.ecr."${region}".amazonaws.com

# Build the docker image locally with the image name and then push it to ECR
# with the full name.

docker build  -t ${image} .
docker tag ${image} ${fullname}

docker push ${fullname}


#!/bin/bash

payload=$1
content=${2:-text/csv}

curl --data-binary @${payload} -H "Content-Type: ${content}" -v http://localhost:8080/invocations


#!/bin/sh

image=$1

docker run -v $(pwd)/test_dir:/opt/ml -p 8080:8080 --rm ${image} serve


#!/bin/sh

image=$1

mkdir -p test_dir/model
mkdir -p test_dir/output

rm test_dir/model/*
rm test_dir/output/*

docker run -v $(pwd)/test_dir:/opt/ml --rm ${image} train


c4b301c88fc1:amazon-sagemaker-examples julkroll$ git diff
diff --git a/advanced_functionality/xgboost_bring_your_own_model/xgboost_bring_your_own_model.ipynb b/advanced_functionality/xgboost_bring_your_own_model/xgboost_bring_your_own_model.ipynb
index 8df40914..f9b5a511 100644
--- a/advanced_functionality/xgboost_bring_your_own_model/xgboost_bring_your_own_model.ipynb
+++ b/advanced_functionality/xgboost_bring_your_own_model/xgboost_bring_your_own_model.ipynb
@@ -1,18 +1,7 @@
 {
  "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "canadian-powell",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "!pip install -Uq xgboost"
-   ]
-  },
   {
    "cell_type": "markdown",
-   "id": "animal-static",
    "metadata": {},
    "source": [
     "# Amazon SageMaker XGBoost Bring Your Own Model\n",
@@ -54,7 +43,6 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "comic-jonathan",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -77,7 +65,6 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "revolutionary-egypt",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -88,30 +75,15 @@
   },
   {
    "cell_type": "markdown",
-   "id": "second-traffic",
    "metadata": {},
    "source": [
     "## Optionally, train a scikit learn XGBoost model\n",
     "\n",
-    "These steps are optional and are needed to generate the scikit-learn model that will eventually be hosted using the SageMaker Algorithm contained. \n",
-    "\n",
-    "### Install XGboost\n",
-    "Note that for conda based installation, you'll need to change the Notebook kernel to the environment with conda and Python3. "
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "expanded-dress",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "!conda install -y -c conda-forge xgboost==0.90"
+    "These steps are optional and are needed to generate the scikit-learn model that will eventually be hosted using the SageMaker Algorithm contained. \n"
    ]
   },
   {
    "cell_type": "markdown",
-   "id": "little-still",
    "metadata": {},
    "source": [
     "### Fetch the dataset"
@@ -120,7 +92,6 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "injured-crawford",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -129,7 +100,6 @@
   },
   {
    "cell_type": "markdown",
-   "id": "tough-facial",
    "metadata": {},
    "source": [
     "### Prepare the dataset for training"
@@ -138,7 +108,6 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "bright-powder",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -162,7 +131,6 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "wooden-thesis",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -178,7 +146,6 @@
   },
   {
    "cell_type": "markdown",
-   "id": "strong-height",
    "metadata": {},
    "source": [
     "### Train the XGBClassifier"
@@ -187,7 +154,6 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "sought-genome",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -202,7 +168,6 @@
   },
   {
    "cell_type": "markdown",
-   "id": "patient-endorsement",
    "metadata": {},
    "source": [
     "### Save the trained model file\n",
@@ -212,7 +177,6 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "arctic-retail",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -223,7 +187,6 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "fatty-chapel",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -232,7 +195,6 @@
   },
   {
    "cell_type": "markdown",
-   "id": "forced-illustration",
    "metadata": {},
    "source": [
     "## Upload the pre-trained model to S3"
@@ -241,7 +203,6 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "molecular-admission",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -252,7 +213,6 @@
   },
   {
    "cell_type": "markdown",
-   "id": "willing-miami",
    "metadata": {},
    "source": [
     "## Set up hosting for the model\n",
@@ -264,19 +224,17 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "juvenile-glossary",
    "metadata": {},
    "outputs": [],
    "source": [
-    "from sagemaker.amazon.amazon_estimator import get_image_uri\n",
+    "from sagemaker import image_uris\n",
     "\n",
-    "container = get_image_uri(boto3.Session().region_name, \"xgboost\", \"0.90-2\")"
+    "container = image_uris.retrieve(region=boto3.Session().region_name, framework=\"xgboost\", version=\"0.90-2\")"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "competitive-mozambique",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -303,7 +261,6 @@
   },
   {
    "cell_type": "markdown",
-   "id": "announced-affect",
    "metadata": {},
    "source": [
     "### Create endpoint configuration\n",
@@ -314,7 +271,6 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "alike-experience",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -340,7 +296,6 @@
   },
   {
    "cell_type": "markdown",
-   "id": "otherwise-wiring",
    "metadata": {},
    "source": [
     "### Create endpoint\n",
@@ -350,7 +305,6 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "experienced-makeup",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -380,7 +334,6 @@
   },
   {
    "cell_type": "markdown",
-   "id": "specific-sheffield",
    "metadata": {},
    "source": [
     "## Validate the model for use\n",
@@ -390,7 +343,6 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "pediatric-subject",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -399,7 +351,6 @@
   },
   {
    "cell_type": "markdown",
-   "id": "saving-ghost",
    "metadata": {},
    "source": [
     "Lets generate the prediction for a single datapoint. We'll pick one from the test data generated earlier."
@@ -408,7 +359,6 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "polish-laugh",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -423,7 +373,6 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "reported-coalition",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -447,7 +396,6 @@
   },
   {
    "cell_type": "markdown",
-   "id": "pursuant-cemetery",
    "metadata": {},
    "source": [
     "### Post process the output\n",
@@ -457,7 +405,6 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "broken-individual",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -469,7 +416,6 @@
   },
   {
    "cell_type": "markdown",
-   "id": "going-popularity",
    "metadata": {},
    "source": [
     "### (Optional) Delete the Endpoint\n",
@@ -480,7 +426,6 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "foster-steps",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -490,10 +435,11 @@
  ],
  "metadata": {
   "anaconda-cloud": {},
+  "instance_type": "ml.t3.medium",
   "kernelspec": {
-   "display_name": "Environment (conda_anaconda3)",
+   "display_name": "Python 3 (Data Science)",
    "language": "python",
-   "name": "conda_anaconda3"
+   "name": "python3__SAGEMAKER_INTERNAL__arn:aws:sagemaker:us-west-2:236514542706:image/datascience-1.0"
   },
   "language_info": {
    "codemirror_mode": {
@@ -504,10 +450,11 @@
    "mimetype": "text/x-python",
    "name": "python",
    "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3"
+   "pygments_lexer": "ipython3",
+   "version": "3.6.13"
   },
   "notice": "Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.  Licensed under the Apache License, Version 2.0 (the \"License\"). You may not use this file except in compliance with the License. A copy of the License is located at http://aws.amazon.com/apache2.0/ or in the \"license\" file accompanying this file. This file is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License."
  },
  "nbformat": 4,
  "nbformat_minor": 5
-}
\ No newline at end of file
+}
c4b301c88fc1:amazon-sagemaker-examples julkroll$


#!/usr/bin/env bash

# This script shows how to build the Docker image and push it to ECR to be ready for use
# by SageMaker.

# The argument to this script is the image name. This will be used as the image on the local
# machine and combined with the account and region to form the repository name for ECR.
image=$1

if [ "$image" == "" ]
then
    echo "Usage: $0 <image-name>"
    exit 1
fi

chmod +x decision_trees/train
chmod +x decision_trees/serve

# Get the account number associated with the current IAM credentials
account=$(aws sts get-caller-identity --query Account --output text)

if [ $? -ne 0 ]
then
    exit 255
fi

# Get the region defined in the current configuration (default to us-west-2 if none defined)
region=$(aws configure get region)
region=${region:-us-west-2}

fullname="${account}.dkr.ecr.${region}.amazonaws.com/${image}:latest"

# If the repository doesn't exist in ECR, create it.

aws ecr describe-repositories --repository-names "${image}" > /dev/null 2>&1

if [ $? -ne 0 ]
then
    aws ecr create-repository --repository-name "${image}" > /dev/null
fi

# Get the login command from ECR and execute it directly
$(aws ecr get-login --region ${region} --no-include-email)

# Build the docker image locally with the image name and then push it to ECR
# with the full name.

docker build  -t ${image} .
docker tag ${image} ${fullname}

docker push ${fullname}


#!/bin/bash

# List all processes running from streamlit
echo "Processes running from streamlit:"
ps -Al | grep streamlit

# Kill all processes running from streamlit
echo "Killing all processes running from streamlit"
pkill -9 streamlit

# Delete the file temp.txt
rm temp.txt


#!/bin/sh
CURRENTDATE=`date +"%Y-%m-%d %T"`
RED='\033[0;31m'
CYAN='\033[1;36m'
GREEN='\033[1;32m'
NC='\033[0m'
S3_PATH=$1

# Run the Streamlit app and save the output to "temp.txt"
streamlit run app.py > temp.txt &

# Read the text file using cat
echo "Getting the URL to view your Streamlit app in the browser"

# Extract the last four digits of the port number from the Network URL
sleep 5
PORT=$(grep "Network URL" temp.txt | awk -F':' '{print $NF}' | awk '{print $1}' | tail -c 5)
echo -e "${CYAN}${CURRENTDATE}: [INFO]:${NC} Port Number ${PORT}"

# Get Studio domain information
DOMAIN_ID=$(jq .DomainId /opt/ml/metadata/resource-metadata.json || exit 1)
RESOURCE_NAME=$(jq .ResourceName /opt/ml/metadata/resource-metadata.json || exit 1)
RESOURCE_ARN=$(jq .ResourceArn /opt/ml/metadata/resource-metadata.json || exit 1)

# Remove quotes from string
DOMAIN_ID=`sed -e 's/^"//' -e 's/"$//' <<< "$DOMAIN_ID"`
RESOURCE_NAME=`sed -e 's/^"//' -e 's/"$//' <<< "$RESOURCE_NAME"`
RESOURCE_ARN=`sed -e 's/^"//' -e 's/"$//' <<< "$RESOURCE_ARN"`
RESOURCE_ARN_ARRAY=($(echo "$RESOURCE_ARN" | tr ':' '\n'))

# Get Studio domain region
REGION=$(echo "${RESOURCE_ARN_ARRAY[3]}")

# Check if it's Collaborative Space
SPACE_NAME=$(jq .SpaceName /opt/ml/metadata/resource-metadata.json || exit 1)

# if it's not a collaborative space
if [ -z "$SPACE_NAME" ] || [ $SPACE_NAME == "null" ] ;
then
    # If it's a user-profile access
    echo -e "${CYAN}${CURRENTDATE}: [INFO]:${NC} Domain Id ${DOMAIN_ID}"
    STUDIO_URL="https://${DOMAIN_ID}.studio.${REGION}.sagemaker.aws"

# It is a collaborative space
else

    SEM=true
    SPACE_ID=

    # Check if Space Id was previously configured
    if [ -f /tmp/space-metadata.json ]; then
        SAVED_SPACE_ID=$(jq .SpaceId /tmp/space-metadata.json || exit 1)
        SAVED_SPACE_ID=`sed -e 's/^"//' -e 's/"$//' <<< "$SAVED_SPACE_ID"`

        if [ -z "$SAVED_SPACE_ID" ] || [ $SAVED_SPACE_ID == "null" ]; then
            ASK_INPUT=true
        else
            ASK_INPUT=false
        fi
    else
        ASK_INPUT=true
    fi

    # If Space Id is not available, ask for it
    while [[ $SPACE_ID = "" ]] ; do
        # If Space Id already configured, skeep the ask
        if [ "$ASK_INPUT" = true ]; then
            echo -e "${CYAN}${CURRENTDATE}: [INFO]:${NC} Please insert the Space Id from your url. e.g. https://${GREEN}<SPACE_ID>${NC}.studio.${REGION}.sagemaker.aws/jupyter/default/lab"
            read SPACE_ID
            SEM=true
        else
            SPACE_ID=$SAVED_SPACE_ID
        fi

        if ! [ -z "$SPACE_ID" ] && ! [ $SPACE_ID == "null" ] ;
        then
            while $SEM; do
                echo "${SPACE_ID}"
                read -p "Should this be used as Space Id? (y/N) " yn
                case $yn in
                    [Yy]* )
                        echo -e "${CYAN}${CURRENTDATE}: [INFO]:${NC} Domain Id ${DOMAIN_ID}"
                        echo -e "${CYAN}${CURRENTDATE}: [INFO]:${NC} Space Id ${SPACE_ID}"

                        jq -n --arg space_id $SPACE_ID '{"SpaceId":$space_id}' > /tmp/space-metadata.json

                        STUDIO_URL="https://${SPACE_ID}.studio.${REGION}.sagemaker.aws"

                        SEM=false
                        ;;
                    [Nn]* )
                        SPACE_ID=
                        ASK_INPUT=true
                        SEM=false
                        ;;
                    * ) echo "Please answer yes or no.";;
                esac
            done
        fi
    done
fi

echo -e "${CYAN}${CURRENTDATE}: [INFO]:${NC} Studio Url ${STUDIO_URL}"

link="${STUDIO_URL}/jupyter/${RESOURCE_NAME}/proxy/${PORT}/"

echo -e "${CYAN}${CURRENTDATE}: [INFO]:${NC} Starting Streamlit App"
echo -e "${CYAN}${CURRENTDATE}: [INFO]: ${GREEN}${link}${NC}"

exit 0
fi


pip install --no-cache-dir -r requirements.txt
sudo yum install -y iproute
sudo yum install -y jq
sudo yum install -y lsof


#!/bin/bash
# Get all port numbers in use by Streamlit and output the URLs for all running streamlit apps.

CURRENTDATE=`date +"%Y-%m-%d %T"`
RED='\033[0;31m'
CYAN='\033[1;36m'
GREEN='\033[1;32m'
NC='\033[0m'
S3_PATH=$1

# Get Studio domain information
DOMAIN_ID=$(jq .DomainId /opt/ml/metadata/resource-metadata.json || exit 1)
RESOURCE_NAME=$(jq .ResourceName /opt/ml/metadata/resource-metadata.json || exit 1)
RESOURCE_ARN=$(jq .ResourceArn /opt/ml/metadata/resource-metadata.json || exit 1)

# Remove quotes from string
DOMAIN_ID=`sed -e 's/^"//' -e 's/"$//' <<< "$DOMAIN_ID"`
RESOURCE_NAME=`sed -e 's/^"//' -e 's/"$//' <<< "$RESOURCE_NAME"`
RESOURCE_ARN=`sed -e 's/^"//' -e 's/"$//' <<< "$RESOURCE_ARN"`
RESOURCE_ARN_ARRAY=($(echo "$RESOURCE_ARN" | tr ':' '\n'))

# Get Studio domain region
REGION=$(echo "${RESOURCE_ARN_ARRAY[3]}")

# Check if it's Collaborative Space
SPACE_NAME=$(jq .SpaceName /opt/ml/metadata/resource-metadata.json || exit 1)

# Find the process IDs of all running Streamlit instances
streamlit_pids=$(pgrep streamlit)

# Initialize an empty array to hold the port numbers
port_array=()
# Loop through each process ID and find the port number in use
for pid in $streamlit_pids; do
    # Get the port number for the process
    port=$(lsof -a -i -P -n -p $pid | grep LISTEN | awk -F':' '{print $2}' | cut -d ' ' -f 1)\
    # port=$(lsof -a -i -P -n -p $pid | grep ".*:\d\+.*(LISTEN)$" | awk -F':' '{print $2}' | cut -d ' ' -f 1)
    port="${port:0:4}"

    # Add the port number to the array
    port_array+=("$port")

done

echo "These are the Streamlit Apps Currently Running: "
# Loop through the array and print each port number
for PORT in "${port_array[@]}"; do

    # if it's not a collaborative space
    if [ -z "$SPACE_NAME" ] || [ $SPACE_NAME == "null" ] ;
    then
        # If it's a user-profile access
        STUDIO_URL="https://${DOMAIN_ID}.studio.${REGION}.sagemaker.aws"

    # It is a collaborative space
    else

        SEM=true
        SPACE_ID=

        # Check if Space Id was previously configured
        if [ -f /tmp/space-metadata.json ]; then
            SAVED_SPACE_ID=$(jq .SpaceId /tmp/space-metadata.json || exit 1)
            SAVED_SPACE_ID=`sed -e 's/^"//' -e 's/"$//' <<< "$SAVED_SPACE_ID"`

            if [ -z "$SAVED_SPACE_ID" ] || [ $SAVED_SPACE_ID == "null" ]; then
                ASK_INPUT=true
            else
                ASK_INPUT=false
            fi
        else
            ASK_INPUT=true
        fi

        # If Space Id is not available, ask for it
        while [[ $SPACE_ID = "" ]] ; do
            # If Space Id already configured, skeep the ask
            if [ "$ASK_INPUT" = true ]; then
                echo -e "${CYAN}${CURRENTDATE}: [INFO]:${NC} Please insert the Space Id from your url. e.g. https://${GREEN}<SPACE_ID>${NC}.studio.${REGION}.sagemaker.aws/jupyter/default/lab"
                read SPACE_ID
                SEM=true
            else
                SPACE_ID=$SAVED_SPACE_ID
            fi

            if ! [ -z "$SPACE_ID" ] && ! [ $SPACE_ID == "null" ] ;
            then
                while $SEM; do
                    echo "${SPACE_ID}"
                    read -p "Should this be used as Space Id? (y/N) " yn
                    case $yn in
                        [Yy]* )

                            jq -n --arg space_id $SPACE_ID '{"SpaceId":$space_id}' > /tmp/space-metadata.json

                            STUDIO_URL="https://${SPACE_ID}.studio.${REGION}.sagemaker.aws"

                            SEM=false
                            ;;
                        [Nn]* )
                            SPACE_ID=
                            ASK_INPUT=true
                            SEM=false
                            ;;
                        * ) echo "Please answer yes or no.";;
                    esac
                done
            fi
        done
    fi

    link="${STUDIO_URL}/jupyter/${RESOURCE_NAME}/proxy/${PORT}/"

    echo -e "${GREEN}${link}${NC}"
done
exit 0
fi


ACCOUNT_ID=$1
REGION=$2
REPO_NAME=$3

docker build -f ../docker/Dockerfile -t $REPO_NAME ../docker

docker tag $REPO_NAME $ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com/$REPO_NAME:latest

aws ecr get-login-password --region $REGION | docker login --username AWS --password-stdin $ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com

aws ecr describe-repositories --repository-names $REPO_NAME || aws ecr create-repository --repository-name $REPO_NAME

docker push $ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com/$REPO_NAME:latest


#!/bin/bash

payload=$1
content=${2:-text/csv}

curl -d @${payload} -H "Content-Type: ${content}" http://localhost:8080/invocations


#!/bin/bash
# COLLABORATORS: Jeremy Feltracco(jpfelt@), Vidya Sagar Ravipati (ravividy@)

# Immediately stop if any intermediate step fails.
set -e

REPO_ROOT_PATH="$(dirname $0)/.."

PREFIX="smgt-workflow"

echo "[START] LAUNCHING SMGT-WORKFLOW"

if [[ -z "${AWS_PROFILE}" ]];
then
  read -p "Enter your AWS CLI IAM AWS_PROFILE : " AWS_PROFILE
fi

if [[ -z "${COGNITO_POOL_ID}" ]];
then
  read -p "Enter your SageMaker Workforce cognito user pool id: " COGNITO_POOL_ID
fi

if [[ -z "${COGNITO_POOL_CLIENT_ID}" ]];
then
  read -p "Enter your SageMaker Workforce cognito client id: " COGNITO_POOL_CLIENT_ID
fi

stack_name="${PREFIX}-v0"
account=$(aws sts get-caller-identity --query Account --output text --profile ${AWS_PROFILE})
region=$(aws configure get region --profile ${AWS_PROFILE})
echo "We will be launching resources in ${region} in account ${account} with AWS CLI Profile ${AWS_PROFILE}."

DEPLOY_BUCKET_NAME="${PREFIX}-${account}-${region}"
aws s3 mb s3://${DEPLOY_BUCKET_NAME} --profile ${AWS_PROFILE}

 sam build \
     -t main-merged.yml \
     --use-container \
     --skip-pull-image

 sam package \
     --template-file .aws-sam/build/template.yaml \
     --output-template-file main-packaged.yml \
     --s3-bucket ${DEPLOY_BUCKET_NAME} \
     --profile ${AWS_PROFILE}

 sam deploy \
     --template-file main-packaged.yml \
     --stack-name ${stack_name} \
     --region ${region} \
     --tags Project=${stack_name} \
     --s3-bucket ${DEPLOY_BUCKET_NAME} \
     --parameter-overrides \
       ParameterKey=Prefix,ParameterValue=${PREFIX} \
       ParameterKey=CognitoUserPoolId,ParameterValue=${COGNITO_POOL_ID} \
       ParameterKey=CognitoUserPoolClientId,ParameterValue=${COGNITO_POOL_CLIENT_ID} \
     --capabilities CAPABILITY_IAM CAPABILITY_AUTO_EXPAND CAPABILITY_NAMED_IAM \
     --profile ${AWS_PROFILE}

echo "[SUCCESS] DONE BUILDING SMGT-WORKFLOW"


#!/bin/bash

SCRIPT_PATH="`dirname \"$0\"`"
INSTALL_PATH="$SCRIPT_PATH/target/python"

rm -rf $INSTALL_PATH
mkdir -p $INSTALL_PATH
pip3 install -t $INSTALL_PATH -r $SCRIPT_PATH/requirements.txt


#!/bin/bash

SCRIPT_PATH="`dirname \"$0\"`"

git clean -f -x -d $SCRIPT_PATH


#!/bin/bash

sudo -n true
if [ $? -eq 0 ]; then
  echo "The user has root access."
else
  echo "The user does not have root access. Everything required to run the notebook is already installed and setup. We are good to go!"
  exit 0
fi

# Do we have GPU support?
nvidia-smi > /dev/null 2>&1
if [ $? -eq 0 ]; then
  # check if we have nvidia-docker
  NVIDIA_DOCKER=`rpm -qa | grep -c nvidia-docker2`
  if [ $NVIDIA_DOCKER -eq 0 ]; then
    # Install nvidia-docker2
    #sudo pkill -SIGHUP dockerd
    sudo yum -y remove docker
    sudo yum -y install docker-17.09.1ce-1.111.amzn1

    sudo /etc/init.d/docker start

    curl -s -L https://nvidia.github.io/nvidia-docker/amzn1/nvidia-docker.repo | sudo tee /etc/yum.repos.d/nvidia-docker.repo
    sudo yum install -y nvidia-docker2
    sudo cp daemon.json /etc/docker/daemon.json
    sudo pkill -SIGHUP dockerd
    echo "installed nvidia-docker2"
  else
    echo "nvidia-docker2 already installed. We are good to go!"
  fi
fi

# This is common for both GPU and CPU instances

# check if we have docker-compose
docker-compose version >/dev/null 2>&1
if [ $? -ne 0 ]; then
  # install docker compose
  pip install docker-compose
fi

# check if we need to configure our docker interface
SAGEMAKER_NETWORK=`docker network ls | grep -c sagemaker-local`
if [ $SAGEMAKER_NETWORK -eq 0 ]; then
  docker network create --driver bridge sagemaker-local
fi

# Notebook instance Docker networking fixes
RUNNING_ON_NOTEBOOK_INSTANCE=`sudo iptables -S OUTPUT -t nat | grep -c 169.254.0.2`

# Get the Docker Network CIDR and IP for the sagemaker-local docker interface.
SAGEMAKER_INTERFACE=br-`docker network ls | grep sagemaker-local | cut -d' ' -f1`
DOCKER_NET=`ip route | grep $SAGEMAKER_INTERFACE | cut -d" " -f1`
DOCKER_IP=`ip route | grep $SAGEMAKER_INTERFACE | cut -d" " -f12`

# check if both IPTables and the Route Table are OK.
IPTABLES_PATCHED=`sudo iptables -S PREROUTING -t nat | grep -c 169.254.0.2`
ROUTE_TABLE_PATCHED=`sudo ip route show table agent | grep -c $SAGEMAKER_INTERFACE`

if [ $RUNNING_ON_NOTEBOOK_INSTANCE -gt 0 ]; then

  if [ $ROUTE_TABLE_PATCHED -eq 0 ]; then
    # fix routing
    sudo ip route add $DOCKER_NET via $DOCKER_IP dev $SAGEMAKER_INTERFACE table agent
  else
    echo "SageMaker instance route table setup is ok. We are good to go."
  fi

  if [ $IPTABLES_PATCHED -eq 0 ]; then
    sudo iptables -t nat -A PREROUTING  -i $SAGEMAKER_INTERFACE -d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j DNAT --to-destination 169.254.0.2:9081
    echo "iptables for Docker setup done"
  else
    echo "SageMaker instance routing for Docker is ok. We are good to go!"
  fi
fi


#!/bin/bash
source activate rapids
if [[ "$1" == "serve" ]]; then
    echo -e "@ entrypoint -> launching serving script \n"
    python serve.py
else
    echo -e "@ entrypoint -> launching training script \n"
    python train.py
fi


#!/bin/bash

# use this script to save ResNet50 model as a .pt file
python pt_exporter.py

# Optional Scripts
# use this script to convert Pytorch model to ONNX format
# python onnx_exporter.py


#!/bin/bash
python onnx_exporter.py
python pt_exporter.py
trtexec --onnx=model.onnx --saveEngine=model.plan --explicitBatch --minShapes=input:1x3x224x224 --optShapes=input:128x3x224x224 --maxShapes=input:128x3x224x224 --fp16 --verbose | tee conversion.txt


#!/bin/bash

# use this script to save ResNet50 model as a .pt file
#python pt_exporter.py

# Optional Scripts
# use this script to convert Pytorch model to ONNX format
 python onnx_exporter.py

#use this command to generate a model plan that will be used to host SageMaker Endpoint
trtexec --onnx=model.onnx --saveEngine=model.plan --explicitBatch --minShapes=input:1x3x224x224 --optShapes=input:128x3x224x224 --maxShapes=input:128x3x224x224 --fp16 --verbose | tee conversion.txt


#!/bin/bash
python -m pip install transformers==4.26.1
python onnx_exporter.py

trtexec --onnx=model.onnx --saveEngine=model_bs16.plan --minShapes=token_ids:1x128,attn_mask:1x128 --optShapes=token_ids:16x128,attn_mask:16x128 --maxShapes=token_ids:128x128,attn_mask:128x128 --fp16 --verbose --workspace=14000 | tee conversion_bs16_dy.txt


#!/bin/bash

conda create -y -n hf_env python=3.8
source ~/anaconda3/etc/profile.d/conda.sh
source activate hf_env
export PYTHONNOUSERSITE=True
pip install torch --extra-index-url https://download.pytorch.org/whl/cu116
pip install transformers[sentencepiece]
pip install conda-pack
conda-pack


#!/bin/bash

freeMemory=$(awk '/MemFree/ { printf "%.0f \n", $2 }' /proc/meminfo)
totalMemory=$(awk '/MemTotal/ { printf "%.0f \n", $2 }' /proc/meminfo)
availableMemory=$(awk '/MemAvailable/ { printf "%.0f \n", $2 }' /proc/meminfo)

echo freeMemory = $freeMemory
echo availableMemory = $availableMemory
echo totalMemory = $totalMemory

heapMemory=$((availableMemory*80/100))

echo Using 80% of availableMemory = $heapMemory

heapMemoryInMBs=$((heapMemory/1024))M

echo heapMemoryInMBs = $heapMemoryInMBs

echo "[server-startup] Starting java application"

exec java -Xmx$heapMemoryInMBs -Djava.security.egd=file:/dev/./urandom -Dapp.port=8080 -jar /work/app.jar


#!/bin/bash

conda create -y -n preprocessing_env python=3.8
source /opt/conda/etc/profile.d/conda.sh
conda activate preprocessing_env
export PYTHONNOUSERSITE=True
conda install -y -c conda-forge pandas scikit-learn
pip install conda-pack
conda-pack


#!/bin/bash

set -e

sudo -u ec2-user -i <<'EOF'

mkdir -p rapids_kernel
cd rapids_kernel
wget -q https://rapidsai-data.s3.us-east-2.amazonaws.com/conda-pack/rapidsai/rapids21.06_cuda11.0_py3.8.tar.gz
echo "wget completed"
tar -xzf *.gz
echo "unzip completed"
source /home/ec2-user/rapids_kernel/bin/activate
conda-unpack
echo "unpack completed"
python -m ipykernel install --user --name rapids-2106
echo "kernel install completed"

EOF


#!/usr/bin/env bash

# This script shows how to build the Docker image and push it to ECR to be ready for use
# by SageMaker.

# The argument to this script is the image name. This will be used as the image on the local
# machine and combined with the account and region to form the repository name for ECR.
reponame=$1
versiontag=$2
baseimage=$3
regionname=$4
account=$5

if [ "$reponame" == "" ] || [ "$versiontag" == "" ]  || [ "$baseimage" == "" ] || [ "$regionname" == "" ] || [ "$account" == "" ]
then
    echo "Usage: $0 <repo-name> <version-tag> <base-image> <region> <account>"
    exit 1
fi

if [ $? -ne 0 ]
then
    exit 255
fi

fullname="${account}.dkr.ecr.${regionname}.amazonaws.com/${reponame}:${versiontag}"

# If the repository doesn't exist in ECR, create it.
aws ecr describe-repositories --repository-names "${reponame}" > /dev/null 2>&1

if [ $? -ne 0 ]
then
    aws ecr create-repository --repository-name "${reponame}" > /dev/null
fi

# Get the login command from ECR in order to pull down the SageMaker PyTorch image
aws ecr get-login-password --region $regionname | docker login --username AWS --password-stdin ${baseimage}

aws ecr get-login-password --region $regionname | docker login --username AWS --password-stdin ${account}.dkr.ecr."${regionname}".amazonaws.com

# Build the docker image locally with the image name and then push it to ECR
# with the full name.
docker build  -t ${reponame} . --build-arg BASE_IMAGE=${baseimage}
docker tag ${reponame} ${fullname}

# Get the login command from ECR in order to pull down the SageMaker PyTorch image
aws ecr get-login-password --region $regionname | docker login --username AWS --password-stdin ${account}.dkr.ecr."${regionname}".amazonaws.com

docker push ${fullname}
echo "${fullname}"


#!/bin/bash
#  ./patch_ssd.sh s3://my-bucket/my-model-output 5 s3://my-bucket/my-patched-output

# This script requires you to have already cloned the incubator-mxnet project
# to obtain the deploy.py script
#git clone https://github.com/apache/incubator-mxnet

BUCKET_AND_PREFIX=$1 # should enforce that target must contain 'deeplens', else import fails
NUM_CLASSES=$2       # could be extracted from hyperparams.json for more robust implementation
TARGET_PREFIX=$3

# download and unpack the model artifacts
rm -rf tmp && mkdir tmp
aws s3 cp $BUCKET_AND_PREFIX/model.tar.gz tmp
gunzip -k -c tmp/model.tar.gz | tar -C tmp -xopf -
ls -l tmp/*

# copy your parameters file and symbol file into the SSD model directory
mv tmp/*-0000.params tmp/ssd_resnet50_512-0000.params
mv tmp/*-symbol.json tmp/ssd_resnet50_512-symbol.json

# run the deploy Python script, which creates patched versions of
# your parameters and symbol file with a new prefix of "deploy"
python incubator-mxnet/example/ssd/deploy.py --network resnet50 \
  --data-shape 512 --num-class $NUM_CLASSES --prefix tmp/ssd_

# now re-package your updated model artifacts
rm tmp/ssd_*  &&  rm tmp/model.tar.gz
tar -cvzf ./patched_model.tar.gz -C tmp \
  ./deploy_ssd_resnet50_512-0000.params \
  ./deploy_ssd_resnet50_512-symbol.json \
  ./hyperparams.json

# move the new artifacts to S3 for DeepLens model creation. Use a new
# prefix to keep it distinct from the original model artifacts.
aws s3 cp patched_model.tar.gz $TARGET_PREFIX/

# clean up
rm -rf tmp
rm *.gz


#!/bin/bash

sudo -n true
if [ $? -eq 0 ]; then
  echo "The user has root access."
else
  echo "The user does not have root access. Everything required to run the notebook is already installed and setup. We are good to go!"
  exit 0
fi

# Do we have GPU support?
nvidia-smi > /dev/null 2>&1
if [ $? -eq 0 ]; then
  # check if we have nvidia-docker
  NVIDIA_DOCKER=`rpm -qa | grep -c nvidia-docker2`
  if [ $NVIDIA_DOCKER -eq 0 ]; then
    # Install nvidia-docker2
    DOCKER_VERSION=`yum list docker | tail -1 | awk '{print $2}' | head -c 2`

    if [ $DOCKER_VERSION -eq 17 ]; then
      DOCKER_PKG_VERSION='17.09.1ce-1.111.amzn1'
      NVIDIA_DOCKER_PKG_VERSION='2.0.3-1.docker17.09.1.ce.amzn1'
    else
      DOCKER_PKG_VERSION='18.06.1ce-3.17.amzn1'
      NVIDIA_DOCKER_PKG_VERSION='2.0.3-1.docker18.06.1.ce.amzn1'
    fi

    sudo yum -y remove docker
    sudo yum -y install docker-$DOCKER_PKG_VERSION

    sudo /etc/init.d/docker start

    curl -s -L https://nvidia.github.io/nvidia-docker/amzn1/nvidia-docker.repo | sudo tee /etc/yum.repos.d/nvidia-docker.repo
    sudo yum install -y nvidia-docker2-$NVIDIA_DOCKER_PKG_VERSION
    sudo cp daemon.json /etc/docker/daemon.json
    sudo pkill -SIGHUP dockerd
    echo "installed nvidia-docker2"
  else
    echo "nvidia-docker2 already installed. We are good to go!"
  fi
fi

# This is common for both GPU and CPU instances

# check if we have docker-compose
docker-compose version >/dev/null 2>&1
if [ $? -ne 0 ]; then
  # install docker compose
  pip install docker-compose
fi

# check if we need to configure our docker interface
SAGEMAKER_NETWORK=`docker network ls | grep -c sagemaker-local`
if [ $SAGEMAKER_NETWORK -eq 0 ]; then
  docker network create --driver bridge sagemaker-local
fi

# Notebook instance Docker networking fixes
RUNNING_ON_NOTEBOOK_INSTANCE=`sudo iptables -S OUTPUT -t nat | grep -c 169.254.0.2`

# Get the Docker Network CIDR and IP for the sagemaker-local docker interface.
SAGEMAKER_INTERFACE=br-`docker network ls | grep sagemaker-local | cut -d' ' -f1`
DOCKER_NET=`ip route | grep $SAGEMAKER_INTERFACE | cut -d" " -f1`
DOCKER_IP=`ip route | grep $SAGEMAKER_INTERFACE | cut -d" " -f12`

# check if both IPTables and the Route Table are OK.
IPTABLES_PATCHED=`sudo iptables -S PREROUTING -t nat | grep -c $SAGEMAKER_INTERFACE`
ROUTE_TABLE_PATCHED=`sudo ip route show table agent | grep -c $SAGEMAKER_INTERFACE`

if [ $RUNNING_ON_NOTEBOOK_INSTANCE -gt 0 ]; then

  if [ $ROUTE_TABLE_PATCHED -eq 0 ]; then
    # fix routing
    sudo ip route add $DOCKER_NET via $DOCKER_IP dev $SAGEMAKER_INTERFACE table agent
  else
    echo "SageMaker instance route table setup is ok. We are good to go."
  fi

  if [ $IPTABLES_PATCHED -eq 0 ]; then
    sudo iptables -t nat -A PREROUTING  -i $SAGEMAKER_INTERFACE -d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j DNAT --to-destination 169.254.0.2:9081
    echo "iptables for Docker setup done"
  else
    echo "SageMaker instance routing for Docker is ok. We are good to go!"
  fi
fi


#!/bin/sh

echo "Starting the spot int handler"

if [ "${QUEUE_URL}" == "" ]; then
  echo '[ERROR] Environment variable `QUEUE_URL` has no value set.' 1>&2
  exit 1
fi

if [ "${NAMESPACE}" == "" ]; then
  echo '[ERROR] Environment variable `NAMESPACE` has no value set.' 1>&2
  exit 1
fi

if [ "${POD_NAME}" == "" ]; then
  echo '[ERROR] Environment variable `POD_NAME` has no value set.' 1>&2
  exit 1
fi

echo "Getting the node name"
NODE_NAME=$(kubectl --namespace ${NAMESPACE} get pod ${POD_NAME} --output jsonpath="{.spec.nodeName}")
echo NODE_NAME=${NODE_NAME}

if [ "${NODE_NAME}" == "" ]; then
  echo "[ERROR] Unable to fetch the name of the node running the pod \"${POD_NAME}\" in the namespace \"${NAMESPACE}\"." 1>&2
  exit 1
fi

echo "Gather some information"
INSTANCE_ID_URL=${INSTANCE_ID_URL:-http://169.254.169.254/latest/meta-data/instance-id}
INSTANCE_ID=$(curl -s ${INSTANCE_ID_URL})
echo INSTANCE_ID_URL=${INSTANCE_ID_URL}
echo INSTANCE_ID=${INSTANCE_ID}

echo "\`kubectl drain ${NODE_NAME}\` will be executed once a termination notice is made."

POLL_INTERVAL=${POLL_INTERVAL:-5}
NOTICE_URL=${NOTICE_URL:-http://169.254.169.254/latest/meta-data/spot/termination-time}

echo "Polling ${NOTICE_URL} every ${POLL_INTERVAL} second(s)"

while http_status=$(curl -o /dev/null -w '%{http_code}' -sL ${NOTICE_URL}); [ ${http_status} -ne 200 ]; do
  echo $(date): ${http_status}
  sleep ${POLL_INTERVAL}
done

echo $(date): ${http_status}

MESSAGE="[{'status': 'spot termination', 'public_hostname': ${NODE_NAME}, 'public_port': NA, 'region': 'us-west'}]"
MESSAGE_GRP_ID="gsGrp_us-west-2"
aws sqs send-message --queue-url ${QUEUE_URL} --message-body "${MESSAGE}" --message-group-id ${MESSAGE_GRP_ID}

echo "Drain the node."
kubectl drain ${NODE_NAME} --force --ignore-daemonsets

echo "Sleep for 200 seconds to prevent raise condition"
# The instance should be terminated by the end of the sleep assumming 120 sec notification time
sleep 200


#!/bin/bash

source account.conf
region="us-west-2"
repo='.dkr.ecr.'$region'.amazonaws.com/autopilot:latest'
repo_url=$account$repo

$(aws ecr get-login --no-include-email --region $region)
docker build -t autopilot .
docker tag autopilot:latest $repo_url
docker push $repo_url


#!/bin/bash

export QUEUE_URL="https://sqs.us-west-2.amazonaws.com/356566070122/gameserver.fifo"
export NAMESPACE="default"
export POD_NAME="my_pod"
export DEPLOY_NAME="minecraft"
export CURRENT_SIZE_METRIC_NAME="num_of_gs"
export NEW_SIZE_METRIC_NAME="new_size"
export FREQUENCY="10"
export MIN_GS_NUM="10"


#!/bin/sh -x
#source ./export_locally.sh
echo "Starting the game server auto-pilot"

# This process runs as daemon set per game server deployment. It attempt to get the required number of game-servers within a region.
# It gets this number from an ML model. For testing purpose, it avoids the ML endpoint and uses random numbers instead.
# In the case of scale down, individual game server pods should protect itself with a termination grace period to avoid player interruption
# In such cases, an extra x min will be given where terminated pods status will turn to 'terminating'.
# Game servers can listen to SIGKILL or SIGTERM and trigger a game-server drain (this is not k8s thing. it is a game-server logic)

if [ "${DEMAND_URL}" == "" ]; then
  echo '[ERROR] Environment variable `DEMAND_URL` has no value set.' 1>&2
  exit 1
fi

if [ "${MODEL_URL}" == "" ]; then
  echo '[ERROR] Environment variable `MODEL_URL` has no value set.' 1>&2
  exit 1
fi

if [ "${QUEUE_URL}" == "" ]; then
  echo '[ERROR] Environment variable `QUEUE_URL` has no value set.' 1>&2
  exit 1
fi

if [ "${NAMESPACE}" == "" ]; then
  echo '[ERROR] Environment variable `NAMESPACE` has no value set.' 1>&2
  exit 1
fi

if [ "${POD_NAME}" == "" ]; then
  echo '[ERROR] Environment variable `POD_NAME` has no value set.' 1>&2
  exit 1
fi

if [ "${DEPLOY_NAME}" == "" ]; then
  echo '[ERROR] Environment variable `DEPLOY_NAME` has no value set.' 1>&2
  exit 1
fi

if [ "${CURRENT_SIZE_METRIC_NAME}" == "" ]; then
  echo '[ERROR] Environment variable `CURRENT_SIZE_METRIC_NAME` has no value set.' 1>&2
  exit 1
fi

if [ "${NEW_SIZE_METRIC_NAME}" == "" ]; then
  echo '[ERROR] Environment variable `NEW_SIZE_METRIC_NAME` has no value set.' 1>&2
  exit 1
fi

if [ "${FREQUENCY}" == "" ]; then
  echo '[ERROR] Environment variable `FREQUENCY` has no value set.' 1>&2
  exit 1
fi

if [ "${MIN_GS_NUM}" == "" ]; then
  echo '[ERROR] Environment variable `MIN_GS_NUM` has no value set.' 1>&2
  exit 1
fi

if [ "${AWS_DEFAULT_REGION}" == "" ]; then
  echo '[ERROR] Environment variable `AWS_DEFAULT_REGION` has no value set.' 1>&2
  exit 1
fi

while true
do

  #Testing only - generating random size for replica set. Realworld solution will get this number from an AI/ML endpoint
  #NEW_RS_SIZE=$RANDOM
  #RANGE=30
  #let "NEW_RS_SIZE %= $RANGE"
  #echo NEW_RS_SIZE=${NEW_RS_SIZE}
  #End testing section

  #Getting predictions from a trained model
  TMP_PREDICTION_JSON="/tmp/prediction.json"
  DEMAND_URL=${DEMAND_URL}
  MODEL_URL=${MODEL_URL}
  MODEL_URL_PARAM="/"
  `curl -w "\n" $MODEL_URL/$MODEL_URL_PARAM > $TMP_PREDICTION_JSON`
  NEW_RS_SIZE_FLOAT=`cat $TMP_PREDICTION_JSON | jq '.Prediction.num_of_gameservers'`
  IS_FALSE_POSITIVE=`cat $TMP_PREDICTION_JSON | jq '.Prediction.is_false_positive'`
  if [ -n "${NEW_RS_SIZE_FLOAT}" ]; then
    NEW_RS_SIZE=${NEW_RS_SIZE_FLOAT%.*}
  else
    echo "api returned null, setting rs default="${MIN_GS_NUM}
    NEW_RS_SIZE=${MIN_GS_NUM}
  fi
  echo NEW_RS_SIZE=${NEW_RS_SIZE}
  CURRENT_RS_SIZE=`kubectl get deploy ${DEPLOY_NAME} -n ${NAMESPACE} -o=jsonpath='{.status.availableReplicas}'`
  echo CURRENT_RS_SIZE=${CURRENT_RS_SIZE}

  kubectl scale deploy/${DEPLOY_NAME} --replicas=${NEW_RS_SIZE} -n ${NAMESPACE}
  echo "sleeping for ${SLEEP_TIME_B4_NEXT_READ} to allow the scale operations"

  NUM_NODES=`kubectl get nodes -o json | jq '.items[].metadata.labels'| grep ${NODE_GROUP} | wc -l`
  echo NUM_NODES=${NUM_NODES}

  TIME_ST="$(date '+%Y-%m-%d %H:%M:%S')"
  echo TIMESTAMP=$TIME_ST

  MESSAGE="[{'type': 'autopilot','timestamp':${TIME_ST},'deployment':${DEPLOY_NAME},'current_rs_size': ${CURRENT_RS_SIZE},'new_rs_size':${NEW_RS_SIZE},'region':${AWS_DEFAULT_REGION}}]"
  CURRENT_DEMAND=`curl -w "\n" $DEMAND_URL | jq '.Prediction.num_of_gameservers'`
  echo CURRENT_DEMAND=${CURRENT_DEMAND}
  #aws sqs send-message --queue-url ${QUEUE_URL} --message-body "${MESSAGE}"
  aws cloudwatch put-metric-data --metric-name ${CURRENT_SIZE_METRIC_NAME} --namespace ${DEPLOY_NAME} --value ${CURRENT_RS_SIZE}
  #aws cloudwatch put-metric-data --metric-name ${NEW_SIZE_METRIC_NAME} --namespace ${DEPLOY_NAME} --value ${NEW_RS_SIZE}
  aws cloudwatch put-metric-data --metric-name current_gs_demand --namespace ${DEPLOY_NAME} --value ${CURRENT_DEMAND}
  aws cloudwatch put-metric-data --metric-name num_of_nodes --namespace ${DEPLOY_NAME} --value ${NUM_NODES}
  aws cloudwatch put-metric-data --metric-name false_positive --namespace ${DEPLOY_NAME} --value ${IS_FALSE_POSITIVE}

  sleep ${FREQUENCY}

done


#!/bin/sh

echo "Creating docker image repositories"
aws cloudformation create-stack --stack-name ecr-repos --template-body file://./ecr-repos.json

echo "Creating DynamoDB table latest-observations"
aws cloudformation create-stack --stack-name latest-observations-dynamodb-table --template-body file://./latest_observations-table.json

echo "Waiting for the DynamoDB table latest-observations to be created"
aws dynamodb wait table-exists --table-name latest_observations

echo "Initializing latest-observations"
aws dynamodb put-item --table-name latest_observations --item file://./init_observation.json


#!/bin/sh

sudo curl --silent --location -o /usr/local/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/v1.13.7/bin/linux/amd64/kubectl

sudo chmod +x /usr/local/bin/kubectl

sudo yum -y install jq gettext

for command in kubectl jq envsubst
  do
    which $command &>/dev/null && echo "$command in path" || echo "$command NOT FOUND"
  done

ssh-keygen -f ~/.ssh/id_rsa -t rsa -N ''

curl --silent --location "https://github.com/weaveworks/eksctl/releases/download/latest_release/eksctl_$(uname -s)_amd64.tar.gz" | tar xz -C /tmp

sudo mv -v /tmp/eksctl /usr/local/bin

eksctl version

rm -vf ${HOME}/.aws/credentials

export ACCOUNT_ID=$(aws sts get-caller-identity --output text --query Account)
export AWS_REGION=$(curl -s 169.254.169.254/latest/dynamic/instance-identity/document | jq -r '.region')

echo "export ACCOUNT_ID=${ACCOUNT_ID}" >> ~/.bash_profile
echo "export AWS_REGION=${AWS_REGION}" >> ~/.bash_profile
aws configure set default.region ${AWS_REGION}
aws configure get default.region


#!/bin/bash

source account.conf
repo='.dkr.ecr.us-west-2.amazonaws.com/minecraft-server:latest'
repo_url=$account$repo

$(aws ecr get-login --no-include-email --region us-west-2)
docker build -t minecraft-server .
docker tag minecraft-server:latest $repo_url
docker push $repo_url


#!/bin/sh

asg_name=`aws autoscaling describe-auto-scaling-groups| jq '.AutoScalingGroups[].AutoScalingGroupName' | grep rl-gs-autopilot`
aws autoscaling update-auto-scaling-group --auto-scaling-group-name $asg_name --mixed-instances-policy file://./mixed-instance-policy.json


#!/bin/bash

#region=us-east-1
region=us-west-2
source account.conf
repo='.dkr.ecr.'$region'.amazonaws.com/spot-sig-handler:latest'
repo_url=$account$repo

$(aws ecr get-login --no-include-email --region $region)
docker build -t spot-sig-handler .
docker tag spot-sig-handler:latest $repo_url
docker push $repo_url


#!/bin/sh

echo "Starting the spot int handler"

if [ "${QUEUE_URL}" == "" ]; then
  echo '[ERROR] Environment variable `QUEUE_URL` has no value set.' 1>&2
  exit 1
fi

if [ "${NAMESPACE}" == "" ]; then
  echo '[ERROR] Environment variable `NAMESPACE` has no value set.' 1>&2
  exit 1
fi

if [ "${POD_NAME}" == "" ]; then
  echo '[ERROR] Environment variable `POD_NAME` has no value set.' 1>&2
  exit 1
fi

echo "Getting the node name"
NODE_NAME=$(kubectl --namespace ${NAMESPACE} get pod ${POD_NAME} --output jsonpath="{.spec.nodeName}")
echo NODE_NAME=${NODE_NAME}

if [ "${NODE_NAME}" == "" ]; then
  echo "[ERROR] Unable to fetch the name of the node running the pod \"${POD_NAME}\" in the namespace \"${NAMESPACE}\"." 1>&2
  exit 1
fi

echo "Gather some information"
INSTANCE_ID_URL=${INSTANCE_ID_URL:-http://169.254.169.254/latest/meta-data/instance-id}
INSTANCE_ID=$(curl -s ${INSTANCE_ID_URL})
echo INSTANCE_ID_URL=${INSTANCE_ID_URL}
echo INSTANCE_ID=${INSTANCE_ID}

INSTANCE_TYPE=$(kubectl get nodes ${NODE_NAME} -L beta.kubernetes.io/instance-type| grep -v INSTANCE-TYPE| awk '{print $NF}')
echo INSTANCE_TYPE=$INSTANCE_TYPE

INSTANCE_AZ=$(kubectl get nodes ${NODE_NAME} -L failure-domain.beta.kubernetes.io/zone| grep -v ZONE| awk '{print $NF}')
echo INSTANCE_AZ=$INSTANCE_AZ

INSTANCE_GROUP=$(kubectl get nodes ${NODE_NAME} -L alpha.eksctl.io/nodegroup-name| grep -v NODEGROUP-NAME| awk '{print $NF}')
echo INSTANCE_GROUP=$INSTANCE_GROUP

TIME_ST="$(date '+%Y-%m-%d %H:%M:%S')"
echo TIMESTAMP=$TIME_ST

echo "\`kubectl drain ${NODE_NAME}\` will be executed once a termination notice is made."

POLL_INTERVAL=${POLL_INTERVAL:-5}
NOTICE_URL=${NOTICE_URL:-http://169.254.169.254/latest/meta-data/spot/termination-time}

#echo "Send test message to queue ${QUEUE_URL} with message ${MESSAGE}"
#MESSAGE="[{'status': 'test spot termination', 'public_hostname': ${NODE_NAME}, 'public_port': NA, 'region': 'us-east'}]"
#MESSAGE_GRP_ID="gsGrp_us-east-1"
#aws sqs send-message --queue-url ${QUEUE_URL} --message-body "${MESSAGE}" --message-group-id ${MESSAGE_GRP_ID}

echo "Polling ${NOTICE_URL} every ${POLL_INTERVAL} second(s)"
while http_status=$(curl -o /dev/null -w '%{http_code}' -sL ${NOTICE_URL}); [ ${http_status} -ne 200 ]; do
  echo $(date): ${http_status}
  sleep ${POLL_INTERVAL}
done

echo $(date): ${http_status}

MESSAGE="[{'status': 'spot termination','instance_az': ${INSTANCE_AZ},'instance_group':${INSTANCE_GROUP},'time':${TIME_ST}, 'instance_type': ${INSTANCE_TYPE}, 'public_hostname': ${NODE_NAME}, 'public_port': NA, 'region': ${AWS_DEFAULT_REGION}]"
echo MESSAGE=$MESSAGE
aws sqs send-message --queue-url ${QUEUE_URL} --message-body "${MESSAGE}"

echo "Drain the node."
kubectl drain ${NODE_NAME} --force --ignore-daemonsets

echo "Sleep for 200 seconds to prevent raise condition"
# The instance should be terminated by the end of the sleep assumming 120 sec notification time
sleep 200


#!/usr/bin/env bash

if [ $1 == 'train' ]
then
    # Remove all nvidia gl libraries if they exists to run training in SageMaker.
    rm -rf /usr/local/nvidia/lib/libGL*
    rm -rf /usr/local/nvidia/lib/libEGL*
    rm -rf /usr/local/nvidia/lib/libOpenGL*
    rm -rf /usr/local/nvidia/lib64/libGL*
    rm -rf /usr/local/nvidia/lib64/libEGL*
    rm -rf /usr/local/nvidia/lib64/libOpenGL*

    CURRENT_HOST=$(jq .current_host  /opt/ml/input/config/resourceconfig.json)

    sed -ie "s/PLACEHOLDER_HOSTNAME/$CURRENT_HOST/g" /changehostname.c

    gcc -o /changehostname.o -c -fPIC -Wall /changehostname.c
    gcc -o /libchangehostname.so -shared -export-dynamic /changehostname.o -ldl

    LD_PRELOAD=/libchangehostname.so xvfb-run --auto-servernum -s "-screen 0 1400x900x24" train
elif [ $1 == 'serve' ]
then
    serve
fi


#!/usr/bin/env bash

sed -ie "s/PLACEHOLDER_HOSTNAME/$1/g" /changehostname.c

gcc -o /changehostname.o -c -fPIC -Wall /changehostname.c
gcc -o /libchangehostname.so -shared -export-dynamic /changehostname.o -ldl


#!/usr/bin/env bash

# This script shows how to build the Docker image and push it to ECR to be ready for use
# by SageMaker.

# The argument to this script is the image name. This will be used as the image on the local
# machine and combined with the account and region to form the repository name for ECR.
image=$1

if [ "$image" == "" ]
then
    echo "Usage: $0 <image-name>"
    exit 1
fi

chmod +x blazing_text/serve

# Get the account number associated with the current IAM credentials
account=$(aws sts get-caller-identity --query Account --output text)

if [ $? -ne 0 ]
then
    exit 255
fi

# Get the region defined in the current configuration (default to us-west-2 if none defined)
region=$(aws configure get region)
region=${region:-us-west-2}

fullname="${account}.dkr.ecr.${region}.amazonaws.com/${image}:latest"

# If the repository doesn't exist in ECR, create it.

aws ecr describe-repositories --repository-names "${image}" > /dev/null 2>&1

if [ $? -ne 0 ]
then
    aws ecr create-repository --repository-name "${image}" > /dev/null
fi

# Get the login command from ECR and execute it directly
aws ecr get-login-password --region "${region}" | docker login --username AWS --password-stdin "${account}".dkr.ecr."${region}".amazonaws.com

# Build the docker image locally with the image name and then push it to ECR
# with the full name.

docker build  -t ${image} .
docker tag ${image} ${fullname}

docker push ${fullname}


#!/bin/bash

payload=$1
content=${2:-text/csv}

curl --data-binary @${payload} -H "Content-Type: ${content}" -v http://localhost:8080/invocations


#!/bin/sh

image=$1

docker run -v $(pwd)/test_dir:/opt/ml -p 8080:8080 --rm ${image} serve


docker build -t test:latest .
python local_test/test_container.py


# get my account name

account=$(aws sts get-caller-identity --query Account --output text)

# Get the region defined in the current configuration (default to us-west-2 if none defined)
region=$(aws configure get region)

# authenticate my account
aws ecr get-login-password --region ${region} | docker login --username AWS \
    --password-stdin ${account}.dkr.ecr.${region}.amazonaws.com

# create a repo in my ecr called test
aws ecr describe-repositories --repository-names "test" > /dev/null 2>&1

if [ $? -ne 0 ]
then
    aws ecr create-repository --repository-name "test" > /dev/null
fi

# tag the test image with account name
fullname="${account}.dkr.ecr.${region}.amazonaws.com/test:latest"
docker tag test:latest ${fullname}
docker push ${fullname}


#!/usr/bin/env bash

# This script shows how to build the Docker image and push it to ECR to be ready for use
# by SageMaker.

# The argument to this script is the image name. This will be used as the image on the local
# machine and combined with the account and region to form the repository name for ECR.
image=$1

if [ "$image" == "" ]
then
    echo "Usage: $0 <image-name>"
    exit 1
fi

chmod +x decision_trees/train
chmod +x decision_trees/serve

# Get the account number associated with the current IAM credentials
account=$(aws sts get-caller-identity --query Account --output text)

if [ $? -ne 0 ]
then
    exit 255
fi

# Get the region defined in the current configuration (default to us-west-2 if none defined)
region=$(aws configure get region)
region=${region:-us-west-2}

fullname="${account}.dkr.ecr.${region}.amazonaws.com/${image}:latest"

# If the repository doesn't exist in ECR, create it.

aws ecr describe-repositories --repository-names "${image}" > /dev/null 2>&1

if [ $? -ne 0 ]
then
    aws ecr create-repository --repository-name "${image}" > /dev/null
fi

# Get the login command from ECR and execute it directly
aws ecr get-login-password --region "${region}" | docker login --username AWS --password-stdin "${account}".dkr.ecr."${region}".amazonaws.com

# Build the docker image locally with the image name and then push it to ECR
# with the full name.

docker build  -t ${image} .
docker tag ${image} ${fullname}

docker push ${fullname}


#!/bin/sh

image=$1

mkdir -p test_dir/model
mkdir -p test_dir/output

rm test_dir/model/*
rm test_dir/output/*

docker run -v $(pwd)/test_dir:/opt/ml --rm ${image} train


# Â© 2021 Amazon Web Services, Inc. or its affiliates. All Rights Reserved.
#
# This AWS Content is provided subject to the terms of the AWS Customer Agreement
# available at http://aws.amazon.com/agreement or other written agreement between
# Customer and either Amazon Web Services, Inc. or Amazon Web Services EMEA SARL or both.

#!/bin/sh

REPO=$1
S3_BUCKET_NAME=$2

NAME=sourcedir
PUSH=true

if [ -z ${REPO} ] ;
then
    echo "Repository not specified"
    exit 1
fi

SCRIPT=$(readlink -f "$0")
# Absolute path this script is in, thus /home/user/bin
SCRIPTPATH=$(dirname "$SCRIPT")

cd $SCRIPTPATH/$REPO/src
tar --exclude='data' -czvf ${NAME}.tar.gz *

rm -rf ../dist/$REPO
mkdir ../dist
mkdir ../dist/$REPO

mv ${NAME}.tar.gz ../dist/$REPO

if [ -z ${S3_BUCKET_NAME} ] ;
then
  echo "S3 Bucket not specified, no upload"
  PUSH=false
fi

if $PUSH ;
then
  echo "Uploading s3://${S3_BUCKET_NAME}/artifact/${REPO}/${NAME}.tar.gz"

  aws s3 cp ../dist/${REPO}/${NAME}.tar.gz s3://${S3_BUCKET_NAME}/artifact/${REPO}/${NAME}.tar.gz
else
  exit 1
fi


#!/bin/sh

# The name of our algorithm
repo=$1
registry_name=$2
image_tag=$3
docker_file=$4
platforms=$5

echo "[INFO]: registry_name=${registry_name}"
echo "[INFO]: image_tag=${image_tag}"
echo "[INFO]: docker_file=${docker_file}"
echo "[INFO]: platforms=${platforms}"

cd $repo

account=$(aws sts get-caller-identity --query Account --output text)

# Get the region defined in the current configuration (default to us-west-2 if none defined)
region=$(aws configure get region)

echo "[INFO]: Region ${region}"

fullname="${account}.dkr.ecr.${region}.amazonaws.com/${registry_name}:${image_tag}"

echo "[INFO]: Image name: ${fullname}"

# If the repository doesn't exist in ECR, create it.

aws ecr describe-repositories --repository-names "${registry_name}" > /dev/null 2>&1

aws ecr create-repository --repository-name "${registry_name}" > /dev/null

## If you are extending Amazon SageMaker Images, you need to login to the account
# Get the login command from ECR and execute it directly
password=$(aws ecr --region ${region} get-login-password)

docker login -u AWS -p ${password} "${account}.dkr.ecr.${region}.amazonaws.com"

if [ -z ${platforms} ]
then
  docker build -t ${fullname} -f ${docker_file} .
else
  echo "Provided platform = ${platforms}"
  docker build -t ${fullname} -f ${docker_file} . --platform=${platforms}
fi

docker push ${fullname}


#!/bin/bash

sudo -n true
if [ $? -eq 0 ]; then
  echo "The user has root access."
else
  echo "The user does not have root access. Everything required to run the notebook is already installed and setup. We are good to go!"
  exit 0
fi

# Do we have GPU support?
nvidia-smi > /dev/null 2>&1
if [ $? -eq 0 ]; then
  # check if we have nvidia-docker
  NVIDIA_DOCKER=`rpm -qa | grep -c nvidia-docker2`
  if [ $NVIDIA_DOCKER -eq 0 ]; then
    # Install nvidia-docker2
    DOCKER_VERSION=`yum list docker | tail -1 | awk '{print $2}' | head -c 2`

    if [ $DOCKER_VERSION -eq 17 ]; then
      DOCKER_PKG_VERSION='17.09.1ce-1.111.amzn1'
      NVIDIA_DOCKER_PKG_VERSION='2.0.3-1.docker17.09.1.ce.amzn1'
    else
      DOCKER_PKG_VERSION='18.06.1ce-3.17.amzn1'
      NVIDIA_DOCKER_PKG_VERSION='2.0.3-1.docker18.06.1.ce.amzn1'
    fi

    sudo yum -y remove docker
    sudo yum -y install docker-$DOCKER_PKG_VERSION

    sudo /etc/init.d/docker start

    curl -s -L https://nvidia.github.io/nvidia-docker/amzn1/nvidia-docker.repo | sudo tee /etc/yum.repos.d/nvidia-docker.repo
    sudo yum install -y nvidia-docker2-$NVIDIA_DOCKER_PKG_VERSION
    sudo cp daemon.json /etc/docker/daemon.json
    sudo pkill -SIGHUP dockerd
    echo "installed nvidia-docker2"
  else
    echo "nvidia-docker2 already installed. We are good to go!"
  fi
fi

# This is common for both GPU and CPU instances

# check if we have docker-compose
docker-compose version >/dev/null 2>&1
if [ $? -ne 0 ]; then
  # install docker compose
  pip install docker-compose
fi

# check if we need to configure our docker interface
SAGEMAKER_NETWORK=`docker network ls | grep -c sagemaker-local`
if [ $SAGEMAKER_NETWORK -eq 0 ]; then
  docker network create --driver bridge sagemaker-local
fi

# Notebook instance Docker networking fixes
RUNNING_ON_NOTEBOOK_INSTANCE=`sudo iptables -S OUTPUT -t nat | grep -c 169.254.0.2`

# Get the Docker Network CIDR and IP for the sagemaker-local docker interface.
SAGEMAKER_INTERFACE=br-`docker network ls | grep sagemaker-local | cut -d' ' -f1`
DOCKER_NET=`ip route | grep $SAGEMAKER_INTERFACE | cut -d" " -f1`
DOCKER_IP=`ip route | grep $SAGEMAKER_INTERFACE | cut -d" " -f12`

# check if both IPTables and the Route Table are OK.
IPTABLES_PATCHED=`sudo iptables -S PREROUTING -t nat | grep -c $SAGEMAKER_INTERFACE`
ROUTE_TABLE_PATCHED=`sudo ip route show table agent | grep -c $SAGEMAKER_INTERFACE`

if [ $RUNNING_ON_NOTEBOOK_INSTANCE -gt 0 ]; then

  if [ $ROUTE_TABLE_PATCHED -eq 0 ]; then
    # fix routing
    sudo ip route add $DOCKER_NET via $DOCKER_IP dev $SAGEMAKER_INTERFACE table agent
  else
    echo "SageMaker instance route table setup is ok. We are good to go."
  fi

  if [ $IPTABLES_PATCHED -eq 0 ]; then
    sudo iptables -t nat -A PREROUTING  -i $SAGEMAKER_INTERFACE -d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j DNAT --to-destination 169.254.0.2:9081
    echo "iptables for Docker setup done"
  else
    echo "SageMaker instance routing for Docker is ok. We are good to go!"
  fi
fi