echo nix/shell.sh: Entering a devShell
export SRC_ROOT=$(pwd)
export PATH=$SRC_ROOT/build/bin:$PATH

banner()
{
    echo "+---------------------------------------------------------+"
    printf "| %-55s |\n" "$1"
    echo "+---------------------------------------------------------+"
}

function clean {
    banner "Cleanup up ./build"
    rm -rf ./build
}

function configure {
    banner "Configuring with cmake"
    cmake -S . -B./build \
          -DBUILD_TESTING=ON \
          -DS2N_INTEG_TESTS=ON \
          -DS2N_INSTALL_S2NC_S2ND=ON \
          -DS2N_INTEG_NIX=ON \
          -DBUILD_SHARED_LIBS=ON \
          -DCMAKE_BUILD_TYPE=RelWithDebInfo
}

function build {
    banner "Running Build"
    javac tests/integrationv2/bin/SSLSocketClient.java
    cmake --build ./build -j $(nproc)
}

function unit {
    if [[ -z "$1" ]]; then
        (cd $SRC_ROOT/build; ctest -L unit -j $(nproc) --verbose)
    else
        (cd $SRC_ROOT/build; ctest -L unit -R $1 -j $(nproc) --verbose)
    fi
}

function integ {
    if [ "$1" == "help" ]; then
        echo "The following tests are not supported:"
        echo " - cross_compatibility"
        echo "    This test depends on s2nc_head and s2nd_head. To run"
        echo "    the test build s2n-tls from the main branch on github."
        echo "    Change the names of s2n[cd] to s2n[cd]_head and add those"
        echo "    binaries to \$PATH."
        echo "- renegotiate_apache"
        echo "   This test requires apache to be running. See codebuild/bin/s2n_apache.sh"
        echo "    for more info."
        return
    fi
    if [[ -z "$1" ]]; then
        banner "Running all integ tests except cross_compatibility, renegotiate_apache."
        (cd $SRC_ROOT/build; ctest -L integrationv2 -E "(integrationv2_cross_compatibility|integrationv2_renegotiate_apache)" --verbose)
    else
        banner "Warning: cross_compatibility & renegotiate_apache are not supported in nix for various reasons integ help for more info."
        for test in $@; do
            ctest --test-dir ./build -L integrationv2 --no-tests=error --output-on-failure -R "$test" --verbose
            if [ "$?" -ne 0 ]; then
               echo "Test failed, stopping execution"
               exit 1
            fi
        done
    fi
}

function check-clang-format {
    banner "Dry run of clang-format"
    (cd $SRC_ROOT;
    include_regex=".*\.(c|h)$";
    src_files=`find ./api -name .git -prune -o -regextype posix-egrep -regex "$include_regex" -print`;
    src_files+=" ";
    src_files+=`find ./bin -name .git -prune -o -regextype posix-egrep -regex "$include_regex" -print`;
    src_files+=" ";
    src_files+=`find ./crypto -name .git -prune -o -regextype posix-egrep -regex "$include_regex" -print`;
    src_files+=" ";
    src_files+=`find ./stuffer -name .git -prune -o -regextype posix-egrep -regex "$include_regex" -print`;
    src_files+=" ";
    src_files+=`find ./error -name .git -prune -o -regextype posix-egrep -regex "$include_regex" -print`;
    src_files+=" ";
    src_files+=`find ./tls -name .git -prune -o -regextype posix-egrep -regex "$include_regex" -print`;
    src_files+=" ";
    src_files+=`find ./utils -name .git -prune -o -regextype posix-egrep -regex "$include_regex" -print`;
    src_files+=" ";
    src_files+=`find ./tests/unit -name .git -prune -o -regextype posix-egrep -regex "$include_regex" -print`;
    src_files+=" ";
    src_files+=`find ./tests/testlib -name .git -prune -o -regextype posix-egrep -regex "$include_regex" -print`;
    echo $src_files | xargs -n 1 -P $(nproc) clang-format --dry-run -style=file)
}
function do-clang-format {
    banner "In place clang-format"
    (cd $SRC_ROOT;
    include_regex=".*\.(c|h)$";
    src_files=`find ./api -name .git -prune -o -regextype posix-egrep -regex "$include_regex" -print`;
    src_files+=" ";
    src_files+=`find ./bin -name .git -prune -o -regextype posix-egrep -regex "$include_regex" -print`;
    src_files+=" ";
    src_files+=`find ./crypto -name .git -prune -o -regextype posix-egrep -regex "$include_regex" -print`;
    src_files+=" ";
    src_files+=`find ./stuffer -name .git -prune -o -regextype posix-egrep -regex "$include_regex" -print`;
    src_files+=" ";
    src_files+=`find ./error -name .git -prune -o -regextype posix-egrep -regex "$include_regex" -print`;
    src_files+=" ";
    src_files+=`find ./tls -name .git -prune -o -regextype posix-egrep -regex "$include_regex" -print`;
    src_files+=" ";
    src_files+=`find ./utils -name .git -prune -o -regextype posix-egrep -regex "$include_regex" -print`;
    src_files+=" ";
    src_files+=`find ./tests/unit -name .git -prune -o -regextype posix-egrep -regex "$include_regex" -print`;
    src_files+=" ";
    src_files+=`find ./tests/testlib -name .git -prune -o -regextype posix-egrep -regex "$include_regex" -print`;
    echo $src_files | xargs -n 1 -P $(nproc) clang-format -style=file -i)
}

function test_toolchain_counts {
    # This is a starting point for a unit test of the devShell.
    # The choosen S2N_LIBCRYPTO should be 2, and the others should be zero.
    banner "Checking the CMAKE_INCLUDE_PATH for libcrypto counts"
    echo $CMAKE_INCLUDE_PATH|gawk 'BEGIN{RS=":"; o10=0; o11=0; o3=0;awslc=0;libre=0}
      /openssl-3.0/{o3++}
      /openssl-1.1/{o11++}
      /openssl-1.0/{o10++}
      /aws-lc/{awslc++}
      /libressl/{libre++}
      END{print "\nOpenssl3:\t",o3,"\nOpenssl1.1:\t",o11,"\nOpenssl1.0.2:\t",o10,"\nAwlc:\t\t",awslc,"\nLibreSSL:\t", libre}'
    banner "Checking tooling counts (these should all be 1)"
    echo -e "\nOpenssl integ:\t $(openssl version|grep -c '1.1.1')"
    echo -e "Corretto 17:\t $(java -version 2>&1|grep -ce 'Runtime.*Corretto-17')"
    echo -e "gnutls-cli:\t $(gnutls-cli --version |grep -c 'gnutls-cli 3.7')"
    echo -e "gnutls-serv:\t $(gnutls-serv --version |grep -c 'gnutls-serv 3.7')"
    echo -e "Nix Python:\t $(which python|grep -c '/nix/store')"
    echo -e "Nix pytest:\t $(which pytest|grep -c '/nix/store')"
    echo -e "Nix sslyze:\t $(which sslyze|grep -c '/nix/store')"
    echo -e "python nassl:\t $(pip freeze|grep -c 'nassl')"
}

function test_nonstandard_compilation {
    # Any script that needs to compile s2n in a non-standard way can run here
    ./codebuild/bin/test_dynamic_load.sh $(mktemp -d)
}


#!/bin/bash

sudo apt-get install python3
sudo apt-get install python3-pip

pip3 install --user cryptography
pip3 install --user pycryptodome
pip3 install --user pycryptodomex

python3 aes_scram.py


#!/usr/bin/env bash

# Idempotently applies a codemod script to update safety macro calls to include the
# context in which they're used.

set -e

SCRIPT_DIR=scripts/s2n_safety_explicit_context

if [ ! -d $SCRIPT_DIR/.venv ]; then
  python3 -m venv $SCRIPT_DIR/.venv
fi

source $SCRIPT_DIR/.venv/bin/activate
pip install -r $SCRIPT_DIR/requirements.txt

set -x

function join { local IFS=","; echo "$*"; }

IGNORES=$(
    join \
        api \
        cmake \
        codebuild \
        coverage \
        docker-images \
        docs \
        error \
        lib \
        libcrypto-build \
        libcrypto-root \
        scram \
        scripts \
        utils/s2n_safety.h \
        utils/s2n_safety_macros.h \
)

function mod {
  codemod \
    --exclude-paths $IGNORES \
    --extensions c,h,patch \
    "$1" "$2"
}

function map {
  mod "( +)$1 *\(" "\1$2("
  mod "^$1 *\(" "$2("
}

# PTR
map BAIL_PTR               PTR_BAIL
map S2N_ERROR_PTR          PTR_BAIL
map ENSURE_PTR             PTR_ENSURE
map ENSURE_REF_PTR         PTR_ENSURE_REF
map ENSURE_MUT_PTR         PTR_ENSURE_MUT
map GUARD_PTR              PTR_GUARD_POSIX
map GUARD_POSIX_PTR        PTR_GUARD_POSIX
map GUARD_NONNULL_PTR      PTR_GUARD_NONNULL
map GUARD_RESULT_PTR       PTR_GUARD_RESULT
map notnull_check_ptr      PTR_ENSURE_REF

# POSIX
map BAIL_POSIX             POSIX_BAIL
map S2N_ERROR              POSIX_BAIL
map ENSURE_POSIX           POSIX_ENSURE
map ENSURE_POSIX_REF       POSIX_ENSURE_REF
map ENSURE_POSIX_MUT       POSIX_ENSURE_MUT
map notnull_check          POSIX_ENSURE_REF
map PRECONDITION_POSIX     POSIX_PRECONDITION
map POSTCONDITION_POSIX    POSIX_POSTCONDITION
map GUARD                  POSIX_GUARD
map GUARD_NONNULL          POSIX_GUARD_PTR
map GUARD_POSIX_NONNULL    POSIX_GUARD_PTR
map GUARD_OSSL             POSIX_GUARD_OSSL
map GUARD_POSIX_OSSL       POSIX_GUARD_OSSL
map GUARD_AS_POSIX         POSIX_GUARD_RESULT
map GUARD_POSIX            POSIX_GUARD
map GUARD_POSIX_STRICT     POSIX_GUARD_STRICT
map gte_check              POSIX_ENSURE_GTE
map lte_check              POSIX_ENSURE_LTE
map gt_check               POSIX_ENSURE_GT
map lt_check               POSIX_ENSURE_LT
map eq_check               POSIX_ENSURE_EQ
map ne_check               POSIX_ENSURE_NE
map inclusive_range_check  POSIX_ENSURE_INCLUSIVE_RANGE
map exclusive_range_check  POSIX_ENSURE_EXCLUSIVE_RANGE
map memcpy_check           POSIX_CHECKED_MEMCPY
map memset_check           POSIX_CHECKED_MEMSET

# GOTO
map GUARD_GOTO             GOTO_GUARD_POSIX
map GUARD_POSIX_GOTO       GOTO_GUARD_POSIX
map GUARD_NONNULL_GOTO     GOTO_GUARD_PTR
map GUARD_RESULT_GOTO      GOTO_GUARD_RESULT

# s2n_result
map GUARD_RESULT           RESULT_GUARD
map GUARD_AS_RESULT        RESULT_GUARD_POSIX
map GUARD_RESULT_PTR       RESULT_GUARD_PTR
map GUARD_RESULT_OSSL      RESULT_GUARD_OSSL
map ENSURE                 RESULT_ENSURE
map ENSURE_REF             RESULT_ENSURE_REF
map ENSURE_MUT             RESULT_ENSURE_MUT
map ENSURE_GTE             RESULT_ENSURE_GTE
map ENSURE_LTE             RESULT_ENSURE_LTE
map ENSURE_GT              RESULT_ENSURE_GT
map ENSURE_LT              RESULT_ENSURE_LT
map ENSURE_NE              RESULT_ENSURE_NE
map ENSURE_EQ              RESULT_ENSURE_EQ
map ENSURE_OK              RESULT_ENSURE_OK
map DEBUG_ENSURE           RESULT_DEBUG_ENSURE
map BAIL                   RESULT_BAIL
map CHECKED_MEMCPY         RESULT_CHECKED_MEMCPY
map CHECKED_MEMSET         RESULT_CHECKED_MEMSET


#!/bin/bash
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License").
# You may not use this file except in compliance with the License.
# A copy of the License is located at
#
#  http://aws.amazon.com/apache2.0
#
# or in the "license" file accompanying this file. This file is distributed
# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
# express or implied. See the License for the specific language governing
# permissions and limitations under the License.
#

set -e

usage() {
    echo "Usage: calcTotalCov.sh"
    exit 1
}

if [ "$#" -ne "0" ]; then
    usage
fi

if [[ -z "$S2N_ROOT" ]]; then
    S2N_ROOT=../..
fi

FUZZCOV_SOURCES="${S2N_ROOT}/api ${S2N_ROOT}/bin ${S2N_ROOT}/crypto ${S2N_ROOT}/error ${S2N_ROOT}/pq-crypto ${S2N_ROOT}/stuffer ${S2N_ROOT}/tls ${S2N_ROOT}/utils"

# Outputs fuzz coverage results if the FUZZ_COVERAGE environment variable is set
# Total coverage is overlayed on source code in s2n_cov.html and coverage statistics are available in s2n_cov.txt
# If using LLVM version 9 or greater, coverage is output in LCOV format instead of HTML
# All files are stored in the s2n coverage directory
if [[ "$FUZZ_COVERAGE" == "true" ]]; then

    printf "Calculating total s2n coverage... "

    # The llvm-profdata merge command warns that the profraws were created from different binaries (which is true) but
    # works fine for what we care about (the s2n library). Therefore, for user clarity all output is suppressed.
    llvm-profdata merge -sparse ./profiles/*/*.profdata -o ./profiles/s2n_cov.profdata > /dev/null 2>&1
    llvm-cov report -instr-profile=./profiles/s2n_cov.profdata ${S2N_ROOT}/lib/libs2n.so ${FUZZCOV_SOURCES} > ${COVERAGE_DIR}/fuzz/s2n_cov.txt

    # Use LCOV format instead of HTML if the LLVM version we're using supports it
    if [[ $(grep -Eo "[0-9]*" <<< `llvm-cov --version` | head -1) > 8 ]]; then
        llvm-cov export -instr-profile=./profiles/s2n_cov.profdata ${S2N_ROOT}/lib/libs2n.so ${FUZZCOV_SOURCES} -format=lcov > ${COVERAGE_DIR}/fuzz/s2n_cov.info
        genhtml -q -o ${COVERAGE_DIR}/html/overall_fuzz_coverage ${COVERAGE_DIR}/fuzz/s2n_cov.info
    else
        llvm-cov show -instr-profile=./profiles/s2n_cov.profdata ${S2N_ROOT}/lib/libs2n.so ${FUZZCOV_SOURCES} -use-color -format=html > ${COVERAGE_DIR}/fuzz/s2n_cov.html
    fi
    # Generate coverage report compatible with codecov.io
    llvm-cov show -instr-profile=./profiles/s2n_cov.profdata ${S2N_ROOT}/lib/libs2n.so ${FUZZCOV_SOURCES} > ${COVERAGE_DIR}/fuzz/codecov.txt

    S2N_COV=`grep -Eo '[0-9]*\.[0-9]*\%' ${COVERAGE_DIR}/fuzz/s2n_cov.txt | tail -1`
    printf "total s2n coverage from fuzz tests: %s\n" $S2N_COV
fi


#!/bin/bash
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License").
# You may not use this file except in compliance with the License.
# A copy of the License is located at
#
#  http://aws.amazon.com/apache2.0
#
# or in the "license" file accompanying this file. This file is distributed
# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
# express or implied. See the License for the specific language governing
# permissions and limitations under the License.
#

# The timeout command sends a TERM and under normal circumstances returns
# exit code 124. We'll undo this later.
set -e

usage() {
    echo "Usage: runFuzzTest.sh TEST_NAME FUZZ_TIMEOUT_SEC"
    exit 1
}

if [ "$#" -ne "2" ]; then
    usage
fi

TEST_NAME=$1
FUZZ_TIMEOUT_SEC=$2
MIN_TEST_PER_SEC="1000"
MIN_FEATURES_COVERED="100"

# Failures for negative tests on AFL can be ignored.
if [[ $TEST_NAME == *_negative_test && "$AFL_FUZZ" != "true" ]];
then
    EXPECTED_TEST_FAILURE=1
else
    EXPECTED_TEST_FAILURE=0
fi

ASAN_OPTIONS+="symbolize=1"
LSAN_OPTIONS+="log_threads=1"
UBSAN_OPTIONS+="print_stacktrace=1"
NUM_CPU_THREADS=$(nproc)
LIBFUZZER_ARGS+="-timeout=5 -max_len=4096 -print_final_stats=1 -jobs=${NUM_CPU_THREADS} -workers=${NUM_CPU_THREADS} -max_total_time=${FUZZ_TIMEOUT_SEC}"

TEST_SPECIFIC_OVERRIDES="${PWD}/LD_PRELOAD/${TEST_NAME}_overrides.so"
GLOBAL_OVERRIDES="${PWD}/LD_PRELOAD/global_overrides.so"

FUZZCOV_SOURCES="${S2N_ROOT}/api ${S2N_ROOT}/bin ${S2N_ROOT}/crypto ${S2N_ROOT}/error ${S2N_ROOT}/pq-crypto ${S2N_ROOT}/stuffer ${S2N_ROOT}/tls ${S2N_ROOT}/utils"

if [ -e $TEST_SPECIFIC_OVERRIDES ];
then
    export LD_PRELOAD="$TEST_SPECIFIC_OVERRIDES $GLOBAL_OVERRIDES"
else
    export LD_PRELOAD="$GLOBAL_OVERRIDES"
fi

FIPS_TEST_MSG=""
if [ -n "${S2N_TEST_IN_FIPS_MODE}" ];
then
    FIPS_TEST_MSG=" FIPS test"
fi

if [ ! -d "./corpus/${TEST_NAME}" ];
then
  printf "\033[33;1mWARNING!\033[0m ./corpus/${TEST_NAME} directory does not exist, feature coverage may be below minimum.\n\n"
fi

# Make directory if it doesn't exist
mkdir -p "./corpus/${TEST_NAME}"

ACTUAL_TEST_FAILURE=0

# Copy existing Corpus to a temp directory so that new inputs from fuzz tests runs will add new inputs to the temp directory.
# This allows us to minimize new inputs before merging to the original corpus directory.
TEMP_CORPUS_DIR="$(mktemp -d)"
cp -r ./corpus/${TEST_NAME}/. "${TEMP_CORPUS_DIR}"

# Run AFL instead of libfuzzer if AFL_FUZZ is set. Not compatible with fuzz coverage.
if [[ ${AFL_FUZZ} == "true" && ${FUZZ_COVERAGE} != "true" ]]; then
    unset LD_PRELOAD
    # See https://aflplus.plus/docs/env_variables/
    export AFL_NO_UI=true
    export AFL_HARDEN=true
    printf "Running AFL %-s %-40s for %5d sec... " "${FIPS_TEST_MSG}" ${TEST_NAME} ${FUZZ_TIMEOUT_SEC}
    mkdir -p results/${TEST_NAME}
    set +e
    timeout ${FUZZ_TIMEOUT_SEC} ${LIBFUZZER_INSTALL_DIR}/afl-fuzz -i corpus/${TEST_NAME} -o results/${TEST_NAME} -m none ./${TEST_NAME}  2>&1> ./results/${TEST_NAME}/console_output.log
    returncode=$?
    # See the timeout man page for specifics
    if [[ ${returncode} -ne 124 ]]; then
        printf "\033[33;1mWARNING!\033[0m AFL exited with an unexpected return value: %8d" ${returncode}
    fi
    set -e
    CRASH_COUNT=$(sed -n -e 's/^unique_crashes *: //p' ./results/${TEST_NAME}/fuzzer_stats)
    TEST_COUNT=$(sed -n -e 's/^execs_done *: //p' ./results/${TEST_NAME}/fuzzer_stats)
    FLOAT_TESTS_PER_SEC=$(sed -n -e 's/^execs_per_sec *: //p' ./results/${TEST_NAME}/fuzzer_stats)
    TESTS_PER_SEC=$(echo "($FLOAT_TESTS_PER_SEC+.5)/1"|bc)

    if [[ ${TESTS_PER_SEC} -lt 10 ]]; then
        printf "\033[33;1mWARNING!\033[0m %10d tests, only %6d tests per second; test is too slow.\n" ${TEST_COUNT} ${TESTS_PER_SEC}
    fi
    if [[ ${CRASH_COUNT} -gt 0 ]]; then
        ACTUAL_TEST_FAILURE=1
    fi
    if [[ ${ACTUAL_TEST_FAILURE} == ${EXPECTED_TEST_FAILURE} ]]; then
        printf "\033[32;1mPASSED\033[0m %8d tests, %.1f test/sec\n" ${TEST_COUNT} ${TESTS_PER_SEC}
        exit 0
    else
        printf "\033[31;1mFAILED\033[0m %10d tests, %6d unique crashes\n" ${TEST_COUNT} ${CRASH_COUNT}
        exit -1
    fi
else
    printf "Running %-s %-40s for %5d sec with %2d threads... " "${FIPS_TEST_MSG}" ${TEST_NAME} ${FUZZ_TIMEOUT_SEC} ${NUM_CPU_THREADS}
fi

# Setup and clean profile structure if FUZZ_COVERAGE is enabled, otherwise run as normal
if [[ "$FUZZ_COVERAGE" == "true" ]]; then
    mkdir -p "./profiles/${TEST_NAME}"
    rm -f ./profiles/${TEST_NAME}/*.profraw
    LLVM_PROFILE_FILE="./profiles/${TEST_NAME}/${TEST_NAME}.%p.profraw" ./${TEST_NAME} ${LIBFUZZER_ARGS} ${TEMP_CORPUS_DIR} > ${TEST_NAME}_output.txt 2>&1 || ACTUAL_TEST_FAILURE=1
else
    ./${TEST_NAME} ${LIBFUZZER_ARGS} ${TEMP_CORPUS_DIR} > ${TEST_NAME}_output.txt 2>&1 || ACTUAL_TEST_FAILURE=1
fi

TEST_INFO=$(
    grep -o "stat::number_of_executed_units: [0-9]*" ${TEST_NAME}_output.txt | \
    awk -v timeout=$FUZZ_TIMEOUT_SEC '{count += int($2); rate = int(count / timeout)} END {print count, "tests, " rate " test/sec"}' \
)
TESTS_PER_SEC=$(echo "$TEST_INFO" | cut -d ' ' -f 3)
FEATURE_COVERAGE=`grep -o "ft: [0-9]*" ${TEST_NAME}_output.txt | awk '{print $2}' | sort | tail -1`
TARGET_FUNCS=''
declare -i TARGET_TOTAL=0
declare -i TARGET_COV=0

# Outputs fuzz coverage results if the FUZZ_COVERAGE environment variable is set
# Coverage is overlayed on source code in ${TEST_NAME}_cov.html, and coverage statistics are available in ${TEST_NAME}_cov.txt
# If using LLVM version 9 or greater, coverage is output in LCOV format instead of HTML
# All files are stored in the s2n coverage directory
if [[ "$FUZZ_COVERAGE" == "true" ]]; then
    mkdir -p ${COVERAGE_DIR}/fuzz
    llvm-profdata merge -sparse ./profiles/${TEST_NAME}/*.profraw -o ./profiles/${TEST_NAME}/${TEST_NAME}.profdata
    llvm-cov report -instr-profile=./profiles/${TEST_NAME}/${TEST_NAME}.profdata ${S2N_ROOT}/lib/libs2n.so ${FUZZCOV_SOURCES} -show-functions > ${COVERAGE_DIR}/fuzz/${TEST_NAME}_cov.txt

    # Use LCOV format instead of HTML if the LLVM version we're using supports it
    if [[ $(grep -Eo "[0-9]*" <<< `llvm-cov --version` | head -1) -gt 8 ]]; then
        llvm-cov export -instr-profile=./profiles/${TEST_NAME}/${TEST_NAME}.profdata ${S2N_ROOT}/lib/libs2n.so ${FUZZCOV_SOURCES} -format=lcov > ${COVERAGE_DIR}/fuzz/${TEST_NAME}_cov.info
        genhtml -q -o ${COVERAGE_DIR}/html/${TEST_NAME} ${COVERAGE_DIR}/fuzz/${TEST_NAME}_cov.info
    else
        llvm-cov show -instr-profile=./profiles/${TEST_NAME}/${TEST_NAME}.profdata ${S2N_ROOT}/lib/libs2n.so ${FUZZCOV_SOURCES} -use-color -format=html > ${COVERAGE_DIR}/fuzz/${TEST_NAME}_cov.html
    fi

    # Extract target functions from test source
    TARGET_FUNCS=`grep -Pzo "(?<=/\* Target Functions: )[\w\s]*" ${TEST_NAME}.c | tr -d "\0"`

    # Find line coverage statistics for target functions
    if [[ ! -z "$TARGET_FUNCS" ]];
    then
        for TARGET in ${TARGET_FUNCS}
        do
            TARGET_TOTAL+=`sed -n "s/^.*${TARGET} .*% *\([0-9]*\) .*$/\1/p" ${COVERAGE_DIR}/fuzz/${TEST_NAME}_cov.txt`
            TARGET_COV+=`sed -n "s/^.*${TARGET} .*% *[0-9]* *\([0-9]*\) .*$/\1/p" ${COVERAGE_DIR}/fuzz/${TEST_NAME}_cov.txt`
        done
    fi
fi

if [ $ACTUAL_TEST_FAILURE == $EXPECTED_TEST_FAILURE ];
then
    printf "\033[32;1mPASSED\033[0m %s" "$TEST_INFO"

    # Output target function coverage percentage if target functions are defined and fuzzing coverage is enabled
    # Otherwise, print number of features covered
    if [[ "$FUZZ_COVERAGE" == "true" && ! -z "$TARGET_FUNCS" && "$EXPECTED_TEST_FAILURE" != 1 && "$TARGET_TOTAL" != 0 ]];
    then
        printf ", %6.2f%% target coverage" "$(( 10000 * ($TARGET_TOTAL - $TARGET_COV) / $TARGET_TOTAL ))e-2"
    else
        printf ", %5d features covered" $FEATURE_COVERAGE
    fi

    if [ $EXPECTED_TEST_FAILURE == 1 ];
    then
        # Clean up LibFuzzer corpus files if the test is negative.
        printf "\n"
        rm -f leak-* crash-*
    else
        # TEMP_CORPUS_DIR may contain many new inputs that only covers a small set of new branches.
        # Instead of copying all new inputs to the corpus directory,  only copy back minimum number of new inputs that reach new branches.
        ./${TEST_NAME} -merge=1 "./corpus/${TEST_NAME}" "${TEMP_CORPUS_DIR}" > ${TEST_NAME}_results.txt 2>&1

        # Print number of new files and branches found in new Inputs (if any)
        RESULTS=`grep -Eo "[0-9]+ new files .*$" ${TEST_NAME}_results.txt | tail -1`
        printf ", ${RESULTS}\n"

        if [ "$TESTS_PER_SEC" -lt $MIN_TEST_PER_SEC ]; then
            printf "\033[33;1mWARNING!\033[0m ${TEST_NAME} is only ${TESTS_PER_SEC} tests/sec, which is below ${MIN_TEST_PER_SEC}/sec! Fuzz tests are more effective at higher rates.\n\n"
        fi

        if [ "$FEATURE_COVERAGE" -lt $MIN_FEATURES_COVERED ]; then
            printf "\033[31;1mERROR!\033[0m ${TEST_NAME} only covers ${FEATURE_COVERAGE} features, which is below ${MIN_FEATURES_COVERED}! This may be due to missing corpus files or a bug.\n"
            exit -1;
        fi
    fi

else
    cat ${TEST_NAME}_output.txt
    printf "\033[31;1mFAILED\033[0m %s, %6d features covered\n" "$TEST_INFO" $FEATURE_COVERAGE
    exit -1
fi


#!/usr/bin/env bash
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License").
# You may not use this file except in compliance with the License.
# A copy of the License is located at
#
#  http://aws.amazon.com/apache2.0
#
# or in the "license" file accompanying this file. This file is distributed
# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
# express or implied. See the License for the specific language governing
# permissions and limitations under the License.
#

set -eu

usage() {
    echo "gen_self_signed_cert.sh [OPTION]

Options:
--user-type Server or client depending on intended usage of the cert
--hash-alg The type of hash algorithm to use to sign the cert in openssl format, ex; sha256
--dns A DNS type subject alternative name to add to the cert. Can be repeated.
--ip An IP type subject alternative name to add to the cert. Can be repeated.
--uri A URI type subject alternative name to add to the cert. Can be repeated.
--cn  The CN to add to the cert
--key-type The type of key for the generated certificate to have. Either rsa or ecdsa.
--rsa-key-size Size of rsa key for generated certificate.
--curve-name name of the ECC curve to use for the generated ECDSA certificate.
--prefix Prefix for the output certificate and private key file name. Defaults to the CN value.
"
    exit 0;
}

GETOPT="getopt"

# use gnu-getopt on macos
if [[ "$OSTYPE" == "darwin"* ]]; then
    GETOPT="/usr/local/opt/gnu-getopt/bin/getopt"

    if ! [ -x "$(command -v $GETOPT)" ]; then
      echo 'Error: getopt is not installed. Install with `brew install gnu-getopt`' >&2
      exit 1
    fi
fi

# This only works with gnu getopt.
PARSED_OPTS=`$GETOPT -o vdn: --long help,user-type:,rsa-key-size:,curve-name:,hash-alg:,ip:,uri:,dns:,cn:,key-type:,prefix: -n 'parse-options' -- "$@"`
eval set -- "$PARSED_OPTS"

USER_TYPE="server"
KEY_TYPE="rsa"
RSA_KEY_SIZE="2048"
CURVE_NAME="prime256v1"
HASH_ALG="sha256"
SANS=
CN="s2nTestCert"
PREFIX=

while true; do
  case "$1" in
    --help ) usage ;;
    --cn ) CN="$2" ; shift 2 ;;
    --dns ) SANS="$SANS""DNS:$2,"  ; shift 2 ;;
    --ip ) SANS="$SANS""IP:$2,"  ; shift 2 ;;
    --uri ) SANS="$SANS""URI:$2,"  ; shift 2 ;;
    --hash-alg )    HASH_ALG="$2" ; shift 2 ;;
    --key-type ) KEY_TYPE="$2" ; shift 2 ;;
    --rsa-key-size ) RSA_KEY_SIZE="$2" ; shift 2 ;;
    --curve-name ) CURVE_NAME="$2" ; shift 2 ;;
    --user-type ) USER_TYPE="$2" ; shift 2 ;;
    --prefix ) PREFIX="$2" ; shift 2 ;;
    -- ) shift; break ;;
    * ) break ;;
  esac
done

# Trim the railing comma, openssl x509 config expects the last SAN value to have no comma.
SANS=`echo $SANS | sed 's/,*$//g'`
if [ -z "$PREFIX" ]; then
    PREFIX=$CN
fi

# Picked up by cert_config.cfg. It might have been simpler to just generate all certs with both
# serverAuth and clientAuth KeyUsage.
KEY_USAGE=
if [ "$USER_TYPE" == "server" ]
then
    KEY_USAGE="serverAuth";
elif [ "$USER_TYPE" == "client" ]
then
    KEY_USAGE="clientAuth";
elif [ "$USER_TYPE" == "both" ]
then
    KEY_USAGE="serverAuth, clientAuth";
else
    echo "Incorrect user-type: $USER_TYPE"
    usage ;
fi

config="""
[req]
distinguished_name = req_distinguished_name
x509_extensions = v3_req
prompt = no
[req_distinguished_name]
C = US
ST = WA
L = Seattle
O = Amazon
OU = s2n
"""

if [[ ! -z "$CN" ]]; then
    config+="CN = $CN"
fi

config+="""

[v3_req]
keyUsage = keyEncipherment, dataEncipherment, digitalSignature
extendedKeyUsage = $KEY_USAGE
"""

if [[ ! -z "$SANS" ]]; then
    config+="subjectAltName = $SANS"
fi

cert_conf_path=$(mktemp)
echo "$config" > $cert_conf_path

# append an underscore if there's a prefix
if [[ ! -z "$PREFIX" ]]; then
  PREFIX="${PREFIX}_"
fi

if [ "$KEY_TYPE" == "rsa" ]; then
    openssl req -x509 -config "$cert_conf_path" -newkey rsa:${RSA_KEY_SIZE} -${HASH_ALG} -nodes -keyout ${PREFIX}rsa_key.pem -out ${PREFIX}rsa_cert.pem -days 36500
elif [ "$KEY_TYPE" == "ecdsa" ]; then
    openssl ecparam -out "${PREFIX}ecdsa_key.pem" -name "$CURVE_NAME" -genkey
    openssl req -new -config "$cert_conf_path" -days 36500 -nodes -x509 -key "${PREFIX}ecdsa_key.pem" -out "${PREFIX}ecdsa_cert.pem"
else
    echo "Incorrect key-type: $KEY_TYPE"
    usage ;
fi

rm $cert_conf_path


#!/bin/bash
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License").
# You may not use this file except in compliance with the License.
# A copy of the License is located at
#
#  http://aws.amazon.com/apache2.0
#
# or in the "license" file accompanying this file. This file is distributed
# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
# express or implied. See the License for the specific language governing
# permissions and limitations under the License.
#

install_dir="$(pwd)"
openssl_conf_path="$(pwd)/openssl.conf"

get_subj() {
  common_name="$1"
  echo "/C=US/ST=Massachusetts/L=Boston/O=Amazon/OU=s2n/CN=${common_name}"
}

init_ca_dir() {
  ca_dir="$1"
  mkdir -p "${ca_dir}"
  touch "${ca_dir}/index.txt"
  echo 01 > "${ca_dir}/serial"
  echo 01 > "${ca_dir}/crlnumber"
}

generate_key_cert() {
  path="$1"
  type="$2"

  common_name=$(basename "${path}")
  out="csr.pem"
  opts=()

  case "${type}" in
    ca)
      opts=("-x509" "-days" "36500" "-extensions" "v3_ca")
      out="cert.pem"
      ;;
    intermediate)
      ;;
    leaf)
      # Common name must be localhost for s2n-tls default verify host callback
      common_name="localhost"
      ;;
    *)
      echo "invalid type"
      exit
  esac

  pushd "${path}" || exit

  openssl genrsa -out key.pem 4096 || exit
  chmod 400 key.pem

  ca_dir="." \
  openssl req -config "${openssl_conf_path}" \
      -key key.pem \
      -new -sha256 \
      "${opts[@]}" \
      -out "${out}" \
      -subj "$(get_subj ${common_name})" \
      || exit

  # Create a chain with just the root certificate for validating child certificates
  if [ "${type}" = "ca" ]; then
    cp cert.pem chain.pem
  fi

  popd || exit
}

verify_cert() {
  ca_file_path="$1"
  cert_file_path="$2"
  crl_file_path="$3"

  opts=()
  if [ -n "${crl_file_path}" ]; then
      is_revoked=$4
      if [ -z "${is_revoked}" ]; then
        exit
      fi

      opts=("-CRLfile" "${crl_file_path}" "-crl_check")
  fi

  verify_stderr=$(
    openssl verify \
        -CAfile "${ca_file_path}" \
        "${opts[@]}" \
        "${cert_file_path}" \
        2>&1
  )
  verify_ret=$?

  if [ -n "${crl_file_path}" ] && [ ${is_revoked} -eq 1 ]; then
    # Ensure openssl verify failed due to certificate revocation
    if ! [[ "${verify_stderr}" =~ .*"certificate revoked".* ]]; then
      exit
    fi
  else
    # Ensure openssl verify succeeded
    if [ ${verify_ret} -ne 0 ]; then
      exit
    fi
  fi
}

sign_cert() {
  signer_path="$1"
  to_sign_path="$2"
  signer_type="$3"

  opts=()
  case "${signer_type}" in
    ca)
      opts=("-extensions" "v3_intermediate_ca")
      ;;
    intermediate)
      opts=("-extensions" "usr_cert")
      ;;
    *)
      echo "invalid type"
      exit
  esac

  ca_dir="${signer_path}" \
  openssl ca -config "${openssl_conf_path}" -batch \
      -days 36500 -notext -md sha256 \
      "${opts[@]}" \
      -in "${to_sign_path}/csr.pem" \
      -out "${to_sign_path}/cert.pem" \
      || exit

  # Ensure openssl can validate the signed certificate
  verify_cert "${signer_path}/chain.pem" "${to_sign_path}/cert.pem"

  # Create a chain that includes the newly signed certificate and all parent certificates
  # for validating future child certificates
  if [ "${signer_type}" = "ca" ]; then
    cat "${to_sign_path}/cert.pem" "${signer_path}/chain.pem" > "${to_sign_path}/chain.pem"
  fi
}

generate_crl() {
  path="$1"

  pushd "${path}" || exit
  ca_dir="." \
  openssl ca -config "${openssl_conf_path}" -gencrl \
    -out crl.pem \
      || exit
  popd || exit
}

revoke_cert() {
  ca_path="$1"
  cert_path="$2"

  ca_dir="${ca_path}" \
  openssl ca -config "${openssl_conf_path}" \
      -revoke "${cert_path}/cert.pem" \
      || exit
  generate_crl "${ca_path}"
}

base_dir=$(mktemp -d)
pushd "${base_dir}" || exit

#
# Generate certificate chains of valid and revoked certificates with 1 intermediate CA.
#
# root -> valid intermediate   -> valid leaf
#                              -> revoked leaf
#      -> revoked intermediate -> valid leaf
#                              -> revoked leaf
#

# Generate root certificate
init_ca_dir root
generate_key_cert root ca

# Generate intermediate certificates
init_ca_dir root/intermediate
generate_key_cert root/intermediate intermediate
sign_cert root root/intermediate ca

init_ca_dir root/intermediate_revoked
generate_key_cert root/intermediate_revoked intermediate
sign_cert root root/intermediate_revoked ca

# Generate leaf certificates
mkdir root/intermediate/leaf
generate_key_cert root/intermediate/leaf leaf
sign_cert root/intermediate root/intermediate/leaf intermediate

mkdir root/intermediate/leaf_revoked
generate_key_cert root/intermediate/leaf_revoked leaf
sign_cert root/intermediate root/intermediate/leaf_revoked intermediate

mkdir root/intermediate_revoked/leaf
generate_key_cert root/intermediate_revoked/leaf leaf
sign_cert root/intermediate_revoked root/intermediate_revoked/leaf intermediate

mkdir root/intermediate_revoked/leaf_revoked
generate_key_cert root/intermediate_revoked/leaf_revoked leaf
sign_cert root/intermediate_revoked root/intermediate_revoked/leaf_revoked intermediate

# Generate CRLs and revoke the revoked certificates
generate_crl root
revoke_cert root root/intermediate_revoked

generate_crl root/intermediate
revoke_cert root/intermediate root/intermediate/leaf_revoked

generate_crl root/intermediate_revoked
revoke_cert root/intermediate_revoked root/intermediate/leaf_revoked

# Ensure openssl verify reads generated CRLs and properly rejects certificates
verify_cert root/chain.pem root/intermediate/cert.pem root/crl.pem 0
verify_cert root/chain.pem root/intermediate_revoked/cert.pem root/crl.pem 1

verify_cert root/intermediate/chain.pem root/intermediate/leaf/cert.pem root/intermediate/crl.pem 0
verify_cert root/intermediate/chain.pem root/intermediate/leaf_revoked/cert.pem root/intermediate/crl.pem 1

verify_cert root/intermediate_revoked/chain.pem root/intermediate_revoked/leaf/cert.pem root/intermediate_revoked/crl.pem 0
verify_cert root/intermediate_revoked/chain.pem root/intermediate_revoked/leaf_revoked/cert.pem root/intermediate_revoked/crl.pem 1

# Generate CRLs with invalid timestamps. Requires Openssl 3.0.
ca_dir="root/intermediate" \
openssl ca -config "${openssl_conf_path}" -gencrl \
    -crl_lastupdate 21220101000000Z \
    -out "${install_dir}/intermediate_invalid_this_update_crl.pem" \
    || exit

ca_dir="root/intermediate" \
openssl ca -config "${openssl_conf_path}" -gencrl \
    -crl_nextupdate 500101000000Z \
    -out "${install_dir}/intermediate_invalid_next_update_crl.pem" \
    || exit

#
# Create certificate chain pems and copy CRLs and keys
#

cp root/cert.pem "${install_dir}/root_cert.pem"

cat root/intermediate/leaf/cert.pem \
    root/intermediate/cert.pem \
    > "${install_dir}/none_revoked_cert_chain.pem"
cp root/intermediate/leaf/key.pem "${install_dir}/none_revoked_key.pem"

cat root/intermediate/leaf_revoked/cert.pem \
    root/intermediate/cert.pem \
    > "${install_dir}/leaf_revoked_cert_chain.pem"
cp root/intermediate/leaf_revoked/key.pem "${install_dir}/leaf_revoked_key.pem"

cat root/intermediate_revoked/leaf/cert.pem \
    root/intermediate_revoked/cert.pem \
    > "${install_dir}/intermediate_revoked_cert_chain.pem"
cp root/intermediate_revoked/leaf/key.pem "${install_dir}/intermediate_revoked_key.pem"

cat root/intermediate_revoked/leaf_revoked/cert.pem \
    root/intermediate_revoked/cert.pem \
    > "${install_dir}/all_revoked_cert_chain.pem"
cp root/intermediate_revoked/leaf_revoked/key.pem "${install_dir}/all_revoked_key.pem"

cp root/crl.pem "${install_dir}/root_crl.pem"
cp root/intermediate/crl.pem "${install_dir}/intermediate_crl.pem"
cp root/intermediate_revoked/crl.pem "${install_dir}/intermediate_revoked_crl.pem"

popd || exit

# Cleanup
rm -rf "${base_dir}"

# Ensure that s2nc accepts all generated certificates
./test_s2nc.sh || exit

echo "Generation successful."


#!/bin/bash
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License").
# You may not use this file except in compliance with the License.
# A copy of the License is located at
#
#  http://aws.amazon.com/apache2.0
#
# or in the "license" file accompanying this file. This file is distributed
# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
# express or implied. See the License for the specific language governing
# permissions and limitations under the License.
#

# Ensure that s2nc accepts all generated certificates

s2nc_path="../../../bin/s2nc"
s2nd_path="../../../bin/s2nd"

cert_chains=("none_revoked" "leaf_revoked" "intermediate_revoked" "all_revoked")

for cert_chain in "${cert_chains[@]}"; do
  "${s2nd_path}" \
      --self-service-blinding \
      --negotiate \
      --cert "${cert_chain}_cert_chain.pem" \
      --key "${cert_chain}_key.pem" \
      localhost 8888 &
  s2nd_pid=$!

  "${s2nc_path}" \
      --ca-file "root_cert.pem" \
      localhost 8888
  s2nc_success=$?

  kill ${s2nd_pid}

  if [ ${s2nc_success} -ne 0 ]; then
    exit 1
  fi
done


#!/bin/bash
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License").
# You may not use this file except in compliance with the License.
# A copy of the License is located at
#
#  http://aws.amazon.com/apache2.0
#
# or in the "license" file accompanying this file. This file is distributed
# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
# express or implied. See the License for the specific language governing
# permissions and limitations under the License.

# immediately bail if any command fails
set -e

# This script assumes the existence of
# ca_key.pem
# ca_cert.pem
# server_key.pem - the private key for early expiry cert being generated
# config/ca.cnf
# config/server_early_expiry.cnf

echo "generating serial file"
touch config/serial
# the openssl ca command will read this file to figure out the serial number,
# increment it for each certificate that is signed. This is the serial number in
# hex notation.
echo "1024" > config/serial

echo "generating server early expiry CSR"
openssl req  -new -nodes -key server_key.pem -out server_early_expire.csr -config config/server_early_expire.cnf

echo "creating empty CA database"
# it's mandatory for this to exist, although it is initially empty
# this data base will be updated during the ca command
touch certs_early_expire_index.txt

echo "generating server certificate and signing it"
# this directory holds the duplicate certs that the CA command generates
# https://security.stackexchange.com/questions/111448/how-to-avoid-writing-pem-while-signing-a-csr
mkdir -p to_nuke

# use the "batch" option to disable prompting
# the enddate argument is in YYYYMMDDHHMMSS format
# use the "notext" option because s2n can't parse .pem certificates with the
# text information inside of them
openssl ca -batch \
    -in server_early_expire.csr  \
    -out server_cert_early_expire.pem \
    -config config/ca.cnf \
    -outdir to_nuke \
    -enddate 20370101010101Z \
    -extfile config/server_early_expire.cnf \
    -extensions req_ext \
    -notext

echo "verifying generated certificates"
openssl verify -CAfile ca_cert.pem server_cert_early_expire.pem

echo "generating ocsp response"
# This is the target next_update date for the OCSP response. This date needs to
# be before 2038 so that next_update field can be tested on 32 bit platforms.
target_date="2036-01-01"
current_date=$(date +%Y-%m-%d)
expiration_days=$(( ($(date -d "$target_date" +%s) - $(date -d "$current_date" +%s)) / 86400 ))

# The & launches the server in the background
openssl ocsp -port 8889 -text -CA ca_cert.pem \
      -index certs_early_expire_index.txt \
      -rkey ocsp_key.pem \
      -rsigner ocsp_cert.pem \
      -nrequest 1 \
      -ndays $expiration_days \
      &

openssl ocsp -CAfile ca_cert.pem \
      -url http://127.0.0.1:8889 \
      -issuer ca_cert.pem \
      -verify_other ocsp_cert.pem \
      -cert server_cert_early_expire.pem -respout ocsp_response_early_expire.der

echo "removing temporary files"
rm server_early_expire.csr
# openssl generates <file>, <file>.attr, <file>.old, and <file>.attr.old
# delete all of them
rm certs_early_expire_index* -f
rm to_nuke -rf
rm config/serial
rm config/serial.old


#!/usr/bin/env bash
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License").
# You may not use this file except in compliance with the License.
# A copy of the License is located at
#
#  http://aws.amazon.com/apache2.0
#
# or in the "license" file accompanying this file. This file is distributed
# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
# express or implied. See the License for the specific language governing
# permissions and limitations under the License.
#

set -eu

cd $(dirname $0)

../gen_self_signed_cert.sh --cn '' --prefix 'without_cn'
../gen_self_signed_cert.sh --ip '::1' --prefix 'ip_v6_lo'


#!/bin/bash
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License").
# You may not use this file except in compliance with the License.
# A copy of the License is located at
#
#  http://aws.amazon.com/apache2.0
#
# or in the "license" file accompanying this file. This file is distributed
# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
# express or implied. See the License for the specific language governing
# permissions and limitations under the License.
#

make clean
rm -r crypto
rm -r error
rm -r stuffer
rm -r tls
rm -r utils


#!/bin/bash
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License").
# You may not use this file except in compliance with the License.
# A copy of the License is located at
#
#  http://aws.amazon.com/apache2.0
#
# or in the "license" file accompanying this file. This file is distributed
# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
# express or implied. See the License for the specific language governing
# permissions and limitations under the License.
#

set -x
set -e

BASEDIR=$(pwd)
echo $BASEDIR
S2N_BASE="$BASEDIR/../../../.."
echo $S2N_BASE

cd $BASEDIR
mkdir -p crypto
#The hmac should be based off the old hmac, so just apply the patches to add the invarients
cp $S2N_BASE/crypto/s2n_hmac.c crypto/
cp $S2N_BASE/crypto/s2n_hmac.h crypto/
patch -p5 < ../patches/hmac.patch

#the hash uses my stubs for now, so replace the file
cp stubs/s2n_hash.c crypto/
cp stubs/s2n_hash.h crypto/

mkdir -p error
cp ../stubs/s2n_errno.c error/

mkdir -p tls
#add invariants etc needed for the proof to the s2n_cbc code
cp $S2N_BASE/tls/s2n_cbc.c tls/
patch -p5 < ../patches/cbc.patch

mkdir -p utils
cp $S2N_BASE/utils/s2n_result.c utils/
cp $S2N_BASE/utils/s2n_safety.c utils/
patch -p1 < ../patches/safety.patch

cp ../stubs/s2n_annotations.h utils/
cp ../stubs/s2n_ensure.h utils/
cp ../stubs/s2n_ensure.c utils/


#!/bin/bash
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License").
# You may not use this file except in compliance with the License.
# A copy of the License is located at
#
#  http://aws.amazon.com/apache2.0
#
# or in the "license" file accompanying this file. This file is distributed
# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
# express or implied. See the License for the specific language governing
# permissions and limitations under the License.
#

./copy_as_needed.sh
make clean
time make


#!/bin/bash
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License").
# You may not use this file except in compliance with the License.
# A copy of the License is located at
#
#  http://aws.amazon.com/apache2.0
#
# or in the "license" file accompanying this file. This file is distributed
# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
# express or implied. See the License for the specific language governing
# permissions and limitations under the License.
#

make clean
rm -r crypto
rm -r error
rm -r tls
rm -r stuffer
rm -r utils


#!/bin/bash
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License").
# You may not use this file except in compliance with the License.
# A copy of the License is located at
#
#  http://aws.amazon.com/apache2.0
#
# or in the "license" file accompanying this file. This file is distributed
# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
# express or implied. See the License for the specific language governing
# permissions and limitations under the License.
#

set -x
set -e
BASEDIR=$(pwd)
echo $BASEDIR
S2N_BASE="$BASEDIR/../../../.."
echo $S2N_BASE

cd $BASEDIR
mkdir -p crypto
#The hmac should be based off the old hmac, so just apply the patches to add the invarients
cp $S2N_BASE/crypto/s2n_hmac.c crypto/
cp $S2N_BASE/crypto/s2n_hmac.h crypto/
patch -p5 < ../patches/hmac.patch

#the hash uses my stubs for now, so replace the file
cp ../stubs/s2n_hash.c crypto/
cp ../stubs/s2n_hash.h crypto/

mkdir -p error
cp ../stubs/s2n_errno.c error/

mkdir -p stuffer
cp $S2N_BASE/stuffer/s2n_stuffer.c stuffer/
cp $S2N_BASE/stuffer/s2n_stuffer_network_order.c stuffer/

mkdir -p tls
#add invariants etc needed for the proof to the s2n_cbc code
cp $S2N_BASE/tls/s2n_aead.c tls/
cp $S2N_BASE/tls/s2n_cbc.c tls/
cp $S2N_BASE/tls/s2n_record_read_aead.c tls/
patch -p5 < ../patches/cbc.patch

mkdir -p utils
cp $S2N_BASE/utils/s2n_blob.c utils/
cp $S2N_BASE/utils/s2n_result.c utils/
cp $S2N_BASE/utils/s2n_result.h utils/
cp $S2N_BASE/utils/s2n_safety.c utils/
cp $S2N_BASE/utils/s2n_safety.h utils/
patch -p1 < ../patches/safety.patch

cp s2n_annotations.h utils/
cp ../stubs/s2n_ensure.h utils/
cp ../stubs/s2n_ensure.c utils/
cp ../stubs/s2n_mem.c utils/


#!/bin/bash
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License").
# You may not use this file except in compliance with the License.
# A copy of the License is located at
#
#  http://aws.amazon.com/apache2.0
#
# or in the "license" file accompanying this file. This file is distributed
# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
# express or implied. See the License for the specific language governing
# permissions and limitations under the License.
#

set -x
set -e

BASEDIR=$(pwd)
echo $BASEDIR
S2N_BASE="$BASEDIR/../../../.."
echo $S2N_BASE

cd $BASEDIR
mkdir -p crypto
#The hmac should be based off the old hmac, so just apply the patches to add the invarients
cp $S2N_BASE/crypto/s2n_hmac.c crypto/
cp $S2N_BASE/crypto/s2n_hmac.h crypto/
patch -p5 < ../patches/hmac.patch

#the hash uses my stubs for now, so replace the file
cp ../stubs/s2n_hash.c crypto/
cp ../stubs/s2n_hash.h crypto/

mkdir -p error
cp ../stubs/s2n_errno.c error/

mkdir -p stuffer
cp $S2N_BASE/stuffer/s2n_stuffer.c stuffer/
cp $S2N_BASE/stuffer/s2n_stuffer_network_order.c stuffer/

mkdir -p tls
#add invariants etc needed for the proof to the s2n_cbc code
cp stubs/s2n_cbc.c tls/
cp $S2N_BASE/tls/s2n_record_read_cbc.c tls/
patch -p1 < record_read_cbc.patch

mkdir -p utils
cp $S2N_BASE/utils/s2n_result.c utils/
cp $S2N_BASE/utils/s2n_safety.c utils/
patch -p1 < ../patches/safety.patch

cp ../stubs/s2n_annotations.h utils/
cp ../stubs/s2n_ensure.h utils/
cp ../stubs/s2n_ensure.c utils/
cp ../stubs/s2n_mem.c utils/


#!/bin/bash
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License").
# You may not use this file except in compliance with the License.
# A copy of the License is located at
#
#  http://aws.amazon.com/apache2.0
#
# or in the "license" file accompanying this file. This file is distributed
# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
# express or implied. See the License for the specific language governing
# permissions and limitations under the License.
#

set -x
set -e

BASEDIR=$(pwd)
echo $BASEDIR
S2N_BASE="$BASEDIR/../../../.."
echo $S2N_BASE

cd $BASEDIR
mkdir -p crypto
#The hmac should be based off the old hmac, so just apply the patches to add the invarients
cp $S2N_BASE/crypto/s2n_hmac.c crypto/
cp $S2N_BASE/crypto/s2n_hmac.h crypto/
patch -p5 < ../patches/hmac.patch

#the hash uses my stubs for now, so replace the file
cp ../stubs/s2n_hash.c crypto/
cp ../stubs/s2n_hash.h crypto/

mkdir -p error
cp ../stubs/s2n_errno.c error/

mkdir -p stuffer
cp $S2N_BASE/stuffer/s2n_stuffer.c stuffer/
cp $S2N_BASE/stuffer/s2n_stuffer_network_order.c stuffer/

mkdir -p tls
#add invariants etc needed for the proof to the s2n_cbc code
cp $S2N_BASE/tls/s2n_cbc.c tls/
cp $S2N_BASE/tls/s2n_record_read_composite.c tls/
patch -p5 < ../patches/cbc.patch
patch -p1 < record_read.patch

mkdir -p utils
cp $S2N_BASE/utils/s2n_result.c utils/
cp $S2N_BASE/utils/s2n_safety.c utils/
patch -p1 < ../patches/safety.patch

cp ../stubs/s2n_annotations.h utils/
cp ../stubs/s2n_ensure.h utils/
cp ../stubs/s2n_ensure.c utils/
cp ../stubs/s2n_mem.c utils/


#!/bin/bash
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License").
# You may not use this file except in compliance with the License.
# A copy of the License is located at
#
#  http://aws.amazon.com/apache2.0
#
# or in the "license" file accompanying this file. This file is distributed
# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
# express or implied. See the License for the specific language governing
# permissions and limitations under the License.
#

set -x
set -e

BASEDIR=$(pwd)
echo $BASEDIR
S2N_BASE="$BASEDIR/../../../.."
echo $S2N_BASE

cd $BASEDIR
mkdir -p crypto
#The hmac should be based off the old hmac, so just apply the patches to add the invarients
cp $S2N_BASE/crypto/s2n_hmac.c crypto/
cp $S2N_BASE/crypto/s2n_hmac.h crypto/
patch -p5 < ../patches/hmac.patch

#the hash uses my stubs for now, so replace the file
cp ../stubs/s2n_hash.c crypto/
cp ../stubs/s2n_hash.h crypto/

mkdir -p error
cp ../stubs/s2n_errno.c error/

mkdir -p stuffer
cp $S2N_BASE/stuffer/s2n_stuffer.c stuffer/
cp $S2N_BASE/stuffer/s2n_stuffer_network_order.c stuffer/

mkdir -p tls
#add invariants etc needed for the proof to the s2n_cbc code
cp $S2N_BASE/tls/s2n_cbc.c tls/
cp $S2N_BASE/tls/s2n_record_read_stream.c tls/
patch -p5 < ../patches/cbc.patch

mkdir -p utils
cp $S2N_BASE/utils/s2n_result.c utils/
cp $S2N_BASE/utils/s2n_safety.c utils/
patch -p1 < ../patches/safety.patch

cp ../stubs/s2n_annotations.h utils/
cp ../stubs/s2n_ensure.h utils/
cp ../stubs/s2n_ensure.c utils/
cp ../stubs/s2n_mem.c utils/


#!/usr/bin/env bash

sed -ie "s/PLACEHOLDER_HOSTNAME/$1/g" changehostname.c

gcc -o changehostname.o -c -fPIC -Wall changehostname.c
gcc -o libchangehostname.so -shared -export-dynamic changehostname.o -ldl


#!/usr/bin/env bash
# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the 'License'). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
#     http://aws.amazon.com/apache2.0/
#
# or in the 'license' file accompanying this file. This file is
# distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.

set -ex
python script.py


#!/bin/bash

# We need to checkout the version of boto3 that is installed in the mamba environment.

boto3_version=$(micromamba list | grep boto3 | tr -s ' ' | cut -d ' ' -f 3)
# Checkout the corresponding boto3 version
git checkout tags/$boto3_version

# Run the unit and functional tests
pytest tests/unit tests/functional || exit $?


#!/bin/bash

# Ref: https://keras.io/guides/, https://github.com/keras-team/keras-io/tree/master

for file in *.py; do
    if [ "$file" != "transfer_learning.py" ]; then
        # skipping transfer_learning.py because it has 20 epochs and it takes a very long time to execute
        # https://github.com/keras-team/keras-io/blob/master/guides/transfer_learning.py#L562
        python "$file" || exit $?
    fi
done


#!/bin/bash
# Run all the tutorials
for file in *.py; do
     python "$file" || exit $?
done


#!/bin/bash

# Create an empty notebook file for papermill's output
touch nb_output.ipynb

# List of all referenced notebook files in Basics, Core, and In Depth sections of Tensorflow docs, excluding experimentals.
# https://www.tensorflow.org/guide
example_notebooks=('basics.ipynb'
    'tensor.ipynb'
    'variable.ipynb'
    'autodiff.ipynb'
    'intro_to_graphs.ipynb'
    'intro_to_modules.ipynb'
    'basic_training_loops.ipynb'
    'core/quickstart_core.ipynb'
    'core/logistic_regression_core.ipynb'
    'core/mlp_core.ipynb'
    'core/matrix_core.ipynb'
    'core/optimizers_core.ipynb'
    'tensor_slicing.ipynb'
    'advanced_autodiff.ipynb'
    'ragged_tensor.ipynb'
    'sparse_tensor.ipynb'
    'random_numbers.ipynb'
)

for nb in ${example_notebooks[@]}; do
     papermill $nb 'nb_output.ipynb'
done


#!/bin/bash

pushd python >/dev/null
python3 setup.py sdist
popd >/dev/null

docker build -t sagemaker-codebuild:cpu -f Dockerfile .


#!/bin/bash

function get_default_region() {
    if [ -n "$AWS_DEFAULT_REGION" ]; then
        echo "$AWS_DEFAULT_REGION"
    else
        aws configure get region
    fi
}

function get_aws_account() {
    aws --endpoint-url 'https://sts.us-west-2.amazonaws.com' --region 'us-west-2' sts get-caller-identity --query 'Account' --output text
}

aws_region=$(get_default_region)
aws_account=$(get_aws_account)
ecr_password=$(aws ecr get-login-password --region $aws_region)
docker login --username AWS --password $ecr_password $aws_account.dkr.ecr.$aws_region.amazonaws.com

docker tag sagemaker-codebuild:cpu $aws_account.dkr.ecr.$aws_region.amazonaws.com/sagemaker-codebuild:cpu
docker push $aws_account.dkr.ecr.$aws_region.amazonaws.com/sagemaker-codebuild:cpu

docker logout https://$aws_account.dkr.ecr.$aws_region.amazonaws.com


#!/usr/bin/env bash

# The argument to this script is the image name. This will be used as the image on the local
# machine and combined with the account and region to form the repository name for ECR.

prog=$0
default_image="python:3.7-slim-buster"

function usage {
    echo "Usage: $1 [--base <base-image>] <image>"
    echo "       base:  the image to use to build from [default: ${default_image}]"
    echo "       image: the image to build to. Will be pushed to the matching ECR repo in your account"
}

if [ "$1" == "--base" ]
then
    base=$2
    if [ "${base}" == "" ]
    then
       usage ${prog}
       exit 1
    fi
    shift 2
else
    base=${default_image}
fi

image=$1

if [ "${image}" == "" ]
then
    usage ${prog}
    exit 1
fi

echo "Source image ${base}"
echo "Final image ${image}"

# Get the account number associated with the current IAM credentials
account=$(aws sts get-caller-identity --query Account --output text)

if [ $? -ne 0 ]
then
    exit 255
fi

# Get the region defined in the current configuration (default to us-west-2 if none defined)
region=$(aws configure get region)
region=${region:-us-west-2}
echo "Region ${region}"

fullname="${account}.dkr.ecr.${region}.amazonaws.com/${image}:latest"

# If the repository doesn't exist in ECR, create it.

aws ecr describe-repositories --repository-names "${image}" > /dev/null 2>&1

if [ $? -ne 0 ]
then
    aws ecr create-repository --repository-name "${image}" > /dev/null
    if [ $? -ne 0 ]
    then
        exit 255
    fi
fi

# Docker login has changed in aws-cli version 2. We support both flavors.
AWS_CLI_MAJOR_VERSION=$(aws --version | sed 's%^aws-cli/\([0-9]*\)\..*$%\1%')
if [ "${AWS_CLI_MAJOR_VERSION}" == "1" ]
then
  # Get the login command from ECR and execute it directly
  $(aws ecr get-login --region ${region} --no-include-email)
else
  aws ecr get-login-password --region ${region} | docker login --username AWS --password-stdin ${account}.dkr.ecr.${region}.amazonaws.com
fi

# Build the docker image locally with the image name and then push it to ECR
# with the full name.

docker build -t ${image} --build-arg BASE_IMAGE=${base} .

docker tag ${image} ${fullname}

docker push ${fullname}


#!/bin/bash

echo "No custom init script provided"


#!/bin/bash

# Use the container locally to run a Jupyter notebook against a container

image=$1
notebook=$2
outdir=${3:-$(pwd)}

notebook_dir=$(dirname ${notebook})
notebook_file=$(basename ${notebook})

notebook_ext="${notebook_file##*.}"
notebook_base="${notebook_file%.*}"

docker run --rm \
       -v ${notebook_dir}:/opt/ml/processing/input \
       -v ${outdir}:/opt/ml/processing/output \
       -e PAPERMILL_INPUT=/opt/ml/processing/input/${notebook_file} \
       -e PAPERMILL_OUTPUT=/opt/ml/processing/output/${notebook_base}-$(date +%Y-%m-%d-%H-%M-%S).${notebook_ext} \
       -e PAPERMILL_PARAMS="{}" \
       ${image} run_notebook


#!/bin/bash

# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
#     http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.

set -e

if [[ "$1" = "serve" ]]; then
    shift 1
    mxnet-model-server --start --mms-config config.properties
else
    eval "$@"
fi

# prevent docker exit
tail -f /dev/null


#!/usr/bin/env bash
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
#     http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.

set -euo pipefail

echo =================== $1 execution time ===================

start_time=$2
end_time=`date +%s`
total_time=$(expr $end_time - $start_time + 1)
hours=$((total_time/60/60%24))
minutes=$((total_time/60%60))
secs=$((total_time%60))

(( $hours > 0 )) && printf '%d hours ' $hours
(( $minutes > 0 )) && printf '%d minutes ' $minutes
(( $hours > 0 || $minutes > 0 )) && printf 'and '
printf '%d seconds\n\n' $secs


#!/usr/bin/env python
# add_no_index2.py
import fileinput
import sys

for line in fileinput.input(inplace=True):
    if '.. class::' in line or '.. function::' in line or '.. data::' in line or '.. _' in line:
        sys.stdout.write(line + '   :noindex:\n')
    else:
        sys.stdout.write(line)


#!/usr/bin/env bash

python benchmarks/scripts/tf_cnn_benchmarks/tf_cnn_benchmarks.py --num_batches=500 --model vgg16 --variable_update horovod --horovod_device gpu --use_fp16 --summary_verbosity 1 --save_summaries_steps 10 --train_dir /opt/ml/model --eval_dir /opt/ml/model --batch_size 32


echo "This is emr test script..."
sleep 15


#!/bin/bash
#
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
#     http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.
#
# Mounting EFS and FSx for Lustre file systems for integration Tests
FILE_SYSTEM_EFS_ID=$1
FILE_SYSTEM_FSX_ID=$2
REGION=$3
EFS_MOUNT_DIRECTORY=$4
FSX_MOUNT_DIRECTORY=$5

echo "Mounting EFS File Systems"
sudo yum install -y amazon-efs-utils
if [ ! -d $EFS_MOUNT_DIRECTORY ]
then
    echo "$EFS_MOUNT_DIRECTORY does not exist. Creating.."
    sudo mkdir "$EFS_MOUNT_DIRECTORY"
fi

if grep -qs $EFS_MOUNT_DIRECTORY /proc/mounts
then
    echo "$EFS_MOUNT_DIRECTORY already mounted."
else
    sudo mount -t efs "$FILE_SYSTEM_EFS_ID":/ efs
    if [ $? -eq 0 ]
    then
        echo "Successfully mounted $FILE_SYSTEM_EFS_ID to $EFS_MOUNT_DIRECTORY."
    else
        echo "Something went wrong. Could not mount $FILE_SYSTEM_EFS_ID to $EFS_MOUNT_DIRECTORY."
        exit -1
    fi
fi

echo "Creating subfolders for training data from EFS"
if [ ! -d $EFS_MOUNT_DIRECTORY/tensorflow ]
then
    echo "$EFS_MOUNT_DIRECTORY/tensorflow does not exist. Creating.."
    sudo mkdir $EFS_MOUNT_DIRECTORY/tensorflow
fi

if [ ! -d $EFS_MOUNT_DIRECTORY/one_p_mnist ]
then
    echo "$EFS_MOUNT_DIRECTORY/one_p_mnist does not exist. Creating.."
    sudo mkdir $EFS_MOUNT_DIRECTORY/one_p_mnist
fi

echo "Mounting FSx for Lustre File System"
sudo yum install -y lustre-client
if [ ! -d $FSX_MOUNT_DIRECTORY ]
then
    echo "$FSX_MOUNT_DIRECTORY does not exist. Creating.."
    sudo mkdir -p $FSX_MOUNT_DIRECTORY
fi

if grep -qs $FSX_MOUNT_DIRECTORY /proc/mounts;
then
    echo "$FSX_MOUNT_DIRECTORY already mounted."
else
    sudo mount -t lustre -o noatime,flock "$FILE_SYSTEM_FSX_ID".fsx."$REGION".amazonaws.com@tcp:/fsx $FSX_MOUNT_DIRECTORY
    if [ $? -eq 0 ]
    then
        echo "Successfully mounted $FILE_SYSTEM_FSX_ID to $FSX_MOUNT_DIRECTORY."
    else
        echo "Something went wrong. Could not mount $FILE_SYSTEM_FSX_ID to $FSX_MOUNT_DIRECTORY."
        exit -1
    fi
fi

echo "Creating subfolders for training data from Lustre file systems"
if [ ! -d $FSX_MOUNT_DIRECTORY/tensorflow ]
then
    echo "$FSX_MOUNT_DIRECTORY/tensorflow does not exist. Creating.."
    sudo mkdir $FSX_MOUNT_DIRECTORY/tensorflow
fi

if [ ! -d $FSX_MOUNT_DIRECTORY/one_p_mnist ]
then
    echo "$FSX_MOUNT_DIRECTORY/one_p_mnist does not exist. Creating.."
    sudo mkdir $FSX_MOUNT_DIRECTORY/one_p_mnist
fi

echo "Copying files to the mounted folders"
sudo cp temp_tf/* efs/tensorflow
sudo cp temp_tf/* /mnt/fsx/tensorflow
sudo cp temp_one_p/* efs/one_p_mnist/
sudo cp temp_one_p/* /mnt/fsx/one_p_mnist


#!/bin/bash
#
# Run a test against a SageMaker notebook
# Only runs within the SDK's CI/CD environment

function CreateLifeCycleConfig ()
{

echo "Creating life cycle config...."
LIFECYCLE_CONFIG_NAME=$1
LIFECYCLE_CONFIG_CONTENT=$2
aws sagemaker create-notebook-instance-lifecycle-config --notebook-instance-lifecycle-config-name "$LIFECYCLE_CONFIG_NAME" --on-create Content="$LIFECYCLE_CONFIG_CONTENT"

}

function DeleteLifeCycleConfig ()
{

echo "Deleting the existing life cycle config...."
LIFECYCLE_CONFIG_NAME=$1
aws sagemaker delete-notebook-instance-lifecycle-config --notebook-instance-lifecycle-config-name  "$LIFECYCLE_CONFIG_NAME"

}

function CreateLifeCycleConfigContent ()
{
ACCOUNT_ID=$1
COMMIT_ID=$2

TARBALL_DIRECTORY=/tmp/sdk-tarballs
LIFECYCLE_CONFIG_1=$(cat << 'EOF'
#!/bin/bash

set -e
set -x

mkdir "$HOME/.dlami"
touch "$HOME/.dlami/dlami_build_in_progress"
TARBALL_DIRECTORY=/tmp/sdk-tarballs
mkdir -p "$TARBALL_DIRECTORY"

EOF
)

LIFECYCLE_CONFIG_2=$(cat << EOF

aws s3 --region us-west-2 cp "s3://sagemaker-python-sdk-$ACCOUNT_ID/notebook_test/sagemaker-$COMMIT_ID.tar.gz" "$TARBALL_DIRECTORY/sagemaker.tar.gz"

EOF
)

LIFECYCLE_CONFIG_3=$(cat << 'EOF'

# Include "base" separately since it's not a subdirectory.
for env in base /home/ec2-user/anaconda3/envs/*; do
    echo "Updating SageMaker vended software in $env from pre-release SDKs..."

    sudo -u ec2-user -E sh -c 'source /home/ec2-user/anaconda3/bin/activate "$env"'

    echo "Updating SageMaker Python SDK..."
    pip install --upgrade pip
    pip install "$TARBALL_DIRECTORY/sagemaker.tar.gz"

    sudo -u ec2-user -E sh -c 'source /home/ec2-user/anaconda3/bin/deactivate'

    echo "Update of $env is complete."
done

sudo rm -rf "$MODELS_SOURCE_DIRECTORY"
sudo rm -rf "$TARBALL_DIRECTORY"
rm -rf "$HOME/.dlami"

EOF
)

LIFECYCLE_CONFIG_CONTENT=$((echo "$LIFECYCLE_CONFIG_1$LIFECYCLE_CONFIG_2$LIFECYCLE_CONFIG_3"|| echo "")| base64)

echo "$LIFECYCLE_CONFIG_CONTENT"

}

set -euo pipefail

# git doesn't work in codepipeline, use CODEBUILD_RESOLVED_SOURCE_VERSION to get commit id
codebuild_initiator="${CODEBUILD_INITIATOR:-0}"
if [ "${codebuild_initiator:0:12}" == "codepipeline" ]; then
    COMMIT_ID="${CODEBUILD_RESOLVED_SOURCE_VERSION}"
else
    COMMIT_ID=$(git rev-parse --short HEAD)
fi

ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
LIFECYCLE_CONFIG_NAME="install-python-sdk-$COMMIT_ID"

python setup.py sdist

aws s3 --region us-west-2 cp ./dist/sagemaker-*.tar.gz s3://sagemaker-python-sdk-$ACCOUNT_ID/notebook_test/sagemaker-$COMMIT_ID.tar.gz
aws s3 cp s3://sagemaker-python-sdk-cli-$ACCOUNT_ID/mead-nb-test.tar.gz mead-nb-test.tar.gz
tar -xzf mead-nb-test.tar.gz

LIFECYCLE_CONFIG_CONTENT=$(CreateLifeCycleConfigContent "$ACCOUNT_ID" "$COMMIT_ID" )

if !(CreateLifeCycleConfig "$LIFECYCLE_CONFIG_NAME" "$LIFECYCLE_CONFIG_CONTENT")  ; then
(DeleteLifeCycleConfig "$LIFECYCLE_CONFIG_NAME")
(CreateLifeCycleConfig "$LIFECYCLE_CONFIG_NAME" "$LIFECYCLE_CONFIG_CONTENT")
fi

if [ -d amazon-sagemaker-examples ]; then rm -Rf amazon-sagemaker-examples; fi
git clone --depth 1 https://github.com/aws/amazon-sagemaker-examples.git

export JAVA_HOME=$(get-java-home)
echo "set JAVA_HOME=$JAVA_HOME"
export SAGEMAKER_ROLE_ARN=$(aws iam list-roles --output text --query "Roles[?RoleName == 'SageMakerRole'].Arn")
echo "set SAGEMAKER_ROLE_ARN=$SAGEMAKER_ROLE_ARN"

./runtime/bin/mead-run-nb-test \
--instance-type ml.c4.8xlarge \
--region us-west-2 \
--lifecycle-config-name $LIFECYCLE_CONFIG_NAME \
--notebook-instance-role-arn $SAGEMAKER_ROLE_ARN \
--platformIdentifier notebook-al2-v2 \
--consider-skips-failures \
./amazon-sagemaker-examples/sagemaker_processing/spark_distributed_data_processing/sagemaker-spark-processing.ipynb \
./amazon-sagemaker-examples/advanced_functionality/kmeans_bring_your_own_model/kmeans_bring_your_own_model.ipynb \
./amazon-sagemaker-examples/advanced_functionality/tensorflow_iris_byom/tensorflow_BYOM_iris.ipynb \
./amazon-sagemaker-examples/sagemaker-python-sdk/1P_kmeans_highlevel/kmeans_mnist.ipynb \
./amazon-sagemaker-examples/sagemaker-python-sdk/1P_kmeans_lowlevel/kmeans_mnist_lowlevel.ipynb \
./amazon-sagemaker-examples/sagemaker-python-sdk/mxnet_gluon_sentiment/mxnet_sentiment_analysis_with_gluon.ipynb \
./amazon-sagemaker-examples/sagemaker-python-sdk/mxnet_onnx_export/mxnet_onnx_export.ipynb \
./amazon-sagemaker-examples/sagemaker-python-sdk/scikit_learn_randomforest/Sklearn_on_SageMaker_end2end.ipynb \
./amazon-sagemaker-examples/sagemaker-python-sdk/tensorflow_moving_from_framework_mode_to_script_mode/tensorflow_moving_from_framework_mode_to_script_mode.ipynb \
./amazon-sagemaker-examples/sagemaker-python-sdk/tensorflow_script_mode_pipe_mode/tensorflow_script_mode_pipe_mode.ipynb \
./amazon-sagemaker-examples/sagemaker-python-sdk/tensorflow_serving_using_elastic_inference_with_your_own_model/tensorflow_serving_pretrained_model_elastic_inference.ipynb \

(DeleteLifeCycleConfig "$LIFECYCLE_CONFIG_NAME")


#!/usr/bin/env bash

if [ $1 == 'train' ]
then
    CURRENT_HOST=$(jq .current_host  /opt/ml/input/config/resourceconfig.json)

    sed -ie "s/PLACEHOLDER_HOSTNAME/$CURRENT_HOST/g" changehostname.c

    gcc -o changehostname.o -c -fPIC -Wall changehostname.c
    gcc -o libchangehostname.so -shared -export-dynamic changehostname.o -ldl

    LD_PRELOAD=/libchangehostname.so train
else
    serve
fi


#!/usr/bin/env bash

if [[ "$1" = "train" ]]; then
     CURRENT_HOST=$(jq .current_host  /opt/ml/input/config/resourceconfig.json)
     sed -ie "s/PLACEHOLDER_HOSTNAME/$CURRENT_HOST/g" changehostname.c
     gcc -o changehostname.o -c -fPIC -Wall changehostname.c
     gcc -o libchangehostname.so -shared -export-dynamic changehostname.o -ldl
     LD_PRELOAD=/libchangehostname.so train
else
     eval "$@"
fi


#!/usr/bin/env bash

if [ $1 == 'train' ]
then
    # Remove all nvidia gl libraries if they exists to run training in SageMaker.
    rm -rf /usr/local/nvidia/lib/libGL*
    rm -rf /usr/local/nvidia/lib/libEGL*
    rm -rf /usr/local/nvidia/lib/libOpenGL*
    rm -rf /usr/local/nvidia/lib64/libGL*
    rm -rf /usr/local/nvidia/lib64/libEGL*
    rm -rf /usr/local/nvidia/lib64/libOpenGL*

    CURRENT_HOST=$(jq .current_host  /opt/ml/input/config/resourceconfig.json)

    sed -ie "s/PLACEHOLDER_HOSTNAME/$CURRENT_HOST/g" /changehostname.c

    gcc -o /changehostname.o -c -fPIC -Wall /changehostname.c
    gcc -o /libchangehostname.so -shared -export-dynamic /changehostname.o -ldl

    LD_PRELOAD=/libchangehostname.so xvfb-run --auto-servernum -s "-screen 0 1400x900x24" train
elif [ $1 == 'serve' ]
then
    serve
fi


#!/bin/bash
#
# Build the docker images.

set -euo pipefail

source scripts/shared.sh

parse_build_std_args "$@"

echo "pulling previous_image: $base_image_ecr_repo for layer cache... "
$(aws ecr get-login --no-include-email --registry-id $base_image_account_id) &>/dev/null || echo 'warning: ecr login failed'
docker pull $base_image_ecr_repo &>/dev/null || echo 'warning: base image pull failed'
docker logout https://$base_image_account_id.dkr.ecr.$aws_region.amazonaws.com &>/dev/null

echo "building image with version_number: $version_number, and tagging as $local_image_tag ... "
docker build \
    --cache-from $base_image_ecr_repo \
    -f vw/docker/$version_number/Dockerfile \
    -t $local_image_tag .


#!/bin/bash
#
# Pushes a locally tagged image to ECR repo.

set -euo pipefail

source scripts/shared.sh

parse_publish_std_args "$@"

echo "logging into ECR."
$(aws ecr get-login --no-include-email --registry-id $aws_account) &>/dev/null || echo 'warning: ecr login failed'
docker tag $local_image_tag $new_ecr_image_tag

echo "pushing local_image: $local_image_tag to ecr_repo: $new_ecr_image_tag "
docker push $new_ecr_image_tag

docker logout https://$aws_account.dkr.ecr.$aws_region.amazonaws.com &>/dev/null


#!/bin/bash
#
# Utility functions for build/test scripts.

function error() {
    >&2 echo $1
    >&2 echo "build usage: $0 [--version-number <major-version>] [--local-image-tag <tag-for-new-image>] [--base-image-ecr-repo <base ecr repo to cache from>] [--region <aws-region>]"
    >&2 echo "publish usage: $0 [--local-image-tag <local image tag to push>] [--new-ecr-image-repo <ecr repo to push to>] [--region <aws-region>]"
    exit 1
}

function get_default_region() {
    if [ -n "${AWS_DEFAULT_REGION:-}" ]; then
        echo "$AWS_DEFAULT_REGION"
    else
        aws configure get region
    fi
}

function get_aws_account() {
    aws sts get-caller-identity --query 'Account' --output text
}

function parse_build_std_args() {
    # defaults
    aws_region=$(get_default_region)

    while [[ $# -gt 0 ]]; do
    key="$1"

    case $key in
        -b|--base-image-account-id)
        base_image_account_id="$2"
        shift
        shift
        ;;
        -v|--version-number)
        version_number="$2"
        shift
        shift
        ;;
        -r|--region)
        aws_region="$2"
        shift
        shift
        ;;
        -l|--local-image-tag)
        local_image_tag="$2"
        shift
        shift
        ;;
        -b|--base-image-ecr-repo)
        base_image_ecr_repo="$2"
        shift
        shift
        ;;
        *) # unknown option
        error "unknown option: $1"
        shift
        ;;
    esac
    done

    [[ -z "${base_image_account_id// }" ]] && error 'missing base-image-account-id'
    [[ -z "${version_number// }" ]] && error 'missing version-number'
    [[ -z "${aws_region// }" ]] && error 'missing aws region'
    [[ -z "${local_image_tag// }" ]] && error 'missing local tag name for the built image'
    [[ -z "${base_image_ecr_repo// }" ]] && error 'missing base image to build from'

    true
}

function parse_publish_std_args() {
    # defaults
    aws_region=$(get_default_region)
    aws_account=$(get_aws_account)

    while [[ $# -gt 0 ]]; do
    key="$1"

    case $key in
        -l|--local-image-tag)
        local_image_tag="$2"
        shift
        shift
        ;;
        -r|--region)
        aws_region="$2"
        shift
        shift
        ;;
        -n|--new-ecr-image-tag)
        new_ecr_image_tag="$2"
        shift
        shift
        ;;
        *) # unknown option
        error "unknown option: $1"
        shift
        ;;
    esac
    done

    [[ -z "${aws_account// }" ]] && error 'missing aws aws_account'
    [[ -z "${aws_region// }" ]] && error 'missing aws region'
    [[ -z "${local_image_tag// }" ]] && error 'missing local tag name for the built image'
    [[ -z "${new_ecr_image_tag// }" ]] && error 'missing base image to build from'

    true
}


#!/usr/bin/env bash

pip install gym[atari] gym[box2d] box2d-py>=2.3.5
python gym_envs.py "$@"


#!/usr/bin/env bash
# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
#     http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.

set -euo pipefail

echo =================== $1 execution time ===================

start_time=$2
end_time=`date +%s`
total_time=$(expr $end_time - $start_time + 1)
hours=$((total_time/60/60%24))
minutes=$((total_time/60%60))
secs=$((total_time%60))

(( $hours > 0 )) && printf '%d hours ' $hours
(( $minutes > 0 )) && printf '%d minutes ' $minutes
(( $hours > 0 || $minutes > 0 )) && printf 'and '
printf '%d seconds\n\n' $secs


#!/bin/bash

curl -LO https://github.com/apache/spark/archive/branch-2.2.zip
unzip -o branch-2.2.zip "spark-branch-2.2/python/pyspark/*"
cp -r spark-branch-2.2/python/pyspark ../src/


#!/bin/bash
# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
#     http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.

#
# Build the docker images.
#

set -euo pipefail

source scripts/shared.sh

parse_std_args "$@"

aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin 137112412989.dkr.ecr.us-west-2.amazonaws.com

echo "building image ${version} ... "
echo "building image under ${build_context}/docker/${framework_version} ... "
docker build \
    -f ${build_context}/docker/${framework_version}/Dockerfile.${processor} \
    -t ${repository}:${version} \
    --build-arg REGION=${REGION} \
    -t sagemaker-spark:latest \
    ${build_context}

docker logout https://137112412989.dkr.ecr.us-west-2.amazonaws.com


#!/bin/bash
# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
#     http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.

#
# Calls EC2 describe-instance-types to form a json file used by the Spark container for choosing Spark config.
# This will fetch more instance types than needed, but that's OK.
#

set -euo pipefail

source scripts/shared.sh

parse_std_args "$@"

dest_path=${build_context}/aws-config
mkdir -p ${dest_path}
dest_path=${dest_path}/ec2-instance-type-info.json

echo "Fetching EC2 instance type info to ${dest_path} ..."
aws ec2 describe-instance-types --region ${aws_region} | \
  jq -c '[.InstanceTypes[] |
  {
    InstanceType: .InstanceType,
    VCpuInfo: .VCpuInfo,
    MemoryInfo: .MemoryInfo,
    GpuInfo: .GpuInfo
  }] |
  sort_by(.InstanceType)' > ${dest_path}


#!/bin/bash
# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
#     http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.

#
# Publish images to your ECR account.
#

set -euo pipefail

source scripts/shared.sh

parse_std_args "$@"

$(aws ecr get-login --no-include-email --region ${aws_region} --registry-id ${aws_account})
docker tag ${repository}:${version} ${aws_account}.dkr.ecr.${aws_region}.amazonaws.com/${repository}:${version}
docker push ${aws_account}.dkr.ecr.${aws_region}.amazonaws.com/${repository}:${version}
docker logout https://${aws_account}.dkr.ecr.${aws_region}.amazonaws.com


#!/bin/bash
#
# Utility functions for build/test scripts.

function error() {
    >&2 echo $1
    >&2 echo "usage: $0 [--region <aws-region>] [--spark-version <spark MAJOR.MINOR version>] [--processor <cpu|gpu>]\
     [--use-case <processing|training>] [--framework-version <py3|py37>] [--sm-version <sagemaker MAJOR.MINOR version?]"
    exit 1
}

function get_aws_account() {
    aws --region $1 sts --endpoint-url https://sts.$1.amazonaws.com get-caller-identity --query 'Account' --output text
}

function parse_std_args() {
    while [[ $# -gt 0 ]]; do
    key="$1"

    case $key in
        -r|--region)
        aws_region="$2"
        shift
        shift
        ;;
        --spark-version)
        spark_version="$2"
        shift
        shift
        ;;
        --processor)
        processor="$2"
        shift
        shift
        ;;
        --use-case)
        use_case="$2"
        shift
        shift
        ;;
        --framework-version)
        framework_version="$2"
        shift
        shift
        ;;
        --sm-version)
        sm_version="$2"
        shift
        shift
        ;;
        *) # unknown option
        error "unknown option: $1"
        shift
        ;;
    esac
    done

    [[ -z "${aws_region// }" ]] && error 'missing aws region'
    [[ -z "${spark_version// }" ]] && error 'missing spark minor version'
    [[ -z "${processor// }" ]] && processor='cpu'
    [[ -z "${use_case// }" ]] && use_case='processing'
    [[ -z "${framework_version// }" ]] && framework_version='py3'
    [[ -z "${sm_version// }" ]] && sm_version='0.1'

    [[ -z "${DEST_REPO}" ]] && repository=sagemaker-spark-$use_case || repository="${DEST_REPO}"
    [[ -z "${REGION}" ]] && aws_region='us-west-2' || aws_region="${REGION}"

    build_context=./spark/${use_case}/${spark_version}/${framework_version:0:3}
    aws_account=$(get_aws_account $aws_region)
    # spark tagging scheme :
    # sagemaker-spark-[processing|training]:[spark_version]-[processor]-[python/scala version]-v[sm_major].[sm_minor]
    version=${spark_version}-${processor}-${framework_version}-v${sm_version}
    true
}

function error_build_context() {
    >&2 echo $1
    >&2 echo "usage: $0 [--build-context <Docker build context>]"
    exit 1
}

function parse_build_context() {
    # Docker build context
    if [[ $1 == "--build-context" ]]
    then
        build_context="$2"
    fi

    [[ -z "${build_context// }" ]] && error_build_context "missing path to docker build context"

    true
}


#EMPTY FILE AVOID OVERRIDDING ENV VARS
# Specifically, without copying the empty file, SPARK_HISTORY_OPTS will be overriden,
# spark.history.ui.port defaults to 18082, and spark.eventLog.dir defaults to local fs


#!/bin/bash
# This is needed to make sure Java correctly detects CPU/Memory set by the container limits
java -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap -jar /usr/local/lib/sparkml-serving-3.3.jar


find . -name \*.hpp -or -name \*.cpp \
	| xargs cpplint --linelength=120 --filter=-build/c++11 2>&1 \
	| grep -v "Done processing "


#!/bin/bash
#
# Build all the docker images.

set -euo pipefail

DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"

${DIR}/build.sh --version 1.14.0 --arch eia
${DIR}/build.sh --version 1.15.0 --arch cpu
${DIR}/build.sh --version 1.15.0 --arch gpu
${DIR}/build.sh --version 2.1.0 --arch cpu
${DIR}/build.sh --version 2.1.0 --arch gpu


#!/bin/bash
#
# Build the docker images.

set -euo pipefail

source scripts/shared.sh

parse_std_args "$@"

get_ei_executable

echo "pulling previous image for layer cache... "
aws ecr get-login-password --region ${aws_region} \
    | docker login \
        --password-stdin \
        --username AWS \
        "${aws_account}.dkr.ecr.${aws_region}.amazonaws.com/${repository}" &>/dev/null || echo 'warning: ecr login failed'
docker pull $aws_account.dkr.ecr.$aws_region.amazonaws.com/$repository:$full_version-$device &>/dev/null || echo 'warning: pull failed'
docker logout https://$aws_account.dkr.ecr.$aws_region.amazonaws.com &>/dev/null

echo "building image... "
cp -r docker/build_artifacts/* docker/$short_version/
docker build \
    --cache-from $aws_account.dkr.ecr.$aws_region.amazonaws.com/$repository:$full_version-$device \
    --build-arg TFS_VERSION=$full_version \
    --build-arg TFS_SHORT_VERSION=$short_version \
    -f docker/$short_version/Dockerfile.$arch \
    -t $repository:$full_version-$device \
    -t $repository:$short_version-$device \
    docker/$short_version/

remove_ei_executable


#!/bin/bash
#
# Some example curl requests to try on local docker containers.

curl -X POST --data-binary @test/resources/inputs/test.json -H 'Content-Type: application/json' -H 'X-Amzn-SageMaker-Custom-Attributes: tfs-model-name=half_plus_three' http://localhost:8080/invocations
curl -X POST --data-binary @test/resources/inputs/test-gcloud.jsons -H 'Content-Type: application/json' -H 'X-Amzn-SageMaker-Custom-Attributes: tfs-model-name=half_plus_three' http://localhost:8080/invocations
curl -X POST --data-binary @test/resources/inputs/test-generic.json -H 'Content-Type: application/json' -H 'X-Amzn-SageMaker-Custom-Attributes: tfs-model-name=half_plus_three' http://localhost:8080/invocations
curl -X POST --data-binary @test/resources/inputs/test.csv -H 'Content-Type: text/csv' -H 'X-Amzn-SageMaker-Custom-Attributes: tfs-model-name=half_plus_three' http://localhost:8080/invocations
curl -X POST --data-binary @test/resources/inputs/test-cifar.json -H 'Content-Type: application/json' -H 'X-Amzn-SageMaker-Custom-Attributes: tfs-model-name=cifar' http://localhost:8080/invocations


#!/bin/bash
#
# Publish all images to your ECR account.

set -euo pipefail

DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"

${DIR}/publish.sh --version 1.14.0 --arch eia
${DIR}/publish.sh --version 1.15.0 --arch cpu
${DIR}/publish.sh --version 1.15.0 --arch gpu
${DIR}/publish.sh --version 2.1.0 --arch cpu
${DIR}/publish.sh --version 2.1.0 --arch gpu


#!/bin/bash
#
# Publish images to your ECR account.

set -euo pipefail

source scripts/shared.sh

parse_std_args "$@"

aws ecr get-login-password --region ${aws_region} \
    | docker login \
        --password-stdin \
        --username AWS \
        "${aws_account}.dkr.ecr.${aws_region}.amazonaws.com/${repository}"
docker tag $repository:$full_version-$device $aws_account.dkr.ecr.$aws_region.amazonaws.com/$repository:$full_version-$device
docker tag $repository:$full_version-$device $aws_account.dkr.ecr.$aws_region.amazonaws.com/$repository:$short_version-$device
docker push $aws_account.dkr.ecr.$aws_region.amazonaws.com/$repository:$full_version-$device
docker push $aws_account.dkr.ecr.$aws_region.amazonaws.com/$repository:$short_version-$device
docker logout https://$aws_account.dkr.ecr.$aws_region.amazonaws.com


#!/bin/bash
#
# Utility functions for build/test scripts.

function error() {
    >&2 echo $1
    >&2 echo "usage: $0 [--version <major-version>] [--arch (cpu*|gpu|eia)] [--region <aws-region>]"
    exit 1
}

function get_default_region() {
    if [ -n "${AWS_DEFAULT_REGION:-}" ]; then
        echo "$AWS_DEFAULT_REGION"
    else
        aws configure get region
    fi
}

function get_full_version() {
    echo $1 | sed 's#^\([0-9][0-9]*\.[0-9][0-9]*\)$#\1.0#'
}

function get_short_version() {
    echo $1 | sed 's#\([0-9][0-9]*\.[0-9][0-9]*\)\.[0-9][0-9]*#\1#'
}

function get_aws_account() {
    aws --region $AWS_DEFAULT_REGION sts --endpoint-url https://sts.$AWS_DEFAULT_REGION.amazonaws.com get-caller-identity --query 'Account' --output text
}

function get_ei_executable() {
    [[ $arch != 'eia' ]] && return

    if [[ -z $(aws s3 ls 's3://amazonei-tensorflow/tensorflow-serving/v'${short_version}'/ubuntu/latest/') ]]; then
        echo 'ERROR: cannot find this version in S3 bucket.'
        exit 1
    fi

    tmpdir=$(mktemp -d)
    tar_file=$(aws s3 ls "s3://amazonei-tensorflow/tensorflow-serving/v${short_version}/ubuntu/latest/" | awk '{print $4}')
    aws s3 cp "s3://amazonei-tensorflow/tensorflow-serving/v${short_version}/ubuntu/latest/${tar_file}" "$tmpdir/$tar_file"

    tar -C "$tmpdir" -xf "$tmpdir/$tar_file"

    find "$tmpdir" -name amazonei_tensorflow_model_server -exec mv {} docker/build_artifacts/ \;
    rm -rf "$tmpdir"
}

function remove_ei_executable() {
    [[ $arch != 'eia' ]] && return

    rm docker/build_artifacts/amazonei_tensorflow_model_server
}

function get_device_type() {
    if [[ $1 = 'eia' ]]; then
        echo 'cpu'
    else
        echo $1
    fi
}

function parse_std_args() {
    # defaults
    arch='cpu'
    version='1.13.0'
    repository='sagemaker-tensorflow-serving'

    aws_region=$(get_default_region)
    aws_account=$(get_aws_account)

    while [[ $# -gt 0 ]]; do
    key="$1"

    case $key in
        -v|--version)
        version="$2"
        shift
        shift
        ;;
        -a|--arch)
        arch="$2"
        shift
        shift
        ;;
        -r|--region)
        aws_region="$2"
        shift
        shift
        ;;
        -p|--repository)
        repository="$2"
        shift
        shift
        ;;
        *) # unknown option
        error "unknown option: $1"
        shift
        ;;
    esac
    done

    [[ -z "${version// }" ]] && error 'missing version'
    [[ "$arch" =~ ^(cpu|gpu|eia)$ ]] || error "invalid arch: $arch"
    [[ -z "${aws_region// }" ]] && error 'missing aws region'

    [[ "$arch" = eia ]] && repository=$repository'-'$arch

    full_version=$(get_full_version $version)
    short_version=$(get_short_version $version)
    device=$(get_device_type $arch)

    true
}


#!/bin/bash
#
# Start a local docker container.

set -euo pipefail

source scripts/shared.sh

parse_std_args "$@"

if [ "$arch" == 'gpu' ]; then
    docker_command='nvidia-docker'
else
    docker_command='docker'
fi

MODEL_DIR="$(cd "test/resources/models" > /dev/null && pwd)"
$docker_command run \
    -v "$MODEL_DIR":/opt/ml/model:ro \
    -p 8080:8080 \
    -e "SAGEMAKER_TFS_NGINX_LOGLEVEL=error" \
    -e "SAGEMAKER_BIND_TO_PORT=8080" \
    -e "SAGEMAKER_SAFE_PORT_RANGE=9000-9999" \
    $repository:$full_version-$device serve > log.txt 2>&1 &


#!/bin/bash
#
# Stop a local docker container.

set -euo pipefail

source scripts/shared.sh

parse_std_args "$@"

docker kill $(docker ps -q --filter ancestor=$repository:$full_version-$device)


#!/bin/bash

ab -k -n 10000 -c 16 -p test/resources/inputs/test.json -T 'application/json' http://localhost:8080/tfs/v1/models/half_plus_three:predict
ab -k -n 10000 -c 16 -p test/resources/inputs/test.json -T 'application/json' http://localhost:8080/invocations
ab -k -n 10000 -c 16 -p test/resources/inputs/test.jsons -T 'application/json' http://localhost:8080/invocations
ab -k -n 10000 -c 16 -p test/resources/inputs/test.csv -T 'text/csv' http://localhost:8080/invocations
ab -k -n 10000 -c 16 -p test/resources/inputs/test-cifar.json -T 'application/json'     -H 'X-Amzn-SageMaker-Custom-Attributes: tfs-model-name=cifar' http://localhost:8080/invocations

# Larger payloads are generated and removed when this script exits.
TEMPFILE='/tmp/perftest_data'
trap 'rm -f $TEMPFILE' EXIT

echo "Generating data"
# Creates a 10MB file with 10000 columns per line.
python test/perf/data_generator.py -c 'text/csv' -s 10000 -p 10 -u MB > $TEMPFILE || exit $?
ab -k -n 10 -c 1 -p "$TEMPFILE" -T 'text/csv' http://localhost:8080/invocations

python test/perf/data_generator.py -c 'application/json' -s 10000 -p 10 -u MB > $TEMPFILE || exit $?
ab -k -n 10 -c 1 -p "$TEMPFILE" -T 'application/json' http://localhost:8080/invocations

python test/perf/data_generator.py -c 'application/jsonlines' -s 10000 -p 10 -u MB > $TEMPFILE || exit $?
ab -k -n 10 -c 1 -p "$TEMPFILE" -T 'application/jsonlines' http://localhost:8080/invocations


#!/bin/bash

instance_type="${1:-c5.xlarge}"
if [[ "$instance_type" == p* ]]; then
    arch='gpu'
else
    arch='cpu'
fi

endpoint_name=$(echo "sagemaker-tensorflow-serving-$instance_type" | tr . -)

aws sagemaker create-endpoint-config \
    --endpoint-config-name $endpoint_name \
    --production-variants '[{
        "VariantName": "variant-name-1",
        "ModelName": "sagemaker-tensorflow-serving-model-'$arch'",
        "InitialInstanceCount": 1,
        "InstanceType": "ml.'$instance_type'"
    }]'

aws sagemaker create-endpoint --endpoint-name $endpoint_name --endpoint-config-name $endpoint_name


#!/bin/bash

set -e

arch=${1:-'cpu'}
aws_region=$(aws configure get region)
aws_account=$(aws --region $aws_region sts --endpoint-url https://sts.$aws_region.amazonaws.com get-caller-identity --query 'Account' --output text)

# change this to match SageMaker execution role in your account
sagemaker_role="arn:aws:iam::$aws_account:role/service-role/AmazonSageMaker-ExecutionRole-20180510T114550"

tar -C test/resources/models -czf /tmp/sagemaker-tensorflow-serving-model.tar.gz .
aws s3 mb s3://sagemaker-$aws_region-$aws_account || true
aws s3 cp /tmp/sagemaker-tensorflow-serving-model.tar.gz s3://sagemaker-$aws_region-$aws_account/sagemaker-tensorflow-serving/test-models/sagemaker-tensorflow-serving-model.tar.gz
rm /tmp/sagemaker-tensorflow-serving-model.tar.gz

aws sagemaker create-model \
    --model-name sagemaker-tensorflow-serving-model-$arch \
    --primary-container '{
        "Image": "'$aws_account'.dkr.ecr.'$aws_region'.amazonaws.com/sagemaker-tensorflow-serving:1.11.1-'$arch'",
        "ModelDataUrl": "s3://sagemaker-'$aws_region'-'$aws_account'/sagemaker-tensorflow-serving/test-models/sagemaker-tensorflow-serving-model.tar.gz",
        "Environment": {
            "SAGEMAKER_TFS_DEFAULT_MODEL_NAME": "half_plus_three"
        }
    }' \
    --execution-role-arn "$sagemaker_role"


#!/bin/bash

endpoint=${1-'sagemaker-tensorflow-serving-cpu-c5-xlarge'}
aws sagemaker delete-endpoint --endpoint-name $endpoint
aws sagemaker delete-endpoint-config --endpoint-config-name $endpoint


#!/bin/bash

for i in $(seq 1 5); do python perftest_endpoint.py --count 5000 --warmup 100 --workers 4 --model sm-c5xl >> sm-perftest.log; done
for i in $(seq 1 5); do python perftest_endpoint.py --count 5000 --warmup 100 --workers 8 --model sm-c5xl >> sm-perftest.log; done
for i in $(seq 1 5); do python perftest_endpoint.py --count 5000 --warmup 100 --workers 16 --model sm-c5xl >> sm-perftest.log; done
for i in $(seq 1 5); do python perftest_endpoint.py --count 5000 --warmup 100 --workers 32 --model sm-c5xl >> sm-perftest.log; done
for i in $(seq 1 5); do python perftest_endpoint.py --count 5000 --warmup 100 --workers 4 --model sm-c518xl >> sm-perftest.log; done
for i in $(seq 1 5); do python perftest_endpoint.py --count 5000 --warmup 100 --workers 8 --model sm-c518xl >> sm-perftest.log; done
for i in $(seq 1 5); do python perftest_endpoint.py --count 5000 --warmup 100 --workers 16 --model sm-c518xl >> sm-perftest.log; done
for i in $(seq 1 5); do python perftest_endpoint.py --count 5000 --warmup 100 --workers 32 --model sm-c518xl >> sm-perftest.log; done
for i in $(seq 1 5); do python perftest_endpoint.py --count 5000 --warmup 100 --workers 64 --model sm-c518xl >> sm-perftest.log; done

for i in $(seq 1 5); do python perftest_endpoint.py --count 5000 --warmup 100 --workers 4 --model tfs-c5xl >> tfs-perftest.log; done
for i in $(seq 1 5); do python perftest_endpoint.py --count 5000 --warmup 100 --workers 8 --model tfs-c5xl >> tfs-perftest.log; done
for i in $(seq 1 5); do python perftest_endpoint.py --count 5000 --warmup 100 --workers 16 --model tfs-c5xl >> tfs-perftest.log; done
for i in $(seq 1 5); do python perftest_endpoint.py --count 5000 --warmup 100 --workers 32 --model tfs-c5xl >> tfs-perftest.log; done
for i in $(seq 1 5); do python perftest_endpoint.py --count 5000 --warmup 100 --workers 4 --model tfs-c518xl >> tfs-perftest.log; done
for i in $(seq 1 5); do python perftest_endpoint.py --count 5000 --warmup 100 --workers 8 --model tfs-c518xl >> tfs-perftest.log; done
for i in $(seq 1 5); do python perftest_endpoint.py --count 5000 --warmup 100 --workers 16 --model tfs-c518xl >> tfs-perftest.log; done
for i in $(seq 1 5); do python perftest_endpoint.py --count 5000 --warmup 100 --workers 32 --model tfs-c518xl >> tfs-perftest.log; done
for i in $(seq 1 5); do python perftest_endpoint.py --count 5000 --warmup 100 --workers 64 --model tfs-c518xl >> tfs-perftest.log; done
for i in $(seq 1 5); do python perftest_endpoint.py --count 5000 --warmup 100 --workers 128 --model tfs-c518xl >> tfs-perftest.log; done

for i in $(seq 1 5); do python test/perf/perftest_endpoint.py --count 5000 --warmup 100 --workers 4 --model tfs-p2xl >> tfs-perftest.log; done
for i in $(seq 1 5); do python test/perf/perftest_endpoint.py --count 5000 --warmup 100 --workers 8 --model tfs-p2xl >> tfs-perftest.log; done
for i in $(seq 1 5); do python test/perf/perftest_endpoint.py --count 5000 --warmup 100 --workers 16 --model tfs-p2xl >> tfs-perftest.log; done
for i in $(seq 1 5); do python test/perf/perftest_endpoint.py --count 5000 --warmup 100 --workers 32 --model tfs-p2xl >> tfs-perftest.log; done
for i in $(seq 1 5); do python test/perf/perftest_endpoint.py --count 5000 --warmup 100 --workers 64 --model tfs-p2xl >> tfs-perftest.log; done
for i in $(seq 1 5); do python test/perf/perftest_endpoint.py --count 5000 --warmup 100 --workers 4 --model tfs-p316xl >> tfs-perftest.log; done
for i in $(seq 1 5); do python test/perf/perftest_endpoint.py --count 5000 --warmup 100 --workers 8 --model tfs-p316xl >> tfs-perftest.log; done
for i in $(seq 1 5); do python test/perf/perftest_endpoint.py --count 5000 --warmup 100 --workers 16 --model tfs-p316xl >> tfs-perftest.log; done
for i in $(seq 1 5); do python test/perf/perftest_endpoint.py --count 5000 --warmup 100 --workers 32 --model tfs-p316xl >> tfs-perftest.log; done
for i in $(seq 1 5); do python test/perf/perftest_endpoint.py --count 5000 --warmup 100 --workers 64 --model tfs-p316xl >> tfs-perftest.log; done
for i in $(seq 1 5); do python test/perf/perftest_endpoint.py --count 5000 --warmup 100 --workers 128 --model tfs-p316xl >> tfs-perftest.log; done


#!/usr/bin/env bash
# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
#     http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.

./execute_tensorflow_training.py train \
--framework-version  1.12 \
--device gpu \
\
--instance-types ml.p3.16xlarge \
\
--instance-counts 1 \
--instance-counts 2 \
--instance-counts 4 \
--instance-counts 8 \
--instance-counts 16 \
\
--py-versions py3 \
\
--subnets # add subnet id here  \
\
--security-groups # add security-group id here


#!/usr/bin/env bash
# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
#     http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.

set -ex

echo "Launching training job using $SM_NUM_GPUS GPUs"

# p3 instances have larger GPU memory, so a higher batch size can be used
GPU_MEM=`nvidia-smi --query-gpu=memory.total --format=csv,noheader -i 0 | awk '{print $1}'`
if [ $GPU_MEM -gt 15000 ] ; then BATCH_SIZE=256; else BATCH_SIZE=128; fi

# Training
python -W ignore train_imagenet_resnet_hvd.py --num_epochs 90 --synthetic -b $BATCH_SIZE \
	--lr_decay_mode poly --warmup_epochs 10 --clear_log


#!/usr/bin/env bash

execute_tensorflow_training.py train \
--framework-version  1.12 \
--device gpu \
\
--instance-types ml.p3.16xlarge \
\
--instance-counts 1 \
--instance-counts 2 \
--instance-counts 4 \
\
--py-versions py3 \
\
--subnets subnet-125fb674  \
\
--security-groups sg-ce5dd1b4  \
\
--batch-sizes 64 \
\
-- --num_batches=1000 --model vgg16 \
    --variable_update horovod --horovod_device gpu --use_fp16 --summary_verbosity 1 --save_summaries_steps 10


#!/usr/bin/env bash
# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the 'License'). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
#     http://aws.amazon.com/apache2.0/
#
# or in the 'license' file accompanying this file. This file is
# distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.

set -ex
python script.py


#!/bin/bash

dataset_name=example
train_folder_path=../data/${dataset_name}_train_csv/
test_folder_path=../data/${dataset_name}_test_csv/

batch_size=64
batch_size_test=512
selected_layer_idx=-1
pooling_strategy=CLS_TOKEN
max_length=64
lr=2e-4
lr_bert=5e-6
entropy_weight=5
num_warmup_steps=1000
seed=1

model_path=../results/${dataset_name}_pair_BERT-finetune_layer${selected_layer_idx}\
_${pooling_strategy}\
_maxlen${max_length}\
_bs${batch_size}\
_lr-bert${lr_bert}\
_lr${lr}\
_warmup${num_warmup_steps}\
_entropy${entropy_weight}\
_seed${seed}/

python -u ../../main.py \
    --model_path=${model_path} \
    --train_folder_path=${train_folder_path} \
    --test_folder_path=${test_folder_path} \
    --is_csv_header \
    --dim_input=768 \
    --number_clusters=64 \
    --dim_hidden=8 \
    --num_layers_posterior=0 \
    --batch_size=${batch_size} \
    --batch_size_test=${batch_size_test} \
    --lr=${lr} \
    --num_warmup_steps=${num_warmup_steps} \
    --lr_prior=0.1 \
    --num_steps_prior=1 \
    --init=0.0 \
    --clip=1.0 \
    --epochs=1 \
    --log_interval=10 \
    --check_val_test_interval=10000 \
    --save_per_num_epoch=100 \
    --num_bad_epochs=10 \
    --seed=${seed} \
    --entropy_weight=${entropy_weight} \
    --num_workers=0 \
    --cuda \
    --lr_bert=${lr_bert} \
    --max_length=${max_length} \
    --pooling_strategy=${pooling_strategy} \
    --selected_layer_idx=${selected_layer_idx}


#!/bin/sh
set -eux

VENV=.venv_cfn_lint

# Install to separate venv to avoid circular dependency; cfn-lint depends on samtranslator
# See https://github.com/aws/serverless-application-model/issues/1042
if [ ! -d "${VENV}" ]; then
    python3 -m venv "${VENV}"
fi

"${VENV}/bin/python" -m pip install cfn-lint==0.75.0 --upgrade --quiet
"${VENV}/bin/cfn-lint" --format parseable