#!/bin/bash

# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.

# Reads authorized keys blob $3 and prints verified, unexpired keys
# Openssl to use provided as $1
# Signer public key file path provided as $2

# Quick script to generate a CA, intermediate, and end certificate

if [ -z "${1}" ] ; then
    echo "No openssl provided"
    exit 1
fi

if [ -z "${2}" ] ; then
    echo "No target directory provided"
    exit 2
fi

OPENSSL="${1}"
certpath="${2}"

# Configure the CA
mkdir -p "${certpath}/ca.db.certs"
touch "${certpath}/ca.db.index"
echo 01 > "${certpath}/ca.db.serial"

cat > "${certpath}/ca.conf" <<'EOF'
default_ca = ca_default

[ca_default]
dir = REPLACE_WITH_CERTPATH
certs = $dir
new_certs_dir = $dir/ca.db.certs
database = $dir/ca.db.index
serial = $dir/ca.db.serial
RANDFILE = $dir/ca.db.rand
certificate = $dir/ca.crt
private_key = $dir/ca.key
default_days = 1
default_crl_days = 1
default_md = sha256
preserve = no
policy = generic_policy
x509_extensions = usr_cert

[generic_policy]
countryName = optional
stateOrProvinceName = optional
localityName = optional
organizationName = optional
organizationalUnitName = optional
commonName = supplied
emailAddress = optional

[usr_cert]
authorityInfoAccess = OCSP;URI:http://localhost:8080
subjectKeyIdentifier=hash
authorityKeyIdentifier=keyid,issuer

[v3_ocsp]
basicConstraints = CA:FALSE
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
extendedKeyUsage = OCSPSigning

[v3_ca]
subjectKeyIdentifier=hash
authorityKeyIdentifier=keyid:always,issuer:always
basicConstraints = CA:TRUE

[req]
distinguished_name = req_distinguished_name

[req_distinguished_name]

EOF

sed -i "s|REPLACE_WITH_CERTPATH|${certpath}|" "${certpath}/ca.conf"

# Generate the CA
"${OPENSSL}" genrsa -out "${certpath}/ca.key" 2048 > /dev/null 2>&1
"${OPENSSL}" req -x509 -new -nodes -key "${certpath}/ca.key" -sha256 -days 1 -out "${certpath}/ca.crt" -subj "/CN=managedssh.amazonaws.com" > /dev/null 2>&1
"${OPENSSL}" x509 -in "${certpath}/ca.crt" -outform PEM -out "${certpath}/ca.pem"
subject=$("${OPENSSL}" x509 -noout -subject -in "${certpath}/ca.pem" | sed -n -e 's/^.*CN=//p')
# Add "# subject" to start
sed -i '1s;^;# '"$subject"'\n;' "${certpath}/ca.crt"

# Configure the intermediary
mkdir -p "${certpath}/intermediate.db.certs"
touch "${certpath}/intermediate.db.index"
echo 01 > "${certpath}/intermediate.db.serial"

cat > "${certpath}/intermediate.conf" <<'EOF'
default_ca = ca_default

[ca_default]
dir = REPLACE_WITH_CERTPATH
certs = $dir
new_certs_dir = $dir/intermediate.db.certs
database = $dir/intermediate.db.index
serial = $dir/intermediate.db.serial
RANDFILE = $dir/intermediate.db.rand
certificate = $dir/intermediate.crt
private_key = $dir/intermediate.key
default_days = 1
default_crl_days = 1
default_md = sha256
preserve = no
policy = generic_policy
x509_extensions = usr_cert

[generic_policy]
countryName = optional
stateOrProvinceName = optional
localityName = optional
organizationName = optional
organizationalUnitName = optional
commonName = supplied
emailAddress = optional

[usr_cert]
authorityInfoAccess = OCSP;URI:http://localhost:8080
subjectKeyIdentifier=hash
authorityKeyIdentifier=keyid,issuer

[v3_ocsp]
basicConstraints = CA:FALSE
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
extendedKeyUsage = OCSPSigning

[req]
distinguished_name = req_distinguished_name

[req_distinguished_name]

EOF

sed -i "s|REPLACE_WITH_CERTPATH|${certpath}|" "${certpath}/intermediate.conf"

# Generate & sign the intermediary
"${OPENSSL}" genrsa -out "${certpath}/intermediate.key" 2048 > /dev/null 2>&1
"${OPENSSL}" req -new -nodes -config "${certpath}/ca.conf" -key "${certpath}/intermediate.key" -out "${certpath}/intermediate.csr" -extensions v3_ocsp -subj "/CN=intermediate.managedssh.amazonaws.com" > /dev/null 2>&1
yes | "${OPENSSL}" ca -config "${certpath}/ca.conf" -in "${certpath}/intermediate.csr" -cert "${certpath}/ca.crt" -keyfile "${certpath}/ca.key" -out "${certpath}/intermediate.crt" -extensions v3_ocsp -extensions v3_ca > /dev/null 2>&1
"${OPENSSL}" x509 -in "${certpath}/intermediate.crt" -outform PEM -out "${certpath}/intermediate.pem"

# Generate and sign the test cert
"${OPENSSL}" genrsa -out "${certpath}/unittest.key" 2048 > /dev/null 2>&1
"${OPENSSL}" req -new -nodes -config "${certpath}/intermediate.conf" -key "${certpath}/unittest.key" -out "${certpath}/unittest.csr" -subj "/CN=unittest.managedssh.amazonaws.com" > /dev/null 2>&1
yes | "${OPENSSL}" ca -config "${certpath}/intermediate.conf" -in "${certpath}/unittest.csr" -cert "${certpath}/intermediate.crt" -keyfile "${certpath}/intermediate.key" -out "${certpath}/unittest.crt" > /dev/null 2>&1
"${OPENSSL}" x509 -in "${certpath}/unittest.crt" -outform PEM -out "${certpath}/unittest.pem"


#!/bin/bash

# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.

# Reads authorized keys blob $3 and prints verified, unexpired keys
# Openssl to use provided as $1
# Signer public key file path provided as $2

# Unit test helper.  Takes a directory containing key blobs in files, signs the data in each, and consolidates into a final output.

if [ -z "${1}" ] ; then
    echo "No openssl provided"
    exit 1
fi

if [ -z "${2}" ] ; then
    echo "Signing key not provided."
    exit 2
fi

if [ -z "${3}" ] ; then
    echo "Input directory not provided."
    exit 3
fi

if [ -z "${4}" ] ; then
    echo "Target file not provided."
    exit 4
fi

OPENSSL="${1}"
private_key="${2}"
input_dir="${3}"
target="${4}"

signing_temp=$(mktemp -d /dev/shm/tmp-XXXXXXXX)
trap 'rm -rf "${signing_temp}"' EXIT

for file in "${input_dir}"/* ; do
    # Generate the signature
    "${OPENSSL}" dgst -sha256 -sigopt rsa_padding_mode:pss -sigopt rsa_pss_saltlen:32 -sign "${private_key}" -out "${signing_temp}/signature" "${file}"
    # Base64 encode it
    base64 "${signing_temp}/signature" > "${signing_temp}/encoded"
    # Add the input file dump & its signature to the target
    cat "${file}" "${signing_temp}/encoded" >> "${target}"
    # Append a newline
    echo >> "${target}"
done


#!/bin/bash

# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.

# Reads authorized keys blob $3 and prints verified, unexpired keys
# Openssl to use provided as $1
# Signer public key file path provided as $2

# Authorized keys command unit test script.  Takes the script to test, certificate, CA, input, and expected output.

# TODO: Switch to named args/getopts

if [ -z "${1}" ] ; then
    echo "No openssl provided"
    exit 1
fi

if [ -z "${2}" ] ; then
    echo "Script file not provided"
    exit 2
fi

if [ -z "${3}" ] ; then
    echo "No certificate file provided"
    exit 3
fi

if [ -z "${4}" ] ; then
    echo "No CA file provided"
    exit 4
fi

if [ -z "${5}" ] ; then
    echo "OCSP responses directory not provided"
    exit 5
fi

if [ -z "${6}" ] ; then
    echo "No input file provided"
    exit 6
fi

if [ -z "${7}" ] ; then
    echo "No expected output file provided"
    exit 7
fi

OPENSSL=$1
certificate=$(cat "${3}")
filename="${6##*/}"

expected_output=$(cat "${7}")
expected_exit=0
if [ -z "${expected_output}" ] ; then
    # If expected output is empty then we expect the script not to exit success
    expected_exit=255
fi

testdir=$(mktemp -d /dev/shm/tmp-XXXXXXXX)

# The key fingerprint is for a pre-generated ssh key used in several test inputs
# Some test inputs contain other keys as well
# The test outputs, however, *only* match this key - we expect the other keys to be rejected
test_output=$(${2} -x true -p "${6}" -o "${OPENSSL}" -d "${testdir}" -s "${certificate}" -i "i-abcd1234" -c "unittest.managedssh.amazonaws.com" -a "${4}" -v "${5}" -f "SHA256:F3e4S8/QjcVquqrvmyq9AWAhOxIXfpbpnmDVFdA0sPU")
test_status=$?

exit_status=0
if [[ $test_status -eq $expected_exit && "${test_output}" = "${expected_output}" ]] ; then
    echo "${filename} PASSED"
else
    echo "${filename} FAILED"
    echo "EXPECTED: exit ${expected_exit} with output"
    echo "${expected_output}"
    echo "ACTUAL: exit ${test_status} with output"
    echo "${test_output}"
    exit_status=1
fi

rm -rf "${testdir}"

exit "${exit_status}"


#!/bin/bash

# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.

# Reads authorized keys blob $3 and prints verified, unexpired keys
# Openssl to use provided as $1
# Signer public key file path provided as $2

# Test that package uninstall removes customizations and reinstall re-adds them

TOPDIR=$(dirname "$( cd "$( dirname "$( dirname "${BASH_SOURCE[0]}" )" )" && pwd)")

while getopts ":i:p:z:u:k:l:t:" opt ; do
    case "${opt}" in
        i)
            instance_id="${OPTARG}"
            ;;
        p)
            public_ip="${OPTARG}"
            ;;
        z)
            zone="${OPTARG}"
            ;;
        u)
            osuser="${OPTARG}"
            ;;
        k)
            private_key="${OPTARG}"
            ;;
        l)
            distro="${OPTARG}"
            ;;
        t)
            package_path="${OPTARG}"
            ;;
        *)
            ;;
    esac
done

source "${TOPDIR}/integration-test/config/${distro}"

package_name="${package_path##*/}"

# Construct the test script
scriptfile=$(mktemp /tmp/tmp-XXXXXXXX)
trap 'rm -f "${scriptfile}"'  EXIT

if [ "${SSHD_CONFIG_MODIFIED}" -ne 0 ] ; then
    config_check="\"\$(sudo grep \"^AuthorizedKeysCommand[[:blank:]]/opt/aws/bin/eic_run_authorized_keys[[:blank:]]%u[[:blank:]]%f$\" /etc/ssh/sshd_config)\""
else
    config_check="-f /lib/systemd/system/${SSHD_SERVICE}.service.d/ec2-instance-connect.conf"
fi

echo '#!/bin/bash' > "${scriptfile}"
echo "set -e" >> "${scriptfile}"
printf "%s\n%s\n" "echo \"Uninstalling package\"" "${REMOVE} ec2-instance-connect 2>&1" >> "${scriptfile}"
printf "%s\n%s\n%s\n%s\n" "if [ ${config_check} ] ; then" "echo \"Package was not installed or deconfigured correctly\"" "exit 1" "fi" >> "${scriptfile}"
# We need an extra newline before the custom AKC due to some distros, eg RHEL, not having a newline at end of file
printf "%s\n%s\n" "echo \"Adding override config to sshd_config\"" "printf \"\n%s\n\" \"AuthorizedKeysCommand exit\" | sudo tee -a /etc/ssh/sshd_config" >> "${scriptfile}"
printf "%s\n%s\n" "echo \"Installing package\"" "${INSTALL} /tmp/${package_name} 2>&1" >> "${scriptfile}"
printf "%s\n%s\n%s\n%s\n" "if [ ${config_check} ] ; then" "echo \"Package overwrote existing customizations!\"" "exit 1" "fi" >> "${scriptfile}"

echo "scping test script to instance"
scp -i "${private_key}" -o StrictHostKeyChecking=no "${scriptfile}" "${osuser}@${public_ip}:/tmp/eic_custom_config_test.sh" 2>&1
scp_result="${?}"
if [ "${scp_result}" -ne 0 ] ; then
    exit 1
fi

echo "Running test script"
fail=0
ssh -i "${private_key}" -o StrictHostKeyChecking=no "${osuser}@${public_ip}" 'chmod +x /tmp/eic_custom_config_test.sh ; /tmp/eic_custom_config_test.sh' 2>&1
test_result="${?}"
if [ "${test_result}" -ne 0 ] ; then
    fail=1
fi

if [ "${fail}" -eq 0 ] ; then
    echo "Invoking EIC end-to-end test for final validation"
    "${TOPDIR}/integration-test/test/ssh_test.sh" -i "${instance_id}" -p "${public_ip}" -z "${zone}" -u "${osuser}" -k "${private_key}" -l "${distro}" -t "${package_path}" 2>&1
    ssh_result="${?}"
    if [ "${ssh_result}" -eq 0 ] ; then
        echo "ssh via EIC succeeded but should have failed!"
        fail=1
    fi
fi

# Reset back to start-of-test conditions
clean_cmd="${REMOVE} ec2-instance-connect  2>&1 ; sudo sed -i \"\~^AuthorizedKeysCommand[[:blank:]]exit\$~d\" /etc/ssh/sshd_config ; ${INSTALL} /tmp/${package_name} 2>&1"
ssh -i "${private_key}" -o StrictHostKeyChecking=no "${osuser}@${public_ip}" "${clean_cmd}"

exit $fail


#!/bin/bash

# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.

# Reads authorized keys blob $3 and prints verified, unexpired keys
# Openssl to use provided as $1
# Signer public key file path provided as $2

# Basic EIC functionality test: push a key and ssh to the instance

while getopts ":i:p:z:u:" opt ; do
    case "${opt}" in
        i)
            instance_id="${OPTARG}"
            ;;
        p)
            public_ip="${OPTARG}"
            ;;
        z)
            zone="${OPTARG}"
            ;;
        u)
            osuser="${OPTARG}"
            ;;
        k)
            private_key="${OPTARG}"
            ;;
        l)
            distro="${OPTARG}"
            ;;
        t)
            package_path="${OPTARG}"
            ;;
        *)
            ;;
    esac
done

echo "Testing EC2 Instance Connect end-to-end ssh functionality..."

keydir=$(mktemp -d /tmp/tmp-XXXXXXXX)
trap 'rm -rf "${keydir}"' EXIT

keyfile="${keydir}/eic_test_key"
ssh-keygen -t rsa -b 2048 -f "${keyfile}" -P "" -q

# We will make 3 tries as certain elements (eg network calls) may cause transient failure
try="0"
success=1

while [ $try -lt 3 ] ; do
    # Make 3 attempts to push a key
    awstry="0"
    while [ $awstry -lt 3 ] ; do
        aws ec2-instance-connect send-ssh-public-key --region us-west-2 --instance-id "${instance_id}" --availability-zone "${zone}" --instance-os-user "${osuser}" --ssh-public-key "file://${keyfile}.pub" 1>/dev/null
        aws_code=$?
        if [ $aws_code -ne 0 ] ; then
            sleep 5
            awstry=$((awstry+1))
            echo "${instance_id} EIC call ${awstry} failed"
        else
            awstry="3"
        fi
    done
    sshtry="0"
    # Make 3 attempts to ssh
    while [ $sshtry -lt 3 ] ; do
        ssh -q -i "${keyfile}" -o StrictHostKeyChecking=no "${osuser}@${public_ip}" exit 2>&1
        success=$?
        if [ $success -eq 0 ] ; then
            echo "${instance_id} ssh succeeded!"
            sshtry="3"
            try="3"
        else
            sleep 5
            sshtry=$((sshtry+1))
            echo "${instance_id} ssh call ${sshtry} failed"
        fi
    done
    if [ $success -ne 0 ] ; then
        try=$((try+1))
        echo "${instance_id} EIC test attempt ${try} failed"
    fi
done

if [ $success -eq 0 ] ; then
    echo "SUCCESS"
else
    echo "FAILURE"
fi

exit $success


#!/bin/bash

# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.

# Reads authorized keys blob $3 and prints verified, unexpired keys
# Openssl to use provided as $1
# Signer public key file path provided as $2

# Test that package uninstall removes customizations and reinstall re-adds them

TOPDIR=$(dirname "$( cd "$( dirname "$( dirname "${BASH_SOURCE[0]}" )" )" && pwd)")

while getopts ":i:p:z:u:k:l:t:" opt ; do
    case "${opt}" in
        i)
            instance_id="${OPTARG}"
            ;;
        p)
            public_ip="${OPTARG}"
            ;;
        z)
            zone="${OPTARG}"
            ;;
        u)
            osuser="${OPTARG}"
            ;;
        k)
            private_key="${OPTARG}"
            ;;
        l)
            distro="${OPTARG}"
            ;;
        t)
            package_path="${OPTARG}"
            ;;
        *)
            ;;
    esac
done

source "${TOPDIR}/integration-test/config/${distro}"

package_name="${package_path##*/}"

# Construct the test script
scriptfile=$(mktemp /tmp/tmp-XXXXXXXX)
trap 'rm -f "${scriptfile}"'  EXIT

if [ "${SSHD_CONFIG_MODIFIED}" -ne 0 ] ; then
    config_check="grep -q \"^AuthorizedKeysCommand[[:blank:]]/opt/aws/bin/eic_run_authorized_keys[[:blank:]]%u[[:blank:]]%f$\" /etc/ssh/sshd_config"
else
    config_check="-f /lib/systemd/system/${SSHD_SERVICE}.service.d/ec2-instance-connect.conf"
fi

echo "#!/bin/bash" > "${scriptfile}"
echo "set -e" >> "${scriptfile}"
printf "%s\n%s\n" "echo \"Uninstalling package\"" "${REMOVE} ec2-instance-connect 2>&1" >> "${scriptfile}"
printf "%s\n%s\n%s\n%s\n" "if [ ${config_check} ] ; then" "echo \"Package was not installed or deconfigured correctly\"" "exit 1" "fi" >> "${scriptfile}"
printf "%s\n%s\n" "echo \"Reinstalling package\"" "${INSTALL} /tmp/${package_name} 2>&1" >> "${scriptfile}"
printf "%s\n%s\n%s\n%s\n" "if [ ! ${config_check} ] ; then" "echo \"Package not installed/configured correctly\"" "exit 1" "fi" >> "${scriptfile}"

echo "scping test script to instance"
scp -i "${private_key}" -o StrictHostKeyChecking=no "${scriptfile}" "${osuser}@${public_ip}:/tmp/eic_uninstall_reinstall_test.sh" 2>&1
scp_result="${?}"
if [ "${scp_result}" -ne 0 ] ; then
    exit 1
fi

echo "Running test script"
ssh -i "${private_key}" -o StrictHostKeyChecking=no "${osuser}@${public_ip}" 'chmod +x /tmp/eic_uninstall_reinstall_test.sh ; /tmp/eic_uninstall_reinstall_test.sh' 2>&1
test_result="${?}"
if [ "${test_result}" -ne 0 ] ; then
    exit 1
fi

echo "Invoking EIC end-to-end test for final validation"
"${TOPDIR}/integration-test/test/ssh_test.sh" -i "${instance_id}" -p "${public_ip}" -z "${zone}" -u "${osuser}" -k "${private_key}" -l "${distro}" -t "${package_path}" 2>&1

exit "${?}"


#!/usr/bin/env bash
# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.
ARTIFACTS_DIRECTORY="$HOME/awsebcli_artifacts"
GIT_COMMIT=`git rev-parse HEAD`

if [[ ! $GIT_BRANCH ]]; then
    GIT_BRANCH = `git rev-parse --abbrev-ref HEAD`
fi
PYTHON_INSTALLATION=$1
PYTHON_VERSION=`${PYTHON_INSTALLATION} -c 'import sys; print(".".join(map(str, sys.version_info[:3])))'`
VENV_ENV_NAME="$PYTHON_VERSION-$GIT_COMMIT"
USAGE_TEXT=$(cat <<usage
This Bash script runs the unit tests of the current Git commit in an exclusive
virtualenv created from the Python installation described by the first argument
to it. If the current branch is a `master` branch (irrespective of the `remote`),
the script generates an EBCLI artifact tar file available for consumers such as an
end-to-end tests runner, and for the purposes of uploading to PyPi.

Usage:

    ./scripts/jenkins/runner.sh <PYTHON_EXECUTABLE>

e.g., to test this project against Python 2.7 and Python 3.6, you would:

    ./scripts/jenkins/runner.sh /usr/local/bin/python2.7

        and

    ./scripts/jenkins/runner.sh /usr/local/bin/python3.6

usage
)
PYTHON_NOT_FOUND="\"$PYTHON_INSTALLATION --version\" did not work. Is \"$PYTHON_INSTALLATION\" really a Python binary?"
STEP_NUMBER=1
SUBSTEP_NUMBER=1

error_if_required_number_of_arguments_have_not_been_passed()
{
    if [ "$#" -ne 1 ] ; then
        echo "$USAGE_TEXT"
        exit 1
    fi
}

exit_upon_failure()
{
    if [[ $? -ne 0 ]] ; then
        echo "Delete Python $PYTHON_VERSION virtualenv before premature exit"
        rm -rf $VENV_ENV_NAME

        exit 1
    fi
    increment_step_number
    echo ""
}

exit_upon_substep_failure()
{
    if [[ $? -ne 0 ]] ; then
        echo "Delete Python $PYTHON_VERSION virtualenv before premature exit"
        rm -rf $VENV_ENV_NAME

        exit 1
    fi
    increment_substep_number
    echo ""
}

print_help_message_if_requested_and_exit()
{
    for i in $*
    do
        if [ "$i" == "--help" ] ; then
            echo "$USAGE_TEXT"
            exit 0
        fi
    done
}

validate_python_version_name()
{
    if [ "$PYTHON_VERSION" == '' ] ; then
        echo "$PYTHON_NOT_FOUND"
        exit 1
    fi
}

step_title()
{
    echo ""
    echo "******************************************************"
    echo "$STEP_NUMBER. $1"
    echo "******************************************************"
}

substep_title()
{
    echo "    $STEP_NUMBER.$SUBSTEP_NUMBER $1"
}

increment_step_number()
{
    STEP_NUMBER=$(($STEP_NUMBER + 1))
}

increment_substep_number()
{
    SUBSTEP_NUMBER=$((SUBSTEP_NUMBER + 1))
}

print_help_message_if_requested_and_exit $*

step_title "Verifying required number of arguments have been passed"
error_if_required_number_of_arguments_have_not_been_passed $*
increment_step_number

step_title "Verifying Python binary path is valid"
validate_python_version_name
increment_step_number

create_and_load_new_virtualenv()
{
    substep_title "Creating new Python $PYTHON_VERSION virtualenv"
    virtualenv -p $PYTHON_INSTALLATION $VENV_ENV_NAME
    exit_upon_substep_failure

    substep_title "Loading Python $PYTHON_VERSION virtualenv"
    source $VENV_ENV_NAME/bin/activate
    exit_upon_substep_failure

    substep_title "Installing pip 21.1"
    pip install pip=="21.1"
    exit_upon_substep_failure
}

step_title "Checking for CVEs"
check_for_cves()
{
    create_and_load_new_virtualenv

    substep_title "Installing AWSEBCLI and dependencies"
    pip install . --no-cache-dir
    exit_upon_substep_failure

    substep_title "Installing package, 'safety'"
    pip install safety --no-cache-dir
    exit_upon_substep_failure

    substep_title "Checking for known security vulnerabilities"
    safety check
    exit_upon_substep_failure

    substep_title "Deleting Python $PYTHON_VERSION virtualenv"
    rm -rf $VENV_ENV_NAME
    exit_upon_substep_failure
    SUBSTEP_NUMBER=1
}
check_for_cves
increment_step_number

step_title "Ensuring AWSEBCLI installs correctly after AWSCLI"
ensure_awsebcli_installs_correctly_after_awscli()
{
    create_and_load_new_virtualenv

    substep_title "Installing AWSCLI and dependencies"
    pip install awscli --no-cache-dir
    exit_upon_substep_failure

    substep_title "Installing AWSEBCLI and dependencies"
    pip install . --no-cache-dir
    exit_upon_substep_failure

    substep_title "Checking for missing dependencies and dependency mismatches in the package set"
    python tests/test_dependencies_mismatch.py
    exit_upon_substep_failure

    substep_title "Deleting Python $PYTHON_VERSION virtualenv"
    rm -rf $VENV_ENV_NAME
    SUBSTEP_NUMBER=1
}
ensure_awsebcli_installs_correctly_after_awscli
increment_step_number

step_title "Ensuring AWSCLI installs correctly after AWSEBCLI"
ensure_awscli_installs_correctly_after_awsebcli()
{
    create_and_load_new_virtualenv

    substep_title "Installing AWSEBCLI and dependencies"
    pip install . --no-cache-dir
    exit_upon_substep_failure

    substep_title "Checking for missing dependencies and dependency mismatches in the package set"
    python tests/test_dependencies_mismatch.py
    exit_upon_substep_failure

    substep_title "Installing AWSEBCLI and dependencies"
    pip install awsebcli --no-cache-dir
    exit_upon_substep_failure

    substep_title "Checking for missing dependencies and dependency mismatches in the package set"
    python tests/test_dependencies_mismatch.py
    exit_upon_substep_failure

    substep_title "Deleting Python $PYTHON_VERSION virtualenv"
    rm -rf $VENV_ENV_NAME
    SUBSTEP_NUMBER=1
}
ensure_awscli_installs_correctly_after_awsebcli
increment_step_number

step_title "Creating new Python $PYTHON_VERSION virtualenv"
virtualenv -p $PYTHON_INSTALLATION $VENV_ENV_NAME
exit_upon_failure

step_title "Installing pip 21.1"
pip install pip=="21.1"
exit_upon_failure

step_title "Loading Python $PYTHON_VERSION virtualenv"
source $VENV_ENV_NAME/bin/activate
exit_upon_failure

step_title "(Re)Installing AWSEBCLI and dependencies using commit $GIT_BRANCH/$GIT_COMMIT"
python scripts/jenkins/install_dependencies
exit_upon_failure

step_title "Checking for missing dependencies and dependency mismatches in the package set"
python tests/test_dependencies_mismatch.py
exit_upon_failure

step_title "Executing unit tests"
python scripts/jenkins/run_unit_tests
exit_upon_failure

step_title "Checking whether branch is a 'master' branch"
if [[ $GIT_BRANCH =~ 'master' ]]; then
    substep_title "Ensuring $ARTIFACTS_DIRECTORY exists"
    if [ ! -d "$ARTIFACTS_DIRECTORY" ]; then
        mkdir "$ARTIFACTS_DIRECTORY"
    fi
    increment_substep_number

    substep_title "Recreating $ARTIFACTS_DIRECTORY/$PYTHON_VERSION"
    if [ -d $ARTIFACTS_DIRECTORY/$PYTHON_VERSION ]; then
        rm -rf $ARTIFACTS_DIRECTORY/$PYTHON_VERSION
    fi
    mkdir $ARTIFACTS_DIRECTORY/$PYTHON_VERSION
    increment_substep_number

    substep_title "Packaging awsebcli to store in $ARTIFACTS_DIRECTORY/$PYTHON_VERSION"
    python setup.py sdist
    mv dist/* $ARTIFACTS_DIRECTORY/$PYTHON_VERSION/
    increment_substep_number
else
    substep_title "Branch is not a 'master' branch; skipping artifact generation"
    increment_substep_number
fi

step_title "Deleting Python $PYTHON_VERSION virtualenv"
rm -rf $VENV_ENV_NAME


#!/bin/bash
#===============================================================================
#!# script: hbase-snapshot-export
#!# authors: ripani
#!# version: v0.2
#!#
#!# Generate a snapshot of all the tables present in a specific hbase namespace.
#!# The snapshots so created are then copied over an S3 bucket or HDFS path.
#!# For legacy clusters, use the s3a:// schema and specify the AWS programmatic
#!# keys.
#!#
#!# If you're transfering data between kerberized clusters, make sure the
#!# clusters belong to the same Kerberos realm.
#===============================================================================
#?#
#?# usage: ./hbase-snapshot-export.sh <HBASE_NS> <SNAPSHOT_PATH> [S3_ACCESS_KEY] [AWS_SECRET_KEY]
#?#        ./hbase-snapshot-export.sh "default" "hdfs://NN:8020/hbase"
#?#        ./hbase-snapshot-export.sh "default" "s3://BUCKET/PREFIX"
#?#        ./hbase-snapshot-export.sh "default" "s3a://BUCKET/PREFIX" "KEY" "SECRET"
#?#
#?#   HBASE_NS                 HBase namespace to backup
#?#   SNAPSHOT_PATH            HDFS or S3 path.
#?#                            Example: s3://BUCKET or hdfs://NN:8020/user/hbase
#?#   AWS_ACCESS_KEY           [Optional] AWS access key for s3a schema
#?#   AWS_SECRET_KEY           [Optional] AWS secret key for s3a schema
#?#
#===============================================================================

# Print the usage helper using the header as source
function usage() {
	[ "$*" ] && echo "$0: $*"
	sed -n '/^#?#/,/^$/s/^#?# \{0,1\}//p' "$0"
	exit -1
}

[[ $# -lt 2 ]] && echo "error: wrong parameters" && usage

#===============================================================================
# Configurations
#===============================================================================
HBASE_NS="$1"
SNAPSHOT_PATH="$2"
AWS_ACCESS_KEY="$3"
AWS_SECRET_KEY="$4"

if [[ -f "/emr/instance-controller/lib/info/extraInstanceData.json" ]]; then
	HBASE_CMD="sudo -u hbase hbase"
else
	HBASE_CMD="hbase"
fi

# Retrieve list tables for the namespace
readarray -t tables < <(echo "list_namespace_tables '$HBASE_NS'" | $HBASE_CMD shell 2> /dev/null | sed -e '1,/TABLE/d' -e '/seconds/,$d' | while IFS='' read -r line || [[ -n "$line" ]]; do echo "$line"; done)

# Generate Snapshots
label="$(date +"%Y%m%d")-$(date +%s)"
for table in "${tables[@]}"; do
	echo "Creating snapshot for table $HBASE_NS:$table"
	$HBASE_CMD snapshot create -n "$label-$HBASE_NS-$table" -t $HBASE_NS:$table
done

# Copy Snapshots to S3
snapshots=$($HBASE_CMD snapshot info -list-snapshots | grep $label | awk '{print $1}')
for s in ${snapshots}; do
	echo "Transfer snapshot $s to $SNAPSHOT_PATH"
	if [[ -z "$AWS_ACCESS_KEY" && -z "$AWS_SECRET_KEY" ]]; then
		$HBASE_CMD org.apache.hadoop.hbase.snapshot.ExportSnapshot -snapshot ${s} -copy-to $SNAPSHOT_PATH
	else
		$HBASE_CMD org.apache.hadoop.hbase.snapshot.ExportSnapshot -Dfs.s3a.access.key=$AWS_ACCESS_KEY -Dfs.s3a.secret.key=$AWS_SECRET_KEY -snapshot ${s} -copy-to $SNAPSHOT_PATH
	fi
done


#!/bin/bash

# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You may not use
# this file except in compliance with the License. A copy of the License is
# located at
#
#     http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied. See the License for the specific language governing permissions and
# limitations under the License.

# The command used to generate the clang-format config file is:
#   clang-format --style=Google -dump-config

echo "Checking version number"
if command -v clang-format-8; then
    CLANG_FORMAT=clang-format-8
elif command -v clang-format; then
    CLANG_FORMAT=clang-format
else
    echo "clang-format-8 or clang-format not found"
    exit 1
fi
VER=$($CLANG_FORMAT --version | cut -f3 -d' ' | cut -f1 -d'.')

if [ "$VER" -ge 8 ]; then
    set -euxo pipefail
    find . -name '*.h' -or -name '*.c' -or -name '*.cpp' | xargs "$CLANG_FORMAT" -i
else
    echo "clang-format version 8 or greater is needed to read the format file."
    exit 1
fi


#!/bin/bash

# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You may not use
# this file except in compliance with the License. A copy of the License is
# located at
#
#     http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied. See the License for the specific language governing permissions and
# limitations under the License.

set -euxo pipefail

PATH=$PWD/build-tools/bin:$PATH
ROOT=$PWD
NUM_CPU_THREADS=$(nproc)

# End to end tests require valid credentials (instance role, etc..)
# Disable for local runs.
if [ -f "/sys/hypervisor/uuid" ]; then
        ONEC2=$(grep -c ec2 /sys/hypervisor/uuid)
        if [ "${ONEC2}" -gt 0 ]; then
            E2E="ON";
        else
            E2E="OFF";
        fi
elif [ -n "$CODEBUILD_BUILD_ARN" ]; then
    E2E="ON";
else
    E2E="OFF";
fi

debug() {
# If the threading test does in fact fail, it does so by crashing.
# Since this sort of bug might not be reproducible, make sure to dump
# some useful information before failing.
    ulimit -c unlimited
    if ! "$@"; then
        if [ -e core.* ]; then
            gdb -x "$ROOT/codebuild/gdb.commands" "$1" core.*
            exit 1
        fi
    fi
}

run_test() {
    PREFIX_PATH="$1"
    shift

    rm -rf build
    mkdir build
    (cd build
    #TODO: EC2 metadata service fails; fix an re-enable end2end tests.
    cmake -DBUILD_AWS_ENC_SDK_CPP=ON -DAWS_ENC_SDK_END_TO_END_TESTS=${E2E} \
        -DAWS_ENC_SDK_KNOWN_GOOD_TESTS=ON \
        -DAWS_ENC_SDK_END_TO_END_EXAMPLES=${E2E} \
        -DCMAKE_C_FLAGS="$CFLAGS" \
        -DCMAKE_CXX_FLAGS="$CXXFLAGS" \
        -DCMAKE_SHARED_LINKER_FLAGS="$LDFLAGS" \
        -DOPENSSL_ROOT_DIR=/deps/install \
        -DVALGRIND_OPTIONS="--gen-suppressions=all;--suppressions=$ROOT/valgrind.suppressions" \
        -DCMAKE_PREFIX_PATH="$PREFIX_PATH" \
        -DBUILD_SHARED_LIBS="$BUILD_SHARED_LIBS" \
        -GNinja \
        .. "$@" 2>&1|head -n 1000)
    cmake --build $ROOT/build -- -v
    (cd build; ctest --output-on-failure -j${NUM_CPU_THREADS})
    (cd build; debug ./tests/test_local_cache_threading) || exit 1
    "$ROOT/codebuild/bin/test-install.sh" "$PREFIX_PATH" "$PWD/build"
}

# Print env variables for debug purposes
env

case "$TEST_MODE" in
    dynamic)
        # Run the full test suite without valgrind, and as a shared library
        export BUILD_SHARED_LIBS=on
        run_test '/deps/install;/deps/shared/install' -DCMAKE_BUILD_TYPE=RelWithDebInfo
        ;;
    static_debug)
        # Also run the test suite as a debug build (probing for -DNDEBUG issues), and as a static library
        export BUILD_SHARED_LIBS=off
        run_test '/deps/install;/deps/static/install' -DCMAKE_BUILD_TYPE=Debug
        ;;
    static_valgrind)
        # Run a lighter weight test suite under valgrind
        export BUILD_SHARED_LIBS=off
        run_test '/deps/install;/deps/static/install' \
            -DCMAKE_BUILD_TYPE=RelWithDebInfo -DREDUCE_TEST_ITERATIONS=TRUE -DVALGRIND_TEST_SUITE=ON
        ;;
    *)
        echo "unknown TEST_MODE: $TEST_MODE"
        exit 1
        ;;
esac


#!/bin/bash

# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You may not use
# this file except in compliance with the License. A copy of the License is
# located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied. See the License for the specific language governing permissions and
# limitations under the License.

./aws-encryption-sdk-specification/util/report.js \
    'include/**/*.h' \
    'source/**/*.c' \
    'tests/**/*.[ch]' \
    'aws-encryption-sdk-cpp/include/**/*.h' \
    'aws-encryption-sdk-cpp/source/**/*.cpp' \
    'aws-encryption-sdk-cpp/tests/**/*.h' \
    'aws-encryption-sdk-cpp/tests/**/*.cpp' \
    'compliance/*.txt'


#!/bin/bash

# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You may not use
# this file except in compliance with the License. A copy of the License is
# located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied. See the License for the specific language governing permissions and
# limitations under the License.

# This script builds and installs AWS-originated dependencies. Since we want to test
# building against both the static and shared variants, we build them twice, installing
# the results under /deps/static/install and /deps/shared/install

# This script uses the following environment variables:
# CC, CXX, CFLAGS, CXXFLAGS, LDFLAGS - Compiler and flags configuration

set -euxo pipefail

build_pkg() {
    INSTALL_DIR=$1
    GIT_URL=$2
    GIT_REF=$3

    shift; shift; shift

    SRC_DIR=/tmp/$(basename $GIT_URL .git)
    BUILD_DIR=/tmp/build

    if ! [ -e $SRC_DIR ]; then
        mkdir -p "$(dirname "$SRC_DIR")"
        git clone --depth 1 --branch $GIT_REF --recurse-submodules "$GIT_URL" "$SRC_DIR"
    fi

    mkdir -p $BUILD_DIR
    (cd $BUILD_DIR &&
     export LD_LIBRARY_PATH=/deps/install &&
     cmake $SRC_DIR "$@" -DCMAKE_INSTALL_PREFIX=$root/install -DCMAKE_BUILD_TYPE=RelWithDebInfo -GNinja \
        -DCMAKE_PREFIX_PATH=/deps/install \
        -DCMAKE_C_FLAGS="$CFLAGS" \
        -DCMAKE_CXX_FLAGS="$CXXFLAGS" \
        -DCMAKE_SHARED_LINKER_FLAGS="$LDFLAGS")
    cmake --build $BUILD_DIR
    cmake --build $BUILD_DIR --target install
    rm -rf $BUILD_DIR
}

mkdir -p /deps

# These must refer to git tags
AWS_CRT_CPP_VERSION="v0.17.28"
AWS_SDK_CPP_VERSION="1.9.263"

for libtype in shared static; do
    root=/deps/$libtype

    if [ $libtype == shared ]; then
        CMAKE_ARGS="-DBUILD_SHARED_LIBS=ON"
        build_pkg $root/install https://github.com/awslabs/aws-crt-cpp.git $AWS_CRT_CPP_VERSION \
	        $CMAKE_ARGS -DUSE_OPENSSL=ON
    else
        CMAKE_ARGS="-DBUILD_SHARED_LIBS=OFF"
    fi

    build_pkg $root/install https://github.com/aws/aws-sdk-cpp.git $AWS_SDK_CPP_VERSION \
            $CMAKE_ARGS -DBUILD_ONLY=kms
done


#!/bin/bash

# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You may not use
# this file except in compliance with the License. A copy of the License is
# located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied. See the License for the specific language governing permissions and
# limitations under the License.

# This script installs nodejs, which we use for running some tests.

set -euxo pipefail

VERSION="node_14.x"
DISTRO="$(awk -F'=' '/UBUNTU_CODENAME/ { print $2 }' /etc/os-release)"

wget --quiet -O - https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -
echo "deb https://deb.nodesource.com/$VERSION $DISTRO main" | tee /etc/apt/sources.list.d/nodesource.list
echo "deb-src https://deb.nodesource.com/$VERSION $DISTRO main" | tee -a /etc/apt/sources.list.d/nodesource.list

apt-get update
apt-get install nodejs


#!/bin/bash

# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0

# This script is similar to |install-shared-deps.sh| except below differences:
# 1. OpenSSL is replaced with awslc.
# 2. LibCurl version is updated to 7.74.
# After this script, awslc(static and shared) and curl will be installed under /deps/install.

set -euxo pipefail

export AWSLC_SRC_DIR=/tmp/awslc
export INSTALL_DIR=/deps/install
export LD_LIBRARY_PATH=${INSTALL_DIR}
export NUM_CPU_THREADS=$(nproc)

function download_awslc() {
    AWSLC_GIT_URL='https://github.com/awslabs/aws-lc.git'
    AWSLC_TAG='main'
    AWSLC_SHALLOW_SINCE="2023-04-07T00:00:01"
    rm -rf ${AWSLC_SRC_DIR}
    mkdir -p ${AWSLC_SRC_DIR}
    git clone --shallow-since ${AWSLC_SHALLOW_SINCE} --branch ${AWSLC_TAG} "${AWSLC_GIT_URL}" "${AWSLC_SRC_DIR}"
}

function build_awslc() {
    BUILD_DIR=/tmp/build/awslc
    rm -rf ${BUILD_DIR}
    mkdir -p ${BUILD_DIR}
    CMAKE_BUILD_COMMAND="cmake ${AWSLC_SRC_DIR} $@ \
        -GNinja \
        -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \
        -DCMAKE_BUILD_TYPE=RelWithDebInfo"
    if [[ -n "${CFLAGS-}" ]]; then
        CMAKE_BUILD_COMMAND="${CMAKE_BUILD_COMMAND} -DCMAKE_C_FLAGS=${CFLAGS}"
    fi
    if [[ -n "${CXXFLAGS-}" ]]; then
        CMAKE_BUILD_COMMAND="${CMAKE_BUILD_COMMAND} -DCMAKE_CXX_FLAGS=${CXXFLAGS}"
    fi
    (cd ${BUILD_DIR} && ${CMAKE_BUILD_COMMAND})
    cmake --build ${BUILD_DIR}
    cmake --build ${BUILD_DIR} --target install
    rm -rf ${BUILD_DIR}
}

function install_libcurl() {
    mkdir /deps/curl
    cd /deps/curl
    wget https://curl.haxx.se/download/curl-7.74.0.tar.gz
    tar xzf curl-*.tar.gz
    cd curl-*/
    # awslc is forked from boringssl.
    # |OPENSSL_IS_AWSLC| macro is equivalent to |OPENSSL_IS_BORINGSSL|.
    # Replacing OPENSSL_IS_BORINGSSL with OPENSSL_IS_AWSLC.
    #
    # TODO: Remove the extra command below when curl has official support for AWS-LC.
    find ./ -type f -exec sed -i -e 's/OPENSSL_IS_BORINGSSL/OPENSSL_IS_AWSLC/g' {} \;
    ./configure --with-ssl=/deps/install \
        --prefix=/deps/install \
        --disable-ldap \
        --without-libidn \
        --without-gnutls \
        --without-nss \
        --without-gssapi
    make -j"${NUM_CPU_THREADS}"
    make install
    cd /
    rm -rf /deps/curl
}

mkdir -p /deps

download_awslc

build_awslc '-DBUILD_SHARED_LIBS=ON'
build_awslc '-DBUILD_SHARED_LIBS=OFF'

install_libcurl


#!/bin/bash

# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You may not use
# this file except in compliance with the License. A copy of the License is
# located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied. See the License for the specific language governing permissions and
# limitations under the License.

# This script installs non-AWS dependencies. Currently this is openssl and libcurl.

# We install openssl primarily to control the version being used, but also to turn on
# -DPURIFY to silence some valgrind warnings.

# We install libcurl because we need a version that links against the version of openssl
# in use, to avoid version conflicts.

# env variables used:
# $OPENSSL_PLATFORM: The openssl platform name (e.g. linux-generic32)
# CC, CXX, CFLAGS, CXXFLAGS, LDFLAGS: The usual compiler flags (as used by the respective package
#                                     configure scripts)
# OPENSSL_TAG: The tag or branch of openssl to build

set -euxo pipefail

mkdir -p /deps

git clone --depth 1 --branch $OPENSSL_TAG https://github.com/openssl/openssl.git /deps/openssl_src
cd /deps/openssl_src
# -DPURIFY tries to avoid confusing valgrind (e.g. avoiding using uninitialized memory as entropy)
./Configure --prefix=/deps/install shared -DPURIFY $OPENSSL_PLATFORM
make depend
make -j8
make install
cd /
rm -rf /deps/openssl_src

# We need a libcurl compatible with our version of openssl as well
mkdir /deps/curl
cd /deps/curl
wget https://curl.haxx.se/download/curl-7.61.1.tar.gz
tar xzf curl-*.tar.gz
cd curl-*/
./configure --with-ssl=/deps/install --prefix=/deps/install \
    --without-gnutls --without-nss --without-gssapi
make -j8
make install
cd /
rm -rf /deps/curl


#!/bin/bash

# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You may not use
# this file except in compliance with the License. A copy of the License is
# located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied. See the License for the specific language governing permissions and
# limitations under the License.

# This script builds and uploads all linux docker images.
# The ECS_REGISTRY environment variable can be used to override which registry to upload to.

set -euxo pipefail

ECS_SERVER="636124823696.dkr.ecr.us-west-2.amazonaws.com"
ECS_REGISTRY=${ECS_REGISTRY:-${ECS_SERVER}/linux-docker-images}
_AUTH_TOKEN=`aws ecr get-login-password --region us-west-2`
docker login --password ${_AUTH_TOKEN}  --username AWS ${ECS_SERVER}

build_image() {
    docker build -t $1 -f $1.Dockerfile .
    docker tag $1:latest $ECS_REGISTRY:$1
    docker push $ECS_REGISTRY:$1
}

build_image ubuntu-latest-x64
build_image ubuntu-latest-x64-awslc


#!/bin/bash

# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You may not use
# this file except in compliance with the License. A copy of the License is
# located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied. See the License for the specific language governing permissions and
# limitations under the License.

# When doing rapid iteration on dockerfiles, the apt package download step can take
# significant time. This script attempts to use an apt-cacher-ng proxy on the host
# to reduce this time. If such a proxy is not running, we'll fall back to directly
# connecting to the distribution's apt repository.

set -euxo pipefail

# Find the host's IP
HOST_IP=`route -n | grep '^0.0.0.0' | perl -pe 's/^\S+\s+(\S+)\s.*/$1/'`

# Is a proxy running?
if nc -e /bin/true $HOST_IP 3142 < /dev/null; then
    echo "=== Enabling apt proxy on http://$HOST_IP:3142"
    echo "Acquire::http { Proxy \"http://$HOST_IP:3142\"; };" > /etc/apt/apt.conf.d/99proxy
fi


#!/bin/bash

# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You may not use
# this file except in compliance with the License. A copy of the License is
# located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied. See the License for the specific language governing permissions and
# limitations under the License.

# This script installs dependencies from APT. If specific additional dependencies are needed,
# the $EXTRA_PACKAGES environment variable can be set from the Dockerfile to avoid having to
# `apt-get update' twice in the dockerfile.

EXTRA_PACKAGES=${EXTRA_PACKAGES:-}

set -euxo pipefail

apt-get -y update
export DEBIAN_FRONTEND=noninteractive

apt-get -y dist-upgrade
apt-get -y install xutils-dev wget build-essential cmake3 git zlib1g-dev awscli valgrind ninja-build libjson-c-dev $EXTRA_PACKAGES
rm -rf /var/cache/apt/* /var/lib/apt/lists/*


#!/bin/bash

# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You may not use
# this file except in compliance with the License. A copy of the License is
# located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied. See the License for the specific language governing permissions and
# limitations under the License.

# This script installs dependencies from APT. If specific additional dependencies are needed,
# the $EXTRA_PACKAGES environment variable can be set from the Dockerfile to avoid having to
# `apt-get update' twice in the dockerfile.

EXTRA_PACKAGES=${EXTRA_PACKAGES:-}

set -euxo pipefail

apt-get -y update
export DEBIAN_FRONTEND=noninteractive

apt-get -y dist-upgrade
apt-get -y install xutils-dev wget build-essential cmake git zlib1g-dev awscli valgrind ninja-build libjson-c-dev clang-format $EXTRA_PACKAGES
rm -rf /var/cache/apt/* /var/lib/apt/lists/*


#!/bin/bash

# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You may not use
# this file except in compliance with the License. A copy of the License is
# located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied. See the License for the specific language governing permissions and
# limitations under the License.

./aws-encryption-sdk-specification/util/test_conditions \
    -s '-r include/ --include *.h' \
    -s 'source/*.c' \
    -t '-r tests/ --include *.[ch]' \
    -s '-r aws-encryption-sdk-cpp/include/ --include *.h' \
    -s '-r aws-encryption-sdk-cpp/source/ --include *.cpp' \
    -t '-r aws-encryption-sdk-cpp/tests/ --include *.h' \
    -t '-r aws-encryption-sdk-cpp/tests/ --include *.cpp' \
    -s 'compliance/*.txt'


#!/bin/bash

# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You may not use
# this file except in compliance with the License. A copy of the License is
# located at
#
#     http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied. See the License for the specific language governing permissions and
# limitations under the License.

set -euxo pipefail

MY_PATH="$(dirname "$0")"
CODEBUILD_BASE="$MY_PATH/.."

PREFIX_PATH="$1"
BUILD_DIR="$2"
BUILD_SHARED_LIBS=${BUILD_SHARED_LIBS:-off}

rm -rf /tmp/TEST_INSTALL
cmake -DCMAKE_INSTALL_PREFIX=/tmp/TEST_INSTALL -DBUILD_SHARED_LIBS="$BUILD_SHARED_LIBS" "$BUILD_DIR"
cmake --build "$BUILD_DIR" --target install

for i in test-install-project test-install-project-cpp; do
    PROJECT="$CODEBUILD_BASE/$i"
    PROJECT_BUILD="$PROJECT/build"

    rm -rf "$PROJECT_BUILD"
    mkdir "$PROJECT_BUILD"
    (cd "$PROJECT_BUILD"; cmake .. -DBUILD_SHARED_LIBS="$BUILD_SHARED_LIBS" -DCMAKE_PREFIX_PATH="$1;/tmp/TEST_INSTALL")
    cmake --build "$PROJECT_BUILD"
    "$PROJECT_BUILD/testapp"
done


#!/bin/bash

# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You may not use
# this file except in compliance with the License. A copy of the License is
# located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied. See the License for the specific language governing permissions and
# limitations under the License.

set -euxo pipefail

mkdir -p /deps

# Ubuntu 14.04's version of cmake3 is too old for the AWS SDK for C++
# (it fails when we try to use it from the encryption SDK's build scripts).
# We'll use the distro version to bootstrap and build a newer version.

# Note that we could skip the distro version and use the ./bootstrap script
# in the cmake distribution ... but this takes way longer, as it'll end up
# compiling a bootstrap cmake single-threaded, then recompile cmake again
# after configuring itself.
mkdir -p /deps/cmake
(cd /deps/cmake;
 wget https://cmake.org/files/v3.12/cmake-3.12.2.tar.gz;
 tar xzvf cmake-*.tar.gz;
 cd cmake-*;
 mkdir build;
 cd build;
 cmake .. -GNinja -DCMAKE_INSTALL_PREFIX=/usr/local;
 ninja;
 ninja install)
rm -rf /deps/cmake


#!/bin/bash

# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You may not use
# this file except in compliance with the License. A copy of the License is
# located at
#
#     http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied. See the License for the specific language governing permissions and
# limitations under the License.

set +ex

OS=$(uname -s)
AWS_SDK_CPP_VER="1.8.32"  #github versioned branch
BUILDROOT="/var/tmp/build-$(date +%s)"
OPENSSL_VER="openssl@1.1" #note this is a brew label, not an exact version reference
BUILDROOT="/var/tmp/build-$(date +%Y%m%d)"
AWS_ENCRYPTION_SDK_C_VER="master"  # Build from master

deps(){
  echo "Installing dependencies"
  if [ $(which brew|grep -c 'brew') -lt 1 ]; then
    echo "Can't find brew, required to build"
    exit 1
  fi
  brew install ${OPENSSL_VER} cmake || true
  mkdir ${BUILDROOT} || true
  mkdir ${INSTALLROOT} || true
}

build_cpp(){
    echo "Building aws-sdk-cpp"
    cd ${BUILDROOT}
    git clone -b ${AWS_SDK_CPP_VER} https://github.com/aws/aws-sdk-cpp.git
    mkdir -p ${BUILDROOT}/build-aws-sdk-cpp ||true
    mkdir -p ${BUILDROOT}/install || true
    cd ${BUILDROOT}/build-aws-sdk-cpp
    # See https://docs.aws.amazon.com/sdk-for-cpp/v1/developer-guide/cmake-params.html#cmake-target-arch
    cmake -G Xcode -DTARGET_ARCH="APPLE" -DBUILD_SHARED_LIBS=ON -DBUILD_ONLY="kms" -DENABLE_UNITY_BUILD=ON ../aws-sdk-cpp
    # This target runs the cpp tests, but doesn't appear to run the other dependancy tests.
    xcodebuild -target ALL_BUILD
    xcodebuild -target install
}

build_csdk(){
    echo "Building aws-encryption-sdk-c"
    cd ${BUILDROOT}
    git clone -b ${AWS_ENCRYPTION_SDK_C_VER} https://github.com/aws/aws-encryption-sdk-c.git
    mkdir -p ${BUILDROOT}/build-aws-encryption-sdk-c || true
    cd $BUILDROOT/build-aws-encryption-sdk-c
    cmake -G Xcode -DBUILD_SHARED_LIBS=ON -DOPENSSL_ROOT_DIR="/usr/local/opt/${OPENSSL_VER}" ../aws-encryption-sdk-c
    xcodebuild -target ALL_BUILD
    xcodebuild -scheme RUN_TESTS
}

if [ "$OS" == 'Darwin' ]; then
    deps
    build_cpp
    build_csdk
else
	echo "Expecting Mac OS; exiting"
fi


#!/bin/bash

# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You may not use
# this file except in compliance with the License. A copy of the License is
# located at
#
#     http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied. See the License for the specific language governing permissions and
# limitations under the License.

set -euxo pipefail

rm -f hello_world.txt.encrypted
rm -f hello_world.txt.decrypted


#!/bin/sh

# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You may not use
# this file except in compliance with the License. A copy of the License is
# located at
#
#     http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied. See the License for the specific language governing permissions and
# limitations under the License.

set -eux

if [ "$#" -eq 0 ]; then
    echo "Specify option -s to start jobs, -e to end jobs, and -c to cleanup bookkeeping"
fi

while getopts ":sec" opt; do
    case $opt in
        s ) # Start CBMC Batch Jobs
            for job in proofs/*/; do
                job=${job%/} #remove trailing slash
                job=${job#*/} #job name
                echo "Starting job $job"
                cbmc-batch \
                    --no-report \
                    --no-coverage \
                    --wsdir proofs/$job \
                    --srcdir ../ \
                    --jobprefix $job-local \
                    --yaml proofs/$job/cbmc-batch.yaml
            done
            ;;
        e ) # Check CBMC Batch Job Results
            result="results.txt"
            if [ -f $result ]; then
                rm $result
            fi
            for Makefile in Makefile-*-local-*; do
                make -f $Makefile monitor
                make -f $Makefile copy
                dir=${Makefile#*-} # directory name from copy
                job=${dir%-local-*-*} # original job name
                check="$( ./check_result.py $dir proofs/$job/cbmc-batch.yaml )"
                echo "$job: $check" >> $result
            done
            ;;
        c ) # Cleanup
            for Makefile in Makefile-*-local-*; do
                make -f $Makefile cleanup
            done
            ;;
        \?)
            echo "Specify option -s to start jobs, -e to end jobs, and -c to cleanup bookkeeping"
            exit 1
    esac
done


#!/bin/bash

# Basic example invocation of the AWS Encryption CLI to decrypt input from the command line
# See full docs for more details:
#  https://docs.aws.amazon.com/encryption-sdk/latest/developer-guide/crypto-cli-examples.html

# Usage: decrypt_command_line.sh $PLAINTEXT

echo "$1" | aws-encryption-cli --decrypt \
                               --suppress-metadata \
                               --input - \
                               --output - \
                               --decode \
                               --buffer \
                               --wrapping-keys discovery=true


#!/bin/bash

# Basic example invocation of the AWS Encryption CLI to decrypt all files in a directory
# See full docs for more details:
#  https://docs.aws.amazon.com/encryption-sdk/latest/developer-guide/crypto-cli-examples.html

# Usage: decrypt_directory.sh $INPUT_DIRECTORY $CMK_ARN $OUTPUT_DIRECTORY

aws-encryption-cli --decrypt \
                   --input $1 --recursive \
                   --wrapping-keys key=$2 \
                   --commitment-policy require-encrypt-require-decrypt \
                   --encryption-context purpose=test \
                   --suppress-metadata \
                   --max-encrypted-data-keys 1 \
                   --buffer \
                   --output $3


#!/bin/bash

# Basic example invocation of the AWS Encryption CLI to decrypt a file
# See full docs for more details:
#  https://docs.aws.amazon.com/encryption-sdk/latest/developer-guide/crypto-cli-examples.html

# Usage: decrypt_file.sh $INPUT_FILE $CMK_ARN $OUTPUT_DIRECTORY

aws-encryption-cli --decrypt \
                   --input $1 \
                   --wrapping-keys key=$2 \
                   --commitment-policy require-encrypt-require-decrypt \
                   --encryption-context purpose=test \
                   --suppress-metadata \
                   --max-encrypted-data-keys 1 \
                   --buffer \
                   --output $3


#!/bin/bash

# Basic example invocation of the AWS Encryption CLI to decrypt a file that was encrypted with multiple
# master keys
# See full docs for more details:
#  https://docs.aws.amazon.com/encryption-sdk/latest/developer-guide/crypto-cli-examples.html

# Usage: decrypt_file_multiple_keys.sh $INPUT_FILE $CMK_ARN_1 $CMK_ARN_2 $OUTPUT_DIRECTORY

aws-encryption-cli --decrypt \
                   --input $1 \
                   --wrapping-keys key=$2 key=$3 \
                   --commitment-policy require-encrypt-require-decrypt \
                   --encryption-context purpose=test \
                   --suppress-metadata \
                   --max-encrypted-data-keys 2 \
                   --buffer \
                   --output $4


#!/bin/bash

# Basic example invocation of the AWS Encryption CLI to encrypt input from the command line
# See full docs for more details:
#  https://docs.aws.amazon.com/encryption-sdk/latest/developer-guide/crypto-cli-examples.html

# Usage: encrypt_command_line.sh $PLAINTEXT $CMK_ARN

echo "$1" | aws-encryption-cli --encrypt \
                               --suppress-metadata \
                               --input - \
                               --output - \
                               --encode \
                               --wrapping-keys key=$2


#!/bin/bash

# Basic example invocation of the AWS Encryption CLI to encrypt all files in a directory
# See full docs for more details:
#  https://docs.aws.amazon.com/encryption-sdk/latest/developer-guide/crypto-cli-examples.html

# Usage: encrypt_directory.sh $INPUT_DIRECTORY $CMK_ARN $OUTPUT_DIRECTORY

aws-encryption-cli --encrypt \
                   --input $1 --recursive \
                   --wrapping-keys key=$2 \
                   --suppress-metadata \
                   --encryption-context purpose=test \
                   --commitment-policy require-encrypt-require-decrypt \
                   --output $3


#!/bin/bash

# Basic example invocation of the AWS Encryption CLI to encrypt a file
# See full docs for more details:
#  https://docs.aws.amazon.com/encryption-sdk/latest/developer-guide/crypto-cli-examples.html

# Usage: encrypt_file.sh $INPUT_FILE $CMK_ARN $OUTPUT_DIRECTORY

aws-encryption-cli --encrypt \
                   --input $1 \
                   --wrapping-keys key=$2 \
                   --suppress-metadata \
                   --encryption-context purpose=test \
                   --commitment-policy require-encrypt-require-decrypt \
                   --output $3


#!/bin/bash

# Basic example invocation of the AWS Encryption CLI to encrypt a file with multiple master keys
# See full docs for more details:
#  https://docs.aws.amazon.com/encryption-sdk/latest/developer-guide/crypto-cli-examples.html

# Usage: encrypt_file_multiple_keys.sh $INPUT_FILE $CMK_ARN_1 $CMK_ARN_2 $OUTPUT_DIRECTORY

aws-encryption-cli --encrypt \
                   --input $1 \
                   --wrapping-keys key=$2 key=$3 \
                   --suppress-metadata \
                   --encryption-context purpose=test \
                   --commitment-policy require-encrypt-require-decrypt \
                   --output $4


#!/bin/sh

SPEC_ROOT=aws-encryption-sdk-specification

# As we make the duvet framework more robust this logic around extracting and rebuilding
# will probably change. For now, do something simple to unblock ourselves.
REBUILD=false

if [ ! -z $1 ] && [ $1 == "rebuild" ] ; then
  echo "Re-extracting spec because it was explicitly requested"
  REBUILD=true
fi

if [ ! -d $SPEC_ROOT/compliance ] ; then
  echo "Compliance directory missing, extracting spec"
  REBUILD=true
fi

if [ $REBUILD == "true" ] ; then
  cd aws-encryption-sdk-specification
  ./util/specification_extract.sh
  cd ..
fi

$SPEC_ROOT/util/report.js \
  $(find src -name '*.dfy') \
  $(find test -name '*.dfy') \
  $(find aws-encryption-sdk-net/Source -name '*.cs') \
  $(find aws-encryption-sdk-net/Test -name '*.cs') \
  $(find compliance_exceptions -name '*.txt')


## Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
## SPDX-License-Identifier: Apache-2.0

#!bin/bash

VERSION=$1
COUNTER=0
STATUS=1

echo "Looking for version $VERSION"

while [  $STATUS -ne 0 ]; do
    mvn org.apache.maven.plugins:maven-dependency-plugin:3.0.1:get \
        -Dartifact=com.amazonaws:aws-encryption-sdk-java:$VERSION:jar -U

    STATUS=$?
    if [ $STATUS -eq 0 ]; then
        echo "Found version $VERSION in Maven Central :)"
        break
    fi

    if [  $((COUNTER+=1)) -eq 15 ]; then
        echo "It has been an awfully long time, you should check Maven Central for issues"
        exit 1
    fi

    echo "Could not find version $VERSION. Trying again."
    sleep 60
done


#!/bin/bash

# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0

./aws-encryption-sdk-specification/util/report.js \
  'src/main/**/*.java' \
  'src/test/**/*.java' \
  'compliance_exceptions/*.java'


#!/bin/bash

# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0

./aws-encryption-sdk-specification/util/test_conditions \
  -s '-r src/main/ --include *.java' \
  -t '-r src/test/ --include *.java' \
  -s 'compliance_exceptions/*.java'


#!/bin/bash
# Minimal wrapper script for upstream requirements install and freeze.
# We do this here rather than as tox commands because tox does not support output redirection.
if [ -z ${1} ]; then
    exit 1
fi

pip install -r requirements.txt
pip install -r dev_requirements/test-requirements.txt
pip freeze > ${1}


echo -n "AWS for Fluent Bit Container Image Version "
cat /AWS_FOR_FLUENT_BIT_VERSION
exec /fluent-bit/bin/fluent-bit -e /fluent-bit/firehose.so -e /fluent-bit/cloudwatch.so -e /fluent-bit/kinesis.so -c /fluent-bit/etc/fluent-bit.conf


#!/bin/bash
export AWS_REGION="us-west-2"
export PROJECT_ROOT="$(pwd)"
export VOLUME_MOUNT_CONTAINER="/out"
export ARCHITECTURE=$(uname -m)
# For arm, uname evaluates to 'aarch64' but everywhere else in the pipline
# we use 'arm64'
if [ "$ARCHITECTURE" = "aarch64" ]; then
    ARCHITECTURE="arm64"
fi

export CW_INTEG_VALIDATOR_IMAGE="${CW_INTEG_VALIDATOR_IMAGE_BASE}:${ARCHITECTURE}"
export S3_INTEG_VALIDATOR_IMAGE="${S3_INTEG_VALIDATOR_IMAGE_BASE}:${ARCHITECTURE}"

test_cloudwatch() {
	export LOG_GROUP_NAME="fluent-bit-integ-test-${ARCHITECTURE}"
	# Tag is used to name the log stream; each test run has a unique (random) log stream name
	export TAG=$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 10)
	docker-compose --file ./integ/test_cloudwatch/docker-compose.test.yml build
	docker-compose --file ./integ/test_cloudwatch/docker-compose.test.yml up --abort-on-container-exit
	sleep 120

	# Creates a file as a flag for the validation failure
	mkdir -p ./integ/out
	touch ./integ/out/cloudwatch-test

	# Validate that log data is present in CW
	docker-compose --file ./integ/test_cloudwatch/docker-compose.validate.yml build
	docker-compose --file ./integ/test_cloudwatch/docker-compose.validate.yml up --abort-on-container-exit
}

clean_cloudwatch() {
	export LOG_GROUP_NAME="fluent-bit-integ-test-${ARCHITECTURE}"
	# Clean up resources that were created in the test
	docker-compose --file ./integ/test_cloudwatch/docker-compose.clean.yml build
	docker-compose --file ./integ/test_cloudwatch/docker-compose.clean.yml up --abort-on-container-exit
}

validate_or_clean_s3() {
	# Validate: validates that appropirate log data is present in the s3 bucket
	# Clean: deletes all the objects from s3 bucket that were created in the test
	export S3_ACTION="${1}"
	docker-compose --file ./integ/test_kinesis/docker-compose.validate-and-clean-s3.yml build
	docker-compose --file ./integ/test_kinesis/docker-compose.validate-and-clean-s3.yml up --abort-on-container-exit
}

test_kinesis() {
	# Generates log data which will be stored on the s3 bucket
	docker-compose --file ./integ/test_kinesis/docker-compose.test.yml build
	docker-compose --file ./integ/test_kinesis/docker-compose.test.yml up --abort-on-container-exit

    # Giving a pause before running the validation test
    # Firehose delivery stream has a buffer time which causes the delay to send the data
	sleep 120

	# Creates a file as a flag for the validation failure
	mkdir -p ./integ/out
	touch ./integ/out/kinesis-test

	validate_or_clean_s3 validate

	if [ -f ./integ/out/kinesis-test ]; then
		# if the file still exists, test failed
		echo "Test failed for kinesis stream."
		exit 1
	fi
}

test_kinesis_streams() {
	# Generates log data which will be stored on the s3 bucket
	docker-compose --file ./integ/test_kinesis/docker-compose.core.test.yml build
	docker-compose --file ./integ/test_kinesis/docker-compose.core.test.yml up --abort-on-container-exit

    # Giving a pause before running the validation test
    # Firehose delivery stream has a buffer time which causes the delay to send the data
	sleep 120

	# Creates a file as a flag for the validation failure
	mkdir -p ./integ/out
	touch ./integ/out/kinesis-test

	validate_or_clean_s3 validate

	if [ -f ./integ/out/kinesis-test ]; then
		# if the file still exists, test failed
		echo "Test failed for kinesis stream."
		exit 1
	fi
}

test_firehose() {
	# Generates log data which will be stored on the s3 bucket
	docker-compose --file ./integ/test_firehose/docker-compose.test.yml build
	docker-compose --file ./integ/test_firehose/docker-compose.test.yml up --abort-on-container-exit

    # Giving a pause before running the validation test
    # Firehose delivery stream has a buffer time which causes the delay to send the data
	sleep 120

	# Creates a file as a flag for the validation failure
	mkdir -p ./integ/out
	touch ./integ/out/firehose-test

	validate_or_clean_s3 validate

	if [ -f ./integ/out/firehose-test ]; then
		# if the file still exists, test failed
		echo "Test failed for firehose."
		exit 1
	fi
}

test_kinesis_firehose() {
	# Generates log data which will be stored on the s3 bucket
	docker-compose --file ./integ/test_firehose/docker-compose.core.test.yml build
	docker-compose --file ./integ/test_firehose/docker-compose.core.test.yml up --abort-on-container-exit

    # Giving a pause before running the validation test
    # Firehose delivery stream has a buffer time which causes the delay to send the data
	sleep 120

	# Creates a file as a flag for the validation failure
	mkdir -p ./integ/out
	touch ./integ/out/firehose-test

	validate_or_clean_s3 validate

	if [ -f ./integ/out/firehose-test ]; then
		# if the file still exists, test failed
		echo "Test failed for firehose."
		exit 1
	fi
}

test_s3() {
	# different S3 prefix for each test
	export S3_PREFIX_PUT_OBJECT="logs/${ARCHITECTURE}/putobject"
	export S3_PREFIX_MULTIPART="logs/${ARCHITECTURE}/multipart"
	# Tag is used in the s3 keys; each test run has a unique (random) tag
	export TAG=$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 10)
	# Generates log data which will be stored on the s3 bucket
	docker-compose --file ./integ/test_s3/docker-compose.test.yml build
	docker-compose --file ./integ/test_s3/docker-compose.test.yml up --abort-on-container-exit

    # Giving a pause before running the validation test
	sleep 20

	# Creates a file as a flag for the validation failure
	mkdir -p ./integ/out
	touch ./integ/out/s3-test

	export S3_ACTION="validate"
	docker-compose --file ./integ/test_s3/docker-compose.validate-s3-multipart.yml build
	docker-compose --file ./integ/test_s3/docker-compose.validate-s3-multipart.yml up --abort-on-container-exit

	if [ -f ./integ/out/s3-test ]; then
		# if the file still exists, test failed
		echo "Test failed for S3."
		exit 1
	fi

	# Creates a file as a flag for the validation failure
	mkdir -p ./integ/out
	touch ./integ/out/s3-test

	export S3_ACTION="validate"
	docker-compose --file ./integ/test_s3/docker-compose.validate-s3-putobject.yml build
	docker-compose --file ./integ/test_s3/docker-compose.validate-s3-putobject.yml up --abort-on-container-exit

	if [ -f ./integ/out/s3-test ]; then
		# if the file still exists, test failed
		echo "Test failed for S3."
		exit 1
	fi
}

clean_s3() {
	validate_or_clean_s3 clean
}
if [ "${1}" = "cloudwatch" ]; then
	export CW_PLUGIN_UNDER_TEST="cloudwatch"
	test_cloudwatch
	clean_cloudwatch

	if [ -f ./integ/out/cloudwatch-test ]; then
		# if the file still exists, test failed
		echo "Test Failed for Cloudwatch."
		exit 1
	fi
fi

if [ "${1}" = "cloudwatch_logs" ]; then
	export CW_PLUGIN_UNDER_TEST="cloudwatch_logs"
	test_cloudwatch
	clean_cloudwatch

	if [ -f ./integ/out/cloudwatch-test ]; then
		# if the file still exists, test failed
		echo "Test Failed for Cloudwatch."
		exit 1
	fi
fi

if [ "${1}" = "clean-cloudwatch" ]; then
	clean_cloudwatch
fi

if [ "${1}" = "kinesis" ]; then
	export S3_PREFIX="kinesis-test"
	export TEST_FILE="kinesis-test"
	export EXPECTED_EVENTS_LEN="1000"
	source ./integ/resources/create_test_resources.sh
	source ./integ/resources/setup_test_environment.sh

	clean_s3 && test_kinesis
fi

if [ "${1}" = "kinesis_streams" ]; then
	export S3_PREFIX="kinesis-test"
	export TEST_FILE="kinesis-test"
	export EXPECTED_EVENTS_LEN="1000"
	source ./integ/resources/create_test_resources.sh
	source ./integ/resources/setup_test_environment.sh

	clean_s3 && test_kinesis_streams
fi

if [ "${1}" = "firehose" ]; then
	export S3_PREFIX="firehose-test"
	export TEST_FILE="firehose-test"
	export EXPECTED_EVENTS_LEN="1000"
	source ./integ/resources/create_test_resources.sh
	source ./integ/resources/setup_test_environment.sh

	clean_s3 && test_firehose
fi

if [ "${1}" = "kinesis_firehose" ]; then
	export S3_PREFIX="firehose-test"
	export TEST_FILE="firehose-test"
	export EXPECTED_EVENTS_LEN="1000"
	source ./integ/resources/create_test_resources.sh
	source ./integ/resources/setup_test_environment.sh

	clean_s3 && test_kinesis_firehose
fi

if [ "${1}" = "s3" ]; then
	export S3_PREFIX="logs"
	export TEST_FILE="s3-test"
	export EXPECTED_EVENTS_LEN="7717"
	source ./integ/resources/create_test_resources.sh
	source ./integ/resources/setup_test_environment.sh

	clean_s3 && test_s3
fi

if [ "${1}" = "clean-s3" ]; then
	source ./integ/resources/setup_test_environment.sh
	clean_s3
fi

if [ "${1}" = "cicd" ]; then
	export CW_PLUGIN_UNDER_TEST="cloudwatch"
	echo "Running tests on Golang CW Plugin"
	test_cloudwatch && clean_cloudwatch
	if [ -f ./integ/out/cloudwatch-test ]; then
		# if the file still exists, test failed
		echo "Test Failed for Cloudwatch (Golang)."
		exit 1
	fi

	export CW_PLUGIN_UNDER_TEST="cloudwatch_logs"
	echo "Running tests on Core C CW Plugin"
	test_cloudwatch && clean_cloudwatch
	if [ -f ./integ/out/cloudwatch-test ]; then
		# if the file still exists, test failed
		echo "Test Failed for Cloudwatch (Core)."
		exit 1
	fi

	source ./integ/resources/setup_test_environment.sh

	# golang kinesis plugin
	export S3_PREFIX="kinesis-test"
	export TEST_FILE="kinesis-test"
	export EXPECTED_EVENTS_LEN="1000"
	clean_s3 && test_kinesis

	# golang firehose plugin
	export S3_PREFIX="firehose-test"
	export TEST_FILE="firehose-test"
	clean_s3 && test_firehose

	# core kinesis plugin
	export S3_PREFIX="kinesis-test"
	export TEST_FILE="kinesis-test"
	export EXPECTED_EVENTS_LEN="1000"
	clean_s3 && test_kinesis_streams

	# core firehose plugin
	export S3_PREFIX="firehose-test"
	export TEST_FILE="firehose-test"
	clean_s3 && test_kinesis_firehose

	export S3_PREFIX="logs"
	export TEST_FILE="s3-test"
	export EXPECTED_EVENTS_LEN="7717"
	source ./integ/resources/create_test_resources.sh
	source ./integ/resources/setup_test_environment.sh

	clean_s3 && test_s3
fi

if [ "${1}" = "delete" ]; then
	source ./integ/resources/delete_test_resources.sh
fi


#!/bin/bash

# Deploys the CloudFormation template to create the stack and necessary resources- kinesis data stream, s3 bucket, and kinesis firehose delivery stream
# Resource (stream, s3, delivery stream) names will start with the stack name followed by the corresponding architecture. "integ-test-fluent-bit-architecture"
ARCHITECTURE=$(uname -m | tr '_' '-')
# For arm, uname evaluates to 'aarch64' but everywhere else in the pipline
# we use 'arm64'
if [ "$ARCHITECTURE" = "aarch64" ]; then
    ARCHITECTURE="arm64"
fi
aws cloudformation deploy --template-file ./integ/resources/cfn-kinesis-s3-firehose.yml --stack-name integ-test-fluent-bit-${ARCHITECTURE} --region us-west-2 --capabilities CAPABILITY_NAMED_IAM


# Delete the CloudFormation stack which created all the resources for running the integration test
ARCHITECTURE=$(uname -m | tr '_' '-')
# For arm, uname evaluates to 'aarch64' but everywhere else in the pipline
# we use 'arm64'
if [ "$ARCHITECTURE" = "aarch64" ]; then
    ARCHITECTURE="arm64"
fi
aws cloudformation delete-stack --stack-name integ-test-fluent-bit-${ARCHITECTURE}


#!/bin/bash

# Using CloudFormation describe-stacks extracts the output values for kinesis stream and s3 bucket name, and sets them as environment variables
ARCHITECTURE=$(uname -m | tr '_' '-')
# For arm, uname evaluates to 'aarch64' but everywhere else in the pipline
# we use 'arm64'
if [ "$ARCHITECTURE" = "aarch64" ]; then
    ARCHITECTURE="arm64"
fi
stackOutputs=$(aws cloudformation describe-stacks --stack-name integ-test-fluent-bit-${ARCHITECTURE} --output text --query 'Stacks[0].Outputs[*].OutputValue')
read -r -a outputArray <<< "$stackOutputs"
export FIREHOSE_STREAM="${outputArray[0]}"
export KINESIS_STREAM="${outputArray[1]}"
export S3_BUCKET_NAME="${outputArray[2]}"


#!/bin/bash

# Use CloudFormation describe-stacks extracts the output values for cloudwatch log group name, s3 bucket name and ecs cluster name, and sets them as environment variables
stackOutputs=$(aws cloudformation describe-stacks --stack-name ${TESTING_RESOURCES_STACK_NAME} --output text --query 'Stacks[0].Outputs[*].OutputValue')
read -r -a outputArray <<< "$stackOutputs"
if [ $PLATFORM == "ECS" ]
then
    export ECS_CLUSTER_NAME="${outputArray[0]}"
    export CW_LOG_GROUP_NAME="${outputArray[1]}"
else
    export CW_LOG_GROUP_NAME="${outputArray[0]}"
fi
stackOutputs=$(aws cloudformation describe-stacks --stack-name ${LOG_STORAGE_STACK_NAME} --output text --query 'Stacks[0].Outputs[*].OutputValue')
read -r -a outputArray <<< "$stackOutputs"
export S3_BUCKET_NAME="${outputArray[0]}"

export AWS_DEFAULT_REGION=${AWS_REGION}

# Set necessary images as env vars
export FLUENT_BIT_IMAGE="${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:latest"
export ECS_APP_IMAGE="906394416424.dkr.ecr.us-west-2.amazonaws.com/load-test-fluent-bit-ecs-app-image:latest"
export EKS_APP_IMAGE="906394416424.dkr.ecr.us-west-2.amazonaws.com/load-test-fluent-bit-eks-app-image:latest"
export ECS_APP_IMAGE_TCP="906394416424.dkr.ecr.us-west-2.amazonaws.com/load-test-fluent-bit-ecs-app-image-tcp:latest"
# Label EKS nodes
if [ $PLATFORM == "EKS" ]
then
    DestinationArray=("cloudwatch" "s3" "kinesis" "firehose")
    for i in "${!DestinationArray[@]}"; do
        export NODE_NAME=$(kubectl get nodes -o json | jq -r ".items[$i].metadata.name")
        kubectl label --overwrite nodes ${NODE_NAME} destination=${DestinationArray[$i]}
    done
fi


mvn clean
mvn compile
mvn assembly:single
mvn package


mvn compile
mvn assembly:single
mvn package


java -cp target/my-app-1.0-SNAPSHOT-jar-with-dependencies.jar com.mycompany.app.App


#!/bin/bash
# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
# 	http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.

set -euo pipefail

# List of arguments the script can accept
ARGUMENT_LIST=(
  "KINESIS_PLUGIN_CLONE_URL"
  "KINESIS_PLUGIN_TAG"
  "KINESIS_PLUGIN_BRANCH"
  "FIREHOSE_PLUGIN_CLONE_URL"
  "FIREHOSE_PLUGIN_TAG"
  "FIREHOSE_PLUGIN_BRANCH"
  "CLOUDWATCH_PLUGIN_CLONE_URL"
  "CLOUDWATCH_PLUGIN_TAG"
  "CLOUDWATCH_PLUGIN_BRANCH"
  "DOCKER_BUILD_FLAGS"
)

# A variable to hold the build arguments for docker build
PLUGIN_BUILD_ARGS=""

# windows or linux
# set by env var in Makefile right now
# setting this by env var ensures it works even on platforms where getopt and longoptions does not work
OS_TYPE="${OS_TYPE}"
DOCKER_BUILD_FLAGS="${DOCKER_BUILD_FLAGS}"

# Go plugin versions can either be set by args to the script, or they will be sourced
# from the windows.versions or linux.version file
KINESIS_PLUGIN_TAG=""
FIREHOSE_PLUGIN_TAG=""
CLOUDWATCH_PLUGIN_TAG=""

# Method to display usage of the script
usage() {
  echo "Usage: $0 [--KINESIS_PLUGIN_CLONE_URL <string>] [--KINESIS_PLUGIN_TAG <string>] [--KINESIS_PLUGIN_BRANCH <string>]\
  [--FIREHOSE_PLUGIN_CLONE_URL <string>] [--FIREHOSE_PLUGIN_TAG <string>] [--FIREHOSE_PLUGIN_BRANCH <string>]\
  [--CLOUDWATCH_PLUGIN_CLONE_URL <string>] [--CLOUDWATCH_PLUGIN_TAG <string>] [--CLOUDWATCH_PLUGIN_BRANCH <string>] \
  [--DOCKER_BUILD_FLAGS <string>]" 1>&2;
  exit 1;
}

# Read arguments
opts=$(getopt \
  --longoptions "$(printf "%s:," "${ARGUMENT_LIST[@]}")" \
  --name "$(basename "$0")" \
  --options "" \
  -- "$@"
)

eval set -- "$opts"

# Build plugin build arguments
#TODO: this bash does not actually work on Developer Macs for some reason
while [ $# -gt 0 ]
do
  case "$1" in
    --KINESIS_PLUGIN_CLONE_URL)
      if [ -n "$2" ];then PLUGIN_BUILD_ARGS="$PLUGIN_BUILD_ARGS --build-arg KINESIS_PLUGIN_CLONE_URL=$2";fi
      shift 2;;
    --KINESIS_PLUGIN_TAG)
      if [ -n "$2" ];then
        PLUGIN_BUILD_ARGS="$PLUGIN_BUILD_ARGS --build-arg KINESIS_PLUGIN_TAG=$2"
        KINESIS_PLUGIN_TAG="$2"
      fi
      shift 2;;
    --KINESIS_PLUGIN_BRANCH)
      if [ -n "$2" ];then PLUGIN_BUILD_ARGS="$PLUGIN_BUILD_ARGS --build-arg KINESIS_PLUGIN_BRANCH=$2";fi
      shift 2;;
    --FIREHOSE_PLUGIN_CLONE_URL)
      if [ -n "$2" ];then PLUGIN_BUILD_ARGS="$PLUGIN_BUILD_ARGS --build-arg FIREHOSE_PLUGIN_CLONE_URL=$2";fi
      shift 2;;
    --FIREHOSE_PLUGIN_TAG)
      if [ -n "$2" ];then
        PLUGIN_BUILD_ARGS="$PLUGIN_BUILD_ARGS --build-arg FIREHOSE_PLUGIN_TAG=$2"
        FIREHOSE_PLUGIN_TAG="$2"
      fi
      shift 2;;
    --FIREHOSE_PLUGIN_BRANCH)
      if [ -n "$2" ];then PLUGIN_BUILD_ARGS="$PLUGIN_BUILD_ARGS --build-arg FIREHOSE_PLUGIN_BRANCH=$2";fi
      shift 2;;
    --CLOUDWATCH_PLUGIN_CLONE_URL)
      if [ -n "$2" ];then PLUGIN_BUILD_ARGS="$PLUGIN_BUILD_ARGS --build-arg CLOUDWATCH_PLUGIN_CLONE_URL=$2";fi
      shift 2;;
    --CLOUDWATCH_PLUGIN_TAG)
      if [ -n "$2" ];then
        PLUGIN_BUILD_ARGS="$PLUGIN_BUILD_ARGS --build-arg CLOUDWATCH_PLUGIN_TAG=$2"
        CLOUDWATCH_PLUGIN_TAG="$2"
      fi
      shift 2;;
    --CLOUDWATCH_PLUGIN_BRANCH)
      if [ -n "$2" ];then PLUGIN_BUILD_ARGS="$PLUGIN_BUILD_ARGS --build-arg CLOUDWATCH_PLUGIN_BRANCH=$2";fi
      shift 2;;
    --DOCKER_BUILD_FLAGS)
      if [ -n "$2" ];then PLUGIN_BUILD_ARGS="$PLUGIN_BUILD_ARGS $2";fi
      shift 2;;
    # End of arguments. End here and break.
    --) shift; break ;;
    # Any other argument is an invalid arg.
    *) usage;;
  esac
done

# Change the directory to root of the repository
scripts=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
cd "${scripts}/.."

if [ -z "$OS_TYPE" ];
then
  echo "OS_TYPE is required"
  exit 1
fi

# Retrieve the latest version based on the platform.
if [ "$OS_TYPE" == "windows" ];
then
  latest_version=$(cat windows.versions | jq -r '.windows[] | select(.latest==true)')
elif [ "$OS_TYPE" == "linux" ];
then
  latest_version=$(cat linux.version | jq -r '.linux')
fi

# For sanity checking, find the number of latest versions present.
latest_counts=$(echo $latest_version | jq -r '.version' | wc -l)

# If latest version is not found or if multiple entries found, then throw an error.
if [ -z "$latest_version" ]
then
  echo "latest version not found for ${OS_TYPE}"
  exit 1
elif [ "$latest_counts" -ne 1 ]; then
  echo "more than 1 latest versions found for ${OS_TYPE}: ${latest_version}"
  exit 1
fi

# Source go plugin versions from latest version, if not set.
if [ -z "$KINESIS_PLUGIN_TAG" ];
then
  KINESIS_PLUGIN_TAG=$(echo "$latest_version" | jq -r '."kinesis-plugin"')
  PLUGIN_BUILD_ARGS="$PLUGIN_BUILD_ARGS --build-arg KINESIS_PLUGIN_TAG=$KINESIS_PLUGIN_TAG"
fi
if [ -z "$FIREHOSE_PLUGIN_TAG" ];
then
  FIREHOSE_PLUGIN_TAG=$(echo "$latest_version" | jq -r '."firehose-plugin"')
  PLUGIN_BUILD_ARGS="$PLUGIN_BUILD_ARGS --build-arg FIREHOSE_PLUGIN_TAG=$FIREHOSE_PLUGIN_TAG"
fi
if [ -z "$CLOUDWATCH_PLUGIN_TAG" ];
then
  CLOUDWATCH_PLUGIN_TAG=$(echo "$latest_version" | jq -r '."cloudwatch-plugin"')
  PLUGIN_BUILD_ARGS="$PLUGIN_BUILD_ARGS --build-arg CLOUDWATCH_PLUGIN_TAG=$CLOUDWATCH_PLUGIN_TAG"
fi

echo "Plugin build arguments for ${OS_TYPE} are: $PLUGIN_BUILD_ARGS"
echo "Docker build flags are: $DOCKER_BUILD_FLAGS"

# Run platform specific build commands
if [ "$OS_TYPE" == "windows" ];
then
  # Create the build output folder
  mkdir -p ./build/windows
  echo "Created build output folder"

  # Build plugin image and then copy the windows plugins
  docker build $PLUGIN_BUILD_ARGS -t aws-fluent-bit-plugins:latest -f ./scripts/dockerfiles/Dockerfile.plugins-windows .
  docker create -ti --name plugin-build-container aws-fluent-bit-plugins:latest bash
  docker cp plugin-build-container:/plugins_windows.tar ./build/windows/plugins_windows.tar
  docker rm -f plugin-build-container
  echo "Copied plugin archive to the build output folder"
fi

if [ "$OS_TYPE" == "linux" ];
then
  docker build $PLUGIN_BUILD_ARGS -t aws-fluent-bit-plugins:latest -f ./scripts/dockerfiles/Dockerfile.plugins .
fi


#!/bin/bash
# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
# 	http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.

set -uo pipefail

#############################################################################################################
# Helper methods to set the required variables.
#############################################################################################################
# Constant
# Number of presently supported OS versions.
SUPPORTED_WINDOWS_OS_RELEASES_NUMBER=2

# Sets ALL_AWS_FOR_FLUENT_BIT_VERSIONS variable with all the supported fluent-bit versions.
set_all_supported_versions() {
  ALL_AWS_FOR_FLUENT_BIT_VERSIONS=$(cat < ../windows.versions | jq -c -r '.windows[].version')
}

# Sets AWS_FOR_FLUENT_BIT_STABLE_VERSION variable with the stable version.
set_stable_version() {
  # Find the stable version.
  AWS_FOR_FLUENT_BIT_STABLE_VERSION=$(cat < ../windows.versions | jq -r '.windows[] | select(.stable==true)| .version')
  STABLE_VERSION_COUNT=$(echo "$AWS_FOR_FLUENT_BIT_STABLE_VERSION" | wc -l)

  if [ "$STABLE_VERSION_COUNT" -ne 1 ]; then
    echo "Discrepancy in stable version: ${AWS_FOR_FLUENT_BIT_STABLE_VERSION}"
    exit 1
  fi
}

# Sets AWS_FOR_FLUENT_BIT_LATEST_VERSION variable with the latest version.
set_latest_version() {
  # Find the latest version.
  AWS_FOR_FLUENT_BIT_LATEST_VERSION=$(cat < ../windows.versions | jq -r '.windows[] | select(.latest==true)| .version')
  LATEST_VERSION_COUNT=$(echo "$AWS_FOR_FLUENT_BIT_LATEST_VERSION" | wc -l)

  if [ "$LATEST_VERSION_COUNT" -ne 1 ]; then
    echo "Discrepancy in latest version: ${AWS_FOR_FLUENT_BIT_LATEST_VERSION}"
    exit 1
  fi
}

# Sets AWS_FOR_FLUENT_BIT_VERSION_DOCKERHUB variable with the latest version available in dockerhub.
set_dockerhub_version() {
  # Find the dockerhub version.
  docker_hub_image_tags=$(curl -s -S 'https://registry.hub.docker.com/v2/repositories/amazon/aws-for-fluent-bit/tags/?page=1&page_size=250' | jq -r '.results[].name')
  tag_array=(`echo ${docker_hub_image_tags}`)
  AWS_FOR_FLUENT_BIT_VERSION_DOCKERHUB=$(./get_latest_dockerhub_version.py windows latest ${tag_array[@]})
}

# Returns the regional endpoint.
# For Standard partition, it would be "amazonaws.com"
# For China partition, it would be "amazonaws.com.cn"
get_regional_image_endpoint() {
  region=${1}
  endpoint="amazonaws.com"

  if [ "${region}" = "cn-north-1" ] || [ "${region}" = "cn-northwest-1" ]; then
  		endpoint="${endpoint}.cn"
  fi
  echo "${endpoint}"
}

#############################################################################################################
# Functions to compare and validate image manifests
#############################################################################################################

# Checks if the image manifest for the provided image exists in the region.
check_image_manifest_exists() {
  manifest_image=${1}

  # Inspect the manifest of the image.
  # If the same doesn't exist, return 1.
  image_manifest=$(docker manifest inspect "${manifest_image}" || echo "1")

  if [ "${image_manifest}" = 1 ]; then
    echo "No manifest found for ${manifest_image}"
    exit 1
  else
    # If the image manifest exists, then ensure that the number of digests is equal to the supported Windows releases.
    manifest_digests=$(echo "${image_manifest}" | jq -r '.manifests[].digest')
    if [ $(echo "${manifest_digests}" | wc -l) != $SUPPORTED_WINDOWS_OS_RELEASES_NUMBER ]; then
      echo "Incomplete manifest found for ${manifest_image}"
      exit 1
    fi
    echo "Manifest validation completed for ${manifest_image}"
  fi
}

# Compares image manifests of two images based on the image digests.
compare_image_manifests() {
  first_manifest_image=${1}
  second_manifest_image=${2}

  # Retrieve the manifests of both the images. If the same is not found then return 1.
  first_manifest=$(docker manifest inspect "${first_manifest_image}" || echo "1")
  second_manifest=$(docker manifest inspect "${second_manifest_image}" || echo "1")

  if [ "${first_manifest}" = 1 ]; then
    echo "No manifest found for ${first_manifest_image}"
    return 1
  elif [ "${second_manifest}" = 1 ]; then
    echo "No manifest found for ${second_manifest_image}"
    return 1
  fi

  # Obtain the digest entries from the manifest.
  first_manifest_digests=$(echo "${first_manifest}" | jq -r '.manifests[].digest' | sort)
  second_manifest_digests=$(echo "${second_manifest}" | jq -r '.manifests[].digest' | sort)

  # Compare the digest entries of both the manifests.
  if [ $(echo "$first_manifest_digests" | wc -l) != $(echo "$second_manifest_digests" | wc -l) ]; then
    echo "Mismatch in manifests due to different number of manifest entries"
    return 1
  elif ! diff <(echo "${first_manifest_digests}") <(echo "${second_manifest_digests}"); then
    echo "Mismatch in manifests: First: ${first_manifest_digests}, Second: ${second_manifest_digests}"
    return 1
  else
    echo "Manifests are same"
  fi
}

# Set the required variables
set_all_supported_versions
set_latest_version
set_dockerhub_version
set_stable_version


# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
# 	http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.

# This script uploads the following to the provided S3 bucket under S3_KEY_PREFIX
#   <s3_key>_<date>_HOSTNAME_RANDOM.core.zip
#   <s3_key>_<date>_HOSTNAME_RANDOM.executable
#   <s3_key>_<date>_HOSTNAME_RANDOM.stacktrace
#   <s3_key>_<date>_HOSTNAME_RANDOM.all.zip

export S3_BUCKET=$1
export S3_KEY_PREFIX=$2
export RUN_ID=$3
export CRASH_FOLDER="/cores-out" # Add all crash files to a cores-out folder for staging,
                                 # then copy over to the mounted volume /cores

# Folder Structure
# $CRASH_FOLDER: All the crash symbol information is combined and added to this folder.
#    It is uploaded to s3 and added to the /cores folder
# /cores: Files are sent to the cores folder

# Process crash symbols
export CORE_FILENAME=`basename $S3_KEY_PREFIX`_`date +"%FT%H%M%S"`${HOSTNAME+_host-$HOSTNAME}_${RUN_ID}

# Check if there is a corefile
export NO_CORE_STR="ls: cannot access $CRASH_FOLDER/core*: No such file or directory"
export LS_CORE_STR="$(ls $CRASH_FOLDER/core* 2>&1 >/dev/null)"

if [ "$NO_CORE_STR" == "$LS_CORE_STR" ]; then

    # No corefile
    echo "No core file to upload"
else
    # Zip and upload the core file first incase the other processes fail
    mv `ls $CRASH_FOLDER/core*` ${CORE_FILENAME}.core
    zip $CRASH_FOLDER/${CORE_FILENAME}.core.zip $CRASH_FOLDER/${CORE_FILENAME}.core

    # $CRASH_FOLDER
    #   - CORE_FILENAME.core.zip
    #   - CORE_FILENAME.core

    # Upload core zip
    aws s3 cp $CRASH_FOLDER/${CORE_FILENAME}.core.zip s3://${S3_BUCKET}/${S3_KEY_PREFIX}/

    # Remove corefile zip to root folder
    mv $CRASH_FOLDER/${CORE_FILENAME}.core.zip /tmp/${CORE_FILENAME}.core.zip

    # $CRASH_FOLDER
    #   - CORE_FILENAME.core

    # Process corefile, get stack trace and fluent bit executable. Add to zip.
    cp /fluent-bit/bin/fluent-bit ./${CORE_FILENAME}.executable
    gdb -batch -ex 'thread apply all bt full' -ex 'quit' '/fluent-bit/bin/fluent-bit' ${CORE_FILENAME}.core > $CRASH_FOLDER/${CORE_FILENAME}.stacktrace

    # $CRASH_FOLDER
    #   - CORE_FILENAME.core
    #   - CORE_FILENAME.executable
    #   - CORE_FILENAME.stacktrace

    # Zip the crashfiles
    zip -r $CRASH_FOLDER/${CORE_FILENAME}.all.zip $CRASH_FOLDER
    rm $CRASH_FOLDER/${CORE_FILENAME}.core
    mv /tmp/${CORE_FILENAME}.core.zip $CRASH_FOLDER/${CORE_FILENAME}.core.zip

    # $CRASH_FOLDER
    #   - CORE_FILENAME.core.zip
    #   - CORE_FILENAME.executable
    #   - CORE_FILENAME.stacktrace
    #   - CORE_FILENAME.all.zip

    # Send crash files to the /cores/ folder
    cp $CRASH_FOLDER/* /cores/

    # Send crash files to S3
    aws s3 cp $CRASH_FOLDER s3://${S3_BUCKET}/${S3_KEY_PREFIX}/ --recursive

    # Log stacktrace
    echo "Stacktrace - ${CORE_FILENAME}.stacktrace:"
    cat /cores/${CORE_FILENAME}.stacktrace

    # Delay for 1 minute to allow for slow EFS uploads to complete if used
    echo "Waiting 1 minute for EFS core file transfers to complete"
    sleep 60
fi