#!/bin/bash

# Helper Functions
function banner() {
    clear
    echo '           _______   __           _   ______   ________            '
    echo '     /\   |  __ \ \ / /          | | |  _   | |__   ___|           '
    echo '    /  \  | |  | \ V /   ______  | | | |  | |    |  |              '
    echo "   / /\ \ | |  | |> <   |______| | | | |  | |    |  |              "
    echo '  / ____ \| |__| / . \           | | | |__| |    |  |              '
    echo ' /_/    \_\_____/_/_\_\          |_| |_____ |    |_ |              '
    echo '        |__   __| | |                   | |                        '
    echo '           | | ___| | ___ _ __ ___   ___| |_ _ __ _   _            '
    echo "           | |/ _ \ |/ _ \ '_ \` _ \ / _ \ __| '__| | | |          "
    echo '           | |  __/ |  __/ | | | | |  __/ |_| |  | |_| |           '
    echo '           |_|\___|_|\___|_| |_| |_|\___|\__|_|   \__, |           '
    echo '                                                   __/ |           '
    echo '                                                  |___/            '
}

function spinner() {
    local info="$1"
    local pid=$!
    local delay=0.75
    local spinstr='|/-\'
    while kill -0 $pid 2> /dev/null; do
        local temp=${spinstr#?}
        printf " [%c]  $info" "$spinstr"
        local spinstr=$temp${spinstr%"$temp"}
        sleep $delay
        echo -ne "\033[0K\r"
    done
}

function deletePreLine() {
    echo -ne '\033[1A'
    echo -ne "\r\033[0K"
}

# Service Specific Functions
function add_required_extensions() {
    az extension add --name azure-iot --only-show-errors --output none; \
    az extension update --name azure-iot --only-show-errors --output none; \
    az extension add --name kusto --only-show-errors --output none; \
    az extension update --name kusto --only-show-errors --output none
}

function create_resource_group() {
    az group create --name $rgName --location "East US" --only-show-errors --output none
}

function deploy_azure_services() {
    az deployment group create -n $deploymentName -g $rgName \
        --template-file main.bicep \
        --parameters deploymentSuffix=$randomNum principalId=$principalId @iotanalytics.parameters.json \
        --only-show-errors --output none
}

function get_deployment_output() {
    dtName=$(az deployment group show -n $deploymentName -g $rgName --query properties.outputs.digitalTwinName.value --output tsv)
    dtHostName=$(az deployment group show -n $deploymentName -g $rgName --query properties.outputs.digitalTwinHostName.value --output tsv)
    saName=$(az deployment group show -n $deploymentName -g $rgName --query properties.outputs.saName.value --output tsv)
    saKey=$(az storage account keys list --account-name $saName --query [0].value -o tsv)
    saId=$(az deployment group show -n $deploymentName -g $rgName --query properties.outputs.saId.value --output tsv)
    adxName=$(az deployment group show -n $deploymentName -g $rgName --query properties.outputs.adxName.value --output tsv)
    adxResoureId=$(az deployment group show -n $deploymentName -g $rgName --query properties.outputs.adxClusterId.value --output tsv)
    location=$(az deployment group show -n $deploymentName -g $rgName --query properties.outputs.location.value --output tsv)
    eventHubNSId=$(az deployment group show -n $deploymentName -g $rgName --query properties.outputs.eventhubClusterId.value --output tsv)
    eventHubResourceId="$eventHubNSId/eventhubs/iotdata"
    eventHubHistoricId="$eventHubNSId/eventhubs/historicdata"
    iotCentralName=$(az deployment group show -n $deploymentName -g $rgName --query properties.outputs.iotCentralName.value --output tsv)
    iotCentralAppID=$(az iot central app show -n $iotCentralName -g $rgName --query  applicationId --output tsv)
    numDevices=$(az deployment group show -n $deploymentName -g $rgName --query properties.outputs.deviceNumber.value --output tsv)
    eventHubConnectionString=$(az deployment group show -n $deploymentName -g $rgName --query properties.outputs.eventHubConnectionString.value --output tsv)
}

function configure_ADX_cluster() {
    sed -i "s/<dtURI>/$dtHostName/g" config/configDB.kql ;\
    #sed -i "s/<saname>/$saName/g" config/configDB.kql ;\
    #sed -i "s/<sakey>/$saKey/g" config/configDB.kql ;\
    az storage blob upload -f config/configDB.kql -c adxscript -n configDB.kql \
        --account-key $saKey --account-name $saName --only-show-errors --output none  ;\
    blobURI="https://$saName.blob.core.windows.net/adxscript/configDB.kql"  ;\
    blobSAS=$(az storage blob generate-sas --account-name $saName --container-name adxscript \
        --name configDB.kql --permissions acdrw --expiry $tomorrow --account-key $saKey --output tsv)  ;\
    az kusto script create --cluster-name $adxName --database-name IoTAnalytics  \
        --force-update-tag "config1" --script-url $blobURI --script-url-sas-token $blobSAS \
        --resource-group $rgName --name 'configDB' --only-show-errors --output none  ;\
    az kusto data-connection event-hub create --cluster-name $adxName --name "IoTAnalytics" \
        --database-name "IoTAnalytics" --location $location --consumer-group '$Default' \
        --event-hub-resource-id $eventHubResourceId --managed-identity-resource-id $adxResoureId \
        --data-format 'JSON' --table-name 'StageIoTRawData' --mapping-rule-name 'StageIoTRawData_mapping' \
        --compression 'None' --resource-group $rgName --only-show-errors --output none

    az kusto data-connection event-grid create --cluster-name $adxName -g $rgName --database-name "IoTAnalytics" \
        --table-name "Thermostats" --name "HistoricalLoad" --ignore-first-record true --data-format csv  \
        --mapping-rule-name "Thermostats_mapping" --storage-account-resource-id $saId \
        --consumer-group '$Default' --event-hub-resource-id $eventHubHistoricId
    az storage blob upload -f config/Thermostat_January2022.csv -c adxscript -n Thermostat_January2022.csv \
        --account-key $saKey --account-name $saName --only-show-errors --output none  ;\
}

function create_digital_twin_models() {
    az dt model create -n $dtName --from-directory ./dtconfig  --only-show-errors --output none ; \
    az dt twin create -n $dtName --dtmi "dtmi:StageIoTRawData:Office;1" --twin-id Dallas --only-show-errors --output none; \
    az dt twin create -n $dtName --dtmi "dtmi:StageIoTRawData:Office;1" --twin-id Seattle --only-show-errors --output none; \
    az dt twin create -n $dtName --dtmi "dtmi:StageIoTRawData:Office;1" --twin-id Atlanta --only-show-errors --output none; \
    az dt twin create -n $dtName --dtmi "dtmi:StageIoTRawData:Floor;1" --twin-id DAL1 --only-show-errors --output none; \
    az dt twin create -n $dtName --dtmi "dtmi:StageIoTRawData:Floor;1" --twin-id DAL2 --only-show-errors --output none; \
    az dt twin create -n $dtName --dtmi "dtmi:StageIoTRawData:Floor;1" --twin-id DAL3 --only-show-errors --output none; \
    az dt twin create -n $dtName --dtmi "dtmi:StageIoTRawData:Floor;1" --twin-id DAL4 --only-show-errors --output none; \
    az dt twin create -n $dtName --dtmi "dtmi:StageIoTRawData:Floor;1" --twin-id DAL5 --only-show-errors --output none; \
    az dt twin create -n $dtName --dtmi "dtmi:StageIoTRawData:Floor;1" --twin-id DAL6 --only-show-errors --output none; \
    az dt twin create -n $dtName --dtmi "dtmi:StageIoTRawData:Floor;1" --twin-id SEA1 --only-show-errors --output none; \
    az dt twin create -n $dtName --dtmi "dtmi:StageIoTRawData:Floor;1" --twin-id SEA2 --only-show-errors --output none; \
    az dt twin create -n $dtName --dtmi "dtmi:StageIoTRawData:Floor;1" --twin-id SEA3 --only-show-errors --output none; \
    az dt twin create -n $dtName --dtmi "dtmi:StageIoTRawData:Floor;1" --twin-id SEA4 --only-show-errors --output none; \
    az dt twin create -n $dtName --dtmi "dtmi:StageIoTRawData:Floor;1" --twin-id SEA5 --only-show-errors --output none; \
    az dt twin create -n $dtName --dtmi "dtmi:StageIoTRawData:Floor;1" --twin-id SEA6 --only-show-errors --output none; \
    az dt twin create -n $dtName --dtmi "dtmi:StageIoTRawData:Floor;1" --twin-id ATL1 --only-show-errors --output none; \
    az dt twin create -n $dtName --dtmi "dtmi:StageIoTRawData:Floor;1" --twin-id ATL2 --only-show-errors --output none; \
    az dt twin create -n $dtName --dtmi "dtmi:StageIoTRawData:Floor;1" --twin-id ATL3 --only-show-errors --output none; \
    az dt twin create -n $dtName --dtmi "dtmi:StageIoTRawData:Floor;1" --twin-id ATL4 --only-show-errors --output none; \
    az dt twin create -n $dtName --dtmi "dtmi:StageIoTRawData:Floor;1" --twin-id ATL5 --only-show-errors --output none; \
    az dt twin create -n $dtName --dtmi "dtmi:StageIoTRawData:Floor;1" --twin-id ATL6 --only-show-errors --output none; \
    az dt twin relationship create -n $dtName --relationship-id 'DAL_F1'  \
        --relationship 'officecontainsfloors' --source 'Dallas' --target 'DAL1' \
        --only-show-errors --output none; \
    az dt twin relationship create -n $dtName --relationship-id 'DAL_F2'  \
        --relationship 'officecontainsfloors' --source 'Dallas' --target 'DAL2' \
        --only-show-errors --output none; \
    az dt twin relationship create -n $dtName --relationship-id 'DAL_F3'  \
        --relationship 'officecontainsfloors' --source 'Dallas' --target 'DAL3' \
        --only-show-errors --output none; \
    az dt twin relationship create -n $dtName --relationship-id 'DAL_F4'  \
        --relationship 'officecontainsfloors' --source 'Dallas' --target 'DAL4' \
        --only-show-errors --output none; \
    az dt twin relationship create -n $dtName --relationship-id 'DAL_F5'  \
        --relationship 'officecontainsfloors' --source 'Dallas' --target 'DAL5' \
        --only-show-errors --output none; \
    az dt twin relationship create -n $dtName --relationship-id 'DAL_F6'  \
        --relationship 'officecontainsfloors' --source 'Dallas' --target 'DAL6' \
        --only-show-errors --output none; \
    az dt twin relationship create -n $dtName --relationship-id 'SEA_F1'  \
        --relationship 'officecontainsfloors' --source 'Seattle' --target 'SEA1' \
        --only-show-errors --output none; \
    az dt twin relationship create -n $dtName --relationship-id 'SEA_F2'  \
        --relationship 'officecontainsfloors' --source 'Seattle' --target 'SEA2' \
        --only-show-errors --output none; \
    az dt twin relationship create -n $dtName --relationship-id 'SEA_F3'  \
        --relationship 'officecontainsfloors' --source 'Seattle' --target 'SEA3' \
        --only-show-errors --output none; \
    az dt twin relationship create -n $dtName --relationship-id 'SEA_F4'  \
        --relationship 'officecontainsfloors' --source 'Seattle' --target 'SEA4' \
        --only-show-errors --output none; \
    az dt twin relationship create -n $dtName --relationship-id 'SEA_F5'  \
        --relationship 'officecontainsfloors' --source 'Seattle' --target 'SEA5' \
        --only-show-errors --output none; \
    az dt twin relationship create -n $dtName --relationship-id 'SEA_F6'  \
        --relationship 'officecontainsfloors' --source 'Seattle' --target 'SEA6' \
        --only-show-errors --output none; \
    az dt twin relationship create -n $dtName --relationship-id 'ATL_F1'  \
        --relationship 'officecontainsfloors' --source 'Atlanta' --target 'ATL1' \
        --only-show-errors --output none; \
    az dt twin relationship create -n $dtName --relationship-id 'ATL_F2'  \
        --relationship 'officecontainsfloors' --source 'Atlanta' --target 'ATL2' \
        --only-show-errors --output none; \
    az dt twin relationship create -n $dtName --relationship-id 'ATL_F3'  \
        --relationship 'officecontainsfloors' --source 'Atlanta' --target 'ATL3' \
        --only-show-errors --output none; \
    az dt twin relationship create -n $dtName --relationship-id 'ATL_F4'  \
        --relationship 'officecontainsfloors' --source 'Atlanta' --target 'ATL4' \
        --only-show-errors --output none; \
    az dt twin relationship create -n $dtName --relationship-id 'ATL_F5'  \
        --relationship 'officecontainsfloors' --source 'Atlanta' --target 'ATL5' \
        --only-show-errors --output none; \
    az dt twin relationship create -n $dtName --relationship-id 'ATL_F6'  \
        --relationship 'officecontainsfloors' --source 'Atlanta' --target 'ATL6' \
        --only-show-errors --output none; \
}

function deploy_thermostat_devices() {
    for (( c=1; c<=$numDevices; c++ ))
    do
        deviceId=$(cat /proc/sys/kernel/random/uuid)
        az iot central device create --device-id $deviceId --app-id $iotCentralAppID \
            --template dtmi:m43gbjjsrr5:fp1yz0dm0qs --simulated --only-show-errors --output none

        floornum=$(expr $c % 18)

        floor=${floors[$floornum]}

        az dt twin create -n $dtName --dtmi "dtmi:StageIoTRawData:Thermostat;1" --twin-id $deviceId \
            --only-show-errors --output none ;\
        az dt twin relationship create -n $dtName --relationship-id "contains${deviceId}" \
            --relationship 'floorcontainsdevices' --source $floor --target $deviceId --only-show-errors --output none
    done
}

function configure_IoT_Central_output() {
    az iot central export destination create --app-id $iotCentralAppID --dest-id 'eventHubExport' \
        --type eventhubs@v1 --name 'eventHubExport' \
        --authorization '{"type": "connectionString", "connectionString": "'$eventHubConnectionString'" }' \
        --only-show-errors --output none  ; \
    az iot central export create --app-id $iotCentralAppID --export-id 'iotEventHubExport' \
        --display-name 'iotEventHubExport' --source 'telemetry' --destinations '[{"id": "eventHubExport"}]' \
        --only-show-errors --output none
}

# Define required variables
randomNum=$RANDOM
currentDate=$(date)
tomorrow=$(date +"%Y-%m-%dT00:00:00Z" -d "$currentDate +1 days")
deploymentName=ADXIoTAnalyticsDeployment$randomNum
rgName=ADXIoTAnalytics$randomNum
principalId=$(az ad signed-in-user show --query id -o tsv)

# Setup array to utilize when assiging devices to departments and patients
floors=('DAL1' 'DAL2' 'DAL3' 'DAL4' 'DAL5' 'DAL6' 'SEA1' 'SEA2' 'SEA3' 'SEA4' 'SEA5' 'SEA6' 'ATL1' 'ATL2' 'ATL3' 'ATL4' 'ATL5' 'ATL6')

banner # Show Welcome banner

echo '1. Starting solution deployment'
add_required_extensions & # Install/Update required eztensions
spinner "Installing IoT Extensions"
create_resource_group & # Create parent resurce group
spinner "Creating Resource Group"
deploy_azure_services & # Create all additional services using main Bicep template
spinner "Deploying Azure Services"

echo "2. Starting configuration for deployment $deploymentName"
get_deployment_output  # Get Deployment output values

# Start Configuration
configure_ADX_cluster & # Configure ADX cluster
spinner "Configuring ADX Cluster"
# Get/Refresh IoT Central Token
az account get-access-token --resource https://apps.azureiotcentral.com --only-show-errors --output none
create_digital_twin_models & # Create all the models from folder in git repo
spinner "Creating model for Azure Digital Twins $dtName"

# Complete configuration
echo "Creating $numDevices Smart Knee Brace devices on IoT Central: $iotCentralName ($iotCentralAppID) and Digital Twins: $dtName"
deploy_thermostat_devices # Deploy Thermostat simulated devices
configure_IoT_Central_output & # On IoT Central, create an Event Hub export and destination with json payload
spinner " Creating IoT Central App export and destination on IoT Central: $iotCentralName ($iotCentralAppID)"

echo "3. Configuration completed"


#!/bin/bash

set -euxo pipefail

: "${SUBSCRIPTION_ID:=8ecadfc9-d1a3-4ea4-b844-0d9f87e4d7c8}" #Azure Container Service - Test Subscription
: "${RESOURCE_GROUP_NAME:=agentbaker-e2e-tests}"
: "${LOCATION:=eastus}"
: "${CLUSTER_NAME:=agentbaker-e2e-test-cluster}"
: "${AZURE_TENANT_ID:=72f988bf-86f1-41af-91ab-2d7cd011db47}"
: "${TIMEOUT:=30m}"

export SUBSCRIPTION_ID
export RESOURCE_GROUP_NAME
export LOCATION
export CLUSTER_NAME
export AZURE_TENANT_ID

go version
go test -timeout $TIMEOUT -v -run Test_All ./


#!/usr/bin/env bash
set -euox pipefail

retrycmd_get_tarball() {
    tar_retries=$1; wait_sleep=$2; tarball=$3; url=$4
    echo "${tar_retries} retries"
    for i in $(seq 1 $tar_retries); do
        tar -tzf $tarball && break || \
        if [ $i -eq $tar_retries ]; then
            return 1
        else
            timeout 60 curl -fsSL $url -o $tarball
            sleep $wait_sleep
        fi
    done
}

sudo swapoff -a
sudo ctr version
sudo systemctl status containerd

sed '$ d' parts/linux/cloud-init/artifacts/manifest.json > temporary_manifest.json
KUBE_BINARY_VERSIONS="$(jq -r .kubernetes.versions[] temporary_manifest.json)"
for KUBE_BINARY_VERSION in $KUBE_BINARY_VERSIONS; do
    K8S_DOWNLOADS_DIR="/opt/kubernetes/downloads"
    KUBE_BINARY_URL="https://acs-mirror.azureedge.net/kubernetes/v${KUBE_BINARY_VERSION}/binaries/kubernetes-node-linux-amd64.tar.gz"
    mkdir -p ${K8S_DOWNLOADS_DIR}
    K8S_TGZ_TMP=${KUBE_BINARY_URL##*/}
    retrycmd_get_tarball 120 5 "$K8S_DOWNLOADS_DIR/${K8S_TGZ_TMP}" ${KUBE_BINARY_URL} || exit 120
    tar --transform="s|.*|&-${KUBE_BINARY_VERSION}|" --show-transformed-names -xzvf "$K8S_DOWNLOADS_DIR/${K8S_TGZ_TMP}" \
        --strip-components=3 -C /usr/local/bin kubernetes/node/bin/kubelet kubernetes/node/bin/kubectl
    rm -f "$K8S_DOWNLOADS_DIR/${K8S_TGZ_TMP}"
    export KUBE_BINARY_VERSION
    pushd e2e || exit 1
    go run -ldflags="-X main.KUBE_BINARY_VERSION=${KUBE_BINARY_VERSION}" kubelet/main.go
    cat kubelet/${KUBE_BINARY_VERSION}-flags.json
    popd || exit 1
done
rm -f temporary_manifest.json


#!/bin/bash

set -euxo pipefail

source e2e-helper.sh

log "Starting to create windows nodepool"

RESOURCE_GROUP_NAME="$RESOURCE_GROUP_NAME-$WINDOWS_E2E_IMAGE-$K8S_VERSION"

out=$(az aks nodepool list --cluster-name $CLUSTER_NAME -g $RESOURCE_GROUP_NAME | jq '.[].name')

if [[ "$out" != *"winnp"* ]]; then
    log "Creating windows nodepool"
    retval=0
    az aks nodepool add --resource-group $RESOURCE_GROUP_NAME --cluster-name $CLUSTER_NAME --name "winnp" --os-type Windows --os-sku $WINDOWS_E2E_OSSKU --node-vm-size $WINDOWS_E2E_VMSIZE --node-count 1 || retval=$?
    if [ "$retval" -ne 0 ]; then
        log "Other pipeline may be creating the same nodepool, waiting for ready"
    else
        log "Created windows nodepool"
    fi
else
    log "Already create windows nodepool"
fi

provisioning_state=$(az aks nodepool show --cluster-name $CLUSTER_NAME -g $RESOURCE_GROUP_NAME -n "winnp" -ojson | jq '.provisioningState' | tr -d "\"")
if [ "$provisioning_state" == "Creating" ]; then
    log "Other pipeline may be creating the same nodepool, waiting for ready"
    az aks nodepool wait --nodepool-name "winnp" --cluster-name $CLUSTER_NAME --resource-group $RESOURCE_GROUP_NAME --created --interval 60 --timeout 1800
fi

provisioning_state=$(az aks nodepool show --cluster-name $CLUSTER_NAME -g $RESOURCE_GROUP_NAME -n "winnp" -ojson | jq '.provisioningState' | tr -d "\"")
if [ "$provisioning_state" == "Succeeded" ]; then
    log "Windows nodepool is in succeed state"
else
    err "Windows nodepool is not in succeeded state. Current state is $provisioning_state."
    exit 1
fi


#!/bin/bash

log() {
    printf "\\033[1;33m%s\\033[0m\\n" "$*"
}

ok() {
    printf "\\033[1;32m%s\\033[0m\\n" "$*"
}

err() {
    printf "\\033[1;31m%s\\033[0m\\n" "$*"
}

exec_on_host() {
    kubectl exec $(kubectl get pod -l app=debug -o jsonpath="{.items[0].metadata.name}") -- bash -c "nsenter -t 1 -m bash -c \"$1\"" > $2
}

backfill_clean_storage_container() {
    set +x
    # Get supported kubernetes versions
    versions=$(az aks get-versions --location $LOCATION --query "orchestrators[].orchestratorVersion" -o tsv)
    k8s_versions=${versions//./}

    # Get container names e.g. akswinstore2022-1256 (for this $container_version would be "1256")
    container_names=$(az storage container list --account-name $STORAGE_ACCOUNT_NAME --account-key $MAPPED_ACCOUNT_KEY --query "[?starts_with(name, 'akswinstore')].name" -o tsv)
    # Check if the container version is still supported and delete the container if not
    for container_name in $container_names; do
        container_version=$(echo $container_name | cut -d '-' -f 2- | awk '{print tolower($0)}')
        echo "container version is $container_version"
        if [[ $k8s_versions == *"$container_version"* ]]; then
            echo "The version $container_version is available in the $LOCATION region."
        else
            echo "The version $container_version is not available in the $LOCATION region."
            echo "Deleting the container."
            az storage container delete --name $container_name --account-name $STORAGE_ACCOUNT_NAME --account-key $MAPPED_ACCOUNT_KEY
            echo "Deletion completed."
        fi
    done

    set -x
}

create_storage_container() {
    set +x

    # check if the storage container exists and create one if not
    exists=$(az storage container exists --account-name $STORAGE_ACCOUNT_NAME --account-key $MAPPED_ACCOUNT_KEY --name $WINDOWS_E2E_STORAGE_CONTAINER)
    if [[ $exists == *false* ]]; then
        az storage container create -n $WINDOWS_E2E_STORAGE_CONTAINER --account-name $STORAGE_ACCOUNT_NAME --account-key $MAPPED_ACCOUNT_KEY
        echo "Created storage container $WINDOWS_E2E_STORAGE_CONTAINER in $STORAGE_ACCOUNT_NAME"
    else
        # check if the storage container is empty and delete the blobs within one by one if not
        blob_list=$(az storage blob list --account-name $STORAGE_ACCOUNT_NAME --container-name $WINDOWS_E2E_STORAGE_CONTAINER --account-key $MAPPED_ACCOUNT_KEY -o json | jq -r '.[] | .name')
        if [[ -n $blob_list ]]; then
            for blob in $blob_list; do
                az storage blob delete --account-name $STORAGE_ACCOUNT_NAME --container-name $WINDOWS_E2E_STORAGE_CONTAINER --account-key $MAPPED_ACCOUNT_KEY --name $blob
                echo "Deleted blob $blob from storage container $WINDOWS_E2E_STORAGE_CONTAINER in the storage account $STORAGE_ACCOUNT_NAME"
            done
        fi
    fi
    set -x
}

upload_linux_file_to_storage_account() {
    local retval=0
    MC_RESOURCE_GROUP_NAME="MC_${RESOURCE_GROUP_NAME}_${CLUSTER_NAME}_$LOCATION"
    MC_VMSS_NAME=$(az vmss list -g $MC_RESOURCE_GROUP_NAME --query "[?contains(name, 'nodepool')]" -ojson | jq -r '.[0].name')
    VMSS_INSTANCE_ID="$(az vmss list-instances --name $MC_VMSS_NAME -g $MC_RESOURCE_GROUP_NAME | jq -r '.[0].instanceId')"

    set +x
    expiryTime=$(date --date="2 day" +%Y-%m-%d)
    token=$(az storage container generate-sas --account-name $STORAGE_ACCOUNT_NAME --account-key $MAPPED_ACCOUNT_KEY --permissions 'w' --expiry $expiryTime --name $WINDOWS_E2E_STORAGE_CONTAINER)
    linuxFileURL="https://${STORAGE_ACCOUNT_NAME}.blob.core.windows.net/${WINDOWS_E2E_STORAGE_CONTAINER}/${MC_VMSS_NAME}-linux-file.zip?${token}"

    az vmss run-command invoke --command-id RunShellScript \
        --resource-group $MC_RESOURCE_GROUP_NAME \
        --name $MC_VMSS_NAME \
        --instance-id $VMSS_INSTANCE_ID \
        --scripts "cat /etc/kubernetes/azure.json > /home/fields.json; cat /etc/kubernetes/certs/apiserver.crt | base64 -w 0 > /home/apiserver.crt; cat /etc/kubernetes/certs/ca.crt | base64 -w 0 > /home/ca.crt; cat /etc/kubernetes/certs/client.key | base64 -w 0 > /home/client.key; cat /var/lib/kubelet/bootstrap-kubeconfig > /home/bootstrap-kubeconfig; cd /home; zip file.zip fields.json apiserver.crt ca.crt client.key bootstrap-kubeconfig; wget https://aka.ms/downloadazcopy-v10-linux; tar -xvf downloadazcopy-v10-linux; cd ./azcopy_*; ./azcopy copy /home/file.zip $linuxFileURL" || retval=$?

    set -x
    if [ "$retval" -eq 0 ]; then
        log "Upload linux file successfully"
    else
        err "Failed to upload linux file. Error code is $retval."
    fi
}

download_linux_file_from_storage_account() {
    wget https://aka.ms/downloadazcopy-v10-linux
    tar -xvf downloadazcopy-v10-linux

    expiryTime=$(date --date="2 day" +%Y-%m-%d)

    set +x

    token=$(az storage container generate-sas --account-name $STORAGE_ACCOUNT_NAME --account-key $MAPPED_ACCOUNT_KEY --permissions 'rl' --expiry $expiryTime --name $WINDOWS_E2E_STORAGE_CONTAINER)
    tokenWithoutQuote=${token//\"}
    linuxFileURL="https://${STORAGE_ACCOUNT_NAME}.blob.core.windows.net/${WINDOWS_E2E_STORAGE_CONTAINER}/${MC_VMSS_NAME}-linux-file.zip?${tokenWithoutQuote}"

    array=(azcopy_*)
    noExistStr="File count: 0"
    local fileExist="false"
    for i in $(seq 1 20); do
        listResult=$(${array[0]}/azcopy list $linuxFileURL --running-tally)
        if [[ "$listResult" == *"$noExistStr"* ]]; then
            log "Linux file has not been uploaded, waiting..."
            sleep 10
            continue
        fi
        fileExist="true"
        break;
    done
    set -x

    if [ "$fileExist" == "false" ]; then
        err "File does not exist in storage account."
        exit 1
    fi

    set +x
    ${array[0]}/azcopy copy $linuxFileURL file.zip
    set -x

    unzip file.zip
}

addJsonToFile() {
    k=$1; v=$2
    jq -r --arg key $k --arg value $v '. + { ($key) : $value}' < fields.json > dummy.json && mv dummy.json fields.json
}

getAgentPoolProfileValues() {
    declare -a properties=("mode" "name" "nodeImageVersion")

    for property in "${properties[@]}"; do
        value=$(jq -r .agentPoolProfiles[].${property} < cluster_info.json)
        addJsonToFile $property $value
    done
}

getFQDN() {
    fqdn=$(jq -r '.fqdn' < cluster_info.json)
    addJsonToFile "fqdn" $fqdn
}

getMSIResourceID() {
    msiResourceID=$(jq -r '.identityProfile.kubeletidentity.resourceId' < cluster_info.json)
    addJsonToFile "msiResourceID" $msiResourceID
}

getTenantID() {
    tenantID=$(jq -r '.identity.tenantId' < cluster_info.json)
    addJsonToFile "tenantID" $tenantID
}

cleanupOutdatedFiles() {
    # delete blobs created 1 day ago
    EXPIRATION_IN_HOURS=24
    (( expirationInSecs = ${EXPIRATION_IN_HOURS} * 60 * 60 ))
    (( deadline=$(date +%s)-${expirationInSecs%.*} ))
    dateOfdeadline=$(date -d @${deadline} +"%Y-%m-%dT%H:%M:%S+00:00")

    # two containers need to be cleaned up now
    CONTAINER_LIST=("cselogs" "csepackages")

    for CONTAINER_NAME in "${CONTAINER_LIST[@]}"
    do
        result=$(az storage blob list -c $CONTAINER_NAME --account-name $STORAGE_ACCOUNT_NAME --account-key $MAPPED_ACCOUNT_KEY -o json \
        | jq -r --arg time "$dateOfdeadline" '.[] | select(.properties.creationTime < $time)' \
        | jq -r '.name')

        for item in $result
        do
            az storage blob delete -c $CONTAINER_NAME --account-name $STORAGE_ACCOUNT_NAME --account-key $MAPPED_ACCOUNT_KEY -n $item
            echo "Deleted $item in $CONTAINER_NAME"
        done
    done
}


#!/bin/bash

set -euxo pipefail

source e2e-helper.sh

choose() {
    echo ${1:RANDOM%${#1}:1} $RANDOM;
}

collect-logs() {
    local retval
    retval=0
    mkdir -p $SCENARIO_NAME-logs
    VMSS_INSTANCE_ID="$(az vmss list-instances --name $DEPLOYMENT_VMSS_NAME -g $MC_RESOURCE_GROUP_NAME | jq -r '.[0].instanceId')"
    set +x
    expiryTime=$(date --date="2 day" +%Y-%m-%d)
    token=$(az storage container generate-sas --account-name $STORAGE_ACCOUNT_NAME --account-key $MAPPED_ACCOUNT_KEY --permissions 'rwld' --expiry $expiryTime --name $STORAGE_LOG_CONTAINER --https-only)
    # Use .ps1 file to run scripts since single quotes of parameters for --scripts would fail in check-shell
    az vmss run-command invoke --command-id RunPowerShellScript \
        --resource-group $MC_RESOURCE_GROUP_NAME \
        --name $DEPLOYMENT_VMSS_NAME \
        --instance-id $VMSS_INSTANCE_ID \
        --scripts @upload-cse-logs.ps1 \
        --parameters arg1=$STORAGE_ACCOUNT_NAME arg2=$STORAGE_LOG_CONTAINER arg3=$DEPLOYMENT_VMSS_NAME arg4=$token || retval=$?
    if [ "$retval" -ne 0 ]; then
        err "Failed in uploading cse logs. Error code is $retval."
    fi

    tokenWithoutQuote=${token//\"}
    # use array to pass shellcheck
    array=(azcopy_*)
    ${array[0]}/azcopy copy "https://${STORAGE_ACCOUNT_NAME}.blob.core.windows.net/${STORAGE_LOG_CONTAINER}/${DEPLOYMENT_VMSS_NAME}-cse.log?${tokenWithoutQuote}" $SCENARIO_NAME-logs/$WINDOWS_E2E_IMAGE-CustomDataSetupScript.log || retval=$?
    if [ "$retval" -ne 0 ]; then
        err "Failed in downloading cse logs. Error code is $retval."
    else
        log "Collect cse logs done"
        ${array[0]}/azcopy rm "https://${STORAGE_ACCOUNT_NAME}.blob.core.windows.net/${STORAGE_LOG_CONTAINER}/${DEPLOYMENT_VMSS_NAME}-cse.log?${tokenWithoutQuote}" || retval=$?
        if [ "$retval" -ne 0 ]; then
            err "Failed in deleting cse logs in remote storage. Error code is $retval."
        fi
    fi
    set -x
}

RESOURCE_GROUP_NAME="$RESOURCE_GROUP_NAME-$WINDOWS_E2E_IMAGE-$K8S_VERSION"

DEPLOYMENT_VMSS_NAME="$(mktemp -u winXXXXX | tr '[:upper:]' '[:lower:]')"
export DEPLOYMENT_VMSS_NAME

# download azcopy
wget https://aka.ms/downloadazcopy-v10-linux
tar -xvf downloadazcopy-v10-linux

# zip and upload cse package
timeStamp=$(date +%s)
cd ../staging/cse/windows
zip -r ../../../$WINDOWS_E2E_IMAGE/$WINDOWS_E2E_IMAGE-aks-windows-cse-scripts.zip ./* -x ./*.tests.ps1 -x "*azurecnifunc.tests.suites*" -x README -x provisioningscripts/*.md -x debug/update-scripts.ps1
log "Zip cse packages done"

set +x
expiryTime=$(date --date="2 day" +%Y-%m-%d)
token=$(az storage container generate-sas --account-name $STORAGE_ACCOUNT_NAME --account-key $MAPPED_ACCOUNT_KEY --permissions 'rwld' --expiry $expiryTime --name $STORAGE_PACKAGE_CONTAINER)
tokenWithoutQuote=${token//\"}

csePackageURL="https://${STORAGE_ACCOUNT_NAME}.blob.core.windows.net/${STORAGE_PACKAGE_CONTAINER}/${timeStamp}-${DEPLOYMENT_VMSS_NAME}-aks-windows-cse-scripts.zip?${tokenWithoutQuote}"
export csePackageURL

cd ../../../$WINDOWS_E2E_IMAGE

array=(azcopy_*)
noExistStr="File count: 0"
listResult=$(${array[0]}/azcopy list $csePackageURL --running-tally)

for i in $(seq 1 10); do
    if [[ "$listResult" != *"$noExistStr"* ]]; then
        log "Cse package with the same exists, retry $i to use new name..."
        timeStamp=$(date +%s)
        csePackageURL="https://${STORAGE_ACCOUNT_NAME}.blob.core.windows.net/${STORAGE_PACKAGE_CONTAINER}/${timeStamp}-${DEPLOYMENT_VMSS_NAME}-aks-windows-cse-scripts.zip?${tokenWithoutQuote}"
        listResult=$(${array[0]}/azcopy list $csePackageURL --running-tally)
        continue
    fi
    ${array[0]}/azcopy copy $WINDOWS_E2E_IMAGE-aks-windows-cse-scripts.zip $csePackageURL
    break;
done

set -x

listResult=$(${array[0]}/azcopy list $csePackageURL --running-tally)
if [[ "$listResult" == *"$noExistStr"* ]]; then
    err "Failed to upload cse package"
    exit 1
fi

log "Upload cse packages done"

log "Scenario is $SCENARIO_NAME"
log "Windows package version is $WINDOWS_PACKAGE_VERSION"

# Generate vmss cse deployment config for windows nodepool testing
export orchestratorVersion=$WINDOWS_PACKAGE_VERSION
envsubst < scenarios/$SCENARIO_NAME/property-$SCENARIO_NAME-template.json > scenarios/$SCENARIO_NAME/$WINDOWS_E2E_IMAGE-property-$SCENARIO_NAME.json

set +x
WINDOWS_PASSWORD=$({
    choose '0123456789'
    choose 'abcdefghijklmnopqrstuvwxyz'
    choose 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
    choose '#@'
    for i in $(seq 1 16)
    do
        choose '#@0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
    done
} | sort -R | awk '{printf "%s", $1}')
set -x
echo $WINDOWS_PASSWORD

MC_RESOURCE_GROUP_NAME="MC_${RESOURCE_GROUP_NAME}_${CLUSTER_NAME}_eastus"

KUBECONFIG=$(pwd)/kubeconfig
export KUBECONFIG

clientCertificate=$(grep "client-certificate-data" $KUBECONFIG | awk '{print $2}')

tee $SCENARIO_NAME-vmss.json > /dev/null <<EOF
{
    "group": "${MC_RESOURCE_GROUP_NAME}",
    "vmss": "${DEPLOYMENT_VMSS_NAME}"
}
EOF

# Removed the "network-plugin" tag o.w. kubelet error for 1.24.0+ contains "failed to parse kubelet flag: unknown flag: --network-plugin"
# "network-plugin" works for 1.23.15 and below (you won't see this parsing error in kubelet.err.log)
jq --arg clientCrt "$clientCertificate" --arg vmssName $DEPLOYMENT_VMSS_NAME 'del(.KubeletConfig."--pod-manifest-path") | del(.KubeletConfig."--pod-max-pids") | del(.KubeletConfig."--protect-kernel-defaults") | del(.KubeletConfig."--tls-cert-file") | del(.KubeletConfig."--tls-private-key-file") | del(.KubeletConfig."--network-plugin") | .ContainerService.properties.certificateProfile += {"clientCertificate": $clientCrt} | .PrimaryScaleSetName=$vmssName' nodebootstrapping_config.json > $WINDOWS_E2E_IMAGE-nodebootstrapping_config_for_windows.json
jq -s '.[0] * .[1]' $WINDOWS_E2E_IMAGE-nodebootstrapping_config_for_windows.json scenarios/$SCENARIO_NAME/$WINDOWS_E2E_IMAGE-property-$SCENARIO_NAME.json > scenarios/$SCENARIO_NAME/$WINDOWS_E2E_IMAGE-nbc-$SCENARIO_NAME.json

go test -tags bash_e2e -run TestE2EWindows

MC_WIN_VMSS_NAME=$(az vmss list -g $MC_RESOURCE_GROUP_NAME --query "[?contains(name, 'winnp')]" -ojson | jq -r '.[0].name')
VMSS_RESOURCE_Id=$(az resource show --resource-group $MC_RESOURCE_GROUP_NAME --name $MC_WIN_VMSS_NAME --resource-type Microsoft.Compute/virtualMachineScaleSets --query id --output tsv)

az group export --resource-group $MC_RESOURCE_GROUP_NAME --resource-ids $VMSS_RESOURCE_Id --include-parameter-default-value > test.json
# TODO: use prod image
# IMAGE_REFERENCE=$(jq -c '.resources[0].properties.virtualMachineProfile.storageProfile.imageReference.id' test.json)
IMAGE_REFERENCE="[concat(parameters('galleries_AKSWindows_externalid'), '/images/windows-e2e-test-$WINDOWS_E2E_IMAGE/versions/2023.02.07')]"
IMAGE_EXTERNALID="/subscriptions/$IMAGE_SUBSCRIPTION_ID/resourceGroups/akswinvhdbuilderrg/providers/Microsoft.Compute/galleries/AKSWindows"
WINDOWS_VNET=$(jq -c '.parameters | with_entries( select(.key|contains("vnet")))' test.json)
WINDOWS_LOADBALANCER=$(jq -c '.parameters | with_entries( select(.key|contains("loadBalancers")))' test.json)
WINDOWS_IDENTITY=$(jq -c '.resources[0] | with_entries( select(.key|contains("identity")))' test.json)
WINDOWS_SKU=$(jq -c '.resources[0] | with_entries( select(.key|contains("sku")))' test.json)
WINDOWS_OSDISK=$(jq -c '.resources[0].properties.virtualMachineProfile.storageProfile | with_entries( select(.key|contains("osDisk")))' test.json)
NETWORK_PROPERTIES=$(jq -c '.resources[0].properties.virtualMachineProfile.networkProfile.networkInterfaceConfigurations[0] | with_entries( select(.key|contains("properties")))' test.json)
CUSTOM_DATA=$(cat scenarios/$SCENARIO_NAME/$WINDOWS_E2E_IMAGE-$SCENARIO_NAME-cloud-init.txt)
CSE_CMD=$(cat scenarios/$SCENARIO_NAME/$WINDOWS_E2E_IMAGE-$SCENARIO_NAME-cseCmd)

jq --argjson JsonForVnet "$WINDOWS_VNET" \
    --argjson JsonForLB "$WINDOWS_LOADBALANCER" \
    --argjson JsonForIdentity "$WINDOWS_IDENTITY" \
    --argjson JsonForSKU "$WINDOWS_SKU" \
    --argjson JsonForNetwork "$NETWORK_PROPERTIES" \
    --argjson JsonForOSDisk "$WINDOWS_OSDISK" \
    --arg ValueForImageExternalID "$IMAGE_EXTERNALID" \
    --arg ValueForImageReference "$IMAGE_REFERENCE" \
    --arg ValueForAdminPassword "$WINDOWS_PASSWORD" \
    --arg ValueForCustomData "$CUSTOM_DATA" \
    --arg ValueForCSECmd "$CSE_CMD" \
    --arg ValueForVMSS "$DEPLOYMENT_VMSS_NAME" \
    '.parameters += $JsonForVnet | .parameters += $JsonForLB | .parameters.galleries_AKSWindows_externalid.defaultValue=$ValueForImageExternalID | .resources[0] += $JsonForIdentity | .resources[0] += $JsonForSKU | .resources[0].properties.virtualMachineProfile.storageProfile+=$JsonForOSDisk | .resources[0].properties.virtualMachineProfile.networkProfile.networkInterfaceConfigurations[0] += $JsonForNetwork | .resources[0].properties.virtualMachineProfile.storageProfile.imageReference.id=$ValueForImageReference | .resources[0].properties.virtualMachineProfile.osProfile.adminPassword=$ValueForAdminPassword | .resources[0].properties.virtualMachineProfile.osProfile.customData=$ValueForCustomData | .resources[0].properties.virtualMachineProfile.extensionProfile.extensions[0].properties.settings.commandToExecute=$ValueForCSECmd | .parameters.virtualMachineScaleSets_akswin30_name.defaultValue=$ValueForVMSS' \
    windows_vmss_template.json > $DEPLOYMENT_VMSS_NAME-deployment.json

retval=0
set +e
az deployment group create --resource-group $MC_RESOURCE_GROUP_NAME \
         --template-file $DEPLOYMENT_VMSS_NAME-deployment.json || retval=$?
set -e
log "Deployment of windows vmss succeeded."

if [ "$retval" -ne 0 ]; then
    err "Failed to deploy windows vmss. Error code is $retval."
    exit 1
fi

# delete cse package in storage account
${array[0]}/azcopy rm $csePackageURL || retval=$?

if [ "$retval" -ne 0 ]; then
    err "Failed to delete cse package in storage account. Error code is $retval."
else
    log "Delete cse package in storage account done"
fi

log "Collect cse log"
collect-logs

cat $SCENARIO_NAME-vmss.json

VMSS_INSTANCE_NAME=$(az vmss list-instances \
                    -n ${DEPLOYMENT_VMSS_NAME} \
                    -g $MC_RESOURCE_GROUP_NAME \
                    -ojson | \
                    jq -r '.[].osProfile.computerName')
export VMSS_INSTANCE_NAME

# Sleep to let the automatic upgrade of the VM finish
waitForNodeStartTime=$(date +%s)
for i in $(seq 1 10); do
    set +e
    kubectl get nodes | grep $VMSS_INSTANCE_NAME
    retval=$?
    # pipefail interferes with conditional.
    # shellcheck disable=SC2143
    if [ -z "$(kubectl get nodes | grep $VMSS_INSTANCE_NAME | grep -v "NotReady")" ]; then
        log "Retrying attempt $i"
        sleep 10
        continue
    fi
    break;
done
waitForNodeEndTime=$(date +%s)
log "Waited $((waitForNodeEndTime-waitForNodeStartTime)) seconds for node to join"

FAILED=0
# Check if the node joined the cluster
if [ "$retval" -eq 0 ]; then
    ok "Test succeeded, node joined the cluster"
    kubectl get nodes -o wide | grep $VMSS_INSTANCE_NAME
else
    err "Node did not join cluster"
    FAILED=1
fi

# Run a windows servercore pod on the node to check if pod runs
POD_NAME=$(mktemp -u podName-XXXXXXX | tr '[:upper:]' '[:lower:]')
export POD_NAME
if [[ "$WINDOWS_E2E_IMAGE" == *"2019"* ]]; then
    WINDOWS_POD_IMAGE="2019"
else
    WINDOWS_POD_IMAGE="2022"
fi
export WINDOWS_POD_IMAGE
envsubst < pod-windows-template.yaml > pod-windows.yaml

kubectl apply -f pod-windows.yaml

# Sleep to let Pod Status=Running
waitForPodStartTime=$(date +%s)
for i in $(seq 1 20); do
    set +e
    kubectl get pods -o wide | grep $POD_NAME
    kubectl get pods -o wide | grep $POD_NAME | grep 'Running'
    retval=$?
    set -e
    if [ "$retval" -ne 0 ]; then
        log "Retrying attempt $i"
        sleep 15
        continue
    fi
    break;
done
waitForPodEndTime=$(date +%s)
log "Waited $((waitForPodEndTime-waitForPodStartTime)) seconds for pod to come up"

if [ "$retval" -eq 0 ]; then
    ok "Pod ran successfully"
else
    err "Pod pending/not running. Error code is $retval."
    kubectl get pods -o wide | grep $POD_NAME
    kubectl describe pod $POD_NAME
    exit 1
fi

if [ "$FAILED" -ne 0  ] || [ "$retval" -ne 0 ]; then
    log "Reserve vmss and node for failed pipeline"
else
    waitForDeleteStartTime=$(date +%s)

    # Only delete node and vmss since we reuse the resource group and cluster now
    kubectl delete node $VMSS_INSTANCE_NAME
    az vmss delete -g $(jq -r .group $SCENARIO_NAME-vmss.json) -n $(jq -r .vmss $SCENARIO_NAME-vmss.json)

    waitForDeleteEndTime=$(date +%s)
    log "Waited $((waitForDeleteEndTime-waitForDeleteStartTime)) seconds to delete VMSS and node"
fi


#!/bin/bash

set -euxo pipefail

source e2e-helper.sh

log "Starting e2e tests"

# Create a resource group for the cluster
log "Creating resource group"
RESOURCE_GROUP_NAME="$RESOURCE_GROUP_NAME-$WINDOWS_E2E_IMAGE-$K8S_VERSION"

rgStartTime=$(date +%s)
az group create -l $LOCATION -n $RESOURCE_GROUP_NAME --subscription $SUBSCRIPTION_ID -ojson
rgEndTime=$(date +%s)
log "Created resource group in $((rgEndTime-rgStartTime)) seconds"

# Check if there exists a cluster in the RG. If yes, check if the MC_RG associated with it still exists.
# MC_RG gets deleted due to ACS-Test Garbage Collection but the cluster hangs around
out=$(az aks list -g $RESOURCE_GROUP_NAME -ojson | jq '.[].name')
create_cluster="false"
if [ -n "$out" ]; then
    provisioning_state=$(az aks show -n $CLUSTER_NAME -g $RESOURCE_GROUP_NAME -ojson | jq '.provisioningState' | tr -d "\"")
    MC_RG_NAME="MC_${RESOURCE_GROUP_NAME}_${CLUSTER_NAME}_$LOCATION"
    exists=$(az group exists -n $MC_RG_NAME)
    if [ "$exists" == "false" ] || [ "$provisioning_state" == "Failed" ] || [ "$provisioning_state" == "Canceled" ]; then
        # The cluster is in a broken state
        log "Cluster $CLUSTER_NAME is in an unusable state, deleting..."
        clusterDeleteStartTime=$(date +%s)
        az aks delete -n $CLUSTER_NAME -g $RESOURCE_GROUP_NAME --yes
        clusterDeleteEndTime=$(date +%s)
        log "Deleted cluster $CLUSTER_NAME in $((clusterDeleteEndTime-clusterDeleteStartTime)) seconds"
        create_cluster="true"
    elif [ "$provisioning_state" == "Creating" ]; then
        # Other pipeline is creating this cluster
        log "Cluster $CLUSTER_NAME is being created, waiting for ready"
        az aks wait --name $CLUSTER_NAME --resource-group $RESOURCE_GROUP_NAME --created --interval 60 --timeout 1800
        provisioning_state=$(az aks show -n $CLUSTER_NAME -g $RESOURCE_GROUP_NAME -ojson | jq '.provisioningState' | tr -d "\"")
        if [ "$provisioning_state" == "Succeeded" ]; then
            log "Cluster created by other pipeline successfully"
        else
            err "Other pipeline failed to create the cluster. Current state of cluster is $provisioning_state."
            exit 1
        fi
    elif [ "$provisioning_state" == "Updating" ]; then
        # Other pipeline is updating this cluster
        log "Cluster $CLUSTER_NAME is being updated, waiting for ready"
        az aks wait --name $CLUSTER_NAME --resource-group $RESOURCE_GROUP_NAME --updated --interval 60 --timeout 1800
        provisioning_state=$(az aks show -n $CLUSTER_NAME -g $RESOURCE_GROUP_NAME -ojson | jq '.provisioningState' | tr -d "\"")
        if [ "$provisioning_state" == "Succeeded" ]; then
            log "Cluster updated by other pipeline successfully"
        else
            err "Other pipeline failed to update the cluster. Current state of cluster is $provisioning_state."
            exit 1
        fi
    elif [ "$provisioning_state" == "Deleting" ]; then
        log "Cluster $CLUSTER_NAME is being deleted, waiting for ready"
        az aks wait --name $CLUSTER_NAME --resource-group $RESOURCE_GROUP_NAME --deleted --interval 60 --timeout 1800
        retval=0
        az aks show -n $CLUSTER_NAME -g $RESOURCE_GROUP_NAME -ojson || retval=$?
        if [ "$retval" -ne 0  ]; then
            log "Cluster deleted successfully"
            create_cluster="true"
        else
            err "Failed to delete the cluster."
            exit 1
        fi
    fi
else
    create_cluster="true"
fi

# Create the AKS cluster and get the kubeconfig
if [ "$create_cluster" == "true" ]; then
    log "Creating cluster $CLUSTER_NAME"
    clusterCreateStartTime=$(date +%s)
    retval=0

    az aks create -g $RESOURCE_GROUP_NAME -n $CLUSTER_NAME --node-count 1 --generate-ssh-keys --network-plugin azure -ojson || retval=$?

    if [ "$retval" -ne 0  ]; then
        log "Other pipelines may be creating cluster $CLUSTER_NAME, waiting for ready"
        create_cluster="false"
        az aks wait --name $CLUSTER_NAME --resource-group $RESOURCE_GROUP_NAME --created --interval 60 --timeout 1800
    fi
    provisioning_state=$(az aks show -n $CLUSTER_NAME -g $RESOURCE_GROUP_NAME -ojson | jq '.provisioningState' | tr -d "\"")
    if [ "$provisioning_state" == "Succeeded" ]; then
        log "Created cluster successfully"
    else
        err "Failed to create cluster. Current state is $provisioning_state."
        exit 1
    fi
    clusterCreateEndTime=$(date +%s)
    log "Created cluster $CLUSTER_NAME in $((clusterCreateEndTime-clusterCreateStartTime)) seconds"
fi

az aks get-credentials -g $RESOURCE_GROUP_NAME -n $CLUSTER_NAME --file kubeconfig --overwrite-existing
KUBECONFIG=$(pwd)/kubeconfig
export KUBECONFIG

# Store the contents of az aks show to a file to reduce API call overhead
az aks show -n $CLUSTER_NAME -g $RESOURCE_GROUP_NAME -ojson > cluster_info.json

MC_RESOURCE_GROUP_NAME="MC_${RESOURCE_GROUP_NAME}_${CLUSTER_NAME}_eastus"
az vmss list -g $MC_RESOURCE_GROUP_NAME --query "[?contains(name, 'nodepool')]" -otable
MC_VMSS_NAME=$(az vmss list -g $MC_RESOURCE_GROUP_NAME --query "[?contains(name, 'nodepool')]" -ojson | jq -r '.[0].name')
CLUSTER_ID=$(echo $MC_VMSS_NAME | cut -d '-' -f3)

backfill_clean_storage_container
if [ "$create_cluster" == "true" ]; then
    create_storage_container
    upload_linux_file_to_storage_account
    if [ "$WINDOWS_E2E_IMAGE" == "2019-containerd" ]; then
        cleanupOutdatedFiles
    fi
fi
download_linux_file_from_storage_account
log "Download of linux file from storage account completed"

set +x
addJsonToFile "apiserverCrt" "$(cat apiserver.crt)"
addJsonToFile "caCrt" "$(cat ca.crt)"
addJsonToFile "clientKey" "$(cat client.key)"
if [ -f "bootstrap-kubeconfig" ] && [ -n "$(cat bootstrap-kubeconfig)" ]; then
    tlsToken="$(grep "token" < bootstrap-kubeconfig | cut -f2 -d ":" | tr -d '"')"
    addJsonToFile "tlsbootstraptoken" "$tlsToken"
fi
set -x

# # Add other relevant information needed by AgentBaker for bootstrapping later
getAgentPoolProfileValues
getFQDN
getMSIResourceID

addJsonToFile "mcRGName" $MC_RESOURCE_GROUP_NAME
addJsonToFile "clusterID" $CLUSTER_ID
addJsonToFile "subID" $SUBSCRIPTION_ID

set +x
# shellcheck disable=SC2091
$(jq -r 'keys[] as $k | "export \($k)=\(.[$k])"' fields.json)
envsubst < percluster_template.json > percluster_config.json
jq -s '.[0] * .[1]' nodebootstrapping_template.json percluster_config.json > nodebootstrapping_config.json
log "Node bootstrapping config generated"


#!/bin/sh
# This script was originally generated by logrotate automatically and placed in /etc/cron.daily/logrotate
# This will be saved on the target VM within /usr/local/bin/logrotate.sh and invoked by logrotate.service

# Clean non existent log file entries from status file
cd /var/lib/logrotate
test -e status || touch status
head -1 status > status.clean
sed 's/"//g' status | while read logfile date
do
    [ -e "$logfile" ] && echo "\"$logfile\" $date"
done >> status.clean
mv status.clean status

test -x /usr/sbin/logrotate || exit 0
/usr/sbin/logrotate --verbose /etc/logrotate.conf


#!/usr/bin/env bash
set -o errexit
set -o nounset
set -o pipefail
set -x

# Bind mount kubelet to ephemeral storage on startup, as necessary.
#
# This fixes an issue with kubelet's ability to detect allocatable
# capacity for Node ephemeral-storage. On Azure, ephemeral-storage
# should correspond to the temp disk if a VM has one. This script makes
# that true by bind mounting the temp disk to /var/lib/kubelet, so
# kubelet thinks it's located on the temp disk (/dev/sdb). This results
# in correct calculation of ephemeral-storage capacity.

# if aks ever supports alternatives besides temp disk
# this mount point will need to be updated
MOUNT_POINT="/mnt/aks"

KUBELET_MOUNT_POINT="${MOUNT_POINT}/kubelet"
KUBELET_DIR="/var/lib/kubelet"

mkdir -p "${MOUNT_POINT}"

# only move the kubelet directory to alternate location on first boot.
SENTINEL_FILE="/opt/azure/containers/bind-sentinel"
if [ ! -e "$SENTINEL_FILE" ]; then
    mv "$KUBELET_DIR" "$MOUNT_POINT"
    touch "$SENTINEL_FILE"
fi

# on every boot, bind mount the kubelet directory back to the expected
# location before kubelet itself may start.
mkdir -p "${KUBELET_DIR}"
mount --bind "${KUBELET_MOUNT_POINT}" "${KUBELET_DIR}"
chmod a+w "${KUBELET_DIR}"


#!/bin/bash
# Disallow container from reaching out to the special IP address 168.63.129.16
# for TCP protocol (which http uses)
#
# 168.63.129.16 contains protected settings that have priviledged info.
#
# The host can still reach 168.63.129.16 because it goes through the OUTPUT chain, not FORWARD.
#
# Note: we should not block all traffic to 168.63.129.16. For example UDP traffic is still needed
# for DNS.
iptables -I FORWARD -d 168.63.129.16 -p tcp --dport 80 -j DROP


#!/usr/bin/env bash

set -o nounset
set -o pipefail

[ ! -f "/var/run/mdsd-ci/update.status" ] && exit 0
status=$(cat /var/run/mdsd-ci/update.status)

if [[ "$status" == "add" ]]; then
        echo "Status changed to $status."
        [ -f "/var/run/mdsd-ci/70-rsyslog-forward-mdsd-ci.conf" ] && cp /var/run/mdsd-ci/70-rsyslog-forward-mdsd-ci.conf /etc/rsyslog.d
elif [[ "$status" == "remove" ]]; then
        echo "Status changed to $status."
        [ -f "/etc/rsyslog.d/70-rsyslog-forward-mdsd-ci.conf" ] && rm /etc/rsyslog.d/70-rsyslog-forward-mdsd-ci.conf
else
        echo "Unexpected status change to $status. Exiting"
        exit 1
fi

echo "Restarting rsyslog"
systemctl restart rsyslog

exit 0


#!/bin/bash
# This gets us the error codes we use and the os and such.
source /home/packer/provision_source.sh

assignRootPW() {
    if grep '^root:[!*]:' /etc/shadow; then
        VERSION=$(grep DISTRIB_RELEASE /etc/*-release | cut -f 2 -d "=")
        SALT=$(openssl rand -base64 5)
        SECRET=$(openssl rand -base64 37)
        CMD="import crypt, getpass, pwd; print(crypt.crypt('$SECRET', '\$6\$$SALT\$'))"
        if [[ "${VERSION}" == "22.04" ]]; then
            HASH=$(python3 -c "$CMD")
        else
            HASH=$(python -c "$CMD")
        fi

        echo 'root:'$HASH | /usr/sbin/chpasswd -e || exit $ERR_CIS_ASSIGN_ROOT_PW
    fi
}

assignFilePermissions() {
    FILES="
    auth.log
    alternatives.log
    cloud-init.log
    cloud-init-output.log
    daemon.log
    dpkg.log
    kern.log
    lastlog
    waagent.log
    syslog
    unattended-upgrades/unattended-upgrades.log
    unattended-upgrades/unattended-upgrades-dpkg.log
    azure-vnet-ipam.log
    azure-vnet-telemetry.log
    azure-cnimonitor.log
    azure-vnet.log
    kv-driver.log
    blobfuse-driver.log
    blobfuse-flexvol-installer.log
    landscape/sysinfo.log
    "
    for FILE in ${FILES}; do
        FILEPATH="/var/log/${FILE}"
        DIR=$(dirname "${FILEPATH}")
        mkdir -p ${DIR} || exit $ERR_CIS_ASSIGN_FILE_PERMISSION
        touch ${FILEPATH} || exit $ERR_CIS_ASSIGN_FILE_PERMISSION
        chmod 640 ${FILEPATH} || exit $ERR_CIS_ASSIGN_FILE_PERMISSION
    done
    find /var/log -type f -perm '/o+r' -exec chmod 'g-wx,o-rwx' {} \;
    chmod 600 /etc/passwd- || exit $ERR_CIS_ASSIGN_FILE_PERMISSION
    chmod 600 /etc/shadow- || exit $ERR_CIS_ASSIGN_FILE_PERMISSION
    chmod 600 /etc/group- || exit $ERR_CIS_ASSIGN_FILE_PERMISSION

    if [[ -f /etc/default/grub ]]; then
        chmod 644 /etc/default/grub || exit $ERR_CIS_ASSIGN_FILE_PERMISSION
    fi

    if [[ -f /etc/crontab ]]; then
        chmod 0600 /etc/crontab || exit $ERR_CIS_ASSIGN_FILE_PERMISSION
    fi
    for filepath in /etc/cron.hourly /etc/cron.daily /etc/cron.weekly /etc/cron.monthly /etc/cron.d; do
        if [[ -e $filepath ]]; then
            chmod 0600 $filepath || exit $ERR_CIS_ASSIGN_FILE_PERMISSION
        fi
    done

    # Docs: https://www.man7.org/linux/man-pages/man1/crontab.1.html
    # If cron.allow exists, then cron.deny is ignored. To minimize who can use cron, we
    # always want cron.allow and will default it to empty if it doesn't exist.
    # We also need to set appropriate permissions on it.
    # Since it will be ignored anyway, we delete cron.deny.
    touch /etc/cron.allow || exit $ERR_CIS_ASSIGN_FILE_PERMISSION
    chmod 640 /etc/cron.allow || exit $ERR_CIS_ASSIGN_FILE_PERMISSION
    rm -rf /etc/cron.deny || exit $ERR_CIS_ASSIGN_FILE_PERMISSION
}

# Helper function to replace or append settings to a setting file.
# This abstracts the general logic of:
#   1. Search for the setting (via a pattern passed in).
#   2. If it's there, replace it with desired setting line; otherwise append it to the end of the file.
#   3. Validate that there is now exactly one instance of the setting, and that it is the one we want.
replaceOrAppendSetting() {
    local SEARCH_PATTERN=$1
    local SETTING_LINE=$2
    local FILE=$3

    # Search and replace/append.
    if grep -E "$SEARCH_PATTERN" "$FILE" >/dev/null; then
        sed -E -i "s|${SEARCH_PATTERN}|${SETTING_LINE}|g" "$FILE" || exit $ERR_CIS_APPLY_PASSWORD_CONFIG
    else
        echo -e "\n${SETTING_LINE}" >>"$FILE"
    fi

    # After replacement/append, there should be exactly one line that sets the setting,
    # and it must have the value we want.
    # If not, then there's something wrong with this script.
    if [[ $(grep -E "$SEARCH_PATTERN" "$FILE") != "$SETTING_LINE" ]]; then
        echo "replacement was wrong"
        exit $ERR_CIS_APPLY_PASSWORD_CONFIG
    fi
}

# Creates the search pattern and setting lines for login.defs settings, and calls through
# to do the replacement. Note that this uses extended regular expressions, so both
# grep and sed need to be called as such.
#
# The search pattern is:
#   '^#{0,1} {0,1}' -- Line starts with 0 or 1 '#' followed by 0 or 1 space
#   '${1}\s+'       -- Then the setting name followed by one or more whitespace characters
#   '[0-9]+$'       -- Then one more more number, which is the setting value, which is the end of the line.
#
# This is based on a combination of the syntax for the file and real examples we've found.
replaceOrAppendLoginDefs() {
    replaceOrAppendSetting "^#{0,1} {0,1}${1}\s+[0-9]+$" "${1} ${2}" /etc/login.defs
}

# Creates the search pattern and setting lines for useradd default settings, and calls through
# to do the replacement. Note that this uses extended regular expressions, so both
# grep and sed need to be called as such.
#
# The search pattern is:
#   '^#{0,1} {0,1}' -- Line starts with 0 or 1 '#' followed by 0 or 1 space
#   '${1}='         -- Then the setting name followed by '='
#   '.*$'           -- Then 0 or nore of any character which is the end of the line.
#                      Note that this allows for a setting value to be there or not.
#
# This is based on a combination of the syntax for the file and real examples we've found.
replaceOrAppendUserAdd() {
    replaceOrAppendSetting "^#{0,1} {0,1}${1}=.*$" "${1}=${2}" /etc/default/useradd
}

setPWExpiration() {
    replaceOrAppendLoginDefs PASS_MAX_DAYS 90
    replaceOrAppendLoginDefs PASS_MIN_DAYS 7
    replaceOrAppendUserAdd INACTIVE 30
}

# Creates the search pattern and setting lines for the core dump settings, and calls through
# to do the replacement. Note that this uses extended regular expressions, so both
# grep and sed need to be called as such.
#
# The search pattern is:
#  '^#{0,1} {0,1}' -- Line starts with 0 or 1 '#' followed by 0 or 1 space
#  '${1}='         -- Then the setting name followed by '='
#  '.*$'           -- Then 0 or nore of any character which is the end of the line.
#
# This is based on a combination of the syntax for the file (https://www.man7.org/linux/man-pages/man5/coredump.conf.5.html)
# and real examples we've found.
replaceOrAppendCoreDump() {
    replaceOrAppendSetting "^#{0,1} {0,1}${1}=.*$" "${1}=${2}" /etc/systemd/coredump.conf
}

configureCoreDump() {
    replaceOrAppendCoreDump Storage none
    replaceOrAppendCoreDump ProcessSizeMax 0
}

fixDefaultUmaskForAccountCreation() {
    replaceOrAppendLoginDefs UMASK 027
}

function maskNfsServer() {
    # If nfs-server.service exists, we need to mask it per CIS requirement.
    # Note that on ubuntu systems, it isn't installed but on mariner we need it
    # due to a dependency, but disable it by default.
    if systemctl list-unit-files nfs-server.service >/dev/null; then
        systemctl --now mask nfs-server || $ERR_SYSTEMCTL_MASK_FAIL
    fi
}

function addFailLockDir() {
    # Mariner V2 uses pamd faillocking, which requires a directory to store the faillock files.
    # Default is /var/run/faillock, but that's a tmpfs, so we need to use /var/log/faillock instead.
    # But we need to leave settings alone for other skus.
    if [[ "${OS}" == "${MARINER_OS_NAME}" && "${OS_VERSION}" == "2.0" ]]; then
        # Replace or append the dir setting in /etc/security/faillock.conf
        # Docs: https://www.man7.org/linux/man-pages/man5/faillock.conf.5.html
        #
        # Search pattern is:
        # '^#{0,1} {0,1}' -- Line starts with 0 or 1 '#' followed by 0 or 1 space
        # 'dir\s+'        -- Then the setting name followed by one or more whitespace characters
        # '.*$'           -- Then 0 or nore of any character which is the end of the line.
        #
        # This is based on a combination of the syntax for the file and real examples we've found.
        local fail_lock_dir="/var/log/faillock"
        mkdir -p ${fail_lock_dir}
        replaceOrAppendSetting "^#{0,1} {0,1}dir\s+.*$" "dir = ${fail_lock_dir}" /etc/security/faillock.conf
    fi
}

applyCIS() {
    setPWExpiration
    assignRootPW
    assignFilePermissions
    configureCoreDump
    fixDefaultUmaskForAccountCreation
    maskNfsServer
    addFailLockDir
}

applyCIS

#EOF


PROVISION_OUTPUT="/var/log/azure/cluster-provision-cse-output.log";
echo $(date),$(hostname) > ${PROVISION_OUTPUT};
{{if ShouldEnableCustomData}}
cloud-init status --wait > /dev/null 2>&1;
[ $? -ne 0 ] && echo 'cloud-init failed' >> ${PROVISION_OUTPUT} && exit 1;
echo "cloud-init succeeded" >> ${PROVISION_OUTPUT};
{{end}}
{{if IsAKSCustomCloud}}
REPO_DEPOT_ENDPOINT="{{AKSCustomCloudRepoDepotEndpoint}}"
{{GetInitAKSCustomCloudFilepath}} >> /var/log/azure/cluster-provision.log 2>&1;
{{end}}
ADMINUSER={{GetParameter "linuxAdminUsername"}}
MOBY_VERSION={{GetParameter "mobyVersion"}}
TENANT_ID={{GetVariable "tenantID"}}
KUBERNETES_VERSION={{GetParameter "kubernetesVersion"}}
HYPERKUBE_URL={{GetParameter "kubernetesHyperkubeSpec"}}
KUBE_BINARY_URL={{GetParameter "kubeBinaryURL"}}
CUSTOM_KUBE_BINARY_URL={{GetParameter "customKubeBinaryURL"}}
KUBEPROXY_URL={{GetParameter "kubeProxySpec"}}
APISERVER_PUBLIC_KEY={{GetParameter "apiServerCertificate"}}
SUBSCRIPTION_ID={{GetVariable "subscriptionId"}}
RESOURCE_GROUP={{GetVariable "resourceGroup"}}
LOCATION={{GetVariable "location"}}
VM_TYPE={{GetVariable "vmType"}}
SUBNET={{GetVariable "subnetName"}}
NETWORK_SECURITY_GROUP={{GetVariable "nsgName"}}
VIRTUAL_NETWORK={{GetVariable "virtualNetworkName"}}
VIRTUAL_NETWORK_RESOURCE_GROUP={{GetVariable "virtualNetworkResourceGroupName"}}
ROUTE_TABLE={{GetVariable "routeTableName"}}
PRIMARY_AVAILABILITY_SET={{GetVariable "primaryAvailabilitySetName"}}
PRIMARY_SCALE_SET={{GetVariable "primaryScaleSetName"}}
SERVICE_PRINCIPAL_CLIENT_ID={{GetParameter "servicePrincipalClientId"}}
NETWORK_PLUGIN={{GetParameter "networkPlugin"}}
NETWORK_POLICY={{GetParameter "networkPolicy"}}
VNET_CNI_PLUGINS_URL={{GetParameter "vnetCniLinuxPluginsURL"}}
CNI_PLUGINS_URL={{GetParameter "cniPluginsURL"}}
CLOUDPROVIDER_BACKOFF={{GetParameterProperty "cloudproviderConfig" "cloudProviderBackoff"}}
CLOUDPROVIDER_BACKOFF_MODE={{GetParameterProperty "cloudproviderConfig" "cloudProviderBackoffMode"}}
CLOUDPROVIDER_BACKOFF_RETRIES={{GetParameterProperty "cloudproviderConfig" "cloudProviderBackoffRetries"}}
CLOUDPROVIDER_BACKOFF_EXPONENT={{GetParameterProperty "cloudproviderConfig" "cloudProviderBackoffExponent"}}
CLOUDPROVIDER_BACKOFF_DURATION={{GetParameterProperty "cloudproviderConfig" "cloudProviderBackoffDuration"}}
CLOUDPROVIDER_BACKOFF_JITTER={{GetParameterProperty "cloudproviderConfig" "cloudProviderBackoffJitter"}}
CLOUDPROVIDER_RATELIMIT={{GetParameterProperty "cloudproviderConfig" "cloudProviderRateLimit"}}
CLOUDPROVIDER_RATELIMIT_QPS={{GetParameterProperty "cloudproviderConfig" "cloudProviderRateLimitQPS"}}
CLOUDPROVIDER_RATELIMIT_QPS_WRITE={{GetParameterProperty "cloudproviderConfig" "cloudProviderRateLimitQPSWrite"}}
CLOUDPROVIDER_RATELIMIT_BUCKET={{GetParameterProperty "cloudproviderConfig" "cloudProviderRateLimitBucket"}}
CLOUDPROVIDER_RATELIMIT_BUCKET_WRITE={{GetParameterProperty "cloudproviderConfig" "cloudProviderRateLimitBucketWrite"}}
LOAD_BALANCER_DISABLE_OUTBOUND_SNAT={{GetParameterProperty "cloudproviderConfig" "cloudProviderDisableOutboundSNAT"}}
USE_MANAGED_IDENTITY_EXTENSION={{GetVariable "useManagedIdentityExtension"}}
USE_INSTANCE_METADATA={{GetVariable "useInstanceMetadata"}}
LOAD_BALANCER_SKU={{GetVariable "loadBalancerSku"}}
EXCLUDE_MASTER_FROM_STANDARD_LB={{GetVariable "excludeMasterFromStandardLB"}}
MAXIMUM_LOADBALANCER_RULE_COUNT={{GetVariable "maximumLoadBalancerRuleCount"}}
CONTAINER_RUNTIME={{GetParameter "containerRuntime"}}
CLI_TOOL={{GetParameter "cliTool"}}
CONTAINERD_DOWNLOAD_URL_BASE={{GetParameter "containerdDownloadURLBase"}}
NETWORK_MODE={{GetParameter "networkMode"}}
KUBE_BINARY_URL={{GetParameter "kubeBinaryURL"}}
USER_ASSIGNED_IDENTITY_ID={{GetVariable "userAssignedIdentityID"}}
API_SERVER_NAME={{GetKubernetesEndpoint}}
IS_VHD={{GetVariable "isVHD"}}
GPU_NODE={{GetVariable "gpuNode"}}
SGX_NODE={{GetVariable "sgxNode"}}
MIG_NODE={{GetVariable "migNode"}}
CONFIG_GPU_DRIVER_IF_NEEDED={{GetVariable "configGPUDriverIfNeeded"}}
ENABLE_GPU_DEVICE_PLUGIN_IF_NEEDED={{GetVariable "enableGPUDevicePluginIfNeeded"}}
TELEPORTD_PLUGIN_DOWNLOAD_URL={{GetParameter "teleportdPluginURL"}}
CONTAINERD_VERSION={{GetParameter "containerdVersion"}}
CONTAINERD_PACKAGE_URL={{GetParameter "containerdPackageURL"}}
RUNC_VERSION={{GetParameter "runcVersion"}}
RUNC_PACKAGE_URL={{GetParameter "runcPackageURL"}}
ENABLE_HOSTS_CONFIG_AGENT="{{EnableHostsConfigAgent}}"
DISABLE_SSH="{{ShouldDisableSSH}}"
NEEDS_CONTAINERD="{{NeedsContainerd}}"
TELEPORT_ENABLED="{{TeleportEnabled}}"
SHOULD_CONFIGURE_HTTP_PROXY="{{ShouldConfigureHTTPProxy}}"
SHOULD_CONFIGURE_HTTP_PROXY_CA="{{ShouldConfigureHTTPProxyCA}}"
HTTP_PROXY_TRUSTED_CA="{{GetHTTPProxyCA}}"
SHOULD_CONFIGURE_CUSTOM_CA_TRUST="{{ShouldConfigureCustomCATrust}}"
CUSTOM_CA_TRUST_COUNT="{{len GetCustomCATrustConfigCerts}}"
{{range $i, $cert := GetCustomCATrustConfigCerts}}
CUSTOM_CA_CERT_{{$i}}="{{$cert}}"
{{end}}
IS_KRUSTLET="{{IsKrustlet}}"
GPU_NEEDS_FABRIC_MANAGER="{{GPUNeedsFabricManager}}"
NEEDS_DOCKER_LOGIN="{{and IsDockerContainerRuntime HasPrivateAzureRegistryServer}}"
IPV6_DUAL_STACK_ENABLED="{{IsIPv6DualStackFeatureEnabled}}"
OUTBOUND_COMMAND="{{GetOutboundCommand}}"
ENABLE_UNATTENDED_UPGRADES="{{EnableUnattendedUpgrade}}"
ENSURE_NO_DUPE_PROMISCUOUS_BRIDGE="{{ and NeedsContainerd IsKubenet (not HasCalicoNetworkPolicy) }}"
SHOULD_CONFIG_SWAP_FILE="{{ShouldConfigSwapFile}}"
SHOULD_CONFIG_TRANSPARENT_HUGE_PAGE="{{ShouldConfigTransparentHugePage}}"
SHOULD_CONFIG_CONTAINERD_ULIMITS="{{ShouldConfigContainerdUlimits}}"
CONTAINERD_ULIMITS="{{GetContainerdUlimitString}}"
{{/* both CLOUD and ENVIRONMENT have special values when IsAKSCustomCloud == true */}}
{{/* CLOUD uses AzureStackCloud and seems to be used by kubelet, k8s cloud provider */}}
{{/* target environment seems to go to ARM SDK config */}}
{{/* not sure why separate/inconsistent? */}}
{{/* see GetCustomEnvironmentJSON for more weirdness. */}}
TARGET_CLOUD="{{- if IsAKSCustomCloud -}} AzureStackCloud {{- else -}} {{GetTargetEnvironment}} {{- end -}}"
TARGET_ENVIRONMENT="{{GetTargetEnvironment}}"
CUSTOM_ENV_JSON="{{GetBase64EncodedEnvironmentJSON}}"
IS_CUSTOM_CLOUD="{{IsAKSCustomCloud}}"
CSE_HELPERS_FILEPATH="{{GetCSEHelpersScriptFilepath}}"
CSE_DISTRO_HELPERS_FILEPATH="{{GetCSEHelpersScriptDistroFilepath}}"
CSE_INSTALL_FILEPATH="{{GetCSEInstallScriptFilepath}}"
CSE_DISTRO_INSTALL_FILEPATH="{{GetCSEInstallScriptDistroFilepath}}"
CSE_CONFIG_FILEPATH="{{GetCSEConfigScriptFilepath}}"
AZURE_PRIVATE_REGISTRY_SERVER="{{GetPrivateAzureRegistryServer}}"
HAS_CUSTOM_SEARCH_DOMAIN="{{HasCustomSearchDomain}}"
CUSTOM_SEARCH_DOMAIN_FILEPATH="{{GetCustomSearchDomainsCSEScriptFilepath}}"
HTTP_PROXY_URLS="{{GetHTTPProxy}}"
HTTPS_PROXY_URLS="{{GetHTTPSProxy}}"
NO_PROXY_URLS="{{GetNoProxy}}"
PROXY_VARS="{{GetProxyVariables}}"
CLIENT_TLS_BOOTSTRAPPING_ENABLED="{{IsKubeletClientTLSBootstrappingEnabled}}"
DHCPV6_SERVICE_FILEPATH="{{GetDHCPv6ServiceCSEScriptFilepath}}"
DHCPV6_CONFIG_FILEPATH="{{GetDHCPv6ConfigCSEScriptFilepath}}"
THP_ENABLED="{{GetTransparentHugePageEnabled}}"
THP_DEFRAG="{{GetTransparentHugePageDefrag}}"
SERVICE_PRINCIPAL_FILE_CONTENT="{{GetServicePrincipalSecret}}"
KUBELET_CLIENT_CONTENT="{{GetKubeletClientKey}}"
KUBELET_CLIENT_CERT_CONTENT="{{GetKubeletClientCert}}"
KUBELET_CONFIG_FILE_ENABLED="{{IsKubeletConfigFileEnabled}}"
KUBELET_CONFIG_FILE_CONTENT="{{GetKubeletConfigFileContentBase64}}"
SWAP_FILE_SIZE_MB="{{GetSwapFileSizeMB}}"
GPU_DRIVER_VERSION="{{GPUDriverVersion}}"
GPU_INSTANCE_PROFILE="{{GetGPUInstanceProfile}}"
CUSTOM_SEARCH_DOMAIN_NAME="{{GetSearchDomainName}}"
CUSTOM_SEARCH_REALM_USER="{{GetSearchDomainRealmUser}}"
CUSTOM_SEARCH_REALM_PASSWORD="{{GetSearchDomainRealmPassword}}"
MESSAGE_OF_THE_DAY="{{GetMessageOfTheDay}}"
HAS_KUBELET_DISK_TYPE="{{HasKubeletDiskType}}"
NEEDS_CGROUPV2="{{Is2204VHD}}"
TLS_BOOTSTRAP_TOKEN="{{GetTLSBootstrapTokenForKubeConfig}}"
KUBELET_FLAGS="{{GetKubeletConfigKeyVals}}"
NETWORK_POLICY="{{GetParameter "networkPolicy"}}"
{{- if not (IsKubernetesVersionGe "1.17.0")}}
KUBELET_IMAGE="{{GetHyperkubeImageReference}}"
{{end}}
{{if IsKubernetesVersionGe "1.16.0"}}
KUBELET_NODE_LABELS="{{GetAgentKubernetesLabels . }}"
{{else}}
KUBELET_NODE_LABELS="{{GetAgentKubernetesLabelsDeprecated . }}"
{{end}}
AZURE_ENVIRONMENT_FILEPATH="{{- if IsAKSCustomCloud}}/etc/kubernetes/{{GetTargetEnvironment}}.json{{end}}"
KUBE_CA_CRT="{{GetParameter "caCertificate"}}"
KUBENET_TEMPLATE="{{GetKubenetTemplate}}"
CONTAINERD_CONFIG_CONTENT="{{GetContainerdConfigContent}}"
CONTAINERD_CONFIG_NO_GPU_CONTENT="{{GetContainerdConfigNoGPUContent}}"
IS_KATA="{{IsKata}}"
SYSCTL_CONTENT="{{GetSysctlContent}}"
/usr/bin/nohup /bin/bash -c "/bin/bash /opt/azure/containers/provision_start.sh"


#!/bin/bash
NODE_INDEX=$(hostname | tail -c 2)
NODE_NAME=$(hostname)

configureAdminUser(){
    chage -E -1 -I -1 -m 0 -M 99999 "${ADMINUSER}"
    chage -l "${ADMINUSER}"
}

configPrivateClusterHosts() {
    mkdir -p /etc/systemd/system/reconcile-private-hosts.service.d/
    touch /etc/systemd/system/reconcile-private-hosts.service.d/10-fqdn.conf
    tee /etc/systemd/system/reconcile-private-hosts.service.d/10-fqdn.conf > /dev/null <<EOF
[Service]
Environment="KUBE_API_SERVER_NAME=${API_SERVER_NAME}"
EOF
  systemctlEnableAndStart reconcile-private-hosts || exit $ERR_SYSTEMCTL_START_FAIL
}
configureTransparentHugePage() {
    ETC_SYSFS_CONF="/etc/sysfs.conf"
    if [[ "${THP_ENABLED}" != "" ]]; then
        echo "${THP_ENABLED}" > /sys/kernel/mm/transparent_hugepage/enabled
        echo "kernel/mm/transparent_hugepage/enabled=${THP_ENABLED}" >> ${ETC_SYSFS_CONF}
    fi
    if [[ "${THP_DEFRAG}" != "" ]]; then
        echo "${THP_DEFRAG}" > /sys/kernel/mm/transparent_hugepage/defrag
        echo "kernel/mm/transparent_hugepage/defrag=${THP_DEFRAG}" >> ${ETC_SYSFS_CONF}
    fi
}

configureSwapFile() {
    # https://learn.microsoft.com/en-us/troubleshoot/azure/virtual-machines/troubleshoot-device-names-problems#identify-disk-luns
    swap_size_kb=$(expr ${SWAP_FILE_SIZE_MB} \* 1000)
    swap_location=""

    # Attempt to use the resource disk
    if [[ -L /dev/disk/azure/resource-part1 ]]; then
        resource_disk_path=$(findmnt -nr -o target -S $(readlink -f /dev/disk/azure/resource-part1))
        disk_free_kb=$(df ${resource_disk_path} | sed 1d | awk '{print $4}')
        if [[ ${disk_free_kb} -gt ${swap_size_kb} ]]; then
            echo "Will use resource disk for swap file"
            swap_location=${resource_disk_path}/swapfile
        else
            echo "Insufficient disk space on resource disk to create swap file: request ${swap_size_kb} free ${disk_free_kb}, attempting to fall back to OS disk..."
        fi
    fi

    # If we couldn't use the resource disk, attempt to use the OS disk
    if [[ -z "${swap_location}" ]]; then
        # Directly check size on the root directory since we can't rely on 'root-part1' always being the correct label
        os_device=$(readlink -f /dev/disk/azure/root)
        disk_free_kb=$(df -P / | sed 1d | awk '{print $4}')
        if [[ ${disk_free_kb} -gt ${swap_size_kb} ]]; then
            echo "Will use OS disk for swap file"
            swap_location=/swapfile
        else
            echo "Insufficient disk space on OS device ${os_device} to create swap file: request ${swap_size_kb} free ${disk_free_kb}"
            exit $ERR_SWAP_CREATE_INSUFFICIENT_DISK_SPACE
        fi
    fi

    echo "Swap file will be saved to: ${swap_location}"
    retrycmd_if_failure 24 5 25 fallocate -l ${swap_size_kb}K ${swap_location} || exit $ERR_SWAP_CREATE_FAIL
    chmod 600 ${swap_location}
    retrycmd_if_failure 24 5 25 mkswap ${swap_location} || exit $ERR_SWAP_CREATE_FAIL
    retrycmd_if_failure 24 5 25 swapon ${swap_location} || exit $ERR_SWAP_CREATE_FAIL
    retrycmd_if_failure 24 5 25 swapon --show | grep ${swap_location} || exit $ERR_SWAP_CREATE_FAIL
    echo "${swap_location} none swap sw 0 0" >> /etc/fstab
}

configureEtcEnvironment() {
    mkdir -p /etc/systemd/system.conf.d/
    touch /etc/systemd/system.conf.d/proxy.conf
    chmod 0644 /etc/systemd/system.conf.d/proxy.conf

    mkdir -p  /etc/apt/apt.conf.d
    chmod 0644 /etc/apt/apt.conf.d/95proxy
    touch /etc/apt/apt.conf.d/95proxy

    # TODO(ace): this pains me but quick and dirty refactor
    echo "[Manager]" >> /etc/systemd/system.conf.d/proxy.conf
    if [ "${HTTP_PROXY_URLS}" != "" ]; then
        echo "HTTP_PROXY=${HTTP_PROXY_URLS}" >> /etc/environment
        echo "http_proxy=${HTTP_PROXY_URLS}" >> /etc/environment
        echo "Acquire::http::proxy \"${HTTP_PROXY_URLS}\";" >> /etc/apt/apt.conf.d/95proxy
        echo "DefaultEnvironment=\"HTTP_PROXY=${HTTP_PROXY_URLS}\"" >> /etc/systemd/system.conf.d/proxy.conf
        echo "DefaultEnvironment=\"http_proxy=${HTTP_PROXY_URLS}\"" >> /etc/systemd/system.conf.d/proxy.conf
    fi
    if [ "${HTTPS_PROXY_URLS}" != "" ]; then
        echo "HTTPS_PROXY=${HTTPS_PROXY_URLS}" >> /etc/environment
        echo "https_proxy=${HTTPS_PROXY_URLS}" >> /etc/environment
        echo "Acquire::https::proxy \"${HTTPS_PROXY_URLS}\";" >> /etc/apt/apt.conf.d/95proxy
        echo "DefaultEnvironment=\"HTTPS_PROXY=${HTTPS_PROXY_URLS}\"" >> /etc/systemd/system.conf.d/proxy.conf
        echo "DefaultEnvironment=\"https_proxy=${HTTPS_PROXY_URLS}\"" >> /etc/systemd/system.conf.d/proxy.conf
    fi
    if [ "${NO_PROXY_URLS}" != "" ]; then
        echo "NO_PROXY=${NO_PROXY_URLS}" >> /etc/environment
        echo "no_proxy=${NO_PROXY_URLS}" >> /etc/environment
        echo "DefaultEnvironment=\"NO_PROXY=${NO_PROXY_URLS}\"" >> /etc/systemd/system.conf.d/proxy.conf
        echo "DefaultEnvironment=\"no_proxy=${NO_PROXY_URLS}\"" >> /etc/systemd/system.conf.d/proxy.conf
    fi

    # for kubelet to pick up the proxy
    mkdir -p "/etc/systemd/system/kubelet.service.d"
    tee "/etc/systemd/system/kubelet.service.d/10-httpproxy.conf" > /dev/null <<'EOF'
[Service]
EnvironmentFile=/etc/environment
EOF
}

configureHTTPProxyCA() {
    if [[ $OS == $MARINER_OS_NAME ]]; then
        cert_dest="/usr/share/pki/ca-trust-source/anchors"
        update_cmd="update-ca-trust"
    else
        cert_dest="/usr/local/share/ca-certificates"
        update_cmd="update-ca-certificates"
    fi
    echo "${HTTP_PROXY_TRUSTED_CA}" | base64 -d > "${cert_dest}/proxyCA.crt" || exit $ERR_UPDATE_CA_CERTS
    $update_cmd || exit $ERR_UPDATE_CA_CERTS
}

configureCustomCaCertificate() {
    mkdir -p /opt/certs
    for i in $(seq 0 $((${CUSTOM_CA_TRUST_COUNT} - 1))); do
        # directly referring to the variable as "${CUSTOM_CA_CERT_${i}}"
        # causes bad substitution errors in bash
        # dynamically declare and use `!` to add a layer of indirection
        declare varname=CUSTOM_CA_CERT_${i}
        echo "${!varname}" | base64 -d > /opt/certs/00000000000000cert${i}.crt
    done
    # This will block until the service is considered active.
    # Update_certs.service is a oneshot type of unit that
    # is considered active when the ExecStart= command terminates with a zero status code.
    systemctl restart update_certs.service || exit $ERR_UPDATE_CA_CERTS
    # after new certs are added to trust store, containerd will not pick them up properly before restart.
    # aim here is to have this working straight away for a freshly provisioned node
    # so we force a restart after the certs are updated
    # custom CA daemonset copies certs passed by the user to the node, what then triggers update_certs.path unit
    # path unit then triggers the script that copies over cert files to correct location on the node and updates the trust store
    # as a part of this flow we could restart containerd everytime a new cert is added to the trust store using custom CA
    systemctl restart containerd
}

configureContainerdUlimits() {
  CONTAINERD_ULIMIT_DROP_IN_FILE_PATH="/etc/systemd/system/containerd.service.d/set_ulimits.conf"
  touch "${CONTAINERD_ULIMIT_DROP_IN_FILE_PATH}"
  chmod 0600 "${CONTAINERD_ULIMIT_DROP_IN_FILE_PATH}"
  tee "${CONTAINERD_ULIMIT_DROP_IN_FILE_PATH}" > /dev/null <<EOF
$(echo "$CONTAINERD_ULIMITS" | tr ' ' '\n')
EOF

  systemctl daemon-reload
  systemctl restart containerd
}

configureKubeletServerCert() {
    KUBELET_SERVER_PRIVATE_KEY_PATH="/etc/kubernetes/certs/kubeletserver.key"
    KUBELET_SERVER_CERT_PATH="/etc/kubernetes/certs/kubeletserver.crt"

    openssl genrsa -out $KUBELET_SERVER_PRIVATE_KEY_PATH 2048
    openssl req -new -x509 -days 7300 -key $KUBELET_SERVER_PRIVATE_KEY_PATH -out $KUBELET_SERVER_CERT_PATH -subj "/CN=${NODE_NAME}" -addext "subjectAltName=DNS:${NODE_NAME}"
}

configureK8s() {
    APISERVER_PUBLIC_KEY_PATH="/etc/kubernetes/certs/apiserver.crt"
    touch "${APISERVER_PUBLIC_KEY_PATH}"
    chmod 0644 "${APISERVER_PUBLIC_KEY_PATH}"
    chown root:root "${APISERVER_PUBLIC_KEY_PATH}"

    AZURE_JSON_PATH="/etc/kubernetes/azure.json"
    touch "${AZURE_JSON_PATH}"
    chmod 0600 "${AZURE_JSON_PATH}"
    chown root:root "${AZURE_JSON_PATH}"

    mkdir -p "/etc/kubernetes/certs"
    if [ -n "${KUBELET_CLIENT_CONTENT}" ]; then
        echo "${KUBELET_CLIENT_CONTENT}" | base64 -d > /etc/kubernetes/certs/client.key
    fi
    if [ -n "${KUBELET_CLIENT_CERT_CONTENT}" ]; then
        echo "${KUBELET_CLIENT_CERT_CONTENT}" | base64 -d > /etc/kubernetes/certs/client.crt
    fi
    if [ -n "${SERVICE_PRINCIPAL_FILE_CONTENT}" ]; then
        echo "${SERVICE_PRINCIPAL_FILE_CONTENT}" | base64 -d > /etc/kubernetes/sp.txt
    fi

    set +x
    echo "${APISERVER_PUBLIC_KEY}" | base64 --decode > "${APISERVER_PUBLIC_KEY_PATH}"
    # Perform the required JSON escaping
    SP_FILE="/etc/kubernetes/sp.txt"
    SERVICE_PRINCIPAL_CLIENT_SECRET="$(cat "$SP_FILE")"
    SERVICE_PRINCIPAL_CLIENT_SECRET=${SERVICE_PRINCIPAL_CLIENT_SECRET//\\/\\\\}
    SERVICE_PRINCIPAL_CLIENT_SECRET=${SERVICE_PRINCIPAL_CLIENT_SECRET//\"/\\\"}
    rm "$SP_FILE" # unneeded after reading from disk.
    cat << EOF > "${AZURE_JSON_PATH}"
{
    "cloud": "${TARGET_CLOUD}",
    "tenantId": "${TENANT_ID}",
    "subscriptionId": "${SUBSCRIPTION_ID}",
    "aadClientId": "${SERVICE_PRINCIPAL_CLIENT_ID}",
    "aadClientSecret": "${SERVICE_PRINCIPAL_CLIENT_SECRET}",
    "resourceGroup": "${RESOURCE_GROUP}",
    "location": "${LOCATION}",
    "vmType": "${VM_TYPE}",
    "subnetName": "${SUBNET}",
    "securityGroupName": "${NETWORK_SECURITY_GROUP}",
    "vnetName": "${VIRTUAL_NETWORK}",
    "vnetResourceGroup": "${VIRTUAL_NETWORK_RESOURCE_GROUP}",
    "routeTableName": "${ROUTE_TABLE}",
    "primaryAvailabilitySetName": "${PRIMARY_AVAILABILITY_SET}",
    "primaryScaleSetName": "${PRIMARY_SCALE_SET}",
    "cloudProviderBackoffMode": "${CLOUDPROVIDER_BACKOFF_MODE}",
    "cloudProviderBackoff": ${CLOUDPROVIDER_BACKOFF},
    "cloudProviderBackoffRetries": ${CLOUDPROVIDER_BACKOFF_RETRIES},
    "cloudProviderBackoffExponent": ${CLOUDPROVIDER_BACKOFF_EXPONENT},
    "cloudProviderBackoffDuration": ${CLOUDPROVIDER_BACKOFF_DURATION},
    "cloudProviderBackoffJitter": ${CLOUDPROVIDER_BACKOFF_JITTER},
    "cloudProviderRateLimit": ${CLOUDPROVIDER_RATELIMIT},
    "cloudProviderRateLimitQPS": ${CLOUDPROVIDER_RATELIMIT_QPS},
    "cloudProviderRateLimitBucket": ${CLOUDPROVIDER_RATELIMIT_BUCKET},
    "cloudProviderRateLimitQPSWrite": ${CLOUDPROVIDER_RATELIMIT_QPS_WRITE},
    "cloudProviderRateLimitBucketWrite": ${CLOUDPROVIDER_RATELIMIT_BUCKET_WRITE},
    "useManagedIdentityExtension": ${USE_MANAGED_IDENTITY_EXTENSION},
    "userAssignedIdentityID": "${USER_ASSIGNED_IDENTITY_ID}",
    "useInstanceMetadata": ${USE_INSTANCE_METADATA},
    "loadBalancerSku": "${LOAD_BALANCER_SKU}",
    "disableOutboundSNAT": ${LOAD_BALANCER_DISABLE_OUTBOUND_SNAT},
    "excludeMasterFromStandardLB": ${EXCLUDE_MASTER_FROM_STANDARD_LB},
    "providerVaultName": "${KMS_PROVIDER_VAULT_NAME}",
    "maximumLoadBalancerRuleCount": ${MAXIMUM_LOADBALANCER_RULE_COUNT},
    "providerKeyName": "k8s",
    "providerKeyVersion": ""
}
EOF
    set -x
    if [[ "${CLOUDPROVIDER_BACKOFF_MODE}" = "v2" ]]; then
        sed -i "/cloudProviderBackoffExponent/d" /etc/kubernetes/azure.json
        sed -i "/cloudProviderBackoffJitter/d" /etc/kubernetes/azure.json
    fi

    configureKubeletServerCert
    if [ "${IS_CUSTOM_CLOUD}" == "true" ]; then
        set +x
        AKS_CUSTOM_CLOUD_JSON_PATH="/etc/kubernetes/${TARGET_ENVIRONMENT}.json"
        touch "${AKS_CUSTOM_CLOUD_JSON_PATH}"
        chmod 0600 "${AKS_CUSTOM_CLOUD_JSON_PATH}"
        chown root:root "${AKS_CUSTOM_CLOUD_JSON_PATH}"

        echo "${CUSTOM_ENV_JSON}" | base64 -d > "${AKS_CUSTOM_CLOUD_JSON_PATH}"
        set -x
    fi

    if [ "${KUBELET_CONFIG_FILE_ENABLED}" == "true" ]; then
        set +x
        KUBELET_CONFIG_JSON_PATH="/etc/default/kubeletconfig.json"
        touch "${KUBELET_CONFIG_JSON_PATH}"
        chmod 0600 "${KUBELET_CONFIG_JSON_PATH}"
        chown root:root "${KUBELET_CONFIG_JSON_PATH}"
        echo "${KUBELET_CONFIG_FILE_CONTENT}" | base64 -d > "${KUBELET_CONFIG_JSON_PATH}"
        set -x
        KUBELET_CONFIG_DROP_IN="/etc/systemd/system/kubelet.service.d/10-componentconfig.conf"
        touch "${KUBELET_CONFIG_DROP_IN}"
        chmod 0600 "${KUBELET_CONFIG_DROP_IN}"
        tee "${KUBELET_CONFIG_DROP_IN}" > /dev/null <<EOF
[Service]
Environment="KUBELET_CONFIG_FILE_FLAGS=--config /etc/default/kubeletconfig.json"
EOF
    fi
}

configureCNI() {
    # needed for the iptables rules to work on bridges
    retrycmd_if_failure 120 5 25 modprobe br_netfilter || exit $ERR_MODPROBE_FAIL
    echo -n "br_netfilter" > /etc/modules-load.d/br_netfilter.conf
    configureCNIIPTables
}

configureCNIIPTables() {
    if [[ "${NETWORK_PLUGIN}" = "azure" ]]; then
        mv $CNI_BIN_DIR/10-azure.conflist $CNI_CONFIG_DIR/
        chmod 600 $CNI_CONFIG_DIR/10-azure.conflist
        if [[ "${NETWORK_POLICY}" == "calico" ]]; then
          sed -i 's#"mode":"bridge"#"mode":"transparent"#g' $CNI_CONFIG_DIR/10-azure.conflist
        elif [[ "${NETWORK_POLICY}" == "" || "${NETWORK_POLICY}" == "none" ]] && [[ "${NETWORK_MODE}" == "transparent" ]]; then
          sed -i 's#"mode":"bridge"#"mode":"transparent"#g' $CNI_CONFIG_DIR/10-azure.conflist
        fi
        /sbin/ebtables -t nat --list
    fi
}

disableSystemdResolved() {
    ls -ltr /etc/resolv.conf
    cat /etc/resolv.conf
    UBUNTU_RELEASE=$(lsb_release -r -s)
    if [[ "${UBUNTU_RELEASE}" == "18.04" || "${UBUNTU_RELEASE}" == "20.04" || "${UBUNTU_RELEASE}" == "22.04" ]]; then
        echo "Ingorings systemd-resolved query service but using its resolv.conf file"
        echo "This is the simplest approach to workaround resolved issues without completely uninstall it"
        [ -f /run/systemd/resolve/resolv.conf ] && sudo ln -sf /run/systemd/resolve/resolv.conf /etc/resolv.conf
        ls -ltr /etc/resolv.conf
        cat /etc/resolv.conf
    fi
}

ensureContainerd() {
  if [ "${TELEPORT_ENABLED}" == "true" ]; then
    ensureTeleportd
  fi
  mkdir -p "/etc/systemd/system/containerd.service.d"
  tee "/etc/systemd/system/containerd.service.d/exec_start.conf" > /dev/null <<EOF
[Service]
ExecStartPost=/sbin/iptables -P FORWARD ACCEPT
EOF

  mkdir -p /etc/containerd
  if [[ "${GPU_NODE}" = true ]] && [[ "${skip_nvidia_driver_install}" == "true" ]]; then
    echo "Generating non-GPU containerd config for GPU node due to VM tags"
    echo "${CONTAINERD_CONFIG_NO_GPU_CONTENT}" | base64 -d > /etc/containerd/config.toml || exit $ERR_FILE_WATCH_TIMEOUT
  else
    echo "Generating containerd config..."
    echo "${CONTAINERD_CONFIG_CONTENT}" | base64 -d > /etc/containerd/config.toml || exit $ERR_FILE_WATCH_TIMEOUT
  fi

  tee "/etc/sysctl.d/99-force-bridge-forward.conf" > /dev/null <<EOF
net.ipv4.ip_forward = 1
net.ipv4.conf.all.forwarding = 1
net.ipv6.conf.all.forwarding = 1
net.bridge.bridge-nf-call-iptables = 1
EOF
  retrycmd_if_failure 120 5 25 sysctl --system || exit $ERR_SYSCTL_RELOAD
  systemctl is-active --quiet docker && (systemctl_disable 20 30 120 docker || exit $ERR_SYSTEMD_DOCKER_STOP_FAIL)
  systemctlEnableAndStart containerd || exit $ERR_SYSTEMCTL_START_FAIL
}

ensureNoDupOnPromiscuBridge() {
    systemctlEnableAndStart ensure-no-dup || exit $ERR_SYSTEMCTL_START_FAIL
}

ensureTeleportd() {
    systemctlEnableAndStart teleportd || exit $ERR_SYSTEMCTL_START_FAIL
}

ensureDocker() {
    DOCKER_SERVICE_EXEC_START_FILE=/etc/systemd/system/docker.service.d/exec_start.conf
    usermod -aG docker ${ADMINUSER}
    DOCKER_MOUNT_FLAGS_SYSTEMD_FILE=/etc/systemd/system/docker.service.d/clear_mount_propagation_flags.conf
    DOCKER_JSON_FILE=/etc/docker/daemon.json
    for i in $(seq 1 1200); do
        if [ -s $DOCKER_JSON_FILE ]; then
            jq '.' < $DOCKER_JSON_FILE && break
        fi
        if [ $i -eq 1200 ]; then
            exit $ERR_FILE_WATCH_TIMEOUT
        else
            sleep 1
        fi
    done
    systemctl is-active --quiet containerd && (systemctl_disable 20 30 120 containerd || exit $ERR_SYSTEMD_CONTAINERD_STOP_FAIL)
    systemctlEnableAndStart docker || exit $ERR_DOCKER_START_FAIL

}

ensureDHCPv6() {
    systemctlEnableAndStart dhcpv6 || exit $ERR_SYSTEMCTL_START_FAIL
    retrycmd_if_failure 120 5 25 modprobe ip6_tables || exit $ERR_MODPROBE_FAIL
}

ensureKubelet() {
    KUBELET_DEFAULT_FILE=/etc/default/kubelet
    mkdir -p /etc/default
    echo "KUBELET_FLAGS=${KUBELET_FLAGS}" > "${KUBELET_DEFAULT_FILE}"
    echo "KUBELET_REGISTER_SCHEDULABLE=true" >> "${KUBELET_DEFAULT_FILE}"
    echo "NETWORK_POLICY=${NETWORK_POLICY}" >> "${KUBELET_DEFAULT_FILE}"
    echo "KUBELET_IMAGE=${KUBELET_IMAGE}" >> "${KUBELET_DEFAULT_FILE}"
    echo "KUBELET_NODE_LABELS=${KUBELET_NODE_LABELS}" >> "${KUBELET_DEFAULT_FILE}"
    if [ -n "${AZURE_ENVIRONMENT_FILEPATH}" ]; then
        echo "AZURE_ENVIRONMENT_FILEPATH=${AZURE_ENVIRONMENT_FILEPATH}" >> "${KUBELET_DEFAULT_FILE}"
    fi

    KUBE_CA_FILE="/etc/kubernetes/certs/ca.crt"
    mkdir -p "$(dirname "${KUBE_CA_FILE}")"
    echo "${KUBE_CA_CRT}" | base64 -d > "${KUBE_CA_FILE}"
    chmod 0600 "${KUBE_CA_FILE}"

    if [ "${CLIENT_TLS_BOOTSTRAPPING_ENABLED}" == "true" ]; then
        KUBELET_TLS_DROP_IN="/etc/systemd/system/kubelet.service.d/10-tlsbootstrap.conf"
        mkdir -p "$(dirname "${KUBELET_TLS_DROP_IN}")"
        touch "${KUBELET_TLS_DROP_IN}"
        chmod 0600 "${KUBELET_TLS_DROP_IN}"
        tee "${KUBELET_TLS_DROP_IN}" > /dev/null <<EOF
[Service]
Environment="KUBELET_TLS_BOOTSTRAP_FLAGS=--kubeconfig /var/lib/kubelet/kubeconfig --bootstrap-kubeconfig /var/lib/kubelet/bootstrap-kubeconfig"
EOF
        BOOTSTRAP_KUBECONFIG_FILE=/var/lib/kubelet/bootstrap-kubeconfig
        mkdir -p "$(dirname "${BOOTSTRAP_KUBECONFIG_FILE}")"
        touch "${BOOTSTRAP_KUBECONFIG_FILE}"
        chmod 0644 "${BOOTSTRAP_KUBECONFIG_FILE}"
        tee "${BOOTSTRAP_KUBECONFIG_FILE}" > /dev/null <<EOF
apiVersion: v1
kind: Config
clusters:
- name: localcluster
  cluster:
    certificate-authority: /etc/kubernetes/certs/ca.crt
    server: https://${API_SERVER_NAME}:443
users:
- name: kubelet-bootstrap
  user:
    token: "${TLS_BOOTSTRAP_TOKEN}"
contexts:
- context:
    cluster: localcluster
    user: kubelet-bootstrap
  name: bootstrap-context
current-context: bootstrap-context
EOF
    else
        KUBECONFIG_FILE=/var/lib/kubelet/kubeconfig
        mkdir -p "$(dirname "${KUBECONFIG_FILE}")"
        touch "${KUBECONFIG_FILE}"
        chmod 0644 "${KUBECONFIG_FILE}"
        tee "${KUBECONFIG_FILE}" > /dev/null <<EOF
apiVersion: v1
kind: Config
clusters:
- name: localcluster
  cluster:
    certificate-authority: /etc/kubernetes/certs/ca.crt
    server: https://${API_SERVER_NAME}:443
users:
- name: client
  user:
    client-certificate: /etc/kubernetes/certs/client.crt
    client-key: /etc/kubernetes/certs/client.key
contexts:
- context:
    cluster: localcluster
    user: client
  name: localclustercontext
current-context: localclustercontext
EOF
    fi
    KUBELET_RUNTIME_CONFIG_SCRIPT_FILE=/opt/azure/containers/kubelet.sh
    tee "${KUBELET_RUNTIME_CONFIG_SCRIPT_FILE}" > /dev/null <<EOF
#!/bin/bash
# Disallow container from reaching out to the special IP address 168.63.129.16
# for TCP protocol (which http uses)
#
# 168.63.129.16 contains protected settings that have priviledged info.
#
# The host can still reach 168.63.129.16 because it goes through the OUTPUT chain, not FORWARD.
#
# Note: we should not block all traffic to 168.63.129.16. For example UDP traffic is still needed
# for DNS.
iptables -I FORWARD -d 168.63.129.16 -p tcp --dport 80 -j DROP
EOF
    systemctlEnableAndStart kubelet || exit $ERR_KUBELET_START_FAIL
}

ensureMigPartition(){
    mkdir -p /etc/systemd/system/mig-partition.service.d/
    touch /etc/systemd/system/mig-partition.service.d/10-mig-profile.conf
    tee /etc/systemd/system/mig-partition.service.d/10-mig-profile.conf > /dev/null <<EOF
[Service]
Environment="GPU_INSTANCE_PROFILE=${GPU_INSTANCE_PROFILE}"
EOF
    # this is expected to fail and work only on next reboot
    # it MAY succeed, only due to unreliability of systemd
    # service type=Simple, which does not exit non-zero
    # on failure if ExecStart failed to invoke.
    systemctlEnableAndStart mig-partition
}

ensureSysctl() {
    SYSCTL_CONFIG_FILE=/etc/sysctl.d/999-sysctl-aks.conf
    mkdir -p "$(dirname "${SYSCTL_CONFIG_FILE}")"
    touch "${SYSCTL_CONFIG_FILE}"
    chmod 0644 "${SYSCTL_CONFIG_FILE}"
    echo "${SYSCTL_CONTENT}" | base64 -d > "${SYSCTL_CONFIG_FILE}"
    retrycmd_if_failure 24 5 25 sysctl --system
}

ensureK8sControlPlane() {
    if $REBOOTREQUIRED || [ "$NO_OUTBOUND" = "true" ]; then
        return
    fi
    retrycmd_if_failure 120 5 25 $KUBECTL 2>/dev/null cluster-info || exit $ERR_K8S_RUNNING_TIMEOUT
}

createKubeManifestDir() {
    KUBEMANIFESTDIR=/etc/kubernetes/manifests
    mkdir -p $KUBEMANIFESTDIR
}

writeKubeConfig() {
    KUBECONFIGDIR=/home/$ADMINUSER/.kube
    KUBECONFIGFILE=$KUBECONFIGDIR/config
    mkdir -p $KUBECONFIGDIR
    touch $KUBECONFIGFILE
    chown $ADMINUSER:$ADMINUSER $KUBECONFIGDIR
    chown $ADMINUSER:$ADMINUSER $KUBECONFIGFILE
    chmod 700 $KUBECONFIGDIR
    chmod 600 $KUBECONFIGFILE
    set +x
    echo "
---
apiVersion: v1
clusters:
- cluster:
    certificate-authority-data: \"$CA_CERTIFICATE\"
    server: $KUBECONFIG_SERVER
  name: \"$MASTER_FQDN\"
contexts:
- context:
    cluster: \"$MASTER_FQDN\"
    user: \"$MASTER_FQDN-admin\"
  name: \"$MASTER_FQDN\"
current-context: \"$MASTER_FQDN\"
kind: Config
users:
- name: \"$MASTER_FQDN-admin\"
  user:
    client-certificate-data: \"$KUBECONFIG_CERTIFICATE\"
    client-key-data: \"$KUBECONFIG_KEY\"
" > $KUBECONFIGFILE
    set -x
}

configClusterAutoscalerAddon() {
    CLUSTER_AUTOSCALER_ADDON_FILE=/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml
    sed -i "s|<clientID>|$(echo $SERVICE_PRINCIPAL_CLIENT_ID | base64)|g" $CLUSTER_AUTOSCALER_ADDON_FILE
    sed -i "s|<clientSec>|$(echo $SERVICE_PRINCIPAL_CLIENT_SECRET | base64)|g" $CLUSTER_AUTOSCALER_ADDON_FILE
    sed -i "s|<subID>|$(echo $SUBSCRIPTION_ID | base64)|g" $CLUSTER_AUTOSCALER_ADDON_FILE
    sed -i "s|<tenantID>|$(echo $TENANT_ID | base64)|g" $CLUSTER_AUTOSCALER_ADDON_FILE
    sed -i "s|<rg>|$(echo $RESOURCE_GROUP | base64)|g" $CLUSTER_AUTOSCALER_ADDON_FILE
}

configACIConnectorAddon() {
    ACI_CONNECTOR_CREDENTIALS=$(printf "{\"clientId\": \"%s\", \"clientSecret\": \"%s\", \"tenantId\": \"%s\", \"subscriptionId\": \"%s\", \"activeDirectoryEndpointUrl\": \"https://login.microsoftonline.com\",\"resourceManagerEndpointUrl\": \"https://management.azure.com/\", \"activeDirectoryGraphResourceId\": \"https://graph.windows.net/\", \"sqlManagementEndpointUrl\": \"https://management.core.windows.net:8443/\", \"galleryEndpointUrl\": \"https://gallery.azure.com/\", \"managementEndpointUrl\": \"https://management.core.windows.net/\"}" "$SERVICE_PRINCIPAL_CLIENT_ID" "$SERVICE_PRINCIPAL_CLIENT_SECRET" "$TENANT_ID" "$SUBSCRIPTION_ID" | base64 -w 0)

    openssl req -newkey rsa:4096 -new -nodes -x509 -days 3650 -keyout /etc/kubernetes/certs/aci-connector-key.pem -out /etc/kubernetes/certs/aci-connector-cert.pem -subj "/C=US/ST=CA/L=virtualkubelet/O=virtualkubelet/OU=virtualkubelet/CN=virtualkubelet"
    ACI_CONNECTOR_KEY=$(base64 /etc/kubernetes/certs/aci-connector-key.pem -w0)
    ACI_CONNECTOR_CERT=$(base64 /etc/kubernetes/certs/aci-connector-cert.pem -w0)

    ACI_CONNECTOR_ADDON_FILE=/etc/kubernetes/addons/aci-connector-deployment.yaml
    sed -i "s|<creds>|$ACI_CONNECTOR_CREDENTIALS|g" $ACI_CONNECTOR_ADDON_FILE
    sed -i "s|<rgName>|$RESOURCE_GROUP|g" $ACI_CONNECTOR_ADDON_FILE
    sed -i "s|<cert>|$ACI_CONNECTOR_CERT|g" $ACI_CONNECTOR_ADDON_FILE
    sed -i "s|<key>|$ACI_CONNECTOR_KEY|g" $ACI_CONNECTOR_ADDON_FILE
}

configAzurePolicyAddon() {
    AZURE_POLICY_ADDON_FILE=/etc/kubernetes/addons/azure-policy-deployment.yaml
    sed -i "s|<resourceId>|/subscriptions/$SUBSCRIPTION_ID/resourceGroups/$RESOURCE_GROUP|g" $AZURE_POLICY_ADDON_FILE
}

configGPUDrivers() {
    # install gpu driver
    if [[ $OS == $UBUNTU_OS_NAME ]]; then
        mkdir -p /opt/{actions,gpu}
        if [[ "${CONTAINER_RUNTIME}" == "containerd" ]]; then
            ctr image pull $NVIDIA_DRIVER_IMAGE:$NVIDIA_DRIVER_IMAGE_TAG
            retrycmd_if_failure 5 10 600 bash -c "$CTR_GPU_INSTALL_CMD $NVIDIA_DRIVER_IMAGE:$NVIDIA_DRIVER_IMAGE_TAG gpuinstall /entrypoint.sh install"
            ret=$?
            if [[ "$ret" != "0" ]]; then
                echo "Failed to install GPU driver, exiting..."
                exit $ERR_GPU_DRIVERS_START_FAIL
            fi
            ctr images rm --sync $NVIDIA_DRIVER_IMAGE:$NVIDIA_DRIVER_IMAGE_TAG
        else
            bash -c "$DOCKER_GPU_INSTALL_CMD $NVIDIA_DRIVER_IMAGE:$NVIDIA_DRIVER_IMAGE_TAG install"
            ret=$?
            if [[ "$ret" != "0" ]]; then
                echo "Failed to install GPU driver, exiting..."
                exit $ERR_GPU_DRIVERS_START_FAIL
            fi
            docker rmi $NVIDIA_DRIVER_IMAGE:$NVIDIA_DRIVER_IMAGE_TAG
        fi
    elif [[ $OS == $MARINER_OS_NAME ]]; then
        downloadGPUDrivers
        installNvidiaContainerRuntime
    else
        echo "os $OS not supported at this time. skipping configGPUDrivers"
        exit 1
    fi

    # validate on host, already done inside container.
    if [[ $OS == $UBUNTU_OS_NAME ]]; then
        retrycmd_if_failure 120 5 25 nvidia-modprobe -u -c0 || exit $ERR_GPU_DRIVERS_START_FAIL
    fi
    retrycmd_if_failure 120 5 300 nvidia-smi || exit $ERR_GPU_DRIVERS_START_FAIL
    retrycmd_if_failure 120 5 25 ldconfig || exit $ERR_GPU_DRIVERS_START_FAIL

    # reload containerd/dockerd
    if [[ "${CONTAINER_RUNTIME}" == "containerd" ]]; then
        retrycmd_if_failure 120 5 25 pkill -SIGHUP containerd || exit $ERR_GPU_DRIVERS_INSTALL_TIMEOUT
    else
        retrycmd_if_failure 120 5 25 pkill -SIGHUP dockerd || exit $ERR_GPU_DRIVERS_INSTALL_TIMEOUT
    fi
}

validateGPUDrivers() {
    if [[ $(isARM64) == 1 ]]; then
        # no GPU on ARM64
        return
    fi

    retrycmd_if_failure 24 5 25 nvidia-modprobe -u -c0 && echo "gpu driver loaded" || configGPUDrivers || exit $ERR_GPU_DRIVERS_START_FAIL
    which nvidia-smi
    if [[ $? == 0 ]]; then
        SMI_RESULT=$(retrycmd_if_failure 24 5 300 nvidia-smi)
    else
        SMI_RESULT=$(retrycmd_if_failure 24 5 300 $GPU_DEST/bin/nvidia-smi)
    fi
    SMI_STATUS=$?
    if [[ $SMI_STATUS != 0 ]]; then
        if [[ $SMI_RESULT == *"infoROM is corrupted"* ]]; then
            exit $ERR_GPU_INFO_ROM_CORRUPTED
        else
            exit $ERR_GPU_DRIVERS_START_FAIL
        fi
    else
        echo "gpu driver working fine"
    fi
}

ensureGPUDrivers() {
    if [[ $(isARM64) == 1 ]]; then
        # no GPU on ARM64
        return
    fi

    if [[ "${CONFIG_GPU_DRIVER_IF_NEEDED}" = true ]]; then
        logs_to_events "AKS.CSE.ensureGPUDrivers.configGPUDrivers" configGPUDrivers
    else
        logs_to_events "AKS.CSE.ensureGPUDrivers.validateGPUDrivers" validateGPUDrivers
    fi
    if [[ $OS == $UBUNTU_OS_NAME ]]; then
        logs_to_events "AKS.CSE.ensureGPUDrivers.nvidia-modprobe" "systemctlEnableAndStart nvidia-modprobe" || exit $ERR_GPU_DRIVERS_START_FAIL
    fi
}

disableSSH() {
    systemctlDisableAndStop ssh || exit $ERR_DISABLE_SSH
}

#EOF