Can you convert this bash script to golang?
#!/bin/bash
# -*- tab-width: 4; encoding: utf-8 -*-
#
## @file
## @author Izuma Networks
## @brief setup script for k8s cluster bring up and testing on incus and general VMs
## @copyright apache-2
## @version 0.5.0
# Always match this to CHANGELOG.md
SCRIPTVERSION="1.0.0"
# ----------------------------------------------------------------------------------------------------- #
#                 ____       _                    ___     _____                                         #
#                |  _ \  ___| |__  _   _  __ _   ( _ )   | ____|_ __ _ __ ___  _ __ ___                 #
#                | | | |/ _ \ '_ \| | | |/ _` |  / _ \/\ |  _| | '__| '__/ _ \| '__/ __|                #
#                | |_| |  __/ |_) | |_| | (_| | | (_>  < | |___| |  | | | (_) | |  \__ \                #
#                |____/ \___|_.__/ \__,_|\__, |  \___/\/ |_____|_|  |_|  \___/|_|  |___/                #
#                                        |___/                                                          #
#                                        setup and functionality                                        #
# ----------------------------------------------------------------------------------------------------- #
set -e -u -o pipefail
DEBUG=0

function debugModeEnable() {
    echo "Debug level set to: ${DEBUG}"

    # Enable extdebug for enhanced debugging features
    if [[ "${DEBUG}" -gt 0 ]]; then
        shopt -s extdebug
        if [[ "${DEBUG}" -gt 3 ]]; then
            shopt -x
        fi

        find_function_line() {
            local func_name="$1"
            local script_file="$0"
            grep -n -E "^[[:space:]]*function[[:space:]]+${func_name}[[:space:]]*\(\)|^[[:space:]]*${func_name}[[:space:]]*\(\)" "${script_file}" | awk -F: '{print $1}'
        }

        _ERROR_print_all_lineno() {
            local i
            echo "All line numbers:"
            for ((i = 0; i < ${#BASH_LINENO[@]}; i++)); do
                echo "BASH_LINENO[${i}] = ${BASH_LINENO[${i}]}"
            done
        }

        _ERROR_print_stack_trace() {
            echo "Stack trace(s)"
            local callTree="\nCall tree:"
            local callTreeL="\nCall function tree:"
            echo "functions in stack: ${#FUNCNAME[@]}"
            for ((i = 1; i < ${#FUNCNAME[@]}; i++)); do
                callTree="${callTree}\n ${i}: ${BASH_SOURCE[${i}]}:${BASH_LINENO[$((i - 1))]} ${FUNCNAME[${i}]} -> ${FUNCNAME[$((i - 1))]}(...)"
                abline="$(declare -F "${FUNCNAME[${i}]}" | awk '{print $2}' || echo 'Error')"
                callTreeL="${callTreeL}\n ${i}: ${BASH_SOURCE[${i}]}:${abline:-'Unknown'} ${FUNCNAME[${i}]} -> ${FUNCNAME[$((i - 1))]}(...)"
            done
            echo -e "${callTree}"
            echo -e "${callTreeL}"
        }

        # Function to handle exit due to an error
        function _ERROR_method() {
            rc=$?
            if [[ "${rc}" -ne 0 ]]; then
                echo "-------------"
                echo "Detected Error in stack.  Error entry point: $0:${BASH_LINENO[0]}: with return code ${rc}."
                echo "Also tracking $0:${last_line}"
                _ERROR_print_stack_trace
                if [[ "${DEBUG}" -gt 1 ]]; then
                    _ERROR_print_all_lineno
                fi
            fi
        }
        last_line=0
        trap 'last_line=$LINENO' DEBUG

        trap _ERROR_method ERR
        # Use the function for the EXIT trap
        trap _ERROR_method EXIT

        if [[ "${DEBUG}" -gt 2 ]]; then
            # Prints every line of code as its processed
            debug_trap() {
                last_line="${LINENO}"
                echo -e "Executing line $0:${BASH_LINENO[0]}\t ${BASH_COMMAND}"
            }
            trap debug_trap DEBUG
        fi
    fi
}
# ----------------------------------------------------------------------------------------------------- #
#                                ____ _     ___  ____    _    _     ____                                #
#                               / ___| |   / _ \| __ )  / \  | |   / ___|                               #
#                              | |  _| |  | | | |  _ \ / _ \ | |   \___ \                               #
#                              | |_| | |__| |_| | |_) / ___ \| |___ ___) |                              #
#                               \____|_____\___/|____/_/   \_\_____|____/                               #
#                                                                                                       #
#                                          and system defaults                                          #
# ----------------------------------------------------------------------------------------------------- #

DEPLOYMENTTYPES=("k8svanilla" "k3s")
DEFAULT_CPU=2
DEFAULT_DEPLOYMENTTYPE="k8svanilla"
DEFAULT_MEMORY=2
DEFAULT_INSTANCETYPE="vm"
DEFAULT_NODETOTAL=3
DEFAULT_OS="ubuntu"
DEFAULT_OSVERSION="22.04"
incusInfo="$(incus info)"
partOfCluster="$(echo "${incusInfo}" | grep "server_clustered" | awk '{print $2}')"
if [[ "${partOfCluster}" == "true" ]]; then
    clusterNodesdetailed="$(incus cluster list -f csv)"
    clusterNodes="$(echo "${clusterNodesdetailed}" | awk -F "," '{print $1}')"
    clusterNodesArrayPrep="$(echo "${clusterNodes}" | xargs)"
    IFS=' ' read -r -a VALIDTARGETS <<<"${clusterNodesArrayPrep}"
    DEFAULT_TARGET="$(echo "${clusterNodes}" | head -n 1)"
else
    DEFAULT_TARGET=""
fi
DEFAULT_PROJECT="default"
SUPPORTEDK8s=("1.27" "1.28" "1.29" "1.30")
DEFAULT_K8SVERSION="1.29.4-2.1"
SUPPORTEDK8sSversions=("1.30.1-1.1" "1.30.0-1.1" "1.29.5-1.1" "1.29.4-2.1" "1.29.3-1.1" "1.29.2-1.1" "1.29.1-1.1" "1.29.0-1.1" "1.28.10-1.1" "1.28.9-2.1" "1.28.8-1.1" "1.28.7-1.1" "1.28.6-1.1" "1.28.5-1.1" "1.28.4-1.1" "1.28.3-1.1" "1.28.2-1.1" "1.28.1-1.1" "1.28.0-1.1" "1.27.14-1.1" "1.27.13-2.1" "1.27.12-1.1" "1.27.11-1.1" "1.27.10-1.1" "1.27.9-1.1" "1.27.8-1.1" "1.27.7-1.1" "1.27.6-1.1" "1.27.5-1.1" "1.27.4-1.1" "1.27.3-1.1" "1.27.2-1.1" "1.27.1-1.1" "1.27.0-2.1")
CNI_PROVIDERS=("calico" "cilium")
DEFAULT_CNI_PROVIDER="cilium"
DEFAULT_POD_NETWORK_CIDR="192.168.0.0/16"

TEMPDIR=""

CPU="${DEFAULT_CPU}"
DEPLOYMENTTYPE="${DEFAULT_DEPLOYMENTTYPE}"
NODETOTAL="${DEFAULT_NODETOTAL}"
INSTANCETYPE="${DEFAULT_INSTANCETYPE}"
MEMORY="${DEFAULT_MEMORY}"
OS="${DEFAULT_OS}"
OSVERSION="${DEFAULT_OSVERSION}"
TARGET="${DEFAULT_TARGET}"
PROJECT="${DEFAULT_PROJECT}"
K8SVERSION="${DEFAULT_K8SVERSION}"
CNI_PROVIDER="${DEFAULT_CNI_PROVIDER}"
POD_NETWORK_CIDR="${DEFAULT_POD_NETWORK_CIDR}"
LASTIP=""
MASTERIP=""
DEFAULT_OUTPUT_DIR="${HOME}/wrangler/"

function _resetDefaults() {
    LASTIP=""
    MASTERIP=""
    CPU="${DEFAULT_CPU}"
    DEPLOYMENTTYPE="${DEFAULT_DEPLOYMENTTYPE}"
    NODETOTAL="${DEFAULT_NODETOTAL}"
    INSTANCETYPE="${DEFAULT_INSTANCETYPE}"
    MEMORY="${DEFAULT_MEMORY}"
    OS="${DEFAULT_OS}"
    OSVERSION="${DEFAULT_OSVERSION}"
    TARGET="${DEFAULT_TARGET}"
    PROJECT="${DEFAULT_PROJECT}"
    K8SVERSION="${DEFAULT_K8SVERSION}"
    CNI_PROVIDER="${DEFAULT_CNI_PROVIDER}"
    POD_NETWORK_CIDR="${DEFAULT_POD_NETWORK_CIDR}"
}

# ----------------------------------------------------------------------------------------------------- #
#                                                                                                       #
#                                            Test Case Vars                                             #
# ----------------------------------------------------------------------------------------------------- #
TESTTORUN=""
FORCENODNS=0
EXITONERROR=0
PRETEND=0
declare -A TESTMACROS=(
    [all]="test_*"
    [k8s]="test_k8s_*"
    [util]="test_util_*"
    [incus]="test_incus_*"
)

# ----------------------------------------------------------------------------------------------------- #
#                                                                                                       #
#                                                colors                                                 #
# ----------------------------------------------------------------------------------------------------- #
RED='\033[3;31m'
GREEN='\033[3;32m'
BRIGHTRED='\033[1;31m'
YELLOW='\033[0;33m'
BRIGHTYELLOW='\033[1;93m'
DIMYELLOW='\033[2;33m'
UNDERLINE='\033[4m'
NORM='\033[0m'
BOLD='\033[1m'
# ----------------------------------------------------------------------------------------------------- #
#                                                                                                       #
#                                              error codes                                              #
# ----------------------------------------------------------------------------------------------------- #
declare -a ERRORCODES=("EC_SUCCESS" "EC_CLI_ERROR" "EC_INCUS_FAILED_TO_IP" "EC_INCUS_FAILED_TO_DELETE_INSTANCE" "EC_INCUS_NAME_EXISTS" "EC_INTERNAL_INCORRECT_PARAMS" "EC_INTERNAL_INVALID_PARAMS" "EC_K8S_NOT_READY_TIME" "EC_TEST_FAILED_IN_LIST" "EC_TEST_FORCED_EXIT")
EC_SUCCESS=0
EC_CLI_ERROR=2
EC_CLI_BAD_OPTION_PARSE=3
EC_CLI_PRECHECK_FAILED=5
EC_INCUS_FAILED_TO_IP=11
EC_INCUS_FAILED_TO_DELETE_INSTANCE=12
EC_INCUS_NAME_EXISTS=13
EC_INTERNAL_INCORRECT_PARAMS=20
EC_INTERNAL_INVALID_PARAMS=21
EC_K8S_NOT_READY_TIME=30
EC_TEST_FAILED_IN_LIST=40
EC_TEST_FORCED_EXIT=41
EC_TEST_FAILED_CLEANUP=42

declare -a ERRORCODE_MSG=()
ERRORCODE_MSG[EC_SUCCESS]="Success"
ERRORCODE_MSG[EC_CLI_ERROR]="CLI usage error"
ERRORCODE_MSG[EC_CLI_BAD_OPTION_PARSE]="CLI option parsing error"
ERRORCODE_MSG[EC_CLI_PRECHECK_FAILED]="CLI prerequisite checks failed"
ERRORCODE_MSG[EC_INCUS_FAILED_TO_DELETE_INSTANCE]="Failed to delete instance"
ERRORCODE_MSG[EC_INCUS_NAME_EXISTS]="Name already exists"
ERRORCODE_MSG[EC_INCUS_FAILED_TO_IP]="Failed to assign/get IP to incus instance"
ERRORCODE_MSG[EC_INTERNAL_INCORRECT_PARAMS]="Internal function called with incorrect parameters"
ERRORCODE_MSG[EC_INTERNAL_INVALID_PARAMS]="Internal function called with invalid parameters"
ERRORCODE_MSG[EC_K8S_NOT_READY_TIME]="Kubernetes cluster not ready in allocated time"
ERRORCODE_MSG[EC_TEST_FAILED_IN_LIST]="At least one test failed in the test run"
ERRORCODE_MSG[EC_TEST_FORCED_EXIT]="Forced exit trap (-e) in test run"
ERRORCODE_MSG[EC_TEST_FAILED_CLEANUP]="Failed to cleanup after test run"
# ----------------------------------------------------------------------------------------------------- #
#                                      _   _ _____ ___ _     ____                                       #
#                                     | | | |_   _|_ _| |   / ___|                                      #
#                                     | | | | | |  | || |   \___ \                                      #
#                                     | |_| | | |  | || |___ ___) |                                     #
#                                      \___/  |_| |___|_____|____/                                      #
#                                                                                                       #
#                           Utility functions and Internals prefixed with a _                           #
# ----------------------------------------------------------------------------------------------------- #
# ----------------------------------------------------------------------------------------------------- #
#                                                                                                       #
#                                             error control                                             #
# ----------------------------------------------------------------------------------------------------- #
function _handle_error() {
    local errorcode="${1}"
    shift
    local errorMessage
    echo "Error code: ${errorcode}"
    errorMessage="${ERRORCODE_MSG[${errorcode}]}"
    if [[ "${errorcode}" -ne 0 ]]; then
        echo "Error ${errorcode}: ${errorMessage}"
        if [[ -n "$*" ]]; then
            echo "Additional information: $*"
        fi
    else
        echo "Success"
    fi
    echo "Exiting"
    exit "${errorcode}"
}

# ----------------------------------------------------------------------------------------------------- #
#                                                                                                       #
#                                           display cli utils                                           #
# ----------------------------------------------------------------------------------------------------- #

function _banner() {
    echo -e "${BRIGHTYELLOW}${UNDERLINE}${1}${NORM}"
}

function _subbanner() {
    echo -e "${YELLOW}${1}${NORM}"
}

function _subbanner2() {
    echo -e "${DIMYELLOW}${1}${NORM}"
}

# use a tty for logging instead of a echo
function _log() {
    echo -e "${1}" >/dev/tty
}

function _helplist() {
    local inRay=("$@")
    for i in "${!inRay[@]}"; do
        printf "\"%s\"" "${inRay[${i}]}"
        if ((i < ${#inRay[@]} - 1)); then
            printf " | "
        fi
    done
}

# ----------------------------------------------------------------------------------------------------- #
#                                                                                                       #
#                                                 exec                                                  #
# ----------------------------------------------------------------------------------------------------- #

function _remoteExec() {
    local instance="${1}"
    local cmd="${2}"
    #we need the cmd to expand, which is against shellcheck rules
    #shellcheck disable=SC2086
    results=$(incus exec "${instance}" -- ${cmd})
    echo "${results}"
}

function _localExec() {
    local cmd="${1}"
    #we need the cmd to expand, which is against shellcheck rules
    #shellcheck disable=SC2086
    if [[ "${PRETEND}" -eq 1 ]]; then
        echo "Pretend: ${cmd}"
        return
    fi
    eval "${cmd}"
}

function _wrapfunction() {
    local func="${1}"
    local instance="${2}"
    shift 2
    if [[ "${PRETEND}" -eq 1 ]]; then
        echo "Pretend: _wrapfunction ${func} ${instance} $*"
    fi
    # Create a temporary script that includes the function definition and invocation
    declare -f "${func}" >"${TEMPDIR}"/sztemp.sh
    # doesn't like the $@ in the echo but that is the desired effect
    # shellcheck disable=SC2145
    echo "${func} $@" >>"${TEMPDIR}"/sztemp.sh
    _subbanner "Running ${func} on ${instance}"
    _localExec "incus shell \"${instance}\" <\"${TEMPDIR}\"/sztemp.sh"
    # Remove the temporary script
    mv "${TEMPDIR}"/sztemp.sh "${TEMPDIR}"/sz_"${func}".log
}

# ----------------------------------------------------------------------------------------------------- #
#                                                                                                       #
#                                         k8s and cluster utils                                         #
# ----------------------------------------------------------------------------------------------------- #

function _buildNodeList() {
    local prefix="${1}"
    local nodeArray=()
    nodeArray+=("${prefix}-kmaster")
    for ((i = 1; i < "${NODETOTAL}"; i++)); do
        nodeArray+=("${prefix}-kworker${i}")
    done
    echo "${nodeArray[@]}"
}
# ----------------------------------------------------------------------------------------------------- #
#                                                                                                       #
#                                              incus utils                                              #
# ----------------------------------------------------------------------------------------------------- #

function _nameExists() {
    local incusName="${1}"
    local clusterNodesdetailed
    clusterNodesdetailed="$(incus list -f csv -c n)"
    if echo "${clusterNodesdetailed}" | grep -q "${incusName}"; then
        return 0
    else
        return 1
    fi
}

function _exitIfNameInUse() {
    local incusName="${1}"
    #shellcheck disable=SC2310 #this is a test function
    if _nameExists "${incusName}"; then
        _handle_error "${EC_INCUS_NAME_EXISTS}" "${incusName}"
    fi
}

function _countReady() {
    local text="$1"
    local count=0
    while IFS= read -r line; do
        if [[ ${line} =~ (^|[[:space:]])Ready($|[[:space:]]) ]]; then
            ((count++))
        fi
    done <<<"${text}"
    echo "${count}"
}

# ----------------------------------------------------------------------------------------------------- #
#                                                                                                       #
#                                             general utils                                             #
# ----------------------------------------------------------------------------------------------------- #

function _makeTmpDir() {
    TEMPDIR=$(mktemp -d)
}

function _destroyTmpDir() {
    rm -rf "${TEMPDIR}"
}

function _makeArrayFromOption() {
    local var_name="${1}"
    local var_value="${2}"
    local -n array="${3}"

    if [[ "${var_value}" == *","* ]]; then
        IFS=',' read -ra array <<<"${var_value}"
        if [[ "${#array[@]}" -ne "${NODETOTAL}" ]]; then
            displayHelp "Error: Not enough ${var_name} resources were specified for the total number of instances requested (${NODETOTAL})."
        fi
    else
        for ((i = 0; i < "${NODETOTAL}"; i++)); do
            array[i]="${var_value}"
        done
    fi
}

_isWordsInList() {
    local word="${1}"
    local wordRay
    # shellcheck disable=SC2206 #we are building an array
    wordRay=(${word//,/ })
    shift
    local -n list="${1}"
    local errorMessage="${2}"
    for word in "${wordRay[@]}"; do
        found=0
        for element in "${list[@]}"; do
            if [[ "${element}" == "${word}" ]]; then
                found=1
                break
            fi
        done
        if [[ "${found}" -eq 0 ]]; then
            return 1
        fi
    done
}
_validateWordInList() {
    local word="${1}"
    shift
    local -n list="${1}"
    local errorMessage="${2}"
    found=0
    for element in "${list[@]}"; do
        if [[ "${element}" == "${word}" ]]; then
            found=1
            break
        fi
    done

    if [[ "${found}" -eq 0 ]]; then
        displayHelp "${errorMessage}"
    fi
}

# ----------------------------------------------------------------------------------------------------- #
#                                 __  __            _     _                                             #
#                                |  \/  | __ _  ___| |__ (_)_ __   ___                                  #
#                                | |\/| |/ _` |/ __| '_ \| | '_ \ / _ \                                 #
#                                | |  | | (_| | (__| | | | | | | |  __/                                 #
#                                |_|  |_|\__,_|\___|_| |_|_|_| |_|\___|                                 #
#                                                                                                       #
#                                        Getting the OS up stuff                                        #
# ----------------------------------------------------------------------------------------------------- #
function awaitInstanceIP() {
    local instanceName="${1}"
    local count=0
    local description
    local iphas
    local iphead
    local ip
    local ipslash
    local checkTime=5
    local expireTime=90
    local maxCount=$((expireTime / checkTime))
    _subbanner "Waiting for instance ${instanceName} to get first IP ready"
    count=0
    if [[ "${PRETEND}" -eq 1 ]]; then
        echo "Pretend: Waited up to ${expireTime} seconds for instance ${instanceName} to get IP 1.1.1.1"
        LASTIP="1.1.1.1"
        return
    fi
    while true; do
        ip=""
        description=$(incus info "${instanceName}")
        if echo "${description}" | grep -E "\binet\b" | grep -q 'global'; then
            iphas="$(echo "${description}" | grep 'global')"
            iphead=$(echo "${iphas}" | head -n 1)
            ipslash=$(echo "${iphead}" | awk '{print $2}')
            ip=$(echo "${ipslash}" | cut -d'/' -f1)
        fi
        if [[ -n "${ip}" ]]; then
            echo "Instance ${instanceName} is ready with IP: ${ip}"
            break
        else
            elapsed=$((count * checkTime))
            echo "Waiting for instance to get an IP... ${elapsed} seconds elapsed."
            sleep "${checkTime}"
            count=$((count + 1))
            if [[ ${count} -eq ${maxCount} ]]; then
                _handle_error "${EC_INCUS_FAILED_TO_IP}" "Instance ${instanceName}, time: ${expireTime}"
            fi
        fi
    done
    LASTIP="${ip}"
}

function snapshotInstance() {
    local instanceName="${1}"
    local snapshotName="${2}"
    if [[ $# -ne 2 ]]; then
        _handle_error "${EC_INTERNAL_INCORRECT_PARAMS}" "snapshotInstance <instanceName> <snapshotName>"
    fi
    _subbanner "Creating snapshot ${snapshotName} for instance ${instanceName}"
    cmd="incus snapshot create ${instanceName} ${snapshotName}"
    echo "${cmd}"
    _localExec "${cmd}"
}

function launchInstance() {
    local instanceName="${1}"
    local instanceType="${2}"
    local os="${3}"
    local osVersion="${4}"
    local cpu="${5}"
    local mem="${6}"
    local target="${7}"
    local project="${8}"
    if [[ $# -ne 8 ]]; then
        _handle_error "${EC_INTERNAL_INCORRECT_PARAMS}" "launchInstance <instanceName> <instanceType> <os> <osVersion> <cpu> <memory> <target> <project>"
    fi
    if [[ ${instanceType} == "vm" ]]; then
        local typereplacement="--vm"
    elif [[ ${instanceType} == "container" ]]; then
        local typereplacement="--profile default --profile capability-k8s "
    else
        echo "Invalid instance type. Please use either 'vm' or 'container'"
        _handle_error "${EC_INTERNAL_INVALID_PARAMS}" "launchInstance <instanceType> Please use either 'vm' or 'container'"
    fi
    if [[ ${target} != "" ]]; then
        local targetreplacement="--target ${target}"
    else
        local targetreplacement=""
    fi
    mem=${mem%GB}
    echo "Ensuring project is set to ${project}"
    cmd="incus project switch ${project}"
    echo "${cmd}"
    _localExec "${cmd}"
    _exitIfNameInUse "${instanceName}"
    echo "Creating instance ${instanceName}"
    cmd="incus launch images:${os}/${osVersion} ${instanceName} --type c${cpu}-m${mem} ${typereplacement} ${targetreplacement}"
    echo "${cmd}"
    _localExec "${cmd}"
    awaitInstanceIP "${instanceName}"
    echo "Rebooting ${instanceName} to ensure IP connectivity"

    _localExec "incus stop -f ${instanceName}"
    _localExec "incus start ${instanceName}"
    awaitInstanceIP "${instanceName}"
}
# ----------------------------------------------------------------------------------------------------- #
#                              _  _____        ____       _                                             #
#                             | |/ ( _ ) ___  / ___|  ___| |_ _   _ _ __                                #
#                             | ' // _ \/ __| \___ \ / _ \ __| | | | '_ \                               #
#                             | . \ (_) \__ \  ___) |  __/ |_| |_| | |_) |                              #
#                             |_|\_\___/|___/ |____/ \___|\__|\__,_| .__/                               #
#                                                                  |_|                                  #
#                                 Deploying and configuring Kubernetes                                  #
# ----------------------------------------------------------------------------------------------------- #

function setupKubelet() {
    local localversion="${1}"
    echo "Installing kubelet"
    echo "installing version: ${localversion}"
    if [[ "${localversion}" == "" ]]; then
        echo "gone classic (${localversion})" >/tmp/sk.log
        apt-get install -y kubelet kubeadm kubectl
    else
        echo "gone new (${localversion})" >/tmp/sk.log
        apt-get install -y kubelet="${localversion}" kubeadm="${localversion}" kubectl="${localversion}"
    fi
    echo 'KUBELET_EXTRA_ARGS="--fail-swap-on=false"' >/etc/default/kubelet
    systemctl restart kubelet
    systemctl restart kubelet
}
function setupContainerd() {
    local arch
    local lsbdata
    echo "Installing containerd"
    apt-get install -y apt-transport-https ca-certificates
    if [[ ! -f /etc/apt/keyrings/docker.gpg ]]; then
        sudo curl -fsSL -o docker.gpg https://download.docker.com/linux/ubuntu/gpg
        gpg --dearmor --no-tty --batch -o /etc/apt/keyrings/docker.gpg docker.gpg
    fi

    if [[ ! -f /etc/apt/sources.list.d/docker.list ]]; then
        arch=$(dpkg --print-architecture)
        lsbdata="$(lsb_release -cs)"
        echo "deb [arch=${arch} signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu ${lsbdata} stable" | sudo tee /etc/apt/sources.list.d/docker.list
    fi
    apt-get update
    apt-get install -y containerd.io
    containerd config default >/etc/containerd/config.toml
    sed -i 's/SystemdCgroup \= false/SystemdCgroup \= true/g' /etc/containerd/config.toml
    systemctl enable containerd
    systemctl restart containerd
}

function setupGeneral() {
    echo "updating the system"
    {
        echo "label  ::1/128       0"
        echo "label  ::/0          1"
        echo "label  2002::/16     2"
        echo "label ::/96          3"
        echo "label ::ffff:0:0/96  4"
        echo "precedence  ::1/128       50"
        echo "precedence  ::/0          40"
        echo "precedence  2002::/16     30"
        echo "precedence ::/96          20"
        echo "precedence ::ffff:0:0/96  100"
    } >/etc/gai.conf
    apt-get update
    echo "Installing required packages"
    apt-get install -y curl dnsutils gnupg lsb-release net-tools software-properties-common ssh
    echo "Set up kubernetes repo"

    for versionk8s in "$@"; do
        if [[ ! -f /etc/apt/keyrings/kubernetes-apt-keyring-${versionk8s}.gpg ]]; then
            curl -fsSL -o Release.key https://pkgs.k8s.io/core:/stable:/v"${versionk8s}"/deb/Release.key
            sudo gpg --dearmor --no-tty --batch -o /etc/apt/keyrings/kubernetes-apt-keyring-"${versionk8s}".gpg Release.key
        fi
        if [[ ! -f /etc/apt/sources.list.d/kubernetes-"${versionk8s}".list ]]; then
            echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring-${versionk8s}.gpg] https://pkgs.k8s.io/core:/stable:/v${versionk8s}/deb/ /" | sudo tee /etc/apt/sources.list.d/kubernetes-"${versionk8s}".list
        fi
    done
    apt-get update
    echo "Enable ssh password authentication"
    sed -i 's/^#PasswordAuthentication .*/PasswordAuthentication yes/' /etc/ssh/sshd_config
    echo 'PermitRootLogin yes' >>/etc/ssh/sshd_config
    systemctl reload sshd
    echo "set the root password"
    echo -e "kubeadmin\nkubeadmin" | passwd root
}

function setupMasterOnly() {
    echo "Setting up master"
    local local_cni_provider="${1}"
    local local_pod_network_cidr="${2}"
    kubeadm config images pull
    if [[ ! -f /root/kubeinit.log ]]; then
        kubeadm init --pod-network-cidr="${local_pod_network_cidr}" --ignore-preflight-errors=all >>/root/kubeinit.log
    fi
    if [[ ! -f /root/.kube/config ]]; then
        mkdir /root/.kube
        cp /etc/kubernetes/admin.conf /root/.kube/config
    fi

    if [[ ${local_cni_provider} = "calico" ]]; then
        echo "deploy Calico"
        kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/v3.27.0/manifests/tigera-operator.yaml
        # The standard calico setup expects the pod-network-cidr to be 192.168.0.0/16, so change it to be our value.
        wget https://raw.githubusercontent.com/projectcalico/calico/v3.27.0/manifests/custom-resources.yaml
        sed -i "s|192.168.0.0/16|${local_pod_network_cidr}|g" custom-resources.yaml
        kubectl create -f custom-resources.yaml
    fi
    if [[ ${local_cni_provider} = "cilium" ]]; then
        echo "deploy Cilium"
        curl -L --remote-name https://github.com/cilium/cilium-cli/releases/latest/download/cilium-linux-amd64.tar.gz
        tar xzvf cilium-linux-amd64.tar.gz
        sudo mv cilium /usr/local/bin/
        rm cilium-linux-amd64.tar.gz
        cilium install
    fi

    echo "Generate and save k8s cluster join command to /joincluster.sh"
    if [[ ! -f /joincluster.sh ]]; then
        joinCommand=$(kubeadm token create --print-join-command 2>/dev/null)
        ipAddressWithPort="$(echo "${joinCommand}" | awk '{ print $3 }')"
        ipAddress="${ipAddressWithPort%:*}"
        hostToUse="${HOSTNAME}"
        hostreturn="$(host "${hostToUse}")"
        hostToUseIP="$(echo "${hostreturn}" | awk '/has address/ {print $4}')"
        firstOctet="$(echo "${hostToUseIP}" | cut -d. -f1)"
        if [[ -z "${hostToUse}" || "${firstOctet}" == "127" || "${FORCENODNS}" -eq 1 ]]; then
            echo "Using IP address ${ipAddress} instead of hostname ${hostToUse}. Our first octet is ${firstOctet} and FORCENODNS is ${FORCENODNS}"
            hostToUse="${ipAddress}"
        fi
        # Replace the IP address in the join command with the hostname
        joinCommandWithHostname="${joinCommand//${ipAddress}/${hostToUse}}"
        echo "${joinCommandWithHostname} --ignore-preflight-errors=all" >/joincluster.sh
    fi
}

function setupWorker() {
    local master="${1}"
    local masterip="${2}"
    echo "Setting up worker"
    echo "Joining the master: ${master} at ${masterip}"
    apt-get install -qq -y sshpass
    sshpass -p "kubeadmin" scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no "${masterip}":/joincluster.sh /joincluster.sh 2>/var/log/joincluster.log
    bash /joincluster.sh >>/var/log/joincluster.log
}

function setupValidateMaster() {
    local masterName="${1}"
    local nodeCount="${2}"
    local max_attempts=90 # 15 minutes with 10-second intervals
    local remaining_checks=${max_attempts}
    local attempt=1
    local res
    if [[ "${PRETEND}" -eq 1 ]]; then
        echo "Pretend: Checked k8s cluster readiness over ${max_attempts} * 10 seconds period.  Success!"
        return 0
    fi
    while [[ "${attempt}" -le "${max_attempts}" ]]; do
        echo "Checking k8s cluster readiness... Attempts remaining: ${remaining_checks}/${max_attempts}"
        ((remaining_checks--))
        res=$(_remoteExec "${masterName}" "kubectl get nodes")
        #shellcheck disable=2310 #- this is a test function
        if _testRunUtil_expect_grep_count "${res}" "${nodeCount}"; then
            echo "k8s cluster is valid and ready!"
            echo "${res}"
            return 0
        fi

        if [[ "${attempt}" -eq "${max_attempts}" ]]; then
            echo "k8s cluster is not ready after 3 minutes. Exiting."
            return 1
        fi
        attempt=$((attempt + 1))
        sleep 10
    done
}

# ----------------------------------------------------------------------------------------------------- #
#                                   _  _____         ___  ____  ____                                    #
#                                  | |/ ( _ ) ___   / _ \|  _ \/ ___|                                   #
#                                  | ' // _ \/ __| | | | | |_) \___ \                                   #
#                                  | . \ (_) \__ \ | |_| |  __/ ___) |                                  #
#                                  |_|\_\___/|___/  \___/|_|   |____/                                   #
#                                                                                                       #
#                                  Externally operations on Kubernetes                                  #
# ----------------------------------------------------------------------------------------------------- #
function KubeConfigThis() {
    local master="${1}"
    echo "Copying kubeconfig to ${HOME}/.kube/${master}"
    mkdir -p "${HOME}"/.kube/
    _localExec "incus file pull \"${master}\"/etc/kubernetes/admin.conf ~/.kube/\"${master}\""
}

# ----------------------------------------------------------------------------------------------------- #
#                                       __  __    _    ___ _   _                                        #
#                                      |  \/  |  / \  |_ _| \ | |                                       #
#                                      | |\/| | / _ \  | ||  \| |                                       #
#                                      | |  | |/ ___ \ | || |\  |                                       #
#                                      |_|  |_/_/   \_\___|_| \_|                                       #
#                                                                                                       #
#                                Entry points for different deployments                                 #
# ----------------------------------------------------------------------------------------------------- #
function launchClusterInstances() {
    local prefix="${1}"
    local master="${prefix}-kmaster"
    _makeArrayFromOption "target hosts" "${TARGET}" TARGETS
    _makeArrayFromOption "CPU" "${CPU}" CPUS
    _makeArrayFromOption "memory" "${MEMORY}" MEMS
    #parse the TARGET variable to determine the target hosts we will deploy to
    #shellcheck disable=SC2154,SC2153 #due to the way we used a function to build the array
    launchInstance "${master}" "${INSTANCETYPE}" "${OS}" "${OSVERSION}" "${CPUS[0]}" "${MEMS[0]}" "${TARGETS[0]}" "${PROJECT}"
    MASTERIP="${LASTIP}"
    for ((i = 1; i < "${NODETOTAL}"; i++)); do
        local thisNode="${prefix}-kworker${i}"
        launchInstance "${thisNode}" "${INSTANCETYPE}" "${OS}" "${OSVERSION}" "${CPUS[${i}]}" "${MEMS[${i}]}" "${TARGETS[${i}]}" "${PROJECT}"
    done
}

function snapshotClusterInstances() {
    local prefix="${1}"
    local nodeRay
    #shellcheck disable=SC2312 #prefer to error if _buildNodeList fails
    IFS=' ' read -ra nodeRay <<<"$(_buildNodeList "${prefix}")"
    for node in "${nodeRay[@]}"; do
        snapshotInstance "${node}" "${node}_snap0"
    done
}

function buildHelperScripts() {
    local prefix="${1}"
    local nodeRay
    #shellcheck disable=SC2312 #prefer to error if _buildNodeList fails
    IFS=' ' read -ra nodeRay <<<"$(_buildNodeList "${prefix}")"
    if [[ ! -d "${DEFAULT_OUTPUT_DIR}" ]]; then
        mkdir -p "${DEFAULT_OUTPUT_DIR}"
    fi
    echo "#!/bin/bash" >"${DEFAULT_OUTPUT_DIR}${prefix}"-recover.sh
    echo "#!/bin/bash" >"${DEFAULT_OUTPUT_DIR}${prefix}"-restart.sh
    echo "#!/bin/bash" >"${DEFAULT_OUTPUT_DIR}${prefix}"-destroy.sh
    chmod +x "${DEFAULT_OUTPUT_DIR}${prefix}"-recover.sh
    chmod +x "${DEFAULT_OUTPUT_DIR}${prefix}"-restart.sh
    chmod +x "${DEFAULT_OUTPUT_DIR}${prefix}"-destroy.sh
    for node in "${nodeRay[@]}"; do
        echo "incus snapshot restore ${node} ${node}_snap0" >>"${DEFAULT_OUTPUT_DIR}${prefix}"-recover.sh
        echo "incus restart ${node}" >>"${DEFAULT_OUTPUT_DIR}${prefix}"-restart.sh
        echo "incus delete -f ${node}" >>"${DEFAULT_OUTPUT_DIR}${prefix}"-destroy.sh
    done
}

function main_k8svanilla() {
    local prefix="${1}"
    local nodeRay
    #shellcheck disable=SC2312 #prefer to error if _buildNodeList fails
    IFS=' ' read -ra nodeRay <<<"$(_buildNodeList "${prefix}")"
    local master="${nodeRay[0]}"
    export DEBIAN_FRONTEND=noninteractive
    for node in "${nodeRay[@]}"; do
        _banner "Setting up ${node}"
        _wrapfunction setupGeneral "${node}" "${SUPPORTEDK8s[@]}"
        _wrapfunction setupContainerd "${node}"
        _wrapfunction setupKubelet "${node}" "${K8SVERSION}"
        if [[ "${node}" == "${master}" ]]; then
            _banner "Doing master only things"
            _wrapfunction setupMasterOnly "${node}" "${CNI_PROVIDER}" "${POD_NETWORK_CIDR}"
        else
            _subbanner "Doing worker only things"
            _wrapfunction setupWorker "${node}" "${master}" "${MASTERIP}"
        fi
    done
    _banner "Validating deployment ${node}"
    #shellcheck disable=SC2310 #this is a test function
    if setupValidateMaster "${master}" "${NODETOTAL}"; then
        echo "k8s cluster is ready"
        KubeConfigThis "${master}"
    else
        echo "k8s cluster is not ready in allocated time."
        if [[ "${TESTTORUN}" == "" ]]; then
            _handle_error "${EC_K8S_NOT_READY_TIME}"
        else
            return "${EC_K8S_NOT_READY_TIME}"
        fi
    fi
}

function program_main() {
    local prefix="${1}"
    _banner "Creating ${NODETOTAL} instances for deployment type ${DEPLOYMENTTYPE}"
    launchClusterInstances "${prefix}"
    _banner "Setting up the k8s cluster onto the nodes"
    if [[ "${TESTTORUN}" != "" ]]; then
        if main_"${DEPLOYMENTTYPE}" "${prefix}"; then
            :
        else
            rc=$?
            return "${rc}"
        fi
    else
        main_"${DEPLOYMENTTYPE}" "${prefix}"
    fi
    _banner "Taking snapshots of the instances"
    snapshotClusterInstances "${prefix}"
    _banner "Building k8s cluster helper scripts"
    buildHelperScripts "${prefix}"
    echo -e "----------------------------------------------------------------------------------"
    echo -e "  ${GREEN}SUCCESS${NORM}"
    echo -e ""
    echo -e "  The Kubernetes cluster ${prefix} deployed successfully."
    echo -e "  Helper scripts:"
    echo -e "    - restore: ${DEFAULT_OUTPUT_DIR}${prefix}-recover.sh"
    echo -e "    - restart: ${DEFAULT_OUTPUT_DIR}${prefix}-restart.sh"
    echo -e "    - destroy: ${DEFAULT_OUTPUT_DIR}${prefix}-destroy.sh"
    echo -e ""
    echo -e "----------------------------------------------------------------------------------"
}

# ----------------------------------------------------------------------------------------------------- #
#                                     _____ _____ ____ _____ ____                                       #
#                                    |_   _| ____/ ___|_   _/ ___|                                      #
#                                      | | |  _| \___ \ | | \___ \                                      #
#                                      | | | |___ ___) || |  ___) |                                     #
#                                      |_| |_____|____/ |_| |____/                                      #
#                                                                                                       #
#                                           a suite of tests                                            #
# ----------------------------------------------------------------------------------------------------- #
# ----------------------------------------------------------------------------------------------------- #
#                                                                                                       #
#                           testRunner functions to aid in running the tests                            #
# ----------------------------------------------------------------------------------------------------- #

## @fn _testRunner_bracketOut()
## @brief Prints a message with colored brackets.
## @param color Colors code for the brackets.
## @param bracketMessage Message to display inside the brackets.
## @param remainder The rest of the message to display after the brackets via $@
function _testRunner_bracketOut() {
    local color=${1}
    local bracketMessage="${2}"
    shift 2
    local remainder=${*}
    echo -e "[${color}${bracketMessage}${NORM}]\t${remainder}"
}

## @fn _testRunner_success()
## @brief Prints a success message with green brackets.
## @param allArgs Remaining args ($@)  to display as a success.
function _testRunner_success() {
    _testRunner_bracketOut "${GREEN}" "  OK  " "$@"
}

## @fn _testRunner_failure()
## @brief Prints a failure message with red brackets.
## @param allArgs Remaining args ($@)  to display as a failure.
function _testRunner_failure() {
    _testRunner_bracketOut "${RED}" " FAIL " "$@"
}

## @fn _testRunner_warn()
## @brief Prints a warning message with yellow brackets.
## @param allArgs Remaining args ($@)  to display as a warning.
function _testRunner_warn() {
    _testRunner_bracketOut "${YELLOW}" " WARN " "$@"
}

# Initializes an empty test report.cod
TESTREPORT=""

## @fn _testRunner_addTestReport()
## @brief Adds ax string entry to the test report.
## @param nl Text to add to the test report.
function _testRunner_addTestReport() {
    local nl="${1}"
    TESTREPORT="${TESTREPORT}${nl}\n"
}

## @fn _testRunner_displayTestReport()
## @brief Displays the accumulated test report.
function _testRunner_displayTestReport() {
    echo -e "${TESTREPORT}"
}

## @fn _testRunner_raiseError()
## @brief Raises an error with an optional exit.
## @param message Message to display.
## @details If EXITONERROR is set to 1, the script exits with status 1.
function _testRunner_raiseError() {
    local message="${1}"
    echo -e "${NORM}${message}${NORM}"
    if [[ "${EXITONERROR}" -eq 1 ]]; then
        echo -e "${BRIGHTRED}Exiting due to error${NORM}"
        # Display the last function call and line number
        echo "Last function call: ${FUNCNAME[1]} on line ${BASH_LINENO[0]} in file ${BASH_SOURCE[1]}"
        _handle_error "${EC_TEST_FORCED_EXIT}" "${message}"
    fi
}

## Counter for tracking the number of tests.
testcounter=0

## @fn _testRunner_Results()
## @brief Logs the result of a test, incrementing the test counter.
## @param testname Name of the test.
## @param result Result of the test, either "Passed" or any other string for failure.
## @details Increments the global test counter and logs the test result using either the success or failure logging function.
## Adds the result to the test report.
function _testRunner_Results() {
    local testname="${1}"
    local result="${2}"
    local capturedBrackets
    testcounter=$((testcounter + 1))
    if [[ "${result}" == "Passed" ]]; then
        capturedBrackets=$(_testRunner_success "Test ${testcounter}: ${testname}")
    else
        capturedBrackets=$(_testRunner_failure "Test ${testcounter}: ${testname} ${result}")
    fi
    _testRunner_addTestReport "${capturedBrackets}"
}
# ----------------------------------------------------------------------------------------------------- #
#                                                                                                       #
#                                 low level conditional tests and utils                                 #
# ----------------------------------------------------------------------------------------------------- #
## @fn _testRunUtil_cleanup()
## @brief Cleans up resources by deleting nodes with a specified prefix.
## @param prefix The prefix used to identify the nodes to be deleted.
## @details This function lists all nodes using 'incus list', filters them by the specified prefix, and then deletes each node that matches the prefix.
## If any nodes with the specified prefix exist after the deletion attempt, it prints an error message and returns 8; otherwise, it returns 0.
function _testRunUtil_cleanup() {
    prefix="${1}"
    _resetDefaults
    if [[ "${prefix}" == "" ]]; then
        displayHelp "No prefix specified" "<prefix>"
    fi
    local listing
    if [[ "${PRETEND}" -eq 1 ]]; then
        echo "Pretend: Would have deleted all nodes with prefix ${prefix}"
        return 0
    fi
    listing=$(incus list -f csv -c n)
    if echo "${listing}" | grep -q "${prefix}"; then
        filteredListing=$(echo "${listing}" | grep "${prefix}")
        listing=$(echo "${filteredListing}" | xargs)
        for node in ${listing}; do
            incus delete -f "${node}"
        done
        listing=$(incus list)
        if echo "${listing} " | grep -q "${prefix}"; then
            _handle_error "${EC_TEST_FAILED_CLEANUP}" "Failed to delete all nodes with prefix ${prefix}"
        fi
    fi
}

## @fn _testRunUtil_expect_grep_count()
## @brief Checks if the expected count of a substring appears in a given text.
## @param text The text to search within.
## @param expected The substring to count in the text.
## @details Uses grep to count the occurrences of `expected` in `text` and compares it to a hardcoded count of 3.
## If the actual count does not match the expected, an error is raised using `_testRunner_raiseError`.
## @return Returns 0 if the count matches, 1 otherwise.
function _testRunUtil_expect_grep_count() {
    local text="${1}"
    local expected="${2}"
    local readyCount
    readyCount=$(_countReady "${text}")
    if [[ "${readyCount}" -ne "${expected}" && "${PRETEND}" -ne 1 ]]; then
        echo "text: ${text}"
        echo "readyCount: ${readyCount}"
        _testRunner_raiseError "Expected ${expected} instances of 'Ready' but found ${readyCount}"
        return 1
    fi
}

function _testRunUtil_correctReady() {
    local prefix="${1}"
    if [[ "${PRETEND}" -eq 1 ]]; then
        echo "Pretend: Would have checked all nodes with prefix ${prefix} are ready with ${NODETOTAL} nodes"
        _testRunner_Results "${FUNCNAME[1]}" "Passed"
        return 0
    fi
    res=$(_remoteExec "${prefix}-kmaster" "kubectl get nodes")
    #shellcheck disable=2310 #- this is a test function
    if _testRunUtil_expect_grep_count "${res}" "${NODETOTAL}"; then
        _testRunner_Results "${FUNCNAME[1]}" "Passed"
        return 0
    else
        _testRunner_Results "${FUNCNAME[1]}" "failed to get ready nodes"
        diagnose_failing_nodes "${prefix}" "${res}"
        return 1
    fi
}

## @fn _testRunUtil_getNodeMemory()
## @brief Retrieves the memory allocated to a node in GB.
## @param node The node identifier.
## @details Fetches the node configuration using `incus config show` command, parses the memory line, extracts the memory value in MiB, and converts it to GB.
## @return Echoes the memory allocated to the node in GB.
function _testRunUtil_getNodeMemory() {
    local node
    local config
    local memline
    local memCountwq
    local memValue
    local memCount
    node="${1}"
    config=$(incus config show "${node}")
    memline=$(echo "${config}" | grep memory)
    memCountwq=$(echo "${memline}" | awk '{print $2}')
    memValue=${memCountwq%MiB}
    memCount=$(echo "${memValue} / 1024" | bc)
    echo "${memCount}"
}

## @fn _testRunUtil_getNodeCPU()
## @brief Retrieves the number of CPUs allocated to a node.
## @param node The node identifier.
## @details Fetches the node configuration using `incus config show` command, parses the CPU line, and extracts the CPU count.
## @return Echoes the number of CPUs allocated to the node.
function _testRunUtil_getNodeCPU() {
    # Check nbr of arguments received
    if [[ "$#" -ne 1 ]]; then
        echo "_testRunUtil_getNodeCPU: wrong number of arguments $#, expected 1"
        return 1
    fi
    local node
    local config
    local cpuline
    local cpuCount
    node="${1}"
    config=$(incus config show "${node}")
    cpuline=$(echo "${config}" | grep cpu)
    cpuCount=$(echo "${cpuline}" | awk '{print $2}')
    cpuCount="${cpuCount//\"/}" # Remove double quotes
    echo "${cpuCount}"
}
# ----------------------------------------------------------------------------------------------------- #
#                                                                                                       #
#                                              util tests                                               #
# ----------------------------------------------------------------------------------------------------- #
## @fn test_util_buildNodeList()
## @brief Tests the `_buildNodeList` utility function.
## @param prefix The prefix to use for the node list.
## @details Calls the `_buildNodeList` function with the provided prefix and compares the output to the expected node list.
## If the output matches the expected list, the test is marked as "Passed"; otherwise, it is marked as "Failed".
## @return Returns 0 if the test passes, indicating success; otherwise, returns 1.
function test_util_buildNodeList() {
    local prefix="${1}"
    NODETOTAL=6
    local nodeList
    #shellcheck disable=SC2312 #prefer to error if _buildNodeList fails
    mapfile -t nodeList < <(_buildNodeList "${prefix}")
    if [[ "${nodeList[*]}" == "${prefix}-kmaster ${prefix}-kworker1 ${prefix}-kworker2 ${prefix}-kworker3 ${prefix}-kworker4 ${prefix}-kworker5" ]]; then
        _testRunner_Results "${FUNCNAME[0]}" "Passed"
        return 0
    else
        _testRunner_Results "${FUNCNAME[0]}" "Failed"
        return 1
    fi
}
## @fn test_util_cleanup()
## @brief Cleans up resources after tests.
## @details This function cleans up the k8s cluster resources based on the provided prefix. It calls the `_testRunUtil_cleanup` utility function to perform the cleanup. If the cleanup is successful, it logs the result as "Passed"; otherwise, it logs "Failed".
## @param prefix The prefix used for naming the k8s cluster resources to be cleaned up.
## @return Returns 0 if the cleanup is successful, indicating success; otherwise, returns 1.
function test_util_cleanup() {
    prefix="${1}"
    #shellcheck disable=SC2310 #this is a test function
    if _testRunUtil_cleanup "${prefix}"; then
        _testRunner_Results "${FUNCNAME[0]}" "Passed"
        return 0
    else
        _testRunner_Results "${FUNCNAME[0]}" "Failed"
        return 1
    fi
}
## @fn test_util_expectedGrepNotReady()
## @brief Verifies that not all nodes are in the Ready state.
## @details This function checks if the k8s cluster nodes are not all in the Ready state by simulating the output of a k8s cluster status command. It uses the `_testRunUtil_expect_grep_count` utility function to count the occurrences of "Ready" statuses. If the count matches the expected number, indicating that not all nodes are Ready, the test is marked as failed, as it expects at least one node to not be in the Ready state.
## @return Returns 0 if the test passes (at least one node is not Ready), indicating success; otherwise, returns 1.
function test_util_expectedGrepNotReady() {
    local isNotReady="NAME              STATUS     ROLES           AGE    VERSION
    tr-tra-kmaster    Ready      control-plane   2m7s   v1.29.4
    tr-tra-kworker1   Ready      <none>          61s    v1.29.4
    tr-tra-kworker2   NotReady   <none>          1s     v1.29.4"
    # shellcheck disable=2310 #- this is a test function
    if _testRunUtil_expect_grep_count "${isNotReady}" 3; then
        _testRunner_Results "${FUNCNAME[0]}" "Failed" "Should not count NotReady as Ready"
        return 1
    else
        _testRunner_Results "${FUNCNAME[0]}" "Passed"
    fi
}

## @fn test_util_expectedGrepReady()
## @brief Verifies all nodes are in the Ready state.
## @details This function simulates checking the status of k8s cluster nodes to ensure they are all in the "Ready" state. It uses a predefined string that represents the output of a hypothetical k8s cluster status command and checks if the count of "Ready" statuses matches the expected number. If the count is correct, the test passes; otherwise, it fails.
## @return Returns 0 if all nodes are in the Ready state, indicating success; otherwise, returns 1.
function test_util_expectedGrepReady() {
    local isReady="NAME              STATUS   ROLES           AGE    VERSION
    tr-tra-kmaster    Ready    control-plane   2m7s   v1.29.4
    tr-tra-kworker1   Ready    <none>          61s    v1.29.4
    tr-tra-kworker2   Ready    <none>          1s     v1.29.4"
    # shellcheck disable=2310 #- this is a test function
    if _testRunUtil_expect_grep_count "${isReady}" 3; then
        _testRunner_Results "${FUNCNAME[0]}" "Passed"
    else
        _testRunner_Results "${FUNCNAME[0]}" "Failed"
        return 1
    fi
}

## @fn xtest_util_forceFail()
## @brief Forces a test failure.
## @details This function is designed to always fail. It is useful for testing the test framework itself or ensuring that failure handling works as expected. It explicitly logs the test as "Failed" and returns a non-zero exit code.
## @return Always returns 1, indicating failure.
function xtest_util_forceFail() {
    _testRunner_Results "${FUNCNAME[0]}" "Failed" "This test is expected to fail"
    return 1
}

## @fn test_util_forcePass()
## @brief Forces a test pass.
## @details This function is designed to always pass. It is useful for testing the test framework itself or ensuring that success handling works as expected. It explicitly logs the test as "Passed" and returns a zero exit code.
## @return Always returns 0, indicating success.
function test_util_forcePass() {
    _testRunner_Results "${FUNCNAME[0]}" "Passed"
    return 0
}

## @fn test_util_validateWordInList()
## @brief Validates if a word is in a list using the function _validateWordInList.
## @details This function tests if a word is in a list by comparing the word to the list. If the word is found in the list, the test passes; otherwise, it fails.
## @param prefix The prefix used for naming the k8s cluster resources and is always required for a test_ function
## @return Returns 0 if the word is found in the list, indicating success; otherwise, returns 1.
function test_util_validateSingleWordInList() {
    local prefix="${1}"
    #shellcheck disable=SC2310 #this is a test function
    if _isWordsInList "1.27" SUPPORTEDK8s "Invalid version"; then
        _testRunner_Results "${FUNCNAME[0]}" "Passed"
        return 0
    else
        _testRunner_Results "${FUNCNAME[0]}" "Failed"
        return 1
    fi
}

## @fn test_util_validateMultipleWordsInList()
## @brief Validates if multiple words are in a csv list using the function _validateWordInList.
## @details This function tests if a word is in a list by comparing the word to the list. If the word is found in the list, the test passes; otherwise, it fails.
## @param prefix The prefix used for naming the k8s cluster resources and is always required for a test_ function
## @return Returns 0 if the word is found in the list, indicating success; otherwise, returns 1.
function test_util_validateMultipleWordsInList() {
    local prefix="${1}"
    #shellcheck disable=SC2310 #this is a test function
    if _isWordsInList "1.27,1.28" SUPPORTEDK8s "Invalid version xxx"; then
        _testRunner_Results "${FUNCNAME[0]}" "Passed"
        return 0
    else
        _testRunner_Results "${FUNCNAME[0]}" "Failed"
        return 1
    fi
}

# ----------------------------------------------------------------------------------------------------- #
#                                                                                                       #
#                                              incus tests                                              #
# ----------------------------------------------------------------------------------------------------- #
## @fn test_incus_launchNoTarget()
## @brief Tests launching an instance in a non Incus cluster environment.
## @details This function tests launching an instance without a target, which simulates a scenario where no incus cluster is available. It calls the `launchInstance` function with an empty target and checks if the IP address is empty. If the IP address is empty, the test passes; otherwise, it fails.
## @param prefix The prefix used for naming the k8s cluster resources.
## @return Returns 0 if the IP address is empty, indicating success; otherwise, returns 1.
function test_incus_launchNoTarget() {
    local prefix="${1}"
    local instanceName="${prefix}-kmaster"
    local instanceType="vm"
    local os="ubuntu"
    local osVersion="22.04"
    local cpu="2"
    local memory="2"
    local target=""
    local project="default"
    LASTIP=""
    launchInstance "${instanceName}" "${instanceType}" "${os}" "${osVersion}" "${cpu}" "${memory}" "${target}" "${project}"
    if [[ "${LASTIP}" == "" ]]; then
        _testRunner_Results "${FUNCNAME[0]}" "Failed" "No IP address returned"
        return 1
    else
        _testRunner_Results "${FUNCNAME[0]}" "Passed"
        return 0
    fi
}
## @fn diagnose_failing_nodes()
## @brief Try to get diagnostics for nodes that are not ready.
## @details This function runs remotely kubectl describe node & kubectl logs. Runs locally incus info on the node.
## @param prefix The prefix used for naming the k8s cluster resources.
## @param res    The results of the failing kubectl get nodes that triggers this action.
## @return Returns 0 (success) by default (even if _remoteExec or incus fails). Returns with 1 if wrong number arguments provided.
function diagnose_failing_nodes() {
    # Check nbr of arguments received
    if [[ "$#" -ne 2 ]]; then
        echo "diagnose_failing_nodes: wrong number of arguments"
        return 1
    fi
    local prefix
    prefix="${1}"
    local res
    res="${2}"
    failing=$(echo "${res}" | grep -v "Ready")
    for node in ${failing}; do
        #shellcheck disable=2310 #- this is a test function
        _remoteExec "${prefix}-kmaster" "kubectl describe node ${node}" || true
        #shellcheck disable=2310 #- this is a test function
        _remoteExec "${prefix}-kmaster" "kubectl logs ${node}" || true
        incus info "${node}" || true
    done
    return 0
}
# ----------------------------------------------------------------------------------------------------- #
#                                                                                                       #
#                                   k8s bring up and operations tests                                   #
# ----------------------------------------------------------------------------------------------------- #

## @fn test_k8s_calico()
## @brief Tests the k8s cluster setup with Calico as the CNI provider.
## @details This function sets the CNI provider to Calico, deploys a k8s cluster, and checks if all nodes are in the "Ready" state.
## @param prefix The prefix used for naming the k8s cluster resources.
## @return Returns 0 if all nodes are in the "Ready" state, indicating success; otherwise, returns 1.
function test_k8s_calico() {
    prefix="${1}"
    CNI_PROVIDER="calico"
    program_main "${prefix}"
    _testRunUtil_correctReady "${prefix}"
}

## @fn test_k8s_creationWithDns()
## @brief Tests the creation of the environment with DNS enabled.
## @param prefix The prefix used for naming the nodes in the k8s cluster.
## @details This function initiates the environment setup with DNS capabilities and checks if the nodes are in the 'Ready' state.
## It relies on the main function to set up the environment and _remoteExec to execute commands on the remote node.
## @note This function assumes a working DNS environment where nodes can resolve each other.
## @return Exits with 1 on failure if EXITONERROR is set; otherwise, returns 0 on success and 1 for failure.
function test_k8s_creationWithDns() {
    local prefix="${1}"
    local res
    CPU=2
    MEMORY=4
    program_main "${prefix}"
    _testRunUtil_correctReady "${prefix}"
}

## @fn test_k8s_creationWithoutDns()
## @brief Tests the creation of the environment without DNS.
## @param prefix The prefix used for naming the nodes in the k8s cluster.
## @details This function initiates the environment setup without DNS capabilities and checks if the nodes are in the 'Ready' state.
## It sets FORCENODNS to 1 to simulate an environment without DNS and relies on the main function for setup.
## @note This function simulates a non-DNS environment and may give a false positive if DNS is actually required for operation.
## @return Exits with 1 on failure if EXITONERROR is set; otherwise, returns 0 on success and 1 for failure.
function test_k8s_creationWithoutDns() {
    local prefix="${1}"
    FORCENODNS=1
    local res
    program_main "${prefix}"
    _testRunUtil_correctReady "${prefix}"
}

## @fn test_k8s_defaults()
## @brief Tests the default k8s cluster setup.
## @details This function tests the default setup by deploying a k8s cluster and checking if all nodes are in the "Ready" state.
## @param prefix The prefix used for naming the k8s cluster resources.
## @return Returns 0 if all nodes are in the "Ready" state, indicating success; otherwise, returns 1.
function test_k8s_defaults() {
    local prefix="${1}"
    local res
    program_main "${prefix}"
    _testRunUtil_correctReady "${prefix}"
}

## @fn test_incus_ram_cpu_variants()
## @brief Tests RAM and CPU variants for instances when the comma separated variable format.
## @param prefix The prefix used for naming the instances resources.
## @details This function tests the RAM and CPU variants for instances by deploying a k8s cluster with different CPU and RAM values.
## It compares the expected values with the actual values for each node and logs the test as "Passed" if the values match; otherwise, it logs the failure and returns 1.
## @return Returns 0 if the test passes (all nodes have the expected CPU and RAM values), otherwise returns 1.
function test_incus_ram_cpu_variants() {
    local prefix="${1}"
    local CPU MEMORY cpu mem node nodeRay nodeCPU nodeMemory res
    CPU="2,4,6"
    MEMORY="2,4,6"
    launchClusterInstances "${prefix}"
    #shellcheck disable=SC2312 #prefer to error if _buildNodeList fails
    IFS=' ' read -ra nodeRay <<<"$(_buildNodeList "${prefix}")"
    # Convert CSV strings to arrays
    IFS=',' read -r -a CPU_ARRAY <<<"${CPU}"
    IFS=',' read -r -a MEMORY_ARRAY <<<"${MEMORY}"
    # Loop over the indices of the arrays
    for i in "${!CPU_ARRAY[@]}"; do
        cpu="${CPU_ARRAY[${i}]}"
        mem="${MEMORY_ARRAY[${i}]}"
        node="${nodeRay[${i}]}"
        # echo "Checking node $node with CPU $cpu and MEMORY $mem"
        # in pretend mode, we don't actually check the values
        if [[ "${PRETEND}" -eq 1 ]]; then
            nodeCPU="${cpu}"
            nodeMemory="${mem}"
        else
            nodeCPU=$(_testRunUtil_getNodeCPU "${node}")
            nodeMemory=$(_testRunUtil_getNodeMemory "${node}")
        fi
        if [[ "${cpu}" -eq "${nodeCPU}" && "${mem}" -eq "${nodeMemory}" ]]; then
            :
        else
            _testRunner_Results "${FUNCNAME[0]}" "Node ${node} has incorrect CPU ${nodeCPU} (${cpu} expected) or MEMORY ${nodeMemory} (${mem} expected) values"
            return 1
        fi
    done
    _testRunner_Results "${FUNCNAME[0]}" "Passed"
}

## @fn test_k8s_version()
## @brief Tests if the Kubernetes cluster is set up with the specified version.
## @param prefix The prefix used for naming the k8s cluster resources.
## @param version The Kubernetes version to test for.
## @details This function sets the Kubernetes version if specified, initiates the k8s cluster setup using the 'main' function, and then checks if all nodes are in the "Ready" state.
## It uses '_remoteExec' to execute 'kubectl get nodes' on the master node and '_testRunUtil_expect_grep_count' to verify that all nodes are ready.
## If the nodes are ready, it logs the test as passed; otherwise, it logs the failure and returns 1.
## @return Returns 0 if the test passes (all nodes are ready), otherwise returns 1.
function test_k8s_version() {
    local prefix="${1}"
    local version="${2:-}"
    if [[ "${version}" != "" ]]; then
        K8SVERSION="${version}"
    fi
    program_main "${prefix}"
    _testRunUtil_correctReady "${prefix}"
}

## @fn test_k8s_versions()
## @brief Tests k8s cluster setup with different Kubernetes versions.
## @param prefix The prefix used for naming the k8s cluster resources.
## @details Iterates over a list of supported Kubernetes versions, sets each version, and initiates the k8s cluster setup.
## For each version, it checks if all nodes are in the "Ready" state by executing 'kubectl get nodes' on the master node.
## Utilizes '_testRunUtil_expect_grep_count' to verify the count of "Ready" nodes matches the expected count.
## If a version passes the test, it logs the result as "Passed"; otherwise, it logs the failure and returns 1.
## @note This function returns after the first failure without testing remaining versions.
## @return Returns 0 if all nodes are ready for a specific version, otherwise returns 1 on the first failure.
function test_k8s_versions() {
    local prefix="${1}"
    for version in "${SUPPORTEDK8s[@]}"; do
        K8SVERSION="${version}"
        program_main "${prefix}"
        _testRunUtil_correctReady "${prefix}"
    done
}
## @fn testSuite()
## Executes a suite of tests based on the provided test name and prefix.
## This function dynamically identifies and runs test functions within the script.
## It supports running a specific test, multiple tests, or all tests, excluding those
## that are intentionally masked out (e.g., due to external resource constraints).
##
## @param testname The name of the test suite to run.
##                 Use "all" to run all available tests.
##                 Multiple test names can be specified, separated by commas.
## @param prefix The prefix to use for the test run.
##               This prefix is used for identifying or grouping test runs, potentially for isolation or reporting.
## @param additionalArgs Additional arguments to be passed to the test functions.
##
## @details The function first checks for the presence of mandatory parameters (test name and prefix).
##          It then collects all defined functions in the script and filters out those that are test functions
##          (i.e., functions whose names start with "test_"). Depending on the test name parameter, it either runs
##          all test functions (excluding any specifically masked out tests) or runs only the specified test(s).
##          Each test function is called with the prefix and any additional arguments. If any test function returns
##          a non-zero exit code, the overall return code is set to 8. After running the tests, a test report is displayed,
##          and the script exits with the determined return code.
##
## @note The function uses global variables for managing test execution state and assumes the existence of helper functions
##       like _banner, _subbanner, _testRunUtil_cleanup, and _testRunner_displayTestReport for logging and cleanup tasks.
##
## @return Exits with 0 if all tests pass, or 8 if any test fails.
function testSuite() {
    local testname="${1}"
    local prefix="TR-${2}"
    local foundTest=0
    local allFunctions
    local returnCodes=0
    shift 2
    if [[ "${testname}" == "" ]]; then
        displayHelp "No test suite specified" "<testname> <prefix>"
    fi
    if [[ "${prefix}" == "" ]]; then
        displayHelp "No prefix specified" "<prefix>"
    fi
    _banner "Test Suite"
    allFunctions="$(declare -F)"
    testfunctions=()

    # Loop through all function names in allFunctions
    for funcname in ${allFunctions}; do
        # Check if the function name starts with 'test_'
        if [[ "${funcname}" == test_* ]]; then
            # Add the function name to the testfunctions array
            testfunctions+=("${funcname}")
        fi
    done

    # Function to run tests based on pattern
    run_tests() {
        local pattern="$1"
        # Convert wildcard pattern to a regular expression
        local regex_pattern pattern       # Example pattern
        regex_pattern="${pattern//\*/.*}" # Replace '*' with '.*' using native Bash string manipulation
        _banner "Running Test Suites matching pattern: ${pattern}"
        for test in "${testfunctions[@]}"; do
            # Mask out the test_versions test because it currently uses too many docker hub pulls
            if [[ "${test}" != "test_k8s_versions" && "${test}" =~ ${regex_pattern} ]]; then
                _subbanner "Running ${test} with prefix ${prefix}"
                if ! ${test} "${prefix}"; then
                    returnCodes="${EC_TEST_FAILED_IN_LIST}"
                fi
                _testRunUtil_cleanup "${prefix}"
                _subbanner "Completed ${test} with prefix ${prefix}"
            fi
        done
    }
    matchedPattern=""
    for key in "${!TESTMACROS[@]}"; do
        pattern="${TESTMACROS[${key}]}"
        if [[ "${testname}" == "${key}" ]]; then
            matchedPattern=${pattern}
            break
        fi
    done

    if [[ -n "${matchedPattern}" ]]; then
        run_tests "${matchedPattern}"
    else
        IFS=',' read -r -a testNameArray <<<"${testname}"
        for testNameEntry in "${testNameArray[@]}"; do
            foundTest=0
            for funcname in "${testfunctions[@]}"; do
                # Check if the function name is the same as the current testNameEntry
                if [[ "${funcname}" == "${testNameEntry}" ]]; then
                    foundTest=1
                    _subbanner "Running ${testNameEntry} with prefix ${prefix} and args $*"
                    if ! ${testNameEntry} "${prefix}" "$@"; then
                        returnCodes="${EC_TEST_FAILED_IN_LIST}"
                    fi
                    _testRunUtil_cleanup "${prefix}"
                    _subbanner "Completed ${testNameEntry} with prefix ${prefix}"
                    break # This break exits the for loop for testfunctions
                fi
            done
            if [[ "${foundTest}" -eq 0 ]]; then
                displayHelp "Invalid test suite ${testNameEntry}"
            fi
        done
    fi
    _banner "Test Suite Report"
    _testRunner_displayTestReport
    return "${returnCodes}"
}

# ----------------------------------------------------------------------------------------------------- #
#                                             ____ _     ___                                            #
#                                            / ___| |   |_ _|                                           #
#                                           | |   | |    | |                                            #
#                                           | |___| |___ | |                                            #
#                                            \____|_____|___|                                           #
#                                                                                                       #
#                                      Things to make the cli work                                      #
# ----------------------------------------------------------------------------------------------------- #

function displayHelp() {
    local error="${1:-}"
    local errorArg="${2:-}"

    echo -e "# ----------------------------------------------------------------------------------------------------- #"
    echo -e "#                                                             _                                         #"
    echo -e "#                             __      ___ __ __ _ _ __   __ _| | ___ _ __                               #"
    echo -e "#                             \ \ /\ / / '__/ _' | '_ \ / _' | |/ _ \ '__|                              #"
    echo -e "#                              \ V  V /| | | (_| | | | | (_| | |  __/ |                                 #"
    echo -e "#                               \_/\_/ |_|  \__,_|_| |_|\__, |_|\___|_|                                 #"
    echo -e "#                                                       |___/                                           #"
    echo -e "#                                          incus environments                                           #"
    echo -e "# ----------------------------------------------------------------------------------------------------- #"

    if [[ "${error}" != "" ]]; then
        echo -en "${BOLD}${RED}Error:\t${NORM} ${BRIGHTRED}${error}${NORM}\n\n"
    fi
    if [[ "${errorArg}" != "" ]]; then
        echo -en "Usage:\t${CLICALL}${BRIGHTRED} ${errorArg}${NORM}\n"
    else
        echo -e "Usage:\t$0 [options] <prefix>"
    fi
    echo ""
    echo "This script creates K8s clusters using instances (both vm and containers) managed in incus."
    echo ""
    echo "Options:"
    echo "  --cpu <cpu>: CPU count for the nodes. Default: ${DEFAULT_CPU}."
    echo "            --example: --cpu 2 assigns 2 cpus to all nodes. (-kmaster and -kworker(n))"
    echo "      -- multiple cpu counts provided via comma separated list. (must align with node count from --num-nodes)"
    echo "            --example: --cpu 2,4,6 assigns 2 -kmaster and 4 -kworker1 and 6 -kworker2."
    echo "  --containers: use containers instead of the default ${DEFAULT_INSTANCETYPE}"
    echo -n "  --deployment-type <deploymentType>: One of "
    _helplist "${DEPLOYMENTTYPES[@]}"
    echo ". Default: \"${DEFAULT_DEPLOYMENTTYPE}\""
    echo "  --debug-level <Debug_level>: set the debug level. Default: ${DEBUG}"
    echo "      -- 0: No debug output"
    echo "      -- 1: Minimal debug output with stack trace"
    echo "      -- 2: additional stack trace info"
    echo "      -- 3: developer based, printing of each line number"
    echo "      -- 4: shopt -x tracing"
    echo "  --exit-on-error: Exit on error during test module. Essentially disables auto-cleanup."
    echo "  --help: Display this help message."
    echo "  --incus-project <incus project>: Default: ${DEFAULT_PROJECT}"
    echo "  --memory <memory>: Memory for the nodes in GB. Default: ${DEFAULT_MEMORY}"
    echo "             --example: --memory 4 assigns 4GB of memory to all nodes (-kmaster and -kworker(n))."
    echo "      -- multiple memory specs provided via comma separated list. Must align with node count from --num-nodes."
    echo "             --example: --memory 2,4,4 assigns 2GB to -kmaster and 4GB to -kworker1 and -kworker2."
    echo "  --num-nodes <number_Nodes>: Number  of nodes to create for the k8s cluster. Default: ${DEFAULT_NODETOTAL}"
    vcniText=$(_helplist "${CNI_PROVIDERS[@]}")
    echo "  --network-provider <network_provider>: The network provider. Options are ${vcniText}. Default: ${DEFAULT_CNI_PROVIDER}"
    echo "  --os <OS>: Operating system for the instances. Default: ${DEFAULT_OS}. (Ubuntu is the only supported OS as of version 0.2)"
    echo "  --os-version <OS version>: Operating system version for the instances. Default: ${DEFAULT_OSVERSION}"
    echo "  --output-dir <output_dir>: Directory to store the recovery script. Default: ${DEFAULT_OUTPUT_DIR}"
    echo "  --pod-cidr-network <pod-cidr-network>: Default: ${DEFAULT_POD_NETWORK_CIDR}"
    echo "  --pretend: Run the script in pretend mode. No changes will be made."
    echo "  --tests <tests>: Run a specific test, multiple tests or macro defined tests."
    echo "      --example: --tests all will run all tests that start with test_"
    echo "      --example: --tests k8s will run all tests that start with test_k8s_"
    echo "      --example: --tests util will run all tests that start with test_util_"
    echo "      --example: --tests test_util_cleanup will run the test_util_cleanup test"
    echo "      --example: --tests test_util_cleanup,test_util_forcePass will run those two tests"
    echo "  --target-hosts <target incus host(s)>: to deploy to."
    if [[ ${DEFAULT_TARGET} != "" ]]; then
        vtargetText=$(_helplist "${VALIDTARGETS[@]}")
        local countValidTargets=${#VALIDTARGETS[@]}
        echo "      -- Availability: ${vtargetText}. Default: ${DEFAULT_TARGET}"
        echo "             --example: --target-hosts ${VALIDTARGETS[((countValidTargets - 1))]} will deploy all nodes (-kmaster, kworker1, kworker2) to the ${VALIDTARGETS[((countValidTargets - 1))]} host."
        echo "      -- multiple hosts provided via comma separated list. (alignment with --num-nodes flag required)"
        if [[ countValidTargets -ge 3 ]]; then
            echo "             --example: --target-hosts ${VALIDTARGETS[((countValidTargets - 3))]},${VALIDTARGETS[((countValidTargets - 2))]},${VALIDTARGETS[((countValidTargets - 1))]} will -kmaster to ${VALIDTARGETS[((countValidTargets - 3))]}, -kworker1 to ${VALIDTARGETS[((countValidTargets - 2))]} and -kworker2 to ${VALIDTARGETS[((countValidTargets - 1))]}."
        elif [[ countValidTargets -eq 2 ]]; then
            echo "             --example: --target-hosts ${VALIDTARGETS[((countValidTargets - 2))]},${VALIDTARGETS[((countValidTargets - 1))]},${VALIDTARGETS[((countValidTargets - 1))]} will deploy -kmaster to ${VALIDTARGETS[((countValidTargets - 2))]} and -kworker(n) to ${VALIDTARGETS[((countValidTargets - 1))]}"
        else
            echo "             --example: --target-hosts ${VALIDTARGETS[((countValidTargets - 1))]} will deploy all nodes to the ${VALIDTARGETS[((countValidTargets - 1))]}."
        fi
        echo "      -- Not available on this host.  Not a Incus cluster member."
    fi
    echo "  --k8s-version <K8s version>: Sets the version of Kubernetes to run. Currently defaults to: ${DEFAULT_K8SVERSION}."
    echo "      Supported versions are:"
    for version in "${SUPPORTEDK8sSversions[@]}"; do
        echo "      -- ${version}"
    done
    echo "  --version: display the version of this script."
    echo "Arguments:"
    echo "  <K8s_cluster_prefix>: a prefix for each node in the k8s cluster. All nodes base name will be  -kmaster or -kworker(n)."
    echo -e "Example 1:"
    echo "Creates a 4-node Kubernetes cluster with VMs (2 CPUs, 4GB RAM each). Master on H91 (Tcluster-kmaster), workers on H92-H94 (Tcluster-kworker1-3). Generates a recovery script at ${HOME}/K8sWrangler/Tcluster-recover.sh."
    echo "  -- command: $0 --cpu 2 --memory 4 --num-nodes 4 --target-hosts H91,H92,H93,H94 Tcluster"
    echo ""
    echo "Error Codes:"
    for code in "${ERRORCODES[@]}"; do
        msg_index=${code}
        echo "  -- ${!code} ${code}: ${ERRORCODE_MSG[${msg_index}]}"
    done
    echo ""

    if [[ "${error}" == "" ]]; then
        exit 0
    fi
    _handle_error "${EC_CLI_ERROR}" "${error}"
}

function processArgs() {
    OPTIONS=$(getopt -o h --long cpu:,containers,deployment-type:,debug-level:,exit-on-error,help,incus-project:,memory:,num-nodes:,network-provider:,os:,os-version:,output-dir:,pretend,pod-cidr-network:,target-hosts:,tests:,k8s-version:,version -n 'parse-options' -- "$@")
    #shellcheck disable=2181 #we want to use this compare in brackets
    if [[ $? -ne 0 ]]; then
        echo "Failed parsing options." >&2
        _handle_error "${EC_CLI_BAD_OPTION_PARSE}" "Failed parsing options."
    fi
    #echo "OPTIONS=$OPTIONS"
    eval set -- "${OPTIONS}"
    while true; do
        case "$1" in
        --containers)
            DEFAULT_INSTANCETYPE="container"
            shift
            ;;
        --cpu)
            DEFAULT_CPU="$2"
            shift 2
            ;;
        --debug-level)
            DEBUG="$2"
            debugModeEnable
            shift 2
            ;;
        --deployment-type)
            _validateWordInList "$2" DEPLOYMENTTYPES "Invalid deployment type $2"
            DEFAULT_DEPLOYMENTTYPE="$2"
            shift 2
            ;;
        --exit-on-error)
            EXITONERROR=1
            shift
            ;;
        -h | --help)
            displayHelp
            ;;
        --incus-project)
            DEFAULT_PROJECT="$2"
            plist="$(incus DEFAULT_project list -f csv -c "n")"
            echo "${plist}" | grep -q "${DEFAULT_PROJECT}"
            if ! echo "${plist}" | grep -q "${DEFAULT_PROJECT}"; then
                displayHelp "Project ${DEFAULT_PROJECT} does not exist"
            fi
            shift 2
            ;;
        --k8s-version)
            _validateWordInList "$2" SUPPORTEDK8sSversions "Invalid Kubernetes version $2"
            DEFAULT_K8SVERSION="$2"
            shift 2
            ;;
        --memory)
            DEFAULT_MEMORY="$2"
            shift 2
            ;;
        --network-provider)
            _validateWordInList "$2" CNI_PROVIDERS "Invalid CNI provider $2"
            DEFAULT_CNI_PROVIDER="$2"
            shift 2
            ;;
        --num-nodes)
            DEFAULT_NODETOTAL="$2"
            if [[ "${DEFAULT_NODETOTAL}" -lt 3 ]]; then
                displayHelp "You must specify at least 3 nodes (--num-nodes 3)"
            fi
            shift 2
            ;;
        --os)
            DEFAULT_OS="$2"
            shift 2
            ;;
        --os-version)
            DEFAULT_OSVERSION="$2"
            shift 2
            ;;
        --output-dir)
            DEFAULT_OUTPUT_DIR="$2"
            shift 2
            ;;
        --pretend)
            PRETEND=1
            shift
            ;;
        --pod-cidr-network)
            DEFAULT_POD_NETWORK_CIDR="$2"
            shift 2
            ;;
        --target-hosts)
            if [[ "${DEFAULT_TARGET}" == "" ]]; then
                displayHelp "Target --target-hosts not supported.  This host is not a member of an Incus cluster."
            else
                _validateWordInList "$2" VALIDTARGETS "Invalid target-hosts $2"
                DEFAULT_TARGET="$2"
            fi
            shift 2
            ;;
        --tests)
            TESTTORUN="$2"
            shift 2
            ;;
        --version)
            echo "Version: ${SCRIPTVERSION}"
            exit 0
            ;;
        --)
            shift
            break
            ;;
        *)
            echo "Invalid Option: $1" 1>&2
            exit 1
            ;;
        esac
    done
    if [[ $# -ne 1 ]]; then
        displayHelp "Prefix not specified" "<prefix>"
    fi
    MYPREFIX="$1"
    shift
    _resetDefaults
    if [[ "${MYPREFIX}" == "" ]]; then
        displayHelp "Prefix not specified" "<prefix>"
    fi
    if [[ "${TESTTORUN}" != "" ]]; then
        _makeTmpDir
        _testRunUtil_cleanup "${MYPREFIX}"
        testSuite "${TESTTORUN}" "${MYPREFIX}" "$@"
        rc=$?
        _destroyTmpDir
        _handle_error "${rc}" "Tests: ${TESTTORUN} on ${MYPREFIX}"
    fi
    _makeTmpDir
    program_main "${MYPREFIX}"
    _destroyTmpDir
}

# ----------------------------------------------------------------------------------------------------- #
#                                   ____                                                                #
#                                  |  _ \ _ __ ___ _ __ ___  __ _ ___                                   #
#                                  | |_) | '__/ _ \ '__/ _ \/ _` / __|                                  #
#                                  |  __/| | |  __/ | |  __/ (_| \__ \                                  #
#                                  |_|   |_|  \___|_|  \___|\__, |___/                                  #
#                                                              |_|                                      #
#                                   what we need before this can run                                    #
# ----------------------------------------------------------------------------------------------------- #
function prereqs() {
    #make sure incus is installed
    local contflag=0
    local capture
    if ! command -v incus &>/dev/null; then
        capture+="$(_testRunner_failure "incus is  installed")\n"
        contflag="${EC_CLI_PRECHECK_FAILED}"
    else
        capture+="$(_testRunner_success "incus is installed")\n"
    fi
    # test if the current user is a member of incus group
    if ! groups | grep -qE "\bincus\b"; then
        capture+="$(_testRunner_failure "User is a member of the incus group")\n"
        contflag="${EC_CLI_PRECHECK_FAILED}"
    else
        capture+="$(_testRunner_success "User is a member of the incus group")\n"
    fi
    # test if the current user is a member of incus-admin
    if ! groups | grep -q "\bincus-admin\b"; then
        capture+="$(_testRunner_failure "User is a member of the incus-admin group")\n"
        contflag="${EC_CLI_PRECHECK_FAILED}"
    else
        capture+="$(_testRunner_success "User is a member of the incus-admin group")\n"
    fi
    if [[ "${contflag}" -ne 0 ]]; then
        echo -e "${capture}"
    fi
    return "${contflag}"
}

export DEBIAN_FRONTEND=noninteractive
#shellcheck disable=SC2124
CLICALL="$0 $@"
#shellcheck disable=SC2310 #this is a test function
if prereqs; then
    processArgs "$@"
else
    _handle_error "${EC_CLI_PRECHECK_FAILED}"
fi