#!/usr/bin/env bash

# Copyright 2017 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../../..

source "${KUBE_ROOT}/test/kubemark/common/util.sh"

# Wrapper for gcloud compute, running it $RETRIES times in case of failures.
# Args:
# $@: all stuff that goes after 'gcloud compute'
function run-gcloud-compute-with-retries {
  run-cmd-with-retries gcloud compute "$@"
}

function authenticate-docker {
  echo "Configuring registry authentication"
  mkdir -p "${HOME}/.docker"
  gcloud beta auth configure-docker -q
}

function create-kubemark-master {
  # We intentionally override env vars in subshell to preserve original values.
  # shellcheck disable=SC2030,SC2031
  (
    # All calls to e2e-grow-cluster must share temp dir with initial e2e-up.sh.
    kube::util::ensure-temp-dir
    export KUBE_TEMP="${KUBE_TEMP}"

    export KUBECONFIG="${RESOURCE_DIRECTORY}/kubeconfig.kubemark"
    export KUBECONFIG_INTERNAL="${RESOURCE_DIRECTORY}/kubeconfig-internal.kubemark"
    export CLUSTER_NAME="${CLUSTER_NAME}-kubemark"
    export KUBE_CREATE_NODES=false
    export KUBE_GCE_INSTANCE_PREFIX="${KUBE_GCE_INSTANCE_PREFIX}-kubemark"

    # Quite tricky cidr setup: we set KUBE_GCE_ENABLE_IP_ALIASES=true to avoid creating
    # cloud routes and RangeAllocator to assign cidrs by kube-controller-manager.
    export KUBE_GCE_ENABLE_IP_ALIASES=true
    export KUBE_GCE_NODE_IPAM_MODE=RangeAllocator

    # Disable all addons. They are running outside of the kubemark cluster.
    export KUBE_ENABLE_CLUSTER_AUTOSCALER=false
    export KUBE_ENABLE_CLUSTER_DNS=false
    export KUBE_ENABLE_NODE_LOGGING=false
    export KUBE_ENABLE_METRICS_SERVER=false
    export KUBE_ENABLE_L7_LOADBALANCING="none"

    # Unset env variables set by kubetest for 'root cluster'. We need recompute them
    # for kubemark master.
    # TODO(mborsz): Figure out some better way to filter out such env variables than
    # listing them here.
    unset MASTER_SIZE MASTER_DISK_SIZE MASTER_ROOT_DISK_SIZE

    # Set kubemark-specific overrides:
    # for each defined env KUBEMARK_X=Y call export X=Y.
    for var in ${!KUBEMARK_*}; do
      dst_var=${var#KUBEMARK_}
      val=${!var}
      echo "Setting ${dst_var} to '${val}'"
      export "${dst_var}"="${val}"
    done

    "${KUBE_ROOT}/hack/e2e-internal/e2e-up.sh"

    if [[ "${KUBEMARK_HA_MASTER:-}" == "true" && -n "${KUBEMARK_MASTER_ADDITIONAL_ZONES:-}" ]]; then
        for KUBE_GCE_ZONE in ${KUBEMARK_MASTER_ADDITIONAL_ZONES}; do
          KUBE_GCE_ZONE="${KUBE_GCE_ZONE}" KUBE_REPLICATE_EXISTING_MASTER=true \
            "${KUBE_ROOT}/hack/e2e-internal/e2e-grow-cluster.sh"
        done
    fi

    # The e2e-up.sh script is not sourced, so we don't have access to variables that
    # it sets. Instead, we read data which was written to the KUBE_TEMP directory.
    # The cluster-location is either ZONE (say us-east1-a) or REGION (say us-east1).
    # To get REGION from location, only first two parts are matched.
    REGION=$(grep -o "^[a-z]*-[a-z0-9]*" "${KUBE_TEMP}"/cluster-location.txt)
    MASTER_NAME="${KUBE_GCE_INSTANCE_PREFIX}"-master

    if [[ ${GCE_PRIVATE_CLUSTER:-} == "true" ]]; then
      MASTER_INTERNAL_IP=$(gcloud compute addresses describe "${MASTER_NAME}-internal-ip" \
          --project "${PROJECT}" --region "${REGION}" -q --format='value(address)')
    fi
    MASTER_IP=$(gcloud compute addresses describe "${MASTER_NAME}-ip" \
        --project "${PROJECT}" --region "${REGION}" -q --format='value(address)')

    # If cluster uses private master IP, two kubeconfigs are created:
    # - kubeconfig with public IP, which will be used to connect to the cluster
    #     from outside of the cluster network
    # - kubeconfig with private IP (called internal kubeconfig), which will be
    #      used to create hollow nodes.
    #
    # Note that hollow nodes might use either of these kubeconfigs, but
    # using internal one is better from performance and cost perspective, since
    # traffic does not need to go through Cloud NAT.
    if [[ -n "${MASTER_INTERNAL_IP:-}" ]]; then
      echo "Writing internal kubeconfig to '${KUBECONFIG_INTERNAL}'"
      ip_regexp=${MASTER_IP//./\\.} # escape ".", so that sed won't treat it as "any char"
      sed "s/${ip_regexp}/${MASTER_INTERNAL_IP}/g" "${KUBECONFIG}" > "${KUBECONFIG_INTERNAL}"
    fi
    )
}

function delete-kubemark-master {
  # We intentionally override env vars in subshell to preserve original values.
  # shellcheck disable=SC2030,SC2031
  (
    export CLUSTER_NAME="${CLUSTER_NAME}-kubemark"
    export KUBE_GCE_INSTANCE_PREFIX="${KUBE_GCE_INSTANCE_PREFIX}-kubemark"

    export KUBE_DELETE_NETWORK=false

    if [[ "${KUBEMARK_HA_MASTER:-}" == "true" && -n "${KUBEMARK_MASTER_ADDITIONAL_ZONES:-}" ]]; then
      for KUBE_GCE_ZONE in ${KUBEMARK_MASTER_ADDITIONAL_ZONES}; do
        KUBE_GCE_ZONE="${KUBE_GCE_ZONE}" KUBE_REPLICATE_EXISTING_MASTER=true \
          "${KUBE_ROOT}/hack/e2e-internal/e2e-shrink-cluster.sh"
      done
    fi

    "${KUBE_ROOT}/hack/e2e-internal/e2e-down.sh"
  )
}

function calculate-node-labels {
  echo "cloud.google.com/metadata-proxy-ready=true"
}


#!/usr/bin/env bash

# Copyright 2018 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Script that destroys the clusters used, namespace, and deployment.

KUBECTL=kubectl
KUBEMARK_DIRECTORY="${KUBE_ROOT}/test/kubemark"
RESOURCE_DIRECTORY="${KUBEMARK_DIRECTORY}/resources"

# Login to cloud services
complete-login

# Remove resources created for kubemark
# shellcheck disable=SC2154 # Color defined in sourced script
echo -e "${color_yellow}REMOVING RESOURCES${color_norm}"
spawn-config
"${KUBECTL}" delete -f "${RESOURCE_DIRECTORY}/addons" &> /dev/null || true
"${KUBECTL}" delete -f "${RESOURCE_DIRECTORY}/hollow-node.yaml" &> /dev/null || true
"${KUBECTL}" delete -f "${RESOURCE_DIRECTORY}/kubemark-ns.json" &> /dev/null || true
rm -rf "${RESOURCE_DIRECTORY}/addons"
	"${RESOURCE_DIRECTORY}/hollow-node.yaml" &> /dev/null || true

# Remove clusters, namespaces, and deployments
delete-clusters
if [[ -f "${RESOURCE_DIRECTORY}/iks-namespacelist.sh" ]] ; then
  bash "${RESOURCE_DIRECTORY}/iks-namespacelist.sh"
  rm -f "${RESOURCE_DIRECTORY}/iks-namespacelist.sh"
fi
# shellcheck disable=SC2154 # Color defined in sourced script
echo -e "${color_blue}EXECUTION COMPLETE${color_norm}"
exit 0


#!/usr/bin/env bash

# Copyright 2018 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Script that creates a Kubemark cluster for IBM cloud.

KUBECTL="${KUBE_ROOT}/cluster/kubectl.sh"
KUBEMARK_DIRECTORY="${KUBE_ROOT}/test/kubemark"
RESOURCE_DIRECTORY="${KUBEMARK_DIRECTORY}/resources"

# Generate secret and configMap for the hollow-node pods to work, prepare
# manifests of the hollow-node and heapster replication controllers from
# templates, and finally create these resources through kubectl.
function create-kube-hollow-node-resources {
  # Create kubeconfig for Kubelet.
  KUBELET_KUBECONFIG_CONTENTS="$(cat <<EOF
apiVersion: v1
kind: Config
users:
- name: kubelet
  user:
    client-certificate-data: "${KUBELET_CERT_BASE64}"
    client-key-data: "${KUBELET_KEY_BASE64}"
clusters:
- name: kubemark
  cluster:
    certificate-authority-data: "${CA_CERT_BASE64}"
    server: https://${MASTER_IP}
contexts:
- context:
    cluster: kubemark
    user: kubelet
  name: kubemark-context
current-context: kubemark-context
EOF
)"

  # Create kubeconfig for Kubeproxy.
  KUBEPROXY_KUBECONFIG_CONTENTS="$(cat <<EOF
apiVersion: v1
kind: Config
users:
- name: kube-proxy
  user:
    client-certificate-data: "${KUBELET_CERT_BASE64}"
    client-key-data: "${KUBELET_KEY_BASE64}"
clusters:
- name: kubemark
  cluster:
    insecure-skip-tls-verify: true
    server: https://${MASTER_IP}
contexts:
- context:
    cluster: kubemark
    user: kube-proxy
  name: kubemark-context
current-context: kubemark-context
EOF
  )"

  # Create kubeconfig for Heapster.
  HEAPSTER_KUBECONFIG_CONTENTS="$(cat <<EOF
apiVersion: v1
kind: Config
users:
- name: heapster
  user:
    client-certificate-data: "${KUBELET_CERT_BASE64}"
    client-key-data: "${KUBELET_KEY_BASE64}"
clusters:
- name: kubemark
  cluster:
    insecure-skip-tls-verify: true
    server: https://${MASTER_IP}
contexts:
- context:
    cluster: kubemark
    user: heapster
  name: kubemark-context
current-context: kubemark-context
EOF
)"

  # Create kubeconfig for Cluster Autoscaler.
  CLUSTER_AUTOSCALER_KUBECONFIG_CONTENTS="$(cat <<EOF
apiVersion: v1
kind: Config
users:
- name: cluster-autoscaler
  user:
    client-certificate-data: "${KUBELET_CERT_BASE64}"
    client-key-data: "${KUBELET_KEY_BASE64}"
clusters:
- name: kubemark
  cluster:
    insecure-skip-tls-verify: true
    server: https://${MASTER_IP}
contexts:
- context:
    cluster: kubemark
    user: cluster-autoscaler
  name: kubemark-context
current-context: kubemark-context
EOF
)"

  # Create kubeconfig for NodeProblemDetector.
  NPD_KUBECONFIG_CONTENTS="$(cat <<EOF
apiVersion: v1
kind: Config
users:
- name: node-problem-detector
  user:
    client-certificate-data: "${KUBELET_CERT_BASE64}"
    client-key-data: "${KUBELET_KEY_BASE64}"
clusters:
- name: kubemark
  cluster:
    insecure-skip-tls-verify: true
    server: https://${MASTER_IP}
contexts:
- context:
    cluster: kubemark
    user: node-problem-detector
  name: kubemark-context
current-context: kubemark-context
EOF
)"

  # Create kubeconfig for Kube DNS.
  KUBE_DNS_KUBECONFIG_CONTENTS="$(cat <<EOF
apiVersion: v1
kind: Config
users:
- name: kube-dns
  user:
    client-certificate-data: "${KUBELET_CERT_BASE64}"
    client-key-data: "${KUBELET_KEY_BASE64}"
clusters:
- name: kubemark
  cluster:
    insecure-skip-tls-verify: true
    server: https://${MASTER_IP}
contexts:
- context:
    cluster: kubemark
    user: kube-dns
  name: kubemark-context
current-context: kubemark-context
EOF
)"

  # Create kubemark namespace.
  spawn-config
  if kubectl get ns | grep -Fq "kubemark"; then
  	 kubectl delete ns kubemark
  	 while kubectl get ns | grep -Fq "kubemark"
  	 do
  	 	sleep 10
  	 done
  fi
  "${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/kubemark-ns.json"
  # Create configmap for configuring hollow- kubelet, proxy and npd.
  "${KUBECTL}" create configmap "node-configmap" --namespace="kubemark" \
    --from-file=kernel.monitor="${RESOURCE_DIRECTORY}/kernel-monitor.json"

  # Create secret for passing kubeconfigs to kubelet, kubeproxy and npd.
  "${KUBECTL}" create secret generic "kubeconfig" --type=Opaque --namespace="kubemark" \
    --from-literal=kubelet.kubeconfig="${KUBELET_KUBECONFIG_CONTENTS}" \
    --from-literal=kubeproxy.kubeconfig="${KUBEPROXY_KUBECONFIG_CONTENTS}" \
    --from-literal=heapster.kubeconfig="${HEAPSTER_KUBECONFIG_CONTENTS}" \
    --from-literal=cluster_autoscaler.kubeconfig="${CLUSTER_AUTOSCALER_KUBECONFIG_CONTENTS}" \
    --from-literal=npd.kubeconfig="${NPD_KUBECONFIG_CONTENTS}" \
    --from-literal=dns.kubeconfig="${KUBE_DNS_KUBECONFIG_CONTENTS}"

  # Create addon pods.
  # Heapster.
  mkdir -p "${RESOURCE_DIRECTORY}/addons"
  sed "s/{{MASTER_IP}}/${MASTER_IP}/g" "${RESOURCE_DIRECTORY}/heapster_template.json" > "${RESOURCE_DIRECTORY}/addons/heapster.json"
  metrics_mem_per_node=4
  metrics_mem=$((200 + metrics_mem_per_node*NUM_NODES))
  sed -i'' -e "s/{{METRICS_MEM}}/${metrics_mem}/g" "${RESOURCE_DIRECTORY}/addons/heapster.json"
  metrics_cpu_per_node_numerator=${NUM_NODES}
  metrics_cpu_per_node_denominator=2
  metrics_cpu=$((80 + metrics_cpu_per_node_numerator / metrics_cpu_per_node_denominator))
  sed -i'' -e "s/{{METRICS_CPU}}/${metrics_cpu}/g" "${RESOURCE_DIRECTORY}/addons/heapster.json"
  eventer_mem_per_node=500
  eventer_mem=$((200 * 1024 + eventer_mem_per_node*NUM_NODES))
  sed -i'' -e "s/{{EVENTER_MEM}}/${eventer_mem}/g" "${RESOURCE_DIRECTORY}/addons/heapster.json"

  # Cluster Autoscaler.
  if [[ "${ENABLE_KUBEMARK_CLUSTER_AUTOSCALER:-}" == "true" ]]; then
    echo "Setting up Cluster Autoscaler"
    KUBEMARK_AUTOSCALER_MIG_NAME="${KUBEMARK_AUTOSCALER_MIG_NAME:-${NODE_INSTANCE_PREFIX}-group}"
    KUBEMARK_AUTOSCALER_MIN_NODES="${KUBEMARK_AUTOSCALER_MIN_NODES:-0}"
    KUBEMARK_AUTOSCALER_MAX_NODES="${KUBEMARK_AUTOSCALER_MAX_NODES:-${DESIRED_NODES}}"
    NUM_NODES=${KUBEMARK_AUTOSCALER_MAX_NODES}
    echo "Setting maximum cluster size to ${NUM_NODES}."
    KUBEMARK_MIG_CONFIG="autoscaling.k8s.io/nodegroup: ${KUBEMARK_AUTOSCALER_MIG_NAME}"
    sed "s/{{master_ip}}/${MASTER_IP}/g" "${RESOURCE_DIRECTORY}/cluster-autoscaler_template.json" > "${RESOURCE_DIRECTORY}/addons/cluster-autoscaler.json"
    sed -i'' -e "s/{{kubemark_autoscaler_mig_name}}/${KUBEMARK_AUTOSCALER_MIG_NAME}/g" "${RESOURCE_DIRECTORY}/addons/cluster-autoscaler.json"
    sed -i'' -e "s/{{kubemark_autoscaler_min_nodes}}/${KUBEMARK_AUTOSCALER_MIN_NODES}/g" "${RESOURCE_DIRECTORY}/addons/cluster-autoscaler.json"
    sed -i'' -e "s/{{kubemark_autoscaler_max_nodes}}/${KUBEMARK_AUTOSCALER_MAX_NODES}/g" "${RESOURCE_DIRECTORY}/addons/cluster-autoscaler.json"
  fi

  # Kube DNS.
  if [[ "${ENABLE_KUBEMARK_KUBE_DNS:-}" == "true" ]]; then
    echo "Setting up kube-dns"
    sed "s/{{dns_domain}}/${KUBE_DNS_DOMAIN}/g" "${RESOURCE_DIRECTORY}/kube_dns_template.yaml" > "${RESOURCE_DIRECTORY}/addons/kube_dns.yaml"
  fi

  "${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/addons" --namespace="kubemark"
  set-registry-secrets

  # Create the replication controller for hollow-nodes.
  # We allow to override the NUM_REPLICAS when running Cluster Autoscaler.
  NUM_REPLICAS=${NUM_REPLICAS:-${KUBEMARK_NUM_NODES}}
  sed "s/{{numreplicas}}/${NUM_REPLICAS}/g" "${RESOURCE_DIRECTORY}/hollow-node_template.yaml" > "${RESOURCE_DIRECTORY}/hollow-node.yaml"
  proxy_cpu=20
  if [ "${NUM_NODES}" -gt 1000 ]; then
    proxy_cpu=50
  fi
  proxy_mem_per_node=50
  proxy_mem=$((100 * 1024 + proxy_mem_per_node*NUM_NODES))
  hollow_kubelet_params=$(eval "for param in ${HOLLOW_KUBELET_TEST_ARGS:-}; do echo -n \\\"\$param\\\",; done")
  hollow_kubelet_params=${hollow_kubelet_params%?}
  hollow_proxy_params=$(eval "for param in ${HOLLOW_PROXY_TEST_ARGS:-}; do echo -n \\\"\$param\\\",; done")
  hollow_proxy_params=${hollow_proxy_params%?}

  sed -i'' -e "s/{{HOLLOW_PROXY_CPU}}/${proxy_cpu}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
  sed -i'' -e "s/{{HOLLOW_PROXY_MEM}}/${proxy_mem}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
  sed -i'' -e "s'{{kubemark_image_registry}}'${KUBEMARK_IMAGE_REGISTRY}${KUBE_NAMESPACE}'g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
  sed -i'' -e "s/{{kubemark_image_tag}}/${KUBEMARK_IMAGE_TAG}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
  sed -i'' -e "s/{{master_ip}}/${MASTER_IP}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
  sed -i'' -e "s/{{hollow_kubelet_params}}/${hollow_kubelet_params}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
  sed -i'' -e "s/{{hollow_proxy_params}}/${hollow_proxy_params}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
  sed -i'' -e "s'{{kubemark_mig_config}}'${KUBEMARK_MIG_CONFIG:-}'g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
  "${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/hollow-node.yaml" --namespace="kubemark"

  echo "Created secrets, configMaps, replication-controllers required for hollow-nodes."
}

# Wait until all hollow-nodes are running or there is a timeout.
function wait-for-hollow-nodes-to-run-or-timeout {
  echo -n "Waiting for all hollow-nodes to become Running"
  start=$(date +%s)
  # IKS uses a real cluster for hollow master, so need to exclude the real worker nodes
  nodes=$("${KUBECTL}" --kubeconfig="${KUBECONFIG}" get node | grep hollow-node 2> /dev/null) || true
  ready=$(($(echo "${nodes}" | grep -vc "NotReady") - 1))
  until [[ "${ready}" -ge "${NUM_REPLICAS}" ]]; do
    echo -n "."
    sleep 1
    now=$(date +%s)
    # Fail it if it already took more than 30 minutes.
    if [ $((now - start)) -gt 1800 ]; then
      echo ""
      # shellcheck disable=SC2154 # Color defined in sourced script
      echo -e "${color_red} Timeout waiting for all hollow-nodes to become Running. ${color_norm}"
      # Try listing nodes again - if it fails it means that API server is not responding
      if "${KUBECTL}" --kubeconfig="${KUBECONFIG}" get node &> /dev/null; then
        echo "Found only ${ready} ready hollow-nodes while waiting for ${NUM_NODES}."
      else
        echo "Got error while trying to list hollow-nodes. Probably API server is down."
      fi
      spawn-config
      pods=$("${KUBECTL}" get pods -l name=hollow-node --namespace=kubemark) || true
      running=$(($(echo "${pods}" | grep -c "Running")))
      echo "${running} hollow-nodes are reported as 'Running'"
      not_running=$(($(echo "${pods}" | grep -vc "Running") - 1))
      echo "${not_running} hollow-nodes are reported as NOT 'Running'"
      echo "${pods}" | grep -v "Running"
      exit 1
    fi
    nodes=$("${KUBECTL}" --kubeconfig="${KUBECONFIG}" get node 2> /dev/null) || true
    ready=$(($(echo "${nodes}" | grep -vc "NotReady") - 1))
  done
  # shellcheck disable=SC2154 # Color defined in sourced script
  echo -e "${color_green} Done!${color_norm}"
}

############################### Main Function ########################################
# In order for the cluster autoscalar to function, the template file must be changed so that the ":443"
# is removed. This is because the port is already given with the MASTER_IP.

# Create clusters and populate with hollow nodes
complete-login
build-kubemark-image
choose-clusters
generate-values
set-hollow-master
echo "Creating kube hollow node resources"
create-kube-hollow-node-resources
master-config
# shellcheck disable=SC2154 # Color defined in sourced script
echo -e "${color_blue}EXECUTION COMPLETE${color_norm}"

# Check status of Kubemark
# shellcheck disable=SC2154 # Color defined in sourced script
echo -e "${color_yellow}CHECKING STATUS${color_norm}"
wait-for-hollow-nodes-to-run-or-timeout

# Celebrate
echo ""
# shellcheck disable=SC2154 # Color defined in sourced script
echo -e "${color_blue}SUCCESS${color_norm}"
clean-repo
exit 0


#!/usr/bin/env bash

# Copyright 2018 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../../..

# Creates a new kube-spawn cluster
function create-clusters {
	# shellcheck disable=SC2154	# Color defined in sourced script
	echo -e "${color_yellow}CHECKING CLUSTERS${color_norm}"
	if ibmcloud ks clusters | grep -Fq 'deleting'; then
		echo -n "Deleting old clusters"
	fi
	while ibmcloud ks clusters | grep -Fq 'deleting'
	do
		echo -n "."
		sleep 10
	done
	echo ""
	ibmcloud ks region-set us-east >/dev/null
	ibmcloud ks vlans wdc06 >/dev/null
	PRIVLAN=$(ibmcloud ks vlans wdc06 --json | jq '. | .[] | select(.type == "private") | .id' | sed -e "s/\"//g")
	PUBVLAN=$(ibmcloud ks vlans wdc06 --json | jq '. | .[] | select(.type == "public") | .id' | sed -e "s/\"//g")
	if ! ibmcloud ks clusters | grep -Fq 'kubeSpawnTester'; then
		echo "Creating spawning cluster"
		# make number and spec of node workers configurable
		# otherwise it can't afford tests like kubemark-5000
		# TODO: dynamically adjust the number and spec
		ibmcloud ks cluster-create --location "${CLUSTER_LOCATION}" --public-vlan "${PUBVLAN}" --private-vlan "${PRIVLAN}" --workers "${NUM_NODES:-2}" --machine-type "${NODE_SIZE}" --name kubeSpawnTester
	fi
	if ! ibmcloud ks clusters | grep -Fq 'kubeMasterTester'; then
		echo "Creating master cluster"
		# if we can't make it a bare master (workers = 0)
		# then make workers = 1 with the smallest machine spec
		ibmcloud ks cluster-create --location "${CLUSTER_LOCATION}" --public-vlan "${PUBVLAN}" --private-vlan "${PRIVLAN}" --workers 1 --machine-type u2c.2x4 --name kubeMasterTester
	fi
	push-image
	if ! ibmcloud ks clusters | grep 'kubeSpawnTester' | grep -Fq 'normal'; then
		# shellcheck disable=SC2154 # Color defined in sourced script
		echo -e "${color_cyan}Warning: new clusters may take up to 60 minutes to be ready${color_norm}"
		echo -n "Clusters loading"
	fi
	while ! ibmcloud ks clusters | grep 'kubeSpawnTester' | grep -Fq 'normal'
	do
		echo -n "."
		sleep 5
	done
	while ! ibmcloud ks clusters | grep 'kubeMasterTester' | grep -Fq 'normal'
	do
		echo -n "."
		sleep 5
	done
	echo -e "${color_yellow}CLUSTER CREATION COMPLETE${color_norm}"
}

# Builds and pushes image to registry
function push-image {
	if [[ "${ISBUILD}" = "y" ]]; then
		if ! ibmcloud cr namespaces | grep -Fq "${KUBE_NAMESPACE}"; then
			echo "Creating registry namespace"
			ibmcloud cr namespace-add "${KUBE_NAMESPACE}"
			echo "ibmcloud cr namespace-rm ${KUBE_NAMESPACE}" >> "${RESOURCE_DIRECTORY}/iks-namespacelist.sh"
		fi
		docker build -t "${KUBEMARK_INIT_TAG}" "${KUBEMARK_IMAGE_LOCATION}"
		docker tag "${KUBEMARK_INIT_TAG}" "${KUBEMARK_IMAGE_REGISTRY}${KUBE_NAMESPACE}/${PROJECT}:${KUBEMARK_IMAGE_TAG}"
		docker push "${KUBEMARK_IMAGE_REGISTRY}${KUBE_NAMESPACE}/${PROJECT}:${KUBEMARK_IMAGE_TAG}"
		echo "Image pushed"
	else
		KUBEMARK_IMAGE_REGISTRY="${KUBEMARK_IMAGE_REGISTRY:-brandondr96}"
		KUBE_NAMESPACE=""
	fi
}

# Allow user to use existing clusters if desired
function choose-clusters {
	echo -n -e "Do you want to use custom clusters? [y/N]${color_cyan}>${color_norm} "
	read -r USE_EXISTING
	if [[ "${USE_EXISTING}" = "y" ]]; then
		echo -e "${color_yellow}Enter path for desired hollow-node spawning cluster kubeconfig file:${color_norm}"
		read -r CUSTOM_SPAWN_CONFIG
		echo -e "${color_yellow}Enter path for desired hollow-node hosting cluster kubeconfig file:${color_norm}"
		read -r CUSTOM_MASTER_CONFIG
		push-image
	elif [[ "${USE_EXISTING}" = "N" ]]; then
		create-clusters
	else
		# shellcheck disable=SC2154 # Color defined in sourced script
		echo -e "${color_red}Invalid response, please try again:${color_norm}"
		choose-clusters
	fi
}

# Ensure secrets are correctly set
function set-registry-secrets {
	spawn-config
	kubectl get secret bluemix-default-secret-regional -o yaml | sed 's/default/kubemark/g' | kubectl -n kubemark create -f -
	kubectl patch serviceaccount -n kubemark default -p '{"imagePullSecrets": [{"name": "bluemix-kubemark-secret"}]}'
	kubectl -n kubemark get serviceaccounts default -o json | jq 'del(.metadata.resourceVersion)' | jq 'setpath(["imagePullSecrets"];[{"name":"bluemix-kubemark-secret-regional"}])' | kubectl -n kubemark replace serviceaccount default -f -
}

# Sets the hollow-node master
# Exported variables:
#   MASTER_IP - IP Address of the Kubemark master
function set-hollow-master {
	echo -e "${color_yellow}CONFIGURING MASTER${color_norm}"
	master-config
	MASTER_IP=$(grep server "$KUBECONFIG" | awk -F "/" '{print $3}')
	export MASTER_IP
}

# Set up master cluster environment
# Exported variables:
#   KUBECONFIG - Overrides default kube config for the purpose of setting up the Kubemark master components.
function master-config {
	if [[ "${USE_EXISTING}" = "y" ]]; then
		export KUBECONFIG=${CUSTOM_MASTER_CONFIG}
	else
		eval "$(ibmcloud ks cluster-config kubeMasterTester --admin | grep export)"
	fi
}

# Set up spawn cluster environment
# Exported variables:
#    KUBECONFIG - Overrides default kube config for the purpose of setting up the hollow-node cluster.
function spawn-config {
	if [[ "${USE_EXISTING}" = "y" ]]; then
		export KUBECONFIG=${CUSTOM_SPAWN_CONFIG}
	else
		eval "$(ibmcloud ks cluster-config kubeSpawnTester --admin | grep export)"
	fi
}

# Deletes existing clusters
function delete-clusters {
	echo "DELETING CLUSTERS"
	ibmcloud ks cluster-rm kubeSpawnTester
	ibmcloud ks cluster-rm kubeMasterTester
	while ! ibmcloud ks clusters | grep 'kubeSpawnTester' | grep -Fq 'deleting'
	do
		sleep 5
	done
	while ! ibmcloud ks clusters | grep 'kubeMasterTester' | grep -Fq 'deleting'
	do
		sleep 5
	done
	kubectl delete ns kubemark
}

# Login to cloud services
function complete-login {
	echo -e "${color_yellow}LOGGING INTO CLOUD SERVICES${color_norm}"
	echo -n -e "Do you have a federated IBM cloud login? [y/N]${color_cyan}>${color_norm} "
	read -r ISFED
	if [[ "${ISFED}" = "y" ]]; then
		ibmcloud login --sso -a "${REGISTRY_LOGIN_URL}"
	elif [[ "${ISFED}" = "N" ]]; then
		ibmcloud login -a "${REGISTRY_LOGIN_URL}"
	else
		echo -e "${color_red}Invalid response, please try again:${color_norm}"
		complete-login
	fi
	ibmcloud cr login
}

# Generate values to fill the hollow-node configuration templates.
# Exported variables:
#   KUBECTL - The name or path to the kubernetes client binary.
function generate-values {
	echo "Generating values"
	master-config
	KUBECTL=kubectl
	export KUBECTL
	KUBEMARK_DIRECTORY="${KUBE_ROOT}/test/kubemark"
	RESOURCE_DIRECTORY="${KUBEMARK_DIRECTORY}/resources"
	CONFIGPATH=${KUBECONFIG%/*}
	KUBELET_CERT_BASE64="${KUBELET_CERT_BASE64:-$(base64 "${CONFIGPATH}/admin.pem" | tr -d '\r\n')}"
	KUBELET_KEY_BASE64="${KUBELET_KEY_BASE64:-$(base64 "${CONFIGPATH}/admin-key.pem" | tr -d '\r\n')}"
	CA_CERT_BASE64="${CA_CERT_BASE64:-$( base64 "$(find "${CONFIGPATH}" -name "*ca*" | head -n 1)" | tr -d '\r\n')}"

}

# Build image for kubemark
function build-kubemark-image {
	echo -n -e "Do you want to build the kubemark image? [y/N]${color_cyan}>${color_norm} "
	read -r ISBUILD
	if [[ "${ISBUILD}" = "y" ]]; then
		echo -e "${color_yellow}BUILDING IMAGE${color_norm}"
		"${KUBE_ROOT}/build/run.sh" make kubemark
		cp "${KUBE_ROOT}/_output/dockerized/bin/linux/amd64/kubemark" "${KUBEMARK_IMAGE_LOCATION}"
	elif [[ "${ISBUILD}" = "N" ]]; then
		echo -n ""
	else
		echo -e "${color_red}Invalid response, please try again:${color_norm}"
		build-kubemark-image
	fi
}

# Clean up repository
function clean-repo {
	echo -n -e "Do you want to remove build output and binary? [y/N]${color_cyan}>${color_norm} "
	read -r ISCLEAN
	if [[ "${ISCLEAN}" = "y" ]]; then
		echo -e "${color_yellow}CLEANING REPO${color_norm}"
		rm -rf "${KUBE_ROOT}/_output"
		rm -f "${KUBEMARK_IMAGE_LOCATION}/kubemark"
	elif [[ "${ISCLEAN}" = "N" ]]; then
		echo -n ""
	else
		echo -e "${color_red}Invalid response, please try again:${color_norm}"
		clean-repo
	fi
}


#!/usr/bin/env bash

# Copyright 2017 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../../..

source "${KUBE_ROOT}/test/kubemark/common/util.sh"

# Leave the skeleton definition of execute-cmd-on-master-with-retries
# so only the pre-existing provider functions will target this.
function execute-cmd-on-pre-existing-master-with-retries() {
  IP_WITHOUT_PORT=$(echo "${MASTER_IP}" | cut -f 1 -d ':') || "${MASTER_IP}"

  RETRIES="${2:-1}" run-cmd-with-retries ssh kubernetes@"${IP_WITHOUT_PORT}" "${1}"
}


#!/usr/bin/env bash

# Copyright 2015 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Script that starts kubelet on kubemark-master as a supervisord process
# and then runs the master components as pods using kubelet.

set -o errexit
set -o nounset
set -o pipefail

# Define key path variables.
KUBE_ROOT="/home/kubernetes"
KUBE_BINDIR="${KUBE_ROOT}/kubernetes/server/bin"

function config-ip-firewall {
  echo "Configuring IP firewall rules"
  # The GCI image has host firewall which drop most inbound/forwarded packets.
  # We need to add rules to accept all TCP/UDP/ICMP packets.
  if iptables -L INPUT | grep "Chain INPUT (policy DROP)" > /dev/null; then
    echo "Add rules to accept all inbound TCP/UDP/ICMP packets"
    iptables -A INPUT -w -p TCP -j ACCEPT
    iptables -A INPUT -w -p UDP -j ACCEPT
    iptables -A INPUT -w -p ICMP -j ACCEPT
  fi
  if iptables -L FORWARD | grep "Chain FORWARD (policy DROP)" > /dev/null; then
    echo "Add rules to accept all forwarded TCP/UDP/ICMP packets"
    iptables -A FORWARD -w -p TCP -j ACCEPT
    iptables -A FORWARD -w -p UDP -j ACCEPT
    iptables -A FORWARD -w -p ICMP -j ACCEPT
  fi
}

function create-dirs {
	echo "Creating required directories"
	mkdir -p /var/lib/kubelet
	mkdir -p /etc/kubernetes/manifests
	mkdir -p /etc/kubernetes/addons
}

# Setup working directory for kubelet.
function setup-kubelet-dir {
	echo "Making /var/lib/kubelet executable for kubelet"
	mount -B /var/lib/kubelet /var/lib/kubelet/
	mount -B -o remount,exec,suid,dev /var/lib/kubelet
}

# Remove any default etcd config dirs/files.
function delete-default-etcd-configs {
	if [[ -d /etc/etcd ]]; then
		rm -rf /etc/etcd
	fi
	if [[ -e /etc/default/etcd ]]; then
		rm -f /etc/default/etcd
	fi
	if [[ -e /etc/systemd/system/etcd.service ]]; then
		rm -f /etc/systemd/system/etcd.service
	fi
	if [[ -e /etc/init.d/etcd ]]; then
		rm -f /etc/init.d/etcd
	fi
}

# Compute etcd related variables.
function compute-etcd-variables {
	ETCD_IMAGE="${ETCD_IMAGE:-}"
	ETCD_QUOTA_BYTES=""
	if [ "${ETCD_VERSION:0:2}" == "3." ]; then
		# TODO: Set larger quota to see if that helps with
		# 'mvcc: database space exceeded' errors. If so, pipe
		# though our setup scripts.
		ETCD_QUOTA_BYTES=" --quota-backend-bytes=4294967296 "
	fi
}

# Formats the given device ($1) if needed and mounts it at given mount point
# ($2).
function safe-format-and-mount() {
	device=$1
	mountpoint=$2

	# Format only if the disk is not already formatted.
	if ! tune2fs -l "${device}" ; then
		echo "Formatting '${device}'"
		mkfs.ext4 -F "${device}"
	fi

	echo "Mounting '${device}' at '${mountpoint}'"
	mount -o discard,defaults "${device}" "${mountpoint}"
}

# Finds a PD device with name '$1' attached to the master.
function find-attached-pd() {
	local -r pd_name=$1
	if [[ ! -e /dev/disk/by-id/${pd_name} ]]; then
		echo ""
	fi
	device_info=$(ls -l "/dev/disk/by-id/${pd_name}")
	relative_path=${device_info##* }
	echo "/dev/disk/by-id/${relative_path}"
}

# Mounts a persistent disk (formatting if needed) to store the persistent data
# on the master. safe-format-and-mount only formats an unformatted disk, and
# mkdir -p will leave a directory be if it already exists.
function mount-pd() {
	local -r pd_name=$1
	local -r mount_point=$2

	if [[ -z "${find-attached-pd ${pd_name}}" ]]; then
		echo "Can't find ${pd_name}. Skipping mount."
		return
	fi

	local -r pd_path="/dev/disk/by-id/${pd_name}"
	echo "Mounting PD '${pd_path}' at '${mount_point}'"
	# Format and mount the disk, create directories on it for all of the master's
	# persistent data, and link them to where they're used.
	mkdir -p "${mount_point}"
	safe-format-and-mount "${pd_path}" "${mount_point}"
	echo "Mounted PD '${pd_path}' at '${mount_point}'"

	# NOTE: These locations on the PD store persistent data, so to maintain
	# upgradeability, these locations should not change.  If they do, take care
	# to maintain a migration path from these locations to whatever new
	# locations.
}

# Create kubeconfig for controller-manager's service account authentication.
function create-kubecontrollermanager-kubeconfig {
	echo "Creating kube-controller-manager kubeconfig file"
	mkdir -p "${KUBE_ROOT}/k8s_auth_data/kube-controller-manager"
	cat <<EOF >"${KUBE_ROOT}/k8s_auth_data/kube-controller-manager/kubeconfig"
apiVersion: v1
kind: Config
users:
- name: kube-controller-manager
  user:
    token: ${KUBE_CONTROLLER_MANAGER_TOKEN}
clusters:
- name: local
  cluster:
    insecure-skip-tls-verify: true
    server: https://localhost:443
contexts:
- context:
    cluster: local
    user: kube-controller-manager
  name: service-account-context
current-context: service-account-context
EOF
}

function create-kubescheduler-kubeconfig {
  echo "Creating kube-scheduler kubeconfig file"
  mkdir -p "${KUBE_ROOT}/k8s_auth_data/kube-scheduler"
  cat <<EOF >"${KUBE_ROOT}/k8s_auth_data/kube-scheduler/kubeconfig"
apiVersion: v1
kind: Config
users:
- name: kube-scheduler
  user:
    token: ${KUBE_SCHEDULER_TOKEN}
clusters:
- name: local
  cluster:
    insecure-skip-tls-verify: true
    server: https://localhost:443
contexts:
- context:
    cluster: local
    user: kube-scheduler
  name: kube-scheduler
current-context: kube-scheduler
EOF
}

function create-addonmanager-kubeconfig {
  echo "Creating addonmanager kubeconfig file"
  mkdir -p "${KUBE_ROOT}/k8s_auth_data/addon-manager"
  cat <<EOF >"${KUBE_ROOT}/k8s_auth_data/addon-manager/kubeconfig"
apiVersion: v1
kind: Config
users:
- name: addon-manager
  user:
    token: ${ADDON_MANAGER_TOKEN}
clusters:
- name: local
  cluster:
    insecure-skip-tls-verify: true
    server: https://localhost:443
contexts:
- context:
    cluster: local
    user: addon-manager
  name: addon-manager
current-context: addon-manager
EOF
}

function assemble-docker-flags {
	echo "Assemble docker command line flags"
	local docker_opts="-p /var/run/docker.pid --iptables=false --ip-masq=false"
	docker_opts+=" --log-level=debug"  # Since it's a test cluster
	# TODO(shyamjvs): Incorporate network plugin options, etc later.
	echo "DOCKER_OPTS=\"${docker_opts}\"" > /etc/default/docker
	echo "DOCKER_NOFILE=65536" >> /etc/default/docker  # For setting ulimit -n
	systemctl restart docker
}

# A helper function for loading a docker image. It keeps trying up to 5 times.
#
# $1: Full path of the docker image
function try-load-docker-image {
	local -r img=$1
	echo "Try to load docker image file ${img}"
	# Temporarily turn off errexit, because we don't want to exit on first failure.
	set +e
	local -r max_attempts=5
	local -i attempt_num=1
	until timeout 30 docker load -i "${img}"; do
		if [[ "${attempt_num}" == "${max_attempts}" ]]; then
			echo "Fail to load docker image file ${img} after ${max_attempts} retries. Exit!!"
			exit 1
		else
			attempt_num=$((attempt_num+1))
			sleep 5
		fi
	done
	# Re-enable errexit.
	set -e
}

# Loads kube-system docker images. It is better to do it before starting kubelet,
# as kubelet will restart docker daemon, which may interfere with loading images.
function load-docker-images {
	echo "Start loading kube-system docker images"
	local -r img_dir="${KUBE_BINDIR}"
	try-load-docker-image "${img_dir}/kube-apiserver.tar"
	try-load-docker-image "${img_dir}/kube-controller-manager.tar"
	try-load-docker-image "${img_dir}/kube-scheduler.tar"
}

# Computes command line arguments to be passed to kubelet.
function compute-kubelet-params {
	local params="${KUBELET_TEST_ARGS:-}"
	params+=" --cgroup-root=/"
	params+=" --cloud-provider=gce"
	params+=" --pod-manifest-path=/etc/kubernetes/manifests"
	if [[ -n "${KUBELET_PORT:-}" ]]; then
		params+=" --port=${KUBELET_PORT}"
	fi
	params+=" --enable-debugging-handlers=false"
	params+=" --hairpin-mode=none"
	echo "${params}"
}

# Creates the systemd config file for kubelet.service.
function create-kubelet-conf() {
	local -r kubelet_bin="$1"
	local -r kubelet_env_file="/etc/default/kubelet"
	local -r flags=$(compute-kubelet-params)
	echo "KUBELET_OPTS=\"${flags}\"" > "${kubelet_env_file}"

	# Write the systemd service file for kubelet.
	cat <<EOF >/etc/systemd/system/kubelet.service
[Unit]
Description=Kubermark kubelet
Requires=network-online.target
After=network-online.target

[Service]
Restart=always
RestartSec=10
EnvironmentFile=${kubelet_env_file}
ExecStart=${kubelet_bin} \$KUBELET_OPTS

[Install]
WantedBy=multi-user.target
EOF
}

# This function assembles the kubelet systemd service file and starts it using
# systemctl, on the kubemark master.
function start-kubelet {
	# Create systemd config.
	local -r kubelet_bin="/usr/bin/kubelet"
	create-kubelet-conf "${kubelet_bin}"

	# Flush iptables nat table
  	iptables -t nat -F || true

	# Start the kubelet service.
	systemctl start kubelet.service
}

# Create the log file and set its properties.
#
# $1 is the file to create.
function prepare-log-file {
	touch "$1"
	chmod 644 "$1"
	chown root:root "$1"
}

# A helper function for copying addon manifests and set dir/files
# permissions.
#
# $1: addon category under /etc/kubernetes
# $2: manifest source dir
function setup-addon-manifests {
  local -r src_dir="${KUBE_ROOT}/$2"
  local -r dst_dir="/etc/kubernetes/$1/$2"

  if [[ ! -d "${dst_dir}" ]]; then
    mkdir -p "${dst_dir}"
  fi

  local files
  files=$(find "${src_dir}" -maxdepth 1 -name "*.yaml")
  if [[ -n "${files}" ]]; then
    cp "${src_dir}/"*.yaml "${dst_dir}"
  fi
  chown -R root:root "${dst_dir}"
  chmod 755 "${dst_dir}"
  chmod 644 "${dst_dir}"/*
}

# Write the config for the audit policy.
# Note: This duplicates the function in cluster/gce/gci/configure-helper.sh.
# TODO: Get rid of this function when #53321 is fixed.
function create-master-audit-policy {
  local -r path="${1}"
  local -r policy="${2:-}"

  if [[ -n "${policy}" ]]; then
    echo "${policy}" > "${path}"
    return
  fi

  # Known api groups
  local -r known_apis='
      - group: "" # core
      - group: "admissionregistration.k8s.io"
      - group: "apiextensions.k8s.io"
      - group: "apiregistration.k8s.io"
      - group: "apps"
      - group: "authentication.k8s.io"
      - group: "authorization.k8s.io"
      - group: "autoscaling"
      - group: "batch"
      - group: "certificates.k8s.io"
      - group: "extensions"
      - group: "metrics"
      - group: "networking.k8s.io"
      - group: "policy"
      - group: "rbac.authorization.k8s.io"
      - group: "storage.k8s.io"'

  cat <<EOF >"${path}"
apiVersion: audit.k8s.io/v1
kind: Policy
rules:
  # The following requests were manually identified as high-volume and low-risk,
  # so drop them.
  - level: None
    users: ["system:kube-proxy"]
    verbs: ["watch"]
    resources:
      - group: "" # core
        resources: ["endpoints", "services", "services/status"]
  - level: None
    # Ingress controller reads 'configmaps/ingress-uid' through the unsecured port.
    # TODO(#46983): Change this to the ingress controller service account.
    users: ["system:unsecured"]
    namespaces: ["kube-system"]
    verbs: ["get"]
    resources:
      - group: "" # core
        resources: ["configmaps"]
  - level: None
    users: ["kubelet"] # legacy kubelet identity
    verbs: ["get"]
    resources:
      - group: "" # core
        resources: ["nodes", "nodes/status"]
  - level: None
    userGroups: ["system:nodes"]
    verbs: ["get"]
    resources:
      - group: "" # core
        resources: ["nodes", "nodes/status"]
  - level: None
    users:
      - system:kube-controller-manager
      - system:kube-scheduler
      - system:serviceaccount:kube-system:endpoint-controller
    verbs: ["get", "update"]
    namespaces: ["kube-system"]
    resources:
      - group: "" # core
        resources: ["endpoints"]
  - level: None
    users: ["system:apiserver"]
    verbs: ["get"]
    resources:
      - group: "" # core
        resources: ["namespaces", "namespaces/status", "namespaces/finalize"]
  # Don't log HPA fetching metrics.
  - level: None
    users:
      - system:kube-controller-manager
    verbs: ["get", "list"]
    resources:
      - group: "metrics"
  # Don't log these read-only URLs.
  - level: None
    nonResourceURLs:
      - /healthz*
      - /version
      - /swagger*
  # Don't log events requests.
  - level: None
    resources:
      - group: "" # core
        resources: ["events"]
  # node and pod status calls from nodes are high-volume and can be large, don't log responses for expected updates from nodes
  - level: Request
    users: ["kubelet", "system:node-problem-detector", "system:serviceaccount:kube-system:node-problem-detector"]
    verbs: ["update","patch"]
    resources:
      - group: "" # core
        resources: ["nodes/status", "pods/status"]
    omitStages:
      - "RequestReceived"
  - level: Request
    userGroups: ["system:nodes"]
    verbs: ["update","patch"]
    resources:
      - group: "" # core
        resources: ["nodes/status", "pods/status"]
    omitStages:
      - "RequestReceived"
  # deletecollection calls can be large, don't log responses for expected namespace deletions
  - level: Request
    users: ["system:serviceaccount:kube-system:namespace-controller"]
    verbs: ["deletecollection"]
    omitStages:
      - "RequestReceived"
  # Secrets, ConfigMaps, TokenRequest and TokenReviews can contain sensitive & binary data,
  # so only log at the Metadata level.
  - level: Metadata
    resources:
      - group: "" # core
        resources: ["secrets", "configmaps", "serviceaccounts/token"]
      - group: authentication.k8s.io
        resources: ["tokenreviews"]
    omitStages:
      - "RequestReceived"
  # Get responses can be large; skip them.
  - level: Request
    verbs: ["get", "list", "watch"]
    resources: ${known_apis}
    omitStages:
      - "RequestReceived"
  # Default level for known APIs
  - level: RequestResponse
    resources: ${known_apis}
    omitStages:
      - "RequestReceived"
  # Default level for all other requests.
  - level: Metadata
    omitStages:
      - "RequestReceived"
EOF
}

# Computes command line arguments to be passed to etcd.
function compute-etcd-params {
	local params="${ETCD_TEST_ARGS:-}"
	params+=" --name=etcd-$(hostname -s)"
	params+=" --listen-peer-urls=http://127.0.0.1:2380"
	params+=" --advertise-client-urls=http://127.0.0.1:2379"
	params+=" --listen-client-urls=http://0.0.0.0:2379"

	# Enable apiserver->etcd auth.
	params+=" --client-cert-auth"
	params+=" --trusted-ca-file /etc/srv/kubernetes/etcd-apiserver-ca.crt"
	params+=" --cert-file /etc/srv/kubernetes/etcd-apiserver-server.crt"
	params+=" --key-file /etc/srv/kubernetes/etcd-apiserver-server.key"

	params+=" --data-dir=/var/etcd/data"
	params+=" ${ETCD_QUOTA_BYTES}"
	echo "${params}"
}

# Computes command line arguments to be passed to etcd-events.
function compute-etcd-events-params {
	local params="${ETCD_TEST_ARGS:-}"
	params+=" --name=etcd-$(hostname -s)"
	params+=" --listen-peer-urls=http://127.0.0.1:2381"
	params+=" --advertise-client-urls=http://127.0.0.1:4002"
	params+=" --listen-client-urls=http://0.0.0.0:4002"
	params+=" --data-dir=/var/etcd/data-events"
	params+=" ${ETCD_QUOTA_BYTES}"
	echo "${params}"
}

# Computes command line arguments to be passed to apiserver.
function compute-kube-apiserver-params {
	local params="--insecure-bind-address=0.0.0.0"
	params+=" --etcd-servers=${ETCD_SERVERS:-http://127.0.0.1:2379}"
	if [[ -z "${ETCD_SERVERS:-}" ]]; then
		params+=" --etcd-servers-overrides=${ETCD_SERVERS_OVERRIDES:-/events#${EVENT_STORE_URL}}"
	elif [[ -n "${ETCD_SERVERS_OVERRIDES:-}" ]]; then
		params+=" --etcd-servers-overrides=${ETCD_SERVERS_OVERRIDES:-}"
	fi
	# Enable apiserver->etcd auth.
	params+=" --etcd-cafile=/etc/srv/kubernetes/etcd-apiserver-ca.crt"
	params+=" --etcd-certfile=/etc/srv/kubernetes/etcd-apiserver-client.crt"
	params+=" --etcd-keyfile=/etc/srv/kubernetes/etcd-apiserver-client.key"

	params+=" --tls-cert-file=/etc/srv/kubernetes/server.cert"
	params+=" --tls-private-key-file=/etc/srv/kubernetes/server.key"
	params+=" --requestheader-client-ca-file=/etc/srv/kubernetes/aggr_ca.crt"
	params+=" --requestheader-allowed-names=aggregator"
	params+=" --requestheader-extra-headers-prefix=X-Remote-Extra-"
	params+=" --requestheader-group-headers=X-Remote-Group"
	params+=" --requestheader-username-headers=X-Remote-User"
	params+=" --proxy-client-cert-file=/etc/srv/kubernetes/proxy_client.crt"
	params+=" --proxy-client-key-file=/etc/srv/kubernetes/proxy_client.key"
	params+=" --enable-aggregator-routing=true"
	params+=" --client-ca-file=/etc/srv/kubernetes/ca.crt"
	params+=" --token-auth-file=/etc/srv/kubernetes/known_tokens.csv"
	params+=" --secure-port=443"
	params+=" --target-ram-mb=$((NUM_NODES * 60))"
	params+=" --service-cluster-ip-range=${SERVICE_CLUSTER_IP_RANGE}"
	params+=" --admission-control=${CUSTOM_ADMISSION_PLUGINS}"
	params+=" --authorization-mode=Node,RBAC"
	params+=" --allow-privileged=true"
	if [[ -n "${STORAGE_BACKEND:-}" ]]; then
		params+=" --storage-backend=${STORAGE_BACKEND}"
	fi
	if [[ -n "${STORAGE_MEDIA_TYPE:-}" ]]; then
		params+=" --storage-media-type=${STORAGE_MEDIA_TYPE}"
	fi
  if [[ -n "${ETCD_COMPACTION_INTERVAL_SEC:-}" ]]; then
    params+=" --etcd-compaction-interval=${ETCD_COMPACTION_INTERVAL_SEC}s"
  fi
	if [[ -n "${KUBE_APISERVER_REQUEST_TIMEOUT:-}" ]]; then
		params+=" --min-request-timeout=${KUBE_APISERVER_REQUEST_TIMEOUT}"
	fi
	if [[ "${NUM_NODES}" -ge 3000 ]]; then
		params+=" --max-requests-inflight=3000 --max-mutating-requests-inflight=1000"
	elif [[ "${NUM_NODES}" -ge 1000 ]]; then
		params+=" --max-requests-inflight=1500 --max-mutating-requests-inflight=500"
	fi
	if [[ -n "${RUNTIME_CONFIG:-}" ]]; then
		params+=" --runtime-config=${RUNTIME_CONFIG}"
	fi
	if [[ -n "${FEATURE_GATES:-}" ]]; then
		params+=" --feature-gates=${FEATURE_GATES}"
	fi
	if [[ "${ENABLE_APISERVER_ADVANCED_AUDIT:-}" == "true" ]]; then
		# Create the audit policy file, and mount it into the apiserver pod.
		create-master-audit-policy "${audit_policy_file}" "${ADVANCED_AUDIT_POLICY:-}"

		# The config below matches the one in cluster/gce/gci/configure-helper.sh.
		# TODO: Currently supporting just log backend. Support webhook if needed.
		params+=" --audit-policy-file=${audit_policy_file}"
		params+=" --audit-log-path=/var/log/kube-apiserver-audit.log"
		params+=" --audit-log-maxage=0"
		params+=" --audit-log-maxbackup=0"
		params+=" --audit-log-maxsize=2000000000"
	fi
        # Append APISERVER_TEST_ARGS to the end, which will allow for
        # the above defaults to be overridden.
	params+=" ${APISERVER_TEST_ARGS:-}"
	echo "${params}"
}

# Computes command line arguments to be passed to controller-manager.
function compute-kube-controller-manager-params {
	local params="${CONTROLLER_MANAGER_TEST_ARGS:-}"
	params+=" --use-service-account-credentials"
	params+=" --kubeconfig=/etc/srv/kubernetes/kube-controller-manager/kubeconfig"
	params+=" --service-account-private-key-file=/etc/srv/kubernetes/server.key"
	params+=" --root-ca-file=/etc/srv/kubernetes/ca.crt"
	params+=" --allocate-node-cidrs=${ALLOCATE_NODE_CIDRS}"
	params+=" --cluster-cidr=${CLUSTER_IP_RANGE}"
	params+=" --service-cluster-ip-range=${SERVICE_CLUSTER_IP_RANGE}"
	params+=" --terminated-pod-gc-threshold=${TERMINATED_POD_GC_THRESHOLD}"
	echo "${params}"
}

# Computes command line arguments to be passed to scheduler.
function compute-kube-scheduler-params {
	local params="${SCHEDULER_TEST_ARGS:-}"
	params+=" --kubeconfig=/etc/srv/kubernetes/kube-scheduler/kubeconfig"
	echo "${params}"
}

# Computes command line arguments to be passed to addon-manager.
function compute-kube-addon-manager-params {
	echo ""
}

# Start a kubernetes master component '$1' which can be any of the following:
# 1. etcd
# 2. etcd-events
# 3. kube-apiserver
# 4. kube-controller-manager
# 5. kube-scheduler
# 6. kube-addon-manager
#
# It prepares the log file, loads the docker tag, calculates variables, sets them
# in the manifest file, and then copies the manifest file to /etc/kubernetes/manifests.
#
# Assumed vars:
#   DOCKER_REGISTRY
function start-kubemaster-component() {
	echo "Start master component $1"
	local -r component=$1
	prepare-log-file /var/log/"${component}".log
	local -r src_file="${KUBE_ROOT}/${component}.yaml"
	local -r params=$("compute-${component}-params")

	# Evaluate variables.
	sed -i -e "s@{{params}}@${params}@g" "${src_file}"
	sed -i -e "s@{{kube_docker_registry}}@${DOCKER_REGISTRY}@g" "${src_file}"
	sed -i -e "s@{{instance_prefix}}@${INSTANCE_PREFIX}@g" "${src_file}"
	if [ "${component:0:4}" == "etcd" ]; then
		sed -i -e "s@{{etcd_image}}@${ETCD_IMAGE}@g" "${src_file}"
	elif [ "${component}" == "kube-addon-manager" ]; then
		setup-addon-manifests "addons" "kubemark-rbac-bindings"
	else
		local -r component_docker_tag=$(cat "${KUBE_BINDIR}/${component}.docker_tag")
		sed -i -e "s@{{${component}_docker_tag}}@${component_docker_tag}@g" "${src_file}"
		if [ "${component}" == "kube-apiserver" ]; then
			local audit_policy_config_mount=""
			local audit_policy_config_volume=""
			if [[ "${ENABLE_APISERVER_ADVANCED_AUDIT:-}" == "true" ]]; then
				read -r -d '' audit_policy_config_mount << EOF
- name: auditpolicyconfigmount
  mountPath: ${audit_policy_file}
  readOnly: true
EOF
				read -r -d '' audit_policy_config_volume << EOF
- name: auditpolicyconfigmount
  hostPath:
    path: ${audit_policy_file}
    type: FileOrCreate
EOF
			fi
			sed -i -e "s@{{audit_policy_config_mount}}@${audit_policy_config_mount}@g" "${src_file}"
			sed -i -e "s@{{audit_policy_config_volume}}@${audit_policy_config_volume}@g" "${src_file}"
		fi
	fi
	cp "${src_file}" /etc/kubernetes/manifests
}

############################### Main Function ########################################
echo "Start to configure master instance for kubemark"

# Extract files from the server tar and setup master env variables.
cd "${KUBE_ROOT}"
if [[ ! -d "${KUBE_ROOT}/kubernetes" ]]; then
	tar xzf kubernetes-server-linux-amd64.tar.gz
fi
source "${KUBE_ROOT}/kubemark-master-env.sh"

# Setup IP firewall rules, required directory structure and etcd config.
config-ip-firewall
create-dirs
setup-kubelet-dir
delete-default-etcd-configs
compute-etcd-variables

# Setup authentication tokens and kubeconfigs for kube-controller-manager and kube-scheduler,
# only if their kubeconfigs don't already exist as this script could be running on reboot.
if [[ ! -f "${KUBE_ROOT}/k8s_auth_data/kube-controller-manager/kubeconfig" ]]; then
	KUBE_CONTROLLER_MANAGER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
	echo "${KUBE_CONTROLLER_MANAGER_TOKEN},system:kube-controller-manager,uid:system:kube-controller-manager" >> "${KUBE_ROOT}/k8s_auth_data/known_tokens.csv"
	create-kubecontrollermanager-kubeconfig
fi
if [[ ! -f "${KUBE_ROOT}/k8s_auth_data/kube-scheduler/kubeconfig" ]]; then
	KUBE_SCHEDULER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
	echo "${KUBE_SCHEDULER_TOKEN},system:kube-scheduler,uid:system:kube-scheduler" >> "${KUBE_ROOT}/k8s_auth_data/known_tokens.csv"
	create-kubescheduler-kubeconfig
fi

ADDON_MANAGER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
echo "${ADDON_MANAGER_TOKEN},system:addon-manager,admin,system:masters" >> "${KUBE_ROOT}/k8s_auth_data/known_tokens.csv"
create-addonmanager-kubeconfig

# Mount master PD for etcd and create symbolic links to it.
{
	main_etcd_mount_point="/mnt/disks/master-pd"
	mount-pd "google-master-pd" "${main_etcd_mount_point}"
	# Contains all the data stored in etcd.
	mkdir -p "${main_etcd_mount_point}/var/etcd"
	chmod 700 "${main_etcd_mount_point}/var/etcd"
	ln -s -f "${main_etcd_mount_point}/var/etcd" /var/etcd
	mkdir -p /etc/srv
	# Setup the dynamically generated apiserver auth certs and keys to pd.
	mkdir -p "${main_etcd_mount_point}/srv/kubernetes"
	ln -s -f "${main_etcd_mount_point}/srv/kubernetes" /etc/srv/kubernetes
	# Copy the files to the PD only if they don't exist (so we do it only the first time).
	if [[ "$(ls -A ${main_etcd_mount_point}/srv/kubernetes/)" == "" ]]; then
		cp -r "${KUBE_ROOT}"/k8s_auth_data/* "${main_etcd_mount_point}/srv/kubernetes/"
	fi
	# Directory for kube-apiserver to store SSH key (if necessary).
	mkdir -p "${main_etcd_mount_point}/srv/sshproxy"
	ln -s -f "${main_etcd_mount_point}/srv/sshproxy" /etc/srv/sshproxy
}

# Mount master PD for event-etcd (if required) and create symbolic links to it.
{
	EVENT_STORE_IP="${EVENT_STORE_IP:-127.0.0.1}"
	EVENT_STORE_URL="${EVENT_STORE_URL:-http://${EVENT_STORE_IP}:4002}"
	if [ "${EVENT_PD:-}" == "true" ]; then
		event_etcd_mount_point="/mnt/disks/master-event-pd"
		mount-pd "google-master-event-pd" "${event_etcd_mount_point}"
		# Contains all the data stored in event etcd.
		mkdir -p "${event_etcd_mount_point}/var/etcd/events"
		chmod 700 "${event_etcd_mount_point}/var/etcd/events"
		ln -s -f "${event_etcd_mount_point}/var/etcd/events" /var/etcd/events
	fi
}

# Setup docker flags and load images of the master components.
assemble-docker-flags
DOCKER_REGISTRY="k8s.gcr.io"
load-docker-images

readonly audit_policy_file="/etc/audit_policy.config"

# Start kubelet as a supervisord process and master components as pods.
start-kubelet
if [[ -z "${ETCD_SERVERS:-}" ]]; then
	start-kubemaster-component "etcd"
	if [ "${EVENT_STORE_IP:-}" == "127.0.0.1" ]; then
		start-kubemaster-component "etcd-events"
	fi
fi
start-kubemaster-component "kube-apiserver"
start-kubemaster-component "kube-controller-manager"
start-kubemaster-component "kube-scheduler"
start-kubemaster-component "kube-addon-manager"

# Wait till apiserver is working fine or timeout.
echo -n "Waiting for apiserver to be healthy"
start=$(date +%s)
until [ "$(curl 127.0.0.1:8080/healthz 2> /dev/null)" == "ok" ]; do
	echo -n "."
	sleep 1
	now=$(date +%s)
	if [ $((now - start)) -gt 300 ]; then
		echo "Timeout!"
		exit 1
	fi
done

echo "Done for the configuration for kubemark master"


#!/usr/bin/env bash

# Copyright 2017 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This script contains the helper functions that each provider hosting
# Kubermark must implement to use test/kubemark/start-kubemark.sh and
# test/kubemark/stop-kubemark.sh scripts.

# This function should authenticate docker to be able to read/write to
# the right container registry (needed for pushing kubemark image).
function authenticate-docker {
	echo "Configuring registry authentication" 1>&2
}

# This function should create kubemark master and write kubeconfig to
# "${RESOURCE_DIRECTORY}/kubeconfig.kubemark".
# If a cluster uses private master IP, create-kubemark-master might also write
# a second kubeconfig to "${RESOURCE_DIRECTORY}/kubeconfig-internal.kubemark".
# The difference between these two kubeconfigs is that the internal one uses
# private master IP, which might be better suited for setting up hollow nodes.
function create-kubemark-master {
  echo "Creating cluster..."
}

# This function should delete kubemark master.
function delete-kubemark-master {
  echo "Deleting cluster..."
}

# This function should return node labels.
function calculate-node-labels {
  echo ""
}

# Common colors used throughout the kubemark scripts
if [[ -z "${color_start-}" ]]; then
  declare -r color_start="\033["
  # shellcheck disable=SC2034
  declare -r color_red="${color_start}0;31m"
  # shellcheck disable=SC2034
  declare -r color_yellow="${color_start}0;33m"
  # shellcheck disable=SC2034
  declare -r color_green="${color_start}0;32m"
  # shellcheck disable=SC2034
  declare -r color_blue="${color_start}1;34m"
  # shellcheck disable=SC2034
  declare -r color_cyan="${color_start}1;36m"
  # shellcheck disable=SC2034
  declare -r color_norm="${color_start}0m"
fi


#!/usr/bin/env bash
### Copyright 2010 Manuel Carrasco MoÃ±ino. (manolo at apache.org)
###
### Licensed under the Apache License, Version 2.0.
### You may obtain a copy of it at
### http://www.apache.org/licenses/LICENSE-2.0

###
### A library for shell scripts which creates reports in jUnit format.
### These reports can be used in Jenkins, or any other CI.
###
### Usage:
###     - Include this file in your shell script
###     - Use juLog to call your command any time you want to produce a new report
###        Usage:   juLog <options> command arguments
###           options:
###             -class="MyClass" : a class name which will be shown in the junit report
###             -name="TestName" : the test name which will be shown in the junit report
###             -error="RegExp"  : a regexp which sets the test as failure when the output matches it
###             -ierror="RegExp" : same as -error but case insensitive
###             -output="Path"   : path to output directory, defaults to "./results"
###     - Junit reports are left in the folder 'result' under the directory where the script is executed.
###     - Configure Jenkins to parse junit files from the generated folder
###

asserts=00; errors=0; total=0; content=""
date="$(which gdate 2>/dev/null || which date)"

# default output folder
juDIR="$(pwd)/results"

# The name of the suite is calculated based in your script name
suite=""

if LANG=C sed --help 2>&1 | grep -q GNU; then
  SED="sed"
elif which gsed &>/dev/null; then
  SED="gsed"
else
  echo "Failed to find GNU sed as sed or gsed. If you are on Mac: brew install gnu-sed." >&2
  exit 1
fi

# A wrapper for the eval method witch allows catching seg-faults and use tee
errfile=/tmp/evErr.$$.log
function eVal() {
  (eval "$1")
  # stdout and stderr may currently be inverted (see below) so echo may write to stderr
  echo "$?" 2>&1 | tr -d "\n" > "${errfile}"
}

# Method to clean old tests
function juLogClean() {
  echo "+++ Removing old junit reports from: ${juDIR} "
  rm -f "${juDIR}"/junit-*
}

# Execute a command and record its results
function juLog() {
  suite="";
  errfile=/tmp/evErr.$$.log
  date="$(which gdate 2>/dev/null || which date)"
  asserts=00; errors=0; total=0; content=""

  # parse arguments
  ya=""; icase=""
  while [[ -z "$ya" ]]; do
    case "$1" in
      -name=*)   name="$(echo "$1" | ${SED} -e 's/-name=//')";   shift;;
      -class=*)  class="$(echo "$1" | ${SED} -e 's/-class=//')";   shift;;
      -ierror=*) ereg="$(echo "$1" | ${SED} -e 's/-ierror=//')"; icase="-i"; shift;;
      -error=*)  ereg="$(echo "$1" | ${SED} -e 's/-error=//')";  shift;;
      -output=*) juDIR="$(echo "$1" | ${SED} -e 's/-output=//')";  shift;;
      *)         ya=1;;
    esac
  done

  # create output directory
  mkdir -p "${juDIR}" || exit
  # use first arg as name if it was not given
  if [[ -z "${name}" ]]; then
    name="${asserts}-$1"
    shift
  fi

  if [[ "${class}" = "" ]]; then
    class="default"
  fi

  suite=${class}

  # calculate command to eval
  [[ -z "$1" ]] && return
  cmd="$1"; shift
  while [[ -n "${1:-}" ]]
  do
     cmd="${cmd} \"$1\""
     shift
  done

  # eval the command sending output to a file
  outf=/var/tmp/ju$$.txt
  errf=/var/tmp/ju$$-err.txt
  :>${outf}
  echo ""                         | tee -a ${outf}
  echo "+++ Running case: ${class}.${name} " | tee -a ${outf}
  echo "+++ working dir: $(pwd)"           | tee -a ${outf}
  echo "+++ command: ${cmd}"            | tee -a ${outf}
  ini="$(${date} +%s.%N)"
  # execute the command, temporarily swapping stderr and stdout so they can be tee'd to separate files,
  # then swapping them back again so that the streams are written correctly for the invoking process
  ( (eVal "${cmd}" | tee -a ${outf}) 3>&1 1>&2 2>&3 | tee ${errf}) 3>&1 1>&2 2>&3
  evErr="$(cat ${errfile})"
  rm -f ${errfile}
  end="$(${date} +%s.%N)"
  echo "+++ exit code: ${evErr}"        | tee -a ${outf}

  # set the appropriate error, based in the exit code and the regex
  [[ ${evErr} != 0 ]] && err=1 || err=0
  out="$(${SED} -e 's/^\([^+]\)/| \1/g' "$outf")"
  if [ ${err} = 0 ] && [ -n "${ereg:-}" ]; then
      H=$(echo "${out}" | grep -E ${icase} "${ereg}")
      [[ -n "${H}" ]] && err=1
  fi
  [[ ${err} != 0 ]] && echo "+++ error: ${err}"         | tee -a ${outf}
  rm -f ${outf}

  errMsg=$(cat ${errf})
  rm -f ${errf}
  # calculate vars
  asserts=$((asserts+1))
  errors=$((errors+err))
  time=$(echo "${end} ${ini}" | awk '{print $1 - $2}')
  total=$(echo "${total} ${time}" | awk '{print $1 + $2}')

  # write the junit xml report
  ## failure tag
  [[ ${err} = 0 ]] && failure="" || failure="
      <failure type=\"ScriptError\" message=\"Script Error\"><![CDATA[${errMsg}]]></failure>
  "
  ## testcase tag
  content="${content}
    <testcase assertions=\"1\" name=\"${name}\" time=\"${time}\" classname=\"${class}\">
    ${failure}
    <system-err><![CDATA[${errMsg}]]></system-err>
    </testcase>
  "
  ## testsuite block

  if [[ -e "${juDIR}/junit_${suite}.xml" ]]; then
    # file exists. first update the failures count
    failCount=$(${SED} -n "s/.*testsuite.*failures=\"\([0-9]*\)\".*/\1/p" "${juDIR}/junit_${suite}.xml")
    errors=$((failCount+errors))
    ${SED} -i "0,/failures=\"${failCount}\"/ s/failures=\"${failCount}\"/failures=\"${errors}\"/" "${juDIR}/junit_${suite}.xml"
    ${SED} -i "0,/errors=\"${failCount}\"/ s/errors=\"${failCount}\"/errors=\"${errors}\"/" "${juDIR}/junit_${suite}.xml"

    # file exists. Need to append to it. If we remove the testsuite end tag, we can just add it in after.
    ${SED} -i "s^</testsuite>^^g" "${juDIR}/junit_${suite}.xml" ## remove testSuite so we can add it later
    ${SED} -i "s^</testsuites>^^g" "${juDIR}/junit_${suite}.xml"
    cat <<EOF >> "$juDIR/junit_$suite.xml"
     ${content:-}
    </testsuite>
</testsuites>
EOF

  else
    # no file exists. Adding a new file
    cat <<EOF > "${juDIR}/junit_${suite}.xml"
<?xml version="1.0" encoding="UTF-8"?>
<testsuites>
    <testsuite failures="${errors}" assertions="${assertions:-}" name="${suite}" tests="1" errors="${errors}" time="${total}">
    ${content:-}
    </testsuite>
</testsuites>
EOF
  fi

  return ${err}
}


#!/bin/sh
# enable automatic i386/ARM/M68K/MIPS/SPARC/PPC/s390/HPPA/Xtensa/microblaze
# program execution by the kernel
#
# downloaded from https://raw.githubusercontent.com/qemu/qemu/master/scripts/qemu-binfmt-conf.sh

# Enable automatic program execution by the kernel.

qemu_target_list="i386 i486 alpha arm armeb sparc sparc32plus sparc64 \
ppc ppc64 ppc64le m68k mips mipsel mipsn32 mipsn32el mips64 mips64el \
sh4 sh4eb s390x aarch64 aarch64_be hppa riscv32 riscv64 xtensa xtensaeb \
microblaze microblazeel or1k x86_64"

i386_magic='\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x03\x00'
i386_mask='\xff\xff\xff\xff\xff\xfe\xfe\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
i386_family=i386

i486_magic='\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x06\x00'
i486_mask='\xff\xff\xff\xff\xff\xfe\xfe\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
i486_family=i386

x86_64_magic='\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x3e\x00'
x86_64_mask='\xff\xff\xff\xff\xff\xfe\xfe\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
x86_64_family=i386

alpha_magic='\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x26\x90'
alpha_mask='\xff\xff\xff\xff\xff\xfe\xfe\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
alpha_family=alpha

arm_magic='\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x28\x00'
arm_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
arm_family=arm

armeb_magic='\x7fELF\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x28'
armeb_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
armeb_family=armeb

sparc_magic='\x7fELF\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x02'
sparc_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
sparc_family=sparc

sparc32plus_magic='\x7fELF\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x12'
sparc32plus_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
sparc32plus_family=sparc

sparc64_magic='\x7fELF\x02\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x2b'
sparc64_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
sparc64_family=sparc

ppc_magic='\x7fELF\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x14'
ppc_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
ppc_family=ppc

ppc64_magic='\x7fELF\x02\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x15'
ppc64_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
ppc64_family=ppc

ppc64le_magic='\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x15\x00'
ppc64le_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\x00'
ppc64le_family=ppcle

m68k_magic='\x7fELF\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x04'
m68k_mask='\xff\xff\xff\xff\xff\xff\xfe\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
m68k_family=m68k

# FIXME: We could use the other endianness on a MIPS host.

mips_magic='\x7fELF\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x08'
mips_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
mips_family=mips

mipsel_magic='\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x08\x00'
mipsel_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
mipsel_family=mips

mipsn32_magic='\x7fELF\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x08'
mipsn32_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
mipsn32_family=mips

mipsn32el_magic='\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x08\x00'
mipsn32el_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
mipsn32el_family=mips

mips64_magic='\x7fELF\x02\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x08'
mips64_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
mips64_family=mips

mips64el_magic='\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x08\x00'
mips64el_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
mips64el_family=mips

sh4_magic='\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x2a\x00'
sh4_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
sh4_family=sh4

sh4eb_magic='\x7fELF\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x2a'
sh4eb_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
sh4eb_family=sh4

s390x_magic='\x7fELF\x02\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x16'
s390x_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
s390x_family=s390x

aarch64_magic='\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\xb7\x00'
aarch64_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
aarch64_family=arm

aarch64_be_magic='\x7fELF\x02\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\xb7'
aarch64_be_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
aarch64_be_family=armeb

hppa_magic='\x7f\x45\x4c\x46\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x0f'
hppa_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
hppa_family=hppa

riscv32_magic='\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\xf3\x00'
riscv32_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
riscv32_family=riscv

riscv64_magic='\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\xf3\x00'
riscv64_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
riscv64_family=riscv

xtensa_magic='\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x5e\x00'
xtensa_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
xtensa_family=xtensa

xtensaeb_magic='\x7fELF\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x5e'
xtensaeb_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
xtensaeb_family=xtensaeb

microblaze_magic='\x7fELF\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\xba\xab'
microblaze_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
microblaze_family=microblaze

microblazeel_magic='\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\xab\xba'
microblazeel_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
microblazeel_family=microblazeel

or1k_magic='\x7fELF\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x5c'
or1k_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
or1k_family=or1k

qemu_get_family() {
    cpu=${HOST_ARCH:-$(uname -m)}
    case "$cpu" in
    amd64|i386|i486|i586|i686|i86pc|BePC|x86_64)
        echo "i386"
        ;;
    mips*)
        echo "mips"
        ;;
    "Power Macintosh"|ppc64|powerpc|ppc)
        echo "ppc"
        ;;
    ppc64el|ppc64le)
        echo "ppcle"
        ;;
    arm|armel|armhf|arm64|armv[4-9]*l|aarch64)
        echo "arm"
        ;;
    armeb|armv[4-9]*b|aarch64_be)
        echo "armeb"
        ;;
    sparc*)
        echo "sparc"
        ;;
    riscv*)
        echo "riscv"
        ;;
    *)
        echo "$cpu"
        ;;
    esac
}

usage() {
    cat <<EOF
Usage: qemu-binfmt-conf.sh [--qemu-path PATH][--debian][--systemd CPU]
                           [--help][--credential yes|no][--exportdir PATH]
                           [--persistent yes|no][--qemu-suffix SUFFIX]

       Configure binfmt_misc to use qemu interpreter

       --help:        display this usage
       --qemu-path:   set path to qemu interpreter ($QEMU_PATH)
       --qemu-suffix: add a suffix to the default interpreter name
       --debian:      don't write into /proc,
                      instead generate update-binfmts templates
       --systemd:     don't write into /proc,
                      instead generate file for systemd-binfmt.service
                      for the given CPU. If CPU is "ALL", generate a
                      file for all known cpus
       --exportdir:   define where to write configuration files
                      (default: $SYSTEMDDIR or $DEBIANDIR)
       --credential:  if yes, credential and security tokens are
                      calculated according to the binary to interpret
       --persistent:  if yes, the interpreter is loaded when binfmt is
                      configured and remains in memory. All future uses
                      are cloned from the open file.

    To import templates with update-binfmts, use :

        sudo update-binfmts --importdir ${EXPORTDIR:-$DEBIANDIR} --import qemu-CPU

    To remove interpreter, use :

        sudo update-binfmts --package qemu-CPU --remove qemu-CPU $QEMU_PATH

    With systemd, binfmt files are loaded by systemd-binfmt.service

    The environment variable HOST_ARCH allows to override 'uname' to generate
    configuration files for a different architecture than the current one.

    where CPU is one of:

        $qemu_target_list

EOF
}

qemu_check_access() {
    if [ ! -w "$1" ] ; then
        echo "ERROR: cannot write to $1" 1>&2
        exit 1
    fi
}

qemu_check_bintfmt_misc() {
    # load the binfmt_misc module
    if [ ! -d /proc/sys/fs/binfmt_misc ]; then
      if ! /sbin/modprobe binfmt_misc ; then
          exit 1
      fi
    fi
    if [ ! -f /proc/sys/fs/binfmt_misc/register ]; then
      if ! mount binfmt_misc -t binfmt_misc /proc/sys/fs/binfmt_misc ; then
          exit 1
      fi
    fi

    qemu_check_access /proc/sys/fs/binfmt_misc/register
}

installed_dpkg() {
    dpkg --status "$1" > /dev/null 2>&1
}

qemu_check_debian() {
    if [ ! -e /etc/debian_version ] ; then
        echo "WARNING: your system is not a Debian based distro" 1>&2
    elif ! installed_dpkg binfmt-support ; then
        echo "WARNING: package binfmt-support is needed" 1>&2
    fi
    qemu_check_access "$EXPORTDIR"
}

qemu_check_systemd() {
    if ! systemctl -q is-enabled systemd-binfmt.service ; then
        echo "WARNING: systemd-binfmt.service is missing or disabled" 1>&2
    fi
    qemu_check_access "$EXPORTDIR"
}

qemu_generate_register() {
    flags=""
    if [ "$CREDENTIAL" = "yes" ] ; then
        flags="OC"
    fi
    if [ "$PERSISTENT" = "yes" ] ; then
        flags="${flags}F"
    fi

    echo ":qemu-$cpu:M::$magic:$mask:$qemu:$flags"
}

qemu_register_interpreter() {
    echo "Setting $qemu as binfmt interpreter for $cpu"
    qemu_generate_register > /proc/sys/fs/binfmt_misc/register
}

qemu_generate_systemd() {
    echo "Setting $qemu as binfmt interpreter for $cpu for systemd-binfmt.service"
    qemu_generate_register > "$EXPORTDIR/qemu-$cpu.conf"
}

qemu_generate_debian() {
    cat > "$EXPORTDIR/qemu-$cpu" <<EOF
package qemu-$cpu
interpreter $qemu
magic $magic
mask $mask
credential $CREDENTIAL
EOF
}

qemu_set_binfmts() {
    # probe cpu type
    host_family=$(qemu_get_family)

    # register the interpreter for each cpu except for the native one

    for cpu in ${qemu_target_list} ; do
        magic=$(eval echo \$${cpu}_magic)
        mask=$(eval echo \$${cpu}_mask)
        family=$(eval echo \$${cpu}_family)

        if [ "$magic" = "" ] || [ "$mask" = "" ] || [ "$family" = "" ] ; then
            echo "INTERNAL ERROR: unknown cpu $cpu" 1>&2
            continue
        fi

        qemu="$QEMU_PATH/qemu-$cpu"
        if [ "$cpu" = "i486" ] ; then
            qemu="$QEMU_PATH/qemu-i386"
        fi

        qemu="$qemu$QEMU_SUFFIX"
        if [ "$host_family" != "$family" ] ; then
            $BINFMT_SET
        fi
    done
}

CHECK=qemu_check_bintfmt_misc
BINFMT_SET=qemu_register_interpreter

SYSTEMDDIR="/etc/binfmt.d"
DEBIANDIR="/usr/share/binfmts"

QEMU_PATH=/usr/local/bin
CREDENTIAL=no
PERSISTENT=no
QEMU_SUFFIX=""

options=$(getopt -o ds:Q:S:e:hc:p: -l debian,systemd:,qemu-path:,qemu-suffix:,exportdir:,help,credential:,persistent: -- "$@")
eval set -- "$options"

while true ; do
    case "$1" in
    -d|--debian)
        CHECK=qemu_check_debian
        BINFMT_SET=qemu_generate_debian
        EXPORTDIR=${EXPORTDIR:-$DEBIANDIR}
        ;;
    -s|--systemd)
        CHECK=qemu_check_systemd
        BINFMT_SET=qemu_generate_systemd
        EXPORTDIR=${EXPORTDIR:-$SYSTEMDDIR}
        shift
        # check given cpu is in the supported CPU list
        if [ "$1" != "ALL" ] ; then
            for cpu in ${qemu_target_list} ; do
                if [ "$cpu" = "$1" ] ; then
                    break
                fi
            done

            if [ "$cpu" = "$1" ] ; then
                qemu_target_list="$1"
            else
                echo "ERROR: unknown CPU \"$1\"" 1>&2
                usage
                exit 1
            fi
        fi
        ;;
    -Q|--qemu-path)
        shift
        QEMU_PATH="$1"
        ;;
    -F|--qemu-suffix)
        shift
        QEMU_SUFFIX="$1"
        ;;
    -e|--exportdir)
        shift
        EXPORTDIR="$1"
        ;;
    -h|--help)
        usage
        exit 1
        ;;
    -c|--credential)
        shift
        CREDENTIAL="$1"
        ;;
    -p|--persistent)
        shift
        PERSISTENT="$1"
        ;;
    *)
        break
        ;;
    esac
    shift
done

$CHECK
qemu_set_binfmts


#!/usr/bin/env bash

QEMU_BIN_DIR=${QEMU_BIN_DIR:-/usr/bin}

if [ ! -d /proc/sys/fs/binfmt_misc ]; then
    echo "No binfmt support in the kernel."
    echo "  Try: '/sbin/modprobe binfmt_misc' from the host"
    exit 1
fi

if [ ! -f /proc/sys/fs/binfmt_misc/register ]; then
    mount binfmt_misc -t binfmt_misc /proc/sys/fs/binfmt_misc
fi

if [ "${1}" = "--reset" ]; then
    shift
    find /proc/sys/fs/binfmt_misc -type f -name 'qemu-*' -exec sh -c 'echo -1 > {}' \;
fi

exec $(dirname "${BASH_SOURCE[0]}")/qemu-binfmt-conf.sh --qemu-suffix "-static" --qemu-path="${QEMU_BIN_DIR}" "$@"


#!/bin/bash
# Test the current package under a different kernel.
# Requires virtme and qemu to be installed.

set -eu
set -o pipefail

if [[ "${1:-}" = "--in-vm" ]]; then
  shift

  mount -t bpf bpf /sys/fs/bpf
  mount -t tracefs tracefs /sys/kernel/debug/tracing
  export CGO_ENABLED=0
  export GOFLAGS=-mod=readonly
  export GOPATH=/run/go-path
  export GOPROXY=file:///run/go-path/pkg/mod/cache/download
  export GOSUMDB=off
  export GOCACHE=/run/go-cache

  if [[ -d "/run/input/bpf" ]]; then
    export KERNEL_SELFTESTS="/run/input/bpf"
  fi

  readonly output="${1}"
  shift

  echo Running tests...
  go test -v -coverpkg=./... -coverprofile="$output/coverage.txt" -count 1 ./...
  touch "$output/success"
  exit 0
fi

# Pull all dependencies, so that we can run tests without the
# vm having network access.
go mod download

# Use sudo if /dev/kvm isn't accessible by the current user.
sudo=""
if [[ ! -r /dev/kvm || ! -w /dev/kvm ]]; then
  sudo="sudo"
fi
readonly sudo

readonly kernel_version="${1:-}"
if [[ -z "${kernel_version}" ]]; then
  echo "Expecting kernel version as first argument"
  exit 1
fi

readonly kernel="linux-${kernel_version}.bz"
readonly selftests="linux-${kernel_version}-selftests-bpf.bz"
readonly input="$(mktemp -d)"
readonly output="$(mktemp -d)"
readonly tmp_dir="${TMPDIR:-/tmp}"
readonly branch="${BRANCH:-master}"

fetch() {
    echo Fetching "${1}"
    wget -nv -N -P "${tmp_dir}" "https://github.com/cilium/ci-kernels/raw/${branch}/${1}"
}

fetch "${kernel}"

if fetch "${selftests}"; then
  mkdir "${input}/bpf"
  tar --strip-components=4 -xjf "${tmp_dir}/${selftests}" -C "${input}/bpf"
else
  echo "No selftests found, disabling"
fi

echo Testing on "${kernel_version}"
$sudo virtme-run --kimg "${tmp_dir}/${kernel}" --memory 512M --pwd \
  --rw \
  --rwdir=/run/input="${input}" \
  --rwdir=/run/output="${output}" \
  --rodir=/run/go-path="$(go env GOPATH)" \
  --rwdir=/run/go-cache="$(go env GOCACHE)" \
  --script-sh "PATH=\"$PATH\" $(realpath "$0") --in-vm /run/output" \
  --qemu-opts -smp 2 # need at least two CPUs for some tests

if [[ ! -e "${output}/success" ]]; then
  echo "Test failed on ${kernel_version}"
  exit 1
else
  echo "Test successful on ${kernel_version}"
  if [[ -v COVERALLS_TOKEN ]]; then
    goveralls -coverprofile="${output}/coverage.txt" -service=semaphore -repotoken "$COVERALLS_TOKEN"
  fi
fi

$sudo rm -r "${input}"
$sudo rm -r "${output}"


#go test -run=none -file bench_test.go -test.bench . -cpuprofile=bench_test.out

go test -c
./go-restful.test -test.run=none -test.cpuprofile=tmp.prof -test.bench=BenchmarkMany
./go-restful.test -test.run=none -test.cpuprofile=curly.prof -test.bench=BenchmarkManyCurly

#go tool pprof go-restful.test tmp.prof
go tool pprof go-restful.test curly.prof


go test -coverprofile=coverage.out
go tool cover -html=coverage.out


#!/bin/bash
set -e
set -x

if [ ! -d /tmp/build-golang/src/github.com/json-iterator ]; then
    mkdir -p /tmp/build-golang/src/github.com/json-iterator
    ln -s $PWD /tmp/build-golang/src/github.com/json-iterator/go
fi
export GOPATH=/tmp/build-golang
go get -u github.com/golang/dep/cmd/dep
cd /tmp/build-golang/src/github.com/json-iterator/go
exec $GOPATH/bin/dep ensure -update


#!/usr/bin/env bash

set -e
echo "" > coverage.txt

for d in $(go list ./... | grep -v vendor); do
    go test -coverprofile=profile.out -coverpkg=github.com/json-iterator/go $d
    if [ -f profile.out ]; then
        cat profile.out >> coverage.txt
        rm profile.out
    fi
done


#!/bin/bash

# for version in v1.9.1 v1.10.0 v1.10.3 v1.10.12 v1.11.2 v1.11.3 v1.12.0 v1.13.1 v1.14.0 v1.14.1 ; do
for version in v1.10.12 v1.14.1 v1.15.2 ; do
    echo "### $version" > $version.txt
    git checkout -- go.mod && git checkout $version && go test -run=NONE -bench=Benchmark2 >> $version.txt || exit 1
done


#!/usr/bin/env bash

set -e
echo "" > coverage.txt

for d in $(go list ./... | grep -v vendor); do
    go test -coverprofile=profile.out -coverpkg=github.com/modern-go/concurrent $d
    if [ -f profile.out ]; then
        cat profile.out >> coverage.txt
        rm profile.out
    fi
done


#!/usr/bin/env bash

set -e
echo "" > coverage.txt

for d in $(go list github.com/modern-go/reflect2-tests/... | grep -v vendor); do
    go test -coverprofile=profile.out -coverpkg=github.com/modern-go/reflect2 $d
    if [ -f profile.out ]; then
        cat profile.out >> coverage.txt
        rm profile.out
    fi
done


#!/bin/bash -e

pushd "$(dirname "$0")" >/dev/null

{
  cat <<EOF
/*
Copyright (c) 2017 VMware, Inc. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package simulator

// GuestID is the list of valid types.VirtualMachineGuestOsIdentifier
var GuestID = []types.VirtualMachineGuestOsIdentifier{
EOF

  ids=($(grep 'VirtualMachineGuestOsIdentifier(' ../vim25/types/enum.go | grep = | awk '{print $1}'))
  printf "types.%s,\n" "${ids[@]}"

  echo "}"
} > guest_id.go

goimports -w guest_id.go


#!/usr/bin/env bash

# Copyright The OpenTelemetry Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

#
# This script is used for
# a) creating a new tagged release of go.opentelemetry.io/contrib
# b) bumping the referenced version of go.opentelemetry.io/otel
#
# The options can be used together or individually.
#
set -e

declare CONTRIB_TAG
declare OTEL_TAG

help() {
   printf "\n"
   printf "Usage: %s [-o otel_tag] [-t tag]\n" "$0"
   printf "\t-o Otel release tag. Update all go.mod to reference go.opentelemetry.io/otel <otel_tag>.\n"
   printf "\t-t New Contrib unreleased tag. Update all go.mod files with this tag.\n"
   exit 1 # Exit script after printing help
}

while getopts "t:o:" opt
do
   case "$opt" in
      t ) CONTRIB_TAG="$OPTARG" ;;
      o ) OTEL_TAG="$OPTARG" ;;
      ? ) help ;; # Print help
   esac
done

declare -r SEMVER_REGEX="^v(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)(\\-[0-9A-Za-z-]+(\\.[0-9A-Za-z-]+)*)?(\\+[0-9A-Za-z-]+(\\.[0-9A-Za-z-]+)*)?$"

validate_tag() {
    local tag_=$1
    if [[ "${tag_}" =~ ${SEMVER_REGEX} ]]; then
	    printf "%s is valid semver tag.\n" "${tag_}"
    else
	    printf "%s is not a valid semver tag.\n" "${tag_}"
	    return 1
    fi
}

# Print help in case parameters are empty
if [[ -z "$CONTRIB_TAG" && -z "$OTEL_TAG" ]]
then
    printf "At least one of '-o' or '-t' must be specified.\n"
    help
fi

## Validate tags first
if [ -n "${OTEL_TAG}" ]; then
    validate_tag "${OTEL_TAG}" || exit $?

    # check that OTEL_TAG is a currently released tag for go.opentelemetry.io/otel
    TMPDIR=$(mktemp -d "/tmp/otel-contrib.XXXXXX") || exit 1
    trap "rm -fr ${TMPDIR}" EXIT
    (cd "${TMPDIR}" && go mod init tagtest)
    # requires go 1.14 for support of '-modfile'
    if ! go get -modfile="${TMPDIR}/go.mod" -d -v "go.opentelemetry.io/otel@${OTEL_TAG}"; then
        printf "go.opentelemetry.io/otel %s does not exist. Please supply a valid tag\n" "${OTEL_TAG}"
        exit 1
    fi
fi
if [ -n "${CONTRIB_TAG}" ]; then
    validate_tag "${CONTRIB_TAG}" || exit $?
    TAG_FOUND=$(git tag --list "${CONTRIB_TAG}")
    if [[ ${TAG_FOUND} = "${CONTRIB_TAG}" ]] ; then
        printf "Tag %s already exists in this repo\n" "${CONTRIB_TAG}"
        exit 1
    fi
else
    CONTRIB_TAG=${OTEL_TAG}  # if contrib_tag not specified, but OTEL_TAG is, then set it to OTEL_TAG
fi

# Get version for contrib.go
OTEL_CONTRIB_VERSION=$(echo "${CONTRIB_TAG}" | egrep -o "${SEMVER_REGEX}")
# Strip leading v
OTEL_CONTRIB_VERSION="${OTEL_CONTRIB_VERSION#v}"

cd "$(dirname "$0")"

if ! git diff --quiet; then \
    printf "Working tree is not clean, can't proceed\n"
    git status
    git diff
    exit 1
fi

# Update contrib.go version definition
cp contrib.go contrib.go.bak
sed "s/\(return \"\)[0-9]*\.[0-9]*\.[0-9]*\"/\1${OTEL_CONTRIB_VERSION}\"/" ./contrib.go.bak > ./contrib.go
rm -f ./contrib.go.bak

declare -r BRANCH_NAME=pre_release_${CONTRIB_TAG}

patch_gomods() {
    local pkg_=$1
    local tag_=$2
    # now do the same for all the directories underneath
    PACKAGE_DIRS=$(find . -mindepth 2 -type f -name 'go.mod' -exec dirname {} \; | egrep -v 'tools' | sed 's|^\.\/||' | sort)
    # quote any '.' characters in the pkg name
    local quoted_pkg_=${pkg_//./\\.}
    for dir in $PACKAGE_DIRS; do
	    cp "${dir}/go.mod" "${dir}/go.mod.bak"
	    sed "s|${quoted_pkg_}\([^ ]*\) v[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*[^0-9]*.*$|${pkg_}\1 ${tag_}|" "${dir}/go.mod.bak" >"${dir}/go.mod"
	    rm -f "${dir}/go.mod.bak"
    done
}

# branch off from existing main
git checkout -b "${BRANCH_NAME}" main

# Update go.mods
if [ -n "${OTEL_TAG}" ]; then
    # first update the top most module
    go get "go.opentelemetry.io/otel@${OTEL_TAG}"
    patch_gomods go.opentelemetry.io/otel "${OTEL_TAG}"
fi

if [ -n "${CONTRIB_TAG}" ]; then
    patch_gomods go.opentelemetry.io/contrib "${CONTRIB_TAG}"
fi

git diff
# Run lint to update go.sum
make lint

# Add changes and commit.
git add .
make ci
# Check whether registry links are up to date
make registry-links-check

declare COMMIT_MSG=""
if [ -n "${OTEL_TAG}" ]; then
    COMMIT_MSG+="Bumping otel version to ${OTEL_TAG}"
fi
COMMIT_MSG+=".  Prepare for releasing ${CONTRIB_TAG}"
git commit -m "${COMMIT_MSG}"

printf "Now run following to verify the changes.\ngit diff main\n"
printf "\nThen push the changes to upstream\n"


#!/usr/bin/env bash

# Copyright The OpenTelemetry Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

readonly PROGNAME=$(basename "$0")
readonly PROGDIR=$(readlink -m "$(dirname "$0")")

readonly EXCLUDE_PACKAGES="tools"
readonly SEMVER_REGEX="v(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)(\\-[0-9A-Za-z-]+(\\.[0-9A-Za-z-]+)*)?(\\+[0-9A-Za-z-]+(\\.[0-9A-Za-z-]+)*)?"

usage() {
    cat <<- EOF
Usage: $PROGNAME [OPTIONS] SEMVER_TAG COMMIT_HASH

Creates git tag for all Go packages in project.

OPTIONS:
  -h --help        Show this help.

ARGUMENTS:
  SEMVER_TAG       Semantic version to tag with.
  COMMIT_HASH      Git commit hash to tag.
EOF
}

cmdline() {
    local arg commit

    for arg
    do
        local delim=""
        case "$arg" in
            # Translate long form options to short form.
            --help)           args="${args}-h ";;
            # Pass through for everything else.
            *) [[ "${arg:0:1}" == "-" ]] || delim="\""
                args="${args}${delim}${arg}${delim} ";;
        esac
    done

    # Reset and process short form options.
    eval set -- "$args"

    while getopts "h" OPTION
    do
         case $OPTION in
         h)
             usage
             exit 0
             ;;
         *)
             echo "unknown option: $OPTION"
             usage
             exit 1
             ;;
        esac
    done

    # Positional arguments.
    shift $((OPTIND-1))
    readonly TAG="$1"
    if [ -z "$TAG" ]
    then
        echo "missing SEMVER_TAG"
        usage
        exit 1
    fi
    if [[ ! "$TAG" =~ $SEMVER_REGEX ]]
    then
        printf "invalid semantic version: %s\n" "$TAG"
        exit 2
    fi
    if [[ "$( git tag --list "$TAG" )" ]]
    then
        printf "tag already exists: %s\n" "$TAG"
        exit 2
    fi

    shift
    commit="$1"
    if [ -z "$commit" ]
    then
        echo "missing COMMIT_HASH"
        usage
        exit 1
    fi
    # Verify rev is for a commit and unify hashes into a complete SHA1.
    readonly SHA="$( git rev-parse --quiet --verify "${commit}^{commit}" )"
    if [ -z "$SHA" ]
    then
        printf "invalid commit hash: %s\n" "$commit"
        exit 2
    fi
    if [ "$( git merge-base "$SHA" HEAD )" != "$SHA" ]
    then
        printf "commit '%s' not found on this branch\n" "$commit"
        exit 2
    fi
}

package_dirs() {
    # Return a list of package directories in the form:
    #
    #  package/directory/a
    #  package/directory/b
    #  deeper/package/directory/a
    #  ...
    #
    # Making sure to exclude any packages in the EXCLUDE_PACKAGES regexp.
    find . -mindepth 2 -type f -name 'go.mod' -exec dirname {} \; \
        | grep -E -v "$EXCLUDE_PACKAGES" \
        | sed 's/^\.\///' \
        | sort
}

git_tag() {
    local tag="$1"
    local commit="$2"

    git tag -a "$tag" -s -m "Version $tag" "$commit"
}

previous_version() {
    local current="$1"

    # Requires git > 2.0
    git tag -l --sort=v:refname \
        | grep -E "^${SEMVER_REGEX}$" \
        | grep -v "$current" \
        | tail -1
}

print_changes() {
    local tag="$1"
    local previous

    previous="$( previous_version "$tag" )"
    if [ -n "$previous" ]
    then
        printf "\nRaw changes made between %s and %s\n" "$previous" "$tag"
        printf "======================================\n"
        git --no-pager log --pretty=oneline "${previous}..$tag"
    fi
}

main() {
    local dir

    cmdline "$@"

    cd "$PROGDIR" || exit 3

    # Create tag for root package.
    git_tag "$TAG" "$SHA"
    printf "created tag: %s\n" "$TAG"

    # Create tag for all sub-packages.
    for dir in $( package_dirs )
    do
        git_tag "${dir}/$TAG" "$SHA"
        printf "created tag: %s\n" "${dir}/$TAG"
    done

    print_changes "$TAG"
}
main "$@"


#!/usr/bin/env bash

# Copyright The OpenTelemetry Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -euo pipefail

top_dir='.'
if [[ $# -gt 0 ]]; then
    top_dir="${1}"
fi

p=$(pwd)
mod_dirs=()

# Note `mapfile` does not exist in older bash versions:
# https://stackoverflow.com/questions/41475261/need-alternative-to-readarray-mapfile-for-script-on-older-version-of-bash

while IFS= read -r line; do
    mod_dirs+=("$line")
done < <(find "${top_dir}" -type f -name 'go.mod' -exec dirname {} \; | sort)

for mod_dir in "${mod_dirs[@]}"; do
    cd "${mod_dir}"

    while IFS= read -r line; do
        echo ".${line#${p}}"
    done < <(go list --find -f '{{.Name}}|{{.Dir}}' ./... | grep '^main|' | cut -f 2- -d '|')
    cd "${p}"
done


#!/bin/bash

# Copyright The OpenTelemetry Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -e

help()
{
   printf "\n"
   printf "Usage: $0 -t tag\n"
   printf "\t-t Unreleased tag. Update all go.mod with this tag.\n"
   exit 1 # Exit script after printing help
}

while getopts "t:" opt
do
   case "$opt" in
      t ) TAG="$OPTARG" ;;
      ? ) help ;; # Print help
   esac
done

# Print help in case parameters are empty
if [ -z "$TAG" ]
then
   printf "Tag is missing\n";
   help
fi

# Validate semver
SEMVER_REGEX="^v(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)(\\-[0-9A-Za-z-]+(\\.[0-9A-Za-z-]+)*)?(\\+[0-9A-Za-z-]+(\\.[0-9A-Za-z-]+)*)?$"
if [[ "${TAG}" =~ ${SEMVER_REGEX} ]]; then
	printf "${TAG} is valid semver tag.\n"
else
	printf "${TAG} is not a valid semver tag.\n"
	exit -1
fi

TAG_FOUND=`git tag --list ${TAG}`
if [[ ${TAG_FOUND} = ${TAG} ]] ; then
        printf "Tag ${TAG} already exists\n"
        exit -1
fi

# Get version for version.go
OTEL_VERSION=$(echo "${TAG}" | grep -o '^v[0-9]\+\.[0-9]\+\.[0-9]\+')
# Strip leading v
OTEL_VERSION="${OTEL_VERSION#v}"

cd $(dirname $0)

if ! git diff --quiet; then \
	printf "Working tree is not clean, can't proceed with the release process\n"
	git status
	git diff
	exit 1
fi

# Update version.go
cp ./version.go ./version.go.bak
sed "s/\(return \"\)[0-9]*\.[0-9]*\.[0-9]*\"/\1${OTEL_VERSION}\"/" ./version.go.bak >./version.go
rm -f ./version.go.bak

# Update go.mod
git checkout -b pre_release_${TAG} main
PACKAGE_DIRS=$(find . -mindepth 2 -type f -name 'go.mod' -exec dirname {} \; | egrep -v 'tools' | sed 's/^\.\///' | sort)

for dir in $PACKAGE_DIRS; do
	cp "${dir}/go.mod" "${dir}/go.mod.bak"
	sed "s/opentelemetry.io\/otel\([^ ]*\) v[0-9]*\.[0-9]*\.[0-9]/opentelemetry.io\/otel\1 ${TAG}/" "${dir}/go.mod.bak" >"${dir}/go.mod"
	rm -f "${dir}/go.mod.bak"
done

# Run lint to update go.sum
make lint

# Add changes and commit.
git add .
make ci
git commit -m "Prepare for releasing $TAG"

printf "Now run following to verify the changes.\ngit diff main\n"
printf "\nThen push the changes to upstream\n"