#!/usr/bin/env bash

# Copyright 2021 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -o errexit
set -o nounset
set -o pipefail

# This will canonicalize the path
KUBE_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")"/../.. && pwd -P)
cd "${KUBE_ROOT}"

# test sources -> json spec _output/specsummaries.json
test/conformance/gen-specsummaries.sh
# specsummaries.json -> conformance.yaml
test/conformance/spec-to-yaml.sh


#!/usr/bin/env bash

# Copyright 2021 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -o errexit
set -o nounset
set -o pipefail

# This will canonicalize the path
KUBE_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")"/../.. && pwd -P)
cd "${KUBE_ROOT}"

# build ginkgo and e2e.test
# NOTE: we do *not* use `make WHAT=...` because we do *not* want to be running
# make generated_files when diffing things (see: hack/verify-conformance-yaml.sh)
# other update/verify already handle the generated files
hack/make-rules/build.sh vendor/github.com/onsi/ginkgo/ginkgo test/e2e/e2e.test

# dump spec
./_output/bin/ginkgo --dryRun=true --focus='[Conformance]' ./_output/bin/e2e.test -- --spec-dump "${KUBE_ROOT}/_output/specsummaries.json" > /dev/null


#!/usr/bin/env bash

# Copyright 2021 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -o errexit
set -o nounset
set -o pipefail

# This will canonicalize the path
KUBE_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")"/../.. && pwd -P)
cd "${KUBE_ROOT}"

# convert dumped spec (see dump-spec.sh) to conformance.md
go run ./test/conformance/walk.go --source="${KUBE_ROOT}" --docs ./_output/specsummaries.json > ./_output/conformance.md


#!/usr/bin/env bash

# Copyright 2021 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -o errexit
set -o nounset
set -o pipefail

# This will canonicalize the path
KUBE_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")"/../.. && pwd -P)
cd "${KUBE_ROOT}"

# convert dumped spec (see dump-spec.sh) to conformance.yaml
go run ./test/conformance/walk.go --source="${KUBE_ROOT}" ./_output/specsummaries.json > ./_output/conformance.yaml


#!/bin/sh

# Copyright 2021 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This script will update all sidecar RBAC files and the CSI hostpath
# deployment files such that they match what is in a hostpath driver
# release.
#
# Beware that this will wipe out all local modifications!

# Can be a tag or a branch.
script="$0"
hostpath_version="$1"

if ! [ "$hostpath_version" ]; then
    cat >&2 <<EOF
Usage: $0 <hostpath tag or branch name>

Required parameter is missing.
EOF
    exit 1
fi

set -xe
cd "$(dirname "$0")"

# Remove stale files.
rm -rf external-attacher external-provisioner external-resizer external-snapshotter external-health-monitor hostpath csi-driver-host-path

# Check out desired release.
git clone https://github.com/kubernetes-csi/csi-driver-host-path.git
(cd csi-driver-host-path && git checkout "$hostpath_version")
trap "rm -rf csi-driver-host-path" EXIT

# Main YAML files.
mkdir hostpath
cat >hostpath/README.md <<EOF
The files in this directory are exact copys of "kubernetes-latest" in
https://github.com/kubernetes-csi/csi-driver-host-path/tree/$hostpath_version/deploy/

Do not edit manually. Run $script to refresh the content.
EOF
cp -r csi-driver-host-path/deploy/kubernetes-latest/hostpath hostpath/
cat >hostpath/hostpath/e2e-test-rbac.yaml <<EOF
# priviledged Pod Security Policy, previously defined just for gcePD via PrivilegedTestPSPClusterRoleBinding()
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: psp-csi-hostpath-role
subjects:
  # This list of ServiceAccount intentionally covers everything that might
  # be needed. In practice, only some of these accounts are actually
  # used.
  - kind: ServiceAccount
    name: csi-attacher
    namespace: default
  - kind: ServiceAccount
    name: csi-provisioner
    namespace: default
  - kind: ServiceAccount
    name: csi-snapshotter
    namespace: default
  - kind: ServiceAccount
    name: csi-resizer
    namespace: default
  - kind: ServiceAccount
    name: csi-external-health-monitor-controller
    namespace: default
  - kind: ServiceAccount
    name: csi-hostpathplugin-sa
    namespace: default
roleRef:
  kind: ClusterRole
  name: e2e-test-privileged-psp
  apiGroup: rbac.authorization.k8s.io
EOF

download () {
    project="$1"
    path="$2"
    tag="$3"
    rbac="$4"

    mkdir -p "$project/$path"
    url="https://github.com/kubernetes-csi/$project/raw/$tag/deploy/kubernetes/$path/$rbac"
    cat >"$project/$path/$rbac" <<EOF
# Do not edit, downloaded from $url
# for csi-driver-host-path $hostpath_version
# by $script
#
EOF
    curl --fail --location "$url" >>"$project/$path/$rbac"
}

# RBAC files for each sidecar.
# This relies on the convention that "external-something" has "csi-something" as image name.
# external-health-monitor is special, it has two images.
# The repository for each image is ignored.
images=$(grep -r '^ *image:.*csi' hostpath/hostpath | sed -e 's;.*image:.*/;;' | grep -v 'node-driver-registrar' | sort -u)
for image in $images; do
    tag=$(echo "$image" | sed -e 's/.*://')
    path=
    rbac="rbac.yaml"
    case $image in
        csi-external-*)
            # csi-external-health-monitor-agent:v0.2.0
            project=$(echo "$image" | sed -e 's/csi-\(.*\)-[^:]*:.*/\1/')
            path=$(echo "$image" | sed -e 's/csi-\([^:]*\):.*/\1/')
            ;;
        *)
            project=$(echo "$image" | sed -e 's/:.*//' -e 's/^csi/external/')
            case $project in
                external-snapshotter)
                    # Another special case...
                    path="csi-snapshotter"
                    rbac="rbac-csi-snapshotter.yaml"
                    ;;
            esac
            ;;
    esac
    download "$project" "$path" "$tag" "$rbac"
done


#!/usr/bin/env bash

# Copyright 2016 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Make bucket and a folder for e2e-node test logs.
# Populate the folder from the logs stored in /tmp/_artifacts/ in the same way as a
# jenkins build would, and then print the URL to view the test results on Gubernator

set -o errexit
set -o nounset
set -o pipefail

KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../..
source "${KUBE_ROOT}/hack/lib/logging.sh"

if [[ $# -eq 0 || ! $1 =~ ^[Yy]$ ]]; then
  read -r -p "Do you want to run gubernator.sh and upload logs publicly to GCS? [y/n]" yn
  echo
  if [[ ! $yn =~ ^[Yy]$ ]]; then
      exit 1
  fi
fi

# Check that user has gsutil
if [[ $(which gsutil) == "" ]]; then
  echo "Could not find gsutil when running \`which gsutil\`"
  exit 1
fi

# Check that user has gcloud
if [[ $(which gcloud) == "" ]]; then
  echo "Could not find gcloud when running: \`which gcloud\`"
  exit 1
fi

# Check that user has Credentialed Active account
if ! gcloud auth list | grep -q "ACTIVE"; then
  echo "Could not find active account when running: \`gcloud auth list\`"
  exit 1
fi

readonly gcs_acl="public-read"
bucket_name="${USER}-g8r-logs"
echo ""
V=2 kube::log::status "Using bucket ${bucket_name}"

# Check if the bucket exists
if ! gsutil ls gs:// | grep -q "gs://${bucket_name}/"; then
  V=2 kube::log::status "Creating public bucket ${bucket_name}"
  gsutil mb "gs://${bucket_name}/"
  # Make all files in the bucket publicly readable
  gsutil acl ch -u AllUsers:R "gs://${bucket_name}"
else
  V=2 kube::log::status "Bucket already exists"
fi

# Path for e2e-node test results
GCS_JOBS_PATH="gs://${bucket_name}/logs/e2e-node"

ARTIFACTS=${ARTIFACTS:-"/tmp/_artifacts"}
BUILD_LOG_PATH="${ARTIFACTS}/build-log.txt"

if [[ ! -e $BUILD_LOG_PATH ]]; then
  echo "Could not find build-log.txt at ${BUILD_LOG_PATH}"
  exit 1
fi

# Get start and end timestamps based on build-log.txt file contents
# Line where the actual tests start
start_line=$(grep -n -m 1 "^=" "${BUILD_LOG_PATH}" | sed 's/\([0-9]*\).*/\1/')
# Create text file starting where the tests start
after_start=$(tail -n "+${start_line}" "${BUILD_LOG_PATH}")
echo "${after_start}" >> build-log-cut.txt
# Match the first timestamp
start_time_raw=$(grep -m 1 -o '[0-9][0-9][0-9][0-9][[:blank:]][0-9][0-9]:[0-9][0-9]:[0-9][0-9].[0-9]*' build-log-cut.txt)
rm build-log-cut.txt
# Make the date readable by date command (ex: 0101 00:00:00.000 -> 01/01 00:00:00.000)
start_time=$(echo "${start_time_raw}" | sed 's/^.\{2\}/&\//')
V=2 kube::log::status "Started at ${start_time}"
# Match the last timestamp in the build-log file
end_time=$(grep -o '[0-9][0-9][0-9][0-9][[:blank:]][0-9][0-9]:[0-9][0-9]:[0-9][0-9].[0-9]*' "${BUILD_LOG_PATH}" | tail -1 | sed 's/^.\{2\}/&\//')
# Convert to epoch time for Gubernator
start_time_epoch=$(date -d "${start_time}" +%s)
end_time_epoch=$(date -d "${end_time}" +%s)

# Make folder name for build from timestamp
BUILD_STAMP=$(echo "${start_time}" | sed 's/\///' | sed 's/ /_/')

GCS_LOGS_PATH="${GCS_JOBS_PATH}/${BUILD_STAMP}"

# Check if folder for same logs already exists
if gsutil ls "${GCS_JOBS_PATH}" | grep -q "${BUILD_STAMP}"; then
  V=2 kube::log::status "Log files already uploaded"
  echo "Gubernator linked below:"
  echo "k8s-gubernator.appspot.com/build/${GCS_LOGS_PATH}?local=on"
  exit
fi

while IFS= read -r result; do
  if [[ $result != "" && $result != "${ARTIFACTS}/results" && $result != "${ARTIFACTS}" ]]; then
    mv "${result}/"* "${ARTIFACTS}"
  fi
done < <(find "${ARTIFACTS}" -type d -name "results")

# Upload log files
for upload_attempt in $(seq 3); do
  if [[ -d "${ARTIFACTS}" && -n $(ls -A "${ARTIFACTS}") ]]; then
    V=2 kube::log::status "Uploading artifacts"
    gsutil -m -q -o "GSUtil:use_magicfile=True" cp -a "${gcs_acl}" -r -c \
      -z log,xml,json "${ARTIFACTS}" "${GCS_LOGS_PATH}/artifacts" || continue
  fi
  break
done
for upload_attempt in $(seq 3); do
  if [[ -e "${BUILD_LOG_PATH}" ]]; then
    V=2 kube::log::status "Uploading build log"
    gsutil -q cp -Z -a "${gcs_acl}" "${BUILD_LOG_PATH}" "${GCS_LOGS_PATH}" || continue
  fi
  break
done

# Find the k8s version for started.json
version=""
if [[ -e "version" ]]; then
  version=$(cat "version")
elif [[ -e "hack/lib/version.sh" ]]; then
  source "hack/lib/version.sh"
  kube::version::get_version_vars
  version="${KUBE_GIT_VERSION-}"
fi
if [[ -n "${version}" ]]; then
  V=2 kube::log::status "Found Kubernetes version: ${version}"
else
  V=2 kube::log::status "Could not find Kubernetes version"
fi

#Find build result from build-log.txt
if grep -Fxq "Test Suite Passed" "${BUILD_LOG_PATH}"
  then
    build_result="SUCCESS"
else
    build_result="FAILURE"
fi

V=4 kube::log::status "Build result is ${build_result}"

if [[ -e "${ARTIFACTS}/started.json" ]]; then
  rm "${ARTIFACTS}/started.json"
fi

if [[ -e "${ARTIFACTS}/finished.json" ]]; then
  rm "${ARTIFACTS}/finished.json"
fi

V=2 kube::log::status "Constructing started.json and finished.json files"
cat <<EOF >"${ARTIFACTS}/started.json"
{
    "version": "${version}",
    "timestamp": ${start_time_epoch},
    "jenkins-node": "${NODE_NAME:-}"
}
EOF

cat <<EOF >"${ARTIFACTS}/finished.json"
{
    "result": "${build_result}",
    "timestamp": ${end_time_epoch}
}
EOF

# Upload started.json
V=2 kube::log::status "Uploading started.json and finished.json"
V=2 kube::log::status "Run started at ${start_time}"
json_file="${GCS_LOGS_PATH}/started.json"

for upload_attempt in $(seq 3); do
  V=2 kube::log::status "Uploading started.json to ${json_file} (attempt ${upload_attempt})"
  gsutil -q -h "Content-Type:application/json" cp -a "${gcs_acl}" "${ARTIFACTS}/started.json" \
    "${json_file}" || continue
  break
done

# Upload finished.json
for upload_attempt in $(seq 3); do
  V=2 kube::log::status "Uploading finished.json to ${GCS_LOGS_PATH} (attempt ${upload_attempt})"
  gsutil -q -h "Content-Type:application/json" cp -a "${gcs_acl}" "${ARTIFACTS}/finished.json" \
    "${GCS_LOGS_PATH}/finished.json" || continue
  break
done

echo "Gubernator linked below:"
echo "k8s-gubernator.appspot.com/build/${bucket_name}/logs/e2e-node/${BUILD_STAMP}"


#!/usr/bin/env bash

# Copyright 2016 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This script is only for demonstrating how to use the node test container. In
# production environment, kubelet bootstrap will be more complicated, user
# should configure the node test container accordingly.
# In addition, this script will also be used in the node e2e test to let it use
# the containerized test suite.

# TODO(random-liu): Use standard installer to install kubelet.
# TODO(random-liu): Use standard tool to start kubelet in production way (such
# as systemd, supervisord etc.)

# Refresh sudo credentials if needed
if ping -c 1 -q metadata.google.internal &> /dev/null; then
  echo 'Running on GCE, not asking for sudo credentials'
elif sudo --non-interactive "$(which bash)" -c true 2> /dev/null; then
  # if we can run bash without a password, it's a pretty safe bet that either
  # we can run any command without a password, or that sudo credentials
  # are already cached - and they've just been re-cached
  echo 'No need to refresh sudo credentials'
else
  echo 'Updating sudo credentials'
  sudo -v || exit 1
fi

# FOCUS is ginkgo focus to select which tests to run. By default, FOCUS is
# initialized as "\[Conformance\]" in the test container to run all conformance
# test.
FOCUS=${FOCUS:-""}

# SKIP is ginkgo skip to select which tests to skip. By default, SKIP is
# initialized as "\[Flaky\]|\[Serial\]" in the test container skipping all
# flaky and serial test.
SKIP=${SKIP:-""}

# TEST_ARGS is the test arguments. It could be used to override default test
# arguments in the container.
TEST_ARGS=${TEST_ARGS:-""}

# REGISTRY is the image registry for node test image.
REGISTRY=${REGISTRY:-"k8s.gcr.io"}

# ARCH is the architecture of current machine, the script will use this to
# select corresponding test container image.
ARCH=${ARCH:-"amd64"}

# VERSION is the version of the test container image.
VERSION=${VERSION:-"0.2"}

# KUBELET_BIN is the kubelet binary name. If it is not specified, use the
# default binary name "kubelet".
KUBELET_BIN=${KUBELET_BIN:-"kubelet"}

# KUBELET is the kubelet binary path. If it is not specified, assume kubelet is
# in PATH.
KUBELET=${KUBELET:-"$(which "$KUBELET_BIN")"}

# LOG_DIR is the absolute path of the directory where the test will collect all
# logs to. By default, use the current directory.
LOG_DIR=${LOG_DIR:-$(pwd)}
mkdir -p "$LOG_DIR"

# NETWORK_PLUGIN is the network plugin used by kubelet. Do not use network
# plugin by default.
NETWORK_PLUGIN=${NETWORK_PLUGIN:-""}

# CNI_CONF_DIR is the path to network plugin binaries.
CNI_CONF_DIR=${CNI_CONF_DIR:-""}

# CNI_BIN_DIR is the path to network plugin config files.
CNI_BIN_DIR=${CNI_BIN_DIR:-""}

# KUBELET_KUBECONFIG is the path to a kubeconfig file, specifying how to connect to the API server.
KUBELET_KUBECONFIG=${KUBELET_KUBECONFIG:-"/var/lib/kubelet/kubeconfig"}

# Creates a kubeconfig file for the kubelet.
# Args: address (e.g. "http://localhost:8080"), destination file path
function create-kubelet-kubeconfig() {
  local api_addr="${1}"
  local dest="${2}"
  local dest_dir
  dest_dir="$(dirname "${dest}")"
  mkdir -p "${dest_dir}" &>/dev/null || sudo mkdir -p "${dest_dir}"
  sudo=$(test -w "${dest_dir}" || echo "sudo -E")
  cat <<EOF | ${sudo} tee "${dest}" > /dev/null
apiVersion: v1
kind: Config
clusters:
  - cluster:
      server: ${api_addr}
    name: local
contexts:
  - context:
      cluster: local
    name: local
current-context: local
EOF
}

# start_kubelet starts kubelet and redirect kubelet log to $LOG_DIR/kubelet.log.
kubelet_log=kubelet.log
start_kubelet() {
  echo "Creating kubelet.kubeconfig"
  create-kubelet-kubeconfig "http://localhost:8080" "${KUBELET_KUBECONFIG}"
  echo "Starting kubelet..."
  # we want to run this command as root but log the file to a normal user file
  # (so disable SC2024)
  # shellcheck disable=SC2024
  if ! sudo -b "${KUBELET}" "$@" &>"${LOG_DIR}/${kubelet_log}"; then
    echo "Failed to start kubelet"
    exit 1
  fi
}

# wait_kubelet retries for 10 times for kubelet to be ready by checking http://127.0.0.1:10255/healthz.
wait_kubelet() {
  echo "Health checking kubelet..."
  healthCheckURL=http://127.0.0.1:10255/healthz
  local maxRetry=10
  local cur=1
  while [ $cur -le $maxRetry ]; do
    if curl -s $healthCheckURL > /dev/null; then
      echo "Kubelet is ready"
      break
    fi
    if [ $cur -eq $maxRetry ]; then
      echo "Health check exceeds max retry"
      exit 1
    fi
    echo "Kubelet is not ready"
    sleep 1
    ((cur++))
  done
}

# kill_kubelet kills kubelet.
kill_kubelet() {
  echo "Stopping kubelet..."
  if ! sudo pkill "${KUBELET_BIN}"; then
    echo "Failed to stop kubelet."
    exit 1
  fi
}

# run_test runs the node test container.
run_test() {
  env=""
  if [ -n "$FOCUS" ]; then
    env="$env -e FOCUS=\"$FOCUS\""
  fi
  if [ -n "$SKIP" ]; then
    env="$env -e SKIP=\"$SKIP\""
  fi
  if [ -n "$TEST_ARGS" ]; then
    env="$env -e TEST_ARGS=\"$TEST_ARGS\""
  fi
  # The test assumes that inside the container:
  # * kubelet manifest path is mounted to the same path;
  # * log collect directory is mounted to /var/result;
  # * root file system is mounted to /rootfs.
  sudo sh -c "docker run -it --rm --privileged=true --net=host -v /:/rootfs \
    -v $config_dir:$config_dir -v $LOG_DIR:/var/result ${env} $REGISTRY/node-test-$ARCH:$VERSION"
}

# Check whether kubelet is running. If kubelet is running, tell the user to stop
# it before running the test.
pid=$(pidof "${KUBELET_BIN}")
if [ -n "$pid" ]; then
  echo "Kubelet is running (pid=$pid), please stop it before running the test."
  exit 1
fi

volume_stats_agg_period=10s
serialize_image_pulls=false
config_dir=$(mktemp -d)
file_check_frequency=10s
pod_cidr=10.100.0.0/24
log_level=4
start_kubelet --kubeconfig "${KUBELET_KUBECONFIG}" \
  --volume-stats-agg-period $volume_stats_agg_period \
  --serialize-image-pulls=$serialize_image_pulls \
  --pod-manifest-path "${config_dir}" \
  --file-check-frequency $file_check_frequency \
  --pod-cidr=$pod_cidr \
  --runtime-cgroups=/docker-daemon \
  --kubelet-cgroups=/kubelet \
  --system-cgroups=/system \
  --cgroup-root=/ \
  "--network-plugin=${NETWORK_PLUGIN}" \
  "--cni-conf-dir=${CNI_CONF_DIR}" \
  "--cni-bin-dir=${CNI_BIN_DIR}" \
  --v=$log_level \
  --logtostderr

wait_kubelet

run_test

kill_kubelet

# Clean up the kubelet config directory
sudo rm -rf "${config_dir}"


#!/usr/bin/env bash

# Copyright 2016 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Script used to configure node e2e test hosts from gce base images.
# DISCLAIMER: This script is not actively tested or maintained.  No guarantees that this will work
# on any host environment.  Contributions encouraged!  Send PRs to pwittrock (github.com).
#
# At some point has successfully configured the following distros:
# - ubuntu trusty
# - containervm (no-op)
# - rhel 7
# - centos 7
# - debian jessie

# RHEL os detection
cat /etc/*-release | grep "ID=\"rhel\""
OS_RHEL=$?

# On a systemd environment, enable cpu and memory accounting for all processes by default.
if [ -d /etc/systemd ]; then
  cat <<EOF >kubernetes-accounting.conf
[Manager]
DefaultCPUAccounting=yes
DefaultMemoryAccounting=yes
EOF
  sudo mkdir -p /etc/systemd/system.conf.d/
  sudo cp kubernetes-accounting.conf /etc/systemd/system.conf.d
  sudo systemctl daemon-reload
fi

# For coreos, disable updates
if sudo systemctl status update-engine &>/dev/null; then
  sudo systemctl mask update-engine locksmithd
fi

# Fixup sudoers require tty
if ! sudo grep -q "# Defaults    requiretty" /etc/sudoers; then
  sudo sed -i 's/Defaults    requiretty/# Defaults    requiretty/' /etc/sudoers
fi

# Install nsenter for ubuntu images
if cat /etc/*-release | grep "ID=ubuntu"; then
  if ! which nsenter > /dev/null; then
     echo "Do not find nsenter. Install it."
     NSENTER_BUILD_DIR=$(mktemp -d /tmp/nsenter-build-XXXXXX)
     cd "$NSENTER_BUILD_DIR" || exit 1
     curl https://www.kernel.org/pub/linux/utils/util-linux/v2.31/util-linux-2.31.tar.gz | tar -zxf-
     sudo apt-get update
     sudo apt-get --yes install make
     sudo apt-get --yes install gcc
     cd util-linux-2.31 || exit 1
     ./configure --without-ncurses
     make nsenter
     sudo cp nsenter /usr/local/bin
     rm -rf "$NSENTER_BUILD_DIR"
   fi
fi

# Install docker
if ! hash docker 2>/dev/null; then
  # RHEL platforms should always install from RHEL repository
  # This will install the latest supported stable docker platform on RHEL
  if [ $OS_RHEL -eq 0 ]; then
    sudo yum install -y docker-latest
    sudo groupadd docker
    sudo systemctl enable docker-latest.service
    sudo systemctl start docker-latest.service
  else
    curl -fsSL https://get.docker.com/ | sh
    sudo service docker start
    sudo systemctl enable docker.service
  fi
fi

# Allow jenkins access to docker
id jenkins || sudo useradd jenkins -m
sudo usermod -a -G docker jenkins

# install lxc
if ! cat /etc/*-release | grep "ID=debian"; then
  hash apt-get 2>/dev/null
  if [ $? -ne 1 ]; then
    sudo apt-get install lxc -y
    lxc-checkconfig
    sudo sed -i 's/GRUB_CMDLINE_LINUX="\(.*\)"/GRUB_CMDLINE_LINUX="\1 cgroup_enable=memory"/' /etc/default/grub
    sudo update-grub
  fi
fi

# delete init kubelet from containervm so that is doesn't startup
if [ -f /etc/init.d/kubelet ]; then
  sudo rm /etc/init.d/kubelet
fi


#!/usr/bin/env bash

# Copyright 2016 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Usage: copy-e2e-image.sh <image-name> <from-project-name> <to-project-name>

# See *.properties for list of images to copy,
# typically from kubernetes-node-e2e-images

set -e

print_usage() {
    echo "This script helps copy a GCE image from a source to a target project"
    echo -e "\nUsage:\n$0 <from-image-name> <from-project-name> <to-project-name> <to-image-name>\n"
}

if [  $# -ne 4 ]; then
    print_usage
    exit 1
fi

FROM_IMAGE=$1
FROM_PROJECT=$2
TO_PROJECT=$3
TO_IMAGE=$4

echo "Copying image $FROM_IMAGE from project $FROM_PROJECT to project $TO_PROJECT as image $TO_IMAGE..."
gcloud compute --project "${TO_PROJECT}" disks create "${TO_IMAGE}" --image=https://www.googleapis.com/compute/v1/projects/"${FROM_PROJECT}/global/images/""${FROM_IMAGE}"
gcloud compute --project "${TO_PROJECT}" images create "${TO_IMAGE}" \
  --source-disk="${TO_IMAGE}" \
  --description="Cloned from projects/$2/global/images/$1 by $USER on $(date)"
gcloud -q compute --project "${TO_PROJECT}" disks delete "${TO_IMAGE}"


#!/usr/bin/env bash

# Copyright 2016 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Script executed by jenkins to run node e2e tests against gce
# Usage: test/e2e_node/jenkins/e2e-node-jenkins.sh <path to properties>
# Properties files:
# - test/e2e_node/jenkins/jenkins-ci.properties : for running jenkins ci
# - test/e2e_node/jenkins/jenkins-pull.properties : for running jenkins pull request builder
# - test/e2e_node/jenkins/template.properties : template for creating a properties file to run locally

set -e
set -x

: "${1:?Usage test/e2e_node/jenkins/e2e-node-jenkins.sh <path to properties>}"

. "${1}"

# Until all GOPATH references are removed from all build scripts as well,
# explicitly disable module mode to avoid picking up user-set GO111MODULE preferences.
# As individual scripts make use of go modules, they can explicitly set GO111MODULE=on
export GO111MODULE=off

# indirectly generates test/e2e/generated/bindata.go too
make generated_files

# TODO converge build steps with hack/build-go some day if possible.
go build test/e2e_node/environment/conformance.go

PARALLELISM=${PARALLELISM:-8}
WORKSPACE=${WORKSPACE:-"/tmp/"}
ARTIFACTS=${WORKSPACE}/_artifacts
TIMEOUT=${TIMEOUT:-"45m"}

mkdir -p "${ARTIFACTS}"

go run test/e2e_node/runner/remote/run_remote.go  --logtostderr --vmodule=*=4 \
  --ssh-env="gce" --ssh-user="$GCE_USER" --zone="$GCE_ZONE" --project="$GCE_PROJECT" \
  --hosts="$GCE_HOSTS" --images="$GCE_IMAGES" --image-project="$GCE_IMAGE_PROJECT" \
  --image-config-file="$GCE_IMAGE_CONFIG_PATH" --cleanup="$CLEANUP" \
  --results-dir="$ARTIFACTS" --ginkgo-flags="--nodes=$PARALLELISM $GINKGO_FLAGS" \
  --test-timeout="$TIMEOUT" --test_args="$TEST_ARGS --kubelet-flags=\"$KUBELET_ARGS\"" \
  --instance-metadata="$GCE_INSTANCE_METADATA" --system-spec-name="$SYSTEM_SPEC_NAME" \
  --extra-envs="$EXTRA_ENVS"


#!/usr/bin/env bash

# Copyright 2017 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This script is meant to install Nvidia drivers on Ubuntu 14.04 GCE VMs.
# This script is meant to facilitate testing of Nvidia GPU support in Kubernetes.
echo "Checking for CUDA and installing."
# Check for CUDA and try to install.
if ! dpkg-query -W cuda; then
  curl -O http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1404/x86_64/cuda-repo-ubuntu1404_8.0.61-1_amd64.deb
  dpkg -i ./cuda-repo-ubuntu1404_8.0.61-1_amd64.deb
  apt-get update
  apt-get install cuda -y
  apt-get install "linux-headers-$(uname -r)" -y
fi
## Pre-loads kernel modules
nvidia-smi


#!/usr/bin/env bash

# Copyright 2016 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Script executed by jenkins to run node conformance test against gce
# Usage: test/e2e_node/jenkins/conformance-node-jenkins.sh <path to properties>

set -e
set -x

: "${1:?Usage test/e2e_node/jenkins/conformance-node-jenkins.sh <path to properties>}"

. "${1}"

make generated_files

WORKSPACE=${WORKSPACE:-"/tmp/"}
ARTIFACTS=${WORKSPACE}/_artifacts
TIMEOUT=${TIMEOUT:-"45m"}

mkdir -p "${ARTIFACTS}"

go run test/e2e_node/runner/remote/run_remote.go  --test-suite=conformance \
  --logtostderr --vmodule=*=4 --ssh-env="gce" --ssh-user="$GCE_USER" \
  --zone="$GCE_ZONE" --project="$GCE_PROJECT" --hosts="$GCE_HOSTS" \
  --images="$GCE_IMAGES" --image-project="$GCE_IMAGE_PROJECT" \
  --image-config-file="$GCE_IMAGE_CONFIG_PATH" --cleanup="$CLEANUP" \
  --results-dir="$ARTIFACTS" --test-timeout="$TIMEOUT" \
  --test_args="--kubelet-flags=\"$KUBELET_ARGS\"" \
  --instance-metadata="$GCE_INSTANCE_METADATA" \
  --system-spec-name="$SYSTEM_SPEC_NAME" \
  --extra-envs="$EXTRA_ENVS"


#!/usr/bin/env bash

# Copyright 2017 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -o errexit
set -o nounset
set -o pipefail

TASK=${1}
WHAT=${2}

# docker buildx command is still experimental as of Docker 19.03.0
export DOCKER_CLI_EXPERIMENTAL="enabled"

# Connecting to a Remote Docker requires certificates for authentication, which can be found
# at this path. By default, they can be found in the ${HOME} folder. We're expecting to find
# here ".docker-${os_version}" folders which contains the necessary certificates.
DOCKER_CERT_BASE_PATH="${DOCKER_CERT_BASE_PATH:-${HOME}}"

KUBE_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P)"
source "${KUBE_ROOT}/hack/lib/logging.sh"
source "${KUBE_ROOT}/hack/lib/util.sh"

# Mapping of go ARCH to actual architectures shipped part of multiarch/qemu-user-static project
declare -A QEMUARCHS=( ["amd64"]="x86_64" ["arm"]="arm" ["arm64"]="aarch64" ["ppc64le"]="ppc64le" ["s390x"]="s390x" )

windows_os_versions=(1809 1903 1909 2004 20H2)
declare -A WINDOWS_OS_VERSIONS_MAP

initWindowsOsVersions() {
  for os_version in "${windows_os_versions[@]}"; do
    img_base="mcr.microsoft.com/windows/nanoserver:${os_version}"
    full_version=$(docker manifest inspect "${img_base}" | grep "os.version" | head -n 1 | awk '{print $2}') || true
    WINDOWS_OS_VERSIONS_MAP["${os_version}"]="${full_version}"
  done
}

initWindowsOsVersions

# Returns list of all supported architectures from BASEIMAGE file
listOsArchs() {
  local image=${1}
  cut -d "=" -f 1 "${image}"/BASEIMAGE
}

splitOsArch() {
  local image=${1}
  local os_arch=${2}

  if [[ $os_arch =~ .*/.*/.* ]]; then
    # for Windows, we have to support both LTS and SAC channels, so we're building multiple Windows images.
    # the format for this case is: OS/ARCH/OS_VERSION.
    os_name=$(echo "$os_arch" | cut -d "/" -f 1)
    arch=$(echo "$os_arch" | cut -d "/" -f 2)
    os_version=$(echo "$os_arch" | cut -d "/" -f 3)
    suffix="$os_name-$arch-$os_version"
  elif [[ $os_arch =~ .*/.* ]]; then
    os_name=$(echo "$os_arch" | cut -d "/" -f 1)
    arch=$(echo "$os_arch" | cut -d "/" -f 2)
    os_version=""
    suffix="$os_name-$arch"
  else
    echo "The BASEIMAGE file for the ${image} image is not properly formatted. Expected entries to start with 'os/arch', found '${os_arch}' instead."
    exit 1
  fi
}

# Returns baseimage need to used in Dockerfile for any given architecture
getBaseImage() {
  os_arch=$1
  grep "${os_arch}=" BASEIMAGE | cut -d= -f2
}

# This function will build test image for all the architectures
# mentioned in BASEIMAGE file. In the absence of BASEIMAGE file,
# it will build for all the supported arch list - amd64, arm,
# arm64, ppc64le, s390x
build() {
  local image=${1}
  local img_folder=${1}
  local output_type=${2}
  docker_version_check

  if [[ -f "${img_folder}/BASEIMAGE" ]]; then
    os_archs=$(listOsArchs "$image")
  else
    # prepend linux/ to the QEMUARCHS items.
    os_archs=$(printf 'linux/%s\n' "${!QEMUARCHS[@]}")
  fi

  # image tag
  TAG=$(<"${img_folder}/VERSION")

  alias_name="$(cat "${img_folder}/ALIAS" 2>/dev/null || true)"
  if [[ -n "${alias_name}" ]]; then
    echo "Found an alias for '${image}'. Building / tagging image as '${alias_name}.'"
    image="${alias_name}"
  fi

  kube::util::ensure-gnu-sed

  for os_arch in ${os_archs}; do
    splitOsArch "${image}" "${os_arch}"
    if [[ "${os_name}" == "windows" && "${output_type}" == "docker" ]]; then
      echo "Cannot build the image '${image}' for ${os_arch}. Built Windows container images need to be pushed to a registry."
      continue
    fi

    echo "Building image for ${image} OS/ARCH: ${os_arch}..."

    # Create a temporary directory for every architecture and copy the image content
    # and build the image from temporary directory
    mkdir -p "${KUBE_ROOT}"/_tmp
    temp_dir=$(mktemp -d "${KUBE_ROOT}"/_tmp/test-images-build.XXXXXX)
    kube::util::trap_add "rm -rf ${temp_dir}" EXIT

    cp -r "${img_folder}"/* "${temp_dir}"
    if [[ -f ${img_folder}/Makefile ]]; then
      # make bin will take care of all the prerequisites needed
      # for building the docker image
      make -C "${img_folder}" bin OS="${os_name}" ARCH="${arch}" TARGET="${temp_dir}"
    fi
    pushd "${temp_dir}"

    # NOTE(claudiub): Some Windows images might require their own Dockerfile
    # while simpler ones will not. If we're building for Windows, check if
    # "Dockerfile_windows" exists or not.
    dockerfile_name="Dockerfile"
    if [[ "$os_name" = "windows" && -f "Dockerfile_windows" ]]; then
      dockerfile_name="Dockerfile_windows"
    fi

    base_image=""
    if [[ -f BASEIMAGE ]]; then
      base_image=$(getBaseImage "${os_arch}" | "${SED}" "s|REGISTRY|${REGISTRY}|g")
      "${SED}" -i "s|BASEARCH|${arch}|g" $dockerfile_name
    fi

    # Only the cross-build on x86 is guaranteed by far, other arches like aarch64 doesn't support cross-build
    # thus, there is no need to tackle a disability feature on those platforms, and also help to prevent from
    # ending up a wrong image tag on non-amd64 platforms.
    build_arch=$(uname -m)
    if [[ ${build_arch} = 'x86_64' ]]; then
        # copy the qemu-*-static binary to docker image to build the multi architecture image on x86 platform
        if grep -q 'CROSS_BUILD_' Dockerfile; then
          if [[ "${arch}" = 'amd64' ]]; then
            "${SED}" -i '/CROSS_BUILD_/d' Dockerfile
          else
            "${SED}" -i "s|QEMUARCH|${QEMUARCHS[$arch]}|g" Dockerfile
            # Register qemu-*-static for all supported processors except the current one
            echo 'Registering qemu-*-static binaries in the kernel'
            local sudo=""
            if [[ $(id -u) -ne 0 ]]; then
	            sudo="sudo"
            fi
            ${sudo} docker run --rm --privileged tonistiigi/binfmt:latest --install all
            curl -sSL https://github.com/multiarch/qemu-user-static/releases/download/"${QEMUVERSION}"/x86_64_qemu-"${QEMUARCHS[$arch]}"-static.tar.gz | tar -xz -C "${temp_dir}"
            # Ensure we don't get surprised by umask settings
            chmod 0755 "${temp_dir}/qemu-${QEMUARCHS[$arch]}-static"
            "${SED}" -i 's/CROSS_BUILD_//g' Dockerfile
          fi
        fi
    elif [[ "${QEMUARCHS[$arch]}" != "${build_arch}" ]]; then
		echo "skip cross-build $arch on non-supported platform ${build_arch}."
		popd
        continue
    else
        "${SED}" -i '/CROSS_BUILD_/d' Dockerfile
    fi

    docker buildx build --progress=plain --no-cache --pull --output=type="${output_type}" --platform "${os_name}/${arch}" \
        --build-arg BASEIMAGE="${base_image}" --build-arg REGISTRY="${REGISTRY}" --build-arg OS_VERSION="${os_version}" \
        -t "${REGISTRY}/${image}:${TAG}-${suffix}" -f "${dockerfile_name}" .

    popd
  done
}

docker_version_check() {
  # docker buildx has been introduced in 19.03, so we need to make sure we have it.
  docker_version=$(docker version --format '{{.Client.Version}}' | cut -d"-" -f1)
  if [[ ${docker_version} != 19.03.0 && ${docker_version} < 19.03.0 ]]; then
    echo "Minimum docker version 19.03.0 is required for using docker buildx: ${docker_version}]"
    exit 1
  fi
}

# This function will push the docker images
push() {
  local image=${1}
  docker_version_check

  TAG=$(<"${image}"/VERSION)
  if [[ -f ${image}/BASEIMAGE ]]; then
    os_archs=$(listOsArchs "$image")
  else
    # prepend linux/ to the QEMUARCHS items.
    os_archs=$(printf 'linux/%s\n' "${!QEMUARCHS[@]}")
  fi

  pushd "${image}"
  alias_name="$(cat ALIAS 2>/dev/null || true)"
  if [[ -n "${alias_name}" ]]; then
    echo "Found an alias for '${image}'. Pushing image as '${alias_name}.'"
    image="${alias_name}"
  fi

  kube::util::ensure-gnu-sed

  # reset manifest list; needed in case multiple images are being built / pushed.
  manifest=()
  # Make os_archs list into image manifest. Eg: 'linux/amd64 linux/ppc64le' to '${REGISTRY}/${image}:${TAG}-linux-amd64 ${REGISTRY}/${image}:${TAG}-linux-ppc64le'
  while IFS='' read -r line; do manifest+=("$line"); done < <(echo "$os_archs" | "${SED}" "s~\/~-~g" | "${SED}" -e "s~[^ ]*~$REGISTRY\/$image:$TAG\-&~g")
  docker manifest create --amend "${REGISTRY}/${image}:${TAG}" "${manifest[@]}"

  # We will need the full registry name in order to set the "os.version" for Windows images.
  # If the ${REGISTRY} dcesn't have any slashes, it means that it's on dockerhub.
  registry_prefix=""
  if [[ ! $REGISTRY =~ .*/.* ]]; then
    registry_prefix="docker.io/"
  fi
  # The images in the manifest list are stored locally. The folder / file name is almost the same,
  # with a few changes.
  manifest_image_folder=$(echo "${registry_prefix}${REGISTRY}/${image}:${TAG}" | sed "s|/|_|g" | sed "s/:/-/")

  for os_arch in ${os_archs}; do
    splitOsArch "${image}" "${os_arch}"
    docker manifest annotate --os "${os_name}" --arch "${arch}" "${REGISTRY}/${image}:${TAG}" "${REGISTRY}/${image}:${TAG}-${suffix}"

    # For Windows images, we also need to include the "os.version" in the manifest list, so the Windows node
    # can pull the proper image it needs.
    if [[ "$os_name" = "windows" ]]; then
      full_version="${WINDOWS_OS_VERSIONS_MAP[$os_version]}"

      # At the moment, docker manifest annotate doesn't allow us to set the os.version, so we'll have to
      # it ourselves. The manifest list can be found locally as JSONs.
      sed -i -r "s/(\"os\"\:\"windows\")/\0,\"os.version\":$full_version/" \
        "${HOME}/.docker/manifests/${manifest_image_folder}/${manifest_image_folder}-${suffix}"
    fi
  done
  popd
  docker manifest push --purge "${REGISTRY}/${image}:${TAG}"
}

# This function is for building AND pushing images. Useful if ${WHAT} is "all-conformance".
# This will allow images to be pushed immediately after they've been built.
build_and_push() {
  local image=${1}
  build "${image}" "registry"
  push "${image}"
}

# This function is for building the go code
bin() {
  local arch_prefix=""
  if [[ "${ARCH:-}" == "arm" ]]; then
    arch_prefix="GOARM=${GOARM:-7}"
  fi
  for SRC in "$@";
  do
  docker run --rm -v "${TARGET}:${TARGET}:Z" -v "${KUBE_ROOT}":/go/src/k8s.io/kubernetes:Z \
        golang:"${GOLANG_VERSION}" \
        /bin/bash -c "\
                cd /go/src/k8s.io/kubernetes/test/images/${SRC_DIR} && \
                CGO_ENABLED=0 ${arch_prefix} GOOS=${OS} GOARCH=${ARCH} go build -a -installsuffix cgo --ldflags '-w' -o ${TARGET}/${SRC} ./$(dirname "${SRC}")"
  done
}

shift

if [[ "${WHAT}" == "all-conformance" ]]; then
  # NOTE(claudiub): Building *ALL* the images under the kubernetes/test/images folder takes an extremely
  # long time (especially some images), and some images are rarely used and rarely updated, so there's
  # no point in rebuilding all of them every time. This will only build the Conformance-related images.
  # Discussed during Conformance Office Hours Meeting (2019.12.17):
  # https://docs.google.com/document/d/1W31nXh9RYAb_VaYkwuPLd1hFxuRX3iU0DmaQ4lkCsX8/edit#heading=h.l87lu17xm9bh
  shift
  conformance_images=("busybox" "agnhost" "echoserver" "jessie-dnsutils" "kitten" "nautilus" "nonewprivs" "resource-consumer" "sample-apiserver")
  for image in "${conformance_images[@]}"; do
    eval "${TASK}" "${image}" "$@"
  done
else
  eval "${TASK}" "$@"
fi


#!/bin/sh

# Copyright 2020 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

ARCH=$(uname -m)

if [ "${ARCH}" = "s390x" ]; then
    echo "http://dl-cdn.alpinelinux.org/alpine/v3.8/main" > /etc/apk/repositories
    echo "http://dl-cdn.alpinelinux.org/alpine/v3.8/community" >> /etc/apk/repositories
fi

apk add --no-cache openssl nginx-mod-http-lua nginx-mod-http-lua-upstream

exit 0


#!/bin/sh

# Copyright 2018 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

echo "Generating self-signed cert"
mkdir -p /certs
openssl req -x509 -sha256 -nodes -days 365 -newkey rsa:2048 \
-keyout /certs/privateKey.key \
-out /certs/certificate.crt \
-subj "/C=UK/ST=Warwickshire/L=Leamington/O=OrgName/OU=IT Department/CN=example.com"

# If we're running on Windows, skip loading the Linux .so modules.
if [ "$(uname)" = "Windows_NT" ]; then
	sed -i -E "s/^(load_module modules\/ndk_http_module.so;)$/#\1/" conf/nginx.conf
	sed -i -E "s/^(load_module modules\/ngx_http_lua_module.so;)$/#\1/" conf/nginx.conf
	sed -i -E "s/^(load_module modules\/ngx_http_lua_upstream_module.so;)$/#\1/" conf/nginx.conf

	# NOTE(claudiub): on Windows, nginx will take the paths in the nginx.conf file as relative paths.
	cmd /S /C "mklink /D C:\\openresty\\certs C:\\certs"
fi

echo "Starting nginx"
nginx -g "daemon off;"


#!/usr/bin/env bash

# Copyright 2017 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

DEB_ARCH=$(dpkg --print-architecture)

# http://security.debian.org/debian-security/dists/jessie/updates/InRelease is missing
# entries for some platforms, so we just remove the last line in sources.list in
# /etc/apt/sources.list which is "deb http://deb.debian.org/debian jessie-updates main"

case ${DEB_ARCH} in
    s390x|arm64|ppc64el)
        # we have to use the archive mirrors.
        # See: https://lists.debian.org/debian-devel-announce/2019/03/msg00006.html
        echo "deb http://archive.debian.org/debian/ jessie main contrib non-free" | tee /etc/apt/sources.list
        echo "deb-src http://archive.debian.org/debian/ jessie main contrib non-free" | tee -a /etc/apt/sources.list
       ;;
esac

exit 0


#!/bin/bash

# Copyright 2016 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This volume is assumed to exist and is shared with parent of the init
# container. It contains the redis installation.
INSTALL_VOLUME="/opt"

# This volume is assumed to exist and is shared with the peer-finder
# init container. It contains on-start/change configuration scripts.
WORK_DIR="/work-dir"

VERSION="3.2.0"

for i in "$@"
do
case $i in
    -i=*|--install-into=*)
    INSTALL_VOLUME="${i#*=}"
    shift
    ;;
    -w=*|--work-dir=*)
    WORK_DIR="${i#*=}"
    shift
    ;;
    *)
    # unknown option
    ;;
esac
done

echo installing config scripts into "${WORK_DIR}"
mkdir -p "${WORK_DIR}"
cp /on-start.sh "${WORK_DIR}"/
cp /peer-finder "${WORK_DIR}"/

echo installing redis-"${VERSION}" into "${INSTALL_VOLUME}"
mkdir -p "${INSTALL_VOLUME}"
mv /redis "${INSTALL_VOLUME}"/redis


#!/usr/bin/env bash

# Copyright 2016 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -e

CFG=/opt/redis/redis.conf
HOSTNAME=$(hostname)
DATADIR="/data"
# Port on which redis listens for connections.
PORT=6379

# Ping everyone but ourself to see if there's a master. Only one pet starts at
# a time, so if we don't see a master we can assume the position is ours.
while read -ra LINE; do
    if [[ "${LINE[0]}" == *"${HOSTNAME}"* ]]; then
        sed -i -e "s|^bind.*$|bind ${LINE[0]}|" ${CFG}
    elif [ "$(/opt/redis/redis-cli -h "${LINE[0]}" info | grep role | sed 's,\r$,,')" = "role:master" ]; then
        # TODO: More restrictive regex?
        sed -i -e "s|^# slaveof.*$|slaveof ${LINE[0]} ${PORT}|" ${CFG}
    fi
done

# Set the data directory for append only log and snapshot files. This should
# be a persistent volume for consistency.
sed -i -e "s|^.*dir .*$|dir ${DATADIR}|" ${CFG}

# The append only log is written for every SET operation. Without this setting,
# redis just snapshots periodically which is only safe for a cache. This will
# produce an appendonly.aof file in the configured data dir.
sed -i -e "s|^appendonly .*$|appendonly yes|" ${CFG}

# Every write triggers an fsync. Recommended default is "everysec", which
# is only safe for AP applications.
sed -i -e "s|^appendfsync .*$|appendfsync always|" ${CFG}


#! /bin/bash

# Copyright 2016 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This volume is assumed to exist and is shared with parent of the init
# container. It contains the zookeeper installation.
INSTALL_VOLUME="/opt"

# This volume is assumed to exist and is shared with the peer-finder
# init container. It contains on-start/change configuration scripts.
WORKDIR_VOLUME="/work-dir"

# As of April-2016 is 3.4.8 is the latest stable, but versions 3.5.0 onward
# allow dynamic reconfiguration.
VERSION="3.5.0-alpha"

for i in "$@"
do
case $i in
    -i=*|--install-into=*)
    INSTALL_VOLUME="${i#*=}"
    shift
    ;;
    -w=*|--work-dir=*)
    WORKDIR_VOLUME="${i#*=}"
    shift
    ;;
    *)
    # unknown option
    ;;
esac
done

echo installing config scripts into "${WORKDIR_VOLUME}"
mkdir -p "${WORKDIR_VOLUME}"
cp /on-start.sh "${WORKDIR_VOLUME}"/
cp /peer-finder "${WORKDIR_VOLUME}"/

echo installing zookeeper-"${VERSION}" into "${INSTALL_VOLUME}"
mkdir -p "${INSTALL_VOLUME}"
mv /zookeeper "${INSTALL_VOLUME}"/zookeeper
cp "${INSTALL_VOLUME}"/zookeeper/conf/zoo_sample.cfg "${INSTALL_VOLUME}"/zookeeper/conf/zoo.cfg

# TODO: Should dynamic config be tied to the version?
IFS="." read -ra RELEASE <<< "${VERSION}"
if [ "$(("${RELEASE[1]}"))" -gt 4 ]; then
    echo zookeeper-"${VERSION}" supports dynamic reconfiguration, enabling it
    echo "standaloneEnabled=false" >> "${INSTALL_VOLUME}"/zookeeper/conf/zoo.cfg
    echo "dynamicConfigFile=${INSTALL_VOLUME}/zookeeper/conf/zoo.cfg.dynamic" >> "${INSTALL_VOLUME}"/zookeeper/conf/zoo.cfg
fi

# TODO: This is a hack, netcat is convenient to have in the zookeeper container
# I want to avoid using a custom zookeeper image just for this. So copy it.
NC=$(which nc)
if [ "${NC}" != "" ]; then
    echo copying nc into "${INSTALL_VOLUME}"
    cp "${NC}" "${INSTALL_VOLUME}"
fi


#! /bin/bash

# Copyright 2016 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -eo pipefail

# This script configures zookeeper cluster member ship for version of zookeeper
# >= 3.5.0. It should not be used with the on-change.sh script in this example.
# As of April-2016 is 3.4.8 is the latest stable.

# Both /opt and /tmp/zookeeper are assumed to be volumes shared with the parent.
# The format of each line in the dynamic config file is:
# server.<1 based index>=<server-dns-name>:<peer port>:<election port>[:role];[<client port address>:]<client port>
# <1 based index> is the server index that matches the id in datadir/myid
# <peer port> is the port on which peers communicate to agree on updates
# <election port> is the port used for leader election
# [:role] can be set to observer, participant by default
# <client port address> is optional and defaults to 0.0.0.0
# <client port> is the port on which the server accepts client connections

CFG=/opt/zookeeper/conf/zoo.cfg.dynamic
CFG_BAK=/opt/zookeeper/conf/zoo.cfg.bak
MY_ID_FILE=/tmp/zookeeper/myid
HOSTNAME=$(hostname)

while read -ra LINE; do
    PEERS=("${PEERS[@]}" "${LINE[0]}")
done

# Don't add the first member as an observer
if [ ${#PEERS[@]} -eq 1 ]; then
    # We need to write our index in this list of servers into MY_ID_FILE.
    # Note that this may not always coincide with the hostname id.
    echo 1 > "${MY_ID_FILE}"
    echo "server.1=${PEERS[0]}:2888:3888;2181" > "${CFG}"
    # TODO: zkServer-initialize is the safe way to handle changes to datadir
    # because simply starting will create a new datadir, BUT if the user changed
    # pod template they might end up with 2 datadirs and brief split brain.
    exit
fi

# Every subsequent member is added as an observer and promoted to a participant
echo "" > "${CFG_BAK}"
i=0
LEADER=$HOSTNAME
for peer in "${PEERS[@]}"; do
    (( i=i+1 ))
    if [[ "${peer}" == *"${HOSTNAME}"* ]]; then
      MY_ID=$i
      MY_NAME=${peer}
      echo "$i" > "${MY_ID_FILE}"
      echo "server.${i}=${peer}:2888:3888:observer;2181" >> "${CFG_BAK}"
    else
      if [[ $(echo srvr | /opt/nc "${peer}" 2181 | grep Mode) = "Mode: leader" ]]; then
        LEADER="${peer}"
      fi
      echo "server.${i}=${peer}:2888:3888:participant;2181" >> "${CFG_BAK}"
    fi
done

# zookeeper won't start without myid anyway.
# This means our hostname wasn't in the peer list.
if [ ! -f "${MY_ID_FILE}" ]; then
  exit 1
fi

# Once the dynamic config file is written it shouldn't be modified, so the final
# reconfigure needs to happen through the "reconfig" command.
cp ${CFG_BAK} ${CFG}

# TODO: zkServer-initialize is the safe way to handle changes to datadir
# because simply starting will create a new datadir, BUT if the user changed
# pod template they might end up with 2 datadirs and brief split brain.
/opt/zookeeper/bin/zkServer.sh start

# TODO: We shouldn't need to specify the address of the master as long as
# there's quorum. According to the docs the new server is just not allowed to
# vote, it's still allowed to propose config changes, and it knows the
# existing members of the ensemble from *its* config.
ADD_SERVER="server.$MY_ID=$MY_NAME:2888:3888:participant;0.0.0.0:2181"
/opt/zookeeper/bin/zkCli.sh reconfig -s "${LEADER}":2181 -add "${ADD_SERVER}"

# Prove that we've actually joined the running cluster
ITERATION=0
until echo config | /opt/nc localhost 2181 | grep "${ADD_SERVER}" > /dev/null; do
  echo "$ITERATION"] waiting for updated config to sync back to localhost
  sleep 1
  (( ITERATION=ITERATION+1 ))
  if [ "$ITERATION" -eq 20 ]; then
    exit 1
  fi
done

/opt/zookeeper/bin/zkServer.sh stop


#!/usr/bin/env bash

# Copyright 2015 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

DIR="$(mktemp -d)"

function start()
{
    mount -t tmpfs test "$DIR"
    chmod 755 "$DIR"
    cp /vol/* "$DIR/"
    /usr/sbin/glusterd -p /run/glusterd.pid
    gluster volume create test_vol "$(hostname -i):$DIR" force
    gluster volume start test_vol
}

function stop()
{
    gluster --mode=script volume stop test_vol force
    kill "$(cat /run/glusterd.pid)"
    umount "$DIR"
    rm -rf "$DIR"
    exit 0
}

trap stop TERM

start "$@"

while true; do
    sleep 5
done


#!/usr/bin/env bash

# Copyright 2015 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Exit on the first error.
set -e

MNTDIR="$(mktemp -d)"

cleanup()
{
    # Make sure we return the right exit code
    RET=$?
    # Silently remove everything and ignore errors
    set +e
    /bin/umount "$MNTDIR" 2>/dev/null
    /bin/rmdir "$MNTDIR" 2>/dev/null
    /bin/rm block 2>/dev/null
    exit $RET
}

trap cleanup TERM EXIT

# Create 120MB device with ext2
# (volume_io tests need at least 100MB)
dd if=/dev/zero of=block seek=120 count=1 bs=1M
mkfs.ext2 block

# Add index.html to it
mount -o loop block "$MNTDIR"
echo "Hello from iscsi" > "$MNTDIR/index.html"
umount "$MNTDIR"

rm block.tar.gz 2>/dev/null || :
tar cfz block.tar.gz block


#!/usr/bin/env bash

# Copyright 2018 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This script does not run any daemon, it only configures iSCSI target (=server)
# in kernel. It is possible to run this script multiple times on a single
# node, each run will create its own IQN and LUN.

# Kubernetes must provide unique name.
IQN=$1

# targetcli synchronizes over dbus, however it does not work in
# containers. Use flock instead
LOCK=/srv/iscsi/targetcli.lock

function start()
{
    # targetcli need dbus. It may not run on the host, so start a private one
    mkdir /run/dbus
    dbus-daemon  --system

    # Create new IQN (iSCSI Qualified Name)
    flock $LOCK targetcli /iscsi create "$IQN"
    # Run it in demo mode, i.e. no authentication
    flock $LOCK targetcli /iscsi/"$IQN"/tpg1 set attribute authentication=0 demo_mode_write_protect=0 generate_node_acls=1 cache_dynamic_acls=1

    # Create unique "block volume" (i.e. flat file) on the *host*.
    # Having it in the container confuses kernel from some reason
    # and it's not able to server multiple LUNs from different
    # containers.
    # /srv/iscsi must be bind-mount from the host.
    cp /block /srv/iscsi/"$IQN"

    # Make the block volume available through our IQN as LUN 0
    flock $LOCK targetcli /backstores/fileio create block-"$IQN" /srv/iscsi/"$IQN"
    flock $LOCK targetcli /iscsi/"$IQN"/tpg1/luns create /backstores/fileio/block-"$IQN"

    echo "iscsi target started"
}

function stop()
{
    echo "stopping iscsi target"
    # Remove IQN
    flock $LOCK targetcli /iscsi/"$IQN"/tpg1/luns/ delete 0
    flock $LOCK targetcli /iscsi delete "$IQN"
    # Remove block device mapping
    flock $LOCK targetcli /backstores/fileio delete block-"$IQN"
    /bin/rm -f /srv/iscsi/"$IQN"
    echo "iscsi target stopped"
    exit 0
}

trap stop TERM
start

while true; do
    sleep 1
done


#!/usr/bin/env bash

# Copyright 2015 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

function start()
{

    unset gid
    # accept "-G gid" option
    while getopts "G:" opt; do
        case ${opt} in
            G) gid=${OPTARG};;
            *):;;
        esac
    done
    shift $((OPTIND - 1))

    # prepare /etc/exports
    for i in "$@"; do
        # fsid=0: needed for NFSv4
        echo "$i *(rw,fsid=0,insecure,no_root_squash)" >> /etc/exports
        if [ -v gid ] ; then
            chmod 070 "$i"
            chgrp "$gid" "$i"
        fi
        # move index.html to here
        /bin/cp /tmp/index.html "$i/"
        chmod 644 "$i/index.html"
        echo "Serving $i"
    done

    # start rpcbind if it is not started yet
    /usr/sbin/rpcinfo 127.0.0.1 > /dev/null; s=$?
    if [ $s -ne 0 ]; then
       echo "Starting rpcbind"
       /usr/sbin/rpcbind -w
    fi

    mount -t nfsd nfsd /proc/fs/nfsd

    # -V 3: enable NFSv3
    /usr/sbin/rpc.mountd -N 2 -V 3

    /usr/sbin/exportfs -r
    # -G 10 to reduce grace time to 10 seconds (the lowest allowed)
    /usr/sbin/rpc.nfsd -G 10 -N 2 -V 3
    /usr/sbin/rpc.statd --no-notify
    echo "NFS started"
}

function stop()
{
    echo "Stopping NFS"

    /usr/sbin/rpc.nfsd 0
    /usr/sbin/exportfs -au
    /usr/sbin/exportfs -f

    kill "$( pidof rpc.mountd )"
    umount /proc/fs/nfsd
    echo > /etc/exports
    exit 0
}

trap stop TERM

start "$@"

# Ugly hack to do nothing and wait for SIGTERM
while true; do
    sleep 5
done


#!/usr/bin/env bash

# Copyright 2015 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

#
# Bootstraps a CEPH server.
# It creates two OSDs on local machine, creates RBD pool there
# and imports 'block' device there.
#
# We must create fresh OSDs and filesystem here, because shipping it
# in a container would increase the image by ~300MB.
#

# Create /etc/ceph/ceph.conf
sh ./ceph.conf.sh "$(hostname -i)"

# Configure and start ceph-mon
sh ./mon.sh "$(hostname -i)"

# Configure and start 2x ceph-osd
mkdir -p /var/lib/ceph/osd/ceph-0 /var/lib/ceph/osd/ceph-1
sh ./osd.sh 0
sh ./osd.sh 1

# Configure and start cephfs metadata server
sh ./mds.sh

# Prepare a RBD volume "foo" (only with layering feature, the others may
# require newer clients).
# NOTE: we need Ceph kernel modules on the host that runs the client!
# As the default pool `rbd` might not be created on arm64 platform for ceph deployment,
# should create it if it does not exist.
arch=$(uname -m)
if [[ ${arch} = 'aarch64' || ${arch} = 'arm64' ]]; then
    if [[ $(ceph osd lspools) = "" ]]; then
        ceph osd pool create rbd 8
        rbd pool init rbd
    fi
fi
rbd import --image-feature layering block foo

# Prepare a cephfs volume
ceph osd pool create cephfs_data 4
ceph osd pool create cephfs_metadata 4
ceph fs new cephfs cephfs_metadata cephfs_data
# Put index.html into the volume
# It takes a while until the volume created above is mountable,
# 1 second is usually enough, but try indefinetily.
sleep 1
while ! ceph-fuse -m "$(hostname -i):6789" /mnt; do
    echo "Waiting for cephfs to be up"
    sleep 1
done
echo "Hello Ceph!" > /mnt/index.html
chmod 644 /mnt/index.html
umount /mnt

echo "Ceph is ready"

# Wait forever
while true; do
    sleep 10
done


#!/usr/bin/env bash

# Copyright 2015 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

#
# Configures /etc/ceph.conf from a template.
#

echo "
[global]
auth cluster required = none
auth service required = none
auth client required = none

[mon.a]
host = cephbox
mon addr = $1

[osd]
osd journal size = 128
journal dio = false

# allow running on ext4
osd max object name len = 256
osd max object namespace len = 64

[osd.0]
osd host = cephbox
" > /etc/ceph/ceph.conf


#!/usr/bin/env bash

# Copyright 2015 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Create block.tar.gz with a small ext2 filesystem.
# It must be run as root (to mount with '-o loop')!

# Exit on the first error.
set -e

MNTDIR="$(mktemp -d)"

cleanup()
{
    # Make sure we return the right exit code
    RET=$?
    # Silently remove everything and ignore errors
    set +e
    /bin/umount "$MNTDIR" 2>/dev/null
    /bin/rmdir "$MNTDIR" 2>/dev/null
    /bin/rm block 2>/dev/null
    exit $RET
}

trap cleanup TERM EXIT

# Create 120MB device with ext2
# (volume_io tests need at least 100MB)
dd if=/dev/zero of=block seek=120 count=1 bs=1M
mkfs.ext2 block

# Add index.html to it
mount -o loop block "$MNTDIR"
echo "Hello from RBD" > "$MNTDIR/index.html"
umount "$MNTDIR"

rm block.tar.gz 2>/dev/null || :
tar cfz block.tar.gz block


#!/usr/bin/env bash

# Copyright 2017 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

#
# Configures Ceph Metadata Service (mds), needed by CephFS
#
ceph-mds -i cephfs -c /etc/ceph/ceph.conf


#!/usr/bin/env bash

# Copyright 2015 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

#
# Configures and launches a new MON.
#

# monitor setup
monmaptool --create --clobber --fsid "$(uuidgen)" --add a "${1}":6789 /etc/ceph/monmap
mkdir /var/lib/ceph/mon/ceph-a
ceph-mon -i a --mkfs --monmap /etc/ceph/monmap -k /var/lib/ceph/mon/keyring
cp /var/lib/ceph/mon/keyring /var/lib/ceph/mon/ceph-a
ceph-mon -i a --monmap /etc/ceph/monmap -k /var/lib/ceph/mon/ceph-a/keyring

# client setup (handy)
cp /var/lib/ceph/mon/keyring /etc/ceph

# for this test we want to
ceph osd getcrushmap -o /tmp/crushc
crushtool -d /tmp/crushc -o /tmp/crushd
sed -i 's/step chooseleaf firstn 0 type host/step chooseleaf firstn 0 type osd/' /tmp/crushd
crushtool -c /tmp/crushd -o /tmp/crushc
ceph osd setcrushmap -i /tmp/crushc


#!/usr/bin/env bash

# Copyright 2015 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

#
# Configures and launches a new OSD.
#

ceph osd create
ceph-osd -i "${1}" --mkfs --mkkey
ceph auth add "osd.${1}" osd 'allow *' mon 'allow rwx' -i "/var/lib/ceph/osd/ceph-${1}/keyring"
ceph osd crush add "${1}" 1 root=default host=cephbox
ceph-osd -i "${1}" -k "/var/lib/ceph/osd/ceph-${1}/keyring"


#!/usr/bin/env bash

# Copyright 2021 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -o errexit
set -o nounset
set -o pipefail

KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../..
source "${KUBE_ROOT}/hack/lib/init.sh"
source "${KUBE_ROOT}/hack/lib/util.sh"

stability_check_setup() {
  kube::golang::verify_go_version
  kube::util::ensure-temp-dir
  cd "${KUBE_ROOT}"
  export KUBE_EXTRA_GOPATH=$KUBE_TEMP
  kube::golang::setup_env
  pushd "${KUBE_EXTRA_GOPATH}" >/dev/null
    GO111MODULE=on go get "gopkg.in/yaml.v2"
  popd >/dev/null
}

find_files_to_check() {
  find . -not \( \
      \( \
        -wholename './output' \
        -o -wholename './_output' \
        -o -wholename './_gopath' \
        -o -wholename './release' \
        -o -wholename './target' \
        -o -wholename '*/third_party/*' \
        -o -wholename '*/vendor/*' \
        -o -wholename '*/hack/*' \
        -o -wholename '*_test.go' \
        \) -prune \
    \) \
    \( -wholename '**/*.go' \
  \)
}

red=$(tput setaf 1)
green=$(tput setaf 2)
reset=$(tput sgr0)

kube::validate::stablemetrics() {
  stability_check_setup
  temp_file=$(mktemp)
  doValidate=$(find_files_to_check | grep -E ".*.go" | grep -v ".*_test.go" | sort | xargs -L 200 go run "test/instrumentation/main.go" "test/instrumentation/decode_metric.go" "test/instrumentation/find_stable_metric.go" "test/instrumentation/error.go" "test/instrumentation/metric.go" -- 1>"${temp_file}")

  if $doValidate; then
    echo -e "${green}Diffing test/instrumentation/testdata/stable-metrics-list.yaml\n${reset}"
    if diff -u "$KUBE_ROOT/test/instrumentation/testdata/stable-metrics-list.yaml" "$temp_file"; then
      echo -e "${green}\nPASS metrics stability verification ${reset}"
      return 0
    fi
  fi

  echo "${red}!!! Metrics Stability static analysis has failed!${reset}" >&2
  echo "${red}!!! Please run ./hack/update-generated-stable-metrics.sh to update the golden list.${reset}" >&2
  exit 1
}

kube::update::stablemetrics() {
  stability_check_setup
  temp_file=$(mktemp)
  doCheckStability=$(find_files_to_check | grep -E ".*.go" | grep -v ".*_test.go" | sort | xargs -L 200 go run "test/instrumentation/main.go" "test/instrumentation/decode_metric.go" "test/instrumentation/find_stable_metric.go" "test/instrumentation/error.go" "test/instrumentation/metric.go" -- 1>"${temp_file}")

  if ! $doCheckStability; then
    echo "${red}!!! updating golden list of metrics has failed! ${reset}" >&2
    exit 1
  fi
  mv -f "$temp_file" "${KUBE_ROOT}/test/instrumentation/testdata/stable-metrics-list.yaml"
  echo "${green}Updated golden list of stable metrics.${reset}"
}


#!/usr/bin/env bash

# Copyright 2018 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This file is not executable, and therefore it will not be run.
exit 1


#!/usr/bin/env bash

# Copyright 2018 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -o errexit
set -o nounset
set -o pipefail

if [[ -n "${EXEC_PLUGIN_OUTPUT_FILE-""}" ]]; then
  cat "$EXEC_PLUGIN_OUTPUT_FILE"
  exit
fi

echo "${EXEC_PLUGIN_OUTPUT-""}"

exit "${EXEC_PLUGIN_EXEC_CODE-0}"


#!/usr/bin/env bash

# Copyright 2018 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -o errexit
set -o nounset
set -o pipefail

TEST_ARGS=""
RUN_PATTERN=".*"
PROFILE_OPTS=""

function usage() {
  echo "usage: $0 <options>"
  echo " -h display this help message"
  echo " -d enable debug logs in tests"
  echo " -r <pattern> regex pattern to match for tests"
  echo " -o <filename> file to write JSON formatted results to"
  echo " -p <id> enable cpu and memory profiles, output written to mem-<id>.out and cpu-<id>.out"
  echo " -c enable custom test configuration"
  echo " -a <name> allocator name, one of RangeAllocator, CloudAllocator, IPAMFromCluster, IPAMFromCloud"
  echo " -k <num> api server qps for allocator"
  echo " -n <num> number of nodes to simulate"
  echo " -m <num> api server qps for node creation"
  echo " -l <num> gce cloud endpoint qps"
  exit 1
}

while getopts ":hdr:o:p:ca:k:n:m:l:" opt; do
  case ${opt} in
    d) TEST_ARGS="${TEST_ARGS} -v=6"
      ;;
    r) RUN_PATTERN="${OPTARG}"
      ;;
    o) TEST_ARGS="${TEST_ARGS} -log ${OPTARG}"
      ;;
    p) PROFILE_OPTS="-memprofile mem-${OPTARG}.out -cpuprofile cpu-${OPTARG}.out"
      ;;
    c) TEST_ARGS="${TEST_ARGS} -custom"
      ;;
    a) TEST_ARGS="${TEST_ARGS} -allocator ${OPTARG}"
      ;;
    k) TEST_ARGS="${TEST_ARGS} -kube-qps ${OPTARG}"
      ;;
    n) TEST_ARGS="${TEST_ARGS} -num-nodes ${OPTARG}"
      ;;
    m) TEST_ARGS="${TEST_ARGS} -create-qps ${OPTARG}"
      ;;
    l) TEST_ARGS="${TEST_ARGS} -cloud-qps ${OPTARG}"
      ;;
    h) usage
      ;;
    \?) usage
      ;;
  esac
done

KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../../../
source "${KUBE_ROOT}/hack/lib/init.sh"

kube::golang::setup_env

DIR_BASENAME=$(dirname "${BASH_SOURCE[0]}")
pushd "${DIR_BASENAME}"

cleanup() {
  popd 2> /dev/null
  kube::etcd::cleanup
  kube::log::status "performance test cleanup complete"
}

trap cleanup EXIT

kube::etcd::start

# Running IPAM tests. It might take a long time.
kube::log::status "performance test (IPAM) start"
go test "${PROFILE_OPTS}" -test.run="${RUN_PATTERN}" -test.timeout=60m -test.short=false -v -args "${TEST_ARGS}"
kube::log::status "... IPAM tests finished."


#!/usr/bin/env bash

# Copyright 2017 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

CLOUD_PROVIDER="${CLOUD_PROVIDER:-gce}"
CONTAINER_REGISTRY="${CONTAINER_REGISTRY:-gcr.io}"
PROJECT="${PROJECT:-}"
KUBEMARK_IMAGE_REGISTRY="${KUBEMARK_IMAGE_REGISTRY:-}"
KUBEMARK_IMAGE_MAKE_TARGET="${KUBEMARK_IMAGE_MAKE_TARGET:-gcloudpush}"


#!/usr/bin/env bash

# Copyright 2015 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This script assumes that kubectl binary is present in PATH.
kubectl config set-cluster hollow-cluster --server=http://localhost:8080 --insecure-skip-tls-verify=true
kubectl config set-credentials "$(whoami)"
kubectl config set-context hollow-context --cluster=hollow-cluster --user="$(whoami)"


#!/usr/bin/env bash

# Copyright 2015 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

export KUBERNETES_PROVIDER="kubemark"
export KUBE_CONFIG_FILE="config-default.sh"

KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../..

# We need an absolute path to KUBE_ROOT
ABSOLUTE_ROOT=$(readlink -f "${KUBE_ROOT}")

source "${KUBE_ROOT}/cluster/kubemark/util.sh"

echo "Kubemark master name: ${MASTER_NAME}"
echo "Kubemark master ip: ${KUBE_MASTER_IP}"

export KUBE_MASTER_URL="https://${KUBE_MASTER_IP}"
export KUBECONFIG="${ABSOLUTE_ROOT}/test/kubemark/resources/kubeconfig.kubemark"
export E2E_MIN_STARTUP_PODS=0

if [[ -z "$*" ]]; then
	ARGS=('--ginkgo.focus=[Feature:Performance]')
else
	ARGS=("$@")
fi

if [[ "${ENABLE_KUBEMARK_CLUSTER_AUTOSCALER}" == "true" ]]; then
  ARGS+=("--kubemark-external-kubeconfig=${DEFAULT_KUBECONFIG}")
fi

# Running locally.
for ((i=0; i < ${#ARGS[@]}; i++)); do
  ARGS[$i]="$(echo "${ARGS[$i]}" | sed -e 's/\[/\\\[/g' -e 's/\]/\\\]/g' )"
done
"${KUBE_ROOT}/hack/ginkgo-e2e.sh" "--e2e-verify-service-account=false" "--dump-logs-on-failure=false" "${ARGS[@]}"


#!/usr/bin/env bash

# Copyright 2015 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Script that creates a Kubemark cluster for any given cloud provider.

set -o errexit
set -o nounset
set -o pipefail

TMP_ROOT="$(dirname "${BASH_SOURCE[@]}")/../.."
KUBE_ROOT=$(readlink -e "${TMP_ROOT}" 2> /dev/null || perl -MCwd -e 'print Cwd::abs_path shift' "${TMP_ROOT}")

source "${KUBE_ROOT}/test/kubemark/skeleton/util.sh"
source "${KUBE_ROOT}/test/kubemark/cloud-provider-config.sh"
source "${KUBE_ROOT}/test/kubemark/${CLOUD_PROVIDER}/util.sh"
source "${KUBE_ROOT}/cluster/kubemark/${CLOUD_PROVIDER}/config-default.sh"

if [[ -f "${KUBE_ROOT}/test/kubemark/${CLOUD_PROVIDER}/startup.sh" ]] ; then
  source "${KUBE_ROOT}/test/kubemark/${CLOUD_PROVIDER}/startup.sh"
fi

source "${KUBE_ROOT}/cluster/kubemark/util.sh"

KUBECTL="${KUBE_ROOT}/cluster/kubectl.sh"
KUBEMARK_DIRECTORY="${KUBE_ROOT}/test/kubemark"
RESOURCE_DIRECTORY="${KUBEMARK_DIRECTORY}/resources"
LOCAL_KUBECONFIG="${RESOURCE_DIRECTORY}/kubeconfig.kubemark"
INTERNAL_KUBECONFIG="${RESOURCE_DIRECTORY}/kubeconfig-internal.kubemark"

# Generate a random 6-digit alphanumeric tag for the kubemark image.
# Used to uniquify image builds across different invocations of this script.
KUBEMARK_IMAGE_TAG=$(head /dev/urandom | tr -dc 'a-z0-9' | fold -w 6 | head -n 1)

# Create a docker image for hollow-node and upload it to the appropriate docker registry.
function create-and-upload-hollow-node-image {
  authenticate-docker
  KUBEMARK_IMAGE_REGISTRY="${KUBEMARK_IMAGE_REGISTRY:-${CONTAINER_REGISTRY}/${PROJECT}}"
  # Build+push the image through makefile.
  build_cmd=("make" "${KUBEMARK_IMAGE_MAKE_TARGET}")
  MAKE_DIR="${KUBE_ROOT}/cluster/images/kubemark"
  KUBEMARK_BIN="$(kube::util::find-binary-for-platform kubemark linux/amd64)"
  if [[ -z "${KUBEMARK_BIN}" ]]; then
    echo 'Cannot find cmd/kubemark binary'
    exit 1
  fi
  echo "Copying kubemark binary to ${MAKE_DIR}"
  cp "${KUBEMARK_BIN}" "${MAKE_DIR}"
  CURR_DIR=$(pwd)
  cd "${MAKE_DIR}"
  REGISTRY=${KUBEMARK_IMAGE_REGISTRY} IMAGE_TAG=${KUBEMARK_IMAGE_TAG} run-cmd-with-retries "${build_cmd[@]}"
  rm kubemark
  cd "$CURR_DIR"
  echo "Created and uploaded the kubemark hollow-node image to docker registry."
  # Cleanup the kubemark image after the script exits.
  if [[ "${CLEANUP_KUBEMARK_IMAGE:-}" == "true" ]]; then
    trap delete-kubemark-image EXIT
  fi
}

function delete-kubemark-image {
  delete-image "${KUBEMARK_IMAGE_REGISTRY}/kubemark:${KUBEMARK_IMAGE_TAG}"
}

# Generate secret and configMap for the hollow-node pods to work, prepare
# manifests of the hollow-node and heapster replication controllers from
# templates, and finally create these resources through kubectl.
function create-kube-hollow-node-resources {
  # Create kubemark namespace.
  "${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/kubemark-ns.json"

  # Create configmap for configuring hollow- kubelet, proxy and npd.
  "${KUBECTL}" create configmap "node-configmap" --namespace="kubemark" \
    --from-file=kernel.monitor="${RESOURCE_DIRECTORY}/kernel-monitor.json"

  # Create secret for passing kubeconfigs to kubelet, kubeproxy and npd.
  # It's bad that all component shares the same kubeconfig.
  # TODO(https://github.com/kubernetes/kubernetes/issues/79883): Migrate all components to separate credentials.
  "${KUBECTL}" create secret generic "kubeconfig" --type=Opaque --namespace="kubemark" \
    --from-file=kubelet.kubeconfig="${HOLLOWNODE_KUBECONFIG}" \
    --from-file=kubeproxy.kubeconfig="${HOLLOWNODE_KUBECONFIG}" \
    --from-file=npd.kubeconfig="${HOLLOWNODE_KUBECONFIG}" \
    --from-file=heapster.kubeconfig="${HOLLOWNODE_KUBECONFIG}" \
    --from-file=cluster_autoscaler.kubeconfig="${HOLLOWNODE_KUBECONFIG}" \
    --from-file=dns.kubeconfig="${HOLLOWNODE_KUBECONFIG}"

  # Create addon pods.
  # Heapster.
  mkdir -p "${RESOURCE_DIRECTORY}/addons"
  sed "s@{{MASTER_IP}}@${MASTER_IP}@g" "${RESOURCE_DIRECTORY}/heapster_template.json" > "${RESOURCE_DIRECTORY}/addons/heapster.json"
  metrics_mem_per_node=4
  metrics_mem=$((200 + metrics_mem_per_node*NUM_NODES))
  sed -i'' -e "s@{{METRICS_MEM}}@${metrics_mem}@g" "${RESOURCE_DIRECTORY}/addons/heapster.json"
  metrics_cpu_per_node_numerator=${NUM_NODES}
  metrics_cpu_per_node_denominator=2
  metrics_cpu=$((80 + metrics_cpu_per_node_numerator / metrics_cpu_per_node_denominator))
  sed -i'' -e "s@{{METRICS_CPU}}@${metrics_cpu}@g" "${RESOURCE_DIRECTORY}/addons/heapster.json"
  eventer_mem_per_node=500
  eventer_mem=$((200 * 1024 + eventer_mem_per_node*NUM_NODES))
  sed -i'' -e "s@{{EVENTER_MEM}}@${eventer_mem}@g" "${RESOURCE_DIRECTORY}/addons/heapster.json"

  # Cluster Autoscaler.
  if [[ "${ENABLE_KUBEMARK_CLUSTER_AUTOSCALER:-}" == "true" ]]; then
    echo "Setting up Cluster Autoscaler"
    KUBEMARK_AUTOSCALER_MIG_NAME="${KUBEMARK_AUTOSCALER_MIG_NAME:-${NODE_INSTANCE_PREFIX}-group}"
    KUBEMARK_AUTOSCALER_MIN_NODES="${KUBEMARK_AUTOSCALER_MIN_NODES:-0}"
    KUBEMARK_AUTOSCALER_MAX_NODES="${KUBEMARK_AUTOSCALER_MAX_NODES:-10}"
    NUM_NODES=${KUBEMARK_AUTOSCALER_MAX_NODES}
    echo "Setting maximum cluster size to ${NUM_NODES}."
    KUBEMARK_MIG_CONFIG="autoscaling.k8s.io/nodegroup: ${KUBEMARK_AUTOSCALER_MIG_NAME}"
    sed "s/{{master_ip}}/${MASTER_IP}/g" "${RESOURCE_DIRECTORY}/cluster-autoscaler_template.json" > "${RESOURCE_DIRECTORY}/addons/cluster-autoscaler.json"
    sed -i'' -e "s@{{kubemark_autoscaler_mig_name}}@${KUBEMARK_AUTOSCALER_MIG_NAME}@g" "${RESOURCE_DIRECTORY}/addons/cluster-autoscaler.json"
    sed -i'' -e "s@{{kubemark_autoscaler_min_nodes}}@${KUBEMARK_AUTOSCALER_MIN_NODES}@g" "${RESOURCE_DIRECTORY}/addons/cluster-autoscaler.json"
    sed -i'' -e "s@{{kubemark_autoscaler_max_nodes}}@${KUBEMARK_AUTOSCALER_MAX_NODES}@g" "${RESOURCE_DIRECTORY}/addons/cluster-autoscaler.json"
  fi

  # Kube DNS.
  if [[ "${ENABLE_KUBEMARK_KUBE_DNS:-}" == "true" ]]; then
    echo "Setting up kube-dns"
    sed "s@{{dns_domain}}@${KUBE_DNS_DOMAIN}@g" "${RESOURCE_DIRECTORY}/kube_dns_template.yaml" > "${RESOURCE_DIRECTORY}/addons/kube_dns.yaml"
  fi

  "${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/addons" --namespace="kubemark"

  # Create the replication controller for hollow-nodes.
  # We allow to override the NUM_REPLICAS when running Cluster Autoscaler.
  NUM_REPLICAS=${NUM_REPLICAS:-${NUM_NODES}}
  sed "s@{{numreplicas}}@${NUM_REPLICAS}@g" "${RESOURCE_DIRECTORY}/hollow-node_template.yaml" > "${RESOURCE_DIRECTORY}/hollow-node.yaml"
  proxy_cpu=20
  if [ "${NUM_NODES}" -gt 1000 ]; then
    proxy_cpu=50
  fi
  proxy_cpu=${KUBEMARK_HOLLOW_PROXY_MILLICPU:-$proxy_cpu}
  proxy_mem_per_node=${KUBEMARK_HOLLOW_PROXY_MEM_PER_NODE_KB:-50}
  proxy_mem=$((100 * 1024 + proxy_mem_per_node*NUM_NODES))
  hollow_node_labels=${HOLLOW_NODE_LABELS:-$(calculate-node-labels)}
  hollow_kubelet_params=$(eval "for param in ${HOLLOW_KUBELET_TEST_ARGS:-}; do echo -n \\\"\$param\\\",; done")
  hollow_kubelet_params=${hollow_kubelet_params%?}
  hollow_proxy_params=$(eval "for param in ${HOLLOW_PROXY_TEST_ARGS:-}; do echo -n \\\"\$param\\\",; done")
  hollow_proxy_params=${hollow_proxy_params%?}

  sed -i'' -e "s@{{hollow_kubelet_millicpu}}@${KUBEMARK_HOLLOW_KUBELET_MILLICPU:-40}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
  sed -i'' -e "s@{{hollow_kubelet_mem_Ki}}@${KUBEMARK_HOLLOW_KUBELET_MEM_KB:-$((100*1024))}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
  sed -i'' -e "s@{{hollow_proxy_millicpu}}@${proxy_cpu}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
  sed -i'' -e "s@{{hollow_proxy_mem_Ki}}@${proxy_mem}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
  sed -i'' -e "s@{{npd_millicpu}}@${KUBEMARK_NPD_MILLICPU:-20}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
  sed -i'' -e "s@{{npd_mem_Ki}}@${KUBEMARK_NPD_MEM_KB:-$((20*1024))}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
  sed -i'' -e "s@{{kubemark_image_registry}}@${KUBEMARK_IMAGE_REGISTRY}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
  sed -i'' -e "s@{{kubemark_image_tag}}@${KUBEMARK_IMAGE_TAG}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
  sed -i'' -e "s@{{master_ip}}@${MASTER_IP}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
  sed -i'' -e "s@{{hollow_node_labels}}@${hollow_node_labels}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
  sed -i'' -e "s@{{hollow_kubelet_params}}@${hollow_kubelet_params}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
  sed -i'' -e "s@{{hollow_proxy_params}}@${hollow_proxy_params}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
  sed -i'' -e "s@{{kubemark_mig_config}}@${KUBEMARK_MIG_CONFIG:-}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
  "${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/hollow-node.yaml" --namespace="kubemark"

  echo "Created secrets, configMaps, replication-controllers required for hollow-nodes."
}

# Wait until all hollow-nodes are running or there is a timeout.
function wait-for-hollow-nodes-to-run-or-timeout {
  echo -n "Waiting for all hollow-nodes to become Running"
  start=$(date +%s)
  nodes=$("${KUBECTL}" --kubeconfig="${LOCAL_KUBECONFIG}" get node 2> /dev/null) || true
  ready=$(($(echo "${nodes}" | grep -vc "NotReady") - 1))

  until [[ "${ready}" -ge "${NUM_REPLICAS}" ]]; do
    echo -n "."
    sleep 1
    now=$(date +%s)
    # Fail it if it already took more than 30 minutes.
    if [ $((now - start)) -gt 1800 ]; then
      echo ""
      # shellcheck disable=SC2154 # Color defined in sourced script
      echo -e "${color_red} Timeout waiting for all hollow-nodes to become Running. ${color_norm}"
      # Try listing nodes again - if it fails it means that API server is not responding
      if "${KUBECTL}" --kubeconfig="${LOCAL_KUBECONFIG}" get node &> /dev/null; then
        echo "Found only ${ready} ready hollow-nodes while waiting for ${NUM_NODES}."
      else
        echo "Got error while trying to list hollow-nodes. Probably API server is down."
      fi
      pods=$("${KUBECTL}" get pods -l name=hollow-node --namespace=kubemark) || true
      running=$(($(echo "${pods}" | grep -c "Running")))
      echo "${running} hollow-nodes are reported as 'Running'"
      not_running=$(($(echo "${pods}" | grep -vc "Running") - 1))
      echo "${not_running} hollow-nodes are reported as NOT 'Running'"
      echo "${pods}" | grep -v Running
      exit 1
    fi
    nodes=$("${KUBECTL}" --kubeconfig="${LOCAL_KUBECONFIG}" get node 2> /dev/null) || true
    ready=$(($(echo "${nodes}" | grep -vc "NotReady") - 1))
  done
  # shellcheck disable=SC2154 # Color defined in sourced script
  echo -e "${color_green} Done!${color_norm}"
}

############################### Main Function ########################################

# Setup for hollow-nodes.
function start-hollow-nodes {
  # shellcheck disable=SC2154 # Color defined in sourced script
  echo -e "${color_yellow}STARTING SETUP FOR HOLLOW-NODES${color_norm}"
  create-and-upload-hollow-node-image
  create-kube-hollow-node-resources
  wait-for-hollow-nodes-to-run-or-timeout
}

# Annotates the node objects in the kubemark cluster to make their size
# similar to regular nodes.
# TODO(#90833): Replace this with image preloading from ClusterLoader to better
# reflect the reality in kubemark tests.
function resize-node-objects {
  if [[ -z "$KUBEMARK_NODE_OBJECT_SIZE_BYTES" ]]; then
    return 0
  fi

  annotation_size_bytes="${KUBEMARK_NODE_OBJECT_SIZE_BYTES}"
  echo "Annotating node objects with ${annotation_size_bytes} byte label"
  label=$( (< /dev/urandom tr -dc 'a-zA-Z0-9' | fold -w "$annotation_size_bytes"; true) | head -n 1)
  "${KUBECTL}" --kubeconfig="${LOCAL_KUBECONFIG}" get nodes -o name \
    | xargs -n10 -P100 -r -I% "${KUBECTL}" --kubeconfig="${LOCAL_KUBECONFIG}" annotate --overwrite % label="$label" > /dev/null
  echo "Annotating node objects completed"
}

detect-project &> /dev/null
create-kubemark-master

if [ -f "${INTERNAL_KUBECONFIG}" ]; then
    HOLLOWNODE_KUBECONFIG="${INTERNAL_KUBECONFIG}"
else
    HOLLOWNODE_KUBECONFIG="${LOCAL_KUBECONFIG}"
fi

MASTER_IP=$(grep server "${HOLLOWNODE_KUBECONFIG}" | awk -F "/" '{print $3}')

start-hollow-nodes
resize-node-objects

echo ""
echo "Master IP: ${MASTER_IP}"
echo "Kubeconfig for kubemark master is written in ${LOCAL_KUBECONFIG}"


#!/usr/bin/env bash

# Copyright 2015 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Script that destroys Kubemark cluster and deletes all master resources.

KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../..

source "${KUBE_ROOT}/test/kubemark/skeleton/util.sh"
source "${KUBE_ROOT}/test/kubemark/cloud-provider-config.sh"
source "${KUBE_ROOT}/test/kubemark/${CLOUD_PROVIDER}/util.sh"
source "${KUBE_ROOT}/cluster/kubemark/${CLOUD_PROVIDER}/config-default.sh"

if [[ -f "${KUBE_ROOT}/test/kubemark/${CLOUD_PROVIDER}/shutdown.sh" ]] ; then
  source "${KUBE_ROOT}/test/kubemark/${CLOUD_PROVIDER}/shutdown.sh"
fi

source "${KUBE_ROOT}/cluster/kubemark/util.sh"

KUBECTL="${KUBE_ROOT}/cluster/kubectl.sh"
KUBEMARK_DIRECTORY="${KUBE_ROOT}/test/kubemark"
RESOURCE_DIRECTORY="${KUBEMARK_DIRECTORY}/resources"

detect-project &> /dev/null

"${KUBECTL}" delete -f "${RESOURCE_DIRECTORY}/addons" --namespace="kubemark" &> /dev/null || true
"${KUBECTL}" delete -f "${RESOURCE_DIRECTORY}/hollow-node.yaml" --namespace="kubemark" &> /dev/null || true
"${KUBECTL}" delete -f "${RESOURCE_DIRECTORY}/kubemark-ns.json" &> /dev/null || true

rm -rf "${RESOURCE_DIRECTORY}/addons" \
	"${RESOURCE_DIRECTORY}/kubeconfig.kubemark" \
	"${RESOURCE_DIRECTORY}/hollow-node.yaml"  &> /dev/null || true

delete-kubemark-master


#!/usr/bin/env bash

# Copyright 2017 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Running cmd $RETRIES times in case of failures.
function run-cmd-with-retries {
  RETRIES="${RETRIES:-3}"
  for attempt in $(seq 1 "${RETRIES}"); do
    local ret_val=0
    exec 5>&1 # Duplicate &1 to &5 for use below.
    # We don't use 'local' to declare result as then ret_val always gets value 0.
    # We use tee to output to &5 (redirected to stdout) while also storing it in the variable.
    result=$("$@" 2>&1 | tee >(cat - >&5)) || ret_val="$?"
    if [[ "${ret_val:-0}" -ne "0" ]]; then
      if [[ $(echo "${result}" | grep -c "already exists") -gt 0 ]]; then
        if [[ "${attempt}" == 1 ]]; then
          # shellcheck disable=SC2154 # Color defined in sourced script
          echo -e "${color_red}Failed to $1 $2 ${3:-} as the resource hasn't been deleted from a previous run.${color_norm}" >& 2
          exit 1
        fi
        # shellcheck disable=SC2154 # Color defined in sourced script
        echo -e "${color_yellow}Succeeded to $1 $2 ${3:-} in the previous attempt, but status response wasn't received.${color_norm}"
        return 0
      fi
      # shellcheck disable=SC2154 # Color defined in sourced script
      echo -e "${color_yellow}Attempt $attempt failed to $1 $2 ${3:-}. Retrying.${color_norm}" >& 2
      sleep $((attempt * 5))
    else
      # shellcheck disable=SC2154 # Color defined in sourced script
      echo -e "${color_green}Succeeded to $1 $2 ${3:-}.${color_norm}"
      return 0
    fi
  done
  # shellcheck disable=SC2154 # Color defined in sourced script
  echo -e "${color_red}Failed to $1 $2 ${3:-}.${color_norm}" >& 2
  exit 1
}