#!/bin/bash

# Copyright 2019 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# A small smoke test to run against a just-deployed kube-up cluster with Windows
# nodes. Performs checks such as:
#   1) Verifying that all Windows nodes have status Ready.
#   2) Verifying that no system pods are attempting to run on Windows nodes.
#   3) Verifying pairwise connectivity between most of the following: Linux
#      pods, Windows pods, K8s services, and the Internet.
#   4) Verifying that basic DNS resolution works in Windows pods.
#
# This script assumes that it is run from the root of the kubernetes repository.
#
# TODOs:
#   - Implement the node-to-pod checks.
#   - Capture stdout for each command to a file and only print it when the test
#     fails.
#   - Move copy-pasted code into reusable functions.
#   - Continue running all checks after one fails.
#   - Test service connectivity by running a test pod with an http server and
#     exposing it as a service (rather than curl-ing from existing system
#     services that don't serve http requests).
#   - Add test retries for transient errors, such as:
#     "error: unable to upgrade connection: Authorization error
#     (user=kube-apiserver, verb=create, resource=nodes, subresource=proxy)"

# Override this to use a different kubectl binary.
kubectl=kubectl
linux_deployment_timeout=60
windows_deployment_timeout=600
output_file=/tmp/k8s-smoke-test.out

function check_windows_nodes_are_ready {
  # kubectl filtering is the worst.
  statuses=$(${kubectl} get nodes -l kubernetes.io/os=windows \
    -o jsonpath='{.items[*].status.conditions[?(@.type=="Ready")].status}')
  for status in $statuses; do
    if [[ $status == "False" ]]; then
      echo "ERROR: some Windows node has status != Ready"
      echo "kubectl get nodes -l kubernetes.io/os=windows"
      ${kubectl} get nodes -l kubernetes.io/os=windows
      exit 1
    fi
  done
  echo "Verified that all Windows nodes have status Ready"
}

function untaint_windows_nodes {
  # Untaint the windows nodes to allow test workloads without tolerations to be
  # scheduled onto them.
  WINDOWS_NODES=$(${kubectl} get nodes -l kubernetes.io/os=windows -o name)
  for node in $WINDOWS_NODES; do
    ${kubectl} taint node "$node" node.kubernetes.io/os:NoSchedule-
  done
}

function check_no_system_pods_on_windows_nodes {
  windows_system_pods=$(${kubectl} get pods --namespace kube-system \
    -o wide | grep -E "Pending|windows" | wc -w)
  if [[ $windows_system_pods -ne 0 ]]; then
    echo "ERROR: there are kube-system pods trying to run on Windows nodes"
    echo "kubectl get pods --namespace kube-system -o wide"
    ${kubectl} get pods --namespace kube-system -o wide
    exit 1
  fi
  echo "Verified that all system pods are running on Linux nodes"
}

linux_webserver_deployment=linux-nginx
linux_webserver_pod_label=nginx
linux_webserver_replicas=1

function deploy_linux_webserver_pod {
  echo "Writing example deployment to $linux_webserver_deployment.yaml"
  cat <<EOF > $linux_webserver_deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: $linux_webserver_deployment
  labels:
    app: $linux_webserver_pod_label
spec:
  replicas: $linux_webserver_replicas
  selector:
    matchLabels:
      app: $linux_webserver_pod_label
  template:
    metadata:
      labels:
        app: $linux_webserver_pod_label
    spec:
      containers:
      - name: nginx
        image: nginx:1.7.9
      nodeSelector:
        kubernetes.io/os: linux
EOF

  if ! ${kubectl} create -f $linux_webserver_deployment.yaml; then
    echo "kubectl create -f $linux_webserver_deployment.yaml failed"
    exit 1
  fi

  timeout=$linux_deployment_timeout
  while [[ $timeout -gt 0 ]]; do
    echo "Waiting for $linux_webserver_replicas Linux $linux_webserver_pod_label pods to become Ready"
    statuses=$(${kubectl} get pods -l app=$linux_webserver_pod_label \
      -o jsonpath='{.items[*].status.conditions[?(@.type=="Ready")].status}' \
      | grep "True" | wc -w)
    if [[ $statuses -eq $linux_webserver_replicas ]]; then
      break
    else
      sleep 10
      (( timeout=timeout-10 ))
    fi
  done

  if [[ $timeout -gt 0 ]]; then
    echo "All $linux_webserver_pod_label pods became Ready"
  else
    echo "ERROR: Not all $linux_webserver_pod_label pods became Ready"
    echo "kubectl get pods -l app=$linux_webserver_pod_label"
    ${kubectl} get pods -l app=$linux_webserver_pod_label
    cleanup_deployments
    exit 1
  fi
}

# Returns the name of an arbitrary Linux webserver pod.
function get_linux_webserver_pod_name {
  $kubectl get pods -l app=$linux_webserver_pod_label \
    -o jsonpath='{.items[0].metadata.name}'
}

# Returns the IP address of an arbitrary Linux webserver pod.
function get_linux_webserver_pod_ip {
  $kubectl get pods -l app=$linux_webserver_pod_label \
    -o jsonpath='{.items[0].status.podIP}'
}

function undeploy_linux_webserver_pod {
  ${kubectl} delete deployment $linux_webserver_deployment
}

linux_command_deployment=linux-ubuntu
linux_command_pod_label=ubuntu
linux_command_replicas=1

function deploy_linux_command_pod {
  echo "Writing example deployment to $linux_command_deployment.yaml"
  cat <<EOF > $linux_command_deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: $linux_command_deployment
  labels:
    app: $linux_command_pod_label
spec:
  replicas: $linux_command_replicas
  selector:
    matchLabels:
      app: $linux_command_pod_label
  template:
    metadata:
      labels:
        app: $linux_command_pod_label
    spec:
      containers:
      - name: ubuntu
        image: ubuntu
        command: ["sleep", "123456"]
      nodeSelector:
        kubernetes.io/os: linux
EOF

  if ! ${kubectl} create -f $linux_command_deployment.yaml; then
    echo "kubectl create -f $linux_command_deployment.yaml failed"
    exit 1
  fi

  timeout=$linux_deployment_timeout
  while [[ $timeout -gt 0 ]]; do
    echo "Waiting for $linux_command_replicas Linux $linux_command_pod_label pods to become Ready"
    statuses=$(${kubectl} get pods -l app=$linux_command_pod_label \
      -o jsonpath='{.items[*].status.conditions[?(@.type=="Ready")].status}' \
      | grep "True" | wc -w)
    if [[ $statuses -eq $linux_command_replicas ]]; then
      break
    else
      sleep 10
      (( timeout=timeout-10 ))
    fi
  done

  if [[ $timeout -gt 0 ]]; then
    echo "All $linux_command_pod_label pods became Ready"
  else
    echo "ERROR: Not all $linux_command_pod_label pods became Ready"
    echo "kubectl get pods -l app=$linux_command_pod_label"
    ${kubectl} get pods -l app=$linux_command_pod_label
    cleanup_deployments
    exit 1
  fi
}

# Returns the name of an arbitrary Linux command pod.
function get_linux_command_pod_name {
  $kubectl get pods -l app=$linux_command_pod_label \
    -o jsonpath='{.items[0].metadata.name}'
}

# Returns the IP address of an arbitrary Linux command pod.
function get_linux_command_pod_ip {
  $kubectl get pods -l app=$linux_command_pod_label \
    -o jsonpath='{.items[0].status.podIP}'
}

# Installs test executables (ping, curl) in the Linux command pod.
# NOTE: this assumes that there is only one Linux "command pod".
# TODO(pjh): fix this.
function prepare_linux_command_pod {
  local linux_command_pod
  linux_command_pod="$(get_linux_command_pod_name)"

  echo "Installing test utilities in Linux command pod, may take a minute"
  $kubectl exec "$linux_command_pod" -- apt-get update > /dev/null
  $kubectl exec "$linux_command_pod" -- \
    apt-get install -y iputils-ping curl > /dev/null
}

function undeploy_linux_command_pod {
  ${kubectl} delete deployment $linux_command_deployment
}

windows_webserver_deployment=windows-agnhost
windows_webserver_pod_label=agnhost
# The default port for 'agnhost serve-hostname'. The documentation says that
# this can be changed but the --port arg does not seem to work.
windows_webserver_port=9376
windows_webserver_replicas=1

function deploy_windows_webserver_pod {
  echo "Writing example deployment to $windows_webserver_deployment.yaml"
  cat <<EOF > $windows_webserver_deployment.yaml
# A multi-arch Windows container that runs an HTTP server on port
# $windows_webserver_port that serves the container's hostname.
#   curl -s http://<pod_ip>:$windows_webserver_port
apiVersion: apps/v1
kind: Deployment
metadata:
  name: $windows_webserver_deployment
  labels:
    app: $windows_webserver_pod_label
spec:
  replicas: $windows_webserver_replicas
  selector:
    matchLabels:
      app: $windows_webserver_pod_label
  template:
    metadata:
      labels:
        app: $windows_webserver_pod_label
    spec:
      containers:
      - name: agnhost
        image: e2eteam/agnhost:2.26
        args:
        - serve-hostname
      nodeSelector:
        kubernetes.io/os: windows
      tolerations:
      - effect: NoSchedule
        key: node.kubernetes.io/os
        operator: Equal
        value: windows
EOF

  if ! ${kubectl} create -f $windows_webserver_deployment.yaml; then
    echo "kubectl create -f $windows_webserver_deployment.yaml failed"
    exit 1
  fi

  timeout=$windows_deployment_timeout
  while [[ $timeout -gt 0 ]]; do
    echo "Waiting for $windows_webserver_replicas Windows $windows_webserver_pod_label pods to become Ready"
    statuses=$(${kubectl} get pods -l app=$windows_webserver_pod_label \
      -o jsonpath='{.items[*].status.conditions[?(@.type=="Ready")].status}' \
      | grep "True" | wc -w)
    if [[ $statuses -eq $windows_webserver_replicas ]]; then
      break
    else
      sleep 10
      (( timeout=timeout-10 ))
    fi
  done

  if [[ $timeout -gt 0 ]]; then
    echo "All $windows_webserver_pod_label pods became Ready"
  else
    echo "ERROR: Not all $windows_webserver_pod_label pods became Ready"
    echo "kubectl get pods -l app=$windows_webserver_pod_label"
    ${kubectl} get pods -l app=$windows_webserver_pod_label
    cleanup_deployments
    exit 1
  fi
}

function get_windows_webserver_pod_name {
  $kubectl get pods -l app=$windows_webserver_pod_label \
    -o jsonpath='{.items[0].metadata.name}'
}

function get_windows_webserver_pod_ip {
  $kubectl get pods -l app=$windows_webserver_pod_label \
    -o jsonpath='{.items[0].status.podIP}'
}

function undeploy_windows_webserver_pod {
  ${kubectl} delete deployment $windows_webserver_deployment
}

windows_command_deployment=windows-powershell
windows_command_pod_label=powershell
windows_command_replicas=1

# Deploys a multi-arch Windows pod capable of running PowerShell.
function deploy_windows_command_pod {
  echo "Writing example deployment to $windows_command_deployment.yaml"
  cat <<EOF > $windows_command_deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: $windows_command_deployment
  labels:
    app: $windows_command_pod_label
spec:
  replicas: $windows_command_replicas
  selector:
    matchLabels:
      app: $windows_command_pod_label
  template:
    metadata:
      labels:
        app: $windows_command_pod_label
    spec:
      containers:
      - name: pause-win
        image: k8s.gcr.io/pause:3.5
      nodeSelector:
        kubernetes.io/os: windows
      tolerations:
      - effect: NoSchedule
        key: node.kubernetes.io/os
        operator: Equal
        value: windows
EOF

  if ! ${kubectl} create -f $windows_command_deployment.yaml; then
    echo "kubectl create -f $windows_command_deployment.yaml failed"
    exit 1
  fi

  timeout=$windows_deployment_timeout
  while [[ $timeout -gt 0 ]]; do
    echo "Waiting for $windows_command_replicas Windows $windows_command_pod_label pods to become Ready"
    statuses=$(${kubectl} get pods -l app=$windows_command_pod_label \
      -o jsonpath='{.items[*].status.conditions[?(@.type=="Ready")].status}' \
      | grep "True" | wc -w)
    if [[ $statuses -eq $windows_command_replicas ]]; then
      break
    else
      sleep 10
      (( timeout=timeout-10 ))
    fi
  done

  if [[ $timeout -gt 0 ]]; then
    echo "All $windows_command_pod_label pods became Ready"
  else
    echo "ERROR: Not all $windows_command_pod_label pods became Ready"
    echo "kubectl get pods -l app=$windows_command_pod_label"
    ${kubectl} get pods -l app=$windows_command_pod_label
    cleanup_deployments
    exit 1
  fi
}

function get_windows_command_pod_name {
  $kubectl get pods -l app=$windows_command_pod_label \
    -o jsonpath='{.items[0].metadata.name}'
}

function get_windows_command_pod_ip {
  $kubectl get pods -l app=$windows_command_pod_label \
    -o jsonpath='{.items[0].status.podIP}'
}

function undeploy_windows_command_pod {
  ${kubectl} delete deployment $windows_command_deployment
}

function test_linux_node_to_linux_pod {
  echo "TODO: ${FUNCNAME[0]}"
}

function test_linux_node_to_windows_pod {
  echo "TODO: ${FUNCNAME[0]}"
}

function test_linux_pod_to_linux_pod {
  echo "TEST: ${FUNCNAME[0]}"
  local linux_command_pod
  linux_command_pod="$(get_linux_command_pod_name)"
  local linux_webserver_pod_ip
  linux_webserver_pod_ip="$(get_linux_webserver_pod_ip)"

  if ! $kubectl exec "$linux_command_pod" -- curl -s -m 20 \
      "http://$linux_webserver_pod_ip" &> $output_file; then
    cleanup_deployments
    echo "Failing output: $(cat $output_file)"
    echo "FAILED: ${FUNCNAME[0]}"
    exit 1
  fi
}

# TODO(pjh): this test flakily fails on brand-new clusters, not sure why.
# % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
#                                Dload  Upload   Total   Spent    Left  Speed
# 0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
# curl: (6) Could not resolve host:
# command terminated with exit code 6
function test_linux_pod_to_windows_pod {
  echo "TEST: ${FUNCNAME[0]}"
  local linux_command_pod
  linux_command_pod="$(get_linux_command_pod_name)"
  local windows_webserver_pod_ip
  windows_webserver_pod_ip="$(get_windows_webserver_pod_ip)"

  if ! $kubectl exec "$linux_command_pod" -- curl -s -m 20 \
      "http://$windows_webserver_pod_ip:$windows_webserver_port" &> $output_file; then
    cleanup_deployments
    echo "Failing output: $(cat $output_file)"
    echo "FAILED: ${FUNCNAME[0]}"
    echo "This test seems to be flaky. TODO(pjh): investigate."
    exit 1
  fi
}

function test_linux_pod_to_internet {
  echo "TEST: ${FUNCNAME[0]}"
  local linux_command_pod
  linux_command_pod="$(get_linux_command_pod_name)"
  # A stable (hopefully) HTTP server provided by Cloudflare.
  local internet_ip="1.1.1.1"

  if ! $kubectl exec "$linux_command_pod" -- curl -s -m 20 \
      "http://$internet_ip" > $output_file; then
    cleanup_deployments
    echo "Failing output: $(cat $output_file)"
    echo "FAILED: ${FUNCNAME[0]}"
    exit 1
  fi
}

function test_linux_pod_to_k8s_service {
  echo "TEST: ${FUNCNAME[0]}"
  local linux_command_pod
  linux_command_pod="$(get_linux_command_pod_name)"
  local service="metrics-server"
  local service_ip
  service_ip=$($kubectl get service --namespace kube-system $service \
    -o jsonpath='{.spec.clusterIP}')
  local service_port
  service_port=$($kubectl get service --namespace kube-system $service \
    -o jsonpath='{.spec.ports[?(@.protocol=="TCP")].port}')
  echo "curl-ing $service address from Linux pod: $service_ip:$service_port"

  # curl-ing the metrics-server service downloads 14 bytes of unprintable binary
  # data and sets a return code of success (0).
  if ! $kubectl exec "$linux_command_pod" -- \
      curl -s -m 20 --insecure "https://$service_ip:$service_port" &> $output_file; then
    cleanup_deployments
    echo "Failing output: $(cat $output_file)"
    echo "FAILED: ${FUNCNAME[0]}"
    exit 1
  fi
}

function test_windows_node_to_linux_pod {
  echo "TODO: ${FUNCNAME[0]}"
}

function test_windows_node_to_windows_pod {
  echo "TODO: ${FUNCNAME[0]}"
}

# TODO(pjh): this test failed for me once with
#   error: unable to upgrade connection: container not found ("nettest")
# Maybe the container crashed for some reason? Investigate if it happens more.
#
# TODO(pjh): another one-time failure:
#   error: unable to upgrade connection: Authorization error
#   (user=kube-apiserver, verb=create, resource=nodes, subresource=proxy)
function test_windows_pod_to_linux_pod {
  echo "TEST: ${FUNCNAME[0]}"
  local windows_command_pod
  windows_command_pod="$(get_windows_command_pod_name)"
  local linux_webserver_pod_ip
  linux_webserver_pod_ip="$(get_linux_webserver_pod_ip)"

  if ! $kubectl exec "$windows_command_pod" -- powershell.exe \
      "curl -UseBasicParsing http://$linux_webserver_pod_ip" > \
      $output_file; then
    cleanup_deployments
    echo "Failing output: $(cat $output_file)"
    echo "FAILED: ${FUNCNAME[0]}"
    exit 1
  fi
}

function test_windows_pod_to_windows_pod {
  echo "TEST: ${FUNCNAME[0]}"
  local windows_command_pod
  windows_command_pod="$(get_windows_command_pod_name)"
  local windows_webserver_pod_ip
  windows_webserver_pod_ip="$(get_windows_webserver_pod_ip)"

  if ! $kubectl exec "$windows_command_pod" -- powershell.exe \
      "curl -UseBasicParsing http://$windows_webserver_pod_ip:$windows_webserver_port" \
      > $output_file; then
    cleanup_deployments
    echo "Failing output: $(cat $output_file)"
    echo "FAILED: ${FUNCNAME[0]}"
    exit 1
  fi
}

function test_windows_pod_to_internet {
  echo "TEST: ${FUNCNAME[0]}"
  local windows_command_pod
  windows_command_pod="$(get_windows_command_pod_name)"
  # A stable (hopefully) HTTP server provided by Cloudflare. If this ever stops
  # working, we can request from 8.8.8.8 (Google DNS) using https instead.
  local internet_ip="1.1.1.1"

  if ! $kubectl exec "$windows_command_pod" -- powershell.exe \
      "curl -UseBasicParsing http://$internet_ip" > $output_file; then
    cleanup_deployments
    echo "Failing output: $(cat $output_file)"
    echo "FAILED: ${FUNCNAME[0]}"
    exit 1
  fi
}

function test_windows_pod_to_k8s_service {
  echo "TEST: ${FUNCNAME[0]}"
  local windows_command_pod
  windows_command_pod="$(get_windows_command_pod_name)"
  local service="metrics-server"
  local service_ip
  service_ip=$($kubectl get service --namespace kube-system $service \
    -o jsonpath='{.spec.clusterIP}')
  local service_port
  service_port=$($kubectl get service --namespace kube-system $service \
    -o jsonpath='{.spec.ports[?(@.protocol=="TCP")].port}')
  local service_address="$service_ip:$service_port"

  echo "curl-ing $service address from Windows pod: $service_address"
  # curl-ing the metrics-server service results in a ServerProtocolViolation
  # ("The server committed a protocol violation. Section=ResponseStatusLine")
  # exception. Since we don't care about what the metrics-server actually gives
  # back to us, just that we can reach it, we check that we get the expected
  # exception code and not some other exception code.
  # TODO: it might be less fragile to check that we don't get the "Unable to
  # connect to the remote server" exception code (2) instead of specifically
  # expecting the protocol-violation exception code (11).
  if ! $kubectl exec "$windows_command_pod" -- powershell.exe \
      "\$result = try { \`
         curl -UseBasicParsing http://$service_address -ErrorAction Stop \`
       } catch [System.Net.WebException] { \`
         \$_ \`
       }; \`
       if ([int]\$result.Exception.Status -eq 11) { \`
         Write-Host \"curl $service_address got expected exception\"
         exit 0 \`
       } else { \`
         Write-Host \"curl $service_address got unexpected result/exception: \$result\"
         exit 1 \`
       }" > $output_file; then
    cleanup_deployments
    echo "Failing output: $(cat $output_file)"
    echo "FAILED: ${FUNCNAME[0]}"
    exit 1
  fi
}

function test_kube_dns_in_windows_pod {
  echo "TEST: ${FUNCNAME[0]}"
  local windows_command_pod
  windows_command_pod="$(get_windows_command_pod_name)"
  local service="kube-dns"
  local service_ip
  service_ip=$($kubectl get service --namespace kube-system $service \
    -o jsonpath='{.spec.clusterIP}')

  if ! $kubectl exec "$windows_command_pod" -- powershell.exe \
      "Resolve-DnsName www.bing.com -server $service_ip" > $output_file; then
    cleanup_deployments
    echo "Failing output: $(cat $output_file)"
    echo "FAILED: ${FUNCNAME[0]}"
    exit 1
  fi
}

function test_dns_just_works_in_windows_pod {
  echo "TEST: ${FUNCNAME[0]}"
  local windows_command_pod
  windows_command_pod="$(get_windows_command_pod_name)"

  if ! $kubectl exec "$windows_command_pod" -- powershell.exe \
      "curl -UseBasicParsing http://www.bing.com" > $output_file; then
    cleanup_deployments
    echo "Failing output: $(cat $output_file)"
    echo "FAILED: ${FUNCNAME[0]}"
    exit 1
  fi
}

function cleanup_deployments {
  undeploy_linux_webserver_pod
  undeploy_linux_command_pod
  undeploy_windows_webserver_pod
  undeploy_windows_command_pod
}

check_windows_nodes_are_ready
untaint_windows_nodes
check_no_system_pods_on_windows_nodes

deploy_linux_webserver_pod
deploy_linux_command_pod
deploy_windows_webserver_pod
deploy_windows_command_pod
prepare_linux_command_pod
echo ""

test_linux_node_to_linux_pod
test_linux_node_to_windows_pod
test_linux_pod_to_linux_pod
test_linux_pod_to_windows_pod
test_linux_pod_to_k8s_service

# Note: test_windows_node_to_k8s_service is not supported at this time.
# https://docs.microsoft.com/en-us/virtualization/windowscontainers/kubernetes/common-problems#my-windows-node-cannot-access-my-services-using-the-service-ip
test_windows_node_to_linux_pod
test_windows_node_to_windows_pod
test_windows_pod_to_linux_pod
test_windows_pod_to_windows_pod
test_windows_pod_to_internet
test_windows_pod_to_k8s_service
test_kube_dns_in_windows_pod
test_dns_just_works_in_windows_pod
echo ""

cleanup_deployments
echo "All tests passed!"
exit 0


#!/usr/bin/env bash
# Copyright 2018 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -o errexit
set -o nounset
set -o pipefail

kubectl create -f conformance-e2e.yaml
while true; do
  STATUS=$(kubectl -n conformance get pods e2e-conformance-test -o jsonpath="{.status.phase}")
  timestamp=$(date +"[%H:%M:%S]")
  echo "$timestamp Pod status is: ${STATUS}"
  if [[ "$STATUS" == "Succeeded" ]]; then
    echo "$timestamp Done."
    break
  elif [[ "$STATUS" == "Failed" ]]; then
    echo "$timestamp Failed."
    kubectl -n conformance describe pods e2e-conformance-test || true
    kubectl -n conformance logs e2e-conformance-test || true
    exit 1
  else
    sleep 5
  fi
done
echo "Please use 'kubectl logs -n conformance e2e-conformance-test' to view the results"


#!/usr/bin/env bash
# Copyright 2018 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -o errexit
set -o nounset
set -o pipefail

# Shutdown the tests gracefully then save the results
shutdown () {
    E2E_SUITE_PID=$(pgrep e2e.test)
    echo "sending TERM to ${E2E_SUITE_PID}"
    kill -s TERM "${E2E_SUITE_PID}"

    # Kind of a hack to wait for this pid to finish.
    # Since it's not a child of this shell we cannot use wait.
    tail --pid "${E2E_SUITE_PID}" -f /dev/null
    saveResults
}

saveResults() {
    cd "${RESULTS_DIR}" || exit
    tar -czf e2e.tar.gz ./*
    # mark the done file as a termination notice.
    echo -n "${RESULTS_DIR}/e2e.tar.gz" > "${RESULTS_DIR}/done"
}

# Optional Golang runner alternative to the bash script.
# Entry provided via env var to simplify invocation.
if [[ -n ${E2E_USE_GO_RUNNER:-} ]]; then
    set -x
    /gorunner && ret=0 || ret=$?
    exit ${ret}
fi

# We get the TERM from kubernetes and handle it gracefully
trap shutdown TERM

ginkgo_args=()
if [[ -n ${E2E_DRYRUN:-} ]]; then
    ginkgo_args+=("--dryRun=true")
fi

case ${E2E_PARALLEL} in
    'y'|'Y'|'true')
        # The flag '--p' will automatically detect the optimal number of ginkgo nodes.
        ginkgo_args+=("--p")
        # Skip serial tests if parallel mode is enabled.
        E2E_SKIP="\\[Serial\\]|${E2E_SKIP}" ;;
esac

ginkgo_args+=(
    "--focus=${E2E_FOCUS}"
    "--skip=${E2E_SKIP}"
    "--noColor=true"
)

set -x
/usr/local/bin/ginkgo "${ginkgo_args[@]}" /usr/local/bin/e2e.test -- --disable-log-dump --repo-root=/kubernetes --provider="${E2E_PROVIDER}" --report-dir="${RESULTS_DIR}" --kubeconfig="${KUBECONFIG}" -v="${E2E_VERBOSITY}" > >(tee "${RESULTS_DIR}"/e2e.log) && ret=0 || ret=$?
set +x
saveResults
exit ${ret}


#!/bin/sh

# Copyright 2016 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# DEPRECATED:
# The functionality has been moved to migrate binary and this script
# if left for backward compatibility with previous manifests. It will be
# removed in the future.

set -o errexit
set -o nounset

/usr/local/bin/migrate


#!/usr/bin/env bash

# Copyright 2014 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../..
source "${KUBE_ROOT}/test/kubemark/cloud-provider-config.sh"
source "${KUBE_ROOT}/cluster/${CLOUD_PROVIDER}/util.sh"
source "${KUBE_ROOT}/cluster/kubemark/${CLOUD_PROVIDER}/config-default.sh"


#!/usr/bin/env bash

# Copyright 2017 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# A configuration for Kubemark cluster. It doesn't need to be kept in
# sync with gce/config-default.sh (except the filename, because I'm reusing
# gce/util.sh script which assumes config filename), but if some things that
# are enabled by default should not run in hollow clusters, they should be disabled here.

# shellcheck disable=SC2034 # Variables sourced in other scripts.

source "${KUBE_ROOT}/cluster/gce/config-common.sh"

CLEANUP_KUBEMARK_IMAGE=${CLEANUP_KUBEMARK_IMAGE:-true}
TEST_CLUSTER_LOG_LEVEL="${TEST_CLUSTER_LOG_LEVEL:-"--v=4"}"

# If you want to set up multiple kubemark clusters with different "names",
# you should change this env per each start-kubemark.sh invocation.
KUBE_GCE_INSTANCE_PREFIX="${KUBE_GCE_INSTANCE_PREFIX:-"default"}"

# NUM_NODES is used by start-kubemark.sh to determine a correct number of replicas.
NUM_NODES=${KUBEMARK_NUM_NODES:-10}
NUM_WINDOWS_NODES=${KUBEMARK_NUM_WINDOWS_NODES:-0}

HOLLOW_KUBELET_TEST_LOG_LEVEL="${HOLLOW_KUBELET_TEST_LOG_LEVEL:-$TEST_CLUSTER_LOG_LEVEL}"
HOLLOW_PROXY_TEST_LOG_LEVEL="${HOLLOW_PROXY_TEST_LOG_LEVEL:-$TEST_CLUSTER_LOG_LEVEL}"

# Hollow-node components' test arguments.
HOLLOW_KUBELET_TEST_ARGS="${HOLLOW_KUBELET_TEST_ARGS:-} ${HOLLOW_KUBELET_TEST_LOG_LEVEL}"
HOLLOW_PROXY_TEST_ARGS="${HOLLOW_PROXY_TEST_ARGS:-} ${HOLLOW_PROXY_TEST_LOG_LEVEL}"

# Optional: Enable cluster autoscaler.
ENABLE_KUBEMARK_CLUSTER_AUTOSCALER="${ENABLE_KUBEMARK_CLUSTER_AUTOSCALER:-false}"
# When using Cluster Autoscaler, always start with one hollow-node replica.
# NUM_NODES should not be specified by the user. Instead we use
# NUM_NODES=KUBEMARK_AUTOSCALER_MAX_NODES. This gives other cluster components
# (e.g. kubemark master, Heapster) enough resources to handle maximum cluster size.
if [[ "${ENABLE_KUBEMARK_CLUSTER_AUTOSCALER}" == "true" ]]; then
  NUM_REPLICAS=1
  if [[ -n "$NUM_NODES" ]]; then
    echo "WARNING: Using Cluster Autoscaler, ignoring NUM_NODES parameter. Set KUBEMARK_AUTOSCALER_MAX_NODES to specify maximum size of the cluster."
  fi
fi

#Optional: Enable kube dns.
ENABLE_KUBEMARK_KUBE_DNS="${ENABLE_KUBEMARK_KUBE_DNS:-true}"
KUBE_DNS_DOMAIN="${KUBE_DNS_DOMAIN:-cluster.local}"

CLEANUP_KUBEMARK_IMAGE=false


#!/usr/bin/env bash

# Copyright 2018 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Cloud information
RANDGEN=$(dd if=/dev/urandom bs=64 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=16 count=1 2>/dev/null | sed 's/[A-Z]//g')
# shellcheck disable=2034 # Variable sourced in other scripts.
KUBE_NAMESPACE="kubemark_${RANDGEN}"
KUBEMARK_IMAGE_TAG="${KUBEMARK_IMAGE_TAG:-2}"
KUBEMARK_IMAGE_LOCATION="${KUBEMARK_IMAGE_LOCATION:-${KUBE_ROOT}/cluster/images/kubemark}"
KUBEMARK_INIT_TAG="${KUBEMARK_INIT_TAG:-${PROJECT}:${KUBEMARK_IMAGE_TAG}}"
CLUSTER_LOCATION="${CLUSTER_LOCATION:-wdc06}"
REGISTRY_LOGIN_URL="${REGISTRY_LOGIN_URL:-https://api.ng.bluemix.net}"

# User defined
# number of real workers in spawnTester cluster
NUM_NODES="${NUM_NODES:-2}"
# spec of real workers in spawnTester cluster
NODE_SIZE=${NODE_SIZE:-u2c.2x4}
DESIRED_NODES="${DESIRED_NODES:-10}"
# number of hollow nodes
# TODO: once IKS supports `kubetest`, i.e. the base cluster provisioning implemented in `kubetest`
# we can use NUM_NODES=${KUBEMARK_NUM_NODES:-10} to avoid usage of KUBEMARK_NUM_NODES
# more context, see https://github.com/kubernetes/kubernetes/pull/76909#discussion_r277735942
KUBEMARK_NUM_NODES="${KUBEMARK_NUM_NODES:-10}"
ENABLE_KUBEMARK_CLUSTER_AUTOSCALER="${ENABLE_KUBEMARK_CLUSTER_AUTOSCALER:-true}"
ENABLE_KUBEMARK_KUBE_DNS="${ENABLE_KUBEMARK_KUBE_DNS:-false}"
KUBELET_TEST_LOG_LEVEL="${KUBELET_TEST_LOG_LEVEL:-"--v=2"}"
KUBEPROXY_TEST_LOG_LEVEL="${KUBEPROXY_TEST_LOG_LEVEL:-"--v=4"}"
USE_REAL_PROXIER=${USE_REAL_PROXIER:-false}
NODE_INSTANCE_PREFIX=${NODE_INSTANCE_PREFIX:-node}
USE_EXISTING=${USE_EXISTING:-}


#!/usr/bin/env bash
# Copyright 2017 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Configuration for landing a Kubemark cluster on a pre-existing Kubernetes
# cluster.

# Pre-existing provider expects a MASTER_IP.
# If you need to specify a port that's not the default (443), add it to MASTER_IP.
#
# Example: Connect to the Master on the secure port 6443
#          MASTER_IP=192.168.122.5:6443
#
MASTER_IP="${MASTER_IP:-}"

# The container registry and project given to the kubemark container:
#   $CONTAINER_REGISTRY/$PROJECT/kubemark
#
CONTAINER_REGISTRY="${CONTAINER_REGISTRY:-}"
PROJECT="${PROJECT:-}"

NUM_NODES="${NUM_NODES:-1}"

KUBELET_TEST_LOG_LEVEL="${KUBELET_TEST_LOG_LEVEL:-}"
KUBEPROXY_TEST_LOG_LEVEL="${KUBEPROXY_TEST_LOG_LEVEL:-}"
MASTER_NAME="${MASTER_NAME:-}"


#!/usr/bin/env bash

# Copyright 2017 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Call this to dump all master and node logs into the folder specified in $1
# (defaults to _artifacts). Only works if the provider supports SSH.

# TODO(shyamjvs): This script should be moved to test/e2e which is where it ideally belongs.
set -o errexit
set -o nounset
set -o pipefail

readonly report_dir="${1:-_artifacts}"
readonly gcs_artifacts_dir="${2:-}"
readonly logexporter_namespace="${3:-logexporter}"

# In order to more trivially extend log-dump for custom deployments,
# check for a function named log_dump_custom_get_instances. If it's
# defined, we assume the function can me called with one argument, the
# role, which is either "master" or "node".
echo 'Checking for custom logdump instances, if any'
if [[ $(type -t log_dump_custom_get_instances) == "function" ]]; then
  readonly use_custom_instance_list=yes
else
  readonly use_custom_instance_list=
fi

readonly master_ssh_supported_providers="gce aws"
readonly node_ssh_supported_providers="gce gke aws"
readonly gcloud_supported_providers="gce gke"

readonly master_logfiles="kube-apiserver.log kube-apiserver-audit.log kube-scheduler.log kube-controller-manager.log etcd.log etcd-events.log glbc.log cluster-autoscaler.log kube-addon-manager.log konnectivity-server.log fluentd.log kubelet.cov"
readonly node_logfiles="kube-proxy.log fluentd.log node-problem-detector.log kubelet.cov"
readonly node_systemd_services="node-problem-detector"
readonly hollow_node_logfiles="kubelet-hollow-node-*.log kubeproxy-hollow-node-*.log npd-hollow-node-*.log"
readonly aws_logfiles="cloud-init-output.log"
readonly gce_logfiles="startupscript.log"
readonly kern_logfile="kern.log"
readonly initd_logfiles="docker/log"
readonly supervisord_logfiles="kubelet.log supervisor/supervisord.log supervisor/kubelet-stdout.log supervisor/kubelet-stderr.log supervisor/docker-stdout.log supervisor/docker-stderr.log"
readonly systemd_services="kubelet kubelet-monitor kube-container-runtime-monitor ${LOG_DUMP_SYSTEMD_SERVICES:-docker}"
readonly extra_log_files="${LOG_DUMP_EXTRA_FILES:-}"
readonly extra_systemd_services="${LOG_DUMP_SAVE_SERVICES:-}"
readonly dump_systemd_journal="${LOG_DUMP_SYSTEMD_JOURNAL:-false}"
# Log files found in WINDOWS_LOGS_DIR on Windows nodes:
readonly windows_node_logfiles="kubelet.log kube-proxy.log docker.log docker_images.log csi-proxy.log"
# Log files found in other directories on Windows nodes:
readonly windows_node_otherfiles="C:\\Windows\\MEMORY.dmp"

# Limit the number of concurrent node connections so that we don't run out of
# file descriptors for large clusters.
readonly max_dump_processes=25

# Indicator variable whether we experienced a significant failure during
# logexporter creation or execution.
logexporter_failed=0

# Percentage of nodes that must be logexported successfully (otherwise the
# process will exit with a non-zero exit code).
readonly log_dump_expected_success_percentage="${LOG_DUMP_EXPECTED_SUCCESS_PERCENTAGE:-0}"

function print-deprecation-note() {
  local -r dashline=$(printf -- '-%.0s' {1..100})
  echo "${dashline}"
  echo "k/k version of the log-dump.sh script is deprecated!"
  echo "Please migrate your test job to use test-infra's repo version of log-dump.sh!"
  echo "Migration steps can be found in the readme file."
  echo "${dashline}"
}

# TODO: Get rid of all the sourcing of bash dependencies eventually.
function setup() {
  KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../..
  if [[ -z "${use_custom_instance_list}" ]]; then
    : "${KUBE_CONFIG_FILE:=config-test.sh}"
    echo 'Sourcing kube-util.sh'
    source "${KUBE_ROOT}/cluster/kube-util.sh"
    echo 'Detecting project'
    detect-project 2>&1
  elif [[ "${KUBERNETES_PROVIDER}" == "gke" ]]; then
    echo "Using 'use_custom_instance_list' with gke, skipping check for LOG_DUMP_SSH_KEY and LOG_DUMP_SSH_USER"
    # Source the below script for the ssh-to-node utility function.
    # Hack to save and restore the value of the ZONE env as the script overwrites it.
    local gke_zone="${ZONE:-}"
    source "${KUBE_ROOT}/cluster/gce/util.sh"
    ZONE="${gke_zone}"
  elif [[ -z "${LOG_DUMP_SSH_KEY:-}" ]]; then
    echo 'LOG_DUMP_SSH_KEY not set, but required when using log_dump_custom_get_instances'
    exit 1
  elif [[ -z "${LOG_DUMP_SSH_USER:-}" ]]; then
    echo 'LOG_DUMP_SSH_USER not set, but required when using log_dump_custom_get_instances'
    exit 1
  fi
  source "${KUBE_ROOT}/hack/lib/util.sh"
}

function log-dump-ssh() {
  if [[ "${gcloud_supported_providers}" =~ ${KUBERNETES_PROVIDER} ]]; then
    ssh-to-node "$@"
    return
  fi

  local host="$1"
  local cmd="$2"

  ssh -oLogLevel=quiet -oConnectTimeout=30 -oStrictHostKeyChecking=no -i "${LOG_DUMP_SSH_KEY}" "${LOG_DUMP_SSH_USER}@${host}" "${cmd}"
}

# Copy all files /var/log/{$3}.log on node $1 into local dir $2.
# $3 should be a string array of file names.
# This function shouldn't ever trigger errexit, but doesn't block stderr.
function copy-logs-from-node() {
    local -r node="${1}"
    local -r dir="${2}"
    shift
    shift
    local files=("$@")
    # Append "*"
    # The * at the end is needed to also copy rotated logs (which happens
    # in large clusters and long runs).
    files=( "${files[@]/%/*}" )
    # Prepend "/var/log/"
    files=( "${files[@]/#/\/var\/log\/}" )
    # Comma delimit (even the singleton, or scp does the wrong thing), surround by braces.
    local -r scp_files="{$(printf "%s," "${files[@]}")}"

    if [[ "${gcloud_supported_providers}" =~ ${KUBERNETES_PROVIDER} ]]; then
      # get-serial-port-output lets you ask for ports 1-4, but currently (11/21/2016) only port 1 contains useful information
      gcloud compute instances get-serial-port-output --project "${PROJECT}" --zone "${ZONE}" --port 1 "${node}" > "${dir}/serial-1.log" || true
      gcloud compute scp --recurse --project "${PROJECT}" --zone "${ZONE}" "${node}:${scp_files}" "${dir}" > /dev/null || true
    elif  [[ "${KUBERNETES_PROVIDER}" == "aws" ]]; then
      local ip
      ip=$(get_ssh_hostname "${node}")
      scp -oLogLevel=quiet -oConnectTimeout=30 -oStrictHostKeyChecking=no -i "${AWS_SSH_KEY}" "${SSH_USER}@${ip}:${scp_files}" "${dir}" > /dev/null || true
    elif  [[ -n "${use_custom_instance_list}" ]]; then
      scp -oLogLevel=quiet -oConnectTimeout=30 -oStrictHostKeyChecking=no -i "${LOG_DUMP_SSH_KEY}" "${LOG_DUMP_SSH_USER}@${node}:${scp_files}" "${dir}" > /dev/null || true
    else
      echo "Unknown cloud-provider '${KUBERNETES_PROVIDER}' and use_custom_instance_list is unset too - skipping logdump for '${node}'"
    fi
}

# Save logs for node $1 into directory $2. Pass in any non-common files in $3.
# Pass in any non-common systemd services in $4.
# $3 and $4 should be a space-separated list of files.
# Set $5 to true to indicate it is on master. Default to false.
# This function shouldn't ever trigger errexit
function save-logs() {
    local -r node_name="${1}"
    local -r dir="${2}"
    local files=()
    IFS=' ' read -r -a files <<< "$3"
    local opt_systemd_services="${4:-""}"
    local on_master="${5:-"false"}"

    local extra=()
    IFS=' ' read -r -a extra <<< "$extra_log_files"
    files+=("${extra[@]}")
    if [[ -n "${use_custom_instance_list}" ]]; then
      if [[ -n "${LOG_DUMP_SAVE_LOGS:-}" ]]; then
	local dump=()
        IFS=' ' read -r -a dump <<< "${LOG_DUMP_SAVE_LOGS:-}"
        files+=("${dump[@]}")
      fi
    else
      local providerlogs=()
      case "${KUBERNETES_PROVIDER}" in
        gce|gke)
          IFS=' ' read -r -a providerlogs <<< "${gce_logfiles}"
          ;;
        aws)
          IFS=' ' read -r -a providerlogs <<< "${aws_logfiles}"
          ;;
      esac
      files+=("${providerlogs[@]}")
    fi
    local services
    read -r -a services <<< "${systemd_services} ${opt_systemd_services} ${extra_systemd_services}"

    if log-dump-ssh "${node_name}" "command -v journalctl" &> /dev/null; then
        if [[ "${on_master}" == "true" ]]; then
          log-dump-ssh "${node_name}" "sudo journalctl --output=short-precise -u kube-master-installation.service" > "${dir}/kube-master-installation.log" || true
          log-dump-ssh "${node_name}" "sudo journalctl --output=short-precise -u kube-master-configuration.service" > "${dir}/kube-master-configuration.log" || true
        else
          log-dump-ssh "${node_name}" "sudo journalctl --output=short-precise -u kube-node-installation.service" > "${dir}/kube-node-installation.log" || true
          log-dump-ssh "${node_name}" "sudo journalctl --output=short-precise -u kube-node-configuration.service" > "${dir}/kube-node-configuration.log" || true
        fi
        log-dump-ssh "${node_name}" "sudo journalctl --output=short-precise -k" > "${dir}/kern.log" || true

        for svc in "${services[@]}"; do
            log-dump-ssh "${node_name}" "sudo journalctl --output=cat -u ${svc}.service" > "${dir}/${svc}.log" || true
        done

        if [[ "$dump_systemd_journal" == "true" ]]; then
          log-dump-ssh "${node_name}" "sudo journalctl --output=short-precise" > "${dir}/systemd.log" || true
        fi
    else
        local tmpfiles=()
        for f in "${kern_logfile}" "${initd_logfiles}" "${supervisord_logfiles}"; do
	    IFS=' ' read -r -a tmpfiles <<< "$f"
	    files+=("${tmpfiles[@]}")
        done
    fi

    # Try dumping coverage profiles, if it looks like coverage is enabled in the first place.
    if log-dump-ssh "${node_name}" "stat /var/log/kubelet.cov" &> /dev/null; then
      if log-dump-ssh "${node_name}" "command -v docker" &> /dev/null; then
        if [[ "${on_master}" == "true" ]]; then
          run-in-docker-container "${node_name}" "kube-apiserver" "cat /tmp/k8s-kube-apiserver.cov" > "${dir}/kube-apiserver.cov" || true
          run-in-docker-container "${node_name}" "kube-scheduler" "cat /tmp/k8s-kube-scheduler.cov" > "${dir}/kube-scheduler.cov" || true
          run-in-docker-container "${node_name}" "kube-controller-manager" "cat /tmp/k8s-kube-controller-manager.cov" > "${dir}/kube-controller-manager.cov" || true
        else
          run-in-docker-container "${node_name}" "kube-proxy" "cat /tmp/k8s-kube-proxy.cov" > "${dir}/kube-proxy.cov" || true
        fi
      else
        echo 'Coverage profiles seem to exist, but cannot be retrieved from inside containers.'
      fi
    fi

    echo 'Changing logfiles to be world-readable for download'
    log-dump-ssh "${node_name}" "sudo chmod -R a+r /var/log" || true

    echo "Copying '${files[*]}' from ${node_name}"
    copy-logs-from-node "${node_name}" "${dir}" "${files[@]}"
}

# Saves a copy of the Windows Docker event log to ${WINDOWS_LOGS_DIR}\docker.log
# on node $1.
function export-windows-docker-event-log() {
    local -r node="${1}"

    local -r powershell_cmd="powershell.exe -Command \"\$logs=\$(Get-EventLog -LogName Application -Source Docker | Format-Table -Property TimeGenerated, EntryType, Message -Wrap); \$logs | Out-File -FilePath '${WINDOWS_LOGS_DIR}\\docker.log'\""

    # Retry up to 3 times to allow ssh keys to be properly propagated and
    # stored.
    for retry in {1..3}; do
      if gcloud compute ssh --project "${PROJECT}" --zone "${ZONE}" "${node}" \
        --command "$powershell_cmd"; then
        break
      else
        sleep 10
      fi
    done
}

# Saves prepulled Windows Docker images list to ${WINDOWS_LOGS_DIR}\docker_images.log
# on node $1.
function export-windows-docker-images-list() {
    local -r node="${1}"

    local -r powershell_cmd="powershell.exe -Command \"\$logs=\$(docker image list); \$logs | Out-File -FilePath '${WINDOWS_LOGS_DIR}\\docker_images.log'\""

    # Retry up to 3 times to allow ssh keys to be properly propagated and
    # stored.
    for retry in {1..3}; do
      if gcloud compute ssh --project "${PROJECT}" --zone "${ZONE}" "${node}" \
        --command "$powershell_cmd"; then
        break
      else
        sleep 10
      fi
    done
}

# Saves log files from diagnostics tool.(https://github.com/GoogleCloudPlatform/compute-image-tools/tree/master/cli_tools/diagnostics)
function save-windows-logs-via-diagnostics-tool() {
    local node="${1}"
    local dest_dir="${2}"

    gcloud compute instances add-metadata "${node}" --metadata enable-diagnostics=true --project="${PROJECT}" --zone="${ZONE}"
    local logs_archive_in_gcs
    logs_archive_in_gcs=$(gcloud alpha compute diagnose export-logs "${node}" "--zone=${ZONE}" "--project=${PROJECT}" | tail -n 1)
    local temp_local_path="${node}.zip"
    for retry in {1..20}; do
      if gsutil mv "${logs_archive_in_gcs}" "${temp_local_path}"  > /dev/null 2>&1; then
        echo "Downloaded diagnostics log from ${logs_archive_in_gcs}"
        break
      else
        sleep 10
      fi
    done

    if [[ -f "${temp_local_path}" ]]; then
      unzip "${temp_local_path}" -d "${dest_dir}" > /dev/null
      rm -f "${temp_local_path}"
    fi
}

# Saves log files from SSH
function save-windows-logs-via-ssh() {
    local node="${1}"
    local dest_dir="${2}"

    export-windows-docker-event-log "${node}"
    export-windows-docker-images-list "${node}"

    local remote_files=()
    for file in "${windows_node_logfiles[@]}"; do
      remote_files+=( "${WINDOWS_LOGS_DIR}\\${file}" )
    done
    remote_files+=( "${windows_node_otherfiles[@]}" )

    # TODO(pjh, yujuhong): handle rotated logs and copying multiple files at the
    # same time.
    for remote_file in "${remote_files[@]}"; do
      # Retry up to 3 times to allow ssh keys to be properly propagated and
      # stored.
      for retry in {1..3}; do
        if gcloud compute scp --recurse --project "${PROJECT}" \
          --zone "${ZONE}" "${node}:${remote_file}" "${dest_dir}" \
          > /dev/null; then
          break
        else
          sleep 10
        fi
      done
    done
}

# Save log files and serial console output from Windows node $1 into local
# directory $2.
# This function shouldn't ever trigger errexit.
function save-logs-windows() {
    local -r node="${1}"
    local -r dest_dir="${2}"

    if [[ ! "${gcloud_supported_providers}" =~ ${KUBERNETES_PROVIDER} ]]; then
      echo "Not saving logs for ${node}, Windows log dumping requires gcloud support"
      return
    fi

    if [[ "${KUBERNETES_PROVIDER}" == "gke" ]]; then
      save-windows-logs-via-diagnostics-tool "${node}" "${dest_dir}"
    else
      save-windows-logs-via-ssh "${node}" "${dest_dir}"
    fi

    # Serial port 1 contains the Windows console output.
    gcloud compute instances get-serial-port-output --project "${PROJECT}" \
      --zone "${ZONE}" --port 1 "${node}" > "${dest_dir}/serial-1.log" || true
}

# Execute a command in container $2 on node $1.
# Uses docker because the container may not ordinarily permit direct execution.
function run-in-docker-container() {
  local node_name="$1"
  local container="$2"
  shift 2
  log-dump-ssh "${node_name}" "docker exec \"\$(docker ps -f label=io.kubernetes.container.name=${container} --format \"{{.ID}}\")\" $*"
}

function dump_masters() {
  local master_names=()
  if [[ -n "${use_custom_instance_list}" ]]; then
    while IFS='' read -r line; do master_names+=("$line"); done < <(log_dump_custom_get_instances master)
  elif [[ ! "${master_ssh_supported_providers}" =~ ${KUBERNETES_PROVIDER} ]]; then
    echo "Master SSH not supported for ${KUBERNETES_PROVIDER}"
    return
  elif [[ -n "${KUBEMARK_MASTER_NAME:-}" ]]; then
    master_names=( "${KUBEMARK_MASTER_NAME}" )
  else
    if ! (detect-master); then
      echo 'Master not detected. Is the cluster up?'
      return
    fi
    master_names=( "${MASTER_NAME}" )
  fi

  if [[ "${#master_names[@]}" == 0 ]]; then
    echo 'No masters found?'
    return
  fi

  proc=${max_dump_processes}
  for master_name in "${master_names[@]}"; do
    master_dir="${report_dir}/${master_name}"
    mkdir -p "${master_dir}"
    save-logs "${master_name}" "${master_dir}" "${master_logfiles}" "" "true" &

    # We don't want to run more than ${max_dump_processes} at a time, so
    # wait once we hit that many nodes. This isn't ideal, since one might
    # take much longer than the others, but it should help.
    proc=$((proc - 1))
    if [[ proc -eq 0 ]]; then
      proc=${max_dump_processes}
      wait
    fi
  done
  # Wait for any remaining processes.
  if [[ proc -gt 0 && proc -lt ${max_dump_processes} ]]; then
    wait
  fi
}

# Dumps logs from nodes in the cluster. Linux nodes to dump logs from can be
# specified via $1 or $use_custom_instance_list. If not specified then the nodes
# to dump logs for will be detected using detect-node-names(); if Windows nodes
# are present then they will be detected and their logs will be dumped too.
function dump_nodes() {
  local node_names=()
  local windows_node_names=()
  if [[ -n "${1:-}" ]]; then
    echo 'Dumping logs for nodes provided as args to dump_nodes() function'
    node_names=( "$@" )
  elif [[ -n "${use_custom_instance_list}" ]]; then
    echo 'Dumping logs for nodes provided by log_dump_custom_get_instances() function'
    while IFS='' read -r line; do node_names+=("$line"); done < <(log_dump_custom_get_instances node)
  elif [[ ! "${node_ssh_supported_providers}" =~ ${KUBERNETES_PROVIDER} ]]; then
    echo "Node SSH not supported for ${KUBERNETES_PROVIDER}"
    return
  else
    echo 'Detecting nodes in the cluster'
    detect-node-names &> /dev/null
    if [[ -n "${NODE_NAMES:-}" ]]; then
      node_names=( "${NODE_NAMES[@]}" )
    fi
    if [[ -n "${WINDOWS_NODE_NAMES:-}" ]]; then
      windows_node_names=( "${WINDOWS_NODE_NAMES[@]}" )
    fi
  fi

  if [[ "${#node_names[@]}" == 0 && "${#windows_node_names[@]}" == 0 ]]; then
    echo 'No nodes found!'
    return
  fi

  node_logfiles_all="${node_logfiles}"
  if [[ "${ENABLE_HOLLOW_NODE_LOGS:-}" == "true" ]]; then
    node_logfiles_all="${node_logfiles_all} ${hollow_node_logfiles}"
  fi

  linux_nodes_selected_for_logs=()
  if [[ -n "${LOGDUMP_ONLY_N_RANDOM_NODES:-}" ]]; then
    # We randomly choose 'LOGDUMP_ONLY_N_RANDOM_NODES' many nodes for fetching logs.
    for index in $(shuf -i 0-$(( ${#node_names[*]} - 1 )) -n "${LOGDUMP_ONLY_N_RANDOM_NODES}")
    do
      linux_nodes_selected_for_logs+=("${node_names[$index]}")
    done
  else
    linux_nodes_selected_for_logs=( "${node_names[@]}" )
  fi
  all_selected_nodes=( "${linux_nodes_selected_for_logs[@]}" )
  all_selected_nodes+=( "${windows_node_names[@]}" )

  proc=${max_dump_processes}
  start="$(date +%s)"
  # log_dump_ssh_timeout is the maximal number of seconds the log dumping over
  # SSH operation can take. Please note that the logic enforcing the timeout
  # is only a best effort. The actual time of the operation may be longer
  # due to waiting for all the child processes below.
  log_dump_ssh_timeout_seconds="${LOG_DUMP_SSH_TIMEOUT_SECONDS:-}"
  for i in "${!all_selected_nodes[@]}"; do
    node_name="${all_selected_nodes[$i]}"
    node_dir="${report_dir}/${node_name}"
    mkdir -p "${node_dir}"
    if [[ "${i}" -lt "${#linux_nodes_selected_for_logs[@]}" ]]; then
      # Save logs in the background. This speeds up things when there are
      # many nodes.
      save-logs "${node_name}" "${node_dir}" "${node_logfiles_all}" "${node_systemd_services}" &
    else
      save-logs-windows "${node_name}" "${node_dir}" &
    fi

    # We don't want to run more than ${max_dump_processes} at a time, so
    # wait once we hit that many nodes. This isn't ideal, since one might
    # take much longer than the others, but it should help.
    proc=$((proc - 1))
    if [[ proc -eq 0 ]]; then
      proc=${max_dump_processes}
      wait
      now="$(date +%s)"
      if [[ -n "${log_dump_ssh_timeout_seconds}" && $((now - start)) -gt ${log_dump_ssh_timeout_seconds} ]]; then
        echo "WARNING: Hit timeout after ${log_dump_ssh_timeout_seconds} seconds, finishing log dumping over SSH shortly"
        break
      fi
    fi
  done
  # Wait for any remaining processes.
  if [[ proc -gt 0 && proc -lt ${max_dump_processes} ]]; then
    wait
  fi
}

# Collect names of nodes which didn't run logexporter successfully.
# This function examines NODE_NAMES but not WINDOWS_NODE_NAMES since logexporter
# does not run on Windows nodes.
#
# Note: This step is O(#nodes^2) as we check if each node is present in the list of succeeded nodes.
# Making it linear would add code complexity without much benefit (as it just takes ~1s for 5k nodes).
# Assumes:
#   NODE_NAMES
# Sets:
#   NON_LOGEXPORTED_NODES
function find_non_logexported_nodes() {
  local file="${gcs_artifacts_dir}/logexported-nodes-registry"
  echo "Listing marker files ($file) for successful nodes..."
  succeeded_nodes=$(gsutil ls "${file}") || return 1
  echo 'Successfully listed marker files for successful nodes'
  NON_LOGEXPORTED_NODES=()
  for node in "${NODE_NAMES[@]}"; do
    if [[ ! "${succeeded_nodes}" =~ ${node} ]]; then
      NON_LOGEXPORTED_NODES+=("${node}")
    fi
  done
}

# This function examines NODE_NAMES but not WINDOWS_NODE_NAMES since logexporter
# does not run on Windows nodes.
function dump_nodes_with_logexporter() {
  if [[ -n "${use_custom_instance_list}" ]]; then
    echo 'Dumping logs for nodes provided by log_dump_custom_get_instances() function'
    NODE_NAMES=()
    while IFS='' read -r line; do NODE_NAMES+=("$line"); done < <(log_dump_custom_get_instances node)
  else
    echo 'Detecting nodes in the cluster'
    detect-node-names &> /dev/null
  fi

  if [[ -z "${NODE_NAMES:-}" ]]; then
    echo 'No nodes found!'
    return
  fi

  # Obtain parameters required by logexporter.
  local -r service_account_credentials="$(base64 "${GOOGLE_APPLICATION_CREDENTIALS}" | tr -d '\n')"
  local -r cloud_provider="${KUBERNETES_PROVIDER}"
  local -r enable_hollow_node_logs="${ENABLE_HOLLOW_NODE_LOGS:-false}"
  local -r logexport_sleep_seconds="$(( 90 + NUM_NODES / 3 ))"
  if [[ -z "${ZONE_NODE_SELECTOR_DISABLED:-}" ]]; then
    local -r node_selector="${ZONE_NODE_SELECTOR_LABEL:-topology.kubernetes.io/zone}: ${ZONE}"
  fi

  # Fill in the parameters in the logexporter daemonset template.
  local -r tmp="${KUBE_TEMP}/logexporter"
  local -r manifest_yaml="${tmp}/logexporter-daemonset.yaml"
  mkdir -p "${tmp}"
  cp "${KUBE_ROOT}/cluster/log-dump/logexporter-daemonset.yaml" "${manifest_yaml}"

  sed -i'' -e "s@{{.NodeSelector}}@${node_selector:-}@g" "${manifest_yaml}"
  sed -i'' -e "s@{{.LogexporterNamespace}}@${logexporter_namespace}@g" "${manifest_yaml}"
  sed -i'' -e "s@{{.ServiceAccountCredentials}}@${service_account_credentials}@g" "${manifest_yaml}"
  sed -i'' -e "s@{{.CloudProvider}}@${cloud_provider}@g" "${manifest_yaml}"
  sed -i'' -e "s@{{.GCSPath}}@${gcs_artifacts_dir}@g" "${manifest_yaml}"
  sed -i'' -e "s@{{.EnableHollowNodeLogs}}@${enable_hollow_node_logs}@g" "${manifest_yaml}"
  sed -i'' -e "s@{{.DumpSystemdJournal}}@${dump_systemd_journal}@g" "${manifest_yaml}"
  sed -i'' -e "s@{{.ExtraLogFiles}}@${extra_log_files}@g" "${manifest_yaml}"
  sed -i'' -e "s@{{.ExtraSystemdServices}}@${extra_systemd_services}@g" "${manifest_yaml}"

  # Create the logexporter namespace, service-account secret and the logexporter daemonset within that namespace.
  KUBECTL="${KUBE_ROOT}/cluster/kubectl.sh"
  if ! "${KUBECTL}" create -f "${manifest_yaml}"; then
    echo 'Failed to create logexporter daemonset.. falling back to logdump through SSH'
    "${KUBECTL}" delete namespace "${logexporter_namespace}" || true
    dump_nodes "${NODE_NAMES[@]}"
    logexporter_failed=1
    return
  fi

  # Periodically fetch list of already logexported nodes to verify
  # if we aren't already done.
  start="$(date +%s)"
  while true; do
    now="$(date +%s)"
    if [[ $((now - start)) -gt ${logexport_sleep_seconds} ]]; then
      echo 'Waiting for all nodes to be logexported timed out.'
      break
    fi
    if find_non_logexported_nodes; then
      if [[ -z "${NON_LOGEXPORTED_NODES:-}" ]]; then
        break
      fi
    fi
    sleep 15
  done

  # Store logs from logexporter pods to allow debugging log exporting process
  # itself.
  proc=${max_dump_processes}
  "${KUBECTL}" get pods -n "${logexporter_namespace}" -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.spec.nodeName}{"\n"}{end}' | (while read -r pod node; do
    echo "Fetching logs from ${pod} running on ${node}"
    mkdir -p "${report_dir}/${node}"
    "${KUBECTL}" logs -n "${logexporter_namespace}" "${pod}" > "${report_dir}/${node}/${pod}.log" &

    # We don't want to run more than ${max_dump_processes} at a time, so
    # wait once we hit that many nodes. This isn't ideal, since one might
    # take much longer than the others, but it should help.
    proc=$((proc - 1))
    if [[ proc -eq 0 ]]; then
      proc=${max_dump_processes}
      wait
    fi
  # Wait for any remaining processes.
  done; wait)

  # List registry of marker files (of nodes whose logexporter succeeded) from GCS.
  for retry in {1..10}; do
    if find_non_logexported_nodes; then
      break
    else
      echo "Attempt ${retry} failed to list marker files for successful nodes"
      if [[ "${retry}" == 10 ]]; then
        echo 'Final attempt to list marker files failed.. falling back to logdump through SSH'
        "${KUBECTL}" delete namespace "${logexporter_namespace}" || true
        dump_nodes "${NODE_NAMES[@]}"
        logexporter_failed=1
        return
      fi
      sleep 2
    fi
  done

  failed_nodes=()
  # The following if is needed, because defaulting for empty arrays
  # seems to treat them as non-empty with single empty string.
  if [[ -n "${NON_LOGEXPORTED_NODES:-}" ]]; then
    for node in "${NON_LOGEXPORTED_NODES[@]:-}"; do
      echo "Logexporter didn't succeed on node ${node}. Queuing it for logdump through SSH."
      failed_nodes+=("${node}")
    done
  fi

  # If less than a certain ratio of the nodes got logexported, report an error.
  if [[ $(((${#NODE_NAMES[@]} - ${#failed_nodes[@]}) * 100)) -lt $((${#NODE_NAMES[@]} * log_dump_expected_success_percentage )) ]]; then
    logexporter_failed=1
  fi

  # Delete the logexporter resources and dump logs for the failed nodes (if any) through SSH.
  "${KUBECTL}" get pods --namespace "${logexporter_namespace}" || true
  "${KUBECTL}" delete namespace "${logexporter_namespace}" || true
  if [[ "${#failed_nodes[@]}" != 0 ]]; then
    echo -e "Dumping logs through SSH for the following nodes:\n${failed_nodes[*]}"
    dump_nodes "${failed_nodes[@]}"
  fi
}

function detect_node_failures() {
  if ! [[ "${gcloud_supported_providers}" =~ ${KUBERNETES_PROVIDER} ]]; then
    return
  fi

  detect-node-names
  if [[ "${KUBERNETES_PROVIDER}" == "gce" ]]; then
    local all_instance_groups=("${INSTANCE_GROUPS[@]}" "${WINDOWS_INSTANCE_GROUPS[@]}")
  else
    local all_instance_groups=("${INSTANCE_GROUPS[@]}")
  fi

  if [ -z "${all_instance_groups:-}" ]; then
    return
  fi
  for group in "${all_instance_groups[@]}"; do
    local creation_timestamp
    creation_timestamp=$(gcloud compute instance-groups managed describe \
                         "${group}" \
                         --project "${PROJECT}" \
                         --zone "${ZONE}" \
                         --format='value(creationTimestamp)')
    echo "Failures for ${group} (if any):"
    gcloud logging read --order=asc \
          --format='table(timestamp,jsonPayload.resource.name,jsonPayload.event_subtype)' \
          --project "${PROJECT}" \
          "resource.type=\"gce_instance\"
           logName=\"projects/${PROJECT}/logs/compute.googleapis.com%2Factivity_log\"
           (jsonPayload.event_subtype=\"compute.instances.hostError\" OR jsonPayload.event_subtype=\"compute.instances.automaticRestart\")
           jsonPayload.resource.name:\"${group}\"
           timestamp >= \"${creation_timestamp}\""
  done
}

function main() {
  print-deprecation-note
  setup
  kube::util::ensure-temp-dir
  # Copy master logs to artifacts dir locally (through SSH).
  echo "Dumping logs from master locally to '${report_dir}'"
  dump_masters
  if [[ "${DUMP_ONLY_MASTER_LOGS:-}" == "true" ]]; then
    echo 'Skipping dumping of node logs'
    return
  fi

  # Copy logs from nodes to GCS directly or to artifacts dir locally (through SSH).
  if [[ -n "${gcs_artifacts_dir}" ]]; then
    echo "Dumping logs from nodes to GCS directly at '${gcs_artifacts_dir}' using logexporter"
    dump_nodes_with_logexporter
  else
    echo "Dumping logs from nodes locally to '${report_dir}'"
    dump_nodes
  fi

  detect_node_failures
  if [[ ${logexporter_failed} -ne 0 && ${log_dump_expected_success_percentage} -gt 0 ]]; then
    return 1
  fi
}

main


#!/usr/bin/env bash

# Copyright 2017 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# A library of helper functions for landing kubemark containers on a
# pre-existing Kubernetes master. See test/kubemark/pre-existing/README.md
# for me details on using a pre-existing provider.

KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../..

source "${KUBE_ROOT}/cluster/common.sh"
source "${KUBE_ROOT}/hack/lib/util.sh"

function detect-project() {
  if [[ -z "${MASTER_IP:-}" ]]; then
    echo "Set 'MASTER_IP' to the instance assigned to be the Kubernetes master" 1>&2
    exit 1
  fi

  if [[ -z "${PROJECT:-}" ]]; then
    echo "Set 'PROJECT' to the name of the container project: $CONTAINER_REGISTRY/$PROJECT/kubemark" >&2
    exit 1
  fi

  if [[ -z "${SERVICE_CLUSTER_IP_RANGE:-}" ]]; then
    cluster_range=$(echo "${MASTER_IP}" | awk -F '.' '{printf("%d.%d.%d.0", $1, $2, $3)}')
    SERVICE_CLUSTER_IP_RANGE="${SERVICE_CLUSTER_IP_RANGE:-$cluster_range/16}"
  fi
}

function create-certs {
  execute-cmd-on-pre-existing-master-with-retries 'sudo cat /etc/kubernetes/admin.conf' > /tmp/kubeconfig

  # CA_CERT_BASE64, KUBELET_CERT_BASE64 and KUBELET_KEY_BASE64 might be used
  # in test/kubemark/iks/util.sh
  # If it becomes clear that the variables are not used anywhere, then we can
  # remove them:
  CA_CERT_BASE64=$(grep certificate-authority /tmp/kubeconfig | awk '{print $2}' | head -n 1)
  KUBELET_CERT_BASE64=$(grep client-certificate-data /tmp/kubeconfig | awk '{print $2}' | head -n 1)
  KUBELET_KEY_BASE64=$(grep client-key-data /tmp/kubeconfig | awk '{print $2}' | head -n 1)
  export CA_CERT_BASE64
  export KUBELET_CERT_BASE64
  export KUBELET_KEY_BASE64
}


#!/usr/bin/env bash

# Copyright 2016 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This script contains the helper functions that each provider hosting
# Kubernetes must implement to use cluster/kube-*.sh scripts.

# Must ensure that the following ENV vars are set
function detect-master {
	echo "KUBE_MASTER_IP: ${KUBE_MASTER_IP:-}" 1>&2
	echo "KUBE_MASTER: ${KUBE_MASTER:-}" 1>&2
}

# Get node names if they are not static.
function detect-node-names {
	echo "NODE_NAMES: [${NODE_NAMES[*]}]" 1>&2
}

# Get node IP addresses and store in KUBE_NODE_IP_ADDRESSES[]
function detect-nodes {
	echo "KUBE_NODE_IP_ADDRESSES: [${KUBE_NODE_IP_ADDRESSES[*]}]" 1>&2
}

# Verify prereqs on host machine
function verify-prereqs {
	echo "Skeleton Provider: verify-prereqs not implemented" 1>&2
}

# Validate a kubernetes cluster
function validate-cluster {
	# by default call the generic validate-cluster.sh script, customizable by
	# any cluster provider if this does not fit.
	"${KUBE_ROOT}/cluster/validate-cluster.sh"
}

# Instantiate a kubernetes cluster
function kube-up {
	echo "Skeleton Provider: kube-up not implemented" 1>&2
}

# Delete a kubernetes cluster
function kube-down {
	echo "Skeleton Provider: kube-down not implemented" 1>&2
}

# Execute prior to running tests to build a release if required for env
function test-build-release {
	echo "Skeleton Provider: test-build-release not implemented" 1>&2
}

# Execute prior to running tests to initialize required structure
function test-setup {
	echo "Skeleton Provider: test-setup not implemented" 1>&2
}

# Execute after running tests to perform any required clean-up
function test-teardown {
	echo "Skeleton Provider: test-teardown not implemented" 1>&2
}

function prepare-e2e {
	echo "Skeleton Provider: prepare-e2e not implemented" 1>&2
}

function detect-project {
	echo "Skeleton Provider: detect-project not implemented" 1>&2
}


#!/usr/bin/env bash

# Copyright 2014 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This script runs `make test` command with some args for benchmark test.
# The "true" target of this makerule is `hack/make-rules/test.sh`.
# Args:
#   WHAT: Directory names to test.  All *_test.go files under these
#     directories will be run.  If not specified, "everything" will be tested.
# Usage: `hack/benchmark-go.sh`.
# Example: `hack/benchmark-go.sh WHAT=./pkg/kubelet`.

set -o errexit
set -o nounset
set -o pipefail

make test \
    WHAT="$*" \
    KUBE_COVER="" \
    KUBE_RACE=" " \
    KUBE_TEST_ARGS="-- -test.run='^X' -benchtime=1s -bench=. -benchmem" \


#!/usr/bin/env bash

# Copyright 2014 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This script is a vestigial redirection.  Please do not add "real" logic.
# The "true" target of this makerule is `hack/make-rules/cross.sh`.

# This script runs `make cross` command.
# The command sets up a go workspace locally and builds all for all appropriate
# platforms.
# Usage: `hack/build-cross.sh`.

set -o errexit
set -o nounset
set -o pipefail

KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..

echo "NOTE: $0 has been replaced by 'make cross'"
echo
echo "The equivalent of this invocation is: "
echo "    make cross"
echo
echo
make --no-print-directory -C "${KUBE_ROOT}" cross


#!/usr/bin/env bash

# Copyright 2016 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This script is a vestigial redirection.  Please do not add "real" logic.

# This script runs `make all` command.
# The command compiles all Makefile configs.
# Args:
#   WHAT: Directory names to build.  If any of these directories has a 'main'
#     package, the build will produce executable files under $(OUT_DIR)/go/bin.
#     If not specified, "everything" will be built.
# Usage: `hack/build-go.sh`.
# Example: `hack/build-go.sh WHAT=cmd/kubelet`.

set -o errexit
set -o nounset
set -o pipefail

KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..

# For help output
ARGHELP=""
if [[ "$#" -gt 0 ]]; then
    ARGHELP="WHAT='$*'"
fi

echo "NOTE: $0 has been replaced by 'make' or 'make all'"
echo
echo "The equivalent of this invocation is: "
echo "    make ${ARGHELP}"
echo
echo
make --no-print-directory -C "${KUBE_ROOT}" all WHAT="$*"


#!/usr/bin/env bash

# Copyright 2015 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Usage Instructions: https://git.k8s.io/community/contributors/devel/sig-release/cherry-picks.md

# Checkout a PR from GitHub. (Yes, this is sitting in a Git tree. How
# meta.) Assumes you care about pulls from remote "upstream" and
# checks them out to a branch named:
#  automated-cherry-pick-of-<pr>-<target branch>-<timestamp>

set -o errexit
set -o nounset
set -o pipefail

REPO_ROOT="$(git rev-parse --show-toplevel)"
declare -r REPO_ROOT
cd "${REPO_ROOT}"

STARTINGBRANCH=$(git symbolic-ref --short HEAD)
declare -r STARTINGBRANCH
declare -r REBASEMAGIC="${REPO_ROOT}/.git/rebase-apply"
DRY_RUN=${DRY_RUN:-""}
REGENERATE_DOCS=${REGENERATE_DOCS:-""}
UPSTREAM_REMOTE=${UPSTREAM_REMOTE:-upstream}
FORK_REMOTE=${FORK_REMOTE:-origin}
MAIN_REPO_ORG=${MAIN_REPO_ORG:-$(git remote get-url "$UPSTREAM_REMOTE" | awk '{gsub(/http[s]:\/\/|git@/,"")}1' | awk -F'[@:./]' 'NR==1{print $3}')}
MAIN_REPO_NAME=${MAIN_REPO_NAME:-$(git remote get-url "$UPSTREAM_REMOTE" | awk '{gsub(/http[s]:\/\/|git@/,"")}1' | awk -F'[@:./]' 'NR==1{print $4}')}

if [[ -z ${GITHUB_USER:-} ]]; then
  echo "Please export GITHUB_USER=<your-user> (or GH organization, if that's where your fork lives)"
  exit 1
fi

if ! command -v gh > /dev/null; then
  echo "Can't find 'gh' tool in PATH, please install from https://github.com/cli/cli"
  exit 1
fi

if [[ "$#" -lt 2 ]]; then
  echo "${0} <remote branch> <pr-number>...: cherry pick one or more <pr> onto <remote branch> and leave instructions for proposing pull request"
  echo
  echo "  Checks out <remote branch> and handles the cherry-pick of <pr> (possibly multiple) for you."
  echo "  Examples:"
  echo "    $0 upstream/release-3.14 12345        # Cherry-picks PR 12345 onto upstream/release-3.14 and proposes that as a PR."
  echo "    $0 upstream/release-3.14 12345 56789  # Cherry-picks PR 12345, then 56789 and proposes the combination as a single PR."
  echo
  echo "  Set the DRY_RUN environment var to skip git push and creating PR."
  echo "  This is useful for creating patches to a release branch without making a PR."
  echo "  When DRY_RUN is set the script will leave you in a branch containing the commits you cherry-picked."
  echo
  echo "  Set the REGENERATE_DOCS environment var to regenerate documentation for the target branch after picking the specified commits."
  echo "  This is useful when picking commits containing changes to API documentation."
  echo
  echo "  Set UPSTREAM_REMOTE (default: upstream) and FORK_REMOTE (default: origin)"
  echo "  to override the default remote names to what you have locally."
  echo
  echo "  For merge process info, see https://git.k8s.io/community/contributors/devel/sig-release/cherry-picks.md"
  exit 2
fi

# Checks if you are logged in. Will error/bail if you are not.
gh auth status

if git_status=$(git status --porcelain --untracked=no 2>/dev/null) && [[ -n "${git_status}" ]]; then
  echo "!!! Dirty tree. Clean up and try again."
  exit 1
fi

if [[ -e "${REBASEMAGIC}" ]]; then
  echo "!!! 'git rebase' or 'git am' in progress. Clean up and try again."
  exit 1
fi

declare -r BRANCH="$1"
shift 1
declare -r PULLS=( "$@" )

function join { local IFS="$1"; shift; echo "$*"; }
PULLDASH=$(join - "${PULLS[@]/#/#}") # Generates something like "#12345-#56789"
declare -r PULLDASH
PULLSUBJ=$(join " " "${PULLS[@]/#/#}") # Generates something like "#12345 #56789"
declare -r PULLSUBJ

echo "+++ Updating remotes..."
git remote update "${UPSTREAM_REMOTE}" "${FORK_REMOTE}"

if ! git log -n1 --format=%H "${BRANCH}" >/dev/null 2>&1; then
  echo "!!! '${BRANCH}' not found. The second argument should be something like ${UPSTREAM_REMOTE}/release-0.21."
  echo "    (In particular, it needs to be a valid, existing remote branch that I can 'git checkout'.)"
  exit 1
fi

NEWBRANCHREQ="automated-cherry-pick-of-${PULLDASH}" # "Required" portion for tools.
declare -r NEWBRANCHREQ
NEWBRANCH="$(echo "${NEWBRANCHREQ}-${BRANCH}" | sed 's/\//-/g')"
declare -r NEWBRANCH
NEWBRANCHUNIQ="${NEWBRANCH}-$(date +%s)"
declare -r NEWBRANCHUNIQ
echo "+++ Creating local branch ${NEWBRANCHUNIQ}"

cleanbranch=""
gitamcleanup=false
function return_to_kansas {
  if [[ "${gitamcleanup}" == "true" ]]; then
    echo
    echo "+++ Aborting in-progress git am."
    git am --abort >/dev/null 2>&1 || true
  fi

  # return to the starting branch and delete the PR text file
  if [[ -z "${DRY_RUN}" ]]; then
    echo
    echo "+++ Returning you to the ${STARTINGBRANCH} branch and cleaning up."
    git checkout -f "${STARTINGBRANCH}" >/dev/null 2>&1 || true
    if [[ -n "${cleanbranch}" ]]; then
      git branch -D "${cleanbranch}" >/dev/null 2>&1 || true
    fi
  fi
}
trap return_to_kansas EXIT

SUBJECTS=()
function make-a-pr() {
  local rel
  rel="$(basename "${BRANCH}")"
  echo
  echo "+++ Creating a pull request on GitHub at ${GITHUB_USER}:${NEWBRANCH}"

  local numandtitle
  numandtitle=$(printf '%s\n' "${SUBJECTS[@]}")
  prtext=$(cat <<EOF
Cherry pick of ${PULLSUBJ} on ${rel}.

${numandtitle}

For details on the cherry pick process, see the [cherry pick requests](https://git.k8s.io/community/contributors/devel/sig-release/cherry-picks.md) page.
EOF
)

  gh pr create --title="Automated cherry pick of ${numandtitle}" --body="${prtext}" --head "${GITHUB_USER}:${NEWBRANCH}" --base "${rel}" --repo="${MAIN_REPO_ORG}/${MAIN_REPO_NAME}"
}

git checkout -b "${NEWBRANCHUNIQ}" "${BRANCH}"
cleanbranch="${NEWBRANCHUNIQ}"

gitamcleanup=true
for pull in "${PULLS[@]}"; do
  echo "+++ Downloading patch to /tmp/${pull}.patch (in case you need to do this again)"

  curl -o "/tmp/${pull}.patch" -sSL "https://github.com/${MAIN_REPO_ORG}/${MAIN_REPO_NAME}/pull/${pull}.patch"
  echo
  echo "+++ About to attempt cherry pick of PR. To reattempt:"
  echo "  $ git am -3 /tmp/${pull}.patch"
  echo
  git am -3 "/tmp/${pull}.patch" || {
    conflicts=false
    while unmerged=$(git status --porcelain | grep ^U) && [[ -n ${unmerged} ]] \
      || [[ -e "${REBASEMAGIC}" ]]; do
      conflicts=true # <-- We should have detected conflicts once
      echo
      echo "+++ Conflicts detected:"
      echo
      (git status --porcelain | grep ^U) || echo "!!! None. Did you git am --continue?"
      echo
      echo "+++ Please resolve the conflicts in another window (and remember to 'git add / git am --continue')"
      read -p "+++ Proceed (anything but 'y' aborts the cherry-pick)? [y/n] " -r
      echo
      if ! [[ "${REPLY}" =~ ^[yY]$ ]]; then
        echo "Aborting." >&2
        exit 1
      fi
    done

    if [[ "${conflicts}" != "true" ]]; then
      echo "!!! git am failed, likely because of an in-progress 'git am' or 'git rebase'"
      exit 1
    fi
  }

  # set the subject
  subject=$(grep -m 1 "^Subject" "/tmp/${pull}.patch" | sed -e 's/Subject: \[PATCH//g' | sed 's/.*] //')
  SUBJECTS+=("#${pull}: ${subject}")

  # remove the patch file from /tmp
  rm -f "/tmp/${pull}.patch"
done
gitamcleanup=false

# Re-generate docs (if needed)
if [[ -n "${REGENERATE_DOCS}" ]]; then
  echo
  echo "Regenerating docs..."
  if ! hack/generate-docs.sh; then
    echo
    echo "hack/generate-docs.sh FAILED to complete."
    exit 1
  fi
fi

if [[ -n "${DRY_RUN}" ]]; then
  echo "!!! Skipping git push and PR creation because you set DRY_RUN."
  echo "To return to the branch you were in when you invoked this script:"
  echo
  echo "  git checkout ${STARTINGBRANCH}"
  echo
  echo "To delete this branch:"
  echo
  echo "  git branch -D ${NEWBRANCHUNIQ}"
  exit 0
fi

if git remote -v | grep ^"${FORK_REMOTE}" | grep "${MAIN_REPO_ORG}/${MAIN_REPO_NAME}.git"; then
  echo "!!! You have ${FORK_REMOTE} configured as your ${MAIN_REPO_ORG}/${MAIN_REPO_NAME}.git"
  echo "This isn't normal. Leaving you with push instructions:"
  echo
  echo "+++ First manually push the branch this script created:"
  echo
  echo "  git push REMOTE ${NEWBRANCHUNIQ}:${NEWBRANCH}"
  echo
  echo "where REMOTE is your personal fork (maybe ${UPSTREAM_REMOTE}? Consider swapping those.)."
  echo "OR consider setting UPSTREAM_REMOTE and FORK_REMOTE to different values."
  echo
  make-a-pr
  cleanbranch=""
  exit 0
fi

echo
echo "+++ I'm about to do the following to push to GitHub (and I'm assuming ${FORK_REMOTE} is your personal fork):"
echo
echo "  git push ${FORK_REMOTE} ${NEWBRANCHUNIQ}:${NEWBRANCH}"
echo
read -p "+++ Proceed (anything but 'y' aborts the cherry-pick)? [y/n] " -r
if ! [[ "${REPLY}" =~ ^[yY]$ ]]; then
  echo "Aborting." >&2
  exit 1
fi

git push "${FORK_REMOTE}" -f "${NEWBRANCHUNIQ}:${NEWBRANCH}"
make-a-pr


#!/usr/bin/env bash

# Copyright 2014 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This script will build a dev release and push it to an existing cluster.

set -o errexit
set -o nounset
set -o pipefail

KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..

# Build a dev release
if ! make -f "${KUBE_ROOT}"/Makefile quick-release
then
        echo "Building a release failed!"
        exit 1
fi

# Now push this out to the cluster
"${KUBE_ROOT}/cluster/kube-push.sh"


#!/usr/bin/env bash

# Copyright 2014 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This script will build a dev release and bring up a new cluster with that
# release.

set -o errexit
set -o nounset
set -o pipefail

KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..

# Build a dev release

if ! make -f "${KUBE_ROOT}"/Makefile quick-release
then
        echo "Building the release failed!"
        exit 1
fi

# Now bring a new cluster up with that release.
"${KUBE_ROOT}/cluster/kube-up.sh"


#!/usr/bin/env bash

# Copyright 2014 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This script builds some binaries and then the conformance image.
# REGISTRY and VERSION must be set.
# Example usage:
#   $ export REGISTRY=gcr.io/someone
#   $ export VERSION=v1.4.0-testfix
#   ./hack/dev-push-conformance.sh
# That will build and push gcr.io/someone/conformance-amd64:v1.4.0-testfix

set -o errexit
set -o nounset
set -o pipefail

KUBE_ROOT="$(dirname "${BASH_SOURCE[0]}")/.."
source "${KUBE_ROOT}/build/common.sh"

if [[ -z "${REGISTRY:-}" ]]; then
	echo "REGISTRY must be set"
	exit 1
fi
if [[ -z "${VERSION:-}" ]]; then
	echo "VERSION must be set"
	exit 1
fi

IMAGE="${REGISTRY}/conformance-amd64:${VERSION}"

kube::build::verify_prereqs
kube::build::build_image
kube::build::run_build_command make WHAT="vendor/github.com/onsi/ginkgo/ginkgo test/e2e/e2e.test cmd/kubectl cluster/images/conformance/go-runner"
kube::build::copy_output

make -C "${KUBE_ROOT}/cluster/images/conformance" build
docker push "${IMAGE}"


#!/usr/bin/env bash

# Copyright 2016 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This script is a vestigial redirection.  Please do not add "real" logic.
# The "true" target of this makerule is `hack/make-rules/test-e2e-node.sh`.

# This script runs `make test-e2e-node` command.
# The command builds and runs node end-to-end tests.
# Args:
#  FOCUS: Regexp that matches the tests to be run.  Defaults to "".
#  SKIP: Regexp that matches the tests that needs to be skipped.  Defaults
# Usage: `hack/e2e-node-test.sh `.
# Example: `hack/e2e-node-test.sh FOCUS=Kubelet SKIP=container`.

set -o errexit
set -o nounset
set -o pipefail

KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..

# For help output
ARGHELP=""
if [[ -n "${FOCUS:-}" ]]; then
    ARGHELP="FOCUS='${FOCUS}' "
fi
if [[ -n "${SKIP:-}" ]]; then
    ARGHELP="${ARGHELP}SKIP='${SKIP}'"
fi

echo "NOTE: $0 has been replaced by 'make test-e2e-node'"
echo
echo "This script supports a number of parameters passed as environment variables."
echo "Please see the Makefile for more details."
echo
echo "The equivalent of this invocation is: "
echo "    make test-e2e-node ${ARGHELP}"
echo
echo
make --no-print-directory -C "${KUBE_ROOT}" test-e2e-node FOCUS="${FOCUS:-}" SKIP="${SKIP:-}"


#!/usr/bin/env bash

# Copyright 2016 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -o errexit
set -o pipefail
set -o nounset

KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
export KUBE_ROOT
source "${KUBE_ROOT}/hack/lib/init.sh"
source "${KUBE_ROOT}/hack/lib/logging.sh"

if [[ ! -d "${KUBE_ROOT}/pkg" ]]; then
	echo "${KUBE_ROOT}/pkg not detected.  This script should be run from a location where the source dirs are available."
	exit 1
fi

# Ensure that we find the binaries we build before anything else.
export GOBIN="${KUBE_OUTPUT_BINPATH}"
PATH="${GOBIN}:${PATH}"

# Install tools we need, but only from vendor/...
go install k8s.io/kubernetes/vendor/github.com/go-bindata/go-bindata/go-bindata

# run the generation from the root directory for stable output
pushd "${KUBE_ROOT}" >/dev/null

# These are files for e2e tests.
BINDATA_OUTPUT="test/e2e/generated/bindata.go"
go-bindata -nometadata -o "${BINDATA_OUTPUT}.tmp" -pkg generated \
	-ignore .jpg -ignore .png -ignore .md \
	"test/conformance/testdata/..." \
	"test/e2e/testing-manifests/..." \
	"test/e2e_node/testing-manifests/..." \
	"test/images/..." \
	"test/fixtures/..."

gofmt -s -w "${BINDATA_OUTPUT}.tmp"

# Here we compare and overwrite only if different to avoid updating the
# timestamp and triggering a rebuild. The 'cat' redirect trick to preserve file
# permissions of the target file.
if ! cmp -s "${BINDATA_OUTPUT}.tmp" "${BINDATA_OUTPUT}" ; then
	cat "${BINDATA_OUTPUT}.tmp" > "${BINDATA_OUTPUT}"
	V=2 kube::log::info "Generated bindata file : ${BINDATA_OUTPUT} has $(wc -l ${BINDATA_OUTPUT}) lines of lovely automated artifacts"
else
	V=2 kube::log::info "No changes in generated bindata file: ${BINDATA_OUTPUT}"
fi

rm -f "${BINDATA_OUTPUT}.tmp"

# These are files for runtime code
BINDATA_OUTPUT="staging/src/k8s.io/kubectl/pkg/generated/bindata.go"
go-bindata -nometadata -nocompress -o "${BINDATA_OUTPUT}.tmp" -pkg generated \
	-ignore .jpg -ignore .png -ignore .md \
	"translations/..."

gofmt -s -w "${BINDATA_OUTPUT}.tmp"

# Here we compare and overwrite only if different to avoid updating the
# timestamp and triggering a rebuild. The 'cat' redirect trick to preserve file
# permissions of the target file.
if ! cmp -s "${BINDATA_OUTPUT}.tmp" "${BINDATA_OUTPUT}" ; then
	cat "${BINDATA_OUTPUT}.tmp" > "${BINDATA_OUTPUT}"
	V=2 kube::log::info "Generated bindata file : ${BINDATA_OUTPUT} has $(wc -l ${BINDATA_OUTPUT}) lines of lovely automated artifacts"
else
	V=2 kube::log::info "No changes in generated bindata file: ${BINDATA_OUTPUT}"
fi

rm -f "${BINDATA_OUTPUT}.tmp"

popd >/dev/null


#!/usr/bin/env bash

# Copyright 2014 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This file is not intended to be run automatically. It is meant to be run
# immediately before exporting docs. We do not want to check these documents in
# by default.

set -o errexit
set -o nounset
set -o pipefail

KUBE_HACK_ROOT=$(dirname "${BASH_SOURCE[0]}")

echo "WARNING: hack/generate-docs.sh is an alias for hack/update-generated-docs.sh"
echo "and will be removed in a future version."

"${KUBE_HACK_ROOT}"/update-generated-docs.sh


#!/usr/bin/env bash

# Copyright 2015 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This script runs `curl` command to get the kubernetes build file.
# Version number or publication is either a proper version number'
# (e.g. "v1.0.6", "v1.2.0-alpha.1.881+376438b69c7612") or a version'
#  publication of the form <bucket>/<version> (e.g. "release/stable",'
# "ci/latest-1").'

# Usage `hack/get-build.sh [Version]`.
# Example `hack/get-build.sh v1.16.4`.

set -o errexit
set -o nounset
set -o pipefail

KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..

source "${KUBE_ROOT}/cluster/common.sh"

declare -r KUBE_RELEASE_BUCKET_URL="https://storage.googleapis.com/kubernetes-release"
declare -r KUBE_DEV_RELEASE_BUCKET_URL="https://storage.googleapis.com/kubernetes-release-dev"
declare -r KUBE_TAR_NAME="kubernetes.tar.gz"

usage() {
  echo "${0} [-v] <version number or publication>"
  echo "  -v:  Don't get tars, just print the version number"
  echo ""
  echo '  Version number or publication is either a proper version number'
  echo '  (e.g. "v1.0.6", "v1.2.0-alpha.1.881+376438b69c7612") or a version'
  echo '  publication of the form <bucket>/<version> (e.g. "release/stable",'
  echo '  "ci/latest-1").  Some common ones are:'
  echo '    - "release/stable"'
  echo '    - "release/latest"'
  echo '    - "ci/latest"'
  echo '  See the docs on getting builds for more information about version'
  echo '  publication.'
}

print_version=false

while getopts ":vh" opt; do
  case ${opt} in
    v)
      print_version="true"
      ;;
    h)
      usage
      exit 0
      ;;
    \?)
      echo "Invalid option: -$OPTARG" >&2
      usage
      exit 1
      ;;
  esac
done
shift $((OPTIND-1))

if [[ $# -ne 1 ]]; then
    usage
    exit 1
fi

set_binary_version "${1}"

if [[ "${print_version}" == "true" ]]; then
  echo "${KUBE_VERSION}"
else
  echo "Using version at ${1}: ${KUBE_VERSION}" >&2
  if [[ ${KUBE_VERSION} =~ ${KUBE_RELEASE_VERSION_REGEX} ]]; then
    curl --fail -o "kubernetes-${KUBE_VERSION}.tar.gz" "${KUBE_RELEASE_BUCKET_URL}/release/${KUBE_VERSION}/${KUBE_TAR_NAME}"
  elif [[ ${KUBE_VERSION} =~ ${KUBE_CI_VERSION_REGEX} ]]; then
    curl --fail -o "kubernetes-${KUBE_VERSION}.tar.gz" "${KUBE_DEV_RELEASE_BUCKET_URL}/ci/${KUBE_VERSION}/${KUBE_TAR_NAME}"
  else
    echo "Version doesn't match regexp" >&2
    exit 1
  fi
fi


#!/usr/bin/env bash

# Copyright 2014 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This script runs e2e tests on Google Cloud Platform.
# Usage: `hack/ginkgo-e2e.sh`.

set -o errexit
set -o nounset
set -o pipefail

KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
source "${KUBE_ROOT}/cluster/common.sh"
source "${KUBE_ROOT}/hack/lib/init.sh"

# Find the ginkgo binary build as part of the release.
ginkgo=$(kube::util::find-binary "ginkgo")
e2e_test=$(kube::util::find-binary "e2e.test")

# --- Setup some env vars.

GINKGO_PARALLEL=${GINKGO_PARALLEL:-n} # set to 'y' to run tests in parallel
CLOUD_CONFIG=${CLOUD_CONFIG:-""}

# If 'y', Ginkgo's reporter will not print out in color when tests are run
# in parallel
GINKGO_NO_COLOR=${GINKGO_NO_COLOR:-n}

# If 'y', will rerun failed tests once to give them a second chance.
GINKGO_TOLERATE_FLAKES=${GINKGO_TOLERATE_FLAKES:-n}

# If set, the command executed will be:
# - `dlv exec` if set to "delve"
# - `gdb` if set to "gdb"
# NOTE: for this to work the e2e.test binary has to be compiled with
# make WHAT=test/e2e/e2e.test GOGCFLAGS="all=-N -l" GOLDFLAGS=""
E2E_TEST_DEBUG_TOOL=${E2E_TEST_DEBUG_TOOL:-}

: "${KUBECTL:="${KUBE_ROOT}/cluster/kubectl.sh"}"
: "${KUBE_CONFIG_FILE:="config-test.sh"}"

export KUBECTL KUBE_CONFIG_FILE

source "${KUBE_ROOT}/cluster/kube-util.sh"

function detect-master-from-kubeconfig() {
    export KUBECONFIG=${KUBECONFIG:-$DEFAULT_KUBECONFIG}

    local cc
    cc=$("${KUBE_ROOT}/cluster/kubectl.sh" config view -o jsonpath="{.current-context}")
    if [[ -n "${KUBE_CONTEXT:-}" ]]; then
      cc="${KUBE_CONTEXT}"
    fi
    local cluster
    cluster=$("${KUBE_ROOT}/cluster/kubectl.sh" config view -o jsonpath="{.contexts[?(@.name == \"${cc}\")].context.cluster}")
    KUBE_MASTER_URL=$("${KUBE_ROOT}/cluster/kubectl.sh" config view -o jsonpath="{.clusters[?(@.name == \"${cluster}\")].cluster.server}")
}

# ---- Do cloud-provider-specific setup
if [[ -n "${KUBERNETES_CONFORMANCE_TEST:-}" ]]; then
    echo "Conformance test: not doing test setup."
    KUBERNETES_PROVIDER=${KUBERNETES_CONFORMANCE_PROVIDER:-"skeleton"}

    detect-master-from-kubeconfig

    auth_config=(
      "--kubeconfig=${KUBECONFIG}"
    )
else
    echo "Setting up for KUBERNETES_PROVIDER=\"${KUBERNETES_PROVIDER}\"."

    prepare-e2e

    detect-master >/dev/null

    KUBE_MASTER_URL="${KUBE_MASTER_URL:-}"
    if [[ -z "${KUBE_MASTER_URL:-}" && -n "${KUBE_MASTER_IP:-}" ]]; then
      KUBE_MASTER_URL="https://${KUBE_MASTER_IP}"
    fi

    auth_config=(
      "--kubeconfig=${KUBECONFIG:-$DEFAULT_KUBECONFIG}"
    )
fi

if [[ -n "${NODE_INSTANCE_PREFIX:-}" ]]; then
  NODE_INSTANCE_GROUP="${NODE_INSTANCE_PREFIX}-group"
fi

if [[ "${KUBERNETES_PROVIDER}" == "gce" ]]; then
  set_num_migs
  NODE_INSTANCE_GROUP=""
  for ((i=1; i<=NUM_MIGS; i++)); do
    if [[ ${i} == "${NUM_MIGS}" ]]; then
      # We are assigning the same mig names as create-nodes function from cluster/gce/util.sh.
      NODE_INSTANCE_GROUP="${NODE_INSTANCE_GROUP}${NODE_INSTANCE_PREFIX}-group"
    else
      NODE_INSTANCE_GROUP="${NODE_INSTANCE_GROUP}${NODE_INSTANCE_PREFIX}-group-${i},"
    fi
  done
fi

# TODO(kubernetes/test-infra#3330): Allow NODE_INSTANCE_GROUP to be
# set before we get here, which eliminates any cluster/gke use if
# KUBERNETES_CONFORMANCE_PROVIDER is set to "gke".
if [[ -z "${NODE_INSTANCE_GROUP:-}" ]] && [[ "${KUBERNETES_PROVIDER}" == "gke" ]]; then
  detect-node-instance-groups
  NODE_INSTANCE_GROUP=$(kube::util::join , "${NODE_INSTANCE_GROUP[@]}")
fi

if [[ "${KUBERNETES_PROVIDER}" == "azure" ]]; then
    if [[ ${CLOUD_CONFIG} == "" ]]; then
        echo "Missing azure cloud config"
        exit 1
    fi
fi

ginkgo_args=()
if [[ -n "${CONFORMANCE_TEST_SKIP_REGEX:-}" ]]; then
  ginkgo_args+=("--skip=${CONFORMANCE_TEST_SKIP_REGEX}")
  ginkgo_args+=("--seed=1436380640")
fi
if [[ -n "${GINKGO_PARALLEL_NODES:-}" ]]; then
  ginkgo_args+=("--nodes=${GINKGO_PARALLEL_NODES}")
elif [[ ${GINKGO_PARALLEL} =~ ^[yY]$ ]]; then
  ginkgo_args+=("--nodes=25")
fi

if [[ "${GINKGO_UNTIL_IT_FAILS:-}" == true ]]; then
  ginkgo_args+=("--untilItFails=true")
fi

FLAKE_ATTEMPTS=1
if [[ "${GINKGO_TOLERATE_FLAKES}" == "y" ]]; then
  FLAKE_ATTEMPTS=2
fi

if [[ "${GINKGO_NO_COLOR}" == "y" ]]; then
  ginkgo_args+=("--noColor")
fi

CONTAINER_RUNTIME=${CONTAINER_RUNTIME:-${KUBE_CONTAINER_RUNTIME:-}}

# The --host setting is used only when providing --auth_config
# If --kubeconfig is used, the host to use is retrieved from the .kubeconfig
# file and the one provided with --host is ignored.
# Add path for things like running kubectl binary.
PATH=$(dirname "${e2e_test}"):"${PATH}"
export PATH

# Choose the program to execute.
program=("${ginkgo}")
if [[ "${E2E_TEST_DEBUG_TOOL}" == "delve" ]]; then
  program=("dlv" "exec")
elif [[ "${E2E_TEST_DEBUG_TOOL}" == "gdb" ]]; then
  program=("gdb")
fi

"${program[@]}" "${ginkgo_args[@]:+${ginkgo_args[@]}}" "${e2e_test}" -- \
  "${auth_config[@]:+${auth_config[@]}}" \
  --ginkgo.flakeAttempts="${FLAKE_ATTEMPTS}" \
  --host="${KUBE_MASTER_URL}" \
  --provider="${KUBERNETES_PROVIDER}" \
  --gce-project="${PROJECT:-}" \
  --gce-zone="${ZONE:-}" \
  --gce-region="${REGION:-}" \
  --gce-multizone="${MULTIZONE:-false}" \
  --gke-cluster="${CLUSTER_NAME:-}" \
  --kube-master="${KUBE_MASTER:-}" \
  --cluster-tag="${CLUSTER_ID:-}" \
  --cloud-config-file="${CLOUD_CONFIG:-}" \
  --repo-root="${KUBE_ROOT}" \
  --node-instance-group="${NODE_INSTANCE_GROUP:-}" \
  --prefix="${KUBE_GCE_INSTANCE_PREFIX:-e2e}" \
  --network="${KUBE_GCE_NETWORK:-${KUBE_GKE_NETWORK:-e2e}}" \
  --node-tag="${NODE_TAG:-}" \
  --master-tag="${MASTER_TAG:-}" \
  --docker-config-file="${DOCKER_CONFIG_FILE:-}" \
  --dns-domain="${KUBE_DNS_DOMAIN:-cluster.local}" \
  --ginkgo.slowSpecThreshold="${GINKGO_SLOW_SPEC_THRESHOLD:-300}" \
  ${CONTAINER_RUNTIME:+"--container-runtime=${CONTAINER_RUNTIME}"} \
  ${MASTER_OS_DISTRIBUTION:+"--master-os-distro=${MASTER_OS_DISTRIBUTION}"} \
  ${NODE_OS_DISTRIBUTION:+"--node-os-distro=${NODE_OS_DISTRIBUTION}"} \
  ${NUM_NODES:+"--num-nodes=${NUM_NODES}"} \
  ${E2E_REPORT_DIR:+"--report-dir=${E2E_REPORT_DIR}"} \
  ${E2E_REPORT_PREFIX:+"--report-prefix=${E2E_REPORT_PREFIX}"} \
  "${@:-}"