#!/bin/bash

# Copyright 2017 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# usage: generate.sh $LANGUAGE $OUTPUT_DIR
#  current tested languages:
#   * java

dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"

if ! which protoc > /dev/null; then
  ${dir}/install.sh
  proto=./bin/protoc
else
  proto=$(which protoc)
fi

${dir}/dependencies.sh

# The format here is <file-name>;<generated-class-name>
files="k8s.io/apimachinery/pkg/api/resource/generated.proto;Resource \
       k8s.io/apimachinery/pkg/apis/meta/v1/generated.proto;Meta \
       k8s.io/apimachinery/pkg/runtime/generated.proto;Runtime \
       k8s.io/apimachinery/pkg/runtime/schema/generated.proto;RuntimeSchema \
       k8s.io/apimachinery/pkg/util/intstr/generated.proto;IntStr \
       k8s.io/api/rbac/v1alpha1/generated.proto;V1alpha1Rbac \
       k8s.io/api/rbac/v1/generated.proto;V1Rbac \
       k8s.io/api/rbac/v1beta1/generated.proto;V1beta1Rbac \
       k8s.io/api/networking/v1/generated.proto;V1Networking \
       k8s.io/api/settings/v1alpha1/generated.proto;V1alpha1Settings \
       k8s.io/api/admissionregistration/v1beta1/generated.proto;V1beta1Admissionregistration \
       k8s.io/api/scheduling/v1alpha1/generated.proto;V1alpha1Scheduling \
       k8s.io/api/storage/v1/generated.proto;V1Storage \
       k8s.io/api/storage/v1beta1/generated.proto;V1beta1Storage \
       k8s.io/api/batch/v2alpha1/generated.proto;V2alpha1Batch \
       k8s.io/api/batch/v1/generated.proto;V1Batch \
       k8s.io/api/batch/v1beta1/generated.proto;V1beta1Batch \
       k8s.io/api/apps/v1beta2/generated.proto;V1beta2Apps \
       k8s.io/api/apps/v1/generated.proto;V1Apps \
       k8s.io/api/apps/v1beta1/generated.proto;V1beta1Apps \
       k8s.io/api/authentication/v1/generated.proto;V1Authentication \
       k8s.io/api/authentication/v1beta1/generated.proto;V1beta1Authentication \
       k8s.io/api/admission/v1beta1/generated.proto;V1beta1Admission \
       k8s.io/api/policy/v1beta1/generated.proto;V1beta1Policy \
       k8s.io/api/core/v1/generated.proto;V1 \
       k8s.io/api/autoscaling/v1/generated.proto;V1Autoscaling \
       k8s.io/api/autoscaling/v2beta1/generated.proto;V2beta1Autoscaling \
       k8s.io/api/autoscaling/v2beta2/generated.proto;V2beta2Autoscaling \
       k8s.io/api/extensions/v1beta1/generated.proto;V1beta1Extensions \
       k8s.io/api/certificates/v1beta1/generated.proto;V1beta1Certificates \
       k8s.io/api/imagepolicy/v1alpha1/generated.proto;V1alpha1Imagepolicy \
       k8s.io/api/authorization/v1/generated.proto;V1Authorization \
       k8s.io/api/authorization/v1beta1/generated.proto;V1beta1Authorization \
       k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/generated.proto;V1beta1Apiextensions"

proto_files=""

echo 'Munging proto file packages'

# This is a little hacky, but we know the go_package directive is in the
# right place, so add a marker, and then append more package declarations.
# Sorry, I like perl.
for info in ${files}; do
  file=$(echo ${info} | cut -d ";" -f 1)
  class=$(echo ${info} | cut -d ";" -f 2)
  proto_files="${file} ${proto_files}"
  perl -pi -e \
    's/option go_package = "(.*)";/option go_package = "$1";\n\/\/ PKG/' \
    ${file}
  perl -pi -e \
    's/\/\/ PKG/\/\/ PKG\noption java_package = "io.kubernetes.client.proto";/' \
    ${file}
  perl -pi -e \
    "s/\/\/ PKG/\/\/ PKG\noption java_outer_classname = \"${class}\";/" \
    ${file}

  # Other package declarations can go here.
done

echo "Generating code for $1"
${proto} -I${dir} ${proto_files} --${1}_out=${2}


#!/bin/bash

# Copyright 2017 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

echo 'Installing proto compiler'

version=3.6.1
file=protoc-${version}-linux-x86_64.zip

wget https://github.com/google/protobuf/releases/download/v${version}/${file}
unzip ${file}
rm ${file}


apt update && \
apt install -y locales cmake libsm6 libxext6 libxrender-dev && \
locale-gen en_US.UTF-8

export LC_ALL=en_US.UTF-8

# Pip packages for running server and face-aligned (dlib takes a while to install)
pip install flask flask_cors tqdm opencv-python imutils dlib imageio

# Get model weights
curl https://openaipublic.azureedge.net/glow-demo/large3/graph_optimized.pb > graph_optimized.pb

# Get manipulation vectors
curl https://openaipublic.azureedge.net/glow-demo/z_manipulate.npy > z_manipulate.npy

# Get facial landmarks for aligning input faces
curl https://openaipublic.azureedge.net/glow-demo/shape_predictor_68_face_landmarks.dat > shape_predictor_68_face_landmarks.dat

# Pip package for running optimized model with fused kernels
curl https://openaipublic.azureedge.net/glow-demo/blocksparse-1.0.0-py2.py3-none-any.whl > blocksparse-1.0.0-py2.py3-none-any.whl
pip install blocksparse-1.0.0-py2.py3-none-any.whl

# If blocksparse doesn't install, use unoptimized model (and set optimized=False in model.py)
# curl https://openaipublic.azureedge.net/glow-demo/large3/graph_unoptimized.pb > graph_unoptimized.pb


#!/bin/sh

# This file is deprecated. Will be removed shortly.

./build.py


#!/bin/sh -eux

if [ "$1" != "" ]; then
  cd /tmp
  wget http://downloads.sourceforge.net/project/libjpeg-turbo/1.3.1/libjpeg-turbo-official_$1_amd64.deb
  sudo dpkg -i libjpeg-turbo-official_$1_amd64.deb
  find /opt/libjpeg-turbo/lib64
  find /opt/libjpeg-turbo/include
fi


#!/bin/bash

# Update PR refs for testing.
if [[ -n "${CIRCLE_PR_NUMBER}" ]]
then
    FETCH_REFS="${FETCH_REFS} +refs/pull/${CIRCLE_PR_NUMBER}/head:pr/${CIRCLE_PR_NUMBER}/head"
    FETCH_REFS="${FETCH_REFS} +refs/pull/${CIRCLE_PR_NUMBER}/merge:pr/${CIRCLE_PR_NUMBER}/merge"
fi

# Retrieve the refs.
if [[ -n "${CIRCLE_PR_NUMBER}" ]]
then
    git fetch -u origin ${FETCH_REFS}
fi

# Checkout the PR merge ref.
if [[ -n "${CIRCLE_PR_NUMBER}" ]]
then
    git checkout -qf "pr/${CIRCLE_PR_NUMBER}/merge"
fi

# Check for merge conflicts.
if [[ -n "${CIRCLE_PR_NUMBER}" ]]
then
    git branch --merged | grep "pr/${CIRCLE_PR_NUMBER}/head" > /dev/null
fi


#!/usr/bin/env bash

# PLEASE NOTE: This script has been automatically generated by conda-smithy. Any changes here
# will be lost next time ``conda smithy rerender`` is run. If you would like to make permanent
# changes to this script, consider a proposal to conda-smithy so that other feedstocks can also
# benefit from the improvement.

FEEDSTOCK_ROOT=$(cd "$(dirname "$0")/.."; pwd;)
RECIPE_ROOT=$FEEDSTOCK_ROOT/recipe

docker info

config=$(cat <<CONDARC

channels:
 - conda-forge
 - defaults # As we need conda-build

conda-build:
 root-dir: /feedstock_root/build_artefacts

show_channel_urls: true

CONDARC
)

cat << EOF | docker run -i \
                        -v ${RECIPE_ROOT}:/recipe_root \
                        -v ${FEEDSTOCK_ROOT}:/feedstock_root \
                        -a stdin -a stdout -a stderr \
                        condaforge/linux-anvil \
                        bash || exit $?

export BINSTAR_TOKEN=${BINSTAR_TOKEN}
export PYTHONUNBUFFERED=1

echo "$config" > ~/.condarc
# A lock sometimes occurs with incomplete builds. The lock file is stored in build_artefacts.
conda clean --lock

conda install --yes --quiet conda-forge-build-setup
source run_conda_forge_build_setup

# Embarking on 3 case(s).
    set -x
    export CONDA_PY=27
    set +x
    conda build /recipe_root --quiet || exit 1
    upload_or_check_non_existence /recipe_root jeremyschlatter --channel=main || exit 1

    set -x
    export CONDA_PY=34
    set +x
    conda build /recipe_root --quiet || exit 1
    upload_or_check_non_existence /recipe_root jeremyschlatter --channel=main || exit 1

    set -x
    export CONDA_PY=35
    set +x
    conda build /recipe_root --quiet || exit 1
    upload_or_check_non_existence /recipe_root jeremyschlatter --channel=main || exit 1
EOF


GOOS=$(uname | tr '[:upper:]' '[:lower:]')
if [ $ARCH = '64' ]; then
    GOARCH=amd64
else
    GOARCH=386
fi
curl -o go.tar.gz https://storage.googleapis.com/golang/go1.7.4.$GOOS-$GOARCH.tar.gz
tar xf go.tar.gz
export GOROOT=$(pwd)/go
export PATH=$GOROOT/bin:$PATH

export GO_VNCDRIVER_PYTHON="$PYTHON"
export LIBJPG="-L$PREFIX/lib -lturbojpeg"
$PREFIX/bin/pip install .

if [ -n $TWINE_USERNAME ]; then
    echo "Building and uploading wheel to PyPI"
    $PYTHON setup.py bdist_wheel
    twine upload --skip-existing dist/go_vncdriver-*.whl -r https://testpypi.python.org/pypi --repository-url https://testpypi.python.org/pypi
fi


#!/bin/bash

# Set up torch with magma support

DIR="`mktemp -d`"
cd $DIR

# Install deps
conda install -y numpy ninja pyyaml mkl mkl-include setuptools cmake cffi typing_extensions future six requests dataclasses

# Install torch, magma
conda install -y -c pytorch magma-cuda110

# Build torch from scratch
git clone --recursive https://github.com/pytorch/pytorch
cd pytorch

# If updating an existing checkout
# git submodule sync
# git submodule update --init --recursive

export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname $(which conda))/../"}
python setup.py install


#!/bin/bash -e

# If these dependencies do not exist, install them
hash gulp 2>/dev/null || npm i -g gulp
hash tsfmt 2>/dev/null || npm i -g typescript-formatter
hash typings 2>/dev/null || npm i -g typings

(cd src && typings install)


#!/bin/bash -e
tsfmt -r $(find . -name "*.ts*" ! -name "*.d.ts" ! -path "./node_modules/*") || true


export word=imagenet
export pixels=128
echo $HOSTNAME >> ${word}.record.txt
CUDA_VISIBLE_DEVICES=$GPUS python train_${word}.py --dataset imagenet_train --is_train True --checkpoint_dir gan/checkpoint_${word} --image_size ${pixels} --is_crop True --sample_dir gan/samples_${word} --image_width ${pixels} --batch_size 16


#!/bin/bash
python -m torch.distributed.launch --nproc_per_node=2 distributed_data_parallel.py


#!/bin/bash
python -m torch.distributed.launch --nproc_per_node=2 amp_master_params.py

python compare.py


#!/bin/bash

CUDA_VISIBLE_DEVICES=0,1 python -m torch.distributed.launch --nproc_per_node=2 ddp_race_condition_test.py


python single_gpu_unit_test.py
python -m torch.distributed.launch --nproc_per_node=2 two_gpu_unit_test.py
python -m torch.distributed.launch --nproc_per_node=2 two_gpu_unit_test.py --fp64
#beware, you need a system with at least 4 gpus to test group_size<world_size
python -m torch.distributed.launch --nproc_per_node=4 test_groups.py --group_size=2


#!/bin/bash

print_banner() {
  printf "\n\n\n\e[30m\e[42m$1\e[0m\n\n\n\n"
}

print_green() {
  printf "\e[30m\e[42m$1\e[0m\n"
}

print_red() {
  printf "\e[30m\e[41m$1\e[0m\n"
}

images=(
"gitlab-master.nvidia.com:5005/dl/dgx/pytorch:19.03-py3-devel"
"gitlab-master.nvidia.com:5005/dl/dgx/pytorch:master-py3-devel"
"pytorch/pytorch:nightly-devel-cuda10.0-cudnn7"
"pytorch/pytorch:1.1.0-cuda10.0-cudnn7.5-devel"
"pytorch/pytorch:1.0.1-cuda10.0-cudnn7-devel"
"pytorch/pytorch:1.0-cuda10.0-cudnn7-devel"
"pytorch/pytorch:nightly-devel-cuda9.2-cudnn7"
)

branch="master"

# Associative array for exit codes
declare -A exit_codes
for image in images
do
  exit_codes[$image]="None"
done

for image in "${images[@]}"
do
  print_banner "$image"
  set -x
  docker pull $image
  # Trying python setup.py install instead of pip install to ensure direct access to error codes.
  # Maybe pip install would be ok too but this works.
  docker run --runtime=nvidia --rm $image /bin/bash -c "yes | pip uninstall apex; yes | pip uninstall apex; git clone https://github.com/NVIDIA/apex.git; cd apex; git checkout $branch; set -e;  python setup.py install --cuda_ext --cpp_ext"
  exit_code=$?
  set +x
  if [ $exit_code != 0 ]
  then
    print_red "Exit code: $exit_code"
  else
    print_green "Exit code: $exit_code"
  fi
  exit_codes[$image]=$exit_code
done

success=0
for image in "${images[@]}"
do
  exit_code=${exit_codes[$image]}
  if [ $exit_code != 0 ]
  then
    print_red "$image : $exit_code"
    success=1
  else
    print_green "$image : $exit_code"
  fi
done

if [ $success != 0 ]
then
  print_red "Overall status:  failure"
else
  print_green "Overall status:  success"
fi

exit $success


#!/bin/bash

print_banner() {
  printf "\n\n\n\e[30m\e[42m$1\e[0m\n\n\n\n"
}

print_banner "Distributed status:  $1"

echo $2
DATADIR=$2

if [ -n "$3" ]
then
  USE_BASELINE=""
else
  USE_BASELINE="--use_baseline"
fi

if [ "$1" == "single_gpu" ]
then
  BASE_CMD="python main_amp.py -a resnet50 --b 128 --workers 4 --deterministic --prints-to-process 5"
fi

if [ "$1" == "distributed" ]
then
  BASE_CMD="python -m torch.distributed.launch --nproc_per_node=2 main_amp.py -a resnet50 --b 128 --workers 4 --deterministic --prints-to-process 5"
fi

ADAM_ARGS="--opt-level O2 --keep-batchnorm-fp32 False --fused-adam"

keep_batchnorms=(
""
"--keep-batchnorm-fp32 True"
"--keep-batchnorm-fp32 False"
)

loss_scales=(
""
"--loss-scale 1.0"
"--loss-scale 128.0"
"--loss-scale dynamic"
)

opt_levels=(
"O0"
"O1"
"O2"
"O3"
)

rm True*
rm False*

set -e

print_banner "Installing Apex with --cuda_ext and --cpp_ext"

pushd ../../..
pip install -v --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext" .
popd

for opt_level in "${opt_levels[@]}"
do
  for loss_scale in "${loss_scales[@]}"
  do
    for keep_batchnorm in "${keep_batchnorms[@]}"
    do
      if [ "$opt_level" == "O1" ] && [ -n "${keep_batchnorm}" ]
      then
        print_banner "Skipping ${opt_level} ${loss_scale} ${keep_batchnorm}"
        continue
      fi
      print_banner "${BASE_CMD} --opt-level ${opt_level} ${loss_scale} ${keep_batchnorm} --has-ext $DATADIR"
      set -x
      ${BASE_CMD} --opt-level ${opt_level} ${loss_scale} ${keep_batchnorm} --has-ext $DATADIR
      set +x
    done
  done
done

# Handle FusedAdam separately due to limited support.
# FusedAdam will not be tested for bitwise accuracy against the Python implementation.
# The L0 tests already do so.  These tests are here to ensure that it actually runs,
# and get an idea of performance.
for loss_scale in "${loss_scales[@]}"
do
  print_banner "${BASE_CMD} ${ADAM_ARGS} ${loss_scale} --has-ext $DATADIR"
  set -x
  ${BASE_CMD} ${ADAM_ARGS} ${loss_scale} --has-ext $DATADIR
  set +x
done

print_banner "Reinstalling apex without extensions"

pushd ../../..
pip install -v --no-cache-dir .
popd

for opt_level in "${opt_levels[@]}"
do
  for loss_scale in "${loss_scales[@]}"
  do
    for keep_batchnorm in "${keep_batchnorms[@]}"
    do
      if [ "$opt_level" == "O1" ] && [ -n "${keep_batchnorm}" ]
      then
        print_banner "Skipping ${opt_level} ${loss_scale} ${keep_batchnorm}"
        continue
      fi
      print_banner "${BASE_CMD} --opt-level ${opt_level} ${loss_scale} ${keep_batchnorm} $DATADIR"
      set -x
      ${BASE_CMD} --opt-level ${opt_level} ${loss_scale} ${keep_batchnorm} $DATADIR
      set +x
    done
  done
done

print_banner "Checking for bitwise accuracy between Python-only and cpp/cuda extension installs"

for opt_level in "${opt_levels[@]}"
do
  for loss_scale in "${loss_scales[@]}"
  do
    for keep_batchnorm in "${keep_batchnorms[@]}"
    do
      echo ""
      if [ "$opt_level" == "O1" ] && [ -n "${keep_batchnorm}" ]
      then
        echo "Skipping ${opt_level} ${loss_scale} ${keep_batchnorm}"
        continue
      fi
      echo "${BASE_CMD} --opt-level ${opt_level} ${loss_scale} ${keep_batchnorm} [--has-ext] $DATADIR"
      set -x
      python compare.py --opt-level ${opt_level} ${loss_scale} ${keep_batchnorm} --use_baseline
      set +x
    done
  done
done

print_banner "Reinstalling Apex with --cuda_ext and --cpp_ext"

pushd ../../..
pip install -v --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext" .
popd


#!/bin/bash

DATADIR="/home/mcarilli/Desktop/pt18data/apex_stale/examples/imagenet/bare_metal_train_val/"
# DATADIR="/opt/home/apex/examples/imagenet/"
cp ../common/* .
bash run_test.sh single_gpu $1 $DATADIR yes


#!/bin/bash

cp ../common/* .
bash run_test.sh distributed $1


#!/bin/bash

# Exit on error
# set -e

DESIRED_PROTO_VERSION="3.6.1"

# call protoc direclty, if version is not the desired one, download the desired vesrion.

if [ -f "protoc/bin/protoc" ]; then
  PROTOC_BIN="protoc/bin/protoc"
else
  PROTOC_BIN=`which protoc`
fi

echo "using" $PROTOC_BIN

CURRENT_PROTOC_VER=`${PROTOC_BIN} --version`
if [ -z ${PROTOC_BIN} ] || [[ "$CURRENT_PROTOC_VER" != "libprotoc "$DESIRED_PROTO_VERSION ]]; then
  # Download and use the latest version of protoc.
  if [ "$(uname)" == "Darwin" ]; then
    PROTOC_ZIP="protoc-"$DESIRED_PROTO_VERSION"-osx-x86_64.zip"
  else
    PROTOC_ZIP="protoc-"$DESIRED_PROTO_VERSION"-linux-x86_64.zip"
  fi
  WGET_BIN=`which wget`
  if [[ ! -z ${WGET_BIN} ]]; then
    ${WGET_BIN} https://github.com/protocolbuffers/protobuf/releases/download/v"$DESIRED_PROTO_VERSION"/${PROTOC_ZIP}
    rm -rf protoc
    python -c "import zipfile; zipfile.ZipFile('"${PROTOC_ZIP}"','r').extractall('protoc')"
    PROTOC_BIN=protoc/bin/protoc
    chmod +x ${PROTOC_BIN}
  fi
fi

# Regenerate
if [[ ! -z ${PROTOC_BIN} ]]; then
  # Delete all existing Python protobuf (*_pb2.py) output
  rm -rf tensorboardX/proto/*pb2*.py
  ${PROTOC_BIN} tensorboardX/proto/*.proto --python_out=.

  echo "Done generating tensorboardX/proto/*pb2*.py"
else
  echo "protoc not installed so can't regenerate tensorboardX/proto/*pb2*.py, using precompiled version."
fi


#!/bin/bash

set -euo pipefail

gem install nokogiri --quiet --silent

ruby junit.rb /junits/*


#!/usr/bin/env bash

# Copyright 2014 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# shellcheck disable=SC2034 # Variables sourced in other scripts.

# Common utilities, variables and checks for all build scripts.
set -o errexit
set -o nounset
set -o pipefail

# Unset CDPATH, having it set messes up with script import paths
unset CDPATH

USER_ID=$(id -u)
GROUP_ID=$(id -g)

DOCKER_OPTS=${DOCKER_OPTS:-""}
IFS=" " read -r -a DOCKER <<< "docker ${DOCKER_OPTS}"
DOCKER_HOST=${DOCKER_HOST:-""}

# This will canonicalize the path
KUBE_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd -P)

source "${KUBE_ROOT}/hack/lib/init.sh"

# Constants
readonly KUBE_BUILD_IMAGE_REPO=kube-build
readonly KUBE_BUILD_IMAGE_CROSS_TAG="$(cat "${KUBE_ROOT}/build/build-image/cross/VERSION")"

readonly KUBE_DOCKER_REGISTRY="${KUBE_DOCKER_REGISTRY:-k8s.gcr.io}"
readonly KUBE_BASE_IMAGE_REGISTRY="${KUBE_BASE_IMAGE_REGISTRY:-k8s.gcr.io/build-image}"

# This version number is used to cause everyone to rebuild their data containers
# and build image.  This is especially useful for automated build systems like
# Jenkins.
#
# Increment/change this number if you change the build image (anything under
# build/build-image) or change the set of volumes in the data container.
readonly KUBE_BUILD_IMAGE_VERSION_BASE="$(cat "${KUBE_ROOT}/build/build-image/VERSION")"
readonly KUBE_BUILD_IMAGE_VERSION="${KUBE_BUILD_IMAGE_VERSION_BASE}-${KUBE_BUILD_IMAGE_CROSS_TAG}"

# Here we map the output directories across both the local and remote _output
# directories:
#
# *_OUTPUT_ROOT    - the base of all output in that environment.
# *_OUTPUT_SUBPATH - location where golang stuff is built/cached.  Also
#                    persisted across docker runs with a volume mount.
# *_OUTPUT_BINPATH - location where final binaries are placed.  If the remote
#                    is really remote, this is the stuff that has to be copied
#                    back.
# OUT_DIR can come in from the Makefile, so honor it.
readonly LOCAL_OUTPUT_ROOT="${KUBE_ROOT}/${OUT_DIR:-_output}"
readonly LOCAL_OUTPUT_SUBPATH="${LOCAL_OUTPUT_ROOT}/dockerized"
readonly LOCAL_OUTPUT_BINPATH="${LOCAL_OUTPUT_SUBPATH}/bin"
readonly LOCAL_OUTPUT_GOPATH="${LOCAL_OUTPUT_SUBPATH}/go"
readonly LOCAL_OUTPUT_IMAGE_STAGING="${LOCAL_OUTPUT_ROOT}/images"

# This is a symlink to binaries for "this platform" (e.g. build tools).
readonly THIS_PLATFORM_BIN="${LOCAL_OUTPUT_ROOT}/bin"

readonly REMOTE_ROOT="/go/src/${KUBE_GO_PACKAGE}"
readonly REMOTE_OUTPUT_ROOT="${REMOTE_ROOT}/_output"
readonly REMOTE_OUTPUT_SUBPATH="${REMOTE_OUTPUT_ROOT}/dockerized"
readonly REMOTE_OUTPUT_BINPATH="${REMOTE_OUTPUT_SUBPATH}/bin"
readonly REMOTE_OUTPUT_GOPATH="${REMOTE_OUTPUT_SUBPATH}/go"

# This is the port on the workstation host to expose RSYNC on.  Set this if you
# are doing something fancy with ssh tunneling.
readonly KUBE_RSYNC_PORT="${KUBE_RSYNC_PORT:-}"

# This is the port that rsync is running on *inside* the container. This may be
# mapped to KUBE_RSYNC_PORT via docker networking.
readonly KUBE_CONTAINER_RSYNC_PORT=8730

# These are the default versions (image tags) for their respective base images.
readonly __default_debian_iptables_version=buster-v1.6.5
readonly __default_go_runner_version=v2.3.1-go1.16.5-buster.0
readonly __default_setcap_version=buster-v2.0.3

# These are the base images for the Docker-wrapped binaries.
readonly KUBE_GORUNNER_IMAGE="${KUBE_GORUNNER_IMAGE:-$KUBE_BASE_IMAGE_REGISTRY/go-runner:$__default_go_runner_version}"
readonly KUBE_APISERVER_BASE_IMAGE="${KUBE_APISERVER_BASE_IMAGE:-$KUBE_GORUNNER_IMAGE}"
readonly KUBE_CONTROLLER_MANAGER_BASE_IMAGE="${KUBE_CONTROLLER_MANAGER_BASE_IMAGE:-$KUBE_GORUNNER_IMAGE}"
readonly KUBE_SCHEDULER_BASE_IMAGE="${KUBE_SCHEDULER_BASE_IMAGE:-$KUBE_GORUNNER_IMAGE}"
readonly KUBE_PROXY_BASE_IMAGE="${KUBE_PROXY_BASE_IMAGE:-$KUBE_BASE_IMAGE_REGISTRY/debian-iptables:$__default_debian_iptables_version}"

# This is the image used in a multi-stage build to apply capabilities to Docker-wrapped binaries.
readonly KUBE_BUILD_SETCAP_IMAGE="${KUBE_BUILD_SETCAP_IMAGE:-$KUBE_BASE_IMAGE_REGISTRY/setcap:$__default_setcap_version}"

# Get the set of master binaries that run in Docker (on Linux)
# Entry format is "<binary-name>,<base-image>".
# Binaries are placed in /usr/local/bin inside the image.
# `make` users can override any or all of the base images using the associated
# environment variables.
#
# $1 - server architecture
kube::build::get_docker_wrapped_binaries() {
  ### If you change any of these lists, please also update DOCKERIZED_BINARIES
  ### in build/BUILD. And kube::golang::server_image_targets
  local targets=(
    "kube-apiserver,${KUBE_APISERVER_BASE_IMAGE}"
    "kube-controller-manager,${KUBE_CONTROLLER_MANAGER_BASE_IMAGE}"
    "kube-scheduler,${KUBE_SCHEDULER_BASE_IMAGE}"
    "kube-proxy,${KUBE_PROXY_BASE_IMAGE}"
  )

  echo "${targets[@]}"
}

# ---------------------------------------------------------------------------
# Basic setup functions

# Verify that the right utilities and such are installed for building Kube. Set
# up some dynamic constants.
# Args:
#   $1 - boolean of whether to require functioning docker (default true)
#
# Vars set:
#   KUBE_ROOT_HASH
#   KUBE_BUILD_IMAGE_TAG_BASE
#   KUBE_BUILD_IMAGE_TAG
#   KUBE_BUILD_IMAGE
#   KUBE_BUILD_CONTAINER_NAME_BASE
#   KUBE_BUILD_CONTAINER_NAME
#   KUBE_DATA_CONTAINER_NAME_BASE
#   KUBE_DATA_CONTAINER_NAME
#   KUBE_RSYNC_CONTAINER_NAME_BASE
#   KUBE_RSYNC_CONTAINER_NAME
#   DOCKER_MOUNT_ARGS
#   LOCAL_OUTPUT_BUILD_CONTEXT
function kube::build::verify_prereqs() {
  local -r require_docker=${1:-true}
  kube::log::status "Verifying Prerequisites...."
  kube::build::ensure_tar || return 1
  kube::build::ensure_rsync || return 1
  if ${require_docker}; then
    kube::build::ensure_docker_in_path || return 1
    if kube::build::is_osx; then
        kube::build::docker_available_on_osx || return 1
    fi
    kube::util::ensure_docker_daemon_connectivity || return 1

    if (( KUBE_VERBOSE > 6 )); then
      kube::log::status "Docker Version:"
      "${DOCKER[@]}" version | kube::log::info_from_stdin
    fi
  fi

  KUBE_GIT_BRANCH=$(git symbolic-ref --short -q HEAD 2>/dev/null || true)
  KUBE_ROOT_HASH=$(kube::build::short_hash "${HOSTNAME:-}:${KUBE_ROOT}:${KUBE_GIT_BRANCH}")
  KUBE_BUILD_IMAGE_TAG_BASE="build-${KUBE_ROOT_HASH}"
  KUBE_BUILD_IMAGE_TAG="${KUBE_BUILD_IMAGE_TAG_BASE}-${KUBE_BUILD_IMAGE_VERSION}"
  KUBE_BUILD_IMAGE="${KUBE_BUILD_IMAGE_REPO}:${KUBE_BUILD_IMAGE_TAG}"
  KUBE_BUILD_CONTAINER_NAME_BASE="kube-build-${KUBE_ROOT_HASH}"
  KUBE_BUILD_CONTAINER_NAME="${KUBE_BUILD_CONTAINER_NAME_BASE}-${KUBE_BUILD_IMAGE_VERSION}"
  KUBE_RSYNC_CONTAINER_NAME_BASE="kube-rsync-${KUBE_ROOT_HASH}"
  KUBE_RSYNC_CONTAINER_NAME="${KUBE_RSYNC_CONTAINER_NAME_BASE}-${KUBE_BUILD_IMAGE_VERSION}"
  KUBE_DATA_CONTAINER_NAME_BASE="kube-build-data-${KUBE_ROOT_HASH}"
  KUBE_DATA_CONTAINER_NAME="${KUBE_DATA_CONTAINER_NAME_BASE}-${KUBE_BUILD_IMAGE_VERSION}"
  DOCKER_MOUNT_ARGS=(--volumes-from "${KUBE_DATA_CONTAINER_NAME}")
  LOCAL_OUTPUT_BUILD_CONTEXT="${LOCAL_OUTPUT_IMAGE_STAGING}/${KUBE_BUILD_IMAGE}"

  kube::version::get_version_vars
  kube::version::save_version_vars "${KUBE_ROOT}/.dockerized-kube-version-defs"

  # Without this, the user's umask can leak through.
  umask 0022
}

# ---------------------------------------------------------------------------
# Utility functions

function kube::build::docker_available_on_osx() {
  if [[ -z "${DOCKER_HOST}" ]]; then
    if [[ -S "/var/run/docker.sock" ]]; then
      kube::log::status "Using Docker for MacOS"
      return 0
    fi

    kube::log::status "No docker host is set."
    kube::log::status "It looks like you're running Mac OS X, but Docker for Mac cannot be found."
    kube::log::status "See: https://docs.docker.com/engine/installation/mac/ for installation instructions."
    return 1
  fi
}

function kube::build::is_osx() {
  [[ "$(uname)" == "Darwin" ]]
}

function kube::build::is_gnu_sed() {
  [[ $(sed --version 2>&1) == *GNU* ]]
}

function kube::build::ensure_rsync() {
  if [[ -z "$(which rsync)" ]]; then
    kube::log::error "Can't find 'rsync' in PATH, please fix and retry."
    return 1
  fi
}

function kube::build::ensure_docker_in_path() {
  if [[ -z "$(which docker)" ]]; then
    kube::log::error "Can't find 'docker' in PATH, please fix and retry."
    kube::log::error "See https://docs.docker.com/installation/#installation for installation instructions."
    return 1
  fi
}

function kube::build::ensure_tar() {
  if [[ -n "${TAR:-}" ]]; then
    return
  fi

  # Find gnu tar if it is available, bomb out if not.
  TAR=tar
  if which gtar &>/dev/null; then
      TAR=gtar
  else
      if which gnutar &>/dev/null; then
	  TAR=gnutar
      fi
  fi
  if ! "${TAR}" --version | grep -q GNU; then
    echo "  !!! Cannot find GNU tar. Build on Linux or install GNU tar"
    echo "      on Mac OS X (brew install gnu-tar)."
    return 1
  fi
}

function kube::build::has_docker() {
  which docker &> /dev/null
}

function kube::build::has_ip() {
  which ip &> /dev/null && ip -Version | grep 'iproute2' &> /dev/null
}

# Detect if a specific image exists
#
# $1 - image repo name
# $2 - image tag
function kube::build::docker_image_exists() {
  [[ -n $1 && -n $2 ]] || {
    kube::log::error "Internal error. Image not specified in docker_image_exists."
    exit 2
  }

  [[ $("${DOCKER[@]}" images -q "${1}:${2}") ]]
}

# Delete all images that match a tag prefix except for the "current" version
#
# $1: The image repo/name
# $2: The tag base. We consider any image that matches $2*
# $3: The current image not to delete if provided
function kube::build::docker_delete_old_images() {
  # In Docker 1.12, we can replace this with
  #    docker images "$1" --format "{{.Tag}}"
  for tag in $("${DOCKER[@]}" images "${1}" | tail -n +2 | awk '{print $2}') ; do
    if [[ "${tag}" != "${2}"* ]] ; then
      V=3 kube::log::status "Keeping image ${1}:${tag}"
      continue
    fi

    if [[ -z "${3:-}" || "${tag}" != "${3}" ]] ; then
      V=2 kube::log::status "Deleting image ${1}:${tag}"
      "${DOCKER[@]}" rmi "${1}:${tag}" >/dev/null
    else
      V=3 kube::log::status "Keeping image ${1}:${tag}"
    fi
  done
}

# Stop and delete all containers that match a pattern
#
# $1: The base container prefix
# $2: The current container to keep, if provided
function kube::build::docker_delete_old_containers() {
  # In Docker 1.12 we can replace this line with
  #   docker ps -a --format="{{.Names}}"
  for container in $("${DOCKER[@]}" ps -a | tail -n +2 | awk '{print $NF}') ; do
    if [[ "${container}" != "${1}"* ]] ; then
      V=3 kube::log::status "Keeping container ${container}"
      continue
    fi
    if [[ -z "${2:-}" || "${container}" != "${2}" ]] ; then
      V=2 kube::log::status "Deleting container ${container}"
      kube::build::destroy_container "${container}"
    else
      V=3 kube::log::status "Keeping container ${container}"
    fi
  done
}

# Takes $1 and computes a short has for it. Useful for unique tag generation
function kube::build::short_hash() {
  [[ $# -eq 1 ]] || {
    kube::log::error "Internal error.  No data based to short_hash."
    exit 2
  }

  local short_hash
  if which md5 >/dev/null 2>&1; then
    short_hash=$(md5 -q -s "$1")
  else
    short_hash=$(echo -n "$1" | md5sum)
  fi
  echo "${short_hash:0:10}"
}

# Pedantically kill, wait-on and remove a container. The -f -v options
# to rm don't actually seem to get the job done, so force kill the
# container, wait to ensure it's stopped, then try the remove. This is
# a workaround for bug https://github.com/docker/docker/issues/3968.
function kube::build::destroy_container() {
  "${DOCKER[@]}" kill "$1" >/dev/null 2>&1 || true
  if [[ $("${DOCKER[@]}" version --format '{{.Server.Version}}') = 17.06.0* ]]; then
    # Workaround https://github.com/moby/moby/issues/33948.
    # TODO: remove when 17.06.0 is not relevant anymore
    DOCKER_API_VERSION=v1.29 "${DOCKER[@]}" wait "$1" >/dev/null 2>&1 || true
  else
    "${DOCKER[@]}" wait "$1" >/dev/null 2>&1 || true
  fi
  "${DOCKER[@]}" rm -f -v "$1" >/dev/null 2>&1 || true
}

# ---------------------------------------------------------------------------
# Building

function kube::build::clean() {
  if kube::build::has_docker ; then
    kube::build::docker_delete_old_containers "${KUBE_BUILD_CONTAINER_NAME_BASE}"
    kube::build::docker_delete_old_containers "${KUBE_RSYNC_CONTAINER_NAME_BASE}"
    kube::build::docker_delete_old_containers "${KUBE_DATA_CONTAINER_NAME_BASE}"
    kube::build::docker_delete_old_images "${KUBE_BUILD_IMAGE_REPO}" "${KUBE_BUILD_IMAGE_TAG_BASE}"

    V=2 kube::log::status "Cleaning all untagged docker images"
    "${DOCKER[@]}" rmi "$("${DOCKER[@]}" images -q --filter 'dangling=true')" 2> /dev/null || true
  fi

  if [[ -d "${LOCAL_OUTPUT_ROOT}" ]]; then
    kube::log::status "Removing _output directory"
    rm -rf "${LOCAL_OUTPUT_ROOT}"
  fi
}

# Set up the context directory for the kube-build image and build it.
function kube::build::build_image() {
  mkdir -p "${LOCAL_OUTPUT_BUILD_CONTEXT}"
  # Make sure the context directory owned by the right user for syncing sources to container.
  chown -R "${USER_ID}":"${GROUP_ID}" "${LOCAL_OUTPUT_BUILD_CONTEXT}"

  cp /etc/localtime "${LOCAL_OUTPUT_BUILD_CONTEXT}/"
  chmod u+w "${LOCAL_OUTPUT_BUILD_CONTEXT}/localtime"

  cp "${KUBE_ROOT}/build/build-image/Dockerfile" "${LOCAL_OUTPUT_BUILD_CONTEXT}/Dockerfile"
  cp "${KUBE_ROOT}/build/build-image/rsyncd.sh" "${LOCAL_OUTPUT_BUILD_CONTEXT}/"
  dd if=/dev/urandom bs=512 count=1 2>/dev/null | LC_ALL=C tr -dc 'A-Za-z0-9' | dd bs=32 count=1 2>/dev/null > "${LOCAL_OUTPUT_BUILD_CONTEXT}/rsyncd.password"
  chmod go= "${LOCAL_OUTPUT_BUILD_CONTEXT}/rsyncd.password"

  kube::build::docker_build "${KUBE_BUILD_IMAGE}" "${LOCAL_OUTPUT_BUILD_CONTEXT}" 'false' "--build-arg=KUBE_BUILD_IMAGE_CROSS_TAG=${KUBE_BUILD_IMAGE_CROSS_TAG} --build-arg=KUBE_BASE_IMAGE_REGISTRY=${KUBE_BASE_IMAGE_REGISTRY}"

  # Clean up old versions of everything
  kube::build::docker_delete_old_containers "${KUBE_BUILD_CONTAINER_NAME_BASE}" "${KUBE_BUILD_CONTAINER_NAME}"
  kube::build::docker_delete_old_containers "${KUBE_RSYNC_CONTAINER_NAME_BASE}" "${KUBE_RSYNC_CONTAINER_NAME}"
  kube::build::docker_delete_old_containers "${KUBE_DATA_CONTAINER_NAME_BASE}" "${KUBE_DATA_CONTAINER_NAME}"
  kube::build::docker_delete_old_images "${KUBE_BUILD_IMAGE_REPO}" "${KUBE_BUILD_IMAGE_TAG_BASE}" "${KUBE_BUILD_IMAGE_TAG}"

  kube::build::ensure_data_container
  kube::build::sync_to_container
}

# Build a docker image from a Dockerfile.
# $1 is the name of the image to build
# $2 is the location of the "context" directory, with the Dockerfile at the root.
# $3 is the value to set the --pull flag for docker build; true by default
# $4 is the set of --build-args for docker.
function kube::build::docker_build() {
  local -r image=$1
  local -r context_dir=$2
  local -r pull="${3:-true}"
  local build_args
  IFS=" " read -r -a build_args <<< "$4"
  readonly build_args
  local -ra build_cmd=("${DOCKER[@]}" buildx build --load -t "${image}" "--pull=${pull}" "${build_args[@]}" "${context_dir}")

  kube::log::status "Building Docker image ${image}"
  local docker_output
  docker_output=$(DOCKER_CLI_EXPERIMENTAL=enabled "${build_cmd[@]}" 2>&1) || {
    cat <<EOF >&2
+++ Docker build command failed for ${image}

${docker_output}

To retry manually, run:

DOCKER_CLI_EXPERIMENTAL=enabled ${build_cmd[*]}

EOF
    return 1
  }
}

function kube::build::ensure_data_container() {
  # If the data container exists AND exited successfully, we can use it.
  # Otherwise nuke it and start over.
  local ret=0
  local code=0

  code=$(docker inspect \
      -f '{{.State.ExitCode}}' \
      "${KUBE_DATA_CONTAINER_NAME}" 2>/dev/null) || ret=$?
  if [[ "${ret}" == 0 && "${code}" != 0 ]]; then
    kube::build::destroy_container "${KUBE_DATA_CONTAINER_NAME}"
    ret=1
  fi
  if [[ "${ret}" != 0 ]]; then
    kube::log::status "Creating data container ${KUBE_DATA_CONTAINER_NAME}"
    # We have to ensure the directory exists, or else the docker run will
    # create it as root.
    mkdir -p "${LOCAL_OUTPUT_GOPATH}"
    # We want this to run as root to be able to chown, so non-root users can
    # later use the result as a data container.  This run both creates the data
    # container and chowns the GOPATH.
    #
    # The data container creates volumes for all of the directories that store
    # intermediates for the Go build. This enables incremental builds across
    # Docker sessions. The *_cgo paths are re-compiled versions of the go std
    # libraries for true static building.
    local -ra docker_cmd=(
      "${DOCKER[@]}" run
      --volume "${REMOTE_ROOT}"   # white-out the whole output dir
      --volume /usr/local/go/pkg/linux_386_cgo
      --volume /usr/local/go/pkg/linux_amd64_cgo
      --volume /usr/local/go/pkg/linux_arm_cgo
      --volume /usr/local/go/pkg/linux_arm64_cgo
      --volume /usr/local/go/pkg/linux_ppc64le_cgo
      --volume /usr/local/go/pkg/darwin_amd64_cgo
      --volume /usr/local/go/pkg/darwin_386_cgo
      --volume /usr/local/go/pkg/windows_amd64_cgo
      --volume /usr/local/go/pkg/windows_386_cgo
      --name "${KUBE_DATA_CONTAINER_NAME}"
      --hostname "${HOSTNAME}"
      "${KUBE_BUILD_IMAGE}"
      chown -R "${USER_ID}":"${GROUP_ID}"
        "${REMOTE_ROOT}"
        /usr/local/go/pkg/
    )
    "${docker_cmd[@]}"
  fi
}

# Run a command in the kube-build image.  This assumes that the image has
# already been built.
function kube::build::run_build_command() {
  kube::log::status "Running build command..."
  kube::build::run_build_command_ex "${KUBE_BUILD_CONTAINER_NAME}" -- "$@"
}

# Run a command in the kube-build image.  This assumes that the image has
# already been built.
#
# Arguments are in the form of
#  <container name> <extra docker args> -- <command>
function kube::build::run_build_command_ex() {
  [[ $# != 0 ]] || { echo "Invalid input - please specify a container name." >&2; return 4; }
  local container_name="${1}"
  shift

  local -a docker_run_opts=(
    "--name=${container_name}"
    "--user=$(id -u):$(id -g)"
    "--hostname=${HOSTNAME}"
    "${DOCKER_MOUNT_ARGS[@]}"
  )

  local detach=false

  [[ $# != 0 ]] || { echo "Invalid input - please specify docker arguments followed by --." >&2; return 4; }
  # Everything before "--" is an arg to docker
  until [ -z "${1-}" ] ; do
    if [[ "$1" == "--" ]]; then
      shift
      break
    fi
    docker_run_opts+=("$1")
    if [[ "$1" == "-d" || "$1" == "--detach" ]] ; then
      detach=true
    fi
    shift
  done

  # Everything after "--" is the command to run
  [[ $# != 0 ]] || { echo "Invalid input - please specify a command to run." >&2; return 4; }
  local -a cmd=()
  until [ -z "${1-}" ] ; do
    cmd+=("$1")
    shift
  done

  docker_run_opts+=(
    --env "KUBE_FASTBUILD=${KUBE_FASTBUILD:-false}"
    --env "KUBE_BUILDER_OS=${OSTYPE:-notdetected}"
    --env "KUBE_VERBOSE=${KUBE_VERBOSE}"
    --env "KUBE_BUILD_WITH_COVERAGE=${KUBE_BUILD_WITH_COVERAGE:-}"
    --env "KUBE_BUILD_PLATFORMS=${KUBE_BUILD_PLATFORMS:-}"
    --env "GOFLAGS=${GOFLAGS:-}"
    --env "GOGCFLAGS=${GOGCFLAGS:-}"
    --env "SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH:-}"
  )

  # use GOLDFLAGS only if it is set explicitly.
  if [[ -v GOLDFLAGS ]]; then
    docker_run_opts+=(
      --env "GOLDFLAGS=${GOLDFLAGS:-}"
    )
  fi

  if [[ -n "${DOCKER_CGROUP_PARENT:-}" ]]; then
    kube::log::status "Using ${DOCKER_CGROUP_PARENT} as container cgroup parent"
    docker_run_opts+=(--cgroup-parent "${DOCKER_CGROUP_PARENT}")
  fi

  # If we have stdin we can run interactive.  This allows things like 'shell.sh'
  # to work.  However, if we run this way and don't have stdin, then it ends up
  # running in a daemon-ish mode.  So if we don't have a stdin, we explicitly
  # attach stderr/stdout but don't bother asking for a tty.
  if [[ -t 0 ]]; then
    docker_run_opts+=(--interactive --tty)
  elif [[ "${detach}" == false ]]; then
    docker_run_opts+=("--attach=stdout" "--attach=stderr")
  fi

  local -ra docker_cmd=(
    "${DOCKER[@]}" run "${docker_run_opts[@]}" "${KUBE_BUILD_IMAGE}")

  # Clean up container from any previous run
  kube::build::destroy_container "${container_name}"
  "${docker_cmd[@]}" "${cmd[@]}"
  if [[ "${detach}" == false ]]; then
    kube::build::destroy_container "${container_name}"
  fi
}

function kube::build::rsync_probe {
  # Wait unil rsync is up and running.
  local tries=20
  while (( tries > 0 )) ; do
    if rsync "rsync://k8s@${1}:${2}/" \
         --password-file="${LOCAL_OUTPUT_BUILD_CONTEXT}/rsyncd.password" \
         &> /dev/null ; then
      return 0
    fi
    tries=$(( tries - 1))
    sleep 0.1
  done

  return 1
}

# Start up the rsync container in the background. This should be explicitly
# stopped with kube::build::stop_rsyncd_container.
#
# This will set the global var KUBE_RSYNC_ADDR to the effective port that the
# rsync daemon can be reached out.
function kube::build::start_rsyncd_container() {
  IPTOOL=ifconfig
  if kube::build::has_ip ; then
    IPTOOL="ip address"
  fi
  kube::build::stop_rsyncd_container
  V=3 kube::log::status "Starting rsyncd container"
  kube::build::run_build_command_ex \
    "${KUBE_RSYNC_CONTAINER_NAME}" -p 127.0.0.1:"${KUBE_RSYNC_PORT}":"${KUBE_CONTAINER_RSYNC_PORT}" -d \
    -e ALLOW_HOST="$(${IPTOOL} | grep -Eo 'inet (addr:)?([0-9]*\.){3}[0-9]*' | grep -Eo '([0-9]*\.){3}[0-9]*' | grep -v '127.0.0.1')" \
    -- /rsyncd.sh >/dev/null

  local mapped_port
  if ! mapped_port=$("${DOCKER[@]}" port "${KUBE_RSYNC_CONTAINER_NAME}" ${KUBE_CONTAINER_RSYNC_PORT} 2> /dev/null | cut -d: -f 2) ; then
    kube::log::error "Could not get effective rsync port"
    return 1
  fi

  local container_ip
  container_ip=$("${DOCKER[@]}" inspect --format '{{ .NetworkSettings.IPAddress }}' "${KUBE_RSYNC_CONTAINER_NAME}")

  # Sometimes we can reach rsync through localhost and a NAT'd port.  Other
  # times (when we are running in another docker container on the Jenkins
  # machines) we have to talk directly to the container IP.  There is no one
  # strategy that works in all cases so we test to figure out which situation we
  # are in.
  if kube::build::rsync_probe 127.0.0.1 "${mapped_port}"; then
    KUBE_RSYNC_ADDR="127.0.0.1:${mapped_port}"
    return 0
  elif kube::build::rsync_probe "${container_ip}" ${KUBE_CONTAINER_RSYNC_PORT}; then
    KUBE_RSYNC_ADDR="${container_ip}:${KUBE_CONTAINER_RSYNC_PORT}"
    return 0
  fi

  kube::log::error "Could not connect to rsync container."
  return 1
}

function kube::build::stop_rsyncd_container() {
  V=3 kube::log::status "Stopping any currently running rsyncd container"
  unset KUBE_RSYNC_ADDR
  kube::build::destroy_container "${KUBE_RSYNC_CONTAINER_NAME}"
}

function kube::build::rsync {
  local -a rsync_opts=(
    --archive
    "--password-file=${LOCAL_OUTPUT_BUILD_CONTEXT}/rsyncd.password"
  )
  if (( KUBE_VERBOSE >= 6 )); then
    rsync_opts+=("-iv")
  fi
  if (( KUBE_RSYNC_COMPRESS > 0 )); then
     rsync_opts+=("--compress-level=${KUBE_RSYNC_COMPRESS}")
  fi
  V=3 kube::log::status "Running rsync"
  rsync "${rsync_opts[@]}" "$@"
}

# This will launch rsyncd in a container and then sync the source tree to the
# container over the local network.
function kube::build::sync_to_container() {
  kube::log::status "Syncing sources to container"

  kube::build::start_rsyncd_container

  # rsync filters are a bit confusing.  Here we are syncing everything except
  # output only directories and things that are not necessary like the git
  # directory and generated files. The '- /' filter prevents rsync
  # from trying to set the uid/gid/perms on the root of the sync tree.
  # As an exception, we need to sync generated files in staging/, because
  # they will not be re-generated by 'make'. Note that the 'H' filtered files
  # are hidden from rsync so they will be deleted in the target container if
  # they exist. This will allow them to be re-created in the container if
  # necessary.
  kube::build::rsync \
    --delete \
    --filter='H /.git' \
    --filter='- /.make/' \
    --filter='- /_tmp/' \
    --filter='- /_output/' \
    --filter='- /' \
    --filter='H zz_generated.*' \
    --filter='H generated.proto' \
    "${KUBE_ROOT}/" "rsync://k8s@${KUBE_RSYNC_ADDR}/k8s/"

  kube::build::stop_rsyncd_container
}

# Copy all build results back out.
function kube::build::copy_output() {
  kube::log::status "Syncing out of container"

  kube::build::start_rsyncd_container

  # The filter syntax for rsync is a little obscure. It filters on files and
  # directories.  If you don't go in to a directory you won't find any files
  # there.  Rules are evaluated in order.  The last two rules are a little
  # magic. '+ */' says to go in to every directory and '- /**' says to ignore
  # any file or directory that isn't already specifically allowed.
  #
  # We are looking to copy out all of the built binaries along with various
  # generated files.
  kube::build::rsync \
    --prune-empty-dirs \
    --filter='- /_temp/' \
    --filter='+ /vendor/' \
    --filter='+ /staging/***/Godeps/**' \
    --filter='+ /_output/dockerized/bin/**' \
    --filter='+ zz_generated.*' \
    --filter='+ generated.proto' \
    --filter='+ *.pb.go' \
    --filter='+ types.go' \
    --filter='+ */' \
    --filter='- /**' \
    "rsync://k8s@${KUBE_RSYNC_ADDR}/k8s/" "${KUBE_ROOT}"

  kube::build::stop_rsyncd_container
}


#!/usr/bin/env bash

# Copyright 2014 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Copies any built binaries (and other generated files) out of the Docker build container.
set -o errexit
set -o nounset
set -o pipefail

KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
source "${KUBE_ROOT}/build/common.sh"

kube::build::verify_prereqs
kube::build::copy_output


#!/usr/bin/env bash

# Copyright 2014 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Build the docker image necessary for building Kubernetes
#
# This script will package the parts of the repo that we need to build
# Kubernetes into a tar file and put it in the right place in the output
# directory.  It will then copy over the Dockerfile and build the kube-build
# image.
set -o errexit
set -o nounset
set -o pipefail

KUBE_ROOT="$(dirname "${BASH_SOURCE[0]}")/.."
source "${KUBE_ROOT}/build/common.sh"

kube::build::verify_prereqs
kube::build::build_image


#!/usr/bin/env bash

# Copyright 2014 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Clean out the output directory on the docker host.
set -o errexit
set -o nounset
set -o pipefail

KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
source "${KUBE_ROOT}/build/common.sh"

kube::build::verify_prereqs false
kube::build::clean


#!/usr/bin/env bash
# Copyright 2017 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -o errexit
set -o nounset
set -o pipefail

# Complete the release with the standard env
KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
source "${KUBE_ROOT}/build/common.sh"
source "${KUBE_ROOT}/build/lib/release.sh"

kube::build::ensure_tar
kube::version::get_version_vars
kube::release::package_tarballs


#!/usr/bin/env bash

# Copyright 2018 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Build Kubernetes release images. This will build the server target binaries,
# and create wrap them in Docker images, see `make release` for full releases

set -o errexit
set -o nounset
set -o pipefail

KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
source "${KUBE_ROOT}/build/common.sh"
source "${KUBE_ROOT}/build/lib/release.sh"

CMD_TARGETS="${KUBE_SERVER_IMAGE_TARGETS[*]}"
if [[ "${KUBE_BUILD_CONFORMANCE}" =~ [yY] ]]; then
    CMD_TARGETS="${CMD_TARGETS} ${KUBE_CONFORMANCE_IMAGE_TARGETS[*]}"
fi
# include extra WHAT if specified so you can build docker images + binaries
# in one call with a single pass of syncing to the container + generating code
if [[ -n "${KUBE_EXTRA_WHAT:-}" ]]; then
    CMD_TARGETS="${CMD_TARGETS} ${KUBE_EXTRA_WHAT}"
fi

kube::build::verify_prereqs
kube::build::build_image
kube::build::run_build_command make all WHAT="${CMD_TARGETS}" KUBE_BUILD_PLATFORMS="${KUBE_SERVER_PLATFORMS[*]}"

kube::build::copy_output

kube::release::build_server_images


#!/usr/bin/env bash
# Copyright 2017 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -o errexit
set -o nounset
set -o pipefail

# Complete the release with the standard env
KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..

# Check and error if not "in-a-container"
if [[ ! -f /.dockerenv ]]; then
  echo
  echo "'make release-in-a-container' can only be used from a docker container."
  echo
  exit 1
fi

# Other dependencies: Your container should contain docker
if ! type -p docker >/dev/null 2>&1; then
  echo
  echo "'make release-in-a-container' requires a container with" \
       "docker installed."
  echo
  exit 1
fi

# First run make cross-in-a-container
make cross-in-a-container

# at the moment only make test is supported.
if [[ $KUBE_RELEASE_RUN_TESTS =~ ^[yY]$ ]]; then
  make test
fi

"${KUBE_ROOT}/build/package-tarballs.sh"


#!/usr/bin/env bash

# Copyright 2014 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Build a Kubernetes release.  This will build the binaries, create the Docker
# images and other build artifacts.
#
# For pushing these artifacts publicly to Google Cloud Storage or to a registry
# please refer to the kubernetes/release repo at
# https://github.com/kubernetes/release.

set -o errexit
set -o nounset
set -o pipefail

KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
source "${KUBE_ROOT}/build/common.sh"
source "${KUBE_ROOT}/build/lib/release.sh"

KUBE_RELEASE_RUN_TESTS=${KUBE_RELEASE_RUN_TESTS-y}

kube::build::verify_prereqs
kube::build::build_image
kube::build::run_build_command make cross

if [[ $KUBE_RELEASE_RUN_TESTS =~ ^[yY]$ ]]; then
  kube::build::run_build_command make test
  kube::build::run_build_command make test-integration
fi

kube::build::copy_output

kube::release::package_tarballs


#!/usr/bin/env bash

# Copyright 2014 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Run a command in the docker build container.  Typically this will be one of
# the commands in `hack/`.  When running in the build container the user is sure
# to have a consistent reproducible build environment.

set -o errexit
set -o nounset
set -o pipefail

KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
source "$KUBE_ROOT/build/common.sh"

KUBE_RUN_COPY_OUTPUT="${KUBE_RUN_COPY_OUTPUT:-y}"

kube::build::verify_prereqs
kube::build::build_image

if [[ ${KUBE_RUN_COPY_OUTPUT} =~ ^[yY]$ ]]; then
  kube::log::status "Output from this container will be rsynced out upon completion. Set KUBE_RUN_COPY_OUTPUT=n to disable."
else
  kube::log::status "Output from this container will NOT be rsynced out upon completion. Set KUBE_RUN_COPY_OUTPUT=y to enable."
fi

kube::build::run_build_command "$@"

if [[ ${KUBE_RUN_COPY_OUTPUT} =~ ^[yY]$ ]]; then
  kube::build::copy_output
fi


#!/usr/bin/env bash

# Copyright 2014 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Run a bash script in the Docker build image.
#
# This container will have a snapshot of the current sources.

set -o errexit
set -o nounset
set -o pipefail

KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
source "${KUBE_ROOT}/build/common.sh"

KUBE_RUN_COPY_OUTPUT="${KUBE_RUN_COPY_OUTPUT:-n}" "${KUBE_ROOT}/build/run.sh" bash "$@"


#!/usr/bin/env bash

# Copyright 2016 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Common utility functions for build scripts

KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..

function kube::release::semantic_version() {
  # This takes:
  # Client Version: version.Info{Major:"1", Minor:"1+", GitVersion:"v1.1.0-alpha.0.2328+3c0a05de4a38e3", GitCommit:"3c0a05de4a38e355d147dbfb4d85bad6d2d73bb9", GitTreeState:"clean"}
  # and spits back the GitVersion piece in a way that is somewhat
  # resilient to the other fields changing (we hope)
  "${KUBE_ROOT}/cluster/kubectl.sh" version --client | sed "s/, */\\
/g" | grep -E "^GitVersion:" | cut -f2 -d: | cut -f2 -d\"
}

function kube::release::semantic_image_tag_version() {
    printf "%s" "$(kube::release::semantic_version)" | tr + _
}


#!/usr/bin/env bash

# Copyright 2016 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This script will set up and run rsyncd to allow data to move into and out of
# our dockerized build system.  This is used for syncing sources and changes of
# sources into the docker-build-container.  It is also used to transfer built binaries
# and generated files back out.
#
# When run as root (rare) it'll preserve the file ids as sent from the client.
# Usually it'll be run as non-dockerized UID/GID and end up translating all file
# ownership to that.

set -o errexit
set -o nounset
set -o pipefail

# The directory that gets sync'd
VOLUME=${HOME}

# Assume that this is running in Docker on a bridge.  Allow connections from
# anything on the local subnet.
ALLOW=$(ip route | awk  '/^default via/ { reg = "^[0-9./]+ dev "$5 } ; $0 ~ reg { print $1 }')

CONFDIR="/tmp/rsync.k8s"
PIDFILE="${CONFDIR}/rsyncd.pid"
CONFFILE="${CONFDIR}/rsyncd.conf"
SECRETS="${CONFDIR}/rsyncd.secrets"

mkdir -p "${CONFDIR}"

if [[ -f "${PIDFILE}" ]]; then
  PID=$(cat "${PIDFILE}")
  echo "Cleaning up old PID file: ${PIDFILE}"
  kill "${PID}" &> /dev/null || true
  rm "${PIDFILE}"
fi

PASSWORD=$(</rsyncd.password)

cat <<EOF >"${SECRETS}"
k8s:${PASSWORD}
EOF
chmod go= "${SECRETS}"

USER_CONFIG=
if [[ "$(id -u)" == "0" ]]; then
  USER_CONFIG="  uid = 0"$'\n'"  gid = 0"
fi

cat <<EOF >"${CONFFILE}"
pid file = ${PIDFILE}
use chroot = no
log file = /dev/stdout
reverse lookup = no
munge symlinks = no
port = 8730
[k8s]
  numeric ids = true
  $USER_CONFIG
  hosts deny = *
  hosts allow = ${ALLOW} ${ALLOW_HOST-}
  auth users = k8s
  secrets file = ${SECRETS}
  read only = false
  path = ${VOLUME}
  filter = - /.make/ - /_tmp/
EOF

exec /usr/bin/rsync --no-detach --daemon --config="${CONFFILE}" "$@"


#!/usr/bin/env bash

# Copyright 2016 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This file creates release artifacts (tar files, container images) that are
# ready to distribute to install or distribute to end users.

###############################################################################
# Most of the ::release:: namespace functions have been moved to
# github.com/kubernetes/release.  Have a look in that repo and specifically in
# lib/releaselib.sh for ::release::-related functionality.
###############################################################################

# This is where the final release artifacts are created locally
readonly RELEASE_STAGE="${LOCAL_OUTPUT_ROOT}/release-stage"
readonly RELEASE_TARS="${LOCAL_OUTPUT_ROOT}/release-tars"
readonly RELEASE_IMAGES="${LOCAL_OUTPUT_ROOT}/release-images"

KUBE_BUILD_CONFORMANCE=${KUBE_BUILD_CONFORMANCE:-n}
KUBE_BUILD_PULL_LATEST_IMAGES=${KUBE_BUILD_PULL_LATEST_IMAGES:-y}

# Validate a ci version
#
# Globals:
#   None
# Arguments:
#   version
# Returns:
#   If version is a valid ci version
# Sets:                    (e.g. for '1.2.3-alpha.4.56+abcdef12345678')
#   VERSION_MAJOR          (e.g. '1')
#   VERSION_MINOR          (e.g. '2')
#   VERSION_PATCH          (e.g. '3')
#   VERSION_PRERELEASE     (e.g. 'alpha')
#   VERSION_PRERELEASE_REV (e.g. '4')
#   VERSION_BUILD_INFO     (e.g. '.56+abcdef12345678')
#   VERSION_COMMITS        (e.g. '56')
function kube::release::parse_and_validate_ci_version() {
  # Accept things like "v1.2.3-alpha.4.56+abcdef12345678" or "v1.2.3-beta.4"
  local -r version_regex="^v(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)-([a-zA-Z0-9]+)\\.(0|[1-9][0-9]*)(\\.(0|[1-9][0-9]*)\\+[0-9a-f]{7,40})?$"
  local -r version="${1-}"
  [[ "${version}" =~ ${version_regex} ]] || {
    kube::log::error "Invalid ci version: '${version}', must match regex ${version_regex}"
    return 1
  }

  # The VERSION variables are used when this file is sourced, hence
  # the shellcheck SC2034 'appears unused' warning is to be ignored.

  # shellcheck disable=SC2034
  VERSION_MAJOR="${BASH_REMATCH[1]}"
  # shellcheck disable=SC2034
  VERSION_MINOR="${BASH_REMATCH[2]}"
  # shellcheck disable=SC2034
  VERSION_PATCH="${BASH_REMATCH[3]}"
  # shellcheck disable=SC2034
  VERSION_PRERELEASE="${BASH_REMATCH[4]}"
  # shellcheck disable=SC2034
  VERSION_PRERELEASE_REV="${BASH_REMATCH[5]}"
  # shellcheck disable=SC2034
  VERSION_BUILD_INFO="${BASH_REMATCH[6]}"
  # shellcheck disable=SC2034
  VERSION_COMMITS="${BASH_REMATCH[7]}"
}

# ---------------------------------------------------------------------------
# Build final release artifacts
function kube::release::clean_cruft() {
  # Clean out cruft
  find "${RELEASE_STAGE}" -name '*~' -exec rm {} \;
  find "${RELEASE_STAGE}" -name '#*#' -exec rm {} \;
  find "${RELEASE_STAGE}" -name '.DS*' -exec rm {} \;
}

function kube::release::package_tarballs() {
  # Clean out any old releases
  rm -rf "${RELEASE_STAGE}" "${RELEASE_TARS}" "${RELEASE_IMAGES}"
  mkdir -p "${RELEASE_TARS}"
  kube::release::package_src_tarball &
  kube::release::package_client_tarballs &
  kube::release::package_kube_manifests_tarball &
  kube::util::wait-for-jobs || { kube::log::error "previous tarball phase failed"; return 1; }

  # _node and _server tarballs depend on _src tarball
  kube::release::package_node_tarballs &
  kube::release::package_server_tarballs &
  kube::util::wait-for-jobs || { kube::log::error "previous tarball phase failed"; return 1; }

  kube::release::package_final_tarball & # _final depends on some of the previous phases
  kube::release::package_test_tarballs & # _test doesn't depend on anything
  kube::util::wait-for-jobs || { kube::log::error "previous tarball phase failed"; return 1; }
}

# Package the source code we built, for compliance/licensing/audit/yadda.
function kube::release::package_src_tarball() {
  local -r src_tarball="${RELEASE_TARS}/kubernetes-src.tar.gz"
  kube::log::status "Building tarball: src"
  if [[ "${KUBE_GIT_TREE_STATE-}" = 'clean' ]]; then
    git archive -o "${src_tarball}" HEAD
  else
    find "${KUBE_ROOT}" -mindepth 1 -maxdepth 1 \
      ! \( \
        \( -path "${KUBE_ROOT}"/_\*       -o \
           -path "${KUBE_ROOT}"/.git\*    -o \
           -path "${KUBE_ROOT}"/.config\* -o \
           -path "${KUBE_ROOT}"/.gsutil\*    \
        \) -prune \
      \) -print0 \
    | "${TAR}" czf "${src_tarball}" --transform "s|${KUBE_ROOT#/*}|kubernetes|" --null -T -
  fi
}

# Package up all of the cross compiled clients. Over time this should grow into
# a full SDK
function kube::release::package_client_tarballs() {
  # Find all of the built client binaries
  local long_platforms=("${LOCAL_OUTPUT_BINPATH}"/*/*)
  if [[ -n ${KUBE_BUILD_PLATFORMS-} ]]; then
    read -ra long_platforms <<< "${KUBE_BUILD_PLATFORMS}"
  fi

  for platform_long in "${long_platforms[@]}"; do
    local platform
    local platform_tag
    platform=${platform_long##${LOCAL_OUTPUT_BINPATH}/} # Strip LOCAL_OUTPUT_BINPATH
    platform_tag=${platform/\//-} # Replace a "/" for a "-"
    kube::log::status "Starting tarball: client $platform_tag"

    (
      local release_stage="${RELEASE_STAGE}/client/${platform_tag}/kubernetes"
      rm -rf "${release_stage}"
      mkdir -p "${release_stage}/client/bin"

      local client_bins=("${KUBE_CLIENT_BINARIES[@]}")
      if [[ "${platform%/*}" = 'windows' ]]; then
        client_bins=("${KUBE_CLIENT_BINARIES_WIN[@]}")
      fi

      # This fancy expression will expand to prepend a path
      # (${LOCAL_OUTPUT_BINPATH}/${platform}/) to every item in the
      # client_bins array.
      cp "${client_bins[@]/#/${LOCAL_OUTPUT_BINPATH}/${platform}/}" \
        "${release_stage}/client/bin/"

      kube::release::clean_cruft

      local package_name="${RELEASE_TARS}/kubernetes-client-${platform_tag}.tar.gz"
      kube::release::create_tarball "${package_name}" "${release_stage}/.."
    ) &
  done

  kube::log::status "Waiting on tarballs"
  kube::util::wait-for-jobs || { kube::log::error "client tarball creation failed"; exit 1; }
}

# Package up all of the node binaries
function kube::release::package_node_tarballs() {
  local platform
  for platform in "${KUBE_NODE_PLATFORMS[@]}"; do
    local platform_tag
    local arch
    platform_tag=${platform/\//-} # Replace a "/" for a "-"
    arch=$(basename "${platform}")
    kube::log::status "Building tarball: node $platform_tag"

    local release_stage="${RELEASE_STAGE}/node/${platform_tag}/kubernetes"
    rm -rf "${release_stage}"
    mkdir -p "${release_stage}/node/bin"

    local node_bins=("${KUBE_NODE_BINARIES[@]}")
    if [[ "${platform%/*}" = 'windows' ]]; then
      node_bins=("${KUBE_NODE_BINARIES_WIN[@]}")
    fi
    # This fancy expression will expand to prepend a path
    # (${LOCAL_OUTPUT_BINPATH}/${platform}/) to every item in the
    # node_bins array.
    cp "${node_bins[@]/#/${LOCAL_OUTPUT_BINPATH}/${platform}/}" \
      "${release_stage}/node/bin/"

    # TODO: Docker images here
    # kube::release::create_docker_images_for_server "${release_stage}/server/bin" "${arch}"

    # Include the client binaries here too as they are useful debugging tools.
    local client_bins=("${KUBE_CLIENT_BINARIES[@]}")
    if [[ "${platform%/*}" = 'windows' ]]; then
      client_bins=("${KUBE_CLIENT_BINARIES_WIN[@]}")
    fi
    # This fancy expression will expand to prepend a path
    # (${LOCAL_OUTPUT_BINPATH}/${platform}/) to every item in the
    # client_bins array.
    cp "${client_bins[@]/#/${LOCAL_OUTPUT_BINPATH}/${platform}/}" \
      "${release_stage}/node/bin/"

    cp -R "${KUBE_ROOT}/LICENSES" "${release_stage}/"

    cp "${RELEASE_TARS}/kubernetes-src.tar.gz" "${release_stage}/"

    kube::release::clean_cruft

    local package_name="${RELEASE_TARS}/kubernetes-node-${platform_tag}.tar.gz"
    kube::release::create_tarball "${package_name}" "${release_stage}/.."
  done
}

# Package up all of the server binaries in docker images
function kube::release::build_server_images() {
  # Clean out any old images
  rm -rf "${RELEASE_IMAGES}"
  local platform
  for platform in "${KUBE_SERVER_PLATFORMS[@]}"; do
    local platform_tag
    local arch
    platform_tag=${platform/\//-} # Replace a "/" for a "-"
    arch=$(basename "${platform}")
    kube::log::status "Building images: $platform_tag"

    local release_stage
    release_stage="${RELEASE_STAGE}/server/${platform_tag}/kubernetes"
    rm -rf "${release_stage}"
    mkdir -p "${release_stage}/server/bin"

    # This fancy expression will expand to prepend a path
    # (${LOCAL_OUTPUT_BINPATH}/${platform}/) to every item in the
    # KUBE_SERVER_IMAGE_BINARIES array.
    cp "${KUBE_SERVER_IMAGE_BINARIES[@]/#/${LOCAL_OUTPUT_BINPATH}/${platform}/}" \
      "${release_stage}/server/bin/"

    kube::release::create_docker_images_for_server "${release_stage}/server/bin" "${arch}"
  done
}

# Package up all of the server binaries
function kube::release::package_server_tarballs() {
  kube::release::build_server_images
  local platform
  for platform in "${KUBE_SERVER_PLATFORMS[@]}"; do
    local platform_tag
    local arch
    platform_tag=${platform/\//-} # Replace a "/" for a "-"
    arch=$(basename "${platform}")
    kube::log::status "Building tarball: server $platform_tag"

    # NOTE: this directory was setup in kube::release::build_server_images
    local release_stage
    release_stage="${RELEASE_STAGE}/server/${platform_tag}/kubernetes"
    mkdir -p "${release_stage}/addons"

    # This fancy expression will expand to prepend a path
    # (${LOCAL_OUTPUT_BINPATH}/${platform}/) to every item in the
    # KUBE_SERVER_BINARIES array.
    cp "${KUBE_SERVER_BINARIES[@]/#/${LOCAL_OUTPUT_BINPATH}/${platform}/}" \
      "${release_stage}/server/bin/"

    # Include the client binaries here too as they are useful debugging tools.
    local client_bins
    client_bins=("${KUBE_CLIENT_BINARIES[@]}")
    if [[ "${platform%/*}" = 'windows' ]]; then
      client_bins=("${KUBE_CLIENT_BINARIES_WIN[@]}")
    fi
    # This fancy expression will expand to prepend a path
    # (${LOCAL_OUTPUT_BINPATH}/${platform}/) to every item in the
    # client_bins array.
    cp "${client_bins[@]/#/${LOCAL_OUTPUT_BINPATH}/${platform}/}" \
      "${release_stage}/server/bin/"

    cp -R "${KUBE_ROOT}/LICENSES" "${release_stage}/"

    cp "${RELEASE_TARS}/kubernetes-src.tar.gz" "${release_stage}/"

    kube::release::clean_cruft

    local package_name
    package_name="${RELEASE_TARS}/kubernetes-server-${platform_tag}.tar.gz"
    kube::release::create_tarball "${package_name}" "${release_stage}/.."
  done
}

function kube::release::md5() {
  if which md5 >/dev/null 2>&1; then
    md5 -q "$1"
  else
    md5sum "$1" | awk '{ print $1 }'
  fi
}

function kube::release::sha1() {
  if which sha1sum >/dev/null 2>&1; then
    sha1sum "$1" | awk '{ print $1 }'
  else
    shasum -a1 "$1" | awk '{ print $1 }'
  fi
}

function kube::release::build_conformance_image() {
  local -r arch="$1"
  local -r registry="$2"
  local -r version="$3"
  local -r save_dir="${4-}"
  kube::log::status "Building conformance image for arch: ${arch}"
  ARCH="${arch}" REGISTRY="${registry}" VERSION="${version}" \
    make -C cluster/images/conformance/ build >/dev/null

  local conformance_tag
  conformance_tag="${registry}/conformance-${arch}:${version}"
  if [[ -n "${save_dir}" ]]; then
    "${DOCKER[@]}" save "${conformance_tag}" > "${save_dir}/conformance-${arch}.tar"
  fi
  kube::log::status "Deleting conformance image ${conformance_tag}"
  "${DOCKER[@]}" rmi "${conformance_tag}" &>/dev/null || true
}

# This builds all the release docker images (One docker image per binary)
# Args:
#  $1 - binary_dir, the directory to save the tared images to.
#  $2 - arch, architecture for which we are building docker images.
function kube::release::create_docker_images_for_server() {
  # Create a sub-shell so that we don't pollute the outer environment
  (
    local binary_dir
    local arch
    local binaries
    local images_dir
    binary_dir="$1"
    arch="$2"
    binaries=$(kube::build::get_docker_wrapped_binaries)
    images_dir="${RELEASE_IMAGES}/${arch}"
    mkdir -p "${images_dir}"

    # k8s.gcr.io is the constant tag in the docker archives, this is also the default for config scripts in GKE.
    # We can use KUBE_DOCKER_REGISTRY to include and extra registry in the docker archive.
    # If we use KUBE_DOCKER_REGISTRY="k8s.gcr.io", then the extra tag (same) is ignored, see release_docker_image_tag below.
    local -r docker_registry="k8s.gcr.io"
    # Docker tags cannot contain '+'
    local docker_tag="${KUBE_GIT_VERSION/+/_}"
    if [[ -z "${docker_tag}" ]]; then
      kube::log::error "git version information missing; cannot create Docker tag"
      return 1
    fi

    # provide `--pull` argument to `docker build` if `KUBE_BUILD_PULL_LATEST_IMAGES`
    # is set to y or Y; otherwise try to build the image without forcefully
    # pulling the latest base image.
    local docker_build_opts
    docker_build_opts=
    if [[ "${KUBE_BUILD_PULL_LATEST_IMAGES}" =~ [yY] ]]; then
        docker_build_opts='--pull'
    fi

    for wrappable in $binaries; do

      local binary_name=${wrappable%%,*}
      local base_image=${wrappable##*,}
      local binary_file_path="${binary_dir}/${binary_name}"
      local docker_build_path="${binary_file_path}.dockerbuild"
      local docker_image_tag="${docker_registry}/${binary_name}-${arch}:${docker_tag}"

      local docker_file_path="${KUBE_ROOT}/build/server-image/Dockerfile"
      # If this binary has its own Dockerfile use that else use the generic Dockerfile.
      if [[ -f "${KUBE_ROOT}/build/server-image/${binary_name}/Dockerfile" ]]; then
          docker_file_path="${KUBE_ROOT}/build/server-image/${binary_name}/Dockerfile"
      fi

      kube::log::status "Starting docker build for image: ${binary_name}-${arch}"
      (
        rm -rf "${docker_build_path}"
        mkdir -p "${docker_build_path}"
        ln "${binary_file_path}" "${docker_build_path}/${binary_name}"

        local build_log="${docker_build_path}/build.log"
        if ! DOCKER_CLI_EXPERIMENTAL=enabled "${DOCKER[@]}" buildx build \
          -f "${docker_file_path}" \
          --platform linux/"${arch}" \
          --load ${docker_build_opts:+"${docker_build_opts}"} \
          -t "${docker_image_tag}" \
          --build-arg BASEIMAGE="${base_image}" \
          --build-arg SETCAP_IMAGE="${KUBE_BUILD_SETCAP_IMAGE}" \
          --build-arg BINARY="${binary_name}" \
          "${docker_build_path}" >"${build_log}" 2>&1; then
            cat "${build_log}"
            exit 1
        fi
        rm "${build_log}"

        # If we are building an official/alpha/beta release we want to keep
        # docker images and tag them appropriately.
        local -r release_docker_image_tag="${KUBE_DOCKER_REGISTRY-$docker_registry}/${binary_name}-${arch}:${KUBE_DOCKER_IMAGE_TAG-$docker_tag}"
        if [[ "${release_docker_image_tag}" != "${docker_image_tag}" ]]; then
          kube::log::status "Tagging docker image ${docker_image_tag} as ${release_docker_image_tag}"
          "${DOCKER[@]}" rmi "${release_docker_image_tag}" 2>/dev/null || true
          "${DOCKER[@]}" tag "${docker_image_tag}" "${release_docker_image_tag}" 2>/dev/null
        fi
        "${DOCKER[@]}" save -o "${binary_file_path}.tar" "${docker_image_tag}" "${release_docker_image_tag}"
        echo "${docker_tag}" > "${binary_file_path}.docker_tag"
        rm -rf "${docker_build_path}"
        ln "${binary_file_path}.tar" "${images_dir}/"

        kube::log::status "Deleting docker image ${docker_image_tag}"
        "${DOCKER[@]}" rmi "${docker_image_tag}" &>/dev/null || true
      ) &
    done

    if [[ "${KUBE_BUILD_CONFORMANCE}" =~ [yY] ]]; then
      kube::release::build_conformance_image "${arch}" "${docker_registry}" \
        "${docker_tag}" "${images_dir}" &
    fi

    kube::util::wait-for-jobs || { kube::log::error "previous Docker build failed"; return 1; }
    kube::log::status "Docker builds done"
  )

}

# This will pack kube-system manifests files for distros such as COS.
function kube::release::package_kube_manifests_tarball() {
  kube::log::status "Building tarball: manifests"

  local src_dir="${KUBE_ROOT}/cluster/gce/manifests"

  local release_stage="${RELEASE_STAGE}/manifests/kubernetes"
  rm -rf "${release_stage}"

  local dst_dir="${release_stage}/gci-trusty"
  mkdir -p "${dst_dir}"
  cp "${src_dir}/kube-proxy.manifest" "${dst_dir}/"
  cp "${src_dir}/cluster-autoscaler.manifest" "${dst_dir}/"
  cp "${src_dir}/etcd.manifest" "${dst_dir}"
  cp "${src_dir}/kube-scheduler.manifest" "${dst_dir}"
  cp "${src_dir}/kube-apiserver.manifest" "${dst_dir}"
  cp "${src_dir}/konnectivity-server.yaml" "${dst_dir}"
  cp "${src_dir}/abac-authz-policy.jsonl" "${dst_dir}"
  cp "${src_dir}/kube-controller-manager.manifest" "${dst_dir}"
  cp "${src_dir}/kube-addon-manager.yaml" "${dst_dir}"
  cp "${src_dir}/glbc.manifest" "${dst_dir}"
  find "${src_dir}" -name 'internal-*' -exec cp {} "${dst_dir}" \;
  cp "${KUBE_ROOT}/cluster/gce/gci/configure-helper.sh" "${dst_dir}/gci-configure-helper.sh"
  cp "${KUBE_ROOT}/cluster/gce/gci/configure-kubeapiserver.sh" "${dst_dir}/configure-kubeapiserver.sh"
  if [[ -e "${KUBE_ROOT}/cluster/gce/gci/gke-internal-configure-helper.sh" ]]; then
    cp "${KUBE_ROOT}/cluster/gce/gci/gke-internal-configure-helper.sh" "${dst_dir}/"
  fi
  cp "${KUBE_ROOT}/cluster/gce/gci/health-monitor.sh" "${dst_dir}/health-monitor.sh"
  # Merge GCE-specific addons with general purpose addons.
  for d in cluster/addons cluster/gce/addons; do
    find "${KUBE_ROOT}/${d}" \( \( -name \*.yaml -o -name \*.yaml.in -o -name \*.json \) -a ! \( -name \*demo\* \) \) -print0 | "${TAR}" c --transform "s|${KUBE_ROOT#/*}/${d}||" --null -T - | "${TAR}" x -C "${dst_dir}"
  done

  kube::release::clean_cruft

  local package_name="${RELEASE_TARS}/kubernetes-manifests.tar.gz"
  kube::release::create_tarball "${package_name}" "${release_stage}/.."
}

# Builds tarballs for each test platform containing the appropriate binaries.
function kube::release::package_test_platform_tarballs() {
  local platform
  rm -rf "${RELEASE_STAGE}/test"
  # KUBE_TEST_SERVER_PLATFORMS is a subset of KUBE_TEST_PLATFORMS,
  # so process it first.
  for platform in "${KUBE_TEST_SERVER_PLATFORMS[@]}"; do
    local platform_tag=${platform/\//-} # Replace a "/" for a "-"
    local release_stage="${RELEASE_STAGE}/test/${platform_tag}/kubernetes"
    mkdir -p "${release_stage}/test/bin"
    # This fancy expression will expand to prepend a path
    # (${LOCAL_OUTPUT_BINPATH}/${platform}/) to every item in the
    # KUBE_TEST_SERVER_BINARIES array.
    cp "${KUBE_TEST_SERVER_BINARIES[@]/#/${LOCAL_OUTPUT_BINPATH}/${platform}/}" \
      "${release_stage}/test/bin/"
  done
  for platform in "${KUBE_TEST_PLATFORMS[@]}"; do
    (
      local platform_tag=${platform/\//-} # Replace a "/" for a "-"
      kube::log::status "Starting tarball: test $platform_tag"
      local release_stage="${RELEASE_STAGE}/test/${platform_tag}/kubernetes"
      mkdir -p "${release_stage}/test/bin"

      local test_bins=("${KUBE_TEST_BINARIES[@]}")
      if [[ "${platform%/*}" = 'windows' ]]; then
        test_bins=("${KUBE_TEST_BINARIES_WIN[@]}")
      fi
      # This fancy expression will expand to prepend a path
      # (${LOCAL_OUTPUT_BINPATH}/${platform}/) to every item in the
      # test_bins array.
      cp "${test_bins[@]/#/${LOCAL_OUTPUT_BINPATH}/${platform}/}" \
        "${release_stage}/test/bin/"

      local package_name="${RELEASE_TARS}/kubernetes-test-${platform_tag}.tar.gz"
      kube::release::create_tarball "${package_name}" "${release_stage}/.."
    ) &
  done

  kube::log::status "Waiting on test tarballs"
  kube::util::wait-for-jobs || { kube::log::error "test tarball creation failed"; exit 1; }
}

# This is the stuff you need to run tests from the binary distribution.
function kube::release::package_test_tarballs() {
  kube::release::package_test_platform_tarballs

  kube::log::status "Building tarball: test portable"

  local release_stage="${RELEASE_STAGE}/test/kubernetes"
  rm -rf "${release_stage}"
  mkdir -p "${release_stage}"

  # First add test image files and other portable sources so we can create
  # the portable test tarball.
  mkdir -p "${release_stage}/test/images"
  cp -fR "${KUBE_ROOT}/test/images" "${release_stage}/test/"
  "${TAR}" c "${KUBE_TEST_PORTABLE[@]}" | "${TAR}" x -C "${release_stage}"

  kube::release::clean_cruft

  local portable_tarball_name="${RELEASE_TARS}/kubernetes-test-portable.tar.gz"
  kube::release::create_tarball "${portable_tarball_name}" "${release_stage}/.."
}

# This is all the platform-independent stuff you need to run/install kubernetes.
# Arch-specific binaries will need to be downloaded separately (possibly by
# using the bundled cluster/get-kube-binaries.sh script).
# Included in this tarball:
#   - Cluster spin up/down scripts and configs for various cloud providers
#   - Tarballs for manifest configs that are ready to be uploaded
#   - Examples (which may or may not still work)
#   - The remnants of the docs/ directory
function kube::release::package_final_tarball() {
  kube::log::status "Building tarball: final"

  # This isn't a "full" tarball anymore, but the release lib still expects
  # artifacts under "full/kubernetes/"
  local release_stage="${RELEASE_STAGE}/full/kubernetes"
  rm -rf "${release_stage}"
  mkdir -p "${release_stage}"

  mkdir -p "${release_stage}/client"
  cat <<EOF > "${release_stage}/client/README"
Client binaries are no longer included in the Kubernetes final tarball.

Run cluster/get-kube-binaries.sh to download client and server binaries.
EOF

  # We want everything in /cluster.
  cp -R "${KUBE_ROOT}/cluster" "${release_stage}/"

  mkdir -p "${release_stage}/server"
  cp "${RELEASE_TARS}/kubernetes-manifests.tar.gz" "${release_stage}/server/"
  cat <<EOF > "${release_stage}/server/README"
Server binary tarballs are no longer included in the Kubernetes final tarball.

Run cluster/get-kube-binaries.sh to download client and server binaries.
EOF

  # Include hack/lib as a dependency for the cluster/ scripts
  mkdir -p "${release_stage}/hack"
  cp -R "${KUBE_ROOT}/hack/lib" "${release_stage}/hack/"

  cp -R "${KUBE_ROOT}/docs" "${release_stage}/"
  cp "${KUBE_ROOT}/README.md" "${release_stage}/"
  cp -R "${KUBE_ROOT}/LICENSES" "${release_stage}/"

  echo "${KUBE_GIT_VERSION}" > "${release_stage}/version"

  kube::release::clean_cruft

  local package_name="${RELEASE_TARS}/kubernetes.tar.gz"
  kube::release::create_tarball "${package_name}" "${release_stage}/.."
}

# Build a release tarball.  $1 is the output tar name.  $2 is the base directory
# of the files to be packaged.  This assumes that ${2}/kubernetes is what is
# being packaged.
function kube::release::create_tarball() {
  kube::build::ensure_tar

  local tarfile=$1
  local stagingdir=$2

  "${TAR}" czf "${tarfile}" -C "${stagingdir}" kubernetes --owner=0 --group=0
}