Repository: kubernetes/minikube
Pull request #10774 (2021-03-10 18:36:39)

run arm64 integration tests on jenkins
This PR adds jenkins scripts to run arm64 integration tests


Patch:
diff --git a/hack/jenkins/common.sh b/hack/jenkins/common.sh
index 1d3bc108e..feb09d8a1 100755
--- a/hack/jenkins/common.sh
+++ b/hack/jenkins/common.sh
@@ -375,7 +375,7 @@ touch "${HTML_OUT}"
 touch "${SUMMARY_OUT}"
 gopogh_status=$(gopogh -in "${JSON_OUT}" -out_html "${HTML_OUT}" -out_summary "${SUMMARY_OUT}" -name "${JOB_NAME}" -pr "${MINIKUBE_LOCATION}" -repo github.com/kubernetes/minikube/  -details "${COMMIT}") || true
 fail_num=$(echo $gopogh_status | jq '.NumberOfFail')
-test_num=$(echo $gopogh_status | jq '.NumberOfTests')       
+test_num=$(echo $gopogh_status | jq '.NumberOfTests')
 pessimistic_status="${fail_num} / ${test_num} failures"
 description="completed with ${status} in ${elapsed} minute(s)."
 if [ "$status" = "failure" ]; then
@@ -445,7 +445,7 @@ function retry_github_status() {
     echo "HTTP code ${code}! Retrying in ${timeout} .."
     sleep "${timeout}"
     attempt=$(( attempt + 1 ))
-    timeout=$(( timeout * 5 )) 
+    timeout=$(( timeout * 5 ))
   done
 }
 
diff --git a/hack/jenkins/installers/check_install_docker.sh b/hack/jenkins/installers/check_install_docker.sh
index cf2e5e9de..23acba730 100755
--- a/hack/jenkins/installers/check_install_docker.sh
+++ b/hack/jenkins/installers/check_install_docker.sh
@@ -16,6 +16,8 @@
 
 set -eux -o pipefail
 
+ARCH=${ARCH:=amd64}
+
 if [[ "$OSTYPE" == "darwin"* ]]; then
   echo "detected darwin, exiting"
   exit 0
@@ -28,5 +30,5 @@ rm get-docker.sh
 sudo adduser jenkins docker || true
 
 echo "Installing latest kubectl"
-curl -LO "https://storage.googleapis.com/kubernetes-release/release/`curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt`/bin/linux/amd64/kubectl"
+curl -LO "https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/${ARCH}/kubectl"
 sudo install ./kubectl /usr/local/bin/kubectl
diff --git a/hack/jenkins/installers/check_install_golang.sh b/hack/jenkins/installers/check_install_golang.sh
index 40cb0feaf..b01ef8595 100755
--- a/hack/jenkins/installers/check_install_golang.sh
+++ b/hack/jenkins/installers/check_install_golang.sh
@@ -25,6 +25,8 @@ fi
 VERSION_TO_INSTALL=${1}
 INSTALL_PATH=${2}
 
+ARCH=${ARCH:=amd64}
+
 # installs or updates golang if right version doesn't exists
 function check_and_install_golang() {
   if ! go version &>/dev/null; then
@@ -59,7 +61,7 @@ function install_golang() {
   fi
   # using sudo because previously installed versions might have been installed by a different user.
   # as it was the case on jenkins VM.
-  sudo curl -qL -O "https://storage.googleapis.com/golang/go${1}.${INSTALLOS}-amd64.tar.gz" &&
+  sudo curl -qL -O "https://storage.googleapis.com/golang/go${1}.${INSTALLOS}-${ARCH}.tar.gz" &&
     sudo tar -xf go${1}.${INSTALLOS}-amd64.tar.gz &&
     sudo rm -rf "${2}/go" &&
     sudo mv go "${2}/" && sudo chown -R $(whoami): ${2}/go
diff --git a/hack/jenkins/minikube_set_pending.sh b/hack/jenkins/minikube_set_pending.sh
index 63c50dd60..923d58cd0 100755
--- a/hack/jenkins/minikube_set_pending.sh
+++ b/hack/jenkins/minikube_set_pending.sh
@@ -43,6 +43,9 @@ jobs=(
      'KVM_Linux_crio'
      'none_Linux'
      'Docker_Linux'
+     'Docker_Linux_docker_arm64'
+     'Docker_Linux_containerd_arm64'
+     'Docker_Linux_crio_arm64'
      'Docker_Linux_containerd'
      'Docker_Linux_crio'
      # 'Docker_macOS'
diff --git a/hack/jenkins/upload_integration_report.sh b/hack/jenkins/upload_integration_report.sh
new file mode 100644
index 000000000..a96cd1ab8
--- /dev/null
+++ b/hack/jenkins/upload_integration_report.sh
@@ -0,0 +1,52 @@
+#!/bin/bash
+
+# Copyright 2021 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# This script uploads the test reports to the GCS bucket
+
+# The script expects the following env variables:
+# OS_ARCH: The operating system and the architecture separated by a hyphen '-' (e.g. darwin-amd64, linux-amd64, windows-amd64)
+# VM_DRIVER: the driver to use for the test
+# CONTAINER_RUNTIME: the container runtime to use for the test
+# EXTRA_START_ARGS: additional flags to pass into minikube start
+# EXTRA_TEST_ARGS: additional flags to pass into go test
+# JOB_NAME: the name of the logfile and check name to update on github
+
+
+# upload results to GCS
+JOB_GCS_BUCKET="minikube-builds/logs/${MINIKUBE_LOCATION}/${COMMIT:0:7}/${UPSTREAM_JOB}"
+
+ARTIFACTS=artifacts/test_reports
+
+ls -l artifacts
+ls -l artifacts/test_reports
+
+TEST_OUT="$ARTIFACTS/out.txt"
+echo ">> uploading ${TEST_OUT} to gs://${JOB_GCS_BUCKET}.txt"
+gsutil -qm cp "${TEST_OUT}" "gs://${JOB_GCS_BUCKET}.txt" || true
+
+JSON_OUT="$ARTIFACTS/out.json"
+echo ">> uploading ${JSON_OUT}"
+gsutil -qm cp "${JSON_OUT}" "gs://${JOB_GCS_BUCKET}.json" || true
+
+HTML_OUT="$ARTIFACTS/out.html"
+echo ">> uploading ${HTML_OUT}"
+gsutil -qm cp "${HTML_OUT}" "gs://${JOB_GCS_BUCKET}.html" || true
+
+SUMMARY_OUT="$ARTIFACTS/summary.txt"
+echo ">> uploading ${SUMMARY_OUT}"
+gsutil -qm cp "${SUMMARY_OUT}" "gs://${JOB_GCS_BUCKET}_summary.json" || true
+#
diff --git a/pkg/util/utils.go b/pkg/util/utils.go
index f5fe5a3b2..227c8402b 100644
--- a/pkg/util/utils.go
+++ b/pkg/util/utils.go
@@ -29,7 +29,7 @@ import (
 )
 
 const (
-	downloadURL = "https://storage.googleapis.com/minikube/releases/%s/minikube-%s-amd64%s"
+	downloadURL = "https://storage.googleapis.com/minikube/releases/%s/minikube-%s-%s%s"
 )
 
 // CalculateSizeInMB returns the number of MB in the human readable string
@@ -62,13 +62,13 @@ func ConvertUnsignedBytesToMB(byteSize uint64) int64 {
 	return int64(byteSize / units.MiB)
 }
 
-// GetBinaryDownloadURL returns a suitable URL for the platform
-func GetBinaryDownloadURL(version, platform string) string {
+// GetBinaryDownloadURL returns a suitable URL for the platform and arch
+func GetBinaryDownloadURL(version, platform, arch string) string {
 	switch platform {
 	case "windows":
-		return fmt.Sprintf(downloadURL, version, platform, ".exe")
+		return fmt.Sprintf(downloadURL, version, platform, arch, ".exe")
 	default:
-		return fmt.Sprintf(downloadURL, version, platform, "")
+		return fmt.Sprintf(downloadURL, version, platform, arch, "")
 	}
 }
 


Test patch:
diff --git a/deploy/minikube/release_sanity_test.go b/deploy/minikube/release_sanity_test.go
index 8788e9de7..07eba31e8 100644
--- a/deploy/minikube/release_sanity_test.go
+++ b/deploy/minikube/release_sanity_test.go
@@ -23,6 +23,7 @@ import (
 	"encoding/hex"
 	"fmt"
 	"io/ioutil"
+	"runtime"
 	"testing"
 
 	retryablehttp "github.com/hashicorp/go-retryablehttp"
@@ -56,7 +57,7 @@ func TestReleasesJSON(t *testing.T) {
 		fmt.Printf("Checking release: %s\n", r.Name)
 		for platform, sha := range r.Checksums {
 			fmt.Printf("Checking SHA for %s.\n", platform)
-			actualSha, err := getSHAFromURL(util.GetBinaryDownloadURL(r.Name, platform))
+			actualSha, err := getSHAFromURL(util.GetBinaryDownloadURL(r.Name, platform, runtime.GOARCH))
 			if err != nil {
 				t.Errorf("Error calculating SHA for %s-%s. Error: %v", r.Name, platform, err)
 				continue
diff --git a/hack/jenkins/linux_integration_tests_docker_arm64.sh b/hack/jenkins/linux_integration_tests_docker_arm64.sh
new file mode 100644
index 000000000..11a1c5f40
--- /dev/null
+++ b/hack/jenkins/linux_integration_tests_docker_arm64.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+
+# Copyright 2019 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# This script runs the integration tests on a Linux machine for the KVM Driver
+
+# The script expects the following env variables:
+# MINIKUBE_LOCATION: GIT_COMMIT from upstream build.
+# COMMIT: Actual commit ID from upstream build
+# EXTRA_BUILD_ARGS (optional): Extra args to be passed into the minikube integrations tests
+# access_token: The Github API access token. Injected by the Jenkins credential provider.
+set -e
+
+mkdir -p cron && gsutil -qm rsync "gs://minikube-builds/${MINIKUBE_LOCATION}/cron" cron || echo "FAILED TO GET CRON FILES"
+sudo install cron/cleanup_and_reboot_Linux.sh /etc/cron.hourly/cleanup_and_reboot || echo "FAILED TO INSTALL CLEANUP"
+
+# removing possible left over docker containers from previous runs
+docker rm -f -v "$(docker ps -aq)" >/dev/null 2>&1 || true
+
+ARCH="arm64" \
+OS="linux" \
+VM_DRIVER="docker" \
+JOB_NAME="$JOB_NAME" \
+CONTAINER_RUNTIME="docker" \
+source ./run_tests.sh
diff --git a/hack/jenkins/linux_integration_tests_docker_containerd_arm64.sh b/hack/jenkins/linux_integration_tests_docker_containerd_arm64.sh
new file mode 100644
index 000000000..96685cb8c
--- /dev/null
+++ b/hack/jenkins/linux_integration_tests_docker_containerd_arm64.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+
+# Copyright 2019 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# This script runs the integration tests on a Linux machine for the KVM Driver
+
+# The script expects the following env variables:
+# MINIKUBE_LOCATION: GIT_COMMIT from upstream build.
+# COMMIT: Actual commit ID from upstream build
+# EXTRA_BUILD_ARGS (optional): Extra args to be passed into the minikube integrations tests
+# access_token: The Github API access token. Injected by the Jenkins credential provider.
+set -e
+
+mkdir -p cron && gsutil -qm rsync "gs://minikube-builds/${MINIKUBE_LOCATION}/cron" cron || echo "FAILED TO GET CRON FILES"
+sudo install cron/cleanup_and_reboot_Linux.sh /etc/cron.hourly/cleanup_and_reboot || echo "FAILED TO INSTALL CLEANUP"
+
+# removing possible left over docker containers from previous runs
+docker rm -f -v "$(docker ps -aq)" >/dev/null 2>&1 || true
+
+ARCH="arm64" \
+OS="linux" \
+VM_DRIVER="docker" \
+JOB_NAME="$JOB_NAME" \
+CONTAINER_RUNTIME="containerd" \
+source ./run_tests.sh
diff --git a/hack/jenkins/linux_integration_tests_docker_crio_arm64.sh b/hack/jenkins/linux_integration_tests_docker_crio_arm64.sh
new file mode 100644
index 000000000..dd46df66f
--- /dev/null
+++ b/hack/jenkins/linux_integration_tests_docker_crio_arm64.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+
+# Copyright 2019 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# This script runs the integration tests on a Linux machine for the KVM Driver
+
+# The script expects the following env variables:
+# MINIKUBE_LOCATION: GIT_COMMIT from upstream build.
+# COMMIT: Actual commit ID from upstream build
+# EXTRA_BUILD_ARGS (optional): Extra args to be passed into the minikube integrations tests
+# access_token: The Github API access token. Injected by the Jenkins credential provider.
+set -e
+
+mkdir -p cron && gsutil -qm rsync "gs://minikube-builds/${MINIKUBE_LOCATION}/cron" cron || echo "FAILED TO GET CRON FILES"
+sudo install cron/cleanup_and_reboot_Linux.sh /etc/cron.hourly/cleanup_and_reboot || echo "FAILED TO INSTALL CLEANUP"
+
+# removing possible left over docker containers from previous runs
+docker rm -f -v "$(docker ps -aq)" >/dev/null 2>&1 || true
+
+ARCH="arm64" \
+OS="linux" \
+VM_DRIVER="docker" \
+JOB_NAME="$JOB_NAME" \
+CONTAINER_RUNTIME="crio" \
+source ./run_tests.sh
diff --git a/hack/jenkins/run_tests.sh b/hack/jenkins/run_tests.sh
new file mode 100644
index 000000000..c22d2cdd9
--- /dev/null
+++ b/hack/jenkins/run_tests.sh
@@ -0,0 +1,449 @@
+#!/bin/bash
+
+# Copyright 2016 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# This script downloads the test files from the build bucket and makes some executable.
+
+# The script expects the following env variables:
+# OS: The operating system
+# ARCH: The architecture
+# VM_DRIVER: the driver to use for the test
+# CONTAINER_RUNTIME: the container runtime to use for the test
+# EXTRA_START_ARGS: additional flags to pass into minikube start
+# EXTRA_TEST_ARGS: additional flags to pass into go test
+# JOB_NAME: the name of the logfile and check name to update on github
+
+readonly OS_ARCH="${OS}-${ARCH}"
+readonly TEST_ROOT="${HOME}/minikube-integration"
+readonly TEST_HOME="${TEST_ROOT}/${OS_ARCH}-${VM_DRIVER}-${CONTAINER_RUNTIME}-${MINIKUBE_LOCATION}-$$-${COMMIT}"
+
+export GOPATH="$HOME/go"
+export KUBECONFIG="${TEST_HOME}/kubeconfig"
+export PATH=$PATH:"/usr/local/bin/:/usr/local/go/bin/:$GOPATH/bin"
+
+readonly TIMEOUT=${1:-120m}
+
+if [ "$(uname)" != "Darwin" ]; then
+  # install lsof for finding none driver procs, psmisc to use pstree in cronjobs
+  sudo apt-get -y install lsof psmisc
+fi
+
+# installing golang so we could do go get for gopogh
+sudo ARCH="$ARCH"./installers/check_install_golang.sh "1.16" "/usr/local" || true
+
+# install docker and kubectl if not present
+sudo ARCH="$ARCH" ./installers/check_install_docker.sh
+
+# let's just clean all docker artifacts up
+docker system prune --force --volumes || true
+docker system df || true
+
+echo ">> Starting at $(date)"
+echo ""
+echo "arch:      ${OS_ARCH}"
+echo "build:     ${MINIKUBE_LOCATION}"
+echo "driver:    ${VM_DRIVER}"
+echo "runtime:   ${CONTAINER_RUNTIME}"
+echo "job:       ${JOB_NAME}"
+echo "test home: ${TEST_HOME}"
+echo "sudo:      ${SUDO_PREFIX}"
+echo "kernel:    $(uname -v)"
+echo "uptime:    $(uptime)"
+# Setting KUBECONFIG prevents the version check from erroring out due to permission issues
+echo "kubectl:   $(env KUBECONFIG=${TEST_HOME} kubectl version --client --short=true)"
+echo "docker:    $(docker version --format '{{ .Client.Version }}')"
+echo "podman:    $(sudo podman version --format '{{.Version}}' || true)"
+echo "go:        $(go version || true)"
+
+
+case "${VM_DRIVER}" in
+  kvm2)
+    echo "virsh:     $(virsh --version)"
+  ;;
+  virtualbox)
+    echo "vbox:      $(vboxmanage --version)"
+  ;;
+esac
+
+echo ""
+mkdir -p out/ testdata/
+
+# Install gsutil if necessary.
+if ! type -P gsutil >/dev/null; then
+  if [[ ! -x "out/gsutil/gsutil" ]]; then
+    echo "Installing gsutil to $(pwd)/out ..."
+    curl -s https://storage.googleapis.com/pub/gsutil.tar.gz | tar -C out/ -zxf -
+  fi
+  PATH="$(pwd)/out/gsutil:$PATH"
+fi
+
+# Add the out/ directory to the PATH, for using new drivers.
+export PATH="$(pwd)/out/":$PATH
+
+
+echo
+echo ">> Downloading test inputs from ${MINIKUBE_LOCATION} ..."
+gsutil -qm cp \
+  "gs://minikube-builds/${MINIKUBE_LOCATION}/minikube-${OS_ARCH}" \
+  "gs://minikube-builds/${MINIKUBE_LOCATION}/docker-machine-driver"-* \
+  "gs://minikube-builds/${MINIKUBE_LOCATION}/e2e-${OS_ARCH}" out
+
+gsutil -qm cp -r "gs://minikube-builds/${MINIKUBE_LOCATION}/testdata"/* testdata/
+
+gsutil -qm cp "gs://minikube-builds/${MINIKUBE_LOCATION}/gvisor-addon" testdata/
+
+
+# Set the executable bit on the e2e binary and out binary
+export MINIKUBE_BIN="out/minikube-${OS_ARCH}"
+export E2E_BIN="out/e2e-${OS_ARCH}"
+chmod +x "${MINIKUBE_BIN}" "${E2E_BIN}" out/docker-machine-driver-*
+"${MINIKUBE_BIN}" version
+
+procs=$(pgrep "minikube-${OS_ARCH}|e2e-${OS_ARCH}" || true)
+if [[ "${procs}" != "" ]]; then
+  echo "Warning: found stale test processes to kill:"
+  ps -f -p ${procs} || true
+  kill ${procs} || true
+  kill -9 ${procs} || true
+fi
+
+# Quickly notice misconfigured test roots
+mkdir -p "${TEST_ROOT}"
+
+# Cleanup stale test outputs.
+echo
+echo ">> Cleaning up after previous test runs ..."
+for test_path in ${TEST_ROOT}; do
+  ls -lad "${test_path}" || continue
+
+  echo "* Cleaning stale test path: ${test_path}"
+  for tunnel in $(find ${test_path} -name tunnels.json -type f); do
+    env MINIKUBE_HOME="$(dirname ${tunnel})" ${MINIKUBE_BIN} tunnel --cleanup || true
+  done
+
+  for home in $(find ${test_path} -name .minikube -type d); do
+    env MINIKUBE_HOME="$(dirname ${home})" ${MINIKUBE_BIN} delete --all || true
+    sudo rm -Rf "${home}"
+  done
+
+  for kconfig in $(find ${test_path} -name kubeconfig -type f); do
+    sudo rm -f "${kconfig}"
+  done
+
+  ## ultimate shotgun clean up docker after we tried all
+  docker rm -f -v $(docker ps -aq) >/dev/null 2>&1 || true
+
+  # Be very specific to avoid accidentally deleting other items, like wildcards or devices
+  if [[ -d "${test_path}" ]]; then
+    rm -Rf "${test_path}" || true
+  elif [[ -f "${test_path}" ]]; then
+    rm -f "${test_path}" || true
+  fi
+done
+
+function cleanup_procs() {
+  # sometimes tests left over zombie procs that won't exit
+  # for example:
+  # jenkins  20041  0.0  0.0      0     0 ?        Z    Aug19   0:00 [minikube-linux-] <defunct>
+  zombie_defuncts=$(ps -A -ostat,ppid | awk '/[zZ]/ && !a[$2]++ {print $2}')
+  if [[ "${zombie_defuncts}" != "" ]]; then
+    echo "Found zombie defunct procs to kill..."
+    ps -f -p ${zombie_defuncts} || true
+    kill ${zombie_defuncts} || true
+  fi
+
+  if type -P virsh; then
+    virsh -c qemu:///system list --all --uuid \
+      | xargs -I {} sh -c "virsh -c qemu:///system destroy {}; virsh -c qemu:///system undefine {}" \
+      || true
+    echo ">> virsh VM list after clean up (should be empty):"
+    virsh -c qemu:///system list --all || true
+  fi
+
+  if type -P vboxmanage; then
+    killall VBoxHeadless || true
+    sleep 1
+    killall -9 VBoxHeadless || true
+
+    for guid in $(vboxmanage list vms | grep -Eo '\{[a-zA-Z0-9-]+\}'); do
+      echo "- Removing stale VirtualBox VM: $guid"
+      vboxmanage startvm "${guid}" --type emergencystop || true
+      vboxmanage unregistervm "${guid}" || true
+    done
+
+    ifaces=$(vboxmanage list hostonlyifs | grep -E "^Name:" | awk '{ print $2 }')
+    for if in $ifaces; do
+      vboxmanage hostonlyif remove "${if}" || true
+    done
+
+    echo ">> VirtualBox VM list after clean up (should be empty):"
+    vboxmanage list vms || true
+    echo ">> VirtualBox interface list after clean up (should be empty):"
+    vboxmanage list hostonlyifs || true
+  fi
+
+  if type -P hdiutil; then
+    hdiutil info | grep -E "/dev/disk[1-9][^s]" || true
+    hdiutil info \
+        | grep -E "/dev/disk[1-9][^s]" \
+        | awk '{print $1}' \
+        | xargs -I {} sh -c "hdiutil detach {}" \
+        || true
+  fi
+
+  # cleaning up stale hyperkits
+  if type -P hyperkit; then
+    for pid in $(pgrep hyperkit); do
+      echo "Killing stale hyperkit $pid"
+      ps -f -p $pid || true
+      kill $pid || true
+      kill -9 $pid || true
+    done
+  fi
+
+  if [[ "${VM_DRIVER}" == "hyperkit" ]]; then
+    if [[ -e out/docker-machine-driver-hyperkit ]]; then
+      sudo chown root:wheel out/docker-machine-driver-hyperkit || true
+      sudo chmod u+s out/docker-machine-driver-hyperkit || true
+    fi
+  fi
+
+  kprocs=$(pgrep kubectl || true)
+  if [[ "${kprocs}" != "" ]]; then
+    echo "error: killing hung kubectl processes ..."
+    ps -f -p ${kprocs} || true
+    sudo -E kill ${kprocs} || true
+  fi
+
+
+  # clean up none drivers binding on 8443
+  none_procs=$(sudo lsof -i :8443 | tail -n +2 | awk '{print $2}' || true)
+  if [[ "${none_procs}" != "" ]]; then
+    echo "Found stale api servers listening on 8443 processes to kill: "
+    for p in $none_procs
+    do
+      echo "Kiling stale none driver:  $p"
+      sudo -E ps -f -p $p || true
+      sudo -E kill $p || true
+      sudo -E kill -9 $p || true
+    done
+  fi
+}
+
+function cleanup_stale_routes() {
+  local show="netstat -rn -f inet"
+  local del="sudo route -n delete"
+
+  if [[ "$(uname)" == "Linux" ]]; then
+    show="ip route show"
+    del="sudo ip route delete"
+  fi
+
+  local troutes=$($show | awk '{ print $1 }' | grep 10.96.0.0 || true)
+  for route in ${troutes}; do
+    echo "WARNING: deleting stale tunnel route: ${route}"
+    $del "${route}" || true
+  done
+}
+
+cleanup_procs || true
+cleanup_stale_routes || true
+
+mkdir -p "${TEST_HOME}"
+export MINIKUBE_HOME="${TEST_HOME}/.minikube"
+
+
+# Build the gvisor image so that we can integration test changes to pkg/gvisor
+chmod +x ./testdata/gvisor-addon
+# skipping gvisor mac because ofg https://github.com/kubernetes/minikube/issues/5137
+if [ "$(uname)" != "Darwin" ]; then
+  # Should match GVISOR_IMAGE_VERSION in Makefile
+  docker build -t gcr.io/k8s-minikube/gvisor-addon:2 -f testdata/gvisor-addon-Dockerfile ./testdata
+fi
+
+readonly LOAD=$(uptime | egrep -o "load average.*: [0-9]+" | cut -d" " -f3)
+if [[ "${LOAD}" -gt 2 ]]; then
+  echo ""
+  echo "********************** LOAD WARNING ********************************"
+  echo "Load average is very high (${LOAD}), which may cause failures. Top:"
+  if [[ "$(uname)" == "Darwin" ]]; then
+    # Two samples, macOS does not calculate CPU usage on the first one
+    top -l 2 -o cpu -n 5 | tail -n 15
+  else
+    top -b -n1 | head -n 15
+  fi
+  echo "********************** LOAD WARNING ********************************"
+  echo "Sleeping 30s to see if load goes down ...."
+  sleep 30
+  uptime
+fi
+
+readonly TEST_OUT="${TEST_HOME}/testout.txt"
+readonly JSON_OUT="${TEST_HOME}/test.json"
+readonly HTML_OUT="${TEST_HOME}/test.html"
+readonly SUMMARY_OUT="${TEST_HOME}/test_summary.json"
+
+e2e_start_time="$(date -u +%s)"
+echo ""
+echo ">> Starting ${E2E_BIN} at $(date)"
+set -x
+
+if test -f "${TEST_OUT}"; then
+  rm "${TEST_OUT}" || true # clean up previous runs of same build
+fi
+touch "${TEST_OUT}"
+
+if [ ! -z "${CONTAINER_RUNTIME}" ]
+then
+    EXTRA_START_ARGS="${EXTRA_START_ARGS} --container-runtime=${CONTAINER_RUNTIME}"
+fi
+
+${SUDO_PREFIX}${E2E_BIN} \
+  -minikube-start-args="--driver=${VM_DRIVER} ${EXTRA_START_ARGS}" \
+  -test.timeout=${TIMEOUT} -test.v \
+  ${EXTRA_TEST_ARGS} \
+  -binary="${MINIKUBE_BIN}" 2>&1 | tee "${TEST_OUT}"
+
+result=${PIPESTATUS[0]} # capture the exit code of the first cmd in pipe.
+set +x
+echo ">> ${E2E_BIN} exited with ${result} at $(date)"
+echo ""
+
+if [[ $result -eq 0 ]]; then
+  status="success"
+  echo "minikube: SUCCESS"
+else
+  status="failure"
+  echo "minikube: FAIL"
+fi
+
+## caclucate the time took to finish running e2e binary test.
+e2e_end_time="$(date -u +%s)"
+elapsed=$(($e2e_end_time-$e2e_start_time))
+min=$(($elapsed/60))
+sec=$(tail -c 3 <<< $((${elapsed}00/60)))
+elapsed=$min.$sec
+
+
+
+echo ">> Attmpting to convert test logs to json"
+if test -f "${JSON_OUT}"; then
+  rm "${JSON_OUT}" || true # clean up previous runs of same build
+fi
+
+touch "${JSON_OUT}"
+
+# Generate JSON output
+echo ">> Running go test2json"
+go tool test2json -t < "${TEST_OUT}" > "${JSON_OUT}" || true
+
+if ! type "jq" > /dev/null; then
+echo ">> Installing jq"
+    if [ "$(uname)" != "Darwin" ]; then
+      curl -LO https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 && sudo install jq-linux64 /usr/local/bin/jq
+    else
+      curl -LO https://github.com/stedolan/jq/releases/download/jq-1.6/jq-osx-amd64 && sudo install jq-osx-amd64 /usr/local/bin/jq
+    fi
+fi
+
+echo ">> Installing gopogh"
+curl -LO "https://github.com/medyagh/gopogh/releases/download/v0.6.0/gopogh-${OS_ARCH}"
+sudo install "gopogh-${OS_ARCH}" /usr/local/bin/gopogh
+
+
+echo ">> Running gopogh"
+if test -f "${HTML_OUT}"; then
+    rm "${HTML_OUT}" || true # clean up previous runs of same build
+fi
+
+touch "${HTML_OUT}"
+touch "${SUMMARY_OUT}"
+gopogh_status=$(gopogh -in "${JSON_OUT}" -out_html "${HTML_OUT}" -out_summary "${SUMMARY_OUT}" -name "${JOB_NAME}" -pr "${MINIKUBE_LOCATION}" -repo github.com/kubernetes/minikube/  -details "${COMMIT}") || true
+fail_num=$(echo $gopogh_status | jq '.NumberOfFail')
+test_num=$(echo $gopogh_status | jq '.NumberOfTests')
+pessimistic_status="${fail_num} / ${test_num} failures"
+description="completed with ${status} in ${elapsed} minute(s)."
+if [ "$status" = "failure" ]; then
+  description="completed with ${pessimistic_status} in ${elapsed} minute(s)."
+fi
+echo "$description"
+
+
+REPORTS_PATH=test_reports
+mkdir -p "$REPORTS_PATH"
+cp "${TEST_OUT}" "$REPORTS_PATH/out.txt"
+cp "${JSON_OUT}" "$REPORTS_PATH/out.json"
+cp "${HTML_OUT}" "$REPORTS_PATH/out.html"
+cp "${SUMMARY_OUT}" "$REPORTS_PATH/summary.txt"
+
+
+echo ">> Cleaning up after ourselves ..."
+timeout 3m ${SUDO_PREFIX}${MINIKUBE_BIN} tunnel --cleanup || true
+timeout 5m ${SUDO_PREFIX}${MINIKUBE_BIN} delete --all --purge >/dev/null 2>/dev/null || true
+cleanup_stale_routes || true
+
+${SUDO_PREFIX} rm -Rf "${MINIKUBE_HOME}" || true
+${SUDO_PREFIX} rm -f "${KUBECONFIG}" || true
+
+rmdir "${TEST_HOME}" || true
+echo ">> ${TEST_HOME} completed at $(date)"
+
+if [[ "${MINIKUBE_LOCATION}" == "master" ]]; then
+  exit "$result"
+fi
+
+public_log_url="https://storage.googleapis.com/minikube-builds/logs/${MINIKUBE_LOCATION}/${COMMIT:0:7}/${JOB_NAME}.html"
+
+# retry_github_status provides reliable github status updates
+function retry_github_status() {
+  local commit=$1
+  local context=$2
+  local state=$3
+  local token=$4
+  local target=$5
+  local desc=$6
+
+   # Retry in case we hit our GitHub API quota or fail other ways.
+  local attempt=0
+  local timeout=2
+  local code=-1
+
+  echo "set GitHub status $context to $desc"
+
+  while [[ "${attempt}" -lt 8 ]]; do
+    local out=$(mktemp)
+    code=$(curl -o "${out}" -s --write-out "%{http_code}" -L \
+      "https://api.github.com/repos/kubernetes/minikube/statuses/${commit}?access_token=${token}" \
+      -H "Content-Type: application/json" \
+      -X POST \
+      -d "{\"state\": \"${state}\", \"description\": \"Jenkins: ${desc}\", \"target_url\": \"${target}\", \"context\": \"${context}\"}" || echo 999)
+
+    # 2xx HTTP codes
+    if [[ "${code}" =~ ^2 ]]; then
+      break
+    fi
+
+    cat "${out}" && rm -f "${out}"
+    echo "HTTP code ${code}! Retrying in ${timeout} .."
+    sleep "${timeout}"
+    attempt=$(( attempt + 1 ))
+    timeout=$(( timeout * 5 ))
+  done
+}
+
+retry_github_status "${COMMIT}" "${JOB_NAME}" "${status}" "${access_token}" "${public_log_url}" "${description}"
+
+exit "$result"
diff --git a/pkg/util/utils_test.go b/pkg/util/utils_test.go
index 81952d73c..3e5e36965 100644
--- a/pkg/util/utils_test.go
+++ b/pkg/util/utils_test.go
@@ -20,6 +20,7 @@ import (
 	"io/ioutil"
 	"os"
 	"os/user"
+	"runtime"
 	"syscall"
 	"testing"
 
@@ -38,7 +39,7 @@ func TestGetBinaryDownloadURL(t *testing.T) {
 	}
 
 	for _, tt := range testData {
-		url := GetBinaryDownloadURL(tt.version, tt.platform)
+		url := GetBinaryDownloadURL(tt.version, tt.platform, runtime.GOARCH)
 		if url != tt.expectedURL {
 			t.Fatalf("Expected '%s' but got '%s'", tt.expectedURL, url)
 		}
diff --git a/test/integration/addons_test.go b/test/integration/addons_test.go
index 62e982aec..59209f055 100644
--- a/test/integration/addons_test.go
+++ b/test/integration/addons_test.go
@@ -345,6 +345,10 @@ func validateHelmTillerAddon(ctx context.Context, t *testing.T, profile string)
 
 	defer PostMortemLogs(t, profile)
 
+	if arm64Platform() {
+		t.Skip("skip Helm test on arm64")
+	}
+
 	client, err := kapi.Client(profile)
 	if err != nil {
 		t.Fatalf("failed to get Kubernetes client for %s: %v", profile, err)
diff --git a/test/integration/version_upgrade_test.go b/test/integration/version_upgrade_test.go
index 01f02e122..841dc4174 100644
--- a/test/integration/version_upgrade_test.go
+++ b/test/integration/version_upgrade_test.go
@@ -43,7 +43,7 @@ func installRelease(version string) (f *os.File, err error) {
 	}
 	tf.Close()
 
-	url := pkgutil.GetBinaryDownloadURL(version, runtime.GOOS)
+	url := pkgutil.GetBinaryDownloadURL(version, runtime.GOOS, runtime.GOARCH)
 
 	if err := retry.Expo(func() error { return getter.GetFile(tf.Name(), url) }, 3*time.Second, Minutes(3)); err != nil {
 		return tf, err
@@ -79,8 +79,13 @@ func TestRunningBinaryUpgrade(t *testing.T) {
 	// Should be a version from the last 6 months
 	legacyVersion := "v1.6.2"
 	if KicDriver() {
-		// v1.8.0 would be selected, but: https://github.com/kubernetes/minikube/issues/8740
-		legacyVersion = "v1.9.0"
+		if arm64Platform() {
+			// arm64 KIC driver is supported starting from v1.17.0
+			legacyVersion = "v1.17.0"
+		} else {
+			// v1.8.0 would be selected, but: https://github.com/kubernetes/minikube/issues/8740
+			legacyVersion = "v1.9.0"
+		}
 	}
 
 	tf, err := installRelease(legacyVersion)
@@ -147,6 +152,10 @@ func TestStoppedBinaryUpgrade(t *testing.T) {
 	if KicDriver() {
 		// first release with non-experimental KIC
 		legacyVersion = "v1.8.0"
+		if arm64Platform() {
+			// first release with non-experimental arm64 KIC
+			legacyVersion = "v1.17.0"
+		}
 	}
 
 	tf, err := installRelease(legacyVersion)
diff --git a/test/stress/stress_test.go b/test/stress/stress_test.go
index 5bcf8a61c..629119172 100644
--- a/test/stress/stress_test.go
+++ b/test/stress/stress_test.go
@@ -48,7 +48,7 @@ func TestStress(t *testing.T) {
 	}
 
 	oldPath := fmt.Sprintf("../../out/minikube-%s", *upgradeFrom)
-	url := util.GetBinaryDownloadURL(*upgradeFrom, runtime.GOOS)
+	url := util.GetBinaryDownloadURL(*upgradeFrom, runtime.GOOS, runtime.GOARCH)
 	t.Logf("Downloading minikube %s from %s", *upgradeFrom, url)
 	err = getter.GetFile(oldPath, url)
 	if err != nil {