Repository: code-ready/crc
Pull request #2699 (2021-09-06 14:28:15)

[integration] Add container to run integration test within a target host
This PR adds a makefile goal to generate a container image holding the integration tests binary. The container image will manage the installation and run of the integration tests and will pick the results back.

This PR will have a following one having only one crc-test container to hold both e2e and integration binaries, the container will be created per platform to reduce the sizing

openshift-ci[bot]: [APPROVALNOTIFIER] This PR is **NOT APPROVED**

This pull-request has been approved by: *<a href="https://github.com/code-ready/crc/pull/2699#" title="Author self-approved">adrianriobo</a>*
To complete the [pull request process](https://git.k8s.io/community/contributors/guide/owners.md#the-code-review-process), please assign **gbraad** after the PR has been reviewed.
You can assign the PR to them by writing `/assign @gbraad` in a comment when ready.

The full list of commands accepted by this bot can be found [here](https://go.k8s.io/bot-commands?repo=code-ready%2Fcrc).

<details open>
Needs approval from an approver in each of these files:

- **[OWNERS](https://github.com/code-ready/crc/blob/master/OWNERS)**

Approvers can indicate their approval by writing `/approve` in a comment
Approvers can cancel approval by writing `/approve cancel` in a comment
</details>
<!-- META={"approvers":["gbraad"]} -->
openshift-ci[bot]: [APPROVALNOTIFIER] This PR is **APPROVED**

This pull-request has been approved by: *<a href="https://github.com/code-ready/crc/pull/2699#" title="Author self-approved">adrianriobo</a>*, *<a href="https://github.com/code-ready/crc/pull/2699#pullrequestreview-749261809" title="Approved">cfergeau</a>*

The full list of commands accepted by this bot can be found [here](https://go.k8s.io/bot-commands?repo=code-ready%2Fcrc).

The pull request process is described [here](https://git.k8s.io/community/contributors/guide/owners.md#the-code-review-process)

<details >
Needs approval from an approver in each of these files:

- ~~[OWNERS](https://github.com/code-ready/crc/blob/master/OWNERS)~~ [cfergeau]

Approvers can indicate their approval by writing `/approve` in a comment
Approvers can cancel approval by writing `/approve cancel` in a comment
</details>
<!-- META={"approvers":[]} -->

Patch:
diff --git a/Makefile b/Makefile
index a746ce2a9..62457473c 100644
--- a/Makefile
+++ b/Makefile
@@ -160,7 +160,13 @@ build_e2e: $(SOURCES)
 	GOOS=windows go test ./test/e2e/ -c -o $(BUILD_DIR)/windows-amd64/e2e.test.exe
 	GOOS=darwin  go test ./test/e2e/ -c -o $(BUILD_DIR)/macos-amd64/e2e.test
 
-#  Build the container image
+.PHONY: build_integration
+build_integration: $(SOURCES)
+	GOOS=linux   go test ./test/integration/ -c -o $(BUILD_DIR)/linux-amd64/integration.test
+	GOOS=windows go test --ldflags="-X $(REPOPATH)/pkg/crc/version.installerBuild=true" ./test/integration/ -c -o $(BUILD_DIR)/windows-amd64/integration.test.exe
+	GOOS=darwin  go test --ldflags="-X $(REPOPATH)/pkg/crc/version.installerBuild=true" ./test/integration/ -c -o $(BUILD_DIR)/macos-amd64/integration.test
+
+#  Build the container image for e2e
 .PHONY: containerized_e2e
 containerized_e2e:
 ifndef CRC_E2E_IMG_VERSION
@@ -170,6 +176,16 @@ IMG = quay.io/crcont/crc-e2e:$(CRC_E2E_IMG_VERSION)
 containerized_e2e: clean
 	$(CONTAINER_RUNTIME) build -t $(IMG) -f images/build-e2e/Dockerfile .
 
+#  Build the container image for integration
+.PHONY: containerized_integration
+containerized_integration:
+ifndef CRC_INTEGRATION_IMG_VERSION
+CRC_INTEGRATION_IMG_VERSION=v$(CRC_VERSION)-$(COMMIT_SHA)
+endif
+IMG = quay.io/crcont/crc-integration:$(CRC_INTEGRATION_IMG_VERSION)
+containerized_integration: clean
+	$(CONTAINER_RUNTIME) build -t $(IMG) -f images/build-integration/Dockerfile .
+
 .PHONY: integration ## Run integration tests in Ginkgo
 integration:
 ifndef PULL_SECRET_PATH
diff --git a/images/build-integration/Dockerfile b/images/build-integration/Dockerfile
new file mode 100644
index 000000000..307f1a349
--- /dev/null
+++ b/images/build-integration/Dockerfile
@@ -0,0 +1,22 @@
+
+FROM registry.access.redhat.com/ubi8/go-toolset:1.15.13 as builder
+
+USER root
+WORKDIR /workspace
+COPY . .
+RUN make build_integration
+
+FROM registry.access.redhat.com/ubi8/ubi-minimal 
+
+LABEL MAINTAINER "CodeReady Containers <devtools-cdk@redhat.com>"
+
+COPY --from=builder /workspace/images/build-integration/entrypoint.sh /usr/local/bin/
+COPY --from=builder /workspace/out /opt/crc/bin
+
+ENV EPEL https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm
+
+RUN rpm -ivh ${EPEL} \ 
+    && microdnf --enablerepo=epel install -y openssh-clients sshpass \
+    && microdnf clean all
+
+ENTRYPOINT ["entrypoint.sh"]
diff --git a/images/build-integration/README.md b/images/build-integration/README.md
new file mode 100644
index 000000000..8d2946381
--- /dev/null
+++ b/images/build-integration/README.md
@@ -0,0 +1,44 @@
+# Overview
+
+The container includes the integration binary for all 3 platforms plus the required resources to run it.  
+
+The container connects through ssh to the target host and copy the right binary for the platform, run integration tests and pick the results and logs back.
+
+## Envs
+
+**PLATFORM**:*define target platform (windows, macos, linux).*
+**TARGET_HOST**:*dns or ip for the target host.*  
+**TARGET_HOST_USERNAME**:*username for target host.*  
+**TARGET_HOST_KEY_PATH**:*private key for user. (Mandatory if not TARGET_HOST_PASSWORD).*  
+**TARGET_HOST_PASSWORD**:*password for user. (Mandatory if not TARGET_HOST_KEY_PATH).*  
+**PULL_SECRET_FILE_PATH** pull secret file path (local to container).*  
+**BUNDLE_LOCATION**:*(Optional). When testing crc with custom bundle set the bundle location on target server.*  
+**RESULTS_PATH**:*(Optional). Path inside container to pick results and logs from integration execution.*
+
+## Samples
+
+```bash
+# Run integration on macos platform with ssh key and custom bundle
+podman run --rm -it --name crc-integration \
+    -e PLATFORM=macos \
+    -e TARGET_HOST=$IP \
+    -e TARGET_HOST_USERNAME=$USER \
+    -e TARGET_HOST_KEY_PATH=/opt/crc/id_rsa \
+    -e PULL_SECRET_FILE_PATH=/opt/crc/pull-secret \
+    -e BUNDLE_LOCATION=/bundles/crc_hyperv_4.8.0-rc.3.crcbundle \
+    -v $PWD/pull-secret:/opt/crc/pull-secret:Z \
+    -v $PWD/id_rsa:/opt/crc/id_rsa:Z \
+    -v $PWD/output:/output:Z \
+    quay.io/crcont/crc-integration:v1.29.0
+
+# Run integration on windows platform with ssh password and crc released version
+podman run --rm -it --name crc-integration \
+    -e PLATFORM=windows \
+    -e TARGET_HOST=$IP \
+    -e TARGET_HOST_USERNAME=$USER \
+    -e TARGET_HOST_PASSWORD=$PASSWORD \
+    -e PULL_SECRET_FILE_PATH=/opt/crc/pull-secret \
+    -v $PWD/pull-secret:/opt/crc/pull-secret:Z \
+    -v $PWD/output:/output:Z \
+    quay.io/crcont/crc-integration:v1.29.0
+```
diff --git a/images/build-integration/entrypoint.sh b/images/build-integration/entrypoint.sh
new file mode 100755
index 000000000..5673baef3
--- /dev/null
+++ b/images/build-integration/entrypoint.sh
@@ -0,0 +1,102 @@
+#!/bin/sh
+
+# Vars
+BINARY=integration.test
+if [[ ${PLATFORM} == 'windows' ]]; then
+    BINARY=integration.test.exe
+fi
+BINARY_PATH="/opt/crc/bin/${PLATFORM}-amd64/${BINARY}"
+
+# Results
+RESULTS_PATH="${RESULTS_PATH:-/output}"
+
+
+if [ "${DEBUG:-}" = "true" ]; then
+    set -xuo 
+fi
+
+# Validate conf
+validate=true
+[[ -z "${TARGET_HOST+x}" ]] \
+    && echo "TARGET_HOST required" \
+    && validate=false
+
+[[ -z "${TARGET_HOST_USERNAME+x}" ]] \
+    && echo "TARGET_HOST_USERNAME required" \
+    && validate=false
+
+[[ -z "${TARGET_HOST_KEY_PATH+x}" && -z "${TARGET_HOST_PASSWORD+x}" ]] \
+    && echo "TARGET_HOST_KEY_PATH or TARGET_HOST_PASSWORD required" \
+    && validate=false
+
+[[ -z "${PULL_SECRET_FILE_PATH+x}" ]] \
+    && echo "PULL_SECRET_FILE_PATH required" \
+    && validate=false
+
+[[ $validate == false ]] && exit 1
+
+# Define remote connection
+REMOTE="${TARGET_HOST_USERNAME}@${TARGET_HOST}"
+if [[ ! -z "${TARGET_HOST_DOMAIN+x}" ]]; then
+    REMOTE="${TARGET_HOST_USERNAME}@${TARGET_HOST_DOMAIN}@${TARGET_HOST}"
+fi
+
+# Set SCP / SSH command with pass or key
+NO_STRICT='-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'
+if [[ ! -z "${TARGET_HOST_KEY_PATH+x}" ]]; then
+    SCP="scp -r ${NO_STRICT} -i ${TARGET_HOST_KEY_PATH}"
+    SSH="ssh ${NO_STRICT} -i ${TARGET_HOST_KEY_PATH}"
+else
+    SCP="sshpass -p ${TARGET_HOST_PASSWORD} scp -r ${NO_STRICT}" \
+    SSH="sshpass -p ${TARGET_HOST_PASSWORD} ssh ${NO_STRICT}"
+fi
+
+echo "Copy resources to target"
+# Create execution folder 
+EXECUTION_FOLDER="/Users/${TARGET_HOST_USERNAME}/crc-integration"
+if [[ ${PLATFORM} == 'linux' ]]; then
+    EXECUTION_FOLDER="/home/${TARGET_HOST_USERNAME}/crc-integration"
+fi
+DATA_FOLDER="${EXECUTION_FOLDER}/out"
+if [[ ${PLATFORM} == 'windows' ]]; then
+    # Todo change for powershell cmdlet
+    $SSH "${REMOTE}" "powershell.exe -c New-Item -ItemType directory -Path ${EXECUTION_FOLDER}/bin"
+else
+    $SSH "${REMOTE}" "mkdir -p ${EXECUTION_FOLDER}/bin"
+fi
+
+# Copy crc-integration binary and pull-secret
+$SCP "${BINARY_PATH}" "${REMOTE}:${EXECUTION_FOLDER}/bin"
+$SCP "${PULL_SECRET_FILE_PATH}" "${REMOTE}:${EXECUTION_FOLDER}/pull-secret"
+
+echo "Running integration tests"
+# Run integration cmd
+if [[ ! -z "${BUNDLE_LOCATION+x}" ]] && [[ -n "${BUNDLE_LOCATION}" ]]; then
+    BUNDLE_PATH="${BUNDLE_LOCATION}"
+else
+    BUNDLE_PATH="embedded"
+fi
+# Review when pwsh added as powershell supported
+if [[ ${PLATFORM} == 'windows' ]]; then
+    BINARY_EXEC="cd ${EXECUTION_FOLDER}/bin; "
+    BINARY_EXEC+="\$env:SHELL='powershell'; "
+    BINARY_EXEC+="\$env:PULL_SECRET_PATH='${EXECUTION_FOLDER}/pull-secret'; "
+    BINARY_EXEC+="\$env:BUNDLE_PATH='${BUNDLE_PATH}'; "
+else 
+    BINARY_EXEC="cd ${EXECUTION_FOLDER}/bin && "
+    BINARY_EXEC+="PULL_SECRET_PATH=${EXECUTION_FOLDER}/pull-secret "
+    BINARY_EXEC+="BUNDLE_PATH=${BUNDLE_PATH} "
+fi
+BINARY_EXEC+="./${BINARY} > integration.results"
+# Execute command remote
+$SSH "${REMOTE}" "${BINARY_EXEC}"
+
+echo "Getting integration results and logs"
+# Get results
+mkdir -p "${RESULTS_PATH}"
+$SCP "${REMOTE}:${EXECUTION_FOLDER}/bin/integration.results" "${RESULTS_PATH}"
+$SCP "${REMOTE}:${EXECUTION_FOLDER}/bin/out/integration.xml" "${RESULTS_PATH}"
+
+echo "Cleanup target"
+# Clenaup
+$SSH "${REMOTE}" "rm -r ${EXECUTION_FOLDER}"
\ No newline at end of file
diff --git a/pkg/crc/api/api_http.go b/pkg/crc/api/api_http.go
index 7bf221b1f..f53bca968 100644
--- a/pkg/crc/api/api_http.go
+++ b/pkg/crc/api/api_http.go
@@ -11,6 +11,12 @@ import (
 func NewMux(config crcConfig.Storage, machine machine.Client, logger Logger, telemetry Telemetry) http.Handler {
 	handler := NewHandler(config, machine, logger, telemetry)
 
+	server := newServerWithRoutes(handler)
+
+	return server.Handler()
+}
+
+func newServerWithRoutes(handler *Handler) *server {
 	server := newServer()
 
 	server.POST("/start", handler.Start)
@@ -20,6 +26,7 @@ func NewMux(config crcConfig.Storage, machine machine.Client, logger Logger, tel
 	server.GET("/stop", handler.Stop)
 
 	server.POST("/poweroff", handler.PowerOff)
+
 	server.GET("/status", handler.Status)
 
 	server.DELETE("/delete", handler.Delete)
@@ -38,10 +45,10 @@ func NewMux(config crcConfig.Storage, machine machine.Client, logger Logger, tel
 	server.GET("/telemetry", handler.UploadTelemetry)
 	server.POST("/telemetry", handler.UploadTelemetry)
 
-	server.GET("/pull-secret", getPullSecret(config))
+	server.GET("/pull-secret", getPullSecret(handler.Config))
 	server.POST("/pull-secret", setPullSecret())
 
-	return server.Handler()
+	return server
 }
 
 func setPullSecret() func(c *context) error {
diff --git a/pkg/crc/cluster/cert_renewal.go b/pkg/crc/cluster/cert_renewal.go
index a9bd592f6..f1f939581 100644
--- a/pkg/crc/cluster/cert_renewal.go
+++ b/pkg/crc/cluster/cert_renewal.go
@@ -1,6 +1,7 @@
 package cluster
 
 import (
+	"context"
 	"fmt"
 	"time"
 
@@ -16,9 +17,9 @@ func isPending(csr *k8scerts.CertificateSigningRequest) bool {
 	return len(csr.Status.Conditions) == 0 && len(csr.Status.Certificate) == 0
 }
 
-func approvePendingCSRs(ocConfig oc.Config, expectedSignerName string) error {
-	return crcerrors.RetryAfter(8*time.Minute, func() error {
-		csrs, err := getCSRList(ocConfig, expectedSignerName)
+func approvePendingCSRs(ctx context.Context, ocConfig oc.Config, expectedSignerName string) error {
+	return crcerrors.Retry(ctx, 8*time.Minute, func() error {
+		csrs, err := getCSRList(ctx, ocConfig, expectedSignerName)
 		if err != nil {
 			return &crcerrors.RetriableError{Err: err}
 		}
@@ -42,7 +43,7 @@ func approvePendingCSRs(ocConfig oc.Config, expectedSignerName string) error {
 	}, time.Second*5)
 }
 
-func ApproveCSRAndWaitForCertsRenewal(sshRunner *ssh.Runner, ocConfig oc.Config, client, server bool) error {
+func ApproveCSRAndWaitForCertsRenewal(ctx context.Context, sshRunner *ssh.Runner, ocConfig oc.Config, client, server bool) error {
 	const (
 		kubeletClientSignerName = "kubernetes.io/kube-apiserver-client-kubelet"
 		authClientSignerName    = "kubernetes.io/kube-apiserver-client"
@@ -53,7 +54,7 @@ func ApproveCSRAndWaitForCertsRenewal(sshRunner *ssh.Runner, ocConfig oc.Config,
 	// Kubelet stores the cert in /var/lib/kubelet/pki/kubelet-client-current.pem
 	if client {
 		logging.Info("Kubelet client certificate has expired, renewing it... [will take up to 8 minutes]")
-		if err := approvePendingCSRs(ocConfig, kubeletClientSignerName); err != nil {
+		if err := approvePendingCSRs(ctx, ocConfig, kubeletClientSignerName); err != nil {
 			logging.Debugf("Error approving pending kube-apiserver-client-kubelet CSRs: %v", err)
 			return err
 		}
@@ -62,11 +63,11 @@ func ApproveCSRAndWaitForCertsRenewal(sshRunner *ssh.Runner, ocConfig oc.Config,
 		// if the patch backported to 4.8 z stream.
 		// https://github.com/openshift/library-go/pull/1190 and https://github.com/openshift/cluster-authentication-operator/pull/475
 		// https://bugzilla.redhat.com/show_bug.cgi?id=1997906
-		if err := deleteCSR(ocConfig, authClientSignerName); err != nil {
+		if err := deleteCSR(ctx, ocConfig, authClientSignerName); err != nil {
 			logging.Debugf("Error deleting openshift-authenticator csr: %v", err)
 			return err
 		}
-		if err := crcerrors.RetryAfter(5*time.Minute, waitForCertRenewal(sshRunner, KubeletClientCert), time.Second*5); err != nil {
+		if err := crcerrors.Retry(ctx, 5*time.Minute, waitForCertRenewal(sshRunner, KubeletClientCert), time.Second*5); err != nil {
 			logging.Debugf("Error approving pending kube-apiserver-client-kubelet CSR: %v", err)
 			return err
 		}
@@ -77,7 +78,7 @@ func ApproveCSRAndWaitForCertsRenewal(sshRunner *ssh.Runner, ocConfig oc.Config,
 	// This CSR is automatically approved by the cluster-machine-approver. The k8s controller manager issues the cert and kubelet fetches it.
 	if server {
 		logging.Info("Kubelet serving certificate has expired, waiting for automatic renewal... [will take up to 8 minutes]")
-		return crcerrors.RetryAfter(5*time.Minute, waitForCertRenewal(sshRunner, KubeletServerCert), time.Second*5)
+		return crcerrors.Retry(ctx, 5*time.Minute, waitForCertRenewal(sshRunner, KubeletServerCert), time.Second*5)
 	}
 	return nil
 }
diff --git a/pkg/crc/cluster/cluster.go b/pkg/crc/cluster/cluster.go
index d445eb8f7..a782120c1 100644
--- a/pkg/crc/cluster/cluster.go
+++ b/pkg/crc/cluster/cluster.go
@@ -82,13 +82,13 @@ func GetRootPartitionUsage(sshRunner *ssh.Runner) (int64, int64, error) {
 	return diskSize, diskUsage, nil
 }
 
-func EnsureSSHKeyPresentInTheCluster(ocConfig oc.Config, sshPublicKeyPath string) error {
+func EnsureSSHKeyPresentInTheCluster(ctx context.Context, ocConfig oc.Config, sshPublicKeyPath string) error {
 	sshPublicKeyByte, err := ioutil.ReadFile(sshPublicKeyPath)
 	if err != nil {
 		return err
 	}
 	sshPublicKey := strings.TrimRight(string(sshPublicKeyByte), "\r\n")
-	if err := WaitForOpenshiftResource(ocConfig, "machineconfigs"); err != nil {
+	if err := WaitForOpenshiftResource(ctx, ocConfig, "machineconfigs"); err != nil {
 		return err
 	}
 	stdout, stderr, err := ocConfig.RunOcCommand("get", "machineconfigs", "99-master-ssh", "-o", `jsonpath='{.spec.config.passwd.users[0].sshAuthorizedKeys[0]}'`)
@@ -109,8 +109,8 @@ func EnsureSSHKeyPresentInTheCluster(ocConfig oc.Config, sshPublicKeyPath string
 	return nil
 }
 
-func EnsurePullSecretPresentInTheCluster(ocConfig oc.Config, pullSec PullSecretLoader) error {
-	if err := WaitForOpenshiftResource(ocConfig, "secret"); err != nil {
+func EnsurePullSecretPresentInTheCluster(ctx context.Context, ocConfig oc.Config, pullSec PullSecretLoader) error {
+	if err := WaitForOpenshiftResource(ctx, ocConfig, "secret"); err != nil {
 		return err
 	}
 
@@ -143,9 +143,9 @@ func EnsurePullSecretPresentInTheCluster(ocConfig oc.Config, pullSec PullSecretL
 	return nil
 }
 
-func EnsureGeneratedClientCAPresentInTheCluster(ocConfig oc.Config, sshRunner *ssh.Runner, selfSignedCACert *x509.Certificate, adminCert string) error {
+func EnsureGeneratedClientCAPresentInTheCluster(ctx context.Context, ocConfig oc.Config, sshRunner *ssh.Runner, selfSignedCACert *x509.Certificate, adminCert string) error {
 	selfSignedCAPem := crctls.CertToPem(selfSignedCACert)
-	if err := WaitForOpenshiftResource(ocConfig, "configmaps"); err != nil {
+	if err := WaitForOpenshiftResource(ctx, ocConfig, "configmaps"); err != nil {
 		return err
 	}
 	clusterClientCA, stderr, err := ocConfig.RunOcCommand("get", "configmaps", "admin-kubeconfig-client-ca", "-n", "openshift-config", "-o", `jsonpath="{.data.ca-bundle\.crt}"`)
@@ -176,7 +176,7 @@ func EnsureGeneratedClientCAPresentInTheCluster(ocConfig oc.Config, sshRunner *s
 	return nil
 }
 
-func RemovePullSecretFromCluster(ocConfig oc.Config, sshRunner *ssh.Runner) error {
+func RemovePullSecretFromCluster(ctx context.Context, ocConfig oc.Config, sshRunner *ssh.Runner) error {
 	logging.Info("Removing user's pull secret from instance disk and from cluster secret...")
 	cmdArgs := []string{"patch", "secret", "pull-secret", "-p",
 		`'{"data":{".dockerconfigjson":"e30K"}}'`,
@@ -186,10 +186,10 @@ func RemovePullSecretFromCluster(ocConfig oc.Config, sshRunner *ssh.Runner) erro
 	if err != nil {
 		return fmt.Errorf("Failed to remove Pull secret %w: %s", err, stderr)
 	}
-	return waitForPullSecretRemovedFromInstanceDisk(sshRunner)
+	return waitForPullSecretRemovedFromInstanceDisk(ctx, sshRunner)
 }
 
-func waitForPullSecretRemovedFromInstanceDisk(sshRunner *ssh.Runner) error {
+func waitForPullSecretRemovedFromInstanceDisk(ctx context.Context, sshRunner *ssh.Runner) error {
 	logging.Info("Waiting for user's pull secret removed from instance disk...")
 	pullSecretPresentFunc := func() error {
 		stdout, stderr, err := sshRunner.RunPrivate(fmt.Sprintf("sudo cat %s", vmPullSecretPath))
@@ -201,7 +201,7 @@ func waitForPullSecretRemovedFromInstanceDisk(sshRunner *ssh.Runner) error {
 		}
 		return nil
 	}
-	return errors.RetryAfter(1*time.Minute, pullSecretPresentFunc, 2*time.Second)
+	return errors.Retry(ctx, 1*time.Minute, pullSecretPresentFunc, 2*time.Second)
 }
 
 func RemoveOldRenderedMachineConfig(ocConfig oc.Config) error {
@@ -249,8 +249,8 @@ func RemoveOldRenderedMachineConfig(ocConfig oc.Config) error {
 	return nil
 }
 
-func EnsureClusterIDIsNotEmpty(ocConfig oc.Config) error {
-	if err := WaitForOpenshiftResource(ocConfig, "clusterversion"); err != nil {
+func EnsureClusterIDIsNotEmpty(ctx context.Context, ocConfig oc.Config) error {
+	if err := WaitForOpenshiftResource(ctx, ocConfig, "clusterversion"); err != nil {
 		return err
 	}
 
@@ -275,7 +275,7 @@ func EnsureClusterIDIsNotEmpty(ocConfig oc.Config) error {
 	return nil
 }
 
-func AddProxyConfigToCluster(sshRunner *ssh.Runner, ocConfig oc.Config, proxy *network.ProxyConfig) error {
+func AddProxyConfigToCluster(ctx context.Context, sshRunner *ssh.Runner, ocConfig oc.Config, proxy *network.ProxyConfig) error {
 	type trustedCA struct {
 		Name string `json:"name"`
 	}
@@ -299,7 +299,7 @@ func AddProxyConfigToCluster(sshRunner *ssh.Runner, ocConfig oc.Config, proxy *n
 		},
 	}
 
-	if err := WaitForOpenshiftResource(ocConfig, "proxy"); err != nil {
+	if err := WaitForOpenshiftResource(ctx, ocConfig, "proxy"); err != nil {
 		return err
 	}
 
@@ -419,7 +419,7 @@ func EnsurePullSecretPresentOnInstanceDisk(sshRunner *ssh.Runner, pullSecret Pul
 	return sshRunner.CopyData([]byte(content), vmPullSecretPath, 0600)
 }
 
-func WaitForPullSecretPresentOnInstanceDisk(sshRunner *ssh.Runner) error {
+func WaitForPullSecretPresentOnInstanceDisk(ctx context.Context, sshRunner *ssh.Runner) error {
 	logging.Info("Waiting for user's pull secret part of instance disk...")
 	pullSecretPresentFunc := func() error {
 		stdout, stderr, err := sshRunner.RunPrivate(fmt.Sprintf("sudo cat %s", vmPullSecretPath))
@@ -431,10 +431,10 @@ func WaitForPullSecretPresentOnInstanceDisk(sshRunner *ssh.Runner) error {
 		}
 		return nil
 	}
-	return errors.RetryAfter(7*time.Minute, pullSecretPresentFunc, 2*time.Second)
+	return errors.Retry(ctx, 7*time.Minute, pullSecretPresentFunc, 2*time.Second)
 }
 
-func WaitForRequestHeaderClientCaFile(sshRunner *ssh.Runner) error {
+func WaitForRequestHeaderClientCaFile(ctx context.Context, sshRunner *ssh.Runner) error {
 	lookupRequestHeaderClientCa := func() error {
 		expired, err := checkCertValidity(sshRunner, AggregatorClientCert)
 		if err != nil {
@@ -445,7 +445,7 @@ func WaitForRequestHeaderClientCaFile(sshRunner *ssh.Runner) error {
 		}
 		return nil
 	}
-	return errors.RetryAfter(8*time.Minute, lookupRequestHeaderClientCa, 2*time.Second)
+	return errors.Retry(ctx, 8*time.Minute, lookupRequestHeaderClientCa, 2*time.Second)
 }
 
 func WaitForAPIServer(ctx context.Context, ocConfig oc.Config) error {
@@ -459,11 +459,11 @@ func WaitForAPIServer(ctx context.Context, ocConfig oc.Config) error {
 		logging.Debug(stdout)
 		return nil
 	}
-	return errors.RetryAfterWithContext(ctx, 4*time.Minute, waitForAPIServer, time.Second)
+	return errors.Retry(ctx, 4*time.Minute, waitForAPIServer, time.Second)
 }
 
-func DeleteOpenshiftAPIServerPods(ocConfig oc.Config) error {
-	if err := WaitForOpenshiftResource(ocConfig, "pod"); err != nil {
+func DeleteOpenshiftAPIServerPods(ctx context.Context, ocConfig oc.Config) error {
+	if err := WaitForOpenshiftResource(ctx, ocConfig, "pod"); err != nil {
 		return err
 	}
 
@@ -476,7 +476,7 @@ func DeleteOpenshiftAPIServerPods(ocConfig oc.Config) error {
 		return nil
 	}
 
-	return errors.RetryAfter(60*time.Second, deleteOpenshiftAPIServerPods, time.Second)
+	return errors.Retry(ctx, 60*time.Second, deleteOpenshiftAPIServerPods, time.Second)
 }
 
 func CheckProxySettingsForOperator(ocConfig oc.Config, proxy *network.ProxyConfig, deployment, namespace string) (bool, error) {
@@ -495,8 +495,8 @@ func CheckProxySettingsForOperator(ocConfig oc.Config, proxy *network.ProxyConfi
 	return false, nil
 }
 
-func DeleteMCOLeaderLease(ocConfig oc.Config) error {
-	if err := WaitForOpenshiftResource(ocConfig, "cm"); err != nil {
+func DeleteMCOLeaderLease(ctx context.Context, ocConfig oc.Config) error {
+	if err := WaitForOpenshiftResource(ctx, ocConfig, "cm"); err != nil {
 		return err
 	}
 	_, _, err := ocConfig.RunOcCommand("delete", "-n", "openshift-machine-config-operator", "cm", "machine-config-controller")
diff --git a/pkg/crc/cluster/csr.go b/pkg/crc/cluster/csr.go
index ea06b2adf..9ed5e78a6 100644
--- a/pkg/crc/cluster/csr.go
+++ b/pkg/crc/cluster/csr.go
@@ -1,6 +1,7 @@
 package cluster
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"time"
@@ -12,7 +13,7 @@ import (
 	k8scerts "k8s.io/api/certificates/v1beta1"
 )
 
-func WaitForOpenshiftResource(ocConfig oc.Config, resource string) error {
+func WaitForOpenshiftResource(ctx context.Context, ocConfig oc.Config, resource string) error {
 	logging.Debugf("Waiting for availability of resource type '%s'", resource)
 	waitForAPIServer := func() error {
 		stdout, stderr, err := ocConfig.WithFailFast().RunOcCommand("get", resource)
@@ -23,11 +24,11 @@ func WaitForOpenshiftResource(ocConfig oc.Config, resource string) error {
 		logging.Debug(stdout)
 		return nil
 	}
-	return crcerrors.RetryAfter(80*time.Second, waitForAPIServer, time.Second)
+	return crcerrors.Retry(ctx, 80*time.Second, waitForAPIServer, time.Second)
 }
 
-func deleteCSR(ocConfig oc.Config, expectedSignerName string) error {
-	csrs, err := getCSRList(ocConfig, expectedSignerName)
+func deleteCSR(ctx context.Context, ocConfig oc.Config, expectedSignerName string) error {
+	csrs, err := getCSRList(ctx, ocConfig, expectedSignerName)
 	if err != nil {
 		return err
 	}
@@ -41,9 +42,9 @@ func deleteCSR(ocConfig oc.Config, expectedSignerName string) error {
 	return nil
 }
 
-func getCSRList(ocConfig oc.Config, expectedSignerName string) (*k8scerts.CertificateSigningRequestList, error) {
+func getCSRList(ctx context.Context, ocConfig oc.Config, expectedSignerName string) (*k8scerts.CertificateSigningRequestList, error) {
 	var csrs k8scerts.CertificateSigningRequestList
-	if err := WaitForOpenshiftResource(ocConfig, "csr"); err != nil {
+	if err := WaitForOpenshiftResource(ctx, ocConfig, "csr"); err != nil {
 		return nil, err
 	}
 	output, stderr, err := ocConfig.WithFailFast().RunOcCommand("get", "csr", "-ojson")
diff --git a/pkg/crc/cluster/kubeadmin_password.go b/pkg/crc/cluster/kubeadmin_password.go
index 305c4653a..87b0f9a60 100644
--- a/pkg/crc/cluster/kubeadmin_password.go
+++ b/pkg/crc/cluster/kubeadmin_password.go
@@ -3,6 +3,7 @@ package cluster
 import (
 	"bufio"
 	"bytes"
+	"context"
 	"crypto/rand"
 	"encoding/base64"
 	"fmt"
@@ -28,7 +29,7 @@ func GenerateKubeAdminUserPassword() error {
 }
 
 // UpdateKubeAdminUserPassword updates the htpasswd secret
-func UpdateKubeAdminUserPassword(ocConfig oc.Config, newPassword string) error {
+func UpdateKubeAdminUserPassword(ctx context.Context, ocConfig oc.Config, newPassword string) error {
 	if newPassword != "" {
 		logging.Infof("Overriding password for kubeadmin user")
 		if err := ioutil.WriteFile(constants.GetKubeAdminPasswordPath(), []byte(strings.TrimSpace(newPassword)), 0600); err != nil {
@@ -45,7 +46,7 @@ func UpdateKubeAdminUserPassword(ocConfig oc.Config, newPassword string) error {
 		"kubeadmin": kubeAdminPassword,
 	}
 
-	if err := WaitForOpenshiftResource(ocConfig, "secret"); err != nil {
+	if err := WaitForOpenshiftResource(ctx, ocConfig, "secret"); err != nil {
 		return err
 	}
 
diff --git a/pkg/crc/errors/multierror.go b/pkg/crc/errors/multierror.go
index d6daeab28..527a2b918 100644
--- a/pkg/crc/errors/multierror.go
+++ b/pkg/crc/errors/multierror.go
@@ -81,8 +81,8 @@ func (r *RetriableError) Error() string {
 	return "Temporary error: " + r.Err.Error()
 }
 
-// RetryAfterWithContext retries for a certain duration, after a delay
-func RetryAfterWithContext(ctx context.Context, limit time.Duration, callback func() error, d time.Duration) error {
+// Retry retries for a certain duration, after a delay
+func Retry(ctx context.Context, limit time.Duration, callback func() error, d time.Duration) error {
 	if ctx.Err() != nil {
 		return ctx.Err()
 	}
@@ -112,8 +112,3 @@ func RetryAfterWithContext(ctx context.Context, limit time.Duration, callback fu
 	logging.Debugf("RetryAfter timeout after %d tries", attempt)
 	return m
 }
-
-// RetryAfter retries for a certain duration, after a delay
-func RetryAfter(limit time.Duration, callback func() error, d time.Duration) error {
-	return RetryAfterWithContext(context.Background(), limit, callback, d)
-}
diff --git a/pkg/crc/machine/bundle/repository.go b/pkg/crc/machine/bundle/repository.go
index bbe194eca..7c3f5f278 100644
--- a/pkg/crc/machine/bundle/repository.go
+++ b/pkg/crc/machine/bundle/repository.go
@@ -1,6 +1,7 @@
 package bundle
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
@@ -153,12 +154,17 @@ func (repo *Repository) Extract(path string) error {
 	bundleBaseDir := GetBundleNameWithoutExtension(bundleName)
 	bundleDir := filepath.Join(repo.CacheDir, bundleBaseDir)
 	_ = os.RemoveAll(bundleDir)
-	return crcerrors.RetryAfter(time.Minute, func() error {
+	err := crcerrors.Retry(context.Background(), time.Minute, func() error {
 		if err := os.Rename(filepath.Join(tmpDir, bundleBaseDir), bundleDir); err != nil {
 			return &crcerrors.RetriableError{Err: err}
 		}
 		return nil
 	}, 5*time.Second)
+	if err != nil {
+		return err
+	}
+
+	return os.Chmod(bundleDir, 0755)
 }
 
 func (repo *Repository) List() ([]CrcBundleInfo, error) {
diff --git a/pkg/crc/machine/fakemachine/client.go b/pkg/crc/machine/fakemachine/client.go
index f75bc24fb..bb6b52186 100644
--- a/pkg/crc/machine/fakemachine/client.go
+++ b/pkg/crc/machine/fakemachine/client.go
@@ -69,6 +69,9 @@ func (c *Client) PowerOff() error {
 }
 
 func (c *Client) GenerateBundle(forceStop bool) error {
+	if c.Failing {
+		return errors.New("bundle generation failed")
+	}
 	return nil
 }
 
diff --git a/pkg/crc/machine/generate_bundle.go b/pkg/crc/machine/generate_bundle.go
index 7347b2f1b..310bff0b5 100644
--- a/pkg/crc/machine/generate_bundle.go
+++ b/pkg/crc/machine/generate_bundle.go
@@ -1,6 +1,7 @@
 package machine
 
 import (
+	"context"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -23,7 +24,7 @@ func (client *client) GenerateBundle(forceStop bool) error {
 	defer sshRunner.Close()
 
 	ocConfig := oc.UseOCWithSSH(sshRunner)
-	if err := cluster.RemovePullSecretFromCluster(ocConfig, sshRunner); err != nil {
+	if err := cluster.RemovePullSecretFromCluster(context.Background(), ocConfig, sshRunner); err != nil {
 		return errors.Wrap(err, "Error removing pull secret from cluster")
 	}
 
diff --git a/pkg/crc/machine/start.go b/pkg/crc/machine/start.go
index 02c82006d..4dc8eed69 100644
--- a/pkg/crc/machine/start.go
+++ b/pkg/crc/machine/start.go
@@ -338,7 +338,7 @@ func (client *client) Start(ctx context.Context, startConfig types.StartConfig)
 	}
 
 	// Check DNS lookup before starting the kubelet
-	if queryOutput, err := dns.CheckCRCLocalDNSReachable(servicePostStartConfig); err != nil {
+	if queryOutput, err := dns.CheckCRCLocalDNSReachable(ctx, servicePostStartConfig); err != nil {
 		if !client.useVSock() {
 			return nil, errors.Wrapf(err, "Failed internal DNS query: %s", queryOutput)
 		}
@@ -385,7 +385,7 @@ func (client *client) Start(ctx context.Context, startConfig types.StartConfig)
 
 	ocConfig := oc.UseOCWithSSH(sshRunner)
 
-	if err := cluster.ApproveCSRAndWaitForCertsRenewal(sshRunner, ocConfig, certsExpired[cluster.KubeletClientCert], certsExpired[cluster.KubeletServerCert]); err != nil {
+	if err := cluster.ApproveCSRAndWaitForCertsRenewal(ctx, sshRunner, ocConfig, certsExpired[cluster.KubeletClientCert], certsExpired[cluster.KubeletServerCert]); err != nil {
 		logBundleDate(crcBundleMetadata)
 		return nil, errors.Wrap(err, "Failed to renew TLS certificates: please check if a newer CodeReady Containers release is available")
 	}
@@ -394,31 +394,31 @@ func (client *client) Start(ctx context.Context, startConfig types.StartConfig)
 		return nil, errors.Wrap(err, "Error waiting for apiserver")
 	}
 
-	if err := cluster.DeleteMCOLeaderLease(ocConfig); err != nil {
+	if err := cluster.DeleteMCOLeaderLease(ctx, ocConfig); err != nil {
 		return nil, err
 	}
 
-	if err := cluster.EnsurePullSecretPresentInTheCluster(ocConfig, startConfig.PullSecret); err != nil {
+	if err := cluster.EnsurePullSecretPresentInTheCluster(ctx, ocConfig, startConfig.PullSecret); err != nil {
 		return nil, errors.Wrap(err, "Failed to update cluster pull secret")
 	}
 
-	if err := cluster.EnsureSSHKeyPresentInTheCluster(ocConfig, constants.GetPublicKeyPath()); err != nil {
+	if err := cluster.EnsureSSHKeyPresentInTheCluster(ctx, ocConfig, constants.GetPublicKeyPath()); err != nil {
 		return nil, errors.Wrap(err, "Failed to update ssh public key to machine config")
 	}
 
-	if err := cluster.WaitForPullSecretPresentOnInstanceDisk(sshRunner); err != nil {
+	if err := cluster.WaitForPullSecretPresentOnInstanceDisk(ctx, sshRunner); err != nil {
 		return nil, errors.Wrap(err, "Failed to update pull secret on the disk")
 	}
 
-	if err := ensureProxyIsConfiguredInOpenShift(ocConfig, sshRunner, proxyConfig, instanceIP); err != nil {
+	if err := ensureProxyIsConfiguredInOpenShift(ctx, ocConfig, sshRunner, proxyConfig, instanceIP); err != nil {
 		return nil, errors.Wrap(err, "Failed to update cluster proxy configuration")
 	}
 
-	if err := cluster.UpdateKubeAdminUserPassword(ocConfig, startConfig.KubeAdminPassword); err != nil {
+	if err := cluster.UpdateKubeAdminUserPassword(ctx, ocConfig, startConfig.KubeAdminPassword); err != nil {
 		return nil, errors.Wrap(err, "Failed to update kubeadmin user password")
 	}
 
-	if err := cluster.EnsureClusterIDIsNotEmpty(ocConfig); err != nil {
+	if err := cluster.EnsureClusterIDIsNotEmpty(ctx, ocConfig); err != nil {
 		return nil, errors.Wrap(err, "Failed to update cluster ID")
 	}
 
@@ -449,16 +449,16 @@ func (client *client) Start(ctx context.Context, startConfig types.StartConfig)
 	// More info: https://bugzilla.redhat.com/show_bug.cgi?id=1795163
 	if certsExpired[cluster.AggregatorClientCert] {
 		logging.Debug("Waiting for the renewal of the request header client ca...")
-		if err := cluster.WaitForRequestHeaderClientCaFile(sshRunner); err != nil {
+		if err := cluster.WaitForRequestHeaderClientCaFile(ctx, sshRunner); err != nil {
 			return nil, errors.Wrap(err, "Failed to wait for aggregator client ca renewal")
 		}
 
-		if err := cluster.DeleteOpenshiftAPIServerPods(ocConfig); err != nil {
+		if err := cluster.DeleteOpenshiftAPIServerPods(ctx, ocConfig); err != nil {
 			return nil, errors.Wrap(err, "Cannot delete OpenShift API Server pods")
 		}
 	}
 
-	if err := updateKubeconfig(ocConfig, sshRunner, crcBundleMetadata.GetKubeConfigPath()); err != nil {
+	if err := updateKubeconfig(ctx, ocConfig, sshRunner, crcBundleMetadata.GetKubeConfigPath()); err != nil {
 		return nil, errors.Wrap(err, "Failed to update kubeconfig file")
 	}
 
@@ -580,7 +580,7 @@ func startHost(ctx context.Context, api libmachine.API, vm *host.Host) error {
 	}
 
 	logging.Debug("Waiting for machine to be running, this may take a few minutes...")
-	if err := crcerrors.RetryAfterWithContext(ctx, 3*time.Minute, host.MachineInState(vm.Driver, libmachinestate.Running), 3*time.Second); err != nil {
+	if err := crcerrors.Retry(ctx, 3*time.Minute, host.MachineInState(vm.Driver, libmachinestate.Running), 3*time.Second); err != nil {
 		return fmt.Errorf("Error waiting for machine to be running: %s", err)
 	}
 
@@ -644,12 +644,12 @@ func ensureKubeletAndCRIOAreConfiguredForProxy(sshRunner *crcssh.Runner, proxy *
 	return cluster.AddProxyToKubeletAndCriO(sshRunner, proxy)
 }
 
-func ensureProxyIsConfiguredInOpenShift(ocConfig oc.Config, sshRunner *crcssh.Runner, proxy *network.ProxyConfig, instanceIP string) (err error) {
+func ensureProxyIsConfiguredInOpenShift(ctx context.Context, ocConfig oc.Config, sshRunner *crcssh.Runner, proxy *network.ProxyConfig, instanceIP string) (err error) {
 	if !proxy.IsEnabled() {
 		return nil
 	}
 	logging.Info("Adding proxy configuration to the cluster...")
-	return cluster.AddProxyConfigToCluster(sshRunner, ocConfig, proxy)
+	return cluster.AddProxyConfigToCluster(ctx, sshRunner, ocConfig, proxy)
 }
 
 func waitForProxyPropagation(ctx context.Context, ocConfig oc.Config, proxyConfig *network.ProxyConfig) {
@@ -670,7 +670,7 @@ func waitForProxyPropagation(ctx context.Context, ocConfig oc.Config, proxyConfi
 		return nil
 	}
 
-	if err := crcerrors.RetryAfterWithContext(ctx, 300*time.Second, checkProxySettingsForOperator, 2*time.Second); err != nil {
+	if err := crcerrors.Retry(ctx, 300*time.Second, checkProxySettingsForOperator, 2*time.Second); err != nil {
 		logging.Debug("Failed to propagate proxy settings to cluster")
 	}
 }
@@ -719,7 +719,7 @@ func ensureRoutesControllerIsRunning(sshRunner *crcssh.Runner, ocConfig oc.Confi
 	return nil
 }
 
-func updateKubeconfig(ocConfig oc.Config, sshRunner *crcssh.Runner, kubeconfigFilePath string) error {
+func updateKubeconfig(ctx context.Context, ocConfig oc.Config, sshRunner *crcssh.Runner, kubeconfigFilePath string) error {
 	selfSignedCAKey, selfSignedCACert, err := crctls.GetSelfSignedCA()
 	if err != nil {
 		return errors.Wrap(err, "Not able to generate root CA key and Cert")
@@ -731,7 +731,7 @@ func updateKubeconfig(ocConfig oc.Config, sshRunner *crcssh.Runner, kubeconfigFi
 	if err != nil {
 		return errors.Wrap(err, "Not able to get user CA")
 	}
-	if err := cluster.EnsureGeneratedClientCAPresentInTheCluster(ocConfig, sshRunner, selfSignedCACert, adminClientCA); err != nil {
+	if err := cluster.EnsureGeneratedClientCAPresentInTheCluster(ctx, ocConfig, sshRunner, selfSignedCACert, adminClientCA); err != nil {
 		return errors.Wrap(err, "Failed to update user CA to cluster")
 	}
 	return nil
diff --git a/pkg/crc/preflight/preflight_linux.go b/pkg/crc/preflight/preflight_linux.go
index 6db5108fb..af5034019 100644
--- a/pkg/crc/preflight/preflight_linux.go
+++ b/pkg/crc/preflight/preflight_linux.go
@@ -326,7 +326,7 @@ func (filter preflightFilter) SetDistro(distro *linux.OsRelease) {
 
 func (filter preflightFilter) SetSystemdUser(distro *linux.OsRelease) {
 	switch {
-	case distroIsLike(distro, linux.RHEL) && distro.VersionID == "7":
+	case distroIsLike(distro, linux.RHEL) && (distro.VersionID == "7" || strings.HasPrefix(distro.VersionID, "7.")):
 		filter[SystemdUser] = Unsupported
 	default:
 		filter[SystemdUser] = Supported
diff --git a/pkg/crc/services/dns/dns.go b/pkg/crc/services/dns/dns.go
index 2539b77f0..eb7d86a68 100644
--- a/pkg/crc/services/dns/dns.go
+++ b/pkg/crc/services/dns/dns.go
@@ -1,6 +1,7 @@
 package dns
 
 import (
+	"context"
 	"fmt"
 	"time"
 
@@ -95,7 +96,7 @@ func dnsServers(serviceConfig services.ServicePostStartConfig) ([]network.NameSe
 	return append([]network.NameServer{{IPAddress: dnsContainerIP}}, orgResolvValues.NameServers...), nil
 }
 
-func CheckCRCLocalDNSReachable(serviceConfig services.ServicePostStartConfig) (string, error) {
+func CheckCRCLocalDNSReachable(ctx context.Context, serviceConfig services.ServicePostStartConfig) (string, error) {
 	appsURI := fmt.Sprintf("foo.%s", serviceConfig.BundleMetadata.ClusterInfo.AppsDomain)
 	// Try 30 times for 1 second interval, In nested environment most of time crc failed to get
 	// Internal dns query resolved for some time.
@@ -109,7 +110,7 @@ func CheckCRCLocalDNSReachable(serviceConfig services.ServicePostStartConfig) (s
 		return nil
 	}
 
-	if err := errors.RetryAfter(30*time.Second, checkLocalDNSReach, time.Second); err != nil {
+	if err := errors.Retry(ctx, 30*time.Second, checkLocalDNSReach, time.Second); err != nil {
 		return queryOutput, err
 	}
 	return queryOutput, err
diff --git a/pkg/crc/services/dns/dns_darwin.go b/pkg/crc/services/dns/dns_darwin.go
index e3bd12377..4360e1cdf 100644
--- a/pkg/crc/services/dns/dns_darwin.go
+++ b/pkg/crc/services/dns/dns_darwin.go
@@ -2,6 +2,7 @@ package dns
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 	"path/filepath"
 	"strings"
@@ -137,7 +138,7 @@ func waitForNetwork() error {
 		}
 		return nil
 	}
-	if err := crcerrors.RetryAfter(15*time.Second, retriableConnectivityCheck, time.Second); err != nil {
+	if err := crcerrors.Retry(context.Background(), 15*time.Second, retriableConnectivityCheck, time.Second); err != nil {
 		return fmt.Errorf("Host is not connected to internet")
 	}
 
diff --git a/pkg/crc/ssh/ssh.go b/pkg/crc/ssh/ssh.go
index 09d3e2808..ab846dd79 100644
--- a/pkg/crc/ssh/ssh.go
+++ b/pkg/crc/ssh/ssh.go
@@ -105,5 +105,5 @@ func (runner *Runner) WaitForConnectivity(ctx context.Context, timeout time.Dura
 		return nil
 	}
 
-	return errors.RetryAfterWithContext(ctx, timeout, checkSSHConnectivity, time.Second)
+	return errors.Retry(ctx, timeout, checkSSHConnectivity, time.Second)
 }
diff --git a/pkg/libmachine/host/host.go b/pkg/libmachine/host/host.go
index 9319df22f..3efe7f4f9 100644
--- a/pkg/libmachine/host/host.go
+++ b/pkg/libmachine/host/host.go
@@ -1,6 +1,7 @@
 package host
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"net/rpc"
@@ -40,7 +41,7 @@ func (h *Host) runActionForState(action func() error, desiredState state.State)
 		return err
 	}
 
-	return crcerrors.RetryAfter(3*time.Minute, MachineInState(h.Driver, desiredState), 3*time.Second)
+	return crcerrors.Retry(context.Background(), 3*time.Minute, MachineInState(h.Driver, desiredState), 3*time.Second)
 }
 
 func (h *Host) Stop() error {


Test patch:
diff --git a/images/build-e2e/README.md b/images/build-e2e/README.md
index 1d06cd99c..0dc750d9e 100644
--- a/images/build-e2e/README.md
+++ b/images/build-e2e/README.md
@@ -6,41 +6,43 @@ The container connects through ssh to the target host and copy the right binary
 
 ## Envs
 
-* PLATFORM              define target platform (windows, macos, linux).  
-* TARGET_HOST           dns or ip for the target host.  
-* TARGET_HOST_USERNAME  username for target host.  
-* TARGET_HOST_KEY_PATH  private key for user. (Mandatory if not TARGET_HOST_PASSWORD).  
-* TARGET_HOST_PASSWORD  password for user. (Mandatory if not TARGET_HOST_KEY_PATH).  
-* PULL_SECRET_FILE_PATH pull secret file path (local to container).  
-* BUNDLE_VERSION        testing agaisnt crc released version bundle version for crc released version. (Mandatory if not BUNDLE_LOCATION).  
-* BUNDLE_LOCATION       when testing crc with custom bundle set the bundle location on target server. (Mandatory if not BUNDLE_VERSION).  
-* RESULTS_PATH          Optional. Path inside container to pick results and logs from e2e execution.  
+**PLATFORM**:*define target platform (windows, macos, linux).*
+**TARGET_HOST**:*dns or ip for the target host.*  
+**TARGET_HOST_USERNAME**:*username for target host.*  
+**TARGET_HOST_KEY_PATH**:*private key for user. (Mandatory if not TARGET_HOST_PASSWORD).*  
+**TARGET_HOST_PASSWORD**:*password for user. (Mandatory if not TARGET_HOST_KEY_PATH).*  
+**PULL_SECRET_FILE_PATH**: *pull secret file path (local to container).*  
+**BUNDLE_VERSION**:*(Mandatory if not BUNDLE_LOCATION). Testing agaisnt crc released version bundle version for crc released version.*
+**BUNDLE_LOCATION**:*(Mandatory if not BUNDLE_VERSION). When testing crc with custom bundle set the bundle location on target server.*  
+**RESULTS_PATH**:*(Optional). Path inside container to pick results and logs from e2e execution.*  
+**CLEANUP_HOME**:*(Optional). Cleanup crc home folder or keep as it is to run test.*  
+**TESTING_MODE**:*(Optional). Define e2e testing mode (valid values are ux, non-ux. Default is non-ux)*  
 
-# Samples
+## Samples
 
 ```bash
 # Run e2e on macos platform with ssh key and custom bundle
 podman run --rm -it --name crc-e2e \
-		    -e PLATFORM=macos \
-            -e TARGET_HOST=$IP \
-		    -e TARGET_HOST_USERNAME=$USER \
-            -e TARGET_HOST_KEY_PATH=/opt/crc/id_rsa \
-		    -e PULL_SECRET_FILE_PATH=/opt/crc/pull-secret \
-            -e BUNDLE_LOCATION=/bundles/crc_hyperv_4.8.0-rc.3.crcbundle \
-            -v $PWD/pull-secret:/opt/crc/pull-secret:Z \
-            -v $PWD/id_rsa:/opt/crc/id_rsa:Z \
-            -v $PWD/output:/output:Z \
-		    quay.io/crcont/crc-e2e:v1.29.0-465452f4
+    -e PLATFORM=macos \
+    -e TARGET_HOST=$IP \
+    -e TARGET_HOST_USERNAME=$USER \
+    -e TARGET_HOST_KEY_PATH=/opt/crc/id_rsa \
+    -e PULL_SECRET_FILE_PATH=/opt/crc/pull-secret \
+    -e BUNDLE_LOCATION=/bundles/crc_hyperv_4.8.0-rc.3.crcbundle \
+    -v $PWD/pull-secret:/opt/crc/pull-secret:Z \
+    -v $PWD/id_rsa:/opt/crc/id_rsa:Z \
+    -v $PWD/output:/output:Z \
+    quay.io/crcont/crc-e2e:v1.29.0-465452f4
 
 # Run e2e on windows platform with ssh password and crc released version
 podman run --rm -it --name crc-e2e \
-		    -e PLATFORM=windows \
-            -e TARGET_HOST=$IP \
-		    -e TARGET_HOST_USERNAME=$USER \
-            -e TARGET_HOST_PASSWORD=$PASSWORD \
-		    -e PULL_SECRET_FILE_PATH=/opt/crc/pull-secret \
-            -e BUNDLE_VERSION=4.7.18 \
-            -v $PWD/pull-secret:/opt/crc/pull-secret:Z \
-            -v $PWD/output:/output:Z \
-		    quay.io/crcont/crc-e2e:v1.29.0-465452f4
-```
\ No newline at end of file
+    -e PLATFORM=windows \
+    -e TARGET_HOST=$IP \
+    -e TARGET_HOST_USERNAME=$USER \
+    -e TARGET_HOST_PASSWORD=$PASSWORD \
+    -e PULL_SECRET_FILE_PATH=/opt/crc/pull-secret \
+    -e BUNDLE_VERSION=4.7.18 \
+    -v $PWD/pull-secret:/opt/crc/pull-secret:Z \
+    -v $PWD/output:/output:Z \
+    quay.io/crcont/crc-e2e:v1.29.0-465452f4
+```
diff --git a/images/build-e2e/entrypoint.sh b/images/build-e2e/entrypoint.sh
index 8ca821b89..8e7946c35 100755
--- a/images/build-e2e/entrypoint.sh
+++ b/images/build-e2e/entrypoint.sh
@@ -6,13 +6,20 @@ if [[ ${PLATFORM} == 'windows' ]]; then
     BINARY=e2e.test.exe
 fi
 BINARY_PATH="/opt/crc/bin/${PLATFORM}-amd64/${BINARY}"
+
+# Running options
+CLEANUP_HOME="${CLEANUP_HOME:-"true"}"
+# Valid options are non-ux or ux
+TESTING_MODE="${TESTING_MODE:-"non-ux"}"
 # Review this when go 1.16 with embed support
 FEATURES_PATH=/opt/crc/features
 TESTDATA_PATH=/opt/crc/testdata
 UX_RESOURCES_PATH=/opt/crc/ux
+
 # Results
 RESULTS_PATH="${RESULTS_PATH:-/output}"
 
+
 if [ "${DEBUG:-}" = "true" ]; then
     set -xuo 
 fi
@@ -39,6 +46,10 @@ validate=true
     && echo "BUNDLE_VERSION or BUNDLE_LOCATION required" \
     && validate=false
 
+[[ "${TESTING_MODE}" != "ux" && "${TESTING_MODE}" != "non-ux" ]] \
+    && echo "TESTING_MODE should have valid value ux or non-ux" \
+    && validate=false
+
 [[ $validate == false ]] && exit 1
 
 # Define remote connection
@@ -72,8 +83,6 @@ else
 fi
 
 # Copy crc-e2e binary and pull-secret
-# Review this when ux feature cleanup environment 
-rm "${FEATURES_PATH}/ux.feature"
 # Review this when go 1.16 with embed support
 $SCP "${BINARY_PATH}" "${REMOTE}:${EXECUTION_FOLDER}/bin"
 $SCP "${PULL_SECRET_FILE_PATH}" "${REMOTE}:${EXECUTION_FOLDER}/pull-secret"
@@ -86,9 +95,10 @@ if [[ ${PLATFORM} == 'windows' ]]; then
     $SCP "${UX_RESOURCES_PATH}" "${REMOTE}:${REMOTE_RESOURCES_PATH}"
 
 else
-    $SSH "${REMOTE}" "sudo mkdir -p ${REMOTE_RESOURCES_PATH}"
-    $SCP "${UX_RESOURCES_PATH}" "${REMOTE}:${EXECUTION_FOLDER}"
-    $SSH "${REMOTE}" "sudo mv ${EXECUTION_FOLDER}/ux ${REMOTE_RESOURCES_PATH}"
+    # This relies on:  cat /etc/synthetic.conf 
+    #workspace  Users/crcqe/crc-e2e/workspace as symbolic link
+    $SSH "${REMOTE}" "mkdir -p ${EXECUTION_FOLDER}/workspace/test/extended/crc"
+    $SCP "${UX_RESOURCES_PATH}" "${REMOTE}:${EXECUTION_FOLDER}/workspace/test/extended/crc"
 fi
 # Testdata files
 $SCP "${TESTDATA_PATH}" "${REMOTE}:${EXECUTION_FOLDER}"
@@ -104,7 +114,16 @@ OPTIONS+="--pull-secret-file=${EXECUTION_FOLDER}/pull-secret "
 if [[ ${PLATFORM} == 'macos' ]]; then
     PLATFORM="darwin"
 fi
-OPTIONS+="--godog.tags='@${PLATFORM}' --godog.format=junit"
+GODOG_TAGS="@${PLATFORM} "
+if [ "${TESTING_MODE}" = "non-ux" ]; then
+    GODOG_TAGS+="&& ~@ux "
+else 
+    GODOG_TAGS+="&& @ux "
+fi
+OPTIONS+="--godog.tags='${GODOG_TAGS}' --godog.format=junit "
+if [ "${CLEANUP_HOME}" = "false" ]; then
+    OPTIONS+="--cleanup-home=false "
+fi
 # Review when pwsh added as powershell supported
 if [[ ${PLATFORM} == 'windows' ]]; then
     BINARY_EXEC="\$env:SHELL='powershell'; "
@@ -125,7 +144,5 @@ echo "Cleanup target"
 if [[ ${PLATFORM} == 'windows' ]]; then
     # Todo change for powershell cmdlet
     $SSH "${REMOTE}" "rm -r /workspace"
-else
-    $SSH "${REMOTE}" "sudo rm -rf /workspace"
 fi
 $SSH "${REMOTE}" "rm -r ${EXECUTION_FOLDER}"
\ No newline at end of file
diff --git a/pkg/crc/api/api_client_test.go b/pkg/crc/api/api_client_test.go
new file mode 100644
index 000000000..391eea280
--- /dev/null
+++ b/pkg/crc/api/api_client_test.go
@@ -0,0 +1,314 @@
+package api
+
+import (
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"testing"
+
+	apiClient "github.com/code-ready/crc/pkg/crc/api/client"
+	crcConfig "github.com/code-ready/crc/pkg/crc/config"
+	"github.com/code-ready/crc/pkg/crc/constants"
+	"github.com/code-ready/crc/pkg/crc/machine/fakemachine"
+	"github.com/code-ready/crc/pkg/crc/machine/types"
+	"github.com/code-ready/crc/pkg/crc/version"
+	"github.com/stretchr/testify/assert"
+)
+
+type testClient struct {
+	*apiClient.Client
+	config     crcConfig.Storage
+	httpServer *httptest.Server
+}
+
+func (client *testClient) Close() {
+	client.httpServer.Close()
+}
+
+func newTestClient() *testClient {
+	fakeMachine := fakemachine.NewClient()
+	config := setupNewInMemoryConfig()
+
+	ts := httptest.NewServer(NewMux(config, fakeMachine, &mockLogger{}, &mockTelemetry{}))
+
+	return &testClient{
+		apiClient.New(http.DefaultClient, ts.URL),
+		config,
+		ts,
+	}
+}
+
+func TestVersion(t *testing.T) {
+	client := newTestClient()
+	defer client.Close()
+	vr, err := client.Version()
+	assert.NoError(t, err)
+	assert.Equal(
+		t,
+		apiClient.VersionResult{
+			CrcVersion:       version.GetCRCVersion(),
+			OpenshiftVersion: version.GetBundleVersion(),
+			CommitSha:        version.GetCommitSha(),
+			Success:          true,
+		},
+		vr,
+	)
+}
+
+func TestStatus(t *testing.T) {
+	client := newTestClient()
+	defer client.Close()
+	statusResult, err := client.Status()
+	assert.NoError(t, err)
+	assert.Equal(
+		t,
+		apiClient.ClusterStatusResult{
+			CrcStatus:        "Running",
+			OpenshiftStatus:  "Running",
+			OpenshiftVersion: "4.5.1",
+			DiskUse:          int64(10000000000),
+			DiskSize:         int64(20000000000),
+			Success:          true,
+		},
+		statusResult,
+	)
+}
+
+func TestStart(t *testing.T) {
+	client := newTestClient()
+	defer client.Close()
+	startConfig := apiClient.StartConfig{}
+	startResult, err := client.Start(startConfig)
+	assert.NoError(t, err)
+	assert.Equal(
+		t,
+		apiClient.StartResult{
+			Status:         "",
+			Error:          "",
+			KubeletStarted: true,
+			Success:        true,
+			ClusterConfig: types.ClusterConfig{
+				ClusterCACert: "MIIDODCCAiCgAwIBAgIIRVfCKNUa1wIwDQYJ",
+				KubeConfig:    "/tmp/kubeconfig",
+				KubeAdminPass: "foobar",
+				ClusterAPI:    "https://foo.testing:6443",
+				WebConsoleURL: "https://console.foo.testing:6443",
+				ProxyConfig:   nil,
+			},
+		},
+		startResult,
+	)
+}
+
+func TestSetup(t *testing.T) {
+	client := newTestClient()
+	defer client.Close()
+	stopResult, err := client.Stop()
+	assert.NoError(t, err)
+	assert.Equal(
+		t,
+		apiClient.Result{
+			Success: true,
+			Error:   "",
+		},
+		stopResult,
+	)
+}
+
+func TestDelete(t *testing.T) {
+	client := newTestClient()
+	defer client.Close()
+	deleteResult, err := client.Delete()
+	assert.NoError(t, err)
+	assert.Equal(
+		t,
+		apiClient.Result{
+			Success: true,
+			Error:   "",
+		},
+		deleteResult,
+	)
+}
+
+func TestConfigGet(t *testing.T) {
+	client := newTestClient()
+	defer client.Close()
+	configGetResult, err := client.GetConfig([]string{"cpus"})
+	assert.NoError(t, err)
+	assert.Equal(
+		t,
+		apiClient.GetConfigResult{
+			Success: true,
+			Error:   "",
+			Configs: map[string]interface{}{
+				"cpus": float64(4),
+			},
+		},
+		configGetResult,
+	)
+}
+
+func TestConfigSet(t *testing.T) {
+	client := newTestClient()
+	defer client.Close()
+	configSetResult, err := client.SetConfig(apiClient.SetConfigRequest{
+		Properties: map[string]interface{}{
+			"cpus": float64(5),
+		},
+	})
+	assert.NoError(t, err)
+	assert.Equal(
+		t,
+		apiClient.SetOrUnsetConfigResult{
+			Success:    true,
+			Error:      "",
+			Properties: []string{"cpus"},
+		},
+		configSetResult,
+	)
+	// Get the cpus config again after setting it to make sure it is set
+	// properly.
+	configGetAfterSetResult, err := client.GetConfig([]string{"cpus"})
+	assert.NoError(t, err)
+	assert.Equal(
+		t,
+		apiClient.GetConfigResult{
+			Success: true,
+			Error:   "",
+			Configs: map[string]interface{}{
+				"cpus": float64(5),
+			},
+		},
+		configGetAfterSetResult,
+	)
+}
+
+func TestConfigUnset(t *testing.T) {
+	client := newTestClient()
+	defer client.Close()
+	configUnsetResult, err := client.UnsetConfig([]string{"cpus"})
+	assert.NoError(t, err)
+	assert.Equal(
+		t,
+		apiClient.SetOrUnsetConfigResult{
+			Success:    true,
+			Error:      "",
+			Properties: []string{"cpus"},
+		},
+		configUnsetResult,
+	)
+}
+
+func TestConfigGetAll(t *testing.T) {
+	client := newTestClient()
+	defer client.Close()
+	allConfigGetResult, err := client.GetConfig(nil)
+	assert.NoError(t, err)
+	configs := make(map[string]interface{})
+	for k, v := range client.config.AllConfigs() {
+		// This is required because of https://pkg.go.dev/encoding/json#Unmarshal
+		// Unmarshal stores float64 for JSON numbers in case of interface.
+		switch v := v.Value.(type) {
+		case int:
+			configs[k] = float64(v)
+		default:
+			configs[k] = v
+		}
+	}
+	assert.Equal(
+		t,
+		apiClient.GetConfigResult{
+			Success: true,
+			Error:   "",
+			Configs: configs,
+		},
+		allConfigGetResult,
+	)
+}
+
+func TestConfigGetMultiple(t *testing.T) {
+	client := newTestClient()
+	defer client.Close()
+	// Get result of making query for multiple property
+	configGetMultiplePropertyResult, err := client.GetConfig([]string{"cpus", "memory"})
+	assert.NoError(t, err)
+	assert.Equal(
+		t,
+		apiClient.GetConfigResult{
+			Success: true,
+			Error:   "",
+			Configs: map[string]interface{}{
+				"cpus":   float64(4),
+				"memory": float64(9216),
+			},
+		},
+		configGetMultiplePropertyResult,
+	)
+}
+
+func TestConfigGetEscaped(t *testing.T) {
+	client := newTestClient()
+	defer client.Close()
+	// Get result of special query for multiple properties
+	configGetSpecialPropertyResult, err := client.GetConfig([]string{"a&a", "b&&&b"})
+	assert.NoError(t, err)
+	assert.Equal(
+		t,
+		apiClient.GetConfigResult{
+			Success: true,
+			Error:   "",
+			Configs: map[string]interface{}{
+				"a&a":   "foo",
+				"b&&&b": "bar",
+			},
+		},
+		configGetSpecialPropertyResult,
+	)
+}
+
+func TestTelemetry(t *testing.T) {
+	fakeMachine := fakemachine.NewClient()
+	config := setupNewInMemoryConfig()
+
+	telemetry := &mockTelemetry{}
+	ts := httptest.NewServer(NewMux(config, fakeMachine, &mockLogger{}, telemetry))
+	defer ts.Close()
+
+	client := apiClient.New(http.DefaultClient, ts.URL)
+
+	_ = client.Telemetry("click start")
+	_ = client.Telemetry("click stop")
+
+	assert.Equal(t, []string{"click start", "click stop"}, telemetry.actions)
+}
+
+func TestPullSecret(t *testing.T) {
+	dir, err := ioutil.TempDir("", "test-pull-secret")
+	assert.NoError(t, err)
+	defer os.RemoveAll(dir)
+
+	fakeMachine := fakemachine.NewClient()
+	config := setupNewInMemoryConfig()
+
+	ts := httptest.NewServer(NewMux(config, fakeMachine, &mockLogger{}, &mockTelemetry{}))
+	defer ts.Close()
+
+	client := apiClient.New(http.DefaultClient, ts.URL)
+
+	defined, err := client.IsPullSecretDefined()
+	assert.NoError(t, err)
+	assert.False(t, defined)
+
+	pullSecretFile := filepath.Join(dir, "pull-secret.json")
+	assert.NoError(t, ioutil.WriteFile(pullSecretFile, []byte(constants.OkdPullSecret), 0600))
+	_, err = config.Set(crcConfig.PullSecretFile, pullSecretFile)
+	assert.NoError(t, err)
+
+	defined, err = client.IsPullSecretDefined()
+	assert.NoError(t, err)
+	assert.True(t, defined)
+
+	assert.Error(t, client.SetPullSecret("{}")) // invalid
+}
diff --git a/pkg/crc/api/api_http_test.go b/pkg/crc/api/api_http_test.go
index 391eea280..8493b8f73 100644
--- a/pkg/crc/api/api_http_test.go
+++ b/pkg/crc/api/api_http_test.go
@@ -1,314 +1,507 @@
 package api
 
 import (
+	"fmt"
+	"io"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
+	"net/http/httputil"
 	"os"
-	"path/filepath"
+	"strings"
 	"testing"
 
-	apiClient "github.com/code-ready/crc/pkg/crc/api/client"
 	crcConfig "github.com/code-ready/crc/pkg/crc/config"
 	"github.com/code-ready/crc/pkg/crc/constants"
 	"github.com/code-ready/crc/pkg/crc/machine/fakemachine"
-	"github.com/code-ready/crc/pkg/crc/machine/types"
 	"github.com/code-ready/crc/pkg/crc/version"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
-type testClient struct {
-	*apiClient.Client
-	config     crcConfig.Storage
-	httpServer *httptest.Server
+type mockServer struct {
+	*server
+	client *fakemachine.Client
+	config crcConfig.Storage
 }
 
-func (client *testClient) Close() {
-	client.httpServer.Close()
+func createDummyPullSecret(t *testing.T) string {
+	f, err := ioutil.TempFile("", "kubeconfig")
+	assert.NoError(t, err)
+	_, err = f.WriteString(constants.OkdPullSecret)
+	assert.NoError(t, err)
+	err = f.Close()
+	assert.NoError(t, err)
+
+	return f.Name()
 }
 
-func newTestClient() *testClient {
+func removePullSecret(t *testing.T, server *mockServer) {
+	_, err := server.config.Unset(crcConfig.PullSecretFile)
+	assert.NoError(t, err)
+}
+
+func newMockServer(pullSecretPath string) *mockServer {
 	fakeMachine := fakemachine.NewClient()
+
 	config := setupNewInMemoryConfig()
+	_, _ = config.Set(crcConfig.PullSecretFile, pullSecretPath)
 
-	ts := httptest.NewServer(NewMux(config, fakeMachine, &mockLogger{}, &mockTelemetry{}))
+	handler := NewHandler(config, fakeMachine, &mockLogger{}, &mockTelemetry{})
 
-	return &testClient{
-		apiClient.New(http.DefaultClient, ts.URL),
-		config,
-		ts,
+	return &mockServer{
+		server: newServerWithRoutes(handler),
+		client: fakeMachine,
+		config: config,
 	}
 }
 
-func TestVersion(t *testing.T) {
-	client := newTestClient()
-	defer client.Close()
-	vr, err := client.Version()
-	assert.NoError(t, err)
-	assert.Equal(
-		t,
-		apiClient.VersionResult{
-			CrcVersion:       version.GetCRCVersion(),
-			OpenshiftVersion: version.GetBundleVersion(),
-			CommitSha:        version.GetCommitSha(),
-			Success:          true,
-		},
-		vr,
-	)
+func sendRequest(handler http.Handler, request *request) *http.Response {
+	url := fmt.Sprintf("/%s", request.resource)
+	var data io.Reader
+	if request.data != "" {
+		data = strings.NewReader(request.data)
+	} else {
+		data = nil
+	}
+
+	req := httptest.NewRequest(request.httpMethod, url, data)
+	req.Header.Set("Content-Type", "application/json")
+	{
+		requestDump, _ := httputil.DumpRequest(req, true)
+		fmt.Println(string(requestDump))
+	}
+	w := httptest.NewRecorder()
+	handler.ServeHTTP(w, req)
+
+	response := w.Result()
+	{
+		responseDump, _ := httputil.DumpResponse(response, true)
+		fmt.Println(string(responseDump))
+	}
+	return response
 }
 
-func TestStatus(t *testing.T) {
-	client := newTestClient()
-	defer client.Close()
-	statusResult, err := client.Status()
-	assert.NoError(t, err)
-	assert.Equal(
-		t,
-		apiClient.ClusterStatusResult{
-			CrcStatus:        "Running",
-			OpenshiftStatus:  "Running",
-			OpenshiftVersion: "4.5.1",
-			DiskUse:          int64(10000000000),
-			DiskSize:         int64(20000000000),
-			Success:          true,
-		},
-		statusResult,
-	)
+type request struct {
+	httpMethod string
+	resource   string
+	data       string
 }
 
-func TestStart(t *testing.T) {
-	client := newTestClient()
-	defer client.Close()
-	startConfig := apiClient.StartConfig{}
-	startResult, err := client.Start(startConfig)
-	assert.NoError(t, err)
-	assert.Equal(
-		t,
-		apiClient.StartResult{
-			Status:         "",
-			Error:          "",
-			KubeletStarted: true,
-			Success:        true,
-			ClusterConfig: types.ClusterConfig{
-				ClusterCACert: "MIIDODCCAiCgAwIBAgIIRVfCKNUa1wIwDQYJ",
-				KubeConfig:    "/tmp/kubeconfig",
-				KubeAdminPass: "foobar",
-				ClusterAPI:    "https://foo.testing:6443",
-				WebConsoleURL: "https://console.foo.testing:6443",
-				ProxyConfig:   nil,
-			},
-		},
-		startResult,
-	)
+type response struct {
+	statusCode int
+	protoMajor int
+	protoMinor int
+	// headers
+	body string
 }
 
-func TestSetup(t *testing.T) {
-	client := newTestClient()
-	defer client.Close()
-	stopResult, err := client.Stop()
-	assert.NoError(t, err)
-	assert.Equal(
-		t,
-		apiClient.Result{
-			Success: true,
-			Error:   "",
-		},
-		stopResult,
-	)
+type testCase struct {
+	preTestFunc func(t *testing.T, server *mockServer)
+	request     request
+	failRequest bool
+	response    response
 }
 
-func TestDelete(t *testing.T) {
-	client := newTestClient()
-	defer client.Close()
-	deleteResult, err := client.Delete()
-	assert.NoError(t, err)
-	assert.Equal(
-		t,
-		apiClient.Result{
-			Success: true,
-			Error:   "",
-		},
-		deleteResult,
-	)
+func get(resource string) request {
+	return request{
+		httpMethod: http.MethodGet,
+		resource:   resource,
+	}
 }
 
-func TestConfigGet(t *testing.T) {
-	client := newTestClient()
-	defer client.Close()
-	configGetResult, err := client.GetConfig([]string{"cpus"})
-	assert.NoError(t, err)
-	assert.Equal(
-		t,
-		apiClient.GetConfigResult{
-			Success: true,
-			Error:   "",
-			Configs: map[string]interface{}{
-				"cpus": float64(4),
-			},
-		},
-		configGetResult,
-	)
+func post(resource string) request {
+	return request{
+		httpMethod: http.MethodPost,
+		resource:   resource,
+	}
 }
 
-func TestConfigSet(t *testing.T) {
-	client := newTestClient()
-	defer client.Close()
-	configSetResult, err := client.SetConfig(apiClient.SetConfigRequest{
-		Properties: map[string]interface{}{
-			"cpus": float64(5),
-		},
-	})
-	assert.NoError(t, err)
-	assert.Equal(
-		t,
-		apiClient.SetOrUnsetConfigResult{
-			Success:    true,
-			Error:      "",
-			Properties: []string{"cpus"},
-		},
-		configSetResult,
-	)
-	// Get the cpus config again after setting it to make sure it is set
-	// properly.
-	configGetAfterSetResult, err := client.GetConfig([]string{"cpus"})
-	assert.NoError(t, err)
-	assert.Equal(
-		t,
-		apiClient.GetConfigResult{
-			Success: true,
-			Error:   "",
-			Configs: map[string]interface{}{
-				"cpus": float64(5),
-			},
-		},
-		configGetAfterSetResult,
-	)
+func delete(resource string) request {
+	return request{
+		httpMethod: http.MethodDelete,
+		resource:   resource,
+	}
 }
 
-func TestConfigUnset(t *testing.T) {
-	client := newTestClient()
-	defer client.Close()
-	configUnsetResult, err := client.UnsetConfig([]string{"cpus"})
-	assert.NoError(t, err)
-	assert.Equal(
-		t,
-		apiClient.SetOrUnsetConfigResult{
-			Success:    true,
-			Error:      "",
-			Properties: []string{"cpus"},
-		},
-		configUnsetResult,
-	)
+func (req request) String() string {
+	return fmt.Sprintf("%s /%s HTTP/1.1", req.httpMethod, req.resource)
 }
 
-func TestConfigGetAll(t *testing.T) {
-	client := newTestClient()
-	defer client.Close()
-	allConfigGetResult, err := client.GetConfig(nil)
-	assert.NoError(t, err)
-	configs := make(map[string]interface{})
-	for k, v := range client.config.AllConfigs() {
-		// This is required because of https://pkg.go.dev/encoding/json#Unmarshal
-		// Unmarshal stores float64 for JSON numbers in case of interface.
-		switch v := v.Value.(type) {
-		case int:
-			configs[k] = float64(v)
-		default:
-			configs[k] = v
-		}
+func (req request) withBody(data string) request {
+	req.data = data
+	return req
+}
+
+func jSon(data string) response {
+	return response{
+		statusCode: 200,
+		protoMajor: 1,
+		protoMinor: 1,
+		body:       data,
 	}
-	assert.Equal(
-		t,
-		apiClient.GetConfigResult{
-			Success: true,
-			Error:   "",
-			Configs: configs,
-		},
-		allConfigGetResult,
-	)
 }
 
-func TestConfigGetMultiple(t *testing.T) {
-	client := newTestClient()
-	defer client.Close()
-	// Get result of making query for multiple property
-	configGetMultiplePropertyResult, err := client.GetConfig([]string{"cpus", "memory"})
-	assert.NoError(t, err)
-	assert.Equal(
-		t,
-		apiClient.GetConfigResult{
-			Success: true,
-			Error:   "",
-			Configs: map[string]interface{}{
-				"cpus":   float64(4),
-				"memory": float64(9216),
-			},
-		},
-		configGetMultiplePropertyResult,
-	)
+func empty() response {
+	return response{
+		statusCode: 200,
+		protoMajor: 1,
+		protoMinor: 1,
+		body:       `{"Success":true,"Error":""}`,
+	}
 }
 
-func TestConfigGetEscaped(t *testing.T) {
-	client := newTestClient()
-	defer client.Close()
-	// Get result of special query for multiple properties
-	configGetSpecialPropertyResult, err := client.GetConfig([]string{"a&a", "b&&&b"})
-	assert.NoError(t, err)
-	assert.Equal(
-		t,
-		apiClient.GetConfigResult{
-			Success: true,
-			Error:   "",
-			Configs: map[string]interface{}{
-				"a&a":   "foo",
-				"b&&&b": "bar",
-			},
-		},
-		configGetSpecialPropertyResult,
-	)
+func httpError(statusCode int) response {
+	return response{
+		statusCode: statusCode,
+		protoMajor: 1,
+		protoMinor: 1,
+	}
 }
 
-func TestTelemetry(t *testing.T) {
-	fakeMachine := fakemachine.NewClient()
-	config := setupNewInMemoryConfig()
+func (resp response) withBody(body string) response {
+	resp.body = body
+	return resp
+}
+
+var testCases = []testCase{
+	// start
+	{
+		request:  post("start"),
+		response: jSon(`{"Success":true,"Status":"","Error":"","ClusterConfig":{"ClusterCACert":"MIIDODCCAiCgAwIBAgIIRVfCKNUa1wIwDQYJ","KubeConfig":"/tmp/kubeconfig","KubeAdminPass":"foobar","ClusterAPI":"https://foo.testing:6443","WebConsoleURL":"https://console.foo.testing:6443","ProxyConfig":null},"KubeletStarted":true}`),
+	},
+	{
+		request:  get("start"),
+		response: jSon(`{"Success":true,"Status":"","Error":"","ClusterConfig":{"ClusterCACert":"MIIDODCCAiCgAwIBAgIIRVfCKNUa1wIwDQYJ","KubeConfig":"/tmp/kubeconfig","KubeAdminPass":"foobar","ClusterAPI":"https://foo.testing:6443","WebConsoleURL":"https://console.foo.testing:6443","ProxyConfig":null},"KubeletStarted":true}`),
+	},
+
+	// start with failure
+	{
+		request:     post("start"),
+		failRequest: true,
+		response:    httpError(500).withBody("Failed to start\n"),
+	},
+	{
+		request:     get("start"),
+		failRequest: true,
+		response:    httpError(500).withBody("Failed to start\n"),
+	},
+
+	// stop
+	{
+		request:  post("stop"),
+		response: empty(),
+	},
+	{
+		request:  get("stop"),
+		response: empty(),
+	},
+
+	// stop with failure
+	{
+		request:     post("stop"),
+		failRequest: true,
+		// error message comes from fakemachine
+		response: httpError(500).withBody("stop failed\n"),
+	},
+	{
+		request:     get("stop"),
+		failRequest: true,
+		// error message comes from fakemachine
+		response: httpError(500).withBody("stop failed\n"),
+	},
+
+	// poweroff
+	{
+		request:  post("poweroff"),
+		response: empty(),
+	},
 
-	telemetry := &mockTelemetry{}
-	ts := httptest.NewServer(NewMux(config, fakeMachine, &mockLogger{}, telemetry))
-	defer ts.Close()
+	// poweroff with failure
+	{
+		request:     post("poweroff"),
+		failRequest: true,
+		// error message comes from fakemachine
+		response: httpError(500).withBody("poweroff failed\n"),
+	},
 
-	client := apiClient.New(http.DefaultClient, ts.URL)
+	// status
+	{
+		request:  get("status"),
+		response: jSon(`{"CrcStatus":"Running","OpenshiftStatus":"Running","OpenshiftVersion":"4.5.1","DiskUse":10000000000,"DiskSize":20000000000,"Error":"","Success":true}`),
+	},
 
-	_ = client.Telemetry("click start")
-	_ = client.Telemetry("click stop")
+	// status with failure
+	{
+		request:     get("status"),
+		failRequest: true,
+		// error message comes from fakemachine
+		response: httpError(500).withBody("broken\n"),
+	},
 
-	assert.Equal(t, []string{"click start", "click stop"}, telemetry.actions)
+	// delete
+	{
+		request:  delete("delete"),
+		response: empty(),
+	},
+	{
+		request:  get("delete"),
+		response: empty(),
+	},
+
+	// delete with failure
+	{
+		request:     delete("delete"),
+		failRequest: true,
+		// error message comes from fakemachine
+		response: httpError(500).withBody("delete failed\n"),
+	},
+	{
+		request:     get("delete"),
+		failRequest: true,
+		// error message comes from fakemachine
+		response: httpError(500).withBody("delete failed\n"),
+	},
+
+	// version
+	{
+		request:  get("version"),
+		response: jSon(fmt.Sprintf(`{"CrcVersion":"%s","CommitSha":"%s","OpenshiftVersion":"%s","Success":true}`, version.GetCRCVersion(), version.GetCommitSha(), version.GetBundleVersion())),
+	},
+
+	// version never fails
+
+	// webconsoleurl
+	{
+		request:  get("webconsoleurl"),
+		response: jSon(`{"ClusterConfig":{"ClusterCACert":"MIIDODCCAiCgAwIBAgIIRVfCKNUa1wIwDQYJ","KubeConfig":"/tmp/kubeconfig","KubeAdminPass":"foobar","ClusterAPI":"https://foo.testing:6443","WebConsoleURL":"https://console.foo.testing:6443","ProxyConfig":null},"Success":true,"Error":""}`),
+	},
+
+	// webconsoleurl with failure
+	{
+		request:     get("webconsoleurl"),
+		failRequest: true,
+		// error message comes from fakemachine
+		response: httpError(500).withBody("console failed\n"),
+	},
+
+	// config
+	{
+		request:  get("config?cpus"),
+		response: jSon(`{"Success":true,"Error":"","Configs":{"cpus":4}}`),
+	},
+	{
+		request:  post("config?cpus").withBody("xx"),
+		response: httpError(500).withBody("invalid character 'x' looking for beginning of value\n"),
+	},
+	{
+		request:  delete("config?cpus"),
+		response: httpError(500).withBody("unexpected end of JSON input\n"),
+	},
+	{
+		request:  get("config?cpus").withBody("xx"),
+		response: jSon(`{"Success":true,"Error":"","Configs":{"cpus":4}}`),
+	},
+
+	// logs
+	{
+		request:  get("logs"),
+		response: jSon(`{"Success":true,"Messages":["message 1","message 2","message 3"]}`),
+	},
+
+	// logs never fails
+
+	// telemetry
+	{
+		request:  get("telemetry"),
+		response: httpError(500).withBody("unexpected end of JSON input\n"),
+	},
+	{
+		request:  post("telemetry"),
+		response: httpError(500).withBody("unexpected end of JSON input\n"),
+	},
+
+	// telemetry with failure
+	{
+		request:     get("telemetry"),
+		failRequest: true,
+		response:    httpError(500).withBody("unexpected end of JSON input\n"),
+	},
+	{
+		request:     post("telemetry"),
+		failRequest: true,
+		response:    httpError(500).withBody("unexpected end of JSON input\n"),
+	},
+
+	// pull-secret
+	{
+		request: get("pull-secret"),
+		// other 404 return "not found", and others "404 not found"
+		response: empty().withBody(""),
+	},
+	{
+		request:  post("pull-secret"),
+		response: httpError(500).withBody("empty pull secret\n"),
+	},
+
+	// pull-secret with failure
+	{
+		preTestFunc: removePullSecret,
+		request:     get("pull-secret"),
+		failRequest: true,
+		// other 404 return "not found", and others "404 not found"
+		response: httpError(404),
+	},
+	{
+		request:     post("pull-secret"),
+		failRequest: true,
+		response:    httpError(500).withBody("empty pull secret\n"),
+	},
+
+	// not found
+	{
+		request:  get("notfound"),
+		response: httpError(404).withBody("Not Found\n"),
+	},
+
+	// config
+	{
+		request:  get("config?cpus"),
+		response: jSon(`{"Success":true,"Error":"","Configs":{"cpus":4}}`),
+	},
 }
 
-func TestPullSecret(t *testing.T) {
-	dir, err := ioutil.TempDir("", "test-pull-secret")
-	assert.NoError(t, err)
-	defer os.RemoveAll(dir)
+var invalidHTTPMethods = []testCase{
+	// start
+	{
+		request:  delete("start"),
+		response: httpError(404).withBody("Not Found\n"),
+	},
 
-	fakeMachine := fakemachine.NewClient()
-	config := setupNewInMemoryConfig()
+	// stop
+	{
+		request:  delete("stop"),
+		response: httpError(404).withBody("Not Found\n"),
+	},
 
-	ts := httptest.NewServer(NewMux(config, fakeMachine, &mockLogger{}, &mockTelemetry{}))
-	defer ts.Close()
+	// poweroff
+	{
+		request:  get("poweroff"),
+		response: httpError(404).withBody("Not Found\n"),
+	},
+	{
+		request:  delete("poweroff"),
+		response: httpError(404).withBody("Not Found\n"),
+	},
 
-	client := apiClient.New(http.DefaultClient, ts.URL)
+	// status
+	{
+		request:  post("status"),
+		response: httpError(404).withBody("Not Found\n"),
+	},
+	{
+		request:  delete("status"),
+		response: httpError(404).withBody("Not Found\n"),
+	},
 
-	defined, err := client.IsPullSecretDefined()
-	assert.NoError(t, err)
-	assert.False(t, defined)
+	// delete
+	{
+		request:  post("delete"),
+		response: httpError(404).withBody("Not Found\n"),
+	},
 
-	pullSecretFile := filepath.Join(dir, "pull-secret.json")
-	assert.NoError(t, ioutil.WriteFile(pullSecretFile, []byte(constants.OkdPullSecret), 0600))
-	_, err = config.Set(crcConfig.PullSecretFile, pullSecretFile)
-	assert.NoError(t, err)
+	// version
+	{
+		request:  post("version"),
+		response: httpError(404).withBody("Not Found\n"),
+	},
+	{
+		request:  delete("version"),
+		response: httpError(404).withBody("Not Found\n"),
+	},
 
-	defined, err = client.IsPullSecretDefined()
-	assert.NoError(t, err)
-	assert.True(t, defined)
+	// webconsoleurl
+	{
+		request:  post("webconsoleurl"),
+		response: httpError(404).withBody("Not Found\n"),
+	},
+	{
+		request:  delete("webconsoleurl"),
+		response: httpError(404).withBody("Not Found\n"),
+	},
 
-	assert.Error(t, client.SetPullSecret("{}")) // invalid
+	// logs
+	{
+		request:  post("logs"),
+		response: httpError(404).withBody("Not Found\n"),
+	},
+	{
+		request:  delete("logs"),
+		response: httpError(404).withBody("Not Found\n"),
+	},
+
+	// telemetry
+	{
+		request:  delete("telemetry"),
+		response: httpError(404).withBody("Not Found\n"),
+	},
+
+	// pull-secret
+	{
+		request: delete("pull-secret"),
+		// other 404 return "not found", and others "404 not found"
+		response: httpError(404).withBody("Not Found\n"),
+	},
+}
+
+func testOne(t *testing.T, testCase *testCase, server *mockServer) {
+	server.client.Failing = testCase.failRequest
+	if testCase.preTestFunc != nil {
+		testCase.preTestFunc(t, server)
+	}
+	resp := sendRequest(server.Handler(), &testCase.request)
+
+	require.Equal(t, testCase.response.statusCode, resp.StatusCode, testCase.request)
+	require.Equal(t, testCase.response.protoMajor, resp.ProtoMajor, testCase.request)
+	require.Equal(t, testCase.response.protoMinor, resp.ProtoMinor, testCase.request)
+	body, err := ioutil.ReadAll(resp.Body)
+	require.NoError(t, err, testCase.request)
+	require.Equal(t, testCase.response.body, string(body), testCase.request)
+	fmt.Println("-----")
+}
+
+func TestRequests(t *testing.T) {
+	pullSecretPath := createDummyPullSecret(t)
+	defer os.Remove(pullSecretPath)
+	server := newMockServer(pullSecretPath)
+
+	for i := range testCases {
+		testOne(t, &testCases[i], server)
+	}
+
+	for i := range invalidHTTPMethods {
+		testOne(t, &testCases[i], server)
+	}
+}
+
+func TestRoutes(t *testing.T) {
+	// this checks that we have test cases for all routes registered with the `api` entrypoint
+
+	var routes = map[string][]string{}
+	for _, testCase := range testCases {
+		// Add leading '/', remove trailing '?....'
+		pattern := fmt.Sprintf("/%s", strings.SplitN(testCase.request.resource, "?", 2)[0])
+		if _, ok := routes[pattern]; !ok {
+			routes[pattern] = []string{}
+		}
+		routes[pattern] = append(routes[pattern], testCase.request.httpMethod)
+	}
+
+	server := newMockServer("")
+	for pattern, methodMap := range server.routes {
+		assert.Contains(t, routes, pattern)
+		for method := range methodMap {
+			assert.Contains(t, routes[pattern], method, "routes[%s][%s] is missing from the API testcases", pattern, method)
+		}
+	}
 }
diff --git a/pkg/crc/errors/multierror_test.go b/pkg/crc/errors/multierror_test.go
index b051ad423..7710efb32 100644
--- a/pkg/crc/errors/multierror_test.go
+++ b/pkg/crc/errors/multierror_test.go
@@ -1,6 +1,7 @@
 package errors
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"testing"
@@ -9,9 +10,9 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-func TestRetryAfter(t *testing.T) {
+func TestRetryAfterWithContext(t *testing.T) {
 	calls := 0
-	ret := RetryAfter(time.Second, func() error {
+	ret := Retry(context.Background(), time.Second, func() error {
 		calls++
 		return nil
 	}, 0)
@@ -21,7 +22,7 @@ func TestRetryAfter(t *testing.T) {
 
 func TestRetryAfterFailure(t *testing.T) {
 	calls := 0
-	ret := RetryAfter(time.Second, func() error {
+	ret := Retry(context.Background(), time.Second, func() error {
 		calls++
 		return errors.New("failed")
 	}, 0)
@@ -31,7 +32,7 @@ func TestRetryAfterFailure(t *testing.T) {
 
 func TestRetryAfterSlowFailure(t *testing.T) {
 	calls := 0
-	ret := RetryAfter(time.Millisecond, func() error {
+	ret := Retry(context.Background(), time.Millisecond, func() error {
 		time.Sleep(50 * time.Millisecond)
 		calls++
 		if calls < 2 {
@@ -45,7 +46,7 @@ func TestRetryAfterSlowFailure(t *testing.T) {
 
 func TestRetryAfterMaxAttempts(t *testing.T) {
 	calls := 0
-	ret := RetryAfter(10*time.Millisecond, func() error {
+	ret := Retry(context.Background(), 10*time.Millisecond, func() error {
 		calls++
 		return &RetriableError{Err: errors.New("failed")}
 	}, 0)
@@ -55,7 +56,7 @@ func TestRetryAfterMaxAttempts(t *testing.T) {
 
 func TestRetryAfterSuccessAfterFailures(t *testing.T) {
 	calls := 0
-	ret := RetryAfter(time.Second, func() error {
+	ret := Retry(context.Background(), time.Second, func() error {
 		calls++
 		if calls < 3 {
 			return &RetriableError{Err: errors.New("failed")}