Repository: hetznercloud/hcloud-cloud-controller-manager
Pull request #97 (2020-10-19 05:44:58)

Implement e2e tests


Patch:
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c77fab1e..7cbcdaa9 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -8,6 +8,7 @@ include:
 
 stages:
   - test
+  - E2E
   - build
   - release
 
@@ -33,9 +34,51 @@ test:unit:
   variables:
     NODE_NAME: "test"
   script:
-    - GO_PACKAGES=$(command go list ./... | tr "\n" ",")
-    - go test -race -covermode=atomic -coverprofile=.coverage.out ./... 2> /dev/null
-    - go tool cover -func=.coverage.out | tail -n1
-  coverage: '/total:[^\d]+(\d+\.\d+%)/'
+    - go test $(go list ./... | grep -v e2etests) -v
   tags:
     - hc-bladerunner
+
+.e2e:k8s: &testk8se2e
+  stage: E2E
+  image: docker:git
+  variables:
+    K8S_VERSION: 1.17.12
+  before_script:
+    - apk add --no-cache git make musl-dev go openssh-client
+  script:
+    - go test $(go list ./... | grep e2etests) -v -timeout 60m
+  tags:
+    - hc-bladerunner-build
+
+k8s-1.17:
+  <<: *testk8se2e
+  variables:
+    K8S_VERSION: 1.17.12
+
+k8s-1.17 networks:
+  <<: *testk8se2e
+  variables:
+    K8S_VERSION: 1.17.12
+    USE_NETWORKS: "yes"
+
+k8s-1.18:
+  <<: *testk8se2e
+  variables:
+    K8S_VERSION: 1.18.9
+
+k8s-1.18 networks:
+  <<: *testk8se2e
+  variables:
+    K8S_VERSION: 1.18.9
+    USE_NETWORKS: "yes"
+
+k8s-1.19:
+  <<: *testk8se2e
+  variables:
+    K8S_VERSION: 1.19.3
+
+k8s-1.19 networks:
+  <<: *testk8se2e
+  variables:
+    K8S_VERSION: 1.19.3
+    USE_NETWORKS: "yes"
diff --git a/README.md b/README.md
index fbed41a0..ca2a5071 100644
--- a/README.md
+++ b/README.md
@@ -110,6 +110,42 @@ kubectl apply -f  https://raw.githubusercontent.com/hetznercloud/hcloud-cloud-co
 If you want to use the Hetzner Cloud `Networks` Feature, head over to the [Deployment with Networks support documentation](./docs/deploy_with_networks.md).
 
 
+## E2E Tests
+
+The Hetzner Cloud cloud controller manager was tested against all supported Kubernetes versions. You can run the tests with the following commands. Keep in mind, that these tests run on real cloud servers and will create Load Balancers that will be billed. 
+
+**Test Server Setup:** 
+1x CPX21 (Ubuntu 18.04)
+
+**Requirements: Docker and Go 1.15**
+
+1. Configure your environment correctly
+
+```bash
+export HCLOUD_TOKEN=<specifiy a project token>
+export K8S_VERSION=1.19.3 # The specific (latest) version is needed here
+export USE_SSH_KEYS=key1,key2 # Name or IDs of your SSH Keys within the Hetzner Cloud, the servers will be accessable with that keys
+export USE_NETWORKS=yes # if `yes` this identidicates that the tests should provision the server with cilium as CNI and also enable the Network related tests
+## Optional configuration env vars:
+export TEST_DEBUG_MODE=yes # With this env you can toggle the output of the provision and test commands. With `yes` it will log the whole output to stdout
+```
+
+2. Run the tests
+
+```bash
+go test $(go list ./... | grep e2etests) -v -timeout 60m
+```
+The tests will now run and cleanup themselves afterwards. Sometimes it might happen that you need to clean up the project manually via the [Hetzner Cloud Console](https://console.hetzner.cloud) or the [hcloud-cli](https://github.com/hetznercloud/cli) .
+
+For easier debugging on the server we always configure the latest version of the [hcloud-cli](https://github.com/hetznercloud/cli) with the given `HCLOUD_TOKEN` and a few bash aliases on the host:
+
+```bash
+alias k="kubectl"
+alias ksy="kubectl -n kube-system"
+alias kgp="kubectl get pods"
+alias kgs="kubectl get services"
+```
+
 ## License
 
 Apache License, Version 2.0
diff --git a/deploy/dev-ccm-networks.yaml b/deploy/dev-ccm-networks.yaml
index a15be18d..06fec86b 100644
--- a/deploy/dev-ccm-networks.yaml
+++ b/deploy/dev-ccm-networks.yaml
@@ -35,7 +35,7 @@ spec:
       labels:
         app: hcloud-cloud-controller-manager
       annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
+        scheduler.alpha.kubernetes.io/critical-pod: ""
     spec:
       serviceAccountName: cloud-controller-manager
       dnsPolicy: Default
@@ -81,4 +81,4 @@ spec:
               valueFrom:
                 secretKeyRef:
                   name: hcloud
-                  key: network
\ No newline at end of file
+                  key: network
diff --git a/deploy/dev-ccm.yaml b/deploy/dev-ccm.yaml
index 3ecac745..fa6d9b12 100644
--- a/deploy/dev-ccm.yaml
+++ b/deploy/dev-ccm.yaml
@@ -36,7 +36,7 @@ spec:
       labels:
         app: hcloud-cloud-controller-manager
       annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
+        scheduler.alpha.kubernetes.io/critical-pod: ""
     spec:
       serviceAccountName: cloud-controller-manager
       dnsPolicy: Default
diff --git a/go.mod b/go.mod
index d43164cc..b6bb786b 100644
--- a/go.mod
+++ b/go.mod
@@ -9,9 +9,11 @@ require (
 	github.com/hetznercloud/hcloud-go v1.22.0
 	github.com/stretchr/testify v1.6.1
 	github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect
+	golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975
 	gopkg.in/square/go-jose.v2 v2.3.0 // indirect
 	k8s.io/api v0.18.8
 	k8s.io/apimachinery v0.18.8
+	k8s.io/client-go v0.18.8
 	k8s.io/cloud-provider v0.18.8
 	k8s.io/component-base v0.18.8
 	k8s.io/klog/v2 v2.1.0
diff --git a/scripts/delete-token.sh b/scripts/delete-token.sh
new file mode 100755
index 00000000..a3459e12
--- /dev/null
+++ b/scripts/delete-token.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+set -e
+token=$1
+curl -A 'travis-terraform-provider' --header 'Authorization: Bearer '"$TTS_TOKEN"'' -X DELETE https://tt-service.hetzner.cloud/token?token=''"$token"''
diff --git a/scripts/get-token.sh b/scripts/get-token.sh
new file mode 100755
index 00000000..8097c7e1
--- /dev/null
+++ b/scripts/get-token.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -e
+resp=$(curl -A 'travis-terraform-provider' --header 'Authorization: Bearer '"$TTS_TOKEN"'' -X POST https://tt-service.hetzner.cloud/token -o resp.json)
+if grep -q token "resp.json"
+then
+    token=$(cat resp.json | jq -r '.token')
+    echo $token
+else
+    exit 1
+fi


Test patch:
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index cc6012a9..4a1eedc6 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -12,4 +12,4 @@ jobs:
       - name: Run tests
         run: |
           go vet ./...
-          go test ./...
+          go test $(go list ./... | grep -v e2etests)
diff --git a/.github/workflows/test_e2e.yml b/.github/workflows/test_e2e.yml
new file mode 100644
index 00000000..3b541a57
--- /dev/null
+++ b/.github/workflows/test_e2e.yml
@@ -0,0 +1,25 @@
+name: Run e2e tests
+on: [push]
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        k8s: [ 1.17.13, 1.18.10, 1.19.3]
+        networks: [yes, no]
+    name: ${{ matrix.k8s }}- networks - ${{ matrix.networks }}
+    steps:
+    - uses: actions/setup-go@v2
+      with:
+        go-version: '1.15'
+    - uses: actions/checkout@master
+    - name: Run tests
+      env:
+        K8S_VERSION: ${{ matrix.k8s }}
+        TTS_TOKEN: ${{ secrets.TTS_TOKEN }}
+        USE_NETWORKS: ${{ matrix.networks }}
+      run: |
+        export HCLOUD_TOKEN=$(./scripts/get-token.sh)
+        cat resp.json
+        go test $(go list ./... | grep e2etests) -v -timeout 60m
+        ./scripts/delete-token.sh $HCLOUD_TOKEN
diff --git a/e2etests/.gitignore b/e2etests/.gitignore
new file mode 100644
index 00000000..4456063e
--- /dev/null
+++ b/e2etests/.gitignore
@@ -0,0 +1,3 @@
+ci-hcloud-ccm.tar
+ssh_key
+kubeconfig
diff --git a/e2etests/e2e_test.go b/e2etests/e2e_test.go
new file mode 100644
index 00000000..400d72e5
--- /dev/null
+++ b/e2etests/e2e_test.go
@@ -0,0 +1,133 @@
+package e2etests
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/hetznercloud/hcloud-cloud-controller-manager/internal/annotation"
+	"github.com/stretchr/testify/assert"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+var testCluster TestCluster
+
+func TestMain(m *testing.M) {
+	if err := testCluster.Start(); err != nil {
+		fmt.Printf("%v\n", err)
+		os.Exit(1)
+	}
+
+	rc := m.Run()
+
+	if err := testCluster.Stop(); err != nil {
+		fmt.Printf("%v\n", err)
+		os.Exit(1)
+	}
+	os.Exit(rc)
+}
+
+func TestCloudControllerManagerPodIsPresent(t *testing.T) {
+	t.Run("hcloud-cloud-controller-manager pod is present in kube-system", func(t *testing.T) {
+		pods, err := testCluster.k8sClient.CoreV1().Pods("kube-system").List(context.Background(), metav1.ListOptions{})
+		assert.NoError(t, err)
+
+		found := false
+		for _, pod := range pods.Items {
+			if strings.Contains(pod.Name, "hcloud-cloud-controller-manager") {
+				found = true
+				break
+			}
+		}
+		if !found {
+			t.Error("kube-system does not contain a pod named hcloud-cloud-controller-manager")
+		}
+	})
+
+	t.Run("pod with app=hcloud-cloud-controller-manager is present in kube-system", func(t *testing.T) {
+		pods, err := testCluster.k8sClient.CoreV1().Pods("kube-system").List(context.Background(), metav1.ListOptions{LabelSelector: "app=hcloud-cloud-controller-manager"})
+		assert.NoError(t, err)
+
+		if len(pods.Items) == 0 {
+			t.Fatal("kube-system does not contain a pod with label app=hcloud-cloud-controller-manager")
+		}
+	})
+}
+
+func TestCloudControllerManagerSetCorrectNodeLabels(t *testing.T) {
+	node, err := testCluster.k8sClient.CoreV1().Nodes().Get(context.Background(), testCluster.setup.server.Name, metav1.GetOptions{})
+	assert.NoError(t, err)
+
+	labels := node.Labels
+	expectedLabels := map[string]string{
+		"node.kubernetes.io/instance-type": testCluster.setup.server.ServerType.Name,
+		"topology.kubernetes.io/region":    testCluster.setup.server.Datacenter.Location.Name,
+		"topology.kubernetes.io/zone":      testCluster.setup.server.Datacenter.Name,
+	}
+	for expectedLabel, expectedValue := range expectedLabels {
+		if labelValue, ok := labels[expectedLabel]; !ok || labelValue != expectedValue {
+			t.Errorf("node have a not expected label %s, ok: %v, given value %s, expected value %s", expectedLabel, ok, labelValue, expectedValue)
+		}
+	}
+}
+
+func TestCloudControllerManagerLoadBalancersMinimalSetup(t *testing.T) {
+	lbTest := lbTestHelper{t: t, K8sClient: testCluster.k8sClient, podName: "loadbalancer-minimal"}
+
+	pod := lbTest.DeployTestPod()
+
+	lbSvc := lbTest.ServiceDefinition(pod, map[string]string{
+		string(annotation.LBLocation): "nbg1",
+	})
+
+	lbSvc, err := lbTest.CreateService(lbSvc)
+	if err != nil {
+		t.Fatalf("deploying test svc: %s", err)
+	}
+
+	ingressIP := lbSvc.Status.LoadBalancer.Ingress[0].IP // Index 0 is always the public IP of the LB
+	lbTest.WaitForHttpAvailable(ingressIP)
+
+	lbTest.TearDown()
+}
+
+func TestCloudControllerManagerLoadBalancersWithPrivateNetwork(t *testing.T) {
+	if testCluster.useNetworks == false {
+		t.Skipf("Private Networks test is disabled")
+	}
+
+	lbTest := lbTestHelper{t: t, K8sClient: testCluster.k8sClient, podName: "loadbalancer-private-network"}
+
+	pod := lbTest.DeployTestPod()
+
+	lbSvcDefinition := lbTest.ServiceDefinition(pod, map[string]string{
+		string(annotation.LBLocation):     "nbg1",
+		string(annotation.LBUsePrivateIP): "true",
+	})
+
+	lbSvc, err := lbTest.CreateService(lbSvcDefinition)
+	if err != nil {
+		t.Fatalf("deploying test svc: %s", err)
+	}
+
+	ingressIP := lbSvc.Status.LoadBalancer.Ingress[0].IP // Index 0 is always the public IP of the LB
+	lbTest.WaitForHttpAvailable(ingressIP)
+
+	lbTest.TearDown()
+}
+
+func TestCloudControllerManagerNetworksPodIPsAreAccessible(t *testing.T) {
+	if testCluster.useNetworks == false {
+		t.Skipf("Private Networks test is disabled")
+	}
+
+	nwTest := nwTestHelper{t: t, K8sClient: testCluster.k8sClient, server: testCluster.setup.server, privateKey: testCluster.setup.privKey, podName: "network-routes-accessible"}
+
+	pod := nwTest.DeployTestPod()
+
+	nwTest.WaitForHttpAvailable(pod.Status.PodIP)
+
+	nwTest.TearDown()
+}
diff --git a/e2etests/setup.go b/e2etests/setup.go
new file mode 100644
index 00000000..3725a7cd
--- /dev/null
+++ b/e2etests/setup.go
@@ -0,0 +1,430 @@
+package e2etests
+
+import (
+	"bytes"
+	"context"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"encoding/pem"
+	"fmt"
+	"html/template"
+	"io"
+	"io/ioutil"
+	"net"
+	"os"
+	"os/exec"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/hetznercloud/hcloud-go/hcloud"
+	"golang.org/x/crypto/ssh"
+)
+
+type hcloudK8sSetup struct {
+	Hcloud         *hcloud.Client
+	HcloudToken    string
+	K8sVersion     string
+	TestIdentifier string
+	privKey        string
+	server         *hcloud.Server
+	sshKey         *hcloud.SSHKey
+	network        *hcloud.Network
+}
+
+type cloudInitTmpl struct {
+	K8sVersion    string
+	HcloudToken   string
+	HcloudNetwork string
+}
+
+// PrepareTestEnv setups a test environment for the Cloud Controller Manager
+// This includes the creation of a Network, SSH Key and Server.
+// The server will be created with a Cloud Init UserData
+// The template can be found under e2etests/templates/cloudinit.ixt.tpl
+func (s *hcloudK8sSetup) PrepareTestEnv(ctx context.Context, additionalSSHKeys []*hcloud.SSHKey) error {
+	const op = "hcloudK8sSetup/PrepareTestEnv"
+
+	err := s.getSSHKey(ctx)
+	if err != nil {
+		return fmt.Errorf("%s getSSHKey: %s", op, err)
+	}
+
+	err = s.getNetwork(ctx)
+	if err != nil {
+		return fmt.Errorf("%s getNetwork: %s", op, err)
+	}
+	userData, err := s.getCloudInitConfig()
+	if err != nil {
+		return fmt.Errorf("%s getCloudInitConfig: %s", op, err)
+	}
+	sshKeys := []*hcloud.SSHKey{s.sshKey}
+	for _, additionalSSHKey := range additionalSSHKeys {
+		sshKeys = append(sshKeys, additionalSSHKey)
+	}
+
+	res, _, err := s.Hcloud.Server.Create(ctx, hcloud.ServerCreateOpts{
+		Name:       fmt.Sprintf("srv-%s", s.TestIdentifier),
+		ServerType: &hcloud.ServerType{Name: "cpx21"},
+		Image:      &hcloud.Image{Name: "ubuntu-20.04"},
+		SSHKeys:    sshKeys,
+		UserData:   userData,
+		Labels:     map[string]string{"K8sVersion": s.K8sVersion, "test": s.TestIdentifier},
+		Networks:   []*hcloud.Network{s.network},
+	})
+	if err != nil {
+		return fmt.Errorf("%s Hcloud.Server.Create: %s", op, err)
+	}
+
+	_, errCh := s.Hcloud.Action.WatchProgress(ctx, res.Action)
+	if err := <-errCh; err != nil {
+		return fmt.Errorf("%s WatchProgress Action %s: %s", op, res.Action.Command, err)
+	}
+
+	for _, nextAction := range res.NextActions {
+		_, errCh := s.Hcloud.Action.WatchProgress(ctx, nextAction)
+		if err := <-errCh; err != nil {
+			return fmt.Errorf("%s WatchProgress NextAction %s: %s", op, nextAction.Command, err)
+		}
+	}
+	s.server = res.Server
+	fmt.Printf("%s Waiting for server to be sshable:", op)
+	for {
+		conn, err := net.Dial("tcp", fmt.Sprintf("%s:22", s.server.PublicNet.IPv4.IP.String()))
+		if err != nil {
+			fmt.Print(".")
+			time.Sleep(1 * time.Second)
+			continue
+		}
+		_ = conn.Close()
+		fmt.Print("Connection successful\n")
+		break
+	}
+	err = s.waitForCloudInit()
+	if err != nil {
+		return err
+	}
+
+	err = s.transferCCMDockerImage()
+	if err != nil {
+		return fmt.Errorf("%s: %s", op, err)
+	}
+
+	fmt.Printf("%s Load Image:\n", op)
+	err = RunCommandOnServer(s.privKey, s.server, fmt.Sprintf("docker load --input ci-hcloud-ccm.tar"))
+	if err != nil {
+		return fmt.Errorf("%s:  Load image %s", op, err)
+	}
+
+	return nil
+}
+
+// PrepareK8s patches an existing kubernetes cluster with a CNI and the correct
+// Cloud Controller Manager version from this test run
+func (s *hcloudK8sSetup) PrepareK8s(withNetworks bool) (string, error) {
+	const op = "hcloudK8sSetup/PrepareK8s"
+
+	if withNetworks {
+		err := s.deployCilium()
+		if err != nil {
+			return "", fmt.Errorf("%s: %s", op, err)
+		}
+	} else {
+		err := s.deployFlannel()
+		if err != nil {
+			return "", fmt.Errorf("%s: %s", op, err)
+		}
+	}
+
+	err := s.prepareCCMDeploymentFile(withNetworks)
+	if err != nil {
+		return "", fmt.Errorf("%s: %s", op, err)
+	}
+
+	fmt.Printf("%s: Apply ccm deployment\n", op)
+	err = RunCommandOnServer(s.privKey, s.server, fmt.Sprintf("KUBECONFIG=/root/.kube/config kubectl apply -f ccm.yml"))
+	if err != nil {
+		return "", fmt.Errorf("%s Deploy ccm: %s", op, err)
+	}
+	fmt.Printf("%s: Download kubeconfig\n", op)
+
+	cmd := exec.Command("/usr/bin/scp", "-i", "ssh_key", "-o", "UserKnownHostsFile=/dev/null", "-o", "StrictHostKeyChecking=no", fmt.Sprintf("root@%s:/root/.kube/config", s.server.PublicNet.IPv4.IP.String()), "kubeconfig")
+	if ok := os.Getenv("TEST_DEBUG_MODE"); ok != "" {
+		cmd.Stdout = os.Stdout
+	}
+	err = cmd.Run()
+	if err != nil {
+		return "", fmt.Errorf("%s download kubeconfig: %s", op, err)
+	}
+	return "kubeconfig", nil
+}
+
+// prepareCCMDeploymentFile patches the Cloud Controller Deployment file
+// It replaces the used image and the pull policy to always use the local image
+// from this test run
+func (s *hcloudK8sSetup) prepareCCMDeploymentFile(networks bool) error {
+	const op = "hcloudK8sSetup/prepareCCMDeploymentFile"
+	fmt.Printf("%s: Read master deployment filee\n", op)
+	var deploymentFilePath = "../deploy/dev-ccm.yaml"
+	if networks {
+		deploymentFilePath = "../deploy/dev-ccm-networks.yaml"
+	}
+	deploymentFile, err := ioutil.ReadFile(deploymentFilePath)
+	if err != nil {
+		return fmt.Errorf("%s: read ccm deployment file %s: %v", op, deploymentFilePath, err)
+	}
+
+	fmt.Printf("%s: Prepare deployment file and transfer it\n", op)
+	deploymentFile = []byte(strings.ReplaceAll(string(deploymentFile), "hetznercloud/hcloud-cloud-controller-manager:latest", fmt.Sprintf("hcloud-ccm:ci_%s", s.TestIdentifier)))
+	deploymentFile = []byte(strings.ReplaceAll(string(deploymentFile), " imagePullPolicy: Always", " imagePullPolicy: IfNotPresent"))
+
+	err = RunCommandOnServer(s.privKey, s.server, fmt.Sprintf("echo '%s' >> ccm.yml", deploymentFile))
+	if err != nil {
+		return fmt.Errorf("%s: Prepare deployment file and transfer it: %s", op, err)
+	}
+	return nil
+}
+
+// deployFlannel deploys Flannel as CNI. Flannel is used for all tests where
+// we don't use Hetzner Cloud Networks
+func (s *hcloudK8sSetup) deployFlannel() error {
+	const op = "hcloudK8sSetup/deployFlannel"
+	fmt.Printf("%s: apply flannel deployment\n", op)
+	err := RunCommandOnServer(s.privKey, s.server, "KUBECONFIG=/root/.kube/config kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml")
+	if err != nil {
+		return fmt.Errorf("%s: apply flannel deployment: %s", op, err)
+	}
+	fmt.Printf("%s: patch flannel deployment\n", op)
+	err = RunCommandOnServer(s.privKey, s.server, "KUBECONFIG=/root/.kube/config kubectl -n kube-system patch ds kube-flannel-ds --type json -p '[{\"op\":\"add\",\"path\":\"/spec/template/spec/tolerations/-\",\"value\":{\"key\":\"node.cloudprovider.kubernetes.io/uninitialized\",\"value\":\"true\",\"effect\":\"NoSchedule\"}}]'")
+	if err != nil {
+		return fmt.Errorf("%s: patch flannel deployment: %s", op, err)
+	}
+	return nil
+}
+
+// deployCilium deploys Cilium as CNI. Cilium is used for all tests where
+// we use Hetzner Cloud Networks as Cilium is one of the only CNIs
+// that support Cloud Controllers as source for advertising routes.
+func (s *hcloudK8sSetup) deployCilium() error {
+	const op = "hcloudK8sSetup/deployCilium"
+
+	deploymentFile, err := ioutil.ReadFile("templates/cilium.yml")
+	if err != nil {
+		return fmt.Errorf("%s: read cilium deployment file %s: %v", op, "templates/cilium.yml", err)
+	}
+	err = RunCommandOnServer(s.privKey, s.server, fmt.Sprintf("echo '%s' >> cilium.yml", deploymentFile))
+	if err != nil {
+		return fmt.Errorf("%s: Transfer cilium deployment: %s", op, err)
+	}
+
+	fmt.Printf("%s: apply cilium deployment\n", op)
+	err = RunCommandOnServer(s.privKey, s.server, "KUBECONFIG=/root/.kube/config kubectl apply -f cilium.yml")
+	if err != nil {
+		return fmt.Errorf("%s: apply cilium deployment: %s", op, err)
+	}
+
+	return nil
+}
+
+// transferCCMDockerImage transfers the local build docker image tar via SCP
+func (s *hcloudK8sSetup) transferCCMDockerImage() error {
+	const op = "hcloudK8sSetup/transferCCMDockerImage"
+	fmt.Printf("%s: Transfer docker image\n", op)
+	err := WithSSHSession(s.privKey, s.server.PublicNet.IPv4.IP.String(), func(session *ssh.Session) error {
+		file, err := os.Open("ci-hcloud-ccm.tar")
+		if err != nil {
+			return fmt.Errorf("%s read ci-hcloud-ccm.tar: %s", op, err)
+		}
+		defer file.Close()
+		stat, err := file.Stat()
+		if err != nil {
+			return fmt.Errorf("%s file.Stat: %s", op, err)
+		}
+		wg := sync.WaitGroup{}
+		wg.Add(1)
+
+		go func() {
+			hostIn, _ := session.StdinPipe()
+			defer hostIn.Close()
+			fmt.Fprintf(hostIn, "C0664 %d %s\n", stat.Size(), "ci-hcloud-ccm.tar")
+			io.Copy(hostIn, file)
+			fmt.Fprint(hostIn, "\x00")
+			wg.Done()
+		}()
+
+		err = session.Run("/usr/bin/scp -t /root")
+		if err != nil {
+			return fmt.Errorf("%s copy via scp: %s", op, err)
+		}
+		wg.Wait()
+		return err
+	})
+	return err
+}
+
+// waitForCloudInit waits on cloud init on the server.
+// when cloud init is ready we can assume that the server
+// and the plain k8s installation is ready
+func (s *hcloudK8sSetup) waitForCloudInit() error {
+	const op = "hcloudK8sSetup/PrepareTestEnv"
+	fmt.Printf("%s: Wait for cloud-init\n", op)
+	err := RunCommandOnServer(s.privKey, s.server, fmt.Sprintf("cloud-init status --wait > /dev/null"))
+	if err != nil {
+		return fmt.Errorf("%s: Wait for cloud-init: %s", op, err)
+	}
+	return nil
+}
+
+// TearDown deletes all created resources within the Hetzner Cloud
+// there is no need to "shutdown" the k8s cluster before
+// so we just delete all created resources
+func (s *hcloudK8sSetup) TearDown(ctx context.Context) error {
+	const op = "hcloudK8sSetup/TearDown"
+	_, err := s.Hcloud.Server.Delete(ctx, s.server)
+	if err != nil {
+		return fmt.Errorf("%s Hcloud.Server.Delete: %s", op, err)
+	}
+	s.server = nil
+	_, err = s.Hcloud.SSHKey.Delete(ctx, s.sshKey)
+	if err != nil {
+		return fmt.Errorf("%s Hcloud.SSHKey.Delete: %s", err, err)
+	}
+	s.sshKey = nil
+	_, err = s.Hcloud.Network.Delete(ctx, s.network)
+	if err != nil {
+		return fmt.Errorf("%s Hcloud.Network.Delete: %s", err, err)
+	}
+	s.network = nil
+	return nil
+}
+
+// getCloudInitConfig returns the generated cloud init configuration
+func (s *hcloudK8sSetup) getCloudInitConfig() (string, error) {
+	const op = "hcloudK8sSetup/getCloudInitConfig"
+	str, err := ioutil.ReadFile("templates/cloudinit.txt.tpl")
+	if err != nil {
+		return "", fmt.Errorf("%s: read template file %s: %v", "templates/cloudinit.txt.tpl", op, err)
+	}
+	tmpl, err := template.New("cloud_init").Parse(string(str))
+	if err != nil {
+		return "", fmt.Errorf("%s: parsing template file %s: %v", "templates/cloudinit.txt.tpl", op, err)
+	}
+	var buf bytes.Buffer
+	if err := tmpl.Execute(&buf, cloudInitTmpl{K8sVersion: s.K8sVersion, HcloudToken: s.HcloudToken, HcloudNetwork: s.network.Name}); err != nil {
+		return "", fmt.Errorf("%s: execute template: %v", op, err)
+	}
+	return buf.String(), nil
+}
+
+//getSSHKey create and get the Hetzner Cloud SSH Key for the test
+func (s *hcloudK8sSetup) getSSHKey(ctx context.Context) error {
+	const op = "hcloudK8sSetup/getSSHKey"
+	pubKey, privKey, err := makeSSHKeyPair()
+	if err != nil {
+		return err
+	}
+	sshKey, _, err := s.Hcloud.SSHKey.Create(ctx, hcloud.SSHKeyCreateOpts{
+		Name:      fmt.Sprintf("s-%s", s.TestIdentifier),
+		PublicKey: pubKey,
+		Labels:    map[string]string{"K8sVersion": s.K8sVersion, "test": s.TestIdentifier},
+	})
+	if err != nil {
+		return fmt.Errorf("%s: creating ssh key: %v", op, err)
+	}
+	s.privKey = privKey
+	s.sshKey = sshKey
+	err = ioutil.WriteFile("ssh_key", []byte(s.privKey), 0600)
+	if err != nil {
+		return fmt.Errorf("%s: writing ssh key private key: %v", op, err)
+	}
+	return nil
+}
+
+// getNetwork create a Hetzner Cloud Network for this test
+func (s *hcloudK8sSetup) getNetwork(ctx context.Context) error {
+	const op = "hcloudK8sSetup/getNetwork"
+	_, ipRange, _ := net.ParseCIDR("10.0.0.0/8")
+	_, subnetRange, _ := net.ParseCIDR("10.0.0.0/16")
+	network, _, err := s.Hcloud.Network.Create(ctx, hcloud.NetworkCreateOpts{
+		Name:    fmt.Sprintf("nw-%s", s.TestIdentifier),
+		IPRange: ipRange,
+		Labels:  map[string]string{"K8sVersion": s.K8sVersion, "test": s.TestIdentifier},
+	})
+	if err != nil {
+		return fmt.Errorf("%s: creating network: %v", op, err)
+	}
+	_, _, err = s.Hcloud.Network.AddSubnet(ctx, network, hcloud.NetworkAddSubnetOpts{
+		Subnet: hcloud.NetworkSubnet{
+			Type:        hcloud.NetworkSubnetTypeCloud,
+			IPRange:     subnetRange,
+			NetworkZone: hcloud.NetworkZoneEUCentral,
+		},
+	})
+	if err != nil {
+		return fmt.Errorf("%s: creating subnet: %v", op, err)
+	}
+	s.network = network
+	return nil
+}
+
+// makeSSHKeyPair generate a SSH key pair
+func makeSSHKeyPair() (string, string, error) {
+	privateKey, err := rsa.GenerateKey(rand.Reader, 1024)
+	if err != nil {
+		return "", "", err
+	}
+
+	// generate and write private key as PEM
+	var privKeyBuf strings.Builder
+
+	privateKeyPEM := &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(privateKey)}
+	if err := pem.Encode(&privKeyBuf, privateKeyPEM); err != nil {
+		return "", "", err
+	}
+
+	// generate and write public key
+	pub, err := ssh.NewPublicKey(&privateKey.PublicKey)
+	if err != nil {
+		return "", "", err
+	}
+
+	var pubKeyBuf strings.Builder
+	pubKeyBuf.Write(ssh.MarshalAuthorizedKey(pub))
+
+	return pubKeyBuf.String(), privKeyBuf.String(), nil
+}
+
+func RunCommandOnServer(privKey string, server *hcloud.Server, command string) error {
+	return WithSSHSession(privKey, server.PublicNet.IPv4.IP.String(), func(session *ssh.Session) error {
+		if ok := os.Getenv("TEST_DEBUG_MODE"); ok != "" {
+			session.Stdout = os.Stdout
+		}
+		return session.Run(command)
+	})
+}
+
+func WithSSHSession(privKey string, host string, fn func(*ssh.Session) error) error {
+	signer, err := ssh.ParsePrivateKey([]byte(privKey))
+	if err != nil {
+		return err
+	}
+
+	client, err := ssh.Dial("tcp", net.JoinHostPort(host, "22"), &ssh.ClientConfig{
+		User:            "root",
+		Auth:            []ssh.AuthMethod{ssh.PublicKeys(signer)},
+		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
+		Timeout:         1 * time.Second,
+	})
+	if err != nil {
+		return err
+	}
+
+	session, err := client.NewSession()
+	if err != nil {
+		return err
+	}
+	defer session.Close()
+
+	return fn(session)
+}
diff --git a/e2etests/templates/cilium.yml b/e2etests/templates/cilium.yml
new file mode 100644
index 00000000..b93e1914
--- /dev/null
+++ b/e2etests/templates/cilium.yml
@@ -0,0 +1,679 @@
+---
+# Source: cilium/charts/agent/templates/serviceaccount.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: cilium
+  namespace: kube-system
+---
+# Source: cilium/charts/operator/templates/serviceaccount.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: cilium-operator
+  namespace: kube-system
+---
+# Source: cilium/charts/config/templates/configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cilium-config
+  namespace: kube-system
+data:
+
+  # Identity allocation mode selects how identities are shared between cilium
+  # nodes by setting how they are stored. The options are "crd" or "kvstore".
+  # - "crd" stores identities in kubernetes as CRDs (custom resource definition).
+  #   These can be queried with:
+  #     kubectl get ciliumid
+  # - "kvstore" stores identities in a kvstore, etcd or consul, that is
+  #   configured below. Cilium versions before 1.6 supported only the kvstore
+  #   backend. Upgrades from these older cilium versions should continue using
+  #   the kvstore by commenting out the identity-allocation-mode below, or
+  #   setting it to "kvstore".
+  identity-allocation-mode: crd
+
+  # If you want to run cilium in debug mode change this value to true
+  debug: "false"
+  # If you want metrics enabled in all of your Cilium agents, set the port for
+  # which the Cilium agents will have their metrics exposed.
+  # This option deprecates the "prometheus-serve-addr" in the
+  # "cilium-metrics-config" ConfigMap
+  # NOTE that this will open the port on ALL nodes where Cilium pods are
+  # scheduled.
+  prometheus-serve-addr: ":9100"
+
+  # Enable IPv4 addressing. If enabled, all endpoints are allocated an IPv4
+  # address.
+  enable-ipv4: "true"
+
+  # Enable IPv6 addressing. If enabled, all endpoints are allocated an IPv6
+  # address.
+  enable-ipv6: "false"
+
+  # If you want cilium monitor to aggregate tracing for packets, set this level
+  # to "low", "medium", or "maximum". The higher the level, the less packets
+  # that will be seen in monitor output.
+  monitor-aggregation: medium
+
+  # The monitor aggregation interval governs the typical time between monitor
+  # notification events for each allowed connection.
+  #
+  # Only effective when monitor aggregation is set to "medium" or higher.
+  monitor-aggregation-interval: 5s
+
+  # The monitor aggregation flags determine which TCP flags which, upon the
+  # first observation, cause monitor notifications to be generated.
+  #
+  # Only effective when monitor aggregation is set to "medium" or higher.
+  monitor-aggregation-flags: all
+
+  # ct-global-max-entries-* specifies the maximum number of connections
+  # supported across all endpoints, split by protocol: tcp or other. One pair
+  # of maps uses these values for IPv4 connections, and another pair of maps
+  # use these values for IPv6 connections.
+  #
+  # If these values are modified, then during the next Cilium startup the
+  # tracking of ongoing connections may be disrupted. This may lead to brief
+  # policy drops or a change in loadbalancing decisions for a connection.
+  #
+  # For users upgrading from Cilium 1.2 or earlier, to minimize disruption
+  # during the upgrade process, comment out these options.
+  bpf-ct-global-tcp-max: "524288"
+  bpf-ct-global-any-max: "262144"
+
+  # Pre-allocation of map entries allows per-packet latency to be reduced, at
+  # the expense of up-front memory allocation for the entries in the maps. The
+  # default value below will minimize memory usage in the default installation;
+  # users who are sensitive to latency may consider setting this to "true".
+  #
+  # This option was introduced in Cilium 1.4. Cilium 1.3 and earlier ignore
+  # this option and behave as though it is set to "true".
+  #
+  # If this value is modified, then during the next Cilium startup the restore
+  # of existing endpoints and tracking of ongoing connections may be disrupted.
+  # This may lead to policy drops or a change in loadbalancing decisions for a
+  # connection for some time. Endpoints may need to be recreated to restore
+  # connectivity.
+  #
+  # If this option is set to "false" during an upgrade from 1.3 or earlier to
+  # 1.4 or later, then it may cause one-time disruptions during the upgrade.
+  preallocate-bpf-maps: "false"
+
+  # Regular expression matching compatible Istio sidecar istio-proxy
+  # container image names
+  sidecar-istio-proxy-image: "cilium/istio_proxy"
+
+  # Encapsulation mode for communication between nodes
+  # Possible values:
+  #   - disabled
+  #   - vxlan (default)
+  #   - geneve
+  tunnel: disabled
+
+  # Name of the cluster. Only relevant when building a mesh of clusters.
+  cluster-name: default
+
+  # DNS Polling periodically issues a DNS lookup for each `matchName` from
+  # cilium-agent. The result is used to regenerate endpoint policy.
+  # DNS lookups are repeated with an interval of 5 seconds, and are made for
+  # A(IPv4) and AAAA(IPv6) addresses. Should a lookup fail, the most recent IP
+  # data is used instead. An IP change will trigger a regeneration of the Cilium
+  # policy for each endpoint and increment the per cilium-agent policy
+  # repository revision.
+  #
+  # This option is disabled by default starting from version 1.4.x in favor
+  # of a more powerful DNS proxy-based implementation, see [0] for details.
+  # Enable this option if you want to use FQDN policies but do not want to use
+  # the DNS proxy.
+  #
+  # To ease upgrade, users may opt to set this option to "true".
+  # Otherwise please refer to the Upgrade Guide [1] which explains how to
+  # prepare policy rules for upgrade.
+  #
+  # [0] http://docs.cilium.io/en/stable/policy/language/#dns-based
+  # [1] http://docs.cilium.io/en/stable/install/upgrade/#changes-that-may-require-action
+  tofqdns-enable-poller: "false"
+
+  # wait-bpf-mount makes init container wait until bpf filesystem is mounted
+  wait-bpf-mount: "false"
+
+  masquerade: "true"
+  enable-ipsec: "false"
+  ipsec-key-file: /etc/ipsec/keys
+  encrypt-interface: ens10
+  encrypt-node: "false"
+  enable-xt-socket-fallback: "true"
+  install-iptables-rules: "true"
+  auto-direct-node-routes: "false"
+  kube-proxy-replacement:  "probe"
+  enable-host-reachable-services: "false"
+  enable-external-ips: "false"
+  enable-node-port: "false"
+  enable-well-known-identities: "false"
+  enable-remote-node-identity: "true"
+
+  blacklist-conflicting-routes: "false"
+---
+# Source: cilium/charts/agent/templates/clusterrole.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: cilium
+rules:
+  - apiGroups:
+      - networking.k8s.io
+    resources:
+      - networkpolicies
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - discovery.k8s.io
+    resources:
+      - endpointslices
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - namespaces
+      - services
+      - nodes
+      - endpoints
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - nodes
+    verbs:
+      - get
+      - list
+      - watch
+      - update
+  - apiGroups:
+      - ""
+    resources:
+      - nodes
+      - nodes/status
+    verbs:
+      - patch
+  - apiGroups:
+      - apiextensions.k8s.io
+    resources:
+      - customresourcedefinitions
+    verbs:
+      - create
+      - get
+      - list
+      - watch
+      - update
+  - apiGroups:
+      - cilium.io
+    resources:
+      - ciliumnetworkpolicies
+      - ciliumnetworkpolicies/status
+      - ciliumclusterwidenetworkpolicies
+      - ciliumclusterwidenetworkpolicies/status
+      - ciliumendpoints
+      - ciliumendpoints/status
+      - ciliumnodes
+      - ciliumnodes/status
+      - ciliumidentities
+      - ciliumidentities/status
+    verbs:
+      - "*"
+---
+# Source: cilium/charts/operator/templates/clusterrole.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: cilium-operator
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      # to automatically delete [core|kube]dns pods so that are starting to being
+      # managed by Cilium
+      - pods
+    verbs:
+      - get
+      - list
+      - watch
+      - delete
+  - apiGroups:
+      - discovery.k8s.io
+    resources:
+      - endpointslices
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      # to automatically read from k8s and import the node"s pod CIDR to cilium"s
+      # etcd so all nodes know how to reach another pod running in in a different
+      # node.
+      - nodes
+      # to perform the translation of a CNP that contains `ToGroup` to its endpoints
+      - services
+      - endpoints
+      # to check apiserver connectivity
+      - namespaces
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - cilium.io
+    resources:
+      - ciliumnetworkpolicies
+      - ciliumnetworkpolicies/status
+      - ciliumclusterwidenetworkpolicies
+      - ciliumclusterwidenetworkpolicies/status
+      - ciliumendpoints
+      - ciliumendpoints/status
+      - ciliumnodes
+      - ciliumnodes/status
+      - ciliumidentities
+      - ciliumidentities/status
+    verbs:
+      - "*"
+---
+# Source: cilium/charts/agent/templates/clusterrolebinding.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: cilium
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cilium
+subjects:
+  - kind: ServiceAccount
+    name: cilium
+    namespace: kube-system
+---
+# Source: cilium/charts/operator/templates/clusterrolebinding.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: cilium-operator
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cilium-operator
+subjects:
+  - kind: ServiceAccount
+    name: cilium-operator
+    namespace: kube-system
+---
+# Source: cilium/charts/agent/templates/daemonset.yaml
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  labels:
+    k8s-app: cilium
+  name: cilium
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      k8s-app: cilium
+  template:
+    metadata:
+      annotations:
+        prometheus.io/port: "9100"
+        prometheus.io/scrape: "true"
+        # This annotation plus the CriticalAddonsOnly toleration makes
+        # cilium to be a critical pod in the cluster, which ensures cilium
+        # gets priority scheduling.
+        # https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/
+        scheduler.alpha.kubernetes.io/critical-pod: ""
+      labels:
+        k8s-app: cilium
+    spec:
+      containers:
+        - args:
+            - --config-dir=/tmp/cilium/config-map
+          command:
+            - cilium-agent
+          livenessProbe:
+            exec:
+              command:
+                - cilium
+                - status
+                - --brief
+            failureThreshold: 10
+            # The initial delay for the liveness probe is intentionally large to
+            # avoid an endless kill & restart cycle if in the event that the initial
+            # bootstrapping takes longer than expected.
+            initialDelaySeconds: 120
+            periodSeconds: 30
+            successThreshold: 1
+            timeoutSeconds: 5
+          readinessProbe:
+            exec:
+              command:
+                - cilium
+                - status
+                - --brief
+            failureThreshold: 3
+            initialDelaySeconds: 5
+            periodSeconds: 30
+            successThreshold: 1
+            timeoutSeconds: 5
+          env:
+            - name: K8S_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: spec.nodeName
+            - name: CILIUM_K8S_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: metadata.namespace
+            - name: CILIUM_FLANNEL_MASTER_DEVICE
+              valueFrom:
+                configMapKeyRef:
+                  key: flannel-master-device
+                  name: cilium-config
+                  optional: true
+            - name: CILIUM_FLANNEL_UNINSTALL_ON_EXIT
+              valueFrom:
+                configMapKeyRef:
+                  key: flannel-uninstall-on-exit
+                  name: cilium-config
+                  optional: true
+            - name: CILIUM_CLUSTERMESH_CONFIG
+              value: /var/lib/cilium/clustermesh/
+            - name: CILIUM_CNI_CHAINING_MODE
+              valueFrom:
+                configMapKeyRef:
+                  key: cni-chaining-mode
+                  name: cilium-config
+                  optional: true
+            - name: CILIUM_CUSTOM_CNI_CONF
+              valueFrom:
+                configMapKeyRef:
+                  key: custom-cni-conf
+                  name: cilium-config
+                  optional: true
+          image: "docker.io/cilium/cilium:v1.7.1"
+          imagePullPolicy: IfNotPresent
+          lifecycle:
+            postStart:
+              exec:
+                command:
+                  - "/cni-install.sh"
+                  - "--enable-debug=false"
+            preStop:
+              exec:
+                command:
+                  - /cni-uninstall.sh
+          name: cilium-agent
+          ports:
+            - containerPort: 9100
+              hostPort: 9100
+              name: prometheus
+              protocol: TCP
+          securityContext:
+            capabilities:
+              add:
+                - NET_ADMIN
+                - SYS_MODULE
+            privileged: true
+          volumeMounts:
+            - mountPath: /sys/fs/bpf
+              name: bpf-maps
+            - mountPath: /var/run/cilium
+              name: cilium-run
+            - mountPath: /host/opt/cni/bin
+              name: cni-path
+            - mountPath: /host/etc/cni/net.d
+              name: etc-cni-netd
+            - mountPath: /var/lib/cilium/clustermesh
+              name: clustermesh-secrets
+              readOnly: true
+            - mountPath: /tmp/cilium/config-map
+              name: cilium-config-path
+              readOnly: true
+              # Needed to be able to load kernel modules
+            - mountPath: /lib/modules
+              name: lib-modules
+              readOnly: true
+            - mountPath: /run/xtables.lock
+              name: xtables-lock
+      hostNetwork: true
+      initContainers:
+        - command:
+            - /init-container.sh
+          env:
+            - name: CILIUM_ALL_STATE
+              valueFrom:
+                configMapKeyRef:
+                  key: clean-cilium-state
+                  name: cilium-config
+                  optional: true
+            - name: CILIUM_BPF_STATE
+              valueFrom:
+                configMapKeyRef:
+                  key: clean-cilium-bpf-state
+                  name: cilium-config
+                  optional: true
+            - name: CILIUM_WAIT_BPF_MOUNT
+              valueFrom:
+                configMapKeyRef:
+                  key: wait-bpf-mount
+                  name: cilium-config
+                  optional: true
+          image: "docker.io/cilium/cilium:v1.7.1"
+          imagePullPolicy: IfNotPresent
+          name: clean-cilium-state
+          securityContext:
+            capabilities:
+              add:
+                - NET_ADMIN
+            privileged: true
+          volumeMounts:
+            - mountPath: /sys/fs/bpf
+              name: bpf-maps
+              mountPropagation: HostToContainer
+            - mountPath: /var/run/cilium
+              name: cilium-run
+      restartPolicy: Always
+      priorityClassName: system-node-critical
+      serviceAccount: cilium
+      serviceAccountName: cilium
+      terminationGracePeriodSeconds: 1
+      tolerations:
+        - operator: Exists
+      volumes:
+        # To keep state between restarts / upgrades
+        - hostPath:
+            path: /var/run/cilium
+            type: DirectoryOrCreate
+          name: cilium-run
+          # To keep state between restarts / upgrades for bpf maps
+        - hostPath:
+            path: /sys/fs/bpf
+            type: DirectoryOrCreate
+          name: bpf-maps
+        # To install cilium cni plugin in the host
+        - hostPath:
+            path:  /opt/cni/bin
+            type: DirectoryOrCreate
+          name: cni-path
+          # To install cilium cni configuration in the host
+        - hostPath:
+            path: /etc/cni/net.d
+            type: DirectoryOrCreate
+          name: etc-cni-netd
+          # To be able to load kernel modules
+        - hostPath:
+            path: /lib/modules
+          name: lib-modules
+          # To access iptables concurrently with other processes (e.g. kube-proxy)
+        - hostPath:
+            path: /run/xtables.lock
+            type: FileOrCreate
+          name: xtables-lock
+          # To read the clustermesh configuration
+        - name: clustermesh-secrets
+          secret:
+            defaultMode: 420
+            optional: true
+            secretName: cilium-clustermesh
+          # To read the configuration from the config map
+        - configMap:
+            name: cilium-config
+          name: cilium-config-path
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: 2
+    type: RollingUpdate
+---
+# Source: cilium/charts/operator/templates/deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    io.cilium/app: operator
+    name: cilium-operator
+  name: cilium-operator
+  namespace: kube-system
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      io.cilium/app: operator
+      name: cilium-operator
+  strategy:
+    rollingUpdate:
+      maxSurge: 1
+      maxUnavailable: 1
+    type: RollingUpdate
+  template:
+    metadata:
+      annotations:
+        prometheus.io/port: "6942"
+        prometheus.io/scrape: "true"
+      labels:
+        io.cilium/app: operator
+        name: cilium-operator
+    spec:
+      containers:
+        - args:
+            - --debug=$(CILIUM_DEBUG)
+            - --identity-allocation-mode=$(CILIUM_IDENTITY_ALLOCATION_MODE)
+            - --enable-metrics
+            - --synchronize-k8s-nodes=true
+          command:
+            - cilium-operator
+          env:
+            - name: CILIUM_K8S_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: metadata.namespace
+            - name: K8S_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: spec.nodeName
+            - name: CILIUM_DEBUG
+              valueFrom:
+                configMapKeyRef:
+                  key: debug
+                  name: cilium-config
+                  optional: true
+            - name: CILIUM_CLUSTER_NAME
+              valueFrom:
+                configMapKeyRef:
+                  key: cluster-name
+                  name: cilium-config
+                  optional: true
+            - name: CILIUM_CLUSTER_ID
+              valueFrom:
+                configMapKeyRef:
+                  key: cluster-id
+                  name: cilium-config
+                  optional: true
+            - name: CILIUM_IPAM
+              valueFrom:
+                configMapKeyRef:
+                  key: ipam
+                  name: cilium-config
+                  optional: true
+            - name: CILIUM_DISABLE_ENDPOINT_CRD
+              valueFrom:
+                configMapKeyRef:
+                  key: disable-endpoint-crd
+                  name: cilium-config
+                  optional: true
+            - name: CILIUM_KVSTORE
+              valueFrom:
+                configMapKeyRef:
+                  key: kvstore
+                  name: cilium-config
+                  optional: true
+            - name: CILIUM_KVSTORE_OPT
+              valueFrom:
+                configMapKeyRef:
+                  key: kvstore-opt
+                  name: cilium-config
+                  optional: true
+            - name: AWS_ACCESS_KEY_ID
+              valueFrom:
+                secretKeyRef:
+                  key: AWS_ACCESS_KEY_ID
+                  name: cilium-aws
+                  optional: true
+            - name: AWS_SECRET_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  key: AWS_SECRET_ACCESS_KEY
+                  name: cilium-aws
+                  optional: true
+            - name: AWS_DEFAULT_REGION
+              valueFrom:
+                secretKeyRef:
+                  key: AWS_DEFAULT_REGION
+                  name: cilium-aws
+                  optional: true
+            - name: CILIUM_IDENTITY_ALLOCATION_MODE
+              valueFrom:
+                configMapKeyRef:
+                  key: identity-allocation-mode
+                  name: cilium-config
+                  optional: true
+          image: "docker.io/cilium/operator:v1.7.1"
+          imagePullPolicy: IfNotPresent
+          name: cilium-operator
+          ports:
+            - containerPort: 6942
+              hostPort: 6942
+              name: prometheus
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              host: "127.0.0.1"
+              path: /healthz
+              port: 9234
+              scheme: HTTP
+            initialDelaySeconds: 60
+            periodSeconds: 10
+            timeoutSeconds: 3
+      hostNetwork: true
+      restartPolicy: Always
+      serviceAccount: cilium-operator
+      serviceAccountName: cilium-operator
diff --git a/e2etests/templates/cloudinit.txt.tpl b/e2etests/templates/cloudinit.txt.tpl
new file mode 100644
index 00000000..28dfb4b7
--- /dev/null
+++ b/e2etests/templates/cloudinit.txt.tpl
@@ -0,0 +1,44 @@
+#cloud-config
+write_files:
+- content: |
+    net.bridge.bridge-nf-call-ip6tables = 1
+    net.bridge.bridge-nf-call-iptables = 1
+  path: /etc/sysctl.d/k8s.conf
+- content: |
+    apiVersion: kubeadm.k8s.io/v1beta2
+    kind: ClusterConfiguration
+    kubernetesVersion: v{{.K8sVersion}}
+    networking:
+      podSubnet: "10.244.0.0/16"
+  path: /tmp/kubeadm-config.yaml
+- content: |
+    [Service]
+    Environment="KUBELET_EXTRA_ARGS=--cloud-provider=external"
+  path: /etc/systemd/system/kubelet.service.d/20-hcloud.conf
+- content: |
+    alias k="kubectl"
+    alias ksy="kubectl -n kube-system"
+    alias kgp="kubectl get pods"
+    alias kgs="kubectl get services"
+    export HCLOUD_TOKEN={{.HcloudToken}}
+  path: /root/.bashrc
+runcmd:
+- sysctl --system
+- apt install -y apt-transport-https curl
+- curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
+- echo "deb https://apt.kubernetes.io/ kubernetes-xenial main" > /etc/apt/sources.list.d/kubernetes.list
+- apt update
+- apt install -y kubectl={{.K8sVersion}}-00 kubeadm={{.K8sVersion}}-00 kubelet={{.K8sVersion}}-00 docker.io
+- systemctl daemon-reload
+- systemctl restart kubelet
+- kubeadm init  --config /tmp/kubeadm-config.yaml
+- mkdir -p /root/.kube
+- cp -i /etc/kubernetes/admin.conf /root/.kube/config
+- until KUBECONFIG=/root/.kube/config kubectl get node; do sleep 2;done
+- KUBECONFIG=/root/.kube/config kubectl -n kube-system create secret generic hcloud --from-literal=token={{.HcloudToken}} --from-literal=network={{.HcloudNetwork}}
+# Remove all master taints and labels, this is special for the one-node cluster we use within the tests
+- KUBECONFIG=/root/.kube/config kubectl taint nodes --all node-role.kubernetes.io/master-
+- KUBECONFIG=/root/.kube/config kubectl label nodes --all node-role.kubernetes.io/master-
+# Download and install latest hcloud cli release for easier debugging on host
+- curl -s https://api.github.com/repos/hetznercloud/cli/releases/latest | grep browser_download_url | grep linux-amd64 | cut -d '"' -f 4 | wget -qi -
+- tar xvzf hcloud-linux-amd64.tar.gz && cp hcloud /usr/bin/hcloud && chmod +x /usr/bin/hcloud
diff --git a/e2etests/testing.go b/e2etests/testing.go
new file mode 100644
index 00000000..3b31f0ea
--- /dev/null
+++ b/e2etests/testing.go
@@ -0,0 +1,469 @@
+package e2etests
+
+import (
+	"context"
+	"fmt"
+	"math/rand"
+	"net/http"
+	"os"
+	"os/exec"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"k8s.io/client-go/tools/clientcmd"
+
+	"github.com/hetznercloud/hcloud-go/hcloud"
+	corev1 "k8s.io/api/core/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/kubernetes"
+)
+
+var rng *rand.Rand
+
+func init() {
+	rng = rand.New(rand.NewSource(time.Now().UnixNano()))
+}
+
+type TestCluster struct {
+	once        sync.Once
+	useNetworks bool
+	setup       *hcloudK8sSetup
+	k8sClient   *kubernetes.Clientset
+	started     bool
+	err         error
+}
+
+func (tc *TestCluster) initialize() error {
+	const op = "TestCluster/initialize"
+	tc.once.Do(func() {
+		fmt.Printf("%s: Starting CCM Testsuite\n", op)
+		networksSupport := os.Getenv("USE_NETWORKS")
+		if networksSupport == "yes" {
+			tc.useNetworks = true
+		}
+		isUsingGithubActions := os.Getenv("GITHUB_ACTIONS")
+		isUsingGitlabCI := os.Getenv("CI_JOB_ID")
+		testIdentifier := ""
+		if isUsingGithubActions == "true" {
+			testIdentifier = fmt.Sprintf("gh-%s-%d", os.Getenv("GITHUB_RUN_ID"), rng.Int())
+			fmt.Printf("%s: Running in Github Action\n", op)
+		}
+		if isUsingGitlabCI != "" {
+			testIdentifier = fmt.Sprintf("gl-%s", isUsingGitlabCI)
+			fmt.Printf("%s: Running in Gitlab CI\n", op)
+		}
+		if testIdentifier == "" {
+			testIdentifier = fmt.Sprintf("local-%d", rng.Int())
+			fmt.Printf("%s: Running local\n", op)
+		}
+
+		k8sVersion := os.Getenv("K8S_VERSION")
+		if k8sVersion == "" {
+			k8sVersion = "1.18.9"
+		}
+		token := os.Getenv("HCLOUD_TOKEN")
+		if len(token) != 64 {
+			tc.err = fmt.Errorf("%s: No valid HCLOUD_TOKEN found\n", op)
+			return
+		}
+
+		var additionalSSHKeys []*hcloud.SSHKey
+
+		opts := []hcloud.ClientOption{
+			hcloud.WithToken(token),
+			hcloud.WithApplication("hcloud-ccm-testsuite", "1.0"),
+		}
+		hcloudClient := hcloud.NewClient(opts...)
+		additionalSSHKeysIdOrName := os.Getenv("USE_SSH_KEYS")
+		if additionalSSHKeysIdOrName != "" {
+			idsOrNames := strings.Split(additionalSSHKeysIdOrName, ",")
+			for _, idOrName := range idsOrNames {
+				additionalSSHKey, _, err := hcloudClient.SSHKey.Get(context.Background(), idOrName)
+				if err != nil {
+					tc.err = fmt.Errorf("%s:%s\n", op, err)
+					return
+				}
+				additionalSSHKeys = append(additionalSSHKeys, additionalSSHKey)
+			}
+
+		}
+
+		fmt.Printf("Test against k8s %s\n", k8sVersion)
+
+		fmt.Println("Building ccm image")
+		cmd := exec.Command("docker", "build", "-t", fmt.Sprintf("hcloud-ccm:ci_%s", testIdentifier), "../")
+		cmd.Stdout = os.Stdout
+		err := cmd.Run()
+		if err != nil {
+			tc.err = fmt.Errorf("%s:%s\n", op, err)
+			return
+		}
+
+		fmt.Println("Saving ccm image to disk")
+		cmd = exec.Command("docker", "save", "--output", "ci-hcloud-ccm.tar", fmt.Sprintf("hcloud-ccm:ci_%s", testIdentifier))
+		cmd.Stdout = os.Stdout
+		err = cmd.Run()
+		if err != nil {
+			tc.err = fmt.Errorf("%s:%s\n", op, err)
+			return
+		}
+
+		tc.setup = &hcloudK8sSetup{Hcloud: hcloudClient, K8sVersion: k8sVersion, TestIdentifier: testIdentifier, HcloudToken: token}
+		fmt.Println("Setting up test env")
+
+		err = tc.setup.PrepareTestEnv(context.Background(), additionalSSHKeys)
+		if err != nil {
+			tc.err = fmt.Errorf("%s:%s\n", op, err)
+			return
+		}
+
+		kubeconfigPath, err := tc.setup.PrepareK8s(tc.useNetworks)
+		if err != nil {
+			tc.err = fmt.Errorf("%s:%s\n", op, err)
+			return
+		}
+
+		config, err := clientcmd.BuildConfigFromFlags("", kubeconfigPath)
+		if err != nil {
+			fmt.Printf("%s: clientcmd.BuildConfigFromFlags: %s", op, err)
+		}
+
+		tc.k8sClient, err = kubernetes.NewForConfig(config)
+		if err != nil {
+			fmt.Printf("%s: kubernetes.NewForConfig: %s", op, err)
+		}
+
+		tc.started = true
+	})
+	return tc.err
+}
+
+func (tc *TestCluster) Start() error {
+	if err := tc.initialize(); err != nil {
+		return fmt.Errorf("start test cluster: %v", err)
+	}
+	if err := tc.ensureNodesAreReady(); err != nil {
+		return fmt.Errorf("start test cluster: %v", err)
+	}
+	return nil
+}
+
+func (tc *TestCluster) Stop() error {
+	const op = "TestCluster/Stop"
+	if tc.err != nil {
+		return fmt.Errorf("%s: %s", op, tc.err)
+	}
+	if !tc.started {
+		return nil
+	}
+	err := tc.setup.TearDown(context.Background())
+	if err != nil {
+		fmt.Printf("%s: Tear Down: %s", op, err)
+	}
+	return nil
+}
+
+func (tc *TestCluster) ensureNodesAreReady() error {
+	const op = "ensureNodesAreReady"
+	err := wait.Poll(1*time.Second, 5*time.Minute, func() (bool, error) {
+		nodes, err := tc.k8sClient.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{})
+		if err != nil {
+			return false, err
+		}
+		available := false
+		for _, node := range nodes.Items {
+			for _, cond := range node.Status.Conditions {
+				if cond.Type == corev1.NodeReady && cond.Status == corev1.ConditionTrue {
+					available = true
+				}
+			}
+		}
+		return available, nil
+	})
+	if err != nil {
+		return fmt.Errorf("%s: Nodes did not be ready after at least 5 minutes: %s", op, err)
+	}
+
+	return nil
+}
+
+type lbTestHelper struct {
+	podName   string
+	K8sClient *kubernetes.Clientset
+	t         *testing.T
+}
+
+// DeployTestPod deploys a basic nginx pod within the k8s cluster
+// and waits until it is "ready"
+func (l *lbTestHelper) DeployTestPod() *corev1.Pod {
+	const op = "lbTestHelper/DeployTestPod"
+	podName := fmt.Sprintf("pod-%s", l.podName)
+	testPod := corev1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: podName,
+			Labels: map[string]string{
+				"app": podName,
+			},
+		},
+		Spec: corev1.PodSpec{
+			Containers: []corev1.Container{
+				{
+					Name:  "nginx-hello-world",
+					Image: "nginxdemos/hello:plain-text",
+					Ports: []corev1.ContainerPort{
+						{
+							ContainerPort: 80,
+							Name:          "http",
+						},
+					},
+				},
+			},
+		},
+	}
+
+	pod, err := l.K8sClient.CoreV1().Pods(corev1.NamespaceDefault).Create(context.Background(), &testPod, metav1.CreateOptions{})
+	if err != nil {
+		l.t.Fatalf("%s: could not create test pod: %s", op, err)
+	}
+	err = wait.Poll(1*time.Second, 1*time.Minute, func() (done bool, err error) {
+		p, err := l.K8sClient.CoreV1().Pods(corev1.NamespaceDefault).Get(context.Background(), podName, metav1.GetOptions{})
+		if err != nil {
+			return false, err
+		}
+		for _, condition := range p.Status.Conditions {
+			if condition.Type == corev1.PodReady && condition.Status == corev1.ConditionTrue {
+				return true, nil
+			}
+		}
+		pod = p
+		return false, nil
+	})
+	if err != nil {
+		l.t.Fatalf("%s: pod %s did not come up after 1 minute: %s", op, podName, err)
+	}
+	return pod
+}
+
+// ServiceDefinition returns a service definition for a Hetzner Cloud Load Balancer (k8s service)
+func (l *lbTestHelper) ServiceDefinition(pod *corev1.Pod, annotations map[string]string) *corev1.Service {
+	return &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:        fmt.Sprintf("svc-%s", l.podName),
+			Annotations: annotations,
+		},
+		Spec: corev1.ServiceSpec{
+			Selector: map[string]string{
+				"app": pod.Name,
+			},
+			Type: corev1.ServiceTypeLoadBalancer,
+			Ports: []corev1.ServicePort{
+				{
+					Port: 80,
+					Name: "http",
+				},
+			},
+			ExternalTrafficPolicy: corev1.ServiceExternalTrafficPolicyTypeLocal,
+		},
+	}
+}
+
+// CreateService creates a k8s service based on the given service definition
+// and waits until it is "ready"
+func (l *lbTestHelper) CreateService(lbSvc *corev1.Service) (*corev1.Service, error) {
+	const op = "lbTestHelper/CreateService"
+	_, err := l.K8sClient.CoreV1().Services(corev1.NamespaceDefault).Create(context.Background(), lbSvc, metav1.CreateOptions{})
+	if err != nil {
+		return nil, fmt.Errorf("%s: could not create service: %s", op, err)
+	}
+
+	err = wait.Poll(1*time.Second, 5*time.Minute, func() (done bool, err error) {
+		svc, err := l.K8sClient.CoreV1().Services(corev1.NamespaceDefault).Get(context.Background(), lbSvc.Name, metav1.GetOptions{})
+		if err != nil {
+			return false, err
+		}
+		ingressIPs := svc.Status.LoadBalancer.Ingress
+		if len(ingressIPs) > 0 {
+			lbSvc = svc
+			return true, nil
+		}
+		return false, nil
+	})
+	if err != nil {
+		return nil, fmt.Errorf("%s: test service (load balancer) did not come up after 1 minute: %s", op, err)
+	}
+	return lbSvc, nil
+}
+
+// TearDown deletes the created pod and service
+func (l *lbTestHelper) TearDown() {
+	const op = "lbTestHelper/TearDown"
+	svcName := fmt.Sprintf("svc-%s", l.podName)
+	err := l.K8sClient.CoreV1().Services(corev1.NamespaceDefault).Delete(context.Background(), svcName, metav1.DeleteOptions{})
+	if err != nil && !k8serrors.IsNotFound(err) {
+		l.t.Errorf("%s: deleting test svc failed: %s", op, err)
+	}
+
+	err = wait.Poll(1*time.Second, 1*time.Minute, func() (done bool, err error) {
+		_, err = l.K8sClient.CoreV1().Services(corev1.NamespaceDefault).Get(context.Background(), svcName, metav1.GetOptions{})
+		if err != nil {
+			if k8serrors.IsNotFound(err) {
+				return true, nil
+			}
+			return false, err
+		}
+		return false, nil
+	})
+	if err != nil {
+		l.t.Errorf("%s: test service was not removed after 1 minute: %s", op, err)
+	}
+
+	podName := fmt.Sprintf("pod-%s", l.podName)
+	err = l.K8sClient.CoreV1().Pods(corev1.NamespaceDefault).Delete(context.Background(), podName, metav1.DeleteOptions{})
+	if err != nil && !k8serrors.IsNotFound(err) {
+		l.t.Errorf("%s: deleting test pod failed: %s", op, err)
+	}
+	err = wait.Poll(1*time.Second, 1*time.Minute, func() (done bool, err error) {
+		_, err = l.K8sClient.CoreV1().Pods(corev1.NamespaceDefault).Get(context.Background(), podName, metav1.GetOptions{})
+		if err != nil {
+			if k8serrors.IsNotFound(err) {
+				return true, nil
+			}
+			return false, err
+		}
+		return false, nil
+	})
+	if err != nil {
+		l.t.Errorf("%s: test pod not removed after 1 minute: %s", op, err)
+	}
+}
+
+// WaitForHttpAvailable tries to connect to the given IP via http
+// It tries it for 2 minutes, if after two minutes the connection
+// wasn't successful and it wasn't a HTTP 200 response it will fail
+func (l *lbTestHelper) WaitForHttpAvailable(ingressIP string) {
+	const op = "lbTestHelper/WaitForHttpAvailable"
+	client := &http.Client{
+		Timeout: 1 * time.Second,
+	}
+	err := wait.Poll(1*time.Second, 2*time.Minute, func() (bool, error) {
+		resp, err := client.Get(fmt.Sprintf("http://%s", ingressIP))
+		if err != nil {
+			return false, nil
+		}
+		defer resp.Body.Close()
+		if resp.StatusCode != http.StatusOK {
+			return false, fmt.Errorf("%s: got HTTP Code %d instead of 200", op, resp.StatusCode)
+		}
+		return true, nil
+	})
+	if err != nil {
+		l.t.Errorf("%s: not available via client.Get: %s", op, err)
+	}
+}
+
+type nwTestHelper struct {
+	podName    string
+	K8sClient  *kubernetes.Clientset
+	privateKey string
+	server     *hcloud.Server
+	t          *testing.T
+}
+
+// DeployTestPod deploys a basic nginx pod within the k8s cluster
+// and waits until it is "ready"
+func (n *nwTestHelper) DeployTestPod() *corev1.Pod {
+	const op = "nwTestHelper/DeployTestPod"
+	podName := fmt.Sprintf("pod-%s", n.podName)
+	testPod := corev1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: podName,
+			Labels: map[string]string{
+				"app": podName,
+			},
+		},
+		Spec: corev1.PodSpec{
+			Containers: []corev1.Container{
+				{
+					Name:  "nginx-hello-world",
+					Image: "nginxdemos/hello:plain-text",
+					Ports: []corev1.ContainerPort{
+						{
+							ContainerPort: 80,
+							Name:          "http",
+						},
+					},
+				},
+			},
+		},
+	}
+
+	pod, err := n.K8sClient.CoreV1().Pods(corev1.NamespaceDefault).Create(context.Background(), &testPod, metav1.CreateOptions{})
+	if err != nil {
+		n.t.Fatalf("%s: could not create test pod: %s", op, err)
+	}
+	err = wait.Poll(1*time.Second, 1*time.Minute, func() (done bool, err error) {
+		p, err := n.K8sClient.CoreV1().Pods(corev1.NamespaceDefault).Get(context.Background(), podName, metav1.GetOptions{})
+		if err != nil {
+			return false, err
+		}
+		for _, condition := range p.Status.Conditions {
+			if condition.Type == corev1.PodReady && condition.Status == corev1.ConditionTrue {
+				return true, nil
+			}
+		}
+		pod = p
+		return false, nil
+	})
+	if err != nil {
+		n.t.Fatalf("%s: pod %s did not come up after 1 minute: %s", op, podName, err)
+	}
+	pod, err = n.K8sClient.CoreV1().Pods(corev1.NamespaceDefault).Get(context.Background(), podName, metav1.GetOptions{})
+	if err != nil {
+		n.t.Fatalf("%s: could not create test pod: %s", op, err)
+	}
+	return pod
+}
+
+// WaitForHttpAvailable tries to connect to the given IP via curl
+// It tries it for 2 minutes, if after two minutes the connection
+// wasn't successful and it wasn't a HTTP 200 response it will fail
+func (n *nwTestHelper) WaitForHttpAvailable(podIp string) {
+	const op = "nwTestHelper/WaitForHttpAvailable"
+	err := wait.Poll(1*time.Second, 2*time.Minute, func() (bool, error) {
+		err := RunCommandOnServer(n.privateKey, n.server, fmt.Sprintf("curl http://%s", podIp))
+		if err != nil {
+			return false, nil
+		}
+		return true, nil
+	})
+	if err != nil {
+		n.t.Errorf("%s: not available via curl: %s", op, err)
+	}
+}
+
+// TearDown deletes the created pod
+func (n *nwTestHelper) TearDown() {
+	const op = "nwTestHelper/TearDown"
+	podName := fmt.Sprintf("pod-%s", n.podName)
+	err := n.K8sClient.CoreV1().Pods(corev1.NamespaceDefault).Delete(context.Background(), podName, metav1.DeleteOptions{})
+	if err != nil && !k8serrors.IsNotFound(err) {
+		n.t.Errorf("%s: deleting test pod failed: %s", op, err)
+	}
+	err = wait.Poll(1*time.Second, 1*time.Minute, func() (done bool, err error) {
+		_, err = n.K8sClient.CoreV1().Pods(corev1.NamespaceDefault).Get(context.Background(), podName, metav1.GetOptions{})
+		if err != nil {
+			if k8serrors.IsNotFound(err) {
+				return true, nil
+			}
+			return false, err
+		}
+		return false, nil
+	})
+	if err != nil {
+		n.t.Errorf("%s: test pod not removed after 1 minute: %s", op, err)
+	}
+}