Repository: hetznercloud/hcloud-cloud-controller-manager
Pull request #106 (2020-10-28 07:02:23)

Add option to keep server after e2e test failure
This allows manual analysis of the environment after a test failure.


Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8502e33b..de0603a1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,11 @@
 Changelog
 =========
 
+v1.8.1
+----------
+
+* Fix: excessive calls to `/v1/servers` endpoint.
+
 v1.8.0
 ------
 * Fix: nil pointer dereference when Load Balancers were disabled
diff --git a/README.md b/README.md
index d5e5cbd4..7d13c91c 100644
--- a/README.md
+++ b/README.md
@@ -112,9 +112,9 @@ If you want to use the Hetzner Cloud `Networks` Feature, head over to the [Deplo
 
 ## E2E Tests
 
-The Hetzner Cloud cloud controller manager was tested against all supported Kubernetes versions. You can run the tests with the following commands. Keep in mind, that these tests run on real cloud servers and will create Load Balancers that will be billed. 
+The Hetzner Cloud cloud controller manager was tested against all supported Kubernetes versions. You can run the tests with the following commands. Keep in mind, that these tests run on real cloud servers and will create Load Balancers that will be billed.
 
-**Test Server Setup:** 
+**Test Server Setup:**
 1x CPX21 (Ubuntu 18.04)
 
 **Requirements: Docker and Go 1.15**
@@ -128,6 +128,7 @@ export USE_SSH_KEYS=key1,key2 # Name or IDs of your SSH Keys within the Hetzner
 export USE_NETWORKS=yes # if `yes` this identidicates that the tests should provision the server with cilium as CNI and also enable the Network related tests
 ## Optional configuration env vars:
 export TEST_DEBUG_MODE=yes # With this env you can toggle the output of the provision and test commands. With `yes` it will log the whole output to stdout
+export KEEP_SERVER_ON_FAILURE=yes # Keep the test server after a test failure.
 ```
 
 2. Run the tests
diff --git a/deploy/ccm-networks.yaml b/deploy/ccm-networks.yaml
index 5d12277a..796ca1b6 100644
--- a/deploy/ccm-networks.yaml
+++ b/deploy/ccm-networks.yaml
@@ -54,7 +54,7 @@ spec:
           effect: "NoSchedule"
       hostNetwork: true
       containers:
-        - image: hetznercloud/hcloud-cloud-controller-manager:v1.8.0
+        - image: hetznercloud/hcloud-cloud-controller-manager:v1.8.1
           name: hcloud-cloud-controller-manager
           command:
             - "/bin/hcloud-cloud-controller-manager"
diff --git a/deploy/ccm.yaml b/deploy/ccm.yaml
index 4972b685..6fd3818f 100644
--- a/deploy/ccm.yaml
+++ b/deploy/ccm.yaml
@@ -54,7 +54,7 @@ spec:
         - key: "node.kubernetes.io/not-ready"
           effect: "NoSchedule"
       containers:
-        - image: hetznercloud/hcloud-cloud-controller-manager:v1.8.0
+        - image: hetznercloud/hcloud-cloud-controller-manager:v1.8.1
           name: hcloud-cloud-controller-manager
           command:
             - "/bin/hcloud-cloud-controller-manager"
diff --git a/hcloud/cloud.go b/hcloud/cloud.go
index 5b54fded..3995338e 100644
--- a/hcloud/cloud.go
+++ b/hcloud/cloud.go
@@ -36,7 +36,7 @@ const (
 	hcloudLoadBalancersEnabledENVVar = "HCLOUD_LOAD_BALANCERS_ENABLED"
 	nodeNameENVVar                   = "NODE_NAME"
 	providerName                     = "hcloud"
-	providerVersion                  = "v1.8.0"
+	providerVersion                  = "v1.8.1"
 )
 
 type cloud struct {
diff --git a/hcloud/routes.go b/hcloud/routes.go
index 10d4f866..6cfb24bb 100644
--- a/hcloud/routes.go
+++ b/hcloud/routes.go
@@ -16,8 +16,7 @@ import (
 type routes struct {
 	client      *hcloud.Client
 	network     *hcloud.Network
-	serverNames map[string]string
-	serverIPs   map[string]net.IP
+	serverCache *hcops.AllServersCache
 }
 
 func newRoutes(client *hcloud.Client, networkID int) (*routes, error) {
@@ -31,7 +30,11 @@ func newRoutes(client *hcloud.Client, networkID int) (*routes, error) {
 		return nil, fmt.Errorf("network not found: %d", networkID)
 	}
 
-	return &routes{client, networkObj, make(map[string]string), make(map[string]net.IP)}, nil
+	return &routes{
+		client:      client,
+		network:     networkObj,
+		serverCache: &hcops.AllServersCache{LoadFunc: client.Server.All},
+	}, nil
 }
 
 func (r *routes) reloadNetwork(ctx context.Context) error {
@@ -48,45 +51,21 @@ func (r *routes) reloadNetwork(ctx context.Context) error {
 	return nil
 }
 
-func (r *routes) loadServers(ctx context.Context) error {
-	const op = "hcloud/loadServers"
-
-	serversRaw, err := r.client.Server.All(ctx)
-	if err != nil {
-		return fmt.Errorf("%s: %w", op, err)
-	}
-
-	for _, server := range serversRaw {
-		for _, privateNet := range server.PrivateNet {
-			if privateNet.Network.ID == r.network.ID {
-				r.serverNames[privateNet.IP.String()] = server.Name
-				r.serverIPs[server.Name] = privateNet.IP
-				break
-			}
-		}
-	}
-	return nil
-}
-
 // ListRoutes lists all managed routes that belong to the specified clusterName
 func (r *routes) ListRoutes(ctx context.Context, clusterName string) ([]*cloudprovider.Route, error) {
 	const op = "hcloud/ListRoutes"
-	var routes []*cloudprovider.Route
 
 	if err := r.reloadNetwork(ctx); err != nil {
-		return routes, fmt.Errorf("%s: %w", op, err)
-	}
-
-	if err := r.loadServers(ctx); err != nil {
-		return routes, fmt.Errorf("%s: %w", op, err)
+		return nil, fmt.Errorf("%s: %w", op, err)
 	}
 
-	for _, route := range r.network.Routes {
+	routes := make([]*cloudprovider.Route, len(r.network.Routes))
+	for i, route := range r.network.Routes {
 		r, err := r.hcloudRouteToRoute(route)
 		if err != nil {
 			return routes, fmt.Errorf("%s: %w", op, err)
 		}
-		routes = append(routes, r)
+		routes[i] = r
 	}
 	return routes, nil
 }
@@ -97,14 +76,16 @@ func (r *routes) ListRoutes(ctx context.Context, clusterName string) ([]*cloudpr
 func (r *routes) CreateRoute(ctx context.Context, clusterName string, nameHint string, route *cloudprovider.Route) error {
 	const op = "hcloud/CreateRoute"
 
-	if err := r.loadServers(ctx); err != nil {
-		return fmt.Errorf("%s: %w", op, err)
+	srv, err := r.serverCache.ByName(string(route.TargetNode))
+	if err != nil {
+		return fmt.Errorf("%s: %v", op, err)
 	}
 
-	ip, ok := r.serverIPs[string(route.TargetNode)]
+	privNet, ok := findServerPrivateNetByID(srv, r.network.ID)
 	if !ok {
-		return fmt.Errorf("%s: server %v: not found", op, route.TargetNode)
+		return fmt.Errorf("%s: server %v: no network with id: %d", op, route.TargetNode, r.network.ID)
 	}
+	ip := privNet.IP
 
 	_, cidr, err := net.ParseCIDR(route.DestinationCIDR)
 	if err != nil {
@@ -148,14 +129,15 @@ func (r *routes) CreateRoute(ctx context.Context, clusterName string, nameHint s
 func (r *routes) DeleteRoute(ctx context.Context, clusterName string, route *cloudprovider.Route) error {
 	const op = "hcloud/DeleteRoute"
 
-	if err := r.loadServers(ctx); err != nil {
-		return fmt.Errorf("%s: %w", op, err)
+	srv, err := r.serverCache.ByName(string(route.TargetNode))
+	if err != nil {
+		return fmt.Errorf("%s: %v", op, err)
 	}
-
-	ip, ok := r.serverIPs[string(route.TargetNode)]
+	privNet, ok := findServerPrivateNetByID(srv, r.network.ID)
 	if !ok {
-		return fmt.Errorf("%s: server %v: not found", op, route.TargetNode)
+		return fmt.Errorf("%s: server %v: no network with id: %d", op, route.TargetNode, r.network.ID)
 	}
+	ip := privNet.IP
 
 	_, cidr, err := net.ParseCIDR(route.DestinationCIDR)
 	if err != nil {
@@ -189,10 +171,11 @@ func (r *routes) DeleteRoute(ctx context.Context, clusterName string, route *clo
 func (r *routes) hcloudRouteToRoute(route hcloud.NetworkRoute) (*cloudprovider.Route, error) {
 	const op = "hcloud/hcloudRouteToRoute"
 
-	nodeName, ok := r.serverNames[route.Gateway.String()]
-	if !ok {
-		return nil, fmt.Errorf("%s: server with IP %v: not found", op, route.Gateway)
+	srv, err := r.serverCache.ByPrivateIP(route.Gateway)
+	if err != nil {
+		return nil, fmt.Errorf("%s: %v", op, err)
 	}
+	nodeName := srv.Name
 
 	return &cloudprovider.Route{
 		DestinationCIDR: route.Destination.String(),
@@ -210,10 +193,15 @@ func (r *routes) checkIfRouteAlreadyExists(ctx context.Context, route *cloudprov
 
 	for _, _route := range r.network.Routes {
 		if _route.Destination.String() == route.DestinationCIDR {
-			ip, ok := r.serverIPs[string(route.TargetNode)]
+			srv, err := r.serverCache.ByName(string(route.TargetNode))
+			if err != nil {
+				return false, fmt.Errorf("%s: %v", op, err)
+			}
+			privNet, ok := findServerPrivateNetByID(srv, r.network.ID)
 			if !ok {
-				return false, fmt.Errorf("%s: server %v: not found", op, string(route.TargetNode))
+				return false, fmt.Errorf("%s: server %v: no network with id: %d", op, route.TargetNode, r.network.ID)
 			}
+			ip := privNet.IP
 
 			if !_route.Gateway.Equal(ip) {
 				action, _, err := r.client.Network.DeleteRoute(context.Background(), r.network, hcloud.NetworkDeleteRouteOpts{
@@ -233,3 +221,12 @@ func (r *routes) checkIfRouteAlreadyExists(ctx context.Context, route *cloudprov
 	}
 	return false, nil
 }
+
+func findServerPrivateNetByID(srv *hcloud.Server, id int) (hcloud.ServerPrivateNet, bool) {
+	for _, n := range srv.PrivateNet {
+		if n.Network.ID == id {
+			return n, true
+		}
+	}
+	return hcloud.ServerPrivateNet{}, false
+}
diff --git a/internal/hcops/server.go b/internal/hcops/server.go
new file mode 100644
index 00000000..3aec08c7
--- /dev/null
+++ b/internal/hcops/server.go
@@ -0,0 +1,126 @@
+package hcops
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"sync"
+	"time"
+
+	"github.com/hetznercloud/hcloud-go/hcloud"
+)
+
+// AllServersCache caches the result of the LoadFunc and provides random access
+// to servers using select hcloud.Server attributes.
+//
+// To simplify things the allServersCache reloads all servers on every cache
+// miss, or whenever a timeout expired.
+type AllServersCache struct {
+	LoadFunc    func(context.Context) ([]*hcloud.Server, error)
+	LoadTimeout time.Duration
+	MaxAge      time.Duration
+
+	lastRefresh time.Time
+	byPrivIP    map[string]*hcloud.Server
+	byName      map[string]*hcloud.Server
+
+	mu sync.Mutex // protects by* maps
+}
+
+// ByPrivateIP obtains a server from the cache using the IP of one of its
+// private networks.
+//
+// Note that a pointer to the object stored in the cache is returned. Modifying
+// this object affects the cache and all other code parts holding a reference.
+// Furthermore modifying the returned server is not concurrency safe.
+func (c *AllServersCache) ByPrivateIP(ip net.IP) (*hcloud.Server, error) {
+	const op = "hcops/AllServersCache.ByPrivateIP"
+
+	srv, err := c.getCache(func() (*hcloud.Server, bool) {
+		srv, ok := c.byPrivIP[ip.String()]
+		return srv, ok
+	})
+	if err != nil {
+		return nil, fmt.Errorf("%s: %w", op, err)
+	}
+
+	return srv, nil
+}
+
+// ByName obtains a server from the cache using the servers name.
+//
+// Note that a pointer to the object stored in the cache is returned. Modifying
+// this object affects the cache and all other code parts holding a reference.
+// Furthermore modifying the returned server is not concurrency safe.
+func (c *AllServersCache) ByName(name string) (*hcloud.Server, error) {
+	const op = "hcops/AllServersCache.ByName"
+
+	srv, err := c.getCache(func() (*hcloud.Server, bool) {
+		srv, ok := c.byName[name]
+		return srv, ok
+	})
+	if err != nil {
+		return nil, fmt.Errorf("%s: %w", op, err)
+	}
+
+	return srv, nil
+}
+
+func (c *AllServersCache) getCache(getSrv func() (*hcloud.Server, bool)) (*hcloud.Server, error) {
+	const op = "hcops/AllServersCache.getCache"
+
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	// First try to get the value from the cache if the cache is not yet
+	// expired.
+	if srv, ok := getSrv(); ok && !c.isExpired() {
+		return srv, nil
+	}
+
+	// Reload from the backend API if we didn't find srv.
+	to := c.LoadTimeout
+	if to == 0 {
+		to = 5 * time.Second
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), to)
+	defer cancel()
+
+	srvs, err := c.LoadFunc(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("%s: %w", op, err)
+	}
+
+	// Re-initialize all maps. This effectively clears the current cache.
+	c.byPrivIP = make(map[string]*hcloud.Server)
+	c.byName = make(map[string]*hcloud.Server)
+
+	for _, srv := range srvs {
+		// Index servers by the IPs of their private networks
+		for _, n := range srv.PrivateNet {
+			if n.IP == nil {
+				continue
+			}
+			c.byPrivIP[n.IP.String()] = srv
+		}
+
+		// Index servers by their names.
+		c.byName[srv.Name] = srv
+	}
+
+	c.lastRefresh = time.Now()
+
+	// Re-try to find the server after the reload.
+	if srv, ok := getSrv(); ok {
+		return srv, nil
+	}
+	return nil, fmt.Errorf("%s: %w", op, ErrNotFound)
+}
+
+func (c *AllServersCache) isExpired() bool {
+	maxAge := c.MaxAge
+	if maxAge == 0 {
+		maxAge = 10 * time.Minute
+	}
+	return time.Now().After(c.lastRefresh.Add(maxAge))
+}
diff --git a/internal/mocks/server.go b/internal/mocks/server.go
new file mode 100644
index 00000000..32d6d712
--- /dev/null
+++ b/internal/mocks/server.go
@@ -0,0 +1,43 @@
+package mocks
+
+import (
+	"context"
+	"testing"
+
+	"github.com/hetznercloud/hcloud-go/hcloud"
+	"github.com/stretchr/testify/mock"
+)
+
+// ServerClient is a mock implementation of the hcloud.ServerClient
+type ServerClient struct {
+	mock.Mock
+	T *testing.T
+}
+
+// NewServerClient creates a new mock server client ready for use.
+func NewServerClient(t *testing.T) *ServerClient {
+	m := &ServerClient{T: t}
+	m.Test(t)
+	return m
+}
+
+// All registers a call to obtain all servers from the Hetzner Cloud API.
+func (m *ServerClient) All(ctx context.Context) ([]*hcloud.Server, error) {
+	args := m.Called(ctx)
+	return serverPtrSlice(m.T, args.Get(0)), args.Error(1)
+}
+
+func serverPtrSlice(t *testing.T, v interface{}) []*hcloud.Server {
+	const op = "mocks/serverPtrSlice"
+
+	t.Helper()
+
+	if v == nil {
+		return nil
+	}
+	ss, ok := v.([]*hcloud.Server)
+	if !ok {
+		t.Fatalf("%s: not a []*Server: %t", op, v)
+	}
+	return ss
+}


Test patch:
diff --git a/e2etests/e2e_test.go b/e2etests/e2e_test.go
index bb89b9b8..364d83c1 100644
--- a/e2etests/e2e_test.go
+++ b/e2etests/e2e_test.go
@@ -23,7 +23,7 @@ func TestMain(m *testing.M) {
 
 	rc := m.Run()
 
-	if err := testCluster.Stop(); err != nil {
+	if err := testCluster.Stop(rc > 0); err != nil {
 		fmt.Printf("%v\n", err)
 		os.Exit(1)
 	}
diff --git a/e2etests/setup.go b/e2etests/setup.go
index 02ef7343..3066b22a 100644
--- a/e2etests/setup.go
+++ b/e2etests/setup.go
@@ -27,6 +27,7 @@ type hcloudK8sSetup struct {
 	HcloudToken    string
 	K8sVersion     string
 	TestIdentifier string
+	KeepOnFailure  bool
 	privKey        string
 	server         *hcloud.Server
 	sshKey         *hcloud.SSHKey
@@ -283,8 +284,17 @@ func (s *hcloudK8sSetup) waitForCloudInit() error {
 // TearDown deletes all created resources within the Hetzner Cloud
 // there is no need to "shutdown" the k8s cluster before
 // so we just delete all created resources
-func (s *hcloudK8sSetup) TearDown(ctx context.Context) error {
+func (s *hcloudK8sSetup) TearDown(testFailed bool) error {
 	const op = "hcloudK8sSetup/TearDown"
+
+	if s.KeepOnFailure && testFailed {
+		fmt.Println("Skipping tear-down for further analysis.")
+		fmt.Println("Please clean-up afterwards ;-)")
+		return nil
+	}
+
+	ctx := context.Background()
+
 	_, err := s.Hcloud.Server.Delete(ctx, s.server)
 	if err != nil {
 		return fmt.Errorf("%s Hcloud.Server.Delete: %s", op, err)
diff --git a/e2etests/testing.go b/e2etests/testing.go
index 3b31f0ea..fd14ba42 100644
--- a/e2etests/testing.go
+++ b/e2etests/testing.go
@@ -67,9 +67,10 @@ func (tc *TestCluster) initialize() error {
 		}
 		token := os.Getenv("HCLOUD_TOKEN")
 		if len(token) != 64 {
-			tc.err = fmt.Errorf("%s: No valid HCLOUD_TOKEN found\n", op)
+			tc.err = fmt.Errorf("%s: No valid HCLOUD_TOKEN found", op)
 			return
 		}
+		keepOnFailure := os.Getenv("KEEP_SERVER_ON_FAILURE") == "yes"
 
 		var additionalSSHKeys []*hcloud.SSHKey
 
@@ -78,18 +79,17 @@ func (tc *TestCluster) initialize() error {
 			hcloud.WithApplication("hcloud-ccm-testsuite", "1.0"),
 		}
 		hcloudClient := hcloud.NewClient(opts...)
-		additionalSSHKeysIdOrName := os.Getenv("USE_SSH_KEYS")
-		if additionalSSHKeysIdOrName != "" {
-			idsOrNames := strings.Split(additionalSSHKeysIdOrName, ",")
+		additionalSSHKeysIDOrName := os.Getenv("USE_SSH_KEYS")
+		if additionalSSHKeysIDOrName != "" {
+			idsOrNames := strings.Split(additionalSSHKeysIDOrName, ",")
 			for _, idOrName := range idsOrNames {
 				additionalSSHKey, _, err := hcloudClient.SSHKey.Get(context.Background(), idOrName)
 				if err != nil {
-					tc.err = fmt.Errorf("%s:%s\n", op, err)
+					tc.err = fmt.Errorf("%s: %s", op, err)
 					return
 				}
 				additionalSSHKeys = append(additionalSSHKeys, additionalSSHKey)
 			}
-
 		}
 
 		fmt.Printf("Test against k8s %s\n", k8sVersion)
@@ -99,7 +99,7 @@ func (tc *TestCluster) initialize() error {
 		cmd.Stdout = os.Stdout
 		err := cmd.Run()
 		if err != nil {
-			tc.err = fmt.Errorf("%s:%s\n", op, err)
+			tc.err = fmt.Errorf("%s: %s", op, err)
 			return
 		}
 
@@ -108,22 +108,28 @@ func (tc *TestCluster) initialize() error {
 		cmd.Stdout = os.Stdout
 		err = cmd.Run()
 		if err != nil {
-			tc.err = fmt.Errorf("%s:%s\n", op, err)
+			tc.err = fmt.Errorf("%s: %s", op, err)
 			return
 		}
 
-		tc.setup = &hcloudK8sSetup{Hcloud: hcloudClient, K8sVersion: k8sVersion, TestIdentifier: testIdentifier, HcloudToken: token}
+		tc.setup = &hcloudK8sSetup{
+			Hcloud:         hcloudClient,
+			K8sVersion:     k8sVersion,
+			TestIdentifier: testIdentifier,
+			HcloudToken:    token,
+			KeepOnFailure:  keepOnFailure,
+		}
 		fmt.Println("Setting up test env")
 
 		err = tc.setup.PrepareTestEnv(context.Background(), additionalSSHKeys)
 		if err != nil {
-			tc.err = fmt.Errorf("%s:%s\n", op, err)
+			tc.err = fmt.Errorf("%s: %s", op, err)
 			return
 		}
 
 		kubeconfigPath, err := tc.setup.PrepareK8s(tc.useNetworks)
 		if err != nil {
-			tc.err = fmt.Errorf("%s:%s\n", op, err)
+			tc.err = fmt.Errorf("%s: %s", op, err)
 			return
 		}
 
@@ -152,7 +158,7 @@ func (tc *TestCluster) Start() error {
 	return nil
 }
 
-func (tc *TestCluster) Stop() error {
+func (tc *TestCluster) Stop(testFailed bool) error {
 	const op = "TestCluster/Stop"
 	if tc.err != nil {
 		return fmt.Errorf("%s: %s", op, tc.err)
@@ -160,7 +166,7 @@ func (tc *TestCluster) Stop() error {
 	if !tc.started {
 		return nil
 	}
-	err := tc.setup.TearDown(context.Background())
+	err := tc.setup.TearDown(testFailed)
 	if err != nil {
 		fmt.Printf("%s: Tear Down: %s", op, err)
 	}
@@ -182,6 +188,18 @@ func (tc *TestCluster) ensureNodesAreReady() error {
 				}
 			}
 		}
+
+		pods, err := tc.k8sClient.CoreV1().Pods("kube-system").List(context.Background(), metav1.ListOptions{})
+		if err != nil {
+			return false, err
+		}
+		for _, pod := range pods.Items {
+			for _, cond := range pod.Status.Conditions {
+				if cond.Type == corev1.PodReady && cond.Status == corev1.ConditionTrue {
+					available = available && true
+				}
+			}
+		}
 		return available, nil
 	})
 	if err != nil {
diff --git a/internal/hcops/server_test.go b/internal/hcops/server_test.go
new file mode 100644
index 00000000..22de059d
--- /dev/null
+++ b/internal/hcops/server_test.go
@@ -0,0 +1,231 @@
+package hcops_test
+
+import (
+	"errors"
+	"fmt"
+	"net"
+	"testing"
+	"time"
+
+	"github.com/hetznercloud/hcloud-cloud-controller-manager/internal/hcops"
+	"github.com/hetznercloud/hcloud-cloud-controller-manager/internal/mocks"
+	"github.com/hetznercloud/hcloud-go/hcloud"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+)
+
+func TestAllServersCache_CacheMiss(t *testing.T) {
+	srv := &hcloud.Server{
+		ID:   12345,
+		Name: "cache-miss",
+		PrivateNet: []hcloud.ServerPrivateNet{
+			{
+				IP: net.ParseIP("10.0.0.2"),
+			},
+		},
+	}
+	cacheOps := newAllServersCacheOps(t, srv)
+	tmpl := allServersCacheTestCase{
+		SetUp: func(t *testing.T, tt *allServersCacheTestCase) {
+			tt.ServerClient.
+				On("All", mock.Anything).
+				Return([]*hcloud.Server{srv}, nil)
+		},
+		Expected: srv,
+	}
+
+	runAllServersCacheTests(t, "Cache miss", tmpl, cacheOps)
+}
+
+func TestAllServersCache_CacheHit(t *testing.T) {
+	srv := &hcloud.Server{
+		ID:   54321,
+		Name: "cache-hit",
+		PrivateNet: []hcloud.ServerPrivateNet{
+			{
+				IP: net.ParseIP("10.0.0.3"),
+			},
+		},
+	}
+	cacheOps := newAllServersCacheOps(t, srv)
+	tmpl := allServersCacheTestCase{
+		SetUp: func(t *testing.T, tt *allServersCacheTestCase) {
+			tt.ServerClient.
+				On("All", mock.Anything).
+				Return([]*hcloud.Server{srv}, nil).
+				Once()
+
+			// Perform any cache op to initialize caches
+			if _, err := tt.Cache.ByName(srv.Name); err != nil {
+				t.Fatalf("SetUp: %v", err)
+			}
+		},
+		Assert: func(t *testing.T, tt *allServersCacheTestCase) {
+			// All must be called only once. This call has happened during the
+			// test SetUp method. All additional calls indicate an error.
+			tt.ServerClient.AssertNumberOfCalls(t, "All", 1)
+		},
+		Expected: srv,
+	}
+
+	runAllServersCacheTests(t, "Cache hit", tmpl, cacheOps)
+}
+
+func TestAllServersCache_CacheRefresh(t *testing.T) {
+	srv := &hcloud.Server{
+		ID:   56789,
+		Name: "cache-refresh",
+		PrivateNet: []hcloud.ServerPrivateNet{
+			{
+				IP: net.ParseIP("10.0.0.9"),
+			},
+		},
+	}
+	cacheOps := newAllServersCacheOps(t, srv)
+	tmpl := allServersCacheTestCase{
+		SetUp: func(t *testing.T, tt *allServersCacheTestCase) {
+			tt.ServerClient.
+				On("All", mock.Anything).
+				Return([]*hcloud.Server{srv}, nil)
+
+			if _, err := tt.Cache.ByName(srv.Name); err != nil {
+				t.Fatalf("SetUp: %v", err)
+			}
+			// Set the maximum cache age to a ridiculously low time to
+			// speed the test up.
+			tt.Cache.MaxAge = time.Nanosecond
+			time.Sleep(2 * tt.Cache.MaxAge)
+		},
+		Assert: func(t *testing.T, tt *allServersCacheTestCase) {
+			// All must be called only twice. Once during set-up and once
+			// during the refresh because of the cache age expired.
+			tt.ServerClient.AssertNumberOfCalls(t, "All", 2)
+		},
+		Expected: srv,
+	}
+
+	runAllServersCacheTests(t, "Cache refresh", tmpl, cacheOps)
+}
+
+func TestAllServersCache_NotFound(t *testing.T) {
+	srv := &hcloud.Server{
+		ID:   101010,
+		Name: "not-found",
+		PrivateNet: []hcloud.ServerPrivateNet{
+			{
+				IP: net.ParseIP("10.0.0.4"),
+			},
+		},
+	}
+	cacheOps := newAllServersCacheOps(t, srv)
+	tmpl := allServersCacheTestCase{
+		SetUp: func(t *testing.T, tt *allServersCacheTestCase) {
+			tt.ServerClient.
+				On("All", mock.Anything).
+				Return(nil, nil)
+		},
+		ExpectedErr: hcops.ErrNotFound,
+	}
+
+	runAllServersCacheTests(t, "Not found", tmpl, cacheOps)
+}
+
+func TestAllServersCache_ClientError(t *testing.T) {
+	err := errors.New("client-error")
+	srv := &hcloud.Server{
+		ID:   202020,
+		Name: "client-error",
+		PrivateNet: []hcloud.ServerPrivateNet{
+			{
+				IP: net.ParseIP("10.0.0.5"),
+			},
+		},
+	}
+	cacheOps := newAllServersCacheOps(t, srv)
+	tmpl := allServersCacheTestCase{
+		SetUp: func(t *testing.T, tt *allServersCacheTestCase) {
+			tt.ServerClient.
+				On("All", mock.Anything).
+				Return(nil, err)
+		},
+		ExpectedErr: err,
+	}
+
+	runAllServersCacheTests(t, "Not found", tmpl, cacheOps)
+}
+
+type allServersCacheOp func(c *hcops.AllServersCache) (*hcloud.Server, error)
+
+func newAllServersCacheOps(t *testing.T, srv *hcloud.Server) map[string]allServersCacheOp {
+	return map[string]allServersCacheOp{
+		"ByPrivateIP": func(c *hcops.AllServersCache) (*hcloud.Server, error) {
+			if len(srv.PrivateNet) == 0 {
+				t.Fatal("ByPrivateIP: server has no private net")
+			}
+			if len(srv.PrivateNet) > 1 {
+				t.Fatal("ByPrivateIP: server more than one private net")
+			}
+			ip := srv.PrivateNet[0].IP
+			if ip == nil {
+				t.Fatal("ByPrivateIP: server has no private ip")
+			}
+			return c.ByPrivateIP(ip)
+		},
+		"ByName": func(c *hcops.AllServersCache) (*hcloud.Server, error) {
+			if srv.Name == "" {
+				t.Fatal("ByName: server has no name")
+			}
+			return c.ByName(srv.Name)
+		},
+	}
+}
+
+type allServersCacheTestCase struct {
+	SetUp       func(t *testing.T, tt *allServersCacheTestCase)
+	CacheOp     allServersCacheOp
+	Expected    *hcloud.Server
+	ExpectedErr error
+	Assert      func(t *testing.T, tt *allServersCacheTestCase)
+
+	// set in run method.
+	ServerClient *mocks.ServerClient
+	Cache        *hcops.AllServersCache
+}
+
+func (tt *allServersCacheTestCase) run(t *testing.T) {
+	tt.ServerClient = mocks.NewServerClient(t)
+	tt.Cache = &hcops.AllServersCache{
+		LoadFunc: tt.ServerClient.All,
+	}
+
+	if tt.SetUp != nil {
+		tt.SetUp(t, tt)
+	}
+	actual, err := tt.CacheOp(tt.Cache)
+	if tt.ExpectedErr == nil && !assert.NoError(t, err) {
+		return
+	}
+	if tt.ExpectedErr != nil {
+		assert.Truef(t, errors.Is(err, tt.ExpectedErr), "expected error: %v; got %v", tt.ExpectedErr, err)
+		return
+	}
+	assert.Equal(t, tt.Expected, actual)
+
+	if tt.Assert != nil {
+		tt.Assert(t, tt)
+	}
+	tt.ServerClient.AssertExpectations(t)
+}
+
+func runAllServersCacheTests(
+	t *testing.T, name string, tmpl allServersCacheTestCase, cacheOps map[string]allServersCacheOp,
+) {
+	for opName, op := range cacheOps {
+		assert.Nil(t, tmpl.CacheOp) // just make sure tmpl was not modified
+
+		// Copy tmpl as we do not want to modify the original
+		tt := tmpl
+		tt.CacheOp = op
+		t.Run(fmt.Sprintf("%s: %s", name, opName), tt.run)
+	}
+}
diff --git a/scripts/e2etest-local.sh b/scripts/e2etest-local.sh
new file mode 100755
index 00000000..e29a43a4
--- /dev/null
+++ b/scripts/e2etest-local.sh
@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+set -e
+
+function test_k8s_version() {
+    if [[ -z "$1" ]]; then
+        echo "Usage: $0 <k8s-version>"
+        return 1
+    fi
+
+    export K8S_VERSION="$1"
+
+    echo "Testing K8S $K8S_VERSION without network support"
+    export USE_NETWORKS="no"
+    if ! go test -count=1 -v -timeout 60m ./e2etests; then
+        return 2
+    fi
+
+    echo
+    echo
+    echo "Testing K8S $K8S_VERSION with network support"
+    export USE_NETWORKS="yes"
+    if ! go test -count=1 -v -timeout 60m ./e2etests; then
+        return 2
+    fi
+}
+
+if [[ -z "$HCLOUD_TOKEN" ]]; then
+    echo "HCLOUD_TOKEN not set! Aborting tests."
+    exit 1
+fi
+
+K8S_VERSIONS=("1.17.12" "1.18.9" "1.19.3")
+for v in "${K8S_VERSIONS[@]}"; do
+    test_k8s_version "$v"
+done