Repository: hetznercloud/hcloud-cloud-controller-manager
Pull request #556 (2023-11-08 07:06:20)

chore(dev-env): make private network routes feature optional
This feature will not work with the new Robot integration, so we need to make it optional in the dev env.

Related to #526 


Patch:
diff --git a/hack/dev-up.sh b/hack/dev-up.sh
index f276c5d2..921672c9 100755
--- a/hack/dev-up.sh
+++ b/hack/dev-up.sh
@@ -29,6 +29,7 @@ if [[ -n "${DEBUG:-}" ]]; then set -x; fi
   network_cidr=${NETWORK_CIDR:-10.0.0.0/8}
   subnet_cidr=${SUBNET_CIDR:-10.0.0.0/24}
   cluster_cidr=${CLUSTER_CIDR:-10.244.0.0/16}
+  routes_enabled=${ROUTES_ENABLED:-true}
   scope="${SCOPE:-dev}"
   scope=${scope//[^a-zA-Z0-9_]/-}
   scope_name=hccm-${scope}
@@ -95,7 +96,12 @@ if [[ -n "${DEBUG:-}" ]]; then set -x; fi
       $ssh_command root@$ip 'mkdir -p /etc/rancher/k3s && cat > /etc/rancher/k3s/registries.yaml' < $SCRIPT_DIR/k3s-registries.yaml
 
       private_ip=$(hcloud server describe $server_name -o format="{{ (index .PrivateNet 0).IP }}")
-      k3s_node_ip_opts="--node-external-ip ${ip} --node-ip ${private_ip}"
+      k3s_node_ip_opts="--node-ip ${ip}"
+      if [[ "$routes_enabled" == "true" ]]; then
+        # Only advertise the private IP if we have routing enabled, to avoid issues where the nodes can
+        # not communicate with each other on the advertised addresses (ie. Robot Servers)
+        k3s_node_ip_opts="--node-external-ip ${ip} --node-ip ${private_ip}"
+      fi
 
       if [[ "$num" == "1" ]]; then
         # First node is control plane.
@@ -142,10 +148,17 @@ if [[ -n "${DEBUG:-}" ]]; then set -x; fi
     # Install Cilium.
     ( trap error ERR
       if ! helm status -n kube-system cilium >/dev/null 2>&1; then
-        helm upgrade -install cilium cilium --repo https://helm.cilium.io/ -n kube-system --version 1.13.1 \
-          --set tunnel=disabled \
-          --set ipv4NativeRoutingCIDR=$cluster_cidr \
+        values=(
           --set ipam.mode=kubernetes
+        )
+        if [[ "$routes_enabled" == "true" ]]; then
+          # When using the Network Routes, we do not need (or want) Cilium to handle these ranges
+          values+=(
+            --set tunnel=disabled
+            --set ipv4NativeRoutingCIDR="$cluster_cidr"
+          )
+        fi
+        helm upgrade -install cilium cilium --repo https://helm.cilium.io/ -n kube-system --version 1.13.1 "${values[@]}"
       fi) &
 
     # Create HCLOUD_TOKEN Secret for hcloud-cloud-controller-manager.
diff --git a/hcloud/cloud.go b/hcloud/cloud.go
index 200ac333..4a3748be 100644
--- a/hcloud/cloud.go
+++ b/hcloud/cloud.go
@@ -144,7 +144,7 @@ func (c *cloud) LoadBalancer() (cloudprovider.LoadBalancer, bool) {
 		ActionClient:  &c.client.Action,
 		NetworkClient: &c.client.Network,
 		NetworkID:     c.networkID,
-		Cfg:           c.cfg.LoadBalancer,
+		Cfg:           c.cfg,
 	}
 
 	return newLoadBalancers(lbOps, c.cfg.LoadBalancer.DisablePrivateIngress, c.cfg.LoadBalancer.DisableIPv6), true
diff --git a/internal/hcops/load_balancer.go b/internal/hcops/load_balancer.go
index 2a6aa33f..c1c4ca4f 100644
--- a/internal/hcops/load_balancer.go
+++ b/internal/hcops/load_balancer.go
@@ -71,7 +71,7 @@ type LoadBalancerOps struct {
 	CertOps       *CertificateOps
 	RetryDelay    time.Duration
 	NetworkID     int64
-	Cfg           config.LoadBalancerConfiguration
+	Cfg           config.HCCMConfiguration
 }
 
 // GetByK8SServiceUID tries to find a Load Balancer by its Kubernetes service
@@ -164,8 +164,8 @@ func (l *LoadBalancerOps) Create(
 	if v, ok := annotation.LBType.StringFromService(svc); ok {
 		opts.LoadBalancerType.Name = v
 	}
-	if l.Cfg.Location != "" {
-		opts.Location = &hcloud.Location{Name: l.Cfg.Location}
+	if l.Cfg.LoadBalancer.Location != "" {
+		opts.Location = &hcloud.Location{Name: l.Cfg.LoadBalancer.Location}
 	}
 	if v, ok := annotation.LBLocation.StringFromService(svc); ok {
 		if v == "" {
@@ -176,7 +176,7 @@ func (l *LoadBalancerOps) Create(
 			opts.Location = &hcloud.Location{Name: v}
 		}
 	}
-	opts.NetworkZone = hcloud.NetworkZone(l.Cfg.NetworkZone)
+	opts.NetworkZone = hcloud.NetworkZone(l.Cfg.LoadBalancer.NetworkZone)
 	if v, ok := annotation.LBNetworkZone.StringFromService(svc); ok {
 		opts.NetworkZone = hcloud.NetworkZone(v)
 	}
@@ -658,7 +658,7 @@ func (l *LoadBalancerOps) getUsePrivateIP(svc *corev1.Service) (bool, error) {
 	usePrivateIP, err := annotation.LBUsePrivateIP.BoolFromService(svc)
 	if err != nil {
 		if errors.Is(err, annotation.ErrNotSet) {
-			return l.Cfg.UsePrivateIP, nil
+			return l.Cfg.LoadBalancer.UsePrivateIP, nil
 		}
 		return false, err
 	}


Test patch:
diff --git a/internal/hcops/load_balancer_test.go b/internal/hcops/load_balancer_test.go
index f8bc5a2a..fa579f7b 100644
--- a/internal/hcops/load_balancer_test.go
+++ b/internal/hcops/load_balancer_test.go
@@ -244,7 +244,7 @@ func TestGetByK8SServiceUID(t *testing.T) {
 func TestLoadBalancerOps_Create(t *testing.T) {
 	type testCase struct {
 		name               string
-		cfg                config.LoadBalancerConfiguration
+		cfg                config.HCCMConfiguration
 		serviceAnnotations map[annotation.Name]interface{}
 		createOpts         hcloud.LoadBalancerCreateOpts
 		mock               func(t *testing.T, tt *testCase, fx *hcops.LoadBalancerOpsFixture)
@@ -254,8 +254,10 @@ func TestLoadBalancerOps_Create(t *testing.T) {
 	tests := []testCase{
 		{
 			name: "create with with location name (and default set)",
-			cfg: config.LoadBalancerConfiguration{
-				Location: "hel1",
+			cfg: config.HCCMConfiguration{
+				LoadBalancer: config.LoadBalancerConfiguration{
+					Location: "hel1",
+				},
 			},
 			serviceAnnotations: map[annotation.Name]interface{}{
 				annotation.LBLocation: "fsn1",
@@ -274,8 +276,10 @@ func TestLoadBalancerOps_Create(t *testing.T) {
 		},
 		{
 			name: "create with network zone name only (and default set)",
-			cfg: config.LoadBalancerConfiguration{
-				NetworkZone: "eu-central",
+			cfg: config.HCCMConfiguration{
+				LoadBalancer: config.LoadBalancerConfiguration{
+					NetworkZone: "eu-central",
+				},
 			},
 			serviceAnnotations: map[annotation.Name]interface{}{
 				annotation.LBNetworkZone: "eu-central",
@@ -292,8 +296,10 @@ func TestLoadBalancerOps_Create(t *testing.T) {
 		},
 		{
 			name: "create with location as default only",
-			cfg: config.LoadBalancerConfiguration{
-				Location: "fsn1",
+			cfg: config.HCCMConfiguration{
+				LoadBalancer: config.LoadBalancerConfiguration{
+					Location: "fsn1",
+				},
 			},
 			createOpts: hcloud.LoadBalancerCreateOpts{
 				Name:             "some-lb",
@@ -309,8 +315,10 @@ func TestLoadBalancerOps_Create(t *testing.T) {
 		},
 		{
 			name: "create with network zone as default only",
-			cfg: config.LoadBalancerConfiguration{
-				NetworkZone: "eu-central",
+			cfg: config.HCCMConfiguration{
+				LoadBalancer: config.LoadBalancerConfiguration{
+					NetworkZone: "eu-central",
+				},
 			},
 			createOpts: hcloud.LoadBalancerCreateOpts{
 				Name:             "some-lb",
@@ -324,8 +332,10 @@ func TestLoadBalancerOps_Create(t *testing.T) {
 		},
 		{
 			name: "create with network zone and reset default location",
-			cfg: config.LoadBalancerConfiguration{
-				Location: "hel1",
+			cfg: config.HCCMConfiguration{
+				LoadBalancer: config.LoadBalancerConfiguration{
+					Location: "hel1",
+				},
 			},
 			serviceAnnotations: map[annotation.Name]interface{}{
 				annotation.LBLocation:    "",
@@ -343,8 +353,10 @@ func TestLoadBalancerOps_Create(t *testing.T) {
 		},
 		{
 			name: "create with location and reset default network zone",
-			cfg: config.LoadBalancerConfiguration{
-				NetworkZone: "eu-central",
+			cfg: config.HCCMConfiguration{
+				LoadBalancer: config.LoadBalancerConfiguration{
+					NetworkZone: "eu-central",
+				},
 			},
 			serviceAnnotations: map[annotation.Name]interface{}{
 				annotation.LBLocation:    "fsn1",
@@ -590,7 +602,7 @@ func TestLoadBalancerOps_Delete(t *testing.T) {
 
 type LBReconcilementTestCase struct {
 	name               string
-	cfg                config.LoadBalancerConfiguration
+	cfg                config.HCCMConfiguration
 	serviceUID         string
 	serviceAnnotations map[annotation.Name]interface{}
 	servicePorts       []corev1.ServicePort
@@ -1189,9 +1201,11 @@ func TestLoadBalancerOps_ReconcileHCLBTargets(t *testing.T) {
 		},
 		{
 			name: "enable use of private network via default",
-			cfg: config.LoadBalancerConfiguration{
-				// Make sure the annotation overrides the default
-				UsePrivateIP: true,
+			cfg: config.HCCMConfiguration{
+				LoadBalancer: config.LoadBalancerConfiguration{
+					// Make sure the annotation overrides the default
+					UsePrivateIP: true,
+				},
 			},
 			k8sNodes: []*corev1.Node{
 				{Spec: corev1.NodeSpec{ProviderID: "hcloud://1"}},
@@ -1219,9 +1233,11 @@ func TestLoadBalancerOps_ReconcileHCLBTargets(t *testing.T) {
 		},
 		{
 			name: "enable use of private network via annotation",
-			cfg: config.LoadBalancerConfiguration{
-				// Make sure the annotation overrides the default
-				UsePrivateIP: false,
+			cfg: config.HCCMConfiguration{
+				LoadBalancer: config.LoadBalancerConfiguration{
+					// Make sure the annotation overrides the default
+					UsePrivateIP: false,
+				},
 			},
 			k8sNodes: []*corev1.Node{
 				{Spec: corev1.NodeSpec{ProviderID: "hcloud://1"}},
@@ -1252,9 +1268,11 @@ func TestLoadBalancerOps_ReconcileHCLBTargets(t *testing.T) {
 		},
 		{
 			name: "disable use of private network via annotation",
-			cfg: config.LoadBalancerConfiguration{
-				// Make sure the annotation overrides the default
-				UsePrivateIP: true,
+			cfg: config.HCCMConfiguration{
+				LoadBalancer: config.LoadBalancerConfiguration{
+					// Make sure the annotation overrides the default
+					UsePrivateIP: true,
+				},
 			},
 			k8sNodes: []*corev1.Node{
 				{Spec: corev1.NodeSpec{ProviderID: "hcloud://1"}},
diff --git a/tests/e2e/cloud_test.go b/tests/e2e/cloud_test.go
new file mode 100644
index 00000000..d18ab167
--- /dev/null
+++ b/tests/e2e/cloud_test.go
@@ -0,0 +1,303 @@
+//go:build e2e
+
+package e2e
+
+import (
+	"context"
+	"fmt"
+	"math/rand"
+	"net"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+
+	"github.com/hetznercloud/hcloud-cloud-controller-manager/internal/annotation"
+	"github.com/hetznercloud/hcloud-cloud-controller-manager/internal/hcops"
+	"github.com/hetznercloud/hcloud-go/v2/hcloud"
+)
+
+func TestNodeSetCorrectNodeLabelsAndIPAddresses(t *testing.T) {
+	t.Parallel()
+
+	ctx := context.Background()
+
+	node, err := testCluster.k8sClient.CoreV1().Nodes().Get(ctx, "hccm-"+testCluster.scope+"-1", metav1.GetOptions{})
+	assert.NoError(t, err)
+
+	server, _, err := testCluster.hcloud.Server.Get(ctx, "hccm-"+testCluster.scope+"-1")
+	if err != nil {
+		return
+	}
+
+	labels := node.Labels
+	expectedLabels := map[string]string{
+		"node.kubernetes.io/instance-type": server.ServerType.Name,
+		"topology.kubernetes.io/region":    server.Datacenter.Location.Name,
+		"topology.kubernetes.io/zone":      server.Datacenter.Name,
+		"kubernetes.io/hostname":           server.Name,
+		"kubernetes.io/os":                 "linux",
+		"kubernetes.io/arch":               "amd64",
+	}
+	for expectedLabel, expectedValue := range expectedLabels {
+		if labelValue, ok := labels[expectedLabel]; !ok || labelValue != expectedValue {
+			t.Errorf("node have a not expected label %s, ok: %v, given value %s, expected value %s", expectedLabel, ok, labelValue, expectedValue)
+		}
+	}
+
+	for _, address := range node.Status.Addresses {
+		if address.Type == corev1.NodeExternalIP {
+			expectedIP := server.PublicNet.IPv4.IP.String()
+			if expectedIP != address.Address {
+				t.Errorf("Got %s as NodeExternalIP but expected %s", address.Address, expectedIP)
+			}
+		}
+	}
+
+	for _, address := range node.Status.Addresses {
+		if address.Type == corev1.NodeInternalIP {
+			expectedIP := server.PrivateNet[0].IP.String()
+			if expectedIP != address.Address {
+				t.Errorf("Got %s as NodeInternalIP but expected %s", address.Address, expectedIP)
+			}
+		}
+	}
+}
+
+func TestServiceLoadBalancersMinimalSetup(t *testing.T) {
+	t.Parallel()
+
+	lbTest := lbTestHelper{
+		t:         t,
+		K8sClient: testCluster.k8sClient,
+		podName:   "loadbalancer-minimal",
+	}
+
+	pod := lbTest.DeployTestPod()
+
+	lbSvc := lbTest.ServiceDefinition(pod, map[string]string{
+		string(annotation.LBLocation): "nbg1",
+	})
+
+	lbSvc, err := lbTest.CreateService(lbSvc)
+	if err != nil {
+		t.Fatalf("deploying test svc: %s", err)
+	}
+
+	WaitForHTTPAvailable(t, lbSvc.Status.LoadBalancer.Ingress[0].IP, false)
+
+	lbTest.TearDown()
+}
+
+func TestServiceLoadBalancersHTTPS(t *testing.T) {
+	t.Parallel()
+
+	cert := testCluster.CreateTLSCertificate(t, "loadbalancer-https")
+	lbTest := lbTestHelper{
+		t:         t,
+		K8sClient: testCluster.k8sClient,
+		podName:   "loadbalancer-https",
+		port:      443,
+	}
+
+	pod := lbTest.DeployTestPod()
+
+	lbSvc := lbTest.ServiceDefinition(pod, map[string]string{
+		string(annotation.LBLocation):            "nbg1",
+		string(annotation.LBSvcHTTPCertificates): cert.Name,
+		string(annotation.LBSvcProtocol):         "https",
+	})
+
+	lbSvc, err := lbTest.CreateService(lbSvc)
+	if err != nil {
+		t.Fatalf("deploying test svc: %s", err)
+	}
+
+	WaitForHTTPAvailable(t, lbSvc.Status.LoadBalancer.Ingress[0].IP, true)
+
+	lbTest.TearDown()
+}
+
+func TestServiceLoadBalancersHTTPSWithManagedCertificate(t *testing.T) {
+	t.Parallel()
+
+	ctx := context.Background()
+
+	if testCluster.certDomain == "" {
+		t.Skip("Skipping because CERT_DOMAIN is not set")
+	}
+
+	domainName := fmt.Sprintf("%d-ccm-test.%s", rand.Int(), testCluster.certDomain)
+	lbTest := lbTestHelper{
+		t:         t,
+		K8sClient: testCluster.k8sClient,
+		podName:   "loadbalancer-https",
+		port:      443,
+	}
+
+	pod := lbTest.DeployTestPod()
+
+	lbSvc := lbTest.ServiceDefinition(pod, map[string]string{
+		string(annotation.LBLocation):                                "nbg1",
+		string(annotation.LBSvcHTTPCertificateType):                  "managed",
+		string(annotation.LBSvcHTTPManagedCertificateDomains):        domainName,
+		string(annotation.LBSvcProtocol):                             "https",
+		string(annotation.LBSvcHTTPManagedCertificateUseACMEStaging): "true",
+	})
+
+	lbSvc, err := lbTest.CreateService(lbSvc)
+	if err != nil {
+		t.Fatalf("deploying test svc: %s", err)
+	}
+	certs, err := testCluster.hcloud.Certificate.AllWithOpts(ctx, hcloud.CertificateListOpts{
+		ListOpts: hcloud.ListOpts{
+			LabelSelector: fmt.Sprintf("%s=%s", hcops.LabelServiceUID, lbSvc.ObjectMeta.UID),
+		},
+	})
+	assert.NoError(t, err)
+	assert.Len(t, certs, 1)
+
+	lbTest.TearDown()
+	_, err = testCluster.hcloud.Certificate.Delete(ctx, certs[0])
+	assert.NoError(t, err)
+}
+
+func TestServiceLoadBalancersWithPrivateNetwork(t *testing.T) {
+	t.Parallel()
+
+	lbTest := lbTestHelper{t: t, K8sClient: testCluster.k8sClient, podName: "loadbalancer-private-network"}
+
+	pod := lbTest.DeployTestPod()
+
+	lbSvcDefinition := lbTest.ServiceDefinition(pod, map[string]string{
+		string(annotation.LBLocation):     "nbg1",
+		string(annotation.LBUsePrivateIP): "true",
+	})
+
+	lbSvc, err := lbTest.CreateService(lbSvcDefinition)
+	if err != nil {
+		t.Fatalf("deploying test svc: %s", err)
+	}
+
+	WaitForHTTPAvailable(t, lbSvc.Status.LoadBalancer.Ingress[0].IP, false)
+
+	lbTest.TearDown()
+}
+
+func TestRouteNetworksPodIPsAreAccessible(t *testing.T) {
+	t.Parallel()
+
+	err := wait.PollUntilContextTimeout(context.Background(), 1*time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) {
+		node, err := testCluster.k8sClient.CoreV1().Nodes().Get(ctx, "hccm-"+testCluster.scope+"-1", metav1.GetOptions{})
+		if err != nil {
+			return false, err
+		}
+
+		network, _, err := testCluster.hcloud.Network.Get(ctx, "hccm-"+testCluster.scope)
+		if err != nil {
+			return false, err
+		}
+		for _, route := range network.Routes {
+			if route.Destination.String() == node.Spec.PodCIDR {
+				for _, a := range node.Status.Addresses {
+					if a.Type == corev1.NodeInternalIP {
+						assert.Equal(t, a.Address, route.Gateway.String())
+					}
+				}
+				return true, nil
+			}
+		}
+		return false, nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestRouteDeleteCorrectRoutes(t *testing.T) {
+	t.Parallel()
+
+	// This test tests for:
+	// - hccm is keeping routes that are outside of its scope (e.g. 0.0.0.0/0)
+	// - hccm is deleting routes that are inside the cluster cidr, but for which no server exists (10.254.0.0/24)
+	//
+	// Testing for no-op (keep route) is hard, but by combining the test with a deletion test we can be reasonably sure
+	// that it was not tried in the same reconcile loop.
+
+	ctx := context.Background()
+
+	testGateway := net.ParseIP("10.0.0.254")
+	_, defaultDestination, err := net.ParseCIDR("0.0.0.0/0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, outdatedDestination, err := net.ParseCIDR("10.244.254.0/24")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	network, _, err := testCluster.hcloud.Network.Get(ctx, "hccm-"+testCluster.scope)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Remove routes from previous tests
+	for _, route := range network.Routes {
+		if route.Gateway.Equal(testGateway) {
+			action, _, err := testCluster.hcloud.Network.DeleteRoute(ctx, network, hcloud.NetworkDeleteRouteOpts{Route: route})
+			if err != nil {
+				t.Fatal(err)
+			}
+			if err := hcops.WatchAction(ctx, &testCluster.hcloud.Action, action); err != nil {
+				t.Fatal(err)
+			}
+		}
+	}
+
+	action, _, err := testCluster.hcloud.Network.AddRoute(ctx, network, hcloud.NetworkAddRouteOpts{Route: hcloud.NetworkRoute{
+		Destination: defaultDestination,
+		Gateway:     testGateway,
+	}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := hcops.WatchAction(ctx, &testCluster.hcloud.Action, action); err != nil {
+		t.Fatal(err)
+	}
+
+	action, _, err = testCluster.hcloud.Network.AddRoute(ctx, network, hcloud.NetworkAddRouteOpts{Route: hcloud.NetworkRoute{
+		Destination: outdatedDestination,
+		Gateway:     testGateway,
+	}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := hcops.WatchAction(ctx, &testCluster.hcloud.Action, action); err != nil {
+		t.Fatal(err)
+	}
+
+	err = wait.PollUntilContextTimeout(ctx, 1*time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) {
+		network, _, err = testCluster.hcloud.Network.Get(ctx, "hccm-"+testCluster.scope)
+		if err != nil {
+			return false, err
+		}
+
+		hasDefaultRoute := false
+		for _, route := range network.Routes {
+			switch route.Destination.String() {
+			case defaultDestination.String():
+				hasDefaultRoute = true
+			case outdatedDestination.String():
+				// Route for outdated destination still exists
+				return false, nil
+			}
+		}
+		return hasDefaultRoute, nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+}
diff --git a/tests/e2e/e2e_test.go b/tests/e2e/e2e_test.go
index a7b8c6bd..2bce4a6f 100644
--- a/tests/e2e/e2e_test.go
+++ b/tests/e2e/e2e_test.go
@@ -5,21 +5,12 @@ package e2e
 import (
 	"context"
 	"fmt"
-	"math/rand"
-	"net"
 	"os"
 	"strings"
 	"testing"
-	"time"
 
 	"github.com/stretchr/testify/assert"
-	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/util/wait"
-
-	"github.com/hetznercloud/hcloud-cloud-controller-manager/internal/annotation"
-	"github.com/hetznercloud/hcloud-cloud-controller-manager/internal/hcops"
-	"github.com/hetznercloud/hcloud-go/v2/hcloud"
 )
 
 var testCluster TestCluster
@@ -70,281 +61,3 @@ func TestPodIsPresent(t *testing.T) {
 		}
 	})
 }
-
-func TestNodeSetCorrectNodeLabelsAndIPAddresses(t *testing.T) {
-	t.Parallel()
-
-	node, err := testCluster.k8sClient.CoreV1().Nodes().Get(context.Background(), "hccm-"+testCluster.scope+"-1", metav1.GetOptions{})
-	assert.NoError(t, err)
-
-	server, _, err := testCluster.hcloud.Server.Get(context.TODO(), "hccm-"+testCluster.scope+"-1")
-	if err != nil {
-		return
-	}
-
-	labels := node.Labels
-	expectedLabels := map[string]string{
-		"node.kubernetes.io/instance-type": server.ServerType.Name,
-		"topology.kubernetes.io/region":    server.Datacenter.Location.Name,
-		"topology.kubernetes.io/zone":      server.Datacenter.Name,
-		"kubernetes.io/hostname":           server.Name,
-		"kubernetes.io/os":                 "linux",
-		"kubernetes.io/arch":               "amd64",
-	}
-	for expectedLabel, expectedValue := range expectedLabels {
-		if labelValue, ok := labels[expectedLabel]; !ok || labelValue != expectedValue {
-			t.Errorf("node have a not expected label %s, ok: %v, given value %s, expected value %s", expectedLabel, ok, labelValue, expectedValue)
-		}
-	}
-
-	for _, address := range node.Status.Addresses {
-		if address.Type == corev1.NodeExternalIP {
-			expectedIP := server.PublicNet.IPv4.IP.String()
-			if expectedIP != address.Address {
-				t.Errorf("Got %s as NodeExternalIP but expected %s", address.Address, expectedIP)
-			}
-		}
-	}
-
-	for _, address := range node.Status.Addresses {
-		if address.Type == corev1.NodeInternalIP {
-			expectedIP := server.PrivateNet[0].IP.String()
-			if expectedIP != address.Address {
-				t.Errorf("Got %s as NodeInternalIP but expected %s", address.Address, expectedIP)
-			}
-		}
-	}
-}
-
-func TestServiceLoadBalancersMinimalSetup(t *testing.T) {
-	t.Parallel()
-
-	lbTest := lbTestHelper{
-		t:         t,
-		K8sClient: testCluster.k8sClient,
-		podName:   "loadbalancer-minimal",
-	}
-
-	pod := lbTest.DeployTestPod()
-
-	lbSvc := lbTest.ServiceDefinition(pod, map[string]string{
-		string(annotation.LBLocation): "nbg1",
-	})
-
-	lbSvc, err := lbTest.CreateService(lbSvc)
-	if err != nil {
-		t.Fatalf("deploying test svc: %s", err)
-	}
-
-	WaitForHTTPAvailable(t, lbSvc.Status.LoadBalancer.Ingress[0].IP, false)
-
-	lbTest.TearDown()
-}
-
-func TestServiceLoadBalancersHTTPS(t *testing.T) {
-	t.Parallel()
-
-	cert := testCluster.CreateTLSCertificate(t, "loadbalancer-https")
-	lbTest := lbTestHelper{
-		t:         t,
-		K8sClient: testCluster.k8sClient,
-		podName:   "loadbalancer-https",
-		port:      443,
-	}
-
-	pod := lbTest.DeployTestPod()
-
-	lbSvc := lbTest.ServiceDefinition(pod, map[string]string{
-		string(annotation.LBLocation):            "nbg1",
-		string(annotation.LBSvcHTTPCertificates): cert.Name,
-		string(annotation.LBSvcProtocol):         "https",
-	})
-
-	lbSvc, err := lbTest.CreateService(lbSvc)
-	if err != nil {
-		t.Fatalf("deploying test svc: %s", err)
-	}
-
-	WaitForHTTPAvailable(t, lbSvc.Status.LoadBalancer.Ingress[0].IP, true)
-
-	lbTest.TearDown()
-}
-
-func TestServiceLoadBalancersHTTPSWithManagedCertificate(t *testing.T) {
-	t.Parallel()
-
-	if testCluster.certDomain == "" {
-		t.Skip("Skipping because CERT_DOMAIN is not set")
-	}
-
-	domainName := fmt.Sprintf("%d-ccm-test.%s", rand.Int(), testCluster.certDomain)
-	lbTest := lbTestHelper{
-		t:         t,
-		K8sClient: testCluster.k8sClient,
-		podName:   "loadbalancer-https",
-		port:      443,
-	}
-
-	pod := lbTest.DeployTestPod()
-
-	lbSvc := lbTest.ServiceDefinition(pod, map[string]string{
-		string(annotation.LBLocation):                                "nbg1",
-		string(annotation.LBSvcHTTPCertificateType):                  "managed",
-		string(annotation.LBSvcHTTPManagedCertificateDomains):        domainName,
-		string(annotation.LBSvcProtocol):                             "https",
-		string(annotation.LBSvcHTTPManagedCertificateUseACMEStaging): "true",
-	})
-
-	lbSvc, err := lbTest.CreateService(lbSvc)
-	if err != nil {
-		t.Fatalf("deploying test svc: %s", err)
-	}
-	certs, err := testCluster.hcloud.Certificate.AllWithOpts(context.Background(), hcloud.CertificateListOpts{
-		ListOpts: hcloud.ListOpts{
-			LabelSelector: fmt.Sprintf("%s=%s", hcops.LabelServiceUID, lbSvc.ObjectMeta.UID),
-		},
-	})
-	assert.NoError(t, err)
-	assert.Len(t, certs, 1)
-
-	lbTest.TearDown()
-	_, err = testCluster.hcloud.Certificate.Delete(context.Background(), certs[0])
-	assert.NoError(t, err)
-}
-
-func TestServiceLoadBalancersWithPrivateNetwork(t *testing.T) {
-	t.Parallel()
-
-	lbTest := lbTestHelper{t: t, K8sClient: testCluster.k8sClient, podName: "loadbalancer-private-network"}
-
-	pod := lbTest.DeployTestPod()
-
-	lbSvcDefinition := lbTest.ServiceDefinition(pod, map[string]string{
-		string(annotation.LBLocation):     "nbg1",
-		string(annotation.LBUsePrivateIP): "true",
-	})
-
-	lbSvc, err := lbTest.CreateService(lbSvcDefinition)
-	if err != nil {
-		t.Fatalf("deploying test svc: %s", err)
-	}
-
-	WaitForHTTPAvailable(t, lbSvc.Status.LoadBalancer.Ingress[0].IP, false)
-
-	lbTest.TearDown()
-}
-
-func TestRouteNetworksPodIPsAreAccessible(t *testing.T) {
-	t.Parallel()
-
-	err := wait.PollUntilContextTimeout(context.Background(), 1*time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) {
-		node, err := testCluster.k8sClient.CoreV1().Nodes().Get(ctx, "hccm-"+testCluster.scope+"-1", metav1.GetOptions{})
-		if err != nil {
-			return false, err
-		}
-
-		network, _, err := testCluster.hcloud.Network.Get(context.TODO(), "hccm-"+testCluster.scope)
-		if err != nil {
-			return false, err
-		}
-		for _, route := range network.Routes {
-			if route.Destination.String() == node.Spec.PodCIDR {
-				for _, a := range node.Status.Addresses {
-					if a.Type == corev1.NodeInternalIP {
-						assert.Equal(t, a.Address, route.Gateway.String())
-					}
-				}
-				return true, nil
-			}
-		}
-		return false, nil
-	})
-	if err != nil {
-		t.Fatal(err)
-	}
-}
-
-func TestRouteDeleteCorrectRoutes(t *testing.T) {
-	t.Parallel()
-
-	// This test tests for:
-	// - hccm is keeping routes that are outside of its scope (e.g. 0.0.0.0/0)
-	// - hccm is deleting routes that are inside the cluster cidr, but for which no server exists (10.254.0.0/24)
-	//
-	// Testing for no-op (keep route) is hard, but by combining the test with a deletion test we can be reasonably sure
-	// that it was not tried in the same reconcile loop.
-
-	ctx := context.Background()
-
-	testGateway := net.ParseIP("10.0.0.254")
-	_, defaultDestination, err := net.ParseCIDR("0.0.0.0/0")
-	if err != nil {
-		t.Fatal(err)
-	}
-	_, outdatedDestination, err := net.ParseCIDR("10.244.254.0/24")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	network, _, err := testCluster.hcloud.Network.Get(ctx, "hccm-"+testCluster.scope)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Remove routes from previous tests
-	for _, route := range network.Routes {
-		if route.Gateway.Equal(testGateway) {
-			action, _, err := testCluster.hcloud.Network.DeleteRoute(ctx, network, hcloud.NetworkDeleteRouteOpts{Route: route})
-			if err != nil {
-				t.Fatal(err)
-			}
-			if err := hcops.WatchAction(ctx, &testCluster.hcloud.Action, action); err != nil {
-				t.Fatal(err)
-			}
-		}
-	}
-
-	action, _, err := testCluster.hcloud.Network.AddRoute(ctx, network, hcloud.NetworkAddRouteOpts{Route: hcloud.NetworkRoute{
-		Destination: defaultDestination,
-		Gateway:     testGateway,
-	}})
-	if err != nil {
-		t.Fatal(err)
-	}
-	if err := hcops.WatchAction(ctx, &testCluster.hcloud.Action, action); err != nil {
-		t.Fatal(err)
-	}
-
-	action, _, err = testCluster.hcloud.Network.AddRoute(ctx, network, hcloud.NetworkAddRouteOpts{Route: hcloud.NetworkRoute{
-		Destination: outdatedDestination,
-		Gateway:     testGateway,
-	}})
-	if err != nil {
-		t.Fatal(err)
-	}
-	if err := hcops.WatchAction(ctx, &testCluster.hcloud.Action, action); err != nil {
-		t.Fatal(err)
-	}
-
-	err = wait.PollUntilContextTimeout(ctx, 1*time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) {
-		network, _, err = testCluster.hcloud.Network.Get(ctx, "hccm-"+testCluster.scope)
-		if err != nil {
-			return false, err
-		}
-
-		hasDefaultRoute := false
-		for _, route := range network.Routes {
-			switch route.Destination.String() {
-			case defaultDestination.String():
-				hasDefaultRoute = true
-			case outdatedDestination.String():
-				// Route for outdated destination still exists
-				return false, nil
-			}
-		}
-		return hasDefaultRoute, nil
-	})
-	if err != nil {
-		t.Fatal(err)
-	}
-}