Repository: hetznercloud/hcloud-cloud-controller-manager
Pull request #204 (2021-05-30 18:43:55)

Support for IPv6 NodeAddresses
This PR adds support for setting an IPv6 address of the node for IPv6-only and dualstack clusters behind an optional environment variable which defaults to IPv4-only.

Context is [this Terraform module](https://github.com/tibordp/terraform-hcloud-dualstack-k8s) which sets up a dual-stack cluster on Hetzner Cloud. The [network plugin](https://github.com/tibordp/wigglenet) that it uses needs instance addresses in order to identify where to terminate the Wireguard tunnel for pod-to-pod traffic and if the primary address family for the cluster is IPv6, it makes sense for the ExternalIP addresses to be also.

tibordp: Hey, @fhofherr and @LKaemmerling. Do you think this is an appropriate feature to have in Hetzner CCM and if so, is this an OK approach to make it available?

I am happy to address any feedback on the PR / add docs / etc. if so

Thank you!
fhofherr: Hi @tibordp, 

thank you for your PR. It looks good to me as well and I'd like to merge it soon. Unfortunately a minor merge conflict introduced itself since you opened your PR.  Could you please resolve it?
tibordp: Hey @fhofherr. I rebased on top of recent changes. I also tested manually in a real cluster using a private build with `HCLOUD_INSTANCES_ADDRESS_FAMILY=dualstack` and the results are as expected:

```
$ kubectl get node -o wide
NAME           STATUS   ROLES                  AGE   VERSION   INTERNAL-IP   EXTERNAL-IP            OS-IMAGE             KERNEL-VERSION     CONTAINER-RUNTIME
k8s-master-0   Ready    control-plane,master   12m   v1.21.1   <none>        2a01:4f9:c011:446c::   Ubuntu 20.04.2 LTS   5.4.0-72-generic   containerd://1.4.6
k8s-master-1   Ready    control-plane,master   11m   v1.21.1   <none>        2a01:4f9:c011:446b::   Ubuntu 20.04.2 LTS   5.4.0-72-generic   containerd://1.4.6
k8s-worker-0   Ready    <none>                 11m   v1.21.1   <none>        2a01:4f9:c011:446d::   Ubuntu 20.04.2 LTS   5.4.0-72-generic   containerd://1.4.6
k8s-worker-1   Ready    <none>                 11m   v1.21.1   <none>        2a01:4f9:c011:446e::   Ubuntu 20.04.2 LTS   5.4.0-72-generic   containerd://1.4.6
$ kubectl describe node k8s-master-0
Name:               k8s-master-0
Roles:              control-plane,master

<...>

Addresses:
  Hostname:    k8s-master-0
  ExternalIP:  2a01:4f9:c011:446c::
  ExternalIP:  95.217.216.210

<...>
```
tibordp: I saw that the hcloud Terraform provider just takes the ::1 of the IPv6 subnet, so I went ahead and did the same

https://github.com/hetznercloud/terraform-provider-hcloud/blob/e87fbb565f698587c2336e36caa7c55348ad2bdc/internal/server/resource.go#L706
fhofherr: Using `<prefix>::1` should be a fair assumption to make. I will merge the PR now. Thanks for your contribution :-) 

Patch:
diff --git a/hcloud/cloud.go b/hcloud/cloud.go
index a8ae11c0..43c725ef 100644
--- a/hcloud/cloud.go
+++ b/hcloud/cloud.go
@@ -40,6 +40,7 @@ const (
 	hcloudDebugENVVar    = "HCLOUD_DEBUG"
 	// Disable the "master/server is attached to the network" check against the metadata service.
 	hcloudNetworkDisableAttachedCheckENVVar  = "HCLOUD_NETWORK_DISABLE_ATTACHED_CHECK"
+	hcloudInstancesAddressFamily             = "HCLOUD_INSTANCES_ADDRESS_FAMILY"
 	hcloudLoadBalancersEnabledENVVar         = "HCLOUD_LOAD_BALANCERS_ENABLED"
 	hcloudLoadBalancersLocation              = "HCLOUD_LOAD_BALANCERS_LOCATION"
 	hcloudLoadBalancersNetworkZone           = "HCLOUD_LOAD_BALANCERS_NETWORK_ZONE"
@@ -140,10 +141,16 @@ func newCloud(config io.Reader) (cloudprovider.Interface, error) {
 	if os.Getenv(hcloudLoadBalancersEnabledENVVar) == "false" {
 		loadBalancers = nil
 	}
+
+	instancesAddressFamily, err := addressFamilyFromEnv()
+	if err != nil {
+		return nil, fmt.Errorf("%s: %w", op, err)
+	}
+
 	return &cloud{
 		client:       client,
 		zones:        newZones(client, nodeName),
-		instances:    newInstances(client),
+		instances:    newInstances(client, instancesAddressFamily),
 		loadBalancer: loadBalancers,
 		routes:       nil,
 		networkID:    networkID,
@@ -237,6 +244,27 @@ func serverIsAttachedToNetwork(networkId int) (bool, error) {
 	return strings.Contains(string(body), fmt.Sprintf("network_id: %d\n", networkId)), nil
 }
 
+// addressFamilyFromEnv returns the address family for the instance address from the environment
+// variable. Returns AddressFamilyIPv4 if unset.
+func addressFamilyFromEnv() (addressFamily, error) {
+	family, ok := os.LookupEnv(hcloudInstancesAddressFamily)
+	if !ok {
+		return AddressFamilyIPv4, nil
+	}
+
+	switch strings.ToLower(family) {
+	case "ipv6":
+		return AddressFamilyIPv6, nil
+	case "ipv4":
+		return AddressFamilyIPv4, nil
+	case "dualstack":
+		return AddressFamilyDualStack, nil
+	default:
+		return -1, fmt.Errorf(
+			"%v: Invalid value, expected one of: ipv4,ipv6,dualstack", hcloudInstancesAddressFamily)
+	}
+}
+
 // getEnvBool returns the boolean parsed from the environment variable with the given key and a potential error
 // parsing the var. Returns false if the env var is unset.
 func getEnvBool(key string) (bool, error) {
diff --git a/hcloud/instances.go b/hcloud/instances.go
index 2fbb1d87..47cb9dcb 100644
--- a/hcloud/instances.go
+++ b/hcloud/instances.go
@@ -28,12 +28,21 @@ import (
 	cloudprovider "k8s.io/cloud-provider"
 )
 
+type addressFamily int
+
+const (
+	AddressFamilyDualStack addressFamily = iota
+	AddressFamilyIPv6
+	AddressFamilyIPv4
+)
+
 type instances struct {
-	client *hcloud.Client
+	client        *hcloud.Client
+	addressFamily addressFamily
 }
 
-func newInstances(client *hcloud.Client) *instances {
-	return &instances{client}
+func newInstances(client *hcloud.Client, addressFamily addressFamily) *instances {
+	return &instances{client, addressFamily}
 }
 
 func (i *instances) NodeAddressesByProviderID(ctx context.Context, providerID string) ([]v1.NodeAddress, error) {
@@ -149,8 +158,26 @@ func (i *instances) nodeAddresses(ctx context.Context, server *hcloud.Server) []
 	addresses = append(
 		addresses,
 		v1.NodeAddress{Type: v1.NodeHostName, Address: server.Name},
-		v1.NodeAddress{Type: v1.NodeExternalIP, Address: server.PublicNet.IPv4.IP.String()},
 	)
+
+	if i.addressFamily == AddressFamilyIPv6 || i.addressFamily == AddressFamilyDualStack {
+		// For a given IPv6 network of 2001:db8:1234::/64, the instance address is 2001:db8:1234::1
+		host_address := server.PublicNet.IPv6.IP
+		host_address[len(host_address)-1] |= 0x01
+
+		addresses = append(
+			addresses,
+			v1.NodeAddress{Type: v1.NodeExternalIP, Address: host_address.String()},
+		)
+	}
+
+	if i.addressFamily == AddressFamilyIPv4 || i.addressFamily == AddressFamilyDualStack {
+		addresses = append(
+			addresses,
+			v1.NodeAddress{Type: v1.NodeExternalIP, Address: server.PublicNet.IPv4.IP.String()},
+		)
+	}
+
 	n := os.Getenv(hcloudNetworkENVVar)
 	if len(n) > 0 {
 		network, _, _ := i.client.Network.Get(ctx, n)


Test patch:
diff --git a/hcloud/instances_test.go b/hcloud/instances_test.go
index 484592c8..faa80c56 100644
--- a/hcloud/instances_test.go
+++ b/hcloud/instances_test.go
@@ -36,6 +36,9 @@ func TestNodeAddressesByProviderID(t *testing.T) {
 				ID:   1,
 				Name: "node15",
 				PublicNet: schema.ServerPublicNet{
+					IPv6: schema.ServerPublicNetIPv6{
+						IP: "2001:db8:1234::/64",
+					},
 					IPv4: schema.ServerPublicNetIPv4{
 						IP: "131.232.99.1",
 					},
@@ -44,7 +47,7 @@ func TestNodeAddressesByProviderID(t *testing.T) {
 		})
 	})
 
-	instances := newInstances(env.Client)
+	instances := newInstances(env.Client, AddressFamilyIPv4)
 	addr, err := instances.NodeAddressesByProviderID(context.TODO(), "hcloud://1")
 	if err != nil {
 		t.Fatalf("Unexpected error: %v", err)
@@ -69,6 +72,9 @@ func TestNodeAddresses(t *testing.T) {
 					ID:   1,
 					Name: "node15",
 					PublicNet: schema.ServerPublicNet{
+						IPv6: schema.ServerPublicNetIPv6{
+							IP: "2001:db8:1234::",
+						},
 						IPv4: schema.ServerPublicNetIPv4{
 							IP: "131.232.99.1",
 						},
@@ -78,7 +84,7 @@ func TestNodeAddresses(t *testing.T) {
 		})
 	})
 
-	instances := newInstances(env.Client)
+	instances := newInstances(env.Client, AddressFamilyIPv4)
 	addr, err := instances.NodeAddresses(context.TODO(), "node15")
 	if err != nil {
 		t.Fatalf("Unexpected error: %v", err)
@@ -90,6 +96,81 @@ func TestNodeAddresses(t *testing.T) {
 	}
 }
 
+func TestNodeAddressesIPv6(t *testing.T) {
+	env := newTestEnv()
+	defer env.Teardown()
+	env.Mux.HandleFunc("/servers", func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.RawQuery != "name=node15" {
+			t.Fatal("missing name query")
+		}
+		json.NewEncoder(w).Encode(schema.ServerListResponse{
+			Servers: []schema.Server{
+				{
+					ID:   1,
+					Name: "node15",
+					PublicNet: schema.ServerPublicNet{
+						IPv6: schema.ServerPublicNetIPv6{
+							IP: "2001:db8:1234::/64",
+						},
+						IPv4: schema.ServerPublicNetIPv4{
+							IP: "131.232.99.1",
+						},
+					},
+				},
+			},
+		})
+	})
+
+	instances := newInstances(env.Client, AddressFamilyIPv6)
+	addr, err := instances.NodeAddresses(context.TODO(), "node15")
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+	if len(addr) != 2 ||
+		addr[0].Type != v1.NodeHostName || addr[0].Address != "node15" ||
+		addr[1].Type != v1.NodeExternalIP || addr[1].Address != "2001:db8:1234::1" {
+		t.Errorf("Unexpected node addresses: %v", addr)
+	}
+}
+
+func TestNodeAddressesDualStack(t *testing.T) {
+	env := newTestEnv()
+	defer env.Teardown()
+	env.Mux.HandleFunc("/servers", func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.RawQuery != "name=node15" {
+			t.Fatal("missing name query")
+		}
+		json.NewEncoder(w).Encode(schema.ServerListResponse{
+			Servers: []schema.Server{
+				{
+					ID:   1,
+					Name: "node15",
+					PublicNet: schema.ServerPublicNet{
+						IPv6: schema.ServerPublicNetIPv6{
+							IP: "2001:db8:1234::/64",
+						},
+						IPv4: schema.ServerPublicNetIPv4{
+							IP: "131.232.99.1",
+						},
+					},
+				},
+			},
+		})
+	})
+
+	instances := newInstances(env.Client, AddressFamilyDualStack)
+	addr, err := instances.NodeAddresses(context.TODO(), "node15")
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+	if len(addr) != 3 ||
+		addr[0].Type != v1.NodeHostName || addr[0].Address != "node15" ||
+		addr[1].Type != v1.NodeExternalIP || addr[1].Address != "2001:db8:1234::1" ||
+		addr[2].Type != v1.NodeExternalIP || addr[2].Address != "131.232.99.1" {
+		t.Errorf("Unexpected node addresses: %v", addr)
+	}
+}
+
 func TestExternalID(t *testing.T) {
 	env := newTestEnv()
 	defer env.Teardown()
@@ -106,7 +187,7 @@ func TestExternalID(t *testing.T) {
 		})
 	})
 
-	instances := newInstances(env.Client)
+	instances := newInstances(env.Client, AddressFamilyIPv4)
 	id, err := instances.ExternalID(context.TODO(), "node15")
 	if err != nil {
 		t.Fatalf("Unexpected error: %v", err)
@@ -135,7 +216,7 @@ func TestInstanceType(t *testing.T) {
 		})
 	})
 
-	instances := newInstances(env.Client)
+	instances := newInstances(env.Client, AddressFamilyIPv4)
 	serverType, err := instances.InstanceType(context.TODO(), "node15")
 	if err != nil {
 		t.Fatalf("Unexpected error: %v", err)
@@ -160,7 +241,7 @@ func TestInstanceTypeByProviderID(t *testing.T) {
 		})
 	})
 
-	instances := newInstances(env.Client)
+	instances := newInstances(env.Client, AddressFamilyIPv4)
 	instanceType, err := instances.InstanceTypeByProviderID(context.TODO(), "hcloud://1")
 	if err != nil {
 		t.Fatalf("Unexpected error: %v", err)
@@ -186,7 +267,7 @@ func TestInstanceExistsByProviderID(t *testing.T) {
 			})
 		})
 
-		instances := newInstances(env.Client)
+		instances := newInstances(env.Client, AddressFamilyIPv4)
 		exists, err := instances.InstanceExistsByProviderID(context.TODO(), "hcloud://1")
 		if err != nil {
 			t.Fatalf("Unexpected error: %v", err)
@@ -209,7 +290,7 @@ func TestInstanceExistsByProviderID(t *testing.T) {
 			})
 		})
 
-		instances := newInstances(env.Client)
+		instances := newInstances(env.Client, AddressFamilyIPv4)
 		exists, err := instances.InstanceExistsByProviderID(context.TODO(), "hcloud://1")
 		if err != nil {
 			t.Fatalf("Unexpected error: %v", err)
@@ -233,7 +314,7 @@ func TestInstanceShutdownByProviderID(t *testing.T) {
 			})
 		})
 
-		instances := newInstances(env.Client)
+		instances := newInstances(env.Client, AddressFamilyIPv4)
 		isOff, err := instances.InstanceShutdownByProviderID(context.TODO(), "hcloud://1")
 		if !isOff {
 			t.Errorf("Unexpected isOff state: %v", isOff)
@@ -255,7 +336,7 @@ func TestInstanceShutdownByProviderID(t *testing.T) {
 			})
 		})
 
-		instances := newInstances(env.Client)
+		instances := newInstances(env.Client, AddressFamilyIPv4)
 		isOff, err := instances.InstanceShutdownByProviderID(context.TODO(), "hcloud://1")
 		if isOff {
 			t.Errorf("Unexpected isOff state: %v", isOff)
@@ -279,7 +360,7 @@ func TestInstanceShutdownByProviderID(t *testing.T) {
 			})
 		})
 
-		instances := newInstances(env.Client)
+		instances := newInstances(env.Client, AddressFamilyIPv4)
 		isOff, err := instances.InstanceShutdownByProviderID(context.TODO(), "hcloud://1")
 		if isOff {
 			t.Errorf("Unexpected isOff state: %v", isOff)
@@ -293,7 +374,7 @@ func TestInstanceShutdownByProviderID(t *testing.T) {
 func TestCurrentNodeName(t *testing.T) {
 	env := newTestEnv()
 	defer env.Teardown()
-	instances := newInstances(env.Client)
+	instances := newInstances(env.Client, AddressFamilyIPv4)
 	nodeName, err := instances.CurrentNodeName(context.TODO(), "hostname")
 	if err != nil {
 		t.Fatalf("Unexpected error: %v", err)