Repository: hetznercloud/hcloud-cloud-controller-manager (go)

Issue:
Support secret injection with HashiCorp Vault
I would like to see support for HashiCorp Vault.
More specifically, the Vault Agent Injector, which can inject secrets from Vault into pods.

To do this, HCCM must either read the HCLOUD_TOKEN from a file or enable the sourcing of a file before startup (so that injecting environment variables via vault is possible, see https://developer.hashicorp.com/vault/docs/platform/k8s/injector/examples#environment-variable-example).

Another solution would be to use the [Vault Secrets Operator](https://github.com/hashicorp/vault-secrets-operator/), which synchronises Vault Secrets with Kubernetes Secrets. However, I think loading on pod startup is better (because it can use the pod service account. This can also be done with the VSO but this requires 2 CRDs for each secret that should be loaded (`VaultAuth` and `VaultStaticSecret`))

What do the developers and the community think about this change?

Pull request:
feat: read HCLOUD_TOKEN from file
This allows the `HCLOUD_TOKEN` (and `ROBOT_USER` and `ROBOT_PASSWORD`) to be read from a file. This can be useful if the token is injected using secret injection (e.g. with the vault agent injector).

I tested the changes on my dev cluster. I cannot test the robot account changes, because I only use hcloud. But because they use the same function inside the code I think its working too :). If someone is interested in using this with the vault agent injector, I used the following helm values:

```yaml
image:
  repository: <custom-image-because-changes-are-not-released>
  tag: <custom-image-because-changes-are-not-released>
podAnnotations:
  vault.hashicorp.com/agent-inject: "true"
  vault.hashicorp.com/log-format: json
  vault.hashicorp.com/role: <your-vault-role-name>
  vault.hashicorp.com/secret-volume-path-token: /vault/secrets
  vault.hashicorp.com/agent-inject-file-token: token
  vault.hashicorp.com/agent-inject-secret-token: <your-vault-mount>/data/<your-vault-path>
  vault.hashicorp.com/agent-inject-template-token: |
    {{ with secret "<your-vault-mount>/data/<your-vault-path>" -}}
     {{ .Data.data.token }}
    {{- end }}
env:
  HCLOUD_TOKEN_FILE:
    value: "/vault/secrets/token"
  HCLOUD_TOKEN: null # must be set because helm results in using value and valueFrom and that results in an error
```

This change is inspired from [external-dns cloudflare provider](https://github.com/kubernetes-sigs/external-dns/blob/master/provider/cloudflare/cloudflare.go#L171). I requested the same change for the [csi-driver](https://github.com/hetznercloud/csi-driver/pull/617) to keep consistency in reading HCLOUD_TOKEN from file.

Closes #595 

Patch:
diff --git a/chart/templates/daemonset.yaml b/chart/templates/daemonset.yaml
index 12242b8b..35d5b351 100644
--- a/chart/templates/daemonset.yaml
+++ b/chart/templates/daemonset.yaml
@@ -13,6 +13,13 @@ spec:
     metadata:
       labels:
         {{- include "hcloud-cloud-controller-manager.selectorLabels" . | nindent 8 }}
+        {{- if .Values.podLabels }}
+        {{- toYaml .Values.podLabels | nindent 8 }}
+        {{- end }}
+      {{- if .Values.podAnnotations }}
+      annotations:
+        {{- toYaml .Values.podAnnotations | nindent 8 }}
+      {{- end }}
     spec:
       serviceAccountName: {{ include "hcloud-cloud-controller-manager.name" . }}
       dnsPolicy: Default
diff --git a/chart/templates/deployment.yaml b/chart/templates/deployment.yaml
index b59daa2a..782a58df 100644
--- a/chart/templates/deployment.yaml
+++ b/chart/templates/deployment.yaml
@@ -14,6 +14,13 @@ spec:
     metadata:
       labels:
         {{- include "hcloud-cloud-controller-manager.selectorLabels" . | nindent 8 }}
+        {{- if .Values.podLabels }}
+        {{- toYaml .Values.podLabels | nindent 8 }}
+        {{- end }}
+      {{- if .Values.podAnnotations }}
+      annotations:
+        {{- toYaml .Values.podAnnotations | nindent 8 }}
+      {{- end }}
     spec:
       serviceAccountName: {{ include "hcloud-cloud-controller-manager.name" . }}
       dnsPolicy: Default
diff --git a/chart/values.yaml b/chart/values.yaml
index 795109d5..041cb4ea 100644
--- a/chart/values.yaml
+++ b/chart/values.yaml
@@ -29,6 +29,16 @@ env:
   # HCLOUD_NETWORK - see networking.enabled
   # ROBOT_ENABLED - see robot.enabled
 
+  # You can also use a file to provide secrets to the hcloud-cloud-controller-manager.
+  # This is currently possible for HCLOUD_TOKEN, ROBOT_USER, and ROBOT_PASSWORD.
+  # Use the env var appended with _FILE (e.g. HCLOUD_TOKEN_FILE) and set the value to the file path that should be read
+  # The file must be provided externally (e.g. via secret injection).
+  # Example:
+  # HCLOUD_TOKEN_FILE:
+  #   value: "/etc/hetzner/token"
+  # to disable reading the token from the secret you have to disable the original env var:
+  # HCLOUD_TOKEN: null
+
   HCLOUD_TOKEN:
     valueFrom:
       secretKeyRef:
@@ -103,3 +113,7 @@ nodeSelector: {}
 robot:
   # Set to true to enable support for Robot (Dedicated) servers.
   enabled: false
+
+podLabels: {}
+
+podAnnotations: {}
diff --git a/internal/config/config.go b/internal/config/config.go
index 9c97e2a6..504858a4 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"os"
 	"strconv"
+	"strings"
 	"time"
 )
 
@@ -96,6 +97,32 @@ type HCCMConfiguration struct {
 	Route        RouteConfiguration
 }
 
+// read values from environment variables or from file set via _FILE env var
+// values set directly via env var take precedence over values set via file.
+func readFromEnvOrFile(envVar string) (string, error) {
+	// check if the value is set directly via env (e.g. HCLOUD_TOKEN)
+	value, ok := os.LookupEnv(envVar)
+	if ok {
+		return value, nil
+	}
+
+	// check if the value is set via a file (e.g. HCLOUD_TOKEN_FILE)
+	value, ok = os.LookupEnv(envVar + "_FILE")
+	if !ok {
+		// return no error here, the values could be optional
+		// and the function "Validate()" below checks that all required variables are set
+		return "", nil
+	}
+
+	// read file content
+	valueBytes, err := os.ReadFile(value)
+	if err != nil {
+		return "", fmt.Errorf("failed to read %s: %w", envVar+"_FILE", err)
+	}
+
+	return strings.TrimSpace(string(valueBytes)), nil
+}
+
 // Read evaluates all environment variables and returns a [HCCMConfiguration]. It only validates as far as
 // it needs to parse the values. For business logic validation, check out [HCCMConfiguration.Validate].
 func Read() (HCCMConfiguration, error) {
@@ -106,7 +133,10 @@ func Read() (HCCMConfiguration, error) {
 	var errs []error
 	var cfg HCCMConfiguration
 
-	cfg.HCloudClient.Token = os.Getenv(hcloudToken)
+	cfg.HCloudClient.Token, err = readFromEnvOrFile(hcloudToken)
+	if err != nil {
+		errs = append(errs, err)
+	}
 	cfg.HCloudClient.Endpoint = os.Getenv(hcloudEndpoint)
 	cfg.HCloudClient.Debug, err = getEnvBool(hcloudDebug, false)
 	if err != nil {
@@ -117,8 +147,14 @@ func Read() (HCCMConfiguration, error) {
 	if err != nil {
 		errs = append(errs, err)
 	}
-	cfg.Robot.User = os.Getenv(robotUser)
-	cfg.Robot.Password = os.Getenv(robotPassword)
+	cfg.Robot.User, err = readFromEnvOrFile(robotUser)
+	if err != nil {
+		errs = append(errs, err)
+	}
+	cfg.Robot.Password, err = readFromEnvOrFile(robotPassword)
+	if err != nil {
+		errs = append(errs, err)
+	}
 	cfg.Robot.CacheTimeout, err = getEnvDuration(robotCacheTimeout)
 	if err != nil {
 		errs = append(errs, err)


Test patch:
diff --git a/internal/config/config_test.go b/internal/config/config_test.go
index 367b640d..ac8d6f97 100644
--- a/internal/config/config_test.go
+++ b/internal/config/config_test.go
@@ -14,6 +14,7 @@ func TestRead(t *testing.T) {
 	tests := []struct {
 		name    string
 		env     []string
+		files   map[string]string
 		want    HCCMConfiguration
 		wantErr error
 	}{
@@ -48,6 +49,54 @@ func TestRead(t *testing.T) {
 			},
 			wantErr: nil,
 		},
+		{
+			name: "secrets from file",
+			env: []string{
+				"HCLOUD_TOKEN_FILE", "/tmp/hetzner-token",
+				"ROBOT_USER_FILE", "/tmp/hetzner-user",
+				"ROBOT_PASSWORD_FILE", "/tmp/hetzner-password",
+			},
+			files: map[string]string{
+				"hetzner-token":    "jr5g7ZHpPptyhJzZyHw2Pqu4g9gTqDvEceYpngPf79jN_NOT_VALID_dzhepnahq",
+				"hetzner-user":     "foobar",
+				"hetzner-password": `secret-password`,
+			},
+			want: HCCMConfiguration{
+				HCloudClient: HCloudClientConfiguration{Token: "jr5g7ZHpPptyhJzZyHw2Pqu4g9gTqDvEceYpngPf79jN_NOT_VALID_dzhepnahq"},
+				Robot: RobotConfiguration{
+					Enabled:           false,
+					User:              "foobar",
+					Password:          "secret-password",
+					CacheTimeout:      5 * time.Minute,
+					RateLimitWaitTime: 0,
+				},
+				Metrics:      MetricsConfiguration{Enabled: true, Address: ":8233"},
+				Instance:     InstanceConfiguration{AddressFamily: AddressFamilyIPv4},
+				LoadBalancer: LoadBalancerConfiguration{Enabled: true},
+				Route:        RouteConfiguration{Enabled: false},
+			},
+			wantErr: nil,
+		},
+		{
+			name: "secrets from unknown file",
+			env: []string{
+				"HCLOUD_TOKEN_FILE", "/tmp/hetzner-token",
+				"ROBOT_USER_FILE", "/tmp/hetzner-user",
+				"ROBOT_PASSWORD_FILE", "/tmp/hetzner-password",
+			},
+			files: map[string]string{}, // don't create files
+			want: HCCMConfiguration{
+				HCloudClient: HCloudClientConfiguration{Token: ""},
+				Robot:        RobotConfiguration{User: "", Password: "", CacheTimeout: 0},
+				Metrics:      MetricsConfiguration{Enabled: false},
+				Instance:     InstanceConfiguration{},
+				LoadBalancer: LoadBalancerConfiguration{Enabled: false},
+				Route:        RouteConfiguration{Enabled: false},
+			},
+			wantErr: errors.New(`failed to read HCLOUD_TOKEN_FILE: open /tmp/hetzner-token: no such file or directory
+failed to read ROBOT_USER_FILE: open /tmp/hetzner-user: no such file or directory
+failed to read ROBOT_PASSWORD_FILE: open /tmp/hetzner-password: no such file or directory`),
+		},
 		{
 			name: "client",
 			env: []string{
@@ -207,6 +256,8 @@ failed to parse ROBOT_RATE_LIMIT_WAIT_TIME: time: unknown unit "fortnights" in d
 		t.Run(tt.name, func(t *testing.T) {
 			resetEnv := testsupport.Setenv(t, tt.env...)
 			defer resetEnv()
+			resetFiles := testsupport.SetFiles(t, tt.files)
+			defer resetFiles()
 
 			got, err := Read()
 			if tt.wantErr == nil {
diff --git a/internal/testsupport/files.go b/internal/testsupport/files.go
new file mode 100644
index 00000000..1872d73d
--- /dev/null
+++ b/internal/testsupport/files.go
@@ -0,0 +1,45 @@
+package testsupport
+
+import (
+	"os"
+	"testing"
+)
+
+// SetFiles can be used to temporarily create files on the local file system.
+// It returns a function that will clean up all files it created.
+func SetFiles(t *testing.T, files map[string]string) func() {
+	for file, content := range files {
+		filepath := os.TempDir() + "/" + file
+
+		// check if file exists
+		_, err := os.Stat(filepath)
+		if err == nil {
+			t.Fatalf("Trying to set file %s, but it already exists. Please choose another filepath for the test.", filepath)
+		}
+
+		// create file
+		f, err := os.Create(filepath)
+		if err != nil {
+			t.Fatalf("Failed to create file %s: %v", filepath, err)
+		}
+
+		// write content to file
+		_, err = f.WriteString(content)
+		if err != nil {
+			t.Fatalf("Failed to write to file %s: %v", filepath, err)
+		}
+
+		// close file
+		f.Close()
+	}
+
+	return func() {
+		for file := range files {
+			filepath := os.TempDir() + "/" + file
+			err := os.Remove(filepath)
+			if err != nil {
+				t.Fatalf("Failed to remove file %s: %v", filepath, err)
+			}
+		}
+	}
+}