Repository: GoogleCloudPlatform/cloud-sql-proxy-operator
Pull request #213 (2023-02-21 18:31:03)

feat: Add configuration for the admin api port and debug.
Enable the user to configure the Admin Port for the proxy through the AuthProxyWorkload resource.

Related to #45 


Patch:
diff --git a/config/crd/bases/cloudsql.cloud.google.com_authproxyworkloads.yaml b/config/crd/bases/cloudsql.cloud.google.com_authproxyworkloads.yaml
index 68559d5..fae1a6a 100644
--- a/config/crd/bases/cloudsql.cloud.google.com_authproxyworkloads.yaml
+++ b/config/crd/bases/cloudsql.cloud.google.com_authproxyworkloads.yaml
@@ -46,6 +46,24 @@ spec:
                 authProxyContainer:
                   description: AuthProxyContainer describes the resources and config for the Auth Proxy container.
                   properties:
+                    adminServer:
+                      description: AdminServer specifies the config for the proxy's admin service which is available to other containers in the same pod.
+                      properties:
+                        enableAPIs:
+                          description: 'EnableAPIs specifies the list of admin APIs to enable. At least one API must be enabled. Possible values: - "Debug" will enable pprof debugging by setting the `--debug` cli flag. - "QuitQuitQuit" will enable pprof debugging by setting the `--quitquitquit` cli flag.'
+                          enum:
+                            - Debug
+                            - QuitQuitQuit
+                          items:
+                            type: string
+                          minItems: 1
+                          type: array
+                        port:
+                          description: Port the port for the proxy's localhost-only admin server. This sets the proxy container's CLI argument `--admin-port`
+                          format: int32
+                          minimum: 1
+                          type: integer
+                      type: object
                     container:
                       description: Container is debugging parameter that when specified will override the proxy container with a completely custom Container spec.
                       properties:
diff --git a/docs/api.md b/docs/api.md
index 9f13bd4..81e7065 100644
--- a/docs/api.md
+++ b/docs/api.md
@@ -15,6 +15,21 @@ the custom resource AuthProxyWorkload version v1alpha1.
 
 
 
+#### AdminServerSpec
+
+
+
+AdminServerSpec specifies how to start the proxy's admin server: which port and whether to enable debugging or quitquitquit. It controls to the proxy's --admin-port, --debug, and --quitquitquit CLI flags.
+
+_Appears in:_
+- [AuthProxyContainerSpec](#authproxycontainerspec)
+
+| Field | Description |
+| --- | --- |
+| `port` _integer_ | Port the port for the proxy's localhost-only admin server. This sets the proxy container's CLI argument `--admin-port` |
+| `enableAPIs` _string array_ | EnableAPIs specifies the list of admin APIs to enable. At least one API must be enabled. Possible values: - "Debug" will enable pprof debugging by setting the `--debug` cli flag. - "QuitQuitQuit" will enable pprof debugging by setting the `--quitquitquit`   cli flag. |
+
+
 #### AuthProxyContainerSpec
 
 
@@ -29,6 +44,7 @@ _Appears in:_
 | `container` _[Container](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#container-v1-core)_ | Container is debugging parameter that when specified will override the proxy container with a completely custom Container spec. |
 | `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#resourcerequirements-v1-core)_ | Resources specifies the resources required for the proxy pod. |
 | `telemetry` _[TelemetrySpec](#telemetryspec)_ | Telemetry specifies how the proxy should expose telemetry. Optional, by default |
+| `adminServer` _[AdminServerSpec](#adminserverspec)_ | AdminServer specifies the config for the proxy's admin service which is available to other containers in the same pod. |
 | `maxConnections` _integer_ | MaxConnections limits the number of connections. Default value is no limit. This sets the proxy container's CLI argument `--max-connections` |
 | `maxSigtermDelay` _integer_ | MaxSigtermDelay is the maximum number of seconds to wait for connections to close after receiving a TERM signal. This sets the proxy container's CLI argument `--max-sigterm-delay` and configures `terminationGracePeriodSeconds` on the workload's PodSpec. |
 | `sqlAdminAPIEndpoint` _string_ | SQLAdminAPIEndpoint is a debugging parameter that when specified will change the Google Cloud api endpoint used by the proxy. |
diff --git a/installer/cloud-sql-proxy-operator.yaml b/installer/cloud-sql-proxy-operator.yaml
index 8a59b6f..f8d0e85 100644
--- a/installer/cloud-sql-proxy-operator.yaml
+++ b/installer/cloud-sql-proxy-operator.yaml
@@ -64,6 +64,24 @@ spec:
                 authProxyContainer:
                   description: AuthProxyContainer describes the resources and config for the Auth Proxy container.
                   properties:
+                    adminServer:
+                      description: AdminServer specifies the config for the proxy's admin service which is available to other containers in the same pod.
+                      properties:
+                        enableAPIs:
+                          description: 'EnableAPIs specifies the list of admin APIs to enable. At least one API must be enabled. Possible values: - "Debug" will enable pprof debugging by setting the `--debug` cli flag. - "QuitQuitQuit" will enable pprof debugging by setting the `--quitquitquit` cli flag.'
+                          enum:
+                            - Debug
+                            - QuitQuitQuit
+                          items:
+                            type: string
+                          minItems: 1
+                          type: array
+                        port:
+                          description: Port the port for the proxy's localhost-only admin server. This sets the proxy container's CLI argument `--admin-port`
+                          format: int32
+                          minimum: 1
+                          type: integer
+                      type: object
                     container:
                       description: Container is debugging parameter that when specified will override the proxy container with a completely custom Container spec.
                       properties:
diff --git a/internal/api/v1alpha1/authproxyworkload_types.go b/internal/api/v1alpha1/authproxyworkload_types.go
index 0d46149..ac9b1fc 100644
--- a/internal/api/v1alpha1/authproxyworkload_types.go
+++ b/internal/api/v1alpha1/authproxyworkload_types.go
@@ -155,6 +155,10 @@ type AuthProxyContainerSpec struct {
 	//+kubebuilder:validation:Optional
 	Telemetry *TelemetrySpec `json:"telemetry,omitempty"`
 
+	// AdminServer specifies the config for the proxy's admin service which is
+	// available to other containers in the same pod.
+	AdminServer *AdminServerSpec `json:"adminServer,omitempty"`
+
 	// MaxConnections limits the number of connections. Default value is no limit.
 	// This sets the proxy container's CLI argument `--max-connections`
 	//+kubebuilder:validation:Optional
@@ -193,6 +197,27 @@ type AuthProxyContainerSpec struct {
 	RolloutStrategy string `json:"rolloutStrategy,omitempty"`
 }
 
+// AdminServerSpec specifies how to start the proxy's admin server:
+// which port and whether to enable debugging or quitquitquit. It controls
+// to the proxy's --admin-port, --debug, and --quitquitquit CLI flags.
+type AdminServerSpec struct {
+
+	// Port the port for the proxy's localhost-only admin server.
+	// This sets the proxy container's CLI argument `--admin-port`
+	//+kubebuilder:validation:required
+	//+kubebuilder:validation:Minimum=1
+	Port int32 `json:"port,omitempty"`
+
+	// EnableAPIs specifies the list of admin APIs to enable. At least one
+	// API must be enabled. Possible values:
+	// - "Debug" will enable pprof debugging by setting the `--debug` cli flag.
+	// - "QuitQuitQuit" will enable pprof debugging by setting the `--quitquitquit`
+	//   cli flag.
+	//+kubebuilder:validation:MinItems:=1
+	//+kubebuilder:validation:Enum=Debug;QuitQuitQuit
+	EnableAPIs []string `json:"enableAPIs,omitempty"`
+}
+
 // TelemetrySpec specifies how the proxy container will expose telemetry.
 type TelemetrySpec struct {
 	// HTTPPort the port for Prometheus and health check server.
diff --git a/internal/api/v1alpha1/authproxyworkload_webhook.go b/internal/api/v1alpha1/authproxyworkload_webhook.go
index d0bc0bc..0307b3a 100644
--- a/internal/api/v1alpha1/authproxyworkload_webhook.go
+++ b/internal/api/v1alpha1/authproxyworkload_webhook.go
@@ -100,11 +100,35 @@ func (r *AuthProxyWorkload) validate() field.ErrorList {
 	allErrs = append(allErrs, validation.ValidateLabelName(r.Name, field.NewPath("metadata", "name"))...)
 	allErrs = append(allErrs, validateWorkload(&r.Spec.Workload, field.NewPath("spec", "workload"))...)
 	allErrs = append(allErrs, validateInstances(&r.Spec.Instances, field.NewPath("spec", "instances"))...)
+	allErrs = append(allErrs, validateContainer(r.Spec.AuthProxyContainer, field.NewPath("spec", "authProxyContainer"))...)
 
 	return allErrs
 
 }
 
+func validateContainer(spec *AuthProxyContainerSpec, f *field.Path) field.ErrorList {
+	if spec == nil {
+		return nil
+	}
+
+	var allErrs field.ErrorList
+	if spec.AdminServer != nil && len(spec.AdminServer.EnableAPIs) == 0 {
+		allErrs = append(allErrs, field.Invalid(
+			f.Child("adminServer", "enableAPIs"), nil,
+			"enableAPIs must have at least one valid element: Debug or QuitQuitQuit"))
+	}
+	if spec.AdminServer != nil {
+		errors := apivalidation.IsValidPortNum(int(spec.AdminServer.Port))
+		for _, e := range errors {
+			allErrs = append(allErrs, field.Invalid(
+				f.Child("adminServer", "port"),
+				spec.AdminServer.Port, e))
+		}
+	}
+
+	return allErrs
+}
+
 // validateUpdateFrom checks that an update to an AuthProxyWorkload resource
 // adheres to these rules:
 // - No changes to the workload selector
diff --git a/internal/api/v1alpha1/zz_generated.deepcopy.go b/internal/api/v1alpha1/zz_generated.deepcopy.go
index 4d1e3a8..4e9e3b5 100644
--- a/internal/api/v1alpha1/zz_generated.deepcopy.go
+++ b/internal/api/v1alpha1/zz_generated.deepcopy.go
@@ -25,6 +25,26 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 )
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *AdminServerSpec) DeepCopyInto(out *AdminServerSpec) {
+	*out = *in
+	if in.EnableAPIs != nil {
+		in, out := &in.EnableAPIs, &out.EnableAPIs
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AdminServerSpec.
+func (in *AdminServerSpec) DeepCopy() *AdminServerSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(AdminServerSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *AuthProxyContainerSpec) DeepCopyInto(out *AuthProxyContainerSpec) {
 	*out = *in
@@ -43,6 +63,11 @@ func (in *AuthProxyContainerSpec) DeepCopyInto(out *AuthProxyContainerSpec) {
 		*out = new(TelemetrySpec)
 		(*in).DeepCopyInto(*out)
 	}
+	if in.AdminServer != nil {
+		in, out := &in.AdminServer, &out.AdminServer
+		*out = new(AdminServerSpec)
+		(*in).DeepCopyInto(*out)
+	}
 	if in.MaxConnections != nil {
 		in, out := &in.MaxConnections, &out.MaxConnections
 		*out = new(int64)
diff --git a/internal/workload/podspec_updates.go b/internal/workload/podspec_updates.go
index 4c0a90a..6467338 100644
--- a/internal/workload/podspec_updates.go
+++ b/internal/workload/podspec_updates.go
@@ -45,6 +45,10 @@ const (
 	// DefaultHealthCheckPort is the used by the proxy to expose prometheus
 	// and kubernetes health checks.
 	DefaultHealthCheckPort int32 = 9801
+
+	// DefaultAdminPort is the used by the proxy to expose prometheus
+	// and kubernetes health checks.
+	DefaultAdminPort int32 = 9802
 )
 
 var l = logf.Log.WithName("internal.workload")
@@ -504,6 +508,9 @@ func (s *updateState) updateContainer(p *cloudsqlapi.AuthProxyWorkload, wl Workl
 	// always enable http port healthchecks on 0.0.0.0 and structured logs
 	s.addHealthCheck(p, c)
 
+	// enable the proxy's admin service
+	s.addAdminServer(p)
+
 	// add the user agent
 	s.addProxyContainerEnvVar(p, "CSQL_PROXY_USER_AGENT", s.updater.userAgent)
 
@@ -663,7 +670,7 @@ func (s *updateState) addHealthCheck(p *cloudsqlapi.AuthProxyWorkload, c *corev1
 
 	cs := p.Spec.AuthProxyContainer
 
-	// if the TelemetrySpec.exists, get Port values
+	// if the TelemetrySpec.exists, get Port and Port values
 	if cs != nil && cs.Telemetry != nil {
 		if cs.Telemetry.HTTPPort != nil {
 			portPtr = cs.Telemetry.HTTPPort
@@ -694,11 +701,30 @@ func (s *updateState) addHealthCheck(p *cloudsqlapi.AuthProxyWorkload, c *corev1
 		PeriodSeconds: 30,
 	}
 	// Add a port that is associated with the proxy, but not a specific db instance
-	s.addPort(port, proxyInstanceID{AuthProxyWorkload: types.NamespacedName{Namespace: p.Namespace, Name: p.Name}})
+	s.addProxyPort(port, p)
 	s.addProxyContainerEnvVar(p, "CSQL_PROXY_HTTP_PORT", fmt.Sprintf("%d", port))
 	s.addProxyContainerEnvVar(p, "CSQL_PROXY_HTTP_ADDRESS", "0.0.0.0")
 	s.addProxyContainerEnvVar(p, "CSQL_PROXY_HEALTH_CHECK", "true")
-	return
+}
+
+func (s *updateState) addAdminServer(p *cloudsqlapi.AuthProxyWorkload) {
+
+	if p.Spec.AuthProxyContainer == nil || p.Spec.AuthProxyContainer.AdminServer == nil {
+		return
+	}
+
+	cs := p.Spec.AuthProxyContainer.AdminServer
+	s.addProxyPort(cs.Port, p)
+	s.addProxyContainerEnvVar(p, "CSQL_PROXY_ADMIN_PORT", fmt.Sprintf("%d", cs.Port))
+	for _, name := range cs.EnableAPIs {
+		switch name {
+		case "Debug":
+			s.addProxyContainerEnvVar(p, "CSQL_PROXY_DEBUG", "true")
+		case "QuitQuitQuit":
+			s.addProxyContainerEnvVar(p, "CSQL_PROXY_QUITQUITQUIT", "true")
+		}
+	}
+
 }
 
 func (s *updateState) addVolumeMount(p *cloudsqlapi.AuthProxyWorkload, is *cloudsqlapi.InstanceSpec, m corev1.VolumeMount, v corev1.Volume) {


Test patch:
diff --git a/internal/api/v1alpha1/authproxyworkload_test.go b/internal/api/v1alpha1/authproxyworkload_test.go
index f708733..102816e 100644
--- a/internal/api/v1alpha1/authproxyworkload_test.go
+++ b/internal/api/v1alpha1/authproxyworkload_test.go
@@ -22,11 +22,12 @@ import (
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-func ptr[T int | int32 | int64 | string](i T) *T {
+func ptr[T int | int32 | int64 | string | bool](i T) *T {
 	return &i
 }
 
-func TestAuthProxyWorkload_ValidateCreate_Instances(t *testing.T) {
+func TestAuthProxyWorkload_ValidateCreate_InstanceSpec(t *testing.T) {
+
 	data := []struct {
 		desc      string
 		spec      []cloudsqlapi.InstanceSpec
@@ -217,6 +218,77 @@ func TestAuthProxyWorkload_ValidateCreate_WorkloadSpec(t *testing.T) {
 		})
 	}
 }
+func TestAuthProxyWorkload_ValidateCreate_AuthProxyContainerSpec(t *testing.T) {
+	wantPort := int32(9393)
+
+	data := []struct {
+		desc      string
+		spec      cloudsqlapi.AuthProxyContainerSpec
+		wantValid bool
+	}{
+
+		{
+			desc: "Valid, Debug and AdminPort set",
+			spec: cloudsqlapi.AuthProxyContainerSpec{
+				AdminServer: &cloudsqlapi.AdminServerSpec{
+					EnableAPIs: []string{"Debug"},
+					Port:       wantPort,
+				},
+			},
+			wantValid: true,
+		},
+		{
+			desc: "Invalid, Debug set without AdminPort",
+			spec: cloudsqlapi.AuthProxyContainerSpec{
+				AdminServer: &cloudsqlapi.AdminServerSpec{
+					EnableAPIs: []string{"Debug"},
+				},
+			},
+			wantValid: false,
+		},
+		{
+			desc: "Invalid, EnableAPIs is empty",
+			spec: cloudsqlapi.AuthProxyContainerSpec{
+				AdminServer: &cloudsqlapi.AdminServerSpec{
+					Port: wantPort,
+				},
+			},
+			wantValid: false,
+		},
+	}
+
+	for _, tc := range data {
+		t.Run(tc.desc, func(t *testing.T) {
+			p := cloudsqlapi.AuthProxyWorkload{
+				ObjectMeta: v1.ObjectMeta{Name: "sample"},
+				Spec: cloudsqlapi.AuthProxyWorkloadSpec{
+					Workload: cloudsqlapi.WorkloadSelectorSpec{
+						Kind: "Deployment",
+						Name: "webapp",
+					},
+					AuthProxyContainer: &tc.spec,
+					Instances: []cloudsqlapi.InstanceSpec{{
+						ConnectionString: "proj:region:db2",
+						Port:             ptr(int32(2443)),
+					}},
+				},
+			}
+			p.Default()
+			err := p.ValidateCreate()
+			gotValid := err == nil
+			switch {
+			case tc.wantValid && !gotValid:
+				t.Errorf("wants create valid, got error %v", err)
+				printFieldErrors(t, err)
+			case !tc.wantValid && gotValid:
+				t.Errorf("wants an error on create, got no error")
+			default:
+				t.Logf("create passed %s", tc.desc)
+				// test passes, do nothing.
+			}
+		})
+	}
+}
 
 func TestAuthProxyWorkload_ValidateUpdate(t *testing.T) {
 	data := []struct {
diff --git a/internal/workload/podspec_updates_test.go b/internal/workload/podspec_updates_test.go
index 85af0bf..2e6d911 100644
--- a/internal/workload/podspec_updates_test.go
+++ b/internal/workload/podspec_updates_test.go
@@ -461,7 +461,7 @@ func TestContainerReplaced(t *testing.T) {
 
 }
 
-func ptr[T int | int32 | int64 | string](i T) *T {
+func ptr[T int | int32 | int64 | string | bool](i T) *T {
 	return &i
 }
 
@@ -512,19 +512,19 @@ func TestResourcesFromSpec(t *testing.T) {
 }
 
 func TestProxyCLIArgs(t *testing.T) {
-	type testParam struct {
-		desc                 string
-		proxySpec            v1alpha1.AuthProxyWorkloadSpec
-		wantProxyArgContains []string
-		wantErrorCodes       []string
-		wantWorkloadEnv      map[string]string
-	}
 	wantTrue := true
 	wantFalse := false
 
 	var wantPort int32 = 5000
 
-	var testcases = []testParam{
+	testcases := []struct {
+		desc                 string
+		proxySpec            v1alpha1.AuthProxyWorkloadSpec
+		wantProxyArgContains []string
+		wantErrorCodes       []string
+		wantWorkloadEnv      map[string]string
+		dontWantEnvSet       []string
+	}{
 		{
 			desc: "default cli config",
 			proxySpec: v1alpha1.AuthProxyWorkloadSpec{
@@ -630,11 +630,16 @@ func TestProxyCLIArgs(t *testing.T) {
 					Telemetry: &v1alpha1.TelemetrySpec{
 						HTTPPort: ptr(int32(9092)),
 					},
+					AdminServer: &v1alpha1.AdminServerSpec{
+						EnableAPIs: []string{"Debug", "QuitQuitQuit"},
+						Port:       int32(9091),
+					},
 					MaxConnections:  ptr(int64(10)),
 					MaxSigtermDelay: ptr(int64(20)),
 				},
 				Instances: []v1alpha1.InstanceSpec{{
 					ConnectionString: "hello:world:one",
+					Port:             ptr(int32(5000)),
 				}},
 			},
 			wantProxyArgContains: []string{
@@ -643,11 +648,31 @@ func TestProxyCLIArgs(t *testing.T) {
 			wantWorkloadEnv: map[string]string{
 				"CSQL_PROXY_SQLADMIN_API_ENDPOINT": "https://example.com",
 				"CSQL_PROXY_HTTP_PORT":             "9092",
+				"CSQL_PROXY_ADMIN_PORT":            "9091",
+				"CSQL_PROXY_DEBUG":                 "true",
+				"CSQL_PROXY_QUITQUITQUIT":          "true",
 				"CSQL_PROXY_HEALTH_CHECK":          "true",
 				"CSQL_PROXY_MAX_CONNECTIONS":       "10",
 				"CSQL_PROXY_MAX_SIGTERM_DELAY":     "20",
 			},
 		},
+		{
+			desc: "No admin port enabled when AdminServerSpec is nil",
+			proxySpec: v1alpha1.AuthProxyWorkloadSpec{
+				AuthProxyContainer: &v1alpha1.AuthProxyContainerSpec{},
+				Instances: []v1alpha1.InstanceSpec{{
+					ConnectionString: "hello:world:one",
+					Port:             ptr(int32(5000)),
+				}},
+			},
+			wantProxyArgContains: []string{
+				fmt.Sprintf("hello:world:one?port=%d", workload.DefaultFirstPort),
+			},
+			wantWorkloadEnv: map[string]string{
+				"CSQL_PROXY_HEALTH_CHECK": "true",
+			},
+			dontWantEnvSet: []string{"CSQL_PROXY_DEBUG", "CSQL_PROXY_ADMIN_PORT"},
+		},
 		{
 			desc: "port conflict with other instance causes error",
 			proxySpec: v1alpha1.AuthProxyWorkloadSpec{
@@ -682,8 +707,7 @@ func TestProxyCLIArgs(t *testing.T) {
 		},
 	}
 
-	for i := 0; i < len(testcases); i++ {
-		tc := &testcases[i]
+	for _, tc := range testcases {
 		t.Run(tc.desc, func(t *testing.T) {
 			u := workload.NewUpdater("cloud-sql-proxy-operator/dev")
 
@@ -700,6 +724,12 @@ func TestProxyCLIArgs(t *testing.T) {
 			// Create a AuthProxyWorkload that matches the deployment
 			csqls := []*v1alpha1.AuthProxyWorkload{authProxyWorkloadFromSpec("instance1", tc.proxySpec)}
 
+			// ensure valid
+			err := csqls[0].ValidateCreate()
+			if err != nil {
+				t.Fatal("Invalid AuthProxyWorkload resource", err)
+			}
+
 			// update the containers
 			updateErr := configureProxies(u, wl, csqls)
 
@@ -734,6 +764,13 @@ func TestProxyCLIArgs(t *testing.T) {
 					t.Errorf("got %v, wants %v workload env var %v", gotEnvVar, wantValue, wantKey)
 				}
 			}
+			for _, dontWantKey := range tc.dontWantEnvSet {
+				gotEnvVar, err := findEnvVar(wl, csqlContainer.Name, dontWantKey)
+				if err != nil {
+					continue
+				}
+				t.Errorf("got env %v=%v, wants no env var set", dontWantKey, gotEnvVar)
+			}
 
 		})
 	}