Repository: GoogleCloudPlatform/cloud-sql-proxy-operator (go)

Issue:
Delete workloads pods that are missing an proxy container when they are in an error state
The operator uses a webhook to add the proxy container to pods. The webhook is setup with `failurePolicy: Ignore`. This was an intentional design choice. It allows pods to be created when the operator is not receiving webhook requests.  (If you create a webhook for all pods with `failurePolicy: Fail`, then when the pods serving that webhook are terminated, the cluster will be unable to start new pods, including the replacement pods for that webhook.)

Unfortunately this means that occasionally a pod related to a workload will be created without the operator adding the proxy sidecar container. This could happen if there is a bug in the webhook causing it to return an error response. This could also happen when all the operator pods are offline, for example when the cluster replaces a node which happens to be running all the operator pods. 

The best solution for now is to program the operator to delete improperly configured pods. The operator will listen for events on all pods related to known AuthProxyWorkloads. If such a pod exits in error, the operator will check the pod resource for the proxy sidecar container. If the proxy sidecar container is missing, the operator will delete the pod. 

In the future, we can use the webhook [matchConditions](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchconditions) with `failurePolicy: Fail` to ensure that the webhook only filters pods related to AuthProxyWorklod, not all pods in the cluster. This k8s feature is alpha in v1.27, so we can't use it yet.


Pull request description:
fix: delete misconfigured pods in error state.
The pod cannot be configured to reliably instrument every single pod creation event due to limitations in the K8s API. 
Thus, the operator needs to listen for pod create and update events, to check if the pod is configured correctly. If the
pod is not configured correctly, and the pod is in an error or waiting state, the operator will delete the pod so that it can
be replaced. The replacement pod will be correctly configured by the webhook as it is created. 

Fixes #337

Patch:
diff --git a/internal/controller/pod_controller.go b/internal/controller/pod_controller.go
index a1fd0de..c3a8165 100644
--- a/internal/controller/pod_controller.go
+++ b/internal/controller/pod_controller.go
@@ -22,9 +22,12 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/json"
+	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
 
 	cloudsqlapi "github.com/GoogleCloudPlatform/cloud-sql-proxy-operator/internal/api/v1"
@@ -74,6 +77,7 @@ func (a *PodAdmissionWebhook) Handle(ctx context.Context, req admission.Request)
 		return admission.Errored(http.StatusInternalServerError,
 			fmt.Errorf("unable to marshal workload result"))
 	}
+	l.Info("updated proxy on pod", "Operation", req.Operation, "Namespace", req.Namespace, "Name", req.Name)
 
 	return admission.PatchResponseFromRaw(req.Object.Raw, marshaledRes)
 }
@@ -81,18 +85,43 @@ func (a *PodAdmissionWebhook) Handle(ctx context.Context, req admission.Request)
 // handleCreatePodRequest Finds relevant AuthProxyWorkload resources and updates the pod
 // with matching resources, returning a non-nil pod when the pod was updated.
 func (a *PodAdmissionWebhook) handleCreatePodRequest(ctx context.Context, p corev1.Pod) (*corev1.Pod, error) {
+	l := logf.FromContext(ctx)
+	wl := &workload.PodWorkload{Pod: &p}
+
+	proxies, err := findMatchingProxies(ctx, a.Client, a.updater, wl)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(proxies) == 0 {
+		return nil, nil
+	}
+
+	// Configure the pod, adding containers for each of the proxies
+	wlConfigErr := a.updater.ConfigureWorkload(wl, proxies)
+
+	if wlConfigErr != nil {
+		l.Error(wlConfigErr, "Unable to reconcile workload result in webhook: "+wlConfigErr.Error(),
+			"kind", wl.Pod.Kind, "ns", wl.Pod.Namespace, "name", wl.Pod.Name)
+		return nil, fmt.Errorf("there is an AuthProxyWorkloadConfiguration error reconciling this workload %v", wlConfigErr)
+	}
+
+	return wl.Pod, nil // updated pod
+}
+
+// findMatchingProxies lists all AuthProxyWorkloads that are related to this pod
+// or its owners.
+func findMatchingProxies(ctx context.Context, c client.Client, u *workload.Updater, wl *workload.PodWorkload) ([]*cloudsqlapi.AuthProxyWorkload, error) {
 	var (
-		instList    = &cloudsqlapi.AuthProxyWorkloadList{}
-		proxies     []*cloudsqlapi.AuthProxyWorkload
-		wlConfigErr error
-		l           = logf.FromContext(ctx)
-		wl          = &workload.PodWorkload{Pod: &p}
+		instList = &cloudsqlapi.AuthProxyWorkloadList{}
+		proxies  []*cloudsqlapi.AuthProxyWorkload
+		l        = logf.FromContext(ctx)
 	)
 
 	// List all the AuthProxyWorkloads in the same namespace.
 	// To avoid privilege escalation, the operator requires that the AuthProxyWorkload
 	// may only affect pods in the same namespace.
-	err := a.Client.List(ctx, instList, client.InNamespace(wl.Object().GetNamespace()))
+	err := c.List(ctx, instList, client.InNamespace(wl.Object().GetNamespace()))
 	if err != nil {
 		l.Error(err, "Unable to list CloudSqlClient resources in webhook",
 			"kind", wl.Pod.Kind, "ns", wl.Pod.Namespace, "name", wl.Pod.Name)
@@ -100,32 +129,24 @@ func (a *PodAdmissionWebhook) handleCreatePodRequest(ctx context.Context, p core
 	}
 
 	// List the owners of this pod.
-	owners, err := a.listOwners(ctx, wl.Object())
+	owners, err := listOwners(ctx, c, wl.Object())
 	if err != nil {
 		return nil, fmt.Errorf("there is an AuthProxyWorkloadConfiguration error reconciling this workload %v", err)
 	}
 
 	// Find matching AuthProxyWorkloads for this pod
-	proxies = a.updater.FindMatchingAuthProxyWorkloads(instList, wl, owners)
+	proxies = u.FindMatchingAuthProxyWorkloads(instList, wl, owners)
 	if len(proxies) == 0 {
 		return nil, nil // no change
 	}
 
-	// Configure the pod, adding containers for each of the proxies
-	wlConfigErr = a.updater.ConfigureWorkload(wl, proxies)
-
-	if wlConfigErr != nil {
-		l.Error(wlConfigErr, "Unable to reconcile workload result in webhook: "+wlConfigErr.Error(),
-			"kind", wl.Pod.Kind, "ns", wl.Pod.Namespace, "name", wl.Pod.Name)
-		return nil, fmt.Errorf("there is an AuthProxyWorkloadConfiguration error reconciling this workload %v", wlConfigErr)
-	}
+	return proxies, nil
 
-	return wl.Pod, nil // updated
 }
 
 // listOwners returns the list of this object's owners and its extended owners.
 // Warning: this is a recursive function
-func (a *PodAdmissionWebhook) listOwners(ctx context.Context, object client.Object) ([]workload.Workload, error) {
+func listOwners(ctx context.Context, c client.Client, object client.Object) ([]workload.Workload, error) {
 	l := logf.FromContext(ctx)
 	var owners []workload.Workload
 
@@ -143,7 +164,7 @@ func (a *PodAdmissionWebhook) listOwners(ctx context.Context, object client.Obje
 		owners = append(owners, wl)
 		owner = wl.Object()
 
-		err = a.Client.Get(ctx, key, owner)
+		err = c.Get(ctx, key, owner)
 		if err != nil {
 			switch t := err.(type) {
 			case *apierrors.StatusError:
@@ -157,9 +178,9 @@ func (a *PodAdmissionWebhook) listOwners(ctx context.Context, object client.Obje
 			return nil, err
 		}
 
-		// recursively call for the owners of the owner, and append those.
+		// Recursively call for the owners of the owner, and append those.
 		// So that we reach Pod --> ReplicaSet --> Deployment
-		ownerOwners, err := a.listOwners(ctx, owner)
+		ownerOwners, err := listOwners(ctx, c, owner)
 		if err != nil {
 			return nil, err
 		}
@@ -168,3 +189,85 @@ func (a *PodAdmissionWebhook) listOwners(ctx context.Context, object client.Obje
 	}
 	return owners, nil
 }
+
+type podDeleteController struct {
+	client.Client
+	Scheme  *runtime.Scheme
+	updater *workload.Updater
+}
+
+// newDeletePodController constructs a podDeleteController.
+func newPodDeleteController(mgr ctrl.Manager, u *workload.Updater) (*podDeleteController, error) {
+	r := &podDeleteController{
+		Client:  mgr.GetClient(),
+		Scheme:  mgr.GetScheme(),
+		updater: u,
+	}
+	err := r.setupWithManager(mgr)
+	return r, err
+}
+
+// setupWithManager adds this AuthProxyWorkload controller to the controller-runtime
+// manager.
+func (r *podDeleteController) setupWithManager(mgr ctrl.Manager) error {
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&corev1.Pod{}).
+		Complete(r)
+}
+
+func (r *podDeleteController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	// Read the ReplicaSet
+	pod := &corev1.Pod{}
+	err := r.Client.Get(ctx, req.NamespacedName, pod)
+	if err != nil {
+		return reconcile.Result{}, client.IgnoreNotFound(err)
+	}
+	// Pod is already deleted, ignore this change event.
+	if !pod.ObjectMeta.DeletionTimestamp.IsZero() {
+		return reconcile.Result{}, nil
+	}
+
+	err = r.handlePodChanged(ctx, pod)
+	if err != nil {
+		return reconcile.Result{Requeue: true}, err
+	}
+
+	return reconcile.Result{}, nil
+}
+
+// handlePodChanged Deletes pods that meet the following criteria:
+// 1. The pod is in Error or CrashLoopBackoff state.
+// 2. The pod matches one or more AuthProxyWorkload resources.
+// 3. The pod is missing one or more proxy sidecar containers for the resources.
+func (r *podDeleteController) handlePodChanged(ctx context.Context, pod *corev1.Pod) error {
+
+	wl := &workload.PodWorkload{Pod: pod}
+
+	// Find all proxies that match this pod
+	proxies, err := findMatchingProxies(ctx, r.Client, r.updater, wl)
+	if err != nil {
+		return fmt.Errorf("unable to find proxies when pod changed, %v", err)
+	}
+
+	// There are no proxies, ignore this event.
+	if len(proxies) == 0 {
+		return nil
+	}
+
+	// Check if this pod is in an error or waiting state and is missing
+	// proxy containers.
+	wlConfigErr := r.updater.CheckWorkloadContainers(wl, proxies)
+
+	// If the pod is misconfigured, attempt to delete it.
+	if wlConfigErr != nil {
+		l := logf.FromContext(ctx)
+		l.Info("Pod configured incorrectly. Deleting.",
+			"Namespace", pod.Namespace, "Name", pod.Name, "Status", pod.Status)
+		err = r.Client.Delete(ctx, pod)
+		if err != nil && !apierrors.IsNotFound(err) {
+			return fmt.Errorf("unable to delete pod %v/%v, %v", pod.Namespace, pod.Name, err)
+		}
+	}
+
+	return nil
+}
diff --git a/internal/controller/setup.go b/internal/controller/setup.go
index ac55215..bb582aa 100644
--- a/internal/controller/setup.go
+++ b/internal/controller/setup.go
@@ -36,7 +36,7 @@ func InitScheme(scheme *runtime.Scheme) {
 	//+kubebuilder:scaffold:scheme
 }
 
-// SetupManagers was moved out of ../main.go to here so that it can be invoked
+// SetupManagers was moved out of ../main.go here so that it can be invoked
 // from the testintegration tests AND from the actual operator.
 func SetupManagers(mgr manager.Manager, userAgent, defaultProxyImage string) error {
 	u := workload.NewUpdater(userAgent, defaultProxyImage)
@@ -72,6 +72,14 @@ func SetupManagers(mgr manager.Manager, userAgent, defaultProxyImage string) err
 		return err
 	}
 
+	// Register the podDeleteController, which will listen for pod changes and
+	// delete misconfigured pods that in a waiting or error state.
+	_, err = newPodDeleteController(mgr, u)
+	if err != nil {
+		setupLog.Error(err, "unable to create pod informer")
+		return err
+	}
+
 	// Add the runnable task that will upgrade the proxy image on workloads with
 	// default container image when  the operator first starts.
 	err = mgr.Add(&upgradeDefaultProxyOnStartup{c: mgr.GetClient()})
diff --git a/internal/workload/podspec_updates.go b/internal/workload/podspec_updates.go
index 6532970..aeb4269 100644
--- a/internal/workload/podspec_updates.go
+++ b/internal/workload/podspec_updates.go
@@ -203,16 +203,55 @@ func (u *Updater) filterMatchingInstances(pl *cloudsqlapi.AuthProxyWorkloadList,
 			}
 
 			matchingAuthProxyWorkloads = append(matchingAuthProxyWorkloads, p)
-			// need to update wl
-			l.Info("Found matching wl",
-				"wl", wl.GetNamespace()+"/"+wl.GetName(),
-				"wlSelector", p.Spec.Workload,
-				"AuthProxyWorkload", p.Namespace+"/"+p.Name)
 		}
 	}
 	return matchingAuthProxyWorkloads
 }
 
+// CheckWorkloadContainers determines if a pod is configured incorrectly and
+// therefore needs to be deleted. Pods must be (1) missing one or more proxy
+// sidecar containers and (2) have a terminated container.
+func (u *Updater) CheckWorkloadContainers(wl *PodWorkload, matches []*cloudsqlapi.AuthProxyWorkload) error {
+
+	// Find the names of all AuthProxyWorkload resources that should have a
+	// container on this pod, but there is no container.
+	var missing []string
+	for _, p := range matches {
+		wantName := ContainerName(p)
+		var found bool
+		for _, c := range wl.PodSpec().Containers {
+			if c.Name == wantName {
+				found = true
+			}
+		}
+		if !found {
+			missing = append(missing, p.Name)
+			break
+		}
+	}
+
+	// If no containers are missing, then there is no error, return nil.
+	if len(missing) == 0 {
+		return nil
+	}
+
+	missingSidecars := strings.Join(missing, ", ")
+
+	// Some proxy containers are missing. Are the remaining pod containers failing?
+	for _, cs := range wl.Pod.Status.ContainerStatuses {
+		if cs.State.Terminated != nil && cs.State.Terminated.Reason == "Error" {
+			return fmt.Errorf("pod is in an error state and missing sidecar containers %v", missingSidecars)
+		}
+		if cs.State.Waiting != nil && cs.State.Waiting.Reason == "CrashLoopBackoff" {
+			return fmt.Errorf("pod is in a CrashLoopBackoff state and missing sidecar containers %v", missingSidecars)
+		}
+	}
+
+	// Pod's other containers are not in an error state. Operator should not
+	// interrupt running containers.
+	return nil
+}
+
 // ConfigureWorkload applies the proxy containers from all of the
 // instances listed in matchingAuthProxyWorkloads to the workload
 func (u *Updater) ConfigureWorkload(wl *PodWorkload, matches []*cloudsqlapi.AuthProxyWorkload) error {


Test patch:
diff --git a/internal/controller/pod_controller_test.go b/internal/controller/pod_controller_test.go
index e5cbd2f..de92f9f 100644
--- a/internal/controller/pod_controller_test.go
+++ b/internal/controller/pod_controller_test.go
@@ -22,8 +22,11 @@ import (
 	"github.com/GoogleCloudPlatform/cloud-sql-proxy-operator/internal/testhelpers"
 	"github.com/GoogleCloudPlatform/cloud-sql-proxy-operator/internal/workload"
 	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	"sigs.k8s.io/controller-runtime/pkg/log"
@@ -156,3 +159,144 @@ func podWebhookController(cb client.Client) (*PodAdmissionWebhook, context.Conte
 
 	return r, ctx, nil
 }
+
+func TestPodDeleteController(t *testing.T) {
+	// Proxy workload
+	p := testhelpers.BuildAuthProxyWorkload(types.NamespacedName{
+		Namespace: "default",
+		Name:      "test",
+	}, "project:region:db")
+	addFinalizers(p)
+	addSelectorWorkload(p, "Deployment", "app", "webapp")
+
+	// Deployment that matches the proxy
+	dMatch := testhelpers.BuildDeployment(types.NamespacedName{
+		Namespace: "default",
+		Name:      "test",
+	}, "webapp")
+	dMatch.ObjectMeta.Labels = map[string]string{"app": "webapp"}
+
+	// Deployment that does not match the proxy
+	dNoMatch := testhelpers.BuildDeployment(types.NamespacedName{
+		Namespace: "default",
+		Name:      "test",
+	}, "webapp")
+	dNoMatch.ObjectMeta.Labels = map[string]string{"app": "other"}
+
+	data := []struct {
+		name                 string
+		d                    *appsv1.Deployment
+		wantNotFound         bool
+		setPodError          bool
+		setSidecarContainers bool
+	}{
+		{
+			name:         "matching pod with error gets deleted",
+			d:            dMatch,
+			setPodError:  true,
+			wantNotFound: true,
+		},
+		{
+			name:         "matching pod with no error",
+			d:            dMatch,
+			setPodError:  false,
+			wantNotFound: false,
+		},
+		{
+			name:         "no matching workload, pod ok",
+			d:            dNoMatch,
+			setPodError:  false,
+			wantNotFound: false,
+		},
+		{
+			name:         "no matching workload, pod error",
+			d:            dNoMatch,
+			setPodError:  true,
+			wantNotFound: false,
+		},
+		{
+			name:                 "matching workload, pod error, has containers",
+			d:                    dNoMatch,
+			setPodError:          true,
+			setSidecarContainers: true,
+			wantNotFound:         false,
+		},
+		{
+			name:                 "matching workload, pod ok, has containers",
+			d:                    dNoMatch,
+			setPodError:          false,
+			setSidecarContainers: true,
+			wantNotFound:         false,
+		},
+	}
+
+	for _, tc := range data {
+		t.Run(tc.name, func(t *testing.T) {
+
+			cb, scheme, err := clientBuilder()
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			rs, hash, err := testhelpers.BuildDeploymentReplicaSet(tc.d, scheme)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			pods, err := testhelpers.BuildDeploymentReplicaSetPods(tc.d, rs, hash, scheme)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if tc.setPodError {
+				pods[0].Status.ContainerStatuses = []corev1.ContainerStatus{{
+					Name:  pods[0].Spec.Containers[0].Name,
+					State: corev1.ContainerState{Waiting: &corev1.ContainerStateWaiting{Reason: "CrashLoopBackoff"}},
+				}}
+			}
+
+			cb = cb.WithObjects(p)
+			cb = cb.WithObjects(tc.d)
+			cb = cb.WithObjects(rs)
+
+			for _, p := range pods {
+				cb = cb.WithObjects(p)
+			}
+			c := cb.Build()
+			h, ctx := podDeleteControllerForTest(c)
+			if tc.setSidecarContainers {
+				h.updater.ConfigureWorkload(&workload.PodWorkload{Pod: pods[0]}, []*cloudsqlapi.AuthProxyWorkload{p})
+			}
+
+			req := ctrl.Request{
+				NamespacedName: types.NamespacedName{
+					Namespace: pods[0].Namespace,
+					Name:      pods[0].Name,
+				},
+			}
+
+			h.Reconcile(ctx, req)
+
+			var deletedPod corev1.Pod
+			err = c.Get(ctx, client.ObjectKeyFromObject(pods[0]), &deletedPod)
+
+			if !tc.wantNotFound && errors.IsNotFound(err) {
+				t.Fatalf("want not found error, got %v", err)
+			}
+			if tc.wantNotFound && !errors.IsNotFound(err) {
+				t.Fatalf("want no error, found error, got %v", err)
+			}
+
+		})
+	}
+
+}
+
+func podDeleteControllerForTest(c client.Client) (*podDeleteController, context.Context) {
+	ctx := log.IntoContext(context.Background(), logger)
+	r := &podDeleteController{
+		Client:  c,
+		Scheme:  c.Scheme(),
+		updater: workload.NewUpdater("cloud-sql-proxy-operator/dev", "proxy:1.0"),
+	}
+	return r, ctx
+}
diff --git a/internal/workload/podspec_updates_test.go b/internal/workload/podspec_updates_test.go
index 6720540..3005da8 100644
--- a/internal/workload/podspec_updates_test.go
+++ b/internal/workload/podspec_updates_test.go
@@ -1016,3 +1016,110 @@ func TestWorkloadUnixVolume(t *testing.T) {
 	}
 
 }
+
+func TestUpdater_CheckWorkloadContainers(t *testing.T) {
+	var (
+		wantsInstanceName = "project:server:db"
+		u                 = workload.NewUpdater("cloud-sql-proxy-operator/dev", workload.DefaultProxyImage)
+	)
+
+	// Create a AuthProxyWorkloads to match the pods
+	p1 := simpleAuthProxy("instance1", wantsInstanceName)
+	p2 := simpleAuthProxy("instance2", wantsInstanceName)
+	csqls := []*cloudsqlapi.AuthProxyWorkload{p1, p2}
+
+	// Pod configuration says it is running, and has all proxy containers
+	wlCfgRunning := podWorkload()
+	err := configureProxies(u, wlCfgRunning, csqls)
+	if err != nil {
+		t.Fatal(err)
+	}
+	wlCfgRunning.Pod.Status.ContainerStatuses = []corev1.ContainerStatus{{
+		Name:  wlCfgRunning.Pod.Spec.Containers[0].Name,
+		State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{StartedAt: metav1.Now()}},
+	}}
+
+	// Pod configuration says it is in error, and has all proxy containers
+	wlCfgError := podWorkload()
+	err = configureProxies(u, wlCfgError, csqls)
+	if err != nil {
+		t.Fatal(err)
+	}
+	wlCfgError.Pod.Status.ContainerStatuses = []corev1.ContainerStatus{{
+		Name:  wlCfgError.Pod.Spec.Containers[0].Name,
+		State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{Reason: "Error", ExitCode: 2}},
+	}}
+
+	// Pod configuration says it is waiting, and has all proxy containers
+	wlCfgWait := podWorkload()
+	err = configureProxies(u, wlCfgWait, csqls)
+	if err != nil {
+		t.Fatal(err)
+	}
+	wlCfgWait.Pod.Status.ContainerStatuses = []corev1.ContainerStatus{{
+		Name:  wlCfgWait.Pod.Spec.Containers[0].Name,
+		State: corev1.ContainerState{Waiting: &corev1.ContainerStateWaiting{Reason: "CrashLoopBackoff"}},
+	}}
+
+	// Pod configuration says it is running and is missing its proxy containers
+	wlCfgMissingRunning := podWorkload()
+	wlCfgMissingRunning.Pod.Status.ContainerStatuses = []corev1.ContainerStatus{{
+		Name:  wlCfgMissingRunning.Pod.Spec.Containers[0].Name,
+		State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{StartedAt: metav1.Now()}},
+	}}
+
+	// Pod configuration says it is in error and is missing its proxy containers
+	wlCfgMissingError := podWorkload()
+	wlCfgMissingError.Pod.Status.ContainerStatuses = []corev1.ContainerStatus{{
+		Name:  wlCfgMissingError.Pod.Spec.Containers[0].Name,
+		State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{Reason: "Error", ExitCode: 2}},
+	}}
+
+	// Pod configuration says it is waiting is missing its proxy containers
+	wlCfgMissingWait := podWorkload()
+	wlCfgMissingWait.Pod.Status.ContainerStatuses = []corev1.ContainerStatus{{
+		Name:  wlCfgMissingWait.Pod.Spec.Containers[0].Name,
+		State: corev1.ContainerState{Waiting: &corev1.ContainerStateWaiting{Reason: "CrashLoopBackoff"}},
+	}}
+
+	// Pod configuration says it is in error and is missing one of the 2 sidecar
+	// containers.
+	wlCfgOutOfDateError := podWorkload()
+	// Only configure 1 of the 2 expected AuthProxyWorkload sidecar containers
+	err = configureProxies(u, wlCfgWait, []*cloudsqlapi.AuthProxyWorkload{p1})
+	if err != nil {
+		t.Fatal(err)
+	}
+	wlCfgOutOfDateError.Pod.Status.ContainerStatuses = []corev1.ContainerStatus{{
+		Name:  wlCfgOutOfDateError.Pod.Spec.Containers[0].Name,
+		State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{Reason: "Error", ExitCode: 2}},
+	}}
+
+	tests := []struct {
+		name    string
+		wl      *workload.PodWorkload
+		wantErr bool
+	}{
+		{name: "Workload Configured, state running", wl: wlCfgRunning},
+		{name: "Workload Configured, state terminated", wl: wlCfgError},
+		{name: "Workload Configured, state waiting", wl: wlCfgWait},
+		{name: "Config missing, state running", wl: wlCfgMissingRunning},
+		{name: "Config missing, state terminated", wl: wlCfgMissingError, wantErr: true},
+		{name: "Config missing, state waiting", wl: wlCfgMissingWait, wantErr: true},
+		{name: "One container missing, state error", wl: wlCfgOutOfDateError, wantErr: true},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			// update the containers
+			err := u.CheckWorkloadContainers(test.wl, csqls)
+
+			if test.wantErr && err == nil {
+				t.Fatal("want not nil, got nil. err")
+			}
+			if !test.wantErr && err != nil {
+				t.Fatalf("want nil, got %v. err", err)
+			}
+
+		})
+	}
+}