Repository: GoogleCloudPlatform/cloud-sql-proxy-operator
Pull request #200 (2023-02-07 22:10:12)

feat: automatic  changes to workloads when an AuthProxyWorload is deleted
When an AuthProxyWorkload is deleted, automatically update all related workloads that support automatic
rollout: Deployment, StatefulSet, DaemonSet, ReplicaSet.

hessjcg: /gcbrun


Patch:
diff --git a/internal/controller/authproxyworkload_controller.go b/internal/controller/authproxyworkload_controller.go
index e3a9ad5..e808bfc 100644
--- a/internal/controller/authproxyworkload_controller.go
+++ b/internal/controller/authproxyworkload_controller.go
@@ -156,7 +156,7 @@ func (r *AuthProxyWorkloadReconciler) Reconcile(ctx context.Context, req ctrl.Re
 			"gen", resource.GetGeneration())
 		r.recentlyDeleted.set(req.NamespacedName, true)
 		// the object has been deleted
-		return r.doDelete(ctx, resource, l)
+		return r.doDelete(ctx, resource)
 	}
 
 	l.Info("Reconcile add/update for AuthProxyWorkload",
@@ -169,10 +169,15 @@ func (r *AuthProxyWorkloadReconciler) Reconcile(ctx context.Context, req ctrl.Re
 
 // doDelete removes our finalizer and updates the related workloads
 // when the reconcile loop receives an AuthProxyWorkload that was deleted.
-func (r *AuthProxyWorkloadReconciler) doDelete(ctx context.Context, resource *cloudsqlapi.AuthProxyWorkload, l logr.Logger) (ctrl.Result, error) {
+func (r *AuthProxyWorkloadReconciler) doDelete(ctx context.Context, resource *cloudsqlapi.AuthProxyWorkload) (ctrl.Result, error) {
 
 	// Mark all related workloads as needing to be updated
-	_, err := r.updateWorkloadStatus(ctx, l, resource)
+	allWorkloads, err := r.updateWorkloadStatus(ctx, resource)
+	if err != nil {
+		return requeueNow, err
+	}
+
+	_, err = r.updateWorkloadAnnotations(ctx, resource, allWorkloads)
 	if err != nil {
 		return requeueNow, err
 	}
@@ -231,7 +236,7 @@ func (r *AuthProxyWorkloadReconciler) doCreateUpdate(ctx context.Context, l logr
 	}
 
 	// find all workloads that relate to this AuthProxyWorkload resource
-	allWorkloads, err := r.updateWorkloadStatus(ctx, l, resource)
+	allWorkloads, err := r.updateWorkloadStatus(ctx, resource)
 	if err != nil {
 		// State 1.2 - unable to read workloads, abort and try again after a delay.
 		return requeueWithDelay, err
@@ -247,25 +252,9 @@ func (r *AuthProxyWorkloadReconciler) doCreateUpdate(ctx context.Context, l logr
 
 	// State 3.*: Workloads already exist. Some may need to be updated to roll out
 	// changes.
-	var outOfDateCount int
-	for _, wl := range allWorkloads {
-		wlChanged := r.needsAnnotationUpdate(wl, resource)
-		if !wlChanged {
-			continue
-		}
-
-		outOfDateCount++
-		_, err = controllerutil.CreateOrPatch(ctx, r.Client, wl.Object(), func() error {
-			r.updateAnnotation(wl, resource)
-			return nil
-		})
-
-		// State 3.1 Failed to update one of the workloads PodTemplateSpec annotations, requeue.
-		if err != nil {
-			message := fmt.Sprintf("Reconciled %d matching workloads. Error updating workload %v: %v", len(allWorkloads), wl.Object().GetName(), err)
-			return r.reconcileResult(ctx, l, resource, orig, cloudsqlapi.ReasonWorkloadNeedsUpdate, message, false)
-		}
-
+	outOfDateCount, err := r.updateWorkloadAnnotations(ctx, resource, allWorkloads)
+	if err != nil {
+		return requeueNow, err
 	}
 
 	// State 3.2 Successfully updated all workload PodTemplateSpec annotations, requeue
@@ -282,14 +271,12 @@ func (r *AuthProxyWorkloadReconciler) doCreateUpdate(ctx context.Context, l logr
 // needsAnnotationUpdate returns true when the workload was annotated with
 // a different generation of the resource.
 func (r *AuthProxyWorkloadReconciler) needsAnnotationUpdate(wl workload.Workload, resource *cloudsqlapi.AuthProxyWorkload) bool {
-
 	// This workload is not mutable. Ignore it.
 	if _, ok := wl.(workload.WithMutablePodTemplate); !ok {
 		return false
 	}
 
 	k, v := workload.PodAnnotation(resource)
-
 	// Check if the correct annotation exists
 	an := wl.PodTemplateAnnotations()
 	if an != nil && an[k] == v {
@@ -388,16 +375,15 @@ func (r *AuthProxyWorkloadReconciler) patchAuthProxyWorkloadStatus(
 // updates the needs update annotations using internal.UpdateWorkloadAnnotation.
 // Once the workload is saved, the workload admission mutate webhook will
 // apply the correct containers to this instance.
-func (r *AuthProxyWorkloadReconciler) updateWorkloadStatus(ctx context.Context, _ logr.Logger, resource *cloudsqlapi.AuthProxyWorkload) (matching []workload.Workload, retErr error) {
+func (r *AuthProxyWorkloadReconciler) updateWorkloadStatus(ctx context.Context, resource *cloudsqlapi.AuthProxyWorkload) (matching []workload.Workload, retErr error) {
 
 	matching, err := r.listWorkloads(ctx, resource.Spec.Workload, resource.GetNamespace())
 	if err != nil {
 		return nil, err
 	}
 
-	// all matching workloads get a new annotation that will be removed
-	// when the reconcile loop for outOfDate is completed.
 	for _, wl := range matching {
+		// update the status condition for a workload
 		s := newStatus(wl)
 		s.Conditions = replaceCondition(s.Conditions, &metav1.Condition{
 			Type:               cloudsqlapi.ConditionWorkloadUpToDate,
@@ -531,3 +517,25 @@ func (r *AuthProxyWorkloadReconciler) loadByLabelSelector(ctx context.Context, w
 	return wl.Workloads(), nil
 
 }
+
+func (r *AuthProxyWorkloadReconciler) updateWorkloadAnnotations(ctx context.Context, resource *cloudsqlapi.AuthProxyWorkload, workloads []workload.Workload) (int, error) {
+	var outOfDate int
+	for _, wl := range workloads {
+		if r.needsAnnotationUpdate(wl, resource) {
+			outOfDate++
+
+			_, err := controllerutil.CreateOrPatch(ctx, r.Client, wl.Object(), func() error {
+				r.updateAnnotation(wl, resource)
+				return nil
+			})
+
+			// Failed to update one of the workloads PodTemplateSpec annotations.
+			if err != nil {
+				return 0, fmt.Errorf("reconciled %d matching workloads. Error removing proxy from workload %v: %v", len(workloads), wl.Object().GetName(), err)
+			}
+		}
+	}
+
+	return outOfDate, nil
+
+}
diff --git a/internal/workload/podspec_updates.go b/internal/workload/podspec_updates.go
index 2a175c6..4607ba4 100644
--- a/internal/workload/podspec_updates.go
+++ b/internal/workload/podspec_updates.go
@@ -18,6 +18,7 @@ import (
 	"fmt"
 	"sort"
 	"strings"
+	"time"
 
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
@@ -47,12 +48,18 @@ const (
 
 var l = logf.Log.WithName("internal.workload")
 
-// PodAnnotation returns the key and value for an annotation on a pod
-// to indicate the specific AuthProxyWorkload resource that was configured
-// on the pod.
+// PodAnnotation returns the annotation (key, value) that should be added to
+// pods that are configured with this AuthProxyWorkload resource. This takes
+// into account whether the AuthProxyWorkload exists or was recently deleted.
 func PodAnnotation(r *cloudsqlapi.AuthProxyWorkload) (string, string) {
-	return fmt.Sprintf("%s/%s", cloudsqlapi.AnnotationPrefix, r.Name),
-		fmt.Sprintf("%d", r.Generation)
+	k := fmt.Sprintf("%s/%s", cloudsqlapi.AnnotationPrefix, r.Name)
+	v := fmt.Sprintf("%d", r.Generation)
+	// if r was deleted, use a different value
+	if r.GetDeletionTimestamp() != nil {
+		v = fmt.Sprintf("%d-deleted-%s", r.Generation, r.GetDeletionTimestamp().Format(time.RFC3339))
+	}
+
+	return k, v
 }
 
 // Updater holds global state used while reconciling workloads.


Test patch:
diff --git a/internal/controller/authproxyworkload_controller_test.go b/internal/controller/authproxyworkload_controller_test.go
index 2e013fc..b3c2a31 100644
--- a/internal/controller/authproxyworkload_controller_test.go
+++ b/internal/controller/authproxyworkload_controller_test.go
@@ -18,6 +18,7 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"strings"
 	"testing"
 
 	"go.uber.org/zap/zapcore"
@@ -69,11 +70,8 @@ func TestReconcileDeleted(t *testing.T) {
 		Namespace: "default",
 		Name:      "test",
 	}, "project:region:db")
-	p.Finalizers = []string{finalizerName}
-	p.Spec.Workload = v1alpha1.WorkloadSelectorSpec{
-		Kind: "Pod",
-		Name: "thing",
-	}
+	addFinalizers(p)
+	addPodWorkload(p)
 
 	cb, err := clientBuilder()
 	if err != nil {
@@ -114,11 +112,8 @@ func TestReconcileState21ByName(t *testing.T) {
 		Namespace: "default",
 		Name:      "test",
 	}, "project:region:db")
-	p.Finalizers = []string{finalizerName}
-	p.Spec.Workload = v1alpha1.WorkloadSelectorSpec{
-		Kind: "Pod",
-		Name: "testpod",
-	}
+	addFinalizers(p)
+	addPodWorkload(p)
 
 	err := runReconcileTestcase(p, []client.Object{p}, false, metav1.ConditionTrue, v1alpha1.ReasonNoWorkloadsFound)
 	if err != nil {
@@ -131,13 +126,8 @@ func TestReconcileState21BySelector(t *testing.T) {
 		Namespace: "default",
 		Name:      "test",
 	}, "project:region:db")
-	p.Finalizers = []string{finalizerName}
-	p.Spec.Workload = v1alpha1.WorkloadSelectorSpec{
-		Kind: "Pod",
-		Selector: &metav1.LabelSelector{
-			MatchLabels: map[string]string{"app": "things"},
-		},
-	}
+	addFinalizers(p)
+	addSelectorWorkload(p, "Pod", "app", "things")
 
 	err := runReconcileTestcase(p, []client.Object{p}, false, metav1.ConditionTrue, v1alpha1.ReasonNoWorkloadsFound)
 	if err != nil {
@@ -147,27 +137,20 @@ func TestReconcileState21BySelector(t *testing.T) {
 }
 
 func TestReconcileState32(t *testing.T) {
-	wantRequeue := true
-	wantStatus := metav1.ConditionFalse
-	wantReason := v1alpha1.ReasonWorkloadNeedsUpdate
-
+	const (
+		wantRequeue = true
+		wantStatus  = metav1.ConditionFalse
+		wantReason  = v1alpha1.ReasonWorkloadNeedsUpdate
+		labelK      = "app"
+		labelV      = "things"
+	)
 	p := testhelpers.BuildAuthProxyWorkload(types.NamespacedName{
 		Namespace: "default",
 		Name:      "test",
 	}, "project:region:db")
 	p.Generation = 2
-	p.Finalizers = []string{finalizerName}
-	p.Spec.Workload = v1alpha1.WorkloadSelectorSpec{
-		Kind: "Deployment",
-		Selector: &metav1.LabelSelector{
-			MatchLabels: map[string]string{"app": "things"},
-		},
-	}
-	p.Status.Conditions = []*metav1.Condition{{
-		Type:   v1alpha1.ConditionUpToDate,
-		Reason: v1alpha1.ReasonStartedReconcile,
-		Status: metav1.ConditionFalse,
-	}}
+	addFinalizers(p)
+	addSelectorWorkload(p, "Deployment", labelK, labelV)
 
 	// mimic a pod that was updated by the webhook
 	reqName := v1alpha1.AnnotationPrefix + "/" + p.Name
@@ -175,7 +158,7 @@ func TestReconcileState32(t *testing.T) {
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "thing",
 			Namespace: "default",
-			Labels:    map[string]string{"app": "things"},
+			Labels:    map[string]string{labelK: labelV},
 		},
 		Spec: appsv1.DeploymentSpec{Template: corev1.PodTemplateSpec{
 			ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{reqName: "1"}},
@@ -190,27 +173,21 @@ func TestReconcileState32(t *testing.T) {
 }
 
 func TestReconcileState33(t *testing.T) {
-	wantRequeue := false
-	wantStatus := metav1.ConditionTrue
-	wantReason := v1alpha1.ReasonFinishedReconcile
+	const (
+		wantRequeue = false
+		wantStatus  = metav1.ConditionTrue
+		wantReason  = v1alpha1.ReasonFinishedReconcile
+		labelK      = "app"
+		labelV      = "things"
+	)
 
 	p := testhelpers.BuildAuthProxyWorkload(types.NamespacedName{
 		Namespace: "default",
 		Name:      "test",
 	}, "project:region:db")
 	p.Generation = 1
-	p.Finalizers = []string{finalizerName}
-	p.Spec.Workload = v1alpha1.WorkloadSelectorSpec{
-		Kind: "Deployment",
-		Selector: &metav1.LabelSelector{
-			MatchLabels: map[string]string{"app": "things"},
-		},
-	}
-	p.Status.Conditions = []*metav1.Condition{{
-		Type:   v1alpha1.ConditionUpToDate,
-		Reason: v1alpha1.ReasonStartedReconcile,
-		Status: metav1.ConditionFalse,
-	}}
+	addFinalizers(p)
+	addSelectorWorkload(p, "Deployment", labelK, labelV)
 
 	// mimic a pod that was updated by the webhook
 	reqName := v1alpha1.AnnotationPrefix + "/" + p.Name
@@ -218,7 +195,7 @@ func TestReconcileState33(t *testing.T) {
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "thing",
 			Namespace: "default",
-			Labels:    map[string]string{"app": "things"},
+			Labels:    map[string]string{labelK: labelV},
 		},
 		Spec: appsv1.DeploymentSpec{Template: corev1.PodTemplateSpec{
 			ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{reqName: "1"}},
@@ -232,6 +209,86 @@ func TestReconcileState33(t *testing.T) {
 
 }
 
+func TestReconcileDeleteUpdatesWorkload(t *testing.T) {
+	const (
+		labelK = "app"
+		labelV = "things"
+	)
+	resource := testhelpers.BuildAuthProxyWorkload(types.NamespacedName{
+		Namespace: "default",
+		Name:      "test",
+	}, "project:region:db")
+	resource.Generation = 1
+	addFinalizers(resource)
+	addSelectorWorkload(resource, "Deployment", labelK, labelV)
+
+	k, v := workload.PodAnnotation(resource)
+
+	// mimic a deployment that was updated by the webhook
+	deployment := &appsv1.Deployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "thing",
+			Namespace: "default",
+			Labels:    map[string]string{labelK: labelV},
+		},
+		Spec: appsv1.DeploymentSpec{Template: corev1.PodTemplateSpec{
+			ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{k: v}},
+		}},
+	}
+
+	// Build a client with the resource and deployment
+	cb, err := clientBuilder()
+	if err != nil {
+		t.Error(err) // shouldn't ever happen
+	}
+	c := cb.WithObjects(resource, deployment).Build()
+	r, req, ctx := reconciler(resource, c)
+
+	// Delete the resource
+	c.Delete(ctx, resource)
+	if err != nil {
+		t.Error(err)
+	}
+
+	// Run Reconcile on the deleted resource
+	res, err := r.Reconcile(ctx, req)
+	if err != nil {
+		t.Error(err)
+	}
+	if res.Requeue {
+		t.Errorf("got %v, want %v for requeue", res.Requeue, false)
+	}
+
+	// Check that the resource doesn't exist anymore
+	err = c.Get(ctx, types.NamespacedName{
+		Namespace: resource.GetNamespace(),
+		Name:      resource.GetName(),
+	}, resource)
+	if err != nil {
+		if !errors.IsNotFound(err) {
+			t.Errorf("wants not found error, got %v", err)
+		}
+	} else {
+		t.Error("wants not found error, got no error")
+	}
+
+	// Fetch the deployment and make sure the annotations show the
+	// deleted resource.
+	d := &appsv1.Deployment{}
+	err = c.Get(ctx, types.NamespacedName{
+		Namespace: deployment.GetNamespace(),
+		Name:      deployment.GetName(),
+	}, d)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if got, want := d.Spec.Template.ObjectMeta.Annotations[k], "1-deleted-"; !strings.HasPrefix(got, want) {
+		t.Fatalf("got %v, wants annotation value to have prefix %v", got, want)
+	}
+
+}
+
 func runReconcileTestcase(p *v1alpha1.AuthProxyWorkload, clientObjects []client.Object, wantRequeue bool, wantStatus metav1.ConditionStatus, wantReason string) error {
 	cb, err := clientBuilder()
 	if err != nil {
@@ -304,3 +361,21 @@ func reconciler(p *v1alpha1.AuthProxyWorkload, cb client.Client) (*AuthProxyWork
 	}
 	return r, req, ctx
 }
+
+func addFinalizers(p *v1alpha1.AuthProxyWorkload) {
+	p.Finalizers = []string{finalizerName}
+}
+func addPodWorkload(p *v1alpha1.AuthProxyWorkload) {
+	p.Spec.Workload = v1alpha1.WorkloadSelectorSpec{
+		Kind: "Pod",
+		Name: "testpod",
+	}
+}
+func addSelectorWorkload(p *v1alpha1.AuthProxyWorkload, kind, labelK, labelV string) {
+	p.Spec.Workload = v1alpha1.WorkloadSelectorSpec{
+		Kind: kind,
+		Selector: &metav1.LabelSelector{
+			MatchLabels: map[string]string{labelK: labelV},
+		},
+	}
+}
diff --git a/internal/testhelpers/resources.go b/internal/testhelpers/resources.go
index 7b8687a..d976f25 100644
--- a/internal/testhelpers/resources.go
+++ b/internal/testhelpers/resources.go
@@ -595,7 +595,7 @@ func BuildAuthProxyWorkload(key types.NamespacedName, connectionString string) *
 }
 
 // CreateAuthProxyWorkload creates an AuthProxyWorkload in the kubernetes cluster.
-func (cc *TestCaseClient) CreateAuthProxyWorkload(ctx context.Context, key types.NamespacedName, appLabel string, connectionString string, kind string) error {
+func (cc *TestCaseClient) CreateAuthProxyWorkload(ctx context.Context, key types.NamespacedName, appLabel string, connectionString string, kind string) (*v1alpha1.AuthProxyWorkload, error) {
 	proxy := BuildAuthProxyWorkload(key, connectionString)
 	proxy.Spec.Workload = v1alpha1.WorkloadSelectorSpec{
 		Kind: kind,
@@ -613,9 +613,9 @@ func (cc *TestCaseClient) CreateAuthProxyWorkload(ctx context.Context, key types
 	}
 	err := cc.Client.Create(ctx, proxy)
 	if err != nil {
-		return fmt.Errorf("Unable to create entity %v", err)
+		return nil, fmt.Errorf("Unable to create entity %v", err)
 	}
-	return nil
+	return proxy, nil
 }
 
 // GetConditionStatus finds a condition where Condition.Type == condType and returns
diff --git a/internal/testhelpers/testcases.go b/internal/testhelpers/testcases.go
index d27784c..10317ad 100644
--- a/internal/testhelpers/testcases.go
+++ b/internal/testhelpers/testcases.go
@@ -53,7 +53,7 @@ func (cc *TestCaseClient) CreateResource(ctx context.Context) (*cloudsqlapi.Auth
 		return nil, fmt.Errorf("can't create namespace, %v", err)
 	}
 	key := types.NamespacedName{Name: name, Namespace: ns}
-	err = cc.CreateAuthProxyWorkload(ctx, key, "app", expectedConnStr, "Deployment")
+	_, err = cc.CreateAuthProxyWorkload(ctx, key, "app", expectedConnStr, "Deployment")
 	if err != nil {
 		return nil, fmt.Errorf("unable to create auth proxy workload %v", err)
 	}
diff --git a/internal/testintegration/integration_test.go b/internal/testintegration/integration_test.go
index 2c1a5c7..8181c92 100644
--- a/internal/testintegration/integration_test.go
+++ b/internal/testintegration/integration_test.go
@@ -85,7 +85,7 @@ func TestModifiesNewDeployment(t *testing.T) {
 	key := types.NamespacedName{Name: pwlName, Namespace: tcc.Namespace}
 
 	t.Log("Creating AuthProxyWorkload")
-	err = tcc.CreateAuthProxyWorkload(ctx, key, deploymentAppLabel, tcc.ConnectionString, "Deployment")
+	_, err = tcc.CreateAuthProxyWorkload(ctx, key, deploymentAppLabel, tcc.ConnectionString, "Deployment")
 	if err != nil {
 		t.Error(err)
 		return
@@ -158,7 +158,7 @@ func TestModifiesExistingDeployment(t *testing.T) {
 	}
 
 	t.Log("Creating cloud sql instance")
-	err = tcc.CreateAuthProxyWorkload(ctx, pKey, deploymentAppLabel, tcc.ConnectionString, "Deployment")
+	_, err = tcc.CreateAuthProxyWorkload(ctx, pKey, deploymentAppLabel, tcc.ConnectionString, "Deployment")
 	if err != nil {
 		t.Fatal(err)
 	}
diff --git a/internal/workload/podspec_updates_test.go b/internal/workload/podspec_updates_test.go
index 763fb81..37d0ac8 100644
--- a/internal/workload/podspec_updates_test.go
+++ b/internal/workload/podspec_updates_test.go
@@ -19,6 +19,7 @@ import (
 	"reflect"
 	"strconv"
 	"testing"
+	"time"
 
 	"github.com/GoogleCloudPlatform/cloud-sql-proxy-operator/internal/api/v1alpha1"
 	"github.com/GoogleCloudPlatform/cloud-sql-proxy-operator/internal/workload"
@@ -777,7 +778,10 @@ func assertContainerArgsContains(t *testing.T, gotArgs, wantArgs []string) {
 }
 
 func TestPodTemplateAnnotations(t *testing.T) {
+
 	var (
+		now = metav1.Now()
+
 		wantAnnotations = map[string]string{
 			"cloudsql.cloud.google.com/instance1": "1",
 			"cloudsql.cloud.google.com/instance2": "2",
@@ -794,9 +798,13 @@ func TestPodTemplateAnnotations(t *testing.T) {
 	// Create a AuthProxyWorkload that matches the deployment
 	csqls := []*v1alpha1.AuthProxyWorkload{
 		simpleAuthProxy("instance1", "project:server:db"),
-		simpleAuthProxy("instance2", "project:server2:db2")}
+		simpleAuthProxy("instance2", "project:server2:db2"),
+		simpleAuthProxy("instance3", "project:server3:db3")}
+
 	csqls[0].ObjectMeta.Generation = 1
 	csqls[1].ObjectMeta.Generation = 2
+	csqls[2].ObjectMeta.Generation = 3
+	csqls[2].ObjectMeta.DeletionTimestamp = &now
 
 	// update the containers
 	err := configureProxies(u, wl, csqls)
@@ -810,3 +818,38 @@ func TestPodTemplateAnnotations(t *testing.T) {
 	}
 
 }
+
+func TestPodAnnotation(t *testing.T) {
+	now := metav1.Now()
+	server := &v1alpha1.AuthProxyWorkload{ObjectMeta: metav1.ObjectMeta{Name: "instance1", Generation: 1}}
+	deletedServer := &v1alpha1.AuthProxyWorkload{ObjectMeta: metav1.ObjectMeta{Name: "instance2", Generation: 2, DeletionTimestamp: &now}}
+
+	var testcases = []struct {
+		name  string
+		r     *v1alpha1.AuthProxyWorkload
+		wantK string
+		wantV string
+	}{
+		{
+			name:  "instance1",
+			r:     server,
+			wantK: "cloudsql.cloud.google.com/instance1",
+			wantV: "1",
+		}, {
+			name:  "instance2",
+			r:     deletedServer,
+			wantK: "cloudsql.cloud.google.com/instance2",
+			wantV: fmt.Sprintf("2-deleted-%s", now.Format(time.RFC3339)),
+		},
+	}
+
+	for _, tc := range testcases {
+		gotK, gotV := workload.PodAnnotation(tc.r)
+		if tc.wantK != gotK {
+			t.Errorf("got %v, want %v for key", gotK, tc.wantK)
+		}
+		if tc.wantV != gotV {
+			t.Errorf("got %v, want %v for value", gotV, tc.wantV)
+		}
+	}
+}
diff --git a/tests/e2e_test.go b/tests/e2e_test.go
index aa7eacd..57f0d54 100644
--- a/tests/e2e_test.go
+++ b/tests/e2e_test.go
@@ -128,7 +128,7 @@ func TestProxyAppliedOnNewWorkload(t *testing.T) {
 			key := types.NamespacedName{Name: pwlName, Namespace: tp.Namespace}
 
 			t.Log("Creating AuthProxyWorkload")
-			err = tp.CreateAuthProxyWorkload(ctx, key, appLabel, tp.ConnectionString, kind)
+			_, err = tp.CreateAuthProxyWorkload(ctx, key, appLabel, tp.ConnectionString, kind)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -243,7 +243,7 @@ func TestProxyAppliedOnExistingWorkload(t *testing.T) {
 			}
 
 			t.Log("Creating AuthProxyWorkload")
-			err = tp.CreateAuthProxyWorkload(ctx, key, appLabel, tp.ConnectionString, kind)
+			_, err = tp.CreateAuthProxyWorkload(ctx, key, appLabel, tp.ConnectionString, kind)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -335,7 +335,7 @@ func TestPublicDBConnections(t *testing.T) {
 			wl.Deployment.Spec.Template = test.podTemplate
 			t.Log("Creating AuthProxyWorkload")
 
-			err = tp.CreateAuthProxyWorkload(ctx, key, appLabel, tp.ConnectionString, kind)
+			_, err = tp.CreateAuthProxyWorkload(ctx, key, appLabel, tp.ConnectionString, kind)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -373,3 +373,88 @@ func TestPublicDBConnections(t *testing.T) {
 	}
 
 }
+
+func TestUpdateWorkloadOnDelete(t *testing.T) {
+
+	const (
+		pwlName  = "newss"
+		appLabel = "busybox"
+		name     = "app"
+		allOrAny = "all"
+	)
+	// Use a deployment workload
+	wl := &workload.DeploymentWorkload{Deployment: testhelpers.BuildDeployment(types.NamespacedName{}, "busybox")}
+	o := wl.Object()
+	kind := o.GetObjectKind().GroupVersionKind().Kind
+
+	// Set up the e2e test namespace
+	skipCleanup := loadValue("SKIP_CLEANUP", "", "false") == "true"
+	ctx := testContext()
+	tp := newPublicPostgresClient("new" + strings.ToLower(kind))
+
+	err := tp.CreateOrPatchNamespace(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Cleanup(func() {
+		if skipCleanup {
+			return
+		}
+
+		err = tp.DeleteNamespace(ctx)
+		if err != nil {
+			t.Fatal(err)
+		}
+	})
+
+	// Create AuthProxyWorkload
+	key := types.NamespacedName{Name: pwlName, Namespace: tp.Namespace}
+
+	t.Log("Creating AuthProxyWorkload")
+	proxy, err := tp.CreateAuthProxyWorkload(ctx, key, appLabel, tp.ConnectionString, kind)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Log("Waiting for AuthProxyWorkload operator to begin the reconcile loop")
+	_, err = tp.GetAuthProxyWorkloadAfterReconcile(ctx, key)
+	if err != nil {
+		t.Fatal("unable to create AuthProxyWorkload", err)
+	}
+
+	// Create deployment
+	t.Log("Creating ", kind)
+	o.SetNamespace(tp.Namespace)
+	o.SetName(name)
+	err = tp.CreateWorkload(ctx, o)
+	if err != nil {
+		t.Fatal("unable to create ", kind, err)
+	}
+	selector := &metav1.LabelSelector{
+		MatchLabels: map[string]string{"app": appLabel},
+	}
+
+	// Check that the deployment pods are configured with the proxy: pods
+	// have 2 containers.
+	t.Log("Checking for container counts", kind)
+	err = tp.ExpectPodContainerCount(ctx, selector, 2, allOrAny)
+	if err != nil {
+		t.Error(err)
+	}
+	t.Log("Workload Created. Removing AuthProxyWorkload", kind)
+
+	// Delete the AuthProxyWorkload
+	err = tp.Client.Delete(ctx, proxy)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Check that deployment pods are configured without the proxy: pods have
+	// 1 container.
+	t.Log("Checking for container counts after delete", kind)
+	err = tp.ExpectPodContainerCount(ctx, selector, 1, allOrAny)
+	if err != nil {
+		t.Error(err)
+	}
+}