Repository: knative/pkg
Pull request #1431 (2020-06-20 00:05:58)

Fix gcs ListChildrenFiles and ListDirectChildren
Previous right trim and "/" addition prevented the discovery of
top-level directories under the bucket.

/cc @chizhg @chaodaiG 

knative-prow-robot: [APPROVALNOTIFIER] This PR is **NOT APPROVED**

This pull-request has been approved by: *<a href="https://github.com/knative/pkg/pull/1431#" title="Author self-approved">peterfeifanchen</a>*
To complete the [pull request process](https://git.k8s.io/community/contributors/guide/owners.md#the-code-review-process), please assign **chizhg**
You can assign the PR to them by writing `/assign @chizhg` in a comment when ready.

The full list of commands accepted by this bot can be found [here](https://go.k8s.io/bot-commands?repo=knative%2Fpkg).

<details open>
Needs approval from an approver in each of these files:

- **[test/OWNERS](https://github.com/knative/pkg/blob/master/test/OWNERS)**

Approvers can indicate their approval by writing `/approve` in a comment
Approvers can cancel approval by writing `/approve cancel` in a comment
</details>
<!-- META={"approvers":["chizhg"]} -->

Patch:
diff --git a/OWNERS_ALIASES b/OWNERS_ALIASES
index 35fc90205..98bd1203f 100644
--- a/OWNERS_ALIASES
+++ b/OWNERS_ALIASES
@@ -8,14 +8,20 @@ aliases:
   - mattmoor
   - vaikas
   - n3wscott
+  apis-reviewers:
+  - whaught
 
   apis-duck-approvers:
   - mattmoor
   - vaikas
+  apis-duck-reviewers:
+  - whaught
 
   codegen-approvers:
   - mattmoor
   - n3wscott
+  codegen-reviewers:
+  - whaught
 
   configmap-approvers:
   - mattmoor
@@ -28,6 +34,8 @@ aliases:
   - mattmoor
   - tcnghia
   - vagababov
+  controller-reviewers:
+  - whaught
 
   kmeta-approvers:
   - mattmoor
@@ -65,4 +73,5 @@ aliases:
   - dprotaso
   - mattmoor
   - tcnghia
-
+  webhook-reviewers:
+  - whaught
diff --git a/apis/OWNERS b/apis/OWNERS
index a25420ebc..1227b2b9e 100644
--- a/apis/OWNERS
+++ b/apis/OWNERS
@@ -2,3 +2,6 @@
 
 approvers:
 - apis-approvers
+
+reviewers:
+- apis-reviewers
diff --git a/apis/duck/OWNERS b/apis/duck/OWNERS
index ad4d83c51..8df611ba0 100644
--- a/apis/duck/OWNERS
+++ b/apis/duck/OWNERS
@@ -2,3 +2,6 @@
 
 approvers:
 - apis-duck-approvers
+
+reviewers:
+- apis-duck-reviewers
diff --git a/apis/duck/register.go b/apis/duck/register.go
index d10adc21c..d84cd49d1 100644
--- a/apis/duck/register.go
+++ b/apis/duck/register.go
@@ -18,4 +18,12 @@ package duck
 
 const (
 	GroupName = "duck.knative.dev"
+
+	// AddressableDuckVersionLabel is the label we use to declare
+	// that a type conforms to the Addressable duck type.
+	AddressableDuckVersionLabel = "duck.knative.dev/addressable"
+
+	// SourceDuckVersionLabel is the label we use to declare
+	// that a type conforms to the Source duck type.
+	SourceDuckVersionLabel = "duck.knative.dev/source"
 )
diff --git a/client/injection/apiextensions/reconciler/apiextensions/v1/customresourcedefinition/controller.go b/client/injection/apiextensions/reconciler/apiextensions/v1/customresourcedefinition/controller.go
index 1130d910d..315985b6b 100644
--- a/client/injection/apiextensions/reconciler/apiextensions/v1/customresourcedefinition/controller.go
+++ b/client/injection/apiextensions/reconciler/apiextensions/v1/customresourcedefinition/controller.go
@@ -26,6 +26,8 @@ import (
 
 	corev1 "k8s.io/api/core/v1"
 	clientsetscheme "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/scheme"
+	labels "k8s.io/apimachinery/pkg/labels"
+	types "k8s.io/apimachinery/pkg/types"
 	watch "k8s.io/apimachinery/pkg/watch"
 	scheme "k8s.io/client-go/kubernetes/scheme"
 	v1 "k8s.io/client-go/kubernetes/typed/core/v1"
@@ -35,6 +37,7 @@ import (
 	kubeclient "knative.dev/pkg/client/injection/kube/client"
 	controller "knative.dev/pkg/controller"
 	logging "knative.dev/pkg/logging"
+	reconciler "knative.dev/pkg/reconciler"
 )
 
 const (
@@ -56,9 +59,27 @@ func NewImpl(ctx context.Context, r Interface, optionsFns ...controller.OptionsF
 
 	customresourcedefinitionInformer := customresourcedefinition.Get(ctx)
 
+	lister := customresourcedefinitionInformer.Lister()
+
 	rec := &reconcilerImpl{
+		LeaderAwareFuncs: reconciler.LeaderAwareFuncs{
+			PromoteFunc: func(bkt reconciler.Bucket, enq func(reconciler.Bucket, types.NamespacedName)) error {
+				all, err := lister.List(labels.Everything())
+				if err != nil {
+					return err
+				}
+				for _, elt := range all {
+					// TODO: Consider letting users specify a filter in options.
+					enq(bkt, types.NamespacedName{
+						Namespace: elt.GetNamespace(),
+						Name:      elt.GetName(),
+					})
+				}
+				return nil
+			},
+		},
 		Client:        client.Get(ctx),
-		Lister:        customresourcedefinitionInformer.Lister(),
+		Lister:        lister,
 		reconciler:    r,
 		finalizerName: defaultFinalizerName,
 	}
diff --git a/client/injection/apiextensions/reconciler/apiextensions/v1/customresourcedefinition/reconciler.go b/client/injection/apiextensions/reconciler/apiextensions/v1/customresourcedefinition/reconciler.go
index 4f858c9c2..e0cd0fc1e 100644
--- a/client/injection/apiextensions/reconciler/apiextensions/v1/customresourcedefinition/reconciler.go
+++ b/client/injection/apiextensions/reconciler/apiextensions/v1/customresourcedefinition/reconciler.go
@@ -21,6 +21,7 @@ package customresourcedefinition
 import (
 	context "context"
 	json "encoding/json"
+	fmt "fmt"
 	reflect "reflect"
 
 	zap "go.uber.org/zap"
@@ -31,6 +32,7 @@ import (
 	equality "k8s.io/apimachinery/pkg/api/equality"
 	errors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	labels "k8s.io/apimachinery/pkg/labels"
 	types "k8s.io/apimachinery/pkg/types"
 	sets "k8s.io/apimachinery/pkg/util/sets"
 	cache "k8s.io/client-go/tools/cache"
@@ -63,8 +65,30 @@ type Finalizer interface {
 	FinalizeKind(ctx context.Context, o *v1.CustomResourceDefinition) reconciler.Event
 }
 
+// ReadOnlyInterface defines the strongly typed interfaces to be implemented by a
+// controller reconciling v1.CustomResourceDefinition if they want to process resources for which
+// they are not the leader.
+type ReadOnlyInterface interface {
+	// ObserveKind implements logic to observe v1.CustomResourceDefinition.
+	// This method should not write to the API.
+	ObserveKind(ctx context.Context, o *v1.CustomResourceDefinition) reconciler.Event
+}
+
+// ReadOnlyFinalizer defines the strongly typed interfaces to be implemented by a
+// controller finalizing v1.CustomResourceDefinition if they want to process tombstoned resources
+// even when they are not the leader.  Due to the nature of how finalizers are handled
+// there are no guarantees that this will be called.
+type ReadOnlyFinalizer interface {
+	// ObserveFinalizeKind implements custom logic to observe the final state of v1.CustomResourceDefinition.
+	// This method should not write to the API.
+	ObserveFinalizeKind(ctx context.Context, o *v1.CustomResourceDefinition) reconciler.Event
+}
+
 // reconcilerImpl implements controller.Reconciler for v1.CustomResourceDefinition resources.
 type reconcilerImpl struct {
+	// LeaderAwareFuncs is inlined to help us implement reconciler.LeaderAware
+	reconciler.LeaderAwareFuncs
+
 	// Client is used to write back status updates.
 	Client clientset.Interface
 
@@ -89,13 +113,39 @@ type reconcilerImpl struct {
 // Check that our Reconciler implements controller.Reconciler
 var _ controller.Reconciler = (*reconcilerImpl)(nil)
 
+// Check that our generated Reconciler is always LeaderAware.
+var _ reconciler.LeaderAware = (*reconcilerImpl)(nil)
+
 func NewReconciler(ctx context.Context, logger *zap.SugaredLogger, client clientset.Interface, lister apiextensionsv1.CustomResourceDefinitionLister, recorder record.EventRecorder, r Interface, options ...controller.Options) controller.Reconciler {
 	// Check the options function input. It should be 0 or 1.
 	if len(options) > 1 {
 		logger.Fatalf("up to one options struct is supported, found %d", len(options))
 	}
 
+	// Fail fast when users inadvertently implement the other LeaderAware interface.
+	// For the typed reconcilers, Promote shouldn't take any arguments.
+	if _, ok := r.(reconciler.LeaderAware); ok {
+		logger.Fatalf("%T implements the incorrect LeaderAware interface.  Promote() should not take an argument as genreconciler handles the enqueuing automatically.", r)
+	}
+	// TODO: Consider validating when folks implement ReadOnlyFinalizer, but not Finalizer.
+
 	rec := &reconcilerImpl{
+		LeaderAwareFuncs: reconciler.LeaderAwareFuncs{
+			PromoteFunc: func(bkt reconciler.Bucket, enq func(reconciler.Bucket, types.NamespacedName)) error {
+				all, err := lister.List(labels.Everything())
+				if err != nil {
+					return err
+				}
+				for _, elt := range all {
+					// TODO: Consider letting users specify a filter in options.
+					enq(bkt, types.NamespacedName{
+						Namespace: elt.GetNamespace(),
+						Name:      elt.GetName(),
+					})
+				}
+				return nil
+			},
+		},
 		Client:        client,
 		Lister:        lister,
 		Recorder:      recorder,
@@ -119,6 +169,25 @@ func NewReconciler(ctx context.Context, logger *zap.SugaredLogger, client client
 func (r *reconcilerImpl) Reconcile(ctx context.Context, key string) error {
 	logger := logging.FromContext(ctx)
 
+	// Convert the namespace/name string into a distinct namespace and name
+	namespace, name, err := cache.SplitMetaNamespaceKey(key)
+	if err != nil {
+		logger.Errorf("invalid resource key: %s", key)
+		return nil
+	}
+	// Establish whether we are the leader for use below.
+	isLeader := r.IsLeaderFor(types.NamespacedName{
+		Namespace: namespace,
+		Name:      name,
+	})
+	roi, isROI := r.reconciler.(ReadOnlyInterface)
+	rof, isROF := r.reconciler.(ReadOnlyFinalizer)
+	if !isLeader && !isROI && !isROF {
+		// If we are not the leader, and we don't implement either ReadOnly
+		// interface, then take a fast-path out.
+		return nil
+	}
+
 	// If configStore is set, attach the frozen configuration to the context.
 	if r.configStore != nil {
 		ctx = r.configStore.ToContext(ctx)
@@ -127,15 +196,6 @@ func (r *reconcilerImpl) Reconcile(ctx context.Context, key string) error {
 	// Add the recorder to context.
 	ctx = controller.WithEventRecorder(ctx, r.Recorder)
 
-	// Convert the namespace/name string into a distinct namespace and name
-
-	_, name, err := cache.SplitMetaNamespaceKey(key)
-
-	if err != nil {
-		logger.Errorf("invalid resource key: %s", key)
-		return nil
-	}
-
 	// Get the resource with this namespace/name.
 
 	getter := r.Lister
@@ -155,20 +215,28 @@ func (r *reconcilerImpl) Reconcile(ctx context.Context, key string) error {
 
 	var reconcileEvent reconciler.Event
 	if resource.GetDeletionTimestamp().IsZero() {
-		// Append the target method to the logger.
-		logger = logger.With(zap.String("targetMethod", "ReconcileKind"))
+		if isLeader {
+			// Append the target method to the logger.
+			logger = logger.With(zap.String("targetMethod", "ReconcileKind"))
+
+			// Set and update the finalizer on resource if r.reconciler
+			// implements Finalizer.
+			if resource, err = r.setFinalizerIfFinalizer(ctx, resource); err != nil {
+				return fmt.Errorf("failed to set finalizers: %w", err)
+			}
 
-		// Set and update the finalizer on resource if r.reconciler
-		// implements Finalizer.
-		if resource, err = r.setFinalizerIfFinalizer(ctx, resource); err != nil {
-			logger.Warnw("Failed to set finalizers", zap.Error(err))
-		}
+			// Reconcile this copy of the resource and then write back any status
+			// updates regardless of whether the reconciliation errored out.
+			reconcileEvent = r.reconciler.ReconcileKind(ctx, resource)
 
-		// Reconcile this copy of the resource and then write back any status
-		// updates regardless of whether the reconciliation errored out.
-		reconcileEvent = r.reconciler.ReconcileKind(ctx, resource)
+		} else if isROI {
+			// Append the target method to the logger.
+			logger = logger.With(zap.String("targetMethod", "ObserveKind"))
 
-	} else if fin, ok := r.reconciler.(Finalizer); ok {
+			// Observe any changes to this resource, since we are not the leader.
+			reconcileEvent = roi.ObserveKind(ctx, resource)
+		}
+	} else if fin, ok := r.reconciler.(Finalizer); isLeader && ok {
 		// Append the target method to the logger.
 		logger = logger.With(zap.String("targetMethod", "FinalizeKind"))
 
@@ -176,8 +244,14 @@ func (r *reconcilerImpl) Reconcile(ctx context.Context, key string) error {
 		// and reconciled cleanly (nil or normal event), remove the finalizer.
 		reconcileEvent = fin.FinalizeKind(ctx, resource)
 		if resource, err = r.clearFinalizer(ctx, resource, reconcileEvent); err != nil {
-			logger.Warnw("Failed to clear finalizers", zap.Error(err))
+			return fmt.Errorf("failed to clear finalizers: %w", err)
 		}
+	} else if !isLeader && isROF {
+		// Append the target method to the logger.
+		logger = logger.With(zap.String("targetMethod", "ObserveFinalizeKind"))
+
+		// For finalizing reconcilers, just observe when we aren't the leader.
+		reconcileEvent = rof.ObserveFinalizeKind(ctx, resource)
 	}
 
 	// Synchronize the status.
@@ -186,6 +260,9 @@ func (r *reconcilerImpl) Reconcile(ctx context.Context, key string) error {
 		// This is important because the copy we loaded from the injectionInformer's
 		// cache may be stale and we don't want to overwrite a prior update
 		// to status with this stale state.
+	} else if !isLeader {
+		logger.Warn("Saw status changes when we aren't the leader!")
+		// TODO: Consider logging the diff at Debug?
 	} else if err = r.updateStatus(original, resource); err != nil {
 		logger.Warnw("Failed to update resource status", zap.Error(err))
 		r.Recorder.Eventf(resource, corev1.EventTypeWarning, "UpdateFailed",
diff --git a/client/injection/apiextensions/reconciler/apiextensions/v1/customresourcedefinition/stub/reconciler.go b/client/injection/apiextensions/reconciler/apiextensions/v1/customresourcedefinition/stub/reconciler.go
index 2f95a95a2..16ae428b8 100644
--- a/client/injection/apiextensions/reconciler/apiextensions/v1/customresourcedefinition/stub/reconciler.go
+++ b/client/injection/apiextensions/reconciler/apiextensions/v1/customresourcedefinition/stub/reconciler.go
@@ -46,6 +46,15 @@ var _ customresourcedefinition.Interface = (*Reconciler)(nil)
 // Optionally check that our Reconciler implements Finalizer
 //var _ customresourcedefinition.Finalizer = (*Reconciler)(nil)
 
+// Optionally check that our Reconciler implements ReadOnlyInterface
+// Implement this to observe resources even when we are not the leader.
+//var _ customresourcedefinition.ReadOnlyInterface = (*Reconciler)(nil)
+
+// Optionally check that our Reconciler implements ReadOnlyFinalizer
+// Implement this to observe tombstoned resources even when we are not
+// the leader (best effort).
+//var _ customresourcedefinition.ReadOnlyFinalizer = (*Reconciler)(nil)
+
 // ReconcileKind implements Interface.ReconcileKind.
 func (r *Reconciler) ReconcileKind(ctx context.Context, o *apiextensionsv1.CustomResourceDefinition) reconciler.Event {
 	// TODO: use this if the resource implements InitializeConditions.
@@ -64,3 +73,15 @@ func (r *Reconciler) ReconcileKind(ctx context.Context, o *apiextensionsv1.Custo
 //	// TODO: add custom finalization logic here.
 //	return nil
 //}
+
+// Optionally, use ObserveKind to observe the resource when we are not the leader.
+// func (r *Reconciler) ObserveKind(ctx context.Context, o *apiextensionsv1.CustomResourceDefinition) reconciler.Event {
+// 	// TODO: add custom observation logic here.
+// 	return nil
+// }
+
+// Optionally, use ObserveFinalizeKind to observe resources being finalized when we are no the leader.
+//func (r *Reconciler) ObserveFinalizeKind(ctx context.Context, o *apiextensionsv1.CustomResourceDefinition) reconciler.Event {
+// 	// TODO: add custom observation logic here.
+//	return nil
+//}
diff --git a/client/injection/apiextensions/reconciler/apiextensions/v1beta1/customresourcedefinition/controller.go b/client/injection/apiextensions/reconciler/apiextensions/v1beta1/customresourcedefinition/controller.go
index 4f20eeb03..548b8e160 100644
--- a/client/injection/apiextensions/reconciler/apiextensions/v1beta1/customresourcedefinition/controller.go
+++ b/client/injection/apiextensions/reconciler/apiextensions/v1beta1/customresourcedefinition/controller.go
@@ -26,6 +26,8 @@ import (
 
 	corev1 "k8s.io/api/core/v1"
 	clientsetscheme "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/scheme"
+	labels "k8s.io/apimachinery/pkg/labels"
+	types "k8s.io/apimachinery/pkg/types"
 	watch "k8s.io/apimachinery/pkg/watch"
 	scheme "k8s.io/client-go/kubernetes/scheme"
 	v1 "k8s.io/client-go/kubernetes/typed/core/v1"
@@ -35,6 +37,7 @@ import (
 	kubeclient "knative.dev/pkg/client/injection/kube/client"
 	controller "knative.dev/pkg/controller"
 	logging "knative.dev/pkg/logging"
+	reconciler "knative.dev/pkg/reconciler"
 )
 
 const (
@@ -56,9 +59,27 @@ func NewImpl(ctx context.Context, r Interface, optionsFns ...controller.OptionsF
 
 	customresourcedefinitionInformer := customresourcedefinition.Get(ctx)
 
+	lister := customresourcedefinitionInformer.Lister()
+
 	rec := &reconcilerImpl{
+		LeaderAwareFuncs: reconciler.LeaderAwareFuncs{
+			PromoteFunc: func(bkt reconciler.Bucket, enq func(reconciler.Bucket, types.NamespacedName)) error {
+				all, err := lister.List(labels.Everything())
+				if err != nil {
+					return err
+				}
+				for _, elt := range all {
+					// TODO: Consider letting users specify a filter in options.
+					enq(bkt, types.NamespacedName{
+						Namespace: elt.GetNamespace(),
+						Name:      elt.GetName(),
+					})
+				}
+				return nil
+			},
+		},
 		Client:        client.Get(ctx),
-		Lister:        customresourcedefinitionInformer.Lister(),
+		Lister:        lister,
 		reconciler:    r,
 		finalizerName: defaultFinalizerName,
 	}
diff --git a/client/injection/apiextensions/reconciler/apiextensions/v1beta1/customresourcedefinition/reconciler.go b/client/injection/apiextensions/reconciler/apiextensions/v1beta1/customresourcedefinition/reconciler.go
index 30d0fa9cb..46db0ef98 100644
--- a/client/injection/apiextensions/reconciler/apiextensions/v1beta1/customresourcedefinition/reconciler.go
+++ b/client/injection/apiextensions/reconciler/apiextensions/v1beta1/customresourcedefinition/reconciler.go
@@ -32,6 +32,7 @@ import (
 	equality "k8s.io/apimachinery/pkg/api/equality"
 	errors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	labels "k8s.io/apimachinery/pkg/labels"
 	types "k8s.io/apimachinery/pkg/types"
 	sets "k8s.io/apimachinery/pkg/util/sets"
 	cache "k8s.io/client-go/tools/cache"
@@ -64,8 +65,30 @@ type Finalizer interface {
 	FinalizeKind(ctx context.Context, o *v1beta1.CustomResourceDefinition) reconciler.Event
 }
 
+// ReadOnlyInterface defines the strongly typed interfaces to be implemented by a
+// controller reconciling v1beta1.CustomResourceDefinition if they want to process resources for which
+// they are not the leader.
+type ReadOnlyInterface interface {
+	// ObserveKind implements logic to observe v1beta1.CustomResourceDefinition.
+	// This method should not write to the API.
+	ObserveKind(ctx context.Context, o *v1beta1.CustomResourceDefinition) reconciler.Event
+}
+
+// ReadOnlyFinalizer defines the strongly typed interfaces to be implemented by a
+// controller finalizing v1beta1.CustomResourceDefinition if they want to process tombstoned resources
+// even when they are not the leader.  Due to the nature of how finalizers are handled
+// there are no guarantees that this will be called.
+type ReadOnlyFinalizer interface {
+	// ObserveFinalizeKind implements custom logic to observe the final state of v1beta1.CustomResourceDefinition.
+	// This method should not write to the API.
+	ObserveFinalizeKind(ctx context.Context, o *v1beta1.CustomResourceDefinition) reconciler.Event
+}
+
 // reconcilerImpl implements controller.Reconciler for v1beta1.CustomResourceDefinition resources.
 type reconcilerImpl struct {
+	// LeaderAwareFuncs is inlined to help us implement reconciler.LeaderAware
+	reconciler.LeaderAwareFuncs
+
 	// Client is used to write back status updates.
 	Client clientset.Interface
 
@@ -90,13 +113,39 @@ type reconcilerImpl struct {
 // Check that our Reconciler implements controller.Reconciler
 var _ controller.Reconciler = (*reconcilerImpl)(nil)
 
+// Check that our generated Reconciler is always LeaderAware.
+var _ reconciler.LeaderAware = (*reconcilerImpl)(nil)
+
 func NewReconciler(ctx context.Context, logger *zap.SugaredLogger, client clientset.Interface, lister apiextensionsv1beta1.CustomResourceDefinitionLister, recorder record.EventRecorder, r Interface, options ...controller.Options) controller.Reconciler {
 	// Check the options function input. It should be 0 or 1.
 	if len(options) > 1 {
 		logger.Fatalf("up to one options struct is supported, found %d", len(options))
 	}
 
+	// Fail fast when users inadvertently implement the other LeaderAware interface.
+	// For the typed reconcilers, Promote shouldn't take any arguments.
+	if _, ok := r.(reconciler.LeaderAware); ok {
+		logger.Fatalf("%T implements the incorrect LeaderAware interface.  Promote() should not take an argument as genreconciler handles the enqueuing automatically.", r)
+	}
+	// TODO: Consider validating when folks implement ReadOnlyFinalizer, but not Finalizer.
+
 	rec := &reconcilerImpl{
+		LeaderAwareFuncs: reconciler.LeaderAwareFuncs{
+			PromoteFunc: func(bkt reconciler.Bucket, enq func(reconciler.Bucket, types.NamespacedName)) error {
+				all, err := lister.List(labels.Everything())
+				if err != nil {
+					return err
+				}
+				for _, elt := range all {
+					// TODO: Consider letting users specify a filter in options.
+					enq(bkt, types.NamespacedName{
+						Namespace: elt.GetNamespace(),
+						Name:      elt.GetName(),
+					})
+				}
+				return nil
+			},
+		},
 		Client:        client,
 		Lister:        lister,
 		Recorder:      recorder,
@@ -120,6 +169,25 @@ func NewReconciler(ctx context.Context, logger *zap.SugaredLogger, client client
 func (r *reconcilerImpl) Reconcile(ctx context.Context, key string) error {
 	logger := logging.FromContext(ctx)
 
+	// Convert the namespace/name string into a distinct namespace and name
+	namespace, name, err := cache.SplitMetaNamespaceKey(key)
+	if err != nil {
+		logger.Errorf("invalid resource key: %s", key)
+		return nil
+	}
+	// Establish whether we are the leader for use below.
+	isLeader := r.IsLeaderFor(types.NamespacedName{
+		Namespace: namespace,
+		Name:      name,
+	})
+	roi, isROI := r.reconciler.(ReadOnlyInterface)
+	rof, isROF := r.reconciler.(ReadOnlyFinalizer)
+	if !isLeader && !isROI && !isROF {
+		// If we are not the leader, and we don't implement either ReadOnly
+		// interface, then take a fast-path out.
+		return nil
+	}
+
 	// If configStore is set, attach the frozen configuration to the context.
 	if r.configStore != nil {
 		ctx = r.configStore.ToContext(ctx)
@@ -128,15 +196,6 @@ func (r *reconcilerImpl) Reconcile(ctx context.Context, key string) error {
 	// Add the recorder to context.
 	ctx = controller.WithEventRecorder(ctx, r.Recorder)
 
-	// Convert the namespace/name string into a distinct namespace and name
-
-	_, name, err := cache.SplitMetaNamespaceKey(key)
-
-	if err != nil {
-		logger.Errorf("invalid resource key: %s", key)
-		return nil
-	}
-
 	// Get the resource with this namespace/name.
 
 	getter := r.Lister
@@ -156,20 +215,28 @@ func (r *reconcilerImpl) Reconcile(ctx context.Context, key string) error {
 
 	var reconcileEvent reconciler.Event
 	if resource.GetDeletionTimestamp().IsZero() {
-		// Append the target method to the logger.
-		logger = logger.With(zap.String("targetMethod", "ReconcileKind"))
+		if isLeader {
+			// Append the target method to the logger.
+			logger = logger.With(zap.String("targetMethod", "ReconcileKind"))
+
+			// Set and update the finalizer on resource if r.reconciler
+			// implements Finalizer.
+			if resource, err = r.setFinalizerIfFinalizer(ctx, resource); err != nil {
+				return fmt.Errorf("failed to set finalizers: %w", err)
+			}
 
-		// Set and update the finalizer on resource if r.reconciler
-		// implements Finalizer.
-		if resource, err = r.setFinalizerIfFinalizer(ctx, resource); err != nil {
-			return fmt.Errorf("failed to set finalizers: %w", err)
-		}
+			// Reconcile this copy of the resource and then write back any status
+			// updates regardless of whether the reconciliation errored out.
+			reconcileEvent = r.reconciler.ReconcileKind(ctx, resource)
 
-		// Reconcile this copy of the resource and then write back any status
-		// updates regardless of whether the reconciliation errored out.
-		reconcileEvent = r.reconciler.ReconcileKind(ctx, resource)
+		} else if isROI {
+			// Append the target method to the logger.
+			logger = logger.With(zap.String("targetMethod", "ObserveKind"))
 
-	} else if fin, ok := r.reconciler.(Finalizer); ok {
+			// Observe any changes to this resource, since we are not the leader.
+			reconcileEvent = roi.ObserveKind(ctx, resource)
+		}
+	} else if fin, ok := r.reconciler.(Finalizer); isLeader && ok {
 		// Append the target method to the logger.
 		logger = logger.With(zap.String("targetMethod", "FinalizeKind"))
 
@@ -179,6 +246,12 @@ func (r *reconcilerImpl) Reconcile(ctx context.Context, key string) error {
 		if resource, err = r.clearFinalizer(ctx, resource, reconcileEvent); err != nil {
 			return fmt.Errorf("failed to clear finalizers: %w", err)
 		}
+	} else if !isLeader && isROF {
+		// Append the target method to the logger.
+		logger = logger.With(zap.String("targetMethod", "ObserveFinalizeKind"))
+
+		// For finalizing reconcilers, just observe when we aren't the leader.
+		reconcileEvent = rof.ObserveFinalizeKind(ctx, resource)
 	}
 
 	// Synchronize the status.
@@ -187,6 +260,9 @@ func (r *reconcilerImpl) Reconcile(ctx context.Context, key string) error {
 		// This is important because the copy we loaded from the injectionInformer's
 		// cache may be stale and we don't want to overwrite a prior update
 		// to status with this stale state.
+	} else if !isLeader {
+		logger.Warn("Saw status changes when we aren't the leader!")
+		// TODO: Consider logging the diff at Debug?
 	} else if err = r.updateStatus(original, resource); err != nil {
 		logger.Warnw("Failed to update resource status", zap.Error(err))
 		r.Recorder.Eventf(resource, v1.EventTypeWarning, "UpdateFailed",
diff --git a/client/injection/apiextensions/reconciler/apiextensions/v1beta1/customresourcedefinition/stub/reconciler.go b/client/injection/apiextensions/reconciler/apiextensions/v1beta1/customresourcedefinition/stub/reconciler.go
index be5bd027c..fdd7f5a6a 100644
--- a/client/injection/apiextensions/reconciler/apiextensions/v1beta1/customresourcedefinition/stub/reconciler.go
+++ b/client/injection/apiextensions/reconciler/apiextensions/v1beta1/customresourcedefinition/stub/reconciler.go
@@ -46,6 +46,15 @@ var _ customresourcedefinition.Interface = (*Reconciler)(nil)
 // Optionally check that our Reconciler implements Finalizer
 //var _ customresourcedefinition.Finalizer = (*Reconciler)(nil)
 
+// Optionally check that our Reconciler implements ReadOnlyInterface
+// Implement this to observe resources even when we are not the leader.
+//var _ customresourcedefinition.ReadOnlyInterface = (*Reconciler)(nil)
+
+// Optionally check that our Reconciler implements ReadOnlyFinalizer
+// Implement this to observe tombstoned resources even when we are not
+// the leader (best effort).
+//var _ customresourcedefinition.ReadOnlyFinalizer = (*Reconciler)(nil)
+
 // ReconcileKind implements Interface.ReconcileKind.
 func (r *Reconciler) ReconcileKind(ctx context.Context, o *v1beta1.CustomResourceDefinition) reconciler.Event {
 	// TODO: use this if the resource implements InitializeConditions.
@@ -64,3 +73,15 @@ func (r *Reconciler) ReconcileKind(ctx context.Context, o *v1beta1.CustomResourc
 //	// TODO: add custom finalization logic here.
 //	return nil
 //}
+
+// Optionally, use ObserveKind to observe the resource when we are not the leader.
+// func (r *Reconciler) ObserveKind(ctx context.Context, o *v1beta1.CustomResourceDefinition) reconciler.Event {
+// 	// TODO: add custom observation logic here.
+// 	return nil
+// }
+
+// Optionally, use ObserveFinalizeKind to observe resources being finalized when we are no the leader.
+//func (r *Reconciler) ObserveFinalizeKind(ctx context.Context, o *v1beta1.CustomResourceDefinition) reconciler.Event {
+// 	// TODO: add custom observation logic here.
+//	return nil
+//}
diff --git a/client/injection/kube/reconciler/core/v1/namespace/controller.go b/client/injection/kube/reconciler/core/v1/namespace/controller.go
index 4cee36b43..a21da03bb 100644
--- a/client/injection/kube/reconciler/core/v1/namespace/controller.go
+++ b/client/injection/kube/reconciler/core/v1/namespace/controller.go
@@ -25,6 +25,8 @@ import (
 	strings "strings"
 
 	corev1 "k8s.io/api/core/v1"
+	labels "k8s.io/apimachinery/pkg/labels"
+	types "k8s.io/apimachinery/pkg/types"
 	watch "k8s.io/apimachinery/pkg/watch"
 	scheme "k8s.io/client-go/kubernetes/scheme"
 	v1 "k8s.io/client-go/kubernetes/typed/core/v1"
@@ -33,6 +35,7 @@ import (
 	namespace "knative.dev/pkg/client/injection/kube/informers/core/v1/namespace"
 	controller "knative.dev/pkg/controller"
 	logging "knative.dev/pkg/logging"
+	reconciler "knative.dev/pkg/reconciler"
 )
 
 const (
@@ -54,9 +57,27 @@ func NewImpl(ctx context.Context, r Interface, optionsFns ...controller.OptionsF
 
 	namespaceInformer := namespace.Get(ctx)
 
+	lister := namespaceInformer.Lister()
+
 	rec := &reconcilerImpl{
+		LeaderAwareFuncs: reconciler.LeaderAwareFuncs{
+			PromoteFunc: func(bkt reconciler.Bucket, enq func(reconciler.Bucket, types.NamespacedName)) error {
+				all, err := lister.List(labels.Everything())
+				if err != nil {
+					return err
+				}
+				for _, elt := range all {
+					// TODO: Consider letting users specify a filter in options.
+					enq(bkt, types.NamespacedName{
+						Namespace: elt.GetNamespace(),
+						Name:      elt.GetName(),
+					})
+				}
+				return nil
+			},
+		},
 		Client:        client.Get(ctx),
-		Lister:        namespaceInformer.Lister(),
+		Lister:        lister,
 		reconciler:    r,
 		finalizerName: defaultFinalizerName,
 	}
diff --git a/client/injection/kube/reconciler/core/v1/namespace/reconciler.go b/client/injection/kube/reconciler/core/v1/namespace/reconciler.go
index 00c761a1d..e7e27c017 100644
--- a/client/injection/kube/reconciler/core/v1/namespace/reconciler.go
+++ b/client/injection/kube/reconciler/core/v1/namespace/reconciler.go
@@ -29,6 +29,7 @@ import (
 	equality "k8s.io/apimachinery/pkg/api/equality"
 	errors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	labels "k8s.io/apimachinery/pkg/labels"
 	types "k8s.io/apimachinery/pkg/types"
 	sets "k8s.io/apimachinery/pkg/util/sets"
 	kubernetes "k8s.io/client-go/kubernetes"
@@ -63,8 +64,30 @@ type Finalizer interface {
 	FinalizeKind(ctx context.Context, o *v1.Namespace) reconciler.Event
 }
 
+// ReadOnlyInterface defines the strongly typed interfaces to be implemented by a
+// controller reconciling v1.Namespace if they want to process resources for which
+// they are not the leader.
+type ReadOnlyInterface interface {
+	// ObserveKind implements logic to observe v1.Namespace.
+	// This method should not write to the API.
+	ObserveKind(ctx context.Context, o *v1.Namespace) reconciler.Event
+}
+
+// ReadOnlyFinalizer defines the strongly typed interfaces to be implemented by a
+// controller finalizing v1.Namespace if they want to process tombstoned resources
+// even when they are not the leader.  Due to the nature of how finalizers are handled
+// there are no guarantees that this will be called.
+type ReadOnlyFinalizer interface {
+	// ObserveFinalizeKind implements custom logic to observe the final state of v1.Namespace.
+	// This method should not write to the API.
+	ObserveFinalizeKind(ctx context.Context, o *v1.Namespace) reconciler.Event
+}
+
 // reconcilerImpl implements controller.Reconciler for v1.Namespace resources.
 type reconcilerImpl struct {
+	// LeaderAwareFuncs is inlined to help us implement reconciler.LeaderAware
+	reconciler.LeaderAwareFuncs
+
 	// Client is used to write back status updates.
 	Client kubernetes.Interface
 
@@ -89,13 +112,39 @@ type reconcilerImpl struct {
 // Check that our Reconciler implements controller.Reconciler
 var _ controller.Reconciler = (*reconcilerImpl)(nil)
 
+// Check that our generated Reconciler is always LeaderAware.
+var _ reconciler.LeaderAware = (*reconcilerImpl)(nil)
+
 func NewReconciler(ctx context.Context, logger *zap.SugaredLogger, client kubernetes.Interface, lister corev1.NamespaceLister, recorder record.EventRecorder, r Interface, options ...controller.Options) controller.Reconciler {
 	// Check the options function input. It should be 0 or 1.
 	if len(options) > 1 {
 		logger.Fatalf("up to one options struct is supported, found %d", len(options))
 	}
 
+	// Fail fast when users inadvertently implement the other LeaderAware interface.
+	// For the typed reconcilers, Promote shouldn't take any arguments.
+	if _, ok := r.(reconciler.LeaderAware); ok {
+		logger.Fatalf("%T implements the incorrect LeaderAware interface.  Promote() should not take an argument as genreconciler handles the enqueuing automatically.", r)
+	}
+	// TODO: Consider validating when folks implement ReadOnlyFinalizer, but not Finalizer.
+
 	rec := &reconcilerImpl{
+		LeaderAwareFuncs: reconciler.LeaderAwareFuncs{
+			PromoteFunc: func(bkt reconciler.Bucket, enq func(reconciler.Bucket, types.NamespacedName)) error {
+				all, err := lister.List(labels.Everything())
+				if err != nil {
+					return err
+				}
+				for _, elt := range all {
+					// TODO: Consider letting users specify a filter in options.
+					enq(bkt, types.NamespacedName{
+						Namespace: elt.GetNamespace(),
+						Name:      elt.GetName(),
+					})
+				}
+				return nil
+			},
+		},
 		Client:        client,
 		Lister:        lister,
 		Recorder:      recorder,
@@ -119,6 +168,25 @@ func NewReconciler(ctx context.Context, logger *zap.SugaredLogger, client kubern
 func (r *reconcilerImpl) Reconcile(ctx context.Context, key string) error {
 	logger := logging.FromContext(ctx)
 
+	// Convert the namespace/name string into a distinct namespace and name
+	namespace, name, err := cache.SplitMetaNamespaceKey(key)
+	if err != nil {
+		logger.Errorf("invalid resource key: %s", key)
+		return nil
+	}
+	// Establish whether we are the leader for use below.
+	isLeader := r.IsLeaderFor(types.NamespacedName{
+		Namespace: namespace,
+		Name:      name,
+	})
+	roi, isROI := r.reconciler.(ReadOnlyInterface)
+	rof, isROF := r.reconciler.(ReadOnlyFinalizer)
+	if !isLeader && !isROI && !isROF {
+		// If we are not the leader, and we don't implement either ReadOnly
+		// interface, then take a fast-path out.
+		return nil
+	}
+
 	// If configStore is set, attach the frozen configuration to the context.
 	if r.configStore != nil {
 		ctx = r.configStore.ToContext(ctx)
@@ -127,15 +195,6 @@ func (r *reconcilerImpl) Reconcile(ctx context.Context, key string) error {
 	// Add the recorder to context.
 	ctx = controller.WithEventRecorder(ctx, r.Recorder)
 
-	// Convert the namespace/name string into a distinct namespace and name
-
-	_, name, err := cache.SplitMetaNamespaceKey(key)
-
-	if err != nil {
-		logger.Errorf("invalid resource key: %s", key)
-		return nil
-	}
-
 	// Get the resource with this namespace/name.
 
 	getter := r.Lister
@@ -155,20 +214,28 @@ func (r *reconcilerImpl) Reconcile(ctx context.Context, key string) error {
 
 	var reconcileEvent reconciler.Event
 	if resource.GetDeletionTimestamp().IsZero() {
-		// Append the target method to the logger.
-		logger = logger.With(zap.String("targetMethod", "ReconcileKind"))
+		if isLeader {
+			// Append the target method to the logger.
+			logger = logger.With(zap.String("targetMethod", "ReconcileKind"))
+
+			// Set and update the finalizer on resource if r.reconciler
+			// implements Finalizer.
+			if resource, err = r.setFinalizerIfFinalizer(ctx, resource); err != nil {
+				return fmt.Errorf("failed to set finalizers: %w", err)
+			}
 
-		// Set and update the finalizer on resource if r.reconciler
-		// implements Finalizer.
-		if resource, err = r.setFinalizerIfFinalizer(ctx, resource); err != nil {
-			return fmt.Errorf("failed to set finalizers: %w", err)
-		}
+			// Reconcile this copy of the resource and then write back any status
+			// updates regardless of whether the reconciliation errored out.
+			reconcileEvent = r.reconciler.ReconcileKind(ctx, resource)
 
-		// Reconcile this copy of the resource and then write back any status
-		// updates regardless of whether the reconciliation errored out.
-		reconcileEvent = r.reconciler.ReconcileKind(ctx, resource)
+		} else if isROI {
+			// Append the target method to the logger.
+			logger = logger.With(zap.String("targetMethod", "ObserveKind"))
 
-	} else if fin, ok := r.reconciler.(Finalizer); ok {
+			// Observe any changes to this resource, since we are not the leader.
+			reconcileEvent = roi.ObserveKind(ctx, resource)
+		}
+	} else if fin, ok := r.reconciler.(Finalizer); isLeader && ok {
 		// Append the target method to the logger.
 		logger = logger.With(zap.String("targetMethod", "FinalizeKind"))
 
@@ -178,6 +245,12 @@ func (r *reconcilerImpl) Reconcile(ctx context.Context, key string) error {
 		if resource, err = r.clearFinalizer(ctx, resource, reconcileEvent); err != nil {
 			return fmt.Errorf("failed to clear finalizers: %w", err)
 		}
+	} else if !isLeader && isROF {
+		// Append the target method to the logger.
+		logger = logger.With(zap.String("targetMethod", "ObserveFinalizeKind"))
+
+		// For finalizing reconcilers, just observe when we aren't the leader.
+		reconcileEvent = rof.ObserveFinalizeKind(ctx, resource)
 	}
 
 	// Synchronize the status.
@@ -186,6 +259,9 @@ func (r *reconcilerImpl) Reconcile(ctx context.Context, key string) error {
 		// This is important because the copy we loaded from the injectionInformer's
 		// cache may be stale and we don't want to overwrite a prior update
 		// to status with this stale state.
+	} else if !isLeader {
+		logger.Warn("Saw status changes when we aren't the leader!")
+		// TODO: Consider logging the diff at Debug?
 	} else if err = r.updateStatus(original, resource); err != nil {
 		logger.Warnw("Failed to update resource status", zap.Error(err))
 		r.Recorder.Eventf(resource, v1.EventTypeWarning, "UpdateFailed",
diff --git a/client/injection/kube/reconciler/core/v1/namespace/stub/reconciler.go b/client/injection/kube/reconciler/core/v1/namespace/stub/reconciler.go
index 21f788287..570fe1122 100644
--- a/client/injection/kube/reconciler/core/v1/namespace/stub/reconciler.go
+++ b/client/injection/kube/reconciler/core/v1/namespace/stub/reconciler.go
@@ -45,6 +45,15 @@ var _ namespace.Interface = (*Reconciler)(nil)
 // Optionally check that our Reconciler implements Finalizer
 //var _ namespace.Finalizer = (*Reconciler)(nil)
 
+// Optionally check that our Reconciler implements ReadOnlyInterface
+// Implement this to observe resources even when we are not the leader.
+//var _ namespace.ReadOnlyInterface = (*Reconciler)(nil)
+
+// Optionally check that our Reconciler implements ReadOnlyFinalizer
+// Implement this to observe tombstoned resources even when we are not
+// the leader (best effort).
+//var _ namespace.ReadOnlyFinalizer = (*Reconciler)(nil)
+
 // ReconcileKind implements Interface.ReconcileKind.
 func (r *Reconciler) ReconcileKind(ctx context.Context, o *v1.Namespace) reconciler.Event {
 	// TODO: use this if the resource implements InitializeConditions.
@@ -63,3 +72,15 @@ func (r *Reconciler) ReconcileKind(ctx context.Context, o *v1.Namespace) reconci
 //	// TODO: add custom finalization logic here.
 //	return nil
 //}
+
+// Optionally, use ObserveKind to observe the resource when we are not the leader.
+// func (r *Reconciler) ObserveKind(ctx context.Context, o *v1.Namespace) reconciler.Event {
+// 	// TODO: add custom observation logic here.
+// 	return nil
+// }
+
+// Optionally, use ObserveFinalizeKind to observe resources being finalized when we are no the leader.
+//func (r *Reconciler) ObserveFinalizeKind(ctx context.Context, o *v1.Namespace) reconciler.Event {
+// 	// TODO: add custom observation logic here.
+//	return nil
+//}
diff --git a/codegen/OWNERS b/codegen/OWNERS
index 18af7f343..9053435aa 100644
--- a/codegen/OWNERS
+++ b/codegen/OWNERS
@@ -2,3 +2,6 @@
 
 approvers:
 - codegen-approvers
+
+reviewers:
+- codegen-reviewers
diff --git a/codegen/cmd/injection-gen/generators/reconciler_controller.go b/codegen/cmd/injection-gen/generators/reconciler_controller.go
index bf3ca04fe..ac1c9fac3 100644
--- a/codegen/cmd/injection-gen/generators/reconciler_controller.go
+++ b/codegen/cmd/injection-gen/generators/reconciler_controller.go
@@ -142,9 +142,21 @@ func (g *reconcilerControllerGenerator) GenerateType(c *generator.Context, t *ty
 			Package: "context",
 			Name:    "Context",
 		}),
-		"fmtSprintf": c.Universe.Function(types.Name{
-			Package: "fmt",
-			Name:    "Sprintf",
+		"reconcilerLeaderAwareFuncs": c.Universe.Type(types.Name{
+			Package: "knative.dev/pkg/reconciler",
+			Name:    "LeaderAwareFuncs",
+		}),
+		"reconcilerBucket": c.Universe.Type(types.Name{
+			Package: "knative.dev/pkg/reconciler",
+			Name:    "Bucket",
+		}),
+		"typesNamespacedName": c.Universe.Type(types.Name{
+			Package: "k8s.io/apimachinery/pkg/types",
+			Name:    "NamespacedName",
+		}),
+		"labelsEverything": c.Universe.Function(types.Name{
+			Package: "k8s.io/apimachinery/pkg/labels",
+			Name:    "Everything",
 		}),
 		"stringsReplaceAll": c.Universe.Function(types.Name{
 			Package: "strings",
@@ -154,6 +166,10 @@ func (g *reconcilerControllerGenerator) GenerateType(c *generator.Context, t *ty
 			Package: "reflect",
 			Name:    "TypeOf",
 		}),
+		"fmtSprintf": c.Universe.Function(types.Name{
+			Package: "fmt",
+			Name:    "Sprintf",
+		}),
 	}
 
 	sw.Do(reconcilerControllerNewImpl, m)
@@ -185,9 +201,27 @@ func NewImpl(ctx {{.contextContext|raw}}, r Interface{{if .hasClass}}, classValu
 
 	{{.type|lowercaseSingular}}Informer := {{.informerGet|raw}}(ctx)
 
+	lister := {{.type|lowercaseSingular}}Informer.Lister()
+
 	rec := &reconcilerImpl{
+		LeaderAwareFuncs: {{.reconcilerLeaderAwareFuncs|raw}}{
+			PromoteFunc: func(bkt {{.reconcilerBucket|raw}}, enq func({{.reconcilerBucket|raw}}, {{.typesNamespacedName|raw}})) error {
+				all, err := lister.List({{.labelsEverything|raw}}())
+				if err != nil {
+					return err
+				}
+				for _, elt := range all {
+					// TODO: Consider letting users specify a filter in options.
+					enq(bkt, {{.typesNamespacedName|raw}}{
+						Namespace: elt.GetNamespace(),
+						Name: elt.GetName(),
+					})
+				}
+				return nil
+			},
+		},
 		Client:  {{.clientGet|raw}}(ctx),
-		Lister:  {{.type|lowercaseSingular}}Informer.Lister(),
+		Lister:  lister,
 		reconciler:    r,
 		finalizerName: defaultFinalizerName,
 		{{if .hasClass}}classValue: classValue,{{end}}
diff --git a/codegen/cmd/injection-gen/generators/reconciler_reconciler.go b/codegen/cmd/injection-gen/generators/reconciler_reconciler.go
index df4d595ab..b3d3596a1 100644
--- a/codegen/cmd/injection-gen/generators/reconciler_reconciler.go
+++ b/codegen/cmd/injection-gen/generators/reconciler_reconciler.go
@@ -152,6 +152,30 @@ func (g *reconcilerReconcilerGenerator) GenerateType(c *generator.Context, t *ty
 		"equalitySemantic":    c.Universe.Package("k8s.io/apimachinery/pkg/api/equality").Variable("Semantic"),
 		"jsonMarshal":         c.Universe.Package("encoding/json").Function("Marshal"),
 		"typesMergePatchType": c.Universe.Package("k8s.io/apimachinery/pkg/types").Constant("MergePatchType"),
+		"syncRWMutex": c.Universe.Type(types.Name{
+			Package: "sync",
+			Name:    "RWMutex",
+		}),
+		"reconcilerLeaderAware": c.Universe.Type(types.Name{
+			Package: "knative.dev/pkg/reconciler",
+			Name:    "LeaderAware",
+		}),
+		"reconcilerLeaderAwareFuncs": c.Universe.Type(types.Name{
+			Package: "knative.dev/pkg/reconciler",
+			Name:    "LeaderAwareFuncs",
+		}),
+		"reconcilerBucket": c.Universe.Type(types.Name{
+			Package: "knative.dev/pkg/reconciler",
+			Name:    "Bucket",
+		}),
+		"typesNamespacedName": c.Universe.Type(types.Name{
+			Package: "k8s.io/apimachinery/pkg/types",
+			Name:    "NamespacedName",
+		}),
+		"labelsEverything": c.Universe.Function(types.Name{
+			Package: "k8s.io/apimachinery/pkg/labels",
+			Name:    "Everything",
+		}),
 	}
 
 	sw.Do(reconcilerInterfaceFactory, m)
@@ -187,8 +211,30 @@ type Finalizer interface {
 	FinalizeKind(ctx {{.contextContext|raw}}, o *{{.type|raw}}) {{.reconcilerEvent|raw}}
 }
 
+// ReadOnlyInterface defines the strongly typed interfaces to be implemented by a
+// controller reconciling {{.type|raw}} if they want to process resources for which
+// they are not the leader.
+type ReadOnlyInterface interface {
+	// ObserveKind implements logic to observe {{.type|raw}}.
+	// This method should not write to the API.
+	ObserveKind(ctx {{.contextContext|raw}}, o *{{.type|raw}}) {{.reconcilerEvent|raw}}
+}
+
+// ReadOnlyFinalizer defines the strongly typed interfaces to be implemented by a
+// controller finalizing {{.type|raw}} if they want to process tombstoned resources
+// even when they are not the leader.  Due to the nature of how finalizers are handled
+// there are no guarantees that this will be called.
+type ReadOnlyFinalizer interface {
+	// ObserveFinalizeKind implements custom logic to observe the final state of {{.type|raw}}.
+	// This method should not write to the API.
+	ObserveFinalizeKind(ctx {{.contextContext|raw}}, o *{{.type|raw}}) {{.reconcilerEvent|raw}}
+}
+
 // reconcilerImpl implements controller.Reconciler for {{.type|raw}} resources.
 type reconcilerImpl struct {
+	// LeaderAwareFuncs is inlined to help us implement {{.reconcilerLeaderAware|raw}}
+	{{.reconcilerLeaderAwareFuncs|raw}}
+
 	// Client is used to write back status updates.
 	Client {{.clientsetInterface|raw}}
 
@@ -217,6 +263,8 @@ type reconcilerImpl struct {
 
 // Check that our Reconciler implements controller.Reconciler
 var _ controller.Reconciler = (*reconcilerImpl)(nil)
+// Check that our generated Reconciler is always LeaderAware.
+var _ {{.reconcilerLeaderAware|raw}}  = (*reconcilerImpl)(nil)
 
 `
 
@@ -227,7 +275,30 @@ func NewReconciler(ctx {{.contextContext|raw}}, logger *{{.zapSugaredLogger|raw}
 		logger.Fatalf("up to one options struct is supported, found %d", len(options))
 	}
 
+	// Fail fast when users inadvertently implement the other LeaderAware interface.
+	// For the typed reconcilers, Promote shouldn't take any arguments.
+	if _, ok := r.({{.reconcilerLeaderAware|raw}}); ok {
+		logger.Fatalf("%T implements the incorrect LeaderAware interface.  Promote() should not take an argument as genreconciler handles the enqueuing automatically.", r)
+	}
+	// TODO: Consider validating when folks implement ReadOnlyFinalizer, but not Finalizer.
+
 	rec := &reconcilerImpl{
+		LeaderAwareFuncs: {{.reconcilerLeaderAwareFuncs|raw}}{
+			PromoteFunc: func(bkt {{.reconcilerBucket|raw}}, enq func({{.reconcilerBucket|raw}}, {{.typesNamespacedName|raw}})) error {
+				all, err := lister.List({{.labelsEverything|raw}}())
+				if err != nil {
+					return err
+				}
+				for _, elt := range all {
+					// TODO: Consider letting users specify a filter in options.
+					enq(bkt, {{.typesNamespacedName|raw}}{
+						Namespace: elt.GetNamespace(),
+						Name: elt.GetName(),
+					})
+				}
+				return nil
+			},
+		},
 		Client: client,
 		Lister: lister,
 		Recorder: recorder,
@@ -254,6 +325,25 @@ var reconcilerImplFactory = `
 func (r *reconcilerImpl) Reconcile(ctx {{.contextContext|raw}}, key string) error {
 	logger := {{.loggingFromContext|raw}}(ctx)
 
+	// Convert the namespace/name string into a distinct namespace and name
+	namespace, name, err := {{.cacheSplitMetaNamespaceKey|raw}}(key)
+	if err != nil {
+		logger.Errorf("invalid resource key: %s", key)
+		return nil
+	}
+	// Establish whether we are the leader for use below.
+	isLeader := r.IsLeaderFor({{.typesNamespacedName|raw}}{
+		Namespace: namespace,
+		Name: name,
+	})
+	roi, isROI := r.reconciler.(ReadOnlyInterface)
+	rof, isROF := r.reconciler.(ReadOnlyFinalizer);
+	if !isLeader && !isROI && !isROF {
+		// If we are not the leader, and we don't implement either ReadOnly
+		// interface, then take a fast-path out.
+		return nil
+	}
+
 	// If configStore is set, attach the frozen configuration to the context.
 	if r.configStore != nil {
 		ctx = r.configStore.ToContext(ctx)
@@ -262,19 +352,7 @@ func (r *reconcilerImpl) Reconcile(ctx {{.contextContext|raw}}, key string) erro
 	// Add the recorder to context.
 	ctx = {{.controllerWithEventRecorder|raw}}(ctx, r.Recorder)
 
-	// Convert the namespace/name string into a distinct namespace and name
-	{{if .nonNamespaced}}
-	_, name, err := {{.cacheSplitMetaNamespaceKey|raw}}(key)
-	{{else}}
-	namespace, name, err := {{.cacheSplitMetaNamespaceKey|raw}}(key)
-	{{end}}
-	if err != nil {
-		logger.Errorf("invalid resource key: %s", key)
-		return nil
-	}
-
 	// Get the resource with this namespace/name.
-
 	{{if .nonNamespaced}}
 	getter := r.Lister
 	{{else}}
@@ -303,27 +381,34 @@ func (r *reconcilerImpl) Reconcile(ctx {{.contextContext|raw}}, key string) erro
 
 	var reconcileEvent {{.reconcilerEvent|raw}}
 	if resource.GetDeletionTimestamp().IsZero() {
-		// Append the target method to the logger.
-		logger = logger.With(zap.String("targetMethod", "ReconcileKind"))
-
-		// Set and update the finalizer on resource if r.reconciler
-		// implements Finalizer.
-		if resource, err = r.setFinalizerIfFinalizer(ctx, resource); err != nil {
-			return {{.fmtErrorf|raw}}("failed to set finalizers: %w", err)
-		}
+		if isLeader {
+			// Append the target method to the logger.
+			logger = logger.With(zap.String("targetMethod", "ReconcileKind"))
+
+			// Set and update the finalizer on resource if r.reconciler
+			// implements Finalizer.
+			if resource, err = r.setFinalizerIfFinalizer(ctx, resource); err != nil {
+				return {{.fmtErrorf|raw}}("failed to set finalizers: %w", err)
+			}
+			{{if .isKRShaped}}
+			reconciler.PreProcessReconcile(ctx, resource)
+			{{end}}
 
-		{{if .isKRShaped}}
-		reconciler.PreProcessReconcile(ctx, resource)
-		{{end}}
+			// Reconcile this copy of the resource and then write back any status
+			// updates regardless of whether the reconciliation errored out.
+			reconcileEvent = r.reconciler.ReconcileKind(ctx, resource)
 
-		// Reconcile this copy of the resource and then write back any status
-		// updates regardless of whether the reconciliation errored out.
-		reconcileEvent = r.reconciler.ReconcileKind(ctx, resource)
+			{{if .isKRShaped}}
+			reconciler.PostProcessReconcile(ctx, resource, original)
+			{{end}}
+		} else if isROI {
+			// Append the target method to the logger.
+			logger = logger.With(zap.String("targetMethod", "ObserveKind"))
 
-		{{if .isKRShaped}}
-		reconciler.PostProcessReconcile(ctx, resource, original)
-		{{end}}
-	} else if fin, ok := r.reconciler.(Finalizer); ok {
+			// Observe any changes to this resource, since we are not the leader.
+			reconcileEvent = roi.ObserveKind(ctx, resource)
+		}
+	} else if fin, ok := r.reconciler.(Finalizer); isLeader && ok {
 		// Append the target method to the logger.
 		logger = logger.With(zap.String("targetMethod", "FinalizeKind"))
 
@@ -333,6 +418,12 @@ func (r *reconcilerImpl) Reconcile(ctx {{.contextContext|raw}}, key string) erro
 		if resource, err = r.clearFinalizer(ctx, resource, reconcileEvent); err != nil {
 			return {{.fmtErrorf|raw}}("failed to clear finalizers: %w", err)
 		}
+	} else if !isLeader && isROF {
+		// Append the target method to the logger.
+		logger = logger.With(zap.String("targetMethod", "ObserveFinalizeKind"))
+
+		// For finalizing reconcilers, just observe when we aren't the leader.
+		reconcileEvent = rof.ObserveFinalizeKind(ctx, resource)
 	}
 
 	// Synchronize the status.
@@ -341,6 +432,9 @@ func (r *reconcilerImpl) Reconcile(ctx {{.contextContext|raw}}, key string) erro
 		// This is important because the copy we loaded from the injectionInformer's
 		// cache may be stale and we don't want to overwrite a prior update
 		// to status with this stale state.
+	} else if !isLeader {
+		logger.Warn("Saw status changes when we aren't the leader!")
+		// TODO: Consider logging the diff at Debug?
 	} else if err = r.updateStatus(original, resource); err != nil {
 		logger.Warnw("Failed to update resource status", zap.Error(err))
 		r.Recorder.Eventf(resource, {{.corev1EventTypeWarning|raw}}, "UpdateFailed",
@@ -520,4 +614,5 @@ func (r *reconcilerImpl) clearFinalizer(ctx {{.contextContext|raw}}, resource *{
 	// Synchronize the finalizers filtered by r.finalizerName.
 	return r.updateFinalizersFiltered(ctx, resource)
 }
+
 `
diff --git a/codegen/cmd/injection-gen/generators/reconciler_reconciler_stub.go b/codegen/cmd/injection-gen/generators/reconciler_reconciler_stub.go
index 3571ae2d1..62805e3d2 100644
--- a/codegen/cmd/injection-gen/generators/reconciler_reconciler_stub.go
+++ b/codegen/cmd/injection-gen/generators/reconciler_reconciler_stub.go
@@ -77,6 +77,14 @@ func (g *reconcilerReconcilerStubGenerator) GenerateType(c *generator.Context, t
 			Package: g.reconcilerPkg,
 			Name:    "Finalizer",
 		}),
+		"reconcilerReadOnlyInterface": c.Universe.Type(types.Name{
+			Package: g.reconcilerPkg,
+			Name:    "ReadOnlyInterface",
+		}),
+		"reconcilerReadOnlyFinalizer": c.Universe.Type(types.Name{
+			Package: g.reconcilerPkg,
+			Name:    "ReadOnlyFinalizer",
+		}),
 		"corev1EventTypeNormal": c.Universe.Type(types.Name{
 			Package: "k8s.io/api/core/v1",
 			Name:    "EventTypeNormal",
@@ -112,6 +120,14 @@ var _ {{.reconcilerInterface|raw}} = (*Reconciler)(nil)
 // Optionally check that our Reconciler implements Finalizer
 //var _ {{.reconcilerFinalizer|raw}} = (*Reconciler)(nil)
 
+// Optionally check that our Reconciler implements ReadOnlyInterface
+// Implement this to observe resources even when we are not the leader.
+//var _ {{.reconcilerReadOnlyInterface|raw}} = (*Reconciler)(nil)
+
+// Optionally check that our Reconciler implements ReadOnlyFinalizer
+// Implement this to observe tombstoned resources even when we are not
+// the leader (best effort).
+//var _ {{.reconcilerReadOnlyFinalizer|raw}} = (*Reconciler)(nil)
 
 // ReconcileKind implements Interface.ReconcileKind.
 func (r *Reconciler) ReconcileKind(ctx {{.contextContext|raw}}, o *{{.type|raw}}) {{.reconcilerEvent|raw}} {
@@ -133,4 +149,16 @@ func (r *Reconciler) ReconcileKind(ctx {{.contextContext|raw}}, o *{{.type|raw}}
 //	// TODO: add custom finalization logic here.
 //	return nil
 //}
+
+// Optionally, use ObserveKind to observe the resource when we are not the leader.
+// func (r *Reconciler) ObserveKind(ctx {{.contextContext|raw}}, o *{{.type|raw}}) {{.reconcilerEvent|raw}} {
+// 	// TODO: add custom observation logic here.
+// 	return nil
+// }
+
+// Optionally, use ObserveFinalizeKind to observe resources being finalized when we are no the leader.
+//func (r *Reconciler) ObserveFinalizeKind(ctx {{.contextContext|raw}}, o *{{.type|raw}}) {{.reconcilerEvent|raw}} {
+// 	// TODO: add custom observation logic here.
+//	return nil
+//}
 `
diff --git a/configmap/manual_watcher.go b/configmap/manual_watcher.go
index ad39bb8b9..b1eb27633 100644
--- a/configmap/manual_watcher.go
+++ b/configmap/manual_watcher.go
@@ -26,9 +26,8 @@ import (
 type ManualWatcher struct {
 	Namespace string
 
-	// Guards mutations to defaultImpl fields
-	m sync.RWMutex
-
+	// Guards observers
+	m         sync.RWMutex
 	observers map[string][]Observer
 }
 
@@ -40,7 +39,7 @@ func (w *ManualWatcher) Watch(name string, o ...Observer) {
 	defer w.m.Unlock()
 
 	if w.observers == nil {
-		w.observers = make(map[string][]Observer, len(o))
+		w.observers = make(map[string][]Observer, 1)
 	}
 	w.observers[name] = append(w.observers[name], o...)
 }
@@ -58,13 +57,8 @@ func (w *ManualWatcher) OnChange(configMap *corev1.ConfigMap) {
 	// Within our namespace, take the lock and see if there are any registered observers.
 	w.m.RLock()
 	defer w.m.RUnlock()
-	observers, ok := w.observers[configMap.Name]
-	if !ok {
-		return // No observers.
-	}
-
 	// Iterate over the observers and invoke their callbacks.
-	for _, o := range observers {
+	for _, o := range w.observers[configMap.Name] {
 		o(configMap)
 	}
 }
diff --git a/controller/OWNERS b/controller/OWNERS
index afa22257a..0b270d53a 100644
--- a/controller/OWNERS
+++ b/controller/OWNERS
@@ -2,3 +2,6 @@
 
 approvers:
 - controller-approvers
+
+reviewers:
+- controller-reviewers
diff --git a/hack/update-codegen.sh b/hack/update-codegen.sh
index 1337f8110..8a2198335 100755
--- a/hack/update-codegen.sh
+++ b/hack/update-codegen.sh
@@ -19,6 +19,12 @@ set -o nounset
 set -o pipefail
 
 export GO111MODULE=on
+# If we run with -mod=vendor here, then generate-groups.sh looks for vendor files in the wrong place.
+export GOFLAGS=-mod=
+
+if [ -z "${GOPATH:-}" ]; then
+  export GOPATH=$(go env GOPATH)
+fi
 
 source $(dirname $0)/../vendor/knative.dev/test-infra/scripts/library.sh
 
diff --git a/pool/OWNERS b/pool/OWNERS
index f4a5f11dc..5c0a9d035 100644
--- a/pool/OWNERS
+++ b/pool/OWNERS
@@ -3,5 +3,8 @@
 approvers:
 - controller-approvers
 
+reviewers:
+- controller-reviewers
+
 labels:
 - area/API
diff --git a/reconciler/OWNERS b/reconciler/OWNERS
index afa22257a..0b270d53a 100644
--- a/reconciler/OWNERS
+++ b/reconciler/OWNERS
@@ -2,3 +2,6 @@
 
 approvers:
 - controller-approvers
+
+reviewers:
+- controller-reviewers
diff --git a/webhook/OWNERS b/webhook/OWNERS
index b87878d94..001df221f 100644
--- a/webhook/OWNERS
+++ b/webhook/OWNERS
@@ -2,3 +2,6 @@
 
 approvers:
 - webhook-approvers
+
+reviewers:
+- webhook-reviewers
diff --git a/webhook/conversion.go b/webhook/conversion.go
index 146ac259f..73cddd7a4 100644
--- a/webhook/conversion.go
+++ b/webhook/conversion.go
@@ -54,6 +54,10 @@ func conversionHandler(rootLogger *zap.SugaredLogger, stats StatsReporter, c Con
 
 		ctx := logging.WithLogger(r.Context(), logger)
 		response := apixv1.ConversionReview{
+			// Use the same type meta as the request - this is required by the K8s API
+			// note: v1beta1 & v1 ConversionReview shapes are identical so even though
+			// we're using v1 types we still support v1beta1 conversion requests
+			TypeMeta: review.TypeMeta,
 			Response: c.Convert(ctx, review.Request),
 		}
 


Test patch:
diff --git a/configmap/manual_watcher_test.go b/configmap/manual_watcher_test.go
index 72f35ee37..dfe0c64e9 100644
--- a/configmap/manual_watcher_test.go
+++ b/configmap/manual_watcher_test.go
@@ -37,57 +37,43 @@ func TestCallbackInvoked(t *testing.T) {
 		Namespace: "default",
 	}
 
-	observer := counter{}
-
-	watcher.Watch("foo", observer.callback)
+	// Verify empty works as designed.
 	watcher.OnChange(&corev1.ConfigMap{
 		ObjectMeta: metav1.ObjectMeta{
 			Namespace: "default",
 			Name:      "foo",
 		},
 	})
-
-	if observer.count() == 0 {
-		t.Errorf("Expected callback to be invoked - got invocations %v", observer.count())
-	}
-}
-
-func TestDifferentNamespace(t *testing.T) {
-	watcher := ManualWatcher{
-		Namespace: "default",
-	}
-
 	observer := counter{}
 
 	watcher.Watch("foo", observer.callback)
 	watcher.OnChange(&corev1.ConfigMap{
 		ObjectMeta: metav1.ObjectMeta{
-			Namespace: "not-default",
+			Namespace: "default",
 			Name:      "foo",
 		},
 	})
 
-	if observer.count() != 0 {
-		t.Errorf("Expected callback to be not be invoked - got invocations %v", observer.count())
+	if observer.count() == 0 {
+		t.Errorf("Expected callback to be invoked - got invocations %v", observer.count())
 	}
 }
 
-func TestLateRegistration(t *testing.T) {
+func TestDifferentNamespace(t *testing.T) {
 	watcher := ManualWatcher{
 		Namespace: "default",
 	}
 
 	observer := counter{}
 
+	watcher.Watch("foo", observer.callback)
 	watcher.OnChange(&corev1.ConfigMap{
 		ObjectMeta: metav1.ObjectMeta{
-			Namespace: "default",
+			Namespace: "not-default",
 			Name:      "foo",
 		},
 	})
 
-	watcher.Watch("foo", observer.callback)
-
 	if observer.count() != 0 {
 		t.Errorf("Expected callback to be not be invoked - got invocations %v", observer.count())
 	}
diff --git a/test/gcs/gcs.go b/test/gcs/gcs.go
index b2798ac6a..13ae206fa 100644
--- a/test/gcs/gcs.go
+++ b/test/gcs/gcs.go
@@ -146,9 +146,17 @@ func (g *GCSClient) getObjectsAttrs(ctx context.Context, bucketName, storagePath
 	return allAttrs, nil
 }
 
+func (g *GCSClient) listChildren(ctx context.Context, bucketName, dirPath, exclusionFilter string) ([]string, error) {
+	if dirPath != "" {
+		dirPath = strings.TrimRight(dirPath, " /") + "/"
+	}
+
+	return g.list(ctx, bucketName, dirPath, exclusionFilter)
+}
+
 // ListChildrenFiles recursively lists all children files.
 func (g *GCSClient) ListChildrenFiles(ctx context.Context, bucketName, dirPath string) ([]string, error) {
-	return g.list(ctx, bucketName, strings.TrimRight(dirPath, " /")+"/", "")
+	return g.listChildren(ctx, bucketName, dirPath, "")
 }
 
 // ListDirectChildren lists direct children paths (including files and directories).
@@ -156,7 +164,7 @@ func (g *GCSClient) ListDirectChildren(ctx context.Context, bucketName, dirPath
 	// If there are 2 directories named "foo" and "foobar",
 	// then given storagePath "foo" will get files both under "foo" and "foobar".
 	// Add trailling slash to storagePath, so that only gets children under given directory.
-	return g.list(ctx, bucketName, strings.TrimRight(dirPath, " /")+"/", "/")
+	return g.listChildren(ctx, bucketName, dirPath, "/")
 }
 
 // CopyObject copies objects from one location to another. Assumes both source and destination buckets exist.
diff --git a/testutils/clustermanager/perf-tests/main.go b/testutils/clustermanager/perf-tests/main.go
index 70b0e88ba..f8084c078 100644
--- a/testutils/clustermanager/perf-tests/main.go
+++ b/testutils/clustermanager/perf-tests/main.go
@@ -27,6 +27,7 @@ import (
 var (
 	isRecreate          bool
 	isReconcile         bool
+	isDelete            bool
 	gcpProjectName      string
 	repoName            string
 	benchmarkRootFolder string
@@ -40,10 +41,11 @@ func main() {
 	flag.StringVar(&benchmarkRootFolder, "benchmark-root", "", "root folder of the benchmarks")
 	flag.BoolVar(&isRecreate, "recreate", false, "is recreate operation or not")
 	flag.BoolVar(&isReconcile, "reconcile", false, "is reconcile operation or not")
+	flag.BoolVar(&isDelete, "delete", false, "is delete operation or not")
 	flag.Parse()
 
-	if isRecreate && isReconcile {
-		log.Fatal("Only one operation can be specified, either recreate or reconcile")
+	if (isRecreate && isReconcile) || (isRecreate && isDelete) || (isReconcile && isDelete) {
+		log.Fatal("--recreate, --reconcile and --delete are mutually exclusive")
 	}
 
 	client, err := testPkg.NewClient(gkeEnvironment)
@@ -61,7 +63,12 @@ func main() {
 			log.Fatalf("Failed reconciling clusters for repo %q: %v", repoName, err)
 		}
 		log.Printf("Done with reconciling clusters for repo %q", repoName)
+	case isDelete:
+		if err := client.DeleteClusters(gcpProjectName, repoName, benchmarkRootFolder); err != nil {
+			log.Fatalf("Failed deleting clusters for repo %q: %v", repoName, err)
+		}
+		log.Printf("Done with deleting clusters for repo %q", repoName)
 	default:
-		log.Fatal("One operation must be specified, either recreate or reconcile")
+		log.Fatal("One operation must be specified, either recreate, reconcile or delete")
 	}
 }
diff --git a/testutils/clustermanager/perf-tests/pkg/cluster.go b/testutils/clustermanager/perf-tests/pkg/cluster.go
index a388c5d75..90f0ccfc5 100644
--- a/testutils/clustermanager/perf-tests/pkg/cluster.go
+++ b/testutils/clustermanager/perf-tests/pkg/cluster.go
@@ -103,6 +103,19 @@ func (gc *gkeClient) ReconcileClusters(gcpProject, repo, benchmarkRoot string) e
 	return gc.processClusters(gcpProject, repo, benchmarkRoot, handleExistingCluster, handleNewClusterConfig)
 }
 
+// DeleteClusters will delete all existing clusters.
+func (gc *gkeClient) DeleteClusters(gcpProject, repo, benchmarkRoot string) error {
+	handleExistingCluster := func(cluster container.Cluster, configExists bool, config ClusterConfig) error {
+		// retain the cluster, if the cluster config is unchanged
+		return gc.deleteClusterWithRetries(gcpProject, cluster)
+	}
+	handleNewClusterConfig := func(clusterName string, clusterConfig ClusterConfig) error {
+		// do nothing
+		return nil
+	}
+	return gc.processClusters(gcpProject, repo, benchmarkRoot, handleExistingCluster, handleNewClusterConfig)
+}
+
 // processClusters will process existing clusters and configs for new clusters,
 // with the corresponding functions provided by callers.
 func (gc *gkeClient) processClusters(
diff --git a/testutils/clustermanager/perf-tests/pkg/cluster_test.go b/testutils/clustermanager/perf-tests/pkg/cluster_test.go
index 8b221a917..e6580f10f 100644
--- a/testutils/clustermanager/perf-tests/pkg/cluster_test.go
+++ b/testutils/clustermanager/perf-tests/pkg/cluster_test.go
@@ -23,6 +23,7 @@ import (
 
 	"github.com/google/go-cmp/cmp"
 	container "google.golang.org/api/container/v1beta1"
+
 	"knative.dev/pkg/test/gke"
 	gkeFake "knative.dev/pkg/test/gke/fake"
 )
@@ -291,7 +292,103 @@ func TestReconcileClusters(t *testing.T) {
 				tc.testName, fakeProject, fakeRepository, tc.benchmarkRoot, diff)
 		}
 	}
+}
+
+func TestDeleteClusters(t *testing.T) {
+	testCases := []struct {
+		testName           string
+		benchmarkRoot      string
+		precreatedClusters map[string]ClusterConfig
+		expectedClusters   map[string]ClusterConfig
+	}{
+		// nothing will be done if there is no cluster at the beginning
+		{
+			testName:           "all related clusters will be deleted",
+			benchmarkRoot:      testBenchmarkRoot,
+			precreatedClusters: make(map[string]ClusterConfig),
+			expectedClusters:   make(map[string]ClusterConfig),
+		},
+		// all clusters will be created if there is no cluster at the beginning
+		{
+			testName:      "all related clusters will be deleted",
+			benchmarkRoot: testBenchmarkRoot,
+			precreatedClusters: map[string]ClusterConfig{
+				clusterNameForBenchmark("test-benchmark1", fakeRepository): {
+					Location:  "us-central1",
+					NodeCount: 2,
+					NodeType:  "n1-standard-4",
+				},
+				clusterNameForBenchmark("test-benchmark2", fakeRepository): {
+					Location:  "us-west1",
+					NodeCount: 2,
+					NodeType:  "n1-standard-8",
+				},
+			},
+			expectedClusters: make(map[string]ClusterConfig),
+		},
+		// clusters that do not belong to this repo will not be touched
+		{
+			testName:      "clusters that do not belong to this repo will not be touched",
+			benchmarkRoot: testBenchmarkRoot,
+			precreatedClusters: map[string]ClusterConfig{
+				clusterNameForBenchmark("test-benchmark1", fakeRepository): {
+					Location:  "us-central1",
+					NodeCount: 2,
+					NodeType:  "n1-standard-4",
+				},
+				"unrelated-cluster": {
+					Location:  "us-central1",
+					NodeCount: 3,
+					NodeType:  "n1-standard-4",
+				},
+			},
+			expectedClusters: map[string]ClusterConfig{
+				"unrelated-cluster": {
+					Location:  "us-central1",
+					NodeCount: 3,
+					NodeType:  "n1-standard-4",
+				},
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		client := setupFakeGKEClient()
+		for name, config := range tc.precreatedClusters {
+			region, zone := gke.RegionZoneFromLoc(config.Location)
+			var addons []string
+			if strings.TrimSpace(config.Addons) != "" {
+				addons = strings.Split(config.Addons, ",")
+			}
+			req := &gke.Request{
+				ClusterName: name,
+				MinNodes:    config.NodeCount,
+				MaxNodes:    config.NodeCount,
+				NodeType:    config.NodeType,
+				Addons:      addons,
+			}
+			creq, _ := gke.NewCreateClusterRequest(req)
+			client.ops.CreateCluster(fakeProject, region, zone, creq)
+		}
+		err := client.DeleteClusters(fakeProject, fakeRepository, testBenchmarkRoot)
+		fmt.Println(err)
+
+		clusters, _ := client.ops.ListClustersInProject(fakeProject)
+		actual := make(map[string]ClusterConfig)
+		for _, cluster := range clusters {
+			actual[cluster.Name] = ClusterConfig{
+				Location:  cluster.Location,
+				NodeCount: cluster.NodePools[0].Autoscaling.MaxNodeCount,
+				NodeType:  cluster.NodePools[0].Config.MachineType,
+				Addons:    getAddonsForCluster(cluster),
+			}
+		}
 
+		if diff := cmp.Diff(tc.expectedClusters, actual); diff != "" {
+			t.Fatalf("Test %q fails, DeleteClusters(%q, %q, %q) returns wrong result (-want +got):\n%s",
+				tc.testName, fakeProject, fakeRepository, tc.benchmarkRoot, diff)
+		}
+	}
 }
 
 // Return addons as a string slice for the given cluster.
diff --git a/webhook/conversion_integration_test.go b/webhook/conversion_integration_test.go
index 67879caff..422326a2e 100644
--- a/webhook/conversion_integration_test.go
+++ b/webhook/conversion_integration_test.go
@@ -25,6 +25,7 @@ import (
 	"net/http"
 	"testing"
 
+	"github.com/google/go-cmp/cmp"
 	"golang.org/x/sync/errgroup"
 	apixv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -136,6 +137,10 @@ func TestConversionValidResponse(t *testing.T) {
 	if reviewResponse.Response.UID != "some-uid" {
 		t.Errorf("expected the response uid to be the stubbed version")
 	}
+
+	if diff := cmp.Diff(review.TypeMeta, reviewResponse.TypeMeta); diff != "" {
+		t.Errorf("expected the response typeMeta to be the same as the request (-want, +got)\n%s", diff)
+	}
 }
 
 func TestConversionInvalidResponse(t *testing.T) {