Repository: kubernetes-sigs/cluster-api
Pull request #3004 (2020-05-05 10:40:21)

🏃CAPD automatically re-create a machine if there is an error during provisioning
**What this PR does / why we need it**:
This PR makes CAPD recover from conditions when the docker provider creates a container for a machine, but for some reason, the container is not fully operational/does not completes all the provisioning steps.

More specifically, given that the CAPD provisioning time is small, the PR cleanups containers with provisioning errors and re-provision from scratch

**Which issue(s) this PR fixes**:
Fixes https://github.com/kubernetes-sigs/cluster-api/issues/2999
Fixes https://github.com/kubernetes-sigs/cluster-api/issues/2341

/assing @vincepri 
/assing @sedefsavas 


Patch:
diff --git a/bootstrap/kubeadm/controllers/kubeadmconfig_controller.go b/bootstrap/kubeadm/controllers/kubeadmconfig_controller.go
index a2f99eb47..fec09f452 100644
--- a/bootstrap/kubeadm/controllers/kubeadmconfig_controller.go
+++ b/bootstrap/kubeadm/controllers/kubeadmconfig_controller.go
@@ -42,9 +42,7 @@ import (
 	expv1 "sigs.k8s.io/cluster-api/exp/api/v1alpha3"
 	"sigs.k8s.io/cluster-api/feature"
 	"sigs.k8s.io/cluster-api/util"
-	"sigs.k8s.io/cluster-api/util/annotations"
 	"sigs.k8s.io/cluster-api/util/patch"
-	"sigs.k8s.io/cluster-api/util/predicates"
 	"sigs.k8s.io/cluster-api/util/secret"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -92,19 +90,24 @@ func (r *KubeadmConfigReconciler) SetupWithManager(mgr ctrl.Manager, option cont
 
 	r.scheme = mgr.GetScheme()
 
-	b := ctrl.NewControllerManagedBy(mgr).
+	builder := ctrl.NewControllerManagedBy(mgr).
 		For(&bootstrapv1.KubeadmConfig{}).
 		WithOptions(option).
-		WithEventFilter(predicates.ResourceNotPaused(r.Log)).
 		Watches(
 			&source.Kind{Type: &clusterv1.Machine{}},
 			&handler.EnqueueRequestsFromMapFunc{
 				ToRequests: handler.ToRequestsFunc(r.MachineToBootstrapMapFunc),
 			},
+		).
+		Watches(
+			&source.Kind{Type: &clusterv1.Cluster{}},
+			&handler.EnqueueRequestsFromMapFunc{
+				ToRequests: handler.ToRequestsFunc(r.ClusterToKubeadmConfigs),
+			},
 		)
 
 	if feature.Gates.Enabled(feature.MachinePool) {
-		b = b.Watches(
+		builder = builder.Watches(
 			&source.Kind{Type: &expv1.MachinePool{}},
 			&handler.EnqueueRequestsFromMapFunc{
 				ToRequests: handler.ToRequestsFunc(r.MachinePoolToBootstrapMapFunc),
@@ -112,22 +115,10 @@ func (r *KubeadmConfigReconciler) SetupWithManager(mgr ctrl.Manager, option cont
 		)
 	}
 
-	c, err := b.Build(r)
-	if err != nil {
+	if err := builder.Complete(r); err != nil {
 		return errors.Wrap(err, "failed setting up with a controller manager")
 	}
 
-	err = c.Watch(
-		&source.Kind{Type: &clusterv1.Cluster{}},
-		&handler.EnqueueRequestsFromMapFunc{
-			ToRequests: handler.ToRequestsFunc(r.ClusterToKubeadmConfigs),
-		},
-		predicates.ClusterUnpausedAndInfrastructureReady(r.Log),
-	)
-	if err != nil {
-		return errors.Wrap(err, "failed adding Watch for Clusters to controller manager")
-	}
-
 	return nil
 }
 
@@ -177,11 +168,6 @@ func (r *KubeadmConfigReconciler) Reconcile(req ctrl.Request) (_ ctrl.Result, re
 		return ctrl.Result{}, err
 	}
 
-	if annotations.IsPaused(cluster, config) {
-		log.Info("Reconciliation is paused for this object")
-		return ctrl.Result{}, nil
-	}
-
 	scope := &Scope{
 		Logger:      log,
 		Config:      config,
diff --git a/controllers/cluster_controller.go b/controllers/cluster_controller.go
index cd4541e4e..977bcc966 100644
--- a/controllers/cluster_controller.go
+++ b/controllers/cluster_controller.go
@@ -35,9 +35,7 @@ import (
 	"sigs.k8s.io/cluster-api/controllers/metrics"
 	capierrors "sigs.k8s.io/cluster-api/errors"
 	"sigs.k8s.io/cluster-api/util"
-	"sigs.k8s.io/cluster-api/util/annotations"
 	"sigs.k8s.io/cluster-api/util/patch"
-	"sigs.k8s.io/cluster-api/util/predicates"
 	"sigs.k8s.io/cluster-api/util/secret"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -79,7 +77,6 @@ func (r *ClusterReconciler) SetupWithManager(mgr ctrl.Manager, options controlle
 			&handler.EnqueueRequestsFromMapFunc{ToRequests: handler.ToRequestsFunc(r.controlPlaneMachineToCluster)},
 		).
 		WithOptions(options).
-		WithEventFilter(predicates.ResourceNotPaused(r.Log)).
 		Build(r)
 
 	if err != nil {
@@ -112,7 +109,7 @@ func (r *ClusterReconciler) Reconcile(req ctrl.Request) (_ ctrl.Result, reterr e
 	}
 
 	// Return early if the object or Cluster is paused.
-	if annotations.IsPaused(cluster, cluster) {
+	if util.IsPaused(cluster, cluster) {
 		logger.Info("Reconciliation is paused for this object")
 		return ctrl.Result{}, nil
 	}
diff --git a/controllers/cluster_controller_phases.go b/controllers/cluster_controller_phases.go
index 6f815fc52..7888f5ac7 100644
--- a/controllers/cluster_controller_phases.go
+++ b/controllers/cluster_controller_phases.go
@@ -29,7 +29,6 @@ import (
 	"sigs.k8s.io/cluster-api/controllers/external"
 	capierrors "sigs.k8s.io/cluster-api/errors"
 	"sigs.k8s.io/cluster-api/util"
-	"sigs.k8s.io/cluster-api/util/annotations"
 	utilconversion "sigs.k8s.io/cluster-api/util/conversion"
 	"sigs.k8s.io/cluster-api/util/kubeconfig"
 	"sigs.k8s.io/cluster-api/util/patch"
@@ -79,7 +78,7 @@ func (r *ClusterReconciler) reconcileExternal(ctx context.Context, cluster *clus
 	}
 
 	// if external ref is paused, return error.
-	if annotations.IsPaused(cluster, obj) {
+	if util.IsPaused(cluster, obj) {
 		logger.V(3).Info("External object referenced is paused")
 		return external.ReconcileOutput{Paused: true}, nil
 	}
diff --git a/controllers/external/tracker.go b/controllers/external/tracker.go
index 29088fdac..bbc097c60 100644
--- a/controllers/external/tracker.go
+++ b/controllers/external/tracker.go
@@ -23,7 +23,6 @@ import (
 	"github.com/pkg/errors"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
-	"sigs.k8s.io/cluster-api/util/predicates"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"sigs.k8s.io/controller-runtime/pkg/source"
@@ -56,7 +55,6 @@ func (o *ObjectTracker) Watch(log logr.Logger, obj runtime.Object, handler handl
 	err := o.Controller.Watch(
 		&source.Kind{Type: u},
 		handler,
-		predicates.ResourceNotPaused(log),
 	)
 	if err != nil {
 		o.m.Delete(obj)
diff --git a/controllers/machine_controller.go b/controllers/machine_controller.go
index ab7feca5d..2298b70f0 100644
--- a/controllers/machine_controller.go
+++ b/controllers/machine_controller.go
@@ -42,16 +42,13 @@ import (
 	capierrors "sigs.k8s.io/cluster-api/errors"
 	kubedrain "sigs.k8s.io/cluster-api/third_party/kubernetes-drain"
 	"sigs.k8s.io/cluster-api/util"
-	"sigs.k8s.io/cluster-api/util/annotations"
 	"sigs.k8s.io/cluster-api/util/patch"
-	"sigs.k8s.io/cluster-api/util/predicates"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
-	"sigs.k8s.io/controller-runtime/pkg/source"
 )
 
 var (
@@ -80,30 +77,22 @@ type MachineReconciler struct {
 }
 
 func (r *MachineReconciler) SetupWithManager(mgr ctrl.Manager, options controller.Options) error {
-	clusterToMachines, err := util.ClusterToObjectsMapper(mgr.GetClient(), &clusterv1.MachineList{}, mgr.GetScheme())
-	if err != nil {
-		return err
-	}
-
 	controller, err := ctrl.NewControllerManagedBy(mgr).
 		For(&clusterv1.Machine{}).
 		WithOptions(options).
-		WithEventFilter(predicates.ResourceNotPaused(r.Log)).
 		Build(r)
+
 	if err != nil {
 		return errors.Wrap(err, "failed setting up with a controller manager")
 	}
 
-	err = controller.Watch(
-		&source.Kind{Type: &clusterv1.Cluster{}},
-		&handler.EnqueueRequestsFromMapFunc{
-			ToRequests: clusterToMachines,
-		},
-		// TODO: should this wait for Cluster.Status.InfrastructureReady similar to Infra Machine resources?
-		predicates.ClusterUnpaused(r.Log),
-	)
+	// Add a watch on clusterv1.Cluster object for paused notifications.
+	clusterToMachines, err := util.ClusterToObjectsMapper(mgr.GetClient(), &clusterv1.MachineList{}, mgr.GetScheme())
 	if err != nil {
-		return errors.Wrap(err, "failed to add Watch for Clusters to controller manager")
+		return err
+	}
+	if err := util.WatchOnClusterPaused(controller, clusterToMachines); err != nil {
+		return err
 	}
 
 	r.recorder = mgr.GetEventRecorderFor("machine-controller")
@@ -154,7 +143,7 @@ func (r *MachineReconciler) Reconcile(req ctrl.Request) (_ ctrl.Result, reterr e
 	}
 
 	// Return early if the object or Cluster is paused.
-	if annotations.IsPaused(cluster, m) {
+	if util.IsPaused(cluster, m) {
 		logger.Info("Reconciliation is paused for this object")
 		return ctrl.Result{}, nil
 	}
diff --git a/controllers/machine_controller_phases.go b/controllers/machine_controller_phases.go
index 73cf303a9..1f6df26a8 100644
--- a/controllers/machine_controller_phases.go
+++ b/controllers/machine_controller_phases.go
@@ -28,7 +28,6 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/utils/pointer"
-	"sigs.k8s.io/cluster-api/util/annotations"
 	utilconversion "sigs.k8s.io/cluster-api/util/conversion"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	"sigs.k8s.io/controller-runtime/pkg/handler"
@@ -103,7 +102,7 @@ func (r *MachineReconciler) reconcileExternal(ctx context.Context, cluster *clus
 	}
 
 	// if external ref is paused, return error.
-	if annotations.IsPaused(cluster, obj) {
+	if util.IsPaused(cluster, obj) {
 		logger.V(3).Info("External object referenced is paused")
 		return external.ReconcileOutput{Paused: true}, nil
 	}
diff --git a/controllers/machinedeployment_controller.go b/controllers/machinedeployment_controller.go
index 6404c3e80..b0ced4ecb 100644
--- a/controllers/machinedeployment_controller.go
+++ b/controllers/machinedeployment_controller.go
@@ -30,9 +30,7 @@ import (
 	"k8s.io/client-go/tools/record"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3"
 	"sigs.k8s.io/cluster-api/util"
-	"sigs.k8s.io/cluster-api/util/annotations"
 	"sigs.k8s.io/cluster-api/util/patch"
-	"sigs.k8s.io/cluster-api/util/predicates"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
@@ -60,12 +58,7 @@ type MachineDeploymentReconciler struct {
 }
 
 func (r *MachineDeploymentReconciler) SetupWithManager(mgr ctrl.Manager, options controller.Options) error {
-	clusterToMachineDeployments, err := util.ClusterToObjectsMapper(mgr.GetClient(), &clusterv1.MachineDeploymentList{}, mgr.GetScheme())
-	if err != nil {
-		return err
-	}
-
-	c, err := ctrl.NewControllerManagedBy(mgr).
+	controller, err := ctrl.NewControllerManagedBy(mgr).
 		For(&clusterv1.MachineDeployment{}).
 		Owns(&clusterv1.MachineSet{}).
 		Watches(
@@ -73,22 +66,19 @@ func (r *MachineDeploymentReconciler) SetupWithManager(mgr ctrl.Manager, options
 			&handler.EnqueueRequestsFromMapFunc{ToRequests: handler.ToRequestsFunc(r.MachineSetToDeployments)},
 		).
 		WithOptions(options).
-		WithEventFilter(predicates.ResourceNotPaused(r.Log)).
 		Build(r)
+
 	if err != nil {
 		return errors.Wrap(err, "failed setting up with a controller manager")
 	}
 
-	err = c.Watch(
-		&source.Kind{Type: &clusterv1.Cluster{}},
-		&handler.EnqueueRequestsFromMapFunc{
-			ToRequests: clusterToMachineDeployments,
-		},
-		// TODO: should this wait for Cluster.Status.InfrastructureReady similar to Infra Machine resources?
-		predicates.ClusterUnpaused(r.Log),
-	)
+	// Add a watch on clusterv1.Cluster object for paused notifications.
+	clusterToMachineDeployments, err := util.ClusterToObjectsMapper(mgr.GetClient(), &clusterv1.MachineDeploymentList{}, mgr.GetScheme())
 	if err != nil {
-		return errors.Wrap(err, "failed to add Watch for Clusters to controller manager")
+		return err
+	}
+	if err := util.WatchOnClusterPaused(controller, clusterToMachineDeployments); err != nil {
+		return err
 	}
 
 	r.recorder = mgr.GetEventRecorderFor("machinedeployment-controller")
@@ -117,7 +107,7 @@ func (r *MachineDeploymentReconciler) Reconcile(req ctrl.Request) (_ ctrl.Result
 	}
 
 	// Return early if the object or Cluster is paused.
-	if annotations.IsPaused(cluster, deployment) {
+	if util.IsPaused(cluster, deployment) {
 		logger.Info("Reconciliation is paused for this object")
 		return ctrl.Result{}, nil
 	}
diff --git a/controllers/machinehealthcheck_controller.go b/controllers/machinehealthcheck_controller.go
index f6de9b728..5bd74f2a8 100644
--- a/controllers/machinehealthcheck_controller.go
+++ b/controllers/machinehealthcheck_controller.go
@@ -35,9 +35,7 @@ import (
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3"
 	"sigs.k8s.io/cluster-api/controllers/remote"
 	"sigs.k8s.io/cluster-api/util"
-	"sigs.k8s.io/cluster-api/util/annotations"
 	"sigs.k8s.io/cluster-api/util/patch"
-	"sigs.k8s.io/cluster-api/util/predicates"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -78,25 +76,20 @@ type MachineHealthCheckReconciler struct {
 func (r *MachineHealthCheckReconciler) SetupWithManager(mgr ctrl.Manager, options controller.Options) error {
 	controller, err := ctrl.NewControllerManagedBy(mgr).
 		For(&clusterv1.MachineHealthCheck{}).
+		Watches(
+			&source.Kind{Type: &clusterv1.Cluster{}},
+			&handler.EnqueueRequestsFromMapFunc{ToRequests: handler.ToRequestsFunc(r.clusterToMachineHealthCheck)},
+		).
 		Watches(
 			&source.Kind{Type: &clusterv1.Machine{}},
 			&handler.EnqueueRequestsFromMapFunc{ToRequests: handler.ToRequestsFunc(r.machineToMachineHealthCheck)},
 		).
 		WithOptions(options).
-		WithEventFilter(predicates.ResourceNotPaused(r.Log)).
 		Build(r)
+
 	if err != nil {
 		return errors.Wrap(err, "failed setting up with a controller manager")
 	}
-	err = controller.Watch(
-		&source.Kind{Type: &clusterv1.Cluster{}},
-		&handler.EnqueueRequestsFromMapFunc{ToRequests: handler.ToRequestsFunc(r.clusterToMachineHealthCheck)},
-		// TODO: should this wait for Cluster.Status.InfrastructureReady similar to Infra Machine resources?
-		predicates.ClusterUnpaused(r.Log),
-	)
-	if err != nil {
-		return errors.Wrap(err, "failed to add Watch for Clusters to controller manager")
-	}
 
 	// Add index to MachineHealthCheck for listing by Cluster Name
 	if err := mgr.GetCache().IndexField(&clusterv1.MachineHealthCheck{},
@@ -147,7 +140,7 @@ func (r *MachineHealthCheckReconciler) Reconcile(req ctrl.Request) (_ ctrl.Resul
 	}
 
 	// Return early if the object or Cluster is paused.
-	if annotations.IsPaused(cluster, m) {
+	if util.IsPaused(cluster, m) {
 		logger.Info("Reconciliation is paused for this object")
 		return ctrl.Result{}, nil
 	}
diff --git a/controllers/machineset_controller.go b/controllers/machineset_controller.go
index 2d3b16089..dedbe249a 100644
--- a/controllers/machineset_controller.go
+++ b/controllers/machineset_controller.go
@@ -38,10 +38,8 @@ import (
 	"sigs.k8s.io/cluster-api/controllers/noderefutil"
 	"sigs.k8s.io/cluster-api/controllers/remote"
 	"sigs.k8s.io/cluster-api/util"
-	"sigs.k8s.io/cluster-api/util/annotations"
 	utilconversion "sigs.k8s.io/cluster-api/util/conversion"
 	"sigs.k8s.io/cluster-api/util/patch"
-	"sigs.k8s.io/cluster-api/util/predicates"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
@@ -77,12 +75,7 @@ type MachineSetReconciler struct {
 }
 
 func (r *MachineSetReconciler) SetupWithManager(mgr ctrl.Manager, options controller.Options) error {
-	clusterToMachineSets, err := util.ClusterToObjectsMapper(mgr.GetClient(), &clusterv1.MachineSetList{}, mgr.GetScheme())
-	if err != nil {
-		return err
-	}
-
-	c, err := ctrl.NewControllerManagedBy(mgr).
+	controller, err := ctrl.NewControllerManagedBy(mgr).
 		For(&clusterv1.MachineSet{}).
 		Owns(&clusterv1.Machine{}).
 		Watches(
@@ -90,20 +83,19 @@ func (r *MachineSetReconciler) SetupWithManager(mgr ctrl.Manager, options contro
 			&handler.EnqueueRequestsFromMapFunc{ToRequests: handler.ToRequestsFunc(r.MachineToMachineSets)},
 		).
 		WithOptions(options).
-		WithEventFilter(predicates.ResourceNotPaused(r.Log)).
 		Build(r)
+
 	if err != nil {
 		return errors.Wrap(err, "failed setting up with a controller manager")
 	}
 
-	err = c.Watch(
-		&source.Kind{Type: &clusterv1.Cluster{}},
-		&handler.EnqueueRequestsFromMapFunc{ToRequests: clusterToMachineSets},
-		// TODO: should this wait for Cluster.Status.InfrastructureReady similar to Infra Machine resources?
-		predicates.ClusterUnpaused(r.Log),
-	)
+	// Add a watch on clusterv1.Cluster object for paused notifications.
+	clusterToMachineSets, err := util.ClusterToObjectsMapper(mgr.GetClient(), &clusterv1.MachineSetList{}, mgr.GetScheme())
 	if err != nil {
-		return errors.Wrap(err, "failed to add Watch for Clusters to controller manager")
+		return err
+	}
+	if err := util.WatchOnClusterPaused(controller, clusterToMachineSets); err != nil {
+		return err
 	}
 
 	r.recorder = mgr.GetEventRecorderFor("machineset-controller")
@@ -132,7 +124,7 @@ func (r *MachineSetReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error)
 	}
 
 	// Return early if the object or Cluster is paused.
-	if annotations.IsPaused(cluster, machineSet) {
+	if util.IsPaused(cluster, machineSet) {
 		logger.Info("Reconciliation is paused for this object")
 		return ctrl.Result{}, nil
 	}
diff --git a/controllers/remote/cluster.go b/controllers/remote/cluster.go
index 131bcf6a6..190098070 100644
--- a/controllers/remote/cluster.go
+++ b/controllers/remote/cluster.go
@@ -40,11 +40,12 @@ func NewClusterClient(ctx context.Context, c client.Client, cluster client.Objec
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to create client for Cluster %s/%s", cluster.Namespace, cluster.Name)
 	}
+
 	return ret, nil
 }
 
 // RESTConfig returns a configuration instance to be used with a Kubernetes client.
-func RESTConfig(ctx context.Context, c client.Reader, cluster client.ObjectKey) (*restclient.Config, error) {
+func RESTConfig(ctx context.Context, c client.Client, cluster client.ObjectKey) (*restclient.Config, error) {
 	kubeConfig, err := kcfg.FromSecret(ctx, c, cluster)
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to retrieve kubeconfig secret for Cluster %s/%s", cluster.Namespace, cluster.Name)
diff --git a/controlplane/kubeadm/controllers/controller.go b/controlplane/kubeadm/controllers/controller.go
index a601526cb..7831c6dec 100644
--- a/controlplane/kubeadm/controllers/controller.go
+++ b/controlplane/kubeadm/controllers/controller.go
@@ -36,9 +36,7 @@ import (
 	"sigs.k8s.io/cluster-api/controlplane/kubeadm/internal/machinefilters"
 	capierrors "sigs.k8s.io/cluster-api/errors"
 	"sigs.k8s.io/cluster-api/util"
-	"sigs.k8s.io/cluster-api/util/annotations"
 	"sigs.k8s.io/cluster-api/util/patch"
-	"sigs.k8s.io/cluster-api/util/predicates"
 	"sigs.k8s.io/cluster-api/util/secret"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -65,30 +63,22 @@ type KubeadmControlPlaneReconciler struct {
 	controller controller.Controller
 	recorder   record.EventRecorder
 
-	managementCluster         internal.ManagementCluster
-	managementClusterUncached internal.ManagementCluster
+	managementCluster internal.ManagementCluster
 }
 
 func (r *KubeadmControlPlaneReconciler) SetupWithManager(mgr ctrl.Manager, options controller.Options) error {
 	c, err := ctrl.NewControllerManagedBy(mgr).
 		For(&controlplanev1.KubeadmControlPlane{}).
 		Owns(&clusterv1.Machine{}).
+		Watches(
+			&source.Kind{Type: &clusterv1.Cluster{}},
+			&handler.EnqueueRequestsFromMapFunc{ToRequests: handler.ToRequestsFunc(r.ClusterToKubeadmControlPlane)},
+		).
 		WithOptions(options).
-		WithEventFilter(predicates.ResourceNotPaused(r.Log)).
 		Build(r)
-	if err != nil {
-		return errors.Wrap(err, "failed setting up with a controller manager")
-	}
 
-	err = c.Watch(
-		&source.Kind{Type: &clusterv1.Cluster{}},
-		&handler.EnqueueRequestsFromMapFunc{
-			ToRequests: handler.ToRequestsFunc(r.ClusterToKubeadmControlPlane),
-		},
-		predicates.ClusterUnpausedAndInfrastructureReady(r.Log),
-	)
 	if err != nil {
-		return errors.Wrap(err, "failed adding Watch for Clusters to controller manager")
+		return errors.Wrap(err, "failed setting up with a controller manager")
 	}
 
 	r.scheme = mgr.GetScheme()
@@ -98,9 +88,6 @@ func (r *KubeadmControlPlaneReconciler) SetupWithManager(mgr ctrl.Manager, optio
 	if r.managementCluster == nil {
 		r.managementCluster = &internal.Management{Client: r.Client}
 	}
-	if r.managementClusterUncached == nil {
-		r.managementClusterUncached = &internal.Management{Client: mgr.GetAPIReader()}
-	}
 
 	return nil
 }
@@ -130,7 +117,7 @@ func (r *KubeadmControlPlaneReconciler) Reconcile(req ctrl.Request) (res ctrl.Re
 	}
 	logger = logger.WithValues("cluster", cluster.Name)
 
-	if annotations.IsPaused(cluster, kcp) {
+	if util.IsPaused(cluster, kcp) {
 		logger.Info("Reconciliation is paused for this object")
 		return ctrl.Result{}, nil
 	}
diff --git a/controlplane/kubeadm/controllers/scale.go b/controlplane/kubeadm/controllers/scale.go
index 8b2a4c95f..8cb1332f3 100644
--- a/controlplane/kubeadm/controllers/scale.go
+++ b/controlplane/kubeadm/controllers/scale.go
@@ -25,7 +25,6 @@ import (
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3"
 	controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1alpha3"
 	"sigs.k8s.io/cluster-api/controlplane/kubeadm/internal"
-	"sigs.k8s.io/cluster-api/controlplane/kubeadm/internal/machinefilters"
 	capierrors "sigs.k8s.io/cluster-api/errors"
 	"sigs.k8s.io/cluster-api/util"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -34,20 +33,6 @@ import (
 func (r *KubeadmControlPlaneReconciler) initializeControlPlane(ctx context.Context, cluster *clusterv1.Cluster, kcp *controlplanev1.KubeadmControlPlane, controlPlane *internal.ControlPlane) (ctrl.Result, error) {
 	logger := controlPlane.Logger()
 
-	// Perform an uncached read of all the owned machines. This check is in place to make sure
-	// that the controller cache is not misbehaving and we end up initializing the cluster more than once.
-	ownedMachines, err := r.managementClusterUncached.GetMachinesForCluster(ctx, util.ObjectKey(cluster), machinefilters.OwnedControlPlaneMachines(kcp.Name))
-	if err != nil {
-		logger.Error(err, "failed to perform an uncached read of control plane machines for cluster")
-		return ctrl.Result{}, err
-	}
-	if len(ownedMachines) > 0 {
-		return ctrl.Result{}, errors.Errorf(
-			"control plane has already been initialized, found %d owned machine for cluster %s/%s: controller cache or management cluster is misbehaving",
-			len(ownedMachines), cluster.Namespace, cluster.Name,
-		)
-	}
-
 	bootstrapSpec := controlPlane.InitialControlPlaneConfig()
 	fd := controlPlane.FailureDomainWithFewestMachines()
 	if err := r.cloneConfigsAndGenerateMachine(ctx, cluster, kcp, bootstrapSpec, fd); err != nil {
diff --git a/controlplane/kubeadm/internal/cluster.go b/controlplane/kubeadm/internal/cluster.go
index 24cf31000..40a6be3d3 100644
--- a/controlplane/kubeadm/internal/cluster.go
+++ b/controlplane/kubeadm/internal/cluster.go
@@ -46,7 +46,7 @@ type ManagementCluster interface {
 
 // Management holds operations on the management cluster.
 type Management struct {
-	Client ctrlclient.Reader
+	Client ctrlclient.Client
 }
 
 // GetMachinesForCluster returns a list of machines that can be filtered or not.
diff --git a/docs/book/src/developer/providers/v1alpha2-to-v1alpha3.md b/docs/book/src/developer/providers/v1alpha2-to-v1alpha3.md
index 6c3b287a5..d51bbc2ff 100644
--- a/docs/book/src/developer/providers/v1alpha2-to-v1alpha3.md
+++ b/docs/book/src/developer/providers/v1alpha2-to-v1alpha3.md
@@ -162,45 +162,18 @@ outside of the existing module.
   }
   ```
 - Unless your controller is already watching Clusters, add a Watch to get notifications when Cluster.Spec.Paused field changes.
-  In most cases, `predicates.ClusterUnpaused` and `util.ClusterToObjectsMapper` can be used like in the example below:
+  In most cases, `util.WatchOnClusterPaused` and `util.ClusterToObjectsMapper` can be used like in the example below:
   ```go
   // Add a watch on clusterv1.Cluster object for paused notifications.
   clusterToObjectFunc, err := util.ClusterToObjectsMapper(mgr.GetClient(), <List object here>, mgr.GetScheme())
   if err != nil {
     return err
   }
-  err = controller.Watch(
-      &source.Kind{Type: &cluserv1.Cluster{}},
-      &handler.EnqueueRequestsFromMapFunc{
-          ToRequests: clusterToObjectFunc,
-      },
-      predicates.ClusterUnpaused(r.Log),
-  )
-  if err != nil {
+  if err := util.WatchOnClusterPaused(controller, clusterToObjectFunc); err != nil {
     return err
   }
   ```
   NB: You need to have `cluster.x-k8s.io/cluster-name` applied to all your objects for the mapper to function.
-- In some cases, you'll want to not just watch on Cluster.Spec.Paused changes, but also on
-  Cluster.Status.InfrastructureReady. For those cases `predicates.ClusterUnpausedAndInfrastructureReady` should be used
-  instead. 
-  ```go
-  // Add a watch on clusterv1.Cluster object for paused and infrastructureReady notifications.
-  clusterToObjectFunc, err := util.ClusterToObjectsMapper(mgr.GetClient(), <List object here>, mgr.GetScheme())
-  if err != nil {
-    return err
-  }
-  err = controller.Watch(
-        &source.Kind{Type: &cluserv1.Cluster{}},
-        &handler.EnqueueRequestsFromMapFunc{
-            ToRequests: clusterToObjectFunc,
-        },
-        predicates.ClusterUnpausedAndInfrastructureReady(r.Log),
-    )
-    if err != nil {
-      return err
-    }
-    ```
 
 ## [OPTIONAL] Support failure domains.
 
diff --git a/exp/controllers/machinepool_controller.go b/exp/controllers/machinepool_controller.go
index 9f8030317..1d1d64ffc 100644
--- a/exp/controllers/machinepool_controller.go
+++ b/exp/controllers/machinepool_controller.go
@@ -36,15 +36,11 @@ import (
 	capierrors "sigs.k8s.io/cluster-api/errors"
 	expv1 "sigs.k8s.io/cluster-api/exp/api/v1alpha3"
 	"sigs.k8s.io/cluster-api/util"
-	"sigs.k8s.io/cluster-api/util/annotations"
 	"sigs.k8s.io/cluster-api/util/patch"
-	"sigs.k8s.io/cluster-api/util/predicates"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
-	"sigs.k8s.io/controller-runtime/pkg/handler"
-	"sigs.k8s.io/controller-runtime/pkg/source"
 )
 
 // +kubebuilder:rbac:groups=core,resources=events,verbs=get;list;watch;create;patch
@@ -66,30 +62,13 @@ type MachinePoolReconciler struct {
 }
 
 func (r *MachinePoolReconciler) SetupWithManager(mgr ctrl.Manager, options controller.Options) error {
-	clusterToMachinePools, err := util.ClusterToObjectsMapper(mgr.GetClient(), &expv1.MachinePoolList{}, mgr.GetScheme())
-	if err != nil {
-		return err
-	}
-
 	c, err := ctrl.NewControllerManagedBy(mgr).
 		For(&expv1.MachinePool{}).
 		WithOptions(options).
-		WithEventFilter(predicates.ResourceNotPaused(r.Log)).
 		Build(r)
 	if err != nil {
 		return errors.Wrap(err, "failed setting up with a controller manager")
 	}
-	err = c.Watch(
-		&source.Kind{Type: &clusterv1.Cluster{}},
-		&handler.EnqueueRequestsFromMapFunc{
-			ToRequests: clusterToMachinePools,
-		},
-		// TODO: should this wait for Cluster.Status.InfrastructureReady similar to Infra Machine resources?
-		predicates.ClusterUnpaused(r.Log),
-	)
-	if err != nil {
-		return errors.Wrap(err, "failed adding Watch for Cluster to controller manager")
-	}
 
 	r.controller = c
 	r.recorder = mgr.GetEventRecorderFor("machinepool-controller")
@@ -121,7 +100,7 @@ func (r *MachinePoolReconciler) Reconcile(req ctrl.Request) (_ ctrl.Result, rete
 	}
 
 	// Return early if the object or Cluster is paused.
-	if annotations.IsPaused(cluster, mp) {
+	if util.IsPaused(cluster, mp) {
 		logger.Info("Reconciliation is paused for this object")
 		return ctrl.Result{}, nil
 	}
diff --git a/exp/controllers/machinepool_controller_phases.go b/exp/controllers/machinepool_controller_phases.go
index ec0af7ddf..7ed61f1f0 100644
--- a/exp/controllers/machinepool_controller_phases.go
+++ b/exp/controllers/machinepool_controller_phases.go
@@ -33,7 +33,6 @@ import (
 	capierrors "sigs.k8s.io/cluster-api/errors"
 	expv1 "sigs.k8s.io/cluster-api/exp/api/v1alpha3"
 	"sigs.k8s.io/cluster-api/util"
-	"sigs.k8s.io/cluster-api/util/annotations"
 	"sigs.k8s.io/cluster-api/util/patch"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	"sigs.k8s.io/controller-runtime/pkg/handler"
@@ -91,7 +90,7 @@ func (r *MachinePoolReconciler) reconcileExternal(ctx context.Context, cluster *
 	}
 
 	// if external ref is paused, return error.
-	if annotations.IsPaused(cluster, obj) {
+	if util.IsPaused(cluster, obj) {
 		logger.V(3).Info("External object referenced is paused")
 		return external.ReconcileOutput{Paused: true}, nil
 	}
diff --git a/go.mod b/go.mod
index 48337f501..e58637016 100644
--- a/go.mod
+++ b/go.mod
@@ -11,9 +11,12 @@ require (
 	github.com/evanphx/json-patch v4.5.0+incompatible
 	github.com/go-logr/logr v0.1.0
 	github.com/gogo/protobuf v1.3.1
+	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
 	github.com/google/go-github v17.0.0+incompatible
 	github.com/google/go-querystring v1.0.0 // indirect
 	github.com/google/gofuzz v1.1.0
+	github.com/hashicorp/golang-lru v0.5.4 // indirect
+	github.com/imdario/mergo v0.3.8 // indirect
 	github.com/onsi/ginkgo v1.12.0
 	github.com/onsi/gomega v1.9.0
 	github.com/opencontainers/go-digest v1.0.0-rc1 // indirect
@@ -24,8 +27,10 @@ require (
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.6.2
 	go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738
+	golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073 // indirect
 	golang.org/x/net v0.0.0-20200301022130-244492dfa37a
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
+	golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect
 	google.golang.org/appengine v1.6.1 // indirect
 	google.golang.org/grpc v1.26.0
 	k8s.io/api v0.17.2
diff --git a/go.sum b/go.sum
index c468b50c2..e129e3537 100644
--- a/go.sum
+++ b/go.sum
@@ -185,8 +185,9 @@ github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -239,15 +240,17 @@ github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t
 github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
 github.com/hashicorp/golang-lru v0.0.0-20180201235237-0fb14efe8c47/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
+github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
-github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28=
 github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
+github.com/imdario/mergo v0.3.8 h1:CGgOkSJeqMRmt0D9XLWExdT4m4F1vd3FV3VPt+0VxkQ=
+github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jimstudt/http-authentication v0.0.0-20140401203705-3eca13d6893a/go.mod h1:wK6yTYYcgjHE1Z1QtXACPDjcFJyBskHEdagmnq3vsP8=
@@ -462,8 +465,9 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073 h1:xMPOj6Pz6UipU1wXLkrtqpHbR0AVFnyPEQq/wRWz9lM=
+golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -538,8 +542,9 @@ golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
diff --git a/util/annotations/paused.go b/util/annotations/paused.go
deleted file mode 100644
index acb817a05..000000000
--- a/util/annotations/paused.go
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
-Copyright 2020 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package annotations
-
-import (
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3"
-)
-
-// IsPaused returns true if the Cluster is paused or the object has the `paused` annotation.
-func IsPaused(cluster *clusterv1.Cluster, o metav1.Object) bool {
-	if cluster.Spec.Paused {
-		return true
-	}
-	return HasPausedAnnotation(o)
-}
-
-// HasPausedAnnotation returns true if the object has the `paused` annotation.
-func HasPausedAnnotation(o metav1.Object) bool {
-	annotations := o.GetAnnotations()
-	if annotations == nil {
-		return false
-	}
-	_, ok := annotations[clusterv1.PausedAnnotation]
-	return ok
-}
diff --git a/util/kubeconfig/kubeconfig.go b/util/kubeconfig/kubeconfig.go
index a5ffc11f2..32a51d7a2 100644
--- a/util/kubeconfig/kubeconfig.go
+++ b/util/kubeconfig/kubeconfig.go
@@ -40,7 +40,7 @@ var (
 )
 
 // FromSecret fetches the Kubeconfig for a Cluster.
-func FromSecret(ctx context.Context, c client.Reader, cluster client.ObjectKey) ([]byte, error) {
+func FromSecret(ctx context.Context, c client.Client, cluster client.ObjectKey) ([]byte, error) {
 	out, err := secret.Get(ctx, c, cluster, secret.Kubeconfig)
 	if err != nil {
 		return nil, err
diff --git a/util/predicates/cluster_predicates.go b/util/predicates/cluster_predicates.go
deleted file mode 100644
index 06b76ee96..000000000
--- a/util/predicates/cluster_predicates.go
+++ /dev/null
@@ -1,195 +0,0 @@
-/*
-Copyright 2020 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package predicates
-
-import (
-	"github.com/go-logr/logr"
-	clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3"
-	"sigs.k8s.io/controller-runtime/pkg/event"
-	"sigs.k8s.io/controller-runtime/pkg/predicate"
-)
-
-// ClusterCreateInfraReady returns a predicate that returns true for a create event when a cluster has Status.InfrastructureReady set as true
-// it also returns true if the resource provided is not a Cluster to allow for use with controller-runtime NewControllerManagedBy
-func ClusterCreateInfraReady(logger logr.Logger) predicate.Funcs {
-	log := logger.WithValues("predicate", "ClusterCreateInfraReady")
-	return predicate.Funcs{
-		CreateFunc: func(e event.CreateEvent) bool {
-			log = log.WithValues("eventType", "create")
-
-			c, ok := e.Object.(*clusterv1.Cluster)
-			if !ok {
-
-				log.V(4).Info("Expected Cluster", "type", e.Object.GetObjectKind().GroupVersionKind().String())
-				return false
-			}
-			log = log.WithValues("namespace", c.Namespace, "cluster", c.Name)
-
-			// Only need to trigger a reconcile if the Cluster.Status.InfrastructureReady is true
-			if c.Status.InfrastructureReady {
-				log.V(4).Info("Cluster infrastructure is ready, allowing further processing")
-				return true
-			}
-
-			log.V(4).Info("Cluster infrastructure is not ready, blocking further processing")
-			return false
-		},
-		UpdateFunc:  func(e event.UpdateEvent) bool { return false },
-		DeleteFunc:  func(e event.DeleteEvent) bool { return false },
-		GenericFunc: func(e event.GenericEvent) bool { return false },
-	}
-}
-
-// ClusterCreateNotPaused returns a predicate that returns true for a create event when a cluster has Spec.Paused set as false
-// it also returns true if the resource provided is not a Cluster to allow for use with controller-runtime NewControllerManagedBy
-func ClusterCreateNotPaused(logger logr.Logger) predicate.Funcs {
-	log := logger.WithValues("predicate", "ClusterCreateNotPaused")
-	return predicate.Funcs{
-		CreateFunc: func(e event.CreateEvent) bool {
-			log = log.WithValues("eventType", "create")
-
-			c, ok := e.Object.(*clusterv1.Cluster)
-			if !ok {
-
-				log.V(4).Info("Expected Cluster", "type", e.Object.GetObjectKind().GroupVersionKind().String())
-				return false
-			}
-			log = log.WithValues("namespace", c.Namespace, "cluster", c.Name)
-
-			// Only need to trigger a reconcile if the Cluster.Spec.Paused is false
-			if !c.Spec.Paused {
-				log.V(4).Info("Cluster is not paused, allowing further processing")
-				return true
-			}
-
-			log.V(4).Info("Cluster is not paused, blocking further processing")
-			return false
-		},
-		UpdateFunc:  func(e event.UpdateEvent) bool { return false },
-		DeleteFunc:  func(e event.DeleteEvent) bool { return false },
-		GenericFunc: func(e event.GenericEvent) bool { return false },
-	}
-}
-
-// ClusterUpdateInfraReady returns a predicate that returns true for an update event when a cluster has Status.InfrastructureReady changed from false to true
-// it also returns true if the resource provided is not a Cluster to allow for use with controller-runtime NewControllerManagedBy
-func ClusterUpdateInfraReady(logger logr.Logger) predicate.Funcs {
-	log := logger.WithValues("predicate", "ClusterUpdateInfraReady")
-	return predicate.Funcs{
-		UpdateFunc: func(e event.UpdateEvent) bool {
-			log = log.WithValues("eventType", "update")
-
-			oldCluster, ok := e.ObjectOld.(*clusterv1.Cluster)
-			if !ok {
-
-				log.V(4).Info("Expected Cluster", "type", e.ObjectOld.GetObjectKind().GroupVersionKind().String())
-				return false
-			}
-			log = log.WithValues("namespace", oldCluster.Namespace, "cluster", oldCluster.Name)
-
-			newCluster := e.ObjectNew.(*clusterv1.Cluster)
-
-			if !oldCluster.Status.InfrastructureReady && newCluster.Status.InfrastructureReady {
-				log.V(4).Info("Cluster infrastructure became ready, allowing further processing")
-				return true
-			}
-
-			log.V(4).Info("Cluster infrastructure did not become ready, blocking further processing")
-			return false
-		},
-		CreateFunc:  func(e event.CreateEvent) bool { return false },
-		DeleteFunc:  func(e event.DeleteEvent) bool { return false },
-		GenericFunc: func(e event.GenericEvent) bool { return false },
-	}
-}
-
-// ClusterUpdateUnpaused returns a predicate that returns true for an update event when a cluster has Spec.Paused changed from true to false
-// it also returns true if the resource provided is not a Cluster to allow for use with controller-runtime NewControllerManagedBy
-func ClusterUpdateUnpaused(logger logr.Logger) predicate.Funcs {
-	log := logger.WithValues("predicate", "ClusterUpdateUnpaused")
-	return predicate.Funcs{
-		UpdateFunc: func(e event.UpdateEvent) bool {
-			log = log.WithValues("eventType", "update")
-
-			oldCluster, ok := e.ObjectOld.(*clusterv1.Cluster)
-			if !ok {
-
-				log.V(4).Info("Expected Cluster", "type", e.ObjectOld.GetObjectKind().GroupVersionKind().String())
-				return false
-			}
-			log = log.WithValues("namespace", oldCluster.Namespace, "cluster", oldCluster.Name)
-
-			newCluster := e.ObjectNew.(*clusterv1.Cluster)
-
-			if oldCluster.Spec.Paused && !newCluster.Spec.Paused {
-				log.V(4).Info("Cluster was unpaused, allowing further processing")
-				return true
-			}
-
-			log.V(4).Info("Cluster was not unpaused, blocking further processing")
-			return false
-		},
-		CreateFunc:  func(e event.CreateEvent) bool { return false },
-		DeleteFunc:  func(e event.DeleteEvent) bool { return false },
-		GenericFunc: func(e event.GenericEvent) bool { return false },
-	}
-}
-
-// ClusterUnpaused returns a Predicate that returns true on Cluster creation events where Cluster.Spec.Paused is false
-// and Update events when Cluster.Spec.Paused transitions to false.
-// This implements a common requirement for many cluster-api and provider controllers (such as Cluster Infrastructure
-// controllers) to resume reconciliation when the Cluster is unpaused.
-// Example use:
-//  err := controller.Watch(
-//      &source.Kind{Type: &clusterv1.Cluster{}},
-//      &handler.EnqueueRequestsFromMapFunc{
-//          ToRequests: clusterToMachines,
-//      },
-//      predicates.ClusterUnpaused(r.Log),
-//  )
-func ClusterUnpaused(logger logr.Logger) predicate.Funcs {
-	log := logger.WithValues("predicate", "ClusterUnpaused")
-
-	// Use any to ensure we process either create or update events we care about
-	return Any(log, ClusterCreateNotPaused(log), ClusterUpdateUnpaused(log))
-}
-
-// ClusterUnpausedAndInfrastructureReady returns a Predicate that returns true on Cluster creation events where
-// both Cluster.Spec.Paused is false and Cluster.Status.InfrastructureReady is true and Update events when
-// either Cluster.Spec.Paused transitions to false or Cluster.Status.InfrastructureReady transitions to true.
-// This implements a common requirement for some cluster-api and provider controllers (such as Machine Infrastructure
-// controllers) to resume reconciliation when the Cluster is unpaused and when the infrastructure becomes ready.
-// Example use:
-//  err := controller.Watch(
-//      &source.Kind{Type: &clusterv1.Cluster{}},
-//      &handler.EnqueueRequestsFromMapFunc{
-//          ToRequests: clusterToMachines,
-//      },
-//      predicates.ClusterUnpausedAndInfrastructureReady(r.Log),
-//  )
-func ClusterUnpausedAndInfrastructureReady(logger logr.Logger) predicate.Funcs {
-	log := logger.WithValues("predicate", "ClusterUnpausedAndInfrastructureReady")
-
-	// Only continue processing create events if both not paused and infrastructure is ready
-	createPredicates := All(log, ClusterCreateNotPaused(log), ClusterCreateInfraReady(log))
-
-	// Process update events if either Cluster is unpaused or infrastructure becomes ready
-	updatePredicates := Any(log, ClusterUpdateUnpaused(log), ClusterUpdateInfraReady(log))
-
-	// Use any to ensure we process either create or update events we care about
-	return Any(log, createPredicates, updatePredicates)
-}
diff --git a/util/predicates/generic_predicates.go b/util/predicates/generic_predicates.go
deleted file mode 100644
index 6fac48261..000000000
--- a/util/predicates/generic_predicates.go
+++ /dev/null
@@ -1,164 +0,0 @@
-/*
-Copyright 2020 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package predicates
-
-import (
-	"strings"
-
-	"github.com/go-logr/logr"
-	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/runtime"
-	"sigs.k8s.io/cluster-api/util/annotations"
-
-	"sigs.k8s.io/controller-runtime/pkg/event"
-	"sigs.k8s.io/controller-runtime/pkg/predicate"
-)
-
-// All returns a predicate that returns true only if all given predicates return true
-func All(logger logr.Logger, predicates ...predicate.Funcs) predicate.Funcs {
-	log := logger.WithValues("predicateAggregation", "All")
-	return predicate.Funcs{
-		UpdateFunc: func(e event.UpdateEvent) bool {
-			for _, p := range predicates {
-				if !p.UpdateFunc(e) {
-					log.V(4).Info("One of the provided predicates returned false, blocking further processing")
-					return false
-				}
-			}
-			log.V(4).Info("All provided predicates returned true, allowing further processing")
-			return true
-		},
-		CreateFunc: func(e event.CreateEvent) bool {
-			for _, p := range predicates {
-				if !p.CreateFunc(e) {
-					log.V(4).Info("One of the provided predicates returned false, blocking further processing")
-					return false
-				}
-			}
-			log.V(4).Info("All provided predicates returned true, allowing further processing")
-			return true
-		},
-		DeleteFunc: func(e event.DeleteEvent) bool {
-			for _, p := range predicates {
-				if !p.DeleteFunc(e) {
-					log.V(4).Info("One of the provided predicates returned false, blocking further processing")
-					return false
-				}
-			}
-			log.V(4).Info("All provided predicates returned true, allowing further processing")
-			return true
-		},
-		GenericFunc: func(e event.GenericEvent) bool {
-			for _, p := range predicates {
-				if !p.GenericFunc(e) {
-					log.V(4).Info("One of the provided predicates returned false, blocking further processing")
-					return false
-				}
-			}
-			log.V(4).Info("All provided predicates returned true, allowing further processing")
-			return true
-		},
-	}
-}
-
-// Any returns a predicate that returns true only if any given predicate returns true
-func Any(logger logr.Logger, predicates ...predicate.Funcs) predicate.Funcs {
-	log := logger.WithValues("predicateAggregation", "Any")
-	return predicate.Funcs{
-		UpdateFunc: func(e event.UpdateEvent) bool {
-			for _, p := range predicates {
-				if p.UpdateFunc(e) {
-					log.V(4).Info("One of the provided predicates returned true, allowing further processing")
-					return true
-				}
-			}
-			log.V(4).Info("All of the provided predicates returned false, blocking further processing")
-			return false
-		},
-		CreateFunc: func(e event.CreateEvent) bool {
-			for _, p := range predicates {
-				if p.CreateFunc(e) {
-					log.V(4).Info("One of the provided predicates returned true, allowing further processing")
-					return true
-				}
-			}
-			log.V(4).Info("All of the provided predicates returned false, blocking further processing")
-			return false
-		},
-		DeleteFunc: func(e event.DeleteEvent) bool {
-			for _, p := range predicates {
-				if p.DeleteFunc(e) {
-					log.V(4).Info("One of the provided predicates returned true, allowing further processing")
-					return true
-				}
-			}
-			log.V(4).Info("All of the provided predicates returned false, blocking further processing")
-			return false
-		},
-		GenericFunc: func(e event.GenericEvent) bool {
-			for _, p := range predicates {
-				if p.GenericFunc(e) {
-					log.V(4).Info("One of the provided predicates returned true, allowing further processing")
-					return true
-				}
-			}
-			log.V(4).Info("All of the provided predicates returned false, blocking further processing")
-			return false
-		},
-	}
-}
-
-// ResourceNotPaused returns a Predicate that returns true only if the provided resource does not contain the
-// paused annotation.
-// This implements a common requirement for all cluster-api and provider controllers skip reconciliation when the paused
-// annotation is present for a resource.
-// Example use:
-//	func (r *MyReconciler) SetupWithManager(mgr ctrl.Manager, options controller.Options) error {
-//		controller, err := ctrl.NewControllerManagedBy(mgr).
-//			For(&v1.MyType{}).
-//			WithOptions(options).
-//			WithEventFilter(util.ResourceNotPaused(r.Log)).
-//			Build(r)
-//		return err
-//	}
-func ResourceNotPaused(logger logr.Logger) predicate.Funcs {
-	return predicate.Funcs{
-		UpdateFunc: func(e event.UpdateEvent) bool {
-			return processIfNotPaused(logger.WithValues("predicate", "updateEvent"), e.ObjectNew, e.MetaNew)
-		},
-		CreateFunc: func(e event.CreateEvent) bool {
-			return processIfNotPaused(logger.WithValues("predicate", "createEvent"), e.Object, e.Meta)
-		},
-		DeleteFunc: func(e event.DeleteEvent) bool {
-			return processIfNotPaused(logger.WithValues("predicate", "deleteEvent"), e.Object, e.Meta)
-		},
-		GenericFunc: func(e event.GenericEvent) bool {
-			return processIfNotPaused(logger.WithValues("predicate", "genericEvent"), e.Object, e.Meta)
-		},
-	}
-}
-
-func processIfNotPaused(logger logr.Logger, obj runtime.Object, meta v1.Object) bool {
-	kind := strings.ToLower(obj.GetObjectKind().GroupVersionKind().Kind)
-	log := logger.WithValues("namespace", meta.GetNamespace(), kind, meta.GetName())
-	if annotations.HasPausedAnnotation(meta) {
-		log.V(4).Info("Resource is paused, will not attempt to map resource")
-		return false
-	}
-	log.V(4).Info("Resource is not paused, will attempt to map resource")
-	return true
-}
diff --git a/util/secret/secret.go b/util/secret/secret.go
index 5af9fd9f5..a17b3f430 100644
--- a/util/secret/secret.go
+++ b/util/secret/secret.go
@@ -28,13 +28,13 @@ import (
 
 // Get retrieves the specified Secret (if any) from the given
 // cluster name and namespace.
-func Get(ctx context.Context, c client.Reader, cluster client.ObjectKey, purpose Purpose) (*corev1.Secret, error) {
+func Get(ctx context.Context, c client.Client, cluster client.ObjectKey, purpose Purpose) (*corev1.Secret, error) {
 	return GetFromNamespacedName(ctx, c, cluster, purpose)
 }
 
 // GetFromNamespacedName retrieves the specified Secret (if any) from the given
 // cluster name and namespace.
-func GetFromNamespacedName(ctx context.Context, c client.Reader, clusterName client.ObjectKey, purpose Purpose) (*corev1.Secret, error) {
+func GetFromNamespacedName(ctx context.Context, c client.Client, clusterName client.ObjectKey, purpose Purpose) (*corev1.Secret, error) {
 	secret := &corev1.Secret{}
 	secretKey := client.ObjectKey{
 		Namespace: clusterName.Namespace,
diff --git a/util/util.go b/util/util.go
index e455b8af3..efce5c790 100644
--- a/util/util.go
+++ b/util/util.go
@@ -37,16 +37,15 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/version"
-	"k8s.io/klog/klogr"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3"
-	"sigs.k8s.io/cluster-api/util/annotations"
 	"sigs.k8s.io/cluster-api/util/container"
-	"sigs.k8s.io/cluster-api/util/predicates"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
+	"sigs.k8s.io/controller-runtime/pkg/event"
 	"sigs.k8s.io/controller-runtime/pkg/handler"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 	"sigs.k8s.io/controller-runtime/pkg/source"
 )
@@ -434,15 +433,23 @@ func HasOwner(refList []metav1.OwnerReference, apiVersion string, kinds []string
 	return false
 }
 
-var (
-	// IsPaused returns true if the Cluster is paused or the object has the `paused` annotation.
-	// Deprecated: use util/annotations/IsPaused instead
-	IsPaused = annotations.IsPaused
+// IsPaused returns true if the Cluster is paused or the object has the `paused` annotation.
+func IsPaused(cluster *clusterv1.Cluster, o metav1.Object) bool {
+	if cluster.Spec.Paused {
+		return true
+	}
+	return HasPausedAnnotation(o)
+}
 
-	// HasPausedAnnotation returns true if the object has the `paused` annotation.
-	// Deprecated: use util/annotations/HasPausedAnnotation instead
-	HasPausedAnnotation = annotations.HasPausedAnnotation
-)
+// HasPausedAnnotation returns true if the object has the `paused` annotation.
+func HasPausedAnnotation(o metav1.Object) bool {
+	annotations := o.GetAnnotations()
+	if annotations == nil {
+		return false
+	}
+	_, ok := annotations[clusterv1.PausedAnnotation]
+	return ok
+}
 
 // GetCRDWithContract retrieves a list of CustomResourceDefinitions from using controller-runtime Client,
 // filtering with the `contract` label passed in.
@@ -498,16 +505,19 @@ func (o MachinesByCreationTimestamp) Less(i, j int) bool {
 // WatchOnClusterPaused adds a conditional watch to the controlled given as input
 // that sends watch notifications on any create or delete, and only updates
 // that toggle Cluster.Spec.Cluster.
-// Deprecated: Instead add the Watch directly and use predicates.ClusterUnpaused or
-// predicates.ClusterUnpausedAndInfrastructureReady depending on your use case.
 func WatchOnClusterPaused(c controller.Controller, mapFunc handler.Mapper) error {
-	log := klogr.New().WithName("WatchOnClusterPaused")
 	return c.Watch(
 		&source.Kind{Type: &clusterv1.Cluster{}},
 		&handler.EnqueueRequestsFromMapFunc{
 			ToRequests: mapFunc,
 		},
-		predicates.ClusterUnpaused(log),
+		predicate.Funcs{
+			UpdateFunc: func(e event.UpdateEvent) bool {
+				oldCluster := e.ObjectOld.(*clusterv1.Cluster)
+				newCluster := e.ObjectNew.(*clusterv1.Cluster)
+				return oldCluster.Spec.Paused && !newCluster.Spec.Paused
+			},
+		},
 	)
 }
 


Test patch:
diff --git a/controlplane/kubeadm/controllers/controller_test.go b/controlplane/kubeadm/controllers/controller_test.go
index 18669f08d..f5423e8ff 100644
--- a/controlplane/kubeadm/controllers/controller_test.go
+++ b/controlplane/kubeadm/controllers/controller_test.go
@@ -525,15 +525,6 @@ kubernetesVersion: metav1.16.1`,
 				Status: internal.ClusterStatus{},
 			},
 		},
-		managementClusterUncached: &fakeManagementCluster{
-			Management: &internal.Management{Client: fakeClient},
-			Workload: fakeWorkloadCluster{
-				Workload: &internal.Workload{
-					Client: fakeClient,
-				},
-				Status: internal.ClusterStatus{},
-			},
-		},
 	}
 
 	result, err := r.Reconcile(ctrl.Request{NamespacedName: util.ObjectKey(kcp)})
diff --git a/controlplane/kubeadm/controllers/scale_test.go b/controlplane/kubeadm/controllers/scale_test.go
index 72fe8688d..088d37167 100644
--- a/controlplane/kubeadm/controllers/scale_test.go
+++ b/controlplane/kubeadm/controllers/scale_test.go
@@ -50,10 +50,6 @@ func TestKubeadmControlPlaneReconciler_initializeControlPlane(t *testing.T) {
 		Client:   fakeClient,
 		Log:      log.Log,
 		recorder: record.NewFakeRecorder(32),
-		managementClusterUncached: &fakeManagementCluster{
-			Management: &internal.Management{Client: fakeClient},
-			Workload:   fakeWorkloadCluster{},
-		},
 	}
 	controlPlane := &internal.ControlPlane{
 		Cluster: cluster,
@@ -105,11 +101,10 @@ func TestKubeadmControlPlaneReconciler_scaleUpControlPlane(t *testing.T) {
 		fakeClient := newFakeClient(g, initObjs...)
 
 		r := &KubeadmControlPlaneReconciler{
-			Client:                    fakeClient,
-			managementCluster:         fmc,
-			managementClusterUncached: fmc,
-			Log:                       log.Log,
-			recorder:                  record.NewFakeRecorder(32),
+			Client:            fakeClient,
+			managementCluster: fmc,
+			Log:               log.Log,
+			recorder:          record.NewFakeRecorder(32),
 		}
 		controlPlane := &internal.ControlPlane{
 			KCP:      kcp,
@@ -161,11 +156,10 @@ func TestKubeadmControlPlaneReconciler_scaleUpControlPlane(t *testing.T) {
 			}
 
 			r := &KubeadmControlPlaneReconciler{
-				Client:                    fakeClient,
-				managementCluster:         fmc,
-				managementClusterUncached: fmc,
-				Log:                       log.Log,
-				recorder:                  record.NewFakeRecorder(32),
+				Client:            fakeClient,
+				managementCluster: fmc,
+				Log:               log.Log,
+				recorder:          record.NewFakeRecorder(32),
 			}
 			controlPlane := &internal.ControlPlane{
 				KCP:      kcp,
diff --git a/controlplane/kubeadm/controllers/upgrade_test.go b/controlplane/kubeadm/controllers/upgrade_test.go
index 7dbd859c0..5136054e5 100644
--- a/controlplane/kubeadm/controllers/upgrade_test.go
+++ b/controlplane/kubeadm/controllers/upgrade_test.go
@@ -53,12 +53,6 @@ func TestKubeadmControlPlaneReconciler_upgradeControlPlane(t *testing.T) {
 			ControlPlaneHealthy: true,
 			EtcdHealthy:         true,
 		},
-		managementClusterUncached: &fakeManagementCluster{
-			Management:          &internal.Management{Client: fakeClient},
-			Workload:            fakeWorkloadCluster{Status: internal.ClusterStatus{Nodes: 1}},
-			ControlPlaneHealthy: true,
-			EtcdHealthy:         true,
-		},
 	}
 	controlPlane := &internal.ControlPlane{
 		KCP:      kcp,
diff --git a/test/infrastructure/docker/controllers/dockercluster_controller.go b/test/infrastructure/docker/controllers/dockercluster_controller.go
index 4c0192aeb..2871a095a 100644
--- a/test/infrastructure/docker/controllers/dockercluster_controller.go
+++ b/test/infrastructure/docker/controllers/dockercluster_controller.go
@@ -27,7 +27,6 @@ import (
 	"sigs.k8s.io/cluster-api/test/infrastructure/docker/docker"
 	"sigs.k8s.io/cluster-api/util"
 	"sigs.k8s.io/cluster-api/util/patch"
-	"sigs.k8s.io/cluster-api/util/predicates"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
@@ -151,18 +150,13 @@ func reconcileDelete(ctx context.Context, dockerCluster *infrav1.DockerCluster,
 
 // SetupWithManager will add watches for this controller
 func (r *DockerClusterReconciler) SetupWithManager(mgr ctrl.Manager) error {
-	c, err := ctrl.NewControllerManagedBy(mgr).
+	return ctrl.NewControllerManagedBy(mgr).
 		For(&infrav1.DockerCluster{}).
-		WithEventFilter(predicates.ResourceNotPaused(r.Log)).
-		Build(r)
-	if err != nil {
-		return err
-	}
-	return c.Watch(
-		&source.Kind{Type: &clusterv1.Cluster{}},
-		&handler.EnqueueRequestsFromMapFunc{
-			ToRequests: util.ClusterToInfrastructureMapFunc(infrav1.GroupVersion.WithKind("DockerCluster")),
-		},
-		predicates.ClusterUnpaused(r.Log),
-	)
+		Watches(
+			&source.Kind{Type: &clusterv1.Cluster{}},
+			&handler.EnqueueRequestsFromMapFunc{
+				ToRequests: util.ClusterToInfrastructureMapFunc(infrav1.GroupVersion.WithKind("DockerCluster")),
+			},
+		).
+		Complete(r)
 }
diff --git a/test/infrastructure/docker/controllers/dockermachine_controller.go b/test/infrastructure/docker/controllers/dockermachine_controller.go
index ad8d5c590..6a8ba5a26 100644
--- a/test/infrastructure/docker/controllers/dockermachine_controller.go
+++ b/test/infrastructure/docker/controllers/dockermachine_controller.go
@@ -31,7 +31,6 @@ import (
 	"sigs.k8s.io/cluster-api/test/infrastructure/docker/docker"
 	"sigs.k8s.io/cluster-api/util"
 	"sigs.k8s.io/cluster-api/util/patch"
-	"sigs.k8s.io/cluster-api/util/predicates"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
@@ -153,7 +152,7 @@ func (r *DockerMachineReconciler) Reconcile(req ctrl.Request) (_ ctrl.Result, re
 	return r.reconcileNormal(ctx, machine, dockerMachine, externalMachine, externalLoadBalancer, log)
 }
 
-func (r *DockerMachineReconciler) reconcileNormal(ctx context.Context, machine *clusterv1.Machine, dockerMachine *infrav1.DockerMachine, externalMachine *docker.Machine, externalLoadBalancer *docker.LoadBalancer, log logr.Logger) (ctrl.Result, error) {
+func (r *DockerMachineReconciler) reconcileNormal(ctx context.Context, machine *clusterv1.Machine, dockerMachine *infrav1.DockerMachine, externalMachine *docker.Machine, externalLoadBalancer *docker.LoadBalancer, log logr.Logger) (res ctrl.Result, retErr error) {
 	// If the DockerMachine doesn't have finalizer, add it.
 	controllerutil.AddFinalizer(dockerMachine, infrav1.MachineFinalizer)
 
@@ -177,6 +176,19 @@ func (r *DockerMachineReconciler) reconcileNormal(ctx context.Context, machine *
 		role = constants.ControlPlaneNodeRoleValue
 	}
 
+	// Defining a cleanup func that will delete a machine when there are error during provisioning, so the operation
+	// can be re-tried from a clean state when the next reconcile happens (in 10 seconds)
+	defer func() {
+		if retErr != nil && !dockerMachine.Spec.Bootstrapped {
+			log.Info(fmt.Sprintf("%v, cleaning up so we can re-provision from a clean state", retErr))
+			if err := externalMachine.Delete(ctx); err != nil {
+				log.Info("Failed to cleanup machine")
+			}
+			res = ctrl.Result{RequeueAfter: 10 * time.Second}
+			retErr = nil
+		}
+	}()
+
 	if err := externalMachine.Create(ctx, role, machine.Spec.Version, dockerMachine.Spec.ExtraMounts); err != nil {
 		return ctrl.Result{}, errors.Wrap(err, "failed to create worker DockerMachine")
 	}
@@ -252,15 +264,8 @@ func (r *DockerMachineReconciler) reconcileDelete(ctx context.Context, machine *
 
 // SetupWithManager will add watches for this controller
 func (r *DockerMachineReconciler) SetupWithManager(mgr ctrl.Manager, options controller.Options) error {
-	clusterToDockerMachines, err := util.ClusterToObjectsMapper(mgr.GetClient(), &infrav1.DockerMachineList{}, mgr.GetScheme())
-	if err != nil {
-		return err
-	}
-
-	c, err := ctrl.NewControllerManagedBy(mgr).
+	return ctrl.NewControllerManagedBy(mgr).
 		For(&infrav1.DockerMachine{}).
-		WithOptions(options).
-		WithEventFilter(predicates.ResourceNotPaused(r.Log)).
 		Watches(
 			&source.Kind{Type: &clusterv1.Machine{}},
 			&handler.EnqueueRequestsFromMapFunc{
@@ -273,17 +278,8 @@ func (r *DockerMachineReconciler) SetupWithManager(mgr ctrl.Manager, options con
 				ToRequests: handler.ToRequestsFunc(r.DockerClusterToDockerMachines),
 			},
 		).
-		Build(r)
-	if err != nil {
-		return err
-	}
-	return c.Watch(
-		&source.Kind{Type: &clusterv1.Cluster{}},
-		&handler.EnqueueRequestsFromMapFunc{
-			ToRequests: clusterToDockerMachines,
-		},
-		predicates.ClusterUnpausedAndInfrastructureReady(r.Log),
-	)
+		WithOptions(options).
+		Complete(r)
 }
 
 // DockerClusterToDockerMachines is a handler.ToRequestsFunc to be used to enqeue
diff --git a/test/infrastructure/docker/docker/machine.go b/test/infrastructure/docker/docker/machine.go
index f7987ef40..472e0a1be 100644
--- a/test/infrastructure/docker/docker/machine.go
+++ b/test/infrastructure/docker/docker/machine.go
@@ -142,7 +142,7 @@ func (m *Machine) Create(ctx context.Context, role string, version *string, moun
 		}
 		// After creating a node we need to wait a small amount of time until crictl does not return an error.
 		// This fixes an issue where we try to kubeadm init too quickly after creating the container.
-		err = wait.PollImmediate(500*time.Millisecond, 2*time.Second, func() (bool, error) {
+		err = wait.PollImmediate(500*time.Millisecond, 4*time.Second, func() (bool, error) {
 			ps := m.container.Commander.Command("crictl", "ps")
 			return ps.Run(ctx) == nil, nil
 		})
diff --git a/test/infrastructure/docker/go.sum b/test/infrastructure/docker/go.sum
index 261f6c684..99a7cb227 100644
--- a/test/infrastructure/docker/go.sum
+++ b/test/infrastructure/docker/go.sum
@@ -161,6 +161,8 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU
 github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -205,12 +207,16 @@ github.com/hashicorp/golang-lru v0.0.0-20180201235237-0fb14efe8c47/go.mod h1:/m3
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
+github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28=
 github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
+github.com/imdario/mergo v0.3.8 h1:CGgOkSJeqMRmt0D9XLWExdT4m4F1vd3FV3VPt+0VxkQ=
+github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jimstudt/http-authentication v0.0.0-20140401203705-3eca13d6893a/go.mod h1:wK6yTYYcgjHE1Z1QtXACPDjcFJyBskHEdagmnq3vsP8=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
@@ -397,8 +403,9 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073 h1:xMPOj6Pz6UipU1wXLkrtqpHbR0AVFnyPEQq/wRWz9lM=
+golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -474,6 +481,8 @@ golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxb
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=