Repository: kubernetes-sigs/cluster-api
Pull request #4570 (2021-05-04 22:01:30)

🐛 fix clusterctl config cluster regression
**What this PR does / why we need it**:
This PR fixes a clusterctl regression preventing to use the clusterctl library for clusterctl config cluster before the management cluster is created or when the management cluster is not yet initialised. 

**Which issue(s) this PR fixes**:
Fixes https://github.com/kubernetes-sigs/cluster-api/issues/4569

/assign @vincepri 

k8s-ci-robot: [APPROVALNOTIFIER] This PR is **NOT APPROVED**

This pull-request has been approved by:
To complete the [pull request process](https://git.k8s.io/community/contributors/guide/owners.md#the-code-review-process), please ask for approval from **fabriziopandini** after the PR has been reviewed.

The full list of commands accepted by this bot can be found [here](https://go.k8s.io/bot-commands?repo=kubernetes-sigs%2Fcluster-api).

<details open>
Needs approval from an approver in each of these files:

- **[cmd/clusterctl/OWNERS](https://github.com/kubernetes-sigs/cluster-api/blob/master/cmd/clusterctl/OWNERS)**

Approvers can indicate their approval by writing `/approve` in a comment
Approvers can cancel approval by writing `/approve cancel` in a comment
</details>
<!-- META={"approvers":["fabriziopandini"]} -->

Patch:
diff --git a/cmd/clusterctl/client/config.go b/cmd/clusterctl/client/config.go
index 6f599e245..c6347f804 100644
--- a/cmd/clusterctl/client/config.go
+++ b/cmd/clusterctl/client/config.go
@@ -223,14 +223,14 @@ func (c *clusterctlClient) GetClusterTemplate(options GetClusterTemplateOptions)
 	}
 
 	// Gets  the client for the current management cluster
-	cluster, err := c.clusterClientFactory(ClusterClientFactoryInput{options.Kubeconfig, options.YamlProcessor})
+	clusterClient, err := c.clusterClientFactory(ClusterClientFactoryInput{options.Kubeconfig, options.YamlProcessor})
 	if err != nil {
 		return nil, err
 	}
 
 	// If the option specifying the targetNamespace is empty, try to detect it.
 	if options.TargetNamespace == "" {
-		currentNamespace, err := cluster.Proxy().CurrentNamespace()
+		currentNamespace, err := clusterClient.Proxy().CurrentNamespace()
 		if err != nil {
 			return nil, err
 		}
@@ -248,16 +248,21 @@ func (c *clusterctlClient) GetClusterTemplate(options GetClusterTemplateOptions)
 	// Gets the workload cluster template from the selected source
 	if options.ProviderRepositorySource != nil {
 		// Ensure this command only runs against management clusters with the current Cluster API contract.
-		if err := cluster.ProviderInventory().CheckCAPIContract(); err != nil {
-			return nil, err
+		// NOTE: This command tolerates also not existing cluster (Kubeconfig.Path=="") or clusters not yet initialized in order to allow
+		// users to dry-run the command and take a look at what the cluster will look like; in both scenarios, it is required
+		// to pass provider:version given that auto-discovery can't work without a provider inventory installed in a cluster.
+		if options.Kubeconfig.Path != "" {
+			if err := clusterClient.ProviderInventory().CheckCAPIContract(cluster.AllowCAPINotInstalled{}); err != nil {
+				return nil, err
+			}
 		}
-		return c.getTemplateFromRepository(cluster, options)
+		return c.getTemplateFromRepository(clusterClient, options)
 	}
 	if options.ConfigMapSource != nil {
-		return c.getTemplateFromConfigMap(cluster, *options.ConfigMapSource, options.TargetNamespace, options.ListVariablesOnly)
+		return c.getTemplateFromConfigMap(clusterClient, *options.ConfigMapSource, options.TargetNamespace, options.ListVariablesOnly)
 	}
 	if options.URLSource != nil {
-		return c.getTemplateFromURL(cluster, *options.URLSource, options.TargetNamespace, options.ListVariablesOnly)
+		return c.getTemplateFromURL(clusterClient, *options.URLSource, options.TargetNamespace, options.ListVariablesOnly)
 	}
 
 	return nil, errors.New("unable to read custom template. Please specify a template source")
diff --git a/controllers/machine_controller.go b/controllers/machine_controller.go
index 1e7a44d61..cc7466fef 100644
--- a/controllers/machine_controller.go
+++ b/controllers/machine_controller.go
@@ -19,6 +19,7 @@ package controllers
 import (
 	"context"
 	"fmt"
+	"sigs.k8s.io/cluster-api/util/collections"
 	"time"
 
 	"github.com/pkg/errors"
@@ -39,7 +40,6 @@ import (
 	kubedrain "sigs.k8s.io/cluster-api/third_party/kubernetes-drain"
 	"sigs.k8s.io/cluster-api/util"
 	"sigs.k8s.io/cluster-api/util/annotations"
-	"sigs.k8s.io/cluster-api/util/collections"
 	"sigs.k8s.io/cluster-api/util/conditions"
 	"sigs.k8s.io/cluster-api/util/patch"
 	"sigs.k8s.io/cluster-api/util/predicates"
diff --git a/controllers/machinedeployment_rolling.go b/controllers/machinedeployment_rolling.go
index 1694b3268..b64c14404 100644
--- a/controllers/machinedeployment_rolling.go
+++ b/controllers/machinedeployment_rolling.go
@@ -28,7 +28,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
-// rolloutRolling implements the logic for rolling a new MachineSet.
+// rolloutRolling implements the logic for rolling a new machine set.
 func (r *MachineDeploymentReconciler) rolloutRolling(ctx context.Context, d *clusterv1.MachineDeployment, msList []*clusterv1.MachineSet) error {
 	newMS, oldMSs, err := r.getAllMachineSetsAndSyncRevision(ctx, d, msList, true)
 	if err != nil {
@@ -73,11 +73,11 @@ func (r *MachineDeploymentReconciler) rolloutRolling(ctx context.Context, d *clu
 
 func (r *MachineDeploymentReconciler) reconcileNewMachineSet(ctx context.Context, allMSs []*clusterv1.MachineSet, newMS *clusterv1.MachineSet, deployment *clusterv1.MachineDeployment) error {
 	if deployment.Spec.Replicas == nil {
-		return errors.Errorf("spec.replicas for MachineDeployment %v is nil, this is unexpected", client.ObjectKeyFromObject(deployment))
+		return errors.Errorf("spec replicas for MachineDeployment %s is nil, this is unexpected", client.ObjectKeyFromObject(deployment).String())
 	}
 
 	if newMS.Spec.Replicas == nil {
-		return errors.Errorf("spec.replicas for MachineSet %v is nil, this is unexpected", client.ObjectKeyFromObject(newMS))
+		return errors.Errorf("spec replicas for MachineSet %s is nil, this is unexpected", client.ObjectKeyFromObject(newMS).String())
 	}
 
 	if *(newMS.Spec.Replicas) == *(deployment.Spec.Replicas) {
@@ -101,13 +101,13 @@ func (r *MachineDeploymentReconciler) reconcileOldMachineSets(ctx context.Contex
 	log := ctrl.LoggerFrom(ctx)
 
 	if deployment.Spec.Replicas == nil {
-		return errors.Errorf("spec.replicas for MachineDeployment %v is nil, this is unexpected",
-			client.ObjectKeyFromObject(deployment))
+		return errors.Errorf("spec replicas for MachineDeployment %q/%q is nil, this is unexpected",
+			deployment.Namespace, deployment.Name)
 	}
 
 	if newMS.Spec.Replicas == nil {
-		return errors.Errorf("spec.replicas for MachineSet %v is nil, this is unexpected",
-			client.ObjectKeyFromObject(newMS))
+		return errors.Errorf("spec replicas for MachineSet %q/%q is nil, this is unexpected",
+			newMS.Namespace, newMS.Name)
 	}
 
 	oldMachinesCount := mdutil.GetReplicaCountForMachineSets(oldMSs)
@@ -117,18 +117,18 @@ func (r *MachineDeploymentReconciler) reconcileOldMachineSets(ctx context.Contex
 	}
 
 	allMachinesCount := mdutil.GetReplicaCountForMachineSets(allMSs)
-	log.V(4).Info("New MachineSet has available machines",
-		"machineset", client.ObjectKeyFromObject(newMS).String(), "available-replicas", newMS.Status.AvailableReplicas)
+	log.V(4).Info("New machine set has available machines",
+		"machineset", newMS.Name, "count", newMS.Status.AvailableReplicas)
 	maxUnavailable := mdutil.MaxUnavailable(*deployment)
 
 	// Check if we can scale down. We can scale down in the following 2 cases:
-	// * Some old MachineSets have unhealthy replicas, we could safely scale down those unhealthy replicas since that won't further
+	// * Some old machine sets have unhealthy replicas, we could safely scale down those unhealthy replicas since that won't further
 	//  increase unavailability.
-	// * New MachineSet has scaled up and it's replicas becomes ready, then we can scale down old MachineSets in a further step.
+	// * New machine set has scaled up and it's replicas becomes ready, then we can scale down old machine sets in a further step.
 	//
 	// maxScaledDown := allMachinesCount - minAvailable - newMachineSetMachinesUnavailable
 	// take into account not only maxUnavailable and any surge machines that have been created, but also unavailable machines from
-	// the newMS, so that the unavailable machines from the newMS would not make us scale down old MachineSets in a further
+	// the newMS, so that the unavailable machines from the newMS would not make us scale down old machine sets in a further
 	// step(that will increase unavailability).
 	//
 	// Concrete example:
@@ -139,18 +139,18 @@ func (r *MachineDeploymentReconciler) reconcileOldMachineSets(ctx context.Contex
 	//
 	// case 1:
 	// * Deployment is updated, newMS is created with 3 replicas, oldMS is scaled down to 8, and newMS is scaled up to 5.
-	// * The new MachineSet machines crashloop and never become available.
+	// * The new machine set machines crashloop and never become available.
 	// * allMachinesCount is 13. minAvailable is 8. newMSMachinesUnavailable is 5.
 	// * A node fails and causes one of the oldMS machines to become unavailable. However, 13 - 8 - 5 = 0, so the oldMS won't be scaled down.
 	// * The user notices the crashloop and does kubectl rollout undo to rollback.
-	// * newMSMachinesUnavailable is 1, since we rolled back to the good MachineSet, so maxScaledDown = 13 - 8 - 1 = 4. 4 of the crashlooping machines will be scaled down.
+	// * newMSMachinesUnavailable is 1, since we rolled back to the good machine set, so maxScaledDown = 13 - 8 - 1 = 4. 4 of the crashlooping machines will be scaled down.
 	// * The total number of machines will then be 9 and the newMS can be scaled up to 10.
 	//
 	// case 2:
 	// Same example, but pushing a new machine template instead of rolling back (aka "roll over"):
-	// * The new MachineSet created must start with 0 replicas because allMachinesCount is already at 13.
-	// * However, newMSMachinesUnavailable would also be 0, so the 2 old MachineSets could be scaled down by 5 (13 - 8 - 0), which would then
-	// allow the new MachineSet to be scaled up by 5.
+	// * The new machine set created must start with 0 replicas because allMachinesCount is already at 13.
+	// * However, newMSMachinesUnavailable would also be 0, so the 2 old machine sets could be scaled down by 5 (13 - 8 - 0), which would then
+	// allow the new machine set to be scaled up by 5.
 	minAvailable := *(deployment.Spec.Replicas) - maxUnavailable
 	newMSUnavailableMachineCount := *(newMS.Spec.Replicas) - newMS.Status.AvailableReplicas
 	maxScaledDown := allMachinesCount - minAvailable - newMSUnavailableMachineCount
@@ -167,7 +167,7 @@ func (r *MachineDeploymentReconciler) reconcileOldMachineSets(ctx context.Contex
 
 	log.V(4).Info("Cleaned up unhealthy replicas from old MachineSets", "count", cleanupCount)
 
-	// Scale down old MachineSets, need check maxUnavailable to ensure we can scale down
+	// Scale down old machine sets, need check maxUnavailable to ensure we can scale down
 	allMSs = oldMSs
 	allMSs = append(allMSs, newMS)
 	scaledDownCount, err := r.scaleDownOldMachineSetsForRollingUpdate(ctx, allMSs, oldMSs, deployment)
@@ -175,25 +175,24 @@ func (r *MachineDeploymentReconciler) reconcileOldMachineSets(ctx context.Contex
 		return err
 	}
 
-	log.V(4).Info("Scaled down old MachineSets of MachineDeployment", "count", scaledDownCount)
+	log.V(4).Info("Scaled down old MachineSets of deployment", "count", scaledDownCount)
 	return nil
 }
 
-// cleanupUnhealthyReplicas will scale down old MachineSets with unhealthy replicas, so that all unhealthy replicas will be deleted.
+// cleanupUnhealthyReplicas will scale down old machine sets with unhealthy replicas, so that all unhealthy replicas will be deleted.
 func (r *MachineDeploymentReconciler) cleanupUnhealthyReplicas(ctx context.Context, oldMSs []*clusterv1.MachineSet, deployment *clusterv1.MachineDeployment, maxCleanupCount int32) ([]*clusterv1.MachineSet, int32, error) {
 	log := ctrl.LoggerFrom(ctx)
 
 	sort.Sort(mdutil.MachineSetsByCreationTimestamp(oldMSs))
 
-	// Scale down all old MachineSets with any unhealthy replicas. MachineSet will honour Spec.DeletePolicy
-	// for deleting Machines. Machines with a deletion timestamp, with a failure message or without a nodeRef
-	// are preferred for all strategies.
-	// This results in a best effort to remove machines backing unhealthy nodes.
+	// Safely scale down all old machine sets with unhealthy replicas. Replica set will sort the machines in the order
+	// such that not-ready < ready, unscheduled < scheduled, and pending < running. This ensures that unhealthy replicas will
+	// been deleted first and won't increase unavailability.
 	totalScaledDown := int32(0)
 
 	for _, targetMS := range oldMSs {
 		if targetMS.Spec.Replicas == nil {
-			return nil, 0, errors.Errorf("spec.replicas for MachineSet %v is nil, this is unexpected", client.ObjectKeyFromObject(targetMS))
+			return nil, 0, errors.Errorf("spec replicas for machine set %v is nil, this is unexpected", targetMS.Name)
 		}
 
 		if totalScaledDown >= maxCleanupCount {
@@ -202,13 +201,12 @@ func (r *MachineDeploymentReconciler) cleanupUnhealthyReplicas(ctx context.Conte
 
 		oldMSReplicas := *(targetMS.Spec.Replicas)
 		if oldMSReplicas == 0 {
-			// cannot scale down this MachineSet.
+			// cannot scale down this machine set.
 			continue
 		}
 
 		oldMSAvailableReplicas := targetMS.Status.AvailableReplicas
-		log.V(4).Info("Found available Machines in old MachineSet",
-			"count", oldMSAvailableReplicas, "target-machineset", client.ObjectKeyFromObject(targetMS).String())
+		log.V(4).Info("Found available machines in old MS", "count", oldMSAvailableReplicas, "target-machineset", targetMS.Name)
 		if oldMSReplicas == oldMSAvailableReplicas {
 			// no unhealthy replicas found, no scaling required.
 			continue
@@ -220,8 +218,7 @@ func (r *MachineDeploymentReconciler) cleanupUnhealthyReplicas(ctx context.Conte
 		newReplicasCount := oldMSReplicas - scaledDownCount
 
 		if newReplicasCount > oldMSReplicas {
-			return nil, 0, errors.Errorf("when cleaning up unhealthy replicas, got invalid request to scale down %v: %d -> %d",
-				client.ObjectKeyFromObject(targetMS), oldMSReplicas, newReplicasCount)
+			return nil, 0, errors.Errorf("when cleaning up unhealthy replicas, got invalid request to scale down %s/%s %d -> %d", targetMS.Namespace, targetMS.Name, oldMSReplicas, newReplicasCount)
 		}
 
 		if err := r.scaleMachineSet(ctx, targetMS, newReplicasCount, deployment); err != nil {
@@ -234,22 +231,22 @@ func (r *MachineDeploymentReconciler) cleanupUnhealthyReplicas(ctx context.Conte
 	return oldMSs, totalScaledDown, nil
 }
 
-// scaleDownOldMachineSetsForRollingUpdate scales down old MachineSets when deployment strategy is "RollingUpdate".
+// scaleDownOldMachineSetsForRollingUpdate scales down old machine sets when deployment strategy is "RollingUpdate".
 // Need check maxUnavailable to ensure availability.
 func (r *MachineDeploymentReconciler) scaleDownOldMachineSetsForRollingUpdate(ctx context.Context, allMSs []*clusterv1.MachineSet, oldMSs []*clusterv1.MachineSet, deployment *clusterv1.MachineDeployment) (int32, error) {
 	log := ctrl.LoggerFrom(ctx)
 
 	if deployment.Spec.Replicas == nil {
-		return 0, errors.Errorf("spec.replicas for MachineDeployment %v is nil, this is unexpected", client.ObjectKeyFromObject(deployment))
+		return 0, errors.Errorf("spec replicas for deployment %v is nil, this is unexpected", deployment.Name)
 	}
 
 	maxUnavailable := mdutil.MaxUnavailable(*deployment)
+
+	// Check if we can scale down.
 	minAvailable := *(deployment.Spec.Replicas) - maxUnavailable
 
 	// Find the number of available machines.
 	availableMachineCount := mdutil.GetAvailableReplicaCountForMachineSets(allMSs)
-
-	// Check if we can scale down.
 	if availableMachineCount <= minAvailable {
 		// Cannot scale down.
 		return 0, nil
@@ -263,7 +260,7 @@ func (r *MachineDeploymentReconciler) scaleDownOldMachineSetsForRollingUpdate(ct
 	totalScaleDownCount := availableMachineCount - minAvailable
 	for _, targetMS := range oldMSs {
 		if targetMS.Spec.Replicas == nil {
-			return 0, errors.Errorf("spec.replicas for MachineSet %v is nil, this is unexpected", client.ObjectKeyFromObject(targetMS))
+			return 0, errors.Errorf("spec replicas for machine set %v is nil, this is unexpected", targetMS.Name)
 		}
 
 		if totalScaledDown >= totalScaleDownCount {
@@ -280,8 +277,7 @@ func (r *MachineDeploymentReconciler) scaleDownOldMachineSetsForRollingUpdate(ct
 		scaleDownCount := integer.Int32Min(*(targetMS.Spec.Replicas), totalScaleDownCount-totalScaledDown)
 		newReplicasCount := *(targetMS.Spec.Replicas) - scaleDownCount
 		if newReplicasCount > *(targetMS.Spec.Replicas) {
-			return totalScaledDown, errors.Errorf("when scaling down old MachineSet, got invalid request to scale down %v: %d -> %d",
-				client.ObjectKeyFromObject(targetMS), *(targetMS.Spec.Replicas), newReplicasCount)
+			return totalScaledDown, errors.Errorf("when scaling down old MS, got invalid request to scale down %s/%s %d -> %d", targetMS.Namespace, targetMS.Name, *(targetMS.Spec.Replicas), newReplicasCount)
 		}
 
 		if err := r.scaleMachineSet(ctx, targetMS, newReplicasCount, deployment); err != nil {
diff --git a/go.mod b/go.mod
index 18012fb36..918a6b0ca 100644
--- a/go.mod
+++ b/go.mod
@@ -36,7 +36,7 @@ require (
 	k8s.io/klog/v2 v2.8.0
 	k8s.io/kubectl v0.21.0
 	k8s.io/utils v0.0.0-20210305010621-2afb4311ab10
-	sigs.k8s.io/controller-runtime v0.9.0-beta.1
+	sigs.k8s.io/controller-runtime v0.9.0-beta.0
 	sigs.k8s.io/kind v0.9.0
 	sigs.k8s.io/yaml v1.2.0
 )
diff --git a/go.sum b/go.sum
index 0b523486f..10f0a999a 100644
--- a/go.sum
+++ b/go.sum
@@ -1070,8 +1070,8 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg=
-sigs.k8s.io/controller-runtime v0.9.0-beta.1 h1:pmJVlNQeVMtkszjrRP59XT55Ny7+1it7ri3yyukwYTE=
-sigs.k8s.io/controller-runtime v0.9.0-beta.1/go.mod h1:ufPDuvefw2Y1KnBgHQrLdOjueYlj+XJV2AszbT+WTxs=
+sigs.k8s.io/controller-runtime v0.9.0-beta.0 h1:GMy39zuf8ywrQyNuc24wR1mdiJCSGJnxaaqakZldaug=
+sigs.k8s.io/controller-runtime v0.9.0-beta.0/go.mod h1:ufPDuvefw2Y1KnBgHQrLdOjueYlj+XJV2AszbT+WTxs=
 sigs.k8s.io/kind v0.9.0 h1:SoDlXq6pEc7dGagHULNRCCBYrLH6xOi7lqXTRXeAlg4=
 sigs.k8s.io/kind v0.9.0/go.mod h1:cxKQWwmbtRDzQ+RNKnR6gZG6fjbeTtItp5cGf+ww+1Y=
 sigs.k8s.io/kustomize/api v0.8.5/go.mod h1:M377apnKT5ZHJS++6H4rQoCHmWtt6qTpp3mbe7p6OLY=


Test patch:
diff --git a/cmd/clusterctl/client/config_test.go b/cmd/clusterctl/client/config_test.go
index 67a23f44f..78baf0c32 100644
--- a/cmd/clusterctl/client/config_test.go
+++ b/cmd/clusterctl/client/config_test.go
@@ -25,14 +25,14 @@ import (
 
 	. "github.com/onsi/gomega"
 	"github.com/pkg/errors"
-	"sigs.k8s.io/cluster-api/cmd/clusterctl/internal/test"
-
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/utils/pointer"
 	clusterctlv1 "sigs.k8s.io/cluster-api/cmd/clusterctl/api/v1alpha3"
 	"sigs.k8s.io/cluster-api/cmd/clusterctl/client/cluster"
 	"sigs.k8s.io/cluster-api/cmd/clusterctl/client/config"
+	"sigs.k8s.io/cluster-api/cmd/clusterctl/client/repository"
+	"sigs.k8s.io/cluster-api/cmd/clusterctl/internal/test"
 )
 
 func Test_clusterctlClient_GetProvidersConfig(t *testing.T) {
@@ -648,8 +648,14 @@ func Test_clusterctlClient_GetClusterTemplate_onEmptyCluster(t *testing.T) {
 	cluster1 := newFakeCluster(cluster.Kubeconfig{Path: "kubeconfig", Context: "mgmt-context"}, config1).
 		WithObjs(configMap)
 
+	repository1 := newFakeRepository(infraProviderConfig, config1).
+		WithPaths("root", "components").
+		WithDefaultVersion("v3.0.0").
+		WithFile("v3.0.0", "cluster-template.yaml", rawTemplate)
+
 	client := newFakeClient(config1).
-		WithCluster(cluster1)
+		WithCluster(cluster1).
+		WithRepository(repository1)
 
 	type args struct {
 		options GetClusterTemplateOptions
@@ -668,7 +674,7 @@ func Test_clusterctlClient_GetClusterTemplate_onEmptyCluster(t *testing.T) {
 		wantErr bool
 	}{
 		{
-			name: "repository source - fails because the cluster is not initialized/not v1alpha3",
+			name: "repository source - pass if the cluster is not initialized but infra provider:version are specified",
 			args: args{
 				options: GetClusterTemplateOptions{
 					Kubeconfig: Kubeconfig{Path: "kubeconfig", Context: "mgmt-context"},
@@ -681,6 +687,26 @@ func Test_clusterctlClient_GetClusterTemplate_onEmptyCluster(t *testing.T) {
 					ControlPlaneMachineCount: pointer.Int64Ptr(1),
 				},
 			},
+			want: templateValues{
+				variables:       []string{"CLUSTER_NAME"}, // variable detected
+				targetNamespace: "ns1",
+				yaml:            templateYAML("ns1", "test"), // original template modified with target namespace and variable replacement
+			},
+		},
+		{
+			name: "repository source - fails if the cluster is not initialized and infra provider:version are not specified",
+			args: args{
+				options: GetClusterTemplateOptions{
+					Kubeconfig: Kubeconfig{Path: "kubeconfig", Context: "mgmt-context"},
+					ProviderRepositorySource: &ProviderRepositorySourceOptions{
+						InfrastructureProvider: "",
+						Flavor:                 "",
+					},
+					ClusterName:              "test",
+					TargetNamespace:          "ns1",
+					ControlPlaneMachineCount: pointer.Int64Ptr(1),
+				},
+			},
 			wantErr: true,
 		},
 		{
@@ -745,6 +771,117 @@ func Test_clusterctlClient_GetClusterTemplate_onEmptyCluster(t *testing.T) {
 	}
 }
 
+func newFakeClientWithoutCluster(configClient config.Client) *fakeClient {
+	fake := &fakeClient{
+		configClient: configClient,
+		repositories: map[string]repository.Client{},
+	}
+
+	var err error
+	fake.internalClient, err = newClusterctlClient("fake-config",
+		InjectConfig(fake.configClient),
+		InjectRepositoryFactory(func(input RepositoryClientFactoryInput) (repository.Client, error) {
+			if _, ok := fake.repositories[input.Provider.ManifestLabel()]; !ok {
+				return nil, errors.Errorf("Repository for kubeconfig %q does not exist.", input.Provider.ManifestLabel())
+			}
+			return fake.repositories[input.Provider.ManifestLabel()], nil
+		}),
+	)
+	if err != nil {
+		panic(err)
+	}
+
+	return fake
+}
+
+func Test_clusterctlClient_GetClusterTemplate_withoutCluster(t *testing.T) {
+	rawTemplate := templateYAML("ns3", "${ CLUSTER_NAME }")
+
+	config1 := newFakeConfig().
+		WithProvider(infraProviderConfig)
+
+	repository1 := newFakeRepository(infraProviderConfig, config1).
+		WithPaths("root", "components").
+		WithDefaultVersion("v3.0.0").
+		WithFile("v3.0.0", "cluster-template.yaml", rawTemplate)
+
+	client := newFakeClientWithoutCluster(config1).
+		WithRepository(repository1)
+
+	type args struct {
+		options GetClusterTemplateOptions
+	}
+
+	type templateValues struct {
+		variables       []string
+		targetNamespace string
+		yaml            []byte
+	}
+
+	tests := []struct {
+		name    string
+		args    args
+		want    templateValues
+		wantErr bool
+	}{
+		{
+			name: "repository source - pass without kubeconfig but infra provider:version are specified",
+			args: args{
+				options: GetClusterTemplateOptions{
+					Kubeconfig: Kubeconfig{Path: "", Context: ""},
+					ProviderRepositorySource: &ProviderRepositorySourceOptions{
+						InfrastructureProvider: "infra:v3.0.0",
+						Flavor:                 "",
+					},
+					ClusterName:              "test",
+					TargetNamespace:          "ns1",
+					ControlPlaneMachineCount: pointer.Int64Ptr(1),
+				},
+			},
+			want: templateValues{
+				variables:       []string{"CLUSTER_NAME"}, // variable detected
+				targetNamespace: "ns1",
+				yaml:            templateYAML("ns1", "test"), // original template modified with target namespace and variable replacement
+			},
+		},
+		{
+			name: "repository source - fails without kubeconfig and infra provider:version are not specified",
+			args: args{
+				options: GetClusterTemplateOptions{
+					Kubeconfig: Kubeconfig{Path: "", Context: ""},
+					ProviderRepositorySource: &ProviderRepositorySourceOptions{
+						InfrastructureProvider: "",
+						Flavor:                 "",
+					},
+					ClusterName:              "test",
+					TargetNamespace:          "ns1",
+					ControlPlaneMachineCount: pointer.Int64Ptr(1),
+				},
+			},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gs := NewWithT(t)
+
+			got, err := client.GetClusterTemplate(tt.args.options)
+			if tt.wantErr {
+				gs.Expect(err).To(HaveOccurred())
+				return
+			}
+			gs.Expect(err).NotTo(HaveOccurred())
+
+			gs.Expect(got.Variables()).To(Equal(tt.want.variables))
+			gs.Expect(got.TargetNamespace()).To(Equal(tt.want.targetNamespace))
+
+			gotYaml, err := got.Yaml()
+			gs.Expect(err).NotTo(HaveOccurred())
+			gs.Expect(gotYaml).To(Equal(tt.want.yaml))
+		})
+	}
+}
+
 func Test_clusterctlClient_ProcessYAML(t *testing.T) {
 	g := NewWithT(t)
 	template := `v1: ${VAR1:=default1}
diff --git a/controllers/machinedeployment_rolling_test.go b/controllers/machinedeployment_rolling_test.go
index 0f0af8fe9..f34af73f8 100644
--- a/controllers/machinedeployment_rolling_test.go
+++ b/controllers/machinedeployment_rolling_test.go
@@ -54,7 +54,7 @@ func TestReconcileNewMachineSet(t *testing.T) {
 					Replicas: pointer.Int32Ptr(2),
 				},
 			},
-			error: errors.Errorf("spec.replicas for MachineDeployment foo/bar is nil, this is unexpected"),
+			error: errors.Errorf("spec replicas for MachineDeployment foo/bar is nil, this is unexpected"),
 		},
 		{
 			name: "It fails when new machineSet has no replicas",
@@ -69,7 +69,7 @@ func TestReconcileNewMachineSet(t *testing.T) {
 					Name:      "bar",
 				},
 			},
-			error: errors.Errorf("spec.replicas for MachineSet foo/bar is nil, this is unexpected"),
+			error: errors.Errorf("spec replicas for MachineSet foo/bar is nil, this is unexpected"),
 		},
 		{
 			name: "RollingUpdate strategy: Scale up: 0 -> 2",
@@ -197,7 +197,6 @@ func TestReconcileNewMachineSet(t *testing.T) {
 
 			err := r.reconcileNewMachineSet(ctx, allMachineSets, tc.newMachineSet, tc.machineDeployment)
 			if tc.error != nil {
-				g.Expect(err).To(HaveOccurred())
 				g.Expect(err.Error()).To(BeEquivalentTo(tc.error.Error()))
 				return
 			}
@@ -220,189 +219,3 @@ func TestReconcileNewMachineSet(t *testing.T) {
 		})
 	}
 }
-
-func TestReconcileOldMachineSets(t *testing.T) {
-	testCases := []struct {
-		name                           string
-		machineDeployment              *clusterv1.MachineDeployment
-		newMachineSet                  *clusterv1.MachineSet
-		oldMachineSets                 []*clusterv1.MachineSet
-		expectedOldMachineSetsReplicas int
-		error                          error
-	}{
-		{
-			name: "It fails when machineDeployment has no replicas",
-			machineDeployment: &clusterv1.MachineDeployment{
-				ObjectMeta: metav1.ObjectMeta{
-					Namespace: "foo",
-					Name:      "bar",
-				},
-			},
-			newMachineSet: &clusterv1.MachineSet{
-				Spec: clusterv1.MachineSetSpec{
-					Replicas: pointer.Int32Ptr(2),
-				},
-			},
-			error: errors.Errorf("spec.replicas for MachineDeployment foo/bar is nil, this is unexpected"),
-		},
-		{
-			name: "It fails when new machineSet has no replicas",
-			machineDeployment: &clusterv1.MachineDeployment{
-				Spec: clusterv1.MachineDeploymentSpec{
-					Replicas: pointer.Int32Ptr(2),
-				},
-			},
-			newMachineSet: &clusterv1.MachineSet{
-				ObjectMeta: metav1.ObjectMeta{
-					Namespace: "foo",
-					Name:      "bar",
-				},
-			},
-			error: errors.Errorf("spec.replicas for MachineSet foo/bar is nil, this is unexpected"),
-		},
-		{
-			name: "RollingUpdate strategy: Scale down old MachineSets when all new replicas are available",
-			machineDeployment: &clusterv1.MachineDeployment{
-				ObjectMeta: metav1.ObjectMeta{
-					Namespace: "foo",
-					Name:      "bar",
-				},
-				Spec: clusterv1.MachineDeploymentSpec{
-					Strategy: &clusterv1.MachineDeploymentStrategy{
-						Type: clusterv1.RollingUpdateMachineDeploymentStrategyType,
-						RollingUpdate: &clusterv1.MachineRollingUpdateDeployment{
-							MaxUnavailable: intOrStrPtr(1),
-							MaxSurge:       intOrStrPtr(3),
-						},
-					},
-					Replicas: pointer.Int32Ptr(2),
-				},
-			},
-			newMachineSet: &clusterv1.MachineSet{
-				ObjectMeta: metav1.ObjectMeta{
-					Namespace: "foo",
-					Name:      "bar",
-				},
-				Spec: clusterv1.MachineSetSpec{
-					Replicas: pointer.Int32Ptr(0),
-				},
-				Status: clusterv1.MachineSetStatus{
-					AvailableReplicas: 2,
-				},
-			},
-			oldMachineSets: []*clusterv1.MachineSet{
-				{
-					ObjectMeta: metav1.ObjectMeta{
-						Namespace: "foo",
-						Name:      "2replicas",
-					},
-					Spec: clusterv1.MachineSetSpec{
-						Replicas: pointer.Int32Ptr(2),
-					},
-					Status: clusterv1.MachineSetStatus{
-						AvailableReplicas: 2,
-					},
-				},
-				{
-					ObjectMeta: metav1.ObjectMeta{
-						Namespace: "foo",
-						Name:      "1replicas",
-					},
-					Spec: clusterv1.MachineSetSpec{
-						Replicas: pointer.Int32Ptr(1),
-					},
-					Status: clusterv1.MachineSetStatus{
-						AvailableReplicas: 1,
-					},
-				},
-			},
-			expectedOldMachineSetsReplicas: 0,
-		},
-		{
-			name: "RollingUpdate strategy: It does not scale down old MachineSets when above maxUnavailable",
-			machineDeployment: &clusterv1.MachineDeployment{
-				ObjectMeta: metav1.ObjectMeta{
-					Namespace: "foo",
-					Name:      "bar",
-				},
-				Spec: clusterv1.MachineDeploymentSpec{
-					Strategy: &clusterv1.MachineDeploymentStrategy{
-						Type: clusterv1.RollingUpdateMachineDeploymentStrategyType,
-						RollingUpdate: &clusterv1.MachineRollingUpdateDeployment{
-							MaxUnavailable: intOrStrPtr(2),
-							MaxSurge:       intOrStrPtr(3),
-						},
-					},
-					Replicas: pointer.Int32Ptr(10),
-				},
-			},
-			newMachineSet: &clusterv1.MachineSet{
-				ObjectMeta: metav1.ObjectMeta{
-					Namespace: "foo",
-					Name:      "bar",
-				},
-				Spec: clusterv1.MachineSetSpec{
-					Replicas: pointer.Int32Ptr(5),
-				},
-				Status: clusterv1.MachineSetStatus{
-					Replicas:          5,
-					ReadyReplicas:     0,
-					AvailableReplicas: 0,
-				},
-			},
-			oldMachineSets: []*clusterv1.MachineSet{
-				{
-					ObjectMeta: metav1.ObjectMeta{
-						Namespace: "foo",
-						Name:      "8replicas",
-					},
-					Spec: clusterv1.MachineSetSpec{
-						Replicas: pointer.Int32Ptr(8),
-					},
-					Status: clusterv1.MachineSetStatus{
-						Replicas:          10,
-						ReadyReplicas:     8,
-						AvailableReplicas: 8,
-					},
-				},
-			},
-			expectedOldMachineSetsReplicas: 8,
-		},
-	}
-	for _, tc := range testCases {
-		t.Run(tc.name, func(t *testing.T) {
-			g := NewWithT(t)
-
-			g.Expect(clusterv1.AddToScheme(scheme.Scheme)).To(Succeed())
-
-			resources := []client.Object{
-				tc.machineDeployment,
-			}
-
-			allMachineSets := append(tc.oldMachineSets, tc.newMachineSet)
-			for key := range allMachineSets {
-				resources = append(resources, allMachineSets[key])
-			}
-
-			r := &MachineDeploymentReconciler{
-				Client:   fake.NewClientBuilder().WithObjects(resources...).Build(),
-				recorder: record.NewFakeRecorder(32),
-			}
-
-			err := r.reconcileOldMachineSets(ctx, allMachineSets, tc.oldMachineSets, tc.newMachineSet, tc.machineDeployment)
-			if tc.error != nil {
-				g.Expect(err).To(HaveOccurred())
-				g.Expect(err.Error()).To(BeEquivalentTo(tc.error.Error()))
-				return
-			}
-
-			g.Expect(err).ToNot(HaveOccurred())
-			for key := range tc.oldMachineSets {
-				freshOldMachineSet := &clusterv1.MachineSet{}
-				err = r.Client.Get(ctx, client.ObjectKeyFromObject(tc.oldMachineSets[key]), freshOldMachineSet)
-				g.Expect(err).ToNot(HaveOccurred())
-				g.Expect(*freshOldMachineSet.Spec.Replicas).To(BeEquivalentTo(tc.expectedOldMachineSetsReplicas))
-			}
-		})
-	}
-}
diff --git a/test/e2e/config/docker.yaml b/test/e2e/config/docker.yaml
index 178e622cd..fa016b881 100644
--- a/test/e2e/config/docker.yaml
+++ b/test/e2e/config/docker.yaml
@@ -10,13 +10,13 @@
 images:
 # Use local dev images built source tree;
 - name: gcr.io/k8s-staging-cluster-api/cluster-api-controller-amd64:dev
-  loadBehavior: tryLoad
+  loadBehavior: mustLoad
 - name: gcr.io/k8s-staging-cluster-api/kubeadm-bootstrap-controller-amd64:dev
-  loadBehavior: tryLoad
+  loadBehavior: mustLoad
 - name: gcr.io/k8s-staging-cluster-api/kubeadm-control-plane-controller-amd64:dev
-  loadBehavior: tryLoad
+  loadBehavior: mustLoad
 - name: gcr.io/k8s-staging-cluster-api/capd-manager-amd64:dev
-  loadBehavior: tryLoad
+  loadBehavior: mustLoad
 - name: quay.io/jetstack/cert-manager-cainjector:v1.1.0
   loadBehavior: tryLoad
 - name: quay.io/jetstack/cert-manager-webhook:v1.1.0
diff --git a/test/infrastructure/docker/config/default/manager_image_patch.yaml b/test/infrastructure/docker/config/default/manager_image_patch.yaml
index 2b0a3fe80..0fee551ee 100644
--- a/test/infrastructure/docker/config/default/manager_image_patch.yaml
+++ b/test/infrastructure/docker/config/default/manager_image_patch.yaml
@@ -8,5 +8,5 @@ spec:
     spec:
       containers:
       # Change the value of image field below to your controller image URL
-      - image: gcr.io/k8s-staging-cluster-api/capd-manager:master
+      - image: gcr.io/k8s-staging-cluster-api/capd-manager:dev
         name: manager
diff --git a/test/infrastructure/docker/go.mod b/test/infrastructure/docker/go.mod
index bc8591317..e0ecf22fa 100644
--- a/test/infrastructure/docker/go.mod
+++ b/test/infrastructure/docker/go.mod
@@ -14,7 +14,7 @@ require (
 	k8s.io/klog/v2 v2.8.0
 	k8s.io/utils v0.0.0-20210305010621-2afb4311ab10
 	sigs.k8s.io/cluster-api v0.3.3
-	sigs.k8s.io/controller-runtime v0.9.0-beta.1
+	sigs.k8s.io/controller-runtime v0.9.0-beta.0
 	sigs.k8s.io/kind v0.9.0
 	sigs.k8s.io/yaml v1.2.0
 )
diff --git a/test/infrastructure/docker/go.sum b/test/infrastructure/docker/go.sum
index b7b4bdc90..e7bcec740 100644
--- a/test/infrastructure/docker/go.sum
+++ b/test/infrastructure/docker/go.sum
@@ -999,8 +999,8 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg=
-sigs.k8s.io/controller-runtime v0.9.0-beta.1 h1:pmJVlNQeVMtkszjrRP59XT55Ny7+1it7ri3yyukwYTE=
-sigs.k8s.io/controller-runtime v0.9.0-beta.1/go.mod h1:ufPDuvefw2Y1KnBgHQrLdOjueYlj+XJV2AszbT+WTxs=
+sigs.k8s.io/controller-runtime v0.9.0-beta.0 h1:GMy39zuf8ywrQyNuc24wR1mdiJCSGJnxaaqakZldaug=
+sigs.k8s.io/controller-runtime v0.9.0-beta.0/go.mod h1:ufPDuvefw2Y1KnBgHQrLdOjueYlj+XJV2AszbT+WTxs=
 sigs.k8s.io/kind v0.9.0 h1:SoDlXq6pEc7dGagHULNRCCBYrLH6xOi7lqXTRXeAlg4=
 sigs.k8s.io/kind v0.9.0/go.mod h1:cxKQWwmbtRDzQ+RNKnR6gZG6fjbeTtItp5cGf+ww+1Y=
 sigs.k8s.io/kustomize/api v0.8.5/go.mod h1:M377apnKT5ZHJS++6H4rQoCHmWtt6qTpp3mbe7p6OLY=