Repository: kurator-dev/kurator
Pull request #239 (2023-04-07 07:29:48)

[release-0.3] customcluster: realization of upgrading of cluster version
This is an automated cherry-pick of #209

```release-note

```


Patch:
diff --git a/examples/infra/customcluster/cc-kcp.yaml b/examples/infra/customcluster/cc-kcp.yaml
index aed5103..bf17b2f 100644
--- a/examples/infra/customcluster/cc-kcp.yaml
+++ b/examples/infra/customcluster/cc-kcp.yaml
@@ -14,4 +14,4 @@ spec:
       kind: customMachine
       name: cc-custommachine
       namespace: default
-  version: v1.25.0
+  version: v1.24.6
diff --git a/go.mod b/go.mod
index 0b40901..ad773e7 100644
--- a/go.mod
+++ b/go.mod
@@ -11,6 +11,7 @@ require (
 	github.com/awslabs/goformation/v4 v4.19.5
 	github.com/bramvdbogaerde/go-scp v1.2.0
 	github.com/cert-manager/cert-manager v1.10.0
+	github.com/coreos/go-semver v0.3.0
 	github.com/go-logr/logr v1.2.3
 	github.com/gosuri/uitable v0.0.4
 	github.com/hashicorp/go-multierror v1.1.1
@@ -89,7 +90,6 @@ require (
 	github.com/cncf/xds/go v0.0.0-20220520190051-1e77728a1eaa // indirect
 	github.com/coredns/caddy v1.1.0 // indirect
 	github.com/coredns/corefile-migration v1.0.17 // indirect
-	github.com/coreos/go-semver v0.3.0 // indirect
 	github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect
 	github.com/coreos/go-systemd/v22 v22.3.2 // indirect
 	github.com/cyphar/filepath-securejoin v0.2.3 // indirect
diff --git a/pkg/apis/infra/v1alpha1/cluster.go b/pkg/apis/infra/v1alpha1/cluster.go
index 9b2735e..d37403a 100644
--- a/pkg/apis/infra/v1alpha1/cluster.go
+++ b/pkg/apis/infra/v1alpha1/cluster.go
@@ -85,6 +85,9 @@ const (
 
 	// ScalingDownPhase represents the cluster is removing the worker nodes.
 	ScalingDownPhase CustomClusterPhase = "ScalingDown"
+
+	// UpgradingPhase represents the kubernetes version of cluster is upgrading.
+	UpgradingPhase CustomClusterPhase = "Upgrading"
 )
 
 const (
@@ -109,6 +112,13 @@ const (
 	// ScaleDownWorkerRunFailedReason (Severity=Error) documents that the scale down worker run failed.
 	ScaleDownWorkerRunFailedReason = "ScaleDownWorkerRunFailed"
 
+	// UpgradeCondition reports on whether the cluster Kubernetes version is upgraded.
+	UpgradeCondition capiv1.ConditionType = "Upgraded"
+	// UpgradeWorkerCreateFailed (Severity=Error) documents that the upgrade worker failed to create.
+	UpgradeWorkerCreateFailed = "UpgradeWorkerFailedCreate"
+	// UpgradeWorkerRunFailedReason (Severity=Error) documents that the upgrade worker run failed.
+	UpgradeWorkerRunFailedReason = "UpgradeWorkerRunFailed"
+
 	// TerminatedCondition reports on whether the cluster is terminated. If this condition meet, then the customCluster will be deleted and there won't be any marking as true.
 	TerminatedCondition capiv1.ConditionType = "Terminated"
 	// FailedCreateTerminateWorker (Severity=Error) documents that the terminal worker failed to create.
diff --git a/pkg/controllers/customcluster_controller.go b/pkg/controllers/customcluster_controller.go
index 6387a42..aca86dc 100644
--- a/pkg/controllers/customcluster_controller.go
+++ b/pkg/controllers/customcluster_controller.go
@@ -62,6 +62,7 @@ type NodeInfo struct {
 // ClusterInfo represents the information of the cluster on VMs.
 type ClusterInfo struct {
 	WorkerNodes []NodeInfo
+	KubeVersion string
 }
 
 const (
@@ -84,13 +85,23 @@ const (
 	CustomClusterScaleDownAction customClusterManageAction = "scale-down"
 	KubesprayScaleDownCMDPrefix  customClusterManageCMD    = KubesprayCMDPrefix + "remove-node.yml -vvv -e skip_confirmation=yes"
 
-	// TODO: support custom this in CustomCluster/CustomMachine
-	DefaultKubesprayImage = "quay.io/kubespray/kubespray:v2.20.0"
+	CustomClusterUpgradeAction customClusterManageAction = "upgrade"
+	KubesprayUpgradeCMDPrefix  customClusterManageCMD    = KubesprayCMDPrefix + "upgrade-cluster.yml -vvv "
 
 	// CustomClusterFinalizer is the finalizer applied to crd.
 	CustomClusterFinalizer = "customcluster.cluster.kurator.dev"
 	// custom configmap finalizer requires at least one slash.
 	CustomClusterConfigMapFinalizer = CustomClusterFinalizer + "/configmap"
+
+	// KubeVersionPrefix is the prefix string of version of kubernetes
+	KubeVersionPrefix = "kube_version: "
+	// DefaultKubesprayVersion is the version of Kubespray used by the customCluster manager worker pod.
+	DefaultKubesprayVersion = "v2.20.0"
+	// DefaultMaxVersion is the max version of Kubernetes that is supported by DefaultKubesprayVersion.
+	DefaultMaxVersion = "v1.24.6"
+	// DefaultMinVersion is the min version of Kubernetes that is supported by DefaultKubesprayVersion.
+	DefaultMinVersion = "v1.22.0"
+	// TODO: support more version to remove limited DefaultMaxVersion and DefaultMinVersion
 )
 
 // SetupWithManager sets up the controller with the Manager.
@@ -253,21 +264,30 @@ func (r *CustomClusterController) reconcile(ctx context.Context, customCluster *
 	log := ctrl.LoggerFrom(ctx)
 	phase := customCluster.Status.Phase
 
+	// desiredClusterInfo contains information retrieved from configured CRDs such as "customMachine" and "kcp".
+	desiredClusterInfo := getDesiredClusterInfo(customMachine, kcp)
+	// desiredVersion is the one recorded in kcp.version.
+	desiredVersion := desiredClusterInfo.KubeVersion
+	// If the desired version is not within a specified range of versions, return directly.
+	if !isSupportedVersion(desiredVersion, DefaultMinVersion, DefaultMaxVersion) {
+		log.Error(fmt.Errorf("the kubespray image version is %s. Then the minimum supported version of Kubernetes is %s, and the maximum is %s. However, the desired version of Kubernetes is %s ", DefaultKubesprayVersion, DefaultMinVersion, DefaultMaxVersion, desiredVersion), "")
+		return ctrl.Result{}, nil
+	}
+
 	// Handle cluster provision.
 	if phase == v1alpha1.PendingPhase || phase == v1alpha1.ProvisionFailedPhase || phase == v1alpha1.ProvisioningPhase {
 		return r.reconcileProvision(ctx, customCluster, customMachine, cluster, kcp)
 	}
 
-	// desiredClusterInfo contains information retrieved from configured CRDs such as "customMachine" and "kcp".
-	var desiredClusterInfo *ClusterInfo
 	// provisionedClusterInfo contains information retrieved from configmap that represent provisioned cluster.
 	var provisionedClusterInfo *ClusterInfo
 	// scaleUpWorkerNodes is the nodes where desiredCluster is more than provisionedCluster.
 	var scaleUpWorkerNodes []NodeInfo
 	// scaleUpWorkerNodes is the nodes where desiredCluster is less than provisionedCluster.
 	var scaleDownWorkerNodes []NodeInfo
+	// provisionedVersion is the one recorded in configmap cluster-config.data.kube_version.
+	var provisionedVersion string
 	if phase == v1alpha1.ProvisionedPhase {
-		desiredClusterInfo = getDesiredClusterInfo(customMachine, kcp)
 		var err error
 		provisionedClusterInfo, err = r.getProvisionedClusterInfo(ctx, customCluster)
 		if err != nil {
@@ -276,6 +296,7 @@ func (r *CustomClusterController) reconcile(ctx context.Context, customCluster *
 		}
 		scaleUpWorkerNodes = findScaleUpWorkerNodes(provisionedClusterInfo.WorkerNodes, desiredClusterInfo.WorkerNodes)
 		scaleDownWorkerNodes = findScaleDownWorkerNodes(provisionedClusterInfo.WorkerNodes, desiredClusterInfo.WorkerNodes)
+		provisionedVersion = provisionedClusterInfo.KubeVersion
 	}
 
 	// Handle worker nodes scaling.
@@ -287,6 +308,19 @@ func (r *CustomClusterController) reconcile(ctx context.Context, customCluster *
 		return r.reconcileScaleDown(ctx, customCluster, customMachine, scaleDownWorkerNodes)
 	}
 
+	// Handle cluster upgrade.
+	if desiredVersion != provisionedVersion {
+		// If the desired version upgrade is not supported by Kubeadm, return directly.
+		if !isKubeadmUpgradeSupported(provisionedVersion, desiredVersion) {
+			log.Error(fmt.Errorf(" skipping MINOR versions when upgrading is unsupported with kubeadm, you can not upgrade kubernetes version from %s to %s. ", provisionedVersion, desiredVersion), "")
+			return ctrl.Result{}, nil
+		}
+		// Start reconcileUpgrade.
+		if phase == v1alpha1.ProvisionedPhase || phase == v1alpha1.UpgradingPhase {
+			return r.reconcileUpgrade(ctx, customCluster, desiredVersion)
+		}
+	}
+
 	return ctrl.Result{}, nil
 }
 
diff --git a/pkg/controllers/customcluster_helper.go b/pkg/controllers/customcluster_helper.go
index 6630c17..9a6d6f1 100644
--- a/pkg/controllers/customcluster_helper.go
+++ b/pkg/controllers/customcluster_helper.go
@@ -39,7 +39,7 @@ func generateClusterManageWorker(customCluster *v1alpha1.CustomCluster, manageAc
 	podName := customCluster.Name + "-" + string(manageAction)
 	namespace := customCluster.Namespace
 	defaultMode := int32(0o600)
-
+	kubesprayImage := getKubesprayImage(DefaultKubesprayVersion)
 	managerWorker := &corev1.Pod{
 		ObjectMeta: metav1.ObjectMeta{
 			Namespace: namespace,
@@ -54,7 +54,7 @@ func generateClusterManageWorker(customCluster *v1alpha1.CustomCluster, manageAc
 			Containers: []corev1.Container{
 				{
 					Name:    podName,
-					Image:   DefaultKubesprayImage,
+					Image:   kubesprayImage,
 					Command: []string{"/bin/sh", "-c"},
 					Args:    []string{string(manageCMD)},
 
@@ -271,12 +271,13 @@ func generateOwnerRefFromCustomCluster(customCluster *v1alpha1.CustomCluster) me
 	}
 }
 
-// getDesiredClusterInfo get desired cluster info from crd configuration。
+// getDesiredClusterInfo get desired cluster info from crd configuration.
 func getDesiredClusterInfo(customMachine *v1alpha1.CustomMachine, kcp *controlplanev1.KubeadmControlPlane) *ClusterInfo {
 	workerNodes := getWorkerNodesFromCustomMachine(customMachine)
 
 	clusterInfo := &ClusterInfo{
 		WorkerNodes: workerNodes,
+		KubeVersion: kcp.Spec.Version,
 	}
 
 	return clusterInfo
@@ -289,13 +290,22 @@ func (r *CustomClusterController) getProvisionedClusterInfo(ctx context.Context,
 	if err := r.Client.Get(ctx, generateClusterHostsKey(customCluster), clusterHosts); err != nil {
 		return nil, err
 	}
-
 	// get workerNode from cluster-host
 	workerNodes := getWorkerNodeInfoFromClusterHosts(clusterHosts)
 
-	// create workerNodes from cluster-host Configmap
+	// get current cluster-config configMap
+	clusterConfig := &corev1.ConfigMap{}
+	if err := r.Client.Get(ctx, generateClusterConfigKey(customCluster), clusterConfig); err != nil {
+		return nil, err
+	}
+
+	// get provisioned version from cluster-config
+	provisionedVersion := getKubeVersionFromCM(clusterConfig)
+
+	// get the provisioned cluster info
 	clusterInfo := &ClusterInfo{
 		WorkerNodes: workerNodes,
+		KubeVersion: provisionedVersion,
 	}
 
 	return clusterInfo, nil
@@ -383,3 +393,9 @@ func (r *CustomClusterController) ensureWorkerPodCreated(ctx context.Context, cu
 	}
 	return workerPod, nil
 }
+
+// getKubesprayImage take in kubesprayVersion return the kubespray image url of this version.
+func getKubesprayImage(kubesprayVersion string) string {
+	imagePath := "quay.io/kubespray/kubespray:" + kubesprayVersion
+	return imagePath
+}
diff --git a/pkg/controllers/customcluster_upgrade.go b/pkg/controllers/customcluster_upgrade.go
new file mode 100644
index 0000000..3f4b8c1
--- /dev/null
+++ b/pkg/controllers/customcluster_upgrade.go
@@ -0,0 +1,198 @@
+/*
+Copyright Kurator Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllers
+
+import (
+	"context"
+	"regexp"
+	"strings"
+
+	"github.com/coreos/go-semver/semver"
+	corev1 "k8s.io/api/core/v1"
+	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
+	"sigs.k8s.io/cluster-api/util/conditions"
+	ctrl "sigs.k8s.io/controller-runtime"
+
+	"kurator.dev/kurator/pkg/apis/infra/v1alpha1"
+)
+
+// reconcileUpgrade is responsible for handling the customCluster reconciliation process of cluster upgrading to targetVersion
+func (r *CustomClusterController) reconcileUpgrade(ctx context.Context, customCluster *v1alpha1.CustomCluster, targetVersion string) (ctrl.Result, error) {
+	log := ctrl.LoggerFrom(ctx)
+
+	cmd := generateUpgradeManageCMD(targetVersion)
+	// Checks whether the worker node for upgrading already exists. If it does not exist, then create it.
+	workerPod, err1 := r.ensureWorkerPodCreated(ctx, customCluster, CustomClusterUpgradeAction, cmd, generateClusterHostsName(customCluster), generateClusterConfigName(customCluster))
+	if err1 != nil {
+		conditions.MarkFalse(customCluster, v1alpha1.UpgradeCondition, v1alpha1.UpgradeWorkerCreateFailed,
+			clusterv1.ConditionSeverityWarning, "upgrade worker is failed to create %s/%s.", customCluster.Namespace, customCluster.Name)
+		log.Error(err1, "failed to ensure that upgrade WorkerPod is created", "customCluster", customCluster.Name)
+		return ctrl.Result{}, err1
+	}
+
+	// Check the current customCluster status.
+	if customCluster.Status.Phase != v1alpha1.UpgradingPhase {
+		log.Info("phase changes", "prevPhase", customCluster.Status.Phase, "currentPhase", v1alpha1.UpgradingPhase)
+		customCluster.Status.Phase = v1alpha1.UpgradingPhase
+	}
+
+	// Determine the progress of upgrading based on the status of the workerPod.
+	if workerPod.Status.Phase == corev1.PodSucceeded {
+		// Update the cluster-config to ensure that the current cluster-config (provisioned cluster info) represents the cluster after upgrading.
+		if err := r.updateKubeVersion(ctx, customCluster, targetVersion); err != nil {
+			log.Error(err, "failed to update the kubeVersion of configmap cluster-config after upgrading")
+			return ctrl.Result{}, err
+		}
+		// Restore the workerPod's status to "provisioned" after upgrading.
+		log.Info("phase changes", "prevPhase", customCluster.Status.Phase, "currentPhase", v1alpha1.ProvisionedPhase)
+		customCluster.Status.Phase = v1alpha1.ProvisionedPhase
+
+		// Delete the upgrading worker.
+		if err := r.ensureWorkerPodDeleted(ctx, generateWorkerKey(customCluster, CustomClusterUpgradeAction)); err != nil {
+			log.Error(err, "failed to delete upgrade worker pod", "worker", generateWorkerKey(customCluster, CustomClusterUpgradeAction))
+			return ctrl.Result{}, err
+		}
+		conditions.MarkTrue(customCluster, v1alpha1.UpgradeCondition)
+
+		return ctrl.Result{}, nil
+	}
+
+	// When upgrade worker pod runs failed, the status of customCluster will change into "provisioned". Deleting this error one will trigger the creation of a new upgrade worker pod.
+	if workerPod.Status.Phase == corev1.PodFailed {
+		log.Info("upgrade worker runs failed, customCluster phase changes", "prevPhase", customCluster.Status.Phase, "currentPhase", v1alpha1.ProvisionedPhase)
+		customCluster.Status.Phase = v1alpha1.UnknownPhase
+		conditions.MarkFalse(customCluster, v1alpha1.UpgradeCondition, v1alpha1.UpgradeWorkerRunFailedReason,
+			clusterv1.ConditionSeverityWarning, "upgrade worker run failed %s/%s", customCluster.Namespace, customCluster.Name)
+		return ctrl.Result{}, nil
+	}
+
+	return ctrl.Result{}, nil
+}
+
+// generateScaleDownManageCMD generate a kubespray cmd to upgrade cluster to desired kubeVersion.
+func generateUpgradeManageCMD(kubeVersion string) customClusterManageCMD {
+	if len(kubeVersion) == 0 {
+		return ""
+	}
+
+	cmd := string(KubesprayUpgradeCMDPrefix) + " -e kube_version=v" + strings.TrimPrefix(kubeVersion, "v")
+
+	return customClusterManageCMD(cmd)
+}
+
+// isSupportedVersion checks if a desired version is within a specified range of versions.
+func isSupportedVersion(desiredVersion, minVersion, maxVersion string) bool {
+	desiredVersion = strings.TrimPrefix(desiredVersion, "v")
+	minVersion = strings.TrimPrefix(minVersion, "v")
+	maxVersion = strings.TrimPrefix(maxVersion, "v")
+
+	// Parse the version strings using semver package.
+	desired, err := semver.NewVersion(desiredVersion)
+	if err != nil {
+		return false
+	}
+
+	min, err1 := semver.NewVersion(minVersion)
+	if err1 != nil {
+		return false
+	}
+
+	max, err2 := semver.NewVersion(maxVersion)
+	if err2 != nil {
+		return false
+	}
+
+	// Check the desiredVersion is in the correct scope.
+	if desired.Compare(*min) < 0 || desired.Compare(*max) > 0 {
+		return false
+	}
+
+	return true
+}
+
+// isKubeadmUpgradeSupported check if this upgrading is supported to Kubeadm. kubespray using kubeadm to upgrade, but it is not supported to skip MINOR versions during the upgrade process using Kubeadm.
+func isKubeadmUpgradeSupported(originVersion, targetVersion string) bool {
+	originVersion = strings.TrimPrefix(originVersion, "v")
+	targetVersion = strings.TrimPrefix(targetVersion, "v")
+
+	// Parse the version strings using semver package.
+	origin, err := semver.NewVersion(originVersion)
+	if err != nil {
+		return false
+	}
+	target, err1 := semver.NewVersion(targetVersion)
+	if err1 != nil {
+		return false
+	}
+
+	// Compare the major versions.
+	if origin.Major != target.Major {
+		return false
+	}
+
+	// Compare the minor versions. Check if the minor version difference is at most 1
+	if origin.Minor-target.Minor > 1 || target.Minor-origin.Minor > 1 {
+		return false
+	}
+
+	return true
+}
+
+// updateKubeVersion update the kubeVersion in configmap.
+func (r *CustomClusterController) updateKubeVersion(ctx context.Context, customCluster *v1alpha1.CustomCluster, newKubeVersion string) error {
+	// get cm.
+	cm := &corev1.ConfigMap{}
+	if err := r.Client.Get(ctx, generateClusterConfigKey(customCluster), cm); err != nil {
+		return err
+	}
+
+	cm.Data[ClusterConfigName] = getUpdatedKubeVersionConfigData(cm, newKubeVersion)
+
+	// update cm.
+	if err := r.Client.Update(ctx, cm); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// getUpdatedKubeVersionConfigData get the configuration data that represents the upgraded version of Kubernetes.
+func getUpdatedKubeVersionConfigData(clusterConfig *corev1.ConfigMap, newKubeVersion string) string {
+	clusterConfigData := strings.TrimSpace(clusterConfig.Data[ClusterConfigName])
+
+	// add KubeVersionPrefix to avoid confusion with other configurations.
+	oldStr := KubeVersionPrefix + getKubeVersionFromCM(clusterConfig)
+	newStr := KubeVersionPrefix + newKubeVersion
+
+	return strings.TrimSpace(strings.Replace(clusterConfigData, oldStr, newStr, -1))
+}
+
+// getKubeVersionFromCM get the provisioned k8s version from the cluster-config configmap.
+func getKubeVersionFromCM(clusterConfig *corev1.ConfigMap) string {
+	clusterConfigData := clusterConfig.Data[ClusterConfigName]
+	clusterConfigData = strings.TrimSpace(clusterConfigData)
+
+	zp := regexp.MustCompile(`[\t\n\f\r]`)
+	clusterHostDateArr := zp.Split(clusterConfigData, -1)
+
+	for _, configStr := range clusterHostDateArr {
+		if strings.HasPrefix(configStr, KubeVersionPrefix) {
+			return strings.TrimSpace(strings.Replace(configStr, KubeVersionPrefix, "", -1))
+		}
+	}
+	return ""
+}


Test patch:
diff --git a/pkg/controllers/customcluster_helper_test.go b/pkg/controllers/customcluster_helper_test.go
index 1900b99..6f71035 100644
--- a/pkg/controllers/customcluster_helper_test.go
+++ b/pkg/controllers/customcluster_helper_test.go
@@ -87,6 +87,7 @@ var targetWorkerNodesSingle = []NodeInfo{
 
 var targetClusterInfoSingle = &ClusterInfo{
 	WorkerNodes: targetWorkerNodesSingle,
+	KubeVersion: "v1.20.0",
 }
 
 var targetWorkerNodesMulti = []NodeInfo{
@@ -104,6 +105,7 @@ var targetWorkerNodesMulti = []NodeInfo{
 
 var targetClusterInfoMulti = &ClusterInfo{
 	WorkerNodes: targetWorkerNodesMulti,
+	KubeVersion: "v1.25.0",
 }
 
 func TestGetWorkerNodesFromCustomMachine(t *testing.T) {
diff --git a/pkg/controllers/customcluster_upgrade_test.go b/pkg/controllers/customcluster_upgrade_test.go
new file mode 100644
index 0000000..f8459bd
--- /dev/null
+++ b/pkg/controllers/customcluster_upgrade_test.go
@@ -0,0 +1,161 @@
+/*
+Copyright Kurator Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllers
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestIsKubeadmUpgradeSupported(t *testing.T) {
+	testCases := []struct {
+		name          string
+		originVersion string
+		targetVersion string
+		expected      bool
+	}{
+		{
+			name:          "Same major and minor version",
+			originVersion: "v1.18.0",
+			targetVersion: "v1.18.1",
+			expected:      true,
+		},
+		{
+			name:          "Skip one minor version",
+			originVersion: "v1.18.0",
+			targetVersion: "v1.19.0",
+			expected:      true,
+		},
+		{
+			name:          "Skip two minor versions",
+			originVersion: "v1.18.0",
+			targetVersion: "v1.20.0",
+			expected:      false,
+		},
+		{
+			name:          "Invalid origin version",
+			originVersion: "invalid",
+			targetVersion: "v1.18.1",
+			expected:      false,
+		},
+		{
+			name:          "Invalid target version",
+			originVersion: "v1.18.0",
+			targetVersion: "invalid",
+			expected:      false,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			actual := isKubeadmUpgradeSupported(tc.originVersion, tc.targetVersion)
+			assert.Equal(t, tc.expected, actual)
+		})
+	}
+}
+
+func TestIsSupportedVersion(t *testing.T) {
+	testCases := []struct {
+		name           string
+		desiredVersion string
+		minVersion     string
+		maxVersion     string
+		expected       bool
+	}{
+		{
+			name:           "valid version within range",
+			desiredVersion: "v1.2.3",
+			minVersion:     "v1.0.0",
+			maxVersion:     "v2.0.0",
+			expected:       true,
+		},
+		{
+			name:           "invalid desired version",
+			desiredVersion: "invalid",
+			minVersion:     "v1.0.0",
+			maxVersion:     "v2.0.0",
+			expected:       false,
+		},
+		{
+			name:           "invalid min version",
+			desiredVersion: "v1.2.3",
+			minVersion:     "invalid",
+			maxVersion:     "v2.0.0",
+			expected:       false,
+		},
+		{
+			name:           "invalid max version",
+			desiredVersion: "v1.2.3",
+			minVersion:     "v1.0.0",
+			maxVersion:     "invalid",
+			expected:       false,
+		},
+		{
+			name:           "desired version below range",
+			desiredVersion: "v0.9.9",
+			minVersion:     "v1.0.0",
+			maxVersion:     "v2.0.0",
+			expected:       false,
+		},
+		{
+			name:           "desired version above range",
+			desiredVersion: "v2.1.0",
+			minVersion:     "v1.0.0",
+			maxVersion:     "v2.0.0",
+			expected:       false,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			result := isSupportedVersion(tc.desiredVersion, tc.minVersion, tc.maxVersion)
+			assert.Equal(t, tc.expected, result)
+		})
+	}
+}
+
+func TestGenerateUpgradeManageCMD(t *testing.T) {
+	tests := []struct {
+		name        string
+		kubeVersion string
+		expected    customClusterManageCMD
+	}{
+		{
+			name:        "valid kube version",
+			kubeVersion: "v1.20.0",
+			expected:    "ansible-playbook -i inventory/cluster-hosts --private-key /root/.ssh/ssh-privatekey upgrade-cluster.yml -vvv  -e kube_version=v1.20.0",
+		},
+		{
+			name:        "non-prefixed kube version",
+			kubeVersion: "1.23.4",
+			expected:    "ansible-playbook -i inventory/cluster-hosts --private-key /root/.ssh/ssh-privatekey upgrade-cluster.yml -vvv  -e kube_version=v1.23.4",
+		},
+		{
+			name:        "empty kube version",
+			kubeVersion: "",
+			expected:    "",
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			got := generateUpgradeManageCMD(tc.kubeVersion)
+			assert.Equal(t, tc.expected, got)
+		})
+	}
+}