Repository: clusternet/clusternet
Pull request #673 (2023-04-23 07:28:57)

taint cluster with not-ready conditions
<!--  Thanks for sending a pull request!

-->

#### What type of PR is this?
kind/feature

#### What this PR does / why we need it:

#### Which issue(s) this PR fixes:
<!--
*Automatically closes linked issue when PR is merged.
Usage: `Fixes #<issue number>`, or `Fixes (paste link of issue)`.
-->
Fixes #

#### Special notes for your reviewer:


codecov-commenter: ## [Codecov](https://codecov.io/gh/clusternet/clusternet/pull/673?src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=clusternet) Report
> Merging [#673](https://codecov.io/gh/clusternet/clusternet/pull/673?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=clusternet) (ec63856) into [main](https://codecov.io/gh/clusternet/clusternet/commit/e2ffa5fa54a7e81045fbef497d0dfd68a4bcc7f6?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=clusternet) (e2ffa5f) will **decrease** coverage by `0.32%`.
> The diff coverage is `n/a`.

:mega: This organization is not using Codecov’s [GitHub App Integration](https://github.com/apps/codecov). We recommend you install it so Codecov can continue to function properly for your repositories. [Learn more](https://about.codecov.io/blog/codecov-is-updating-its-github-integration/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=clusternet)

```diff
@@            Coverage Diff             @@
##             main     #673      +/-   ##
==========================================
- Coverage   13.60%   13.29%   -0.32%     
==========================================
  Files          66       71       +5     
  Lines        7492     7825     +333     
==========================================
+ Hits         1019     1040      +21     
- Misses       6391     6701     +310     
- Partials       82       84       +2     
```


[see 40 files with indirect coverage changes](https://codecov.io/gh/clusternet/clusternet/pull/673/indirect-changes?src=pr&el=tree-more&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=clusternet)

Help us with your feedback. Take ten seconds to tell us [how you rate us](https://about.codecov.io/nps?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=clusternet). Have a feature suggestion? [Share it here.](https://app.codecov.io/gh/feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=clusternet)


Patch:
diff --git a/go.mod b/go.mod
index 973cc97..7056e40 100644
--- a/go.mod
+++ b/go.mod
@@ -4,7 +4,7 @@ go 1.19
 
 require (
 	github.com/davecgh/go-spew v1.1.1
-	github.com/dixudx/yacht v0.4.0
+	github.com/dixudx/yacht v0.4.1
 	github.com/emicklei/go-restful/v3 v3.9.0
 	github.com/evanphx/json-patch v5.6.0+incompatible
 	github.com/google/go-cmp v0.5.9
diff --git a/go.sum b/go.sum
index 91f9c3f..74432d1 100644
--- a/go.sum
+++ b/go.sum
@@ -143,8 +143,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/denisenkom/go-mssqldb v0.9.0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
 github.com/distribution/distribution/v3 v3.0.0-20220526142353-ffbd94cbe269 h1:hbCT8ZPPMqefiAWD2ZKjn7ypokIGViTvBBg/ExLSdCk=
-github.com/dixudx/yacht v0.4.0 h1:bEK7D41OQiLR5LpheZM2kvhdXSoIIVTO3z2W/44srYw=
-github.com/dixudx/yacht v0.4.0/go.mod h1:u0y5AFBD9swk3T0jspITZCPfu+tmzrlaRBoWb2Hv35k=
+github.com/dixudx/yacht v0.4.1 h1:o7WNtHZM84xkqOLSYqa5QygP6rFETowLH0i7ycstlrg=
+github.com/dixudx/yacht v0.4.1/go.mod h1:u0y5AFBD9swk3T0jspITZCPfu+tmzrlaRBoWb2Hv35k=
 github.com/docker/cli v20.10.17+incompatible h1:eO2KS7ZFeov5UJeaDmIs1NFEDRf32PaqRpvoEkKBy5M=
 github.com/docker/cli v20.10.17+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
 github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68=
diff --git a/pkg/controllermanager/controllermanager.go b/pkg/controllermanager/controllermanager.go
index 19511e5..ec13aac 100644
--- a/pkg/controllermanager/controllermanager.go
+++ b/pkg/controllermanager/controllermanager.go
@@ -233,7 +233,7 @@ func startClusterLifecycleController(controllerCtx *controllercontext.Controller
 	if err != nil {
 		return false, err
 	}
-	go clusterLifecycle.Run(controllerCtx.Opts.Threadiness, ctx.Done())
+	go clusterLifecycle.Run(controllerCtx.Opts.Threadiness, ctx)
 	return true, nil
 }
 
diff --git a/pkg/controllers/clusters/clusterlifecycle/clusterlifecycle_controller.go b/pkg/controllers/clusters/clusterlifecycle/clusterlifecycle_controller.go
index 8d4d272..0cd94ea 100644
--- a/pkg/controllers/clusters/clusterlifecycle/clusterlifecycle_controller.go
+++ b/pkg/controllers/clusters/clusterlifecycle/clusterlifecycle_controller.go
@@ -18,24 +18,28 @@ package clusterlifecycle
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"time"
 
+	"github.com/dixudx/yacht"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/equality"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
-	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apimachinery/pkg/util/strategicpatch"
 	"k8s.io/client-go/tools/cache"
 	"k8s.io/client-go/tools/record"
 	"k8s.io/client-go/util/workqueue"
 	"k8s.io/klog/v2"
 
 	clusterapi "github.com/clusternet/clusternet/pkg/apis/clusters/v1beta1"
-	clusternetClientSet "github.com/clusternet/clusternet/pkg/generated/clientset/versioned"
+	clusternetclientset "github.com/clusternet/clusternet/pkg/generated/clientset/versioned"
 	clusterinformers "github.com/clusternet/clusternet/pkg/generated/informers/externalversions/clusters/v1beta1"
-	clusterlisters "github.com/clusternet/clusternet/pkg/generated/listers/clusters/v1beta1"
+	"github.com/clusternet/clusternet/pkg/known"
+	taintutils "github.com/clusternet/clusternet/pkg/utils/taints"
 )
 
 const (
@@ -45,9 +49,26 @@ const (
 	ClusterNotReadyThreshold = 3
 )
 
+var (
+	// map {ConditionType: {ConditionStatus: TaintKey}}
+	// represents which ConditionType under which ConditionStatus should be
+	// tainted with which TaintKey
+	// for certain ConditionType, there are multiple {ConditionStatus,TaintKey} pairs
+	clusterConditionToTaintKeyStatusMap = map[string]map[metav1.ConditionStatus]string{
+		clusterapi.ClusterReady: {
+			metav1.ConditionFalse:   known.TaintClusterUnschedulable,
+			metav1.ConditionUnknown: known.TaintClusterUnschedulable,
+		},
+	}
+
+	taintKeyToNodeConditionMap = map[string]string{
+		known.TaintClusterUnschedulable: clusterapi.ClusterReady,
+	}
+)
+
 // Controller is a controller that manages cluster's lifecycle
 type Controller struct {
-	clusternetClient clusternetClientSet.Interface
+	clusternetClient clusternetclientset.Interface
 
 	// workqueue is a rate limited work queue. This is used to queue work to be
 	// processed instead of performing it as soon as a change happens. This
@@ -56,203 +77,108 @@ type Controller struct {
 	// simultaneously in two different workers.
 	workqueue workqueue.RateLimitingInterface
 
-	clusterLister clusterlisters.ManagedClusterLister
-	clusterSynced cache.InformerSynced
-
-	recorder record.EventRecorder
+	recorder        record.EventRecorder
+	yachtController *yacht.Controller
+	clusterInformer clusterinformers.ManagedClusterInformer
 }
 
-func NewController(clusternetClient clusternetClientSet.Interface,
+func NewController(clusternetClient clusternetclientset.Interface,
 	clusterInformer clusterinformers.ManagedClusterInformer, recorder record.EventRecorder) (*Controller, error) {
 	c := &Controller{
 		clusternetClient: clusternetClient,
 		workqueue:        workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "ManagedCluster"),
-		clusterLister:    clusterInformer.Lister(),
-		clusterSynced:    clusterInformer.Informer().HasSynced,
 		recorder:         recorder,
+		clusterInformer:  clusterInformer,
 	}
 
-	_, err := clusterInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
-		AddFunc:    c.addCluster,
-		UpdateFunc: c.updateCluster,
-	})
+	yachtController := yacht.NewController("cluster-lifecycle").
+		WithHandlerFunc(c.handle).
+		WithEnqueueFilterFunc(func(oldObj, newObj interface{}) (bool, error) {
+			// update event
+			if oldObj != nil && newObj != nil {
+				oldMcls := oldObj.(*clusterapi.ManagedCluster)
+				newMcls := newObj.(*clusterapi.ManagedCluster)
+
+				// Decide whether discovery has reported a status change.
+				if equality.Semantic.DeepEqual(oldMcls.Status, newMcls.Status) {
+					klog.V(4).Infof("no updates on the status of ManagedCluster %s, skipping syncing", klog.KObj(oldMcls))
+					return false, nil
+				}
+			}
+			return true, nil
+		})
+
+	// add event handler for ManagedCluster on the addition/update
+	clusterEventHandler := yachtController.DefaultResourceEventHandlerFuncs()
+	clusterEventHandler.DeleteFunc = nil // only care cluster add/update
+	_, err := clusterInformer.Informer().AddEventHandler(clusterEventHandler)
 	if err != nil {
 		return nil, err
 	}
-
+	c.yachtController = yachtController
 	return c, nil
 }
 
-func (c *Controller) Run(workers int, stopCh <-chan struct{}) {
+func (c *Controller) Run(workers int, ctx context.Context) {
 	defer utilruntime.HandleCrash()
 
 	klog.Infof("Starting cluster lifecycle controller")
 	defer klog.Infof("Shutting down cluster lifecycle controller")
 
-	// Wait for the caches to be synced before starting monitor lifecycle
-	if !cache.WaitForNamedCacheSync("cluster-lifecycle-controller", stopCh, c.clusterSynced) {
-		return
-	}
-
-	klog.V(5).Infof("starting %d worker threads", workers)
-	// Launch workers to process ManagedCluster resources
-	for i := 0; i < workers; i++ {
-		go wait.Until(c.runWorker, time.Second, stopCh)
-	}
-
-	<-stopCh
-}
-
-func (c *Controller) addCluster(obj interface{}) {
-	mcls := obj.(*clusterapi.ManagedCluster)
-	klog.V(4).Infof("adding ManagedCluster %q", klog.KObj(mcls))
-	c.enqueue(mcls)
-}
-
-func (c *Controller) updateCluster(old, cur interface{}) {
-	oldMcls := old.(*clusterapi.ManagedCluster)
-	newMcls := cur.(*clusterapi.ManagedCluster)
-
-	if newMcls.DeletionTimestamp != nil {
-		return
-	}
-
-	// Decide whether discovery has reported a status change.
-	if equality.Semantic.DeepEqual(oldMcls.Status, newMcls.Status) {
-		klog.V(4).Infof("no updates on the status of ManagedCluster %s, skipping syncing", klog.KObj(oldMcls))
-		return
-	}
-
-	klog.V(5).Infof("updating ManagedCluster %q", klog.KObj(oldMcls))
-	c.enqueue(newMcls)
+	c.yachtController.WithWorkers(workers).Run(ctx)
 }
 
-// runWorker is a long-running function that will continually call the
-// processNextWorkItem function in order to read and process a message on the
-// workqueue.
-func (c *Controller) runWorker() {
-	for c.processNextWorkItem() {
-	}
-}
-
-// processNextWorkItem will read a single work item off the workqueue and
-// attempt to process it, by calling the syncHandler.
-func (c *Controller) processNextWorkItem() bool {
-	obj, shutdown := c.workqueue.Get()
-
-	if shutdown {
-		return false
-	}
-
-	// We wrap this block in a func so we can defer c.workqueue.Done.
-	err := func(obj interface{}) error {
-		// We call Done here so the workqueue knows we have finished
-		// processing this item. We also must remember to call Forget if we
-		// do not want this work item being re-queued. For example, we do
-		// not call Forget if a transient error occurs, instead the item is
-		// put back on the workqueue and attempted again after a back-off
-		// period.
-		defer c.workqueue.Done(obj)
-		var key string
-		var ok bool
-		// We expect strings to come off the workqueue. These are of the
-		// form namespace/name. We do this as the delayed nature of the
-		// workqueue means the items in the informer cache may actually be
-		// more up to date that when the item was initially put onto the
-		// workqueue.
-		if key, ok = obj.(string); !ok {
-			// As the item in the workqueue is actually invalid, we call
-			// Forget here else we'd go into a loop of attempting to
-			// process a work item that is invalid.
-			c.workqueue.Forget(obj)
-			utilruntime.HandleError(fmt.Errorf("expected string in workqueue but got %#v", obj))
-			return nil
-		}
-		// Run the syncHandler, passing it the namespace/name string of the
-		// ManagedCluster resource to be synced.
-		requeueAfter, err := c.syncHandler(key)
-		switch {
-		case err != nil:
-			// Put the item back on the workqueue to handle any transient errors.
-			c.workqueue.AddRateLimited(key)
-			return fmt.Errorf("error syncing '%s': %s, requeuing", key, err.Error())
-		case requeueAfter > 0:
-			// Put the item back on the workqueue with delay.
-			c.workqueue.Forget(obj)
-			c.workqueue.AddAfter(obj, requeueAfter)
-		default:
-			c.workqueue.Forget(obj)
-		}
-		klog.Infof("successfully synced ManagedCluster %q", key)
-		return nil
-	}(obj)
-
-	if err != nil {
-		utilruntime.HandleError(err)
-		return true
-	}
-
-	return true
-}
-
-// syncHandler compares the actual state with the desired, and attempts to
-// converge the two. It then updates the Status block of the ManagedCluster resource
-// with the current status of the resource.
-// Duration <=0 means don't need to requeue this obj.
-func (c *Controller) syncHandler(key string) (time.Duration, error) {
-	// If an error occurs during handling, we'll requeue the item so we can
-	// attempt processing again later. This could have been caused by a
-	// temporary network failure, or any other transient reason.
-
+func (c *Controller) handle(key interface{}) (requeueAfter *time.Duration, err error) {
 	// Convert the namespace/name string into a distinct namespace and name
-	ns, name, err := cache.SplitMetaNamespaceKey(key)
+	ns, name, err := cache.SplitMetaNamespaceKey(key.(string))
 	if err != nil {
 		utilruntime.HandleError(fmt.Errorf("invalid resource key: %s", key))
-		return time.Duration(0), nil
+		return nil, nil
 	}
 
 	klog.V(4).Infof("start processing ManagedCluster %q", key)
 	// Get the ManagedCluster resource with this name
-	mcls, err := c.clusterLister.ManagedClusters(ns).Get(name)
+	mcls, err := c.clusterInformer.Lister().ManagedClusters(ns).Get(name)
 	// The ManagedCluster resource may no longer exist, in which case we stop processing.
 	if apierrors.IsNotFound(err) {
 		klog.V(2).Infof("ManagedCluster %q has been deleted", key)
-		return time.Duration(0), nil
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	if mcls.DeletionTimestamp != nil {
+		klog.V(2).Infof("ManagedCluster %q is being deleted. Skip it", key)
+		return nil, nil
 	}
+
+	updatedMcls, err := c.updateClusterCondition(context.TODO(), mcls.DeepCopy())
 	if err != nil {
-		return time.Duration(0), err
+		msg := fmt.Sprintf("failed to update conditions of ManagedCluster status: %v", err)
+		klog.WarningDepth(2, msg)
+		c.recorder.Event(mcls, corev1.EventTypeWarning, "FailedUpdatingClusterStatus", msg)
+		return nil, err
 	}
 
-	err = c.updateClusterCondition(context.TODO(), mcls.DeepCopy())
+	err = c.updateClusterTaints(context.TODO(), updatedMcls)
 	if err != nil {
-		return time.Duration(0), err
+		return nil, err
 	}
+
 	// requeueAfter is the duration that we consider a cluster to be Unknown
 	// the duration is the same as the grace period for cluster monitoring
 	return getClusterMonitorGracePeriod(mcls.Status), nil
 }
 
-// enqueue takes a ManagedCluster resource and converts it into a namespace/name
-// string which is then put onto the work queue. This method should *not* be
-// passed resources of any type other than ManagedCluster.
-func (c *Controller) enqueue(cluster *clusterapi.ManagedCluster) {
-	key, err := cache.MetaNamespaceKeyFunc(cluster)
-	if err != nil {
-		utilruntime.HandleError(err)
-		return
-	}
-	c.workqueue.Add(key)
-}
-
-func (c *Controller) updateClusterCondition(ctx context.Context, cluster *clusterapi.ManagedCluster) error {
+func (c *Controller) updateClusterCondition(ctx context.Context, cluster *clusterapi.ManagedCluster) (*clusterapi.ManagedCluster, error) {
 	currentReadyCondition := GetClusterCondition(&cluster.Status, clusterapi.ClusterReady)
-	observedReadyCondition := c.generateClusterReadyCondition(cluster)
+	observedReadyCondition := generateClusterReadyCondition(cluster)
 	if observedReadyCondition == nil || equality.Semantic.DeepEqual(currentReadyCondition, observedReadyCondition) {
-		return nil
+		return nil, nil
 	}
 
-	// update Readyz,Livez,Healthz to false when cluster status is unknown
-	if observedReadyCondition.Status == metav1.ConditionUnknown {
+	// update Readyz,Livez,Healthz to false when cluster status is not ready
+	if observedReadyCondition.Status != metav1.ConditionTrue {
 		cluster.Status.Readyz = false
 		cluster.Status.Livez = false
 		cluster.Status.Healthz = false
@@ -260,19 +186,88 @@ func (c *Controller) updateClusterCondition(ctx context.Context, cluster *cluste
 
 	// TODO: multiple cluster conditions
 	cluster.Status.Conditions = []metav1.Condition{*observedReadyCondition}
-	_, err := c.clusternetClient.ClustersV1beta1().ManagedClusters(cluster.Namespace).UpdateStatus(ctx, cluster, metav1.UpdateOptions{})
+	return c.clusternetClient.ClustersV1beta1().ManagedClusters(cluster.Namespace).UpdateStatus(ctx,
+		cluster, metav1.UpdateOptions{})
+}
+
+func (c *Controller) updateClusterTaints(ctx context.Context, cluster *clusterapi.ManagedCluster) error {
+	// Map cluster's condition to Taints.
+	var taints []corev1.Taint
+	for _, condition := range cluster.Status.Conditions {
+		if taintMap, found := clusterConditionToTaintKeyStatusMap[condition.Type]; found {
+			if taintKey, found2 := taintMap[condition.Status]; found2 {
+				taints = append(taints, corev1.Taint{
+					Key:    taintKey,
+					Effect: corev1.TaintEffectNoSchedule,
+				})
+			}
+		}
+	}
+
+	// Get exist taints of cluster.
+	clusterTaints := taintutils.TaintSetFilter(cluster.Spec.Taints, func(t *corev1.Taint) bool {
+		// only NoSchedule taints are candidates to be compared with "taints" later
+		if t.Effect != corev1.TaintEffectNoSchedule {
+			return false
+		}
+		// Find unschedulable taint of cluster.
+		if t.Key == known.TaintClusterUnschedulable {
+			return true
+		}
+		// Find cluster condition taints of cluster.
+		_, found := taintKeyToNodeConditionMap[t.Key]
+		return found
+	})
+	taintsToAdd, taintsToDel := taintutils.TaintSetDiff(taints, clusterTaints)
+	// If nothing to add or delete, return true directly.
+	if len(taintsToAdd) == 0 && len(taintsToDel) == 0 {
+		return nil
+	}
+
+	// patch taints
+	return patchClusterTaints(ctx, c.clusternetClient, cluster, taintsToAdd, taintsToDel)
+}
+
+func patchClusterTaints(ctx context.Context, client clusternetclientset.Interface, cluster *clusterapi.ManagedCluster,
+	taintsToAdd, taintsToRemove []*corev1.Taint) error {
+	var err error
+	var newCluster *clusterapi.ManagedCluster
+	for _, taint := range taintsToRemove {
+		newCluster, _, err = taintutils.RemoveTaint(cluster, taint)
+		if err != nil {
+			return err
+		}
+	}
+	for _, taint := range taintsToAdd {
+		newCluster, _, err = taintutils.AddOrUpdateTaint(newCluster, taint)
+		if err != nil {
+			return err
+		}
+	}
+
+	// patch ManagedCluster
+	oldData, err := json.Marshal(cluster)
 	if err != nil {
-		msg := fmt.Sprintf("failed to update conditions of ManagedCluster status: %v", err)
-		klog.WarningDepth(2, msg)
-		c.recorder.Event(cluster, corev1.EventTypeWarning, "FailedUpdatingClusterStatus", msg)
 		return err
 	}
-
-	return nil
+	newData, err := json.Marshal(newCluster)
+	if err != nil {
+		return err
+	}
+	patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, clusterapi.ManagedCluster{})
+	if err != nil {
+		return err
+	}
+	if len(patchBytes) == 0 {
+		return nil
+	}
+	_, err = client.ClustersV1beta1().ManagedClusters(cluster.Namespace).Patch(ctx, cluster.Name, types.MergePatchType,
+		patchBytes, metav1.PatchOptions{})
+	return err
 }
 
 // generateClusterReadyCondition return cluster unknown condition based on cluster status and grace period.
-func (c *Controller) generateClusterReadyCondition(cluster *clusterapi.ManagedCluster) *metav1.Condition {
+func generateClusterReadyCondition(cluster *clusterapi.ManagedCluster) *metav1.Condition {
 	lastObservedTime := cluster.Status.LastObservedTime
 	probeTimestamp := lastObservedTime
 	if lastObservedTime.IsZero() {
@@ -281,7 +276,7 @@ func (c *Controller) generateClusterReadyCondition(cluster *clusterapi.ManagedCl
 		probeTimestamp = cluster.CreationTimestamp
 	}
 
-	if !metav1.Now().After(probeTimestamp.Add(getClusterMonitorGracePeriod(cluster.Status))) {
+	if !metav1.Now().After(probeTimestamp.Add(*getClusterMonitorGracePeriod(cluster.Status))) {
 		return nil
 	}
 
@@ -319,10 +314,10 @@ func GetClusterCondition(status *clusterapi.ManagedClusterStatus, conditionType
 }
 
 // getClusterMonitorGracePeriod calculate grace period for cluster monitoring, use DefaultClusterMonitorGracePeriod if HeartbeatFrequencySeconds is undefined.
-func getClusterMonitorGracePeriod(status clusterapi.ManagedClusterStatus) time.Duration {
+func getClusterMonitorGracePeriod(status clusterapi.ManagedClusterStatus) *time.Duration {
 	gracePeriod := DefaultClusterMonitorGracePeriod
 	if status.HeartbeatFrequencySeconds != nil {
 		gracePeriod = time.Second * time.Duration(*status.HeartbeatFrequencySeconds) * ClusterNotReadyThreshold
 	}
-	return gracePeriod
+	return &gracePeriod
 }
diff --git a/pkg/controllers/mcs/serviceimport_controller.go b/pkg/controllers/mcs/serviceimport_controller.go
index 3f9793e..f86f671 100644
--- a/pkg/controllers/mcs/serviceimport_controller.go
+++ b/pkg/controllers/mcs/serviceimport_controller.go
@@ -350,8 +350,16 @@ func servicePorts(svcImport *v1alpha1.ServiceImport) []corev1.ServicePort {
 }
 
 // preFilter filter ServiceImport if has no label known.LabelServiceName and known.LabelServiceNameSpace
-func preFilter(oldObj, _ interface{}) (bool, error) {
-	si := oldObj.(*v1alpha1.ServiceImport)
+func preFilter(oldObj, newObj interface{}) (bool, error) {
+	var si *v1alpha1.ServiceImport
+	if newObj == nil {
+		// Delete
+		si = oldObj.(*v1alpha1.ServiceImport)
+	} else {
+		// Add or Update
+		si = newObj.(*v1alpha1.ServiceImport)
+	}
+
 	if si.Spec.Type != v1alpha1.ClusterSetIP {
 		return false, nil
 	}
diff --git a/pkg/known/taints.go b/pkg/known/taints.go
new file mode 100644
index 0000000..e720f3d
--- /dev/null
+++ b/pkg/known/taints.go
@@ -0,0 +1,23 @@
+/*
+Copyright 2023 The Clusternet Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package known
+
+const (
+	// TaintClusterUnschedulable will be added when cluster becomes unschedulable
+	// and removed when cluster becomes schedulable.
+	TaintClusterUnschedulable = "clusters.clusternet.io/unschedulable"
+)
diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go
index 389c917..54bbb61 100644
--- a/pkg/scheduler/scheduler.go
+++ b/pkg/scheduler/scheduler.go
@@ -534,9 +534,9 @@ func (sched *Scheduler) addAllEventHandlers() error {
 
 		for key, subscribers := range sched.subscribersMap {
 			for _, subscriber := range subscribers {
-				selector, err := metav1.LabelSelectorAsSelector(subscriber.ClusterAffinity)
-				if err != nil {
-					klog.ErrorDepth(5, fmt.Sprintf("failed to parse labelSelector in Subscription %s: %v", key, err))
+				selector, err2 := metav1.LabelSelectorAsSelector(subscriber.ClusterAffinity)
+				if err2 != nil {
+					klog.ErrorDepth(5, fmt.Sprintf("failed to parse labelSelector in Subscription %s: %v", key, err2))
 					continue
 				}
 				if newMcls != nil && oldMcls == nil {
diff --git a/pkg/utils/deployer.go b/pkg/utils/deployer.go
index 39672bf..2ce781f 100644
--- a/pkg/utils/deployer.go
+++ b/pkg/utils/deployer.go
@@ -788,9 +788,18 @@ func IsClusterLost(clusterID, namespace string, clusterLister clusterlisters.Man
 		return true
 	}
 
-	for _, condition := range mcls[0].Status.Conditions {
+	return !ClusterHasReadyCondition(mcls[0])
+}
+
+func ClusterHasReadyCondition(mc *clusterapi.ManagedCluster) bool {
+	// in case the conditions are not updated yet
+	if len(mc.Status.Conditions) == 0 {
+		return true
+	}
+
+	for _, condition := range mc.Status.Conditions {
 		if condition.Type == clusterapi.ClusterReady {
-			return condition.Status == metav1.ConditionUnknown
+			return condition.Status == metav1.ConditionTrue
 		}
 	}
 
diff --git a/pkg/utils/taints/taints.go b/pkg/utils/taints/taints.go
new file mode 100644
index 0000000..ec64a7b
--- /dev/null
+++ b/pkg/utils/taints/taints.go
@@ -0,0 +1,137 @@
+/*
+Copyright 2023 The Clusternet Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Copied from k8s.io/kubernetes/pkg/util/taints/taints.go and modified
+// Some unused functions are deleted as well
+
+// package taints implements utilities for working with taints
+
+package taints
+
+import (
+	"reflect"
+
+	v1 "k8s.io/api/core/v1"
+
+	clusterapi "github.com/clusternet/clusternet/pkg/apis/clusters/v1beta1"
+)
+
+// DeleteTaint removes all the taints that have the same key and effect to given taintToDelete.
+func DeleteTaint(taints []v1.Taint, taintToDelete *v1.Taint) ([]v1.Taint, bool) {
+	newTaints := []v1.Taint{}
+	deleted := false
+	for i := range taints {
+		if taintToDelete.MatchTaint(&taints[i]) {
+			deleted = true
+			continue
+		}
+		newTaints = append(newTaints, taints[i])
+	}
+	return newTaints, deleted
+}
+
+// RemoveTaint tries to remove a taint from annotations list.
+// Returns a new copy of updated Cluster and true if something was updated false otherwise.
+func RemoveTaint(cluster *clusterapi.ManagedCluster, taint *v1.Taint) (*clusterapi.ManagedCluster, bool, error) {
+	newCluster := cluster.DeepCopy()
+	clusterTaints := newCluster.Spec.Taints
+	if len(clusterTaints) == 0 {
+		return newCluster, false, nil
+	}
+
+	if !TaintExists(clusterTaints, taint) {
+		return newCluster, false, nil
+	}
+
+	newTaints, _ := DeleteTaint(clusterTaints, taint)
+	newCluster.Spec.Taints = newTaints
+	return newCluster, true, nil
+}
+
+// AddOrUpdateTaint tries to add a taint to annotations list.
+// Returns a new copy of updated Cluster and true if something was updated false otherwise.
+func AddOrUpdateTaint(cluster *clusterapi.ManagedCluster, taint *v1.Taint) (*clusterapi.ManagedCluster, bool, error) {
+	newCluster := cluster.DeepCopy()
+	clusterTaints := newCluster.Spec.Taints
+
+	var newTaints []v1.Taint
+	updated := false
+	for i := range clusterTaints {
+		if taint.MatchTaint(&clusterTaints[i]) {
+			if reflect.DeepEqual(*taint, clusterTaints[i]) {
+				return newCluster, false, nil
+			}
+			newTaints = append(newTaints, *taint)
+			updated = true
+			continue
+		}
+
+		newTaints = append(newTaints, clusterTaints[i])
+	}
+
+	if !updated {
+		newTaints = append(newTaints, *taint)
+	}
+
+	newCluster.Spec.Taints = newTaints
+	return newCluster, true, nil
+}
+
+// TaintExists checks if the given taint exists in list of taints. Returns true if exists false otherwise.
+func TaintExists(taints []v1.Taint, taintToFind *v1.Taint) bool {
+	for _, taint := range taints {
+		if taint.MatchTaint(taintToFind) {
+			return true
+		}
+	}
+	return false
+}
+
+// TaintSetDiff finds the difference between two taint slices and
+// returns all new and removed elements of the new slice relative to the old slice.
+// for example:
+// input: taintsNew=[a b] taintsOld=[a c]
+// output: taintsToAdd=[b] taintsToRemove=[c]
+func TaintSetDiff(taintsNew, taintsOld []v1.Taint) (taintsToAdd []*v1.Taint, taintsToRemove []*v1.Taint) {
+	for _, taint := range taintsNew {
+		if !TaintExists(taintsOld, &taint) {
+			t := taint
+			taintsToAdd = append(taintsToAdd, &t)
+		}
+	}
+
+	for _, taint := range taintsOld {
+		if !TaintExists(taintsNew, &taint) {
+			t := taint
+			taintsToRemove = append(taintsToRemove, &t)
+		}
+	}
+
+	return
+}
+
+// TaintSetFilter filters from the taint slice according to the passed fn function to get the filtered taint slice.
+func TaintSetFilter(taints []v1.Taint, fn func(*v1.Taint) bool) []v1.Taint {
+	res := []v1.Taint{}
+
+	for _, taint := range taints {
+		if fn(&taint) {
+			res = append(res, taint)
+		}
+	}
+
+	return res
+}


Test patch:
diff --git a/pkg/controllers/clusters/clusterlifecycle/clusterlifecycle_controller_test.go b/pkg/controllers/clusters/clusterlifecycle/clusterlifecycle_controller_test.go
index 8c6c92b..379f7a5 100644
--- a/pkg/controllers/clusters/clusterlifecycle/clusterlifecycle_controller_test.go
+++ b/pkg/controllers/clusters/clusterlifecycle/clusterlifecycle_controller_test.go
@@ -26,7 +26,6 @@ import (
 )
 
 func TestGetClusterReadyCondition(t *testing.T) {
-	c := Controller{}
 	heartbeatFrequencySeconds := int64(60)
 
 	tests := []struct {
@@ -116,7 +115,7 @@ func TestGetClusterReadyCondition(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			ClusterConditionAsserts(t, tt.want, c.generateClusterReadyCondition(tt.input))
+			ClusterConditionAsserts(t, tt.want, generateClusterReadyCondition(tt.input))
 		})
 	}
 }
diff --git a/pkg/scheduler/framework/plugins/tainttoleration/taint_toleration_test.go b/pkg/scheduler/framework/plugins/tainttoleration/taint_toleration_test.go
index abfd75d..ba37828 100644
--- a/pkg/scheduler/framework/plugins/tainttoleration/taint_toleration_test.go
+++ b/pkg/scheduler/framework/plugins/tainttoleration/taint_toleration_test.go
@@ -73,23 +73,38 @@ func TestTaintTolerationScore(t *testing.T) {
 		// basic test case
 		{
 			name: "cluster with taints tolerated by the subscription, gets a higher score than those cluster with intolerable taints",
-			subscription: subscriptionWithTolerations("sub1", []v1.Toleration{{
-				Key:      "foo",
-				Operator: v1.TolerationOpEqual,
-				Value:    "bar",
-				Effect:   v1.TaintEffectPreferNoSchedule,
-			}}),
+			subscription: subscriptionWithTolerations(
+				"sub1",
+				[]v1.Toleration{
+					{
+						Key:      "foo",
+						Operator: v1.TolerationOpEqual,
+						Value:    "bar",
+						Effect:   v1.TaintEffectPreferNoSchedule,
+					},
+				},
+			),
 			clusters: []*clusterapi.ManagedCluster{
-				clusterWithTaints("cluster-ns-01", []v1.Taint{{
-					Key:    "foo",
-					Value:  "bar",
-					Effect: v1.TaintEffectPreferNoSchedule,
-				}}),
-				clusterWithTaints("cluster-ns-02", []v1.Taint{{
-					Key:    "foo",
-					Value:  "blah",
-					Effect: v1.TaintEffectPreferNoSchedule,
-				}}),
+				clusterWithTaints(
+					"cluster-ns-01",
+					[]v1.Taint{
+						{
+							Key:    "foo",
+							Value:  "bar",
+							Effect: v1.TaintEffectPreferNoSchedule,
+						},
+					},
+				),
+				clusterWithTaints(
+					"cluster-ns-02",
+					[]v1.Taint{
+						{
+							Key:    "foo",
+							Value:  "blah",
+							Effect: v1.TaintEffectPreferNoSchedule,
+						},
+					},
+				),
 			},
 			expectedList: []framework.ClusterScore{
 				{NamespacedName: "cluster-ns-01/cluster-test", Score: framework.MaxClusterScore},
@@ -105,7 +120,8 @@ func TestTaintTolerationScore(t *testing.T) {
 					Operator: v1.TolerationOpEqual,
 					Value:    "arm64",
 					Effect:   v1.TaintEffectPreferNoSchedule,
-				}, {
+				},
+				{
 					Key:      "disk-type",
 					Operator: v1.TolerationOpEqual,
 					Value:    "ssd",
@@ -114,24 +130,30 @@ func TestTaintTolerationScore(t *testing.T) {
 			}),
 			clusters: []*clusterapi.ManagedCluster{
 				clusterWithTaints("cluster-ns-01", []v1.Taint{}),
-				clusterWithTaints("cluster-ns-02", []v1.Taint{
-					{
-						Key:    "cpu-type",
-						Value:  "arm64",
-						Effect: v1.TaintEffectPreferNoSchedule,
+				clusterWithTaints(
+					"cluster-ns-02",
+					[]v1.Taint{
+						{
+							Key:    "cpu-type",
+							Value:  "arm64",
+							Effect: v1.TaintEffectPreferNoSchedule,
+						},
 					},
-				}),
-				clusterWithTaints("cluster-ns-03", []v1.Taint{
-					{
-						Key:    "cpu-type",
-						Value:  "arm64",
-						Effect: v1.TaintEffectPreferNoSchedule,
-					}, {
-						Key:    "disk-type",
-						Value:  "ssd",
-						Effect: v1.TaintEffectPreferNoSchedule,
+				),
+				clusterWithTaints(
+					"cluster-ns-03",
+					[]v1.Taint{
+						{
+							Key:    "cpu-type",
+							Value:  "arm64",
+							Effect: v1.TaintEffectPreferNoSchedule,
+						}, {
+							Key:    "disk-type",
+							Value:  "ssd",
+							Effect: v1.TaintEffectPreferNoSchedule,
+						},
 					},
-				}),
+				),
 			},
 			expectedList: []framework.ClusterScore{
 				{NamespacedName: "cluster-ns-01/cluster-test", Score: framework.MaxClusterScore},
@@ -142,32 +164,43 @@ func TestTaintTolerationScore(t *testing.T) {
 		// the count of taints on a cluster that are not tolerated by subscription, matters.
 		{
 			name: "the more intolerable taints a cluster has, the lower score it gets.",
-			subscription: subscriptionWithTolerations("sub1", []v1.Toleration{{
-				Key:      "foo",
-				Operator: v1.TolerationOpEqual,
-				Value:    "bar",
-				Effect:   v1.TaintEffectPreferNoSchedule,
-			}}),
+			subscription: subscriptionWithTolerations(
+				"sub1",
+				[]v1.Toleration{
+					{
+						Key:      "foo",
+						Operator: v1.TolerationOpEqual,
+						Value:    "bar",
+						Effect:   v1.TaintEffectPreferNoSchedule,
+					},
+				},
+			),
 			clusters: []*clusterapi.ManagedCluster{
 				clusterWithTaints("cluster-ns-01", []v1.Taint{}),
-				clusterWithTaints("cluster-ns-02", []v1.Taint{
-					{
-						Key:    "cpu-type",
-						Value:  "arm64",
-						Effect: v1.TaintEffectPreferNoSchedule,
+				clusterWithTaints(
+					"cluster-ns-02",
+					[]v1.Taint{
+						{
+							Key:    "cpu-type",
+							Value:  "arm64",
+							Effect: v1.TaintEffectPreferNoSchedule,
+						},
 					},
-				}),
-				clusterWithTaints("cluster-ns-03", []v1.Taint{
-					{
-						Key:    "cpu-type",
-						Value:  "arm64",
-						Effect: v1.TaintEffectPreferNoSchedule,
-					}, {
-						Key:    "disk-type",
-						Value:  "ssd",
-						Effect: v1.TaintEffectPreferNoSchedule,
+				),
+				clusterWithTaints(
+					"cluster-ns-03",
+					[]v1.Taint{
+						{
+							Key:    "cpu-type",
+							Value:  "arm64",
+							Effect: v1.TaintEffectPreferNoSchedule,
+						}, {
+							Key:    "disk-type",
+							Value:  "ssd",
+							Effect: v1.TaintEffectPreferNoSchedule,
+						},
 					},
-				}),
+				),
 			},
 			expectedList: []framework.ClusterScore{
 				{NamespacedName: "cluster-ns-01/cluster-test", Score: framework.MaxClusterScore},
@@ -178,39 +211,48 @@ func TestTaintTolerationScore(t *testing.T) {
 		// taints-tolerations priority only takes care about the taints and tolerations that have effect PreferNoSchedule
 		{
 			name: "only taints and tolerations that have effect PreferNoSchedule are checked by taints-tolerations priority function",
-			subscription: subscriptionWithTolerations("sub1", []v1.Toleration{
-				{
-					Key:      "cpu-type",
-					Operator: v1.TolerationOpEqual,
-					Value:    "arm64",
-					Effect:   v1.TaintEffectNoSchedule,
-				}, {
-					Key:      "disk-type",
-					Operator: v1.TolerationOpEqual,
-					Value:    "ssd",
-					Effect:   v1.TaintEffectNoSchedule,
+			subscription: subscriptionWithTolerations(
+				"sub1",
+				[]v1.Toleration{
+					{
+						Key:      "cpu-type",
+						Operator: v1.TolerationOpEqual,
+						Value:    "arm64",
+						Effect:   v1.TaintEffectNoSchedule,
+					}, {
+						Key:      "disk-type",
+						Operator: v1.TolerationOpEqual,
+						Value:    "ssd",
+						Effect:   v1.TaintEffectNoSchedule,
+					},
 				},
-			}),
+			),
 			clusters: []*clusterapi.ManagedCluster{
 				clusterWithTaints("cluster-ns-01", []v1.Taint{}),
-				clusterWithTaints("cluster-ns-02", []v1.Taint{
-					{
-						Key:    "cpu-type",
-						Value:  "arm64",
-						Effect: v1.TaintEffectNoSchedule,
+				clusterWithTaints(
+					"cluster-ns-02",
+					[]v1.Taint{
+						{
+							Key:    "cpu-type",
+							Value:  "arm64",
+							Effect: v1.TaintEffectNoSchedule,
+						},
 					},
-				}),
-				clusterWithTaints("cluster-ns-03", []v1.Taint{
-					{
-						Key:    "cpu-type",
-						Value:  "arm64",
-						Effect: v1.TaintEffectPreferNoSchedule,
-					}, {
-						Key:    "disk-type",
-						Value:  "ssd",
-						Effect: v1.TaintEffectPreferNoSchedule,
+				),
+				clusterWithTaints(
+					"cluster-ns-03",
+					[]v1.Taint{
+						{
+							Key:    "cpu-type",
+							Value:  "arm64",
+							Effect: v1.TaintEffectPreferNoSchedule,
+						}, {
+							Key:    "disk-type",
+							Value:  "ssd",
+							Effect: v1.TaintEffectPreferNoSchedule,
+						},
 					},
-				}),
+				),
 			},
 			expectedList: []framework.ClusterScore{
 				{NamespacedName: "cluster-ns-01/cluster-test", Score: framework.MaxClusterScore},
@@ -225,13 +267,16 @@ func TestTaintTolerationScore(t *testing.T) {
 			clusters: []*clusterapi.ManagedCluster{
 				// Cluster without taints
 				clusterWithTaints("cluster-ns-01", []v1.Taint{}),
-				clusterWithTaints("cluster-ns-02", []v1.Taint{
-					{
-						Key:    "cpu-type",
-						Value:  "arm64",
-						Effect: v1.TaintEffectPreferNoSchedule,
+				clusterWithTaints(
+					"cluster-ns-02",
+					[]v1.Taint{
+						{
+							Key:    "cpu-type",
+							Value:  "arm64",
+							Effect: v1.TaintEffectPreferNoSchedule,
+						},
 					},
-				}),
+				),
 			},
 			expectedList: []framework.ClusterScore{
 				{NamespacedName: "cluster-ns-01/cluster-test", Score: framework.MaxClusterScore},
@@ -285,63 +330,111 @@ func TestTaintTolerationFilter(t *testing.T) {
 		{
 			name:         "A subscription having no tolerations can't be scheduled onto a cluster with nonempty taints",
 			subscription: subscriptionWithTolerations("sub1", []v1.Toleration{}),
-			cluster:      clusterWithTaints("clusterA", []v1.Taint{{Key: "dedicated", Value: "user1", Effect: "NoSchedule"}}),
+			cluster: clusterWithTaints(
+				"clusterA",
+				[]v1.Taint{{Key: "dedicated", Value: "user1", Effect: "NoSchedule"}},
+			),
 			wantStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable,
 				"clusters(s) had taint {dedicated: user1}, that the subscription didn't tolerate"),
 		},
 		{
-			name:         "A subscription which can be scheduled on a dedicated cluster assigned to user1 with effect NoSchedule",
-			subscription: subscriptionWithTolerations("sub1", []v1.Toleration{{Key: "dedicated", Value: "user1", Effect: "NoSchedule"}}),
-			cluster:      clusterWithTaints("clusterA", []v1.Taint{{Key: "dedicated", Value: "user1", Effect: "NoSchedule"}}),
+			name: "A subscription which can be scheduled on a dedicated cluster assigned to user1 with effect NoSchedule",
+			subscription: subscriptionWithTolerations(
+				"sub1",
+				[]v1.Toleration{{Key: "dedicated", Value: "user1", Effect: "NoSchedule"}},
+			),
+			cluster: clusterWithTaints(
+				"clusterA",
+				[]v1.Taint{{Key: "dedicated", Value: "user1", Effect: "NoSchedule"}},
+			),
 		},
 		{
-			name:         "A subscription which can't be scheduled on a dedicated cluster assigned to user2 with effect NoSchedule",
-			subscription: subscriptionWithTolerations("sub1", []v1.Toleration{{Key: "dedicated", Operator: "Equal", Value: "user2", Effect: "NoSchedule"}}),
-			cluster:      clusterWithTaints("clusterA", []v1.Taint{{Key: "dedicated", Value: "user1", Effect: "NoSchedule"}}),
+			name: "A subscription which can't be scheduled on a dedicated cluster assigned to user2 with effect NoSchedule",
+			subscription: subscriptionWithTolerations(
+				"sub1",
+				[]v1.Toleration{{Key: "dedicated", Operator: "Equal", Value: "user2", Effect: "NoSchedule"}},
+			),
+			cluster: clusterWithTaints(
+				"clusterA",
+				[]v1.Taint{{Key: "dedicated", Value: "user1", Effect: "NoSchedule"}},
+			),
 			wantStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable,
 				"clusters(s) had taint {dedicated: user1}, that the subscription didn't tolerate"),
 		},
 		{
-			name:         "A subscription can be scheduled onto the cluster, with a toleration uses operator Exists that tolerates the taints on the cluster",
-			subscription: subscriptionWithTolerations("sub1", []v1.Toleration{{Key: "foo", Operator: "Exists", Effect: "NoSchedule"}}),
-			cluster:      clusterWithTaints("clusterA", []v1.Taint{{Key: "foo", Value: "bar", Effect: "NoSchedule"}}),
+			name: "A subscription can be scheduled onto the cluster, with a toleration uses operator Exists that tolerates the taints on the cluster",
+			subscription: subscriptionWithTolerations(
+				"sub1",
+				[]v1.Toleration{{Key: "foo", Operator: "Exists", Effect: "NoSchedule"}},
+			),
+			cluster: clusterWithTaints(
+				"clusterA",
+				[]v1.Taint{{Key: "foo", Value: "bar", Effect: "NoSchedule"}},
+			),
 		},
 		{
 			name: "A subscription has multiple tolerations, cluster has multiple taints, all the taints are tolerated, subscription can be scheduled onto the cluster",
-			subscription: subscriptionWithTolerations("sub1", []v1.Toleration{
-				{Key: "dedicated", Operator: "Equal", Value: "user2", Effect: "NoSchedule"},
-				{Key: "foo", Operator: "Exists", Effect: "NoSchedule"},
-			}),
-			cluster: clusterWithTaints("clusterA", []v1.Taint{
-				{Key: "dedicated", Value: "user2", Effect: "NoSchedule"},
-				{Key: "foo", Value: "bar", Effect: "NoSchedule"},
-			}),
+			subscription: subscriptionWithTolerations(
+				"sub1",
+				[]v1.Toleration{
+					{Key: "dedicated", Operator: "Equal", Value: "user2", Effect: "NoSchedule"},
+					{Key: "foo", Operator: "Exists", Effect: "NoSchedule"},
+				},
+			),
+			cluster: clusterWithTaints(
+				"clusterA",
+				[]v1.Taint{
+					{Key: "dedicated", Value: "user2", Effect: "NoSchedule"},
+					{Key: "foo", Value: "bar", Effect: "NoSchedule"},
+				},
+			),
 		},
 		{
 			name: "A subscription has a toleration that keys and values match the taint on the cluster, but (non-empty) effect doesn't match, " +
 				"can't be scheduled onto the cluster",
-			subscription: subscriptionWithTolerations("sub1", []v1.Toleration{{Key: "foo", Operator: "Equal", Value: "bar", Effect: "PreferNoSchedule"}}),
-			cluster:      clusterWithTaints("clusterA", []v1.Taint{{Key: "foo", Value: "bar", Effect: "NoSchedule"}}),
+			subscription: subscriptionWithTolerations(
+				"sub1",
+				[]v1.Toleration{{Key: "foo", Operator: "Equal", Value: "bar", Effect: "PreferNoSchedule"}},
+			),
+			cluster: clusterWithTaints(
+				"clusterA",
+				[]v1.Taint{{Key: "foo", Value: "bar", Effect: "NoSchedule"}},
+			),
 			wantStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable,
 				"clusters(s) had taint {foo: bar}, that the subscription didn't tolerate"),
 		},
 		{
 			name: "The subscription has a toleration that keys and values match the taint on the cluster, the effect of toleration is empty, " +
 				"and the effect of taint is NoSchedule. Subscription can be scheduled onto the cluster",
-			subscription: subscriptionWithTolerations("sub1", []v1.Toleration{{Key: "foo", Operator: "Equal", Value: "bar"}}),
-			cluster:      clusterWithTaints("clusterA", []v1.Taint{{Key: "foo", Value: "bar", Effect: "NoSchedule"}}),
+			subscription: subscriptionWithTolerations(
+				"sub1",
+				[]v1.Toleration{{Key: "foo", Operator: "Equal", Value: "bar"}},
+			),
+			cluster: clusterWithTaints(
+				"clusterA",
+				[]v1.Taint{{Key: "foo", Value: "bar", Effect: "NoSchedule"}},
+			),
 		},
 		{
 			name: "The subscription has a toleration that key and value don't match the taint on the cluster, " +
 				"but the effect of taint on cluster is PreferNoSchedule. Subscription can be scheduled onto the cluster",
-			subscription: subscriptionWithTolerations("sub1", []v1.Toleration{{Key: "dedicated", Operator: "Equal", Value: "user2", Effect: "NoSchedule"}}),
-			cluster:      clusterWithTaints("clusterA", []v1.Taint{{Key: "dedicated", Value: "user1", Effect: "PreferNoSchedule"}}),
+			subscription: subscriptionWithTolerations(
+				"sub1",
+				[]v1.Toleration{{Key: "dedicated", Operator: "Equal", Value: "user2", Effect: "NoSchedule"}},
+			),
+			cluster: clusterWithTaints(
+				"clusterA",
+				[]v1.Taint{{Key: "dedicated", Value: "user1", Effect: "PreferNoSchedule"}},
+			),
 		},
 		{
 			name: "The subscription has no toleration, " +
 				"but the effect of taint on cluster is PreferNoSchedule. Subscription can be scheduled onto the cluster",
 			subscription: subscriptionWithTolerations("sub1", []v1.Toleration{}),
-			cluster:      clusterWithTaints("clusterA", []v1.Taint{{Key: "dedicated", Value: "user1", Effect: "PreferNoSchedule"}}),
+			cluster: clusterWithTaints(
+				"clusterA",
+				[]v1.Taint{{Key: "dedicated", Value: "user1", Effect: "PreferNoSchedule"}},
+			),
 		},
 	}
 	for _, test := range tests {
diff --git a/pkg/utils/deployer_test.go b/pkg/utils/deployer_test.go
index 618c252..5e6ca9a 100644
--- a/pkg/utils/deployer_test.go
+++ b/pkg/utils/deployer_test.go
@@ -19,7 +19,10 @@ package utils
 import (
 	"testing"
 
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+
+	clusterapi "github.com/clusternet/clusternet/pkg/apis/clusters/v1beta1"
 )
 
 func TestResourceNeedResync(t *testing.T) {
@@ -234,3 +237,84 @@ func TestResourceNeedResync(t *testing.T) {
 		})
 	}
 }
+
+func TestClusterHasReadyCondition(t *testing.T) {
+	tests := []struct {
+		name         string
+		mc           *clusterapi.ManagedCluster
+		clusterReady bool
+	}{
+		{
+			mc: &clusterapi.ManagedCluster{
+				TypeMeta:   metav1.TypeMeta{},
+				ObjectMeta: metav1.ObjectMeta{},
+				Spec:       clusterapi.ManagedClusterSpec{},
+			},
+			clusterReady: true,
+		},
+		{
+			mc: &clusterapi.ManagedCluster{
+				TypeMeta:   metav1.TypeMeta{},
+				ObjectMeta: metav1.ObjectMeta{},
+				Spec:       clusterapi.ManagedClusterSpec{},
+				Status: clusterapi.ManagedClusterStatus{
+					Conditions: []metav1.Condition{
+						{
+							LastTransitionTime: metav1.Now(),
+							Message:            "managed cluster is ready.",
+							Reason:             "ManagedClusterReady",
+							Status:             metav1.ConditionTrue,
+							Type:               clusterapi.ClusterReady,
+						},
+					},
+				},
+			},
+			clusterReady: true,
+		},
+		{
+			mc: &clusterapi.ManagedCluster{
+				TypeMeta:   metav1.TypeMeta{},
+				ObjectMeta: metav1.ObjectMeta{},
+				Spec:       clusterapi.ManagedClusterSpec{},
+				Status: clusterapi.ManagedClusterStatus{
+					Conditions: []metav1.Condition{
+						{
+							LastTransitionTime: metav1.Now(),
+							Message:            "managed cluster is in unknown status",
+							Reason:             "ManagedClusterUnknown",
+							Status:             metav1.ConditionUnknown,
+							Type:               clusterapi.ClusterReady,
+						},
+					},
+				},
+			},
+			clusterReady: false,
+		},
+		{
+			mc: &clusterapi.ManagedCluster{
+				TypeMeta:   metav1.TypeMeta{},
+				ObjectMeta: metav1.ObjectMeta{},
+				Spec:       clusterapi.ManagedClusterSpec{},
+				Status: clusterapi.ManagedClusterStatus{
+					Conditions: []metav1.Condition{
+						{
+							LastTransitionTime: metav1.Now(),
+							Message:            "managed cluster is not ready",
+							Reason:             "ManagedClusterNotReady",
+							Status:             metav1.ConditionFalse,
+							Type:               clusterapi.ClusterReady,
+						},
+					},
+				},
+			},
+			clusterReady: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if isReady := ClusterHasReadyCondition(tt.mc); isReady != tt.clusterReady {
+				t.Errorf("ClusterHasReadyCondition() = %v, want %v", isReady, tt.clusterReady)
+			}
+		})
+	}
+}
diff --git a/pkg/utils/taints/taints_test.go b/pkg/utils/taints/taints_test.go
new file mode 100644
index 0000000..eada8da
--- /dev/null
+++ b/pkg/utils/taints/taints_test.go
@@ -0,0 +1,366 @@
+/*
+Copyright 2023 The Clusternet Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Copied from k8s.io/kubernetes/pkg/util/taints/taints_test.go and modified
+// Some unused functions are deleted as well
+
+package taints
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	corev1 "k8s.io/api/core/v1"
+
+	clusterapi "github.com/clusternet/clusternet/pkg/apis/clusters/v1beta1"
+)
+
+func TestAddOrUpdateTaint(t *testing.T) {
+	taint := corev1.Taint{
+		Key:    "foo",
+		Value:  "bar",
+		Effect: corev1.TaintEffectNoSchedule,
+	}
+
+	taintNew := corev1.Taint{
+		Key:    "foo_1",
+		Value:  "bar_1",
+		Effect: corev1.TaintEffectNoSchedule,
+	}
+
+	taintUpdateValue := taint
+	taintUpdateValue.Value = "bar_1"
+
+	testcases := []struct {
+		name           string
+		cluster        *clusterapi.ManagedCluster
+		taint          *corev1.Taint
+		expectedUpdate bool
+		expectedTaints []corev1.Taint
+	}{
+		{
+			name:           "add a new taint",
+			cluster:        &clusterapi.ManagedCluster{},
+			taint:          &taint,
+			expectedUpdate: true,
+			expectedTaints: []corev1.Taint{taint},
+		},
+		{
+			name: "add a unique taint",
+			cluster: &clusterapi.ManagedCluster{
+				Spec: clusterapi.ManagedClusterSpec{Taints: []corev1.Taint{taint}},
+			},
+			taint:          &taintNew,
+			expectedUpdate: true,
+			expectedTaints: []corev1.Taint{taint, taintNew},
+		},
+		{
+			name: "add duplicate taint",
+			cluster: &clusterapi.ManagedCluster{
+				Spec: clusterapi.ManagedClusterSpec{Taints: []corev1.Taint{taint}},
+			},
+			taint:          &taint,
+			expectedUpdate: false,
+			expectedTaints: []corev1.Taint{taint},
+		},
+		{
+			name: "update taint value",
+			cluster: &clusterapi.ManagedCluster{
+				Spec: clusterapi.ManagedClusterSpec{Taints: []corev1.Taint{taint}},
+			},
+			taint:          &taintUpdateValue,
+			expectedUpdate: true,
+			expectedTaints: []corev1.Taint{taintUpdateValue},
+		},
+	}
+
+	for _, tc := range testcases {
+		t.Run(tc.name, func(t *testing.T) {
+			newNode, updated, err := AddOrUpdateTaint(tc.cluster, tc.taint)
+			if err != nil {
+				t.Errorf("[%s] should not raise error but got %v", tc.name, err)
+			}
+			if updated != tc.expectedUpdate {
+				t.Errorf("[%s] expected taints to not be updated", tc.name)
+			}
+			if diff := cmp.Diff(newNode.Spec.Taints, tc.expectedTaints); diff != "" {
+				t.Errorf("Unexpected result (-want, +got):\n%s", diff)
+			}
+		})
+	}
+}
+
+func TestTaintExists(t *testing.T) {
+	testingTaints := []corev1.Taint{
+		{
+			Key:    "foo_1",
+			Value:  "bar_1",
+			Effect: corev1.TaintEffectNoExecute,
+		},
+		{
+			Key:    "foo_2",
+			Value:  "bar_2",
+			Effect: corev1.TaintEffectNoSchedule,
+		},
+	}
+
+	cases := []struct {
+		name           string
+		taintToFind    *corev1.Taint
+		expectedResult bool
+	}{
+		{
+			name:           "taint exists",
+			taintToFind:    &corev1.Taint{Key: "foo_1", Value: "bar_1", Effect: corev1.TaintEffectNoExecute},
+			expectedResult: true,
+		},
+		{
+			name:           "different key",
+			taintToFind:    &corev1.Taint{Key: "no_such_key", Value: "bar_1", Effect: corev1.TaintEffectNoExecute},
+			expectedResult: false,
+		},
+		{
+			name:           "different effect",
+			taintToFind:    &corev1.Taint{Key: "foo_1", Value: "bar_1", Effect: corev1.TaintEffectNoSchedule},
+			expectedResult: false,
+		},
+	}
+
+	for _, c := range cases {
+		result := TaintExists(testingTaints, c.taintToFind)
+
+		if result != c.expectedResult {
+			t.Errorf("[%s] unexpected results: %v", c.name, result)
+			continue
+		}
+	}
+}
+
+func TestTaintSetFilter(t *testing.T) {
+	testTaint1 := corev1.Taint{
+		Key:    "foo_1",
+		Value:  "bar_1",
+		Effect: corev1.TaintEffectNoExecute,
+	}
+	testTaint2 := corev1.Taint{
+		Key:    "foo_2",
+		Value:  "bar_2",
+		Effect: corev1.TaintEffectNoSchedule,
+	}
+
+	testTaint3 := corev1.Taint{
+		Key:    "foo_3",
+		Value:  "bar_3",
+		Effect: corev1.TaintEffectNoSchedule,
+	}
+	testTaints := []corev1.Taint{testTaint1, testTaint2, testTaint3}
+
+	testcases := []struct {
+		name           string
+		fn             func(t *corev1.Taint) bool
+		expectedTaints []corev1.Taint
+	}{
+		{
+			name: "Filter out nothing",
+			fn: func(t *corev1.Taint) bool {
+				if t.Key == corev1.TaintNodeUnschedulable {
+					return true
+				}
+				return false
+			},
+			expectedTaints: []corev1.Taint{},
+		},
+		{
+			name: "Filter out a subset",
+			fn: func(t *corev1.Taint) bool {
+				if t.Effect == corev1.TaintEffectNoExecute {
+					return true
+				}
+				return false
+			},
+			expectedTaints: []corev1.Taint{testTaint1},
+		},
+		{
+			name:           "Filter out everything",
+			fn:             func(t *corev1.Taint) bool { return true },
+			expectedTaints: []corev1.Taint{testTaint1, testTaint2, testTaint3},
+		},
+	}
+	for _, tc := range testcases {
+		t.Run(tc.name, func(t *testing.T) {
+			taintsAfterFilter := TaintSetFilter(testTaints, tc.fn)
+			if diff := cmp.Diff(tc.expectedTaints, taintsAfterFilter); diff != "" {
+				t.Errorf("Unexpected postFilterResult (-want, +got):\n%s", diff)
+			}
+		})
+	}
+}
+
+func TestRemoveTaint(t *testing.T) {
+	cases := []struct {
+		name           string
+		cluster        *clusterapi.ManagedCluster
+		taintToRemove  *corev1.Taint
+		expectedTaints []corev1.Taint
+		expectedResult bool
+	}{
+		{
+			name: "remove taint unsuccessfully",
+			cluster: &clusterapi.ManagedCluster{
+				Spec: clusterapi.ManagedClusterSpec{
+					Taints: []corev1.Taint{
+						{
+							Key:    "foo",
+							Effect: corev1.TaintEffectNoSchedule,
+						},
+					},
+				},
+			},
+			taintToRemove: &corev1.Taint{
+				Key:    "foo_1",
+				Effect: corev1.TaintEffectNoSchedule,
+			},
+			expectedTaints: []corev1.Taint{
+				{
+					Key:    "foo",
+					Effect: corev1.TaintEffectNoSchedule,
+				},
+			},
+			expectedResult: false,
+		},
+		{
+			name: "remove taint successfully",
+			cluster: &clusterapi.ManagedCluster{
+				Spec: clusterapi.ManagedClusterSpec{
+					Taints: []corev1.Taint{
+						{
+							Key:    "foo",
+							Effect: corev1.TaintEffectNoSchedule,
+						},
+					},
+				},
+			},
+			taintToRemove: &corev1.Taint{
+				Key:    "foo",
+				Effect: corev1.TaintEffectNoSchedule,
+			},
+			expectedTaints: []corev1.Taint{},
+			expectedResult: true,
+		},
+		{
+			name: "remove taint from cluster with no taint",
+			cluster: &clusterapi.ManagedCluster{
+				Spec: clusterapi.ManagedClusterSpec{
+					Taints: []corev1.Taint{},
+				},
+			},
+			taintToRemove: &corev1.Taint{
+				Key:    "foo",
+				Effect: corev1.TaintEffectNoSchedule,
+			},
+			expectedTaints: []corev1.Taint{},
+			expectedResult: false,
+		},
+	}
+
+	for _, c := range cases {
+		newNode, result, err := RemoveTaint(c.cluster, c.taintToRemove)
+		if err != nil {
+			t.Errorf("[%s] should not raise error but got: %v", c.name, err)
+		}
+		if result != c.expectedResult {
+			t.Errorf("[%s] should return %t, but got: %t", c.name, c.expectedResult, result)
+		}
+		if !reflect.DeepEqual(newNode.Spec.Taints, c.expectedTaints) {
+			t.Errorf("[%s] the new cluster object should have taints %v, but got: %v", c.name, c.expectedTaints, newNode.Spec.Taints)
+		}
+	}
+}
+
+func TestDeleteTaint(t *testing.T) {
+	cases := []struct {
+		name           string
+		taints         []corev1.Taint
+		taintToDelete  *corev1.Taint
+		expectedTaints []corev1.Taint
+		expectedResult bool
+	}{
+		{
+			name: "delete taint with different name",
+			taints: []corev1.Taint{
+				{
+					Key:    "foo",
+					Effect: corev1.TaintEffectNoSchedule,
+				},
+			},
+			taintToDelete: &corev1.Taint{Key: "foo_1", Effect: corev1.TaintEffectNoSchedule},
+			expectedTaints: []corev1.Taint{
+				{
+					Key:    "foo",
+					Effect: corev1.TaintEffectNoSchedule,
+				},
+			},
+			expectedResult: false,
+		},
+		{
+			name: "delete taint with different effect",
+			taints: []corev1.Taint{
+				{
+					Key:    "foo",
+					Effect: corev1.TaintEffectNoSchedule,
+				},
+			},
+			taintToDelete: &corev1.Taint{Key: "foo", Effect: corev1.TaintEffectNoExecute},
+			expectedTaints: []corev1.Taint{
+				{
+					Key:    "foo",
+					Effect: corev1.TaintEffectNoSchedule,
+				},
+			},
+			expectedResult: false,
+		},
+		{
+			name: "delete taint successfully",
+			taints: []corev1.Taint{
+				{
+					Key:    "foo",
+					Effect: corev1.TaintEffectNoSchedule,
+				},
+			},
+			taintToDelete:  &corev1.Taint{Key: "foo", Effect: corev1.TaintEffectNoSchedule},
+			expectedTaints: []corev1.Taint{},
+			expectedResult: true,
+		},
+		{
+			name:           "delete taint from empty taint array",
+			taints:         []corev1.Taint{},
+			taintToDelete:  &corev1.Taint{Key: "foo", Effect: corev1.TaintEffectNoSchedule},
+			expectedTaints: []corev1.Taint{},
+			expectedResult: false,
+		},
+	}
+
+	for _, c := range cases {
+		taints, result := DeleteTaint(c.taints, c.taintToDelete)
+		if result != c.expectedResult {
+			t.Errorf("[%s] should return %t, but got: %t", c.name, c.expectedResult, result)
+		}
+		if !reflect.DeepEqual(taints, c.expectedTaints) {
+			t.Errorf("[%s] the result taints should be %v, but got: %v", c.name, c.expectedTaints, taints)
+		}
+	}
+}