Repository: apecloud/kubeblocks

Issue:
[BUG] Failed to adopt existing Replicas to templates
1. create a mysql cluster
```yaml
kind: Cluster
metadata:
  name: mycluster
  namespace: demo
spec:
  terminationPolicy: Delete
  componentSpecs:
    - name: mysql
      componentDef: mysql-8.0
      serviceVersion: "8.0.35"
      replicas: 2
      flatInstanceOrdinal: true  # Enable flat ordinal naming
      resources:
        limits:
          cpu: '0.5'
          memory: 0.5Gi
        requests:
          cpu: '0.5'
          memory: 0.5Gi
      volumeClaimTemplates:
        - name: data
          spec:
            storageClassName: ""
            accessModes:
              - ReadWriteOnce
            resources:
              requests:
                storage: 10Gi
```

2. adopt replica-0 to template tpl-0, everything works as expected.
```yaml
apiVersion: apps.kubeblocks.io/v1
kind: Cluster
metadata:
  name: mycluster
  namespace: demo
spec:
  terminationPolicy: Delete
  componentSpecs:
    - name: mysql
      componentDef: mysql-8.0
      serviceVersion: "8.0.35"
      replicas: 2
      flatInstanceOrdinal: true  # Enable flat ordinal naming
      resources:
        limits:
          cpu: '0.5'
          memory: 0.5Gi
        requests:
          cpu: '0.5'
          memory: 0.5Gi
      volumeClaimTemplates:
        - name: data
          spec:
            storageClassName: ""
            accessModes:
              - ReadWriteOnce
            resources:
              requests:
                storage: 20Gi
      instances:
        - name: tpl-0
          replicas: 1
          ordinals:
            discrete: [0]  # Take over pod mysql-1
          volumeClaimTemplates:
            - name: data
              spec:
                storageClassName: ""
                accessModes:
                  - ReadWriteOnce
                resources:
                  requests:
                    storage: 20Gi   
```

Replica-0 is adopted by template tpl-0 and its status is 
```yaml
❯ k get its mycluster2-mysql -oyaml | yq .status
availableReplicas: 2
currentReplicas: 2
initReplicas: 0
instanceStatus:
  - podName: mycluster2-mysql-0
    role: secondary
  - podName: mycluster2-mysql-1
    role: primary
ordinals:
  - 1    # 1 is not taken over yet, using default template.
readyReplicas: 2
replicas: 2
templatesStatus:
  - availableReplicas: 1   # status for tpl-0, it take over replica-0
    currentReplicas: 1
    name: tpl-0
    ordinals:
      - 0
    readyReplicas: 1
    replicas: 1
    updatedReplicas: 1
updatedReplicas: 2
```

3. continue to adopt replica-1 to tpl-1
```yaml
apiVersion: apps.kubeblocks.io/v1
kind: Cluster
metadata:
  name: mycluster
  namespace: demo
spec:
  terminationPolicy: Delete
  componentSpecs:
    - name: mysql
      componentDef: mysql-8.0
      serviceVersion: "8.0.35"
      replicas: 2
      flatInstanceOrdinal: true  # Enable flat ordinal naming
      resources:
        limits:
          cpu: '0.5'
          memory: 0.5Gi
        requests:
          cpu: '0.5'
          memory: 0.5Gi
      volumeClaimTemplates:
        - name: data
          spec:
            storageClassName: ""
            accessModes:
              - ReadWriteOnce
            resources:
              requests:
                storage: 20Gi
      instances:
        - name: tpl-0
          replicas: 1
          ordinals:
            discrete: [0]  # Take over pod mysql-0
          volumeClaimTemplates:
            - name: data
              spec:
                storageClassName: ""
                accessModes:
                  - ReadWriteOnce
                resources:
                  requests:
                    storage: 20Gi                                   
        - name: tpl-1
          replicas: 1
          ordinals:
            discrete: [1]  # Take over pod mysql-1
          volumeClaimTemplates:
            - name: data
              spec:
                storageClassName: ""
                accessModes:
                  - ReadWriteOnce
                resources:
                  requests:
                    storage: 20Gi                           

```
replicas-1 will be in loops of delete -> create -> delete -> create -> delete

ITS status is 
```yaml
availableReplicas: 1
initReplicas: 0
instanceStatus:
  - podName: mycluster2-mysql-0
    role: primary
  - podName: mycluster2-mysql-1
observedGeneration: 6
ordinals:
  - 1      #  1 is not taken over
readyReplicas: 1
replicas: 2
templatesStatus:          # and there is no status for tpl-1 in templatesStatus
  - availableReplicas: 1
    currentReplicas: 1
    name: tpl-0
    ordinals:
      - 0
    readyReplicas: 1
    replicas: 1
    updatedReplicas: 1
updatedReplicas: 1
```

replicas-1 should be adopted by tpl-1, the  pod created is labeled with `apps.kubeblocks.io/instance-template: tpl-1` as expected.
but the ITS status is incorrect and there is no `tpl-1` in `templatesStatus`.

Expected ITS status is 
```yaml
status:
  availableReplicas: 2
  initReplicas: 0
  instanceStatus:
  - podName: mycluster-mysql-0
    role: primary
  - podName: mycluster-mysql-1
    role: secondary
  observedGeneration: 1
  readyReplicas: 2
  replicas: 2
  templatesStatus:
  - availableReplicas: 1
    currentReplicas: 1
    name: tpl-0
    ordinals:
    - 0
    readyReplicas: 1
    replicas: 1
    updatedReplicas: 1
  - availableReplicas: 1    # missing tpl-1 in templatesStatus
    currentReplicas: 1
    name: tpl-1
    ordinals:
    - 1
    readyReplicas: 1
    replicas: 1
    updatedReplicas: 1
  updatedReplicas: 2
```

Discussion:
@shanshanying: The disered Tree expected only ONE available replicas (which is wrong), and it keeps deleting newly created pod.@shanshanying: and failed to release the adoptions from tpl
e.g. create pods with one template

```yaml
apiVersion: apps.kubeblocks.io/v1
kind: Cluster
metadata:
  name: mycluster
  namespace: demo
spec:
  terminationPolicy: Delete
  componentSpecs:
    - name: mysql
      componentDef: mysql-8.0
      serviceVersion: "8.0.35"
      replicas: 2
      flatInstanceOrdinal: true  # Enable flat ordinal naming
      resources:
        limits:
          cpu: '0.5'
          memory: 0.5Gi
        requests:
          cpu: '0.5'
          memory: 0.5Gi
      volumeClaimTemplates:
        - name: data
          spec:
            storageClassName: ""
            accessModes:
              - ReadWriteOnce
            resources:
              requests:
                storage: 20Gi
      instances:
        - name: tpl-0
          replicas: 1
          ordinals:
            discrete: [0]  # Take over pod mysql-0
          volumeClaimTemplates:
            - name: data
              spec:
                storageClassName: ""
                accessModes:
                  - ReadWriteOnce
                resources:
                  requests:
                    storage: 20Gi      
```

If we remove instance template, it will 
1. createa new pod pod-2 first
2. delete pod-0 later.

Expected behavior is :   replica-0 using the default template, instead of creating replcia-2.

@leon-inf: > and failed to release the adoptions from tpl e.g. create pods with one template
> 
> apiVersion: apps.kubeblocks.io/v1
> kind: Cluster
> metadata:
>   name: mycluster
>   namespace: demo
> spec:
>   terminationPolicy: Delete
>   componentSpecs:
>     - name: mysql
>       componentDef: mysql-8.0
>       serviceVersion: "8.0.35"
>       replicas: 2
>       flatInstanceOrdinal: true  # Enable flat ordinal naming
>       resources:
>         limits:
>           cpu: '0.5'
>           memory: 0.5Gi
>         requests:
>           cpu: '0.5'
>           memory: 0.5Gi
>       volumeClaimTemplates:
>         - name: data
>           spec:
>             storageClassName: ""
>             accessModes:
>               - ReadWriteOnce
>             resources:
>               requests:
>                 storage: 20Gi
>       instances:
>         - name: tpl-0
>           replicas: 1
>           ordinals:
>             discrete: [0]  # Take over pod mysql-0
>           volumeClaimTemplates:
>             - name: data
>               spec:
>                 storageClassName: ""
>                 accessModes:
>                   - ReadWriteOnce
>                 resources:
>                   requests:
>                     storage: 20Gi      
> If we remove instance template, it will
> 
> 1. createa new pod pod-2 first
> 2. delete pod-0 later.
> 
> Expected behavior is : replica-0 using the default template, instead of creating replcia-2.

Using ordinal 2 is by design. There is no support yet for keeping the ordinal unchanged when taking back.


Solution patch:
diff --git a/apis/apps/v1/cluster_types.go b/apis/apps/v1/cluster_types.go
index b189dc841..668509125 100644
--- a/apis/apps/v1/cluster_types.go
+++ b/apis/apps/v1/cluster_types.go
@@ -494,6 +494,13 @@ type ClusterComponentSpec struct {
 	// +listMapKey=name
 	Instances []InstanceTemplate `json:"instances,omitempty" patchStrategy:"merge,retainKeys" patchMergeKey:"name"`
 
+	// Specifies the desired Ordinals.
+	// The Ordinals used to specify the ordinal of the instance (pod) names to be generated under this component.
+	// If Ordinals are defined, their number must be equal to or more than the corresponding replicas.
+	//
+	// +optional
+	Ordinals Ordinals `json:"ordinals,omitempty"`
+
 	// flatInstanceOrdinal controls whether the naming of instances(pods) under this component uses a flattened,
 	// globally uniquely ordinal scheme, regardless of the instance template.
 	//
@@ -746,6 +753,11 @@ type ShardTemplate struct {
 	// +optional
 	Instances []InstanceTemplate `json:"instances,omitempty"`
 
+	// Specifies an override for the desired Ordinals of the shard.
+	//
+	// +optional
+	Ordinals *Ordinals `json:"ordinals,omitempty"`
+
 	// Specifies an override for the instance naming of the shard.
 	//
 	// +optional
diff --git a/apis/apps/v1/component_types.go b/apis/apps/v1/component_types.go
index 201401bf6..b692b1d2a 100644
--- a/apis/apps/v1/component_types.go
+++ b/apis/apps/v1/component_types.go
@@ -279,6 +279,13 @@ type ComponentSpec struct {
 	// +optional
 	Instances []InstanceTemplate `json:"instances,omitempty" patchStrategy:"merge,retainKeys" patchMergeKey:"name"`
 
+	// Specifies the desired Ordinals.
+	// The Ordinals used to specify the ordinal of the instance (pod) names to be generated under this component.
+	// If Ordinals are defined, their number must be equal to or more than the corresponding replicas.
+	//
+	// +optional
+	Ordinals Ordinals `json:"ordinals,omitempty"`
+
 	// flatInstanceOrdinal controls whether the naming of instances(pods) under this component uses a flattened,
 	// globally uniquely ordinal scheme, regardless of the instance template.
 	//
diff --git a/apis/apps/v1/zz_generated.deepcopy.go b/apis/apps/v1/zz_generated.deepcopy.go
index b76820cde..5d78c0bd6 100644
--- a/apis/apps/v1/zz_generated.deepcopy.go
+++ b/apis/apps/v1/zz_generated.deepcopy.go
@@ -388,6 +388,7 @@ func (in *ClusterComponentSpec) DeepCopyInto(out *ClusterComponentSpec) {
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
 	}
+	in.Ordinals.DeepCopyInto(&out.Ordinals)
 	if in.OfflineInstances != nil {
 		in, out := &in.OfflineInstances, &out.OfflineInstances
 		*out = make([]string, len(*in))
@@ -1598,6 +1599,7 @@ func (in *ComponentSpec) DeepCopyInto(out *ComponentSpec) {
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
 	}
+	in.Ordinals.DeepCopyInto(&out.Ordinals)
 	if in.OfflineInstances != nil {
 		in, out := &in.OfflineInstances, &out.OfflineInstances
 		*out = make([]string, len(*in))
@@ -3291,6 +3293,11 @@ func (in *ShardTemplate) DeepCopyInto(out *ShardTemplate) {
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
 	}
+	if in.Ordinals != nil {
+		in, out := &in.Ordinals, &out.Ordinals
+		*out = new(Ordinals)
+		(*in).DeepCopyInto(*out)
+	}
 	if in.FlatInstanceOrdinal != nil {
 		in, out := &in.FlatInstanceOrdinal, &out.FlatInstanceOrdinal
 		*out = new(bool)
diff --git a/config/crd/bases/apps.kubeblocks.io_clusters.yaml b/config/crd/bases/apps.kubeblocks.io_clusters.yaml
index 099f93f54..83ce3fafc 100644
--- a/config/crd/bases/apps.kubeblocks.io_clusters.yaml
+++ b/config/crd/bases/apps.kubeblocks.io_clusters.yaml
@@ -2828,6 +2828,35 @@ spec:
                       items:
                         type: string
                       type: array
+                    ordinals:
+                      description: |-
+                        Specifies the desired Ordinals.
+                        The Ordinals used to specify the ordinal of the instance (pod) names to be generated under this component.
+                        If Ordinals are defined, their number must be equal to or more than the corresponding replicas.
+                      properties:
+                        discrete:
+                          items:
+                            format: int32
+                            type: integer
+                          type: array
+                        ranges:
+                          items:
+                            description: |-
+                              Range represents a range with a start and an end value. Both start and end are included.
+                              It is used to define a continuous segment.
+                            properties:
+                              end:
+                                format: int32
+                                type: integer
+                              start:
+                                format: int32
+                                type: integer
+                            required:
+                            - end
+                            - start
+                            type: object
+                          type: array
+                      type: object
                     parallelPodManagementConcurrency:
                       anyOf:
                       - type: integer
@@ -9892,6 +9921,33 @@ spec:
                             maxLength: 15
                             pattern: ^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$
                             type: string
+                          ordinals:
+                            description: Specifies an override for the desired Ordinals
+                              of the shard.
+                            properties:
+                              discrete:
+                                items:
+                                  format: int32
+                                  type: integer
+                                type: array
+                              ranges:
+                                items:
+                                  description: |-
+                                    Range represents a range with a start and an end value. Both start and end are included.
+                                    It is used to define a continuous segment.
+                                  properties:
+                                    end:
+                                      format: int32
+                                      type: integer
+                                    start:
+                                      format: int32
+                                      type: integer
+                                  required:
+                                  - end
+                                  - start
+                                  type: object
+                                type: array
+                            type: object
                           replicas:
                             default: 1
                             description: Specifies the desired number of replicas
@@ -14112,6 +14168,35 @@ spec:
                           items:
                             type: string
                           type: array
+                        ordinals:
+                          description: |-
+                            Specifies the desired Ordinals.
+                            The Ordinals used to specify the ordinal of the instance (pod) names to be generated under this component.
+                            If Ordinals are defined, their number must be equal to or more than the corresponding replicas.
+                          properties:
+                            discrete:
+                              items:
+                                format: int32
+                                type: integer
+                              type: array
+                            ranges:
+                              items:
+                                description: |-
+                                  Range represents a range with a start and an end value. Both start and end are included.
+                                  It is used to define a continuous segment.
+                                properties:
+                                  end:
+                                    format: int32
+                                    type: integer
+                                  start:
+                                    format: int32
+                                    type: integer
+                                required:
+                                - end
+                                - start
+                                type: object
+                              type: array
+                          type: object
                         parallelPodManagementConcurrency:
                           anyOf:
                           - type: integer
diff --git a/config/crd/bases/apps.kubeblocks.io_components.yaml b/config/crd/bases/apps.kubeblocks.io_components.yaml
index 1ba48b17b..d7a42b250 100644
--- a/config/crd/bases/apps.kubeblocks.io_components.yaml
+++ b/config/crd/bases/apps.kubeblocks.io_components.yaml
@@ -2609,6 +2609,35 @@ spec:
                 items:
                   type: string
                 type: array
+              ordinals:
+                description: |-
+                  Specifies the desired Ordinals.
+                  The Ordinals used to specify the ordinal of the instance (pod) names to be generated under this component.
+                  If Ordinals are defined, their number must be equal to or more than the corresponding replicas.
+                properties:
+                  discrete:
+                    items:
+                      format: int32
+                      type: integer
+                    type: array
+                  ranges:
+                    items:
+                      description: |-
+                        Range represents a range with a start and an end value. Both start and end are included.
+                        It is used to define a continuous segment.
+                      properties:
+                        end:
+                          format: int32
+                          type: integer
+                        start:
+                          format: int32
+                          type: integer
+                      required:
+                      - end
+                      - start
+                      type: object
+                    type: array
+                type: object
               parallelPodManagementConcurrency:
                 anyOf:
                 - type: integer
diff --git a/controllers/apps/cluster/transformer_cluster_component.go b/controllers/apps/cluster/transformer_cluster_component.go
index e0aab523d..0d1eea622 100644
--- a/controllers/apps/cluster/transformer_cluster_component.go
+++ b/controllers/apps/cluster/transformer_cluster_component.go
@@ -279,6 +279,7 @@ func copyAndMergeComponent(oldCompObj, newCompObj *appsv1.Component) *appsv1.Com
 	compObjCopy.Spec.SchedulingPolicy = compProto.Spec.SchedulingPolicy
 	compObjCopy.Spec.TLSConfig = compProto.Spec.TLSConfig
 	compObjCopy.Spec.Instances = compProto.Spec.Instances
+	compObjCopy.Spec.Ordinals = compProto.Spec.Ordinals
 	compObjCopy.Spec.FlatInstanceOrdinal = compProto.Spec.FlatInstanceOrdinal
 	compObjCopy.Spec.OfflineInstances = compProto.Spec.OfflineInstances
 	compObjCopy.Spec.RuntimeClassName = compProto.Spec.RuntimeClassName
diff --git a/controllers/apps/component/transformer_component_template.go b/controllers/apps/component/transformer_component_template.go
index f0049bb97..2b21040be 100644
--- a/controllers/apps/component/transformer_component_template.go
+++ b/controllers/apps/component/transformer_component_template.go
@@ -21,7 +21,6 @@ package component
 
 import (
 	"fmt"
-	"maps"
 	"reflect"
 	"strings"
 	"text/template"
@@ -258,9 +257,9 @@ func renderFileTemplateData(transCtx *componentTransformContext,
 		rendered        = make(map[string]string)
 	)
 
-	variables := make(map[string]string)
-	if synthesizedComp.TemplateVars != nil {
-		maps.Copy(variables, synthesizedComp.TemplateVars)
+	variables := make(map[string]any)
+	for k, v := range synthesizedComp.TemplateVars {
+		variables[k] = v
 	}
 	for k, v := range fileTemplate.Variables {
 		variables[k] = v // override
diff --git a/controllers/apps/component/transformer_component_workload.go b/controllers/apps/component/transformer_component_workload.go
index 127d3f26a..7b4b0e2a1 100644
--- a/controllers/apps/component/transformer_component_workload.go
+++ b/controllers/apps/component/transformer_component_workload.go
@@ -342,6 +342,7 @@ func copyAndMergeITS(oldITS, newITS *workloads.InstanceSet) *workloads.InstanceS
 	itsObjCopy.Spec.Roles = itsProto.Spec.Roles
 	itsObjCopy.Spec.LifecycleActions = itsProto.Spec.LifecycleActions
 	itsObjCopy.Spec.Instances = itsProto.Spec.Instances
+	itsObjCopy.Spec.DefaultTemplateOrdinals = itsProto.Spec.DefaultTemplateOrdinals
 	itsObjCopy.Spec.FlatInstanceOrdinal = itsProto.Spec.FlatInstanceOrdinal
 	itsObjCopy.Spec.OfflineInstances = itsProto.Spec.OfflineInstances
 	itsObjCopy.Spec.MinReadySeconds = itsProto.Spec.MinReadySeconds
diff --git a/deploy/helm/crds/apps.kubeblocks.io_clusters.yaml b/deploy/helm/crds/apps.kubeblocks.io_clusters.yaml
index 099f93f54..83ce3fafc 100644
--- a/deploy/helm/crds/apps.kubeblocks.io_clusters.yaml
+++ b/deploy/helm/crds/apps.kubeblocks.io_clusters.yaml
@@ -2828,6 +2828,35 @@ spec:
                       items:
                         type: string
                       type: array
+                    ordinals:
+                      description: |-
+                        Specifies the desired Ordinals.
+                        The Ordinals used to specify the ordinal of the instance (pod) names to be generated under this component.
+                        If Ordinals are defined, their number must be equal to or more than the corresponding replicas.
+                      properties:
+                        discrete:
+                          items:
+                            format: int32
+                            type: integer
+                          type: array
+                        ranges:
+                          items:
+                            description: |-
+                              Range represents a range with a start and an end value. Both start and end are included.
+                              It is used to define a continuous segment.
+                            properties:
+                              end:
+                                format: int32
+                                type: integer
+                              start:
+                                format: int32
+                                type: integer
+                            required:
+                            - end
+                            - start
+                            type: object
+                          type: array
+                      type: object
                     parallelPodManagementConcurrency:
                       anyOf:
                       - type: integer
@@ -9892,6 +9921,33 @@ spec:
                             maxLength: 15
                             pattern: ^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$
                             type: string
+                          ordinals:
+                            description: Specifies an override for the desired Ordinals
+                              of the shard.
+                            properties:
+                              discrete:
+                                items:
+                                  format: int32
+                                  type: integer
+                                type: array
+                              ranges:
+                                items:
+                                  description: |-
+                                    Range represents a range with a start and an end value. Both start and end are included.
+                                    It is used to define a continuous segment.
+                                  properties:
+                                    end:
+                                      format: int32
+                                      type: integer
+                                    start:
+                                      format: int32
+                                      type: integer
+                                  required:
+                                  - end
+                                  - start
+                                  type: object
+                                type: array
+                            type: object
                           replicas:
                             default: 1
                             description: Specifies the desired number of replicas
@@ -14112,6 +14168,35 @@ spec:
                           items:
                             type: string
                           type: array
+                        ordinals:
+                          description: |-
+                            Specifies the desired Ordinals.
+                            The Ordinals used to specify the ordinal of the instance (pod) names to be generated under this component.
+                            If Ordinals are defined, their number must be equal to or more than the corresponding replicas.
+                          properties:
+                            discrete:
+                              items:
+                                format: int32
+                                type: integer
+                              type: array
+                            ranges:
+                              items:
+                                description: |-
+                                  Range represents a range with a start and an end value. Both start and end are included.
+                                  It is used to define a continuous segment.
+                                properties:
+                                  end:
+                                    format: int32
+                                    type: integer
+                                  start:
+                                    format: int32
+                                    type: integer
+                                required:
+                                - end
+                                - start
+                                type: object
+                              type: array
+                          type: object
                         parallelPodManagementConcurrency:
                           anyOf:
                           - type: integer
diff --git a/deploy/helm/crds/apps.kubeblocks.io_components.yaml b/deploy/helm/crds/apps.kubeblocks.io_components.yaml
index 1ba48b17b..d7a42b250 100644
--- a/deploy/helm/crds/apps.kubeblocks.io_components.yaml
+++ b/deploy/helm/crds/apps.kubeblocks.io_components.yaml
@@ -2609,6 +2609,35 @@ spec:
                 items:
                   type: string
                 type: array
+              ordinals:
+                description: |-
+                  Specifies the desired Ordinals.
+                  The Ordinals used to specify the ordinal of the instance (pod) names to be generated under this component.
+                  If Ordinals are defined, their number must be equal to or more than the corresponding replicas.
+                properties:
+                  discrete:
+                    items:
+                      format: int32
+                      type: integer
+                    type: array
+                  ranges:
+                    items:
+                      description: |-
+                        Range represents a range with a start and an end value. Both start and end are included.
+                        It is used to define a continuous segment.
+                      properties:
+                        end:
+                          format: int32
+                          type: integer
+                        start:
+                          format: int32
+                          type: integer
+                      required:
+                      - end
+                      - start
+                      type: object
+                    type: array
+                type: object
               parallelPodManagementConcurrency:
                 anyOf:
                 - type: integer
diff --git a/docs/developer_docs/api-reference/cluster.md b/docs/developer_docs/api-reference/cluster.md
index bd5f4f5ae..f4363db5a 100644
--- a/docs/developer_docs/api-reference/cluster.md
+++ b/docs/developer_docs/api-reference/cluster.md
@@ -855,6 +855,22 @@ Any remaining replicas will be generated using the default template and will fol
 </tr>
 <tr>
 <td>
+<code>ordinals</code><br/>
+<em>
+<a href="#apps.kubeblocks.io/v1.Ordinals">
+Ordinals
+</a>
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>Specifies the desired Ordinals.
+The Ordinals used to specify the ordinal of the instance (pod) names to be generated under this component.
+If Ordinals are defined, their number must be equal to or more than the corresponding replicas.</p>
+</td>
+</tr>
+<tr>
+<td>
 <code>flatInstanceOrdinal</code><br/>
 <em>
 bool
@@ -3392,6 +3408,22 @@ Any remaining replicas will be generated using the default template and will fol
 </tr>
 <tr>
 <td>
+<code>ordinals</code><br/>
+<em>
+<a href="#apps.kubeblocks.io/v1.Ordinals">
+Ordinals
+</a>
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>Specifies the desired Ordinals.
+The Ordinals used to specify the ordinal of the instance (pod) names to be generated under this component.
+If Ordinals are defined, their number must be equal to or more than the corresponding replicas.</p>
+</td>
+</tr>
+<tr>
+<td>
 <code>flatInstanceOrdinal</code><br/>
 <em>
 bool
@@ -6727,6 +6759,22 @@ Any remaining replicas will be generated using the default template and will fol
 </tr>
 <tr>
 <td>
+<code>ordinals</code><br/>
+<em>
+<a href="#apps.kubeblocks.io/v1.Ordinals">
+Ordinals
+</a>
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>Specifies the desired Ordinals.
+The Ordinals used to specify the ordinal of the instance (pod) names to be generated under this component.
+If Ordinals are defined, their number must be equal to or more than the corresponding replicas.</p>
+</td>
+</tr>
+<tr>
+<td>
 <code>flatInstanceOrdinal</code><br/>
 <em>
 bool
@@ -9160,7 +9208,7 @@ VarOption
 <h3 id="apps.kubeblocks.io/v1.Ordinals">Ordinals
 </h3>
 <p>
-(<em>Appears on:</em><a href="#apps.kubeblocks.io/v1.InstanceTemplate">InstanceTemplate</a>, <a href="#workloads.kubeblocks.io/v1.InstanceSetSpec">InstanceSetSpec</a>, <a href="#workloads.kubeblocks.io/v1.InstanceTemplate">InstanceTemplate</a>)
+(<em>Appears on:</em><a href="#apps.kubeblocks.io/v1.ClusterComponentSpec">ClusterComponentSpec</a>, <a href="#apps.kubeblocks.io/v1.ComponentSpec">ComponentSpec</a>, <a href="#apps.kubeblocks.io/v1.InstanceTemplate">InstanceTemplate</a>, <a href="#apps.kubeblocks.io/v1.ShardTemplate">ShardTemplate</a>, <a href="#workloads.kubeblocks.io/v1.InstanceSetSpec">InstanceSetSpec</a>, <a href="#workloads.kubeblocks.io/v1.InstanceTemplate">InstanceTemplate</a>)
 </p>
 <div>
 <p>Ordinals represents a combination of continuous segments and individual values.</p>
@@ -11933,6 +11981,20 @@ Kubernetes core/v1.ResourceRequirements
 </tr>
 <tr>
 <td>
+<code>ordinals</code><br/>
+<em>
+<a href="#apps.kubeblocks.io/v1.Ordinals">
+Ordinals
+</a>
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>Specifies an override for the desired Ordinals of the shard.</p>
+</td>
+</tr>
+<tr>
+<td>
 <code>flatInstanceOrdinal</code><br/>
 <em>
 bool
diff --git a/pkg/controller/builder/builder_component.go b/pkg/controller/builder/builder_component.go
index 47531ebad..f4a5a0af5 100644
--- a/pkg/controller/builder/builder_component.go
+++ b/pkg/controller/builder/builder_component.go
@@ -179,6 +179,12 @@ func (builder *ComponentBuilder) SetInstances(instances []appsv1.InstanceTemplat
 	builder.get().Spec.Instances = instances
 	return builder
 }
+
+func (builder *ComponentBuilder) SetOrdinals(ordinals appsv1.Ordinals) *ComponentBuilder {
+	builder.get().Spec.Ordinals = ordinals
+	return builder
+}
+
 func (builder *ComponentBuilder) SetFlatInstanceOrdinal(flatInstanceOrdinal bool) *ComponentBuilder {
 	builder.get().Spec.FlatInstanceOrdinal = flatInstanceOrdinal
 	return builder
diff --git a/pkg/controller/builder/builder_instance_set.go b/pkg/controller/builder/builder_instance_set.go
index 903e092aa..741fb8d05 100644
--- a/pkg/controller/builder/builder_instance_set.go
+++ b/pkg/controller/builder/builder_instance_set.go
@@ -154,6 +154,11 @@ func (builder *InstanceSetBuilder) SetInstances(instances []workloads.InstanceTe
 	return builder
 }
 
+func (builder *InstanceSetBuilder) SetOrdinals(ordinals kbappsv1.Ordinals) *InstanceSetBuilder {
+	builder.get().Spec.DefaultTemplateOrdinals = ordinals
+	return builder
+}
+
 func (builder *InstanceSetBuilder) SetFlatInstanceOrdinal(flatInstanceOrdinal bool) *InstanceSetBuilder {
 	builder.get().Spec.FlatInstanceOrdinal = flatInstanceOrdinal
 	return builder
diff --git a/pkg/controller/component/component.go b/pkg/controller/component/component.go
index cf70d545b..88e4a9ab4 100644
--- a/pkg/controller/component/component.go
+++ b/pkg/controller/component/component.go
@@ -90,6 +90,7 @@ func BuildComponent(cluster *appsv1.Cluster, compSpec *appsv1.ClusterComponentSp
 		SetServiceRefs(compSpec.ServiceRefs).
 		SetTLSConfig(compSpec.TLS, compSpec.Issuer).
 		SetInstances(compSpec.Instances).
+		SetOrdinals(compSpec.Ordinals).
 		SetFlatInstanceOrdinal(compSpec.FlatInstanceOrdinal).
 		SetOfflineInstances(compSpec.OfflineInstances).
 		SetRuntimeClassName(cluster.Spec.RuntimeClassName).
diff --git a/pkg/controller/component/synthesize_component.go b/pkg/controller/component/synthesize_component.go
index 9fc52a96b..b6ccfb627 100644
--- a/pkg/controller/component/synthesize_component.go
+++ b/pkg/controller/component/synthesize_component.go
@@ -110,6 +110,7 @@ func BuildSynthesizedComponent(ctx context.Context, cli client.Reader,
 		TLSConfig:                        comp.Spec.TLSConfig,
 		ServiceAccountName:               comp.Spec.ServiceAccountName,
 		Instances:                        comp.Spec.Instances,
+		Ordinals:                         comp.Spec.Ordinals,
 		FlatInstanceOrdinal:              comp.Spec.FlatInstanceOrdinal,
 		InstanceImages:                   make(map[string]map[string]string),
 		OfflineInstances:                 comp.Spec.OfflineInstances,
diff --git a/pkg/controller/component/type.go b/pkg/controller/component/type.go
index 829aefd83..8ed3363c9 100644
--- a/pkg/controller/component/type.go
+++ b/pkg/controller/component/type.go
@@ -61,6 +61,7 @@ type SynthesizedComponent struct {
 	EnvVars                          []corev1.EnvVar                        `json:"envVars,omitempty"`
 	EnvFromSources                   []corev1.EnvFromSource                 `json:"envFromSources,omitempty"`
 	Instances                        []kbappsv1.InstanceTemplate            `json:"instances,omitempty"`
+	Ordinals                         kbappsv1.Ordinals
 	FlatInstanceOrdinal              bool
 	InstanceImages                   map[string]map[string]string    `json:"instanceImages,omitempty"`
 	OfflineInstances                 []string                        `json:"offlineInstances,omitempty"`
diff --git a/pkg/controller/component/vars.go b/pkg/controller/component/vars.go
index 6232eec00..070003179 100644
--- a/pkg/controller/component/vars.go
+++ b/pkg/controller/component/vars.go
@@ -239,7 +239,7 @@ func resolveValueReferenceNEscaping(templateVars, credentialVars map[string]core
 func evaluateObjectVarsExpression(definedVars []appsv1.EnvVar, credentialVars []corev1.EnvVar, vars *[]corev1.EnvVar) error {
 	var (
 		isValues = make(map[string]bool)
-		values   = make(map[string]string)
+		values   = make(map[string]any)
 	)
 	normalize := func(name string) string {
 		return strings.ReplaceAll(name, "-", "_")
diff --git a/pkg/controller/component/workload_utils.go b/pkg/controller/component/workload_utils.go
index 5b4701c4b..2d7607d36 100644
--- a/pkg/controller/component/workload_utils.go
+++ b/pkg/controller/component/workload_utils.go
@@ -147,10 +147,11 @@ func GeneratePodNamesByComp(comp *appsv1.Component) ([]string, error) {
 			Annotations: comp.Annotations,
 		},
 		Spec: workloads.InstanceSetSpec{
-			Replicas:            &comp.Spec.Replicas,
-			Instances:           instanceTemplates(),
-			FlatInstanceOrdinal: comp.Spec.FlatInstanceOrdinal,
-			OfflineInstances:    comp.Spec.OfflineInstances,
+			Replicas:                &comp.Spec.Replicas,
+			Instances:               instanceTemplates(),
+			DefaultTemplateOrdinals: comp.Spec.Ordinals,
+			FlatInstanceOrdinal:     comp.Spec.FlatInstanceOrdinal,
+			OfflineInstances:        comp.Spec.OfflineInstances,
 		},
 	}
 	itsExt, err := instancetemplate.BuildInstanceSetExt(its, nil)
diff --git a/pkg/controller/factory/builder.go b/pkg/controller/factory/builder.go
index c7ead4959..8ec8632f0 100644
--- a/pkg/controller/factory/builder.go
+++ b/pkg/controller/factory/builder.go
@@ -74,6 +74,7 @@ func BuildInstanceSet(synthesizedComp *component.SynthesizedComponent, compDef *
 		SetPVCRetentionPolicy(&synthesizedComp.PVCRetentionPolicy).
 		SetMinReadySeconds(synthesizedComp.MinReadySeconds).
 		SetInstances(getInstanceTemplates(synthesizedComp)).
+		SetOrdinals(synthesizedComp.Ordinals).
 		SetFlatInstanceOrdinal(synthesizedComp.FlatInstanceOrdinal).
 		SetOfflineInstances(synthesizedComp.OfflineInstances).
 		SetRoles(synthesizedComp.Roles).
diff --git a/pkg/controller/instanceset/in_place_update_util.go b/pkg/controller/instanceset/in_place_update_util.go
index c0b677c11..58b8c9576 100644
--- a/pkg/controller/instanceset/in_place_update_util.go
+++ b/pkg/controller/instanceset/in_place_update_util.go
@@ -25,6 +25,7 @@ import (
 	"slices"
 	"strings"
 
+	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	"k8s.io/utils/ptr"
@@ -44,6 +45,10 @@ const (
 	inPlaceUpdatePolicy podUpdatePolicy = "inPlaceUpdate"
 )
 
+var (
+	errTemplateNotFound = fmt.Errorf("no template found for pod")
+)
+
 func supportPodVerticalScaling() bool {
 	return viper.GetBool(constant.FeatureGateInPlacePodVerticalScaling)
 }
@@ -314,7 +319,7 @@ func getPodUpdatePolicy(its *workloads.InstanceSet, pod *corev1.Pod) (podUpdateP
 		return templateName == templateExt.Name
 	})
 	if index < 0 {
-		return noOpsPolicy, "", fmt.Errorf("no corresponding template found for instance %s", pod.Name)
+		return noOpsPolicy, "", errors.Wrapf(errTemplateNotFound, "pod: %s/%s", pod.Namespace, pod.Name)
 	}
 	newPod, err := buildInstancePodByTemplate(pod.Name, templateList[index], its, getPodRevision(pod))
 	if err != nil {
@@ -371,13 +376,19 @@ func getTemplateNameByPod(itsExt *instancetemplate.InstanceSetExt, pod *corev1.P
 	if ok {
 		return tplExt.Name, nil
 	}
-	return "", fmt.Errorf("no template found for pod %s/%s", pod.Namespace, pod.Name)
+	return "", errors.Wrapf(errTemplateNotFound, "pod: %s/%s", pod.Namespace, pod.Name)
 }
 
-// IsPodUpdated tells whether the pod's spec is as expected in the InstanceSet.
+// isPodUpdated tells whether the pod's spec is as expected in the InstanceSet.
 // This function is meant to replace the old fashion `GetPodRevision(pod) == updateRevision`,
 // as the pod template revision has been redefined in instanceset.
-func IsPodUpdated(its *workloads.InstanceSet, pod *corev1.Pod) (bool, error) {
+func isPodUpdated(its *workloads.InstanceSet, pod *corev1.Pod) (bool, error) {
 	policy, _, err := getPodUpdatePolicy(its, pod)
+	if err != nil {
+		if errors.Is(err, errTemplateNotFound) {
+			return true, nil
+		}
+		return false, err
+	}
 	return policy == noOpsPolicy, err
 }
diff --git a/pkg/controller/instanceset/reconciler_revision_update.go b/pkg/controller/instanceset/reconciler_revision_update.go
index 081f6b0f1..6166bd630 100644
--- a/pkg/controller/instanceset/reconciler_revision_update.go
+++ b/pkg/controller/instanceset/reconciler_revision_update.go
@@ -106,7 +106,7 @@ func calculateUpdatedReplicas(its *workloads.InstanceSet, pods []client.Object)
 	updatedReplicas := int32(0)
 	for i := range pods {
 		pod, _ := pods[i].(*corev1.Pod)
-		updated, err := IsPodUpdated(its, pod)
+		updated, err := isPodUpdated(its, pod)
 		if err != nil {
 			return 0, nil
 		}
diff --git a/pkg/controller/instanceset/reconciler_status.go b/pkg/controller/instanceset/reconciler_status.go
index fb71b4be3..73a73d1da 100644
--- a/pkg/controller/instanceset/reconciler_status.go
+++ b/pkg/controller/instanceset/reconciler_status.go
@@ -124,7 +124,7 @@ func (r *statusReconciler) Reconcile(tree *kubebuilderx.ObjectTree) (kubebuilder
 			}
 		}
 		if isCreated(pod) && !isTerminating(pod) {
-			isPodUpdated, err := IsPodUpdated(its, pod)
+			isPodUpdated, err := isPodUpdated(its, pod)
 			if err != nil {
 				return kubebuilderx.Continue, err
 			}
diff --git a/pkg/controller/instancetemplate/name_builder_default.go b/pkg/controller/instancetemplate/name_builder_default.go
index 2e36fe7c3..74e899c3c 100644
--- a/pkg/controller/instancetemplate/name_builder_default.go
+++ b/pkg/controller/instancetemplate/name_builder_default.go
@@ -120,7 +120,7 @@ func getOrdinalListByTemplateName(its *workloads.InstanceSet, templateName strin
 }
 
 func getOrdinalsByTemplateName(its *workloads.InstanceSet, templateName string) (kbappsv1.Ordinals, error) {
-	if templateName == "" {
+	if templateName == defaultTemplateName {
 		return its.Spec.DefaultTemplateOrdinals, nil
 	}
 	for _, template := range its.Spec.Instances {
diff --git a/pkg/controller/instancetemplate/name_builder_flat.go b/pkg/controller/instancetemplate/name_builder_flat.go
index 4d9502cf1..32cb080e9 100644
--- a/pkg/controller/instancetemplate/name_builder_flat.go
+++ b/pkg/controller/instancetemplate/name_builder_flat.go
@@ -158,6 +158,7 @@ func generateTemplateName2OrdinalMap(itsExt *InstanceSetExt) (map[string]sets.Se
 		instanceTemplatesList = append(instanceTemplatesList, instanceTemplate)
 		template2OrdinalSetMap[instanceTemplate.Name] = sets.New[int32]()
 	}
+	template2OrdinalSetMap[defaultTemplateName] = sets.New[int32]() // always add the default instance template
 	slices.SortFunc(instanceTemplatesList, func(a, b *workloads.InstanceTemplate) int {
 		return strings.Compare(a.Name, b.Name)
 	})
@@ -177,20 +178,17 @@ func generateTemplateName2OrdinalMap(itsExt *InstanceSetExt) (map[string]sets.Se
 		defaultTemplateUnavailableOrdinalSet = defaultTemplateUnavailableOrdinalSet.Union(availableOrdinalSet)
 	}
 
-	if _, ok := template2OrdinalSetMap[""]; ok {
-		template2OrdinalSetMap[""].Insert(itsExt.InstanceSet.Status.Ordinals...)
-	}
+	template2OrdinalSetMap[defaultTemplateName].Insert(itsExt.InstanceSet.Status.Ordinals...)
 	globalUsedOrdinalSet.Insert(itsExt.InstanceSet.Status.Ordinals...)
 	for _, status := range itsExt.InstanceSet.Status.TemplatesStatus {
-		if _, ok := template2OrdinalSetMap[status.Name]; ok {
-			template2OrdinalSetMap[status.Name].Insert(status.Ordinals...)
+		if _, ok := template2OrdinalSetMap[status.Name]; !ok {
+			template2OrdinalSetMap[status.Name] = sets.New[int32]()
 		}
+		template2OrdinalSetMap[status.Name].Insert(status.Ordinals...)
 		globalUsedOrdinalSet.Insert(status.Ordinals...)
 	}
 
-	generateWithOrdinalsDefined := func(
-		current, available sets.Set[int32], instanceTemplate *workloads.InstanceTemplate,
-	) (sets.Set[int32], error) {
+	generateWithOrdinalsDefined := func(current, available sets.Set[int32], instanceTemplate *workloads.InstanceTemplate) (sets.Set[int32], error) {
 		// delete any ordinal that doesn't in the available
 		current = current.Intersection(available)
 
@@ -208,6 +206,24 @@ func generateTemplateName2OrdinalMap(itsExt *InstanceSetExt) (map[string]sets.Se
 			return current, nil
 		}
 
+		// move from the default instance template to a named instance template
+		isTakeOver := func(ordinal int32) bool {
+			return instanceTemplate.Name != defaultTemplateName &&
+				template2OrdinalSetMap[defaultTemplateName].Has(ordinal)
+		}
+
+		// move from a named instance template to the default instance template
+		isTakeBack := func(ordinal int32) bool {
+			if instanceTemplate.Name == defaultTemplateName {
+				for name, ordinals := range template2OrdinalSetMap {
+					if name != defaultTemplateName && ordinals.Has(ordinal) {
+						return true
+					}
+				}
+			}
+			return false
+		}
+
 		// if current is too little, add some
 		availableWithoutCurrent := available.Difference(current)
 		availableList := convertOrdinalSetToSortedList(availableWithoutCurrent)
@@ -218,7 +234,7 @@ func generateTemplateName2OrdinalMap(itsExt *InstanceSetExt) (map[string]sets.Se
 					return current, ErrOrdinalsNotEnough
 				}
 				ordinal := availableList[cur]
-				if !globalUsedOrdinalSet.Has(ordinal) || (!current.Has(ordinal) && template2OrdinalSetMap[""].Has(ordinal)) {
+				if !globalUsedOrdinalSet.Has(ordinal) || (!current.Has(ordinal) && (isTakeOver(ordinal) || isTakeBack(ordinal))) {
 					globalUsedOrdinalSet.Insert(ordinal)
 					current.Insert(ordinal)
 					break
@@ -229,9 +245,7 @@ func generateTemplateName2OrdinalMap(itsExt *InstanceSetExt) (map[string]sets.Se
 		return current, nil
 	}
 
-	generateWithoutOrdinalsDefined := func(
-		current sets.Set[int32], instanceTemplate *workloads.InstanceTemplate,
-	) (sets.Set[int32], error) {
+	generateWithoutOrdinalsDefined := func(current sets.Set[int32], instanceTemplate *workloads.InstanceTemplate) (sets.Set[int32], error) {
 		// delete any ordinal that is taken by an template with ordinals defined
 		current = current.Difference(defaultTemplateUnavailableOrdinalSet)
 		// delete any offline ordinals
diff --git a/pkg/controller/instancetemplate/types.go b/pkg/controller/instancetemplate/types.go
index 119560d25..f3d8b5393 100644
--- a/pkg/controller/instancetemplate/types.go
+++ b/pkg/controller/instancetemplate/types.go
@@ -31,6 +31,8 @@ const (
 	TemplateRefDataKey       = "instances"
 
 	TemplateNameLabelKey = "workloads.kubeblocks.io/template-name"
+
+	defaultTemplateName = ""
 )
 
 // InstanceSetExt has a complete list of instance templates.
diff --git a/pkg/controller/sharding/utils.go b/pkg/controller/sharding/utils.go
index d2db153dc..b0c1c0b25 100644
--- a/pkg/controller/sharding/utils.go
+++ b/pkg/controller/sharding/utils.go
@@ -140,6 +140,9 @@ func buildShardTemplates(clusterName string, sharding *appsv1.ClusterSharding, s
 		if tpl.Instances != nil {
 			spec.Instances = tpl.Instances
 		}
+		if tpl.Ordinals != nil {
+			spec.Ordinals = *tpl.Ordinals
+		}
 		if tpl.FlatInstanceOrdinal != nil {
 			spec.FlatInstanceOrdinal = *tpl.FlatInstanceOrdinal
 		}
diff --git a/pkg/kbagent/service/action_utils.go b/pkg/kbagent/service/action_utils.go
index 71e5c6204..7695ec442 100644
--- a/pkg/kbagent/service/action_utils.go
+++ b/pkg/kbagent/service/action_utils.go
@@ -24,7 +24,6 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"maps"
 	"net"
 	"net/http"
 	"os"
@@ -601,9 +600,11 @@ func renderTemplateData(action string, parameters map[string]string, data string
 	return buf.String(), nil
 }
 
-func mergeEnvWith(parameters map[string]string) map[string]string {
-	result := make(map[string]string)
-	maps.Copy(result, parameters)
+func mergeEnvWith(parameters map[string]string) map[string]any {
+	result := make(map[string]any)
+	for k, v := range parameters {
+		result[k] = v
+	}
 	for _, e := range os.Environ() {
 		kv := strings.Split(e, "=")
 		if _, ok := result[kv[0]]; !ok {
diff --git a/pkg/kbagent/service/probe.go b/pkg/kbagent/service/probe.go
index a4c1c009f..1337eaace 100644
--- a/pkg/kbagent/service/probe.go
+++ b/pkg/kbagent/service/probe.go
@@ -40,6 +40,11 @@ const (
 	defaultProbePeriodSeconds = 60
 )
 
+var (
+	defaultRetrySendEventInterval = 1 * time.Minute
+	retrySendEventInterval        = defaultRetrySendEventInterval
+)
+
 func newProbeService(logger logr.Logger, actionService *actionService, probes []proto.Probe) (*probeService, error) {
 	sp := &probeService{
 		logger:        logger,
@@ -58,10 +63,11 @@ func newProbeService(logger logr.Logger, actionService *actionService, probes []
 }
 
 type probeService struct {
-	logger        logr.Logger
-	actionService *actionService
-	probes        map[string]*proto.Probe
-	runners       map[string]*probeRunner
+	logger               logr.Logger
+	actionService        *actionService
+	probes               map[string]*proto.Probe
+	runners              map[string]*probeRunner
+	sendEventWithMessage func(logger *logr.Logger, reason string, message string, sync bool) error
 }
 
 var _ Service = &probeService{}
@@ -77,9 +83,10 @@ func (s *probeService) URI() string {
 func (s *probeService) Start() error {
 	for name := range s.probes {
 		runner := &probeRunner{
-			logger:        s.logger.WithValues("probe", name),
-			actionService: s.actionService,
-			latestEvent:   make(chan proto.ProbeEvent, 1),
+			logger:               s.logger.WithValues("probe", name),
+			actionService:        s.actionService,
+			latestEvent:          make(chan proto.ProbeEvent, 1),
+			sendEventWithMessage: s.sendEventWithMessage,
 		}
 		go runner.run(s.probes[name])
 		s.runners[name] = runner
@@ -87,22 +94,23 @@ func (s *probeService) Start() error {
 	return nil
 }
 
-func (s *probeService) HandleConn(ctx context.Context, conn net.Conn) error {
+func (s *probeService) HandleConn(context.Context, net.Conn) error {
 	return nil
 }
 
-func (s *probeService) HandleRequest(ctx context.Context, payload []byte) ([]byte, error) {
+func (s *probeService) HandleRequest(context.Context, []byte) ([]byte, error) {
 	return nil, errors.Wrapf(proto.ErrNotImplemented, "service %s does not support request handling", s.Kind())
 }
 
 type probeRunner struct {
-	logger        logr.Logger
-	actionService *actionService
-	ticker        *time.Ticker
-	succeedCount  int64
-	failedCount   int64
-	latestOutput  []byte
-	latestEvent   chan proto.ProbeEvent
+	logger               logr.Logger
+	actionService        *actionService
+	ticker               *time.Ticker
+	succeedCount         int64
+	failedCount          int64
+	latestOutput         []byte
+	latestEvent          chan proto.ProbeEvent
+	sendEventWithMessage func(logger *logr.Logger, reason string, message string, sync bool) error
 }
 
 func (r *probeRunner) run(probe *proto.Probe) {
@@ -112,23 +120,23 @@ func (r *probeRunner) run(probe *proto.Probe) {
 		time.Sleep(time.Duration(probe.InitialDelaySeconds) * time.Second)
 	}
 
-	// launch report loop first
+	// launch the report loop first
 	r.launchReportLoop(probe)
 
-	r.launchRunLoop(probe)
+	r.launchProbeLoop(probe)
 }
 
-func (r *probeRunner) launchRunLoop(probe *proto.Probe) {
+func (r *probeRunner) launchProbeLoop(probe *proto.Probe) {
 	if probe.PeriodSeconds <= 0 {
 		probe.PeriodSeconds = defaultProbePeriodSeconds
 	}
 	r.ticker = time.NewTicker(time.Duration(probe.PeriodSeconds) * time.Second)
 	defer r.ticker.Stop()
 
-	r.runLoop(probe)
+	r.probeLoop(probe)
 }
 
-func (r *probeRunner) runLoop(probe *proto.Probe) {
+func (r *probeRunner) probeLoop(probe *proto.Probe) {
 	once := func() {
 		output, err := r.actionService.handleRequest(context.Background(), &proto.ActionRequest{Action: probe.Action})
 		if err == nil {
@@ -154,45 +162,16 @@ func (r *probeRunner) runLoop(probe *proto.Probe) {
 	}
 }
 
-func (r *probeRunner) launchReportLoop(probe *proto.Probe) {
-	if probe.ReportPeriodSeconds <= 0 {
-		return
-	}
-
-	if probe.ReportPeriodSeconds < probe.PeriodSeconds {
-		probe.ReportPeriodSeconds = probe.PeriodSeconds
-	}
-
-	go func() {
-		ticker := time.NewTicker(time.Duration(probe.ReportPeriodSeconds) * time.Second)
-		defer ticker.Stop()
-
-		var latestReportedEvent *proto.ProbeEvent
-		for range ticker.C {
-			latestEvent := gather(r.latestEvent)
-			if latestEvent == nil && latestReportedEvent != nil {
-				latestEvent = latestReportedEvent
-			}
-			if latestEvent != nil {
-				r.logger.Info("report probe event periodically",
-					"code", latestEvent.Code, "output", outputPrefix(latestEvent.Output), "message", latestEvent.Message)
-				r.sendEvent(latestEvent)
-			}
-			latestReportedEvent = latestEvent
-		}
-	}()
-}
-
 func (r *probeRunner) report(probe *proto.Probe, output []byte, err error) {
 	var latestEvent *proto.ProbeEvent
 
 	succeed, thresholdPoint := r.succeed(probe)
 	if succeed && thresholdPoint ||
 		succeed && !thresholdPoint && !reflect.DeepEqual(output, r.latestOutput) {
-		latestEvent = r.buildNSendEvent(probe.Instance, probe.Action, 0, output, "")
+		latestEvent = r.buildEvent(probe.Instance, probe.Action, 0, output, "")
 	}
 	if r.fail(probe) {
-		latestEvent = r.buildNSendEvent(probe.Instance, probe.Action, -1, r.latestOutput, err.Error())
+		latestEvent = r.buildEvent(probe.Instance, probe.Action, -1, r.latestOutput, err.Error())
 	}
 
 	if latestEvent != nil {
@@ -227,29 +206,88 @@ func (r *probeRunner) fail(probe *proto.Probe) bool {
 	return false
 }
 
-func (r *probeRunner) buildNSendEvent(instance, probe string, code int32, output []byte, message string) *proto.ProbeEvent {
-	r.logger.Info("send probe event", "probe", probe, "code", code, "output", outputPrefix(output), "message", message)
-	event := &proto.ProbeEvent{
+func (r *probeRunner) buildEvent(instance, probe string, code int32, output []byte, message string) *proto.ProbeEvent {
+	return &proto.ProbeEvent{
 		Instance: instance,
 		Probe:    probe,
 		Code:     code,
 		Output:   output,
 		Message:  message,
 	}
-	r.sendEvent(event)
-	return event
 }
 
-func (r *probeRunner) sendEvent(event *proto.ProbeEvent) {
-	msg, err := json.Marshal(&event)
-	if err == nil {
-		_ = util.SendEventWithMessage(&r.logger, event.Probe, string(msg), false)
-	} else {
-		r.logger.Error(err, fmt.Sprintf("failed to marshal probe event, probe: %s", event.Probe))
-	}
-}
+func (r *probeRunner) launchReportLoop(probe *proto.Probe) {
+	go func() {
+		var reportChan <-chan time.Time
+		if probe.ReportPeriodSeconds > 0 {
+			if probe.ReportPeriodSeconds < probe.PeriodSeconds {
+				probe.ReportPeriodSeconds = probe.PeriodSeconds
+			}
+			ticker := time.NewTicker(time.Duration(probe.ReportPeriodSeconds) * time.Second)
+			defer ticker.Stop()
+			reportChan = ticker.C
+		}
+
+		retryTicker := time.NewTicker(retrySendEventInterval)
+		defer retryTicker.Stop()
+
+		var event proto.ProbeEvent
+		var hasEvent bool
+
+		output := func() string {
+			prefixLen := min(len(event.Output), 32)
+			return string(event.Output[:prefixLen])
+		}
+
+		log := func(err error, msg string, retry, periodically bool) {
+			if err == nil {
+				r.logger.Info(msg, "retry", retry, "periodically", periodically,
+					"probe", probe.Action, "code", event.Code, "output", output(), "message", event.Message)
+			} else {
+				r.logger.Error(err, msg, "retry", retry, "periodically", periodically,
+					"probe", probe.Action, "code", event.Code, "output", output(), "message", event.Message)
+			}
+		}
+
+		trySend := func(retry, periodically bool) bool {
+			if !hasEvent {
+				return true
+			}
 
-func outputPrefix(output []byte) string {
-	prefixLen := min(len(output), 32)
-	return string(output[:prefixLen])
+			msg, err := json.Marshal(event)
+			if err != nil {
+				log(err, "failed to marshal the probe event", retry, periodically)
+				return true
+			}
+
+			if r.sendEventWithMessage == nil {
+				r.sendEventWithMessage = util.SendEventWithMessage
+			}
+			err = r.sendEventWithMessage(&r.logger, event.Probe, string(msg), true)
+			if err == nil {
+				log(nil, "succeed to send the probe event", retry, periodically)
+			} else {
+				log(err, "failed to send the probe event, will retry later", retry, periodically)
+			}
+			return err == nil
+		}
+
+		needsRetry := false
+		for {
+			select {
+			case latest := <-r.latestEvent:
+				event = latest
+				hasEvent = true
+				needsRetry = !trySend(false, false)
+
+			case <-reportChan:
+				needsRetry = !trySend(false, true)
+
+			case <-retryTicker.C:
+				if needsRetry {
+					needsRetry = !trySend(true, false)
+				}
+			}
+		}
+	}()
 }
diff --git a/pkg/kbagent/util/event.go b/pkg/kbagent/util/event.go
index aaffd8a43..315a7eb7e 100644
--- a/pkg/kbagent/util/event.go
+++ b/pkg/kbagent/util/event.go
@@ -38,25 +38,26 @@ import (
 )
 
 const (
-	maxRetryAttempts = 30
-	retryInterval    = 10 * time.Second
+	defaultRetryInterval    = 10 * time.Second
+	defaultMaxRetryAttempts = 30
+
+	syncRetryInterval    = 200 * time.Millisecond
+	syncMaxRetryAttempts = 3
 )
 
 func SendEventWithMessage(logger *logr.Logger, reason string, message string, sync bool) error {
-	send := func() error {
-		err := createOrUpdateEvent(reason, message)
-		if logger != nil && err != nil {
-			logger.Error(err, "failed to send event",
-				"reason", reason,
-				"message", message)
+	send := func(retryInterval time.Duration, retryAttempts int32) error {
+		err := createOrUpdateEvent(reason, message, retryInterval, retryAttempts)
+		if err != nil && logger != nil {
+			logger.Error(err, "failed to send event", "reason", reason, "message", message)
 		}
 		return err
 	}
 	if sync {
-		return send()
+		return send(syncRetryInterval, syncMaxRetryAttempts)
 	}
 	go func() {
-		_ = send()
+		_ = send(defaultRetryInterval, defaultMaxRetryAttempts)
 	}()
 	return nil
 }
@@ -92,7 +93,7 @@ func newEvent(reason string, message string) *corev1.Event {
 	}
 }
 
-func createOrUpdateEvent(reason, message string) error {
+func createOrUpdateEvent(reason, message string, retryInterval time.Duration, retryAttempts int32) error {
 	clientSet, err := getK8sClientSet()
 	if err != nil {
 		return err
@@ -101,30 +102,28 @@ func createOrUpdateEvent(reason, message string) error {
 	eventName := generateEventName(reason, message)
 
 	var event *corev1.Event
-	for i := 0; i < maxRetryAttempts; i++ {
+	attempts := max(retryAttempts, 1)
+	for i := int32(0); i < attempts; i++ {
 		event, err = eventsClient.Get(context.Background(), eventName, metav1.GetOptions{})
 		if err == nil {
-			// update
 			event.Count++
 			// the granularity of lastTimestamp is second and it is not enough for the event.
 			// there may multiple events in the same second, so we need to use EventTime here.
 			// event.LastTimestamp = metav1.Now()
 			event.EventTime = metav1.NowMicro()
 			_, err = eventsClient.Update(context.Background(), event, metav1.UpdateOptions{})
-			if err == nil {
-				return nil
-			}
 		} else if k8serrors.IsNotFound(err) {
-			// create
 			event = newEvent(reason, message)
 			_, err = eventsClient.Create(context.Background(), event, metav1.CreateOptions{})
-			if err == nil {
-				return nil
-			}
 		}
-		time.Sleep(retryInterval)
+		if err == nil {
+			return nil
+		}
+		if retryInterval > 0 && i < attempts-1 {
+			time.Sleep(retryInterval)
+		}
 	}
-	return errors.Wrapf(err, "failed to handle event after %d attempts", maxRetryAttempts)
+	return errors.Wrapf(err, "failed to create or update event after %d attempts", retryAttempts)
 }
 
 func getK8sClientSet() (*kubernetes.Clientset, error) {



Test patch:
diff --git a/pkg/controller/instancetemplate/name_builder_flat_test.go b/pkg/controller/instancetemplate/name_builder_flat_test.go
index a9409fc79..08a102f22 100644
--- a/pkg/controller/instancetemplate/name_builder_flat_test.go
+++ b/pkg/controller/instancetemplate/name_builder_flat_test.go
@@ -66,9 +66,9 @@ var _ = Describe("flat name builder tests", func() {
 				},
 			},
 		}, map[string]sets.Set[int32]{
-			"":   sets.New[int32](0, 1),
-			"t1": sets.New[int32](2, 3),
-			"t2": sets.New[int32](4),
+			defaultTemplateName: sets.New[int32](0, 1),
+			"t1":                sets.New[int32](2, 3),
+			"t2":                sets.New[int32](4),
 		}, false),
 
 		Entry("with running instances", &workloads.InstanceSet{
@@ -95,9 +95,9 @@ var _ = Describe("flat name builder tests", func() {
 				},
 			},
 		}, map[string]sets.Set[int32]{
-			"":   sets.New[int32](0, 1),
-			"t1": sets.New[int32](2, 4),
-			"t2": sets.New[int32](3),
+			defaultTemplateName: sets.New[int32](0, 1),
+			"t1":                sets.New[int32](2, 4),
+			"t2":                sets.New[int32](3),
 		}, false),
 
 		Entry("deal with scale in", &workloads.InstanceSet{
@@ -124,9 +124,9 @@ var _ = Describe("flat name builder tests", func() {
 				},
 			},
 		}, map[string]sets.Set[int32]{
-			"":   sets.New[int32](0, 1, 5),
-			"t1": sets.New[int32](2),
-			"t2": sets.New[int32](6),
+			defaultTemplateName: sets.New[int32](0, 1, 5),
+			"t1":                sets.New[int32](2),
+			"t2":                sets.New[int32](6),
 		}, false),
 
 		Entry("with ordinal spec", &workloads.InstanceSet{
@@ -188,8 +188,8 @@ var _ = Describe("flat name builder tests", func() {
 				},
 			},
 		}, map[string]sets.Set[int32]{
-			"":   sets.New[int32](0, 1),
-			"t1": sets.New[int32](100, 101),
+			defaultTemplateName: sets.New[int32](0, 1),
+			"t1":                sets.New[int32](100, 101),
 		}, false),
 
 		Entry("with ordinal spec - a newly defined ordinal spec takes an ordinal of the default template", &workloads.InstanceSet{
@@ -209,8 +209,8 @@ var _ = Describe("flat name builder tests", func() {
 				Ordinals: []int32{0, 1},
 			},
 		}, map[string]sets.Set[int32]{
-			"":   sets.New[int32](0, 3, 4),
-			"t1": sets.New[int32](1, 2),
+			defaultTemplateName: sets.New[int32](0, 3, 4),
+			"t1":                sets.New[int32](1, 2),
 		}, false),
 
 		Entry("with ordinal spec - replicas < length of ordinals range", &workloads.InstanceSet{
@@ -252,9 +252,9 @@ var _ = Describe("flat name builder tests", func() {
 				},
 			},
 		}, map[string]sets.Set[int32]{
-			"":   sets.New[int32](0, 1),
-			"t1": sets.New[int32](100, 150),
-			"t2": sets.New[int32](500),
+			defaultTemplateName: sets.New[int32](0, 1),
+			"t1":                sets.New[int32](100, 150),
+			"t2":                sets.New[int32](500),
 		}, false),
 
 		Entry("with ordinal spec - zero replica", &workloads.InstanceSet{
@@ -335,8 +335,8 @@ var _ = Describe("flat name builder tests", func() {
 				Ordinals: []int32{0, 1, 2},
 			},
 		}, map[string]sets.Set[int32]{
-			"t1": sets.New[int32](2, 3),
-			"":   sets.New[int32](0, 1),
+			"t1":                sets.New[int32](2, 3),
+			defaultTemplateName: sets.New[int32](0, 1),
 		}, false),
 
 		Entry("partially exchange ordinal spec", &workloads.InstanceSet{
@@ -397,8 +397,8 @@ var _ = Describe("flat name builder tests", func() {
 				},
 			},
 		}, map[string]sets.Set[int32]{
-			"":   sets.New[int32](0, 1, 4),
-			"t1": sets.New[int32](3, 5),
+			defaultTemplateName: sets.New[int32](0, 1, 4),
+			"t1":                sets.New[int32](3, 5),
 		}, false),
 
 		Entry("with offline instances conflicts template ordinals", &workloads.InstanceSet{
@@ -438,9 +438,9 @@ var _ = Describe("flat name builder tests", func() {
 				Ordinals: []int32{0, 1, 2, 3, 4},
 			},
 		}, map[string]sets.Set[int32]{
-			"":   sets.New[int32](0, 1, 2),
-			"t1": sets.New[int32](5),
-			"t2": sets.New[int32](6),
+			defaultTemplateName: sets.New[int32](0, 1, 2),
+			"t1":                sets.New[int32](5),
+			"t2":                sets.New[int32](6),
 		}, false),
 
 		Entry("move replicas into instance templates - take over", &workloads.InstanceSet{
@@ -467,22 +467,46 @@ var _ = Describe("flat name builder tests", func() {
 				Ordinals: []int32{0, 1, 2, 3, 4},
 			},
 		}, map[string]sets.Set[int32]{
-			"":   sets.New[int32](0, 1, 2),
-			"t1": sets.New[int32](3),
-			"t2": sets.New[int32](4),
+			defaultTemplateName: sets.New[int32](0, 1, 2),
+			"t1":                sets.New[int32](3),
+			"t2":                sets.New[int32](4),
 		}, false),
 
-		Entry("move replicas into instance templates - mixed", &workloads.InstanceSet{
+		Entry("move replicas into instance templates - take over last one", &workloads.InstanceSet{
 			Spec: workloads.InstanceSetSpec{
 				Replicas: ptr.To[int32](5),
 				Instances: []workloads.InstanceTemplate{
 					{
 						Name:     "t1",
 						Replicas: ptr.To[int32](1),
+						Ordinals: workloads.Ordinals{
+							Discrete: []int32{0},
+						},
 					},
 					{
 						Name:     "t2",
 						Replicas: ptr.To[int32](1),
+						Ordinals: workloads.Ordinals{
+							Discrete: []int32{1},
+						},
+					},
+					{
+						Name:     "t3",
+						Replicas: ptr.To[int32](1),
+						Ordinals: workloads.Ordinals{
+							Discrete: []int32{2},
+						},
+					},
+					{
+						Name:     "t4",
+						Replicas: ptr.To[int32](1),
+						Ordinals: workloads.Ordinals{
+							Discrete: []int32{3},
+						},
+					},
+					{
+						Name:     "t5",
+						Replicas: ptr.To[int32](1),
 						Ordinals: workloads.Ordinals{
 							Discrete: []int32{4},
 						},
@@ -493,9 +517,37 @@ var _ = Describe("flat name builder tests", func() {
 				Ordinals: []int32{0, 1, 2, 3, 4},
 			},
 		}, map[string]sets.Set[int32]{
-			"":   sets.New[int32](0, 1, 2),
-			"t1": sets.New[int32](5),
-			"t2": sets.New[int32](4),
+			"t1": sets.New[int32](0),
+			"t2": sets.New[int32](1),
+			"t3": sets.New[int32](2),
+			"t4": sets.New[int32](3),
+			"t5": sets.New[int32](4),
+		}, false),
+
+		Entry("move replicas into instance templates - take over and replace", &workloads.InstanceSet{
+			Spec: workloads.InstanceSetSpec{
+				Replicas: ptr.To[int32](5),
+				Instances: []workloads.InstanceTemplate{
+					{
+						Name:     "t1",
+						Replicas: ptr.To[int32](1),
+					},
+					{
+						Name:     "t2",
+						Replicas: ptr.To[int32](1),
+						Ordinals: workloads.Ordinals{
+							Discrete: []int32{4},
+						},
+					},
+				},
+			},
+			Status: workloads.InstanceSetStatus{
+				Ordinals: []int32{0, 1, 2, 3, 4},
+			},
+		}, map[string]sets.Set[int32]{
+			defaultTemplateName: sets.New[int32](0, 1, 2),
+			"t1":                sets.New[int32](5),
+			"t2":                sets.New[int32](4),
 		}, false),
 
 		Entry("move replicas out of instance templates", &workloads.InstanceSet{
@@ -507,58 +559,70 @@ var _ = Describe("flat name builder tests", func() {
 				TemplatesStatus: []workloads.InstanceTemplateStatus{
 					{
 						Name:     "t1",
+						Replicas: 1,
 						Ordinals: []int32{3},
 					},
 					{
 						Name:     "t2",
+						Replicas: 1,
 						Ordinals: []int32{4},
 					},
 				},
 			},
 		}, map[string]sets.Set[int32]{
-			"": sets.New[int32](0, 1, 2, 5, 6),
+			defaultTemplateName: sets.New[int32](0, 1, 2, 5, 6), // replace 3 and 4 with 5 and 6
 		}, false),
 
-		PEntry("move replicas out of instance templates - taken over instances", &workloads.InstanceSet{
+		Entry("move replicas out of instance templates - take back", &workloads.InstanceSet{
 			Spec: workloads.InstanceSetSpec{
 				Replicas: ptr.To[int32](5),
+				DefaultTemplateOrdinals: workloads.Ordinals{
+					Discrete: []int32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9},
+				},
 			},
 			Status: workloads.InstanceSetStatus{
 				Ordinals: []int32{0, 1, 2},
 				TemplatesStatus: []workloads.InstanceTemplateStatus{
 					{
 						Name:     "t1",
+						Replicas: 1,
 						Ordinals: []int32{3},
 					},
 					{
 						Name:     "t2",
+						Replicas: 1,
 						Ordinals: []int32{4},
 					},
 				},
 			},
 		}, map[string]sets.Set[int32]{
-			"": sets.New[int32](0, 1, 2, 3, 4),
+			defaultTemplateName: sets.New[int32](0, 1, 2, 3, 4), // take back 3 and 4
 		}, false),
 
-		PEntry("move replicas out of instance templates - mixed", &workloads.InstanceSet{
+		Entry("move replicas out of instance templates - take back and replace", &workloads.InstanceSet{
 			Spec: workloads.InstanceSetSpec{
 				Replicas: ptr.To[int32](5),
+				DefaultTemplateOrdinals: workloads.Ordinals{
+					Discrete: []int32{0, 1, 2, 3, 5, 6, 7, 8, 9}, // without ordinal 4
+				},
 			},
 			Status: workloads.InstanceSetStatus{
 				Ordinals: []int32{0, 1, 2},
 				TemplatesStatus: []workloads.InstanceTemplateStatus{
 					{
 						Name:     "t1",
+						Replicas: 1,
 						Ordinals: []int32{3},
 					},
 					{
 						Name:     "t2",
+						Replicas: 1,
 						Ordinals: []int32{4},
 					},
 				},
 			},
 		}, map[string]sets.Set[int32]{
-			"": sets.New[int32](0, 1, 2, 5, 4),
+			defaultTemplateName: sets.New[int32](0, 1, 2, 3, 5), // take back 3 and replace 4 with 5
 		}, false),
 	)
 
diff --git a/pkg/kbagent/service/probe_test.go b/pkg/kbagent/service/probe_test.go
index cb69fd274..87fdab662 100644
--- a/pkg/kbagent/service/probe_test.go
+++ b/pkg/kbagent/service/probe_test.go
@@ -20,6 +20,8 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package service
 
 import (
+	"encoding/json"
+	"fmt"
 	"time"
 
 	. "github.com/onsi/ginkgo/v2"
@@ -34,9 +36,10 @@ import (
 var _ = Describe("probe", func() {
 	Context("probe", func() {
 		var (
-			actions = []proto.Action{
+			probeName = "roleProbe"
+			actions   = []proto.Action{
 				{
-					Name: "roleProbe",
+					Name: probeName,
 					Exec: &proto.ExecAction{
 						Commands: []string{"/bin/bash", "-c", "echo -n leader"},
 					},
@@ -44,7 +47,7 @@ var _ = Describe("probe", func() {
 			}
 			probes = []proto.Probe{
 				{
-					Action:              "roleProbe",
+					Action:              probeName,
 					InitialDelaySeconds: 0,
 					PeriodSeconds:       1,
 					SuccessThreshold:    1,
@@ -90,6 +93,8 @@ var _ = Describe("probe", func() {
 
 		It("initial delay seconds", func() {
 			probes[0].InitialDelaySeconds = 60
+			defer func() { probes[0].InitialDelaySeconds = 0 }()
+
 			service, err := newProbeService(logr.New(nil), actionSvc, probes)
 			Expect(err).Should(BeNil())
 			Expect(service).ShouldNot(BeNil())
@@ -97,11 +102,107 @@ var _ = Describe("probe", func() {
 			Expect(service.Start()).Should(Succeed())
 
 			time.Sleep(1 * time.Second)
-			r := service.runners["roleProbe"]
+			r := service.runners[probeName]
 			Expect(r).ShouldNot(BeNil())
 			Expect(r.ticker).Should(BeNil())
 		})
 
+		It("send event", func() {
+			By("create probe service")
+			service, err := newProbeService(logr.New(nil), actionSvc, probes)
+			Expect(err).Should(BeNil())
+			Expect(service).ShouldNot(BeNil())
+
+			By("mock send event function")
+			eventChan := make(chan struct {
+				reason  string
+				message string
+			}, 128)
+			service.sendEventWithMessage = func(_ *logr.Logger, reason string, message string, _ bool) error {
+				eventChan <- struct{ reason, message string }{reason, message}
+				return nil
+			}
+
+			By("start probe service")
+			Expect(service.Start()).Should(Succeed())
+
+			By("check received event")
+			var receivedData struct{ reason, message string }
+			Eventually(eventChan).Should(Receive(&receivedData))
+			Expect(receivedData.reason).Should(Equal(probeName))
+			var event proto.ProbeEvent
+			Expect(json.Unmarshal([]byte(receivedData.message), &event)).Should(Succeed())
+			Eventually(event.Probe).Should(Equal(probeName))
+			Eventually(event.Code).Should(Equal(int32(0)))
+			Eventually(event.Output).Should(Equal([]byte("leader")))
+		})
+
+		It("send event - API server error", func() {
+			By("create probe service")
+			service, err := newProbeService(logr.New(nil), actionSvc, probes)
+			Expect(err).Should(BeNil())
+			Expect(service).ShouldNot(BeNil())
+
+			By("mock send event function with error")
+			var (
+				count = 0
+			)
+			service.sendEventWithMessage = func(_ *logr.Logger, reason string, message string, _ bool) error {
+				count += 1
+				return fmt.Errorf("API server error")
+			}
+			retrySendEventInterval = 1 * time.Second
+			defer func() { retrySendEventInterval = defaultRetrySendEventInterval }()
+
+			By("start probe service")
+			Expect(service.Start()).Should(Succeed())
+
+			By("wait for probe to send event error")
+			Eventually(func() int { return count }, 2*retrySendEventInterval).Should(BeNumerically(">", 1))
+		})
+
+		It("send event - after API server recover", func() {
+			By("create probe service")
+			service, err := newProbeService(logr.New(nil), actionSvc, probes)
+			Expect(err).Should(BeNil())
+			Expect(service).ShouldNot(BeNil())
+
+			By("mock send event function with temporary error")
+			var (
+				count     = 0
+				eventChan = make(chan struct {
+					reason  string
+					message string
+				}, 128)
+			)
+			service.sendEventWithMessage = func(_ *logr.Logger, reason string, message string, _ bool) error {
+				count += 1
+				if count <= 2 {
+					return fmt.Errorf("API server error")
+				}
+				eventChan <- struct{ reason, message string }{reason, message}
+				return nil
+			}
+			retrySendEventInterval = 1 * time.Second
+			defer func() { retrySendEventInterval = defaultRetrySendEventInterval }()
+
+			By("start probe service")
+			Expect(service.Start()).Should(Succeed())
+
+			By("wait for probe to send event error")
+			Eventually(func() int { return count }, 2*retrySendEventInterval).Should(BeNumerically(">", 1))
+
+			By("check received event after recover")
+			var receivedData struct{ reason, message string }
+			Eventually(eventChan, 2*retrySendEventInterval).Should(Receive(&receivedData))
+			Expect(receivedData.reason).Should(Equal(probeName))
+			var event proto.ProbeEvent
+			Expect(json.Unmarshal([]byte(receivedData.message), &event)).Should(Succeed())
+			Eventually(event.Probe).Should(Equal(probeName))
+			Eventually(event.Code).Should(Equal(int32(0)))
+			Eventually(event.Output).Should(Equal([]byte("leader")))
+		})
+
 		// TODO: more test cases
 	})
 })