Repository: kubean-io/kubean
Pull request #956 (2023-09-13 08:00:43)

webhook for clusterOperations
<!--  

Thanks for sending a pull request!  Here are some tips for you:

* make sure your commit is signed off

-->

#### What type of PR is this?

<!--
Add one of the following kinds:
/kind bug
/kind cleanup
/kind documentation
/kind enhancement
/kind feature

Optionally add one or more of the following kinds if applicable:
/kind api-change
/kind deprecation
/kind failing-test
/kind flake
/kind design
/kind regression
-->

/kind api-change

#### What this PR does / why we need it:

#### Which issue(s) this PR fixes:
<!--
*Automatically closes linked issue when PR is merged.
Usage: `Fixes #<issue number>`, or `Fixes (paste link of issue)`.
_If PR is about `failing-tests or flakes`, please post the related issues/tests in a comment and do not use `Fixes`_*
-->
Fixes #

#### Special notes for your reviewer:


hangscer8: install:

![企业微信截图_3a4a6fad-43d9-4956-b6b4-d5e47657fc73](https://github.com/kubean-io/kubean/assets/28976713/0d3a17f7-72ac-4704-add6-0d641b93f6e8)

running:

![企业微信截图_ab4cd1c3-956c-41a0-9ce5-747b9e7caf6b](https://github.com/kubean-io/kubean/assets/28976713/fabbf3cc-9b94-48d5-8a52-65f993555a49)

webhook:

![企业微信截图_8bc3e7c2-8953-4267-8bfa-d666da274013](https://github.com/kubean-io/kubean/assets/28976713/7541b5fc-c950-46de-9d9b-ed0c09d69fb3)



Patch:
diff --git a/.github/workflows/auto-pr-ci.yaml b/.github/workflows/auto-pr-ci.yaml
index 120f6d435..c1e2048dc 100644
--- a/.github/workflows/auto-pr-ci.yaml
+++ b/.github/workflows/auto-pr-ci.yaml
@@ -12,6 +12,7 @@ on:
 
 env:
   KUKEAN_OPERATOR_IMAGE_NAME: kubean-operator
+  KUBEAN_ADMISSION_IMAGE_NAME: kubean-admission
   SPRAY_JOB_IMAGE_NAME: spray-job
   AIRGAP_PATCH_IMAGE_NAME: airgap-patch
   KUBESPRAY_TAG: latest
@@ -121,6 +122,21 @@ jobs:
           docker tag $KUKEAN_OPERATOR_IMAGE_NAME $IMAGE_ID:$VERSION
           docker push $IMAGE_ID:$VERSION
 
+      - name: kubean-admission Build image
+        run: docker build . --file ./build/images/$KUBEAN_ADMISSION_IMAGE_NAME/Dockerfile --tag $KUBEAN_ADMISSION_IMAGE_NAME
+
+      - name: kubean-admission Push image
+        run: |
+          IMAGE_ID=ghcr.io/${{ env.REPO }}/$KUBEAN_ADMISSION_IMAGE_NAME
+          # Change all uppercase to lowercase
+          IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
+          # Strip git ref prefix from version
+          VERSION="$(git describe --tags --abbrev=8 --dirty)-e2e"
+          echo IMAGE_ID=$IMAGE_ID
+          echo VERSION=$VERSION
+          docker tag $KUBEAN_ADMISSION_IMAGE_NAME $IMAGE_ID:$VERSION
+          docker push $IMAGE_ID:$VERSION          
+
       - name: spray-job Build image
         run: docker build . --file ./build/images/$SPRAY_JOB_IMAGE_NAME/Dockerfile --tag $SPRAY_JOB_IMAGE_NAME  --build-arg SPRAY_TAG=$KUBESPRAY_TAG
 
diff --git a/.github/workflows/call-build-imgs-for-kubean.yaml b/.github/workflows/call-build-imgs-for-kubean.yaml
index 3e14f2ffe..15a10d1fc 100644
--- a/.github/workflows/call-build-imgs-for-kubean.yaml
+++ b/.github/workflows/call-build-imgs-for-kubean.yaml
@@ -42,6 +42,12 @@ jobs:
             ghcr.io/${{ inputs.REPO }}/kubean-operator:${{ inputs.BUILD_IMG_TAG }}
             ghcr.io/${{ inputs.REPO }}/kubean-operator:latest
           build-args: ""
+        - name: kubean-admission
+          dockerfile: build/images/kubean-admission/Dockerfile
+          tags: |
+            ghcr.io/${{ inputs.REPO }}/kubean-admission:${{ inputs.BUILD_IMG_TAG }}
+            ghcr.io/${{ inputs.REPO }}/kubean-admission:latest
+          build-args: ""
 
     steps:
     - uses: actions/checkout@v3
diff --git a/.github/workflows/call-sync-repos.yaml b/.github/workflows/call-sync-repos.yaml
index 279f18dfd..914ee0259 100644
--- a/.github/workflows/call-sync-repos.yaml
+++ b/.github/workflows/call-sync-repos.yaml
@@ -27,6 +27,7 @@ jobs:
         yq -i ".appVersion=\"${GITHUB_REF_NAME}\"" charts/kubean/Chart.yaml
         yq -i ".kubeanOperator.image.tag=\"${GITHUB_REF_NAME}\"" charts/kubean/values.yaml
         yq -i ".sprayJob.image.tag=\"${GITHUB_REF_NAME}\"" charts/kubean/values.yaml
+        yq -i ".kubeanAdmission.image.tag=\"${GITHUB_REF_NAME}\"" charts/kubean/values.yaml
 
     - name: helm checkout
       run: |
diff --git a/Makefile b/Makefile
index d08e72a15..7d0762f6b 100644
--- a/Makefile
+++ b/Makefile
@@ -57,6 +57,21 @@ kubean-operator: $(SOURCES)
 			--load \
 			.
 
+.PHONY: kubean-admission
+kubean-admission: $(SOURCES)
+	echo "Building kubean-admission for arch = $(BUILD_ARCH)"
+	export DOCKER_CLI_EXPERIMENTAL=enabled ;\
+	! ( docker buildx ls | grep kubean-admission-multi-platform-builder ) && docker buildx create --use --platform=$(BUILD_ARCH) --name kubean-admission-multi-platform-builder ;\
+	docker buildx build \
+			--build-arg kubean_version=$(KUBEAN_VERSION) \
+			--builder kubean-admission-multi-platform-builder \
+			--platform $(BUILD_ARCH) \
+			--tag $(REGISTRY_REPO)/kubean-admission:$(KUBEAN_IMAGE_VERSION)  \
+			--tag $(REGISTRY_REPO)/kubean-admission:latest  \
+			-f ./build/images/kubean-admission/Dockerfile \
+			--load \
+			.
+
 .PHONY: spray-job
 spray-job: $(SOURCES)
 	echo "Building spray-job for arch = $(BUILD_ARCH)"
diff --git a/api/apis/clusteroperation/v1alpha1/types.go b/api/apis/clusteroperation/v1alpha1/types.go
index 0dff80483..732d2e641 100644
--- a/api/apis/clusteroperation/v1alpha1/types.go
+++ b/api/apis/clusteroperation/v1alpha1/types.go
@@ -115,7 +115,6 @@ const (
 	RunningStatus   OpsStatus = "Running"
 	SucceededStatus OpsStatus = "Succeeded"
 	FailedStatus    OpsStatus = "Failed"
-	BlockedStatus   OpsStatus = "Blocked"
 )
 
 // Status contains information about the current status of a
diff --git a/api/constants/constants.go b/api/constants/constants.go
index c2211bbf6..87444c9e0 100644
--- a/api/constants/constants.go
+++ b/api/constants/constants.go
@@ -9,3 +9,5 @@ const Hosts_yml = "hosts.yml"
 const Group_vars_yml = "group_vars.yml"
 
 const SSH_privatekey = "ssh-privatekey"
+
+const KubeanClusterHasCompleted = "hasCompleted"
diff --git a/build/images/kubean-admission/Dockerfile b/build/images/kubean-admission/Dockerfile
new file mode 100644
index 000000000..3f44c27f2
--- /dev/null
+++ b/build/images/kubean-admission/Dockerfile
@@ -0,0 +1,18 @@
+FROM golang:1.20.4 as build
+
+WORKDIR /kubean
+
+ENV GO111MODULE=on \
+    GOPROXY=https://goproxy.cn,direct
+
+COPY . .
+
+RUN CGO_ENABLED=0 GOOS=linux go build -mod vendor -o kubean-admission ./cmd/kubean-admission/main.go
+
+FROM alpine:3.15
+
+RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories
+
+COPY --from=build /kubean/kubean-admission /bin/
+
+CMD ["/bin/kubean-admission"]
diff --git a/charts/kubean/.relok8s-images.yaml b/charts/kubean/.relok8s-images.yaml
index 631b1a112..81731e52d 100644
--- a/charts/kubean/.relok8s-images.yaml
+++ b/charts/kubean/.relok8s-images.yaml
@@ -2,3 +2,5 @@
 - "{{ .kubeanOperator.image.registry }}/{{ .kubeanOperator.image.repository }}:{{ .kubeanOperator.image.tag }}"
 # spray-job
 - "{{ .sprayJob.image.registry }}/{{ .sprayJob.image.repository }}:{{ .sprayJob.image.tag }}"
+# kubean-admission
+- "{{ .kubeanAdmission.image.registry }}/{{ .kubeanAdmission.image.repository }}:{{ .kubeanAdmission.image.tag }}"
diff --git a/charts/kubean/templates/_helpers.tpl b/charts/kubean/templates/_helpers.tpl
index b614ed572..c278c5a28 100644
--- a/charts/kubean/templates/_helpers.tpl
+++ b/charts/kubean/templates/_helpers.tpl
@@ -27,6 +27,10 @@ If release name contains chart name it will be used as a full name.
 {{- end }}
 {{- end }}
 
+{{- define "kubean.admissionName" -}}
+{{- printf "%s-admission" .Release.Name }}
+{{- end }}
+
 {{/*
 Create chart name and version as used by the chart label.
 */}}
@@ -38,18 +42,27 @@ Create chart name and version as used by the chart label.
 Common labels
 */}}
 {{- define "kubean.labels" -}}
-{{ include "kubean.selectorLabels" . }}
+{{ include "kubeanOperator.selectorLabels" . }}
 app.kubernetes.io/managed-by: {{ .Release.Service }}
 {{- end }}
 
 {{/*
 Selector labels
 */}}
-{{- define "kubean.selectorLabels" -}}
+{{- define "kubeanOperator.selectorLabels" -}}
 app.kubernetes.io/name: {{ include "kubean.name" . }}
 app.kubernetes.io/instance: {{ .Release.Name }}
 {{- end }}
 
+{{- define "kubeanAdmission.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "kubean.name" . }}
+app.kubernetes.io/instance: {{ printf "%s-admission-controller" .Release.Name }}
+{{- end }}
+
+{{- define "kubeanAdmission.svcName" -}}
+{{- printf "%s-admission" .Chart.Name }}
+{{- end }}
+
 {{/*
 Create the name of the service account to use
 */}}
diff --git a/charts/kubean/templates/clusterrole.yaml b/charts/kubean/templates/clusterrole.yaml
index 29caf0750..e6866a87b 100644
--- a/charts/kubean/templates/clusterrole.yaml
+++ b/charts/kubean/templates/clusterrole.yaml
@@ -20,5 +20,8 @@ rules:
     resources: ['*']
     verbs: ['*']
   - apiGroups: ['authorization.k8s.io']
+    resources: ['*']
+    verbs: ['*']
+  - apiGroups: ['admissionregistration.k8s.io']
     resources: ['*']
     verbs: ['*']
\ No newline at end of file
diff --git a/charts/kubean/templates/deployment.yaml b/charts/kubean/templates/deployment.yaml
index a05aa7f27..e8a105061 100644
--- a/charts/kubean/templates/deployment.yaml
+++ b/charts/kubean/templates/deployment.yaml
@@ -9,7 +9,7 @@ spec:
   replicas: {{ .Values.kubeanOperator.replicaCount }}
   selector:
     matchLabels:
-      {{- include "kubean.selectorLabels" . | nindent 6 }}
+      {{- include "kubeanOperator.selectorLabels" . | nindent 6 }}
   template:
     metadata:
       {{- with .Values.kubeanOperator.podAnnotations }}
@@ -17,7 +17,7 @@ spec:
         {{- toYaml . | nindent 8 }}
       {{- end }}
       labels:
-        {{- include "kubean.selectorLabels" . | nindent 8 }}
+        {{- include "kubeanOperator.selectorLabels" . | nindent 8 }}
     spec:
       {{- with .Values.kubeanOperator.image.pullSecrets }}
       imagePullSecrets:
diff --git a/charts/kubean/templates/deployment_webhook.yaml b/charts/kubean/templates/deployment_webhook.yaml
new file mode 100644
index 000000000..7da5c7223
--- /dev/null
+++ b/charts/kubean/templates/deployment_webhook.yaml
@@ -0,0 +1,59 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "kubean.admissionName" . }}
+  namespace: {{ include "kubean.namespace" . }}
+  labels:
+    {{- include "kubean.labels" . | nindent 4 }}
+spec:
+  replicas: {{ .Values.kubeanAdmission.replicaCount }}
+  selector:
+    matchLabels:
+      {{- include "kubeanAdmission.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      {{- with .Values.kubeanOperator.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        {{- include "kubeanAdmission.selectorLabels" . | nindent 8 }}
+    spec:
+      {{- with .Values.kubeanOperator.image.pullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      serviceAccountName: {{ include "kubean.serviceAccountName" . }}
+      securityContext:
+        {{- toYaml .Values.kubeanOperator.podSecurityContext | nindent 8 }}
+      containers:
+        - name: {{ .Chart.Name }}
+          securityContext:
+            {{- toYaml .Values.kubeanOperator.securityContext | nindent 12 }}
+          image: "{{ .Values.kubeanAdmission.image.registry }}/{{ .Values.kubeanAdmission.image.repository }}:{{ .Values.kubeanAdmission.image.tag | default .Chart.AppVersion }}"
+          imagePullPolicy: {{ .Values.kubeanOperator.image.pullPolicy }}
+          env:
+            - name: WEBHOOK_SERVICE_NAME
+              value: {{ include "kubeanAdmission.svcName" . }}
+            - name: WEBHOOK_SERVICE_NAMESPACE
+              value: {{ include "kubean.namespace" . }}
+            - name: WEBHOOK_FAILURE_POLICY
+              value: Ignore
+          ports:
+            - name: webhook-port
+              containerPort: 10443
+              protocol: TCP
+          resources:
+            {{- toYaml .Values.kubeanOperator.resources | nindent 12 }}
+      {{- with .Values.kubeanOperator.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.kubeanOperator.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.kubeanOperator.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
diff --git a/charts/kubean/templates/service.yaml b/charts/kubean/templates/service.yaml
index d24a7da45..b4ea9376a 100644
--- a/charts/kubean/templates/service.yaml
+++ b/charts/kubean/templates/service.yaml
@@ -13,4 +13,4 @@ spec:
       protocol: TCP
       name: http
   selector:
-    {{- include "kubean.selectorLabels" . | nindent 4 }}
+    {{- include "kubeanOperator.selectorLabels" . | nindent 4 }}
diff --git a/charts/kubean/templates/service_webhook.yaml b/charts/kubean/templates/service_webhook.yaml
new file mode 100644
index 000000000..52f2e02f8
--- /dev/null
+++ b/charts/kubean/templates/service_webhook.yaml
@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "kubeanAdmission.svcName" . }}
+  namespace: {{ include "kubean.namespace" . }}
+  labels:
+    {{- include "kubean.labels" . | nindent 4 }}
+spec:
+  type: ClusterIP
+  ports:
+    - port: 443
+      targetPort: webhook-port
+      protocol: TCP
+      name: webhook-port
+  selector:
+    {{- include "kubeanAdmission.selectorLabels" . | nindent 4 }}
diff --git a/charts/kubean/templates/webhook.yaml b/charts/kubean/templates/webhook.yaml
new file mode 100644
index 000000000..7f76e2ca1
--- /dev/null
+++ b/charts/kubean/templates/webhook.yaml
@@ -0,0 +1,4 @@
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingWebhookConfiguration
+metadata:
+  name: kubean-admission-webhook
diff --git a/charts/kubean/values.yaml b/charts/kubean/values.yaml
index 9bc49f1ec..23729c6f2 100644
--- a/charts/kubean/values.yaml
+++ b/charts/kubean/values.yaml
@@ -70,12 +70,23 @@ kubeanOperator:
         podAffinityTerm:
           labelSelector:
             matchExpressions:
-            - key: app.kubernetes.io/instance
+            - key: app.kubernetes.io/name
               operator: In
               values:
               - kubean
           topologyKey: kubernetes.io/hostname
 
+kubeanAdmission:
+  replicaCount: 1
+  ## define admission image
+  image:
+    # -- spray-job image registry
+    registry: ghcr.io
+    # -- spray-job image registry
+    repository: kubean-io/kubean-admission
+    # -- the image tag whose default is the chart appVersion
+    tag: ""
+
 ## define spray-job configuration
 sprayJob:
   ## define spray-job image
diff --git a/cmd/kubean-admission/app/app.go b/cmd/kubean-admission/app/app.go
new file mode 100644
index 000000000..6efdc64b2
--- /dev/null
+++ b/cmd/kubean-admission/app/app.go
@@ -0,0 +1,81 @@
+// Copyright 2023 Authors of kubean-io
+// SPDX-License-Identifier: Apache-2.0
+
+package app
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"os"
+
+	kubeanClusterOperationClientSet "github.com/kubean-io/kubean-api/generated/clusteroperation/clientset/versioned"
+	clusteropswebhook "github.com/kubean-io/kubean/pkg/webhooks/clusterops"
+
+	"github.com/kubean-io/kubean/pkg/version"
+
+	"github.com/spf13/cobra"
+	"k8s.io/client-go/kubernetes"
+	rest "k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+	klog "k8s.io/klog/v2"
+)
+
+func NewCommand(ctx context.Context) *cobra.Command {
+	opts := NewOptions()
+	cmd := &cobra.Command{
+		Use:  "kubean-admission",
+		Long: "run admission controller for ClusterOperation",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if errs := opts.Validate(); len(errs) != 0 {
+				return errs.ToAggregate()
+			}
+			return Run(ctx, opts)
+		},
+	}
+	versionCmd := &cobra.Command{
+		Use:   "version",
+		Short: "Print the version of admission controller",
+		Run: func(cmd *cobra.Command, args []string) {
+			cmd.Println(version.Get())
+		},
+	}
+	cmd.Flags().AddGoFlagSet(flag.CommandLine)
+	cmd.AddCommand(versionCmd)
+	opts.AddFlags(cmd.Flags())
+	return cmd
+}
+
+func Run(ctx context.Context, opt *Options) error {
+	klog.Warningf("Start Admission Controller")
+	resetConfig, err := rest.InClusterConfig()
+	if err != nil {
+		klog.ErrorS(err, " can not load k8s config")
+		return err
+	}
+	resetConfig.QPS = opt.KubeAPIQPS
+	resetConfig.Burst = opt.KubeAPIBurst
+	if err != nil {
+		resetConfig, err = clientcmd.BuildConfigFromFlags("", os.Getenv("HOME")+"/.kube/config")
+		if err != nil {
+			return err
+		}
+	}
+	ClientSet, err := kubernetes.NewForConfig(resetConfig)
+	if err != nil {
+		return err
+	}
+	clusterClientOperationSet, err := kubeanClusterOperationClientSet.NewForConfig(resetConfig)
+	if err != nil {
+		return err
+	}
+	go func() {
+		clusteropswebhook.CreateHTTPSCASecretWithLock(ctx, ClientSet)
+	}()
+	CASecret := clusteropswebhook.WaitForCASecretExist(ClientSet)
+	if err := clusteropswebhook.CreateHTTPSCAFilesFromSecret(CASecret); err != nil {
+		return err
+	}
+	clusteropswebhook.StartWebHookHTTPSServer(clusteropswebhook.PrepareWebHookHTTPSServer(clusterClientOperationSet))
+	return fmt.Errorf("admission has exited")
+}
diff --git a/cmd/kubean-admission/app/options.go b/cmd/kubean-admission/app/options.go
new file mode 100644
index 000000000..4ecb6ac0a
--- /dev/null
+++ b/cmd/kubean-admission/app/options.go
@@ -0,0 +1,32 @@
+// Copyright 2023 Authors of kubean-io
+// SPDX-License-Identifier: Apache-2.0
+
+package app
+
+import (
+	"github.com/spf13/pflag"
+	"k8s.io/apimachinery/pkg/util/validation/field"
+)
+
+type Options struct {
+	KubeAPIQPS float32
+	// KubeAPIBurst is the burst to allow while talking with kubean-apiserver.
+	KubeAPIBurst int
+}
+
+func NewOptions() *Options {
+	return &Options{
+		KubeAPIQPS:   100.0,
+		KubeAPIBurst: 100,
+	}
+}
+
+func (o *Options) AddFlags(flags *pflag.FlagSet) {
+	flags.Float32Var(&o.KubeAPIQPS, "kube-api-qps", 100.0, "QPS to use while talking with kubean-apiserver. Doesn't cover events and node heartbeat apis which rate limiting is controlled by a different set of flags.")
+	flags.IntVar(&o.KubeAPIBurst, "kube-api-burst", 100, "Burst to use while talking with kubean-apiserver. Doesn't cover events and node heartbeat apis which rate limiting is controlled by a different set of flags.")
+}
+
+func (o *Options) Validate() field.ErrorList {
+	errs := field.ErrorList{}
+	return errs
+}
diff --git a/cmd/kubean-admission/main.go b/cmd/kubean-admission/main.go
new file mode 100644
index 000000000..cf5085e33
--- /dev/null
+++ b/cmd/kubean-admission/main.go
@@ -0,0 +1,24 @@
+// Copyright 2023 Authors of kubean-io
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+	"os"
+
+	apiserver "k8s.io/apiserver/pkg/server"
+	"k8s.io/component-base/logs"
+	klog "k8s.io/klog/v2"
+
+	"github.com/kubean-io/kubean/cmd/kubean-admission/app"
+)
+
+func main() {
+	logs.InitLogs()
+	defer logs.FlushLogs()
+	ctx := apiserver.SetupSignalContext()
+	if err := app.NewCommand(ctx).Execute(); err != nil {
+		klog.Error(err.Error())
+		os.Exit(1)
+	}
+}
diff --git a/hack/deploy.sh b/hack/deploy.sh
index f55ba6f68..b3bbadea0 100755
--- a/hack/deploy.sh
+++ b/hack/deploy.sh
@@ -32,10 +32,10 @@ LOCAL_RELEASE_NAME=kubean
 # replace the default values.yaml, the image repo or image revision
 value_override=""
 if [ "${IMG_REGISTRY}" != "" ]; then
-    value_override=" $value_override --set kubeanOperator.image.registry=${IMG_REGISTRY} --set sprayJob.image.registry=${IMG_REGISTRY}"
+    value_override=" $value_override --set kubeanOperator.image.registry=${IMG_REGISTRY} --set kubeanAdmission.image.registry=${IMG_REGISTRY} --set sprayJob.image.registry=${IMG_REGISTRY}"
 fi
 if [ "${IMG_VER}" != "" ]; then
-    value_override=" $value_override --set kubeanOperator.image.tag=${IMG_VER} --set sprayJob.image.tag=${IMG_VER}"
+    value_override=" $value_override --set kubeanOperator.image.tag=${IMG_VER} --set kubeanAdmission.image.tag=${IMG_VER} --set sprayJob.image.tag=${IMG_VER}"
 fi
 
 echo "value_override: ${value_override}  ##########"
diff --git a/hack/local-chart-to-deploy.sh b/hack/local-chart-to-deploy.sh
index 1ad6fdb1b..399c9c1b5 100644
--- a/hack/local-chart-to-deploy.sh
+++ b/hack/local-chart-to-deploy.sh
@@ -25,8 +25,10 @@ helm upgrade \
     --create-namespace \
     --cleanup-on-fail \
     --set kubeanOperator.image.registry="${IMAGE_REGISTRY}" \
+    --set kubeanAdmission.image.registry="${IMAGE_REGISTRY}" \
     --set sprayJob.image.registry="${IMAGE_REGISTRY}" \
     --set kubeanOperator.image.tag="${IMAGE_TAG}" \
+    --set kubeanAdmission.image.tag="${IMAGE_TAG}" \
     --set sprayJob.image.tag="${IMAGE_TAG}" \
     --kubeconfig "${KUBECONFIG_PATH}" \
     --timeout 1200s \
diff --git a/pkg/controllers/clusterops/controller.go b/pkg/controllers/clusterops/controller.go
index f498fa3f7..289dc4c74 100644
--- a/pkg/controllers/clusterops/controller.go
+++ b/pkg/controllers/clusterops/controller.go
@@ -183,42 +183,6 @@ func (c *Controller) FetchJobConditionStatusAndCompletionTime(clusterOps *cluste
 	return clusteroperationv1alpha1.RunningStatus, nil, nil
 }
 
-func (c *Controller) ListClusterOps(clusterName string) ([]clusteroperationv1alpha1.ClusterOperation, error) {
-	list, err := c.KubeanClusterOpsSet.KubeanV1alpha1().ClusterOperations().List(context.Background(), metav1.ListOptions{LabelSelector: labels.SelectorFromSet(map[string]string{constants.KubeanClusterLabelKey: clusterName}).String()})
-	if err != nil {
-		return nil, err
-	}
-	return list.Items, nil
-}
-
-func (c *Controller) CurrentJobNeedBlock(clusterOps *clusteroperationv1alpha1.ClusterOperation, listClusterOps func(clusterName string) ([]clusteroperationv1alpha1.ClusterOperation, error)) (bool, error) {
-	if clusterOps.Status.Status == clusteroperationv1alpha1.SucceededStatus || clusterOps.Status.Status == clusteroperationv1alpha1.FailedStatus {
-		return false, nil
-	}
-	clusterOpsList, err := listClusterOps(clusterOps.Spec.Cluster)
-	if err != nil {
-		return false, err
-	}
-	filter := func(ops clusteroperationv1alpha1.ClusterOperation) bool {
-		// todo: clusterOps has the group label and number label, first find the early group and then find the before number in the same group if possible
-		// try to find the early running clusterOps job in the same cluster
-		return ops.Name != clusterOps.Name &&
-			ops.CreationTimestamp.UnixMilli() < clusterOps.CreationTimestamp.UnixMilli() && // <= or < ? , use "<" to avoid two jobs with the same createTime waiting for each others(blocked by each others) ,createTimes is base on second not mills.
-			(ops.Status.Status == clusteroperationv1alpha1.RunningStatus || ops.Status.JobRef.IsEmpty()) // Empty jobRef means the job is blocked or ready to run.
-	}
-	runningClusterOpsList := make([]clusteroperationv1alpha1.ClusterOperation, 0)
-	for i := range clusterOpsList {
-		if filter(clusterOpsList[i]) {
-			runningClusterOpsList = append(runningClusterOpsList, clusterOpsList[i])
-		}
-	}
-	if len(runningClusterOpsList) != 0 {
-		klog.Warningf("clusterOps %s blocked by %s", clusterOps.Name, runningClusterOpsList[0].Name)
-		return true, nil
-	}
-	return false, nil
-}
-
 func IsValidImageName(image string) bool {
 	isNumberOrLetter := func(r rune) bool {
 		return unicode.IsLetter(r) || unicode.IsNumber(r)
@@ -329,22 +293,6 @@ func (c *Controller) Reconcile(ctx context.Context, req controllerruntime.Reques
 		klog.ErrorS(err, "failed to update the ownReference configData or secretData", "clusterOps", clusterOps.Name)
 		return controllerruntime.Result{RequeueAfter: RequeueAfter}, nil
 	}
-	needBlock, err := c.CurrentJobNeedBlock(clusterOps, c.ListClusterOps)
-	if err != nil {
-		klog.ErrorS(err, "failed to list clusterOps", "cluster", clusterOps.Spec.Cluster)
-		return controllerruntime.Result{RequeueAfter: RequeueAfter}, nil
-	}
-	if needBlock {
-		klog.Infof("clusterOps %s is blocked and waiting for other clusterOps completed", clusterOps.Name)
-		if clusterOps.Status.Status != clusteroperationv1alpha1.BlockedStatus {
-			clusterOps.Status.Status = clusteroperationv1alpha1.BlockedStatus
-			if err := c.Client.Status().Update(context.Background(), clusterOps); err != nil {
-				klog.Warningf("clusterOps %s update Status to Blocked but %s", clusterOps.Name, err.Error())
-				return controllerruntime.Result{RequeueAfter: RequeueAfter}, nil
-			}
-		}
-		return controllerruntime.Result{RequeueAfter: LoopForJobStatus}, nil
-	}
 
 	needRequeue, err = c.CreateKubeSprayJob(clusterOps)
 	if err != nil {
@@ -362,6 +310,10 @@ func (c *Controller) Reconcile(ctx context.Context, req controllerruntime.Reques
 	if needRequeue {
 		return controllerruntime.Result{RequeueAfter: LoopForJobStatus}, nil
 	}
+	if err := c.UpdateStatusForLabel(clusterOps); err != nil {
+		klog.Error(err)
+		return controllerruntime.Result{RequeueAfter: RequeueAfter}, nil
+	}
 	return controllerruntime.Result{}, nil
 }
 
@@ -858,6 +810,22 @@ func (c *Controller) UpdateOwnReferenceToClusterOps(clusterOps *clusteroperation
 	)
 }
 
+func (c *Controller) UpdateStatusForLabel(ops *clusteroperationv1alpha1.ClusterOperation) error {
+	if ops.Labels == nil {
+		ops.Labels = make(map[string]string)
+	}
+	if ops.Labels[constants.KubeanClusterHasCompleted] == "done" {
+		return nil
+	}
+	if ops.Status.Status == clusteroperationv1alpha1.SucceededStatus || ops.Status.Status == clusteroperationv1alpha1.FailedStatus {
+		ops.Labels[constants.KubeanClusterHasCompleted] = "done"
+		if err := c.Client.Update(context.Background(), ops); err != nil {
+			return err
+		} // update label for ListOperation
+	}
+	return nil
+}
+
 // BackUpDataRef perform the backup of configRef and secretRef and return (needRequeue,error).
 func (c *Controller) BackUpDataRef(clusterOps *clusteroperationv1alpha1.ClusterOperation, cluster *clusterv1alpha1.Cluster) (bool, error) {
 	timestamp := fmt.Sprintf("-%d", time.Now().UnixMilli())
diff --git a/pkg/util/certmanager.go b/pkg/util/certmanager.go
new file mode 100644
index 000000000..f6a2c2a95
--- /dev/null
+++ b/pkg/util/certmanager.go
@@ -0,0 +1,120 @@
+// Copyright 2023 Authors of kubean-io
+// SPDX-License-Identifier: Apache-2.0
+
+package util
+
+import (
+	"bytes"
+	cryptorand "crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"math/big"
+	"time"
+)
+
+// certManager helps to generate certs.
+type certManager struct {
+	Organizations []string      `json:"organizations"`
+	EffectiveTime time.Duration `json:"effectiveTime"`
+	DNSNames      []string      `json:"DNSNames"`
+	CommonName    string        `json:"commonName"`
+}
+
+func NewCertManager(
+	Orz []string,
+	effectiveTime time.Duration,
+	dnsNames []string,
+	commonName string,
+) *certManager {
+	return &certManager{
+		Organizations: Orz,
+		EffectiveTime: effectiveTime,
+		DNSNames:      dnsNames,
+		CommonName:    commonName,
+	}
+}
+
+// GenerateSelfSignedCerts return self-signed certs according to provided dns.
+func (m *certManager) GenerateSelfSignedCerts() (*bytes.Buffer, *bytes.Buffer, error) {
+	var serverCertPEM *bytes.Buffer
+	var serverPrivateKeyPEM *bytes.Buffer
+	var err error
+	// CA config
+	ca := &x509.Certificate{
+		SerialNumber: big.NewInt(2021),
+		Subject: pkix.Name{
+			Organization: m.Organizations,
+		},
+		NotBefore:             time.Now(),
+		NotAfter:              time.Now().Add(m.EffectiveTime),
+		IsCA:                  true,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
+		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
+		BasicConstraintsValid: true,
+	}
+
+	var caPrivateKey *rsa.PrivateKey
+	caPrivateKey, err = rsa.GenerateKey(cryptorand.Reader, 4096)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// self signed CA certificate
+	var caBytes []byte
+	caBytes, err = x509.CreateCertificate(cryptorand.Reader, ca, ca, &caPrivateKey.PublicKey, caPrivateKey)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// PEM encode CA cert
+	caPEM := new(bytes.Buffer)
+	_ = pem.Encode(caPEM, &pem.Block{
+		Type:  "CERTIFICATE",
+		Bytes: caBytes,
+	})
+
+	// server cert config
+	cert := &x509.Certificate{
+		DNSNames:     m.DNSNames,
+		SerialNumber: big.NewInt(1658),
+		Subject: pkix.Name{
+			CommonName:   m.CommonName,
+			Organization: m.Organizations,
+		},
+		NotBefore:    time.Now(),
+		NotAfter:     time.Now().AddDate(1, 0, 0),
+		SubjectKeyId: []byte{1, 2, 3, 4, 6},
+		ExtKeyUsage:  []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
+		KeyUsage:     x509.KeyUsageDigitalSignature,
+	}
+
+	// server private key
+	var serverPrivateKey *rsa.PrivateKey
+	serverPrivateKey, err = rsa.GenerateKey(cryptorand.Reader, 4096)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// sign the server cert
+	var serverCertBytes []byte
+	serverCertBytes, err = x509.CreateCertificate(cryptorand.Reader, cert, ca, &serverPrivateKey.PublicKey, caPrivateKey)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// PEM encode the server cert and key
+	serverCertPEM = new(bytes.Buffer)
+	_ = pem.Encode(serverCertPEM, &pem.Block{
+		Type:  "CERTIFICATE",
+		Bytes: serverCertBytes,
+	})
+	serverPrivateKeyPEM = new(bytes.Buffer)
+	_ = pem.Encode(serverPrivateKeyPEM, &pem.Block{
+		Type:  "RSA PRIVATE KEY",
+		Bytes: x509.MarshalPKCS1PrivateKey(serverPrivateKey),
+	})
+
+	return serverCertPEM, serverPrivateKeyPEM, err
+}
diff --git a/pkg/util/cluster.go b/pkg/util/cluster.go
index df183bda0..4dee83d3e 100644
--- a/pkg/util/cluster.go
+++ b/pkg/util/cluster.go
@@ -112,3 +112,10 @@ func UpdateOwnReference(client kubernetes.Interface, configMapList []*apis.Confi
 	}
 	return nil
 }
+
+func GetCurrentRunningPodName() string {
+	if name := os.Getenv("HOSTNAME"); name != "" {
+		return name
+	}
+	return "default"
+}
diff --git a/pkg/util/files.go b/pkg/util/files.go
new file mode 100644
index 000000000..92bf570f7
--- /dev/null
+++ b/pkg/util/files.go
@@ -0,0 +1,30 @@
+// Copyright 2023 Authors of kubean-io
+// SPDX-License-Identifier: Apache-2.0
+
+package util
+
+import (
+	"os"
+)
+
+func WriteFile(filepath string, bytes []byte) error {
+	f, err := os.Create(filepath)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	_, err = f.Write(bytes)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func IsExist(filepath string) bool {
+	if _, err := os.Stat(filepath); err == nil {
+		return true
+	}
+	return false
+}
diff --git a/pkg/webhooks/clusterops/clusterops_webhook.go b/pkg/webhooks/clusterops/clusterops_webhook.go
new file mode 100644
index 000000000..fc10fedff
--- /dev/null
+++ b/pkg/webhooks/clusterops/clusterops_webhook.go
@@ -0,0 +1,370 @@
+// Copyright 2023 Authors of kubean-io
+// SPDX-License-Identifier: Apache-2.0
+
+package clusterops
+
+import (
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"os"
+	"path/filepath"
+	"time"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/selection"
+	"k8s.io/client-go/tools/leaderelection"
+	"k8s.io/client-go/tools/leaderelection/resourcelock"
+
+	clusteroperationv1alpha1 "github.com/kubean-io/kubean-api/apis/clusteroperation/v1alpha1"
+	"github.com/kubean-io/kubean-api/constants"
+	clusterOperationClientSet "github.com/kubean-io/kubean-api/generated/clusteroperation/clientset/versioned"
+
+	admissionv1 "k8s.io/api/admission/v1"
+	admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/klog/v2"
+
+	"github.com/kubean-io/kubean/pkg/util"
+)
+
+var (
+	certsDir          = "/etc/webhook/certs"
+	certKey           = "tls.key"
+	certFile          = "tls.crt"
+	Organization      = "kubean.io"
+	DefaultEffectTime = 10 * 365 * 24 * time.Hour
+	CAStoreSecret     = "webhook-http-ca-secret"
+
+	WebHookPath             = "/webhook"
+	WebhookSVCNamespace, _  = os.LookupEnv("WEBHOOK_SERVICE_NAMESPACE")
+	WebhookSVCName, _       = os.LookupEnv("WEBHOOK_SERVICE_NAME")
+	ClusterOperationWebhook = "kubean-admission-webhook"
+	FailurePolicy, _        = os.LookupEnv("WEBHOOK_FAILURE_POLICY")
+	dnsNames                = []string{
+		WebhookSVCName,
+		WebhookSVCName + "." + WebhookSVCNamespace,
+		WebhookSVCName + "." + WebhookSVCNamespace + "." + "svc",
+		WebhookSVCName + "." + WebhookSVCNamespace + "." + "svc.cluster.local",
+	}
+	commonName = WebhookSVCName + "." + WebhookSVCNamespace + "." + "svc"
+)
+
+func CreateHTTPSCASecretWithLock(ctx context.Context, client kubernetes.Interface) error {
+	defer func() {
+		if r := recover(); r != nil {
+			klog.Errorf("webhook RunWithLeaseLock but error %v", r)
+		}
+	}()
+	lock := &resourcelock.LeaseLock{
+		LeaseMeta: metav1.ObjectMeta{
+			Name:      "lease-for-kubean-webhook-ca-create",
+			Namespace: util.GetCurrentNSOrDefault(),
+		},
+		Client: client.CoordinationV1(),
+		LockConfig: resourcelock.ResourceLockConfig{
+			Identity: util.GetCurrentRunningPodName(), // podName as ID
+		},
+	}
+	leaderelection.RunOrDie(ctx, leaderelection.LeaderElectionConfig{
+		Lock:            lock,
+		ReleaseOnCancel: true,
+		LeaseDuration:   time.Second * 60,
+		RenewDeadline:   time.Second * 30,
+		RetryPeriod:     time.Second * 20,
+		Callbacks: leaderelection.LeaderCallbacks{
+			OnStartedLeading: func(ctx context.Context) {
+				klog.Warningf("webhook create CA OnStartedLeading on %s", util.GetCurrentRunningPodName())
+				if err := EnsureCASecretExist(client); err == nil {
+					UpdateClusterOperationWebhook(client)
+				}
+				<-ctx.Done()
+			},
+			OnNewLeader: func(identity string) {
+				klog.Warningf("webhook create CA OnNewLeader on %s", identity)
+			},
+			OnStoppedLeading: func() {
+			},
+		},
+	})
+	return nil
+}
+
+func WaitForCASecretExist(client kubernetes.Interface) *corev1.Secret {
+	var result *corev1.Secret
+	for {
+		time.Sleep(time.Second * 2)
+		result, _ = client.CoreV1().Secrets(util.GetCurrentNSOrDefault()).Get(context.Background(), CAStoreSecret, metav1.GetOptions{})
+		if result != nil && len(result.Data) == 2 {
+			return result
+		}
+	}
+}
+
+func EnsureCASecretExist(client kubernetes.Interface) error {
+	_, err := client.CoreV1().Secrets(util.GetCurrentNSOrDefault()).Get(context.Background(), CAStoreSecret, metav1.GetOptions{})
+	if err != nil && apierrors.IsNotFound(err) {
+		newSecret, err := createHTTPSCAInSecret()
+		if err != nil {
+			klog.Error(err)
+			return err
+		}
+		klog.Warning("webhook create secret for https CA data")
+		if _, err := client.CoreV1().Secrets(util.GetCurrentNSOrDefault()).Create(context.Background(), newSecret, metav1.CreateOptions{}); err != nil {
+			klog.Error(err)
+			return err
+		}
+		return nil
+	}
+	return nil
+}
+
+func createHTTPSCAInSecret() (*corev1.Secret, error) {
+	serverCertPEM, serverPrivateKeyPEM, err := util.NewCertManager(
+		[]string{Organization},
+		DefaultEffectTime,
+		dnsNames,
+		commonName,
+	).GenerateSelfSignedCerts()
+	if err != nil {
+		return nil, err
+	}
+	secret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      CAStoreSecret,
+			Namespace: util.GetCurrentNSOrDefault(),
+		},
+		Type: corev1.SecretTypeOpaque,
+		Data: map[string][]byte{
+			"crt": []byte(base64.StdEncoding.EncodeToString(serverCertPEM.Bytes())),
+			"key": []byte(base64.StdEncoding.EncodeToString(serverPrivateKeyPEM.Bytes())),
+		},
+	}
+	return secret, nil
+}
+
+func CreateHTTPSCAFilesFromSecret(secret *corev1.Secret) error {
+	if certsDir == "" {
+		return errors.New("empty certsDir")
+	}
+	if util.IsExist(filepath.Join(certsDir, certFile)) && util.IsExist(filepath.Join(certsDir, certKey)) {
+		// need not create CA files for https.
+		return nil
+	}
+	if err := os.MkdirAll(certsDir, 0o666); err != nil {
+		return err
+	}
+	serverCertPEM, err := base64.StdEncoding.DecodeString(string(secret.Data["crt"]))
+	if err != nil {
+		klog.ErrorS(err, "can not read crt data from secret")
+		return err
+	}
+	serverPrivateKeyPEM, err := base64.StdEncoding.DecodeString(string(secret.Data["key"]))
+	if err != nil {
+		klog.ErrorS(err, "can not read key data from secret")
+		return err
+	}
+	err = util.WriteFile(filepath.Join(certsDir, certFile), serverCertPEM)
+	if err != nil {
+		klog.ErrorS(err, "failed to write tls.cert")
+		return err
+	}
+
+	err = util.WriteFile(filepath.Join(certsDir, certKey), serverPrivateKeyPEM)
+	if err != nil {
+		klog.ErrorS(err, "failed to write tls.key")
+		return err
+	}
+	return nil
+}
+
+type PingHandler struct{}
+
+func (p PingHandler) ServeHTTP(writer http.ResponseWriter, request *http.Request) {
+	writer.WriteHeader(http.StatusOK) // call WriteHandler before Write
+	writer.Write([]byte("pong"))
+}
+
+type AdmissionReviewHandler struct {
+	KubeanClusterOpsSet clusterOperationClientSet.Interface
+}
+
+func (handler AdmissionReviewHandler) ServeHTTP(writer http.ResponseWriter, request *http.Request) {
+	klog.Warning("receive webhook request")
+	admissionReviewReq := admissionv1.AdmissionReview{}
+	if err := json.NewDecoder(request.Body).Decode(&admissionReviewReq); err != nil {
+		klog.ErrorS(err, "parse http body to AdmissionReview")
+		writer.WriteHeader(http.StatusBadRequest)
+		writer.Write([]byte(fmt.Sprint(err, "parse http body to AdmissionReview")))
+		return
+	}
+	if admissionReviewReq.Request == nil || len(admissionReviewReq.Request.Object.Raw) == 0 { // for validate create ,so Object.Raw is not empty
+		klog.Error("parse http body to AdmissionReview but no object")
+		writer.WriteHeader(http.StatusBadRequest)
+		writer.Write([]byte("parse http body to AdmissionReview but no object"))
+		return
+	}
+	clusterOperation := clusteroperationv1alpha1.ClusterOperation{}
+	if err := json.Unmarshal(admissionReviewReq.Request.Object.Raw, &clusterOperation); err != nil {
+		klog.ErrorS(err, "parse AdmissionReview.Object.Raw in ClusterOperation but failed")
+		writer.WriteHeader(http.StatusBadRequest)
+		writer.Write([]byte(fmt.Sprint(err, "parse AdmissionReview.Object.Raw in ClusterOperation but failed")))
+		return
+	}
+	if clusterOperation.Spec.Cluster == "" {
+		klog.Error("parse AdmissionReview.Object.Raw in ClusterOperation but spec.Cluster is empty")
+		writer.WriteHeader(http.StatusBadRequest)
+		writer.Write([]byte("parse AdmissionReview.Object.Raw in ClusterOperation but spec.Cluster is empty"))
+		return
+	}
+	klog.Warningf("receive webhook request for clusterOperation %s", clusterOperation.Name)
+	requirement, err := labels.NewRequirement(constants.KubeanClusterHasCompleted, selection.DoesNotExist, []string{}) // only when the operation has succeed or failed , then has this label
+	if err != nil {                                                                                                    // todo
+		klog.Error(err)
+		return
+	}
+	selector := labels.NewSelector()
+	selector.Add(*requirement)
+	opsList, err := handler.KubeanClusterOpsSet.KubeanV1alpha1().ClusterOperations().List(context.Background(), metav1.ListOptions{LabelSelector: selector.String()})
+	if err != nil {
+		klog.ErrorS(err, "fetch ClusterOperations but failed")
+		writer.WriteHeader(http.StatusBadRequest)
+		writer.Write([]byte(fmt.Sprint(err, "fetch ClusterOperations but failed")))
+		return
+	}
+	admissionReviewResponse := admissionv1.AdmissionReview{
+		TypeMeta: admissionReviewReq.TypeMeta,
+		Response: &admissionv1.AdmissionResponse{
+			UID: admissionReviewReq.Request.UID,
+		},
+	}
+	// allow default
+	admissionReviewResponse.Response.Allowed = true
+
+	for _, ops := range opsList.Items {
+		if ops.Status.Status == clusteroperationv1alpha1.FailedStatus || ops.Status.Status == clusteroperationv1alpha1.SucceededStatus {
+			continue // ignore
+		}
+		if ops.Name != clusterOperation.Name && ops.Spec.Cluster == clusterOperation.Spec.Cluster &&
+			(ops.Status.Status == "" || ops.Status.Status == clusteroperationv1alpha1.RunningStatus) { // belongs to the same cluster and still running
+			// not allow
+			admissionReviewResponse.Response.Allowed = false
+			admissionReviewResponse.Response.Result = &metav1.Status{
+				Message: fmt.Sprintf("Not Accept %s , because clusterOperation %s has not completed which belongs to Cluster %s", clusterOperation.Name, ops.Name, clusterOperation.Spec.Cluster),
+				Reason:  metav1.StatusReasonNotAcceptable,
+				Code:    http.StatusNotAcceptable,
+			}
+			break
+		}
+	}
+	httpResult, _ := json.Marshal(admissionReviewResponse)
+	writer.WriteHeader(http.StatusOK)
+	writer.Write(httpResult)
+}
+
+func PrepareWebHookHTTPSServer(KubeanClusterOpsSet clusterOperationClientSet.Interface) *http.Server {
+	mux := http.NewServeMux()
+	mux.Handle(WebHookPath, AdmissionReviewHandler{KubeanClusterOpsSet: KubeanClusterOpsSet})
+	mux.Handle("/ping", PingHandler{})
+	server := &http.Server{
+		Addr:    ":10443",
+		Handler: mux,
+	}
+	return server
+}
+
+func StartWebHookHTTPSServer(server *http.Server) {
+	certPath := filepath.Join(certsDir, certFile)
+	keyPath := filepath.Join(certsDir, certKey)
+	klog.Warning("start https server for webhook")
+	if err := server.ListenAndServeTLS(certPath, keyPath); err != nil {
+		klog.ErrorS(err, "start https server for webhook but failed")
+	}
+}
+
+func UpdateClusterOperationWebhook(clientSet kubernetes.Interface) error {
+	if ClusterOperationWebhook == "" {
+		return errors.New("ClusterOperationWebhook empty")
+	}
+	secret, err := clientSet.CoreV1().Secrets(util.GetCurrentNSOrDefault()).Get(context.Background(), CAStoreSecret, metav1.GetOptions{})
+	if err != nil {
+		klog.Error(err)
+		return err
+	}
+	caCertData, err := base64.StdEncoding.DecodeString(string(secret.Data["crt"]))
+	if err != nil {
+		klog.Error(err)
+		return err
+	}
+	m, err := clientSet.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get(context.Background(), ClusterOperationWebhook, metav1.GetOptions{})
+	if err == nil && len(m.Webhooks) > 0 && string(m.Webhooks[0].ClientConfig.CABundle) == string(caCertData) {
+		// need not update mutating-webhook
+		return nil
+	}
+	newWebHook := &admissionregistrationv1.ValidatingWebhookConfiguration{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: ClusterOperationWebhook,
+		},
+		Webhooks: []admissionregistrationv1.ValidatingWebhook{
+			{
+				Name: Organization + ".webhook",
+				ClientConfig: admissionregistrationv1.WebhookClientConfig{
+					CABundle: caCertData, // CA bundle created earlier
+					Service: &admissionregistrationv1.ServiceReference{
+						Name:      WebhookSVCName,
+						Namespace: WebhookSVCNamespace,
+						Path:      &WebHookPath,
+						Port: func() *int32 {
+							httpsPort := int32(443)
+							return &httpsPort
+						}(),
+					},
+				},
+				Rules: []admissionregistrationv1.RuleWithOperations{
+					{
+						Operations: []admissionregistrationv1.OperationType{
+							admissionregistrationv1.Create, // only for create , not for update or delete
+						},
+						Rule: admissionregistrationv1.Rule{
+							APIGroups:   []string{"kubean.io"},
+							APIVersions: []string{"v1alpha1"},
+							Resources:   []string{"clusteroperations"},
+						},
+					},
+				},
+				FailurePolicy: func() *admissionregistrationv1.FailurePolicyType {
+					policy := admissionregistrationv1.FailurePolicyType(FailurePolicy)
+					return &policy
+				}(),
+				TimeoutSeconds: func() *int32 {
+					timeout := int32(10)
+					return &timeout
+				}(),
+				AdmissionReviewVersions: []string{"v1", "v1beta1"},
+				SideEffects: func() *admissionregistrationv1.SideEffectClass {
+					se := admissionregistrationv1.SideEffectClassNone
+					return &se
+				}(),
+			},
+		},
+	}
+	if err != nil && apierrors.IsNotFound(err) { // create
+		if _, err := clientSet.AdmissionregistrationV1().ValidatingWebhookConfigurations().Create(context.Background(), newWebHook, metav1.CreateOptions{}); err != nil {
+			klog.Error(err)
+			return err
+		}
+		return nil
+	}
+	newWebHook.ResourceVersion = m.ResourceVersion // update
+	if _, err := clientSet.AdmissionregistrationV1().ValidatingWebhookConfigurations().Update(context.Background(), newWebHook, metav1.UpdateOptions{}); err != nil {
+		klog.Error(err)
+		return err
+	}
+	return nil
+}
diff --git a/vendor/github.com/kubean-io/kubean-api/apis/clusteroperation/v1alpha1/types.go b/vendor/github.com/kubean-io/kubean-api/apis/clusteroperation/v1alpha1/types.go
index 0dff80483..732d2e641 100644
--- a/vendor/github.com/kubean-io/kubean-api/apis/clusteroperation/v1alpha1/types.go
+++ b/vendor/github.com/kubean-io/kubean-api/apis/clusteroperation/v1alpha1/types.go
@@ -115,7 +115,6 @@ const (
 	RunningStatus   OpsStatus = "Running"
 	SucceededStatus OpsStatus = "Succeeded"
 	FailedStatus    OpsStatus = "Failed"
-	BlockedStatus   OpsStatus = "Blocked"
 )
 
 // Status contains information about the current status of a
diff --git a/vendor/github.com/kubean-io/kubean-api/constants/constants.go b/vendor/github.com/kubean-io/kubean-api/constants/constants.go
index c2211bbf6..87444c9e0 100644
--- a/vendor/github.com/kubean-io/kubean-api/constants/constants.go
+++ b/vendor/github.com/kubean-io/kubean-api/constants/constants.go
@@ -9,3 +9,5 @@ const Hosts_yml = "hosts.yml"
 const Group_vars_yml = "group_vars.yml"
 
 const SSH_privatekey = "ssh-privatekey"
+
+const KubeanClusterHasCompleted = "hasCompleted"


Test patch:
diff --git a/pkg/controllers/clusterops/controller_test.go b/pkg/controllers/clusterops/controller_test.go
index cdfe6c313..7bfca8935 100644
--- a/pkg/controllers/clusterops/controller_test.go
+++ b/pkg/controllers/clusterops/controller_test.go
@@ -661,178 +661,6 @@ func TestController_HookCustomAction(t *testing.T) {
 	}
 }
 
-func TestCurrentJobNeedBlock(t *testing.T) {
-	controller := Controller{}
-	clusterOps := &clusteroperationv1alpha1.ClusterOperation{}
-	clusterOps.Name = "the target one clusterOps"
-	clusterOps.CreationTimestamp = metav1.Time{Time: time.UnixMilli(1000)}
-	testCases := []struct {
-		name string
-		args func() bool
-		want bool
-	}{
-		{
-			name: "it occurs error",
-			args: func() bool {
-				_, err := controller.CurrentJobNeedBlock(clusterOps, func(clusterName string) ([]clusteroperationv1alpha1.ClusterOperation, error) {
-					return nil, fmt.Errorf("error")
-				})
-				return err == nil
-			},
-			want: false,
-		},
-		{
-			name: "it returns none jobs running before the target clusterOps",
-			args: func() bool {
-				needBlock, err := controller.CurrentJobNeedBlock(clusterOps, func(clusterName string) ([]clusteroperationv1alpha1.ClusterOperation, error) {
-					return nil, nil
-				})
-				return err == nil && !needBlock
-			},
-			want: true,
-		},
-		{
-			name: "it returns none jobs running before the target clusterOps again",
-			args: func() bool {
-				needBlock, err := controller.CurrentJobNeedBlock(clusterOps, func(clusterName string) ([]clusteroperationv1alpha1.ClusterOperation, error) {
-					ops1 := clusteroperationv1alpha1.ClusterOperation{}
-					ops1.Name = "ops1"
-					ops1.CreationTimestamp = metav1.Time{Time: time.UnixMilli(1000 + 10000)}
-					ops2 := clusteroperationv1alpha1.ClusterOperation{}
-					ops2.Name = "ops2"
-					ops2.CreationTimestamp = metav1.Time{Time: time.UnixMilli(1000 + 10000)}
-					return []clusteroperationv1alpha1.ClusterOperation{ops1, ops2}, nil
-				})
-				return err == nil && !needBlock
-			},
-			want: true,
-		},
-		{
-			name: "it returns some jobs running before the target clusterOps",
-			args: func() bool {
-				needBlock, err := controller.CurrentJobNeedBlock(clusterOps, func(clusterName string) ([]clusteroperationv1alpha1.ClusterOperation, error) {
-					ops1 := clusteroperationv1alpha1.ClusterOperation{}
-					ops1.Name = "ops1"
-					ops1.CreationTimestamp = metav1.Time{Time: time.UnixMilli(0)}
-					ops2 := clusteroperationv1alpha1.ClusterOperation{}
-					ops2.Name = "ops2"
-					ops2.CreationTimestamp = metav1.Time{Time: time.UnixMilli(1000 + 10000)}
-					return []clusteroperationv1alpha1.ClusterOperation{ops1, ops2}, nil
-				})
-				return err == nil && needBlock
-			},
-			want: true,
-		},
-		{
-			name: "it returns some jobs running before the target clusterOps with the same createTime",
-			args: func() bool {
-				needBlock, err := controller.CurrentJobNeedBlock(clusterOps, func(clusterName string) ([]clusteroperationv1alpha1.ClusterOperation, error) {
-					ops1 := clusteroperationv1alpha1.ClusterOperation{}
-					ops1.Name = "ops1"
-					ops1.CreationTimestamp = metav1.Time{Time: time.UnixMilli(1000)}
-					ops1.Status.Status = clusteroperationv1alpha1.RunningStatus
-					ops2 := clusteroperationv1alpha1.ClusterOperation{}
-					ops2.Name = "ops2"
-					ops2.CreationTimestamp = metav1.Time{Time: time.UnixMilli(1000 + 10000)}
-					return []clusteroperationv1alpha1.ClusterOperation{ops1, ops2}, nil
-				})
-				return err == nil && !needBlock
-			},
-			want: true,
-		},
-		{
-			name: "it returns some jobs completed before the target clusterOps",
-			args: func() bool {
-				needBlock, err := controller.CurrentJobNeedBlock(clusterOps, func(clusterName string) ([]clusteroperationv1alpha1.ClusterOperation, error) {
-					ops1 := clusteroperationv1alpha1.ClusterOperation{}
-					ops1.Name = "ops1"
-					ops1.CreationTimestamp = metav1.Time{Time: time.UnixMilli(0)}
-					ops1.Status.Status = clusteroperationv1alpha1.SucceededStatus
-					ops1.Status.JobRef = &apis.JobRef{Name: "ok", NameSpace: "ok"}
-					ops2 := clusteroperationv1alpha1.ClusterOperation{}
-					ops2.Name = "ops2"
-					ops2.CreationTimestamp = metav1.Time{Time: time.UnixMilli(1000 + 10000)}
-					return []clusteroperationv1alpha1.ClusterOperation{ops1, ops2}, nil
-				})
-				return err == nil && !needBlock
-			},
-			want: true,
-		},
-		{
-			name: "it returns some jobs failed before the target clusterOps",
-			args: func() bool {
-				needBlock, err := controller.CurrentJobNeedBlock(clusterOps, func(clusterName string) ([]clusteroperationv1alpha1.ClusterOperation, error) {
-					ops1 := clusteroperationv1alpha1.ClusterOperation{}
-					ops1.Name = "ops1"
-					ops1.CreationTimestamp = metav1.Time{Time: time.UnixMilli(0)}
-					ops1.Status.Status = clusteroperationv1alpha1.FailedStatus
-					ops1.Status.JobRef = &apis.JobRef{Name: "ok", NameSpace: "ok"}
-					ops2 := clusteroperationv1alpha1.ClusterOperation{}
-					ops2.Name = "ops2"
-					ops2.CreationTimestamp = metav1.Time{Time: time.UnixMilli(1000 + 10000)}
-					return []clusteroperationv1alpha1.ClusterOperation{ops1, ops2}, nil
-				})
-				return err == nil && !needBlock
-			},
-			want: true,
-		},
-		{
-			name: "it returns some jobs blocked before the target clusterOps",
-			args: func() bool {
-				needBlock, err := controller.CurrentJobNeedBlock(clusterOps, func(clusterName string) ([]clusteroperationv1alpha1.ClusterOperation, error) {
-					ops1 := clusteroperationv1alpha1.ClusterOperation{}
-					ops1.Name = "ops1"
-					ops1.CreationTimestamp = metav1.Time{Time: time.UnixMilli(0)}
-					ops1.Status.Status = clusteroperationv1alpha1.BlockedStatus
-					ops2 := clusteroperationv1alpha1.ClusterOperation{}
-					ops2.Name = "ops2"
-					ops2.CreationTimestamp = metav1.Time{Time: time.UnixMilli(1000 + 10000)}
-					return []clusteroperationv1alpha1.ClusterOperation{ops1, ops2}, nil
-				})
-				return err == nil && needBlock
-			},
-			want: true,
-		},
-		{
-			name: "it returns some jobs blocked at the same createTime",
-			args: func() bool {
-				needBlock, err := controller.CurrentJobNeedBlock(clusterOps, func(clusterName string) ([]clusteroperationv1alpha1.ClusterOperation, error) {
-					ops1 := clusteroperationv1alpha1.ClusterOperation{}
-					ops1.Name = "ops1"
-					ops1.CreationTimestamp = metav1.Time{Time: time.UnixMilli(1000)}
-					ops1.Status.Status = clusteroperationv1alpha1.BlockedStatus
-					ops2 := clusteroperationv1alpha1.ClusterOperation{}
-					ops2.Name = "ops2"
-					ops2.CreationTimestamp = metav1.Time{Time: time.UnixMilli(1000 + 10000)}
-					return []clusteroperationv1alpha1.ClusterOperation{ops1, ops2}, nil
-				})
-				return err == nil && !needBlock
-			},
-			want: true,
-		},
-		{
-			name: "ops succeeded",
-			args: func() bool {
-				clusterOps1 := *clusterOps
-				clusterOps1.Status.Status = clusteroperationv1alpha1.SucceededStatus
-				needBlock, err := controller.CurrentJobNeedBlock(&clusterOps1, func(clusterName string) ([]clusteroperationv1alpha1.ClusterOperation, error) {
-					return nil, nil
-				})
-				return err == nil && !needBlock
-			},
-			want: true,
-		},
-	}
-
-	for _, testCase := range testCases {
-		t.Run(testCase.name, func(t *testing.T) {
-			if testCase.args() != testCase.want {
-				t.Fatal()
-			}
-		})
-	}
-}
-
 func TestIsValidImageName(t *testing.T) {
 	testCases := []struct {
 		name string
@@ -2351,67 +2179,6 @@ func Test_FetchJobStatus(t *testing.T) {
 	}
 }
 
-func Test_ListClusterOps(t *testing.T) {
-	controller := Controller{
-		Client:              newFakeClient(),
-		ClientSet:           clientsetfake.NewSimpleClientset(),
-		KubeanClusterSet:    clusterv1alpha1fake.NewSimpleClientset(),
-		KubeanClusterOpsSet: clusteroperationv1alpha1fake.NewSimpleClientset(),
-	}
-	tests := []struct {
-		name string
-		args func() bool
-		want bool
-	}{
-		{
-			name: "list and get nothing",
-			args: func() bool {
-				result, err := controller.ListClusterOps("kubeanCluster-123")
-				return len(result) == 0 && err == nil
-			},
-			want: true,
-		},
-		{
-			name: "list and get something",
-			args: func() bool {
-				clusterOps := &clusteroperationv1alpha1.ClusterOperation{
-					TypeMeta: metav1.TypeMeta{
-						Kind:       "ClusterOperation",
-						APIVersion: "kubean.io/v1alpha1",
-					},
-					ObjectMeta: metav1.ObjectMeta{
-						Name:   "my_kubean_ops_cluster",
-						Labels: map[string]string{constants.KubeanClusterLabelKey: "kubeanCluster-123"},
-					},
-				}
-				controller.KubeanClusterOpsSet.KubeanV1alpha1().ClusterOperations().Create(context.Background(), clusterOps, metav1.CreateOptions{})
-				result, err := controller.ListClusterOps("kubeanCluster-123")
-				return len(result) > 0 && err == nil
-			},
-			want: true,
-		},
-		{
-			name: "get error",
-			args: func() bool {
-				fetchTestingFake(controller.KubeanClusterOpsSet.KubeanV1alpha1()).PrependReactor("list", "clusteroperations", func(action k8stesting.Action) (handled bool, ret runtime.Object, err error) {
-					return true, nil, fmt.Errorf("this is error")
-				})
-				_, err := controller.ListClusterOps("kubeanCluster-123")
-				removeReactorFromTestingTake(controller.KubeanClusterOpsSet.KubeanV1alpha1(), "list", "clusteroperations")
-				return err != nil && err.Error() == "this is error"
-			},
-			want: true,
-		},
-	}
-	for _, test := range tests {
-		t.Run(test.name, func(t *testing.T) {
-			if test.args() != test.want {
-				t.Fatal()
-			}
-		})
-	}
-}
-
 func Test_CopyConfigMap(t *testing.T) {
 	controller := Controller{
 		Client:              newFakeClient(),
diff --git a/pkg/util/certmanager_test.go b/pkg/util/certmanager_test.go
new file mode 100644
index 000000000..665235b36
--- /dev/null
+++ b/pkg/util/certmanager_test.go
@@ -0,0 +1,27 @@
+// Copyright 2023 Authors of kubean-io
+// SPDX-License-Identifier: Apache-2.0
+
+package util
+
+import (
+	"testing"
+	"time"
+)
+
+func TestNewCertManager(t *testing.T) {
+	cert, key, err := NewCertManager(
+		[]string{"Org1"},
+		time.Hour*100,
+		[]string{"a1.com", "a2.com"},
+		"a1.com",
+	).GenerateSelfSignedCerts()
+	if err != nil {
+		t.Fatal()
+	}
+	if cert.String() == "" {
+		t.Fatal()
+	}
+	if key.String() == "" {
+		t.Fatal()
+	}
+}
diff --git a/pkg/webhooks/clusterops/clusterops_webhook_test.go b/pkg/webhooks/clusterops/clusterops_webhook_test.go
new file mode 100644
index 000000000..71afb0aca
--- /dev/null
+++ b/pkg/webhooks/clusterops/clusterops_webhook_test.go
@@ -0,0 +1,217 @@
+// Copyright 2023 Authors of kubean-io
+// SPDX-License-Identifier: Apache-2.0
+
+package clusterops
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	admissionv1 "k8s.io/api/admission/v1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apiserver/pkg/endpoints/responsewriter"
+
+	clusteroperationv1alpha1 "github.com/kubean-io/kubean-api/apis/clusteroperation/v1alpha1"
+	clusteroperationv1alpha1fake "github.com/kubean-io/kubean-api/generated/clusteroperation/clientset/versioned/fake"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	clientsetfake "k8s.io/client-go/kubernetes/fake"
+
+	"github.com/kubean-io/kubean/pkg/util"
+)
+
+func TestCreateHTTPSCAFiles(t *testing.T) {
+	certsDir = strings.TrimRight(os.TempDir(), "/")
+	if err := CreateHTTPSCAFilesFromSecret(&corev1.Secret{}); err != nil {
+		t.Fatal(err)
+	}
+	if err := CreateHTTPSCAFilesFromSecret(&corev1.Secret{}); err != nil {
+		t.Fatal(err)
+	}
+	if !util.IsExist(filepath.Join(certsDir, certFile)) {
+		t.Fatal()
+	}
+	if !util.IsExist(filepath.Join(certsDir, certKey)) {
+		t.Fatal()
+	}
+}
+
+func TestPrepareWebHookHTTPSServer(t *testing.T) {
+	server := PrepareWebHookHTTPSServer(nil)
+	if server == nil {
+		t.Fatal()
+	}
+}
+
+func TestUpdateClusterOperationWebhook(t *testing.T) {
+	certsDir = strings.TrimRight(os.TempDir(), "/")
+	ClusterOperationWebhook = "my-webhook-abc"
+	CreateHTTPSCAFilesFromSecret(&corev1.Secret{})
+	fakeClientSet := clientsetfake.NewSimpleClientset()
+	EnsureCASecretExist(fakeClientSet)
+	if err := UpdateClusterOperationWebhook(fakeClientSet); err != nil {
+		t.Fatal()
+	}
+	os.Remove(filepath.Join(certsDir, certFile))
+	CreateHTTPSCAFilesFromSecret(&corev1.Secret{})
+	if err := UpdateClusterOperationWebhook(fakeClientSet); err != nil {
+		t.Fatal()
+	}
+	result, err := fakeClientSet.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get(context.Background(), ClusterOperationWebhook, metav1.GetOptions{})
+	if err != nil {
+		t.Fatal()
+	}
+	if len(result.Webhooks) == 0 {
+		t.Fatal()
+	}
+}
+
+type FakeResponseWriter struct {
+	code   int
+	result string
+	responsewriter.FakeResponseWriter
+}
+
+func (fw *FakeResponseWriter) WriteHeader(code int) {
+	fw.code = code
+}
+
+func (fw *FakeResponseWriter) Write(bs []byte) (int, error) {
+	fw.result = string(bs)
+	return len(bs), nil
+}
+
+func TestAdmissionReviewHandlerHttp(t *testing.T) {
+	clusterOperationClientSet := clusteroperationv1alpha1fake.NewSimpleClientset()
+	handler := AdmissionReviewHandler{clusterOperationClientSet}
+
+	tests := []struct {
+		name string
+		args func() bool
+		want bool
+	}{
+		{
+			name: "bad body parse",
+			args: func() bool {
+				response := &FakeResponseWriter{}
+				request, _ := http.NewRequest("", "", strings.NewReader("{"))
+				handler.ServeHTTP(response, request)
+				return response.code == http.StatusBadRequest
+			},
+			want: true,
+		},
+		{
+			name: "empty object",
+			args: func() bool {
+				response := &FakeResponseWriter{}
+				request, _ := http.NewRequest("", "", strings.NewReader("{}"))
+				handler.ServeHTTP(response, request)
+				return response.code == http.StatusBadRequest
+			},
+			want: true,
+		},
+		{
+			name: "empty spec.Cluster",
+			args: func() bool {
+				response := &FakeResponseWriter{}
+				clusterOps := &clusteroperationv1alpha1.ClusterOperation{
+					TypeMeta: metav1.TypeMeta{
+						Kind:       "ClusterOperation",
+						APIVersion: "kubean.io/v1alpha1",
+					},
+					ObjectMeta: metav1.ObjectMeta{
+						Name: "my_kubean_ops_cluster",
+					},
+				}
+				raw, _ := json.Marshal(clusterOps)
+				admissionReview := admissionv1.AdmissionReview{Request: &admissionv1.AdmissionRequest{Object: runtime.RawExtension{Raw: raw}}}
+				admissionReviewBytes, _ := json.Marshal(admissionReview)
+				request, _ := http.NewRequest("", "", bytes.NewReader(admissionReviewBytes))
+				handler.ServeHTTP(response, request)
+				return response.code == http.StatusBadRequest && strings.Contains(response.result, "spec.Cluster is empty")
+			},
+			want: true,
+		},
+		{
+			name: "allow",
+			args: func() bool {
+				response := &FakeResponseWriter{}
+				clusterOps := &clusteroperationv1alpha1.ClusterOperation{
+					TypeMeta: metav1.TypeMeta{
+						Kind:       "ClusterOperation",
+						APIVersion: "kubean.io/v1alpha1",
+					},
+					ObjectMeta: metav1.ObjectMeta{
+						Name: "my_kubean_ops_cluster",
+					},
+					Spec: clusteroperationv1alpha1.Spec{
+						Cluster: "abc_cluster",
+					},
+				}
+				raw, _ := json.Marshal(clusterOps)
+				admissionReview := admissionv1.AdmissionReview{Request: &admissionv1.AdmissionRequest{Object: runtime.RawExtension{Raw: raw}}}
+				admissionReviewBytes, _ := json.Marshal(admissionReview)
+				request, _ := http.NewRequest("", "", bytes.NewReader(admissionReviewBytes))
+				handler.ServeHTTP(response, request)
+				admissionReviewResponse := &admissionv1.AdmissionReview{}
+				json.Unmarshal([]byte(response.result), admissionReviewResponse)
+				return response.code == http.StatusOK && admissionReviewResponse.Response.Allowed
+			},
+			want: true,
+		},
+		{
+			name: "not allow",
+			args: func() bool {
+				response := &FakeResponseWriter{}
+				clusterOps1 := &clusteroperationv1alpha1.ClusterOperation{
+					TypeMeta: metav1.TypeMeta{
+						Kind:       "ClusterOperation",
+						APIVersion: "kubean.io/v1alpha1",
+					},
+					ObjectMeta: metav1.ObjectMeta{
+						Name: "my_kubean_ops_cluster_1",
+					},
+					Spec: clusteroperationv1alpha1.Spec{
+						Cluster: "abc_cluster",
+					},
+				}
+				clusterOperationClientSet.KubeanV1alpha1().ClusterOperations().Create(context.Background(), clusterOps1, metav1.CreateOptions{})
+				clusterOps2 := &clusteroperationv1alpha1.ClusterOperation{
+					TypeMeta: metav1.TypeMeta{
+						Kind:       "ClusterOperation",
+						APIVersion: "kubean.io/v1alpha1",
+					},
+					ObjectMeta: metav1.ObjectMeta{
+						Name: "my_kubean_ops_cluster_2",
+					},
+					Spec: clusteroperationv1alpha1.Spec{
+						Cluster: "abc_cluster",
+					},
+				}
+				raw, _ := json.Marshal(clusterOps2)
+				admissionReview := admissionv1.AdmissionReview{Request: &admissionv1.AdmissionRequest{Object: runtime.RawExtension{Raw: raw}}}
+				admissionReviewBytes, _ := json.Marshal(admissionReview)
+				request, _ := http.NewRequest("", "", bytes.NewReader(admissionReviewBytes))
+				handler.ServeHTTP(response, request)
+				admissionReviewResponse := &admissionv1.AdmissionReview{}
+				json.Unmarshal([]byte(response.result), admissionReviewResponse)
+				return response.code == http.StatusOK && !admissionReviewResponse.Response.Allowed && admissionReviewResponse.Response.Result.Reason == metav1.StatusReasonNotAcceptable
+			},
+			want: true,
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			if test.args() != test.want {
+				t.Fatal()
+			}
+		})
+	}
+}