Repository: rook/rook
Pull request #14936 (2024-10-30 07:48:29)

doc: clarify dataDirHostPath must be unique for each cluster
If there are multiple clusters, `dataDirHostPath` must be unique for each cluster. However, it's not documented yet. In addition, `ceph-teardown.md` was also updated because the original document assumes that `dataDirHostPath` is `/var/lib/rook` and the files for a cluster `rook-ceph` is under `/var/lib/rook/rook-ceph`.

**Checklist:**

- [x] **Commit Message Formatting**: Commit titles and messages follow guidelines in the [developer guide](https://rook.io/docs/rook/latest/Contributing/development-flow/#commit-structure).
- [x] Reviewed the developer guide on [Submitting a Pull Request](https://rook.io/docs/rook/latest/Contributing/development-flow/#submitting-a-pull-request)
- [ ] [Pending release notes](https://github.com/rook/rook/blob/master/PendingReleaseNotes.md) updated with breaking and/or notable changes for the next minor release.
- [x] Documentation has been updated, if necessary.
- [ ] Unit tests have been added, if necessary.
- [ ] Integration tests have been added, if necessary.


travisn: Commonly doc changes are only backported to the latest release, but could be backported to 1.14 if you think necessary.

Patch:
diff --git a/.github/workflows/multus.yaml b/.github/workflows/multus.yaml
index 0be92c9e8..230f36303 100644
--- a/.github/workflows/multus.yaml
+++ b/.github/workflows/multus.yaml
@@ -55,7 +55,6 @@ jobs:
         uses: ./.github/workflows/tmate_debug
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
-          debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
 
       - name: Setup multus
         run: ./tests/scripts/multus/setup-multus.sh
diff --git a/.github/workflows/tmate_debug/action.yml b/.github/workflows/tmate_debug/action.yml
index 93c2d5772..b6a32e5f5 100644
--- a/.github/workflows/tmate_debug/action.yml
+++ b/.github/workflows/tmate_debug/action.yml
@@ -4,15 +4,12 @@ inputs:
   use-tmate:
     description: "boolean for enabling TMATE"
     required: true
-  debug-ci:
-    description: "boolean for debug-ci label in PR"
-    requnred: false
 runs:
   using: "composite"
   steps:
     - name: consider debugging
       shell: bash --noprofile --norc -eo pipefail -x {0}
-      if: runner.debug || inputs.debug-ci == 'true'
+      if: runner.debug || contains(github.event.pull_request.labels.*.name, 'debug-ci')
       run: |
         # Enable tmate only in the Rook fork, where the USE_TMATE secret is set in the repo, or if the action is re-run
         if [ "$GITHUB_REPOSITORY_OWNER" = "rook" ] || [ -n "${{ inputs.use-tmate }}" ] || [ "$GITHUB_RUN_ATTEMPT" -gt 1 ]; then
diff --git a/Documentation/CRDs/Cluster/ceph-cluster-crd.md b/Documentation/CRDs/Cluster/ceph-cluster-crd.md
index fce660364..9268858b2 100755
--- a/Documentation/CRDs/Cluster/ceph-cluster-crd.md
+++ b/Documentation/CRDs/Cluster/ceph-cluster-crd.md
@@ -33,7 +33,7 @@ Settings can be specified at the global level to apply to the cluster as a whole
         Using the general `v19` tag is not recommended in production because it may lead to inconsistent versions of the image running across different nodes in the cluster.
     * `allowUnsupported`: If `true`, allow an unsupported major version of the Ceph release. Currently Reef and Squid are supported. Future versions such as Tentacle (v20) would require this to be set to `true`. Should be set to `false` in production.
     * `imagePullPolicy`: The image pull policy for the ceph daemon pods. Possible values are `Always`, `IfNotPresent`, and `Never`. The default is `IfNotPresent`.
-* `dataDirHostPath`: The path on the host ([hostPath](https://kubernetes.io/docs/concepts/storage/volumes/#hostpath)) where config and data should be stored for each of the services. If the directory does not exist, it will be created. Because this directory persists on the host, it will remain after pods are deleted. Following paths and any of their subpaths **must not be used**: `/etc/ceph`, `/rook` or `/var/log/ceph`.
+* `dataDirHostPath`: The path on the host ([hostPath](https://kubernetes.io/docs/concepts/storage/volumes/#hostpath)) where config and data should be stored for each of the services. If there are multiple clusters, the directory must be unique for each cluster. If the directory does not exist, it will be created. Because this directory persists on the host, it will remain after pods are deleted. Following paths and any of their subpaths **must not be used**: `/etc/ceph`, `/rook` or `/var/log/ceph`.
     * **WARNING**: For test scenarios, if you delete a cluster and start a new cluster on the same hosts, the path used by `dataDirHostPath` must be deleted. Otherwise, stale keys and other config will remain from the previous cluster and the new mons will fail to start.
 If this value is empty, each pod will get an ephemeral directory to store their config files that is tied to the lifetime of the pod running on that node. More details can be found in the Kubernetes [empty dir docs](https://kubernetes.io/docs/concepts/storage/volumes/#emptydir).
 * `skipUpgradeChecks`: if set to true Rook won't perform any upgrade checks on Ceph daemons during an upgrade. Use this at **YOUR OWN RISK**, only if you know what you're doing. To understand Rook's upgrade process of Ceph, read the [upgrade doc](../../Upgrade/rook-upgrade.md#ceph-version-upgrades).
diff --git a/Documentation/Helm-Charts/operator-chart.md b/Documentation/Helm-Charts/operator-chart.md
index 7fc7ec887..63ec4f542 100644
--- a/Documentation/Helm-Charts/operator-chart.md
+++ b/Documentation/Helm-Charts/operator-chart.md
@@ -52,6 +52,7 @@ The following table lists the configurable parameters of the rook-operator chart
 | `cephCommandsTimeoutSeconds` | The timeout for ceph commands in seconds | `"15"` |
 | `containerSecurityContext` | Set the container security context for the operator | `{"capabilities":{"drop":["ALL"]},"runAsGroup":2016,"runAsNonRoot":true,"runAsUser":2016}` |
 | `crds.enabled` | Whether the helm chart should create and update the CRDs. If false, the CRDs must be managed independently with deploy/examples/crds.yaml. **WARNING** Only set during first deployment. If later disabled the cluster may be DESTROYED. If the CRDs are deleted in this case, see [the disaster recovery guide](https://rook.io/docs/rook/latest/Troubleshooting/disaster-recovery/#restoring-crds-after-deletion) to restore them. | `true` |
+| `csi.allowUnsupportedVersion` | Allow starting an unsupported ceph-csi image | `false` |
 | `csi.attacher.repository` | Kubernetes CSI Attacher image repository | `"registry.k8s.io/sig-storage/csi-attacher"` |
 | `csi.attacher.tag` | Attacher image tag | `"v4.6.1"` |
 | `csi.cephFSAttachRequired` | Whether to skip any attach operation altogether for CephFS PVCs. See more details [here](https://kubernetes-csi.github.io/docs/skip-attach.html#skip-attach-with-csi-driver-object). If cephFSAttachRequired is set to false it skips the volume attachments and makes the creation of pods using the CephFS PVC fast. **WARNING** It's highly discouraged to use this for CephFS RWO volumes. Refer to this [issue](https://github.com/kubernetes/kubernetes/issues/103305) for more details. | `true` |
diff --git a/Documentation/Storage-Configuration/ceph-teardown.md b/Documentation/Storage-Configuration/ceph-teardown.md
index 6196854ed..702417841 100644
--- a/Documentation/Storage-Configuration/ceph-teardown.md
+++ b/Documentation/Storage-Configuration/ceph-teardown.md
@@ -12,7 +12,7 @@ Rook provides the following clean up options:
 To tear down the cluster, the following resources need to be cleaned up:
 
 * The resources created under Rook's namespace (default `rook-ceph`) such as the Rook operator created by `operator.yaml` and the cluster CR `cluster.yaml`.
-* `/var/lib/rook/rook-ceph`: Path on each host in the cluster where configuration is stored by the ceph mons and osds
+* All files under `dataDirHostPath` (default `/var/lib/rook`): Path on each host in the cluster where configuration is stored by ceph daemons.
 * Devices used by the OSDs
 
 If the default namespaces or paths such as `dataDirHostPath` are changed in the example yaml files, these namespaces and paths will need to be changed throughout these instructions.
@@ -68,7 +68,7 @@ kubectl delete storageclass csi-cephfs
 
     These jobs will perform the following operations:
 
-    * Delete the namespace directory under `dataDirHostPath`, for example `/var/lib/rook/rook-ceph`, on all the nodes
+    * Delete the all files under `dataDirHostPath` on all the nodes
     * Wipe the data on the drives on all the nodes where OSDs were running in this cluster
 
 !!! note
@@ -92,7 +92,7 @@ kubectl delete -f crds.yaml
 
 If the `cleanupPolicy` was not added to the CephCluster CR before deleting the cluster, these manual steps are required to tear down the cluster.
 
-Connect to each machine and delete the namespace directory under `dataDirHostPath`, for example `/var/lib/rook/rook-ceph`.
+Connect to each machine and delete all files under `dataDirHostPath`.
 
 #### Zapping Devices
 
diff --git a/Makefile b/Makefile
index 413f33f61..6fef6948b 100644
--- a/Makefile
+++ b/Makefile
@@ -161,7 +161,6 @@ fmt: ## Check formatting of go sources.
 	@$(MAKE) go.init
 	@$(MAKE) go.fmt
 
-gen.codegen: codegen
 codegen: ${CODE_GENERATOR} ## Run code generators.
 	@build/codegen/codegen.sh
 
@@ -179,21 +178,19 @@ distclean: clean ## Remove all files that are created by building or configuring
 prune: ## Prune cached artifacts.
 	@$(MAKE) -C images prune
 
-gen.crds: crds
+docs: helm-docs
+
 crds: $(CONTROLLER_GEN) $(YQ)
 	@echo Updating CRD manifests
 	@build/crds/build-crds.sh $(CONTROLLER_GEN) $(YQ)
 	@GOBIN=$(GOBIN) build/crds/generate-crd-docs.sh
 
-gen.rbac: gen-rbac
 gen-rbac: $(HELM) $(YQ) ## Generate RBAC from Helm charts
 	@# output only stdout to the file; stderr for debugging should keep going to stderr
 	HELM=$(HELM) ./build/rbac/gen-common.sh
 	HELM=$(HELM) ./build/rbac/gen-nfs-rbac.sh
 	HELM=$(HELM) ./build/rbac/gen-psp.sh
 
-gen.docs: docs
-docs: helm-docs
 helm-docs: $(HELM_DOCS) ## Use helm-docs to generate documentation from helm charts
 	$(HELM_DOCS) -c deploy/charts/rook-ceph \
 		-o ../../../Documentation/Helm-Charts/operator-chart.md \
@@ -221,15 +218,11 @@ docs-preview: ## Preview the documentation through mkdocs
 docs-build:  ## Build the documentation to the `site/` directory
 	mkdocs build --strict
 
-gen.crd-docs: generate-docs-crds
 generate-docs-crds: ## Build the documentation for CRD
 	@GOBIN=$(GOBIN) build/crds/generate-crd-docs.sh
 
-generate: gen.codegen gen.crds gen.rbac gen.docs gen.crd-docs ## Update all generated files (code, manifests, charts, and docs).
-
-
 .PHONY: all build.common
-.PHONY: build build.all install test check vet fmt codegen gen.codegen gen.rbac gen.crds gen.crd-docs gen.docs generate mod.check clean distclean prune
+.PHONY: build build.all install test check vet fmt codegen mod.check clean distclean prune
 
 # ====================================================================================
 # Help
diff --git a/deploy/charts/rook-ceph-cluster/values.yaml b/deploy/charts/rook-ceph-cluster/values.yaml
index 7de9fdebb..47297069d 100644
--- a/deploy/charts/rook-ceph-cluster/values.yaml
+++ b/deploy/charts/rook-ceph-cluster/values.yaml
@@ -100,7 +100,7 @@ cephClusterSpec:
     # Do not set to true in production.
     allowUnsupported: false
 
-  # The path on the host where configuration files will be persisted. Must be specified.
+  # The path on the host where configuration files will be persisted. Must be specified. If there are multiple clusters, the directory must be unique for each cluster.
   # Important: if you reinstall the cluster, make sure you delete this directory from each host or else the mons will fail to start on the new cluster.
   # In Minikube, the '/data' directory is configured to persist across reboots. Use "/data/rook" in Minikube environment.
   dataDirHostPath: /var/lib/rook
diff --git a/deploy/charts/rook-ceph/templates/configmap.yaml b/deploy/charts/rook-ceph/templates/configmap.yaml
index 1acc2c5e8..98b99741d 100644
--- a/deploy/charts/rook-ceph/templates/configmap.yaml
+++ b/deploy/charts/rook-ceph/templates/configmap.yaml
@@ -169,6 +169,9 @@ data:
 {{- if .Values.csi.nfsProvisionerNodeAffinity }}
   CSI_NFS_PROVISIONER_NODE_AFFINITY: {{ .Values.csi.nfsProvisionerNodeAffinity }}
 {{- end }}
+{{- if .Values.csi.allowUnsupportedVersion }}
+  ROOK_CSI_ALLOW_UNSUPPORTED_VERSION: {{ .Values.csi.allowUnsupportedVersion | quote }}
+{{- end }}
 {{- if .Values.csi.pluginTolerations }}
   CSI_PLUGIN_TOLERATIONS: {{ toYaml .Values.csi.pluginTolerations | quote }}
 {{- end }}
diff --git a/deploy/charts/rook-ceph/values.yaml b/deploy/charts/rook-ceph/values.yaml
index 8b5ceb82f..5b48fc868 100644
--- a/deploy/charts/rook-ceph/values.yaml
+++ b/deploy/charts/rook-ceph/values.yaml
@@ -179,6 +179,9 @@ csi:
   # -- Set GRPC timeout for csi containers (in seconds). It should be >= 120. If this value is not set or is invalid, it defaults to 150
   grpcTimeoutInSeconds: 150
 
+  # -- Allow starting an unsupported ceph-csi image
+  allowUnsupportedVersion: false
+
   # -- Burst to use while communicating with the kubernetes apiserver.
   kubeApiBurst:
 
diff --git a/deploy/examples/cluster-external-management.yaml b/deploy/examples/cluster-external-management.yaml
index 156e968df..f1ef32560 100644
--- a/deploy/examples/cluster-external-management.yaml
+++ b/deploy/examples/cluster-external-management.yaml
@@ -16,6 +16,7 @@ metadata:
 spec:
   external:
     enable: true
+  # If there are multiple clusters, the directory must be unique for each cluster.
   dataDirHostPath: /var/lib/rook
   # providing an image is required, if you want to create other CRs (rgw, mds, nfs)
   cephVersion:
diff --git a/deploy/examples/cluster-on-local-pvc.yaml b/deploy/examples/cluster-on-local-pvc.yaml
index 0d92793f4..c93b52af8 100644
--- a/deploy/examples/cluster-on-local-pvc.yaml
+++ b/deploy/examples/cluster-on-local-pvc.yaml
@@ -162,6 +162,7 @@ metadata:
   name: rook-ceph
   namespace: rook-ceph # namespace:cluster
 spec:
+  # If there are multiple clusters, the directory must be unique for each cluster.
   dataDirHostPath: /var/lib/rook
   mon:
     count: 3
diff --git a/deploy/examples/cluster-on-pvc-minikube.yaml b/deploy/examples/cluster-on-pvc-minikube.yaml
index 3900fc3ef..00f56a726 100644
--- a/deploy/examples/cluster-on-pvc-minikube.yaml
+++ b/deploy/examples/cluster-on-pvc-minikube.yaml
@@ -128,6 +128,7 @@ metadata:
   name: my-cluster
   namespace: rook-ceph # namespace:cluster
 spec:
+  # If there are multiple clusters, the directory must be unique for each cluster.
   dataDirHostPath: /var/lib/rook
   mon:
     count: 1
diff --git a/deploy/examples/cluster-on-pvc.yaml b/deploy/examples/cluster-on-pvc.yaml
index 01017800c..398de65cf 100644
--- a/deploy/examples/cluster-on-pvc.yaml
+++ b/deploy/examples/cluster-on-pvc.yaml
@@ -12,6 +12,7 @@ metadata:
   name: rook-ceph
   namespace: rook-ceph # namespace:cluster
 spec:
+  # If there are multiple clusters, the directory must be unique for each cluster.
   dataDirHostPath: /var/lib/rook
   mon:
     # Set the number of mons to be started. Generally recommended to be 3.
diff --git a/deploy/examples/cluster-stretched-aws.yaml b/deploy/examples/cluster-stretched-aws.yaml
index c286e7dcc..48796dfe2 100644
--- a/deploy/examples/cluster-stretched-aws.yaml
+++ b/deploy/examples/cluster-stretched-aws.yaml
@@ -14,6 +14,7 @@ metadata:
   name: rook-ceph
   namespace: rook-ceph # namespace:cluster
 spec:
+  # If there are multiple clusters, the directory must be unique for each cluster.
   dataDirHostPath: /var/lib/rook
   mon:
     # Five mons must be created for stretch mode
diff --git a/deploy/examples/cluster-stretched.yaml b/deploy/examples/cluster-stretched.yaml
index 7f582754a..5d595f3ca 100644
--- a/deploy/examples/cluster-stretched.yaml
+++ b/deploy/examples/cluster-stretched.yaml
@@ -14,6 +14,7 @@ metadata:
   name: rook-ceph
   namespace: rook-ceph # namespace:cluster
 spec:
+  # If there are multiple clusters, the directory must be unique for each cluster.
   dataDirHostPath: /var/lib/rook
   mon:
     # Five mons must be created for stretch mode
diff --git a/deploy/examples/cluster.yaml b/deploy/examples/cluster.yaml
index 345897fb5..3263448ea 100644
--- a/deploy/examples/cluster.yaml
+++ b/deploy/examples/cluster.yaml
@@ -26,7 +26,7 @@ spec:
     # Future versions such as Tentacle (v20) would require this to be set to `true`.
     # Do not set to true in production.
     allowUnsupported: false
-  # The path on the host where configuration files will be persisted. Must be specified.
+  # The path on the host where configuration files will be persisted. Must be specified. If there are multiple clusters, the directory must be unique for each cluster.
   # Important: if you reinstall the cluster, make sure you delete this directory from each host or else the mons will fail to start on the new cluster.
   # In Minikube, the '/data' directory is configured to persist across reboots. Use "/data/rook" in Minikube environment.
   dataDirHostPath: /var/lib/rook
diff --git a/deploy/examples/operator-openshift.yaml b/deploy/examples/operator-openshift.yaml
index 1186fd184..239b75b16 100644
--- a/deploy/examples/operator-openshift.yaml
+++ b/deploy/examples/operator-openshift.yaml
@@ -183,6 +183,9 @@ data:
   # supported values are documented at https://kubernetes-csi.github.io/docs/support-fsgroup.html
   CSI_NFS_FSGROUPPOLICY: "File"
 
+  # (Optional) Allow starting unsupported ceph-csi image
+  ROOK_CSI_ALLOW_UNSUPPORTED_VERSION: "false"
+
   # (Optional) control the host mount of /etc/selinux for csi plugin pods.
   CSI_PLUGIN_ENABLE_SELINUX_HOST_MOUNT: "false"
 
diff --git a/deploy/examples/operator.yaml b/deploy/examples/operator.yaml
index b11549be5..1a90a5193 100644
--- a/deploy/examples/operator.yaml
+++ b/deploy/examples/operator.yaml
@@ -113,6 +113,9 @@ data:
   # supported values are documented at https://kubernetes-csi.github.io/docs/support-fsgroup.html
   CSI_NFS_FSGROUPPOLICY: "File"
 
+  # (Optional) Allow starting unsupported ceph-csi image
+  ROOK_CSI_ALLOW_UNSUPPORTED_VERSION: "false"
+
   # (Optional) control the host mount of /etc/selinux for csi plugin pods.
   CSI_PLUGIN_ENABLE_SELINUX_HOST_MOUNT: "false"
 
diff --git a/pkg/operator/ceph/csi/controller.go b/pkg/operator/ceph/csi/controller.go
index b9c6c5034..e2fc9e39f 100644
--- a/pkg/operator/ceph/csi/controller.go
+++ b/pkg/operator/ceph/csi/controller.go
@@ -43,6 +43,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/version"
 )
 
 const (
@@ -177,6 +178,11 @@ func (r *ReconcileCSI) reconcile(request reconcile.Request) (reconcile.Result, e
 		r.opConfig.Parameters = opConfig.Data
 	}
 
+	serverVersion, err := r.context.Clientset.Discovery().ServerVersion()
+	if err != nil {
+		return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to get server version")
+	}
+
 	enableCSIOperator, err = strconv.ParseBool(k8sutil.GetValue(r.opConfig.Parameters, "ROOK_USE_CSI_OPERATOR", "false"))
 	if err != nil {
 		return reconcileResult, errors.Wrap(err, "unable to parse value for 'ROOK_USE_CSI_OPERATOR'")
@@ -197,7 +203,7 @@ func (r *ReconcileCSI) reconcile(request reconcile.Request) (reconcile.Result, e
 		if kerrors.IsNotFound(err) {
 			logger.Debug("no ceph cluster found not deploying ceph csi driver")
 			EnableRBD, EnableCephFS, EnableNFS = false, false, false
-			err = r.stopDrivers()
+			err = r.stopDrivers(serverVersion)
 			if err != nil {
 				return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to stop Drivers")
 			}
@@ -212,7 +218,7 @@ func (r *ReconcileCSI) reconcile(request reconcile.Request) (reconcile.Result, e
 	if len(cephClusters.Items) == 0 {
 		logger.Debug("no ceph cluster found not deploying ceph csi driver")
 		EnableRBD, EnableCephFS, EnableNFS = false, false, false
-		err = r.stopDrivers()
+		err = r.stopDrivers(serverVersion)
 		if err != nil {
 			return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to stop Drivers")
 		}
@@ -274,11 +280,11 @@ func (r *ReconcileCSI) reconcile(request reconcile.Request) (reconcile.Result, e
 		// disable Rook-managed CSI drivers if CSI operator is enabled
 		if EnableCSIOperator() {
 			logger.Info("disabling csi-driver since EnableCSIOperator is true")
-			err := r.stopDrivers()
+			err := r.stopDrivers(serverVersion)
 			if err != nil {
 				return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to stop csi Drivers")
 			}
-			err = r.reconcileOperatorConfig(cluster, clusterInfo)
+			err = r.reconcileOperatorConfig(cluster, clusterInfo, serverVersion)
 			if err != nil {
 				return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to reconcile csi-op config CR")
 			}
@@ -287,7 +293,7 @@ func (r *ReconcileCSI) reconcile(request reconcile.Request) (reconcile.Result, e
 	}
 
 	if !disableCSI && !EnableCSIOperator() {
-		err = r.validateAndConfigureDrivers(ownerInfo)
+		err = r.validateAndConfigureDrivers(serverVersion, ownerInfo)
 		if err != nil {
 			return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to configure ceph csi")
 		}
@@ -296,8 +302,8 @@ func (r *ReconcileCSI) reconcile(request reconcile.Request) (reconcile.Result, e
 	return reconcileResult, nil
 }
 
-func (r *ReconcileCSI) reconcileOperatorConfig(cluster cephv1.CephCluster, clusterInfo *cephclient.ClusterInfo) error {
-	if err := r.setParams(); err != nil {
+func (r *ReconcileCSI) reconcileOperatorConfig(cluster cephv1.CephCluster, clusterInfo *cephclient.ClusterInfo, serverVersion *version.Info) error {
+	if err := r.setParams(serverVersion); err != nil {
 		return errors.Wrapf(err, "failed to configure CSI parameters")
 	}
 
diff --git a/pkg/operator/ceph/csi/csi.go b/pkg/operator/ceph/csi/csi.go
index b1ab03618..4313eb0ef 100644
--- a/pkg/operator/ceph/csi/csi.go
+++ b/pkg/operator/ceph/csi/csi.go
@@ -28,14 +28,16 @@ import (
 	"github.com/pkg/errors"
 	kerrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/version"
 )
 
-func (r *ReconcileCSI) validateAndConfigureDrivers(ownerInfo *k8sutil.OwnerInfo) error {
+func (r *ReconcileCSI) validateAndConfigureDrivers(serverVersion *version.Info, ownerInfo *k8sutil.OwnerInfo) error {
 	var (
+		v   *CephCSIVersion
 		err error
 	)
 
-	if err = r.setParams(); err != nil {
+	if err = r.setParams(serverVersion); err != nil {
 		return errors.Wrapf(err, "failed to configure CSI parameters")
 	}
 
@@ -43,17 +45,25 @@ func (r *ReconcileCSI) validateAndConfigureDrivers(ownerInfo *k8sutil.OwnerInfo)
 		return errors.Wrapf(err, "failed to validate CSI parameters")
 	}
 
+	if !AllowUnsupported && CSIEnabled() {
+		if v, err = r.validateCSIVersion(ownerInfo); err != nil {
+			return errors.Wrapf(err, "invalid csi version")
+		}
+	} else {
+		logger.Info("skipping csi version check, since unsupported versions are allowed or csi is disabled")
+	}
+
 	if CSIEnabled() {
-		if err = r.startDrivers(ownerInfo); err != nil {
+		if err = r.startDrivers(serverVersion, ownerInfo, v); err != nil {
 			return errors.Wrap(err, "failed to start ceph csi drivers")
 		}
 	}
 
 	// Check whether RBD or CephFS needs to be disabled
-	return r.stopDrivers()
+	return r.stopDrivers(serverVersion)
 }
 
-func (r *ReconcileCSI) setParams() error {
+func (r *ReconcileCSI) setParams(ver *version.Info) error {
 	var err error
 
 	if EnableRBD, err = strconv.ParseBool(k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_ENABLE_RBD", "true")); err != nil {
@@ -68,6 +78,10 @@ func (r *ReconcileCSI) setParams() error {
 		return errors.Wrap(err, "unable to parse value for 'ROOK_CSI_ENABLE_NFS'")
 	}
 
+	if AllowUnsupported, err = strconv.ParseBool(k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_ALLOW_UNSUPPORTED_VERSION", "false")); err != nil {
+		return errors.Wrap(err, "unable to parse value for 'ROOK_CSI_ALLOW_UNSUPPORTED_VERSION'")
+	}
+
 	if CSIParam.EnableCSIHostNetwork, err = strconv.ParseBool(k8sutil.GetValue(r.opConfig.Parameters, "CSI_ENABLE_HOST_NETWORK", "true")); err != nil {
 		return errors.Wrap(err, "failed to parse value for 'CSI_ENABLE_HOST_NETWORK'")
 	}
@@ -125,7 +139,10 @@ func (r *ReconcileCSI) setParams() error {
 		CSIParam.EnableOMAPGenerator = true
 	}
 
-	CSIParam.EnableCSIDriverSeLinuxMount = true
+	// CSIDriver SeLinuxMount option is only available from kubernetes version 1.25.
+	if ver.Major == kubeMinMajor && ver.Minor >= kubeMinVerForCSIDriverSeLinuxMount {
+		CSIParam.EnableCSIDriverSeLinuxMount = true
+	}
 
 	CSIParam.EnableRBDSnapshotter = true
 	if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_ENABLE_RBD_SNAPSHOTTER", "true"), "false") {
@@ -196,6 +213,8 @@ func (r *ReconcileCSI) setParams() error {
 		CSIParam.EnablePluginSelinuxHostMount = true
 	}
 
+	logger.Infof("Kubernetes version is %s.%s", ver.Major, ver.Minor)
+
 	logLevel := k8sutil.GetValue(r.opConfig.Parameters, "CSI_LOG_LEVEL", "")
 	CSIParam.LogLevel = defaultLogLevel
 	if logLevel != "" {
diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go
index f3e14a681..59af7f601 100644
--- a/pkg/operator/ceph/csi/spec.go
+++ b/pkg/operator/ceph/csi/spec.go
@@ -24,8 +24,11 @@ import (
 	"strings"
 	"time"
 
+	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
+	"github.com/rook/rook/pkg/operator/ceph/cluster/telemetry"
 	opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
 	"github.com/rook/rook/pkg/operator/k8sutil"
+	"github.com/rook/rook/pkg/operator/k8sutil/cmdreporter"
 
 	"github.com/pkg/errors"
 	apps "k8s.io/api/apps/v1"
@@ -33,6 +36,7 @@ import (
 	k8scsi "k8s.io/api/storage/v1beta1"
 	kerrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/version"
 	"k8s.io/client-go/kubernetes"
 )
 
@@ -113,6 +117,7 @@ var (
 	EnableCephFS              = false
 	EnableNFS                 = false
 	enableCSIOperator         = false
+	AllowUnsupported          = false
 	CustomCSICephConfigExists = false
 
 	// driver names
@@ -173,6 +178,9 @@ var (
 )
 
 const (
+	kubeMinMajor                       = "1"
+	kubeMinVerForCSIDriverSeLinuxMount = "25"
+
 	// common tolerations and node affinity
 	provisionerTolerationsEnv  = "CSI_PROVISIONER_TOLERATIONS"
 	provisionerNodeAffinityEnv = "CSI_PROVISIONER_NODE_AFFINITY"
@@ -226,6 +234,7 @@ const (
 	DefaultRBDLivenessMerticsPort    uint16 = 9080
 	DefaultCSIAddonsPort             uint16 = 9070
 
+	detectCSIVersionName = "rook-ceph-csi-detect-version"
 	// default log level for csi containers
 	defaultLogLevel        uint8 = 0
 	defaultSidecarLogLevel uint8 = 0
@@ -295,7 +304,7 @@ func validateCSIParam() error {
 	return nil
 }
 
-func (r *ReconcileCSI) startDrivers(ownerInfo *k8sutil.OwnerInfo) error {
+func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerInfo, v *CephCSIVersion) error {
 	var (
 		err                                                                             error
 		rbdPlugin, cephfsPlugin, nfsPlugin                                              *apps.DaemonSet
@@ -650,14 +659,14 @@ func (r *ReconcileCSI) startDrivers(ownerInfo *k8sutil.OwnerInfo) error {
 	return nil
 }
 
-func (r *ReconcileCSI) stopDrivers() error {
+func (r *ReconcileCSI) stopDrivers(ver *version.Info) error {
 	RBDDriverName = fmt.Sprintf("%s.rbd.csi.ceph.com", r.opConfig.OperatorNamespace)
 	CephFSDriverName = fmt.Sprintf("%s.cephfs.csi.ceph.com", r.opConfig.OperatorNamespace)
 	NFSDriverName = fmt.Sprintf("%s.nfs.csi.ceph.com", r.opConfig.OperatorNamespace)
 
 	if !EnableRBD || EnableCSIOperator() {
 		logger.Debugf("either EnableRBD if `false` or EnableCSIOperator is `true`, `EnableRBD is %t` and `EnableCSIOperator is %t", EnableRBD, EnableCSIOperator())
-		err := r.deleteCSIDriverResources(CsiRBDPlugin, csiRBDProvisioner, "csi-rbdplugin-metrics", RBDDriverName)
+		err := r.deleteCSIDriverResources(ver, CsiRBDPlugin, csiRBDProvisioner, "csi-rbdplugin-metrics", RBDDriverName)
 		if err != nil {
 			return errors.Wrap(err, "failed to remove CSI Ceph RBD driver")
 		}
@@ -666,7 +675,7 @@ func (r *ReconcileCSI) stopDrivers() error {
 
 	if !EnableCephFS || EnableCSIOperator() {
 		logger.Debugf("either EnableCephFS if `false` or EnableCSIOperator is `true`, `EnableCephFS is %t` and `EnableCSIOperator is %t", EnableRBD, EnableCSIOperator())
-		err := r.deleteCSIDriverResources(CsiCephFSPlugin, csiCephFSProvisioner, "csi-cephfsplugin-metrics", CephFSDriverName)
+		err := r.deleteCSIDriverResources(ver, CsiCephFSPlugin, csiCephFSProvisioner, "csi-cephfsplugin-metrics", CephFSDriverName)
 		if err != nil {
 			return errors.Wrap(err, "failed to remove CSI CephFS driver")
 		}
@@ -675,7 +684,7 @@ func (r *ReconcileCSI) stopDrivers() error {
 
 	if !EnableNFS || EnableCSIOperator() {
 		logger.Debugf("either EnableNFS if `false` or EnableCSIOperator is `true`, `EnableNFS is %t` and `EnableCSIOperator is %t", EnableRBD, EnableCSIOperator())
-		err := r.deleteCSIDriverResources(CsiNFSPlugin, csiNFSProvisioner, "csi-nfsplugin-metrics", NFSDriverName)
+		err := r.deleteCSIDriverResources(ver, CsiNFSPlugin, csiNFSProvisioner, "csi-nfsplugin-metrics", NFSDriverName)
 		if err != nil {
 			return errors.Wrap(err, "failed to remove CSI NFS driver")
 		}
@@ -685,7 +694,7 @@ func (r *ReconcileCSI) stopDrivers() error {
 	return nil
 }
 
-func (r *ReconcileCSI) deleteCSIDriverResources(daemonset, deployment, service, driverName string) error {
+func (r *ReconcileCSI) deleteCSIDriverResources(ver *version.Info, daemonset, deployment, service, driverName string) error {
 	csiDriverobj := v1CsiDriver{}
 	err := k8sutil.DeleteDaemonset(r.opManagerContext, r.context.Clientset, r.opConfig.OperatorNamespace, daemonset)
 	if err != nil {
@@ -729,6 +738,66 @@ func (r *ReconcileCSI) applyCephClusterNetworkConfig(ctx context.Context, object
 	return nil
 }
 
+// ValidateCSIVersion checks if the configured ceph-csi image is supported
+func (r *ReconcileCSI) validateCSIVersion(ownerInfo *k8sutil.OwnerInfo) (*CephCSIVersion, error) {
+	timeout := 15 * time.Minute
+
+	logger.Infof("detecting the ceph csi image version for image %q", CSIParam.CSIPluginImage)
+
+	versionReporter, err := cmdreporter.New(
+		r.context.Clientset,
+		ownerInfo,
+		detectCSIVersionName,
+		detectCSIVersionName,
+		r.opConfig.OperatorNamespace,
+		[]string{"cephcsi"},
+		[]string{"--version"},
+		r.opConfig.Image,
+		CSIParam.CSIPluginImage,
+		corev1.PullPolicy(CSIParam.ImagePullPolicy),
+	)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to set up ceph CSI version job")
+	}
+
+	job := versionReporter.Job()
+	job.Spec.Template.Spec.ServiceAccountName = r.opConfig.ServiceAccount
+
+	// Apply csi provisioner toleration and affinity for csi version check job
+	job.Spec.Template.Spec.Tolerations = getToleration(r.opConfig.Parameters, provisionerTolerationsEnv, []corev1.Toleration{})
+	job.Spec.Template.Spec.Affinity = &corev1.Affinity{
+		NodeAffinity: getNodeAffinity(r.opConfig.Parameters, provisionerNodeAffinityEnv, &corev1.NodeAffinity{}),
+	}
+	if r.firstCephCluster != nil {
+		cephv1.GetCmdReporterAnnotations(r.firstCephCluster.Annotations).ApplyToObjectMeta(&job.Spec.Template.ObjectMeta)
+		cephv1.GetCmdReporterLabels(r.firstCephCluster.Labels).ApplyToObjectMeta(&job.Spec.Template.ObjectMeta)
+	}
+
+	stdout, _, retcode, err := versionReporter.Run(r.opManagerContext, timeout)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to complete ceph CSI version job")
+	}
+
+	if retcode != 0 {
+		return nil, errors.Errorf("ceph CSI version job returned %d", retcode)
+	}
+
+	version, err := extractCephCSIVersion(stdout)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to extract ceph CSI version")
+	}
+	logger.Infof("Detected ceph CSI image version: %q", version)
+
+	// Set the CSI version on a package variable so that each cluster reporting the
+	// telemetry can report the same CSI version
+	telemetry.CSIVersion = version.String()
+
+	if !version.Supported() {
+		return nil, errors.Errorf("ceph CSI image needs to be at least version %q", minimum.String())
+	}
+	return version, nil
+}
+
 func validateCSIDriverNamePrefix(ctx context.Context, clientset kubernetes.Interface, namespace, driverNamePrefix string) error {
 	if EnableRBD {
 		rbdDriverNamePrefix, err := getCSIDriverNamePrefixFromDeployment(ctx, clientset, namespace, csiRBDProvisioner, "csi-rbdplugin")
diff --git a/pkg/operator/ceph/csi/version.go b/pkg/operator/ceph/csi/version.go
new file mode 100644
index 000000000..aa4d9bb26
--- /dev/null
+++ b/pkg/operator/ceph/csi/version.go
@@ -0,0 +1,114 @@
+/*
+Copyright 2020 The Rook Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package csi
+
+import (
+	"fmt"
+	"regexp"
+	"strconv"
+
+	"github.com/pkg/errors"
+)
+
+var (
+	//minimum supported version is 3.10.0
+	minimum = CephCSIVersion{3, 11, 0}
+	//supportedCSIVersions are versions that rook supports
+	releasev312 = CephCSIVersion{3, 12, 0}
+
+	supportedCSIVersions = []CephCSIVersion{
+		minimum,
+		releasev312,
+	}
+
+	// for parsing the output of `cephcsi`
+	versionCSIPattern = regexp.MustCompile(`v(\d+)\.(\d+)\.(\d+)`)
+)
+
+// CephCSIVersion represents the Ceph CSI version format
+type CephCSIVersion struct {
+	Major  int
+	Minor  int
+	Bugfix int
+}
+
+func (v *CephCSIVersion) String() string {
+	return fmt.Sprintf("v%d.%d.%d",
+		v.Major, v.Minor, v.Bugfix)
+}
+
+// Supported checks if the detected version is part of the known supported CSI versions
+func (v *CephCSIVersion) Supported() bool {
+	if !v.isAtLeast(&minimum) {
+		return false
+	}
+
+	// if AllowUnsupported is set also a csi-image greater than the supported ones are allowed
+	if AllowUnsupported {
+		return true
+	}
+	for _, sv := range supportedCSIVersions {
+		if v.Major == sv.Major {
+			if v.Minor == sv.Minor {
+				if v.Bugfix >= sv.Bugfix {
+					return true
+				}
+			}
+		}
+	}
+	return false
+}
+
+func (v *CephCSIVersion) isAtLeast(version *CephCSIVersion) bool {
+	if v.Major > version.Major {
+		return true
+	}
+	if v.Major == version.Major && v.Minor >= version.Minor {
+		if v.Minor > version.Minor {
+			return true
+		}
+		if v.Bugfix >= version.Bugfix {
+			return true
+		}
+	}
+	return false
+}
+
+// extractCephCSIVersion extracts the major, minor and extra digit of a Ceph CSI release
+func extractCephCSIVersion(src string) (*CephCSIVersion, error) {
+	m := versionCSIPattern.FindStringSubmatch(src)
+	if m == nil || len(m) < 3 {
+		return nil, errors.Errorf("failed to parse version from: %q", CSIParam.CSIPluginImage)
+	}
+
+	major, err := strconv.Atoi(m[1])
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to parse version major part: %q", m[0])
+	}
+
+	minor, err := strconv.Atoi(m[2])
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to parse version minor part: %q", m[1])
+	}
+
+	bugfix, err := strconv.Atoi(m[3])
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to parse version bugfix part: %q", m[2])
+	}
+
+	return &CephCSIVersion{major, minor, bugfix}, nil
+}


Test patch:
diff --git a/.github/workflows/canary-integration-test.yml b/.github/workflows/canary-integration-test.yml
index ad5e75125..3b493eeff 100644
--- a/.github/workflows/canary-integration-test.yml
+++ b/.github/workflows/canary-integration-test.yml
@@ -33,7 +33,6 @@ jobs:
         uses: ./.github/workflows/tmate_debug
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
-          debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
 
       - name: setup cluster resources
         uses: ./.github/workflows/canary-test-config
@@ -343,7 +342,6 @@ jobs:
         uses: ./.github/workflows/tmate_debug
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
-          debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
 
       - name: setup cluster resources
         uses: ./.github/workflows/canary-test-config
@@ -421,7 +419,6 @@ jobs:
         uses: ./.github/workflows/tmate_debug
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
-          debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
 
       - name: setup cluster resources
         uses: ./.github/workflows/canary-test-config
@@ -474,7 +471,6 @@ jobs:
         uses: ./.github/workflows/tmate_debug
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
-          debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
 
       - name: setup cluster resources
         uses: ./.github/workflows/canary-test-config
@@ -524,7 +520,6 @@ jobs:
         uses: ./.github/workflows/tmate_debug
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
-          debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
 
       - name: setup cluster resources
         uses: ./.github/workflows/canary-test-config
@@ -583,7 +578,6 @@ jobs:
         uses: ./.github/workflows/tmate_debug
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
-          debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
 
       - name: setup cluster resources
         uses: ./.github/workflows/canary-test-config
@@ -636,7 +630,6 @@ jobs:
         uses: ./.github/workflows/tmate_debug
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
-          debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
 
       - name: setup cluster resources
         uses: ./.github/workflows/canary-test-config
@@ -694,7 +687,6 @@ jobs:
         uses: ./.github/workflows/tmate_debug
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
-          debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
 
       - name: setup cluster resources
         uses: ./.github/workflows/canary-test-config
@@ -771,7 +763,6 @@ jobs:
         uses: ./.github/workflows/tmate_debug
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
-          debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
 
       - name: setup cluster resources
         uses: ./.github/workflows/canary-test-config
@@ -824,7 +815,6 @@ jobs:
         uses: ./.github/workflows/tmate_debug
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
-          debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
 
       - name: setup cluster resources
         uses: ./.github/workflows/canary-test-config
@@ -880,7 +870,6 @@ jobs:
         uses: ./.github/workflows/tmate_debug
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
-          debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
 
       - name: setup cluster resources
         uses: ./.github/workflows/canary-test-config
@@ -949,7 +938,6 @@ jobs:
         uses: ./.github/workflows/tmate_debug
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
-          debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
 
       - name: setup cluster resources
         uses: ./.github/workflows/canary-test-config
@@ -1004,7 +992,6 @@ jobs:
         uses: ./.github/workflows/tmate_debug
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
-          debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
 
       - name: setup cluster resources
         uses: ./.github/workflows/canary-test-config
@@ -1069,7 +1056,6 @@ jobs:
         uses: ./.github/workflows/tmate_debug
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
-          debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
 
       - name: setup cluster resources
         uses: ./.github/workflows/canary-test-config
@@ -1160,7 +1146,6 @@ jobs:
         uses: ./.github/workflows/tmate_debug
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
-          debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
 
       - name: setup cluster resources
         uses: ./.github/workflows/canary-test-config
@@ -1231,7 +1216,6 @@ jobs:
         uses: ./.github/workflows/tmate_debug
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
-          debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
 
       - name: setup cluster resources
         uses: ./.github/workflows/canary-test-config
@@ -1289,7 +1273,6 @@ jobs:
         uses: ./.github/workflows/tmate_debug
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
-          debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
 
       - name: setup cluster resources
         uses: ./.github/workflows/canary-test-config
@@ -1549,7 +1532,6 @@ jobs:
         uses: ./.github/workflows/tmate_debug
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
-          debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
 
       - name: setup cluster resources
         uses: ./.github/workflows/canary-test-config
@@ -1645,7 +1627,6 @@ jobs:
         uses: ./.github/workflows/tmate_debug
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
-          debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
 
       - name: run encryption KMS IBM Key Protect
         uses: ./.github/workflows/encryption-pvc-kms-ibm-kp
@@ -1676,7 +1657,6 @@ jobs:
         uses: ./.github/workflows/tmate_debug
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
-          debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
 
       - name: setup cluster resources
         uses: ./.github/workflows/canary-test-config
@@ -1758,7 +1738,6 @@ jobs:
         uses: ./.github/workflows/tmate_debug
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
-          debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
 
       - name: setup cluster resources
         uses: ./.github/workflows/canary-test-config
diff --git a/.github/workflows/integration-test-helm-suite.yaml b/.github/workflows/integration-test-helm-suite.yaml
index 8d30d681c..962f98f8d 100644
--- a/.github/workflows/integration-test-helm-suite.yaml
+++ b/.github/workflows/integration-test-helm-suite.yaml
@@ -39,7 +39,6 @@ jobs:
         uses: ./.github/workflows/tmate_debug
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
-          debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
 
       - name: setup latest cluster resources
         uses: ./.github/workflows/integration-test-config-latest-k8s
diff --git a/.github/workflows/integration-test-keystone-auth-suite.yaml b/.github/workflows/integration-test-keystone-auth-suite.yaml
index 5d4db9c53..24da4dd58 100644
--- a/.github/workflows/integration-test-keystone-auth-suite.yaml
+++ b/.github/workflows/integration-test-keystone-auth-suite.yaml
@@ -39,7 +39,6 @@ jobs:
         uses: ./.github/workflows/tmate_debug
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
-          debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
 
       - name: setup latest cluster resources
         uses: ./.github/workflows/integration-test-config-latest-k8s
diff --git a/.github/workflows/integration-test-mgr-suite.yaml b/.github/workflows/integration-test-mgr-suite.yaml
index 68a56d486..cf8623cba 100644
--- a/.github/workflows/integration-test-mgr-suite.yaml
+++ b/.github/workflows/integration-test-mgr-suite.yaml
@@ -38,7 +38,6 @@ jobs:
         uses: ./.github/workflows/tmate_debug
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
-          debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
 
       - name: setup cluster resources
         uses: ./.github/workflows/integration-test-config-latest-k8s
diff --git a/.github/workflows/integration-test-multi-cluster-suite.yaml b/.github/workflows/integration-test-multi-cluster-suite.yaml
index 05233cf08..71c73b018 100644
--- a/.github/workflows/integration-test-multi-cluster-suite.yaml
+++ b/.github/workflows/integration-test-multi-cluster-suite.yaml
@@ -39,7 +39,6 @@ jobs:
         uses: ./.github/workflows/tmate_debug
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
-          debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
 
       - name: setup cluster resources
         uses: ./.github/workflows/integration-test-config-latest-k8s
diff --git a/.github/workflows/integration-test-object-suite.yaml b/.github/workflows/integration-test-object-suite.yaml
index fa13bcd91..be07cdb0f 100644
--- a/.github/workflows/integration-test-object-suite.yaml
+++ b/.github/workflows/integration-test-object-suite.yaml
@@ -39,7 +39,6 @@ jobs:
         uses: ./.github/workflows/tmate_debug
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
-          debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
 
       - name: setup latest cluster resources
         uses: ./.github/workflows/integration-test-config-latest-k8s
diff --git a/.github/workflows/integration-test-smoke-suite.yaml b/.github/workflows/integration-test-smoke-suite.yaml
index 5712ed754..0ccf3fe1a 100644
--- a/.github/workflows/integration-test-smoke-suite.yaml
+++ b/.github/workflows/integration-test-smoke-suite.yaml
@@ -39,7 +39,6 @@ jobs:
         uses: ./.github/workflows/tmate_debug
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
-          debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
 
       - name: setup cluster resources
         uses: ./.github/workflows/integration-test-config-latest-k8s
diff --git a/.github/workflows/integration-test-upgrade-suite.yaml b/.github/workflows/integration-test-upgrade-suite.yaml
index c4c384e87..0c88af1a2 100644
--- a/.github/workflows/integration-test-upgrade-suite.yaml
+++ b/.github/workflows/integration-test-upgrade-suite.yaml
@@ -39,7 +39,6 @@ jobs:
         uses: ./.github/workflows/tmate_debug
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
-          debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
 
       - name: setup cluster resources
         uses: ./.github/workflows/integration-test-config-latest-k8s
@@ -84,7 +83,6 @@ jobs:
         uses: ./.github/workflows/tmate_debug
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
-          debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
 
       - name: setup cluster resources
         uses: ./.github/workflows/integration-test-config-latest-k8s
diff --git a/pkg/operator/ceph/csi/controller_test.go b/pkg/operator/ceph/csi/controller_test.go
index 2df439cef..b1998bab4 100644
--- a/pkg/operator/ceph/csi/controller_test.go
+++ b/pkg/operator/ceph/csi/controller_test.go
@@ -61,6 +61,7 @@ func TestCephCSIController(t *testing.T) {
 	t.Setenv(k8sutil.PodNameEnvVar, "rook-ceph-operator")
 	t.Setenv(k8sutil.PodNamespaceEnvVar, namespace)
 
+	t.Setenv("ROOK_CSI_ALLOW_UNSUPPORTED_VERSION", "true")
 	CSIParam = Param{
 		CSIPluginImage:   "image",
 		RegistrarImage:   "image",
diff --git a/pkg/operator/ceph/csi/version_test.go b/pkg/operator/ceph/csi/version_test.go
new file mode 100644
index 000000000..7bd518c6b
--- /dev/null
+++ b/pkg/operator/ceph/csi/version_test.go
@@ -0,0 +1,129 @@
+/*
+Copyright 2020 The Rook Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package csi
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+var (
+	testMinVersion   = CephCSIVersion{3, 11, 0}
+	testReleaseV390  = CephCSIVersion{3, 9, 0}
+	testReleaseV391  = CephCSIVersion{3, 9, 1}
+	testreleasev310  = CephCSIVersion{3, 10, 0}
+	testReleaseV3101 = CephCSIVersion{3, 10, 1}
+	testReleaseV3102 = CephCSIVersion{3, 10, 2}
+	testReleaseV3110 = CephCSIVersion{3, 11, 0}
+	testReleaseV3120 = CephCSIVersion{3, 12, 0}
+
+	testVersionUnsupported = CephCSIVersion{4, 0, 0}
+)
+
+func TestIsAtLeast(t *testing.T) {
+	// Test version which is smaller
+	var version = CephCSIVersion{1, 40, 10}
+	ret := testMinVersion.isAtLeast(&version)
+	assert.Equal(t, true, ret)
+
+	// Test version which is equal
+	ret = testMinVersion.isAtLeast(&testMinVersion)
+	assert.Equal(t, true, ret)
+
+	// Test for 3.9.0
+	ret = testReleaseV390.isAtLeast(&testMinVersion)
+	assert.Equal(t, false, ret)
+
+	// Test for 3.9.1
+	ret = testReleaseV391.isAtLeast(&testMinVersion)
+	assert.Equal(t, false, ret)
+
+	// Test for 3.10.0
+	ret = testreleasev310.isAtLeast(&testReleaseV390)
+	assert.Equal(t, true, ret)
+
+	// Test for 3.10.1
+	ret = testReleaseV3101.isAtLeast(&testReleaseV3101)
+	assert.Equal(t, true, ret)
+
+	// Test for 3.10.2
+	ret = testReleaseV3102.isAtLeast(&testReleaseV3102)
+	assert.Equal(t, true, ret)
+
+	// Test for 3.11.0
+	ret = testReleaseV3110.isAtLeast(&testReleaseV3110)
+	assert.Equal(t, true, ret)
+
+	// Test for 3.12.0
+	ret = testReleaseV3120.isAtLeast(&testReleaseV3120)
+	assert.Equal(t, true, ret)
+
+}
+
+func TestSupported(t *testing.T) {
+	AllowUnsupported = false
+	ret := testMinVersion.Supported()
+	assert.Equal(t, true, ret)
+
+	ret = testVersionUnsupported.Supported()
+	assert.Equal(t, false, ret)
+
+	// 3.9.x is not supported after 3.11.0 release
+	ret = testReleaseV390.Supported()
+	assert.Equal(t, false, ret)
+
+	ret = testReleaseV391.Supported()
+	assert.Equal(t, false, ret)
+
+	ret = testreleasev310.Supported()
+	assert.Equal(t, false, ret)
+
+	ret = testReleaseV3101.Supported()
+	assert.Equal(t, false, ret)
+
+	ret = testReleaseV3110.Supported()
+	assert.Equal(t, true, ret)
+
+	ret = testReleaseV3120.Supported()
+	assert.Equal(t, true, ret)
+}
+
+func Test_extractCephCSIVersion(t *testing.T) {
+	expectedVersion := CephCSIVersion{3, 0, 0}
+	csiString := []byte(`Cephcsi Version: v3.0.0
+		Git Commit: e58d537a07ca0184f67d33db85bf6b4911624b44
+		Go Version: go1.12.15
+		Compiler: gc
+		Platform: linux/amd64
+		`)
+	version, err := extractCephCSIVersion(string(csiString))
+
+	assert.Equal(t, &expectedVersion, version)
+	assert.Nil(t, err)
+
+	csiString = []byte(`Cephcsi Version: rubbish
+	Git Commit: e58d537a07ca0184f67d33db85bf6b4911624b44
+	Go Version: go1.12.15
+	Compiler: gc
+	Platform: linux/amd64
+	`)
+	version, err = extractCephCSIVersion(string(csiString))
+
+	assert.Nil(t, version)
+	assert.Contains(t, err.Error(), "failed to parse version from")
+}