Repository: rook/rook
Pull request #9565 (2022-01-12 11:46:18)

rgw: inject tls certs for bucket notification and topic operations


<!-- Please take a look at our [Contributing](https://rook.io/docs/rook/latest/development-flow.html)
documentation before submitting a Pull Request!
Thank you for contributing to Rook! -->

**Description of your changes:**
The certs for accessing TLS enabled RGW is saved as secrets and inject
them if controllers for notification and topics if the request is sent to
TLS enabled RGW endpoint.

Signed-off-by: Jiffin Tony Thottan <thottanjiffin@gmail.com>

**Which issue is resolved by this Pull Request:**
Resolves #

**Checklist:**

- [ ] **Commit Message Formatting**: Commit titles and messages follow guidelines in the [developer guide](https://rook.io/docs/rook/latest/development-flow.html#commit-structure).
- [ ] **Skip Tests for Docs**: Add the flag for skipping the build if this is only a documentation change. See [here](https://github.com/rook/rook/blob/master/INSTALL.md#skip-ci) for the flag.
- [ ] **Skip Unrelated Tests**: Add a flag to run tests for a specific storage provider. See [test options](https://github.com/rook/rook/blob/master/INSTALL.md#test-storage-provider).
- [ ] Reviewed the developer guide on [Submitting a Pull Request](https://rook.io/docs/rook/latest/development-flow.html#submitting-a-pull-request)
- [ ] Documentation has been updated, if necessary.
- [ ] Unit tests have been added, if necessary.
- [ ] Integration tests have been added, if necessary.
- [ ] Pending release notes updated with breaking and/or notable changes, if necessary.
- [ ] Upgrade from previous release is tested and upgrade user guide is updated, if necessary.
- [ ] Code generation (`make codegen`) has been run to update object specifications, if necessary.


thotz: > I think this looks good, being pretty straightforward, but I have a couple questions...
> 
> I don't see any changes to unit tests. Should we add some validation to the existing unit tests for this to make sure the TLS connections are working as we expect them to?
@BlaineEXE :
I guess it is better to add cases in the integration tests of SSL objectstore than units tests, we check this cephobjectstore user, obc via integration test than unit tests.

> 
> Should we update any documentation for users about how to use certs?

Not really, this is internal between Rook and RGW server. If the RGW server is configured with TLS, then Rook as a s3 client should not fail operations sent to the RGW server.

yuvalif: 
> I don't see any changes to unit tests. Should we add some validation to the existing unit tests for this to make sure the TLS connections are working as we expect them to?

we do not have any unit tests that cover the session creation between the operator and the RGW.
currently, this is covered in the integration tests, we should probably add that there.
yuvalif: `GetBucketNotificationConfiguration` test in failing:
https://github.com/rook/rook/runs/4806381551?check_suite_focus=true#step:4:1371

Patch:
diff --git a/.github/workflows/daily-nightly-jobs.yml b/.github/workflows/daily-nightly-jobs.yml
index 042bb2cdb..7c7df6e28 100644
--- a/.github/workflows/daily-nightly-jobs.yml
+++ b/.github/workflows/daily-nightly-jobs.yml
@@ -324,16 +324,13 @@ jobs:
       with:
         fetch-depth: 0
 
-    - name: run encryption KMS IBM Key Protect
+    - name: run Encryption KMS IBM Key Protect
       uses: ./.github/workflows/encryption-pvc-kms-ibm-kp
-      if: "env.IBM_KP_SERVICE_INSTANCE_ID != '' && env.IBM_KP_SERVICE_API_KEY != ''"
-      env:
-        IBM_KP_SERVICE_INSTANCE_ID: ${{ secrets.IBM_INSTANCE_ID }}
-        IBM_KP_SERVICE_API_KEY: ${{ secrets.IBM_SERVICE_API_KEY }}
+      if: "env.IBM_KP_INSTANCE_ID != '' && env.IBM_KP_SERVICE_API_KEY != ''"
       with:
-        ibm-instance-id: ${{ secrets.IBM_INSTANCE_ID }}
-        ibm-service-api-key: ${{ secrets.IBM_SERVICE_API_KEY }}
         github-token: ${{ secrets.GITHUB_TOKEN }}
+        ibm-instance-id: ${{ secrets.IBM_KP_INSTANCE_ID }}
+        ibm-service-api-key: ${{ secrets.IBM_KP_SERVICE_API_KEY }}
 
     - name: upload test result
       uses: actions/upload-artifact@v2
diff --git a/.github/workflows/encryption-pvc-kms-ibm-kp/action.yml b/.github/workflows/encryption-pvc-kms-ibm-kp/action.yml
index ee8fdc6db..4b83110a8 100644
--- a/.github/workflows/encryption-pvc-kms-ibm-kp/action.yml
+++ b/.github/workflows/encryption-pvc-kms-ibm-kp/action.yml
@@ -2,7 +2,7 @@ name: Encryption KMS IBM Key Protect
 description: Reusable workflow to test Encryption KMS IBM Key Protect
 inputs:
   ibm-instance-id:
-    description: IBM_KP_SERVICE_INSTANCE_ID from the calling workflow
+    description: IBM_KP_INSTANCE_ID from the calling workflow
     required: true
   ibm-service-api-key:
     description: IBM_KP_SERVICE_API_KEY from the calling workflow
@@ -15,12 +15,12 @@ runs:
   using: "composite"
   steps:
     - name: fail if env no present
-      if: "env.IBM_KP_SERVICE_INSTANCE_ID == '' || env.IBM_KP_SERVICE_API_KEY == ''"
+      if: "env.IBM_KP_INSTANCE_ID == '' || env.IBM_KP_SERVICE_API_KEY == ''"
       env:
-        IBM_KP_SERVICE_INSTANCE_ID: ${{ inputs.ibm-instance-id }}
+        IBM_KP_INSTANCE_ID: ${{ inputs.ibm-instance-id }}
         IBM_KP_SERVICE_API_KEY: ${{ inputs.ibm-service-api-key }}
       shell: bash --noprofile --norc -eo pipefail -x {0}
-      run: echo "IBM_KP_SERVICE_INSTANCE_ID and IBM_KP_SERVICE_API_KEY must be set in the environment" && exit 0
+      run: echo "IBM_KP_INSTANCE_ID and IBM_KP_SERVICE_API_KEY must be set in the environment" && exit 0
 
     - name: setup cluster resources
       uses: ./.github/workflows/setup-cluster-resources
@@ -42,7 +42,7 @@ runs:
     - name: deploy cluster
       shell: bash --noprofile --norc -eo pipefail -x {0}
       env:
-        IBM_KP_SERVICE_INSTANCE_ID: ${{ inputs.ibm-instance-id }}
+        IBM_KP_INSTANCE_ID: ${{ inputs.ibm-instance-id }}
         IBM_KP_SERVICE_API_KEY: ${{ inputs.ibm-service-api-key }}
       run: |
         tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml
diff --git a/Documentation/ceph-kms.md b/Documentation/ceph-kms.md
index 24f84d553..24f0d373b 100644
--- a/Documentation/ceph-kms.md
+++ b/Documentation/ceph-kms.md
@@ -266,8 +266,8 @@ security:
   kms:
     # name of the k8s config map containing all the kms connection details
     connectionDetails:
-      KMS_PROVIDER: ibmkeyprotect
-      IBM_KP_SERVICE_INSTANCE_ID: <instance ID that was retrieved in the first paragraph>
+      KMS_PROVIDER: ibm-kp
+      IBM_KP_INSTANCE_ID: <instance ID that was retrieved in the first paragraph>
     # name of the k8s secret containing the service API Key
     tokenSecretName: ibm-kp-svc-api-key
 ```
diff --git a/Documentation/ceph-upgrade.md b/Documentation/ceph-upgrade.md
index b8b1b87eb..feb8ed786 100644
--- a/Documentation/ceph-upgrade.md
+++ b/Documentation/ceph-upgrade.md
@@ -545,14 +545,14 @@ kubectl -n $ROOK_OPERATOR_NAMESPACE edit configmap rook-ceph-operator-config
 The default upstream images are included below, which you can change to your desired images.
 
 ```yaml
-ROOK_CSI_CEPH_IMAGE: "quay.io/cephcsi/cephcsi:v3.5.1"
+ROOK_CSI_CEPH_IMAGE: "quay.io/cephcsi/cephcsi:v3.5.0"
 ROOK_CSI_REGISTRAR_IMAGE: "k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.4.0"
 ROOK_CSI_PROVISIONER_IMAGE: "k8s.gcr.io/sig-storage/csi-provisioner:v3.1.0"
 ROOK_CSI_ATTACHER_IMAGE: "k8s.gcr.io/sig-storage/csi-attacher:v3.4.0"
 ROOK_CSI_RESIZER_IMAGE: "k8s.gcr.io/sig-storage/csi-resizer:v1.3.0"
 ROOK_CSI_SNAPSHOTTER_IMAGE: "k8s.gcr.io/sig-storage/csi-snapshotter:v4.2.0"
 CSI_VOLUME_REPLICATION_IMAGE: "quay.io/csiaddons/volumereplication-operator:v0.1.0"
-ROOK_CSIADDONS_IMAGE: "quay.io/csiaddons/k8s-sidecar:v0.2.1"
+ROOK_CSIADDONS_IMAGE: "quay.io/csiaddons/k8s-sidecar:v0.1.0"
 ```
 
 ### **Use default images**
@@ -576,7 +576,7 @@ k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.4.0
 k8s.gcr.io/sig-storage/csi-provisioner:v3.1.0
 k8s.gcr.io/sig-storage/csi-resizer:v1.3.0
 k8s.gcr.io/sig-storage/csi-snapshotter:v4.2.0
-quay.io/cephcsi/cephcsi:v3.5.1
+quay.io/cephcsi/cephcsi:v3.5.0
 quay.io/csiaddons/volumereplication-operator:v0.1.0
-quay.io/csiaddons/k8s-sidecar:v0.2.1
+quay.io/csiaddons/k8s-sidecar:v0.1.0
 ```
diff --git a/Documentation/helm-operator.md b/Documentation/helm-operator.md
index 0fcab3438..919c5cfc3 100644
--- a/Documentation/helm-operator.md
+++ b/Documentation/helm-operator.md
@@ -134,7 +134,7 @@ The following tables lists the configurable parameters of the rook-operator char
 | `csi.rbdLivenessMetricsPort`        | Ceph CSI RBD driver metrics port.                                                                                           | `8080`                                                    |
 | `csi.forceCephFSKernelClient`       | Enable Ceph Kernel clients on kernel < 4.17 which support quotas for Cephfs.                                                | `true`                                                    |
 | `csi.kubeletDirPath`                | Kubelet root directory path (if the Kubelet uses a different path for the `--root-dir` flag)                                | `/var/lib/kubelet`                                        |
-| `csi.cephcsi.image`                 | Ceph CSI image.                                                                                                             | `quay.io/cephcsi/cephcsi:v3.5.1`                          |
+| `csi.cephcsi.image`                 | Ceph CSI image.                                                                                                             | `quay.io/cephcsi/cephcsi:v3.5.0`                          |
 | `csi.rbdPluginUpdateStrategy`       | CSI Rbd plugin daemonset update strategy, supported values are OnDelete and RollingUpdate.                                  | `OnDelete`                                                |
 | `csi.cephFSPluginUpdateStrategy`    | CSI CephFS plugin daemonset update strategy, supported values are OnDelete and RollingUpdate.                               | `OnDelete`                                                |
 | `csi.registrar.image`               | Kubernetes CSI registrar image.                                                                                             | `k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.4.0` |
@@ -147,7 +147,7 @@ The following tables lists the configurable parameters of the rook-operator char
 | `csi.volumeReplication.enabled`     | Enable Volume Replication.                                                                                                  | `false`                                                   |
 | `csi.volumeReplication.image`       | Volume Replication Controller image.                                                                                        | `quay.io/csiaddons/volumereplication-operator:v0.1.0`     |
 | `csi.csiAddons.enabled`     | Enable CSIAddons                                                                                                  | `false`                                                   |
-| `csi.csiAddons.image`       | CSIAddons Sidecar image.                                                                                        | `quay.io/csiaddons/k8s-sidecar:v0.2.1`     |
+| `csi.csiAddons.image`       | CSIAddons Sidecar image.                                                                                        | `quay.io/csiaddons/k8s-sidecar:v0.1.0`     |
 | `admissionController.tolerations`   | Array of tolerations in YAML format which will be added to admission controller deployment.                                 | <none>                                                    |
 | `admissionController.nodeAffinity`  | The node labels for affinity of the admission controller deployment (***)                                                   | <none>                                                    |
 | `allowMultipleFilesystems`          | **(experimental in  Octopus (v15))** Allows multiple filesystems to be deployed to a Ceph cluster.                          | `false`                                                   |
diff --git a/deploy/charts/rook-ceph/templates/configmap.yaml b/deploy/charts/rook-ceph/templates/configmap.yaml
deleted file mode 100644
index ccc0a3a48..000000000
--- a/deploy/charts/rook-ceph/templates/configmap.yaml
+++ /dev/null
@@ -1,163 +0,0 @@
-# Operator settings that can be updated without an operator restart
-# Operator settings that require an operator restart are found in the operator env vars
-kind: ConfigMap
-apiVersion: v1
-metadata:
-  name: rook-ceph-operator-config
-data:
-  ROOK_LOG_LEVEL: {{ .Values.logLevel | quote }}
-  ROOK_CEPH_COMMANDS_TIMEOUT_SECONDS: {{ .Values.cephCommandsTimeoutSeconds | quote }}
-  ROOK_OBC_WATCH_OPERATOR_NAMESPACE: {{ .Values.enableOBCWatchOperatorNamespace | quote }}
-{{- if .Values.csi }}
-  ROOK_CSI_ENABLE_RBD: {{ .Values.csi.enableRbdDriver | quote }}
-  ROOK_CSI_ENABLE_CEPHFS: {{ .Values.csi.enableCephfsDriver | quote }}
-  CSI_ENABLE_CEPHFS_SNAPSHOTTER: {{ .Values.csi.enableCephfsSnapshotter | quote }}
-  CSI_ENABLE_RBD_SNAPSHOTTER: {{ .Values.csi.enableRBDSnapshotter | quote }}
-  CSI_PLUGIN_ENABLE_SELINUX_HOST_MOUNT: {{ .Values.csi.enablePluginSelinuxHostMount | quote }}
-  CSI_ENABLE_OMAP_GENERATOR: {{ .Values.csi.enableOMAPGenerator | quote }}
-{{- if .Values.csi.pluginPriorityClassName }}
-  CSI_PLUGIN_PRIORITY_CLASSNAME: {{ .Values.csi.pluginPriorityClassName | quote }}
-{{- end }}
-{{- if .Values.csi.provisionerPriorityClassName }}
-  CSI_PROVISIONER_PRIORITY_CLASSNAME: {{ .Values.csi.provisionerPriorityClassName | quote }}
-{{- end }}
-{{- if .Values.csi.enableCSIHostNetwork }}
-  CSI_ENABLE_HOST_NETWORK: {{ .Values.csi.enableCSIHostNetwork | quote }}
-{{- end }}
-{{- if .Values.csi.cephFSPluginUpdateStrategy }}
-  CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY: {{ .Values.csi.cephFSPluginUpdateStrategy | quote }}
-{{- end }}
-{{- if .Values.csi.rbdFSGroupPolicy }}
-  CSI_RBD_FSGROUPPOLICY: {{ .Values.csi.rbdFSGroupPolicy | quote }}
-{{- end }}
-{{- if .Values.csi.cephFSFSGroupPolicy }}
-  CSI_CEPHFS_FSGROUPPOLICY: {{ .Values.csi.cephFSFSGroupPolicy | quote }}
-{{- end }}
-{{- if .Values.csi.rbdPluginUpdateStrategy }}
-  CSI_RBD_PLUGIN_UPDATE_STRATEGY: {{ .Values.csi.rbdPluginUpdateStrategy | quote }}
-{{- end }}
-{{- if .Values.csi.kubeletDirPath }}
-  ROOK_CSI_KUBELET_DIR_PATH: {{ .Values.csi.kubeletDirPath | quote }}
-{{- end }}
-  ROOK_CSI_ENABLE_GRPC_METRICS: {{ .Values.csi.enableGrpcMetrics | quote }}
-{{- if .Values.csi.cephcsi }}
-{{- if .Values.csi.cephcsi.image }}
-  ROOK_CSI_CEPH_IMAGE: {{ .Values.csi.cephcsi.image | quote }}
-{{- end }}
-{{- end }}
-{{- if .Values.csi.registrar }}
-{{- if .Values.csi.registrar.image }}
-  ROOK_CSI_REGISTRAR_IMAGE: {{ .Values.csi.registrar.image | quote }}
-{{- end }}
-{{- end }}
-{{- if .Values.csi.provisioner }}
-{{- if .Values.csi.provisioner.image }}
-  ROOK_CSI_PROVISIONER_IMAGE: {{ .Values.csi.provisioner.image | quote }}
-{{- end }}
-{{- end }}
-{{- if .Values.csi.snapshotter }}
-{{- if .Values.csi.snapshotter.image }}
-  ROOK_CSI_SNAPSHOTTER_IMAGE: {{ .Values.csi.snapshotter.image | quote }}
-{{- end }}
-{{- end }}
-{{- if .Values.csi.attacher }}
-{{- if .Values.csi.attacher.image }}
-  ROOK_CSI_ATTACHER_IMAGE: {{ .Values.csi.attacher.image | quote }}
-{{- end }}
-{{- end }}
-{{- if .Values.csi.resizer }}
-{{- if .Values.csi.resizer.image }}
-  ROOK_CSI_RESIZER_IMAGE: {{ .Values.csi.resizer.image | quote }}
-{{- end }}
-{{- end }}
-{{- if .Values.csi.volumeReplication }}
-  CSI_ENABLE_VOLUME_REPLICATION: {{ .Values.csi.volumeReplication.enabled | quote }}
-{{- if .Values.csi.volumeReplication.image }}
-  CSI_VOLUME_REPLICATION_IMAGE: {{ .Values.csi.volumeReplication.image | quote }}
-{{- end }}
-{{- end }}
-{{- if .Values.csi.csiAddons }}
-  CSI_ENABLE_CSIADDONS: {{ .Values.csi.csiAddons.enabled | quote }}
-{{- if .Values.csi.csiAddons.image }}
-  ROOK_CSIADDONS_IMAGE: {{ .Values.csi.csiAddons.image | quote }}
-{{- end }}
-{{- end }}
-{{- if .Values.csi.cephfsPodLabels }}
-  ROOK_CSI_CEPHFS_POD_LABELS: {{ .Values.csi.cephfsPodLabels | quote }}
-{{- end }}
-{{- if .Values.csi.rbdPodLabels }}
-  ROOK_CSI_RBD_POD_LABELS: {{ .Values.csi.rbdPodLabels | quote }}
-{{- end }}
-{{- if .Values.csi.provisionerTolerations }}
-  CSI_PROVISIONER_TOLERATIONS: {{ toYaml .Values.csi.provisionerTolerations | quote }}
-{{- end }}
-{{- if .Values.csi.provisionerNodeAffinity }}
-  CSI_PROVISIONER_NODE_AFFINITY: {{ .Values.csi.provisionerNodeAffinity }}
-{{- end }}
-{{- if .Values.csi.rbdProvisionerTolerations }}
-  CSI_RBD_PROVISIONER_TOLERATIONS: {{ toYaml .Values.csi.rbdProvisionerTolerations | quote }}
-{{- end }}
-{{- if .Values.csi.rbdProvisionerNodeAffinity }}
-  CSI_RBD_PROVISIONER_NODE_AFFINITY: {{ .Values.csi.rbdProvisionerNodeAffinity }}
-{{- end }}
-{{- if .Values.csi.cephFSProvisionerTolerations }}
-  CSI_CEPHFS_PROVISIONER_TOLERATIONS: {{ toYaml .Values.csi.cephFSProvisionerTolerations | quote }}
-{{- end }}
-{{- if .Values.csi.cephFSProvisionerNodeAffinity }}
-  CSI_CEPHFS_PROVISIONER_NODE_AFFINITY: {{ .Values.csi.cephFSProvisionerNodeAffinity }}
-{{- end }}
-{{- if .Values.csi.allowUnsupportedVersion }}
-  ROOK_CSI_ALLOW_UNSUPPORTED_VERSION: {{ .Values.csi.allowUnsupportedVersion | quote }}
-{{- end }}
-{{- if .Values.csi.pluginTolerations }}
-  CSI_PLUGIN_TOLERATIONS: {{ toYaml .Values.csi.pluginTolerations | quote }}
-{{- end }}
-{{- if .Values.csi.pluginNodeAffinity }}
-  CSI_PLUGIN_NODE_AFFINITY: {{ .Values.csi.pluginNodeAffinity }}
-{{- end }}
-{{- if .Values.csi.rbdPluginTolerations }}
-  CSI_RBD_PLUGIN_TOLERATIONS: {{ toYaml .Values.csi.rbdPluginTolerations | quote }}
-{{- end }}
-{{- if .Values.csi.rbdPluginNodeAffinity }}
-  CSI_RBD_PLUGIN_NODE_AFFINITY: {{ .Values.csi.rbdPluginNodeAffinity }}
-{{- end }}
-{{- if .Values.csi.cephFSPluginTolerations }}
-  CSI_CEPHFS_PLUGIN_TOLERATIONS: {{ toYaml .Values.csi.cephFSPluginTolerations | quote }}
-{{- end }}
-{{- if .Values.csi.cephFSPluginNodeAffinity }}
-  CSI_CEPHFS_PLUGIN_NODE_AFFINITY: {{ .Values.csi.cephFSPluginNodeAffinity }}
-{{- end }}
-{{- if .Values.csi.cephfsGrpcMetricsPort }}
-  CSI_CEPHFS_GRPC_METRICS_PORT: {{ .Values.csi.cephfsGrpcMetricsPort | quote }}
-{{- end }}
-{{- if .Values.csi.cephfsLivenessMetricsPort }}
-  CSI_CEPHFS_LIVENESS_METRICS_PORT: {{ .Values.csi.cephfsLivenessMetricsPort | quote }}
-{{- end }}
-{{- if .Values.csi.rbdGrpcMetricsPort }}
-  CSI_RBD_GRPC_METRICS_PORT: {{ .Values.csi.rbdGrpcMetricsPort | quote }}
-{{- end }}
-{{- if .Values.csi.rbdLivenessMetricsPort }}
-  CSI_RBD_LIVENESS_METRICS_PORT: {{ .Values.csi.rbdLivenessMetricsPort | quote }}
-{{- end }}
-{{- if .Values.csi.forceCephFSKernelClient }}
-  CSI_FORCE_CEPHFS_KERNEL_CLIENT: {{ .Values.csi.forceCephFSKernelClient | quote }}
-{{- end }}
-{{- if .Values.csi.logLevel }}
-  CSI_LOG_LEVEL: {{ .Values.csi.logLevel | quote }}
-{{- end }}
-{{- if .Values.csi.provisionerReplicas }}
-  CSI_PROVISIONER_REPLICAS: {{ .Values.csi.provisionerReplicas | quote }}
-{{- end }}
-{{- if .Values.csi.csiRBDProvisionerResource }}
-  CSI_RBD_PROVISIONER_RESOURCE: {{ .Values.csi.csiRBDProvisionerResource | quote }}
-{{- end }}
-{{- if .Values.csi.csiRBDPluginResource }}
-  CSI_RBD_PLUGIN_RESOURCE: {{ .Values.csi.csiRBDPluginResource | quote }}
-{{- end }}
-{{- if .Values.csi.csiCephFSProvisionerResource }}
-  CSI_CEPHFS_PROVISIONER_RESOURCE: {{ .Values.csi.csiCephFSProvisionerResource | quote }}
-{{- end }}
-{{- if .Values.csi.csiCephFSPluginResource }}
-  CSI_CEPHFS_PLUGIN_RESOURCE: {{ .Values.csi.csiCephFSPluginResource | quote }}
-{{- end }}
-{{- end }}
diff --git a/deploy/charts/rook-ceph/templates/deployment.yaml b/deploy/charts/rook-ceph/templates/deployment.yaml
index 9e98f86f7..446e59b24 100644
--- a/deploy/charts/rook-ceph/templates/deployment.yaml
+++ b/deploy/charts/rook-ceph/templates/deployment.yaml
@@ -68,12 +68,222 @@ spec:
 {{- end }}
         - name: ROOK_HOSTPATH_REQUIRES_PRIVILEGED
           value: "{{ .Values.hostpathRequiresPrivileged }}"
+        - name: ROOK_LOG_LEVEL
+          value: {{ .Values.logLevel }}
         - name: ROOK_ENABLE_SELINUX_RELABELING
           value: "{{ .Values.enableSelinuxRelabeling }}"
         - name: ROOK_DISABLE_DEVICE_HOTPLUG
           value: "{{ .Values.disableDeviceHotplug }}"
+{{- if .Values.csi }}
+        - name: ROOK_CSI_ENABLE_RBD
+          value: {{ .Values.csi.enableRbdDriver | quote }}
+        - name: ROOK_CSI_ENABLE_CEPHFS
+          value: {{ .Values.csi.enableCephfsDriver | quote }}
+        - name: CSI_ENABLE_CEPHFS_SNAPSHOTTER
+          value: {{ .Values.csi.enableCephfsSnapshotter | quote }}
+        - name: CSI_ENABLE_RBD_SNAPSHOTTER
+          value: {{ .Values.csi.enableRBDSnapshotter | quote }}
+        - name: CSI_PLUGIN_PRIORITY_CLASSNAME
+          value: {{ .Values.csi.pluginPriorityClassName | quote }}
+        - name: CSI_PROVISIONER_PRIORITY_CLASSNAME
+          value: {{ .Values.csi.provisionerPriorityClassName | quote }}
+        - name: CSI_ENABLE_OMAP_GENERATOR
+          value: {{ .Values.csi.enableOMAPGenerator | quote }}
+        - name: CSI_ENABLE_VOLUME_REPLICATION
+          value: {{ .Values.csi.volumeReplication.enabled | quote }}
+        - name: CSI_ENABLE_CSIADDONS
+          value: {{ .Values.csi.csiAddons.enabled | quote }}
+        - name: CSI_PLUGIN_ENABLE_SELINUX_HOST_MOUNT
+          value: {{ .Values.csi.enablePluginSelinuxHostMount | quote }}
+{{- if .Values.csi.enableCSIHostNetwork }}
+        - name: CSI_ENABLE_HOST_NETWORK
+          value: {{ .Values.csi.enableCSIHostNetwork | quote }}
+{{- end }}
+{{- if .Values.csi.cephFSPluginUpdateStrategy }}
+        - name: CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY
+          value: {{ .Values.csi.cephFSPluginUpdateStrategy | quote }}
+{{- end }}
+{{- if .Values.csi.rbdFSGroupPolicy }}
+        - name: CSI_RBD_FSGROUPPOLICY
+          value: {{ .Values.csi.rbdFSGroupPolicy | quote }}
+{{- end }}
+{{- if .Values.csi.cephFSFSGroupPolicy }}
+        - name: CSI_CEPHFS_FSGROUPPOLICY
+          value: {{ .Values.csi.cephFSFSGroupPolicy | quote }}
+{{- end }}
+{{- if .Values.csi.rbdPluginUpdateStrategy }}
+        - name: CSI_RBD_PLUGIN_UPDATE_STRATEGY
+          value: {{ .Values.csi.rbdPluginUpdateStrategy | quote }}
+{{- end }}
+{{- if .Values.csi.kubeletDirPath }}
+        - name: ROOK_CSI_KUBELET_DIR_PATH
+          value: {{ .Values.csi.kubeletDirPath | quote }}
+{{- end }}
+        - name: ROOK_CSI_ENABLE_GRPC_METRICS
+          value: {{ .Values.csi.enableGrpcMetrics | quote }}
+{{- if .Values.csi.cephcsi }}
+{{- if .Values.csi.cephcsi.image }}
+        - name: ROOK_CSI_CEPH_IMAGE
+          value: {{ .Values.csi.cephcsi.image | quote }}
+{{- end }}
+{{- end }}
+{{- if .Values.csi.registrar }}
+{{- if .Values.csi.registrar.image }}
+        - name: ROOK_CSI_REGISTRAR_IMAGE
+          value: {{ .Values.csi.registrar.image | quote }}
+{{- end }}
+{{- end }}
+{{- if .Values.csi.provisioner }}
+{{- if .Values.csi.provisioner.image }}
+        - name: ROOK_CSI_PROVISIONER_IMAGE
+          value: {{ .Values.csi.provisioner.image | quote }}
+{{- end }}
+{{- end }}
+{{- if .Values.csi.snapshotter }}
+{{- if .Values.csi.snapshotter.image }}
+        - name: ROOK_CSI_SNAPSHOTTER_IMAGE
+          value: {{ .Values.csi.snapshotter.image | quote }}
+{{- end }}
+{{- end }}
+{{- if .Values.csi.attacher }}
+{{- if .Values.csi.attacher.image }}
+        - name: ROOK_CSI_ATTACHER_IMAGE
+          value: {{ .Values.csi.attacher.image | quote }}
+{{- end }}
+{{- end }}
+{{- if .Values.csi.resizer }}
+{{- if .Values.csi.resizer.image }}
+        - name: ROOK_CSI_RESIZER_IMAGE
+          value: {{ .Values.csi.resizer.image | quote }}
+{{- end }}
+{{- end }}
+{{- if .Values.csi.volumeReplication }}
+{{- if .Values.csi.volumeReplication.image }}
+        - name: CSI_VOLUME_REPLICATION_IMAGE
+          value: {{ .Values.csi.volumeReplication.image | quote }}
+{{- end }}
+{{- end }}
+{{- if .Values.csi.csiAddons }}
+{{- if .Values.csi.csiAddons.image }}
+        - name: ROOK_CSIADDONS_IMAGE
+          value: {{ .Values.csi.csiAddons.image | quote }}
+{{- end }}
+{{- if .Values.csi.csiAddonsPort }}
+        - name: CSIADDONS_PORT
+          value: {{ .Values.csi.csiAddonsPort | quote }}
+{{- end }}
+{{- end }}
+{{- if .Values.csi.cephfsPodLabels }}
+        - name: ROOK_CSI_CEPHFS_POD_LABELS
+          value: {{ .Values.csi.cephfsPodLabels | quote }}
+{{- end }}
+{{- if .Values.csi.rbdPodLabels }}
+        - name: ROOK_CSI_RBD_POD_LABELS
+          value: {{ .Values.csi.rbdPodLabels | quote }}
+{{- end }}
+{{- if .Values.csi.provisionerTolerations }}
+        - name: CSI_PROVISIONER_TOLERATIONS
+          value: {{ toYaml .Values.csi.provisionerTolerations | quote }}
+{{- end }}
+{{- if .Values.csi.provisionerNodeAffinity }}
+        - name: CSI_PROVISIONER_NODE_AFFINITY
+          value: {{ .Values.csi.provisionerNodeAffinity }}
+{{- end }}
+{{- if .Values.csi.rbdProvisionerTolerations }}
+        - name: CSI_RBD_PROVISIONER_TOLERATIONS
+          value: {{ toYaml .Values.csi.rbdProvisionerTolerations | quote }}
+{{- end }}
+{{- if .Values.csi.rbdProvisionerNodeAffinity }}
+        - name: CSI_RBD_PROVISIONER_NODE_AFFINITY
+          value: {{ .Values.csi.rbdProvisionerNodeAffinity }}
+{{- end }}
+{{- if .Values.csi.cephFSProvisionerTolerations }}
+        - name: CSI_CEPHFS_PROVISIONER_TOLERATIONS
+          value: {{ toYaml .Values.csi.cephFSProvisionerTolerations | quote }}
+{{- end }}
+{{- if .Values.csi.cephFSProvisionerNodeAffinity }}
+        - name: CSI_CEPHFS_PROVISIONER_NODE_AFFINITY
+          value: {{ .Values.csi.cephFSProvisionerNodeAffinity }}
+{{- end }}
+{{- if .Values.csi.allowUnsupportedVersion }}
+        - name: ROOK_CSI_ALLOW_UNSUPPORTED_VERSION
+          value: {{ .Values.csi.allowUnsupportedVersion | quote }}
+{{- end }}
+{{- if .Values.csi.pluginTolerations }}
+        - name: CSI_PLUGIN_TOLERATIONS
+          value: {{ toYaml .Values.csi.pluginTolerations | quote }}
+{{- end }}
+{{- if .Values.csi.pluginNodeAffinity }}
+        - name: CSI_PLUGIN_NODE_AFFINITY
+          value: {{ .Values.csi.pluginNodeAffinity }}
+{{- end }}
+{{- if .Values.csi.rbdPluginTolerations }}
+        - name: CSI_RBD_PLUGIN_TOLERATIONS
+          value: {{ toYaml .Values.csi.rbdPluginTolerations | quote }}
+{{- end }}
+{{- if .Values.csi.rbdPluginNodeAffinity }}
+        - name: CSI_RBD_PLUGIN_NODE_AFFINITY
+          value: {{ .Values.csi.rbdPluginNodeAffinity }}
+{{- end }}
+{{- if .Values.csi.cephFSPluginTolerations }}
+        - name: CSI_CEPHFS_PLUGIN_TOLERATIONS
+          value: {{ toYaml .Values.csi.cephFSPluginTolerations | quote }}
+{{- end }}
+{{- if .Values.csi.cephFSPluginNodeAffinity }}
+        - name: CSI_CEPHFS_PLUGIN_NODE_AFFINITY
+          value: {{ .Values.csi.cephFSPluginNodeAffinity }}
+{{- end }}
+{{- if .Values.csi.cephfsGrpcMetricsPort }}
+        - name: CSI_CEPHFS_GRPC_METRICS_PORT
+          value: {{ .Values.csi.cephfsGrpcMetricsPort | quote }}
+{{- end }}
+{{- if .Values.csi.cephfsLivenessMetricsPort }}
+        - name: CSI_CEPHFS_LIVENESS_METRICS_PORT
+          value: {{ .Values.csi.cephfsLivenessMetricsPort | quote }}
+{{- end }}
+{{- if .Values.csi.rbdGrpcMetricsPort }}
+        - name: CSI_RBD_GRPC_METRICS_PORT
+          value: {{ .Values.csi.rbdGrpcMetricsPort | quote }}
+{{- end }}
+{{- if .Values.csi.rbdLivenessMetricsPort }}
+        - name: CSI_RBD_LIVENESS_METRICS_PORT
+          value: {{ .Values.csi.rbdLivenessMetricsPort | quote }}
+{{- end }}
+{{- if .Values.csi.forceCephFSKernelClient }}
+        - name: CSI_FORCE_CEPHFS_KERNEL_CLIENT
+          value: {{ .Values.csi.forceCephFSKernelClient | quote }}
+{{- end }}
+{{- if .Values.csi.logLevel }}
+        - name: CSI_LOG_LEVEL
+          value: {{ .Values.csi.logLevel | quote }}
+{{- end }}
+{{- if .Values.csi.provisionerReplicas }}
+        - name: CSI_PROVISIONER_REPLICAS
+          value: {{ .Values.csi.provisionerReplicas | quote }}
+{{- end }}
+{{- if .Values.csi.csiRBDProvisionerResource }}
+        - name: CSI_RBD_PROVISIONER_RESOURCE
+          value: {{ .Values.csi.csiRBDProvisionerResource | quote }}
+{{- end }}
+{{- if .Values.csi.csiRBDPluginResource }}
+        - name: CSI_RBD_PLUGIN_RESOURCE
+          value: {{ .Values.csi.csiRBDPluginResource | quote }}
+{{- end }}
+{{- if .Values.csi.csiCephFSProvisionerResource }}
+        - name: CSI_CEPHFS_PROVISIONER_RESOURCE
+          value: {{ .Values.csi.csiCephFSProvisionerResource | quote }}
+{{- end }}
+{{- if .Values.csi.csiCephFSPluginResource }}
+        - name: CSI_CEPHFS_PLUGIN_RESOURCE
+          value: {{ .Values.csi.csiCephFSPluginResource | quote }}
+{{- end }}
+{{- end }}
         - name: ROOK_ENABLE_DISCOVERY_DAEMON
           value: "{{ .Values.enableDiscoveryDaemon }}"
+        - name: ROOK_CEPH_COMMANDS_TIMEOUT_SECONDS
+          value: "{{ .Values.cephCommandsTimeoutSeconds }}"
+        - name: ROOK_OBC_WATCH_OPERATOR_NAMESPACE
+          value: "{{ .Values.enableOBCWatchOperatorNamespace }}"
 
         - name: NODE_NAME
           valueFrom:
diff --git a/deploy/charts/rook-ceph/values.yaml b/deploy/charts/rook-ceph/values.yaml
index 1213e5cf8..8a869b761 100644
--- a/deploy/charts/rook-ceph/values.yaml
+++ b/deploy/charts/rook-ceph/values.yaml
@@ -276,7 +276,7 @@ csi:
   #rbdLivenessMetricsPort: 9080
   #kubeletDirPath: /var/lib/kubelet
   #cephcsi:
-    #image: quay.io/cephcsi/cephcsi:v3.5.1
+    #image: quay.io/cephcsi/cephcsi:v3.5.0
   #registrar:
     #image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.4.0
   #provisioner:
@@ -300,7 +300,7 @@ csi:
   # Enable the CSIAddons sidecar.
   csiAddons:
     enabled: false
-    #image: "quay.io/csiaddons/k8s-sidecar:v0.2.1"
+    #image: "quay.io/csiaddons/k8s-sidecar:v0.1.0"
 enableDiscoveryDaemon: false
 cephCommandsTimeoutSeconds: "15"
 
diff --git a/deploy/examples/images.txt b/deploy/examples/images.txt
index f29ca0f18..dac22620e 100644
--- a/deploy/examples/images.txt
+++ b/deploy/examples/images.txt
@@ -4,7 +4,7 @@
  k8s.gcr.io/sig-storage/csi-resizer:v1.3.0
  k8s.gcr.io/sig-storage/csi-snapshotter:v4.2.0
  quay.io/ceph/ceph:v16.2.7
- quay.io/cephcsi/cephcsi:v3.5.1
- quay.io/csiaddons/k8s-sidecar:v0.2.1
+ quay.io/cephcsi/cephcsi:v3.5.0
+ quay.io/csiaddons/k8s-sidecar:v0.1.0
  quay.io/csiaddons/volumereplication-operator:v0.1.0
  rook/ceph:master
diff --git a/deploy/examples/operator-openshift.yaml b/deploy/examples/operator-openshift.yaml
index 5d56d60e8..d1340d6b4 100644
--- a/deploy/examples/operator-openshift.yaml
+++ b/deploy/examples/operator-openshift.yaml
@@ -161,7 +161,7 @@ data:
   # The default version of CSI supported by Rook will be started. To change the version
   # of the CSI driver to something other than what is officially supported, change
   # these images to the desired release of the CSI driver.
-  # ROOK_CSI_CEPH_IMAGE: "quay.io/cephcsi/cephcsi:v3.5.1"
+  # ROOK_CSI_CEPH_IMAGE: "quay.io/cephcsi/cephcsi:v3.5.0"
   # ROOK_CSI_REGISTRAR_IMAGE: "k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.4.0"
   # ROOK_CSI_RESIZER_IMAGE: "k8s.gcr.io/sig-storage/csi-resizer:v1.3.0"
   # ROOK_CSI_PROVISIONER_IMAGE: "k8s.gcr.io/sig-storage/csi-provisioner:v3.1.0"
@@ -420,7 +420,7 @@ data:
   # CSI_VOLUME_REPLICATION_IMAGE: "quay.io/csiaddons/volumereplication-operator:v0.1.0"
   # Enable the csi addons sidecar.
   CSI_ENABLE_CSIADDONS: "false"
-  # ROOK_CSIADDONS_IMAGE: "quay.io/csiaddons/k8s-sidecar:v0.2.1"
+  # ROOK_CSIADDONS_IMAGE: "quay.io/csiaddons/k8s-sidecar:v0.1.0"
 ---
 # The deployment for the rook operator
 # OLM: BEGIN OPERATOR DEPLOYMENT
diff --git a/deploy/examples/operator.yaml b/deploy/examples/operator.yaml
index 38297d53c..c81ad7a9e 100644
--- a/deploy/examples/operator.yaml
+++ b/deploy/examples/operator.yaml
@@ -79,7 +79,7 @@ data:
   # The default version of CSI supported by Rook will be started. To change the version
   # of the CSI driver to something other than what is officially supported, change
   # these images to the desired release of the CSI driver.
-  # ROOK_CSI_CEPH_IMAGE: "quay.io/cephcsi/cephcsi:v3.5.1"
+  # ROOK_CSI_CEPH_IMAGE: "quay.io/cephcsi/cephcsi:v3.5.0"
   # ROOK_CSI_REGISTRAR_IMAGE: "k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.4.0"
   # ROOK_CSI_RESIZER_IMAGE: "k8s.gcr.io/sig-storage/csi-resizer:v1.3.0"
   # ROOK_CSI_PROVISIONER_IMAGE: "k8s.gcr.io/sig-storage/csi-provisioner:v3.1.0"
@@ -338,7 +338,7 @@ data:
   # CSI_VOLUME_REPLICATION_IMAGE: "quay.io/csiaddons/volumereplication-operator:v0.1.0"
   # Enable the csi addons sidecar.
   CSI_ENABLE_CSIADDONS: "false"
-  # ROOK_CSIADDONS_IMAGE: "quay.io/csiaddons/k8s-sidecar:v0.2.1"
+  # ROOK_CSIADDONS_IMAGE: "quay.io/csiaddons/k8s-sidecar:v0.1.0"
 ---
 # OLM: BEGIN OPERATOR DEPLOYMENT
 apiVersion: apps/v1
diff --git a/design/ceph/object/swift-and-keystone-integration.md b/design/ceph/object/swift-and-keystone-integration.md
deleted file mode 100644
index 27717ade5..000000000
--- a/design/ceph/object/swift-and-keystone-integration.md
+++ /dev/null
@@ -1,323 +0,0 @@
----
-target-version: release-1.9
----
-
-# Swift and Keystone Integration
-
-## Summary
-
-### Goals
-
-The goal of this proposal is to allow configuring the Swift API and
-the Keystone integration of Ceph RGW natively via the Object Store
-CRD, which allows native integration of the Rook operated Ceph RGW into
-OpenStack clouds.
-
-Both changes are bundled together as one proposal because they will
-typically deployed together. It is unlikely to encounter a Swift
-object store API outside an OpenStack cloud and in the context of an
-OpenStack cloud the Swift API needs to be integrated with the Keystone
-authentication service.
-
-The Keystone integration must support the current [OpenStack Identity
-API version 3](https://docs.openstack.org/api-ref/identity/v3/).
-
-It must be possible to serve S3 and Swift for the same object store
-pool.
-
-It must be possible to [obtain S3 credentials via OpenStack](
-https://docs.ceph.com/en/octopus/radosgw/keystone/#keystone-integration-with-the-s3-api).
-
-Any changes to the CRD must be future safe and cleanly allow extension
-to further technologies (such as LDAP authentication).
-
-### Non-Goals
-
-* Support for OpenStack Identity API versions below v3. API version v2 has long
-  been deprecated and [has been removed in the "queens" version of
-  Keystone](https://docs.openstack.org/keystone/xena/contributor/http-api.html)
-  which was released in 2018 and is now in extended maintenance mode
-  (which means it gets no more points releases and only sporadic bug
-  fixes/security fixes).
-
-* Authenticating Ceph RGW to Keystone via admin token (a.k.a. shared secret).
-  This is a deliberate choice as [admin tokens should not be used in production environments](
-  https://docs.openstack.org/keystone/rocky/admin/identity-bootstrap.html#using-a-shared-secret).
-
-* Support for APIs beside S3 and Swift.
-
-* Interaction of Swift with OBCs is out of scope of this document.  If
-  you need to access a bucket created by OBC via Swift you need to
-  create a separate `cephobjectstoreuser`, configure its access rights
-  to the bucket and use those credentials.
-
-* Support for authentication technologies other than Keystone (e.g. LDAP)
-
-* Exposing options that disable security features (e.g. TLS verification)
-
-## Proposal details
-
-The Object Store CRD will have to be extended to accommodate the new
-settings.
-
-### Keystone integration
-
-A new optional section `auth.keystone` is added to the Object Store
-CRD to configure the keystone integration:
-
-```yaml
-auth:
-  keystone:
-    url: https://keystone:5000/                     [1, 2]
-    acceptedRoles: ["_member_", "service", "admin"] [1, 2]
-    implicitTenants: swift                          [1]
-    tokenCacheSize: 1000                            [1]
-    revocationInterval: 1200                        [1]
-    serviceUserSecretName: rgw-service-user         [3, 2]
-```
-Annotations:
-* `[1]` These options map directly to [RGW configuration
-  options](https://docs.ceph.com/en/octopus/radosgw/config-ref/#keystone-settings),
-  the corresponding RGW option is formed by prefixing it with
-  `rgw_keystone_` and replacing upper case letters by their lower case
-  letter followed by an underscore. E.g. `tokenCacheSize` maps to
-  `rgw_keystone_token_cache_size`.
-* `[2]` These settings are required in the `keystone` section if
-  present.
-* `[1]` The name of the secret containing the credentials for the
-  service user account used by RGW. It has to be in the same namespace
-  as the object store resource.
-
-The `rgw_keystone_api_version` option is not exposed to the user, as
-only version 3 of the OpenStack Identity API is supported for now. If
-a newer version of the Openstack Identity should be released at some
-point, it will be easy to extend the CR to accommodate it.
-
-The certificate to verify the Keystone endpoint can't be explicitly
-configured in Ceph RGW. Instead, the system configuration of the pod
-running RGW is used. You can add to the system certificate store via
-the `gateway.caBundleRef` setting of the object store resource.
-
-The credentials for the Keystone service account used by Ceph RGW are
-supplied in a Secret that contains a mapping of OpenStack [openrc
-environment variables](https://docs.openstack.org/python-openstackclient/xena/cli/man/openstack.html#environment-variables).
-`password` is the only authentication type that is supported, this is
-a limitation of RGW which does not support other Keystone authentication
-types. Example:
-
-```yaml
-apiVersion: v1
-kind: Secret
-metadata:
-  name: rgw-service-user
-data:
-  OS_PASSWORD: "horse staples battery correct"
-  OS_USERNAME: "ceph-rgw"
-  OS_PROJECT_NAME: "admin"
-  OS_USER_DOMAIN_NAME: "Default"
-  OS_PROJECT_DOMAIN_NAME: "Default"
-  OS_AUTH_TYPE: "password"
-```
-
-This format is chosen because it is a natural and interoperable way
-that keystone credentials are represented and for the supported auth
-type it maps naturally to the Ceph RGW configuration.
-
-The following constraints must be fulfilled by the secret:
-* `OS_AUTH_TYPE` must be `password` or omitted.
-* `OS_USER_DOMAIN_NAME` must equal `OS_PROJECT_DOMAIN_NAME`. This is a
-  restriction of Ceph RGW, which does not support configuring separate
-  domains for the user and project.
-* All openrc variables not in the example above are ignored. The API
-  version (`OS_IDENTITY_API_VERSION`) is assumed to be `3` and
-  Keystone endpoint `OS_AUTH_URL` is taken from the
-  `keystone.apiVersion` configuration in the object store resource.
-
-The mapping to
-[RGW configuration options](https://docs.ceph.com/en/octopus/radosgw/config-ref/#keystone-settings)
-is done as follows:
-* `OS_USERNAME` -> `rgw_keystone_admin_user`
-* `OS_PROJECT_NAME` -> `rgw_keystone_admin_project`
-* `OS_PROJECT_DOMAIN_NAME`, `OS_USER_DOMAIN_NAME` -> `rgw_keystone_admin_domain`
-* `OS_PASSWORD` -> `rgw_keystone_admin_password`
-
-### Swift integration
-
-The currently ignored `gateway.type` option is deprecated and from now on
-explicitly ignored by rook.
-
-The other `gateway` settings are kept as they are: They do not directly
-relate to Swift or S3 but are common configuration of RGW.
-
-The Swift API is enabled and configured via a new `protocols` section:
-```yaml
-protocols:
-  swift:                      [1]
-    accountInUrl: true        [4]
-    urlPrefix: /example       [4]
-    versioningEnabled: false  [4]
-  s3:
-    enabled: false            [2]
-    authUseKeystone: true     [3]
-```
-Annotations:
-* `[1]` Swift will be enabled, if `protocols.swift` is present.
-* `[2]` This defaults to `true` (even if `protocols.s3` is not present
-  in the CRD). This maintains backwards compatibility – by default S3
-  is enabled.
-* `[3]` This option maps directly to the `rgw_s3_auth_use_keystone` option.
-  Enabling it allows generating S3 credentials via an OpenStack API call, see the
-  [docs](https://docs.ceph.com/en/octopus/radosgw/keystone/#keystone-integration-with-the-s3-api).
-   If not given, the defaults of the corresponding RGW option apply.
-* `[4]` These options map directly to [RGW configuration
-  options](https://docs.ceph.com/en/octopus/radosgw/config-ref/#swift-settings),
-  the corresponding RGW option is formed by prefixing it with
-  `rgw_swift_` and replacing upper case letters by their lower case
-  letter followed by an underscore. E.g. `urlPrefix` maps to
-  `rgw_swift_url_prefix`. They are optional. If not given, the defaults
-  of the corresponding RGW option apply.
-
-The access to the Swift API is granted by creating a subuser of an RGW
-user. While commonly the access is granted via projects
-mapped from Keystone, explicit creation of subusers is supported by
-extending the `cephobjectstoreuser` resource with a new optional section
-`spec.subUsers`:
-```yaml
-apiVersion: ceph.rook.io/v1
-kind: CephObjectStoreUser
-metadata:
-  name: my-user
-  namespace: rook-ceph
-spec:
-  store: my-store
-  displayName: my-display-name
-  quotas:
-    maxBuckets: 100
-    maxSize: 10G
-    maxObjects: 10000
-  capabilities:
-    user: "*"
-    bucket: "*"
-  subUsers:
-  - name: swift  [1]
-    access: full [2]
-```
-
-Annotations:
-* `[1]` This is the name of the subuser without the `username:` prefix
-  (see below for more explanation). The `name` must be unique within
-  the `subUsers` list.
-
-* `[2]` The possible values are: `read`, `write`, `readwrite`,
-  `full`. These values take their meanings from the possible values of
-  the `--access-level` option of `radosgw-admin subuser create`, as
-  documented in the [radosgw admin guide](
-  https://docs.ceph.com/en/octopus/radosgw/admin/#create-a-subuser).
-
-* `name` and `access` are required for each item in `subUsers`.
-
-When changing the subuser-configuration in the CR, this is reflected
-on the RGW side:
-* Subsers are deleted and created to match the list of subusers in the
-  resource.
-* If the access level for an existing user is changed no new
-  credentials are created, but the existing credentials are kept.
-* If a subuser is deleted the corresponding credential secret is
-  deleted as well.
-* Changing only the order of the subuser list does not trigger a
-  reconcile.
-
-The subusers are not mapped to a separate CR for the
-following reasons:
-
-* The full subuser names are prefixed with the username like
-  `my-user:subusername`, so being unique within the CR guarantees
-  global uniqueness.
-
-  Unlike `radosgw-admin` the subuser name in the CRD must be provided
-  *without the prefix* (radosgw-admin allows both, to omit or include
-  the prefix).
-
-* The subuser abstraction is very simple – only a name and an access
-  level can be configured, so a separate resource would not be
-  appropriate complexity wise.
-
-Like for the S3 access keys for the users, the swift keys created for the
-sub-users will be automatically injected into Secret objects. The
-credentials for the subusers are mapped to separate secrets, in the
-case of the example the following secret will be created:
-```yaml
-apiVersion:
-kind: Secret
-metadata:
-  name: rook-ceph-object-subuser-my-store-my-user-swift [1]
-  namespace: rook-ceph
-data:
-  SWIFT_USER: my-user:swift                             [2]
-  SWIFT_SECRET_KEY: $KEY                                [3]
-  SWIFT_AUTH_ENDPOINT: https://rgw.example:6000/auth    [4]
-```
-Annotations:
-* `[1]` The name is constructed by joining the following elements with dashes
-  (compare [the corresponding name of the secret for the object store users](
-  https://github.com/rook/rook/blob/376ca62f8ad07540d9ddffe9dc0ee53f4ac35e29/pkg/operator/ceph/object/user/controller.go#L416)):
-      - the literal `rook-ceph-object-subuser`
-      - the name of the object store resource
-      - the name of the user
-      - the name of the subuser (without `username:`-prefix)
-* `[2]` The full name of the subuser (including the `username:`-prefix).
-* `[3]` The generated swift access secret.
-* `[4]` The API endpoint for [swift auth](https://docs.ceph.com/en/octopus/radosgw/swift/auth/#auth-get).
-
-### Risks and Mitigation
-
-As long as the Object Store CRD changes are well thought out the
-overall risk is minimal.
-
-* If Swift is enabled by accident this could lead to an increased and
-  unexpected attack surface, especially since the authentication and
-  authorization for Swift and S3 may differ depending on how Ceph RGW
-  is configured. This is mitigated by requiring explicit configuration
-  to enable Swift.
-
-* A misconfigured Keystone integration may allow users to gain access
-  to objects they should not be authorized to access (e.g. if the
-  `rgw keystone accepted roles` setting is too broad). This will be
-  mitigated by proper documentation to make the operator aware of the
-  risk.
-
-* Ceph RGW allows to disable TLS verification when querying Keystone,
-  we deliberately choose not to expose this config option to the user.
-
-* The mapping from store, username and subuser-name to the name of the
-  secret with the credentials is not injective. This means that the
-  subusers of two different users may map to the same secret
-  (e.g. `user:a-b` and `user-a:b`).
-
-  This is potentially a vector for leaks of credentials to
-  unauthorized entities. A simple workaround is to avoid dashes in the
-  names of users and subuser managed by the CephObjectStoreUser CR.
-
-  Documenting the problem is deemed sufficient since a similar
-  problem already exists for the secret created for the users (in that
-  case for users from different object stores, e.g. the secret for
-  user `foo` in `my-store` collides with the one for user `store-foo`
-  in `my`).
-
-## Drawbacks
-
-* As shown in [#4754](https://github.com/rook/rook/issues/4754)
-  keystone can be integrated via config override so
-  it is not strictly necessary to support configuring it via the
-  Object Store CRD. Adding it to the CRD complicates things with
-  minor gain.
-
-## Alternatives
-
-* There is no workable alternative to extending the CRD for Swift
-  support.
-
-* Keystone support could be configured via Rook config override as shown
-  in [#4754](https://github.com/rook/rook/issues/4754).
-
-## Open Questions
diff --git a/pkg/apis/ceph.rook.io/v1/security.go b/pkg/apis/ceph.rook.io/v1/security.go
index 135c97e0b..01f678898 100644
--- a/pkg/apis/ceph.rook.io/v1/security.go
+++ b/pkg/apis/ceph.rook.io/v1/security.go
@@ -50,7 +50,7 @@ func (kms *KeyManagementServiceSpec) IsVaultKMS() bool {
 
 // IsIBMKeyProtectKMS return whether IBM Key Protect KMS is configured
 func (kms *KeyManagementServiceSpec) IsIBMKeyProtectKMS() bool {
-	return getParam(kms.ConnectionDetails, "KMS_PROVIDER") == "ibmkeyprotect"
+	return getParam(kms.ConnectionDetails, "KMS_PROVIDER") == secrets.TypeIBM
 }
 
 // IsTLSEnabled return KMS TLS details are configured
diff --git a/pkg/daemon/ceph/osd/kms/ibm_key_protect.go b/pkg/daemon/ceph/osd/kms/ibm_key_protect.go
index c673372f4..675a5380d 100644
--- a/pkg/daemon/ceph/osd/kms/ibm_key_protect.go
+++ b/pkg/daemon/ceph/osd/kms/ibm_key_protect.go
@@ -18,15 +18,15 @@ package kms
 
 import (
 	kp "github.com/IBM/keyprotect-go-client"
+	"github.com/libopenstorage/secrets"
 	"github.com/pkg/errors"
 )
 
 const (
-	TypeIBM = "ibmkeyprotect"
 	// IbmKeyProtectServiceApiKey is the IBM Key Protect service API key
 	IbmKeyProtectServiceApiKey = "IBM_KP_SERVICE_API_KEY"
 	// IbmKeyProtectInstanceIdKey is the IBM Key Protect instance id
-	IbmKeyProtectInstanceIdKey = "IBM_KP_SERVICE_INSTANCE_ID"
+	IbmKeyProtectInstanceIdKey = "IBM_KP_INSTANCE_ID"
 	// IbmKeyProtectBaseUrlKey is the IBM Key Protect base url
 	IbmKeyProtectBaseUrlKey = "IBM_KP_BASE_URL"
 	// IbmKeyProtectTokenUrlKey is the IBM Key Protect token url
@@ -39,7 +39,7 @@ var (
 	kmsIBMKeyProtectMandatoryConnectionDetails = []string{IbmKeyProtectInstanceIdKey, IbmKeyProtectServiceApiKey}
 	// ErrIbmServiceApiKeyNotSet is returned when IBM_KP_SERVICE_API_KEY is not set
 	ErrIbmServiceApiKeyNotSet = errors.Errorf("%s not set.", IbmKeyProtectServiceApiKey)
-	// ErrIbmInstanceIdKeyNotSet is returned when IBM_KP_SERVICE_INSTANCE_ID is not set
+	// ErrIbmInstanceIdKeyNotSet is returned when IBM_KP_INSTANCE_ID is not set
 	ErrIbmInstanceIdKeyNotSet = errors.Errorf("%s not set.", IbmKeyProtectInstanceIdKey)
 )
 
@@ -83,4 +83,4 @@ func InitKeyProtect(config map[string]string) (*kp.Client, error) {
 }
 
 // IsIBMKeyProtect determines whether the configured KMS is IBM Key Protect
-func (c *Config) IsIBMKeyProtect() bool { return c.Provider == TypeIBM }
+func (c *Config) IsIBMKeyProtect() bool { return c.Provider == secrets.TypeIBM }
diff --git a/pkg/daemon/ceph/osd/kms/kms.go b/pkg/daemon/ceph/osd/kms/kms.go
index cf1734284..ab005c2bf 100644
--- a/pkg/daemon/ceph/osd/kms/kms.go
+++ b/pkg/daemon/ceph/osd/kms/kms.go
@@ -65,8 +65,8 @@ func NewConfig(context *clusterd.Context, clusterSpec *cephv1.ClusterSpec, clust
 		config.Provider = secrets.TypeK8s
 	case secrets.TypeVault:
 		config.Provider = secrets.TypeVault
-	case TypeIBM:
-		config.Provider = TypeIBM
+	case secrets.TypeIBM:
+		config.Provider = secrets.TypeIBM
 	default:
 		logger.Errorf("unsupported kms type %q", Provider)
 	}
@@ -248,7 +248,7 @@ func ValidateConnectionDetails(ctx context.Context, clusterdContext *clusterd.Co
 				return errors.Wrap(err, "failed to set vault kms token to an env var")
 			}
 
-		case TypeIBM:
+		case secrets.TypeIBM:
 			for _, config := range kmsIBMKeyProtectMandatoryTokenDetails {
 				v, ok := kmsToken.Data[config]
 				if !ok || len(v) == 0 {
@@ -281,7 +281,7 @@ func ValidateConnectionDetails(ctx context.Context, clusterdContext *clusterd.Co
 			}
 		}
 
-	case TypeIBM:
+	case secrets.TypeIBM:
 		for _, config := range kmsIBMKeyProtectMandatoryConnectionDetails {
 			if GetParam(securitySpec.KeyManagementService.ConnectionDetails, config) == "" {
 				return errors.Errorf("failed to validate kms config %q. cannot be empty", config)
diff --git a/pkg/operator/ceph/cluster/cluster.go b/pkg/operator/ceph/cluster/cluster.go
index c9cf926f4..3a6ffdb6b 100755
--- a/pkg/operator/ceph/cluster/cluster.go
+++ b/pkg/operator/ceph/cluster/cluster.go
@@ -22,6 +22,7 @@ import (
 	"fmt"
 	"os/exec"
 	"path"
+	"sync"
 	"syscall"
 
 	"github.com/pkg/errors"
@@ -66,7 +67,7 @@ type clusterHealth struct {
 	internalCancel context.CancelFunc
 }
 
-func newCluster(c *cephv1.CephCluster, context *clusterd.Context, ownerInfo *k8sutil.OwnerInfo) *cluster {
+func newCluster(c *cephv1.CephCluster, context *clusterd.Context, csiMutex *sync.Mutex, ownerInfo *k8sutil.OwnerInfo) *cluster {
 	return &cluster{
 		// at this phase of the cluster creation process, the identity components of the cluster are
 		// not yet established. we reserve this struct which is filled in as soon as the cluster's
@@ -78,7 +79,7 @@ func newCluster(c *cephv1.CephCluster, context *clusterd.Context, ownerInfo *k8s
 		namespacedName:     types.NamespacedName{Namespace: c.Namespace, Name: c.Name},
 		monitoringRoutines: make(map[string]*clusterHealth),
 		ownerInfo:          ownerInfo,
-		mons:               mon.New(context, c.Namespace, c.Spec, ownerInfo),
+		mons:               mon.New(context, c.Namespace, c.Spec, ownerInfo, csiMutex),
 	}
 }
 
diff --git a/pkg/operator/ceph/cluster/cluster_external.go b/pkg/operator/ceph/cluster/cluster_external.go
index fb97a9adf..284d34a9a 100644
--- a/pkg/operator/ceph/cluster/cluster_external.go
+++ b/pkg/operator/ceph/cluster/cluster_external.go
@@ -111,7 +111,7 @@ func (c *ClusterController) configureExternalCephCluster(cluster *cluster) error
 	}
 
 	// Save CSI configmap
-	err = csi.SaveClusterConfig(c.context.Clientset, c.namespacedName.Namespace, cluster.ClusterInfo, &csi.CsiClusterConfigEntry{Monitors: csi.MonEndpoints(cluster.ClusterInfo.Monitors)})
+	err = csi.SaveClusterConfig(c.context.Clientset, c.namespacedName.Namespace, cluster.ClusterInfo, c.csiConfigMutex)
 	if err != nil {
 		return errors.Wrap(err, "failed to update csi cluster config")
 	}
diff --git a/pkg/operator/ceph/cluster/controller.go b/pkg/operator/ceph/cluster/controller.go
index f1fa42874..d232a821f 100644
--- a/pkg/operator/ceph/cluster/controller.go
+++ b/pkg/operator/ceph/cluster/controller.go
@@ -21,6 +21,7 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"sync"
 
 	"github.com/coreos/pkg/capnslog"
 	"github.com/pkg/errors"
@@ -84,6 +85,7 @@ type ClusterController struct {
 	context        *clusterd.Context
 	rookImage      string
 	clusterMap     map[string]*cluster
+	csiConfigMutex *sync.Mutex
 	osdChecker     *osd.OSDHealthMonitor
 	client         client.Client
 	namespacedName types.NamespacedName
@@ -325,9 +327,10 @@ func (r *ReconcileCephCluster) reconcileDelete(cephCluster *cephv1.CephCluster)
 // NewClusterController create controller for watching cluster custom resources created
 func NewClusterController(context *clusterd.Context, rookImage string) *ClusterController {
 	return &ClusterController{
-		context:    context,
-		rookImage:  rookImage,
-		clusterMap: make(map[string]*cluster),
+		context:        context,
+		rookImage:      rookImage,
+		clusterMap:     make(map[string]*cluster),
+		csiConfigMutex: &sync.Mutex{},
 	}
 }
 
@@ -340,7 +343,7 @@ func (c *ClusterController) reconcileCephCluster(clusterObj *cephv1.CephCluster,
 	cluster, ok := c.clusterMap[clusterObj.Namespace]
 	if !ok {
 		// It's a new cluster so let's populate the struct
-		cluster = newCluster(clusterObj, c.context, ownerInfo)
+		cluster = newCluster(clusterObj, c.context, c.csiConfigMutex, ownerInfo)
 	}
 	cluster.namespacedName = c.namespacedName
 
diff --git a/pkg/operator/ceph/cluster/mon/mon.go b/pkg/operator/ceph/cluster/mon/mon.go
index bf7988ba3..070ce2edc 100644
--- a/pkg/operator/ceph/cluster/mon/mon.go
+++ b/pkg/operator/ceph/cluster/mon/mon.go
@@ -120,6 +120,7 @@ type Cluster struct {
 	monTimeoutList     map[string]time.Time
 	mapping            *Mapping
 	ownerInfo          *k8sutil.OwnerInfo
+	csiConfigMutex     *sync.Mutex
 	isUpgrade          bool
 	arbiterMon         string
 }
@@ -163,7 +164,7 @@ type SchedulingResult struct {
 }
 
 // New creates an instance of a mon cluster
-func New(context *clusterd.Context, namespace string, spec cephv1.ClusterSpec, ownerInfo *k8sutil.OwnerInfo) *Cluster {
+func New(context *clusterd.Context, namespace string, spec cephv1.ClusterSpec, ownerInfo *k8sutil.OwnerInfo, csiConfigMutex *sync.Mutex) *Cluster {
 	return &Cluster{
 		context:        context,
 		spec:           spec,
@@ -174,7 +175,8 @@ func New(context *clusterd.Context, namespace string, spec cephv1.ClusterSpec, o
 		mapping: &Mapping{
 			Schedule: map[string]*MonScheduleInfo{},
 		},
-		ownerInfo: ownerInfo,
+		ownerInfo:      ownerInfo,
+		csiConfigMutex: csiConfigMutex,
 	}
 }
 
@@ -1040,7 +1042,7 @@ func (c *Cluster) saveMonConfig() error {
 		return errors.Wrap(err, "failed to write connection config for new mons")
 	}
 
-	if err := csi.SaveClusterConfig(c.context.Clientset, c.Namespace, c.ClusterInfo, &csi.CsiClusterConfigEntry{Monitors: csi.MonEndpoints(c.ClusterInfo.Monitors)}); err != nil {
+	if err := csi.SaveClusterConfig(c.context.Clientset, c.Namespace, c.ClusterInfo, c.csiConfigMutex); err != nil {
 		return errors.Wrap(err, "failed to update csi cluster config")
 	}
 
diff --git a/pkg/operator/ceph/controller/predicate.go b/pkg/operator/ceph/controller/predicate.go
index 7c2ed2011..77eeac7ec 100644
--- a/pkg/operator/ceph/controller/predicate.go
+++ b/pkg/operator/ceph/controller/predicate.go
@@ -745,8 +745,6 @@ func DuplicateCephClusters(ctx context.Context, c client.Client, object client.O
 		return true
 	}
 
-	logger.Debugf("found %d ceph clusters in namespace %q", len(cephClusterList.Items), object.GetNamespace())
-
 	// This check is needed when the operator is down and a cluster was created
 	if len(cephClusterList.Items) > 1 {
 		// Since multiple predicate are using this function we don't want all of them to log the
diff --git a/pkg/operator/ceph/csi/cluster_config.go b/pkg/operator/ceph/csi/cluster_config.go
index 3f4ad3b8c..03e76934f 100644
--- a/pkg/operator/ceph/csi/cluster_config.go
+++ b/pkg/operator/ceph/csi/cluster_config.go
@@ -33,22 +33,15 @@ import (
 )
 
 var (
-	logger      = capnslog.NewPackageLogger("github.com/rook/rook", "ceph-csi")
-	configMutex sync.Mutex
+	logger = capnslog.NewPackageLogger("github.com/rook/rook", "ceph-csi")
 )
 
-type CsiClusterConfigEntry struct {
-	ClusterID      string         `json:"clusterID"`
-	Monitors       []string       `json:"monitors"`
-	CephFS         *CsiCephFSSpec `json:"cephFS,omitempty"`
-	RadosNamespace string         `json:"radosNamespace,omitempty"`
+type csiClusterConfigEntry struct {
+	ClusterID string   `json:"clusterID"`
+	Monitors  []string `json:"monitors"`
 }
 
-type CsiCephFSSpec struct {
-	SubvolumeGroup string `json:"subvolumeGroup,omitempty"`
-}
-
-type csiClusterConfig []CsiClusterConfigEntry
+type csiClusterConfig []csiClusterConfigEntry
 
 // FormatCsiClusterConfig returns a json-formatted string containing
 // the cluster-to-mon mapping required to configure ceph csi.
@@ -86,7 +79,7 @@ func formatCsiClusterConfig(cc csiClusterConfig) (string, error) {
 	return string(ccJson), nil
 }
 
-func MonEndpoints(mons map[string]*cephclient.MonInfo) []string {
+func monEndpoints(mons map[string]*cephclient.MonInfo) []string {
 	endpoints := make([]string, 0)
 	for _, m := range mons {
 		endpoints = append(endpoints, m.Endpoint)
@@ -96,59 +89,32 @@ func MonEndpoints(mons map[string]*cephclient.MonInfo) []string {
 
 // updateCsiClusterConfig returns a json-formatted string containing
 // the cluster-to-mon mapping required to configure ceph csi.
-func updateCsiClusterConfig(curr, clusterKey string, newCsiClusterConfigEntry *CsiClusterConfigEntry) (string, error) {
+func updateCsiClusterConfig(
+	curr, clusterKey string, mons map[string]*cephclient.MonInfo) (string, error) {
+
 	var (
 		cc     csiClusterConfig
-		centry CsiClusterConfigEntry
+		centry csiClusterConfigEntry
 		found  bool
 	)
-
 	cc, err := parseCsiClusterConfig(curr)
 	if err != nil {
 		return "", errors.Wrap(err, "failed to parse current csi cluster config")
 	}
 
-	// Regardless of which controllers call updateCsiClusterConfig(), the values will be preserved since
-	// a lock is acquired for the update operation. So concurrent updates (rare event) will block and
-	// wait for the other update to complete. Monitors and Subvolumegroup will be updated
-	// independently and won't collide.
 	for i, centry := range cc {
 		if centry.ClusterID == clusterKey {
-			// If the new entry is nil, this means the entry is being deleted so remove it from the list
-			if newCsiClusterConfigEntry == nil {
-				cc = append(cc[:i], cc[i+1:]...)
-				found = true
-				break
-			}
-			centry.Monitors = newCsiClusterConfigEntry.Monitors
-			if newCsiClusterConfigEntry.CephFS != nil && newCsiClusterConfigEntry.CephFS.SubvolumeGroup != "" {
-				centry.CephFS = newCsiClusterConfigEntry.CephFS
-			}
-			if newCsiClusterConfigEntry.RadosNamespace != "" {
-				centry.RadosNamespace = newCsiClusterConfigEntry.RadosNamespace
-			}
+			centry.Monitors = monEndpoints(mons)
 			found = true
 			cc[i] = centry
 			break
 		}
 	}
 	if !found {
-		// If it's the first time we create the cluster, the entry does not exist, so the removal
-		// will fail with a dangling pointer
-		if newCsiClusterConfigEntry != nil {
-			centry.ClusterID = clusterKey
-			centry.Monitors = newCsiClusterConfigEntry.Monitors
-			// Add a condition not to fill with empty values
-			if newCsiClusterConfigEntry.CephFS != nil && newCsiClusterConfigEntry.CephFS.SubvolumeGroup != "" {
-				centry.CephFS = newCsiClusterConfigEntry.CephFS
-			}
-			if newCsiClusterConfigEntry.RadosNamespace != "" {
-				centry.RadosNamespace = newCsiClusterConfigEntry.RadosNamespace
-			}
-			cc = append(cc, centry)
-		}
+		centry.ClusterID = clusterKey
+		centry.Monitors = monEndpoints(mons)
+		cc = append(cc, centry)
 	}
-
 	return formatCsiClusterConfig(cc)
 }
 
@@ -189,24 +155,24 @@ func CreateCsiConfigMap(namespace string, clientset kubernetes.Interface, ownerI
 // value that is provided to ceph-csi uses in the storage class.
 // The locker l is typically a mutex and is used to prevent the config
 // map from being updated for multiple clusters simultaneously.
-func SaveClusterConfig(clientset kubernetes.Interface, clusterNamespace string, clusterInfo *cephclient.ClusterInfo, newCsiClusterConfigEntry *CsiClusterConfigEntry) error {
+func SaveClusterConfig(
+	clientset kubernetes.Interface, clusterNamespace string,
+	clusterInfo *cephclient.ClusterInfo, l sync.Locker) error {
 	if !CSIEnabled() {
-		logger.Debug("skipping csi config update because csi is not enabled")
 		return nil
 	}
-
-	configMutex.Lock()
-	defer configMutex.Unlock()
-
+	l.Lock()
+	defer l.Unlock()
 	// csi is deployed into the same namespace as the operator
 	csiNamespace := os.Getenv(k8sutil.PodNamespaceEnvVar)
 	if csiNamespace == "" {
-		return errors.Errorf("namespace value missing for %q", k8sutil.PodNamespaceEnvVar)
+		return errors.Errorf("namespace value missing for %s", k8sutil.PodNamespaceEnvVar)
 	}
-	logger.Debugf("using %q for csi configmap namespace", csiNamespace)
+	logger.Debugf("Using %+v for CSI ConfigMap Namespace", csiNamespace)
 
 	// fetch current ConfigMap contents
-	configMap, err := clientset.CoreV1().ConfigMaps(csiNamespace).Get(clusterInfo.Context, ConfigName, metav1.GetOptions{})
+	configMap, err := clientset.CoreV1().ConfigMaps(csiNamespace).Get(clusterInfo.Context,
+		ConfigName, metav1.GetOptions{})
 	if err != nil {
 		return errors.Wrap(err, "failed to fetch current csi config map")
 	}
@@ -216,8 +182,8 @@ func SaveClusterConfig(clientset kubernetes.Interface, clusterNamespace string,
 	if currData == "" {
 		currData = "[]"
 	}
-
-	newData, err := updateCsiClusterConfig(currData, clusterNamespace, newCsiClusterConfigEntry)
+	newData, err := updateCsiClusterConfig(
+		currData, clusterNamespace, clusterInfo.Monitors)
 	if err != nil {
 		return errors.Wrap(err, "failed to update csi config map data")
 	}
@@ -225,7 +191,7 @@ func SaveClusterConfig(clientset kubernetes.Interface, clusterNamespace string,
 
 	// update ConfigMap with new contents
 	if _, err := clientset.CoreV1().ConfigMaps(csiNamespace).Update(clusterInfo.Context, configMap, metav1.UpdateOptions{}); err != nil {
-		return errors.Wrap(err, "failed to update csi config map")
+		return errors.Wrapf(err, "failed to update csi config map")
 	}
 
 	return nil
diff --git a/pkg/operator/ceph/csi/controller.go b/pkg/operator/ceph/csi/controller.go
index a905f1ddd..8fd48c264 100644
--- a/pkg/operator/ceph/csi/controller.go
+++ b/pkg/operator/ceph/csi/controller.go
@@ -54,7 +54,7 @@ type ReconcileCSI struct {
 // Add creates a new Ceph CSI Controller and adds it to the Manager. The Manager will set fields on the Controller
 // and Start it when the Manager is Started.
 func Add(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) error {
-	return add(opManagerContext, mgr, newReconciler(mgr, context, opManagerContext, opConfig), opConfig)
+	return add(opManagerContext, mgr, newReconciler(mgr, context, opManagerContext, opConfig))
 }
 
 // newReconciler returns a new reconcile.Reconciler
@@ -67,7 +67,7 @@ func newReconciler(mgr manager.Manager, context *clusterd.Context, opManagerCont
 	}
 }
 
-func add(ctx context.Context, mgr manager.Manager, r reconcile.Reconciler, opConfig opcontroller.OperatorConfig) error {
+func add(ctx context.Context, mgr manager.Manager, r reconcile.Reconciler) error {
 	// Create a new controller
 	c, err := controller.New(controllerName, mgr, controller.Options{Reconciler: r})
 	if err != nil {
@@ -77,14 +77,14 @@ func add(ctx context.Context, mgr manager.Manager, r reconcile.Reconciler, opCon
 
 	// Watch for ConfigMap (operator config)
 	err = c.Watch(&source.Kind{
-		Type: &v1.ConfigMap{TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: v1.SchemeGroupVersion.String()}}}, &handler.EnqueueRequestForObject{}, predicateController(ctx, mgr.GetClient(), opConfig.OperatorNamespace))
+		Type: &v1.ConfigMap{TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: v1.SchemeGroupVersion.String()}}}, &handler.EnqueueRequestForObject{}, predicateController(ctx, mgr.GetClient()))
 	if err != nil {
 		return err
 	}
 
 	// Watch for CephCluster
 	err = c.Watch(&source.Kind{
-		Type: &cephv1.CephCluster{TypeMeta: metav1.TypeMeta{Kind: "CephCluster", APIVersion: v1.SchemeGroupVersion.String()}}}, &handler.EnqueueRequestForObject{}, predicateController(ctx, mgr.GetClient(), opConfig.OperatorNamespace))
+		Type: &cephv1.CephCluster{TypeMeta: metav1.TypeMeta{Kind: "CephCluster", APIVersion: v1.SchemeGroupVersion.String()}}}, &handler.EnqueueRequestForObject{}, predicateController(ctx, mgr.GetClient()))
 	if err != nil {
 		return err
 	}
diff --git a/pkg/operator/ceph/csi/predicate.go b/pkg/operator/ceph/csi/predicate.go
index bfe4055e8..c6b3c539d 100644
--- a/pkg/operator/ceph/csi/predicate.go
+++ b/pkg/operator/ceph/csi/predicate.go
@@ -22,18 +22,17 @@ import (
 
 	"github.com/google/go-cmp/cmp"
 	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
+	"github.com/rook/rook/pkg/operator/ceph/controller"
 	opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
 	v1 "k8s.io/api/core/v1"
-	kerrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/resource"
-	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/event"
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
 )
 
 // predicateController is the predicate function to trigger reconcile on operator configuration cm change
-func predicateController(ctx context.Context, c client.Client, opNamespace string) predicate.Funcs {
+func predicateController(ctx context.Context, c client.Client) predicate.Funcs {
 	return predicate.Funcs{
 		CreateFunc: func(e event.CreateEvent) bool {
 			// if the operator configuration file is created we want to reconcile
@@ -46,27 +45,15 @@ func predicateController(ctx context.Context, c client.Client, opNamespace strin
 			// If a Ceph Cluster is created we want to reconcile the csi driver
 			if cephCluster, ok := e.Object.(*cephv1.CephCluster); ok {
 				// If there are more than one ceph cluster in the same namespace do not reconcile
-				if opcontroller.DuplicateCephClusters(ctx, c, e.Object, false) {
+				if controller.DuplicateCephClusters(ctx, c, e.Object, false) {
 					return false
 				}
-
-				// We still have users that don't use the ConfigMap to configure the
-				// operator/csi and still rely on environment variables from the operator pod's
-				// spec. So we still want to reconcile the controller if the ConfigMap cannot be found.
-				err := c.Get(ctx, types.NamespacedName{Name: opcontroller.OperatorSettingConfigMapName, Namespace: opNamespace}, &v1.ConfigMap{})
-				if err != nil && kerrors.IsNotFound(err) {
-					logger.Debugf("could not find operator configuration ConfigMap, will reconcile the csi controller")
-					return true
-				}
-
 				// This allows us to avoid a double reconcile of the CSI controller if this is not
 				// the first generation of the CephCluster. So only return true if this is the very
 				// first instance of the CephCluster
 				// Corner case is when the cluster is created but the operator is down AND the cm
 				// does not exist... However, these days the operator config map is part of the
 				// operator.yaml so it's probably acceptable?
-				// This does not account for the case where the cephcluster is already deployed and
-				// the upgrade the operator or restart it. However, the CM check above should catch that
 				return cephCluster.Generation == 1
 			}
 
@@ -89,11 +76,6 @@ func predicateController(ctx context.Context, c client.Client, opNamespace strin
 		},
 
 		DeleteFunc: func(e event.DeleteEvent) bool {
-			// if the operator configuration file is deleted we want to reconcile to apply the
-			// configuration based on environment variables present in the operator's pod spec
-			if cm, ok := e.Object.(*v1.ConfigMap); ok {
-				return cm.Name == opcontroller.OperatorSettingConfigMapName
-			}
 			return false
 		},
 
diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go
index de394bef7..cad77a8f1 100644
--- a/pkg/operator/ceph/csi/spec.go
+++ b/pkg/operator/ceph/csi/spec.go
@@ -101,14 +101,14 @@ var (
 // manually challenging.
 var (
 	// image names
-	DefaultCSIPluginImage         = "quay.io/cephcsi/cephcsi:v3.5.1"
+	DefaultCSIPluginImage         = "quay.io/cephcsi/cephcsi:v3.5.0"
 	DefaultRegistrarImage         = "k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.4.0"
 	DefaultProvisionerImage       = "k8s.gcr.io/sig-storage/csi-provisioner:v3.1.0"
 	DefaultAttacherImage          = "k8s.gcr.io/sig-storage/csi-attacher:v3.4.0"
 	DefaultSnapshotterImage       = "k8s.gcr.io/sig-storage/csi-snapshotter:v4.2.0"
 	DefaultResizerImage           = "k8s.gcr.io/sig-storage/csi-resizer:v1.3.0"
 	DefaultVolumeReplicationImage = "quay.io/csiaddons/volumereplication-operator:v0.1.0"
-	DefaultCSIAddonsImage         = "quay.io/csiaddons/k8s-sidecar:v0.2.1"
+	DefaultCSIAddonsImage         = "quay.io/csiaddons/k8s-sidecar:v0.1.0"
 
 	// Local package template path for RBD
 	//go:embed template/rbd/csi-rbdplugin.yaml
diff --git a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml
index 9088d158d..b586ffb94 100644
--- a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml
+++ b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml
@@ -139,7 +139,7 @@ spec:
             - "--pod=$(POD_NAME)"
             - "--namespace=$(POD_NAMESPACE)"
             - "--pod-uid=$(POD_UID)"
-            - "--stagingpath={{ .KubeletDirPath }}/plugins/kubernetes.io/csi/"
+            - "--stagingpath={{ .KubeletDirPath }}/plugins/kubernetes.io/csi/pv/"
           ports:
             - containerPort: {{ .CSIAddonsPort }}
           env:
diff --git a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml
index e688e6af2..2b1ae4fcb 100644
--- a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml
+++ b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml
@@ -132,7 +132,7 @@ spec:
             - "--pod=$(POD_NAME)"
             - "--namespace=$(POD_NAMESPACE)"
             - "--pod-uid=$(POD_UID)"
-            - "--stagingpath={{ .KubeletDirPath }}/plugins/kubernetes.io/csi/"
+            - "--stagingpath={{ .KubeletDirPath }}/plugins/kubernetes.io/csi/pv/"
           ports:
             - containerPort: {{ .CSIAddonsPort }}
           env:
diff --git a/pkg/operator/ceph/file/subvolumegroup/controller.go b/pkg/operator/ceph/file/subvolumegroup/controller.go
index 9c9af8fa6..c65434723 100644
--- a/pkg/operator/ceph/file/subvolumegroup/controller.go
+++ b/pkg/operator/ceph/file/subvolumegroup/controller.go
@@ -40,9 +40,7 @@ import (
 	"github.com/rook/rook/pkg/clusterd"
 	"github.com/rook/rook/pkg/operator/ceph/cluster/mon"
 	opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
-	"github.com/rook/rook/pkg/operator/ceph/csi"
 	"github.com/rook/rook/pkg/operator/ceph/reporting"
-	"github.com/rook/rook/pkg/operator/k8sutil"
 	kerrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -177,18 +175,9 @@ func (r *ReconcileCephFilesystemSubVolumeGroup) reconcile(request reconcile.Requ
 		logger.Debugf("deleting subvolume group %q", cephFilesystemSubVolumeGroup.Name)
 		err := r.deleteSubVolumeGroup(cephFilesystemSubVolumeGroup)
 		if err != nil {
-			if strings.Contains(err.Error(), opcontroller.UninitializedCephConfigError) {
-				logger.Info(opcontroller.OperatorNotInitializedMessage)
-				return opcontroller.WaitForRequeueIfOperatorNotInitialized, nil
-			}
 			return reconcile.Result{}, errors.Wrapf(err, "failed to delete ceph ceph filesystem subvolume group %q", cephFilesystemSubVolumeGroup.Name)
 		}
 
-		err = csi.SaveClusterConfig(r.context.Clientset, buildClusterID(cephFilesystemSubVolumeGroup), r.clusterInfo, nil)
-		if err != nil {
-			return reconcile.Result{}, errors.Wrap(err, "failed to save cluster config")
-		}
-
 		// Remove finalizer
 		err = opcontroller.RemoveFinalizer(r.opManagerContext, r.client, cephFilesystemSubVolumeGroup)
 		if err != nil {
@@ -229,20 +218,6 @@ func (r *ReconcileCephFilesystemSubVolumeGroup) reconcile(request reconcile.Requ
 		return reconcile.Result{}, errors.Wrapf(err, "failed to create or update ceph filesystem subvolume group %q", cephFilesystemSubVolumeGroup.Name)
 	}
 
-	// Update CSI config map
-	// If the mon endpoints change, the mon health check go routine will take care of updating the
-	// config map, so no special care is needed in this controller
-	csiClusterConfigEntry := csi.CsiClusterConfigEntry{
-		Monitors: csi.MonEndpoints(r.clusterInfo.Monitors),
-		CephFS: &csi.CsiCephFSSpec{
-			SubvolumeGroup: cephFilesystemSubVolumeGroup.Name,
-		},
-	}
-	err = csi.SaveClusterConfig(r.context.Clientset, buildClusterID(cephFilesystemSubVolumeGroup), r.clusterInfo, &csiClusterConfigEntry)
-	if err != nil {
-		return reconcile.Result{}, errors.Wrap(err, "failed to save cluster config")
-	}
-
 	// Success! Let's update the status
 	r.updateStatus(r.client, request.NamespacedName, cephv1.ConditionReady)
 
@@ -266,17 +241,12 @@ func (r *ReconcileCephFilesystemSubVolumeGroup) createOrUpdateSubVolumeGroup(cep
 // Delete the ceph filesystem subvolume group
 func (r *ReconcileCephFilesystemSubVolumeGroup) deleteSubVolumeGroup(cephFilesystemSubVolumeGroup *cephv1.CephFilesystemSubVolumeGroup) error {
 	logger.Infof("deleting ceph filesystem subvolume group object %q", cephFilesystemSubVolumeGroup.Name)
-	if err := cephclient.DeleteCephFSSubVolumeGroup(r.context, r.clusterInfo, cephFilesystemSubVolumeGroup.Spec.FilesystemName, cephFilesystemSubVolumeGroup.Name); err != nil {
+	if err := cephclient.DeleteCephFSSubVolumeGroup(r.context, r.clusterInfo, cephFilesystemSubVolumeGroup.Name, cephFilesystemSubVolumeGroup.Spec.FilesystemName); err != nil {
 		code, ok := exec.ExitStatus(err)
-		// If the subvolumegroup does not exit, we should not return an error
-		if ok && code == int(syscall.ENOENT) {
-			logger.Debugf("ceph filesystem subvolume group %q do not exist", cephFilesystemSubVolumeGroup.Name)
-			return nil
-		}
 		// If the subvolumegroup has subvolumes the command will fail with:
 		// Error ENOTEMPTY: error in rmdir /volumes/csi
-		if ok && (code == int(syscall.ENOTEMPTY)) {
-			return errors.Wrapf(err, "failed to delete ceph filesystem subvolume group %q, remove the subvolumes first", cephFilesystemSubVolumeGroup.Name)
+		if ok && code != int(syscall.ENOTEMPTY) {
+			return errors.Wrapf(err, "failed to delete ceph filesystem subvolume group %q, remove the subvolumes first.", cephFilesystemSubVolumeGroup.Name)
 		}
 
 		return errors.Wrapf(err, "failed to delete ceph filesystem subvolume group %q", cephFilesystemSubVolumeGroup.Name)
@@ -302,15 +272,9 @@ func (r *ReconcileCephFilesystemSubVolumeGroup) updateStatus(client client.Clien
 	}
 
 	cephFilesystemSubVolumeGroup.Status.Phase = status
-	cephFilesystemSubVolumeGroup.Status.Info = map[string]string{"clusterID": buildClusterID(cephFilesystemSubVolumeGroup)}
 	if err := reporting.UpdateStatus(client, cephFilesystemSubVolumeGroup); err != nil {
 		logger.Errorf("failed to set ceph ceph filesystem subvolume group %q status to %q. %v", name, status, err)
 		return
 	}
 	logger.Debugf("ceph ceph filesystem subvolume group %q status updated to %q", name, status)
 }
-
-func buildClusterID(cephFilesystemSubVolumeGroup *cephv1.CephFilesystemSubVolumeGroup) string {
-	clusterID := fmt.Sprintf("%s-%s-file-%s", cephFilesystemSubVolumeGroup.Namespace, cephFilesystemSubVolumeGroup.Spec.FilesystemName, cephFilesystemSubVolumeGroup.Name)
-	return k8sutil.Hash(clusterID)
-}
diff --git a/pkg/operator/ceph/nfs/nfs.go b/pkg/operator/ceph/nfs/nfs.go
index 2da06a4b8..236140003 100644
--- a/pkg/operator/ceph/nfs/nfs.go
+++ b/pkg/operator/ceph/nfs/nfs.go
@@ -18,6 +18,7 @@ limitations under the License.
 package nfs
 
 import (
+	"crypto/sha256"
 	"fmt"
 
 	"github.com/banzaicloud/k8s-objectmatcher/patch"
@@ -214,7 +215,13 @@ func (r *ReconcileCephNFS) createConfigMap(n *cephv1.CephNFS, name string) (stri
 		}
 	}
 
-	return configMap.Name, k8sutil.Hash(fmt.Sprintf("%v", configMap.Data)), nil
+	h := sha256.New()
+	if _, err := h.Write([]byte(fmt.Sprintf("%v", configMap.Data))); err != nil {
+		return "", "", errors.Wrapf(err, "failed to get hash of ganesha config map")
+	}
+	configHash := fmt.Sprintf("%x", h.Sum(nil))
+
+	return configMap.Name, configHash, nil
 }
 
 // Down scale the ganesha server
diff --git a/pkg/operator/ceph/object/bucket/rgw-handlers.go b/pkg/operator/ceph/object/bucket/rgw-handlers.go
index 6fc13dd78..bcce7954e 100644
--- a/pkg/operator/ceph/object/bucket/rgw-handlers.go
+++ b/pkg/operator/ceph/object/bucket/rgw-handlers.go
@@ -101,16 +101,6 @@ func (p *Provisioner) deleteOBCResource(bucketName string) error {
 		} else if errors.Is(err, admin.ErrNoSuchBucket) {
 			// opinion: "not found" is not an error
 			logger.Infof("bucket %q does not exist", p.bucketName)
-		} else if errors.Is(err, admin.ErrNoSuchKey) {
-			// ceph might return NoSuchKey than NoSuchBucket when the target bucket does not exist.
-			// then we can use GetBucketInfo() to judge the existence of the bucket.
-			// see: https://github.com/ceph/ceph/pull/44413
-			_, err2 := p.adminOpsClient.GetBucketInfo(p.clusterInfo.Context, admin.Bucket{Bucket: bucketName, PurgeObject: &thePurge})
-			if errors.Is(err2, admin.ErrNoSuchBucket) {
-				logger.Infof("bucket info %q does not exist", p.bucketName)
-			} else {
-				return errors.Wrapf(err, "failed to delete bucket %q (could not get bucket info)", bucketName)
-			}
 		} else {
 			return errors.Wrapf(err, "failed to delete bucket %q", bucketName)
 		}
diff --git a/pkg/operator/ceph/object/notification/provisioner.go b/pkg/operator/ceph/object/notification/provisioner.go
index 5558085b4..0331dceae 100644
--- a/pkg/operator/ceph/object/notification/provisioner.go
+++ b/pkg/operator/ceph/object/notification/provisioner.go
@@ -19,7 +19,6 @@ package notification
 
 import (
 	"context"
-	"net/http"
 
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/ceph/go-ceph/rgw/admin"
@@ -44,26 +43,14 @@ type provisioner struct {
 	objectStoreName  types.NamespacedName
 }
 
-func getUserCredentials(opManagerContext context.Context, username string, objStore *cephv1.CephObjectStore, objContext *object.Context) (accessKey string, secretKey string, err error) {
+func getUserCredentials(adminOpsCtx *object.AdminOpsContext, opManagerContext context.Context, username string) (accessKey string, secretKey string, err error) {
 	if len(username) == 0 {
 		err = errors.New("no user name provided")
 		return
 	}
 
-	adminAccessKey, adminSecretKey, err := object.GetAdminOPSUserCredentials(objContext, &objStore.Spec)
-	if err != nil {
-		err = errors.Wrapf(err, "failed to get Ceph RGW admin ops user credentials when getting user %q", username)
-		return
-	}
-
-	adminOpsClient, err := admin.New(objContext.Endpoint, adminAccessKey, adminSecretKey, &http.Client{})
-	if err != nil {
-		err = errors.Wrapf(err, "failed to build admin ops API connection to get user %q", username)
-		return
-	}
-
 	var u admin.User
-	u, err = adminOpsClient.GetUser(opManagerContext, admin.User{ID: username})
+	u, err = adminOpsCtx.AdminOpsClient.GetUser(opManagerContext, admin.User{ID: username})
 	if err != nil {
 		err = errors.Wrapf(err, "failed to get ceph user %q", username)
 		return
@@ -88,12 +75,24 @@ func newS3Agent(p provisioner) (*object.S3Agent, error) {
 	// CephClusterSpec is needed for GetAdminOPSUserCredentials()
 	objContext.CephClusterSpec = *p.clusterSpec
 
-	accessKey, secretKey, err := getUserCredentials(p.opManagerContext, p.owner, objStore, objContext)
+	adminOpsCtx, err := object.NewMultisiteAdminOpsContext(objContext, &objStore.Spec)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to get admin Ops context for CephObjectStore %q", p.objectStoreName)
+
+	}
+	accessKey, secretKey, err := getUserCredentials(adminOpsCtx, p.opManagerContext, p.owner)
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to get owner credentials for %q", p.owner)
 	}
+	tlsCert := make([]byte, 0)
+	if objStore.Spec.IsTLSEnabled() {
+		tlsCert, _, err = object.GetTlsCaCert(objContext, &objStore.Spec)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to fetch TLS certificate for the object store")
+		}
+	}
 
-	return object.NewS3Agent(accessKey, secretKey, objContext.Endpoint, objContext.ZoneGroup, logger.LevelAt(capnslog.DEBUG), objContext.Context.KubeConfig.CertData)
+	return object.NewS3Agent(accessKey, secretKey, objContext.Endpoint, objContext.ZoneGroup, logger.LevelAt(capnslog.DEBUG), tlsCert)
 }
 
 // TODO: convert all rules without restrictions once the AWS SDK supports that
diff --git a/pkg/operator/ceph/object/topic/provisioner.go b/pkg/operator/ceph/object/topic/provisioner.go
index e6f9ea347..66f7124bb 100644
--- a/pkg/operator/ceph/object/topic/provisioner.go
+++ b/pkg/operator/ceph/object/topic/provisioner.go
@@ -88,7 +88,10 @@ func createSNSClient(p provisioner, objectStoreName types.NamespacedName) (*sns.
 	}
 	tlsEnabled := objStore.Spec.IsTLSEnabled()
 	if tlsEnabled {
-		tlsCert := objContext.Context.KubeConfig.CertData
+		tlsCert, _, err := object.GetTlsCaCert(objContext, &objStore.Spec)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to get TLS certificate for the object store")
+		}
 		if len(tlsCert) > 0 {
 			client.Transport = object.BuildTransportTLS(tlsCert, false)
 		}
@@ -101,6 +104,7 @@ func createSNSClient(p provisioner, objectStoreName types.NamespacedName) (*sns.
 			WithEndpoint(objContext.Endpoint).
 			WithMaxRetries(3).
 			WithDisableSSL(!tlsEnabled).
+			WithHTTPClient(&client).
 			WithLogLevel(logLevel),
 	)
 	if err != nil {


Test patch:
diff --git a/.github/workflows/canary-integration-test.yml b/.github/workflows/canary-integration-test.yml
index 9aa1c7d30..fc1ea7be8 100644
--- a/.github/workflows/canary-integration-test.yml
+++ b/.github/workflows/canary-integration-test.yml
@@ -838,12 +838,9 @@ jobs:
       with:
         fetch-depth: 0
 
-    - name: run encryption KMS IBM Key Protect
+    - name: run Encryption KMS IBM Key Protect
       uses: ./.github/workflows/encryption-pvc-kms-ibm-kp
-      if: "env.IBM_KP_SERVICE_INSTANCE_ID != '' && env.IBM_KP_SERVICE_API_KEY != ''"
-      env:
-        IBM_KP_SERVICE_INSTANCE_ID: ${{ secrets.IBM_INSTANCE_ID }}
-        IBM_KP_SERVICE_API_KEY: ${{ secrets.IBM_SERVICE_API_KEY }}
+      if: "env.IBM_KP_INSTANCE_ID != '' && env.IBM_KP_SERVICE_API_KEY != ''"
       with:
         ibm-instance-id: ${{ secrets.IBM_INSTANCE_ID }}
         ibm-service-api-key: ${{ secrets.IBM_SERVICE_API_KEY }}
diff --git a/pkg/daemon/ceph/client/config_test.go b/pkg/daemon/ceph/client/config_test.go
index 4d62fcc65..8528741ea 100644
--- a/pkg/daemon/ceph/client/config_test.go
+++ b/pkg/daemon/ceph/client/config_test.go
@@ -18,6 +18,8 @@ package client
 
 import (
 	"context"
+	"io/ioutil"
+	"os"
 	"path/filepath"
 	"strings"
 	"testing"
@@ -64,7 +66,11 @@ func TestCreateDefaultCephConfig(t *testing.T) {
 func TestGenerateConfigFile(t *testing.T) {
 	ctx := context.TODO()
 	// set up a temporary config directory that will be cleaned up after test
-	configDir := t.TempDir()
+	configDir, err := ioutil.TempDir("", "TestGenerateConfigFile")
+	if err != nil {
+		t.Fatalf("failed to create temp config dir: %+v", err)
+	}
+	defer os.RemoveAll(configDir)
 
 	// create mocked cluster context and info
 	clientset := test.New(t, 3)
@@ -84,7 +90,7 @@ func TestGenerateConfigFile(t *testing.T) {
 		},
 		Data: data,
 	}
-	_, err := clientset.CoreV1().ConfigMaps(ns).Create(ctx, cm, metav1.CreateOptions{})
+	_, err = clientset.CoreV1().ConfigMaps(ns).Create(ctx, cm, metav1.CreateOptions{})
 	assert.NoError(t, err)
 
 	clusterInfo := &ClusterInfo{
diff --git a/pkg/daemon/ceph/osd/device_test.go b/pkg/daemon/ceph/osd/device_test.go
index 44e7ea1c4..55c3f6a6a 100644
--- a/pkg/daemon/ceph/osd/device_test.go
+++ b/pkg/daemon/ceph/osd/device_test.go
@@ -17,6 +17,7 @@ package osd
 
 import (
 	"io/ioutil"
+	"os"
 	"path"
 	"strings"
 	"testing"
@@ -28,7 +29,8 @@ import (
 )
 
 func TestOSDBootstrap(t *testing.T) {
-	configDir := t.TempDir()
+	configDir, _ := ioutil.TempDir("", "")
+	defer os.RemoveAll(configDir)
 
 	executor := &exectest.MockExecutor{
 		MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) {
@@ -37,6 +39,7 @@ func TestOSDBootstrap(t *testing.T) {
 	}
 
 	context := &clusterd.Context{Executor: executor, ConfigDir: configDir}
+	defer os.RemoveAll(context.ConfigDir)
 	err := createOSDBootstrapKeyring(context, client.AdminTestClusterInfo("mycluster"), configDir)
 	assert.Nil(t, err)
 
diff --git a/pkg/daemon/ceph/osd/kms/envs_test.go b/pkg/daemon/ceph/osd/kms/envs_test.go
index a8c562f93..aafb8b312 100644
--- a/pkg/daemon/ceph/osd/kms/envs_test.go
+++ b/pkg/daemon/ceph/osd/kms/envs_test.go
@@ -63,7 +63,7 @@ func TestVaultTLSEnvVarFromSecret(t *testing.T) {
 	t.Run("ibm kp", func(t *testing.T) {
 		spec := cephv1.ClusterSpec{Security: cephv1.SecuritySpec{KeyManagementService: cephv1.KeyManagementServiceSpec{
 			TokenSecretName:   "ibm-kp-token",
-			ConnectionDetails: map[string]string{"KMS_PROVIDER": TypeIBM, "IBM_KP_SERVICE_INSTANCE_ID": "1"}}},
+			ConnectionDetails: map[string]string{"KMS_PROVIDER": "ibm-kp", "IBM_KP_INSTANCE_ID": "1"}}},
 		}
 		envVars := ConfigToEnvVar(spec)
 		areEnvVarsSorted := sort.SliceIsSorted(envVars, func(i, j int) bool {
@@ -71,8 +71,8 @@ func TestVaultTLSEnvVarFromSecret(t *testing.T) {
 		})
 		assert.True(t, areEnvVarsSorted)
 		assert.Equal(t, 3, len(envVars))
-		assert.Contains(t, envVars, v1.EnvVar{Name: "KMS_PROVIDER", Value: TypeIBM})
-		assert.Contains(t, envVars, v1.EnvVar{Name: "IBM_KP_SERVICE_INSTANCE_ID", Value: "1"})
+		assert.Contains(t, envVars, v1.EnvVar{Name: "KMS_PROVIDER", Value: "ibm-kp"})
+		assert.Contains(t, envVars, v1.EnvVar{Name: "IBM_KP_INSTANCE_ID", Value: "1"})
 		assert.Contains(t, envVars, v1.EnvVar{Name: "IBM_KP_SERVICE_API_KEY", ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: "ibm-kp-token"}, Key: "IBM_KP_SERVICE_API_KEY"}}})
 	})
 }
@@ -140,11 +140,11 @@ func TestVaultConfigToEnvVar(t *testing.T) {
 		},
 		{
 			"ibm kp - IBM_KP_SERVICE_API_KEY is removed from the details",
-			args{spec: cephv1.ClusterSpec{Security: cephv1.SecuritySpec{KeyManagementService: cephv1.KeyManagementServiceSpec{ConnectionDetails: map[string]string{"KMS_PROVIDER": TypeIBM, "IBM_KP_SERVICE_API_KEY": "foo", "IBM_KP_SERVICE_INSTANCE_ID": "1"}, TokenSecretName: "ibm-kp-token"}}}},
+			args{spec: cephv1.ClusterSpec{Security: cephv1.SecuritySpec{KeyManagementService: cephv1.KeyManagementServiceSpec{ConnectionDetails: map[string]string{"KMS_PROVIDER": "ibm-kp", "IBM_KP_SERVICE_API_KEY": "foo", "IBM_KP_INSTANCE_ID": "1"}, TokenSecretName: "ibm-kp-token"}}}},
 			[]v1.EnvVar{
+				{Name: "IBM_KP_INSTANCE_ID", Value: "1"},
 				{Name: "IBM_KP_SERVICE_API_KEY", ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: "ibm-kp-token"}, Key: "IBM_KP_SERVICE_API_KEY"}}},
-				{Name: "IBM_KP_SERVICE_INSTANCE_ID", Value: "1"},
-				{Name: "KMS_PROVIDER", Value: TypeIBM},
+				{Name: "KMS_PROVIDER", Value: "ibm-kp"},
 			},
 		},
 	}
diff --git a/pkg/daemon/ceph/osd/kms/ibm_key_protect_test.go b/pkg/daemon/ceph/osd/kms/ibm_key_protect_test.go
index aeeaeb948..5a19b164a 100644
--- a/pkg/daemon/ceph/osd/kms/ibm_key_protect_test.go
+++ b/pkg/daemon/ceph/osd/kms/ibm_key_protect_test.go
@@ -35,7 +35,7 @@ func TestInitKeyProtect(t *testing.T) {
 		config[IbmKeyProtectServiceApiKey] = "foo"
 	})
 
-	t.Run("IBM_KP_SERVICE_INSTANCE_ID not set", func(t *testing.T) {
+	t.Run("IBM_KP_INSTANCE_ID not set", func(t *testing.T) {
 		_, err := InitKeyProtect(config)
 		assert.Error(t, err)
 		assert.ErrorIs(t, err, ErrIbmInstanceIdKeyNotSet)
diff --git a/pkg/daemon/ceph/osd/kms/kms_test.go b/pkg/daemon/ceph/osd/kms/kms_test.go
index 459d8718b..180f9b32b 100644
--- a/pkg/daemon/ceph/osd/kms/kms_test.go
+++ b/pkg/daemon/ceph/osd/kms/kms_test.go
@@ -58,7 +58,7 @@ func TestValidateConnectionDetails(t *testing.T) {
 	}
 	ibmSecuritySpec := &cephv1.SecuritySpec{KeyManagementService: cephv1.KeyManagementServiceSpec{
 		ConnectionDetails: map[string]string{
-			"KMS_PROVIDER": TypeIBM,
+			"KMS_PROVIDER": "ibm-kp",
 		},
 	}}
 
@@ -183,8 +183,8 @@ func TestValidateConnectionDetails(t *testing.T) {
 		assert.NoError(t, err)
 		err = ValidateConnectionDetails(ctx, context, ibmSecuritySpec, ns)
 		assert.Error(t, err, "")
-		assert.EqualError(t, err, "failed to validate kms config \"IBM_KP_SERVICE_INSTANCE_ID\". cannot be empty")
-		ibmSecuritySpec.KeyManagementService.ConnectionDetails["IBM_KP_SERVICE_INSTANCE_ID"] = "foo"
+		assert.EqualError(t, err, "failed to validate kms config \"IBM_KP_INSTANCE_ID\". cannot be empty")
+		ibmSecuritySpec.KeyManagementService.ConnectionDetails["IBM_KP_INSTANCE_ID"] = "foo"
 	})
 
 	t.Run("ibm kp - success", func(t *testing.T) {
diff --git a/pkg/daemon/ceph/osd/volume_test.go b/pkg/daemon/ceph/osd/volume_test.go
index 34d32b4b4..d77f57a21 100644
--- a/pkg/daemon/ceph/osd/volume_test.go
+++ b/pkg/daemon/ceph/osd/volume_test.go
@@ -324,8 +324,12 @@ func TestConfigureCVDevices(t *testing.T) {
 	}()
 
 	originalCephConfigDir := cephConfigDir
-	cephConfigDir = t.TempDir()
+	cephConfigDir, err = ioutil.TempDir("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
 	defer func() {
+		os.RemoveAll(cephConfigDir)
 		cephConfigDir = originalCephConfigDir
 	}()
 
diff --git a/pkg/daemon/util/copybins_test.go b/pkg/daemon/util/copybins_test.go
index eeec30362..ca85855ee 100644
--- a/pkg/daemon/util/copybins_test.go
+++ b/pkg/daemon/util/copybins_test.go
@@ -58,13 +58,15 @@ func TestCopyBinary(t *testing.T) {
 	// testRootDir/
 	//   bin/
 	//   copy-to-dir/
-	testRootDir := t.TempDir()
+	testRootDir, err := ioutil.TempDir("", "rook-cmd-reporter-copy-binaries-test")
+	assert.NoError(t, err)
+	defer os.RemoveAll(testRootDir)
 	// create a test PATH="testRootDir/bin"
 	envPath := path.Join(testRootDir, "bin")
 	mkdir(envPath)
 	oPath := os.Getenv("PATH")
 	defer os.Setenv("PATH", oPath)
-	err := os.Setenv("PATH", envPath)
+	err = os.Setenv("PATH", envPath)
 	assert.NoError(t, err)
 	// create initial copy-to-dir
 	copyToDir := path.Join(testRootDir, "copy-to-dir")
diff --git a/pkg/operator/ceph/cluster/mgr/mgr_test.go b/pkg/operator/ceph/cluster/mgr/mgr_test.go
index 3eb562427..b846c2f26 100644
--- a/pkg/operator/ceph/cluster/mgr/mgr_test.go
+++ b/pkg/operator/ceph/cluster/mgr/mgr_test.go
@@ -19,6 +19,7 @@ package mgr
 import (
 	"context"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"testing"
 	"time"
@@ -64,7 +65,7 @@ func TestStartMgr(t *testing.T) {
 	}
 
 	clientset := testop.New(t, 3)
-	configDir := t.TempDir()
+	configDir, _ := ioutil.TempDir("", "")
 	scheme := scheme.Scheme
 	err := policyv1.AddToScheme(scheme)
 	assert.NoError(t, err)
@@ -72,6 +73,7 @@ func TestStartMgr(t *testing.T) {
 	assert.NoError(t, err)
 	cl := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects().Build()
 
+	defer os.RemoveAll(configDir)
 	ctx := &clusterd.Context{
 		Executor:  executor,
 		ConfigDir: configDir,
@@ -255,7 +257,8 @@ func TestMgrSidecarReconcile(t *testing.T) {
 		},
 	}
 	clientset := testop.New(t, 3)
-	configDir := t.TempDir()
+	configDir, _ := ioutil.TempDir("", "")
+	defer os.RemoveAll(configDir)
 	ctx := &clusterd.Context{
 		Executor:  executor,
 		ConfigDir: configDir,
diff --git a/pkg/operator/ceph/cluster/mon/drain_test.go b/pkg/operator/ceph/cluster/mon/drain_test.go
index 2ff565d21..f64e1f6e7 100644
--- a/pkg/operator/ceph/cluster/mon/drain_test.go
+++ b/pkg/operator/ceph/cluster/mon/drain_test.go
@@ -18,6 +18,7 @@ package mon
 
 import (
 	"context"
+	"sync"
 	"testing"
 
 	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
@@ -47,7 +48,7 @@ func createFakeCluster(t *testing.T, cephClusterObj *cephv1.CephCluster, k8sVers
 
 	cl := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects().Build()
 	clientset := test.New(t, 3)
-	c := New(&clusterd.Context{Client: cl, Clientset: clientset}, mockNamespace, cephClusterObj.Spec, ownerInfo)
+	c := New(&clusterd.Context{Client: cl, Clientset: clientset}, mockNamespace, cephClusterObj.Spec, ownerInfo, &sync.Mutex{})
 	test.SetFakeKubernetesVersion(clientset, k8sVersion)
 	return c
 }
diff --git a/pkg/operator/ceph/cluster/mon/health_test.go b/pkg/operator/ceph/cluster/mon/health_test.go
index 0f16cff07..f27a1a75b 100644
--- a/pkg/operator/ceph/cluster/mon/health_test.go
+++ b/pkg/operator/ceph/cluster/mon/health_test.go
@@ -19,8 +19,10 @@ package mon
 import (
 	"context"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"reflect"
+	"sync"
 	"testing"
 	"time"
 
@@ -56,14 +58,15 @@ func TestCheckHealth(t *testing.T) {
 		},
 	}
 	clientset := test.New(t, 1)
-	configDir := t.TempDir()
+	configDir, _ := ioutil.TempDir("", "")
+	defer os.RemoveAll(configDir)
 	context := &clusterd.Context{
 		Clientset: clientset,
 		ConfigDir: configDir,
 		Executor:  executor,
 	}
 	ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef()
-	c := New(context, "ns", cephv1.ClusterSpec{}, ownerInfo)
+	c := New(context, "ns", cephv1.ClusterSpec{}, ownerInfo, &sync.Mutex{})
 	// clusterInfo is nil so we return err
 	err := c.checkHealth(ctx)
 	assert.NotNil(t, err)
@@ -76,6 +79,7 @@ func TestCheckHealth(t *testing.T) {
 	c.spec.Mon.Count = 3
 	logger.Infof("initial mons: %v", c.ClusterInfo.Monitors)
 	c.waitForStart = false
+	defer os.RemoveAll(c.context.ConfigDir)
 
 	c.mapping.Schedule["f"] = &MonScheduleInfo{
 		Name:    "node0",
@@ -151,7 +155,7 @@ func TestCheckHealth(t *testing.T) {
 }
 
 func TestSkipMonFailover(t *testing.T) {
-	c := New(&clusterd.Context{}, "ns", cephv1.ClusterSpec{}, nil)
+	c := New(&clusterd.Context{}, "ns", cephv1.ClusterSpec{}, nil, nil)
 	c.ClusterInfo = clienttest.CreateTestClusterInfo(1)
 	monName := "arb"
 
@@ -187,7 +191,8 @@ func TestSkipMonFailover(t *testing.T) {
 func TestEvictMonOnSameNode(t *testing.T) {
 	ctx := context.TODO()
 	clientset := test.New(t, 1)
-	configDir := t.TempDir()
+	configDir, _ := ioutil.TempDir("", "")
+	defer os.RemoveAll(configDir)
 	executor := &exectest.MockExecutor{
 		MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) {
 			logger.Infof("executing command: %s %+v", command, args)
@@ -196,7 +201,7 @@ func TestEvictMonOnSameNode(t *testing.T) {
 	}
 	context := &clusterd.Context{Clientset: clientset, ConfigDir: configDir, Executor: executor}
 	ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef()
-	c := New(context, "ns", cephv1.ClusterSpec{}, ownerInfo)
+	c := New(context, "ns", cephv1.ClusterSpec{}, ownerInfo, &sync.Mutex{})
 	setCommonMonProperties(c, 1, cephv1.MonSpec{Count: 0}, "myversion")
 	c.maxMonID = 2
 	c.waitForStart = false
@@ -251,7 +256,7 @@ func TestScaleMonDeployment(t *testing.T) {
 	clientset := test.New(t, 1)
 	context := &clusterd.Context{Clientset: clientset}
 	ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef()
-	c := New(context, "ns", cephv1.ClusterSpec{}, ownerInfo)
+	c := New(context, "ns", cephv1.ClusterSpec{}, ownerInfo, &sync.Mutex{})
 	setCommonMonProperties(c, 1, cephv1.MonSpec{Count: 0, AllowMultiplePerNode: true}, "myversion")
 
 	name := "a"
@@ -294,16 +299,18 @@ func TestCheckHealthNotFound(t *testing.T) {
 		},
 	}
 	clientset := test.New(t, 1)
-	configDir := t.TempDir()
+	configDir, _ := ioutil.TempDir("", "")
+	defer os.RemoveAll(configDir)
 	context := &clusterd.Context{
 		Clientset: clientset,
 		ConfigDir: configDir,
 		Executor:  executor,
 	}
 	ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef()
-	c := New(context, "ns", cephv1.ClusterSpec{}, ownerInfo)
+	c := New(context, "ns", cephv1.ClusterSpec{}, ownerInfo, &sync.Mutex{})
 	setCommonMonProperties(c, 2, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "myversion")
 	c.waitForStart = false
+	defer os.RemoveAll(c.context.ConfigDir)
 
 	c.mapping.Schedule["a"] = &MonScheduleInfo{
 		Name: "node0",
@@ -355,17 +362,19 @@ func TestAddRemoveMons(t *testing.T) {
 		},
 	}
 	clientset := test.New(t, 1)
-	configDir := t.TempDir()
+	configDir, _ := ioutil.TempDir("", "")
+	defer os.RemoveAll(configDir)
 	context := &clusterd.Context{
 		Clientset: clientset,
 		ConfigDir: configDir,
 		Executor:  executor,
 	}
 	ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef()
-	c := New(context, "ns", cephv1.ClusterSpec{}, ownerInfo)
+	c := New(context, "ns", cephv1.ClusterSpec{}, ownerInfo, &sync.Mutex{})
 	setCommonMonProperties(c, 0, cephv1.MonSpec{Count: 5, AllowMultiplePerNode: true}, "myversion")
 	c.maxMonID = 0 // "a" is max mon id
 	c.waitForStart = false
+	defer os.RemoveAll(c.context.ConfigDir)
 
 	// checking the health will increase the mons as desired all in one go
 	err := c.checkHealth(ctx)
diff --git a/pkg/operator/ceph/cluster/mon/mon_test.go b/pkg/operator/ceph/cluster/mon/mon_test.go
index 068c77cea..14e6f37db 100644
--- a/pkg/operator/ceph/cluster/mon/mon_test.go
+++ b/pkg/operator/ceph/cluster/mon/mon_test.go
@@ -19,11 +19,13 @@ package mon
 import (
 	"context"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"path"
 	"reflect"
 	"strconv"
 	"strings"
+	"sync"
 	"testing"
 	"time"
 
@@ -79,7 +81,8 @@ func newTestStartCluster(t *testing.T, namespace string) (*clusterd.Context, err
 
 func newTestStartClusterWithQuorumResponse(t *testing.T, namespace string, monResponse func() (string, error)) (*clusterd.Context, error) {
 	clientset := test.New(t, 3)
-	configDir := t.TempDir()
+	configDir, _ := ioutil.TempDir("", "")
+	defer os.RemoveAll(configDir)
 	executor := &exectest.MockExecutor{
 		MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) {
 			if strings.Contains(command, "ceph-authtool") {
@@ -270,7 +273,7 @@ func validateStart(t *testing.T, c *Cluster) {
 func TestPersistMons(t *testing.T) {
 	clientset := test.New(t, 1)
 	ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef()
-	c := New(&clusterd.Context{Clientset: clientset}, "ns", cephv1.ClusterSpec{Annotations: cephv1.AnnotationsSpec{cephv1.KeyClusterMetadata: cephv1.Annotations{"key": "value"}}}, ownerInfo)
+	c := New(&clusterd.Context{Clientset: clientset}, "ns", cephv1.ClusterSpec{Annotations: cephv1.AnnotationsSpec{cephv1.KeyClusterMetadata: cephv1.Annotations{"key": "value"}}}, ownerInfo, &sync.Mutex{})
 	setCommonMonProperties(c, 1, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "myversion")
 
 	// Persist mon a
@@ -296,9 +299,10 @@ func TestPersistMons(t *testing.T) {
 func TestSaveMonEndpoints(t *testing.T) {
 	ctx := context.TODO()
 	clientset := test.New(t, 1)
-	configDir := t.TempDir()
+	configDir, _ := ioutil.TempDir("", "")
+	defer os.RemoveAll(configDir)
 	ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef()
-	c := New(&clusterd.Context{Clientset: clientset, ConfigDir: configDir}, "ns", cephv1.ClusterSpec{}, ownerInfo)
+	c := New(&clusterd.Context{Clientset: clientset, ConfigDir: configDir}, "ns", cephv1.ClusterSpec{}, ownerInfo, &sync.Mutex{})
 	setCommonMonProperties(c, 1, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "myversion")
 
 	// create the initial config map
@@ -343,9 +347,10 @@ func TestSaveMonEndpoints(t *testing.T) {
 
 func TestMaxMonID(t *testing.T) {
 	clientset := test.New(t, 1)
-	configDir := t.TempDir()
+	configDir, _ := ioutil.TempDir("", "")
+	defer os.RemoveAll(configDir)
 	ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef()
-	c := New(&clusterd.Context{Clientset: clientset, ConfigDir: configDir}, "ns", cephv1.ClusterSpec{}, ownerInfo)
+	c := New(&clusterd.Context{Clientset: clientset, ConfigDir: configDir}, "ns", cephv1.ClusterSpec{}, ownerInfo, &sync.Mutex{})
 	c.ClusterInfo = clienttest.CreateTestClusterInfo(1)
 
 	// when the configmap is not found, the maxMonID is -1
diff --git a/pkg/operator/ceph/cluster/mon/node_test.go b/pkg/operator/ceph/cluster/mon/node_test.go
index 98bf7fbe8..8fa24cf59 100644
--- a/pkg/operator/ceph/cluster/mon/node_test.go
+++ b/pkg/operator/ceph/cluster/mon/node_test.go
@@ -19,6 +19,7 @@ package mon
 import (
 	"context"
 	"strings"
+	"sync"
 	"testing"
 
 	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
@@ -36,7 +37,7 @@ import (
 func TestNodeAffinity(t *testing.T) {
 	ctx := context.TODO()
 	clientset := test.New(t, 4)
-	c := New(&clusterd.Context{Clientset: clientset}, "ns", cephv1.ClusterSpec{}, &k8sutil.OwnerInfo{})
+	c := New(&clusterd.Context{Clientset: clientset}, "ns", cephv1.ClusterSpec{}, &k8sutil.OwnerInfo{}, &sync.Mutex{})
 	setCommonMonProperties(c, 0, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "myversion")
 
 	c.spec.Placement = map[cephv1.KeyType]cephv1.Placement{}
@@ -166,7 +167,7 @@ func TestPodMemory(t *testing.T) {
 
 func TestHostNetwork(t *testing.T) {
 	clientset := test.New(t, 3)
-	c := New(&clusterd.Context{Clientset: clientset}, "ns", cephv1.ClusterSpec{}, &k8sutil.OwnerInfo{})
+	c := New(&clusterd.Context{Clientset: clientset}, "ns", cephv1.ClusterSpec{}, &k8sutil.OwnerInfo{}, &sync.Mutex{})
 	setCommonMonProperties(c, 0, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "myversion")
 
 	c.spec.Network.HostNetwork = true
diff --git a/pkg/operator/ceph/cluster/mon/service_test.go b/pkg/operator/ceph/cluster/mon/service_test.go
index c8cf31e36..62b5dc574 100644
--- a/pkg/operator/ceph/cluster/mon/service_test.go
+++ b/pkg/operator/ceph/cluster/mon/service_test.go
@@ -18,6 +18,7 @@ package mon
 
 import (
 	"context"
+	"sync"
 	"testing"
 
 	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
@@ -32,7 +33,7 @@ import (
 func TestCreateService(t *testing.T) {
 	ctx := context.TODO()
 	clientset := test.New(t, 1)
-	c := New(&clusterd.Context{Clientset: clientset}, "ns", cephv1.ClusterSpec{}, &k8sutil.OwnerInfo{})
+	c := New(&clusterd.Context{Clientset: clientset}, "ns", cephv1.ClusterSpec{}, &k8sutil.OwnerInfo{}, &sync.Mutex{})
 	c.ClusterInfo = client.AdminTestClusterInfo("rook-ceph")
 	m := &monConfig{ResourceName: "rook-ceph-mon-b", DaemonName: "b"}
 	clusterIP, err := c.createService(m)
diff --git a/pkg/operator/ceph/cluster/mon/spec_test.go b/pkg/operator/ceph/cluster/mon/spec_test.go
index c1f58ad9f..2b213e2be 100644
--- a/pkg/operator/ceph/cluster/mon/spec_test.go
+++ b/pkg/operator/ceph/cluster/mon/spec_test.go
@@ -17,6 +17,7 @@ limitations under the License.
 package mon
 
 import (
+	"sync"
 	"testing"
 
 	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
@@ -47,6 +48,7 @@ func testPodSpec(t *testing.T, monID string, pvc bool) {
 		"ns",
 		cephv1.ClusterSpec{},
 		ownerInfo,
+		&sync.Mutex{},
 	)
 	setCommonMonProperties(c, 0, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "rook/rook:myversion")
 	c.spec.CephVersion = cephv1.CephVersionSpec{Image: "quay.io/ceph/ceph:myceph"}
@@ -96,6 +98,7 @@ func TestDeploymentPVCSpec(t *testing.T) {
 		"ns",
 		cephv1.ClusterSpec{},
 		ownerInfo,
+		&sync.Mutex{},
 	)
 	setCommonMonProperties(c, 0, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "rook/rook:myversion")
 	c.spec.CephVersion = cephv1.CephVersionSpec{Image: "quay.io/ceph/ceph:myceph"}
@@ -155,6 +158,7 @@ func testRequiredDuringScheduling(t *testing.T, hostNetwork, allowMultiplePerNod
 		"ns",
 		cephv1.ClusterSpec{},
 		&k8sutil.OwnerInfo{},
+		&sync.Mutex{},
 	)
 
 	c.spec.Network.HostNetwork = hostNetwork
diff --git a/pkg/operator/ceph/csi/cluster_config_test.go b/pkg/operator/ceph/csi/cluster_config_test.go
index 08c9e1cad..afa53e074 100644
--- a/pkg/operator/ceph/csi/cluster_config_test.go
+++ b/pkg/operator/ceph/csi/cluster_config_test.go
@@ -19,142 +19,71 @@ package csi
 import (
 	"testing"
 
+	cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
 	"github.com/stretchr/testify/assert"
 )
 
 func TestUpdateCsiClusterConfig(t *testing.T) {
-	csiClusterConfigEntry := CsiClusterConfigEntry{
-		Monitors: []string{"1.2.3.4:5000"},
+	// initialize an empty list & add a simple mons list
+	mons := map[string]*cephclient.MonInfo{
+		"foo": {Name: "foo", Endpoint: "1.2.3.4:5000"},
 	}
-	csiClusterConfigEntry2 := CsiClusterConfigEntry{
-		Monitors: []string{"20.1.1.1:5000", "20.1.1.2:5000", "20.1.1.3:5000"},
+	s, err := updateCsiClusterConfig("[]", "alpha", mons)
+	assert.NoError(t, err)
+	assert.Equal(t, s,
+		`[{"clusterID":"alpha","monitors":["1.2.3.4:5000"]}]`)
+
+	// add a 2nd mon to the current cluster
+	mons["bar"] = &cephclient.MonInfo{
+		Name: "bar", Endpoint: "10.11.12.13:5000"}
+	s, err = updateCsiClusterConfig(s, "alpha", mons)
+	assert.NoError(t, err)
+	cc, err := parseCsiClusterConfig(s)
+	assert.NoError(t, err)
+	assert.Equal(t, len(cc), 1)
+	assert.Equal(t, cc[0].ClusterID, "alpha")
+	assert.Contains(t, cc[0].Monitors, "1.2.3.4:5000")
+	assert.Contains(t, cc[0].Monitors, "10.11.12.13:5000")
+	assert.Equal(t, len(cc[0].Monitors), 2)
+
+	// add a 2nd cluster with 3 mons
+	mons2 := map[string]*cephclient.MonInfo{
+		"flim": {Name: "flim", Endpoint: "20.1.1.1:5000"},
+		"flam": {Name: "flam", Endpoint: "20.1.1.2:5000"},
+		"blam": {Name: "blam", Endpoint: "20.1.1.3:5000"},
 	}
-	csiClusterConfigEntry3 := CsiClusterConfigEntry{
-		Monitors: []string{"10.1.1.1:5000", "10.1.1.2:5000", "10.1.1.3:5000"},
-		CephFS: &CsiCephFSSpec{
-			SubvolumeGroup: "mygroup",
-		},
-	}
-	var s string
-	var err error
-
-	t.Run("add a simple mons list", func(t *testing.T) {
-		s, err = updateCsiClusterConfig("[]", "alpha", &csiClusterConfigEntry)
-		assert.NoError(t, err)
-		assert.Equal(t, s,
-			`[{"clusterID":"alpha","monitors":["1.2.3.4:5000"]}]`)
-	})
-
-	t.Run("add a 2nd mon to the current cluster", func(t *testing.T) {
-		csiClusterConfigEntry.Monitors = append(csiClusterConfigEntry.Monitors, "10.11.12.13:5000")
-		s, err = updateCsiClusterConfig(s, "alpha", &csiClusterConfigEntry)
-		assert.NoError(t, err)
-		cc, err := parseCsiClusterConfig(s)
-		assert.NoError(t, err)
-		assert.Equal(t, 1, len(cc))
-		assert.Equal(t, "alpha", cc[0].ClusterID)
-		assert.Contains(t, cc[0].Monitors, "1.2.3.4:5000")
-		assert.Contains(t, cc[0].Monitors, "10.11.12.13:5000")
-		assert.Equal(t, 2, len(cc[0].Monitors))
-	})
-
-	t.Run("add a 2nd cluster with 3 mons", func(t *testing.T) {
-		s, err = updateCsiClusterConfig(s, "beta", &csiClusterConfigEntry2)
-		assert.NoError(t, err)
-		cc, err := parseCsiClusterConfig(s)
-		assert.NoError(t, err)
-		assert.Equal(t, 2, len(cc))
-		assert.Equal(t, "alpha", cc[0].ClusterID)
-		assert.Contains(t, cc[0].Monitors, "1.2.3.4:5000")
-		assert.Contains(t, cc[0].Monitors, "10.11.12.13:5000")
-		assert.Equal(t, 2, len(cc[0].Monitors))
-		assert.Equal(t, "beta", cc[1].ClusterID)
-		assert.Contains(t, cc[1].Monitors, "20.1.1.1:5000")
-		assert.Contains(t, cc[1].Monitors, "20.1.1.2:5000")
-		assert.Contains(t, cc[1].Monitors, "20.1.1.3:5000")
-		assert.Equal(t, 3, len(cc[1].Monitors))
-	})
-
-	t.Run("remove a mon from the 2nd cluster", func(t *testing.T) {
-		i := 2
-		// Remove last element of the slice
-		csiClusterConfigEntry2.Monitors = append(csiClusterConfigEntry2.Monitors[:i], csiClusterConfigEntry2.Monitors[i+1:]...)
-		s, err = updateCsiClusterConfig(s, "beta", &csiClusterConfigEntry2)
-		assert.NoError(t, err)
-		cc, err := parseCsiClusterConfig(s)
-		assert.NoError(t, err)
-		assert.Equal(t, 2, len(cc))
-		assert.Equal(t, "alpha", cc[0].ClusterID)
-		assert.Contains(t, cc[0].Monitors, "1.2.3.4:5000")
-		assert.Contains(t, cc[0].Monitors, "10.11.12.13:5000")
-		assert.Equal(t, 2, len(cc[0].Monitors))
-		assert.Equal(t, "beta", cc[1].ClusterID)
-		assert.Contains(t, cc[1].Monitors, "20.1.1.1:5000")
-		assert.Contains(t, cc[1].Monitors, "20.1.1.2:5000")
-		assert.Equal(t, 2, len(cc[1].Monitors))
-	})
-
-	t.Run("add a 3rd cluster with subvolumegroup", func(t *testing.T) {
-		s, err = updateCsiClusterConfig(s, "baba", &csiClusterConfigEntry3)
-		assert.NoError(t, err)
-		cc, err := parseCsiClusterConfig(s)
-		assert.NoError(t, err)
-		assert.Equal(t, 3, len(cc))
-		assert.Equal(t, "alpha", cc[0].ClusterID)
-		assert.Equal(t, 2, len(cc[0].Monitors))
-		assert.Equal(t, "beta", cc[1].ClusterID)
-		assert.Equal(t, "baba", cc[2].ClusterID)
-		assert.Equal(t, "10.1.1.1:5000", cc[2].Monitors[0])
-		assert.Equal(t, 3, len(cc[2].Monitors))
-		assert.Equal(t, "mygroup", cc[2].CephFS.SubvolumeGroup)
-
-	})
-
-	t.Run("add a 4th mon to the 3rd cluster and subvolumegroup is preserved", func(t *testing.T) {
-		csiClusterConfigEntry3.Monitors = append(csiClusterConfigEntry3.Monitors, "10.11.12.13:5000")
-		s, err = updateCsiClusterConfig(s, "baba", &csiClusterConfigEntry3)
-		assert.NoError(t, err)
-		cc, err := parseCsiClusterConfig(s)
-		assert.NoError(t, err)
-		assert.Equal(t, 3, len(cc))
-		assert.Equal(t, 4, len(cc[2].Monitors))
-		assert.Equal(t, "mygroup", cc[2].CephFS.SubvolumeGroup)
-	})
-
-	t.Run("remove subvolumegroup", func(t *testing.T) {
-		csiClusterConfigEntry3.CephFS.SubvolumeGroup = ""
-		s, err = updateCsiClusterConfig(s, "baba", nil)
-		assert.NoError(t, err)
-		cc, err := parseCsiClusterConfig(s)
-		assert.NoError(t, err)
-		assert.Equal(t, 2, len(cc)) // we only have 2 clusters now
-	})
-
-	t.Run("add subvolumegroup and mons after", func(t *testing.T) {
-		csiClusterConfigEntry4 := CsiClusterConfigEntry{
-			CephFS: &CsiCephFSSpec{
-				SubvolumeGroup: "mygroup2",
-			},
-		}
-		s, err = updateCsiClusterConfig(s, "quatre", &csiClusterConfigEntry4)
-		assert.NoError(t, err)
-		cc, err := parseCsiClusterConfig(s)
-		assert.NoError(t, err)
-		assert.Equal(t, 3, len(cc), cc)
-		assert.Equal(t, 0, len(cc[2].Monitors))
-		assert.Equal(t, "mygroup2", cc[2].CephFS.SubvolumeGroup, cc)
-
-		csiClusterConfigEntry4.Monitors = []string{"10.1.1.1:5000", "10.1.1.2:5000", "10.1.1.3:5000"}
-		s, err = updateCsiClusterConfig(s, "quatre", &csiClusterConfigEntry4)
-		assert.NoError(t, err)
-		cc, err = parseCsiClusterConfig(s)
-		assert.NoError(t, err)
-		assert.Equal(t, "mygroup2", cc[2].CephFS.SubvolumeGroup)
-		assert.Equal(t, 3, len(cc[2].Monitors))
-	})
-
-	t.Run("does it return error on garbage input?", func(t *testing.T) {
-		_, err = updateCsiClusterConfig("qqq", "beta", &csiClusterConfigEntry2)
-		assert.Error(t, err)
-	})
+	s, err = updateCsiClusterConfig(s, "beta", mons2)
+	assert.NoError(t, err)
+	cc, err = parseCsiClusterConfig(s)
+	assert.NoError(t, err)
+	assert.Equal(t, len(cc), 2)
+	assert.Equal(t, cc[0].ClusterID, "alpha")
+	assert.Contains(t, cc[0].Monitors, "1.2.3.4:5000")
+	assert.Contains(t, cc[0].Monitors, "10.11.12.13:5000")
+	assert.Equal(t, len(cc[0].Monitors), 2)
+	assert.Equal(t, cc[1].ClusterID, "beta")
+	assert.Contains(t, cc[1].Monitors, "20.1.1.1:5000")
+	assert.Contains(t, cc[1].Monitors, "20.1.1.2:5000")
+	assert.Contains(t, cc[1].Monitors, "20.1.1.3:5000")
+	assert.Equal(t, len(cc[1].Monitors), 3)
+
+	// remove a mon from the 2nd cluster
+	delete(mons2, "blam")
+	s, err = updateCsiClusterConfig(s, "beta", mons2)
+	assert.NoError(t, err)
+	cc, err = parseCsiClusterConfig(s)
+	assert.NoError(t, err)
+	assert.Equal(t, len(cc), 2)
+	assert.Equal(t, cc[0].ClusterID, "alpha")
+	assert.Contains(t, cc[0].Monitors, "1.2.3.4:5000")
+	assert.Contains(t, cc[0].Monitors, "10.11.12.13:5000")
+	assert.Equal(t, len(cc[0].Monitors), 2)
+	assert.Equal(t, cc[1].ClusterID, "beta")
+	assert.Contains(t, cc[1].Monitors, "20.1.1.1:5000")
+	assert.Contains(t, cc[1].Monitors, "20.1.1.2:5000")
+	assert.Equal(t, len(cc[1].Monitors), 2)
+
+	// does it return error on garbage input?
+	_, err = updateCsiClusterConfig("qqq", "beta", mons2)
+	assert.Error(t, err)
 }
diff --git a/pkg/operator/ceph/csi/predicate_test.go b/pkg/operator/ceph/csi/predicate_test.go
index 95f86820d..af718e63d 100644
--- a/pkg/operator/ceph/csi/predicate_test.go
+++ b/pkg/operator/ceph/csi/predicate_test.go
@@ -17,18 +17,9 @@ limitations under the License.
 package csi
 
 import (
-	"context"
 	"testing"
 
-	"github.com/coreos/pkg/capnslog"
-	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
-	"github.com/rook/rook/pkg/client/clientset/versioned/scheme"
 	"github.com/stretchr/testify/assert"
-	v1 "k8s.io/api/core/v1"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/client/fake"
-	"sigs.k8s.io/controller-runtime/pkg/event"
-	"sigs.k8s.io/controller-runtime/pkg/predicate"
 )
 
 func Test_findCSIChange(t *testing.T) {
@@ -132,119 +123,3 @@ func Test_findCSIChange(t *testing.T) {
 		assert.True(t, b)
 	})
 }
-
-func Test_predicateController(t *testing.T) {
-	capnslog.SetGlobalLogLevel(capnslog.DEBUG)
-	var (
-		client    client.WithWatch
-		namespace v1.Namespace
-		p         predicate.Funcs
-		c         event.CreateEvent
-		d         event.DeleteEvent
-		u         event.UpdateEvent
-		cm        v1.ConfigMap
-		cm2       v1.ConfigMap
-		cluster   cephv1.CephCluster
-		cluster2  cephv1.CephCluster
-	)
-
-	s := scheme.Scheme
-	s.AddKnownTypes(cephv1.SchemeGroupVersion, &v1.ConfigMap{}, &v1.ConfigMapList{}, &cephv1.CephCluster{}, &cephv1.CephClusterList{})
-	client = fake.NewClientBuilder().WithScheme(s).Build()
-
-	t.Run("create event is not the one we are looking for", func(t *testing.T) {
-		c = event.CreateEvent{Object: &namespace}
-		p = predicateController(context.TODO(), client, "rook-ceph")
-		assert.False(t, p.Create(c))
-	})
-
-	t.Run("create event is a CM but not the operator config", func(t *testing.T) {
-		c = event.CreateEvent{Object: &cm}
-		p = predicateController(context.TODO(), client, "rook-ceph")
-		assert.False(t, p.Create(c))
-	})
-
-	t.Run("delete event is a CM and it's not the operator config", func(t *testing.T) {
-		d = event.DeleteEvent{Object: &cm}
-		p = predicateController(context.TODO(), client, "rook-ceph")
-		assert.False(t, p.Delete(d))
-	})
-
-	t.Run("create event is a CM and it's the operator config", func(t *testing.T) {
-		cm.Name = "rook-ceph-operator-config"
-		c = event.CreateEvent{Object: &cm}
-		p = predicateController(context.TODO(), client, "rook-ceph")
-		assert.True(t, p.Create(c))
-	})
-
-	t.Run("delete event is a CM and it's the operator config", func(t *testing.T) {
-		d = event.DeleteEvent{Object: &cm}
-		p = predicateController(context.TODO(), client, "rook-ceph")
-		assert.True(t, p.Delete(d))
-	})
-
-	t.Run("update event is a CM and it's the operator config but nothing changed", func(t *testing.T) {
-		u = event.UpdateEvent{ObjectOld: &cm, ObjectNew: &cm}
-		p = predicateController(context.TODO(), client, "rook-ceph")
-		assert.False(t, p.Update(u))
-	})
-
-	t.Run("update event is a CM and the content changed but with incorrect setting", func(t *testing.T) {
-		cm.Data = map[string]string{"foo": "bar"}
-		cm2.Name = "rook-ceph-operator-config"
-		cm2.Data = map[string]string{"foo": "ooo"}
-		u = event.UpdateEvent{ObjectOld: &cm, ObjectNew: &cm2}
-		p = predicateController(context.TODO(), client, "rook-ceph")
-		assert.False(t, p.Update(u))
-	})
-
-	t.Run("update event is a CM and the content changed with correct setting", func(t *testing.T) {
-		cm.Data = map[string]string{"foo": "bar"}
-		cm2.Name = "rook-ceph-operator-config"
-		cm2.Data = map[string]string{"CSI_PROVISIONER_REPLICAS": "2"}
-		u = event.UpdateEvent{ObjectOld: &cm, ObjectNew: &cm2}
-		p = predicateController(context.TODO(), client, "rook-ceph")
-		assert.True(t, p.Update(u))
-	})
-
-	t.Run("create event is a CephCluster and it's the first instance and a cm is present", func(t *testing.T) {
-		cm.Namespace = "rook-ceph"
-		err := client.Create(context.TODO(), &cm)
-		assert.NoError(t, err)
-		cluster.Generation = 1
-		c = event.CreateEvent{Object: &cluster}
-		p = predicateController(context.TODO(), client, "rook-ceph")
-		assert.True(t, p.Create(c))
-	})
-
-	t.Run("create event is a CephCluster and it's not the first instance", func(t *testing.T) {
-		cluster.Generation = 2
-		c = event.CreateEvent{Object: &cluster}
-		p = predicateController(context.TODO(), client, "rook-ceph")
-		assert.False(t, p.Create(c))
-	})
-
-	t.Run("create event is a CephCluster and it's not the first instance but no cm so we reconcile", func(t *testing.T) {
-		err := client.Delete(context.TODO(), &cm)
-		assert.NoError(t, err)
-		cluster.Generation = 2
-		c = event.CreateEvent{Object: &cluster}
-		p = predicateController(context.TODO(), client, "rook-ceph")
-		assert.True(t, p.Create(c))
-	})
-
-	t.Run("create event is a CephCluster and a cluster already exists - not reconciling", func(t *testing.T) {
-		cluster.Name = "ceph1"
-		cluster2.Name = "ceph2"
-		cluster.Namespace = "rook-ceph"
-		cluster2.Namespace = "rook-ceph"
-		err := client.Create(context.TODO(), &cluster)
-		assert.NoError(t, err)
-		err = client.Create(context.TODO(), &cluster2)
-		assert.NoError(t, err)
-		c = event.CreateEvent{Object: &cluster2}
-		assert.Equal(t, "rook-ceph", c.Object.GetNamespace())
-		p = predicateController(context.TODO(), client, "rook-ceph")
-		assert.False(t, p.Create(c))
-	})
-}
diff --git a/pkg/operator/ceph/file/filesystem_test.go b/pkg/operator/ceph/file/filesystem_test.go
index 64abb1715..63b6a3971 100644
--- a/pkg/operator/ceph/file/filesystem_test.go
+++ b/pkg/operator/ceph/file/filesystem_test.go
@@ -21,6 +21,8 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io/ioutil"
+	"os"
 	"path"
 	"reflect"
 	"strings"
@@ -317,11 +319,12 @@ func TestCreateFilesystem(t *testing.T) {
 	ctx := context.TODO()
 	var deploymentsUpdated *[]*apps.Deployment
 	mds.UpdateDeploymentAndWait, deploymentsUpdated = testopk8s.UpdateDeploymentAndWaitStub()
-	configDir := t.TempDir()
+	configDir, _ := ioutil.TempDir("", "")
 	fsName := "myfs"
 	addDataPoolCount := 0
 	createDataPoolCount := 0
 	executor := fsExecutor(t, fsName, configDir, false, &createDataPoolCount, &addDataPoolCount)
+	defer os.RemoveAll(configDir)
 	clientset := testop.New(t, 1)
 	context := &clusterd.Context{
 		Executor:  executor,
@@ -395,12 +398,13 @@ func TestUpgradeFilesystem(t *testing.T) {
 	ctx := context.TODO()
 	var deploymentsUpdated *[]*apps.Deployment
 	mds.UpdateDeploymentAndWait, deploymentsUpdated = testopk8s.UpdateDeploymentAndWaitStub()
-	configDir := t.TempDir()
+	configDir, _ := ioutil.TempDir("", "")
 
 	fsName := "myfs"
 	addDataPoolCount := 0
 	createDataPoolCount := 0
 	executor := fsExecutor(t, fsName, configDir, false, &createDataPoolCount, &addDataPoolCount)
+	defer os.RemoveAll(configDir)
 	clientset := testop.New(t, 1)
 	context := &clusterd.Context{
 		Executor:  executor,
@@ -514,7 +518,7 @@ func TestUpgradeFilesystem(t *testing.T) {
 func TestCreateNopoolFilesystem(t *testing.T) {
 	ctx := context.TODO()
 	clientset := testop.New(t, 3)
-	configDir := t.TempDir()
+	configDir, _ := ioutil.TempDir("", "")
 	executor := &exectest.MockExecutor{
 		MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) {
 			if strings.Contains(command, "ceph-authtool") {
@@ -526,6 +530,7 @@ func TestCreateNopoolFilesystem(t *testing.T) {
 			return "", errors.New("unknown command error")
 		},
 	}
+	defer os.RemoveAll(configDir)
 	context := &clusterd.Context{
 		Executor:  executor,
 		ConfigDir: configDir,
diff --git a/pkg/operator/ceph/file/subvolumegroup/controller_test.go b/pkg/operator/ceph/file/subvolumegroup/controller_test.go
index 1d0e5a555..6fffbdea5 100644
--- a/pkg/operator/ceph/file/subvolumegroup/controller_test.go
+++ b/pkg/operator/ceph/file/subvolumegroup/controller_test.go
@@ -26,7 +26,6 @@ import (
 	rookclient "github.com/rook/rook/pkg/client/clientset/versioned/fake"
 	"github.com/rook/rook/pkg/client/clientset/versioned/scheme"
 	"github.com/rook/rook/pkg/clusterd"
-	"github.com/rook/rook/pkg/operator/ceph/csi"
 	"github.com/rook/rook/pkg/operator/k8sutil"
 	testop "github.com/rook/rook/pkg/operator/test"
 	exectest "github.com/rook/rook/pkg/util/exec/test"
@@ -207,39 +206,15 @@ func TestCephClientController(t *testing.T) {
 			client:           cl,
 			scheme:           s,
 			context:          c,
-			opManagerContext: ctx,
+			opManagerContext: context.TODO(),
 		}
 
-		// Enable CSI
-		csi.EnableRBD = true
-		os.Setenv("POD_NAMESPACE", namespace)
-		// Create CSI config map
-		ownerRef := &metav1.OwnerReference{}
-		ownerInfo := k8sutil.NewOwnerInfoWithOwnerRef(ownerRef, "")
-		err = csi.CreateCsiConfigMap(namespace, c.Clientset, ownerInfo)
-		assert.NoError(t, err)
-
 		res, err := r.Reconcile(ctx, req)
 		assert.NoError(t, err)
 		assert.False(t, res.Requeue)
 
-		err = r.client.Get(ctx, req.NamespacedName, cephFilesystemSubVolumeGroup)
+		err = r.client.Get(context.TODO(), req.NamespacedName, cephFilesystemSubVolumeGroup)
 		assert.NoError(t, err)
 		assert.Equal(t, cephv1.ConditionReady, cephFilesystemSubVolumeGroup.Status.Phase)
-		assert.NotEmpty(t, cephFilesystemSubVolumeGroup.Status.Info["clusterID"])
-
-		// test that csi configmap is created
-		cm, err := c.Clientset.CoreV1().ConfigMaps(namespace).Get(ctx, csi.ConfigName, metav1.GetOptions{})
-		assert.NoError(t, err)
-		assert.NotEmpty(t, cm.Data[csi.ConfigKey])
-		assert.Contains(t, cm.Data[csi.ConfigKey], "clusterID")
-		assert.Contains(t, cm.Data[csi.ConfigKey], "group-a")
 	})
 }
-
-func Test_buildClusterID(t *testing.T) {
-	longName := "foooooooooooooooooooooooooooooooooooooooooooo"
-	cephFilesystemSubVolumeGroup := &cephv1.CephFilesystemSubVolumeGroup{ObjectMeta: metav1.ObjectMeta{Namespace: "rook-ceph", Name: longName}, Spec: cephv1.CephFilesystemSubVolumeGroupSpec{FilesystemName: "myfs"}}
-	clusterID := buildClusterID(cephFilesystemSubVolumeGroup)
-	assert.Equal(t, "29e92135b7e8c014079b9f9f3566777d", clusterID)
-}
diff --git a/pkg/operator/ceph/object/bucket/rgw-handlers_test.go b/pkg/operator/ceph/object/bucket/rgw-handlers_test.go
deleted file mode 100644
index ba487b8d1..000000000
--- a/pkg/operator/ceph/object/bucket/rgw-handlers_test.go
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
-Copyright 2022 The Rook Authors. All rights reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-	http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package bucket
-
-import (
-	"bytes"
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
-	"net/http"
-	"testing"
-
-	"github.com/ceph/go-ceph/rgw/admin"
-	rookclient "github.com/rook/rook/pkg/client/clientset/versioned/fake"
-	"github.com/rook/rook/pkg/clusterd"
-	"github.com/rook/rook/pkg/daemon/ceph/client"
-	cephobject "github.com/rook/rook/pkg/operator/ceph/object"
-	"github.com/rook/rook/pkg/operator/test"
-	"github.com/stretchr/testify/assert"
-)
-
-type statusError struct {
-	Code      string `json:"Code,omitempty"`
-	RequestID string `json:"RequestId,omitempty"`
-	HostID    string `json:"HostId,omitempty"`
-}
-
-func TestDeleteOBCResource(t *testing.T) {
-	clusterInfo := client.AdminTestClusterInfo("ns")
-	p := NewProvisioner(&clusterd.Context{RookClientset: rookclient.NewSimpleClientset(), Clientset: test.New(t, 1)}, clusterInfo)
-	mockClient := func(errCodeRemoveBucket string, errCodeGetBucketInfo string) *cephobject.MockClient {
-		return &cephobject.MockClient{
-			MockDo: func(req *http.Request) (*http.Response, error) {
-				if req.URL.Path == "rook-ceph-rgw-my-store.mycluster.svc/admin/bucket" {
-					if req.Method == http.MethodDelete {
-						status, _ := json.Marshal(statusError{errCodeRemoveBucket, "requestid", "hostid"})
-						return &http.Response{
-							StatusCode: 404,
-							Body:       ioutil.NopCloser(bytes.NewReader([]byte(status))),
-						}, nil
-					}
-					if req.Method == http.MethodGet {
-						status, _ := json.Marshal(statusError{errCodeGetBucketInfo, "requestid", "hostid"})
-						return &http.Response{
-							StatusCode: 404,
-							Body:       ioutil.NopCloser(bytes.NewReader([]byte(status))),
-						}, nil
-					}
-				}
-				return nil, fmt.Errorf("unexpected request: %q. method %q. path %q", req.URL.RawQuery, req.Method, req.URL.Path)
-			},
-		}
-	}
-
-	t.Run("remove bucket returns NoSuchBucket", func(t *testing.T) {
-		adminClient, err := admin.New("rook-ceph-rgw-my-store.mycluster.svc", "53S6B9S809NUP19IJ2K3", "1bXPegzsGClvoGAiJdHQD1uOW2sQBLAZM9j9VtXR", mockClient("NoSuchBucket", ""))
-		assert.NoError(t, err)
-		p.adminOpsClient = adminClient
-		err = p.deleteOBCResource("bucket")
-		assert.NoError(t, err)
-	})
-
-	t.Run("remove bucket returns NoSuchKey and get bucket info returns NoSuchBucket", func(t *testing.T) {
-		adminClient, err := admin.New("rook-ceph-rgw-my-store.mycluster.svc", "53S6B9S809NUP19IJ2K3", "1bXPegzsGClvoGAiJdHQD1uOW2sQBLAZM9j9VtXR", mockClient("NoSuchKey", "NoSuchBucket"))
-		assert.NoError(t, err)
-		p.adminOpsClient = adminClient
-		err = p.deleteOBCResource("bucket")
-		assert.NoError(t, err)
-	})
-
-	t.Run("remove bucket returns NoSuchKey and get bucket info returns an error other than NoSuchBucket", func(t *testing.T) {
-		adminClient, err := admin.New("rook-ceph-rgw-my-store.mycluster.svc", "53S6B9S809NUP19IJ2K3", "1bXPegzsGClvoGAiJdHQD1uOW2sQBLAZM9j9VtXR", mockClient("NoSuchKey", "NoSuchKey"))
-		assert.NoError(t, err)
-		p.adminOpsClient = adminClient
-		err = p.deleteOBCResource("bucket")
-		assert.Error(t, err)
-	})
-}
diff --git a/pkg/operator/ceph/object/rgw_test.go b/pkg/operator/ceph/object/rgw_test.go
index e90092382..e720bcd1a 100644
--- a/pkg/operator/ceph/object/rgw_test.go
+++ b/pkg/operator/ceph/object/rgw_test.go
@@ -18,6 +18,8 @@ package object
 
 import (
 	"context"
+	"io/ioutil"
+	"os"
 	"testing"
 
 	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
@@ -49,7 +51,8 @@ func TestStartRGW(t *testing.T) {
 		},
 	}
 
-	configDir := t.TempDir()
+	configDir, _ := ioutil.TempDir("", "")
+	defer os.RemoveAll(configDir)
 	info := clienttest.CreateTestClusterInfo(1)
 	context := &clusterd.Context{Clientset: clientset, Executor: executor, ConfigDir: configDir}
 	store := simpleStore()
diff --git a/tests/framework/clients/bucket.go b/tests/framework/clients/bucket.go
index cfcc61ad4..037bcdf19 100644
--- a/tests/framework/clients/bucket.go
+++ b/tests/framework/clients/bucket.go
@@ -150,18 +150,22 @@ func (b *BucketOperation) CheckOBMaxObject(obcName, maxobject string) bool {
 }
 
 // Checks the bucket notifications set on RGW backend bucket
-func (b *BucketOperation) CheckBucketNotificationSetonRGW(namespace, storeName, obcName, bucketname, notificationName, s3endpoint string) bool {
+func (b *BucketOperation) CheckBucketNotificationSetonRGW(namespace, storeName, obcName, bucketname, notificationName string, helper *TestClient, tlsEnabled bool) bool {
 	var s3client *rgw.S3Agent
-	s3AccessKey, _ := b.GetAccessKey(obcName)
-	s3SecretKey, _ := b.GetSecretKey(obcName)
-
-	//TODO : add TLS check
-	s3client, err := rgw.NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, "", true, nil)
+	var err error
+	s3endpoint, _ := helper.ObjectClient.GetEndPointUrl(namespace, storeName)
+	s3AccessKey, _ := helper.BucketClient.GetAccessKey(obcName)
+	s3SecretKey, _ := helper.BucketClient.GetSecretKey(obcName)
+	if tlsEnabled {
+		s3client, err = rgw.NewInsecureS3Agent(s3AccessKey, s3SecretKey, s3endpoint, "", true)
+	} else {
+		s3client, err = rgw.NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, "", true, nil)
+	}
 	if err != nil {
-		logger.Errorf("S3 client creation failed with error %v", err)
+		logger.Infof("failed to s3client due to %v", err)
 		return false
 	}
-
+	logger.Infof("endpoint (%s) Accesskey (%s) secret (%s)", s3endpoint, s3AccessKey, s3SecretKey)
 	notifications, err := s3client.Client.GetBucketNotificationConfiguration(&s3.GetBucketNotificationConfigurationRequest{
 		Bucket: &bucketname,
 	})
diff --git a/tests/framework/installer/ceph_helm_installer.go b/tests/framework/installer/ceph_helm_installer.go
index fd07352d4..f61a6feed 100644
--- a/tests/framework/installer/ceph_helm_installer.go
+++ b/tests/framework/installer/ceph_helm_installer.go
@@ -62,6 +62,9 @@ func (h *CephInstaller) configureRookOperatorViaHelm(upgrade bool) error {
 	if err := h.k8shelper.CreateNamespace(h.settings.OperatorNamespace); err != nil {
 		return errors.Errorf("failed to create namespace %s. %v", h.settings.Namespace, err)
 	}
+	if err := h.k8shelper.CreateOpConfigMap(h.settings.OperatorNamespace); err != nil {
+		return errors.Errorf("failed to create operator config map. %v", err)
+	}
 	if err := h.startAdmissionController(); err != nil {
 		return errors.Errorf("failed to start admission controllers. %v", err)
 	}
diff --git a/tests/framework/utils/k8s_helper.go b/tests/framework/utils/k8s_helper.go
index f57440c11..8d4d71f57 100644
--- a/tests/framework/utils/k8s_helper.go
+++ b/tests/framework/utils/k8s_helper.go
@@ -37,6 +37,7 @@ import (
 	rookclient "github.com/rook/rook/pkg/client/clientset/versioned"
 	"github.com/rook/rook/pkg/clusterd"
 	"github.com/rook/rook/pkg/operator/ceph/cluster/crash"
+	"github.com/rook/rook/pkg/operator/ceph/controller"
 	"github.com/rook/rook/pkg/operator/k8sutil"
 	"github.com/rook/rook/pkg/util/exec"
 	"github.com/stretchr/testify/require"
@@ -374,6 +375,22 @@ func (k8sh *K8sHelper) CreateNamespace(namespace string) error {
 	return nil
 }
 
+func (k8sh *K8sHelper) CreateOpConfigMap(namespace string) error {
+	ctx := context.TODO()
+	cm := &v1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      controller.OperatorSettingConfigMapName,
+			Namespace: namespace,
+		},
+	}
+	_, err := k8sh.Clientset.CoreV1().ConfigMaps(namespace).Create(ctx, cm, metav1.CreateOptions{})
+	if err != nil && !kerrors.IsAlreadyExists(err) {
+		return fmt.Errorf("failed to create operator config map %s. %+v", controller.OperatorSettingConfigMapName, err)
+	}
+
+	return nil
+}
+
 func (k8sh *K8sHelper) CountPodsWithLabel(label string, namespace string) (int, error) {
 	ctx := context.TODO()
 	options := metav1.ListOptions{LabelSelector: label}
diff --git a/tests/integration/ceph_base_object_test.go b/tests/integration/ceph_base_object_test.go
index 7e650e21a..9f1c8cc86 100644
--- a/tests/integration/ceph_base_object_test.go
+++ b/tests/integration/ceph_base_object_test.go
@@ -21,6 +21,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"os"
 	"path/filepath"
 	"strconv"
 	"testing"
@@ -254,7 +255,9 @@ func generateRgwTlsCertSecret(t *testing.T, helper *clients.TestClient, k8sh *ut
 	ctx := context.TODO()
 	root, err := utils.FindRookRoot()
 	require.NoError(t, err, "failed to get rook root")
-	tlscertdir := t.TempDir()
+	tlscertdir, err := ioutil.TempDir(root, "tlscertdir")
+	require.NoError(t, err, "failed to create directory for TLS certs")
+	defer os.RemoveAll(tlscertdir)
 	cmdArgs := utils.CommandArgs{Command: filepath.Join(root, "tests/scripts/generate-tls-config.sh"),
 		CmdArgs: []string{tlscertdir, rgwServiceName, namespace}}
 	cmdOut := utils.ExecuteCommand(cmdArgs)
diff --git a/tests/integration/ceph_bucket_notification_test.go b/tests/integration/ceph_bucket_notification_test.go
index 91b285b1c..21297f7b2 100644
--- a/tests/integration/ceph_bucket_notification_test.go
+++ b/tests/integration/ceph_bucket_notification_test.go
@@ -23,26 +23,20 @@ import (
 
 	"github.com/rook/rook/pkg/daemon/ceph/client"
 	rgw "github.com/rook/rook/pkg/operator/ceph/object"
+	"github.com/rook/rook/tests/framework/clients"
 	"github.com/rook/rook/tests/framework/utils"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/suite"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-func (s *ObjectSuite) TestBucketNotifications() {
+func testBucketNotifications(s suite.Suite, helper *clients.TestClient, k8sh *utils.K8sHelper, namespace, storeName string) {
 	if utils.IsPlatformOpenShift() {
 		s.T().Skip("bucket notification tests skipped on openshift")
 	}
 
-	objectStoreServicePrefix = objectStoreServicePrefixUniq
 	bucketNotificationLabelPrefix := "bucket-notification-"
-	storeName := "test-store-bucket-notification"
-	tlsEnable := false
-	namespace := s.settings.Namespace
 	obcNamespace := "default"
-	helper := s.helper
-	k8sh := s.k8sh
-	logger.Infof("Running on Rook Cluster %s", namespace)
-	createCephObjectStore(s.T(), helper, k8sh, namespace, storeName, 3, tlsEnable)
 
 	ctx := context.TODO()
 	clusterInfo := client.AdminTestClusterInfo(namespace)
@@ -57,7 +51,7 @@ func (s *ObjectSuite) TestBucketNotifications() {
 	notificationName := "my-notification"
 	topicName := "my-topic"
 	httpEndpointService := "my-notification-sink"
-	s3endpoint, _ := helper.ObjectClient.GetEndPointUrl(namespace, storeName)
+	logger.Infof("Testing Bucket Notifications on %s", storeName)
 
 	t.Run("create CephBucketTopic", func(t *testing.T) {
 		err := helper.TopicClient.CreateTopic(topicName, storeName, httpEndpointService)
@@ -111,7 +105,7 @@ func (s *ObjectSuite) TestBucketNotifications() {
 
 	t.Run("check CephBucketNotification created for bucket", func(t *testing.T) {
 		notificationPresent := utils.Retry(12, 2*time.Second, "notification is created for bucket", func() bool {
-			return helper.BucketClient.CheckBucketNotificationSetonRGW(namespace, storeName, obcName, bucketname, notificationName, s3endpoint)
+			return helper.BucketClient.CheckBucketNotificationSetonRGW(namespace, storeName, obcName, bucketname, notificationName, helper, objectStore.Spec.IsTLSEnabled())
 		})
 		assert.True(t, notificationPresent)
 		logger.Info("CephBucketNotification created successfully on bucket")
@@ -127,7 +121,7 @@ func (s *ObjectSuite) TestBucketNotifications() {
 		notificationPresent := utils.Retry(12, 2*time.Second, "notification is created for bucket", func() bool {
 			// TODO : add api to fetch all the notification from backend to see if it is unaffected
 			t.Skipped()
-			return helper.BucketClient.CheckBucketNotificationSetonRGW(namespace, storeName, obcName, bucketname, notificationName, s3endpoint)
+			return helper.BucketClient.CheckBucketNotificationSetonRGW(namespace, storeName, obcName, bucketname, notificationName, helper, objectStore.Spec.IsTLSEnabled())
 		})
 		assert.True(t, notificationPresent)
 	})
@@ -142,7 +136,7 @@ func (s *ObjectSuite) TestBucketNotifications() {
 		notificationPresent := utils.Retry(12, 2*time.Second, "notification is created for bucket", func() bool {
 			// TODO : add api to fetch all the notification from backend to see if it is unaffected
 			t.Skipped()
-			return helper.BucketClient.CheckBucketNotificationSetonRGW(namespace, storeName, obcName, bucketname, notificationName, s3endpoint)
+			return helper.BucketClient.CheckBucketNotificationSetonRGW(namespace, storeName, obcName, bucketname, notificationName, helper, objectStore.Spec.IsTLSEnabled())
 		})
 		assert.True(t, notificationPresent)
 	})
@@ -158,7 +152,7 @@ func (s *ObjectSuite) TestBucketNotifications() {
 		// check whether existing bucket notification uneffected
 		var notificationPresent bool
 		for i := 0; i < 4; i++ {
-			notificationPresent = helper.BucketClient.CheckBucketNotificationSetonRGW(namespace, storeName, obcName, bucketname, notificationName, s3endpoint)
+			notificationPresent = helper.BucketClient.CheckBucketNotificationSetonRGW(namespace, storeName, obcName, bucketname, notificationName, helper, objectStore.Spec.IsTLSEnabled())
 			if !notificationPresent {
 				break
 			}
@@ -196,7 +190,7 @@ func (s *ObjectSuite) TestBucketNotifications() {
 		t.Run("new-notification should be configured for bucket", func(t *testing.T) {
 			// check whether bucket notification added
 			notificationPresent := utils.Retry(12, 2*time.Second, "notification is created for bucket", func() bool {
-				return helper.BucketClient.CheckBucketNotificationSetonRGW(namespace, storeName, obcName, bucketname, newNotificationName, s3endpoint)
+				return helper.BucketClient.CheckBucketNotificationSetonRGW(namespace, storeName, obcName, bucketname, newNotificationName, helper, objectStore.Spec.IsTLSEnabled())
 			})
 			assert.True(t, notificationPresent)
 		})
@@ -269,7 +263,7 @@ func (s *ObjectSuite) TestBucketNotifications() {
 		t.Run("notification should be configured after creating the topic", func(t *testing.T) {
 			// check whether bucket notification added, should pass since topic got created
 			notificationPresent := utils.Retry(12, 2*time.Second, "notification is created for bucket", func() bool {
-				return helper.BucketClient.CheckBucketNotificationSetonRGW(namespace, storeName, reverseOBCName, reverseBucketName, reverseNotificationName, s3endpoint)
+				return helper.BucketClient.CheckBucketNotificationSetonRGW(namespace, storeName, reverseOBCName, reverseBucketName, reverseNotificationName, helper, objectStore.Spec.IsTLSEnabled())
 			})
 			assert.True(t, notificationPresent)
 		})
diff --git a/tests/integration/ceph_object_test.go b/tests/integration/ceph_object_test.go
index 49e0f723b..268dd2729 100644
--- a/tests/integration/ceph_object_test.go
+++ b/tests/integration/ceph_object_test.go
@@ -153,6 +153,10 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite
 
 	// now test operation of the first object store
 	testObjectStoreOperations(s, helper, k8sh, namespace, storeName)
+
+	bucketNotificationTestStoreName := "bucket-notification-" + storeName
+	createCephObjectStore(s.T(), helper, k8sh, namespace, bucketNotificationTestStoreName, 1, tlsEnable)
+	testBucketNotifications(s, helper, k8sh, namespace, bucketNotificationTestStoreName)
 }
 
 func testObjectStoreOperations(s suite.Suite, helper *clients.TestClient, k8sh *utils.K8sHelper, namespace, storeName string) {
@@ -160,6 +164,7 @@ func testObjectStoreOperations(s suite.Suite, helper *clients.TestClient, k8sh *
 	clusterInfo := client.AdminTestClusterInfo(namespace)
 	t := s.T()
 
+	logger.Infof("Testing Object Operations on %s", storeName)
 	t.Run("create CephObjectStoreUser", func(t *testing.T) {
 		createCephObjectUser(s, helper, k8sh, namespace, storeName, userid, true, true)
 		i := 0
@@ -358,7 +363,8 @@ func testObjectStoreOperations(s suite.Suite, helper *clients.TestClient, k8sh *
 		assert.True(t, k8sh.CheckPodCountAndState("rook-ceph-mgr", namespace, 1, "Running"))
 	})
 
-	t.Run("CephObjectStore should delete now that dependents are gone", func(t *testing.T) {
+	// tests are complete, now delete the objectstore
+	s.T().Run("CephObjectStore should delete now that dependents are gone", func(t *testing.T) {
 		// wait initially since it will almost never detect on the first try without this.
 		time.Sleep(3 * time.Second)
 
diff --git a/tests/manifests/test-kms-ibm-kp-spec.in b/tests/manifests/test-kms-ibm-kp-spec.in
index 6bbebd77d..4070971a8 100644
--- a/tests/manifests/test-kms-ibm-kp-spec.in
+++ b/tests/manifests/test-kms-ibm-kp-spec.in
@@ -2,6 +2,6 @@ spec:
   security:
     kms:
       connectionDetails:
-        KMS_PROVIDER: ibmkeyprotect
-        IBM_KP_SERVICE_INSTANCE_ID: $IBM_KP_SERVICE_INSTANCE_ID
+        KMS_PROVIDER: ibm-kp
+        IBM_KP_INSTANCE_ID: $IBM_KP_INSTANCE_ID
       tokenSecretName: rook-ibm-kp-token