Repository: rook/rook
Pull request #13741 (2024-02-09 15:59:16)

mgr: remove privileged security context from mgr sidecar container
<!-- Thank you for contributing to Rook! -->

<!-- STEPS TO FOLLOW:
  1. Add a description of the changes (frequently the same as the commit description)
  2. Enter the issue number next to "Resolves #" below (if there is no tracking issue resolved, **remove that section**)
  3. Review our Contributing documentation at https://rook.io/docs/rook/latest/Contributing/development-flow/
  4. Follow the steps in the checklist below, starting with the **Commit Message Formatting**.
-->

Mgr sidecar was using a privileged security context because it needed
to create the `/var/lib/rook` directory, generate the config on it, and
then copy it to the `/etc/ceph` directory. This change creates empty
directories on `/var/lib/rook` and `/etc/ceph` so the mgr sidecar can use
them, removing the need for using a privileged security context.

**Issue resolved by this Pull Request:**
Fixes: https://github.com/rook/rook/issues/13719

**Manual testing:**

```
k get pod | grep mgr | awk '{print $1}' | (while read x; do echo "======= $x =====" ; k get pod $x -o=json | jq -r '.spec.containers[] | "\(.name) - \(.securityContext)"'; done)

======= rook-ceph-mgr-a-74f9fd45d4-lpdhl =====
mgr - {"privileged":false}
watch-active - {"privileged":false}
======= rook-ceph-mgr-b-6549f7bcd6-wz9r7 =====
mgr - {"privileged":false}
watch-active - {"privileged":false}
```

Mgr sidecar `watch-active` is running without issues both on the active and on the mgr standby pods:

```
k logs rook-ceph-mgr-a-74f9fd45d4-lpdhl -c watch-active | tail -10                                         ✔   
2024-02-12 09:52:00.872858 I | cephcmd: successfully checked mgr_role label. checking again in 15s
2024-02-12 09:52:15.887326 I | op-mgr: Checking mgr_role label value of daemon a (prev active mgr was a)
2024-02-12 09:52:16.352018 I | op-mgr: active mgr is still the same (a). No need to update mgr_role label on daemon a.
2024-02-12 09:52:16.352034 I | cephcmd: successfully checked mgr_role label. checking again in 15s
2024-02-12 09:52:31.365846 I | op-mgr: Checking mgr_role label value of daemon a (prev active mgr was a)
2024-02-12 09:52:31.828295 I | op-mgr: active mgr is still the same (a). No need to update mgr_role label on daemon a.
2024-02-12 09:52:31.828314 I | cephcmd: successfully checked mgr_role label. checking again in 15s
2024-02-12 09:52:46.840292 I | op-mgr: Checking mgr_role label value of daemon a (prev active mgr was a)
2024-02-12 09:52:47.259273 I | op-mgr: active mgr is still the same (a). No need to update mgr_role label on daemon a.
2024-02-12 09:52:47.259293 I | cephcmd: successfully checked mgr_role label. checking again in 15s
```

```
k logs rook-ceph-mgr-b-6549f7bcd6-wz9r7 -c watch-active | tail -10                                         ✔   
2024-02-12 09:52:32.478736 I | cephcmd: successfully checked mgr_role label. checking again in 15s
2024-02-12 09:52:47.478856 I | op-mgr: Checking mgr_role label value of daemon b (prev active mgr was a)
2024-02-12 09:52:47.872795 I | op-mgr: active mgr is still the same (a). No need to update mgr_role label on daemon b.
2024-02-12 09:52:47.872813 I | cephcmd: successfully checked mgr_role label. checking again in 15s
2024-02-12 09:53:02.886029 I | op-mgr: Checking mgr_role label value of daemon b (prev active mgr was a)
2024-02-12 09:53:03.371399 I | op-mgr: active mgr is still the same (a). No need to update mgr_role label on daemon b.
2024-02-12 09:53:03.371469 I | cephcmd: successfully checked mgr_role label. checking again in 15s
2024-02-12 09:53:18.381855 I | op-mgr: Checking mgr_role label value of daemon b (prev active mgr was a)
2024-02-12 09:53:18.789335 I | op-mgr: active mgr is still the same (a). No need to update mgr_role label on daemon b.
2024-02-12 09:53:18.789353 I | cephcmd: successfully checked mgr_role label. checking again in 15s

```

**Checklist:**

- [x] **Commit Message Formatting**: Commit titles and messages follow guidelines in the [developer guide](https://rook.io/docs/rook/latest/Contributing/development-flow/#commit-structure).
- [x] Reviewed the developer guide on [Submitting a Pull Request](https://rook.io/docs/rook/latest/Contributing/development-flow/#submitting-a-pull-request)
- [ ] [Pending release notes](https://github.com/rook/rook/blob/master/PendingReleaseNotes.md) updated with breaking and/or notable changes for the next minor release.
- [ ] Documentation has been updated, if necessary.
- [ ] Unit tests have been added, if necessary.
- [ ] Integration tests have been added, if necessary.



Patch:
diff --git a/.github/workflows/daily-nightly-jobs.yml b/.github/workflows/daily-nightly-jobs.yml
index b5e28acb5..e361880f2 100644
--- a/.github/workflows/daily-nightly-jobs.yml
+++ b/.github/workflows/daily-nightly-jobs.yml
@@ -2,7 +2,6 @@ name: Daily nightly jobs
 on:
   schedule:
     - cron: "0 0 * * *" # every day at midnight
-  workflow_dispatch: {}
 
 defaults:
   run:
@@ -395,18 +394,64 @@ jobs:
           name: ceph-upgrade-suite-quincy-artifact
           path: /home/runner/work/rook/rook/tests/integration/_output/tests/
 
-  # run default canary suite
-  canary-tests:
-    uses: ./.github/workflows/canary-integration-test.yml
-    secrets: inherit
-
-  # run canary suite with relevant ceph devel versions
-  canary-tests-devel:
-    uses: ./.github/workflows/canary-integration-test.yml
+  rgw-multisite-test-with-ceph-devel:
+    if: github.repository == 'rook/rook'
+    runs-on: ubuntu-20.04
     strategy:
       matrix:
         ceph-image-tag:
           ["latest-main-devel", "latest-quincy-devel", "latest-reef-devel"]
-    with:
-      ceph-image: quay.io/ceph/daemon-base:${{ matrix.ceph-image-tag }}
-    secrets: inherit
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: consider debugging
+        uses: ./.github/workflows/tmate_debug
+        with:
+          use-tmate: ${{ secrets.USE_TMATE }}
+
+      - name: run RGW multisite test
+        uses: ./.github/workflows/rgw-multisite-test
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          ceph-image: quay.io/ceph/daemon-base:${{ matrix.ceph-image-tag }}
+
+      - name: upload test result
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: rgw-multisite-testing-ceph-${{ matrix.ceph-image-tag }}
+          path: test
+
+  encryption-pvc-kms-ibm-kp:
+    if: github.repository == 'rook/rook'
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: consider debugging
+        uses: ./.github/workflows/tmate_debug
+        with:
+          use-tmate: ${{ secrets.USE_TMATE }}
+
+      - name: run encryption KMS IBM Key Protect
+        uses: ./.github/workflows/encryption-pvc-kms-ibm-kp
+        if: "env.IBM_KP_SERVICE_INSTANCE_ID != '' && env.IBM_KP_SERVICE_API_KEY != ''"
+        env:
+          IBM_KP_SERVICE_INSTANCE_ID: ${{ secrets.IBM_INSTANCE_ID }}
+          IBM_KP_SERVICE_API_KEY: ${{ secrets.IBM_SERVICE_API_KEY }}
+        with:
+          ibm-instance-id: ${{ secrets.IBM_INSTANCE_ID }}
+          ibm-service-api-key: ${{ secrets.IBM_SERVICE_API_KEY }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: upload test result
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: encryption-pvc-kms-ibm-kp
+          path: test
diff --git a/.github/workflows/golangci-lint.yaml b/.github/workflows/golangci-lint.yaml
index 5bf7d673d..73dab46ef 100644
--- a/.github/workflows/golangci-lint.yaml
+++ b/.github/workflows/golangci-lint.yaml
@@ -31,7 +31,7 @@ jobs:
         with:
           go-version: "1.21"
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@v4
+        uses: golangci/golangci-lint-action@v3
         with:
           # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
           version: v1.55
diff --git a/.github/workflows/multus.yaml b/.github/workflows/multus.yaml
index a31584577..cfe1cbaa0 100644
--- a/.github/workflows/multus.yaml
+++ b/.github/workflows/multus.yaml
@@ -42,7 +42,7 @@ jobs:
           go-version: "1.21"
 
       - name: Create KinD Cluster
-        uses: helm/kind-action@v1.9.0
+        uses: helm/kind-action@v1.8.0
         with:
           config: tests/scripts/multus/kind-config.yaml
           cluster_name: kind
diff --git a/Documentation/CRDs/Cluster/external-cluster.md b/Documentation/CRDs/Cluster/external-cluster.md
index 8a803fe33..2dd19fede 100644
--- a/Documentation/CRDs/Cluster/external-cluster.md
+++ b/Documentation/CRDs/Cluster/external-cluster.md
@@ -127,16 +127,7 @@ export RGW_POOL_PREFIX=default
 
 1. Paste the above output from `create-external-cluster-resources.py` into your current shell to allow importing the source data.
 
-1. The import script in the next step uses the current kubeconfig context by
-   default. If you want to specify the kubernetes cluster to use without
-   changing the current context, you can specify the cluster name by setting
-   the KUBECONTEXT environment variable.
-
-   ```console
-   export KUBECONTEXT=hub-cluster
-   ```
-
-1. Run the [import](https://github.com/rook/rook/blob/master/deploy/examples/import-external-cluster.sh) script.
+2. Run the [import](https://github.com/rook/rook/blob/master/deploy/examples/import-external-cluster.sh) script.
 
    !!! note
        If your Rook cluster nodes are running a kernel earlier than or equivalent to 5.4, remove
diff --git a/Documentation/Helm-Charts/operator-chart.md b/Documentation/Helm-Charts/operator-chart.md
index d0b7d43af..39dfab6af 100644
--- a/Documentation/Helm-Charts/operator-chart.md
+++ b/Documentation/Helm-Charts/operator-chart.md
@@ -59,7 +59,7 @@ The following table lists the configurable parameters of the rook-operator chart
 | `csi.cephFSKernelMountOptions` | Set CephFS Kernel mount options to use https://docs.ceph.com/en/latest/man/8/mount.ceph/#options. Set to "ms_mode=secure" when connections.encrypted is enabled in CephCluster CR | `nil` |
 | `csi.cephFSPluginUpdateStrategy` | CSI CephFS plugin daemonset update strategy, supported values are OnDelete and RollingUpdate | `RollingUpdate` |
 | `csi.cephFSPluginUpdateStrategyMaxUnavailable` | A maxUnavailable parameter of CSI cephFS plugin daemonset update strategy. | `1` |
-| `csi.cephcsi.image` | Ceph CSI image | `quay.io/cephcsi/cephcsi:v3.10.2` |
+| `csi.cephcsi.image` | Ceph CSI image | `quay.io/cephcsi/cephcsi:v3.10.1` |
 | `csi.cephfsLivenessMetricsPort` | CSI CephFS driver metrics port | `9081` |
 | `csi.cephfsPodLabels` | Labels to add to the CSI CephFS Deployments and DaemonSets Pods | `nil` |
 | `csi.clusterName` | Cluster name identifier to set as metadata on the CephFS subvolume and RBD images. This will be useful in cases like for example, when two container orchestrator clusters (Kubernetes/OCP) are using a single ceph cluster | `nil` |
@@ -115,6 +115,8 @@ The following table lists the configurable parameters of the rook-operator chart
 | `csi.rbdPluginUpdateStrategy` | CSI RBD plugin daemonset update strategy, supported values are OnDelete and RollingUpdate | `RollingUpdate` |
 | `csi.rbdPluginUpdateStrategyMaxUnavailable` | A maxUnavailable parameter of CSI RBD plugin daemonset update strategy. | `1` |
 | `csi.rbdPodLabels` | Labels to add to the CSI RBD Deployments and DaemonSets Pods | `nil` |
+| `csi.readAffinity.crushLocationLabels` | Define which node labels to use as CRUSH location. This should correspond to the values set in the CRUSH map. | labels listed [here](../CRDs/Cluster/ceph-cluster-crd.md#osd-topology) |
+| `csi.readAffinity.enabled` | Enable read affinity for RBD volumes. Recommended to set to true if running kernel 5.8 or newer. | `false` |
 | `csi.registrar.image` | Kubernetes CSI registrar image | `registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.9.1` |
 | `csi.resizer.image` | Kubernetes CSI resizer image | `registry.k8s.io/sig-storage/csi-resizer:v1.9.2` |
 | `csi.serviceMonitor.enabled` | Enable ServiceMonitor for Ceph CSI drivers | `false` |
diff --git a/Documentation/Storage-Configuration/Ceph-CSI/custom-images.md b/Documentation/Storage-Configuration/Ceph-CSI/custom-images.md
index 703f85b45..67bc50084 100644
--- a/Documentation/Storage-Configuration/Ceph-CSI/custom-images.md
+++ b/Documentation/Storage-Configuration/Ceph-CSI/custom-images.md
@@ -18,7 +18,7 @@ kubectl -n $ROOK_OPERATOR_NAMESPACE edit configmap rook-ceph-operator-config
 The default upstream images are included below, which you can change to your desired images.
 
 ```yaml
-ROOK_CSI_CEPH_IMAGE: "quay.io/cephcsi/cephcsi:v3.10.2"
+ROOK_CSI_CEPH_IMAGE: "quay.io/cephcsi/cephcsi:v3.10.1"
 ROOK_CSI_REGISTRAR_IMAGE: "registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.9.1"
 ROOK_CSI_PROVISIONER_IMAGE: "registry.k8s.io/sig-storage/csi-provisioner:v3.6.3"
 ROOK_CSI_ATTACHER_IMAGE: "registry.k8s.io/sig-storage/csi-attacher:v4.4.2"
@@ -32,7 +32,7 @@ ROOK_CSIADDONS_IMAGE: "quay.io/csiaddons/k8s-sidecar:v0.8.0"
 If image version is not passed along with the image name in any of the variables above,
 Rook will add the corresponding default version to that image.
 Example: if `ROOK_CSI_CEPH_IMAGE: "quay.io/private-repo/cephcsi"` is passed,
-Rook will add internal default version and consume it as `"quay.io/private-repo/cephcsi:v3.10.2"`.
+Rook will add internal default version and consume it as `"quay.io/private-repo/cephcsi:v3.10.1"`.
 
 ### **Use default images**
 
diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md
index 609e93295..0fa402039 100644
--- a/PendingReleaseNotes.md
+++ b/PendingReleaseNotes.md
@@ -1,7 +1,6 @@
 # v1.14 Pending Release Notes
 
 ## Breaking Changes
-- The removal of `CSI_ENABLE_READ_AFFINITY` option and its replacement with per-cluster
-read affinity setting in cephCluster CR (CSIDriverOptions section) in [PR](https://github.com/rook/rook/pull/13665)
+
 
 ## Features
diff --git a/deploy/charts/rook-ceph/templates/configmap.yaml b/deploy/charts/rook-ceph/templates/configmap.yaml
index d6af5cc79..ee7e48142 100644
--- a/deploy/charts/rook-ceph/templates/configmap.yaml
+++ b/deploy/charts/rook-ceph/templates/configmap.yaml
@@ -118,6 +118,12 @@ data:
   CSI_TOPOLOGY_DOMAIN_LABELS: {{ .Values.csi.topology.domainLabels | join "," }}
 {{- end }}
 {{- end }}
+{{- if .Values.csi.readAffinity }}
+  CSI_ENABLE_READ_AFFINITY: {{ .Values.csi.readAffinity.enabled | quote }}
+{{- if .Values.csi.readAffinity.crushLocationLabels }}
+  CSI_CRUSH_LOCATION_LABELS: {{ .Values.csi.readAffinity.crushLocationLabels | join "," }}
+{{- end }}
+{{- end }}
 {{- if .Values.csi.nfs }}
   ROOK_CSI_ENABLE_NFS: {{ .Values.csi.nfs.enabled | quote }}
 {{- end }}
diff --git a/deploy/charts/rook-ceph/values.yaml b/deploy/charts/rook-ceph/values.yaml
index 0781cf3b6..2be950904 100644
--- a/deploy/charts/rook-ceph/values.yaml
+++ b/deploy/charts/rook-ceph/values.yaml
@@ -469,7 +469,7 @@ csi:
 
   cephcsi:
     # -- Ceph CSI image
-    # @default -- `quay.io/cephcsi/cephcsi:v3.10.2`
+    # @default -- `quay.io/cephcsi/cephcsi:v3.10.1`
     image:
 
   registrar:
@@ -531,6 +531,17 @@ csi:
     # - topology.kubernetes.io/zone
     # - topology.rook.io/rack
 
+  readAffinity:
+    # -- Enable read affinity for RBD volumes. Recommended to
+    # set to true if running kernel 5.8 or newer.
+    # @default -- `false`
+    enabled: false
+    # -- Define which node labels to use
+    # as CRUSH location. This should correspond to the values set
+    # in the CRUSH map.
+    # @default -- labels listed [here](../CRDs/Cluster/ceph-cluster-crd.md#osd-topology)
+    crushLocationLabels:
+
   # -- Whether to skip any attach operation altogether for CephFS PVCs. See more details
   # [here](https://kubernetes-csi.github.io/docs/skip-attach.html#skip-attach-with-csi-driver-object).
   # If cephFSAttachRequired is set to false it skips the volume attachments and makes the creation
diff --git a/deploy/examples/images.txt b/deploy/examples/images.txt
index 741f75b73..f7e14f7f7 100644
--- a/deploy/examples/images.txt
+++ b/deploy/examples/images.txt
@@ -1,7 +1,7 @@
  gcr.io/k8s-staging-sig-storage/objectstorage-sidecar/objectstorage-sidecar:v20230130-v0.1.0-24-gc0cf995
  quay.io/ceph/ceph:v18.2.1
  quay.io/ceph/cosi:v0.1.1
- quay.io/cephcsi/cephcsi:v3.10.2
+ quay.io/cephcsi/cephcsi:v3.10.1
  quay.io/csiaddons/k8s-sidecar:v0.8.0
  registry.k8s.io/sig-storage/csi-attacher:v4.4.2
  registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.9.1
diff --git a/deploy/examples/import-external-cluster.sh b/deploy/examples/import-external-cluster.sh
index fac910aa1..241f53333 100644
--- a/deploy/examples/import-external-cluster.sh
+++ b/deploy/examples/import-external-cluster.sh
@@ -66,17 +66,11 @@ function checkEnvVars() {
     echo "Providing both ROOK_EXTERNAL_ADMIN_SECRET and ROOK_EXTERNAL_USER_SECRET is not supported, choose one only."
     exit 1
   fi
-  if [ -n "$KUBECONTEXT" ]; then
-    echo "Using $KUBECONTEXT as the context value for kubectl commands"
-    KUBECTL="kubectl --context=$KUBECONTEXT"
-  else
-    KUBECTL="kubectl"
-  fi
 }
 
 function createClusterNamespace() {
-  if ! $KUBECTL get namespace "$NAMESPACE" &>/dev/null; then
-    $KUBECTL \
+  if ! kubectl get namespace "$NAMESPACE" &>/dev/null; then
+    kubectl \
       create \
       namespace \
       "$NAMESPACE"
@@ -86,8 +80,8 @@ function createClusterNamespace() {
 }
 
 function createRadosNamespaceCR() {
-  if ! $KUBECTL -n "$NAMESPACE" get CephBlockPoolRadosNamespace $RADOS_NAMESPACE &>/dev/null; then
-    cat <<eof | $KUBECTL create -f -
+  if ! kubectl -n "$NAMESPACE" get CephBlockPoolRadosNamespace $RADOS_NAMESPACE &>/dev/null; then
+    cat <<eof | kubectl create -f -
 apiVersion: ceph.rook.io/v1
 kind: CephBlockPoolRadosNamespace
 metadata:
@@ -103,8 +97,8 @@ eof
 }
 
 function createSubvolumeGroupCR() {
-  if ! $KUBECTL -n "$NAMESPACE" get CephFilesystemSubVolumeGroup $SUBVOLUME_GROUP &>/dev/null; then
-    cat <<eof | $KUBECTL create -f -
+  if ! kubectl -n "$NAMESPACE" get CephFilesystemSubVolumeGroup $SUBVOLUME_GROUP &>/dev/null; then
+    cat <<eof | kubectl create -f -
 ---
 apiVersion: ceph.rook.io/v1
 kind: CephFilesystemSubVolumeGroup
@@ -123,19 +117,19 @@ eof
 function importClusterID() {
   if [ -n "$RADOS_NAMESPACE" ]; then
     createRadosNamespaceCR
-    timeout 20 sh -c "until [ $($KUBECTL -n "$NAMESPACE" get CephBlockPoolRadosNamespace/"$RADOS_NAMESPACE" -o jsonpath='{.status.phase}' | grep -c "Ready") -eq 1 ]; do echo "waiting for radosNamespace to get created" && sleep 1; done"
-    CLUSTER_ID_RBD=$($KUBECTL -n "$NAMESPACE" get cephblockpoolradosnamespace.ceph.rook.io/"$RADOS_NAMESPACE" -o jsonpath='{.status.info.clusterID}')
+    timeout 20 sh -c "until [ $(kubectl -n "$NAMESPACE" get CephBlockPoolRadosNamespace/"$RADOS_NAMESPACE" -o jsonpath='{.status.phase}' | grep -c "Ready") -eq 1 ]; do echo "waiting for radosNamespace to get created" && sleep 1; done"
+    CLUSTER_ID_RBD=$(kubectl -n "$NAMESPACE" get cephblockpoolradosnamespace.ceph.rook.io/"$RADOS_NAMESPACE" -o jsonpath='{.status.info.clusterID}')
   fi
   if [ -n "$SUBVOLUME_GROUP" ]; then
     createSubvolumeGroupCR
-    timeout 20 sh -c "until [ $($KUBECTL -n "$NAMESPACE" get CephFilesystemSubVolumeGroup/"$SUBVOLUME_GROUP" -o jsonpath='{.status.phase}' | grep -c "Ready") -eq 1 ]; do echo "waiting for radosNamespace to get created" && sleep 1; done"
-    CLUSTER_ID_CEPHFS=$($KUBECTL -n "$NAMESPACE" get cephfilesystemsubvolumegroup.ceph.rook.io/"$SUBVOLUME_GROUP" -o jsonpath='{.status.info.clusterID}')
+    timeout 20 sh -c "until [ $(kubectl -n "$NAMESPACE" get CephFilesystemSubVolumeGroup/"$SUBVOLUME_GROUP" -o jsonpath='{.status.phase}' | grep -c "Ready") -eq 1 ]; do echo "waiting for radosNamespace to get created" && sleep 1; done"
+    CLUSTER_ID_CEPHFS=$(kubectl -n "$NAMESPACE" get cephfilesystemsubvolumegroup.ceph.rook.io/"$SUBVOLUME_GROUP" -o jsonpath='{.status.info.clusterID}')
   fi
 }
 
 function importSecret() {
-  if ! $KUBECTL -n "$NAMESPACE" get secret "$MON_SECRET_NAME" &>/dev/null; then
-    $KUBECTL -n "$NAMESPACE" \
+  if ! kubectl -n "$NAMESPACE" get secret "$MON_SECRET_NAME" &>/dev/null; then
+    kubectl -n "$NAMESPACE" \
       create \
       secret \
       generic \
@@ -153,8 +147,8 @@ function importSecret() {
 }
 
 function importConfigMap() {
-  if ! $KUBECTL -n "$NAMESPACE" get configmap "$MON_ENDPOINT_CONFIGMAP_NAME" &>/dev/null; then
-    $KUBECTL -n "$NAMESPACE" \
+  if ! kubectl -n "$NAMESPACE" get configmap "$MON_ENDPOINT_CONFIGMAP_NAME" &>/dev/null; then
+    kubectl -n "$NAMESPACE" \
       create \
       configmap \
       "$MON_ENDPOINT_CONFIGMAP_NAME" \
@@ -167,8 +161,8 @@ function importConfigMap() {
 }
 
 function importCsiRBDNodeSecret() {
-  if ! $KUBECTL -n "$NAMESPACE" get secret "rook-$CSI_RBD_NODE_SECRET_NAME" &>/dev/null; then
-    $KUBECTL -n "$NAMESPACE" \
+  if ! kubectl -n "$NAMESPACE" get secret "rook-$CSI_RBD_NODE_SECRET_NAME" &>/dev/null; then
+    kubectl -n "$NAMESPACE" \
       create \
       secret \
       generic \
@@ -182,8 +176,8 @@ function importCsiRBDNodeSecret() {
 }
 
 function importCsiRBDProvisionerSecret() {
-  if ! $KUBECTL -n "$NAMESPACE" get secret "rook-$CSI_RBD_PROVISIONER_SECRET_NAME" &>/dev/null; then
-    $KUBECTL -n "$NAMESPACE" \
+  if ! kubectl -n "$NAMESPACE" get secret "rook-$CSI_RBD_PROVISIONER_SECRET_NAME" &>/dev/null; then
+    kubectl -n "$NAMESPACE" \
       create \
       secret \
       generic \
@@ -197,8 +191,8 @@ function importCsiRBDProvisionerSecret() {
 }
 
 function importCsiCephFSNodeSecret() {
-  if ! $KUBECTL -n "$NAMESPACE" get secret "rook-$CSI_CEPHFS_NODE_SECRET_NAME" &>/dev/null; then
-    $KUBECTL -n "$NAMESPACE" \
+  if ! kubectl -n "$NAMESPACE" get secret "rook-$CSI_CEPHFS_NODE_SECRET_NAME" &>/dev/null; then
+    kubectl -n "$NAMESPACE" \
       create \
       secret \
       generic \
@@ -212,8 +206,8 @@ function importCsiCephFSNodeSecret() {
 }
 
 function importCsiCephFSProvisionerSecret() {
-  if ! $KUBECTL -n "$NAMESPACE" get secret "rook-$CSI_CEPHFS_PROVISIONER_SECRET_NAME" &>/dev/null; then
-    $KUBECTL -n "$NAMESPACE" \
+  if ! kubectl -n "$NAMESPACE" get secret "rook-$CSI_CEPHFS_PROVISIONER_SECRET_NAME" &>/dev/null; then
+    kubectl -n "$NAMESPACE" \
       create \
       secret \
       generic \
@@ -227,8 +221,8 @@ function importCsiCephFSProvisionerSecret() {
 }
 
 function importRGWAdminOpsUser() {
-  if ! $KUBECTL -n "$NAMESPACE" get secret "$RGW_ADMIN_OPS_USER_SECRET_NAME" &>/dev/null; then
-    $KUBECTL -n "$NAMESPACE" \
+  if ! kubectl -n "$NAMESPACE" get secret "$RGW_ADMIN_OPS_USER_SECRET_NAME" &>/dev/null; then
+    kubectl -n "$NAMESPACE" \
       create \
       secret \
       generic \
@@ -242,8 +236,8 @@ function importRGWAdminOpsUser() {
 }
 
 function createECRBDStorageClass() {
-  if ! $KUBECTL -n "$NAMESPACE" get storageclass $RBD_STORAGE_CLASS_NAME &>/dev/null; then
-    cat <<eof | $KUBECTL create -f -
+  if ! kubectl -n "$NAMESPACE" get storageclass $RBD_STORAGE_CLASS_NAME &>/dev/null; then
+    cat <<eof | kubectl create -f -
 apiVersion: storage.k8s.io/v1
 kind: StorageClass
 metadata:
@@ -271,8 +265,8 @@ eof
 }
 
 function createRBDStorageClass() {
-  if ! $KUBECTL -n "$NAMESPACE" get storageclass $RBD_STORAGE_CLASS_NAME &>/dev/null; then
-    cat <<eof | $KUBECTL create -f -
+  if ! kubectl -n "$NAMESPACE" get storageclass $RBD_STORAGE_CLASS_NAME &>/dev/null; then
+    cat <<eof | kubectl create -f -
 apiVersion: storage.k8s.io/v1
 kind: StorageClass
 metadata:
@@ -299,8 +293,8 @@ eof
 }
 
 function createCephFSStorageClass() {
-  if ! $KUBECTL -n "$NAMESPACE" get storageclass $CEPHFS_STORAGE_CLASS_NAME &>/dev/null; then
-    cat <<eof | $KUBECTL create -f -
+  if ! kubectl -n "$NAMESPACE" get storageclass $CEPHFS_STORAGE_CLASS_NAME &>/dev/null; then
+    cat <<eof | kubectl create -f -
 apiVersion: storage.k8s.io/v1
 kind: StorageClass
 metadata:
diff --git a/deploy/examples/operator-openshift.yaml b/deploy/examples/operator-openshift.yaml
index 3d6f48049..bb32be5d0 100644
--- a/deploy/examples/operator-openshift.yaml
+++ b/deploy/examples/operator-openshift.yaml
@@ -190,7 +190,7 @@ data:
   # The default version of CSI supported by Rook will be started. To change the version
   # of the CSI driver to something other than what is officially supported, change
   # these images to the desired release of the CSI driver.
-  # ROOK_CSI_CEPH_IMAGE: "quay.io/cephcsi/cephcsi:v3.10.2"
+  # ROOK_CSI_CEPH_IMAGE: "quay.io/cephcsi/cephcsi:v3.10.1"
   # ROOK_CSI_REGISTRAR_IMAGE: "registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.9.1"
   # ROOK_CSI_RESIZER_IMAGE: "registry.k8s.io/sig-storage/csi-resizer:v1.9.2"
   # ROOK_CSI_PROVISIONER_IMAGE: "registry.k8s.io/sig-storage/csi-provisioner:v3.6.3"
@@ -560,6 +560,16 @@ data:
   # updated with node labels that define domains of interest
   # CSI_TOPOLOGY_DOMAIN_LABELS: "kubernetes.io/hostname,topology.kubernetes.io/zone,topology.rook.io/rack"
 
+  # Enable read affinity for RBD volumes. Recommended to
+  # set to true if running kernel 5.8 or newer.
+  CSI_ENABLE_READ_AFFINITY: "false"
+  # CRUSH location labels define which node labels to use
+  # as CRUSH location. This should correspond to the values set in
+  # the CRUSH map.
+  # Defaults to all the labels mentioned in
+  # https://rook.io/docs/rook/latest/CRDs/Cluster/ceph-cluster-crd/#osd-topology
+  # CSI_CRUSH_LOCATION_LABELS: "kubernetes.io/hostname,topology.kubernetes.io/zone,topology.rook.io/rack"
+
   # Whether to skip any attach operation altogether for CephCSI PVCs.
   # See more details [here](https://kubernetes-csi.github.io/docs/skip-attach.html#skip-attach-with-csi-driver-object).
   # If set to false it skips the volume attachments and makes the creation of pods using the CephCSI PVC fast.
diff --git a/deploy/examples/operator.yaml b/deploy/examples/operator.yaml
index b5f16fee8..840481993 100644
--- a/deploy/examples/operator.yaml
+++ b/deploy/examples/operator.yaml
@@ -112,7 +112,7 @@ data:
   # The default version of CSI supported by Rook will be started. To change the version
   # of the CSI driver to something other than what is officially supported, change
   # these images to the desired release of the CSI driver.
-  # ROOK_CSI_CEPH_IMAGE: "quay.io/cephcsi/cephcsi:v3.10.2"
+  # ROOK_CSI_CEPH_IMAGE: "quay.io/cephcsi/cephcsi:v3.10.1"
   # ROOK_CSI_REGISTRAR_IMAGE: "registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.9.1"
   # ROOK_CSI_RESIZER_IMAGE: "registry.k8s.io/sig-storage/csi-resizer:v1.9.2"
   # ROOK_CSI_PROVISIONER_IMAGE: "registry.k8s.io/sig-storage/csi-provisioner:v3.6.3"
@@ -502,6 +502,16 @@ data:
   # updated with node labels that define domains of interest
   # CSI_TOPOLOGY_DOMAIN_LABELS: "kubernetes.io/hostname,topology.kubernetes.io/zone,topology.rook.io/rack"
 
+  # Enable read affinity for RBD volumes. Recommended to
+  # set to true if running kernel 5.8 or newer.
+  CSI_ENABLE_READ_AFFINITY: "false"
+  # CRUSH location labels define which node labels to use
+  # as CRUSH location. This should correspond to the values set in
+  # the CRUSH map.
+  # Defaults to all the labels mentioned in
+  # https://rook.io/docs/rook/latest/CRDs/Cluster/ceph-cluster-crd/#osd-topology
+  # CSI_CRUSH_LOCATION_LABELS: "kubernetes.io/hostname,topology.kubernetes.io/zone,topology.rook.io/rack"
+
   # Whether to skip any attach operation altogether for CephCSI PVCs.
   # See more details [here](https://kubernetes-csi.github.io/docs/skip-attach.html#skip-attach-with-csi-driver-object).
   # If set to false it skips the volume attachments and makes the creation of pods using the CephCSI PVC fast.
diff --git a/go.mod b/go.mod
index d082a906f..ebd7b31a5 100644
--- a/go.mod
+++ b/go.mod
@@ -39,7 +39,7 @@ require (
 	k8s.io/client-go v0.29.1
 	k8s.io/cloud-provider v0.29.1
 	k8s.io/utils v0.0.0-20231127182322-b307cd553661
-	sigs.k8s.io/controller-runtime v0.17.1
+	sigs.k8s.io/controller-runtime v0.17.0
 	sigs.k8s.io/mcs-api v0.1.0
 	sigs.k8s.io/yaml v1.4.0
 )
diff --git a/go.sum b/go.sum
index 605adee08..61150cc2a 100644
--- a/go.sum
+++ b/go.sum
@@ -1475,8 +1475,8 @@ rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.7/go.mod h1:PHgbrJT7lCHcxMU+mDHEm+nx46H4zuuHZkDP6icnhu0=
 sigs.k8s.io/controller-runtime v0.6.1/go.mod h1:XRYBPdbf5XJu9kpS84VJiZ7h/u1hF3gEORz0efEja7A=
-sigs.k8s.io/controller-runtime v0.17.1 h1:V1dQELMGVk46YVXXQUbTFujU7u4DQj6YUj9Rb6cuzz8=
-sigs.k8s.io/controller-runtime v0.17.1/go.mod h1:+MngTvIQQQhfXtwfdGw/UOQ/aIaqsYywfCINOtwMO/s=
+sigs.k8s.io/controller-runtime v0.17.0 h1:fjJQf8Ukya+VjogLO6/bNX9HE6Y2xpsO5+fyS26ur/s=
+sigs.k8s.io/controller-runtime v0.17.0/go.mod h1:+MngTvIQQQhfXtwfdGw/UOQ/aIaqsYywfCINOtwMO/s=
 sigs.k8s.io/controller-tools v0.3.0/go.mod h1:enhtKGfxZD1GFEoMgP8Fdbu+uKQ/cq1/WGJhdVChfvI=
 sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs=
 sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
diff --git a/pkg/operator/ceph/cluster/mgr/spec.go b/pkg/operator/ceph/cluster/mgr/spec.go
index d2167c81e..4a452a70a 100644
--- a/pkg/operator/ceph/cluster/mgr/spec.go
+++ b/pkg/operator/ceph/cluster/mgr/spec.go
@@ -74,7 +74,19 @@ func (c *Cluster) makeDeployment(mgrConfig *mgrConfig) (*apps.Deployment, error)
 	if c.spec.Mgr.Count > 1 {
 		podSpec.Spec.Containers = append(podSpec.Spec.Containers, c.makeMgrSidecarContainer(mgrConfig))
 		matchLabels := controller.AppLabels(AppName, c.clusterInfo.Namespace)
-		podSpec.Spec.Volumes = append(podSpec.Spec.Volumes, mon.CephSecretVolume())
+		// append ceph secret volume and some empty volumes needed by the mgr sidecar
+		podSpec.Spec.Volumes = append(podSpec.Spec.Volumes,
+			v1.Volume{
+				Name: "rook-config",
+				VolumeSource: v1.VolumeSource{
+					EmptyDir: &v1.EmptyDirVolumeSource{},
+				}},
+			v1.Volume{
+				Name: "default-config-dir",
+				VolumeSource: v1.VolumeSource{
+					EmptyDir: &v1.EmptyDirVolumeSource{},
+				}},
+			mon.CephSecretVolume())
 
 		// Stretch the mgrs across hosts by default, or across a bigger failure domain for when zones are required like in case of stretched cluster
 		topologyKey := v1.LabelHostname
@@ -225,6 +237,18 @@ func (c *Cluster) makeMgrSidecarContainer(mgrConfig *mgrConfig) v1.Container {
 		{Name: "ROOK_CEPH_VERSION", Value: "ceph version " + c.clusterInfo.CephVersion.String()},
 	}
 
+	volumeMounts := []v1.VolumeMount{
+		{
+			MountPath: "/var/lib/rook",
+			Name:      "rook-config",
+		},
+		{
+			MountPath: "/etc/ceph",
+			Name:      "default-config-dir",
+		},
+	}
+	volumeMounts = append(volumeMounts, mon.CephSecretVolumeMount())
+
 	return v1.Container{
 		Args:            []string{"ceph", "mgr", "watch-active"},
 		Name:            "watch-active",
@@ -232,8 +256,8 @@ func (c *Cluster) makeMgrSidecarContainer(mgrConfig *mgrConfig) v1.Container {
 		ImagePullPolicy: controller.GetContainerImagePullPolicy(c.spec.CephVersion.ImagePullPolicy),
 		Env:             envVars,
 		Resources:       cephv1.GetMgrSidecarResources(c.spec.Resources),
-		SecurityContext: controller.PrivilegedContext(true),
-		VolumeMounts:    []v1.VolumeMount{mon.CephSecretVolumeMount()},
+		SecurityContext: controller.PodSecurityContext(),
+		VolumeMounts:    volumeMounts,
 	}
 }
 
diff --git a/pkg/operator/ceph/csi/csi.go b/pkg/operator/ceph/csi/csi.go
index 48bc7d8eb..5415ed626 100644
--- a/pkg/operator/ceph/csi/csi.go
+++ b/pkg/operator/ceph/csi/csi.go
@@ -21,6 +21,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/rook/rook/pkg/operator/ceph/cluster/osd/topology"
 	"github.com/rook/rook/pkg/operator/k8sutil"
 
 	"github.com/pkg/errors"
@@ -83,6 +84,11 @@ func (r *ReconcileCSI) setParams(ver *version.Info) error {
 		return errors.Wrap(err, "failed to parse value for 'CSI_ENABLE_HOST_NETWORK'")
 	}
 
+	if CSIParam.EnableReadAffinity, err = strconv.ParseBool(k8sutil.GetValue(r.opConfig.Parameters, "CSI_ENABLE_READ_AFFINITY", "false")); err != nil {
+		return errors.Wrap(err, "failed to parse value for 'CSI_ENABLE_READ_AFFINITY'")
+	}
+	CSIParam.CrushLocationLabels = k8sutil.GetValue(r.opConfig.Parameters, "CSI_CRUSH_LOCATION_LABELS", topology.GetDefaultTopologyLabels())
+
 	// If not set or set to anything but "false", the kernel client will be enabled
 	if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_FORCE_CEPHFS_KERNEL_CLIENT", "true"), "false") {
 		CSIParam.ForceCephFSKernelClient = "false"
diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go
index 4e08c1aaf..f4ef79865 100644
--- a/pkg/operator/ceph/csi/spec.go
+++ b/pkg/operator/ceph/csi/spec.go
@@ -65,6 +65,7 @@ type Param struct {
 	ImagePullPolicy                          string
 	CSIClusterName                           string
 	CSIDomainLabels                          string
+	CrushLocationLabels                      string
 	GRPCTimeout                              time.Duration
 	CSIEnableMetadata                        bool
 	EnablePluginSelinuxHostMount             bool
@@ -79,6 +80,7 @@ type Param struct {
 	EnableCSIEncryption                      bool
 	EnableCSITopology                        bool
 	EnableLiveness                           bool
+	EnableReadAffinity                       bool
 	CephFSAttachRequired                     bool
 	RBDAttachRequired                        bool
 	NFSAttachRequired                        bool
@@ -136,7 +138,7 @@ var (
 // manually challenging.
 var (
 	// image names
-	DefaultCSIPluginImage   = "quay.io/cephcsi/cephcsi:v3.10.2"
+	DefaultCSIPluginImage   = "quay.io/cephcsi/cephcsi:v3.10.1"
 	DefaultRegistrarImage   = "registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.9.1"
 	DefaultProvisionerImage = "registry.k8s.io/sig-storage/csi-provisioner:v3.6.3"
 	DefaultAttacherImage    = "registry.k8s.io/sig-storage/csi-attacher:v4.4.2"
diff --git a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml
index 960598ace..95153f402 100644
--- a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml
+++ b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml
@@ -79,6 +79,10 @@ spec:
             {{- if .EnableCSITopology }}
             - "--domainlabels={{ .CSIDomainLabels }}"
             {{- end }}
+            {{- if .EnableReadAffinity }}
+            - "--enable-read-affinity=true"
+            - "--crush-location-labels={{ .CrushLocationLabels }}"
+            {{- end }}
           env:
             - name: POD_IP
               valueFrom:


Test patch:
diff --git a/.github/workflows/canary-integration-test.yml b/.github/workflows/canary-integration-test.yml
index e200aa74e..af0b3964c 100644
--- a/.github/workflows/canary-integration-test.yml
+++ b/.github/workflows/canary-integration-test.yml
@@ -13,12 +13,11 @@ on:
     paths-ignore:
       - "Documentation/**"
       - "design/**"
-  workflow_call:
+  workflow_dispatch:
     inputs:
       ceph-image:
         description: 'Ceph image for creating Ceph cluster'
         default: 'quay.io/ceph/ceph:v18'
-        type: string
 
 defaults:
   run:
@@ -1405,6 +1404,7 @@ jobs:
           fetch-depth: 0
 
       - name: consider debugging
+
         uses: ./.github/workflows/tmate_debug
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
@@ -1431,6 +1431,7 @@ jobs:
           fetch-depth: 0
 
       - name: consider debugging
+
         uses: ./.github/workflows/tmate_debug
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
@@ -1463,6 +1464,7 @@ jobs:
           fetch-depth: 0
 
       - name: consider debugging
+
         uses: ./.github/workflows/tmate_debug
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
@@ -1544,6 +1546,7 @@ jobs:
           fetch-depth: 0
 
       - name: consider debugging
+
         uses: ./.github/workflows/tmate_debug
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
diff --git a/pkg/operator/ceph/csi/util_test.go b/pkg/operator/ceph/csi/util_test.go
index 1ee73d58c..66ba19e74 100644
--- a/pkg/operator/ceph/csi/util_test.go
+++ b/pkg/operator/ceph/csi/util_test.go
@@ -267,7 +267,7 @@ func Test_getImage(t *testing.T) {
 			args: args{
 				data:         map[string]string{},
 				settingName:  "ROOK_CSI_CEPH_IMAGE",
-				defaultImage: "quay.io/cephcsi/cephcsi:v3.10.2",
+				defaultImage: "quay.io/cephcsi/cephcsi:v3.10.1",
 			},
 			want: DefaultCSIPluginImage,
 		},
@@ -278,7 +278,7 @@ func Test_getImage(t *testing.T) {
 					"ROOK_CSI_CEPH_IMAGE": "registry.io/private/cephcsi:v8",
 				},
 				settingName:  "ROOK_CSI_CEPH_IMAGE",
-				defaultImage: "quay.io/cephcsi/cephcsi:v3.10.2",
+				defaultImage: "quay.io/cephcsi/cephcsi:v3.10.1",
 			},
 			want: "registry.io/private/cephcsi:v8",
 		},
@@ -289,9 +289,9 @@ func Test_getImage(t *testing.T) {
 					"ROOK_CSI_CEPH_IMAGE": "registry.io/private/cephcsi",
 				},
 				settingName:  "ROOK_CSI_CEPH_IMAGE",
-				defaultImage: "quay.io/cephcsi/cephcsi:v3.10.2",
+				defaultImage: "quay.io/cephcsi/cephcsi:v3.10.1",
 			},
-			want: "registry.io/private/cephcsi:v3.10.2",
+			want: "registry.io/private/cephcsi:v3.10.1",
 		},
 	}
 	for _, tt := range tests {
diff --git a/pkg/operator/ceph/csi/version_test.go b/pkg/operator/ceph/csi/version_test.go
index 8877104fc..cf2fa2eb7 100644
--- a/pkg/operator/ceph/csi/version_test.go
+++ b/pkg/operator/ceph/csi/version_test.go
@@ -30,7 +30,6 @@ var (
 	testReleaseV391  = CephCSIVersion{3, 9, 1}
 	testreleasev310  = CephCSIVersion{3, 10, 0}
 	testReleaseV3101 = CephCSIVersion{3, 10, 1}
-	testReleaseV3102 = CephCSIVersion{3, 10, 2}
 
 	testVersionUnsupported = CephCSIVersion{4, 0, 0}
 )
@@ -61,10 +60,6 @@ func TestIsAtLeast(t *testing.T) {
 	ret = testReleaseV3101.isAtLeast(&testReleaseV3101)
 	assert.Equal(t, true, ret)
 
-	// Test for 3.10.2
-	ret = testReleaseV3102.isAtLeast(&testReleaseV3102)
-	assert.Equal(t, true, ret)
-
 }
 
 func TestSupported(t *testing.T) {