Repository: rook/rook (go)

Issue:
CephFS PVCs fail to mount with encryption enabled
**Is this a bug report or feature request?**
* Bug Report

**Deviation from expected behavior:** CephFS PVCs cannot mount if encryption is enabled in the storageclass.

**Expected behavior:** PVCs should mount.

**How to reproduce it (minimal and precise):**
Follow (documentation)[https://rook.io/docs/rook/latest-release/Storage-Configuration/Ceph-CSI/ceph-csi-drivers/?h=encr#enable-rbd-and-cephfs-encryption-support] to enable encryption on a CephFS storageclass. Create a PVC. Create a pod using that PVC. Mounting fails. 

**File(s) to submit**:

cluster.yaml

```
apiVersion: ceph.rook.io/v1
kind: CephCluster
metadata:
  name: cluster
  namespace: rook-ceph
spec:
  dataDirHostPath: /var/lib/rook-init
  cephVersion:
    image: quay.io/ceph/ceph:v19
    allowUnsupported: true
  mon:
    count: 3
    allowMultiplePerNode: false
  mgr:
    count: 1
    allowMultiplePerNode: true
    modules:
      - name: rook
        enabled: true
  dashboard:
    enabled: true
  crashCollector:
    disable: true
  placement:
    all:
      tolerations:
        - key: spacemule.net/low-resource
          operator: Exists
  network:
    connections:
      requireMsgr2: true
  storage:
    useAllNodes: false
    useAllDevices: false
    allowDeviceClassUpdate: true
    allowOsdCrushWeightUpdate: true
    nodes:
      - name: "zimablade-4cpu-16gb-3"
        resources:
          limits:
            memory: 4096Mi
          requests:
            memory: 1024Mi
            cpu: 100m
        devices: # specific devices to use for storage can be specified for each node
          - name: "/dev/disk/by-id/ata-WDC_WD20EZAZ-00L9GB0_WD-WX12AB030D77"
            config:
              deviceClass: hdd
          - name: "/dev/disk/by-id/nvme-KINGSTON_SNV2S1000G_50026B73835A88A6"
            config:
              deviceClass: nvme
      - name: "zimablade-4cpu-16gb-2"
        resources:
          limits:
            memory: 4096Mi
          requests:
            memory: 1024Mi
            cpu: 100m
        devices: # specific devices to use for storage can be specified for each node
          - name: "/dev/disk/by-id/ata-WDC_WD20EZAZ-00GGJB0_WD-WXL2A90L34ET"
            config:
              deviceClass: hdd
          - name: "/dev/disk/by-id/nvme-KINGSTON_SNV2S1000G_50026B73835AAA2C"
            config:
              deviceClass: nvme
      - name: "zimablade-4cpu-16gb-1"
        resources:
          limits:
            memory: 4096Mi
          requests:
            memory: 1024Mi
            cpu: 100m
        devices: # specific devices to use for storage can be specified for each node
          - name: "/dev/disk/by-id/ata-ST4000VN006-3CW104_WW644P2W"
            config:
              deviceClass: hdd
          - name: "/dev/disk/by-id/nvme-KINGSTON_SNV2S1000G_50026B7785B8A552"
            config:
              deviceClass: nvme
      - name: "zimablade-2cpu-8gb-1"
        resources:
          limits:
            memory: 2048Mi
          requests:
            memory: 1024Mi
            cpu: 100m
        devices: # specific devices to use for storage can be specified for each node
          - name: "/dev/disk/by-id/ata-ST4000VN006-3CW104_WW6277CG"
            config:
              deviceClass: hdd
          - name: "/dev/disk/by-id/nvme-KINGSTON_SNV2S1000G_50026B7785BE2DF9"
            config:
              deviceClass: nvme
  monitoring:
    enabled: false
  healthCheck:
    daemonHealth:
      mon:
        interval: 45s
        timeout: 600s
  priorityClassNames:
    all: system-node-critical
    mgr: system-cluster-critical
  disruptionManagement:
    managePodBudgets: true
    pgHealthyRegex: >-
      ^(active\+clean|active\+clean\+scrubbing|active\+clean\+scrubbing\+deep|active\+clean\+remapped)$
  continueUpgradeAfterChecksEvenIfNotHealthy: true
  csi:
    cephfs:
      kernelMountOptions: "ms_mode=prefer-crc"
    readAffinity:
      enabled: true
  cephConfig:
    global:
      osd_pool_default_size: "2"
      mon_warn_on_pool_no_redundancy: "false"
      bdev_flock_retry: "20"
      bluefs_buffered_io: "false"
      mon_data_avail_warn: "10"
      mon_allow_pool_size_one: "true"
  resources:
    mon:
      requests:
        cpu: 100m
        memory: 1024Mi
      limits:
        memory: 2048Mi
```

storage-class.yaml

```
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: rook-ceph-rwx-encrypted
provisioner: rook-ceph.cephfs.csi.ceph.com # csi-provisioner-name
parameters:
  # clusterID is the namespace where the rook cluster is running
  # If you change this namespace, also change the namespace below where the secret namespaces are defined
  clusterID: rook-ceph # namespace:cluster

  # CephFS filesystem name into which the volume shall be created
  fsName: ceph-rwx-encrypted

  # Ceph pool into which the volume shall be created
  # Required for provisionVolume: "true"
  pool: ceph-rwx-encrypted-replicated

  # The secrets contain Ceph admin credentials. These are generated automatically by the operator
  # in the same namespace as the cluster.
  csi.storage.k8s.io/provisioner-secret-name: rook-csi-cephfs-provisioner
  csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph # namespace:cluster
  csi.storage.k8s.io/controller-expand-secret-name: rook-csi-cephfs-provisioner
  csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph # namespace:cluster
  csi.storage.k8s.io/node-stage-secret-name: rook-csi-cephfs-node
  csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph # namespace:cluster
  encrypted: "true"
  encryptionKMSID: "luks-metadata"

  # (optional) The driver can use either ceph-fuse (fuse) or ceph kernel client (kernel)
  # If omitted, default volume mounter will be used - this is determined by probing for ceph-fuse
  # or by setting the default mounter explicitly via --volumemounter command-line argument.
  # mounter: kernel
reclaimPolicy: Delete
allowVolumeExpansion: true
```

KMS config:

```
apiVersion: v1
kind: ConfigMap
metadata:
  name: rook-ceph-csi-kms-config
  namespace: rook-ceph
data:
  config.json: |-
    {
      "luks-metadata": {
        "encryptionKMSType": "metadata",
        "secretName": "luks"
      }
    }
```

**Logs to submit**:

These three lines repeat in the csi-cephplugin pod on the node:

```
E0116 13:03:53.193036       1 fscrypt.go:355] ID: 214 Req-ID: 0001-0009-rook-ceph-0000000000000007-a5d07d13-0431-413e-b95d-e6e28e973446 fscrypt: config file "/etc/fscrypt.conf" already exists. Skipping fscrypt node setup
E0116 13:03:53.194798       1 nodeserver.go:157] ID: 214 Req-ID: 0001-0009-rook-ceph-0000000000000007-a5d07d13-0431-413e-b95d-e6e28e973446 failed to create lock for volume ID 0001-0009-rook-ceph-0000000000000007-a5d07d13-0431-413e-b95d-e6e28e973446: failed to lock volume ID 0001-0009-rook-ceph-0000000000000007-a5d07d13-0431-413e-b95d-e6e28e973446: rados: ret=-1, Operation not permitted
E0116 13:03:53.194888       1 utils.go:271] ID: 214 Req-ID: 0001-0009-rook-ceph-0000000000000007-a5d07d13-0431-413e-b95d-e6e28e973446 GRPC error: rpc error: code = Internal desc = failed to lock volume ID 0001-0009-rook-ceph-0000000000000007-a5d07d13-0431-413e-b95d-e6e28e973446: rados: ret=-1, Operation not permitted

```

The cluster is healthy, and I can create and mount encrypted and unencrypted rbd PVCs as well as unencrypted CephFS PVCs .

**Environment**:
* OS (e.g. from /etc/os-release): openSUSE Tumbleweed (MicroOS)
* Kernel (e.g. `uname -a`): 6.12.8-2
* Cloud provider or hardware configuration: Home cluster on zimablades. 4cpu, 16GB ram
* Rook version (use `rook version` inside of a Rook Pod): v1.16.1
* Storage backend version (e.g. for ceph do `ceph -v`): 18.2.4 (e7ad5345525c7aa95470c26863873b581076945d) reef (stable)
* Kubernetes version (use `kubectl version`): 1.31.4
* Kubernetes cluster type (e.g. Tectonic, GKE, OpenShift): K3s
* Storage backend status (e.g. for Ceph use `ceph health` in the [Rook Ceph toolbox](https://rook.io/docs/rook/latest-release/Troubleshooting/ceph-toolbox/#interactive-toolbox)): HEALTH_OK


Pull request description:
ceph: Update caps to include executable permission
ceph-csi uses an exclusive lock for cryptfs operations. This needs the x capability on the metadata pool.

See also https://github.com/ceph/ceph-csi/issues/4728#issuecomment-2835111872

**Issue resolved by this Pull Request:**
Resolves #15283

**Checklist:**

- [x] **Commit Message Formatting**: Commit titles and messages follow guidelines in the [developer guide](https://rook.io/docs/rook/latest/Contributing/development-flow/#commit-structure).
- [x] Reviewed the developer guide on [Submitting a Pull Request](https://rook.io/docs/rook/latest/Contributing/development-flow/#submitting-a-pull-request)
- [x] [Pending release notes](https://github.com/rook/rook/blob/master/PendingReleaseNotes.md) updated with breaking and/or notable changes for the next minor release.
- [x] Documentation has been updated, if necessary.
- [x] Unit tests have been added, if necessary.
- [x] Integration tests have been added, if necessary.


Patch:
diff --git a/Documentation/Getting-Started/Prerequisites/prerequisites.md b/Documentation/Getting-Started/Prerequisites/prerequisites.md
index 127b2a411..fa040450a 100644
--- a/Documentation/Getting-Started/Prerequisites/prerequisites.md
+++ b/Documentation/Getting-Started/Prerequisites/prerequisites.md
@@ -7,7 +7,7 @@ and Rook is granted the required privileges (see below for more information).
 
 ## Kubernetes Version
 
-Kubernetes versions **v1.28** through **v1.33** are supported.
+Kubernetes versions **v1.28** through **v1.32** are supported.
 
 ## CPU Architecture
 
diff --git a/Documentation/Getting-Started/quickstart.md b/Documentation/Getting-Started/quickstart.md
index 2890c7674..4a03835b1 100644
--- a/Documentation/Getting-Started/quickstart.md
+++ b/Documentation/Getting-Started/quickstart.md
@@ -12,7 +12,7 @@ This guide will walk through the basic setup of a Ceph cluster and enable K8s ap
 
 ## Kubernetes Version
 
-Kubernetes versions **v1.28** through **v1.33** are supported.
+Kubernetes versions **v1.28** through **v1.32** are supported.
 
 ## CPU Architecture
 
diff --git a/pkg/operator/ceph/csi/secrets.go b/pkg/operator/ceph/csi/secrets.go
index 8cbe2646f..86739101f 100644
--- a/pkg/operator/ceph/csi/secrets.go
+++ b/pkg/operator/ceph/csi/secrets.go
@@ -98,7 +98,7 @@ func cephCSIKeyringCephFSNodeCaps() []string {
 	return []string{
 		"mon", "allow r",
 		"mgr", "allow rw",
-		"osd", "allow rw tag cephfs *=*",
+		"osd", "allow rwx tag cephfs metadata=*, allow rw tag cephfs data=*",
 		"mds", "allow rw",
 	}
 }
diff --git a/pkg/operator/ceph/pool/radosnamespace/controller.go b/pkg/operator/ceph/pool/radosnamespace/controller.go
index 49f1be3d6..0d3c34edd 100644
--- a/pkg/operator/ceph/pool/radosnamespace/controller.go
+++ b/pkg/operator/ceph/pool/radosnamespace/controller.go
@@ -383,7 +383,7 @@ func (r *ReconcileCephBlockPoolRadosNamespace) createOrUpdateRadosNamespace(ceph
 	logger.Infof("creating ceph blockpool rados namespace %q", namespacedName)
 
 	if getRadosNamespaceName(cephBlockPoolRadosNamespace) == "" {
-		logger.Info("can't create empty radosnamespace %q in the namespace %q as it is already present", cephBlockPoolRadosNamespace.Name, cephBlockPoolRadosNamespace.Namespace)
+		logger.Info("can't create empty radosnamepace as it is already present.")
 		return nil
 	}
 	err := cephclient.CreateRadosNamespace(r.context, r.clusterInfo, cephBlockPoolRadosNamespace.Spec.BlockPoolName, getRadosNamespaceName(cephBlockPoolRadosNamespace))


Test patch:
diff --git a/.github/workflows/canary-test-config/action.yaml b/.github/workflows/canary-test-config/action.yaml
index d59fcedab..28f08e6c6 100644
--- a/.github/workflows/canary-test-config/action.yaml
+++ b/.github/workflows/canary-test-config/action.yaml
@@ -12,7 +12,7 @@ runs:
     - name: Setup Minikube
       shell: bash --noprofile --norc -eo pipefail -x {0}
       run: |
-        tests/scripts/github-action-helper.sh install_minikube_with_none_driver v1.33.0
+        tests/scripts/github-action-helper.sh install_minikube_with_none_driver v1.32.0
 
     - name: install deps
       shell: bash --noprofile --norc -eo pipefail -x {0}
diff --git a/.github/workflows/integration-test-helm-suite.yaml b/.github/workflows/integration-test-helm-suite.yaml
index 4541593ee..f10a3cfda 100644
--- a/.github/workflows/integration-test-helm-suite.yaml
+++ b/.github/workflows/integration-test-helm-suite.yaml
@@ -28,7 +28,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.28.15", "v1.33.0"]
+        kubernetes-versions: ["v1.28.15", "v1.32.3"]
     steps:
       - name: checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
diff --git a/.github/workflows/integration-test-keystone-auth-suite.yaml b/.github/workflows/integration-test-keystone-auth-suite.yaml
index a0458482d..0e253610e 100644
--- a/.github/workflows/integration-test-keystone-auth-suite.yaml
+++ b/.github/workflows/integration-test-keystone-auth-suite.yaml
@@ -28,7 +28,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.28.15", "v1.33.0"]
+        kubernetes-versions: ["v1.28.15", "v1.32.3"]
     steps:
       - name: checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
diff --git a/.github/workflows/integration-test-mgr-suite.yaml b/.github/workflows/integration-test-mgr-suite.yaml
index 3f67e8a29..3c889669d 100644
--- a/.github/workflows/integration-test-mgr-suite.yaml
+++ b/.github/workflows/integration-test-mgr-suite.yaml
@@ -27,7 +27,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.33.0"]
+        kubernetes-versions: ["v1.32.3"]
     steps:
       - name: checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
diff --git a/.github/workflows/integration-test-multi-cluster-suite.yaml b/.github/workflows/integration-test-multi-cluster-suite.yaml
index 3842f579a..fe1088c24 100644
--- a/.github/workflows/integration-test-multi-cluster-suite.yaml
+++ b/.github/workflows/integration-test-multi-cluster-suite.yaml
@@ -28,7 +28,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.33.0"]
+        kubernetes-versions: ["v1.32.3"]
     steps:
       - name: checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
diff --git a/.github/workflows/integration-test-object-suite.yaml b/.github/workflows/integration-test-object-suite.yaml
index a34370c53..7b6d85a45 100644
--- a/.github/workflows/integration-test-object-suite.yaml
+++ b/.github/workflows/integration-test-object-suite.yaml
@@ -28,7 +28,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.28.15", "v1.33.0"]
+        kubernetes-versions: ["v1.28.15", "v1.32.3"]
     steps:
       - name: checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
diff --git a/.github/workflows/integration-test-smoke-suite.yaml b/.github/workflows/integration-test-smoke-suite.yaml
index 5bb99fd7b..67e1d7c7f 100644
--- a/.github/workflows/integration-test-smoke-suite.yaml
+++ b/.github/workflows/integration-test-smoke-suite.yaml
@@ -28,7 +28,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.28.15", "v1.33.0"]
+        kubernetes-versions: ["v1.28.15", "v1.32.3"]
     steps:
       - name: checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
diff --git a/.github/workflows/integration-test-upgrade-suite.yaml b/.github/workflows/integration-test-upgrade-suite.yaml
index 9715df243..cd7538f5b 100644
--- a/.github/workflows/integration-test-upgrade-suite.yaml
+++ b/.github/workflows/integration-test-upgrade-suite.yaml
@@ -28,7 +28,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.28.15", "v1.33.0"]
+        kubernetes-versions: ["v1.28.15", "v1.32.3"]
     steps:
       - name: checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
@@ -73,7 +73,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.28.15", "v1.33.0"]
+        kubernetes-versions: ["v1.28.15", "v1.32.3"]
     steps:
       - name: checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
diff --git a/.github/workflows/integration-tests-on-release.yaml b/.github/workflows/integration-tests-on-release.yaml
index 1da912d1c..37bf7900b 100644
--- a/.github/workflows/integration-tests-on-release.yaml
+++ b/.github/workflows/integration-tests-on-release.yaml
@@ -21,7 +21,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.28.15", "v1.30.12", "v1.32.4", "v1.33.0"]
+        kubernetes-versions: ["v1.28.15", "v1.29.15", "v1.30.11", "v1.32.3"]
     steps:
       - name: checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
@@ -61,7 +61,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.28.15", "v1.30.12", "v1.32.4", "v1.33.0"]
+        kubernetes-versions: ["v1.28.15", "v1.29.15", "v1.30.11", "v1.32.3"]
     steps:
       - name: checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
@@ -102,7 +102,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.28.15", "v1.30.12", "v1.32.4", "v1.33.0"]
+        kubernetes-versions: ["v1.28.15", "v1.29.15", "v1.30.11", "v1.32.3"]
     steps:
       - name: checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
@@ -140,7 +140,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.28.15", "v1.30.12", "v1.32.4", "v1.33.0"]
+        kubernetes-versions: ["v1.28.15", "v1.29.15", "v1.30.11", "v1.32.3"]
     steps:
       - name: checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
@@ -178,7 +178,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.28.15", "v1.30.12", "v1.32.4", "v1.33.0"]
+        kubernetes-versions: ["v1.28.15", "v1.29.15", "v1.30.11", "v1.32.3"]
     steps:
       - name: checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
@@ -219,7 +219,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.28.15", "v1.33.0"]
+        kubernetes-versions: ["v1.28.15", "v1.32.3"]
     steps:
       - name: checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
diff --git a/pkg/operator/ceph/csi/secrets_test.go b/pkg/operator/ceph/csi/secrets_test.go
index d76b9db2a..7b3d2116a 100644
--- a/pkg/operator/ceph/csi/secrets_test.go
+++ b/pkg/operator/ceph/csi/secrets_test.go
@@ -34,7 +34,7 @@ func TestCephCSIKeyringRBDProvisionerCaps(t *testing.T) {
 
 func TestCephCSIKeyringCephFSNodeCaps(t *testing.T) {
 	caps := cephCSIKeyringCephFSNodeCaps()
-	assert.Equal(t, caps, []string{"mon", "allow r", "mgr", "allow rw", "osd", "allow rw tag cephfs *=*", "mds", "allow rw"})
+	assert.Equal(t, caps, []string{"mon", "allow r", "mgr", "allow rw", "osd", "allow rwx tag cephfs metadata=*, allow rw tag cephfs data=*", "mds", "allow rw"})
 }
 
 func TestCephCSIKeyringCephFSProvisionerCaps(t *testing.T) {
diff --git a/tests/scripts/github-action-helper.sh b/tests/scripts/github-action-helper.sh
index 1bd8e8339..f7d19efea 100755
--- a/tests/scripts/github-action-helper.sh
+++ b/tests/scripts/github-action-helper.sh
@@ -708,8 +708,8 @@ function test_csi_nfs_workload {
 }
 
 function install_minikube_with_none_driver() {
-  CRICTL_VERSION="v1.33.0"
-  MINIKUBE_VERSION="v1.35.0"
+  CRICTL_VERSION="v1.32.0"
+  MINIKUBE_VERSION="v1.34.0"
 
   sudo apt update
   sudo apt install -y conntrack socat
@@ -717,16 +717,16 @@ function install_minikube_with_none_driver() {
   sudo dpkg -i minikube_latest_amd64.deb
   rm -f minikube_latest_amd64.deb
 
-  curl -LO https://github.com/Mirantis/cri-dockerd/releases/download/v0.4.0/cri-dockerd_0.4.0.3-0.ubuntu-focal_amd64.deb
-  sudo dpkg -i cri-dockerd_0.4.0.3-0.ubuntu-focal_amd64.deb
-  rm -f cri-dockerd_0.4.0.3-0.ubuntu-focal_amd64.deb
+  curl -LO https://github.com/Mirantis/cri-dockerd/releases/download/v0.3.15/cri-dockerd_0.3.15.3-0.ubuntu-focal_amd64.deb
+  sudo dpkg -i cri-dockerd_0.3.15.3-0.ubuntu-focal_amd64.deb
+  rm -f cri-dockerd_0.3.15.3-0.ubuntu-focal_amd64.deb
 
   wget https://github.com/kubernetes-sigs/cri-tools/releases/download/$CRICTL_VERSION/crictl-$CRICTL_VERSION-linux-amd64.tar.gz
   sudo tar zxvf crictl-$CRICTL_VERSION-linux-amd64.tar.gz -C /usr/local/bin
   rm -f crictl-$CRICTL_VERSION-linux-amd64.tar.gz
   sudo sysctl fs.protected_regular=0
 
-  CNI_PLUGIN_VERSION="v1.7.1"
+  CNI_PLUGIN_VERSION="v1.5.1"
   CNI_PLUGIN_TAR="cni-plugins-linux-amd64-$CNI_PLUGIN_VERSION.tgz" # change arch if not on amd64
   CNI_PLUGIN_INSTALL_DIR="/opt/cni/bin"