Repository: scylladb/scylla-operator
Pull request #1106 (2022-11-25 10:54:48)

Bump Kubernetes version to 1.24.9 in CI
Currently used minikube with `none` driver requires dockershim coming
from 3rd party provider when Kubernetes version above 1.24 is requested.
The main reason why we used minikube it's storage provisioner which
comes with it. Even though it's not perfect and we have issues with it (https://github.com/scylladb/scylla-operator/issues/974).
Adding new dependency doesn't seem right for our usecase.

This change introduces spawning Kube cluster using dedicated tool `kubeadm`.
It gives us more customizable setup, then the previous one, some of the
existing hacks are now a configuration option.
It also allows us to choose different CRI, in this case I used cri-o.
Missing storage provisioner was replaced with battle tested upstream
kubernetes-sigs/sig-storage-local-static-provisioner.

Fixes https://github.com/scylladb/scylla-operator/issues/1137
Fixes https://github.com/scylladb/scylla-operator/issues/975

Requires:

- [x] https://github.com/scylladb/scylla-operator/pull/1099

mykaul: Need to ensure we also test on the K7S version EKS has. 
mykaul: Unless this is critical, let's defer this to future milestones.
zimnx: > Unless this is critical, let's defer this to future milestones.

Our CI uses 1.23.x which will be considered End of Life in 3 months. 
rzetelskik: Should we also create a tracking issue to switch kubernetes-sigs/sig-storage-local-static-provisioner to https://github.com/scylladb/k8s-local-volume-provisioner once we'll have moved to 1.25?
zimnx: @tnozicka Thanks for detailed review, all your comments are addressed. PTAL

Patch:
diff --git a/.github/actions/setup-kubernetes/action.yaml b/.github/actions/setup-kubernetes/action.yaml
new file mode 100644
index 000000000..5f4f7c294
--- /dev/null
+++ b/.github/actions/setup-kubernetes/action.yaml
@@ -0,0 +1,155 @@
+name: "Setup Kubernetes"
+description: 'Installs Kubernetes cluster'
+inputs:
+  kubernetesVersion:
+    description: "Kubernetes version"
+    default: 1.24.9
+runs:
+  using: "composite"
+  steps:
+  - name: Configure system
+    shell: bash
+    run: |
+      set -euExo pipefail
+      shopt -s inherit_errexit
+      
+      # Github Action workers have DROP on FORWARD policy by default, Kube requires packet forwarding
+      sudo iptables -w -P FORWARD ACCEPT
+      
+      # Required by kubeadm
+      sudo modprobe overlay
+      sudo modprobe br_netfilter
+      sudo swapoff -a
+      echo 'net.ipv4.ip_forward = 1' | sudo tee /etc/sysctl.d/90-kube.conf >/dev/null
+      echo 'net.bridge.bridge-nf-call-iptables  = 1' | sudo tee -a /etc/sysctl.d/90-kube.conf >/dev/null
+      echo 'net.bridge.bridge-nf-call-ip6tables = 1' | sudo tee -a /etc/sysctl.d/90-kube.conf >/dev/null
+      
+      sudo sysctl --system
+
+  - name: Install cri-o
+    shell: bash
+    run: |
+      set -euExo pipefail
+      shopt -s inherit_errexit
+      
+      source /etc/os-release
+      KUBERNETES_VERSION_SHORT=$( echo ${{ inputs.kubernetesVersion }} | cut -d'.' -f1-2 )
+      export KUBERNETES_VERSION_SHORT
+      export KUBERNETES_VERSION=${{ inputs.kubernetesVersion }}
+      
+      sudo apt-get update
+      sudo apt-get install -y apt-transport-https ca-certificates curl gpg
+      
+      sudo mkdir -p /etc/apt/keyrings/
+      
+      curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_${VERSION_ID}/Release.key | sudo gpg --dearmor -o /etc/apt/keyrings/libcontainers-archive-keyring.gpg
+      curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/${KUBERNETES_VERSION_SHORT}/xUbuntu_${VERSION_ID}/Release.key | sudo gpg --dearmor -o /etc/apt/keyrings/libcontainers-crio-archive-keyring.gpg
+      
+      echo "deb [signed-by=/etc/apt/keyrings/libcontainers-archive-keyring.gpg] https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_${VERSION_ID}/ /" | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list
+      echo "deb [signed-by=/etc/apt/keyrings/libcontainers-crio-archive-keyring.gpg] https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/${KUBERNETES_VERSION_SHORT}/xUbuntu_${VERSION_ID}/ /" | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable:cri-o:${KUBERNETES_VERSION_SHORT}.list
+      
+      sudo apt-get update
+      sudo apt-get install -y --no-install-recommends cri-o cri-o-runc
+      
+      cat << EOF | sudo tee /etc/crio/crio.conf.d/02-cgroup-manager.conf
+      [crio.runtime]
+      conmon_cgroup = "pod"
+      cgroup_manager = "systemd"
+      EOF
+      
+      sudo systemctl enable --now crio
+
+  - name: Install kubernetes
+    shell: bash
+    run: |
+      set -euExo pipefail
+      shopt -s inherit_errexit
+      
+      export KUBERNETES_PKG_VERSION=${{ inputs.kubernetesVersion }}-00
+      
+      sudo apt-get update
+      sudo apt-get install -y --no-install-recommends conntrack socat ebtables
+      
+      sudo curl -fsSLo /etc/apt/keyrings/kubernetes-archive-keyring.gpg https://packages.cloud.google.com/apt/doc/apt-key.gpg
+      echo "deb [signed-by=/etc/apt/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | sudo tee /etc/apt/sources.list.d/kubernetes.list
+      sudo apt-get update
+      
+      # Remove conflicting package
+      sudo apt remove containernetworking-plugins
+      
+      sudo apt-get install -y --no-install-recommends kubelet="${KUBERNETES_PKG_VERSION}" kubeadm="${KUBERNETES_PKG_VERSION}" kubectl="${KUBERNETES_PKG_VERSION}"
+
+  - name: Start Kubernetes
+    shell: bash
+    run: |
+      set -euExo pipefail
+      shopt -s inherit_errexit
+      
+      cat << EOF | sudo tee /etc/ssl/certs/audit-policy.yaml > /dev/null
+      apiVersion: audit.k8s.io/v1
+      kind: Policy
+      rules:
+      - level: RequestResponse
+      EOF
+      
+      cat << EOF | sudo tee /root/kubeadm-config.yaml > /dev/null
+      kind: ClusterConfiguration
+      apiVersion: kubeadm.k8s.io/v1beta3
+      kubernetesVersion: v${{ inputs.kubernetesVersion }}
+      apiServer:
+        extraArgs:
+          event-ttl: 24h
+          audit-policy-file: /etc/ssl/certs/audit-policy.yaml
+          audit-log-path: /var/log/kube-apiserver-audit.log
+        extraVolumes:
+        - name: logs
+          hostPath: /var/log
+          mountPath: /var/log
+      ---
+      apiVersion: kubeadm.k8s.io/v1beta3
+      kind: InitConfiguration
+      nodeRegistration:
+        criSocket: "unix:///var/run/crio/crio.sock"
+      ---
+      kind: KubeletConfiguration
+      apiVersion: kubelet.config.k8s.io/v1beta1
+      cgroupDriver: systemd
+      EOF
+      
+      sudo kubeadm init --config=/root/kubeadm-config.yaml
+      
+      mkdir -p "${HOME}/.kube"
+      sudo cat /etc/kubernetes/admin.conf > "${HOME}/.kube/config"
+      sudo chown $( id -u ):$( id -g ) "${HOME}/.kube/config"
+      
+      kubectl version
+      
+      # master taint is deprecated in 1.24 and removed in 1.25
+      # control-plane taint is added in 1.25
+      kubectl taint nodes --selector node-role.kubernetes.io/control-plane= node-role.kubernetes.io/control-plane-
+      kubectl taint nodes --selector node-role.kubernetes.io/control-plane= node-role.kubernetes.io/master-
+      
+      kubectl -n kube-system rollout status --timeout=5m deployment.apps/coredns
+
+  - name: Tweak Kubernetes
+    shell: bash
+    run: |
+      set -euExo pipefail
+      shopt -s inherit_errexit
+      
+      # Allow admin to use ephemeralcontainers
+      kubectl apply --server-side -f - <<EOF
+        apiVersion: rbac.authorization.k8s.io/v1
+        kind: ClusterRole
+        metadata:
+          name: scylladb-e2e:hotfixes
+          labels:
+            rbac.authorization.k8s.io/aggregate-to-admin: "true"
+        rules:
+        - apiGroups:
+          - ""
+          resources:
+          - pods/ephemeralcontainers
+          verbs:
+          - patch
+      EOF
diff --git a/.github/actions/setup-local-volume-provisioner/action.yaml b/.github/actions/setup-local-volume-provisioner/action.yaml
new file mode 100644
index 000000000..710f25c1e
--- /dev/null
+++ b/.github/actions/setup-local-volume-provisioner/action.yaml
@@ -0,0 +1,34 @@
+name: "Setup Local Volume Provisioner"
+description: 'Installs Local Volume Provisioner'
+inputs:
+  repositoryPath:
+    description: "Path to where the project code lives"
+    required: true
+  numberOfPersistentVolumes:
+    description: "Number of persistent volumes that will be provisioned"
+    required: true
+runs:
+  using: "composite"
+  steps:
+  - name: Prepare host directories
+    shell: bash
+    run: |
+      set -euExo pipefail
+      shopt -s inherit_errexit
+      
+      sudo mkdir /mnt/persistent-volumes
+      
+      for i in $( seq 1 "${{ inputs.numberOfPersistentVolumes }}" ); do
+        sudo mkdir "/mnt/persistent-volumes/vol${i}" "/mnt/persistent-volumes/disk_vol${i}"
+        sudo mount --bind "/mnt/persistent-volumes/vol${i}" "/mnt/persistent-volumes/disk_vol${i}"
+      done
+
+  - name: Install Local Static Provisioner
+    shell: bash
+    working-directory: ${{ inputs.repositoryPath }}
+    run: |
+      set -euExo pipefail
+      shopt -s inherit_errexit
+      
+      kubectl -n local-volume-provisioner apply --server-side -f ./.github/actions/setup-local-volume-provisioner/manifests/
+      kubectl -n local-volume-provisioner rollout status --timeout=5m daemonset.apps/local-volume-provisioner
diff --git a/.github/actions/setup-local-volume-provisioner/manifests/00_clusterrole.yaml b/.github/actions/setup-local-volume-provisioner/manifests/00_clusterrole.yaml
new file mode 100644
index 000000000..30745e6ff
--- /dev/null
+++ b/.github/actions/setup-local-volume-provisioner/manifests/00_clusterrole.yaml
@@ -0,0 +1,11 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: local-storage-provisioner-node-clusterrole
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - "nodes"
+  verbs:
+  - "get"
diff --git a/.github/actions/setup-local-volume-provisioner/manifests/00_namespace.yaml b/.github/actions/setup-local-volume-provisioner/manifests/00_namespace.yaml
new file mode 100644
index 000000000..4132337d8
--- /dev/null
+++ b/.github/actions/setup-local-volume-provisioner/manifests/00_namespace.yaml
@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: local-volume-provisioner
diff --git a/.github/actions/setup-local-volume-provisioner/manifests/00_storageclass.yaml b/.github/actions/setup-local-volume-provisioner/manifests/00_storageclass.yaml
new file mode 100644
index 000000000..467faea44
--- /dev/null
+++ b/.github/actions/setup-local-volume-provisioner/manifests/00_storageclass.yaml
@@ -0,0 +1,9 @@
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: local-storage
+  annotations:
+    storageclass.kubernetes.io/is-default-class: "true"
+provisioner: kubernetes.io/no-provisioner
+volumeBindingMode: WaitForFirstConsumer
+reclaimPolicy: Delete
diff --git a/.github/actions/setup-local-volume-provisioner/manifests/10_configmap.yaml b/.github/actions/setup-local-volume-provisioner/manifests/10_configmap.yaml
new file mode 100644
index 000000000..b068523b4
--- /dev/null
+++ b/.github/actions/setup-local-volume-provisioner/manifests/10_configmap.yaml
@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: local-provisioner-config
+data:
+  storageClassMap: |
+    local-storage:
+      hostDir: /mnt/persistent-volumes
+      mountDir: /mnt/persistent-volumes
+      blockCleanerCommand:
+      - "/scripts/shred.sh"
+      - "2"
+      volumeMode: Filesystem
+      fsType: ext4
+      namePattern: "disk_*"
diff --git a/.github/actions/setup-local-volume-provisioner/manifests/10_serviceaccount.yaml b/.github/actions/setup-local-volume-provisioner/manifests/10_serviceaccount.yaml
new file mode 100644
index 000000000..b3841b4c3
--- /dev/null
+++ b/.github/actions/setup-local-volume-provisioner/manifests/10_serviceaccount.yaml
@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: local-storage-admin
diff --git a/.github/actions/setup-local-volume-provisioner/manifests/20_node.clusterrolebinding.yaml b/.github/actions/setup-local-volume-provisioner/manifests/20_node.clusterrolebinding.yaml
new file mode 100644
index 000000000..a1e6046fa
--- /dev/null
+++ b/.github/actions/setup-local-volume-provisioner/manifests/20_node.clusterrolebinding.yaml
@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: local-storage-provisioner-node-binding
+subjects:
+- kind: ServiceAccount
+  name: local-storage-admin
+  namespace: local-volume-provisioner
+roleRef:
+  kind: ClusterRole
+  name: local-storage-provisioner-node-clusterrole
+  apiGroup: rbac.authorization.k8s.io
diff --git a/.github/actions/setup-local-volume-provisioner/manifests/20_persistent-volume-provisioner.clusterrolebinding.yaml b/.github/actions/setup-local-volume-provisioner/manifests/20_persistent-volume-provisioner.clusterrolebinding.yaml
new file mode 100644
index 000000000..f0b09734a
--- /dev/null
+++ b/.github/actions/setup-local-volume-provisioner/manifests/20_persistent-volume-provisioner.clusterrolebinding.yaml
@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: local-storage-provisioner-pv-binding
+subjects:
+- kind: ServiceAccount
+  name: local-storage-admin
+  namespace: local-volume-provisioner
+roleRef:
+  kind: ClusterRole
+  name: system:persistent-volume-provisioner
+  apiGroup: rbac.authorization.k8s.io
diff --git a/.github/actions/setup-local-volume-provisioner/manifests/50_daemonset.yaml b/.github/actions/setup-local-volume-provisioner/manifests/50_daemonset.yaml
new file mode 100644
index 000000000..6b527d3fd
--- /dev/null
+++ b/.github/actions/setup-local-volume-provisioner/manifests/50_daemonset.yaml
@@ -0,0 +1,46 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: local-volume-provisioner
+  labels:
+    app: local-volume-provisioner
+spec:
+  selector:
+    matchLabels:
+      app: local-volume-provisioner
+  template:
+    metadata:
+      labels:
+        app: local-volume-provisioner
+    spec:
+      serviceAccountName: local-storage-admin
+      containers:
+      - image: "registry.k8s.io/sig-storage/local-volume-provisioner:v2.5.0"
+        imagePullPolicy: "Always"
+        name: provisioner
+        securityContext:
+          privileged: true
+        env:
+        - name: MY_NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        volumeMounts:
+        - mountPath: /etc/provisioner/config
+          name: provisioner-config
+          readOnly: true
+        - mountPath: /mnt/persistent-volumes
+          name: persistent-volumes
+          mountPropagation: "HostToContainer"
+        - mountPath: /dev
+          name: provisioner-dev
+      volumes:
+      - name: provisioner-config
+        configMap:
+          name: local-provisioner-config
+      - name: persistent-volumes
+        hostPath:
+          path: /mnt/persistent-volumes
+      - name: provisioner-dev
+        hostPath:
+          path: /dev
diff --git a/.github/actions/setup-minikube/action.yaml b/.github/actions/setup-minikube/action.yaml
deleted file mode 100644
index fec4c9f17..000000000
--- a/.github/actions/setup-minikube/action.yaml
+++ /dev/null
@@ -1,122 +0,0 @@
-name: "Setup Minikube"
-description: 'Installs minikube Kubernetes cluster'
-runs:
-  using: "composite"
-  steps:
-  - name: Configure system
-    shell: bash
-    run: |
-      set -euExo pipefail
-
-      echo 'fs.aio-max-nr = 10485760' | sudo tee /etc/sysctl.d/90-scylla.conf >/dev/null
-      sudo sysctl --system
-  - name: Install minikube
-    shell: bash
-    run: |
-      set -x
-      sudo apt-get install -y --no-install-recommends conntrack
-      sudo curl -s --fail -L https://storage.googleapis.com/minikube/releases/v1.25.2/minikube-linux-amd64 -o /usr/local/bin/minikube
-      sudo chmod +x /usr/local/bin/minikube
-
-      sudo curl -s --fail -L "https://dl.k8s.io/release/v1.23.0/bin/linux/amd64/kubectl" -o /usr/local/bin/kubectl
-      sudo chmod +x /usr/local/bin/kubectl
-      sudo curl -s --fail -L https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.23.0/crictl-v1.23.0-linux-amd64.tar.gz | tar -f - -xz -C /usr/local/bin crictl
-  - name: Configure minikube
-    shell: bash
-    run: |
-      set -x
-      sudo minikube config set driver none
-      sudo minikube config set EmbedCerts true
-      sudo ip link set docker0 promisc on
-  - name: Start minikube
-    shell: bash
-    run: |
-      set -euEx -o pipefail
-
-      # Hack /etc/ssl/certs/ as it's passed down to the KAS static pod.
-      cat << EOF | sudo tee /etc/ssl/certs/audit-policy.yaml > /dev/null
-      apiVersion: audit.k8s.io/v1
-      kind: Policy
-      rules:
-      - level: RequestResponse
-      EOF
-
-      sudo minikube start --wait=all --extra-config='apiserver.event-ttl=24h' --extra-config='apiserver.audit-policy-file=/etc/ssl/certs/audit-policy.yaml' --extra-config='apiserver.audit-log-path=/var/log/kube-apiserver-audit.log'
-
-      mkdir -p ~/.kube/
-      sudo cat /root/.kube/config > ~/.kube/config
-      sudo minikube version
-      kubectl version
-
-      # We can't hack an existing dir for logs because they are read only.
-      sudo $( which yq ) -i e '.spec.volumes += {"name": "logs","hostPath": {"path": "/var/log"}} | .spec.containers[0].volumeMounts += {"name": "logs", "mountPath": "/var/log"}' /etc/kubernetes/manifests/kube-apiserver.yaml
-
-      # We need to wait for the apiserver to be restarted.
-      until sudo test -f /var/log/kube-apiserver-audit.log; do
-        echo 'Waiting for /var/log/kube-apiserver-audit.log to be created.'
-        sleep 1;
-      done
-
-      until kubectl get namespace kube-system 2>/dev/null 1>&2; do
-        echo 'Waiting for apiserver to come up.'
-        sleep 1
-      done
-
-      # Hotfix pods/ephemeralcontainers perms needed by e2e tests
-      # (Upstream fix in https://github.com/kubernetes/kubernetes/pull/109332)
-      kubectl apply --server-side -f - <<EOF
-        apiVersion: rbac.authorization.k8s.io/v1
-        kind: ClusterRole
-        metadata:
-          name: scylladb-e2e:hotfixes
-          labels:
-            rbac.authorization.k8s.io/aggregate-to-admin: "true"
-        rules:
-        - apiGroups:
-          - ""
-          resources:
-          - pods/ephemeralcontainers
-          verbs:
-          - patch
-      EOF
-
-      kubectl -n kube-system rollout status --timeout=5m deployment.apps/coredns
-      
-      # DaemonSet resposible for removing stale TCP conntrack entries causing issues in gossip.
-      # https://github.com/scylladb/scylla-operator/issues/975
-      # TODO: remove when GitHub Actions upgrades to kernel version 5.14.
-      kubectl -n kube-system create -f - <<EOF
-        apiVersion: apps/v1
-        kind: DaemonSet
-        metadata:
-          name: conntrack-cleaner-agent
-          namespace: kube-system
-        spec:
-          selector:
-            matchLabels:
-              k8s-app: conntrack-cleaner-agent
-          template:
-            metadata:
-              labels:
-                k8s-app: conntrack-cleaner-agent
-            spec:
-              hostNetwork: true
-              containers:
-              - name: conntrack-cleaner-agent
-                image: docker.io/hanamantagoud/conntrack-cleaner@sha256:8e57d55bd7672166f2f1ae5f2f6fb7b1945ab5e97959200d71f0963218c3e771
-                env:
-                # Values for CONNTRACK_TABLE_DUMP_FREQUENCY should contain duration such as 1s,100ms.
-                # NOTE: Dont set values in nonosecond (ns), microsecond(ms).
-                - name: CONNTRACK_TABLE_DUMP_FREQUENCY
-                  value: "1s"
-                - name: CONNECTION_RENEWAL_THRESHOLD
-                  value: "3"
-                securityContext:
-                  privileged: false
-                  capabilities:
-                    add: ["NET_ADMIN", "NET_RAW"]
-      EOF
-      
-      kubectl -n kube-system rollout status --timeout=5m daemonset.apps/conntrack-cleaner-agent
-      
-      kubectl label nodes scylla.scylladb.com/node-type=scylla --all
diff --git a/.github/workflows/go.yaml b/.github/workflows/go.yaml
index 62ade3ea5..cf4ed4e79 100644
--- a/.github/workflows/go.yaml
+++ b/.github/workflows/go.yaml
@@ -235,7 +235,7 @@ jobs:
     runs-on: ubuntu-20.04
     needs: images
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
       with:
         path: ${{ env.git_repo_path }}
         fetch-depth: 0  # also fetch tags
@@ -252,7 +252,7 @@ jobs:
     runs-on: ubuntu-20.04
     needs: images
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
       with:
         path: ${{ env.git_repo_path }}
         fetch-depth: 0  # also fetch tags
diff --git a/pkg/pointer/pointer.go b/pkg/pointer/pointer.go
index aa0899eff..d09ffac07 100644
--- a/pkg/pointer/pointer.go
+++ b/pkg/pointer/pointer.go
@@ -13,3 +13,7 @@ func KeyUsage(u x509.KeyUsage) *x509.KeyUsage {
 func Time(t metav1.Time) *metav1.Time {
 	return &t
 }
+
+func Ptr[T any](v T) *T {
+	return &v
+}
diff --git a/vendor/k8s.io/component-helpers/storage/volume/helpers.go b/vendor/k8s.io/component-helpers/storage/volume/helpers.go
new file mode 100644
index 000000000..7ec376f34
--- /dev/null
+++ b/vendor/k8s.io/component-helpers/storage/volume/helpers.go
@@ -0,0 +1,70 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package volume
+
+import (
+	"fmt"
+
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/component-helpers/scheduling/corev1"
+)
+
+// GetPersistentVolumeClaimClass returns StorageClassName. If no storage class was
+// requested, it returns "".
+func GetPersistentVolumeClaimClass(claim *v1.PersistentVolumeClaim) string {
+	// Use beta annotation first
+	if class, found := claim.Annotations[v1.BetaStorageClassAnnotation]; found {
+		return class
+	}
+
+	if claim.Spec.StorageClassName != nil {
+		return *claim.Spec.StorageClassName
+	}
+
+	return ""
+}
+
+// GetPersistentVolumeClass returns StorageClassName.
+func GetPersistentVolumeClass(volume *v1.PersistentVolume) string {
+	// Use beta annotation first
+	if class, found := volume.Annotations[v1.BetaStorageClassAnnotation]; found {
+		return class
+	}
+
+	return volume.Spec.StorageClassName
+}
+
+// CheckNodeAffinity looks at the PV node affinity, and checks if the node has the same corresponding labels
+// This ensures that we don't mount a volume that doesn't belong to this node
+func CheckNodeAffinity(pv *v1.PersistentVolume, nodeLabels map[string]string) error {
+	if pv.Spec.NodeAffinity == nil {
+		return nil
+	}
+
+	if pv.Spec.NodeAffinity.Required != nil {
+		node := &v1.Node{ObjectMeta: metav1.ObjectMeta{Labels: nodeLabels}}
+		terms := pv.Spec.NodeAffinity.Required
+		if matches, err := corev1.MatchNodeSelectorTerms(node, terms); err != nil {
+			return err
+		} else if !matches {
+			return fmt.Errorf("no matching NodeSelectorTerms")
+		}
+	}
+
+	return nil
+}
diff --git a/vendor/k8s.io/component-helpers/storage/volume/pv_helpers.go b/vendor/k8s.io/component-helpers/storage/volume/pv_helpers.go
new file mode 100644
index 000000000..f927b7231
--- /dev/null
+++ b/vendor/k8s.io/component-helpers/storage/volume/pv_helpers.go
@@ -0,0 +1,342 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package volume
+
+import (
+	"fmt"
+
+	v1 "k8s.io/api/core/v1"
+	storage "k8s.io/api/storage/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/client-go/kubernetes/scheme"
+	storagelisters "k8s.io/client-go/listers/storage/v1"
+	"k8s.io/client-go/tools/reference"
+)
+
+const (
+	// AnnBindCompleted Annotation applies to PVCs. It indicates that the lifecycle
+	// of the PVC has passed through the initial setup. This information changes how
+	// we interpret some observations of the state of the objects. Value of this
+	// Annotation does not matter.
+	AnnBindCompleted = "pv.kubernetes.io/bind-completed"
+
+	// AnnBoundByController annotation applies to PVs and PVCs. It indicates that
+	// the binding (PV->PVC or PVC->PV) was installed by the controller. The
+	// absence of this annotation means the binding was done by the user (i.e.
+	// pre-bound). Value of this annotation does not matter.
+	// External PV binders must bind PV the same way as PV controller, otherwise PV
+	// controller may not handle it correctly.
+	AnnBoundByController = "pv.kubernetes.io/bound-by-controller"
+
+	// AnnSelectedNode annotation is added to a PVC that has been triggered by scheduler to
+	// be dynamically provisioned. Its value is the name of the selected node.
+	AnnSelectedNode = "volume.kubernetes.io/selected-node"
+
+	// NotSupportedProvisioner is a special provisioner name which can be set
+	// in storage class to indicate dynamic provisioning is not supported by
+	// the storage.
+	NotSupportedProvisioner = "kubernetes.io/no-provisioner"
+
+	// AnnDynamicallyProvisioned annotation is added to a PV that has been dynamically provisioned by
+	// Kubernetes. Its value is name of volume plugin that created the volume.
+	// It serves both user (to show where a PV comes from) and Kubernetes (to
+	// recognize dynamically provisioned PVs in its decisions).
+	AnnDynamicallyProvisioned = "pv.kubernetes.io/provisioned-by"
+
+	// AnnMigratedTo annotation is added to a PVC and PV that is supposed to be
+	// dynamically provisioned/deleted by by its corresponding CSI driver
+	// through the CSIMigration feature flags. When this annotation is set the
+	// Kubernetes components will "stand-down" and the external-provisioner will
+	// act on the objects
+	AnnMigratedTo = "pv.kubernetes.io/migrated-to"
+
+	// AnnStorageProvisioner annotation is added to a PVC that is supposed to be dynamically
+	// provisioned. Its value is name of volume plugin that is supposed to provision
+	// a volume for this PVC.
+	// TODO: remove beta anno once deprecation period ends
+	AnnStorageProvisioner     = "volume.kubernetes.io/storage-provisioner"
+	AnnBetaStorageProvisioner = "volume.beta.kubernetes.io/storage-provisioner"
+
+	//PVDeletionProtectionFinalizer is the finalizer added by the external-provisioner on the PV
+	PVDeletionProtectionFinalizer = "external-provisioner.volume.kubernetes.io/finalizer"
+
+	// PVDeletionInTreeProtectionFinalizer is the finalizer added to protect PV deletion for in-tree volumes.
+	PVDeletionInTreeProtectionFinalizer = "kubernetes.io/pv-controller"
+)
+
+// IsDelayBindingProvisioning checks if claim provisioning with selected-node annotation
+func IsDelayBindingProvisioning(claim *v1.PersistentVolumeClaim) bool {
+	// When feature VolumeScheduling enabled,
+	// Scheduler signal to the PV controller to start dynamic
+	// provisioning by setting the "AnnSelectedNode" annotation
+	// in the PVC
+	_, ok := claim.Annotations[AnnSelectedNode]
+	return ok
+}
+
+// IsDelayBindingMode checks if claim is in delay binding mode.
+func IsDelayBindingMode(claim *v1.PersistentVolumeClaim, classLister storagelisters.StorageClassLister) (bool, error) {
+	className := GetPersistentVolumeClaimClass(claim)
+	if className == "" {
+		return false, nil
+	}
+
+	class, err := classLister.Get(className)
+	if err != nil {
+		if apierrors.IsNotFound(err) {
+			return false, nil
+		}
+		return false, err
+	}
+
+	if class.VolumeBindingMode == nil {
+		return false, fmt.Errorf("VolumeBindingMode not set for StorageClass %q", className)
+	}
+
+	return *class.VolumeBindingMode == storage.VolumeBindingWaitForFirstConsumer, nil
+}
+
+// GetBindVolumeToClaim returns a new volume which is bound to given claim. In
+// addition, it returns a bool which indicates whether we made modification on
+// original volume.
+func GetBindVolumeToClaim(volume *v1.PersistentVolume, claim *v1.PersistentVolumeClaim) (*v1.PersistentVolume, bool, error) {
+	dirty := false
+
+	// Check if the volume was already bound (either by user or by controller)
+	shouldSetBoundByController := false
+	if !IsVolumeBoundToClaim(volume, claim) {
+		shouldSetBoundByController = true
+	}
+
+	// The volume from method args can be pointing to watcher cache. We must not
+	// modify these, therefore create a copy.
+	volumeClone := volume.DeepCopy()
+
+	// Bind the volume to the claim if it is not bound yet
+	if volume.Spec.ClaimRef == nil ||
+		volume.Spec.ClaimRef.Name != claim.Name ||
+		volume.Spec.ClaimRef.Namespace != claim.Namespace ||
+		volume.Spec.ClaimRef.UID != claim.UID {
+
+		claimRef, err := reference.GetReference(scheme.Scheme, claim)
+		if err != nil {
+			return nil, false, fmt.Errorf("unexpected error getting claim reference: %w", err)
+		}
+		volumeClone.Spec.ClaimRef = claimRef
+		dirty = true
+	}
+
+	// Set AnnBoundByController if it is not set yet
+	if shouldSetBoundByController && !metav1.HasAnnotation(volumeClone.ObjectMeta, AnnBoundByController) {
+		metav1.SetMetaDataAnnotation(&volumeClone.ObjectMeta, AnnBoundByController, "yes")
+		dirty = true
+	}
+
+	return volumeClone, dirty, nil
+}
+
+// IsVolumeBoundToClaim returns true, if given volume is pre-bound or bound
+// to specific claim. Both claim.Name and claim.Namespace must be equal.
+// If claim.UID is present in volume.Spec.ClaimRef, it must be equal too.
+func IsVolumeBoundToClaim(volume *v1.PersistentVolume, claim *v1.PersistentVolumeClaim) bool {
+	if volume.Spec.ClaimRef == nil {
+		return false
+	}
+	if claim.Name != volume.Spec.ClaimRef.Name || claim.Namespace != volume.Spec.ClaimRef.Namespace {
+		return false
+	}
+	if volume.Spec.ClaimRef.UID != "" && claim.UID != volume.Spec.ClaimRef.UID {
+		return false
+	}
+	return true
+}
+
+// FindMatchingVolume goes through the list of volumes to find the best matching volume
+// for the claim.
+//
+// This function is used by both the PV controller and scheduler.
+//
+// delayBinding is true only in the PV controller path.  When set, prebound PVs are still returned
+// as a match for the claim, but unbound PVs are skipped.
+//
+// node is set only in the scheduler path. When set, the PV node affinity is checked against
+// the node's labels.
+//
+// excludedVolumes is only used in the scheduler path, and is needed for evaluating multiple
+// unbound PVCs for a single Pod at one time.  As each PVC finds a matching PV, the chosen
+// PV needs to be excluded from future matching.
+func FindMatchingVolume(
+	claim *v1.PersistentVolumeClaim,
+	volumes []*v1.PersistentVolume,
+	node *v1.Node,
+	excludedVolumes map[string]*v1.PersistentVolume,
+	delayBinding bool) (*v1.PersistentVolume, error) {
+
+	var smallestVolume *v1.PersistentVolume
+	var smallestVolumeQty resource.Quantity
+	requestedQty := claim.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)]
+	requestedClass := GetPersistentVolumeClaimClass(claim)
+
+	var selector labels.Selector
+	if claim.Spec.Selector != nil {
+		internalSelector, err := metav1.LabelSelectorAsSelector(claim.Spec.Selector)
+		if err != nil {
+			return nil, fmt.Errorf("error creating internal label selector for claim: %v: %v", claimToClaimKey(claim), err)
+		}
+		selector = internalSelector
+	}
+
+	// Go through all available volumes with two goals:
+	// - find a volume that is either pre-bound by user or dynamically
+	//   provisioned for this claim. Because of this we need to loop through
+	//   all volumes.
+	// - find the smallest matching one if there is no volume pre-bound to
+	//   the claim.
+	for _, volume := range volumes {
+		if _, ok := excludedVolumes[volume.Name]; ok {
+			// Skip volumes in the excluded list
+			continue
+		}
+		if volume.Spec.ClaimRef != nil && !IsVolumeBoundToClaim(volume, claim) {
+			continue
+		}
+
+		volumeQty := volume.Spec.Capacity[v1.ResourceStorage]
+		if volumeQty.Cmp(requestedQty) < 0 {
+			continue
+		}
+		// filter out mismatching volumeModes
+		if CheckVolumeModeMismatches(&claim.Spec, &volume.Spec) {
+			continue
+		}
+
+		// check if PV's DeletionTimeStamp is set, if so, skip this volume.
+		if volume.ObjectMeta.DeletionTimestamp != nil {
+			continue
+		}
+
+		nodeAffinityValid := true
+		if node != nil {
+			// Scheduler path, check that the PV NodeAffinity
+			// is satisfied by the node
+			// CheckNodeAffinity is the most expensive call in this loop.
+			// We should check cheaper conditions first or consider optimizing this function.
+			err := CheckNodeAffinity(volume, node.Labels)
+			if err != nil {
+				nodeAffinityValid = false
+			}
+		}
+
+		if IsVolumeBoundToClaim(volume, claim) {
+			// If PV node affinity is invalid, return no match.
+			// This means the prebound PV (and therefore PVC)
+			// is not suitable for this node.
+			if !nodeAffinityValid {
+				return nil, nil
+			}
+
+			return volume, nil
+		}
+
+		if node == nil && delayBinding {
+			// PV controller does not bind this claim.
+			// Scheduler will handle binding unbound volumes
+			// Scheduler path will have node != nil
+			continue
+		}
+
+		// filter out:
+		// - volumes in non-available phase
+		// - volumes whose labels don't match the claim's selector, if specified
+		// - volumes in Class that is not requested
+		// - volumes whose NodeAffinity does not match the node
+		if volume.Status.Phase != v1.VolumeAvailable {
+			// We ignore volumes in non-available phase, because volumes that
+			// satisfies matching criteria will be updated to available, binding
+			// them now has high chance of encountering unnecessary failures
+			// due to API conflicts.
+			continue
+		} else if selector != nil && !selector.Matches(labels.Set(volume.Labels)) {
+			continue
+		}
+		if GetPersistentVolumeClass(volume) != requestedClass {
+			continue
+		}
+		if !nodeAffinityValid {
+			continue
+		}
+
+		if node != nil {
+			// Scheduler path
+			// Check that the access modes match
+			if !CheckAccessModes(claim, volume) {
+				continue
+			}
+		}
+
+		if smallestVolume == nil || smallestVolumeQty.Cmp(volumeQty) > 0 {
+			smallestVolume = volume
+			smallestVolumeQty = volumeQty
+		}
+	}
+
+	if smallestVolume != nil {
+		// Found a matching volume
+		return smallestVolume, nil
+	}
+
+	return nil, nil
+}
+
+// CheckVolumeModeMismatches is a convenience method that checks volumeMode for PersistentVolume
+// and PersistentVolumeClaims
+func CheckVolumeModeMismatches(pvcSpec *v1.PersistentVolumeClaimSpec, pvSpec *v1.PersistentVolumeSpec) bool {
+	// In HA upgrades, we cannot guarantee that the apiserver is on a version >= controller-manager.
+	// So we default a nil volumeMode to filesystem
+	requestedVolumeMode := v1.PersistentVolumeFilesystem
+	if pvcSpec.VolumeMode != nil {
+		requestedVolumeMode = *pvcSpec.VolumeMode
+	}
+	pvVolumeMode := v1.PersistentVolumeFilesystem
+	if pvSpec.VolumeMode != nil {
+		pvVolumeMode = *pvSpec.VolumeMode
+	}
+	return requestedVolumeMode != pvVolumeMode
+}
+
+// CheckAccessModes returns true if PV satisfies all the PVC's requested AccessModes
+func CheckAccessModes(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) bool {
+	pvModesMap := map[v1.PersistentVolumeAccessMode]bool{}
+	for _, mode := range volume.Spec.AccessModes {
+		pvModesMap[mode] = true
+	}
+
+	for _, mode := range claim.Spec.AccessModes {
+		_, ok := pvModesMap[mode]
+		if !ok {
+			return false
+		}
+	}
+	return true
+}
+
+func claimToClaimKey(claim *v1.PersistentVolumeClaim) string {
+	return fmt.Sprintf("%s/%s", claim.Namespace, claim.Name)
+}
diff --git a/vendor/modules.txt b/vendor/modules.txt
index 88bf0e31f..9c8994440 100644
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@@ -1187,6 +1187,7 @@ k8s.io/component-base/version
 ## explicit; go 1.19
 k8s.io/component-helpers/scheduling/corev1
 k8s.io/component-helpers/scheduling/corev1/nodeaffinity
+k8s.io/component-helpers/storage/volume
 # k8s.io/cri-api v0.25.3
 ## explicit; go 1.19
 k8s.io/cri-api/pkg/apis/runtime/v1alpha2


Test patch:
diff --git a/.github/actions/run-e2e/action.yaml b/.github/actions/run-e2e/action.yaml
index f7ddc26ae..06372f6f2 100644
--- a/.github/actions/run-e2e/action.yaml
+++ b/.github/actions/run-e2e/action.yaml
@@ -50,23 +50,33 @@ runs:
       set -euExo pipefail
       shopt -s inherit_errexit
       
-      unlz4 ~/operatorimage.tar.lz4 - | docker load
-      # docker looses the registry part on save/load
-      docker tag "$( echo "${image_repo_ref}:ci" | sed -E -e 's~[^/]+/(.*)~\1~' )" "${image_repo_ref}:ci"
-      docker images '${{ env.image_repo_ref }}:ci'
+      unlz4 ~/operatorimage.tar.lz4 - | sudo podman load
+      
+      sudo podman images '${{ env.image_repo_ref }}:ci'
   - name: Setup go
     uses: actions/setup-go@v3
     with:
       go-version: ${{ env.go_version }}
-  - name: Install tools
+  - name: Setup Kubernetes
+    uses: ./go/src/github.com/scylladb/scylla-operator/.github/actions/setup-kubernetes
+  - name: Setup Local Volume Provisioner
+    uses: ./go/src/github.com/scylladb/scylla-operator/.github/actions/setup-local-volume-provisioner
+    with:
+      repositoryPath: ${{ inputs.repositoryPath }}
+      numberOfPersistentVolumes: 100
+  - name: Tweak node and Kubernetes
     shell: bash
     run: |
       set -euExo pipefail
       shopt -s inherit_errexit
       
-      go install github.com/mikefarah/yq/v4@v4.6.1
-  - name: Setup minikube
-    uses: ./go/src/github.com/scylladb/scylla-operator/.github/actions/setup-minikube
+      # Label nodes with label required by e2e tests.
+      kubectl label nodes scylla.scylladb.com/node-type=scylla --all
+      
+      # Increase sysctl required by ScyllaDB
+      echo 'fs.aio-max-nr = 300000000' | sudo tee /etc/sysctl.d/90-scylla.conf >/dev/null
+      
+      sudo sysctl --system
   - name: Deploy scylla-operator
     shell: bash
     working-directory: ${{ inputs.repositoryPath }}
@@ -99,18 +109,19 @@ runs:
     run: |
       set -euExo pipefail
       shopt -s inherit_errexit
-
+      
       e2e_timeout_minutes='${{ inputs.baseTimeoutMinutes }}'
       flake_attempts=0
       if [[ -v 'FLAKE_ATTEMPTS' ]]; then
         flake_attempts="${FLAKE_ATTEMPTS}"
         e2e_timeout_minutes="$(( ${e2e_timeout_minutes} + ${flake_attempts} * 10 ))"
       fi
-
+      
       user="$( id -u )"
       group="$( id -g )"
       ingress_address="$( kubectl -n haproxy-ingress get svc haproxy-ingress --template='{{ .spec.clusterIP }}' )"
-      docker run --user="${user}:${group}" --rm \
+      
+      sudo podman run --user="${user}:${group}" --rm \
       --entrypoint=/usr/bin/scylla-operator-tests \
       -v="${ARTIFACTS_DIR}:${ARTIFACTS_DIR}:rw" \
       -v="${HOME}/.kube/config:/kubeconfig:ro" -e='KUBECONFIG=/kubeconfig' \
@@ -139,15 +150,19 @@ runs:
     run: |
       set -euExo pipefail
       shopt -s inherit_errexit
-
-      docker info > docker.info
-      docker images -a > docker.images
-      docker stats -a --no-stream --no-trunc > docker.stats
+      
+      sudo podman info > podman.info
+      sudo podman images -a > podman.images
+      sudo crictl images > crictl.images
+      sudo podman stats -a --no-stream --no-reset > podman.stats
       free -h > free
       journalctl -u kubelet > kubelet.log
-
+      journalctl -u crio > crio.log
+      sudo iptables -L > iptables.log
+      sysctl --all > sysctls.log
+      
       sudo tar -c --use-compress-program=lz4 -f ./kubernetes.tar.lz4 "/etc/kubernetes"
-
+      
       mkdir container-logs
       for ns in kube-system; do
         mkdir "container-logs/${ns}"
diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go
index 5eba9c5f9..c0a5526c0 100644
--- a/test/e2e/framework/framework.go
+++ b/test/e2e/framework/framework.go
@@ -134,6 +134,13 @@ func (f *Framework) ScyllaAdminClient() *scyllaclientset.Clientset {
 	return client
 }
 
+func (f *Framework) CommonLabels() map[string]string {
+	return map[string]string{
+		"e2e":       "scylla-operator",
+		"framework": f.name,
+	}
+}
+
 func (f *Framework) setupNamespace(ctx context.Context) {
 	By("Creating a new namespace")
 	var ns *corev1.Namespace
@@ -148,11 +155,8 @@ func (f *Framework) setupNamespace(ctx context.Context) {
 			ctx,
 			&corev1.Namespace{
 				ObjectMeta: metav1.ObjectMeta{
-					Name: name,
-					Labels: map[string]string{
-						"e2e":       "scylla-operator",
-						"framework": f.name,
-					},
+					Name:   name,
+					Labels: f.CommonLabels(),
 				},
 			},
 			metav1.CreateOptions{},
diff --git a/test/e2e/set/scyllacluster/scyllacluster_pv.go b/test/e2e/set/scyllacluster/scyllacluster_pv.go
index 88f020b77..f84337cfd 100644
--- a/test/e2e/set/scyllacluster/scyllacluster_pv.go
+++ b/test/e2e/set/scyllacluster/scyllacluster_pv.go
@@ -4,36 +4,258 @@ package scyllacluster
 
 import (
 	"context"
+	"errors"
 	"fmt"
+	"strings"
+	"sync"
 
 	g "github.com/onsi/ginkgo/v2"
 	o "github.com/onsi/gomega"
 	scyllav1 "github.com/scylladb/scylla-operator/pkg/api/scylla/v1"
 	"github.com/scylladb/scylla-operator/pkg/controllerhelpers"
+	"github.com/scylladb/scylla-operator/pkg/helpers"
 	"github.com/scylladb/scylla-operator/pkg/naming"
+	"github.com/scylladb/scylla-operator/pkg/pointer"
 	scyllafixture "github.com/scylladb/scylla-operator/test/e2e/fixture/scylla"
 	"github.com/scylladb/scylla-operator/test/e2e/framework"
 	"github.com/scylladb/scylla-operator/test/e2e/utils"
+	"github.com/scylladb/scylla-operator/test/e2e/utils/image"
 	corev1 "k8s.io/api/core/v1"
+	storagev1 "k8s.io/api/storage/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
+	utilerrors "k8s.io/apimachinery/pkg/util/errors"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/client-go/tools/cache"
+	watchtools "k8s.io/client-go/tools/watch"
+	"k8s.io/component-helpers/storage/volume"
 )
 
-var _ = g.Describe("ScyllaCluster Orphaned PV", func() {
-	defer g.GinkgoRecover()
-
+var _ = g.Describe("ScyllaCluster Orphaned PV controller", func() {
 	f := framework.NewFramework("scyllacluster")
 
+	const cloneLabelKey = "e2e.operator.scylladb.com/orphaned-pv-test"
+
+	g.JustBeforeEach(func() {
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		testStorageClass := &storagev1.StorageClass{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:   f.Namespace(),
+				Labels: f.CommonLabels(),
+			},
+			Provisioner:       volume.NotSupportedProvisioner,
+			ReclaimPolicy:     pointer.Ptr(corev1.PersistentVolumeReclaimDelete),
+			VolumeBindingMode: pointer.Ptr(storagev1.VolumeBindingWaitForFirstConsumer),
+		}
+
+		_, err := f.KubeAdminClient().StorageV1().StorageClasses().Create(ctx, testStorageClass, metav1.CreateOptions{})
+		o.Expect(err).NotTo(o.HaveOccurred())
+	})
+
+	g.JustAfterEach(func() {
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		errs := make([]error, 0, 2)
+		errs = append(errs, f.KubeAdminClient().StorageV1().StorageClasses().Delete(ctx, f.Namespace(), metav1.DeleteOptions{}))
+		errs = append(errs, f.KubeAdminClient().CoreV1().PersistentVolumes().DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{
+			LabelSelector: labels.SelectorFromSet(map[string]string{
+				cloneLabelKey: f.Namespace(),
+			}).String(),
+		}))
+
+		o.Expect(utilerrors.NewAggregate(errs)).NotTo(o.HaveOccurred())
+	})
+
 	g.It("should replace a node with orphaned PV", func() {
 		ctx, cancel := context.WithTimeout(context.Background(), testTimeout)
 		defer cancel()
 
+		nodes, err := f.KubeAdminClient().CoreV1().Nodes().List(ctx, metav1.ListOptions{})
+		o.Expect(err).NotTo(o.HaveOccurred())
+
+		if len(nodes.Items) != 1 {
+			g.Skip(
+				"This test supports running at most 1 Node, as it relies on the fact that all Pods land on the same Node." +
+					"Replace with actual Node removal logic when CI env is bigger",
+			)
+		}
+
+		testStorageClassName := f.Namespace()
+
 		sc := scyllafixture.BasicScyllaCluster.ReadOrFail()
-		sc.Spec.Datacenter.Racks[0].Members = 3
 		sc.Spec.AutomaticOrphanedNodeCleanup = true
+		sc.Spec.Datacenter.Racks[0].Members = 3
+		sc.Spec.Datacenter.Racks[0].Storage.StorageClassName = pointer.Ptr(testStorageClassName)
+
+		// This test to trigger the orphaned PV cleanup is updating NodeAffinity of PV used by ScyllaCluster to not exiting Node.
+		// The problem is, that local volume storage provisioners which we use in CI, may set this field, which is immutable.
+		// To overcome this, the ScyllaCluster is using custom StorageClass not backed by any provisioner running in the cluster.
+		// ScyllaCluster is going to request a PVC from that StorageClass, and the test is going to request a clone of the original PVC
+		// from the default StorageClass to get any storage. Then the bound PV is rebounded to the original PVC
+		// but with empty NodeAffinity. This allows the test to trigger the orphaned PV cleanup logic.
+		provisionerCtx, provisionerCancel := context.WithCancel(ctx)
+		defer provisionerCancel()
+
+		var wg sync.WaitGroup
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			defer g.GinkgoRecover()
+
+			lw := &cache.ListWatch{
+				ListFunc: helpers.UncachedListFunc(func(options metav1.ListOptions) (runtime.Object, error) {
+					return f.KubeClient().CoreV1().PersistentVolumeClaims(f.Namespace()).List(ctx, options)
+				}),
+				WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
+					return f.KubeClient().CoreV1().PersistentVolumeClaims(f.Namespace()).Watch(ctx, options)
+				},
+			}
+
+			_, err := watchtools.UntilWithSync(provisionerCtx, lw, &corev1.PersistentVolumeClaim{}, nil, func(e watch.Event) (bool, error) {
+				switch t := e.Type; t {
+				case watch.Added:
+					pvc := e.Object.(*corev1.PersistentVolumeClaim)
+					if pvc.Spec.StorageClassName == nil || *pvc.Spec.StorageClassName != testStorageClassName {
+						return false, nil
+					}
+
+					pvcClone := &corev1.PersistentVolumeClaim{
+						ObjectMeta: metav1.ObjectMeta{
+							Name: fmt.Sprintf("clone-%s", pvc.Name),
+						},
+						Spec:   *pvc.Spec.DeepCopy(),
+						Status: *pvc.Status.DeepCopy(),
+					}
+					pvcClone.Spec.VolumeName = ""
+					pvcClone.Spec.StorageClassName = nil
+
+					framework.Infof("Creating clone PVC for %q", pvc.Name)
+					pvcClone, err = f.KubeClient().CoreV1().PersistentVolumeClaims(f.Namespace()).Create(ctx, pvcClone, metav1.CreateOptions{})
+					o.Expect(err).NotTo(o.HaveOccurred())
+
+					framework.Infof("Creating PVC clone consumer Pod")
+					consumerPodName := fmt.Sprintf("consumer-%s", pvcClone.Name)
+					consumerPod := &corev1.Pod{
+						ObjectMeta: metav1.ObjectMeta{
+							Name: consumerPodName,
+						},
+						Spec: corev1.PodSpec{
+							Containers: []corev1.Container{
+								{
+									Name:  "consumer",
+									Image: image.GetE2EImage(image.BusyBox),
+									Command: []string{
+										"/bin/sh",
+										"-c",
+										"sleep 3600",
+									},
+								},
+							},
+							Volumes: []corev1.Volume{
+								{
+									Name: "data",
+									VolumeSource: corev1.VolumeSource{
+										PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
+											ClaimName: pvcClone.Name,
+										},
+									},
+								},
+							},
+							TerminationGracePeriodSeconds: pointer.Ptr(int64(1)),
+							RestartPolicy:                 corev1.RestartPolicyNever,
+						},
+					}
+					_, err = f.KubeClient().CoreV1().Pods(f.Namespace()).Create(ctx, consumerPod, metav1.CreateOptions{})
+					o.Expect(err).NotTo(o.HaveOccurred())
+				case watch.Modified:
+					pvc := e.Object.(*corev1.PersistentVolumeClaim)
+					if pvc.Spec.StorageClassName == nil {
+						return false, nil
+					}
+					if pvc.DeletionTimestamp != nil {
+						return false, nil
+					}
+
+					if *pvc.Spec.StorageClassName != testStorageClassName && len(pvc.Spec.VolumeName) != 0 {
+						realPvc, err := f.KubeClient().CoreV1().PersistentVolumeClaims(f.Namespace()).Get(ctx, strings.TrimPrefix(pvc.Name, "clone-"), metav1.GetOptions{})
+						o.Expect(err).NotTo(o.HaveOccurred())
+
+						if len(realPvc.Spec.VolumeName) != 0 {
+							return false, nil
+						}
+
+						pv, err := f.KubeAdminClient().CoreV1().PersistentVolumes().Get(ctx, pvc.Spec.VolumeName, metav1.GetOptions{})
+						o.Expect(err).NotTo(o.HaveOccurred())
+
+						pvClone := &corev1.PersistentVolume{
+							ObjectMeta: metav1.ObjectMeta{
+								Name:   fmt.Sprintf("clone-%s", pv.Name),
+								Labels: f.CommonLabels(),
+							},
+							Spec:   *pv.Spec.DeepCopy(),
+							Status: *pv.Status.DeepCopy(),
+						}
+						pvClone.Labels[cloneLabelKey] = f.Namespace()
+						pvClone.Spec.StorageClassName = testStorageClassName
+						pvClone.Spec.NodeAffinity = nil
+						pvClone.Spec.ClaimRef = nil
+						if pvClone.Spec.PersistentVolumeSource.Local != nil {
+							pvClone.Spec.PersistentVolumeSource.HostPath = &corev1.HostPathVolumeSource{
+								Path: pvClone.Spec.PersistentVolumeSource.Local.Path,
+							}
+							pvClone.Spec.PersistentVolumeSource.Local = nil
+						}
+
+						framework.Infof("Creating clone PV for %q", pv.Name)
+						_, err = f.KubeAdminClient().CoreV1().PersistentVolumes().Create(ctx, pvClone, metav1.CreateOptions{})
+						o.Expect(err).NotTo(o.HaveOccurred())
+
+						realPvcCopy := realPvc.DeepCopy()
+						realPvcCopy.Spec.VolumeName = pvClone.Name
+
+						framework.Infof("Binding ScyllaCluster PVC %q with clone PV %q", realPvc.Name, pvClone.Name)
+						_, err = f.KubeClient().CoreV1().PersistentVolumeClaims(realPvc.Namespace).Update(ctx, realPvcCopy, metav1.UpdateOptions{})
+						o.Expect(err).NotTo(o.HaveOccurred())
+					}
+				case watch.Deleted:
+					// If original PVC is deleted, delete the clone too.
+					pvc := e.Object.(*corev1.PersistentVolumeClaim)
+					if strings.HasPrefix(pvc.Name, "clone-") {
+						return false, nil
+					}
+
+					clonePVCName := fmt.Sprintf("clone-%s", pvc.Name)
+					clonePVC, err := f.KubeClient().CoreV1().PersistentVolumeClaims(pvc.Namespace).Get(ctx, clonePVCName, metav1.GetOptions{})
+
+					framework.Infof("Deleting clone PVC %q", clonePVCName)
+					err = f.KubeClient().CoreV1().PersistentVolumeClaims(f.Namespace()).Delete(ctx, clonePVCName, metav1.DeleteOptions{})
+					o.Expect(err).NotTo(o.HaveOccurred())
+
+					consumerPodName := fmt.Sprintf("consumer-%s", clonePVCName)
+					framework.Infof("Deleting Pod consumer of clone PVC %q", consumerPodName)
+
+					err = f.KubeClient().CoreV1().Pods(f.Namespace()).Delete(ctx, consumerPodName, metav1.DeleteOptions{})
+					o.Expect(err).NotTo(o.HaveOccurred())
+
+					err = framework.WaitForObjectDeletion(ctx, f.DynamicAdminClient(), corev1.SchemeGroupVersion.WithResource("persistentvolumeclaims"), f.Namespace(), clonePVC.Name, &clonePVC.UID)
+				}
+				return false, nil
+			})
+			if errors.Is(err, wait.ErrWaitTimeout) && errors.Is(provisionerCtx.Err(), context.Canceled) {
+				return
+			}
+			o.Expect(err).NotTo(o.HaveOccurred())
+		}()
+		defer wg.Wait()
 
 		framework.By("Creating a ScyllaCluster")
-		sc, err := f.ScyllaClient().ScyllaV1().ScyllaClusters(f.Namespace()).Create(ctx, sc, metav1.CreateOptions{})
+		sc, err = f.ScyllaClient().ScyllaV1().ScyllaClusters(f.Namespace()).Create(ctx, sc, metav1.CreateOptions{})
 		o.Expect(err).NotTo(o.HaveOccurred())
 
 		framework.By("Waiting for the ScyllaCluster to rollout (RV=%s)", sc.ResourceVersion)
@@ -126,5 +348,9 @@ var _ = g.Describe("ScyllaCluster Orphaned PV", func() {
 		err = di.SetClientEndpoints(hosts)
 		o.Expect(err).NotTo(o.HaveOccurred())
 		verifyCQLData(ctx, di)
+
+		// Stop fake provisioner
+		provisionerCancel()
+		wg.Wait()
 	})
 })
diff --git a/test/e2e/utils/image/manifests.go b/test/e2e/utils/image/manifests.go
new file mode 100644
index 000000000..e5df10a21
--- /dev/null
+++ b/test/e2e/utils/image/manifests.go
@@ -0,0 +1,49 @@
+// Copyright (c) 2023 ScyllaDB.
+
+package image
+
+import "fmt"
+
+// RegistryList holds public and private image registries
+type RegistryList struct {
+	DockerLibraryRegistry string `json:"dockerLibraryRegistry"`
+}
+
+// Config holds an images registry, name, and version
+type Config struct {
+	registry string
+	name     string
+	version  string
+}
+
+var (
+	registry = initRegistry()
+
+	imageConfigs = initImageConfigs(registry)
+)
+
+func initRegistry() RegistryList {
+	return RegistryList{
+		DockerLibraryRegistry: "docker.io/library",
+	}
+}
+
+const (
+	// None is used as unset image
+	None = iota
+
+	// BusyBox image
+	BusyBox
+)
+
+func initImageConfigs(list RegistryList) map[int]Config {
+	configs := map[int]Config{}
+	configs[BusyBox] = Config{list.DockerLibraryRegistry, "busybox", "1.35"}
+
+	return configs
+}
+
+// GetE2EImage returns the fully qualified URI to an image (including version)
+func GetE2EImage(image int) string {
+	return fmt.Sprintf("%s/%s:%s", imageConfigs[image].registry, imageConfigs[image].name, imageConfigs[image].version)
+}