Repository: kubernetes/kubernetes (language: go)

Issue:
PV nodeAffinity matchExpressions problem with array items and `in` operator since 1.27.0
### What happened?

Since v1.27.0, the `nodeAffinity` for `PersistentVolumes` appear to have added extra validation on whether the values exist, despite using the `in` operator. This works as expected on 1.26.14, and no longer works on 1.27.0 onwards (tested up to 1.29.2).

I am unsure whether this is an intentional breaking change in 1.27.0 or something wrong in our manifests and we've just so happened to get away with it for a few years. The only thing that stands out to me in the changelog for 1.27.0 is some changes to the schedulers `Filter` plugin but this may well be unrelated.

### What did you expect to happen?

When using the `in` operator, I'd expect the expression to only apply to nodes where the match is truey.

### How can we reproduce it (as minimally and precisely as possible)?

Note, using Kind for easy reproduction

## v1.26.14 -  working

### creating a kind cluster
~~~ shell
cat <<EOF > kind_conf
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
nodes:
- role: control-plane
  image: kindest/node:v1.26.14
- role: worker
  image: kindest/node:v1.26.14
- role: worker
  image: kindest/node:v1.26.14
EOF

$ kind create cluster --name 12614-test --config kind_conf
$ k get no
NAME                       STATUS   ROLES           AGE   VERSION
12614-test-control-plane   Ready    control-plane   8h    v1.26.14
12614-test-worker          Ready    <none>          8h    v1.26.14
12614-test-worker2         Ready    <none>          8h    v1.26.14
# labelling a node to ensure pod is scheduled on specific worker
$ k label node 12614-test-worker nginx=nginx
~~~

~~~ shell
cat <<EOF > manifest.yaml
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  labels:
    app: nginx
  name: nginx
  namespace: default
spec:
  replicas: 1
  selector:
    matchLabels:
      app: nginx
  serviceName: nginx
  template:
    metadata:
      labels:
        app: nginx
    spec:
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
            - matchExpressions:
              - key: nginx
                operator: Exists
      containers:
      - name: nginx
        image: nginx:latest
        imagePullPolicy: IfNotPresent
        volumeMounts:
        - mountPath: /var/lib/nginx
          name: default-nginx-data
  volumeClaimTemplates:
  - apiVersion: v1
    kind: PersistentVolumeClaim
    metadata:
      name: default-nginx-data
    spec:
      accessModes:
      - ReadWriteMany
      resources:
        requests:
          storage: 5Gi
      storageClassName: my_sc
      volumeMode: Filesystem
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: default-nginx-data-nginx-0
  namespace: default
  labels:
    app: default-nginx-data-nginx-0
spec:
  storageClassName: my_sc
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteMany
  hostPath:
    path: /srv/k8s/default/nginx-0
  claimRef:
    namespace: default
    name: default-nginx-data-nginx-0
  nodeAffinity:
    required:
      nodeSelectorTerms:
      - matchExpressions:
        - key: kubernetes.io/hostname
          operator: In
          values:
            - 12614-test-worker
            - 12614-prod-worker
EOF
# Apply the manifest
$ k apply -f manifest.yaml
~~~

Since `12614-test-worker` exists in the hostname values of this cluster, the pod schedules as expected. `12614-prod-worker` node does not exist in this environment, and so is ignored.

~~~ shell
$ k get po
NAME      READY   STATUS    RESTARTS   AGE
nginx-0   1/1     Running   0          6m2s
~~~
## v1.27.0 -  not working

### creating a kind cluster
~~~ shell
cat <<EOF > kind_conf
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
nodes:
- role: control-plane
  image: kindest/node:v1.27.0
- role: worker
  image: kindest/node:v1.27.0
- role: worker
  image: kindest/node:v1.27.0
EOF

$ kind create cluster --name 1270-test --config kind_conf
# labelling a node to ensure pod is scheduled on specific worker
$ k label node 1270-test-worker nginx=nginx
~~~

~~~ shell
cat <<EOF > manifest.yaml
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  labels:
    app: nginx
  name: nginx
  namespace: default
spec:
  replicas: 1
  selector:
    matchLabels:
      app: nginx
  serviceName: nginx
  template:
    metadata:
      labels:
        app: nginx
    spec:
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
            - matchExpressions:
              - key: nginx
                operator: Exists
      containers:
      - name: nginx
        image: nginx:latest
        imagePullPolicy: IfNotPresent
        volumeMounts:
        - mountPath: /var/lib/nginx
          name: default-nginx-data
  volumeClaimTemplates:
  - apiVersion: v1
    kind: PersistentVolumeClaim
    metadata:
      name: default-nginx-data
    spec:
      accessModes:
      - ReadWriteMany
      resources:
        requests:
          storage: 5Gi
      storageClassName: my_sc
      volumeMode: Filesystem
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: default-nginx-data-nginx-0
  namespace: default
  labels:
    app: default-nginx-data-nginx-0
spec:
  storageClassName: my_sc
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteMany
  hostPath:
    path: /srv/k8s/default/nginx-0
  claimRef:
    namespace: default
    name: default-nginx-data-nginx-0
  nodeAffinity:
    required:
      nodeSelectorTerms:
      - matchExpressions:
        - key: kubernetes.io/hostname
          operator: In
          values:
            - 1270-test-worker
            - 1270-prod-worker
EOF
# Apply the manifest
$ k apply -f manifest.yaml
~~~

Same expectations as 1.26.14, but in this case, the pod stays stuck in pending state due to the following;

~~~ shell
$ k get no
NAME                      STATUS   ROLES           AGE     VERSION
1270-test-control-plane   Ready    control-plane   5m1s    v1.27.0
1270-test-worker          Ready    <none>          4m34s   v1.27.0
1270-test-worker2         Ready    <none>          4m39s   v1.27.0
$ k get po
NAME      READY   STATUS    RESTARTS   AGE
nginx-0   0/1     Pending   0          5m56s
$ kubectl events --for pod/nginx-0
LAST SEEN   TYPE      REASON             OBJECT        MESSAGE
3m6s        Warning   FailedScheduling   Pod/nginx-0   nodeinfo not found for node name "1270-prod-worker"
~~~

I have tested this all the way up to 1.29.2.

### Anything else we need to know?

_No response_

### Kubernetes version

v1.27.0 through to v1.29.2

### Cloud provider

<details>

</details>


### OS version

<details>

```console
[root@stage1 ~]# cat /etc/os-release
NAME="CentOS Linux"
VERSION="7 (Core)"
ID="centos"
ID_LIKE="rhel fedora"
VERSION_ID="7"
PRETTY_NAME="CentOS Linux 7 (Core)"
ANSI_COLOR="0;31"
CPE_NAME="cpe:/o:centos:centos:7"
HOME_URL="https://www.centos.org/"
BUG_REPORT_URL="https://bugs.centos.org/"

CENTOS_MANTISBT_PROJECT="CentOS-7"
CENTOS_MANTISBT_PROJECT_VERSION="7"
REDHAT_SUPPORT_PRODUCT="centos"
REDHAT_SUPPORT_PRODUCT_VERSION="7"

[root@stage1 ~]# uname -a
Linux stage1.domain.com 3.10.0-1160.108.1.el7.x86_64 #1 SMP Thu Jan 25 16:17:31 UTC 2024 x86_64 x86_64 x86_64 GNU/Linux
```

</details>


### Install tools

<details>

</details>


### Container runtime (CRI) and version (if applicable)

<details>

</details>


### Related plugins (CNI, CSI, ...) and versions (if applicable)

<details>

</details>



Discussion / hints:
This issue is currently awaiting triage.

If a SIG or subproject determines this is a relevant issue, they will accept it by applying the `triage/accepted` label and provide further guidance.

The `triage/accepted` label can be added by org members by writing `/triage accepted` in a comment.


<details>

Instructions for interacting with me using PR comments are available [here](https://git.k8s.io/community/contributors/guide/pull-requests.md).  If you have questions or suggestions related to my behavior, please file an issue against the [kubernetes/test-infra](https://github.com/kubernetes/test-infra/issues/new?title=Prow%20issue:) repository.
</details>
/sig storage
In a similar vein, we have noticed the same issue in K8s v1.27 when the `kubernetes.io/hostname` label doesn't match the node name. We see `nodeinfo not found for <hostname>`, despite (I think) the nodeinfo util supposed to be called using a node name.
/sig scheduling
This is because the non-existent node `1270-prod-worker` is obtained from the cache, which has been fixed by #119779

before 119779, get node will return error:
https://github.com/kubernetes/kubernetes/blob/bd2cb7205ee631b7d6c32ba5d7514ef75f011622/pkg/scheduler/schedule_one.go#L434-L444

after 119779 it's fixed
https://github.com/kubernetes/kubernetes/blob/da6be3b71826ca100d3aeeeea289a1a3272d8aca/pkg/scheduler/schedule_one.go#L485-L497

since version 1.30, it can run normally

@sanposhiho PTAL,  need to cherry-pick for the previous version?
Looks like my PR https://github.com/kubernetes/kubernetes/pull/119779 accidentally fixed this issue. 😅 

https://github.com/kubernetes/kubernetes/commit/380c7f248e4ecc2a104b11652b6171ab65501cba is the trigger of this issue, introduced in v1.27; volumebinding plugin just returns node names in PreFilterResult based on nodeAffinity in PV, and thus PreFilterResult could contain node names actually doesn't exist.

---

@kubernetes/sig-scheduling-leads 

One Line Summary: If nodeaffinity in PV has non-existing node name in `In` with `kubernetes.io/hostname`, the scheduling always fails.
And, this bug is already coincidentally fixed by #119779. 

Questions:
- In the first place, is it a bug worth cherry-picking?
- If Yes, should we cherry-pick #119779, which actually isn't the PR motivated for this issue? Or should we go another way
If it's a regression, we should consider cherry-pick.

🤔
https://github.com/kubernetes/kubernetes/pull/119779 looks big, but the actual code changes are about 10 lines (the rest are comments and tests), so I think it would be ok to cherry-pick.

Can you explain why your PR fixes the problem?
> Can you explain why your PR fixes the problem?

Before my PR, we take NodeInfo of nodes in preFilterResult via `sched.nodeInfoSnapshot.NodeInfos().Get(n)` ([ref](https://github.com/kubernetes/kubernetes/blob/bd2cb7205ee631b7d6c32ba5d7514ef75f011622/pkg/scheduler/schedule_one.go#L434-L444)). Consequently, we could get not found errors from it when preFilterResult has non-existing node name(s).

And, after my PR, we are not using `sched.nodeInfoSnapshot.NodeInfos().Get(n)` anymore.
The pitfall is that when PreFilterResult was introduced, scheduler directly fetch the nodeNames returned by PreFilter() in its node snapshot, so technically speaking, when a plugin (in-tree or out-of-tree) return _non-existent/illegal_ nodes, the pod scheduling flow will abort immediately.

And the PR #109877 happened to stomp on this pitfall.
@Huang-Wei do you mean that you would like to see a smaller version of #119779 for cherry-pick? Note that it is only about 10 lines of actual change. The rest is comments and tests.

Since this is already released code, I would suggest cherry-picking as-is.

Also, worth testing E2E whether the issue reproduces in v1.30.

@dsgnr do you think you can give it a try?
> @Huang-Wei do you mean that you would like to see a smaller version of #119779 for cherry-pick? Note that it is only about 10 lines of actual change. The rest is comments and tests.

yup, i'm in favor of picking the _minimum_ fix, along with a minimum test #124539:

```go
		for _, n := range allNodes {
			if !preRes.NodeNames.Has(n.Node().Name) {
				// We consider Nodes that are filtered out by PreFilterResult as rejected via UnschedulableAndUnresolvable.
				// We have to record them in NodeToStatusMap so that they won't be considered as candidates in the preemption.
				diagnosis.NodeToStatusMap[n.Node().Name] = framework.NewStatus(framework.UnschedulableAndUnresolvable, "node is filtered out by the prefilter result")
				continue
			}
			nodes = append(nodes, n)
```

b/c in addition to the "fix" part, #119779 actually introduced a slight behavior change. @sanposhiho do you think the above ^^ minimum fix + #124539 makes sense?
> Also, worth testing E2E whether the issue reproduces in v1.30.

#124539 can guard it.
> > @Huang-Wei do you mean that you would like to see a smaller version of #119779 for cherry-pick? Note that it is only about 10 lines of actual change. The rest is comments and tests.
> 
> yup, i'm in favor of picking the _minimum_ fix, along with a minimum test #124539:
> 
> ```go
> 		for _, n := range allNodes {
> 			if !preRes.NodeNames.Has(n.Node().Name) {
> 				// We consider Nodes that are filtered out by PreFilterResult as rejected via UnschedulableAndUnresolvable.
> 				// We have to record them in NodeToStatusMap so that they won't be considered as candidates in the preemption.
> 				diagnosis.NodeToStatusMap[n.Node().Name] = framework.NewStatus(framework.UnschedulableAndUnresolvable, "node is filtered out by the prefilter result")
> 				continue
> 			}
> 			nodes = append(nodes, n)
> ```
> 
> b/c in addition to the "fix" part, #119779 actually introduced a slight behavior change. @sanposhiho do you think the above ^^ minimum fix + #124539 makes sense?

@Huang-Wei @alculquicondor  @sanposhiho  Could you let me try it?，picking the minimum fix and add e2e test
A simplified version of the sample is used for testing,manifest.yaml:
```
apiVersion: v1
kind: PersistentVolume
metadata:
  name: data
spec:
  capacity:
    storage: 10Gi
  accessModes:
    - ReadWriteMany
  hostPath:
    path: /tmp/data
  claimRef:
    namespace: default
    name: data
  nodeAffinity:
    required:
      nodeSelectorTerms:
        - matchExpressions:
          - key: kubernetes.io/hostname
            operator: In
            values:
            - 1280-test-worker
            - 1280-prod-worker

---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: data
  namespace: default
spec:
  accessModes:
  - ReadWriteMany
  resources:
    requests:
      storage: 5Gi
  volumeName: data
---
apiVersion: v1
kind: Pod
metadata:
  name: pause
  labels:
    app: pause
spec:
  containers:
  - name: pause
    image: registry.k8s.io/pause:3.6
    volumeMounts:
    - mountPath: /tmp
      name: data
  volumes:
  - name: data
    persistentVolumeClaim:
      claimName: data

```
also get this failedScheduling error, kubectl describe pod pause:
![image](https://github.com/kubernetes/kubernetes/assets/30427474/d7c8db96-c10d-41b5-a9da-f8e39975ac7f)



> @sanposhiho do you think the above ^^ minimum fix + https://github.com/kubernetes/kubernetes/pull/124539 makes sense?

@Huang-Wei That small fix should work.
I don't have a strong opinion on whether we should have a minimum fix vs cherry-pick as it is. 
Defer to you both @alculquicondor @Huang-Wei on that point.

Fix patch:
diff --git a/pkg/scheduler/schedule_one.go b/pkg/scheduler/schedule_one.go
index b9ded88732cd6..8a01a259e65f1 100644
--- a/pkg/scheduler/schedule_one.go
+++ b/pkg/scheduler/schedule_one.go
@@ -439,13 +439,16 @@ func (sched *Scheduler) schedulePod(ctx context.Context, fwk framework.Framework
 // filter plugins and filter extenders.
 func (sched *Scheduler) findNodesThatFitPod(ctx context.Context, fwk framework.Framework, state *framework.CycleState, pod *v1.Pod) ([]*v1.Node, framework.Diagnosis, error) {
 	logger := klog.FromContext(ctx)
-	diagnosis := framework.Diagnosis{
-		NodeToStatusMap: make(framework.NodeToStatusMap),
-	}
 
 	allNodes, err := sched.nodeInfoSnapshot.NodeInfos().List()
 	if err != nil {
-		return nil, diagnosis, err
+		return nil, framework.Diagnosis{
+			NodeToStatusMap: make(framework.NodeToStatusMap),
+		}, err
+	}
+
+	diagnosis := framework.Diagnosis{
+		NodeToStatusMap: make(framework.NodeToStatusMap, len(allNodes)),
 	}
 	// Run "prefilter" plugins.
 	preRes, s := fwk.RunPreFilterPlugins(ctx, state, pod)
@@ -483,12 +486,14 @@ func (sched *Scheduler) findNodesThatFitPod(ctx context.Context, fwk framework.F
 	nodes := allNodes
 	if !preRes.AllNodes() {
 		nodes = make([]*framework.NodeInfo, 0, len(preRes.NodeNames))
-		for n := range preRes.NodeNames {
-			nInfo, err := sched.nodeInfoSnapshot.NodeInfos().Get(n)
-			if err != nil {
-				return nil, diagnosis, err
+		for _, n := range allNodes {
+			if !preRes.NodeNames.Has(n.Node().Name) {
+				// We consider Nodes that are filtered out by PreFilterResult as rejected via UnschedulableAndUnresolvable.
+				// We have to record them in NodeToStatusMap so that they won't be considered as candidates in the preemption.
+				diagnosis.NodeToStatusMap[n.Node().Name] = framework.NewStatus(framework.UnschedulableAndUnresolvable, "node is filtered out by the prefilter result")
+				continue
 			}
-			nodes = append(nodes, nInfo)
+			nodes = append(nodes, n)
 		}
 	}
 	feasibleNodes, err := sched.findNodesThatPassFilters(ctx, fwk, state, pod, &diagnosis, nodes)


Test patch:
diff --git a/pkg/scheduler/schedule_one_test.go b/pkg/scheduler/schedule_one_test.go
index 9a70a62921a18..f07ebc42e90e1 100644
--- a/pkg/scheduler/schedule_one_test.go
+++ b/pkg/scheduler/schedule_one_test.go
@@ -2181,7 +2181,7 @@ func TestSchedulerSchedulePod(t *testing.T) {
 			nodes:              []string{"node1", "node2", "node3"},
 			pod:                st.MakePod().Name("test-prefilter").UID("test-prefilter").Obj(),
 			wantNodes:          sets.New("node2"),
-			wantEvaluatedNodes: ptr.To[int32](1),
+			wantEvaluatedNodes: ptr.To[int32](3),
 		},
 		{
 			name: "test prefilter plugin returning non-intersecting nodes",
diff --git a/pkg/scheduler/testing/wrappers.go b/pkg/scheduler/testing/wrappers.go
index b564bd0cafb03..eb0e62e1cd9c5 100644
--- a/pkg/scheduler/testing/wrappers.go
+++ b/pkg/scheduler/testing/wrappers.go
@@ -851,6 +851,20 @@ func (p *PersistentVolumeWrapper) HostPathVolumeSource(src *v1.HostPathVolumeSou
 	return p
 }
 
+// NodeAffinityIn creates a HARD node affinity (with the operator In)
+// and injects into the pv.
+func (p *PersistentVolumeWrapper) NodeAffinityIn(key string, vals []string) *PersistentVolumeWrapper {
+	if p.Spec.NodeAffinity == nil {
+		p.Spec.NodeAffinity = &v1.VolumeNodeAffinity{}
+	}
+	if p.Spec.NodeAffinity.Required == nil {
+		p.Spec.NodeAffinity.Required = &v1.NodeSelector{}
+	}
+	nodeSelector := MakeNodeSelector().In(key, vals).Obj()
+	p.Spec.NodeAffinity.Required.NodeSelectorTerms = append(p.Spec.NodeAffinity.Required.NodeSelectorTerms, nodeSelector.NodeSelectorTerms...)
+	return p
+}
+
 // ResourceClaimWrapper wraps a ResourceClaim inside.
 type ResourceClaimWrapper struct{ resourcev1alpha2.ResourceClaim }
 
diff --git a/test/e2e/scheduling/predicates.go b/test/e2e/scheduling/predicates.go
index 3761cde4a2201..526bb35499692 100644
--- a/test/e2e/scheduling/predicates.go
+++ b/test/e2e/scheduling/predicates.go
@@ -39,6 +39,7 @@ import (
 	e2enode "k8s.io/kubernetes/test/e2e/framework/node"
 	e2eruntimeclass "k8s.io/kubernetes/test/e2e/framework/node/runtimeclass"
 	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
+	e2epv "k8s.io/kubernetes/test/e2e/framework/pv"
 	e2erc "k8s.io/kubernetes/test/e2e/framework/rc"
 	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
 	testutils "k8s.io/kubernetes/test/utils"
@@ -857,6 +858,78 @@ var _ = SIGDescribe("SchedulerPredicates", framework.WithSerial(), func() {
 		ginkgo.By("Expect all pods are scheduled and running")
 		framework.ExpectNoError(e2epod.WaitForPodsRunning(cs, ns, replicas, time.Minute))
 	})
+
+	// Regression test for an extended scenario for https://issues.k8s.io/123465
+	ginkgo.It("when PVC has node-affinity to non-existent/illegal nodes, the pod should be scheduled normally if suitable nodes exist", func(ctx context.Context) {
+		nodeName := GetNodeThatCanRunPod(ctx, f)
+		nonExistentNodeName1 := string(uuid.NewUUID())
+		nonExistentNodeName2 := string(uuid.NewUUID())
+		hostLabel := "kubernetes.io/hostname"
+		localPath := "/tmp"
+		podName := "bind-pv-with-non-existent-nodes"
+		pvcName := "pvc-" + string(uuid.NewUUID())
+		_, pvc, err := e2epv.CreatePVPVC(ctx, cs, f.Timeouts, e2epv.PersistentVolumeConfig{
+			PVSource: v1.PersistentVolumeSource{
+				Local: &v1.LocalVolumeSource{
+					Path: localPath,
+				},
+			},
+			Prebind: &v1.PersistentVolumeClaim{
+				ObjectMeta: metav1.ObjectMeta{Name: pvcName, Namespace: ns},
+			},
+			NodeAffinity: &v1.VolumeNodeAffinity{
+				Required: &v1.NodeSelector{
+					NodeSelectorTerms: []v1.NodeSelectorTerm{
+						{
+							MatchExpressions: []v1.NodeSelectorRequirement{
+								{
+									Key:      hostLabel,
+									Operator: v1.NodeSelectorOpIn,
+									// add non-existent nodes to the list
+									Values: []string{nodeName, nonExistentNodeName1, nonExistentNodeName2},
+								},
+							},
+						},
+					},
+				},
+			},
+		}, e2epv.PersistentVolumeClaimConfig{
+			Name: pvcName,
+		}, ns, true)
+		framework.ExpectNoError(err)
+		bindPvPod := &v1.Pod{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: podName,
+			},
+			Spec: v1.PodSpec{
+				Containers: []v1.Container{
+					{
+						Name:  "pause",
+						Image: imageutils.GetE2EImage(imageutils.Pause),
+						VolumeMounts: []v1.VolumeMount{
+							{
+								Name:      "data",
+								MountPath: "/tmp",
+							},
+						},
+					},
+				},
+				Volumes: []v1.Volume{
+					{
+						Name: "data",
+						VolumeSource: v1.VolumeSource{
+							PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
+								ClaimName: pvc.Name,
+							},
+						},
+					},
+				},
+			},
+		}
+		_, err = f.ClientSet.CoreV1().Pods(ns).Create(ctx, bindPvPod, metav1.CreateOptions{})
+		framework.ExpectNoError(err)
+		framework.ExpectNoError(e2epod.WaitForPodNotPending(ctx, f.ClientSet, ns, podName))
+	})
 })
 
 func patchPod(cs clientset.Interface, old, new *v1.Pod) (*v1.Pod, error) {
diff --git a/test/integration/scheduler/filters/filters_test.go b/test/integration/scheduler/filters/filters_test.go
index 6677740faab01..4bded10523f92 100644
--- a/test/integration/scheduler/filters/filters_test.go
+++ b/test/integration/scheduler/filters/filters_test.go
@@ -43,6 +43,7 @@ var (
 	createAndWaitForNodesInCache = testutils.CreateAndWaitForNodesInCache
 	createNamespacesWithLabels   = testutils.CreateNamespacesWithLabels
 	createNode                   = testutils.CreateNode
+	updateNode                   = testutils.UpdateNode
 	createPausePod               = testutils.CreatePausePod
 	deletePod                    = testutils.DeletePod
 	getPod                       = testutils.GetPod
@@ -2042,6 +2043,111 @@ func TestUnschedulablePodBecomesSchedulable(t *testing.T) {
 				return deletePod(cs, "pod-to-be-deleted", ns)
 			},
 		},
+		{
+			name: "pod with pvc has node-affinity to non-existent/illegal nodes",
+			init: func(cs kubernetes.Interface, ns string) error {
+				storage := v1.VolumeResourceRequirements{Requests: v1.ResourceList{v1.ResourceStorage: resource.MustParse("1Mi")}}
+				volType := v1.HostPathDirectoryOrCreate
+				pv, err := testutils.CreatePV(cs, st.MakePersistentVolume().
+					Name("pv-has-non-existent-nodes").
+					AccessModes([]v1.PersistentVolumeAccessMode{v1.ReadWriteOncePod}).
+					Capacity(storage.Requests).
+					HostPathVolumeSource(&v1.HostPathVolumeSource{Path: "/tmp", Type: &volType}).
+					NodeAffinityIn("kubernetes.io/hostname", []string{"node-available", "non-existing"}). // one node exist, one doesn't
+					Obj())
+				if err != nil {
+					return fmt.Errorf("cannot create pv: %w", err)
+				}
+				_, err = testutils.CreatePVC(cs, st.MakePersistentVolumeClaim().
+					Name("pvc-has-non-existent-nodes").
+					Namespace(ns).
+					Annotation(volume.AnnBindCompleted, "true").
+					VolumeName(pv.Name).
+					AccessModes([]v1.PersistentVolumeAccessMode{v1.ReadWriteOncePod}).
+					Resources(storage).
+					Obj())
+				if err != nil {
+					return fmt.Errorf("cannot create pvc: %w", err)
+				}
+				return nil
+			},
+			pod: &testutils.PausePodConfig{
+				Name: "pod-with-pvc-has-non-existent-nodes",
+				Volumes: []v1.Volume{{
+					Name: "volume",
+					VolumeSource: v1.VolumeSource{
+						PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
+							ClaimName: "pvc-has-non-existent-nodes",
+						},
+					},
+				}},
+			},
+			update: func(cs kubernetes.Interface, ns string) error {
+				_, err := createNode(cs, st.MakeNode().Label("kubernetes.io/hostname", "node-available").Name("node-available").Obj())
+				if err != nil {
+					return fmt.Errorf("cannot create node: %w", err)
+				}
+				return nil
+			},
+		},
+		{
+			name: "pod with pvc got scheduled after node updated it's label",
+			init: func(cs kubernetes.Interface, ns string) error {
+				_, err := createNode(cs, st.MakeNode().Label("foo", "foo").Name("node-foo").Obj())
+				if err != nil {
+					return fmt.Errorf("cannot create node: %w", err)
+				}
+				storage := v1.VolumeResourceRequirements{Requests: v1.ResourceList{v1.ResourceStorage: resource.MustParse("1Mi")}}
+				volType := v1.HostPathDirectoryOrCreate
+				pv, err := testutils.CreatePV(cs, st.MakePersistentVolume().
+					Name("pv-foo").
+					AccessModes([]v1.PersistentVolumeAccessMode{v1.ReadWriteOncePod}).
+					Capacity(storage.Requests).
+					HostPathVolumeSource(&v1.HostPathVolumeSource{Path: "/tmp", Type: &volType}).
+					NodeAffinityIn("foo", []string{"bar"}).
+					Obj())
+				if err != nil {
+					return fmt.Errorf("cannot create pv: %w", err)
+				}
+				_, err = testutils.CreatePVC(cs, st.MakePersistentVolumeClaim().
+					Name("pvc-foo").
+					Namespace(ns).
+					Annotation(volume.AnnBindCompleted, "true").
+					VolumeName(pv.Name).
+					AccessModes([]v1.PersistentVolumeAccessMode{v1.ReadWriteOncePod}).
+					Resources(storage).
+					Obj())
+				if err != nil {
+					return fmt.Errorf("cannot create pvc: %w", err)
+				}
+				return nil
+			},
+			pod: &testutils.PausePodConfig{
+				Name: "pod-with-pvc-foo",
+				Volumes: []v1.Volume{{
+					Name: "volume",
+					VolumeSource: v1.VolumeSource{
+						PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
+							ClaimName: "pvc-foo",
+						},
+					},
+				}},
+			},
+			update: func(cs kubernetes.Interface, ns string) error {
+				_, err := updateNode(cs, &v1.Node{
+					ObjectMeta: metav1.ObjectMeta{
+						Name: "node-foo",
+						Labels: map[string]string{
+							"foo": "bar",
+						},
+					},
+				})
+				if err != nil {
+					return fmt.Errorf("cannot update node: %w", err)
+				}
+				return nil
+			},
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
diff --git a/test/integration/scheduler/plugins/plugins_test.go b/test/integration/scheduler/plugins/plugins_test.go
index 791d1bdd72536..31afc964020c8 100644
--- a/test/integration/scheduler/plugins/plugins_test.go
+++ b/test/integration/scheduler/plugins/plugins_test.go
@@ -32,6 +32,7 @@ import (
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apiserver/pkg/util/feature"
 	clientset "k8s.io/client-go/kubernetes"
@@ -80,9 +81,10 @@ type PreEnqueuePlugin struct {
 }
 
 type PreFilterPlugin struct {
-	numPreFilterCalled int
-	failPreFilter      bool
-	rejectPreFilter    bool
+	numPreFilterCalled   int
+	failPreFilter        bool
+	rejectPreFilter      bool
+	preFilterResultNodes sets.Set[string]
 }
 
 type ScorePlugin struct {
@@ -524,6 +526,9 @@ func (pp *PreFilterPlugin) PreFilter(ctx context.Context, state *framework.Cycle
 	if pp.rejectPreFilter {
 		return nil, framework.NewStatus(framework.Unschedulable, fmt.Sprintf("reject pod %v", pod.Name))
 	}
+	if len(pp.preFilterResultNodes) != 0 {
+		return &framework.PreFilterResult{NodeNames: pp.preFilterResultNodes}, nil
+	}
 	return nil, nil
 }
 
@@ -627,9 +632,10 @@ func TestPreFilterPlugin(t *testing.T) {
 	testContext := testutils.InitTestAPIServer(t, "prefilter-plugin", nil)
 
 	tests := []struct {
-		name   string
-		fail   bool
-		reject bool
+		name                 string
+		fail                 bool
+		reject               bool
+		preFilterResultNodes sets.Set[string]
 	}{
 		{
 			name:   "disable fail and reject flags",
@@ -646,6 +652,18 @@ func TestPreFilterPlugin(t *testing.T) {
 			fail:   false,
 			reject: true,
 		},
+		{
+			name:                 "inject legal node names in PreFilterResult",
+			fail:                 false,
+			reject:               false,
+			preFilterResultNodes: sets.New[string]("test-node-0", "test-node-1"),
+		},
+		{
+			name:                 "inject legal and illegal node names in PreFilterResult",
+			fail:                 false,
+			reject:               false,
+			preFilterResultNodes: sets.New[string]("test-node-0", "non-existent-node"),
+		},
 	}
 
 	for _, test := range tests {
@@ -661,6 +679,7 @@ func TestPreFilterPlugin(t *testing.T) {
 
 			preFilterPlugin.failPreFilter = test.fail
 			preFilterPlugin.rejectPreFilter = test.reject
+			preFilterPlugin.preFilterResultNodes = test.preFilterResultNodes
 			// Create a best effort pod.
 			pod, err := testutils.CreatePausePod(testCtx.ClientSet,
 				testutils.InitPausePod(&testutils.PausePodConfig{Name: "test-pod", Namespace: testCtx.NS.Name}))