Repository: kubernetes/kubernetes (language: go)

Issue:
v1.30: kube-scheduler crashes with: Observed a panic: "integer divide by zero"
### What happened?

On Kubernetes v1.30.0 (and v1.30.1), `kube-scheduler` can crash with the following panic if a pod is defined in a certain way:

```
W0514 09:09:41.391780       1 feature_gate.go:246] Setting GA feature gate MinDomainsInPodTopologySpread=true. It will be removed in a future release.
I0514 09:09:43.191448       1 serving.go:380] Generated self-signed cert in-memory
W0514 09:09:43.574824       1 authentication.go:446] failed to read in-cluster kubeconfig for delegated authentication: open /var/run/secrets/kubernetes.io/serviceaccount/token: no such file or directory
W0514 09:09:43.574862       1 authentication.go:339] No authentication-kubeconfig provided in order to lookup client-ca-file in configmap/extension-apiserver-authentication in kube-system, so client certificate authentication won't work.
W0514 09:09:43.574871       1 authentication.go:363] No authentication-kubeconfig provided in order to lookup requestheader-client-ca-file in configmap/extension-apiserver-authentication in kube-system, so request-header client certificate authentication won't work.
W0514 09:09:43.574885       1 authorization.go:225] failed to read in-cluster kubeconfig for delegated authorization: open /var/run/secrets/kubernetes.io/serviceaccount/token: no such file or directory
W0514 09:09:43.574992       1 authorization.go:193] No authorization-kubeconfig provided, so SubjectAccessReview of authorization tokens won't work.
I0514 09:09:43.586307       1 server.go:154] "Starting Kubernetes Scheduler" version="v1.30.0-zalando-master-117-dirty"
I0514 09:09:43.586327       1 server.go:156] "Golang settings" GOGC="" GOMAXPROCS="" GOTRACEBACK=""
I0514 09:09:43.587758       1 secure_serving.go:213] Serving securely on [::]:10259
I0514 09:09:43.587944       1 tlsconfig.go:240] "Starting DynamicServingCertificateController"
I0514 09:09:43.688733       1 leaderelection.go:250] attempting to acquire leader lease kube-system/kube-scheduler...
I0514 09:09:59.030033       1 leaderelection.go:260] successfully acquired lease kube-system/kube-scheduler
E0514 09:09:59.122043       1 runtime.go:79] Observed a panic: "integer divide by zero" (runtime error: integer divide by zero)
goroutine 330 [running]:
k8s.io/apimachinery/pkg/util/runtime.logPanic({0x1b68bc0, 0x35e1c40})
	k8s.io/apimachinery/pkg/util/runtime/runtime.go:75 +0x7c
k8s.io/apimachinery/pkg/util/runtime.HandleCrash({0x0, 0x0, 0x4000be8e00?})
	k8s.io/apimachinery/pkg/util/runtime/runtime.go:49 +0x78
panic({0x1b68bc0?, 0x35e1c40?})
	runtime/panic.go:770 +0x124
k8s.io/kubernetes/pkg/scheduler.(*Scheduler).findNodesThatFitPod(0x40000d6900, {0x22aae08, 0x40012b9630}, {0x22d2bb8, 0x4000320fc8}, 0x4000347dc0, 0x4000cb4488)
	k8s.io/kubernetes/pkg/scheduler/schedule_one.go:502 +0x88c
k8s.io/kubernetes/pkg/scheduler.(*Scheduler).schedulePod(0x40000d6900, {0x22aae08, 0x40012b9630}, {0x22d2bb8, 0x4000320fc8}, 0x4000347dc0, 0x4000cb4488)
	k8s.io/kubernetes/pkg/scheduler/schedule_one.go:402 +0x25c
k8s.io/kubernetes/pkg/scheduler.(*Scheduler).schedulingCycle(0x40000d6900, {0x22aae08, 0x40012b9630}, 0x4000347dc0, {0x22d2bb8, 0x4000320fc8}, 0x4000c92e10, {0x2?, 0x2?, 0x36352c0?}, ...)
	k8s.io/kubernetes/pkg/scheduler/schedule_one.go:149 +0xb8
k8s.io/kubernetes/pkg/scheduler.(*Scheduler).ScheduleOne(0x40000d6900, {0x22aae08, 0x40012b83c0})
	k8s.io/kubernetes/pkg/scheduler/schedule_one.go:111 +0x4c0
k8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext.func1()
	k8s.io/apimachinery/pkg/util/wait/backoff.go:259 +0x2c
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x400161dec8?)
	k8s.io/apimachinery/pkg/util/wait/backoff.go:226 +0x40
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x400161df68, {0x2286fe0, 0x4001398b70}, 0x1, 0x40012bab40)
	k8s.io/apimachinery/pkg/util/wait/backoff.go:227 +0x90
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0x4000669f68, 0x0, 0x0, 0x1, 0x40012bab40)
	k8s.io/apimachinery/pkg/util/wait/backoff.go:204 +0x80
k8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext({0x22aae08, 0x40012b83c0}, 0x4001385720, 0x0, 0x0, 0x1)
	k8s.io/apimachinery/pkg/util/wait/backoff.go:259 +0x80
k8s.io/apimachinery/pkg/util/wait.UntilWithContext(...)
	k8s.io/apimachinery/pkg/util/wait/backoff.go:170
created by k8s.io/kubernetes/pkg/scheduler.(*Scheduler).Run in goroutine 352
	k8s.io/kubernetes/pkg/scheduler/scheduler.go:445 +0x104
panic: runtime error: integer divide by zero [recovered]
	panic: runtime error: integer divide by zero

goroutine 330 [running]:
k8s.io/apimachinery/pkg/util/runtime.HandleCrash({0x0, 0x0, 0x4000be8e00?})
	k8s.io/apimachinery/pkg/util/runtime/runtime.go:56 +0xe0
panic({0x1b68bc0?, 0x35e1c40?})
	runtime/panic.go:770 +0x124
k8s.io/kubernetes/pkg/scheduler.(*Scheduler).findNodesThatFitPod(0x40000d6900, {0x22aae08, 0x40012b9630}, {0x22d2bb8, 0x4000320fc8}, 0x4000347dc0, 0x4000cb4488)
	k8s.io/kubernetes/pkg/scheduler/schedule_one.go:502 +0x88c
k8s.io/kubernetes/pkg/scheduler.(*Scheduler).schedulePod(0x40000d6900, {0x22aae08, 0x40012b9630}, {0x22d2bb8, 0x4000320fc8}, 0x4000347dc0, 0x4000cb4488)
	k8s.io/kubernetes/pkg/scheduler/schedule_one.go:402 +0x25c
k8s.io/kubernetes/pkg/scheduler.(*Scheduler).schedulingCycle(0x40000d6900, {0x22aae08, 0x40012b9630}, 0x4000347dc0, {0x22d2bb8, 0x4000320fc8}, 0x4000c92e10, {0x2?, 0x2?, 0x36352c0?}, ...)
	k8s.io/kubernetes/pkg/scheduler/schedule_one.go:149 +0xb8
k8s.io/kubernetes/pkg/scheduler.(*Scheduler).ScheduleOne(0x40000d6900, {0x22aae08, 0x40012b83c0})
	k8s.io/kubernetes/pkg/scheduler/schedule_one.go:111 +0x4c0
k8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext.func1()
	k8s.io/apimachinery/pkg/util/wait/backoff.go:259 +0x2c
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x400161dec8?)
	k8s.io/apimachinery/pkg/util/wait/backoff.go:226 +0x40
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x400161df68, {0x2286fe0, 0x4001398b70}, 0x1, 0x40012bab40)
	k8s.io/apimachinery/pkg/util/wait/backoff.go:227 +0x90
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0x4000669f68, 0x0, 0x0, 0x1, 0x40012bab40)
	k8s.io/apimachinery/pkg/util/wait/backoff.go:204 +0x80
k8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext({0x22aae08, 0x40012b83c0}, 0x4001385720, 0x0, 0x0, 0x1)
	k8s.io/apimachinery/pkg/util/wait/backoff.go:259 +0x80
k8s.io/apimachinery/pkg/util/wait.UntilWithContext(...)
	k8s.io/apimachinery/pkg/util/wait/backoff.go:170
created by k8s.io/kubernetes/pkg/scheduler.(*Scheduler).Run in goroutine 352
	k8s.io/kubernetes/pkg/scheduler/scheduler.go:445 +0x104
```

The crash happens [here](https://github.com/kubernetes/kubernetes/blob/7c48c2bd72b9bf5c44d21d7338cc7bea77d0ad2a/pkg/scheduler/schedule_one.go#L502) because the `len(nodes)` is `0` in certain cases. 

### What did you expect to happen?

`kube-scheduler` should not crash, on v1.29.4 it doesn't happen.

On v1.29.4 the `kube-scheduler` is just printing these error logs, but doesn't crash:

```
E0517 13:19:26.611504       1 schedule_one.go:1003] "Error scheduling pod; retrying" err="nodeinfo not found for node name \"invalid-node\"" pod="default/break-kube-scheduler"
```

### How can we reproduce it (as minimally and precisely as possible)?

Create a pod like this:

```yaml
apiVersion: v1
kind: Pod
metadata:
  name: break-kube-scheduler
spec:
  affinity:
    nodeAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
        nodeSelectorTerms:
        - matchFields:
          - key: metadata.name
            operator: In
            values:
            - invalid-node # a node that doesn't exist
  containers:
  - name: main
    image: alpine
    command: ["cat"]
    stdin: true
```
The important part is that the affinity doesn't match a real/valid node.

Once this pod is being processed by `kube-scheduler` it will crash and continue to do so until the pod is deleted.

### Anything else we need to know?

This issue was triggered during the rotation of control-plane nodes. In our setup we update the control plane by scaling from 1 to 2 instances. Once both a ready, the old is being terminated via EC2 API. At this stage the kube-controller-manager _sometimes_ manages to create a replacement daemonset pod once the old node is being deleted. This results in a pod targeting a no longer existing node via `affinity` as illustrated in the example above. For some reason, when the `kube-scheduler` is crashing the `kube-controller-manager` doesn't delete the extra/invalid daemonset pod. Not sure if this is another issue or it has always happened in our setup, but only v1.30 makes `kube-scheduler` crash which causes an actual issue.

### Kubernetes version

<details>

```console
$ kubectl version
Client Version: v1.29.2
Kustomize Version: v5.0.4-0.20230601165947-6ce0bf390ce3
Server Version: v1.30.1
```

</details>


### Cloud provider

<details>
We run a custom Kubernetes setup on AWS EC2 (not EKS).
</details>


### OS version

<details>

```console
# On Linux:
$ cat /etc/os-release
PRETTY_NAME="Ubuntu 22.04.4 LTS"
NAME="Ubuntu"
VERSION_ID="22.04"
VERSION="22.04.4 LTS (Jammy Jellyfish)"
VERSION_CODENAME=jammy
ID=ubuntu
ID_LIKE=debian
HOME_URL="https://www.ubuntu.com/"
SUPPORT_URL="https://help.ubuntu.com/"
BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/"
PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy"
UBUNTU_CODENAME=jammy
$ uname -a
Linux ip-10-149-91-92 6.5.0-1020-aws #20~22.04.1-Ubuntu SMP Wed May  1 16:38:06 UTC 2024 aarch64 aarch64 aarch64 GNU/Linux

# On Windows:
C:\> wmic os get Caption, Version, BuildNumber, OSArchitecture
# paste output here
```

</details>


### Install tools

<details>
Custom tooling: https://github.com/zalando-incubator/kubernetes-on-aws
</details>


### Container runtime (CRI) and version (if applicable)

<details>
</details>


### Related plugins (CNI, CSI, ...) and versions (if applicable)

<details>

</details>



Discussion / hints:
This issue is currently awaiting triage.

If a SIG or subproject determines this is a relevant issue, they will accept it by applying the `triage/accepted` label and provide further guidance.

The `triage/accepted` label can be added by org members by writing `/triage accepted` in a comment.


<details>

Instructions for interacting with me using PR comments are available [here](https://git.k8s.io/community/contributors/guide/pull-requests.md).  If you have questions or suggestions related to my behavior, please file an issue against the [kubernetes-sigs/prow](https://github.com/kubernetes-sigs/prow/issues/new?title=Prow%20issue:) repository.
</details>
/sig scheduling
cc @alculquicondor 
/priority important-soon
@chengjoey @AxeZhan can you take a look?

I believe all the latest patch releases are affected too, because of https://github.com/kubernetes/kubernetes/pull/124559
cc @sanposhiho 
/priority critical-urgent
This crash happens when preFilter plugins filtered out all nodes ...

@alculquicondor 
```
sched.nextStartNodeIndex = (sched.nextStartNodeIndex + processedNodes) % len(nodes)
```

If preFilter filtered out some nodes, here len(nodes) will be a subset of allNodes.
Why are we using this subset at first? I think `%allNodes` should works fine?(seems logical to me)
I think so. In general, we just want to try a different set of nodes.

For the case of Daemonsets, it doesn't really matter, as we will just test one node.
/assign

Successfully catched this with a unit test.
I'll file a small pr with fix https://github.com/kubernetes/kubernetes/issues/124930#issuecomment-2117909102 and a newly added UT.

Fix patch:
diff --git a/pkg/scheduler/schedule_one.go b/pkg/scheduler/schedule_one.go
index 67aaac59da01b..74b1ed2b2fa78 100644
--- a/pkg/scheduler/schedule_one.go
+++ b/pkg/scheduler/schedule_one.go
@@ -500,7 +500,7 @@ func (sched *Scheduler) findNodesThatFitPod(ctx context.Context, fwk framework.F
 	// always try to update the sched.nextStartNodeIndex regardless of whether an error has occurred
 	// this is helpful to make sure that all the nodes have a chance to be searched
 	processedNodes := len(feasibleNodes) + len(diagnosis.NodeToStatusMap)
-	sched.nextStartNodeIndex = (sched.nextStartNodeIndex + processedNodes) % len(nodes)
+	sched.nextStartNodeIndex = (sched.nextStartNodeIndex + processedNodes) % len(allNodes)
 	if err != nil {
 		return nil, diagnosis, err
 	}


Test patch:
diff --git a/pkg/scheduler/schedule_one_test.go b/pkg/scheduler/schedule_one_test.go
index 9b487b1da10ec..83b42556410d6 100644
--- a/pkg/scheduler/schedule_one_test.go
+++ b/pkg/scheduler/schedule_one_test.go
@@ -2431,6 +2431,33 @@ func TestSchedulerSchedulePod(t *testing.T) {
 			// since this case has no score plugin, we'll only try to find one node in Filter stage
 			wantEvaluatedNodes: ptr.To[int32](1),
 		},
+		{
+			name: "test prefilter plugin returned an invalid node",
+			registerPlugins: []tf.RegisterPluginFunc{
+				tf.RegisterQueueSortPlugin(queuesort.Name, queuesort.New),
+				tf.RegisterPreFilterPlugin(
+					"FakePreFilter",
+					tf.NewFakePreFilterPlugin("FakePreFilter", &framework.PreFilterResult{
+						NodeNames: sets.New("invalid-node"),
+					}, nil),
+				),
+				tf.RegisterFilterPlugin("TrueFilter", tf.NewTrueFilterPlugin),
+				tf.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New),
+			},
+			nodes:     []string{"1", "2"},
+			pod:       st.MakePod().Name("test-prefilter").UID("test-prefilter").Obj(),
+			wantNodes: nil,
+			wErr: &framework.FitError{
+				Pod:         st.MakePod().Name("test-prefilter").UID("test-prefilter").Obj(),
+				NumAllNodes: 2,
+				Diagnosis: framework.Diagnosis{
+					NodeToStatusMap: framework.NodeToStatusMap{
+						"1": framework.NewStatus(framework.UnschedulableAndUnresolvable, "node is filtered out by the prefilter result"),
+						"2": framework.NewStatus(framework.UnschedulableAndUnresolvable, "node is filtered out by the prefilter result"),
+					},
+				},
+			},
+		},
 	}
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
diff --git a/test/integration/scheduler/scheduler_test.go b/test/integration/scheduler/scheduler_test.go
index 46e3df821071d..6beac80044543 100644
--- a/test/integration/scheduler/scheduler_test.go
+++ b/test/integration/scheduler/scheduler_test.go
@@ -480,6 +480,33 @@ func TestSchedulerInformers(t *testing.T) {
 			}),
 			preemptedPodIndexes: map[int]struct{}{2: {}},
 		},
+		{
+			name:         "The pod cannot be scheduled when nodeAffinity specifies a non-existent node.",
+			nodes:        []*nodeConfig{{name: "node-1", res: defaultNodeRes}},
+			existingPods: []*v1.Pod{},
+			pod: testutils.InitPausePod(&testutils.PausePodConfig{
+				Name:      "unschedulable-pod",
+				Namespace: testCtx.NS.Name,
+				Affinity: &v1.Affinity{
+					NodeAffinity: &v1.NodeAffinity{
+						RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
+							NodeSelectorTerms: []v1.NodeSelectorTerm{
+								{
+									MatchFields: []v1.NodeSelectorRequirement{
+										{
+											Key:      "metadata.name",
+											Operator: v1.NodeSelectorOpIn,
+											Values:   []string{"invalid-node"},
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+				Resources: defaultPodRes,
+			}),
+		},
 	}
 
 	for _, test := range tests {