Repository: kubernetes-sigs/kueue (Go)

Issue:
TAS: Kueue crashes with panic when the node running a workload is deleted 
**What happened**:

Kueue crashes with panic when a new workload is scheduled after a node is deleted, which was hosting another workload.

**What you expected to happen**:

No panic, admission of new workloads continues.

**How to reproduce it (as minimally and precisely as possible)**:

1. create a TAS workload
2. delete one of the nodes hosting one of the pods of the workload
3. schedule another workload
*Issue*:  kueue panics with the following stacktrace:

```
E1129 18:33:53.869062       1 panic.go:262] "Observed a panic" panic="runtime error: invalid memory address or nil pointer dereference" panicGoValue="\"invalid memory address or nil pointer dereference\"" stacktrace=<
	goroutine 478 [running]:
	k8s.io/apimachinery/pkg/util/runtime.logPanic({0x302f5d0, 0x49be620}, {0x27352c0, 0x49259f0})
		/workspace/vendor/k8s.io/apimachinery/pkg/util/runtime/runtime.go:107 +0xbc
	k8s.io/apimachinery/pkg/util/runtime.handleCrash({0x302f5d0, 0x49be620}, {0x27352c0, 0x49259f0}, {0x49be620, 0x0, 0x10000000043b665?})
		/workspace/vendor/k8s.io/apimachinery/pkg/util/runtime/runtime.go:82 +0x5e
	k8s.io/apimachinery/pkg/util/runtime.HandleCrash({0x0, 0x0, 0xc000baf500?})
		/workspace/vendor/k8s.io/apimachinery/pkg/util/runtime/runtime.go:59 +0x108
	panic({0x27352c0?, 0x49259f0?})
		/usr/local/go/src/runtime/panic.go:785 +0x132
	sigs.k8s.io/kueue/pkg/cache.(*TASFlavorSnapshot).initializeFreeCapacityPerDomain(...)
		/workspace/pkg/cache/tas_flavor_snapshot.go:198
	sigs.k8s.io/kueue/pkg/cache.(*TASFlavorSnapshot).addUsage(...)
		/workspace/pkg/cache/tas_flavor_snapshot.go:193
	sigs.k8s.io/kueue/pkg/cache.(*TASFlavorCache).snapshotForNodes(0xc001702ba0, {{0x3038760?, 0xc001897800?}, 0x0?}, {0xc0018ac408, 0x3, 0x3?}, {0xc0012c5008, 0x10, 0x10})
		/workspace/pkg/cache/tas_flavor.go:121 +0x730
	sigs.k8s.io/kueue/pkg/cache.(*TASFlavorCache).snapshot(0xc001702ba0, {0x302f4f0, 0xc0016c9140})
		/workspace/pkg/cache/tas_flavor.go:105 +0x64e
	sigs.k8s.io/kueue/pkg/cache.(*Cache).Snapshot(0xc000828d20, {0x302f4f0, 0xc0016c9140})
		/workspace/pkg/cache/snapshot.go:103 +0x4c5
	sigs.k8s.io/kueue/pkg/scheduler.(*Scheduler).schedule(0xc000a48820, {0x302f4f0, 0xc0015d6120})
		/workspace/pkg/scheduler/scheduler.go:192 +0x225
	sigs.k8s.io/kueue/pkg/util/wait.untilWithBackoff.func1()
		/workspace/pkg/util/wait/backoff.go:43 +0x2b
	k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc000c8c078?)
		/workspace/vendor/k8s.io/apimachinery/pkg/util/wait/backoff.go:226 +0x33
	k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc001ec5f30, {0x30018c0, 0xc000c8c078}, 0x0, 0xc001b161c0)
		/workspace/vendor/k8s.io/apimachinery/pkg/util/wait/backoff.go:227 +0xaf
	sigs.k8s.io/kueue/pkg/util/wait.untilWithBackoff({0x302f4f0, 0xc0015d6120}, 0xc000f8c050, {0x3028410, 0xc000ff2000})
		/workspace/pkg/util/wait/backoff.go:42 +0xd3
	sigs.k8s.io/kueue/pkg/util/wait.UntilWithBackoff({0x302f4f0, 0xc0015d6120}, 0xc000f8c050)
		/workspace/pkg/util/wait/backoff.go:34 +0x8f
	created by sigs.k8s.io/kueue/pkg/scheduler.(*Scheduler).Start in goroutine 1123
		/workspace/pkg/scheduler/scheduler.go:147 +0x131
```

**Anything else we need to know?**:

The reason is [here](https://github.com/kubernetes-sigs/kueue/blob/cac26f0b8fba6fae0da089ca3f8d04106bbe16de/pkg/cache/tas_flavor_snapshot.go#L192-L194). If the node is deleted, then we didn't initialize the domainID, then we try to addUsage coming from workload in cache. I believe the simplest fix is to just skip adding usage for such domains. This will prevent panic and allow the workloads to schedule on existing nodes. 



Discussion:
cc @PBundyra @mwysokin @mbobrovskyi 
/assign swastik959
/assign mimowo
@mimowo: GitHub didn't allow me to assign the following users: swastik959.

Note that only [kubernetes-sigs members](https://github.com/orgs/kubernetes-sigs/people) with read permissions, repo collaborators and people who have commented on this issue/PR can be assigned. Additionally, issues/PRs can only have 10 assignees at the same time.
For more information please see [the contributor guide](https://git.k8s.io/community/contributors/guide/first-contribution.md#issue-assignment-in-github)

<details>

In response to [this](https://github.com/kubernetes-sigs/kueue/issues/3693#issuecomment-2510744079):

>/assign swastik959
>/assign mimowo


Instructions for interacting with me using PR comments are available [here](https://git.k8s.io/community/contributors/guide/pull-requests.md).  If you have questions or suggestions related to my behavior, please file an issue against the [kubernetes-sigs/prow](https://github.com/kubernetes-sigs/prow/issues/new?title=Prow%20issue:) repository.
</details>



Patch:
diff --git a/pkg/cache/tas_flavor_snapshot.go b/pkg/cache/tas_flavor_snapshot.go
index db2275e9d11..456d2d3661d 100644
--- a/pkg/cache/tas_flavor_snapshot.go
+++ b/pkg/cache/tas_flavor_snapshot.go
@@ -185,19 +185,23 @@ func (s *TASFlavorSnapshot) initializeHelper(dom *domain) {
 }
 
 func (s *TASFlavorSnapshot) addCapacity(domainID utiltas.TopologyDomainID, capacity resources.Requests) {
-	s.initializeFreeCapacityPerDomain(domainID)
+	if s.leaves[domainID].freeCapacity == nil {
+		s.leaves[domainID].freeCapacity = resources.Requests{}
+	}
 	s.leaves[domainID].freeCapacity.Add(capacity)
 }
 
 func (s *TASFlavorSnapshot) addUsage(domainID utiltas.TopologyDomainID, usage resources.Requests) {
-	s.initializeFreeCapacityPerDomain(domainID)
-	s.leaves[domainID].freeCapacity.Sub(usage)
-}
-
-func (s *TASFlavorSnapshot) initializeFreeCapacityPerDomain(domainID utiltas.TopologyDomainID) {
-	if s.leaves[domainID].freeCapacity == nil {
-		s.leaves[domainID].freeCapacity = resources.Requests{}
+	if s.leaves[domainID] == nil {
+		// this can happen if there is an admitted workload for which the
+		// backing node was deleted or is no longer Ready (so the addCapacity
+		// function was not called).
+		s.log.Info("skip accounting for usage in domain", "domain", domainID, "usage", usage)
+		return
 	}
+	// If the leaf domain exists the freeCapacity is already initialized by
+	// the addCapacity function
+	s.leaves[domainID].freeCapacity.Sub(usage)
 }
 
 // Algorithm overview:


Test patch:
diff --git a/pkg/scheduler/scheduler_test.go b/pkg/scheduler/scheduler_test.go
index 67a37ce0ca5..b99e53b20aa 100644
--- a/pkg/scheduler/scheduler_test.go
+++ b/pkg/scheduler/scheduler_test.go
@@ -4544,6 +4544,94 @@ func TestScheduleForTAS(t *testing.T) {
 				},
 			},
 		},
+		"scheduling workload when the node for another admitted workload is deleted": {
+			// Here we have the "bar-admitted" workload, which is admitted and
+			// is using the "x1" node, which is deleted. Still, we have the y1
+			// node which allows to schedule the "foo" workload.
+			nodes: []corev1.Node{
+				*testingnode.MakeNode("y1").
+					Label("tas-node", "true").
+					Label(corev1.LabelHostname, "y1").
+					StatusAllocatable(corev1.ResourceList{
+						corev1.ResourceCPU: resource.MustParse("1"),
+					}).
+					Ready().
+					Obj(),
+			},
+			topologies:      []kueuealpha.Topology{defaultSingleLevelTopology},
+			resourceFlavors: []kueue.ResourceFlavor{defaultTASFlavor},
+			clusterQueues: []kueue.ClusterQueue{
+				*utiltesting.MakeClusterQueue("tas-main").
+					ResourceGroup(
+						*utiltesting.MakeFlavorQuotas("tas-default").
+							Resource(corev1.ResourceCPU, "50").Obj()).
+					Obj(),
+			},
+			workloads: []kueue.Workload{
+				*utiltesting.MakeWorkload("foo", "default").
+					Queue("tas-main").
+					PodSets(
+						*utiltesting.MakePodSet("one", 1).
+							PreferredTopologyRequest(corev1.LabelHostname).
+							Request(corev1.ResourceCPU, "1").
+							Obj(),
+					).
+					Obj(),
+				*utiltesting.MakeWorkload("bar-admitted", "default").
+					Queue("tas-main").
+					ReserveQuota(
+						utiltesting.MakeAdmission("tas-main", "one").
+							Assignment(corev1.ResourceCPU, "tas-default", "1000m").
+							AssignmentPodCount(1).
+							TopologyAssignment(&kueue.TopologyAssignment{
+								Levels: utiltas.Levels(&defaultSingleLevelTopology),
+								Domains: []kueue.TopologyDomainAssignment{
+									{
+										Count: 1,
+										Values: []string{
+											"x1",
+										},
+									},
+								},
+							}).Obj(),
+					).
+					Admitted(true).
+					PodSets(*utiltesting.MakePodSet("one", 1).
+						RequiredTopologyRequest(corev1.LabelHostname).
+						Request(corev1.ResourceCPU, "1").
+						Obj()).
+					Obj(),
+			},
+			wantNewAssignments: map[string]kueue.Admission{
+				"default/foo": *utiltesting.MakeAdmission("tas-main", "one").
+					Assignment(corev1.ResourceCPU, "tas-default", "1000m").
+					AssignmentPodCount(1).
+					TopologyAssignment(&kueue.TopologyAssignment{
+						Levels: utiltas.Levels(&defaultSingleLevelTopology),
+						Domains: []kueue.TopologyDomainAssignment{
+							{
+								Count: 1,
+								Values: []string{
+									"y1",
+								},
+							},
+						},
+					}).Obj(),
+			},
+			eventCmpOpts: []cmp.Option{eventIgnoreMessage},
+			wantEvents: []utiltesting.EventRecord{
+				{
+					Key:       types.NamespacedName{Namespace: "default", Name: "foo"},
+					Reason:    "QuotaReserved",
+					EventType: corev1.EventTypeNormal,
+				},
+				{
+					Key:       types.NamespacedName{Namespace: "default", Name: "foo"},
+					Reason:    "Admitted",
+					EventType: corev1.EventTypeNormal,
+				},
+			},
+		},
 	}
 	for name, tc := range cases {
 		t.Run(name, func(t *testing.T) {