Repository: armadaproject/armada

Issue:
Scheduler panics
<!-- Feel free to skip the sections if they are not applicable. -->

Scheduler panics with the message in logs:

```
2025-08-20T03:54:47.056Z ERROR grpc.go:139 Request triggered panic with cause runtime error: invalid memory address or nil pointer dereference 
goroutine 192233 [running]:
runtime/debug.Stack()
    /opt/hostedtoolcache/go/1.24.6/x64/src/runtime/debug/stack.go:26 +0x5e
github.com/armadaproject/armada/internal/common/grpc.panicRecoveryHandler({0x1f5aac0, 0x38f1be0})
    /home/runner/work/armada/armada/internal/common/grpc/grpc.go:139 +0x25
github.com/armadaproject/armada/internal/common/grpc.CreateGrpcServer.WithRecoveryHandler.func14.1({0xc000c26868?, 0x4194b4?}, {0x1f5aac0?, 0x38f1be0?})
    /home/runner/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware/v2@v2.3.1/interceptors/recovery/options.go:36 +0x27
github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/recovery.recoverFrom({0x265df78?, 0xc001116b40?}, {0x1f5aac0?, 0x38f1be0?}, 0xc00257afc0?)
    /home/runner/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware/v2@v2.3.1/interceptors/recovery/interceptors.go:54 +0xea
github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/recovery.StreamServerInterceptor.func1.1()
    /home/runner/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware/v2@v2.3.1/interceptors/recovery/interceptors.go:44 +0x7b
panic({0x1f5aac0?, 0x38f1be0?})
    /opt/hostedtoolcache/go/1.24.6/x64/src/runtime/panic.go:792 +0x132
k8s.io/apimachinery/pkg/api/resource.(*Quantity).Sub(0x1fb5160?, {{0x0, 0x0}, {0x0}, {0x2690e20, 0x1}, {0x22b676f, 0x9}})
    /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.28.15/pkg/api/resource/quantity.go:585 +0x34
github.com/armadaproject/armada/internal/scheduler/schedulerobjects.(*ResourceList).Sub(0xc000d15508, 0xc000c26c58?)
    /home/runner/work/armada/armada/internal/scheduler/schedulerobjects/resourcelist.go:38 +0x16a
github.com/armadaproject/armada/internal/scheduler/schedulerobjects.AllocatableByPriorityAndResourceType.MarkAllocated(...)
    /home/runner/work/armada/armada/internal/scheduler/schedulerobjects/resourcelist.go:103
github.com/armadaproject/armada/pkg/executorapi.NewNodeFromNodeInfo(0xc001db40a0, {0xc002480158, 0x8}, {0xc0009c4f00, 0x1, 0x8?}, {0xc000700808?, 0xc000d14ae0?, 0x0?})
    /home/runner/work/armada/armada/pkg/executorapi/util.go:36 +0xb16
github.com/armadaproject/armada/internal/scheduler.(*ExecutorApi).executorFromLeaseRequest(0xc0004fa750, 0xc000c00150, 0xc00042f5e0)
    /home/runner/work/armada/armada/internal/scheduler/api.go:376 +0x1ac
github.com/armadaproject/armada/internal/scheduler.(*ExecutorApi).LeaseJobRuns(0xc0004fa750, {0x2668c28, 0xc0006bcd60})
    /home/runner/work/armada/armada/internal/scheduler/api.go:101 +0x1cb
github.com/armadaproject/armada/pkg/executorapi._ExecutorApi_LeaseJobRuns_Handler({0x2143840?, 0xc0004fa750}, {0x2664590, 0xc001116b70})
    /home/runner/work/armada/armada/pkg/executorapi/executorapi.pb.go:977 +0xd8
github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/recovery.StreamServerInterceptor.func1({0x2143840?, 0xc0004fa750?}, {0x2664590?, 0xc001116b70?}, 0x4194b4?, 0xc?)
    /home/runner/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware/v2@v2.3.1/interceptors/recovery/interceptors.go:48 +0x82
google.golang.org/grpc.getChainStreamHandler.func1({0x2143840?, 0xc0004fa750?}, {0x2664590?, 0xc001116b70?})
    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.71.0/server.go:1548 +0x153
github.com/armadaproject/armada/internal/common/grpc.CreateGrpcServer.StreamServerInterceptor.func13({0x2143840?, 0xc0004fa750?}, {0x2664590, 0xc001116b70}, 0xc000c014b8, 0x23f3660?)
    /home/runner/work/armada/armada/internal/common/armadaerrors/errors.go:355 +0x4b
google.golang.org/grpc.getChainStreamHandler.func1.getChainStreamHandler.1({0x2143840, 0xc0004fa750}, {0x2664590, 0xc001116b70})
    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.71.0/server.go:1548 +0xa8
github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/logging.StreamServerInterceptor.StreamServerInterceptor.func2({0x2143840, 0xc0004fa750}, {0x26645d8, 0xc000b846a0}, 0xc001116ae0?, 0xc0011792c0)
    /home/runner/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware/v2@v2.3.1/interceptors/server.go:35 +0x2b5
google.golang.org/grpc.getChainStreamHandler.func1({0x2143840?, 0xc0004fa750?}, {0x26645d8?, 0xc000b846a0?})
    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.71.0/server.go:1548 +0x153
github.com/armadaproject/armada/internal/common/grpc.CreateGrpcServer.StreamServerInterceptor.func11({0x2143840, 0xc0004fa750}, {0x26645d8, 0xc000b846a0}, 0xc000c014b8, 0xc001179280)
    /home/runner/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware/v2@v2.3.1/interceptors/auth/auth.go:68 +0x126
google.golang.org/grpc.getChainStreamHandler.func1.getChainStreamHandler.1({0x2143840, 0xc0004fa750}, {0x26645d8, 0xc000b846a0})
    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.71.0/server.go:1548 +0xa8
github.com/armadaproject/armada/internal/common/grpc.CreateGrpcServer.StreamServerInterceptor.func10({0x2143840, 0xc0004fa750}, {0x2664590, 0xc0011169c0}, 0x30?, 0xc001179240)
    /home/runner/work/armada/armada/internal/common/requestid/interceptors.go:78 +0xe7
google.golang.org/grpc.getChainStreamHandler.func1({0x2143840?, 0xc0004fa750?}, {0x2664590?, 0xc0011169c0?})
    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.71.0/server.go:1548 +0x153
github.com/armadaproject/armada/internal/common/grpc.CreateGrpcServer.(*ServerMetrics).StreamServerInterceptor.StreamServerInterceptor.func17({0x2143840, 0xc0004fa750}, {0x2664b30, 0xc000f60380}, 0xc000c014b8?, 0xc001179200)
    /home/runner/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware/v2@v2.3.1/interceptors/server.go:35 +0x2b5
google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x2143840, 0xc0004fa750}, {0x2664b30, 0xc000f60380}, 0xc000c014b8, 0xc0006bcaf0?)
    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.71.0/server.go:1539 +0x7c
google.golang.org/grpc.(*Server).processStreamingRPC(0xc0009dc200, {0x265df78, 0xc001116810}, 0xc003249aa0, 0xc0009c9500, 0x3907a60, 0x0)
    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.71.0/server.go:1702 +0x1231
google.golang.org/grpc.(*Server).handleStream(0xc0009dc200, {0x265f420, 0xc000a4c000}, 0xc003249aa0)
    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.71.0/server.go:1819 +0xb47
google.golang.org/grpc.(*Server).serveStreams.func2.1()
    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.71.0/server.go:1035 +0x7f
created by google.golang.org/grpc.(*Server).serveStreams.func2 in goroutine 510
    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.71.0/server.go:1046 +0x11d

2025-08-20T03:54:47.057Z ERROR grpc.go:162 finished call grpc.code=Internal grpc.component=server grpc.error="rpc error: code = Internal desc = Internal server error caused by runtime error: invalid memory address or nil pointer dereference" grpc.method=LeaseJobRuns grpc.method_type=bidi_stream grpc.request.deadline=2025-08-20T03:55:16Z grpc.service=executorapi.ExecutorApi grpc.start_time=2025-08-20T03:54:46Z grpc.time_ms=116.211 peer.address=10.244.12.177:54372 protocol=grpc requestId=5CK7HJx8epUZqoQw97G8bf user=anonymous
```

<!-- A clear and concise description of what the bug is. -->

**Files Or Markdown that can reproduce the issue**

Not sure.. Just installed following the docs

**Expected behavior**

Does not panic


Hints from issue comments:
@dejanzele: Closing as resolved

Solution patch:
diff --git a/internal/scheduler/scheduler.go b/internal/scheduler/scheduler.go
index bc1c0a48..214fac20 100644
--- a/internal/scheduler/scheduler.go
+++ b/internal/scheduler/scheduler.go
@@ -407,7 +407,7 @@ func (s *Scheduler) syncState(ctx *armadacontext.Context, initial, fullJobGc boo
 
 		if len(initialRuns) > 0 {
 			updatedRuns = initialRuns
-		} else if maxJobSerial != nil {
+		} else if maxRunSerial != nil {
 			s.runsSerial = *maxRunSerial // Allow the next sync to start from the highest serial possible.
 		}
 
diff --git a/internal/scheduler/schedulerobjects/resourcelist.go b/internal/scheduler/schedulerobjects/resourcelist.go
index 8afa277c..e439899e 100644
--- a/internal/scheduler/schedulerobjects/resourcelist.go
+++ b/internal/scheduler/schedulerobjects/resourcelist.go
@@ -35,6 +35,13 @@ func (a *ResourceList) Sub(b *ResourceList) {
 	a.initialise()
 	for t, qb := range b.Resources {
 		qa := a.Resources[t]
+		if qa == nil {
+			// Skip resources that don't exist in the resource list.
+			// This is an edge case which can happen with aliased resource advertisements.
+			// E.g. if a node advertises "nvidia.com/a100" instead of "nvidia.com/gpu",
+			// but Kubernetes still schedules pods with "nvidia.com/gpu" requests.
+			continue
+		}
 		qa.Sub(*qb)
 		a.Resources[t] = qa
 	}



Test patch:
diff --git a/internal/scheduler/schedulerobjects/resourcelist_test.go b/internal/scheduler/schedulerobjects/resourcelist_test.go
index 9553db56..9af20b5f 100644
--- a/internal/scheduler/schedulerobjects/resourcelist_test.go
+++ b/internal/scheduler/schedulerobjects/resourcelist_test.go
@@ -119,3 +119,74 @@ func TestResourceListEqual(t *testing.T) {
 		})
 	}
 }
+
+func TestResourceListSub(t *testing.T) {
+	tests := map[string]struct {
+		a        *ResourceList
+		b        *ResourceList
+		expected *ResourceList
+	}{
+		"basic subtraction": {
+			a: &ResourceList{
+				Resources: map[string]*resource.Quantity{
+					"cpu":    pointer.MustParseResource("10"),
+					"memory": pointer.MustParseResource("100Gi"),
+				},
+			},
+			b: &ResourceList{
+				Resources: map[string]*resource.Quantity{
+					"cpu":    pointer.MustParseResource("3"),
+					"memory": pointer.MustParseResource("20Gi"),
+				},
+			},
+			expected: &ResourceList{
+				Resources: map[string]*resource.Quantity{
+					"cpu":    pointer.MustParseResource("7"),
+					"memory": pointer.MustParseResource("80Gi"),
+				},
+			},
+		},
+		"subtract with missing resource type": {
+			a: &ResourceList{
+				Resources: map[string]*resource.Quantity{
+					"cpu":             pointer.MustParseResource("10"),
+					"nvidia.com/a100": pointer.MustParseResource("8"),
+				},
+			},
+			b: &ResourceList{
+				Resources: map[string]*resource.Quantity{
+					"cpu":            pointer.MustParseResource("2"),
+					"nvidia.com/gpu": pointer.MustParseResource("1"),
+				},
+			},
+			expected: &ResourceList{
+				Resources: map[string]*resource.Quantity{
+					"cpu":             pointer.MustParseResource("8"),
+					"nvidia.com/a100": pointer.MustParseResource("8"),
+				},
+			},
+		},
+		"subtract empty from non-empty": {
+			a: &ResourceList{
+				Resources: map[string]*resource.Quantity{
+					"cpu": pointer.MustParseResource("10"),
+				},
+			},
+			b: &ResourceList{
+				Resources: map[string]*resource.Quantity{},
+			},
+			expected: &ResourceList{
+				Resources: map[string]*resource.Quantity{
+					"cpu": pointer.MustParseResource("10"),
+				},
+			},
+		},
+	}
+
+	for name, tc := range tests {
+		t.Run(name, func(t *testing.T) {
+			tc.a.Sub(tc.b)
+			assert.True(t, tc.a.Equal(tc.expected))
+		})
+	}
+}