Repository: kubernetes
Issue: scheduler: fix and test "unschedulable_pods" metric
Labels: kind/bug,sig/scheduling,lifecycle/frozen,needs-triage

### What happened?

While debugging a problem with pods getting stuck in the unschedulable queue, I looked at the code which handles the "unschedulable_pods". It's probably not working as intended and lacks unit tests.

See https://github.com/kubernetes/kubernetes/pull/120334#discussion_r1314302936.

### What did you expect to happen?

Let's add unit tests and then fix the code.

### How can we reproduce it (as minimally and precisely as possible)?

Here is a preliminary patch which adds tests.

Note that this does *not* cover https://github.com/kubernetes/kubernetes/blob/cd04f47e8b287068e8bf0bfae6596b2606d54929/pkg/scheduler/internal/queue/scheduling_queue.go#L469

It's not clear whether that `Inc` is needed. It might lead to double counting of the same plugin, once when adding it to `PodInfo`, once when enqueuing that pod.

```patch
diff --git a/pkg/scheduler/internal/queue/scheduling_queue_test.go b/pkg/scheduler/internal/queue/scheduling_queue_test.go
index 541f3f7d42c..bf1d6572401 100644
--- a/pkg/scheduler/internal/queue/scheduling_queue_test.go
+++ b/pkg/scheduler/internal/queue/scheduling_queue_test.go
@@ -54,6 +54,10 @@ const queueMetricMetadata = `
 		# HELP scheduler_queue_incoming_pods_total [STABLE] Number of pods added to scheduling queues by event and queue type.
 		# TYPE scheduler_queue_incoming_pods_total counter
 	`
+const queueMetricMetadataUnschedulable = `
+                # HELP scheduler_unschedulable_pods [ALPHA] The number of unschedulable pods broken down by plugin name. A pod will increment the gauge for all plugins that caused it to not schedule and so this metric have meaning only when broken down by plugin.
+                # TYPE scheduler_unschedulable_pods gauge
+        `
 
 var (
 	TestEvent    = framework.ClusterEvent{Resource: "test"}
@@ -2664,6 +2668,96 @@ func TestIncomingPodsMetrics(t *testing.T) {
 	}
 }
 
+func TestUnschedulablePodsMetrics(t *testing.T) {
+	timestamp := time.Now()
+	unschedulablePlg := "unschedulable_plugin"
+	metrics.Register()
+	var pInfos = make([]*framework.QueuedPodInfo, 0, 3)
+	for i := 1; i <= 3; i++ {
+		p := &framework.QueuedPodInfo{
+			PodInfo: mustNewTestPodInfo(t,
+				st.MakePod().Name(fmt.Sprintf("test-pod-%d", i)).Namespace(fmt.Sprintf("ns%d", i)).UID(fmt.Sprintf("tp-%d", i)).Obj()),
+			Timestamp:            timestamp,
+			UnschedulablePlugins: sets.New(unschedulablePlg),
+		}
+		pInfos = append(pInfos, p)
+	}
+	tests := []struct {
+		name       string
+		operations []operation
+		want       string
+	}{
+		{
+			name: "add pods to activeQ",
+			operations: []operation{
+				add,
+			},
+			want: ``,
+		},
+		{
+			name: "add pods to unschedulablePods",
+			operations: []operation{
+				addUnschedulablePodBackToUnschedulablePods,
+			},
+			want: `
+             scheduler_unschedulable_pods{plugin="unschedulable_plugin",profile=""} 3
+`,
+		},
+		{
+			name: "add pods to unschedulablePods and then move all to backoffQ",
+			operations: []operation{
+				addUnschedulablePodBackToUnschedulablePods,
+				moveAllToActiveOrBackoffQ,
+			},
+			want: `
+             scheduler_unschedulable_pods{plugin="unschedulable_plugin",profile=""} 0
+`,
+		},
+		{
+			name: "add pods to unschedulablePods and then move all to activeQ",
+			operations: []operation{
+				addUnschedulablePodBackToUnschedulablePods,
+				moveClockForward,
+				moveAllToActiveOrBackoffQ,
+			},
+			want: `
+             scheduler_unschedulable_pods{plugin="unschedulable_plugin",profile=""} 0
+`,
+		},
+		{
+			name: "make some pods subject to backoff and add them to backoffQ, then flush backoffQ",
+			operations: []operation{
+				addUnschedulablePodBackToBackoffQ,
+				moveClockForward,
+				flushBackoffQ,
+			},
+			want: `
+             scheduler_unschedulable_pods{plugin="unschedulable_plugin",profile=""} 0
+`,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			metrics.SchedulerQueueIncomingPods.Reset()
+			logger, ctx := ktesting.NewTestContext(t)
+			ctx, cancel := context.WithCancel(ctx)
+			defer cancel()
+			queue := NewTestQueue(ctx, newDefaultQueueSort(), WithClock(testingclock.NewFakeClock(timestamp)))
+			for _, op := range test.operations {
+				for _, pInfo := range pInfos {
+					op(logger, queue, pInfo)
+				}
+			}
+			metricName := metrics.SchedulerSubsystem + "_" + metrics.UnschedulableReasons.Name
+			if err := testutil.CollectAndCompare(metrics.UnschedulableReasons, strings.NewReader(queueMetricMetadataUnschedulable+test.want), metricName); err != nil {
+				t.Errorf("unexpected collecting result:\n%s", err)
+			}
+
+		})
+	}
+}
+
 func TestBackOffFlow(t *testing.T) {
 	cl := testingclock.NewFakeClock(time.Now())
 	logger, ctx := ktesting.NewTestContext(t)
diff --git a/pkg/scheduler/metrics/metrics.go b/pkg/scheduler/metrics/metrics.go
index d4871e70d7f..115045fd1ad 100644
--- a/pkg/scheduler/metrics/metrics.go
+++ b/pkg/scheduler/metrics/metrics.go
@@ -197,7 +197,7 @@ var (
 			StabilityLevel: metrics.ALPHA,
 		}, []string{"type"})
 
-	unschedulableReasons = metrics.NewGaugeVec(
+	UnschedulableReasons = metrics.NewGaugeVec(
 		&metrics.GaugeOpts{
 			Subsystem:      SchedulerSubsystem,
 			Name:           "unschedulable_pods",
@@ -229,7 +229,7 @@ var (
 		Goroutines,
 		PermitWaitDuration,
 		CacheSize,
-		unschedulableReasons,
+		UnschedulableReasons,
 		PluginEvaluationTotal,
 	}
 )
@@ -284,5 +284,5 @@ func SinceInSeconds(start time.Time) float64 {
 }
 
 func UnschedulableReason(plugin string, profile string) metrics.GaugeMetric {
-	return unschedulableReasons.With(metrics.Labels{"plugin": plugin, "profile": profile})
+	return UnschedulableReasons.With(metrics.Labels{"plugin": plugin, "profile": profile})
 }
```

### Anything else we need to know?

_No response_

### Kubernetes version

>= 1.28


### Cloud provider

n/a

### OS version

_No response_

### Install tools

_No response_

### Container runtime (CRI) and version (if applicable)

_No response_

### Related plugins (CNI, CSI, ...) and versions (if applicable)

_No response_