Repository: kubernetes/kubernetes (language: go)

Issue:
allow-metric-labels can not filter metrics
### What happened?

```
apiVersion: v1
kind: Pod
metadata:
  creationTimestamp: null
  labels:
    component: kube-scheduler
    tier: control-plane
  name: kube-scheduler
  namespace: kube-system
spec:
  containers:
  - command:
    - kube-scheduler
    - --bind-address=0.0.0.0
    - --config=/etc/kubernetes/scheduler-config.yaml
    - --v=2
    - --authentication-kubeconfig=/etc/kubernetes/scheduler.conf
    - --authorization-always-allow-paths=/healthz,/readyz,/livez,/configz,/metrics
    - --authorization-kubeconfig=/etc/kubernetes/scheduler.conf
    - --allow-metric-labels="kubernetes_feature_enabled,stage='BETA'","kubernetes_feature_enabled,name='test'"
    image: registry.aliyuncs.com/google_containers/kube-scheduler:v1.30.0
    imagePullPolicy: IfNotPresent
    livenessProbe:
      failureThreshold: 8
      httpGet:
        host: 127.0.0.1
        path: /healthz
        port: 10259
        scheme: HTTPS
      initialDelaySeconds: 10
      periodSeconds: 10
      timeoutSeconds: 15
    name: kube-scheduler
    resources:
      requests:
        cpu: 100m
    startupProbe:
      failureThreshold: 24
      httpGet:
        host: 127.0.0.1
        path: /healthz
        port: 10259
        scheme: HTTPS
      initialDelaySeconds: 10
      periodSeconds: 10
      timeoutSeconds: 15
    volumeMounts:
    - mountPath: /etc/kubernetes/scheduler.conf
      name: kubeconfig
      readOnly: true
    - mountPath: /etc/kubernetes/scheduler-config.yaml
      name: scheduler-config
      readOnly: true
    - mountPath: /etc/kubernetes/extender/
      name: cert
      readOnly: true
    - mountPath: /etc/kubernetes/certs/
      name: scheduler-cert
      readOnly: true     
  hostNetwork: true
  priority: 2000001000
  priorityClassName: system-node-critical
  securityContext:
    seccompProfile:
      type: RuntimeDefault
  volumes:
  - hostPath:
      path: /etc/kubernetes/scheduler.conf
      type: FileOrCreate
    name: kubeconfig
  - hostPath:
      path: /etc/kubernetes/scheduler-config.yaml
      type: FileOrCreate
    name: scheduler-config
  - hostPath:
      path: /etc/kubernetes/extender
      type: DirectoryOrCreate
    name: cert
  - hostPath:
      path: /etc/kubernetes/certs
      type: DirectoryOrCreate
    name: scheduler-cert
---


bernetes_feature_enabled{name="LogarithmicScaleDown",stage="BETA"} 1
kubernetes_feature_enabled{name="LoggingAlphaOptions",stage="ALPHA"} 0
kubernetes_feature_enabled{name="LoggingBetaOptions",stage="BETA"} 1
kubernetes_feature_enabled{name="MatchLabelKeysInPodAffinity",stage="ALPHA"} 0
kubernetes_feature_enabled{name="MatchLabelKeysInPodTopologySpread",stage="BETA"} 1
kubernetes_feature_enabled{name="MaxUnavailableStatefulSet",stage="ALPHA"} 0
kubernetes_feature_enabled{name="MemoryManager",stage="BETA"} 1
kubernetes_feature_enabled{name="MemoryQoS",stage="ALPHA"} 0
kubernetes_feature_enabled{name="MinDomainsInPodTopologySpread",stage=""} 1
kubernetes_feature_enabled{name="MultiCIDRServiceAllocator",stage="ALPHA"} 0
kubernetes_feature_enabled{name="MutatingAdmissionPolicy",stage="ALPHA"} 0
kubernetes_feature_enabled{name="NFTablesProxyMode",stage="ALPHA"} 0
kubernetes_feature_enabled{name="NewVolumeManagerReconstruction",stage=""} 1
kubernetes_feature_enabled{name="NodeInclusionPolicyInPodTopologySpread",stage="BETA"} 1
kubernetes_feature_enabled{name="NodeLogQuery",stage="BETA"} 0
kubernetes_feature_enabled{name="NodeOutOfServiceVolumeDetach",stage=""} 1
kubernetes_feature_enabled{name="NodeSwap",stage="BETA"} 1
kubernetes_feature_enabled{name="OpenAPIEnums",stage="BETA"} 1
kubernetes_feature_enabled{name="PDBUnhealthyPodEvictionPolicy",stage="BETA"} 1
kubernetes_feature_enabled{name="PersistentVolumeLastPhaseTransitionTime",stage="BETA"} 1
kubernetes_feature_enabled{name="PodAndContainerStatsFromCRI",stage="ALPHA"} 0
kubernetes_feature_enabled{name="PodDeletionCost",stage="BETA"} 1
kubernetes_feature_enabled{name="PodDisruptionConditions",stage="
```

### What did you expect to happen?

filter metrics

### How can we reproduce it (as minimally and precisely as possible)?

change kube-scheduler cmd line

### Anything else we need to know?

_No response_

### Kubernetes version

<details>

1.30.0

</details>


### Cloud provider

<details>
wmware workstation
</details>


### OS version

<details>

```console
# On Linux:
$ cat /etc/os-release
# paste output here
$ uname -a
# paste output here

# On Windows:
C:\> wmic os get Caption, Version, BuildNumber, OSArchitecture
# paste output here
```

</details>


### Install tools

<details>

</details>


### Container runtime (CRI) and version (if applicable)

<details>

</details>


### Related plugins (CNI, CSI, ...) and versions (if applicable)

<details>

</details>



Discussion / hints:
/sig node

hi @13567436138 
it might be due to incorrect formatting or unsupported values in the `allow-metric-labels` flag.
To troubleshoot, ensure that the `allow-metric-labels` flag is correctly formatted. 
You might need to check if the metrics you're trying to filter are supported by this flag. 
> it might be due to incorrect formatting or unsupported values in the `allow-metric-labels` flag.

you can specify the allow-list for metric labels using the `--allow-metric-labels` flag. The key format should be `<MetricName>,<LabelName>`, and the value format should be `<allowed_value>,<allowed_value>... (e.g., metric1,label1='v1,v2,v3')`.
you can refer to these pages 
- https://kubernetes.io/docs/reference/scheduling/config/https://kubernetes.io/docs/reference/scheduling/config/
- https://kubernetes.io/docs/concepts/cluster-administration/system-metrics/
Ensure that the allow-metric-labels flag is correctly formatted in your configuration. Here’s an example of the proper format:
```--allow-metric-labels="kubernetes_feature_enabled,stage='BETA'","kubernetes_feature_enabled,name='test'" ```
Additionally, check if the metrics you are trying to filter are supported by this flag.

i think this could also be assigned to sig instrumentation.
/sig instrumentation
How to check,wheather supported

you can check the official Kubernetes documentation for the metrics exposed by the kube-scheduler. 
check this page - https://kubernetes.io/docs/concepts/cluster-administration/system-metrics/
/assign @richabanker 
/triage accepted
the args about `allow-metric-labels` shoue be:
```
--allow-metric-labels=kubernetes_feature_enabled,stage='BETA'
--allow-metric-labels=kubernetes_feature_enabled,name='test'
```
Thanks for catching that!

> the args about `allow-metric-labels` shoue be:
> 
> ```
> --allow-metric-labels=kubernetes_feature_enabled,stage='BETA'
> --allow-metric-labels=kubernetes_feature_enabled,name='test'
> ```


/kind support
/triage accepted
Given https://github.com/kubernetes/kubernetes/issues/126526#issuecomment-2277446781, I think we can close.

/close
@kannon92: Closing this issue.

<details>

In response to [this](https://github.com/kubernetes/kubernetes/issues/126526#issuecomment-2290479694):

>Given https://github.com/kubernetes/kubernetes/issues/126526#issuecomment-2277446781, I think we can close.
>
>/close


Instructions for interacting with me using PR comments are available [here](https://git.k8s.io/community/contributors/guide/pull-requests.md).  If you have questions or suggestions related to my behavior, please file an issue against the [kubernetes-sigs/prow](https://github.com/kubernetes-sigs/prow/issues/new?title=Prow%20issue:) repository.
</details>
/reopen

I saw it was assigned so I'll leave it open.
@kannon92: Reopened this issue.

<details>

In response to [this](https://github.com/kubernetes/kubernetes/issues/126526#issuecomment-2290480074):

>/reopen
>
>I saw it was assigned so I'll leave it open.


Instructions for interacting with me using PR comments are available [here](https://git.k8s.io/community/contributors/guide/pull-requests.md).  If you have questions or suggestions related to my behavior, please file an issue against the [kubernetes-sigs/prow](https://github.com/kubernetes-sigs/prow/issues/new?title=Prow%20issue:) repository.
</details>

Fix patch:
diff --git a/pkg/scheduler/metrics/metrics.go b/pkg/scheduler/metrics/metrics.go
index a2fd58ba801af..84a1bb9e7e51c 100644
--- a/pkg/scheduler/metrics/metrics.go
+++ b/pkg/scheduler/metrics/metrics.go
@@ -87,6 +87,51 @@ const (
 
 // All the histogram based metrics have 1ms as size for the smallest bucket.
 var (
+	scheduleAttempts           *metrics.CounterVec
+	EventHandlingLatency       *metrics.HistogramVec
+	schedulingLatency          *metrics.HistogramVec
+	SchedulingAlgorithmLatency *metrics.Histogram
+	PreemptionVictims          *metrics.Histogram
+	PreemptionAttempts         *metrics.Counter
+	pendingPods                *metrics.GaugeVec
+	InFlightEvents             *metrics.GaugeVec
+	Goroutines                 *metrics.GaugeVec
+
+	// PodSchedulingDuration is deprecated as of Kubernetes v1.28, and will be removed
+	// in v1.31. Please use PodSchedulingSLIDuration instead.
+	PodSchedulingDuration           *metrics.HistogramVec
+	PodSchedulingSLIDuration        *metrics.HistogramVec
+	PodSchedulingAttempts           *metrics.Histogram
+	FrameworkExtensionPointDuration *metrics.HistogramVec
+	PluginExecutionDuration         *metrics.HistogramVec
+
+	// This is only available when the QHint feature gate is enabled.
+	queueingHintExecutionDuration *metrics.HistogramVec
+	SchedulerQueueIncomingPods    *metrics.CounterVec
+	PermitWaitDuration            *metrics.HistogramVec
+	CacheSize                     *metrics.GaugeVec
+	unschedulableReasons          *metrics.GaugeVec
+	PluginEvaluationTotal         *metrics.CounterVec
+	metricsList                   []metrics.Registerable
+)
+
+var registerMetrics sync.Once
+
+// Register all metrics.
+func Register() {
+	// Register the metrics.
+	registerMetrics.Do(func() {
+		InitMetrics()
+		RegisterMetrics(metricsList...)
+		if utilfeature.DefaultFeatureGate.Enabled(features.SchedulerQueueingHints) {
+			RegisterMetrics(queueingHintExecutionDuration)
+			RegisterMetrics(InFlightEvents)
+		}
+		volumebindingmetrics.RegisterVolumeSchedulingMetrics()
+	})
+}
+
+func InitMetrics() {
 	scheduleAttempts = metrics.NewCounterVec(
 		&metrics.CounterOpts{
 			Subsystem:      SchedulerSubsystem,
@@ -292,21 +337,6 @@ var (
 		unschedulableReasons,
 		PluginEvaluationTotal,
 	}
-)
-
-var registerMetrics sync.Once
-
-// Register all metrics.
-func Register() {
-	// Register the metrics.
-	registerMetrics.Do(func() {
-		RegisterMetrics(metricsList...)
-		if utilfeature.DefaultFeatureGate.Enabled(features.SchedulerQueueingHints) {
-			RegisterMetrics(queueingHintExecutionDuration)
-			RegisterMetrics(InFlightEvents)
-		}
-		volumebindingmetrics.RegisterVolumeSchedulingMetrics()
-	})
 }
 
 // RegisterMetrics registers a list of metrics.


Test patch:
diff --git a/pkg/scheduler/backend/cache/cache_test.go b/pkg/scheduler/backend/cache/cache_test.go
index 61fe56000eb7b..b28579f65dc92 100644
--- a/pkg/scheduler/backend/cache/cache_test.go
+++ b/pkg/scheduler/backend/cache/cache_test.go
@@ -33,6 +33,7 @@ import (
 	"k8s.io/klog/v2"
 	"k8s.io/klog/v2/ktesting"
 	"k8s.io/kubernetes/pkg/scheduler/framework"
+	"k8s.io/kubernetes/pkg/scheduler/metrics"
 	st "k8s.io/kubernetes/pkg/scheduler/testing"
 	schedutil "k8s.io/kubernetes/pkg/scheduler/util"
 )
@@ -264,6 +265,7 @@ func assumeAndFinishBinding(logger klog.Logger, cache *cacheImpl, pod *v1.Pod, a
 // TestExpirePod tests that assumed pods will be removed if expired.
 // The removal will be reflected in node info.
 func TestExpirePod(t *testing.T) {
+	metrics.Register()
 	nodeName := "node"
 	testPods := []*v1.Pod{
 		makeBasePod(t, nodeName, "test-1", "100m", "500", "", []v1.ContainerPort{{HostIP: "127.0.0.1", HostPort: 80, Protocol: "TCP"}}),
diff --git a/pkg/scheduler/backend/queue/active_queue_test.go b/pkg/scheduler/backend/queue/active_queue_test.go
index c22abe67294b2..cdb0aa0af3ef0 100644
--- a/pkg/scheduler/backend/queue/active_queue_test.go
+++ b/pkg/scheduler/backend/queue/active_queue_test.go
@@ -29,6 +29,7 @@ import (
 
 func TestClose(t *testing.T) {
 	logger, ctx := ktesting.NewTestContext(t)
+	metrics.Register()
 	rr := metrics.NewMetricsAsyncRecorder(10, time.Second, ctx.Done())
 	aq := newActiveQueue(heap.NewWithRecorder(podInfoKeyFunc, heap.LessFunc[*framework.QueuedPodInfo](newDefaultQueueSort()), metrics.NewActivePodsRecorder()), true, *rr)
 
diff --git a/pkg/scheduler/backend/queue/scheduling_queue_test.go b/pkg/scheduler/backend/queue/scheduling_queue_test.go
index 5ea16250197fc..5450526ebf1d8 100644
--- a/pkg/scheduler/backend/queue/scheduling_queue_test.go
+++ b/pkg/scheduler/backend/queue/scheduling_queue_test.go
@@ -120,6 +120,7 @@ func TestPriorityQueue_Add(t *testing.T) {
 	logger, ctx := ktesting.NewTestContext(t)
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
+	metrics.Register()
 	q := NewTestQueueWithObjects(ctx, newDefaultQueueSort(), objs)
 	q.Add(logger, medPriorityPodInfo.Pod)
 	q.Add(logger, unschedulablePodInfo.Pod)
@@ -2926,7 +2927,6 @@ func TestPodTimestamp(t *testing.T) {
 // TestPendingPodsMetric tests Prometheus metrics related with pending pods
 func TestPendingPodsMetric(t *testing.T) {
 	timestamp := time.Now()
-	metrics.Register()
 	total := 60
 	queueableNum := 50
 	queueable, failme := "queueable", "failme"
@@ -2951,6 +2951,7 @@ func TestPendingPodsMetric(t *testing.T) {
 			pInfosWithDelay[i].Attempts = 0
 		}
 	}
+	metrics.Register()
 
 	tests := []struct {
 		name                       string
@@ -3185,11 +3186,11 @@ scheduler_plugin_execution_duration_seconds_count{extension_point="PreEnqueue",p
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			resetMetrics()
-			resetPodInfos()
 			logger, ctx := ktesting.NewTestContext(t)
 			ctx, cancel := context.WithCancel(ctx)
 			defer cancel()
+			resetMetrics()
+			resetPodInfos()
 
 			m := map[string][]framework.PreEnqueuePlugin{"": {&preEnqueuePlugin{allowlists: []string{queueable}}}}
 			recorder := metrics.NewMetricsAsyncRecorder(3, 20*time.Microsecond, ctx.Done())
@@ -3326,8 +3327,8 @@ func TestPerPodSchedulingMetrics(t *testing.T) {
 func TestIncomingPodsMetrics(t *testing.T) {
 	timestamp := time.Now()
 	unschedulablePlg := "unschedulable_plugin"
-	metrics.Register()
 	var pInfos = make([]*framework.QueuedPodInfo, 0, 3)
+	metrics.Register()
 	for i := 1; i <= 3; i++ {
 		p := &framework.QueuedPodInfo{
 			PodInfo: mustNewTestPodInfo(t,
@@ -3399,10 +3400,10 @@ func TestIncomingPodsMetrics(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			metrics.SchedulerQueueIncomingPods.Reset()
 			logger, ctx := ktesting.NewTestContext(t)
 			ctx, cancel := context.WithCancel(ctx)
 			defer cancel()
+			metrics.SchedulerQueueIncomingPods.Reset()
 			queue := NewTestQueue(ctx, newDefaultQueueSort(), WithClock(testingclock.NewFakeClock(timestamp)))
 			for _, op := range test.operations {
 				for _, pInfo := range pInfos {
diff --git a/pkg/scheduler/eventhandlers_test.go b/pkg/scheduler/eventhandlers_test.go
index bc2537defd4b1..52766571dd551 100644
--- a/pkg/scheduler/eventhandlers_test.go
+++ b/pkg/scheduler/eventhandlers_test.go
@@ -48,6 +48,7 @@ import (
 	"k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodename"
 	"k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeports"
 	"k8s.io/kubernetes/pkg/scheduler/framework/plugins/noderesources"
+	"k8s.io/kubernetes/pkg/scheduler/metrics"
 	st "k8s.io/kubernetes/pkg/scheduler/testing"
 	"k8s.io/kubernetes/pkg/scheduler/util/assumecache"
 )
@@ -55,6 +56,7 @@ import (
 func TestUpdatePodInCache(t *testing.T) {
 	ttl := 10 * time.Second
 	nodeName := "node"
+	metrics.Register()
 
 	tests := []struct {
 		name   string
diff --git a/pkg/scheduler/framework/plugins/defaultpreemption/default_preemption_test.go b/pkg/scheduler/framework/plugins/defaultpreemption/default_preemption_test.go
index eed1896ae4321..933b24683ee2f 100644
--- a/pkg/scheduler/framework/plugins/defaultpreemption/default_preemption_test.go
+++ b/pkg/scheduler/framework/plugins/defaultpreemption/default_preemption_test.go
@@ -59,6 +59,7 @@ import (
 	"k8s.io/kubernetes/pkg/scheduler/framework/plugins/tainttoleration"
 	"k8s.io/kubernetes/pkg/scheduler/framework/preemption"
 	frameworkruntime "k8s.io/kubernetes/pkg/scheduler/framework/runtime"
+	"k8s.io/kubernetes/pkg/scheduler/metrics"
 	st "k8s.io/kubernetes/pkg/scheduler/testing"
 	tf "k8s.io/kubernetes/pkg/scheduler/testing/framework"
 )
@@ -143,6 +144,7 @@ func (pl *TestPlugin) Filter(ctx context.Context, state *framework.CycleState, p
 }
 
 func TestPostFilter(t *testing.T) {
+	metrics.Register()
 	onePodRes := map[v1.ResourceName]string{v1.ResourcePods: "1"}
 	nodeRes := map[v1.ResourceName]string{v1.ResourceCPU: "200m", v1.ResourceMemory: "400"}
 	tests := []struct {
@@ -426,6 +428,7 @@ type candidate struct {
 }
 
 func TestDryRunPreemption(t *testing.T) {
+	metrics.Register()
 	tests := []struct {
 		name                    string
 		args                    *config.DefaultPreemptionArgs
diff --git a/pkg/scheduler/framework/plugins/interpodaffinity/filtering_test.go b/pkg/scheduler/framework/plugins/interpodaffinity/filtering_test.go
index 8f3d88a2e2560..e6c272856b1c6 100644
--- a/pkg/scheduler/framework/plugins/interpodaffinity/filtering_test.go
+++ b/pkg/scheduler/framework/plugins/interpodaffinity/filtering_test.go
@@ -31,6 +31,7 @@ import (
 	"k8s.io/kubernetes/pkg/scheduler/backend/cache"
 	"k8s.io/kubernetes/pkg/scheduler/framework"
 	plugintesting "k8s.io/kubernetes/pkg/scheduler/framework/plugins/testing"
+	"k8s.io/kubernetes/pkg/scheduler/metrics"
 	st "k8s.io/kubernetes/pkg/scheduler/testing"
 )
 
@@ -68,6 +69,7 @@ func TestRequiredAffinitySingleNode(t *testing.T) {
 	}
 	podLabel2 := map[string]string{"security": "S1"}
 	node1 := v1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node1", Labels: labels1}}
+	metrics.Register()
 
 	tests := []struct {
 		pod                 *v1.Pod
diff --git a/pkg/scheduler/framework/plugins/podtopologyspread/filtering_test.go b/pkg/scheduler/framework/plugins/podtopologyspread/filtering_test.go
index cf7ea3569d9de..fd8c5002be43d 100644
--- a/pkg/scheduler/framework/plugins/podtopologyspread/filtering_test.go
+++ b/pkg/scheduler/framework/plugins/podtopologyspread/filtering_test.go
@@ -35,6 +35,7 @@ import (
 	"k8s.io/kubernetes/pkg/scheduler/framework/plugins/feature"
 	plugintesting "k8s.io/kubernetes/pkg/scheduler/framework/plugins/testing"
 	frameworkruntime "k8s.io/kubernetes/pkg/scheduler/framework/runtime"
+	"k8s.io/kubernetes/pkg/scheduler/metrics"
 	st "k8s.io/kubernetes/pkg/scheduler/testing"
 	"k8s.io/utils/ptr"
 )
@@ -68,6 +69,7 @@ func (p *criticalPaths) sort() {
 }
 
 func TestPreFilterState(t *testing.T) {
+	metrics.Register()
 	tests := []struct {
 		name                      string
 		pod                       *v1.Pod
@@ -2388,6 +2390,7 @@ func TestPreFilterStateRemovePod(t *testing.T) {
 }
 
 func BenchmarkFilter(b *testing.B) {
+	metrics.Register()
 	tests := []struct {
 		name             string
 		pod              *v1.Pod
diff --git a/pkg/scheduler/framework/preemption/preemption_test.go b/pkg/scheduler/framework/preemption/preemption_test.go
index f4a0bff99c931..f0189d2095298 100644
--- a/pkg/scheduler/framework/preemption/preemption_test.go
+++ b/pkg/scheduler/framework/preemption/preemption_test.go
@@ -36,6 +36,7 @@ import (
 	"k8s.io/kubernetes/pkg/scheduler/framework/plugins/defaultbinder"
 	"k8s.io/kubernetes/pkg/scheduler/framework/plugins/queuesort"
 	frameworkruntime "k8s.io/kubernetes/pkg/scheduler/framework/runtime"
+	"k8s.io/kubernetes/pkg/scheduler/metrics"
 	st "k8s.io/kubernetes/pkg/scheduler/testing"
 	tf "k8s.io/kubernetes/pkg/scheduler/testing/framework"
 )
@@ -113,6 +114,7 @@ func (pl *FakePreemptionScorePostFilterPlugin) OrderedScoreFuncs(ctx context.Con
 }
 
 func TestDryRunPreemption(t *testing.T) {
+	metrics.Register()
 	tests := []struct {
 		name               string
 		nodes              []*v1.Node
diff --git a/pkg/scheduler/framework/runtime/framework_test.go b/pkg/scheduler/framework/runtime/framework_test.go
index 7b0f3ae63e6ea..d3f27bdb79987 100644
--- a/pkg/scheduler/framework/runtime/framework_test.go
+++ b/pkg/scheduler/framework/runtime/framework_test.go
@@ -457,6 +457,7 @@ func newFrameworkWithQueueSortAndBind(ctx context.Context, r Registry, profile c
 }
 
 func TestInitFrameworkWithScorePlugins(t *testing.T) {
+	metrics.Register()
 	tests := []struct {
 		name    string
 		plugins *config.Plugins
@@ -2900,7 +2901,7 @@ func withMetricsRecorder(recorder *metrics.MetricAsyncRecorder) Option {
 func TestRecordingMetrics(t *testing.T) {
 	state := &framework.CycleState{}
 	state.SetRecordPluginMetrics(true)
-
+	metrics.Register()
 	tests := []struct {
 		name               string
 		action             func(ctx context.Context, f framework.Framework)
@@ -3027,7 +3028,8 @@ func TestRecordingMetrics(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			metrics.Register()
+			_, ctx := ktesting.NewTestContext(t)
+			ctx, cancel := context.WithCancel(ctx)
 			metrics.FrameworkExtensionPointDuration.Reset()
 			metrics.PluginExecutionDuration.Reset()
 
@@ -3050,9 +3052,6 @@ func TestRecordingMetrics(t *testing.T) {
 				PostBind:  pluginSet,
 			}
 
-			_, ctx := ktesting.NewTestContext(t)
-			ctx, cancel := context.WithCancel(ctx)
-
 			recorder := metrics.NewMetricsAsyncRecorder(100, time.Nanosecond, ctx.Done())
 			profile := config.KubeSchedulerProfile{
 				PercentageOfNodesToScore: ptr.To[int32](testPercentageOfNodesToScore),
@@ -3086,6 +3085,7 @@ func TestRecordingMetrics(t *testing.T) {
 }
 
 func TestRunBindPlugins(t *testing.T) {
+	metrics.Register()
 	tests := []struct {
 		name       string
 		injects    []framework.Code
@@ -3154,7 +3154,6 @@ func TestRunBindPlugins(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			metrics.Register()
 			metrics.FrameworkExtensionPointDuration.Reset()
 			metrics.PluginExecutionDuration.Reset()
 
@@ -3203,6 +3202,7 @@ func TestRunBindPlugins(t *testing.T) {
 }
 
 func TestPermitWaitDurationMetric(t *testing.T) {
+	metrics.Register()
 	tests := []struct {
 		name    string
 		inject  injectedResult
@@ -3221,7 +3221,6 @@ func TestPermitWaitDurationMetric(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			_, ctx := ktesting.NewTestContext(t)
-			metrics.Register()
 			metrics.PermitWaitDuration.Reset()
 
 			plugin := &TestPlugin{name: testPlugin, inj: tt.inject}
diff --git a/pkg/scheduler/metrics/metric_recorder_test.go b/pkg/scheduler/metrics/metric_recorder_test.go
index ad12c032ed171..2734b6873a221 100644
--- a/pkg/scheduler/metrics/metric_recorder_test.go
+++ b/pkg/scheduler/metrics/metric_recorder_test.go
@@ -108,6 +108,7 @@ func TestClear(t *testing.T) {
 }
 
 func TestInFlightEventAsync(t *testing.T) {
+	Register()
 	r := &MetricAsyncRecorder{
 		aggregatedInflightEventMetric:              map[gaugeVecMetricKey]int{},
 		aggregatedInflightEventMetricLastFlushTime: time.Now(),