Repository: kubernetes/kubernetes (language: go)

Issue:
allow-metric-labels can not filter metrics
### What happened?

```
apiVersion: v1
kind: Pod
metadata:
  creationTimestamp: null
  labels:
    component: kube-scheduler
    tier: control-plane
  name: kube-scheduler
  namespace: kube-system
spec:
  containers:
  - command:
    - kube-scheduler
    - --bind-address=0.0.0.0
    - --config=/etc/kubernetes/scheduler-config.yaml
    - --v=2
    - --authentication-kubeconfig=/etc/kubernetes/scheduler.conf
    - --authorization-always-allow-paths=/healthz,/readyz,/livez,/configz,/metrics
    - --authorization-kubeconfig=/etc/kubernetes/scheduler.conf
    - --allow-metric-labels="kubernetes_feature_enabled,stage='BETA'","kubernetes_feature_enabled,name='test'"
    image: registry.aliyuncs.com/google_containers/kube-scheduler:v1.30.0
    imagePullPolicy: IfNotPresent
    livenessProbe:
      failureThreshold: 8
      httpGet:
        host: 127.0.0.1
        path: /healthz
        port: 10259
        scheme: HTTPS
      initialDelaySeconds: 10
      periodSeconds: 10
      timeoutSeconds: 15
    name: kube-scheduler
    resources:
      requests:
        cpu: 100m
    startupProbe:
      failureThreshold: 24
      httpGet:
        host: 127.0.0.1
        path: /healthz
        port: 10259
        scheme: HTTPS
      initialDelaySeconds: 10
      periodSeconds: 10
      timeoutSeconds: 15
    volumeMounts:
    - mountPath: /etc/kubernetes/scheduler.conf
      name: kubeconfig
      readOnly: true
    - mountPath: /etc/kubernetes/scheduler-config.yaml
      name: scheduler-config
      readOnly: true
    - mountPath: /etc/kubernetes/extender/
      name: cert
      readOnly: true
    - mountPath: /etc/kubernetes/certs/
      name: scheduler-cert
      readOnly: true     
  hostNetwork: true
  priority: 2000001000
  priorityClassName: system-node-critical
  securityContext:
    seccompProfile:
      type: RuntimeDefault
  volumes:
  - hostPath:
      path: /etc/kubernetes/scheduler.conf
      type: FileOrCreate
    name: kubeconfig
  - hostPath:
      path: /etc/kubernetes/scheduler-config.yaml
      type: FileOrCreate
    name: scheduler-config
  - hostPath:
      path: /etc/kubernetes/extender
      type: DirectoryOrCreate
    name: cert
  - hostPath:
      path: /etc/kubernetes/certs
      type: DirectoryOrCreate
    name: scheduler-cert
---


bernetes_feature_enabled{name="LogarithmicScaleDown",stage="BETA"} 1
kubernetes_feature_enabled{name="LoggingAlphaOptions",stage="ALPHA"} 0
kubernetes_feature_enabled{name="LoggingBetaOptions",stage="BETA"} 1
kubernetes_feature_enabled{name="MatchLabelKeysInPodAffinity",stage="ALPHA"} 0
kubernetes_feature_enabled{name="MatchLabelKeysInPodTopologySpread",stage="BETA"} 1
kubernetes_feature_enabled{name="MaxUnavailableStatefulSet",stage="ALPHA"} 0
kubernetes_feature_enabled{name="MemoryManager",stage="BETA"} 1
kubernetes_feature_enabled{name="MemoryQoS",stage="ALPHA"} 0
kubernetes_feature_enabled{name="MinDomainsInPodTopologySpread",stage=""} 1
kubernetes_feature_enabled{name="MultiCIDRServiceAllocator",stage="ALPHA"} 0
kubernetes_feature_enabled{name="MutatingAdmissionPolicy",stage="ALPHA"} 0
kubernetes_feature_enabled{name="NFTablesProxyMode",stage="ALPHA"} 0
kubernetes_feature_enabled{name="NewVolumeManagerReconstruction",stage=""} 1
kubernetes_feature_enabled{name="NodeInclusionPolicyInPodTopologySpread",stage="BETA"} 1
kubernetes_feature_enabled{name="NodeLogQuery",stage="BETA"} 0
kubernetes_feature_enabled{name="NodeOutOfServiceVolumeDetach",stage=""} 1
kubernetes_feature_enabled{name="NodeSwap",stage="BETA"} 1
kubernetes_feature_enabled{name="OpenAPIEnums",stage="BETA"} 1
kubernetes_feature_enabled{name="PDBUnhealthyPodEvictionPolicy",stage="BETA"} 1
kubernetes_feature_enabled{name="PersistentVolumeLastPhaseTransitionTime",stage="BETA"} 1
kubernetes_feature_enabled{name="PodAndContainerStatsFromCRI",stage="ALPHA"} 0
kubernetes_feature_enabled{name="PodDeletionCost",stage="BETA"} 1
kubernetes_feature_enabled{name="PodDisruptionConditions",stage="
```

### What did you expect to happen?

filter metrics

### How can we reproduce it (as minimally and precisely as possible)?

change kube-scheduler cmd line

### Anything else we need to know?

_No response_

### Kubernetes version

<details>

1.30.0

</details>


### Cloud provider

<details>
wmware workstation
</details>


### OS version

<details>

```console
# On Linux:
$ cat /etc/os-release
# paste output here
$ uname -a
# paste output here

# On Windows:
C:\> wmic os get Caption, Version, BuildNumber, OSArchitecture
# paste output here
```

</details>


### Install tools

<details>

</details>


### Container runtime (CRI) and version (if applicable)

<details>

</details>


### Related plugins (CNI, CSI, ...) and versions (if applicable)

<details>

</details>



Discussion / hints:
/sig node

hi @13567436138 
it might be due to incorrect formatting or unsupported values in the `allow-metric-labels` flag.
To troubleshoot, ensure that the `allow-metric-labels` flag is correctly formatted. 
You might need to check if the metrics you're trying to filter are supported by this flag. 
> it might be due to incorrect formatting or unsupported values in the `allow-metric-labels` flag.

you can specify the allow-list for metric labels using the `--allow-metric-labels` flag. The key format should be `<MetricName>,<LabelName>`, and the value format should be `<allowed_value>,<allowed_value>... (e.g., metric1,label1='v1,v2,v3')`.
you can refer to these pages 
- https://kubernetes.io/docs/reference/scheduling/config/https://kubernetes.io/docs/reference/scheduling/config/
- https://kubernetes.io/docs/concepts/cluster-administration/system-metrics/
Ensure that the allow-metric-labels flag is correctly formatted in your configuration. Here’s an example of the proper format:
```--allow-metric-labels="kubernetes_feature_enabled,stage='BETA'","kubernetes_feature_enabled,name='test'" ```
Additionally, check if the metrics you are trying to filter are supported by this flag.

i think this could also be assigned to sig instrumentation.
/sig instrumentation
How to check,wheather supported

you can check the official Kubernetes documentation for the metrics exposed by the kube-scheduler. 
check this page - https://kubernetes.io/docs/concepts/cluster-administration/system-metrics/
/assign @richabanker 
/triage accepted
the args about `allow-metric-labels` shoue be:
```
--allow-metric-labels=kubernetes_feature_enabled,stage='BETA'
--allow-metric-labels=kubernetes_feature_enabled,name='test'
```
Thanks for catching that!

> the args about `allow-metric-labels` shoue be:
> 
> ```
> --allow-metric-labels=kubernetes_feature_enabled,stage='BETA'
> --allow-metric-labels=kubernetes_feature_enabled,name='test'
> ```


/kind support
/triage accepted
Given https://github.com/kubernetes/kubernetes/issues/126526#issuecomment-2277446781, I think we can close.

/close
@kannon92: Closing this issue.

<details>

In response to [this](https://github.com/kubernetes/kubernetes/issues/126526#issuecomment-2290479694):

>Given https://github.com/kubernetes/kubernetes/issues/126526#issuecomment-2277446781, I think we can close.
>
>/close


Instructions for interacting with me using PR comments are available [here](https://git.k8s.io/community/contributors/guide/pull-requests.md).  If you have questions or suggestions related to my behavior, please file an issue against the [kubernetes-sigs/prow](https://github.com/kubernetes-sigs/prow/issues/new?title=Prow%20issue:) repository.
</details>
/reopen

I saw it was assigned so I'll leave it open.
@kannon92: Reopened this issue.

<details>

In response to [this](https://github.com/kubernetes/kubernetes/issues/126526#issuecomment-2290480074):

>/reopen
>
>I saw it was assigned so I'll leave it open.


Instructions for interacting with me using PR comments are available [here](https://git.k8s.io/community/contributors/guide/pull-requests.md).  If you have questions or suggestions related to my behavior, please file an issue against the [kubernetes-sigs/prow](https://github.com/kubernetes-sigs/prow/issues/new?title=Prow%20issue:) repository.
</details>
I think this is a legit issue, I tried it on my end and was able to confirm that the flag `--allow-metric-labels` is not being respected for both [scheduler metrics](https://github.com/kubernetes/kubernetes/blob/release-1.31/pkg/scheduler/metrics/metrics.go) & [feature enablement metrics](https://github.com/kubernetes/kubernetes/blob/release-1.31/staging/src/k8s.io/component-base/metrics/prometheus/feature/metrics.go). I also suspect this issue is affecting all component metrics currently.

I also wonder why this wasn't caught so far. Maybe we should add an e2e test for this metric.

Tried some fixes for kube-scheduler's metrics & feature enablement metrics in [PR1](https://github.com/kubernetes/kubernetes/pull/126762) & [PR2](https://github.com/kubernetes/kubernetes/pull/126763), maybe there's a better way to apply the flag correctly to metrics here.

cc @logicalhan for visibility
/reopen

https://github.com/kubernetes/kubernetes/pull/126762 only fixes the issue for scheduler metrics. We need a similar fix for all the remaining components, so reopening this issue.
@richabanker: Reopened this issue.

<details>

In response to [this](https://github.com/kubernetes/kubernetes/issues/126526#issuecomment-2350516080):

>/reopen
>
>https://github.com/kubernetes/kubernetes/pull/126762 only fixes the issue for scheduler metrics. We need a similar fix for all the remaining components, so reopening this issue.


Instructions for interacting with me using PR comments are available [here](https://git.k8s.io/community/contributors/guide/pull-requests.md).  If you have questions or suggestions related to my behavior, please file an issue against the [kubernetes-sigs/prow](https://github.com/kubernetes-sigs/prow/issues/new?title=Prow%20issue:) repository.
</details>
/remove-sig node

this is not sig node-specific and applies to all metrics in all components

Fix patch:
diff --git a/staging/src/k8s.io/component-base/metrics/counter.go b/staging/src/k8s.io/component-base/metrics/counter.go
index 5664a68a90039..875b0312a0449 100644
--- a/staging/src/k8s.io/component-base/metrics/counter.go
+++ b/staging/src/k8s.io/component-base/metrics/counter.go
@@ -119,11 +119,6 @@ func NewCounterVec(opts *CounterOpts, labels []string) *CounterVec {
 	opts.StabilityLevel.setDefaults()
 
 	fqName := BuildFQName(opts.Namespace, opts.Subsystem, opts.Name)
-	allowListLock.RLock()
-	if allowList, ok := labelValueAllowLists[fqName]; ok {
-		opts.LabelValueAllowLists = allowList
-	}
-	allowListLock.RUnlock()
 
 	cv := &CounterVec{
 		CounterVec:     noopCounterVec,
@@ -176,7 +171,17 @@ func (v *CounterVec) WithLabelValues(lvs ...string) CounterMetric {
 	}
 	if v.LabelValueAllowLists != nil {
 		v.LabelValueAllowLists.ConstrainToAllowedList(v.originalLabels, lvs)
+	} else {
+		v.initializeLabelAllowListsOnce.Do(func() {
+			allowListLock.RLock()
+			if allowList, ok := labelValueAllowLists[v.FQName()]; ok {
+				v.LabelValueAllowLists = allowList
+				allowList.ConstrainToAllowedList(v.originalLabels, lvs)
+			}
+			allowListLock.RUnlock()
+		})
 	}
+
 	return v.CounterVec.WithLabelValues(lvs...)
 }
 
@@ -190,6 +195,15 @@ func (v *CounterVec) With(labels map[string]string) CounterMetric {
 	}
 	if v.LabelValueAllowLists != nil {
 		v.LabelValueAllowLists.ConstrainLabelMap(labels)
+	} else {
+		v.initializeLabelAllowListsOnce.Do(func() {
+			allowListLock.RLock()
+			if allowList, ok := labelValueAllowLists[v.FQName()]; ok {
+				v.LabelValueAllowLists = allowList
+				allowList.ConstrainLabelMap(labels)
+			}
+			allowListLock.RUnlock()
+		})
 	}
 	return v.CounterVec.With(labels)
 }
diff --git a/staging/src/k8s.io/component-base/metrics/gauge.go b/staging/src/k8s.io/component-base/metrics/gauge.go
index 89631115acbd4..ac33897f70842 100644
--- a/staging/src/k8s.io/component-base/metrics/gauge.go
+++ b/staging/src/k8s.io/component-base/metrics/gauge.go
@@ -105,11 +105,6 @@ func NewGaugeVec(opts *GaugeOpts, labels []string) *GaugeVec {
 	opts.StabilityLevel.setDefaults()
 
 	fqName := BuildFQName(opts.Namespace, opts.Subsystem, opts.Name)
-	allowListLock.RLock()
-	if allowList, ok := labelValueAllowLists[fqName]; ok {
-		opts.LabelValueAllowLists = allowList
-	}
-	allowListLock.RUnlock()
 
 	cv := &GaugeVec{
 		GaugeVec:       noopGaugeVec,
@@ -149,6 +144,15 @@ func (v *GaugeVec) WithLabelValuesChecked(lvs ...string) (GaugeMetric, error) {
 	}
 	if v.LabelValueAllowLists != nil {
 		v.LabelValueAllowLists.ConstrainToAllowedList(v.originalLabels, lvs)
+	} else {
+		v.initializeLabelAllowListsOnce.Do(func() {
+			allowListLock.RLock()
+			if allowList, ok := labelValueAllowLists[v.FQName()]; ok {
+				v.LabelValueAllowLists = allowList
+				allowList.ConstrainToAllowedList(v.originalLabels, lvs)
+			}
+			allowListLock.RUnlock()
+		})
 	}
 	elt, err := v.GaugeVec.GetMetricWithLabelValues(lvs...)
 	return elt, err
@@ -186,6 +190,15 @@ func (v *GaugeVec) WithChecked(labels map[string]string) (GaugeMetric, error) {
 	}
 	if v.LabelValueAllowLists != nil {
 		v.LabelValueAllowLists.ConstrainLabelMap(labels)
+	} else {
+		v.initializeLabelAllowListsOnce.Do(func() {
+			allowListLock.RLock()
+			if allowList, ok := labelValueAllowLists[v.FQName()]; ok {
+				v.LabelValueAllowLists = allowList
+				allowList.ConstrainLabelMap(labels)
+			}
+			allowListLock.RUnlock()
+		})
 	}
 	elt, err := v.GaugeVec.GetMetricWith(labels)
 	return elt, err
diff --git a/staging/src/k8s.io/component-base/metrics/histogram.go b/staging/src/k8s.io/component-base/metrics/histogram.go
index e6884f35c6f8b..2ffbe3192bc5b 100644
--- a/staging/src/k8s.io/component-base/metrics/histogram.go
+++ b/staging/src/k8s.io/component-base/metrics/histogram.go
@@ -96,11 +96,6 @@ func NewHistogramVec(opts *HistogramOpts, labels []string) *HistogramVec {
 	opts.StabilityLevel.setDefaults()
 
 	fqName := BuildFQName(opts.Namespace, opts.Subsystem, opts.Name)
-	allowListLock.RLock()
-	if allowList, ok := labelValueAllowLists[fqName]; ok {
-		opts.LabelValueAllowLists = allowList
-	}
-	allowListLock.RUnlock()
 
 	v := &HistogramVec{
 		HistogramVec:   noopHistogramVec,
@@ -148,6 +143,15 @@ func (v *HistogramVec) WithLabelValues(lvs ...string) ObserverMetric {
 	}
 	if v.LabelValueAllowLists != nil {
 		v.LabelValueAllowLists.ConstrainToAllowedList(v.originalLabels, lvs)
+	} else {
+		v.initializeLabelAllowListsOnce.Do(func() {
+			allowListLock.RLock()
+			if allowList, ok := labelValueAllowLists[v.FQName()]; ok {
+				v.LabelValueAllowLists = allowList
+				allowList.ConstrainToAllowedList(v.originalLabels, lvs)
+			}
+			allowListLock.RUnlock()
+		})
 	}
 	return v.HistogramVec.WithLabelValues(lvs...)
 }
@@ -162,6 +166,15 @@ func (v *HistogramVec) With(labels map[string]string) ObserverMetric {
 	}
 	if v.LabelValueAllowLists != nil {
 		v.LabelValueAllowLists.ConstrainLabelMap(labels)
+	} else {
+		v.initializeLabelAllowListsOnce.Do(func() {
+			allowListLock.RLock()
+			if allowList, ok := labelValueAllowLists[v.FQName()]; ok {
+				v.LabelValueAllowLists = allowList
+				allowList.ConstrainLabelMap(labels)
+			}
+			allowListLock.RUnlock()
+		})
 	}
 	return v.HistogramVec.With(labels)
 }
diff --git a/staging/src/k8s.io/component-base/metrics/opts.go b/staging/src/k8s.io/component-base/metrics/opts.go
index 59994e3db9d7b..596851cfd9e1c 100644
--- a/staging/src/k8s.io/component-base/metrics/opts.go
+++ b/staging/src/k8s.io/component-base/metrics/opts.go
@@ -44,16 +44,17 @@ var (
 // Name must be set to a non-empty string. DeprecatedVersion is defined only
 // if the metric for which this options applies is, in fact, deprecated.
 type KubeOpts struct {
-	Namespace            string
-	Subsystem            string
-	Name                 string
-	Help                 string
-	ConstLabels          map[string]string
-	DeprecatedVersion    string
-	deprecateOnce        sync.Once
-	annotateOnce         sync.Once
-	StabilityLevel       StabilityLevel
-	LabelValueAllowLists *MetricLabelAllowList
+	Namespace                     string
+	Subsystem                     string
+	Name                          string
+	Help                          string
+	ConstLabels                   map[string]string
+	DeprecatedVersion             string
+	deprecateOnce                 sync.Once
+	annotateOnce                  sync.Once
+	StabilityLevel                StabilityLevel
+	initializeLabelAllowListsOnce sync.Once
+	LabelValueAllowLists          *MetricLabelAllowList
 }
 
 // BuildFQName joins the given three name components by "_". Empty name
@@ -160,17 +161,18 @@ func (o *GaugeOpts) toPromGaugeOpts() prometheus.GaugeOpts {
 // and can safely be left at their zero value, although it is strongly
 // encouraged to set a Help string.
 type HistogramOpts struct {
-	Namespace            string
-	Subsystem            string
-	Name                 string
-	Help                 string
-	ConstLabels          map[string]string
-	Buckets              []float64
-	DeprecatedVersion    string
-	deprecateOnce        sync.Once
-	annotateOnce         sync.Once
-	StabilityLevel       StabilityLevel
-	LabelValueAllowLists *MetricLabelAllowList
+	Namespace                     string
+	Subsystem                     string
+	Name                          string
+	Help                          string
+	ConstLabels                   map[string]string
+	Buckets                       []float64
+	DeprecatedVersion             string
+	deprecateOnce                 sync.Once
+	annotateOnce                  sync.Once
+	StabilityLevel                StabilityLevel
+	initializeLabelAllowListsOnce sync.Once
+	LabelValueAllowLists          *MetricLabelAllowList
 }
 
 // Modify help description on the metric description.
@@ -206,18 +208,19 @@ func (o *HistogramOpts) toPromHistogramOpts() prometheus.HistogramOpts {
 // and can safely be left at their zero value, although it is strongly
 // encouraged to set a Help string.
 type TimingHistogramOpts struct {
-	Namespace            string
-	Subsystem            string
-	Name                 string
-	Help                 string
-	ConstLabels          map[string]string
-	Buckets              []float64
-	InitialValue         float64
-	DeprecatedVersion    string
-	deprecateOnce        sync.Once
-	annotateOnce         sync.Once
-	StabilityLevel       StabilityLevel
-	LabelValueAllowLists *MetricLabelAllowList
+	Namespace                     string
+	Subsystem                     string
+	Name                          string
+	Help                          string
+	ConstLabels                   map[string]string
+	Buckets                       []float64
+	InitialValue                  float64
+	DeprecatedVersion             string
+	deprecateOnce                 sync.Once
+	annotateOnce                  sync.Once
+	StabilityLevel                StabilityLevel
+	initializeLabelAllowListsOnce sync.Once
+	LabelValueAllowLists          *MetricLabelAllowList
 }
 
 // Modify help description on the metric description.
@@ -255,20 +258,21 @@ func (o *TimingHistogramOpts) toPromHistogramOpts() promext.TimingHistogramOpts
 // a help string and to explicitly set the Objectives field to the desired value
 // as the default value will change in the upcoming v0.10 of the library.
 type SummaryOpts struct {
-	Namespace            string
-	Subsystem            string
-	Name                 string
-	Help                 string
-	ConstLabels          map[string]string
-	Objectives           map[float64]float64
-	MaxAge               time.Duration
-	AgeBuckets           uint32
-	BufCap               uint32
-	DeprecatedVersion    string
-	deprecateOnce        sync.Once
-	annotateOnce         sync.Once
-	StabilityLevel       StabilityLevel
-	LabelValueAllowLists *MetricLabelAllowList
+	Namespace                     string
+	Subsystem                     string
+	Name                          string
+	Help                          string
+	ConstLabels                   map[string]string
+	Objectives                    map[float64]float64
+	MaxAge                        time.Duration
+	AgeBuckets                    uint32
+	BufCap                        uint32
+	DeprecatedVersion             string
+	deprecateOnce                 sync.Once
+	annotateOnce                  sync.Once
+	StabilityLevel                StabilityLevel
+	initializeLabelAllowListsOnce sync.Once
+	LabelValueAllowLists          *MetricLabelAllowList
 }
 
 // Modify help description on the metric description.
diff --git a/staging/src/k8s.io/component-base/metrics/summary.go b/staging/src/k8s.io/component-base/metrics/summary.go
index d40421645af97..57b60338372a8 100644
--- a/staging/src/k8s.io/component-base/metrics/summary.go
+++ b/staging/src/k8s.io/component-base/metrics/summary.go
@@ -109,11 +109,6 @@ func NewSummaryVec(opts *SummaryOpts, labels []string) *SummaryVec {
 	opts.StabilityLevel.setDefaults()
 
 	fqName := BuildFQName(opts.Namespace, opts.Subsystem, opts.Name)
-	allowListLock.RLock()
-	if allowList, ok := labelValueAllowLists[fqName]; ok {
-		opts.LabelValueAllowLists = allowList
-	}
-	allowListLock.RUnlock()
 
 	v := &SummaryVec{
 		SummaryOpts:    opts,
@@ -160,6 +155,15 @@ func (v *SummaryVec) WithLabelValues(lvs ...string) ObserverMetric {
 	}
 	if v.LabelValueAllowLists != nil {
 		v.LabelValueAllowLists.ConstrainToAllowedList(v.originalLabels, lvs)
+	} else {
+		v.initializeLabelAllowListsOnce.Do(func() {
+			allowListLock.RLock()
+			if allowList, ok := labelValueAllowLists[v.FQName()]; ok {
+				v.LabelValueAllowLists = allowList
+				allowList.ConstrainToAllowedList(v.originalLabels, lvs)
+			}
+			allowListLock.RUnlock()
+		})
 	}
 	return v.SummaryVec.WithLabelValues(lvs...)
 }
@@ -174,6 +178,15 @@ func (v *SummaryVec) With(labels map[string]string) ObserverMetric {
 	}
 	if v.LabelValueAllowLists != nil {
 		v.LabelValueAllowLists.ConstrainLabelMap(labels)
+	} else {
+		v.initializeLabelAllowListsOnce.Do(func() {
+			allowListLock.RLock()
+			if allowList, ok := labelValueAllowLists[v.FQName()]; ok {
+				v.LabelValueAllowLists = allowList
+				allowList.ConstrainLabelMap(labels)
+			}
+			allowListLock.RUnlock()
+		})
 	}
 	return v.SummaryVec.With(labels)
 }
diff --git a/staging/src/k8s.io/component-base/metrics/timing_histogram.go b/staging/src/k8s.io/component-base/metrics/timing_histogram.go
index a0f0b253c7d83..08751ecbb2bd2 100644
--- a/staging/src/k8s.io/component-base/metrics/timing_histogram.go
+++ b/staging/src/k8s.io/component-base/metrics/timing_histogram.go
@@ -125,11 +125,6 @@ func NewTestableTimingHistogramVec(nowFunc func() time.Time, opts *TimingHistogr
 	opts.StabilityLevel.setDefaults()
 
 	fqName := BuildFQName(opts.Namespace, opts.Subsystem, opts.Name)
-	allowListLock.RLock()
-	if allowList, ok := labelValueAllowLists[fqName]; ok {
-		opts.LabelValueAllowLists = allowList
-	}
-	allowListLock.RUnlock()
 
 	v := &TimingHistogramVec{
 		TimingHistogramVec:  noopTimingHistogramVec,
@@ -175,6 +170,15 @@ func (v *TimingHistogramVec) WithLabelValuesChecked(lvs ...string) (GaugeMetric,
 	}
 	if v.LabelValueAllowLists != nil {
 		v.LabelValueAllowLists.ConstrainToAllowedList(v.originalLabels, lvs)
+	} else {
+		v.initializeLabelAllowListsOnce.Do(func() {
+			allowListLock.RLock()
+			if allowList, ok := labelValueAllowLists[v.FQName()]; ok {
+				v.LabelValueAllowLists = allowList
+				allowList.ConstrainToAllowedList(v.originalLabels, lvs)
+			}
+			allowListLock.RUnlock()
+		})
 	}
 	ops, err := v.TimingHistogramVec.GetMetricWithLabelValues(lvs...)
 	if err != nil {
@@ -214,6 +218,15 @@ func (v *TimingHistogramVec) WithChecked(labels map[string]string) (GaugeMetric,
 	}
 	if v.LabelValueAllowLists != nil {
 		v.LabelValueAllowLists.ConstrainLabelMap(labels)
+	} else {
+		v.initializeLabelAllowListsOnce.Do(func() {
+			allowListLock.RLock()
+			if allowList, ok := labelValueAllowLists[v.FQName()]; ok {
+				v.LabelValueAllowLists = allowList
+				allowList.ConstrainLabelMap(labels)
+			}
+			allowListLock.RUnlock()
+		})
 	}
 	ops, err := v.TimingHistogramVec.GetMetricWith(labels)
 	return ops.(GaugeMetric), err


Test patch:
diff --git a/staging/src/k8s.io/component-base/metrics/counter_test.go b/staging/src/k8s.io/component-base/metrics/counter_test.go
index d7196f48c9f82..5e89e45d00dd7 100644
--- a/staging/src/k8s.io/component-base/metrics/counter_test.go
+++ b/staging/src/k8s.io/component-base/metrics/counter_test.go
@@ -245,7 +245,8 @@ func TestCounterWithLabelValueAllowList(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.desc, func(t *testing.T) {
-			SetLabelAllowListFromCLI(labelAllowValues)
+			labelValueAllowLists = map[string]*MetricLabelAllowList{}
+
 			registry := newKubeRegistry(apimachineryversion.Info{
 				Major:      "1",
 				Minor:      "15",
@@ -253,7 +254,7 @@ func TestCounterWithLabelValueAllowList(t *testing.T) {
 			})
 			c := NewCounterVec(opts, labels)
 			registry.MustRegister(c)
-
+			SetLabelAllowListFromCLI(labelAllowValues)
 			for _, lv := range test.labelValues {
 				c.WithLabelValues(lv...).Inc()
 			}
diff --git a/staging/src/k8s.io/component-base/metrics/gauge_test.go b/staging/src/k8s.io/component-base/metrics/gauge_test.go
index c6c09dfed1aae..d41baebb8f1dc 100644
--- a/staging/src/k8s.io/component-base/metrics/gauge_test.go
+++ b/staging/src/k8s.io/component-base/metrics/gauge_test.go
@@ -305,7 +305,8 @@ func TestGaugeWithLabelValueAllowList(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.desc, func(t *testing.T) {
-			SetLabelAllowListFromCLI(labelAllowValues)
+			labelValueAllowLists = map[string]*MetricLabelAllowList{}
+
 			registry := newKubeRegistry(apimachineryversion.Info{
 				Major:      "1",
 				Minor:      "15",
@@ -313,7 +314,7 @@ func TestGaugeWithLabelValueAllowList(t *testing.T) {
 			})
 			g := NewGaugeVec(opts, labels)
 			registry.MustRegister(g)
-
+			SetLabelAllowListFromCLI(labelAllowValues)
 			for _, lv := range test.labelValues {
 				g.WithLabelValues(lv...).Set(100.0)
 			}
diff --git a/staging/src/k8s.io/component-base/metrics/histogram_test.go b/staging/src/k8s.io/component-base/metrics/histogram_test.go
index 51c5601f31280..d5fc24f7c7cbe 100644
--- a/staging/src/k8s.io/component-base/metrics/histogram_test.go
+++ b/staging/src/k8s.io/component-base/metrics/histogram_test.go
@@ -271,7 +271,7 @@ func TestHistogramWithLabelValueAllowList(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.desc, func(t *testing.T) {
-			SetLabelAllowListFromCLI(labelAllowValues)
+			labelValueAllowLists = map[string]*MetricLabelAllowList{}
 			registry := newKubeRegistry(apimachineryversion.Info{
 				Major:      "1",
 				Minor:      "15",
@@ -279,6 +279,7 @@ func TestHistogramWithLabelValueAllowList(t *testing.T) {
 			})
 			c := NewHistogramVec(opts, labels)
 			registry.MustRegister(c)
+			SetLabelAllowListFromCLI(labelAllowValues)
 
 			for _, lv := range test.labelValues {
 				c.WithLabelValues(lv...).Observe(1.0)
diff --git a/staging/src/k8s.io/component-base/metrics/summary_test.go b/staging/src/k8s.io/component-base/metrics/summary_test.go
index ca9b50b7981ad..02fa895e7a204 100644
--- a/staging/src/k8s.io/component-base/metrics/summary_test.go
+++ b/staging/src/k8s.io/component-base/metrics/summary_test.go
@@ -235,7 +235,7 @@ func TestSummaryWithLabelValueAllowList(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.desc, func(t *testing.T) {
-			SetLabelAllowListFromCLI(labelAllowValues)
+			labelValueAllowLists = map[string]*MetricLabelAllowList{}
 			registry := newKubeRegistry(apimachineryversion.Info{
 				Major:      "1",
 				Minor:      "15",
@@ -243,6 +243,7 @@ func TestSummaryWithLabelValueAllowList(t *testing.T) {
 			})
 			c := NewSummaryVec(opts, labels)
 			registry.MustRegister(c)
+			SetLabelAllowListFromCLI(labelAllowValues)
 
 			for _, lv := range test.labelValues {
 				c.WithLabelValues(lv...).Observe(1.0)
diff --git a/staging/src/k8s.io/component-base/metrics/timing_histogram_test.go b/staging/src/k8s.io/component-base/metrics/timing_histogram_test.go
index de10a87bd79cd..80fcb6340c040 100644
--- a/staging/src/k8s.io/component-base/metrics/timing_histogram_test.go
+++ b/staging/src/k8s.io/component-base/metrics/timing_histogram_test.go
@@ -313,7 +313,8 @@ func TestTimingHistogramWithLabelValueAllowList(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.desc, func(t *testing.T) {
-			SetLabelAllowListFromCLI(labelAllowValues)
+			labelValueAllowLists = map[string]*MetricLabelAllowList{}
+
 			registry := newKubeRegistry(apimachineryversion.Info{
 				Major:      "1",
 				Minor:      "15",
@@ -323,6 +324,7 @@ func TestTimingHistogramWithLabelValueAllowList(t *testing.T) {
 			clk := testclock.NewFakePassiveClock(t0)
 			c := NewTestableTimingHistogramVec(clk.Now, opts, labels)
 			registry.MustRegister(c)
+			SetLabelAllowListFromCLI(labelAllowValues)
 			var v0 float64 = 13
 			for _, lv := range test.labelValues {
 				c.WithLabelValues(lv...).Set(v0)