Repository: kubernetes/kubernetes (language: go)

Issue:
allow-metric-labels can not filter metrics
### What happened?

```
apiVersion: v1
kind: Pod
metadata:
  creationTimestamp: null
  labels:
    component: kube-scheduler
    tier: control-plane
  name: kube-scheduler
  namespace: kube-system
spec:
  containers:
  - command:
    - kube-scheduler
    - --bind-address=0.0.0.0
    - --config=/etc/kubernetes/scheduler-config.yaml
    - --v=2
    - --authentication-kubeconfig=/etc/kubernetes/scheduler.conf
    - --authorization-always-allow-paths=/healthz,/readyz,/livez,/configz,/metrics
    - --authorization-kubeconfig=/etc/kubernetes/scheduler.conf
    - --allow-metric-labels="kubernetes_feature_enabled,stage='BETA'","kubernetes_feature_enabled,name='test'"
    image: registry.aliyuncs.com/google_containers/kube-scheduler:v1.30.0
    imagePullPolicy: IfNotPresent
    livenessProbe:
      failureThreshold: 8
      httpGet:
        host: 127.0.0.1
        path: /healthz
        port: 10259
        scheme: HTTPS
      initialDelaySeconds: 10
      periodSeconds: 10
      timeoutSeconds: 15
    name: kube-scheduler
    resources:
      requests:
        cpu: 100m
    startupProbe:
      failureThreshold: 24
      httpGet:
        host: 127.0.0.1
        path: /healthz
        port: 10259
        scheme: HTTPS
      initialDelaySeconds: 10
      periodSeconds: 10
      timeoutSeconds: 15
    volumeMounts:
    - mountPath: /etc/kubernetes/scheduler.conf
      name: kubeconfig
      readOnly: true
    - mountPath: /etc/kubernetes/scheduler-config.yaml
      name: scheduler-config
      readOnly: true
    - mountPath: /etc/kubernetes/extender/
      name: cert
      readOnly: true
    - mountPath: /etc/kubernetes/certs/
      name: scheduler-cert
      readOnly: true     
  hostNetwork: true
  priority: 2000001000
  priorityClassName: system-node-critical
  securityContext:
    seccompProfile:
      type: RuntimeDefault
  volumes:
  - hostPath:
      path: /etc/kubernetes/scheduler.conf
      type: FileOrCreate
    name: kubeconfig
  - hostPath:
      path: /etc/kubernetes/scheduler-config.yaml
      type: FileOrCreate
    name: scheduler-config
  - hostPath:
      path: /etc/kubernetes/extender
      type: DirectoryOrCreate
    name: cert
  - hostPath:
      path: /etc/kubernetes/certs
      type: DirectoryOrCreate
    name: scheduler-cert
---


bernetes_feature_enabled{name="LogarithmicScaleDown",stage="BETA"} 1
kubernetes_feature_enabled{name="LoggingAlphaOptions",stage="ALPHA"} 0
kubernetes_feature_enabled{name="LoggingBetaOptions",stage="BETA"} 1
kubernetes_feature_enabled{name="MatchLabelKeysInPodAffinity",stage="ALPHA"} 0
kubernetes_feature_enabled{name="MatchLabelKeysInPodTopologySpread",stage="BETA"} 1
kubernetes_feature_enabled{name="MaxUnavailableStatefulSet",stage="ALPHA"} 0
kubernetes_feature_enabled{name="MemoryManager",stage="BETA"} 1
kubernetes_feature_enabled{name="MemoryQoS",stage="ALPHA"} 0
kubernetes_feature_enabled{name="MinDomainsInPodTopologySpread",stage=""} 1
kubernetes_feature_enabled{name="MultiCIDRServiceAllocator",stage="ALPHA"} 0
kubernetes_feature_enabled{name="MutatingAdmissionPolicy",stage="ALPHA"} 0
kubernetes_feature_enabled{name="NFTablesProxyMode",stage="ALPHA"} 0
kubernetes_feature_enabled{name="NewVolumeManagerReconstruction",stage=""} 1
kubernetes_feature_enabled{name="NodeInclusionPolicyInPodTopologySpread",stage="BETA"} 1
kubernetes_feature_enabled{name="NodeLogQuery",stage="BETA"} 0
kubernetes_feature_enabled{name="NodeOutOfServiceVolumeDetach",stage=""} 1
kubernetes_feature_enabled{name="NodeSwap",stage="BETA"} 1
kubernetes_feature_enabled{name="OpenAPIEnums",stage="BETA"} 1
kubernetes_feature_enabled{name="PDBUnhealthyPodEvictionPolicy",stage="BETA"} 1
kubernetes_feature_enabled{name="PersistentVolumeLastPhaseTransitionTime",stage="BETA"} 1
kubernetes_feature_enabled{name="PodAndContainerStatsFromCRI",stage="ALPHA"} 0
kubernetes_feature_enabled{name="PodDeletionCost",stage="BETA"} 1
kubernetes_feature_enabled{name="PodDisruptionConditions",stage="
```

### What did you expect to happen?

filter metrics

### How can we reproduce it (as minimally and precisely as possible)?

change kube-scheduler cmd line

### Anything else we need to know?

_No response_

### Kubernetes version

<details>

1.30.0

</details>


### Cloud provider

<details>
wmware workstation
</details>


### OS version

<details>

```console
# On Linux:
$ cat /etc/os-release
# paste output here
$ uname -a
# paste output here

# On Windows:
C:\> wmic os get Caption, Version, BuildNumber, OSArchitecture
# paste output here
```

</details>


### Install tools

<details>

</details>


### Container runtime (CRI) and version (if applicable)

<details>

</details>


### Related plugins (CNI, CSI, ...) and versions (if applicable)

<details>

</details>



Discussion / hints:
/sig node

hi @13567436138 
it might be due to incorrect formatting or unsupported values in the `allow-metric-labels` flag.
To troubleshoot, ensure that the `allow-metric-labels` flag is correctly formatted. 
You might need to check if the metrics you're trying to filter are supported by this flag. 
> it might be due to incorrect formatting or unsupported values in the `allow-metric-labels` flag.

you can specify the allow-list for metric labels using the `--allow-metric-labels` flag. The key format should be `<MetricName>,<LabelName>`, and the value format should be `<allowed_value>,<allowed_value>... (e.g., metric1,label1='v1,v2,v3')`.
you can refer to these pages 
- https://kubernetes.io/docs/reference/scheduling/config/https://kubernetes.io/docs/reference/scheduling/config/
- https://kubernetes.io/docs/concepts/cluster-administration/system-metrics/
Ensure that the allow-metric-labels flag is correctly formatted in your configuration. Here’s an example of the proper format:
```--allow-metric-labels="kubernetes_feature_enabled,stage='BETA'","kubernetes_feature_enabled,name='test'" ```
Additionally, check if the metrics you are trying to filter are supported by this flag.

i think this could also be assigned to sig instrumentation.
/sig instrumentation
How to check,wheather supported

you can check the official Kubernetes documentation for the metrics exposed by the kube-scheduler. 
check this page - https://kubernetes.io/docs/concepts/cluster-administration/system-metrics/
/assign @richabanker 
/triage accepted
the args about `allow-metric-labels` shoue be:
```
--allow-metric-labels=kubernetes_feature_enabled,stage='BETA'
--allow-metric-labels=kubernetes_feature_enabled,name='test'
```
Thanks for catching that!

> the args about `allow-metric-labels` shoue be:
> 
> ```
> --allow-metric-labels=kubernetes_feature_enabled,stage='BETA'
> --allow-metric-labels=kubernetes_feature_enabled,name='test'
> ```


/kind support
/triage accepted
Given https://github.com/kubernetes/kubernetes/issues/126526#issuecomment-2277446781, I think we can close.

/close
@kannon92: Closing this issue.

<details>

In response to [this](https://github.com/kubernetes/kubernetes/issues/126526#issuecomment-2290479694):

>Given https://github.com/kubernetes/kubernetes/issues/126526#issuecomment-2277446781, I think we can close.
>
>/close


Instructions for interacting with me using PR comments are available [here](https://git.k8s.io/community/contributors/guide/pull-requests.md).  If you have questions or suggestions related to my behavior, please file an issue against the [kubernetes-sigs/prow](https://github.com/kubernetes-sigs/prow/issues/new?title=Prow%20issue:) repository.
</details>
/reopen

I saw it was assigned so I'll leave it open.
@kannon92: Reopened this issue.

<details>

In response to [this](https://github.com/kubernetes/kubernetes/issues/126526#issuecomment-2290480074):

>/reopen
>
>I saw it was assigned so I'll leave it open.


Instructions for interacting with me using PR comments are available [here](https://git.k8s.io/community/contributors/guide/pull-requests.md).  If you have questions or suggestions related to my behavior, please file an issue against the [kubernetes-sigs/prow](https://github.com/kubernetes-sigs/prow/issues/new?title=Prow%20issue:) repository.
</details>
I think this is a legit issue, I tried it on my end and was able to confirm that the flag `--allow-metric-labels` is not being respected for both [scheduler metrics](https://github.com/kubernetes/kubernetes/blob/release-1.31/pkg/scheduler/metrics/metrics.go) & [feature enablement metrics](https://github.com/kubernetes/kubernetes/blob/release-1.31/staging/src/k8s.io/component-base/metrics/prometheus/feature/metrics.go). I also suspect this issue is affecting all component metrics currently.

I also wonder why this wasn't caught so far. Maybe we should add an e2e test for this metric.

Tried some fixes for kube-scheduler's metrics & feature enablement metrics in [PR1](https://github.com/kubernetes/kubernetes/pull/126762) & [PR2](https://github.com/kubernetes/kubernetes/pull/126763), maybe there's a better way to apply the flag correctly to metrics here.

cc @logicalhan for visibility
/reopen

https://github.com/kubernetes/kubernetes/pull/126762 only fixes the issue for scheduler metrics. We need a similar fix for all the remaining components, so reopening this issue.
@richabanker: Reopened this issue.

<details>

In response to [this](https://github.com/kubernetes/kubernetes/issues/126526#issuecomment-2350516080):

>/reopen
>
>https://github.com/kubernetes/kubernetes/pull/126762 only fixes the issue for scheduler metrics. We need a similar fix for all the remaining components, so reopening this issue.


Instructions for interacting with me using PR comments are available [here](https://git.k8s.io/community/contributors/guide/pull-requests.md).  If you have questions or suggestions related to my behavior, please file an issue against the [kubernetes-sigs/prow](https://github.com/kubernetes-sigs/prow/issues/new?title=Prow%20issue:) repository.
</details>
/remove-sig node

this is not sig node-specific and applies to all metrics in all components

Fix patch:
diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/metrics/metrics.go b/staging/src/k8s.io/apiserver/pkg/endpoints/metrics/metrics.go
index 48fc951adeedf..0c7a3f6b92c37 100644
--- a/staging/src/k8s.io/apiserver/pkg/endpoints/metrics/metrics.go
+++ b/staging/src/k8s.io/apiserver/pkg/endpoints/metrics/metrics.go
@@ -416,6 +416,33 @@ func Reset() {
 	}
 }
 
+// ResetLabelAllowLists resets the label allow lists for all metrics.
+// NOTE: This is only used for testing.
+func ResetLabelAllowLists() {
+	for _, metric := range metrics {
+		if counterVec, ok := metric.(*compbasemetrics.CounterVec); ok {
+			counterVec.ResetLabelAllowLists()
+			continue
+		}
+		if gaugeVec, ok := metric.(*compbasemetrics.GaugeVec); ok {
+			gaugeVec.ResetLabelAllowLists()
+			continue
+		}
+		if histogramVec, ok := metric.(*compbasemetrics.HistogramVec); ok {
+			histogramVec.ResetLabelAllowLists()
+			continue
+		}
+		if summaryVec, ok := metric.(*compbasemetrics.SummaryVec); ok {
+			summaryVec.ResetLabelAllowLists()
+			continue
+		}
+		if timingHistogramVec, ok := metric.(*compbasemetrics.TimingHistogramVec); ok {
+			timingHistogramVec.ResetLabelAllowLists()
+			continue
+		}
+	}
+}
+
 // UpdateInflightRequestMetrics reports concurrency metrics classified by
 // mutating vs Readonly.
 func UpdateInflightRequestMetrics(phase string, nonmutating, mutating int) {


Test patch:
diff --git a/test/integration/metrics/metrics_test.go b/test/integration/metrics/metrics_test.go
index c36188944f905..423db1f210da0 100644
--- a/test/integration/metrics/metrics_test.go
+++ b/test/integration/metrics/metrics_test.go
@@ -21,13 +21,17 @@ import (
 	"errors"
 	"fmt"
 	"runtime"
+	"slices"
+	"strings"
 	"testing"
 
 	"github.com/prometheus/common/model"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apiserver/pkg/endpoints/metrics"
 	clientset "k8s.io/client-go/kubernetes"
 	restclient "k8s.io/client-go/rest"
+	compbasemetrics "k8s.io/component-base/metrics"
 	"k8s.io/component-base/metrics/testutil"
 	kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
 	"k8s.io/kubernetes/test/integration/framework"
@@ -261,6 +265,101 @@ func TestAPIServerMetricsLabels(t *testing.T) {
 	}
 }
 
+func TestAPIServerMetricsLabelsWithAllowList(t *testing.T) {
+	testCases := []struct {
+		name        string
+		metricName  string
+		labelName   model.LabelName
+		allowValues model.LabelValues
+		isHistogram bool
+	}{
+		{
+			name:        "check CounterVec metric",
+			metricName:  "apiserver_request_total",
+			labelName:   "code",
+			allowValues: model.LabelValues{"201", "500"},
+		},
+		{
+			name:        "check GaugeVec metric",
+			metricName:  "apiserver_current_inflight_requests",
+			labelName:   "request_kind",
+			allowValues: model.LabelValues{"mutating"},
+		},
+		{
+			name:        "check Histogram metric",
+			metricName:  "apiserver_request_duration_seconds",
+			labelName:   "verb",
+			allowValues: model.LabelValues{"POST", "LIST"},
+			isHistogram: true,
+		},
+	}
+
+	// Assemble the allow-metric-labels flag.
+	var allowMetricLabelFlagStrs []string
+	for _, tc := range testCases {
+		var allowValuesStr []string
+		for _, allowValue := range tc.allowValues {
+			allowValuesStr = append(allowValuesStr, string(allowValue))
+		}
+		allowMetricLabelFlagStrs = append(allowMetricLabelFlagStrs, fmt.Sprintf("\"%s,%s=%s\"", tc.metricName, tc.labelName, strings.Join(allowValuesStr, ",")))
+	}
+	allowMetricLabelsFlag := "--allow-metric-labels=" + strings.Join(allowMetricLabelFlagStrs, ",")
+
+	testServerFlags := framework.DefaultTestServerFlags()
+	testServerFlags = append(testServerFlags, allowMetricLabelsFlag)
+
+	// TODO: have a better way to setup and teardown for all the other tests.
+	metrics.Reset()
+	defer metrics.Reset()
+	metrics.ResetLabelAllowLists()
+	defer metrics.ResetLabelAllowLists()
+	defer compbasemetrics.ResetLabelValueAllowLists()
+
+	// KUBE_APISERVER_SERVE_REMOVED_APIS_FOR_ONE_RELEASE allows for APIs pending removal to not block tests
+	t.Setenv("KUBE_APISERVER_SERVE_REMOVED_APIS_FOR_ONE_RELEASE", "true")
+	s := kubeapiservertesting.StartTestServerOrDie(t, nil, testServerFlags, framework.SharedEtcd())
+	defer s.TearDownFn()
+
+	metrics, err := scrapeMetrics(s)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			metricName := tc.metricName
+			if tc.isHistogram {
+				metricName += "_sum"
+			}
+			samples, found := metrics[metricName]
+			if !found {
+				t.Fatalf("metric %q not found", metricName)
+			}
+			for _, sample := range samples {
+				if value, ok := sample.Metric[tc.labelName]; ok {
+					if !slices.Contains(tc.allowValues, value) && value != "unexpected" {
+						t.Fatalf("value %q is not allowed for label %q", value, tc.labelName)
+					}
+				}
+			}
+		})
+
+	}
+
+	t.Run("check cardinality_enforcement_unexpected_categorizations_total", func(t *testing.T) {
+		samples, found := metrics["cardinality_enforcement_unexpected_categorizations_total"]
+		if !found {
+			t.Fatal("metric cardinality_enforcement_unexpected_categorizations_total not found")
+		}
+		if len(samples) != 1 {
+			t.Fatalf("Unexpected number of samples in cardinality_enforcement_unexpected_categorizations_total")
+		}
+		if samples[0].Value <= 0 {
+			t.Fatalf("Unexpected non-positive cardinality_enforcement_unexpected_categorizations_total, got: %s", samples[0].Value)
+		}
+
+	})
+}
+
 func TestAPIServerMetricsPods(t *testing.T) {
 	callOrDie := func(_ interface{}, err error) {
 		if err != nil {