Repository: kubernetes/kube-state-metrics (2026-04-15T04:17:07Z)

Issue:
v2.17.0 breaks custom_resource_config_file config option
**What happened**:

Updating to kube-state-metrics version 2.17.0 broke the `custom_resource_config_file` configuration option in `config.yaml`

**What you expected to happen**:

```
    custom_resource_config_file: "/etc/kube-state-metrics/custom-resource-config.yaml"
```

to be equivalent to 

```
--custom-resource-state-config-file=/etc/kube-state-metrics/custom-resource-config.yaml
```

like in prior versions.

**How to reproduce it (as minimally and precisely as possible)**:

```bash
minikube start
...

helm install \
  cert-manager oci://quay.io/jetstack/charts/cert-manager \
  --version v1.19.1 \
  --namespace cert-manager \
  --create-namespace \
  --set crds.enabled=true

kubectl apply -f - <<'__END__'
apiVersion: cert-manager.io/v1
kind: CertificateRequest
metadata:
  name: my-ca-cr
spec:
  request: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURSBSRVFVRVNULS0tLS0KTUlJQzNqQ0NBY1lDQVFBd2daZ3hDekFKQmdOVkJBWVRBbHBhTVE4d0RRWURWUVFJREFaQmNHOXNiRzh4RFRBTApCZ05WQkFjTUJFMXZiMjR4RVRBUEJnTlZCQW9NQ0VwbGRITjBZV05yTVJVd0V3WURWUVFMREF4alpYSjBMVzFoCmJtRm5aWEl4RVRBUEJnTlZCQU1NQ0dwdmMyaDJZVzVzTVN3d0tnWUpLb1pJaHZjTkFRa0JGaDFxYjNOb2RXRXUKZG1GdWJHVmxkWGRsYmtCcVpYUnpkR0ZqYXk1cGJ6Q0NBU0l3RFFZSktvWklodmNOQVFFQkJRQURnZ0VQQURDQwpBUW9DZ2dFQkFLd01tTFhuQkNiRStZdTIvMlFtRGsxalRWQ3BvbHU3TlZmQlVFUWl1bDhFMHI2NFBLcDRZQ0c5Cmx2N2kwOHdFMEdJQUgydnJRQmxVd3p6ZW1SUWZ4YmQvYVNybzRHNUFBYTJsY2NMaFpqUlh2NEVMaER0aVg4N3IKaTQ0MWJ2Y01OM0ZPTlRuczJhRkJYcllLWGxpNG4rc0RzTEVuZmpWdXRiV01Zeis3M3ptaGZzclRJUjRzTXo3cQpmSzM2WFM4UkRjNW5oVVcyYU9BZ3lnbFZSOVVXRkxXNjNXYXVhcHg2QUpBR1RoZnJYdVVHZXlZUUVBSENxZmZmCjhyOEt3YTFYK1NwYm9YK1ppSVE0Nk5jQ043OFZnL2dQVHNLZmphZURoNWcyNlk1dEVidHd3MWdRbWlhK0MyRHIKWHpYNU13RzJGNHN0cG5kUnRQckZrU1VnMW1zd0xuc0NBd0VBQWFBQU1BMEdDU3FHU0liM0RRRUJDd1VBQTRJQgpBUUFXR0JuRnhaZ0gzd0N3TG5IQ0xjb0l5RHJrMUVvYkRjN3BJK1VVWEJIS2JBWk9IWEFhaGJ5RFFLL2RuTHN3CjJkZ0J3bmlJR3kxNElwQlNxaDBJUE03eHk5WjI4VW9oR3piN0FVakRJWHlNdmkvYTJyTVhjWjI1d1NVQmxGc28Kd005dE1QU2JwcEVvRERsa3NsOUIwT1BPdkFyQ0NKNnZGaU1UbS9wMUJIUWJSOExNQW53U0lUYVVNSFByRzJVMgpjTjEvRGNMWjZ2enEyeENjYVoxemh2bzBpY1VIUm9UWmV1ZEp6MkxmR0VHM1VOb2ppbXpBNUZHd0RhS3BySWp3ClVkd1JmZWZ1T29MT1dNVnFNbGRBcTlyT24wNHJaT3Jnak1HSE9tTWxleVdPS1AySllhaDNrVDdKU01zTHhYcFYKV0ExQjRsLzFFQkhWeGlKQi9Zby9JQWVsCi0tLS0tRU5EIENFUlRJRklDQVRFIFJFUVVFU1QtLS0tLQo=
  isCA: false
  usages:
  - signing
  - digital signature
  - server auth
  # 90 days
  duration: 2160h
  issuerRef:
    name: ca-issuer
    # We can reference ClusterIssuers by changing the kind here.
    # The default value is Issuer (i.e. a locally namespaced Issuer)
    kind: Issuer
    group: cert-manager.io
__END__


cat > config.yaml <<'__END__'
custom_resource_config_file: custom-resource-config.yaml
__END__

cat > custom-resource-config.yaml <<'__END__'
kind: CustomResourceStateMetrics
spec:
  resources:
    - groupVersionKind:
        group: cert-manager.io
        kind: "CertificateRequest"
        version: "v1"
      metricNamePrefix: certmanager_certificaterequest
      labelsFromPath:
        certificaterequest: [metadata, name]
        namespace: [metadata, namespace]
      metrics:
        - name: creation_time
          help: "created timestamp seconds"
          each:
            type: Gauge
            gauge:
              path: [metadata, creationTimestamp]
__END__

go run main.go --config config.yaml --kubeconfig ~/.kube/config

curl -sSLf1 http://localhost:8080/metrics | grep '^kube_customresource_creation_time'
```

If you `git checkout v2.16.0` and follow these steps you will see a log line like:

```
I1029 16:54:39.268274  516434 config.go:85] "Using custom resource plural" resource="cert-manager.io_v1_CertificateRequest" plural="certificaterequests"
I1029 16:54:39.268824  516434 custom_resource_metrics.go:79] "Custom resource state added metrics" familyNames=["certmanager_certificaterequest_creation_time"]
I1029 16:54:39.269603  516434 builder.go:294] "Active resources" activeStoreNames="cert-manager.io/v1, Resource=certificaterequests,[...]"
```

and will see metrics like

```
$ curl -sSLf1 http://localhost:8080/metrics | grep '^certmanager_certificaterequest_creation_time'
...
certmanager_certificaterequest_creation_time{certificaterequest="my-ca-cr",customresource_group="cert-manager.io",customresource_kind="CertificateRequest",customresource_version="v1",namespace="default"} 1.76170942e+09
...
```

If you check out `main` or `v2.17.0` you will not see such output.

```
$ curl -sSLf1 http://localhost:8080/metrics | grep 'my-ca-cr'
$
```

If you append `--custom-resource-state-config-file=./custom-resource-config.yaml` to the CLI in v2.17.0, the CRs will be detected again.


**Anything else we need to know?**:

Works on v2.16.0 breaks on v2.17.0

Strongly suspect regression in one of

*  https://github.com/kubernetes/kube-state-metrics/pull/2705
* https://github.com/kubernetes/kube-state-metrics/pull/2703

**Environment**:

* kube-state-metrics version:  v2.17.0
* Kubernetes version (use `kubectl version`): 
        $ kubectl version
        Client Version: v1.32.1
        Kustomize Version: v5.5.0
        Server Version: v1.32.9-eks-ba24e9c
* Cloud provider or hardware configuration: N/A
* Other info: N/A



Discussion:
```
➜  kube-state-metrics git:(f9f0a178) ✗ git bisect start v2.17.0 v2.16.0                       
HEAD is now at f9f0a178 Merge pull request #2757 from Rishab87/prep-v2.17.0
Bisecting: 32 revisions left to test after this (roughly 5 steps)
[1556ac589fe7de1041bc4f1f3566fc7bddb7d930] Merge pull request #2723 from ydFu/patch-1
➜  kube-state-metrics git:(1556ac58) ✗ go run main.go --config=config.yaml --kubeconfig=/home/craig/projects/EDB/UPM/upm-stack/appliance-access/kind-beaconator/.kubeconfig
...
I1029 17:04:39.267979  541795 custom_resource_metrics.go:79] "Custom resource state added metrics" familyNames=["certmanager_certificaterequest_creation_time"]
...
➜  kube-state-metrics git:(1556ac58) ✗ git bisect good
Bisecting: 15 revisions left to test after this (roughly 4 steps)
[bdcb1e852acd3969420b03aae1bd673100aea9b5] Merge pull request #2675 from IgorIgnatevBolt/feat-hpa-created-metric
➜  kube-state-metrics git:(bdcb1e85) ✗ go run main.go --config=config.yaml --kubeconfig=/home/craig/projects/EDB/UPM/upm-stack/appliance-access/kind-beaconator/.kubeconfig
...
➜  kube-state-metrics git:(bdcb1e85) ✗ git bisect bad
Bisecting: 7 revisions left to test after this (roughly 3 steps)
[9de0b5237b5922f4c4af5c17b107922a705d5680] Merge pull request #2737 from kubernetes/dependabot/go_modules/k8s-dependencies-9dfc872f75
➜  kube-state-metrics git:(9de0b523) ✗ go run main.go --config=config.yaml --kubeconfig=/home/craig/projects/EDB/UPM/upm-stack/appliance-access/kind-beaconator/.kubeconfig
...
I1029 17:05:34.277548  544332 custom_resource_metrics.go:79] "Custom resource state added metrics" familyNames=["certmanager_certificaterequest_creation_time"]
➜  kube-state-metrics git:(9de0b523) ✗ git bisect good
Bisecting: 3 revisions left to test after this (roughly 2 steps)
[160b09bf7d6386f537ab8bfa96fe6eba3bcbe88a] Merge pull request #2703 from rashmichandrashekar/rashmi/ksm-cfg
➜  kube-state-metrics git:(160b09bf) ✗ go run main.go --config=config.yaml --kubeconfig=/home/craig/projects/EDB/UPM/upm-stack/appliance-access/kind-beaconator/.kubeconfig
...
➜  kube-state-metrics git:(160b09bf) ✗ git bisect bad
Bisecting: 1 revision left to test after this (roughly 1 step)
[66296c38198bbb502fd1b43006a41b72044ad1a8] Merge pull request #2738 from kubernetes/dependabot/github_actions/amannn/action-semantic-pull-request-6.0.1
➜  kube-state-metrics git:(66296c38) ✗ go run main.go --config=config.yaml --kubeconfig=/home/craig/projects/EDB/UPM/upm-stack/appliance-access/kind-beaconator/.kubeconfig
...
I1029 17:06:45.776031  547197 custom_resource_metrics.go:79] "Custom resource state added metrics" familyNames=["certmanager_certificaterequest_creation_time"]
...
➜  kube-state-metrics git:(66296c38) ✗ git bisect good                                                                                                                     
Bisecting: 0 revisions left to test after this (roughly 0 steps)
[141c1f05bcdd28f4b0242f563f21612eecfa7f30] changes for continuing to run when file doesnt exist
```


Cause is https://github.com/kubernetes/kube-state-metrics/pull/2703
```
➜  kube-state-metrics git:(f9f0a178) ✗ dlv debug main.go -- --config=config.yaml --kubeconfig=$KUBECONFIG
(dlv) break ./internal/wrapper.go:83
Breakpoint 3 set at 0x2c73812 for k8s.io/kube-state-metrics/v2/internal.RunKubeStateMetricsWrapper() ./internal/wrapper.go:83
(dlv) r
Process restarted with PID 637192

> [Breakpoint 3] k8s.io/kube-state-metrics/v2/internal.RunKubeStateMetricsWrapper() ./internal/wrapper.go:83 (hits goroutine(1):1 total:1) (PC: 0x2c73812)
    78:				configFile, err := os.ReadFile(filepath.Clean(file))
    79:				if err != nil {
    80:					klog.ErrorS(err, "failed to read options configuration file", "file", file)
    81:				}
    82:	
=>  83:				yaml.Unmarshal(configFile, opts)
    84:			}
    85:		}
    86:		if opts.CustomResourceConfigFile != "" {
    87:			crcViper := viper.New()
    88:			crcViper.SetConfigType("yaml")
(dlv) p configFile
[]uint8 len: 57, cap: 512, [99,117,115,116,111,109,95,114,101,115,111,117,114,99,101,95,99,111,110,102,105,103,95,102,105,108,101,58,32,99,117,115,116,111,109,45,114,101,115,111,117,114,99,101,45,99,111,110,102,105,103,46,121,97,109,108,10]
(dlv) p string(configFile)
"custom_resource_config_file: custom-resource-config.yaml\n"
(dlv) p opts.CustomResourceConfigFile
""
(dlv) step
> go.yaml.in/yaml/v3.Unmarshal() /home/craig/go/pkg/mod/go.yaml.in/yaml/v3@v3.0.3/yaml.go:87 (PC: 0xb5c7d0)
    82:	//	yaml.Unmarshal([]byte("a: 1\nb: 2"), &t)
    83:	//
    84:	// See the documentation of Marshal for the format of tags and a list of
    85:	// supported tag options.
    86:	func Unmarshal(in []byte, out interface{}) (err error) {
=>  87:		return unmarshal(in, out, false)
    88:	}
    89:	
    90:	// A Decoder reads and decodes YAML values from an input stream.
    91:	type Decoder struct {
    92:		parser      *parser
(dlv) step
(dlv) stepout
> go.yaml.in/yaml/v3.Unmarshal() /home/craig/go/pkg/mod/go.yaml.in/yaml/v3@v3.0.3/yaml.go:87 (PC: 0xb5c7f4)
Values returned:
	err: error nil

    82:	//	yaml.Unmarshal([]byte("a: 1\nb: 2"), &t)
    83:	//
    84:	// See the documentation of Marshal for the format of tags and a list of
    85:	// supported tag options.
    86:	func Unmarshal(in []byte, out interface{}) (err error) {
=>  87:		return unmarshal(in, out, false)
    88:	}
    89:	
    90:	// A Decoder reads and decodes YAML values from an input stream.
    91:	type Decoder struct {
    92:		parser      *parser
(dlv) p in
[]uint8 len: 57, cap: 512, [99,117,115,116,111,109,95,114,101,115,111,117,114,99,101,95,99,111,110,102,105,103,95,102,105,108,101,58,32,99,117,115,116,111,109,45,114,101,115,111,117,114,99,101,45,99,111,110,102,105,103,46,121,97,109,108,10]
(dlv) p string(in)
"custom_resource_config_file: custom-resource-config.yaml\n"
(dlv) p out.CustomResourceConfigFile
""
```

It looks like the config unmarshal just isn't loading the CR config file from the config.yaml ?

(The unmarshal also fails to check for error return, but that does not appear to be the issue)
Comparing to 2.16.0

```
➜  kube-state-metrics git:(291a9628) ✗ dlv debug main.go -- --config=config.yaml --kubeconfig=$KUBECONFIG
➜  kube-state-metrics git:(291a9628) ✗ dlv debug main.go -- --config=config.yaml --kubeconfig=/home/craig/projects/EDB/UPM/upm-stack/appliance-access/kind-beaconator/.kubeconfig
Type 'help' for list of commands.
(dlv) break internal/wrapper.go:74
Breakpoint 1 set at 0x2b13167 for k8s.io/kube-state-metrics/v2/internal.RunKubeStateMetricsWrapper() ./internal/wrapper.go:74
(dlv) c
> [Breakpoint 1] k8s.io/kube-state-metrics/v2/internal.RunKubeStateMetricsWrapper() ./internal/wrapper.go:74 (hits goroutine(1):1 total:1) (PC: 0x2b13167)
    69:			configFile, err := os.ReadFile(filepath.Clean(file))
    70:			if err != nil {
    71:				klog.ErrorS(err, "failed to read options configuration file", "file", file)
    72:			}
    73:	
=>  74:			yaml.Unmarshal(configFile, opts)
    75:		}
    76:		if opts.CustomResourceConfigFile != "" {
    77:			crcViper := viper.New()
    78:			crcViper.SetConfigType("yaml")
    79:			crcViper.SetConfigFile(opts.CustomResourceConfigFile)
(dlv) p configFile
[]uint8 len: 57, cap: 512, [99,117,115,116,111,109,95,114,101,115,111,117,114,99,101,95,99,111,110,102,105,103,95,102,105,108,101,58,32,99,117,115,116,111,109,45,114,101,115,111,117,114,99,101,45,99,111,110,102,105,103,46,121,97,109,108,10]
(dlv) p string(configFile)
"custom_resource_config_file: custom-resource-config.yaml\n"
(dlv) p opts.CustomResourceConfigFile
""
(dlv) s
> sigs.k8s.io/yaml/goyaml%2ev3.Unmarshal() /home/craig/go/pkg/mod/sigs.k8s.io/yaml@v1.4.0/goyaml.v3/yaml.go:88 (PC: 0xb5c04a)
    83:	//     yaml.Unmarshal([]byte("a: 1\nb: 2"), &t)
    84:	//
    85:	// See the documentation of Marshal for the format of tags and a list of
    86:	// supported tag options.
    87:	//
=>  88:	func Unmarshal(in []byte, out interface{}) (err error) {
    89:		return unmarshal(in, out, false)
    90:	}
    91:	
    92:	// A Decoder reads and decodes YAML values from an input stream.
    93:	type Decoder struct {
(dlv) stepout
> k8s.io/kube-state-metrics/v2/internal.RunKubeStateMetricsWrapper() ./internal/wrapper.go:74 (PC: 0x2b13193)
Values returned:
	err: error nil

    69:			configFile, err := os.ReadFile(filepath.Clean(file))
    70:			if err != nil {
    71:				klog.ErrorS(err, "failed to read options configuration file", "file", file)
    72:			}
    73:	
=>  74:			yaml.Unmarshal(configFile, opts)
    75:		}
    76:		if opts.CustomResourceConfigFile != "" {
    77:			crcViper := viper.New()
    78:			crcViper.SetConfigType("yaml")
    79:			crcViper.SetConfigFile(opts.CustomResourceConfigFile)
(dlv) p opts.CustomResourceConfigFile
"custom-resource-config.yaml"
(dlv) 
```

it unmarshals correctly.
Found the cause.

https://github.com/kubernetes/kube-state-metrics/pull/2703/files#r2475892642

The option was renamed from `custom_resource_config_file` to `custom_resource_state_config_file`.
thanks for debugging this @ringerc and apologies for the inconvenience caused by this. Do you want to create a PR to rename it back? We should have the options in the config match the cli arg.
@mrueg Given that it was never formally documented I don't think it's strictly a regression. I might see about raising one to document it at least.

If you want it renamed the same time let me know.

Once I figured out what broke it was easy enough to work around. Usually I'd say that such a renaming should come with backwards compat to respect the old name and log a deprecation warning, but again, it was never formally documented so that's less of a big deal here.
/triage accepted
/assign @mrueg 



Patch:
diff --git a/docs/metrics/extend/customresourcestate-metrics.md b/docs/metrics/extend/customresourcestate-metrics.md
index 9e493a9a2a..66649b564b 100644
--- a/docs/metrics/extend/customresourcestate-metrics.md
+++ b/docs/metrics/extend/customresourcestate-metrics.md
@@ -12,6 +12,8 @@ Two flags can be used:
 * `--custom-resource-state-config "inline yaml (see example)"` or
 * `--custom-resource-state-config-file /path/to/config.yaml`
 
+When using a `--config` file, the equivalent YAML key is `custom_resource_state_config_file`. The pre-v2.17 key `custom_resource_config_file` is still honored as a deprecated alias and will be removed in a future release; please migrate to `custom_resource_state_config_file`.
+
 If both flags are provided, the inline configuration will take precedence.
 When multiple entries for the same resource exist, kube-state-metrics will exit with an error.
 This includes configuration which refers to a different API version.
diff --git a/internal/wrapper.go b/internal/wrapper.go
index ba2089a60b..19bee1b94b 100644
--- a/internal/wrapper.go
+++ b/internal/wrapper.go
@@ -79,9 +79,23 @@ func RunKubeStateMetricsWrapper(opts *options.Options) {
 			configFile, err := os.ReadFile(filepath.Clean(file))
 			if err != nil {
 				klog.ErrorS(err, "failed to read options configuration file", "file", file)
+				if !opts.ContinueWithoutConfig {
+					klog.FlushAndExit(klog.ExitFlushTimeout, 1)
+				}
+			} else {
+				if err := yaml.Unmarshal(configFile, opts); err != nil {
+					klog.ErrorS(err, "failed to unmarshal options configuration file", "file", file)
+					if !opts.ContinueWithoutConfig {
+						klog.FlushAndExit(klog.ExitFlushTimeout, 1)
+					}
+					// TODO: Remove deprecated custom_resource_config_file support in KSM 2.21.
+				} else if opts.CustomResourceConfigFileDeprecated != "" {
+					klog.Warning("The 'custom_resource_config_file' config key is deprecated and will be removed in a future release; use 'custom_resource_state_config_file' instead")
+					if opts.CustomResourceConfigFile == "" {
+						opts.CustomResourceConfigFile = opts.CustomResourceConfigFileDeprecated
+					}
+				}
 			}
-
-			yaml.Unmarshal(configFile, opts)
 		}
 	}
 	if opts.CustomResourceConfigFile != "" {
diff --git a/pkg/options/options.go b/pkg/options/options.go
index e57f4ba4d3..ad718ea72e 100644
--- a/pkg/options/options.go
+++ b/pkg/options/options.go
@@ -47,10 +47,15 @@ type Options struct {
 	MetricOptInList      MetricSet       `yaml:"metric_opt_in_list"`
 	Resources            ResourceSet     `yaml:"resources"`
 
-	cmd                                     *cobra.Command
-	Apiserver                               string   `yaml:"apiserver"`
-	CustomResourceConfig                    string   `yaml:"custom_resource_config"`
-	CustomResourceConfigFile                string   `yaml:"custom_resource_state_config_file"`
+	cmd                      *cobra.Command
+	Apiserver                string `yaml:"apiserver"`
+	CustomResourceConfig     string `yaml:"custom_resource_config"`
+	CustomResourceConfigFile string `yaml:"custom_resource_state_config_file"`
+	// CustomResourceConfigFileDeprecated preserves backward compatibility with the
+	// pre-v2.17 config key `custom_resource_config_file`. Honored as an alias when
+	// the canonical `custom_resource_state_config_file` is not set; emits a
+	// deprecation warning at startup. Will be removed in a future release.
+	CustomResourceConfigFileDeprecated      string   `yaml:"custom_resource_config_file"`
 	ContinueWithoutCustomResourceConfigFile bool     `yaml:"continue_without_custom_resource_state_config_file"`
 	Host                                    string   `yaml:"host"`
 	Kubeconfig                              string   `yaml:"kubeconfig"`


Test patch:
diff --git a/pkg/options/options_test.go b/pkg/options/options_test.go
index 0e17c0c5a8..ad21c32257 100644
--- a/pkg/options/options_test.go
+++ b/pkg/options/options_test.go
@@ -19,6 +19,8 @@ package options
 import (
 	"os"
 	"testing"
+
+	yaml "sigs.k8s.io/yaml/goyaml.v3"
 )
 
 func TestOptionsParse(t *testing.T) {
@@ -63,3 +65,32 @@ func TestOptionsParse(t *testing.T) {
 		})
 	}
 }
+
+func TestCustomResourceConfigFileDeprecatedAlias(t *testing.T) {
+	t.Run("deprecated key populates alias field", func(t *testing.T) {
+		opts := NewOptions()
+		if err := yaml.Unmarshal([]byte("custom_resource_config_file: /etc/ksm/crs.yaml\n"), opts); err != nil {
+			t.Fatalf("unmarshal: %v", err)
+		}
+		if opts.CustomResourceConfigFileDeprecated != "/etc/ksm/crs.yaml" {
+			t.Fatalf("expected deprecated alias to be set, got %q", opts.CustomResourceConfigFileDeprecated)
+		}
+		if opts.CustomResourceConfigFile != "" {
+			t.Fatalf("expected canonical field to remain empty pre-merge, got %q", opts.CustomResourceConfigFile)
+		}
+	})
+
+	t.Run("canonical key takes precedence when both are set", func(t *testing.T) {
+		opts := NewOptions()
+		yamlIn := "custom_resource_config_file: /old.yaml\ncustom_resource_state_config_file: /new.yaml\n"
+		if err := yaml.Unmarshal([]byte(yamlIn), opts); err != nil {
+			t.Fatalf("unmarshal: %v", err)
+		}
+		if opts.CustomResourceConfigFile != "/new.yaml" {
+			t.Fatalf("expected canonical field to win, got %q", opts.CustomResourceConfigFile)
+		}
+		if opts.CustomResourceConfigFileDeprecated != "/old.yaml" {
+			t.Fatalf("expected deprecated field to retain its value, got %q", opts.CustomResourceConfigFileDeprecated)
+		}
+	})
+}