Repository: prometheus/prometheus (language: go)

Issue:
None of kubernetes jobs in up state
### What did you do?

start prometheus, access /targets in browser.


### What did you expect to see?

kubernetes related targets should all  in up state, I verified this in version v2.42.0.

### What did you see instead? Under which circumstances?

But with version v2.43.0-rc.0
[kube-state-metrics (0/2 up)]
[kubernetes-apiservers (0/3 up)]
[kubernetes-nodes (0/11 up)]
[kubernetes-nodes-cadvisor (0/11 up)]
[kubernetes-nodes-kubelet (0/11 up)]
[kubernetes-pods (0/15 up)]
[kubernetes-service-endpoints (0/8 up)]
[prometheus (1/1 up)]

### System information

Linux 3.10.0-1127.el7.x86_64 x86_64

### Prometheus version

```text
prometheus, version 2.43.0-rc.0 (branch: HEAD, revision: 001ee2620e094970e5657ce39275b2fccdbd1359)
  build user:       root@1dd654b3d754
  build date:       20230309-15:11:28
  go version:       go1.19.7
  platform:         linux/amd64
  tags:             netgo,builtinassets
```


### Prometheus configuration file

```yaml
global:
  scrape_interval:     2m
  evaluation_interval: 1m
  scrape_timeout:      30s

rule_files:
- /etc/config/rules/*.yml

alerting:
  alertmanagers:
  - static_configs:
      - targets:
        - alertmanager:80


#scrape_config_files:
#- /etc/config/scrapes/*.yml

scrape_configs:
- job_name: prometheus
  static_configs:
  - targets:
    - localhost:9090

- job_name: kubernetes-nodes
  kubernetes_sd_configs:
  - role: node
  relabel_configs:
  - action: labelmap
    regex: __meta_kubernetes_node_label_(.+)
  - action: labelmap
    regex: __meta_kubernetes_node_label_kubernetes_io_hostname
    replacement: node
  - action: replace
    source_labels:
    - __address__
    regex: (.*?):(.*?)
    target_label: __address__
    replacement: ${1}:9100

- job_name: kubernetes-apiservers
  kubernetes_sd_configs:
  - role: endpoints
    namespaces:
      names:
      - default
  relabel_configs:
  - action: keep
    regex: default;kubernetes;https
    source_labels:
    - __meta_kubernetes_namespace
    - __meta_kubernetes_service_name
    - __meta_kubernetes_endpoint_port_name
  scheme: https
  tls_config:
    ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
    insecure_skip_verify: true
  bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token

- job_name: kubernetes-nodes-kubelet
  kubernetes_sd_configs:
  - role: node
  relabel_configs:
  - action: labelmap
    regex: __meta_kubernetes_node_label_(.+)
  scheme: https
  tls_config:
    ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
    insecure_skip_verify: true
  bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token

- job_name: kubernetes-nodes-cadvisor
  kubernetes_sd_configs:
  - role: node
  relabel_configs:
  - action: labelmap
    regex: __meta_kubernetes_node_label_(.+)
  - action: replace
    target_label: __metrics_path__
    replacement: /metrics/cadvisor
  - source_labels: [instance]
    separator: ;
    regex: (.+)
    target_label: node
    replacement: $1
    action: replace
  scheme: https
  tls_config:
    ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
    insecure_skip_verify: true
  bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token

- job_name: kube-state-metrics
  kubernetes_sd_configs:
  - role: endpoints
    namespaces:
      names:
      - dtg-ops
  relabel_configs:
  - action: keep
    regex: true
    source_labels:
    - __meta_kubernetes_service_annotation_prometheus_io_scrape
  - action: keep
    source_labels:
    - __meta_kubernetes_pod_container_name
    regex: kube-state-metrics
  - action: labelmap
    regex: __meta_kubernetes_service_label_(.+)
  - action: replace
    source_labels:
    - __meta_kubernetes_namespace
    target_label: kubernetes_namespace
  - action: replace
    source_labels:
    - __meta_kubernetes_service_name
    target_label: kubernetes_service
  - action: replace
    source_labels:
    - __meta_kubernetes_pod_name
    target_label: kubernetes_pod
  - action: replace
    source_labels:
    - __meta_kubernetes_endpoints_name
    target_label: kubernetes_endpoint

- job_name: kubernetes-service-endpoints
  kubernetes_sd_configs:
  - role: endpoints
  relabel_configs:
  - action: keep
    regex: true
    source_labels:
    - __meta_kubernetes_service_annotation_prometheus_io_scrape
  - action: drop
    regex: kube-state-metrics
    source_labels:
    - __meta_kubernetes_endpoints_name
  - action: replace
    regex: (https?)
    source_labels:
    - _kubernetes_service_annotation_prometheus_io_scheme
    target_label: __scheme__
  - action: replace
    regex: (.+)
    source_labels:
    - __meta_kubernetes_service_annotation_prometheus_io_path
    target_label: __metrics_path__
  - action: replace
    regex: ([^:]+)(?::\d+)?;(\d+)
    replacement: $1:$2
    source_labels:
    - __address__
    - __meta_kubernetes_service_annotation_prometheus_io_port
    target_label: __address__
  - action: labelmap
    regex: __meta_kubernetes_service_label_(.+)
  - action: replace
    source_labels:
    - __meta_kubernetes_namespace
    target_label: kubernetes_namespace
  - action: replace
    source_labels:
    - __meta_kubernetes_service_name
    target_label: kubernetes_service_name
  - action: replace
    source_labels:
    - __meta_kubernetes_pod_name
    target_label: kubernetes_pod
  - action: replace
    source_labels:
    - __meta_kubernetes_endpoints_name
    target_label: kubernetes_endpoint

- job_name: kubernetes-services
  kubernetes_sd_configs:
  - role: service
  metrics_path: /probe
  params:
    module:
    - http_2xx
  relabel_configs:
  - action: keep
    regex: true
    source_labels:
    - __meta_kubernetes_service_annotation_prometheus_io_probe
  - source_labels:
    - __address__
    target_label: __param_target
  - replacement: blackbox
    target_label: __address__
  - source_labels:
    - __param_target
    target_label: instance
  - action: labelmap
    regex: __meta_kubernetes_service_label_(.+)
  - action: replace
    source_labels:
    - __meta_kubernetes_namespace
    target_label: kubernetes_namespace
  - action: replace
    source_labels:
    - __meta_kubernetes_service_name
    target_label: kubernetes_service_name

- job_name: kubernetes-pods
  kubernetes_sd_configs:
  - role: pod
  relabel_configs:
  - action: keep
    regex: true
    source_labels:
    - __meta_kubernetes_pod_annotation_prometheus_io_scrape
  - action: replace
    regex: (.+)
    source_labels:
    - __meta_kubernetes_pod_annotation_prometheus_io_path
    target_label: __metrics_path__
  - action: replace
    regex: ([^:]+)(?::\d+)?;(\d+)
    replacement: $1:$2
    source_labels:
    - __address__
    - __meta_kubernetes_pod_annotation_prometheus_io_port
    target_label: __address__
  - action: labelmap
    regex: __meta_kubernetes_pod_label_(.+)
  - action: replace
    source_labels:
    - __meta_kubernetes_namespace
    target_label: kubernetes_namespace
  - action: replace
    source_labels:
    - __meta_kubernetes_pod_name
    target_label: kubernetes_pod_name
```


### Alertmanager version

_No response_

### Alertmanager configuration file

_No response_

### Logs

```text
ts=2023-03-13T09:27:44.863Z caller=main.go:509 level=warn deprecation_notice="'storage.tsdb.retention' flag is deprecated use 'storage.tsdb.retention.time' instead."
ts=2023-03-13T09:27:44.863Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0-rc.0, branch=HEAD, revision=001ee2620e094970e5657ce39275b2fccdbd1359)"
ts=2023-03-13T09:27:44.863Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@1dd654b3d754, date=20230309-15:11:28, tags=netgo,builtinassets)"
ts=2023-03-13T09:27:44.863Z caller=main.go:570 level=info host_details="(Linux 3.10.0-1127.el7.x86_64 #1 SMP Tue Mar 31 23:36:51 UTC 2020 x86_64 prometheus-0 (none))"
ts=2023-03-13T09:27:44.863Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)"
ts=2023-03-13T09:27:44.863Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)"
ts=2023-03-13T09:27:44.870Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=0.0.0.0:9090
ts=2023-03-13T09:27:44.871Z caller=main.go:1005 level=info msg="Starting TSDB ..."
ts=2023-03-13T09:27:44.873Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9090
ts=2023-03-13T09:27:44.873Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9090
ts=2023-03-13T09:27:44.874Z caller=repair.go:56 level=info component=tsdb msg="Found healthy block" mint=1677801600051 maxt=1677866400000 ulid=01GTMV8V2RCPKHBDR6S00FPQ4P
ts=2023-03-13T09:27:44.875Z caller=repair.go:56 level=info component=tsdb msg="Found healthy block" mint=1677866400168 maxt=1677931200000 ulid=01GTPS2FTVKY1YCWPCP102KYGQ
ts=2023-03-13T09:27:44.876Z caller=repair.go:56 level=info component=tsdb msg="Found healthy block" mint=1677931200047 maxt=1677996000000 ulid=01GTRPVXRT3RMZJFW2MCF190GG
ts=2023-03-13T09:27:44.877Z caller=repair.go:56 level=info component=tsdb msg="Found healthy block" mint=1677996000239 maxt=1678060800000 ulid=01GTTMNJK7A24W6F9ANT5WE93M
ts=2023-03-13T09:27:44.878Z caller=repair.go:56 level=info component=tsdb msg="Found healthy block" mint=1678060800891 maxt=1678125600000 ulid=01GTWJF1VY4D00XN1PCSTDH4BA
ts=2023-03-13T09:27:44.879Z caller=repair.go:56 level=info component=tsdb msg="Found healthy block" mint=1678125600535 maxt=1678190400000 ulid=01GTYG8KGAAQBM5W9XVJKF0ADK
ts=2023-03-13T09:27:44.882Z caller=repair.go:56 level=info component=tsdb msg="Found healthy block" mint=1678190400891 maxt=1678255200000 ulid=01GV0E2A1K5T74TMFESAH5MH7K
ts=2023-03-13T09:27:44.883Z caller=repair.go:56 level=info component=tsdb msg="Found healthy block" mint=1678255200795 maxt=1678320000000 ulid=01GV2BVN377HTDH1B7MGXMB9CN
ts=2023-03-13T09:27:44.884Z caller=repair.go:56 level=info component=tsdb msg="Found healthy block" mint=1678320000781 maxt=1678384800000 ulid=01GV49NB7TX97BB91VMZ221JPB
ts=2023-03-13T09:27:44.885Z caller=repair.go:56 level=info component=tsdb msg="Found healthy block" mint=1678384800491 maxt=1678449600000 ulid=01GV67EVDQEN7VJ702XKY66BA4
ts=2023-03-13T09:27:44.886Z caller=repair.go:56 level=info component=tsdb msg="Found healthy block" mint=1678449600767 maxt=1678514400000 ulid=01GV858EMFACH5PH1NWKG2PEB0
ts=2023-03-13T09:27:44.887Z caller=repair.go:56 level=info component=tsdb msg="Found healthy block" mint=1678514400475 maxt=1678579200000 ulid=01GVA31VBBPTX9QD6G4WGTNAFK
ts=2023-03-13T09:27:44.889Z caller=repair.go:56 level=info component=tsdb msg="Found healthy block" mint=1678579200891 maxt=1678644000000 ulid=01GVC0VG76CK5MK3F9WP0YYJY9
ts=2023-03-13T09:27:44.890Z caller=repair.go:56 level=info component=tsdb msg="Found healthy block" mint=1678665600891 maxt=1678672800000 ulid=01GVCEJ6TQSB14JR4Z2T0RZ41V
ts=2023-03-13T09:27:44.891Z caller=repair.go:56 level=info component=tsdb msg="Found healthy block" mint=1678672800883 maxt=1678680000000 ulid=01GVCNDY02JBEHBNC1ST0X9P24
ts=2023-03-13T09:27:44.892Z caller=repair.go:56 level=info component=tsdb msg="Found healthy block" mint=1678644000187 maxt=1678665600000 ulid=01GVCNE5S416NSH4RRNSP1N673
ts=2023-03-13T09:27:44.893Z caller=repair.go:56 level=info component=tsdb msg="Found healthy block" mint=1678680000000 maxt=1678687200000 ulid=01GVCW9N82MQ4EWEPEYT6FMVBX
ts=2023-03-13T09:27:44.894Z caller=repair.go:56 level=info component=tsdb msg="Found healthy block" mint=1678687200568 maxt=1678694400000 ulid=01GVD35CM2G4BNEG4YB2C7XXA0
ts=2023-03-13T09:27:45.918Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any"
ts=2023-03-13T09:27:46.033Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=114.233152ms
ts=2023-03-13T09:27:46.033Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while"
ts=2023-03-13T09:27:49.078Z caller=head.go:700 level=info component=tsdb msg="WAL checkpoint loaded"
ts=2023-03-13T09:27:49.903Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=158 maxSegment=163
ts=2023-03-13T09:27:50.488Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=159 maxSegment=163
ts=2023-03-13T09:27:50.813Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=160 maxSegment=163
ts=2023-03-13T09:27:51.104Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=161 maxSegment=163
ts=2023-03-13T09:27:51.399Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=162 maxSegment=163
ts=2023-03-13T09:27:51.401Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=163 maxSegment=163
ts=2023-03-13T09:27:51.401Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=3.044806776s wal_replay_duration=2.323654298s wbl_replay_duration=241ns total_replay_duration=5.482767536s
ts=2023-03-13T09:27:52.264Z caller=main.go:1024 level=warn fs_type=NFS_SUPER_MAGIC msg="This filesystem is not supported and may lead to data corruption and data loss. Please carefully read https://prometheus.io/docs/prometheus/latest/storage/ to learn more about supported filesystems."
ts=2023-03-13T09:27:52.264Z caller=main.go:1029 level=info msg="TSDB started"
ts=2023-03-13T09:27:52.264Z caller=main.go:1030 level=debug msg="TSDB options" MinBlockDuration=2h MaxBlockDuration=1d MaxBytes=0B NoLockfile=false RetentionDuration=10d WALSegmentSize=0B WALCompression=true
ts=2023-03-13T09:27:52.264Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/config/prometheus.yml
ts=2023-03-13T09:27:52.267Z caller=kubernetes.go:326 level=info component="discovery manager scrape" discovery=kubernetes config=kubernetes-apiservers msg="Using pod service account via in-cluster config"
ts=2023-03-13T09:27:52.267Z caller=kubernetes.go:326 level=info component="discovery manager scrape" discovery=kubernetes config=kubernetes-nodes-kubelet msg="Using pod service account via in-cluster config"
ts=2023-03-13T09:27:52.268Z caller=kubernetes.go:326 level=info component="discovery manager scrape" discovery=kubernetes config=kubernetes-services msg="Using pod service account via in-cluster config"
ts=2023-03-13T09:27:52.268Z caller=kubernetes.go:326 level=info component="discovery manager scrape" discovery=kubernetes config=kubernetes-pods msg="Using pod service account via in-cluster config"
ts=2023-03-13T09:27:52.268Z caller=kubernetes.go:326 level=info component="discovery manager scrape" discovery=kubernetes config=kube-state-metrics msg="Using pod service account via in-cluster config"
ts=2023-03-13T09:27:52.269Z caller=kubernetes.go:326 level=info component="discovery manager scrape" discovery=kubernetes config=kubernetes-service-endpoints msg="Using pod service account via in-cluster config"
ts=2023-03-13T09:27:52.269Z caller=manager.go:196 level=debug component="discovery manager scrape" msg="Starting provider" provider=kubernetes/0 subs=[kubernetes-apiservers]
ts=2023-03-13T09:27:52.269Z caller=manager.go:196 level=debug component="discovery manager scrape" msg="Starting provider" provider=kubernetes/1 subs="[kubernetes-nodes-kubelet kubernetes-nodes-cadvisor kubernetes-nodes]"
ts=2023-03-13T09:27:52.269Z caller=manager.go:196 level=debug component="discovery manager scrape" msg="Starting provider" provider=dns/2 subs=[non-k8s-TEST-java-application]
ts=2023-03-13T09:27:52.269Z caller=manager.go:196 level=debug component="discovery manager scrape" msg="Starting provider" provider=kubernetes/3 subs=[kubernetes-services]
ts=2023-03-13T09:27:52.269Z caller=manager.go:196 level=debug component="discovery manager scrape" msg="Starting provider" provider=kubernetes/4 subs=[kubernetes-pods]
ts=2023-03-13T09:27:52.269Z caller=manager.go:196 level=debug component="discovery manager scrape" msg="Starting provider" provider=static/5 subs=[prometheus]
ts=2023-03-13T09:27:52.269Z caller=manager.go:196 level=debug component="discovery manager scrape" msg="Starting provider" provider=kubernetes/6 subs=[kube-state-metrics]
ts=2023-03-13T09:27:52.269Z caller=manager.go:196 level=debug component="discovery manager scrape" msg="Starting provider" provider=kubernetes/7 subs=[kubernetes-service-endpoints]
ts=2023-03-13T09:27:52.269Z caller=manager.go:196 level=debug component="discovery manager notify" msg="Starting provider" provider=static/0 subs=[config-0]
ts=2023-03-13T09:27:52.270Z caller=klog.go:55 level=debug component=k8s_client_runtime func=Verbose.Infof msg="Starting reflector *v1.Pod (0s) from pkg/mod/k8s.io/client-go@v0.26.2/tools/cache/reflector.go:169"
ts=2023-03-13T09:27:52.270Z caller=klog.go:55 level=debug component=k8s_client_runtime func=Verbose.Infof msg="Listing and watching *v1.Pod from pkg/mod/k8s.io/client-go@v0.26.2/tools/cache/reflector.go:169"
ts=2023-03-13T09:27:52.270Z caller=klog.go:55 level=debug component=k8s_client_runtime func=Verbose.Infof msg="Starting reflector *v1.Node (0s) from pkg/mod/k8s.io/client-go@v0.26.2/tools/cache/reflector.go:169"
ts=2023-03-13T09:27:52.270Z caller=klog.go:55 level=debug component=k8s_client_runtime func=Verbose.Infof msg="Listing and watching *v1.Node from pkg/mod/k8s.io/client-go@v0.26.2/tools/cache/reflector.go:169"
ts=2023-03-13T09:27:52.270Z caller=manager.go:214 level=debug component="discovery manager scrape" msg="Discoverer channel closed" provider=static/5
ts=2023-03-13T09:27:52.270Z caller=klog.go:55 level=debug component=k8s_client_runtime func=Verbose.Infof msg="Starting reflector *v1.Endpoints (0s) from pkg/mod/k8s.io/client-go@v0.26.2/tools/cache/reflector.go:169"
ts=2023-03-13T09:27:52.270Z caller=klog.go:55 level=debug component=k8s_client_runtime func=Verbose.Infof msg="Starting reflector *v1.Pod (0s) from pkg/mod/k8s.io/client-go@v0.26.2/tools/cache/reflector.go:169"
ts=2023-03-13T09:27:52.270Z caller=klog.go:55 level=debug component=k8s_client_runtime func=Verbose.Infof msg="Listing and watching *v1.Endpoints from pkg/mod/k8s.io/client-go@v0.26.2/tools/cache/reflector.go:169"
ts=2023-03-13T09:27:52.270Z caller=klog.go:55 level=debug component=k8s_client_runtime func=Verbose.Infof msg="Listing and watching *v1.Pod from pkg/mod/k8s.io/client-go@v0.26.2/tools/cache/reflector.go:169"
ts=2023-03-13T09:27:52.270Z caller=klog.go:55 level=debug component=k8s_client_runtime func=Verbose.Infof msg="Starting reflector *v1.Endpoints (0s) from pkg/mod/k8s.io/client-go@v0.26.2/tools/cache/reflector.go:169"
ts=2023-03-13T09:27:52.270Z caller=klog.go:55 level=debug component=k8s_client_runtime func=Verbose.Infof msg="Listing and watching *v1.Endpoints from pkg/mod/k8s.io/client-go@v0.26.2/tools/cache/reflector.go:169"
ts=2023-03-13T09:27:52.270Z caller=klog.go:55 level=debug component=k8s_client_runtime func=Verbose.Infof msg="Starting reflector *v1.Service (0s) from pkg/mod/k8s.io/client-go@v0.26.2/tools/cache/reflector.go:169"
ts=2023-03-13T09:27:52.270Z caller=klog.go:55 level=debug component=k8s_client_runtime func=Verbose.Infof msg="Listing and watching *v1.Service from pkg/mod/k8s.io/client-go@v0.26.2/tools/cache/reflector.go:169"
ts=2023-03-13T09:27:52.270Z caller=klog.go:55 level=debug component=k8s_client_runtime func=Verbose.Infof msg="Starting reflector *v1.Endpoints (0s) from pkg/mod/k8s.io/client-go@v0.26.2/tools/cache/reflector.go:169"
ts=2023-03-13T09:27:52.270Z caller=klog.go:55 level=debug component=k8s_client_runtime func=Verbose.Infof msg="Listing and watching *v1.Endpoints from pkg/mod/k8s.io/client-go@v0.26.2/tools/cache/reflector.go:169"
ts=2023-03-13T09:27:52.271Z caller=klog.go:55 level=debug component=k8s_client_runtime func=Verbose.Infof msg="Starting reflector *v1.Pod (0s) from pkg/mod/k8s.io/client-go@v0.26.2/tools/cache/reflector.go:169"
ts=2023-03-13T09:27:52.271Z caller=klog.go:55 level=debug component=k8s_client_runtime func=Verbose.Infof msg="Listing and watching *v1.Pod from pkg/mod/k8s.io/client-go@v0.26.2/tools/cache/reflector.go:169"
ts=2023-03-13T09:27:52.271Z caller=klog.go:55 level=debug component=k8s_client_runtime func=Verbose.Infof msg="Starting reflector *v1.Service (0s) from pkg/mod/k8s.io/client-go@v0.26.2/tools/cache/reflector.go:169"
ts=2023-03-13T09:27:52.271Z caller=klog.go:55 level=debug component=k8s_client_runtime func=Verbose.Infof msg="Listing and watching *v1.Service from pkg/mod/k8s.io/client-go@v0.26.2/tools/cache/reflector.go:169"
ts=2023-03-13T09:27:52.271Z caller=klog.go:55 level=debug component=k8s_client_runtime func=Verbose.Infof msg="Starting reflector *v1.Pod (0s) from pkg/mod/k8s.io/client-go@v0.26.2/tools/cache/reflector.go:169"
ts=2023-03-13T09:27:52.270Z caller=klog.go:55 level=debug component=k8s_client_runtime func=Verbose.Infof msg="Starting reflector *v1.Service (0s) from pkg/mod/k8s.io/client-go@v0.26.2/tools/cache/reflector.go:169"
ts=2023-03-13T09:27:52.271Z caller=klog.go:55 level=debug component=k8s_client_runtime func=Verbose.Infof msg="Listing and watching *v1.Pod from pkg/mod/k8s.io/client-go@v0.26.2/tools/cache/reflector.go:169"
ts=2023-03-13T09:27:52.271Z caller=klog.go:55 level=debug component=k8s_client_runtime func=Verbose.Infof msg="Starting reflector *v1.Service (0s) from pkg/mod/k8s.io/client-go@v0.26.2/tools/cache/reflector.go:169"
ts=2023-03-13T09:27:52.271Z caller=klog.go:55 level=debug component=k8s_client_runtime func=Verbose.Infof msg="Listing and watching *v1.Service from pkg/mod/k8s.io/client-go@v0.26.2/tools/cache/reflector.go:169"
ts=2023-03-13T09:27:52.271Z caller=manager.go:214 level=debug component="discovery manager notify" msg="Discoverer channel closed" provider=static/0
ts=2023-03-13T09:27:52.271Z caller=klog.go:55 level=debug component=k8s_client_runtime func=Verbose.Infof msg="Listing and watching *v1.Service from pkg/mod/k8s.io/client-go@v0.26.2/tools/cache/reflector.go:169"
ts=2023-03-13T09:27:52.286Z caller=klog.go:84 level=debug component=k8s_client_runtime func=Infof msg="GET https://10.1.0.1:443/api/v1/namespaces/default/endpoints?limit=500&resourceVersion=0 200 OK in 14 milliseconds"
ts=2023-03-13T09:27:52.286Z caller=klog.go:84 level=debug component=k8s_client_runtime func=Infof msg="GET https://10.1.0.1:443/api/v1/namespaces/dtg-ops/endpoints?limit=500&resourceVersion=0 200 OK in 15 milliseconds"
ts=2023-03-13T09:27:52.286Z caller=klog.go:84 level=debug component=k8s_client_runtime func=Infof msg="GET https://10.1.0.1:443/api/v1/namespaces/dtg-ops/services?limit=500&resourceVersion=0 200 OK in 15 milliseconds"
ts=2023-03-13T09:27:52.287Z caller=klog.go:84 level=debug component=k8s_client_runtime func=Infof msg="GET https://10.1.0.1:443/api/v1/nodes?limit=500&resourceVersion=0 200 OK in 16 milliseconds"
ts=2023-03-13T09:27:52.287Z caller=klog.go:84 level=debug component=k8s_client_runtime func=Infof msg="GET https://10.1.0.1:443/api/v1/namespaces/default/pods?limit=500&resourceVersion=0 200 OK in 16 milliseconds"
ts=2023-03-13T09:27:52.287Z caller=klog.go:84 level=debug component=k8s_client_runtime func=Infof msg="GET https://10.1.0.1:443/api/v1/namespaces/default/services?limit=500&resourceVersion=0 200 OK in 15 milliseconds"
ts=2023-03-13T09:27:52.289Z caller=klog.go:84 level=debug component=k8s_client_runtime func=Infof msg="GET https://10.1.0.1:443/api/v1/endpoints?limit=500&resourceVersion=0 200 OK in 16 milliseconds"
ts=2023-03-13T09:27:52.292Z caller=klog.go:84 level=debug component=k8s_client_runtime func=Infof msg="GET https://10.1.0.1:443/api/v1/namespaces/default/endpoints?allowWatchBookmarks=true&resourceVersion=262681676&timeout=6m32s&timeoutSeconds=392&watch=true 200 OK in 5 milliseconds"
ts=2023-03-13T09:27:52.292Z caller=klog.go:84 level=debug component=k8s_client_runtime func=Infof msg="GET https://10.1.0.1:443/api/v1/namespaces/default/pods?allowWatchBookmarks=true&resourceVersion=262681633&timeout=7m35s&timeoutSeconds=455&watch=true 200 OK in 4 milliseconds"
ts=2023-03-13T09:27:52.292Z caller=klog.go:84 level=debug component=k8s_client_runtime func=Infof msg="GET https://10.1.0.1:443/api/v1/namespaces/dtg-ops/services?allowWatchBookmarks=true&resourceVersion=262663021&timeout=6m8s&timeoutSeconds=368&watch=true 200 OK in 2 milliseconds"
ts=2023-03-13T09:27:52.292Z caller=klog.go:84 level=debug component=k8s_client_runtime func=Infof msg="GET https://10.1.0.1:443/api/v1/namespaces/dtg-ops/pods?limit=500&resourceVersion=0 200 OK in 21 milliseconds"
ts=2023-03-13T09:27:52.293Z caller=klog.go:84 level=debug component=k8s_client_runtime func=Infof msg="GET https://10.1.0.1:443/api/v1/namespaces/dtg-ops/endpoints?allowWatchBookmarks=true&resourceVersion=262681676&timeout=9m50s&timeoutSeconds=590&watch=true 200 OK in 5 milliseconds"
ts=2023-03-13T09:27:52.293Z caller=klog.go:84 level=debug component=k8s_client_runtime func=Infof msg="GET https://10.1.0.1:443/api/v1/namespaces/default/services?allowWatchBookmarks=true&resourceVersion=262663021&timeout=5m10s&timeoutSeconds=310&watch=true 200 OK in 5 milliseconds"
ts=2023-03-13T09:27:52.293Z caller=klog.go:84 level=debug component=k8s_client_runtime func=Infof msg="GET https://10.1.0.1:443/api/v1/nodes?allowWatchBookmarks=true&resourceVersion=262681226&timeout=7m30s&timeoutSeconds=450&watch=true 200 OK in 1 milliseconds"
ts=2023-03-13T09:27:52.293Z caller=klog.go:84 level=debug component=k8s_client_runtime func=Infof msg="GET https://10.1.0.1:443/api/v1/services?limit=500&resourceVersion=0 200 OK in 22 milliseconds"
ts=2023-03-13T09:27:52.294Z caller=klog.go:84 level=debug component=k8s_client_runtime func=Infof msg="GET https://10.1.0.1:443/api/v1/services?limit=500&resourceVersion=0 200 OK in 22 milliseconds"
ts=2023-03-13T09:27:52.299Z caller=klog.go:84 level=debug component=k8s_client_runtime func=Infof msg="GET https://10.1.0.1:443/api/v1/endpoints?allowWatchBookmarks=true&resourceVersion=262681676&timeout=5m20s&timeoutSeconds=320&watch=true 200 OK in 0 milliseconds"
ts=2023-03-13T09:27:52.300Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/config/prometheus.yml totalDuration=35.984169ms db_storage=7.389µs remote_storage=4.928µs web_handler=1.005µs query_engine=1.489µs scrape=426.009µs scrape_sd=2.70695ms notify=39.764µs notify_sd=348.324µs rules=30.028552ms tracing=25.237µs
ts=2023-03-13T09:27:52.300Z caller=main.go:990 level=info msg="Server is ready to receive web requests."
ts=2023-03-13T09:27:52.300Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..."
ts=2023-03-13T09:27:52.302Z caller=klog.go:84 level=debug component=k8s_client_runtime func=Infof msg="GET https://10.1.0.1:443/api/v1/namespaces/dtg-ops/pods?allowWatchBookmarks=true&resourceVersion=262681633&timeout=9m17s&timeoutSeconds=557&watch=true 200 OK in 0 milliseconds"
ts=2023-03-13T09:27:52.305Z caller=klog.go:84 level=debug component=k8s_client_runtime func=Infof msg="GET https://10.1.0.1:443/api/v1/services?allowWatchBookmarks=true&resourceVersion=262663021&timeout=7m20s&timeoutSeconds=440&watch=true 200 OK in 0 milliseconds"
ts=2023-03-13T09:27:52.306Z caller=klog.go:84 level=debug component=k8s_client_runtime func=Infof msg="GET https://10.1.0.1:443/api/v1/pods?limit=500&resourceVersion=0 200 OK in 34 milliseconds"
ts=2023-03-13T09:27:52.307Z caller=klog.go:84 level=debug component=k8s_client_runtime func=Infof msg="GET https://10.1.0.1:443/api/v1/pods?limit=500&resourceVersion=0 200 OK in 36 milliseconds"
ts=2023-03-13T09:27:52.315Z caller=klog.go:84 level=debug component=k8s_client_runtime func=Infof msg="GET https://10.1.0.1:443/api/v1/services?allowWatchBookmarks=true&resourceVersion=262663021&timeout=9m40s&timeoutSeconds=580&watch=true 200 OK in 0 milliseconds"
ts=2023-03-13T09:27:52.371Z caller=klog.go:55 level=debug component=k8s_client_runtime func=Verbose.Infof msg="caches populated"
ts=2023-03-13T09:27:52.371Z caller=klog.go:55 level=debug component=k8s_client_runtime func=Verbose.Infof msg="caches populated"
ts=2023-03-13T09:27:52.371Z caller=klog.go:55 level=debug component=k8s_client_runtime func=Verbose.Infof msg="caches populated"
ts=2023-03-13T09:27:52.371Z caller=klog.go:55 level=debug component=k8s_client_runtime func=Verbose.Infof msg="caches populated"
ts=2023-03-13T09:27:52.387Z caller=klog.go:84 level=debug component=k8s_client_runtime func=Infof msg="GET https://10.1.0.1:443/api/v1/pods?allowWatchBookmarks=true&resourceVersion=262681633&timeout=6m7s&timeoutSeconds=367&watch=true 200 OK in 1 milliseconds"
ts=2023-03-13T09:27:52.414Z caller=klog.go:84 level=debug component=k8s_client_runtime func=Infof msg="GET https://10.1.0.1:443/api/v1/pods?allowWatchBookmarks=true&resourceVersion=262681633&timeout=7m44s&timeoutSeconds=464&watch=true 200 OK in 1 milliseconds"
ts=2023-03-13T09:27:52.470Z caller=klog.go:55 level=debug component=k8s_client_runtime func=Verbose.Infof msg="caches populated"
ts=2023-03-13T09:27:52.470Z caller=klog.go:55 level=debug component=k8s_client_runtime func=Verbose.Infof msg="caches populated"
```



Discussion / hints:
Could you please provide more details regarding the error message or behavior that you are experiencing in the UI? Also, have you tried using an all-in-one configuration file instead of a separate file for scraping configuration? Please test both and see if the issue persists.

I attempted to reproduce an issue with `scrape_config_files` and `static_configs`, but I was unsuccessful. Additionally, we are testing the release in kubernetes, and it seems to be functioning properly there (http://prombench.prometheus.io/grafana/d/7gmLoNDmz/prombench?orgId=1&var-pr-number=12114).
Image: prom/prometheus:v2.43.0-rc.0
kubernetes: v1.20.4
The scrape_config_files parts I already commented out.
I tried with debug log, But didn't find any hints in logs.
All these not-up target's state were unknown.
I noticed that, within a few seconds when prometheus first started, at least one target were up:
>"scrapeUrl":"https://10.89.79.225:10250/metrics","globalUrl":"https://10.89.79.225:10250/metrics","lastError":"","lastScrape":"2023-03-14T10:41:01.828798393+08:00","lastScrapeDuration":0.049263632,"health":"up",

But a few seconds later, when I refreshed page again, it's health state changed to unknown:
>"scrapeUrl":"https://10.89.79.225:10250/metrics","globalUrl":"https://10.89.79.225:10250/metrics","lastError":"","lastScrape":"0001-01-01T00:00:00Z","lastScrapeDuration":0,"health":"unknown"

It's weird.
> 
> I attempted to reproduce an issue with `scrape_config_files` and `static_configs`, but I was unsuccessful. Additionally, we are testing the release in kubernetes, and it seems to be functioning properly there (http://prombench.prometheus.io/grafana/d/7gmLoNDmz/prombench?orgId=1&var-pr-number=12114).

To reproduce, you should use kubernetes_sd_configs, and the total number of targets matters. When less than 10 targets, it'll be OK. When more than 61 targets (In my case, all related to kubernetes_sd_configs), it up on some targets, then quickly be reset by some unknown reason, and up again, reset again.

After a couple of git bisect steps, it said:

> 5cfe759348d227a703afa6b68e14f78ba328b742 is the first bad commit
commit 5cfe759348d227a703afa6b68e14f78ba328b742
Author: Bryan Boreham <bjboreham@gmail.com>
Date:   Tue Feb 28 19:35:31 2023 +0000

    scrape: make TargetsFromGroup work with Builder not []Label
    
    Save work converting to `Labels` then to `Builder`.
    `PopulateLabels()` now takes as Builder as input.
    
    Signed-off-by: Bryan Boreham <bjboreham@gmail.com>


cc @bboreham any idea here?
> cc @bboreham any idea here?

In `func (sp *scrapePool) sync(targets []*Target)`,  this hash value `hash := t.hash()` changed during every call.
Nothing comes to mind so far, sorry.
It sounds like some slice is getting overwritten due to re-use, perhaps, but I can't spot that in the code.

Also I have been running a build of Prometheus with that change in dev for 2 weeks. Maybe it depends on exactly how SD is working to trigger the bug.

@ouyangde any chance you can take your understanding of the problem and adjust one of the unit-test cases so it fails in the same way?
@bboreham I only noticed that the hash value changes, and lack a deep understanding of the code. A test case about this is easy to made, but may not be the root cause. 
A running instance works well because it scrapes too quickly before the targets being reset. 
@bboreham It's harder than I imagined to write a test case, since I haven't coded in golang.
But now I can explain what the problem is.
In `PopulateLabels` these lines try to delete Meta labels:

> 	lb.Range(func(l labels.Label) {
		if strings.HasPrefix(l.Name, model.MetaLabelPrefix) {
			lb.Del(l.Name)
		}
	})

But the delete operation is in a loop, which caused that some of meta labels not get deleted.
In kubernetes, there are some meta labels changes between discovery-reload, so the hash changed accordingly.

Your commit changed
> lset := labels.New(lbls...)
lb := labels.NewBuilder(lset)

to
>lb := labels.NewBuilder(labels.EmptyLabels())
lb.Set(...)

which made the difference.

The Del function only changed `b.add`, keep `b.base` unchanged.
Since `labels.NewBuilder(lset)` store labels in `b.base`, it works well.
And your methods store labels in `b.add`. So...


OK so this is a bug in `labels.Builder` ?

I don't quite get what you are saying.  Can you give an example in terms of the labels added and deleted?
I think I see a bug in `labels.Builder.Range()`.  But it would still be useful to have your example.

Fix patch:
diff --git a/model/labels/labels.go b/model/labels/labels.go
index 5e06e3b8da4..6de001c3ce1 100644
--- a/model/labels/labels.go
+++ b/model/labels/labels.go
@@ -545,9 +545,12 @@ func (b *Builder) Get(n string) string {
 }
 
 // Range calls f on each label in the Builder.
-// If f calls Set or Del on b then this may affect what callbacks subsequently happen.
 func (b *Builder) Range(f func(l Label)) {
-	origAdd, origDel := b.add, b.del
+	// Stack-based arrays to avoid heap allocation in most cases.
+	var addStack [1024]Label
+	var delStack [1024]string
+	// Take a copy of add and del, so they are unaffected by calls to Set() or Del().
+	origAdd, origDel := append(addStack[:0], b.add...), append(delStack[:0], b.del...)
 	b.base.Range(func(l Label) {
 		if !slices.Contains(origDel, l.Name) && !contains(origAdd, l.Name) {
 			f(l)
diff --git a/model/labels/labels_string.go b/model/labels/labels_string.go
index 6fe14bedc26..98db29d2540 100644
--- a/model/labels/labels_string.go
+++ b/model/labels/labels_string.go
@@ -599,9 +599,12 @@ func (b *Builder) Get(n string) string {
 }
 
 // Range calls f on each label in the Builder.
-// If f calls Set or Del on b then this may affect what callbacks subsequently happen.
 func (b *Builder) Range(f func(l Label)) {
-	origAdd, origDel := b.add, b.del
+	// Stack-based arrays to avoid heap allocation in most cases.
+	var addStack [1024]Label
+	var delStack [1024]string
+	// Take a copy of add and del, so they are unaffected by calls to Set() or Del().
+	origAdd, origDel := append(addStack[:0], b.add...), append(delStack[:0], b.del...)
 	b.base.Range(func(l Label) {
 		if !slices.Contains(origDel, l.Name) && !contains(origAdd, l.Name) {
 			f(l)


Test patch:
diff --git a/model/labels/labels_test.go b/model/labels/labels_test.go
index 4832be33758..588a84b9844 100644
--- a/model/labels/labels_test.go
+++ b/model/labels/labels_test.go
@@ -529,6 +529,11 @@ func TestBuilder(t *testing.T) {
 			base: FromStrings("aaa", "111"),
 			want: FromStrings("aaa", "111"),
 		},
+		{
+			base: FromStrings("aaa", "111", "bbb", "222", "ccc", "333"),
+			set:  []Label{{"aaa", "444"}, {"bbb", "555"}, {"ccc", "666"}},
+			want: FromStrings("aaa", "444", "bbb", "555", "ccc", "666"),
+		},
 		{
 			base: FromStrings("aaa", "111", "bbb", "222", "ccc", "333"),
 			del:  []string{"bbb"},
@@ -591,7 +596,15 @@ func TestBuilder(t *testing.T) {
 				b.Keep(tcase.keep...)
 			}
 			b.Del(tcase.del...)
-			require.Equal(t, tcase.want, b.Labels(tcase.base))
+			require.Equal(t, tcase.want, b.Labels(EmptyLabels()))
+
+			// Check what happens when we call Range and mutate the builder.
+			b.Range(func(l Label) {
+				if l.Name == "aaa" || l.Name == "bbb" {
+					b.Del(l.Name)
+				}
+			})
+			require.Equal(t, tcase.want.BytesWithoutLabels(nil, "aaa", "bbb"), b.Labels(tcase.base).Bytes(nil))
 		})
 	}
 }