Repository: VictoriaMetrics/VictoriaMetrics
Pull request #4235 (2023-05-02 07:23:46)

lib/promscrape/discovery/kubernetes: add common labels to all ports discovered from endpoints
Sets `__meta_kubernetes_endpoints_name` and `__meta_kubernetes_namespace` labels to all ports of pod. Prometheus sets those labels to all ports in pod (https://github.com/prometheus/prometheus/blob/0ab95536115adfe50af249d36d73674be694ca3f/discovery/kubernetes/endpoints.go#L267C15-L269) even if port is not matching any service. 
See: #4154


Patch:
diff --git a/deployment/docker/alerts-health.yml b/deployment/docker/alerts-health.yml
index 551e5d1df..63f66c8c2 100644
--- a/deployment/docker/alerts-health.yml
+++ b/deployment/docker/alerts-health.yml
@@ -6,7 +6,7 @@ groups:
     # note the `job` filter and update accordingly to your setup
     rules:
       - alert: TooManyRestarts
-        expr: changes(process_start_time_seconds{job=~"victoriametrics.*|vmselect.*|vminsert.*|vmstorage.*|vmagent.*|vmalert.*"}[15m]) > 2
+        expr: changes(process_start_time_seconds{job=~"victoriametrics.*|vmselect.*|vminsert.*|vmstorage.*|vmagent.*|vmalert.*|vmsingle.*"}[15m]) > 2
         labels:
           severity: critical
         annotations:
@@ -15,7 +15,7 @@ groups:
             It might be crashlooping."
 
       - alert: ServiceDown
-        expr: up{job=~"victoriametrics.*|vmselect.*|vminsert.*|vmstorage.*|vmagent.*|vmalert.*"} == 0
+        expr: up{job=~"victoriametrics.*|vmselect.*|vminsert.*|vmstorage.*|vmagent.*|vmalert.*|vmsingle.*"} == 0
         for: 2m
         labels:
           severity: critical
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index ca4a4e76d..6e3601a5c 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -26,7 +26,6 @@ The following tip changes can be tested by building VictoriaMetrics components f
 
 * SECURITY: serve `/robots.txt` content to disallow indexing of the exposed instances by search engines. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4128) for details.
 
-* FEATURE: open source [Graphite Render API](https://docs.victoriametrics.com/#graphite-render-api-usage). This API allows using VictoriaMetrics as a drop-in replacement for Graphite at both data ingestion and querying sides and reducing infrastructure costs by up to 10x comparing to Graphite. See [this case study](https://docs.victoriametrics.com/CaseStudies.html#grammarly) as an example.
 * FEATURE: [vmbackup](https://docs.victoriametrics.com/vmbackup.html): store backup creation and completion time in `backup_complete.ignore` file of backup contents. This is useful to determine point in time when backup was created and completed.
 * FEATURE: [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager.html): add `created_at` field to the output of `/api/v1/backups` API and `vmbackupmanager backup list` command. See this [doc](https://docs.victoriametrics.com/vmbackupmanager.html#api-methods) for data format details.
 * FEATURE: deprecate `-bigMergeConcurrency` command-line flag, since improper configuration for this flag frequently led to uncontrolled growth of unmerged parts, which, in turn, could lead to queries slowdown and increased CPU usage. The concurrency for [background merges](https://docs.victoriametrics.com/#storage) can be controlled via `-smallMergeConcurrency` command-line flag, though it isn't recommended to do in general case.
@@ -48,7 +47,8 @@ The following tip changes can be tested by building VictoriaMetrics components f
 * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): fix the display of the tenant selector. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4160).
 * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): fix issue where vmui would freeze when adding a query that returned regular rows alongside a heatmap query.
 * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): properly display an error when using `query` function for templating value of `-external.alert.source` flag. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4181).
-* BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): properly return empty slices instead of nil for `/api/v1/rules` and `/api/v1/alerts` API handlers. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4221). 
+* BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): properly return empty slices instead of nil for `/api/v1/rules` and `/api/v1/alerts` API handlers. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4221).
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): add `__meta_kubernetes_endpoints_name` label for all ports discovered from endpoint. Previously, ports not matched by `Service` did not have this label. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4154) for details.
 
 ## [v1.90.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.90.0)
 
diff --git a/lib/promscrape/discovery/kubernetes/endpoints.go b/lib/promscrape/discovery/kubernetes/endpoints.go
index f83526ed0..25497f4f0 100644
--- a/lib/promscrape/discovery/kubernetes/endpoints.go
+++ b/lib/promscrape/discovery/kubernetes/endpoints.go
@@ -114,6 +114,10 @@ func (eps *Endpoints) getTargetLabels(gw *groupWatcher) []*promutils.Labels {
 		logger.Warnf(`the number of targets for "role: endpoints" %q exceeds 1000 and will be truncated in the next k8s releases; please use "role: endpointslice" instead`, eps.Metadata.key())
 	}
 
+	// Prometheus sets endpoints_name and namespace labels for all endpoints
+	// Even if port is not matching service port.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4154
+	commonEpLabels := eps.getCommonLabels()
 	// Append labels for skipped ports on seen pods.
 	portSeen := func(port int, ports []int) bool {
 		for _, p := range ports {
@@ -134,6 +138,7 @@ func (eps *Endpoints) getTargetLabels(gw *groupWatcher) []*promutils.Labels {
 				m.Add("__address__", addr)
 				p.appendCommonLabels(m, gw)
 				p.appendContainerLabels(m, c, &cp)
+				m.AddFrom(commonEpLabels)
 				if svc != nil {
 					svc.appendCommonLabels(m)
 				}
@@ -146,6 +151,13 @@ func (eps *Endpoints) getTargetLabels(gw *groupWatcher) []*promutils.Labels {
 	return ms
 }
 
+func (eps *Endpoints) getCommonLabels() *promutils.Labels {
+	m := promutils.GetLabels()
+	m.Add("__meta_kubernetes_namespace", eps.Metadata.Namespace)
+	m.Add("__meta_kubernetes_endpoints_name", eps.Metadata.Name)
+	return m
+}
+
 func appendEndpointLabelsForAddresses(ms []*promutils.Labels, gw *groupWatcher, podPortsSeen map[*Pod][]int, eps *Endpoints,
 	eas []EndpointAddress, epp EndpointPort, svc *Service, ready string) []*promutils.Labels {
 	for _, ea := range eas {


Test patch:
diff --git a/lib/promscrape/discovery/kubernetes/endpoints_test.go b/lib/promscrape/discovery/kubernetes/endpoints_test.go
index 402bb18ed..bb6b9e8ac 100644
--- a/lib/promscrape/discovery/kubernetes/endpoints_test.go
+++ b/lib/promscrape/discovery/kubernetes/endpoints_test.go
@@ -293,6 +293,7 @@ func TestGetEndpointsLabels(t *testing.T) {
 			}),
 			promutils.NewLabelsFromMap(map[string]string{
 				"__address__":                                    "192.168.15.1:8428",
+				"__meta_kubernetes_endpoints_name":               "test-eps",
 				"__meta_kubernetes_namespace":                    "default",
 				"__meta_kubernetes_node_label_node_label":        "xyz",
 				"__meta_kubernetes_node_labelpresent_node_label": "true",