Repository: VictoriaMetrics/VictoriaMetrics
Pull request #2173 (2022-02-10 17:09:32)

fixes service discovery for kubernetes
now it must take in account all pods that belong to the discovered endpoint and endpointslice
adds simple test for endpoints
https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2134


Patch:
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index bce8f8c6d..baf134e92 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -38,6 +38,7 @@ The following tip changes can be tested by building VictoriaMetrics components f
 * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): fix possible UI freeze after querying `node_uname_info` time series. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2115).
 * BUGFIX: show the original location of the warning or error message when logging throttled messages. Previously the location inside `lib/logger/throttler.go` was shown. This could increase the complexity of debugging.
 * BUGFIX: vmalert: fix links at web UI. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2167).
+* BUGFIX: vmagent: properly discover pods without exposed ports for the given service for `role: endpoints` and `role: endpointslice` in [kubernetes_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2134).
 
 
 ## [v1.72.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.72.0)
diff --git a/lib/promscrape/discovery/kubernetes/endpoints.go b/lib/promscrape/discovery/kubernetes/endpoints.go
index 8c1fe6b9d..a9ddc6ecc 100644
--- a/lib/promscrape/discovery/kubernetes/endpoints.go
+++ b/lib/promscrape/discovery/kubernetes/endpoints.go
@@ -169,11 +169,16 @@ func getEndpointLabelsForAddressAndPort(podPortsSeen map[*Pod][]int, eps *Endpoi
 		return m
 	}
 	p.appendCommonLabels(m)
+	// always add pod targetRef, even if epp port doesn't match container port
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2134
+	if _, ok := podPortsSeen[p]; !ok {
+		podPortsSeen[p] = []int{}
+	}
 	for _, c := range p.Spec.Containers {
 		for _, cp := range c.Ports {
 			if cp.ContainerPort == epp.Port {
-				p.appendContainerLabels(m, c, &cp)
 				podPortsSeen[p] = append(podPortsSeen[p], cp.ContainerPort)
+				p.appendContainerLabels(m, c, &cp)
 				break
 			}
 		}
diff --git a/lib/promscrape/discovery/kubernetes/endpointslice.go b/lib/promscrape/discovery/kubernetes/endpointslice.go
index 37053de79..727edd2fc 100644
--- a/lib/promscrape/discovery/kubernetes/endpointslice.go
+++ b/lib/promscrape/discovery/kubernetes/endpointslice.go
@@ -101,12 +101,17 @@ func getEndpointSliceLabelsForAddressAndPort(podPortsSeen map[*Pod][]int, addr s
 	if ea.TargetRef.Kind != "Pod" || p == nil {
 		return m
 	}
+	// always add pod targetRef, even if epp port doesn't match container port.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2134
+	if _, ok := podPortsSeen[p]; !ok {
+		podPortsSeen[p] = []int{}
+	}
 	p.appendCommonLabels(m)
 	for _, c := range p.Spec.Containers {
 		for _, cp := range c.Ports {
 			if cp.ContainerPort == epp.Port {
-				p.appendContainerLabels(m, c, &cp)
 				podPortsSeen[p] = append(podPortsSeen[p], cp.ContainerPort)
+				p.appendContainerLabels(m, c, &cp)
 				break
 			}
 		}
@@ -117,7 +122,6 @@ func getEndpointSliceLabelsForAddressAndPort(podPortsSeen map[*Pod][]int, addr s
 
 // //getEndpointSliceLabels builds labels for given EndpointSlice
 func getEndpointSliceLabels(eps *EndpointSlice, addr string, ea Endpoint, epp EndpointPort) map[string]string {
-
 	addr = discoveryutils.JoinHostPort(addr, epp.Port)
 	m := map[string]string{
 		"__address__":                                               addr,


Test patch:
diff --git a/lib/promscrape/discovery/kubernetes/endpoints_test.go b/lib/promscrape/discovery/kubernetes/endpoints_test.go
index 51b30d091..f26fe405c 100644
--- a/lib/promscrape/discovery/kubernetes/endpoints_test.go
+++ b/lib/promscrape/discovery/kubernetes/endpoints_test.go
@@ -113,3 +113,203 @@ func TestParseEndpointsListSuccess(t *testing.T) {
 		t.Fatalf("unexpected labels:\ngot\n%v\nwant\n%v", sortedLabelss, expectedLabelss)
 	}
 }
+
+func TestGetEndpointLabels(t *testing.T) {
+	type testArgs struct {
+		containerPorts map[string][]ContainerPort
+		endpointPorts  []EndpointPort
+	}
+	f := func(name string, args testArgs, wantLabels [][]prompbmarshal.Label) {
+		t.Run(name, func(t *testing.T) {
+			eps := Endpoints{
+				Metadata: ObjectMeta{
+					Name:      "test-eps",
+					Namespace: "default",
+				},
+				Subsets: []EndpointSubset{
+					{
+						Ports: args.endpointPorts,
+						Addresses: []EndpointAddress{
+							{
+								IP: "10.13.15.15",
+								TargetRef: ObjectReference{
+									Kind:      "Pod",
+									Namespace: "default",
+									Name:      "test-pod",
+								},
+							},
+						},
+					},
+				},
+			}
+			svc := Service{
+				Metadata: ObjectMeta{
+					Name:      "test-eps",
+					Namespace: "default",
+				},
+				Spec: ServiceSpec{
+					Ports: []ServicePort{
+						{
+							Name: "test-port",
+							Port: 8081,
+						},
+					},
+				},
+			}
+			pod := Pod{
+				Metadata: ObjectMeta{
+					Name:      "test-pod",
+					Namespace: "default",
+				},
+				Status: PodStatus{PodIP: "192.168.15.1"},
+			}
+			for cn, ports := range args.containerPorts {
+				pod.Spec.Containers = append(pod.Spec.Containers, Container{Name: cn, Ports: ports})
+			}
+			var gw groupWatcher
+			gw.m = map[string]*urlWatcher{
+				"pod": &urlWatcher{
+					role: "pod",
+					objectsByKey: map[string]object{
+						"default/test-pod": &pod,
+					},
+				},
+				"service": &urlWatcher{
+					role: "service",
+					objectsByKey: map[string]object{
+						"default/test-eps": &svc,
+					},
+				},
+			}
+			var sortedLabelss [][]prompbmarshal.Label
+			gotLabels := eps.getTargetLabels(&gw)
+			for _, lbs := range gotLabels {
+				sortedLabelss = append(sortedLabelss, discoveryutils.GetSortedLabels(lbs))
+			}
+			if !areEqualLabelss(sortedLabelss, wantLabels) {
+				t.Fatalf("unexpected labels:\ngot\n%v\nwant\n%v", sortedLabelss, wantLabels)
+
+			}
+		})
+	}
+
+	f("1 port from endpoint", testArgs{
+		endpointPorts: []EndpointPort{
+			{
+				Name: "web",
+				Port: 8081,
+			},
+		},
+	}, [][]prompbmarshal.Label{
+		discoveryutils.GetSortedLabels(map[string]string{
+			"__address__": "10.13.15.15:8081",
+			"__meta_kubernetes_endpoint_address_target_kind": "Pod",
+			"__meta_kubernetes_endpoint_address_target_name": "test-pod",
+			"__meta_kubernetes_endpoint_port_name":           "web",
+			"__meta_kubernetes_endpoint_port_protocol":       "",
+			"__meta_kubernetes_endpoint_ready":               "true",
+			"__meta_kubernetes_endpoints_name":               "test-eps",
+			"__meta_kubernetes_namespace":                    "default",
+			"__meta_kubernetes_pod_host_ip":                  "",
+			"__meta_kubernetes_pod_ip":                       "192.168.15.1",
+			"__meta_kubernetes_pod_name":                     "test-pod",
+			"__meta_kubernetes_pod_node_name":                "",
+			"__meta_kubernetes_pod_phase":                    "",
+			"__meta_kubernetes_pod_ready":                    "unknown",
+			"__meta_kubernetes_pod_uid":                      "",
+			"__meta_kubernetes_service_cluster_ip":           "",
+			"__meta_kubernetes_service_name":                 "test-eps",
+			"__meta_kubernetes_service_type":                 "",
+		}),
+	})
+
+	f("1 port from endpoint and 1 from pod", testArgs{
+		containerPorts: map[string][]ContainerPort{"metrics": []ContainerPort{{
+			Name:          "http-metrics",
+			ContainerPort: 8428,
+		}}},
+		endpointPorts: []EndpointPort{
+			{
+				Name: "web",
+				Port: 8081,
+			},
+		},
+	}, [][]prompbmarshal.Label{
+		discoveryutils.GetSortedLabels(map[string]string{
+			"__address__": "10.13.15.15:8081",
+			"__meta_kubernetes_endpoint_address_target_kind": "Pod",
+			"__meta_kubernetes_endpoint_address_target_name": "test-pod",
+			"__meta_kubernetes_endpoint_port_name":           "web",
+			"__meta_kubernetes_endpoint_port_protocol":       "",
+			"__meta_kubernetes_endpoint_ready":               "true",
+			"__meta_kubernetes_endpoints_name":               "test-eps",
+			"__meta_kubernetes_namespace":                    "default",
+			"__meta_kubernetes_pod_host_ip":                  "",
+			"__meta_kubernetes_pod_ip":                       "192.168.15.1",
+			"__meta_kubernetes_pod_name":                     "test-pod",
+			"__meta_kubernetes_pod_node_name":                "",
+			"__meta_kubernetes_pod_phase":                    "",
+			"__meta_kubernetes_pod_ready":                    "unknown",
+			"__meta_kubernetes_pod_uid":                      "",
+			"__meta_kubernetes_service_cluster_ip":           "",
+			"__meta_kubernetes_service_name":                 "test-eps",
+			"__meta_kubernetes_service_type":                 "",
+		}),
+		discoveryutils.GetSortedLabels(map[string]string{
+			"__address__":                                   "192.168.15.1:8428",
+			"__meta_kubernetes_namespace":                   "default",
+			"__meta_kubernetes_pod_container_name":          "metrics",
+			"__meta_kubernetes_pod_container_port_name":     "http-metrics",
+			"__meta_kubernetes_pod_container_port_number":   "8428",
+			"__meta_kubernetes_pod_container_port_protocol": "",
+			"__meta_kubernetes_pod_host_ip":                 "",
+			"__meta_kubernetes_pod_ip":                      "192.168.15.1",
+			"__meta_kubernetes_pod_name":                    "test-pod",
+			"__meta_kubernetes_pod_node_name":               "",
+			"__meta_kubernetes_pod_phase":                   "",
+			"__meta_kubernetes_pod_ready":                   "unknown",
+			"__meta_kubernetes_pod_uid":                     "",
+			"__meta_kubernetes_service_cluster_ip":          "",
+			"__meta_kubernetes_service_name":                "test-eps",
+			"__meta_kubernetes_service_type":                "",
+		}),
+	})
+
+	f("1 port from endpoint", testArgs{
+		containerPorts: map[string][]ContainerPort{"metrics": []ContainerPort{{
+			Name:          "web",
+			ContainerPort: 8428,
+		}}},
+		endpointPorts: []EndpointPort{
+			{
+				Name: "web",
+				Port: 8428,
+			},
+		},
+	}, [][]prompbmarshal.Label{
+		discoveryutils.GetSortedLabels(map[string]string{
+			"__address__": "10.13.15.15:8428",
+			"__meta_kubernetes_endpoint_address_target_kind": "Pod",
+			"__meta_kubernetes_endpoint_address_target_name": "test-pod",
+			"__meta_kubernetes_endpoint_port_name":           "web",
+			"__meta_kubernetes_endpoint_port_protocol":       "",
+			"__meta_kubernetes_endpoint_ready":               "true",
+			"__meta_kubernetes_endpoints_name":               "test-eps",
+			"__meta_kubernetes_namespace":                    "default",
+			"__meta_kubernetes_pod_container_name":           "metrics",
+			"__meta_kubernetes_pod_container_port_name":      "web",
+			"__meta_kubernetes_pod_container_port_number":    "8428",
+			"__meta_kubernetes_pod_container_port_protocol":  "",
+			"__meta_kubernetes_pod_host_ip":                  "",
+			"__meta_kubernetes_pod_ip":                       "192.168.15.1",
+			"__meta_kubernetes_pod_name":                     "test-pod",
+			"__meta_kubernetes_pod_node_name":                "",
+			"__meta_kubernetes_pod_phase":                    "",
+			"__meta_kubernetes_pod_ready":                    "unknown",
+			"__meta_kubernetes_pod_uid":                      "",
+			"__meta_kubernetes_service_cluster_ip":           "",
+			"__meta_kubernetes_service_name":                 "test-eps",
+			"__meta_kubernetes_service_type":                 "",
+		}),
+	})
+}