Repository: kumahq/kuma (2026-03-20T10:14:20Z)

Issue:
feat(dns): add kuma_workload label to DNS metrics
## Motivation

Kuma DNS metrics (`kuma_dp_dns_request_duration_seconds`, `kuma_dp_dns_upstream_request_duration_seconds`) lack `kuma_workload` and `k8s_kuma_io_namespace` labels that are present on all Envoy metrics.

This forces dashboard queries to use expensive cross-metric joins (`* on(pod) group_left() envoy_server_live{kuma_workload="$workload"}`) to correlate DNS metrics with a workload.

## Implementation information

DNS metrics are emitted by `kuma-dp` process, not Envoy, so they don't go through the same labeling pipeline. The fix should add `kuma_workload`, `k8s_kuma_io_namespace`, `mesh`, and `kuma_io_zone` labels to DNS metrics at the source, matching Envoy metric labels.

> Changelog: skip


Discussion:

Patch:
diff --git a/app/kuma-dp/pkg/dataplane/dnsproxy/component.go b/app/kuma-dp/pkg/dataplane/dnsproxy/component.go
index 495769a04a9a..2161c696911e 100644
--- a/app/kuma-dp/pkg/dataplane/dnsproxy/component.go
+++ b/app/kuma-dp/pkg/dataplane/dnsproxy/component.go
@@ -7,10 +7,12 @@ import (
 	"io"
 	"net"
 	"strings"
+	"sync"
 	"sync/atomic"
 	"time"
 
 	"github.com/miekg/dns"
+	"github.com/prometheus/client_golang/prometheus"
 
 	"github.com/kumahq/kuma/v2/pkg/core"
 	"github.com/kumahq/kuma/v2/pkg/core/runtime/component"
@@ -24,7 +26,9 @@ type Server struct {
 	address       string
 	componentDone chan struct{}
 	dnsMap        atomic.Pointer[dnsMap]
-	metrics       *metrics
+	metrics       atomic.Pointer[metrics]
+	registerer    prometheus.Registerer
+	metricsOnce   sync.Once
 	// upstreamClient is used for testing purposes
 	upstreamClient func(msg *dns.Msg) (*dns.Msg, error)
 	ready          chan struct{}
@@ -50,16 +54,27 @@ func NewServer(address string) (*Server, error) {
 	return NewServerWithCustomClient(address, handler), nil
 }
 
+type ServerOption func(*Server)
+
+func WithRegisterer(r prometheus.Registerer) ServerOption {
+	return func(s *Server) {
+		s.registerer = r
+	}
+}
+
 // NewServerWithCustomClient is used for testing purposes
-func NewServerWithCustomClient(address string, upstreamClient func(msg *dns.Msg) (*dns.Msg, error)) *Server {
+func NewServerWithCustomClient(address string, upstreamClient func(msg *dns.Msg) (*dns.Msg, error), opts ...ServerOption) *Server {
 	s := Server{
 		address:        address,
 		componentDone:  make(chan struct{}),
 		dnsMap:         atomic.Pointer[dnsMap]{},
 		ready:          make(chan struct{}),
-		metrics:        newMetrics(),
+		registerer:     prometheus.DefaultRegisterer,
 		upstreamClient: upstreamClient,
 	}
+	for _, opt := range opts {
+		opt(&s)
+	}
 	s.dnsMap.Store(&dnsMap{ARecords: make(map[string]*dnsEntry), AAAARecords: make(map[string]*dnsEntry)})
 	return &s
 }
@@ -79,7 +94,9 @@ var _ component.GracefulComponent = &Server{}
 func (s *Server) Handler(res dns.ResponseWriter, req *dns.Msg) {
 	start := time.Now()
 	defer func() {
-		s.metrics.RequestDuration.Observe(time.Since(start).Seconds())
+		if m := s.metrics.Load(); m != nil {
+			m.RequestDuration.Observe(time.Since(start).Seconds())
+		}
 		if err := recover(); err != nil {
 			log.Error(fmt.Errorf("handler panic %v", err), "panic in DNS handler")
 			response := new(dns.Msg)
@@ -113,14 +130,18 @@ func (s *Server) Handler(res dns.ResponseWriter, req *dns.Msg) {
 		proxyStart := time.Now()
 		resp, err := s.upstreamClient(req)
 		if err != nil {
-			s.metrics.UpstreamRequestFailureCount.Inc()
+			if m := s.metrics.Load(); m != nil {
+				m.UpstreamRequestFailureCount.Inc()
+			}
 			log.Error(err, "failed to write message to upstream")
 			response = new(dns.Msg)
 			response.SetRcode(req, dns.RcodeServerFailure)
 		} else {
 			response = resp
 		}
-		s.metrics.UpstreamRequestDuration.Observe(time.Since(proxyStart).Seconds())
+		if m := s.metrics.Load(); m != nil {
+			m.UpstreamRequestDuration.Observe(time.Since(proxyStart).Seconds())
+		}
 	}
 	err := res.WriteMsg(response)
 	if err != nil {
@@ -177,6 +198,12 @@ func (s *Server) ReloadMap(ctx context.Context, reader io.Reader) error {
 	if err != nil {
 		return err
 	}
+	if configuration.ExtraLabels != nil {
+		s.metricsOnce.Do(func() {
+			m := newMetrics(s.registerer, configuration.ExtraLabels)
+			s.metrics.Store(m)
+		})
+	}
 	res := dnsMap{
 		ARecords:    make(map[string]*dnsEntry),
 		AAAARecords: make(map[string]*dnsEntry),
diff --git a/app/kuma-dp/pkg/dataplane/dnsproxy/metrics.go b/app/kuma-dp/pkg/dataplane/dnsproxy/metrics.go
index 56b17b555f31..d3587a10dd47 100644
--- a/app/kuma-dp/pkg/dataplane/dnsproxy/metrics.go
+++ b/app/kuma-dp/pkg/dataplane/dnsproxy/metrics.go
@@ -10,30 +10,28 @@ type metrics struct {
 	UpstreamRequestFailureCount prometheus.Counter
 }
 
-var m *metrics
-
-func newMetrics() *metrics {
-	if m != nil {
-		return m
-	}
+func newMetrics(registerer prometheus.Registerer, constLabels prometheus.Labels) *metrics {
 	upstreamRequestDuration := prometheus.NewHistogram(prometheus.HistogramOpts{
-		Name: "kuma_dp_dns_upstream_request_duration_seconds",
-		Help: "The duration of the proxied requests.",
+		Name:        "kuma_dp_dns_upstream_request_duration_seconds",
+		Help:        "The duration of the proxied requests.",
+		ConstLabels: constLabels,
 	})
-	prometheus.MustRegister(upstreamRequestDuration)
+	registerer.MustRegister(upstreamRequestDuration)
 	requestDuration := prometheus.NewHistogram(prometheus.HistogramOpts{
-		Name: "kuma_dp_dns_request_duration_seconds",
-		Help: "The duration of the request (inclusive of request that use internal DNS map).",
+		Name:        "kuma_dp_dns_request_duration_seconds",
+		Help:        "The duration of the request (inclusive of request that use internal DNS map).",
+		ConstLabels: constLabels,
 	})
-	prometheus.MustRegister(requestDuration)
+	registerer.MustRegister(requestDuration)
 	upstreamRequestFailureCount := prometheus.NewCounter(prometheus.CounterOpts{
-		Name: "kuma_dp_dns_upstream_request_failure_count",
-		Help: "The total number of failed upstream requests.",
+		Name:        "kuma_dp_dns_upstream_request_failure_count",
+		Help:        "The total number of failed upstream requests.",
+		ConstLabels: constLabels,
 	})
-	m = &metrics{
+	registerer.MustRegister(upstreamRequestFailureCount)
+	return &metrics{
 		RequestDuration:             requestDuration,
 		UpstreamRequestDuration:     upstreamRequestDuration,
 		UpstreamRequestFailureCount: upstreamRequestFailureCount,
 	}
-	return m
 }
diff --git a/pkg/dns/dpapi/api.go b/pkg/dns/dpapi/api.go
index b9eb0a261e6c..07a2b6e80e88 100644
--- a/pkg/dns/dpapi/api.go
+++ b/pkg/dns/dpapi/api.go
@@ -11,6 +11,7 @@ type DNSRecord struct {
 }
 
 type DNSProxyConfig struct {
-	Records []DNSRecord `json:"records"`
-	TTL     int         `json:"ttl"`
+	Records     []DNSRecord       `json:"records"`
+	TTL         int               `json:"ttl"`
+	ExtraLabels map[string]string `json:"extraLabels,omitempty"`
 }
diff --git a/pkg/xds/generator/dns_generator.go b/pkg/xds/generator/dns_generator.go
index ef2943021bf5..87005d7b5b8f 100644
--- a/pkg/xds/generator/dns_generator.go
+++ b/pkg/xds/generator/dns_generator.go
@@ -6,11 +6,13 @@ import (
 	"encoding/json"
 	"slices"
 
+	mesh_proto "github.com/kumahq/kuma/v2/api/mesh/v1alpha1"
 	unified_naming "github.com/kumahq/kuma/v2/pkg/core/naming/unified-naming"
 	core_system_names "github.com/kumahq/kuma/v2/pkg/core/system_names"
 	core_xds "github.com/kumahq/kuma/v2/pkg/core/xds"
 	xds_types "github.com/kumahq/kuma/v2/pkg/core/xds/types"
 	"github.com/kumahq/kuma/v2/pkg/dns/dpapi"
+	k8s_metadata "github.com/kumahq/kuma/v2/pkg/plugins/runtime/k8s/metadata"
 	util_net "github.com/kumahq/kuma/v2/pkg/util/net"
 	xds_context "github.com/kumahq/kuma/v2/pkg/xds/context"
 	"github.com/kumahq/kuma/v2/pkg/xds/dynconf"
@@ -53,7 +55,7 @@ func (g DNSGenerator) Generate(_ context.Context, rs *core_xds.ResourceSet, xdsC
 	if proxy.Metadata.HasFeature(xds_types.FeatureEmbeddedDNS) {
 		// This is purposefully set to 30s to avoid DNS cache stale with ExternalService and Kong Gateway see: https://github.com/kumahq/kuma/issues/13353.
 		// https://github.com/kumahq/kuma/issues/13463
-		dnsInfo := dpapi.DNSProxyConfig{TTL: 30, Records: []dpapi.DNSRecord{}}
+		dnsInfo := dpapi.DNSProxyConfig{TTL: 30, Records: []dpapi.DNSRecord{}, ExtraLabels: dnsExtraLabels(proxy)}
 		for name, addresses := range vips {
 			dnsInfo.Records = append(dnsInfo.Records, dpapi.DNSRecord{
 				Name: name,
@@ -102,3 +104,20 @@ func (g DNSGenerator) Generate(_ context.Context, rs *core_xds.ResourceSet, xdsC
 	})
 	return resources, nil
 }
+
+func dnsExtraLabels(proxy *core_xds.Proxy) map[string]string {
+	labels := map[string]string{}
+	dpLabels := proxy.Dataplane.GetMeta().GetLabels()
+
+	if workloadName := dpLabels[k8s_metadata.KumaWorkload]; workloadName != "" {
+		labels["kuma_workload"] = workloadName
+	}
+	if ns := dpLabels[mesh_proto.KubeNamespaceTag]; ns != "" {
+		labels["k8s_kuma_io_namespace"] = ns
+	}
+	labels["mesh"] = proxy.Dataplane.GetMeta().GetMesh()
+	if zone := dpLabels[mesh_proto.ZoneTag]; zone != "" {
+		labels["kuma_io_zone"] = zone
+	}
+	return labels
+}


Test patch:
diff --git a/app/kuma-dp/pkg/dataplane/dnsproxy/component_test.go b/app/kuma-dp/pkg/dataplane/dnsproxy/component_test.go
index 62468d2386a4..60c1b2d1267d 100644
--- a/app/kuma-dp/pkg/dataplane/dnsproxy/component_test.go
+++ b/app/kuma-dp/pkg/dataplane/dnsproxy/component_test.go
@@ -12,6 +12,8 @@ import (
 	"github.com/miekg/dns"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	"github.com/prometheus/client_golang/prometheus"
+	io_prometheus "github.com/prometheus/client_model/go"
 
 	"github.com/kumahq/kuma/v2/app/kuma-dp/pkg/dataplane/dnsproxy"
 	"github.com/kumahq/kuma/v2/pkg/test"
@@ -23,6 +25,7 @@ var _ = Describe("components", func() {
 	var server *dnsproxy.Server
 	var address string
 	var mock atomic.Pointer[func(*dns.Msg) (*dns.Msg, error)]
+	var registry *prometheus.Registry
 
 	BeforeEach(func() {
 		port, err := test.FindFreePort("127.0.0.1")
@@ -31,13 +34,14 @@ var _ = Describe("components", func() {
 		done = make(chan struct{})
 		wg = sync.WaitGroup{}
 		wg.Add(1)
+		registry = prometheus.NewRegistry()
 
 		address = net.JoinHostPort("127.0.0.1", strconv.Itoa(int(port)))
 		server = dnsproxy.NewServerWithCustomClient(address, func(msg *dns.Msg) (*dns.Msg, error) {
 			defer GinkgoRecover()
 			f := *mock.Load()
 			return f(msg)
-		})
+		}, dnsproxy.WithRegisterer(registry))
 
 		go func() {
 			defer GinkgoRecover()
@@ -118,10 +122,11 @@ var _ = Describe("components", func() {
 		}
 		mock.Store(&f)
 
-		_ = server.ReloadMap(context.Background(), bytes.NewBuffer([]byte(`{
+		Expect(server.ReloadMap(context.Background(), bytes.NewBuffer([]byte(`{
 "ttl": 123,
-"records": [{"name": "example.com", "ips": ["240.0.0.1", "::2"]}]
-}`)))
+"records": [{"name": "example.com", "ips": ["240.0.0.1", "::2"]}],
+"extraLabels": {"mesh": "default", "kuma_workload": "backend"}
+}`)))).To(Succeed())
 		c := new(dns.Client)
 
 		By("in the map A")
@@ -151,4 +156,47 @@ var _ = Describe("components", func() {
 		Expect(res).To(HaveField("Rcode", Equal(dns.RcodeSuccess)))
 		Expect(res.Answer[0].String()).To(Equal("foo.com.\t125\tIN\tA\t17.0.0.1"))
 	})
+	It("metrics have extra labels after ReloadMap", func() {
+		f := func(req *dns.Msg) (*dns.Msg, error) {
+			return nil, fmt.Errorf("should not call upstream for local entry")
+		}
+		mock.Store(&f)
+
+		Expect(server.ReloadMap(context.Background(), bytes.NewBuffer([]byte(`{
+"ttl": 123,
+"records": [{"name": "example.com", "ips": ["240.0.0.1"]}],
+"extraLabels": {"mesh": "default", "kuma_workload": "backend", "k8s_kuma_io_namespace": "test-ns"}
+}`)))).To(Succeed())
+
+		msg := &dns.Msg{}
+		msg.SetQuestion("example.com.", dns.TypeA)
+		c := new(dns.Client)
+		_, _, err := c.Exchange(msg, address)
+		Expect(err).ToNot(HaveOccurred())
+
+		families, err := registry.Gather()
+		Expect(err).ToNot(HaveOccurred())
+
+		findMetric := func(name string) *io_prometheus.MetricFamily {
+			for _, mf := range families {
+				if mf.GetName() == name {
+					return mf
+				}
+			}
+			return nil
+		}
+
+		reqDuration := findMetric("kuma_dp_dns_request_duration_seconds")
+		Expect(reqDuration).ToNot(BeNil())
+		labels := reqDuration.GetMetric()[0].GetLabel()
+		labelMap := map[string]string{}
+		for _, l := range labels {
+			labelMap[l.GetName()] = l.GetValue()
+		}
+		Expect(labelMap).To(Equal(map[string]string{
+			"mesh":                  "default",
+			"kuma_workload":         "backend",
+			"k8s_kuma_io_namespace": "test-ns",
+		}))
+	})
 })
diff --git a/pkg/xds/generator/dns_generator_test.go b/pkg/xds/generator/dns_generator_test.go
index 6e185fa68900..6044ebb72d58 100644
--- a/pkg/xds/generator/dns_generator_test.go
+++ b/pkg/xds/generator/dns_generator_test.go
@@ -26,6 +26,8 @@ var _ = Describe("DNSGenerator", func() {
 		expected         string
 		features         map[string]bool
 		meshServicesMode mesh_proto.Mesh_MeshServices_Mode
+		dpLabels         map[string]string
+		dpMesh           string
 	}
 
 	DescribeTable("Generate Envoy xDS resources",
@@ -62,6 +64,8 @@ var _ = Describe("DNSGenerator", func() {
 				Dataplane: &core_mesh.DataplaneResource{
 					Meta: &test_model.ResourceMeta{
 						Version: "1",
+						Mesh:    given.dpMesh,
+						Labels:  given.dpLabels,
 					},
 					Spec: &dataplane,
 				},
@@ -112,6 +116,12 @@ var _ = Describe("DNSGenerator", func() {
 			},
 			dataplaneFile: "4-dataplane.input.yaml",
 			expected:      "4-envoy-config.golden.yaml",
+			dpLabels: map[string]string{
+				"kuma.io/workload":      "backend",
+				"k8s.kuma.io/namespace": "test-ns",
+				"kuma.io/zone":          "zone-1",
+			},
+			dpMesh: "default",
 		}),
 		Entry("05. DNS using proxy with unified naming", testCase{
 			dataplaneFile:    "5-dataplane.input.yaml",
@@ -121,6 +131,12 @@ var _ = Describe("DNSGenerator", func() {
 				"feature-embedded-dns":            true,
 				"feature-unified-resource-naming": true,
 			},
+			dpLabels: map[string]string{
+				"kuma.io/workload":      "backend",
+				"k8s.kuma.io/namespace": "test-ns",
+				"kuma.io/zone":          "zone-1",
+			},
+			dpMesh: "default",
 		}),
 		Entry("06. DNS enabled with unified naming", testCase{
 			dataplaneFile: "6-dataplane.input.yaml",
diff --git a/pkg/xds/generator/testdata/dns/4-envoy-config.golden.yaml b/pkg/xds/generator/testdata/dns/4-envoy-config.golden.yaml
index d568251004da..369e328ac745 100644
--- a/pkg/xds/generator/testdata/dns/4-envoy-config.golden.yaml
+++ b/pkg/xds/generator/testdata/dns/4-envoy-config.golden.yaml
@@ -19,7 +19,7 @@ resources:
             - addressPrefix: 127.0.0.1
               prefixLen: 32
           routeConfig:
-            maxDirectResponseBodySizeBytes: 236
+            maxDirectResponseBodySizeBytes: 353
             virtualHosts:
             - domains:
               - '*'
@@ -31,11 +31,11 @@ resources:
                   headers:
                   - name: If-None-Match
                     stringMatch:
-                      exact: 639772d25f9c22e338f3c54ebe8be0faeae7e30c57f1acbf2d8ba5eb191007fb
+                      exact: 3a7819834364a9832e4a7b58cf52f72bc5b33bb1766e86dfd5cfe5afab562693
                   path: /dns
               - directResponse:
                   body:
-                    inlineString: '{"records":[{"name":"backend.test-ns.svc.8080.mesh","ips":["240.0.0.0","::ffff:f000:0"]},{"name":"backend_test-ns_svc_8080.mesh","ips":["240.0.0.0","::ffff:f000:0"]},{"name":"httpbin.mesh","ips":["240.0.0.1","::ffff:f000:1"]}],"ttl":30}'
+                    inlineString: '{"records":[{"name":"backend.test-ns.svc.8080.mesh","ips":["240.0.0.0","::ffff:f000:0"]},{"name":"backend_test-ns_svc_8080.mesh","ips":["240.0.0.0","::ffff:f000:0"]},{"name":"httpbin.mesh","ips":["240.0.0.1","::ffff:f000:1"]}],"ttl":30,"extraLabels":{"k8s_kuma_io_namespace":"test-ns","kuma_io_zone":"zone-1","kuma_workload":"backend","mesh":"default"}}'
                   status: 200
                 match:
                   path: /dns
@@ -43,6 +43,6 @@ resources:
                 responseHeadersToAdd:
                 - header:
                     key: Etag
-                    value: 639772d25f9c22e338f3c54ebe8be0faeae7e30c57f1acbf2d8ba5eb191007fb
+                    value: 3a7819834364a9832e4a7b58cf52f72bc5b33bb1766e86dfd5cfe5afab562693
           statPrefix: _kuma_dynamicconfig
     name: _kuma:dynamicconfig
diff --git a/pkg/xds/generator/testdata/dns/5-envoy-config.golden.yaml b/pkg/xds/generator/testdata/dns/5-envoy-config.golden.yaml
index 452e1d85b7b1..2e4cfd5afdf9 100644
--- a/pkg/xds/generator/testdata/dns/5-envoy-config.golden.yaml
+++ b/pkg/xds/generator/testdata/dns/5-envoy-config.golden.yaml
@@ -19,7 +19,7 @@ resources:
             - addressPrefix: 127.0.0.1
               prefixLen: 32
           routeConfig:
-            maxDirectResponseBodySizeBytes: 236
+            maxDirectResponseBodySizeBytes: 353
             virtualHosts:
             - domains:
               - '*'
@@ -31,12 +31,12 @@ resources:
                   headers:
                   - name: If-None-Match
                     stringMatch:
-                      exact: 639772d25f9c22e338f3c54ebe8be0faeae7e30c57f1acbf2d8ba5eb191007fb
+                      exact: 3a7819834364a9832e4a7b58cf52f72bc5b33bb1766e86dfd5cfe5afab562693
                   path: /dns
                 name: system_dynamicconfig_dns_not_modified
               - directResponse:
                   body:
-                    inlineString: '{"records":[{"name":"backend.test-ns.svc.8080.mesh","ips":["240.0.0.0","::ffff:f000:0"]},{"name":"backend_test-ns_svc_8080.mesh","ips":["240.0.0.0","::ffff:f000:0"]},{"name":"httpbin.mesh","ips":["240.0.0.1","::ffff:f000:1"]}],"ttl":30}'
+                    inlineString: '{"records":[{"name":"backend.test-ns.svc.8080.mesh","ips":["240.0.0.0","::ffff:f000:0"]},{"name":"backend_test-ns_svc_8080.mesh","ips":["240.0.0.0","::ffff:f000:0"]},{"name":"httpbin.mesh","ips":["240.0.0.1","::ffff:f000:1"]}],"ttl":30,"extraLabels":{"k8s_kuma_io_namespace":"test-ns","kuma_io_zone":"zone-1","kuma_workload":"backend","mesh":"default"}}'
                   status: 200
                 match:
                   path: /dns
@@ -44,7 +44,7 @@ resources:
                 responseHeadersToAdd:
                 - header:
                     key: Etag
-                    value: 639772d25f9c22e338f3c54ebe8be0faeae7e30c57f1acbf2d8ba5eb191007fb
+                    value: 3a7819834364a9832e4a7b58cf52f72bc5b33bb1766e86dfd5cfe5afab562693
           statPrefix: system_dynamicconfig
     name: system_dynamicconfig
     statPrefix: system_dynamicconfig