Repository: honeycombio/honeycomb-kubernetes-agent
Pull request #200 (2021-12-22 00:40:41)

add node metadata to metrics events
## Which problem is this PR solving?

- When viewing metrics for our cluster, we'd like to see node metadata such as availability zone, instance type, and node group. This will give us a better perspective of what's happening as pods are scheduled onto nodes or nodes are drained.

## Short description of the changes

- Adds `includeNodeLabels` flag to the metrics config to determine whether or not node metadata is added to metric events. Node labels will respect the `OmitLabels` list. Node metadata is pulled from the Kubernetes API.


asdvalenzuela: I made Dean's change and I think this is ready to go! Thanks, @MikeGoldsmith!

Patch:
diff --git a/.circleci/config.yml b/.circleci/config.yml
index 8af2f32..5c4da56 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -18,7 +18,7 @@ jobs:
       - run: go install github.com/google/ko@latest
       - run: echo "VERSION=$(cat version.txt | tr -d '\n')" >> $BASH_ENV
       - run: ./build/build.sh
-      - run: ./build/test.sh
+      - run: ./build/test.sh .
       - run: e2e-tests/test.sh
       - run:
           name: "Deploy on tagged master push"
diff --git a/config/config.go b/config/config.go
index 56053f0..0e7105d 100644
--- a/config/config.go
+++ b/config/config.go
@@ -1,10 +1,11 @@
 package config
 
 import (
-	"github.com/honeycombio/honeycomb-kubernetes-agent/metrics"
 	"io/ioutil"
 	"time"
 
+	"github.com/honeycombio/honeycomb-kubernetes-agent/metrics"
+
 	yaml "gopkg.in/yaml.v2"
 )
 
@@ -49,6 +50,9 @@ type MetricsConfig struct {
 	// Labels to omit from becoming fields in Honeycomb
 	// By default `controller-revision-hash` is omitted
 	OmitLabels []metrics.OmitLabel `yaml:"omitLabels"`
+	// Include node metadata such as 'node.kubernetes.io/instance-type' or
+	// 'topology.kubernetes.io/region'
+	IncludeNodeLabels bool `yaml:"includeNodeLabels"`
 	// MetricGroupsToCollect provides a list of metrics groups to collect metrics from.
 	// "container", "pod", "node" and "volume" are the only valid groups.
 	MetricGroups     []metrics.MetricGroup  `yaml:"metricGroups"`
diff --git a/main.go b/main.go
index cf10722..f9f3411 100644
--- a/main.go
+++ b/main.go
@@ -207,6 +207,11 @@ func createLogTailers(config *config.Config) ([]*tailer.PathWatcher, []*podtaile
 func startMetricsService(config *config.MetricsConfig) error {
 	if config.Enabled {
 
+		kubeClient, err := newKubeClient()
+		if err != nil {
+			logrus.WithError(err).Fatal("Error instantiating kube client")
+		}
+
 		if config.Interval == 0 {
 			config.Interval = 10 * time.Second
 		}
@@ -227,7 +232,7 @@ func startMetricsService(config *config.MetricsConfig) error {
 
 		logger := logrus.StandardLogger()
 
-		svc, err := service.NewMetricsService(config, builder, logger)
+		svc, err := service.NewMetricsService(config, builder, logger, kubeClient)
 		if err != nil {
 			return err
 		}
diff --git a/metrics/metadata.go b/metrics/metadata.go
index 392fba0..542911f 100644
--- a/metrics/metadata.go
+++ b/metrics/metadata.go
@@ -2,28 +2,46 @@ package metrics
 
 import (
 	"errors"
+	"strconv"
+
 	"github.com/sirupsen/logrus"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
-	"strconv"
 )
 
 type OmitLabel string
 
 type Metadata struct {
-	PodsMetadata *v1.PodList
-	OmitLabels   []OmitLabel
-	logger       *logrus.Logger
+	PodsMetadata      *v1.PodList
+	NodesMetadata     *v1.NodeList
+	OmitLabels        []OmitLabel
+	IncludeNodeLabels bool
+	logger            *logrus.Logger
 }
 
-func NewMetadata(podsMetadata *v1.PodList, omitLabels []OmitLabel, logger *logrus.Logger) *Metadata {
+func NewMetadata(podsMetadata *v1.PodList, nodesMetadata *v1.NodeList, omitLabels []OmitLabel, includeNodeLabels bool, logger *logrus.Logger) *Metadata {
 	return &Metadata{
-		PodsMetadata: podsMetadata,
-		OmitLabels:   omitLabels,
-		logger:       logger,
+		PodsMetadata:      podsMetadata,
+		NodesMetadata:     nodesMetadata,
+		OmitLabels:        omitLabels,
+		IncludeNodeLabels: includeNodeLabels,
+		logger:            logger,
 	}
 }
 
+func (m *Metadata) GetNodeMetadataByName(name string) (*NodeMetadata, error) {
+	for _, n := range m.NodesMetadata.Items {
+		if n.ObjectMeta.Name == name {
+			return &NodeMetadata{m, &n}, nil
+		}
+	}
+
+	m.logger.WithFields(logrus.Fields{
+		"nodeName": name,
+	}).Error("Metadata: Node not found")
+	return &NodeMetadata{}, errors.New("node not found")
+}
+
 func (m *Metadata) GetPodMetadataByUid(uid types.UID) (*PodMetadata, error) {
 	for _, p := range m.PodsMetadata.Items {
 		if p.UID == uid {
@@ -37,6 +55,28 @@ func (m *Metadata) GetPodMetadataByUid(uid types.UID) (*PodMetadata, error) {
 	return &PodMetadata{}, errors.New("pod not found")
 }
 
+type NodeMetadata struct {
+	Metadata *Metadata
+	Node     *v1.Node
+}
+
+func (n *NodeMetadata) GetLabels() map[string]string {
+	labels := make(map[string]string)
+	for k, v := range n.Node.Labels {
+		keep := true
+		for _, o := range n.Metadata.OmitLabels {
+			if o == OmitLabel(k) {
+				keep = false
+				break
+			}
+		}
+		if keep {
+			labels[k] = v
+		}
+	}
+	return labels
+}
+
 type PodMetadata struct {
 	Metadata *Metadata
 	Pod      *v1.Pod
diff --git a/metrics/processor.go b/metrics/processor.go
index f356ff8..18f81a7 100644
--- a/metrics/processor.go
+++ b/metrics/processor.go
@@ -33,7 +33,7 @@ func (p *Processor) GenerateMetricsData(summary *stats.Summary, metadata *Metada
 		logger:                p.logger,
 	}
 
-	nodeResource := getNodeResource(summary.Node)
+	nodeResource := getNodeResource(summary.Node, metadata)
 	acc.nodeStats(nodeResource, summary.Node)
 
 	for _, podStats := range summary.Pods {
diff --git a/metrics/resource.go b/metrics/resource.go
index 0604a37..16f1ee8 100644
--- a/metrics/resource.go
+++ b/metrics/resource.go
@@ -16,11 +16,20 @@ type Resource struct {
 	PodMetadata *PodMetadata
 }
 
-func getNodeResource(s stats.NodeStats) *Resource {
+func getNodeResource(s stats.NodeStats, metadata *Metadata) *Resource {
 	labels := map[string]string{
 		LabelNodeName: s.NodeName,
 	}
 
+	if metadata.IncludeNodeLabels {
+		nodeMetadata, err := metadata.GetNodeMetadataByName(s.NodeName)
+		if err == nil {
+            for k, v := range nodeMetadata.GetLabels() {
+                labels[PrefixLabel+k] = v
+            }
+        }
+	}
+
 	return &Resource{
 		Type:      "node",
 		Name:      s.NodeName,
diff --git a/service/runnable.go b/service/runnable.go
index d91c476..b61a83b 100644
--- a/service/runnable.go
+++ b/service/runnable.go
@@ -4,12 +4,16 @@
 package service
 
 import (
+	"context"
+	"time"
+
 	"github.com/honeycombio/honeycomb-kubernetes-agent/metrics"
 	"github.com/honeycombio/libhoney-go"
 	"github.com/sirupsen/logrus"
-	"time"
 
 	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	corev1 "k8s.io/client-go/kubernetes/typed/core/v1"
 
 	"github.com/honeycombio/honeycomb-kubernetes-agent/interval"
 	"github.com/honeycombio/honeycomb-kubernetes-agent/kubelet"
@@ -18,25 +22,29 @@ import (
 var _ interval.Runnable = (*runnable)(nil)
 
 type runnable struct {
-	statsProvider         *kubelet.StatsProvider
-	metadataProvider      *kubelet.MetadataProvider
-	metricsProvider       *metrics.Processor
-	interval              time.Duration
-	k8sClusterName        string
-	restClient            kubelet.RestClient
-	builder               *libhoney.Builder
-	omitLabels            []metrics.OmitLabel
+	statsProvider     *kubelet.StatsProvider
+	metadataProvider  *kubelet.MetadataProvider
+	metricsProvider   *metrics.Processor
+	interval          time.Duration
+	k8sClusterName    string
+	restClient        kubelet.RestClient
+	builder           *libhoney.Builder
+	omitLabels        []metrics.OmitLabel
+	includeNodeLabels bool
+
 	metricGroupsToCollect map[metrics.MetricGroup]bool
 	logger                *logrus.Logger
+	apiClient             corev1.NodesGetter
 }
 
-func newRunnable(rc kubelet.RestClient, builder *libhoney.Builder, opt Options, logger *logrus.Logger) *runnable {
+func newRunnable(rc kubelet.RestClient, builder *libhoney.Builder, opt Options, logger *logrus.Logger, client *corev1.CoreV1Client) *runnable {
 	return &runnable{
 		interval:              opt.Interval,
 		k8sClusterName:        opt.ClusterName,
 		restClient:            rc,
 		builder:               builder,
 		omitLabels:            opt.OmitLabels,
+		includeNodeLabels:     opt.IncludeNodeLabels,
 		metricGroupsToCollect: opt.MetricGroupsToCollect,
 		logger:                logger,
 	}
@@ -70,15 +78,28 @@ func (r *runnable) Run() error {
 	var podsMetadata *v1.PodList
 	podsMetadata, err = r.metadataProvider.Pods()
 	if err != nil {
-		r.logger.WithError(err).Error("Could not retrieve metadata")
+		r.logger.WithError(err).Error("Could not retrieve pod metadata")
 		return nil
 	}
 	r.logger.WithFields(logrus.Fields{
 		"podCount": len(podsMetadata.Items),
-	}).Debug("Retrieved Metadata")
+	}).Debug("Retrieved Pod Metadata")
+
+	var nodesMetadata *v1.NodeList
+	// fetch metadata for all nodes
+	if r.includeNodeLabels {
+		nodesMetadata, err = r.apiClient.Nodes().List(context.TODO(), metav1.ListOptions{})
+		if err != nil {
+			r.logger.WithError(err).Error("Could not retrieve node metadata")
+			return nil
+		}
+		r.logger.WithFields(logrus.Fields{
+			"nodeCount": len(nodesMetadata.Items),
+		}).Debug("Retrieved Node Metadata")
+	}
 
 	// Get resource metrics
-	metadata := metrics.NewMetadata(podsMetadata, r.omitLabels, r.logger)
+	metadata := metrics.NewMetadata(podsMetadata, nodesMetadata, r.omitLabels, r.includeNodeLabels, r.logger)
 	resourceMetrics := r.metricsProvider.GenerateMetricsData(summary, metadata, r.metricGroupsToCollect)
 	r.logger.WithFields(logrus.Fields{
 		"resourceCount": len(resourceMetrics),
diff --git a/service/service.go b/service/service.go
index adb44c4..481e081 100644
--- a/service/service.go
+++ b/service/service.go
@@ -5,14 +5,16 @@ package service
 
 import (
 	"errors"
+	"os"
+	"time"
+
 	"github.com/honeycombio/honeycomb-kubernetes-agent/config"
 	"github.com/honeycombio/honeycomb-kubernetes-agent/interval"
 	"github.com/honeycombio/honeycomb-kubernetes-agent/kubelet"
 	"github.com/honeycombio/honeycomb-kubernetes-agent/metrics"
 	"github.com/honeycombio/libhoney-go"
 	"github.com/sirupsen/logrus"
-	"os"
-	"time"
+	corev1 "k8s.io/client-go/kubernetes/typed/core/v1"
 )
 
 var defaultOmitLabels []metrics.OmitLabel
@@ -31,16 +33,18 @@ type Service struct {
 	options    Options
 	builder    *libhoney.Builder
 	logger     *logrus.Logger
+	client     *corev1.CoreV1Client
 }
 
 type Options struct {
 	Interval              time.Duration
 	ClusterName           string
 	OmitLabels            []metrics.OmitLabel
+	IncludeNodeLabels     bool
 	MetricGroupsToCollect map[metrics.MetricGroup]bool
 }
 
-func NewMetricsService(cfg *config.MetricsConfig, builder *libhoney.Builder, logger *logrus.Logger) (*Service, error) {
+func NewMetricsService(cfg *config.MetricsConfig, builder *libhoney.Builder, logger *logrus.Logger, client *corev1.CoreV1Client) (*Service, error) {
 
 	if cfg.Endpoint == "" {
 		// Not all Managed K8s offerings allow the nodename to be DNS reachable, try by IP if available.
@@ -70,6 +74,7 @@ func NewMetricsService(cfg *config.MetricsConfig, builder *libhoney.Builder, log
 		Interval:              cfg.Interval,
 		ClusterName:           cfg.ClusterName,
 		OmitLabels:            cfg.OmitLabels,
+		IncludeNodeLabels:     cfg.IncludeNodeLabels,
 		MetricGroupsToCollect: mg,
 	}
 
@@ -79,6 +84,7 @@ func NewMetricsService(cfg *config.MetricsConfig, builder *libhoney.Builder, log
 		options:    *opt,
 		builder:    builder,
 		logger:     logger,
+		client:     client,
 	}, nil
 }
 
@@ -125,11 +131,12 @@ func (s *Service) Start() error {
 		"interval":     s.options.Interval,
 		"clusterName":  s.options.ClusterName,
 		"omitLabels":   s.options.OmitLabels,
+		"includeNodeLabels": s.options.IncludeNodeLabels,
 		"metricGroups": s.options.MetricGroupsToCollect,
 	}).Info("Creating Metrics Service Runner...")
 
 	// setup primary interval runner for metrics service
-	runnable := newRunnable(s.restClient, s.builder, s.options, s.logger)
+	runnable := newRunnable(s.restClient, s.builder, s.options, s.logger, s.client)
 	s.runner = interval.NewRunner("k8s-stats", s.options.Interval, runnable)
 	s.logger.Debug("Metrics Service Runner created")
 


Test patch:
diff --git a/metrics/metrics_test.go b/metrics/metrics_test.go
index ec6383a..92af40a 100644
--- a/metrics/metrics_test.go
+++ b/metrics/metrics_test.go
@@ -1,20 +1,23 @@
 package metrics
 
 import (
+	"encoding/json"
+	"io/ioutil"
 	"math"
 	"testing"
 	"time"
 
 	"github.com/honeycombio/honeycomb-kubernetes-agent/kubelet"
-	"github.com/honeycombio/honeycomb-kubernetes-agent/metrics/mock"
+	metrics_mock "github.com/honeycombio/honeycomb-kubernetes-agent/metrics/mock"
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
+	v1 "k8s.io/api/core/v1"
 	stats "k8s.io/kubelet/pkg/apis/stats/v1alpha1"
 
 	"github.com/stretchr/testify/require"
 )
 
-func createMockSourceAssets(isSum bool, isMeta bool, omitLabels []OmitLabel) (*stats.Summary, *Metadata) {
+func createMockSourceAssets(isSum bool, isMeta bool, omitLabels []OmitLabel, includeNodeLabels bool) (*stats.Summary, *Metadata) {
 	rc := &metrics_mock.MockRestClient{}
 
 	var summary *stats.Summary
@@ -27,12 +30,20 @@ func createMockSourceAssets(isSum bool, isMeta bool, omitLabels []OmitLabel) (*s
 	if isMeta {
 		metadataProvider := kubelet.NewMetadataProvider(rc)
 		podsMetadata, _ := metadataProvider.Pods()
-		metadata = NewMetadata(podsMetadata, omitLabels, logrus.StandardLogger())
+		nodesMetadata := getMockNodesResponse()
+		metadata = NewMetadata(podsMetadata, &nodesMetadata, omitLabels, includeNodeLabels, logrus.StandardLogger())
 	}
 
 	return summary, metadata
 }
 
+func getMockNodesResponse() v1.NodeList {
+	var nodesMetadata v1.NodeList
+	mockResponse, _ := ioutil.ReadFile("../testdata/nodes.json")
+	json.Unmarshal(mockResponse, &nodesMetadata)
+	return nodesMetadata
+}
+
 func createMockAccumulator(metadata *Metadata, mg map[MetricGroup]bool) *MetricDataAccumulator {
 
 	p := NewMetricsProcessor(10*time.Second, logrus.StandardLogger())
@@ -74,7 +85,7 @@ func TestGetLabels(t *testing.T) {
 	_, md := createMockSourceAssets(false, true, []OmitLabel{
 		"controller-revision-hash",
 		"pod-template-generation",
-	})
+	}, false)
 	pmd, _ := md.GetPodMetadataByUid("5c69ffd4-73d1-47df-87d7-08ae860e75ae")
 
 	labels := pmd.GetLabels()
@@ -84,8 +95,24 @@ func TestGetLabels(t *testing.T) {
 	assert.Equal(t, "honeycomb", labels["app.kubernetes.io/name"])
 }
 
+func TestGetNodeLabels(t *testing.T) {
+
+	_, md := createMockSourceAssets(false, true, []OmitLabel{
+		"beta.kubernetes.io/arch",
+		"beta.kubernetes.io/instance-type",
+		"beta.kubernetes.io/os",
+	}, true)
+	nmd, _ := md.GetNodeMetadataByName("duckboat-01")
+
+	labels := nmd.GetLabels()
+
+	assert.Equal(t, 8, len(labels))
+
+	assert.Equal(t, "us-east-1", labels["topology.kubernetes.io/region"])
+}
+
 func TestGetCpuLimit(t *testing.T) {
-	_, md := createMockSourceAssets(false, true, nil)
+	_, md := createMockSourceAssets(false, true, nil, false)
 
 	pmd, _ := md.GetPodMetadataByUid("5997ad9b-1d2a-43cf-ab57-a98d8796dc34")
 	limit := pmd.GetCpuLimit()
@@ -97,7 +124,7 @@ func TestGetCpuLimit(t *testing.T) {
 }
 
 func TestGetCpuLimitForContainer(t *testing.T) {
-	_, md := createMockSourceAssets(false, true, nil)
+	_, md := createMockSourceAssets(false, true, nil, false)
 
 	pmd, _ := md.GetPodMetadataByUid("5997ad9b-1d2a-43cf-ab57-a98d8796dc34")
 	limit := pmd.GetCpuLimitForContainer("speaker")
@@ -105,7 +132,7 @@ func TestGetCpuLimitForContainer(t *testing.T) {
 }
 
 func TestGetMemoryLimit(t *testing.T) {
-	_, md := createMockSourceAssets(false, true, nil)
+	_, md := createMockSourceAssets(false, true, nil, false)
 
 	pmd, _ := md.GetPodMetadataByUid("5997ad9b-1d2a-43cf-ab57-a98d8796dc34")
 	limit := pmd.GetMemoryLimit()
@@ -117,7 +144,7 @@ func TestGetMemoryLimit(t *testing.T) {
 }
 
 func TestGetMemoryLimitForContainer(t *testing.T) {
-	_, md := createMockSourceAssets(false, true, nil)
+	_, md := createMockSourceAssets(false, true, nil, false)
 
 	pmd, _ := md.GetPodMetadataByUid("5997ad9b-1d2a-43cf-ab57-a98d8796dc34")
 	limit := pmd.GetMemoryLimitForContainer("speaker")
@@ -125,7 +152,7 @@ func TestGetMemoryLimitForContainer(t *testing.T) {
 }
 
 func TestGetStatus(t *testing.T) {
-	_, md := createMockSourceAssets(false, true, nil)
+	_, md := createMockSourceAssets(false, true, nil, false)
 
 	pmd, _ := md.GetPodMetadataByUid("5997ad9b-1d2a-43cf-ab57-a98d8796dc34")
 	status := pmd.GetStatus()
@@ -133,7 +160,7 @@ func TestGetStatus(t *testing.T) {
 }
 
 func TestGetStatusForContainer(t *testing.T) {
-	_, md := createMockSourceAssets(false, true, nil)
+	_, md := createMockSourceAssets(false, true, nil, false)
 
 	pmd, _ := md.GetPodMetadataByUid("5997ad9b-1d2a-43cf-ab57-a98d8796dc34")
 	status := pmd.GetStatusForContainer("speaker")
@@ -143,7 +170,7 @@ func TestGetStatusForContainer(t *testing.T) {
 }
 
 func TestGenerateMetricsData(t *testing.T) {
-	summary, metadata := createMockSourceAssets(true, true, nil)
+	summary, metadata := createMockSourceAssets(true, true, nil, false)
 
 	p := NewMetricsProcessor(10*time.Second, logrus.StandardLogger())
 
@@ -163,7 +190,7 @@ func TestGenerateMetricsData(t *testing.T) {
 }
 
 func TestCpuMetrics(t *testing.T) {
-	summary, _ := createMockSourceAssets(true, false, nil)
+	summary, _ := createMockSourceAssets(true, false, nil, false)
 
 	p := NewMetricsProcessor(10*time.Second, logrus.StandardLogger())
 
@@ -176,7 +203,7 @@ func TestCpuMetrics(t *testing.T) {
 }
 
 func TestMemMetrics(t *testing.T) {
-	summary, _ := createMockSourceAssets(true, false, nil)
+	summary, _ := createMockSourceAssets(true, false, nil, false)
 
 	p := NewMetricsProcessor(10*time.Second, logrus.StandardLogger())
 
@@ -190,7 +217,7 @@ func TestMemMetrics(t *testing.T) {
 }
 
 func TestNetworkMetrics(t *testing.T) {
-	summary, _ := createMockSourceAssets(true, false, nil)
+	summary, _ := createMockSourceAssets(true, false, nil, false)
 
 	p := NewMetricsProcessor(10*time.Second, logrus.StandardLogger())
 
@@ -203,7 +230,7 @@ func TestNetworkMetrics(t *testing.T) {
 }
 
 func TestVolumeMetrics(t *testing.T) {
-	summary, _ := createMockSourceAssets(true, false, nil)
+	summary, _ := createMockSourceAssets(true, false, nil, false)
 
 	p := NewMetricsProcessor(10*time.Second, logrus.StandardLogger())
 
@@ -217,10 +244,10 @@ func TestVolumeMetrics(t *testing.T) {
 }
 
 func TestCounterMetrics(t *testing.T) {
-	summary, metadata := createMockSourceAssets(true, true, nil)
+	summary, metadata := createMockSourceAssets(true, true, nil, false)
 
 	podStats := getPodStatsByUID(summary.Pods, "5997ad9b-1d2a-43cf-ab57-a98d8796dc34")
-	node := getNodeResource(summary.Node)
+	node := getNodeResource(summary.Node, metadata)
 	pod := getPodResource(node, podStats, metadata)
 
 	p := NewMetricsProcessor(10*time.Second, logrus.StandardLogger())
@@ -242,10 +269,10 @@ func TestCounterMetrics(t *testing.T) {
 }
 
 func TestExpiredCounterMetrics(t *testing.T) {
-	summary, metadata := createMockSourceAssets(true, true, nil)
+	summary, metadata := createMockSourceAssets(true, true, nil, false)
 
 	podStats := getPodStatsByUID(summary.Pods, "5997ad9b-1d2a-43cf-ab57-a98d8796dc34")
-	node := getNodeResource(summary.Node)
+	node := getNodeResource(summary.Node, metadata)
 	pod := getPodResource(node, podStats, metadata)
 
 	// short expiration time to allow test to run faster
@@ -271,28 +298,39 @@ func TestExpiredCounterMetrics(t *testing.T) {
 }
 
 func TestNodeStats(t *testing.T) {
-	summary, metadata := createMockSourceAssets(true, true, nil)
+	// with includeNodeLabels
+	summary, metadata := createMockSourceAssets(true, true, nil, true)
 	acc := createMockAccumulator(metadata, ValidMetricGroups)
 
-	node := getNodeResource(summary.Node)
+	node := getNodeResource(summary.Node, metadata)
 
 	acc.nodeStats(node, summary.Node)
 	data := acc.Data[0]
-
 	assert.Equal(t, "duckboat-01", data.Resource.Name)
 	assert.Equal(t, 0, len(data.Resource.Status))
-	assert.Equal(t, 1, len(data.Resource.Labels))
+	assert.Equal(t, 9, len(data.Resource.Labels))
+	assert.Equal(t, "us-east-1", data.Resource.Labels[PrefixLabel+"topology.kubernetes.io/region"])
 	assert.Equal(t, 17, len(data.Metrics))
 	assert.Equal(t, float64(388954406)/1000000000, data.Metrics[MeasureCpuUsage].GetValue())
 
+	// without includeNodeLabels
+	summary, metadata = createMockSourceAssets(true, true, nil, false)
+	acc = createMockAccumulator(metadata, ValidMetricGroups)
+
+	node = getNodeResource(summary.Node, metadata)
+
+	acc.nodeStats(node, summary.Node)
+	data = acc.Data[0]
+	assert.Equal(t, "duckboat-01", data.Resource.Name)
+	assert.Equal(t, 1, len(data.Resource.Labels))
 }
 
 func TestPodStats(t *testing.T) {
-	summary, metadata := createMockSourceAssets(true, true, nil)
+	summary, metadata := createMockSourceAssets(true, true, nil, true)
 	acc := createMockAccumulator(metadata, ValidMetricGroups)
 
 	podStats := getPodStatsByUID(summary.Pods, "5997ad9b-1d2a-43cf-ab57-a98d8796dc34")
-	node := getNodeResource(summary.Node)
+	node := getNodeResource(summary.Node, metadata)
 	pod := getPodResource(node, podStats, metadata)
 
 	acc.podStats(pod, podStats)
@@ -300,20 +338,21 @@ func TestPodStats(t *testing.T) {
 
 	assert.Equal(t, "speaker-cpxhz", data.Resource.Name)
 	assert.Equal(t, 4, len(data.Resource.Status))
-	assert.Equal(t, 8, len(data.Resource.Labels))
+	assert.Equal(t, 16, len(data.Resource.Labels))
 	assert.Equal(t, "metallb", data.Resource.Labels[PrefixLabel+"app"])
 	assert.Equal(t, "metallb-system", data.Resource.Labels[LabelNamespaceName])
+	assert.Equal(t, "us-east-1", data.Resource.Labels[PrefixLabel+"topology.kubernetes.io/region"])
 	assert.Equal(t, 17, len(data.Metrics))
 	assert.Equal(t, float64(7919180)/1000000000, data.Metrics[MeasureCpuUsage].GetValue())
 
 }
 
 func TestContainerStats(t *testing.T) {
-	summary, metadata := createMockSourceAssets(true, true, nil)
+	summary, metadata := createMockSourceAssets(true, true, nil, true)
 	acc := createMockAccumulator(metadata, ValidMetricGroups)
 
 	podStats := getPodStatsByUID(summary.Pods, "5997ad9b-1d2a-43cf-ab57-a98d8796dc34")
-	node := getNodeResource(summary.Node)
+	node := getNodeResource(summary.Node, metadata)
 	pod := getPodResource(node, podStats, metadata)
 
 	acc.containerStats(pod, podStats.Containers[0])
@@ -322,20 +361,21 @@ func TestContainerStats(t *testing.T) {
 	assert.Equal(t, "speaker", data.Resource.Name)
 	assert.Equal(t, 4, len(data.Resource.Status))
 	assert.Equal(t, "running", data.Resource.Status[StatusState])
-	assert.Equal(t, 9, len(data.Resource.Labels))
+	assert.Equal(t, 17, len(data.Resource.Labels))
 	assert.Equal(t, "metallb", data.Resource.Labels[PrefixLabel+"app"])
 	assert.Equal(t, "metallb-system", data.Resource.Labels[LabelNamespaceName])
+	assert.Equal(t, "us-east-1", data.Resource.Labels[PrefixLabel+"topology.kubernetes.io/region"])
 	assert.Equal(t, 13, len(data.Metrics))
 	assert.Equal(t, float64(13107067)/1000000000, data.Metrics[MeasureCpuUsage].GetValue())
 
 }
 
 func TestVolumeStats(t *testing.T) {
-	summary, metadata := createMockSourceAssets(true, true, nil)
+	summary, metadata := createMockSourceAssets(true, true, nil, true)
 	acc := createMockAccumulator(metadata, ValidMetricGroups)
 
 	podStats := getPodStatsByUID(summary.Pods, "5997ad9b-1d2a-43cf-ab57-a98d8796dc34")
-	node := getNodeResource(summary.Node)
+	node := getNodeResource(summary.Node, metadata)
 	pod := getPodResource(node, podStats, metadata)
 
 	acc.volumeStats(pod, podStats.VolumeStats[0])
@@ -343,9 +383,10 @@ func TestVolumeStats(t *testing.T) {
 
 	assert.Equal(t, "speaker-token-kpzds", data.Resource.Name)
 	assert.Equal(t, 0, len(data.Resource.Status))
-	assert.Equal(t, 9, len(data.Resource.Labels))
+	assert.Equal(t, 17, len(data.Resource.Labels))
 	assert.Equal(t, "metallb", data.Resource.Labels[PrefixLabel+"app"])
 	assert.Equal(t, "metallb-system", data.Resource.Labels[LabelNamespaceName])
+	assert.Equal(t, "us-east-1", data.Resource.Labels[PrefixLabel+"topology.kubernetes.io/region"])
 	assert.Equal(t, 6, len(data.Metrics))
 
 }
diff --git a/testdata/nodes.json b/testdata/nodes.json
new file mode 100644
index 0000000..8d4af53
--- /dev/null
+++ b/testdata/nodes.json
@@ -0,0 +1,409 @@
+{
+  "kind": "NodeList",
+  "apiVersion": "v1",
+  "metadata": {
+    "resourceVersion": "105324751"
+  },
+  "items": [
+    {
+      "metadata": {
+        "name": "duckboat-01",
+        "uid": "757c743d-83c3-4bd7-9966-1eeac736ef84",
+        "resourceVersion": "105322474",
+        "creationTimestamp": "2021-12-08T01:46:51Z",
+        "labels": {
+          "eks.amazonaws.com/capacityType": "ON_DEMAND",
+          "eks.amazonaws.com/nodegroup": "custom-1",
+          "kubernetes.io/arch": "arm64",
+          "kubernetes.io/hostname": "duckboat-01",
+          "kubernetes.io/os": "linux",
+          "node.kubernetes.io/instance-type": "m6gd.2xlarge",
+          "topology.kubernetes.io/region": "us-east-1",
+          "topology.kubernetes.io/zone": "us-east-1a"
+        },
+        "annotations": {
+          "node.alpha.kubernetes.io/ttl": "0",
+          "volumes.kubernetes.io/controller-managed-attach-detach": "true"
+        },
+        "managedFields": [
+          {
+            "manager": "kube-controller-manager",
+            "operation": "Update",
+            "apiVersion": "v1",
+            "time": "2021-12-08T01:46:54Z",
+            "fieldsType": "FieldsV1",
+            "fieldsV1": {
+              "f:metadata": {
+                "f:annotations": { "f:node.alpha.kubernetes.io/ttl": {} }
+              },
+              "f:spec": { "f:taints": {} }
+            }
+          },
+          {
+            "manager": "kubelet",
+            "operation": "Update",
+            "apiVersion": "v1",
+            "time": "2021-12-08T01:47:21Z",
+            "fieldsType": "FieldsV1",
+            "fieldsV1": {
+              "f:metadata": {
+                "f:annotations": {
+                  ".": {},
+                  "f:volumes.kubernetes.io/controller-managed-attach-detach": {}
+                },
+                "f:labels": {
+                  ".": {},
+                  "f:eks.amazonaws.com/capacityType": {},
+                  "f:eks.amazonaws.com/nodegroup": {},
+                  "f:kubernetes.io/arch": {},
+                  "f:kubernetes.io/hostname": {},
+                  "f:kubernetes.io/os": {},
+                  "f:node.kubernetes.io/instance-type": {},
+                  "f:topology.kubernetes.io/region": {},
+                  "f:topology.kubernetes.io/zone": {}
+                }
+              },
+              "f:spec": { "f:providerID": {} },
+              "f:status": {
+                "f:allocatable": { "f:ephemeral-storage": {} },
+                "f:capacity": { "f:ephemeral-storage": {} },
+                "f:conditions": {
+                  "k:{\"type\":\"DiskPressure\"}": {
+                    "f:lastHeartbeatTime": {}
+                  },
+                  "k:{\"type\":\"MemoryPressure\"}": {
+                    "f:lastHeartbeatTime": {}
+                  },
+                  "k:{\"type\":\"PIDPressure\"}": { "f:lastHeartbeatTime": {} },
+                  "k:{\"type\":\"Ready\"}": {
+                    "f:lastHeartbeatTime": {},
+                    "f:lastTransitionTime": {},
+                    "f:message": {},
+                    "f:reason": {},
+                    "f:status": {}
+                  }
+                },
+                "f:images": {}
+              }
+            }
+          }
+        ]
+      },
+      "spec": {
+        "providerID": "aws:///us-east-1a/i-03dc801fd988ab620",
+        "taints": [
+          {
+            "key": "type",
+            "value": "hound",
+            "effect": "NoSchedule"
+          }
+        ]
+      },
+      "status": {
+        "capacity": {
+          "attachable-volumes-aws-ebs": "39",
+          "cpu": "8",
+          "ephemeral-storage": "134196204Ki",
+          "hugepages-1Gi": "0",
+          "hugepages-2Mi": "0",
+          "hugepages-32Mi": "0",
+          "hugepages-64Ki": "0",
+          "memory": "32144656Ki",
+          "pods": "58"
+        },
+        "allocatable": {
+          "attachable-volumes-aws-ebs": "39",
+          "cpu": "7910m",
+          "ephemeral-storage": "122601479578",
+          "hugepages-1Gi": "0",
+          "hugepages-2Mi": "0",
+          "hugepages-32Mi": "0",
+          "hugepages-64Ki": "0",
+          "memory": "31127824Ki",
+          "pods": "58"
+        },
+        "conditions": [
+          {
+            "type": "MemoryPressure",
+            "status": "False",
+            "lastHeartbeatTime": "2021-12-20T23:58:47Z",
+            "lastTransitionTime": "2021-12-08T01:46:48Z",
+            "reason": "KubeletHasSufficientMemory",
+            "message": "kubelet has sufficient memory available"
+          },
+          {
+            "type": "DiskPressure",
+            "status": "False",
+            "lastHeartbeatTime": "2021-12-20T23:58:47Z",
+            "lastTransitionTime": "2021-12-08T01:46:48Z",
+            "reason": "KubeletHasNoDiskPressure",
+            "message": "kubelet has no disk pressure"
+          },
+          {
+            "type": "PIDPressure",
+            "status": "False",
+            "lastHeartbeatTime": "2021-12-20T23:58:47Z",
+            "lastTransitionTime": "2021-12-08T01:46:48Z",
+            "reason": "KubeletHasSufficientPID",
+            "message": "kubelet has sufficient PID available"
+          },
+          {
+            "type": "Ready",
+            "status": "True",
+            "lastHeartbeatTime": "2021-12-20T23:58:47Z",
+            "lastTransitionTime": "2021-12-08T01:47:11Z",
+            "reason": "KubeletReady",
+            "message": "kubelet is posting ready status"
+          }
+        ],
+        "addresses": [
+          {
+            "type": "InternalIP",
+            "address": "10.9.150.105"
+          },
+          {
+            "type": "Hostname",
+            "address": "ip-10-9-150-105.ec2.internal"
+          },
+          {
+            "type": "InternalDNS",
+            "address": "ip-10-9-150-105.ec2.internal"
+          }
+        ],
+        "daemonEndpoints": {
+          "kubeletEndpoint": {
+            "Port": 10250
+          }
+        },
+        "nodeInfo": {
+          "machineID": "4a12cb49ef28482da9264bcd25ff2026",
+          "systemUUID": "ec2a33d7-9726-820c-79f2-cc8e5771fbf5",
+          "bootID": "b9ae1db7-2461-430f-8937-a401b3883812",
+          "kernelVersion": "5.4.129-63.229.amzn2.aarch64",
+          "osImage": "Amazon Linux 2",
+          "containerRuntimeVersion": "docker://19.3.13",
+          "kubeletVersion": "v1.21.2-13+d2965f0db10712",
+          "kubeProxyVersion": "v1.21.2-13+d2965f0db10712",
+          "operatingSystem": "linux",
+          "architecture": "arm64"
+        },
+        "images": [
+          {
+            "names": [
+              "12341234.dkr.ecr.us-east-1.amazonaws.com/spike@sha256:asdfdasfdasfdafd",
+              "12341234.dkr.ecr.us-east-1.amazonaws.com/spike:40129"
+            ],
+            "sizeBytes": 1461303813
+          },
+          {
+            "names": [
+              "12341234.dkr.ecr.us-east-1.amazonaws.com/spike@sha256:asdfdawer24234",
+              "12341234.dkr.ecr.us-east-1.amazonaws.com/spike:40120"
+            ],
+            "sizeBytes": 1461297117
+          }
+        ]
+      }
+    },
+    {
+      "metadata": {
+        "name": "duckboat-02",
+        "uid": "757c743d-83c3-4bd7-9966-1eeac736es34",
+        "resourceVersion": "105322474",
+        "creationTimestamp": "2021-12-08T01:46:51Z",
+        "labels": {
+          "eks.amazonaws.com/capacityType": "ON_DEMAND",
+          "eks.amazonaws.com/nodegroup": "custom-1",
+          "kubernetes.io/arch": "arm64",
+          "kubernetes.io/hostname": "duckboat-02",
+          "kubernetes.io/os": "linux",
+          "node.kubernetes.io/instance-type": "m6gd.2xlarge",
+          "topology.kubernetes.io/region": "us-east-2",
+          "topology.kubernetes.io/zone": "us-east-2d"
+        },
+        "annotations": {
+          "node.alpha.kubernetes.io/ttl": "0",
+          "volumes.kubernetes.io/controller-managed-attach-detach": "true"
+        },
+        "managedFields": [
+          {
+            "manager": "kube-controller-manager",
+            "operation": "Update",
+            "apiVersion": "v1",
+            "time": "2021-12-08T01:46:54Z",
+            "fieldsType": "FieldsV1",
+            "fieldsV1": {
+              "f:metadata": {
+                "f:annotations": { "f:node.alpha.kubernetes.io/ttl": {} }
+              },
+              "f:spec": { "f:taints": {} }
+            }
+          },
+          {
+            "manager": "kubelet",
+            "operation": "Update",
+            "apiVersion": "v1",
+            "time": "2021-12-08T01:47:21Z",
+            "fieldsType": "FieldsV1",
+            "fieldsV1": {
+              "f:metadata": {
+                "f:annotations": {
+                  ".": {},
+                  "f:volumes.kubernetes.io/controller-managed-attach-detach": {}
+                },
+                "f:labels": {
+                  ".": {},
+                  "f:eks.amazonaws.com/capacityType": {},
+                  "f:eks.amazonaws.com/nodegroup": {},
+                  "f:kubernetes.io/arch": {},
+                  "f:kubernetes.io/hostname": {},
+                  "f:kubernetes.io/os": {},
+                  "f:node.kubernetes.io/instance-type": {},
+                  "f:topology.kubernetes.io/region": {},
+                  "f:topology.kubernetes.io/zone": {}
+                }
+              },
+              "f:spec": { "f:providerID": {} },
+              "f:status": {
+                "f:allocatable": { "f:ephemeral-storage": {} },
+                "f:capacity": { "f:ephemeral-storage": {} },
+                "f:conditions": {
+                  "k:{\"type\":\"DiskPressure\"}": {
+                    "f:lastHeartbeatTime": {}
+                  },
+                  "k:{\"type\":\"MemoryPressure\"}": {
+                    "f:lastHeartbeatTime": {}
+                  },
+                  "k:{\"type\":\"PIDPressure\"}": { "f:lastHeartbeatTime": {} },
+                  "k:{\"type\":\"Ready\"}": {
+                    "f:lastHeartbeatTime": {},
+                    "f:lastTransitionTime": {},
+                    "f:message": {},
+                    "f:reason": {},
+                    "f:status": {}
+                  }
+                },
+                "f:images": {}
+              }
+            }
+          }
+        ]
+      },
+      "spec": {
+        "providerID": "aws:///us-east-1a/i-03dc801fd988ab620",
+        "taints": [
+          {
+            "key": "type",
+            "value": "hound",
+            "effect": "NoSchedule"
+          }
+        ]
+      },
+      "status": {
+        "capacity": {
+          "attachable-volumes-aws-ebs": "39",
+          "cpu": "8",
+          "ephemeral-storage": "134196204Ki",
+          "hugepages-1Gi": "0",
+          "hugepages-2Mi": "0",
+          "hugepages-32Mi": "0",
+          "hugepages-64Ki": "0",
+          "memory": "32144656Ki",
+          "pods": "58"
+        },
+        "allocatable": {
+          "attachable-volumes-aws-ebs": "39",
+          "cpu": "7910m",
+          "ephemeral-storage": "122601479578",
+          "hugepages-1Gi": "0",
+          "hugepages-2Mi": "0",
+          "hugepages-32Mi": "0",
+          "hugepages-64Ki": "0",
+          "memory": "31127824Ki",
+          "pods": "58"
+        },
+        "conditions": [
+          {
+            "type": "MemoryPressure",
+            "status": "False",
+            "lastHeartbeatTime": "2021-12-20T23:58:47Z",
+            "lastTransitionTime": "2021-12-08T01:46:48Z",
+            "reason": "KubeletHasSufficientMemory",
+            "message": "kubelet has sufficient memory available"
+          },
+          {
+            "type": "DiskPressure",
+            "status": "False",
+            "lastHeartbeatTime": "2021-12-20T23:58:47Z",
+            "lastTransitionTime": "2021-12-08T01:46:48Z",
+            "reason": "KubeletHasNoDiskPressure",
+            "message": "kubelet has no disk pressure"
+          },
+          {
+            "type": "PIDPressure",
+            "status": "False",
+            "lastHeartbeatTime": "2021-12-20T23:58:47Z",
+            "lastTransitionTime": "2021-12-08T01:46:48Z",
+            "reason": "KubeletHasSufficientPID",
+            "message": "kubelet has sufficient PID available"
+          },
+          {
+            "type": "Ready",
+            "status": "True",
+            "lastHeartbeatTime": "2021-12-20T23:58:47Z",
+            "lastTransitionTime": "2021-12-08T01:47:11Z",
+            "reason": "KubeletReady",
+            "message": "kubelet is posting ready status"
+          }
+        ],
+        "addresses": [
+          {
+            "type": "InternalIP",
+            "address": "10.9.150.105"
+          },
+          {
+            "type": "Hostname",
+            "address": "ip-10-9-150-105.ec2.internal"
+          },
+          {
+            "type": "InternalDNS",
+            "address": "ip-10-9-150-105.ec2.internal"
+          }
+        ],
+        "daemonEndpoints": {
+          "kubeletEndpoint": {
+            "Port": 10250
+          }
+        },
+        "nodeInfo": {
+          "machineID": "4a12cb49ef28482da9264bcd25ff2026",
+          "systemUUID": "ec2a33d7-9726-820c-79f2-cc8e5771fbf5",
+          "bootID": "b9ae1db7-2461-430f-8937-a401b3883812",
+          "kernelVersion": "5.4.129-63.229.amzn2.aarch64",
+          "osImage": "Amazon Linux 2",
+          "containerRuntimeVersion": "docker://19.3.13",
+          "kubeletVersion": "v1.21.2-13+d2965f0db10712",
+          "kubeProxyVersion": "v1.21.2-13+d2965f0db10712",
+          "operatingSystem": "linux",
+          "architecture": "arm64"
+        },
+        "images": [
+          {
+            "names": [
+              "12341234.dkr.ecr.us-east-1.amazonaws.com/spike@sha256:asdfdasfdasfdafd",
+              "12341234.dkr.ecr.us-east-1.amazonaws.com/spike:40129"
+            ],
+            "sizeBytes": 1461303813
+          },
+          {
+            "names": [
+              "12341234.dkr.ecr.us-east-1.amazonaws.com/spike@sha256:asdfdawer24234",
+              "12341234.dkr.ecr.us-east-1.amazonaws.com/spike:40120"
+            ],
+            "sizeBytes": 1461297117
+          }
+        ]
+      }
+    }
+  ]
+}