Repository: honeycombio/honeycomb-kubernetes-agent (go)

Issue:
agent is restarting when pod is not found
https://github.com/honeycombio/honeycomb-kubernetes-agent/blob/66864979093c63a6dc423f435633b553a77d4e40/metrics/metadata.go#L52

this error is not handled gracefully.
We checked one pod ids and it was a CI runner that was done with the job.

```
level=error msg="Metadata: Pod not found" podUid=2fd21870-xxxx-xxxx-xxxx-4eea5b905015
panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x1 addr=0x8 pc=0xa89ce9]
```

Pull request description:
fix: do not accumulate if pod is not there anymore #333
<!--
Thank you for contributing to the project! 💜
Please make sure to:
- Chat with us first if this is a big change
  - Open a new issue (or comment on an existing one)
  - We want to make sure you don't spend time implementing something we might have to say No to
- Add unit tests
- Mention any relevant issues in the PR description (e.g. "Fixes #123")

Please see our [OSS process document](https://github.com/honeycombio/home/blob/main/honeycomb-oss-lifecycle-and-practices.md#) to get an idea of how we operate.
-->

## Which problem is this PR solving?

- Accumulator is trying to pull metrics for pods that do not exist.

## Short description of the changes

- also abort accumulation if no pod data is available any more.

Fixes #333 

Patch:
diff --git a/metrics/accumulator.go b/metrics/accumulator.go
index 9dde373..46a2caf 100644
--- a/metrics/accumulator.go
+++ b/metrics/accumulator.go
@@ -59,6 +59,12 @@ func (a *MetricDataAccumulator) podStats(podResource *Resource, s stats.PodStats
 	if !a.metricGroupsToCollect[PodMetricGroup] {
 		return
 	}
+	
+	// Metatdata can be nil if pod is terminated before metadata is fetched.
+	// no metrics are needed.
+	if podResource.PodMetadata == nil {
+		return
+	}
 
 	a.accumulate(
 		podResource,
diff --git a/metrics/metadata.go b/metrics/metadata.go
index ac453b6..11f330b 100644
--- a/metrics/metadata.go
+++ b/metrics/metadata.go
@@ -50,7 +50,7 @@ func (m *Metadata) GetPodMetadataByUid(uid types.UID) (*PodMetadata, error) {
 	logrus.WithFields(logrus.Fields{
 		"podUid": uid,
 	}).Error("Metadata: Pod not found")
-	return &PodMetadata{}, errors.New("pod not found")
+	return nil, errors.New("pod not found")
 }
 
 type NodeMetadata struct {


Test patch:
diff --git a/metrics/metrics_test.go b/metrics/metrics_test.go
index c791ecb..6455f8b 100644
--- a/metrics/metrics_test.go
+++ b/metrics/metrics_test.go
@@ -360,6 +360,20 @@ func TestNodeStats(t *testing.T) {
 	assert.Equal(t, 1, len(data.Resource.Labels))
 }
 
+func TestPodStatsOnDeadPod(t *testing.T) {
+	summary, metadata := createMockSourceAssets(true, true, nil, true)
+	acc := createMockAccumulator(metadata, ValidMetricGroups)
+
+	podStats := getPodStatsByUID(summary.Pods, "5997ad9b-1d2a-43cf-ab57-a98d8796dc34")
+	node := getNodeResource(summary.Node, metadata)
+	pod := getPodResource(node, podStats, metadata)
+	pod.PodMetadata = nil
+
+	acc.podStats(pod, podStats)
+	assert.Equal(t, 0, len(acc.Data))
+
+}
+
 func TestPodStats(t *testing.T) {
 	summary, metadata := createMockSourceAssets(true, true, nil, true)
 	acc := createMockAccumulator(metadata, ValidMetricGroups)