Repository: honeycombio/honeycomb-kubernetes-agent
Pull request #246 (2022-04-01 06:00:59)

Bump k8s.io/api from 0.23.3 to 0.23.5
Bumps [k8s.io/api](https://github.com/kubernetes/api) from 0.23.3 to 0.23.5.
<details>
<summary>Commits</summary>
<ul>
<li><a href="https://github.com/kubernetes/api/commit/cd271e7523ef7b6b485aa42c0218c294ca00a001"><code>cd271e7</code></a> Update dependencies to v0.23.5 tag</li>
<li><a href="https://github.com/kubernetes/api/commit/a6b22073a6bd7b71e4e0f651c47ae6aada780dfd"><code>a6b2207</code></a> Merge pull request <a href="https://github-redirect.dependabot.com/kubernetes/api/issues/108139">#108139</a> from liggitt/automated-cherry-pick-of-#108138-upst...</li>
<li><a href="https://github.com/kubernetes/api/commit/80b93d425638f3e420236bc9a9010d8a0eab9314"><code>80b93d4</code></a> Revert v1beta1 PodDisruptionBudget select patchStrategy</li>
<li>See full diff in <a href="https://github.com/kubernetes/api/compare/v0.23.3...v0.23.5">compare view</a></li>
</ul>
</details>
<br />


[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=k8s.io/api&package-manager=go_modules&previous-version=0.23.3&new-version=0.23.5)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores)

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`.

[//]: # (dependabot-automerge-start)
[//]: # (dependabot-automerge-end)

---

<details>
<summary>Dependabot commands and options</summary>
<br />

You can trigger Dependabot actions by commenting on this PR:
- `@dependabot rebase` will rebase this PR
- `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it
- `@dependabot merge` will merge this PR after your CI passes on it
- `@dependabot squash and merge` will squash and merge this PR after your CI passes on it
- `@dependabot cancel merge` will cancel a previously requested merge and block automerging
- `@dependabot reopen` will reopen this PR if it is closed
- `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually
- `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself)
- `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself)
- `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)


</details>


Patch:
diff --git a/config/config.go b/config/config.go
index 879c8fb..f1d7119 100644
--- a/config/config.go
+++ b/config/config.go
@@ -34,6 +34,7 @@ type WatcherConfig struct {
 	// string means "select all pods".
 	LabelSelector *string  `yaml:"labelSelector"`
 	FilePaths     []string `yaml:"paths"`
+	ExcludePaths  []string `yaml:"exclude"`
 	ContainerName string   `yaml:"containerName"`
 	Processors    []map[string]map[string]interface{}
 }
diff --git a/go.mod b/go.mod
index 67048dc..28f11f4 100644
--- a/go.mod
+++ b/go.mod
@@ -3,6 +3,7 @@ module github.com/honeycombio/honeycomb-kubernetes-agent
 go 1.12
 
 require (
+	github.com/bmatcuk/doublestar/v4 v4.0.2
 	github.com/boltdb/bolt v1.3.1
 	github.com/hashicorp/golang-lru v0.5.4
 	github.com/honeycombio/dynsampler-go v0.2.1
@@ -20,8 +21,8 @@ require (
 	github.com/stretchr/testify v1.7.0
 	gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b // indirect
 	gopkg.in/yaml.v2 v2.4.0
-	k8s.io/api v0.23.3
-	k8s.io/apimachinery v0.23.3
+	k8s.io/api v0.23.5
+	k8s.io/apimachinery v0.23.5
 	k8s.io/client-go v0.23.3
 	k8s.io/kubelet v0.23.3
 )
diff --git a/go.sum b/go.sum
index 51238b6..2fa68e0 100644
--- a/go.sum
+++ b/go.sum
@@ -72,6 +72,8 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
 github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM=
 github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
+github.com/bmatcuk/doublestar/v4 v4.0.2 h1:X0krlUVAVmtr2cRoTqR8aDMrDqnB36ht8wpWTiQ3jsA=
+github.com/bmatcuk/doublestar/v4 v4.0.2/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
 github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
 github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
@@ -848,10 +850,12 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-k8s.io/api v0.23.3 h1:KNrME8KHGr12Ozjf8ytOewKzZh6hl/hHUZeHddT3a38=
 k8s.io/api v0.23.3/go.mod h1:w258XdGyvCmnBj/vGzQMj6kzdufJZVUwEM1U2fRJwSQ=
-k8s.io/apimachinery v0.23.3 h1:7IW6jxNzrXTsP0c8yXz2E5Yx/WTzVPTsHIx/2Vm0cIk=
+k8s.io/api v0.23.5 h1:zno3LUiMubxD/V1Zw3ijyKO3wxrhbUF1Ck+VjBvfaoA=
+k8s.io/api v0.23.5/go.mod h1:Na4XuKng8PXJ2JsploYYrivXrINeTaycCGcYgF91Xm8=
 k8s.io/apimachinery v0.23.3/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM=
+k8s.io/apimachinery v0.23.5 h1:Va7dwhp8wgkUPWsEXk6XglXWU4IKYLKNlv8VkX7SDM0=
+k8s.io/apimachinery v0.23.5/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM=
 k8s.io/client-go v0.23.3 h1:23QYUmCQ/W6hW78xIwm3XqZrrKZM+LWDqW2zfo+szJs=
 k8s.io/client-go v0.23.3/go.mod h1:47oMd+YvAOqZM7pcQ6neJtBiFH7alOyfunYN48VsmwE=
 k8s.io/component-base v0.23.3/go.mod h1:1Smc4C60rWG7d3HjSYpIwEbySQ3YWg0uzH5a2AtaTLg=
diff --git a/k8sagent/watcher.go b/k8sagent/watcher.go
index f04440d..7342960 100644
--- a/k8sagent/watcher.go
+++ b/k8sagent/watcher.go
@@ -9,7 +9,7 @@ import (
 
 	"github.com/sirupsen/logrus"
 
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	v1types "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
diff --git a/main.go b/main.go
index aee10c4..48615b6 100644
--- a/main.go
+++ b/main.go
@@ -3,13 +3,14 @@ package main
 import (
 	"bytes"
 	"fmt"
-	"github.com/honeycombio/honeycomb-kubernetes-agent/service"
 	"os"
 	"os/signal"
 	"strings"
 	"syscall"
 	"time"
 
+	"github.com/honeycombio/honeycomb-kubernetes-agent/service"
+
 	"github.com/honeycombio/honeycomb-kubernetes-agent/config"
 	"github.com/honeycombio/honeycomb-kubernetes-agent/handlers"
 	"github.com/honeycombio/honeycomb-kubernetes-agent/podtailer"
diff --git a/metrics/processor.go b/metrics/processor.go
index 6e3a124..eaabd8c 100644
--- a/metrics/processor.go
+++ b/metrics/processor.go
@@ -149,7 +149,7 @@ func (p *Processor) MemMetrics(s *stats.MemoryStats, limit float64) Metrics {
 	var utilization float64
 	if limit > 0 {
 		if s.UsageBytes != nil {
-			usage := float64(*s.UsageBytes)
+			usage := float64(*s.WorkingSetBytes)
 			utilization = (usage / limit) * 100
 		}
 	}
diff --git a/podtailer/podtailer.go b/podtailer/podtailer.go
index 48ae09e..12d5384 100644
--- a/podtailer/podtailer.go
+++ b/podtailer/podtailer.go
@@ -10,9 +10,10 @@ import (
 	"sync"
 	"time"
 
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
 
+	"github.com/bmatcuk/doublestar/v4"
 	"github.com/honeycombio/honeycomb-kubernetes-agent/config"
 	"github.com/honeycombio/honeycomb-kubernetes-agent/handlers"
 	"github.com/honeycombio/honeycomb-kubernetes-agent/k8sagent"
@@ -292,12 +293,39 @@ func determineLogPattern(pod *v1.Pod, basePath string, legacyLogPaths bool) (str
 	return "", fmt.Errorf("Could not find specified log path for pod %s", pod.UID)
 }
 
-func determineFilterFunc(pod *v1.Pod, basePath string, containerName string, legacyLogPaths bool) func(fileName string) bool {
+func determineFilterFunc(pod *v1.Pod, basePath string, containerName string, legacyLogPaths bool, excludePaths []string) func(fileName string) bool {
+	// this is an exclude function that conforms to the filter specs -- it returns true
+	// if the file should be *included*. By default, it's nil (which is treated as true)
+	var exclusionFilter func(string) bool
+	if len(excludePaths) > 0 {
+		excludes := make([]string, 0)
+		for _, exclude := range excludePaths {
+			// don't try to evaluate patterns that we can't even parse
+			if doublestar.ValidatePattern(exclude) {
+				excludes = append(excludes, exclude)
+				logrus.WithField("exclude", exclude).Debug("Excluding path")
+			}
+		}
+		exclusionFilter = func(fileName string) bool {
+			// check if we should exclude this path
+			for _, exclude := range excludes {
+				shouldExclude, _ := doublestar.Match(exclude, fileName)
+				if shouldExclude {
+					return false // false means exclude
+				}
+			}
+			return true
+		}
+	}
 
 	if containerName == "" {
-		logrus.Debug("No container name specified, no filter function needed")
-		return nil
+		if len(excludePaths) == 0 {
+			logrus.Debug("No container name or exclude paths specified, no filter function needed")
+			return nil
+		}
+		return exclusionFilter
 	}
+
 	if legacyLogPaths {
 		re := fmt.Sprintf(
 			"^%s/containers/%s_%s_%s-.+\\.log",
@@ -311,6 +339,11 @@ func determineFilterFunc(pod *v1.Pod, basePath string, containerName string, leg
 		}).Debug("Container filter function")
 
 		return func(fileName string) bool {
+			// first check if we should exclude this path
+			// If exclusionFilter says no, we're going to say no.
+			if exclusionFilter != nil && !exclusionFilter(fileName) {
+				return false
+			}
 			ok, _ := regexp.Match(re, []byte(fileName))
 			return ok
 		}
@@ -336,15 +369,21 @@ func determineFilterFunc(pod *v1.Pod, basePath string, containerName string, leg
 	}).Debug("Container filter function")
 
 	return func(fileName string) bool {
-		ok, _ := regexp.Match(re1, []byte(fileName))
+		// first check if we should exclude this path
+		// If exclusionFilter says no, we're going to say no.
+		if exclusionFilter != nil && !exclusionFilter(fileName) {
+			return false
+		}
+		// now check our patterns -- if any of them succeeds, we're good
+		ok, _ := regexp.MatchString(re1, fileName)
 		if ok {
 			return ok
 		}
-		ok, _ = regexp.Match(re2, []byte(fileName))
+		ok, _ = regexp.MatchString(re2, fileName)
 		if ok {
 			return ok
 		}
-		ok, _ = regexp.Match(re3, []byte(fileName))
+		ok, _ = regexp.MatchString(re3, fileName)
 		return ok
 	}
 }
@@ -358,7 +397,7 @@ func (pt *PodSetTailer) watcherForPod(pod *v1.Pod, containerName string, podWatc
 
 	// only watch logs for containers matching the given name, if
 	// one is specified
-	filterFunc := determineFilterFunc(pod, logsBasePath, containerName, pt.legacyLogPaths)
+	filterFunc := determineFilterFunc(pod, logsBasePath, containerName, pt.legacyLogPaths, pt.config.ExcludePaths)
 
 	additionalProcessors := []processors.Processor{
 		&processors.KubernetesMetadataProcessor{
diff --git a/tailer/tailer.go b/tailer/tailer.go
index d6dd6fb..38b0bd9 100644
--- a/tailer/tailer.go
+++ b/tailer/tailer.go
@@ -3,16 +3,16 @@
 package tailer
 
 import (
+	"os"
 	"path/filepath"
 	"sync"
 	"time"
 
+	"github.com/bmatcuk/doublestar/v4"
 	"github.com/honeycombio/honeycomb-kubernetes-agent/handlers"
 	"github.com/hpcloud/tail"
 
 	"github.com/sirupsen/logrus"
-
-	"k8s.io/api/core/v1"
 )
 
 // Tailer tails a single file, passing each line off to the handler.
@@ -123,7 +123,6 @@ type PathWatcher struct {
 	handlerFactory handlers.LineHandlerFactory
 	stateRecorder  StateRecorder
 	checkInterval  time.Duration
-	pod            *v1.Pod
 
 	stop         chan bool
 	savedPattern string
@@ -171,6 +170,22 @@ func (p *PathWatcher) Stop() {
 	}
 }
 
+// This emulates filepath.Glob's behavior using doublestar,
+// which means that it now supports /**/ names in its paths.
+func doublestarGlob(pat string) ([]string, error) {
+	base, pattern := doublestar.SplitPattern(pat)
+	fsys := os.DirFS(base)
+	files, err := doublestar.Glob(fsys, pattern)
+	if err != nil {
+		return nil, err
+	}
+	var result []string
+	for _, f := range files {
+		result = append(result, filepath.Join(base, f))
+	}
+	return result, nil
+}
+
 func (p *PathWatcher) check() {
 	// Have we figured out the pattern yet? If not, run our pattern function
 	if p.savedPattern == "" {
@@ -188,7 +203,7 @@ func (p *PathWatcher) check() {
 			"log pattern": pt,
 		}).Info("Log pattern")
 	}
-	files, err := filepath.Glob(p.savedPattern)
+	files, err := doublestarGlob(p.savedPattern)
 	if err != nil {
 		logrus.WithError(err).Error("Error globbing files")
 	}
@@ -204,7 +219,7 @@ func (p *PathWatcher) check() {
 			if p.filter != nil && !p.filter(file) {
 				logrus.WithFields(logrus.Fields{
 					"file": file,
-				}).Warn("File filtered out of tailing list based on containerName")
+				}).Warn("File filtered out of tailing list")
 				continue
 			}
 			handler := p.handlerFactory.New(file)


Test patch:
diff --git a/handlers/handlers_test.go b/handlers/handlers_test.go
index 954cedc..9208f12 100644
--- a/handlers/handlers_test.go
+++ b/handlers/handlers_test.go
@@ -10,7 +10,7 @@ import (
 	"testing"
 	"time"
 
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	v1types "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 
diff --git a/metrics/metrics_test.go b/metrics/metrics_test.go
index bcd217f..9a61b98 100644
--- a/metrics/metrics_test.go
+++ b/metrics/metrics_test.go
@@ -222,7 +222,7 @@ func TestMemMetrics(t *testing.T) {
 	require.Equal(t, 7, len(metrics))
 
 	assert.Equal(t, float64(2143281152), metrics[MeasureMemoryUsage].GetValue(), "Memory Usage")
-	assert.InDelta(t, 0.000001, 99.804300603, metrics[MeasureMemoryUtilization].GetValue(), "Memory Utilization")
+	assert.Equal(t, 97.79205322265625, metrics[MeasureMemoryUtilization].GetValue(), "Memory Utilization")
 	assert.Equal(t, float64(23191552), metrics[MeasureMemoryRSS].GetValue(), "Memory RSS")
 }
 
diff --git a/podtailer/podtailer_test.go b/podtailer/podtailer_test.go
index d11fa91..95e48db 100644
--- a/podtailer/podtailer_test.go
+++ b/podtailer/podtailer_test.go
@@ -9,7 +9,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 )
 
 // test patterns after https://github.com/kubernetes/kubernetes/pull/74441
@@ -130,7 +130,7 @@ func TestDetermineFilterFuncLegacyLogs(t *testing.T) {
 		return ok
 	}
 
-	f := determineFilterFunc(pod, "/var/log", "wibble", true)
+	f := determineFilterFunc(pod, "/var/log", "wibble", true, nil)
 	// functions should have same output
 	assert.Equal(
 		t,
@@ -160,7 +160,7 @@ func TestDetermineFilterFunc(t *testing.T) {
 		return ok
 	}
 
-	f := determineFilterFunc(pod, "/var/log", "wibble", false)
+	f := determineFilterFunc(pod, "/var/log", "wibble", false, nil)
 	assert.Equal(
 		t,
 		expected("/var/log/pods/123456/wibble_0.log"),
@@ -192,7 +192,7 @@ func TestDetermineFilterFuncK8S1Dot10(t *testing.T) {
 		return ok
 	}
 
-	f := determineFilterFunc(pod, "/var/log", "wibble", false)
+	f := determineFilterFunc(pod, "/var/log", "wibble", false, nil)
 	assert.Equal(
 		t,
 		expected("/var/log/pods/123456/wibble/0.log"),
@@ -236,7 +236,7 @@ func TestDetermineFilterFuncK8S74441(t *testing.T) {
 	_, err = os.Create(file2)
 	assert.Nil(t, err)
 
-	f := determineFilterFunc(pod, basePath, "container1", false)
+	f := determineFilterFunc(pod, basePath, "container1", false, nil)
 
 	assert.Equal(
 		t,
@@ -250,3 +250,56 @@ func TestDetermineFilterFuncK8S74441(t *testing.T) {
 		false,
 	)
 }
+
+func TestDetermineFilterFuncExclude(t *testing.T) {
+	t.Parallel()
+	pod := &v1.Pod{}
+	pod.Namespace = "asdf"
+	pod.Name = "foo"
+	pod.UID = "123456"
+	podDirName := fmt.Sprintf("%s_%s_%s", pod.Namespace, pod.Name, pod.UID)
+
+	basePath, err := ioutil.TempDir("", "")
+	assert.Nil(t, err)
+	defer os.RemoveAll(basePath)
+
+	err = os.Mkdir(filepath.Join(basePath, "pods"), 0777)
+	assert.Nil(t, err)
+	err = os.Mkdir(filepath.Join(basePath, "pods", podDirName), 0777)
+	assert.Nil(t, err)
+	err = os.Mkdir(filepath.Join(basePath, "pods", podDirName, "container1"), 0777)
+	assert.Nil(t, err)
+	err = os.Mkdir(filepath.Join(basePath, "pods", podDirName, "container2"), 0777)
+	assert.Nil(t, err)
+
+	tests := []struct {
+		path string
+		name string
+		want bool
+	}{
+		{"container1", "0.log", true},
+		{"container1", "1.log.exclude1only", false},
+		{"container1", "1.log.excludeAlways", false},
+		{"container2", "0.log", true},
+		{"container2", "0.log.excludeAlways", false},
+		{"container2", "1.log", true},
+		{"container2", "1.log.exclude1only", true},
+		{"container2", "1.log.excludeAlways", false},
+	}
+
+	f := determineFilterFunc(pod, basePath, "", false, []string{"**/*.excludeAlways", "**/container1/*.exclude1only"})
+
+	for _, tc := range tests {
+		t.Run(filepath.Join(tc.path, tc.name), func(t *testing.T) {
+			file1 := filepath.Join(basePath, "pods", podDirName, tc.path, tc.name)
+			_, err = os.Create(file1)
+			assert.Nil(t, err)
+
+			assert.Equal(
+				t,
+				tc.want,
+				f(file1),
+			)
+		})
+	}
+}
diff --git a/tailer/tailer_test.go b/tailer/tailer_test.go
index e1136dd..d92b885 100644
--- a/tailer/tailer_test.go
+++ b/tailer/tailer_test.go
@@ -5,6 +5,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path"
+	"path/filepath"
 	"testing"
 	"time"
 
@@ -150,8 +151,9 @@ func TestPathWatching(t *testing.T) {
 	time.Sleep(time.Second)
 	watcher.Stop()
 
-	assert.Equal(t, len(handlerFactory.handlers), 2)
-	for _, h := range handlerFactory.handlers {
+	assert.Equal(t, 2, len(handlerFactory.handlers))
+	for k, h := range handlerFactory.handlers {
+		fmt.Println(k)
 		assert.Equal(t, 2, len(h.lines))
 	}
 	assert.Equal(t, len(watcher.tailers), 0)
@@ -165,9 +167,30 @@ func TestTailingWithoutStateRecorder(t *testing.T) {
 	handler := &mockLineHandler{}
 
 	logFile, err := ioutil.TempFile("/tmp", "honeycomb-log-test")
+	assert.NoError(t, err)
 	logFile.Write([]byte("line1\n"))
 	logFile.Sync()
 	tailer := NewTailer(logFile.Name(), handler, stateRecorder)
 	tailer.Run()
 	tailer.Stop()
 }
+
+func TestGlobbing(t *testing.T) {
+	paths := []string{
+		"/var/log/*",
+		"*",
+		".",
+	}
+
+	for _, p := range paths {
+		t.Run(p, func(t *testing.T) {
+			oldfiles, err := filepath.Glob(p)
+			assert.NoError(t, err)
+
+			newfiles, err := doublestarGlob(p)
+			assert.NoError(t, err)
+
+			assert.Equal(t, oldfiles, newfiles)
+		})
+	}
+}