Repository: honeycombio/honeycomb-kubernetes-agent
Pull request #51 (2019-10-03 22:55:43)

Add regex parser


Patch:
diff --git a/.circleci/config.yml b/.circleci/config.yml
index c71c345..dda01a0 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -1,6 +1,12 @@
 # Use the latest 2.1 version of CircleCI pipeline processing engine, see https://circleci.com/docs/2.0/configuration-reference/
 version: 2.1
 
+# required as all of the jobs need to have a tag filter for some reason
+tag_filters: &tag_filters
+  filters:
+      tags:
+        only: /.*/
+
 jobs:
   build_agent:
     # needed to build docker images, attach mount points, etc
@@ -28,4 +34,4 @@ workflows:
   version: 2
   build_and_deploy:
     jobs:
-      - build_agent
+      - build_agent: *tag_filters
diff --git a/docs/configuration-reference.md b/docs/configuration-reference.md
index 6465aae..172ce91 100644
--- a/docs/configuration-reference.md
+++ b/docs/configuration-reference.md
@@ -40,6 +40,20 @@ Currently, the following parsers are supported:
 ### json
 Parses logs in JSON format
 
+### regex
+
+Parses logs using a list of regex expressions. Uses the [RE2 Syntax](https://github.com/google/re2/wiki/Syntax). For each line processed, the first regex in the list to match is used. Note that adding many regex expressions can hurt performance.
+
+```yaml
+parser:
+  name: regex
+  options:
+    # lines contain either tree data or city/state data
+    expressions:
+      - "(?P<species>[A-z]+) (?P<height>[0-9]{2}[0-9]?)"
+      - "(?P<city>[A-z ]+),(?P<state>[A-z]{2})"
+```
+
 ### nginx
 Parses NGINX access logs.
 
@@ -86,7 +100,6 @@ Does no parsing on logs, and returns an event with the entire contents of the lo
 
 More parsers will be added in the future. If you'd like to see support for additional log formats, please open an issue or email support@honeycomb.io!
 
-
 ## Processors
 Processors transform events after they're parsed. Currently, the following processors are supported:
 
diff --git a/examples/quickstart.yaml b/examples/quickstart.yaml
index 0913644..09c053c 100644
--- a/examples/quickstart.yaml
+++ b/examples/quickstart.yaml
@@ -65,8 +65,8 @@ metadata:
   labels:
     k8s-app: honeycomb-agent
     kubernetes.io/cluster-service: 'true'
-    version: v1.3.3
-  name: honeycomb-agent-v1.3.3
+    version: v1.4.0
+  name: honeycomb-agent-v1.4.0
   namespace: kube-system
 spec:
   updateStrategy:
@@ -78,7 +78,7 @@ spec:
       labels:
         k8s-app: honeycomb-agent
         kubernetes.io/cluster-service: 'true'
-        version: v1.3.3
+        version: v1.4.0
     spec:
       tolerations:
         - operator: Exists
@@ -94,7 +94,7 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: spec.nodeName
-        image: honeycombio/honeycomb-kubernetes-agent:1.3.3
+        image: honeycombio/honeycomb-kubernetes-agent:1.4.0
         imagePullPolicy: IfNotPresent
         name: honeycomb-agent
         resources:
diff --git a/parsers/parsers.go b/parsers/parsers.go
index 6088ea7..b9b5b07 100644
--- a/parsers/parsers.go
+++ b/parsers/parsers.go
@@ -36,6 +36,8 @@ func NewParserFactory(config *config.ParserConfig) (ParserFactory, error) {
 		factory = &KeyvalParserFactory{}
 	case "audit":
 		factory = &AuditParserFactory{}
+	case "regex":
+		factory = &RegexFactory{}
 	default:
 		return nil, fmt.Errorf("Unknown parser type %s", config.Name)
 	}
diff --git a/parsers/regex.go b/parsers/regex.go
new file mode 100644
index 0000000..77546da
--- /dev/null
+++ b/parsers/regex.go
@@ -0,0 +1,59 @@
+package parsers
+
+import (
+	"fmt"
+	"regexp"
+)
+
+type RegexFactory struct {
+	expressions []string
+}
+
+func (rf *RegexFactory) Init(options map[string]interface{}) error {
+	if options == nil {
+		return fmt.Errorf("regex parser specified but no options defined")
+	}
+
+	expressions, ok := options["expressions"].([]interface{})
+	if !ok {
+		return fmt.Errorf("regex parser missing patterns option")
+	}
+
+	rf.expressions = make([]string, len(expressions))
+	for i, s := range expressions {
+		expression, ok := s.(string)
+		if !ok {
+			return fmt.Errorf("expected expression %s to be string", s)
+		}
+		rf.expressions[i] = expression
+	}
+	return nil
+}
+
+func (rf *RegexFactory) New() Parser {
+	re := make([]*extRegexp, len(rf.expressions))
+	for i, p := range rf.expressions {
+		re[i] = &extRegexp{regexp.MustCompile(p)}
+	}
+	return &RegexParser{re: re}
+}
+
+type RegexParser struct {
+	re []*extRegexp
+}
+
+func (rp *RegexParser) Parse(line string) (map[string]interface{}, error) {
+	var captures map[string]interface{}
+
+	for _, re := range rp.re {
+		_, captures = re.FindStringSubmatchIfaceMap(line)
+		if captures != nil {
+			break
+		}
+	}
+
+	if captures == nil {
+		return nil, fmt.Errorf("Couldn't parse line with any supplied regexes: %s", line)
+	}
+	return captures, nil
+}
diff --git a/parsers/regexp_extension.go b/parsers/regexp_extension.go
index 7c79f2e..35d4379 100644
--- a/parsers/regexp_extension.go
+++ b/parsers/regexp_extension.go
@@ -39,3 +39,24 @@ func (r *extRegexp) FindStringSubmatchMap(s string) (string, map[string]string)
 	}
 	return match[0], captures
 }
+
+// FindStringSubmatchMapIfaceMap behaves the same as FindStringSubmatchMap except it
+// returns a map[string]interface{}
+func (r *extRegexp) FindStringSubmatchIfaceMap(s string) (string, map[string]interface{}) {
+	match := r.FindStringSubmatch(s)
+	if match == nil {
+		return "", nil
+	}
+
+	captures := make(map[string]interface{})
+	for i, name := range r.SubexpNames() {
+		if i == 0 {
+			continue
+		}
+		if name != "" {
+			// ignore unnamed matches
+			captures[name] = match[i]
+		}
+	}
+	return match[0], captures
+}
diff --git a/version.txt b/version.txt
index 3f44223..467f716 100644
--- a/version.txt
+++ b/version.txt
@@ -1,2 +1,2 @@
-1.3.3
+1.4.0
 


Test patch:
diff --git a/config/config_test.go b/config/config_test.go
index b7dea2e..faa248e 100644
--- a/config/config_test.go
+++ b/config/config_test.go
@@ -40,3 +40,12 @@ func TestAdditionalFieldsParsing(t *testing.T) {
 
 	assert.Equal(t, map[string]interface{}{"foo": "1", "bar": 2}, c.Watchers[0].Processors[0]["additional_fields"])
 }
+
+func TestRegexExpressionsArrayParsing(t *testing.T) {
+	path, _ := filepath.Abs(filepath.Join("testdata", "parser_with_options.yaml"))
+	c, err := ReadFromFile(path)
+	assert.NoError(t, err)
+
+	assert.Equal(t, "regex", c.Watchers[1].Parser.Name)
+	assert.Equal(t, map[string]interface{}{"expressions": []interface{}{"foo", "bar"}}, c.Watchers[1].Parser.Options)
+}
diff --git a/config/testdata/parser_with_options.yaml b/config/testdata/parser_with_options.yaml
index 862c125..acb4542 100644
--- a/config/testdata/parser_with_options.yaml
+++ b/config/testdata/parser_with_options.yaml
@@ -8,3 +8,10 @@ watchers:
       - additional_fields:
           foo: "1"
           bar: 2
+  - dataset: custom
+    parser:
+      name: regex
+      options:
+        expressions:
+          - "foo"
+          - "bar"
diff --git a/parsers/regex_test.go b/parsers/regex_test.go
new file mode 100644
index 0000000..e91ef2e
--- /dev/null
+++ b/parsers/regex_test.go
@@ -0,0 +1,45 @@
+package parsers
+
+import (
+	"testing"
+
+	"github.com/honeycombio/honeycomb-kubernetes-agent/config"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRegexParser(t *testing.T) {
+	cfg := &config.ParserConfig{
+		Name: "regex", Options: map[string]interface{}{
+			"expressions": []interface{}{
+				"(?P<species>[A-z]+) (?P<height>[0-9]{2}[0-9]?)",
+				"(?P<city>[A-z ]+),(?P<state>[A-z]{2})",
+			},
+		},
+	}
+
+	pf, err := NewParserFactory(cfg)
+	assert.Nil(t, err)
+	parser := pf.New()
+
+	tc := []struct {
+		line     string
+		expected map[string]interface{}
+		err      bool
+	}{
+		{line: "walnut 55", expected: map[string]interface{}{"species": "walnut", "height": "55"}, err: false},
+		{line: "douglas 105", expected: map[string]interface{}{"species": "douglas", "height": "105"}, err: false},
+		{line: "San Francisco,Ca", expected: map[string]interface{}{"city": "San Francisco", "state": "Ca"}, err: false},
+		{line: "South Lake Tahoe,CA", expected: map[string]interface{}{"city": "South Lake Tahoe", "state": "CA"}, err: false},
+		{line: "the quick brown fox jumped over the lazy dog", expected: nil, err: true},
+	}
+
+	for _, tt := range tc {
+		parsed, err := parser.Parse(tt.line)
+		assert.Equal(t, parsed, tt.expected)
+		if tt.err {
+			assert.NotNil(t, err)
+		} else {
+			assert.Nil(t, err)
+		}
+	}
+}