Repository: honeydipper/honeydipper
Pull request #420 (2021-06-30 16:37:11)

chore(release): release
* feat(drivers): webhook hmac support secret list (#417)
* feat(drivers): datadog-emitter allow functions (#418)


Patch:
diff --git a/Makefile b/Makefile
index fd76ecb..d46a10c 100644
--- a/Makefile
+++ b/Makefile
@@ -7,7 +7,8 @@ files_require_mocking = internal/workflow/session.go \
 						internal/workflow/store.go \
 						pkg/dipper/rpc.go \
 						internal/api/request_context.go \
-						drivers/cmd/gcloud-secret/main.go
+						drivers/cmd/gcloud-secret/main.go \
+						drivers/cmd/datadog-emitter/statsd.go
 
 ifneq (,$(wildcard ./.env))
 	include ./.env
diff --git a/drivers/cmd/datadog-emitter/main.go b/drivers/cmd/datadog-emitter/main.go
index 3f544cc..eca72ea 100644
--- a/drivers/cmd/datadog-emitter/main.go
+++ b/drivers/cmd/datadog-emitter/main.go
@@ -38,7 +38,8 @@ var (
 	driver         *dipper.Driver
 	log            *logging.Logger
 	datadogOptions DatadogOptions
-	dogstatsd      *statsd.Client
+	dogstatsd      virtualStatsd
+	mockedstatsd   virtualStatsd
 	daemonID       string
 )
 
@@ -50,6 +51,8 @@ func main() {
 	driver = dipper.NewDriver(os.Args[1], "datadog-emitter")
 	driver.RPCHandlers["counter_increment"] = counterIncr
 	driver.RPCHandlers["gauge_set"] = gaugeSet
+	driver.Commands["counter_increment"] = counterIncr
+	driver.Commands["gauge_set"] = gaugeSet
 	driver.Reload = loadOptions
 	driver.Start = loadOptions
 	driver.Run()
@@ -76,7 +79,7 @@ func loadOptions(msg *dipper.Message) {
 	if dogstatsd != nil {
 		dogstatsd.Close()
 	}
-	dogstatsd, err = statsd.New(datadogOptions.StatsdHost + ":" + datadogOptions.StatsdPort)
+	dogstatsd, err = newStatsd(datadogOptions.StatsdHost + ":" + datadogOptions.StatsdPort)
 	if err != nil {
 		panic(err)
 	}
@@ -100,6 +103,9 @@ func counterIncr(msg *dipper.Message) {
 	}
 
 	dipper.Must(dogstatsd.Incr(name, tags, 1))
+	if msg.Reply != nil {
+		msg.Reply <- dipper.Message{}
+	}
 }
 
 func gaugeSet(msg *dipper.Message) {
@@ -119,4 +125,7 @@ func gaugeSet(msg *dipper.Message) {
 	}
 
 	dipper.Must(dogstatsd.Gauge(name, value, tags, 1))
+	if msg.Reply != nil {
+		msg.Reply <- dipper.Message{}
+	}
 }
diff --git a/drivers/cmd/datadog-emitter/statsd.go b/drivers/cmd/datadog-emitter/statsd.go
new file mode 100644
index 0000000..1f9a3a1
--- /dev/null
+++ b/drivers/cmd/datadog-emitter/statsd.go
@@ -0,0 +1,25 @@
+// Copyright 2021 Honey Science Corporation
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, you can obtain one at http://mozilla.org/MPL/2.0/.
+
+// Package datadog-emitter enables Honeydipper to send metrics to datadog
+package main
+
+import "github.com/DataDog/datadog-go/statsd"
+
+type virtualStatsd interface {
+	Close() error
+	Event(*statsd.Event) error
+	Incr(string, []string, float64) error
+	Gauge(string, float64, []string, float64) error
+}
+
+func newStatsd(datadogOptStr string) (virtualStatsd, error) {
+	if mockedstatsd != nil {
+		return mockedstatsd, nil
+	}
+
+	return statsd.New(datadogOptStr)
+}
diff --git a/drivers/cmd/webhook/main.go b/drivers/cmd/webhook/main.go
index b0160bf..bbfd8a0 100644
--- a/drivers/cmd/webhook/main.go
+++ b/drivers/cmd/webhook/main.go
@@ -260,27 +260,37 @@ func verifySystems(eventData map[string]interface{}) {
 		}
 	}
 
-	if signatureHeader != "" {
-		var verifiedSystem []string
-		for name, sys := range sysMap {
-			expectedHeader, ok := sys["signatureHeader"]
-			if !ok || !strings.EqualFold(expectedHeader.(string), signatureHeader) {
-				// ignore unsupported headers or system without signatureHeader
-				continue
-			}
+	if signatureHeader == "" {
+		return
+	}
 
-			secret, ok := dipper.GetMapDataStr(sys, "signatureSecret")
-			if !ok {
-				log.Warningf("[%s] signature secret not defined for system %s", driver.Service, name)
+	var verifiedSystem []string
+	for name, sys := range sysMap {
+		expectedHeader, ok := sys["signatureHeader"]
+		if !ok || !strings.EqualFold(expectedHeader.(string), signatureHeader) {
+			// ignore unsupported headers or system without signatureHeader
+			continue
+		}
 
-				continue
-			}
+		secretValue, ok := dipper.GetMapData(sys, "signatureSecret")
+		if !ok {
+			log.Warningf("[%s] signature secret not defined for system %s", driver.Service, name)
 
-			if verifySignature(signatureHeader, signatureValue, secret, eventData) {
+			continue
+		}
+		secrets, ok := secretValue.([]interface{})
+		if !ok {
+			secrets = []interface{}{secretValue}
+		}
+
+		for _, secret := range secrets {
+			if verifySignature(signatureHeader, signatureValue, secret.(string), eventData) {
 				verifiedSystem = append(verifiedSystem, name)
+
+				break
 			}
 		}
-		log.Infof("[%s] HMAC verified for system(s) %+v", driver.Service, verifiedSystem)
-		eventData["verifiedSystem"] = verifiedSystem
 	}
+	log.Infof("[%s] HMAC verified for system(s) %+v", driver.Service, verifiedSystem)
+	eventData["verifiedSystem"] = verifiedSystem
 }
diff --git a/internal/service/operator.go b/internal/service/operator.go
index 60a608c..e12f203 100644
--- a/internal/service/operator.go
+++ b/internal/service/operator.go
@@ -69,13 +69,16 @@ func handleEventbusCommand(msg *dipper.Message) []RoutedMessage {
 		dipper.Logger.Panicf("[operator] invalid function received")
 	}
 
-	var driver string
-	var params map[string]interface{}
 	dipper.Logger.Debugf("[operator] collapsing function %s %s %+v", function.Target.System, function.Target.Function, function.Parameters)
 	driver, rawaction, params, sysData := collapseFunction(nil, &function)
 	dipper.Logger.Debugf("[operator] collapsed function %s %s %+v", driver, rawaction, params)
 
-	worker := operator.getDriverRuntime("driver:" + driver)
+	feature := "driver:" + driver
+	if strings.HasPrefix(driver, "feature:") {
+		feature = driver[8:]
+	}
+	worker := operator.getDriverRuntime(feature)
+
 	if worker == nil {
 		panic(fmt.Errorf("%w: not defined: %s", ErrOperatorError, driver))
 	}
@@ -114,28 +117,25 @@ func handleEventbusCommand(msg *dipper.Message) []RoutedMessage {
 		"ctx":    ctx,
 		"params": finalParams,
 	})
+	delete(msg.Labels, "retry")
 	if retry != "" {
 		msg.Labels["retry"] = retry
-	} else {
-		delete(msg.Labels, "retry")
 	}
 	backoff := dipper.InterpolateStr("$?ctx.backoff_ms,params.backoff_ms", map[string]interface{}{
 		"ctx":    ctx,
 		"params": finalParams,
 	})
+	delete(msg.Labels, "backoff_ms")
 	if backoff != "" {
 		msg.Labels["backoff_ms"] = backoff
-	} else {
-		delete(msg.Labels, "backoff_ms")
 	}
 	timeout := dipper.InterpolateStr("$?ctx.timeout,params.timeout", map[string]interface{}{
 		"ctx":    ctx,
 		"params": finalParams,
 	})
+	delete(msg.Labels, "timeout")
 	if timeout != "" {
 		msg.Labels["timeout"] = timeout
-	} else {
-		delete(msg.Labels, "timeout")
 	}
 
 	return []RoutedMessage{


Test patch:
diff --git a/drivers/cmd/datadog-emitter/main_test.go b/drivers/cmd/datadog-emitter/main_test.go
new file mode 100644
index 0000000..d4ad604
--- /dev/null
+++ b/drivers/cmd/datadog-emitter/main_test.go
@@ -0,0 +1,94 @@
+// Copyright 2021 Honey Science Corporation
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, you can obtain one at http://mozilla.org/MPL/2.0/.
+
+// +build !integration
+
+package main
+
+import (
+	"os"
+	"testing"
+
+	"github.com/golang/mock/gomock"
+	mock_driver "github.com/honeydipper/honeydipper/drivers/cmd/datadog-emitter/mock_datadog-emitter"
+	"github.com/honeydipper/honeydipper/pkg/dipper"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMain(m *testing.M) {
+	if dipper.Logger == nil {
+		logFile, err := os.Create("test.log")
+		if err != nil {
+			panic(err)
+		}
+		defer logFile.Close()
+		dipper.GetLogger("test", "INFO", logFile, logFile)
+	}
+	driver = &dipper.Driver{Service: "test"}
+	m.Run()
+}
+
+func TestIncrCmd(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	_mockedstatsd := mock_driver.NewMockvirtualStatsd(ctrl)
+	dogstatsd = _mockedstatsd
+	daemonID = "1.1.1.1"
+
+	inMsg := &dipper.Message{
+		Payload: map[string]interface{}{
+			"name": "test.test.counter",
+			"tags": []interface{}{
+				"app:test-app",
+			},
+		},
+		Reply: make(chan dipper.Message, 10),
+	}
+	_mockedstatsd.EXPECT().Incr(gomock.Eq("test.test.counter"), gomock.Eq([]string{"daemon-id:1.1.1.1", "app:test-app"}), float64(1)).Times(1).Return(nil)
+	assert.NotPanics(t, func() { counterIncr(inMsg) }, "counterIncr should not panic")
+	outMsg := <-inMsg.Reply
+	assert.Equal(t, dipper.Message{}, outMsg, "counterIncr should return an empty message upon success")
+}
+
+func TestGaugeCmd(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	_mockedstatsd := mock_driver.NewMockvirtualStatsd(ctrl)
+	dogstatsd = _mockedstatsd
+	daemonID = "1.1.1.1"
+
+	inMsg := &dipper.Message{
+		Payload: map[string]interface{}{
+			"name":  "test.test.counter",
+			"value": "0.6",
+			"tags": []interface{}{
+				"app:test-app",
+			},
+		},
+		Reply: make(chan dipper.Message, 10),
+	}
+	_mockedstatsd.EXPECT().Gauge(gomock.Eq("test.test.counter"), float64(0.6), gomock.Eq([]string{"daemon-id:1.1.1.1", "app:test-app"}), float64(1)).Times(1).Return(nil)
+	assert.NotPanics(t, func() { gaugeSet(inMsg) }, "gaugeSet should not panic")
+	outMsg := <-inMsg.Reply
+	assert.Equal(t, dipper.Message{}, outMsg, "gaugeSet should return an empty message upon success")
+}
+
+func TestLoadOptions(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	_mockedstatsd := mock_driver.NewMockvirtualStatsd(ctrl)
+	mockedstatsd = _mockedstatsd
+	dogstatsd = _mockedstatsd
+	daemonID = "1.1.1.1"
+
+	driver.Options = map[string]interface{}{
+		"data": map[string]interface{}{
+			"statsdHost": "2.2.2.2",
+			"statsdPort": "1234",
+		},
+	}
+
+	_mockedstatsd.EXPECT().Close().Times(1)
+	_mockedstatsd.EXPECT().Event(gomock.Any()).Times(1).Return(nil)
+	assert.NotPanics(t, func() { loadOptions(&dipper.Message{}) }, "loadOptions should not panic")
+}
diff --git a/drivers/cmd/webhook/main_test.go b/drivers/cmd/webhook/main_test.go
index 6d15a2d..2607be4 100644
--- a/drivers/cmd/webhook/main_test.go
+++ b/drivers/cmd/webhook/main_test.go
@@ -157,6 +157,13 @@ func TestHookHandler(t *testing.T) {
 			"signatureHeader": "x-pagerduty-signature",
 			"signatureSecret": "test-secret",
 		},
+		"sys-secret-list": map[string]interface{}{
+			"signatureHeader": "x-pagerduty-signature",
+			"signatureSecret": []interface{}{
+				"test-secret1",
+				"test-secret2",
+			},
+		},
 		"sys-unsupported-header": map[string]interface{}{
 			"signatureHeader": "x-unknown-signature",
 		},
@@ -183,6 +190,11 @@ func TestHookHandler(t *testing.T) {
 				"match": map[string]interface{}{"verifiedSystem": "sys"},
 			},
 		},
+		"sys5.webhook": []interface{}{
+			map[string]interface{}{
+				"match": map[string]interface{}{"verifiedSystem": "sys-secret-list"},
+			},
+		},
 	}
 
 	buf := bytes.NewBuffer(make([]byte, 2048))
@@ -232,4 +244,17 @@ func TestHookHandler(t *testing.T) {
 	hookHandler(resp, req)
 	assert.Equalf(t, 404, resp.status, "should return 404 with unsupported signature header")
 	assert.Zerof(t, buf.Len(), "should not emit event to daemon")
+
+	resp = &mockResponseWriter{header: http.Header{}}
+	req = &http.Request{
+		Method: "POST",
+		URL:    &url.URL{Path: "/test/sys5"},
+		Header: http.Header{"X-Pagerduty-Signature": []string{"v1=7afdf53eec1c15fb75269b28fab95228ba591b11a103d8e0972087e6dee018ca"}},
+		Body:   ioutil.NopCloser(bytes.NewBufferString("hello")),
+	}
+	hookHandler(resp, req)
+	msg = dipper.FetchMessage(buf)
+	assert.Equalf(t, 200, resp.status, "should return 200 on success with proper signature")
+	assert.Equalf(t, "sys-secret-list", dipper.MustGetMapDataStr(msg.Payload, "data.verifiedSystem.0"), "should emit webhook event with verifiedSystem")
+	assert.Equalf(t, 1, len(dipper.MustGetMapData(msg.Payload, "data.verifiedSystem").([]interface{})), "should verify only 1 system")
 }