Repository: honeydipper/honeydipper
Pull request #253 (2020-05-29 17:03:57)

chore(release): release
#### Description

* fix(drivers): driver panic due to closed channel during retry (#251)
* feat(config): allow using params and sysData in function export (#252)
* feat(interpolation): allow indirect access in dollar interpolation (#252)


Patch:
diff --git a/drivers/cmd/gcloud-dataflow/main.go b/drivers/cmd/gcloud-dataflow/main.go
index ee57e03..a4d7b78 100644
--- a/drivers/cmd/gcloud-dataflow/main.go
+++ b/drivers/cmd/gcloud-dataflow/main.go
@@ -272,10 +272,12 @@ func waitForJob(msg *dipper.Message) {
 		result *dataflow.Job
 		err    error
 	)
+
+loop:
 	for {
 		select {
 		case <-expired:
-			break
+			break loop
 		default:
 			func() {
 				execContext, cancel := context.WithTimeout(context.Background(), time.Second*driver.APITimeout)
@@ -293,7 +295,7 @@ func waitForJob(msg *dipper.Message) {
 						"error": fmt.Sprintf("failed to call polling method: %+v", err),
 					},
 				}
-				break
+				break loop
 			}
 
 			if status, ok := terminatedStates[result.CurrentState]; ok {
@@ -306,7 +308,7 @@ func waitForJob(msg *dipper.Message) {
 						"reason": result.CurrentState,
 					},
 				}
-				break
+				break loop
 			}
 			time.Sleep(time.Duration(interval) * time.Second)
 		}
diff --git a/drivers/cmd/kubernetes/main.go b/drivers/cmd/kubernetes/main.go
index dade6b4..73291af 100644
--- a/drivers/cmd/kubernetes/main.go
+++ b/drivers/cmd/kubernetes/main.go
@@ -204,10 +204,11 @@ func waitForJob(m *dipper.Message) {
 
 	jobStatus := StatusSuccess
 	reason := []string{}
+loop:
 	for {
 		select {
 		case <-ctxWatch.Done():
-			break
+			break loop
 		case evt := <-jobstatus.ResultChan():
 			job := evt.Object.(*batchv1.Job)
 			if len(job.Status.Conditions) > 0 && job.Status.Active == 0 {
@@ -226,7 +227,7 @@ func waitForJob(m *dipper.Message) {
 						"reason": strings.Join(reason, "\n"),
 					},
 				}
-				break
+				break loop
 			}
 		}
 	}
diff --git a/go.sum b/go.sum
index 57b316b..1e13d46 100644
--- a/go.sum
+++ b/go.sum
@@ -302,6 +302,7 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
 github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.0/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70=
 github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4=
diff --git a/internal/config/collapsed_function.go b/internal/config/collapsed_function.go
deleted file mode 100644
index 5bcd016..0000000
--- a/internal/config/collapsed_function.go
+++ /dev/null
@@ -1,105 +0,0 @@
-// Copyright 2019 Honey Science Corporation
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, you can obtain one at http://mozilla.org/MPL/2.0/.
-
-package config
-
-import (
-	"strings"
-
-	"github.com/honeydipper/honeydipper/pkg/dipper"
-)
-
-// CollapsedFunction has all the function exports, parameters, sysData allow with the final raw function
-type CollapsedFunction struct {
-	Driver     string
-	RawAction  string
-	SysData    map[string]interface{}
-	Parameters map[string]interface{}
-	Stack      []*Function
-}
-
-// CollapseFunction collapse the function and all its exporting contexts
-func CollapseFunction(s *System, f *Function, cfg *Config) *CollapsedFunction {
-	var ret *CollapsedFunction
-
-	if len(f.Driver) == 0 {
-		childSystem, ok := cfg.DataSet.Systems[f.Target.System]
-		if !ok {
-			dipper.Logger.Panicf("[operator] system not defined %s", f.Target.System)
-		}
-		childFunction, ok := childSystem.Functions[f.Target.Function]
-		if !ok {
-			dipper.Logger.Panicf("[operator] function not defined %s.%s", f.Target.System, f.Target.Function)
-		}
-		ret = CollapseFunction(&childSystem, &childFunction, cfg)
-
-		// split subsystem from system
-		subsystems := strings.Split(f.Target.Function, ".")
-		for _, subsystem := range subsystems[:len(subsystems)-1] {
-			parent := ret.SysData
-			ret.SysData = parent[subsystem].(map[string]interface{})
-			ret.SysData["parent"] = parent
-		}
-	} else {
-		if len(f.Target.System) > 0 {
-			dipper.Logger.Panicf("[operator] function cannot have both driver and target %s.%s %s", f.Target.System, f.Target.Function, f.Driver)
-		}
-		ret = &CollapsedFunction{
-			Driver:    f.Driver,
-			RawAction: f.RawAction,
-		}
-	}
-
-	if s != nil && s.Data != nil {
-		currentSysDataCopy, _ := dipper.DeepCopyMap(s.Data)
-		if ret.SysData == nil {
-			ret.SysData = map[string]interface{}{}
-		}
-		ret.SysData = dipper.MergeMap(ret.SysData, currentSysDataCopy)
-	}
-	if f.Parameters != nil {
-		currentParamCopy, _ := dipper.DeepCopyMap(f.Parameters)
-		if ret.Parameters == nil {
-			ret.Parameters = map[string]interface{}{}
-		}
-		ret.Parameters = dipper.MergeMap(ret.Parameters, currentParamCopy)
-	}
-
-	ret.Stack = append(ret.Stack, f)
-	return ret
-}
-
-// ExportContext create a context data structure based on the collapsed function exports
-func (f *CollapsedFunction) ExportContext(status string, envData map[string]interface{}) (map[string]interface{}, map[string]interface{}) {
-	var exported map[string]interface{}
-	newCtx := envData["ctx"].(map[string]interface{})
-
-	if status != "error" && len(f.Stack) > 0 {
-		oldCtx := newCtx
-		newCtx = dipper.MustDeepCopyMap(oldCtx)
-		envData["ctx"] = newCtx
-
-		for _, layer := range f.Stack {
-			delta := dipper.Interpolate(layer.Export, envData)
-			newCtx = dipper.MergeMap(newCtx, delta)
-			exported = dipper.MergeMap(exported, delta)
-			switch status {
-			case "success":
-				delta := dipper.Interpolate(layer.ExportOnSuccess, envData)
-				newCtx = dipper.MergeMap(newCtx, delta)
-				exported = dipper.MergeMap(exported, delta)
-			case "failure":
-				delta := dipper.Interpolate(layer.ExportOnFailure, envData)
-				newCtx = dipper.MergeMap(newCtx, delta)
-				exported = dipper.MergeMap(exported, delta)
-			}
-		}
-
-		envData["ctx"] = oldCtx
-	}
-
-	return exported, newCtx
-}
diff --git a/internal/config/function_export.go b/internal/config/function_export.go
new file mode 100644
index 0000000..19f20d3
--- /dev/null
+++ b/internal/config/function_export.go
@@ -0,0 +1,108 @@
+// Copyright 2020 Honey Science Corporation
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, you can obtain one at http://mozilla.org/MPL/2.0/.
+
+package config
+
+import (
+	"strings"
+
+	"github.com/honeydipper/honeydipper/pkg/dipper"
+)
+
+// ExportFunctionContext create a context data structure based on the collapsed function exports
+func ExportFunctionContext(s *System, f *Function, envData map[string]interface{}, cfg *Config) map[string]interface{} {
+	var exported map[string]interface{}
+	status := "success"
+	if labelsData, ok := envData["labels"]; ok {
+		if labels, ok := labelsData.(map[string]string); ok {
+			if s, ok := labels["status"]; ok {
+				status = s
+			}
+		}
+	}
+
+	if status != "error" {
+		if len(f.Parameters) > 0 {
+
+			// the parameters need to be squashed before interpolation
+			// we will also need to provide the squashed sysData for interpolation
+			// the outter parameters should override inner parameters, but
+			// the inner sysData should override outter sysData
+			outterParam, _ := envData["params"]
+			envData["params"] = dipper.MergeMap(dipper.MustDeepCopyMap(f.Parameters), outterParam)
+			var outterSysData map[string]interface{}
+			if d, ok := envData["sysData"]; ok && d != nil {
+				outterSysData = d.(map[string]interface{})
+			}
+			if s != nil && s.Data != nil {
+				envData["sysData"] = dipper.MergeMap(outterSysData, dipper.MustDeepCopyMap(s.Data))
+			}
+		}
+
+		var sysData map[string]interface{}
+		if f.Target.System != "" {
+			childSystem, ok := cfg.DataSet.Systems[f.Target.System]
+			if !ok {
+				dipper.Logger.Panicf("[operator] system not defined %s", f.Target.System)
+			}
+			childFunction, ok := childSystem.Functions[f.Target.Function]
+			if !ok {
+				dipper.Logger.Panicf("[operator] function not defined %s.%s", f.Target.System, f.Target.Function)
+			}
+			exported = ExportFunctionContext(&childSystem, &childFunction, envData, cfg)
+
+			// split subsystem from system
+			if data, ok := envData["sysData"]; ok {
+				sysData = data.(map[string]interface{})
+			}
+			subsystems := strings.Split(f.Target.Function, ".")
+			for _, subsystem := range subsystems[:len(subsystems)-1] {
+				parent := sysData
+				sysData = parent[subsystem].(map[string]interface{})
+				sysData["parent"] = parent
+			}
+		} else {
+			if len(f.Target.System) > 0 {
+				dipper.Logger.Panicf("[operator] function cannot have both driver and target %s.%s %s", f.Target.System, f.Target.Function, f.Driver)
+			}
+
+			// here comes the interpolation for the squashed parameters. This interpolation only happens
+			// once, in the inner most function. After that, the parameters can be used for export in all outter
+			// layers.
+			squashedParams, _ := envData["params"]
+			envData["params"] = dipper.Interpolate(squashedParams, envData)
+		}
+
+		// here we abandon the squashed sysData after it is consumed, and use a clean sysData for
+		// interpolating the exported data.
+		if s != nil && s.Data != nil {
+			sysData = dipper.MergeMap(sysData, dipper.MustDeepCopyMap(s.Data))
+		}
+		envData["sysData"] = sysData
+		var newCtx map[string]interface{}
+
+		if newCtxData, ok := envData["ctx"]; ok && newCtxData != nil {
+			newCtx = newCtxData.(map[string]interface{})
+		}
+
+		delta := dipper.Interpolate(f.Export, envData)
+		exported = dipper.MergeMap(exported, dipper.MustDeepCopy(delta))
+		newCtx = dipper.MergeMap(newCtx, delta)
+		switch status {
+		case "success":
+			delta := dipper.Interpolate(f.ExportOnSuccess, envData)
+			exported = dipper.MergeMap(exported, dipper.MustDeepCopy(delta))
+			newCtx = dipper.MergeMap(newCtx, delta)
+		case "failure":
+			delta := dipper.Interpolate(f.ExportOnFailure, envData)
+			exported = dipper.MergeMap(exported, dipper.MustDeepCopy(delta))
+			newCtx = dipper.MergeMap(newCtx, delta)
+		}
+		envData["ctx"] = newCtx
+	}
+
+	return exported
+}
diff --git a/internal/workflow/continue.go b/internal/workflow/continue.go
index 6796703..d0de679 100644
--- a/internal/workflow/continue.go
+++ b/internal/workflow/continue.go
@@ -10,6 +10,7 @@ import (
 	"fmt"
 	"sync/atomic"
 
+	"github.com/honeydipper/honeydipper/internal/config"
 	"github.com/honeydipper/honeydipper/internal/daemon"
 	"github.com/honeydipper/honeydipper/pkg/dipper"
 )
@@ -115,10 +116,9 @@ func (w *Session) processExport(msg *dipper.Message) {
 		envData := w.buildEnvData(msg)
 		status := msg.Labels["status"]
 
-		if w.collapsedFunction != nil && status != SessionStatusError {
-			var export map[string]interface{}
-			export, w.ctx = w.collapsedFunction.ExportContext(status, envData)
-			envData["ctx"] = w.ctx
+		if w.inFlyFunction != nil && status != SessionStatusError {
+			export := config.ExportFunctionContext(nil, w.inFlyFunction, envData, w.store.Helper.GetConfig())
+			delete(envData, "sysData")
 			w.processNoExport(export)
 			if len(export) > 0 {
 				w.exported = append(w.exported, export)
diff --git a/internal/workflow/execute.go b/internal/workflow/execute.go
index 790b9a3..0a49d04 100644
--- a/internal/workflow/execute.go
+++ b/internal/workflow/execute.go
@@ -362,7 +362,7 @@ func (w *Session) callShorthandFunction(f string, msg *dipper.Message) {
 // callFunction makes a call to a function
 func (w *Session) callFunction(f *config.Function, msg *dipper.Message) {
 	// stored for doing export context later
-	w.collapsedFunction = config.CollapseFunction(nil, f, w.store.Helper.GetConfig())
+	w.inFlyFunction = f
 
 	payload := w.buildEnvData(msg)
 	payload["function"] = *f
diff --git a/internal/workflow/session.go b/internal/workflow/session.go
index da117e7..158e579 100644
--- a/internal/workflow/session.go
+++ b/internal/workflow/session.go
@@ -16,23 +16,23 @@ import (
 
 // Session is the data structure about a running workflow and its definition.
 type Session struct {
-	ID                string
-	workflow          *config.Workflow
-	current           int32 // current thread or step
-	iteration         int32 // current item in the iteration list
-	loopCount         int   // counter for looping
-	parent            string
-	ctx               map[string]interface{}
-	event             map[string]interface{}
-	exported          []map[string]interface{}
-	elseBranch        *config.Workflow
-	collapsedFunction *config.CollapsedFunction
-	store             *SessionStore
-	loadedContexts    []string
-	currentHook       string
-	savedMsg          *dipper.Message
-	performing        string
-	isHook            bool
+	ID             string
+	workflow       *config.Workflow
+	current        int32 // current thread or step
+	iteration      int32 // current item in the iteration list
+	loopCount      int   // counter for looping
+	parent         string
+	ctx            map[string]interface{}
+	event          map[string]interface{}
+	exported       []map[string]interface{}
+	elseBranch     *config.Workflow
+	inFlyFunction  *config.Function
+	store          *SessionStore
+	loadedContexts []string
+	currentHook    string
+	savedMsg       *dipper.Message
+	performing     string
+	isHook         bool
 }
 
 // SessionHandler prepare and execute the session provides entry point for SessionStore to invoke and mock for testing
diff --git a/pkg/dipper/commandHandler.go b/pkg/dipper/commandHandler.go
index 8541a24..9904da0 100644
--- a/pkg/dipper/commandHandler.go
+++ b/pkg/dipper/commandHandler.go
@@ -90,18 +90,22 @@ func (p *CommandProvider) Router(msg *Message) {
 	retry, timeout, backoff := p.UnpackLabels(msg)
 
 	var attempt func(chan Message)
-	attempt = func(rchan chan Message) {
-		msg.Reply = rchan
+	attempt = func(replyChannel chan Message) {
+		msg.Reply = replyChannel
+		m := *msg
 
 		go func() {
-			defer close(rchan)
+			defer func() {
+				close(replyChannel)
+				replyChannel = nil
+			}()
 
-			Logger.Debugf("[operaotr] cmd labels %+v", msg.Labels)
+			Logger.Debugf("[operaotr] cmd labels %+v", m.Labels)
 
 			select {
-			case reply := <-msg.Reply:
+			case reply := <-replyChannel:
 				if _, ok := reply.Labels["no-timeout"]; ok {
-					reply = <-msg.Reply
+					reply = <-m.Reply
 				}
 
 				_, hasError := reply.Labels["error"]
@@ -120,15 +124,15 @@ func (p *CommandProvider) Router(msg *Message) {
 		}()
 
 		defer func() {
-			if r := recover(); r != nil {
-				msg.Reply <- Message{
+			if r := recover(); r != nil && replyChannel != nil {
+				replyChannel <- Message{
 					Labels: map[string]string{
 						"error": fmt.Sprintf("%+v", r),
 					},
 				}
 			}
 		}()
-		f(msg)
+		f(&m)
 	}
 
 	attempt(make(chan Message, 1))
diff --git a/pkg/dipper/interpolation.go b/pkg/dipper/interpolation.go
index f7eb8f4..89f6d2b 100644
--- a/pkg/dipper/interpolation.go
+++ b/pkg/dipper/interpolation.go
@@ -72,23 +72,26 @@ func Interpolate(source interface{}, data interface{}) interface{} {
 	case string:
 		if strings.HasPrefix(v, "$") {
 			allowNull := (v[1] == '?')
-			quoteAt := strings.IndexAny(v, "\"'`")
-
-			var keys []string
+			var parsed string
 			if allowNull {
-				if quoteAt > 0 {
-					panic(fmt.Errorf("no need to allow null with default value %s", v))
-				}
-				keys = strings.Split(v[2:], ",")
+				parsed = InterpolateStr(v[2:], data)
 			} else {
-				if quoteAt > 0 {
-					if v[quoteAt-1] != ',' {
-						panic(fmt.Errorf("default value should be separated from the key with comma %s", v))
-					}
-					keys = strings.Split(v[1:quoteAt-1], ",")
-				} else {
-					keys = strings.Split(v[1:], ",")
+				parsed = InterpolateStr(v[1:], data)
+			}
+
+			quote := strings.IndexAny(parsed, "\"'`")
+			if allowNull && quote >= 0 {
+				panic(fmt.Errorf("no need to allow null with default value %s", v))
+			}
+
+			var keys []string
+			if quote > 0 {
+				if parsed[quote-1] != ',' {
+					panic(fmt.Errorf("default value should be separated from the key with comma %s", v))
 				}
+				keys = strings.Split(parsed[:quote-1], ",")
+			} else if quote < 0 {
+				keys = strings.Split(parsed, ",")
 			}
 
 			for _, key := range keys {
@@ -101,11 +104,11 @@ func Interpolate(source interface{}, data interface{}) interface{} {
 				}
 			}
 
-			if quoteAt > 0 {
-				if v[quoteAt] != v[len(v)-1] {
-					panic(fmt.Errorf("quotes not matching for default value %s", v))
+			if quote >= 0 {
+				if parsed[quote] != parsed[len(parsed)-1] {
+					panic(fmt.Errorf("quotes not matching for default value %s", parsed))
 				}
-				return v[quoteAt+1 : len(v)-1]
+				return parsed[quote+1 : len(parsed)-1]
 			}
 
 			if allowNull {


Test patch:
diff --git a/internal/config/function_export_test.go b/internal/config/function_export_test.go
new file mode 100644
index 0000000..3ecc447
--- /dev/null
+++ b/internal/config/function_export_test.go
@@ -0,0 +1,242 @@
+// Copyright 2020 Honey Science Corporation
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, you can obtain one at http://mozilla.org/MPL/2.0/.
+
+// +build !integration
+
+package config
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFunctionExport(t *testing.T) {
+	cfg := Config{
+		DataSet: &DataSet{
+			Systems: map[string]System{
+				"test": System{
+					Data: map[string]interface{}{
+						"key1": "value1",
+						"key2": map[string]string{
+							"key3": "value3",
+						},
+					},
+					Functions: map[string]Function{
+						"testFunc1": Function{
+							Driver:    "testDriver",
+							RawAction: "action",
+							Export: map[string]interface{}{
+								"url": "http://{{ .sysData.key1 }}.{{ .sysData.key2.key3}}/{{ .ctx.path }}",
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	result := ExportFunctionContext(
+		nil,
+		&Function{Target: Action{System: "test", Function: "testFunc1"}},
+		map[string]interface{}{
+			"ctx": map[string]interface{}{"path": "test/path1"},
+		},
+		&cfg,
+	)
+
+	assert.Equal(t, "http://value1.value3/test/path1", result["url"])
+
+	newSys1 := &System{
+		Data: map[string]interface{}{
+			"key1": "value11",
+			"key2": "value22",
+		},
+	}
+
+	result = ExportFunctionContext(
+		newSys1,
+		&Function{Target: Action{System: "test", Function: "testFunc1"}},
+		map[string]interface{}{
+			"ctx": map[string]interface{}{"path": "test/path1"},
+		},
+		&cfg,
+	)
+
+	assert.Equal(t, "http://value1.value3/test/path1", result["url"])
+}
+
+func TestFunctionExportWithParameters(t *testing.T) {
+	cfg := Config{
+		DataSet: &DataSet{
+			Systems: map[string]System{
+				"test": System{
+					Data: map[string]interface{}{
+						"key1": "value1",
+						"key2": map[string]string{
+							"key3": "value3",
+						},
+					},
+					Functions: map[string]Function{
+						"testFunc1": Function{
+							Driver:    "testDriver",
+							RawAction: "action",
+							Parameters: map[string]interface{}{
+								"q1": "param1",
+								"q2": "param2",
+							},
+							Export: map[string]interface{}{
+								"url": "http://{{ .sysData.key1 }}.{{ .sysData.key2.key3}}/{{ .ctx.path }}?q1={{ .params.q1 }}",
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	result := ExportFunctionContext(
+		nil,
+		&Function{Target: Action{System: "test", Function: "testFunc1"}},
+		map[string]interface{}{
+			"ctx": map[string]interface{}{"path": "test/path1"},
+		},
+		&cfg,
+	)
+
+	assert.Equal(t, "http://value1.value3/test/path1?q1=param1", result["url"])
+
+	result = ExportFunctionContext(
+		nil,
+		&Function{
+			Target: Action{System: "test", Function: "testFunc1"},
+			Parameters: map[string]interface{}{
+				"q1": "updated",
+			},
+		},
+		map[string]interface{}{
+			"ctx": map[string]interface{}{"path": "test/path1"},
+		},
+		&cfg,
+	)
+
+	assert.Equal(t, "http://value1.value3/test/path1?q1=updated", result["url"])
+}
+
+func TestFunctionExportWithSquashedParameters(t *testing.T) {
+	cfg := Config{
+		DataSet: &DataSet{
+			Systems: map[string]System{
+				"test": System{
+					Data: map[string]interface{}{
+						"key1": "this_should_not_be_used",
+						"key2": map[string]string{
+							"key3": "value3",
+						},
+					},
+					Functions: map[string]Function{
+						"testFunc1": Function{
+							Driver:    "testDriver",
+							RawAction: "action",
+							Parameters: map[string]interface{}{
+								"q1": "this_should_not_be_used",
+								"q2": "param2",
+							},
+						},
+					},
+				},
+				"test2": System{
+					Data: map[string]interface{}{
+						"key1": "value1",
+						"key2": map[string]string{
+							"key3": "value3",
+						},
+					},
+					Functions: map[string]Function{
+						"testFunc2": Function{
+							Target: Action{
+								System:   "test",
+								Function: "testFunc1",
+							},
+							Parameters: map[string]interface{}{
+								"q1": "param1",
+								"q2": "param2",
+							},
+							Export: map[string]interface{}{
+								"url": "http://{{ .sysData.key1 }}.{{ .sysData.key2.key3}}/{{ .ctx.path }}?q1={{ .params.q1 }}",
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	result := ExportFunctionContext(
+		nil,
+		&Function{Target: Action{System: "test2", Function: "testFunc2"}},
+		map[string]interface{}{
+			"ctx": map[string]interface{}{"path": "test/path1"},
+		},
+		&cfg,
+	)
+
+	assert.Equal(t, "http://value1.value3/test/path1?q1=param1", result["url"])
+}
+
+func TestFunctionExportWithSquashedSysData(t *testing.T) {
+	cfg := Config{
+		DataSet: &DataSet{
+			Systems: map[string]System{
+				"test": System{
+					Data: map[string]interface{}{
+						"key4": "value4",
+					},
+					Functions: map[string]Function{
+						"testFunc1": Function{
+							Driver:    "testDriver",
+							RawAction: "action",
+							Parameters: map[string]interface{}{
+								"q1": "$sysData.key4",
+							},
+						},
+					},
+				},
+				"test2": System{
+					Data: map[string]interface{}{
+						"key1": "value1",
+						"key2": map[string]string{
+							"key3": "value3",
+						},
+						"key4": "this_should_not_be_used",
+					},
+					Functions: map[string]Function{
+						"testFunc2": Function{
+							Target: Action{
+								System:   "test",
+								Function: "testFunc1",
+							},
+							Export: map[string]interface{}{
+								"url": "http://{{ .sysData.key1 }}.{{ .sysData.key2.key3}}/{{ .ctx.path }}?q1={{ .params.q1 }}",
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	result := ExportFunctionContext(
+		nil,
+		&Function{Target: Action{System: "test2", Function: "testFunc2"}},
+		map[string]interface{}{
+			"ctx": map[string]interface{}{"path": "test/path1"},
+		},
+		&cfg,
+	)
+
+	assert.Equal(t, "http://value1.value3/test/path1?q1=value4", result["url"])
+}
diff --git a/internal/config/main_test.go b/internal/config/main_test.go
new file mode 100644
index 0000000..1361dc6
--- /dev/null
+++ b/internal/config/main_test.go
@@ -0,0 +1,26 @@
+// Copyright 2019 Honey Science Corporation
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, you can obtain one at http://mozilla.org/MPL/2.0/.
+
+package config
+
+import (
+	"os"
+	"testing"
+
+	"github.com/honeydipper/honeydipper/pkg/dipper"
+)
+
+func TestMain(m *testing.M) {
+	if dipper.Logger == nil {
+		logFile, err := os.Create("test.log")
+		if err != nil {
+			panic(err)
+		}
+		defer logFile.Close()
+		dipper.GetLogger("test", "INFO", logFile, logFile)
+	}
+	os.Exit(m.Run())
+}
diff --git a/pkg/dipper/commandHandler_test.go b/pkg/dipper/commandHandler_test.go
index 3d852d9..a70086b 100644
--- a/pkg/dipper/commandHandler_test.go
+++ b/pkg/dipper/commandHandler_test.go
@@ -11,13 +11,14 @@ package dipper
 import (
 	"bytes"
 	"fmt"
+	"strconv"
 	"testing"
 	"time"
 
 	"github.com/stretchr/testify/assert"
 )
 
-func TestCommandRetry(t *testing.T) {
+func TestCommandRetrySuccess(t *testing.T) {
 	var b = bytes.Buffer{}
 	counter := 0
 
@@ -61,10 +62,32 @@ func TestCommandRetry(t *testing.T) {
 	assert.Equal(t, 2, counter, "should fail twice")
 	ret := FetchMessage(&b)
 	assert.Equal(t, "success", ret.Labels["status"], "should return success after retry")
+}
+
+func TestCommandRetryFailure(t *testing.T) {
+	var b = bytes.Buffer{}
+	counter := 0
 
 	b.Reset()
 	counter = 0
-	m = &Message{
+
+	subject := CommandProvider{
+		ReturnWriter: &b,
+		Channel:      "test",
+		Subject:      "test",
+
+		Commands: map[string]MessageHandler{
+			"test": func(m *Message) {
+				if counter < 2 {
+					counter++
+					panic(fmt.Errorf("please retry"))
+				}
+				m.Reply <- Message{}
+			},
+		},
+	}
+
+	m := &Message{
 		Labels: map[string]string{
 			"method":     "test",
 			"retry":      "1",
@@ -75,7 +98,7 @@ func TestCommandRetry(t *testing.T) {
 
 	subject.Router(m)
 
-	sleepCount = 0
+	sleepCount := 0
 	for sleepCount < 100 && m.Reply != nil {
 		time.Sleep(time.Millisecond * 5)
 		sleepCount++
@@ -86,6 +109,80 @@ func TestCommandRetry(t *testing.T) {
 	}
 
 	assert.Equal(t, 2, counter, "should fail twice")
-	ret = FetchMessage(&b)
+	ret := FetchMessage(&b)
 	assert.Equal(t, "error", ret.Labels["status"], "should return error after retry error")
 }
+
+func TestCommandRetryRougueFunction(t *testing.T) {
+	var b = bytes.Buffer{}
+	counter := 0
+
+	b.Reset()
+	counter = 0
+
+	subject := CommandProvider{
+		ReturnWriter: &b,
+		Channel:      "test",
+		Subject:      "test",
+
+		Commands: map[string]MessageHandler{
+			"test": func(m *Message) {
+				switch counter {
+				case 0:
+					counter++
+					panic(fmt.Errorf("please retry"))
+				case 1:
+					counter++
+					m.Reply <- Message{
+						Labels: map[string]string{
+							"error": "second failure 1",
+						},
+					}
+					m.Reply <- Message{
+						Labels: map[string]string{
+							"error": "second failure 2",
+						},
+					}
+					m.Reply <- Message{
+						Labels: map[string]string{
+							"error": "second failure 3",
+						},
+					}
+				case 2:
+					counter++
+				}
+				m.Reply <- Message{
+					Payload: map[string]interface{}{
+						"counter": strconv.Itoa(counter),
+					},
+				}
+			},
+		},
+	}
+
+	m := &Message{
+		Labels: map[string]string{
+			"method":     "test",
+			"retry":      "2",
+			"sessionID":  "2",
+			"backoff_ms": "10",
+		},
+	}
+
+	subject.Router(m)
+
+	sleepCount := 0
+	for sleepCount < 100 && m.Reply != nil {
+		time.Sleep(time.Millisecond * 5)
+		sleepCount++
+	}
+
+	if sleepCount == 100 {
+		assert.Fail(t, "timeout waiting for retry to send result")
+	}
+
+	assert.Equal(t, 3, counter, "should fail twice")
+	ret := FetchMessage(&b)
+	assert.Equal(t, "success", ret.Labels["status"], "should return error after retry error")
+	assert.Equal(t, "3", ret.Payload.(map[string]interface{})["counter"], "should return the final counter")
+}
diff --git a/pkg/dipper/interpolation_test.go b/pkg/dipper/interpolation_test.go
index ca3e930..4254104 100644
--- a/pkg/dipper/interpolation_test.go
+++ b/pkg/dipper/interpolation_test.go
@@ -34,6 +34,7 @@ func TestInterpolate(t *testing.T) {
 				"deep": " another {{ index . \"type\" }}",
 			},
 			"default_user": "$ctx.v1,ctx.v2,'default, value with comma'",
+			"item_in_list": "$list.{{ .ptr }}",
 			"yaml_with_template": `:yaml:
 ---
 test:
@@ -44,9 +45,11 @@ test:
 			"user": "test",
 			"type": "direct",
 			"list": map[string]interface{}{
-				"one": "one",
-				"two": "two",
+				"one":   "one",
+				"two":   "two",
+				"three": "the last one",
 			},
+			"ptr": "three",
 		})
 	assert.EqualValues(t,
 		map[string]interface{}{
@@ -55,6 +58,7 @@ test:
 			"map_with_template": map[string]interface{}{
 				"deep": " another direct",
 			},
+			"item_in_list": "the last one",
 			"yaml_with_template": map[string]interface{}{
 				"test": []interface{}{
 					"1 one",