Repository: honeydipper/honeydipper
Pull request #360 (2020-12-18 18:58:19)

chore(release): release
 * d53b0e3 fix(config): avoid concurrent map write in config (#358)
 * a7b8380 fix(workflow): avoid concurrent map write in mergeContext (#357)
 * 6bc1b5d fix(internal): remove wfdata interpolation (#355)
 * 5918d0f fix(deps): update google.golang.org/genproto commit hash to 06b3db8 (#346)
 * 40df4a1 fix(deps): update golang.org/x/oauth2 commit hash to 0b49973 (#345)
 * 81c9043 fix(logging): switch to golang.org/x/term (#350)
 * 76f40b2 chore(deps): update golang docker tag to v1.15.5 (#347)
 * a811f4b refactor: using builtin errors
 * 363dc85 fix(deps): update to aurora v3 (#341)
 * 1417aa8 fix(deps): update golang.org/x/crypto commit hash to 9e8e0b3 (#333)
 * 13a2cb4 fix(deps): update google.golang.org/genproto commit hash to 2e45c02 (#334)


Patch:
diff --git a/.circleci/config.yml b/.circleci/config.yml
index 68d3595..1fda9ab 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -1,14 +1,14 @@
 ---
 defaults: &defaults
   docker:
-    - image: golang:1.15.2-alpine
+    - image: golang:1.15.5-alpine
       environment:
         - GO111MODULE=auto
   working_directory: /go/src/github.com/honeydipper/honeydipper
 
 license_finder: &license_finder
   docker:
-    - image: licensefinder/license_finder:6.8.2
+    - image: licensefinder/license_finder:6.10.0
       environment:
         - GO111MODULE=auto
   working_directory: /root/src/honeydipper
@@ -25,19 +25,10 @@ save_mod: &save_mod
 
 version: 2
 jobs:
-  lint:
-    <<: *defaults
-    steps:
-      - run: apk add git openssh-client
-      - checkout
-      - run: go get -u golang.org/x/lint/golint
-      - run: go get -u mvdan.cc/gofumpt
-      - run: gofumpt -s -w .
-      - run: golint
   golangci-lint:
     <<: *defaults
     docker:
-      - image: golangci/golangci-lint:v1.31.0-alpine
+      - image: golangci/golangci-lint:v1.33.0-alpine
     steps:
       - checkout
       - run: golangci-lint run
@@ -61,8 +52,8 @@ jobs:
   integration:
     <<: *defaults
     docker:
-      - image: golang:1.15.2-alpine
-      - image: redis:5.0.9-alpine
+      - image: golang:1.15.5-alpine
+      - image: redis:5.0.10-alpine
     steps:
       - run:
           name: install dependencies
@@ -94,10 +85,7 @@ workflows:
   version: 2
   run_tests:
     jobs:
-      - lint
-      - golangci-lint:
-          requires:
-            - lint
+      - golangci-lint
       - license_finder
       - test:
           requires:
diff --git a/.dependency_decisions.yml b/.dependency_decisions.yml
index 1359027..8efb251 100644
--- a/.dependency_decisions.yml
+++ b/.dependency_decisions.yml
@@ -233,12 +233,6 @@
     :why: Apache-2.0
     :versions: []
     :when: 2020-05-18 22:08:50.941564000 Z
-- - :approve
-  - github.com/logrusorgru/aurora
-  - :who: 
-    :why: DWTFYWTPL
-    :versions: []
-    :when: 2020-05-18 22:16:03.105598000 Z
 - - :approve
   - github.com/hashicorp/golang-lru
   - :who: 
@@ -251,3 +245,9 @@
     :why: 
     :versions: []
     :when: 2020-06-02 18:09:01.017874000 Z
+- - :approve
+  - github.com/logrusorgru/aurora/v3
+  - :who: wby
+    :why: Unlicense
+    :versions: []
+    :when: 2020-11-07 22:16:03.105598000 Z
diff --git a/.golangci.yml b/.golangci.yml
index 563a48e..870c274 100644
--- a/.golangci.yml
+++ b/.golangci.yml
@@ -1,16 +1,17 @@
 # golangci.com configuration
 # https://github.com/golangci/golangci/wiki/Configuration
 service:
-  golangci-lint-version: 1.29.x # use fixed version to not introduce new linters unexpectedly
+  golangci-lint-version: v1.33.x # use fixed version to not introduce new linters unexpectedly
 
 linters:
   enable-all: true
   disable:
+    - exhaustive
+    - exhaustivestruct
     # code to be refactored at some point
     # but seems necessary to have global variables
     - gochecknoglobals
     - wsl
-    - exhaustive
 
 linters-settings:
   lll:
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 5be1132..a84519e 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -11,7 +11,7 @@ repos:
     - id: golangci-lint
 #   - id: go-critic
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v2.2.3
+  rev: v2.5.0
   hooks:
     - id: check-yaml
       exclude: ^deployments/helm/honeydipper/
diff --git a/build/docker/Dockerfile b/build/docker/Dockerfile
index bb5f79f..d02b56c 100644
--- a/build/docker/Dockerfile
+++ b/build/docker/Dockerfile
@@ -1,4 +1,4 @@
-FROM golang:1.15.2-alpine AS build
+FROM golang:1.15.5-alpine AS build
 
 ARG GOINSTALLOPTS
 ARG GOGC
diff --git a/cmd/honeydipper/configcheck.go b/cmd/honeydipper/configcheck.go
index 8ddd200..6636e36 100644
--- a/cmd/honeydipper/configcheck.go
+++ b/cmd/honeydipper/configcheck.go
@@ -17,7 +17,7 @@ import (
 	"github.com/honeydipper/honeydipper/internal/api"
 	"github.com/honeydipper/honeydipper/internal/config"
 	"github.com/honeydipper/honeydipper/pkg/dipper"
-	"github.com/logrusorgru/aurora"
+	"github.com/logrusorgru/aurora/v3"
 	"github.com/mitchellh/mapstructure"
 )
 
diff --git a/cmd/honeydipper/main.go b/cmd/honeydipper/main.go
index 2114fe4..767e530 100644
--- a/cmd/honeydipper/main.go
+++ b/cmd/honeydipper/main.go
@@ -61,6 +61,9 @@ func initEnv() {
 	initFlags()
 	flag.Parse()
 	cfg = config.Config{InitRepo: config.RepoInfo{}, Services: flag.Args()}
+	if len(cfg.Services) == 0 {
+		cfg.Services = []string{"engine", "receiver", "operator", "api"}
+	}
 
 loop:
 	for _, s := range cfg.Services {
@@ -110,11 +113,7 @@ func start() {
 	// reset logging with configured loglevel
 	getLogger()
 
-	services := cfg.Services
-	if len(services) == 0 {
-		services = []string{"engine", "receiver", "operator", "api"}
-	}
-	for _, s := range services {
+	for _, s := range cfg.Services {
 		switch s {
 		case "engine":
 			service.StartEngine(&cfg)
@@ -140,7 +139,7 @@ func reload() {
 }
 
 func getLogger() {
-	levelstr, ok := cfg.GetDriverDataStr("daemon.loglevel")
+	levelstr, ok := cfg.GetStagedDriverDataStr("daemon.loglevel")
 	if !ok {
 		levelstr = "INFO"
 	}
diff --git a/deployments/helm/honeydipper/.helmignore b/deployments/helm/honeydipper/.helmignore
deleted file mode 100644
index f0c1319..0000000
--- a/deployments/helm/honeydipper/.helmignore
+++ /dev/null
@@ -1,21 +0,0 @@
-# Patterns to ignore when building packages.
-# This supports shell glob matching, relative path matching, and
-# negation (prefixed with !). Only one pattern per line.
-.DS_Store
-# Common VCS dirs
-.git/
-.gitignore
-.bzr/
-.bzrignore
-.hg/
-.hgignore
-.svn/
-# Common backup files
-*.swp
-*.bak
-*.tmp
-*~
-# Various IDEs
-.project
-.idea/
-*.tmproj
diff --git a/deployments/helm/honeydipper/Chart.yaml b/deployments/helm/honeydipper/Chart.yaml
deleted file mode 100644
index 8eb2d0b..0000000
--- a/deployments/helm/honeydipper/Chart.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-apiVersion: v1
-appVersion: "latest"
-description: An event driven orchestration system
-name: honeydipper
-version: 0.1.2
-icon: "https://raw.githubusercontent.com/honeydipper/honeydipper/master/logo/log_medium.png"
diff --git a/deployments/helm/honeydipper/README.md b/deployments/helm/honeydipper/README.md
deleted file mode 100644
index 4e09966..0000000
--- a/deployments/helm/honeydipper/README.md
+++ /dev/null
@@ -1,105 +0,0 @@
-# Honeydipper
-
-[Honeydipper](https://github.com/honeydipper/honeydipper) is basically, your swiss army knife for systems engineering and operations. An event-driven, policy-based orchestration system, with a pluggable open architecture.
-
-## Introduction
-
-This chart creates a deployment to run the Honeydipper daemon, and exposes a service pointing to the webhook listener.
-
-## Prerequisites
-
- * Kubernetes 1.10+
-
-## Required Configuration
-
-Honeydipper daemon loads most of the configurations from git repos. To start, a few environment variables are required to bootstrap repo. Below are the settings required.
-
- * daemon.env.REPO, defaults to `https://github.com/honeydipper/honeydipper-config-essentials.git`
-
-Please refer to [values.yaml](./values.yaml) for detailed example.
-
-## Customization
-
-The following options are supported . See [values.yaml](./values.yaml) for more detailed explanation and examples:
-
-| Parameter | Description | Default |
-| --------- | ----------- | ------- |
-| daemon.useSource | Instead of use prebuilt docker image, compile and run from source code | `false` |
-| daemon.useBranch | If using source code, specify the branch/tag to use | `master` |
-| daemon.sourceRepo | If using source code, specify alternative repo to use, should be a fork of official source code | `https://github.com/honeydipper/honeydipper.git` |
-| daemon.gitIdentityFilePath | If cloning source code from private repo using ssh protocol, use this to specify path to the id_rsa file, not including the file name | `/root` |
-| daemon.githubHostKey | If cloning source code from private repo using ssh protocol, use this to specify the github host key to avoid git ssh failure | current github host key |
-| daemon.spec | Additional pod spec settings for daemon deploy, can be anything other than `containers`, `nodeSelector`, `affinity`, `tolerations` or `volumes` | |
-| daemon.replicaCount | The number of replicas of daemons to start | 1 |
-| daemon.image.repository | The docker image for Honeydipper daemon | `honeydipper/honeydipper` |
-| daemon.image.tag | The version tag of the docker image to use | `latest` |
-| daemon.image.pullPolicy | Pull policy for the docker image | `Always` |
-| daemon.args | Honeydipper daemon args, should be a list of Honeydipper daemon services, empty means all | |
-| daemon.resources | The pod resource spec, cpu limit, memory limit, etc., as defined in a pod sped | |
-| daemon.nodeSelector | A map of node selectors for the pod, as defined in a pod sped | |
-| daemon.affinity | A map of affinity settings for the pod, as defined in a pod sped | |
-| daemon.tolerations | A list of toleration settings for the pod, as defined in a pod sped | |
-| daemon.volumes | A map of volumes for the pod, from volume name to its mount point and a `spec` defining the volume | |
-| daemon.env | A map of environment variables from name to its value, the value is either a string or a data structure used for `valueFrom` as defined in a pod spec | |
-| drivers.webhook.service.type | The exposed service type for the webhook | `LoadBalancer` |
-| drivers.webhook.service.port | The exposed service port for the webhook, needs to match the driver configurations set in the configuration repo | 8080 |
-| drivers.webhook.service.nodePort | The exposed service node port for the webhook. If set to 0, Kubernetes will assign a random port. | 0 |
-| drivers.webhook.ingress.enabled | Use ingress controller for the webhook service | `false` |
-| drivers.webhook.ingress.annotations | If using ingress controllers, specify a map of anotations | |
-| drivers.webhook.ingress.path | If using ingress controllers, specify the path mapped to webhook | |
-| drivers.webhook.ingress.hosts | If using ingress controllers, specify a list of host names for the webhook | |
-| drivers.webhook.ingress.tls | If using ingress controllers, specify the secret that defines the key and certs | |
-| drivers.redis.local | Use redis sidecar container for eventbus, this is only useful for testing. In production, multiple replicas of daemon should share the redis server | true |
-
-### Specifying environment
-
-Using command line:
-
-```helm
-helm --set daemon.env.REPO=git@github.com:example/example.git,daemon.env.BOOTSTRAP_PATH=/submodule,daemon.env.BRANCH=test_branch install test incubator/honeydipper
-```
-
-In values file
-
-```yaml
-daemon:
-  env:
-    REPO: git@github.com:example/example.git
-    BOOTSTRAP_PATH: /submodule
-    BRANCH: test_branch
-    SECRET_VARIABLE:
-      secretRef:
-        name: my-secret
-        key: my-key
-```
-
-### Mounting volumes
-
-Using values file
-
-```yaml
-daemon:
-  volumes:
-    my-config:
-      mountPath: /etc/myconfig/file
-      subPath: file
-      spec:
-        configMap:
-          name: my-config
-```
-
-## Testing the Deployment
-
-To perform a sanity test (i.e. ensure Honeydipper daemon is running, and when you curl the webhook url `http(s)://your-service-url/health`, you should get a `200` response code.)
-
- 1. Install the chart
-
-```bash
-helm install -f test-values.yaml --name trial incubator/honeydipper --debug
-```
-
- 2. Run the tests
-
-```bash
-helm test trial
-```
diff --git a/deployments/helm/honeydipper/templates/NOTES.txt b/deployments/helm/honeydipper/templates/NOTES.txt
deleted file mode 100644
index 477862a..0000000
--- a/deployments/helm/honeydipper/templates/NOTES.txt
+++ /dev/null
@@ -1,19 +0,0 @@
-1. Get the application URL by running these commands:
-{{- if .Values.drivers.webhook.ingress.enabled }}
-{{- range .Values.drivers.webhook.ingress.hosts }}
-  http{{ if $.Values.drivers.webhook.ingress.tls }}s{{ end }}://{{ . }}{{ $.Values.drivers.webhook.ingress.path }}
-{{- end }}
-{{- else if contains "NodePort" .Values.drivers.webhook.service.type }}
-  export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "honeydipper.fullname" . }})
-  export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
-  echo http://$NODE_IP:$NODE_PORT
-{{- else if contains "LoadBalancer" .Values.drivers.webhook.service.type }}
-     NOTE: It may take a few minutes for the LoadBalancer IP to be available.
-           You can watch the status of by running 'kubectl get svc -w {{ include "honeydipper.fullname" . }}'
-  export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "honeydipper.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
-  echo http://$SERVICE_IP:{{ .Values.drivers.webhook.service.port }}
-{{- else if contains "ClusterIP" .Values.drivers.webhook.service.type }}
-  export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "honeydipper.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
-  echo "Visit http://127.0.0.1:8080 to use your application"
-  kubectl port-forward $POD_NAME 8080:8080
-{{- end }}
diff --git a/deployments/helm/honeydipper/templates/_helpers.tpl b/deployments/helm/honeydipper/templates/_helpers.tpl
deleted file mode 100644
index da40fc7..0000000
--- a/deployments/helm/honeydipper/templates/_helpers.tpl
+++ /dev/null
@@ -1,32 +0,0 @@
-{{/* vim: set filetype=mustache: */}}
-{{/*
-Expand the name of the chart.
-*/}}
-{{- define "honeydipper.name" -}}
-{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
-{{- end -}}
-
-{{/*
-Create a default fully qualified app name.
-We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
-If release name contains chart name it will be used as a full name.
-*/}}
-{{- define "honeydipper.fullname" -}}
-{{- if .Values.fullnameOverride -}}
-{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
-{{- else -}}
-{{- $name := default .Chart.Name .Values.nameOverride -}}
-{{- if contains $name .Release.Name -}}
-{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
-{{- else -}}
-{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
-{{- end -}}
-{{- end -}}
-{{- end -}}
-
-{{/*
-Create chart name and version as used by the chart label.
-*/}}
-{{- define "honeydipper.chart" -}}
-{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
-{{- end -}}
diff --git a/deployments/helm/honeydipper/templates/daemon-deployment.yaml b/deployments/helm/honeydipper/templates/daemon-deployment.yaml
deleted file mode 100644
index 1b64035..0000000
--- a/deployments/helm/honeydipper/templates/daemon-deployment.yaml
+++ /dev/null
@@ -1,115 +0,0 @@
-apiVersion: apps/v1beta2
-kind: Deployment
-metadata:
-  name: {{ include "honeydipper.fullname" . }}
-  labels:
-    app.kubernetes.io/name: {{ include "honeydipper.name" . }}
-    helm.sh/chart: {{ include "honeydipper.chart" . }}
-    app.kubernetes.io/instance: {{ .Release.Name }}
-    app.kubernetes.io/managed-by: {{ .Release.Service }}
-spec:
-  replicas: {{ .Values.daemon.replicaCount }}
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: {{ include "honeydipper.name" . }}
-      app.kubernetes.io/instance: {{ .Release.Name }}
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/name: {{ include "honeydipper.name" . }}
-        app.kubernetes.io/instance: {{ .Release.Name }}
-    spec:
-      {{- with .Values.daemon.spec }}
-      {{ toYaml . | indent 6 | trim }}
-      {{- end }}
-      containers:
-        {{- if .Values.drivers.redis.local }}
-        - name: {{ .Chart.Name }}-redis
-          image: "redis:latest"
-        {{- end }}
-        - name: {{ .Chart.Name }}
-          {{- if .Values.daemon.useSource }}
-          image: "golang:alpine"
-          command:
-            - "/bin/sh"
-            - "-c"
-            - |
-              apk add openssh git &&
-              go get -u github.com/golang/dep/cmd/dep &&
-              mkdir -p ~/.ssh &&
-              chmod 700 ~/.ssh &&
-              {{- if .Values.daemon.gitIdentityFilePath }}
-              echo $'Host github.com\n\tIdentityFile\t'{{ .Values.daemon.gitIdentityFilePath }}/id_rsa > ~/.ssh/config &&
-              {{- end }}
-              {{- if .Values.daemon.gitHostKey }}
-              echo 'github.com ssh-rsa {{ .Values.daemon.githubHostKey }}' > ~/.ssh/known_hosts &&
-              {{- end }}
-              mkdir -p /go/src/github.com/honeydipper &&
-              cd /go/src/github.com/honeydipper &&
-              git clone {{ .Values.daemon.sourceRepo }} &&
-              cd honeydipper &&
-              git checkout {{ .Values.daemon.useBranch }} &&
-              dep ensure &&
-              go install ./... &&
-              honeydipper {{ range $k, $v := .Values.daemon.args }}{{ quote $v }}{{ end }} ||
-              sleep 3600
-          {{- else }}
-          image: "{{ .Values.daemon.image.repository }}:{{ .Values.daemon.image.tag }}"
-          {{- end }}
-          imagePullPolicy: {{ .Values.daemon.image.pullPolicy }}
-          {{- with .Values.daemon.args }}
-          args:
-            {{ toYaml . | indent 12 | trim }}
-          {{- end }}
-          {{- with .Values.daemon.env }}
-          env:
-            {{- range $envName, $envValue := . }}
-            - name: {{ $envName }}
-              {{- if kindIs "string" $envValue }}
-              value: {{ $envValue }}
-              {{- else }}
-              valueFrom:
-                {{ toYaml $envValue | indent 16 | trim }}
-              {{- end }}
-            {{- end }}
-          {{- end }}
-          {{- with .Values.daemon.resources }}
-          resources:
-            {{ toYaml .Values.daemon.resources | indent 12 | trim }}
-          {{- end }}
-          {{- with .Values.daemon.volumes }}
-          volumeMounts:
-            {{- range $vName, $vMount := . }}
-            - name: {{ $vName }}
-              {{ omit $vMount "spec" | toYaml | indent 14 | trim }}
-            {{- end }}
-          {{- end }}
-          readinessProbe:
-            tcpSocket:
-              port: {{ .Values.drivers.webhook.service.port }}
-            initialDelaySeconds: 20
-            periodSeconds: 10
-          livenessProbe:
-            tcpSocket:
-              port: {{ .Values.drivers.webhook.service.port }}
-            initialDelaySeconds: 20
-            periodSeconds: 20
-      {{- with .Values.daemon.nodeSelector }}
-      nodeSelector:
-        {{ toYaml . | indent 8 | trim }}
-      {{- end }}
-      {{- with .Values.daemon.affinity }}
-      affinity:
-        {{ toYaml . | indent 8 | trim }}
-      {{- end }}
-      {{- with .Values.daemon.tolerations }}
-      tolerations:
-        {{ toYaml . | indent 8 | trim }}
-      {{- end }}
-      {{- with .Values.daemon.volumes }}
-      volumes:
-        {{- range $vName, $vMount := . }}
-        - name: {{ $vName }}
-          {{ index $vMount "spec" | toYaml | indent 10 | trim }}
-        {{- end }}
-      {{- end }}
diff --git a/deployments/helm/honeydipper/templates/webhook-ingress.yaml b/deployments/helm/honeydipper/templates/webhook-ingress.yaml
deleted file mode 100644
index 1917fd3..0000000
--- a/deployments/helm/honeydipper/templates/webhook-ingress.yaml
+++ /dev/null
@@ -1,38 +0,0 @@
-{{- if .Values.drivers.webhook.ingress.enabled -}}
-{{- $fullName := include "honeydipper.fullname" . -}}
-{{- $ingressPath := .Values.drivers.webhook.ingress.path -}}
-apiVersion: extensions/v1beta1
-kind: Ingress
-metadata:
-  name: {{ $fullName }}
-  labels:
-    app.kubernetes.io/name: {{ include "honeydipper.name" . }}
-    helm.sh/chart: {{ include "honeydipper.chart" . }}
-    app.kubernetes.io/instance: {{ .Release.Name }}
-    app.kubernetes.io/managed-by: {{ .Release.Service }}
-{{- with .Values.drivers.webhook.ingress.annotations }}
-  annotations:
-{{ toYaml . | indent 4 }}
-{{- end }}
-spec:
-{{- if .Values.drivers.webhook.ingress.tls }}
-  tls:
-  {{- range .Values.drivers.webhook.ingress.tls }}
-    - hosts:
-      {{- range .hosts }}
-        - {{ . | quote }}
-      {{- end }}
-      secretName: {{ .secretName }}
-  {{- end }}
-{{- end }}
-  rules:
-  {{- range .Values.drivers.webhook.ingress.hosts }}
-    - host: {{ . | quote }}
-      http:
-        paths:
-          - path: {{ $ingressPath }}
-            backend:
-              serviceName: {{ $fullName }}
-              servicePort: http
-  {{- end }}
-{{- end }}
diff --git a/deployments/helm/honeydipper/templates/webhook-service.yaml b/deployments/helm/honeydipper/templates/webhook-service.yaml
deleted file mode 100644
index c0857b7..0000000
--- a/deployments/helm/honeydipper/templates/webhook-service.yaml
+++ /dev/null
@@ -1,24 +0,0 @@
-{{- if or (has "receiver" .Values.daemon.args) (not .Values.daemon.args) -}}
-apiVersion: v1
-kind: Service
-metadata:
-  name: {{ include "honeydipper.fullname" . }}
-  labels:
-    app.kubernetes.io/name: {{ include "honeydipper.name" . }}
-    helm.sh/chart: {{ include "honeydipper.chart" . }}
-    app.kubernetes.io/instance: {{ .Release.Name }}
-    app.kubernetes.io/managed-by: {{ .Release.Service }}
-spec:
-  type: {{ .Values.drivers.webhook.service.type }}
-  ports:
-    - port: {{ .Values.drivers.webhook.service.port }}
-      targetPort: {{ .Values.drivers.webhook.service.port }}
-      {{- if .Values.drivers.webhook.service.nodePort }}
-      nodePort: {{ .Values.drivers.webhook.service.nodePort }}
-      {{- end }}
-      protocol: TCP
-      name: webhook
-  selector:
-    app.kubernetes.io/name: {{ include "honeydipper.name" . }}
-    app.kubernetes.io/instance: {{ .Release.Name }}
-{{- end -}}
diff --git a/deployments/helm/honeydipper/values.yaml b/deployments/helm/honeydipper/values.yaml
deleted file mode 100644
index 986221d..0000000
--- a/deployments/helm/honeydipper/values.yaml
+++ /dev/null
@@ -1,103 +0,0 @@
-# Default values for honeydipper.
-# This is a YAML-formatted file.
-# Declare variables to be passed into your templates.
-
-# nameOverride is used to override the chart name part of the release name
-nameOverride: ""
-# fullnameOverride is used to override the release name
-fullnameOverride: ""
-
-# daemon is used to specify the settings for the daemon
-daemon:
-  # useSource true means building the daemon using source code instead of prebuilt image
-  useSource: false
-  # useBranch specifies which branch of source code to use
-  useBranch: "master"
-  # sourceRepo specifies the repo to download source code from, could be a clone of original https://github.com/honeydipper/honeydipper.git
-  sourceRepo: "https://github.com/honeydipper/honeydipper.git"
-
-  # gitIdentityFilePath points to the ssh key to use for cloning source code from private repo
-  gitIdentityFilePath: "/root"
-  # githubHostKey is the github.com host key
-  githubHostKey: "AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ=="
-
-  # spec is used to create the container spec in the manifest file
-  spec: {}
-
-  # replicaCount specifies how many replica daemon should be running
-  replicaCount: 1
-
-  # image specifies what image to use to start the daemon
-  image:
-    # repository is the image repository for the daemon container
-    repository: honeydipper/honeydipper
-    # tag is the version of the docker image to use
-    tag: "latest"
-    # pullPolicy specify when to refresh the image
-    pullPolicy: Always
-
-  # args are the arguments passed to the daemon, a list of services to run in the daemon
-  args: []
-
-  # resources specifies the resource limitation for the daemon container
-  resources: {}
-    # We usually recommend not to specify default resources and to leave this as a conscious
-    # choice for the user. This also increases chances charts run on environments with little
-    # resources, such as Minikube. If you do want to specify resources, uncomment the following
-    # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
-    # limits:
-    #  cpu: 100m
-    #  memory: 128Mi
-    # requests:
-    #  cpu: 100m
-    #  memory: 128Mi
-
-  # nodeSelector is used for specifying what nodes to place the daemon containers on
-  nodeSelector: {}
-
-  # tolerations is used in the daemon pod speicification
-  tolerations: []
-
-  # tolerations is used in the daemon pod speicification
-  affinity: {}
-
-  # env is a list of environment variables for the daemon container
-  env:
-    REPO: https://github.com/honeydipper/honeydipper-config-essentials
-
-  # volumes is a list of volumes to be mounted on the daemon container
-  volumes: {}
-
-# drivers containers the settings for the drivers
-drivers:
-  # redis settings for the redis related drivers
-  redis:
-    # local means run a local redis side car container, only useful for testing
-    local: true
-  # webhook settings for the webhook driver
-  webhook:
-    # service specifies how the webhook driver service is exposed
-    service:
-      # type is the type of kubernetes service for the webhook
-      type: LoadBalancer
-      port: 8080
-      nodePort: 0
-
-    # ingress controller settings
-    ingress:
-      # enabled is used for creating an ingress kubernetes resouce
-      enabled: false
-      # annotations are the annotations for the  ingress resource
-      annotations: {}
-        # kubernetes.io/ingress.class: nginx
-        # kubernetes.io/tls-acme: "true"
-      # path specify the path of the ingress
-      path: /
-      # hosts specify the hosts of the ingress
-      hosts:
-        - chart-example.local
-      # tls is used for specifying the ssl cert using secret
-      tls: []
-      #  - secretName: chart-example-tls
-      #    hosts:
-      #      - chart-example.local
diff --git a/docs/developer.md b/docs/developer.md
index 31193d5..516270d 100644
--- a/docs/developer.md
+++ b/docs/developer.md
@@ -382,7 +382,7 @@ workflows:
                 system: lightswitch
                 function: turn_on
               parameters:
-                device_id: '{{ `{{ .wfdata.current }}` }}'
+                device_id: '{{ `{{ .ctx.current }}` }}'
 ```
 <!-- {% endraw %} -->
 
diff --git a/go.mod b/go.mod
index 79906cf..eb15ab9 100644
--- a/go.mod
+++ b/go.mod
@@ -22,17 +22,18 @@ require (
 	github.com/googleapis/gnostic v0.3.1 // indirect
 	github.com/huandu/xstrings v1.3.1 // indirect
 	github.com/imdario/mergo v0.3.8
-	github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381
+	github.com/logrusorgru/aurora/v3 v3.0.0
 	github.com/mitchellh/copystructure v1.0.0 // indirect
 	github.com/mitchellh/mapstructure v1.3.2
 	github.com/mitchellh/reflectwalk v1.0.1 // indirect
 	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
 	github.com/qiangmzsx/string-adapter/v2 v2.1.0
 	github.com/stretchr/testify v1.6.1
-	golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0
-	golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43
+	golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
+	golang.org/x/oauth2 v0.0.0-20201203001011-0b49973bad19
+	golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1
 	google.golang.org/api v0.32.0
-	google.golang.org/genproto v0.0.0-20201002142447-3860012362da
+	google.golang.org/genproto v0.0.0-20201204160425-06b3db808446
 	gopkg.in/h2non/gock.v1 v1.0.15
 	gopkg.in/src-d/go-git.v4 v4.13.1
 	gopkg.in/yaml.v2 v2.3.0 // indirect
diff --git a/go.sum b/go.sum
index ebd99be..885971d 100644
--- a/go.sum
+++ b/go.sum
@@ -270,8 +270,8 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
 github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
-github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381 h1:bqDmpDG49ZRnB5PcgP0RXtQvnMSgIF14M7CBd2shtXs=
-github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
+github.com/logrusorgru/aurora/v3 v3.0.0 h1:R6zcoZZbvVcGMvDCKo45A9U/lzYyzl5NfYIvznmDfE4=
+github.com/logrusorgru/aurora/v3 v3.0.0/go.mod h1:vsR12bk5grlLvLXAYrBsb5Oc/N+LxAlxggSjiwMnCUc=
 github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
@@ -356,9 +356,8 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0 h1:hb9wdF1z5waM+dSIICn1l0DkLVDT3hqhhQsDNUmHPRE=
-golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -435,6 +434,8 @@ golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BG
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43 h1:ld7aEMNHoBnnDAX15v1T6z31v8HwR2A9FYOuAhWqkwc=
 golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20201203001011-0b49973bad19 h1:ZD+2Sd/BnevwJp8PSli8WgGAGzb9IZtxBsv1iZMYeEA=
+golang.org/x/oauth2 v0.0.0-20201203001011-0b49973bad19/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -480,6 +481,10 @@ golang.org/x/sys v0.0.0-20200803210538-64077c9b5642 h1:B6caxRw+hozq68X2MY7jEpZh/
 golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f h1:Fqb3ao1hUmOR3GkUOg/Y+BadLwykBIzs5q8Ez2SbHyc=
 golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -609,6 +614,8 @@ google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d h1:92D1fum1bJLKSdr
 google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20201002142447-3860012362da h1:DTQYk4u7nICKkkVZsBv0/0po0ChISxAJ5CTAfUhO0PQ=
 google.golang.org/genproto v0.0.0-20201002142447-3860012362da/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201204160425-06b3db808446 h1:65ppmIPdaZE+BO34gntwqexoTYr30IRNGmS0OGOHu3A=
+google.golang.org/genproto v0.0.0-20201204160425-06b3db808446/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
diff --git a/internal/api/request.go b/internal/api/request.go
index 332c873..2cfec24 100644
--- a/internal/api/request.go
+++ b/internal/api/request.go
@@ -12,13 +12,8 @@ import (
 	"github.com/honeydipper/honeydipper/pkg/dipper"
 )
 
-const (
-	// APIErrorNoAck means not able to receive ACK for the API call.
-	APIErrorNoAck dipper.Error = "no ack"
-
-	// DefaultAPIReapTimeout is the timeout in seconds before a API result is abandoned.
-	DefaultAPIReapTimeout time.Duration = 30
-)
+// DefaultAPIReapTimeout is the timeout in seconds before a API result is abandoned.
+const DefaultAPIReapTimeout time.Duration = 30
 
 // Request represents a live API call.
 type Request struct {
@@ -110,11 +105,11 @@ func (a *Request) postACK() {
 	defer close(a.received)
 	switch {
 	case a.reqType != TypeFirst && len(a.acks) == 0:
-		a.err = APIErrorNoAck
+		a.err = ErrAPINoACK
 	case a.timeout != InfiniteDuration:
 		select {
 		case <-time.After(a.timeout):
-			a.err = dipper.TimeoutError
+			a.err = dipper.ErrTimeout
 		case <-a.received:
 		}
 	default:
diff --git a/internal/api/store.go b/internal/api/store.go
index f9b1ccf..e2a60c6 100644
--- a/internal/api/store.go
+++ b/internal/api/store.go
@@ -25,9 +25,6 @@ const (
 	// DefaultAPIAckTimeout is the number of milliseconds to wait for acks.
 	DefaultAPIAckTimeout time.Duration = 10
 
-	// APIError is the base for all API related error.
-	APIError dipper.Error = "API error"
-
 	// DefaultAPIWriteTimeout is the default timeout in seconds for responding to the request.
 	DefaultAPIWriteTimeout time.Duration = 10
 
@@ -38,6 +35,14 @@ const (
 	ACLDeny = "deny"
 )
 
+var (
+	// ErrAPIError is the base for all API related error.
+	ErrAPIError = errors.New("API error")
+
+	// ErrAPINoACK means not able to receive ACK for the API call.
+	ErrAPINoACK = fmt.Errorf("%w: no ACK", ErrAPIError)
+)
+
 // Store stores the live API calls in memory.
 type Store struct {
 	requests        *sync.Map
@@ -58,17 +63,17 @@ func (l *Store) HandleAPIACK(m *dipper.Message) {
 
 	uuid, ok := m.Labels["uuid"]
 	if !ok {
-		panic(fmt.Errorf("uuid missing in ack: %w", APIError))
+		panic(fmt.Errorf("%w: uuid missing ACK", ErrAPIError))
 	}
-	dipper.Logger.Debugf("handling api ack for %s", uuid)
+	dipper.Logger.Debugf("handling api ACK for %s", uuid)
 	a, ok := l.requests.Load(uuid)
 	if !ok {
-		panic(fmt.Errorf("request not found: %w", APIError))
+		panic(fmt.Errorf("%w: request not found", ErrAPIError))
 	}
 	api := a.(*Request)
 	responder, ok := m.Labels["from"]
 	if !ok {
-		panic(fmt.Errorf("from label missing in ack: %w", APIError))
+		panic(fmt.Errorf("%w: missing from label in ACK", ErrAPIError))
 	}
 
 	switch api.reqType {
@@ -78,7 +83,7 @@ func (l *Store) HandleAPIACK(m *dipper.Message) {
 		api.acks = append(api.acks, responder)
 		api.firstACK <- 1
 	case TypeFirst:
-		panic(fmt.Errorf("TypeFirst APIs do not expect acks: %w", APIError))
+		panic(fmt.Errorf("%w: TypeFirst APIs do not expect ACKs", ErrAPIError))
 	}
 }
 
@@ -89,20 +94,20 @@ func (l *Store) HandleAPIReturn(m *dipper.Message) {
 	m = dipper.DeserializePayload(m)
 	uuid, ok := m.Labels["uuid"]
 	if !ok {
-		panic(fmt.Errorf("uuid missing in ack: %w", APIError))
+		panic(fmt.Errorf("%w: uuid missing return", ErrAPIError))
 	}
 	a, ok := l.requests.Load(uuid)
 	if !ok {
-		panic(fmt.Errorf("request not found: %w", APIError))
+		panic(fmt.Errorf("%w: request not found", ErrAPIError))
 	}
 	api := a.(*Request)
 	responder, ok := m.Labels["from"]
 	if !ok {
-		panic(fmt.Errorf("from label missing in ack: %w", APIError))
+		panic(fmt.Errorf("%w: missing from label in return", ErrAPIError))
 	}
 
 	if errmsg, ok := m.Labels["error"]; ok {
-		api.err = fmt.Errorf("%w: from [%s]: %s", APIError, responder, errmsg)
+		api.err = fmt.Errorf("%w: from [%s]: %s", ErrAPIError, responder, errmsg)
 		api.received <- 1
 
 		return
@@ -243,7 +248,7 @@ func (l *Store) HandleHTTPRequest(c RequestContext, def Def) {
 	select {
 	case <-r.ready:
 		if r.err != nil {
-			if errors.Is(r.err, APIErrorNoAck) {
+			if errors.Is(r.err, ErrAPINoACK) {
 				c.AbortWithStatusJSON(http.StatusNotFound, map[string]interface{}{"error": "object not found"})
 			} else {
 				c.AbortWithStatusJSON(http.StatusInternalServerError, map[string]interface{}{"error": r.err.Error()})
@@ -255,7 +260,7 @@ func (l *Store) HandleHTTPRequest(c RequestContext, def Def) {
 		if r.method == http.MethodGet && (r.timeout == InfiniteDuration || r.timeout > l.writeTimeout) {
 			c.IndentedJSON(http.StatusAccepted, map[string]interface{}{"uuid": r.uuid, "results": r.getResults()})
 		} else {
-			c.AbortWithStatusJSON(http.StatusInternalServerError, map[string]interface{}{"error": dipper.TimeoutError.Error()})
+			c.AbortWithStatusJSON(http.StatusInternalServerError, map[string]interface{}{"error": dipper.ErrTimeout.Error()})
 		}
 	}
 }
diff --git a/internal/config/config.go b/internal/config/config.go
index 97ed107..b34e11b 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -11,7 +11,9 @@ package config
 import (
 	"bytes"
 	"encoding/gob"
+	"fmt"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/go-errors/errors"
@@ -19,10 +21,36 @@ import (
 	"github.com/imdario/mergo"
 )
 
+const (
+	// StageLoading means that the config is being loaded.
+	StageLoading = iota
+	// StageBooting means that the consumers are loading only essential config.
+	StageBooting
+	// StageDiscovering means discovering dynamic config data for drivers.
+	StageDiscovering
+	// StageServing means all config should be processed and serving.
+	StageServing
+)
+
+var (
+	// StageNames maps to the actual stage integers.
+	StageNames = []string{
+		"Loading",
+		"Booting",
+		"Discovering",
+		"Serving",
+	}
+
+	// ErrConfigRollback happens when daemon decides to rollback during reload.
+	ErrConfigRollback = errors.New("config rollback")
+)
+
 //nolint:gochecknoinits
 func init() {
 	gob.Register(map[string]interface{}{})
 	gob.Register([]interface{}{})
+	gob.Register(System{})
+	gob.Register(DataSet{})
 }
 
 // Config is a wrapper around the final complete configration of the daemon.
@@ -43,11 +71,45 @@ type Config struct {
 	IsDocGen      bool
 	DocSrc        string
 	DocDst        string
+	Locker        *sync.Mutex
+	RawData       *DataSet
+	Stage         int
+	Staged        *DataSet
+	StageWG       []*sync.WaitGroup
+}
+
+// ResetStage resets the stage of the config.
+func (c *Config) ResetStage() {
+	locker := &sync.Mutex{}
+	locker.Lock()
+	defer locker.Unlock()
+	if c.Locker != nil {
+		c.Locker.Lock()
+		defer c.Locker.Unlock()
+	}
+	c.Locker = locker
+	c.Stage = StageLoading
+
+	if c.StageWG != nil {
+		for _, wg := range c.StageWG {
+			dipper.WaitGroupDoneAll(wg)
+		}
+	}
+	c.StageWG = make([]*sync.WaitGroup, 3)
+	c.StageWG[StageLoading] = &sync.WaitGroup{}
+	c.StageWG[StageLoading].Add(len(c.Services))
+	c.StageWG[StageBooting] = &sync.WaitGroup{}
+	c.StageWG[StageBooting].Add(len(c.Services))
+	c.StageWG[StageDiscovering] = &sync.WaitGroup{}
+	c.StageWG[StageDiscovering].Add(len(c.Services))
 }
 
 // Bootstrap loads the configuration during daemon bootstrap.
 // WorkingDir is where git will clone remote repo into.
 func (c *Config) Bootstrap(wd string) {
+	if !c.IsConfigCheck && !c.IsDocGen {
+		c.ResetStage()
+	}
 	c.WorkingDir = wd
 	c.loadRepo(c.InitRepo)
 	c.assemble()
@@ -58,7 +120,7 @@ func (c *Config) Bootstrap(wd string) {
 func (c *Config) Watch() {
 	for {
 		interval := time.Minute
-		if intervalStr, ok := c.GetDriverDataStr("daemon.configCheckInterval"); ok {
+		if intervalStr, ok := c.GetStagedDriverDataStr("daemon.configCheckInterval"); ok {
 			value, err := time.ParseDuration(intervalStr)
 			if err != nil {
 				dipper.Logger.Warningf("invalid drivers.daemon.configCheckInterval %v", err)
@@ -68,7 +130,7 @@ func (c *Config) Watch() {
 		}
 		time.Sleep(interval)
 
-		if watch, ok := dipper.GetMapDataBool(c.DataSet.Drivers, "daemon.watchConfig"); !ok || watch {
+		if watch, ok := dipper.GetMapDataBool(c.Staged.Drivers, "daemon.watchConfig"); !ok || watch {
 			c.Refresh()
 		}
 	}
@@ -82,7 +144,8 @@ func (c *Config) Refresh() {
 		changeDetected = (repoRuntime.refreshRepo() || changeDetected)
 	}
 	if changeDetected {
-		c.LastRunningConfig.DataSet = c.DataSet
+		c.ResetStage()
+		c.LastRunningConfig.DataSet = c.RawData
 		c.LastRunningConfig.Loaded = map[RepoInfo]*Repo{}
 		for k, v := range c.Loaded {
 			c.LastRunningConfig.Loaded[k] = v
@@ -108,12 +171,13 @@ func (c *Config) Refresh() {
 // It requires that the OnChange variable is defined prior to invoking the function.
 func (c *Config) RollBack() {
 	if c.LastRunningConfig.DataSet != nil && c.LastRunningConfig.DataSet != c.DataSet {
-		c.DataSet = c.LastRunningConfig.DataSet
+		c.Staged = c.LastRunningConfig.DataSet
 		c.Loaded = map[RepoInfo]*Repo{}
 		for k, v := range c.LastRunningConfig.Loaded {
 			c.Loaded[k] = v
 		}
 		dipper.Logger.Warning("config rolled back to last running version")
+		c.ResetStage()
 		if c.OnChange != nil {
 			c.OnChange()
 		}
@@ -121,20 +185,73 @@ func (c *Config) RollBack() {
 }
 
 func (c *Config) assemble() {
-	c.DataSet, c.Loaded = c.Loaded[c.InitRepo].assemble(&(DataSet{}), map[RepoInfo]*Repo{})
-	c.extendAllSystems()
-	c.parseWorkflowRegex()
+	c.Staged, c.Loaded = c.Loaded[c.InitRepo].assemble(&(DataSet{}), map[RepoInfo]*Repo{})
+}
+
+// AdvanceStage processes the config and advances the config into the new stage.
+func (c *Config) AdvanceStage(service string, stage int, fns ...dipper.ItemProcessor) {
+	c.Locker.Lock()
+	defer c.Locker.Unlock()
+	locker := c.Locker
+	stageWG := c.StageWG[c.Stage]
+
+	switch {
+	case c.Stage >= stage:
+		return
+	case c.Stage < stage-1:
+		panic(ErrConfigRollback)
+	default:
+		if !dipper.WaitGroupDone(stageWG) {
+			panic(ErrConfigRollback)
+		}
+		locker.Unlock()
+		stageWG.Wait()
+		locker.Lock()
+
+		if locker != c.Locker {
+			panic(ErrConfigRollback)
+		}
+
+		dipper.Logger.Infof("[%s] service reached stage %s", service, StageNames[stage])
+		for _, fn := range fns {
+			c.RecursiveStaged(fn)
+		}
+
+		switch stage {
+		case StageBooting:
+			c.RawData = &DataSet{}
+			dipper.Must(DeepCopy(c.Staged, c.RawData))
+		case StageDiscovering:
+			c.extendAllSystems()
+		case StageServing:
+			c.RecursiveStaged(dipper.RegexParser)
+
+			c.DataSet = c.Staged
+		}
+		c.Stage = stage
+	}
 }
 
-func (c *Config) parseWorkflowRegex() {
-	var processor func(key string, val interface{}) (interface{}, bool)
+// recursive iterates all items and their children to parse the values with give processor.
+func (c *Config) recursive(f dipper.ItemProcessor, stage bool) {
+	var processor dipper.ItemProcessor
+	target := c.DataSet
+	if stage {
+		target = c.Staged
+	}
 
 	processor = func(name string, val interface{}) (interface{}, bool) {
 		switch v := val.(type) {
 		case string:
-			return dipper.RegexParser(name, val)
+			nv, ok := f(name, val)
+			if stage {
+				return nv, ok
+			}
+			// only staged data can be changed.
+			return nil, false
 		case Rule:
 			dipper.Recursive(&v.Do, processor)
+			dipper.Recursive(&v.When, processor)
 		case Workflow:
 			dipper.Recursive(v.Match, processor)
 			dipper.Recursive(v.UnlessMatch, processor)
@@ -142,14 +259,44 @@ func (c *Config) parseWorkflowRegex() {
 			dipper.Recursive(v.Threads, processor)
 			dipper.Recursive(v.Else, processor)
 			dipper.Recursive(v.Cases, processor)
+		case Trigger:
+			dipper.Recursive(v.Match, processor)
+			dipper.Recursive(v.Parameters, processor)
+		case Function:
+			dipper.Recursive(v.Parameters, processor)
+		case System:
+			dipper.Recursive(v.Data, processor)
+			dipper.Recursive(v.Triggers, processor)
+			dipper.Recursive(v.Functions, processor)
 		}
 
 		return nil, false
 	}
 
-	dipper.Recursive(c.DataSet.Workflows, processor)
-	dipper.Recursive(c.DataSet.Rules, processor)
-	dipper.Recursive(c.DataSet.Contexts, dipper.RegexParser)
+	dipper.Logger.Debugf("[config] recursively processing workflows ...")
+	dipper.Recursive(target.Workflows, processor)
+	dipper.Logger.Debugf("[config] recursively processing rules ...")
+	dipper.Recursive(target.Rules, processor)
+	dipper.Logger.Debugf("[config] recursively processing systems ...")
+	dipper.Recursive(target.Systems, processor)
+	dipper.Logger.Debugf("[config] recursively processing contexts ...")
+	dipper.Recursive(target.Contexts, f)
+	dipper.Logger.Debugf("[config] recursively processing drivers ...")
+	for k, v := range target.Drivers {
+		if k != "daemon" {
+			dipper.Recursive(v, f)
+		}
+	}
+}
+
+// RecursiveStaged recursively process staged data.
+func (c *Config) RecursiveStaged(f dipper.ItemProcessor) {
+	c.recursive(f, true)
+}
+
+// Recursive recursively process readonly config data.
+func (c *Config) Recursive(f dipper.ItemProcessor) {
+	c.recursive(f, false)
 }
 
 func (c *Config) isRepoLoaded(repo RepoInfo) bool {
@@ -169,6 +316,29 @@ func (c *Config) loadRepo(repo RepoInfo) {
 	}
 }
 
+// GetStagedDriverData gets an item from a staged driver's data block.
+//   conn,ok := c.GetStagedDriverData("redis.connection")
+// The function returns an interface{} that could be anything.
+func (c *Config) GetStagedDriverData(path string) (ret interface{}, ok bool) {
+	if c.Staged == nil || c.Staged.Drivers == nil {
+		return nil, false
+	}
+
+	return dipper.GetMapData(c.Staged.Drivers, path)
+}
+
+// GetStagedDriverDataStr gets an item from a staged driver's data block.
+//   logLevel,ok := c.GetStagedDriverData("daemon.loglevel")
+// The function assume the return value is a string will do a type assertion.
+// upon returning.
+func (c *Config) GetStagedDriverDataStr(path string) (ret string, ok bool) {
+	if c.Staged == nil || c.Staged.Drivers == nil {
+		return "", false
+	}
+
+	return dipper.GetMapDataStr(c.Staged.Drivers, path)
+}
+
 // GetDriverData gets an item from a driver's data block.
 //   conn,ok := c.GetDriverData("redis.connection")
 // The function returns an interface{} that could be anything.
@@ -194,7 +364,7 @@ func (c *Config) GetDriverDataStr(path string) (ret string, ok bool) {
 
 func (c *Config) extendSystem(processed map[string]bool, system string) {
 	var merged System
-	current := c.DataSet.Systems[system]
+	current := c.Staged.Systems[system]
 	for _, extend := range current.Extends {
 		parts := strings.Split(extend, "=")
 		var base, subKey string
@@ -209,8 +379,9 @@ func (c *Config) extendSystem(processed map[string]bool, system string) {
 			c.extendSystem(processed, base)
 		}
 
-		baseSys := c.DataSet.Systems[base]
-		baseCopy := dipper.Must(SystemCopy(&baseSys)).(*System)
+		baseSys := c.Staged.Systems[base]
+		baseCopy := &System{}
+		dipper.Must(DeepCopy(&baseSys, baseCopy))
 
 		if subKey == "" {
 			dipper.Must(mergeSystem(&merged, *baseCopy))
@@ -219,60 +390,62 @@ func (c *Config) extendSystem(processed map[string]bool, system string) {
 		}
 	}
 	dipper.Must(mergeSystem(&merged, current))
-	c.DataSet.Systems[system] = merged
+	c.Staged.Systems[system] = merged
 	processed[system] = true
 }
 
 func (c *Config) extendAllSystems() {
 	processed := map[string]bool{}
-	for name := range c.DataSet.Systems {
+	for name := range c.Staged.Systems {
 		if _, ok := processed[name]; !ok {
 			c.extendSystem(processed, name)
 		}
 	}
 }
 
-// SystemCopy performs a deep copy of the given system.
-func SystemCopy(s *System) (*System, error) {
-	var buf bytes.Buffer
+// DeepCopy performs a deep copy of the object.
+func DeepCopy(s interface{}, d interface{}) error {
 	if s == nil {
-		return nil, nil
+		return nil
 	}
+
+	var buf bytes.Buffer
 	enc := gob.NewEncoder(&buf)
-	dec := gob.NewDecoder(&buf)
-	err := enc.Encode(*s)
-	if err != nil {
-		return nil, err
+	e := enc.Encode(s)
+	if e != nil {
+		return fmt.Errorf("failed to encode for deep copy: %w", e)
 	}
-	var scopy System
-	err = dec.Decode(&scopy)
-	if err != nil {
-		return nil, err
+
+	dec := gob.NewDecoder(&buf)
+	e = dec.Decode(d)
+	if e != nil {
+		return fmt.Errorf("failed to decode for deep copy: %w", e)
 	}
 
-	return &scopy, nil
+	return nil
 }
 
 func mergeDataSet(d *DataSet, s DataSet) error {
-	for name, system := range s.Systems {
+	for name := range s.Systems {
+		system := s.Systems[name]
+		srcCopy := &System{}
+		dipper.Must(DeepCopy(&system, srcCopy))
 		exist, ok := d.Systems[name]
 		if ok {
-			dipper.Must(mergeSystem(&exist, system))
+			dipper.Must(mergeSystem(&exist, *srcCopy))
 		} else {
-			exist = system
+			exist = *srcCopy
 		}
 		if d.Systems == nil {
 			d.Systems = map[string]System{}
 		}
 		d.Systems[name] = exist
 	}
-
 	s.Systems = map[string]System{}
 	s.Contexts = dipper.MustDeepCopyMap(s.Contexts)
 	s.Drivers = dipper.MustDeepCopyMap(s.Drivers)
-	err := mergo.Merge(d, s, mergo.WithOverride, mergo.WithAppendSlice)
 
-	return err
+	return mergo.Merge(d, s, mergo.WithOverride, mergo.WithAppendSlice)
 }
 
 func addSubsystem(d *System, s System, key string) {
@@ -304,7 +477,7 @@ func mergeSystem(d *System, s System) error {
 		if exist, ok := d.Triggers[name]; ok {
 			err := mergo.Merge(&exist, trigger, mergo.WithOverride, mergo.WithAppendSlice)
 			if err != nil {
-				return err
+				return fmt.Errorf("%w", err)
 			}
 			if exist.Description == "" {
 				exist.Description = trigger.Description
@@ -326,7 +499,7 @@ func mergeSystem(d *System, s System) error {
 		if ok {
 			err := mergo.Merge(&exist, function, mergo.WithOverride, mergo.WithAppendSlice)
 			if err != nil {
-				return err
+				return fmt.Errorf("%w", err)
 			}
 			if exist.Description == "" {
 				exist.Description = function.Description
@@ -342,7 +515,7 @@ func mergeSystem(d *System, s System) error {
 
 	err := mergo.Merge(&d.Data, s.Data, mergo.WithOverride, mergo.WithAppendSlice)
 	if err != nil {
-		return err
+		return fmt.Errorf("%w", err)
 	}
 
 	d.Extends = append(d.Extends, s.Extends...)
diff --git a/internal/config/file_reference.go b/internal/config/file_reference.go
index a386fe2..7b430d8 100644
--- a/internal/config/file_reference.go
+++ b/internal/config/file_reference.go
@@ -7,6 +7,7 @@
 package config
 
 import (
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"path"
@@ -15,15 +16,13 @@ import (
 	"github.com/honeydipper/honeydipper/pkg/dipper"
 )
 
-const (
-	// FileError is all errors when accessing a file.
-	FileError dipper.Error = "file error"
-)
+// ErrFileError is all errors when accessing a file.
+var ErrFileError = errors.New("file error")
 
 func (c *Repo) normalizeFilePath(cwd string, file string) string {
 	fullpath := path.Clean(path.Join(c.root, cwd, file))
 	if !strings.HasPrefix(fullpath, c.root+"/") {
-		panic(fmt.Errorf("invalid path: %s: %w", fullpath, FileError))
+		panic(fmt.Errorf("%w: invalid path: %s", ErrFileError, fullpath))
 	}
 
 	return fullpath
diff --git a/internal/driver/builtin.go b/internal/driver/builtin.go
index f18be56..d96a7f8 100644
--- a/internal/driver/builtin.go
+++ b/internal/driver/builtin.go
@@ -42,10 +42,10 @@ func builtinPath() string {
 func (d *BuiltinDriver) Acquire() {
 	shortName, ok := d.meta.HandlerData["shortName"].(string)
 	if !ok || shortName == "" {
-		panic(fmt.Errorf("shortName is missing for builtin driver: %s: %w", d.meta.Name, DriverError))
+		panic(fmt.Errorf("%w: shortName is missing for builtin driver: %s", ErrDriverError, d.meta.Name))
 	}
 	if strings.ContainsRune(shortName, os.PathSeparator) {
-		panic(fmt.Errorf("shortName has path separator in driver: %s: %w", d.meta.Name, DriverError))
+		panic(fmt.Errorf("%w: shortName has path separator in driver: %s", ErrDriverError, d.meta.Name))
 	}
 
 	d.meta.Executable = filepath.Join(builtinPath(), shortName)
@@ -65,7 +65,7 @@ func (d *BuiltinDriver) Prepare() {
 	}
 
 	if !ok {
-		panic(fmt.Errorf("arguments in driver %s should be a list of strings: %w", d.meta.Name, DriverError))
+		panic(fmt.Errorf("%w: arguments in driver %s should be a list of strings", ErrDriverError, d.meta.Name))
 	}
 
 	for _, v := range argsList {
diff --git a/internal/driver/driver.go b/internal/driver/driver.go
index 7cfedbb..89a288a 100644
--- a/internal/driver/driver.go
+++ b/internal/driver/driver.go
@@ -7,6 +7,7 @@
 package driver
 
 import (
+	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -17,13 +18,11 @@ import (
 	"github.com/mitchellh/mapstructure"
 )
 
-const (
-	// DriverError is the base for all driver related error.
-	DriverError dipper.Error = "driver error"
+// ErrDriverError is the base for all driver related error.
+var ErrDriverError = errors.New("driver error")
 
-	// DriverMessageBuffer is the size of the driver message buffer.
-	DriverMessageBuffer = 10
-)
+// DriverMessageBuffer is the size of the driver message buffer.
+const DriverMessageBuffer = 10
 
 // Handler provides common functions for handling a driver.
 type Handler interface {
@@ -44,7 +43,7 @@ func NewDriver(data map[string]interface{}) Handler {
 	}
 
 	if meta.Name == "" {
-		panic(fmt.Errorf("driver name missing: %+v: %w", meta, DriverError))
+		panic(fmt.Errorf("%w: driver name missing: %+v", ErrDriverError, meta))
 	}
 
 	var dh Handler
@@ -53,14 +52,14 @@ func NewDriver(data map[string]interface{}) Handler {
 	case "builtin":
 		dh = NewBuiltinDriver(&meta)
 	default:
-		panic(fmt.Errorf("unsupported driver type: %s: %w", meta.Type, DriverError))
+		panic(fmt.Errorf("%w: unsupported driver type: %s", ErrDriverError, meta.Type))
 	}
 
 	dh.Acquire()
 	dh.Prepare()
 
 	if meta.Executable == "" {
-		panic(fmt.Errorf("executable not defined for driver: %s: %w", meta.Name, DriverError))
+		panic(fmt.Errorf("%w: executable not defined for driver: %s", ErrDriverError, meta.Name))
 	}
 
 	return dh
diff --git a/internal/service/api.go b/internal/service/api.go
index cad2e76..95ac272 100644
--- a/internal/service/api.go
+++ b/internal/service/api.go
@@ -40,12 +40,10 @@ var (
 func StartAPI(cfg *config.Config) {
 	API = NewService(cfg, "api")
 	API.ServiceReload = reloadAPI
+	API.DiscoverFeatures = APIFeatures
+	API.addResponder("eventbus:api", handleAPIMessage)
 	Services["api"] = API
-	loadAPIConfig(cfg)
 	APIRequestStore = api.NewStore(API)
-	API.addResponder("eventbus:api", handleAPIMessage)
-	API.DiscoverFeatures = APIFeatures
-	startAPIListener()
 	API.start()
 }
 
@@ -100,9 +98,13 @@ func startAPIListener() {
 // reloadAPI reloads config and restarts the listener.
 func reloadAPI(cfg *config.Config) {
 	if loadAPIConfig(cfg) {
-		ctx, cancel := context.WithTimeout(context.Background(), APIServerGracefulTimeout*time.Second)
-		defer cancel()
-		_ = APIServer.Shutdown(ctx)
+		if APIServer == nil {
+			startAPIListener()
+		} else {
+			ctx, cancel := context.WithTimeout(context.Background(), APIServerGracefulTimeout*time.Second)
+			defer cancel()
+			_ = APIServer.Shutdown(ctx)
+		}
 	}
 }
 
diff --git a/internal/service/engine.go b/internal/service/engine.go
index 87d81c5..e9392e3 100644
--- a/internal/service/engine.go
+++ b/internal/service/engine.go
@@ -62,7 +62,6 @@ func StartEngine(cfg *config.Config) {
 	engine.addResponder("eventbus:return", continueSession)
 	setupEngineAPIs()
 
-	buildRuleMap(cfg)
 	engine.start()
 }
 
@@ -80,7 +79,6 @@ func createSessions(d *driver.Runtime, msg *dipper.Message) {
 		rules, ok := ruleMap[event]
 		if ok && rules != nil {
 			for _, rule := range rules {
-				dipper.Recursive(rule.Trigger.Match, engine.decryptDriverData)
 				if dipper.CompareAll(data, rule.Trigger.Match) {
 					dipper.Logger.Infof("[engine] raw event triggers an event %s.%s",
 						rule.OriginalRule.When.Source.System,
diff --git a/internal/service/operator.go b/internal/service/operator.go
index c78dcd7..6a3d0c3 100644
--- a/internal/service/operator.go
+++ b/internal/service/operator.go
@@ -7,6 +7,7 @@
 package service
 
 import (
+	"errors"
 	"fmt"
 	"strings"
 
@@ -16,12 +17,12 @@ import (
 	"github.com/mitchellh/mapstructure"
 )
 
-const (
-	// OperatorError is the base for all operator related error.
-	OperatorError dipper.Error = "operator error"
-)
+var (
+	// ErrOperatorError is the base for all operator related error.
+	ErrOperatorError = errors.New("operator error")
 
-var operator *Service
+	operator *Service
+)
 
 // StartOperator starts the operator service.
 func StartOperator(cfg *config.Config) {
@@ -76,7 +77,7 @@ func handleEventbusCommand(msg *dipper.Message) []RoutedMessage {
 
 	worker := operator.getDriverRuntime("driver:" + driver)
 	if worker == nil {
-		panic(fmt.Errorf("not defined: %s: %w", driver, OperatorError))
+		panic(fmt.Errorf("%w: not defined: %s", ErrOperatorError, driver))
 	}
 	finalParams := params
 	if params != nil {
@@ -87,7 +88,6 @@ func handleEventbusCommand(msg *dipper.Message) []RoutedMessage {
 				"data":    data,
 				"event":   event,
 				"labels":  msg.Labels,
-				"wfdata":  ctx,
 				"ctx":     ctx,
 				"params":  params,
 			}).(map[string]interface{})
@@ -99,12 +99,10 @@ func handleEventbusCommand(msg *dipper.Message) []RoutedMessage {
 			"event":   event,
 			"labels":  msg.Labels,
 			"ctx":     ctx,
-			"wfdata":  ctx,
 			"params":  params,
 		}).(map[string]interface{})
 	}
 	dipper.Logger.Debugf("[operator] interpolated function call %+v", finalParams)
-	dipper.Recursive(finalParams, operator.decryptDriverData)
 
 	msg.Payload = finalParams
 	if msg.Labels == nil {
@@ -197,7 +195,6 @@ func collapseFunction(s *config.System, f *config.Function) (string, string, map
 	}
 
 	if s != nil && s.Data != nil {
-		dipper.Recursive(s.Data, operator.decryptDriverData)
 		currentSysDataCopy, _ := dipper.DeepCopy(s.Data)
 		if sysData == nil {
 			sysData = map[string]interface{}{}
diff --git a/internal/service/service.go b/internal/service/service.go
index 33f93ce..a1232dc 100644
--- a/internal/service/service.go
+++ b/internal/service/service.go
@@ -8,6 +8,7 @@ package service
 
 import (
 	"encoding/base64"
+	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -29,9 +30,6 @@ import (
 const (
 	FeatureEmitter = "emitter"
 
-	// ServiceError indicates error condition when rendering service.
-	ServiceError dipper.Error = "service error"
-
 	// DriverGracefulTimeout is the timeout in milliseconds for a driver to gracefully shutdown.
 	DriverGracefulTimeout time.Duration = 50
 
@@ -79,8 +77,13 @@ type Service struct {
 	healthy            bool
 }
 
-// Services holds a catalog of running services in this daemon process.
-var Services = map[string]*Service{}
+var (
+	// Services holds a catalog of running services in this daemon process.
+	Services = map[string]*Service{}
+
+	// ErrServiceError indicates error condition when rendering service.
+	ErrServiceError = errors.New("service error")
+)
 
 // NewService creates a service with given config and name.
 func NewService(cfg *config.Config, name string) *Service {
@@ -126,18 +129,18 @@ func (s *Service) GetStream(feature string) io.Writer {
 		}
 
 		if runtime.State != driver.DriverAlive {
-			panic(fmt.Errorf("feature failed or loading timeout: %s: %w", feature, ServiceError))
+			panic(fmt.Errorf("%w: feature failed or loading timeout: %s", ErrServiceError, feature))
 		}
 
 		return runtime.Output
 	}
-	panic(fmt.Errorf("feature not loaded: %s: %w", feature, ServiceError))
+	panic(fmt.Errorf("%w: feature not loaded: %s", ErrServiceError, feature))
 }
 
 func (s *Service) decryptDriverData(key string, val interface{}) (ret interface{}, replace bool) {
-	dipper.Logger.Debugf("[%s] decrypting %s", s.name, key)
 	if str, ok := val.(string); ok {
 		if strings.HasPrefix(str, "ENC[") {
+			dipper.Logger.Debugf("[%s] decrypting %s", s.name, key)
 			parts := strings.SplitN(str[4:len(str)-1], ",", 2)
 			encDriver := parts[0]
 			data := []byte(parts[1])
@@ -166,7 +169,7 @@ func (s *Service) loadFeature(feature string) (affected bool, driverName string,
 			if err, ok := r.(error); ok {
 				rerr = err
 			} else {
-				rerr = fmt.Errorf("%+v: %w", r, ServiceError)
+				rerr = fmt.Errorf("%w: %+v", ErrServiceError, r)
 			}
 		}
 	}()
@@ -182,9 +185,9 @@ func (s *Service) loadFeature(feature string) (affected bool, driverName string,
 	if strings.HasPrefix(feature, "driver:") {
 		driverName = feature[7:]
 	} else {
-		driverName, ok = s.config.GetDriverDataStr(fmt.Sprintf("daemon.featureMap.%s.%s", s.name, feature))
+		driverName, ok = s.config.GetStagedDriverDataStr(fmt.Sprintf("daemon.featureMap.%s.%s", s.name, feature))
 		if !ok {
-			driverName, ok = s.config.GetDriverDataStr(fmt.Sprintf("daemon.featureMap.global.%s", feature))
+			driverName, ok = s.config.GetStagedDriverDataStr(fmt.Sprintf("daemon.featureMap.global.%s", feature))
 		}
 		if !ok {
 			panic("driver not defined for the feature")
@@ -192,22 +195,19 @@ func (s *Service) loadFeature(feature string) (affected bool, driverName string,
 	}
 	dipper.Logger.Infof("[%s] mapping feature %s to driver %s", s.name, feature, driverName)
 
-	driverData, _ := s.config.GetDriverData(driverName)
+	driverData, _ := s.config.GetStagedDriverData(driverName)
 	var dynamicData interface{}
 	if strings.HasPrefix(feature, "driver:") {
 		dynamicData = s.dynamicFeatureData[feature]
 	}
 
 	var driverHandler driver.Handler
-	if driverMeta, ok := s.config.GetDriverData(fmt.Sprintf("daemon.drivers.%s", driverName)); ok {
+	if driverMeta, ok := s.config.GetStagedDriverData(fmt.Sprintf("daemon.drivers.%s", driverName)); ok {
 		driverHandler = driver.NewDriver(driverMeta.(map[string]interface{}))
 	} else {
 		panic("unable to get driver metadata")
 	}
 
-	dipper.Recursive(driverData, s.decryptDriverData)
-	dipper.Recursive(dynamicData, s.decryptDriverData)
-
 	dipper.Logger.Debugf("[%s] driver %s meta %v", s.name, driverName, driverHandler.Meta())
 	driverRuntime := driver.Runtime{
 		Feature:     feature,
@@ -272,11 +272,17 @@ func (s *Service) coldReload(driverRuntime *driver.Runtime, oldRuntime *driver.R
 func (s *Service) start() {
 	go func() {
 		dipper.Logger.Infof("[%s] starting service", s.name)
+		s.config.AdvanceStage(s.name, config.StageBooting)
 		featureList := s.getFeatureList()
 		s.loadRequiredFeatures(featureList, true)
 		go s.serviceLoop()
 		time.Sleep(time.Second)
+		s.config.AdvanceStage(s.name, config.StageDiscovering, s.decryptDriverData)
 		s.loadAdditionalFeatures(featureList)
+		s.config.AdvanceStage(s.name, config.StageServing)
+		if s.ServiceReload != nil {
+			s.ServiceReload(s.config)
+		}
 		s.healthy = true
 		go s.metricsLoop()
 	}()
@@ -284,50 +290,47 @@ func (s *Service) start() {
 
 // Reload the service when configuration changes are detected.
 func (s *Service) Reload() {
-	dipper.Logger.Infof("[%s] reloading service", s.name)
-	var featureList map[string]bool
-
-	func() {
-		defer func() {
-			if r := recover(); r != nil {
-				s.healthy = false
-				dipper.Logger.Errorf("[%s] reverting config due to fatal failure %v", s.name, r)
-				s.config.RollBack()
+	defer func() {
+		if r := recover(); r != nil {
+			s.healthy = false
+			if errors.Is(r.(error), config.ErrConfigRollback) {
+				dipper.Logger.Errorf("[%s] reverting config initiated outside of the service", s.name)
+
+				return
 			}
-		}()
-		if s.ServiceReload != nil {
-			s.ServiceReload(s.config)
+			dipper.Logger.Errorf("[%s] reverting config due to fatal failure %v", s.name, r)
+			s.config.RollBack()
 		}
-		featureList = s.getFeatureList()
-		s.loadRequiredFeatures(featureList, false)
 	}()
-
+	dipper.Logger.Infof("[%s] reloading service", s.name)
+	s.config.AdvanceStage(s.name, config.StageBooting)
+	featureList := s.getFeatureList()
+	s.loadRequiredFeatures(featureList, false)
+	s.config.AdvanceStage(s.name, config.StageDiscovering, s.decryptDriverData)
 	s.loadAdditionalFeatures(featureList)
+	s.config.AdvanceStage(s.name, config.StageServing)
+	if s.ServiceReload != nil {
+		s.ServiceReload(s.config)
+	}
 	s.healthy = true
 	s.removeUnusedFeatures(featureList)
 }
 
 func (s *Service) getFeatureList() map[string]bool {
 	featureList := map[string]bool{}
-	if cfgItem, ok := s.config.GetDriverData("daemon.features.global"); ok {
+	if cfgItem, ok := s.config.GetStagedDriverData("daemon.features.global"); ok {
 		for _, feature := range cfgItem.([]interface{}) {
 			featureName := feature.(map[string]interface{})["name"].(string)
 			featureList[featureName], _ = dipper.GetMapDataBool(feature, "required")
 		}
 	}
-	if cfgItem, ok := s.config.GetDriverData("daemon.features." + s.name); ok {
+	if cfgItem, ok := s.config.GetStagedDriverData("daemon.features." + s.name); ok {
 		for _, feature := range cfgItem.([]interface{}) {
 			featureName := feature.(map[string]interface{})["name"].(string)
 			featureList[featureName], _ = dipper.GetMapDataBool(feature, "required")
 		}
 	}
-	if s.DiscoverFeatures != nil {
-		s.dynamicFeatureData = s.DiscoverFeatures(s.config.DataSet)
-		for name := range s.dynamicFeatureData {
-			featureList[name] = false
-		}
-	}
-	dipper.Logger.Debugf("[%s] final feature list %+v", s.name, featureList)
+	dipper.Logger.Debugf("[%s] preliminary feature list %+v", s.name, featureList)
 
 	return featureList
 }
@@ -394,6 +397,14 @@ func (s *Service) loadRequiredFeatures(featureList map[string]bool, boot bool) {
 }
 
 func (s *Service) loadAdditionalFeatures(featureList map[string]bool) {
+	if s.DiscoverFeatures != nil {
+		s.dynamicFeatureData = s.DiscoverFeatures(s.config.Staged)
+		for name := range s.dynamicFeatureData {
+			featureList[name] = false
+		}
+	}
+	dipper.Logger.Debugf("[%s] final feature list %+v", s.name, featureList)
+
 	for feature, required := range featureList {
 		if !required {
 			affected, driverName, err := s.loadFeature(feature)
@@ -426,6 +437,7 @@ func (s *Service) loadAdditionalFeatures(featureList map[string]bool) {
 func (s *Service) serviceLoop() {
 	daemon.Children.Add(1)
 	defer daemon.Children.Done()
+
 	for !daemon.ShuttingDown {
 		var cases []reflect.SelectCase
 		var orderedRuntimes []*driver.Runtime
diff --git a/internal/workflow/continue.go b/internal/workflow/continue.go
index 6beba83..76852de 100644
--- a/internal/workflow/continue.go
+++ b/internal/workflow/continue.go
@@ -87,6 +87,11 @@ func (w *Session) routeNext(msg *dipper.Message) int {
 
 // mergeContext merges child workflow exported context to parent workflow.
 func (w *Session) mergeContext(exports []map[string]interface{}) {
+	// for multi-thread workflows, we might have multiple child workflows
+	// complete at the same time, and try merging context back at the same
+	// time and causing concurrent map write, thus using lock to avoid that.
+	w.ctxLock.Lock()
+	defer w.ctxLock.Unlock()
 	for _, export := range exports {
 		w.ctx = dipper.MergeMap(w.ctx, export)
 		w.processNoExport(export)
diff --git a/internal/workflow/session.go b/internal/workflow/session.go
index 3f06429..5aac96a 100644
--- a/internal/workflow/session.go
+++ b/internal/workflow/session.go
@@ -8,17 +8,17 @@ package workflow
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"reflect"
+	"sync"
 
 	"github.com/honeydipper/honeydipper/internal/config"
 	"github.com/honeydipper/honeydipper/pkg/dipper"
 )
 
-const (
-	// WorkflowError is the base error for all workflow related errors.
-	WorkflowError dipper.Error = "workflow error"
-)
+// ErrWorkflowError is the base error for all workflow related errors.
+var ErrWorkflowError = errors.New("workflow error")
 
 // Session is the data structure about a running workflow and its definition.
 type Session struct {
@@ -30,6 +30,7 @@ type Session struct {
 	loopCount      int   // counter for looping
 	parent         string
 	ctx            map[string]interface{}
+	ctxLock        *sync.Mutex
 	event          map[string]interface{}
 	exported       []map[string]interface{}
 	elseBranch     *config.Workflow
@@ -222,7 +223,7 @@ func (w *Session) initCTX(msg *dipper.Message) {
 			}
 			name, ok := n.(string)
 			if !ok {
-				panic(fmt.Errorf("expected list of strings in contexts in workflow: %s: %w", w.workflow.Name, WorkflowError))
+				panic(fmt.Errorf("%w: expected list of strings in contexts in workflow: %s", ErrWorkflowError, w.workflow.Name))
 			}
 			if name != "" {
 				// at this stage the hooks flag is added only through `context` not `contexts`
@@ -413,7 +414,7 @@ func (w *Session) prepare(msg *dipper.Message, parent interface{}, ctx map[strin
 func (w *Session) createChildSessionWithName(name string, msg *dipper.Message) *Session {
 	src, ok := w.store.Helper.GetConfig().DataSet.Workflows[name]
 	if !ok {
-		panic(fmt.Errorf("not defined: %s: %w", name, WorkflowError))
+		panic(fmt.Errorf("%w: not defined: %s", ErrWorkflowError, name))
 	}
 	if src.Name == "" {
 		src.Name = name
diff --git a/internal/workflow/store.go b/internal/workflow/store.go
index e437fca..0451aac 100644
--- a/internal/workflow/store.go
+++ b/internal/workflow/store.go
@@ -7,6 +7,8 @@
 package workflow
 
 import (
+	"sync"
+
 	"github.com/honeydipper/honeydipper/internal/config"
 	"github.com/honeydipper/honeydipper/internal/daemon"
 	"github.com/honeydipper/honeydipper/pkg/dipper"
@@ -50,6 +52,7 @@ func (s *SessionStore) newSession(parent string, eventUUID string, wf *config.Wo
 		store:    s,
 		EventID:  eventUUID,
 		workflow: wf,
+		ctxLock:  &sync.Mutex{},
 	}
 
 	performing := w.setPerforming("")
diff --git a/pkg/dipper/comm.go b/pkg/dipper/comm.go
index d2105b8..1358381 100644
--- a/pkg/dipper/comm.go
+++ b/pkg/dipper/comm.go
@@ -11,6 +11,7 @@ import (
 	"bytes"
 	"encoding/gob"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"strings"
@@ -124,7 +125,7 @@ func FetchRawMessage(in io.Reader) (msg *Message) {
 	)
 
 	_, err := fmt.Fscanln(in, &channel, &subject, &numLabels, &size)
-	if err == io.EOF {
+	if errors.Is(err, io.EOF) {
 		panic(err)
 	} else if err != nil {
 		errMsg := fmt.Sprintf("%+v", err)
@@ -258,12 +259,12 @@ func MessageCopy(m *Message) (*Message, error) {
 	dec := gob.NewDecoder(&buf)
 	err := enc.Encode(*m)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("%w", err)
 	}
 	var mcopy Message
 	err = dec.Decode(&mcopy)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("%w", err)
 	}
 
 	return &mcopy, nil
diff --git a/pkg/dipper/commandHandler.go b/pkg/dipper/commandHandler.go
index 2c5925c..59cdf33 100644
--- a/pkg/dipper/commandHandler.go
+++ b/pkg/dipper/commandHandler.go
@@ -50,7 +50,7 @@ func (p *CommandProvider) ReturnError(call *Message, pattern string, args ...int
 	})
 	Logger.Warningf("[operator] %s", errText)
 
-	return Error(errText)
+	return errors.New(errText)
 }
 
 // Return : return a value to rpc caller.
diff --git a/pkg/dipper/errorHandling.go b/pkg/dipper/errorHandling.go
index 4b7f1d9..2878630 100644
--- a/pkg/dipper/errorHandling.go
+++ b/pkg/dipper/errorHandling.go
@@ -8,23 +8,6 @@ package dipper
 
 import "github.com/go-errors/errors"
 
-// Error type is a simple error type used by dipper.
-type Error string
-
-// Error gives the message of the Error.
-func (e Error) Error() string {
-	return string(e)
-}
-
-// Is check if a given error matches this error.
-func (e Error) Is(target error) bool {
-	if t, ok := target.(Error); ok {
-		return t == e
-	}
-
-	return false
-}
-
 // SafeExitOnError : use this function in defer statement to ignore errors.
 func SafeExitOnError(args ...interface{}) {
 	if r := recover(); r != nil {
diff --git a/pkg/dipper/interpolation.go b/pkg/dipper/interpolation.go
index fd45ab9..1fb7cd6 100644
--- a/pkg/dipper/interpolation.go
+++ b/pkg/dipper/interpolation.go
@@ -8,6 +8,7 @@ package dipper
 
 import (
 	"bytes"
+	"errors"
 	"fmt"
 	"strings"
 	"text/template"
@@ -17,10 +18,8 @@ import (
 	"github.com/ghodss/yaml"
 )
 
-const (
-	// InterpolationError represents all errors thrown due to failure to interpolate.
-	InterpolationError Error = "config error"
-)
+// ErrInterpolationError represents all errors thrown due to failure to interpolate.
+var ErrInterpolationError = errors.New("config error")
 
 // FuncMap : used to add functions to the go templates.
 var FuncMap = template.FuncMap{
@@ -87,13 +86,13 @@ func InterpolateDollarStr(v string, data interface{}) interface{} {
 
 	quote := strings.IndexAny(parsed, "\"'`")
 	if allowNull && quote >= 0 {
-		panic(fmt.Errorf("allow nil combine with default value: %s: %w", v, InterpolationError))
+		panic(fmt.Errorf("%w: allow nil combine with default value: %s", ErrInterpolationError, v))
 	}
 
 	var keys []string
 	if quote > 0 {
 		if parsed[quote-1] != ',' {
-			panic(fmt.Errorf("missing comma: %s: %w", v, InterpolationError))
+			panic(fmt.Errorf("%w: missing comma: %s", ErrInterpolationError, v))
 		}
 		keys = strings.Split(parsed[:quote-1], ",")
 	} else if quote < 0 {
@@ -113,7 +112,7 @@ func InterpolateDollarStr(v string, data interface{}) interface{} {
 
 	if quote >= 0 {
 		if parsed[quote] != parsed[len(parsed)-1] {
-			panic(fmt.Errorf("quotes not matching: %s: %w", parsed, InterpolationError))
+			panic(fmt.Errorf("%w: quotes not matching: %s", ErrInterpolationError, parsed))
 		}
 
 		return parsed[quote+1 : len(parsed)-1]
@@ -122,7 +121,7 @@ func InterpolateDollarStr(v string, data interface{}) interface{} {
 	if allowNull {
 		return nil
 	}
-	panic(fmt.Errorf("invalid path: %s: %w", v[1:], InterpolationError))
+	panic(fmt.Errorf("%w: invalid path: %s", ErrInterpolationError, v[1:]))
 }
 
 // Interpolate : go through the map data structure to find and parse all the templates.
diff --git a/pkg/dipper/logging.go b/pkg/dipper/logging.go
index fa428e6..bc805b0 100644
--- a/pkg/dipper/logging.go
+++ b/pkg/dipper/logging.go
@@ -12,7 +12,7 @@ import (
 	"strings"
 
 	"github.com/op/go-logging"
-	"golang.org/x/crypto/ssh/terminal"
+	"golang.org/x/term"
 )
 
 // Logger provides methods to log to the configured logger backend.
@@ -28,7 +28,7 @@ func initLogBackend(level logging.Level, logFile *os.File) logging.Backend {
 	backend := logging.NewLogBackend(logFile, "", 0)
 
 	formatStr := `%{time:15:04:05.000} %{module}.%{shortfunc} ▶ %{level:.4s} %{id:03x} %{message}`
-	if terminal.IsTerminal(int(logFile.Fd())) {
+	if term.IsTerminal(int(logFile.Fd())) {
 		formatStr = `%{color}` + formatStr + `%{color:reset}`
 	}
 	format := logging.MustStringFormatter(formatStr)
diff --git a/pkg/dipper/mapUtils.go b/pkg/dipper/mapUtils.go
index 9ca6231..85ae472 100644
--- a/pkg/dipper/mapUtils.go
+++ b/pkg/dipper/mapUtils.go
@@ -7,6 +7,7 @@
 package dipper
 
 import (
+	"errors"
 	"fmt"
 	"reflect"
 	"strconv"
@@ -16,13 +17,8 @@ import (
 	"github.com/imdario/mergo"
 )
 
-const (
-	// MapError are all errors thrown in map manipulation.
-	MapError Error = "map error"
-
-	// MergeError are all errors throw during map merge and combin.
-	MergeError Error = "merge error"
-)
+// ErrMapError are all errors thrown in map manipulation.
+var ErrMapError = errors.New("map error")
 
 // GetMapData : get the data from the deep map following a KV path.
 func GetMapData(from interface{}, path string) (ret interface{}, ok bool) {
@@ -65,7 +61,7 @@ func GetMapData(from interface{}, path string) (ret interface{}, ok bool) {
 func MustGetMapData(from interface{}, path string) interface{} {
 	ret, ok := GetMapData(from, path)
 	if !ok {
-		panic(fmt.Errorf("path not valid: %s: %w", path, MapError))
+		panic(fmt.Errorf("%w: path not valid: %s", ErrMapError, path))
 	}
 
 	return ret
@@ -129,11 +125,14 @@ func MustGetMapDataBool(from interface{}, path string) bool {
 			return flag
 		}
 	}
-	panic(fmt.Errorf("not a bool: %s: %w", path, MapError))
+	panic(fmt.Errorf("%w: not a bool: %s", ErrMapError, path))
 }
 
+// ItemProcessor is a function processing one of the items in a data structure.
+type ItemProcessor func(key string, val interface{}) (newval interface{}, ok bool)
+
 // Recursive : enumerate all the data element deep into the map call the function provided.
-func Recursive(from interface{}, process func(key string, val interface{}) (newval interface{}, ok bool)) {
+func Recursive(from interface{}, process ItemProcessor) {
 	RecursiveWithPrefix(nil, "", "", from, process)
 }
 
@@ -143,7 +142,7 @@ func RecursiveWithPrefix(
 	prefixes string,
 	key interface{},
 	from interface{},
-	process func(key string, val interface{}) (newval interface{}, ok bool),
+	process ItemProcessor,
 ) {
 	keyStr := fmt.Sprintf("%v", key)
 	newPrefixes := keyStr
@@ -192,7 +191,7 @@ func RecursiveWithPrefix(
 					vparent.Field(key.(int)).Set(vval)
 				}
 			default:
-				panic(fmt.Errorf("unable to change: %s in %s: %w", key, prefixes, MapError))
+				panic(fmt.Errorf("%w: unable to change: %s in %s", ErrMapError, key, prefixes))
 			}
 		}
 	}
@@ -273,7 +272,7 @@ func DeepCopyMap(m map[string]interface{}) (map[string]interface{}, error) {
 	}
 	retMap, ok := ret.(map[string]interface{})
 	if !ok {
-		return nil, fmt.Errorf("not a map: %w", MapError)
+		return nil, fmt.Errorf("%w: not a map", ErrMapError)
 	}
 
 	return retMap, nil
diff --git a/pkg/dipper/rpc.go b/pkg/dipper/rpc.go
index 4433f27..ecca318 100644
--- a/pkg/dipper/rpc.go
+++ b/pkg/dipper/rpc.go
@@ -7,21 +7,22 @@
 package dipper
 
 import (
+	"errors"
 	"fmt"
 	"io"
 	"sync"
 	"time"
 )
 
-const (
-	// TimeoutError indicates a timeout error.
-	TimeoutError Error = "timeout"
+// DefaultRPCTimeout is the default timeout in seconds for RPC calls.
+const DefaultRPCTimeout time.Duration = 10
 
-	// RPCError indicates errors happened during RPC call.
-	RPCError Error = "rpc error"
+var (
+	// ErrTimeout indicates a timeout error.
+	ErrTimeout = errors.New("timeout")
 
-	// DefaultRPCTimeout is the default timeout in seconds for RPC calls.
-	DefaultRPCTimeout time.Duration = 10
+	// ErrRPCError indicates errors happened during RPC call.
+	ErrRPCError = errors.New("rpc error")
 )
 
 // RPCHandler : a type of functions that handle RPC calls between drivers.
@@ -101,7 +102,7 @@ func (c *RPCCallerBase) CallRaw(feature string, method string, params []byte) ([
 
 		return msg.([]byte), nil
 	case <-time.After(time.Second * DefaultRPCTimeout):
-		return nil, TimeoutError
+		return nil, ErrTimeout
 	}
 }
 
@@ -119,7 +120,7 @@ func (c *RPCCallerBase) CallRawNoWait(feature string, method string, params []by
 
 	out := c.Parent.GetStream(feature)
 	if out == nil {
-		return fmt.Errorf("feature not available: %s: %w", feature, RPCError)
+		return fmt.Errorf("%w: feature not available: %s", ErrRPCError, feature)
 	}
 
 	// making the call by sending a message
@@ -147,7 +148,7 @@ func (c *RPCCallerBase) HandleReturn(m *Message) {
 	reason, ok := m.Labels["error"]
 
 	if ok {
-		result <- fmt.Errorf("reason: %s: %w", reason, RPCError)
+		result <- fmt.Errorf("%w: reason: %s", ErrRPCError, reason)
 	} else {
 		result <- m.Payload
 	}
diff --git a/pkg/dipper/sync.go b/pkg/dipper/sync.go
new file mode 100644
index 0000000..75b1a6f
--- /dev/null
+++ b/pkg/dipper/sync.go
@@ -0,0 +1,35 @@
+// Copyright 2019 Honey Science Corporation
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, you can obtain one at http://mozilla.org/MPL/2.0/.
+
+// Package dipper is a library used for developing drivers for Honeydipper.
+package dipper
+
+import "sync"
+
+// WaitGroupDone is a way to safely decrement the counter for a WaitGroup.
+func WaitGroupDone(wg *sync.WaitGroup) (ok bool) {
+	defer func() {
+		if recover() != nil {
+			ok = false
+		}
+	}()
+
+	wg.Done()
+	ok = true
+
+	return ok
+}
+
+// WaitGroupDoneAll is a way to safely release the whole WaitGroup.
+func WaitGroupDoneAll(wg *sync.WaitGroup) {
+	defer func() {
+		_ = recover()
+	}()
+
+	for {
+		wg.Done()
+	}
+}


Test patch:
diff --git a/cmd/honeydipper/configcheck_test.go b/cmd/honeydipper/configcheck_test.go
index 05f3309..ba72337 100644
--- a/cmd/honeydipper/configcheck_test.go
+++ b/cmd/honeydipper/configcheck_test.go
@@ -120,7 +120,7 @@ func TestCheckWorkflowDriverFunctionDriver(t *testing.T) {
 	checkWorkflowDriver(workflow, cfg)
 }
 
-// TestCheckWorkflowFunction
+// TestCheckWorkflowFunction.
 var wfFunctionTestCases = []struct {
 	wf  config.Workflow
 	cfg *config.Config
@@ -181,7 +181,7 @@ func testCheckWorkflowFunctionHelper(t *testing.T, wf config.Workflow, cfg *conf
 	checkWorkflowFunction(wf, cfg)
 }
 
-// TestCheckWorkflowActions
+// TestCheckWorkflowActions.
 var wfActionTestCases = []struct {
 	in  config.Workflow
 	out string
@@ -219,7 +219,7 @@ func testCheckWorkflowActionsHelper(t *testing.T, wf config.Workflow, out string
 	checkWorkflowActions(wf)
 }
 
-// TestWorkFlowConditions
+// TestWorkFlowConditions.
 var wfConditionsTestCases = []struct {
 	in  config.Workflow
 	out string
@@ -299,7 +299,7 @@ func TestHasInterpolation(t *testing.T) {
 	}
 }
 
-// helper func
+// helper func.
 func recoverAssertion(out string, t *testing.T) {
 	expected := out
 	var msg string
diff --git a/cmd/honeydipper/integration_test.go b/cmd/honeydipper/integration_test.go
index 2110693..14ec492 100644
--- a/cmd/honeydipper/integration_test.go
+++ b/cmd/honeydipper/integration_test.go
@@ -60,6 +60,7 @@ func intTestDaemonStartup(t *testing.T) {
 		workingBranch = ref[len(ref)-1]
 	}
 	cfg := config.Config{
+		Services: []string{"engine", "receiver", "operator"},
 		InitRepo: config.RepoInfo{
 			Repo:   "../..",
 			Branch: workingBranch,
diff --git a/deployments/helm/honeydipper/templates/tests/test-webhook-service.yaml b/deployments/helm/honeydipper/templates/tests/test-webhook-service.yaml
deleted file mode 100644
index 925c9fc..0000000
--- a/deployments/helm/honeydipper/templates/tests/test-webhook-service.yaml
+++ /dev/null
@@ -1,16 +0,0 @@
-apiVersion: v1
-kind: Pod
-metadata:
-  name: "{{ .Release.Name }}-webhook-test"
-  annotations:
-    "helm.sh/hook": test-success
-spec:
-  containers:
-  - name: {{ .Release.Name }}-webhook-test
-    image: alpine:3.8
-    command:
-      - "sh"
-      - "-c"
-      - |
-        wget -nv http://{{ include "honeydipper.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.drivers.webhook.service.port }}/health
-  restartPolicy: Never
diff --git a/deployments/helm/honeydipper/test-values.yaml b/deployments/helm/honeydipper/test-values.yaml
deleted file mode 100644
index dfcbd83..0000000
--- a/deployments/helm/honeydipper/test-values.yaml
+++ /dev/null
@@ -1,4 +0,0 @@
-daemon:
-  env:
-    REPO: https://github.com/honeydipper/honeydipper
-    BOOTSTRAP_PATH: /deployments/helm/honeydipper/test_fixtures
diff --git a/deployments/helm/honeydipper/test_fixtures/init.yaml b/deployments/helm/honeydipper/test_fixtures/init.yaml
deleted file mode 100644
index 5ec812c..0000000
--- a/deployments/helm/honeydipper/test_fixtures/init.yaml
+++ /dev/null
@@ -1,11 +0,0 @@
----
-repos:
-  - repo: https://github.com/honeydipper/honeydipper-config-essentials
-
-rules:
-  - when:
-      driver: webhook
-      conditions:
-        url: "/health"
-    do:
-      content: noop
diff --git a/internal/api/request_test.go b/internal/api/request_test.go
index 363b1ca..45052d9 100644
--- a/internal/api/request_test.go
+++ b/internal/api/request_test.go
@@ -49,10 +49,10 @@ func requestTest(t *testing.T, caseName string) (*Store, *RequestTestCase) {
 	dipper.Must(mapstructure.Decode(buffer, c))
 
 	// convert all times from test definition to milliseconds
-	c.Def.AckTimeout = c.Def.AckTimeout * time.Millisecond
-	c.Def.Timeout = c.Def.Timeout * time.Millisecond
+	c.Def.AckTimeout *= time.Millisecond
+	c.Def.Timeout *= time.Millisecond
 	for i := range c.Returns {
-		c.Returns[i].Delay = c.Returns[i].Delay * time.Millisecond
+		c.Returns[i].Delay *= time.Millisecond
 	}
 
 	ctrl := gomock.NewController(t)
@@ -75,6 +75,7 @@ func requestTest(t *testing.T, caseName string) (*Store, *RequestTestCase) {
 	nextUUID := func() string {
 		uuid := uuids[0]
 		uuids = uuids[1:]
+
 		return uuid
 	}
 	l.newUUID = nextUUID
@@ -114,6 +115,7 @@ func requestTest(t *testing.T, caseName string) (*Store, *RequestTestCase) {
 	}
 
 	l.HandleHTTPRequest(mockReqCtx, c.Def)
+
 	return l, c
 }
 
diff --git a/internal/api/response_test.go b/internal/api/response_test.go
index 2106287..f642bfd 100644
--- a/internal/api/response_test.go
+++ b/internal/api/response_test.go
@@ -52,6 +52,7 @@ func responseTest(t *testing.T, c *ResponseTestCase) {
 		case <-msgAvailable:
 		case <-time.After(delay):
 		}
+
 		return m
 	}
 
@@ -106,7 +107,7 @@ func TestTypeAllAPIResponse(t *testing.T) {
 		},
 		returnMessages: []ReturnMessage{
 			{
-				Delay: 30 * time.Millisecond,
+				Delay: 3000 * time.Millisecond,
 				Msg: &dipper.Message{
 					Channel: "eventbus",
 					Subject: "api",
@@ -118,7 +119,7 @@ func TestTypeAllAPIResponse(t *testing.T) {
 				},
 			},
 			{
-				Delay: 30 * time.Millisecond,
+				Delay: 3000 * time.Millisecond,
 				Msg: &dipper.Message{
 					Channel: "eventbus",
 					Subject: "api",
@@ -163,7 +164,7 @@ func TestTypeFirstAPIResponse(t *testing.T) {
 		},
 		returnMessages: []ReturnMessage{
 			{
-				Delay: 30 * time.Millisecond,
+				Delay: 3000 * time.Millisecond,
 				Msg: &dipper.Message{
 					Channel: "eventbus",
 					Subject: "api",
@@ -209,7 +210,7 @@ func TestTypeMatchAPIResponse(t *testing.T) {
 		},
 		returnMessages: []ReturnMessage{
 			{
-				Delay: 30 * time.Millisecond,
+				Delay: 3000 * time.Millisecond,
 				Msg: &dipper.Message{
 					Channel: "eventbus",
 					Subject: "api",
@@ -221,7 +222,7 @@ func TestTypeMatchAPIResponse(t *testing.T) {
 				},
 			},
 			{
-				Delay: 30 * time.Millisecond,
+				Delay: 3000 * time.Millisecond,
 				Msg: &dipper.Message{
 					Channel: "eventbus",
 					Subject: "api",
@@ -265,7 +266,7 @@ func TestTypeMatchAPIResponseReturnError(t *testing.T) {
 		},
 		returnMessages: []ReturnMessage{
 			{
-				Delay: 30 * time.Millisecond,
+				Delay: 3000 * time.Millisecond,
 				Msg: &dipper.Message{
 					Channel: "eventbus",
 					Subject: "api",
@@ -277,7 +278,7 @@ func TestTypeMatchAPIResponseReturnError(t *testing.T) {
 				},
 			},
 			{
-				Delay: 30 * time.Millisecond,
+				Delay: 3000 * time.Millisecond,
 				Msg: &dipper.Message{
 					Channel: "eventbus",
 					Subject: "api",
diff --git a/internal/api/test_fixtures/TypeAllAPI.yaml b/internal/api/test_fixtures/TypeAllAPI.yaml
index 98e4e70..b6f4128 100644
--- a/internal/api/test_fixtures/TypeAllAPI.yaml
+++ b/internal/api/test_fixtures/TypeAllAPI.yaml
@@ -8,6 +8,11 @@ def:
   reqType: 1
   service: foo
   object: event
+  ackTimeout: 1000
+
+# store config
+config:
+  writeTimeout: 2000
 
 # mock incoming call
 path: /test_type_all
diff --git a/internal/api/test_fixtures/TypeAllAPITimeout.yaml b/internal/api/test_fixtures/TypeAllAPITimeout.yaml
index d063320..23e8e4e 100644
--- a/internal/api/test_fixtures/TypeAllAPITimeout.yaml
+++ b/internal/api/test_fixtures/TypeAllAPITimeout.yaml
@@ -8,8 +8,13 @@ def:
   reqType: 1
   service: foo
   object: event
+  ackTimeout: 1000
   timeout: 1
 
+# store config
+config:
+  writeTimeout: 2000
+
 # mock incoming call
 path: /test_type_all
 
diff --git a/internal/api/test_fixtures/TypeFirstAPI.yaml b/internal/api/test_fixtures/TypeFirstAPI.yaml
index ad2a733..69edafb 100644
--- a/internal/api/test_fixtures/TypeFirstAPI.yaml
+++ b/internal/api/test_fixtures/TypeFirstAPI.yaml
@@ -8,6 +8,11 @@ def:
   reqType: 0
   service: foo
   object: event
+  ackTimeout: 1000
+
+# store config
+config:
+  writeTimeout: 2000
 
 # mock incoming call
 path: /test_type_first
diff --git a/internal/config/config_stage_test.go b/internal/config/config_stage_test.go
new file mode 100644
index 0000000..d186ce8
--- /dev/null
+++ b/internal/config/config_stage_test.go
@@ -0,0 +1,80 @@
+// Copyright 2019 Honey Science Corporation
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, you can obtain one at http://mozilla.org/MPL/2.0/.
+
+// +build !integration
+
+package config
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestAdvanceStage(t *testing.T) {
+	cfg := &Config{
+		Services: []string{"testsvc1", "testsvc2"},
+		Staged: &DataSet{
+			Drivers: map[string]interface{}{
+				"driver1": map[string]interface{}{
+					"k1": "v1",
+				},
+			},
+		},
+	}
+	cfg.ResetStage()
+
+	processor := func(name string, val interface{}) (interface{}, bool) {
+		if v, ok := val.(string); ok && v == "v1" {
+			return "v1+", true
+		}
+		return nil, false
+	}
+
+	assert.PanicsWithValue(t, ErrConfigRollback, func() { cfg.AdvanceStage("testsvc1", StageDiscovering) }, "cannot skipping booting stage")
+	go cfg.AdvanceStage("testsvc1", StageBooting, processor)
+	time.Sleep(time.Millisecond)
+	assert.NotEqual(t, StageBooting, cfg.Stage, "config stage not advanced")
+	assert.Equal(t, "v1", cfg.Staged.Drivers["driver1"].(map[string]interface{})["k1"], "config not touched, waiting")
+	cfg.AdvanceStage("testsvc2", StageBooting, processor)
+	assert.Equal(t, StageBooting, cfg.Stage, "config stage advanced")
+	assert.Equal(t, "v1+", cfg.Staged.Drivers["driver1"].(map[string]interface{})["k1"], "config touched after advance stage")
+}
+
+func TestRollBack(t *testing.T) {
+	cfg := &Config{
+		Services: []string{"testsvc1", "testsvc2"},
+		Staged:   &DataSet{},
+		LastRunningConfig: struct {
+			DataSet *DataSet
+			Loaded  map[RepoInfo]*Repo
+		}{
+			DataSet: &DataSet{
+				Drivers: map[string]interface{}{
+					"driver1": map[string]interface{}{
+						"k1": "v1",
+					},
+				},
+			},
+			Loaded: map[RepoInfo]*Repo{},
+		},
+	}
+	cfg.ResetStage()
+	cfg.OnChange = func() {
+		assert.Equal(t, "v1", cfg.Staged.Drivers["driver1"].(map[string]interface{})["k1"], "config set to last used data")
+		go cfg.AdvanceStage("testsvc1", StageBooting)
+		cfg.AdvanceStage("testsvc2", StageBooting)
+		assert.Equal(t, StageBooting, cfg.Stage, "config stage advanced")
+	}
+
+	go cfg.AdvanceStage("testsvc1", StageBooting)
+	cfg.AdvanceStage("testsvc2", StageBooting)
+	go func() {
+		assert.PanicsWithValue(t, ErrConfigRollback, func() { cfg.AdvanceStage("testsvc1", StageDiscovering) }, "cancelled due to rollback")
+	}()
+	cfg.RollBack()
+}
diff --git a/internal/config/config_test.go b/internal/config/config_test.go
index 9395f08..c54dfca 100644
--- a/internal/config/config_test.go
+++ b/internal/config/config_test.go
@@ -12,6 +12,7 @@ import (
 	"regexp"
 	"testing"
 
+	"github.com/honeydipper/honeydipper/pkg/dipper"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -47,7 +48,7 @@ func TestConfigGetDriverData(t *testing.T) {
 
 func TestRegexParsing(t *testing.T) {
 	config := &Config{
-		DataSet: &DataSet{
+		Staged: &DataSet{
 			Workflows: map[string]Workflow{
 				"test-workflow": {
 					Match: map[string]interface{}{
@@ -73,11 +74,10 @@ func TestRegexParsing(t *testing.T) {
 		},
 	}
 
-	assert.NotPanics(t, func() { config.parseWorkflowRegex() }, "parsing regex in config should not panic")
+	assert.NotPanics(t, func() { config.RecursiveStaged(dipper.RegexParser); config.DataSet = config.Staged }, "parsing regex in config should not panic")
 	assert.IsType(t, &regexp.Regexp{}, config.DataSet.Workflows["test-workflow"].Match.(map[string]interface{})["key1"], "workflow match regex should be parsed")
 	assert.Equal(t, "non regex", config.DataSet.Workflows["test-workflow"].Match.(map[string]interface{})["key2"], "workflow match non-regex should remain")
 	assert.IsType(t, &regexp.Regexp{}, config.DataSet.Workflows["test-workflow"].UnlessMatch.(map[string]interface{})["key3"], "workflow unless_match regex should be parsed")
 	assert.Equal(t, "non regex", config.DataSet.Workflows["test-workflow"].UnlessMatch.(map[string]interface{})["key4"], "workflow unless_match non-regex should remain")
-	assert.Equal(t, ":regex:test3", config.DataSet.Rules[0].When.Match["key5"], "rule match regex should remain for later driver parsing")
 	assert.Equal(t, "non regex", config.DataSet.Rules[0].When.Match["key6"], "rule match non-regex should remain")
 }
diff --git a/internal/config/function_export_test.go b/internal/config/function_export_test.go
index 5d69bcb..2208cf9 100644
--- a/internal/config/function_export_test.go
+++ b/internal/config/function_export_test.go
@@ -239,8 +239,8 @@ func TestFunctionExportWithSquashedSysData(t *testing.T) {
 }
 
 func TestFunctionExportWithSubsystem(t *testing.T) {
-	cfg := Config{
-		DataSet: &DataSet{
+	cfg := &Config{
+		Staged: &DataSet{
 			Systems: map[string]System{
 				"outter": {
 					Extends: []string{
@@ -271,22 +271,23 @@ func TestFunctionExportWithSubsystem(t *testing.T) {
 		},
 	}
 
-	(&cfg).extendAllSystems()
+	cfg.extendAllSystems()
+	cfg.DataSet = cfg.Staged
 
 	result := ExportFunctionContext(
 		&Function{Target: Action{System: "outter", Function: "inner.testFunc1"}},
 		map[string]interface{}{
 			"ctx": map[string]interface{}{"path": "test/path1"},
 		},
-		&cfg,
+		cfg,
 	)
 
 	assert.Equal(t, "http://value1.value3/test/path1", result["url"])
 }
 
 func TestFunctionExportWithSubsystemParameters(t *testing.T) {
-	cfg := Config{
-		DataSet: &DataSet{
+	cfg := &Config{
+		Staged: &DataSet{
 			Systems: map[string]System{
 				"outter": {
 					Extends: []string{
@@ -321,22 +322,23 @@ func TestFunctionExportWithSubsystemParameters(t *testing.T) {
 		},
 	}
 
-	(&cfg).extendAllSystems()
+	cfg.extendAllSystems()
+	cfg.DataSet = cfg.Staged
 
 	result := ExportFunctionContext(
 		&Function{Target: Action{System: "outter", Function: "inner.testFunc1"}},
 		map[string]interface{}{
 			"ctx": map[string]interface{}{"path": "test/path1"},
 		},
-		&cfg,
+		cfg,
 	)
 
 	assert.Equal(t, "http://value1.value3/test/path1?internal-data", result["url"])
 }
 
 func TestFunctionExportWithSubsystemParentData(t *testing.T) {
-	cfg := Config{
-		DataSet: &DataSet{
+	cfg := &Config{
+		Staged: &DataSet{
 			Systems: map[string]System{
 				"outter": {
 					Extends: []string{
@@ -369,14 +371,15 @@ func TestFunctionExportWithSubsystemParentData(t *testing.T) {
 		},
 	}
 
-	(&cfg).extendAllSystems()
+	cfg.extendAllSystems()
+	cfg.DataSet = cfg.Staged
 
 	result := ExportFunctionContext(
 		&Function{Target: Action{System: "outter", Function: "inner.testFunc1"}},
 		map[string]interface{}{
 			"ctx": map[string]interface{}{"path": "test/path1"},
 		},
-		&cfg,
+		cfg,
 	)
 
 	assert.Equal(t, "http://value1.value3/test/path1?belong-to-parent", result["url"])
diff --git a/internal/driver/builtin_test.go b/internal/driver/builtin_test.go
index 5edb979..8b8f8e7 100644
--- a/internal/driver/builtin_test.go
+++ b/internal/driver/builtin_test.go
@@ -32,15 +32,15 @@ func TestBuiltinAcquire(t *testing.T) {
 	testCases := map[string]interface{}{
 		"panic when shortName is missing": []interface{}{
 			&BuiltinDriver{meta: &Meta{HandlerData: map[string]interface{}{}}},
-			"shortName is missing ", // error prefix
+			"driver error: shortName is missing ", // error prefix
 		},
 		"panic when using relative path with / in shortName": []interface{}{
 			&BuiltinDriver{meta: &Meta{HandlerData: map[string]interface{}{"shortName": "../fakecmd"}}},
-			"shortName has path separator ", // error prefix
+			"driver error: shortName has path separator ", // error prefix
 		},
 		"panic when using absolute path with / in shortName": []interface{}{
 			&BuiltinDriver{meta: &Meta{HandlerData: map[string]interface{}{"shortName": "/usr/bin/fakecmd"}}},
-			"shortName has path separator ", // error prefix
+			"driver error: shortName has path separator ", // error prefix
 		},
 		"has full path in Executable": []interface{}{
 			&BuiltinDriver{meta: &Meta{HandlerData: map[string]interface{}{"shortName": "testDriver"}}},
@@ -88,7 +88,7 @@ func TestBuiltinPrepare(t *testing.T) {
 		},
 		"panic when arguments is not a list": []interface{}{
 			&BuiltinDriver{meta: &Meta{HandlerData: map[string]interface{}{"arguments": false}}},
-			"arguments in driver ", // error prefix
+			"driver error: arguments in driver ", // error prefix
 		},
 	}
 
diff --git a/internal/driver/driver_test.go b/internal/driver/driver_test.go
index c820e62..b28ff1f 100644
--- a/internal/driver/driver_test.go
+++ b/internal/driver/driver_test.go
@@ -20,20 +20,20 @@ func TestDriverNewDriver(t *testing.T) {
 	// test new driver: table driven
 	testCases := map[string]interface{}{
 		"panic when driver name is missing": []interface{}{
-			map[string]interface{}{}, // driver meta
-			"driver name missing: ",  // error prefix
+			map[string]interface{}{},              // driver meta
+			"driver error: driver name missing: ", // error prefix
 		},
 		"panic when driver type is missing": []interface{}{
-			map[string]interface{}{"name": "test"}, // driver meta
-			"unsupported driver type: ",            // error prefix
+			map[string]interface{}{"name": "test"},    // driver meta
+			"driver error: unsupported driver type: ", // error prefix
 		},
 		"panic when driver type is unknown": []interface{}{
 			map[string]interface{}{"name": "test", "type": "faketype"}, // driver meta
-			"unsupported driver type: ",                                // error prefix
+			"driver error: unsupported driver type: ",                  // error prefix
 		},
 		"panic when builtin driver data missing": []interface{}{
 			map[string]interface{}{"name": "test", "type": "builtin"}, // driver meta
-			"shortName is missing for builtin driver: ",               // error prefix
+			"driver error: shortName is missing for builtin driver: ", // error prefix
 		},
 	}
 
diff --git a/internal/service/service_test.go b/internal/service/service_test.go
index 569af0a..43be4b5 100644
--- a/internal/service/service_test.go
+++ b/internal/service/service_test.go
@@ -202,7 +202,7 @@ func TestServiceRemoveEmitter(t *testing.T) {
 		defer daemon.Children.Done()
 
 		assert.NotPanics(t, func() {
-			for i := 0; i < 50; i = i + 1 {
+			for i := 0; i < 50; i++ {
 				select {
 				case svc.driverRuntimes["driver:d1"].Stream <- dipper.Message{
 					Channel: "test",
@@ -218,7 +218,8 @@ func TestServiceRemoveEmitter(t *testing.T) {
 	}()
 
 	newCfg := &config.Config{
-		DataSet: &config.DataSet{
+		Services: []string{"testsvc"},
+		Staged: &config.DataSet{
 			Drivers: map[string]interface{}{
 				"daemon": map[string]interface{}{
 					"features": map[string]interface{}{
@@ -239,6 +240,7 @@ func TestServiceRemoveEmitter(t *testing.T) {
 		},
 	}
 	svc.config = newCfg
+	svc.config.ResetStage()
 
 	time.Sleep(100 * time.Millisecond)
 	assert.NotPanics(t, svc.Reload, "service reload should not panic when emitter is removed")
@@ -299,7 +301,7 @@ func TestServiceEmitterCrashing(t *testing.T) {
 		defer daemon.Children.Done()
 
 		assert.NotPanics(t, func() {
-			for i := 0; i < 50; i = i + 1 {
+			for i := 0; i < 50; i++ {
 				select {
 				case svc.driverRuntimes["driver:d1"].Stream <- dipper.Message{
 					Channel: "test",
@@ -376,7 +378,7 @@ func TestServiceReplaceEmitter(t *testing.T) {
 		defer daemon.Children.Done()
 
 		assert.NotPanics(t, func() {
-			for i := 0; i < 50; i = i + 1 {
+			for i := 0; i < 50; i++ {
 				select {
 				case svc.driverRuntimes["driver:d1"].Stream <- dipper.Message{
 					Channel: "test",
@@ -392,7 +394,8 @@ func TestServiceReplaceEmitter(t *testing.T) {
 	}()
 
 	newCfg := &config.Config{
-		DataSet: &config.DataSet{
+		Services: []string{"testsvc"},
+		Staged: &config.DataSet{
 			Drivers: map[string]interface{}{
 				"daemon": map[string]interface{}{
 					"featureMap": map[string]interface{}{
@@ -421,6 +424,7 @@ func TestServiceReplaceEmitter(t *testing.T) {
 		},
 	}
 	svc.config = newCfg
+	svc.config.ResetStage()
 
 	time.Sleep(100 * time.Millisecond)
 	assert.NotPanics(t, svc.Reload, "service reload should not panic when emitter is changed")