Repository: argoproj/argo (go)

Issue:
Panic when volumeMount does not exist
Checklist:

 * [x] I've included the version.
 * [ ] I've included reproduction steps.
 * [ ] I've included the workflow YAML.
 * [x] I've included the logs.
 
**What happened**:

~~When submitting a workflow that depends on a `WorkflowTemplate`, the workflow crashes and immediately fails (without jumping to onExit).~~

Turns out that the crash actually occurred due to a `volumeMount` not existing.

The message in `argo watch` is "Message:             runtime error: invalid memory address or nil pointer dereference"

**What you expected to happen**:

**How to reproduce it (as minimally and precisely as possible)**:

**Anything else we need to know?**:

The same workflow used to work in Argo v2.8.2

**Environment**:

- Argo version:
```
$ argo version

argo: v2.9.2
  BuildDate: 2020-07-08T23:55:01Z
  GitCommit: 65c2bd44e45c11e0a0b03adeef8d6800b72cd551
  GitTreeState: clean
  GitTag: v2.9.2
  GoVersion: go1.13.4
  Compiler: gc
  Platform: darwin/amd64
```
- Kubernetes version :
```
$ kubectl version -o yaml

clientVersion:
  buildDate: "2020-05-21T14:51:23Z"
  compiler: gc
  gitCommit: 2e7996e3e2712684bc73f0dec0200d64eec7fe40
  gitTreeState: clean
  gitVersion: v1.18.3
  goVersion: go1.14.3
  major: "1"
  minor: "18"
  platform: darwin/amd64
serverVersion:
  buildDate: "2020-03-12T20:55:23Z"
  compiler: gc
  gitCommit: 8d8aa39598534325ad77120c120a22b3a990b5ea
  gitTreeState: clean
  gitVersion: v1.17.4
  goVersion: go1.13.8
  major: "1"
  minor: "17"
  platform: linux/amd64
```

**Other debugging information (if applicable)**:

- workflow-controller logs:

```
kubectl logs -n argo $(kubectl get pods -l app=workflow-controller -n argo -o name)

2020-07-09T17:27:32.816871832Z time="2020-07-09T17:27:32Z" level=info msg="Processing workflow" namespace=default workflow=test-workflow-xqv44
2020-07-09T17:27:32.816928736Z time="2020-07-09T17:27:32Z" level=info msg="Updated phase  -> Running" namespace=default workflow=test-workflow-xqv44
2020-07-09T17:27:32.825983852Z time="2020-07-09T17:27:32Z" level=info msg="Steps node test-workflow-xqv44 initialized Running" namespace=default workflow=test-workflow-xqv44
2020-07-09T17:27:32.826032384Z time="2020-07-09T17:27:32Z" level=info msg="StepGroup node test-workflow-xqv44-3078889322 initialized Running" namespace=default workflow=test-workflow-xqv44
2020-07-09T17:27:32.827444929Z time="2020-07-09T17:27:32Z" level=info msg="Steps node test-workflow-xqv44-4071128617 initialized Running" namespace=default workflow=test-workflow-xqv44
2020-07-09T17:27:32.827485903Z time="2020-07-09T17:27:32Z" level=info msg="StepGroup node test-workflow-xqv44-985411005 initialized Running" namespace=default workflow=test-workflow-xqv44
2020-07-09T17:27:32.828114123Z time="2020-07-09T17:27:32Z" level=info msg="Pod node test-workflow-xqv44-137232839 initialized Pending" namespace=default workflow=test-workflow-xqv44
2020-07-09T17:27:32.828794253Z time="2020-07-09T17:27:32Z" level=error msg="Recovered from panic" namespace=default r="runtime error: invalid memory address or nil pointer dereference" stack="goroutine 532 [running]:
runtime/debug.Stack(0xc08d9b674e, 0x173d160, 0x28710d0)
    /usr/local/go/src/runtime/debug/stack.go:24
+0x9d
github.com/argoproj/argo/workflow/controller.(*wfOperationCtx).operate.func2(0xc001ea0580)
    /go/src/github.com/argoproj/argo/workflow/controller/operator.go:161
+0xc4
panic(0x173d160, 0x28710d0)
    /usr/local/go/src/runtime/panic.go:679 +0x1b2
github.com/argoproj/argo/workflow/controller.(*wfOperationCtx).executeTemplate(0xc001ea0580, 0xc00cfd0f80, 0x77, 0x1bb9bc0, 0xc0006e4420, 0xc00ab578c0, 0xc0064cac60, 0x5, 0x6, 0x0, ...)
    /go/src/github.com/argoproj/argo/workflow/controller/operator.go:1510
+0x1866
github.com/argoproj/argo/workflow/controller.(*wfOperationCtx).executeStepGroup(0xc001ea0580, 0xc0064caf20, 0x2, 0x4, 0xc002908380, 0x68, 0xc0012563b8, 0x1bb9b00)
    /go/src/github.com/argoproj/argo/workflow/controller/steps.go:233 +0x11e1
github.com/argoproj/argo/workflow/controller.(*wfOperationCtx).executeSteps(0xc001ea0580, 0xc0029081c0, 0x65, 0xc00ab578c0, 0xc009296cf0, 0x23, 0xc00a583440, 0x1bb9bc0, 0xc00069d970, 0xc0012585b8, ...)
    /go/src/github.com/argoproj/argo/workflow/controller/steps.go:91
+0x46e
github.com/argoproj/argo/workflow/controller.(*wfOperationCtx).executeTemplate(0xc001ea0580, 0xc0029081c0, 0x65, 0x1bb9bc0, 0xc00069d970, 0xc00ab56c40, 0xc000988580, 0x6, 0x6, 0x0, ...)
    /go/src/github.com/argoproj/argo/workflow/controller/operator.go:1496
+0x19a4
github.com/argoproj/argo/workflow/controller.(*wfOperationCtx).executeStepGroup(0xc001ea0580, 0xc00069d8c0, 0x1, 0x4, 0xc009cc1880, 0x20, 0xc001259950, 0x1bb9b00)
    /go/src/github.com/argoproj/argo/workflow/controller/steps.go:233
+0x11e1
github.com/argoproj/argo/workflow/controller.(*wfOperationCtx).executeSteps(0xc001ea0580, 0xc00e5583a0, 0x1d, 0xc00ab56c40, 0xc009296870, 0x23, 0xc00a582240, 0x1bb9bc0, 0xc001ea0a50, 0xc0006bfef0, ...)
    /go/src/github.com/argoproj/argo/workflow/controller/steps.go:91
+0x46e
github.com/argoproj/argo/workflow/controller.(*wfOperationCtx).executeTemplate(0xc001ea0580, 0xc00e5583a0, 0x1d, 0x1bb9bc0, 0xc001ea0a50, 0xc00ab56ac0, 0xc001ea0790, 0x3, 0x3, 0x0, ...)
    /go/src/github.com/argoproj/argo/workflow/controller/operator.go:1496
+0x19a4
github.com/argoproj/argo/workflow/controller.(*wfOperationCtx).operate(0xc001ea0580)
    /go/src/github.com/argoproj/argo/workflow/controller/operator.go:287 +0x125d
github.com/argoproj/argo/workflow/controller.(*WorkflowController).processNextItem(0xc0004d1180, 0x0)
    /go/src/github.com/argoproj/argo/workflow/controller/controller.go:433 +0x8c2
github.com/argoproj/argo/workflow/controller.(*WorkflowController).runWorker(0xc0004d1180)
    /go/src/github.com/argoproj/argo/workflow/controller/controller.go:359 +0x2b
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc00108c1f0)
    /go/pkg/mod/k8s.io/apimachinery@v0.16.7-beta.0/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc00108c1f0, 0x3b9aca00, 0x0, 0x1, 0xc0006aa9c0)
    /go/pkg/mod/k8s.io/apimachinery@v0.16.7-beta.0/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(0xc00108c1f0, 0x3b9aca00, 0xc0006aa9c0)
    /go/pkg/mod/k8s.io/apimachinery@v0.16.7-beta.0/pkg/util/wait/wait.go:88
+0x4d
created by github.com/argoproj/argo/workflow/controller.(*WorkflowController).Run
    /go/src/github.com/argoproj/argo/workflow/controller/controller.go:172 +0x975
" workflow=test-workflow-xqv44

2020-07-09T17:27:32.828856307Z time="2020-07-09T17:27:32Z" level=info msg="Updated phase Running -> Error" namespace=default workflow=test-workflow-xqv44
2020-07-09T17:27:32.828865379Z time="2020-07-09T17:27:32Z" level=info msg="Updated message  -> runtime error: invalid memory address or nil pointer dereference" namespace=default workflow=test-workflow-xqv44
2020-07-09T17:27:32.828873448Z time="2020-07-09T17:27:32Z" level=info msg="Marking workflow completed" namespace=default workflow=test-workflow-xqv44
2020-07-09T17:27:32.828881787Z time="2020-07-09T17:27:32Z" level=info msg="Checking daemoned children of " namespace=default workflow=test-workflow-xqv44
2020-07-09T17:27:32.839810882Z time="2020-07-09T17:27:32Z" level=info msg="Workflow update successful" namespace=default phase=Error resourceVersion=51728589 workflow=test-workflow-xqv44
```

---
<!-- Issue Author: Don't delete this message to encourage other users to support your issue! -->
**Message from the maintainers**:

If you are impacted by this bug please add a 👍 reaction to this issue! We often sort issues this way to know what to prioritize.


Pull request:
fix(controller): Prevent panic on nil node. Fixes #3436
Checklist:

* [x] Either (a) I've created an [enhancement proposal](https://github.com/argoproj/argo/issues/new/choose) and discussed it with the community, (b) this is a bug fix, or (c) this is a chore.
* [x] The title of the PR is (a) [conventional](https://www.conventionalcommits.org/en/v1.0.0/), (b) states what changed, and (c) suffixes the related issues number. E.g. `"fix(controller): Updates such and such. Fixes #1234"`.  
* [x] I've signed the CLA.
* [x] I have written unit and/or e2e tests for my change. PRs without these are unlikely to be merged.
* [x] My builds are green. Try syncing with master if they are not. 
* [x] My organization is added to [USERS.md](https://github.com/argoproj/argo/blob/master/USERS.md).

Fixes #3436

Patch:
diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml
index 478ccec15..aa8cc1ae2 100644
--- a/.github/workflows/docker-image.yml
+++ b/.github/workflows/docker-image.yml
@@ -24,9 +24,9 @@ jobs:
           password: ${{ secrets.DOCKERIO_PASSWORD }}
       - name: Build & Push Linux Docker Images
         env:
-          DOCKERIO_USERNAME: ${{ secrets.DOCKERIO_USERNAME }}
+          DOCKERIO_ORG: ${{ secrets.DOCKERIO_ORG }}
         run: |
-          docker_org=$DOCKERIO_USERNAME
+          docker_org=$DOCKERIO_ORG
 
           tag=$(basename $GITHUB_REF)
           if [ $tag = "master" ]; then
@@ -52,9 +52,9 @@ jobs:
           password: ${{ secrets.DOCKERIO_PASSWORD }}
       - name: Build & Push Windows Docker Images
         env:
-          DOCKERIO_USERNAME: ${{ secrets.DOCKERIO_USERNAME }}
+          DOCKERIO_ORG: ${{ secrets.DOCKERIO_ORG }}
         run: |
-          docker_org=$DOCKERIO_USERNAME
+          docker_org=$DOCKERIO_ORG
 
           tag=$(basename $GITHUB_REF)
           if [ $tag = "master" ]; then
@@ -81,11 +81,11 @@ jobs:
           password: ${{ secrets.DOCKERIO_PASSWORD }}
       - name: Push Multiarch Image
         env:
-          DOCKERIO_USERNAME: ${{ secrets.DOCKERIO_USERNAME }}
+          DOCKERIO_ORG: ${{ secrets.DOCKERIO_ORG }}
         run: |
           echo $(jq -c '. + { "experimental": "enabled" }' ${DOCKER_CONFIG}/config.json) > ${DOCKER_CONFIG}/config.json
 
-          docker_org=$DOCKERIO_USERNAME
+          docker_org=$DOCKERIO_ORG
 
           tag=$(basename $GITHUB_REF)
           if [ $tag = "master" ]; then
diff --git a/USERS.md b/USERS.md
index 4d764e333..a79af3f9b 100644
--- a/USERS.md
+++ b/USERS.md
@@ -70,6 +70,7 @@ Currently, the following organizations are **officially** using Argo Workflows:
 1. [SAP Hybris](https://cx.sap.com/)
 1. [SegmentStream](https://segmentstream.com)
 1. [Sidecar Technologies](https://hello.getsidecar.com/)
+1. [Sohu](https://www.sohu.com/)
 1. [Styra](https://www.styra.com/)
 1. [Threekit](https://www.threekit.com/)
 1. [Tiger Analytics](https://www.tigeranalytics.com/)
diff --git a/cmd/workflow-controller/main.go b/cmd/workflow-controller/main.go
index 616fdd589..f0eea34dc 100644
--- a/cmd/workflow-controller/main.go
+++ b/cmd/workflow-controller/main.go
@@ -6,11 +6,10 @@ import (
 	"os"
 	"time"
 
-	log "github.com/sirupsen/logrus"
-
 	"github.com/argoproj/pkg/cli"
 	kubecli "github.com/argoproj/pkg/kube/cli"
 	"github.com/argoproj/pkg/stats"
+	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"k8s.io/client-go/kubernetes"
 	_ "k8s.io/client-go/plugin/pkg/client/auth/azure"
diff --git a/pkg/apis/workflow/v1alpha1/workflow_types.go b/pkg/apis/workflow/v1alpha1/workflow_types.go
index f2d2a2b48..6309f06ed 100644
--- a/pkg/apis/workflow/v1alpha1/workflow_types.go
+++ b/pkg/apis/workflow/v1alpha1/workflow_types.go
@@ -1812,6 +1812,15 @@ func (wf *Workflow) GetTemplateByName(name string) *Template {
 	return nil
 }
 
+func (wf *Workflow) GetNodeByName(nodeName string) *NodeStatus {
+	nodeID := wf.NodeID(nodeName)
+	node, ok := wf.Status.Nodes[nodeID]
+	if !ok {
+		return nil
+	}
+	return &node
+}
+
 // GetResourceScope returns the template scope of workflow.
 func (wf *Workflow) GetResourceScope() ResourceScope {
 	return ResourceScopeLocal
diff --git a/workflow/common/util.go b/workflow/common/util.go
index 79a87e804..6998dc3ca 100644
--- a/workflow/common/util.go
+++ b/workflow/common/util.go
@@ -694,3 +694,7 @@ func SplitClusterWorkflowTemplateYAMLFile(body []byte, strict bool) ([]wfv1.Clus
 	}
 	return manifests, nil
 }
+
+func GenerateOnExitNodeName(parentDisplayName string) string {
+	return fmt.Sprintf("%s.onExit", parentDisplayName)
+}
diff --git a/workflow/controller/dag.go b/workflow/controller/dag.go
index bfb9a0129..7436fc7df 100644
--- a/workflow/controller/dag.go
+++ b/workflow/controller/dag.go
@@ -193,7 +193,7 @@ func (d *dagContext) assessDAGPhase(targetTasks []string, nodes wfv1.Nodes) wfv1
 }
 
 func (woc *wfOperationCtx) executeDAG(nodeName string, tmplCtx *templateresolution.Context, templateScope string, tmpl *wfv1.Template, orgTmpl wfv1.TemplateReferenceHolder, opts *executeTemplateOpts) (*wfv1.NodeStatus, error) {
-	node := woc.getNodeByName(nodeName)
+	node := woc.wf.GetNodeByName(nodeName)
 	if node == nil {
 		node = woc.initializeExecutableNode(nodeName, wfv1.NodeTypeDAG, templateScope, tmpl, orgTmpl, opts.boundaryID, wfv1.NodeRunning)
 	}
@@ -261,7 +261,7 @@ func (woc *wfOperationCtx) executeDAG(nodeName string, tmplCtx *templateresoluti
 		return node, err
 	}
 	if outputs != nil {
-		node = woc.getNodeByName(nodeName)
+		node = woc.wf.GetNodeByName(nodeName)
 		node.Outputs = outputs
 		woc.wf.Status.Nodes[node.ID] = *node
 	}
@@ -283,7 +283,7 @@ func (woc *wfOperationCtx) updateOutboundNodesForTargetTasks(dagCtx *dagContext,
 		outboundNodeIDs := woc.getOutboundNodes(depNode.ID)
 		outbound = append(outbound, outboundNodeIDs...)
 	}
-	node := woc.getNodeByName(nodeName)
+	node := woc.wf.GetNodeByName(nodeName)
 	node.OutboundNodes = outbound
 	woc.wf.Status.Nodes[node.ID] = *node
 	woc.log.Infof("Outbound nodes of %s set to %s", node.ID, outbound)
@@ -317,7 +317,7 @@ func (woc *wfOperationCtx) executeDAGTask(dagCtx *dagContext, taskName string) {
 	nodeName := dagCtx.taskNodeName(taskName)
 	taskDependencies := dagCtx.GetTaskDependencies(taskName)
 
-	taskGroupNode := woc.getNodeByName(nodeName)
+	taskGroupNode := woc.wf.GetNodeByName(nodeName)
 	if taskGroupNode != nil && taskGroupNode.Type != wfv1.NodeTypeTaskGroup {
 		taskGroupNode = nil
 	}
@@ -345,16 +345,16 @@ func (woc *wfOperationCtx) executeDAGTask(dagCtx *dagContext, taskName string) {
 	}
 
 	if dagCtx.GetTaskDependsLogic(taskName) != "" {
+		// Recurse into all of this node's dependencies
+		for _, dep := range taskDependencies {
+			woc.executeDAGTask(dagCtx, dep)
+		}
 		execute, proceed, err := dagCtx.evaluateDependsLogic(taskName)
 		if err != nil {
 			woc.initializeNode(nodeName, wfv1.NodeTypeSkipped, dagTemplateScope, task, dagCtx.boundaryID, wfv1.NodeError, err.Error())
 			connectDependencies(nodeName)
 			return
 		}
-		// Recurse into all of this node's dependencies
-		for _, dep := range taskDependencies {
-			woc.executeDAGTask(dagCtx, dep)
-		}
 		if !proceed {
 			// This node's dependencies are not completed yet, return
 			return
@@ -624,6 +624,13 @@ func (d *dagContext) evaluateDependsLogic(taskName string) (bool, bool, error) {
 			return false, false, nil
 		}
 
+		// If a task happens to have an onExit node, don't proceed until the onExit node is fulfilled
+		if onExitNode := d.wf.GetNodeByName(common.GenerateOnExitNodeName(taskName)); onExitNode != nil {
+			if !onExitNode.Fulfilled() {
+				return false, false, nil
+			}
+		}
+
 		evalTaskName := strings.Replace(taskName, "-", "_", -1)
 		if _, ok := evalScope[evalTaskName]; ok {
 			continue
diff --git a/workflow/controller/operator.go b/workflow/controller/operator.go
index dcf98a622..5e0fee95f 100644
--- a/workflow/controller/operator.go
+++ b/workflow/controller/operator.go
@@ -338,7 +338,7 @@ func (woc *wfOperationCtx) operate() {
 		woc.globalParams[common.GlobalVarWorkflowFailures] = strconv.Quote(string(failedNodeBytes))
 
 		woc.log.Infof("Running OnExit handler: %s", woc.wfSpec.OnExit)
-		onExitNodeName := woc.wf.ObjectMeta.Name + ".onExit"
+		onExitNodeName := common.GenerateOnExitNodeName(woc.wf.ObjectMeta.Name)
 		onExitNode, err = woc.executeTemplate(onExitNodeName, &wfv1.WorkflowStep{Template: woc.wfSpec.OnExit}, tmplCtx, woc.wfSpec.Arguments, &executeTemplateOpts{onExitTemplate: true})
 		if err != nil {
 			// the error are handled in the callee so just log it.
@@ -451,15 +451,6 @@ func (woc *wfOperationCtx) setGlobalParameters(executionParameters wfv1.Argument
 	}
 }
 
-func (woc *wfOperationCtx) getNodeByName(nodeName string) *wfv1.NodeStatus {
-	nodeID := woc.wf.NodeID(nodeName)
-	node, ok := woc.wf.Status.Nodes[nodeID]
-	if !ok {
-		return nil
-	}
-	return &node
-}
-
 // persistUpdates will update a workflow with any updates made during workflow operation.
 // It also labels any pods as completed if we have extracted everything we need from it.
 // NOTE: a previous implementation used Patch instead of Update, but Patch does not work with
@@ -1379,7 +1370,7 @@ type executeTemplateOpts struct {
 func (woc *wfOperationCtx) executeTemplate(nodeName string, orgTmpl wfv1.TemplateReferenceHolder, tmplCtx *templateresolution.Context, args wfv1.Arguments, opts *executeTemplateOpts) (*wfv1.NodeStatus, error) {
 	woc.log.Debugf("Evaluating node %s: template: %s, boundaryID: %s", nodeName, common.GetTemplateHolderString(orgTmpl), opts.boundaryID)
 
-	node := woc.getNodeByName(nodeName)
+	node := woc.wf.GetNodeByName(nodeName)
 
 	// Set templateScope from which the template resolution starts.
 	templateScope := tmplCtx.GetTemplateScope()
@@ -1508,7 +1499,7 @@ func (woc *wfOperationCtx) executeTemplate(nodeName string, orgTmpl wfv1.Templat
 		return woc.initializeNode(nodeName, wfv1.NodeTypeSkipped, templateScope, orgTmpl, opts.boundaryID, wfv1.NodeError, err.Error()), err
 	}
 	if err != nil {
-		node = woc.markNodeError(node.Name, err)
+		node = woc.markNodeError(nodeName, err)
 		// If retry policy is not set, or if it is not set to Always or OnError, we won't attempt to retry an errored container
 		// and we return instead.
 		if processedTmpl.RetryStrategy == nil ||
@@ -1541,11 +1532,11 @@ func (woc *wfOperationCtx) executeTemplate(nodeName string, orgTmpl wfv1.Templat
 		}
 	}
 
-	node = woc.getNodeByName(node.Name)
+	node = woc.wf.GetNodeByName(node.Name)
 
 	// Swap the node back to retry node
 	if retryNodeName != "" {
-		retryNode := woc.getNodeByName(retryNodeName)
+		retryNode := woc.wf.GetNodeByName(retryNodeName)
 		if !retryNode.Fulfilled() && node.Fulfilled() { //if the retry child has completed we need to update outself
 			node, err = woc.executeTemplate(retryNodeName, orgTmpl, tmplCtx, args, opts)
 			if err != nil {
@@ -1723,7 +1714,7 @@ func (woc *wfOperationCtx) initializeNode(nodeName string, nodeType wfv1.NodeTyp
 
 // markNodePhase marks a node with the given phase, creating the node if necessary and handles timestamps
 func (woc *wfOperationCtx) markNodePhase(nodeName string, phase wfv1.NodePhase, message ...string) *wfv1.NodeStatus {
-	node := woc.getNodeByName(nodeName)
+	node := woc.wf.GetNodeByName(nodeName)
 	if node == nil {
 		panic(fmt.Sprintf("workflow '%s' node '%s' uninitialized when marking as %v: %s", woc.wf.Name, nodeName, phase, message))
 	}
@@ -1784,7 +1775,7 @@ func (woc *wfOperationCtx) markNodeError(nodeName string, err error) *wfv1.NodeS
 // markNodePending is a convenience method to mark a node and set the message from the error
 func (woc *wfOperationCtx) markNodePending(nodeName string, err error) *wfv1.NodeStatus {
 	woc.log.Infof("Mark node %s as Pending, due to: %+v", nodeName, err)
-	node := woc.getNodeByName(nodeName)
+	node := woc.wf.GetNodeByName(nodeName)
 	return woc.markNodePhase(nodeName, wfv1.NodePending, fmt.Sprintf("Pending %s", time.Since(node.StartedAt.Time)))
 }
 
@@ -1840,7 +1831,7 @@ func (woc *wfOperationCtx) checkParallelism(tmpl *wfv1.Template, node *wfv1.Node
 }
 
 func (woc *wfOperationCtx) executeContainer(nodeName string, templateScope string, tmpl *wfv1.Template, orgTmpl wfv1.TemplateReferenceHolder, opts *executeTemplateOpts) (*wfv1.NodeStatus, error) {
-	node := woc.getNodeByName(nodeName)
+	node := woc.wf.GetNodeByName(nodeName)
 	if node == nil {
 		node = woc.initializeExecutableNode(nodeName, wfv1.NodeTypePod, templateScope, tmpl, orgTmpl, opts.boundaryID, wfv1.NodePending)
 	} else if !isResubmitAllowed(tmpl) || !node.Pending() {
@@ -1982,7 +1973,7 @@ func getStepOrDAGTaskName(nodeName string) string {
 }
 
 func (woc *wfOperationCtx) executeScript(nodeName string, templateScope string, tmpl *wfv1.Template, orgTmpl wfv1.TemplateReferenceHolder, opts *executeTemplateOpts) (*wfv1.NodeStatus, error) {
-	node := woc.getNodeByName(nodeName)
+	node := woc.wf.GetNodeByName(nodeName)
 	if node != nil {
 		return node, nil
 	}
@@ -2247,7 +2238,7 @@ func (woc *wfOperationCtx) addChildNode(parent string, child string) {
 
 // executeResource is runs a kubectl command against a manifest
 func (woc *wfOperationCtx) executeResource(nodeName string, templateScope string, tmpl *wfv1.Template, orgTmpl wfv1.TemplateReferenceHolder, opts *executeTemplateOpts) (*wfv1.NodeStatus, error) {
-	node := woc.getNodeByName(nodeName)
+	node := woc.wf.GetNodeByName(nodeName)
 	if node != nil {
 		return node, nil
 	}
@@ -2283,7 +2274,7 @@ func (woc *wfOperationCtx) executeResource(nodeName string, templateScope string
 }
 
 func (woc *wfOperationCtx) executeSuspend(nodeName string, templateScope string, tmpl *wfv1.Template, orgTmpl wfv1.TemplateReferenceHolder, opts *executeTemplateOpts) (*wfv1.NodeStatus, error) {
-	node := woc.getNodeByName(nodeName)
+	node := woc.wf.GetNodeByName(nodeName)
 	if node == nil {
 		node = woc.initializeExecutableNode(nodeName, wfv1.NodeTypeSuspend, templateScope, tmpl, orgTmpl, opts.boundaryID, wfv1.NodePending)
 	}
@@ -2294,7 +2285,7 @@ func (woc *wfOperationCtx) executeSuspend(nodeName string, templateScope string,
 	var requeueTime *time.Time
 
 	if tmpl.Suspend.Duration != "" {
-		node := woc.getNodeByName(nodeName)
+		node := woc.wf.GetNodeByName(nodeName)
 		suspendDuration, err := parseStringToDuration(tmpl.Suspend.Duration)
 		if err != nil {
 			return node, err
@@ -2497,7 +2488,7 @@ func (woc *wfOperationCtx) createTemplateContext(scope wfv1.ResourceScope, resou
 func (woc *wfOperationCtx) runOnExitNode(templateRef, parentDisplayName, parentNodeName, boundaryID string, tmplCtx *templateresolution.Context) (bool, *wfv1.NodeStatus, error) {
 	if templateRef != "" && woc.wf.Spec.Shutdown.ShouldExecute(true) {
 		woc.log.Infof("Running OnExit handler: %s", templateRef)
-		onExitNodeName := parentDisplayName + ".onExit"
+		onExitNodeName := common.GenerateOnExitNodeName(parentDisplayName)
 		onExitNode, err := woc.executeTemplate(onExitNodeName, &wfv1.WorkflowStep{Template: templateRef}, tmplCtx, woc.wfSpec.Arguments, &executeTemplateOpts{
 			boundaryID:     boundaryID,
 			onExitTemplate: true,
diff --git a/workflow/controller/steps.go b/workflow/controller/steps.go
index 786cf97fa..06e057a56 100644
--- a/workflow/controller/steps.go
+++ b/workflow/controller/steps.go
@@ -32,7 +32,7 @@ type stepsContext struct {
 }
 
 func (woc *wfOperationCtx) executeSteps(nodeName string, tmplCtx *templateresolution.Context, templateScope string, tmpl *wfv1.Template, orgTmpl wfv1.TemplateReferenceHolder, opts *executeTemplateOpts) (*wfv1.NodeStatus, error) {
-	node := woc.getNodeByName(nodeName)
+	node := woc.wf.GetNodeByName(nodeName)
 	if node == nil {
 		node = woc.initializeExecutableNode(nodeName, wfv1.NodeTypeSteps, templateScope, tmpl, orgTmpl, opts.boundaryID, wfv1.NodeRunning)
 	}
@@ -59,7 +59,7 @@ func (woc *wfOperationCtx) executeSteps(nodeName string, tmplCtx *templateresolu
 
 	for i, stepGroup := range tmpl.Steps {
 		sgNodeName := fmt.Sprintf("%s[%d]", nodeName, i)
-		if woc.getNodeByName(sgNodeName) == nil {
+		if woc.wf.GetNodeByName(sgNodeName) == nil {
 			_ = woc.initializeNode(sgNodeName, wfv1.NodeTypeStepGroup, stepTemplateScope, tmpl, stepsCtx.boundaryID, wfv1.NodeRunning)
 		} else {
 			_ = woc.markNodePhase(sgNodeName, wfv1.NodeRunning)
@@ -72,7 +72,7 @@ func (woc *wfOperationCtx) executeSteps(nodeName string, tmplCtx *templateresolu
 			// Otherwise connect all the outbound nodes of the previous step group as parents to
 			// the current step group node.
 			prevStepGroupName := fmt.Sprintf("%s[%d]", nodeName, i-1)
-			prevStepGroupNode := woc.getNodeByName(prevStepGroupName)
+			prevStepGroupNode := woc.wf.GetNodeByName(prevStepGroupName)
 			if len(prevStepGroupNode.Children) == 0 {
 				// corner case which connects an empty StepGroup (e.g. due to empty withParams) to
 				// the previous StepGroup node
@@ -105,7 +105,7 @@ func (woc *wfOperationCtx) executeSteps(nodeName string, tmplCtx *templateresolu
 		// Add all outputs of each step in the group to the scope
 		for _, step := range stepGroup.Steps {
 			childNodeName := fmt.Sprintf("%s.%s", sgNodeName, step.Name)
-			childNode := woc.getNodeByName(childNodeName)
+			childNode := woc.wf.GetNodeByName(childNodeName)
 			prefix := fmt.Sprintf("steps.%s", step.Name)
 			if childNode == nil {
 				// This happens when there was `withItem/withParam` expansion.
@@ -146,7 +146,7 @@ func (woc *wfOperationCtx) executeSteps(nodeName string, tmplCtx *templateresolu
 		return node, err
 	}
 	if outputs != nil {
-		node := woc.getNodeByName(nodeName)
+		node := woc.wf.GetNodeByName(nodeName)
 		node.Outputs = outputs
 		woc.addOutputsToGlobalScope(node.Outputs)
 		woc.wf.Status.Nodes[node.ID] = *node
@@ -160,7 +160,7 @@ func (woc *wfOperationCtx) updateOutboundNodes(nodeName string, tmpl *wfv1.Templ
 	// Find the last, initialized stepgroup node
 	var lastSGNode *wfv1.NodeStatus
 	for i := len(tmpl.Steps) - 1; i >= 0; i-- {
-		sgNode := woc.getNodeByName(fmt.Sprintf("%s[%d]", nodeName, i))
+		sgNode := woc.wf.GetNodeByName(fmt.Sprintf("%s[%d]", nodeName, i))
 		if sgNode != nil {
 			lastSGNode = sgNode
 			break
@@ -175,7 +175,7 @@ func (woc *wfOperationCtx) updateOutboundNodes(nodeName string, tmpl *wfv1.Templ
 		woc.log.Infof("Outbound nodes of %s is %s", childID, outboundNodeIDs)
 		outbound = append(outbound, outboundNodeIDs...)
 	}
-	node := woc.getNodeByName(nodeName)
+	node := woc.wf.GetNodeByName(nodeName)
 	woc.log.Infof("Outbound nodes of %s is %s", node.ID, outbound)
 	node.OutboundNodes = outbound
 	woc.wf.Status.Nodes[node.ID] = *node
@@ -184,7 +184,7 @@ func (woc *wfOperationCtx) updateOutboundNodes(nodeName string, tmpl *wfv1.Templ
 // executeStepGroup examines a list of parallel steps and executes them in parallel.
 // Handles referencing of variables in scope, expands `withItem` clauses, and evaluates `when` expressions
 func (woc *wfOperationCtx) executeStepGroup(stepGroup []wfv1.WorkflowStep, sgNodeName string, stepsCtx *stepsContext) *wfv1.NodeStatus {
-	node := woc.getNodeByName(sgNodeName)
+	node := woc.wf.GetNodeByName(sgNodeName)
 	if node.Fulfilled() {
 		woc.log.Debugf("Step group node %v already marked completed", node)
 		return node
@@ -221,7 +221,7 @@ func (woc *wfOperationCtx) executeStepGroup(stepGroup []wfv1.WorkflowStep, sgNod
 			return woc.markNodeError(sgNodeName, err)
 		}
 		if !proceed {
-			if woc.getNodeByName(childNodeName) == nil {
+			if woc.wf.GetNodeByName(childNodeName) == nil {
 				skipReason := fmt.Sprintf("when '%s' evaluated false", step.When)
 				woc.log.Infof("Skipping %s: %s", childNodeName, skipReason)
 				woc.initializeNode(childNodeName, wfv1.NodeTypeSkipped, stepTemplateScope, &step, stepsCtx.boundaryID, wfv1.NodeSkipped, skipReason)
@@ -249,7 +249,7 @@ func (woc *wfOperationCtx) executeStepGroup(stepGroup []wfv1.WorkflowStep, sgNod
 		}
 	}
 
-	node = woc.getNodeByName(sgNodeName)
+	node = woc.wf.GetNodeByName(sgNodeName)
 	// Return if not all children completed
 	completed := true
 	for _, childNodeID := range node.Children {
@@ -412,7 +412,7 @@ func (woc *wfOperationCtx) expandStepGroup(sgNodeName string, stepGroup []wfv1.W
 		if len(expandedStep) == 0 {
 			// Empty list
 			childNodeName := fmt.Sprintf("%s.%s", sgNodeName, step.Name)
-			if woc.getNodeByName(childNodeName) == nil {
+			if woc.wf.GetNodeByName(childNodeName) == nil {
 				stepTemplateScope := stepsCtx.tmplCtx.GetTemplateScope()
 				skipReason := fmt.Sprint("Skipped, empty params")
 				woc.log.Infof("Skipping %s: %s", childNodeName, skipReason)
diff --git a/workflow/metrics/metrics.go b/workflow/metrics/metrics.go
index 39b3b4af0..e1050504d 100644
--- a/workflow/metrics/metrics.go
+++ b/workflow/metrics/metrics.go
@@ -6,6 +6,7 @@ import (
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
+	log "github.com/sirupsen/logrus"
 	"k8s.io/client-go/util/workqueue"
 
 	"github.com/argoproj/argo/pkg/apis/workflow/v1alpha1"
@@ -51,6 +52,16 @@ type Metrics struct {
 
 	// Used to quickly check if a metric desc is already used by the system
 	defaultMetricDescs map[string]bool
+	logMetric          *prometheus.CounterVec
+}
+
+func (m *Metrics) Levels() []log.Level {
+	return []log.Level{log.InfoLevel, log.WarnLevel, log.ErrorLevel}
+}
+
+func (m *Metrics) Fire(entry *log.Entry) error {
+	m.logMetric.WithLabelValues(entry.Level.String()).Inc()
+	return nil
 }
 
 var _ prometheus.Collector = &Metrics{}
@@ -67,12 +78,22 @@ func New(metricsConfig, telemetryConfig ServerConfig) *Metrics {
 		customMetrics:      make(map[string]metric),
 		workqueueMetrics:   make(map[string]prometheus.Metric),
 		defaultMetricDescs: make(map[string]bool),
+		logMetric: prometheus.NewCounterVec(prometheus.CounterOpts{
+			Name: "log_messages",
+			Help: "Total number of log messages.",
+		}, []string{"level"}),
 	}
 
 	for _, metric := range metrics.allMetrics() {
 		metrics.defaultMetricDescs[metric.Desc().String()] = true
 	}
 
+	for _, level := range metrics.Levels() {
+		metrics.logMetric.WithLabelValues(level.String())
+	}
+
+	log.AddHook(metrics)
+
 	return metrics
 }
 
@@ -96,7 +117,6 @@ func (m *Metrics) allMetrics() []prometheus.Metric {
 	for _, metric := range m.customMetrics {
 		allMetrics = append(allMetrics, metric.metric)
 	}
-
 	return allMetrics
 }
 
diff --git a/workflow/metrics/server.go b/workflow/metrics/server.go
index eb4285623..973332a57 100644
--- a/workflow/metrics/server.go
+++ b/workflow/metrics/server.go
@@ -7,7 +7,6 @@ import (
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
-
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	log "github.com/sirupsen/logrus"
 )
@@ -70,12 +69,14 @@ func (m *Metrics) Describe(ch chan<- *prometheus.Desc) {
 	for _, metric := range m.allMetrics() {
 		ch <- metric.Desc()
 	}
+	m.logMetric.Describe(ch)
 }
 
 func (m *Metrics) Collect(ch chan<- prometheus.Metric) {
 	for _, metric := range m.allMetrics() {
 		ch <- metric
 	}
+	m.logMetric.Collect(ch)
 }
 
 func (m *Metrics) garbageCollector(ctx context.Context) {


Test patch:
diff --git a/test/e2e/metrics_test.go b/test/e2e/metrics_test.go
index 26300b57e..e46f18679 100644
--- a/test/e2e/metrics_test.go
+++ b/test/e2e/metrics_test.go
@@ -39,7 +39,11 @@ func (s *MetricsSuite) TestMetricsEndpoint() {
 	s.Run("Metrics", func() {
 		s.e(s.T()).GET("").
 			Expect().
-			Status(200)
+			Status(200).
+			Body().
+			Contains(`log_messages{level="info"}`).
+			Contains(`log_messages{level="warning"}`).
+			Contains(`log_messages{level="error"}`)
 	})
 }
 
diff --git a/workflow/controller/dag_test.go b/workflow/controller/dag_test.go
index 9cb21e162..ab72311c7 100644
--- a/workflow/controller/dag_test.go
+++ b/workflow/controller/dag_test.go
@@ -110,6 +110,8 @@ func TestSingleDependency(t *testing.T) {
 		// Mark the status of the pod according to the test
 		if _, ok := statusMap[status]; ok {
 			makePodsPhase(t, statusMap[status], controller.kubeclientset, wf.ObjectMeta.Namespace)
+		} else {
+			makePodsPhase(t, v1.PodPending, controller.kubeclientset, wf.ObjectMeta.Namespace)
 		}
 
 		woc.operate()
@@ -121,9 +123,9 @@ func TestSingleDependency(t *testing.T) {
 			}
 		}
 		assert.True(t, found)
-	}
-	if closer != nil {
-		closer()
+		if closer != nil {
+			closer()
+		}
 	}
 }
 
@@ -1275,7 +1277,7 @@ func TestTerminateDAGWithMaxDurationLimitExpiredAndMoreAttempts(t *testing.T) {
 
 	woc.operate()
 
-	retryNode := woc.getNodeByName("dag-diamond-dj7q5.A")
+	retryNode := woc.wf.GetNodeByName("dag-diamond-dj7q5.A")
 	if assert.NotNil(t, retryNode) {
 		assert.Equal(t, wfv1.NodeFailed, retryNode.Phase)
 		assert.Contains(t, retryNode.Message, "Max duration limit exceeded")
@@ -1461,18 +1463,12 @@ func TestRetryStrategyNodes(t *testing.T) {
 	woc := newWorkflowOperationCtx(wf, controller)
 
 	woc.operate()
-	retryNode := woc.getNodeByName("wf-retry-pol")
-	if assert.NotNil(t, retryNode) {
-		assert.Equal(t, wfv1.NodeRunning, retryNode.Phase)
-	}
-
-	woc.operate()
-	retryNode = woc.getNodeByName("wf-retry-pol")
+	retryNode := woc.wf.GetNodeByName("wf-retry-pol")
 	if assert.NotNil(t, retryNode) {
 		assert.Equal(t, wfv1.NodeFailed, retryNode.Phase)
 	}
 
-	onExitNode := woc.getNodeByName("wf-retry-pol.onExit")
+	onExitNode := woc.wf.GetNodeByName("wf-retry-pol.onExit")
 	if assert.NotNil(t, onExitNode) {
 		assert.Equal(t, wfv1.NodePending, onExitNode.Phase)
 	}
@@ -1631,12 +1627,12 @@ func TestOnExitDAGPhase(t *testing.T) {
 	woc := newWorkflowOperationCtx(wf, controller)
 
 	woc.operate()
-	retryNode := woc.getNodeByName("dag-diamond-88trp")
+	retryNode := woc.wf.GetNodeByName("dag-diamond-88trp")
 	if assert.NotNil(t, retryNode) {
 		assert.Equal(t, wfv1.NodeRunning, retryNode.Phase)
 	}
 
-	retryNode = woc.getNodeByName("B.onExit")
+	retryNode = woc.wf.GetNodeByName("B.onExit")
 	if assert.NotNil(t, retryNode) {
 		assert.Equal(t, wfv1.NodePending, retryNode.Phase)
 	}
@@ -1760,11 +1756,17 @@ func TestOnExitNonLeaf(t *testing.T) {
 	woc := newWorkflowOperationCtx(wf, controller)
 
 	woc.operate()
-	retryNode := woc.getNodeByName("step-2.onExit")
+	retryNode := woc.wf.GetNodeByName("step-2.onExit")
 	if assert.NotNil(t, retryNode) {
 		assert.Equal(t, wfv1.NodePending, retryNode.Phase)
 	}
-	retryNode = woc.getNodeByName("exit-handler-bug-example.step-3")
+
+	assert.Nil(t, woc.wf.GetNodeByName("exit-handler-bug-example.step-3"))
+
+	retryNode.Phase = wfv1.NodeSucceeded
+	woc.wf.Status.Nodes[retryNode.ID] = *retryNode
+	woc.operate()
+	retryNode = woc.wf.GetNodeByName("exit-handler-bug-example.step-3")
 	if assert.NotNil(t, retryNode) {
 		assert.Equal(t, wfv1.NodePending, retryNode.Phase)
 	}
diff --git a/workflow/controller/operator_test.go b/workflow/controller/operator_test.go
index fc0b6a3e4..49b1c52ca 100644
--- a/workflow/controller/operator_test.go
+++ b/workflow/controller/operator_test.go
@@ -198,7 +198,7 @@ func TestProcessNodesWithRetries(t *testing.T) {
 		woc.addChildNode(nodeName, childNode)
 	}
 
-	n := woc.getNodeByName(nodeName)
+	n := woc.wf.GetNodeByName(nodeName)
 	lastChild = getChildNodeIndex(n, woc.wf.Status.Nodes, -1)
 	assert.NotNil(t, lastChild)
 
@@ -220,14 +220,14 @@ func TestProcessNodesWithRetries(t *testing.T) {
 	woc.markNodePhase(lastChild.Name, wfv1.NodeFailed)
 	_, _, err = woc.processNodeRetries(n, retries, &executeTemplateOpts{})
 	assert.NoError(t, err)
-	n = woc.getNodeByName(nodeName)
+	n = woc.wf.GetNodeByName(nodeName)
 	assert.Equal(t, n.Phase, wfv1.NodeRunning)
 
 	// Add a third node that has failed.
 	childNode := "child-node-3"
 	woc.initializeNode(childNode, wfv1.NodeTypePod, "", &wfv1.Template{}, "", wfv1.NodeFailed)
 	woc.addChildNode(nodeName, childNode)
-	n = woc.getNodeByName(nodeName)
+	n = woc.wf.GetNodeByName(nodeName)
 	n, _, err = woc.processNodeRetries(n, retries, &executeTemplateOpts{})
 	assert.NoError(t, err)
 	assert.Equal(t, n.Phase, wfv1.NodeFailed)
@@ -270,7 +270,7 @@ func TestProcessNodesWithRetriesOnErrors(t *testing.T) {
 		woc.addChildNode(nodeName, childNode)
 	}
 
-	n := woc.getNodeByName(nodeName)
+	n := woc.wf.GetNodeByName(nodeName)
 	lastChild = getChildNodeIndex(n, woc.wf.Status.Nodes, -1)
 	assert.NotNil(t, lastChild)
 
@@ -292,14 +292,14 @@ func TestProcessNodesWithRetriesOnErrors(t *testing.T) {
 	woc.markNodePhase(lastChild.Name, wfv1.NodeError)
 	_, _, err = woc.processNodeRetries(n, retries, &executeTemplateOpts{})
 	assert.NoError(t, err)
-	n = woc.getNodeByName(nodeName)
+	n = woc.wf.GetNodeByName(nodeName)
 	assert.Equal(t, n.Phase, wfv1.NodeRunning)
 
 	// Add a third node that has errored.
 	childNode := "child-node-3"
 	woc.initializeNode(childNode, wfv1.NodeTypePod, "", &wfv1.Template{}, "", wfv1.NodeError)
 	woc.addChildNode(nodeName, childNode)
-	n = woc.getNodeByName(nodeName)
+	n = woc.wf.GetNodeByName(nodeName)
 	n, _, err = woc.processNodeRetries(n, retries, &executeTemplateOpts{})
 	assert.Nil(t, err)
 	assert.Equal(t, n.Phase, wfv1.NodeError)
@@ -343,7 +343,7 @@ func TestProcessNodesWithRetriesWithBackoff(t *testing.T) {
 	woc.initializeNode("child-node-1", wfv1.NodeTypePod, "", &wfv1.Template{}, "", wfv1.NodeRunning)
 	woc.addChildNode(nodeName, "child-node-1")
 
-	n := woc.getNodeByName(nodeName)
+	n := woc.wf.GetNodeByName(nodeName)
 	lastChild = getChildNodeIndex(n, woc.wf.Status.Nodes, -1)
 	assert.NotNil(t, lastChild)
 
@@ -398,7 +398,7 @@ func TestProcessNodesNoRetryWithError(t *testing.T) {
 		woc.addChildNode(nodeName, childNode)
 	}
 
-	n := woc.getNodeByName(nodeName)
+	n := woc.wf.GetNodeByName(nodeName)
 	lastChild = getChildNodeIndex(n, woc.wf.Status.Nodes, -1)
 	assert.NotNil(t, lastChild)
 
@@ -421,7 +421,7 @@ func TestProcessNodesNoRetryWithError(t *testing.T) {
 	woc.markNodePhase(lastChild.Name, wfv1.NodeError)
 	_, _, err = woc.processNodeRetries(n, retries, &executeTemplateOpts{})
 	assert.NoError(t, err)
-	n = woc.getNodeByName(nodeName)
+	n = woc.wf.GetNodeByName(nodeName)
 	assert.Equal(t, wfv1.NodeError, n.Phase)
 }
 
@@ -557,7 +557,7 @@ func TestBackoffMessage(t *testing.T) {
 	assert.NotNil(t, woc)
 	_, _, err := woc.loadExecutionSpec()
 	assert.NoError(t, err)
-	retryNode := woc.getNodeByName("retry-backoff-s69z6")
+	retryNode := woc.wf.GetNodeByName("retry-backoff-s69z6")
 
 	// Simulate backoff of 4 secods
 	firstNode := getChildNodeIndex(retryNode, woc.wf.Status.Nodes, 0)
@@ -3033,7 +3033,7 @@ func TestRetryNodeOutputs(t *testing.T) {
 	assert.NoError(t, err)
 	woc := newWorkflowOperationCtx(wf, controller)
 
-	retryNode := woc.getNodeByName("daemon-step-dvbnn[0].influx")
+	retryNode := woc.wf.GetNodeByName("daemon-step-dvbnn[0].influx")
 	assert.NotNil(t, retryNode)
 	fmt.Println(retryNode)
 	scope := &wfScope{
@@ -3637,7 +3637,7 @@ func TestNoOnExitWhenSkipped(t *testing.T) {
 
 	woc := newWoc(*wf)
 	woc.operate()
-	assert.Nil(t, woc.getNodeByName("B.onExit"))
+	assert.Nil(t, woc.wf.GetNodeByName("B.onExit"))
 }
 
 func TestGenerateNodeName(t *testing.T) {
@@ -3898,7 +3898,7 @@ func TestPropagateMaxDurationProcess(t *testing.T) {
 	woc.addChildNode(nodeName, childNode)
 
 	var opts executeTemplateOpts
-	n := woc.getNodeByName(nodeName)
+	n := woc.wf.GetNodeByName(nodeName)
 	_, _, err = woc.processNodeRetries(n, retries, &opts)
 	if assert.NoError(t, err) {
 		assert.Equal(t, n.StartedAt.Add(20*time.Second).Round(time.Second).String(), opts.executionDeadline.Round(time.Second).String())