Repository: argoproj/argo (go)

Issue:
Introduce exit handler vairable to be filled with the name and output of step failed
# Summary

Introduce exit handler vairable to be filled with the name and output of step failed

# Motivation

As suggested on Slack:

> Trying to write a "generic" exit-handler that can pull out the information about what task failed (maybe its stderr, or something like that) and give me a nice email/slack message 

# Proposal

Introduce exit handler vairable to be filled with the name and output of step failed

---
**Message from the maintainers**:

If you wish to see this enhancement implemented please add a 👍 reaction to this issue! We often sort issues this way to know what to prioritize.

Pull request:
feat: Add failed node info to exit handler
Closes #1994 

Adds failed node information to the exit handler as a JSON list under the variable `workflow.failures`.

Example:
```yaml
# Example on continue-on-fail, where the workflow will continue running even
# some steps fail.
apiVersion: argoproj.io/v1alpha1
kind: Workflow
metadata:
  generateName: continue-on-fail-
spec:
  entrypoint: workflow-ignore
  onExit: send-email
  parallelism: 1
  templates:
  - name: workflow-ignore
    steps:
    - - name: A
        template: intentional-fail
        continueOn:
          failed: true
    - - name: B
        template: whalesay
      - name: C
        template: intentional-fail
        continueOn:
          failed: true
    - - name: D
        template: whalesay

  - name: whalesay
    container:
      image: docker/whalesay:latest
      command: [cowsay]
      args: ["hello world"]

  - name: intentional-fail
    container:
      image: alpine:latest
      command: [sh, -c]
      args: ["echo intentional failure; exit 1"]

  - name: send-email
    container:
      image: alpine:latest
      command: [sh, -c]
      args: ["echo send e-mail: {{workflow.name}} {{workflow.status}}. Failed steps {{workflow.failures}}"]
```

`workflow.failures` contents:
```json
[
  {
    "name": "C",
    "message": "failed with exit code 1",
    "templateName": "intentional-fail",
    "finishedAt": "2020-02-04 18:07:48 +0000 UTC",
    "phase": "Failed"
  },
  {
    "name": "A",
    "message": "failed with exit code 1",
    "templateName": "intentional-fail",
    "finishedAt": "2020-02-04 18:07:37 +0000 UTC",
    "phase": "Failed"
  }
]
```

Patch:
diff --git a/docs/variables.md b/docs/variables.md
index d108d1336..bad5b30ea 100644
--- a/docs/variables.md
+++ b/docs/variables.md
@@ -60,3 +60,4 @@ The following variables are made available to reference various metadata of a wo
 | Variable | Description|
 |----------|------------|
 | `workflow.status` | Workflow status. One of: `Succeeded`, `Failed`, `Error` |
+| `workflow.failures` | A list of JSON objects containing information about nodes that failed or errored during execution. Includes `name`, `message`, `templateName`, `finishedAt`, and `phase`. |
diff --git a/examples/exit-handlers.yaml b/examples/exit-handlers.yaml
index 3b99dca4e..2694fd06e 100644
--- a/examples/exit-handlers.yaml
+++ b/examples/exit-handlers.yaml
@@ -39,7 +39,14 @@ spec:
     container:
       image: alpine:latest
       command: [sh, -c]
-      args: ["echo send e-mail: {{workflow.name}} {{workflow.status}}"]
+      # Tip: {{workflow.failures}} is a JSON list. If you're using bash to read it, we recommend using jq to manipulate
+      # it. For example:
+      #
+      # echo "{{workflow.failures}}" | jq -r '.[] | "Failed Step: \(.displayName)\tMessage: \(.message)"'
+      #
+      # Will print a list of all the failed steps and their messages. For more info look up the jq docs.
+      # Note: jq is not installed by default on the "alpine:latest" image, however it can be installed with "apk add jq"
+      args: ["echo send e-mail: {{workflow.name}} {{workflow.status}}. Failed steps {{workflow.failures}}"]
   - name: celebrate
     container:
       image: alpine:latest
diff --git a/workflow/common/common.go b/workflow/common/common.go
index b900c35e5..8355920fa 100644
--- a/workflow/common/common.go
+++ b/workflow/common/common.go
@@ -118,6 +118,8 @@ const (
 	GlobalVarWorkflowCreationTimestamp = "workflow.creationTimestamp"
 	// GlobalVarWorkflowPriority is the workflow variable referencing the workflow's priority field
 	GlobalVarWorkflowPriority = "workflow.priority"
+	// GlobalVarWorkflowFailures is a global variable of a JSON map referencing the workflow's failed nodes
+	GlobalVarWorkflowFailures = "workflow.failures"
 	// LocalVarPodName is a step level variable that references the name of the pod
 	LocalVarPodName = "pod.name"
 
diff --git a/workflow/controller/operator.go b/workflow/controller/operator.go
index 5537f947b..4fc4899df 100644
--- a/workflow/controller/operator.go
+++ b/workflow/controller/operator.go
@@ -95,6 +95,16 @@ var (
 // for before requeuing the workflow onto the workqueue.
 const maxOperationTime = 10 * time.Second
 
+// failedNodeStatus is a subset of NodeStatus that is only used to Marshal certain fields into a JSON of failed nodes
+type failedNodeStatus struct {
+	DisplayName  string      `json:"displayName"`
+	Message      string      `json:"message"`
+	TemplateName string      `json:"templateName"`
+	Phase        string      `json:"phase"`
+	PodName      string      `json:"podName"`
+	FinishedAt   metav1.Time `json:"finishedAt"`
+}
+
 // newWorkflowOperationCtx creates and initializes a new wfOperationCtx object.
 func newWorkflowOperationCtx(wf *wfv1.Workflow, wfc *WorkflowController) *wfOperationCtx {
 	// NEVER modify objects from the store. It's a read-only, local cache.
@@ -262,6 +272,29 @@ func (woc *wfOperationCtx) operate() {
 		} else {
 			woc.globalParams[common.GlobalVarWorkflowStatus] = string(workflowStatus)
 		}
+
+		var failures []failedNodeStatus
+		for _, node := range woc.wf.Status.Nodes {
+			if node.Phase == wfv1.NodeFailed || node.Phase == wfv1.NodeError {
+				failures = append(failures,
+					failedNodeStatus{
+						DisplayName:  node.DisplayName,
+						Message:      node.Message,
+						TemplateName: node.TemplateName,
+						Phase:        string(node.Phase),
+						PodName:      node.ID,
+						FinishedAt:   node.FinishedAt,
+					})
+			}
+		}
+		failedNodeBytes, err := json.Marshal(failures)
+		if err != nil {
+			woc.log.Errorf("Error marshalling failed nodes list: %+v", err)
+			// No need to return here
+		}
+		// This strconv.Quote is necessary so that the escaped quotes are not removed during parameter substitution
+		woc.globalParams[common.GlobalVarWorkflowFailures] = strconv.Quote(string(failedNodeBytes))
+
 		woc.log.Infof("Running OnExit handler: %s", woc.wf.Spec.OnExit)
 		onExitNodeName := woc.wf.ObjectMeta.Name + ".onExit"
 		onExitNode, err = woc.executeTemplate(onExitNodeName, &wfv1.Template{Template: woc.wf.Spec.OnExit}, tmplCtx, woc.wf.Spec.Arguments, "")
diff --git a/workflow/validate/validate.go b/workflow/validate/validate.go
index 112b2f4f5..d4d89a515 100644
--- a/workflow/validate/validate.go
+++ b/workflow/validate/validate.go
@@ -136,6 +136,7 @@ func ValidateWorkflow(wftmplGetter templateresolution.WorkflowTemplateNamespaced
 	if wf.Spec.OnExit != "" {
 		// now when validating onExit, {{workflow.status}} is now available as a global
 		ctx.globalParams[common.GlobalVarWorkflowStatus] = placeholderGenerator.NextPlaceholder()
+		ctx.globalParams[common.GlobalVarWorkflowFailures] = placeholderGenerator.NextPlaceholder()
 		_, err = ctx.validateTemplateHolder(&wfv1.Template{Template: wf.Spec.OnExit}, tmplCtx, &wf.Spec.Arguments, map[string]interface{}{})
 		if err != nil {
 			return err


Test patch:
diff --git a/workflow/controller/operator_test.go b/workflow/controller/operator_test.go
index 38ca46514..4ea23489a 100644
--- a/workflow/controller/operator_test.go
+++ b/workflow/controller/operator_test.go
@@ -1990,6 +1990,44 @@ func TestStepsOnExit(t *testing.T) {
 	assert.True(t, onExitNodeIsPresent)
 }
 
+var onExitFailures = `
+apiVersion: argoproj.io/v1alpha1
+kind: Workflow
+metadata:
+  name: exit-handlers
+spec:
+  entrypoint: intentional-fail
+  onExit: exit-handler
+  templates:
+  - name: intentional-fail
+    container:
+      image: alpine:latest
+      command: [sh, -c]
+      args: ["echo intentional failure; exit 1"]
+  - name: exit-handler
+    container:
+      image: alpine:latest
+      command: [sh, -c]
+      args: ["echo send e-mail: {{workflow.name}} {{workflow.status}}. Failed steps {{workflow.failures}}"]
+`
+
+func TestStepsOnExitFailures(t *testing.T) {
+	controller := newController()
+	wfcset := controller.wfclientset.ArgoprojV1alpha1().Workflows("")
+
+	// Test list expansion
+	wf := unmarshalWF(onExitFailures)
+	wf, err := wfcset.Create(wf)
+	assert.Nil(t, err)
+	woc := newWorkflowOperationCtx(wf, controller)
+
+	woc.operate()
+	woc.operate()
+
+	fmt.Println(woc.globalParams)
+	assert.Contains(t, woc.globalParams[common.GlobalVarWorkflowFailures], `[{\"displayName\":\"exit-handlers\",\"message\":\"Unexpected pod phase for exit-handlers: \",\"templateName\":\"intentional-fail\",\"phase\":\"Error\",\"podName\":\"exit-handlers\"`)
+}
+
 var invalidSpec = `
 apiVersion: argoproj.io/v1alpha1
 kind: Workflow