Repository: argoproj/argo (go)

Issue:
UI crash when viewing YAML of nodes in workflows using `workflowTemplateRef`
## Summary 
View the yaml def of archived workflows does not work in 2.12-rc4, and will render a stack trace error in the UI
Images from 2.12-rc4 (BTW, the same problem occurs if click on the Containers tab)
![image](https://user-images.githubusercontent.com/9726307/101751356-783ffc80-3ad0-11eb-8fcc-51900344fbc9.png)
![image](https://user-images.githubusercontent.com/9726307/101751554-ad4c4f00-3ad0-11eb-9d8c-8a8357319600.png)

The same use case does work in 2.11.8
![image](https://user-images.githubusercontent.com/9726307/101751638-c6ed9680-3ad0-11eb-9d0a-2555b0b0fedd.png)
![image](https://user-images.githubusercontent.com/9726307/101751767-f3091780-3ad0-11eb-99a4-689b371b6365.png)

## Diagnostics
Kops k8s 1.17.13 on AWS
2.12-rc4

There are no errors in the server logs and the workflow controller logs.

<!-- Issue Author: Don't delete this message to encourage other users to support your issue! -->
**Message from the maintainers**:

Impacted by this bug? Give it a 👍. We prioritise the issues with the most 👍.


Pull request description:
fix(ui): Fix UI YAML for in workflows with stored spec. Fixes #4691
Signed-off-by: Alex Collins <alex_collins@intuit.com>
Fixes #4691

Checklist:

* [x] My organization is added to [USERS.md](https://github.com/argoproj/argo/blob/master/USERS.md).

<!--

Please leave your PR in draft if you don't need a review yet. 

To fix failing `CI / Codegen`, run `make codegen`. 

-->


Patch:
diff --git a/docs/configure-artifact-repository.md b/docs/configure-artifact-repository.md
index f096d1bc5..bd694e1e0 100644
--- a/docs/configure-artifact-repository.md
+++ b/docs/configure-artifact-repository.md
@@ -154,7 +154,7 @@ access is on a per project rather than per bucket basis.
 ```yaml
 artifacts:
   - name: my-output-artifact
-    path: /my-ouput-artifact
+    path: /my-output-artifact
     s3:
       endpoint: storage.googleapis.com
       bucket: my-gcs-bucket-name
@@ -320,7 +320,7 @@ configuring the default artifact repository described previously.
     outputs:
       artifacts:
       - name: my-output-artifact
-        path: /my-ouput-artifact
+        path: /my-output-artifact
         s3:
           endpoint: storage.googleapis.com
           bucket: my-gcs-bucket-name
diff --git a/server/auth/gatekeeper.go b/server/auth/gatekeeper.go
index 55c135dc7..00403413a 100644
--- a/server/auth/gatekeeper.go
+++ b/server/auth/gatekeeper.go
@@ -131,7 +131,7 @@ func (s gatekeeper) getClients(ctx context.Context) (versioned.Interface, kubern
 	authorization := getAuthHeader(md)
 	mode, valid := s.Modes.GetMode(authorization)
 	if !valid {
-		return nil, nil, nil, status.Error(codes.Unauthenticated, "token not valid for requested mode")
+		return nil, nil, nil, status.Error(codes.Unauthenticated, "token not valid for running mode")
 	}
 	switch mode {
 	case Client:
diff --git a/server/auth/mode.go b/server/auth/mode.go
index 7e5e836f4..bd244527d 100644
--- a/server/auth/mode.go
+++ b/server/auth/mode.go
@@ -31,11 +31,14 @@ func (m Modes) Add(value string) error {
 }
 
 func (m Modes) GetMode(authorisation string) (Mode, bool) {
-	if strings.HasPrefix(authorisation, sso.Prefix) {
-		return SSO, m[SSO]
+	if m[SSO] && strings.HasPrefix(authorisation, sso.Prefix) {
+		return SSO, true
 	}
-	if strings.HasPrefix(authorisation, "Bearer ") || strings.HasPrefix(authorisation, "Basic ") {
-		return Client, m[Client]
+	if m[Client] && strings.HasPrefix(authorisation, "Bearer ") || strings.HasPrefix(authorisation, "Basic ") {
+		return Client, true
 	}
-	return Server, m[Server]
+	if m[Server] {
+		return Server, true
+	}
+	return "", false
 }
diff --git a/ui/src/app/shared/template-resolution.ts b/ui/src/app/shared/template-resolution.ts
index 316b463ec..8e2c1ab42 100644
--- a/ui/src/app/shared/template-resolution.ts
+++ b/ui/src/app/shared/template-resolution.ts
@@ -16,7 +16,7 @@ export function getResolvedTemplates(workflow: models.Workflow, node: models.Nod
         if (templRef.StorageNeeded) {
             tmpl = workflow.status.storedTemplates[templRef.StoredTemplateName];
         } else if (tmpTemplate.template) {
-            tmpl = workflow.spec.templates.find(item => item.name === tmpTemplate.template);
+            tmpl = (workflow.status.storedWorkflowTemplateSpec || workflow.spec).templates.find(item => item.name === tmpTemplate.template);
         }
         if (!tmpl) {
             const name = templRef.StoredTemplateName || tmpTemplate.template;
diff --git a/ui/src/models/workflows.ts b/ui/src/models/workflows.ts
index 77f5a2f0a..8218ab907 100644
--- a/ui/src/models/workflows.ts
+++ b/ui/src/models/workflows.ts
@@ -611,9 +611,9 @@ export interface WorkflowStatus {
     conditions?: Condition[];
 
     /**
-     * StoredWorkflowSpec is a Workflow Spec of top level WorkflowTemplate.
+     * StoredWorkflowTemplateSpec is a Workflow Spec of top level WorkflowTemplate.
      */
-    storedWorkflowSpec?: WorkflowSpec;
+    storedWorkflowTemplateSpec?: WorkflowSpec;
 }
 
 export interface Condition {
diff --git a/workflow/controller/operator.go b/workflow/controller/operator.go
index d00dba8de..acf441606 100644
--- a/workflow/controller/operator.go
+++ b/workflow/controller/operator.go
@@ -909,21 +909,18 @@ func (woc *wfOperationCtx) podReconciliation() error {
 		}
 		if _, ok := seenPods[nodeID]; !ok {
 
+			// grace-period to allow informer sync
+			recentlyStarted := recentlyStarted(node)
+			woc.log.WithFields(log.Fields{"nodeName": node.Name, "nodePhase": node.Phase, "recentlyStarted": recentlyStarted}).Info("Workflow pod is missing")
+			metrics.PodMissingMetric.WithLabelValues(strconv.FormatBool(recentlyStarted), string(node.Phase)).Inc()
+
 			// If the node is pending and the pod does not exist, it could be the case that we want to try to submit it
 			// again instead of marking it as an error. Check if that's the case.
-			if node.Pending() {
+			if node.Pending() || recentlyStarted {
 				continue
 			}
 
-			node.Message = "pod deleted"
-			node.Phase = wfv1.NodeError
-			// FinishedAt must be set since retry strategy depends on it to determine the backoff duration.
-			// See processNodeRetries for more details.
-			node.FinishedAt = metav1.Time{Time: time.Now().UTC()}
-			woc.wf.Status.Nodes[nodeID] = node
-			woc.log.WithField("displayName", node.DisplayName).WithField("templateName", node.TemplateName).
-				WithField("node", node.Name).Error("Pod for node deleted")
-			woc.updated = true
+			woc.markNodePhase(node.Name, wfv1.NodeError, "pod deleted")
 		} else {
 			// At this point we are certain that the pod associated with our node is running or has been run;
 			// it is safe to extract the k8s-node information given this knowledge.
@@ -937,6 +934,10 @@ func (woc *wfOperationCtx) podReconciliation() error {
 	return nil
 }
 
+func recentlyStarted(node wfv1.NodeStatus) bool {
+	return time.Since(node.StartedAt.Time) <= envutil.LookupEnvDurationOr("RECENTLY_STARTED_POD_DURATION", 10*time.Second)
+}
+
 // shouldPrintPodSpec return eligible to print to the pod spec
 func (woc *wfOperationCtx) shouldPrintPodSpec(node wfv1.NodeStatus) bool {
 	return woc.controller.Config.PodSpecLogStrategy.AllPods ||
diff --git a/workflow/executor/common/wait/wait.go b/workflow/executor/common/wait/wait.go
index 0b5910c5b..63bded23a 100644
--- a/workflow/executor/common/wait/wait.go
+++ b/workflow/executor/common/wait/wait.go
@@ -26,22 +26,32 @@ func UntilTerminated(kubernetesInterface kubernetes.Interface, namespace, podNam
 }
 
 func untilTerminatedAux(podInterface v1.PodInterface, containerID string, listOptions metav1.ListOptions) (bool, error) {
+	for {
+		timedOut, done, err := doWatch(podInterface, containerID, listOptions)
+		if !timedOut {
+			return done, err
+		}
+		log.Infof("Pod watch timed out, restarting watch on %s", containerID)
+	}
+}
+
+func doWatch(podInterface v1.PodInterface, containerID string, listOptions metav1.ListOptions) (bool, bool, error) {
 	w, err := podInterface.Watch(listOptions)
 	if err != nil {
-		return true, fmt.Errorf("could not watch pod: %w", err)
+		return false, true, fmt.Errorf("could not watch pod: %w", err)
 	}
 	defer w.Stop()
 	for event := range w.ResultChan() {
 		pod, ok := event.Object.(*corev1.Pod)
 		if !ok {
-			return false, apierrors.FromObject(event.Object)
+			return false, false, apierrors.FromObject(event.Object)
 		}
 		for _, s := range pod.Status.ContainerStatuses {
 			if common.GetContainerID(&s) == containerID && s.State.Terminated != nil {
-				return true, nil
+				return false, true, nil
 			}
 		}
 		listOptions.ResourceVersion = pod.ResourceVersion
 	}
-	return true, nil
+	return true, false, nil
 }
diff --git a/workflow/metrics/pod_missing_metric.go b/workflow/metrics/pod_missing_metric.go
new file mode 100644
index 000000000..0ba2bd20c
--- /dev/null
+++ b/workflow/metrics/pod_missing_metric.go
@@ -0,0 +1,14 @@
+package metrics
+
+import "github.com/prometheus/client_golang/prometheus"
+
+var (
+	PodMissingMetric = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: argoNamespace,
+			Name:      "pod_missing",
+			Help:      "Incidents of pod missing",
+		},
+		[]string{"recently_started", "node_phase"},
+	)
+)
diff --git a/workflow/metrics/server.go b/workflow/metrics/server.go
index ce06cd3c7..d6223fb9c 100644
--- a/workflow/metrics/server.go
+++ b/workflow/metrics/server.go
@@ -70,6 +70,7 @@ func (m *Metrics) Describe(ch chan<- *prometheus.Desc) {
 		ch <- metric.Desc()
 	}
 	m.logMetric.Describe(ch)
+	PodMissingMetric.Describe(ch)
 }
 
 func (m *Metrics) Collect(ch chan<- prometheus.Metric) {
@@ -77,6 +78,7 @@ func (m *Metrics) Collect(ch chan<- prometheus.Metric) {
 		ch <- metric
 	}
 	m.logMetric.Collect(ch)
+	PodMissingMetric.Collect(ch)
 }
 
 func (m *Metrics) garbageCollector(ctx context.Context) {


Test patch:
diff --git a/server/auth/mode_test.go b/server/auth/mode_test.go
index 2fe5bd9b6..2a55e0e9f 100644
--- a/server/auth/mode_test.go
+++ b/server/auth/mode_test.go
@@ -60,4 +60,16 @@ func TestModes_GetMode(t *testing.T) {
 			assert.Equal(t, SSO, mode)
 		}
 	})
+
+	m = Modes{
+		Client: false,
+		SSO:    false,
+		Server: true,
+	}
+	t.Run("Server and Auth", func(t *testing.T) {
+		mode, valid := m.GetMode("Bearer ")
+		if assert.True(t, valid) {
+			assert.Equal(t, Server, mode)
+		}
+	})
 }