Repository: argoproj/argo
Pull request #4729 (2020-12-12 18:46:50)

docs: Update SSO docs to explain how to use the Argo CD Dex instance with Argo Workflows Server
I am fairly certain this is the correct place for this documentation update but would like to know if there are more places to update as well as this took me quite a while to figure out.

Checklist:

* [ ] My organization is added to [USERS.md](https://github.com/argoproj/argo/blob/master/USERS.md).

<!--

Please leave your PR in draft if you don't need a review yet. 

To fix failing `CI / Codegen`, run `make codegen`. 

-->



Patch:
diff --git a/docs/argo-server-sso.md b/docs/argo-server-sso.md
index 6a0482935..3c2e053c4 100644
--- a/docs/argo-server-sso.md
+++ b/docs/argo-server-sso.md
@@ -107,3 +107,53 @@ If no rule matches, we deny the user access.
     ```
     
     The precedence must be the lowest of all your service accounts. 
+
+## Sharing the Argo CD Dex Instance using Oauth2
+
+It is possible to have the Argo Workflows Server use the Argo CD Dex instance for SSO, for instance if you use Okta with SAML which cannot integrate with Argo Workflows directly. In order to make this happen, you will need the following:
+
+- You must be using at least Dex [v2.23.0](https://github.com/dexidp/dex/releases/tag/v2.23.0), because that's when `staticClients[].secretEnv` was added.
+- A secret created above with a `client-id` and `client-secret` to be used by both Dex and Argo Workflows Server. It is called `argo-workflows-sso` in this example.
+- `--auth-mode=sso` server argument added
+- A Dex `staticClients` configured for `argo-workflows-sso`
+- The `sso` configuration filled out in Argo Workflows Server to match
+
+What this might look like in your chart configuration:
+
+`argo-cd/values.yaml`:
+```yaml
+     dex:
+       image:
+         tag: v2.23.0
+       env:
+         - name: ARGO_WORKFLOWS_SSO_CLIENT_SECRET
+           valueFrom:
+             secretKeyRef:
+               name: argo-workflows-sso
+               key: client-secret
+     server:
+       config:
+         dex.config: |
+           staticClients:
+           - id: argo-workflows-sso
+             name: Argo Workflow
+             redirectURIs:
+               - https://argo-workflows.mydomain.com/oauth2/callback
+             secretEnv: ARGO_WORKFLOWS_SSO_CLIENT_SECRET
+```
+
+`argo/values.yaml`:
+```yaml
+     server:
+       extraArgs:
+         - --auth-mode=sso
+       sso:
+         issuer: https://argo-cd.mydomain.com/api/dex
+         clientId:
+           name: argo-workflows-sso
+           key: client-id
+         clientSecret:
+           name: argo-workflows-sso
+           key: client-secret
+         redirectUrl: https://argo-workflows.mydomain.com/oauth2/callback
+```
diff --git a/docs/configure-artifact-repository.md b/docs/configure-artifact-repository.md
index f096d1bc5..bd694e1e0 100644
--- a/docs/configure-artifact-repository.md
+++ b/docs/configure-artifact-repository.md
@@ -154,7 +154,7 @@ access is on a per project rather than per bucket basis.
 ```yaml
 artifacts:
   - name: my-output-artifact
-    path: /my-ouput-artifact
+    path: /my-output-artifact
     s3:
       endpoint: storage.googleapis.com
       bucket: my-gcs-bucket-name
@@ -320,7 +320,7 @@ configuring the default artifact repository described previously.
     outputs:
       artifacts:
       - name: my-output-artifact
-        path: /my-ouput-artifact
+        path: /my-output-artifact
         s3:
           endpoint: storage.googleapis.com
           bucket: my-gcs-bucket-name
diff --git a/server/auth/gatekeeper.go b/server/auth/gatekeeper.go
index 55c135dc7..00403413a 100644
--- a/server/auth/gatekeeper.go
+++ b/server/auth/gatekeeper.go
@@ -131,7 +131,7 @@ func (s gatekeeper) getClients(ctx context.Context) (versioned.Interface, kubern
 	authorization := getAuthHeader(md)
 	mode, valid := s.Modes.GetMode(authorization)
 	if !valid {
-		return nil, nil, nil, status.Error(codes.Unauthenticated, "token not valid for requested mode")
+		return nil, nil, nil, status.Error(codes.Unauthenticated, "token not valid for running mode")
 	}
 	switch mode {
 	case Client:
diff --git a/server/auth/mode.go b/server/auth/mode.go
index 7e5e836f4..bd244527d 100644
--- a/server/auth/mode.go
+++ b/server/auth/mode.go
@@ -31,11 +31,14 @@ func (m Modes) Add(value string) error {
 }
 
 func (m Modes) GetMode(authorisation string) (Mode, bool) {
-	if strings.HasPrefix(authorisation, sso.Prefix) {
-		return SSO, m[SSO]
+	if m[SSO] && strings.HasPrefix(authorisation, sso.Prefix) {
+		return SSO, true
 	}
-	if strings.HasPrefix(authorisation, "Bearer ") || strings.HasPrefix(authorisation, "Basic ") {
-		return Client, m[Client]
+	if m[Client] && strings.HasPrefix(authorisation, "Bearer ") || strings.HasPrefix(authorisation, "Basic ") {
+		return Client, true
 	}
-	return Server, m[Server]
+	if m[Server] {
+		return Server, true
+	}
+	return "", false
 }
diff --git a/ui/src/app/shared/template-resolution.ts b/ui/src/app/shared/template-resolution.ts
index 316b463ec..8e2c1ab42 100644
--- a/ui/src/app/shared/template-resolution.ts
+++ b/ui/src/app/shared/template-resolution.ts
@@ -16,7 +16,7 @@ export function getResolvedTemplates(workflow: models.Workflow, node: models.Nod
         if (templRef.StorageNeeded) {
             tmpl = workflow.status.storedTemplates[templRef.StoredTemplateName];
         } else if (tmpTemplate.template) {
-            tmpl = workflow.spec.templates.find(item => item.name === tmpTemplate.template);
+            tmpl = (workflow.status.storedWorkflowTemplateSpec || workflow.spec).templates.find(item => item.name === tmpTemplate.template);
         }
         if (!tmpl) {
             const name = templRef.StoredTemplateName || tmpTemplate.template;
diff --git a/ui/src/models/workflows.ts b/ui/src/models/workflows.ts
index 77f5a2f0a..8218ab907 100644
--- a/ui/src/models/workflows.ts
+++ b/ui/src/models/workflows.ts
@@ -611,9 +611,9 @@ export interface WorkflowStatus {
     conditions?: Condition[];
 
     /**
-     * StoredWorkflowSpec is a Workflow Spec of top level WorkflowTemplate.
+     * StoredWorkflowTemplateSpec is a Workflow Spec of top level WorkflowTemplate.
      */
-    storedWorkflowSpec?: WorkflowSpec;
+    storedWorkflowTemplateSpec?: WorkflowSpec;
 }
 
 export interface Condition {
diff --git a/workflow/controller/operator.go b/workflow/controller/operator.go
index d00dba8de..acf441606 100644
--- a/workflow/controller/operator.go
+++ b/workflow/controller/operator.go
@@ -909,21 +909,18 @@ func (woc *wfOperationCtx) podReconciliation() error {
 		}
 		if _, ok := seenPods[nodeID]; !ok {
 
+			// grace-period to allow informer sync
+			recentlyStarted := recentlyStarted(node)
+			woc.log.WithFields(log.Fields{"nodeName": node.Name, "nodePhase": node.Phase, "recentlyStarted": recentlyStarted}).Info("Workflow pod is missing")
+			metrics.PodMissingMetric.WithLabelValues(strconv.FormatBool(recentlyStarted), string(node.Phase)).Inc()
+
 			// If the node is pending and the pod does not exist, it could be the case that we want to try to submit it
 			// again instead of marking it as an error. Check if that's the case.
-			if node.Pending() {
+			if node.Pending() || recentlyStarted {
 				continue
 			}
 
-			node.Message = "pod deleted"
-			node.Phase = wfv1.NodeError
-			// FinishedAt must be set since retry strategy depends on it to determine the backoff duration.
-			// See processNodeRetries for more details.
-			node.FinishedAt = metav1.Time{Time: time.Now().UTC()}
-			woc.wf.Status.Nodes[nodeID] = node
-			woc.log.WithField("displayName", node.DisplayName).WithField("templateName", node.TemplateName).
-				WithField("node", node.Name).Error("Pod for node deleted")
-			woc.updated = true
+			woc.markNodePhase(node.Name, wfv1.NodeError, "pod deleted")
 		} else {
 			// At this point we are certain that the pod associated with our node is running or has been run;
 			// it is safe to extract the k8s-node information given this knowledge.
@@ -937,6 +934,10 @@ func (woc *wfOperationCtx) podReconciliation() error {
 	return nil
 }
 
+func recentlyStarted(node wfv1.NodeStatus) bool {
+	return time.Since(node.StartedAt.Time) <= envutil.LookupEnvDurationOr("RECENTLY_STARTED_POD_DURATION", 10*time.Second)
+}
+
 // shouldPrintPodSpec return eligible to print to the pod spec
 func (woc *wfOperationCtx) shouldPrintPodSpec(node wfv1.NodeStatus) bool {
 	return woc.controller.Config.PodSpecLogStrategy.AllPods ||
diff --git a/workflow/executor/common/wait/wait.go b/workflow/executor/common/wait/wait.go
index 0b5910c5b..63bded23a 100644
--- a/workflow/executor/common/wait/wait.go
+++ b/workflow/executor/common/wait/wait.go
@@ -26,22 +26,32 @@ func UntilTerminated(kubernetesInterface kubernetes.Interface, namespace, podNam
 }
 
 func untilTerminatedAux(podInterface v1.PodInterface, containerID string, listOptions metav1.ListOptions) (bool, error) {
+	for {
+		timedOut, done, err := doWatch(podInterface, containerID, listOptions)
+		if !timedOut {
+			return done, err
+		}
+		log.Infof("Pod watch timed out, restarting watch on %s", containerID)
+	}
+}
+
+func doWatch(podInterface v1.PodInterface, containerID string, listOptions metav1.ListOptions) (bool, bool, error) {
 	w, err := podInterface.Watch(listOptions)
 	if err != nil {
-		return true, fmt.Errorf("could not watch pod: %w", err)
+		return false, true, fmt.Errorf("could not watch pod: %w", err)
 	}
 	defer w.Stop()
 	for event := range w.ResultChan() {
 		pod, ok := event.Object.(*corev1.Pod)
 		if !ok {
-			return false, apierrors.FromObject(event.Object)
+			return false, false, apierrors.FromObject(event.Object)
 		}
 		for _, s := range pod.Status.ContainerStatuses {
 			if common.GetContainerID(&s) == containerID && s.State.Terminated != nil {
-				return true, nil
+				return false, true, nil
 			}
 		}
 		listOptions.ResourceVersion = pod.ResourceVersion
 	}
-	return true, nil
+	return true, false, nil
 }
diff --git a/workflow/metrics/pod_missing_metric.go b/workflow/metrics/pod_missing_metric.go
new file mode 100644
index 000000000..0ba2bd20c
--- /dev/null
+++ b/workflow/metrics/pod_missing_metric.go
@@ -0,0 +1,14 @@
+package metrics
+
+import "github.com/prometheus/client_golang/prometheus"
+
+var (
+	PodMissingMetric = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: argoNamespace,
+			Name:      "pod_missing",
+			Help:      "Incidents of pod missing",
+		},
+		[]string{"recently_started", "node_phase"},
+	)
+)
diff --git a/workflow/metrics/server.go b/workflow/metrics/server.go
index ce06cd3c7..d6223fb9c 100644
--- a/workflow/metrics/server.go
+++ b/workflow/metrics/server.go
@@ -70,6 +70,7 @@ func (m *Metrics) Describe(ch chan<- *prometheus.Desc) {
 		ch <- metric.Desc()
 	}
 	m.logMetric.Describe(ch)
+	PodMissingMetric.Describe(ch)
 }
 
 func (m *Metrics) Collect(ch chan<- prometheus.Metric) {
@@ -77,6 +78,7 @@ func (m *Metrics) Collect(ch chan<- prometheus.Metric) {
 		ch <- metric
 	}
 	m.logMetric.Collect(ch)
+	PodMissingMetric.Collect(ch)
 }
 
 func (m *Metrics) garbageCollector(ctx context.Context) {
diff --git a/workflow/ttlcontroller/ttlcontroller.go b/workflow/ttlcontroller/ttlcontroller.go
index e6cafe1d7..dc4597fa4 100644
--- a/workflow/ttlcontroller/ttlcontroller.go
+++ b/workflow/ttlcontroller/ttlcontroller.go
@@ -41,7 +41,7 @@ func NewController(wfClientset wfclientset.Interface, wfInformer cache.SharedInd
 	controller := &Controller{
 		wfclientset:  wfClientset,
 		wfInformer:   wfInformer,
-		workqueue:    workqueue.NewDelayingQueue(),
+		workqueue:    workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "workflow_ttl_queue"),
 		resyncPeriod: workflowTTLResyncPeriod,
 		clock:        clock.RealClock{},
 	}
@@ -129,6 +129,9 @@ func (c *Controller) enqueueWF(obj interface{}) {
 		log.Warnf("'%v' is not an unstructured", obj)
 		return
 	}
+	if un.GetDeletionTimestamp() != nil {
+		return
+	}
 	wf, err := util.FromUnstructured(un)
 	if err != nil {
 		log.Warnf("Failed to unmarshal workflow %v object: %v", obj, err)


Test patch:
diff --git a/server/auth/mode_test.go b/server/auth/mode_test.go
index 2fe5bd9b6..2a55e0e9f 100644
--- a/server/auth/mode_test.go
+++ b/server/auth/mode_test.go
@@ -60,4 +60,16 @@ func TestModes_GetMode(t *testing.T) {
 			assert.Equal(t, SSO, mode)
 		}
 	})
+
+	m = Modes{
+		Client: false,
+		SSO:    false,
+		Server: true,
+	}
+	t.Run("Server and Auth", func(t *testing.T) {
+		mode, valid := m.GetMode("Bearer ")
+		if assert.True(t, valid) {
+			assert.Equal(t, Server, mode)
+		}
+	})
 }