Repository: argoproj/argo (go)

Issue:
EKS workflowDefaults inconsistencies - kubelet executor
## Summary 

What happened/what you expected to happen?
I expected workflowDefaults to be honored the same with with the following commands:
```
argo submit steps.yaml -n test
argo submit --from workflowtemplate/dag-diamond-xyyxx -n test
```
When I run the the submission directly serviceaccount is not honored
When I run the submission from workflowtemplate ttl strategy is not honored

Running a fresh install of argo on a fresh EKS cluster:
```
kubectl create namespace argo
kubectl apply -n argo -f https://raw.githubusercontent.com/argoproj/argo/v2.11.1/manifests/install.yaml
kubectl create ns test
kubectl create sa -n test argo-test
kubectl create clusterrolebinding delete-me --clusterrole=cluster-admin --serviceaccount=test:argo-test
```

```
 k get cm -n argo workflow-controller-configmap -oyaml


apiVersion: v1
data:
  config: |
    containerRuntimeExecutor: kubelet
    workflowDefaults:
      spec:
        serviceAccountName: argo-test
        ttlStrategy:
          secondsAfterCompletion: 3600
          secondsAfterFailure: 3600
          secondsAfterSuccess: 3600
kind: ConfigMap
```


## Diagnostics

What Kubernetes provider are you using? 
EKS 
Client Version: version.Info{Major:"1", Minor:"18", GitVersion:"v1.18.0", GitCommit:"9e991415386e4cf155a24b1da15becaa390438d8", GitTreeState:"clean", BuildDate:"2020-03-26T06:16:15Z", GoVersion:"go1.14", Compiler:"gc", Platform:"darwin/amd64"}
Server Version: version.Info{Major:"1", Minor:"17+", GitVersion:"v1.17.9-eks-4c6976", GitCommit:"4c6976793196d70bc5cd29d56ce5440c9473648e", GitTreeState:"clean", BuildDate:"2020-07-17T18:46:04Z", GoVersion:"go1.13.9", Compiler:"gc", Platform:"linux/amd64"}

What version of Argo Workflows are you running? 
Argo client and server 2.11.1

Bug where ttl is honored but not serviceAccount:
(workflow fails and wait logs complain about the default not having enough permissions)
argo submit steps.yaml -n test
```yaml
apiVersion: argoproj.io/v1alpha1
kind: Workflow
metadata:
  creationTimestamp: "2020-10-02T02:12:37Z"
  generateName: dag-diamond-
  generation: 5
  labels:
    workflows.argoproj.io/completed: "true"
    workflows.argoproj.io/phase: Error
  name: dag-diamond-fz54p
  namespace: test
  resourceVersion: "10049"
  selfLink: /apis/argoproj.io/v1alpha1/namespaces/test/workflows/dag-diamond-fz54p
  uid: 8c5b2fe8-149d-4b23-9e71-16eda92ae401
spec:
  arguments: {}
  entrypoint: diamond
  serviceAccountName: argo-test
  templates:
  - arguments: {}
    container:
      command:
      - echo
      - '{{inputs.parameters.message}}'
      image: alpine:3.7
      name: ""
      resources: {}
    inputs:
      parameters:
      - name: message
    metadata: {}
    name: echo
    outputs: {}
  - arguments: {}
    dag:
      tasks:
      - arguments:
          parameters:
          - name: message
            value: A
        name: A
        template: echo
      - arguments:
          parameters:
          - name: message
            value: B
        dependencies:
        - A
        name: B
        template: echo
      - arguments:
          parameters:
          - name: message
            value: C
        dependencies:
        - A
        name: C
        template: echo
      - arguments:
          parameters:
          - name: message
            value: D
        dependencies:
        - B
        - C
        name: D
        template: echo
    inputs: {}
    metadata: {}
    name: diamond
    outputs: {}
  ttlStrategy:
    secondsAfterCompletion: 3600
    secondsAfterFailure: 3600
    secondsAfterSuccess: 3600
status:
  conditions:
  - status: "True"
    type: Completed
  finishedAt: "2020-10-02T02:12:47Z"
  nodes:
    dag-diamond-fz54p:
      children:
      - dag-diamond-fz54p-4207938145
      displayName: dag-diamond-fz54p
      finishedAt: "2020-10-02T02:12:47Z"
      id: dag-diamond-fz54p
      name: dag-diamond-fz54p
      outboundNodes:
      - dag-diamond-fz54p-4258271002
      phase: Error
      startedAt: "2020-10-02T02:12:37Z"
      templateName: diamond
      templateScope: local/dag-diamond-fz54p
      type: DAG
    dag-diamond-fz54p-4157605288:
      boundaryID: dag-diamond-fz54p
      children:
      - dag-diamond-fz54p-4258271002
      displayName: B
      finishedAt: "2020-10-02T02:12:47Z"
      id: dag-diamond-fz54p-4157605288
      message: 'omitted: depends condition not met'
      name: dag-diamond-fz54p.B
      phase: Omitted
      startedAt: "2020-10-02T02:12:47Z"
      templateName: echo
      templateScope: local/dag-diamond-fz54p
      type: Skipped
    dag-diamond-fz54p-4174382907:
      boundaryID: dag-diamond-fz54p
      children:
      - dag-diamond-fz54p-4258271002
      displayName: C
      finishedAt: "2020-10-02T02:12:47Z"
      id: dag-diamond-fz54p-4174382907
      message: 'omitted: depends condition not met'
      name: dag-diamond-fz54p.C
      phase: Omitted
      startedAt: "2020-10-02T02:12:47Z"
      templateName: echo
      templateScope: local/dag-diamond-fz54p
      type: Skipped
    dag-diamond-fz54p-4207938145:
      boundaryID: dag-diamond-fz54p
      children:
      - dag-diamond-fz54p-4157605288
      - dag-diamond-fz54p-4174382907
      displayName: A
      finishedAt: "2020-10-02T02:12:46Z"
      hostNodeName: ip-10-0-2-189.us-west-2.compute.internal
      id: dag-diamond-fz54p-4207938145
      inputs:
        parameters:
        - name: message
          value: A
      message: 'failed to save outputs: Failed to establish pod watch: timed out waiting
        for the condition'
      name: dag-diamond-fz54p.A
      phase: Error
      resourcesDuration:
        cpu: 8
        memory: 8
      startedAt: "2020-10-02T02:12:37Z"
      templateName: echo
      templateScope: local/dag-diamond-fz54p
      type: Pod
    dag-diamond-fz54p-4258271002:
      boundaryID: dag-diamond-fz54p
      displayName: D
      finishedAt: "2020-10-02T02:12:47Z"
      id: dag-diamond-fz54p-4258271002
      message: 'omitted: depends condition not met'
      name: dag-diamond-fz54p.D
      phase: Omitted
      startedAt: "2020-10-02T02:12:47Z"
      templateName: echo
      templateScope: local/dag-diamond-fz54p
      type: Skipped
  phase: Error
  resourcesDuration:
    cpu: 8
    memory: 8
  startedAt: "2020-10-02T02:12:37Z"
```

Bug where service account is honored and not ttl:
argo submit --from workflowtemplate/dag-diamond-sf8vp -n test
```yaml
apiVersion: argoproj.io/v1alpha1
kind: Workflow
metadata:
  creationTimestamp: "2020-10-02T02:14:26Z"
  generateName: dag-diamond-sf8vp-
  generation: 13
  labels:
    workflows.argoproj.io/completed: "true"
    workflows.argoproj.io/phase: Succeeded
    workflows.argoproj.io/workflow-template: dag-diamond-sf8vp
  name: dag-diamond-sf8vp-dxnjn
  namespace: test
  resourceVersion: "10464"
  selfLink: /apis/argoproj.io/v1alpha1/namespaces/test/workflows/dag-diamond-sf8vp-dxnjn
  uid: 319ff382-0ce6-4f10-acdc-abffedabb0e3
spec:
  arguments: {}
  workflowTemplateRef:
    name: dag-diamond-sf8vp
status:
  conditions:
  - status: "True"
    type: Completed
  finishedAt: "2020-10-02T02:14:40Z"
  nodes:
    dag-diamond-sf8vp-dxnjn:
      children:
      - dag-diamond-sf8vp-dxnjn-126754612
      displayName: dag-diamond-sf8vp-dxnjn
      finishedAt: "2020-10-02T02:14:40Z"
      id: dag-diamond-sf8vp-dxnjn
      name: dag-diamond-sf8vp-dxnjn
      outboundNodes:
      - dag-diamond-sf8vp-dxnjn-76421755
      phase: Succeeded
      startedAt: "2020-10-02T02:14:26Z"
      templateRef:
        name: dag-diamond-sf8vp
        template: diamond
      templateScope: local/
      type: DAG
    dag-diamond-sf8vp-dxnjn-76421755:
      boundaryID: dag-diamond-sf8vp-dxnjn
      displayName: D
      finishedAt: "2020-10-02T02:14:40Z"
      hostNodeName: ip-10-0-2-189.us-west-2.compute.internal
      id: dag-diamond-sf8vp-dxnjn-76421755
      inputs:
        parameters:
        - name: message
          value: D
      name: dag-diamond-sf8vp-dxnjn.D
      outputs:
        exitCode: "0"
      phase: Succeeded
      resourcesDuration:
        cpu: 3
        memory: 3
      startedAt: "2020-10-02T02:14:36Z"
      templateName: echo
      templateScope: namespaced/dag-diamond-sf8vp
      type: Pod
    dag-diamond-sf8vp-dxnjn-126754612:
      boundaryID: dag-diamond-sf8vp-dxnjn
      children:
      - dag-diamond-sf8vp-dxnjn-177087469
      - dag-diamond-sf8vp-dxnjn-160309850
      displayName: A
      finishedAt: "2020-10-02T02:14:29Z"
      hostNodeName: ip-10-0-2-189.us-west-2.compute.internal
      id: dag-diamond-sf8vp-dxnjn-126754612
      inputs:
        parameters:
        - name: message
          value: A
      name: dag-diamond-sf8vp-dxnjn.A
      outputs:
        exitCode: "0"
      phase: Succeeded
      resourcesDuration:
        cpu: 2
        memory: 2
      startedAt: "2020-10-02T02:14:26Z"
      templateName: echo
      templateScope: namespaced/dag-diamond-sf8vp
      type: Pod
    dag-diamond-sf8vp-dxnjn-160309850:
      boundaryID: dag-diamond-sf8vp-dxnjn
      children:
      - dag-diamond-sf8vp-dxnjn-76421755
      displayName: C
      finishedAt: "2020-10-02T02:14:34Z"
      hostNodeName: ip-10-0-2-189.us-west-2.compute.internal
      id: dag-diamond-sf8vp-dxnjn-160309850
      inputs:
        parameters:
        - name: message
          value: C
      name: dag-diamond-sf8vp-dxnjn.C
      outputs:
        exitCode: "0"
      phase: Succeeded
      resourcesDuration:
        cpu: 2
        memory: 2
      startedAt: "2020-10-02T02:14:31Z"
      templateName: echo
      templateScope: namespaced/dag-diamond-sf8vp
      type: Pod
    dag-diamond-sf8vp-dxnjn-177087469:
      boundaryID: dag-diamond-sf8vp-dxnjn
      children:
      - dag-diamond-sf8vp-dxnjn-76421755
      displayName: B
      finishedAt: "2020-10-02T02:14:34Z"
      hostNodeName: ip-10-0-2-189.us-west-2.compute.internal
      id: dag-diamond-sf8vp-dxnjn-177087469
      inputs:
        parameters:
        - name: message
          value: B
      name: dag-diamond-sf8vp-dxnjn.B
      outputs:
        exitCode: "0"
      phase: Succeeded
      resourcesDuration:
        cpu: 2
        memory: 2
      startedAt: "2020-10-02T02:14:31Z"
      templateName: echo
      templateScope: namespaced/dag-diamond-sf8vp
      type: Pod
  phase: Succeeded
  resourcesDuration:
    cpu: 9
    memory: 9
  startedAt: "2020-10-02T02:14:26Z"
  storedTemplates:
    namespaced/dag-diamond-sf8vp/diamond:
      arguments: {}
      dag:
        tasks:
        - arguments:
            parameters:
            - name: message
              value: A
          name: A
          template: echo
        - arguments:
            parameters:
            - name: message
              value: B
          dependencies:
          - A
          name: B
          template: echo
        - arguments:
            parameters:
            - name: message
              value: C
          dependencies:
          - A
          name: C
          template: echo
        - arguments:
            parameters:
            - name: message
              value: D
          dependencies:
          - B
          - C
          name: D
          template: echo
      inputs: {}
      metadata: {}
      name: diamond
      outputs: {}
    namespaced/dag-diamond-sf8vp/echo:
      arguments: {}
      container:
        command:
        - echo
        - '{{inputs.parameters.message}}'
        image: alpine:3.7
        name: ""
        resources: {}
      inputs:
        parameters:
        - name: message
      metadata: {}
      name: echo
      outputs: {}
  storedWorkflowTemplateSpec:
    arguments: {}
    entrypoint: diamond
    templates:
    - arguments: {}
      container:
        command:
        - echo
        - '{{inputs.parameters.message}}'
        image: alpine:3.7
        name: ""
        resources: {}
      inputs:
        parameters:
        - name: message
      metadata: {}
      name: echo
      outputs: {}
    - arguments: {}
      dag:
        tasks:
        - arguments:
            parameters:
            - name: message
              value: A
          name: A
          template: echo
        - arguments:
            parameters:
            - name: message
              value: B
          dependencies:
          - A
          name: B
          template: echo
        - arguments:
            parameters:
            - name: message
              value: C
          dependencies:
          - A
          name: C
          template: echo
        - arguments:
            parameters:
            - name: message
              value: D
          dependencies:
          - B
          - C
          name: D
          template: echo
      inputs: {}
      metadata: {}
      name: diamond
      outputs: {}
```

ttl honored but not serviceAccount:
```
Paste the logs from the workflow controller:
kubectl logs -n argo $(kubectl get pods -l app=workflow-controller -n argo -o name) | grep ${workflow}

time="2020-10-02T02:12:37Z" level=info msg="Processing workflow" namespace=test workflow=dag-diamond-fz54p
time="2020-10-02T02:12:37Z" level=info msg="Updated phase  -> Running" namespace=test workflow=dag-diamond-fz54p
time="2020-10-02T02:12:37Z" level=info msg="DAG node dag-diamond-fz54p initialized Running" namespace=test workflow=dag-diamond-fz54p
time="2020-10-02T02:12:37Z" level=info msg="All of node dag-diamond-fz54p.A dependencies [] completed" namespace=test workflow=dag-diamond-fz54p
time="2020-10-02T02:12:37Z" level=info msg="Pod node dag-diamond-fz54p-4207938145 initialized Pending" namespace=test workflow=dag-diamond-fz54p
time="2020-10-02T02:12:37Z" level=info msg="Created pod: dag-diamond-fz54p.A (dag-diamond-fz54p-4207938145)" namespace=test workflow=dag-diamond-fz54p
time="2020-10-02T02:12:37Z" level=info msg="Workflow update successful" namespace=test phase=Running resourceVersion=10005 workflow=dag-diamond-fz54p
time="2020-10-02T02:12:38Z" level=info msg="Processing workflow" namespace=test workflow=dag-diamond-fz54p
time="2020-10-02T02:12:38Z" level=info msg="Updating node dag-diamond-fz54p-4207938145 message: ContainerCreating"
time="2020-10-02T02:12:38Z" level=info msg="Skipped pod dag-diamond-fz54p.A (dag-diamond-fz54p-4207938145) creation: already exists" namespace=test podPhase=Pending workflow=dag-diamond-fz54p
time="2020-10-02T02:12:38Z" level=info msg="Workflow update successful" namespace=test phase=Running resourceVersion=10013 workflow=dag-diamond-fz54p
time="2020-10-02T02:12:39Z" level=info msg="Processing workflow" namespace=test workflow=dag-diamond-fz54p
time="2020-10-02T02:12:39Z" level=info msg="Skipped pod dag-diamond-fz54p.A (dag-diamond-fz54p-4207938145) creation: already exists" namespace=test podPhase=Pending workflow=dag-diamond-fz54p
time="2020-10-02T02:12:40Z" level=info msg="Processing workflow" namespace=test workflow=dag-diamond-fz54p
time="2020-10-02T02:12:40Z" level=info msg="Updating node dag-diamond-fz54p-4207938145 status Pending -> Running"
time="2020-10-02T02:12:40Z" level=info msg="Workflow update successful" namespace=test phase=Running resourceVersion=10026 workflow=dag-diamond-fz54p
time="2020-10-02T02:12:41Z" level=info msg="Processing workflow" namespace=test workflow=dag-diamond-fz54p
time="2020-10-02T02:12:47Z" level=info msg="Processing workflow" namespace=test workflow=dag-diamond-fz54p
time="2020-10-02T02:12:47Z" level=info msg="Updating node dag-diamond-fz54p-4207938145 status Running -> Error"
time="2020-10-02T02:12:47Z" level=info msg="Updating node dag-diamond-fz54p-4207938145 message: failed to save outputs: Failed to establish pod watch: timed out waiting for the condition"
time="2020-10-02T02:12:47Z" level=info msg="Skipped node dag-diamond-fz54p-4157605288 initialized Omitted (message: omitted: depends condition not met)" namespace=test workflow=dag-diamond-fz54p
time="2020-10-02T02:12:47Z" level=info msg="Skipped node dag-diamond-fz54p-4174382907 initialized Omitted (message: omitted: depends condition not met)" namespace=test workflow=dag-diamond-fz54p
time="2020-10-02T02:12:47Z" level=info msg="Skipped node dag-diamond-fz54p-4258271002 initialized Omitted (message: omitted: depends condition not met)" namespace=test workflow=dag-diamond-fz54p
time="2020-10-02T02:12:47Z" level=info msg="Outbound nodes of dag-diamond-fz54p set to [dag-diamond-fz54p-4258271002]" namespace=test workflow=dag-diamond-fz54p
time="2020-10-02T02:12:47Z" level=info msg="node dag-diamond-fz54p phase Running -> Error" namespace=test workflow=dag-diamond-fz54p
time="2020-10-02T02:12:47Z" level=info msg="node dag-diamond-fz54p finished: 2020-10-02 02:12:47.879673481 +0000 UTC" namespace=test workflow=dag-diamond-fz54p
time="2020-10-02T02:12:47Z" level=info msg="Checking daemoned children of dag-diamond-fz54p" namespace=test workflow=dag-diamond-fz54p
time="2020-10-02T02:12:47Z" level=info msg="Updated phase Running -> Error" namespace=test workflow=dag-diamond-fz54p
time="2020-10-02T02:12:47Z" level=info msg="Marking workflow completed" namespace=test workflow=dag-diamond-fz54p
time="2020-10-02T02:12:47Z" level=info msg="Checking daemoned children of " namespace=test workflow=dag-diamond-fz54p
time="2020-10-02T02:12:47Z" level=info msg="Workflow update successful" namespace=test phase=Error resourceVersion=10049 workflow=dag-diamond-fz54p
time="2020-10-02T02:12:48Z" level=info msg="Labeled pod completed" namespace=test pod=dag-diamond-fz54p-4207938145
```
serviceAccount honored but not ttl:
```
time="2020-10-02T02:12:37Z" level=info msg="Processing workflow" namespace=test workflow=dag-diamond-fz54p
time="2020-10-02T02:12:37Z" level=info msg="Updated phase  -> Running" namespace=test workflow=dag-diamond-fz54p
time="2020-10-02T02:12:37Z" level=info msg="DAG node dag-diamond-fz54p initialized Running" namespace=test workflow=dag-diamond-fz54p
time="2020-10-02T02:12:37Z" level=info msg="All of node dag-diamond-fz54p.A dependencies [] completed" namespace=test workflow=dag-diamond-fz54p
time="2020-10-02T02:12:37Z" level=info msg="Pod node dag-diamond-fz54p-4207938145 initialized Pending" namespace=test workflow=dag-diamond-fz54p
time="2020-10-02T02:12:37Z" level=info msg="Created pod: dag-diamond-fz54p.A (dag-diamond-fz54p-4207938145)" namespace=test workflow=dag-diamond-fz54p
time="2020-10-02T02:12:37Z" level=info msg="Workflow update successful" namespace=test phase=Running resourceVersion=10005 workflow=dag-diamond-fz54p
time="2020-10-02T02:12:38Z" level=info msg="Processing workflow" namespace=test workflow=dag-diamond-fz54p
time="2020-10-02T02:12:38Z" level=info msg="Updating node dag-diamond-fz54p-4207938145 message: ContainerCreating"
time="2020-10-02T02:12:38Z" level=info msg="Skipped pod dag-diamond-fz54p.A (dag-diamond-fz54p-4207938145) creation: already exists" namespace=test podPhase=Pending workflow=dag-diamond-fz54p
time="2020-10-02T02:12:38Z" level=info msg="Workflow update successful" namespace=test phase=Running resourceVersion=10013 workflow=dag-diamond-fz54p
time="2020-10-02T02:12:39Z" level=info msg="Processing workflow" namespace=test workflow=dag-diamond-fz54p
time="2020-10-02T02:12:39Z" level=info msg="Skipped pod dag-diamond-fz54p.A (dag-diamond-fz54p-4207938145) creation: already exists" namespace=test podPhase=Pending workflow=dag-diamond-fz54p
time="2020-10-02T02:12:40Z" level=info msg="Processing workflow" namespace=test workflow=dag-diamond-fz54p
time="2020-10-02T02:12:40Z" level=info msg="Updating node dag-diamond-fz54p-4207938145 status Pending -> Running"
time="2020-10-02T02:12:40Z" level=info msg="Workflow update successful" namespace=test phase=Running resourceVersion=10026 workflow=dag-diamond-fz54p
time="2020-10-02T02:12:41Z" level=info msg="Processing workflow" namespace=test workflow=dag-diamond-fz54p
time="2020-10-02T02:12:47Z" level=info msg="Processing workflow" namespace=test workflow=dag-diamond-fz54p
time="2020-10-02T02:12:47Z" level=info msg="Updating node dag-diamond-fz54p-4207938145 status Running -> Error"
time="2020-10-02T02:12:47Z" level=info msg="Updating node dag-diamond-fz54p-4207938145 message: failed to save outputs: Failed to establish pod watch: timed out waiting for the condition"
time="2020-10-02T02:12:47Z" level=info msg="Skipped node dag-diamond-fz54p-4157605288 initialized Omitted (message: omitted: depends condition not met)" namespace=test workflow=dag-diamond-fz54p
time="2020-10-02T02:12:47Z" level=info msg="Skipped node dag-diamond-fz54p-4174382907 initialized Omitted (message: omitted: depends condition not met)" namespace=test workflow=dag-diamond-fz54p
time="2020-10-02T02:12:47Z" level=info msg="Skipped node dag-diamond-fz54p-4258271002 initialized Omitted (message: omitted: depends condition not met)" namespace=test workflow=dag-diamond-fz54p
time="2020-10-02T02:12:47Z" level=info msg="Outbound nodes of dag-diamond-fz54p set to [dag-diamond-fz54p-4258271002]" namespace=test workflow=dag-diamond-fz54p
time="2020-10-02T02:12:47Z" level=info msg="node dag-diamond-fz54p phase Running -> Error" namespace=test workflow=dag-diamond-fz54p
time="2020-10-02T02:12:47Z" level=info msg="node dag-diamond-fz54p finished: 2020-10-02 02:12:47.879673481 +0000 UTC" namespace=test workflow=dag-diamond-fz54p
time="2020-10-02T02:12:47Z" level=info msg="Checking daemoned children of dag-diamond-fz54p" namespace=test workflow=dag-diamond-fz54p
time="2020-10-02T02:12:47Z" level=info msg="Updated phase Running -> Error" namespace=test workflow=dag-diamond-fz54p
time="2020-10-02T02:12:47Z" level=info msg="Marking workflow completed" namespace=test workflow=dag-diamond-fz54p
time="2020-10-02T02:12:47Z" level=info msg="Checking daemoned children of " namespace=test workflow=dag-diamond-fz54p
time="2020-10-02T02:12:47Z" level=info msg="Workflow update successful" namespace=test phase=Error resourceVersion=10049 workflow=dag-diamond-fz54p
time="2020-10-02T02:12:48Z" level=info msg="Labeled pod completed" namespace=test pod=dag-diamond-fz54p-4207938145
    ~  kubectl logs -n argo $(kubectl get pods -l app=workflow-controller -n argo -o name) | grep dag-diamond-sf8vp-dxnjn                      ✔  py3   example ⎈  20:16:28 
time="2020-10-02T02:14:26Z" level=info msg="Processing workflow" namespace=test workflow=dag-diamond-sf8vp-dxnjn
time="2020-10-02T02:14:26Z" level=info msg="Updated phase  -> Running" namespace=test workflow=dag-diamond-sf8vp-dxnjn
time="2020-10-02T02:14:26Z" level=info msg="DAG node dag-diamond-sf8vp-dxnjn initialized Running" namespace=test workflow=dag-diamond-sf8vp-dxnjn
time="2020-10-02T02:14:26Z" level=info msg="All of node dag-diamond-sf8vp-dxnjn.A dependencies [] completed" namespace=test workflow=dag-diamond-sf8vp-dxnjn
time="2020-10-02T02:14:26Z" level=info msg="Pod node dag-diamond-sf8vp-dxnjn-126754612 initialized Pending" namespace=test workflow=dag-diamond-sf8vp-dxnjn
time="2020-10-02T02:14:26Z" level=info msg="Created pod: dag-diamond-sf8vp-dxnjn.A (dag-diamond-sf8vp-dxnjn-126754612)" namespace=test workflow=dag-diamond-sf8vp-dxnjn
time="2020-10-02T02:14:26Z" level=info msg="Workflow update successful" namespace=test phase=Running resourceVersion=10351 workflow=dag-diamond-sf8vp-dxnjn
time="2020-10-02T02:14:27Z" level=info msg="Processing workflow" namespace=test workflow=dag-diamond-sf8vp-dxnjn
time="2020-10-02T02:14:27Z" level=info msg="Updating node dag-diamond-sf8vp-dxnjn-126754612 message: ContainerCreating"
time="2020-10-02T02:14:27Z" level=info msg="Skipped pod dag-diamond-sf8vp-dxnjn.A (dag-diamond-sf8vp-dxnjn-126754612) creation: already exists" namespace=test podPhase=Pending workflow=dag-diamond-sf8vp-dxnjn
time="2020-10-02T02:14:27Z" level=info msg="Workflow update successful" namespace=test phase=Running resourceVersion=10357 workflow=dag-diamond-sf8vp-dxnjn
time="2020-10-02T02:14:28Z" level=info msg="Processing workflow" namespace=test workflow=dag-diamond-sf8vp-dxnjn
time="2020-10-02T02:14:28Z" level=info msg="Skipped pod dag-diamond-sf8vp-dxnjn.A (dag-diamond-sf8vp-dxnjn-126754612) creation: already exists" namespace=test podPhase=Pending workflow=dag-diamond-sf8vp-dxnjn
time="2020-10-02T02:14:29Z" level=info msg="Processing workflow" namespace=test workflow=dag-diamond-sf8vp-dxnjn
time="2020-10-02T02:14:29Z" level=info msg="Updating node dag-diamond-sf8vp-dxnjn-126754612 status Pending -> Running"
time="2020-10-02T02:14:29Z" level=info msg="Workflow update successful" namespace=test phase=Running resourceVersion=10373 workflow=dag-diamond-sf8vp-dxnjn
time="2020-10-02T02:14:30Z" level=info msg="Processing workflow" namespace=test workflow=dag-diamond-sf8vp-dxnjn
time="2020-10-02T02:14:30Z" level=info msg="Setting node dag-diamond-sf8vp-dxnjn-126754612 outputs"
time="2020-10-02T02:14:30Z" level=info msg="Workflow update successful" namespace=test phase=Running resourceVersion=10378 workflow=dag-diamond-sf8vp-dxnjn
time="2020-10-02T02:14:31Z" level=info msg="Processing workflow" namespace=test workflow=dag-diamond-sf8vp-dxnjn
time="2020-10-02T02:14:31Z" level=info msg="Updating node dag-diamond-sf8vp-dxnjn-126754612 status Running -> Succeeded"
time="2020-10-02T02:14:31Z" level=info msg="All of node dag-diamond-sf8vp-dxnjn.B dependencies [A] completed" namespace=test workflow=dag-diamond-sf8vp-dxnjn
time="2020-10-02T02:14:31Z" level=info msg="Pod node dag-diamond-sf8vp-dxnjn-177087469 initialized Pending" namespace=test workflow=dag-diamond-sf8vp-dxnjn
time="2020-10-02T02:14:31Z" level=info msg="Created pod: dag-diamond-sf8vp-dxnjn.B (dag-diamond-sf8vp-dxnjn-177087469)" namespace=test workflow=dag-diamond-sf8vp-dxnjn
time="2020-10-02T02:14:31Z" level=info msg="All of node dag-diamond-sf8vp-dxnjn.C dependencies [A] completed" namespace=test workflow=dag-diamond-sf8vp-dxnjn
time="2020-10-02T02:14:31Z" level=info msg="Pod node dag-diamond-sf8vp-dxnjn-160309850 initialized Pending" namespace=test workflow=dag-diamond-sf8vp-dxnjn
time="2020-10-02T02:14:31Z" level=info msg="Created pod: dag-diamond-sf8vp-dxnjn.C (dag-diamond-sf8vp-dxnjn-160309850)" namespace=test workflow=dag-diamond-sf8vp-dxnjn
time="2020-10-02T02:14:31Z" level=info msg="Workflow update successful" namespace=test phase=Running resourceVersion=10389 workflow=dag-diamond-sf8vp-dxnjn
time="2020-10-02T02:14:32Z" level=info msg="Processing workflow" namespace=test workflow=dag-diamond-sf8vp-dxnjn
time="2020-10-02T02:14:32Z" level=info msg="Updating node dag-diamond-sf8vp-dxnjn-160309850 message: ContainerCreating"
time="2020-10-02T02:14:32Z" level=info msg="Updating node dag-diamond-sf8vp-dxnjn-177087469 message: ContainerCreating"
time="2020-10-02T02:14:32Z" level=info msg="Labeled pod completed" namespace=test pod=dag-diamond-sf8vp-dxnjn-126754612
time="2020-10-02T02:14:32Z" level=info msg="Skipped pod dag-diamond-sf8vp-dxnjn.B (dag-diamond-sf8vp-dxnjn-177087469) creation: already exists" namespace=test podPhase=Pending workflow=dag-diamond-sf8vp-dxnjn
time="2020-10-02T02:14:32Z" level=info msg="Skipped pod dag-diamond-sf8vp-dxnjn.C (dag-diamond-sf8vp-dxnjn-160309850) creation: already exists" namespace=test podPhase=Pending workflow=dag-diamond-sf8vp-dxnjn
time="2020-10-02T02:14:32Z" level=info msg="Workflow update successful" namespace=test phase=Running resourceVersion=10398 workflow=dag-diamond-sf8vp-dxnjn
time="2020-10-02T02:14:33Z" level=info msg="Processing workflow" namespace=test workflow=dag-diamond-sf8vp-dxnjn
time="2020-10-02T02:14:33Z" level=info msg="Labeled pod completed" namespace=test pod=dag-diamond-sf8vp-dxnjn-126754612
time="2020-10-02T02:14:33Z" level=info msg="Skipped pod dag-diamond-sf8vp-dxnjn.B (dag-diamond-sf8vp-dxnjn-177087469) creation: already exists" namespace=test podPhase=Pending workflow=dag-diamond-sf8vp-dxnjn
time="2020-10-02T02:14:33Z" level=info msg="Skipped pod dag-diamond-sf8vp-dxnjn.C (dag-diamond-sf8vp-dxnjn-160309850) creation: already exists" namespace=test podPhase=Pending workflow=dag-diamond-sf8vp-dxnjn
time="2020-10-02T02:14:34Z" level=info msg="Processing workflow" namespace=test workflow=dag-diamond-sf8vp-dxnjn
time="2020-10-02T02:14:34Z" level=info msg="Updating node dag-diamond-sf8vp-dxnjn-177087469 status Pending -> Running"
time="2020-10-02T02:14:34Z" level=info msg="Updating node dag-diamond-sf8vp-dxnjn-160309850 status Pending -> Running"
time="2020-10-02T02:14:34Z" level=info msg="Workflow update successful" namespace=test phase=Running resourceVersion=10416 workflow=dag-diamond-sf8vp-dxnjn
time="2020-10-02T02:14:35Z" level=info msg="Processing workflow" namespace=test workflow=dag-diamond-sf8vp-dxnjn
time="2020-10-02T02:14:35Z" level=info msg="Setting node dag-diamond-sf8vp-dxnjn-177087469 outputs"
time="2020-10-02T02:14:35Z" level=info msg="Setting node dag-diamond-sf8vp-dxnjn-160309850 outputs"
time="2020-10-02T02:14:35Z" level=info msg="Workflow update successful" namespace=test phase=Running resourceVersion=10426 workflow=dag-diamond-sf8vp-dxnjn
time="2020-10-02T02:14:36Z" level=info msg="Processing workflow" namespace=test workflow=dag-diamond-sf8vp-dxnjn
time="2020-10-02T02:14:36Z" level=info msg="Updating node dag-diamond-sf8vp-dxnjn-177087469 status Running -> Succeeded"
time="2020-10-02T02:14:36Z" level=info msg="Updating node dag-diamond-sf8vp-dxnjn-160309850 status Running -> Succeeded"
time="2020-10-02T02:14:36Z" level=info msg="All of node dag-diamond-sf8vp-dxnjn.D dependencies [B C] completed" namespace=test workflow=dag-diamond-sf8vp-dxnjn
time="2020-10-02T02:14:36Z" level=info msg="Pod node dag-diamond-sf8vp-dxnjn-76421755 initialized Pending" namespace=test workflow=dag-diamond-sf8vp-dxnjn
time="2020-10-02T02:14:36Z" level=info msg="Created pod: dag-diamond-sf8vp-dxnjn.D (dag-diamond-sf8vp-dxnjn-76421755)" namespace=test workflow=dag-diamond-sf8vp-dxnjn
time="2020-10-02T02:14:36Z" level=info msg="Workflow update successful" namespace=test phase=Running resourceVersion=10433 workflow=dag-diamond-sf8vp-dxnjn
time="2020-10-02T02:14:37Z" level=info msg="Processing workflow" namespace=test workflow=dag-diamond-sf8vp-dxnjn
time="2020-10-02T02:14:37Z" level=info msg="Labeled pod completed" namespace=test pod=dag-diamond-sf8vp-dxnjn-177087469
time="2020-10-02T02:14:37Z" level=info msg="Updating node dag-diamond-sf8vp-dxnjn-76421755 message: ContainerCreating"
time="2020-10-02T02:14:37Z" level=info msg="Skipped pod dag-diamond-sf8vp-dxnjn.D (dag-diamond-sf8vp-dxnjn-76421755) creation: already exists" namespace=test podPhase=Pending workflow=dag-diamond-sf8vp-dxnjn
time="2020-10-02T02:14:37Z" level=info msg="Labeled pod completed" namespace=test pod=dag-diamond-sf8vp-dxnjn-160309850
time="2020-10-02T02:14:37Z" level=info msg="Workflow update successful" namespace=test phase=Running resourceVersion=10444 workflow=dag-diamond-sf8vp-dxnjn
time="2020-10-02T02:14:38Z" level=info msg="Processing workflow" namespace=test workflow=dag-diamond-sf8vp-dxnjn
time="2020-10-02T02:14:38Z" level=info msg="Skipped pod dag-diamond-sf8vp-dxnjn.D (dag-diamond-sf8vp-dxnjn-76421755) creation: already exists" namespace=test podPhase=Pending workflow=dag-diamond-sf8vp-dxnjn
time="2020-10-02T02:14:38Z" level=info msg="Labeled pod completed" namespace=test pod=dag-diamond-sf8vp-dxnjn-160309850
time="2020-10-02T02:14:38Z" level=info msg="Labeled pod completed" namespace=test pod=dag-diamond-sf8vp-dxnjn-177087469
time="2020-10-02T02:14:39Z" level=info msg="Processing workflow" namespace=test workflow=dag-diamond-sf8vp-dxnjn
time="2020-10-02T02:14:39Z" level=info msg="Updating node dag-diamond-sf8vp-dxnjn-76421755 status Pending -> Running"
time="2020-10-02T02:14:39Z" level=info msg="Workflow update successful" namespace=test phase=Running resourceVersion=10456 workflow=dag-diamond-sf8vp-dxnjn
time="2020-10-02T02:14:40Z" level=info msg="Processing workflow" namespace=test workflow=dag-diamond-sf8vp-dxnjn
time="2020-10-02T02:14:40Z" level=info msg="Setting node dag-diamond-sf8vp-dxnjn-76421755 outputs"
time="2020-10-02T02:14:40Z" level=info msg="Updating node dag-diamond-sf8vp-dxnjn-76421755 status Running -> Succeeded"
time="2020-10-02T02:14:40Z" level=info msg="Outbound nodes of dag-diamond-sf8vp-dxnjn set to [dag-diamond-sf8vp-dxnjn-76421755]" namespace=test workflow=dag-diamond-sf8vp-dxnjn
time="2020-10-02T02:14:40Z" level=info msg="node dag-diamond-sf8vp-dxnjn phase Running -> Succeeded" namespace=test workflow=dag-diamond-sf8vp-dxnjn
time="2020-10-02T02:14:40Z" level=info msg="node dag-diamond-sf8vp-dxnjn finished: 2020-10-02 02:14:40.982669012 +0000 UTC" namespace=test workflow=dag-diamond-sf8vp-dxnjn
time="2020-10-02T02:14:40Z" level=info msg="Checking daemoned children of dag-diamond-sf8vp-dxnjn" namespace=test workflow=dag-diamond-sf8vp-dxnjn
time="2020-10-02T02:14:40Z" level=info msg="Updated phase Running -> Succeeded" namespace=test workflow=dag-diamond-sf8vp-dxnjn
time="2020-10-02T02:14:40Z" level=info msg="Marking workflow completed" namespace=test workflow=dag-diamond-sf8vp-dxnjn
time="2020-10-02T02:14:40Z" level=info msg="Checking daemoned children of " namespace=test workflow=dag-diamond-sf8vp-dxnjn
time="2020-10-02T02:14:40Z" level=info msg="Workflow update successful" namespace=test phase=Succeeded resourceVersion=10464 workflow=dag-diamond-sf8vp-dxnjn
time="2020-10-02T02:14:42Z" level=info msg="Labeled pod completed" namespace=test pod=dag-diamond-sf8vp-dxnjn-76421755
```


The example I used for this is and also turned it into a workflowTemplate
```
apiVersion: argoproj.io/v1alpha1
kind: Workflow
metadata:
  generateName: dag-diamond-
spec:
  entrypoint: diamond
  templates:
  - name: echo
    inputs:
      parameters:
      - name: message
    container:
      image: alpine:3.7
      command: [echo, "{{inputs.parameters.message}}"]
  - name: diamond
    dag:
      tasks:
      - name: A
        template: echo
        arguments:
          parameters: [{name: message, value: A}]
      - name: B
        dependencies: [A]
        template: echo
        arguments:
          parameters: [{name: message, value: B}]
      - name: C
        dependencies: [A]
        template: echo
        arguments:
          parameters: [{name: message, value: C}]
      - name: D
        dependencies: [B, C]
        template: echo
        arguments:
          parameters: [{name: message, value: D}]
```

---
<!-- Issue Author: Don't delete this message to encourage other users to support your issue! -->
**Message from the maintainers**:

Impacted by this bug? Give it a 👍. We prioritise the issues with the most 👍.


Pull request:
fix: Support workflowDefaults on TTLController for WorkflowTemplateRef Fixes #4188
Checklist:

* [ ] Either (a) I've created an [enhancement proposal](https://github.com/argoproj/argo/issues/new/choose) and discussed it with the community, (b) this is a bug fix, or (c) this is a chore.
* [ ] The title of the PR is (a) [conventional](https://www.conventionalcommits.org/en/v1.0.0/), (b) states what changed, and (c) suffixes the related issues number. E.g. `"fix(controller): Updates such and such. Fixes #1234"`.  
* [ ] My organization is added to [USERS.md](https://github.com/argoproj/argo/blob/master/USERS.md).
* [ ] I've signed the CLA.
* [ ] I have written unit and/or e2e tests for my change. PRs without these are unlikely to be merged.
* [ ] My builds are green. Try syncing with master if they are not. 

Fixes #4188 



Patch:
diff --git a/pkg/apis/workflow/v1alpha1/workflow_types.go b/pkg/apis/workflow/v1alpha1/workflow_types.go
index c9d77fc1e..b8af83835 100644
--- a/pkg/apis/workflow/v1alpha1/workflow_types.go
+++ b/pkg/apis/workflow/v1alpha1/workflow_types.go
@@ -144,6 +144,25 @@ func (w Workflows) Filter(predicate WorkflowPredicate) Workflows {
 	return out
 }
 
+// GetTTLStrategy return TTLStrategy based on Order of precedence:
+//1. Workflow, 2. WorkflowTemplate, 3. Workflowdefault
+func (w *Workflow) GetTTLStrategy(defaultTTLStrategy *TTLStrategy) *TTLStrategy {
+	var ttlStrategy *TTLStrategy
+	// TTLStrategy from Workflow default from Config
+	if defaultTTLStrategy != nil {
+		ttlStrategy = defaultTTLStrategy
+	}
+	// TTLStrategy from WorkflowTemplate
+	if w.Status.StoredWorkflowSpec != nil && w.Status.StoredWorkflowSpec.GetTTLStrategy() != nil {
+		ttlStrategy = w.Status.StoredWorkflowSpec.GetTTLStrategy()
+	}
+	//TTLStrategy from Workflow
+	if w.Spec.GetTTLStrategy() != nil {
+		ttlStrategy = w.Spec.GetTTLStrategy()
+	}
+	return ttlStrategy
+}
+
 var (
 	WorkflowCreatedAfter = func(t time.Time) WorkflowPredicate {
 		return func(wf Workflow) bool {
@@ -356,6 +375,18 @@ func (wfs WorkflowSpec) GetVolumeClaimGC() *VolumeClaimGC {
 	return wfs.VolumeClaimGC
 }
 
+func (wfs WorkflowSpec) GetTTLStrategy() *TTLStrategy {
+	if wfs.TTLSecondsAfterFinished != nil {
+		if wfs.TTLStrategy == nil {
+			ttlstrategy := TTLStrategy{SecondsAfterCompletion: wfs.TTLSecondsAfterFinished}
+			wfs.TTLStrategy = &ttlstrategy
+		} else if wfs.TTLStrategy.SecondsAfterCompletion == nil {
+			wfs.TTLStrategy.SecondsAfterCompletion = wfs.TTLSecondsAfterFinished
+		}
+	}
+	return wfs.TTLStrategy
+}
+
 type ShutdownStrategy string
 
 const (
diff --git a/workflow/controller/controller.go b/workflow/controller/controller.go
index 21e80fc6b..543ca9596 100644
--- a/workflow/controller/controller.go
+++ b/workflow/controller/controller.go
@@ -148,7 +148,9 @@ func NewWorkflowController(restConfig *rest.Config, kubeclientset kubernetes.Int
 
 // RunTTLController runs the workflow TTL controller
 func (wfc *WorkflowController) runTTLController(ctx context.Context) {
-	ttlCtrl := ttlcontroller.NewController(wfc.wfclientset, wfc.wfInformer)
+	ttlCtrl := ttlcontroller.NewController(wfc.wfclientset, wfc.wfInformer, func() *config.Config {
+		return &wfc.Config
+	})
 	err := ttlCtrl.Run(ctx.Done())
 	if err != nil {
 		panic(err)
diff --git a/workflow/ttlcontroller/ttlcontroller.go b/workflow/ttlcontroller/ttlcontroller.go
index 294b4bcc0..386415956 100644
--- a/workflow/ttlcontroller/ttlcontroller.go
+++ b/workflow/ttlcontroller/ttlcontroller.go
@@ -14,6 +14,7 @@ import (
 	"k8s.io/client-go/tools/cache"
 	"k8s.io/client-go/util/workqueue"
 
+	"github.com/argoproj/argo/config"
 	wfv1 "github.com/argoproj/argo/pkg/apis/workflow/v1alpha1"
 	wfclientset "github.com/argoproj/argo/pkg/client/clientset/versioned"
 	commonutil "github.com/argoproj/argo/util"
@@ -25,22 +26,26 @@ const (
 	workflowTTLResyncPeriod = 20 * time.Minute
 )
 
+type ConfigSupplier func() *config.Config
+
 type Controller struct {
-	wfclientset  wfclientset.Interface
-	wfInformer   cache.SharedIndexInformer
-	workqueue    workqueue.DelayingInterface
-	resyncPeriod time.Duration
-	clock        clock.Clock
+	wfclientset    wfclientset.Interface
+	wfInformer     cache.SharedIndexInformer
+	workqueue      workqueue.DelayingInterface
+	resyncPeriod   time.Duration
+	clock          clock.Clock
+	configSupplier ConfigSupplier
 }
 
 // NewController returns a new workflow ttl controller
-func NewController(wfClientset wfclientset.Interface, wfInformer cache.SharedIndexInformer) *Controller {
+func NewController(wfClientset wfclientset.Interface, wfInformer cache.SharedIndexInformer, configSupplier ConfigSupplier) *Controller {
 	controller := &Controller{
-		wfclientset:  wfClientset,
-		wfInformer:   wfInformer,
-		workqueue:    workqueue.NewDelayingQueue(),
-		resyncPeriod: workflowTTLResyncPeriod,
-		clock:        clock.RealClock{},
+		wfclientset:    wfClientset,
+		wfInformer:     wfInformer,
+		workqueue:      workqueue.NewDelayingQueue(),
+		resyncPeriod:   workflowTTLResyncPeriod,
+		clock:          clock.RealClock{},
+		configSupplier: configSupplier,
 	}
 
 	wfInformer.AddEventHandler(cache.FilteringResourceEventHandler{
@@ -132,7 +137,7 @@ func (c *Controller) enqueueWF(obj interface{}) {
 		return
 	}
 	now := c.clock.Now()
-	remaining, expiration := timeLeft(wf, &now)
+	remaining, expiration := timeLeft(wf, &now, c.getDefaultTTLStrategy())
 	if remaining == nil || *remaining > c.resyncPeriod {
 		return
 	}
@@ -187,9 +192,17 @@ func (c *Controller) deleteWorkflow(key string) error {
 	}
 	return nil
 }
+func (c *Controller) getDefaultTTLStrategy() *wfv1.TTLStrategy {
+	wfDefault := c.configSupplier().WorkflowDefaults
+	if wfDefault != nil {
+		return wfDefault.Spec.GetTTLStrategy()
+	}
+	return nil
+}
 
 func (c *Controller) ttlExpired(wf *wfv1.Workflow) bool {
-	ttlStrategy := getTTLStrategy(wf)
+	ttlStrategy := wf.GetTTLStrategy(c.getDefaultTTLStrategy())
+
 	// We don't care about the Workflows that are going to be deleted, or the ones that don't need clean up.
 	if wf.DeletionTimestamp != nil || ttlStrategy == nil || wf.Status.FinishedAt.IsZero() {
 		return false
@@ -208,8 +221,8 @@ func (c *Controller) ttlExpired(wf *wfv1.Workflow) bool {
 	}
 }
 
-func timeLeft(wf *wfv1.Workflow, since *time.Time) (*time.Duration, *time.Time) {
-	ttlStrategy := getTTLStrategy(wf)
+func timeLeft(wf *wfv1.Workflow, since *time.Time, defaultTTLStrategy *wfv1.TTLStrategy) (*time.Duration, *time.Time) {
+	ttlStrategy := wf.GetTTLStrategy(defaultTTLStrategy)
 	if wf.DeletionTimestamp != nil || ttlStrategy == nil || wf.Status.FinishedAt.IsZero() {
 		return nil, nil
 	}
@@ -235,12 +248,3 @@ func timeLeft(wf *wfv1.Workflow, since *time.Time) (*time.Duration, *time.Time)
 		return nil, nil
 	}
 }
-
-func getTTLStrategy(wf *wfv1.Workflow) *wfv1.TTLStrategy {
-	ttlStrategy := wf.Spec.TTLStrategy
-
-	if ttlStrategy == nil && wf.Status.StoredWorkflowSpec != nil && wf.Status.StoredWorkflowSpec.TTLStrategy != nil {
-		return wf.Status.StoredWorkflowSpec.TTLStrategy
-	}
-	return ttlStrategy
-}
diff --git a/workflow/util/util.go b/workflow/util/util.go
index 462089425..95a1fd74c 100644
--- a/workflow/util/util.go
+++ b/workflow/util/util.go
@@ -120,14 +120,6 @@ func NewWorkflowLister(informer cache.SharedIndexInformer) WorkflowLister {
 func FromUnstructured(un *unstructured.Unstructured) (*wfv1.Workflow, error) {
 	var wf wfv1.Workflow
 	err := runtime.DefaultUnstructuredConverter.FromUnstructured(un.Object, &wf)
-	if wf.Spec.TTLSecondsAfterFinished != nil {
-		if wf.Spec.TTLStrategy == nil {
-			ttlstrategy := wfv1.TTLStrategy{SecondsAfterCompletion: wf.Spec.TTLSecondsAfterFinished}
-			wf.Spec.TTLStrategy = &ttlstrategy
-		} else if wf.Spec.TTLStrategy.SecondsAfterCompletion == nil {
-			wf.Spec.TTLStrategy.SecondsAfterCompletion = wf.Spec.TTLSecondsAfterFinished
-		}
-	}
 	return &wf, err
 }
 


Test patch:
diff --git a/pkg/apis/workflow/v1alpha1/workflow_types_test.go b/pkg/apis/workflow/v1alpha1/workflow_types_test.go
index d192fc5d5..9bcdaeb85 100644
--- a/pkg/apis/workflow/v1alpha1/workflow_types_test.go
+++ b/pkg/apis/workflow/v1alpha1/workflow_types_test.go
@@ -222,3 +222,35 @@ func TestWorkflowSpec_GetVolumeGC(t *testing.T) {
 	assert.NotNil(t, spec.GetVolumeClaimGC())
 	assert.Equal(t, &VolumeClaimGC{Strategy: VolumeClaimGCOnSuccess}, spec.GetVolumeClaimGC())
 }
+
+func TestGetTTLStrategy(t *testing.T) {
+
+	spec := WorkflowSpec{TTLSecondsAfterFinished: pointer.Int32Ptr(10)}
+	ttl := spec.GetTTLStrategy()
+	assert.Equal(t, int32(10), *ttl.SecondsAfterCompletion)
+
+	spec = WorkflowSpec{TTLSecondsAfterFinished: pointer.Int32Ptr(10), TTLStrategy: &TTLStrategy{SecondsAfterCompletion: pointer.Int32Ptr(20)}}
+	ttl = spec.GetTTLStrategy()
+	assert.Equal(t, int32(20), *ttl.SecondsAfterCompletion)
+
+	spec = WorkflowSpec{TTLStrategy: &TTLStrategy{SecondsAfterCompletion: pointer.Int32Ptr(20)}}
+	ttl = spec.GetTTLStrategy()
+	assert.Equal(t, int32(20), *ttl.SecondsAfterCompletion)
+}
+
+func TestWfGetTTLStrategy(t *testing.T) {
+	ttl := TTLStrategy{
+		SecondsAfterCompletion: pointer.Int32Ptr(10),
+	}
+	wf := Workflow{}
+	result := wf.GetTTLStrategy(&ttl)
+	assert.Equal(t, int32(10), *result.SecondsAfterCompletion)
+
+	wf.Status.StoredWorkflowSpec = &WorkflowSpec{TTLStrategy: &TTLStrategy{SecondsAfterCompletion: pointer.Int32Ptr(20)}}
+	result = wf.GetTTLStrategy(&ttl)
+	assert.Equal(t, int32(20), *result.SecondsAfterCompletion)
+
+	wf.Spec.TTLStrategy = &TTLStrategy{SecondsAfterCompletion: pointer.Int32Ptr(30)}
+	result = wf.GetTTLStrategy(&ttl)
+	assert.Equal(t, int32(30), *result.SecondsAfterCompletion)
+}
diff --git a/workflow/ttlcontroller/ttlcontroller_test.go b/workflow/ttlcontroller/ttlcontroller_test.go
index 7951c97bc..a07e90f20 100644
--- a/workflow/ttlcontroller/ttlcontroller_test.go
+++ b/workflow/ttlcontroller/ttlcontroller_test.go
@@ -10,7 +10,9 @@ import (
 	"k8s.io/apimachinery/pkg/util/clock"
 	"k8s.io/client-go/tools/cache"
 	"k8s.io/client-go/util/workqueue"
+	"k8s.io/utils/pointer"
 
+	"github.com/argoproj/argo/config"
 	wfv1 "github.com/argoproj/argo/pkg/apis/workflow/v1alpha1"
 	fakewfclientset "github.com/argoproj/argo/pkg/client/clientset/versioned/fake"
 	"github.com/argoproj/argo/test"
@@ -338,6 +340,7 @@ status:
     ttlStrategy:
       secondsAfterCompletion: 60
 `
+var testConfig config.Config
 
 func newTTLController() *Controller {
 	clock := clock.NewFakeClock(time.Now())
@@ -349,6 +352,9 @@ func newTTLController() *Controller {
 		resyncPeriod: workflowTTLResyncPeriod,
 		clock:        clock,
 		workqueue:    workqueue.NewDelayingQueue(),
+		configSupplier: func() *config.Config {
+			return &testConfig
+		},
 	}
 }
 
@@ -599,29 +605,62 @@ func TestTTLlExpired(t *testing.T) {
 
 func TestGetTTLStrategy(t *testing.T) {
 	var ten int32 = 10
-	wf := test.LoadWorkflowFromBytes([]byte(succeededWf))
-	wf.Spec.TTLStrategy = &wfv1.TTLStrategy{
-		SecondsAfterCompletion: &ten,
+	var twenty int32 = 20
+	var thirty int32 = 30
+	defaultTTLStrategy := &wfv1.TTLStrategy{
+		SecondsAfterCompletion: &thirty,
 	}
+	t.Run("TTLFromWfDefaults", func(t *testing.T) {
+		wf := test.LoadWorkflowFromBytes([]byte(succeededWf))
+		ttl := wf.GetTTLStrategy(defaultTTLStrategy)
+		assert.NotNil(t, ttl)
+		assert.Equal(t, thirty, *ttl.SecondsAfterCompletion)
+	})
+
+	t.Run("TTLFromWorkflow", func(t *testing.T) {
+		wf := test.LoadWorkflowFromBytes([]byte(succeededWf))
+		wf.Spec.TTLStrategy = &wfv1.TTLStrategy{
+			SecondsAfterCompletion: &ten,
+		}
+		ttl := wf.GetTTLStrategy(defaultTTLStrategy)
+		assert.NotNil(t, ttl)
+		assert.Equal(t, ten, *ttl.SecondsAfterCompletion)
+	})
+
+	t.Run("TTLInWfwithWorkflowTemplate", func(t *testing.T) {
+		wf1 := test.LoadWorkflowFromBytes([]byte(wftRefWithTTLinWF))
+		ttl := wf1.GetTTLStrategy(defaultTTLStrategy)
+		assert.NotNil(t, ttl)
+		assert.Equal(t, ten, *ttl.SecondsAfterCompletion)
+
+		wf1.Spec.TTLStrategy = nil
+		wf1.Status.StoredWorkflowSpec.TTLStrategy = nil
+		ttl = wf1.GetTTLStrategy(nil)
+		assert.Nil(t, ttl)
+	})
+	t.Run("TTLwithWorkflowTemplate", func(t *testing.T) {
+		wf2 := test.LoadWorkflowFromBytes([]byte(wftRefWithTTLinWFT))
+		wf2.Spec.TTLSecondsAfterFinished = nil
+		wf2.Status.StoredWorkflowSpec.TTLSecondsAfterFinished = &twenty
+		ttl := wf2.GetTTLStrategy(defaultTTLStrategy)
+		assert.NotNil(t, ttl)
+		assert.Equal(t, ten, *ttl.SecondsAfterCompletion)
+		wf2.Status.StoredWorkflowSpec.TTLSecondsAfterFinished = nil
+		wf2.Status.StoredWorkflowSpec.TTLStrategy = nil
+		ttl = wf2.GetTTLStrategy(nil)
+		assert.Nil(t, ttl)
+	})
+}
 
-	ttl := getTTLStrategy(wf)
-	assert.NotNil(t, ttl)
-	assert.Equal(t, ten, *ttl.SecondsAfterCompletion)
-
-	wf1 := test.LoadWorkflowFromBytes([]byte(wftRefWithTTLinWF))
-	ttl = getTTLStrategy(wf1)
-	assert.NotNil(t, ttl)
-	assert.Equal(t, ten, *ttl.SecondsAfterCompletion)
-	wf1.Spec.TTLStrategy = nil
-	wf1.Status.StoredWorkflowSpec.TTLStrategy = nil
-	ttl = getTTLStrategy(wf1)
-	assert.Nil(t, ttl)
-
-	wf2 := test.LoadWorkflowFromBytes([]byte(wftRefWithTTLinWFT))
-	ttl = getTTLStrategy(wf2)
-	assert.NotNil(t, ttl)
-	assert.Equal(t, ten, *ttl.SecondsAfterCompletion)
-	wf2.Status.StoredWorkflowSpec.TTLStrategy = nil
-	ttl = getTTLStrategy(wf2)
+func TestGetDefaultTTLStrategy(t *testing.T) {
+	controller := newTTLController()
+	testConfig = config.Config{WorkflowDefaults: &wfv1.Workflow{Spec: wfv1.WorkflowSpec{TTLSecondsAfterFinished: pointer.Int32Ptr(10)}}}
+	ttl := controller.getDefaultTTLStrategy()
+	assert.Equal(t, int32(10), *ttl.SecondsAfterCompletion)
+	testConfig = config.Config{WorkflowDefaults: &wfv1.Workflow{Spec: wfv1.WorkflowSpec{TTLSecondsAfterFinished: pointer.Int32Ptr(20)}}}
+	ttl = controller.getDefaultTTLStrategy()
+	assert.Equal(t, int32(20), *ttl.SecondsAfterCompletion)
+	testConfig = config.Config{WorkflowDefaults: nil}
+	ttl = controller.getDefaultTTLStrategy()
 	assert.Nil(t, ttl)
 }