Repository: argoproj/argo (go)

Issue:
Pods in retried workflow with PVC after a step that succeeded never get scheduled
## Summary 

### What happened:

Given there is a workflow with a PersistentVolumeClaim mounted on two templates, and the workflow is run, and template A succeeds, and template B fails, when I retry the workflow, then template B's pod never gets scheduled on the retried workflow.

### What I expect to happen:

Template B is run.

## Diagnostics

What Kubernetes provider are you using? 

Docker

What version of Argo Workflows are you running? 

2.11.3

```yaml
apiVersion: argoproj.io/v1alpha1
kind: Workflow
metadata:
  creationTimestamp: "2020-10-14T13:22:54Z"
  generateName: test-pvc-retry-problem-
  generation: 12
  labels:
    workflows.argoproj.io/phase: Running
  name: test-pvc-retry-problem-6sq8m
  namespace: argo-next
  resourceVersion: "1450"
  selfLink: /apis/argoproj.io/v1alpha1/namespaces/argo-next/workflows/test-pvc-retry-problem-6sq8m
  uid: 704d7c20-dbd7-4538-9f9b-21cee76a500e
spec:
  arguments: {}
  entrypoint: run
  templates:
  - arguments: {}
    inputs: {}
    metadata: {}
    name: run
    outputs: {}
    steps:
    - - arguments: {}
        name: first
        template: succeed
    - - arguments: {}
        name: second
        template: fail-on-purpose
  - arguments: {}
    inputs: {}
    metadata: {}
    name: fail-on-purpose
    outputs: {}
    script:
      command:
      - sh
      image: alpine
      name: ""
      resources: {}
      source: |
        echo "I am failing intentionally"
        exit 1
      volumeMounts:
      - mountPath: /data
        name: data
  - arguments: {}
    inputs: {}
    metadata: {}
    name: succeed
    outputs: {}
    script:
      command:
      - sh
      image: alpine
      name: ""
      resources: {}
      source: |
        echo "I am successful"
      volumeMounts:
      - mountPath: /data
        name: data
  ttlSecondsAfterFinished: 600
  volumeClaimTemplates:
  - metadata:
      creationTimestamp: null
      name: data
    spec:
      accessModes:
      - ReadWriteOnce
      resources:
        requests:
          storage: 5Gi
    status: {}
status:
  conditions:
  - status: "False"
    type: Completed
  finishedAt: null
  nodes:
    test-pvc-retry-problem-6sq8m:
      children:
      - test-pvc-retry-problem-6sq8m-2082576660
      displayName: test-pvc-retry-problem-6sq8m
      finishedAt: null
      id: test-pvc-retry-problem-6sq8m
      name: test-pvc-retry-problem-6sq8m
      phase: Running
      startedAt: "2020-10-14T13:22:54Z"
      templateName: run
      templateScope: local/test-pvc-retry-problem-6sq8m
      type: Steps
    test-pvc-retry-problem-6sq8m-928724940:
      boundaryID: test-pvc-retry-problem-6sq8m
      children:
      - test-pvc-retry-problem-6sq8m-3223307657
      displayName: first
      finishedAt: "2020-10-14T13:22:58Z"
      hostNodeName: minikube
      id: test-pvc-retry-problem-6sq8m-928724940
      name: test-pvc-retry-problem-6sq8m[0].first
      outputs:
        artifacts:
        - archiveLogs: true
          name: main-logs
          s3:
            accessKeySecret:
              key: accesskey
              name: argo-minio
            bucket: argo-artifacts
            endpoint: argo-minio:9000
            insecure: true
            key: test-pvc-retry-problem-6sq8m/test-pvc-retry-problem-6sq8m-928724940/main.log
            secretKeySecret:
              key: secretkey
              name: argo-minio
        exitCode: "0"
      phase: Succeeded
      resourcesDuration:
        cpu: 1
        memory: 1
      startedAt: "2020-10-14T13:22:54Z"
      templateName: succeed
      templateScope: local/test-pvc-retry-problem-6sq8m
      type: Pod
    test-pvc-retry-problem-6sq8m-1089601533:
      boundaryID: test-pvc-retry-problem-6sq8m
      displayName: second
      finishedAt: null
      id: test-pvc-retry-problem-6sq8m-1089601533
      message: 'Unschedulable: persistentvolumeclaim "test-pvc-retry-problem-6sq8m-data"
        is being deleted'
      name: test-pvc-retry-problem-6sq8m[1].second
      phase: Pending
      startedAt: "2020-10-14T13:23:06Z"
      templateName: fail-on-purpose
      templateScope: local/test-pvc-retry-problem-6sq8m
      type: Pod
    test-pvc-retry-problem-6sq8m-2082576660:
      boundaryID: test-pvc-retry-problem-6sq8m
      children:
      - test-pvc-retry-problem-6sq8m-928724940
      displayName: '[0]'
      finishedAt: "2020-10-14T13:23:00Z"
      id: test-pvc-retry-problem-6sq8m-2082576660
      name: test-pvc-retry-problem-6sq8m[0]
      phase: Succeeded
      startedAt: "2020-10-14T13:22:54Z"
      templateName: run
      templateScope: local/test-pvc-retry-problem-6sq8m
      type: StepGroup
    test-pvc-retry-problem-6sq8m-3223307657:
      boundaryID: test-pvc-retry-problem-6sq8m
      children:
      - test-pvc-retry-problem-6sq8m-1089601533
      displayName: '[1]'
      finishedAt: null
      id: test-pvc-retry-problem-6sq8m-3223307657
      name: test-pvc-retry-problem-6sq8m[1]
      phase: Running
      startedAt: "2020-10-14T13:23:00Z"
      templateName: run
      templateScope: local/test-pvc-retry-problem-6sq8m
      type: StepGroup
  persistentVolumeClaims:
  - name: data
    persistentVolumeClaim:
      claimName: test-pvc-retry-problem-6sq8m-data
  phase: Running
  resourcesDuration:
    cpu: 2
    memory: 2
  startedAt: "2020-10-14T13:22:54Z"
```


```bash
$ kubectl logs -nargo-next $(kubectl get pods -l app=argo-workflow-controller -n argo-next -o name) | grep test-pvc-retry-problem-6sq8m
time="2020-10-14T13:22:54Z" level=info msg="Processing workflow" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:22:54Z" level=info msg="Updated phase  -> Running" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:22:54Z" level=info msg="Creating pvc test-pvc-retry-problem-6sq8m-data" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:22:54Z" level=info msg="Steps node test-pvc-retry-problem-6sq8m initialized Running" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:22:54Z" level=info msg="StepGroup node test-pvc-retry-problem-6sq8m-2082576660 initialized Running" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:22:54Z" level=info msg="Pod node test-pvc-retry-problem-6sq8m-928724940 initialized Pending" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:22:54Z" level=info msg="Created pod: test-pvc-retry-problem-6sq8m[0].first (test-pvc-retry-problem-6sq8m-928724940)" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:22:54Z" level=info msg="Workflow step group node test-pvc-retry-problem-6sq8m-2082576660 not yet completed" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:22:54Z" level=info msg="Workflow update successful" namespace=argo-next phase=Running resourceVersion=1370 workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:22:55Z" level=info msg="Processing workflow" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:22:55Z" level=info msg="Updating node test-pvc-retry-problem-6sq8m-928724940 message: Unschedulable: pod has unbound immediate PersistentVolumeClaims"
time="2020-10-14T13:22:55Z" level=info msg="Skipped pod test-pvc-retry-problem-6sq8m[0].first (test-pvc-retry-problem-6sq8m-928724940) creation: already exists" namespace=argo-next podPhase=Pending workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:22:55Z" level=info msg="Workflow step group node test-pvc-retry-problem-6sq8m-2082576660 not yet completed" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:22:55Z" level=info msg="Workflow update successful" namespace=argo-next phase=Running resourceVersion=1381 workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:22:56Z" level=info msg="Processing workflow" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:22:56Z" level=info msg="Updating node test-pvc-retry-problem-6sq8m-928724940 message: PodInitializing"
time="2020-10-14T13:22:56Z" level=info msg="Skipped pod test-pvc-retry-problem-6sq8m[0].first (test-pvc-retry-problem-6sq8m-928724940) creation: already exists" namespace=argo-next podPhase=Pending workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:22:56Z" level=info msg="Workflow step group node test-pvc-retry-problem-6sq8m-2082576660 not yet completed" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:22:56Z" level=info msg="Workflow update successful" namespace=argo-next phase=Running resourceVersion=1390 workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:22:57Z" level=info msg="Processing workflow" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:22:57Z" level=info msg="Skipped pod test-pvc-retry-problem-6sq8m[0].first (test-pvc-retry-problem-6sq8m-928724940) creation: already exists" namespace=argo-next podPhase=Pending workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:22:57Z" level=info msg="Workflow step group node test-pvc-retry-problem-6sq8m-2082576660 not yet completed" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:22:58Z" level=info msg="Processing workflow" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:22:58Z" level=info msg="Skipped pod test-pvc-retry-problem-6sq8m[0].first (test-pvc-retry-problem-6sq8m-928724940) creation: already exists" namespace=argo-next podPhase=Pending workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:22:58Z" level=info msg="Workflow step group node test-pvc-retry-problem-6sq8m-2082576660 not yet completed" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:22:59Z" level=info msg="Processing workflow" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:22:59Z" level=info msg="Setting node test-pvc-retry-problem-6sq8m-928724940 outputs"
time="2020-10-14T13:22:59Z" level=info msg="Updating node test-pvc-retry-problem-6sq8m-928724940 status Pending -> Running"
time="2020-10-14T13:22:59Z" level=info msg="Workflow step group node test-pvc-retry-problem-6sq8m-2082576660 not yet completed" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:22:59Z" level=info msg="Workflow update successful" namespace=argo-next phase=Running resourceVersion=1402 workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:00Z" level=info msg="Processing workflow" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:00Z" level=info msg="Updating node test-pvc-retry-problem-6sq8m-928724940 status Running -> Succeeded"
time="2020-10-14T13:23:00Z" level=info msg="Step group node &NodeStatus{ID:test-pvc-retry-problem-6sq8m-2082576660,Name:test-pvc-retry-problem-6sq8m[0],DisplayName:[0],Type:StepGroup,TemplateName:run,TemplateRef:nil,Phase:Running,Boundary
ID:test-pvc-retry-problem-6sq8m,Message:,StartedAt:2020-10-14 13:22:54 +0000 UTC,FinishedAt:0001-01-01 00:00:00 +0000 UTC,PodIP:,Daemoned:nil,Inputs:nil,Outputs:nil,Children:[test-pvc-retry-problem-6sq8m-928724940],OutboundNodes:[],Stored
TemplateID:,WorkflowTemplateName:,TemplateScope:local/test-pvc-retry-problem-6sq8m,ResourcesDuration:ResourcesDuration{},HostNodeName:,MemoizationStatus:nil,SynchronizationStatus:nil,} successful" namespace=argo-next workflow=test-pvc-ret
ry-problem-6sq8m
time="2020-10-14T13:23:00Z" level=info msg="node test-pvc-retry-problem-6sq8m-2082576660 phase Running -> Succeeded" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:00Z" level=info msg="node test-pvc-retry-problem-6sq8m-2082576660 finished: 2020-10-14 13:23:00.256965897 +0000 UTC" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:00Z" level=info msg="StepGroup node test-pvc-retry-problem-6sq8m-3223307657 initialized Running" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:00Z" level=info msg="SG Outbound nodes of test-pvc-retry-problem-6sq8m-928724940 are [test-pvc-retry-problem-6sq8m-928724940]" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:00Z" level=info msg="Pod node test-pvc-retry-problem-6sq8m-1089601533 initialized Pending" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:00Z" level=info msg="Created pod: test-pvc-retry-problem-6sq8m[1].second (test-pvc-retry-problem-6sq8m-1089601533)" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:00Z" level=info msg="Workflow step group node test-pvc-retry-problem-6sq8m-3223307657 not yet completed" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:00Z" level=info msg="Workflow update successful" namespace=argo-next phase=Running resourceVersion=1409 workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:01Z" level=info msg="Processing workflow" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:01Z" level=info msg="Labeled pod completed" namespace=argo-next pod=test-pvc-retry-problem-6sq8m-928724940
time="2020-10-14T13:23:01Z" level=info msg="Updating node test-pvc-retry-problem-6sq8m-1089601533 message: PodInitializing"
time="2020-10-14T13:23:01Z" level=info msg="node test-pvc-retry-problem-6sq8m-2082576660 phase Succeeded -> Running" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:01Z" level=info msg="Step group node &NodeStatus{ID:test-pvc-retry-problem-6sq8m-2082576660,Name:test-pvc-retry-problem-6sq8m[0],DisplayName:[0],Type:StepGroup,TemplateName:run,TemplateRef:nil,Phase:Running,Boundary
ID:test-pvc-retry-problem-6sq8m,Message:,StartedAt:2020-10-14 13:22:54 +0000 UTC,FinishedAt:2020-10-14 13:23:00 +0000 UTC,PodIP:,Daemoned:nil,Inputs:nil,Outputs:nil,Children:[test-pvc-retry-problem-6sq8m-928724940],OutboundNodes:[],Stored
TemplateID:,WorkflowTemplateName:,TemplateScope:local/test-pvc-retry-problem-6sq8m,ResourcesDuration:ResourcesDuration{},HostNodeName:,MemoizationStatus:nil,SynchronizationStatus:nil,} successful" namespace=argo-next workflow=test-pvc-ret
ry-problem-6sq8m
time="2020-10-14T13:23:01Z" level=info msg="node test-pvc-retry-problem-6sq8m-2082576660 phase Running -> Succeeded" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:01Z" level=info msg="SG Outbound nodes of test-pvc-retry-problem-6sq8m-928724940 are [test-pvc-retry-problem-6sq8m-928724940]" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:01Z" level=info msg="Skipped pod test-pvc-retry-problem-6sq8m[1].second (test-pvc-retry-problem-6sq8m-1089601533) creation: already exists" namespace=argo-next podPhase=Pending workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:01Z" level=info msg="Workflow step group node test-pvc-retry-problem-6sq8m-3223307657 not yet completed" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:01Z" level=info msg="Workflow update successful" namespace=argo-next phase=Running resourceVersion=1417 workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:02Z" level=info msg="Processing workflow" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:02Z" level=info msg="node test-pvc-retry-problem-6sq8m-2082576660 phase Succeeded -> Running" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:02Z" level=info msg="Step group node &NodeStatus{ID:test-pvc-retry-problem-6sq8m-2082576660,Name:test-pvc-retry-problem-6sq8m[0],DisplayName:[0],Type:StepGroup,TemplateName:run,TemplateRef:nil,Phase:Running,Boundary
ID:test-pvc-retry-problem-6sq8m,Message:,StartedAt:2020-10-14 13:22:54 +0000 UTC,FinishedAt:2020-10-14 13:23:00 +0000 UTC,PodIP:,Daemoned:nil,Inputs:nil,Outputs:nil,Children:[test-pvc-retry-problem-6sq8m-928724940],OutboundNodes:[],Stored
TemplateID:,WorkflowTemplateName:,TemplateScope:local/test-pvc-retry-problem-6sq8m,ResourcesDuration:ResourcesDuration{},HostNodeName:,MemoizationStatus:nil,SynchronizationStatus:nil,} successful" namespace=argo-next workflow=test-pvc-ret
ry-problem-6sq8m
time="2020-10-14T13:23:02Z" level=info msg="node test-pvc-retry-problem-6sq8m-2082576660 phase Running -> Succeeded" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:02Z" level=info msg="SG Outbound nodes of test-pvc-retry-problem-6sq8m-928724940 are [test-pvc-retry-problem-6sq8m-928724940]" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:02Z" level=info msg="Skipped pod test-pvc-retry-problem-6sq8m[1].second (test-pvc-retry-problem-6sq8m-1089601533) creation: already exists" namespace=argo-next podPhase=Pending workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:02Z" level=info msg="Workflow step group node test-pvc-retry-problem-6sq8m-3223307657 not yet completed" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:02Z" level=info msg="Workflow update successful" namespace=argo-next phase=Running resourceVersion=1417 workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:03Z" level=info msg="Processing workflow" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:03Z" level=info msg="Setting node test-pvc-retry-problem-6sq8m-1089601533 outputs"
time="2020-10-14T13:23:03Z" level=info msg="Updating node test-pvc-retry-problem-6sq8m-1089601533 status Pending -> Running"
time="2020-10-14T13:23:03Z" level=info msg="node test-pvc-retry-problem-6sq8m-2082576660 phase Succeeded -> Running" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:03Z" level=info msg="Step group node &NodeStatus{ID:test-pvc-retry-problem-6sq8m-2082576660,Name:test-pvc-retry-problem-6sq8m[0],DisplayName:[0],Type:StepGroup,TemplateName:run,TemplateRef:nil,Phase:Running,Boundary
ID:test-pvc-retry-problem-6sq8m,Message:,StartedAt:2020-10-14 13:22:54 +0000 UTC,FinishedAt:2020-10-14 13:23:00 +0000 UTC,PodIP:,Daemoned:nil,Inputs:nil,Outputs:nil,Children:[test-pvc-retry-problem-6sq8m-928724940],OutboundNodes:[],Stored
TemplateID:,WorkflowTemplateName:,TemplateScope:local/test-pvc-retry-problem-6sq8m,ResourcesDuration:ResourcesDuration{},HostNodeName:,MemoizationStatus:nil,SynchronizationStatus:nil,} successful" namespace=argo-next workflow=test-pvc-ret
ry-problem-6sq8m
time="2020-10-14T13:23:03Z" level=info msg="node test-pvc-retry-problem-6sq8m-2082576660 phase Running -> Succeeded" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:03Z" level=info msg="SG Outbound nodes of test-pvc-retry-problem-6sq8m-928724940 are [test-pvc-retry-problem-6sq8m-928724940]" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:03Z" level=info msg="Workflow step group node test-pvc-retry-problem-6sq8m-3223307657 not yet completed" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:03Z" level=info msg="Workflow update successful" namespace=argo-next phase=Running resourceVersion=1430 workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:04Z" level=info msg="Processing workflow" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:04Z" level=info msg="Pod failed" displayName=second namespace=argo-next pod=test-pvc-retry-problem-6sq8m-1089601533 templateName=fail-on-purpose workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:04Z" level=info msg="Updating node test-pvc-retry-problem-6sq8m-1089601533 status Running -> Failed"
time="2020-10-14T13:23:04Z" level=info msg="Updating node test-pvc-retry-problem-6sq8m-1089601533 message: failed with exit code 1"
time="2020-10-14T13:23:04Z" level=info msg="node test-pvc-retry-problem-6sq8m-2082576660 phase Succeeded -> Running" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:04Z" level=info msg="Step group node &NodeStatus{ID:test-pvc-retry-problem-6sq8m-2082576660,Name:test-pvc-retry-problem-6sq8m[0],DisplayName:[0],Type:StepGroup,TemplateName:run,TemplateRef:nil,Phase:Running,Boundary
ID:test-pvc-retry-problem-6sq8m,Message:,StartedAt:2020-10-14 13:22:54 +0000 UTC,FinishedAt:2020-10-14 13:23:00 +0000 UTC,PodIP:,Daemoned:nil,Inputs:nil,Outputs:nil,Children:[test-pvc-retry-problem-6sq8m-928724940],OutboundNodes:[],Stored
TemplateID:,WorkflowTemplateName:,TemplateScope:local/test-pvc-retry-problem-6sq8m,ResourcesDuration:ResourcesDuration{},HostNodeName:,MemoizationStatus:nil,SynchronizationStatus:nil,} successful" namespace=argo-next workflow=test-pvc-ret
ry-problem-6sq8m
time="2020-10-14T13:23:04Z" level=info msg="node test-pvc-retry-problem-6sq8m-2082576660 phase Running -> Succeeded" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:04Z" level=info msg="SG Outbound nodes of test-pvc-retry-problem-6sq8m-928724940 are [test-pvc-retry-problem-6sq8m-928724940]" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:04Z" level=info msg="Step group node test-pvc-retry-problem-6sq8m-3223307657 deemed failed: child 'test-pvc-retry-problem-6sq8m-1089601533' failed" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:04Z" level=info msg="node test-pvc-retry-problem-6sq8m-3223307657 phase Running -> Failed" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:04Z" level=info msg="node test-pvc-retry-problem-6sq8m-3223307657 message: child 'test-pvc-retry-problem-6sq8m-1089601533' failed" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:04Z" level=info msg="node test-pvc-retry-problem-6sq8m-3223307657 finished: 2020-10-14 13:23:04.333943157 +0000 UTC" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:04Z" level=info msg="step group test-pvc-retry-problem-6sq8m-3223307657 was unsuccessful: child 'test-pvc-retry-problem-6sq8m-1089601533' failed" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:04Z" level=info msg="Outbound nodes of test-pvc-retry-problem-6sq8m-1089601533 is [test-pvc-retry-problem-6sq8m-1089601533]" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:04Z" level=info msg="Outbound nodes of test-pvc-retry-problem-6sq8m is [test-pvc-retry-problem-6sq8m-1089601533]" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:04Z" level=info msg="node test-pvc-retry-problem-6sq8m phase Running -> Failed" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:04Z" level=info msg="node test-pvc-retry-problem-6sq8m message: child 'test-pvc-retry-problem-6sq8m-1089601533' failed" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:04Z" level=info msg="node test-pvc-retry-problem-6sq8m finished: 2020-10-14 13:23:04.334261487 +0000 UTC" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:04Z" level=info msg="Checking daemoned children of test-pvc-retry-problem-6sq8m" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:04Z" level=info msg="Deleting PVC test-pvc-retry-problem-6sq8m-data" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:04Z" level=info msg="Deleted 1/1 PVCs" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:04Z" level=info msg="Updated phase Running -> Failed" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:04Z" level=info msg="Updated message  -> child 'test-pvc-retry-problem-6sq8m-1089601533' failed" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:04Z" level=info msg="Marking workflow completed" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:04Z" level=info msg="Checking daemoned children of " namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:04Z" level=info msg="Found Workflow argo-next/test-pvc-retry-problem-6sq8m set expire at 2020-10-14 13:33:04 +0000 UTC (9m59.628630617s from now)"
time="2020-10-14T13:23:04Z" level=info msg="Queueing workflow argo-next/test-pvc-retry-problem-6sq8m for delete in 9m59.628630617s"
time="2020-10-14T13:23:04Z" level=info msg="Found Workflow argo-next/test-pvc-retry-problem-6sq8m set expire at 2020-10-14 13:33:04 +0000 UTC (9m59.627813513s from now)"
time="2020-10-14T13:23:04Z" level=info msg="Queueing workflow argo-next/test-pvc-retry-problem-6sq8m for delete in 9m59.627813513s"
time="2020-10-14T13:23:04Z" level=info msg="Workflow update successful" namespace=argo-next phase=Failed resourceVersion=1436 workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:05Z" level=info msg="Labeled pod completed" namespace=argo-next pod=test-pvc-retry-problem-6sq8m-1089601533
time="2020-10-14T13:23:06Z" level=info msg="Found Workflow argo-next/test-pvc-retry-problem-6sq8m set expire at 2020-10-14 13:33:04 +0000 UTC (9m57.045543148s from now)"
time="2020-10-14T13:23:06Z" level=info msg="Queueing workflow argo-next/test-pvc-retry-problem-6sq8m for delete in 9m57.045543148s"
time="2020-10-14T13:23:06Z" level=info msg="Processing workflow" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:06Z" level=info msg="Creating pvc test-pvc-retry-problem-6sq8m-data" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:06Z" level=info msg="pvc already exists. Workflow is re-using it" namespace=argo-next pvc=test-pvc-retry-problem-6sq8m-data workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:06Z" level=info msg="node test-pvc-retry-problem-6sq8m-2082576660 phase Succeeded -> Running" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:06Z" level=info msg="Step group node &NodeStatus{ID:test-pvc-retry-problem-6sq8m-2082576660,Name:test-pvc-retry-problem-6sq8m[0],DisplayName:[0],Type:StepGroup,TemplateName:run,TemplateRef:nil,Phase:Running,BoundaryID:test-pvc-retry-problem-6sq8m,Message:,StartedAt:2020-10-14 13:22:54 +0000 UTC,FinishedAt:2020-10-14 13:23:00 +0000 UTC,PodIP:,Daemoned:nil,Inputs:nil,Outputs:nil,Children:[test-pvc-retry-problem-6sq8m-928724940],OutboundNodes:[],StoredTemplateID:,WorkflowTemplateName:,TemplateScope:local/test-pvc-retry-problem-6sq8m,ResourcesDuration:ResourcesDuration{},HostNodeName:,MemoizationStatus:nil,SynchronizationStatus:nil,} successful" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:06Z" level=info msg="node test-pvc-retry-problem-6sq8m-2082576660 phase Running -> Succeeded" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:06Z" level=info msg="SG Outbound nodes of test-pvc-retry-problem-6sq8m-928724940 are [test-pvc-retry-problem-6sq8m-928724940]" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:06Z" level=info msg="Pod node test-pvc-retry-problem-6sq8m-1089601533 initialized Pending" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:06Z" level=info msg="Created pod: test-pvc-retry-problem-6sq8m[1].second (test-pvc-retry-problem-6sq8m-1089601533)" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:06Z" level=info msg="Workflow step group node test-pvc-retry-problem-6sq8m-3223307657 not yet completed" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:06Z" level=info msg="Workflow update successful" namespace=argo-next phase=Running resourceVersion=1448 workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:07Z" level=info msg="Processing workflow" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:07Z" level=info msg="Updating node test-pvc-retry-problem-6sq8m-1089601533 message: Unschedulable: persistentvolumeclaim \"test-pvc-retry-problem-6sq8m-data\" is being deleted"
time="2020-10-14T13:23:07Z" level=info msg="node test-pvc-retry-problem-6sq8m-2082576660 phase Succeeded -> Running" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:07Z" level=info msg="Step group node &NodeStatus{ID:test-pvc-retry-problem-6sq8m-2082576660,Name:test-pvc-retry-problem-6sq8m[0],DisplayName:[0],Type:StepGroup,TemplateName:run,TemplateRef:nil,Phase:Running,BoundaryID:test-pvc-retry-problem-6sq8m,Message:,StartedAt:2020-10-14 13:22:54 +0000 UTC,FinishedAt:2020-10-14 13:23:00 +0000 UTC,PodIP:,Daemoned:nil,Inputs:nil,Outputs:nil,Children:[test-pvc-retry-problem-6sq8m-928724940],OutboundNodes:[],StoredTemplateID:,WorkflowTemplateName:,TemplateScope:local/test-pvc-retry-problem-6sq8m,ResourcesDuration:ResourcesDuration{},HostNodeName:,MemoizationStatus:nil,SynchronizationStatus:nil,} successful" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:07Z" level=info msg="node test-pvc-retry-problem-6sq8m-2082576660 phase Running -> Succeeded" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:07Z" level=info msg="SG Outbound nodes of test-pvc-retry-problem-6sq8m-928724940 are [test-pvc-retry-problem-6sq8m-928724940]" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:07Z" level=info msg="Skipped pod test-pvc-retry-problem-6sq8m[1].second (test-pvc-retry-problem-6sq8m-1089601533) creation: already exists" namespace=argo-next podPhase=Pending workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:07Z" level=info msg="Workflow step group node test-pvc-retry-problem-6sq8m-3223307657 not yet completed" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:07Z" level=info msg="Workflow update successful" namespace=argo-next phase=Running resourceVersion=1450 workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:08Z" level=info msg="Processing workflow" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:08Z" level=info msg="node test-pvc-retry-problem-6sq8m-2082576660 phase Succeeded -> Running" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:08Z" level=info msg="Step group node &NodeStatus{ID:test-pvc-retry-problem-6sq8m-2082576660,Name:test-pvc-retry-problem-6sq8m[0],DisplayName:[0],Type:StepGroup,TemplateName:run,TemplateRef:nil,Phase:Running,BoundaryID:test-pvc-retry-problem-6sq8m,Message:,StartedAt:2020-10-14 13:22:54 +0000 UTC,FinishedAt:2020-10-14 13:23:00 +0000 UTC,PodIP:,Daemoned:nil,Inputs:nil,Outputs:nil,Children:[test-pvc-retry-problem-6sq8m-928724940],OutboundNodes:[],StoredTemplateID:,WorkflowTemplateName:,TemplateScope:local/test-pvc-retry-problem-6sq8m,ResourcesDuration:ResourcesDuration{},HostNodeName:,MemoizationStatus:nil,SynchronizationStatus:nil,} successful" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:08Z" level=info msg="node test-pvc-retry-problem-6sq8m-2082576660 phase Running -> Succeeded" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:08Z" level=info msg="SG Outbound nodes of test-pvc-retry-problem-6sq8m-928724940 are [test-pvc-retry-problem-6sq8m-928724940]" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:08Z" level=info msg="Skipped pod test-pvc-retry-problem-6sq8m[1].second (test-pvc-retry-problem-6sq8m-1089601533) creation: already exists" namespace=argo-next podPhase=Pending workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:08Z" level=info msg="Workflow step group node test-pvc-retry-problem-6sq8m-3223307657 not yet completed" namespace=argo-next workflow=test-pvc-retry-problem-6sq8m
time="2020-10-14T13:23:08Z" level=info msg="Workflow update successful" namespace=argo-next phase=Running resourceVersion=1450 workflow=test-pvc-retry-problem-6sq8m
```
---
<!-- Issue Author: Don't delete this message to encourage other users to support your issue! -->
**Message from the maintainers**:

Impacted by this bug? Give it a 👍. We prioritise the issues with the most 👍.


Pull request description:
fix(controller): Fix argo retry with PVCs. Fixes #4275
Checklist:

* [x] Either (a) I've created an [enhancement proposal](https://github.com/argoproj/argo/issues/new/choose) and discussed it with the community, (b) this is a bug fix, or (c) this is a chore.
* [x] The title of the PR is (a) [conventional](https://www.conventionalcommits.org/en/v1.0.0/), (b) states what changed, and (c) suffixes the related issues number. E.g. `"fix(controller): Updates such and such. Fixes #1234"`.  
* [x] My organization is added to [USERS.md](https://github.com/argoproj/argo/blob/master/USERS.md).
* [x] I've signed the CLA.
* [x] I have written unit and/or e2e tests for my change. PRs without these are unlikely to be merged.
* [x] My builds are green. Try syncing with master if they are not. 


Patch:
diff --git a/workflow/controller/operator.go b/workflow/controller/operator.go
index 34dc8d878..78d1cb1ac 100644
--- a/workflow/controller/operator.go
+++ b/workflow/controller/operator.go
@@ -388,18 +388,6 @@ func (woc *wfOperationCtx) operate() {
 		}
 	}
 
-	err = woc.deletePVCs()
-	if err != nil {
-		msg := "failed to delete PVCs"
-		woc.log.WithError(err).Errorf(msg)
-		// Mark the workflow with an error message and return, but intentionally do not
-		// markCompletion so that we can retry PVC deletion (TODO: use workqueue.ReAdd())
-		// This error phase may be cleared if a subsequent delete attempt is successful.
-		woc.markWorkflowError(err, false)
-		woc.eventRecorder.Event(woc.wf, apiv1.EventTypeWarning, "WorkflowFailed", fmt.Sprintf("%s %s: %+v", woc.wf.ObjectMeta.Name, msg, err))
-		return
-	}
-
 	var workflowMessage string
 	if node.FailedOrError() && woc.execWf.Spec.Shutdown != "" {
 		workflowMessage = fmt.Sprintf("Stopped with strategy '%s'", woc.execWf.Spec.Shutdown)
@@ -428,8 +416,8 @@ func (woc *wfOperationCtx) operate() {
 		woc.markWorkflowPhase(wfv1.NodeError, true, workflowMessage)
 		woc.eventRecorder.Event(woc.wf, apiv1.EventTypeWarning, "WorkflowFailed", workflowMessage)
 	default:
-		// NOTE: we should never make it here because if the the node was 'Running'
-		// we should have returned earlier.
+		// NOTE: we should never make it here because if the node was 'Running' we should have
+		// returned earlier.
 		err = errors.InternalErrorf("Unexpected node phase %s: %+v", woc.wf.ObjectMeta.Name, err)
 		woc.markWorkflowError(err, true)
 	}
@@ -441,6 +429,11 @@ func (woc *wfOperationCtx) operate() {
 		woc.globalParams[common.GlobalVarWorkflowStatus] = string(workflowStatus)
 		woc.computeMetrics(woc.execWf.Spec.Metrics.Prometheus, woc.globalParams, realTimeScope, false)
 	}
+
+	err = woc.deletePVCs()
+	if err != nil {
+		woc.log.WithError(err).Warn("failed to delete PVCs")
+	}
 }
 
 func (woc *wfOperationCtx) getWorkflowDeadline() *time.Time {


Test patch:
diff --git a/workflow/controller/operator_test.go b/workflow/controller/operator_test.go
index 1682d5778..b365fd50e 100644
--- a/workflow/controller/operator_test.go
+++ b/workflow/controller/operator_test.go
@@ -3379,6 +3379,88 @@ func TestRetryNodeOutputs(t *testing.T) {
 	assert.Contains(t, scope.scope, "steps.influx.finishedAt")
 }
 
+var workflowWithPVCAndFailingStep = `
+apiVersion: argoproj.io/v1alpha1
+kind: Workflow
+metadata:
+  name: wf-with-pvc
+spec:
+  entrypoint: entrypoint
+  templates:
+  - name: entrypoint
+    steps:
+    - - name: succeed
+        template: succeed
+    - - name: failure
+        template: failure
+  - name: succeed
+    script:
+      args: [success]
+      command: [cowsay]
+      image: docker/whalesay:latest
+      volumeMounts:
+      - mountPath: /data
+        name: data
+  - name: failure
+    script:
+      command: [sh]
+      image: alpine
+      args: [exit, "1"]
+      volumeMounts:
+      - mountPath: /data
+        name: data
+  volumeClaimTemplates:
+  - metadata:
+      name: data
+    spec:
+      accessModes:
+      - ReadWriteOnce
+      resources:
+        requests:
+          storage: 1Gi
+status:
+  nodes:
+    wf-with-pvc:
+      name: wf-with-pvc
+      phase: Failed
+    wf-with-pvc-2390440388:
+      name: wf-with-pvc(0)[0].succeed
+      phase: Succeeded
+    wf-with-pvc-3099954303:
+      name: wf-with-pvc(0)[1].failure
+      phase: Failed
+  persistentVolumeClaims:
+  - name: data
+    persistentVolumeClaim:
+      claimName: wf-with-pvc-data
+`
+
+// This test ensures that the PVCs used in the steps are not deleted when
+// the workflow fails
+func TestDeletePVCDoesNotDeletePVCOnFailedWorkflow(t *testing.T) {
+	assert := assert.New(t)
+
+	wf := unmarshalWF(workflowWithPVCAndFailingStep)
+	cancel, controller := newController(wf)
+	defer cancel()
+
+	woc := newWorkflowOperationCtx(wf, controller)
+
+	assert.Len(woc.wf.Status.PersistentVolumeClaims, 1, "1 PVC before operating")
+
+	woc.operate()
+
+	node1 := woc.wf.GetNodeByName("wf-with-pvc(0)[0].succeed")
+	node2 := woc.wf.GetNodeByName("wf-with-pvc(0)[1].failure")
+
+	// Node 1 Succeeded
+	assert.Equal(node1.Phase, wfv1.NodeSucceeded)
+	// Node 2 Failed
+	assert.Equal(node2.Phase, wfv1.NodeFailed)
+	// Hence, PVCs should stick around
+	assert.Len(woc.wf.Status.PersistentVolumeClaims, 1, "PVCs not deleted")
+}
+
 var containerOutputsResult = `
 apiVersion: argoproj.io/v1alpha1
 kind: Workflow