Change:
framework.ExpectNoError(err) }) }) <ins> ginkgo.Describe("Automatically recreate PVC for pending pod when PVC is missing", func() { ssName := "ss" labels := map[string]string{ "foo": "bar", "baz": "blah", } headlessSvcName := "test" var statefulPodMounts []v1.VolumeMount var ss *appsv1.StatefulSet ginkgo.BeforeEach(func(ctx context.Context) { statefulPodMounts = []v1.VolumeMount{{Name: "datadir", MountPath: "/data/"}} ss = e2estatefulset.NewStatefulSet(ssName, ns, headlessSvcName, 1, statefulPodMounts, nil, labels) }) ginkgo.AfterEach(func(ctx context.Context) { if ginkgo.CurrentSpecReport().Failed() { e2eoutput.DumpDebugInfo(ctx, c, ns) } framework.Logf("Deleting all statefulset in ns %v", ns) e2estatefulset.DeleteAllStatefulSets(ctx, c, ns) }) ginkgo.It("PVC should be recreated when pod is pending due to missing PVC [Disruptive][Serial]", func(ctx context.Context) { e2epv.SkipIfNoDefaultStorageClass(ctx, c) readyNode, err := e2enode.GetRandomReadySchedulableNode(ctx, c) framework.ExpectNoError(err) hostLabel := "kubernetes.io/hostname" hostLabelVal := readyNode.Labels[hostLabel] ss.Spec.Template.Spec.NodeSelector = map[string]string{hostLabel: hostLabelVal} // force the pod on a specific node ginkgo.By("Creating statefulset " + ssName + " in namespace " + ns) _, err = c.AppsV1().StatefulSets(ns).Create(context.TODO(), ss, metav1.CreateOptions{}) framework.ExpectNoError(err) ginkgo.By("Confirming PVC exists") err = verifyStatefulSetPVCsExist(ctx, c, ss, []int{0}) framework.ExpectNoError(err) ginkgo.By("Confirming Pod is ready") e2estatefulset.WaitForStatusReadyReplicas(ctx, c, ss, 1) podName := getStatefulSetPodNameAtIndex(0, ss) pod, err := c.CoreV1().Pods(ns).Get(context.TODO(), podName, metav1.GetOptions{}) framework.ExpectNoError(err) nodeName := pod.Spec.NodeName framework.ExpectEqual(nodeName, readyNode.Name) node, err := c.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{}) framework.ExpectNoError(err) oldData, err := json.Marshal(node) framework.ExpectNoError(err) node.Spec.Unschedulable = true newData, err := json.Marshal(node) framework.ExpectNoError(err) // cordon node, to make sure pod does not get scheduled to the node until the pvc is deleted patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, v1.Node{}) framework.ExpectNoError(err) ginkgo.By("Cordoning Node") _, err = c.CoreV1().Nodes().Patch(context.TODO(), nodeName, types.StrategicMergePatchType, patchBytes, metav1.PatchOptions{}) framework.ExpectNoError(err) cordoned := true defer func() { if cordoned { uncordonNode(c, oldData, newData, nodeName) } }() // wait for the node to be unschedulable e2enode.WaitForNodeSchedulable(c, nodeName, 10*time.Second, false) ginkgo.By("Deleting Pod") err = c.CoreV1().Pods(ns).Delete(context.TODO(), podName, metav1.DeleteOptions{}) framework.ExpectNoError(err) // wait for the pod to be recreated e2estatefulset.WaitForStatusCurrentReplicas(c, ss, 1) _, err = c.CoreV1().Pods(ns).Get(context.TODO(), podName, metav1.GetOptions{}) framework.ExpectNoError(err) pvcList, err := c.CoreV1().PersistentVolumeClaims(ns).List(context.TODO(), metav1.ListOptions{LabelSelector: klabels.Everything().String()}) framework.ExpectNoError(err) framework.ExpectEqual(len(pvcList.Items), 1) pvcName := pvcList.Items[0].Name ginkgo.By("Deleting PVC") err = c.CoreV1().PersistentVolumeClaims(ns).Delete(context.TODO(), pvcName, metav1.DeleteOptions{}) framework.ExpectNoError(err) uncordonNode(c, oldData, newData, nodeName) cordoned = false ginkgo.By("Confirming PVC recreated") err = verifyStatefulSetPVCsExist(ctx, c, ss, []int{0}) framework.ExpectNoError(err) ginkgo.By("Confirming Pod is ready after being recreated") e2estatefulset.WaitForStatusReadyReplicas(ctx, c, ss, 1) pod, err = c.CoreV1().Pods(ns).Get(context.TODO(), podName, metav1.GetOptions{}) framework.ExpectNoError(err) framework.ExpectEqual(pod.Spec.NodeName, readyNode.Name) // confirm the pod was scheduled back to the original node }) }) </ins> }) <ins> func uncordonNode(c clientset.Interface, oldData, newData []byte, nodeName string) { ginkgo.By("Uncordoning Node") // uncordon node, by reverting patch revertPatchBytes, err := strategicpatch.CreateTwoWayMergePatch(newData, oldData, v1.Node{}) framework.ExpectNoError(err) _, err = c.CoreV1().Nodes().Patch(context.TODO(), nodeName, types.StrategicMergePatchType, revertPatchBytes, metav1.PatchOptions{}) framework.ExpectNoError(err) } </ins> func kubectlExecWithRetries(ns string, args ...string) (out string) { var err error for i := 0; i < 3; i++ {

Discussion:
User cordons a node with a pvc bound to a local persistent volume User deletes a stateful set pod from that node The pod is in pending because the pvc is still bound to a local pv on the cordoned node (expected) The node dies and the pvc is deleted The pod is still stuck in pending because the pvc is missing (bug) The pod now needs to be manually deleted so that the pod+pvc can be recreated After the pvc was deleted, the statefulset controller should have recreated it and bound it to a new node's local pv so that the pending pod could be scheduled. Currently the only way to get the pod out of pending is to manually delete it so that it recreates itself and its pvc We used for the local PVs, but the problem is not limited to this particular provisioner create a stateful set with a pvc that is bound to a local PV use to cordon a node with a local pv delete a statefulset pod from that node delete the pvc from that node the pod is now stuck in pending due to a missing pvc There was a related issue but this only fixed the issue when the order of operations was delete pvc -delete pod. but in this case, it is delete pod -delete pvc <details</details<detailsTested on aws and gcp </details<details</details<details</details<details</details<details</details
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig apps storage
The Kubernetes project currently lacks enough contributors to adequately respond to all issues and PRs.

This bot triages issues and PRs according to the following rules: After 90d of inactivity, is applied After 30d of inactivity since was applied, is applied After 30d of inactivity since was applied, the issue is closed You can: Mark this issue or PR as fresh with Mark this issue or PR as rotten with Close this issue or PR with Offer to help out with [Issue Triage][1] Please send feedback to sig-contributor-experience at . /lifecycle stale [1]:
/remove-lifecycle stale
/reopen
Reopened this issue. <detailsIn response to : /reopen Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
The Kubernetes project currently lacks enough contributors to adequately respond to all issues. This bot triages un-triaged issues according to the following rules: After 90d of inactivity, is applied After 30d of inactivity since was applied, is applied After 30d of inactivity since was applied, the issue is closed You can: Mark this issue as fresh with Close this issue with Offer to help out with [Issue Triage][1] Please send feedback to sig-contributor-experience at . /lifecycle stale [1]:
This issue was fixed by The new issues look different. /close
Closing this issue. <detailsIn response to : This issue was fixed by The new issues look different. Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details