Change:
<!-- This form is for bug reports and feature requests ONLY! If you're looking for help check and the . If the matter is security related, please disclose it privately via --Is this a BUG REPORT or FEATURE REQUEST?: What happened: Testgrid: https://k8s-
/assign
The test's assumption about test parameter is wrong. Previously it assumed the group name is the same across all zones, but they are different in regional clusters. Looking into using node taints as the permanent fix, as suggested by the TODO comment in the test.
Hm I think taint based evictions is still an alpha feature.
OK. I propose the following solution: Get the zone of Schedule the StatefulSet pod to that zone using node affinity; wait for the pod to run Remove the node affinity from StatefulSet Delete
Can it be simplified? Schedule the pod Figure out which zone that pod got scheduled in Delete the instance group of that zone
Unfortunately I don't know of a good way to look up the instance group given the zone, since there could be instance groups from multiple clusters in the same project
How about: Schedule the pod Figure out the zone Taint all the nodes in the zone Kill the pod
Taint with NoSchedule? yeah that sounds good

Discussion:
"//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/rand:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", <ins> "//staging/src/k8s.io/apimachinery/pkg/util/strategicpatch:go_default_library", </ins> "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/version:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",

"strings" "time" <ins> "encoding/json" </ins> appsv1 "k8s.io/api/apps/v1" "k8s.io/api/core/v1" storage "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" <ins> "k8s.io/apimachinery/pkg/types" </ins> "k8s.io/apimachinery/pkg/util/sets" <ins> "k8s.io/apimachinery/pkg/util/strategicpatch" "k8s.io/apimachinery/pkg/util/wait" </ins> clientset "k8s.io/client-go/kubernetes" podutil "k8s.io/kubernetes/pkg/api/v1/pod" "k8s.io/kubernetes/pkg/kubelet/apis" kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis" "k8s.io/kubernetes/pkg/volume/util" "k8s.io/kubernetes/test/e2e/framework" <del> "k8s.io/kubernetes/test/e2e/framework/providers/gce" </del> "k8s.io/kubernetes/test/e2e/storage/testsuites" "k8s.io/kubernetes/test/e2e/storage/utils" imageutils "k8s.io/kubernetes/test/utils/image"

const ( pvDeletionTimeout       = 3 * time.Minute statefulSetReadyTimeout = 3 * time.Minute <ins> taintKeyPrefix          = "zoneTaint_" </ins> ) var _ = utils.SIGDescribe("Regional PD", func() {

} func testZonalFailover(c clientset.Interface, ns string) { <del> nodes := framework.GetReadySchedulableNodesOrDie(c) nodeCount := len(nodes.Items) </del> cloudZones := getTwoRandomZones(c) class := newRegionalStorageClass(ns, cloudZones) claimTemplate := newClaimTemplate(ns)

Expect(err).ToNot(HaveOccurred()) podZone := node.Labels[apis.LabelZoneFailureDomain] <del> // TODO (verult) Consider using node taints to simulate zonal failure instead. By("deleting instance group belonging to pod's zone") // Asynchronously detect a pod reschedule is triggered during/after instance group deletion. waitStatus := make(chan error) go func() { waitStatus <- waitForStatefulSetReplicasNotReady(statefulSet.Name, ns, c) }() cloud, err := gce.GetGCECloud() if err != nil { Expect(err).NotTo(HaveOccurred()) } instanceGroupName := framework.TestContext.CloudConfig.NodeInstanceGroup instanceGroup, err := cloud.GetInstanceGroup(instanceGroupName, podZone) Expect(err).NotTo(HaveOccurred(), "Error getting instance group %s in zone %s", instanceGroupName, podZone) templateName, err := framework.GetManagedInstanceGroupTemplateName(podZone) Expect(err).NotTo(HaveOccurred(), "Error getting instance group template in zone %s", podZone) err = framework.DeleteManagedInstanceGroup(podZone) Expect(err).NotTo(HaveOccurred(), "Error deleting instance group in zone %s", podZone) </del> <ins> By("tainting nodes in the zone the pod is scheduled in") selector := labels.SelectorFromSet(labels.Set(map[string]string{apis.LabelZoneFailureDomain: podZone})) nodesInZone, err := c.CoreV1().Nodes().List(metav1.ListOptions{LabelSelector: selector.String()}) Expect(err).ToNot(HaveOccurred()) removeTaintFunc := addTaint(c, ns, nodesInZone.Items, podZone) </ins> defer func() { <del> framework.Logf("recreating instance group %s", instanceGroup.Name) framework.ExpectNoError(framework.CreateManagedInstanceGroup(instanceGroup.Size, podZone, templateName), "Error recreating instance group %s in zone %s", instanceGroup.Name, podZone) framework.ExpectNoError(framework.WaitForReadyNodes(c, nodeCount, framework.RestartNodeReadyAgainTimeout), "Error waiting for nodes from the new instance group to become ready.") </del> <ins> framework.Logf("removing previously added node taints") removeTaintFunc() </ins> }() <del> err = <-waitStatus Expect(err).ToNot(HaveOccurred(), "Error waiting for replica to be deleted during failover: %v", err) </del> <ins> By("deleting StatefulSet pod") err = c.CoreV1().Pods(ns).Delete(pod.Name, &metav1.DeleteOptions{}) // Verify the pod is scheduled in the other zone. By("verifying the pod is scheduled in a different zone.") var otherZone string if cloudZones[0] == podZone { otherZone = cloudZones[1] } else { otherZone = cloudZones[0] } err = wait.PollImmediate(framework.Poll, statefulSetReadyTimeout, func() (bool, error) { framework.Logf("checking whether new pod is scheduled in zone %q", otherZone) pod = getPod(c, ns, regionalPDLabels) nodeName = pod.Spec.NodeName node, err = c.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{}) if err != nil { return false, nil } newPodZone := node.Labels[apis.LabelZoneFailureDomain] return newPodZone == otherZone, nil }) Expect(err).NotTo(HaveOccurred(), "Error waiting for pod to be scheduled in a different zone (%q): %v", otherZone, err) </ins> err = framework.WaitForStatefulSetReplicasReady(statefulSet.Name, ns, c, 3*time.Second, framework.RestartPodReadyAgainTimeout) if err != nil {

"The same PVC should be used after failover.") By("verifying the container output has 2 lines, indicating the pod has been created twice using the same regional PD.") <del> pod = getPod(c, ns, regionalPDLabels) </del> logs, err := framework.GetPodLogs(c, ns, pod.Name, "") Expect(err).NotTo(HaveOccurred(), "Error getting logs from pod %s in namespace %s", pod.Name, ns)

Expect(lineCount).To(Equal(expectedLineCount), "Line count of the written file should be %d.", expectedLineCount) <del> // Verify the pod is scheduled in the other zone. By("verifying the pod is scheduled in a different zone.") var otherZone string if cloudZones[0] == podZone { otherZone = cloudZones[1] } else { otherZone = cloudZones[0] </del> <ins> } func addTaint(c clientset.Interface, ns string, nodes []v1.Node, podZone string) (removeTaint func()) { reversePatches := make(map[string][]byte) for _, node := range nodes { oldData, err := json.Marshal(node) Expect(err).NotTo(HaveOccurred()) node.Spec.Taints = append(node.Spec.Taints, v1.Taint{ Key:    taintKeyPrefix + ns, Value:  podZone, Effect: v1.TaintEffectNoSchedule, }) newData, err := json.Marshal(node) Expect(err).NotTo(HaveOccurred()) patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, v1.Node{}) Expect(err).NotTo(HaveOccurred()) reversePatchBytes, err := strategicpatch.CreateTwoWayMergePatch(newData, oldData, v1.Node{}) Expect(err).NotTo(HaveOccurred()) reversePatches[node.Name] = reversePatchBytes _, err = c.CoreV1().Nodes().Patch(node.Name, types.StrategicMergePatchType, patchBytes) Expect(err).ToNot(HaveOccurred()) </ins> } <del> nodeName = pod.Spec.NodeName node, err = c.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{}) Expect(err).ToNot(HaveOccurred()) newPodZone := node.Labels[apis.LabelZoneFailureDomain] Expect(newPodZone).To(Equal(otherZone), "The pod should be scheduled in zone %s after all nodes in zone %s have been deleted", otherZone, podZone) </del> <ins> return func() { for nodeName, reversePatch := range reversePatches { _, err := c.CoreV1().Nodes().Patch(nodeName, types.StrategicMergePatchType, reversePatch) Expect(err).ToNot(HaveOccurred()) } } </ins> } func testRegionalDelayedBinding(c clientset.Interface, ns string) {