Change:
<!-- Please only use this template for submitting reports about flaky tests or jobs (pass or fail with no underlying change in code) in Kubernetes CI --Integration tests are sometimes flaking locally: PreferNominatedNode Once the node gets deleted, the nodelifecycle controller is racing to update pod status and the pod deletion logic is failing causing tests to flake. Ultimately, we call testCtx.CleanUp() which cleans the nodes and the pods involved the test namespace so we are not going to leak any resources. links to appreciated links to specific failures in spyglass appreciated <!-- You can identify the SIG from the "prowjobconfigurl" on the testgrid dashboard for a test --/sig scheduling <!-- Please see the deflaking doc () for more guidance! --/assign
cc Tried addressing it via
/triage accepted thanks!

Discussion:
// enabled. func TestPreferNominatedNode(t *testing.T) { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PreferNominatedNode, true)() <del> testCtx := initTestPreferNominatedNode(t, "perfer-nominated-node") t.Cleanup(func() { testutils.CleanupTest(t, testCtx) }) cs := testCtx.ClientSet defaultPodRes := &v1.ResourceRequirements{Requests: v1.ResourceList{ v1.ResourceCPU:    *resource.NewMilliQuantity(100, resource.DecimalSI), v1.ResourceMemory: *resource.NewQuantity(100, resource.DecimalSI)}, } </del> defaultNodeRes := map[v1.ResourceName]string{ v1.ResourcePods:   "32", v1.ResourceCPU:    "500m", v1.ResourceMemory: "500", } <del> type nodeConfig struct { name string res  map[v1.ResourceName]string </del> <ins> defaultPodRes := &v1.ResourceRequirements{Requests: v1.ResourceList{ v1.ResourceCPU:    *resource.NewMilliQuantity(100, resource.DecimalSI), v1.ResourceMemory: *resource.NewQuantity(100, resource.DecimalSI)}, </ins> } tests := []struct { name         string <del> nodes        []*nodeConfig </del> <ins> nodeNames    []string </ins> existingPods []*v1.Pod pod          *v1.Pod runnningNode string }{ { <del> name: "nominated node released all resource, preemptor is scheduled to the nominated node", nodes: []*nodeConfig{ {name: "node-1", res: defaultNodeRes}, {name: "node-2", res: defaultNodeRes}, }, </del> <ins> name:      "nominated node released all resource, preemptor is scheduled to the nominated node", nodeNames: []string{"node-1", "node-2"}, </ins> existingPods: []*v1.Pod{ initPausePod(&pausePodConfig{ Name:      "low-pod1", <del> Namespace: testCtx.NS.Name, </del> Priority:  &lowPriority, NodeName:  "node-2", Resources: defaultPodRes, }), }, pod: initPausePod(&pausePodConfig{ <del> Name:      "preemptor-pod", Namespace: testCtx.NS.Name, Priority:  &highPriority, </del> <ins> Name:     "preemptor-pod", Priority: &highPriority, </ins> Resources: &v1.ResourceRequirements{Requests: v1.ResourceList{ v1.ResourceCPU:    *resource.NewMilliQuantity(500, resource.DecimalSI), v1.ResourceMemory: *resource.NewQuantity(200, resource.DecimalSI)},

runnningNode: "node-1", }, { <del> name: "nominated node cannot pass all the filters, preemptor should find a different node", nodes: []*nodeConfig{ {name: "node-1", res: defaultNodeRes}, {name: "node-2", res: defaultNodeRes}, }, </del> <ins> name:      "nominated node cannot pass all the filters, preemptor should find a different node", nodeNames: []string{"node-1", "node-2"}, </ins> existingPods: []*v1.Pod{ initPausePod(&pausePodConfig{ <del> Name:      "low-pod1", Namespace: testCtx.NS.Name, </del> <ins> Name:      "low-pod", </ins> Priority:  &lowPriority, Resources: defaultPodRes, NodeName:  "node-1", }), }, pod: initPausePod(&pausePodConfig{ <del> Name:      "preemptor-pod", Namespace: testCtx.NS.Name, Priority:  &highPriority, </del> <ins> Name:     "preemptor-pod1", Priority: &highPriority, </ins> Resources: &v1.ResourceRequirements{Requests: v1.ResourceList{ v1.ResourceCPU:    *resource.NewMilliQuantity(500, resource.DecimalSI), v1.ResourceMemory: *resource.NewQuantity(200, resource.DecimalSI)},

for _, test := range tests { t.Run(test.name, func(t *testing.T) { <ins> testCtx := initTestPreferNominatedNode(t, "perfer-nominated-node") t.Cleanup(func() { testutils.CleanupTest(t, testCtx) }) cs := testCtx.ClientSet nsName := testCtx.NS.Name </ins> var err error var preemptor *v1.Pod <del> for _, nodeConf := range test.nodes { _, err := createNode(cs, st.MakeNode().Name(nodeConf.name).Capacity(nodeConf.res).Obj()) </del> <ins> for _, nodeName := range test.nodeNames { _, err := createNode(cs, st.MakeNode().Name(nodeName).Capacity(defaultNodeRes).Obj()) </ins> if err != nil { <del> t.Fatalf("Error creating node %v: %v", nodeConf.name, err) </del> <ins> t.Fatalf("Error creating node %v: %v", nodeName, err) </ins> } } pods := make([]*v1.Pod, len(test.existingPods)) // Create and run existingPods. for i, p := range test.existingPods { <ins> p.Namespace = nsName </ins> pods[i], err = runPausePod(cs, p) if err != nil { t.Fatalf("Error running pause pod: %v", err) } } <ins> test.pod.Namespace = nsName </ins> preemptor, err = createPausePod(cs, test.pod) if err != nil { t.Errorf("Error while creating high priority pod: %v", err)

if preemptor.Spec.NodeName != test.runnningNode { t.Errorf("Expect pod running on %v, got %v.", test.runnningNode, preemptor.Spec.NodeName) } <del> pods = append(pods, preemptor) // cleanup defer testutils.CleanupPods(cs, t, pods) cs.CoreV1().Nodes().DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{}) </del> }) } }