Change:
<!-- Please only use this template for submitting reports about flaky tests or jobs (pass or fail with no underlying change in code) in Kubernetes CI --Which jobs are flaking: Which test(s) are flaking: Testgrid link: Reason for failure: Unknown yet. Anything else we need to know: links to appreciated links to specific failures in spyglass appreciated /sig scheduling /help
Can I give it a try?
Sure, feel free to take it. /assign ahmad-diaa /remove-help
I keep trying to get it fail locally by running the following command: but it never does, am I doing something wrong? Is there something else that I should be doing and would be more helpful?
Sometimes the flake is hard to reproduce in local env. You may have to read the error log of the flakes thoroughly to get some clues. BTW: you can also run the test multiple times using the directive:
After looking through the logs I came up with some information that might be useful As far as I understand, the reason for the flake is the of the medium priority pod not being cleared That is because the update request for removing returns the following error: So my current guess is that might be outdated in some cases As I am not really familiar with the logic, I would like to know whether or not those are valid points so that I could dig deeper. What do you think?
If that's the case, you can set here: You can raise a PR and manually try for ~20 times to see you can reproduce the flake.
found another issue tracking this too. it seems it has been flaky for a while.
Sorry, didn't mean to take this one for you. I worked the duplicate issue and submitted a PR. Feel free to review that PR, or submit your own if you have a different solution.
This sometimes happens. But we usually take the first PR unless it's clearly worked by someone. In terms of which style to resolve the Conflict error, I'm asking here:
Sorry, didn't mean to take this one for you. I worked the duplicate issue and submitted a PR. Feel free to review that PR, or submit your own if you have a different solution. No worries, I haven't started implementing yet and your PR seems to be following the correct approach as well according to this:

Discussion:
"//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", <ins> "//staging/src/k8s.io/apimachinery/pkg/util/strategicpatch:go_default_library", </ins> "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/client-go/informers:go_default_library",

import ( "context" <ins> "encoding/json" </ins> "fmt" "io/ioutil" "math/rand"

v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" <ins> "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/strategicpatch" </ins> "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/informers" coreinformers "k8s.io/client-go/informers/core/v1"

func (p *podConditionUpdaterImpl) update(pod *v1.Pod, condition *v1.PodCondition) error { klog.V(3).Infof("Updating pod condition for %s/%s to (%s==%s, Reason=%s)", pod.Namespace, pod.Name, condition.Type, condition.Status, condition.Reason) <del> if podutil.UpdatePodCondition(&pod.Status, condition) { _, err := p.Client.CoreV1().Pods(pod.Namespace).UpdateStatus(context.TODO(), pod, metav1.UpdateOptions{}) </del> <ins> oldData, err := json.Marshal(pod) if err != nil { </ins> return err } <del> return nil </del> <ins> if !podutil.UpdatePodCondition(&pod.Status, condition) { return nil } newData, err := json.Marshal(pod) if err != nil { return err } patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, &v1.Pod{}) if err != nil { return fmt.Errorf("failed to create merge patch for pod %q/%q: %v", pod.Namespace, pod.Name, err) } _, err = p.Client.CoreV1().Pods(pod.Namespace).Patch(context.TODO(), pod.Name, types.StrategicMergePatchType, patchBytes, metav1.PatchOptions{}, "status") return err </ins> } type podPreemptorImpl struct {

} func (p *podPreemptorImpl) setNominatedNodeName(pod *v1.Pod, nominatedNodeName string) error { <ins> klog.V(3).Infof("Setting nominated node name for %s/%s to "%s"", pod.Namespace, pod.Name, nominatedNodeName) if pod.Status.NominatedNodeName == nominatedNodeName { return nil } </ins> podCopy := pod.DeepCopy() <ins> oldData, err := json.Marshal(podCopy) if err != nil { return err } </ins> podCopy.Status.NominatedNodeName = nominatedNodeName <del> _, err := p.Client.CoreV1().Pods(pod.Namespace).UpdateStatus(context.TODO(), podCopy, metav1.UpdateOptions{}) </del> <ins> newData, err := json.Marshal(podCopy) if err != nil { return err } patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, &v1.Pod{}) if err != nil { return fmt.Errorf("failed to create merge patch for pod %q/%q: %v", pod.Namespace, pod.Name, err) } _, err = p.Client.CoreV1().Pods(pod.Namespace).Patch(context.TODO(), pod.Name, types.StrategicMergePatchType, patchBytes, metav1.PatchOptions{}, "status") </ins> return err }

"os" "path" "reflect" <ins> "regexp" </ins> "sort" "strings" "sync"

} } } <ins> func TestSetNominatedNodeName(t *testing.T) { tests := []struct { name                     string currentNominatedNodeName string newNominatedNodeName     string expectedPatchRequests    int expectedPatchData        string }{ { name:                     "Should make patch request to set node name", currentNominatedNodeName: "", newNominatedNodeName:     "node1", expectedPatchRequests:    1, expectedPatchData:        `{"status":{"nominatedNodeName":"node1"}}`, }, { name:                     "Should make patch request to clear node name", currentNominatedNodeName: "node1", newNominatedNodeName:     "", expectedPatchRequests:    1, expectedPatchData:        `{"status":{"nominatedNodeName":null}}`, }, { name:                     "Should not make patch request if nominated node is already set to the specified value", currentNominatedNodeName: "node1", newNominatedNodeName:     "node1", expectedPatchRequests:    0, }, { name:                     "Should not make patch request if nominated node is already cleared", currentNominatedNodeName: "", newNominatedNodeName:     "", expectedPatchRequests:    0, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { actualPatchRequests := 0 var actualPatchData string cs := &clientsetfake.Clientset{} cs.AddReactor("patch", "pods", func(action clienttesting.Action) (bool, runtime.Object, error) { actualPatchRequests++ patch := action.(clienttesting.PatchAction) actualPatchData = string(patch.GetPatch()) // For this test, we don't care about the result of the patched pod, just that we got the expected // patch request, so just returning &v1.Pod{} here is OK because scheduler doesn't use the response. return true, &v1.Pod{}, nil }) pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{Name: "foo"}, Status:     v1.PodStatus{NominatedNodeName: test.currentNominatedNodeName}, } preemptor := &podPreemptorImpl{Client: cs} if err := preemptor.setNominatedNodeName(pod, test.newNominatedNodeName); err != nil { t.Fatalf("Error calling setNominatedNodeName: %v", err) } if actualPatchRequests != test.expectedPatchRequests { t.Fatalf("Actual patch requests (%d) dos not equal expected patch requests (%d)", actualPatchRequests, test.expectedPatchRequests) } if test.expectedPatchRequests > 0 && actualPatchData != test.expectedPatchData { t.Fatalf("Patch data mismatch: Actual was %v, but expected %v", actualPatchData, test.expectedPatchData) } }) } } func TestUpdatePodCondition(t *testing.T) { tests := []struct { name                     string currentPodConditions     []v1.PodCondition newPodCondition          *v1.PodCondition expectedPatchRequests    int expectedPatchDataPattern string }{ { name:                 "Should make patch request to add pod condition when there are none currently", currentPodConditions: []v1.PodCondition{}, newPodCondition: &v1.PodCondition{ Type:               "newType", Status:             "newStatus", LastProbeTime:      metav1.NewTime(time.Date(2020, 5, 13, 1, 1, 1, 1, time.UTC)), LastTransitionTime: metav1.NewTime(time.Date(2020, 5, 12, 1, 1, 1, 1, time.UTC)), Reason:             "newReason", Message:            "newMessage", }, expectedPatchRequests:    1, expectedPatchDataPattern: `{"status":{"conditions":[{"lastProbeTime":"2020-05-13T01:01:01Z","lastTransitionTime":".*","message":"newMessage","reason":"newReason","status":"newStatus","type":"newType"}]}}`, }, { name: "Should make patch request to add a new pod condition when there is already one with another type", currentPodConditions: []v1.PodCondition{ { Type:               "someOtherType", Status:             "someOtherTypeStatus", LastProbeTime:      metav1.NewTime(time.Date(2020, 5, 11, 0, 0, 0, 0, time.UTC)), LastTransitionTime: metav1.NewTime(time.Date(2020, 5, 10, 0, 0, 0, 0, time.UTC)), Reason:             "someOtherTypeReason", Message:            "someOtherTypeMessage", }, }, newPodCondition: &v1.PodCondition{ Type:               "newType", Status:             "newStatus", LastProbeTime:      metav1.NewTime(time.Date(2020, 5, 13, 1, 1, 1, 1, time.UTC)), LastTransitionTime: metav1.NewTime(time.Date(2020, 5, 12, 1, 1, 1, 1, time.UTC)), Reason:             "newReason", Message:            "newMessage", }, expectedPatchRequests:    1, expectedPatchDataPattern: `{"status":{"$setElementOrder/conditions":[{"type":"someOtherType"},{"type":"newType"}],"conditions":[{"lastProbeTime":"2020-05-13T01:01:01Z","lastTransitionTime":".*","message":"newMessage","reason":"newReason","status":"newStatus","type":"newType"}]}}`, }, { name: "Should make patch request to update an existing pod condition", currentPodConditions: []v1.PodCondition{ { Type:               "currentType", Status:             "currentStatus", LastProbeTime:      metav1.NewTime(time.Date(2020, 5, 13, 0, 0, 0, 0, time.UTC)), LastTransitionTime: metav1.NewTime(time.Date(2020, 5, 12, 0, 0, 0, 0, time.UTC)), Reason:             "currentReason", Message:            "currentMessage", }, }, newPodCondition: &v1.PodCondition{ Type:               "currentType", Status:             "newStatus", LastProbeTime:      metav1.NewTime(time.Date(2020, 5, 13, 1, 1, 1, 1, time.UTC)), LastTransitionTime: metav1.NewTime(time.Date(2020, 5, 12, 1, 1, 1, 1, time.UTC)), Reason:             "newReason", Message:            "newMessage", }, expectedPatchRequests:    1, expectedPatchDataPattern: `{"status":{"$setElementOrder/conditions":[{"type":"currentType"}],"conditions":[{"lastProbeTime":"2020-05-13T01:01:01Z","lastTransitionTime":".*","message":"newMessage","reason":"newReason","status":"newStatus","type":"currentType"}]}}`, }, { name: "Should make patch request to update an existing pod condition, but the transition time should remain unchanged because the status is the same", currentPodConditions: []v1.PodCondition{ { Type:               "currentType", Status:             "currentStatus", LastProbeTime:      metav1.NewTime(time.Date(2020, 5, 13, 0, 0, 0, 0, time.UTC)), LastTransitionTime: metav1.NewTime(time.Date(2020, 5, 12, 0, 0, 0, 0, time.UTC)), Reason:             "currentReason", Message:            "currentMessage", }, }, newPodCondition: &v1.PodCondition{ Type:               "currentType", Status:             "currentStatus", LastProbeTime:      metav1.NewTime(time.Date(2020, 5, 13, 1, 1, 1, 1, time.UTC)), LastTransitionTime: metav1.NewTime(time.Date(2020, 5, 12, 0, 0, 0, 0, time.UTC)), Reason:             "newReason", Message:            "newMessage", }, expectedPatchRequests:    1, expectedPatchDataPattern: `{"status":{"$setElementOrder/conditions":[{"type":"currentType"}],"conditions":[{"lastProbeTime":"2020-05-13T01:01:01Z","message":"newMessage","reason":"newReason","type":"currentType"}]}}`, }, { name: "Should not make patch request if pod condition already exists and is identical", currentPodConditions: []v1.PodCondition{ { Type:               "currentType", Status:             "currentStatus", LastProbeTime:      metav1.NewTime(time.Date(2020, 5, 13, 0, 0, 0, 0, time.UTC)), LastTransitionTime: metav1.NewTime(time.Date(2020, 5, 12, 0, 0, 0, 0, time.UTC)), Reason:             "currentReason", Message:            "currentMessage", }, }, newPodCondition: &v1.PodCondition{ Type:               "currentType", Status:             "currentStatus", LastProbeTime:      metav1.NewTime(time.Date(2020, 5, 13, 0, 0, 0, 0, time.UTC)), LastTransitionTime: metav1.NewTime(time.Date(2020, 5, 12, 0, 0, 0, 0, time.UTC)), Reason:             "currentReason", Message:            "currentMessage", }, expectedPatchRequests: 0, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { actualPatchRequests := 0 var actualPatchData string cs := &clientsetfake.Clientset{} cs.AddReactor("patch", "pods", func(action clienttesting.Action) (bool, runtime.Object, error) { actualPatchRequests++ patch := action.(clienttesting.PatchAction) actualPatchData = string(patch.GetPatch()) // For this test, we don't care about the result of the patched pod, just that we got the expected // patch request, so just returning &v1.Pod{} here is OK because scheduler doesn't use the response. return true, &v1.Pod{}, nil }) pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{Name: "foo"}, Status:     v1.PodStatus{Conditions: test.currentPodConditions}, } updater := &podConditionUpdaterImpl{Client: cs} if err := updater.update(pod, test.newPodCondition); err != nil { t.Fatalf("Error calling update: %v", err) } if actualPatchRequests != test.expectedPatchRequests { t.Fatalf("Actual patch requests (%d) dos not equal expected patch requests (%d)", actualPatchRequests, test.expectedPatchRequests) } regex, err := regexp.Compile(test.expectedPatchDataPattern) if err != nil { t.Fatalf("Error compiling regexp for %v: %v", test.expectedPatchDataPattern, err) } if test.expectedPatchRequests > 0 && !regex.MatchString(actualPatchData) { t.Fatalf("Patch data mismatch: Actual was %v, but expected to match regexp %v", actualPatchData, test.expectedPatchDataPattern) } }) } } </ins>

framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" "k8s.io/kubernetes/plugin/pkg/admission/priority" testutils "k8s.io/kubernetes/test/integration/util" <del> utils "k8s.io/kubernetes/test/utils" </del> <ins> "k8s.io/kubernetes/test/utils" </ins> ) var lowPriority, mediumPriority, highPriority = int32(100), int32(200), int32(300)

Labels:    map[string]string{"pod": name}, Resources: defaultPodRes, }) <del> // Setting grace period to zero. Otherwise, we may never see the actual deletion // of the pods in integration tests. </del> pod.Spec.TerminationGracePeriodSeconds = &grace return pod }

} // Step 5. Check that nominated node name of the high priority pod is set. if err := waitForNominatedNodeName(cs, highPriPod); err != nil { <del> t.Errorf("NominatedNodeName annotation was not set for pod %v/%v: %v", medPriPod.Namespace, medPriPod.Name, err) </del> <ins> t.Errorf("NominatedNodeName annotation was not set for pod %v/%v: %v", highPriPod.Namespace, highPriPod.Name, err) </ins> } // And the nominated node name of the medium priority pod is cleared. if err := wait.Poll(100*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) {