Change:
<!-- Please only use this template for submitting reports about continuously failing tests or jobs in Kubernetes CI --ci-kubernetes-e2e-gci-gce-serial (gce-cos-master-serial) Kubernetes e2e suite.[sig-scheduling] SchedulerPriorities [Serial] Pod should avoid nodes that have avoidPod annotation June 19th 2021 () Testgrid: https://k8s- Prow history: Looks like pods are scheduled where they shouldn't be? : The tests passes in maybe 20% of the builds, but it looks random, so I don't treat it as a flake. At the moment there are also sig-storage tests consistently failing in this job: , They don't look related though. /sig scheduling /kind failing-test
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/milestone v1.22
try to have a look.
try to have a look.
/cc
This is expected. The feature (which was alpha) is now disabled by default. Users can still enable it by switching to a v1beta1 scheduler component config. Let's remove the test.

Discussion:
"k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" <del> "k8s.io/apimachinery/pkg/fields" </del> "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/watch"

// Action is a function to be performed by the system. type Action func() error <del> // observeNodeUpdateAfterAction returns true if a node update matching the predicate was emitted // from the system after performing the supplied action. func observeNodeUpdateAfterAction(c clientset.Interface, nodeName string, nodePredicate func(*v1.Node) bool, action Action) (bool, error) { observedMatchingNode := false nodeSelector := fields.OneTermEqualSelector("metadata.name", nodeName) informerStartedChan := make(chan struct{}) var informerStartedGuard sync.Once _, controller := cache.NewInformer( &cache.ListWatch{ ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { options.FieldSelector = nodeSelector.String() ls, err := c.CoreV1().Nodes().List(context.TODO(), options) return ls, err }, WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { // Signal parent goroutine that watching has begun. defer informerStartedGuard.Do(func() { close(informerStartedChan) }) options.FieldSelector = nodeSelector.String() w, err := c.CoreV1().Nodes().Watch(context.TODO(), options) return w, err }, }, &v1.Node{}, 0, cache.ResourceEventHandlerFuncs{ UpdateFunc: func(oldObj, newObj interface{}) { n, ok := newObj.(*v1.Node) framework.ExpectEqual(ok, true) if nodePredicate(n) { observedMatchingNode = true } }, }, ) // Start the informer and block this goroutine waiting for the started signal. informerStopChan := make(chan struct{}) defer func() { close(informerStopChan) }() go controller.Run(informerStopChan) <-informerStartedChan // Invoke the action function. err := action() if err != nil { return false, err } // Poll whether the informer has found a matching node update with a timeout. // Wait up 2 minutes polling every second. timeout := 2 * time.Minute interval := 1 * time.Second err = wait.Poll(interval, timeout, func() (bool, error) { return observedMatchingNode, nil }) return err == nil, err } </del> // observeEventAfterAction returns true if an event matching the predicate was emitted // from the system after performing the supplied action. func observeEventAfterAction(c clientset.Interface, ns string, eventPredicate func(*v1.Event) bool, action Action) (bool, error) {

import ( "context" <del> "encoding/json" </del> "fmt" "math" "sync"

_ "github.com/stretchr/testify/assert" v1 "k8s.io/api/core/v1" <del> apierrors "k8s.io/apimachinery/pkg/api/errors" </del> "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels"

"k8s.io/kubernetes/test/e2e/framework" e2enode "k8s.io/kubernetes/test/e2e/framework/node" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" <del> e2erc "k8s.io/kubernetes/test/e2e/framework/rc" </del> e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" testutils "k8s.io/kubernetes/test/utils" <del> imageutils "k8s.io/kubernetes/test/utils/image" </del> ) // Resource is a collection of compute resource.

}, } <del> // addOrUpdateAvoidPodOnNode adds avoidPods annotations to node, will override if it exists func addOrUpdateAvoidPodOnNode(c clientset.Interface, nodeName string, avoidPods v1.AvoidPods) { err := wait.PollImmediate(framework.Poll, framework.SingleCallTimeout, func() (bool, error) { node, err := c.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{}) if err != nil { return false, err } taintsData, err := json.Marshal(avoidPods) framework.ExpectNoError(err) if node.Annotations == nil { node.Annotations = make(map[string]string) } node.Annotations[v1.PreferAvoidPodsAnnotationKey] = string(taintsData) _, err = c.CoreV1().Nodes().Update(context.TODO(), node, metav1.UpdateOptions{}) if err != nil { if !apierrors.IsConflict(err) { framework.ExpectNoError(err) } else { framework.Logf("Conflict when trying to add/update avoidPods %v to %v with error %v", avoidPods, nodeName, err) return false, nil } } return true, nil }) framework.ExpectNoError(err) } // removeAvoidPodsOffNode removes AvoidPods annotations from the node. It does not fail if no such annotation exists. func removeAvoidPodsOffNode(c clientset.Interface, nodeName string) { err := wait.PollImmediate(framework.Poll, framework.SingleCallTimeout, func() (bool, error) { node, err := c.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{}) if err != nil { return false, err } if node.Annotations == nil { return true, nil } delete(node.Annotations, v1.PreferAvoidPodsAnnotationKey) _, err = c.CoreV1().Nodes().Update(context.TODO(), node, metav1.UpdateOptions{}) if err != nil { if !apierrors.IsConflict(err) { framework.ExpectNoError(err) } else { framework.Logf("Conflict when trying to remove avoidPods to %v", nodeName) return false, nil } } return true, nil }) framework.ExpectNoError(err) } </del> // nodesAreTooUtilized ensures that each node can support 2*crioMinMemLimit // We check for double because it needs to support at least the cri-o minimum // plus whatever delta between node usages (which could be up to or at least crioMinMemLimit)

framework.ExpectNotEqual(labelPod.Spec.NodeName, nodeName) }) <del> ginkgo.It("Pod should avoid nodes that have avoidPod annotation", func() { nodeName := nodeList.Items[0].Name // make the nodes have balanced cpu,mem usage cleanUp, err := createBalancedPodForNodes(f, cs, ns, nodeList.Items, podRequestedResource, 0.5) defer cleanUp() framework.ExpectNoError(err) ginkgo.By("Create a RC, with 0 replicas") rc := createRC(ns, "scheduler-priority-avoid-pod", int32(0), map[string]string{"name": "scheduler-priority-avoid-pod"}, f, podRequestedResource) // Cleanup the replication controller when we are done. defer func() { // Resize the replication controller to zero to get rid of pods. if err := e2erc.DeleteRCAndWaitForGC(f.ClientSet, f.Namespace.Name, rc.Name); err != nil { framework.Logf("Failed to cleanup replication controller %v: %v.", rc.Name, err) } }() ginkgo.By("Trying to apply avoidPod annotations on the first node.") avoidPod := v1.AvoidPods{ PreferAvoidPods: []v1.PreferAvoidPodsEntry{ { PodSignature: v1.PodSignature{ PodController: &metav1.OwnerReference{ APIVersion: "v1", Kind:       "ReplicationController", Name:       rc.Name, UID:        rc.UID, Controller: func() *bool { b := true; return &b }(), }, }, Reason:  "some reson", Message: "some message", }, }, } action := func() error { addOrUpdateAvoidPodOnNode(cs, nodeName, avoidPod) return nil } predicate := func(node *v1.Node) bool { val, err := json.Marshal(avoidPod) if err != nil { return false } return node.Annotations[v1.PreferAvoidPodsAnnotationKey] == string(val) } success, err := observeNodeUpdateAfterAction(f.ClientSet, nodeName, predicate, action) framework.ExpectNoError(err) framework.ExpectEqual(success, true) defer removeAvoidPodsOffNode(cs, nodeName) ginkgo.By(fmt.Sprintf("Scale the RC: %s to len(nodeList.Item)-1 : %v.", rc.Name, len(nodeList.Items)-1)) e2erc.ScaleRC(f.ClientSet, f.ScalesGetter, ns, rc.Name, uint(len(nodeList.Items)-1), true) testPods, err := cs.CoreV1().Pods(ns).List(context.TODO(), metav1.ListOptions{ LabelSelector: "name=scheduler-priority-avoid-pod", }) framework.ExpectNoError(err) ginkgo.By(fmt.Sprintf("Verify the pods should not scheduled to the node: %s", nodeName)) for _, pod := range testPods.Items { framework.ExpectNotEqual(pod.Spec.NodeName, nodeName) } }) </del> ginkgo.It("Pod should be preferably scheduled to nodes pod can tolerate", func() { // make the nodes have balanced cpu,mem usage ratio cleanUp, err := createBalancedPodForNodes(f, cs, ns, nodeList.Items, podRequestedResource, 0.5)

return result } <del> func createRC(ns, rsName string, replicas int32, rcPodLabels map[string]string, f *framework.Framework, resource *v1.ResourceRequirements) *v1.ReplicationController { rc := &v1.ReplicationController{ TypeMeta: metav1.TypeMeta{ Kind:       "ReplicationController", APIVersion: "v1", }, ObjectMeta: metav1.ObjectMeta{ Name: rsName, }, Spec: v1.ReplicationControllerSpec{ Replicas: &replicas, Template: &v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: rcPodLabels, }, Spec: v1.PodSpec{ Containers: []v1.Container{ { Name:      rsName, Image:     imageutils.GetPauseImageName(), Resources: *resource, }, }, }, }, }, } rc, err := f.ClientSet.CoreV1().ReplicationControllers(ns).Create(context.TODO(), rc, metav1.CreateOptions{}) framework.ExpectNoError(err) return rc } </del> func getRandomTaint() v1.Taint { return v1.Taint{ Key:    fmt.Sprintf("kubernetes.io/e2e-scheduling-priorities-%s", string(uuid.NewUUID()[:23])),