Change:
When Extender filters out some Nodes, we don't set any unschedulable plugins at all. It means Extender is completely ignored during the requeueing process. So, what's happening is: If Extender filters out all Nodes during scheduling, this Pod is soon retried because this Pod doesn't have any plugin name in unschedulable plugins. If Extender filters out some Nodes during scheduling and plugins filter out all other Nodes, this Pod is retried based on plugins' QueueingHint. Even if any cluster events happen and it could change Extender's decision (but not any plugins' decision), this Pod won't be requeued to activeQ/backoffQ. The latter case is serious because it could make Pods being stuck in unschedulable pod pool in 5min. We should have a short-term solution for the latter case. We can requeue Pods, which were rejected by Extender, by any kind of cluster events because we cannot know which events make Pods schedulable. Eventually, this issue makes us wonder how Pods rejected by Extender should be requeued. We cannot implement QHint equivalent in Extenders because it'd be too slow to call Extender every time any cluster events happen. Probably, somehow implementing EventsToRegister equivalent in Extender? Use Extender which does something in Filter. No response master <detailsn/a </details<details</details<details</details<details</details<details</details
/assign
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig scheduling
/reopen We'll close it when all cherry-picks are done.
Reopened this issue. <detailsIn response to : /reopen Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/close Cherry-picks are done.

Discussion:
logger:               logger, } <ins> if len(f.extenders) > 0 { // Extender doesn't support any kind of requeueing feature like EnqueueExtensions in the scheduling framework. // We register a defaultEnqueueExtension to framework.ExtenderName here. // And, in the scheduling cycle, when Extenders reject some Nodes and the pod ends up being unschedulable, // we put framework.ExtenderName to pInfo.UnschedulablePlugins. f.enqueueExtensions = []framework.EnqueueExtensions{&defaultEnqueueExtension{pluginName: framework.ExtenderName}} } </ins> if profile == nil { return f, nil }

Weight int32 } <ins> // ExtenderName is a fake plugin name put in UnschedulablePlugins when Extender rejected some Nodes. const ExtenderName = "Extender" </ins> // Diagnosis records the details to diagnose a scheduling failure. type Diagnosis struct { NodeToStatusMap NodeToStatusMap

return nil, diagnosis, err } <del> feasibleNodes, err = findNodesThatPassExtenders(ctx, sched.Extenders, pod, feasibleNodes, diagnosis.NodeToStatusMap) </del> <ins> feasibleNodesAfterExtender, err := findNodesThatPassExtenders(ctx, sched.Extenders, pod, feasibleNodes, diagnosis.NodeToStatusMap) </ins> if err != nil { return nil, diagnosis, err } <del> return feasibleNodes, diagnosis, nil </del> <ins> if len(feasibleNodesAfterExtender) != len(feasibleNodes) { // Extenders filtered out some nodes. // // Extender doesn't support any kind of requeueing feature like EnqueueExtensions in the scheduling framework. // When Extenders reject some Nodes and the pod ends up being unschedulable, // we put framework.ExtenderName to pInfo.UnschedulablePlugins. // This Pod will be requeued from unschedulable pod pool to activeQ/backoffQ // by any kind of cluster events. // https://github.com/kubernetes/kubernetes/issues/122019 if diagnosis.UnschedulablePlugins == nil { diagnosis.UnschedulablePlugins = sets.New[string]() } diagnosis.UnschedulablePlugins.Insert(framework.ExtenderName) } return feasibleNodesAfterExtender, diagnosis, nil </ins> } func (sched *Scheduler) evaluateNominatedNode(ctx context.Context, pod *v1.Pod, fwk framework.Framework, state *framework.CycleState, diagnosis framework.Diagnosis) ([]*v1.Node, error) {

tests := []struct { name               string registerPlugins    []tf.RegisterPluginFunc <ins> extenders          []tf.FakeExtender </ins> nodes              []string pvcs               []v1.PersistentVolumeClaim pod                *v1.Pod

}, }, { <ins> name: "test with extender which filters out some Nodes", registerPlugins: []tf.RegisterPluginFunc{ tf.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), tf.RegisterFilterPlugin( "FakeFilter", tf.NewFakeFilterPlugin(map[string]framework.Code{"3": framework.Unschedulable}), ), tf.RegisterScorePlugin("NumericMap", newNumericMapPlugin(), 1), tf.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, extenders: []tf.FakeExtender{ { ExtenderName: "FakeExtender1", Predicates:   []tf.FitPredicate{tf.FalsePredicateExtender}, }, }, nodes:     []string{"1", "2", "3"}, pod:       st.MakePod().Name("test-filter").UID("test-filter").Obj(), wantNodes: nil, wErr: &framework.FitError{ Pod:         st.MakePod().Name("test-filter").UID("test-filter").Obj(), NumAllNodes: 3, Diagnosis: framework.Diagnosis{ NodeToStatusMap: framework.NodeToStatusMap{ "1": framework.NewStatus(framework.Unschedulable, `FakeExtender: node "1" failed`), "2": framework.NewStatus(framework.Unschedulable, `FakeExtender: node "2" failed`), "3": framework.NewStatus(framework.Unschedulable, "injecting failure for pod test-filter").WithPlugin("FakeFilter"), }, UnschedulablePlugins: sets.New("FakeFilter", framework.ExtenderName), }, }, }, { </ins> name: "test with filter plugin returning UnschedulableAndUnresolvable status", registerPlugins: []tf.RegisterPluginFunc{ tf.RegisterQueueSortPlugin(queuesort.Name, queuesort.New),

t.Fatal(err) } <ins> var extenders []framework.Extender for ii := range test.extenders { extenders = append(extenders, &test.extenders[ii]) } </ins> sched := &Scheduler{ Cache:                    cache, nodeInfoSnapshot:         snapshot, percentageOfNodesToScore: schedulerapi.DefaultPercentageOfNodesToScore, <ins> Extenders:                extenders, </ins> } sched.applyDefaultHandlers()

gotFitErr, gotOK := err.(*framework.FitError) wantFitErr, wantOK := test.wErr.(*framework.FitError) if gotOK != wantOK { <del> t.Errorf("Expected err to be FitError: %v, but got %v", wantOK, gotOK) </del> <ins> t.Errorf("Expected err to be FitError: %v, but got %v (error: %v)", wantOK, gotOK, err) </ins> } else if gotOK { if diff := cmp.Diff(gotFitErr, wantFitErr); diff != "" { t.Errorf("Unexpected fitErr: (-want, +got): %s", diff)