Code change:
/sig scheduling /kind regression caused a performance degradation in the 5k nodes scalability tests ! The PR changed how to split the load among parallel threads. Previously, we would parallelize over the number of nodes. The PR does it over the number of plugins, which is much smaller. Performance to improve or remain the same https://perf- No response 28 N/A <details</details<details</details<details</details<details</details
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/assign The quickest fix is to revert, but I'm open to other suggestions.
/priority critical-urgent
Can we just use another Parallelizer to handle the score process for each plugin? I ran the UT locally and it seems fine, but I'm not familar with
Depending on how many cores you have available, you might just overload the routines. I think it's better to revert the parallizer over plugins while keeping the change from map to slice. Although I don't expect a major impact for the latter. You might want to run a profiler for the benchmark with a higher number of nodes.
I'll create a revert version first :)
Let's revert this first since we reached the code freeze. We can do the refactor later.
Sorry for causing this trouble and thanks everyone for a prompt action

Related discussion:
text: allNodePluginScores := make([]framework.NodePluginScores, len(nodes)) numPlugins := len(f.scorePlugins) - state.SkipScorePlugins.Len() plugins := make([]framework.ScorePlugin, 0, numPlugins) <del> pluginToNodeScores := make([]framework.NodeScoreList, numPlugins) </del> <ins> pluginToNodeScores := make(map[string]framework.NodeScoreList, numPlugins) </ins> for _, pl := range f.scorePlugins { if state.SkipScorePlugins.Has(pl.Name()) { continue } plugins = append(plugins, pl) <ins> pluginToNodeScores[pl.Name()] = make(framework.NodeScoreList, len(nodes)) </ins> } ctx, cancel := context.WithCancel(ctx) defer cancel() errCh := parallelize.NewErrorChannel() <del> logger := klog.FromContext(ctx) logger = klog.LoggerWithName(logger, "Score") // TODO(knelasevero): Remove duplicated keys from log entry calls // When contextualized logging hits GA // https://github.com/kubernetes/kubernetes/issues/111672 logger = klog.LoggerWithValues(logger, "pod", klog.KObj(pod)) // Run Score method for each node in parallel. f.Parallelizer().Until(ctx, len(plugins), func(i int) { pl := plugins[i] logger := klog.LoggerWithName(logger, pl.Name()) nodeScores := make(framework.NodeScoreList, len(nodes)) for index, node := range nodes { nodeName := node.Name </del> <ins> if len(plugins) > 0 { logger := klog.FromContext(ctx) logger = klog.LoggerWithName(logger, "Score") // TODO(knelasevero): Remove duplicated keys from log entry calls // When contextualized logging hits GA // https://github.com/kubernetes/kubernetes/issues/111672 logger = klog.LoggerWithValues(logger, "pod", klog.KObj(pod)) // Run Score method for each node in parallel. f.Parallelizer().Until(ctx, len(nodes), func(index int) { nodeName := nodes[index].Name </ins> logger := klog.LoggerWithValues(logger, "node", klog.ObjectRef{Name: nodeName}) <del> ctx := klog.NewContext(ctx, logger) s, status := f.runScorePlugin(ctx, pl, state, pod, nodeName) if !status.IsSuccess() { err := fmt.Errorf("plugin %q failed with: %w", pl.Name(), status.AsError()) errCh.SendErrorWithCancel(err, cancel) return } nodeScores[index] = framework.NodeScore{ Name:  nodeName, Score: s, </del> <ins> for _, pl := range plugins { logger := klog.LoggerWithName(logger, pl.Name()) ctx := klog.NewContext(ctx, logger) s, status := f.runScorePlugin(ctx, pl, state, pod, nodeName) if !status.IsSuccess() { err := fmt.Errorf("plugin %q failed with: %w", pl.Name(), status.AsError()) errCh.SendErrorWithCancel(err, cancel) return } pluginToNodeScores[pl.Name()][index] = framework.NodeScore{ Name:  nodeName, Score: s, } </ins> } <ins> }, metrics.Score) if err := errCh.ReceiveError(); err != nil { return nil, framework.AsStatus(fmt.Errorf("running Score plugins: %w", err)) </ins> } <ins> } // Run NormalizeScore method for each ScorePlugin in parallel. f.Parallelizer().Until(ctx, len(plugins), func(index int) { pl := plugins[index] </ins> if pl.ScoreExtensions() == nil { <del> pluginToNodeScores[i] = nodeScores </del> return } <del> status := f.runScoreExtension(ctx, pl, state, pod, nodeScores) </del> <ins> nodeScoreList := pluginToNodeScores[pl.Name()] status := f.runScoreExtension(ctx, pl, state, pod, nodeScoreList) </ins> if !status.IsSuccess() { err := fmt.Errorf("plugin %q failed with: %w", pl.Name(), status.AsError()) errCh.SendErrorWithCancel(err, cancel) return } <del> pluginToNodeScores[i] = nodeScores </del> }, metrics.Score) if err := errCh.ReceiveError(); err != nil { return nil, framework.AsStatus(fmt.Errorf("running Normalize on Score plugins: %w", err))