Change:
Test: [sig-node] Density [Serial] [Slow] create a sequence of pods latency/resource should be within limit when create 10 pods with 50 background pods /kind flake <!-- If this is a flake, please add: /kind flake --/sig node
/triage accepted /priority important-soon
Although the pod polling exhausted 4 times (interval 2s, total 6s+) to get pod Running, but from the kubelet log, the time from the pod found to Running status reported is almost less than 5s, so wo may reduce the poll interval to 1s, or make this time more relaxed
http://perf- ! This is caused by as . The commit will be reverted, and it make fail yesterday. ! EDITED I mean if this flake points out there are some other performance downgrades in v1.24. Only recently, no version 99% -- -- v1.20 ~2200ms v1.21 ~2200ms v1.22 ~2300ms v1.23 ~2650ms v1.24(before) ~2200ms v1.24(after ) ~6600ms
Editing title as this was also failing on containerd: /milestone v1.24 I think this confirms that the revert above fixed the issue, as the failures stopped after the revert merged. /close
Closing this issue. <detailsIn response to : Editing title as this was also failing on containerd: Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details

Discussion:
"sync" "time" <ins> "k8s.io/apimachinery/pkg/util/wait" "github.com/onsi/ginkgo" "github.com/onsi/gomega" </ins> v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels"

e2emetrics "k8s.io/kubernetes/test/e2e/framework/metrics" imageutils "k8s.io/kubernetes/test/utils/image" admissionapi "k8s.io/pod-security-admission/api" <del> "github.com/onsi/ginkgo" "github.com/onsi/gomega" </del> ) const (

rc.Start() // Create pods sequentially (back-to-back). e2eLags have been sorted. <del> batchlag, e2eLags := createBatchPodSequential(f, testPods) </del> <ins> batchlag, e2eLags := createBatchPodSequential(f, testPods, podType) </ins> rc.Stop() deletePodsSync(f, append(bgPods, testPods...))

} // createBatchPodSequential creates pods back-to-back in sequence. <del> func createBatchPodSequential(f *framework.Framework, pods []*v1.Pod) (time.Duration, []e2emetrics.PodLatencyData) { </del> <ins> func createBatchPodSequential(f *framework.Framework, pods []*v1.Pod, podType string) (time.Duration, []e2emetrics.PodLatencyData) { var ( mutex       = &sync.Mutex{} watchTimes  = make(map[string]metav1.Time, 0) stopCh      = make(chan struct{}) firstCreate metav1.Time lastRunning metav1.Time init        = true ) // the controller watches the change of pod status controller := newInformerWatchPod(f, mutex, watchTimes, podType) go controller.Run(stopCh) defer close(stopCh) </ins> batchStartTime := metav1.Now() e2eLags := make([]e2emetrics.PodLatencyData, 0) <ins> createTimes := make(map[string]metav1.Time) </ins> for _, pod := range pods { create := metav1.Now() <del> f.PodClient().CreateSync(pod) </del> <ins> createTimes[pod.Name] = create p := f.PodClient().Create(pod) framework.ExpectNoError(wait.PollImmediate(2*time.Second, framework.PodStartTimeout, podWatchedRunning(watchTimes, p.Name))) </ins> e2eLags = append(e2eLags, <del> e2emetrics.PodLatencyData{Name: pod.Name, Latency: metav1.Now().Time.Sub(create.Time)}) </del> <ins> e2emetrics.PodLatencyData{Name: pod.Name, Latency: watchTimes[pod.Name].Time.Sub(create.Time)}) </ins> } <del> batchLag := metav1.Now().Time.Sub(batchStartTime.Time) </del> <ins> for name, create := range createTimes { watch, ok := watchTimes[name] framework.ExpectEqual(ok, true) if !init { if firstCreate.Time.After(create.Time) { firstCreate = create } if lastRunning.Time.Before(watch.Time) { lastRunning = watch } } else { init = false firstCreate, lastRunning = create, watch } } batchLag := lastRunning.Time.Sub(batchStartTime.Time) </ins> sort.Sort(e2emetrics.LatencySlice(e2eLags)) return batchLag, e2eLags } <ins> // podWatchedRunning verifies whether the pod becomes Running, as the watchTime was set by informer func podWatchedRunning(watchTimes map[string]metav1.Time, podName string) wait.ConditionFunc { return func() (done bool, err error) { if _, found := watchTimes[podName]; found { return true, nil } return false, nil } } </ins> // verifyLatencyWithinThreshold verifies whether 50, 90 and 99th percentiles of a latency metric are // within the expected threshold. func verifyLatencyWithinThreshold(threshold, actual e2emetrics.LatencyMetric, metricName string) error {