Change:
https://prow.ppc64le- TestStatefulSetControl/CreatePodFailure/Burst/ScaleDownOnly/StatefulSetAutoDeletePVCEnabled of Have started observing more flakes recently since last week. No response_ Able to reproduce using stress tool. /sig testing /sig apps
/triage accepted /assign
opened a PR with a fix:

Discussion:
"context" "errors" "fmt" <ins> "math" </ins> "math/rand" "reflect" "runtime"

err      error after    int <del> parallelLock sync.Mutex parallel     int maxParallel  int delay time.Duration </del> <ins> // this block should be updated consistently parallelLock                sync.Mutex shouldTrackParallelRequests bool parallelRequests            int maxParallelRequests         int parallelRequestDelay        time.Duration </ins> } <del> func (rt *requestTracker) errorReady() bool { rt.Lock() defer rt.Unlock() return rt.err != nil && rt.requests >= rt.after } func (rt *requestTracker) inc() { rt.parallelLock.Lock() rt.parallel++ if rt.maxParallel < rt.parallel { rt.maxParallel = rt.parallel </del> <ins> func (rt *requestTracker) trackParallelRequests() { if !rt.shouldTrackParallelRequests { // do not track parallel requests unless specifically enabled return </ins> } <del> rt.parallelLock.Unlock() rt.Lock() defer rt.Unlock() rt.requests++ if rt.delay != 0 { time.Sleep(rt.delay) </del> <ins> if rt.parallelLock.TryLock() { // lock acquired: we are the only or the first concurrent request // initialize the next set of parallel requests rt.parallelRequests = 1 } else { // lock is held by other requests // now wait for the lock to increase the parallelRequests rt.parallelLock.Lock() rt.parallelRequests++ } defer rt.parallelLock.Unlock() // update the local maximum of parallel collisions if rt.maxParallelRequests < rt.parallelRequests { rt.maxParallelRequests = rt.parallelRequests } // increase the chance of collisions if rt.parallelRequestDelay > 0 { time.Sleep(rt.parallelRequestDelay) </ins> } } <del> func (rt *requestTracker) reset() { rt.parallelLock.Lock() rt.parallel = 0 rt.parallelLock.Unlock() rt.Lock() defer rt.Unlock() rt.err = nil rt.after = 0 rt.delay = 0 } func (rt *requestTracker) getErr() error { </del> <ins> func (rt *requestTracker) incWithOptionalError() error { </ins> rt.Lock() defer rt.Unlock() <del> return rt.err </del> <ins> rt.requests++ if rt.err != nil && rt.requests >= rt.after { // reset and pass the error defer func() { rt.err = nil rt.after = 0 }() return rt.err } return nil </ins> } func newRequestTracker(requests int, err error, after int) requestTracker {

} func (om *fakeObjectManager) CreatePod(ctx context.Context, pod *v1.Pod) error { <del> defer om.createPodTracker.inc() if om.createPodTracker.errorReady() { defer om.createPodTracker.reset() return om.createPodTracker.getErr() </del> <ins> defer om.createPodTracker.trackParallelRequests() if err := om.createPodTracker.incWithOptionalError(); err != nil { return err </ins> } pod.SetUID(types.UID(pod.Name + "-uid")) return om.podsIndexer.Update(pod)

} func (om *fakeObjectManager) UpdatePod(pod *v1.Pod) error { <del> defer om.updatePodTracker.inc() if om.updatePodTracker.errorReady() { defer om.updatePodTracker.reset() return om.updatePodTracker.getErr() </del> <ins> defer om.updatePodTracker.trackParallelRequests() if err := om.updatePodTracker.incWithOptionalError(); err != nil { return err </ins> } return om.podsIndexer.Update(pod) } func (om *fakeObjectManager) DeletePod(pod *v1.Pod) error { <del> defer om.deletePodTracker.inc() if om.deletePodTracker.errorReady() { defer om.deletePodTracker.reset() return om.deletePodTracker.getErr() </del> <ins> defer om.deletePodTracker.trackParallelRequests() if err := om.deletePodTracker.incWithOptionalError(); err != nil { return err </ins> } if key, err := controller.KeyFunc(pod); err != nil { return err

} func (ssu *fakeStatefulSetStatusUpdater) UpdateStatefulSetStatus(ctx context.Context, set *apps.StatefulSet, status *apps.StatefulSetStatus) error { <del> defer ssu.updateStatusTracker.inc() if ssu.updateStatusTracker.errorReady() { defer ssu.updateStatusTracker.reset() return ssu.updateStatusTracker.err </del> <ins> defer ssu.updateStatusTracker.trackParallelRequests() if err := ssu.updateStatusTracker.incWithOptionalError(); err != nil { return err </ins> } set.Status = *status ssu.setsIndexer.Update(set)

obj.SetResourceVersion(strconv.FormatInt(intValue+1, 10)) } } func TestParallelScale(t *testing.T) { for _, tc := range []struct { <del> desc            string replicas        int32 desiredReplicas int32 </del> <ins> desc                        string replicas                    int32 desiredReplicas             int32 expectedMinParallelRequests int </ins> }{ { <del> desc:            "scale up from 3 to 30", replicas:        3, desiredReplicas: 30, </del> <ins> desc:                        "scale up from 3 to 30", replicas:                    3, desiredReplicas:             30, expectedMinParallelRequests: 2, </ins> }, { <del> desc:            "scale down from 10 to 1", replicas:        10, desiredReplicas: 1, </del> <ins> desc:                        "scale down from 10 to 1", replicas:                    10, desiredReplicas:             1, expectedMinParallelRequests: 2, </ins> }, { <del> desc:            "scale down to 0", replicas:        501, desiredReplicas: 0, </del> <ins> desc:                        "scale down to 0", replicas:                    501, desiredReplicas:             0, expectedMinParallelRequests: 10, </ins> }, { <del> desc:            "scale up from 0", replicas:        0, desiredReplicas: 1000, </del> <ins> desc:                        "scale up from 0", replicas:                    0, desiredReplicas:             1000, expectedMinParallelRequests: 20, </ins> }, } { t.Run(tc.desc, func(t *testing.T) { set := burst(newStatefulSet(0)) set.Spec.VolumeClaimTemplates[0].ObjectMeta.Labels = map[string]string{"test": "test"} <del> parallelScale(t, set, tc.replicas, tc.desiredReplicas, assertBurstInvariants) </del> <ins> parallelScale(t, set, tc.replicas, tc.desiredReplicas, tc.expectedMinParallelRequests, assertBurstInvariants) </ins> }) } } <del> func parallelScale(t *testing.T, set *apps.StatefulSet, replicas, desiredReplicas int32, invariants invariantFunc) { </del> <ins> func parallelScale(t *testing.T, set *apps.StatefulSet, replicas, desiredReplicas int32, expectedMinParallelRequests int, invariants invariantFunc) { </ins> var err error diff := desiredReplicas - replicas <ins> // maxParallelRequests: MaxBatchSize of the controller is 500, We divide the diff by 4 to allow maximum of the half of the last batch. if maxParallelRequests := min(500, math.Abs(float64(diff))/4); expectedMinParallelRequests < 2 || float64(expectedMinParallelRequests) > maxParallelRequests { t.Fatalf("expectedMinParallelRequests should be between 2 and %v. Batch size of the controller is expontially increasing until 500. "+ "Got expectedMinParallelRequests %v, ", maxParallelRequests, expectedMinParallelRequests) } </ins> client := fake.NewSimpleClientset(set) om, _, ssc := setupController(client) <del> om.createPodTracker.delay = time.Millisecond </del> <ins> om.createPodTracker.shouldTrackParallelRequests = true om.createPodTracker.parallelRequestDelay = time.Millisecond </ins> *set.Spec.Replicas = replicas if err := parallelScaleUpStatefulSetControl(set, ssc, om, invariants); err != nil {

t.Errorf("Failed to scale statefulset to %v replicas, got %v replicas", desiredReplicas, set.Status.Replicas) } <del> if (diff < -1 || diff > 1) && om.createPodTracker.maxParallel <= 1 { t.Errorf("want max parallel requests > 1, got %v", om.createPodTracker.maxParallel) </del> <ins> if om.createPodTracker.maxParallelRequests < expectedMinParallelRequests { t.Errorf("want max parallelRequests requests >= %v, got %v", expectedMinParallelRequests, om.createPodTracker.maxParallelRequests) </ins> } }