Change:
Which jobs are failing: TestExpectationsOnRecreate unit test, in along with expectations bugfix during 1.19 and cherry-picked to 1.18/1.17 Which test(s) are failing: Reason for failure: Anything else we need to know: These failures are masked in CI because bazel test runs with flake retries (), so the test can fail 2/3 times and the job can still pass (cc
adding to the milestone for triage, in case this is a real bug and not just a test issue /milestone v1.19
cc
/assign
looks like the panic is due to the test continuing in the !exists case (rather than exiting with a Fatal): however, that still means the test is failing in three distinct places:
it looks like the same failure existed in 1.18, so this probably isn't release blocking, but it does need to be resolved in order for us to stop ignoring unit test flakes in CI runs
it actually looks like this test was during the 1.19 dev cycle and cherry-picked back to 1.18/1.17, which is why release-1.18 is failing :-/
figured it out... the controller sync is updating the DS, which is triggering an update event and adding an extra enqueue which the test did not account for good news is that it appears to be a test-only issue
possible test deflake in

Discussion:
t.Fatal(err) } <ins> expectStableQueueLength := func(expected int) { t.Helper() for i := 0; i < 5; i++ { if actual := dsc.queue.Len(); actual != expected { t.Fatalf("expected queue len to remain at %d, got %d", expected, actual) } time.Sleep(10 * time.Millisecond) } } waitForQueueLength := func(expected int, msg string) { t.Helper() i := 0 err = wait.PollImmediate(100*time.Millisecond, informerSyncTimeout, func() (bool, error) { current := dsc.queue.Len() switch { case current == expected: return true, nil case current > expected: return false, fmt.Errorf("queue length %d exceeded expected length %d", current, expected) default: i++ if i > 1 { t.Logf("Waiting for queue to have %d item, currently has: %d", expected, current) } return false, nil } }) if err != nil { t.Fatalf("%s: %v", msg, err) } expectStableQueueLength(expected) } </ins> fakeRecorder := record.NewFakeRecorder(100) dsc.eventRecorder = fakeRecorder

t.Fatal("caches failed to sync") } <del> if dsc.queue.Len() != 0 { t.Fatal("Unexpected item in the queue") } </del> <ins> expectStableQueueLength(0) </ins> oldDS := newDaemonSet("test") oldDS, err = client.AppsV1().DaemonSets(oldDS.Namespace).Create(context.Background(), oldDS, metav1.CreateOptions{})

t.Fatal(err) } <del> err = wait.PollImmediate(100*time.Millisecond, informerSyncTimeout, func() (bool, error) { klog.V(8).Infof("Waiting for queue to have 1 item, currently has: %d", dsc.queue.Len()) return dsc.queue.Len() == 1, nil }) if err != nil { t.Fatalf("initial DS didn't result in new item in the queue: %v", err) } </del> <ins> // create of DS adds to queue, processes waitForQueueLength(1, "created DS") </ins> ok = dsc.processNextWorkItem() if !ok { t.Fatal("queue is shutting down")

t.Fatal(err) } if !exists { <del> t.Errorf("No expectations found for DaemonSet %q", oldDSKey) </del> <ins> t.Fatalf("No expectations found for DaemonSet %q", oldDSKey) </ins> } if dsExp.Fulfilled() { t.Errorf("There should be unfulfiled expectation for creating new pods for DaemonSet %q", oldDSKey) } <del> if dsc.queue.Len() != 0 { t.Fatal("Unexpected item in the queue") </del> <ins> // process updates DS, update adds to queue waitForQueueLength(1, "updated DS") ok = dsc.processNextWorkItem() if !ok { t.Fatal("queue is shutting down") </ins> } <ins> // process does not re-update the DS expectStableQueueLength(0) </ins> err = client.AppsV1().DaemonSets(oldDS.Namespace).Delete(context.Background(), oldDS.Name, metav1.DeleteOptions{}) if err != nil { t.Fatal(err) } <del> err = wait.PollImmediate(100*time.Millisecond, informerSyncTimeout, func() (bool, error) { klog.V(8).Infof("Waiting for queue to have 1 item, currently has: %d", dsc.queue.Len()) return dsc.queue.Len() == 1, nil }) if err != nil { t.Fatalf("Deleting DS didn't result in new item in the queue: %v", err) } </del> <ins> waitForQueueLength(1, "deleted DS") </ins> _, exists, err = dsc.expectations.GetExpectations(oldDSKey) if err != nil {

t.Fatal("Keys should be equal!") } <del> if dsc.queue.Len() != 0 { t.Fatal("Unexpected item in the queue") } </del> <ins> expectStableQueueLength(0) </ins> newDS := oldDS.DeepCopy() newDS.UID = uuid.NewUUID()

t.Fatal("New DS has the same UID as the old one!") } <del> err = wait.PollImmediate(100*time.Millisecond, informerSyncTimeout, func() (bool, error) { klog.V(8).Infof("Waiting for queue to have 1 item, currently has: %d", dsc.queue.Len()) return dsc.queue.Len() == 1, nil }) if err != nil { t.Fatalf("Re-creating DS didn't result in new item in the queue: %v", err) } </del> <ins> waitForQueueLength(1, "recreated DS") </ins> ok = dsc.processNextWorkItem() if !ok { t.Fatal("Queue is shutting down!")

t.Fatal(err) } if !exists { <del> t.Errorf("No expectations found for DaemonSet %q", oldDSKey) </del> <ins> t.Fatalf("No expectations found for DaemonSet %q", oldDSKey) </ins> } if dsExp.Fulfilled() { t.Errorf("There should be unfulfiled expectation for creating new pods for DaemonSet %q", oldDSKey)