Code change:
expectMetrics(t, expectedMetrics{desiredSlices: 1, actualSlices: 1, desiredEndpoints: 10, addedPerSync: 20, removedPerSync: 10, numCreated: 1, numUpdated: 1, numDeleted: 0}) } <ins> // When a Service has a non-nil deletionTimestamp we want to avoid creating any // new EndpointSlices but continue to allow updates and deletes through. This // test uses 3 EndpointSlices, 1 "to-create", 1 "to-update", and 1 "to-delete". // Each test case exercises different combinations of calls to finalize with // those resources. func TestReconcilerFinalizeSvcDeletionTimestamp(t *testing.T) { now := metav1.Now() testCases := []struct { name               string deletionTimestamp  *metav1.Time attemptCreate      bool attemptUpdate      bool attemptDelete      bool expectCreatedSlice bool expectUpdatedSlice bool expectDeletedSlice bool }{{ name:               "Attempt create and update, nil deletion timestamp", deletionTimestamp:  nil, attemptCreate:      true, attemptUpdate:      true, expectCreatedSlice: true, expectUpdatedSlice: true, expectDeletedSlice: true, }, { name:               "Attempt create and update, deletion timestamp set", deletionTimestamp:  &now, attemptCreate:      true, attemptUpdate:      true, expectCreatedSlice: false, expectUpdatedSlice: true, expectDeletedSlice: true, }, { // Slice scheduled for creation is transitioned to update of Slice // scheduled for deletion. name:               "Attempt create, update, and delete, nil deletion timestamp, recycling in action", deletionTimestamp:  nil, attemptCreate:      true, attemptUpdate:      true, attemptDelete:      true, expectCreatedSlice: false, expectUpdatedSlice: true, expectDeletedSlice: true, }, { // Slice scheduled for creation is transitioned to update of Slice // scheduled for deletion. name:               "Attempt create, update, and delete, deletion timestamp set, recycling in action", deletionTimestamp:  &now, attemptCreate:      true, attemptUpdate:      true, attemptDelete:      true, expectCreatedSlice: false, expectUpdatedSlice: true, expectDeletedSlice: true, }, { // Update and delete continue to work when deletionTimestamp is set. name:               "Attempt update delete, deletion timestamp set", deletionTimestamp:  &now, attemptCreate:      false, attemptUpdate:      true, attemptDelete:      true, expectCreatedSlice: false, expectUpdatedSlice: true, expectDeletedSlice: false, }} for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { client := newClientset() setupMetrics() r := newReconciler(client, []*corev1.Node{{ObjectMeta: metav1.ObjectMeta{Name: "node-1"}}}, defaultMaxEndpointsPerSlice) namespace := "test" svc, endpointMeta := newServiceAndEndpointMeta("foo", namespace) svc.DeletionTimestamp = tc.deletionTimestamp esToCreate := &discovery.EndpointSlice{ ObjectMeta:  metav1.ObjectMeta{Name: "to-create"}, AddressType: endpointMeta.AddressType, Ports:       endpointMeta.Ports, } // Add EndpointSlice that can be updated. esToUpdate, err := client.DiscoveryV1beta1().EndpointSlices(namespace).Create(context.TODO(), &discovery.EndpointSlice{ ObjectMeta:  metav1.ObjectMeta{Name: "to-update"}, AddressType: endpointMeta.AddressType, Ports:       endpointMeta.Ports, }, metav1.CreateOptions{}) if err != nil { t.Fatalf("Expected no error creating EndpointSlice during test setup, got %v", err) } // Add an endpoint so we can see if this has actually been updated by // finalize func. esToUpdate.Endpoints = []discovery.Endpoint{{Addresses: []string{"10.2.3.4"}}} // Add EndpointSlice that can be deleted. esToDelete, err := client.DiscoveryV1beta1().EndpointSlices(namespace).Create(context.TODO(), &discovery.EndpointSlice{ ObjectMeta:  metav1.ObjectMeta{Name: "to-delete"}, AddressType: endpointMeta.AddressType, Ports:       endpointMeta.Ports, }, metav1.CreateOptions{}) if err != nil { t.Fatalf("Expected no error creating EndpointSlice during test setup, got %v", err) } slicesToCreate := []*discovery.EndpointSlice{} if tc.attemptCreate { slicesToCreate = append(slicesToCreate, esToCreate.DeepCopy()) } slicesToUpdate := []*discovery.EndpointSlice{} if tc.attemptUpdate { slicesToUpdate = append(slicesToUpdate, esToUpdate.DeepCopy()) } slicesToDelete := []*discovery.EndpointSlice{} if tc.attemptDelete { slicesToDelete = append(slicesToDelete, esToDelete.DeepCopy()) } err = r.finalize(&svc, slicesToCreate, slicesToUpdate, slicesToDelete, time.Now()) if err != nil { t.Errorf("Error calling r.finalize(): %v", err) } fetchedSlices := fetchEndpointSlices(t, client, namespace) createdSliceFound := false updatedSliceFound := false deletedSliceFound := false for _, epSlice := range fetchedSlices { if epSlice.Name == esToCreate.Name { createdSliceFound = true } if epSlice.Name == esToUpdate.Name { updatedSliceFound = true if tc.attemptUpdate && len(epSlice.Endpoints) != len(esToUpdate.Endpoints) { t.Errorf("Expected EndpointSlice to be updated with %d endpoints, got %d endpoints", len(esToUpdate.Endpoints), len(epSlice.Endpoints)) } } if epSlice.Name == esToDelete.Name { deletedSliceFound = true } } if createdSliceFound != tc.expectCreatedSlice { t.Errorf("Expected created EndpointSlice existence to be %t, got %t", tc.expectCreatedSlice, createdSliceFound) } if updatedSliceFound != tc.expectUpdatedSlice { t.Errorf("Expected updated EndpointSlice existence to be %t, got %t", tc.expectUpdatedSlice, updatedSliceFound) } if deletedSliceFound != tc.expectDeletedSlice { t.Errorf("Expected deleted EndpointSlice existence to be %t, got %t", tc.expectDeletedSlice, deletedSliceFound) } }) } } </ins> // Test Helpers func newReconciler(client *fake.Clientset, nodes []*corev1.Node, maxEndpointsPerSlice int32) *reconciler {

Related discussion:
text: One peculiarity may be that we don't actually want the pods to start during these tests because we only want to verify object creation, not the usage of the result, and therefore all pods reference non-existent images. All objects immediately get deleted once the test has verified that they exist, so the timing is probably very different from "normal" usage patterns. I suspect that it is more important to create and delete services quickly than it is to have many endpointslices per service.
I had tried to come up with a script that triggers the problem, without success. That was before I realized that endpointslices are involved. On Monday I can have another look.
That was because I thought that (the default) enables foreground deletion. It doesn't, and there's also no option to select it. After explicitly selecting it via curl, the following script failed for me eventually: Output: Note that for the endpointslice is greater than the of its service.