Change:
} func TestWatchDeleteEventObjectHaveLatestRV(t *testing.T) { <del> ctx, store, client := testSetup(t) </del> <ins> ctx, store, _ := testSetup(t) </ins> key, storedObj := storagetesting.TestPropagateStore(ctx, t, store, &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo"}}) <del> w, err := store.Watch(ctx, key, storage.ListOptions{ResourceVersion: storedObj.ResourceVersion, Predicate: storage.Everything}) </del> <ins> watchCtx, _ := context.WithTimeout(ctx, wait.ForeverTestTimeout) w, err := store.Watch(watchCtx, key, storage.ListOptions{ResourceVersion: storedObj.ResourceVersion, Predicate: storage.Everything}) </ins> if err != nil { t.Fatalf("Watch failed: %v", err) } <del> rv, err := storage.APIObjectVersioner{}.ObjectResourceVersion(storedObj) if err != nil { t.Fatalf("failed to parse resourceVersion on stored object: %v", err) } etcdW := client.Watch(ctx, key, clientv3.WithRev(int64(rv))) if err := store.Delete(ctx, key, &example.Pod{}, &storage.Preconditions{}, storage.ValidateAllObjectFunc, nil); err != nil { </del> <ins> deletedObj := &example.Pod{} if err := store.Delete(ctx, key, deletedObj, &storage.Preconditions{}, storage.ValidateAllObjectFunc, nil); err != nil { </ins> t.Fatalf("Delete failed: %v", err) } <del> var e watch.Event watchCtx, _ := context.WithTimeout(ctx, wait.ForeverTestTimeout) select { case e = <-w.ResultChan(): case <-watchCtx.Done(): t.Fatalf("timed out waiting for watch event") } deletedRV, err := deletedRevision(watchCtx, etcdW) if err != nil { t.Fatalf("did not see delete event in raw watch: %v", err) } watchedDeleteObj := e.Object.(*example.Pod) watchedDeleteRev, err := store.versioner.ParseResourceVersion(watchedDeleteObj.ResourceVersion) if err != nil { t.Fatalf("ParseWatchResourceVersion failed: %v", err) } if int64(watchedDeleteRev) != deletedRV { t.Errorf("Object from delete event have version: %v, should be the same as etcd delete's mod rev: %d", watchedDeleteRev, deletedRV) </del> <ins> // Verify that ResourceVersion has changed on deletion. if storedObj.ResourceVersion == deletedObj.ResourceVersion { t.Fatalf("ResourceVersion didn't changed on deletion: %s", deletedObj.ResourceVersion) </ins> } <del> } func deletedRevision(ctx context.Context, watch <-chan clientv3.WatchResponse) (int64, error) { for { select { case <-ctx.Done(): return 0, ctx.Err() case wres := <-watch: for _, evt := range wres.Events { if evt.Type == mvccpb.DELETE && evt.Kv != nil { return evt.Kv.ModRevision, nil } } </del> <ins> select { case event := <-w.ResultChan(): watchedDeleteObj := event.Object.(*example.Pod) if e, a := deletedObj.ResourceVersion, watchedDeleteObj.ResourceVersion; e != a { t.Errorf("Unexpected resource version: %v, expected %v", a, e) </ins> } } }

Discussion:
While working on generalizing tests to somehow better define spec for storage interface (and share them across all implementation of storage.interface - in particular etcd3 and watchcache), I found an unexpected inconsistency in Delete operation in etcd3. What is happening is that it effectively returns the state of the object from just before the deletion: However, this is inconsistent with how our watch is working, which is returning the state of the object from just before the deletion, but it sets the RV to the moment of deletion (which Delete call isn't doing). This sounds like a bug to me, but given this is a long-standing behavior we may need to take additional care when fixing this. /kind bug /sig api-machinery /priority important-soon /cc
I opened as a fix for it.
/triage accepted