Change:
While working on generalizing tests to somehow better define spec for storage interface (and share them across all implementation of storage.interface - in particular etcd3 and watchcache), I found an unexpected inconsistency in Delete operation in etcd3. What is happening is that it effectively returns the state of the object from just before the deletion: However, this is inconsistent with how our watch is working, which is returning the state of the object from just before the deletion, but it sets the RV to the moment of deletion (which Delete call isn't doing). This sounds like a bug to me, but given this is a long-standing behavior we may need to take additional care when fixing this. /kind bug /sig api-machinery /priority important-soon /cc
I opened as a fix for it.
/triage accepted

Discussion:
origStateIsCurrent = true continue } <del> return decode(s.codec, s.versioner, origState.data, out, origState.rev) </del> <ins> if len(txnResp.Responses) == 0 || txnResp.Responses[0].GetResponseDeleteRange() == nil { return errors.New(fmt.Sprintf("invalid DeleteRange response: %v", txnResp.Responses)) } deleteResp := txnResp.Responses[0].GetResponseDeleteRange() if deleteResp.Header == nil { return errors.New("invalid DeleteRange response - nil header") } return decode(s.codec, s.versioner, origState.data, out, deleteResp.Header.Revision) </ins> } }

"testing" "time" <del> "go.etcd.io/etcd/api/v3/mvccpb" clientv3 "go.etcd.io/etcd/client/v3" </del> storagetesting "k8s.io/apiserver/pkg/storage/testing" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

origCtx, store, _ := testSetup(t) ctx, cancel := context.WithCancel(origCtx) w := store.watcher.createWatchChan(ctx, "/abc", 0, false, false, newTestTransformer(), storage.Everything) <del> // make resutlChan and errChan blocking to ensure ordering. </del> <ins> // make resultChan and errChan blocking to ensure ordering. </ins> w.resultChan = make(chan watch.Event) w.errChan = make(chan error) // The event flow goes like:

} func TestWatchDeleteEventObjectHaveLatestRV(t *testing.T) { <del> ctx, store, client := testSetup(t) </del> <ins> ctx, store, _ := testSetup(t) </ins> key, storedObj := storagetesting.TestPropagateStore(ctx, t, store, &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo"}}) <del> w, err := store.Watch(ctx, key, storage.ListOptions{ResourceVersion: storedObj.ResourceVersion, Predicate: storage.Everything}) </del> <ins> watchCtx, _ := context.WithTimeout(ctx, wait.ForeverTestTimeout) w, err := store.Watch(watchCtx, key, storage.ListOptions{ResourceVersion: storedObj.ResourceVersion, Predicate: storage.Everything}) </ins> if err != nil { t.Fatalf("Watch failed: %v", err) } <del> rv, err := storage.APIObjectVersioner{}.ObjectResourceVersion(storedObj) if err != nil { t.Fatalf("failed to parse resourceVersion on stored object: %v", err) } etcdW := client.Watch(ctx, key, clientv3.WithRev(int64(rv))) if err := store.Delete(ctx, key, &example.Pod{}, &storage.Preconditions{}, storage.ValidateAllObjectFunc, nil); err != nil { </del> <ins> deletedObj := &example.Pod{} if err := store.Delete(ctx, key, deletedObj, &storage.Preconditions{}, storage.ValidateAllObjectFunc, nil); err != nil { </ins> t.Fatalf("Delete failed: %v", err) } <del> var e watch.Event watchCtx, _ := context.WithTimeout(ctx, wait.ForeverTestTimeout) select { case e = <-w.ResultChan(): case <-watchCtx.Done(): t.Fatalf("timed out waiting for watch event") } deletedRV, err := deletedRevision(watchCtx, etcdW) if err != nil { t.Fatalf("did not see delete event in raw watch: %v", err) } watchedDeleteObj := e.Object.(*example.Pod) watchedDeleteRev, err := store.versioner.ParseResourceVersion(watchedDeleteObj.ResourceVersion) if err != nil { t.Fatalf("ParseWatchResourceVersion failed: %v", err) } if int64(watchedDeleteRev) != deletedRV { t.Errorf("Object from delete event have version: %v, should be the same as etcd delete's mod rev: %d", watchedDeleteRev, deletedRV) </del> <ins> // Verify that ResourceVersion has changed on deletion. if storedObj.ResourceVersion == deletedObj.ResourceVersion { t.Fatalf("ResourceVersion didn't changed on deletion: %s", deletedObj.ResourceVersion) </ins> } <del> } func deletedRevision(ctx context.Context, watch <-chan clientv3.WatchResponse) (int64, error) { for { select { case <-ctx.Done(): return 0, ctx.Err() case wres := <-watch: for _, evt := range wres.Events { if evt.Type == mvccpb.DELETE && evt.Kv != nil { return evt.Kv.ModRevision, nil } } </del> <ins> select { case event := <-w.ResultChan(): watchedDeleteObj := event.Object.(*example.Pod) if e, a := deletedObj.ResourceVersion, watchedDeleteObj.ResourceVersion; e != a { t.Errorf("Unexpected resource version: %v, expected %v", a, e) </ins> } } }

err := store.Delete(ctx, tt.key, out, nil, storage.ValidateAllObjectFunc, nil) if tt.expectNotFoundErr { if err == nil || !storage.IsNotFound(err) { <del> t.Errorf("%s: expecting not found error, but get: %s", tt.name, err) </del> <ins> t.Errorf("expecting not found error, but get: %s", err) </ins> } return } if err != nil { <del> t.Fatalf("%s: Delete failed: %v", tt.name, err) </del> <ins> t.Fatalf("Delete failed: %v", err) </ins> } <del> ExpectNoDiff(t, fmt.Sprintf("%s: incorrect pod:", tt.name), tt.expectedObj, out) </del> <ins> // We expect the resource version of the returned object to be // updated compared to the last existing object. if storedObj.ResourceVersion == out.ResourceVersion { t.Errorf("expecting resource version to be updated, but get: %s", out.ResourceVersion) } out.ResourceVersion = storedObj.ResourceVersion ExpectNoDiff(t, "incorrect pod:", tt.expectedObj, out) </ins> }) } }

err := store.Delete(ctx, key, out, tt.precondition, storage.ValidateAllObjectFunc, nil) if tt.expectInvalidObjErr { if err == nil || !storage.IsInvalidObj(err) { <del> t.Errorf("%s: expecting invalid UID error, but get: %s", tt.name, err) </del> <ins> t.Errorf("expecting invalid UID error, but get: %s", err) </ins> } return } if err != nil { <del> t.Fatalf("%s: Delete failed: %v", tt.name, err) </del> <ins> t.Fatalf("Delete failed: %v", err) } // We expect the resource version of the returned object to be // updated compared to the last existing object. if storedObj.ResourceVersion == out.ResourceVersion { t.Errorf("expecting resource version to be updated, but get: %s", out.ResourceVersion) </ins> } <del> ExpectNoDiff(t, fmt.Sprintf("%s: incorrect pod", tt.name), storedObj, out) </del> <ins> out.ResourceVersion = storedObj.ResourceVersion ExpectNoDiff(t, "incorrect pod:", storedObj, out) </ins> key, storedObj = TestPropagateStore(ctx, t, store, &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo", UID: "A"}}) }) }