Change:
Currently, (best-effort) watch bookmarks are sent just right before the given watch is supposed to timeout: In we're adjusting how the size of watch cache is determined and make it dynamic and more duration-oriented (rather than based on some magic constants) - we're targeting to always keep 5 minutes of history in watch cache. With that change, we should get back to our bookmark policy: change it to (additionally to right before timeout) also send bookmarks every N seconds (setting N to slightly less than expected watch cache history period, so slightly less than X minutes we will use there) scale-test this change (to see if it doesn't introduce non-negligible overhead) if so permanently introduce this change /sig scalability api-machinery
/cc
/cc
/cc
I'm reverting as it broke bookmarks. I'm going to fix and re-submit for review hopefully later this week.

Discussion:
// storageWatchListPageSize is the cacher's request chunk size of // initial and resync watch lists to storage. storageWatchListPageSize = int64(10000) <ins> // defaultBookmarkFrequency defines how frequently watch bookmarks should be send // in addition to sending a bookmark right before watch deadline defaultBookmarkFrequency = time.Minute </ins> ) // Config contains the configuration for a given Cache.

// second in a bucket, and pop up them once at the timeout. To be more specific, // if you set fire time at X, you can get the bookmark within (X-1,X+1) period. type watcherBookmarkTimeBuckets struct { <del> lock            sync.Mutex watchersBuckets map[int64][]*cacheWatcher startBucketID   int64 clock           clock.Clock </del> <ins> lock              sync.Mutex watchersBuckets   map[int64][]*cacheWatcher startBucketID     int64 clock             clock.Clock bookmarkFrequency time.Duration </ins> } <del> func newTimeBucketWatchers(clock clock.Clock) *watcherBookmarkTimeBuckets { </del> <ins> func newTimeBucketWatchers(clock clock.Clock, bookmarkFrequency time.Duration) *watcherBookmarkTimeBuckets { </ins> return &watcherBookmarkTimeBuckets{ <del> watchersBuckets: make(map[int64][]*cacheWatcher), startBucketID:   clock.Now().Unix(), clock:           clock, </del> <ins> watchersBuckets:   make(map[int64][]*cacheWatcher), startBucketID:     clock.Now().Unix(), clock:             clock, bookmarkFrequency: bookmarkFrequency, </ins> } } // adds a watcher to the bucket, if the deadline is before the start, it will be // added to the first one. func (t *watcherBookmarkTimeBuckets) addWatcher(w *cacheWatcher) bool { <del> nextTime, ok := w.nextBookmarkTime(t.clock.Now()) </del> <ins> nextTime, ok := w.nextBookmarkTime(t.clock.Now(), t.bookmarkFrequency) </ins> if !ok { return false }

stopCh:           stopCh, clock:            clock, timer:            time.NewTimer(time.Duration(0)), <del> bookmarkWatchers: newTimeBucketWatchers(clock), </del> <ins> bookmarkWatchers: newTimeBucketWatchers(clock, defaultBookmarkFrequency), </ins> } // Ensure that timer is stopped.

continue } c.watchersBuffer = append(c.watchersBuffer, watcher) <del> // Given that we send bookmark event once at deadline-2s, never push again // after the watcher pops up from the buckets. Once we decide to change the // strategy to more sophisticated, we may need it here. </del> <ins> // Requeue the watcher for the next bookmark if needed. c.bookmarkWatchers.addWatcher(watcher) </ins> } } }

} } <del> func (c *cacheWatcher) nextBookmarkTime(now time.Time) (time.Time, bool) { // For now we return 2s before deadline (and maybe +infinity is now already passed this time) // but it gives us extensibility for the future(false when deadline is not set). </del> <ins> func (c *cacheWatcher) nextBookmarkTime(now time.Time, bookmarkFrequency time.Duration) (time.Time, bool) { // We try to send bookmarks: // (a) roughly every minute // (b) right before the watcher timeout - for now we simply set it 2s before //     the deadline // The former gives us periodicity if the watch breaks due to unexpected // conditions, the later ensures that on timeout the watcher is as close to // now as possible - this covers 99% of cases. heartbeatTime := now.Add(bookmarkFrequency) </ins> if c.deadline.IsZero() { <del> return c.deadline, false </del> <ins> // Timeout is set by our client libraries (e.g. reflector) as well as defaulted by // apiserver if properly configured. So this shoudln't happen in practice. return heartbeatTime, true } if pretimeoutTime := c.deadline.Add(-2 * time.Second); pretimeoutTime.Before(heartbeatTime) { heartbeatTime = pretimeoutTime } if heartbeatTime.Before(now) { return time.Time{}, false </ins> } <del> return c.deadline.Add(-2 * time.Second), true </del> <ins> return heartbeatTime, true </ins> } func getEventObject(object runtime.Object) runtime.Object {

} clock := clock.NewFakeClock(time.Now()) <del> watchers := newTimeBucketWatchers(clock) </del> <ins> watchers := newTimeBucketWatchers(clock, defaultBookmarkFrequency) </ins> now := clock.Now() watchers.addWatcher(newWatcher(now.Add(10 * time.Second))) watchers.addWatcher(newWatcher(now.Add(20 * time.Second)))

} } <ins> func TestCacherSendsMultipleWatchBookmarks(t *testing.T) { backingStorage := &dummyStorage{} cacher, _, err := newTestCacher(backingStorage, 1000) if err != nil { t.Fatalf("Couldn't create cacher: %v", err) } defer cacher.Stop() // Update bookmarkFrequency to speed up test. // Note that the frequency lower than 1s doesn't change much due to // resolution how frequency we recompute. cacher.bookmarkWatchers.bookmarkFrequency = time.Second // Wait until cacher is initialized. cacher.ready.wait() pred := storage.Everything pred.AllowWatchBookmarks = true makePod := func(index int) *examplev1.Pod { return &examplev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name:            fmt.Sprintf("pod-%d", index), Namespace:       "ns", ResourceVersion: fmt.Sprintf("%v", 100+index), }, } } // Create pod to initialize watch cache. if err := cacher.watchCache.Add(makePod(0)); err != nil { t.Fatalf("failed to add a pod: %v", err) } ctx, _ := context.WithTimeout(context.Background(), 10*time.Second) w, err := cacher.Watch(ctx, "pods/ns", "100", pred) if err != nil { t.Fatalf("Failed to create watch: %v", err) } // Create one more pod, to ensure that current RV is higher and thus // bookmarks will be delievere (events are delivered for RV higher // than the max from init events). if err := cacher.watchCache.Add(makePod(1)); err != nil { t.Fatalf("failed to add a pod: %v", err) } timeoutCh := time.After(5 * time.Second) lastObservedRV := uint64(0) // Ensure that a watcher gets two bookmarks. for observedBookmarks := 0; observedBookmarks < 2; { select { case event, ok := <-w.ResultChan(): if !ok { t.Fatal("Unexpected closed") } rv, err := cacher.versioner.ObjectResourceVersion(event.Object) if err != nil { t.Errorf("failed to parse resource version from %#v: %v", event.Object, err) } if event.Type == watch.Bookmark { observedBookmarks++ if rv < lastObservedRV { t.Errorf("Unexpected bookmark event resource version %v (last %v)", rv, lastObservedRV) } } lastObservedRV = rv case <-timeoutCh: t.Fatal("Unexpected timeout to receive bookmark events") } } } </ins> func TestDispatchingBookmarkEventsWithConcurrentStop(t *testing.T) { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.WatchBookmark, true)() backingStorage := &dummyStorage{}

allowWatchBookmark bool }{ { // test old client won't get Bookmark event <del> timeout:            2 * time.Second, </del> <ins> timeout:            3 * time.Second, </ins> expected:           false, allowWatchBookmark: false, }, { <del> timeout:            2 * time.Second, </del> <ins> timeout:            3 * time.Second, </ins> expected:           true, allowWatchBookmark: true, },

pred := storage.Everything pred.AllowWatchBookmarks = true <del> ctx, _ := context.WithTimeout(context.Background(), 2*time.Second) </del> <ins> ctx, _ := context.WithTimeout(context.Background(), 3*time.Second) </ins> watcher, err := cacher.WatchList(ctx, "pods/ns", "0", pred) if err != nil { t.Fatalf("Unexpected error: %v", err)