Change:
Reproduced on 1.11.7 as well as master-ish () Deleting a CRD doesn't seem to cause the API server to terminate watches referencing the corresponding CRs, but the watches also don't properly resume if the CRD is created again, meaning that you have to wait for watch timeouts to actually get new watch events. We should probably kill open watches on CRD deletion, so that clients are notified and are forced to re-watch. Either that, or seamlessly resume watches if a CRD is recreated. <details</summary</details<details</summary</detailsthe above CRD: the above CR: (elsewhere) and grab a resource version (elsewhere) the lack of watch events After watches time out, things can re-establish, but that can be a while.
/assign
/cc sttts
/area custom-resources
it's sort a related thread w/ . for now CRD's update will terminate CR's watch connection, the other way round we might fix'em together. one possible fix suggested by months ago is that we make the storage to be a runnable object, e.g. by which we can clearily define the lifecycle of CR's storage
fixed in

Discussion:
"k8s.io/apiextensions-apiserver/test/integration/fixtures" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" <ins> "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/watch" </ins> "k8s.io/client-go/dynamic" )

} }() <del> // Set up 100 loops creating and reading custom resources </del> <ins> // Set up 100 loops creating and reading and watching custom resources </ins> for i := 0; i < 100; i++ { wg.Add(1) go func(i int) {

time.Sleep(10 * time.Millisecond) } }(i) <ins> wg.Add(1) go func(i int) { defer wg.Done() for { select { case <-stopChan: return default: w, err := noxuNamespacedResourceClient.Watch(metav1.ListOptions{}) if err != nil { t.Fatalf("unexpected error establishing watch: %v", err) } for event := range w.ResultChan() { switch event.Type { case watch.Added, watch.Modified, watch.Deleted: // all expected default: t.Errorf("unexpected watch event: %#v", event) } } } time.Sleep(10 * time.Millisecond) } }(i) </ins> } // Let all the established get request loops soak

close(stopChan) // Let loops drain <del> wg.Wait() </del> <ins> drained := make(chan struct{}) go func() { defer close(drained) wg.Wait() }() select { case <-drained: case <-time.After(wait.ForeverTestTimeout): t.Error("timed out waiting for clients to complete") } </ins> }

cacher.stopWg.Add(1) go func() { defer cacher.stopWg.Done() <ins> defer cacher.terminateAllWatchers() </ins> wait.Until( func() { if !cacher.isStopped() {

} } <ins> func TestCacheWatcherStoppedOnDestroy(t *testing.T) { backingStorage := &dummyStorage{} cacher, _ := newTestCacher(backingStorage, 1000) defer cacher.Stop() // Wait until cacher is initialized. cacher.ready.wait() w, err := cacher.Watch(context.Background(), "pods/ns", "0", storage.Everything) if err != nil { t.Fatalf("Failed to create watch: %v", err) } watchClosed := make(chan struct{}) go func() { defer close(watchClosed) for event := range w.ResultChan() { switch event.Type { case watch.Added, watch.Modified, watch.Deleted: // ok default: t.Errorf("unexpected event %#v", event) } } }() cacher.Stop() select { case <-watchClosed: case <-time.After(wait.ForeverTestTimeout): t.Errorf("timed out waiting for watch to close") } } </ins> func TestTimeBucketWatchersBasic(t *testing.T) { filter := func(_ string, _ labels.Set, _ fields.Set) bool { return true