Change:
Which jobs are failing: pull-kubernetes-integration Which test(s) are failing: TestWebhookConverterWithDefaulting /assign /priority important-soon /area custom-resources /sig api-machinery /kind flake /milestone v1.16
/cc
several of the webhook converter tests have similar flakes. it looks like the wait for storage version to become effective times out occasionally Rare flakes seen in all of these tests: TestWebhookConverterWithWatchCache/nontrivial-converter TestWebhookConverterWithWatchCache/metadata-mutating-converter TestWebhookConverterWithDefaulting/metadata-mutating-converter TestWebhookConverterWithWatchCache/noop-converter TestWebhookConverterWithDefaulting/empty-response TestWebhookConverterWithoutWatchCache/nontrivial-converter/check-2 TestWebhookConverterWithoutWatchCache/noop-converter/check-0/v1beta2 TestWebhookConverterWithoutWatchCache/nontrivial-converter/check-0/v1beta2
cc
Yikes, I’ll sort it out.
iiuc sending an empty patch gets short-circuited and doesn't bump CR generation, even if the storage version gets changed, which means we do not re-write to etcd and go through the encoding path this would be racing and doing noop's if the first pass of didn't get the expected storage version I will send a fix to make the patch do actual mutation
It doesn't bump generation, but the bytes to persist in etcd would still be different if the storage version had changed, so they should persist. You can verify that by changing the storage version on a CRD yourself, do an empty patch, and see if the resourceVersion changes.
you're right. Generation is for non-metadata change only, and the RV changes. I queried etcd locally and the apiVersion did change for an empty patch (sidetracking: another observation is after changing the storage version (say v1 -v2; noop convertor), empty-patching v2 endpoint would bump generation, while empty-patching v1 endpoint wouldn't)
another theory: our cached storage strategy could be stale if the CRD informer merges multiple events together (e.g. the informer does a re-list). Two scenarios: -create-update events merged into an update event: strategy with UID:a was created on demand for CRD Foo Foo was deleted and recreated strategy with UID:b was created on demand for the new Foo new Foo got updated (e.g. switching storage version) the strategy with UID:b is supposed to be deleted and , to reflect the last update. But if the CRD informer merges the delete-create-update events into , the strategy with UID:a will be deleted, but the strategy with UID:b won't. -update events merged into a create event: strategy with UID:a was created on demand for CRD Foo Foo was updated (e.g. switching storage version) the strategy with UID:a is supposed to be deleted and , to reflect the last update. But if the CRD informer merges the create-update events into , the strategy with UID:a won't be deleted (as we don't react on create events). another race found in , which could happen in a single update.
I think I :

Discussion:
minRequestTimeout:       minRequestTimeout, } crdInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ <ins> AddFunc:    ret.createCustomResourceDefinition, </ins> UpdateFunc: ret.updateCustomResourceDefinition, DeleteFunc: func(obj interface{}) { ret.removeDeadStorage()

terminating := apiextensions.IsCRDConditionTrue(crd, apiextensions.Terminating) <del> crdInfo, err := r.getOrCreateServingInfoFor(crd) </del> <ins> crdInfo, err := r.getOrCreateServingInfoFor(crd.UID, crd.Name) if apierrors.IsNotFound(err) { r.delegate.ServeHTTP(w, req) return } </ins> if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } <ins> if !hasServedCRDVersion(crdInfo.spec, requestInfo.APIVersion) { r.delegate.ServeHTTP(w, req) return } </ins> verb := strings.ToUpper(requestInfo.Verb) resource := requestInfo.Resource

} } <ins> // createCustomResourceDefinition removes potentially stale storage so it gets re-created func (r *crdHandler) createCustomResourceDefinition(obj interface{}) { crd := obj.(*apiextensions.CustomResourceDefinition) r.customStorageLock.Lock() defer r.customStorageLock.Unlock() // this could happen if the create event is merged from create-update events r.removeStorage_locked(crd.UID) } // updateCustomResourceDefinition removes potentially stale storage so it gets re-created </ins> func (r *crdHandler) updateCustomResourceDefinition(oldObj, newObj interface{}) { oldCRD := oldObj.(*apiextensions.CustomResourceDefinition) newCRD := newObj.(*apiextensions.CustomResourceDefinition)

} } <ins> if oldCRD.UID != newCRD.UID { r.removeStorage_locked(oldCRD.UID) } </ins> storageMap := r.customStorage.Load().(crdStorageMap) oldInfo, found := storageMap[newCRD.UID] if !found {

return } <del> klog.V(4).Infof("Updating customresourcedefinition %s", oldCRD.Name) </del> <ins> klog.V(4).Infof("Updating customresourcedefinition %s", newCRD.Name) r.removeStorage_locked(newCRD.UID) } </ins> <del> if oldInfo, ok := storageMap[types.UID(oldCRD.UID)]; ok { </del> <ins> // removeStorage_locked removes the cached storage with the given uid as key from the storage map. This function // updates r.customStorage with the cleaned-up storageMap and tears down the old storage. // NOTE: Caller MUST hold r.customStorageLock to write r.customStorage thread-safely. func (r *crdHandler) removeStorage_locked(uid types.UID) { storageMap := r.customStorage.Load().(crdStorageMap) if oldInfo, ok := storageMap[uid]; ok { </ins> // Copy because we cannot write to storageMap without a race // as it is used without locking elsewhere. storageMap2 := storageMap.clone() // Remove from the CRD info map and store the map <del> delete(storageMap2, types.UID(oldCRD.UID)) </del> <ins> delete(storageMap2, uid) </ins> r.customStorage.Store(storageMap2) // Tear down the old storage

// GetCustomResourceListerCollectionDeleter returns the ListerCollectionDeleter of // the given crd. func (r *crdHandler) GetCustomResourceListerCollectionDeleter(crd *apiextensions.CustomResourceDefinition) (finalizer.ListerCollectionDeleter, error) { <del> info, err := r.getOrCreateServingInfoFor(crd) </del> <ins> info, err := r.getOrCreateServingInfoFor(crd.UID, crd.Name) </ins> if err != nil { return nil, err } return info.storages[info.storageVersion].CustomResource, nil } <del> func (r *crdHandler) getOrCreateServingInfoFor(crd *apiextensions.CustomResourceDefinition) (*crdInfo, error) { </del> <ins> // getOrCreateServingInfoFor gets the CRD serving info for the given CRD UID if the key exists in the storage map. // Otherwise the function fetches the up-to-date CRD using the given CRD name and creates CRD serving info. func (r *crdHandler) getOrCreateServingInfoFor(uid types.UID, name string) (*crdInfo, error) { </ins> storageMap := r.customStorage.Load().(crdStorageMap) <del> if ret, ok := storageMap[crd.UID]; ok { </del> <ins> if ret, ok := storageMap[uid]; ok { </ins> return ret, nil } r.customStorageLock.Lock() defer r.customStorageLock.Unlock() <ins> // Get the up-to-date CRD when we have the lock, to avoid racing with updateCustomResourceDefinition. // If updateCustomResourceDefinition sees an update and happens later, the storage will be deleted and // we will re-create the updated storage on demand. If updateCustomResourceDefinition happens before, // we make sure that we observe the same up-to-date CRD. crd, err := r.crdLister.Get(name) if err != nil { return nil, err } </ins> storageMap = r.customStorage.Load().(crdStorageMap) if ret, ok := storageMap[crd.UID]; ok { return ret, nil

return nil } <ins> // hasServedCRDVersion returns true if the given version is in the list of CRD's versions and the Served flag is set. func hasServedCRDVersion(spec *apiextensions.CustomResourceDefinitionSpec, version string) bool { for _, v := range spec.Versions { if v.Name == version { return v.Served } } return false } </ins>