Change:
Recently we observed high latencies on POST pods on our scalability tests. For more context see: It appeared to be throttling related, e.g.: As an example look at resource quota admission controller. In there was some fallback to use lru cache in case of on objects. However, not what happens if we are sending multiple POST pods at the same time to apiserver. What happens in Admit() if there are no results in Indexer, we list ResourceQuotas. But if there are multiple calls at the same time, there can be multiple LISTs before the cache is updated: But this can cause throttling of these requests. What we should do is: if there is List() in flight, we should wait until it is finished and lookupCache is updated.
- thanks for the analysis. - fyi, this is an interesting side-effect of the LRU cache pattern, not sure if we have adopted it in other places as well, but good to keep in mind.
To clarify, it was adopted in and admission plug-ins in separate PRs.
- it took me some time, but was useful experience. Thanks for assigning yourself.
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. Prevent issues from auto-closing with an comment. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
Stale issues rot after 30d of inactivity. Mark the issue as fresh with . Rotten issues close after an additional 30d of inactivity. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle rotten /remove-lifecycle stale
Rotten issues close after 30d of inactivity. Reopen the issue with . Mark the issue as fresh with . Send feedback to sig-testing, kubernetes/test-infra and/or . /close
/reopen
You can't reopen an issue/PR unless you authored it or you are a collaborator. <detailsIn response to : /reopen Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details

Discussion:
"strings" "time" <ins> "golang.org/x/sync/singleflight" </ins> corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/api/resource"

// This let's us handle the case of latent caches, by looking up actual results for a namespace on cache miss/no results. // We track the lookup result here so that for repeated requests, we don't look it up very often. liveLookupCache *lru.Cache <ins> group           singleflight.Group </ins> liveTTL         time.Duration }

if len(items) == 0 { lruItemObj, ok := l.liveLookupCache.Get(a.GetNamespace()) if !ok || lruItemObj.(liveLookupEntry).expiry.Before(time.Now()) { <del> // TODO: If there are multiple operations at the same time and cache has just expired, // this may cause multiple List operations being issued at the same time. // If there is already in-flight List() for a given namespace, we should wait until // it is finished and cache is updated instead of doing the same, also to avoid // throttling - see #22422 for details. liveList, err := l.client.CoreV1().LimitRanges(a.GetNamespace()).List(context.TODO(), metav1.ListOptions{}) </del> <ins> // Fixed: #22422 // use singleflight to alleviate simultaneous calls to lruItemObj, err, _ = l.group.Do(a.GetNamespace(), func() (interface{}, error) { liveList, err := l.client.CoreV1().LimitRanges(a.GetNamespace()).List(context.TODO(), metav1.ListOptions{}) if err != nil { return nil, admission.NewForbidden(a, err) } newEntry := liveLookupEntry{expiry: time.Now().Add(l.liveTTL)} for i := range liveList.Items { newEntry.items = append(newEntry.items, &liveList.Items[i]) } l.liveLookupCache.Add(a.GetNamespace(), newEntry) return newEntry, nil }) </ins> if err != nil { <del> return nil, admission.NewForbidden(a, err) } newEntry := liveLookupEntry{expiry: time.Now().Add(l.liveTTL)} for i := range liveList.Items { newEntry.items = append(newEntry.items, &liveList.Items[i]) </del> <ins> return nil, err </ins> } <del> l.liveLookupCache.Add(a.GetNamespace(), newEntry) lruItemObj = newEntry </del> } lruEntry := lruItemObj.(liveLookupEntry)

"context" "fmt" "strconv" <ins> "sync" "sync/atomic" </ins> "testing" "time"

} } } <ins> // TestLimitRanger_GetLimitRangesFixed22422 Fixed Admission controllers can cause unnecessary significant load on apiserver #22422 func TestLimitRanger_GetLimitRangesFixed22422(t *testing.T) { limitRange := validLimitRangeNoDefaults() limitRanges := []corev1.LimitRange{limitRange} var count int64 mockClient := &fake.Clientset{} unhold := make(chan struct{}) mockClient.AddReactor("list", "limitranges", func(action core.Action) (bool, runtime.Object, error) { atomic.AddInt64(&count, 1) limitRangeList := &corev1.LimitRangeList{ ListMeta: metav1.ListMeta{ ResourceVersion: fmt.Sprintf("%d", len(limitRanges)), }, } for index, value := range limitRanges { value.ResourceVersion = fmt.Sprintf("%d", index) value.Namespace = action.GetNamespace() limitRangeList.Items = append(limitRangeList.Items, value) } // he always blocking before sending the signal <-unhold return true, limitRangeList, nil }) handler, _, err := newHandlerForTest(mockClient) if err != nil { t.Errorf("unexpected error initializing handler: %v", err) } attributes := admission.NewAttributesRecord(nil, nil, api.Kind("kind").WithVersion("version"), "test", "name", api.Resource("resource").WithVersion("version"), "subresource", admission.Create, &metav1.CreateOptions{}, false, nil) attributesTest1 := admission.NewAttributesRecord(nil, nil, api.Kind("kind").WithVersion("version"), "test1", "name", api.Resource("resource").WithVersion("version"), "subresource", admission.Create, &metav1.CreateOptions{}, false, nil) wg := sync.WaitGroup{} for i := 0; i < 10; i++ { wg.Add(2) // simulating concurrent calls after a cache failure go func() { defer wg.Done() ret, err := handler.GetLimitRanges(attributes) if err != nil { t.Errorf("unexpected error: %v", err) } for _, c := range ret { if c.Namespace != attributes.GetNamespace() { t.Errorf("Expected %s namespace, got %s", attributes.GetNamespace(), c.Namespace) } } }() // simulation of different namespaces is not a call go func() { defer wg.Done() ret, err := handler.GetLimitRanges(attributesTest1) if err != nil { t.Errorf("unexpected error: %v", err) } for _, c := range ret { if c.Namespace != attributesTest1.GetNamespace() { t.Errorf("Expected %s namespace, got %s", attributesTest1.GetNamespace(), c.Namespace) } } }() } // unhold all the calls with the same namespace handler.GetLimitRanges(attributes) calls, that have to be aggregated unhold <- struct{}{} go func() { unhold <- struct{}{} }() // and here we wait for all the goroutines wg.Wait() // since all the calls with the same namespace will be holded, they must be catched on the singleflight group, // There are two different sets of namespace calls // hence only 2 if count != 2 { t.Errorf("Expected 1 limit range, got %d", count) } // invalidate the cache handler.liveLookupCache.Remove(attributes.GetNamespace()) go func() { // unhold it is blocking until GetLimitRanges is executed unhold <- struct{}{} }() _, err = handler.GetLimitRanges(attributes) if err != nil { t.Errorf("unexpected error: %v", err) } close(unhold) if count != 3 { t.Errorf("Expected 2 limit range, got %d", count) } } </ins>