Change:
There's some practical limit on the number of concurrent access tokens that can be used in a Kubernetes cluster before it begins to spend all of its time in the authn stack verifying them. Absent fixes to it was < 500; with the fixes linked there it's still not infinitely scalable-- I still hit (what I believe to be) with ~ 5k unique tokens @ ~700QPS. This request is to this dimension an official name & definition, and list it with others, e.g. here: it it have a useful value (I think it should be 10k) /kind bug /sig scalability
/cc
oh and 4. add tests which verify it is as high as believed.
cc since projected tokens will increase the number of unique tokens
/sig auth
LRUs are not good at scaling and sharding has some major pitfalls as pointed out. I would suggest that we try to drop the LRU cache for successful authentication results in the token cache. The main benefit of using an LRU here is bound cache size. Entries in the token cache have a fast expiration independent of the LRU policy. The records probably take up O(2kb) (guesstimate) of memory. Given that these are successful attempts with presumably valid tokens, I don't think we are getting any benefit from the LRU. If we don't have a constraint on fixed cache size, we can back the token cache with a sync.Map and run a single background goroutine to periodically walk the map and remove expired entries. This would fix the problematic locking and eviction due to nonuniform distribution of tokens over shards. The failure cache is less important and we don't cache failures by default. If we consider this the "slowpath", we can probably leave it as. If that's not good enough, we have a couple options (with various tradeoffs) to improve it.
If off by default is not enough, more reasoning why the failed cache is not super useful: it might help with an accidental DoS but it doesn't prevent anything actively malicious DoS. I think we can focus our improvements on the success cache.
/assign He is already working on adding an explicit test for it.
has taken it over and is actively working on this. Unfortunately, I cannot assign this issue to him as he's not a k8s member yet
It's not closed - all testes must be submitted and appropriate documentation must be
Thanks everyone!!

Discussion:
} func (dc *DeploymentController) syncRecreateDeployment(deployment extensions.Deployment) error { <del> // TODO: implement me. </del> <ins> newRC, err := dc.getNewRC(deployment) if err != nil { return err } oldRCs, err := dc.getOldRCs(deployment) if err != nil { return err } allRCs := append(oldRCs, newRC) // scale down old rcs scaledDown, err := dc.scaleDownOldRCsForRecreate(oldRCs, deployment) if err != nil { return err } if scaledDown { // Update DeploymentStatus return dc.updateDeploymentStatus(allRCs, newRC, deployment) } // scale up new rc scaledUp, err := dc.scaleUpNewRCForRecreate(newRC, deployment) if err != nil { return err } if scaledUp { // Update DeploymentStatus return dc.updateDeploymentStatus(allRCs, newRC, deployment) } </ins> return nil }

return true, err } <ins> // scaleDownOldRCsForRecreate scales down old rcs when deployment strategy is "Recreate" func (dc *DeploymentController) scaleDownOldRCsForRecreate(oldRCs []*api.ReplicationController, deployment extensions.Deployment) (bool, error) { scaled := false for _, rc := range oldRCs { // Scaling not required. if rc.Spec.Replicas == 0 { continue } _, err := dc.scaleRCAndRecordEvent(rc, 0, deployment) if err != nil { return false, err } scaled = true } return scaled, nil } // scaleUpNewRCForRecreate scales up new rc when deployment strategy is "Recreate" func (dc *DeploymentController) scaleUpNewRCForRecreate(newRC *api.ReplicationController, deployment extensions.Deployment) (bool, error) { if newRC.Spec.Replicas == deployment.Spec.Replicas { // Scaling not required. return false, nil } _, err := dc.scaleRCAndRecordEvent(newRC, deployment.Spec.Replicas, deployment) return true, err } </ins> func (dc *DeploymentController) updateDeploymentStatus(allRCs []*api.ReplicationController, newRC *api.ReplicationController, deployment extensions.Deployment) error { totalReplicas := deploymentutil.GetReplicaCountForRCs(allRCs) updatedReplicas := deploymentutil.GetReplicaCountForRCs([]*api.ReplicationController{newRC})

Namespace: api.NamespaceDefault, }, Spec: api.ReplicationControllerSpec{ <del> Replicas: 0, </del> <ins> Replicas: replicas, </ins> Template: &d.Spec.Template, }, }

It("deployment should create new pods", func() { testNewDeployment(f) }) <del> It("deployment should delete old pods and create new ones [Flaky]", func() { </del> <ins> It("RollingUpdateDeployment should delete old pods and create new ones [Flaky]", func() { </ins> testRollingUpdateDeployment(f) }) <del> It("deployment should scale up and down in the right order [Flaky]", func() { </del> <ins> It("RollingUpdateDeployment should scale up and down in the right order [Flaky]", func() { </ins> testRollingUpdateDeploymentEvents(f) }) <ins> It("RecreateDeployment should delete old pods and create new ones [Flaky]", func() { testRecreateDeployment(f) }) </ins> It("deployment should support rollover [Flaky]", func() { testRolloverDeployment(f) })

} } <del> func newDeployment(deploymentName string, replicas int, podLabels map[string]string, imageName string, image string) *extensions.Deployment { </del> <ins> func newDeployment(deploymentName string, replicas int, podLabels map[string]string, imageName string, image string, strategyType extensions.DeploymentStrategyType) *extensions.Deployment { </ins> return &extensions.Deployment{ ObjectMeta: api.ObjectMeta{ Name: deploymentName, }, Spec: extensions.DeploymentSpec{ <del> Replicas:       replicas, Selector:       podLabels, </del> <ins> Replicas: replicas, Selector: podLabels, Strategy: extensions.DeploymentStrategy{ Type: strategyType, }, </ins> UniqueLabelKey: extensions.DefaultDeploymentUniqueLabelKey, Template: api.PodTemplateSpec{ ObjectMeta: api.ObjectMeta{

podLabels := map[string]string{"name": "nginx"} replicas := 1 Logf("Creating simple deployment %s", deploymentName) <del> _, err := c.Deployments(ns).Create(newDeployment(deploymentName, replicas, podLabels, "nginx", "nginx")) </del> <ins> _, err := c.Deployments(ns).Create(newDeployment(deploymentName, replicas, podLabels, "nginx", "nginx", extensions.RollingUpdateDeploymentStrategyType)) </ins> Expect(err).NotTo(HaveOccurred()) defer func() { deployment, err := c.Deployments(ns).Get(deploymentName)

// Create a deployment to delete nginx pods and instead bring up redis pods. deploymentName := "redis-deployment" Logf("Creating deployment %s", deploymentName) <del> _, err = c.Deployments(ns).Create(newDeployment(deploymentName, replicas, deploymentPodLabels, "redis", "redis")) </del> <ins> _, err = c.Deployments(ns).Create(newDeployment(deploymentName, replicas, deploymentPodLabels, "redis", "redis", extensions.RollingUpdateDeploymentStrategyType)) </ins> Expect(err).NotTo(HaveOccurred()) defer func() { deployment, err := c.Deployments(ns).Get(deploymentName)

// Create a deployment to delete nginx pods and instead bring up redis pods. deploymentName := "redis-deployment-2" Logf("Creating deployment %s", deploymentName) <del> _, err = c.Deployments(ns).Create(newDeployment(deploymentName, replicas, deploymentPodLabels, "redis", "redis")) </del> <ins> _, err = c.Deployments(ns).Create(newDeployment(deploymentName, replicas, deploymentPodLabels, "redis", "redis", extensions.RollingUpdateDeploymentStrategyType)) </ins> Expect(err).NotTo(HaveOccurred()) defer func() { deployment, err := c.Deployments(ns).Get(deploymentName)

Expect(events.Items[1].Message).Should(Equal(fmt.Sprintf("Scaled down rc %s to 0", rcName))) } <ins> func testRecreateDeployment(f *Framework) { ns := f.Namespace.Name c := f.Client // Create nginx pods. deploymentPodLabels := map[string]string{"name": "sample-pod-3"} rcPodLabels := map[string]string{ "name": "sample-pod-3", "pod":  "nginx", } rcName := "nginx-controller" replicas := 3 _, err := c.ReplicationControllers(ns).Create(newRC(rcName, replicas, rcPodLabels, "nginx", "nginx")) Expect(err).NotTo(HaveOccurred()) defer func() { Logf("deleting replication controller %s", rcName) Expect(c.ReplicationControllers(ns).Delete(rcName)).NotTo(HaveOccurred()) }() // Verify that the required pods have come up. err = verifyPods(c, ns, "sample-pod-3", false, 3) if err != nil { Logf("error in waiting for pods to come up: %s", err) Expect(err).NotTo(HaveOccurred()) } // Create a deployment to delete nginx pods and instead bring up redis pods. deploymentName := "redis-deployment-3" Logf("Creating deployment %s", deploymentName) _, err = c.Deployments(ns).Create(newDeployment(deploymentName, replicas, deploymentPodLabels, "redis", "redis", extensions.RecreateDeploymentStrategyType)) Expect(err).NotTo(HaveOccurred()) defer func() { deployment, err := c.Deployments(ns).Get(deploymentName) Expect(err).NotTo(HaveOccurred()) Logf("deleting deployment %s", deploymentName) Expect(c.Deployments(ns).Delete(deploymentName, nil)).NotTo(HaveOccurred()) // TODO: remove this once we can delete rcs with deployment newRC, err := deploymentutil.GetNewRC(*deployment, c) Expect(err).NotTo(HaveOccurred()) Expect(c.ReplicationControllers(ns).Delete(newRC.Name)).NotTo(HaveOccurred()) }() err = waitForDeploymentStatus(c, ns, deploymentName, replicas, 0, replicas, 0) Expect(err).NotTo(HaveOccurred()) // Verify that the pods were scaled up and down as expected. We use events to verify that. deployment, err := c.Deployments(ns).Get(deploymentName) Expect(err).NotTo(HaveOccurred()) waitForEvents(c, ns, deployment, 2) events, err := c.Events(ns).Search(deployment) if err != nil { Logf("error in listing events: %s", err) Expect(err).NotTo(HaveOccurred()) } // There should be 2 events, one to scale up the new RC and then to scale down the old RC. Expect(len(events.Items)).Should(Equal(2)) newRC, err := deploymentutil.GetNewRC(*deployment, c) Expect(err).NotTo(HaveOccurred()) Expect(newRC).NotTo(Equal(nil)) Expect(events.Items[0].Message).Should(Equal(fmt.Sprintf("Scaled down rc %s to 0", rcName))) Expect(events.Items[1].Message).Should(Equal(fmt.Sprintf("Scaled up rc %s to 3", newRC.Name))) } </ins> // testRolloverDeployment tests that deployment supports rollover. // i.e. we can change desired state and kick off rolling update, then change desired state again before it finishes. func testRolloverDeployment(f *Framework) {

deploymentReplicas := 4 deploymentImage := "gcr.io/google_samples/gb-redisslave:v1" deploymentMinReadySeconds := 5 <ins> deploymentStrategyType := extensions.RollingUpdateDeploymentStrategyType </ins> Logf("Creating deployment %s", deploymentName) <del> newDeployment := newDeployment(deploymentName, deploymentReplicas, deploymentPodLabels, deploymentImageName, deploymentImage) </del> <ins> newDeployment := newDeployment(deploymentName, deploymentReplicas, deploymentPodLabels, deploymentImageName, deploymentImage, deploymentStrategyType) </ins> newDeployment.Spec.Strategy.RollingUpdate = &extensions.RollingUpdateDeployment{ MaxUnavailable:  intstr.FromInt(1), MaxSurge:        intstr.FromInt(1),