Change:
<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks!--What happened: Resync period for HorizontalPodAutoscaler controller is set to 15 seconds in order to update HPAs every 15 seconds. Sometimes updates are skipped and period between two updates is 30 seconds. What you expected to happen: HPA should be updated every 15 seconds. How to reproduce it (as minimally and precisely as possible): Add debugs to HPA controller showing how often HPAs are processed. Create HPA. Anything else we need to know?: HPA controller uses rate RateLimitingQueue: New requests are always with 15 seconds delay: If request for the given HPA is in the queue then newly request is skipped. Requests are and removed from the queue at the same moment so there is a race condition. See following scenario: 00:00:00 - Resync adds request to queue to be executed at 00:00:15 00:00:15 - Resync adds next request to queue to be executed at 00:00:30 but it is skipped because previous request is still in the queue 00:00:15 - Request at 00:00:00 is executed 00:00:30 - Resync adds next request to queue to be executed at 00:00:45 00:00:45 - Request at 00:00:30 is executed In above scenario HPA wasn't updated between 00:00:15 and 00:00:45. Environment: Kubernetes version (use ): HEAD Cloud provider or hardware configuration: GKE <!-- DO NOT EDIT BELOW THIS LINE --/kind bug
/sig autoscaling
/assign krzysztof-jastrzebski

Discussion:
} defer a.queue.Done(key) <del> err := a.reconcileKey(key.(string)) if err == nil { // don't "forget" here because we want to only process a given HPA once per resync interval return true </del> <ins> deleted, err := a.reconcileKey(key.(string)) if err != nil { utilruntime.HandleError(err) } // Add request processing HPA after resync interval just in case last resync didn't insert // request into the queue. Request is not inserted into queue by resync if previous one wasn't processed yet. // This happens quite often because requests from previous resync are removed from the queue at the same moment // as next resync inserts new requests. if !deleted { a.queue.AddRateLimited(key) </ins> } <del> a.queue.AddRateLimited(key) utilruntime.HandleError(err) </del> return true }

return replicas, metric, statuses, timestamp, nil } <del> func (a *HorizontalController) reconcileKey(key string) error { </del> <ins> func (a *HorizontalController) reconcileKey(key string) (deleted bool, err error) { </ins> namespace, name, err := cache.SplitMetaNamespaceKey(key) if err != nil { <del> return err </del> <ins> return true, err </ins> } hpa, err := a.hpaLister.HorizontalPodAutoscalers(namespace).Get(name) if errors.IsNotFound(err) { klog.Infof("Horizontal Pod Autoscaler %s has been deleted in %s", name, namespace) delete(a.recommendations, key) <del> return nil </del> <ins> return true, nil </ins> } <del> return a.reconcileAutoscaler(hpa, key) </del> <ins> return false, a.reconcileAutoscaler(hpa, key) </ins> } // computeStatusForObjectMetric computes the desired number of replicas for the specified metric of type ObjectMetricSourceType.

finalPods := int32(math.Ceil(resourcesUsedRatio * float64(startPods))) // To breach tolerance we will create a utilization ratio difference of tolerance to usageRatioToleranceValue) <del> tc := testCase{ </del> <ins> tc1 := testCase{ </ins> minReplicas:             0, maxReplicas:             1000, initialReplicas:         startPods,

useMetricsAPI:   true, recommendations: []timestampedRecommendation{}, } <ins> tc1.runTest(t) </ins> <del> tc.runTest(t) // Reuse the data structure above, now testing "unscaling". // Now, we test that no scaling happens if we are in a very close margin to the tolerance </del> target = math.Abs(1/(requestedToUsed*(1-defaultTestingTolerance))) + .004 finalCPUPercentTarget = int32(target * 100) <del> tc.CPUTarget = finalCPUPercentTarget tc.initialReplicas = startPods tc.expectedDesiredReplicas = startPods tc.expectedConditions = statusOkWithOverrides(autoscalingv2.HorizontalPodAutoscalerCondition{ Type:   autoscalingv2.AbleToScale, Status: v1.ConditionTrue, Reason: "ReadyForNewScale", }) tc.runTest(t) </del> <ins> tc2 := testCase{ minReplicas:             0, maxReplicas:             1000, initialReplicas:         startPods, expectedDesiredReplicas: startPods, CPUTarget:               finalCPUPercentTarget, reportedLevels: []uint64{ totalUsedCPUOfAllPods / 10, totalUsedCPUOfAllPods / 10, totalUsedCPUOfAllPods / 10, totalUsedCPUOfAllPods / 10, totalUsedCPUOfAllPods / 10, totalUsedCPUOfAllPods / 10, totalUsedCPUOfAllPods / 10, totalUsedCPUOfAllPods / 10, totalUsedCPUOfAllPods / 10, totalUsedCPUOfAllPods / 10, }, reportedCPURequests: []resource.Quantity{ resource.MustParse(fmt.Sprint(perPodRequested+100) + "m"), resource.MustParse(fmt.Sprint(perPodRequested-100) + "m"), resource.MustParse(fmt.Sprint(perPodRequested+10) + "m"), resource.MustParse(fmt.Sprint(perPodRequested-10) + "m"), resource.MustParse(fmt.Sprint(perPodRequested+2) + "m"), resource.MustParse(fmt.Sprint(perPodRequested-2) + "m"), resource.MustParse(fmt.Sprint(perPodRequested+1) + "m"), resource.MustParse(fmt.Sprint(perPodRequested-1) + "m"), resource.MustParse(fmt.Sprint(perPodRequested) + "m"), resource.MustParse(fmt.Sprint(perPodRequested) + "m"), }, useMetricsAPI:   true, recommendations: []timestampedRecommendation{}, expectedConditions: statusOkWithOverrides(autoscalingv2.HorizontalPodAutoscalerCondition{ Type:   autoscalingv2.AbleToScale, Status: v1.ConditionTrue, Reason: "ReadyForNewScale", }), } tc2.runTest(t) </ins> } func TestScaleUpRCImmediately(t *testing.T) {

// Last scale time lastScaleTime   *metav1.Time recommendations []timestampedRecommendation <ins> finished bool </ins> } // Needs to be called under a lock.

func (tc *legacyTestCase) runTest(t *testing.T) { testClient, testScaleClient := tc.prepareTestClient(t) metricsClient := metrics.NewHeapsterMetricsClient(testClient, metrics.DefaultHeapsterNamespace, metrics.DefaultHeapsterScheme, metrics.DefaultHeapsterService, metrics.DefaultHeapsterPort) eventClient := &fake.Clientset{} eventClient.AddReactor("*", "events", func(action core.Action) (handled bool, ret runtime.Object, err error) { tc.Lock() defer tc.Unlock() <ins> if tc.finished { return true, &v1.Event{}, nil } </ins> obj := action.(core.CreateAction).GetObject().(*v1.Event) if tc.verifyEvents { switch obj.Reason {

informerFactory.Start(stop) go hpaController.Run(stop) <ins> // Wait for HPA to be processed. <-tc.processed </ins> tc.Lock() <ins> tc.finished = true </ins> if tc.verifyEvents { tc.Unlock() // We need to wait for events to be broadcasted (sleep for longer than record.sleepDuration).

} else { tc.Unlock() } <del> // Wait for HPA to be processed. <-tc.processed </del> tc.verifyResults(t) } func TestLegacyScaleUp(t *testing.T) {