Change:
<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: When the HPA scales up beyond the capacity of the cluster, many unscheduled pods are created. When load is removed, the usage ratio drops and the HPA tries to scale down. The unscheduled pods do not produce metrics, so the HPA fills in the 100% usage and re-evaluates the usage ratio. Because of the many fake 100% metrics, the new usage ratio is above the target value and the direction of scale is reversed. In this case, the HPA decides not to make any change to the current scale. What you expected to happen: I would expect the HPA to not get stuck when load is removed. How to reproduce it (as minimally and precisely as possible): and put it under load in a cluster without cluster autoscaling. the Deployment to increase the CPU request beyond the capability of the cluster (e.g. 500m). This should produce many Pending pods. load. The Pending pods will keep the HPA scaled indefinitely. Anything else we need to know?: Environment: Kubernetes version (use ): Client Version: version.Info{Major:"1", Minor:"14", GitVersion:"v1.14.3", GitCommit:"", GitTreeState:"clean", BuildDate:"2019-06-06T01:44:30Z", GoVersion:"go1.12.5", Compiler:"gc", Platform:"linux/amd64"} Server Version: version.Info{Major:"1", Minor:"12+", GitVersion:"v1.12.9-gke.3", GitCommit:"", GitTreeState:"clean", BuildDate:"2019-06-06T21:30:53Z", GoVersion:"go1.10.8b4", Compiler:"gc", Platform:"linux/amd64"} Cloud provider or hardware configuration: OS (e.g: ): Kernel (e.g. ): Install tools: Network plugin and version (if this is a network-related bug): Others:
/sig autoscaling
This might Previous I suspected that this was the same issues as (HPA positive feedback loop). However it turns out to be different, that Pending pods cause HPA to get stuck.
Original reason for using 100% utilization for missing metrics when scaling down:

Discussion:
"time" autoscaling "k8s.io/api/autoscaling/v2beta2" <del> "k8s.io/api/core/v1" </del> <ins> v1 "k8s.io/api/core/v1" </ins> metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/sets"

if pod.DeletionTimestamp != nil || pod.Status.Phase == v1.PodFailed { continue } <ins> // Pending pods are ignored. if pod.Status.Phase == v1.PodPending { ignoredPods.Insert(pod.Name) continue } // Pods missing metrics. </ins> metric, found := metrics[pod.Name] if !found { missingPods.Insert(pod.Name) continue } <ins> // Unready pods are ignored. </ins> if resource == v1.ResourceCPU { var ignorePod bool _, condition := podutil.GetPodCondition(&pod.Status, v1.PodReady)

"time" autoscalingv2 "k8s.io/api/autoscaling/v2beta2" <del> "k8s.io/api/core/v1" </del> <ins> v1 "k8s.io/api/core/v1" </ins> "k8s.io/apimachinery/pkg/api/meta/testrestmapper" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

tc.runTest(t) } <ins> func TestReplicaCalcScaleDownExcludeUnscheduledPods(t *testing.T) { tc := replicaCalcTestCase{ currentReplicas:  5, expectedReplicas: 1, podReadiness:     []v1.ConditionStatus{v1.ConditionTrue, v1.ConditionFalse, v1.ConditionFalse, v1.ConditionFalse, v1.ConditionFalse}, podPhase:         []v1.PodPhase{v1.PodRunning, v1.PodPending, v1.PodPending, v1.PodPending, v1.PodPending}, resource: &resourceInfo{ name:     v1.ResourceCPU, requests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")}, levels:   []int64{100}, targetUtilization:   50, expectedUtilization: 10, expectedValue:       numContainersPerPod * 100, }, } tc.runTest(t) } </ins> func TestReplicaCalcScaleDownIgnoreHotCpuPods(t *testing.T) { tc := replicaCalcTestCase{ currentReplicas:  5,

sets.NewString("lucretius"), sets.NewString("epicurus"), }, <ins> { name: "pending pods are ignored", pods: []*v1.Pod{ { ObjectMeta: metav1.ObjectMeta{ Name: "unscheduled", }, Status: v1.PodStatus{ Phase: v1.PodPending, }, }, }, metrics:             metricsclient.PodMetricsInfo{}, resource:            v1.ResourceCPU, expectReadyPodCount: 0, expectIgnoredPods:   sets.NewString("unscheduled"), expectMissingPods:   sets.NewString(), }, </ins> } for _, tc := range tests { <del> readyPodCount, ignoredPods, missingPods := groupPods(tc.pods, tc.metrics, tc.resource, defaultTestingCpuInitializationPeriod, defaultTestingDelayOfInitialReadinessStatus) if readyPodCount != tc.expectReadyPodCount { t.Errorf("%s got readyPodCount %d, expected %d", tc.name, readyPodCount, tc.expectReadyPodCount) } if !ignoredPods.Equal(tc.expectIgnoredPods) { t.Errorf("%s got unreadyPods %v, expected %v", tc.name, ignoredPods, tc.expectIgnoredPods) } if !missingPods.Equal(tc.expectMissingPods) { t.Errorf("%s got missingPods %v, expected %v", tc.name, missingPods, tc.expectMissingPods) } </del> <ins> t.Run(tc.name, func(t *testing.T) { readyPodCount, ignoredPods, missingPods := groupPods(tc.pods, tc.metrics, tc.resource, defaultTestingCpuInitializationPeriod, defaultTestingDelayOfInitialReadinessStatus) if readyPodCount != tc.expectReadyPodCount { t.Errorf("%s got readyPodCount %d, expected %d", tc.name, readyPodCount, tc.expectReadyPodCount) } if !ignoredPods.Equal(tc.expectIgnoredPods) { t.Errorf("%s got unreadyPods %v, expected %v", tc.name, ignoredPods, tc.expectIgnoredPods) } if !missingPods.Equal(tc.expectMissingPods) { t.Errorf("%s got missingPods %v, expected %v", tc.name, missingPods, tc.expectMissingPods) } }) </ins> } }