Change:
/kind bug What happened: HPA fails to launch additional pods due to pods in evicted state, even with utilization significantly over target. What you expected to happen: When HPA pods reach the target utilization, new pods should be allocated and provisioned. How to reproduce it (as minimally and precisely as possible): Launch a HPA, let the pods fill the disk on the host which will trigger evictions due to "Pod The node was low on resource: [DiskPressure]", HPA will exceed target and not allocate additional pods. Anything else we need to know?: Seems related to and though those are not specific to HPA. Environment: Kubernetes version (use ):Cloud provider or hardware configuration: GKE OS (e.g. from /etc/os-release): cos-stable-57-9202-64-0 kernel commit id Kernel (e.g. ):Linux gke-prod1-use1b-v2--35s4 4.4.70+ SMP Mon Sep 18 12:43:08 PDT 2017 x86_64 Intel(R) Xeon(R) CPU @ 2.30GHz GenuineIntel GNU/Linux
/sig scheduling
Are there any workarounds while we wait for a fix? We have ran into this on a 1.8.4 GKE cluster.
/sig autoscaling /remove-sig scheduling
I am thinking of fixing this by excluding pods with Phase=Failed from the unreadyPods that are taken into account. It seems the right thing to do as they will not become running again. Do you see any issues?
that seems decent -- anything that will never be running again should not be factored into our calculations.
/status approved-for-milestone Adding to milestone -- if the HPA isn't functioning in the case of evictions, that seems like a decently severe HPA bug.
oops, missing priority was my fault, moving back into milestone: /priority critical-urgent /remove-priority important-soon
[MILESTONENOTIFIER] Milestone Issue Needs Attention Action required: During code freeze, issues in the milestone should be in progress. If this issue is not being actively worked on, please remove it from the milestone. If it is being worked on, please add the label so it can be tracked with other in-flight issues. Note: This issue is marked as , and must be updated every 1 day during code freeze. Example update: <details<summaryIssue Labels</summary- : Issue will be escalated to these SIGs if needed. : Never automatically move issue out of a release milestone; continually escalate to contributor and SIG through all available channels. : Fixes a bug discovered during the current release. </details<details<summaryHelp</summary<ul<li<a href=""Additional instructions</a</li<li<a href=""Commands for setting labels</a</li</ul</details

Discussion:
reportedLevels       []uint64 reportedCPURequests  []resource.Quantity reportedPodReadiness []v1.ConditionStatus <ins> reportedPodPhase     []v1.PodPhase </ins> scaleUpdated         bool statusUpdated        bool eventCreated         bool

if tc.reportedPodReadiness != nil { podReadiness = tc.reportedPodReadiness[i] } <ins> podPhase := v1.PodRunning if tc.reportedPodPhase != nil { podPhase = tc.reportedPodPhase[i] } </ins> podName := fmt.Sprintf("%s-%d", podNamePrefix, i) pod := v1.Pod{ Status: v1.PodStatus{ <del> Phase: v1.PodRunning, </del> <ins> Phase: podPhase, </ins> Conditions: []v1.PodCondition{ { Type:   v1.PodReady,

tc.runTest(t) } <ins> func TestScaleUpIgnoresFailedPods(t *testing.T) { tc := testCase{ minReplicas:          2, maxReplicas:          6, initialReplicas:      2, desiredReplicas:      4, CPUTarget:            30, CPUCurrent:           60, verifyCPUCurrent:     true, reportedLevels:       []uint64{500, 700}, reportedCPURequests:  []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")}, reportedPodReadiness: []v1.ConditionStatus{v1.ConditionTrue, v1.ConditionTrue, v1.ConditionFalse, v1.ConditionFalse}, reportedPodPhase:     []v1.PodPhase{v1.PodRunning, v1.PodRunning, v1.PodFailed, v1.PodFailed}, useMetricsAPI:        true, } tc.runTest(t) } </ins> func TestScaleUpDeployment(t *testing.T) { tc := testCase{ minReplicas:         2,

tc.runTest(t) } <ins> func TestScaleDownIgnoresFailedPods(t *testing.T) { tc := testCase{ minReplicas:          2, maxReplicas:          6, initialReplicas:      5, desiredReplicas:      3, CPUTarget:            50, CPUCurrent:           28, verifyCPUCurrent:     true, reportedLevels:       []uint64{100, 300, 500, 250, 250}, reportedCPURequests:  []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")}, useMetricsAPI:        true, reportedPodReadiness: []v1.ConditionStatus{v1.ConditionTrue, v1.ConditionTrue, v1.ConditionTrue, v1.ConditionTrue, v1.ConditionTrue, v1.ConditionFalse, v1.ConditionFalse}, reportedPodPhase:     []v1.PodPhase{v1.PodRunning, v1.PodRunning, v1.PodRunning, v1.PodRunning, v1.PodRunning, v1.PodFailed, v1.PodFailed}, } tc.runTest(t) } </ins> func TestTolerance(t *testing.T) { tc := testCase{ minReplicas:         1,

if pod.Status.Phase != v1.PodRunning || !podutil.IsPodReady(&pod) { // save this pod name for later, but pretend it doesn't exist for now <del> unreadyPods.Insert(pod.Name) </del> <ins> if pod.Status.Phase != v1.PodFailed { // Failed pods should not be counted as unready pods as they will // not become running anymore. unreadyPods.Insert(pod.Name) } </ins> delete(metrics, pod.Name) continue }

metric   *metricInfo podReadiness []v1.ConditionStatus <ins> podPhase     []v1.PodPhase </ins> } const (

fakeClient := &fake.Clientset{} fakeClient.AddReactor("list", "pods", func(action core.Action) (handled bool, ret runtime.Object, err error) { obj := &v1.PodList{} <del> for i := 0; i < int(tc.currentReplicas); i++ { </del> <ins> podsCount := int(tc.currentReplicas) // Failed pods are not included in tc.currentReplicas if tc.podPhase != nil && len(tc.podPhase) > podsCount { podsCount = len(tc.podPhase) } for i := 0; i < podsCount; i++ { </ins> podReadiness := v1.ConditionTrue <del> if tc.podReadiness != nil { </del> <ins> if tc.podReadiness != nil && i < len(tc.podReadiness) { </ins> podReadiness = tc.podReadiness[i] } <ins> podPhase := v1.PodRunning if tc.podPhase != nil { podPhase = tc.podPhase[i] } </ins> podName := fmt.Sprintf("%s-%d", podNamePrefix, i) pod := v1.Pod{ Status: v1.PodStatus{ <del> Phase: v1.PodRunning, </del> <ins> Phase: podPhase, </ins> Conditions: []v1.PodCondition{ { Type:   v1.PodReady,

tc.runTest(t) } <ins> func TestReplicaCalcScaleUpIgnoresFailedPods(t *testing.T) { tc := replicaCalcTestCase{ currentReplicas:  2, expectedReplicas: 4, podReadiness:     []v1.ConditionStatus{v1.ConditionTrue, v1.ConditionTrue, v1.ConditionFalse, v1.ConditionFalse}, podPhase:         []v1.PodPhase{v1.PodRunning, v1.PodRunning, v1.PodFailed, v1.PodFailed}, resource: &resourceInfo{ name:     v1.ResourceCPU, requests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")}, levels:   []int64{500, 700}, targetUtilization:   30, expectedUtilization: 60, expectedValue:       numContainersPerPod * 600, }, } tc.runTest(t) } </ins> func TestReplicaCalcScaleUpCM(t *testing.T) { tc := replicaCalcTestCase{ currentReplicas:  3,

tc.runTest(t) } <ins> func TestReplicaCalcScaleDownIgnoresFailedPods(t *testing.T) { tc := replicaCalcTestCase{ currentReplicas:  5, expectedReplicas: 3, podReadiness:     []v1.ConditionStatus{v1.ConditionTrue, v1.ConditionTrue, v1.ConditionTrue, v1.ConditionTrue, v1.ConditionTrue, v1.ConditionFalse, v1.ConditionFalse}, podPhase:         []v1.PodPhase{v1.PodRunning, v1.PodRunning, v1.PodRunning, v1.PodRunning, v1.PodRunning, v1.PodFailed, v1.PodFailed}, resource: &resourceInfo{ name:     v1.ResourceCPU, requests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")}, levels:   []int64{100, 300, 500, 250, 250}, targetUtilization:   50, expectedUtilization: 28, expectedValue:       numContainersPerPod * 280, }, } tc.runTest(t) } </ins> func TestReplicaCalcTolerance(t *testing.T) { tc := replicaCalcTestCase{ currentReplicas:  3,