Change:
Is this fixed via the merged PR?
There are two issues: one is fixed by and another is fixed by , however it requires also new release of Heapster which hasn't been done yet so that I haven't closed this issue.
related PRs ,
It looks like you're on top of this, but it's been failing pretty frequently this morning as well, all on "Horizontal pod autoscaling [Skipped][Autoscaling Suite] should scale from 5 pods to 3 pods and from 3 to 1 (scale resource: Memory)"
The issue is about cluster size autoscaling tests, which haven't failed since was merged. I'll close this issue tomorrow once there will be no more failures. Also there is another issue with Horizontal Pod Autoscaling test - tracking issue
The test is stable now so closing the issue.
This problem is occurring again:
blocking merges; raising to P0
looks like was merged to hopefully address this. Lowering to p1.
Yep, disabled the test, tracking issue and there is fix in flight
Closing as is resolved.

Discussion:
<del> /* Copyright 2015 The Kubernetes Authors All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package e2e import ( "fmt" "os/exec" "time" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/fields" "k8s.io/kubernetes/pkg/labels" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) const ( scaleUpTimeout   = 20 * time.Minute scaleDownTimeout = 30 * time.Minute ) var _ = Describe("Autoscaling", func() { f := NewFramework("autoscaling") var nodeCount int var coresPerNode int var memCapacityMb int BeforeEach(func() { SkipUnlessProviderIs("gce") nodes, err := f.Client.Nodes().List(labels.Everything(), fields.Everything()) expectNoError(err) nodeCount = len(nodes.Items) Expect(nodeCount).NotTo(BeZero()) cpu := nodes.Items[0].Status.Capacity[api.ResourceCPU] mem := nodes.Items[0].Status.Capacity[api.ResourceMemory] coresPerNode = int((&cpu).MilliValue() / 1000) memCapacityMb = int((&mem).Value() / 1024 / 1024) }) AfterEach(func() { cleanUpAutoscaler() }) It("[Skipped][Autoscaling Suite] should scale cluster size based on cpu utilization", func() { setUpAutoscaler("cpu/node_utilization", 0.4, nodeCount, nodeCount+1) // Consume 50% CPU millicoresPerReplica := 500 rc := NewStaticResourceConsumer("cpu-utilization", nodeCount*coresPerNode, millicoresPerReplica*nodeCount*coresPerNode, 0, int64(millicoresPerReplica), 100, f) expectNoError(waitForClusterSize(f.Client, nodeCount+1, scaleUpTimeout)) rc.CleanUp() expectNoError(waitForClusterSize(f.Client, nodeCount, scaleDownTimeout)) }) It("[Skipped][Autoscaling Suite] should scale cluster size based on cpu reservation", func() { setUpAutoscaler("cpu/node_reservation", 0.5, nodeCount, nodeCount+1) ReserveCpu(f, "cpu-reservation", 600*nodeCount*coresPerNode) expectNoError(waitForClusterSize(f.Client, nodeCount+1, scaleUpTimeout)) expectNoError(DeleteRC(f.Client, f.Namespace.Name, "cpu-reservation")) expectNoError(waitForClusterSize(f.Client, nodeCount, scaleDownTimeout)) }) It("[Skipped][Autoscaling Suite] should scale cluster size based on memory utilization", func() { setUpAutoscaler("memory/node_utilization", 0.6, nodeCount, nodeCount+1) // Consume 60% of total memory capacity megabytesPerReplica := int(memCapacityMb * 6 / 10 / coresPerNode) rc := NewStaticResourceConsumer("mem-utilization", nodeCount*coresPerNode, 0, megabytesPerReplica*nodeCount*coresPerNode, 100, int64(megabytesPerReplica+100), f) expectNoError(waitForClusterSize(f.Client, nodeCount+1, scaleUpTimeout)) rc.CleanUp() expectNoError(waitForClusterSize(f.Client, nodeCount, scaleDownTimeout)) }) It("[Skipped][Autoscaling Suite] should scale cluster size based on memory reservation", func() { setUpAutoscaler("memory/node_reservation", 0.5, nodeCount, nodeCount+1) ReserveMemory(f, "memory-reservation", nodeCount*memCapacityMb*6/10) expectNoError(waitForClusterSize(f.Client, nodeCount+1, scaleUpTimeout)) expectNoError(DeleteRC(f.Client, f.Namespace.Name, "memory-reservation")) expectNoError(waitForClusterSize(f.Client, nodeCount, scaleDownTimeout)) }) }) func setUpAutoscaler(metric string, target float64, min, max int) { // TODO integrate with kube-up.sh script once it will support autoscaler setup. By("Setting up autoscaler to scale based on " + metric) out, err := exec.Command("gcloud", "compute", "instance-groups", "managed", "set-autoscaling", testContext.CloudConfig.NodeInstanceGroup, "--project="+testContext.CloudConfig.ProjectID, "--zone="+testContext.CloudConfig.Zone, "--custom-metric-utilization=metric=custom.cloudmonitoring.googleapis.com/kubernetes.io/"+metric+fmt.Sprintf(",utilization-target=%v", target)+",utilization-target-type=GAUGE", fmt.Sprintf("--min-num-replicas=%v", min), fmt.Sprintf("--max-num-replicas=%v", max), ).CombinedOutput() expectNoError(err, "Output: "+string(out)) } func cleanUpAutoscaler() { By("Removing autoscaler") out, err := exec.Command("gcloud", "compute", "instance-groups", "managed", "stop-autoscaling", testContext.CloudConfig.NodeInstanceGroup, "--project="+testContext.CloudConfig.ProjectID, "--zone="+testContext.CloudConfig.Zone, ).CombinedOutput() expectNoError(err, "Output: "+string(out)) } func ReserveCpu(f *Framework, id string, millicores int) { By(fmt.Sprintf("Running RC which reserves %v millicores", millicores)) config := &RCConfig{ Client:     f.Client, Name:       id, Namespace:  f.Namespace.Name, Timeout:    10 * time.Minute, Image:      "beta.gcr.io/google_containers/pause:2.0", Replicas:   millicores / 100, CpuRequest: 100, } expectNoError(RunRC(*config)) } func ReserveMemory(f *Framework, id string, megabytes int) { By(fmt.Sprintf("Running RC which reserves %v MB of memory", megabytes)) config := &RCConfig{ Client:     f.Client, Name:       id, Namespace:  f.Namespace.Name, Timeout:    10 * time.Minute, Image:      "beta.gcr.io/google_containers/pause:2.0", Replicas:   megabytes / 500, MemRequest: 500 * 1024 * 1024, } expectNoError(RunRC(*config)) } </del>

requestSizeInMegabytes   int } <del> func NewDynamicResourceConsumer(name string, replicas, initCPU, initMemory int, cpuLimit, memLimit int64, framework *Framework) *ResourceConsumer { return newResourceConsumer(name, replicas, initCPU, initMemory, dynamicConsumptionTimeInSeconds, dynamicRequestSizeInMillicores, dynamicRequestSizeInMegabytes, cpuLimit, memLimit, framework) </del> <ins> func NewDynamicResourceConsumer(name string, replicas, initCPUTotal, initMemoryTotal int, cpuLimit, memLimit int64, framework *Framework) *ResourceConsumer { return newResourceConsumer(name, replicas, initCPUTotal, initMemoryTotal, dynamicConsumptionTimeInSeconds, dynamicRequestSizeInMillicores, dynamicRequestSizeInMegabytes, cpuLimit, memLimit, framework) </ins> } <del> func NewStaticResourceConsumer(name string, replicas, initCPU, initMemory int, cpuLimit, memLimit int64, framework *Framework) *ResourceConsumer { return newResourceConsumer(name, replicas, initCPU, initMemory, staticConsumptionTimeInSeconds, initCPU/replicas, initMemory/replicas, cpuLimit, memLimit, framework) </del> <ins> func NewStaticResourceConsumer(name string, replicas, initCPUTotal, initMemoryTotal int, cpuLimit, memLimit int64, framework *Framework) *ResourceConsumer { return newResourceConsumer(name, replicas, initCPUTotal, initMemoryTotal, staticConsumptionTimeInSeconds, initCPUTotal/replicas, initMemoryTotal/replicas, cpuLimit, memLimit, framework) </ins> } /* NewResourceConsumer creates new ResourceConsumer <del> initCPU argument is in millicores initMemory argument is in megabytes </del> <ins> initCPUTotal argument is in millicores initMemoryTotal argument is in megabytes </ins> memLimit argument is in megabytes, memLimit is a maximum amount of memory that can be consumed by a single pod cpuLimit argument is in millicores, cpuLimit is a maximum amount of cpu that can be consumed by a single pod */ <del> func newResourceConsumer(name string, replicas, initCPU, initMemory, consumptionTimeInSeconds, requestSizeInMillicores, requestSizeInMegabytes int, cpuLimit, memLimit int64, framework *Framework) *ResourceConsumer { </del> <ins> func newResourceConsumer(name string, replicas, initCPUTotal, initMemoryTotal, consumptionTimeInSeconds, requestSizeInMillicores, requestSizeInMegabytes int, cpuLimit, memLimit int64, framework *Framework) *ResourceConsumer { </ins> runServiceAndRCForResourceConsumer(framework.Client, framework.Namespace.Name, name, replicas, cpuLimit, memLimit) rc := &ResourceConsumer{ name:                     name,

requestSizeInMegabytes:   requestSizeInMegabytes, } go rc.makeConsumeCPURequests() <del> rc.ConsumeCPU(initCPU) </del> <ins> rc.ConsumeCPU(initCPUTotal) </ins> go rc.makeConsumeMemRequests() <del> rc.ConsumeMem(initMemory) </del> <ins> rc.ConsumeMem(initMemoryTotal) </ins> return rc }

<ins> /* Copyright 2015 The Kubernetes Authors All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package e2e import ( "fmt" "os/exec" "time" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/fields" "k8s.io/kubernetes/pkg/labels" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) const ( scaleUpTimeout   = 20 * time.Minute scaleDownTimeout = 30 * time.Minute ) var _ = Describe("Autoscaling", func() { f := NewFramework("autoscaling") var nodeCount int var coresPerNode int var memCapacityMb int BeforeEach(func() { SkipUnlessProviderIs("gce") nodes, err := f.Client.Nodes().List(labels.Everything(), fields.Everything()) expectNoError(err) nodeCount = len(nodes.Items) Expect(nodeCount).NotTo(BeZero()) cpu := nodes.Items[0].Status.Capacity[api.ResourceCPU] mem := nodes.Items[0].Status.Capacity[api.ResourceMemory] coresPerNode = int((&cpu).MilliValue() / 1000) memCapacityMb = int((&mem).Value() / 1024 / 1024) }) AfterEach(func() { cleanUpAutoscaler() }) It("[Skipped][Autoscaling Suite] should scale cluster size based on cpu utilization", func() { setUpAutoscaler("cpu/node_utilization", 0.4, nodeCount, nodeCount+1) // Consume 50% CPU rcs := createConsumingRCs(f, "cpu-utilization", nodeCount*coresPerNode, 500, 0) err := waitForClusterSize(f.Client, nodeCount+1, scaleUpTimeout) for _, rc := range rcs { rc.CleanUp() } expectNoError(err) expectNoError(waitForClusterSize(f.Client, nodeCount, scaleDownTimeout)) }) It("[Skipped][Autoscaling Suite] should scale cluster size based on cpu reservation", func() { setUpAutoscaler("cpu/node_reservation", 0.5, nodeCount, nodeCount+1) ReserveCpu(f, "cpu-reservation", 600*nodeCount*coresPerNode) expectNoError(waitForClusterSize(f.Client, nodeCount+1, scaleUpTimeout)) expectNoError(DeleteRC(f.Client, f.Namespace.Name, "cpu-reservation")) expectNoError(waitForClusterSize(f.Client, nodeCount, scaleDownTimeout)) }) It("[Skipped][Autoscaling Suite] should scale cluster size based on memory utilization", func() { setUpAutoscaler("memory/node_utilization", 0.6, nodeCount, nodeCount+1) // Consume 60% of total memory capacity megabytesPerReplica := int(memCapacityMb * 6 / 10 / coresPerNode) rcs := createConsumingRCs(f, "mem-utilization", nodeCount*coresPerNode, 0, megabytesPerReplica) err := waitForClusterSize(f.Client, nodeCount+1, scaleUpTimeout) for _, rc := range rcs { rc.CleanUp() } expectNoError(err) expectNoError(waitForClusterSize(f.Client, nodeCount, scaleDownTimeout)) }) It("[Skipped][Autoscaling Suite] should scale cluster size based on memory reservation", func() { setUpAutoscaler("memory/node_reservation", 0.5, nodeCount, nodeCount+1) ReserveMemory(f, "memory-reservation", nodeCount*memCapacityMb*6/10) expectNoError(waitForClusterSize(f.Client, nodeCount+1, scaleUpTimeout)) expectNoError(DeleteRC(f.Client, f.Namespace.Name, "memory-reservation")) expectNoError(waitForClusterSize(f.Client, nodeCount, scaleDownTimeout)) }) }) func setUpAutoscaler(metric string, target float64, min, max int) { // TODO integrate with kube-up.sh script once it will support autoscaler setup. By("Setting up autoscaler to scale based on " + metric) out, err := exec.Command("gcloud", "compute", "instance-groups", "managed", "set-autoscaling", testContext.CloudConfig.NodeInstanceGroup, "--project="+testContext.CloudConfig.ProjectID, "--zone="+testContext.CloudConfig.Zone, "--custom-metric-utilization=metric=custom.cloudmonitoring.googleapis.com/kubernetes.io/"+metric+fmt.Sprintf(",utilization-target=%v", target)+",utilization-target-type=GAUGE", fmt.Sprintf("--min-num-replicas=%v", min), fmt.Sprintf("--max-num-replicas=%v", max), ).CombinedOutput() expectNoError(err, "Output: "+string(out)) } func createConsumingRCs(f *Framework, name string, count, cpuPerReplica, memPerReplica int) []*ResourceConsumer { var res []*ResourceConsumer for i := 1; i <= count; i++ { name := fmt.Sprintf("%s-%d", name, i) res = append(res, NewStaticResourceConsumer(name, 1, cpuPerReplica, memPerReplica, int64(cpuPerReplica), int64(memPerReplica+100), f)) } return res } func cleanUpAutoscaler() { By("Removing autoscaler") out, err := exec.Command("gcloud", "compute", "instance-groups", "managed", "stop-autoscaling", testContext.CloudConfig.NodeInstanceGroup, "--project="+testContext.CloudConfig.ProjectID, "--zone="+testContext.CloudConfig.Zone, ).CombinedOutput() expectNoError(err, "Output: "+string(out)) } func ReserveCpu(f *Framework, id string, millicores int) { By(fmt.Sprintf("Running RC which reserves %v millicores", millicores)) config := &RCConfig{ Client:     f.Client, Name:       id, Namespace:  f.Namespace.Name, Timeout:    10 * time.Minute, Image:      "beta.gcr.io/google_containers/pause:2.0", Replicas:   millicores / 100, CpuRequest: 100, } expectNoError(RunRC(*config)) } func ReserveMemory(f *Framework, id string, megabytes int) { By(fmt.Sprintf("Running RC which reserves %v MB of memory", megabytes)) config := &RCConfig{ Client:     f.Client, Name:       id, Namespace:  f.Namespace.Name, Timeout:    10 * time.Minute, Image:      "beta.gcr.io/google_containers/pause:2.0", Replicas:   megabytes / 500, MemRequest: 500 * 1024 * 1024, } expectNoError(RunRC(*config)) } </ins>