Code change:
// return the current replicas if the change would be too small return currentReplicas, utilization, timestamp, nil } <del> replicaCount = int32(math.Ceil(usageRatio * float64(currentReplicas))) </del> <ins> readyPodCount, err := c.getReadyPodsCount(namespace, selector) if err != nil { return 0, 0, time.Time{}, fmt.Errorf("unable to calculate ready pods: %s", err) } replicaCount = int32(math.Ceil(usageRatio * float64(readyPodCount))) </ins> return replicaCount, utilization, timestamp, nil } <ins> // @TODO(mattjmcnaughton) Many different functions in this module use variations // of this function. Make this function generic, so we don't repeat the same // logic in multiple places. func (c *ReplicaCalculator) getReadyPodsCount(namespace string, selector labels.Selector) (int64, error) { podList, err := c.podsGetter.Pods(namespace).List(metav1.ListOptions{LabelSelector: selector.String()}) if err != nil { return 0, fmt.Errorf("unable to get pods while calculating replica count: %v", err) } if len(podList.Items) == 0 { return 0, fmt.Errorf("no pods returned by selector while calculating replica count") } readyPodCount := 0 for _, pod := range podList.Items { if pod.Status.Phase == v1.PodRunning && podutil.IsPodReady(&pod) { readyPodCount++ } } return int64(readyPodCount), nil } </ins> // GetExternalMetricReplicas calculates the desired replica count based on a // target metric value (as a milli-value) for the external metric in the given // namespace, and the current replica count. <del> func (c *ReplicaCalculator) GetExternalMetricReplicas(currentReplicas int32, targetUtilization int64, metricName, namespace string, selector *metav1.LabelSelector) (replicaCount int32, utilization int64, timestamp time.Time, err error) { labelSelector, err := metav1.LabelSelectorAsSelector(selector) </del> <ins> func (c *ReplicaCalculator) GetExternalMetricReplicas(currentReplicas int32, targetUtilization int64, metricName, namespace string, metricSelector *metav1.LabelSelector, podSelector labels.Selector) (replicaCount int32, utilization int64, timestamp time.Time, err error) { metricLabelSelector, err := metav1.LabelSelectorAsSelector(metricSelector) </ins> if err != nil { return 0, 0, time.Time{}, err } <del> metrics, timestamp, err := c.metricsClient.GetExternalMetric(metricName, namespace, labelSelector) </del> <ins> metrics, timestamp, err := c.metricsClient.GetExternalMetric(metricName, namespace, metricLabelSelector) </ins> if err != nil { <del> return 0, 0, time.Time{}, fmt.Errorf("unable to get external metric %s/%s/%+v: %s", namespace, metricName, selector, err) </del> <ins> return 0, 0, time.Time{}, fmt.Errorf("unable to get external metric %s/%s/%+v: %s", namespace, metricName, metricSelector, err) </ins> } utilization = 0 for _, val := range metrics { utilization = utilization + val } <ins> readyPodCount, err := c.getReadyPodsCount(namespace, podSelector) if err != nil { return 0, 0, time.Time{}, fmt.Errorf("unable to calculate ready pods: %s", err) } </ins> usageRatio := float64(utilization) / float64(targetUtilization) if math.Abs(1.0-usageRatio) <= c.tolerance { // return the current replicas if the change would be too small return currentReplicas, utilization, timestamp, nil } <del> return int32(math.Ceil(usageRatio * float64(currentReplicas))), utilization, timestamp, nil </del> <ins> return int32(math.Ceil(usageRatio * float64(readyPodCount))), utilization, timestamp, nil </ins> } // GetExternalPerPodMetricReplicas calculates the desired replica count based on a // target metric value per pod (as a milli-value) for the external metric in the // given namespace, and the current replica count. <del> func (c *ReplicaCalculator) GetExternalPerPodMetricReplicas(currentReplicas int32, targetUtilizationPerPod int64, metricName, namespace string, selector *metav1.LabelSelector) (replicaCount int32, utilization int64, timestamp time.Time, err error) { labelSelector, err := metav1.LabelSelectorAsSelector(selector) </del> <ins> func (c *ReplicaCalculator) GetExternalPerPodMetricReplicas(currentReplicas int32, targetUtilizationPerPod int64, metricName, namespace string, metricSelector *metav1.LabelSelector) (replicaCount int32, utilization int64, timestamp time.Time, err error) { metricLabelSelector, err := metav1.LabelSelectorAsSelector(metricSelector) </ins> if err != nil { return 0, 0, time.Time{}, err } <del> metrics, timestamp, err := c.metricsClient.GetExternalMetric(metricName, namespace, labelSelector) </del> <ins> metrics, timestamp, err := c.metricsClient.GetExternalMetric(metricName, namespace, metricLabelSelector) </ins> if err != nil { <del> return 0, 0, time.Time{}, fmt.Errorf("unable to get external metric %s/%s/%+v: %s", namespace, metricName, selector, err) </del> <ins> return 0, 0, time.Time{}, fmt.Errorf("unable to get external metric %s/%s/%+v: %s", namespace, metricName, metricSelector, err) </ins> } utilization = 0 for _, val := range metrics {

Related discussion:
text: WDYT
Ah, yeah, I missed that it was object metrics before. I'm not entirely convinced we need a flag here -- it might just make sense to bring Object metrics in line with all the other metrics types. On the other hand, there's a subtlety here -- the other metric types don't quite ignore unready pods -- they just ignore the metrics from unready pods, which causes a heavy bias in the direction of not scaling, but an overwhelming spike can still cause scaling. That behavior is not as achievable using object metrics. I'm inclined to just ignore that subtlety, though, and say that object metrics should ignore unready pods entirely.
I think this is what we should do long term. I was worried about breaking someone who relies on targetValue + averaged metric to get effect similar to targetAverageValue and though about providing something for transitional period, but if you're ok with it I'd rather just change it as you say.
I think it's safe to change as-is -- that feature's still in beta, and it's not an API change, so it shouldn't just hard-break anyone. Plus, we've change stuff like this before (when we did the initial resource metrics change).
As said, we are more than ready to help with the fix/test if required. This since this is a major blocking for us in the implementation of the HPA in our side
is anyone actively working on implementing this? If not, I'm happy to. I think I have a fairly clear picture of the change we need to make and will likely have time early this week to get it done.
(Also, from the conversation above, to we want to tag this as a release blocker for 1.10)?
I probably won't be able to work on this before the end of this week. Happy to do a review if you jump in and implement it.
I don't think this qualifies as a release blocking bug (but I'm willing to have someone try and convince me otherwise ;-)). There doesn't seem to be any behavior here that's explicitly broken -- it's just sub-optimal in some use cases. I'd definitely like to address this in 1.11, though.
it's not release blocker, but to share our experience ;).