Change:
What happened: The unit test for Horizontal Pod Autoscaler frequently deadlocked on the OpenShift CI system. What you expected to happen: The unit test should be robust and succeed without having to win a race condition to avoid deadlock. How to reproduce it (as minimally and precisely as possible): I'm not sure. Anything else we need to know?: Here are some logs of the problem happening: https://openshift-gce- The deadlocks seem to happen when the chan fills up while waiting for events. If that happens, there's a race where immediately after an event is read but before it can acquire the lock to verify the test, 2 event reactors are able to acquire the lock and write to the chan. The second will block forever since the chan is full and there's nobody to read from it. I've opened a PR with the fix. See
/sig autoscaling
/assign

Discussion:
}) fakeClient.AddReactor("update", "horizontalpodautoscalers", func(action core.Action) (handled bool, ret runtime.Object, err error) { <del> tc.Lock() defer tc.Unlock() obj := action.(core.UpdateAction).GetObject().(*autoscalingv1.HorizontalPodAutoscaler) assert.Equal(t, namespace, obj.Namespace, "the HPA namespace should be as expected") assert.Equal(t, hpaName, obj.Name, "the HPA name should be as expected") assert.Equal(t, tc.expectedDesiredReplicas, obj.Status.DesiredReplicas, "the desired replica count reported in the object status should be as expected") if tc.verifyCPUCurrent { if assert.NotNil(t, obj.Status.CurrentCPUUtilizationPercentage, "the reported CPU utilization percentage should be non-nil") { assert.Equal(t, tc.CPUCurrent, *obj.Status.CurrentCPUUtilizationPercentage, "the report CPU utilization percentage should be as expected") </del> <ins> handled, obj, err := func() (handled bool, ret *autoscalingv1.HorizontalPodAutoscaler, err error) { tc.Lock() defer tc.Unlock() obj := action.(core.UpdateAction).GetObject().(*autoscalingv1.HorizontalPodAutoscaler) assert.Equal(t, namespace, obj.Namespace, "the HPA namespace should be as expected") assert.Equal(t, hpaName, obj.Name, "the HPA name should be as expected") assert.Equal(t, tc.expectedDesiredReplicas, obj.Status.DesiredReplicas, "the desired replica count reported in the object status should be as expected") if tc.verifyCPUCurrent { if assert.NotNil(t, obj.Status.CurrentCPUUtilizationPercentage, "the reported CPU utilization percentage should be non-nil") { assert.Equal(t, tc.CPUCurrent, *obj.Status.CurrentCPUUtilizationPercentage, "the report CPU utilization percentage should be as expected") } </ins> } <ins> var actualConditions []autoscalingv1.HorizontalPodAutoscalerCondition if err := json.Unmarshal([]byte(obj.ObjectMeta.Annotations[autoscaling.HorizontalPodAutoscalerConditionsAnnotation]), &actualConditions); err != nil { return true, nil, err } // TODO: it's ok not to sort these becaues statusOk // contains all the conditions, so we'll never be appending. // Default to statusOk when missing any specific conditions if tc.expectedConditions == nil { tc.expectedConditions = statusOkWithOverrides() } // clear the message so that we can easily compare for i := range actualConditions { actualConditions[i].Message = "" actualConditions[i].LastTransitionTime = metav1.Time{} } assert.Equal(t, tc.expectedConditions, actualConditions, "the status conditions should have been as expected") tc.statusUpdated = true // Every time we reconcile HPA object we are updating status. return true, obj, nil }() if obj != nil { tc.processed <- obj.Name </ins> } <del> var actualConditions []autoscalingv1.HorizontalPodAutoscalerCondition if err := json.Unmarshal([]byte(obj.ObjectMeta.Annotations[autoscaling.HorizontalPodAutoscalerConditionsAnnotation]), &actualConditions); err != nil { return true, nil, err } // TODO: it's ok not to sort these becaues statusOk // contains all the conditions, so we'll never be appending. // Default to statusOk when missing any specific conditions if tc.expectedConditions == nil { tc.expectedConditions = statusOkWithOverrides() } // clear the message so that we can easily compare for i := range actualConditions { actualConditions[i].Message = "" actualConditions[i].LastTransitionTime = metav1.Time{} } assert.Equal(t, tc.expectedConditions, actualConditions, "the status conditions should have been as expected") tc.statusUpdated = true // Every time we reconcile HPA object we are updating status. tc.processed <- obj.Name return true, obj, nil </del> <ins> return handled, obj, err </ins> }) fakeScaleClient := &scalefake.FakeScaleClient{}

go hpaController.Run(stop) tc.Lock() <del> if tc.verifyEvents { tc.Unlock() </del> <ins> shouldWait := tc.verifyEvents tc.Unlock() if shouldWait { </ins> // We need to wait for events to be broadcasted (sleep for longer than record.sleepDuration). <del> time.Sleep(2 * time.Second) </del> <ins> timeoutTime := time.Now().Add(2 * time.Second) for now := time.Now(); timeoutTime.After(now); now = time.Now() { sleepUntil := timeoutTime.Sub(now) select { case <-tc.processed: // drain the chan of any sent events to keep it from filling before the timeout case <-time.After(sleepUntil): // timeout reached, ready to verifyResults } } </ins> } else { <del> tc.Unlock() </del> <ins> // Wait for HPA to be processed. <-tc.processed </ins> } <del> // Wait for HPA to be processed. <-tc.processed </del> tc.verifyResults(t) }

// wait a tick and then mark that we're finished (otherwise, we have no // way to indicate that we're finished, because the function decides not to do anything) time.Sleep(1 * time.Second) <ins> tc.Lock() </ins> tc.statusUpdated = true <ins> tc.Unlock() </ins> tc.processed <- "test-hpa" }()

return true, objv1, nil }) testClient.PrependReactor("update", "horizontalpodautoscalers", func(action core.Action) (handled bool, ret runtime.Object, err error) { <del> tc.Lock() defer tc.Unlock() </del> assert.Fail(t, "should not have attempted to update the HPA when nothing changed") // mark that we've processed this HPA tc.processed <- ""

}) fakeClient.AddReactor("update", "horizontalpodautoscalers", func(action core.Action) (handled bool, ret runtime.Object, err error) { <del> tc.Lock() defer tc.Unlock() obj := action.(core.UpdateAction).GetObject().(*autoscalingv1.HorizontalPodAutoscaler) assert.Equal(t, namespace, obj.Namespace, "the HPA namespace should be as expected") assert.Equal(t, hpaName, obj.Name, "the HPA name should be as expected") assert.Equal(t, tc.desiredReplicas, obj.Status.DesiredReplicas, "the desired replica count reported in the object status should be as expected") if tc.verifyCPUCurrent { if assert.NotNil(t, obj.Status.CurrentCPUUtilizationPercentage, "the reported CPU utilization percentage should be non-nil") { assert.Equal(t, tc.CPUCurrent, *obj.Status.CurrentCPUUtilizationPercentage, "the report CPU utilization percentage should be as expected") </del> <ins> obj := func() *autoscalingv1.HorizontalPodAutoscaler { tc.Lock() defer tc.Unlock() obj := action.(core.UpdateAction).GetObject().(*autoscalingv1.HorizontalPodAutoscaler) assert.Equal(t, namespace, obj.Namespace, "the HPA namespace should be as expected") assert.Equal(t, hpaName, obj.Name, "the HPA name should be as expected") assert.Equal(t, tc.desiredReplicas, obj.Status.DesiredReplicas, "the desired replica count reported in the object status should be as expected") if tc.verifyCPUCurrent { if assert.NotNil(t, obj.Status.CurrentCPUUtilizationPercentage, "the reported CPU utilization percentage should be non-nil") { assert.Equal(t, tc.CPUCurrent, *obj.Status.CurrentCPUUtilizationPercentage, "the report CPU utilization percentage should be as expected") } </ins> } <del> } tc.statusUpdated = true </del> <ins> tc.statusUpdated = true return obj }() </ins> // Every time we reconcile HPA object we are updating status. tc.processed <- obj.Name return true, obj, nil