Change:
}) fakeClient.AddReactor("update", "horizontalpodautoscalers", func(action core.Action) (handled bool, ret runtime.Object, err error) { <del> tc.Lock() defer tc.Unlock() obj := action.(core.UpdateAction).GetObject().(*autoscalingv1.HorizontalPodAutoscaler) assert.Equal(t, namespace, obj.Namespace, "the HPA namespace should be as expected") assert.Equal(t, hpaName, obj.Name, "the HPA name should be as expected") assert.Equal(t, tc.expectedDesiredReplicas, obj.Status.DesiredReplicas, "the desired replica count reported in the object status should be as expected") if tc.verifyCPUCurrent { if assert.NotNil(t, obj.Status.CurrentCPUUtilizationPercentage, "the reported CPU utilization percentage should be non-nil") { assert.Equal(t, tc.CPUCurrent, *obj.Status.CurrentCPUUtilizationPercentage, "the report CPU utilization percentage should be as expected") </del> <ins> handled, obj, err := func() (handled bool, ret *autoscalingv1.HorizontalPodAutoscaler, err error) { tc.Lock() defer tc.Unlock() obj := action.(core.UpdateAction).GetObject().(*autoscalingv1.HorizontalPodAutoscaler) assert.Equal(t, namespace, obj.Namespace, "the HPA namespace should be as expected") assert.Equal(t, hpaName, obj.Name, "the HPA name should be as expected") assert.Equal(t, tc.expectedDesiredReplicas, obj.Status.DesiredReplicas, "the desired replica count reported in the object status should be as expected") if tc.verifyCPUCurrent { if assert.NotNil(t, obj.Status.CurrentCPUUtilizationPercentage, "the reported CPU utilization percentage should be non-nil") { assert.Equal(t, tc.CPUCurrent, *obj.Status.CurrentCPUUtilizationPercentage, "the report CPU utilization percentage should be as expected") } </ins> } <ins> var actualConditions []autoscalingv1.HorizontalPodAutoscalerCondition if err := json.Unmarshal([]byte(obj.ObjectMeta.Annotations[autoscaling.HorizontalPodAutoscalerConditionsAnnotation]), &actualConditions); err != nil { return true, nil, err } // TODO: it's ok not to sort these becaues statusOk // contains all the conditions, so we'll never be appending. // Default to statusOk when missing any specific conditions if tc.expectedConditions == nil { tc.expectedConditions = statusOkWithOverrides() } // clear the message so that we can easily compare for i := range actualConditions { actualConditions[i].Message = "" actualConditions[i].LastTransitionTime = metav1.Time{} } assert.Equal(t, tc.expectedConditions, actualConditions, "the status conditions should have been as expected") tc.statusUpdated = true // Every time we reconcile HPA object we are updating status. return true, obj, nil }() if obj != nil { tc.processed <- obj.Name </ins> } <del> var actualConditions []autoscalingv1.HorizontalPodAutoscalerCondition if err := json.Unmarshal([]byte(obj.ObjectMeta.Annotations[autoscaling.HorizontalPodAutoscalerConditionsAnnotation]), &actualConditions); err != nil { return true, nil, err } // TODO: it's ok not to sort these becaues statusOk // contains all the conditions, so we'll never be appending. // Default to statusOk when missing any specific conditions if tc.expectedConditions == nil { tc.expectedConditions = statusOkWithOverrides() } // clear the message so that we can easily compare for i := range actualConditions { actualConditions[i].Message = "" actualConditions[i].LastTransitionTime = metav1.Time{} } assert.Equal(t, tc.expectedConditions, actualConditions, "the status conditions should have been as expected") tc.statusUpdated = true // Every time we reconcile HPA object we are updating status. tc.processed <- obj.Name return true, obj, nil </del> <ins> return handled, obj, err </ins> }) fakeScaleClient := &scalefake.FakeScaleClient{}

Discussion:
What happened: The unit test for Horizontal Pod Autoscaler frequently deadlocked on the OpenShift CI system. What you expected to happen: The unit test should be robust and succeed without having to win a race condition to avoid deadlock. How to reproduce it (as minimally and precisely as possible): I'm not sure. Anything else we need to know?: Here are some logs of the problem happening: https://openshift-gce- The deadlocks seem to happen when the chan fills up while waiting for events. If that happens, there's a race where immediately after an event is read but before it can acquire the lock to verify the test, 2 event reactors are able to acquire the lock and write to the chan. The second will block forever since the chan is full and there's nobody to read from it. I've opened a PR with the fix. See
/sig autoscaling
/assign