Change:
}, }, }, <ins> { name: "retry patch request when an 'connection refused' error is returned", client: func() *clientsetfake.Clientset { client := clientsetfake.NewSimpleClientset() reqcount := 0 client.PrependReactor("patch", "pods", func(action clienttesting.Action) (bool, runtime.Object, error) { defer func() { reqcount++ }() if reqcount == 0 { // return an connection refused error for the first patch request. return true, &v1.Pod{}, fmt.Errorf("connection refused: %w", syscall.ECONNREFUSED) } if reqcount == 1 { // not return error for the second patch request. return false, &v1.Pod{}, nil } // return error if requests comes in more than three times. return true, nil, errors.New("requests comes in more than three times.") }) return client }(), pod: v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: "ns", Name:      "pod1", }, Spec: v1.PodSpec{ ImagePullSecrets: []v1.LocalObjectReference{{Name: "foo"}}, }, }, statusToUpdate: v1.PodStatus{ Conditions: []v1.PodCondition{ { Type:   v1.PodScheduled, Status: v1.ConditionFalse, }, }, }, }, { name: "only 4 retries at most", client: func() *clientsetfake.Clientset { client := clientsetfake.NewSimpleClientset() reqcount := 0 client.PrependReactor("patch", "pods", func(action clienttesting.Action) (bool, runtime.Object, error) { defer func() { reqcount++ }() if reqcount >= 4 { // return error if requests comes in more than four times. return true, nil, errors.New("requests comes in more than four times.") } // return an connection refused error for the first patch request. return true, &v1.Pod{}, fmt.Errorf("connection refused: %w", syscall.ECONNREFUSED) }) return client }(), pod: v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: "ns", Name:      "pod1", }, Spec: v1.PodSpec{ ImagePullSecrets: []v1.LocalObjectReference{{Name: "foo"}}, }, }, validateErr: net.IsConnectionRefused, statusToUpdate: v1.PodStatus{ Conditions: []v1.PodCondition{ { Type:   v1.PodScheduled, Status: v1.ConditionFalse, }, }, }, }, </ins> } <del> client := clientsetfake.NewSimpleClientset() </del> for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { <ins> client := tc.client </ins> _, err := client.CoreV1().Pods(tc.pod.Namespace).Create(context.TODO(), &tc.pod, metav1.CreateOptions{}) if err != nil { t.Fatal(err)

Discussion:
If there is a connection failure when updating the Pod status, we don't retry. Furthermore, due to , we won't have another chance until much later. The problem is that if we don't mark the Pod as Unschedulable, other controllers (such as cluster-autoscaler) wouldn't react to these pods. We should have stronger guarantees to mark a Pod as Unschedulable. You need an unschedulable Pod (for example, a Pod with a non matching node affinity) and a flaky connection to apiserver. No response master This is particularly problematic in 1.24, due to . Any <details</details<details</details<details</details<details</details
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig scheduling
That is bad, retrying in place on transient errors should probably be ok.
I'll address it. /assign
Thanks!