Change:
If there is a connection failure when updating the Pod status, we don't retry. Furthermore, due to , we won't have another chance until much later. The problem is that if we don't mark the Pod as Unschedulable, other controllers (such as cluster-autoscaler) wouldn't react to these pods. We should have stronger guarantees to mark a Pod as Unschedulable. You need an unschedulable Pod (for example, a Pod with a non matching node affinity) and a flaky connection to apiserver. No response master This is particularly problematic in 1.24, due to . Any <details</details<details</details<details</details<details</details
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig scheduling
That is bad, retrying in place on transient errors should probably be ok.
I'll address it. /assign
Thanks!

Discussion:
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" utilerrors "k8s.io/apimachinery/pkg/util/errors" <ins> "k8s.io/apimachinery/pkg/util/net" </ins> "k8s.io/apimachinery/pkg/util/strategicpatch" "k8s.io/client-go/kubernetes" <ins> "k8s.io/client-go/util/retry" </ins> corev1helpers "k8s.io/component-helpers/scheduling/corev1" "k8s.io/klog/v2" extenderv1 "k8s.io/kube-scheduler/extender/v1"

return nil } <del> _, err = cs.CoreV1().Pods(old.Namespace).Patch(ctx, old.Name, types.StrategicMergePatchType, patchBytes, metav1.PatchOptions{}, "status") return err </del> <ins> patchFn := func() error { _, err := cs.CoreV1().Pods(old.Namespace).Patch(ctx, old.Name, types.StrategicMergePatchType, patchBytes, metav1.PatchOptions{}, "status") return err } return retry.OnError(retry.DefaultBackoff, net.IsConnectionRefused, patchFn) </ins> } // DeletePod deletes the given <pod> from API server

import ( "context" <ins> "errors" </ins> "fmt" <ins> "syscall" </ins> "testing" "time"

v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" <ins> "k8s.io/apimachinery/pkg/util/net" </ins> clientsetfake "k8s.io/client-go/kubernetes/fake" clienttesting "k8s.io/client-go/testing" extenderv1 "k8s.io/kube-scheduler/extender/v1"

func TestPatchPodStatus(t *testing.T) { tests := []struct { <del> name           string pod            v1.Pod </del> <ins> name   string pod    v1.Pod client *clientsetfake.Clientset // validateErr checks if error returned from PatchPodStatus is expected one or not. // (true means error is expected one.) validateErr    func(goterr error) bool </ins> statusToUpdate v1.PodStatus }{ { <del> name: "Should update pod conditions successfully", </del> <ins> name:   "Should update pod conditions successfully", client: clientsetfake.NewSimpleClientset(), </ins> pod: v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: "ns",

// ref: #101697, #94626 - ImagePullSecrets are allowed to have empty secret names // which would fail the 2-way merge patch generation on Pod patches // due to the mergeKey being the name field <del> name: "Should update pod conditions successfully on a pod Spec with secrets with empty name", </del> <ins> name:   "Should update pod conditions successfully on a pod Spec with secrets with empty name", client: clientsetfake.NewSimpleClientset(), </ins> pod: v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: "ns", <del> Name:      "pod2", </del> <ins> Name:      "pod1", </ins> }, Spec: v1.PodSpec{ // this will serialize to imagePullSecrets:[{}]

}, }, }, <ins> { name: "retry patch request when an 'connection refused' error is returned", client: func() *clientsetfake.Clientset { client := clientsetfake.NewSimpleClientset() reqcount := 0 client.PrependReactor("patch", "pods", func(action clienttesting.Action) (bool, runtime.Object, error) { defer func() { reqcount++ }() if reqcount == 0 { // return an connection refused error for the first patch request. return true, &v1.Pod{}, fmt.Errorf("connection refused: %w", syscall.ECONNREFUSED) } if reqcount == 1 { // not return error for the second patch request. return false, &v1.Pod{}, nil } // return error if requests comes in more than three times. return true, nil, errors.New("requests comes in more than three times.") }) return client }(), pod: v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: "ns", Name:      "pod1", }, Spec: v1.PodSpec{ ImagePullSecrets: []v1.LocalObjectReference{{Name: "foo"}}, }, }, statusToUpdate: v1.PodStatus{ Conditions: []v1.PodCondition{ { Type:   v1.PodScheduled, Status: v1.ConditionFalse, }, }, }, }, { name: "only 4 retries at most", client: func() *clientsetfake.Clientset { client := clientsetfake.NewSimpleClientset() reqcount := 0 client.PrependReactor("patch", "pods", func(action clienttesting.Action) (bool, runtime.Object, error) { defer func() { reqcount++ }() if reqcount >= 4 { // return error if requests comes in more than four times. return true, nil, errors.New("requests comes in more than four times.") } // return an connection refused error for the first patch request. return true, &v1.Pod{}, fmt.Errorf("connection refused: %w", syscall.ECONNREFUSED) }) return client }(), pod: v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: "ns", Name:      "pod1", }, Spec: v1.PodSpec{ ImagePullSecrets: []v1.LocalObjectReference{{Name: "foo"}}, }, }, validateErr: net.IsConnectionRefused, statusToUpdate: v1.PodStatus{ Conditions: []v1.PodCondition{ { Type:   v1.PodScheduled, Status: v1.ConditionFalse, }, }, }, }, </ins> } <del> client := clientsetfake.NewSimpleClientset() </del> for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { <ins> client := tc.client </ins> _, err := client.CoreV1().Pods(tc.pod.Namespace).Create(context.TODO(), &tc.pod, metav1.CreateOptions{}) if err != nil { t.Fatal(err)

ctx, cancel := context.WithCancel(context.Background()) defer cancel() err = PatchPodStatus(ctx, client, &tc.pod, &tc.statusToUpdate) <del> if err != nil { </del> <ins> if err != nil && tc.validateErr == nil { // shouldn't be error </ins> t.Fatal(err) } <ins> if tc.validateErr != nil { if !tc.validateErr(err) { t.Fatalf("Returned unexpected error: %v", err) } return } </ins> retrievedPod, err := client.CoreV1().Pods(tc.pod.Namespace).Get(ctx, tc.pod.Name, metav1.GetOptions{}) if err != nil {