Change:
My application is clustered and I have 2 pods running. I am using a rolling upgrade strategy and my partitioning number is 1. First I need to upgrade the pod with the ordinal number 1 so I triggered that upgrade with partitioning number 1 and my upgrade gets completed. After this, my pod having the ordinal number 0 gets restarted and it's automatically upgraded to a new image. This is not an expected behavior because still, my partition number is 1 and pod-0 should come up with an older image only. I am facing this issue for the v1.27 cluster. Prior to this, it was working fine. pod-0 should not upgrade to new image until I upgrade the partition number to 0.a Do helm install and upgrade pod-1 only keeping partition number to 1. No response <detailsconsole $ cat /etc/os-release $ uname -a C:wmic os get Caption, Version, BuildNumber, OSArchitecture </details<details</details<details</details<details</details
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig/api-machinery
/sig apps /sig storage
Hi, Can someone suggest here? As this is introduced in my setup after the v1.27 upgrade? Is there any change in partition logic?
This can be reproduced by using the said STS. 1 - Deploy STS using attached that has a replica count is 2 and updatingstrategy.rolling.partition as 1. 2 - Do edit STS and update the image to 1.15. This will upgrade, pod-1 with image 1.15. 3 - Next delete the pod pod-0 and check that POD-0 came up with a new image which is not the expected behavior. POD 0 should not be updated to a new image until the partition is changed to 0. We have noticed this behavior change in v1.27. Pla suggest : apiVersion: v1 kind: Service metadata: name: nginx labels: app: nginx spec: ports: port: 80 name: web clusterIP: None selector: app: nginx apiVersion: apps/v1 kind: StatefulSet metadata: name: web-am spec: updateStrategy: type: RollingUpdate rollingUpdate: partition: 1 serviceName: "nginx" replicas: 2 selector: matchLabels: app: nginx template: metadata: labels: app: nginx spec: containers: name: nginx args: /bin/sh -c sleep 30;touch /tmp/healthy; sleep 3600 image: nginx:1.14.2 ports: containerPort: 80 name: web

Discussion:
// are set to 0. func completeRollingUpdate(set *apps.StatefulSet, status *apps.StatefulSetStatus) { if set.Spec.UpdateStrategy.Type == apps.RollingUpdateStatefulSetStrategyType && <del> status.UpdatedReplicas == status.Replicas && status.ReadyReplicas == status.Replicas { </del> <ins> status.UpdatedReplicas == *set.Spec.Replicas && status.ReadyReplicas == *set.Spec.Replicas && status.Replicas == *set.Spec.Replicas { </ins> status.CurrentReplicas = status.UpdatedReplicas status.CurrentRevision = status.UpdateRevision }

statefulSetTimeout = 10 * time.Minute // statefulPodTimeout is a timeout for stateful pods to change state statefulPodTimeout = 5 * time.Minute <ins> testFinalizer = "example.com/test-finalizer" </ins> ) var httpProbe = &v1.Probe{

}) <ins> ginkgo.It("should perform canary updates and phased rolling updates of template modifications for partiton1 and delete pod-0 without failing container", func(ctx context.Context) { ginkgo.By("Creating a new StatefulSet without failing container") ss := e2estatefulset.NewStatefulSet("ss2", ns, headlessSvcName, 3, nil, nil, labels) deletingPodForRollingUpdatePartitionTest(ctx, f, c, ns, ss) }) ginkgo.It("should perform canary updates and phased rolling updates of template modifications for partiton1 and delete pod-0 with failing container", func(ctx context.Context) { ginkgo.By("Creating a new StatefulSet with failing container") ss := e2estatefulset.NewStatefulSet("ss3", ns, headlessSvcName, 3, nil, nil, labels) ss.Spec.Template.Spec.Containers = append(ss.Spec.Template.Spec.Containers, v1.Container{ Name:    "sleep-exit-with-1", Image:   imageutils.GetE2EImage(imageutils.BusyBox), Command: []string{"sh", "-c"}, Args: []string{` echo "Running in pod $POD_NAME" _term(){ echo "Received SIGTERM signal" if [ "${POD_NAME}" = "ss3-0" ]; then exit 1 else exit 0 fi } trap _term SIGTERM while true; do echo "Running in infinite loop in $POD_NAME" sleep 1 done `, }, Env: []v1.EnvVar{ { Name: "POD_NAME", ValueFrom: &v1.EnvVarSource{ FieldRef: &v1.ObjectFieldSelector{ APIVersion: "v1", FieldPath:  "metadata.name", }, }, }, }, }) deletingPodForRollingUpdatePartitionTest(ctx, f, c, ns, ss) }) </ins> // Do not mark this as Conformance. // The legacy OnDelete strategy only exists for backward compatibility with pre-v1 APIs. ginkgo.It("should implement legacy replacement when the update strategy is OnDelete", func(ctx context.Context) {

} } <ins> // This function is used canary updates and phased rolling updates of template modifications for partiton1 and delete pod-0 func deletingPodForRollingUpdatePartitionTest(ctx context.Context, f *framework.Framework, c clientset.Interface, ns string, ss *appsv1.StatefulSet) { setHTTPProbe(ss) ss.Spec.UpdateStrategy = appsv1.StatefulSetUpdateStrategy{ Type: appsv1.RollingUpdateStatefulSetStrategyType, RollingUpdate: func() *appsv1.RollingUpdateStatefulSetStrategy { return &appsv1.RollingUpdateStatefulSetStrategy{ Partition: pointer.Int32(1), } }(), } ss, err := c.AppsV1().StatefulSets(ns).Create(ctx, ss, metav1.CreateOptions{}) framework.ExpectNoError(err) e2estatefulset.WaitForRunningAndReady(ctx, c, *ss.Spec.Replicas, ss) ss = waitForStatus(ctx, c, ss) currentRevision, updateRevision := ss.Status.CurrentRevision, ss.Status.UpdateRevision gomega.Expect(currentRevision).To(gomega.Equal(updateRevision), fmt.Sprintf("StatefulSet %s/%s created with update revision %s not equal to current revision %s", ss.Namespace, ss.Name, updateRevision, currentRevision)) pods := e2estatefulset.GetPodList(ctx, c, ss) for i := range pods.Items { gomega.Expect(pods.Items[i].Labels[appsv1.StatefulSetRevisionLabel]).To(gomega.Equal(currentRevision), fmt.Sprintf("Pod %s/%s revision %s is not equal to currentRevision %s", pods.Items[i].Namespace, pods.Items[i].Name, pods.Items[i].Labels[appsv1.StatefulSetRevisionLabel], currentRevision)) } ginkgo.By("Adding finalizer for pod-0") pod0name := getStatefulSetPodNameAtIndex(0, ss) pod0, err := c.CoreV1().Pods(ns).Get(ctx, pod0name, metav1.GetOptions{}) framework.ExpectNoError(err) pod0.Finalizers = append(pod0.Finalizers, testFinalizer) pod0, err = c.CoreV1().Pods(ss.Namespace).Update(ctx, pod0, metav1.UpdateOptions{}) framework.ExpectNoError(err) pods.Items[0] = *pod0 defer e2epod.NewPodClient(f).RemoveFinalizer(ctx, pod0.Name, testFinalizer) ginkgo.By("Updating image on StatefulSet") newImage := NewWebserverImage oldImage := ss.Spec.Template.Spec.Containers[0].Image ginkgo.By(fmt.Sprintf("Updating stateful set template: update image from %s to %s", oldImage, newImage)) gomega.Expect(oldImage).ToNot(gomega.Equal(newImage), "Incorrect test setup: should update to a different image") ss, err = updateStatefulSetWithRetries(ctx, c, ns, ss.Name, func(update *appsv1.StatefulSet) { update.Spec.Template.Spec.Containers[0].Image = newImage }) framework.ExpectNoError(err) ginkgo.By("Creating a new revision") ss = waitForStatus(ctx, c, ss) currentRevision, updateRevision = ss.Status.CurrentRevision, ss.Status.UpdateRevision gomega.Expect(currentRevision).ToNot(gomega.Equal(updateRevision), "Current revision should not equal update revision during rolling update") ginkgo.By("Await for all replicas running, all are updated but pod-0") e2estatefulset.WaitForState(ctx, c, ss, func(set2 *appsv1.StatefulSet, pods2 *v1.PodList) (bool, error) { ss = set2 pods = pods2 if ss.Status.UpdatedReplicas == *ss.Spec.Replicas-1 && ss.Status.Replicas == *ss.Spec.Replicas && ss.Status.ReadyReplicas == *ss.Spec.Replicas { // rolling updated is not completed, because replica 0 isn't ready return true, nil } return false, nil }) ginkgo.By("Verify pod images before pod-0 deletion and recreation") for i := range pods.Items { if i < int(*ss.Spec.UpdateStrategy.RollingUpdate.Partition) { gomega.Expect(pods.Items[i].Spec.Containers[0].Image).To(gomega.Equal(oldImage), fmt.Sprintf("Pod %s/%s has image %s not equal to oldimage image %s", pods.Items[i].Namespace, pods.Items[i].Name, pods.Items[i].Spec.Containers[0].Image, oldImage)) gomega.Expect(pods.Items[i].Labels[appsv1.StatefulSetRevisionLabel]).To(gomega.Equal(currentRevision), fmt.Sprintf("Pod %s/%s has revision %s not equal to current revision %s", pods.Items[i].Namespace, pods.Items[i].Name, pods.Items[i].Labels[appsv1.StatefulSetRevisionLabel], currentRevision)) } else { gomega.Expect(pods.Items[i].Spec.Containers[0].Image).To(gomega.Equal(newImage), fmt.Sprintf("Pod %s/%s has image %s not equal to new image  %s", pods.Items[i].Namespace, pods.Items[i].Name, pods.Items[i].Spec.Containers[0].Image, newImage)) gomega.Expect(pods.Items[i].Labels[appsv1.StatefulSetRevisionLabel]).To(gomega.Equal(updateRevision), fmt.Sprintf("Pod %s/%s has revision %s not equal to new revision %s", pods.Items[i].Namespace, pods.Items[i].Name, pods.Items[i].Labels[appsv1.StatefulSetRevisionLabel], updateRevision)) } } ginkgo.By("Deleting the pod-0 so that kubelet terminates it and StatefulSet controller recreates it") deleteStatefulPodAtIndex(ctx, c, 0, ss) ginkgo.By("Await for two replicas to be updated, while the pod-0 is not running") e2estatefulset.WaitForState(ctx, c, ss, func(set2 *appsv1.StatefulSet, pods2 *v1.PodList) (bool, error) { ss = set2 pods = pods2 return ss.Status.ReadyReplicas == *ss.Spec.Replicas-1, nil }) ginkgo.By(fmt.Sprintf("Removing finalizer from pod-0 (%v/%v) to allow recreation", pod0.Namespace, pod0.Name)) e2epod.NewPodClient(f).RemoveFinalizer(ctx, pod0.Name, testFinalizer) ginkgo.By("Await for recreation of pod-0, so that all replicas are running") e2estatefulset.WaitForState(ctx, c, ss, func(set2 *appsv1.StatefulSet, pods2 *v1.PodList) (bool, error) { ss = set2 pods = pods2 return ss.Status.ReadyReplicas == *ss.Spec.Replicas, nil }) ginkgo.By("Verify pod images after pod-0 deletion and recreation") pods = e2estatefulset.GetPodList(ctx, c, ss) for i := range pods.Items { if i < int(*ss.Spec.UpdateStrategy.RollingUpdate.Partition) { gomega.Expect(pods.Items[i].Spec.Containers[0].Image).To(gomega.Equal(oldImage), fmt.Sprintf("Pod %s/%s has image %s not equal to current image %s", pods.Items[i].Namespace, pods.Items[i].Name, pods.Items[i].Spec.Containers[0].Image, oldImage)) gomega.Expect(pods.Items[i].Labels[appsv1.StatefulSetRevisionLabel]).To(gomega.Equal(currentRevision), fmt.Sprintf("Pod %s/%s has revision %s not equal to current revision %s", pods.Items[i].Namespace, pods.Items[i].Name, pods.Items[i].Labels[appsv1.StatefulSetRevisionLabel], currentRevision)) } else { gomega.Expect(pods.Items[i].Spec.Containers[0].Image).To(gomega.Equal(newImage), fmt.Sprintf("Pod %s/%s has image %s not equal to new image  %s", pods.Items[i].Namespace, pods.Items[i].Name, pods.Items[i].Spec.Containers[0].Image, newImage)) gomega.Expect(pods.Items[i].Labels[appsv1.StatefulSetRevisionLabel]).To(gomega.Equal(updateRevision), fmt.Sprintf("Pod %s/%s has revision %s not equal to new revision %s", pods.Items[i].Namespace, pods.Items[i].Name, pods.Items[i].Labels[appsv1.StatefulSetRevisionLabel], updateRevision)) } } } </ins> // confirmStatefulPodCount asserts that the current number of Pods in ss is count, waiting up to timeout for ss to // scale to count. func confirmStatefulPodCount(ctx context.Context, c clientset.Interface, count int, ss *appsv1.StatefulSet, timeout time.Duration, hard bool) {

"k8s.io/kubernetes/pkg/controlplane" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/test/integration/framework" <ins> "k8s.io/utils/ptr" </ins> ) const (

} } <ins> func TestDeletingPodForRollingUpdatePartition(t *testing.T) { _, ctx := ktesting.NewTestContext(t) closeFn, rm, informers, c := scSetup(ctx, t) defer closeFn() ns := framework.CreateNamespaceOrDie(c, "test-deleting-pod-for-rolling-update-partition", t) defer framework.DeleteNamespaceOrDie(c, ns, t) cancel := runControllerAndInformers(rm, informers) defer cancel() labelMap := labelMap() sts := newSTS("sts", ns.Name, 2) sts.Spec.UpdateStrategy = appsv1.StatefulSetUpdateStrategy{ Type: appsv1.RollingUpdateStatefulSetStrategyType, RollingUpdate: func() *appsv1.RollingUpdateStatefulSetStrategy { return &appsv1.RollingUpdateStatefulSetStrategy{ Partition: ptr.To[int32](1), } }(), } stss, _ := createSTSsPods(t, c, []*appsv1.StatefulSet{sts}, []*v1.Pod{}) sts = stss[0] waitSTSStable(t, c, sts) // Verify STS creates 2 pods podClient := c.CoreV1().Pods(ns.Name) pods := getPods(t, podClient, labelMap) if len(pods.Items) != 2 { t.Fatalf("len(pods) = %d, want 2", len(pods.Items)) } // Setting all pods in Running, Ready, and Available setPodsReadyCondition(t, c, &v1.PodList{Items: pods.Items}, v1.ConditionTrue, time.Now()) // 1. Roll out a new image. oldImage := sts.Spec.Template.Spec.Containers[0].Image newImage := "new-image" if oldImage == newImage { t.Fatalf("bad test setup, statefulSet %s roll out with the same image", sts.Name) } // Set finalizers for the pod-0 to trigger pod recreation failure while the status UpdateRevision is bumped pod0 := &pods.Items[0] updatePod(t, podClient, pod0.Name, func(pod *v1.Pod) { pod.Finalizers = []string{"fake.example.com/blockDeletion"} }) stsClient := c.AppsV1().StatefulSets(ns.Name) _ = updateSTS(t, stsClient, sts.Name, func(sts *appsv1.StatefulSet) { sts.Spec.Template.Spec.Containers[0].Image = newImage }) // Await for the pod-1 to be recreated, while pod-0 remains running if err := wait.PollUntilContextTimeout(ctx, pollInterval, pollTimeout, false, func(ctx context.Context) (bool, error) { ss, err := stsClient.Get(ctx, sts.Name, metav1.GetOptions{}) if err != nil { return false, err } pods := getPods(t, podClient, labelMap) recreatedPods := v1.PodList{} for _, pod := range pods.Items { if pod.Status.Phase == v1.PodPending { recreatedPods.Items = append(recreatedPods.Items, pod) } } setPodsReadyCondition(t, c, &v1.PodList{Items: recreatedPods.Items}, v1.ConditionTrue, time.Now()) return ss.Status.UpdatedReplicas == *ss.Spec.Replicas-*sts.Spec.UpdateStrategy.RollingUpdate.Partition && ss.Status.Replicas == *ss.Spec.Replicas && ss.Status.ReadyReplicas == *ss.Spec.Replicas, nil }); err != nil { t.Fatalf("failed to await for pod-1 to be recreated by sts %s: %v", sts.Name, err) } // Mark pod-0 as terminal and not ready updatePodStatus(t, podClient, pod0.Name, func(pod *v1.Pod) { pod.Status.Phase = v1.PodFailed }) // Make sure pod-0 gets deletion timestamp so that it is recreated if err := c.CoreV1().Pods(ns.Name).Delete(context.TODO(), pod0.Name, metav1.DeleteOptions{}); err != nil { t.Fatalf("error deleting pod %s: %v", pod0.Name, err) } // Await for pod-0 to be not ready if err := wait.PollUntilContextTimeout(ctx, pollInterval, pollTimeout, false, func(ctx context.Context) (bool, error) { ss, err := stsClient.Get(ctx, sts.Name, metav1.GetOptions{}) if err != nil { return false, err } return ss.Status.ReadyReplicas == *ss.Spec.Replicas-1, nil }); err != nil { t.Fatalf("failed to await for pod-0 to be not counted as ready in status of sts %s: %v", sts.Name, err) } // Remove the finalizer to allow recreation updatePod(t, podClient, pod0.Name, func(pod *v1.Pod) { pod.Finalizers = []string{} }) // Await for pod-0 to be recreated and make it running if err := wait.PollUntilContextTimeout(ctx, pollInterval, pollTimeout, false, func(ctx context.Context) (bool, error) { pods := getPods(t, podClient, labelMap) recreatedPods := v1.PodList{} for _, pod := range pods.Items { if pod.Status.Phase == v1.PodPending { recreatedPods.Items = append(recreatedPods.Items, pod) } } setPodsReadyCondition(t, c, &v1.PodList{Items: recreatedPods.Items}, v1.ConditionTrue, time.Now().Add(-120*time.Minute)) return len(recreatedPods.Items) > 0, nil }); err != nil { t.Fatalf("failed to await for pod-0 to be recreated by sts %s: %v", sts.Name, err) } // Await for all stateful set status to record all replicas as ready if err := wait.PollUntilContextTimeout(ctx, pollInterval, pollTimeout, false, func(ctx context.Context) (bool, error) { ss, err := stsClient.Get(ctx, sts.Name, metav1.GetOptions{}) if err != nil { return false, err } return ss.Status.ReadyReplicas == *ss.Spec.Replicas, nil }); err != nil { t.Fatalf("failed to verify .Spec.Template.Spec.Containers[0].Image is updated for sts %s: %v", sts.Name, err) } // Verify 3 pods exist pods = getPods(t, podClient, labelMap) if len(pods.Items) != int(*sts.Spec.Replicas) { t.Fatalf("Unexpected number of pods") } // Verify pod images for i := range pods.Items { if i < int(*sts.Spec.UpdateStrategy.RollingUpdate.Partition) { if pods.Items[i].Spec.Containers[0].Image != oldImage { t.Fatalf("Pod %s has image %s not equal to old image %s", pods.Items[i].Name, pods.Items[i].Spec.Containers[0].Image, oldImage) } } else { if pods.Items[i].Spec.Containers[0].Image != newImage { t.Fatalf("Pod %s has image %s not equal to new image %s", pods.Items[i].Name, pods.Items[i].Spec.Containers[0].Image, newImage) } } } } </ins> func TestStatefulSetStartOrdinal(t *testing.T) { tests := []struct { ordinals         *appsv1.StatefulSetOrdinals