Change:
I created a pod with sidecar container with following manifest: watch the pod readiness. in , it's mentioned "If a readinessProbe is specified for this init container, its result will be used to determine the ready state of the Pod." it's true only for = Always, sidecar container readniess result will be used in Pod ready state, neither "Never" or "OnFailure". I tested with v1.29.0 I expect "If a readinessProbe is specified for this init container, its result will be used to determine the ready state of the Pod." also applicable to = "Never" or "OnFailure". Set = "Never" or "OnFailure". No response <details</details<details</details<details</details<details</details<detailscontainerd --version containerd v1.7.11 </details<details</details
/sig node
/remove-kind bug /kind feature I think this is a feature request, because you want to do something different than the documentation.
can you please take a look It well may be a bug /cc
/assign
confirmed that this issue is reproduced. I'll take a look. /triage accepted This happens in a container with Always containerRestartPolicy from the pod with Never or OnFailure restartPolicy. I think this is a bug. /remove-kind feature /kind bug
/priority important-soon
After a quick debug, it seems that the problem is the apiserver who is preventing the initContainerStatus from being changed from terminated to non-terminated state.
created a PR to address this.
Just a small comment on whether we need a cherry pick. This only affects specific pods with the new beta feature enabled. I don't think it is justifiable to cherry-pick this
do you think this bug is critical enough to cherry-pick. Just discussed on sidecar WG and it feels that impact may be more dangerous than I undertsood. will summarize
IIUC it doesn't seem that critical to Istio, epsecially if its high risk to backport
the fix is relatively easy, but I am operating from the assumption that everything is high risk to backport =)
I confirmed that the bug is not only limited to the sidecar container with . Kubernetes cannot update the terminated sidecar container' statuses from Job without the bugfix (regardless of the ). The sidecar container can continually restart but its container status will be stuck in the first terminated state. It seems like a quite critical bug for me.
Updated the PR description. 1.29: 1.28:

Discussion:
return allErrs } <ins> // ValidateInitContainerStateTransition test to if any illegal init container state transitions are being attempted func ValidateInitContainerStateTransition(newStatuses, oldStatuses []core.ContainerStatus, fldpath *field.Path, podSpec *core.PodSpec) field.ErrorList { allErrs := field.ErrorList{} // If we should always restart, containers are allowed to leave the terminated state if podSpec.RestartPolicy == core.RestartPolicyAlways { return allErrs } for i, oldStatus := range oldStatuses { // Skip any container that is not terminated if oldStatus.State.Terminated == nil { continue } // Skip any container that failed but is allowed to restart if oldStatus.State.Terminated.ExitCode != 0 && podSpec.RestartPolicy == core.RestartPolicyOnFailure { continue } // Skip any restartable init container that is allowed to restart isRestartableInitContainer := false for _, c := range podSpec.InitContainers { if oldStatus.Name == c.Name { if c.RestartPolicy != nil && *c.RestartPolicy == core.ContainerRestartPolicyAlways { isRestartableInitContainer = true } break } } if isRestartableInitContainer { continue } for _, newStatus := range newStatuses { if oldStatus.Name == newStatus.Name && newStatus.State.Terminated == nil { allErrs = append(allErrs, field.Forbidden(fldpath.Index(i).Child("state"), "may not be transitioned to non-terminated state")) } } } return allErrs } </ins> // ValidatePodStatusUpdate checks for changes to status that shouldn't occur in normal operation. func ValidatePodStatusUpdate(newPod, oldPod *core.Pod, opts PodValidationOptions) field.ErrorList { fldPath := field.NewPath("metadata")

// If pod should not restart, make sure the status update does not transition // any terminated containers to a non-terminated state. allErrs = append(allErrs, ValidateContainerStateTransition(newPod.Status.ContainerStatuses, oldPod.Status.ContainerStatuses, fldPath.Child("containerStatuses"), oldPod.Spec.RestartPolicy)...) <del> allErrs = append(allErrs, ValidateContainerStateTransition(newPod.Status.InitContainerStatuses, oldPod.Status.InitContainerStatuses, fldPath.Child("initContainerStatuses"), oldPod.Spec.RestartPolicy)...) </del> <ins> allErrs = append(allErrs, ValidateInitContainerStateTransition(newPod.Status.InitContainerStatuses, oldPod.Status.InitContainerStatuses, fldPath.Child("initContainerStatuses"), &oldPod.Spec)...) </ins> // The kubelet will never restart ephemeral containers, so treat them like they have an implicit RestartPolicyNever. allErrs = append(allErrs, ValidateContainerStateTransition(newPod.Status.EphemeralContainerStatuses, oldPod.Status.EphemeralContainerStatuses, fldPath.Child("ephemeralContainerStatuses"), core.RestartPolicyNever)...) allErrs = append(allErrs, validatePodResourceClaimStatuses(newPod.Status.ResourceClaimStatuses, newPod.Spec.ResourceClaims, fldPath.Child("resourceClaimStatuses"))...)

}, "", "ResourceClaimStatuses okay", <ins> }, { core.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "foo", }, Spec: core.PodSpec{ InitContainers: []core.Container{ { Name: "init", }, }, Containers: []core.Container{ { Name: "nginx", }, }, }, Status: core.PodStatus{ InitContainerStatuses: []core.ContainerStatus{{ ContainerID: "docker://numbers", Image:       "alpine", ImageID:     "docker-pullable://nginx@sha256:d0gf00d", Name:        "init", Ready:       true, State: core.ContainerState{ Running: &core.ContainerStateRunning{ StartedAt: metav1.NewTime(time.Now()), }, }, }}, ContainerStatuses: []core.ContainerStatus{{ ContainerID: "docker://numbers", Image:       "nginx:alpine", ImageID:     "docker-pullable://nginx@sha256:d0gf00d", Name:        "nginx", Ready:       true, Started:     proto.Bool(true), State: core.ContainerState{ Running: &core.ContainerStateRunning{ StartedAt: metav1.NewTime(time.Now()), }, }, }}, }, }, core.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "foo", }, Spec: core.PodSpec{ InitContainers: []core.Container{ { Name: "init", }, }, Containers: []core.Container{ { Name: "nginx", }, }, RestartPolicy: core.RestartPolicyNever, }, Status: core.PodStatus{ InitContainerStatuses: []core.ContainerStatus{{ ContainerID: "docker://numbers", Image:       "alpine", ImageID:     "docker-pullable://nginx@sha256:d0gf00d", Name:        "init", Ready:       false, State: core.ContainerState{ Terminated: &core.ContainerStateTerminated{ ContainerID: "docker://numbers", Reason:      "Completed", }, }, }}, ContainerStatuses: []core.ContainerStatus{{ ContainerID: "docker://numbers", Image:       "nginx:alpine", ImageID:     "docker-pullable://nginx@sha256:d0gf00d", Name:        "nginx", Ready:       true, Started:     proto.Bool(true), State: core.ContainerState{ Running: &core.ContainerStateRunning{ StartedAt: metav1.NewTime(time.Now()), }, }, }}, }, }, `status.initContainerStatuses[0].state: Forbidden: may not be transitioned to non-terminated state`, "init container cannot restart if RestartPolicyNever", }, { core.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "foo", }, Spec: core.PodSpec{ InitContainers: []core.Container{ { Name:          "restartable-init", RestartPolicy: &containerRestartPolicyAlways, }, }, Containers: []core.Container{ { Name: "nginx", }, }, RestartPolicy: core.RestartPolicyNever, }, Status: core.PodStatus{ InitContainerStatuses: []core.ContainerStatus{{ ContainerID: "docker://numbers", Image:       "alpine", ImageID:     "docker-pullable://nginx@sha256:d0gf00d", Name:        "restartable-init", Ready:       true, State: core.ContainerState{ Running: &core.ContainerStateRunning{ StartedAt: metav1.NewTime(time.Now()), }, }, }}, ContainerStatuses: []core.ContainerStatus{{ ContainerID: "docker://numbers", Image:       "nginx:alpine", ImageID:     "docker-pullable://nginx@sha256:d0gf00d", Name:        "nginx", Ready:       true, Started:     proto.Bool(true), State: core.ContainerState{ Running: &core.ContainerStateRunning{ StartedAt: metav1.NewTime(time.Now()), }, }, }}, }, }, core.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "foo", }, Spec: core.PodSpec{ InitContainers: []core.Container{ { Name:          "restartable-init", RestartPolicy: &containerRestartPolicyAlways, }, }, Containers: []core.Container{ { Name: "nginx", }, }, RestartPolicy: core.RestartPolicyNever, }, Status: core.PodStatus{ InitContainerStatuses: []core.ContainerStatus{{ ContainerID: "docker://numbers", Image:       "alpine", ImageID:     "docker-pullable://nginx@sha256:d0gf00d", Name:        "restartable-init", Ready:       false, State: core.ContainerState{ Terminated: &core.ContainerStateTerminated{ ContainerID: "docker://numbers", Reason:      "Completed", }, }, }}, ContainerStatuses: []core.ContainerStatus{{ ContainerID: "docker://numbers", Image:       "nginx:alpine", ImageID:     "docker-pullable://nginx@sha256:d0gf00d", Name:        "nginx", Ready:       true, Started:     proto.Bool(true), State: core.ContainerState{ Running: &core.ContainerStateRunning{ StartedAt: metav1.NewTime(time.Now()), }, }, }}, }, }, "", "restartable init container can restart if RestartPolicyNever", }, { core.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "foo", }, Spec: core.PodSpec{ InitContainers: []core.Container{ { Name:          "restartable-init", RestartPolicy: &containerRestartPolicyAlways, }, }, Containers: []core.Container{ { Name: "nginx", }, }, RestartPolicy: core.RestartPolicyOnFailure, }, Status: core.PodStatus{ InitContainerStatuses: []core.ContainerStatus{{ ContainerID: "docker://numbers", Image:       "alpine", ImageID:     "docker-pullable://nginx@sha256:d0gf00d", Name:        "restartable-init", Ready:       true, State: core.ContainerState{ Running: &core.ContainerStateRunning{ StartedAt: metav1.NewTime(time.Now()), }, }, }}, ContainerStatuses: []core.ContainerStatus{{ ContainerID: "docker://numbers", Image:       "nginx:alpine", ImageID:     "docker-pullable://nginx@sha256:d0gf00d", Name:        "nginx", Ready:       true, Started:     proto.Bool(true), State: core.ContainerState{ Running: &core.ContainerStateRunning{ StartedAt: metav1.NewTime(time.Now()), }, }, }}, }, }, core.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "foo", }, Spec: core.PodSpec{ InitContainers: []core.Container{ { Name:          "restartable-init", RestartPolicy: &containerRestartPolicyAlways, }, }, Containers: []core.Container{ { Name: "nginx", }, }, RestartPolicy: core.RestartPolicyOnFailure, }, Status: core.PodStatus{ InitContainerStatuses: []core.ContainerStatus{{ ContainerID: "docker://numbers", Image:       "alpine", ImageID:     "docker-pullable://nginx@sha256:d0gf00d", Name:        "restartable-init", Ready:       false, State: core.ContainerState{ Terminated: &core.ContainerStateTerminated{ ContainerID: "docker://numbers", Reason:      "Completed", }, }, }}, ContainerStatuses: []core.ContainerStatus{{ ContainerID: "docker://numbers", Image:       "nginx:alpine", ImageID:     "docker-pullable://nginx@sha256:d0gf00d", Name:        "nginx", Ready:       true, Started:     proto.Bool(true), State: core.ContainerState{ Running: &core.ContainerStateRunning{ StartedAt: metav1.NewTime(time.Now()), }, }, }}, }, }, "", "restartable init container can restart if RestartPolicyOnFailure", }, { core.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "foo", }, Spec: core.PodSpec{ InitContainers: []core.Container{ { Name:          "restartable-init", RestartPolicy: &containerRestartPolicyAlways, }, }, Containers: []core.Container{ { Name: "nginx", }, }, RestartPolicy: core.RestartPolicyAlways, }, Status: core.PodStatus{ InitContainerStatuses: []core.ContainerStatus{{ ContainerID: "docker://numbers", Image:       "alpine", ImageID:     "docker-pullable://nginx@sha256:d0gf00d", Name:        "restartable-init", Ready:       true, State: core.ContainerState{ Running: &core.ContainerStateRunning{ StartedAt: metav1.NewTime(time.Now()), }, }, }}, ContainerStatuses: []core.ContainerStatus{{ ContainerID: "docker://numbers", Image:       "nginx:alpine", ImageID:     "docker-pullable://nginx@sha256:d0gf00d", Name:        "nginx", Ready:       true, Started:     proto.Bool(true), State: core.ContainerState{ Running: &core.ContainerStateRunning{ StartedAt: metav1.NewTime(time.Now()), }, }, }}, }, }, core.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "foo", }, Spec: core.PodSpec{ InitContainers: []core.Container{ { Name:          "restartable-init", RestartPolicy: &containerRestartPolicyAlways, }, }, Containers: []core.Container{ { Name: "nginx", }, }, RestartPolicy: core.RestartPolicyAlways, }, Status: core.PodStatus{ InitContainerStatuses: []core.ContainerStatus{{ ContainerID: "docker://numbers", Image:       "alpine", ImageID:     "docker-pullable://nginx@sha256:d0gf00d", Name:        "restartable-init", Ready:       false, State: core.ContainerState{ Terminated: &core.ContainerStateTerminated{ ContainerID: "docker://numbers", Reason:      "Completed", }, }, }}, ContainerStatuses: []core.ContainerStatus{{ ContainerID: "docker://numbers", Image:       "nginx:alpine", ImageID:     "docker-pullable://nginx@sha256:d0gf00d", Name:        "nginx", Ready:       true, Started:     proto.Bool(true), State: core.ContainerState{ Running: &core.ContainerStateRunning{ StartedAt: metav1.NewTime(time.Now()), }, }, }}, }, }, "", "restartable init container can restart if RestartPolicyAlways", </ins> }, }