Code change:
pod.Status.ContainerStatuses = test.oldStatuses podStatus.ContainerStatuses = test.statuses apiStatus := kubelet.generateAPIPodStatus(pod, podStatus) <del> verifyStatus(t, i, apiStatus, test.expectedState, test.expectedLastTerminationState) </del> <ins> assert.NoError(t, verifyContainerStatuses(apiStatus.ContainerStatuses, test.expectedState, test.expectedLastTerminationState), "case %d", i) } // Everything should be the same for init containers for i, test := range tests { kubelet.reasonCache = NewReasonCache() for n, e := range test.reasons { kubelet.reasonCache.add(pod.UID, n, e, "") } pod.Spec.InitContainers = test.containers pod.Status.InitContainerStatuses = test.oldStatuses podStatus.ContainerStatuses = test.statuses apiStatus := kubelet.generateAPIPodStatus(pod, podStatus) expectedState := test.expectedState if test.expectedInitState != nil { expectedState = test.expectedInitState } assert.NoError(t, verifyContainerStatuses(apiStatus.InitContainerStatuses, expectedState, test.expectedLastTerminationState), "case %d", i) } } // Test generateAPIPodStatus with different restart policies. func TestGenerateAPIPodStatusWithDifferentRestartPolicies(t *testing.T) { testErrorReason := fmt.Errorf("test-error") emptyContainerID := (&kubecontainer.ContainerID{}).String() testKubelet := newTestKubelet(t) kubelet := testKubelet.kubelet pod := podWithUidNameNs("12345678", "foo", "new") containers := []api.Container{{Name: "succeed"}, {Name: "failed"}} podStatus := &kubecontainer.PodStatus{ ID:        pod.UID, Name:      pod.Name, Namespace: pod.Namespace, ContainerStatuses: []*kubecontainer.ContainerStatus{ { Name:     "succeed", State:    kubecontainer.ContainerStateExited, ExitCode: 0, }, { Name:     "failed", State:    kubecontainer.ContainerStateExited, ExitCode: 1, }, { Name:     "succeed", State:    kubecontainer.ContainerStateExited, ExitCode: 2, }, { Name:     "failed", State:    kubecontainer.ContainerStateExited, ExitCode: 3, }, }, } kubelet.reasonCache.add(pod.UID, "succeed", testErrorReason, "") kubelet.reasonCache.add(pod.UID, "failed", testErrorReason, "") for c, test := range []struct { restartPolicy                api.RestartPolicy expectedState                map[string]api.ContainerState expectedLastTerminationState map[string]api.ContainerState // Only set expectedInitState when it is different from expectedState expectedInitState map[string]api.ContainerState // Only set expectedInitLastTerminationState when it is different from expectedLastTerminationState expectedInitLastTerminationState map[string]api.ContainerState }{ { restartPolicy: api.RestartPolicyNever, expectedState: map[string]api.ContainerState{ "succeed": {Terminated: &api.ContainerStateTerminated{ ExitCode:    0, ContainerID: emptyContainerID, }}, "failed": {Terminated: &api.ContainerStateTerminated{ ExitCode:    1, ContainerID: emptyContainerID, }}, }, expectedLastTerminationState: map[string]api.ContainerState{ "succeed": {Terminated: &api.ContainerStateTerminated{ ExitCode:    2, ContainerID: emptyContainerID, }}, "failed": {Terminated: &api.ContainerStateTerminated{ ExitCode:    3, ContainerID: emptyContainerID, }}, }, }, { restartPolicy: api.RestartPolicyOnFailure, expectedState: map[string]api.ContainerState{ "succeed": {Terminated: &api.ContainerStateTerminated{ ExitCode:    0, ContainerID: emptyContainerID, }}, "failed": {Waiting: &api.ContainerStateWaiting{Reason: testErrorReason.Error()}}, }, expectedLastTerminationState: map[string]api.ContainerState{ "succeed": {Terminated: &api.ContainerStateTerminated{ ExitCode:    2, ContainerID: emptyContainerID, }}, "failed": {Terminated: &api.ContainerStateTerminated{ ExitCode:    1, ContainerID: emptyContainerID, }}, }, }, { restartPolicy: api.RestartPolicyAlways, expectedState: map[string]api.ContainerState{ "succeed": {Waiting: &api.ContainerStateWaiting{Reason: testErrorReason.Error()}}, "failed":  {Waiting: &api.ContainerStateWaiting{Reason: testErrorReason.Error()}}, }, expectedLastTerminationState: map[string]api.ContainerState{ "succeed": {Terminated: &api.ContainerStateTerminated{ ExitCode:    0, ContainerID: emptyContainerID, }}, "failed": {Terminated: &api.ContainerStateTerminated{ ExitCode:    1, ContainerID: emptyContainerID, }}, }, // If the init container is terminated with exit code 0, it won't be restarted even when the // restart policy is RestartAlways. expectedInitState: map[string]api.ContainerState{ "succeed": {Terminated: &api.ContainerStateTerminated{ ExitCode:    0, ContainerID: emptyContainerID, }}, "failed": {Waiting: &api.ContainerStateWaiting{Reason: testErrorReason.Error()}}, }, expectedInitLastTerminationState: map[string]api.ContainerState{ "succeed": {Terminated: &api.ContainerStateTerminated{ ExitCode:    2, ContainerID: emptyContainerID, }}, "failed": {Terminated: &api.ContainerStateTerminated{ ExitCode:    1, ContainerID: emptyContainerID, }}, }, }, } { pod.Spec.RestartPolicy = test.restartPolicy // Test normal containers pod.Spec.Containers = containers apiStatus := kubelet.generateAPIPodStatus(pod, podStatus) expectedState, expectedLastTerminationState := test.expectedState, test.expectedLastTerminationState assert.NoError(t, verifyContainerStatuses(apiStatus.ContainerStatuses, expectedState, expectedLastTerminationState), "case %d", c) pod.Spec.Containers = nil // Test init containers pod.Spec.InitContainers = containers apiStatus = kubelet.generateAPIPodStatus(pod, podStatus) if test.expectedInitState != nil { expectedState = test.expectedInitState } if test.expectedInitLastTerminationState != nil { expectedLastTerminationState = test.expectedInitLastTerminationState } assert.NoError(t, verifyContainerStatuses(apiStatus.InitContainerStatuses, expectedState, expectedLastTerminationState), "case %d", c) pod.Spec.InitContainers = nil </ins> } }

Related discussion:
text: Each syncPod invocation, we inspect the list of init container in order looking for lastTerminationState. There could be a bug in that logic, or if we didn't get the full status, we might make a bad decision. When determining which container to run next, we always look for the last init container in the chain that has success, then run the next container (unless the pod is RestartNever). , Yu-Ju Hong wrote:
Saw this flake again. How do we verify that container status is not lost between syn pods?
... between calls to syncPods
The second sync clearly saw the init1 container, and decided to launch init2. See "InitContainersToKeep" in the message below.
containers to keep can include failed containers. , Yu-Ju Hong wrote:
Discussed on slack. In summary, the problem has to do with how we convert the kubecontainer.PodStatus to the api.PodStatus. If a container is terminated, the function makes the decisions: the pod restart policy is "Always", and there is a "failure reason" recorded for the given container, this means the pod worker tried to (re)start the container but failed. Set the state to "Waiting" and populate the reason. , the container state is simply "terminated". Init containers don't obey the restart policy like regular containers, and the logic will need to be amended to work with them. is working on a PR. In addition to that, kubelet should not error out if a user container exited before it could apply the oom score (which is best effort in this case). fixes that.
Ahh, thanks for digging into the issue. This is the exact the concern I had when I reviewed the initial implementation of init container, which blocks pr for a while. I even asked to help me verify this case, unfortunately we didn't reproduce the issue back then. :-(
I wish I'd seen the flakes in the runs prior to merge.
We didn't notice this until we started running the tests against docker v1.11.1, which is much faster and hit the race condition differently.
I suppose that's a good thing...
It is hard to reproduce this.