Code change:
pod.Status.ContainerStatuses = test.oldStatuses podStatus.ContainerStatuses = test.statuses apiStatus := kubelet.generateAPIPodStatus(pod, podStatus) <del> verifyStatus(t, i, apiStatus, test.expectedState, test.expectedLastTerminationState) </del> <ins> assert.NoError(t, verifyContainerStatuses(apiStatus.ContainerStatuses, test.expectedState, test.expectedLastTerminationState), "case %d", i) } // Everything should be the same for init containers for i, test := range tests { kubelet.reasonCache = NewReasonCache() for n, e := range test.reasons { kubelet.reasonCache.add(pod.UID, n, e, "") } pod.Spec.InitContainers = test.containers pod.Status.InitContainerStatuses = test.oldStatuses podStatus.ContainerStatuses = test.statuses apiStatus := kubelet.generateAPIPodStatus(pod, podStatus) expectedState := test.expectedState if test.expectedInitState != nil { expectedState = test.expectedInitState } assert.NoError(t, verifyContainerStatuses(apiStatus.InitContainerStatuses, expectedState, test.expectedLastTerminationState), "case %d", i) } } // Test generateAPIPodStatus with different restart policies. func TestGenerateAPIPodStatusWithDifferentRestartPolicies(t *testing.T) { testErrorReason := fmt.Errorf("test-error") emptyContainerID := (&kubecontainer.ContainerID{}).String() testKubelet := newTestKubelet(t) kubelet := testKubelet.kubelet pod := podWithUidNameNs("12345678", "foo", "new") containers := []api.Container{{Name: "succeed"}, {Name: "failed"}} podStatus := &kubecontainer.PodStatus{ ID:        pod.UID, Name:      pod.Name, Namespace: pod.Namespace, ContainerStatuses: []*kubecontainer.ContainerStatus{ { Name:     "succeed", State:    kubecontainer.ContainerStateExited, ExitCode: 0, }, { Name:     "failed", State:    kubecontainer.ContainerStateExited, ExitCode: 1, }, { Name:     "succeed", State:    kubecontainer.ContainerStateExited, ExitCode: 2, }, { Name:     "failed", State:    kubecontainer.ContainerStateExited, ExitCode: 3, }, }, } kubelet.reasonCache.add(pod.UID, "succeed", testErrorReason, "") kubelet.reasonCache.add(pod.UID, "failed", testErrorReason, "") for c, test := range []struct { restartPolicy                api.RestartPolicy expectedState                map[string]api.ContainerState expectedLastTerminationState map[string]api.ContainerState // Only set expectedInitState when it is different from expectedState expectedInitState map[string]api.ContainerState // Only set expectedInitLastTerminationState when it is different from expectedLastTerminationState expectedInitLastTerminationState map[string]api.ContainerState }{ { restartPolicy: api.RestartPolicyNever, expectedState: map[string]api.ContainerState{ "succeed": {Terminated: &api.ContainerStateTerminated{ ExitCode:    0, ContainerID: emptyContainerID, }}, "failed": {Terminated: &api.ContainerStateTerminated{ ExitCode:    1, ContainerID: emptyContainerID, }}, }, expectedLastTerminationState: map[string]api.ContainerState{ "succeed": {Terminated: &api.ContainerStateTerminated{ ExitCode:    2, ContainerID: emptyContainerID, }}, "failed": {Terminated: &api.ContainerStateTerminated{ ExitCode:    3, ContainerID: emptyContainerID, }}, }, }, { restartPolicy: api.RestartPolicyOnFailure, expectedState: map[string]api.ContainerState{ "succeed": {Terminated: &api.ContainerStateTerminated{ ExitCode:    0, ContainerID: emptyContainerID, }}, "failed": {Waiting: &api.ContainerStateWaiting{Reason: testErrorReason.Error()}}, }, expectedLastTerminationState: map[string]api.ContainerState{ "succeed": {Terminated: &api.ContainerStateTerminated{ ExitCode:    2, ContainerID: emptyContainerID, }}, "failed": {Terminated: &api.ContainerStateTerminated{ ExitCode:    1, ContainerID: emptyContainerID, }}, }, }, { restartPolicy: api.RestartPolicyAlways, expectedState: map[string]api.ContainerState{ "succeed": {Waiting: &api.ContainerStateWaiting{Reason: testErrorReason.Error()}}, "failed":  {Waiting: &api.ContainerStateWaiting{Reason: testErrorReason.Error()}}, }, expectedLastTerminationState: map[string]api.ContainerState{ "succeed": {Terminated: &api.ContainerStateTerminated{ ExitCode:    0, ContainerID: emptyContainerID, }}, "failed": {Terminated: &api.ContainerStateTerminated{ ExitCode:    1, ContainerID: emptyContainerID, }}, }, // If the init container is terminated with exit code 0, it won't be restarted even when the // restart policy is RestartAlways. expectedInitState: map[string]api.ContainerState{ "succeed": {Terminated: &api.ContainerStateTerminated{ ExitCode:    0, ContainerID: emptyContainerID, }}, "failed": {Waiting: &api.ContainerStateWaiting{Reason: testErrorReason.Error()}}, }, expectedInitLastTerminationState: map[string]api.ContainerState{ "succeed": {Terminated: &api.ContainerStateTerminated{ ExitCode:    2, ContainerID: emptyContainerID, }}, "failed": {Terminated: &api.ContainerStateTerminated{ ExitCode:    1, ContainerID: emptyContainerID, }}, }, }, } { pod.Spec.RestartPolicy = test.restartPolicy // Test normal containers pod.Spec.Containers = containers apiStatus := kubelet.generateAPIPodStatus(pod, podStatus) expectedState, expectedLastTerminationState := test.expectedState, test.expectedLastTerminationState assert.NoError(t, verifyContainerStatuses(apiStatus.ContainerStatuses, expectedState, expectedLastTerminationState), "case %d", c) pod.Spec.Containers = nil // Test init containers pod.Spec.InitContainers = containers apiStatus = kubelet.generateAPIPodStatus(pod, podStatus) if test.expectedInitState != nil { expectedState = test.expectedInitState } if test.expectedInitLastTerminationState != nil { expectedLastTerminationState = test.expectedInitLastTerminationState } assert.NoError(t, verifyContainerStatuses(apiStatus.InitContainerStatuses, expectedState, expectedLastTerminationState), "case %d", c) pod.Spec.InitContainers = nil </ins> } }

Related discussion:
text: Kubelet issues a docker inspect call after creating a container, but for very short lived containers there is a window in which they could exit before being inspected in the docker manager. If the container PID is 0, we should check the ExitCode and other relevant fields and handle them appropriately. [1] This is the suspected cause of several flakes: /cc
looking at the logs 2 weird things: race described second container ran, which suggests inconsistent reporting of initialized status of init1 and choosing init2 as the next container ( vs ). I think 1 got it right and 2 got it wrong. From logs: I'd expect the second one to not have run in this case? Node logs:
Init containers have an implicit assumption that container statuses include results of previous SyncPod invocations. If that's not the case we may need to fix that (or at least mitigate it). On May 18, 2016, at 8:28 PM, Prashanth B wrote: looking at the logs 2 weird things: race described second container ran, which suggests inconsistent reporting of initialized status of init1 and choosing init2 as the next container ( vs ). I think 1 got it right and 2 got it wrong. From logs: May 18 15:58:33.380: INFO: pod-init--1d4b-11e6-80a4- started at 2016-05-18T15:58:04-07:00 (2+1 container statuses recorded) May 18 15:58:33.380: INFO: Init container init1 ready: false, restart count 0 May 18 15:58:33.380: INFO: Init container init2 ready: true, restart count 0 I'd expect the second one to not have run in this case? Node logs: — You are receiving this because you were mentioned. Reply to this email directly or view it on GitHub
This should be true. Is there any evidence that suggests the opposite? only shows conflicting view of whether init1 has run.
If the container exits already and it's not an infra container, kubelet should not error out. Although even if it does, I failed to see how this could affect the readiness of the init container without checking the code further.