Code change:
pod.Status.ContainerStatuses = test.oldStatuses podStatus.ContainerStatuses = test.statuses apiStatus := kubelet.generateAPIPodStatus(pod, podStatus) <del> verifyStatus(t, i, apiStatus, test.expectedState, test.expectedLastTerminationState) </del> <ins> assert.NoError(t, verifyContainerStatuses(apiStatus.ContainerStatuses, test.expectedState, test.expectedLastTerminationState), "case %d", i) } // Everything should be the same for init containers for i, test := range tests { kubelet.reasonCache = NewReasonCache() for n, e := range test.reasons { kubelet.reasonCache.add(pod.UID, n, e, "") } pod.Spec.InitContainers = test.containers pod.Status.InitContainerStatuses = test.oldStatuses podStatus.ContainerStatuses = test.statuses apiStatus := kubelet.generateAPIPodStatus(pod, podStatus) expectedState := test.expectedState if test.expectedInitState != nil { expectedState = test.expectedInitState } assert.NoError(t, verifyContainerStatuses(apiStatus.InitContainerStatuses, expectedState, test.expectedLastTerminationState), "case %d", i) } } // Test generateAPIPodStatus with different restart policies. func TestGenerateAPIPodStatusWithDifferentRestartPolicies(t *testing.T) { testErrorReason := fmt.Errorf("test-error") emptyContainerID := (&kubecontainer.ContainerID{}).String() testKubelet := newTestKubelet(t) kubelet := testKubelet.kubelet pod := podWithUidNameNs("12345678", "foo", "new") containers := []api.Container{{Name: "succeed"}, {Name: "failed"}} podStatus := &kubecontainer.PodStatus{ ID:        pod.UID, Name:      pod.Name, Namespace: pod.Namespace, ContainerStatuses: []*kubecontainer.ContainerStatus{ { Name:     "succeed", State:    kubecontainer.ContainerStateExited, ExitCode: 0, }, { Name:     "failed", State:    kubecontainer.ContainerStateExited, ExitCode: 1, }, { Name:     "succeed", State:    kubecontainer.ContainerStateExited, ExitCode: 2, }, { Name:     "failed", State:    kubecontainer.ContainerStateExited, ExitCode: 3, }, }, } kubelet.reasonCache.add(pod.UID, "succeed", testErrorReason, "") kubelet.reasonCache.add(pod.UID, "failed", testErrorReason, "") for c, test := range []struct { restartPolicy                api.RestartPolicy expectedState                map[string]api.ContainerState expectedLastTerminationState map[string]api.ContainerState // Only set expectedInitState when it is different from expectedState expectedInitState map[string]api.ContainerState // Only set expectedInitLastTerminationState when it is different from expectedLastTerminationState expectedInitLastTerminationState map[string]api.ContainerState }{ { restartPolicy: api.RestartPolicyNever, expectedState: map[string]api.ContainerState{ "succeed": {Terminated: &api.ContainerStateTerminated{ ExitCode:    0, ContainerID: emptyContainerID, }}, "failed": {Terminated: &api.ContainerStateTerminated{ ExitCode:    1, ContainerID: emptyContainerID, }}, }, expectedLastTerminationState: map[string]api.ContainerState{ "succeed": {Terminated: &api.ContainerStateTerminated{ ExitCode:    2, ContainerID: emptyContainerID, }}, "failed": {Terminated: &api.ContainerStateTerminated{ ExitCode:    3, ContainerID: emptyContainerID, }}, }, }, { restartPolicy: api.RestartPolicyOnFailure, expectedState: map[string]api.ContainerState{ "succeed": {Terminated: &api.ContainerStateTerminated{ ExitCode:    0, ContainerID: emptyContainerID, }}, "failed": {Waiting: &api.ContainerStateWaiting{Reason: testErrorReason.Error()}}, }, expectedLastTerminationState: map[string]api.ContainerState{ "succeed": {Terminated: &api.ContainerStateTerminated{ ExitCode:    2, ContainerID: emptyContainerID, }}, "failed": {Terminated: &api.ContainerStateTerminated{ ExitCode:    1, ContainerID: emptyContainerID, }}, }, }, { restartPolicy: api.RestartPolicyAlways, expectedState: map[string]api.ContainerState{ "succeed": {Waiting: &api.ContainerStateWaiting{Reason: testErrorReason.Error()}}, "failed":  {Waiting: &api.ContainerStateWaiting{Reason: testErrorReason.Error()}}, }, expectedLastTerminationState: map[string]api.ContainerState{ "succeed": {Terminated: &api.ContainerStateTerminated{ ExitCode:    0, ContainerID: emptyContainerID, }}, "failed": {Terminated: &api.ContainerStateTerminated{ ExitCode:    1, ContainerID: emptyContainerID, }}, }, // If the init container is terminated with exit code 0, it won't be restarted even when the // restart policy is RestartAlways. expectedInitState: map[string]api.ContainerState{ "succeed": {Terminated: &api.ContainerStateTerminated{ ExitCode:    0, ContainerID: emptyContainerID, }}, "failed": {Waiting: &api.ContainerStateWaiting{Reason: testErrorReason.Error()}}, }, expectedInitLastTerminationState: map[string]api.ContainerState{ "succeed": {Terminated: &api.ContainerStateTerminated{ ExitCode:    2, ContainerID: emptyContainerID, }}, "failed": {Terminated: &api.ContainerStateTerminated{ ExitCode:    1, ContainerID: emptyContainerID, }}, }, }, } { pod.Spec.RestartPolicy = test.restartPolicy // Test normal containers pod.Spec.Containers = containers apiStatus := kubelet.generateAPIPodStatus(pod, podStatus) expectedState, expectedLastTerminationState := test.expectedState, test.expectedLastTerminationState assert.NoError(t, verifyContainerStatuses(apiStatus.ContainerStatuses, expectedState, expectedLastTerminationState), "case %d", c) pod.Spec.Containers = nil // Test init containers pod.Spec.InitContainers = containers apiStatus = kubelet.generateAPIPodStatus(pod, podStatus) if test.expectedInitState != nil { expectedState = test.expectedInitState } if test.expectedInitLastTerminationState != nil { expectedLastTerminationState = test.expectedInitLastTerminationState } assert.NoError(t, verifyContainerStatuses(apiStatus.InitContainerStatuses, expectedState, expectedLastTerminationState), "case %d", c) pod.Spec.InitContainers = nil </ins> } }

Related discussion:
text: The extra check of last terminated container in your initial code makes me nervous, but we couldn't reproduce it due to that small race windows until we have a faster docker running on the node. :-)
I got "Pods should invoke init containers on a RestartAlways pod" flake (referenced from ): Logs:
All the fixes went in. Please reopen this issue if you see any more flakes.