Change:
is this a Node bug? cc as next oncall to follow up on this.
This is a node bug due to init containers
I may need help debugging - I see:
So I think the problem is in the error case - we're not properly recording that the start failed, and the next time we come in we somehow think the first container succeeded. I'll add some debugging to see if we can't recreate it.
Let's discuss on instead.

Discussion:
// Handle the containers failed to be started, which should be in Waiting state. for _, container := range containers { <ins> if isInitContainer { // If the init container is terminated with exit code 0, it won't be restarted. // TODO(random-liu): Handle this in a cleaner way. s := podStatus.FindContainerStatusByName(container.Name) if s != nil && s.State == kubecontainer.ContainerStateExited && s.ExitCode == 0 { continue } } </ins> // If a container should be restarted in next syncpod, it is *Waiting*. if !kubecontainer.ShouldContainerBeRestarted(&container, pod, podStatus) { continue

cadvisorapi "github.com/google/cadvisor/info/v1" cadvisorapiv2 "github.com/google/cadvisor/info/v2" <ins> "github.com/stretchr/testify/assert" </ins> "k8s.io/kubernetes/pkg/api" apierrors "k8s.io/kubernetes/pkg/api/errors" "k8s.io/kubernetes/pkg/api/resource"

} } <ins> func verifyContainerStatuses(statuses []api.ContainerStatus, state, lastTerminationState map[string]api.ContainerState) error { for _, s := range statuses { if !reflect.DeepEqual(s.State, state[s.Name]) { return fmt.Errorf("unexpected state: %s", diff.ObjectDiff(state[s.Name], s.State)) } if !reflect.DeepEqual(s.LastTerminationState, lastTerminationState[s.Name]) { return fmt.Errorf("unexpected last termination state %s", diff.ObjectDiff( lastTerminationState[s.Name], s.LastTerminationState)) } } return nil } // Test generateAPIPodStatus with different reason cache and old api pod status. </ins> func TestGenerateAPIPodStatusWithReasonCache(t *testing.T) { // The following waiting reason and message  are generated in convertStatusToAPIStatus() startWaitingReason := "ContainerCreating" <ins> initWaitingReason := "PodInitializing" </ins> testTimestamp := time.Unix(123456789, 987654321) testErrorReason := fmt.Errorf("test-error") emptyContainerID := (&kubecontainer.ContainerID{}).String() <del> verifyStatus := func(t *testing.T, c int, podStatus api.PodStatus, state, lastTerminationState map[string]api.ContainerState) { statuses := podStatus.ContainerStatuses for _, s := range statuses { if !reflect.DeepEqual(s.State, state[s.Name]) { t.Errorf("case #%d, unexpected state: %s", c, diff.ObjectDiff(state[s.Name], s.State)) } if !reflect.DeepEqual(s.LastTerminationState, lastTerminationState[s.Name]) { t.Errorf("case #%d, unexpected last termination state %s", c, diff.ObjectDiff( lastTerminationState[s.Name], s.LastTerminationState)) } } } </del> testKubelet := newTestKubelet(t) kubelet := testKubelet.kubelet pod := podWithUidNameNs("12345678", "foo", "new")

Namespace: pod.Namespace, } tests := []struct { <del> containers                   []api.Container statuses                     []*kubecontainer.ContainerStatus reasons                      map[string]error oldStatuses                  []api.ContainerStatus expectedState                map[string]api.ContainerState </del> <ins> containers    []api.Container statuses      []*kubecontainer.ContainerStatus reasons       map[string]error oldStatuses   []api.ContainerStatus expectedState map[string]api.ContainerState // Only set expectedInitState when it is different from expectedState expectedInitState            map[string]api.ContainerState </ins> expectedLastTerminationState map[string]api.ContainerState }{ // For container with no historical record, State should be Waiting, LastTerminationState should be retrieved from // old status from apiserver. { <del> []api.Container{{Name: "without-old-record"}, {Name: "with-old-record"}}, []*kubecontainer.ContainerStatus{}, map[string]error{}, []api.ContainerStatus{{ </del> <ins> containers: []api.Container{{Name: "without-old-record"}, {Name: "with-old-record"}}, statuses:   []*kubecontainer.ContainerStatus{}, reasons:    map[string]error{}, oldStatuses: []api.ContainerStatus{{ </ins> Name:                 "with-old-record", LastTerminationState: api.ContainerState{Terminated: &api.ContainerStateTerminated{}}, }}, <del> map[string]api.ContainerState{ </del> <ins> expectedState: map[string]api.ContainerState{ </ins> "without-old-record": {Waiting: &api.ContainerStateWaiting{ Reason: startWaitingReason, }},

Reason: startWaitingReason, }}, }, <del> map[string]api.ContainerState{ </del> <ins> expectedInitState: map[string]api.ContainerState{ "without-old-record": {Waiting: &api.ContainerStateWaiting{ Reason: initWaitingReason, }}, "with-old-record": {Waiting: &api.ContainerStateWaiting{ Reason: initWaitingReason, }}, }, expectedLastTerminationState: map[string]api.ContainerState{ </ins> "with-old-record": {Terminated: &api.ContainerStateTerminated{}}, }, }, // For running container, State should be Running, LastTerminationState should be retrieved from latest terminated status. { <del> []api.Container{{Name: "running"}}, []*kubecontainer.ContainerStatus{ </del> <ins> containers: []api.Container{{Name: "running"}}, statuses: []*kubecontainer.ContainerStatus{ </ins> { Name:      "running", State:     kubecontainer.ContainerStateRunning,

ExitCode: 1, }, }, <del> map[string]error{}, []api.ContainerStatus{}, map[string]api.ContainerState{ </del> <ins> reasons:     map[string]error{}, oldStatuses: []api.ContainerStatus{}, expectedState: map[string]api.ContainerState{ </ins> "running": {Running: &api.ContainerStateRunning{ StartedAt: unversioned.NewTime(testTimestamp), }}, }, <del> map[string]api.ContainerState{ </del> <ins> expectedLastTerminationState: map[string]api.ContainerState{ </ins> "running": {Terminated: &api.ContainerStateTerminated{ ExitCode:    1, ContainerID: emptyContainerID,

// recent start error or not, State should be Terminated, LastTerminationState should be retrieved from second latest // terminated status. { <del> []api.Container{{Name: "without-reason"}, {Name: "with-reason"}}, []*kubecontainer.ContainerStatus{ </del> <ins> containers: []api.Container{{Name: "without-reason"}, {Name: "with-reason"}}, statuses: []*kubecontainer.ContainerStatus{ </ins> { Name:     "without-reason", State:    kubecontainer.ContainerStateExited,

ExitCode: 5, }, }, <del> map[string]error{"with-reason": testErrorReason, "succeed": testErrorReason}, []api.ContainerStatus{}, map[string]api.ContainerState{ </del> <ins> reasons:     map[string]error{"with-reason": testErrorReason, "succeed": testErrorReason}, oldStatuses: []api.ContainerStatus{}, expectedState: map[string]api.ContainerState{ </ins> "without-reason": {Terminated: &api.ContainerStateTerminated{ ExitCode:    1, ContainerID: emptyContainerID,

ContainerID: emptyContainerID, }}, }, <del> map[string]api.ContainerState{ </del> <ins> expectedLastTerminationState: map[string]api.ContainerState{ </ins> "without-reason": {Terminated: &api.ContainerStateTerminated{ ExitCode:    3, ContainerID: emptyContainerID,

pod.Status.ContainerStatuses = test.oldStatuses podStatus.ContainerStatuses = test.statuses apiStatus := kubelet.generateAPIPodStatus(pod, podStatus) <del> verifyStatus(t, i, apiStatus, test.expectedState, test.expectedLastTerminationState) </del> <ins> assert.NoError(t, verifyContainerStatuses(apiStatus.ContainerStatuses, test.expectedState, test.expectedLastTerminationState), "case %d", i) } // Everything should be the same for init containers for i, test := range tests { kubelet.reasonCache = NewReasonCache() for n, e := range test.reasons { kubelet.reasonCache.add(pod.UID, n, e, "") } pod.Spec.InitContainers = test.containers pod.Status.InitContainerStatuses = test.oldStatuses podStatus.ContainerStatuses = test.statuses apiStatus := kubelet.generateAPIPodStatus(pod, podStatus) expectedState := test.expectedState if test.expectedInitState != nil { expectedState = test.expectedInitState } assert.NoError(t, verifyContainerStatuses(apiStatus.InitContainerStatuses, expectedState, test.expectedLastTerminationState), "case %d", i) } } // Test generateAPIPodStatus with different restart policies. func TestGenerateAPIPodStatusWithDifferentRestartPolicies(t *testing.T) { testErrorReason := fmt.Errorf("test-error") emptyContainerID := (&kubecontainer.ContainerID{}).String() testKubelet := newTestKubelet(t) kubelet := testKubelet.kubelet pod := podWithUidNameNs("12345678", "foo", "new") containers := []api.Container{{Name: "succeed"}, {Name: "failed"}} podStatus := &kubecontainer.PodStatus{ ID:        pod.UID, Name:      pod.Name, Namespace: pod.Namespace, ContainerStatuses: []*kubecontainer.ContainerStatus{ { Name:     "succeed", State:    kubecontainer.ContainerStateExited, ExitCode: 0, }, { Name:     "failed", State:    kubecontainer.ContainerStateExited, ExitCode: 1, }, { Name:     "succeed", State:    kubecontainer.ContainerStateExited, ExitCode: 2, }, { Name:     "failed", State:    kubecontainer.ContainerStateExited, ExitCode: 3, }, }, } kubelet.reasonCache.add(pod.UID, "succeed", testErrorReason, "") kubelet.reasonCache.add(pod.UID, "failed", testErrorReason, "") for c, test := range []struct { restartPolicy                api.RestartPolicy expectedState                map[string]api.ContainerState expectedLastTerminationState map[string]api.ContainerState // Only set expectedInitState when it is different from expectedState expectedInitState map[string]api.ContainerState // Only set expectedInitLastTerminationState when it is different from expectedLastTerminationState expectedInitLastTerminationState map[string]api.ContainerState }{ { restartPolicy: api.RestartPolicyNever, expectedState: map[string]api.ContainerState{ "succeed": {Terminated: &api.ContainerStateTerminated{ ExitCode:    0, ContainerID: emptyContainerID, }}, "failed": {Terminated: &api.ContainerStateTerminated{ ExitCode:    1, ContainerID: emptyContainerID, }}, }, expectedLastTerminationState: map[string]api.ContainerState{ "succeed": {Terminated: &api.ContainerStateTerminated{ ExitCode:    2, ContainerID: emptyContainerID, }}, "failed": {Terminated: &api.ContainerStateTerminated{ ExitCode:    3, ContainerID: emptyContainerID, }}, }, }, { restartPolicy: api.RestartPolicyOnFailure, expectedState: map[string]api.ContainerState{ "succeed": {Terminated: &api.ContainerStateTerminated{ ExitCode:    0, ContainerID: emptyContainerID, }}, "failed": {Waiting: &api.ContainerStateWaiting{Reason: testErrorReason.Error()}}, }, expectedLastTerminationState: map[string]api.ContainerState{ "succeed": {Terminated: &api.ContainerStateTerminated{ ExitCode:    2, ContainerID: emptyContainerID, }}, "failed": {Terminated: &api.ContainerStateTerminated{ ExitCode:    1, ContainerID: emptyContainerID, }}, }, }, { restartPolicy: api.RestartPolicyAlways, expectedState: map[string]api.ContainerState{ "succeed": {Waiting: &api.ContainerStateWaiting{Reason: testErrorReason.Error()}}, "failed":  {Waiting: &api.ContainerStateWaiting{Reason: testErrorReason.Error()}}, }, expectedLastTerminationState: map[string]api.ContainerState{ "succeed": {Terminated: &api.ContainerStateTerminated{ ExitCode:    0, ContainerID: emptyContainerID, }}, "failed": {Terminated: &api.ContainerStateTerminated{ ExitCode:    1, ContainerID: emptyContainerID, }}, }, // If the init container is terminated with exit code 0, it won't be restarted even when the // restart policy is RestartAlways. expectedInitState: map[string]api.ContainerState{ "succeed": {Terminated: &api.ContainerStateTerminated{ ExitCode:    0, ContainerID: emptyContainerID, }}, "failed": {Waiting: &api.ContainerStateWaiting{Reason: testErrorReason.Error()}}, }, expectedInitLastTerminationState: map[string]api.ContainerState{ "succeed": {Terminated: &api.ContainerStateTerminated{ ExitCode:    2, ContainerID: emptyContainerID, }}, "failed": {Terminated: &api.ContainerStateTerminated{ ExitCode:    1, ContainerID: emptyContainerID, }}, }, }, } { pod.Spec.RestartPolicy = test.restartPolicy // Test normal containers pod.Spec.Containers = containers apiStatus := kubelet.generateAPIPodStatus(pod, podStatus) expectedState, expectedLastTerminationState := test.expectedState, test.expectedLastTerminationState assert.NoError(t, verifyContainerStatuses(apiStatus.ContainerStatuses, expectedState, expectedLastTerminationState), "case %d", c) pod.Spec.Containers = nil // Test init containers pod.Spec.InitContainers = containers apiStatus = kubelet.generateAPIPodStatus(pod, podStatus) if test.expectedInitState != nil { expectedState = test.expectedInitState } if test.expectedInitLastTerminationState != nil { expectedLastTerminationState = test.expectedInitLastTerminationState } assert.NoError(t, verifyContainerStatuses(apiStatus.InitContainerStatuses, expectedState, expectedLastTerminationState), "case %d", c) pod.Spec.InitContainers = nil </ins> } }