Change:
I'm actually not sure if it's by design. But certainly replication controller doesn't behave like it expects this.
If restart policy is to restart always, they should be restarted, but in that case they shouldn't be reported as failed.
This is a known issue, and by design based on upper-level will observe the status reported from kubelet, but not check with RestartPolicy and generate PodCondition / PodStatus. When RestartPolicy is Always, even Kubelet reports all containers are at failed state, apiserver shouldn't mark Pod as failed state. This is why initially I want kubelet report Pod state / status / condiction.
As a quick fix, we could let apiserver populate the right state / status based on containerStatus and RestartPolicy for now. In a long term, I still think we should make kubelet smarter and atomic and self-contained component, not just a wrapper around docker.
Short term fix should be P1 then we can de-prio for a "realer" fix.
Yes, I am working on the short term fix, and should have it included in 0.6 cut.
are you still working on this?
Yes, I am working on this. Should finish it today.

Discussion:
if pod.Status.Info == nil { return api.PodPending, nil } <ins> // TODO(dchen1107): move the entire logic to kubelet? </ins> running := 0 <ins> waiting := 0 </ins> stopped := 0 <ins> failed := 0 succeeded := 0 </ins> unknown := 0 for _, container := range pod.Spec.Containers { if containerStatus, ok := pod.Status.Info[container.Name]; ok {

running++ } else if containerStatus.State.Termination != nil { stopped++ <ins> if containerStatus.State.Termination.ExitCode == 0 { succeeded++ } else { failed++ } } else if containerStatus.State.Waiting != nil { waiting++ </ins> } else { unknown++ }

} } switch { <ins> case waiting > 0: // One or more containers has not been started return api.PodPending, nil </ins> case running > 0 && unknown == 0: <ins> // All containers have been started, and at least // one container is running </ins> return api.PodRunning, nil case running == 0 && stopped > 0 && unknown == 0: <del> return api.PodFailed, nil case running == 0 && stopped == 0 && unknown > 0: return api.PodPending, nil </del> <ins> // All containers are terminated if pod.Spec.RestartPolicy.Always != nil { // All containers are in the process of restarting return api.PodRunning, nil } if stopped == succeeded { // RestartPolicy is not Always, and all // containers are terminated in success return api.PodSucceeded, nil } if pod.Spec.RestartPolicy.Never != nil { // RestartPolicy is Never, and all containers are // terminated with at least one in failure return api.PodFailed, nil } // RestartPolicy is OnFailure, and at least one in failure // and in the process of restarting return api.PodRunning, nil </ins> default: return api.PodPending, nil }

} } <del> func TestMakePodStatus(t *testing.T) { </del> <ins> func TestPodStatusWithBadNode(t *testing.T) { </ins> fakeClient := client.Fake{ MinionsList: api.NodeList{ Items: []api.Node{

{Name: "containerA"}, {Name: "containerB"}, }, <del> } currentState := api.PodStatus{ Host: "machine", </del> <ins> RestartPolicy: api.RestartPolicy{Always: &api.RestartPolicyAlways{}}, </ins> } runningState := api.ContainerStatus{ State: api.ContainerState{

status api.PodPhase test   string }{ <del> {&api.Pod{Spec: desiredState, Status: currentState}, api.PodPending, "waiting"}, </del> { &api.Pod{ Spec: desiredState,

"containerA": runningState, "containerB": runningState, }, <del> Host: "machine", </del> <ins> Host: "machine-two", </ins> }, }, <del> api.PodRunning, "all running", </del> <ins> api.PodFailed, "all running but minion is missing", </ins> }, { &api.Pod{ Spec: desiredState, Status: api.PodStatus{ Info: map[string]api.ContainerStatus{ <del> "containerA": runningState, "containerB": runningState, </del> <ins> "containerA": stoppedState, "containerB": stoppedState, </ins> }, Host: "machine-two", }, }, api.PodFailed, <del> "all running but minion is missing", </del> <ins> "all stopped but minion missing", }, } for _, test := range tests { if status, err := getPodStatus(test.pod, fakeClient.Nodes()); status != test.status { t.Errorf("In test %s, expected %v, got %v", test.test, test.status, status) if err != nil { t.Errorf("In test %s, unexpected error: %v", test.test, err) } } } } func TestPodStatusWithRestartAlways(t *testing.T) { fakeClient := client.Fake{ MinionsList: api.NodeList{ Items: []api.Node{ { ObjectMeta: api.ObjectMeta{Name: "machine"}, }, }, }, } desiredState := api.PodSpec{ Containers: []api.Container{ {Name: "containerA"}, {Name: "containerB"}, }, RestartPolicy: api.RestartPolicy{Always: &api.RestartPolicyAlways{}}, } currentState := api.PodStatus{ Host: "machine", } runningState := api.ContainerStatus{ State: api.ContainerState{ Running: &api.ContainerStateRunning{}, }, } stoppedState := api.ContainerStatus{ State: api.ContainerState{ Termination: &api.ContainerStateTerminated{}, }, } tests := []struct { pod    *api.Pod status api.PodPhase test   string }{ {&api.Pod{Spec: desiredState, Status: currentState}, api.PodPending, "waiting"}, { &api.Pod{ Spec: desiredState, Status: api.PodStatus{ Info: map[string]api.ContainerStatus{ "containerA": runningState, "containerB": runningState, }, Host: "machine", }, }, api.PodRunning, "all running", </ins> }, { &api.Pod{

Host: "machine", }, }, <del> api.PodFailed, "all stopped", </del> <ins> api.PodRunning, "all stopped with restart always", </ins> }, { &api.Pod{ Spec: desiredState, Status: api.PodStatus{ Info: map[string]api.ContainerStatus{ <del> "containerA": stoppedState, </del> <ins> "containerA": runningState, </ins> "containerB": stoppedState, }, <del> Host: "machine-two", </del> <ins> Host: "machine", }, }, api.PodRunning, "mixed state #1 with restart always", }, { &api.Pod{ Spec: desiredState, Status: api.PodStatus{ Info: map[string]api.ContainerStatus{ "containerA": runningState, }, Host: "machine", }, }, api.PodPending, "mixed state #2 with restart always", }, } for _, test := range tests { if status, err := getPodStatus(test.pod, fakeClient.Nodes()); status != test.status { t.Errorf("In test %s, expected %v, got %v", test.test, test.status, status) if err != nil { t.Errorf("In test %s, unexpected error: %v", test.test, err) } } } } func TestPodStatusWithRestartNever(t *testing.T) { fakeClient := client.Fake{ MinionsList: api.NodeList{ Items: []api.Node{ { ObjectMeta: api.ObjectMeta{Name: "machine"}, }, }, }, } desiredState := api.PodSpec{ Containers: []api.Container{ {Name: "containerA"}, {Name: "containerB"}, }, RestartPolicy: api.RestartPolicy{Never: &api.RestartPolicyNever{}}, } currentState := api.PodStatus{ Host: "machine", } runningState := api.ContainerStatus{ State: api.ContainerState{ Running: &api.ContainerStateRunning{}, }, } succeededState := api.ContainerStatus{ State: api.ContainerState{ Termination: &api.ContainerStateTerminated{ ExitCode: 0, }, }, } failedState := api.ContainerStatus{ State: api.ContainerState{ Termination: &api.ContainerStateTerminated{ ExitCode: -1, }, }, } tests := []struct { pod    *api.Pod status api.PodPhase test   string }{ {&api.Pod{Spec: desiredState, Status: currentState}, api.PodPending, "waiting"}, { &api.Pod{ Spec: desiredState, Status: api.PodStatus{ Info: map[string]api.ContainerStatus{ "containerA": runningState, "containerB": runningState, }, Host: "machine", }, }, api.PodRunning, "all running with restart never", }, { &api.Pod{ Spec: desiredState, Status: api.PodStatus{ Info: map[string]api.ContainerStatus{ "containerA": succeededState, "containerB": succeededState, }, Host: "machine", }, }, api.PodSucceeded, "all succeeded with restart never", }, { &api.Pod{ Spec: desiredState, Status: api.PodStatus{ Info: map[string]api.ContainerStatus{ "containerA": failedState, "containerB": failedState, }, Host: "machine", </ins> }, }, api.PodFailed, <del> "all stopped but minion missing", </del> <ins> "all failed with restart never", </ins> }, { &api.Pod{

Status: api.PodStatus{ Info: map[string]api.ContainerStatus{ "containerA": runningState, <del> "containerB": stoppedState, </del> <ins> "containerB": succeededState, }, Host: "machine", }, }, api.PodRunning, "mixed state #1 with restart never", }, { &api.Pod{ Spec: desiredState, Status: api.PodStatus{ Info: map[string]api.ContainerStatus{ "containerA": runningState, }, Host: "machine", }, }, api.PodPending, "mixed state #2 with restart never", }, } for _, test := range tests { if status, err := getPodStatus(test.pod, fakeClient.Nodes()); status != test.status { t.Errorf("In test %s, expected %v, got %v", test.test, test.status, status) if err != nil { t.Errorf("In test %s, unexpected error: %v", test.test, err) } } } } func TestPodStatusWithRestartOnFailure(t *testing.T) { fakeClient := client.Fake{ MinionsList: api.NodeList{ Items: []api.Node{ { ObjectMeta: api.ObjectMeta{Name: "machine"}, }, }, }, } desiredState := api.PodSpec{ Containers: []api.Container{ {Name: "containerA"}, {Name: "containerB"}, }, RestartPolicy: api.RestartPolicy{OnFailure: &api.RestartPolicyOnFailure{}}, } currentState := api.PodStatus{ Host: "machine", } runningState := api.ContainerStatus{ State: api.ContainerState{ Running: &api.ContainerStateRunning{}, }, } succeededState := api.ContainerStatus{ State: api.ContainerState{ Termination: &api.ContainerStateTerminated{ ExitCode: 0, }, }, } failedState := api.ContainerStatus{ State: api.ContainerState{ Termination: &api.ContainerStateTerminated{ ExitCode: -1, }, }, } tests := []struct { pod    *api.Pod status api.PodPhase test   string }{ {&api.Pod{Spec: desiredState, Status: currentState}, api.PodPending, "waiting"}, { &api.Pod{ Spec: desiredState, Status: api.PodStatus{ Info: map[string]api.ContainerStatus{ "containerA": runningState, "containerB": runningState, }, Host: "machine", }, }, api.PodRunning, "all running with restart onfailure", }, { &api.Pod{ Spec: desiredState, Status: api.PodStatus{ Info: map[string]api.ContainerStatus{ "containerA": succeededState, "containerB": succeededState, }, Host: "machine", }, }, api.PodSucceeded, "all succeeded with restart onfailure", }, { &api.Pod{ Spec: desiredState, Status: api.PodStatus{ Info: map[string]api.ContainerStatus{ "containerA": failedState, "containerB": failedState, }, Host: "machine", }, }, api.PodRunning, "all failed with restart never", }, { &api.Pod{ Spec: desiredState, Status: api.PodStatus{ Info: map[string]api.ContainerStatus{ "containerA": runningState, "containerB": succeededState, </ins> }, Host: "machine", }, }, api.PodRunning, <del> "mixed state #1", </del> <ins> "mixed state #1 with restart onfailure", </ins> }, { &api.Pod{

}, }, api.PodPending, <del> "mixed state #2", </del> <ins> "mixed state #2 with restart onfailure", </ins> }, } for _, test := range tests {