Change:
Successfully ran containers with "onFailure" restartPolicy are restarting after purging oldest docker containers. Kubelet purges/removes oldest containers periodically from docker. After purging, we loose the exit code and history of previously ran containers and restart containers as if they were never run.
Thanks & I can help fix this. Let me know. Thanks
Thanks for reporting Just to make sure I understand the issue: You ran some containers with an "onFailure" restartPolicy. When the Kubelet purged old containers the container was restarted (unexpectedly).
Ah, after reading again I think the issue is that after the containers are purged, when the container exits successfully it get restarted although it shouldn't. Is that correct?
After reading through the code I found a bug in the implementation that is most likely causing this error. Should have a PR soon.
Yes. Correct. After initial container run, it should never be run again, since it already ran to completion. Will wait for your PR.
PR is out. This should fix the issue. Feel free to test if you can :)

Discussion:
rand.Uint32()) } <ins> // TODO(vmarmol): This should probably return an error. </ins> // Unpacks a container name, returning the pod full name and container name we would have used to // construct the docker name. If the docker name isn't the one we created, we may return empty strings. func ParseDockerName(name string) (podFullName string, podUID types.UID, containerName string, hash uint64) {

} uidToIDMap := map[string][]string{} for _, container := range containers { <del> _, uid, name, _ := dockertools.ParseDockerName(container.ID) </del> <ins> _, uid, name, _ := dockertools.ParseDockerName(container.Names[0]) </ins> uidName := string(uid) + "." + name uidToIDMap[uidName] = append(uidToIDMap[uidName], container.ID) }

} } <ins> func verifyStringArrayEqualsAnyOrder(t *testing.T, actual, expected []string) { invalid := len(actual) != len(expected) if !invalid { for _, exp := range expected { found := false for _, act := range actual { if exp == act { found = true break } } if !found { t.Errorf("Expected element %s not found in %#v", exp, actual) } } } if invalid { t.Errorf("Expected: %#v, Actual: %#v", expected, actual) } } </ins> func verifyBoolean(t *testing.T, expected, value bool) { if expected != value { t.Errorf("Unexpected boolean.  Expected %t.  Found %t", expected, value)

containerDetails map[string]*docker.Container expectedRemoved  []string }{ <ins> // Remove oldest containers. </ins> { containers: []docker.APIContainers{ {

Names: []string{"/k8s_POD_foo.new.test_.deadbeef_42"}, ID:    "3876", }, <ins> }, containerDetails: map[string]*docker.Container{ "1876": { State: docker.State{ Running: false, }, ID:      "1876", Created: time.Now(), }, }, expectedRemoved: []string{"1876"}, }, // Only remove non-running containers. { containers: []docker.APIContainers{ </ins> { // pod infra container Names: []string{"/k8s_POD_foo.new.test_.deadbeef_42"}, <del> ID:    "4876", </del> <ins> ID:    "1876", }, { // pod infra container Names: []string{"/k8s_POD_foo.new.test_.deadbeef_42"}, ID:    "2876", </ins> }, { // pod infra container Names: []string{"/k8s_POD_foo.new.test_.deadbeef_42"}, <del> ID:    "5876", </del> <ins> ID:    "3876", </ins> }, { // pod infra container Names: []string{"/k8s_POD_foo.new.test_.deadbeef_42"}, <del> ID:    "6876", </del> <ins> ID:    "4876", </ins> }, }, containerDetails: map[string]*docker.Container{ "1876": { State: docker.State{ <del> Running: false, </del> <ins> Running: true, </ins> }, ID:      "1876", Created: time.Now(), }, <ins> "2876": { State: docker.State{ Running: false, }, ID:      "2876", Created: time.Now(), }, </ins> }, <del> expectedRemoved: []string{"1876"}, </del> <ins> expectedRemoved: []string{"2876"}, </ins> }, <ins> // Less than maxContainerCount doesn't delete any. </ins> { containers: []docker.APIContainers{ {

Names: []string{"/k8s_POD_foo.new.test_.deadbeef_42"}, ID:    "1876", }, <ins> }, }, // maxContainerCount applies per container.. { containers: []docker.APIContainers{ </ins> { // pod infra container <del> Names: []string{"/k8s_POD_foo.new.test_.deadbeef_42"}, ID:    "2876", </del> <ins> Names: []string{"/k8s_POD_foo2.new.test_.beefbeef_40"}, ID:    "1706", </ins> }, { // pod infra container <del> Names: []string{"/k8s_POD_foo.new.test_.deadbeef_42"}, ID:    "3876", </del> <ins> Names: []string{"/k8s_POD_foo2.new.test_.beefbeef_40"}, ID:    "2706", </ins> }, { // pod infra container <del> Names: []string{"/k8s_POD_foo.new.test_.deadbeef_42"}, ID:    "4876", </del> <ins> Names: []string{"/k8s_POD_foo2.new.test_.beefbeef_40"}, ID:    "3706", </ins> }, { // pod infra container Names: []string{"/k8s_POD_foo.new.test_.deadbeef_42"}, <del> ID:    "5876", </del> <ins> ID:    "1876", </ins> }, { // pod infra container Names: []string{"/k8s_POD_foo.new.test_.deadbeef_42"}, <del> ID:    "6876", </del> <ins> ID:    "2876", </ins> }, { // pod infra container Names: []string{"/k8s_POD_foo.new.test_.deadbeef_42"}, <del> ID:    "7876", </del> <ins> ID:    "3876", </ins> }, }, containerDetails: map[string]*docker.Container{ <del> "1876": { </del> <ins> "1706": { </ins> State: docker.State{ <del> Running: true, </del> <ins> Running: false, </ins> }, <del> ID:      "1876", </del> <ins> ID:      "1706", </ins> Created: time.Now(), }, <del> "2876": { </del> <ins> "1876": { </ins> State: docker.State{ Running: false, }, <del> ID:      "2876", </del> <ins> ID:      "1876", </ins> Created: time.Now(), }, }, <del> expectedRemoved: []string{"2876"}, }, { containers: []docker.APIContainers{ { // pod infra container Names: []string{"/k8s_POD_foo.new.test_.deadbeef_42"}, ID:    "1876", }, }, </del> <ins> expectedRemoved: []string{"1706", "1876"}, </ins> }, } for _, test := range tests { kubelet, fakeDocker := newTestKubelet(t) <del> kubelet.maxContainerCount = 5 </del> <ins> kubelet.maxContainerCount = 2 </ins> fakeDocker.ContainerList = test.containers fakeDocker.ContainerMap = test.containerDetails fakeDocker.Container = &docker.Container{ID: "error", Created: time.Now()}

if err != nil { t.Errorf("unexpected error: %v", err) } <del> if !reflect.DeepEqual(fakeDocker.Removed, test.expectedRemoved) { t.Errorf("expected: %v, got: %v", test.expectedRemoved, fakeDocker.Removed) } </del> <ins> verifyStringArrayEqualsAnyOrder(t, test.expectedRemoved, fakeDocker.Removed) </ins> } }