Change:
}) }) <ins> var _ = framework.KubeDescribe("Pet set recreate [Slow] [Feature:PetSet]", func() { f := framework.NewDefaultFramework("pet-set-recreate") var c *client.Client var ns string labels := map[string]string{ "foo": "bar", "baz": "blah", } headlessSvcName := "test" podName := "test-pod" petSetName := "web" petPodName := "web-0" BeforeEach(func() { framework.SkipUnlessProviderIs("gce", "vagrant") By("creating service " + headlessSvcName + " in namespace " + f.Namespace.Name) headlessService := createServiceSpec(headlessSvcName, "", true, labels) _, err := f.Client.Services(f.Namespace.Name).Create(headlessService) framework.ExpectNoError(err) c = f.Client ns = f.Namespace.Name }) AfterEach(func() { if CurrentGinkgoTestDescription().Failed { dumpDebugInfo(c, ns) } By("Deleting all petset in ns " + ns) deleteAllPetSets(c, ns) }) It("should recreate evicted petset", func() { By("looking for a node to schedule pet set and pod") nodes := framework.GetReadySchedulableNodesOrDie(f.Client) node := nodes.Items[0] By("creating pod with conflicting port in namespace " + f.Namespace.Name) conflictingPort := api.ContainerPort{HostPort: 21017, ContainerPort: 21017, Name: "conflict"} pod := &api.Pod{ ObjectMeta: api.ObjectMeta{ Name: podName, }, Spec: api.PodSpec{ Containers: []api.Container{ { Name:  "nginx", Image: "gcr.io/google_containers/nginx-slim:0.7", Ports: []api.ContainerPort{conflictingPort}, }, }, NodeName: node.Name, }, } pod, err := f.Client.Pods(f.Namespace.Name).Create(pod) framework.ExpectNoError(err) By("creating petset with conflicting port in namespace " + f.Namespace.Name) ps := newPetSet(petSetName, f.Namespace.Name, headlessSvcName, 1, nil, nil, labels) petContainer := &ps.Spec.Template.Spec.Containers[0] petContainer.Ports = append(petContainer.Ports, conflictingPort) ps.Spec.Template.Spec.NodeName = node.Name _, err = f.Client.Apps().PetSets(f.Namespace.Name).Create(ps) framework.ExpectNoError(err) By("waiting until pod " + podName + " will start running in namespace " + f.Namespace.Name) if err := f.WaitForPodRunning(podName); err != nil { framework.Failf("Pod %v did not start running: %v", podName, err) } var initialPetPodUID types.UID By("waiting until pet pod " + petPodName + " will be recreated and deleted at least once in namespace " + f.Namespace.Name) w, err := f.Client.Pods(f.Namespace.Name).Watch(api.SingleObject(api.ObjectMeta{Name: petPodName})) framework.ExpectNoError(err) // we need to get UID from pod in any state and wait until pet set controller will remove pod atleast once _, err = watch.Until(petPodTimeout, w, func(event watch.Event) (bool, error) { pod := event.Object.(*api.Pod) switch event.Type { case watch.Deleted: framework.Logf("Observed delete event for pet pod %v in namespace %v", pod.Name, pod.Namespace) if initialPetPodUID == "" { return false, nil } return true, nil } framework.Logf("Observed pet pod in namespace: %v, name: %v, uid: %v, status phase: %v. Waiting for petset controller to delete.", pod.Namespace, pod.Name, pod.UID, pod.Status.Phase) initialPetPodUID = pod.UID return false, nil }) if err != nil { framework.Failf("Pod %v expected to be re-created atleast once", petPodName) } By("removing pod with conflicting port in namespace " + f.Namespace.Name) err = f.Client.Pods(f.Namespace.Name).Delete(pod.Name, api.NewDeleteOptions(0)) framework.ExpectNoError(err) By("waiting when pet pod " + petPodName + " will be recreated in namespace " + f.Namespace.Name + " and will be in running state") // we may catch delete event, thats why we are waiting for running phase like this, and not with watch.Until Eventually(func() error { petPod, err := f.Client.Pods(f.Namespace.Name).Get(petPodName) if err != nil { return err } if petPod.Status.Phase != api.PodRunning { return fmt.Errorf("Pod %v is not in running phase: %v", petPod.Name, petPod.Status.Phase) } else if petPod.UID == initialPetPodUID { return fmt.Errorf("Pod %v wasn't recreated: %v == %v", petPod.Name, petPod.UID, initialPetPodUID) } return nil }, petPodTimeout, 2*time.Second).Should(BeNil()) }) }) </ins> func dumpDebugInfo(c *client.Client, ns string) { pl, _ := c.Pods(ns).List(api.ListOptions{LabelSelector: labels.Everything()}) for _, p := range pl.Items {

Discussion:
Description of problem Create a petset, when pod is evicted, the petset can't create a new pod on other node and make sure pod is running. Kubernetes version (use ): v1.4.0-alpha.2.1427+ How to reproduce it (as minimally and precisely as possible): kubele eviction-hard="memory.available<${value}" a peset and wait all pod is running node MemoryPressure=true all pod status What happened: is evicted and petset can't create new pod and make sure pod is running. What you expected to happen: create new pod on other node and make sure pod is running.
Also when node come back with sufficient memory, petset can't make pod running again.
FYI
cc
Was able to reproduce it. Pet set controller just loops with the following logs I0830 12:32:52. 9 ] Syncing PetSet default/cockroachdb with 3 pets I0830 12:32:52. 9 ] PetSet cockroachdb blocked from scaling on pet cockroachdb-0 I0830 12:32:52. 9 ] PetSet cockroachdb waiting on unhealthy pet cockroachdb-0 I0830 12:32:52. 9 ] PetSet cockroachdb waiting on unhealthy pet cockroachdb-1 I0830 12:32:52. 9 ] PetSet cockroachdb waiting on unhealthy pet cockroachdb-2
-- do you have the petset YAML that we can evaluate? What is the quality of service tier for the pods produced by the petset? Can you include the of an evicted petset pod so we can see the final resource requirements spec?
I am able to reproduce it with nginx pet set in e2e test. Here is yaml output of it Also it seems that i have working fix and almost finished debugging e2e test (patchset is linked above). Will be grateful for review.
Ah, this helps clear things up... the linked pr shows the problem in petset controller and not kubelet...
Does it make sense to set milestone 1.4 for this issue? I think the fix is ready for review