Change:
I've noticed a issue where if you have multiple replicated slave pods that don't have a HOST assigned and you reduce the number of replicas in the replicationController definition for those pods, sometimes the currently assigned (and running) pods get killed and the unassigned pod will get assigned to run in the killed pods place. The issue can be reproduced using the guestbook example provided in the repo. Steps to reproduce: (Note this is not always deterministic and may need multiple attempts) the master pod the master service the replicated slaves the Slave Controller definition to only have 1 replicas the Slave Controller with new definition The expected behavior should be that the scheduler is able to determine that unassigned pods are more viable candidates when reducing number of replicas. It seems to be more likely that since this issue is not deterministic, the ordering in which the command returns result is not always deterministically in the same order
: please can you look at this or suggested someone else? Thank you.
Unassign myself from this bug since I cannot get working on this soon. Also remove kubelet from the label since it is not kubelet issue. This is obviously a replicaController issue. cc/ Instead of destroying pod in pending state, replicaController unbound running pod from a node. Kubelet running on such node has this new desire state and kill unwanted pod to match it.
Yes, the replication controller should be smarter, taking into account Phase, Readiness, recency, ... This will be a significant problem for auto-scalers, but I expect manual scaling to be infrequent. I doubt we'll get to this soon. A PR would be welcome. Place to change is here:
Hi, I am final year Master's student of International Institute of Information Technology, Hyderabad. I am familiar with docker and containers. I have basic understanding of Go language with some hands on Docker API. This project seems interesting and a solution to this will also be a good addition to Kubernetes. So I would like to pick this as my GSoC project. I have started reading about Replication Controller and Pods and going though the code. I would appreciate if someone has any inputs for me on this. Thanks, Swapnil Patil

Discussion:
} return ContainerStatus{} } <ins> // IsPodReady retruns true if a pod is ready; false otherwise. func IsPodReady(pod *Pod) bool { for _, c := range pod.Status.Conditions { if c.Type == PodReady && c.Status == ConditionTrue { return true } } return false } </ins>

import ( "fmt" <ins> "sort" </ins> "sync" "time"

if !ok { if status, ok := event.Object.(*api.Status); ok { if status.Status == api.StatusFailure { <del> glog.Errorf("failed to watch: %v", status) </del> <ins> glog.Errorf("Failed to watch: %v", status) </ins> // Clear resource version here, as above, this won't hurt consistency, but we // should consider introspecting more carefully here. (or make the apiserver smarter) // "why not both?"

*resourceVersion = rc.ResourceVersion // Sync even if this is a deletion event, to ensure that we leave // it in the desired state. <del> glog.V(4).Infof("About to sync from watch: %v", rc.Name) </del> <ins> glog.V(4).Infof("About to sync from watch: %q", rc.Name) </ins> if err := rm.syncHandler(*rc); err != nil { util.HandleError(fmt.Errorf("unexpected sync error: %v", err)) }

} } <del> // Helper function. Also used in pkg/registry/controller, for now. func FilterActivePods(pods []api.Pod) []*api.Pod { </del> <ins> // filterActivePods returns pods that have not terminated. func filterActivePods(pods []api.Pod) []*api.Pod { </ins> var result []*api.Pod <del> for i := range pods { value := &pods[i] </del> <ins> for _, value := range pods { </ins> if api.PodSucceeded != value.Status.Phase && api.PodFailed != value.Status.Phase { <del> result = append(result, value) </del> <ins> result = append(result, &value) </ins> } } return result } <ins> type activePods []*api.Pod func (s activePods) Len() int      { return len(s) } func (s activePods) Swap(i, j int) { s[i], s[j] = s[j], s[i] } func (s activePods) Less(i, j int) bool { // Unassigned < assigned if s[i].Spec.Host == "" && s[j].Spec.Host != "" { return true } // PodPending < PodUnknown < PodRunning m := map[api.PodPhase]int{api.PodPending: 0, api.PodUnknown: 1, api.PodRunning: 2} if m[s[i].Status.Phase] != m[s[j].Status.Phase] { return m[s[i].Status.Phase] < m[s[j].Status.Phase] } // Not ready < ready if !api.IsPodReady(s[i]) && api.IsPodReady(s[j]) { return true } return false } </ins> func (rm *ReplicationManager) syncReplicationController(controller api.ReplicationController) error { s := labels.Set(controller.Spec.Selector).AsSelector() podList, err := rm.kubeClient.Pods(controller.Namespace).List(s) if err != nil { return err } <del> filteredList := FilterActivePods(podList.Items) activePods := len(filteredList) diff := activePods - controller.Spec.Replicas </del> <ins> filteredList := filterActivePods(podList.Items) numActivePods := len(filteredList) diff := numActivePods - controller.Spec.Replicas </ins> if diff < 0 { diff *= -1 wait := sync.WaitGroup{} wait.Add(diff) <del> glog.V(2).Infof("Too few "%s" replicas, creating %dn", controller.Name, diff) </del> <ins> glog.V(2).Infof("Too few %q replicas, creating %d", controller.Name, diff) </ins> for i := 0; i < diff; i++ { go func() { defer wait.Done()

} wait.Wait() } else if diff > 0 { <del> glog.V(2).Infof("Too many "%s" replicas, deleting %dn", controller.Name, diff) </del> <ins> glog.V(2).Infof("Too many %q replicas, deleting %d", controller.Name, diff) // Sort the pods in the order such that not-ready < ready, unscheduled // < scheduled, and pending < running. This ensures that we delete pods // in the earlier stages whenever possible. sort.Sort(activePods(filteredList)) </ins> wait := sync.WaitGroup{} wait.Add(diff) for i := 0; i < diff; i++ {

} wait.Wait() } <del> if controller.Status.Replicas != activePods { controller.Status.Replicas = activePods </del> <ins> if controller.Status.Replicas != numActivePods { controller.Status.Replicas = numActivePods </ins> _, err = rm.kubeClient.ReplicationControllers(controller.Namespace).Update(&controller) if err != nil { return err

import ( "fmt" <ins> "math/rand" </ins> "net/http" "net/http/httptest" <ins> "reflect" "sort" </ins> "sync" "testing" "time"

t.Errorf("Expected 1 call but got 0") } } <ins> func TestSortingActivePods(t *testing.T) { numPods := 5 podList := newPodList(numPods) pods := make([]*api.Pod, len(podList.Items)) for i := range podList.Items { pods[i] = &podList.Items[i] } // pods[0] is not scheduled yet. pods[0].Spec.Host = "" pods[0].Status.Phase = api.PodPending // pods[1] is scheduled but pending. pods[1].Spec.Host = "bar" pods[1].Status.Phase = api.PodPending // pods[2] is unknown. pods[2].Spec.Host = "foo" pods[2].Status.Phase = api.PodUnknown // pods[3] is running but not ready. pods[3].Spec.Host = "foo" pods[3].Status.Phase = api.PodRunning // pods[4] is running and ready. pods[4].Spec.Host = "foo" pods[4].Status.Phase = api.PodRunning pods[4].Status.Conditions = []api.PodCondition{{Type: api.PodReady, Status: api.ConditionTrue}} getOrder := func(pods []*api.Pod) []string { names := make([]string, len(pods)) for i := range pods { names[i] = pods[i].Name } return names } expected := getOrder(pods) for i := 0; i < 20; i++ { idx := rand.Perm(numPods) randomizedPods := make([]*api.Pod, numPods) for j := 0; j < numPods; j++ { randomizedPods[j] = pods[idx[j]] } sort.Sort(activePods(randomizedPods)) actual := getOrder(randomizedPods) if !reflect.DeepEqual(actual, expected) { t.Errorf("expected %v, got %v", expected, actual) } } } </ins>

continue } <del> inService := false for _, c := range pod.Status.Conditions { if c.Type == api.PodReady && c.Status == api.ConditionTrue { inService = true break } } if !inService { </del> <ins> if !api.IsPodReady(pod) { </ins> glog.V(5).Infof("Pod is out of service: %v/%v", pod.Namespace, pod.Name) continue }