Change:
The test is in flaky suite and no one seems to care. We need an owner for this test. cc Last failure log:
do you think you can take a look at this?
This is happening because of a bad assumption. The assumption is: if the reflector's store is consistent with etcd, the store given back to clients of the framework.Controller is too. I'd really like a general fix applicable to all controllers for this problem, but fixing it just for the RCs is pretty easy. In more detail: This function reports when the reflector has listed from apiserver. But the store given to the reflector is the fifo of the framework.Controller. The store given back to the caller is still not consistent, and will not be till the framework.Controller puts all the objects from the fifo into the client state store (). The rc specific fix: Before starting the rc manager, seed the expectations store with an expectation like {rc:foo, adds: foo.Status.Replicas, deletes:0}. This will wait for status.Replicas number of ADD events till a timeout. The general fix: Somehow expose this through framework.Controller.HasSynced (clients don't care about the fifo anyway, shouldn't it just be an implementation detail?) ideas? cc'ing people who have written controllers that might be interested I don't think this (and generally anything flaky for 1 release) is a p0, I can fix it if you don't mind dealing with the flakes till I've cleared things off my plate. I'd be more than happy to review a fix.
Well, it's a P0 because test is flaky. E.g last run:
- did you have chance to look into it?
The "DaemonRestart" tests are still labeled at . I will send a PR to promote them.
are you still planning to send aPR to promote the DaemonRestart tests out of flaky? I think this is the right thing to do.
Ack; sorry, I lost this in the muck.

Discussion:
"k8s.io/kubernetes/pkg/client/record" client "k8s.io/kubernetes/pkg/client/unversioned" "k8s.io/kubernetes/pkg/controller/framework" <ins> "k8s.io/kubernetes/pkg/fields" </ins> "k8s.io/kubernetes/pkg/labels" "k8s.io/kubernetes/pkg/runtime" )

func FilterActivePods(pods []api.Pod) []*api.Pod { var result []*api.Pod for i := range pods { <del> if api.PodSucceeded != pods[i].Status.Phase && api.PodFailed != pods[i].Status.Phase && pods[i].DeletionTimestamp == nil { result = append(result, &pods[i]) </del> <ins> p := pods[i] if api.PodSucceeded != p.Status.Phase && api.PodFailed != p.Status.Phase && p.DeletionTimestamp == nil { result = append(result, &p) } else { glog.V(4).Infof("Ignoring inactive pod %v/%v in state %v, deletion time %v", p.Namespace, p.Name, p.Status.Phase, p.DeletionTimestamp) </ins> } } return result } <ins> // SyncAllPodsWithStore lists all pods and inserts them into the given store. // Though this function is written in a generic manner, it is only used by the // controllers for a specific purpose, to synchronously populate the store // with the first batch of pods that would otherwise be sent by the Informer. // Doing this avoids more complicated forms of synchronization with the // Informer, though it also means that the controller calling this function // will receive "OnUpdate" events for all the pods in the store, instead of // "OnAdd". This should be ok, since most controllers are level triggered // and make decisions based on the contents of the store. // // TODO: Extend this logic to load arbitrary local state for the controllers // instead of just pods. func SyncAllPodsWithStore(kubeClient client.Interface, store cache.Store) { var allPods *api.PodList var err error listOptions := api.ListOptions{LabelSelector: labels.Everything(), FieldSelector: fields.Everything()} for { if allPods, err = kubeClient.Pods(api.NamespaceAll).List(listOptions); err != nil { glog.Warningf("Retrying pod list: %v", err) continue } break } pods := []interface{}{} for i := range allPods.Items { p := allPods.Items[i] glog.V(4).Infof("Initializing store with pod %v/%v", p.Namespace, p.Name) pods = append(pods, &p) } store.Replace(pods, allPods.ResourceVersion) return } </ins>

// Run begins watching and syncing daemon sets. func (dsc *DaemonSetsController) Run(workers int, stopCh <-chan struct{}) { defer util.HandleCrash() <ins> glog.Infof("Starting Daemon Sets controller manager") controller.SyncAllPodsWithStore(dsc.kubeClient, dsc.podStore.Store) </ins> go dsc.dsController.Run(stopCh) go dsc.podController.Run(stopCh) go dsc.nodeController.Run(stopCh)

if ds.Status.DesiredNumberScheduled == desiredNumberScheduled && ds.Status.CurrentNumberScheduled == currentNumberScheduled && ds.Status.NumberMisscheduled == numberMisscheduled { return nil } var updateErr, getErr error for i := 0; i <= StatusUpdateRetries; i++ { ds.Status.DesiredNumberScheduled = desiredNumberScheduled

import ( "fmt" <ins> "net/http/httptest" </ins> "testing" "k8s.io/kubernetes/pkg/api"

"k8s.io/kubernetes/pkg/client/cache" client "k8s.io/kubernetes/pkg/client/unversioned" "k8s.io/kubernetes/pkg/controller" <ins> "k8s.io/kubernetes/pkg/runtime" </ins> "k8s.io/kubernetes/pkg/securitycontext" <ins> utiltesting "k8s.io/kubernetes/pkg/util/testing" </ins> ) var (

manager.podStoreSynced = alwaysReady syncAndValidateDaemonSets(t, manager, ds, podControl, 1, 0) } <ins> func TestDSManagerInit(t *testing.T) { // Insert a stable daemon set and make sure we don't create an extra pod // for the one node which already has a daemon after a simulated restart. ds := newDaemonSet("test") ds.Status = extensions.DaemonSetStatus{ CurrentNumberScheduled: 1, NumberMisscheduled:     0, DesiredNumberScheduled: 1, } nodeName := "only-node" podList := &api.PodList{ Items: []api.Pod{ *newPod("podname", nodeName, simpleDaemonSetLabel), }} response := runtime.EncodeOrDie(testapi.Default.Codec(), podList) fakeHandler := utiltesting.FakeHandler{ StatusCode:   200, ResponseBody: response, } testServer := httptest.NewServer(&fakeHandler) // TODO: Uncomment when fix #19254 // defer testServer.Close() client := client.NewOrDie(&client.Config{Host: testServer.URL, GroupVersion: testapi.Default.GroupVersion()}) manager := NewDaemonSetsController(client, controller.NoResyncPeriodFunc) manager.dsStore.Add(ds) manager.nodeStore.Add(newNode(nodeName, nil)) manager.podStoreSynced = alwaysReady controller.SyncAllPodsWithStore(manager.kubeClient, manager.podStore.Store) fakePodControl := &controller.FakePodControl{} manager.podControl = fakePodControl manager.syncHandler(getKey(ds, t)) validateSyncDaemonSets(t, fakePodControl, 0, 0) } </ins>

// Run begins watching and syncing. func (rm *ReplicationManager) Run(workers int, stopCh <-chan struct{}) { defer util.HandleCrash() <ins> glog.Infof("Starting RC Manager") controller.SyncAllPodsWithStore(rm.kubeClient, rm.podStore.Store) </ins> go rm.rcController.Run(stopCh) go rm.podController.Run(stopCh) for i := 0; i < workers; i++ {

} } } <ins> func TestRCManagerInit(t *testing.T) { // Insert a stable rc into the replication manager's store and make sure // it syncs pods with the apiserver before making any decisions. rc := newReplicationController(2) response := runtime.EncodeOrDie(testapi.Default.Codec(), newPodList(nil, 2, api.PodRunning, rc)) fakeHandler := utiltesting.FakeHandler{ StatusCode:   200, ResponseBody: response, } testServer := httptest.NewServer(&fakeHandler) // TODO: Uncomment when fix #19254 // defer testServer.Close() client := client.NewOrDie(&client.Config{Host: testServer.URL, GroupVersion: testapi.Default.GroupVersion()}) manager := NewReplicationManager(client, controller.NoResyncPeriodFunc, BurstReplicas) manager.rcStore.Store.Add(rc) manager.podStoreSynced = alwaysReady controller.SyncAllPodsWithStore(manager.kubeClient, manager.podStore.Store) fakePodControl := &controller.FakePodControl{} manager.podControl = fakePodControl manager.syncReplicationController(getKey(rc, t)) validateSyncReplication(t, fakePodControl, 0, 0) } </ins>