Change:
Ran a test executable with more than 20,000 times and no failures. Any hints as to how to get it to show up? I'm running 100 jobs at a time, so my system is stressed already.
I to am unable to reproduce this. I get similar errors if I reduce the timeout at the top of the file. I spent a bit looking at how to rewrite the tests without the timeout, but nothing jumped out at me. Looking at it though, it's really just a 500ms timeout for the test to complete. I'm just going to remove them. Go has a default timeout of 10 minutes. If it's going to pass, it will pass then. If not, Go will give you a nice traceback where it deadlocked.

Discussion:
framework.ResourceEventHandlerFuncs{ AddFunc: jm.enqueueController, UpdateFunc: func(old, cur interface{}) { <del> job := cur.(*experimental.Job) for _, c := range job.Status.Conditions { if c.Type == experimental.JobComplete && c.Status == api.ConditionTrue { return } </del> <ins> if job := cur.(*experimental.Job); !isJobFinished(job) { jm.enqueueController(job) </ins> } <del> jm.enqueueController(cur) </del> }, DeleteFunc: jm.enqueueController, },

return result } <ins> func isJobFinished(j *experimental.Job) bool { for _, c := range j.Status.Conditions { if c.Type == experimental.JobComplete && c.Status == api.ConditionTrue { return true } } return false } </ins> // byCreationTimestamp sorts a list by creation timestamp, using their names as a tie breaker. type byCreationTimestamp []experimental.Job

"k8s.io/kubernetes/pkg/watch" ) <del> // Give each test that starts a background controller up to 1/2 a second. // Since we need to start up a goroutine to test watch, this routine needs // to get cpu before the test can complete. If the test is starved of cpu, // the watch test will take up to 1/2 a second before timing out. const controllerTimeout = 500 * time.Millisecond </del> var alwaysReady = func() bool { return true } func newJob(parallelism, completions int) *experimental.Job {

if err != nil { t.Errorf("Unxpected error when syncing jobs, got %v", err) } <del> ch := make(chan interface{}) go func() { item, _ := manager.queue.Get() ch <- item }() select { case key := <-ch: expectedKey := getKey(job, t) if key != expectedKey { t.Errorf("Expected requeue of job with key %s got %s", expectedKey, key) } case <-time.After(controllerTimeout): manager.queue.ShutDown() t.Errorf("Expected to find a job in the queue, found none.") </del> <ins> t.Log("Waiting for a job in the queue") key, _ := manager.queue.Get() expectedKey := getKey(job, t) if key != expectedKey { t.Errorf("Expected requeue of job with key %s got %s", expectedKey, key) </ins> } }

manager.podStoreSynced = alwaysReady var testJob experimental.Job <del> received := make(chan string) </del> <ins> received := make(chan struct{}) </ins> // The update sent through the fakeWatcher should make its way into the workqueue, // and eventually into the syncHandler.

if !api.Semantic.DeepDerivative(job, testJob) { t.Errorf("Expected %#v, but got %#v", testJob, job) } <del> received <- key </del> <ins> close(received) </ins> return nil } // Start only the job watcher and the workqueue, send a watch event,

// We're sending new job to see if it reaches syncHandler. testJob.Name = "foo" fakeWatch.Add(&testJob) <del> select { case <-received: case <-time.After(controllerTimeout): t.Errorf("Expected 1 call but got 0") } </del> <ins> t.Log("Waiting for job to reach syncHandler") <-received } </ins> <del> // We're sending fake finished job, to see if it reaches syncHandler - it should not, // since we're filtering out finished jobs. testJobv2 := experimental.Job{ ObjectMeta: api.ObjectMeta{Name: "foo"}, </del> <ins> func TestIsJobFinished(t *testing.T) { job := &experimental.Job{ </ins> Status: experimental.JobStatus{ Conditions: []experimental.JobCondition{{ <del> Type:               experimental.JobComplete, Status:             api.ConditionTrue, LastProbeTime:      unversioned.Now(), LastTransitionTime: unversioned.Now(), </del> <ins> Type:   experimental.JobComplete, Status: api.ConditionTrue, </ins> }}, }, } <del> fakeWatch.Modify(&testJobv2) select { case <-received: t.Errorf("Expected 0 call but got 1") case <-time.After(controllerTimeout): </del> <ins> if !isJobFinished(job) { t.Error("Job was expected to be finished") } job.Status.Conditions[0].Status = api.ConditionFalse if isJobFinished(job) { t.Error("Job was not expected to be finished") } job.Status.Conditions[0].Status = api.ConditionUnknown if isJobFinished(job) { t.Error("Job was not expected to be finished") </ins> } }

// Put one job and one pod into the store testJob := newJob(2, 2) manager.jobStore.Store.Add(testJob) <del> received := make(chan string) </del> <ins> received := make(chan struct{}) </ins> // The pod update sent through the fakeWatcher should figure out the managing job and // send it into the syncHandler. manager.syncHandler = func(key string) error {

testPod.Status.Phase = api.PodFailed fakeWatch.Add(&testPod) <del> select { case <-received: case <-time.After(controllerTimeout): t.Errorf("Expected 1 call but got 0") } </del> <ins> t.Log("Waiting for pod to reach syncHandler") <-received </ins> }