Change:
We noticed many goroutines stalled at in a large cluster (~200 nodes, ~1100 pods). It's because if the WatchServer timeout when the channel is full, will be blocked on the write to the even though is , because WatchServer stops consuming the channel after the timeout.
Great finding - thanks for debugging

Discussion:
go_test( name = "go_default_test", srcs = [ <ins> "cacher_whitebox_test.go", </ins> "selection_predicate_test.go", "time_budget_test.go", "util_test.go",

sync.Mutex input   chan watchCacheEvent result  chan watch.Event <ins> done    chan struct{} </ins> filter  watchFilterFunc stopped bool forget  func(bool)

watcher := &cacheWatcher{ input:   make(chan watchCacheEvent, chanSize), result:  make(chan watch.Event, chanSize), <ins> done:    make(chan struct{}), </ins> filter:  filter, stopped: false, forget:  forget,

defer c.Unlock() if !c.stopped { c.stopped = true <ins> close(c.done) </ins> close(c.input) } }

glog.Errorf("unexpected copy error: %v", err) return } <ins> var watchEvent watch.Event </ins> switch { case curObjPasses && !oldObjPasses: <del> c.result <- watch.Event{Type: watch.Added, Object: object} </del> <ins> watchEvent = watch.Event{Type: watch.Added, Object: object} </ins> case curObjPasses && oldObjPasses: <del> c.result <- watch.Event{Type: watch.Modified, Object: object} </del> <ins> watchEvent = watch.Event{Type: watch.Modified, Object: object} </ins> case !curObjPasses && oldObjPasses: <del> c.result <- watch.Event{Type: watch.Deleted, Object: object} </del> <ins> watchEvent = watch.Event{Type: watch.Deleted, Object: object} } select { case c.result <- watchEvent: // don't block on c.result if c.done is closed case <-c.done: </ins> } }

<ins> /* Copyright 2016 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package storage import ( "sync" "testing" "time" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/fields" "k8s.io/kubernetes/pkg/labels" "k8s.io/kubernetes/pkg/util/wait" ) // verifies the cacheWatcher.process goroutine is properly cleaned up even if // the writes to cacheWatcher.result channel is blocked. func TestCacheWatcherCleanupNotBlockedByResult(t *testing.T) { var lock sync.RWMutex count := 0 filter := func(string, labels.Set, fields.Set) bool { return true } forget := func(bool) { lock.Lock() defer lock.Unlock() count++ } initEvents := []watchCacheEvent{ {Object: &api.Pod{}}, {Object: &api.Pod{}}, } // set the size of the buffer of w.result to 0, so that the writes to // w.result is blocked. w := newCacheWatcher(0, 0, initEvents, filter, forget) w.Stop() if err := wait.PollImmediate(1*time.Second, 5*time.Second, func() (bool, error) { lock.RLock() defer lock.RUnlock() return count == 2, nil }); err != nil { t.Fatalf("expected forget() to be called twice, because sendWatchCacheEvent should not be blocked by the result channel: %v", err) } } </ins>