Change:
pull-kubernetes-unit sometimes fails with data race. TestDirty I think the test was 3 months ago with the flaking state. There are two issues. First, updating and reading a bool-type variable is not thread-safe. and there is also another problem checking the state before completely syncing it. No response /sig api-machinery
/assign
/triage accepted

Discussion:
"sort" "strconv" "strings" <ins> "sync" "sync/atomic" </ins> "testing" "time"

discoveryendpoint "k8s.io/apiserver/pkg/endpoints/discovery/aggregated" scheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/cache" <ins> "k8s.io/client-go/util/workqueue" </ins> apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" ) func newDiscoveryManager(rm discoveryendpoint.ResourceManager) *discoveryManager { <del> return NewDiscoveryManager(rm).(*discoveryManager) </del> <ins> dm := NewDiscoveryManager(rm).(*discoveryManager) dm.dirtyAPIServiceQueue = newCompleterWorkqueue(dm.dirtyAPIServiceQueue) return dm </ins> } <del> // Returns true if the queue of services to sync empty this means everything has // been reconciled and placed into merged document func waitForEmptyQueue(stopCh <-chan struct{}, dm *discoveryManager) bool { </del> <ins> // Returns true if the queue of services to sync is complete which means // everything has been reconciled and placed into merged document func waitForQueueComplete(stopCh <-chan struct{}, dm *discoveryManager) bool { </ins> return cache.WaitForCacheSync(stopCh, func() bool { <del> // Once items have successfully synced they are removed from queue. return dm.dirtyAPIServiceQueue.Len() == 0 </del> <ins> return dm.dirtyAPIServiceQueue.(*completerWorkqueue).isComplete() </ins> }) }

go aggregatedManager.Run(testCtx.Done()) <del> require.True(t, waitForEmptyQueue(testCtx.Done(), aggregatedManager)) </del> <ins> require.True(t, waitForQueueComplete(testCtx.Done(), aggregatedManager)) </ins> response, _, parsed := fetchPath(aggregatedResourceManager, "") if response.StatusCode != 200 {

// Test that a handler associated with an APIService gets pinged after the // APIService has been marked as dirty func TestDirty(t *testing.T) { <del> pinged := false </del> <ins> var pinged atomic.Bool service := discoveryendpoint.NewResourceManager() aggregatedResourceManager := discoveryendpoint.NewResourceManager() aggregatedManager := newDiscoveryManager(aggregatedResourceManager) aggregatedManager.AddAPIService(&apiregistrationv1.APIService{ ObjectMeta: metav1.ObjectMeta{ Name: "v1.stable.example.com", }, Spec: apiregistrationv1.APIServiceSpec{ Group:   "stable.example.com", Version: "v1", Service: &apiregistrationv1.ServiceReference{ Name: "test-service", }, }, }, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { pinged.Store(true) service.ServeHTTP(w, r) })) testCtx, cancel := context.WithCancel(context.Background()) defer cancel() go aggregatedManager.Run(testCtx.Done()) require.True(t, waitForQueueComplete(testCtx.Done(), aggregatedManager)) // immediately check for ping, since Run() should block for local services if !pinged.Load() { t.Errorf("service handler never pinged") } } // Shows that waitForQueueComplete also waits for syncing to // complete by artificially making the sync handler take a long time func TestWaitForSync(t *testing.T) { pinged := atomic.Bool{} </ins> service := discoveryendpoint.NewResourceManager() aggregatedResourceManager := discoveryendpoint.NewResourceManager()

}, }, }, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { <del> pinged = true </del> <ins> time.Sleep(3 * time.Second) pinged.Store(true) </ins> service.ServeHTTP(w, r) })) testCtx, cancel := context.WithCancel(context.Background()) defer cancel() go aggregatedManager.Run(testCtx.Done()) <del> require.True(t, waitForEmptyQueue(testCtx.Done(), aggregatedManager)) </del> <ins> require.True(t, waitForQueueComplete(testCtx.Done(), aggregatedManager)) </ins> // immediately check for ping, since Run() should block for local services <del> if !pinged { </del> <ins> if !pinged.Load() { </ins> t.Errorf("service handler never pinged") } }

aggregatedManager.RemoveAPIService(s.Name) } <del> require.True(t, waitForEmptyQueue(testCtx.Done(), aggregatedManager)) </del> <ins> require.True(t, waitForQueueComplete(testCtx.Done(), aggregatedManager)) </ins> response, _, parsed := fetchPath(aggyService, "") if response.StatusCode != 200 {

defer cancel() go aggregatedManager.Run(testCtx.Done()) <del> require.True(t, waitForEmptyQueue(testCtx.Done(), aggregatedManager)) </del> <ins> require.True(t, waitForQueueComplete(testCtx.Done(), aggregatedManager)) </ins> // At this point external services have synced. Check if discovery document // includes the legacy resources

// Important to wait here to ensure we prime the cache with the initial list // of documents in order to exercise 304 Not Modified <del> require.True(t, waitForEmptyQueue(testCtx.Done(), aggregatedManager)) </del> <ins> require.True(t, waitForQueueComplete(testCtx.Done(), aggregatedManager)) </ins> <del> // Now add all groups. We excluded one group before so that AllServicesSynced // could include it in this round. Now, if AllServicesSynced ever returns </del> <ins> // Now add all groups. We excluded one group before so that waitForQueueIsComplete // could include it in this round. Now, if waitForQueueIsComplete ever returns </ins> // true, it must have synced all the pre-existing groups before, which would // return 304 Not Modified for _, s := range apiServices {

} // This would wait the full timeout on 1.26.0. <del> require.True(t, waitForEmptyQueue(testCtx.Done(), aggregatedManager)) </del> <ins> require.True(t, waitForQueueComplete(testCtx.Done(), aggregatedManager)) </ins> } // copied from staging/src/k8s.io/apiserver/pkg/endpoints/discovery/v2/handler_test.go

return w.Result(), bytes, decoded } <ins> // completerWorkqueue is a workqueue.RateLimitingInterface that implements // isComplete type completerWorkqueue struct { lock sync.Mutex workqueue.RateLimitingInterface processing map[interface{}]struct{} } var _ = workqueue.RateLimitingInterface(&completerWorkqueue{}) func newCompleterWorkqueue(wq workqueue.RateLimitingInterface) *completerWorkqueue { return &completerWorkqueue{ RateLimitingInterface: wq, processing:            make(map[interface{}]struct{}), } } func (q *completerWorkqueue) Add(item interface{}) { q.lock.Lock() defer q.lock.Unlock() q.processing[item] = struct{}{} q.RateLimitingInterface.Add(item) } func (q *completerWorkqueue) AddAfter(item interface{}, duration time.Duration) { q.Add(item) } func (q *completerWorkqueue) AddRateLimited(item interface{}) { q.Add(item) } func (q *completerWorkqueue) Done(item interface{}) { q.lock.Lock() defer q.lock.Unlock() delete(q.processing, item) q.RateLimitingInterface.Done(item) } func (q *completerWorkqueue) isComplete() bool { q.lock.Lock() defer q.lock.Unlock() return q.Len() == 0 && len(q.processing) == 0 } </ins>