Repository: kubernetes/kubernetes
File: staging/src/k8s.io/client-go/tools/cache/the_real_fifo.go (Go)

Code before:
	// Queue depth never goes high because processing an item is locking the queue,
	// and new items can't be added until processing finish.
	// https://github.com/kubernetes/kubernetes/issues/103789
	if len(f.items) > 10 {
		id, _ := f.keyOf(item)
		trace := utiltrace.New("RealFIFO Pop Process",
			utiltrace.Field{Key: "ID", Value: id},
			utiltrace.Field{Key: "Depth", Value: len(f.items)},
			utiltrace.Field{Key: "Reason", Value: "slow event handlers blocking the queue"})
		defer trace.LogIfLong(100 * time.Millisecond)
	}

	// Process the item, this may unlock the lock, and allow other goroutines to add items to the queue.
	err := f.whileProcessing_locked(func() error {
		// we wrap in Deltas here to be compatible with preview Pop functions and those interpreting the return value.
		return process(Deltas{item}, isInInitialList)
	})
	return Deltas{item}, err
}

// whileProcessing_locked processes the given function while unlocking the lock if we are able to, namely if we are emitting atomic events.
func (f *RealFIFO) whileProcessing_locked(process func() error) error {
	// If we are emitting atomic events and we have the , we can optimize the processing by unlocking the lock before calling process so that the
	// reflector is not blocked while we process the events. This is because we never access the store in the FIFO, and the store will
	// stay consistent while we process the events.
	if f.unlockWhileProcessing {
		f.lock.Unlock()
		defer f.lock.Lock()
	}
	return process()
}

// batchable stores the delta types that can be batched
var batchable = map[DeltaType]bool{
	Sync:     true,
	Replaced: true,
	Added:    true,
	Updated:  true,
	Deleted:  true,
}

// PopBatch pops as many items as possible to be processed as a batch using processBatch,
// or pop a single item using processSingle if multiple items cannot be batched.
func (f *RealFIFO) PopBatch(processBatch ProcessBatchFunc, processSingle PopProcessFunc) error {
	f.lock.Lock()
	defer f.lock.Unlock()

	for len(f.items) == 0 {
		// When the queue is empty, invocation of Pop() is blocked until new item is enqueued.
		// When Close() is called, the f.closed is set and the condition is broadcasted.
		// Which causes this loop to continue and return from the Pop().

Review comment (performance):
Doing this to allow a burst of events to queue even while we're processing seems useful

If incoming events are outrunning our processing in steady-state, this will let `f.items` grow to the point of OOMing, which seems bad.

Should we also make this conditional on the length of f.items, so a burst could be absorbed, but otherwise we keep new stuff from queuing? I don't really know how to pick a safe/reasonable burst size for this. Shared informers have an initial buffer size of 1024. Batch event processing combines up to 1000 items. Would relaxing if `f.items` is under ~1000ish items might be a decent spot to start?

Queue-depth metrics as in https://github.com/kubernetes/kubernetes/pull/135782 would help a particular operator understand their normal distributions, but that is still likely to be unique to particular clusters and types.

Code after:
	// This is done in a defer so we only do this *after* processing is complete,
	// so concurrent calls to hasSynced will not incorrectly return true while processing is still happening.
	defer func() {
		if f.initialPopulationCount > 0 {
			f.initialPopulationCount--
		}
	}()

	// Only log traces if the queue depth is greater than 10 and it takes more than
	// 100 milliseconds to process one item from the queue.
	// Queue depth never goes high because processing an item is locking the queue,
	// and new items can't be added until processing finish.
	// https://github.com/kubernetes/kubernetes/issues/103789
	if len(f.items) > 10 {
		id, _ := f.keyOf(item)
		trace := utiltrace.New("RealFIFO Pop Process",
			utiltrace.Field{Key: "ID", Value: id},
			utiltrace.Field{Key: "Depth", Value: len(f.items)},
			utiltrace.Field{Key: "Reason", Value: "slow event handlers blocking the queue"})
		defer trace.LogIfLong(100 * time.Millisecond)
	}

	// Process the item, this may unlock the lock, and allow other goroutines to add items to the queue.
	err := f.whileProcessing_locked(func() error {
		// we wrap in Deltas here to be compatible with preview Pop functions and those interpreting the return value.
		return process(Deltas{item}, isInInitialList)
	})
	return Deltas{item}, err
}

// whileProcessing_locked calls the `process` function.
// The lock must be held before calling `whileProcessing_locked`, and is held when `whileProcessing_locked` returns.
// whileProcessing_locked releases the lock during the call to `process` if f.unlockWhileProcessing is true and the f.items queue is not too long.
func (f *RealFIFO) whileProcessing_locked(process func() error) error {
	// Unlock before calling `process` so new items can be enqueued during processing.
	// Only do this if the queue contains less than 2 full batches of items,
	// to prevent the queue from growing unboundedly.
	if f.unlockWhileProcessing && len(f.items) < f.batchSize*2 {
		f.lock.Unlock()
		defer f.lock.Lock()
	}
	return process()
}

// batchable stores the delta types that can be batched
var batchable = map[DeltaType]bool{
	Sync:     true,
	Replaced: true,
	Added:    true,
	Updated:  true,
	Deleted:  true,