Change:
PV controller appears to be processing same object multiple times for same PVC delete event. The reason is mainly because when a PVC is deleted and PV is synced(either because of an event or because of resync), then each update to PV object causes another event which gets processed until PV object is removed from informer/store and really marked as deleted. This results in multiple delete requests being sent to storage provider for same volume. For example - In above gist you can see that volume deletion is being done more than once for pv - /sig storage cc
It looks like this is to some extent by design because events are pulled from workqueue by one goroutine and then do some proccessing and then put into another goroutine via nestedpendingoperations thingy. The problem is, this is inherently "racy" because first goroutine might be pulling events that were triggered by second goroutine and hence causing processing of events that are still in-progress. I take we can't use workqueues with a fixed number of workers because that would kinda limit number of PVs we can provision concurrently, but can we put the item pulled from queue directly into nested_pendingoperations so as we don't have this weird race condition?

Discussion:
} } <ins> func withVolumeDeletionTimestamp(pvs []*v1.PersistentVolume) []*v1.PersistentVolume { result := []*v1.PersistentVolume{} for _, pv := range pvs { // Using time.Now() here will cause mismatching deletion timestamps in tests deleteTime := metav1.Date(2020, time.February, 18, 10, 30, 30, 10, time.UTC) pv.SetDeletionTimestamp(&deleteTime) result = append(result, pv) } return result } </ins> // newClaim returns a new claim with given attributes func newClaim(name, claimUID, capacity, boundToVolume string, phase v1.PersistentVolumeClaimPhase, class *string, annotations ...string) *v1.PersistentVolumeClaim { fs := v1.PersistentVolumeFilesystem

klog.V(3).Infof("error reading persistent volume %q: %v", volume.Name, err) return "", nil } <ins> if newVolume.GetDeletionTimestamp() != nil { klog.V(3).Infof("Volume %q is already being deleted", volume.Name) return "", nil } </ins> needsReclaim, err := ctrl.isVolumeReleased(newVolume) if err != nil { klog.V(3).Infof("error reading claim for volume %q: %v", volume.Name, err)

return nil }, }, <ins> { // delete success(?) - volume has deletion timestamp before doDelete() starts "8-13 - volume is has deletion timestamp and is not processed", withVolumeDeletionTimestamp(newVolumeArray("volume8-13", "1Gi", "uid8-13", "claim8-13", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty)), withVolumeDeletionTimestamp(newVolumeArray("volume8-13", "1Gi", "uid8-13", "claim8-13", v1.VolumeReleased, v1.PersistentVolumeReclaimDelete, classEmpty)), noclaims, noclaims, noevents, noerrors, // We don't need to do anything in test function because deletion will be noticed automatically and synced. // Attempting to use testSyncVolume here will cause an error because of race condition between manually // calling testSyncVolume and volume loop running. func(ctrl *PersistentVolumeController, reactor *pvtesting.VolumeReactor, test controllerTest) error { return nil }, }, </ins> } for _, test := range tests {