Change:
<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks!--What happened: In this test, volume mount failed because the driver-registrar took a minute to come up. However, it seems like volume mount operation is retrying without any sort of backoff. This message appears in 400+ times in less than a minute. What you expected to happen: Operation executor should not be retrying operations so frequently. <!-- DO NOT EDIT BELOW THIS LINE --/kind bug
I'll take a look at this. /assign
Problem is that reconciler depends on operation executor to do backoff. In the normal case operation executor returns a method that does the actual work. Any errors by that method result in exponential backoff. But in this case, the error is coming from the code creating the mount method which is not handled by the back off logic, and is called again and again by the reconciler resulting in log and event spam. This didn't use to be a problem because initialization errors were uncommon (plugins were built in), but with CSI the driver may not exist and these initialization errors are expected. I'll look in to cleaning this up.
Is this something that will affect users in 1.13 and needs CPing?
This will result in kubelet log spam and possibly kubernetes event spam. It happens when someone creates a pod referencing a CSI volume before the CSI driver for that volume is installed. Which happens frequently in E2E tests because they install/remove driver with every test. But should be rare in real world usage since most users are likely to install a driver first before using it.
If release team is worried about this, we could create a quick PR to suppress logging/event generation of this these errors. Otherwise we plan to address this more holistically for 1.14 and 1.13.1.

Discussion:
if fsVolume { // Filesystem volume case // Mount/remount a volume when a volume is attached <del> generatedOperations, err = oe.operationGenerator.GenerateMountVolumeFunc( </del> <ins> generatedOperations = oe.operationGenerator.GenerateMountVolumeFunc( </ins> waitForAttachTimeout, volumeToMount, actualStateOfWorld, isRemount) } else {

} } <del> func (fopg *fakeOperationGenerator) GenerateMountVolumeFunc(waitForAttachTimeout time.Duration, volumeToMount VolumeToMount, actualStateOfWorldMounterUpdater ActualStateOfWorldMounterUpdater, isRemount bool) (volumetypes.GeneratedOperations, error) { </del> <ins> func (fopg *fakeOperationGenerator) GenerateMountVolumeFunc(waitForAttachTimeout time.Duration, volumeToMount VolumeToMount, actualStateOfWorldMounterUpdater ActualStateOfWorldMounterUpdater, isRemount bool) volumetypes.GeneratedOperations { </ins> opFunc := func() (error, error) { startOperationAndBlock(fopg.ch, fopg.quit) return nil, nil } return volumetypes.GeneratedOperations{ OperationFunc: opFunc, <del> }, nil </del> <ins> } </ins> } func (fopg *fakeOperationGenerator) GenerateUnmountVolumeFunc(volumeToUnmount MountedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater, podsDir string) (volumetypes.GeneratedOperations, error) { opFunc := func() (error, error) {

"k8s.io/kubernetes/pkg/volume/util/volumepathhandler" ) <ins> const ( unknownVolumePlugin string = "UnknownVolumePlugin" ) </ins> var _ OperationGenerator = &operationGenerator{} type operationGenerator struct {

// OperationGenerator interface that extracts out the functions from operation_executor to make it dependency injectable type OperationGenerator interface { // Generates the MountVolume function needed to perform the mount of a volume plugin <del> GenerateMountVolumeFunc(waitForAttachTimeout time.Duration, volumeToMount VolumeToMount, actualStateOfWorldMounterUpdater ActualStateOfWorldMounterUpdater, isRemount bool) (volumetypes.GeneratedOperations, error) </del> <ins> GenerateMountVolumeFunc(waitForAttachTimeout time.Duration, volumeToMount VolumeToMount, actualStateOfWorldMounterUpdater ActualStateOfWorldMounterUpdater, isRemount bool) volumetypes.GeneratedOperations </ins> // Generates the UnmountVolume function needed to perform the unmount of a volume plugin GenerateUnmountVolumeFunc(volumeToUnmount MountedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater, podsDir string) (volumetypes.GeneratedOperations, error)

waitForAttachTimeout time.Duration, volumeToMount VolumeToMount, actualStateOfWorld ActualStateOfWorldMounterUpdater, <del> isRemount bool) (volumetypes.GeneratedOperations, error) { </del> <ins> isRemount bool) volumetypes.GeneratedOperations { </ins> // Get mounter plugin <ins> volumePluginName := unknownVolumePlugin </ins> volumePlugin, err := og.volumePluginMgr.FindPluginBySpec(volumeToMount.VolumeSpec) <del> if err != nil || volumePlugin == nil { return volumetypes.GeneratedOperations{}, volumeToMount.GenerateErrorDetailed("MountVolume.FindPluginBySpec failed", err) </del> <ins> if err == nil && volumePlugin != nil { volumePluginName = volumePlugin.GetPluginName() </ins> } <del> affinityErr := checkNodeAffinity(og, volumeToMount, volumePlugin) if affinityErr != nil { eventErr, detailedErr := volumeToMount.GenerateError("MountVolume.NodeAffinity check failed", affinityErr) og.recorder.Eventf(volumeToMount.Pod, v1.EventTypeWarning, kevents.FailedMountVolume, eventErr.Error()) return volumetypes.GeneratedOperations{}, detailedErr } </del> <ins> mountVolumeFunc := func() (error, error) { if err != nil || volumePlugin == nil { return volumeToMount.GenerateError("MountVolume.FindPluginBySpec failed", err) } </ins> <del> volumeMounter, newMounterErr := volumePlugin.NewMounter( volumeToMount.VolumeSpec, volumeToMount.Pod, volume.VolumeOptions{}) if newMounterErr != nil { eventErr, detailedErr := volumeToMount.GenerateError("MountVolume.NewMounter initialization failed", newMounterErr) og.recorder.Eventf(volumeToMount.Pod, v1.EventTypeWarning, kevents.FailedMountVolume, eventErr.Error()) return volumetypes.GeneratedOperations{}, detailedErr } </del> <ins> affinityErr := checkNodeAffinity(og, volumeToMount, volumePlugin) if affinityErr != nil { return volumeToMount.GenerateError("MountVolume.NodeAffinity check failed", affinityErr) } </ins> <del> mountCheckError := checkMountOptionSupport(og, volumeToMount, volumePlugin) </del> <ins> volumeMounter, newMounterErr := volumePlugin.NewMounter( volumeToMount.VolumeSpec, volumeToMount.Pod, volume.VolumeOptions{}) if newMounterErr != nil { return volumeToMount.GenerateError("MountVolume.NewMounter initialization failed", newMounterErr) </ins> <del> if mountCheckError != nil { eventErr, detailedErr := volumeToMount.GenerateError("MountVolume.MountOptionSupport check failed", mountCheckError) og.recorder.Eventf(volumeToMount.Pod, v1.EventTypeWarning, kevents.UnsupportedMountOption, eventErr.Error()) return volumetypes.GeneratedOperations{}, detailedErr } </del> <ins> } </ins> <del> // Get attacher, if possible attachableVolumePlugin, _ := og.volumePluginMgr.FindAttachablePluginBySpec(volumeToMount.VolumeSpec) var volumeAttacher volume.Attacher if attachableVolumePlugin != nil { volumeAttacher, _ = attachableVolumePlugin.NewAttacher() } </del> <ins> mountCheckError := checkMountOptionSupport(og, volumeToMount, volumePlugin) </ins> <del> // get deviceMounter, if possible deviceMountableVolumePlugin, _ := og.volumePluginMgr.FindDeviceMountablePluginBySpec(volumeToMount.VolumeSpec) var volumeDeviceMounter volume.DeviceMounter if deviceMountableVolumePlugin != nil { volumeDeviceMounter, _ = deviceMountableVolumePlugin.NewDeviceMounter() } </del> <ins> if mountCheckError != nil { return volumeToMount.GenerateError("MountVolume.MountOptionSupport check failed", mountCheckError) } </ins> <del> var fsGroup *int64 if volumeToMount.Pod.Spec.SecurityContext != nil && volumeToMount.Pod.Spec.SecurityContext.FSGroup != nil { fsGroup = volumeToMount.Pod.Spec.SecurityContext.FSGroup } </del> <ins> // Get attacher, if possible attachableVolumePlugin, _ := og.volumePluginMgr.FindAttachablePluginBySpec(volumeToMount.VolumeSpec) var volumeAttacher volume.Attacher if attachableVolumePlugin != nil { volumeAttacher, _ = attachableVolumePlugin.NewAttacher() } // get deviceMounter, if possible deviceMountableVolumePlugin, _ := og.volumePluginMgr.FindDeviceMountablePluginBySpec(volumeToMount.VolumeSpec) var volumeDeviceMounter volume.DeviceMounter if deviceMountableVolumePlugin != nil { volumeDeviceMounter, _ = deviceMountableVolumePlugin.NewDeviceMounter() } var fsGroup *int64 if volumeToMount.Pod.Spec.SecurityContext != nil && volumeToMount.Pod.Spec.SecurityContext.FSGroup != nil { fsGroup = volumeToMount.Pod.Spec.SecurityContext.FSGroup } </ins> <del> mountVolumeFunc := func() (error, error) { </del> devicePath := volumeToMount.DevicePath if volumeAttacher != nil { // Wait for attachable volumes to finish attaching

// resizeFileSystem will resize the file system if user has requested a resize of // underlying persistent volume and is allowed to do so. <del> resizeSimpleError, resizeDetailedError := og.resizeFileSystem(volumeToMount, devicePath, deviceMountPath, volumePlugin.GetPluginName()) </del> <ins> resizeSimpleError, resizeDetailedError := og.resizeFileSystem(volumeToMount, devicePath, deviceMountPath, volumePluginName) </ins> if resizeSimpleError != nil || resizeDetailedError != nil { return resizeSimpleError, resizeDetailedError

return volumetypes.GeneratedOperations{ OperationFunc:     mountVolumeFunc, EventRecorderFunc: eventRecorderFunc, <del> CompleteFunc:      util.OperationCompleteHook(util.GetFullQualifiedPluginNameForVolume(volumePlugin.GetPluginName(), volumeToMount.VolumeSpec), "volume_mount"), }, nil </del> <ins> CompleteFunc:      util.OperationCompleteHook(util.GetFullQualifiedPluginNameForVolume(volumePluginName, volumeToMount.VolumeSpec), "volume_mount"), } </ins> } func (og *operationGenerator) resizeFileSystem(volumeToMount VolumeToMount, devicePath, deviceMountPath, pluginName string) (simpleErr, detailedErr error) {