Change:
It happens during the "delete" phase in the load test. ! In perf-dash we can see that in total there are ~20M requests like that during the load test. ! It started in Sep 2019 (sic!), so probably when we enabled PVs in our load tests. We haven't detected it so far as, strangely, it doesn't seem to be impacting our scalability SLIs. We should investigate this. /sig scalability
I checked and we create 35 PersistentVolumeClaims per 100 nodes. So in case of 5K node test there will be 1750 PVCs in the cluster. This is close to the 2K of GET PVC requests we see.
Let's analyze the last run of 5K node test in more depths - Grafana: ! As I wrote already somewhere here, it happens during the delete phase. We reach the peak of ~2K QPS around 2:30 AM UTC in that run. Let's check the logs (api-server, kubelets) around that time.
Looking at the api-server logs one can easily tell that something is wrong. It's swamped by GET persistentvolumeclaim requests:
Restricting the logs to a single stateful set on a single node (single client ip) - it also doesn't look right. Total such requests from this IP address: So it looks like a single kubelet issued almost 20K GET requests for this PVC within ~75min, which gives us ~4 QPS from a single kubelet...
From the scheduler logs it looks like the statefulset pod for this PVC was bound on the node Let's check the kubelet logs there
Kubelet logs: Bingo, around the time when the issue happens, it's full of the following logs:
Looks like the errors disappeared exactly when the pod was deleted So, what happens here is partially caused by our test as it may happen that we'll first delete PVC and won't delete the corresponding pod for the next O(1h). Nevertheless, the code in the should be fixed so it doesn't cause kubelet to send ~4QPS of GET PVC requests in such case.
/sig node /sig storage
There is a pending discussion about ways to fix it in
Thanks for fixing that! I know how much work was it and I really appreciate!

Discussion:
type DesiredStateOfWorldPopulator interface { Run(sourcesReady config.SourcesReady, stopCh <-chan struct{}) <del> // ReprocessPod removes the specified pod from the list of processedPods // (if it exists) forcing it to be reprocessed. This is required to enable </del> <ins> // ReprocessPod sets value for the specified pod in processedPods // to false, forcing it to be reprocessed. This is required to enable </ins> // remounting volumes on pod updates (volumes like Downward API volumes // depend on this behavior to ensure volume content is updated). ReprocessPod(podName volumetypes.UniquePodName)

func (dswp *desiredStateOfWorldPopulator) ReprocessPod( podName volumetypes.UniquePodName) { <del> dswp.deleteProcessedPod(podName) </del> <ins> dswp.markPodProcessingFailed(podName) </ins> } func (dswp *desiredStateOfWorldPopulator) HasAddedPods() bool {

dswp.actualStateOfWorld.MarkRemountRequired(uniquePodName) // Remove any stored errors for the pod, everything went well in this processPodVolumes dswp.desiredStateOfWorld.PopPodErrors(uniquePodName) <ins> } else if dswp.podHasBeenSeenOnce(uniquePodName) { // For the Pod which has been processed at least once, even though some volumes // may not have been reprocessed successfully this round, we still mark it as processed to avoid // processing it at a very high frequency. The pod will be reprocessed when volume manager calls // ReprocessPod() which is triggered by SyncPod. dswp.markPodProcessed(uniquePodName) </ins> } }

} // podPreviouslyProcessed returns true if the volumes for this pod have already <del> // been processed by the populator </del> <ins> // been processed/reprocessed by the populator. Otherwise, the volumes for this pod need to // be reprocessed. </ins> func (dswp *desiredStateOfWorldPopulator) podPreviouslyProcessed( podName volumetypes.UniquePodName) bool { dswp.pods.RLock() defer dswp.pods.RUnlock() <del> _, exists := dswp.pods.processedPods[podName] return exists </del> <ins> return dswp.pods.processedPods[podName] } // markPodProcessingFailed marks the specified pod from processedPods as false to indicate that it failed processing func (dswp *desiredStateOfWorldPopulator) markPodProcessingFailed( podName volumetypes.UniquePodName) { dswp.pods.Lock() dswp.pods.processedPods[podName] = false dswp.pods.Unlock() } // podHasBeenSeenOnce returns true if the pod has been seen by the popoulator // at least once. func (dswp *desiredStateOfWorldPopulator) podHasBeenSeenOnce( podName volumetypes.UniquePodName) bool { dswp.pods.RLock() _, exist := dswp.pods.processedPods[podName] dswp.pods.RUnlock() return exist </ins> } // markPodProcessed records that the volumes for the specified pod have been

"k8s.io/kubernetes/pkg/volume/util/types" ) <ins> func pluginPVOmittingClient(dswp *desiredStateOfWorldPopulator) { fakeClient := &fake.Clientset{} fakeClient.AddReactor("get", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) { return false, nil, nil }) fakeClient.AddReactor("get", "persistentvolumes", func(action core.Action) (bool, runtime.Object, error) { return false, nil, nil }) dswp.kubeClient = fakeClient } func prepareDswpWithVolume(t *testing.T) (*desiredStateOfWorldPopulator, kubepod.Manager) { // create dswp mode := v1.PersistentVolumeFilesystem pv := &v1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "dswp-test-volume-name", }, Spec: v1.PersistentVolumeSpec{ ClaimRef:   &v1.ObjectReference{Namespace: "ns", Name: "file-bound"}, VolumeMode: &mode, }, } pvc := &v1.PersistentVolumeClaim{ Spec: v1.PersistentVolumeClaimSpec{ VolumeName: "dswp-test-volume-name", }, Status: v1.PersistentVolumeClaimStatus{ Phase: v1.ClaimBound, }, } dswp, fakePodManager, _ := createDswpWithVolume(t, pv, pvc) return dswp, fakePodManager } func TestFindAndAddNewPods_WithReprocessPodAndVolumeRetrievalError(t *testing.T) { // create dswp dswp, fakePodManager := prepareDswpWithVolume(t) // create pod containers := []v1.Container{ { VolumeMounts: []v1.VolumeMount{ { Name:      "dswp-test-volume-name", MountPath: "/mnt", }, }, }, } pod := createPodWithVolume("dswp-test-pod", "dswp-test-volume-name", "file-bound", containers) fakePodManager.AddPod(pod) podName := util.GetUniquePodName(pod) dswp.findAndAddNewPods() if !dswp.podPreviouslyProcessed(podName) { t.Fatalf("Failed to record that the volumes for the specified pod: %s have been processed by the populator", podName) } pluginPVOmittingClient(dswp) dswp.ReprocessPod(podName) dswp.findAndAddNewPods() if !dswp.podPreviouslyProcessed(podName) { t.Fatalf("Failed to record that the volumes for the specified pod: %s have been processed by the populator", podName) } fakePodManager.DeletePod(pod) } func TestFindAndAddNewPods_WithVolumeRetrievalError(t *testing.T) { // create dswp dswp, fakePodManager := prepareDswpWithVolume(t) pluginPVOmittingClient(dswp) // create pod containers := []v1.Container{ { VolumeMounts: []v1.VolumeMount{ { Name:      "dswp-test-volume-name", MountPath: "/mnt", }, }, }, } pod := createPodWithVolume("dswp-test-pod", "dswp-test-volume-name", "file-bound", containers) fakePodManager.AddPod(pod) podName := util.GetUniquePodName(pod) dswp.findAndAddNewPods() if dswp.podPreviouslyProcessed(podName) { t.Fatalf("The volumes for the specified pod: %s should not have been processed by the populator", podName) } if dswp.podHasBeenSeenOnce(podName) { t.Fatalf("The volumes for the specified pod: %s should not have been processed by the populator", podName) } } </ins> func TestFindAndAddNewPods_FindAndRemoveDeletedPods(t *testing.T) { // create dswp mode := v1.PersistentVolumeFilesystem