Change:
"k8s.io/kubernetes/pkg/volume/util/types" ) <ins> func pluginPVOmittingClient(dswp *desiredStateOfWorldPopulator) { fakeClient := &fake.Clientset{} fakeClient.AddReactor("get", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) { return false, nil, nil }) fakeClient.AddReactor("get", "persistentvolumes", func(action core.Action) (bool, runtime.Object, error) { return false, nil, nil }) dswp.kubeClient = fakeClient } func prepareDswpWithVolume(t *testing.T) (*desiredStateOfWorldPopulator, kubepod.Manager) { // create dswp mode := v1.PersistentVolumeFilesystem pv := &v1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "dswp-test-volume-name", }, Spec: v1.PersistentVolumeSpec{ ClaimRef:   &v1.ObjectReference{Namespace: "ns", Name: "file-bound"}, VolumeMode: &mode, }, } pvc := &v1.PersistentVolumeClaim{ Spec: v1.PersistentVolumeClaimSpec{ VolumeName: "dswp-test-volume-name", }, Status: v1.PersistentVolumeClaimStatus{ Phase: v1.ClaimBound, }, } dswp, fakePodManager, _ := createDswpWithVolume(t, pv, pvc) return dswp, fakePodManager } func TestFindAndAddNewPods_WithReprocessPodAndVolumeRetrievalError(t *testing.T) { // create dswp dswp, fakePodManager := prepareDswpWithVolume(t) // create pod containers := []v1.Container{ { VolumeMounts: []v1.VolumeMount{ { Name:      "dswp-test-volume-name", MountPath: "/mnt", }, }, }, } pod := createPodWithVolume("dswp-test-pod", "dswp-test-volume-name", "file-bound", containers) fakePodManager.AddPod(pod) podName := util.GetUniquePodName(pod) dswp.findAndAddNewPods() if !dswp.podPreviouslyProcessed(podName) { t.Fatalf("Failed to record that the volumes for the specified pod: %s have been processed by the populator", podName) } pluginPVOmittingClient(dswp) dswp.ReprocessPod(podName) dswp.findAndAddNewPods() if !dswp.podPreviouslyProcessed(podName) { t.Fatalf("Failed to record that the volumes for the specified pod: %s have been processed by the populator", podName) } fakePodManager.DeletePod(pod) } func TestFindAndAddNewPods_WithVolumeRetrievalError(t *testing.T) { // create dswp dswp, fakePodManager := prepareDswpWithVolume(t) pluginPVOmittingClient(dswp) // create pod containers := []v1.Container{ { VolumeMounts: []v1.VolumeMount{ { Name:      "dswp-test-volume-name", MountPath: "/mnt", }, }, }, } pod := createPodWithVolume("dswp-test-pod", "dswp-test-volume-name", "file-bound", containers) fakePodManager.AddPod(pod) podName := util.GetUniquePodName(pod) dswp.findAndAddNewPods() if dswp.podPreviouslyProcessed(podName) { t.Fatalf("The volumes for the specified pod: %s should not have been processed by the populator", podName) } if dswp.podHasBeenSeenOnce(podName) { t.Fatalf("The volumes for the specified pod: %s should not have been processed by the populator", podName) } } </ins> func TestFindAndAddNewPods_FindAndRemoveDeletedPods(t *testing.T) { // create dswp mode := v1.PersistentVolumeFilesystem

Discussion:
It happens during the "delete" phase in the load test. ! In perf-dash we can see that in total there are ~20M requests like that during the load test. ! It started in Sep 2019 (sic!), so probably when we enabled PVs in our load tests. We haven't detected it so far as, strangely, it doesn't seem to be impacting our scalability SLIs. We should investigate this. /sig scalability
I checked and we create 35 PersistentVolumeClaims per 100 nodes. So in case of 5K node test there will be 1750 PVCs in the cluster. This is close to the 2K of GET PVC requests we see.
Let's analyze the last run of 5K node test in more depths - Grafana: ! As I wrote already somewhere here, it happens during the delete phase. We reach the peak of ~2K QPS around 2:30 AM UTC in that run. Let's check the logs (api-server, kubelets) around that time.
Looking at the api-server logs one can easily tell that something is wrong. It's swamped by GET persistentvolumeclaim requests:
Restricting the logs to a single stateful set on a single node (single client ip) - it also doesn't look right. Total such requests from this IP address: So it looks like a single kubelet issued almost 20K GET requests for this PVC within ~75min, which gives us ~4 QPS from a single kubelet...
From the scheduler logs it looks like the statefulset pod for this PVC was bound on the node Let's check the kubelet logs there
Kubelet logs: Bingo, around the time when the issue happens, it's full of the following logs:
Looks like the errors disappeared exactly when the pod was deleted So, what happens here is partially caused by our test as it may happen that we'll first delete PVC and won't delete the corresponding pod for the next O(1h). Nevertheless, the code in the should be fixed so it doesn't cause kubelet to send ~4QPS of GET PVC requests in such case.
/sig node /sig storage
There is a pending discussion about ways to fix it in
Thanks for fixing that! I know how much work was it and I really appreciate!