Change:
The following test failures started happening on 30th March: Kubernetes e2e suite.[sig-storage] In-tree Volumes [Driver: aws] [Testpattern: Inline-volume (default fs)] volumes should allow exec of files on the volume Kubernetes e2e suite.[sig-storage] In-tree Volumes [Driver: aws] [Testpattern: Inline-volume (default fs)] volumes should store data Kubernetes e2e suite.[sig-storage] In-tree Volumes [Driver: aws] [Testpattern: Inline-volume (ext4)] volumes should allow exec of files on the volume Kubernetes e2e suite.[sig-storage] In-tree Volumes [Driver: aws] [Testpattern: Inline-volume (ext4)] volumes should store data See This seems related to the changes in kubelet fails with the following: Tests passing See the e2e logs above No response <detailsLatest CI build </details<detailsAWS </details<details</details<details</details<details</details<detailsEBS CSI driver 1.5.0 </details
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/cc
/sig storage
/assign
not having debugging kops builds for awhile, are kubelet logs attached in prow? I don't see them...
Spoke with offline. It looks like we don't have kubelet logs captured in these tests. I need complete kubelet logs to make sense of these errors because testgrid shows inline volume tests as failing and code in question should not be called or inline volumes (because they can not be expanded). If that code fails, it would have been failing before too. I will wait for logs from and in the meanwhile try and see if I can reproduce these failures in single node AWS cluster setup that I can build manually.
thanks
Looks like this is happening for inline volumes if CSI migration is enabled for them. We construct a "fake" object on the fly for migration enabled plugins even though volume is inline, which is causing this condition to fail. It should be simple fix.
opened to fix the problem. The code change is just one line but has tests.

Discussion:
volumeSpec *volume.Spec, uniquePodName volumetypes.UniquePodName, mountedVolumesForPod map[volumetypes.UniquePodName]map[string]cache.MountedVolume) { <del> if podVolume.PersistentVolumeClaim == nil { </del> <ins> if podVolume.PersistentVolumeClaim == nil || pvc == nil { </ins> // Only PVC supports resize operation. return }

import ( "crypto/md5" "fmt" <ins> csitrans "k8s.io/csi-translation-lib" "k8s.io/kubernetes/pkg/volume/csimigration" </ins> "testing" "time"

} // Populates desiredStateOfWorld cache with one volume/pod. <ins> // Calls Run() // Verifies there is are attach/mount/etc calls and no detach/unmount calls. func Test_Run_Positive_VolumeAttachAndMountMigrationEnabled(t *testing.T) { // Arrange intreeToCSITranslator := csitrans.New() node := &v1.Node{ ObjectMeta: metav1.ObjectMeta{ Name: string(nodeName), }, Spec: v1.NodeSpec{}, Status: v1.NodeStatus{ VolumesAttached: []v1.AttachedVolume{ { Name:       v1.UniqueVolumeName(fmt.Sprintf("fake-plugin/%s", "pd.csi.storage.gke.io-fake-device1")), DevicePath: "fake/path", }, }, }, } volumePluginMgr, fakePlugin := volumetesting.GetTestKubeletVolumePluginMgrWithNode(t, node) dsw := cache.NewDesiredStateOfWorld(volumePluginMgr) asw := cache.NewActualStateOfWorld(nodeName, volumePluginMgr) kubeClient := createTestClient(v1.AttachedVolume{ Name:       v1.UniqueVolumeName(fmt.Sprintf("fake-plugin/%s", "pd.csi.storage.gke.io-fake-device1")), DevicePath: "fake/path", }) fakeRecorder := &record.FakeRecorder{} fakeHandler := volumetesting.NewBlockVolumePathHandler() oex := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator( kubeClient, volumePluginMgr, fakeRecorder, fakeHandler)) reconciler := NewReconciler( kubeClient, true, /* controllerAttachDetachEnabled */ reconcilerLoopSleepDuration, waitForAttachTimeout, nodeName, dsw, asw, hasAddedPods, oex, mount.NewFakeMounter(nil), hostutil.NewFakeHostUtil(nil), volumePluginMgr, kubeletPodsDir) pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "pod1", UID:  "pod1uid", }, Spec: v1.PodSpec{ Volumes: []v1.Volume{ { Name: "volume-name", VolumeSource: v1.VolumeSource{ GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{ PDName: "fake-device1", }, }, }, }, }, } volumeSpec := &volume.Spec{Volume: &pod.Spec.Volumes[0]} migratedSpec, err := csimigration.TranslateInTreeSpecToCSI(volumeSpec, pod.Namespace, intreeToCSITranslator) if err != nil { t.Fatalf("unexpected error while translating spec %v: %v", volumeSpec, err) } podName := util.GetUniquePodName(pod) generatedVolumeName, err := dsw.AddPodToVolume( podName, pod, migratedSpec, migratedSpec.Name(), "" /* volumeGidValue */) // Assert if err != nil { t.Fatalf("AddPodToVolume failed. Expected: <no error> Actual: <%v>", err) } dsw.MarkVolumesReportedInUse([]v1.UniqueVolumeName{generatedVolumeName}) // Act runReconciler(reconciler) waitForMount(t, fakePlugin, generatedVolumeName, asw) // Assert assert.NoError(t, volumetesting.VerifyWaitForAttachCallCount( 1 /* expectedWaitForAttachCallCount */, fakePlugin)) assert.NoError(t, volumetesting.VerifyMountDeviceCallCount( 1 /* expectedMountDeviceCallCount */, fakePlugin)) assert.NoError(t, volumetesting.VerifySetUpCallCount( 1 /* expectedSetUpCallCount */, fakePlugin)) assert.NoError(t, volumetesting.VerifyZeroTearDownCallCount(fakePlugin)) assert.NoError(t, volumetesting.VerifyZeroDetachCallCount(fakePlugin)) } // Populates desiredStateOfWorld cache with one volume/pod. </ins> // Enables controllerAttachDetachEnabled. // Calls Run() // Verifies there is one mount call and no unmount calls.

return wait.ExponentialBackoff(backoff, fn) } <del> func createTestClient() *fake.Clientset { </del> <ins> func createTestClient(attachedVolumes ...v1.AttachedVolume) *fake.Clientset { </ins> fakeClient := &fake.Clientset{} <ins> if len(attachedVolumes) == 0 { attachedVolumes = append(attachedVolumes, v1.AttachedVolume{ Name:       "fake-plugin/fake-device1", DevicePath: "fake/path", }) } </ins> fakeClient.AddReactor("get", "nodes", func(action core.Action) (bool, runtime.Object, error) { return true, &v1.Node{ ObjectMeta: metav1.ObjectMeta{Name: string(nodeName)}, Status: v1.NodeStatus{ <del> VolumesAttached: []v1.AttachedVolume{ { Name:       "fake-plugin/fake-device1", DevicePath: "/fake/path", }, }}, </del> <ins> VolumesAttached: attachedVolumes, }, </ins> }, nil <del> }) </del> <ins> }, ) </ins> fakeClient.AddReactor("*", "*", func(action core.Action) (bool, runtime.Object, error) { return true, nil, fmt.Errorf("no reaction implemented for %s", action) })

claimSize := actualStateOfWorld.GetClaimSize(volumeToMount.VolumeName) // only fetch claimSize if it was not set previously <del> if volumeToMount.VolumeSpec.PersistentVolume != nil && claimSize == nil { </del> <ins> if volumeToMount.VolumeSpec.PersistentVolume != nil && claimSize == nil && !volumeToMount.VolumeSpec.InlineVolumeSpecForCSIMigration { </ins> pv := volumeToMount.VolumeSpec.PersistentVolume pvc, err := og.kubeClient.CoreV1().PersistentVolumeClaims(pv.Spec.ClaimRef.Namespace).Get(context.TODO(), pv.Spec.ClaimRef.Name, metav1.GetOptions{}) if err != nil {