Change:
What happened: Running two pods that use the same iSCSI volume, one of them fails with: What you expected to happen: Both pods should be running Anything else we need to know?: It's caused by iscsi volume plugin persiting iSCSI settings into the mount directory itself (e.g. ): volume plugin creates the global mount directory. volume plugin creates there. volume plugin mounts the volume there. This shadows the file, but it's fine, as long as no other pod uses the same volume on the same node. creates a second pod that uses the volume. volume plugin tries to create in the global volume directory. Since the volume is mounted, it will either writes the file to the volume (!) or fails, if the volume is read-only.
/sig storage /assign /priority impotant-soon
The label(s) cannot be applied, because the repository doesn't have them <detailsIn response to : /sig storage /assign /priority impotant-soon Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/priority important-soon
Our e2e did not catch this, because they always create only one pod when testing read-only volumes.

Discussion:
klog.Errorf("iscsi: failed to mkdir %s, error", globalPDPath) return err } <ins> if b.volumeMode == v1.PersistentVolumeFilesystem { notMnt, err := b.mounter.IsLikelyNotMountPoint(globalPDPath) if err != nil { return err } if !notMnt { // The volume is already mounted, therefore the previous WaitForAttach must have // persisted the volume metadata. In addition, the metadata is actually *inside* // globalPDPath and we can't write it here, because it was shadowed by the volume // mount. klog.V(4).Infof("Skipping persistISCSI, the volume is already mounted at %s", globalPDPath) return nil } } </ins> // Persist iscsi disk config to json file for DetachDisk path return util.persistISCSIFile(*(b.iscsiDisk), globalPDPath) }

type Config struct { NS                  string PVCs                []*v1.PersistentVolumeClaim <ins> PVCsReadOnly        bool </ins> InlineVolumeSources []*v1.VolumeSource IsPrivileged        bool Command             string

volumeMounts = append(volumeMounts, v1.VolumeMount{Name: volumename, MountPath: "/mnt/" + volumename}) } <del> volumes[volumeIndex] = v1.Volume{Name: volumename, VolumeSource: v1.VolumeSource{PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ClaimName: pvclaim.Name, ReadOnly: false}}} </del> <ins> volumes[volumeIndex] = v1.Volume{Name: volumename, VolumeSource: v1.VolumeSource{PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ClaimName: pvclaim.Name, ReadOnly: podConfig.PVCsReadOnly}}} </ins> volumeIndex++ } for _, src := range podConfig.InlineVolumeSources {

// Test access to the volume from pods on different node TestConcurrentAccessToSingleVolume(l.config.Framework, l.cs, l.ns.Name, <del> l.config.ClientNodeSelection, resource.Pvc, numPods, true /* sameNode */) </del> <ins> l.config.ClientNodeSelection, resource.Pvc, numPods, true /* sameNode */, false /* readOnly */) }) // This tests below configuration: // [pod1] [pod2] // [   node1   ] //         /     <- same volume mode (read only) //   [volume1] ginkgo.It("should concurrently access the single read-only volume from pods on the same node", func() { init() defer cleanup() numPods := 2 if !l.driver.GetDriverInfo().Capabilities[CapMultiPODs] { e2eskipper.Skipf("Driver %q does not support multiple concurrent pods - skipping", dInfo.Name) } // Create volume testVolumeSizeRange := t.GetTestSuiteInfo().SupportedSizeRange resource := CreateVolumeResource(l.driver, l.config, pattern, testVolumeSizeRange) l.resources = append(l.resources, resource) // Initialize the volume with a filesystem - it's going to be mounted as read-only below. initializeVolume(l.cs, l.ns.Name, resource.Pvc, l.config.ClientNodeSelection) // Test access to the volume from pods on a single node TestConcurrentAccessToSingleVolume(l.config.Framework, l.cs, l.ns.Name, l.config.ClientNodeSelection, resource.Pvc, numPods, true /* sameNode */, true /* readOnly */) </ins> }) // This tests below configuration:

// Test access to the volume from pods on different node TestConcurrentAccessToSingleVolume(l.config.Framework, l.cs, l.ns.Name, <del> l.config.ClientNodeSelection, resource.Pvc, numPods, false /* sameNode */) </del> <ins> l.config.ClientNodeSelection, resource.Pvc, numPods, false /* sameNode */, false /* readOnly */) </ins> }) }

// pod deletion doesn't affect. Pods are deployed on the same node or different nodes depending on requiresSameNode. // Read/write check are done across pod, by check reading both what pod{n-1} and pod{n} wrote from pod{n}. func TestConcurrentAccessToSingleVolume(f *framework.Framework, cs clientset.Interface, ns string, <del> node e2epod.NodeSelection, pvc *v1.PersistentVolumeClaim, numPods int, requiresSameNode bool) { </del> <ins> node e2epod.NodeSelection, pvc *v1.PersistentVolumeClaim, numPods int, requiresSameNode bool, readOnly bool) { </ins> var pods []*v1.Pod

PVCs:          []*v1.PersistentVolumeClaim{pvc}, SeLinuxLabel:  e2epv.SELinuxLabel, NodeSelection: node, <ins> PVCsReadOnly:  readOnly, </ins> } pod, err := e2epod.CreateSecPodWithNodeSelection(cs, &podConfig, framework.PodStartTimeout) defer func() {

ginkgo.By(fmt.Sprintf("Checking if the volume in pod%d exists as expected volume mode (%s)", index, *pvc.Spec.VolumeMode)) utils.CheckVolumeModeOfPath(f, pod, *pvc.Spec.VolumeMode, path) <ins> if readOnly { ginkgo.By("Skipping volume content checks, volume is read-only") continue } </ins> if i != 0 { ginkgo.By(fmt.Sprintf("From pod%d, checking if reading the data that pod%d write works properly", index, index-1)) // For 1st pod, no one has written data yet, so pass the read check

ginkgo.By(fmt.Sprintf("Rechecking if the volume in pod%d exists as expected volume mode (%s)", index, *pvc.Spec.VolumeMode)) utils.CheckVolumeModeOfPath(f, pod, *pvc.Spec.VolumeMode, "/mnt/volume1") <ins> if readOnly { ginkgo.By("Skipping volume content checks, volume is read-only") continue } </ins> if i == 0 { // This time there should be data that last pod wrote, for 1st pod ginkgo.By(fmt.Sprintf("From pod%d, rechecking if reading the data that last pod write works properly", index))

return nil } <ins> // initializeVolume creates a filesystem on given volume, so it can be used as read-only later func initializeVolume(cs clientset.Interface, ns string, pvc *v1.PersistentVolumeClaim, node e2epod.NodeSelection) { if pvc.Spec.VolumeMode != nil && *pvc.Spec.VolumeMode == v1.PersistentVolumeBlock { // Block volumes do not need to be initialized. return } ginkgo.By(fmt.Sprintf("Initializing a filesystem on PVC %s", pvc.Name)) // Just create a pod with the volume as read-write. Kubernetes will create a filesystem there // if it does not exist yet. podConfig := e2epod.Config{ NS:            ns, PVCs:          []*v1.PersistentVolumeClaim{pvc}, SeLinuxLabel:  e2epv.SELinuxLabel, NodeSelection: node, } pod, err := e2epod.CreateSecPod(cs, &podConfig, framework.PodStartTimeout) defer func() { framework.ExpectNoError(e2epod.DeletePodWithWait(cs, pod)) }() framework.ExpectNoError(err) } </ins>