Repository: ceph/ceph-csi
Pull request #2674 (2021-11-23 08:09:03)

ci: have mergify label PRs for `actions/` with `ci/skip/e2e`
The matching checks the list of modified files, and if it does not
contain anything outside the `actions/` directory, the labels are
applied.

Actions are not tested with e2e, so changes can be labelled to skip
these time and resource intensive tests.

---

<details>
<summary>Show available bot commands</summary>

These commands are normally not required, but in case of issues, leave any of
the following bot commands in an otherwise empty comment in this PR:

- `/retest ci/centos/<job-name>`: retest the `<job-name>` after unrelated
  failure (please report the failure too!)
- `/retest all`: run this in case the CentOS CI failed to start/report any test
  progress or results

</details>


nixpanic: This PR needs manually merging, but there is no need to hurry. Check the [Mergify dashboard](https://dashboard.mergify.com/github/ceph/repo/ceph-csi/queues) to make sure Mergify does not have anything queued, and prevent needing to re-test other PRs.

Patch:
diff --git a/.mergify.yml b/.mergify.yml
index 5c70c1580..bc6d4d833 100644
--- a/.mergify.yml
+++ b/.mergify.yml
@@ -310,9 +310,18 @@ pull_request_rules:
           - ci/skip/e2e
   - name: title contains Mergify
     conditions:
-      - "title~=mergify"
+      - "title~=(?i)mergify"
     actions:
       label:
         add:
           - Repo activity
           - ci/skip/e2e
+  - name: label PR that only changes actions
+    conditions:
+      - -files~=^(!?actions/)
+    actions:
+      label:
+        add:
+          - component/build
+          - component/testing
+          - ci/skip/e2e
diff --git a/charts/ceph-csi-rbd/values.yaml b/charts/ceph-csi-rbd/values.yaml
index 2bf97d554..740ff12db 100644
--- a/charts/ceph-csi-rbd/values.yaml
+++ b/charts/ceph-csi-rbd/values.yaml
@@ -362,6 +362,11 @@ storageClass:
   # https://docs.ceph.com/docs/master/man/8/rbd/#kernel-rbd-krbd-options
   # For nbd options refer
   # https://docs.ceph.com/docs/master/man/8/rbd-nbd/#options
+  # Format:
+  # mapOptions: "<mounter>:op1,op2;<mounter>:op1,op2"
+  # An empty mounter field is treated as krbd type for compatibility.
+  # eg:
+  # mapOptions: "krbd:lock_on_read,queue_depth=1024;nbd:try-netlink"
   mapOptions: ""
 
   # (optional) unmapOptions is a comma-separated list of unmap options.
@@ -369,6 +374,11 @@ storageClass:
   # https://docs.ceph.com/docs/master/man/8/rbd/#kernel-rbd-krbd-options
   # For nbd options refer
   # https://docs.ceph.com/docs/master/man/8/rbd-nbd/#options
+  # Format:
+  # unmapOptions: "<mounter>:op1,op2;<mounter>:op1,op2"
+  # An empty mounter field is treated as krbd type for compatibility.
+  # eg:
+  # unmapOptions: "krbd:force;nbd:force"
   unmapOptions: ""
 
   # The secrets have to contain Ceph credentials with required access
diff --git a/examples/rbd/storageclass.yaml b/examples/rbd/storageclass.yaml
index baba539b2..65ce181ab 100644
--- a/examples/rbd/storageclass.yaml
+++ b/examples/rbd/storageclass.yaml
@@ -53,14 +53,22 @@ parameters:
    # https://docs.ceph.com/docs/master/man/8/rbd/#kernel-rbd-krbd-options
    # For nbd options refer
    # https://docs.ceph.com/docs/master/man/8/rbd-nbd/#options
-   # mapOptions: lock_on_read,queue_depth=1024
+   # Format:
+   # mapOptions: "<mounter>:op1,op2;<mounter>:op1,op2"
+   # An empty mounter field is treated as krbd type for compatibility.
+   # eg:
+   # mapOptions: "krbd:lock_on_read,queue_depth=1024;nbd:try-netlink"
 
    # (optional) unmapOptions is a comma-separated list of unmap options.
    # For krbd options refer
    # https://docs.ceph.com/docs/master/man/8/rbd/#kernel-rbd-krbd-options
    # For nbd options refer
    # https://docs.ceph.com/docs/master/man/8/rbd-nbd/#options
-   # unmapOptions: force
+   # Format:
+   # unmapOptions: "<mounter>:op1,op2;<mounter>:op1,op2"
+   # An empty mounter field is treated as krbd type for compatibility.
+   # eg:
+   # unmapOptions: "krbd:force;nbd:force"
 
    # The secrets have to contain Ceph credentials with required access
    # to the 'pool'.
diff --git a/internal/rbd/nodeserver.go b/internal/rbd/nodeserver.go
index 0425278f7..8242751cd 100644
--- a/internal/rbd/nodeserver.go
+++ b/internal/rbd/nodeserver.go
@@ -224,12 +224,14 @@ func populateRbdVol(
 			return nil, status.Errorf(codes.Internal, "unsupported krbd Feature")
 		}
 		// fallback to rbd-nbd,
-		// ignore the mapOptions and unmapOptions as they are meant for krbd use.
 		rv.Mounter = rbdNbdMounter
 	} else {
 		rv.Mounter = req.GetVolumeContext()["mounter"]
-		rv.MapOptions = req.GetVolumeContext()["mapOptions"]
-		rv.UnmapOptions = req.GetVolumeContext()["unmapOptions"]
+	}
+
+	err = getMapOptions(req, rv)
+	if err != nil {
+		return nil, err
 	}
 
 	rv.VolID = volID
diff --git a/internal/rbd/rbd_attach.go b/internal/rbd/rbd_attach.go
index 3008d68f7..c9ffb6899 100644
--- a/internal/rbd/rbd_attach.go
+++ b/internal/rbd/rbd_attach.go
@@ -29,6 +29,7 @@ import (
 	"github.com/ceph/ceph-csi/internal/util"
 	"github.com/ceph/ceph-csi/internal/util/log"
 
+	"github.com/container-storage-interface/spec/lib/go/csi"
 	"k8s.io/apimachinery/pkg/util/wait"
 )
 
@@ -222,6 +223,65 @@ func setRbdNbdToolFeatures() {
 	log.DefaultLog("NBD module loaded: %t, rbd-nbd supported features, cookie: %t", hasNBD, hasNBDCookieSupport)
 }
 
+// parseMapOptions helps parse formatted mapOptions and unmapOptions and
+// returns mounter specific options.
+func parseMapOptions(mapOptions string) (string, string, error) {
+	var krbdMapOptions, nbdMapOptions string
+	const (
+		noKeyLength = 1
+		validLength = 2
+	)
+	for _, item := range strings.Split(mapOptions, ";") {
+		var mounter, options string
+		if item == "" {
+			continue
+		}
+		s := strings.Split(item, ":")
+		switch len(s) {
+		case noKeyLength:
+			options = strings.TrimSpace(s[0])
+			krbdMapOptions = options
+		case validLength:
+			mounter = strings.TrimSpace(s[0])
+			options = strings.TrimSpace(s[1])
+			switch strings.ToLower(mounter) {
+			case accessTypeKRbd:
+				krbdMapOptions = options
+			case accessTypeNbd:
+				nbdMapOptions = options
+			default:
+				return "", "", fmt.Errorf("unknown mounter type: %q", mounter)
+			}
+		default:
+			return "", "", fmt.Errorf("badly formatted map/unmap options: %q", mapOptions)
+		}
+	}
+
+	return krbdMapOptions, nbdMapOptions, nil
+}
+
+// getMapOptions is a wrapper func, calls parse map/unmap funcs and feeds the
+// rbdVolume object.
+func getMapOptions(req *csi.NodeStageVolumeRequest, rv *rbdVolume) error {
+	krbdMapOptions, nbdMapOptions, err := parseMapOptions(req.GetVolumeContext()["mapOptions"])
+	if err != nil {
+		return err
+	}
+	krbdUnmapOptions, nbdUnmapOptions, err := parseMapOptions(req.GetVolumeContext()["unmapOptions"])
+	if err != nil {
+		return err
+	}
+	if rv.Mounter == rbdDefaultMounter {
+		rv.MapOptions = krbdMapOptions
+		rv.UnmapOptions = krbdUnmapOptions
+	} else if rv.Mounter == rbdNbdMounter {
+		rv.MapOptions = nbdMapOptions
+		rv.UnmapOptions = nbdUnmapOptions
+	}
+
+	return nil
+}
+
 func attachRBDImage(ctx context.Context, volOptions *rbdVolume, device string, cr *util.Credentials) (string, error) {
 	var err error
 
diff --git a/internal/rbd/replicationcontrollerserver.go b/internal/rbd/replicationcontrollerserver.go
index f1729e2d1..7d6659abc 100644
--- a/internal/rbd/replicationcontrollerserver.go
+++ b/internal/rbd/replicationcontrollerserver.go
@@ -290,9 +290,9 @@ func createDummyImage(ctx context.Context, rbdVol *rbdVolume) error {
 		if err != nil {
 			return err
 		}
-		dummyVol := rbdVol
+		dummyVol := *rbdVol
 		dummyVol.RbdImageName = imgName
-		err = createImage(ctx, dummyVol, dummyVol.conn.Creds)
+		err = createImage(ctx, &dummyVol, dummyVol.conn.Creds)
 		if err != nil && !strings.Contains(err.Error(), "File exists") {
 			return err
 		}
@@ -310,7 +310,7 @@ func tickleMirroringOnDummyImage(rbdVol *rbdVolume, mirroringMode librbd.ImageMi
 	if err != nil {
 		return err
 	}
-	dummyVol := rbdVol
+	dummyVol := *rbdVol
 	dummyVol.RbdImageName = imgName
 
 	dummyImageOpsLock.Lock()
@@ -523,12 +523,6 @@ func (rs *ReplicationServer) PromoteVolume(ctx context.Context,
 		return nil, status.Errorf(codes.Internal, "failed to get mirroring mode %s", err.Error())
 	}
 
-	log.DebugLog(ctx, "Attempting to tickle dummy image for restarting RBD schedules")
-	err = tickleMirroringOnDummyImage(rbdVol, mode)
-	if err != nil {
-		return nil, status.Errorf(codes.Internal, "failed to enable mirroring on dummy image %s", err.Error())
-	}
-
 	interval, startTime := getSchedulingDetails(req.GetParameters())
 	if interval != admin.NoInterval {
 		err = rbdVol.addSnapshotScheduling(interval, startTime)
@@ -543,6 +537,12 @@ func (rs *ReplicationServer) PromoteVolume(ctx context.Context,
 			rbdVol)
 	}
 
+	log.DebugLog(ctx, "attempting to tickle dummy image for restarting RBD schedules")
+	err = tickleMirroringOnDummyImage(rbdVol, mode)
+	if err != nil {
+		return nil, status.Errorf(codes.Internal, "failed to enable mirroring on dummy image %s", err.Error())
+	}
+
 	return &replication.PromoteVolumeResponse{}, nil
 }
 


Test patch:
diff --git a/e2e/cephfs.go b/e2e/cephfs.go
index dbbc1d2e4..b340969da 100644
--- a/e2e/cephfs.go
+++ b/e2e/cephfs.go
@@ -282,7 +282,7 @@ var _ = Describe("cephfs", func() {
 			appEphemeralPath := cephFSExamplePath + "pod-ephemeral.yaml"
 
 			By("checking provisioner deployment is running", func() {
-				err := waitForDeploymentComplete(cephFSDeploymentName, cephCSINamespace, f.ClientSet, deployTimeout)
+				err := waitForDeploymentComplete(f.ClientSet, cephFSDeploymentName, cephCSINamespace, deployTimeout)
 				if err != nil {
 					e2elog.Failf("timeout waiting for deployment %s: %v", cephFSDeploymentName, err)
 				}
diff --git a/e2e/deployment.go b/e2e/deployment.go
new file mode 100644
index 000000000..b70bf1904
--- /dev/null
+++ b/e2e/deployment.go
@@ -0,0 +1,172 @@
+/*
+Copyright 2021 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	appsv1 "k8s.io/api/apps/v1"
+	apierrs "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/kubernetes"
+	deploymentutil "k8s.io/kubernetes/pkg/controller/deployment/util"
+	"k8s.io/kubernetes/test/e2e/framework"
+	e2elog "k8s.io/kubernetes/test/e2e/framework/log"
+)
+
+// execCommandInPodWithName run command in pod using podName.
+func execCommandInPodWithName(
+	f *framework.Framework,
+	cmdString,
+	podName,
+	containerName,
+	nameSpace string) (string, string, error) {
+	cmd := []string{"/bin/sh", "-c", cmdString}
+	podOpt := framework.ExecOptions{
+		Command:            cmd,
+		PodName:            podName,
+		Namespace:          nameSpace,
+		ContainerName:      containerName,
+		Stdin:              nil,
+		CaptureStdout:      true,
+		CaptureStderr:      true,
+		PreserveWhitespace: true,
+	}
+
+	return f.ExecWithOptions(podOpt)
+}
+
+// loadAppDeployment loads the deployment app config and return deployment
+// object.
+func loadAppDeployment(path string) (*appsv1.Deployment, error) {
+	deploy := appsv1.Deployment{}
+	if err := unmarshal(path, &deploy); err != nil {
+		return nil, err
+	}
+
+	return &deploy, nil
+}
+
+// createDeploymentApp creates the deployment object and waits for it to be in
+// Available state.
+func createDeploymentApp(clientSet kubernetes.Interface, app *appsv1.Deployment, deployTimeout int) error {
+	_, err := clientSet.AppsV1().Deployments(app.Namespace).Create(context.TODO(), app, metav1.CreateOptions{})
+	if err != nil {
+		return fmt.Errorf("failed to create deploy: %w", err)
+	}
+
+	return waitForDeploymentInAvailableState(clientSet, app.Name, app.Namespace, deployTimeout)
+}
+
+// deleteDeploymentApp deletes the deployment object.
+func deleteDeploymentApp(clientSet kubernetes.Interface, name, ns string, deployTimeout int) error {
+	timeout := time.Duration(deployTimeout) * time.Minute
+	err := clientSet.AppsV1().Deployments(ns).Delete(context.TODO(), name, metav1.DeleteOptions{})
+	if err != nil {
+		return fmt.Errorf("failed to delete deployment: %w", err)
+	}
+	start := time.Now()
+	e2elog.Logf("Waiting for deployment %q to be deleted", name)
+
+	return wait.PollImmediate(poll, timeout, func() (bool, error) {
+		_, err := clientSet.AppsV1().Deployments(ns).Get(context.TODO(), name, metav1.GetOptions{})
+		if err != nil {
+			if isRetryableAPIError(err) {
+				return false, nil
+			}
+			if apierrs.IsNotFound(err) {
+				return true, nil
+			}
+			e2elog.Logf("%q deployment to be deleted (%d seconds elapsed)", name, int(time.Since(start).Seconds()))
+
+			return false, fmt.Errorf("failed to get deployment: %w", err)
+		}
+
+		return false, nil
+	})
+}
+
+// waitForDeploymentInAvailableState wait for deployment to be in Available state.
+func waitForDeploymentInAvailableState(clientSet kubernetes.Interface, name, ns string, deployTimeout int) error {
+	timeout := time.Duration(deployTimeout) * time.Minute
+	start := time.Now()
+	e2elog.Logf("Waiting up to %q to be in Available state", name)
+
+	return wait.PollImmediate(poll, timeout, func() (bool, error) {
+		d, err := clientSet.AppsV1().Deployments(ns).Get(context.TODO(), name, metav1.GetOptions{})
+		if err != nil {
+			if isRetryableAPIError(err) {
+				return false, nil
+			}
+			e2elog.Logf("%q deployment to be Available (%d seconds elapsed)", name, int(time.Since(start).Seconds()))
+
+			return false, err
+		}
+		cond := deploymentutil.GetDeploymentCondition(d.Status, appsv1.DeploymentAvailable)
+
+		return cond != nil, nil
+	})
+}
+
+// Waits for the deployment to complete.
+func waitForDeploymentComplete(clientSet kubernetes.Interface, name, ns string, deployTimeout int) error {
+	var (
+		deployment *appsv1.Deployment
+		reason     string
+		err        error
+	)
+	timeout := time.Duration(deployTimeout) * time.Minute
+	err = wait.PollImmediate(poll, timeout, func() (bool, error) {
+		deployment, err = clientSet.AppsV1().Deployments(ns).Get(context.TODO(), name, metav1.GetOptions{})
+		if err != nil {
+			if isRetryableAPIError(err) {
+				return false, nil
+			}
+			e2elog.Logf("deployment error: %v", err)
+
+			return false, err
+		}
+
+		// TODO need to check rolling update
+
+		// When the deployment status and its underlying resources reach the
+		// desired state, we're done
+		if deployment.Status.Replicas == deployment.Status.ReadyReplicas {
+			return true, nil
+		}
+		e2elog.Logf(
+			"deployment status: expected replica count %d running replica count %d",
+			deployment.Status.Replicas,
+			deployment.Status.ReadyReplicas)
+		reason = fmt.Sprintf("deployment status: %#v", deployment.Status.String())
+
+		return false, nil
+	})
+
+	if errors.Is(err, wait.ErrWaitTimeout) {
+		err = fmt.Errorf("%s", reason)
+	}
+	if err != nil {
+		return fmt.Errorf("error waiting for deployment %q status to match desired state: %w", name, err)
+	}
+
+	return nil
+}
diff --git a/e2e/pod.go b/e2e/pod.go
index a8242bd6a..9f0127d09 100644
--- a/e2e/pod.go
+++ b/e2e/pod.go
@@ -7,7 +7,6 @@ import (
 	"strings"
 	"time"
 
-	appsv1 "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
 	apierrs "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -73,52 +72,6 @@ func waitForDaemonSets(name, ns string, c kubernetes.Interface, t int) error {
 	})
 }
 
-// Waits for the deployment to complete.
-
-func waitForDeploymentComplete(name, ns string, c kubernetes.Interface, t int) error {
-	var (
-		deployment *appsv1.Deployment
-		reason     string
-		err        error
-	)
-	timeout := time.Duration(t) * time.Minute
-	err = wait.PollImmediate(poll, timeout, func() (bool, error) {
-		deployment, err = c.AppsV1().Deployments(ns).Get(context.TODO(), name, metav1.GetOptions{})
-		if err != nil {
-			if isRetryableAPIError(err) {
-				return false, nil
-			}
-			e2elog.Logf("deployment error: %v", err)
-
-			return false, err
-		}
-
-		// TODO need to check rolling update
-
-		// When the deployment status and its underlying resources reach the
-		// desired state, we're done
-		if deployment.Status.Replicas == deployment.Status.ReadyReplicas {
-			return true, nil
-		}
-		e2elog.Logf(
-			"deployment status: expected replica count %d running replica count %d",
-			deployment.Status.Replicas,
-			deployment.Status.ReadyReplicas)
-		reason = fmt.Sprintf("deployment status: %#v", deployment.Status.String())
-
-		return false, nil
-	})
-
-	if errors.Is(err, wait.ErrWaitTimeout) {
-		err = fmt.Errorf("%s", reason)
-	}
-	if err != nil {
-		return fmt.Errorf("error waiting for deployment %q status to match expectation: %w", name, err)
-	}
-
-	return nil
-}
-
 func findPodAndContainerName(f *framework.Framework, ns, cn string, opt *metav1.ListOptions) (string, string, error) {
 	podList, err := f.PodClientNS(ns).List(context.TODO(), *opt)
 	if err != nil {
diff --git a/e2e/rbd.go b/e2e/rbd.go
index c381325ab..a7cc6d4f5 100644
--- a/e2e/rbd.go
+++ b/e2e/rbd.go
@@ -33,6 +33,7 @@ var (
 	rbdDirPath         = "../deploy/rbd/kubernetes/"
 	examplePath        = "../examples/"
 	rbdExamplePath     = examplePath + "/rbd/"
+	e2eTemplatesPath   = "../e2e/templates/"
 	rbdDeploymentName  = "csi-rbdplugin-provisioner"
 	rbdDaemonsetName   = "csi-rbdplugin"
 	defaultRBDPool     = "replicapool"
@@ -59,9 +60,11 @@ var (
 	appBlockSmartClonePath = rbdExamplePath + "block-pod-clone.yaml"
 	appEphemeralPath       = rbdExamplePath + "pod-ephemeral.yaml"
 	snapshotPath           = rbdExamplePath + "snapshot.yaml"
+	deployFSAppPath        = e2eTemplatesPath + "rbd-fs-deployment.yaml"
+	deployBlockAppPath     = e2eTemplatesPath + "rbd-block-deployment.yaml"
 	defaultCloneCount      = 10
 
-	nbdMapOptions             = "debug-rbd=20"
+	nbdMapOptions             = "nbd:debug-rbd=20"
 	e2eDefaultCephLogStrategy = "preserve"
 )
 
@@ -254,7 +257,7 @@ var _ = Describe("RBD", func() {
 		deployVault(f.ClientSet, deployTimeout)
 
 		// wait for provisioner deployment
-		err = waitForDeploymentComplete(rbdDeploymentName, cephCSINamespace, f.ClientSet, deployTimeout)
+		err = waitForDeploymentComplete(f.ClientSet, rbdDeploymentName, cephCSINamespace, deployTimeout)
 		if err != nil {
 			e2elog.Failf("timeout waiting for deployment %s: %v", rbdDeploymentName, err)
 		}
@@ -271,7 +274,7 @@ var _ = Describe("RBD", func() {
 		}
 		// default io-timeout=0, needs kernel >= 5.4
 		if !util.CheckKernelSupport(kernelRelease, nbdZeroIOtimeoutSupport) {
-			nbdMapOptions = "debug-rbd=20,io-timeout=330"
+			nbdMapOptions = "nbd:debug-rbd=20,io-timeout=330"
 		}
 	})
 
@@ -576,6 +579,337 @@ var _ = Describe("RBD", func() {
 				}
 			})
 
+			// NOTE: RWX is restricted for FileSystem VolumeMode at ceph-csi,
+			// see pull#261 for more details.
+			By("Create RWX+Block Mode PVC and bind to multiple pods via deployment using rbd-nbd mounter", func() {
+				err := deleteResource(rbdExamplePath + "storageclass.yaml")
+				if err != nil {
+					e2elog.Failf("failed to delete storageclass: %v", err)
+				}
+				// Storage class with rbd-nbd mounter
+				err = createRBDStorageClass(
+					f.ClientSet,
+					f,
+					defaultSCName,
+					nil,
+					map[string]string{
+						"mounter":         "rbd-nbd",
+						"mapOptions":      nbdMapOptions,
+						"cephLogStrategy": e2eDefaultCephLogStrategy,
+					},
+					deletePolicy)
+				if err != nil {
+					e2elog.Failf("failed to create storageclass: %v", err)
+				}
+				pvc, err := loadPVC(rawPvcPath)
+				if err != nil {
+					e2elog.Failf("failed to load PVC: %v", err)
+				}
+				pvc.Namespace = f.UniqueName
+				pvc.Spec.AccessModes = []v1.PersistentVolumeAccessMode{v1.ReadWriteMany}
+
+				app, err := loadAppDeployment(deployBlockAppPath)
+				if err != nil {
+					e2elog.Failf("failed to load application deployment: %v", err)
+				}
+				app.Namespace = f.UniqueName
+
+				err = createPVCAndDeploymentApp(f, "", pvc, app, deployTimeout)
+				if err != nil {
+					e2elog.Failf("failed to create PVC and application: %v", err)
+				}
+
+				err = waitForDeploymentComplete(f.ClientSet, app.Name, app.Namespace, deployTimeout)
+				if err != nil {
+					e2elog.Failf("timeout waiting for deployment to be in running state: %v", err)
+				}
+
+				devPath := app.Spec.Template.Spec.Containers[0].VolumeDevices[0].DevicePath
+				cmd := fmt.Sprintf("dd if=/dev/zero of=%s bs=1M count=10", devPath)
+
+				opt := metav1.ListOptions{
+					LabelSelector: fmt.Sprintf("app=%s", app.Name),
+				}
+				podList, err := f.PodClientNS(app.Namespace).List(context.TODO(), opt)
+				if err != nil {
+					e2elog.Failf("get pod list failed: %v", err)
+				}
+				if len(podList.Items) != int(*app.Spec.Replicas) {
+					e2elog.Failf("podlist contains %d items, expected %d items", len(podList.Items), *app.Spec.Replicas)
+				}
+				for _, pod := range podList.Items {
+					_, _, err = execCommandInPodWithName(f, cmd, pod.Name, pod.Spec.Containers[0].Name, app.Namespace)
+					if err != nil {
+						e2elog.Failf("command %q failed: %v", cmd, err)
+					}
+				}
+
+				err = deletePVCAndDeploymentApp(f, "", pvc, app)
+				if err != nil {
+					e2elog.Failf("failed to delete PVC and application: %v", err)
+				}
+				// validate created backend rbd images
+				validateRBDImageCount(f, 0, defaultRBDPool)
+				// validate images in trash
+				err = waitToRemoveImagesFromTrash(f, defaultRBDPool, deployTimeout)
+				if err != nil {
+					e2elog.Failf("failed to validate rbd images in pool %s trash: %v", rbdOptions(defaultRBDPool), err)
+				}
+				err = deleteResource(rbdExamplePath + "storageclass.yaml")
+				if err != nil {
+					e2elog.Failf("failed to delete storageclass: %v", err)
+				}
+				err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, nil, deletePolicy)
+				if err != nil {
+					e2elog.Failf("failed to create storageclass: %v", err)
+				}
+			})
+
+			By("Create ROX+FS Mode PVC and bind to multiple pods via deployment using rbd-nbd mounter", func() {
+				err := deleteResource(rbdExamplePath + "storageclass.yaml")
+				if err != nil {
+					e2elog.Failf("failed to delete storageclass: %v", err)
+				}
+				// Storage class with rbd-nbd mounter
+				err = createRBDStorageClass(
+					f.ClientSet,
+					f,
+					defaultSCName,
+					nil,
+					map[string]string{
+						"mounter":         "rbd-nbd",
+						"mapOptions":      nbdMapOptions,
+						"cephLogStrategy": e2eDefaultCephLogStrategy,
+					},
+					deletePolicy)
+				if err != nil {
+					e2elog.Failf("failed to create storageclass: %v", err)
+				}
+
+				// create PVC and bind it to an app
+				pvc, err := loadPVC(pvcPath)
+				if err != nil {
+					e2elog.Failf("failed to load PVC: %v", err)
+				}
+				pvc.Namespace = f.UniqueName
+				app, err := loadApp(appPath)
+				if err != nil {
+					e2elog.Failf("failed to load application: %v", err)
+				}
+				app.Namespace = f.UniqueName
+				err = createPVCAndApp("", f, pvc, app, deployTimeout)
+				if err != nil {
+					e2elog.Failf("failed to create PVC and application: %v", err)
+				}
+				// validate created backend rbd images
+				validateRBDImageCount(f, 1, defaultRBDPool)
+				err = deletePod(app.Name, app.Namespace, f.ClientSet, deployTimeout)
+				if err != nil {
+					e2elog.Failf("failed to delete application: %v", err)
+				}
+
+				// create clone PVC as ROX
+				pvcClone, err := loadPVC(pvcSmartClonePath)
+				if err != nil {
+					e2elog.Failf("failed to load PVC: %v", err)
+				}
+				pvcClone.Spec.DataSource.Name = pvc.Name
+				pvcClone.Namespace = f.UniqueName
+				pvcClone.Spec.AccessModes = []v1.PersistentVolumeAccessMode{v1.ReadOnlyMany}
+				appClone, err := loadAppDeployment(deployFSAppPath)
+				if err != nil {
+					e2elog.Failf("failed to load application deployment: %v", err)
+				}
+				appClone.Namespace = f.UniqueName
+				appClone.Spec.Template.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvcClone.Name
+				appClone.Spec.Template.Spec.Volumes[0].PersistentVolumeClaim.ReadOnly = true
+				err = createPVCAndDeploymentApp(f, "", pvcClone, appClone, deployTimeout)
+				if err != nil {
+					e2elog.Failf("failed to create PVC and application: %v", err)
+				}
+
+				err = waitForDeploymentComplete(f.ClientSet, appClone.Name, appClone.Namespace, deployTimeout)
+				if err != nil {
+					e2elog.Failf("timeout waiting for deployment to be in running state: %v", err)
+				}
+
+				// validate created backend rbd images
+				validateRBDImageCount(f, 3, defaultRBDPool)
+
+				filePath := appClone.Spec.Template.Spec.Containers[0].VolumeMounts[0].MountPath + "/test"
+				cmd := fmt.Sprintf("echo 'Hello World' > %s", filePath)
+
+				opt := metav1.ListOptions{
+					LabelSelector: fmt.Sprintf("app=%s", appClone.Name),
+				}
+				podList, err := f.PodClientNS(appClone.Namespace).List(context.TODO(), opt)
+				if err != nil {
+					e2elog.Failf("get pod list failed: %v", err)
+				}
+				if len(podList.Items) != int(*appClone.Spec.Replicas) {
+					e2elog.Failf("podlist contains %d items, expected %d items", len(podList.Items), *appClone.Spec.Replicas)
+				}
+				for _, pod := range podList.Items {
+					var stdErr string
+					_, stdErr, err = execCommandInPodWithName(f, cmd, pod.Name, pod.Spec.Containers[0].Name, appClone.Namespace)
+					if err != nil {
+						e2elog.Logf("command %q failed: %v", cmd, err)
+					}
+					readOnlyErr := fmt.Sprintf("cannot create %s: Read-only file system", filePath)
+					if !strings.Contains(stdErr, readOnlyErr) {
+						e2elog.Failf(stdErr)
+					}
+				}
+
+				err = deletePVCAndDeploymentApp(f, "", pvcClone, appClone)
+				if err != nil {
+					e2elog.Failf("failed to delete PVC and application: %v", err)
+				}
+				// delete parent pvc
+				err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout)
+				if err != nil {
+					e2elog.Failf("failed to delete PVC: %v", err)
+				}
+				// validate created backend rbd images
+				validateRBDImageCount(f, 0, defaultRBDPool)
+				// validate images in trash
+				err = waitToRemoveImagesFromTrash(f, defaultRBDPool, deployTimeout)
+				if err != nil {
+					e2elog.Failf("failed to validate rbd images in pool %s trash: %v", rbdOptions(defaultRBDPool), err)
+				}
+				err = deleteResource(rbdExamplePath + "storageclass.yaml")
+				if err != nil {
+					e2elog.Failf("failed to delete storageclass: %v", err)
+				}
+				err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, nil, deletePolicy)
+				if err != nil {
+					e2elog.Failf("failed to create storageclass: %v", err)
+				}
+			})
+
+			By("Create ROX+Block Mode PVC and bind to multiple pods via deployment using rbd-nbd mounter", func() {
+				err := deleteResource(rbdExamplePath + "storageclass.yaml")
+				if err != nil {
+					e2elog.Failf("failed to delete storageclass: %v", err)
+				}
+				// Storage class with rbd-nbd mounter
+				err = createRBDStorageClass(
+					f.ClientSet,
+					f,
+					defaultSCName,
+					nil,
+					map[string]string{
+						"mounter":         "rbd-nbd",
+						"mapOptions":      nbdMapOptions,
+						"cephLogStrategy": e2eDefaultCephLogStrategy,
+					},
+					deletePolicy)
+				if err != nil {
+					e2elog.Failf("failed to create storageclass: %v", err)
+				}
+
+				// create PVC and bind it to an app
+				pvc, err := loadPVC(rawPvcPath)
+				if err != nil {
+					e2elog.Failf("failed to load PVC: %v", err)
+				}
+				pvc.Namespace = f.UniqueName
+				app, err := loadApp(rawAppPath)
+				if err != nil {
+					e2elog.Failf("failed to load application: %v", err)
+				}
+				app.Namespace = f.UniqueName
+				err = createPVCAndApp("", f, pvc, app, deployTimeout)
+				if err != nil {
+					e2elog.Failf("failed to create PVC and application: %v", err)
+				}
+				// validate created backend rbd images
+				validateRBDImageCount(f, 1, defaultRBDPool)
+				err = deletePod(app.Name, app.Namespace, f.ClientSet, deployTimeout)
+				if err != nil {
+					e2elog.Failf("failed to delete application: %v", err)
+				}
+
+				// create clone PVC as ROX
+				pvcClone, err := loadPVC(pvcBlockSmartClonePath)
+				if err != nil {
+					e2elog.Failf("failed to load PVC: %v", err)
+				}
+				pvcClone.Spec.DataSource.Name = pvc.Name
+				pvcClone.Namespace = f.UniqueName
+				pvcClone.Spec.AccessModes = []v1.PersistentVolumeAccessMode{v1.ReadOnlyMany}
+				volumeMode := v1.PersistentVolumeBlock
+				pvcClone.Spec.VolumeMode = &volumeMode
+				appClone, err := loadAppDeployment(deployBlockAppPath)
+				if err != nil {
+					e2elog.Failf("failed to load application deployment: %v", err)
+				}
+				appClone.Namespace = f.UniqueName
+				appClone.Spec.Template.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvcClone.Name
+				appClone.Spec.Template.Spec.Volumes[0].PersistentVolumeClaim.ReadOnly = true
+				err = createPVCAndDeploymentApp(f, "", pvcClone, appClone, deployTimeout)
+				if err != nil {
+					e2elog.Failf("failed to create PVC and application: %v", err)
+				}
+
+				err = waitForDeploymentComplete(f.ClientSet, appClone.Name, appClone.Namespace, deployTimeout)
+				if err != nil {
+					e2elog.Failf("timeout waiting for deployment to be in running state: %v", err)
+				}
+
+				// validate created backend rbd images
+				validateRBDImageCount(f, 3, defaultRBDPool)
+
+				devPath := appClone.Spec.Template.Spec.Containers[0].VolumeDevices[0].DevicePath
+				cmd := fmt.Sprintf("dd if=/dev/zero of=%s bs=1M count=10", devPath)
+
+				opt := metav1.ListOptions{
+					LabelSelector: fmt.Sprintf("app=%s", appClone.Name),
+				}
+				podList, err := f.PodClientNS(appClone.Namespace).List(context.TODO(), opt)
+				if err != nil {
+					e2elog.Failf("get pod list failed: %v", err)
+				}
+				if len(podList.Items) != int(*appClone.Spec.Replicas) {
+					e2elog.Failf("podlist contains %d items, expected %d items", len(podList.Items), *appClone.Spec.Replicas)
+				}
+				for _, pod := range podList.Items {
+					var stdErr string
+					_, stdErr, err = execCommandInPodWithName(f, cmd, pod.Name, pod.Spec.Containers[0].Name, appClone.Namespace)
+					if err != nil {
+						e2elog.Logf("command %q failed: %v", cmd, err)
+					}
+					readOnlyErr := fmt.Sprintf("'%s': Operation not permitted", devPath)
+					if !strings.Contains(stdErr, readOnlyErr) {
+						e2elog.Failf(stdErr)
+					}
+				}
+				err = deletePVCAndDeploymentApp(f, "", pvcClone, appClone)
+				if err != nil {
+					e2elog.Failf("failed to delete PVC and application: %v", err)
+				}
+				// delete parent pvc
+				err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout)
+				if err != nil {
+					e2elog.Failf("failed to delete PVC: %v", err)
+				}
+				// validate created backend rbd images
+				validateRBDImageCount(f, 0, defaultRBDPool)
+				// validate images in trash
+				err = waitToRemoveImagesFromTrash(f, defaultRBDPool, deployTimeout)
+				if err != nil {
+					e2elog.Failf("failed to validate rbd images in pool %s trash: %v", rbdOptions(defaultRBDPool), err)
+				}
+				err = deleteResource(rbdExamplePath + "storageclass.yaml")
+				if err != nil {
+					e2elog.Failf("failed to delete storageclass: %v", err)
+				}
+				err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, nil, deletePolicy)
+				if err != nil {
+					e2elog.Failf("failed to create storageclass: %v", err)
+				}
+			})
+
 			By("perform IO on rbd-nbd volume after nodeplugin restart", func() {
 				err := deleteResource(rbdExamplePath + "storageclass.yaml")
 				if err != nil {
@@ -2414,7 +2748,7 @@ var _ = Describe("RBD", func() {
 					if err != nil {
 						e2elog.Failf("timeout waiting for daemonset pods: %v", err)
 					}
-					err = waitForDeploymentComplete(rbdDeploymentName, cephCSINamespace, f.ClientSet, deployTimeout)
+					err = waitForDeploymentComplete(f.ClientSet, rbdDeploymentName, cephCSINamespace, deployTimeout)
 					if err != nil {
 						e2elog.Failf("timeout waiting for deployment to be in running state: %v", err)
 					}
diff --git a/e2e/rbd_helper.go b/e2e/rbd_helper.go
index 08661c792..6d6950fb4 100644
--- a/e2e/rbd_helper.go
+++ b/e2e/rbd_helper.go
@@ -997,7 +997,7 @@ func recreateCSIRBDPods(f *framework.Framework) error {
 	if err != nil {
 		return fmt.Errorf("timeout waiting for daemonset pods: %w", err)
 	}
-	err = waitForDeploymentComplete(rbdDeploymentName, cephCSINamespace, f.ClientSet, deployTimeout)
+	err = waitForDeploymentComplete(f.ClientSet, rbdDeploymentName, cephCSINamespace, deployTimeout)
 	if err != nil {
 		return fmt.Errorf("timeout waiting for deployment to be in running state: %w", err)
 	}
diff --git a/e2e/templates/rbd-block-deployment.yaml b/e2e/templates/rbd-block-deployment.yaml
new file mode 100644
index 000000000..0fcf2a5df
--- /dev/null
+++ b/e2e/templates/rbd-block-deployment.yaml
@@ -0,0 +1,30 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: pod-block-rx-volume
+  labels:
+    app: pod-block-rx-volume
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: pod-block-rx-volume
+  template:
+    metadata:
+      labels:
+        app: pod-block-rx-volume
+    spec:
+      containers:
+        - name: centos
+          image: quay.io/centos/centos:latest
+          imagePullPolicy: IfNotPresent
+          command: ["/bin/sleep", "infinity"]
+          volumeDevices:
+            - name: data
+              devicePath: /dev/xvda
+      volumes:
+        - name: data
+          persistentVolumeClaim:
+            claimName: raw-block-pvc
+            readOnly: false
diff --git a/e2e/templates/rbd-fs-deployment.yaml b/e2e/templates/rbd-fs-deployment.yaml
new file mode 100644
index 000000000..8ee3cbb9c
--- /dev/null
+++ b/e2e/templates/rbd-fs-deployment.yaml
@@ -0,0 +1,29 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: pod-fs-rx-volume
+  labels:
+    app: pod-fs-rx-volume
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: pod-fs-rx-volume
+  template:
+    metadata:
+      labels:
+        app: pod-fs-rx-volume
+    spec:
+      containers:
+        - name: web-server
+          image: docker.io/library/nginx:latest
+          imagePullPolicy: IfNotPresent
+          volumeMounts:
+            - name: mypvc
+              mountPath: /var/lib/www/html
+      volumes:
+        - name: mypvc
+          persistentVolumeClaim:
+            claimName: rbd-pvc
+            readOnly: false
diff --git a/e2e/upgrade-cephfs.go b/e2e/upgrade-cephfs.go
index e01926c6b..236675841 100644
--- a/e2e/upgrade-cephfs.go
+++ b/e2e/upgrade-cephfs.go
@@ -148,7 +148,7 @@ var _ = Describe("CephFS Upgrade Testing", func() {
 	Context("Cephfs Upgrade Test", func() {
 		It("Cephfs Upgrade Test", func() {
 			By("checking provisioner deployment is running", func() {
-				err = waitForDeploymentComplete(cephFSDeploymentName, cephCSINamespace, f.ClientSet, deployTimeout)
+				err = waitForDeploymentComplete(f.ClientSet, cephFSDeploymentName, cephCSINamespace, deployTimeout)
 				if err != nil {
 					e2elog.Failf("timeout waiting for deployment %s: %v", cephFSDeploymentName, err)
 				}
@@ -241,7 +241,7 @@ var _ = Describe("CephFS Upgrade Testing", func() {
 				}
 				deployCephfsPlugin()
 
-				err = waitForDeploymentComplete(cephFSDeploymentName, cephCSINamespace, f.ClientSet, deployTimeout)
+				err = waitForDeploymentComplete(f.ClientSet, cephFSDeploymentName, cephCSINamespace, deployTimeout)
 				if err != nil {
 					e2elog.Failf("timeout waiting for upgraded deployment %s: %v", cephFSDeploymentName, err)
 				}
diff --git a/e2e/upgrade-rbd.go b/e2e/upgrade-rbd.go
index 67c45ab10..2db1e419a 100644
--- a/e2e/upgrade-rbd.go
+++ b/e2e/upgrade-rbd.go
@@ -165,7 +165,7 @@ var _ = Describe("RBD Upgrade Testing", func() {
 			appPath := rbdExamplePath + "pod.yaml"
 
 			By("checking provisioner deployment is running", func() {
-				err := waitForDeploymentComplete(rbdDeploymentName, cephCSINamespace, f.ClientSet, deployTimeout)
+				err := waitForDeploymentComplete(f.ClientSet, rbdDeploymentName, cephCSINamespace, deployTimeout)
 				if err != nil {
 					e2elog.Failf("timeout waiting for deployment %s: %v", rbdDeploymentName, err)
 				}
@@ -260,7 +260,7 @@ var _ = Describe("RBD Upgrade Testing", func() {
 
 				deployRBDPlugin()
 
-				err = waitForDeploymentComplete(rbdDeploymentName, cephCSINamespace, f.ClientSet, deployTimeout)
+				err = waitForDeploymentComplete(f.ClientSet, rbdDeploymentName, cephCSINamespace, deployTimeout)
 				if err != nil {
 					e2elog.Failf("timeout waiting for upgraded deployment %s: %v", rbdDeploymentName, err)
 				}
diff --git a/e2e/utils.go b/e2e/utils.go
index 58a13853e..9af1a42a6 100644
--- a/e2e/utils.go
+++ b/e2e/utils.go
@@ -15,6 +15,7 @@ import (
 	"time"
 
 	snapapi "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1"
+	appsv1 "k8s.io/api/apps/v1"
 	batch "k8s.io/api/batch/v1"
 	v1 "k8s.io/api/core/v1"
 	scv1 "k8s.io/api/storage/v1"
@@ -177,6 +178,50 @@ func createPVCAndApp(
 	return err
 }
 
+// createPVCAndDeploymentApp creates pvc and deployment, if name is not empty
+// same will be set as pvc and app name.
+func createPVCAndDeploymentApp(
+	f *framework.Framework,
+	name string,
+	pvc *v1.PersistentVolumeClaim,
+	app *appsv1.Deployment,
+	pvcTimeout int) error {
+	if name != "" {
+		pvc.Name = name
+		app.Name = name
+		app.Spec.Template.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = name
+	}
+	err := createPVCAndvalidatePV(f.ClientSet, pvc, pvcTimeout)
+	if err != nil {
+		return err
+	}
+	err = createDeploymentApp(f.ClientSet, app, deployTimeout)
+
+	return err
+}
+
+// DeletePVCAndDeploymentApp deletes pvc and deployment, if name is not empty
+// same will be set as pvc and app name.
+func deletePVCAndDeploymentApp(
+	f *framework.Framework,
+	name string,
+	pvc *v1.PersistentVolumeClaim,
+	app *appsv1.Deployment) error {
+	if name != "" {
+		pvc.Name = name
+		app.Name = name
+		app.Spec.Template.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = name
+	}
+
+	err := deleteDeploymentApp(f.ClientSet, app.Name, app.Namespace, deployTimeout)
+	if err != nil {
+		return err
+	}
+	err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout)
+
+	return err
+}
+
 // deletePVCAndApp delete pvc and pod
 // if name is not empty same will be set as pvc and app name.
 func deletePVCAndApp(name string, f *framework.Framework, pvc *v1.PersistentVolumeClaim, app *v1.Pod) error {
diff --git a/internal/rbd/rbd_attach_test.go b/internal/rbd/rbd_attach_test.go
new file mode 100644
index 000000000..07749530d
--- /dev/null
+++ b/internal/rbd/rbd_attach_test.go
@@ -0,0 +1,99 @@
+/*
+Copyright 2021 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package rbd
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestParseMapOptions(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name              string
+		mapOption         string
+		expectKrbdOptions string
+		expectNbdOptions  string
+		expectErr         string
+	}{
+		{
+			name:              "with old format",
+			mapOption:         "kOp1,kOp2",
+			expectKrbdOptions: "kOp1,kOp2",
+			expectNbdOptions:  "",
+			expectErr:         "",
+		},
+		{
+			name:              "with new format",
+			mapOption:         "krbd:kOp1,kOp2;nbd:nOp1,nOp2",
+			expectKrbdOptions: "kOp1,kOp2",
+			expectNbdOptions:  "nOp1,nOp2",
+			expectErr:         "",
+		},
+		{
+			name:              "without krbd: label",
+			mapOption:         "kOp1,kOp2;nbd:nOp1,nOp2",
+			expectKrbdOptions: "kOp1,kOp2",
+			expectNbdOptions:  "nOp1,nOp2",
+			expectErr:         "",
+		},
+		{
+			name:              "with only nbd label",
+			mapOption:         "nbd:nOp1,nOp2",
+			expectKrbdOptions: "",
+			expectNbdOptions:  "nOp1,nOp2",
+			expectErr:         "",
+		},
+		{
+			name:              "unknown mounter used",
+			mapOption:         "xyz:xOp1,xOp2",
+			expectKrbdOptions: "",
+			expectNbdOptions:  "",
+			expectErr:         "unknown mounter type",
+		},
+		{
+			name:              "bad formatted options",
+			mapOption:         "nbd:nOp1:nOp2;",
+			expectKrbdOptions: "",
+			expectNbdOptions:  "",
+			expectErr:         "badly formatted map/unmap options",
+		},
+	}
+	for _, tt := range tests {
+		tc := tt
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			krbdOpts, nbdOpts, err := parseMapOptions(tc.mapOption)
+			if err != nil && !strings.Contains(err.Error(), tc.expectErr) {
+				// returned error
+				t.Errorf("parseMapOptions(%s) returned error, expected: %v, got: %v",
+					tc.mapOption, tc.expectErr, err)
+			}
+			if krbdOpts != tc.expectKrbdOptions {
+				// unexpected krbd option error
+				t.Errorf("parseMapOptions(%s) returned unexpected krbd options, expected :%q, got: %q",
+					tc.mapOption, tc.expectKrbdOptions, krbdOpts)
+			}
+			if nbdOpts != tc.expectNbdOptions {
+				// unexpected nbd option error
+				t.Errorf("parseMapOptions(%s) returned unexpected nbd options, expected: %q, got: %q",
+					tc.mapOption, tc.expectNbdOptions, nbdOpts)
+			}
+		})
+	}
+}