Repository: kubernetes-sigs/gcp-compute-persistent-disk-csi-driver
Pull request #946 (2022-03-28 15:48:17)

Automated cherry pick of #900: add nvme script and rules
#933: Add support for NVMe NodePublishVolume
Cherry pick of #900 #933 on release-1.4.

What type of PR is this?
/kind feature

#900: add nvme script and rules
#933: Add support for NVMe NodePublishVolume

For details on the cherry pick process, see the [cherry pick requests](https://git.k8s.io/community/contributors/devel/sig-release/cherry-picks.md) page.

```release-note
Add support for NVMe persistent disks
```


Patch:
diff --git a/Dockerfile b/Dockerfile
index 3a68a02b..db80c088 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -26,7 +26,8 @@ RUN GOARCH=$(echo $TARGETPLATFORM | cut -f2 -d '/') GCE_PD_CSI_STAGING_VERSION=$
 # Start from Kubernetes Debian base.
 FROM k8s.gcr.io/build-image/debian-base:buster-v1.9.0 as debian
 # Install necessary dependencies
-RUN clean-install util-linux e2fsprogs mount ca-certificates udev xfsprogs
+# google_nvme_id script depends on the following packages: nvme-cli, xxd, bash
+RUN clean-install util-linux e2fsprogs mount ca-certificates udev xfsprogs nvme-cli xxd bash
 # Since we're leveraging apt to pull in dependencies, we use `gcr.io/distroless/base` because it includes glibc.
 FROM gcr.io/distroless/base-debian11
 # Copy necessary dependencies into distroless base.
@@ -50,6 +51,14 @@ COPY --from=debian /sbin/xfs_repair /sbin/xfs_repair
 COPY --from=debian /usr/include/xfs /usr/include/xfs
 COPY --from=debian /usr/lib/xfsprogs/xfs* /usr/lib/xfsprogs/
 COPY --from=debian /usr/sbin/xfs* /usr/sbin/
+# Add dependencies for /lib/udev_containerized/google_nvme_id script
+COPY --from=debian /usr/sbin/nvme /usr/sbin/nvme
+COPY --from=debian /usr/bin/xxd /usr/bin/xxd
+COPY --from=debian /bin/bash /bin/bash
+COPY --from=debian /bin/date /bin/date
+COPY --from=debian /bin/grep /bin/grep
+COPY --from=debian /bin/sed /bin/sed
+COPY --from=debian /bin/ln /bin/ln
 
 # Copy x86 shared libraries into distroless base.
 COPY --from=debian /lib/x86_64-linux-gnu/libblkid.so.1 /lib/x86_64-linux-gnu/libblkid.so.1
@@ -63,4 +72,7 @@ COPY --from=debian /lib/x86_64-linux-gnu/libselinux.so.1 /lib/x86_64-linux-gnu/l
 COPY --from=debian /lib/x86_64-linux-gnu/libtinfo.so.6 /lib/x86_64-linux-gnu/libtinfo.so.6
 COPY --from=debian /lib/x86_64-linux-gnu/libuuid.so.1 /lib/x86_64-linux-gnu/libuuid.so.1
 
+# Copy NVME support required script and rules into distroless base.
+COPY deploy/kubernetes/udev/google_nvme_id /lib/udev_containerized/google_nvme_id
+
 ENTRYPOINT ["/gce-pd-csi-driver"]
\ No newline at end of file
diff --git a/Dockerfile.arm64 b/Dockerfile.arm64
index 9052db4d..480819d5 100644
--- a/Dockerfile.arm64
+++ b/Dockerfile.arm64
@@ -32,7 +32,11 @@ RUN clean-install udev
 FROM k8s.gcr.io/build-image/debian-base:buster-v1.9.0
 COPY --from=builder /go/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/bin/gce-pd-csi-driver /gce-pd-csi-driver
 # Install necessary dependencies
-RUN clean-install util-linux e2fsprogs mount ca-certificates udev xfsprogs
+# google_nvme_id script depends on the following packages: nvme-cli, xxd, bash
+RUN clean-install util-linux e2fsprogs mount ca-certificates udev xfsprogs nvme-cli xxd bash
 COPY --from=mad-hack /lib/udev/scsi_id /lib/udev_containerized/scsi_id
 
+# Copy google_nvme_id script, which is needed for NVMe support.
+COPY deploy/kubernetes/udev/google_nvme_id /lib/udev_containerized/google_nvme_id
+
 ENTRYPOINT ["/gce-pd-csi-driver"]
diff --git a/deploy/kubernetes/README.md b/deploy/kubernetes/README.md
index c21de286..f9fe6825 100644
--- a/deploy/kubernetes/README.md
+++ b/deploy/kubernetes/README.md
@@ -23,3 +23,7 @@ The current structure for kustomization is as follows. Note that Windows support
   * `prow-gke-release-staging-rc-{k8s-minor}`: Used for prow tests. Contains deployment specs of a driver for given k8s    minor version release.
   * `prow-gke-release-staging-rc-head`: Used for prow tests. Contains deployment specs of a driver with latest sidecar images, for latest k8s master.
   * `stable`, `prow-gke-release-staging-rc`: Soon to be removed!
+
+  ## NVME support
+  udev folder under deploy/kubernetes contains google_nvme_id script required for NVME support. Source is downloaded from [GoogleCloudPlatform/guest-configs](https://github.com/GoogleCloudPlatform/guest-configs). README file in the folder contains the downloaded version.
+  Execute deploy/kubernetes/update-nvme.sh to update. See releases available in [GoogleCloudPlatform/guest-configs/releases](https://github.com/GoogleCloudPlatform/guest-configs/releases).
diff --git a/deploy/kubernetes/udev/README b/deploy/kubernetes/udev/README
new file mode 100644
index 00000000..35a53f39
--- /dev/null
+++ b/deploy/kubernetes/udev/README
@@ -0,0 +1,1 @@
+GUEST_CONFIGS_VERSION:  20211116.00
diff --git a/deploy/kubernetes/udev/google_nvme_id b/deploy/kubernetes/udev/google_nvme_id
new file mode 100755
index 00000000..4f8c43f4
--- /dev/null
+++ b/deploy/kubernetes/udev/google_nvme_id
@@ -0,0 +1,245 @@
+#!/bin/bash
+# Copyright 2020 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Used to generate symlinks for PD-NVMe devices using the disk names reported by
+# the metadata server
+
+# Locations of the script's dependencies
+readonly nvme_cli_bin=/usr/sbin/nvme
+
+# Bash regex to parse device paths and controller identification
+readonly NAMESPACE_NUMBER_REGEX="/dev/nvme[[:digit:]]+n([[:digit:]]+).*"
+readonly PARTITION_NUMBER_REGEX="/dev/nvme[[:digit:]]+n[[:digit:]]+p([[:digit:]]+)"
+readonly PD_NVME_REGEX="sn[[:space:]]+:[[:space]]+nvme_card-pd"
+
+# Globals used to generate the symlinks for a PD-NVMe disk.  These are populated
+# by the identify_pd_disk function and exported for consumption by udev rules.
+ID_SERIAL=''
+ID_SERIAL_SHORT=''
+
+#######################################
+# Helper function to log an error message to stderr.
+# Globals:
+#   None
+# Arguments:
+#   String to print as the log message
+# Outputs:
+#   Writes error to STDERR
+#######################################
+function err() {
+  echo "[$(date +'%Y-%m-%dT%H:%M:%S%z')]: $*" >&2
+}
+
+#######################################
+# Retrieves the device name for an NVMe namespace using nvme-cli.
+# Globals:
+#   Uses nvme_cli_bin
+# Arguments:
+#   The path to the nvme namespace (/dev/nvme0n?)
+# Outputs:
+#   The device name parsed from the JSON in the vendor ext of the ns-id command.
+# Returns:
+#   0 if the device name for the namespace could be retrieved, 1 otherwise
+#######################################
+function get_namespace_device_name() {
+  local nvme_json
+  nvme_json="$("$nvme_cli_bin" id-ns -b "$1" | xxd -p -seek 384 | xxd -p -r)"
+  if [[ $? -ne 0 ]]; then
+    return 1
+  fi
+
+  if [[ -z "$nvme_json" ]]; then
+    err "NVMe Vendor Extension disk information not present"
+    return 1
+  fi
+
+  local device_name
+  device_name="$(echo "$nvme_json" | grep device_name | sed -e 's/.*"device_name":[  \t]*"\([a-zA-Z0-9_-]\+\)".*/\1/')"
+
+  # Error if our device name is empty
+  if [[ -z "$device_name" ]]; then
+    err "Empty name"
+    return 1
+  fi
+
+  echo "$device_name"
+  return 0
+}
+
+#######################################
+# Retrieves the nsid for an NVMe namespace
+# Globals:
+#   None
+# Arguments:
+#   The path to the nvme namespace (/dev/nvme0n*)
+# Outputs:
+#   The namespace number/id
+# Returns:
+#   0 if the namespace id could be retrieved, 1 otherwise
+#######################################
+function get_namespace_number() {
+  local dev_path="$1"
+  local namespace_number
+  if [[ "$dev_path" =~ $NAMESPACE_NUMBER_REGEX ]]; then
+    namespace_number="${BASH_REMATCH[1]}"
+  else
+    return 1
+  fi
+
+  echo "$namespace_number"
+  return 0
+}
+
+#######################################
+# Retrieves the partition number for a device path if it exists
+# Globals:
+#   None
+# Arguments:
+#   The path to the device partition (/dev/nvme0n*p*)
+# Outputs:
+#   The value after 'p' in the device path, or an empty string if the path has
+#   no partition.
+#######################################
+function get_partition_number() {
+  local dev_path="$1"
+  local partition_number
+  if [[ "$dev_path" =~ $PARTITION_NUMBER_REGEX ]]; then
+    partition_number="${BASH_REMATCH[1]}"
+    echo "$partition_number"
+  else
+    echo ''
+  fi
+  return 0
+}
+
+#######################################
+# Generates a symlink for a PD-NVMe device using the metadata's disk name.
+# Primarily used for testing but can be used if the script is directly invoked.
+# Globals:
+#   Uses ID_SERIAL_SHORT (can be populated by identify_pd_disk)
+# Arguments:
+#   The device path for the disk
+#######################################
+function gen_symlink() {
+  local dev_path="$1"
+  local partition_number="$(get_partition_number "$dev_path")"
+
+  if [[ -n "$partition_number" ]]; then
+    ln -s "$dev_path" /dev/disk/by-id/google-"$ID_SERIAL_SHORT"-part"$partition_number" > /dev/null 2>&1
+  else
+    ln -s "$dev_path" /dev/disk/by-id/google-"$ID_SERIAL_SHORT" > /dev/null 2>&1
+  fi
+
+  return 0
+}
+
+#######################################
+# Populates the ID_* global variables with a disk's device name and namespace
+# Globals:
+#   Populates ID_SERIAL_SHORT, and ID_SERIAL
+# Arguments:
+#   The device path for the disk
+# Returns:
+#   0 on success and 1 if an error occurrs
+#######################################
+function identify_pd_disk() {
+  local dev_path="$1"
+  local dev_name
+  dev_name="$(get_namespace_device_name "$dev_path")"
+  if [[ $? -ne 0 ]]; then
+    return 1
+  fi
+
+  ID_SERIAL_SHORT="$dev_name"
+  ID_SERIAL="Google_PersistentDisk_${ID_SERIAL_SHORT}"
+  return 0
+}
+
+function print_help_message() {
+  echo "Usage: google_nvme_id [-s] [-h] -d device_path"
+  echo "  -d <device_path> (Required): Specifies the path to generate a name"
+  echo "        for.  This needs to be a path to an nvme device or namespace"
+  echo "  -s: Create symbolic link for the disk under /dev/disk/by-id."
+  echo "        Otherwise, the disk name will be printed to STDOUT"
+  echo "  -h: Print this help message"
+}
+
+function main() {
+  local opt_gen_symlink='false'
+  local device_path=''
+
+  while getopts :d:sh flag; do
+    case "$flag" in
+      d) device_path="$OPTARG";;
+      s) opt_gen_symlink='true';;
+      h) print_help_message
+         return 0
+         ;;
+      :) echo "Invalid option: ${OPTARG} requires an argument" 1>&2
+         return 1
+         ;;
+      *) return 1
+    esac
+  done
+
+  if [[ -z "$device_path" ]]; then
+    echo "Device path (-d) argument required. Use -h for full usage." 1>&2
+    exit 1
+  fi
+
+  # Ensure the nvme-cli command is installed
+  command -v "$nvme_cli_bin" > /dev/null 2>&1
+  if [[ $? -ne 0 ]]; then
+    err "The nvme utility (/usr/sbin/nvme) was not found. You may need to run \
+with sudo or install nvme-cli."
+    return 1
+  fi
+
+  # Ensure the passed device is actually an NVMe device
+  "$nvme_cli_bin" id-ctrl "$device_path" &>/dev/null
+  if [[ $? -ne 0 ]]; then
+    err "Passed device was not an NVMe device.  (You may need to run this \
+script as root/with sudo)."
+    return 1
+  fi
+
+  # Detect the type of attached nvme device
+  local controller_id
+  controller_id=$("$nvme_cli_bin" id-ctrl "$device_path")
+  if [[ ! "$controller_id" =~ nvme_card-pd ]] ; then
+    err "Device is not a PD-NVMe device"
+    return 1
+  fi
+
+  # Fill the global variables for the id command for the given disk type
+  # Error messages will be printed closer to error, no need to reprint here
+  identify_pd_disk "$device_path"
+  if [[ $? -ne 0 ]]; then
+    return $?
+  fi
+
+  # Gen symlinks or print out the globals set by the identify command
+  if [[ "$opt_gen_symlink" == 'true' ]]; then
+    gen_symlink "$device_path"
+  else
+    # These will be consumed by udev
+    echo "ID_SERIAL_SHORT=${ID_SERIAL_SHORT}"
+    echo "ID_SERIAL=${ID_SERIAL}"
+  fi
+
+  return $?
+
+}
+main "$@"
diff --git a/deploy/kubernetes/update-nvme.sh b/deploy/kubernetes/update-nvme.sh
new file mode 100755
index 00000000..dd6ffd82
--- /dev/null
+++ b/deploy/kubernetes/update-nvme.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+# This scripts is to download source of a release from GoogleCloudPlatform/guest-configs repo
+# and copy script and rules required for NVMe support to be under deploy/kubernetes 
+#
+# Args:
+# GUEST_CONFIGS_VERSION: The version of the guest configs release
+
+set -o nounset
+set -o errexit
+
+readonly PKGDIR="${GOPATH}/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver"
+readonly INSTALL_DIR="${PKGDIR}/tmp"
+readonly DEST="${PKGDIR}/deploy/kubernetes/udev"
+readonly SOURCE_FOLDER_NAME="guest-configs-${GUEST_CONFIGS_VERSION}"
+
+function cleanup {
+  rm -rf "$INSTALL_DIR"
+}
+
+trap cleanup EXIT
+
+curl --create-dirs -O --output-dir ${INSTALL_DIR} -L https://github.com/GoogleCloudPlatform/guest-configs/archive/refs/tags/${GUEST_CONFIGS_VERSION}.tar.gz
+tar -xzf ${INSTALL_DIR}/${GUEST_CONFIGS_VERSION}.tar.gz -C ${INSTALL_DIR}
+cp -R ${INSTALL_DIR}/${SOURCE_FOLDER_NAME}/src/lib/udev/google_nvme_id ${DEST}
+echo "GUEST_CONFIGS_VERSION: " ${GUEST_CONFIGS_VERSION} >  ${DEST}/README
diff --git a/pkg/mount-manager/device-utils.go b/pkg/mount-manager/device-utils.go
index 1ea1ebed..60ff099b 100644
--- a/pkg/mount-manager/device-utils.go
+++ b/pkg/mount-manager/device-utils.go
@@ -36,6 +36,8 @@ const (
 	diskPartitionSuffix  = "-part"
 	diskSDPath           = "/dev/sd"
 	diskSDPattern        = "/dev/sd*"
+	diskNvmePath         = "/dev/nvme"
+	diskNvmePattern      = "/dev/nvme*"
 	// How many times to retry for a consistent read of /proc/mounts.
 	maxListTries = 3
 	// Number of fields per line in /proc/mounts as per the fstab man page.
@@ -52,11 +54,20 @@ const (
 	// scsi_id output should be in the form of:
 	// 0Google PersistentDisk <disk name>
 	scsiPattern = `^0Google\s+PersistentDisk\s+([\S]+)\s*$`
+	// google_nvme_id output should be in the form of:
+	// ID_SERIAL_SHORT=<disk name>
+	// Note: The google_nvme_id tool prints out multiple lines, hence we don't
+	// use '^' and '$' to wrap nvmePattern as is done in scsiPattern.
+	nvmePattern = `ID_SERIAL_SHORT=([\S]+)\s*`
+	scsiIdPath  = "/lib/udev_containerized/scsi_id"
+	nvmeIdPath  = "/lib/udev_containerized/google_nvme_id"
 )
 
 var (
 	// regex to parse scsi_id output and extract the serial
 	scsiRegex = regexp.MustCompile(scsiPattern)
+	// regex to parse google_nvme_id output and extract the serial
+	nvmeRegex = regexp.MustCompile(nvmePattern)
 )
 
 // DeviceUtils are a collection of methods that act on the devices attached
@@ -107,13 +118,13 @@ func existingDevicePath(devicePaths []string) (string, error) {
 	return "", nil
 }
 
-// getScsiSerial assumes that /lib/udev/scsi_id exists and will error if it
+// getScsiSerial assumes that scsiIdPath exists and will error if it
 // doesnt. It is the callers responsibility to verify the existence of this
 // tool. Calls scsi_id on the given devicePath to get the serial number reported
 // by that device.
 func getScsiSerial(devicePath string) (string, error) {
 	out, err := exec.Command(
-		"/lib/udev_containerized/scsi_id",
+		scsiIdPath,
 		"--page=0x83",
 		"--whitelisted",
 		fmt.Sprintf("--device=%v", devicePath)).CombinedOutput()
@@ -134,9 +145,58 @@ func parseScsiSerial(output string) (string, error) {
 	return substrings[1], nil
 }
 
+// getNvmeSerial calls google_nvme_id on the given devicePath to get the serial
+// number reported by that device.
+// NOTE: getNvmeSerial assumes that nvmeIdPath exists and will error if it
+// doesn't. It is the caller's responsibility to verify the existence of this
+// tool.
+func getNvmeSerial(devicePath string) (string, error) {
+	out, err := exec.Command(
+		nvmeIdPath,
+		fmt.Sprintf("-d%s", devicePath)).CombinedOutput()
+	if err != nil {
+		return "", fmt.Errorf("google_nvme_id failed for device %q with output %v: %v", devicePath, out, err)
+	}
+
+	return parseNvmeSerial(string(out))
+}
+
+// Parse the output returned by google_nvme_id and extract the serial number
+func parseNvmeSerial(output string) (string, error) {
+	substrings := nvmeRegex.FindStringSubmatch(output)
+	if substrings == nil {
+		return "", fmt.Errorf("google_nvme_id output cannot be parsed: %q", output)
+	}
+
+	return substrings[1], nil
+}
+
+func ensureUdevToolExists(toolPath string) error {
+	exists, err := pathutils.Exists(pathutils.CheckFollowSymlink, toolPath)
+	if err != nil {
+		return fmt.Errorf("failed to check existence of %q: %v", toolPath, err)
+	}
+	if !exists {
+		// The driver should be containerized with the tool so maybe something is
+		// wrong with the build process
+		return fmt.Errorf("could not find tool at %q, unable to verify device paths", nvmeIdPath)
+	}
+	return nil
+}
+
+func ensureUdevToolsExist() error {
+	if err := ensureUdevToolExists(scsiIdPath); err != nil {
+		return err
+	}
+	if err := ensureUdevToolExists(nvmeIdPath); err != nil {
+		return err
+	}
+	return nil
+}
+
 // VerifyDevicePath returns the first devicePath that maps to a real disk in the
-// candidate devicePaths or an empty string if none is found. If
-// /lib/udev_containerized/scsi_id exists it will attempt to fix any issues
+// candidate devicePaths or an empty string if none is found.
+// If the device is not found, it will attempt to fix any issues
 // caused by missing paths or mismatched devices by running a udevadm --trigger.
 func (m *deviceUtils) VerifyDevicePath(devicePaths []string, deviceName string) (string, error) {
 	var devicePath string
@@ -146,15 +206,10 @@ func (m *deviceUtils) VerifyDevicePath(devicePaths []string, deviceName string)
 		pollTimeout  = 3 * time.Second
 	)
 
-	scsiIDPath := "/lib/udev_containerized/scsi_id"
-	exists, err := pathutils.Exists(pathutils.CheckFollowSymlink, scsiIDPath)
+	// Ensure tools in /lib/udev_containerized directory exist
+	err = ensureUdevToolsExist()
 	if err != nil {
-		return "", fmt.Errorf("failed to check scsi_id existence: %v", err)
-	}
-	if !exists {
-		// No SCSI ID tool, the driver should be containerized with the tool so
-		// maybe something is wrong with the build process
-		return "", fmt.Errorf("could not find scsi_id tool at %s, unable to verify device paths", scsiIDPath)
+		return "", err
 	}
 
 	err = wait.Poll(pollInterval, pollTimeout, func() (bool, error) {
@@ -166,40 +221,41 @@ func (m *deviceUtils) VerifyDevicePath(devicePaths []string, deviceName string)
 		}
 
 		if len(devicePath) == 0 {
-			// Couldn't find the path so we need to find a /dev/sdx with the SCSI
-			// serial that matches deviceName. Then we run udevadm trigger on that
-			// device to get the device to show up in /dev/by-id/
+			// Couldn't find a /dev/disk/by-id path for this deviceName, so we need to
+			// find a /dev/* with a serial that matches deviceName. Then we attempt
+			// to repair the symlink.
 			innerErr := udevadmTriggerForDiskIfExists(deviceName)
 			if innerErr != nil {
-				return false, fmt.Errorf("failed to trigger udevadm fix: %v", innerErr)
+				return false, fmt.Errorf("failed to trigger udevadm fix of non existent disk for %q: %v", deviceName, innerErr)
 			}
 			// Go to next retry loop to get the deviceName again after
 			// potentially fixing it with the udev command
 			return false, nil
 		}
 
-		// If there exists a devicePath we make sure disk at /dev/sdx matches the
-		// expected disk at devicePath by matching SCSI Serial to the disk name
-		devSDX, innerErr := filepath.EvalSymlinks(devicePath)
+		// If there exists a devicePath we make sure disk at /dev/* matches the
+		// expected disk at devicePath by matching device Serial to the disk name
+		devFsPath, innerErr := filepath.EvalSymlinks(devicePath)
 		if innerErr != nil {
 			return false, fmt.Errorf("filepath.EvalSymlinks(%q) failed with %v", devicePath, innerErr)
 		}
-		// Check to make sure device path maps to the correct disk
-		if strings.Contains(devSDX, diskSDPath) {
-			scsiSerial, innerErr := getScsiSerial(devSDX)
-			if innerErr != nil {
-				return false, fmt.Errorf("couldn't get SCSI serial number for disk %s: %v", deviceName, innerErr)
-			}
-			// SUCCESS! devicePath points to a /dev/sdx that has a SCSI serial
-			// equivalent to our disk name
-			if scsiSerial == deviceName {
-				return true, nil
-			}
+
+		devFsSerial, innerErr := getDevFsSerial(devFsPath)
+		if innerErr != nil {
+			return false, fmt.Errorf("couldn't get serial number for disk %s at path %s: %v", deviceName, devFsPath, innerErr)
+		}
+		// SUCCESS! devicePath points to a /dev/* path that has a serial
+		// equivalent to our disk name
+		if len(devFsSerial) != 0 && devFsSerial == deviceName {
+			return true, nil
 		}
-		// The devicePath is not mapped to the correct disk
+
+		// A /dev/* path exists, but is either not a recognized /dev prefix type
+		// (/dev/nvme* or /dev/sd*) or devicePath is not mapped to the correct disk.
+		// Attempt a repair
 		innerErr = udevadmTriggerForDiskIfExists(deviceName)
 		if innerErr != nil {
-			return false, fmt.Errorf("failed to trigger udevadm fix: %v", innerErr)
+			return false, fmt.Errorf("failed to trigger udevadm fix of misconfigured disk for %q: %v", deviceName, innerErr)
 		}
 		// Go to next retry loop to get the deviceName again after
 		// potentially fixing it with the udev command
@@ -213,49 +269,78 @@ func (m *deviceUtils) VerifyDevicePath(devicePaths []string, deviceName string)
 	return devicePath, nil
 }
 
+// getDevFsSerial returns the serial number of the /dev/* path at devFsPath.
+// If devFsPath does not start with a known prefix, returns the empty string.
+func getDevFsSerial(devFsPath string) (string, error) {
+	switch {
+	case strings.HasPrefix(devFsPath, diskSDPath):
+		return getScsiSerial(devFsPath)
+	case strings.HasPrefix(devFsPath, diskNvmePath):
+		return getNvmeSerial(devFsPath)
+	default:
+		return "", nil
+	}
+}
+
+func findAvailableDevFsPaths() ([]string, error) {
+	diskSDPaths, err := filepath.Glob(diskSDPattern)
+	if err != nil {
+		return nil, fmt.Errorf("failed to filepath.Glob(\"%s\"): %v", diskSDPattern, err)
+	}
+	diskNvmePaths, err := filepath.Glob(diskNvmePattern)
+	if err != nil {
+		return nil, fmt.Errorf("failed to filepath.Glob(\"%s\"): %v", diskNvmePattern, err)
+	}
+	return append(diskSDPaths, diskNvmePaths...), nil
+}
+
 func udevadmTriggerForDiskIfExists(deviceName string) error {
-	devToSCSI := map[string]string{}
-	sds, err := filepath.Glob(diskSDPattern)
+	devFsPathToSerial := map[string]string{}
+	devFsPaths, err := findAvailableDevFsPaths()
 	if err != nil {
-		return fmt.Errorf("failed to filepath.Glob(\"%s\"): %v", diskSDPattern, err)
+		return err
 	}
-	for _, devSDX := range sds {
-		scsiSerial, err := getScsiSerial(devSDX)
-		if err != nil {
-			return fmt.Errorf("failed to get SCSI Serial num: %v", err)
+	for _, devFsPath := range devFsPaths {
+		devFsSerial, err := getDevFsSerial(devFsPath)
+		if err != nil || len(devFsSerial) == 0 {
+			// If we get an error, ignore. Either this isn't a block device, or it
+			// isn't something we can get a serial number from
+			klog.V(7).Infof("failed to get Serial num for disk %s at path %s: %v", deviceName, devFsPath, err)
+			continue
 		}
-		devToSCSI[devSDX] = scsiSerial
-		if scsiSerial == deviceName {
+		devFsPathToSerial[devFsPath] = devFsSerial
+		if devFsSerial == deviceName {
 			// Found the disk that we're looking for so run a trigger on it
 			// to resolve its /dev/by-id/ path
-			klog.Warningf("udevadm --trigger running to fix disk at path %s which has SCSI ID %s", devSDX, scsiSerial)
-			err := udevadmChangeToDrive(devSDX)
+			klog.Warningf("udevadm --trigger running to fix disk at path %s which has serial numberID %s", devFsPath, devFsSerial)
+			err := udevadmChangeToDrive(devFsPath)
 			if err != nil {
-				return fmt.Errorf("failed to fix disk which has SCSI ID %s: %v", scsiSerial, err)
+				return fmt.Errorf("failed to fix disk which has serial numberID %s: %v", devFsSerial, err)
 			}
 			return nil
 		}
 	}
-	klog.Warningf("udevadm --trigger requested to fix disk %s but no such disk was found in %v", deviceName, devToSCSI)
+	klog.Warningf("udevadm --trigger requested to fix disk %s but no such disk was found in %v", deviceName, devFsPathToSerial)
 	return fmt.Errorf("udevadm --trigger requested to fix disk %s but no such disk was found", deviceName)
 }
 
 // Calls "udevadm trigger --action=change" on the specified drive. drivePath
-// must be the block device path to trigger on, in the format "/dev/sd*", or a
-// symlink to it. This is workaround for Issue #7972. Once the underlying issue
-// has been resolved, this may be removed.
+// must be the block device path to trigger on, in the format "/dev/*", or a
+// symlink to it. This is workaround for Issue #7972
+// (https://github.com/kubernetes/kubernetes/issues/7972). Once the underlying
+// issue has been resolved, this may be removed.
 // udevadm takes a little bit to work its magic in the background so any callers
 // should not expect the trigger to complete instantly and may need to poll for
 // the change
-func udevadmChangeToDrive(devSDX string) error {
-	// Call "udevadm trigger --action=change --property-match=DEVNAME=/dev/sd..."
+func udevadmChangeToDrive(devFsPath string) error {
+	// Call "udevadm trigger --action=change --property-match=DEVNAME=/dev/..."
 	out, err := exec.Command(
 		"udevadm",
 		"trigger",
 		"--action=change",
-		fmt.Sprintf("--property-match=DEVNAME=%s", devSDX)).CombinedOutput()
+		fmt.Sprintf("--property-match=DEVNAME=%s", devFsPath)).CombinedOutput()
 	if err != nil {
-		return fmt.Errorf("udevadmChangeToDrive: udevadm trigger failed for drive %q with output %s: %v.", devSDX, string(out), err)
+		return fmt.Errorf("udevadmChangeToDrive: udevadm trigger failed for drive %q with output %s: %v.", devFsPath, string(out), err)
 	}
 	return nil
 }


Test patch:
diff --git a/pkg/mount-manager/device-utils_test.go b/pkg/mount-manager/device-utils_test.go
new file mode 100644
index 00000000..dd3d22b0
--- /dev/null
+++ b/pkg/mount-manager/device-utils_test.go
@@ -0,0 +1,47 @@
+package mountmanager
+
+import (
+	"testing"
+)
+
+func TestParseNvmeSerial(t *testing.T) {
+	testCases := []struct {
+		name   string
+		output string
+		serial string
+		expErr bool
+	}{
+		{
+			name:   "valid google_nvme_id response",
+			output: "line 58: warning: command substitution: ignored null byte in input\nID_SERIAL_SHORT=pvc-8ee0cf44-6acd-456e-9f3b-95ccd65065b9\nID_SERIAL=Google_PersistentDisk_pvc-8ee0cf44-6acd-456e-9f3b-95ccd65065b9",
+			serial: "pvc-8ee0cf44-6acd-456e-9f3b-95ccd65065b9",
+			expErr: false,
+		},
+		{
+			name:   "valid google_nvme_id boot disk response",
+			output: "line 58: warning: command substitution: ignored null byte in input\nID_SERIAL_SHORT=persistent-disk-0\nID_SERIAL=Google_PersistentDisk_persistent-disk-0",
+			serial: "persistent-disk-0",
+			expErr: false,
+		},
+		{
+			name:   "invalid google_nvme_id response",
+			output: "Error: requesting namespace-id from non-block device\nNVMe Status:INVALID_NS: The namespace or the format of that namespace is invalid(b) NSID:0\nxxd: sorry cannot seek.\n[2022-03-12T04:17:17+0000]: NVMe Vendor Extension disk information not present",
+			serial: "",
+			expErr: true,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Logf("Running test: %v", tc.name)
+		actualSerial, err := parseNvmeSerial(tc.output)
+		if tc.expErr && err == nil {
+			t.Fatalf("Expected error but didn't get any")
+		}
+		if !tc.expErr && err != nil {
+			t.Fatalf("Got unexpected error: %s", err)
+		}
+		if actualSerial != tc.serial {
+			t.Fatalf("Expected '%s' but got '%s'", tc.serial, actualSerial)
+		}
+	}
+}
diff --git a/test/e2e/tests/setup_e2e_test.go b/test/e2e/tests/setup_e2e_test.go
index ce15319f..06c3d454 100644
--- a/test/e2e/tests/setup_e2e_test.go
+++ b/test/e2e/tests/setup_e2e_test.go
@@ -104,6 +104,11 @@ var _ = BeforeSuite(func() {
 				klog.Fatalf("could not copy scsi_id to containerized directory: %v", err)
 			}
 
+			err = testutils.CopyFile(i, "/lib/udev/google_nvme_id", "/lib/udev_containerized/google_nvme_id")
+			if err != nil {
+				klog.Fatalf("could not copy google_nvme_id to containerized directory: %v", err)
+			}
+
 			klog.Infof("Creating new driver and client for node %s\n", i.GetName())
 			// Create new driver and client
 			testContext, err := testutils.GCEClientAndDriverSetup(i)
diff --git a/test/e2e/tests/single_zone_e2e_test.go b/test/e2e/tests/single_zone_e2e_test.go
index 0e7041fd..89c56098 100644
--- a/test/e2e/tests/single_zone_e2e_test.go
+++ b/test/e2e/tests/single_zone_e2e_test.go
@@ -87,15 +87,13 @@ var _ = Describe("GCE PD CSI Driver", func() {
 		Expect(err).To(BeNil(), "Failed to go through volume lifecycle")
 	})
 
-	It("Should automatically fix symlink errors between /dev/sdx and /dev/by-id if disk is not found", func() {
+	It("Should automatically fix the symlink between /dev/* and /dev/by-id if the disk does not match", func() {
 		testContext := getRandomTestContext()
 
 		p, z, _ := testContext.Instance.GetIdentity()
 		client := testContext.Client
 		instance := testContext.Instance
 
-		// Set-up instance to have scsi_id where we expect it
-
 		// Create Disk
 		volName, volID := createAndValidateUniqueZonalDisk(client, p, z)
 
@@ -124,6 +122,78 @@ var _ = Describe("GCE PD CSI Driver", func() {
 
 		// MESS UP THE symlink
 		devicePaths := mountmanager.NewDeviceUtils().GetDiskByIdPaths(volName, "")
+		for _, devicePath := range devicePaths {
+			err = testutils.RmAll(instance, devicePath)
+			Expect(err).To(BeNil(), "failed to remove /dev/by-id folder")
+			err = testutils.Symlink(instance, "/dev/null", devicePath)
+			Expect(err).To(BeNil(), "failed to add invalid symlink /dev/by-id folder")
+		}
+
+		// Stage Disk
+		stageDir := filepath.Join("/tmp/", volName, "stage")
+		err = client.NodeStageExt4Volume(volID, stageDir)
+		Expect(err).To(BeNil(), "failed to repair /dev/by-id symlink and stage volume")
+
+		// Validate that the link is correct
+		var validated bool
+		for _, devicePath := range devicePaths {
+			validated, err = testutils.ValidateLogicalLinkIsDisk(instance, devicePath, volName)
+			Expect(err).To(BeNil(), "failed to validate link %s is disk %s: %v", stageDir, volName, err)
+			if validated {
+				break
+			}
+		}
+		Expect(validated).To(BeTrue(), "could not find device in %v that links to volume %s", devicePaths, volName)
+
+		defer func() {
+			// Unstage Disk
+			err = client.NodeUnstageVolume(volID, stageDir)
+			if err != nil {
+				klog.Errorf("Failed to unstage volume: %v", err)
+			}
+			fp := filepath.Join("/tmp/", volName)
+			err = testutils.RmAll(instance, fp)
+			if err != nil {
+				klog.Errorf("Failed to rm file path %s: %v", fp, err)
+			}
+		}()
+	})
+
+	It("Should automatically add a symlink between /dev/* and /dev/by-id if disk is not found", func() {
+		testContext := getRandomTestContext()
+
+		p, z, _ := testContext.Instance.GetIdentity()
+		client := testContext.Client
+		instance := testContext.Instance
+
+		// Create Disk
+		volName, volID := createAndValidateUniqueZonalDisk(client, p, z)
+
+		defer func() {
+			// Delete Disk
+			err := client.DeleteVolume(volID)
+			Expect(err).To(BeNil(), "DeleteVolume failed")
+
+			// Validate Disk Deleted
+			_, err = computeService.Disks.Get(p, z, volName).Do()
+			Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found")
+		}()
+
+		// Attach Disk
+		err := client.ControllerPublishVolume(volID, instance.GetNodeID())
+		Expect(err).To(BeNil(), "ControllerPublishVolume failed with error for disk %v on node %v: %v", volID, instance.GetNodeID())
+
+		defer func() {
+			// Detach Disk
+			err = client.ControllerUnpublishVolume(volID, instance.GetNodeID())
+			if err != nil {
+				klog.Errorf("Failed to detach disk: %v", err)
+			}
+
+		}()
+
+		// DELETE THE symlink
+		devicePaths := mountmanager.NewDeviceUtils().GetDiskByIdPaths(volName, "")
 		for _, devicePath := range devicePaths {
 			err = testutils.RmAll(instance, devicePath)
 			Expect(err).To(BeNil(), "failed to remove /dev/by-id folder")
diff --git a/test/e2e/utils/utils.go b/test/e2e/utils/utils.go
index 6074de51..a3068ebe 100644
--- a/test/e2e/utils/utils.go
+++ b/test/e2e/utils/utils.go
@@ -251,36 +251,57 @@ func CopyFile(instance *remote.InstanceInfo, src, dest string) error {
 }
 
 // ValidateLogicalLinkIsDisk takes a symlink location at "link" and finds the
-// link location - it then finds the backing PD using scsi_id and validates that
-// it is the same as diskName
+// link location - it then finds the backing PD using either scsi_id or
+// google_nvme_id (depending on the /dev path) and validates that it is the
+// same as diskName
 func ValidateLogicalLinkIsDisk(instance *remote.InstanceInfo, link, diskName string) (bool, error) {
 	const (
-		scsiPattern = `^0Google\s+PersistentDisk\s+([\S]+)\s*$`
-		sdPattern   = "sd\\w"
+		scsiPattern       = `^0Google\s+PersistentDisk\s+([\S]+)\s*$`
+		sdPattern         = `sd\w+`
+		nvmeSerialPattern = `ID_SERIAL_SHORT=([\S]+)\s*`
+		nvmeDevPattern    = `nvme\w+`
 	)
 	// regex to parse scsi_id output and extract the serial
 	scsiRegex := regexp.MustCompile(scsiPattern)
 	sdRegex := regexp.MustCompile(sdPattern)
+	nvmeSerialRegex := regexp.MustCompile(nvmeSerialPattern)
+	nvmeDevRegex := regexp.MustCompile(nvmeDevPattern)
 
-	devSDX, err := instance.SSH("find", link, "-printf", "'%l'")
+	devFsPath, err := instance.SSH("find", link, "-printf", "'%l'")
 	if err != nil {
-		return false, fmt.Errorf("failed to find %s's symbolic link. Output: %v, errror: %v", link, devSDX, err)
+		return false, fmt.Errorf("failed to find symbolic link for %s. Output: %v, errror: %v", link, devFsPath, err)
 	}
-	if len(devSDX) == 0 {
+	if len(devFsPath) == 0 {
 		return false, nil
 	}
-	sdx := sdRegex.Find([]byte(devSDX))
-	fullDevPath := path.Join("/dev/", string(sdx))
-	scsiIDOut, err := instance.SSH("/lib/udev_containerized/scsi_id", "--page=0x83", "--whitelisted", fmt.Sprintf("--device=%v", fullDevPath))
-	if err != nil {
-		return false, fmt.Errorf("failed to find %s's SCSI ID. Output: %v, errror: %v", devSDX, scsiIDOut, err)
-	}
-	scsiID := scsiRegex.FindStringSubmatch(scsiIDOut)
-	if len(scsiID) == 0 {
-		return false, fmt.Errorf("scsi_id output cannot be parsed: %s", scsiID)
-	}
-	if scsiID[1] != diskName {
-		return false, fmt.Errorf("scsiID %s did not match expected diskName %s", scsiID, diskName)
+	if sdx := sdRegex.FindString(devFsPath); len(sdx) != 0 {
+		fullDevPath := path.Join("/dev/", string(sdx))
+		scsiIDOut, err := instance.SSH("/lib/udev_containerized/scsi_id", "--page=0x83", "--whitelisted", fmt.Sprintf("--device=%v", fullDevPath))
+		if err != nil {
+			return false, fmt.Errorf("failed to find %s's SCSI ID. Output: %v, errror: %v", devFsPath, scsiIDOut, err)
+		}
+		scsiID := scsiRegex.FindStringSubmatch(scsiIDOut)
+		if len(scsiID) == 0 {
+			return false, fmt.Errorf("scsi_id output cannot be parsed: %s. Output: %v", scsiID, scsiIDOut)
+		}
+		if scsiID[1] != diskName {
+			return false, fmt.Errorf("scsiID %s did not match expected diskName %s", scsiID, diskName)
+		}
+		return true, nil
+	} else if nvmex := nvmeDevRegex.FindString(devFsPath); len(nvmex) != 0 {
+		fullDevPath := path.Join("/dev/", string(nvmex))
+		nvmeIDOut, err := instance.SSH("/lib/udev_containerized/google_nvme_id", fmt.Sprintf("-d%v", fullDevPath))
+		if err != nil {
+			return false, fmt.Errorf("failed to find %s's NVME ID. Output: %v, errror: %v", devFsPath, nvmeIDOut, err)
+		}
+		nvmeID := nvmeSerialRegex.FindStringSubmatch(nvmeIDOut)
+		if len(nvmeID) == 0 {
+			return false, fmt.Errorf("google_nvme_id output cannot be parsed: %s. Output: %v", nvmeID, nvmeIDOut)
+		}
+		if nvmeID[1] != diskName {
+			return false, fmt.Errorf("nvmeID %s did not match expected diskName %s", nvmeID, diskName)
+		}
+		return true, nil
 	}
-	return true, nil
+	return false, fmt.Errorf("symlinked disk %s for diskName %s does not match a supported /dev/sd* or /dev/nvme* path", devFsPath, diskName)
 }
diff --git a/test/remote/instance.go b/test/remote/instance.go
index af4f0eea..1c912cb2 100644
--- a/test/remote/instance.go
+++ b/test/remote/instance.go
@@ -91,7 +91,7 @@ func (i *InstanceInfo) CreateOrGetInstance(serviceAccount string) error {
 		return fmt.Errorf("Failed to create firewall rule: %v", err)
 	}
 
-	imageURL := "projects/debian-cloud/global/images/family/debian-9"
+	imageURL := "projects/debian-cloud/global/images/family/debian-11"
 	inst := &compute.Instance{
 		Name:        i.name,
 		MachineType: machineType(i.zone, ""),