Repository: kubernetes-sigs/gcp-compute-persistent-disk-csi-driver

Issue:
Device Cache initialization slowing down csi-node when `--node-name` isn't specified
This PR https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/2141 introduced a new deviceCache that relies on the `--node-name` flag to work. However, nothing prevent the user from not configuring this flag and when this happens, the node pods initialization get delayed significantly because it has to wait for the full backoff:

```
gcp-pd-csi-driver-node-vbl4c csi-plugin I1014 07:32:24.784762       1 main.go:125] Operating compute environment set to: production and computeEndpoint is set to: <nil>
gcp-pd-csi-driver-node-vbl4c csi-plugin I1014 07:32:24.785339       1 main.go:134] Sys info: NumCPU: 15 MAXPROC: 1
gcp-pd-csi-driver-node-vbl4c csi-plugin I1014 07:32:24.785357       1 main.go:139] Driver vendor version v1.21.4-dd.202541
gcp-pd-csi-driver-node-vbl4c csi-plugin I1014 07:32:24.786456       1 mount_linux.go:316] Cannot create temp dir to detect safe 'not mounted' behavior: mkdir /tmp/kubelet-detect-safe-umount3379456297: read-only file system
...
gcp-pd-csi-driver-node-vbl4c csi-plugin I1014 07:32:24.790026       1 request.go:1178] Error in request: resource name may not be empty
gcp-pd-csi-driver-node-vbl4c csi-plugin W1014 07:32:24.790071       1 node.go:37] Error getting node : resource name may not be empty, retrying...
gcp-pd-csi-driver-node-vbl4c csi-plugin I1014 07:32:25.790198       1 request.go:1178] Error in request: resource name may not be empty
gcp-pd-csi-driver-node-vbl4c csi-plugin W1014 07:32:25.790245       1 node.go:37] Error getting node : resource name may not be empty, retrying...
gcp-pd-csi-driver-node-vbl4c csi-plugin I1014 07:32:27.791374       1 request.go:1178] Error in request: resource name may not be empty
gcp-pd-csi-driver-node-vbl4c csi-plugin W1014 07:32:27.791414       1 node.go:37] Error getting node : resource name may not be empty, retrying...
gcp-pd-csi-driver-node-vbl4c csi-plugin I1014 07:32:31.793285       1 request.go:1178] Error in request: resource name may not be empty
gcp-pd-csi-driver-node-vbl4c csi-plugin W1014 07:32:31.793325       1 node.go:37] Error getting node : resource name may not be empty, retrying...
gcp-pd-csi-driver-node-vbl4c csi-plugin I1014 07:32:39.793657       1 request.go:1178] Error in request: resource name may not be empty
gcp-pd-csi-driver-node-vbl4c csi-plugin W1014 07:32:39.793724       1 node.go:37] Error getting node : resource name may not be empty, retrying...
gcp-pd-csi-driver-node-vbl4c csi-plugin E1014 07:32:39.793731       1 node.go:46] Failed to get node  after retries: timed out waiting for the condition
gcp-pd-csi-driver-node-vbl4c csi-plugin W1014 07:32:39.793767       1 main.go:283] Failed to create device cache: failed to get node : timed out waiting for the condition
...
gcp-pd-csi-driver-node-vbl4c csi-plugin I1014 07:32:39.794316       1 gce-pd-driver.go:187] Driver: pd.csi.storage.gke.io
```

That's 15s lost which could be avoided and that's slowing down significantly the rollout of the driver on large clusters.
I know this can be avoided by configuring the flag but I think the situation could also be improved with a saner default.

I have 2 ideas:
1. check the nodeName isn't an empty string before running [NewDeviceCacheForNode](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/2141/files#diff-a81f078f88d79f44c944252605b241dc262580acf27c26d395a981c1b9f6241eR284)
2. or if the deviceCache is needed to be running in all cases, then the flag should become mandatory

What do you think?

Discussion:
@sunnylovestiramisu: /assign @cemakd 

Solution patch:
diff --git a/Dockerfile b/Dockerfile
index aaccf4de..e11278d3 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM --platform=$BUILDPLATFORM golang:1.25.3 AS builder
+FROM --platform=$BUILDPLATFORM golang:1.25.4 AS builder
 
 ARG STAGINGVERSION
 ARG TARGETPLATFORM
diff --git a/Dockerfile.Windows b/Dockerfile.Windows
index b47f95ba..43e6f3d4 100644
--- a/Dockerfile.Windows
+++ b/Dockerfile.Windows
@@ -13,7 +13,7 @@
 # limitations under the License.
 
 ARG BASE_IMAGE
-FROM --platform=$BUILDPLATFORM golang:1.25.3 AS builder
+FROM --platform=$BUILDPLATFORM golang:1.25.4 AS builder
 
 ARG TARGETPLATFORM
 ARG STAGINGVERSION
diff --git a/Dockerfile.debug b/Dockerfile.debug
index 241e3d4d..0735b282 100644
--- a/Dockerfile.debug
+++ b/Dockerfile.debug
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM golang:1.25.3 as builder
+FROM golang:1.25.4 as builder
 WORKDIR /go/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver
 ADD . .
 
diff --git a/cmd/gce-pd-csi-driver/main.go b/cmd/gce-pd-csi-driver/main.go
index f2b2e884..726a6950 100644
--- a/cmd/gce-pd-csi-driver/main.go
+++ b/cmd/gce-pd-csi-driver/main.go
@@ -99,6 +99,8 @@ var (
 
 	diskTopology = flag.Bool("disk-topology", false, "If set to true, the driver will add a disk-type.gke.io/[disk-type] topology label when the StorageClass has the use-allowed-disk-topology parameter set to true. That topology label is included in the Topologies returned in CreateVolumeResponse. This flag is disabled by default.")
 
+	dynamicVolumes = flag.Bool("dynamic-volumes", false, "If set to true, the CSI driver will automatically select a compatible disk type based on the presence of the dynamic-volume parameter and disk types defined in the StorageClass. Disabled by default.")
+
 	diskCacheSyncPeriod = flag.Duration("disk-cache-sync-period", 10*time.Minute, "Period for the disk cache to check the /dev/disk/by-id/ directory and evaluate the symlinks")
 
 	enableDiskSizeValidation = flag.Bool("enable-disk-size-validation", false, "If set to true, the driver will validate that the requested disk size is matches the physical disk size. This flag is disabled by default.")
@@ -257,6 +259,7 @@ func handle() {
 		args := &driver.GCEControllerServerArgs{
 			EnableDiskTopology:       *diskTopology,
 			EnableDiskSizeValidation: *enableDiskSizeValidation,
+			EnableDynamicVolumes:     *dynamicVolumes,
 		}
 
 		controllerServer = driver.NewControllerServer(gceDriver, cloudProvider, initialBackoffDuration, maxBackoffDuration, fallbackRequisiteZones, *enableStoragePoolsFlag, *enableDataCacheFlag, multiZoneVolumeHandleConfig, listVolumesConfig, provisionableDisksConfig, *enableHdHAFlag, args)
@@ -299,6 +302,7 @@ func handle() {
 			SysfsPath:                "/sys",
 			MetricsManager:           metricsManager,
 			DeviceCache:              deviceCache,
+			EnableDynamicVolumes:     *dynamicVolumes,
 		}
 		nodeServer = driver.NewNodeServer(gceDriver, mounter, deviceUtils, meta, statter, nsArgs)
 
diff --git a/pkg/common/utils.go b/pkg/common/utils.go
index 74bbb59a..923c93e0 100644
--- a/pkg/common/utils.go
+++ b/pkg/common/utils.go
@@ -492,6 +492,11 @@ func MapNumber(vCPUs int64, limitMap []constants.MachineHyperdiskLimit) int64 {
 	return 15
 }
 
+// HasDiskTypeLabelKeyPrefix checks if the label key starts with the DiskTypeKeyPrefix.
+func HasDiskTypeLabelKeyPrefix(labelKey string) bool {
+	return strings.HasPrefix(labelKey, constants.DiskTypeKeyPrefix)
+}
+
 func DiskTypeLabelKey(diskType string) string {
 	return fmt.Sprintf("%s/%s", constants.DiskTypeKeyPrefix, diskType)
 }
diff --git a/pkg/deviceutils/device-utils.go b/pkg/deviceutils/device-utils.go
index 1f7a8fde..53e1881c 100644
--- a/pkg/deviceutils/device-utils.go
+++ b/pkg/deviceutils/device-utils.go
@@ -290,7 +290,10 @@ func (m *deviceUtils) VerifyDevicePath(devicePaths []string, deviceName string)
 	})
 
 	if err != nil {
-		return "", fmt.Errorf("failed to find and re-link disk %s with udevadm after retrying for %v: %w", deviceName, pollTimeout, err)
+		klog.Warningf("For device %s udevadmin failed: %v. Trying to manually link", deviceName, err)
+		if err := manuallySetDevicePath(deviceName); err != nil {
+			return "", fmt.Errorf("failed to manually set link for disk %s: %w", deviceName, err)
+		}
 	}
 
 	return devicePath, nil
@@ -338,11 +341,11 @@ func findAvailableDevFsPaths() ([]string, error) {
 	return append(diskSDPaths, diskNvmePaths...), nil
 }
 
-func udevadmTriggerForDiskIfExists(deviceName string) error {
+func findDevice(deviceName string) (string, string, error) {
 	devFsPathToSerial := map[string]string{}
 	devFsPaths, err := findAvailableDevFsPaths()
 	if err != nil {
-		return err
+		return "", "", err
 	}
 	for _, devFsPath := range devFsPaths {
 		devFsSerial, err := getDevFsSerial(devFsPath)
@@ -355,17 +358,33 @@ func udevadmTriggerForDiskIfExists(deviceName string) error {
 		klog.V(4).Infof("device path %s, serial number %v", devFsPath, devFsSerial)
 		devFsPathToSerial[devFsPath] = devFsSerial
 		if devFsSerial == deviceName {
-			// Found the disk that we're looking for so run a trigger on it
-			// to resolve its /dev/by-id/ path
-			klog.Warningf("udevadm --trigger running to fix disk at path %s which has serial number %s", devFsPath, devFsSerial)
-			err := udevadmChangeToDrive(devFsPath)
-			if err != nil {
-				return fmt.Errorf("udevadm --trigger failed to fix device path %s which has serial number %s: %w", devFsPath, devFsSerial, err)
-			}
-			return nil
+			return devFsPath, devFsSerial, nil
 		}
 	}
-	return fmt.Errorf("udevadm --trigger requested to fix disk %s but no such disk was found in device path %v", deviceName, devFsPathToSerial)
+	return "", "", fmt.Errorf("udevadm --trigger requested to fix disk %s but no such disk was found in device path %v", deviceName, devFsPathToSerial)
+}
+
+func manuallySetDevicePath(deviceName string) error {
+	devFsPath, devFsSerial, err := findDevice(deviceName)
+	if err != nil {
+		return err
+	}
+	return os.Symlink(devFsPath, path.Join(diskByIdPath, diskGooglePrefix+devFsSerial))
+}
+
+func udevadmTriggerForDiskIfExists(deviceName string) error {
+	devFsPath, devFsSerial, err := findDevice(deviceName)
+	if err != nil {
+		return err
+	}
+	// Found the disk that we're looking for so run a trigger on it
+	// to resolve its /dev/by-id/ path
+	klog.Warningf("udevadm --trigger running to fix disk at path %s which has serial number %s", devFsPath, devFsSerial)
+	err = udevadmChangeToDrive(devFsPath)
+	if err != nil {
+		return fmt.Errorf("udevadm --trigger failed to fix device path %s which has serial number %s: %w", devFsPath, devFsSerial, err)
+	}
+	return nil
 }
 
 // Calls "udevadm trigger --action=change" on the specified drive. drivePath
diff --git a/pkg/gce-pd-csi-driver/controller.go b/pkg/gce-pd-csi-driver/controller.go
index 6c226d78..b29721e1 100644
--- a/pkg/gce-pd-csi-driver/controller.go
+++ b/pkg/gce-pd-csi-driver/controller.go
@@ -132,6 +132,7 @@ type GCEControllerServer struct {
 type GCEControllerServerArgs struct {
 	EnableDiskTopology       bool
 	EnableDiskSizeValidation bool
+	EnableDynamicVolumes     bool
 }
 
 type MultiZoneVolumeHandleConfig struct {
@@ -320,7 +321,7 @@ func (gceCS *GCEControllerServer) createVolumeInternal(ctx context.Context, req
 	var err error
 	// Apply Parameters (case-insensitive). We leave validation of
 	// the values to the cloud provider.
-	params, dataCacheParams, err := gceCS.parameterProcessor().ExtractAndDefaultParameters(req.GetParameters(), gceCS.Driver.extraVolumeLabels, gceCS.enableDataCache, gceCS.Driver.extraTags)
+	params, dataCacheParams, err := gceCS.parameterProcessor().ExtractAndDefaultParameters(req.GetParameters())
 	metrics.UpdateRequestMetadataFromParams(ctx, params)
 	if err != nil {
 		return nil, status.Errorf(codes.InvalidArgument, "failed to extract parameters: %v", err.Error())
@@ -1349,6 +1350,9 @@ func (gceCS *GCEControllerServer) parameterProcessor() *parameters.ParameterProc
 		EnableMultiZone:    gceCS.multiZoneVolumeHandleConfig.Enable,
 		EnableHdHA:         gceCS.enableHdHA,
 		EnableDiskTopology: gceCS.EnableDiskTopology,
+		ExtraVolumeLabels:  gceCS.Driver.extraVolumeLabels,
+		EnableDataCache:    gceCS.enableDataCache,
+		ExtraTags:          gceCS.Driver.extraTags,
 	}
 }
 
@@ -1394,7 +1398,7 @@ func (gceCS *GCEControllerServer) ValidateVolumeCapabilities(ctx context.Context
 	}
 
 	// Validate the disk parameters match the disk we GET
-	params, _, err := gceCS.parameterProcessor().ExtractAndDefaultParameters(req.GetParameters(), gceCS.Driver.extraVolumeLabels, gceCS.enableDataCache, gceCS.Driver.extraTags)
+	params, _, err := gceCS.parameterProcessor().ExtractAndDefaultParameters(req.GetParameters())
 	if err != nil {
 		return nil, status.Errorf(codes.InvalidArgument, "failed to extract parameters: %v", err.Error())
 	}
diff --git a/pkg/gce-pd-csi-driver/gce-pd-driver.go b/pkg/gce-pd-csi-driver/gce-pd-driver.go
index c1740df8..9e4ff02e 100644
--- a/pkg/gce-pd-csi-driver/gce-pd-driver.go
+++ b/pkg/gce-pd-csi-driver/gce-pd-driver.go
@@ -160,6 +160,7 @@ func NewNodeServer(gceDriver *GCEDriver, mounter *mount.SafeFormatAndMount, devi
 		SysfsPath:                args.SysfsPath,
 		metricsManager:           args.MetricsManager,
 		DeviceCache:              args.DeviceCache,
+		EnableDynamicVolumes:     args.EnableDynamicVolumes,
 	}
 }
 
diff --git a/pkg/gce-pd-csi-driver/node.go b/pkg/gce-pd-csi-driver/node.go
index a0577400..3412cfe4 100644
--- a/pkg/gce-pd-csi-driver/node.go
+++ b/pkg/gce-pd-csi-driver/node.go
@@ -32,9 +32,11 @@ import (
 
 	csi "github.com/container-storage-interface/spec/lib/go/csi"
 
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	volumehelpers "k8s.io/cloud-provider/volume/helpers"
 	"k8s.io/klog/v2"
 	"k8s.io/mount-utils"
-
 	"sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/common"
 	"sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/constants"
 	"sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/deviceutils"
@@ -83,6 +85,8 @@ type GCENodeServer struct {
 	metricsManager *metrics.MetricsManager
 	// A cache of the device paths for the volumes that are attached to the node.
 	DeviceCache *linkcache.DeviceCache
+
+	EnableDynamicVolumes bool
 }
 
 type NodeServerArgs struct {
@@ -101,6 +105,8 @@ type NodeServerArgs struct {
 
 	MetricsManager *metrics.MetricsManager
 	DeviceCache    *linkcache.DeviceCache
+
+	EnableDynamicVolumes bool
 }
 
 var _ csi.NodeServer = &GCENodeServer{}
@@ -451,14 +457,22 @@ func (ns *GCENodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStage
 	}
 
 	// If a disk size is provided in the publish context, ensure it matches the actual device size.
-	if expectedSize := req.GetPublishContext()[constants.ContextDiskSizeGB]; expectedSize != "" {
-		actualSize, err := getBlockSizeBytes(devicePath, ns.Mounter)
+	expectedDiskSizeStr := req.GetPublishContext()[constants.ContextDiskSizeGB]
+	if expectedSizeGib, err := strconv.ParseInt(expectedDiskSizeStr, 10, 64); err == nil {
+		actualSizeBytes, err := getBlockSizeBytes(devicePath, ns.Mounter)
 		if err != nil {
 			return nil, status.Error(codes.Internal, fmt.Sprintf("failed to get block size for '%s': %v", devicePath, err.Error()))
 		}
-		if expectedSize != strconv.FormatInt(actualSize, 10) {
-			return nil, status.Error(codes.Internal, fmt.Sprintf("expected block size %q, got %q", expectedSize, strconv.FormatInt(actualSize, 10)))
+		actualSizeGib, err := volumehelpers.RoundUpToGiB(*resource.NewQuantity(actualSizeBytes, resource.DecimalSI))
+		if err != nil {
+			return nil, status.Error(codes.Internal, fmt.Sprintf("failed to convert block size '%d' to GiB: %v", actualSizeBytes, err.Error()))
+		}
+
+		if expectedSizeGib > actualSizeGib {
+			return nil, status.Error(codes.Internal, fmt.Sprintf("expected block size %q, got %q", expectedSizeGib, actualSizeGib))
 		}
+	} else {
+		klog.V(4).Infof("skipping disk size validation due to invalid expected size: %v", err)
 	}
 
 	err = ns.formatAndMount(devicePath, stagingTargetPath, fstype, options, ns.Mounter)
@@ -717,9 +731,25 @@ func (ns *GCENodeServer) NodeGetInfo(ctx context.Context, req *csi.NodeGetInfoRe
 		Segments: map[string]string{constants.TopologyKeyZone: ns.MetadataService.GetZone()},
 	}
 
+	node, err := k8sclient.GetNodeWithRetry(ctx, ns.MetadataService.GetName())
+	if err != nil {
+		klog.Errorf("Failed to get node %s: %v. The error is ignored so that the driver can register", ns.MetadataService.GetName(), err.Error())
+	}
+
+	if ns.EnableDynamicVolumes {
+		labels, err := ns.getDiskTypeLabels(node)
+		if err != nil {
+			klog.Errorf("Failed to fetch disk type topology labels: %v", err)
+		}
+
+		for k, v := range labels {
+			top.Segments[k] = v
+		}
+	}
+
 	nodeID := common.CreateNodeID(ns.MetadataService.GetProject(), ns.MetadataService.GetZone(), ns.MetadataService.GetName())
 
-	volumeLimits, err := ns.GetVolumeLimits(ctx)
+	volumeLimits, err := ns.getVolumeLimits(ctx, node)
 	if err != nil {
 		klog.Errorf("GetVolumeLimits failed: %v. The error is ignored so that the driver can register", err.Error())
 		// No error should be returned from NodeGetInfo, otherwise the driver will not register
@@ -881,7 +911,7 @@ func (ns *GCENodeServer) NodeExpandVolume(ctx context.Context, req *csi.NodeExpa
 	}, nil
 }
 
-func (ns *GCENodeServer) GetVolumeLimits(ctx context.Context) (int64, error) {
+func (ns *GCENodeServer) getVolumeLimits(ctx context.Context, node *corev1.Node) (int64, error) {
 	// Machine-type format: n1-type-CPUS or custom-CPUS-RAM or f1/g1-type
 	machineType := ns.MetadataService.GetMachineType()
 
@@ -893,7 +923,7 @@ func (ns *GCENodeServer) GetVolumeLimits(ctx context.Context) (int64, error) {
 	}
 
 	// Get attach limit override from label
-	attachLimitOverride, err := GetAttachLimitsOverrideFromNodeLabel(ctx, ns.MetadataService.GetName())
+	attachLimitOverride, err := getAttachLimitsOverrideFromNodeLabel(node)
 	if err == nil && attachLimitOverride > 0 && attachLimitOverride < 128 {
 		return attachLimitOverride, nil
 	} else {
@@ -955,11 +985,12 @@ func (ns *GCENodeServer) GetVolumeLimits(ctx context.Context) (int64, error) {
 	return volumeLimitBig, nil
 }
 
-func GetAttachLimitsOverrideFromNodeLabel(ctx context.Context, nodeName string) (int64, error) {
-	node, err := k8sclient.GetNodeWithRetry(ctx, nodeName)
-	if err != nil {
-		return 0, err
+func getAttachLimitsOverrideFromNodeLabel(node *corev1.Node) (int64, error) {
+	// If then node is nil, return 0 which means there is no override
+	if node == nil {
+		return 0, fmt.Errorf("node is nil")
 	}
+
 	if val, found := node.GetLabels()[fmt.Sprintf(constants.NodeRestrictionLabelPrefix, constants.AttachLimitOverrideLabel)]; found {
 		attachLimitOverrideForNode, err := strconv.ParseInt(val, 10, 64)
 		if err != nil {
@@ -970,3 +1001,17 @@ func GetAttachLimitsOverrideFromNodeLabel(ctx context.Context, nodeName string)
 	}
 	return 0, nil
 }
+
+func (ns *GCENodeServer) getDiskTypeLabels(node *corev1.Node) (map[string]string, error) {
+	if node == nil {
+		return nil, fmt.Errorf("node is nil")
+	}
+	lbls := make(map[string]string)
+	for k, v := range node.GetLabels() {
+		if common.HasDiskTypeLabelKeyPrefix(k) {
+			lbls[k] = v
+		}
+	}
+
+	return lbls, nil
+}
diff --git a/pkg/k8sclient/node.go b/pkg/k8sclient/node.go
index 1c4fa9b8..7a67c061 100644
--- a/pkg/k8sclient/node.go
+++ b/pkg/k8sclient/node.go
@@ -2,6 +2,7 @@ package k8sclient
 
 import (
 	"context"
+	"fmt"
 	"time"
 
 	v1 "k8s.io/api/core/v1"
@@ -13,6 +14,9 @@ import (
 )
 
 func GetNodeWithRetry(ctx context.Context, nodeName string) (*v1.Node, error) {
+	if nodeName == "" {
+		return nil, fmt.Errorf("node name is empty")
+	}
 	cfg, err := rest.InClusterConfig()
 	if err != nil {
 		return nil, err
diff --git a/pkg/parameters/parameters.go b/pkg/parameters/parameters.go
index 3734f14d..bfab5a27 100644
--- a/pkg/parameters/parameters.go
+++ b/pkg/parameters/parameters.go
@@ -29,9 +29,7 @@ import (
 
 // ExtractAndDefaultParameters will take the relevant parameters from a map and
 // put them into a well defined struct making sure to default unspecified fields.
-// extraVolumeLabels are added as labels; if there are also labels specified in
-// parameters, any matching extraVolumeLabels will be overridden.
-func (pp *ParameterProcessor) ExtractAndDefaultParameters(parameters map[string]string, extraVolumeLabels map[string]string, enableDataCache bool, extraTags map[string]string) (DiskParameters, DataCacheParameters, error) {
+func (pp *ParameterProcessor) ExtractAndDefaultParameters(parameters map[string]string) (DiskParameters, DataCacheParameters, error) {
 	p := DiskParameters{
 		DiskType:             "pd-standard",           // Default
 		ReplicationType:      replicationTypeNone,     // Default
@@ -43,15 +41,15 @@ func (pp *ParameterProcessor) ExtractAndDefaultParameters(parameters map[string]
 
 	// Set data cache mode default
 	d := DataCacheParameters{}
-	if enableDataCache && parameters[ParameterKeyDataCacheSize] != "" {
+	if pp.EnableDataCache && parameters[ParameterKeyDataCacheSize] != "" {
 		d.DataCacheMode = constants.DataCacheModeWriteThrough
 	}
 
-	for k, v := range extraVolumeLabels {
+	for k, v := range pp.ExtraVolumeLabels {
 		p.Labels[k] = v
 	}
 
-	for k, v := range extraTags {
+	for k, v := range pp.ExtraTags {
 		p.ResourceTags[k] = v
 	}
 
@@ -64,7 +62,7 @@ func (pp *ParameterProcessor) ExtractAndDefaultParameters(parameters map[string]
 		case ParameterKeyType:
 			if v != "" {
 				p.DiskType = strings.ToLower(v)
-				if !pp.EnableHdHA && p.DiskType == DiskTypeHdHA {
+				if pp.isHDHADisabled() && p.DiskType == DiskTypeHdHA {
 					return p, d, fmt.Errorf("parameters contain invalid disk type %s", DiskTypeHdHA)
 				}
 			}
@@ -128,7 +126,7 @@ func (pp *ParameterProcessor) ExtractAndDefaultParameters(parameters map[string]
 
 			p.EnableConfidentialCompute = paramEnableConfidentialCompute
 		case ParameterKeyStoragePools:
-			if !pp.EnableStoragePools {
+			if pp.isStoragePoolDisabled() {
 				return p, d, fmt.Errorf("parameters contains invalid option %q", ParameterKeyStoragePools)
 			}
 			storagePools, err := ParseStoragePools(v)
@@ -137,8 +135,8 @@ func (pp *ParameterProcessor) ExtractAndDefaultParameters(parameters map[string]
 			}
 			p.StoragePools = storagePools
 		case ParameterKeyDataCacheSize:
-			if !enableDataCache {
-				return p, d, fmt.Errorf("data caching enabled: %v; parameters contains invalid option %q", enableDataCache, ParameterKeyDataCacheSize)
+			if pp.isDataCacheDisabled() {
+				return p, d, fmt.Errorf("data caching enabled: %v; parameters contains invalid option %q", pp.EnableDataCache, ParameterKeyDataCacheSize)
 			}
 
 			paramDataCacheSize, err := convert.ConvertGiStringToInt64(v)
@@ -150,8 +148,8 @@ func (pp *ParameterProcessor) ExtractAndDefaultParameters(parameters map[string]
 			}
 			d.DataCacheSize = strconv.FormatInt(paramDataCacheSize, 10)
 		case ParameterKeyDataCacheMode:
-			if !enableDataCache {
-				return p, d, fmt.Errorf("data caching enabled %v; parameters contains invalid option %q", enableDataCache, ParameterKeyDataCacheMode)
+			if pp.isDataCacheDisabled() {
+				return p, d, fmt.Errorf("data caching enabled %v; parameters contains invalid option %q", pp.EnableDataCache, ParameterKeyDataCacheMode)
 			}
 			if err := ValidateDataCacheMode(v); err != nil {
 				return p, d, fmt.Errorf("parameters contains invalid option: %s: %w", ParameterKeyDataCacheMode, err)
@@ -162,7 +160,7 @@ func (pp *ParameterProcessor) ExtractAndDefaultParameters(parameters map[string]
 				return p, d, err
 			}
 		case ParameterKeyEnableMultiZoneProvisioning:
-			if !pp.EnableMultiZone {
+			if pp.isMultiZoneDisabled() {
 				return p, d, fmt.Errorf("parameters contains invalid option %q", ParameterKeyEnableMultiZoneProvisioning)
 			}
 			paramEnableMultiZoneProvisioning, err := convert.ConvertStringToBool(v)
@@ -179,7 +177,7 @@ func (pp *ParameterProcessor) ExtractAndDefaultParameters(parameters map[string]
 				p.AccessMode = v
 			}
 		case ParameterKeyUseAllowedDiskTopology:
-			if !pp.EnableDiskTopology {
+			if pp.isDiskTopologyDisabled() {
 				klog.Warningf("parameters contains invalid option %q when disk topology is not enabled", ParameterKeyUseAllowedDiskTopology)
 				continue
 			}
diff --git a/pkg/parameters/sanitize.go b/pkg/parameters/sanitize.go
new file mode 100644
index 00000000..ed19784b
--- /dev/null
+++ b/pkg/parameters/sanitize.go
@@ -0,0 +1,55 @@
+package parameters
+
+import "strings"
+
+// Map of sanitizers for parameters that apply to only a subset of disk types. Each sanitizeer
+// should only be responsible for a single parameter.
+var sanitizers = map[string]sanitizer{
+	"ReplicationType": func(dp *DiskParameters) {
+		// Only sanitize replication type if parameter has been specified.
+		if isHD(dp.DiskType) && dp.ReplicationType != "" {
+			dp.ReplicationType = replicationTypeNone
+		}
+	},
+	"ProvisionedIOPSOnCreate": func(dp *DiskParameters) {
+		if isPD(dp.DiskType) && dp.DiskType != "pd-extreme" {
+			dp.ProvisionedIOPSOnCreate = 0
+		}
+	},
+	"ProvisionedThroughputOnCreate": func(dp *DiskParameters) {
+		if isPD(dp.DiskType) {
+			dp.ProvisionedThroughputOnCreate = 0
+		}
+	},
+	"StoragePools": func(dp *DiskParameters) {
+		if isPD(dp.DiskType) {
+			dp.StoragePools = nil
+		}
+	},
+	"ForceAttach": func(dp *DiskParameters) {
+		if isHD(dp.DiskType) {
+			dp.ForceAttach = false
+		}
+	},
+}
+
+func isHD(diskType string) bool {
+	return strings.HasPrefix(diskType, "hyperdisk-")
+}
+
+func isPD(diskType string) bool {
+	return strings.HasPrefix(diskType, "pd-")
+}
+
+type sanitizer func(dp *DiskParameters)
+
+// sanitizeDiskParameters removes any parameters that are not applicable to the specified disk type,
+// and is intended to only be used for dynamic volumes.
+func sanitizeDiskParameters(dp *DiskParameters) {
+	if dp == nil {
+		return
+	}
+	for _, sanitizer := range sanitizers {
+		sanitizer(dp)
+	}
+}
diff --git a/pkg/parameters/types.go b/pkg/parameters/types.go
index e8331e45..2a27eb92 100644
--- a/pkg/parameters/types.go
+++ b/pkg/parameters/types.go
@@ -42,6 +42,7 @@ type DiskParameters struct {
 	// Values: {bool}
 	// Default: false
 	EnableConfidentialCompute bool
+	// Values: {bool}
 	// Default: false
 	ForceAttach bool
 	// Values: {[]string}
@@ -56,7 +57,7 @@ type DiskParameters struct {
 	// Values: READ_WRITE_SINGLE, READ_ONLY_MANY, READ_WRITE_MANY
 	// Default: READ_WRITE_SINGLE
 	AccessMode string
-	// Values {}
+	// Values: {bool}
 	// Default: false
 	UseAllowedDiskTopology bool
 }
@@ -81,6 +82,29 @@ type ParameterProcessor struct {
 	EnableMultiZone    bool
 	EnableHdHA         bool
 	EnableDiskTopology bool
+	ExtraVolumeLabels  map[string]string
+	EnableDataCache    bool
+	ExtraTags          map[string]string
+}
+
+func (pp *ParameterProcessor) isHDHADisabled() bool {
+	return !pp.EnableHdHA
+}
+
+func (pp *ParameterProcessor) isStoragePoolDisabled() bool {
+	return !pp.EnableStoragePools
+}
+
+func (pp *ParameterProcessor) isDataCacheDisabled() bool {
+	return !pp.EnableDataCache
+}
+
+func (pp *ParameterProcessor) isMultiZoneDisabled() bool {
+	return !pp.EnableMultiZone
+}
+
+func (pp *ParameterProcessor) isDiskTopologyDisabled() bool {
+	return !pp.EnableDiskTopology
 }
 
 type ModifyVolumeParameters struct {



Test patch:
diff --git a/pkg/gce-pd-csi-driver/node_test.go b/pkg/gce-pd-csi-driver/node_test.go
index 23347f22..f7987924 100644
--- a/pkg/gce-pd-csi-driver/node_test.go
+++ b/pkg/gce-pd-csi-driver/node_test.go
@@ -21,6 +21,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"sort"
 	"strings"
 	"testing"
 	"time"
@@ -32,6 +33,9 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
+	"google.golang.org/protobuf/testing/protocmp"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/mount-utils"
 	"sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/constants"
 	"sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/deviceutils"
@@ -237,11 +241,11 @@ func TestNodeGetVolumeStats(t *testing.T) {
 func TestNodeGetVolumeLimits(t *testing.T) {
 	gceDriver := getTestGCEDriver(t)
 	ns := gceDriver.ns
-	req := &csi.NodeGetInfoRequest{}
 
 	testCases := []struct {
 		name           string
 		machineType    string
+		node           *corev1.Node
 		expVolumeLimit int64
 		expectError    bool
 	}{
@@ -431,6 +435,43 @@ func TestNodeGetVolumeLimits(t *testing.T) {
 			name:           "a4x-medgpu-nolssd", // does not exist, testing edge case
 			machineType:    "a4x-medgpu-nolssd",
 			expVolumeLimit: volumeLimitBig,
+			expectError:    true,
+		},
+		{
+			name:        "attach limit override",
+			machineType: "n1-standard-1",
+			node: &corev1.Node{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: map[string]string{
+						fmt.Sprintf(constants.NodeRestrictionLabelPrefix, constants.AttachLimitOverrideLabel): "63",
+					},
+				},
+			},
+			expVolumeLimit: 63,
+		},
+		{
+			name:        "invalid attach limit override",
+			machineType: "n1-standard-1",
+			node: &corev1.Node{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: map[string]string{
+						fmt.Sprintf(constants.NodeRestrictionLabelPrefix, constants.AttachLimitOverrideLabel): "invalid",
+					},
+				},
+			},
+			expVolumeLimit: volumeLimitBig,
+		},
+		{
+			name:        "attach limit override out of bounds",
+			machineType: "n1-standard-1",
+			node: &corev1.Node{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: map[string]string{
+						fmt.Sprintf(constants.NodeRestrictionLabelPrefix, constants.AttachLimitOverrideLabel): "9999",
+					},
+				},
+			},
+			expVolumeLimit: volumeLimitBig,
 		},
 	}
 
@@ -438,18 +479,15 @@ func TestNodeGetVolumeLimits(t *testing.T) {
 		t.Logf("Test case: %s", tc.name)
 
 		metadataservice.SetMachineType(tc.machineType)
-		res, err := ns.NodeGetInfo(context.Background(), req)
+		volumeLimit, err := ns.getVolumeLimits(context.Background(), tc.node)
 		if err != nil && !tc.expectError {
 			t.Fatalf("Failed to get node info: %v", err)
 		}
 
-		volumeLimit := res.GetMaxVolumesPerNode()
 		if volumeLimit != tc.expVolumeLimit {
 			t.Fatalf("Expected volume limit: %v, got %v, for machine-type: %v",
 				tc.expVolumeLimit, volumeLimit, tc.machineType)
 		}
-
-		t.Logf("Get node info: %v", res)
 	}
 }
 
@@ -1125,14 +1163,49 @@ func TestNodeStageVolume(t *testing.T) {
 			},
 		},
 		{
-			name: "Valid request with disk size check",
+			name: "Valid request disk size matches expected size",
+			req: &csi.NodeStageVolumeRequest{
+				VolumeId:          volumeID,
+				StagingTargetPath: stagingPath,
+				VolumeCapability:  stdVolCap,
+				PublishContext:    map[string]string{constants.ContextDiskSizeGB: "6"},
+			},
+			deviceSize:   6442450944,
+			blockExtSize: 1,
+			readonlyBit:  "1",
+			expResize:    false,
+			expCommandList: []fakeCmd{
+				{
+					cmd:    "blockdev",
+					args:   "--getsize64 /dev/disk/fake-path",
+					stdout: "%v",
+				},
+				{
+					cmd:    "blkid",
+					args:   "-p -s TYPE -s PTTYPE -o export /dev/disk/fake-path",
+					stdout: "DEVNAME=/dev/sdb\nTYPE=%v",
+				},
+				{
+					cmd:    "fsck",
+					args:   "-a /dev/disk/fake-path",
+					stdout: "",
+				},
+				{
+					cmd:    "blockdev",
+					args:   "--getro /dev/disk/fake-path",
+					stdout: "%v",
+				},
+			},
+		},
+		{
+			name: "Valid request disk size exceeds expected size",
 			req: &csi.NodeStageVolumeRequest{
 				VolumeId:          volumeID,
 				StagingTargetPath: stagingPath,
 				VolumeCapability:  stdVolCap,
 				PublishContext:    map[string]string{constants.ContextDiskSizeGB: "1"},
 			},
-			deviceSize:   1,
+			deviceSize:   6442450944,
 			blockExtSize: 1,
 			readonlyBit:  "1",
 			expResize:    false,
@@ -1246,7 +1319,7 @@ func TestNodeStageVolume(t *testing.T) {
 				VolumeCapability:  stdVolCap,
 				PublishContext:    map[string]string{constants.ContextDiskSizeGB: "10"},
 			},
-			deviceSize: 5,
+			deviceSize: 6442450944,
 			expErrCode: codes.Internal,
 			expCommandList: []fakeCmd{
 				{
@@ -1702,3 +1775,119 @@ func TestBlockingFormatAndMount(t *testing.T) {
 	gceDriver := getTestBlockingFormatAndMountGCEDriver(t, readyToExecute)
 	runBlockingFormatAndMount(t, gceDriver, readyToExecute)
 }
+
+func TestGetDiskTypeLabels(t *testing.T) {
+	const (
+		nodeName = "test-node"
+		diskA    = constants.DiskTypeKeyPrefix + "/disk-a"
+		diskB    = constants.DiskTypeKeyPrefix + "/disk-b"
+	)
+
+	testCases := []struct {
+		desc      string
+		node      *corev1.Node
+		want      []string
+		wantError bool
+	}{
+		{
+			desc: "no topology labels",
+			node: &corev1.Node{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:   nodeName,
+					Labels: map[string]string{"foo": "bar"},
+				},
+			},
+			want: nil,
+		},
+		{
+			desc: "multiple topology labels",
+			node: &corev1.Node{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: nodeName,
+					Labels: map[string]string{
+						diskA: "true",
+						diskB: "true",
+					},
+				},
+			},
+			want: []string{diskA, diskB},
+		},
+		{
+			desc:      "node not found",
+			node:      nil,
+			wantError: true,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.desc, func(t *testing.T) {
+			gceDriver := getTestGCEDriverWithCustomMounter(t, mountmanager.NewFakeSafeMounter(), &NodeServerArgs{})
+			ns := gceDriver.ns
+
+			lbls, err := ns.getDiskTypeLabels(tc.node)
+			if tc.wantError {
+				if err == nil {
+					t.Fatalf("expected error but got none")
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("unexpected error: %v", err)
+			}
+
+			var got []string
+			for key := range lbls {
+				got = append(got, key)
+			}
+			sort.Strings(got)
+
+			if diff := cmp.Diff(tc.want, got); diff != "" {
+				t.Fatalf("unexpected topology labels (-want +got):\n%s", diff)
+			}
+		})
+	}
+}
+
+func TestNodeGetInfo(t *testing.T) {
+	const (
+		machineType = "n1-standard-4"
+		zone        = "us-central1-b"
+		name        = "test-node"
+	)
+	tests := []struct {
+		desc string
+		want *csi.NodeGetInfoResponse
+	}{
+		{
+			desc: "success",
+			want: &csi.NodeGetInfoResponse{
+				NodeId:            fmt.Sprintf("projects/test-project/zones/%s/instances/%s", zone, name),
+				MaxVolumesPerNode: volumeLimitBig,
+				AccessibleTopology: &csi.Topology{
+					Segments: map[string]string{
+						constants.TopologyKeyZone: zone,
+					},
+				},
+			},
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.desc, func(t *testing.T) {
+			gceDriver := getTestGCEDriver(t)
+			ns := gceDriver.ns
+			req := &csi.NodeGetInfoRequest{}
+			metadataservice.SetMachineType(machineType)
+			metadataservice.SetZone(zone)
+			metadataservice.SetName(node)
+
+			got, err := ns.NodeGetInfo(context.Background(), req)
+			if err != nil {
+				t.Fatalf("Failed to get node info: %v", err)
+			}
+
+			if diff := cmp.Diff(tc.want, got, protocmp.Transform()); diff != "" {
+				t.Fatalf("NodeGetInfo() returned unexpected diff (-want +got):\n%s", diff)
+			}
+		})
+	}
+}
diff --git a/pkg/parameters/parameters_test.go b/pkg/parameters/parameters_test.go
index 23acf8cc..322eb253 100644
--- a/pkg/parameters/parameters_test.go
+++ b/pkg/parameters/parameters_test.go
@@ -543,8 +543,11 @@ func TestExtractAndDefaultParameters(t *testing.T) {
 				EnableMultiZone:    tc.enableMultiZone,
 				EnableHdHA:         tc.enableHdHA,
 				EnableDiskTopology: tc.enableDiskTopology,
+				ExtraVolumeLabels:  tc.labels,
+				EnableDataCache:    tc.enableDataCache,
+				ExtraTags:          tc.extraTags,
 			}
-			p, d, err := pp.ExtractAndDefaultParameters(tc.parameters, tc.labels, tc.enableDataCache, tc.extraTags)
+			p, d, err := pp.ExtractAndDefaultParameters(tc.parameters)
 			if gotErr := err != nil; gotErr != tc.expectErr {
 				t.Fatalf("ExtractAndDefaultParameters(%+v) = %v; expectedErr: %v", tc.parameters, err, tc.expectErr)
 			}
diff --git a/pkg/parameters/sanitize_test.go b/pkg/parameters/sanitize_test.go
new file mode 100644
index 00000000..7c690997
--- /dev/null
+++ b/pkg/parameters/sanitize_test.go
@@ -0,0 +1,201 @@
+package parameters
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+)
+
+func TestSanitizeDiskParameters(t *testing.T) {
+	tests := []struct {
+		desc       string
+		parameters *DiskParameters
+		want       *DiskParameters
+	}{
+		{
+			desc:       "nil parameters",
+			parameters: nil,
+		},
+		{
+			desc:       "empty parameters unchanged",
+			parameters: &DiskParameters{},
+			want:       &DiskParameters{},
+		},
+		{
+			desc: "general parameters unchanged PD",
+			parameters: &DiskParameters{
+				DiskType:                  "pd-balanced",
+				DiskEncryptionKMSKey:      "fake-kms-key",
+				Tags:                      map[string]string{"fake-tag": "fake-resource-tag-value"},
+				Labels:                    map[string]string{"fake-label": "fake-label-value"},
+				EnableConfidentialCompute: true,
+				ResourceTags:              map[string]string{"fake-resource-tag": "fake-resource-tag-value"},
+				AccessMode:                "READ_WRITE_SINGLE",
+				UseAllowedDiskTopology:    true,
+			},
+			want: &DiskParameters{
+				DiskType:                  "pd-balanced",
+				DiskEncryptionKMSKey:      "fake-kms-key",
+				Tags:                      map[string]string{"fake-tag": "fake-resource-tag-value"},
+				Labels:                    map[string]string{"fake-label": "fake-label-value"},
+				EnableConfidentialCompute: true,
+				ResourceTags:              map[string]string{"fake-resource-tag": "fake-resource-tag-value"},
+				AccessMode:                "READ_WRITE_SINGLE",
+				UseAllowedDiskTopology:    true,
+			},
+		},
+		{
+			desc: "general parameters unchanged HD",
+			parameters: &DiskParameters{
+				DiskType:                  "hyperdisk-balanced",
+				DiskEncryptionKMSKey:      "fake-kms-key",
+				Tags:                      map[string]string{"fake-tag": "fake-resource-tag-value"},
+				Labels:                    map[string]string{"fake-label": "fake-label-value"},
+				EnableConfidentialCompute: true,
+				ResourceTags:              map[string]string{"fake-resource-tag": "fake-resource-tag-value"},
+				AccessMode:                "READ_WRITE_SINGLE",
+				UseAllowedDiskTopology:    true,
+			},
+			want: &DiskParameters{
+				DiskType:                  "hyperdisk-balanced",
+				DiskEncryptionKMSKey:      "fake-kms-key",
+				Tags:                      map[string]string{"fake-tag": "fake-resource-tag-value"},
+				Labels:                    map[string]string{"fake-label": "fake-label-value"},
+				EnableConfidentialCompute: true,
+				ResourceTags:              map[string]string{"fake-resource-tag": "fake-resource-tag-value"},
+				AccessMode:                "READ_WRITE_SINGLE",
+				UseAllowedDiskTopology:    true,
+			},
+		},
+		{
+			// For replication type, the sanitized value is "none" instead of an empty string.
+			desc: "ReplicationType sanitized for HD",
+			parameters: &DiskParameters{
+				DiskType:        "hyperdisk-balanced",
+				ReplicationType: "regional-pd",
+			},
+			want: &DiskParameters{
+				DiskType:        "hyperdisk-balanced",
+				ReplicationType: "none",
+			},
+		},
+		{
+			desc: "ReplicationType unchanged for PD",
+			parameters: &DiskParameters{
+				DiskType:        "pd-balanced",
+				ReplicationType: "regional",
+			},
+			want: &DiskParameters{
+				DiskType:        "pd-balanced",
+				ReplicationType: "regional",
+			},
+		},
+		{
+			desc: "ProvisionedIOPSOnCreate sanitized for PD",
+			parameters: &DiskParameters{
+				DiskType:                "pd-balanced",
+				ProvisionedIOPSOnCreate: 1000,
+			},
+			want: &DiskParameters{
+				DiskType: "pd-balanced",
+			},
+		},
+		{
+			desc: "ProvisionedIOPSOnCreate unchanged for HD",
+			parameters: &DiskParameters{
+				DiskType:                "hyperdisk-balanced",
+				ProvisionedIOPSOnCreate: 5000,
+			},
+			want: &DiskParameters{
+				DiskType:                "hyperdisk-balanced",
+				ProvisionedIOPSOnCreate: 5000,
+			},
+		},
+		{
+			desc: "ProvisionedIOPSOnCreate unchanged for pd-extreme",
+			parameters: &DiskParameters{
+				DiskType:                "pd-extreme",
+				ProvisionedIOPSOnCreate: 5000,
+			},
+			want: &DiskParameters{
+				DiskType:                "pd-extreme",
+				ProvisionedIOPSOnCreate: 5000,
+			},
+		},
+		{
+			desc: "ProvisionedThroughputOnCreate sanitized for PD",
+			parameters: &DiskParameters{
+				DiskType:                      "pd-standard",
+				ProvisionedThroughputOnCreate: 200,
+			},
+			want: &DiskParameters{
+				DiskType: "pd-standard",
+			},
+		},
+		{
+			desc: "ProvisionedThroughputOnCreate unchanged for HD",
+			parameters: &DiskParameters{
+				DiskType:                      "hyperdisk-balanced",
+				ProvisionedThroughputOnCreate: 200,
+			},
+			want: &DiskParameters{
+				DiskType:                      "hyperdisk-balanced",
+				ProvisionedThroughputOnCreate: 200,
+			},
+		},
+		{
+			desc: "StoragePools sanitized for PD",
+			parameters: &DiskParameters{
+				DiskType:     "pd-balanced",
+				StoragePools: []StoragePool{{Name: "fake-pool"}},
+			},
+			want: &DiskParameters{
+				DiskType: "pd-balanced",
+			},
+		},
+		{
+			desc: "StoragePools unchanged for HD",
+			parameters: &DiskParameters{
+				DiskType:     "hyperdisk-balanced",
+				StoragePools: []StoragePool{{Name: "fake-pool"}},
+			},
+			want: &DiskParameters{
+				DiskType:     "hyperdisk-balanced",
+				StoragePools: []StoragePool{{Name: "fake-pool"}},
+			},
+		},
+		{
+			desc: "ForceAttach sanitized for HD",
+			parameters: &DiskParameters{
+				DiskType:    "hyperdisk-balanced",
+				ForceAttach: true,
+			},
+			want: &DiskParameters{
+				DiskType: "hyperdisk-balanced",
+			},
+		},
+		{
+			desc: "ForceAttach unchanged for PD",
+			parameters: &DiskParameters{
+				DiskType:    "pd-balanced",
+				ForceAttach: true,
+			},
+			want: &DiskParameters{
+				DiskType:    "pd-balanced",
+				ForceAttach: true,
+			},
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.desc, func(t *testing.T) {
+			sanitizeDiskParameters(tc.parameters)
+			got := tc.parameters
+			if diff := cmp.Diff(tc.want, got); diff != "" {
+				t.Errorf("sanitizeDiskParameters() mismatch (-want +got):\n%s", diff)
+			}
+
+		})
+	}
+
+}
diff --git a/test/e2e/tests/setup_e2e_test.go b/test/e2e/tests/setup_e2e_test.go
index 4de53577..6916ff3d 100644
--- a/test/e2e/tests/setup_e2e_test.go
+++ b/test/e2e/tests/setup_e2e_test.go
@@ -47,8 +47,8 @@ var (
 	serviceAccount            = flag.String("service-account", "", "Service account to bring up instance with")
 	vmNamePrefix              = flag.String("vm-name-prefix", "gce-pd-csi-e2e", "VM name prefix")
 	architecture              = flag.String("arch", "amd64", "Architecture pd csi driver build on")
-	minCpuPlatform            = flag.String("min-cpu-platform", "rome", "Minimum CPU architecture")
-	mwMinCpuPlatform          = flag.String("min-cpu-platform-mw", "sapphirerapids", "Minimum CPU architecture for multiwriter tests")
+	minCpuPlatform            = flag.String("min-cpu-platform", "AMD Rome", "Minimum CPU architecture")
+	mwMinCpuPlatform          = flag.String("min-cpu-platform-mw", "Intel Sapphire Rapids", "Minimum CPU architecture for multiwriter tests")
 	zones                     = flag.String("zones", "us-east4-a,us-east4-c", "Zones to run tests in. If there are multiple zones, separate each by comma")
 	machineType               = flag.String("machine-type", "n2d-standard-4", "Type of machine to provision instance on")
 	imageURL                  = flag.String("image-url", "projects/ubuntu-os-cloud/global/images/family/ubuntu-minimal-2404-lts-amd64", "OS image url to get image from")
@@ -65,8 +65,8 @@ var (
 	// Some architectures don't have local ssd. Give way to opt out of tests like datacache.
 	skipLocalSsdTests = flag.Bool("skip-local-ssd-tests", false, "Skip local ssd tests like datacache")
 
-	testContexts          = []*remote.TestContext{}
-	hyperdiskTestContexts = []*remote.TestContext{}
+	testContexts          []*remote.TestContext
+	hyperdiskTestContexts []*remote.TestContext
 	computeService        *compute.Service
 	computeAlphaService   *computealpha.Service
 	computeBetaService    *computebeta.Service
@@ -87,10 +87,6 @@ var _ = BeforeSuite(func() {
 	var err error
 	numberOfInstancesPerZone := 2
 	zones := strings.Split(*zones, ",")
-	tcc := make(chan *remote.TestContext, len(zones)*numberOfInstancesPerZone)
-	hdtcc := make(chan *remote.TestContext, len(zones))
-	defer close(tcc)
-	defer close(hdtcc)
 
 	rand.Seed(time.Now().UnixNano())
 
@@ -116,42 +112,40 @@ var _ = BeforeSuite(func() {
 
 	klog.Infof("Running in project %v with service account %v", *project, *serviceAccount)
 
-	setupContext := func(zone string) {
-		var wg sync.WaitGroup
+	testContexts = make([]*remote.TestContext, numberOfInstancesPerZone*len(zones))
+	if *hdMachineType != noMachineType {
+		hyperdiskTestContexts = make([]*remote.TestContext, len(zones))
+	}
+	var wg sync.WaitGroup
+	setupContext := func(idx int, zone string) {
 		// Create 2 instances for each zone as we need 2 instances each zone for certain test cases
 		for j := 0; j < numberOfInstancesPerZone; j++ {
 			wg.Add(1)
 			go func(curZone string, randInt int) {
 				defer GinkgoRecover()
 				defer wg.Done()
-				tcc <- NewDefaultTestContext(curZone, strconv.Itoa(randInt))
+				tc := NewDefaultTestContext(curZone, strconv.Itoa(randInt))
+				k := j + idx*numberOfInstancesPerZone
+				testContexts[k] = tc
+				klog.Infof("Added TestContext for node %s at %d", tc.Instance.GetName(), k)
 			}(zone, j)
 		}
-		wg.Add(1)
-		if *hdMachineType != noMachineType {
+		if hyperdiskTestContexts != nil {
+			wg.Add(1)
 			go func(curZone string) {
 				defer GinkgoRecover()
 				defer wg.Done()
-				hdtcc <- NewTestContext(curZone, *hdMinCpuPlatform, *hdMachineType, "0")
+				tc := NewTestContext(curZone, *hdMinCpuPlatform, *hdMachineType, "0")
+				hyperdiskTestContexts[idx] = tc
+				klog.Infof("Added hyperdisk TestContext for node %s at %d", tc.Instance.GetName(), idx)
 			}(zone)
 		}
-		wg.Wait()
 	}
 
-	for _, zone := range zones {
-		setupContext(zone)
-	}
-
-	for i := 0; i < len(zones)*numberOfInstancesPerZone; i++ {
-		tc := <-tcc
-		testContexts = append(testContexts, tc)
-		klog.Infof("Added TestContext for node %s", tc.Instance.GetName())
-	}
-	for i := 0; i < len(zones); i++ {
-		tc := <-hdtcc
-		hyperdiskTestContexts = append(hyperdiskTestContexts, tc)
-		klog.Infof("Added TestContext for node %s", tc.Instance.GetName())
+	for i, zone := range zones {
+		setupContext(i, zone)
 	}
+	wg.Wait()
 })
 
 var _ = AfterSuite(func() {
diff --git a/test/e2e/tests/single_zone_e2e_test.go b/test/e2e/tests/single_zone_e2e_test.go
index 70c46947..6e35b601 100644
--- a/test/e2e/tests/single_zone_e2e_test.go
+++ b/test/e2e/tests/single_zone_e2e_test.go
@@ -1668,6 +1668,94 @@ var _ = Describe("GCE PD CSI Driver", func() {
 		Expect(err).To(BeNil(), "Failed to rm file path %s: %v", fp, err)
 	})
 
+	It("Should mount if udev disabled, and remount if it's enabled again", func() {
+		testContext := getRandomTestContext()
+		p, z, _ := testContext.Instance.GetIdentity()
+		client := testContext.Client
+		instance := testContext.Instance
+
+		klog.Infof("Disabling udev")
+		err := instance.DisableUdev()
+		Expect(err).To(BeNil(), "Failed to disable udev")
+
+		// Create Disk
+		volName, volID := createAndValidateUniqueZonalDisk(client, p, z, standardDiskType)
+		vol2Name, vol2ID := createAndValidateUniqueZonalDisk(client, p, z, standardDiskType)
+
+		defer func() {
+			// Delete Disks
+			err := client.DeleteVolume(volID)
+			Expect(err).To(BeNil(), "DeleteVolume failed")
+
+			err = client.DeleteVolume(vol2ID)
+			Expect(err).To(BeNil(), "DeleteVolume failed")
+
+			// Validate Disks Deleted
+			_, err = computeService.Disks.Get(p, z, volName).Do()
+			Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found")
+			_, err = computeService.Disks.Get(p, z, vol2Name).Do()
+			Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found")
+		}()
+
+		// Attach & detach disk. We retry as we expect the udev repair to take a little bit of time.
+		klog.Infof("Starting attach & detach with disabled udev")
+		err = wait.Poll(10*time.Second, 5*time.Minute, func() (bool, error) {
+			err = testAttachWriteReadDetach(volID, volName, instance, client, false /* readOnly */, false /* detachAndReattach */, false /* setupDataCache */)
+			if err != nil {
+				klog.Infof("Initial udev error, retrying: %v", err)
+			}
+			return err == nil, nil
+		})
+		Expect(err).To(BeNil(), "Failed to go through volume lifecycle")
+
+		// Attach a different disk. The conflicting udev paths should not cause a problem.
+		klog.Infof("Starting second attach & detach with disabled udev")
+		err = wait.Poll(10*time.Second, 5*time.Minute, func() (bool, error) {
+			err = testAttachWriteReadDetach(vol2ID, vol2Name, instance, client, false /* readOnly */, false /* detachAndReattach */, false /* setupDataCache */)
+			if err != nil {
+				klog.Infof("second disk udev error, retrying: %v", err)
+			}
+			return err == nil, nil
+		})
+		Expect(err).To(BeNil(), "Failed to go through second volume lifecycle")
+
+		// Attach, reenable udev, go through lifecycle of second disk, detach first
+		klog.Infof("Starting attach & udev re-enable")
+		var detacher func()
+		var args *verifyArgs
+		err = wait.Poll(10*time.Second, 5*time.Minute, func() (bool, error) {
+			err, detacher, args = testAttachAndMount(volID, volName, instance, client, attachAndMountArgs{})
+			if err != nil {
+				klog.Infof("attach before reenable failed, retrying: %v", err)
+			}
+			return err == nil, nil
+		})
+		Expect(err).To(BeNil(), "Failed second attach")
+		defer detacher()
+
+		klog.Infof("Re-enabling udev")
+		err = instance.EnableUdev()
+		Expect(err).To(BeNil(), "Failed to enable udev")
+
+		// After udev is enabled we expect everything to succeed on the first try.
+
+		klog.Infof("Testing attach & detach with re-enabled udev")
+		err = testAttachWriteReadDetach(vol2ID, vol2Name, instance, client, false /* readOnly */, false /* detachAndReattach */, false /* setupDataCache */)
+		Expect(err).To(BeNil(), "Failed to go through nested volume lifecycle with enabled")
+
+		klog.Infof("Testing detach with re-enabled udev")
+		err = client.NodeUnpublishVolume(volID, args.publishDir)
+		Expect(err).To(BeNil(), "Failed to unpublish first")
+
+		err = client.NodeUnstageVolume(volID, args.stageDir)
+		Expect(err).To(BeNil(), "Failed to unstage first")
+
+		// Go through complete lifecycle again, with udev enabled.
+		klog.Infof("Testing final lifecycle enabled udev")
+		err = testAttachWriteReadDetach(volID, volName, instance, client, false /* readOnly */, false /* detachAndReattach */, false /* setupDataCache */)
+		Expect(err).To(BeNil(), "Failed to go through volume lifecycle with udev enabled")
+	})
+
 	type multiZoneTestConfig struct {
 		diskType          string
 		readOnly          bool
diff --git a/test/k8s-integration/cluster.go b/test/k8s-integration/cluster.go
index e3e5c0ec..6618f1e4 100644
--- a/test/k8s-integration/cluster.go
+++ b/test/k8s-integration/cluster.go
@@ -1,6 +1,7 @@
 package main
 
 import (
+	"bufio"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -280,6 +281,26 @@ func clusterUpGKE(gceZone, gceRegion string, numNodes int, numWindowsNodes int,
 		}
 	}
 
+	cmd = exec.Command("gcloud", "container", "clusters", "describe", *gkeTestClusterName, locationArg, locationVal)
+	out, err = cmd.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("failed to list cluster: %v %s", err, out)
+	}
+	scanner := bufio.NewScanner(strings.NewReader(string(out)))
+	printedHash := false
+	for scanner.Scan() {
+		line := scanner.Text()
+		if strings.HasPrefix(line, "id: ") {
+			klog.Infof("GKE cluster: %s %s", *gkeTestClusterName, locationVal)
+			klog.Infof("GKE cluster hash %s", line)
+			printedHash = true
+			break
+		}
+	}
+	if !printedHash {
+		return fmt.Errorf("failed to find cluster hash in cluster describe: %s", out)
+	}
+
 	return nil
 }
 
diff --git a/test/remote/instance.go b/test/remote/instance.go
index 92ef9459..13ed1c05 100644
--- a/test/remote/instance.go
+++ b/test/remote/instance.go
@@ -92,7 +92,7 @@ func machineTypeMismatch(curInst *compute.Instance, newInst *compute.Instance) b
 	// Ideally we could compare to see if the new instance has a greater minCpuPlatfor
 	// For now we just check it was set and it's different.
 	if curInst.MinCpuPlatform != "" && curInst.MinCpuPlatform != newInst.MinCpuPlatform {
-		klog.Infof("CPU Platform mismatch")
+		klog.Infof("CPU Platform mismatch: cur: %v; new: %v", curInst.MinCpuPlatform, newInst.MinCpuPlatform)
 		return true
 	}
 	if (curInst.ConfidentialInstanceConfig != nil && newInst.ConfidentialInstanceConfig == nil) ||
@@ -102,7 +102,7 @@ func machineTypeMismatch(curInst *compute.Instance, newInst *compute.Instance) b
 		return true
 	}
 	if curInst.SourceMachineImage != newInst.SourceMachineImage {
-		klog.Infof("Source Machine Mismatch")
+		klog.Infof("Source Machine Mismatch: cur: %v; new: %v", curInst.SourceMachineImage, newInst.SourceMachineImage)
 		return true
 	}
 	return false
@@ -131,7 +131,8 @@ func (i *InstanceInfo) CreateOrGetInstance(localSSDCount int) error {
 						Type: "ONE_TO_ONE_NAT",
 						Name: "External NAT",
 					},
-				}},
+				},
+			},
 		},
 		Disks: []*compute.AttachedDisk{
 			{
diff --git a/test/remote/ssh.go b/test/remote/ssh.go
index 7167ee2e..07c6b84f 100644
--- a/test/remote/ssh.go
+++ b/test/remote/ssh.go
@@ -100,6 +100,48 @@ func (i *InstanceInfo) SSHCheckAlive() error {
 	})
 }
 
+func (i *InstanceInfo) DisableUdev() error {
+	return wait.Poll(5*time.Second, time.Minute, func() (bool, error) {
+		_, err := i.SSH("systemctl", "stop", "systemd-udevd")
+		if err != nil {
+			klog.V(2).Infof("(will retry) failed to stop systemd-udevd: %v", err)
+			return false, nil
+		}
+		_, err = i.SSH("systemctl", "stop", "systemd-udevd-kernel.socket")
+		if err != nil {
+			klog.V(2).Infof("(will retry) failed to stop systemd-udevd-kernel.socket: %v", err)
+			return false, nil
+		}
+		_, err = i.SSH("systemctl", "stop", "systemd-udevd-control.socket")
+		if err != nil {
+			klog.V(2).Infof("(will retry) failed to stop systemd-udevd-control.socket: %v", err)
+			return false, nil
+		}
+		return true, nil
+	})
+}
+
+func (i *InstanceInfo) EnableUdev() error {
+	return wait.Poll(5*time.Second, time.Minute, func() (bool, error) {
+		_, err := i.SSH("systemctl", "start", "systemd-udevd")
+		if err != nil {
+			klog.V(2).Infof("(will retry) failed to start systemd-udevd: %v", err)
+			return false, nil
+		}
+		_, err = i.SSH("systemctl", "start", "systemd-udevd-kernel.socket")
+		if err != nil {
+			klog.V(2).Infof("(will retry) failed to start systemd-udevd-kernel.socket: %v", err)
+			return false, nil
+		}
+		_, err = i.SSH("systemctl", "start", "systemd-udevd-control.socket")
+		if err != nil {
+			klog.V(2).Infof("(will retry) failed to start systemd-udevd-control.socket: %v", err)
+			return false, nil
+		}
+		return true, nil
+	})
+}
+
 // runSSHCommand executes the ssh or scp command, adding the flag provided --ssh-options
 func runSSHCommand(cmd string, args ...string) (string, error) {
 	if pk, ok := os.LookupEnv("JENKINS_GCE_SSH_PRIVATE_KEY_FILE"); ok {