Repository: kubernetes-sigs/aws-ebs-csi-driver (go)

Issue:
Implement Deep health check in driver probe
**Is your feature request related to a problem? Please describe.**
Currently, the driver probe returns success as long as the grpc server is running. We should do deep health check in the probe to prevent issues like DNS resolution issue (eg #214)

**Describe the solution you'd like in detail**
TBD

**Additional context**
~We could also combine this with driver state reconstruction when controller starts after crash by calling into AWS to restore to the most recently know good state.~


Pull request description:
add options to enable aws sdk debug log and add more logs when driver…
… start up

**Is this a bug fix or adding new feature?**
Fixes #216 
**What is this PR about? / Why do we need it?**
Add a new driver level option -aws-sdk-debug-log, if set to true, aws go sdk will add debug log to the std output.
Sample for debug log
```
2021/04/15 00:41:50 DEBUG: Send Request ec2/DescribeVolumes failed, attempt 0/8, error RequestError: send request failed
caused by: Post "https://ec2.us-west-2.amazonaws.com/": dial tcp 54.240.253.45:443: i/o timeout
2021/04/15 00:41:50 DEBUG: Request ec2/DescribeVolumes Details:
```
Also adjust and update the klog level across driver.
**What testing is done?** 
unit test
e2e test

Patch:
diff --git a/cmd/main.go b/cmd/main.go
index 5bacc3ab..fca0eb58 100644
--- a/cmd/main.go
+++ b/cmd/main.go
@@ -18,7 +18,6 @@ package main
 
 import (
 	"flag"
-
 	"github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/driver"
 
 	"k8s.io/klog"
@@ -35,6 +34,7 @@ func main() {
 		driver.WithMode(options.DriverMode),
 		driver.WithVolumeAttachLimit(options.NodeOptions.VolumeAttachLimit),
 		driver.WithKubernetesClusterID(options.ControllerOptions.KubernetesClusterID),
+		driver.WithAwsSdkDebugLog(options.ControllerOptions.AwsSdkDebugLog),
 	)
 	if err != nil {
 		klog.Fatalln(err)
diff --git a/cmd/options/controller_options.go b/cmd/options/controller_options.go
index 9b6287f2..91f413ef 100644
--- a/cmd/options/controller_options.go
+++ b/cmd/options/controller_options.go
@@ -33,10 +33,13 @@ type ControllerOptions struct {
 	ExtraVolumeTags map[string]string
 	// ID of the kubernetes cluster.
 	KubernetesClusterID string
+	// flag to enable sdk debug log
+	AwsSdkDebugLog bool
 }
 
 func (s *ControllerOptions) AddFlags(fs *flag.FlagSet) {
 	fs.Var(cliflag.NewMapStringString(&s.ExtraTags), "extra-tags", "Extra tags to attach to each dynamically provisioned resource. It is a comma separated list of key value pairs like '<key1>=<value1>,<key2>=<value2>'")
 	fs.Var(cliflag.NewMapStringString(&s.ExtraVolumeTags), "extra-volume-tags", "DEPRECATED: Please use --extra-tags instead. Extra volume tags to attach to each dynamically provisioned volume. It is a comma separated list of key value pairs like '<key1>=<value1>,<key2>=<value2>'")
 	fs.StringVar(&s.KubernetesClusterID, "k8s-tag-cluster-id", "", "ID of the Kubernetes cluster used for tagging provisioned EBS volumes (optional).")
+	fs.BoolVar(&s.AwsSdkDebugLog, "aws-sdk-debug-log", false, "To enable the aws sdk debug log level (default to false).")
 }
diff --git a/docs/README.md b/docs/README.md
index 09d37857..b030bd33 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -154,6 +154,11 @@ helm upgrade --install aws-ebs-csi-driver \
     --set enableVolumeSnapshot=true \
     aws-ebs-csi-driver/aws-ebs-csi-driver
 ```
+
+#### Deploy driver with debug mode
+To view driver debug logs, run the CSI driver with `-v=5` command line option
+To enable aws sdk debug logs, run the CSI driver with `--aws-sdk-debug-log=true` command line option.
+
 ## Examples
 Make sure you follow the [Prerequisites](README.md#Prerequisites) before the examples:
 * [Dynamic Provisioning](../examples/kubernetes/dynamic-provisioning)
diff --git a/pkg/cloud/cloud.go b/pkg/cloud/cloud.go
index 8835b912..2ed89b2d 100644
--- a/pkg/cloud/cloud.go
+++ b/pkg/cloud/cloud.go
@@ -239,11 +239,11 @@ var _ Cloud = &cloud{}
 
 // NewCloud returns a new instance of AWS cloud
 // It panics if session is invalid
-func NewCloud(region string) (Cloud, error) {
-	return newEC2Cloud(region)
+func NewCloud(region string, awsSdkDebugLog bool) (Cloud, error) {
+	return newEC2Cloud(region, awsSdkDebugLog)
 }
 
-func newEC2Cloud(region string) (Cloud, error) {
+func newEC2Cloud(region string, awsSdkDebugLog bool) (Cloud, error) {
 	awsConfig := &aws.Config{
 		Region:                        aws.String(region),
 		CredentialsChainVerboseErrors: aws.Bool(true),
@@ -256,6 +256,10 @@ func newEC2Cloud(region string) (Cloud, error) {
 		awsConfig.Endpoint = aws.String(endpoint)
 	}
 
+	if awsSdkDebugLog {
+		awsConfig.WithLogLevel(aws.LogDebugWithRequestErrors)
+	}
+
 	return &cloud{
 		region: region,
 		dm:     dm.NewDeviceManager(),
@@ -310,9 +314,9 @@ func (c *cloud) CreateDisk(ctx context.Context, volumeName string, diskOptions *
 
 	zone := diskOptions.AvailabilityZone
 	if zone == "" {
-		klog.V(5).Infof("AZ is not provided. Using node AZ [%s]", zone)
 		var err error
 		zone, err = c.randomAvailabilityZone(ctx)
+		klog.V(5).Infof("[Debug] AZ is not provided. Using node AZ [%s]", zone)
 		if err != nil {
 			return nil, fmt.Errorf("failed to get availability zone %s", err)
 		}
@@ -370,7 +374,7 @@ func (c *cloud) CreateDisk(ctx context.Context, volumeName string, diskOptions *
 		if _, error := c.DeleteDisk(ctx, volumeID); error != nil {
 			klog.Errorf("%v failed to be deleted, this may cause volume leak", volumeID)
 		} else {
-			klog.V(5).Infof("%v is deleted because it is not in desired state within retry limit", volumeID)
+			klog.V(5).Infof("[Debug] %v is deleted because it is not in desired state within retry limit", volumeID)
 		}
 		return nil, fmt.Errorf("failed to get an available volume in EC2: %v", err)
 	}
@@ -419,7 +423,7 @@ func (c *cloud) AttachDisk(ctx context.Context, volumeID, nodeID string) (string
 			}
 			return "", fmt.Errorf("could not attach volume %q to node %q: %v", volumeID, nodeID, err)
 		}
-		klog.V(5).Infof("AttachVolume volume=%q instance=%q request returned %v", volumeID, nodeID, resp)
+		klog.V(5).Infof("[Debug] AttachVolume volume=%q instance=%q request returned %v", volumeID, nodeID, resp)
 
 	}
 
@@ -952,7 +956,7 @@ func (c *cloud) ResizeDisk(ctx context.Context, volumeID string, newSizeBytes in
 	// Even if existing volume size is greater than user requested size, we should ensure that there are no pending
 	// volume modifications objects or volume has completed previously issued modification request.
 	if oldSizeGiB >= newSizeGiB {
-		klog.V(5).Infof("Volume %q current size (%d GiB) is greater or equal to the new size (%d GiB)", volumeID, oldSizeGiB, newSizeGiB)
+		klog.V(5).Infof("[Debug] Volume %q current size (%d GiB) is greater or equal to the new size (%d GiB)", volumeID, oldSizeGiB, newSizeGiB)
 		_, err = c.waitForVolumeSize(ctx, volumeID)
 		if err != nil && err != VolumeNotBeingModified {
 			return oldSizeGiB, err
@@ -965,7 +969,7 @@ func (c *cloud) ResizeDisk(ctx context.Context, volumeID string, newSizeBytes in
 		Size:     aws.Int64(newSizeGiB),
 	}
 
-	klog.Infof("expanding volume %q to size %d", volumeID, newSizeGiB)
+	klog.V(4).Infof("expanding volume %q to size %d", volumeID, newSizeGiB)
 	response, err := c.ec2.ModifyVolumeWithContext(ctx, req)
 	if err != nil {
 		return 0, fmt.Errorf("could not modify AWS volume %q: %v", volumeID, err)
@@ -1106,18 +1110,18 @@ func capIOPS(volumeType string, requestedCapacityGiB int64, requstedIOPSPerGB, m
 	if iops < minTotalIOPS {
 		if allowIncrease {
 			iops = minTotalIOPS
-			klog.V(5).Infof("Increased IOPS for %s %d GB volume to the min supported limit: %d", volumeType, requestedCapacityGiB, iops)
+			klog.V(5).Infof("[Debug] Increased IOPS for %s %d GB volume to the min supported limit: %d", volumeType, requestedCapacityGiB, iops)
 		} else {
 			return 0, fmt.Errorf("invalid combination of volume size %d GB and iopsPerGB %d: the resulting IOPS %d is too low for AWS, it must be at least %d", requestedCapacityGiB, requstedIOPSPerGB, iops, minTotalIOPS)
 		}
 	}
 	if iops > maxTotalIOPS {
 		iops = maxTotalIOPS
-		klog.V(5).Infof("Capped IOPS for %s %d GB volume at the max supported limit: %d", volumeType, requestedCapacityGiB, iops)
+		klog.V(5).Infof("[Debug] Capped IOPS for %s %d GB volume at the max supported limit: %d", volumeType, requestedCapacityGiB, iops)
 	}
 	if iops > maxIOPSPerGB*requestedCapacityGiB {
 		iops = maxIOPSPerGB * requestedCapacityGiB
-		klog.V(5).Infof("Capped IOPS for %s %d GB volume at %d IOPS/GB: %d", volumeType, requestedCapacityGiB, maxIOPSPerGB, iops)
+		klog.V(5).Infof("[Debug] Capped IOPS for %s %d GB volume at %d IOPS/GB: %d", volumeType, requestedCapacityGiB, maxIOPSPerGB, iops)
 	}
 	return iops, nil
 }
diff --git a/pkg/cloud/devicemanager/manager.go b/pkg/cloud/devicemanager/manager.go
index 14f52293..46cf12ab 100644
--- a/pkg/cloud/devicemanager/manager.go
+++ b/pkg/cloud/devicemanager/manager.go
@@ -194,7 +194,7 @@ func (d *deviceManager) release(device *Device) error {
 		return fmt.Errorf("release on device %q assigned to different volume: %q vs %q", device.Path, device.VolumeID, existingVolumeID)
 	}
 
-	klog.V(5).Infof("Releasing in-process attachment entry: %v -> volume %s", device.Path, device.VolumeID)
+	klog.V(5).Infof("[Debug] Releasing in-process attachment entry: %v -> volume %s", device.Path, device.VolumeID)
 	d.inFlight.Del(nodeID, name)
 
 	return nil
diff --git a/pkg/driver/controller.go b/pkg/driver/controller.go
index dd252816..1f307af0 100644
--- a/pkg/driver/controller.go
+++ b/pkg/driver/controller.go
@@ -75,6 +75,7 @@ var (
 func newControllerService(driverOptions *DriverOptions) controllerService {
 	region := os.Getenv("AWS_REGION")
 	if region == "" {
+		klog.V(5).Infof("[Debug] Retrieving region from metadata service")
 		metadata, err := NewMetadataFunc()
 		if err != nil {
 			panic(err)
@@ -82,13 +83,13 @@ func newControllerService(driverOptions *DriverOptions) controllerService {
 		region = metadata.GetRegion()
 	}
 
-	cloud, err := NewCloudFunc(region)
+	cloudSrv, err := NewCloudFunc(region, driverOptions.awsSdkDebugLog)
 	if err != nil {
 		panic(err)
 	}
 
 	return controllerService{
-		cloud:         cloud,
+		cloud:         cloudSrv,
 		inFlight:      internal.NewInFlight(),
 		driverOptions: driverOptions,
 	}
@@ -323,7 +324,7 @@ func (d *controllerService) ControllerPublishVolume(ctx context.Context, req *cs
 		// TODO: Check volume capability matches for ALREADY_EXISTS
 		return nil, status.Errorf(codes.Internal, "Could not attach volume %q to node %q: %v", volumeID, nodeID, err)
 	}
-	klog.V(5).Infof("ControllerPublishVolume: volume %s attached to node %s through device %s", volumeID, nodeID, devicePath)
+	klog.V(5).Infof("[Debug] ControllerPublishVolume: volume %s attached to node %s through device %s", volumeID, nodeID, devicePath)
 
 	pvInfo := map[string]string{DevicePathKey: devicePath}
 	return &csi.ControllerPublishVolumeResponse{PublishContext: pvInfo}, nil
@@ -347,7 +348,7 @@ func (d *controllerService) ControllerUnpublishVolume(ctx context.Context, req *
 		}
 		return nil, status.Errorf(codes.Internal, "Could not detach volume %q from node %q: %v", volumeID, nodeID, err)
 	}
-	klog.V(5).Infof("ControllerUnpublishVolume: volume %s detached from node %s", volumeID, nodeID)
+	klog.V(5).Infof("[Debug] ControllerUnpublishVolume: volume %s detached from node %s", volumeID, nodeID)
 
 	return &csi.ControllerUnpublishVolumeResponse{}, nil
 }
diff --git a/pkg/driver/driver.go b/pkg/driver/driver.go
index 0e586f06..b4bc7fa1 100644
--- a/pkg/driver/driver.go
+++ b/pkg/driver/driver.go
@@ -65,10 +65,11 @@ type DriverOptions struct {
 	mode                Mode
 	volumeAttachLimit   int64
 	kubernetesClusterID string
+	awsSdkDebugLog      bool
 }
 
 func NewDriver(options ...func(*DriverOptions)) (*Driver, error) {
-	klog.Infof("Driver: %v Version: %v", DriverName, driverVersion)
+	klog.V(4).Infof("Driver: %v Version: %v", DriverName, driverVersion)
 
 	driverOptions := DriverOptions{
 		endpoint: DefaultCSIEndpoint,
@@ -138,12 +139,11 @@ func (d *Driver) Run() error {
 		return fmt.Errorf("unknown mode: %s", d.options.mode)
 	}
 
-	klog.Infof("Listening for connections on address: %#v", listener.Addr())
+	klog.V(4).Infof("Listening for connections on address: %#v", listener.Addr())
 	return d.srv.Serve(listener)
 }
 
 func (d *Driver) Stop() {
-	klog.Infof("Stopping server")
 	d.srv.Stop()
 }
 
@@ -185,3 +185,9 @@ func WithKubernetesClusterID(clusterID string) func(*DriverOptions) {
 		o.kubernetesClusterID = clusterID
 	}
 }
+
+func WithAwsSdkDebugLog(enableSdkDebugLog bool) func(*DriverOptions) {
+	return func(o *DriverOptions) {
+		o.awsSdkDebugLog = enableSdkDebugLog
+	}
+}
diff --git a/pkg/driver/node.go b/pkg/driver/node.go
index 1ad3ad72..03d20cf2 100644
--- a/pkg/driver/node.go
+++ b/pkg/driver/node.go
@@ -82,6 +82,7 @@ type nodeService struct {
 // newNodeService creates a new node service
 // it panics if failed to create the service
 func newNodeService(driverOptions *DriverOptions) nodeService {
+	klog.V(5).Infof("[Debug] Retrieving node info from metadata service")
 	metadata, err := cloud.NewMetadata()
 	if err != nil {
 		panic(err)
@@ -209,7 +210,7 @@ func (d *nodeService) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol
 	}
 
 	// FormatAndMount will format only if needed
-	klog.V(5).Infof("NodeStageVolume: formatting %s and mounting at %s with fstype %s", source, target, fsType)
+	klog.V(4).Infof("NodeStageVolume: formatting %s and mounting at %s with fstype %s", source, target, fsType)
 	err = d.mounter.FormatAndMount(source, target, fsType, mountOptions)
 	if err != nil {
 		msg := fmt.Sprintf("could not format %q and mount it at %q: %v", source, target, err)
@@ -251,7 +252,7 @@ func (d *nodeService) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstag
 	// is not staged to the staging_target_path, the Plugin MUST
 	// reply 0 OK.
 	if refCount == 0 {
-		klog.V(5).Infof("NodeUnstageVolume: %s target not mounted", target)
+		klog.V(5).Infof("[Debug] NodeUnstageVolume: %s target not mounted", target)
 		return &csi.NodeUnstageVolumeResponse{}, nil
 	}
 
@@ -259,7 +260,7 @@ func (d *nodeService) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstag
 		klog.Warningf("NodeUnstageVolume: found %d references to device %s mounted at target path %s", refCount, dev, target)
 	}
 
-	klog.V(5).Infof("NodeUnstageVolume: unmounting %s", target)
+	klog.V(4).Infof("NodeUnstageVolume: unmounting %s", target)
 	err = d.mounter.Unmount(target)
 	if err != nil {
 		return nil, status.Errorf(codes.Internal, "Could not unmount target %q: %v", target, err)
@@ -408,7 +409,7 @@ func (d *nodeService) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpu
 		d.inFlight.Delete(volumeID)
 	}()
 
-	klog.V(5).Infof("NodeUnpublishVolume: unmounting %s", target)
+	klog.V(4).Infof("NodeUnpublishVolume: unmounting %s", target)
 	err := d.mounter.Unmount(target)
 	if err != nil {
 		return nil, status.Errorf(codes.Internal, "Could not unmount %q: %v", target, err)
@@ -570,7 +571,7 @@ func (d *nodeService) nodePublishVolumeForBlock(req *csi.NodePublishVolumeReques
 	}
 
 	// Create the mount point as a file since bind mount device node requires it to be a file
-	klog.V(5).Infof("NodePublishVolume [block]: making target file %s", target)
+	klog.V(4).Infof("NodePublishVolume [block]: making target file %s", target)
 	if err := d.mounter.MakeFile(target); err != nil {
 		if removeErr := os.Remove(target); removeErr != nil {
 			return status.Errorf(codes.Internal, "Could not remove mount target %q: %v", target, removeErr)
@@ -578,7 +579,7 @@ func (d *nodeService) nodePublishVolumeForBlock(req *csi.NodePublishVolumeReques
 		return status.Errorf(codes.Internal, "Could not create file %q: %v", target, err)
 	}
 
-	klog.V(5).Infof("NodePublishVolume [block]: mounting %s at %s", source, target)
+	klog.V(4).Infof("NodePublishVolume [block]: mounting %s at %s", source, target)
 	if err := d.mounter.Mount(source, target, "", mountOptions); err != nil {
 		if removeErr := os.Remove(target); removeErr != nil {
 			return status.Errorf(codes.Internal, "Could not remove mount target %q: %v", target, removeErr)
@@ -600,7 +601,7 @@ func (d *nodeService) nodePublishVolumeForFileSystem(req *csi.NodePublishVolumeR
 		}
 	}
 
-	klog.V(5).Infof("NodePublishVolume: creating dir %s", target)
+	klog.V(4).Infof("NodePublishVolume: creating dir %s", target)
 	if err := d.mounter.MakeDir(target); err != nil {
 		return status.Errorf(codes.Internal, "Could not create dir %q: %v", target, err)
 	}
@@ -610,7 +611,7 @@ func (d *nodeService) nodePublishVolumeForFileSystem(req *csi.NodePublishVolumeR
 		fsType = defaultFsType
 	}
 
-	klog.V(5).Infof("NodePublishVolume: mounting %s at %s with option %s as fstype %s", source, target, mountOptions, fsType)
+	klog.V(4).Infof("NodePublishVolume: mounting %s at %s with option %s as fstype %s", source, target, mountOptions, fsType)
 	if err := d.mounter.Mount(source, target, fsType, mountOptions); err != nil {
 		if removeErr := os.Remove(target); removeErr != nil {
 			return status.Errorf(codes.Internal, "Could not remove mount target %q: %v", target, err)
diff --git a/pkg/driver/node_linux.go b/pkg/driver/node_linux.go
index 71f2c53e..4c4a9d09 100644
--- a/pkg/driver/node_linux.go
+++ b/pkg/driver/node_linux.go
@@ -69,7 +69,7 @@ func findNvmeVolume(findName string) (device string, err error) {
 	stat, err := os.Lstat(p)
 	if err != nil {
 		if os.IsNotExist(err) {
-			klog.V(5).Infof("nvme path %q not found", p)
+			klog.V(5).Infof("[Debug] nvme path %q not found", p)
 			return "", fmt.Errorf("nvme path %q not found", p)
 		}
 		return "", fmt.Errorf("error getting stat of %q: %v", p, err)
diff --git a/pkg/mounter/safe_mounter_windows.go b/pkg/mounter/safe_mounter_windows.go
index ce3a4a82..35a3277d 100644
--- a/pkg/mounter/safe_mounter_windows.go
+++ b/pkg/mounter/safe_mounter_windows.go
@@ -171,7 +171,7 @@ func (mounter *CSIProxyMounter) MakeDir(pathname string) error {
 	}
 	_, err := mounter.FsClient.Mkdir(context.Background(), mkdirReq)
 	if err != nil {
-		klog.Infof("Error: %v", err)
+		klog.V(4).Infof("Error: %v", err)
 		return err
 	}
 


Test patch:
diff --git a/cmd/options/controller_options_test.go b/cmd/options/controller_options_test.go
index 9c89ae88..76075fee 100644
--- a/cmd/options/controller_options_test.go
+++ b/cmd/options/controller_options_test.go
@@ -37,6 +37,11 @@ func TestControllerOptions(t *testing.T) {
 			flag:  "k8s-tag-cluster-id",
 			found: true,
 		},
+		{
+			name:  "lookup aws-sdk-debug-log",
+			flag:  "aws-sdk-debug-log",
+			found: true,
+		},
 		{
 			name:  "fail for non-desired flag",
 			flag:  "some-other-flag",
diff --git a/cmd/options_test.go b/cmd/options_test.go
index 37743a90..d810e7b2 100644
--- a/cmd/options_test.go
+++ b/cmd/options_test.go
@@ -46,6 +46,8 @@ func TestGetOptions(t *testing.T) {
 			extraTagKey: extraTagValue,
 		}
 
+		awsSdkDebugFlagName := "aws-sdk-debug-log"
+		awsSdkDebugFlagValue := true
 		VolumeAttachLimitFlagName := "volume-attach-limit"
 		var VolumeAttachLimit int64 = 42
 
@@ -58,6 +60,7 @@ func TestGetOptions(t *testing.T) {
 		}
 		if withControllerOptions {
 			args = append(args, "-"+extraTagsFlagName+"="+extraTagKey+"="+extraTagValue)
+			args = append(args, "-"+awsSdkDebugFlagName+"="+strconv.FormatBool(awsSdkDebugFlagValue))
 		}
 		if withNodeOptions {
 			args = append(args, "-"+VolumeAttachLimitFlagName+"="+strconv.FormatInt(VolumeAttachLimit, 10))
@@ -87,6 +90,13 @@ func TestGetOptions(t *testing.T) {
 			if !reflect.DeepEqual(options.ControllerOptions.ExtraTags, extraTags) {
 				t.Fatalf("expected extra tags to be %q but it is %q", extraTags, options.ControllerOptions.ExtraTags)
 			}
+			awsDebugLogFlag := flagSet.Lookup(awsSdkDebugFlagName)
+			if awsDebugLogFlag == nil {
+				t.Fatalf("expected %q flag to be added but it is not", awsSdkDebugFlagName)
+			}
+			if options.ControllerOptions.AwsSdkDebugLog != awsSdkDebugFlagValue {
+				t.Fatalf("expected sdk debug flag to be %v but it is %v", awsSdkDebugFlagValue, options.ControllerOptions.AwsSdkDebugLog)
+			}
 		}
 
 		if withNodeOptions {
diff --git a/pkg/cloud/cloud_test.go b/pkg/cloud/cloud_test.go
index 28550d1d..74d0f0ee 100644
--- a/pkg/cloud/cloud_test.go
+++ b/pkg/cloud/cloud_test.go
@@ -67,6 +67,22 @@ func TestCreateDisk(t *testing.T) {
 			},
 			expErr: nil,
 		},
+		{
+			name:       "success: normal with gp2 options",
+			volumeName: "vol-test-name",
+			diskOptions: &DiskOptions{
+				CapacityBytes: util.GiBToBytes(1),
+				VolumeType:    VolumeTypeGP2,
+				Tags:          map[string]string{VolumeNameTagKey: "vol-test"},
+			},
+			expCreateVolumeInput: &ec2.CreateVolumeInput{},
+			expDisk: &Disk{
+				VolumeID:         "vol-test",
+				CapacityGiB:      1,
+				AvailabilityZone: defaultZone,
+			},
+			expErr: nil,
+		},
 		{
 			name:       "success: normal with io2 options",
 			volumeName: "vol-test-name",
@@ -531,6 +547,12 @@ func TestAttachDisk(t *testing.T) {
 			nodeID:   "node-1234",
 			expErr:   fmt.Errorf(""),
 		},
+		{
+			name:     "fail: AttachVolume returned error volumeInUse",
+			volumeID: "vol-test-1234",
+			nodeID:   "node-1234",
+			expErr:   awserr.New("VolumeInUse", "Volume is in use", nil),
+		},
 	}
 
 	for _, tc := range testCases {
diff --git a/pkg/driver/controller_test.go b/pkg/driver/controller_test.go
index ab0298ec..ff181f67 100644
--- a/pkg/driver/controller_test.go
+++ b/pkg/driver/controller_test.go
@@ -50,8 +50,8 @@ func TestNewControllerService(t *testing.T) {
 		testErr    = errors.New("test error")
 		testRegion = "test-region"
 
-		getNewCloudFunc = func(expectedRegion string) func(region string) (cloud.Cloud, error) {
-			return func(region string) (cloud.Cloud, error) {
+		getNewCloudFunc = func(expectedRegion string, awsSdkDebugLog bool) func(region string, awsSdkDebugLog bool) (cloud.Cloud, error) {
+			return func(region string, awsSdkDebugLog bool) (cloud.Cloud, error) {
 				if region != expectedRegion {
 					t.Fatalf("expected region %q but got %q", expectedRegion, region)
 				}
@@ -63,30 +63,30 @@ func TestNewControllerService(t *testing.T) {
 	testCases := []struct {
 		name                  string
 		region                string
-		newCloudFunc          func(string) (cloud.Cloud, error)
+		newCloudFunc          func(string, bool) (cloud.Cloud, error)
 		newMetadataFuncErrors bool
 		expectPanic           bool
 	}{
 		{
 			name:         "AWS_REGION variable set, newCloud does not error",
 			region:       "foo",
-			newCloudFunc: getNewCloudFunc("foo"),
+			newCloudFunc: getNewCloudFunc("foo", false),
 		},
 		{
 			name:   "AWS_REGION variable set, newCloud errors",
 			region: "foo",
-			newCloudFunc: func(region string) (cloud.Cloud, error) {
+			newCloudFunc: func(region string, awsSdkDebugLog bool) (cloud.Cloud, error) {
 				return nil, testErr
 			},
 			expectPanic: true,
 		},
 		{
 			name:         "AWS_REGION variable not set, newMetadata does not error",
-			newCloudFunc: getNewCloudFunc(testRegion),
+			newCloudFunc: getNewCloudFunc(testRegion, false),
 		},
 		{
 			name:                  "AWS_REGION variable not set, newMetadata errors",
-			newCloudFunc:          getNewCloudFunc(testRegion),
+			newCloudFunc:          getNewCloudFunc(testRegion, false),
 			newMetadataFuncErrors: true,
 			expectPanic:           true,
 		},
diff --git a/pkg/driver/driver_test.go b/pkg/driver/driver_test.go
index a9bc3f75..b9218abb 100644
--- a/pkg/driver/driver_test.go
+++ b/pkg/driver/driver_test.go
@@ -85,3 +85,12 @@ func TestWithClusterID(t *testing.T) {
 		t.Fatalf("expected kubernetesClusterID option got set to %s but is set to %s", id, options.kubernetesClusterID)
 	}
 }
+
+func TestWithAwsSdkDebugLog(t *testing.T) {
+	var enableSdkDebugLog bool = true
+	options := &DriverOptions{}
+	WithAwsSdkDebugLog(enableSdkDebugLog)(options)
+	if options.awsSdkDebugLog != enableSdkDebugLog {
+		t.Fatalf("expected awsSdkDebugLog option got set to %v but is set to %v", enableSdkDebugLog, options.awsSdkDebugLog)
+	}
+}
diff --git a/tests/e2e/dynamic_provisioning.go b/tests/e2e/dynamic_provisioning.go
index a5ab32d6..16498f5b 100644
--- a/tests/e2e/dynamic_provisioning.go
+++ b/tests/e2e/dynamic_provisioning.go
@@ -366,7 +366,7 @@ var _ = Describe("[ebs-csi-e2e] [single-az] Dynamic Provisioning", func() {
 		availabilityZones := strings.Split(os.Getenv(awsAvailabilityZonesEnv), ",")
 		availabilityZone := availabilityZones[rand.Intn(len(availabilityZones))]
 		region := availabilityZone[0 : len(availabilityZone)-1]
-		cloud, err := awscloud.NewCloud(region)
+		cloud, err := awscloud.NewCloud(region, false)
 		if err != nil {
 			Fail(fmt.Sprintf("could not get NewCloud: %v", err))
 		}
diff --git a/tests/e2e/pre_provsioning.go b/tests/e2e/pre_provsioning.go
index 76122794..618aba3c 100644
--- a/tests/e2e/pre_provsioning.go
+++ b/tests/e2e/pre_provsioning.go
@@ -84,7 +84,7 @@ var _ = Describe("[ebs-csi-e2e] [single-az] Pre-Provisioned", func() {
 			Tags:             map[string]string{awscloud.VolumeNameTagKey: dummyVolumeName},
 		}
 		var err error
-		cloud, err = awscloud.NewCloud(region)
+		cloud, err = awscloud.NewCloud(region, false)
 		if err != nil {
 			Fail(fmt.Sprintf("could not get NewCloud: %v", err))
 		}