Repository: kubernetes-sigs/gcp-compute-persistent-disk-csi-driver

Issue:
csi-snapshotter reconciliation exponential backoff can exceed test timeout limits
See https://github.com/kubernetes-csi/external-snapshotter/issues/1282

The rate limit can very easily be exceeded for the non-error requeue case for snapshot reconciliation. This is causing presubmits and testgrids to flake very consistently.

A short term fix is to reduce `--retry-interval-max` for drivers deployed in test.

Hints from issue comments:

Solution patch:
diff --git a/Makefile b/Makefile
index a827ca7e..8a375b91 100644
--- a/Makefile
+++ b/Makefile
@@ -24,6 +24,7 @@ DRIVERWINDOWSBINARY=${DRIVERBINARY}.exe
 DOCKER=DOCKER_CLI_EXPERIMENTAL=enabled docker
 
 BASE_IMAGE_LTSC2019=mcr.microsoft.com/windows/servercore:ltsc2019
+BASE_IMAGE_LTSC2022=mcr.microsoft.com/windows/servercore:ltsc2022
 
 # Both arrays MUST be index aligned.
 WINDOWS_IMAGE_TAGS=ltsc2019
@@ -60,14 +61,20 @@ build-and-push-windows-container-ltsc2019: require-GCE_PD_CSI_STAGING_IMAGE init
 		--build-arg BASE_IMAGE=$(BASE_IMAGE_LTSC2019) \
 		--build-arg STAGINGVERSION=$(STAGINGVERSION) --push --provenance=false .
 
-build-and-push-multi-arch: build-and-push-container-linux-amd64 build-and-push-container-linux-arm64 build-and-push-windows-container-ltsc2019
-	$(DOCKER) manifest create $(STAGINGIMAGE):$(STAGINGVERSION) $(STAGINGIMAGE):$(STAGINGVERSION)_linux_amd64 $(STAGINGIMAGE):$(STAGINGVERSION)_linux_arm64 $(STAGINGIMAGE):$(STAGINGVERSION)_ltsc2019
-	STAGINGIMAGE="$(STAGINGIMAGE)" STAGINGVERSION="$(STAGINGVERSION)" WINDOWS_IMAGE_TAGS="$(WINDOWS_IMAGE_TAGS)" WINDOWS_BASE_IMAGES="$(WINDOWS_BASE_IMAGES)" ./manifest_osversion.sh
+build-and-push-windows-container-ltsc2022: require-GCE_PD_CSI_STAGING_IMAGE init-buildx
+	$(DOCKER) buildx build --file=Dockerfile.Windows --platform=windows/amd64 \
+		-t $(STAGINGIMAGE):$(STAGINGVERSION)_ltsc2022 \
+		--build-arg BASE_IMAGE=$(BASE_IMAGE_LTSC2022) \
+		--build-arg STAGINGVERSION=$(STAGINGVERSION) --push --provenance=false .
+
+build-and-push-multi-arch: build-and-push-container-linux-amd64 build-and-push-container-linux-arm64 build-and-push-windows-container-ltsc2019 build-and-push-windows-container-ltsc2022
+	$(DOCKER) manifest create $(STAGINGIMAGE):$(STAGINGVERSION) $(STAGINGIMAGE):$(STAGINGVERSION)_linux_amd64 $(STAGINGIMAGE):$(STAGINGVERSION)_linux_arm64 $(STAGINGIMAGE):$(STAGINGVERSION)_ltsc2019 $(STAGINGIMAGE):$(STAGINGVERSION)_ltsc2022
+	STAGINGIMAGE="$(STAGINGIMAGE)" STAGINGVERSION="$(STAGINGVERSION)" WINDOWS_IMAGE_TAGS="ltsc2019, ltsc2022" WINDOWS_BASE_IMAGES="$(BASE_IMAGE_LTSC2019), $(BASE_IMAGE_LTSC2022)" ./manifest_osversion.sh
 	$(DOCKER) manifest push -p $(STAGINGIMAGE):$(STAGINGVERSION)
 
-build-and-push-multi-arch-debug: build-and-push-container-linux-debug build-and-push-windows-container-ltsc2019
-	$(DOCKER) manifest create $(STAGINGIMAGE):$(STAGINGVERSION) $(STAGINGIMAGE):$(STAGINGVERSION)_linux $(STAGINGIMAGE):$(STAGINGVERSION)_ltsc2019
-	STAGINGIMAGE="$(STAGINGIMAGE)" STAGINGVERSION="$(STAGINGVERSION)" WINDOWS_IMAGE_TAGS="ltsc2019" WINDOWS_BASE_IMAGES="$(BASE_IMAGE_LTSC2019)" ./manifest_osversion.sh
+build-and-push-multi-arch-debug: build-and-push-container-linux-debug build-and-push-windows-container-ltsc2019 build-and-push-windows-container-ltsc2022
+	$(DOCKER) manifest create $(STAGINGIMAGE):$(STAGINGVERSION) $(STAGINGIMAGE):$(STAGINGVERSION)_linux $(STAGINGIMAGE):$(STAGINGVERSION)_ltsc2019 $(STAGINGIMAGE):$(STAGINGVERSION)_ltsc2022
+	STAGINGIMAGE="$(STAGINGIMAGE)" STAGINGVERSION="$(STAGINGVERSION)" WINDOWS_IMAGE_TAGS="ltsc2019, ltsc2022" WINDOWS_BASE_IMAGES="$(BASE_IMAGE_LTSC2019), $(BASE_IMAGE_LTSC2022)" ./manifest_osversion.sh
 	$(DOCKER) manifest push -p $(STAGINGIMAGE):$(STAGINGVERSION)
 
 push-container: build-container
diff --git a/deploy/kubernetes/base/controller/controller.yaml b/deploy/kubernetes/base/controller/controller.yaml
index 2a37539d..58c0f4d1 100644
--- a/deploy/kubernetes/base/controller/controller.yaml
+++ b/deploy/kubernetes/base/controller/controller.yaml
@@ -126,6 +126,7 @@ spec:
             - "--leader-election"
             - "--leader-election-namespace=$(PDCSI_NAMESPACE)"
             - "--timeout=300s"
+            - "--retry-interval-max=60s"
           env:
             - name: PDCSI_NAMESPACE
               valueFrom:
diff --git a/go.mod b/go.mod
index 7bb77099..bcf75e4f 100644
--- a/go.mod
+++ b/go.mod
@@ -15,7 +15,7 @@ require (
 	github.com/google/uuid v1.6.0
 	github.com/googleapis/gax-go/v2 v2.12.4
 	github.com/hashicorp/golang-lru/v2 v2.0.7
-	github.com/kubernetes-csi/csi-proxy/client v1.2.0
+	github.com/kubernetes-csi/csi-proxy/client v1.2.1
 	github.com/kubernetes-csi/csi-test/v5 v5.3.1
 	github.com/onsi/ginkgo/v2 v2.22.2
 	github.com/onsi/gomega v1.36.2
diff --git a/go.sum b/go.sum
index c234295f..0feaadb5 100644
--- a/go.sum
+++ b/go.sum
@@ -1034,8 +1034,8 @@ github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/kubernetes-csi/csi-proxy/client v1.2.0 h1:8oagHPy7VHbHqu6iOyZyusUvGCyzbPg8OQQVh8e4Q3E=
-github.com/kubernetes-csi/csi-proxy/client v1.2.0/go.mod h1:SfK4HVKQdMH5KrffivddAWgX5hl3P5KmnuOTBbDNboU=
+github.com/kubernetes-csi/csi-proxy/client v1.2.1 h1:6ApXTKp5Rhb+2PrPfnk6C+TDG7XwnWJ9/xmmq7iN5cw=
+github.com/kubernetes-csi/csi-proxy/client v1.2.1/go.mod h1:SfK4HVKQdMH5KrffivddAWgX5hl3P5KmnuOTBbDNboU=
 github.com/kubernetes-csi/csi-test/v5 v5.3.1 h1:Wiukp1In+kif+BFo6q2ExjgB+MbrAz4jZWzGfijypuY=
 github.com/kubernetes-csi/csi-test/v5 v5.3.1/go.mod h1:7hA2cSYJ6T8CraEZPA6zqkLZwemjBD54XAnPsPC3VpA=
 github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs=
diff --git a/pkg/common/constants.go b/pkg/common/constants.go
index a6003b65..d5e5c101 100644
--- a/pkg/common/constants.go
+++ b/pkg/common/constants.go
@@ -57,3 +57,16 @@ const (
 	NodeLabelPrefix         = "cloud.google.com/%s"
 	DataCacheLssdCountLabel = "gke-data-cache-disk"
 )
+
+// doc https://cloud.google.com/compute/docs/disks/hyperdisks#max-total-disks-per-vm
+var Gen4MachineHyperdiskAttachLimitMap = []struct {
+	max   int64
+	value int64
+}{
+	{max: 4, value: 16},
+	{max: 8, value: 24},
+	{max: 16, value: 32},
+	{max: 32, value: 48},
+	{max: 64, value: 64},
+	{max: 1024, value: 128},
+}
diff --git a/pkg/common/utils.go b/pkg/common/utils.go
index 3abe5dde..719dcb1d 100644
--- a/pkg/common/utils.go
+++ b/pkg/common/utils.go
@@ -754,3 +754,13 @@ func ShortString(s string) string {
 	}
 	return string(short)
 }
+
+// MapNumber is a function to map input cpu number to the Hyperdisk attach limit
+func MapNumber(num int64) int64 {
+	for _, r := range Gen4MachineHyperdiskAttachLimitMap {
+		if num <= r.max {
+			return r.value
+		}
+	}
+	return 0
+}
diff --git a/pkg/gce-cloud-provider/compute/gce-compute.go b/pkg/gce-cloud-provider/compute/gce-compute.go
index 63402cd2..c03ceeac 100644
--- a/pkg/gce-cloud-provider/compute/gce-compute.go
+++ b/pkg/gce-cloud-provider/compute/gce-compute.go
@@ -315,6 +315,7 @@ func (cloud *CloudProvider) ListSnapshots(ctx context.Context, filter string) ([
 			return nil, "", err
 		}
 		items = append(items, snapshotList.Items...)
+		nextPageToken = snapshotList.NextPageToken
 	}
 	return items, "", nil
 }
@@ -1310,6 +1311,7 @@ func (cloud *CloudProvider) ListImages(ctx context.Context, filter string) ([]*c
 			return nil, "", err
 		}
 		items = append(items, imageList.Items...)
+		nextPageToken = imageList.NextPageToken
 	}
 	return items, "", nil
 }
diff --git a/pkg/gce-pd-csi-driver/node.go b/pkg/gce-pd-csi-driver/node.go
index 52abeb8e..f45d159b 100644
--- a/pkg/gce-pd-csi-driver/node.go
+++ b/pkg/gce-pd-csi-driver/node.go
@@ -23,6 +23,7 @@ import (
 	"regexp"
 	"runtime"
 	"strconv"
+	"strings"
 	"time"
 
 	"google.golang.org/grpc/codes"
@@ -95,8 +96,12 @@ var _ csi.NodeServer = &GCENodeServer{}
 // node boot disk is considered an attachable disk so effective attach limit is
 // one less.
 const (
-	volumeLimitSmall     int64 = 15
-	volumeLimitBig       int64 = 127
+	volumeLimitSmall int64 = 15
+	volumeLimitBig   int64 = 127
+	// doc https://cloud.google.com/compute/docs/memory-optimized-machines#x4_disks
+	x4HyperdiskLimit int64 = 40
+	// doc https://cloud.google.com/compute/docs/accelerator-optimized-machines#a4-disks
+	a4HyperdiskLimit     int64 = 128
 	defaultLinuxFsType         = "ext4"
 	defaultWindowsFsType       = "ntfs"
 	fsTypeExt3                 = "ext3"
@@ -567,6 +572,9 @@ func (ns *GCENodeServer) NodeGetInfo(ctx context.Context, req *csi.NodeGetInfoRe
 	nodeID := common.CreateNodeID(ns.MetadataService.GetProject(), ns.MetadataService.GetZone(), ns.MetadataService.GetName())
 
 	volumeLimits, err := ns.GetVolumeLimits()
+	if err != nil {
+		klog.Errorf("GetVolumeLimits failed: %v", err.Error())
+	}
 
 	resp := &csi.NodeGetInfoResponse{
 		NodeId:             nodeID,
@@ -733,5 +741,24 @@ func (ns *GCENodeServer) GetVolumeLimits() (int64, error) {
 			return volumeLimitSmall, nil
 		}
 	}
+	gen4MachineTypesPrefix := []string{"c4a-", "c4-", "n4-"}
+	for _, gen4Prefix := range gen4MachineTypesPrefix {
+		if strings.HasPrefix(machineType, gen4Prefix) {
+			cpuString := machineType[strings.LastIndex(machineType, "-")+1:]
+			cpus, err := strconv.ParseInt(cpuString, 10, 64)
+			if err != nil {
+				return volumeLimitSmall, fmt.Errorf("invalid cpuString %s for machine type: %v", cpuString, machineType)
+			}
+			return common.MapNumber(cpus), nil
+
+		}
+		if strings.HasPrefix(machineType, "x4-") {
+			return x4HyperdiskLimit, nil
+		}
+		if strings.HasPrefix(machineType, "a4-") {
+			return a4HyperdiskLimit, nil
+		}
+	}
+
 	return volumeLimitBig, nil
 }
diff --git a/vendor/modules.txt b/vendor/modules.txt
index c0e003a7..eca8eb43 100644
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@@ -197,7 +197,7 @@ github.com/josharian/intern
 # github.com/json-iterator/go v1.1.12
 ## explicit; go 1.12
 github.com/json-iterator/go
-# github.com/kubernetes-csi/csi-proxy/client v1.2.0
+# github.com/kubernetes-csi/csi-proxy/client v1.2.1
 ## explicit; go 1.16
 github.com/kubernetes-csi/csi-proxy/client
 github.com/kubernetes-csi/csi-proxy/client/api/disk/v1



Test patch:
diff --git a/pkg/gce-pd-csi-driver/node_test.go b/pkg/gce-pd-csi-driver/node_test.go
index ee016df0..57c3e82b 100644
--- a/pkg/gce-pd-csi-driver/node_test.go
+++ b/pkg/gce-pd-csi-driver/node_test.go
@@ -224,6 +224,7 @@ func TestNodeGetVolumeLimits(t *testing.T) {
 		name           string
 		machineType    string
 		expVolumeLimit int64
+		expectError    bool
 	}{
 		{
 			name:           "Predifined standard machine",
@@ -255,13 +256,54 @@ func TestNodeGetVolumeLimits(t *testing.T) {
 			machineType:    "e2-micro",
 			expVolumeLimit: volumeLimitSmall,
 		},
+		{
+			name:           "c4-standard-192",
+			machineType:    "c4-standard-192",
+			expVolumeLimit: 128,
+		},
+		{
+			name:           "c4-standard-48",
+			machineType:    "c4-standard-48",
+			expVolumeLimit: 64,
+		},
+		{
+			name:           "c4a-standard-4",
+			machineType:    "c4a-standard-4",
+			expVolumeLimit: 16,
+		},
+		{
+			name:           "n4-standard-16",
+			machineType:    "n4-standard-16",
+			expVolumeLimit: 32,
+		},
+		{
+			name:           "n4-highcpu-4",
+			machineType:    "n4-highcpu-4",
+			expVolumeLimit: 16,
+		},
+		{
+			name:           "invalid gen4 machine type",
+			machineType:    "n4-highcpu-4xyz",
+			expVolumeLimit: volumeLimitSmall,
+			expectError:    true,
+		},
+		{
+			name:           "x4-megamem-960-metal",
+			machineType:    "x4-megamem-960-metal",
+			expVolumeLimit: x4HyperdiskLimit,
+		},
+		{
+			name:           "a4-highgpu-8g",
+			machineType:    "a4-highgpu-8g",
+			expVolumeLimit: a4HyperdiskLimit,
+		},
 	}
 
 	for _, tc := range testCases {
 		t.Logf("Test case: %s", tc.name)
 		metadataservice.SetMachineType(tc.machineType)
 		res, err := ns.NodeGetInfo(context.Background(), req)
-		if err != nil {
+		if err != nil && !tc.expectError {
 			t.Fatalf("Failed to get node info: %v", err)
 		} else {
 			volumeLimit := res.GetMaxVolumesPerNode()
diff --git a/test/e2e/tests/single_zone_e2e_test.go b/test/e2e/tests/single_zone_e2e_test.go
index 09870a3a..8fa882c2 100644
--- a/test/e2e/tests/single_zone_e2e_test.go
+++ b/test/e2e/tests/single_zone_e2e_test.go
@@ -1065,6 +1065,10 @@ var _ = Describe("GCE PD CSI Driver", func() {
 		})
 		Expect(err).To(BeNil(), "Could not wait for snapshot be ready")
 
+		snapshots, err := client.ListSnapshots()
+		Expect(err).To(BeNil(), "Could not list snapshots")
+		Expect(snapshots).To(ContainElement(snapshotID), "Couldn't find snapshot ID")
+
 		defer func() {
 			// Delete Disk
 			err := client.DeleteVolume(volID)
@@ -1124,6 +1128,10 @@ var _ = Describe("GCE PD CSI Driver", func() {
 		Expect(err).To(BeNil(), "Failed to parse snapshot ID")
 		Expect(snapshotType).To(Equal(common.DiskImageType), "Expected images type in snapshot ID")
 
+		snapshots, err := client.ListSnapshots()
+		Expect(err).To(BeNil(), "Could not list snapshots")
+		Expect(snapshots).To(ContainElement(snapshotID), "Couldn't find snapshot ID")
+
 		defer func() {
 			// Delete Disk
 			err := client.DeleteVolume(volID)
diff --git a/test/k8s-integration/cluster.go b/test/k8s-integration/cluster.go
index 88343c53..b411dd00 100644
--- a/test/k8s-integration/cluster.go
+++ b/test/k8s-integration/cluster.go
@@ -155,6 +155,9 @@ func setImageTypeEnvs(imageType string) error {
 	case "cos":
 	case "cos_containerd":
 	case "gci": // GCI/COS is default type and does not need env vars set
+	case "win2019", "win2022":
+		// These are handled by the WINDOWS_NODE_OS_DISTRIBUTION env var and
+		// ingested in https://github.com/kubernetes/kubernetes/blob/ded2956c832502da8a0678f5392c24af1cc9dfc0/cluster/gce/util.sh#L112.
 	case "ubuntu", "ubuntu_containerd":
 		return errors.New("setting environment vars for bringing up *ubuntu* cluster on GCE is unimplemented")
 		/* TODO(dyzz) figure out how to bring up a Ubuntu cluster on GCE. The below doesn't work.
@@ -201,7 +204,17 @@ func clusterUpGKE(gceZone, gceRegion string, numNodes int, numWindowsNodes int,
 	var cmd *exec.Cmd
 	cmdParams := []string{"container", "clusters", "create", *gkeTestClusterName,
 		locationArg, locationVal, "--num-nodes", strconv.Itoa(numNodes),
-		"--quiet", "--machine-type", "n1-standard-2", "--image-type", imageType, "--no-enable-autoupgrade"}
+		"--quiet", "--machine-type", "n1-standard-2", "--no-enable-autoupgrade"}
+	if imageType == "win2019" || imageType == "win2022" {
+		cmdParams = append(cmdParams, "--image-type", "WINDOWS_LTSC_CONTAINERD")
+		if imageType == "win2019" {
+			cmdParams = append(cmdParams, "--windows-os-version", "ltsc2019")
+		} else {
+			cmdParams = append(cmdParams, "--windows-os-version", "ltsc2022")
+		}
+	} else {
+		cmdParams = append(cmdParams, "--image-type", imageType)
+	}
 	if isVariableSet(gkeClusterVer) {
 		cmdParams = append(cmdParams, "--cluster-version", *gkeClusterVer)
 	} else {
diff --git a/test/k8s-integration/driver.go b/test/k8s-integration/driver.go
index 4b903234..6f51df9d 100644
--- a/test/k8s-integration/driver.go
+++ b/test/k8s-integration/driver.go
@@ -126,7 +126,7 @@ func deleteDriver(testParams *testParameters, deployOverlayName string) error {
 	return nil
 }
 
-func pushImage(pkgDir, stagingImage, stagingVersion, platform string) error {
+func pushImage(pkgDir, stagingImage, stagingVersion, platform, imageType string) error {
 	err := os.Setenv("GCE_PD_CSI_STAGING_VERSION", stagingVersion)
 	if err != nil {
 		return err
@@ -138,7 +138,6 @@ func pushImage(pkgDir, stagingImage, stagingVersion, platform string) error {
 	var cmd *exec.Cmd
 
 	if platform == "windows" {
-		// build multi-arch image which can work for both Linux and Windows
 		cmd = exec.Command("make", "-C", pkgDir, "build-and-push-multi-arch",
 			fmt.Sprintf("GCE_PD_CSI_STAGING_VERSION=%s", stagingVersion),
 			fmt.Sprintf("GCE_PD_CSI_STAGING_IMAGE=%s", stagingImage))
diff --git a/test/k8s-integration/main.go b/test/k8s-integration/main.go
index 9edb0ef5..a3d82668 100644
--- a/test/k8s-integration/main.go
+++ b/test/k8s-integration/main.go
@@ -300,7 +300,7 @@ func handle() error {
 	// Build and push the driver, if required. Defer the driver image deletion.
 	if *doDriverBuild {
 		klog.Infof("Building GCE PD CSI Driver")
-		err := pushImage(testParams.pkgDir, *stagingImage, testParams.stagingVersion, testParams.platform)
+		err := pushImage(testParams.pkgDir, *stagingImage, testParams.stagingVersion, testParams.platform, *imageType)
 		if err != nil {
 			return fmt.Errorf("failed pushing image: %v", err.Error())
 		}
diff --git a/test/remote/client-wrappers.go b/test/remote/client-wrappers.go
index 9994f9ea..9f869623 100644
--- a/test/remote/client-wrappers.go
+++ b/test/remote/client-wrappers.go
@@ -310,6 +310,20 @@ func (c *CsiClient) NodeGetVolumeStats(volumeID, volumePath string) (available,
 	return
 }
 
+func (c *CsiClient) ListSnapshots() ([]string, error) {
+	lsr := &csipb.ListSnapshotsRequest{}
+	resp, err := c.ctrlClient.ListSnapshots(context.Background(), lsr)
+	if err != nil {
+		return nil, err
+	}
+
+	snapshots := []string{}
+	for _, e := range resp.Entries {
+		snapshots = append(snapshots, e.Snapshot.SnapshotId)
+	}
+	return snapshots, nil
+}
+
 func (c *CsiClient) CreateSnapshot(snapshotName, sourceVolumeId string, params map[string]string) (string, error) {
 
 	csr := &csipb.CreateSnapshotRequest{
diff --git a/test/run-windows-k8s-integration.sh b/test/run-windows-k8s-integration.sh
index e10654cd..01f7153d 100755
--- a/test/run-windows-k8s-integration.sh
+++ b/test/run-windows-k8s-integration.sh
@@ -21,6 +21,7 @@ readonly test_version=${TEST_VERSION:-master}
 readonly gce_zone=${GCE_CLUSTER_ZONE:-us-central1-b}
 readonly use_kubetest2=${USE_KUBETEST2:-true}
 readonly num_windows_nodes=${NUM_WINDOWS_NODES:-3}
+readonly windows_distribution=${WINDOWS_NODE_OS_DISTRIBUTION:-win2019}
 
 # build platforms for `make quick-release`
 export KUBE_BUILD_PLATFORMS=${KUBE_BUILD_PLATFORMS:-"linux/amd64 windows/amd64"}
@@ -35,6 +36,11 @@ if [ "$use_kubetest2" = true ]; then
     go install sigs.k8s.io/kubetest2/kubetest2-tester-ginkgo@${kt2_version}
 fi
 
+if [ "$windows_distribution" != "win2019" -a "$windows_distribution" != "win2022"]; then
+    print "Invalid windows distribution $windows_distribution provided. Exiting."
+    exit 1
+fi
+
 ${PKGDIR}/bin/k8s-integration-test \
     --run-in-prow=true \
     --service-account-file="${E2E_GOOGLE_APPLICATION_CREDENTIALS}" \
@@ -54,4 +60,5 @@ ${PKGDIR}/bin/k8s-integration-test \
     --storageclass-files=sc-windows.yaml \
     --snapshotclass-files=pd-volumesnapshotclass.yaml \
     --test-focus='External.Storage' \
-    --use-kubetest2="${use_kubetest2}"
+    --use-kubetest2="${use_kubetest2}" \
+    --image-type="${windows_distribution}"