Repository: kubernetes-sigs/aws-ebs-csi-driver
Pull request #2500 (2025-05-21 18:13:26)

Return `ResourceExhausted` gRPC error code in `ControllerPublishVolume` when `AttachmentLimitExceeded`
#### What type of PR is this?

/kind cleanup
<!--
Add one of the following kinds:
/kind bug
/kind cleanup
/kind documentation
/kind feature
-->

#### What is this PR about? / Why do we need it?

This PR updates the Attach code path to return the `ResourceExhausted` gRPC error code in `ControllerPublishVolume` when the attachment limit is exceeded for the instance. In newer versions of Kubernetes, CSI drivers can opt in to a feature that allows for updating the `CSINode.Allocatables.Count` value when this gRPC error code is reported in the VolumeAttachment object.

#### How was this change tested?

```
make verify && make test
```

Manually, by deploying the driver with `volumeAttachLimit: 200` and scaling up StatefulSet to induce this failure mode. See driver log:

```
E0521 18:02:25.853256       1 driver.go:108] "GRPC error" err="rpc error: code = ResourceExhausted desc = Attachment limit exceeded for volume \"vol-0d8d2bab6fc5dd31b\" on node \"i-00f897f741bb9326a\": attachment limit exceeded: operation error EC2: AttachVolume, https response error StatusCode: 400, RequestID: c56fd5ce-f4d5-4857-b1dc-908c935a705a, api error AttachmentLimitExceeded: You may attach up to 27 devices for this instance type. Please use a larger instance size to attach more volumes. For a detailed list of limits please see https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/volume_limits.html"
```

#### Does this PR introduce a user-facing change?
<!--
If no, just write "NONE" in the release-note block below.
If yes, enter your extended release note in the block below.
-->
```release-note
NONE
```



Patch:
diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml
index 04c00a33..b08ed382 100644
--- a/.github/dependabot.yaml
+++ b/.github/dependabot.yaml
@@ -15,45 +15,45 @@
 version: 2
 enable-beta-ecosystems: true
 updates:
-  - package-ecosystem: gomod
-    directories: ["/", "/tests/e2e/"]
-    allow:
-      - dependency-type: "all"
-    schedule:
-      interval: weekly
-      day: "wednesday"
-      time: "06:00"
-      timezone: "America/New_York"
-    groups:
-      k8s-dependencies:
-        patterns:
-          - "k8s.io*"
-          - "sigs.k8s.io*"
-          - "github.com/kubernetes-csi*"
-      misc-dependencies:
-        patterns:
-          - "*"
-        exclude-patterns:
-          - "k8s.io*"
-          - "sigs.k8s.io*"
-          - "github.com/kubernetes-csi*"
-    labels:
-      - "area/dependency"
-      - "ok-to-test"
-      - "release-note-none"
-  - package-ecosystem: "github-actions"
-    directory: "/"
-    schedule:
-      interval: weekly
-      # Wednesday chosen to minimize time after Windows Patch Tuesdays
-      day: "wednesday"
-      time: "06:00"
-      timezone: "America/New_York"
-    groups:
-      actions:
-        patterns:
-          - "*"
-    labels:
-      - "area/dependency"
-      - "ok-to-test"
-      - "release-note-none"
+- package-ecosystem: gomod
+  directories: ["/", "/tests/e2e/"]
+  allow:
+  - dependency-type: "all"
+  schedule:
+    interval: weekly
+    day: "wednesday"
+    time: "06:00"
+    timezone: "America/New_York"
+  groups:
+    k8s-dependencies:
+      patterns:
+        - "k8s.io*"
+        - "sigs.k8s.io*"
+        - "github.com/kubernetes-csi*"
+    misc-dependencies:
+      patterns:
+        - "*"
+      exclude-patterns:
+        - "k8s.io*"
+        - "sigs.k8s.io*"
+        - "github.com/kubernetes-csi*"
+  labels:
+    - "area/dependency"
+    - "ok-to-test"
+    - "release-note-none"
+- package-ecosystem: "github-actions"
+  directory: "/"
+  schedule:
+    interval: weekly
+    # Wednesday chosen to minimize time after Windows Patch Tuesdays
+    day: "wednesday"
+    time: "06:00"
+    timezone: "America/New_York"
+  groups:
+    actions:
+      patterns:
+        - "*"
+  labels:
+    - "area/dependency"
+    - "ok-to-test"
+    - "release-note-none"
diff --git a/pkg/cloud/cloud.go b/pkg/cloud/cloud.go
index 4bc69064..4b37e806 100644
--- a/pkg/cloud/cloud.go
+++ b/pkg/cloud/cloud.go
@@ -171,6 +171,9 @@ var (
 
 	// ErrInvalidRequest is returned if parameters were rejected by driver.
 	ErrInvalidRequest = errors.New("invalid request")
+
+	// ErrAttachmentLimitExceeded is returned if the attachment limit is exceeded.
+	ErrAttachmentLimitExceeded = errors.New("attachment limit exceeded")
 )
 
 // Set during build time via -ldflags.
@@ -930,6 +933,9 @@ func (c *cloud) AttachDisk(ctx context.Context, volumeID, nodeID string) (string
 				// Store such bad names in the "likely bad" map to be considered last in future attempts
 				likelyBadDeviceNames.Store(device.Path, struct{}{})
 			}
+			if isAWSErrorAttachmentLimitExceeded(attachErr) {
+				return "", fmt.Errorf("%w: %w", ErrAttachmentLimitExceeded, attachErr)
+			}
 			return "", fmt.Errorf("could not attach volume %q to node %q: %w", volumeID, nodeID, attachErr)
 		}
 		likelyBadDeviceNames.Delete(device.Path)
@@ -1662,6 +1668,12 @@ func isAWSErrorBlockDeviceInUse(err error) bool {
 	return false
 }
 
+// isAWSErrorAttachmentLimitExceeded checks if the error is an AttachmentLimitExceeded error.
+// This error is reported when the maximum number of attachments for an instance is exceeded.
+func isAWSErrorAttachmentLimitExceeded(err error) bool {
+	return isAWSError(err, "AttachmentLimitExceeded")
+}
+
 // isAWSErrorInvalidParameter returns a boolean indicating whether the
 // given error is caused by invalid parameters in a EC2 API request.
 func isAWSErrorInvalidParameter(err error) bool {
diff --git a/pkg/driver/controller.go b/pkg/driver/controller.go
index 4f531470..79913307 100644
--- a/pkg/driver/controller.go
+++ b/pkg/driver/controller.go
@@ -422,6 +422,9 @@ func (d *ControllerService) ControllerPublishVolume(ctx context.Context, req *cs
 			klog.InfoS("ControllerPublishVolume: volume not found", "volumeID", volumeID, "nodeID", nodeID)
 			return nil, status.Errorf(codes.NotFound, "Volume %q not found", volumeID)
 		}
+		if errors.Is(err, cloud.ErrAttachmentLimitExceeded) {
+			return nil, status.Errorf(codes.ResourceExhausted, "Attachment limit exceeded for volume %q on node %q: %v", volumeID, nodeID, err)
+		}
 		return nil, status.Errorf(codes.Internal, "Could not attach volume %q to node %q: %v", volumeID, nodeID, err)
 	}
 	klog.InfoS("ControllerPublishVolume: attached", "volumeID", volumeID, "nodeID", nodeID, "devicePath", devicePath)


Test patch:
diff --git a/pkg/cloud/cloud_test.go b/pkg/cloud/cloud_test.go
index 4d1ff280..5c9ccdda 100644
--- a/pkg/cloud/cloud_test.go
+++ b/pkg/cloud/cloud_test.go
@@ -1685,6 +1685,30 @@ func TestAttachDisk(t *testing.T) {
 				)
 			},
 		},
+		{
+			name:     "fail: AttachVolume returned attachment limit exceeded error",
+			volumeID: defaultVolumeID,
+			nodeID:   defaultNodeID,
+			path:     defaultPath,
+			expErr: fmt.Errorf("%w: %w", ErrAttachmentLimitExceeded, &smithy.GenericAPIError{
+				Code:    "AttachmentLimitExceeded",
+				Message: "Volume attachment limit exceeded",
+			}),
+			mockFunc: func(mockEC2 *MockEC2API, ctx context.Context, volumeID, nodeID, nodeID2, path string, dm dm.DeviceManager) {
+				instanceRequest := createInstanceRequest(nodeID)
+				attachRequest := createAttachRequest(volumeID, nodeID, path)
+				attachLimitErr := &smithy.GenericAPIError{
+					Code:    "AttachmentLimitExceeded",
+					Message: "Volume attachment limit exceeded",
+				}
+
+				gomock.InOrder(
+					mockEC2.EXPECT().DescribeInstances(ctx, instanceRequest).Return(newDescribeInstancesOutput(nodeID), nil),
+					mockEC2.EXPECT().AttachVolume(ctx, attachRequest, gomock.Any()).Return(nil, attachLimitErr),
+				)
+			},
+		},
+
 		{
 			name:     "success: AttachVolume multi-attach",
 			volumeID: defaultVolumeID,
diff --git a/pkg/driver/controller_test.go b/pkg/driver/controller_test.go
index 2183bad6..7e17fa25 100644
--- a/pkg/driver/controller_test.go
+++ b/pkg/driver/controller_test.go
@@ -3536,6 +3536,16 @@ func TestControllerPublishVolume(t *testing.T) {
 			},
 			errorCode: codes.OK,
 		},
+		{
+			name:             "ResourceExhausted error when attachment limit is exceeded",
+			volumeID:         "vol-test",
+			nodeID:           expInstanceID,
+			volumeCapability: stdVolCap,
+			mockAttach: func(mockCloud *cloud.MockCloud, ctx context.Context, volumeID string, nodeID string) {
+				mockCloud.EXPECT().AttachDisk(gomock.Eq(ctx), gomock.Eq(volumeID), gomock.Eq(expInstanceID)).Return("", cloud.ErrAttachmentLimitExceeded)
+			},
+			errorCode: codes.ResourceExhausted,
+		},
 		{
 			name:             "AttachDisk when volume is already attached to the node",
 			volumeID:         "vol-test",