Repository: go-to-k/delstack (go)

Issue:
[Feature] Retry handling for other IAM operations
Throttling errors may occur with APIs such as GetRole that are not currently retried.

It could also be applied to ListAttachedRolePolicies.

```
ERR operation error IAM: GetRole, exceeded maximum number of attempts, 3, https response error StatusCode: 400, RequestID: 75592ee3-8859-435f-96b3-dc0c40e5153a, api error Throttling: Rate exceeded
```

Pull request description:
Refactor/82 iam reytry
Closes #82 

Patch:
diff --git a/internal/operation/role.go b/internal/operation/role.go
index 6053ff7..80eb93f 100644
--- a/internal/operation/role.go
+++ b/internal/operation/role.go
@@ -54,7 +54,7 @@ func (o *RoleOperator) DeleteResources(ctx context.Context) error {
 }
 
 func (o *RoleOperator) DeleteRole(ctx context.Context, roleName *string) error {
-	exists, err := o.client.CheckRoleExists(ctx, roleName)
+	exists, err := o.client.CheckRoleExists(ctx, roleName, sleepTimeSecForIam)
 	if err != nil {
 		return err
 	}
@@ -62,7 +62,7 @@ func (o *RoleOperator) DeleteRole(ctx context.Context, roleName *string) error {
 		return nil
 	}
 
-	policies, err := o.client.ListAttachedRolePolicies(ctx, roleName)
+	policies, err := o.client.ListAttachedRolePolicies(ctx, roleName, sleepTimeSecForIam)
 	if err != nil {
 		return err
 	}
diff --git a/pkg/client/iam.go b/pkg/client/iam.go
index a02daf1..f4efcb9 100644
--- a/pkg/client/iam.go
+++ b/pkg/client/iam.go
@@ -10,10 +10,10 @@ import (
 
 type IIam interface {
 	DeleteRole(ctx context.Context, roleName *string, sleepTimeSec int) error
-	ListAttachedRolePolicies(ctx context.Context, roleName *string) ([]types.AttachedPolicy, error)
+	ListAttachedRolePolicies(ctx context.Context, roleName *string, sleepTimeSec int) ([]types.AttachedPolicy, error)
 	DetachRolePolicies(ctx context.Context, roleName *string, policies []types.AttachedPolicy, sleepTimeSec int) error
 	DetachRolePolicy(ctx context.Context, roleName *string, PolicyArn *string, sleepTimeSec int) error
-	CheckRoleExists(ctx context.Context, roleName *string) (bool, error)
+	CheckRoleExists(ctx context.Context, roleName *string, sleepTimeSec int) (bool, error)
 }
 
 var _ IIam = (*Iam)(nil)
@@ -50,7 +50,7 @@ func (i *Iam) DeleteRole(ctx context.Context, roleName *string, sleepTimeSec int
 	return err
 }
 
-func (i *Iam) ListAttachedRolePolicies(ctx context.Context, roleName *string) ([]types.AttachedPolicy, error) {
+func (i *Iam) ListAttachedRolePolicies(ctx context.Context, roleName *string, sleepTimeSec int) ([]types.AttachedPolicy, error) {
 	var marker *string
 	policies := []types.AttachedPolicy{}
 
@@ -66,7 +66,21 @@ func (i *Iam) ListAttachedRolePolicies(ctx context.Context, roleName *string) ([
 			Marker:   marker,
 		}
 
-		output, err := i.client.ListAttachedRolePolicies(ctx, input)
+		retryable := func(err error) bool {
+			return strings.Contains(err.Error(), "api error Throttling: Rate exceeded")
+		}
+
+		output, err := Retry(
+			&RetryInput[iam.ListAttachedRolePoliciesInput, iam.ListAttachedRolePoliciesOutput, iam.Options]{
+				Ctx:              ctx,
+				SleepTimeSec:     sleepTimeSec,
+				TargetResource:   roleName,
+				Input:            input,
+				ApiCaller:        i.client.ListAttachedRolePolicies,
+				RetryableChecker: retryable,
+			},
+		)
+
 		if err != nil {
 			return nil, err
 		}
@@ -115,12 +129,26 @@ func (i *Iam) DetachRolePolicy(ctx context.Context, roleName *string, PolicyArn
 	return err
 }
 
-func (i *Iam) CheckRoleExists(ctx context.Context, roleName *string) (bool, error) {
+func (i *Iam) CheckRoleExists(ctx context.Context, roleName *string, sleepTimeSec int) (bool, error) {
 	input := &iam.GetRoleInput{
 		RoleName: roleName,
 	}
 
-	_, err := i.client.GetRole(ctx, input)
+	retryable := func(err error) bool {
+		return strings.Contains(err.Error(), "api error Throttling: Rate exceeded")
+	}
+
+	_, err := Retry(
+		&RetryInput[iam.GetRoleInput, iam.GetRoleOutput, iam.Options]{
+			Ctx:              ctx,
+			SleepTimeSec:     sleepTimeSec,
+			TargetResource:   roleName,
+			Input:            input,
+			ApiCaller:        i.client.GetRole,
+			RetryableChecker: retryable,
+		},
+	)
+
 	if err != nil && strings.Contains(err.Error(), "NoSuchEntity") {
 		return false, nil
 	}
diff --git a/pkg/client/iam_mock.go b/pkg/client/iam_mock.go
index 7564c34..8b0727c 100644
--- a/pkg/client/iam_mock.go
+++ b/pkg/client/iam_mock.go
@@ -31,7 +31,7 @@ func (m *MockIam) DeleteRole(ctx context.Context, roleName *string, sleepTimeSec
 	return nil
 }
 
-func (m *MockIam) ListAttachedRolePolicies(ctx context.Context, roleName *string) ([]types.AttachedPolicy, error) {
+func (m *MockIam) ListAttachedRolePolicies(ctx context.Context, roleName *string, sleepTimeSec int) ([]types.AttachedPolicy, error) {
 	output := []types.AttachedPolicy{
 		{
 			PolicyArn:  aws.String("PolicyArn1"),
@@ -53,7 +53,7 @@ func (m *MockIam) DetachRolePolicy(ctx context.Context, roleName *string, Policy
 	return nil
 }
 
-func (m *MockIam) CheckRoleExists(ctx context.Context, repositoryName *string) (bool, error) {
+func (m *MockIam) CheckRoleExists(ctx context.Context, repositoryName *string, sleepTimeSec int) (bool, error) {
 	return true, nil
 }
 
@@ -67,7 +67,7 @@ func (m *AllErrorMockIam) DeleteRole(ctx context.Context, roleName *string, slee
 	return fmt.Errorf("DeleteRoleError")
 }
 
-func (m *AllErrorMockIam) ListAttachedRolePolicies(ctx context.Context, roleName *string) ([]types.AttachedPolicy, error) {
+func (m *AllErrorMockIam) ListAttachedRolePolicies(ctx context.Context, roleName *string, sleepTimeSec int) ([]types.AttachedPolicy, error) {
 	return nil, fmt.Errorf("ListAttachedRolePoliciesError")
 }
 
@@ -79,7 +79,7 @@ func (m *AllErrorMockIam) DetachRolePolicy(ctx context.Context, roleName *string
 	return fmt.Errorf("DetachRolePolicyError")
 }
 
-func (m *AllErrorMockIam) CheckRoleExists(ctx context.Context, repositoryName *string) (bool, error) {
+func (m *AllErrorMockIam) CheckRoleExists(ctx context.Context, repositoryName *string, sleepTimeSec int) (bool, error) {
 	return false, fmt.Errorf("GetRoleError")
 }
 
@@ -93,7 +93,7 @@ func (m *DeleteRoleErrorMockIam) DeleteRole(ctx context.Context, roleName *strin
 	return fmt.Errorf("DeleteRoleError")
 }
 
-func (m *DeleteRoleErrorMockIam) ListAttachedRolePolicies(ctx context.Context, roleName *string) ([]types.AttachedPolicy, error) {
+func (m *DeleteRoleErrorMockIam) ListAttachedRolePolicies(ctx context.Context, roleName *string, sleepTimeSec int) ([]types.AttachedPolicy, error) {
 	output := []types.AttachedPolicy{
 		{
 			PolicyArn:  aws.String("PolicyArn1"),
@@ -115,7 +115,7 @@ func (m *DeleteRoleErrorMockIam) DetachRolePolicy(ctx context.Context, roleName
 	return nil
 }
 
-func (m *DeleteRoleErrorMockIam) CheckRoleExists(ctx context.Context, repositoryName *string) (bool, error) {
+func (m *DeleteRoleErrorMockIam) CheckRoleExists(ctx context.Context, repositoryName *string, sleepTimeSec int) (bool, error) {
 	return true, nil
 }
 
@@ -129,7 +129,7 @@ func (m *ListAttachedRolePoliciesErrorMockIam) DeleteRole(ctx context.Context, r
 	return nil
 }
 
-func (m *ListAttachedRolePoliciesErrorMockIam) ListAttachedRolePolicies(ctx context.Context, roleName *string) ([]types.AttachedPolicy, error) {
+func (m *ListAttachedRolePoliciesErrorMockIam) ListAttachedRolePolicies(ctx context.Context, roleName *string, sleepTimeSec int) ([]types.AttachedPolicy, error) {
 	return nil, fmt.Errorf("ListAttachedRolePoliciesError")
 }
 
@@ -141,7 +141,7 @@ func (m *ListAttachedRolePoliciesErrorMockIam) DetachRolePolicy(ctx context.Cont
 	return nil
 }
 
-func (m *ListAttachedRolePoliciesErrorMockIam) CheckRoleExists(ctx context.Context, repositoryName *string) (bool, error) {
+func (m *ListAttachedRolePoliciesErrorMockIam) CheckRoleExists(ctx context.Context, repositoryName *string, sleepTimeSec int) (bool, error) {
 	return true, nil
 }
 
@@ -155,7 +155,7 @@ func (m *DetachRolePoliciesErrorMockIam) DeleteRole(ctx context.Context, roleNam
 	return nil
 }
 
-func (m *DetachRolePoliciesErrorMockIam) ListAttachedRolePolicies(ctx context.Context, roleName *string) ([]types.AttachedPolicy, error) {
+func (m *DetachRolePoliciesErrorMockIam) ListAttachedRolePolicies(ctx context.Context, roleName *string, sleepTimeSec int) ([]types.AttachedPolicy, error) {
 	output := []types.AttachedPolicy{
 		{
 			PolicyArn:  aws.String("PolicyArn1"),
@@ -177,7 +177,7 @@ func (m *DetachRolePoliciesErrorMockIam) DetachRolePolicy(ctx context.Context, r
 	return nil
 }
 
-func (m *DetachRolePoliciesErrorMockIam) CheckRoleExists(ctx context.Context, repositoryName *string) (bool, error) {
+func (m *DetachRolePoliciesErrorMockIam) CheckRoleExists(ctx context.Context, repositoryName *string, sleepTimeSec int) (bool, error) {
 	return true, nil
 }
 
@@ -191,7 +191,7 @@ func (m *DetachRolePoliciesErrorAfterZeroLengthMockIam) DeleteRole(ctx context.C
 	return nil
 }
 
-func (m *DetachRolePoliciesErrorAfterZeroLengthMockIam) ListAttachedRolePolicies(ctx context.Context, roleName *string) ([]types.AttachedPolicy, error) {
+func (m *DetachRolePoliciesErrorAfterZeroLengthMockIam) ListAttachedRolePolicies(ctx context.Context, roleName *string, sleepTimeSec int) ([]types.AttachedPolicy, error) {
 	output := []types.AttachedPolicy{}
 	return output, nil
 }
@@ -204,7 +204,7 @@ func (m *DetachRolePoliciesErrorAfterZeroLengthMockIam) DetachRolePolicy(ctx con
 	return nil
 }
 
-func (m *DetachRolePoliciesErrorAfterZeroLengthMockIam) CheckRoleExists(ctx context.Context, repositoryName *string) (bool, error) {
+func (m *DetachRolePoliciesErrorAfterZeroLengthMockIam) CheckRoleExists(ctx context.Context, repositoryName *string, sleepTimeSec int) (bool, error) {
 	return true, nil
 }
 
@@ -218,7 +218,7 @@ func (m *CheckRoleExistsErrorMockRole) DeleteRole(ctx context.Context, roleName
 	return nil
 }
 
-func (m *CheckRoleExistsErrorMockRole) ListAttachedRolePolicies(ctx context.Context, roleName *string) ([]types.AttachedPolicy, error) {
+func (m *CheckRoleExistsErrorMockRole) ListAttachedRolePolicies(ctx context.Context, roleName *string, sleepTimeSec int) ([]types.AttachedPolicy, error) {
 	output := []types.AttachedPolicy{
 		{
 			PolicyArn:  aws.String("PolicyArn1"),
@@ -240,7 +240,7 @@ func (m *CheckRoleExistsErrorMockRole) DetachRolePolicy(ctx context.Context, rol
 	return nil
 }
 
-func (m *CheckRoleExistsErrorMockRole) CheckRoleExists(ctx context.Context, repositoryName *string) (bool, error) {
+func (m *CheckRoleExistsErrorMockRole) CheckRoleExists(ctx context.Context, repositoryName *string, sleepTimeSec int) (bool, error) {
 	return false, fmt.Errorf("GetRoleError")
 }
 
@@ -254,7 +254,7 @@ func (m *CheckRoleNotExistsMockRole) DeleteRole(ctx context.Context, roleName *s
 	return nil
 }
 
-func (m *CheckRoleNotExistsMockRole) ListAttachedRolePolicies(ctx context.Context, roleName *string) ([]types.AttachedPolicy, error) {
+func (m *CheckRoleNotExistsMockRole) ListAttachedRolePolicies(ctx context.Context, roleName *string, sleepTimeSec int) ([]types.AttachedPolicy, error) {
 	output := []types.AttachedPolicy{
 		{
 			PolicyArn:  aws.String("PolicyArn1"),
@@ -276,6 +276,6 @@ func (m *CheckRoleNotExistsMockRole) DetachRolePolicy(ctx context.Context, roleN
 	return nil
 }
 
-func (m *CheckRoleNotExistsMockRole) CheckRoleExists(ctx context.Context, repositoryName *string) (bool, error) {
+func (m *CheckRoleNotExistsMockRole) CheckRoleExists(ctx context.Context, repositoryName *string, sleepTimeSec int) (bool, error) {
 	return false, nil
 }


Test patch:
diff --git a/pkg/client/iam_test.go b/pkg/client/iam_test.go
index 46465ec..69b98cd 100644
--- a/pkg/client/iam_test.go
+++ b/pkg/client/iam_test.go
@@ -256,6 +256,31 @@ func TestIam_ListAttachedRolePolicies(t *testing.T) {
 			},
 			wantErr: true,
 		},
+		{
+			name: "list attached role policies failure for api error",
+			args: args{
+				ctx:      context.Background(),
+				roleName: aws.String("test"),
+				withAPIOptionsFunc: func(stack *middleware.Stack) error {
+					return stack.Finalize.Add(
+						middleware.FinalizeMiddlewareFunc(
+							"ListAttachedRolePoliciesApiErrorMock",
+							func(context.Context, middleware.FinalizeInput, middleware.FinalizeHandler) (middleware.FinalizeOutput, middleware.Metadata, error) {
+								return middleware.FinalizeOutput{
+									Result: &iam.ListAttachedRolePoliciesOutput{},
+								}, middleware.Metadata{}, fmt.Errorf("api error Throttling: Rate exceeded")
+							},
+						),
+						middleware.Before,
+					)
+				},
+			},
+			want: want{
+				output: nil,
+				err:    fmt.Errorf("RetryCountOverError: test, operation error IAM: ListAttachedRolePolicies, api error Throttling: Rate exceeded\nRetryCount(" + strconv.Itoa(maxRetryCount) + ") over, but failed to delete. "),
+			},
+			wantErr: true,
+		},
 		{
 			name: "list attached role policies with next marker successfully",
 			args: args{
@@ -406,6 +431,66 @@ func TestIam_ListAttachedRolePolicies(t *testing.T) {
 			},
 			wantErr: true,
 		},
+		{
+			name: "list attached role policies with next marker failure for api error",
+			args: args{
+				ctx:      context.Background(),
+				roleName: aws.String("test"),
+				withAPIOptionsFunc: func(stack *middleware.Stack) error {
+					err := stack.Initialize.Add(
+						middleware.InitializeMiddlewareFunc(
+							"GetNextMarker",
+							getNextMarkerForIamInitialize,
+						), middleware.Before,
+					)
+					if err != nil {
+						return err
+					}
+
+					err = stack.Finalize.Add(
+						middleware.FinalizeMiddlewareFunc(
+							"ListAttachedRolePoliciesWithNextMarkerApiErrorMock",
+							func(ctx context.Context, input middleware.FinalizeInput, handler middleware.FinalizeHandler) (middleware.FinalizeOutput, middleware.Metadata, error) {
+								marker := middleware.GetStackValue(ctx, markerKeyForIam{}).(*string)
+
+								var nextMarker *string
+								var attachedPolicies []types.AttachedPolicy
+								if marker == nil {
+									nextMarker = aws.String("NextMarker")
+									attachedPolicies = []types.AttachedPolicy{
+										{
+											PolicyArn:  aws.String("PolicyArn1"),
+											PolicyName: aws.String("PolicyName1"),
+										},
+										{
+											PolicyArn:  aws.String("PolicyArn2"),
+											PolicyName: aws.String("PolicyName2"),
+										},
+									}
+									return middleware.FinalizeOutput{
+										Result: &iam.ListAttachedRolePoliciesOutput{
+											Marker:           nextMarker,
+											AttachedPolicies: attachedPolicies,
+										},
+									}, middleware.Metadata{}, nil
+								} else {
+									return middleware.FinalizeOutput{
+										Result: &iam.ListAttachedRolePoliciesOutput{},
+									}, middleware.Metadata{}, fmt.Errorf("api error Throttling: Rate exceeded")
+								}
+							},
+						),
+						middleware.Before,
+					)
+					return err
+				},
+			},
+			want: want{
+				output: nil,
+				err:    fmt.Errorf("RetryCountOverError: test, operation error IAM: ListAttachedRolePolicies, api error Throttling: Rate exceeded\nRetryCount(" + strconv.Itoa(maxRetryCount) + ") over, but failed to delete. "),
+			},
+			wantErr: true,
+		},
 	}
 
 	for _, tt := range cases {
@@ -422,7 +507,7 @@ func TestIam_ListAttachedRolePolicies(t *testing.T) {
 			client := iam.NewFromConfig(cfg)
 			iamClient := NewIam(client)
 
-			output, err := iamClient.ListAttachedRolePolicies(tt.args.ctx, tt.args.roleName)
+			output, err := iamClient.ListAttachedRolePolicies(tt.args.ctx, tt.args.roleName, sleepTimeSecForIam)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("error = %#v, wantErr %#v", err.Error(), tt.wantErr)
 				return
@@ -807,6 +892,31 @@ func TestIam_CheckRoleExists(t *testing.T) {
 			},
 			wantErr: true,
 		},
+		{
+			name: "check role exists failure for api error",
+			args: args{
+				ctx:      context.Background(),
+				roleName: aws.String("test"),
+				withAPIOptionsFunc: func(stack *middleware.Stack) error {
+					return stack.Finalize.Add(
+						middleware.FinalizeMiddlewareFunc(
+							"GetRoleApiErrorMock",
+							func(context.Context, middleware.FinalizeInput, middleware.FinalizeHandler) (middleware.FinalizeOutput, middleware.Metadata, error) {
+								return middleware.FinalizeOutput{
+									Result: &iam.GetRoleOutput{},
+								}, middleware.Metadata{}, fmt.Errorf("api error Throttling: Rate exceeded")
+							},
+						),
+						middleware.Before,
+					)
+				},
+			},
+			want: want{
+				exists: false,
+				err:    fmt.Errorf("RetryCountOverError: test, operation error IAM: GetRole, api error Throttling: Rate exceeded\nRetryCount(" + strconv.Itoa(maxRetryCount) + ") over, but failed to delete. "),
+			},
+			wantErr: true,
+		},
 	}
 
 	for _, tt := range cases {
@@ -823,7 +933,7 @@ func TestIam_CheckRoleExists(t *testing.T) {
 			client := iam.NewFromConfig(cfg)
 			iamClient := NewIam(client)
 
-			output, err := iamClient.CheckRoleExists(tt.args.ctx, tt.args.roleName)
+			output, err := iamClient.CheckRoleExists(tt.args.ctx, tt.args.roleName, sleepTimeSecForIam)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("error = %#v, wantErr %#v", err.Error(), tt.wantErr)
 				return