Repository: argoproj/argo-rollouts

Issue:
--aws-verify-target-group does not work with EKS automode: needs support for eks.amazonaws.com API group
Checklist:

* [✅] I've included steps to reproduce the bug.
* [✅] I've included the version of argo rollouts.

**Describe the bug**

When using AWS Load Balancer Controller in EKS automode (default installation), enabling the Argo Rollouts `--aws-verify-target-group` option results in the following error:

```
level=error msg="rollout syncHandler error: targetgroupbindings.elbv2.k8s.aws is forbidden: cannot list resource \"targetgroupbindings\" in API group \"elbv2.k8s.aws\" in the namespace \"ns-name\"" namespace=ns-name rollout=rollout-name
```

In EKS automode, TargetGroupBinding resources are created with the `eks.amazonaws.com/v1` API group, but Argo Rollouts only tries to access `elbv2.k8s.aws/v1beta1`, causing the feature to not work in this environment.

**To Reproduce**

1. Deploy AWS Load Balancer Controller in EKS using automode (default installation).
2. Confirm that TargetGroupBinding resources are created with `eks.amazonaws.com/v1`.
3. Run Argo Rollouts controller with `--aws-verify-target-group`.
4. Trigger a rollout and observe the error above in the controller logs.

**Expected behavior**

- When running in EKS automode, Argo Rollouts should automatically detect and use the `eks.amazonaws.com/v1` API group for TargetGroupBinding resources, or allow the API group to be set via a parameter.
- Currently, the feature is hardcoded to use only `elbv2.k8s.aws/v1beta1`, so it does not work in automode environments.

**Screenshots**

N/A

**Version**

- argo-rollouts: v1.8.0
- AWS Load Balancer Controller: EKS automode
- EKS: 1.32

**Logs**

```
level=error msg="rollout syncHandler error: targetgroupbindings.elbv2.k8s.aws is forbidden: cannot list resource \"targetgroupbindings\" in API group \"elbv2.k8s.aws\" in the namespace \"ns-name\"" namespace=ns-name rollout=rollout-name
```


Hints from issue comments:

Solution patch:
diff --git a/cmd/rollouts-controller/main.go b/cmd/rollouts-controller/main.go
index f4769e9c..7c65334d 100644
--- a/cmd/rollouts-controller/main.go
+++ b/cmd/rollouts-controller/main.go
@@ -70,6 +70,8 @@ func newCommand() *cobra.Command {
 		serviceThreads                 int
 		ingressThreads                 int
 		ephemeralMetadataThreads       int
+		targetGroupBindingVersion      string
+		albTagKeyResourceID            string
 		istioVersion                   string
 		trafficSplitVersion            string
 		traefikAPIGroup                string
@@ -108,6 +110,8 @@ func newCommand() *cobra.Command {
 			ctx := signals.SetupSignalHandlerContext()
 
 			defaults.SetVerifyTargetGroup(awsVerifyTargetGroup)
+			defaults.SetTargetGroupBindingAPIVersion(targetGroupBindingVersion)
+			defaults.SetalbTagKeyResourceID(albTagKeyResourceID)
 			defaults.SetIstioAPIVersion(istioVersion)
 			defaults.SetAmbassadorAPIVersion(ambassadorVersion)
 			defaults.SetSMIAPIVersion(trafficSplitVersion)
@@ -298,7 +302,7 @@ func newCommand() *cobra.Command {
 	command.Flags().StringVar(&logLevel, "loglevel", "info", "Set the logging level. One of: debug|info|warn|error")
 	command.Flags().StringVar(&logFormat, "logformat", "", "Set the logging format. One of: text|json")
 	command.Flags().IntVar(&klogLevel, "kloglevel", 0, "Set the klog logging level")
-	command.Flags().IntVar(&metricsPort, "metricsport", controller.DefaultMetricsPort, "Set the port the metrics endpoint should be exposed over")
+	command.Flags().IntVar(&metricsPort, "metricsPort", controller.DefaultMetricsPort, "Set the port the metrics endpoint should be exposed over")
 	command.Flags().IntVar(&healthzPort, "healthzPort", controller.DefaultHealthzPort, "Set the port the healthz endpoint should be exposed over")
 	command.Flags().StringVar(&instanceID, "instance-id", "", "Indicates which argo rollout objects the controller should operate on")
 	command.Flags().Float32Var(&qps, "qps", defaults.DefaultQPS, "Maximum QPS (queries per second) to the K8s API server")
@@ -309,6 +313,8 @@ func newCommand() *cobra.Command {
 	command.Flags().IntVar(&serviceThreads, "service-threads", controller.DefaultServiceThreads, "Set the number of worker threads for the Service controller")
 	command.Flags().IntVar(&ingressThreads, "ingress-threads", controller.DefaultIngressThreads, "Set the number of worker threads for the Ingress controller")
 	command.Flags().IntVar(&ephemeralMetadataThreads, "ephemeral-metadata-threads", rollout.DefaultEphemeralMetadataThreads, "Set the number of worker threads for the Ephemeral Metadata reconciler")
+	command.Flags().StringVar(&targetGroupBindingVersion, "aws-target-group-binding-api-version", defaults.DefaultTargetGroupBindingAPIVersion, "Set the default AWS TargetGroupBinding apiVersion that controller uses when verifying target group weights.")
+	command.Flags().StringVar(&albTagKeyResourceID, "alb-tag-key-resource-id", defaults.DefaultAlbTagKeyResourceID, "Set the default AWS LoadBalancer tag key for resource ID that controller uses when verifying target group weights.")
 	command.Flags().StringVar(&istioVersion, "istio-api-version", defaults.DefaultIstioVersion, "Set the default Istio apiVersion that controller should look when manipulating VirtualServices.")
 	command.Flags().StringVar(&ambassadorVersion, "ambassador-api-version", defaults.DefaultAmbassadorVersion, "Set the Ambassador apiVersion that controller should look when manipulating Ambassador Mappings.")
 	command.Flags().StringVar(&trafficSplitVersion, "traffic-split-api-version", defaults.DefaultSMITrafficSplitVersion, "Set the default TrafficSplit apiVersion that controller uses when creating TrafficSplits.")
diff --git a/manifests/install.yaml b/manifests/install.yaml
index f6aacccf..b576d5ce 100755
--- a/manifests/install.yaml
+++ b/manifests/install.yaml
@@ -16671,6 +16671,7 @@ rules:
   - get
 - apiGroups:
   - elbv2.k8s.aws
+  - eks.amazonaws.com
   resources:
   - targetgroupbindings
   verbs:
diff --git a/manifests/namespace-install.yaml b/manifests/namespace-install.yaml
index 16ea68f9..21c0db78 100644
--- a/manifests/namespace-install.yaml
+++ b/manifests/namespace-install.yaml
@@ -192,6 +192,7 @@ rules:
   - get
 - apiGroups:
   - elbv2.k8s.aws
+  - eks.amazonaws.com
   resources:
   - targetgroupbindings
   verbs:
diff --git a/manifests/role/argo-rollouts-clusterrole.yaml b/manifests/role/argo-rollouts-clusterrole.yaml
index b43ed1ad..a3930122 100644
--- a/manifests/role/argo-rollouts-clusterrole.yaml
+++ b/manifests/role/argo-rollouts-clusterrole.yaml
@@ -197,6 +197,7 @@ rules:
   - get
 - apiGroups:
   - elbv2.k8s.aws
+  - eks.amazonaws.com
   resources:
   - targetgroupbindings
   verbs:
diff --git a/pkg/apis/rollouts/validation/validation.go b/pkg/apis/rollouts/validation/validation.go
index d9c31a37..bebbcb3f 100644
--- a/pkg/apis/rollouts/validation/validation.go
+++ b/pkg/apis/rollouts/validation/validation.go
@@ -228,6 +228,11 @@ func ValidateRolloutStrategyBlueGreen(rollout *v1alpha1.Rollout, fldPath *field.
 	return allErrs
 }
 
+func InvalidWorkloadRef(rollout *v1alpha1.Rollout, err error) error {
+	fldPath := field.NewPath("spec")
+	return field.Invalid(fldPath.Child("workloadRef"), rollout.Spec.WorkloadRef, err.Error())
+}
+
 // requireCanaryStableServices returns true if the rollout requires canary.stableService and
 // canary.canaryService to be defined
 func requireCanaryStableServices(rollout *v1alpha1.Rollout) bool {
diff --git a/rollout/controller.go b/rollout/controller.go
index 8a5ec674..aa51a3a2 100644
--- a/rollout/controller.go
+++ b/rollout/controller.go
@@ -419,8 +419,6 @@ func (c *Controller) syncHandler(ctx context.Context, key string) error {
 		logCtx.WithField("time_ms", duration.Seconds()*1e3).Info("Reconciliation completed")
 	}()
 
-	resolveErr := c.refResolver.Resolve(r)
-	// We could maybe lose setting the error condition from the below if resolveErr != nil {}, and just log the error to clean up the logic
 	roCtx, err := c.newRolloutContext(r)
 	if roCtx == nil {
 		logCtx.Error("newRolloutContext returned nil")
@@ -438,15 +436,6 @@ func (c *Controller) syncHandler(ctx context.Context, key string) error {
 		logCtx.Errorf("newRolloutContext err %v", err)
 		return err
 	}
-	// We should probably delete this if block and just log the error to clean up the logic, a bigger change would be to add a new
-	// field to the status maybe (reconcileErrMsg) and store the errors there from the processNextWorkItem function in controller/controller.go
-	if resolveErr != nil {
-		err := roCtx.createInvalidRolloutCondition(resolveErr, r)
-		if err != nil {
-			return fmt.Errorf("failed to create invalid rollout condition during resolving the rollout: %w", err)
-		}
-		return fmt.Errorf("failed to resolve rollout: %w", resolveErr)
-	}
 
 	// In order to work with HPA, the rollout.Spec.Replica field cannot be nil. As a result, the controller will update
 	// the rollout to have the replicas field set to the default value. see https://github.com/argoproj/argo-rollouts/issues/119
@@ -508,6 +497,14 @@ func (c *Controller) writeBackToInformer(ro *v1alpha1.Rollout) {
 }
 
 func (c *Controller) newRolloutContext(rollout *v1alpha1.Rollout) (*rolloutContext, error) {
+	// This needs to be run before replicasets are found/resolved below.
+	// Additionally, for whatever reason, the tests also have assertions that
+	// fail if we delay finding the replicasets, analysisruns and experiments
+	// until after the roll context is created. Thus any eventual error in
+	// resolving the workload ref will be handled immediately after the
+	// rollcontext is created.
+	resolveErr := c.refResolver.Resolve(rollout)
+
 	rsList, err := c.getReplicaSetsForRollouts(rollout)
 	if err != nil {
 		return nil, err
@@ -560,6 +557,15 @@ func (c *Controller) newRolloutContext(rollout *v1alpha1.Rollout) (*rolloutConte
 		reconcilerBase: c.reconcilerBase,
 	}
 
+	if resolveErr != nil {
+		err := roCtx.createInvalidRolloutCondition(resolveErr, rollout)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create invalid rollout condition when resolving the rollout: %w", err)
+		}
+
+		return &roCtx, validation.InvalidWorkloadRef(roCtx.rollout, resolveErr)
+	}
+
 	// Get Rollout Validation errors
 	err = roCtx.getRolloutValidationErrors()
 	if err != nil {
diff --git a/rollout/trafficrouting/alb/alb.go b/rollout/trafficrouting/alb/alb.go
index d766a840..224a5665 100644
--- a/rollout/trafficrouting/alb/alb.go
+++ b/rollout/trafficrouting/alb/alb.go
@@ -286,7 +286,7 @@ func (r *Reconciler) VerifyWeightPerIngress(desiredWeight int32, ingresses []str
 				updateTargetGroupStatus(&r.cfg.Status.ALBs[i], &tg, canaryResourceID, stableResourceID, r.log)
 				updateTargetGroupStatus(r.cfg.Status.ALB, &tg, canaryResourceID, stableResourceID, r.log)
 				if tg.Weight != nil {
-					if tg.Tags[aws.AWSLoadBalancerV2TagKeyResourceID] == canaryResourceID {
+					if tg.Tags[defaults.GetalbTagKeyResourceID()] == canaryResourceID {
 						logCtx := logCtx.WithField("tg", *tg.TargetGroupArn)
 						logCtx.Infof("canary weight of %s (desired: %d, current: %d)", canaryResourceID, desiredWeight, *tg.Weight)
 						verified := *tg.Weight == desiredWeight
@@ -296,9 +296,9 @@ func (r *Reconciler) VerifyWeightPerIngress(desiredWeight int32, ingresses []str
 						} else {
 							r.cfg.Recorder.Warnf(rollout, record.EventOptions{EventReason: conditions.TargetGroupUnverifiedReason}, conditions.TargetGroupUnverifiedWeightsMessage, canaryService, *tg.TargetGroupArn, desiredWeight, *tg.Weight)
 						}
-					} else if dest, ok := resourceIDToDest[tg.Tags[aws.AWSLoadBalancerV2TagKeyResourceID]]; ok {
+					} else if dest, ok := resourceIDToDest[tg.Tags[defaults.GetalbTagKeyResourceID()]]; ok {
 						logCtx := logCtx.WithField("tg", *tg.TargetGroupArn)
-						logCtx.Infof("%s weight of %s (desired: %d, current: %d)", dest.ServiceName, tg.Tags[aws.AWSLoadBalancerV2TagKeyResourceID], dest.Weight, *tg.Weight)
+						logCtx.Infof("%s weight of %s (desired: %d, current: %d)", dest.ServiceName, tg.Tags[defaults.GetalbTagKeyResourceID()], dest.Weight, *tg.Weight)
 						verified := *tg.Weight == dest.Weight
 						if verified {
 							numVerifiedWeights += 1
@@ -326,7 +326,7 @@ func updateLoadBalancerStatus(status *v1alpha1.ALBStatus, lb *elbv2types.LoadBal
 }
 
 func updateTargetGroupStatus(status *v1alpha1.ALBStatus, tg *aws.TargetGroupMeta, canaryResourceID string, stableResourceID string, log *logrus.Entry) {
-	if tg.Tags[aws.AWSLoadBalancerV2TagKeyResourceID] == canaryResourceID {
+	if tg.Tags[defaults.GetalbTagKeyResourceID()] == canaryResourceID {
 		status.CanaryTargetGroup.Name = *tg.TargetGroupName
 		status.CanaryTargetGroup.ARN = *tg.TargetGroupArn
 		if tgArnParts := strings.Split(*tg.TargetGroupArn, "/"); len(tgArnParts) > 1 {
@@ -335,7 +335,7 @@ func updateTargetGroupStatus(status *v1alpha1.ALBStatus, tg *aws.TargetGroupMeta
 			status.CanaryTargetGroup.FullName = ""
 			log.Errorf("error parsing canary target group arn: '%s'", *tg.TargetGroupArn)
 		}
-	} else if tg.Tags[aws.AWSLoadBalancerV2TagKeyResourceID] == stableResourceID {
+	} else if tg.Tags[defaults.GetalbTagKeyResourceID()] == stableResourceID {
 		status.StableTargetGroup.Name = *tg.TargetGroupName
 		status.StableTargetGroup.ARN = *tg.TargetGroupArn
 		if tgArnParts := strings.Split(*tg.TargetGroupArn, "/"); len(tgArnParts) > 1 {
diff --git a/utils/aws/aws.go b/utils/aws/aws.go
index 42685302..353487da 100644
--- a/utils/aws/aws.go
+++ b/utils/aws/aws.go
@@ -21,15 +21,6 @@ import (
 	"github.com/argoproj/argo-rollouts/utils/defaults"
 )
 
-// AWSLoadBalancerV2TagKeyResourceID is the tag applied to an AWS resource by the AWS Load Balancer
-// controller, to associate it to the corresponding kubernetes resource. This is used by the rollout
-// controller to identify the correct TargetGroups associated with the LoadBalancer. For AWS
-// target group service references, the format is: <namespace>/<ingress-name>-<service-name>:<port>
-// Example: ingress.k8s.aws/resource: default/alb-rollout-ingress-alb-rollout-stable:80
-// See: https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.2/guide/ingress/annotations/#resource-tags
-// https://github.com/kubernetes-sigs/aws-load-balancer-controller/blob/da8951f80521651e0a1ffe1361c011d6baad7706/pkg/deploy/tracking/provider.go#L19
-const AWSLoadBalancerV2TagKeyResourceID = "ingress.k8s.aws/resource"
-
 // TargetType is the targetType of your ELBV2 TargetGroup.
 //
 // * with `instance` TargetType, nodes with nodePort for your service will be registered as targets
diff --git a/utils/defaults/defaults.go b/utils/defaults/defaults.go
index 28431a44..7cbd526f 100644
--- a/utils/defaults/defaults.go
+++ b/utils/defaults/defaults.go
@@ -60,6 +60,7 @@ const (
 	DefaultIstioVersion                 = "v1alpha3"
 	DefaultSMITrafficSplitVersion       = "v1alpha1"
 	DefaultTargetGroupBindingAPIVersion = "elbv2.k8s.aws/v1beta1"
+	DefaultAlbTagKeyResourceID          = "ingress.k8s.aws/resource"
 	DefaultAppMeshCRDVersion            = "v1beta2"
 	DefaultTraefikAPIGroup              = "traefik.containo.us"
 	DefaultTraefikVersion               = "traefik.containo.us/v1alpha1"
@@ -75,6 +76,7 @@ var (
 	ambassadorAPIVersion         = DefaultAmbassadorVersion
 	smiAPIVersion                = DefaultSMITrafficSplitVersion
 	targetGroupBindingAPIVersion = DefaultTargetGroupBindingAPIVersion
+	albTagKeyResourceID          = DefaultAlbTagKeyResourceID
 	appmeshCRDVersion            = DefaultAppMeshCRDVersion
 	defaultMetricCleanupDelay    = DefaultMetricCleanupDelay
 	defaultDescribeTagsLimit     = DefaultDescribeTagsLimit
@@ -325,6 +327,14 @@ func GetTraefikAPIGroup() string {
 	return traefikAPIGroup
 }
 
+func SetalbTagKeyResourceID(tagKey string) {
+	albTagKeyResourceID = tagKey
+}
+
+func GetalbTagKeyResourceID() string {
+	return albTagKeyResourceID
+}
+
 func SetTargetGroupBindingAPIVersion(apiVersion string) {
 	targetGroupBindingAPIVersion = apiVersion
 }



Test patch:
diff --git a/rollout/controller_test.go b/rollout/controller_test.go
index ef998ebe..c8f1e7b7 100644
--- a/rollout/controller_test.go
+++ b/rollout/controller_test.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"net/http"
 	"reflect"
 	"strconv"
 	"strings"
@@ -17,6 +18,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	extensionsv1beta1 "k8s.io/api/extensions/v1beta1"
 	"k8s.io/apimachinery/pkg/api/equality"
+	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -73,7 +75,22 @@ const (
 type FakeWorkloadRefResolver struct {
 }
 
-func (f *FakeWorkloadRefResolver) Resolve(_ *v1alpha1.Rollout) error {
+func (f *FakeWorkloadRefResolver) Resolve(r *v1alpha1.Rollout) error {
+	if r.Spec.WorkloadRef == nil {
+		return nil
+	}
+
+	if r.Spec.WorkloadRef.Kind == "Error" {
+		return &errors.StatusError{
+			ErrStatus: metav1.Status{
+				Status:  metav1.StatusFailure,
+				Code:    http.StatusNotFound,
+				Reason:  metav1.StatusReasonNotFound,
+				Message: "not found",
+			},
+		}
+	}
+
 	return nil
 }
 
@@ -1400,6 +1417,40 @@ func TestSwitchInvalidSpecMessage(t *testing.T) {
 	assert.JSONEq(t, calculatePatch(r, expectedPatch), patch)
 }
 
+func TestInvalidWorkloadRef(t *testing.T) {
+	f := newFixture(t)
+	defer f.Close()
+
+	r := newCanaryRollout("foo", 1, nil, nil, nil, intstr.FromInt(0), intstr.FromInt(1))
+	r.Spec.Selector = nil
+	r.Spec.Template = corev1.PodTemplateSpec{}
+	r.Spec.WorkloadRef = &v1alpha1.ObjectRef{
+		Kind: "Error",
+	}
+
+	f.rolloutLister = append(f.rolloutLister, r)
+	f.objects = append(f.objects, r)
+
+	patchIndex := f.expectPatchRolloutAction(r)
+	f.run(getKey(r, t))
+
+	expectedPatchWithoutSub := `{
+		"status": {
+			"conditions": [%s,%s],
+			"message": "%s: %s",
+			"phase": "Degraded"
+		}
+	}`
+	errmsg := "The Rollout \"foo\" is invalid: not found"
+	_, progressingCond := newProgressingCondition(conditions.ReplicaSetUpdatedReason, r, "")
+	invalidSpecCond := conditions.NewRolloutCondition(v1alpha1.InvalidSpec, corev1.ConditionTrue, conditions.InvalidSpecReason, errmsg)
+	invalidSpecBytes, _ := json.Marshal(invalidSpecCond)
+	expectedPatch := fmt.Sprintf(expectedPatchWithoutSub, progressingCond, string(invalidSpecBytes), conditions.InvalidSpecReason, strings.ReplaceAll(errmsg, "\"", "\\\""))
+
+	patch := f.getPatchedRollout(patchIndex)
+	assert.JSONEq(t, calculatePatch(r, expectedPatch), patch)
+}
+
 // TestPodTemplateHashEquivalence verifies the hash is computed consistently when there are slight
 // variations made to the pod template in equivalent ways.
 func TestPodTemplateHashEquivalence(t *testing.T) {
diff --git a/rollout/trafficrouting/alb/alb_test.go b/rollout/trafficrouting/alb/alb_test.go
index 04d0cb68..e029c4c3 100644
--- a/rollout/trafficrouting/alb/alb_test.go
+++ b/rollout/trafficrouting/alb/alb_test.go
@@ -33,6 +33,7 @@ const STABLE_SVC = "stable-svc"
 const CANARY_SVC = "canary-svc"
 const PING_SVC = "ping-service"
 const PONG_SVC = "pong-service"
+const ALB_TAG_KEY_RESOURCE_ID = "ingress.k8s.aws/resource"
 
 func fakeRollout(stableSvc, canarySvc string, pingPong *v1alpha1.PingPongSpec, stableIng string, port int32) *v1alpha1.Rollout {
 	return &v1alpha1.Rollout{
@@ -893,7 +894,7 @@ func TestVerifyWeight(t *testing.T) {
 				},
 				Weight: ptr.To[int32](11),
 				Tags: map[string]string{
-					aws.AWSLoadBalancerV2TagKeyResourceID: "default/ingress-canary-svc:443",
+					ALB_TAG_KEY_RESOURCE_ID: "default/ingress-canary-svc:443",
 				},
 			},
 			{
@@ -903,7 +904,7 @@ func TestVerifyWeight(t *testing.T) {
 				},
 				Weight: ptr.To[int32](89),
 				Tags: map[string]string{
-					aws.AWSLoadBalancerV2TagKeyResourceID: "default/ingress-stable-svc:443",
+					ALB_TAG_KEY_RESOURCE_ID: "default/ingress-stable-svc:443",
 				},
 			},
 		}
@@ -935,7 +936,7 @@ func TestVerifyWeight(t *testing.T) {
 				},
 				Weight: ptr.To[int32](10),
 				Tags: map[string]string{
-					aws.AWSLoadBalancerV2TagKeyResourceID: "default/ingress-canary-svc:443",
+					ALB_TAG_KEY_RESOURCE_ID: "default/ingress-canary-svc:443",
 				},
 			},
 			{
@@ -945,7 +946,7 @@ func TestVerifyWeight(t *testing.T) {
 				},
 				Weight: ptr.To[int32](11),
 				Tags: map[string]string{
-					aws.AWSLoadBalancerV2TagKeyResourceID: "default/ingress-stable-svc:443",
+					ALB_TAG_KEY_RESOURCE_ID: "default/ingress-stable-svc:443",
 				},
 			},
 		}
@@ -977,7 +978,7 @@ func TestVerifyWeight(t *testing.T) {
 				},
 				Weight: ptr.To[int32](100),
 				Tags: map[string]string{
-					aws.AWSLoadBalancerV2TagKeyResourceID: "default/ingress-canary-svc:443",
+					ALB_TAG_KEY_RESOURCE_ID: "default/ingress-canary-svc:443",
 				},
 			},
 			{
@@ -987,7 +988,7 @@ func TestVerifyWeight(t *testing.T) {
 				},
 				Weight: ptr.To[int32](0),
 				Tags: map[string]string{
-					aws.AWSLoadBalancerV2TagKeyResourceID: "default/ingress-stable-svc:443",
+					ALB_TAG_KEY_RESOURCE_ID: "default/ingress-stable-svc:443",
 				},
 			},
 		}
@@ -1018,7 +1019,7 @@ func TestVerifyWeight(t *testing.T) {
 				},
 				Weight: ptr.To[int32](10),
 				Tags: map[string]string{
-					aws.AWSLoadBalancerV2TagKeyResourceID: "default/ingress-canary-svc:443",
+					ALB_TAG_KEY_RESOURCE_ID: "default/ingress-canary-svc:443",
 				},
 			},
 			{
@@ -1028,7 +1029,7 @@ func TestVerifyWeight(t *testing.T) {
 				},
 				Weight: ptr.To[int32](11),
 				Tags: map[string]string{
-					aws.AWSLoadBalancerV2TagKeyResourceID: "default/ingress-stable-svc:443",
+					ALB_TAG_KEY_RESOURCE_ID: "default/ingress-stable-svc:443",
 				},
 			},
 		}
@@ -1161,7 +1162,7 @@ func TestVerifyWeightMultiIngress(t *testing.T) {
 				},
 				Weight: ptr.To[int32](11),
 				Tags: map[string]string{
-					aws.AWSLoadBalancerV2TagKeyResourceID: "default/ingress-canary-svc:443",
+					ALB_TAG_KEY_RESOURCE_ID: "default/ingress-canary-svc:443",
 				},
 			},
 			{
@@ -1171,7 +1172,7 @@ func TestVerifyWeightMultiIngress(t *testing.T) {
 				},
 				Weight: ptr.To[int32](89),
 				Tags: map[string]string{
-					aws.AWSLoadBalancerV2TagKeyResourceID: "default/ingress-stable-svc:443",
+					ALB_TAG_KEY_RESOURCE_ID: "default/ingress-stable-svc:443",
 				},
 			},
 			{
@@ -1181,7 +1182,7 @@ func TestVerifyWeightMultiIngress(t *testing.T) {
 				},
 				Weight: ptr.To[int32](11),
 				Tags: map[string]string{
-					aws.AWSLoadBalancerV2TagKeyResourceID: "default/multi-ingress-canary-svc:443",
+					ALB_TAG_KEY_RESOURCE_ID: "default/multi-ingress-canary-svc:443",
 				},
 			},
 			{
@@ -1191,7 +1192,7 @@ func TestVerifyWeightMultiIngress(t *testing.T) {
 				},
 				Weight: ptr.To[int32](89),
 				Tags: map[string]string{
-					aws.AWSLoadBalancerV2TagKeyResourceID: "default/multi-ingress-stable-svc:443",
+					ALB_TAG_KEY_RESOURCE_ID: "default/multi-ingress-stable-svc:443",
 				},
 			},
 		}
@@ -1227,7 +1228,7 @@ func TestVerifyWeightMultiIngress(t *testing.T) {
 				},
 				Weight: ptr.To[int32](10),
 				Tags: map[string]string{
-					aws.AWSLoadBalancerV2TagKeyResourceID: "default/ingress-canary-svc:443",
+					ALB_TAG_KEY_RESOURCE_ID: "default/ingress-canary-svc:443",
 				},
 			},
 			{
@@ -1237,7 +1238,7 @@ func TestVerifyWeightMultiIngress(t *testing.T) {
 				},
 				Weight: ptr.To[int32](90),
 				Tags: map[string]string{
-					aws.AWSLoadBalancerV2TagKeyResourceID: "default/ingress-stable-svc:443",
+					ALB_TAG_KEY_RESOURCE_ID: "default/ingress-stable-svc:443",
 				},
 			},
 			{
@@ -1247,7 +1248,7 @@ func TestVerifyWeightMultiIngress(t *testing.T) {
 				},
 				Weight: ptr.To[int32](10),
 				Tags: map[string]string{
-					aws.AWSLoadBalancerV2TagKeyResourceID: "default/multi-ingress-canary-svc:443",
+					ALB_TAG_KEY_RESOURCE_ID: "default/multi-ingress-canary-svc:443",
 				},
 			},
 			{
@@ -1257,7 +1258,7 @@ func TestVerifyWeightMultiIngress(t *testing.T) {
 				},
 				Weight: ptr.To[int32](90),
 				Tags: map[string]string{
-					aws.AWSLoadBalancerV2TagKeyResourceID: "default/multi-ingress-stable-svc:443",
+					ALB_TAG_KEY_RESOURCE_ID: "default/multi-ingress-stable-svc:443",
 				},
 			},
 		}
@@ -1437,7 +1438,7 @@ func TestVerifyWeightWithAdditionalDestinations(t *testing.T) {
 				},
 				Weight: ptr.To[int32](10),
 				Tags: map[string]string{
-					aws.AWSLoadBalancerV2TagKeyResourceID: "default/ingress-canary-svc:443",
+					ALB_TAG_KEY_RESOURCE_ID: "default/ingress-canary-svc:443",
 				},
 			},
 			{
@@ -1447,7 +1448,7 @@ func TestVerifyWeightWithAdditionalDestinations(t *testing.T) {
 				},
 				Weight: ptr.To[int32](90),
 				Tags: map[string]string{
-					aws.AWSLoadBalancerV2TagKeyResourceID: "default/ingress-stable-svc:443",
+					ALB_TAG_KEY_RESOURCE_ID: "default/ingress-stable-svc:443",
 				},
 			},
 		}
@@ -1477,7 +1478,7 @@ func TestVerifyWeightWithAdditionalDestinations(t *testing.T) {
 				},
 				Weight: ptr.To[int32](10),
 				Tags: map[string]string{
-					aws.AWSLoadBalancerV2TagKeyResourceID: "default/ingress-canary-svc:443",
+					ALB_TAG_KEY_RESOURCE_ID: "default/ingress-canary-svc:443",
 				},
 			},
 			{
@@ -1487,7 +1488,7 @@ func TestVerifyWeightWithAdditionalDestinations(t *testing.T) {
 				},
 				Weight: ptr.To[int32](100),
 				Tags: map[string]string{
-					aws.AWSLoadBalancerV2TagKeyResourceID: "default/ingress-ex-svc-1:443",
+					ALB_TAG_KEY_RESOURCE_ID: "default/ingress-ex-svc-1:443",
 				},
 			},
 			{
@@ -1497,7 +1498,7 @@ func TestVerifyWeightWithAdditionalDestinations(t *testing.T) {
 				},
 				Weight: ptr.To[int32](100),
 				Tags: map[string]string{
-					aws.AWSLoadBalancerV2TagKeyResourceID: "default/ingress-ex-svc-2:443",
+					ALB_TAG_KEY_RESOURCE_ID: "default/ingress-ex-svc-2:443",
 				},
 			},
 			{
@@ -1507,7 +1508,7 @@ func TestVerifyWeightWithAdditionalDestinations(t *testing.T) {
 				},
 				Weight: ptr.To[int32](85),
 				Tags: map[string]string{
-					aws.AWSLoadBalancerV2TagKeyResourceID: "default/ingress-stable-svc:443",
+					ALB_TAG_KEY_RESOURCE_ID: "default/ingress-stable-svc:443",
 				},
 			},
 		}
@@ -1537,7 +1538,7 @@ func TestVerifyWeightWithAdditionalDestinations(t *testing.T) {
 				},
 				Weight: ptr.To[int32](10),
 				Tags: map[string]string{
-					aws.AWSLoadBalancerV2TagKeyResourceID: "default/ingress-canary-svc:443",
+					ALB_TAG_KEY_RESOURCE_ID: "default/ingress-canary-svc:443",
 				},
 			},
 			{
@@ -1547,7 +1548,7 @@ func TestVerifyWeightWithAdditionalDestinations(t *testing.T) {
 				},
 				Weight: &weightDestinations[0].Weight,
 				Tags: map[string]string{
-					aws.AWSLoadBalancerV2TagKeyResourceID: "default/ingress-ex-svc-1:443",
+					ALB_TAG_KEY_RESOURCE_ID: "default/ingress-ex-svc-1:443",
 				},
 			},
 			{
@@ -1557,7 +1558,7 @@ func TestVerifyWeightWithAdditionalDestinations(t *testing.T) {
 				},
 				Weight: &weightDestinations[1].Weight,
 				Tags: map[string]string{
-					aws.AWSLoadBalancerV2TagKeyResourceID: "default/ingress-ex-svc-2:443",
+					ALB_TAG_KEY_RESOURCE_ID: "default/ingress-ex-svc-2:443",
 				},
 			},
 			{
@@ -1567,7 +1568,7 @@ func TestVerifyWeightWithAdditionalDestinations(t *testing.T) {
 				},
 				Weight: ptr.To[int32](85),
 				Tags: map[string]string{
-					aws.AWSLoadBalancerV2TagKeyResourceID: "default/ingress-stable-svc:443",
+					ALB_TAG_KEY_RESOURCE_ID: "default/ingress-stable-svc:443",
 				},
 			},
 		}
@@ -1665,7 +1666,7 @@ func TestVerifyWeightWithAdditionalDestinationsMultiIngress(t *testing.T) {
 				},
 				Weight: ptr.To[int32](10),
 				Tags: map[string]string{
-					aws.AWSLoadBalancerV2TagKeyResourceID: "default/ingress-canary-svc:443",
+					ALB_TAG_KEY_RESOURCE_ID: "default/ingress-canary-svc:443",
 				},
 			},
 			{
@@ -1675,7 +1676,7 @@ func TestVerifyWeightWithAdditionalDestinationsMultiIngress(t *testing.T) {
 				},
 				Weight: ptr.To[int32](90),
 				Tags: map[string]string{
-					aws.AWSLoadBalancerV2TagKeyResourceID: "default/ingress-stable-svc:443",
+					ALB_TAG_KEY_RESOURCE_ID: "default/ingress-stable-svc:443",
 				},
 			},
 			{
@@ -1685,7 +1686,7 @@ func TestVerifyWeightWithAdditionalDestinationsMultiIngress(t *testing.T) {
 				},
 				Weight: ptr.To[int32](10),
 				Tags: map[string]string{
-					aws.AWSLoadBalancerV2TagKeyResourceID: "default/multi-ingress-canary-svc:443",
+					ALB_TAG_KEY_RESOURCE_ID: "default/multi-ingress-canary-svc:443",
 				},
 			},
 			{
@@ -1695,7 +1696,7 @@ func TestVerifyWeightWithAdditionalDestinationsMultiIngress(t *testing.T) {
 				},
 				Weight: ptr.To[int32](90),
 				Tags: map[string]string{
-					aws.AWSLoadBalancerV2TagKeyResourceID: "default/multi-ingress-stable-svc:443",
+					ALB_TAG_KEY_RESOURCE_ID: "default/multi-ingress-stable-svc:443",
 				},
 			},
 		}
@@ -1730,7 +1731,7 @@ func TestVerifyWeightWithAdditionalDestinationsMultiIngress(t *testing.T) {
 				},
 				Weight: ptr.To[int32](10),
 				Tags: map[string]string{
-					aws.AWSLoadBalancerV2TagKeyResourceID: "default/ingress-canary-svc:443",
+					ALB_TAG_KEY_RESOURCE_ID: "default/ingress-canary-svc:443",
 				},
 			},
 			{
@@ -1740,7 +1741,7 @@ func TestVerifyWeightWithAdditionalDestinationsMultiIngress(t *testing.T) {
 				},
 				Weight: ptr.To[int32](85),
 				Tags: map[string]string{
-					aws.AWSLoadBalancerV2TagKeyResourceID: "default/ingress-stable-svc:443",
+					ALB_TAG_KEY_RESOURCE_ID: "default/ingress-stable-svc:443",
 				},
 			},
 			{
@@ -1750,7 +1751,7 @@ func TestVerifyWeightWithAdditionalDestinationsMultiIngress(t *testing.T) {
 				},
 				Weight: ptr.To[int32](10),
 				Tags: map[string]string{
-					aws.AWSLoadBalancerV2TagKeyResourceID: "default/multi-ingress-canary-svc:443",
+					ALB_TAG_KEY_RESOURCE_ID: "default/multi-ingress-canary-svc:443",
 				},
 			},
 			{
@@ -1760,7 +1761,7 @@ func TestVerifyWeightWithAdditionalDestinationsMultiIngress(t *testing.T) {
 				},
 				Weight: ptr.To[int32](85),
 				Tags: map[string]string{
-					aws.AWSLoadBalancerV2TagKeyResourceID: "default/multi-ingress-stable-svc:443",
+					ALB_TAG_KEY_RESOURCE_ID: "default/multi-ingress-stable-svc:443",
 				},
 			},
 			{
@@ -1770,7 +1771,7 @@ func TestVerifyWeightWithAdditionalDestinationsMultiIngress(t *testing.T) {
 				},
 				Weight: ptr.To[int32](100),
 				Tags: map[string]string{
-					aws.AWSLoadBalancerV2TagKeyResourceID: "default/ingress-ex-svc-1:443",
+					ALB_TAG_KEY_RESOURCE_ID: "default/ingress-ex-svc-1:443",
 				},
 			},
 			{
@@ -1780,7 +1781,7 @@ func TestVerifyWeightWithAdditionalDestinationsMultiIngress(t *testing.T) {
 				},
 				Weight: ptr.To[int32](100),
 				Tags: map[string]string{
-					aws.AWSLoadBalancerV2TagKeyResourceID: "default/ingress-ex-svc-2:443",
+					ALB_TAG_KEY_RESOURCE_ID: "default/ingress-ex-svc-2:443",
 				},
 			},
 		}
@@ -1816,7 +1817,7 @@ func TestVerifyWeightWithAdditionalDestinationsMultiIngress(t *testing.T) {
 				},
 				Weight: ptr.To[int32](10),
 				Tags: map[string]string{
-					aws.AWSLoadBalancerV2TagKeyResourceID: "default/ingress-canary-svc:443",
+					ALB_TAG_KEY_RESOURCE_ID: "default/ingress-canary-svc:443",
 				},
 			},
 			{
@@ -1826,7 +1827,7 @@ func TestVerifyWeightWithAdditionalDestinationsMultiIngress(t *testing.T) {
 				},
 				Weight: ptr.To[int32](85),
 				Tags: map[string]string{
-					aws.AWSLoadBalancerV2TagKeyResourceID: "default/ingress-stable-svc:443",
+					ALB_TAG_KEY_RESOURCE_ID: "default/ingress-stable-svc:443",
 				},
 			},
 			{
@@ -1836,7 +1837,7 @@ func TestVerifyWeightWithAdditionalDestinationsMultiIngress(t *testing.T) {
 				},
 				Weight: ptr.To[int32](10),
 				Tags: map[string]string{
-					aws.AWSLoadBalancerV2TagKeyResourceID: "default/multi-ingress-canary-svc:443",
+					ALB_TAG_KEY_RESOURCE_ID: "default/multi-ingress-canary-svc:443",
 				},
 			},
 			{
@@ -1846,7 +1847,7 @@ func TestVerifyWeightWithAdditionalDestinationsMultiIngress(t *testing.T) {
 				},
 				Weight: ptr.To[int32](85),
 				Tags: map[string]string{
-					aws.AWSLoadBalancerV2TagKeyResourceID: "default/multi-ingress-stable-svc:443",
+					ALB_TAG_KEY_RESOURCE_ID: "default/multi-ingress-stable-svc:443",
 				},
 			},
 			{
@@ -1856,7 +1857,7 @@ func TestVerifyWeightWithAdditionalDestinationsMultiIngress(t *testing.T) {
 				},
 				Weight: &weightDestinations[0].Weight,
 				Tags: map[string]string{
-					aws.AWSLoadBalancerV2TagKeyResourceID: "default/ingress-ex-svc-1:443",
+					ALB_TAG_KEY_RESOURCE_ID: "default/ingress-ex-svc-1:443",
 				},
 			},
 			{
@@ -1866,7 +1867,7 @@ func TestVerifyWeightWithAdditionalDestinationsMultiIngress(t *testing.T) {
 				},
 				Weight: &weightDestinations[1].Weight,
 				Tags: map[string]string{
-					aws.AWSLoadBalancerV2TagKeyResourceID: "default/ingress-ex-svc-2:443",
+					ALB_TAG_KEY_RESOURCE_ID: "default/ingress-ex-svc-2:443",
 				},
 			},
 		}
diff --git a/utils/aws/aws_test.go b/utils/aws/aws_test.go
index 90ee2492..a3b69e18 100644
--- a/utils/aws/aws_test.go
+++ b/utils/aws/aws_test.go
@@ -26,6 +26,16 @@ func newFakeClient() (*mocks.ELBv2APIClient, Client) {
 	return &fakeELB, awsClient
 }
 
+func TestGetTargetGroupBindingsGVR(t *testing.T) {
+	defaults.SetTargetGroupBindingAPIVersion("eks.amazonaws.com/v1")
+	gvr, err := GetTargetGroupBindingsGVR()
+	assert.NoError(t, err)
+	assert.Equal(t, "eks.amazonaws.com", gvr.Group)
+	assert.Equal(t, "v1", gvr.Version)
+	assert.Equal(t, "targetgroupbindings", gvr.Resource)
+	defaults.SetTargetGroupBindingAPIVersion("elbv2.k8s.aws/v1beta1")
+}
+
 func TestFindLoadBalancerByDNSName(t *testing.T) {
 	// LoadBalancer not found
 	{
diff --git a/utils/defaults/defaults_test.go b/utils/defaults/defaults_test.go
index f98b92f5..7d9f707a 100644
--- a/utils/defaults/defaults_test.go
+++ b/utils/defaults/defaults_test.go
@@ -412,6 +412,11 @@ func TestSetDefaults(t *testing.T) {
 	SetTargetGroupBindingAPIVersion(DefaultTargetGroupBindingAPIVersion)
 	assert.Equal(t, DefaultTargetGroupBindingAPIVersion, GetTargetGroupBindingAPIVersion())
 
+	SetalbTagKeyResourceID("ingress.amazonaws.com/resource")
+	assert.Equal(t, "ingress.amazonaws.com/resource", GetalbTagKeyResourceID())
+	SetalbTagKeyResourceID(DefaultAlbTagKeyResourceID)
+	assert.Equal(t, DefaultAlbTagKeyResourceID, GetalbTagKeyResourceID())
+
 	assert.Equal(t, DefaultAppMeshCRDVersion, GetAppMeshCRDVersion())
 	SetAppMeshCRDVersion("v1beta3")
 	assert.Equal(t, "v1beta3", GetAppMeshCRDVersion())