Repository: aws/aws-application-networking-k8s (language: go)

Issue:
Feature request for controller to acknowledge service networks and services managed by other accounts
Is there a recommended configuration for the k8s controller when lattice services are expected to be shared across accounts?

In an example scenario:
I have account A and account B

- account A uses kubernetes, account B uses ec2 instances. Each account has its own lattice service network already configured.
- account A uses the k8s controller to manage lattice services, account B uses terraform/manual configuration
- account A wants account B to be able to call its lattice service A
- account A shares lattice service A with account B
- account B associates service A with service network B -- but this here it seems the k8s controller does not like this. https://github.com/aws/aws-application-networking-k8s/blob/3927a6ea027d3f84ed363a64e4e43a6da23b806f/pkg/deploy/lattice/service_manager.go#L271-L283 per this code, the controller will delete the service<>service network association since it's not captured in the HTTPRoute of service A

I know there is the annotation that makes kubernetes ignore vpc associations to a gateway[0]. However even with this, the gateway/service network is still created in account A. In this scenario, account B wants to manage its own infrastructure as much as possible. With the current limitations it seems that all lattice resources must be created in 1 account and then shared out to other accounts.

Some possible solutions I can think of

1. consider foreign resources a special case such as having an option to force the controller to not delete a service<>service network association if the account ids of these resources are different
2. similar to the previous idea, allow a reference to a non-existent gateway which refers to a remote service network (on another account)
3. allow k8s controller to manage resources for multiple accounts. (I like this option less since it gives less independence to other accounts to manage their own AWS resources)

I am definitely open to other ideas I am not considering and would be happy to add more context
[0] https://github.com/aws/aws-application-networking-k8s/blob/main/docs/multi-sn.md

Note: after discussion with @liwenwu-amazon on slack is sounds like prefer solution 1

[In her words](https://kubernetes.slack.com/archives/C052EBZPAUR/p1691549259484649): "lattice controller not to disassociate lattice service from a lattice service network , if that lattice service network owned by a different account"



Discussion / hints:
To add to this: we also want to support the other direction, where account B shares a service to be associated with account A's service network.
Adding to this. If Account A is running eks on a shared vpc, then the annotation `application-networking.k8s.aws/lattice-vpc-association` will not work correctly as you cannot associate a shared vpc with a service network. It throws the following error `AccessDeniedException: Vpc ownership validation failed`. 

The solution to this is to allow services shared to account b to be associated with the service network in account b.
One possibly way to implement solution1:

- Grab serviceNetwork owner accountId by ServiceNetworkServiceAssociation’s serviceNetworkArn from [ListServiceNetworkServiceAssociations response at line 278](https://github.com/aws/aws-application-networking-k8s/blob/3927a6ea027d3f84ed363a64e4e43a6da23b806f/pkg/deploy/lattice/service_manager.go#L275),

- [At line 288](https://github.com/aws/aws-application-networking-k8s/blob/3927a6ea027d3f84ed363a64e4e43a6da23b806f/pkg/deploy/lattice/service_manager.go#L288), do an extra condition check, if serviceNetworkOwnerAccount != currentControllerUsingAccount (or just serviceNetworkOwnerAccount != serviceOwnerAccount? ), `serviceNetworkAssociationMgr` just  ignores to delete that ServiceNetworkServiceAssociation

Fix patch:
diff --git a/cmd/aws-application-networking-k8s/main.go b/cmd/aws-application-networking-k8s/main.go
index e74f3080..6f110805 100644
--- a/cmd/aws-application-networking-k8s/main.go
+++ b/cmd/aws-application-networking-k8s/main.go
@@ -109,11 +109,14 @@ func main() {
 		"AccoundId", config.AccountID,
 		"DefaultServiceNetwork", config.DefaultServiceNetwork,
 		"UseLongTgName", config.UseLongTGName,
+		"ClusterName", config.ClusterName,
 	)
 
 	cloud, err := aws.NewCloud(log.Named("cloud"), aws.CloudConfig{
-		VpcId:     config.VpcID,
-		AccountId: config.AccountID,
+		VpcId:       config.VpcID,
+		AccountId:   config.AccountID,
+		Region:      config.Region,
+		ClusterName: config.ClusterName,
 	})
 	if err != nil {
 		setupLog.Fatal("cloud client setup failed: %s", err)
diff --git a/pkg/aws/cloud.go b/pkg/aws/cloud.go
index 416196f0..39785079 100644
--- a/pkg/aws/cloud.go
+++ b/pkg/aws/cloud.go
@@ -1,26 +1,43 @@
 package aws
 
 import (
+	"fmt"
+
 	"github.com/aws/aws-application-networking-k8s/pkg/aws/services"
-	"github.com/aws/aws-application-networking-k8s/pkg/config"
 	"github.com/aws/aws-application-networking-k8s/pkg/utils/gwlog"
-	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/request"
 	"github.com/aws/aws-sdk-go/aws/session"
-	"github.com/aws/aws-sdk-go/service/eks"
+	"github.com/aws/aws-sdk-go/service/vpclattice"
+)
+
+const (
+	TagBase      = "application-networking.k8s.aws/"
+	TagManagedBy = TagBase + "ManagedBy"
 )
 
+type Tags = map[string]*string
+
 //go:generate mockgen -destination cloud_mocks.go -package aws github.com/aws/aws-application-networking-k8s/pkg/aws Cloud
 
 type CloudConfig struct {
-	VpcId     string
-	AccountId string
+	VpcId       string
+	AccountId   string
+	Region      string
+	ClusterName string
 }
 
 type Cloud interface {
 	Config() CloudConfig
 	Lattice() services.Lattice
-	EKS() services.EKS
+
+	// creates lattice tags with default values populated
+	DefaultTags() Tags
+
+	// check if managedBy tag set for lattice resource
+	IsArnManaged(arn string) (bool, error)
+
+	// check if tags map has managedBy tag
+	ContainsManagedBy(tags Tags) bool
 }
 
 // NewCloud constructs new Cloud implementation.
@@ -47,44 +64,58 @@ func NewCloud(log gwlog.Logger, cfg CloudConfig) (Cloud, error) {
 		}
 	})
 
-	lattice := services.NewDefaultLattice(sess, config.Region)
-	eks := services.NewDefaultEKS(sess, config.Region)
-	cl := NewDefaultCloud(lattice, eks, cfg)
+	lattice := services.NewDefaultLattice(sess, cfg.Region)
+	cl := NewDefaultCloud(lattice, cfg)
 	return cl, nil
 }
 
 // Used in testing and mocks
-func NewDefaultCloud(lattice services.Lattice, eks services.EKS, cfg CloudConfig) Cloud {
-	return &defaultCloud{cfg, lattice, eks}
+func NewDefaultCloud(lattice services.Lattice, cfg CloudConfig) Cloud {
+	return &defaultCloud{
+		cfg:          cfg,
+		lattice:      lattice,
+		managedByTag: getManagedByTag(cfg),
+	}
 }
 
 type defaultCloud struct {
-	cfg     CloudConfig
-	lattice services.Lattice
-	eks     services.EKS
+	cfg          CloudConfig
+	lattice      services.Lattice
+	managedByTag string
 }
 
 func (c *defaultCloud) Lattice() services.Lattice {
 	return c.lattice
 }
 
-func (c *defaultCloud) EKS() services.EKS {
-	return c.eks
-}
-
 func (c *defaultCloud) Config() CloudConfig {
 	return c.cfg
 }
 
-func (d *defaultCloud) GetEKSClusterVPC(name string) string {
-	input := &eks.DescribeClusterInput{
-		Name: aws.String(name),
-	}
+func (c *defaultCloud) DefaultTags() Tags {
+	tags := Tags{}
+	tags[TagManagedBy] = &c.managedByTag
+	return tags
+}
 
-	result, err := d.eks.DescribeCluster(input)
+func (c *defaultCloud) ContainsManagedBy(tags Tags) bool {
+	tag, ok := tags[TagManagedBy]
+	if !ok || tag == nil {
+		return false
+	}
+	return *tag == c.managedByTag
+}
 
+func (c *defaultCloud) IsArnManaged(arn string) (bool, error) {
+	tagsReq := &vpclattice.ListTagsForResourceInput{ResourceArn: &arn}
+	resp, err := c.lattice.ListTagsForResource(tagsReq)
 	if err != nil {
-		return ""
+		return false, err
 	}
-	return result.String()
+	isManaged := c.ContainsManagedBy(resp.Tags)
+	return isManaged, nil
+}
+
+func getManagedByTag(cfg CloudConfig) string {
+	return fmt.Sprintf("%s/%s/%s", cfg.AccountId, cfg.ClusterName, cfg.VpcId)
 }
diff --git a/pkg/aws/cloud_mocks.go b/pkg/aws/cloud_mocks.go
index 10ec2248..7e8fd8c5 100644
--- a/pkg/aws/cloud_mocks.go
+++ b/pkg/aws/cloud_mocks.go
@@ -48,18 +48,47 @@ func (mr *MockCloudMockRecorder) Config() *gomock.Call {
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Config", reflect.TypeOf((*MockCloud)(nil).Config))
 }
 
-// EKS mocks base method.
-func (m *MockCloud) EKS() services.EKS {
+// ContainsManagedBy mocks base method.
+func (m *MockCloud) ContainsManagedBy(arg0 map[string]*string) bool {
 	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "EKS")
-	ret0, _ := ret[0].(services.EKS)
+	ret := m.ctrl.Call(m, "ContainsManagedBy", arg0)
+	ret0, _ := ret[0].(bool)
 	return ret0
 }
 
-// EKS indicates an expected call of EKS.
-func (mr *MockCloudMockRecorder) EKS() *gomock.Call {
+// ContainsManagedBy indicates an expected call of ContainsManagedBy.
+func (mr *MockCloudMockRecorder) ContainsManagedBy(arg0 interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EKS", reflect.TypeOf((*MockCloud)(nil).EKS))
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ContainsManagedBy", reflect.TypeOf((*MockCloud)(nil).ContainsManagedBy), arg0)
+}
+
+// DefaultTags mocks base method.
+func (m *MockCloud) DefaultTags() map[string]*string {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "DefaultTags")
+	ret0, _ := ret[0].(map[string]*string)
+	return ret0
+}
+
+// DefaultTags indicates an expected call of DefaultTags.
+func (mr *MockCloudMockRecorder) DefaultTags() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DefaultTags", reflect.TypeOf((*MockCloud)(nil).DefaultTags))
+}
+
+// IsArnManaged mocks base method.
+func (m *MockCloud) IsArnManaged(arg0 string) (bool, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "IsArnManaged", arg0)
+	ret0, _ := ret[0].(bool)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// IsArnManaged indicates an expected call of IsArnManaged.
+func (mr *MockCloudMockRecorder) IsArnManaged(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsArnManaged", reflect.TypeOf((*MockCloud)(nil).IsArnManaged), arg0)
 }
 
 // Lattice mocks base method.
diff --git a/pkg/config/controller_config.go b/pkg/config/controller_config.go
index 35fc1dcb..c55e9f0a 100644
--- a/pkg/config/controller_config.go
+++ b/pkg/config/controller_config.go
@@ -5,7 +5,10 @@ import (
 	"fmt"
 	"os"
 
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/ec2metadata"
 	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/ec2"
 )
 
 const (
@@ -18,6 +21,7 @@ const (
 	NO_DEFAULT_SERVICE_NETWORK      = "NO_DEFAULT_SERVICE_NETWORK"
 	REGION                          = "REGION"
 	CLUSTER_VPC_ID                  = "CLUSTER_VPC_ID"
+	CLUSTER_NAME                    = "CLUSTER_NAME"
 	CLUSTER_LOCAL_GATEWAY           = "CLUSTER_LOCAL_GATEWAY"
 	AWS_ACCOUNT_ID                  = "AWS_ACCOUNT_ID"
 	TARGET_GROUP_NAME_LEN_MODE      = "TARGET_GROUP_NAME_LEN_MODE"
@@ -30,6 +34,7 @@ var Region = ""
 var logLevel = defaultLogLevel
 var DefaultServiceNetwork = ""
 var UseLongTGName = false
+var ClusterName = ""
 
 func GetClusterLocalGateway() (string, error) {
 	if DefaultServiceNetwork == UnknownInput {
@@ -41,10 +46,10 @@ func GetClusterLocalGateway() (string, error) {
 func ConfigInit() error {
 	sess, _ := session.NewSession()
 	metadata := NewEC2Metadata(sess)
-	return configInit(metadata)
+	return configInit(sess, metadata)
 }
 
-func configInit(metadata EC2Metadata) error {
+func configInit(sess *session.Session, metadata EC2Metadata) error {
 	var err error
 
 	// CLUSTER_VPC_ID
@@ -77,6 +82,12 @@ func configInit(metadata EC2Metadata) error {
 	// CLUSTER_LOCAL_GATEWAY
 	DefaultServiceNetwork = os.Getenv(CLUSTER_LOCAL_GATEWAY)
 
+	// CLUSTER_NAME
+	ClusterName, err = getClusterName(sess)
+	if err != nil {
+		return fmt.Errorf("cannot get cluster name: %s", err)
+	}
+
 	// TARGET_GROUP_NAME_LEN_MODE
 	tgNameLengthMode := os.Getenv(TARGET_GROUP_NAME_LEN_MODE)
 
@@ -88,3 +99,37 @@ func configInit(metadata EC2Metadata) error {
 
 	return nil
 }
+
+// try to find cluster name, search in env then in ec2 instance tags
+func getClusterName(sess *session.Session) (string, error) {
+	cn := os.Getenv(CLUSTER_NAME)
+	if cn != "" {
+		return cn, nil
+	}
+	// fallback to ec2 instance tags
+	meta := ec2metadata.New(sess)
+	doc, err := meta.GetInstanceIdentityDocument()
+	if err != nil {
+		return "", err
+	}
+	instanceId := doc.InstanceID
+	region, err := meta.Region()
+	if err != nil {
+		return "", err
+	}
+	ec2Client := ec2.New(sess, &aws.Config{Region: aws.String(region)})
+	tagReq := &ec2.DescribeTagsInput{Filters: []*ec2.Filter{{
+		Name:   aws.String("resource-id"),
+		Values: []*string{aws.String(instanceId)},
+	}}}
+	tagRes, err := ec2Client.DescribeTags(tagReq)
+	if err != nil {
+		return "", err
+	}
+	for _, tag := range tagRes.Tags {
+		if *tag.Key == "aws:eks:cluster-name" {
+			return *tag.Value, nil
+		}
+	}
+	return "", errors.New("not found in env and metadata")
+}
diff --git a/pkg/deploy/lattice/service_manager.go b/pkg/deploy/lattice/service_manager.go
index 5fd9e19b..a1d721e9 100644
--- a/pkg/deploy/lattice/service_manager.go
+++ b/pkg/deploy/lattice/service_manager.go
@@ -86,6 +86,7 @@ func (m *defaultServiceManager) createAssociation(ctx context.Context, svcId *st
 	assocReq := &CreateSnSvcAssocReq{
 		ServiceIdentifier:        svcId,
 		ServiceNetworkIdentifier: aws.String(sn.ID),
+		Tags:                     m.cloud.DefaultTags(),
 	}
 	assocResp, err := m.cloud.Lattice().CreateServiceNetworkServiceAssociationWithContext(ctx, assocReq)
 	if err != nil {
@@ -187,7 +188,16 @@ func (m *defaultServiceManager) updateAssociations(ctx context.Context, svc *Ser
 	}
 
 	for _, assoc := range toDelete {
-		m.deleteAssociation(ctx, assoc.Arn)
+		isManaged, err := m.cloud.IsArnManaged(*assoc.Arn)
+		if err != nil {
+			return err
+		}
+		if isManaged {
+			err = m.deleteAssociation(ctx, assoc.Arn)
+			if err != nil {
+				return err
+			}
+		}
 	}
 
 	return nil


Test patch:
diff --git a/pkg/aws/cloud_test.go b/pkg/aws/cloud_test.go
new file mode 100644
index 00000000..28adead6
--- /dev/null
+++ b/pkg/aws/cloud_test.go
@@ -0,0 +1,81 @@
+package aws
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/aws/aws-application-networking-k8s/pkg/aws/services"
+	"github.com/aws/aws-sdk-go/service/vpclattice"
+	gomock "github.com/golang/mock/gomock"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetManagedByTag(t *testing.T) {
+
+	t.Run("account, cluster name, and vpc", func(t *testing.T) {
+		cfg := CloudConfig{
+			AccountId:   "acc",
+			VpcId:       "vpc",
+			ClusterName: "cluster",
+		}
+		tag := getManagedByTag(cfg)
+		assert.Equal(t, "acc/cluster/vpc", tag)
+	})
+
+}
+
+func TestDefaultTags(t *testing.T) {
+	cfg := CloudConfig{"acc", "vpc", "region", "cluster"}
+	c := NewDefaultCloud(nil, cfg)
+	tags := c.DefaultTags()
+	tagWant := getManagedByTag(cfg)
+	tagGot, exits := tags[TagManagedBy]
+	assert.True(t, exits)
+	assert.Equal(t, tagWant, *tagGot)
+}
+
+func TestIsArnManaged(t *testing.T) {
+	c := gomock.NewController(t)
+	defer c.Finish()
+
+	lat := services.NewMockLattice(c)
+	cfg := CloudConfig{VpcId: "vpc-id", AccountId: "account-id"}
+	cl := NewDefaultCloud(lat, cfg)
+
+	t.Run("arn sent", func(t *testing.T) {
+		arn := "arn"
+		lat.EXPECT().ListTagsForResource(gomock.Any()).
+			DoAndReturn(
+				func(req *vpclattice.ListTagsForResourceInput) (*vpclattice.ListTagsForResourceOutput, error) {
+					assert.Equal(t, arn, *req.ResourceArn)
+					return &vpclattice.ListTagsForResourceOutput{}, nil
+				})
+		cl.IsArnManaged(arn)
+	})
+
+	t.Run("is managed", func(t *testing.T) {
+		arn := "arn"
+		lat.EXPECT().ListTagsForResource(gomock.Any()).
+			Return(&vpclattice.ListTagsForResourceOutput{
+				Tags: cl.DefaultTags(),
+			}, nil)
+		managed, err := cl.IsArnManaged(arn)
+		assert.Nil(t, err)
+		assert.True(t, managed)
+	})
+
+	t.Run("not managed", func(t *testing.T) {
+		lat.EXPECT().ListTagsForResource(gomock.Any()).
+			Return(&vpclattice.ListTagsForResourceOutput{}, nil)
+		managed, err := cl.IsArnManaged("arn")
+		assert.Nil(t, err)
+		assert.False(t, managed)
+	})
+
+	t.Run("error", func(t *testing.T) {
+		lat.EXPECT().ListTagsForResource(gomock.Any()).
+			Return(nil, errors.New(":("))
+		_, err := cl.IsArnManaged("arn")
+		assert.NotNil(t, err)
+	})
+}
diff --git a/pkg/config/controller_config_test.go b/pkg/config/controller_config_test.go
index 6e515ec1..c7955ded 100644
--- a/pkg/config/controller_config_test.go
+++ b/pkg/config/controller_config_test.go
@@ -38,7 +38,7 @@ func Test_config_init_with_partial_env_var(t *testing.T) {
 	os.Setenv(CLUSTER_LOCAL_GATEWAY, testClusterLocalGateway)
 	os.Unsetenv(AWS_ACCOUNT_ID)
 	os.Unsetenv(TARGET_GROUP_NAME_LEN_MODE)
-	err := configInit(ec2MetadataUnavailable())
+	err := configInit(nil, ec2MetadataUnavailable())
 	assert.NotNil(t, err)
 }
 
@@ -48,7 +48,7 @@ func Test_config_init_no_env_var(t *testing.T) {
 	os.Unsetenv(CLUSTER_LOCAL_GATEWAY)
 	os.Unsetenv(AWS_ACCOUNT_ID)
 	os.Unsetenv(TARGET_GROUP_NAME_LEN_MODE)
-	err := configInit(ec2MetadataUnavailable())
+	err := configInit(nil, ec2MetadataUnavailable())
 	assert.NotNil(t, err)
 
 }
@@ -60,16 +60,19 @@ func Test_config_init_with_all_env_var(t *testing.T) {
 	testClusterLocalGateway := "default"
 	testTargetGroupNameLenMode := "long"
 	testAwsAccountId := "12345678"
+	testClusterName := "cluster-name"
 
 	os.Setenv(REGION, testRegion)
 	os.Setenv(CLUSTER_VPC_ID, testClusterVpcId)
 	os.Setenv(CLUSTER_LOCAL_GATEWAY, testClusterLocalGateway)
 	os.Setenv(AWS_ACCOUNT_ID, testAwsAccountId)
 	os.Setenv(TARGET_GROUP_NAME_LEN_MODE, testTargetGroupNameLenMode)
-	configInit(ec2MetadataUnavailable())
+	os.Setenv(CLUSTER_NAME, testClusterName)
+	configInit(nil, ec2MetadataUnavailable())
 	assert.Equal(t, Region, testRegion)
 	assert.Equal(t, VpcID, testClusterVpcId)
 	assert.Equal(t, AccountID, testAwsAccountId)
 	assert.Equal(t, DefaultServiceNetwork, testClusterLocalGateway)
 	assert.Equal(t, UseLongTGName, true)
+	assert.Equal(t, testClusterName, ClusterName)
 }
diff --git a/pkg/deploy/lattice/service_manager_test.go b/pkg/deploy/lattice/service_manager_test.go
index aeda5a1b..4144ccbe 100644
--- a/pkg/deploy/lattice/service_manager_test.go
+++ b/pkg/deploy/lattice/service_manager_test.go
@@ -21,7 +21,7 @@ func TestServiceManagerInteg(t *testing.T) {
 
 	lat := services.NewMockLattice(c)
 	cfg := mocks_aws.CloudConfig{VpcId: "vpc-id", AccountId: "account-id"}
-	cl := mocks_aws.NewDefaultCloud(lat, nil, cfg)
+	cl := mocks_aws.NewDefaultCloud(lat, cfg)
 	ds := latticestore.NewLatticeDataStore()
 	ctx := context.Background()
 	m := NewServiceManager(cl, ds)
@@ -29,6 +29,7 @@ func TestServiceManagerInteg(t *testing.T) {
 	// Case for single service and single sn-svc association
 	// Make sure that we send requests to Lattice for create Service and create Sn-Svc
 	t.Run("create new service and association", func(t *testing.T) {
+
 		svc := &Service{
 			Spec: latticemodel.ServiceSpec{
 				Name:                "svc",
@@ -57,7 +58,8 @@ func TestServiceManagerInteg(t *testing.T) {
 						DnsEntry: &vpclattice.DnsEntry{DomainName: aws.String("dns")},
 						Id:       aws.String("svc-id"),
 					}, nil
-				})
+				}).
+			Times(1)
 
 		// assert that we call create association
 		lat.EXPECT().
@@ -65,10 +67,12 @@ func TestServiceManagerInteg(t *testing.T) {
 			DoAndReturn(
 				func(_ context.Context, req *CreateSnSvcAssocReq, _ ...interface{}) (*CreateSnSvcAssocResp, error) {
 					assert.Equal(t, "sn-id", *req.ServiceNetworkIdentifier)
+					assert.True(t, cl.ContainsManagedBy(req.Tags))
 					return &CreateSnSvcAssocResp{
 						Status: aws.String(vpclattice.ServiceNetworkServiceAssociationStatusActive),
 					}, nil
-				})
+				}).
+			Times(1)
 
 		status, err := m.Create(ctx, svc)
 		assert.Nil(t, err)
@@ -77,19 +81,26 @@ func TestServiceManagerInteg(t *testing.T) {
 
 	// Update is more complex than create, we need to apply diff for Sn-Svc associations
 	// This test covers creation/deletion for multiple SN's
-	// Service is associated with sn-keep and sn-delete,
-	// for update we need to delete sn-delete and add sn-add
+	// sn-keep - no changes
+	// sn-delete - managed by gateway and want to delete
+	// sn-create - managed by gateway and want to create
+	// sn-foreign - not managed by gateway and exists in lattice (should keep)
 	t.Run("update service's associations", func(t *testing.T) {
-		snNames := []string{"sn-keep", "sn-delete", "sn-add"}
+		snKeep := "sn-keep"
+		snDelete := "sn-delete"
+		snAdd := "sn-add"
+		snForeign := "sn-foreign"
+
 		svc := &Service{
 			Spec: latticemodel.ServiceSpec{
 				Name:                "svc",
 				Namespace:           "ns",
-				ServiceNetworkNames: []string{snNames[0], snNames[2]},
+				ServiceNetworkNames: []string{snKeep, snAdd},
 			},
 		}
 
-		for _, sn := range snNames {
+		// populate storage with managed sn's
+		for _, sn := range []string{snKeep, snDelete, snAdd} {
 			ds.AddServiceNetwork(sn, cfg.AccountId, sn+"-arn", sn+"-id", sn+"-status")
 		}
 
@@ -100,42 +111,58 @@ func TestServiceManagerInteg(t *testing.T) {
 				Arn:  aws.String("svc-arn"),
 				Id:   aws.String("svc-id"),
 				Name: aws.String(svc.LatticeName()),
-			}}, nil)
+			}}, nil).
+			Times(1)
 
-		// 2 associations exists, keep and delete
+		// 3 associations exist in lattice: keep, delete, and foreign
 		lat.EXPECT().
 			ListServiceNetworkServiceAssociationsAsList(gomock.Any(), gomock.Any()).
-			Return([]*SnSvcAssocSummary{
-				{
-					Arn:                aws.String(snNames[0] + "-arn"),
-					Id:                 aws.String(snNames[0] + "-id"),
-					ServiceNetworkName: &snNames[0],
-					Status:             aws.String(vpclattice.ServiceNetworkServiceAssociationStatusActive),
-				},
-				{
-					Arn:                aws.String(snNames[1] + "-arn"),
-					Id:                 aws.String(snNames[1] + "-id"),
-					ServiceNetworkName: &snNames[1],
-					Status:             aws.String(vpclattice.ServiceNetworkServiceAssociationStatusActive),
-				},
-			}, nil)
+			DoAndReturn(func(_ context.Context, req *ListSnSvcAssocsReq) ([]*SnSvcAssocSummary, error) {
+				assocs := []*SnSvcAssocSummary{}
+				for _, sn := range []string{snKeep, snDelete, snForeign} {
+					assocs = append(assocs, &SnSvcAssocSummary{
+						Arn:                aws.String(sn + "-arn"),
+						Id:                 aws.String(sn + "-id"),
+						ServiceNetworkName: aws.String(sn),
+						Status:             aws.String(vpclattice.ServiceNetworkServiceAssociationStatusActive),
+					})
+				}
+				return assocs, nil
+			}).
+			Times(1)
+
+		// return managed by gateway controller tags for all associations except for foreign
+		lat.EXPECT().ListTagsForResource(gomock.Any()).
+			DoAndReturn(func(req *vpclattice.ListTagsForResourceInput) (*vpclattice.ListTagsForResourceOutput, error) {
+				if *req.ResourceArn == snForeign+"-arn" {
+					return &vpclattice.ListTagsForResourceOutput{}, nil
+				} else {
+					return &vpclattice.ListTagsForResourceOutput{
+						Tags: cl.DefaultTags(),
+					}, nil
+				}
+			}).
+			Times(2) // delete and foreign
 
-		// assert we call create and delete associations
+		// assert we call create and delete associations on managed resources
 		lat.EXPECT().
 			CreateServiceNetworkServiceAssociationWithContext(gomock.Any(), gomock.Any()).
 			DoAndReturn(
 				func(_ context.Context, req *CreateSnSvcAssocReq, _ ...interface{}) (*CreateSnSvcAssocResp, error) {
-					assert.Equal(t, "sn-add-id", *req.ServiceNetworkIdentifier)
+					assert.Equal(t, snAdd+"-id", *req.ServiceNetworkIdentifier)
 					return &CreateSnSvcAssocResp{Status: aws.String(vpclattice.ServiceNetworkServiceAssociationStatusActive)}, nil
-				})
+				}).
+			Times(1)
 
+		// make sure we call delete only on managed resource, only snDelete should be deleted
 		lat.EXPECT().
 			DeleteServiceNetworkServiceAssociationWithContext(gomock.Any(), gomock.Any(), gomock.Any()).
 			DoAndReturn(
 				func(_ context.Context, req *DelSnSvcAssocReq, _ ...interface{}) (*DelSnSvcAssocResp, error) {
-					assert.Equal(t, "sn-delete-arn", *req.ServiceNetworkServiceAssociationIdentifier)
+					assert.Equal(t, snDelete+"-arn", *req.ServiceNetworkServiceAssociationIdentifier)
 					return &DelSnSvcAssocResp{}, nil
-				})
+				}).
+			Times(1)
 
 		status, err := m.Create(ctx, svc)
 		assert.Nil(t, err)
@@ -173,10 +200,12 @@ func TestServiceManagerInteg(t *testing.T) {
 		// assert we delete association and service
 		lat.EXPECT().
 			DeleteServiceNetworkServiceAssociationWithContext(gomock.Any(), gomock.Any(), gomock.Any()).
-			Return(nil, nil)
+			Return(nil, nil).
+			Times(1)
 
 		lat.EXPECT().
-			DeleteServiceWithContext(gomock.Any(), gomock.Any()).Return(nil, nil)
+			DeleteServiceWithContext(gomock.Any(), gomock.Any()).Return(nil, nil).
+			Times(1)
 
 		err := m.Delete(ctx, svc)
 		assert.Nil(t, err)
@@ -186,7 +215,7 @@ func TestServiceManagerInteg(t *testing.T) {
 
 func TestCreateSvcReq(t *testing.T) {
 	cfg := mocks_aws.CloudConfig{VpcId: "vpc-id", AccountId: "account-id"}
-	cl := mocks_aws.NewDefaultCloud(nil, nil, cfg)
+	cl := mocks_aws.NewDefaultCloud(nil, cfg)
 	ds := latticestore.NewLatticeDataStore()
 	m := NewServiceManager(cl, ds)