Repository: amp-labs/connectors
Pull request #1608 (2025-04-30 00:46:01)

[CON-1667] feat(aws): Metadata
# Description

Using [AWS Golang SDK](https://github.com/aws/aws-sdk-go-v2/tree/main) inferred object schema response. Metadata is stored under `schema.json` associated with each AWS service.
Package `awsidentitystore` matches `identitystore` service and `awsssoadmin` matches `sso-admin`.

# Functionality

When object is queried search for metadata service by service.
AWS Identity Center module combines `identitystore` and `sso-admin`. In future other AWS modules may correspond to one or many AWS services.

# Testing
![image](https://github.com/user-attachments/assets/bc05ba05-4369-4e61-bab4-4428664f3725)


Cobalt0s: > [!WARNING]
> <b>This pull request is not mergeable via GitHub because a downstack PR is open. Once all requirements are satisfied, merge this PR as a stack <a href="https://app.graphite.dev/github/pr/amp-labs/connectors/1608?utm_source=stack-comment-downstack-mergeability-warning" >on Graphite</a>.</b>
> <a href="https://graphite.dev/docs/merge-pull-requests">Learn more</a>

* **#1609** <a href="https://app.graphite.dev/github/pr/amp-labs/connectors/1609?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a>
* **#1608** <a href="https://app.graphite.dev/github/pr/amp-labs/connectors/1608?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a> 👈 <a href="https://app.graphite.dev/github/pr/amp-labs/connectors/1608?utm_source=stack-comment-view-in-graphite" target="_blank">(View in Graphite)</a>
* **#1603** <a href="https://app.graphite.dev/github/pr/amp-labs/connectors/1603?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a>: 1 other dependent PR ([#1604](https://github.com/amp-labs/connectors/pull/1604) <a href="https://app.graphite.dev/github/pr/amp-labs/connectors/1604?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a>)
* **#1378** <a href="https://app.graphite.dev/github/pr/amp-labs/connectors/1378?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a>
* `main`


<h2></h2>

This stack of pull requests is managed by <a href="https://graphite.dev?utm-source=stack-comment"><b>Graphite</b></a>. Learn more about <a href="https://stacking.dev/?utm_source=stack-comment">stacking</a>.
<!-- Current dependencies on/for this PR: -->

Patch:
diff --git a/providers/aws/internal/awscore/commands.go b/providers/aws/internal/core/commands.go
similarity index 95%
rename from providers/aws/internal/awscore/commands.go
rename to providers/aws/internal/core/commands.go
index 71f399ce..d838f0ff 100644
--- a/providers/aws/internal/awscore/commands.go
+++ b/providers/aws/internal/core/commands.go
@@ -1,4 +1,4 @@
-package awscore
+package core
 
 import "github.com/amp-labs/connectors/internal/datautils"
 
diff --git a/providers/aws/internal/awscore/requests.go b/providers/aws/internal/core/requests.go
similarity index 99%
rename from providers/aws/internal/awscore/requests.go
rename to providers/aws/internal/core/requests.go
index 09acee7e..413c708d 100644
--- a/providers/aws/internal/awscore/requests.go
+++ b/providers/aws/internal/core/requests.go
@@ -1,4 +1,4 @@
-package awscore
+package core
 
 import (
 	"bytes"
diff --git a/providers/aws/internal/identitystore/metadata.go b/providers/aws/internal/identitystore/metadata.go
new file mode 100644
index 00000000..76bdd552
--- /dev/null
+++ b/providers/aws/internal/identitystore/metadata.go
@@ -0,0 +1,20 @@
+package identitystore
+
+import (
+	_ "embed"
+
+	"github.com/amp-labs/connectors/internal/staticschema"
+	"github.com/amp-labs/connectors/tools/fileconv"
+	"github.com/amp-labs/connectors/tools/scrapper"
+)
+
+var (
+	//go:embed schemas.json
+	schemas []byte
+
+	FileManager = scrapper.NewMetadataFileManager[staticschema.FieldMetadataMapV1]( // nolint:gochecknoglobals
+		schemas, fileconv.NewSiblingFileLocator())
+
+	// Schemas is cached Object schemas.
+	Schemas = FileManager.MustLoadSchemas() // nolint:gochecknoglobals
+)
diff --git a/providers/aws/internal/identitystore/schemas.json b/providers/aws/internal/identitystore/schemas.json
new file mode 100644
index 00000000..e0091c85
--- /dev/null
+++ b/providers/aws/internal/identitystore/schemas.json
@@ -0,0 +1,51 @@
+{
+  "modules": {
+    "aws-identity-center": {
+      "objects": {
+        "Groups": {
+          "displayName": "Groups",
+          "responseKey": "Groups",
+          "fields": {
+            "GroupId": "Group Id",
+            "IdentityStoreId": "Identity Store Id",
+            "Description": "Description",
+            "DisplayName": "Display Name",
+            "ExternalIds": "External Ids"
+          }
+        },
+        "Users": {
+          "displayName": "Users",
+          "responseKey": "Users",
+          "fields": {
+            "IdentityStoreId": "Identity Store Id",
+            "UserId": "User Id",
+            "Addresses": "Addresses",
+            "DisplayName": "Display Name",
+            "Emails": "Emails",
+            "ExternalIds": "External Ids",
+            "Locale": "Locale",
+            "Name": "Name",
+            "NickName": "NickName",
+            "PhoneNumbers": "Phone Numbers",
+            "PreferredLanguage": "Preferred Language",
+            "ProfileUrl": "Profile URL",
+            "Timezone": "Timezone",
+            "Title": "Title",
+            "UserName": "User Name",
+            "UserType": "User Type"
+          }
+        },
+        "GroupMemberships": {
+          "displayName": "Group Memberships",
+          "responseKey": "GroupMemberships",
+          "fields": {
+            "IdentityStoreId": "Identity Store Id",
+            "GroupId": "Group Id",
+            "MemberId": "Member Id",
+            "MembershipId": "Membership Id"
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/providers/aws/internal/ssoadmin/metadata.go b/providers/aws/internal/ssoadmin/metadata.go
new file mode 100644
index 00000000..d584ddfe
--- /dev/null
+++ b/providers/aws/internal/ssoadmin/metadata.go
@@ -0,0 +1,20 @@
+package ssoadmin
+
+import (
+	_ "embed"
+
+	"github.com/amp-labs/connectors/internal/staticschema"
+	"github.com/amp-labs/connectors/tools/fileconv"
+	"github.com/amp-labs/connectors/tools/scrapper"
+)
+
+var (
+	//go:embed schemas.json
+	schemas []byte
+
+	FileManager = scrapper.NewMetadataFileManager[staticschema.FieldMetadataMapV1]( // nolint:gochecknoglobals
+		schemas, fileconv.NewSiblingFileLocator())
+
+	// Schemas is cached Object schemas.
+	Schemas = FileManager.MustLoadSchemas() // nolint:gochecknoglobals
+)
diff --git a/providers/aws/internal/ssoadmin/schemas.json b/providers/aws/internal/ssoadmin/schemas.json
new file mode 100644
index 00000000..f8ca30cd
--- /dev/null
+++ b/providers/aws/internal/ssoadmin/schemas.json
@@ -0,0 +1,81 @@
+{
+  "modules": {
+    "aws-identity-center": {
+      "objects": {
+        "Instances": {
+          "displayName": "Instances",
+          "responseKey": "Instances",
+          "fields": {
+            "CreatedDate": "Created Date",
+            "IdentityStoreId": "Identity Store Id",
+            "InstanceArn": "Instance Arn",
+            "Name": "Name",
+            "OwnerAccountId": "Owner Account Id",
+            "Status": "Status"
+          }
+        },
+        "Applications": {
+          "displayName": "Applications",
+          "responseKey": "Applications",
+          "fields": {
+            "ApplicationAccount": "Application Account",
+            "ApplicationArn": "Application Arn",
+            "ApplicationProviderArn": "Application Provider Arn",
+            "CreatedDate": "Created Date",
+            "Description": "Description",
+            "InstanceArn": "Instance Arn",
+            "Name": "Name",
+            "PortalOptions": "Portal Options",
+            "Status": "Status"
+          }
+        },
+        "ApplicationProviders": {
+          "displayName": "Application Providers",
+          "responseKey": "ApplicationProviders",
+          "fields": {
+            "ApplicationProviderArn": "Application Provider Arn",
+            "DisplayData": "Display Data",
+            "FederationProtocol": "Federation Protocol",
+            "ResourceServerConfig": "Resource Server Config"
+          }
+        },
+        "AccountAssignmentsCreationStatus": {
+          "displayName": "Account Assignments Creation Status",
+          "responseKey": "AccountAssignmentsCreationStatus",
+          "fields": {
+            "CreatedDate": "Created Date",
+            "RequestId": "Request Id",
+            "Status": "Status"
+          }
+        },
+        "AccountAssignmentsDeletionStatus": {
+          "displayName": "Account Assignments Deletion Status",
+          "responseKey": "AccountAssignmentsDeletionStatus",
+          "fields": {
+            "CreatedDate": "Created Date",
+            "RequestId": "Request Id",
+            "Status": "Status"
+          }
+        },
+        "PermissionSetsProvisioningStatus": {
+          "displayName": "Permission Sets Provisioning Status",
+          "responseKey": "PermissionSetsProvisioningStatus",
+          "fields": {
+            "CreatedDate": "Created Date",
+            "RequestId": "Request Id",
+            "Status": "Status"
+          }
+        },
+        "TrustedTokenIssuers": {
+          "displayName": "Trusted Token Issuers",
+          "responseKey": "TrustedTokenIssuers",
+          "fields": {
+            "Name": "Name",
+            "TrustedTokenIssuerArn": "Trusted Token Issuer Arn",
+            "TrustedTokenIssuerType": "Trusted Token Issuer Type"
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/providers/aws/metadata.go b/providers/aws/metadata.go
new file mode 100644
index 00000000..2e63254d
--- /dev/null
+++ b/providers/aws/metadata.go
@@ -0,0 +1,46 @@
+package aws
+
+import (
+	"context"
+
+	"github.com/amp-labs/connectors"
+	"github.com/amp-labs/connectors/common"
+	"github.com/amp-labs/connectors/providers"
+	"github.com/amp-labs/connectors/providers/aws/internal/identitystore"
+	"github.com/amp-labs/connectors/providers/aws/internal/ssoadmin"
+)
+
+func (c *Connector) ListObjectMetadata(
+	ctx context.Context, objectNames []string,
+) (*connectors.ListObjectMetadataResult, error) {
+	if len(objectNames) == 0 {
+		return nil, common.ErrMissingObjects
+	}
+
+	result := common.NewListObjectMetadataResult()
+
+	for _, name := range objectNames {
+		if data, ok := findObjectMetadata(name); ok {
+			result.Result[name] = *data
+		} else {
+			result.Errors[name] = common.ErrObjectNotSupported
+		}
+	}
+
+	return result, nil
+}
+
+// Try locating metadata for an object across AWS services.
+func findObjectMetadata(objectName string) (*common.ObjectMetadata, bool) {
+	data, err := identitystore.Schemas.SelectOne(providers.ModuleAWSIdentityCenter, objectName)
+	if err == nil {
+		return data, true
+	}
+
+	data, err = ssoadmin.Schemas.SelectOne(providers.ModuleAWSIdentityCenter, objectName)
+	if err == nil {
+		return data, true
+	}
+
+	return nil, false
+}


Test patch:
diff --git a/providers/aws/metadata_test.go b/providers/aws/metadata_test.go
new file mode 100644
index 00000000..6f8f9f34
--- /dev/null
+++ b/providers/aws/metadata_test.go
@@ -0,0 +1,115 @@
+package aws
+
+import (
+	"net/http"
+	"testing"
+
+	"github.com/amp-labs/connectors"
+	"github.com/amp-labs/connectors/common"
+	"github.com/amp-labs/connectors/providers"
+	"github.com/amp-labs/connectors/test/utils/mockutils/mockserver"
+	"github.com/amp-labs/connectors/test/utils/testroutines"
+)
+
+func TestListObjectMetadata(t *testing.T) { // nolint:funlen,gocognit,cyclop
+	t.Parallel()
+
+	tests := []testroutines.Metadata{
+		{
+			Name:         "At least one object name must be queried",
+			Input:        nil,
+			Server:       mockserver.Dummy(),
+			ExpectedErrs: []error{common.ErrMissingObjects},
+		},
+		{
+			Name:       "Unknown object requested",
+			Input:      []string{"butterflies"},
+			Server:     mockserver.Dummy(),
+			Comparator: testroutines.ComparatorSubsetMetadata,
+			Expected: &common.ListObjectMetadataResult{
+				Errors: map[string]error{
+					"butterflies": common.ErrObjectNotSupported,
+				},
+			},
+		},
+		{
+			Name:       "Metadata for objects across AWS Services",
+			Input:      []string{"Users", "Applications", "Helicopters"},
+			Server:     mockserver.Dummy(),
+			Comparator: testroutines.ComparatorSubsetMetadata,
+			Expected: &common.ListObjectMetadataResult{
+				Result: map[string]common.ObjectMetadata{
+					"Users": {
+						DisplayName: "Users",
+						FieldsMap: map[string]string{
+							"IdentityStoreId":   "Identity Store Id",
+							"UserId":            "User Id",
+							"Addresses":         "Addresses",
+							"DisplayName":       "Display Name",
+							"Emails":            "Emails",
+							"ExternalIds":       "External Ids",
+							"Locale":            "Locale",
+							"Name":              "Name",
+							"NickName":          "NickName",
+							"PhoneNumbers":      "Phone Numbers",
+							"PreferredLanguage": "Preferred Language",
+							"ProfileUrl":        "Profile URL",
+							"Timezone":          "Timezone",
+							"Title":             "Title",
+							"UserName":          "User Name",
+							"UserType":          "User Type",
+						},
+					},
+					"Applications": {
+						DisplayName: "Applications",
+						FieldsMap: map[string]string{
+							"ApplicationAccount":     "Application Account",
+							"ApplicationArn":         "Application Arn",
+							"ApplicationProviderArn": "Application Provider Arn",
+							"CreatedDate":            "Created Date",
+							"Description":            "Description",
+							"InstanceArn":            "Instance Arn",
+							"Name":                   "Name",
+							"PortalOptions":          "Portal Options",
+							"Status":                 "Status",
+						},
+					},
+				},
+				Errors: map[string]error{
+					"Helicopters": common.ErrObjectNotSupported,
+				},
+			},
+			ExpectedErrs: nil,
+		},
+	}
+
+	for _, tt := range tests {
+		// nolint:varnamelen
+		t.Run(tt.Name, func(t *testing.T) {
+			t.Parallel()
+
+			tt.Run(t, func() (connectors.ObjectMetadataConnector, error) {
+				return constructTestConnector(tt.Server.URL)
+			})
+		})
+	}
+}
+
+func constructTestConnector(serverURL string) (*Connector, error) {
+	connector, err := NewConnector(common.Parameters{
+		Module:              providers.ModuleAWSIdentityCenter,
+		AuthenticatedClient: http.DefaultClient,
+		Metadata: map[string]string{
+			"region":          "test-region",
+			"identityStoreID": "test-identity-store-id",
+			"instanceArn":     "test-instance-arn",
+		},
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	connector.SetBaseURL(serverURL)
+
+	return connector, nil
+}
diff --git a/test/aws/metadata/main.go b/test/aws/metadata/main.go
new file mode 100644
index 00000000..1d7756d3
--- /dev/null
+++ b/test/aws/metadata/main.go
@@ -0,0 +1,34 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"github.com/amp-labs/connectors/providers"
+	connTest "github.com/amp-labs/connectors/test/aws"
+	"github.com/amp-labs/connectors/test/utils"
+)
+
+func main() {
+	// Handle Ctrl-C gracefully.
+	ctx, done := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
+	defer done()
+
+	// Set up slog logging.
+	utils.SetupLogging()
+
+	conn := connTest.GetAWSConnector(ctx, providers.ModuleAWSIdentityCenter)
+
+	metadata, err := conn.ListObjectMetadata(ctx, []string{
+		"Users", "Applications",
+	})
+	if err != nil {
+		utils.Fail("error listing metadata", "error", err)
+	}
+
+	fmt.Println("Metadata...")
+	utils.DumpJSON(metadata, os.Stdout)
+}