Repository: amp-labs/connectors
Pull request #1603 (2025-04-29 00:16:02)

[CON-1667] feat(aws): Connector
# Features

* Defined AWS provider
  * Do we need to create new AuthType?
  * Provider BaseURL is Not Applicable.
  * Module BaseURL acts as a URL template, where region is known at connector init, service domain scoped to each API request.
* Added `AWSClient`. This depends on `aws-sdk-go-v2` package.
* Connector requires metadata: "region", "identityStoreID", "instanceArn".
* Internal `awscore` can be used by `aws` package only. It is intended for making HTTP calls to AWS.
* GetAWSConnector is an example on how to init a connector. It will be used by tests.

# Logos
![image](https://github.com/user-attachments/assets/c8cf10bf-8f0a-43c0-baeb-b87112fe0bcd)


Cobalt0s: * **#1603** <a href="https://app.graphite.dev/github/pr/amp-labs/connectors/1603?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a> 👈 <a href="https://app.graphite.dev/github/pr/amp-labs/connectors/1603?utm_source=stack-comment-view-in-graphite" target="_blank">(View in Graphite)</a>
* `main`


<h2></h2>

This stack of pull requests is managed by <a href="https://graphite.dev?utm-source=stack-comment"><b>Graphite</b></a>. Learn more about <a href="https://stacking.dev/?utm_source=stack-comment">stacking</a>.
<!-- Current dependencies on/for this PR: -->

Patch:
diff --git a/common/awsClient.go b/common/awsClient.go
new file mode 100644
index 00000000..33f0d985
--- /dev/null
+++ b/common/awsClient.go
@@ -0,0 +1,112 @@
+// nolint:ireturn
+package common
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"io"
+	"net/http"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+)
+
+type ctxKey string
+
+const AWSServiceContextKey ctxKey = "AWSService"
+
+// ErrRequestAWSMissingService is returned when deep connector implementation doesn't attach
+// AWS service name into the context, therefore the request cannot be constructed for sending.
+var ErrRequestAWSMissingService = errors.New("AWS request is missing Service name, supplied via context")
+
+type AWSClient struct {
+	client *http.Client
+	cfg    aws.Config
+	region string
+}
+
+// NewAWSClient constructs an AWSClient using Access Key ID and Secret.
+// Keys can be obtained as described here:
+// https://docs.aws.amazon.com/sdk-for-go/v2/developer-guide/getting-started.html#get-your-aws-access-keys
+//
+// The HTTP client is scoped to a single AWS region but supports multiple AWS services.
+// The request context must pass service domain via context key AWSServiceContextKey.
+//
+// Features:
+// * SHA256 payload hashing
+// * SigV4 request signing: https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html
+func NewAWSClient(ctx context.Context, client *http.Client,
+	accessKeyID, accessKeySecret, region string,
+) (AuthenticatedHTTPClient, error) {
+	sessionToken := "" // empty value signifies permanent credentials
+
+	cfg, err := config.LoadDefaultConfig(ctx,
+		config.WithRegion(region),
+		config.WithCredentialsProvider(
+			credentials.NewStaticCredentialsProvider(accessKeyID, accessKeySecret, sessionToken),
+		),
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	return AWSClient{
+		client: client,
+		cfg:    cfg,
+		region: region,
+	}, nil
+}
+
+func (c AWSClient) Do(req *http.Request) (*http.Response, error) {
+	// Sign the request
+	ctx := req.Context()
+
+	awsService, ok := ctx.Value(AWSServiceContextKey).(string)
+	if !ok || len(awsService) == 0 {
+		return nil, ErrRequestAWSMissingService
+	}
+
+	payload, err := extractPayload(req)
+	if err != nil {
+		return nil, err
+	}
+
+	sum := sha256.Sum256(payload)
+	payloadHash := hex.EncodeToString(sum[:])
+
+	creds, err := c.cfg.Credentials.Retrieve(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	signer := v4.NewSigner()
+
+	err = signer.SignHTTP(ctx, creds, req, payloadHash, awsService, c.region, time.Now())
+	if err != nil {
+		return nil, err
+	}
+
+	return c.client.Do(req)
+}
+
+func (c AWSClient) CloseIdleConnections() {
+	c.client.CloseIdleConnections()
+}
+
+func extractPayload(req *http.Request) ([]byte, error) {
+	bodyBytes, err := io.ReadAll(req.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	req.Body.Close()
+	req.Body = io.NopCloser(bytes.NewReader(bodyBytes))
+
+	return bodyBytes, nil
+}
diff --git a/common/json.go b/common/json.go
index 2e0184d1..3b6684a8 100644
--- a/common/json.go
+++ b/common/json.go
@@ -130,8 +130,9 @@ func ParseJSONResponse(res *http.Response, body []byte) (*JSONHTTPResponse, erro
 		}, nil
 	}
 
-	// Ensure the response is JSON
-	if err := EnsureContentType(`^application/.*json$`, res, false); err != nil {
+	// Ensure the response is JSON.
+	// Starts with application ends with JSON which may be followed by the version.
+	if err := EnsureContentType(`^application/.*json([-0-9.])*$`, res, false); err != nil {
 		return nil, err
 	}
 
diff --git a/go.mod b/go.mod
index b7ae138f..38113968 100644
--- a/go.mod
+++ b/go.mod
@@ -5,6 +5,9 @@ go 1.23.0
 require (
 	github.com/PuerkitoBio/goquery v1.10.3
 	github.com/antchfx/xmlquery v1.4.3
+	github.com/aws/aws-sdk-go-v2 v1.36.3
+	github.com/aws/aws-sdk-go-v2/config v1.29.14
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.67
 	github.com/brianvoe/gofakeit/v6 v6.28.0
 	github.com/chromedp/chromedp v0.13.6
 	github.com/deiu/linkparser v0.0.0-20170608193052-9b6849e15168
@@ -28,6 +31,16 @@ require (
 require (
 	github.com/andybalholm/cascadia v1.3.3 // indirect
 	github.com/antchfx/xpath v1.3.3 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.33.19 // indirect
+	github.com/aws/smithy-go v1.22.2 // indirect
 	github.com/chromedp/cdproto v0.0.0-20250403032234-65de8f5d025b // indirect
 	github.com/chromedp/sysutil v1.1.0 // indirect
 	github.com/fsnotify/fsnotify v1.8.0 // indirect
diff --git a/go.sum b/go.sum
index 266caa91..09f0ecf2 100644
--- a/go.sum
+++ b/go.sum
@@ -6,6 +6,32 @@ github.com/antchfx/xmlquery v1.4.3 h1:f6jhxCzANrWfa93O+NmRWvieVyLs+R2Szfpy+YrZaw
 github.com/antchfx/xmlquery v1.4.3/go.mod h1:AEPEEPYE9GnA2mj5Ur2L5Q5/2PycJ0N9Fusrx9b12fc=
 github.com/antchfx/xpath v1.3.3 h1:tmuPQa1Uye0Ym1Zn65vxPgfltWb/Lxu2jeqIGteJSRs=
 github.com/antchfx/xpath v1.3.3/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
+github.com/aws/aws-sdk-go-v2 v1.36.3 h1:mJoei2CxPutQVxaATCzDUjcZEjVRdpsiiXi2o38yqWM=
+github.com/aws/aws-sdk-go-v2 v1.36.3/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg=
+github.com/aws/aws-sdk-go-v2/config v1.29.14 h1:f+eEi/2cKCg9pqKBoAIwRGzVb70MRKqWX4dg1BDcSJM=
+github.com/aws/aws-sdk-go-v2/config v1.29.14/go.mod h1:wVPHWcIFv3WO89w0rE10gzf17ZYy+UVS1Geq8Iei34g=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.67 h1:9KxtdcIA/5xPNQyZRgUSpYOE6j9Bc4+D7nZua0KGYOM=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.67/go.mod h1:p3C44m+cfnbv763s52gCqrjaqyPikj9Sg47kUVaNZQQ=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 h1:x793wxmUWVDhshP8WW2mlnXuFrO4cOd3HLBroh1paFw=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30/go.mod h1:Jpne2tDnYiFascUEs2AWHJL9Yp7A5ZVy3TNyxaAjD6M=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 h1:ZK5jHhnrioRkUNOc+hOgQKlUL5JeC3S6JgLxtQ+Rm0Q=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34/go.mod h1:p4VfIceZokChbA9FzMbRGz5OV+lekcVtHlPKEO0gSZY=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 h1:SZwFm17ZUNNg5Np0ioo/gq8Mn6u9w19Mri8DnJ15Jf0=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34/go.mod h1:dFZsC0BLo346mvKQLWmoJxT+Sjp+qcVR1tRVHQGOH9Q=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 h1:eAh2A4b5IzM/lum78bZ590jy36+d/aFLgKF/4Vd1xPE=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3/go.mod h1:0yKJC/kb8sAnmlYa6Zs3QVYqaC8ug2AbnNChv5Ox3uA=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 h1:dM9/92u2F1JbDaGooxTq18wmmFzbJRfXfVfy96/1CXM=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15/go.mod h1:SwFBy2vjtA0vZbjjaFtfN045boopadnoVPhu4Fv66vY=
+github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 h1:1Gw+9ajCV1jogloEv1RRnvfRFia2cL6c9cuKV2Ps+G8=
+github.com/aws/aws-sdk-go-v2/service/sso v1.25.3/go.mod h1:qs4a9T5EMLl/Cajiw2TcbNt2UNo/Hqlyp+GiuG4CFDI=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 h1:hXmVKytPfTy5axZ+fYbR5d0cFmC3JvwLm5kM83luako=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1/go.mod h1:MlYRNmYu/fGPoxBQVvBYr9nyr948aY/WLUvwBMBJubs=
+github.com/aws/aws-sdk-go-v2/service/sts v1.33.19 h1:1XuUZ8mYJw9B6lzAkXhqHlJd/XvaX32evhproijJEZY=
+github.com/aws/aws-sdk-go-v2/service/sts v1.33.19/go.mod h1:cQnB8CUnxbMU82JvlqjKR2HBOm3fe9pWorWBza6MBJ4=
+github.com/aws/smithy-go v1.22.2 h1:6D9hW43xKFrRx/tXXfAlIZc4JI+yQe6snnWcQyxSyLQ=
+github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
 github.com/brianvoe/gofakeit/v6 v6.28.0 h1:Xib46XXuQfmlLS2EXRuJpqcw8St6qSZz75OUo0tgAW4=
 github.com/brianvoe/gofakeit/v6 v6.28.0/go.mod h1:Xj58BMSnFqcn/fAQeSK+/PLtC5kSb7FJIq4JyGa8vEs=
 github.com/chromedp/cdproto v0.0.0-20250403032234-65de8f5d025b h1:jJmiCljLNTaq/O1ju9Bzz2MPpFlmiTn0F7LwCoeDZVw=
diff --git a/providers/aws.go b/providers/aws.go
new file mode 100644
index 00000000..ac34b4e0
--- /dev/null
+++ b/providers/aws.go
@@ -0,0 +1,49 @@
+package providers
+
+import "github.com/amp-labs/connectors/common"
+
+const AWS Provider = "aws"
+
+const ModuleAWSIdentityCenter common.ModuleID = "aws-identity-center"
+
+func init() {
+	SetInfo(AWS, ProviderInfo{
+		DisplayName: "Amazon Web Services",
+		AuthType:    Basic,
+		BaseURL:     "https", // TODO is it ok that this value will be empty? Validation enforces some value.
+		Support: Support{
+			BulkWrite: BulkWriteSupport{
+				Insert: false,
+				Update: false,
+				Upsert: false,
+				Delete: false,
+			},
+			Proxy:     false,
+			Read:      false,
+			Subscribe: false,
+			Write:     false,
+		},
+		Modules: &Modules{
+			ModuleAWSIdentityCenter: {
+				// TODO serviceDomain changes based on the request. This is not global to the connector.
+				BaseURL:     "https://{{.serviceDomain}}.{{.region}}.amazonaws.com",
+				DisplayName: "AWS Identity Center",
+				Support: Support{
+					Read:      false,
+					Subscribe: false,
+					Write:     false,
+				},
+			},
+		},
+		Media: &Media{
+			DarkMode: &MediaTypeDarkMode{
+				IconURL: "https://res.cloudinary.com/dycvts6vp/image/upload/v1746046733/media/aws_1746046732.jpg",
+				LogoURL: "https://res.cloudinary.com/dycvts6vp/image/upload/v1746046794/media/aws_1746046793.svg",
+			},
+			Regular: &MediaTypeRegular{
+				IconURL: "https://res.cloudinary.com/dycvts6vp/image/upload/v1746046733/media/aws_1746046732.jpg",
+				LogoURL: "https://res.cloudinary.com/dycvts6vp/image/upload/v1746046777/media/aws_1746046777.svg",
+			},
+		},
+	})
+}
diff --git a/providers/aws/connector.go b/providers/aws/connector.go
new file mode 100644
index 00000000..500bda7b
--- /dev/null
+++ b/providers/aws/connector.go
@@ -0,0 +1,78 @@
+package aws
+
+import (
+	"strings"
+
+	"github.com/amp-labs/connectors/common"
+	"github.com/amp-labs/connectors/internal/components"
+	"github.com/amp-labs/connectors/providers"
+)
+
+type Connector struct {
+	// Basic connector
+	*components.Connector
+
+	// Require authenticated client
+	common.RequireAuthenticatedClient
+	common.RequireModule
+	common.RequireMetadata
+
+	// supported operations
+	components.Reader
+
+	region          string
+	identityStoreId string
+	instanceARN     string
+}
+
+func NewConnector(params common.Parameters) (*Connector, error) {
+	conn, err := components.Initialize(providers.AWS, params,
+		func(connector *components.Connector) (*Connector, error) {
+			var expectedMetadataKeys []string
+			if params.Module == providers.ModuleAWSIdentityCenter {
+				expectedMetadataKeys = []string{"region", "identityStoreId", "instanceARN"}
+			}
+
+			return constructor(connector, expectedMetadataKeys)
+		},
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	// TODO this should be resolved by the ProviderInfo initialization.
+	conn.region = params.Metadata["region"]
+	conn.identityStoreId = params.Metadata["identityStoreId"]
+	conn.instanceARN = params.Metadata["instanceARN"]
+
+	return conn, nil
+}
+
+func constructor(base *components.Connector, expectedMetadataKeys []string) (*Connector, error) {
+	connector := &Connector{
+		Connector: base,
+		RequireModule: common.RequireModule{
+			ExpectedModules: []common.ModuleID{
+				providers.ModuleAWSIdentityCenter,
+			},
+		},
+		RequireMetadata: common.RequireMetadata{
+			ExpectedMetadataKeys: expectedMetadataKeys,
+		},
+	}
+
+	return connector, nil
+}
+
+// nolint:unused
+func (c *Connector) getModuleURL() (string, error) {
+	modules := c.ProviderInfo().Modules
+	if modules == nil {
+		return "", common.ErrInvalidModuleDeclaration
+	}
+
+	baseURL := (*modules)[c.Module()].BaseURL
+	baseURL = strings.Replace(baseURL, "{{.region}}", c.region, 1)
+
+	return baseURL, nil
+}
diff --git a/providers/aws/internal/awscore/commands.go b/providers/aws/internal/awscore/commands.go
new file mode 100644
index 00000000..71f399ce
--- /dev/null
+++ b/providers/aws/internal/awscore/commands.go
@@ -0,0 +1,13 @@
+package awscore
+
+import "github.com/amp-labs/connectors/internal/datautils"
+
+type Command string
+
+func FormatCommand(serviceName string, registry datautils.Map[string, string], objectName string) Command {
+	return Command(serviceName + "." + registry[objectName])
+}
+
+func (c Command) String() string {
+	return string(c)
+}
diff --git a/providers/aws/internal/awscore/requests.go b/providers/aws/internal/awscore/requests.go
new file mode 100644
index 00000000..09acee7e
--- /dev/null
+++ b/providers/aws/internal/awscore/requests.go
@@ -0,0 +1,70 @@
+package awscore
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"net/http"
+	"strings"
+
+	"github.com/amp-labs/connectors/common"
+	"github.com/amp-labs/connectors/internal/goutils"
+)
+
+// nolint:lll
+const (
+	// Mime https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-making-api-requests-json.html#sqs-api-constructing-endpoints-json
+	Mime = "application/x-amz-json-1.1"
+	// MaxPageSize https://docs.aws.amazon.com/cli/latest/userguide/cli-usage-pagination.html
+	MaxPageSize = 1000
+)
+
+// nolint:tagliatelle
+type ReadPayload struct {
+	MaxResults int     `json:"MaxResults"`
+	NextToken  *string `json:"NextToken,omitempty"`
+}
+
+func NewReadPayload(params common.ReadParams) *ReadPayload {
+	var nextToken *string
+	if len(params.NextPage) != 0 {
+		nextToken = goutils.Pointer(params.NextPage.String())
+	}
+
+	return &ReadPayload{
+		MaxResults: MaxPageSize,
+		NextToken:  nextToken,
+	}
+}
+
+func BuildRequest(
+	ctx context.Context, baseURL, serviceDomain, serviceSigningName string, command Command, payload any,
+) (*http.Request, error) {
+	reader, err := getReader(payload)
+	if err != nil {
+		return nil, err
+	}
+
+	ctx = context.WithValue(ctx, common.AWSServiceContextKey, serviceSigningName)
+	baseURL = strings.Replace(baseURL, "{{.serviceDomain}}", serviceDomain, 1)
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, baseURL, reader)
+	if err != nil {
+		return nil, err
+	}
+
+	// Required headers
+	req.Header.Set("Content-Type", Mime)
+	req.Header.Set("X-Amz-Target", command.String())
+
+	return req, nil
+}
+
+func getReader(payload any) (*bytes.Reader, error) {
+	data, err := json.Marshal(payload)
+	if err != nil {
+		return nil, err
+	}
+
+	return bytes.NewReader(data), nil
+}


Test patch:
diff --git a/test/aws/connector.go b/test/aws/connector.go
new file mode 100644
index 00000000..299df089
--- /dev/null
+++ b/test/aws/connector.go
@@ -0,0 +1,82 @@
+package aws
+
+import (
+	"context"
+	"net/http"
+
+	"github.com/amp-labs/connectors/common"
+	"github.com/amp-labs/connectors/common/scanning/credscanning"
+	"github.com/amp-labs/connectors/providers"
+	"github.com/amp-labs/connectors/providers/aws"
+	testUtils "github.com/amp-labs/connectors/test/utils"
+)
+
+/*
+File example:
+	{
+	  "provider": "aws",
+	  "username": ".....", // AWS Access Key ID
+	  "password": ".....", // AWS Access Key Secret
+	  "metadata": {
+		"region": "us-east-2",
+		"identityStoreId": "d-9a670e6550",
+		"instanceARN": "arn:aws:sso:::instance/ssoins-668432c2a02ced8f"
+	  }
+	}
+*/
+
+// nolint:gochecknoglobals
+var (
+	fieldRegion = credscanning.Field{
+		Name:      "region",
+		PathJSON:  "metadata.region",
+		SuffixENV: "REGION",
+	}
+	fieldIdentityStoreID = credscanning.Field{
+		Name:      "identityStoreId",
+		PathJSON:  "metadata.identityStoreId",
+		SuffixENV: "IDENTITY_STORE_ID",
+	}
+	fieldInstanceArn = credscanning.Field{
+		Name:      "instanceARN",
+		PathJSON:  "metadata.instanceARN",
+		SuffixENV: "INSTANCE_ARN",
+	}
+)
+
+func GetAWSConnector(ctx context.Context, module common.ModuleID) *aws.Connector {
+	filePath := credscanning.LoadPath(providers.AWS)
+	reader := testUtils.MustCreateProvCredJSON(filePath, false, false,
+		fieldRegion, fieldIdentityStoreID, fieldInstanceArn,
+	)
+
+	awsRegion := reader.Get(fieldRegion)
+
+	client, err := common.NewAWSClient(ctx,
+		http.DefaultClient,
+		reader.Get(credscanning.Fields.Username),
+		reader.Get(credscanning.Fields.Password),
+		awsRegion,
+	)
+
+	if err != nil {
+		testUtils.Fail(err.Error())
+	}
+
+	conn, err := aws.NewConnector(
+		common.Parameters{
+			Module:              module,
+			AuthenticatedClient: client,
+			Metadata: map[string]string{
+				"region":          awsRegion,
+				"identityStoreId": reader.Get(fieldIdentityStoreID),
+				"instanceARN":     reader.Get(fieldInstanceArn),
+			},
+		},
+	)
+	if err != nil {
+		testUtils.Fail("error creating connector", "error", err)
+	}
+
+	return conn
+}