Repository: amp-labs/connectors
Pull request #1378 (2025-02-18 23:57:31)

feat: Construct proxy connector with metadata+catalogVars
# Background
We are not limited to workspace catalog variables. The Google connector requires support for multiple modules, each differing by subdomain. This can only be communicated via metadata.
AWS connector also relies uppon region and some service specific metadata.

# Changes
* `connectors.NewConnector` now accepts `WithMetadata` option.
* `paramsbuilder.Metadata` returns all values as potential catalog variables.
* Proxy instances, regardless of auth type, capture substitutions and use the `WithMetadata` option.

Cobalt0s: * **#1378** <a href="https://app.graphite.dev/github/pr/amp-labs/connectors/1378?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a> 👈 <a href="https://app.graphite.dev/github/pr/amp-labs/connectors/1378?utm_source=stack-comment-view-in-graphite" target="_blank">(View in Graphite)</a>
* `main`


<h2></h2>

This stack of pull requests is managed by <a href="https://graphite.dev?utm-source=stack-comment"><b>Graphite</b></a>. Learn more about <a href="https://stacking.dev/?utm_source=stack-comment">stacking</a>.
<!-- Current dependencies on/for this PR: -->
laurenzlong: @Cobalt0s please don't merge this until @RajatPawar has had a chance to review - since he's doing some related work on provider metadata. 
Cobalt0s: > Looks good! Please make sure that you also make the equivalent changes in the server to supply this metadata.

There is no connector that supplies metadata this way. This would be relevant for proxy connectors that require metadata. I will save this as todo to see if it can be generalized in server.

Patch:
diff --git a/common/paramsbuilder/catalogVariable.go b/common/paramsbuilder/catalogVariable.go
index f2f42d5f..55c6aec6 100644
--- a/common/paramsbuilder/catalogVariable.go
+++ b/common/paramsbuilder/catalogVariable.go
@@ -1,8 +1,6 @@
 package paramsbuilder
 
 import (
-	"log/slog"
-
 	"github.com/amp-labs/connectors/common/substitutions"
 	"github.com/amp-labs/connectors/common/substitutions/catalogreplacer"
 )
@@ -21,7 +19,12 @@ func NewCatalogVariables[V substitutions.RegistryValue](
 		case catalogreplacer.VariableWorkspace:
 			result = append(result, &Workspace{Name: name})
 		default:
-			slog.Info("unknown substitution SubstitutionPlan for catalog", key, value)
+			result = append(result, &catalogreplacer.CustomCatalogVariable{
+				Plan: catalogreplacer.SubstitutionPlan{
+					From: key,
+					To:   name,
+				},
+			})
 		}
 	}
 
diff --git a/common/paramsbuilder/metadata.go b/common/paramsbuilder/metadata.go
index ead5ec22..3060cb5e 100644
--- a/common/paramsbuilder/metadata.go
+++ b/common/paramsbuilder/metadata.go
@@ -3,6 +3,8 @@ package paramsbuilder
 import (
 	"errors"
 	"fmt"
+
+	"github.com/amp-labs/connectors/common/substitutions/catalogreplacer"
 )
 
 var (
@@ -19,14 +21,14 @@ type Metadata struct {
 	requiredKeys []string
 }
 
-func (p *Metadata) ValidateParams() error {
-	if len(p.requiredKeys) == 0 {
+func (m *Metadata) ValidateParams() error {
+	if len(m.requiredKeys) == 0 {
 		// if metadata is used as a parameter it must have at least one required key.
 		return ErrIncorrectMetadataParamUsage
 	}
 
-	for _, key := range p.requiredKeys {
-		value, ok := p.Map[key]
+	for _, key := range m.requiredKeys {
+		value, ok := m.Map[key]
 		if !ok {
 			return fmt.Errorf("%w, missing key: %v", ErrMissingMetadata, key)
 		}
@@ -39,7 +41,22 @@ func (p *Metadata) ValidateParams() error {
 	return nil
 }
 
-func (p *Metadata) WithMetadata(metadata map[string]string, requiredKeys []string) {
-	p.Map = metadata
-	p.requiredKeys = requiredKeys
+func (m *Metadata) WithMetadata(metadata map[string]string, requiredKeys []string) {
+	m.Map = metadata
+	m.requiredKeys = requiredKeys
+}
+
+func (m *Metadata) GetCatalogVars() []catalogreplacer.CustomCatalogVariable {
+	result := make([]catalogreplacer.CustomCatalogVariable, 0)
+
+	for key, value := range m.Map {
+		result = append(result, catalogreplacer.CustomCatalogVariable{
+			Plan: catalogreplacer.SubstitutionPlan{
+				From: key,
+				To:   value,
+			},
+		})
+	}
+
+	return result
 }
diff --git a/common/substitutions/registry.go b/common/substitutions/registry.go
index 5ad80d9c..286c4d1c 100644
--- a/common/substitutions/registry.go
+++ b/common/substitutions/registry.go
@@ -14,7 +14,7 @@ type RegistryValue interface {
 
 type Registry[V RegistryValue] map[string]V
 
-func (r Registry[V]) convertStrMap() map[string]string {
+func (r Registry[V]) ConvertStrMap() map[string]string {
 	result := make(map[string]string)
 	for k, v := range r {
 		result[k] = RegistryValueToString(v)
@@ -25,7 +25,7 @@ func (r Registry[V]) convertStrMap() map[string]string {
 
 // Apply substitutions to the struct. Creates side effects.
 func (r Registry[V]) Apply(input any) error {
-	err := substituteStruct(input, r.convertStrMap())
+	err := substituteStruct(input, r.ConvertStrMap())
 	if err != nil {
 		return errors.Join(err, ErrSubstitutionFailure)
 	}
diff --git a/common/urlbuilder/url.go b/common/urlbuilder/url.go
index e4c4504e..1c5f6179 100644
--- a/common/urlbuilder/url.go
+++ b/common/urlbuilder/url.go
@@ -35,7 +35,7 @@ func New(base string, path ...string) (*URL, error) {
 	u := &URL{
 		delegate:           delegate,
 		queryParams:        values,
-		encodingExceptions: nil,
+		encodingExceptions: make(map[string]string),
 	}
 	u.AddPath(path...)
 
@@ -79,7 +79,9 @@ func (u *URL) RemoveQueryParam(name string) {
 }
 
 func (u *URL) AddEncodingExceptions(exceptions map[string]string) {
-	u.encodingExceptions = exceptions
+	for key, value := range exceptions {
+		u.encodingExceptions[key] = value
+	}
 }
 
 // ToURL relies on String method.
diff --git a/connector/connector.go b/connector/connector.go
index c5ed8473..fd963684 100644
--- a/connector/connector.go
+++ b/connector/connector.go
@@ -30,7 +30,7 @@ func NewConnector(
 	}
 
 	// Read provider info & replace catalog variables with given substitutions, if any
-	providerInfo, err := providers.ReadInfo(conn.provider, &params.Workspace)
+	providerInfo, err := providers.ReadInfo(conn.provider, params.GetCatalogVars()...)
 	if err != nil {
 		return nil, err
 	}
diff --git a/connector/params.go b/connector/params.go
index 3ac2c30f..2eb054fa 100644
--- a/connector/params.go
+++ b/connector/params.go
@@ -7,6 +7,7 @@ import (
 
 	"github.com/amp-labs/connectors/common"
 	"github.com/amp-labs/connectors/common/paramsbuilder"
+	"github.com/amp-labs/connectors/common/substitutions/catalogreplacer"
 	"github.com/amp-labs/connectors/providers"
 	"golang.org/x/oauth2"
 )
@@ -25,6 +26,7 @@ type parameters struct {
 	provider providers.Provider
 	paramsbuilder.Client
 	paramsbuilder.Workspace
+	paramsbuilder.Metadata
 }
 
 func (p parameters) ValidateParams() error {
@@ -61,3 +63,22 @@ func WithWorkspace(workspaceRef string) Option {
 		params.WithWorkspace(workspaceRef)
 	}
 }
+
+// WithMetadata sets authentication metadata expected by connector.
+func WithMetadata(metadata map[string]string) Option {
+	return func(params *parameters) {
+		params.WithMetadata(metadata, nil)
+	}
+}
+
+func (p parameters) GetCatalogVars() []catalogreplacer.CatalogVariable {
+	variables := []catalogreplacer.CatalogVariable{
+		&p.Workspace,
+	}
+
+	for _, v := range p.Metadata.GetCatalogVars() {
+		variables = append(variables, v)
+	}
+
+	return variables
+}
diff --git a/scripts/proxy/proxy.go b/scripts/proxy/proxy.go
index ef63513a..28eabdff 100644
--- a/scripts/proxy/proxy.go
+++ b/scripts/proxy/proxy.go
@@ -12,8 +12,10 @@ import (
 	"github.com/amp-labs/connectors/common/paramsbuilder"
 	"github.com/amp-labs/connectors/common/scanning"
 	"github.com/amp-labs/connectors/common/scanning/credscanning"
+	"github.com/amp-labs/connectors/common/substitutions"
 	"github.com/amp-labs/connectors/providers"
 	"github.com/amp-labs/connectors/scripts/utils/proxyserv"
+	"github.com/spyzhov/ajson"
 )
 
 // ================================
@@ -81,12 +83,12 @@ func main() {
 
 	provider := registry.MustString(credscanning.Fields.Provider.Name)
 
-	substitutions, err := registry.GetMap(SubstitutionsFieldName)
+	subs, err := registry.GetMap(SubstitutionsFieldName)
 	if err != nil {
 		slog.Warn("no substitutions, ensure that the provider info doesn't have any {{variables}}")
 	}
 
-	catalogVariables := paramsbuilder.NewCatalogVariables(substitutions)
+	catalogVariables := paramsbuilder.NewCatalogVariables(subs)
 
 	info, err := providers.ReadInfo(provider, catalogVariables...)
 	if err != nil {
@@ -101,12 +103,15 @@ func main() {
 	ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
 	defer cancel()
 
+	substitutionsMap := substitutions.Registry[*ajson.Node](subs)
+
 	proxy := createProviderProxy(ctx, info, proxyserv.Factory{
 		Provider:         provider,
 		CatalogVariables: catalogVariables,
 		Debug:            *debug,
 		Registry:         registry,
 		CredsFilePath:    DefaultCredsFile,
+		Substitutions:    substitutionsMap.ConvertStrMap(),
 	})
 
 	proxy.Start(ctx, DefaultPort)
diff --git a/scripts/utils/proxyserv/common.go b/scripts/utils/proxyserv/common.go
index 9add4356..0aa3f62f 100644
--- a/scripts/utils/proxyserv/common.go
+++ b/scripts/utils/proxyserv/common.go
@@ -22,6 +22,7 @@ type Factory struct {
 	Debug            bool
 	Registry         scanning.Registry
 	CredsFilePath    string
+	Substitutions    map[string]string
 }
 
 type ClientAuthParams struct {
diff --git a/scripts/utils/proxyserv/oauthApiKey.go b/scripts/utils/proxyserv/oauthApiKey.go
index a643bdb8..54548944 100644
--- a/scripts/utils/proxyserv/oauthApiKey.go
+++ b/scripts/utils/proxyserv/oauthApiKey.go
@@ -16,7 +16,7 @@ import (
 func (f Factory) CreateProxyAPIKey(ctx context.Context) *Proxy {
 	apiKey := getAPIKey(f.Registry)
 	providerInfo := getProviderConfig(f.Provider, f.CatalogVariables)
-	httpClient := setupAPIKeyHTTPClient(ctx, providerInfo, apiKey, f.Debug)
+	httpClient := setupAPIKeyHTTPClient(ctx, providerInfo, apiKey, f.Debug, f.Substitutions)
 	baseURL := getBaseURL(providerInfo)
 
 	return newProxy(baseURL, httpClient)
@@ -24,6 +24,7 @@ func (f Factory) CreateProxyAPIKey(ctx context.Context) *Proxy {
 
 func setupAPIKeyHTTPClient(
 	ctx context.Context, prov *providers.ProviderInfo, apiKey string, debug bool,
+	metadata map[string]string,
 ) common.AuthenticatedHTTPClient {
 	client, err := prov.NewClient(ctx, &providers.NewClientParams{
 		Debug: debug,
@@ -35,7 +36,10 @@ func setupAPIKeyHTTPClient(
 		panic(err)
 	}
 
-	cc, err := connector.NewConnector(prov.Name, connector.WithAuthenticatedClient(client))
+	cc, err := connector.NewConnector(prov.Name,
+		connector.WithAuthenticatedClient(client),
+		connector.WithMetadata(metadata),
+	)
 	if err != nil {
 		panic(err)
 	}
diff --git a/scripts/utils/proxyserv/oauthAuthCode.go b/scripts/utils/proxyserv/oauthAuthCode.go
index 828b02ac..1e19b8ac 100644
--- a/scripts/utils/proxyserv/oauthAuthCode.go
+++ b/scripts/utils/proxyserv/oauthAuthCode.go
@@ -15,7 +15,7 @@ func (f Factory) CreateProxyOAuth2AuthCode(ctx context.Context) *Proxy {
 	tokens := getTokensFromRegistry(f.CredsFilePath)
 	providerInfo := getProviderConfig(f.Provider, f.CatalogVariables)
 	cfg := configureOAuthAuthCode(params.ID, params.Secret, params.Scopes, providerInfo)
-	httpClient := setupOAuth2AuthCodeHTTPClient(ctx, providerInfo, cfg, tokens, f.Debug)
+	httpClient := setupOAuth2AuthCodeHTTPClient(ctx, providerInfo, cfg, tokens, f.Debug, f.Substitutions)
 	baseURL := getBaseURL(providerInfo)
 
 	return newProxy(baseURL, httpClient)
@@ -39,6 +39,7 @@ func configureOAuthAuthCode(
 // This helps with refreshing tokens automatically.
 func setupOAuth2AuthCodeHTTPClient(
 	ctx context.Context, prov *providers.ProviderInfo, cfg *oauth2.Config, tokens *oauth2.Token, debug bool,
+	metadata map[string]string,
 ) common.AuthenticatedHTTPClient {
 	client, err := prov.NewClient(ctx, &providers.NewClientParams{
 		Debug: debug,
@@ -51,7 +52,10 @@ func setupOAuth2AuthCodeHTTPClient(
 		panic(err)
 	}
 
-	cc, err := connector.NewConnector(prov.Name, connector.WithAuthenticatedClient(client))
+	cc, err := connector.NewConnector(prov.Name,
+		connector.WithAuthenticatedClient(client),
+		connector.WithMetadata(metadata),
+	)
 	if err != nil {
 		panic(err)
 	}
diff --git a/scripts/utils/proxyserv/oauthBasic.go b/scripts/utils/proxyserv/oauthBasic.go
index 7915e4f7..7f43b783 100644
--- a/scripts/utils/proxyserv/oauthBasic.go
+++ b/scripts/utils/proxyserv/oauthBasic.go
@@ -16,7 +16,7 @@ import (
 func (f Factory) CreateProxyBasic(ctx context.Context) *Proxy {
 	params := createBasicParams(f.Registry)
 	providerInfo := getProviderConfig(f.Provider, f.CatalogVariables)
-	httpClient := setupBasicAuthHTTPClient(ctx, providerInfo, params.User, params.Pass, f.Debug)
+	httpClient := setupBasicAuthHTTPClient(ctx, providerInfo, params.User, params.Pass, f.Debug, f.Substitutions)
 	baseURL := getBaseURL(providerInfo)
 
 	return newProxy(baseURL, httpClient)
@@ -46,6 +46,7 @@ func createBasicParams(registry scanning.Registry) *providers.BasicParams {
 
 func setupBasicAuthHTTPClient(
 	ctx context.Context, prov *providers.ProviderInfo, user, pass string, debug bool,
+	metadata map[string]string,
 ) common.AuthenticatedHTTPClient {
 	client, err := prov.NewClient(ctx, &providers.NewClientParams{
 		Debug: debug,
@@ -58,7 +59,10 @@ func setupBasicAuthHTTPClient(
 		panic(err)
 	}
 
-	cc, err := connector.NewConnector(prov.Name, connector.WithAuthenticatedClient(client))
+	cc, err := connector.NewConnector(prov.Name,
+		connector.WithAuthenticatedClient(client),
+		connector.WithMetadata(metadata),
+	)
 	if err != nil {
 		panic(err)
 	}
diff --git a/scripts/utils/proxyserv/oauthClientCred.go b/scripts/utils/proxyserv/oauthClientCred.go
index ade47e10..96c92906 100644
--- a/scripts/utils/proxyserv/oauthClientCred.go
+++ b/scripts/utils/proxyserv/oauthClientCred.go
@@ -15,7 +15,7 @@ func (f Factory) CreateProxyOAuth2ClientCreds(ctx context.Context) *Proxy {
 	params := createClientAuthParams(f.Provider, f.Registry)
 	providerInfo := getProviderConfig(f.Provider, f.CatalogVariables)
 	cfg := configureOAuthClientCredentials(params.ID, params.Secret, params.Scopes, providerInfo)
-	httpClient := setupOAuth2ClientCredentialsHTTPClient(ctx, providerInfo, cfg, f.Debug)
+	httpClient := setupOAuth2ClientCredentialsHTTPClient(ctx, providerInfo, cfg, f.Debug, f.Substitutions)
 	baseURL := getBaseURL(providerInfo)
 
 	return newProxy(baseURL, httpClient)
@@ -44,6 +44,7 @@ func configureOAuthClientCredentials(
 
 func setupOAuth2ClientCredentialsHTTPClient(
 	ctx context.Context, prov *providers.ProviderInfo, cfg *clientcredentials.Config, debug bool,
+	metadata map[string]string,
 ) common.AuthenticatedHTTPClient {
 	client, err := prov.NewClient(ctx, &providers.NewClientParams{
 		Debug: debug,
@@ -55,7 +56,10 @@ func setupOAuth2ClientCredentialsHTTPClient(
 		panic(err)
 	}
 
-	cc, err := connector.NewConnector(prov.Name, connector.WithAuthenticatedClient(client))
+	cc, err := connector.NewConnector(prov.Name,
+		connector.WithAuthenticatedClient(client),
+		connector.WithMetadata(metadata),
+	)
 	if err != nil {
 		panic(err)
 	}


Test patch:
diff --git a/common/paramsbuilder/catalogVariable_test.go b/common/paramsbuilder/catalogVariable_test.go
index bc079acf..06de9f2b 100644
--- a/common/paramsbuilder/catalogVariable_test.go
+++ b/common/paramsbuilder/catalogVariable_test.go
@@ -17,19 +17,17 @@ func TestNewCatalogVariables(t *testing.T) {
 		expected []catalogreplacer.CatalogVariable
 	}{
 		{
-			name: "Unknown substitutions are not translated to Variables",
+			name: "Unknown substitutions is translated to Variables",
 			input: substitutions.Registry[string]{
-				"insect":  "butterfly",
-				"fish":    "catfish",
-				"nothing": "",
-				"":        "something",
+				"subdomain": "www",
+			},
+			expected: []catalogreplacer.CatalogVariable{
+				createCustomVariable("subdomain", "www"),
 			},
-			expected: []catalogreplacer.CatalogVariable{},
 		},
 		{
-			name: "Only workspace Variable is captured",
+			name: "Workspace Variable is captured",
 			input: substitutions.Registry[string]{
-				"insect":    "butterfly",
 				"workspace": "office",
 			},
 			expected: []catalogreplacer.CatalogVariable{
@@ -85,3 +83,12 @@ func TestNewCatalogSubstitutionRegistry(t *testing.T) {
 		})
 	}
 }
+
+func createCustomVariable(from, to string) *catalogreplacer.CustomCatalogVariable {
+	return &catalogreplacer.CustomCatalogVariable{
+		Plan: catalogreplacer.SubstitutionPlan{
+			From: from,
+			To:   to,
+		},
+	}
+}