Repository: amp-labs/connectors
Pull request #615 (2024-06-27 20:48:08)

[ENG-1257] define: CatalogVariable type for substitution
# Changes

Catalog substitutions were changed from `map` to `concrete type`.

We already have connector options defined under `paramsbuilder` package.
Parameters that are of CatalogVarialbe type can be directly passed into variadic `ReadInfo` function.

Any new parameters that will participate in substitution algorithm must return SubstitutionPlan. For example `paramsbuilder.Workspace` returns SubstitutionPlan{from:"workspace", to: "DesiredValue"}.

Also, sometimes ReadInfo or ReadCatalog perform adjustment of default catalog beforehand. This is now reflected in CustomizedCatalog(... CatalogOption). This allows stacking of 2 variadic configurations of CatalogOption and CatalogVariable.

# Improvements

We could clean up providers/utils.go even further. For example clone method is very general in nature and can be used anywhere by anybody. And algorithm to `substituteStruct` can live in dedicated package.


Cobalt0s: @RajatPawar I changed the README. Let me know if it is ok.
Also, can clients implement CatalogVariable? Or is it only us that enforce allowed variables (workspace, region, etc).
The build is failing but the changes are expected to be breaking: moving from map to typed variadic methods.
RajatPawar: FYI, the build and test action is failing

Patch:
diff --git a/.github/PULL_REQUEST_TEMPLATE/new_connector.md b/.github/PULL_REQUEST_TEMPLATE/new_connector.md
index db1902ea..7807bf18 100644
--- a/.github/PULL_REQUEST_TEMPLATE/new_connector.md
+++ b/.github/PULL_REQUEST_TEMPLATE/new_connector.md
@@ -4,8 +4,8 @@ Closes #<issue-number>
 - [ ] Ran Linter
 
 ## Catalog variables
-<Does the provider's info include any template variables such as `{{workspace}}`?>
-* {{workspace}}
+<Does the provider's info include any template variables such as `{{.workspace}}`?>
+* {{.workspace}}
 
 ## Notes
 
diff --git a/README.md b/README.md
index 214bcdb8..3f5e2ea0 100644
--- a/README.md
+++ b/README.md
@@ -42,19 +42,16 @@ To add a new basic connector that allows proxying through the ampersand platform
 
 ### Initialization
 
-**Note**: If your provider requires variables to be replaced in the catalog (providers.yaml), use the `WithCatalogSubstitutions` option to replace placeholders with actual values. 
-If you don't, the provider info will be incomplete and the connector will not work. 
+**Note**: If your provider requires variables to be replaced in the catalog (providers.yaml), there is a defined list of options that will replace placeholders with actual values. 
+Error message will indicate what options are missing. 
 
-For example, a provider may use `{{workspace}}` in an option (maybe in the base URL) which needs to be replaced with an actual customer instance name. In that case, you would initialize the connector like this:
+For example, a provider may use `{{.workspace}}` in the base URL which needs to be replaced with an actual customer instance name. In that case, you would initialize the connector like this:
 
 ```go
 conn, err := connector.NewConnector(
     providers.SomeProvider,
     connector.WithClient(context.Background(), http.DefaultClient, cfg, tok),
-    
-    // WithCatalogSubstitutions allows you to replace placeholders in the catalog (providers.yaml) with actual values.
-    connector.WithCatalogSubstitutions(map[string]string{
-		"workspace": "customer-instance-ref",
-	}),
+    connector.WithWorkspace(Workspace), // {{.workspace}}
 )
 ```
+This will **automatically** replace workspace catalog variable with an actual value that you have specified in the option.
diff --git a/common/paramsbuilder/params.go b/common/paramsbuilder/params.go
index 72fec466..b729e796 100644
--- a/common/paramsbuilder/params.go
+++ b/common/paramsbuilder/params.go
@@ -79,8 +79,11 @@ func (p *Workspace) WithWorkspace(workspaceRef string) {
 	p.Name = workspaceRef
 }
 
-func (p *Workspace) Substitution() map[string]string {
-	return map[string]string{"workspace": p.Name}
+func (p *Workspace) GetSubstitutionPlan() SubstitutionPlan {
+	return SubstitutionPlan{
+		From: variableWorkspace,
+		To:   p.Name,
+	}
 }
 
 // Module params adds suffix to URL controlling API versions.
diff --git a/common/paramsbuilder/variables.go b/common/paramsbuilder/variables.go
new file mode 100644
index 00000000..49ec15ea
--- /dev/null
+++ b/common/paramsbuilder/variables.go
@@ -0,0 +1,71 @@
+package paramsbuilder
+
+import (
+	"log/slog"
+
+	"github.com/spyzhov/ajson"
+)
+
+const (
+	variableWorkspace = "workspace"
+)
+
+// CatalogVariable allows dynamically to replace variables represented with `{{VAR_NAME}}` string.
+type CatalogVariable interface {
+	GetSubstitutionPlan() SubstitutionPlan
+}
+
+// SubstitutionPlan defines an intent to replace `from` with `to`.
+type SubstitutionPlan struct {
+	From string
+	To   string
+}
+
+type RegistryValue interface {
+	string | *ajson.Node
+}
+
+type SubstitutionRegistry[V RegistryValue] map[string]V
+
+func NewCatalogSubstitutionRegistry(vars []CatalogVariable) SubstitutionRegistry[string] {
+	substitutions := make(SubstitutionRegistry[string])
+
+	for _, variable := range vars {
+		s := variable.GetSubstitutionPlan()
+		substitutions[s.From] = s.To
+	}
+
+	return substitutions
+}
+
+// NewCatalogVariables converts JSON into supported list of Catalog Variables.
+// This enforces type checking.
+func NewCatalogVariables[V RegistryValue](registry SubstitutionRegistry[V]) []CatalogVariable {
+	result := make([]CatalogVariable, 0)
+
+	for key, value := range registry {
+		name := registryValueToString(value)
+
+		switch key {
+		case variableWorkspace:
+			result = append(result, &Workspace{Name: name})
+		default:
+			slog.Info("unknown substitution SubstitutionPlan for catalog", key, value)
+		}
+	}
+
+	return result
+}
+
+func registryValueToString[V RegistryValue](value V) string {
+	var name string
+	if v, ok := any(value).(string); ok {
+		name = v
+	}
+
+	if v, ok := any(value).(*ajson.Node); ok {
+		name = v.MustString()
+	}
+
+	return name
+}
diff --git a/connector/connector.go b/connector/connector.go
index 18a4ad2b..bedd764a 100644
--- a/connector/connector.go
+++ b/connector/connector.go
@@ -34,9 +34,7 @@ func NewConnector(
 	}
 
 	// Read provider info & replace catalog variables with given substitutions, if any
-	substitution := params.Workspace.Substitution()
-
-	providerInfo, err := providers.ReadInfo(conn.provider, &substitution)
+	providerInfo, err := providers.ReadInfo(conn.provider, &params.Workspace)
 	if err != nil {
 		return nil, err
 	}
diff --git a/docusign/connector.go b/docusign/connector.go
index d7a58d91..ccedc71f 100644
--- a/docusign/connector.go
+++ b/docusign/connector.go
@@ -28,7 +28,7 @@ func NewConnector(opts ...Option) (conn *Connector, outErr error) {
 	}
 
 	// Read provider info
-	conn.ProviderInfo, err = providers.ReadInfo(providers.Docusign, nil)
+	conn.ProviderInfo, err = providers.ReadInfo(providers.Docusign)
 	if err != nil {
 		return nil, err
 	}
diff --git a/dynamicscrm/connector.go b/dynamicscrm/connector.go
index 26d0aa77..c1f1d48e 100644
--- a/dynamicscrm/connector.go
+++ b/dynamicscrm/connector.go
@@ -39,9 +39,7 @@ func NewConnector(opts ...Option) (conn *Connector, outErr error) {
 		},
 	}
 
-	providerInfo, err := providers.ReadInfo(conn.Provider(), &map[string]string{
-		"workspace": params.Workspace.Name,
-	})
+	providerInfo, err := providers.ReadInfo(conn.Provider(), &params.Workspace)
 	if err != nil {
 		return nil, err
 	}
diff --git a/examples/auth_connectors/salesforce/salesforce.go b/examples/auth_connectors/salesforce/salesforce.go
index 4a25c9a7..f3214934 100644
--- a/examples/auth_connectors/salesforce/salesforce.go
+++ b/examples/auth_connectors/salesforce/salesforce.go
@@ -7,6 +7,7 @@ import (
 	"time"
 
 	"github.com/amp-labs/connectors/common"
+	"github.com/amp-labs/connectors/common/paramsbuilder"
 	"github.com/amp-labs/connectors/connector"
 	"github.com/amp-labs/connectors/examples/utils"
 	"github.com/amp-labs/connectors/providers"
@@ -14,6 +15,7 @@ import (
 
 const (
 	// Replace these with your own values.
+
 	Workspace          = "<workspace>"
 	OAuth2ClientId     = "<client id>"
 	OAuth2ClientSecret = "<client secret>"
@@ -27,11 +29,6 @@ const (
 // it will be set to a day ago to force a refresh.
 var AccessTokenExpiry = time.Now().Add(-24 * time.Hour)
 
-// substitutions is a map of variables that can be used in the provider catalog.
-var substitutions = map[string]string{
-	"workspace": Workspace,
-}
-
 // Run this example with `go run salesforce.go`
 func main() {
 	utils.Run(salesforceAuthExample)
@@ -71,8 +68,8 @@ func salesforceAuthExample(ctx context.Context) error {
 // Create an auth connector with the Salesforce provider.
 func createAuthConnector(ctx context.Context) *connector.Connector {
 	conn, err := connector.NewConnector(providers.Salesforce,
-		connector.WithWorkspace(Workspace),
-		connector.WithAuthenticatedClient(createAuthenticatedHttpClient(ctx)))
+		connector.WithAuthenticatedClient(createAuthenticatedHttpClient(ctx)),
+		connector.WithWorkspace(Workspace))
 	if err != nil {
 		panic(err)
 	}
@@ -82,7 +79,7 @@ func createAuthConnector(ctx context.Context) *connector.Connector {
 
 // Create an OAuth2 authenticated HTTP client for Salesforce.
 func createAuthenticatedHttpClient(ctx context.Context) common.AuthenticatedHTTPClient {
-	info, err := providers.ReadInfo(providers.Salesforce, &substitutions)
+	info, err := providers.ReadInfo(providers.Salesforce, &paramsbuilder.Workspace{Name: Workspace})
 	if err != nil {
 		panic(err)
 	}
diff --git a/examples/deep_connectors/salesforce/salesforce.go b/examples/deep_connectors/salesforce/salesforce.go
index 6aa30c39..39ae9cce 100644
--- a/examples/deep_connectors/salesforce/salesforce.go
+++ b/examples/deep_connectors/salesforce/salesforce.go
@@ -7,6 +7,7 @@ import (
 
 	"github.com/amp-labs/connectors"
 	"github.com/amp-labs/connectors/common"
+	"github.com/amp-labs/connectors/common/paramsbuilder"
 	"github.com/amp-labs/connectors/examples/utils"
 	"github.com/amp-labs/connectors/providers"
 	"github.com/amp-labs/connectors/salesforce"
@@ -27,11 +28,6 @@ const (
 // it will be set to a day ago to force a refresh.
 var AccessTokenExpiry = time.Now().Add(-24 * time.Hour)
 
-// substitutions is a map of variables that can be used in the provider catalog.
-var substitutions = map[string]string{
-	"workspace": Workspace,
-}
-
 // Run this example with `go run salesforce.go`
 func main() {
 	utils.Run(salesforceDeepExample)
@@ -92,7 +88,7 @@ func createDeepConnector(ctx context.Context) *salesforce.Connector {
 
 // Create an OAuth2 authenticated HTTP client for Salesforce.
 func createAuthenticatedHttpClient(ctx context.Context) common.AuthenticatedHTTPClient {
-	info, err := providers.ReadInfo(providers.Salesforce, &substitutions)
+	info, err := providers.ReadInfo(providers.Salesforce, &paramsbuilder.Workspace{Name: Workspace})
 	if err != nil {
 		panic(err)
 	}
diff --git a/gong/connector.go b/gong/connector.go
index 5d2ddb8f..1a7afcba 100644
--- a/gong/connector.go
+++ b/gong/connector.go
@@ -35,7 +35,7 @@ func NewConnector(opts ...Option) (conn *Connector, outErr error) {
 	}
 
 	// Read provider info
-	providerInfo, err := providers.ReadInfo(conn.Provider(), nil)
+	providerInfo, err := providers.ReadInfo(conn.Provider())
 	if err != nil {
 		return nil, err
 	}
diff --git a/hubspot/connector.go b/hubspot/connector.go
index 2fed7862..ba95e9f4 100644
--- a/hubspot/connector.go
+++ b/hubspot/connector.go
@@ -25,7 +25,7 @@ func NewConnector(opts ...Option) (conn *Connector, outErr error) {
 	}
 
 	// Read provider info & replace catalog variables with given substitutions, if any
-	providerInfo, err := providers.ReadInfo(providers.Hubspot, nil)
+	providerInfo, err := providers.ReadInfo(providers.Hubspot)
 	if err != nil {
 		return nil, err
 	}
diff --git a/hubspot/modules.go b/hubspot/modules.go
index de8e5f52..b55e8af1 100644
--- a/hubspot/modules.go
+++ b/hubspot/modules.go
@@ -18,4 +18,7 @@ var ModuleEmpty = paramsbuilder.APIModule{ // nolint: gochecknoglobals
 
 // supportedModules represents currently working and supported modules within the Hubspot connector.
 // Any added module should be appended added here.
-var supportedModules = []paramsbuilder.APIModule{ModuleCRM} // nolint: gochecknoglobals
+var supportedModules = []paramsbuilder.APIModule{ // nolint: gochecknoglobals
+	ModuleEmpty,
+	ModuleCRM,
+}
diff --git a/hubspot/params.go b/hubspot/params.go
index d0ac45ee..e73d68fc 100644
--- a/hubspot/params.go
+++ b/hubspot/params.go
@@ -59,7 +59,7 @@ func WithAuthenticatedClient(client common.AuthenticatedHTTPClient) Option {
 // WithModule sets the hubspot API module to use for the connector. It's required.
 func WithModule(module paramsbuilder.APIModule) Option {
 	return func(params *parameters) {
-		params.WithModule(module, supportedModules, &ModuleCRM)
+		params.WithModule(module, supportedModules, &ModuleEmpty)
 	}
 }
 
diff --git a/intercom/connector.go b/intercom/connector.go
index 88fbeaf3..efeaf764 100644
--- a/intercom/connector.go
+++ b/intercom/connector.go
@@ -33,7 +33,7 @@ func NewConnector(opts ...Option) (conn *Connector, outErr error) {
 		return nil, err
 	}
 
-	providerInfo, err := providers.ReadInfo(providers.Intercom, nil)
+	providerInfo, err := providers.ReadInfo(providers.Intercom)
 	if err != nil {
 		return nil, err
 	}
diff --git a/outreach/connector.go b/outreach/connector.go
index dcffb98b..224f8d30 100644
--- a/outreach/connector.go
+++ b/outreach/connector.go
@@ -28,7 +28,7 @@ func NewConnector(opts ...Option) (conn *Connector, outErr error) {
 	}
 
 	// Read provider info
-	providerInfo, err := providers.ReadInfo(providers.Outreach, nil)
+	providerInfo, err := providers.ReadInfo(providers.Outreach)
 	if err != nil {
 		return nil, err
 	}
diff --git a/providers/utils.go b/providers/utils.go
index bbc08284..79d61c6e 100644
--- a/providers/utils.go
+++ b/providers/utils.go
@@ -13,15 +13,18 @@ import (
 	"text/template" // nosemgrep: go.lang.security.audit.xss.import-text-template.import-text-template
 
 	"github.com/amp-labs/connectors/common"
+	"github.com/amp-labs/connectors/common/paramsbuilder"
 	"github.com/go-playground/validator"
 	"golang.org/x/oauth2"
 	"golang.org/x/oauth2/clientcredentials"
 )
 
 var (
-	ErrProviderCatalogNotFound = errors.New("provider or provider catalog not found")
-	ErrProviderOptionNotFound  = errors.New("provider option not found")
-	ErrClient                  = errors.New("client creation failed")
+	ErrCatalogNotFound        = errors.New("catalog not found")
+	ErrProviderNotFound       = errors.New("provider not found")
+	ErrProviderOptionNotFound = errors.New("provider option not found")
+	ErrClient                 = errors.New("client creation failed")
+	ErrSubstitutionFailure    = errors.New("failed to resolve substitutions")
 )
 
 type CatalogOption func(params *catalogParams)
@@ -37,25 +40,51 @@ func WithCatalog(c CatalogType) CatalogOption {
 	}
 }
 
-func ReadCatalog(opts ...CatalogOption) (CatalogType, error) {
+type CustomCatalog struct {
+	custom CatalogType
+}
+
+// NewCustomCatalog allows to apply modifiers on the base catalog, to tweak its content.
+// Just like the default catalog it supports reading data, resolves variable substitutions.
+func NewCustomCatalog(opts ...CatalogOption) CustomCatalog {
 	params := &catalogParams{catalog: catalog}
 
 	for _, opt := range opts {
 		opt(params)
 	}
 
-	if params.catalog == nil {
-		return nil, ErrProviderCatalogNotFound
+	return CustomCatalog{custom: params.catalog}
+}
+
+func (c CustomCatalog) catalog() (CatalogType, error) {
+	if c.custom == nil {
+		// Null catalog was probably set via options.
+		// This is not allowed.
+		return nil, ErrCatalogNotFound
+	}
+
+	return c.custom, nil
+}
+
+// ReadCatalog is used to get the catalog.
+func ReadCatalog(opts ...CatalogOption) (CatalogType, error) {
+	return NewCustomCatalog(opts...).ReadCatalog()
+}
+
+func (c CustomCatalog) ReadCatalog() (CatalogType, error) {
+	catalogInstance, err := c.catalog()
+	if err != nil {
+		return nil, err
 	}
 
-	catalog, err := clone[CatalogType](params.catalog)
+	catalogCopy, err := clone[CatalogType](catalogInstance)
 	if err != nil {
 		return nil, err
 	}
 
 	// Validate the provider configuration
 	v := validator.New()
-	for provider, providerInfo := range catalog {
+	for provider, providerInfo := range catalogCopy {
 		if err := v.Struct(providerInfo); err != nil {
 			return nil, err
 		}
@@ -63,7 +92,7 @@ func ReadCatalog(opts ...CatalogOption) (CatalogType, error) {
 		providerInfo.Name = provider
 	}
 
-	return catalog, nil
+	return catalogCopy, nil
 }
 
 // SetInfo sets the information for a specific provider in the catalog.
@@ -83,20 +112,20 @@ func SetInfo(provider Provider, info ProviderInfo) {
 
 // ReadInfo reads the information from the catalog for specific provider. It also performs string substitution
 // on the values in the config that are surrounded by {{}}.
-func ReadInfo(provider Provider, substitutions *map[string]string, opts ...CatalogOption) (*ProviderInfo, error) {
-	params := &catalogParams{catalog: catalog}
-
-	for _, opt := range opts {
-		opt(params)
-	}
+// The catalog variable will be applied such that `{{.VAR_NAME}}` string will be replaced with `VAR_VALUE`.
+func ReadInfo(provider Provider, vars ...paramsbuilder.CatalogVariable) (*ProviderInfo, error) {
+	return NewCustomCatalog().ReadInfo(provider, vars...)
+}
 
-	if params.catalog == nil {
-		return nil, ErrProviderCatalogNotFound
+func (c CustomCatalog) ReadInfo(provider Provider, vars ...paramsbuilder.CatalogVariable) (*ProviderInfo, error) {
+	catalogInstance, err := c.catalog()
+	if err != nil {
+		return nil, err
 	}
 
-	pInfo, ok := params.catalog[provider]
+	pInfo, ok := catalogInstance[provider]
 	if !ok {
-		return nil, ErrProviderCatalogNotFound
+		return nil, ErrProviderNotFound
 	}
 
 	// Clone before modifying
@@ -113,21 +142,33 @@ func ReadInfo(provider Provider, substitutions *map[string]string, opts ...Catal
 		return nil, err
 	}
 
-	if substitutions == nil {
-		substitutions = &map[string]string{}
-	}
-
 	// Apply substitutions to the provider configuration values which contain variables in the form of {{var}}.
-	if err := substituteStruct(&providerInfo, substitutions); err != nil {
+	if err := providerInfo.SubstituteWith(vars); err != nil {
 		return nil, err
 	}
 
 	return &providerInfo, nil
 }
 
+func (i *ProviderInfo) SubstituteWith(vars []paramsbuilder.CatalogVariable) error {
+	// TODO catalog substitution algorithm could live outside of this package
+	return applySubstitutions(paramsbuilder.NewCatalogSubstitutionRegistry(vars), i)
+}
+
+func applySubstitutions(substitutions paramsbuilder.SubstitutionRegistry[string], input any) error {
+	// TODO paramsbuilder.SubstitutionRegistry should have this function as a method
+	// (further refactoring maybe not in paramsbuilder package)
+	err := substituteStruct(input, substitutions)
+	if err != nil {
+		return errors.Join(err, ErrSubstitutionFailure)
+	}
+
+	return nil
+}
+
 // substituteStruct performs string substitution on the fields of the input struct
 // using the substitutions map.
-func substituteStruct(input interface{}, substitutions *map[string]string) (err error) { //nolint:gocognit,cyclop,lll
+func substituteStruct(input interface{}, substitutions map[string]string) (err error) { //nolint:gocognit,cyclop,lll
 	configStruct := reflect.ValueOf(input).Elem()
 	for i := 0; i < configStruct.NumField(); i++ {
 		field := configStruct.Field(i)
@@ -180,15 +221,16 @@ func substituteStruct(input interface{}, substitutions *map[string]string) (err
 
 // substitute performs string substitution on the input string
 // using the substitutions map.
-func substitute(input string, substitutions *map[string]string) (string, error) {
-	tmpl, err := template.New("-").Parse(input)
+func substitute(input string, substitutions map[string]string) (string, error) {
+	// missing variables are not allowed, Execute will throw an error.
+	tmpl, err := template.New("-").Option("missingkey=error").Parse(input)
 	if err != nil {
 		return "", err
 	}
 
 	var result strings.Builder
 
-	err = tmpl.Execute(&result, substitutions)
+	err = tmpl.Execute(&result, &substitutions)
 	if err != nil {
 		return "", err
 	}
diff --git a/salesforce/connector.go b/salesforce/connector.go
index ae5413c3..4a063f43 100644
--- a/salesforce/connector.go
+++ b/salesforce/connector.go
@@ -45,9 +45,7 @@ func NewConnector(opts ...Option) (conn *Connector, outErr error) {
 	}
 
 	// Read provider info & replace catalog variables with given substitutions, if any
-	substitution := params.Workspace.Substitution()
-
-	providerInfo, err := providers.ReadInfo(providers.Salesforce, &substitution)
+	providerInfo, err := providers.ReadInfo(providers.Salesforce, &params.Workspace)
 	if err != nil {
 		return nil, err
 	}
diff --git a/salesloft/connector.go b/salesloft/connector.go
index 7f15ec38..230f921d 100644
--- a/salesloft/connector.go
+++ b/salesloft/connector.go
@@ -28,7 +28,7 @@ func NewConnector(opts ...Option) (conn *Connector, outErr error) {
 		return nil, err
 	}
 
-	providerInfo, err := providers.ReadInfo(providers.Salesloft, nil)
+	providerInfo, err := providers.ReadInfo(providers.Salesloft)
 	if err != nil {
 		return nil, err
 	}
diff --git a/scripts/oauth/token.go b/scripts/oauth/token.go
index 3ebd33a0..0798a940 100644
--- a/scripts/oauth/token.go
+++ b/scripts/oauth/token.go
@@ -18,6 +18,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/amp-labs/connectors/common/paramsbuilder"
 	"github.com/amp-labs/connectors/providers"
 	"github.com/amp-labs/connectors/utils"
 	"golang.org/x/oauth2"
@@ -299,15 +300,9 @@ func setup() *OAuthApp {
 		slog.Warn("no substitutions, ensure that the provider info doesn't have any {{variables}}", err)
 	}
 
-	// Cast the substitutions to a map[string]string
-	substitutionsMap := make(map[string]string)
-	for key, val := range substitutions {
-		substitutionsMap[key] = val.MustString()
-	}
-
 	provider := registry.MustString("Provider")
 
-	providerInfo, err := providers.ReadInfo(provider, &substitutionsMap)
+	providerInfo, err := providers.ReadInfo(provider, paramsbuilder.NewCatalogVariables(substitutions)...)
 	if err != nil {
 		slog.Error("failed to read provider config", "error", err)
 
diff --git a/scripts/proxy/proxy.go b/scripts/proxy/proxy.go
index d1a125e2..08fac92d 100644
--- a/scripts/proxy/proxy.go
+++ b/scripts/proxy/proxy.go
@@ -18,6 +18,7 @@ import (
 	"time"
 
 	"github.com/amp-labs/connectors/common"
+	"github.com/amp-labs/connectors/common/paramsbuilder"
 	"github.com/amp-labs/connectors/connector"
 	"github.com/amp-labs/connectors/providers"
 	"github.com/amp-labs/connectors/utils"
@@ -137,13 +138,9 @@ func main() {
 		slog.Warn("no substitutions, ensure that the provider info doesn't have any {{variables}}")
 	}
 
-	// Cast the substitutions to a map[string]string
-	substitutionsMap := make(map[string]string)
-	for key, val := range substitutions {
-		substitutionsMap[key] = val.MustString()
-	}
+	catalogVariables := paramsbuilder.NewCatalogVariables(substitutions)
 
-	info, err := providers.ReadInfo(provider, &substitutionsMap)
+	info, err := providers.ReadInfo(provider, catalogVariables...)
 	if err != nil {
 		log.Fatalf("Error reading provider info: %v", err)
 	}
@@ -164,54 +161,54 @@ func main() {
 
 		switch info.Oauth2Opts.GrantType {
 		case providers.ClientCredentials:
-			mainOAuth2ClientCreds(ctx, provider, substitutionsMap)
+			mainOAuth2ClientCreds(ctx, provider, catalogVariables)
 		case providers.AuthorizationCode:
-			mainOAuth2AuthCode(ctx, provider, substitutionsMap)
+			mainOAuth2AuthCode(ctx, provider, catalogVariables)
 		case providers.Password:
 			// de facto, password grant acts as client credentials,
 			// even so access and refresh tokens were acquired differently.
-			mainOAuth2ClientCreds(ctx, provider, substitutionsMap)
+			mainOAuth2ClientCreds(ctx, provider, catalogVariables)
 		default:
 			log.Fatalf("Unsupported OAuth2 grant type: %s", info.Oauth2Opts.GrantType)
 		}
 	case providers.ApiKey:
-		mainApiKey(ctx, provider, substitutionsMap)
+		mainApiKey(ctx, provider, catalogVariables)
 	case providers.Basic:
-		mainBasic(ctx, provider, substitutionsMap)
+		mainBasic(ctx, provider, catalogVariables)
 	default:
 		log.Fatalf("Unsupported auth type: %s", info.AuthType)
 	}
 }
 
-func mainOAuth2ClientCreds(ctx context.Context, provider string, substitutions map[string]string) {
+func mainOAuth2ClientCreds(ctx context.Context, provider string, catalogVariables []paramsbuilder.CatalogVariable) {
 	params := createClientAuthParams(provider)
 	tokens := getTokensFromRegistry()
-	proxy := buildOAuth2AuthCodeProxy(ctx, provider, params.Scopes, params.ID, params.Secret, substitutions, tokens)
+	proxy := buildOAuth2AuthCodeProxy(ctx, provider, params.Scopes, params.ID, params.Secret, catalogVariables, tokens)
 	startProxy(ctx, proxy, DefaultPort)
 }
 
-func mainOAuth2AuthCode(ctx context.Context, provider string, substitutions map[string]string) {
+func mainOAuth2AuthCode(ctx context.Context, provider string, catalogVariables []paramsbuilder.CatalogVariable) {
 	params := createClientAuthParams(provider)
 	tokens := getTokensFromRegistry()
-	proxy := buildOAuth2AuthCodeProxy(ctx, provider, params.Scopes, params.ID, params.Secret, substitutions, tokens)
+	proxy := buildOAuth2AuthCodeProxy(ctx, provider, params.Scopes, params.ID, params.Secret, catalogVariables, tokens)
 	startProxy(ctx, proxy, DefaultPort)
 }
 
-func mainApiKey(ctx context.Context, provider string, substitutions map[string]string) {
+func mainApiKey(ctx context.Context, provider string, catalogVariables []paramsbuilder.CatalogVariable) {
 	apiKey := registry.MustString("ApiKey")
 	if apiKey == "" {
 		_, _ = fmt.Fprintln(os.Stderr, "api key from registry is empty")
 		os.Exit(1)
 	}
 
-	proxy := buildApiKeyProxy(ctx, provider, substitutions, apiKey)
+	proxy := buildApiKeyProxy(ctx, provider, catalogVariables, apiKey)
 	startProxy(ctx, proxy, DefaultPort)
 }
 
-func mainBasic(ctx context.Context, provider string, substitutions map[string]string) {
+func mainBasic(ctx context.Context, provider string, catalogVariables []paramsbuilder.CatalogVariable) {
 	params := createBasicParams()
 
-	proxy := buildBasicAuthProxy(ctx, provider, substitutions, params.User, params.Pass)
+	proxy := buildBasicAuthProxy(ctx, provider, catalogVariables, params.User, params.Pass)
 	startProxy(ctx, proxy, DefaultPort)
 }
 
@@ -364,8 +361,8 @@ func startProxy(ctx context.Context, proxy *Proxy, port int) {
 	}
 }
 
-func buildOAuth2ClientCredentialsProxy(ctx context.Context, provider string, scopes []string, clientId, clientSecret string, substitutions map[string]string) *Proxy {
-	providerInfo := getProviderConfig(provider, substitutions)
+func buildOAuth2ClientCredentialsProxy(ctx context.Context, provider string, scopes []string, clientId, clientSecret string, catalogVariables []paramsbuilder.CatalogVariable) *Proxy {
+	providerInfo := getProviderConfig(provider, catalogVariables)
 	cfg := configureOAuthClientCredentials(clientId, clientSecret, scopes, providerInfo)
 	httpClient := setupOAuth2ClientCredentialsHttpClient(ctx, providerInfo, cfg)
 
@@ -377,8 +374,8 @@ func buildOAuth2ClientCredentialsProxy(ctx context.Context, provider string, sco
 	return newProxy(target, httpClient)
 }
 
-func buildApiKeyProxy(ctx context.Context, provider string, substitutions map[string]string, apiKey string) *Proxy {
-	providerInfo := getProviderConfig(provider, substitutions)
+func buildApiKeyProxy(ctx context.Context, provider string, catalogVariables []paramsbuilder.CatalogVariable, apiKey string) *Proxy {
+	providerInfo := getProviderConfig(provider, catalogVariables)
 	httpClient := setupApiKeyHttpClient(ctx, providerInfo, apiKey)
 
 	target, err := url.Parse(providerInfo.BaseURL)
@@ -389,8 +386,8 @@ func buildApiKeyProxy(ctx context.Context, provider string, substitutions map[st
 	return newProxy(target, httpClient)
 }
 
-func buildBasicAuthProxy(ctx context.Context, provider string, substitutions map[string]string, user, pass string) *Proxy {
-	providerInfo := getProviderConfig(provider, substitutions)
+func buildBasicAuthProxy(ctx context.Context, provider string, catalogVariables []paramsbuilder.CatalogVariable, user, pass string) *Proxy {
+	providerInfo := getProviderConfig(provider, catalogVariables)
 	httpClient := setupBasicAuthHttpClient(ctx, providerInfo, user, pass)
 
 	target, err := url.Parse(providerInfo.BaseURL)
@@ -401,8 +398,8 @@ func buildBasicAuthProxy(ctx context.Context, provider string, substitutions map
 	return newProxy(target, httpClient)
 }
 
-func buildOAuth2AuthCodeProxy(ctx context.Context, provider string, scopes []string, clientId, clientSecret string, substitutions map[string]string, tokens *oauth2.Token) *Proxy {
-	providerInfo := getProviderConfig(provider, substitutions)
+func buildOAuth2AuthCodeProxy(ctx context.Context, provider string, scopes []string, clientId, clientSecret string, catalogVariables []paramsbuilder.CatalogVariable, tokens *oauth2.Token) *Proxy {
+	providerInfo := getProviderConfig(provider, catalogVariables)
 	cfg := configureOAuthAuthCode(clientId, clientSecret, scopes, providerInfo)
 	httpClient := setupOAuth2AuthCodeHttpClient(ctx, providerInfo, cfg, tokens)
 
@@ -414,8 +411,8 @@ func buildOAuth2AuthCodeProxy(ctx context.Context, provider string, scopes []str
 	return newProxy(target, httpClient)
 }
 
-func getProviderConfig(provider string, substitutions map[string]string) *providers.ProviderInfo {
-	config, err := providers.ReadInfo(provider, &substitutions)
+func getProviderConfig(provider string, catalogVariables []paramsbuilder.CatalogVariable) *providers.ProviderInfo {
+	config, err := providers.ReadInfo(provider, catalogVariables...)
 	if err != nil {
 		panic(fmt.Errorf("%w: %s", err, provider))
 	}


Test patch:
diff --git a/common/paramsbuilder/variables_test.go b/common/paramsbuilder/variables_test.go
new file mode 100644
index 00000000..30fc9319
--- /dev/null
+++ b/common/paramsbuilder/variables_test.go
@@ -0,0 +1,86 @@
+package paramsbuilder
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestNewCatalogVariables(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name     string
+		input    SubstitutionRegistry[string]
+		expected []CatalogVariable
+	}{
+		{
+			name: "Unknown substitutions are not translated to Variables",
+			input: SubstitutionRegistry[string]{
+				"insect":  "butterfly",
+				"fish":    "catfish",
+				"nothing": "",
+				"":        "something",
+			},
+			expected: []CatalogVariable{},
+		},
+		{
+			name: "Only workspace Variable is captured",
+			input: SubstitutionRegistry[string]{
+				"insect":    "butterfly",
+				"workspace": "office",
+			},
+			expected: []CatalogVariable{
+				&Workspace{Name: "office"},
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		tt := tt // rebind, omit loop side effects for parallel goroutine
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			output := NewCatalogVariables(tt.input)
+			if !reflect.DeepEqual(output, tt.expected) {
+				t.Fatalf("%s: expected: (%v), got: (%v)", tt.name, tt.expected, output)
+			}
+		})
+	}
+}
+
+func TestNewCatalogSubstitutionRegistry(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name     string
+		input    []CatalogVariable
+		expected SubstitutionRegistry[string]
+	}{
+		{
+			name:     "No variables - no substitutions",
+			input:    []CatalogVariable{},
+			expected: SubstitutionRegistry[string]{},
+		},
+		{
+			name: "Workspace is translated into substitution",
+			input: []CatalogVariable{
+				&Workspace{Name: "cool organization"},
+			},
+			expected: SubstitutionRegistry[string]{
+				variableWorkspace: "cool organization",
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		tt := tt // rebind, omit loop side effects for parallel goroutine
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			output := NewCatalogSubstitutionRegistry(tt.input)
+			if !reflect.DeepEqual(output, tt.expected) {
+				t.Fatalf("%s: expected: (%v), got: (%v)", tt.name, tt.expected, output)
+			}
+		})
+	}
+}
diff --git a/providers/utils_test.go b/providers/utils_test.go
new file mode 100644
index 00000000..f1203678
--- /dev/null
+++ b/providers/utils_test.go
@@ -0,0 +1,177 @@
+// nolint:ireturn
+package providers
+
+import (
+	"errors"
+	"reflect"
+	"strings"
+	"testing"
+
+	"github.com/amp-labs/connectors/common/paramsbuilder"
+)
+
+var (
+	testCatalog CatalogType = map[string]ProviderInfo{ // nolint:gochecknoglobals
+		"test": {
+			AuthType:    Oauth2,
+			Name:        "Tester Testerson",
+			BaseURL:     "https://{{.workspace}}.test.com",
+			DisplayName: "Super Test",
+		},
+	}
+	customTestCatalogOption = []CatalogOption{ // nolint:gochecknoglobals
+		func(params *catalogParams) {
+			params.catalog = testCatalog
+		},
+	}
+)
+
+func TestNewCustomCatalog(t *testing.T) { //nolint:funlen
+	t.Parallel()
+
+	tests := []struct {
+		name         string
+		input        []CatalogOption
+		expected     CatalogType
+		expectedErrs []error
+	}{
+		{
+			name: "Removing catalog is not allowed",
+			input: []CatalogOption{
+				func(params *catalogParams) {
+					params.catalog = nil
+				},
+			},
+			expected:     nil,
+			expectedErrs: []error{ErrCatalogNotFound},
+		},
+		{
+			name:         "Custom catalog can be set",
+			input:        customTestCatalogOption,
+			expected:     testCatalog,
+			expectedErrs: nil,
+		},
+		{
+			name:         "Builtin catalog is used by default",
+			input:        []CatalogOption{},
+			expected:     catalog,
+			expectedErrs: nil,
+		},
+	}
+
+	for _, tt := range tests {
+		// nolint:varnamelen
+		tt := tt // rebind, omit loop side effects for parallel goroutine
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			output, err := NewCustomCatalog(tt.input...).catalog()
+			if err != nil {
+				if len(tt.expectedErrs) == 0 {
+					t.Fatalf("%s: expected no errors, got: (%v)", tt.name, err)
+				}
+			} else {
+				// check that missing error is what is expected
+				if len(tt.expectedErrs) != 0 {
+					t.Fatalf("%s: expected errors (%v), but got nothing", tt.name, tt.expectedErrs)
+				}
+			}
+
+			for _, expectedErr := range tt.expectedErrs {
+				if !errors.Is(err, expectedErr) && !strings.Contains(err.Error(), expectedErr.Error()) {
+					t.Fatalf("%s: expected Error: (%v), got: (%v)", tt.name, expectedErr, err)
+				}
+			}
+
+			if !reflect.DeepEqual(output, tt.expected) {
+				t.Fatalf("%s: expected: (%v), got: (%v)", tt.name, tt.expected, output)
+			}
+		})
+	}
+}
+
+func TestReadInfo(t *testing.T) { // nolint:funlen
+	t.Parallel()
+
+	type inType struct {
+		options  []CatalogOption
+		provider Provider
+		vars     []paramsbuilder.CatalogVariable
+	}
+
+	tests := []struct {
+		name         string
+		input        inType
+		expected     *ProviderInfo
+		expectedErrs []error
+	}{
+		{
+			name: "Custom catalog has missing provider",
+			input: inType{
+				options:  customTestCatalogOption,
+				provider: "nobody knows",
+				vars:     nil,
+			},
+			expected:     nil,
+			expectedErrs: []error{ErrProviderNotFound},
+		},
+		{
+			name: "Provider requires substitution",
+			input: inType{
+				options:  customTestCatalogOption,
+				provider: "test",
+				vars:     nil,
+			},
+			expected:     nil,
+			expectedErrs: []error{ErrSubstitutionFailure},
+		},
+		{
+			name: "Provider requires substitution",
+			input: inType{
+				options:  customTestCatalogOption,
+				provider: "test",
+				vars: []paramsbuilder.CatalogVariable{
+					&paramsbuilder.Workspace{Name: "europe"},
+				},
+			},
+			expected: &ProviderInfo{
+				AuthType:    Oauth2,
+				Name:        "test",
+				BaseURL:     "https://europe.test.com",
+				DisplayName: "Super Test",
+			},
+			expectedErrs: nil,
+		},
+	}
+
+	for _, tt := range tests {
+		// nolint:varnamelen
+		tt := tt // rebind, omit loop side effects for parallel goroutine
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			output, err := NewCustomCatalog(tt.input.options...).
+				ReadInfo(tt.input.provider, tt.input.vars...)
+			if err != nil {
+				if len(tt.expectedErrs) == 0 {
+					t.Fatalf("%s: expected no errors, got: (%v)", tt.name, err)
+				}
+			} else {
+				// check that missing error is what is expected
+				if len(tt.expectedErrs) != 0 {
+					t.Fatalf("%s: expected errors (%v), but got nothing", tt.name, tt.expectedErrs)
+				}
+			}
+
+			for _, expectedErr := range tt.expectedErrs {
+				if !errors.Is(err, expectedErr) && !strings.Contains(err.Error(), expectedErr.Error()) {
+					t.Fatalf("%s: expected Error: (%v), got: (%v)", tt.name, expectedErr, err)
+				}
+			}
+
+			if !reflect.DeepEqual(output, tt.expected) {
+				t.Fatalf("%s: expected: (%v), got: (%v)", tt.name, tt.expected, output)
+			}
+		})
+	}
+}