Repository: amp-labs/connectors
Pull request #95 (2024-03-15 08:34:30)

[ENG-758] Add metadata fields to catalog
# Context
* OAuth tokens obtained from providers can come with extra metadata that contain information about -
  * The account/workspace that the token is associated with (referred to as the `workspaceRef` here on)
  * The user that the token is associated with (referred to as the `consumerRef` here on)
  * The scopes related to the token

* For example, a salesforce token looks like - 
```
{
        "access_token": "1234",
        "refresh_token": "4321",
        "scope": "refresh_token api id full",
        "instance_url": "https://ampersand-dev-ed.my.salesforce.com",
        "id": "https://login.salesforce.com/id/00D5x0000019AWSGA1/0053x000000orG1ABI",
        "token_type": "Bearer",
        "issued_at": "1680295102278"
    }
```

* and a notion token looks like - 
```
{
        "access_token": "1234",
        "refresh_token": "4321",
        "workspace_id": "65b72915-fddc-4107-b76c-8575109ea65a",
        "owner": {
          "user": {
             "id": "some-user-id"
         }
        "token_type": "Bearer",
        "issued_at": "1680295102278"
    }
```

# Requirement
* In each oauth token obtained from a provider, if available, we are interested in capturing these fields. For Salesforce, `instance_url` is the `workspaceRef`, `id` is the the `consumerRef`, and the `scope` field is as is.
* Since the names of these payload keys can change from provider to provider, we want to store the names of these 3 fields in the catalog.
* In Notion's case, the `workspaceRef` is found under the `workspace_id` field, the `consumerRef` is found under `owner.user.id` field (note that nesting is denoted by a dot) and there is no scope field.

# Changes
This PR introduces a struct called `TokenMetadataFields` under `OAuthOpts` which stores the `WorkspaceRefField`, the `ConsumerRefField` and the `ScopesField`. 

For Salesforce, the `WorkspaceRefField` would be `instance_url`, while for Notion it would be `workspace_id`, and so on. 

This will allow us to be provider-agnostic when extracting token metadata.


Patch:
diff --git a/providers/catalog.go b/providers/catalog.go
index b839a72d..866ed756 100644
--- a/providers/catalog.go
+++ b/providers/catalog.go
@@ -10,11 +10,14 @@ const (
 	LinkedIn   Provider = "linkedIn"
 	Salesloft  Provider = "salesloft"
 	Outreach   Provider = "outreach"
+	Sellsy     Provider = "sellsy"
+	Attio      Provider = "attio"
 	Close      Provider = "close"
 	Keap       Provider = "keap"
 	Asana      Provider = "asana"
 	Dropbox    Provider = "dropbox"
 	Notion     Provider = "notion"
+	Gong       Provider = "gong"
 )
 
 // ================================================================================
@@ -31,6 +34,11 @@ var catalog = CatalogType{ // nolint:gochecknoglobals
 			TokenURL:                  "https://{{.workspace}}.my.salesforce.com/services/oauth2/token",
 			ExplicitScopesRequired:    false,
 			ExplicitWorkspaceRequired: true,
+			TokenMetadataFields: TokenMetadataFields{
+				ConsumerRefField:  "id",
+				WorkspaceRefField: "instance_url",
+				ScopesField:       "scope",
+			},
 		},
 		Support: Support{
 			BulkWrite: true,
@@ -73,6 +81,9 @@ var catalog = CatalogType{ // nolint:gochecknoglobals
 			TokenURL:                  "https://www.linkedin.com/oauth/v2/accessToken",
 			ExplicitScopesRequired:    true,
 			ExplicitWorkspaceRequired: false,
+			TokenMetadataFields: TokenMetadataFields{
+				ScopesField: "scope",
+			},
 		},
 		Support: Support{
 			BulkWrite: false,
@@ -83,7 +94,7 @@ var catalog = CatalogType{ // nolint:gochecknoglobals
 		},
 	},
 
-	// SalesLoft configuration
+	// Salesloft configuration
 	Salesloft: {
 		AuthType: Oauth2,
 		BaseURL:  "https://api.salesloft.com",
@@ -121,12 +132,49 @@ var catalog = CatalogType{ // nolint:gochecknoglobals
 		},
 	},
 
+	// Sellsy configuration
+	Sellsy: {
+		AuthType: Oauth2,
+		BaseURL:  "https://api.sellsy.com",
+		OauthOpts: OauthOpts{
+			AuthURL:                   "https://login.sellsy.com/oauth2/authorization",
+			TokenURL:                  "https://login.sellsy.com/oauth2/access-tokens",
+			ExplicitScopesRequired:    false,
+			ExplicitWorkspaceRequired: false,
+		},
+		Support: Support{
+			BulkWrite: false,
+			Proxy:     false,
+			Read:      false,
+			Subscribe: false,
+			Write:     false,
+		},
+	},
+
+	// Attio configuration
+	Attio: {
+		AuthType: Oauth2,
+		BaseURL:  "https://api.attio.com/api",
+		OauthOpts: OauthOpts{
+			AuthURL:                   "https://app.attio.com/authorize",
+			TokenURL:                  "https://app.attio.com/oauth/token",
+			ExplicitScopesRequired:    true,
+			ExplicitWorkspaceRequired: false,
+		},
+		Support: Support{
+			BulkWrite: false,
+			Proxy:     false,
+			Read:      false,
+			Subscribe: false,
+			Write:     false,
+		},
+	},
+
 	// Close configuration
 	Close: {
 		AuthType: Oauth2,
 		BaseURL:  "https://api.close.com/api",
 		OauthOpts: OauthOpts{
-
 			AuthURL:                   "https://app.close.com/oauth2/authorize",
 			TokenURL:                  "https://api.close.com/oauth2/token",
 			ExplicitScopesRequired:    false,
@@ -146,7 +194,6 @@ var catalog = CatalogType{ // nolint:gochecknoglobals
 		BaseURL:  "https://api.infusionsoft.com",
 
 		OauthOpts: OauthOpts{
-
 			AuthURL:                   "https://accounts.infusionsoft.com/app/oauth/authorize",
 			TokenURL:                  "https://api.infusionsoft.com/token",
 			ExplicitScopesRequired:    false,
@@ -208,6 +255,10 @@ var catalog = CatalogType{ // nolint:gochecknoglobals
 			TokenURL:                  "https://api.notion.com/v1/oauth/token",
 			ExplicitScopesRequired:    false,
 			ExplicitWorkspaceRequired: false,
+			TokenMetadataFields: TokenMetadataFields{
+				ConsumerRefField:  "owner.user.id",
+				WorkspaceRefField: "workspace_id",
+			},
 		},
 		Support: Support{
 			BulkWrite: false,
@@ -217,4 +268,23 @@ var catalog = CatalogType{ // nolint:gochecknoglobals
 			Write:     false,
 		},
 	},
+
+	// Gong configuration
+	Gong: {
+		AuthType: Oauth2,
+		BaseURL:  "https://{{.workspace}}.api.gong.io",
+		OauthOpts: OauthOpts{
+			AuthURL:                   "https://app.gong.io/oauth2/authorize",
+			TokenURL:                  "https://app.gong.io/oauth2/generate-customer-token",
+			ExplicitScopesRequired:    true,
+			ExplicitWorkspaceRequired: false,
+		},
+		Support: Support{
+			BulkWrite: false,
+			Proxy:     false,
+			Read:      false,
+			Subscribe: false,
+			Write:     false,
+		},
+	},
 }
diff --git a/providers/types.gen.go b/providers/types.gen.go
index 7147fe64..cdb00f36 100644
--- a/providers/types.gen.go
+++ b/providers/types.gen.go
@@ -16,10 +16,11 @@ type CatalogType map[string]ProviderInfo
 
 // OauthOpts defines model for OauthOpts.
 type OauthOpts struct {
-	AuthURL                   string `json:"authURL" validate:"required"`
-	ExplicitScopesRequired    bool   `json:"explicitScopesRequired"`
-	ExplicitWorkspaceRequired bool   `json:"explicitWorkspaceRequired"`
-	TokenURL                  string `json:"tokenURL" validate:"required"`
+	AuthURL                   string              `json:"authURL" validate:"required"`
+	ExplicitScopesRequired    bool                `json:"explicitScopesRequired"`
+	ExplicitWorkspaceRequired bool                `json:"explicitWorkspaceRequired"`
+	TokenMetadataFields       TokenMetadataFields `json:"tokenMetadataFields"`
+	TokenURL                  string              `json:"tokenURL" validate:"required"`
 }
 
 // Provider defines model for Provider.
@@ -45,3 +46,10 @@ type Support struct {
 	Subscribe bool `json:"subscribe"`
 	Write     bool `json:"write"`
 }
+
+// TokenMetadataFields defines model for TokenMetadataFields.
+type TokenMetadataFields struct {
+	ConsumerRefField  string `json:"consumerRefField,omitempty"`
+	ScopesField       string `json:"scopesField,omitempty"`
+	WorkspaceRefField string `json:"workspaceRefField,omitempty"`
+}
diff --git a/providers/types.yaml b/providers/types.yaml
index 90cb53e2..a344a1f4 100644
--- a/providers/types.yaml
+++ b/providers/types.yaml
@@ -42,6 +42,22 @@ components:
         write:
           type: boolean
 
+    TokenMetadataFields:
+      type: object
+      properties:
+        workspaceRefField:
+          type: string
+          example: account-id
+          x-go-type-skip-optional-pointer: true
+        consumerRefField:
+          type: string
+          example: user-id
+          x-go-type-skip-optional-pointer: true
+        scopesField:
+          type: string
+          example: scopes
+          x-go-type-skip-optional-pointer: true
+
     OauthOpts:
       type: object
       x-oapi-codegen-extra-tags:
@@ -51,6 +67,7 @@ components:
           - tokenURL
           - explicitScopesRequired
           - explicitWorkspaceRequired
+          - tokenMetadataFields
       properties:
         authURL:
           type: string
@@ -68,6 +85,8 @@ components:
         explicitWorkspaceRequired:
           type: boolean
           example: true
+        tokenMetadataFields:
+          $ref: '#/components/schemas/TokenMetadataFields'
 
     ProviderOpts:
         type: object


Test patch:
diff --git a/providers/catalog_test.go b/providers/catalog_test.go
index 3856ebc1..a7b0f294 100644
--- a/providers/catalog_test.go
+++ b/providers/catalog_test.go
@@ -34,6 +34,11 @@ var testCases = []struct { // nolint
 				TokenURL:                  "https://example.my.salesforce.com/services/oauth2/token",
 				ExplicitWorkspaceRequired: true,
 				ExplicitScopesRequired:    false,
+				TokenMetadataFields: TokenMetadataFields{
+					ConsumerRefField:  "id",
+					WorkspaceRefField: "instance_url",
+					ScopesField:       "scope",
+				},
 			},
 			BaseURL: "https://example.my.salesforce.com",
 			ProviderOpts: ProviderOpts{
@@ -88,6 +93,9 @@ var testCases = []struct { // nolint
 				TokenURL:                  "https://www.linkedin.com/oauth/v2/accessToken",
 				ExplicitScopesRequired:    true,
 				ExplicitWorkspaceRequired: false,
+				TokenMetadataFields: TokenMetadataFields{
+					ScopesField: "scope",
+				},
 			},
 			BaseURL: "https://api.linkedin.com",
 		},
@@ -150,6 +158,54 @@ var testCases = []struct { // nolint
 		expectedErr: nil,
 	},
 
+	{
+		provider: Sellsy,
+		expected: &ProviderInfo{
+			AuthType: Oauth2,
+			OauthOpts: OauthOpts{
+				AuthURL:                   "https://login.sellsy.com/oauth2/authorization",
+				TokenURL:                  "https://login.sellsy.com/oauth2/access-tokens",
+				ExplicitScopesRequired:    false,
+				ExplicitWorkspaceRequired: false,
+			},
+			Support: Support{
+				BulkWrite: false,
+				Proxy:     false,
+				Read:      false,
+				Subscribe: false,
+				Write:     false,
+			},
+			BaseURL: "https://api.sellsy.com",
+		},
+		expectedErr: nil,
+	},
+
+	{
+		provider:    Attio,
+		description: "Valid Attio provider config with non-existent substitutions",
+		substitutions: map[string]string{
+			"nonexistentvar": "abc",
+		},
+		expected: &ProviderInfo{
+			Support: Support{
+				Read:      false,
+				Write:     false,
+				BulkWrite: false,
+				Subscribe: false,
+				Proxy:     false,
+			},
+			AuthType: Oauth2,
+			OauthOpts: OauthOpts{
+				AuthURL:                   "https://app.attio.com/authorize",
+				TokenURL:                  "https://app.attio.com/oauth/token",
+				ExplicitScopesRequired:    true,
+				ExplicitWorkspaceRequired: false,
+			},
+			BaseURL: "https://api.attio.com/api",
+		},
+		expectedErr: nil,
+	},
+
 	{
 		provider:    Close,
 		description: "Valid Close provider config with no substitutions",
@@ -249,7 +305,7 @@ var testCases = []struct { // nolint
 				Write:     false,
 				BulkWrite: false,
 				Subscribe: false,
-				Proxy:     false,
+				Proxy:     true,
 			},
 			AuthType: Oauth2,
 			OauthOpts: OauthOpts{
@@ -257,11 +313,40 @@ var testCases = []struct { // nolint
 				TokenURL:                  "https://api.notion.com/v1/oauth/token",
 				ExplicitScopesRequired:    false,
 				ExplicitWorkspaceRequired: false,
+				TokenMetadataFields: TokenMetadataFields{
+					WorkspaceRefField: "workspace_id",
+					ConsumerRefField:  "owner.user.id",
+				},
 			},
 			BaseURL: "https://api.notion.com",
 		},
 		expectedErr: nil,
 	},
+	{
+		provider:    Gong,
+		description: "Gong provider config with valid substitutions",
+		substitutions: map[string]string{
+			"workspace": "testing",
+		},
+		expected: &ProviderInfo{
+			Support: Support{
+				Read:      false,
+				Write:     false,
+				BulkWrite: false,
+				Subscribe: false,
+				Proxy:     false,
+			},
+			AuthType: Oauth2,
+			OauthOpts: OauthOpts{
+				AuthURL:                   "https://app.gong.io/oauth2/authorize",
+				TokenURL:                  "https://app.gong.io/oauth2/generate-customer-token",
+				ExplicitWorkspaceRequired: false,
+				ExplicitScopesRequired:    true,
+			},
+			BaseURL: "https://testing.api.gong.io",
+		},
+		expectedErr: nil,
+	},
 }
 
 func TestReadInfo(t *testing.T) { // nolint