Repository: amp-labs/connectors
Pull request #497 (2024-06-09 09:04:53)

tidy: Remove methods Gong connector
We have removed methods (to reduce redundancy in #427). This removes methods in gong.


Patch:
diff --git a/.github/workflows/generate-json-catalog.yaml b/.github/workflows/generate-json-catalog.yaml
new file mode 100644
index 00000000..d81ab12f
--- /dev/null
+++ b/.github/workflows/generate-json-catalog.yaml
@@ -0,0 +1,38 @@
+name: Publish JSON catalog
+
+on:
+  pull_request:
+    types: [closed]
+    branches:
+      - 'main'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Check for changes in providers/catalog.go
+        uses: dorny/paths-filter@v3
+        id: filter
+        with:
+          filters: |
+            catalog:
+              - 'providers/catalog.go'
+
+      - name: Run script if catalog has changed
+        if: steps.filter.outputs.catalog == 'true'
+        run: go run scripts/catalog/json.go
+        shell: bash
+
+      - name: Push changes
+        if: steps.filter.outputs.catalog == 'true'
+        run: |
+          git config --global user.email "no-reply@github.com"
+          git config --global user.name "GitHub Actions"
+          git add providers/catalog.json
+          git commit -m "Update catalog.json"
+          git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}
+          git push
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/gong/methods.go b/gong/methods.go
deleted file mode 100644
index a83052f8..00000000
--- a/gong/methods.go
+++ /dev/null
@@ -1,16 +0,0 @@
-package gong
-
-import (
-	"context"
-
-	"github.com/amp-labs/connectors/common"
-)
-
-func (c *Connector) get(ctx context.Context, url string) (*common.JSONHTTPResponse, error) {
-	res, err := c.Client.Get(ctx, url)
-	if err != nil {
-		return nil, c.HandleError(err)
-	}
-
-	return res, nil
-}
diff --git a/gong/read.go b/gong/read.go
index d4e319f0..0473439c 100644
--- a/gong/read.go
+++ b/gong/read.go
@@ -28,7 +28,7 @@ func (c *Connector) Read(ctx context.Context, config common.ReadParams) (*common
 		fields = config.Fields
 	}
 
-	res, err = c.get(ctx, fullURL)
+	res, err = c.Client.Get(ctx, fullURL)
 	if err != nil {
 		return nil, err
 	}
diff --git a/providers/catalog.go b/providers/catalog.go
index c3da0afa..993e20c5 100644
--- a/providers/catalog.go
+++ b/providers/catalog.go
@@ -47,6 +47,7 @@ const (
 	Ironclad                Provider = "ironclad"
 	IroncladDemo            Provider = "ironcladDemo"
 	IroncladEU              Provider = "ironcladEU"
+	Iterable                Provider = "iterable"
 	Keap                    Provider = "keap"
 	Klaviyo                 Provider = "klaviyo"
 	LinkedIn                Provider = "linkedIn"
@@ -1978,8 +1979,11 @@ var catalog = CatalogType{ // nolint:gochecknoglobals
 			GrantType:                 AuthorizationCode,
 			AuthURL:                   "https://github.com/login/oauth/authorize",
 			TokenURL:                  "https://github.com/login/oauth/access_token",
-			ExplicitScopesRequired:    false,
+			ExplicitScopesRequired:    true,
 			ExplicitWorkspaceRequired: false,
+			TokenMetadataFields: TokenMetadataFields{
+				ScopesField: "scope",
+			},
 		},
 		Support: Support{
 			BulkWrite: BulkWriteSupport{
@@ -2122,4 +2126,26 @@ var catalog = CatalogType{ // nolint:gochecknoglobals
 			Write:     false,
 		},
 	},
+
+	// Iterable API Key authentication
+	Iterable: {
+		AuthType: ApiKey,
+		BaseURL:  "https://api.iterable.com",
+		ApiKeyOpts: &ApiKeyOpts{
+			HeaderName: "Api-Key",
+			DocsURL:    "https://app.iterable.com/settings/apiKeys",
+		},
+		Support: Support{
+			BulkWrite: BulkWriteSupport{
+				Insert: false,
+				Update: false,
+				Upsert: false,
+				Delete: false,
+			},
+			Proxy:     false,
+			Read:      false,
+			Subscribe: false,
+			Write:     false,
+		},
+	},
 }
diff --git a/salesloft/read.go b/salesloft/read.go
index 02a96e37..916dbdf9 100644
--- a/salesloft/read.go
+++ b/salesloft/read.go
@@ -3,6 +3,7 @@ package salesloft
 import (
 	"context"
 	"strconv"
+	"time"
 
 	"github.com/amp-labs/connectors/common"
 	"github.com/amp-labs/connectors/common/urlbuilder"
@@ -43,5 +44,14 @@ func (c *Connector) buildReadURL(config common.ReadParams) (*urlbuilder.URL, err
 
 	link.WithQueryParam("per_page", strconv.Itoa(DefaultPageSize))
 
+	if !config.Since.IsZero() {
+		// Documentation states ISO8601, while server accepts different formats
+		// but for consistency we are sticking to one format to be sent.
+		// For the reference any API resource that includes time data type mentions iso8601 string format.
+		// One example, say accounts is https://developers.salesloft.com/docs/api/accounts-index
+		updatedSince := config.Since.Format(time.RFC3339Nano)
+		link.WithQueryParam("updated_at[gte]", updatedSince)
+	}
+
 	return link, nil
 }
diff --git a/scripts/catalog/json.go b/scripts/catalog/json.go
new file mode 100644
index 00000000..d584df75
--- /dev/null
+++ b/scripts/catalog/json.go
@@ -0,0 +1,30 @@
+package main
+
+import (
+	"encoding/json"
+	"log"
+	"os"
+
+	"github.com/amp-labs/connectors/providers"
+)
+
+func main() {
+	catalog, err := providers.ReadCatalog()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	bytes, err := json.Marshal(catalog)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	tempFile := "providers/catalog.json"
+
+	err = os.WriteFile(tempFile, bytes, 0o644)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	log.Printf("Catalog successfully written to: %s\n", tempFile)
+}
diff --git a/scripts/oauth/token.go b/scripts/oauth/token.go
index 905649ce..e219d345 100644
--- a/scripts/oauth/token.go
+++ b/scripts/oauth/token.go
@@ -200,6 +200,7 @@ func (a *OAuthApp) processCallback(writer http.ResponseWriter, request *http.Req
 func (a *OAuthApp) Run() error {
 	if a.GrantType == providers.ClientCredentials {
 		src := a.ClientCredsConfig.TokenSource(context.Background())
+
 		tok, err := src.Token()
 		if err != nil {
 			return err
diff --git a/scripts/proxy/proxy.go b/scripts/proxy/proxy.go
index 8c36137b..b96a08d0 100644
--- a/scripts/proxy/proxy.go
+++ b/scripts/proxy/proxy.go
@@ -190,7 +190,7 @@ func mainOAuth2ClientCreds(ctx context.Context, provider string, substitutionsMa
 
 	oauthScopes := strings.Split(scopes, ",")
 
-	validateRequiredOAuth2Flags(ctx, provider, clientId, clientSecret)
+	validateRequiredOAuth2Flags(provider, clientId, clientSecret)
 	startOAuthClientCredsProxy(ctx, provider, oauthScopes, clientId, clientSecret, substitutionsMap, DefaultPort)
 }
 
@@ -206,7 +206,7 @@ func mainOAuth2AuthCode(ctx context.Context, provider string, substitutionsMap m
 
 	oauthScopes := strings.Split(scopes, ",")
 
-	validateRequiredOAuth2Flags(ctx, provider, clientId, clientSecret)
+	validateRequiredOAuth2Flags(provider, clientId, clientSecret)
 	startOAuthAuthCodeProxy(ctx, provider, oauthScopes, clientId, clientSecret, substitutionsMap, DefaultPort, tokens)
 }
 
@@ -236,6 +236,7 @@ func mainBasic(ctx context.Context, provider string, substitutionsMap map[string
 func getTokensFromRegistry() *oauth2.Token {
 	accessToken := registry.MustString("AccessToken")
 	refreshToken, err := registry.GetString("RefreshToken")
+
 	if err != nil {
 		// we are working without refresh token
 		return &oauth2.Token{
@@ -286,7 +287,7 @@ func parseAccessTokenExpiry(expiryStr, timeFormat string) time.Time {
 	return expiry
 }
 
-func validateRequiredOAuth2Flags(ctx context.Context, provider, clientId, clientSecret string) {
+func validateRequiredOAuth2Flags(provider, clientId, clientSecret string) {
 	if provider == "" || clientId == "" || clientSecret == "" {
 		_, _ = fmt.Fprintln(os.Stderr, "Missing required flags: -provider, -client-id, -client-secret")
 
@@ -436,10 +437,13 @@ func configureOAuthClientCredentials(clientId, clientSecret string, scopes []str
 	cfg := &clientcredentials.Config{
 		ClientID:     clientId,
 		ClientSecret: clientSecret,
-		Scopes:       scopes,
 		TokenURL:     providerInfo.OauthOpts.TokenURL,
 	}
 
+	if providerInfo.OauthOpts.ExplicitScopesRequired {
+		cfg.Scopes = scopes
+	}
+
 	if providerInfo.OauthOpts.Audience != "" {
 		aud := providerInfo.OauthOpts.Audience
 		cfg.EndpointParams = url.Values{"audience": {aud}}


Test patch:
diff --git a/salesloft/read_test.go b/salesloft/read_test.go
index 8f36650a..a0bb2ed5 100644
--- a/salesloft/read_test.go
+++ b/salesloft/read_test.go
@@ -5,9 +5,11 @@ import (
 	"errors"
 	"net/http"
 	"net/http/httptest"
+	"net/url"
 	"reflect"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/amp-labs/connectors/common"
 	"github.com/amp-labs/connectors/common/interpreter"
@@ -22,6 +24,10 @@ func TestRead(t *testing.T) { //nolint:funlen,gocognit,cyclop,maintidx
 	responseEmptyRead := mockutils.DataFromFile(t, "read-empty.json")
 	responseListPeople := mockutils.DataFromFile(t, "read-list-people.json")
 	responseListUsers := mockutils.DataFromFile(t, "read-list-users.json")
+	responseListAccounts := mockutils.DataFromFile(t, "read-list-accounts.json")
+	responseListAccountsSince := mockutils.DataFromFile(t, "read-list-accounts-since.json")
+	accountsSince, err := time.Parse(time.RFC3339Nano, "2024-06-07T10:51:20.851224-04:00")
+	mockutils.NoErrors(t, err)
 
 	tests := []struct {
 		name         string
@@ -193,6 +199,47 @@ func TestRead(t *testing.T) { //nolint:funlen,gocognit,cyclop,maintidx
 			},
 			expectedErrs: nil,
 		},
+		{
+			name:  "Successful read accounts without since query",
+			input: common.ReadParams{ObjectName: "accounts"},
+			server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				w.Header().Set("Content-Type", "application/json")
+				mockutils.RespondToMissingQueryParameters(w, r, []string{"updated_at[gte]"}, func() {
+					w.WriteHeader(http.StatusOK)
+					_, _ = w.Write(responseListAccounts)
+				})
+			})),
+			comparator: func(baseURL string, actual, expected *common.ReadResult) bool {
+				return actual.Rows == expected.Rows
+			},
+			expected: &common.ReadResult{
+				Rows: 4,
+			},
+			expectedErrs: nil,
+		},
+		{
+			name: "Successful read accounts since point in time",
+			input: common.ReadParams{
+				ObjectName: "accounts",
+				Since:      accountsSince,
+			},
+			server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				w.Header().Set("Content-Type", "application/json")
+				mockutils.RespondToQueryParameters(w, r, url.Values{
+					"updated_at[gte]": []string{"2024-06-07T10:51:20.851224-04:00"},
+				}, func() {
+					w.WriteHeader(http.StatusOK)
+					_, _ = w.Write(responseListAccountsSince)
+				})
+			})),
+			comparator: func(baseURL string, actual, expected *common.ReadResult) bool {
+				return actual.Rows == expected.Rows
+			},
+			expected: &common.ReadResult{
+				Rows: 2,
+			},
+			expectedErrs: nil,
+		},
 	}
 
 	for _, tt := range tests {
diff --git a/salesloft/test/read-list-accounts-since.json b/salesloft/test/read-list-accounts-since.json
new file mode 100644
index 00000000..4c2f3568
--- /dev/null
+++ b/salesloft/test/read-list-accounts-since.json
@@ -0,0 +1,67 @@
+{
+  "metadata": {
+    "filtering": {
+      "updated_at": {
+        "gte": "2024-06-07T14:51:20.851224Z"
+      }
+    },
+    "paging": {
+      "per_page": 25,
+      "current_page": 1,
+      "next_page": null,
+      "prev_page": null
+    },
+    "sorting": {
+      "sort_by": "updated_at",
+      "sort_direction": "DESC NULLS LAST"
+    }
+  },
+  "data": [
+    {
+      "id": 48371814,
+      "created_at": "2024-06-07T10:51:48.966580-04:00",
+      "updated_at": "2024-06-07T10:51:48.966580-04:00",
+      "name": "Reebok",
+      "domain": "https://www.reebok.eu/en-gb/",
+      "crm_object_type": "account",
+      "do_not_contact": false,
+      "custom_fields": {},
+      "user_relationships": [],
+      "tags": [],
+      "counts": {
+        "people": null
+      },
+      "owner": {
+        "_href": "https://api.salesloft.com/v2/users/49067",
+        "id": 49067
+      },
+      "creator": {
+        "_href": "https://api.salesloft.com/v2/users/49067",
+        "id": 49067
+      }
+    },
+    {
+      "id": 48371806,
+      "created_at": "2024-06-07T10:51:20.851224-04:00",
+      "updated_at": "2024-06-07T10:51:20.851224-04:00",
+      "name": "Asics",
+      "domain": "https://www.asics.com/gb/en-gb/",
+      "crm_object_type": "account",
+      "do_not_contact": false,
+      "custom_fields": {},
+      "user_relationships": [],
+      "tags": [],
+      "counts": {
+        "people": null
+      },
+      "owner": {
+        "_href": "https://api.salesloft.com/v2/users/49067",
+        "id": 49067
+      },
+      "creator": {
+        "_href": "https://api.salesloft.com/v2/users/49067",
+        "id": 49067
+      }
+    }
+  ]
+}
\ No newline at end of file
diff --git a/salesloft/test/read-list-accounts.json b/salesloft/test/read-list-accounts.json
new file mode 100644
index 00000000..9fea1223
--- /dev/null
+++ b/salesloft/test/read-list-accounts.json
@@ -0,0 +1,109 @@
+{
+  "metadata": {
+    "filtering": {},
+    "paging": {
+      "per_page": 25,
+      "current_page": 1,
+      "next_page": null,
+      "prev_page": null
+    },
+    "sorting": {
+      "sort_by": "updated_at",
+      "sort_direction": "DESC NULLS LAST"
+    }
+  },
+  "data": [
+    {
+      "id": 48371814,
+      "created_at": "2024-06-07T10:51:48.966580-04:00",
+      "updated_at": "2024-06-07T10:51:48.966580-04:00",
+      "name": "Reebok",
+      "domain": "https://www.reebok.eu/en-gb/",
+      "crm_object_type": "account",
+      "do_not_contact": false,
+      "custom_fields": {},
+      "user_relationships": [],
+      "tags": [],
+      "counts": {
+        "people": null
+      },
+      "owner": {
+        "_href": "https://api.salesloft.com/v2/users/49067",
+        "id": 49067
+      },
+      "creator": {
+        "_href": "https://api.salesloft.com/v2/users/49067",
+        "id": 49067
+      }
+    },
+    {
+      "id": 48371806,
+      "created_at": "2024-06-07T10:51:20.851224-04:00",
+      "updated_at": "2024-06-07T10:51:20.851224-04:00",
+      "name": "Asics",
+      "domain": "https://www.asics.com/gb/en-gb/",
+      "crm_object_type": "account",
+      "do_not_contact": false,
+      "custom_fields": {},
+      "user_relationships": [],
+      "tags": [],
+      "counts": {
+        "people": null
+      },
+      "owner": {
+        "_href": "https://api.salesloft.com/v2/users/49067",
+        "id": 49067
+      },
+      "creator": {
+        "_href": "https://api.salesloft.com/v2/users/49067",
+        "id": 49067
+      }
+    },
+    {
+      "id": 48371772,
+      "created_at": "2024-06-07T10:50:43.502576-04:00",
+      "updated_at": "2024-06-07T10:50:43.502576-04:00",
+      "name": "Puma",
+      "domain": "https://us.puma.com/us/en",
+      "crm_object_type": "account",
+      "do_not_contact": false,
+      "custom_fields": {},
+      "user_relationships": [],
+      "tags": [],
+      "counts": {
+        "people": null
+      },
+      "owner": {
+        "_href": "https://api.salesloft.com/v2/users/49067",
+        "id": 49067
+      },
+      "creator": {
+        "_href": "https://api.salesloft.com/v2/users/49067",
+        "id": 49067
+      }
+    },
+    {
+      "id": 48316458,
+      "created_at": "2024-06-06T12:35:41.051972-04:00",
+      "updated_at": "2024-06-06T12:35:41.051972-04:00",
+      "name": "Nike",
+      "domain": "https://www.nike.com",
+      "crm_object_type": "account",
+      "do_not_contact": false,
+      "custom_fields": {},
+      "user_relationships": [],
+      "tags": [],
+      "counts": {
+        "people": null
+      },
+      "owner": {
+        "_href": "https://api.salesloft.com/v2/users/49067",
+        "id": 49067
+      },
+      "creator": {
+        "_href": "https://api.salesloft.com/v2/users/49067",
+        "id": 49067
+      }
+    }
+  ]
+}
\ No newline at end of file
diff --git a/test/utils/mockutils/checks.go b/test/utils/mockutils/checks.go
index 0c20b837..e1b8d5a1 100644
--- a/test/utils/mockutils/checks.go
+++ b/test/utils/mockutils/checks.go
@@ -1,6 +1,9 @@
 package mockutils
 
-import "fmt"
+import (
+	"fmt"
+	"testing"
+)
 
 func DoesObjectCorrespondToString(object any, correspondent string) bool {
 	if object == nil && len(correspondent) == 0 {
@@ -9,3 +12,9 @@ func DoesObjectCorrespondToString(object any, correspondent string) bool {
 
 	return fmt.Sprintf("%v", object) == correspondent
 }
+
+func NoErrors(t *testing.T, err error) {
+	if err != nil {
+		t.Fatalf("failed to start test, %v", err)
+	}
+}
diff --git a/test/utils/mockutils/server.go b/test/utils/mockutils/server.go
index f546bcfc..9743a7e7 100644
--- a/test/utils/mockutils/server.go
+++ b/test/utils/mockutils/server.go
@@ -3,6 +3,7 @@ package mockutils
 import (
 	"fmt"
 	"net/http"
+	"net/url"
 )
 
 func RespondNoContentForMethod(w http.ResponseWriter, r *http.Request, methodName string) {
@@ -42,6 +43,36 @@ func RespondToHeader(w http.ResponseWriter, r *http.Request, header http.Header,
 	}
 }
 
+func RespondToQueryParameters(w http.ResponseWriter, r *http.Request, queries url.Values, onSuccess func()) {
+	// if some query parameters are mismatching return error code so the test will fail
+	if queryParam, ok := queryParamsAreSubset(r.URL.Query(), queries); ok {
+		// if method is matching headers
+		onSuccess()
+	} else {
+		w.WriteHeader(http.StatusBadRequest)
+		WriteBody(w, fmt.Sprintf(`{
+			"error": {
+				"code": "from test",
+				"message": "test server mismatching [%v] query parameter"
+			}}`, queryParam))
+	}
+}
+
+func RespondToMissingQueryParameters(w http.ResponseWriter, r *http.Request, missingQueries []string, onSuccess func()) {
+	// if at least one query parameter exists return error code so the test will fail
+	if queryParam, ok := queryParamsMissing(r.URL.Query(), missingQueries); ok {
+		// if method is matching headers
+		onSuccess()
+	} else {
+		w.WriteHeader(http.StatusBadRequest)
+		WriteBody(w, fmt.Sprintf(`{
+			"error": {
+				"code": "from test",
+				"message": "test server found [%v] query parameter"
+			}}`, queryParam))
+	}
+}
+
 func WriteBody(w http.ResponseWriter, body string) {
 	_, _ = w.Write([]byte(body))
 }
@@ -62,3 +93,37 @@ func headerIsSubset(superset, subset http.Header) (string, bool) {
 
 	return "", true
 }
+
+func queryParamsAreSubset(superset, subset url.Values) (string, bool) {
+	for param, values := range subset {
+		superValues := make(map[string]bool)
+
+		strings, ok := superset[param]
+		if !ok {
+			return param, false
+		}
+
+		for _, v := range strings {
+			superValues[v] = true
+		}
+
+		for _, value := range values {
+			if _, found := superValues[value]; !found {
+				return param, false
+			}
+		}
+	}
+
+	return "", true
+}
+
+func queryParamsMissing(superset url.Values, missing []string) (string, bool) {
+	for _, param := range missing {
+		if superset.Has(param) {
+			// query was found, while should be missing
+			return param, false
+		}
+	}
+
+	return "", true
+}