Repository: amp-labs/connectors
Pull request #494 (2024-06-07 17:30:10)

feature: add Salesloft Read.Since
# Background

Ampersand schedules `read` without time interval by default.

# Salesloft API

About 30% of Read APIs use `updated_at` query parameter. It is **15** out of **48** total objects as of June 10th, 2024.

# Action

Salesloft read method is now capable of returning a subset of data since specified time focusing on recent updates.


Patch:
diff --git a/salesloft/read.go b/salesloft/read.go
index 02a96e37..916dbdf9 100644
--- a/salesloft/read.go
+++ b/salesloft/read.go
@@ -3,6 +3,7 @@ package salesloft
 import (
 	"context"
 	"strconv"
+	"time"
 
 	"github.com/amp-labs/connectors/common"
 	"github.com/amp-labs/connectors/common/urlbuilder"
@@ -43,5 +44,14 @@ func (c *Connector) buildReadURL(config common.ReadParams) (*urlbuilder.URL, err
 
 	link.WithQueryParam("per_page", strconv.Itoa(DefaultPageSize))
 
+	if !config.Since.IsZero() {
+		// Documentation states ISO8601, while server accepts different formats
+		// but for consistency we are sticking to one format to be sent.
+		// For the reference any API resource that includes time data type mentions iso8601 string format.
+		// One example, say accounts is https://developers.salesloft.com/docs/api/accounts-index
+		updatedSince := config.Since.Format(time.RFC3339Nano)
+		link.WithQueryParam("updated_at[gte]", updatedSince)
+	}
+
 	return link, nil
 }
diff --git a/scripts/oauth/token.go b/scripts/oauth/token.go
index 905649ce..e219d345 100644
--- a/scripts/oauth/token.go
+++ b/scripts/oauth/token.go
@@ -200,6 +200,7 @@ func (a *OAuthApp) processCallback(writer http.ResponseWriter, request *http.Req
 func (a *OAuthApp) Run() error {
 	if a.GrantType == providers.ClientCredentials {
 		src := a.ClientCredsConfig.TokenSource(context.Background())
+
 		tok, err := src.Token()
 		if err != nil {
 			return err
diff --git a/scripts/proxy/proxy.go b/scripts/proxy/proxy.go
index 8c36137b..efeb0162 100644
--- a/scripts/proxy/proxy.go
+++ b/scripts/proxy/proxy.go
@@ -236,6 +236,7 @@ func mainBasic(ctx context.Context, provider string, substitutionsMap map[string
 func getTokensFromRegistry() *oauth2.Token {
 	accessToken := registry.MustString("AccessToken")
 	refreshToken, err := registry.GetString("RefreshToken")
+
 	if err != nil {
 		// we are working without refresh token
 		return &oauth2.Token{


Test patch:
diff --git a/salesloft/read_test.go b/salesloft/read_test.go
index 8f36650a..a0bb2ed5 100644
--- a/salesloft/read_test.go
+++ b/salesloft/read_test.go
@@ -5,9 +5,11 @@ import (
 	"errors"
 	"net/http"
 	"net/http/httptest"
+	"net/url"
 	"reflect"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/amp-labs/connectors/common"
 	"github.com/amp-labs/connectors/common/interpreter"
@@ -22,6 +24,10 @@ func TestRead(t *testing.T) { //nolint:funlen,gocognit,cyclop,maintidx
 	responseEmptyRead := mockutils.DataFromFile(t, "read-empty.json")
 	responseListPeople := mockutils.DataFromFile(t, "read-list-people.json")
 	responseListUsers := mockutils.DataFromFile(t, "read-list-users.json")
+	responseListAccounts := mockutils.DataFromFile(t, "read-list-accounts.json")
+	responseListAccountsSince := mockutils.DataFromFile(t, "read-list-accounts-since.json")
+	accountsSince, err := time.Parse(time.RFC3339Nano, "2024-06-07T10:51:20.851224-04:00")
+	mockutils.NoErrors(t, err)
 
 	tests := []struct {
 		name         string
@@ -193,6 +199,47 @@ func TestRead(t *testing.T) { //nolint:funlen,gocognit,cyclop,maintidx
 			},
 			expectedErrs: nil,
 		},
+		{
+			name:  "Successful read accounts without since query",
+			input: common.ReadParams{ObjectName: "accounts"},
+			server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				w.Header().Set("Content-Type", "application/json")
+				mockutils.RespondToMissingQueryParameters(w, r, []string{"updated_at[gte]"}, func() {
+					w.WriteHeader(http.StatusOK)
+					_, _ = w.Write(responseListAccounts)
+				})
+			})),
+			comparator: func(baseURL string, actual, expected *common.ReadResult) bool {
+				return actual.Rows == expected.Rows
+			},
+			expected: &common.ReadResult{
+				Rows: 4,
+			},
+			expectedErrs: nil,
+		},
+		{
+			name: "Successful read accounts since point in time",
+			input: common.ReadParams{
+				ObjectName: "accounts",
+				Since:      accountsSince,
+			},
+			server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				w.Header().Set("Content-Type", "application/json")
+				mockutils.RespondToQueryParameters(w, r, url.Values{
+					"updated_at[gte]": []string{"2024-06-07T10:51:20.851224-04:00"},
+				}, func() {
+					w.WriteHeader(http.StatusOK)
+					_, _ = w.Write(responseListAccountsSince)
+				})
+			})),
+			comparator: func(baseURL string, actual, expected *common.ReadResult) bool {
+				return actual.Rows == expected.Rows
+			},
+			expected: &common.ReadResult{
+				Rows: 2,
+			},
+			expectedErrs: nil,
+		},
 	}
 
 	for _, tt := range tests {
diff --git a/salesloft/test/read-list-accounts-since.json b/salesloft/test/read-list-accounts-since.json
new file mode 100644
index 00000000..4c2f3568
--- /dev/null
+++ b/salesloft/test/read-list-accounts-since.json
@@ -0,0 +1,67 @@
+{
+  "metadata": {
+    "filtering": {
+      "updated_at": {
+        "gte": "2024-06-07T14:51:20.851224Z"
+      }
+    },
+    "paging": {
+      "per_page": 25,
+      "current_page": 1,
+      "next_page": null,
+      "prev_page": null
+    },
+    "sorting": {
+      "sort_by": "updated_at",
+      "sort_direction": "DESC NULLS LAST"
+    }
+  },
+  "data": [
+    {
+      "id": 48371814,
+      "created_at": "2024-06-07T10:51:48.966580-04:00",
+      "updated_at": "2024-06-07T10:51:48.966580-04:00",
+      "name": "Reebok",
+      "domain": "https://www.reebok.eu/en-gb/",
+      "crm_object_type": "account",
+      "do_not_contact": false,
+      "custom_fields": {},
+      "user_relationships": [],
+      "tags": [],
+      "counts": {
+        "people": null
+      },
+      "owner": {
+        "_href": "https://api.salesloft.com/v2/users/49067",
+        "id": 49067
+      },
+      "creator": {
+        "_href": "https://api.salesloft.com/v2/users/49067",
+        "id": 49067
+      }
+    },
+    {
+      "id": 48371806,
+      "created_at": "2024-06-07T10:51:20.851224-04:00",
+      "updated_at": "2024-06-07T10:51:20.851224-04:00",
+      "name": "Asics",
+      "domain": "https://www.asics.com/gb/en-gb/",
+      "crm_object_type": "account",
+      "do_not_contact": false,
+      "custom_fields": {},
+      "user_relationships": [],
+      "tags": [],
+      "counts": {
+        "people": null
+      },
+      "owner": {
+        "_href": "https://api.salesloft.com/v2/users/49067",
+        "id": 49067
+      },
+      "creator": {
+        "_href": "https://api.salesloft.com/v2/users/49067",
+        "id": 49067
+      }
+    }
+  ]
+}
\ No newline at end of file
diff --git a/salesloft/test/read-list-accounts.json b/salesloft/test/read-list-accounts.json
new file mode 100644
index 00000000..9fea1223
--- /dev/null
+++ b/salesloft/test/read-list-accounts.json
@@ -0,0 +1,109 @@
+{
+  "metadata": {
+    "filtering": {},
+    "paging": {
+      "per_page": 25,
+      "current_page": 1,
+      "next_page": null,
+      "prev_page": null
+    },
+    "sorting": {
+      "sort_by": "updated_at",
+      "sort_direction": "DESC NULLS LAST"
+    }
+  },
+  "data": [
+    {
+      "id": 48371814,
+      "created_at": "2024-06-07T10:51:48.966580-04:00",
+      "updated_at": "2024-06-07T10:51:48.966580-04:00",
+      "name": "Reebok",
+      "domain": "https://www.reebok.eu/en-gb/",
+      "crm_object_type": "account",
+      "do_not_contact": false,
+      "custom_fields": {},
+      "user_relationships": [],
+      "tags": [],
+      "counts": {
+        "people": null
+      },
+      "owner": {
+        "_href": "https://api.salesloft.com/v2/users/49067",
+        "id": 49067
+      },
+      "creator": {
+        "_href": "https://api.salesloft.com/v2/users/49067",
+        "id": 49067
+      }
+    },
+    {
+      "id": 48371806,
+      "created_at": "2024-06-07T10:51:20.851224-04:00",
+      "updated_at": "2024-06-07T10:51:20.851224-04:00",
+      "name": "Asics",
+      "domain": "https://www.asics.com/gb/en-gb/",
+      "crm_object_type": "account",
+      "do_not_contact": false,
+      "custom_fields": {},
+      "user_relationships": [],
+      "tags": [],
+      "counts": {
+        "people": null
+      },
+      "owner": {
+        "_href": "https://api.salesloft.com/v2/users/49067",
+        "id": 49067
+      },
+      "creator": {
+        "_href": "https://api.salesloft.com/v2/users/49067",
+        "id": 49067
+      }
+    },
+    {
+      "id": 48371772,
+      "created_at": "2024-06-07T10:50:43.502576-04:00",
+      "updated_at": "2024-06-07T10:50:43.502576-04:00",
+      "name": "Puma",
+      "domain": "https://us.puma.com/us/en",
+      "crm_object_type": "account",
+      "do_not_contact": false,
+      "custom_fields": {},
+      "user_relationships": [],
+      "tags": [],
+      "counts": {
+        "people": null
+      },
+      "owner": {
+        "_href": "https://api.salesloft.com/v2/users/49067",
+        "id": 49067
+      },
+      "creator": {
+        "_href": "https://api.salesloft.com/v2/users/49067",
+        "id": 49067
+      }
+    },
+    {
+      "id": 48316458,
+      "created_at": "2024-06-06T12:35:41.051972-04:00",
+      "updated_at": "2024-06-06T12:35:41.051972-04:00",
+      "name": "Nike",
+      "domain": "https://www.nike.com",
+      "crm_object_type": "account",
+      "do_not_contact": false,
+      "custom_fields": {},
+      "user_relationships": [],
+      "tags": [],
+      "counts": {
+        "people": null
+      },
+      "owner": {
+        "_href": "https://api.salesloft.com/v2/users/49067",
+        "id": 49067
+      },
+      "creator": {
+        "_href": "https://api.salesloft.com/v2/users/49067",
+        "id": 49067
+      }
+    }
+  ]
+}
\ No newline at end of file
diff --git a/test/utils/mockutils/checks.go b/test/utils/mockutils/checks.go
index 0c20b837..e1b8d5a1 100644
--- a/test/utils/mockutils/checks.go
+++ b/test/utils/mockutils/checks.go
@@ -1,6 +1,9 @@
 package mockutils
 
-import "fmt"
+import (
+	"fmt"
+	"testing"
+)
 
 func DoesObjectCorrespondToString(object any, correspondent string) bool {
 	if object == nil && len(correspondent) == 0 {
@@ -9,3 +12,9 @@ func DoesObjectCorrespondToString(object any, correspondent string) bool {
 
 	return fmt.Sprintf("%v", object) == correspondent
 }
+
+func NoErrors(t *testing.T, err error) {
+	if err != nil {
+		t.Fatalf("failed to start test, %v", err)
+	}
+}
diff --git a/test/utils/mockutils/server.go b/test/utils/mockutils/server.go
index f546bcfc..9743a7e7 100644
--- a/test/utils/mockutils/server.go
+++ b/test/utils/mockutils/server.go
@@ -3,6 +3,7 @@ package mockutils
 import (
 	"fmt"
 	"net/http"
+	"net/url"
 )
 
 func RespondNoContentForMethod(w http.ResponseWriter, r *http.Request, methodName string) {
@@ -42,6 +43,36 @@ func RespondToHeader(w http.ResponseWriter, r *http.Request, header http.Header,
 	}
 }
 
+func RespondToQueryParameters(w http.ResponseWriter, r *http.Request, queries url.Values, onSuccess func()) {
+	// if some query parameters are mismatching return error code so the test will fail
+	if queryParam, ok := queryParamsAreSubset(r.URL.Query(), queries); ok {
+		// if method is matching headers
+		onSuccess()
+	} else {
+		w.WriteHeader(http.StatusBadRequest)
+		WriteBody(w, fmt.Sprintf(`{
+			"error": {
+				"code": "from test",
+				"message": "test server mismatching [%v] query parameter"
+			}}`, queryParam))
+	}
+}
+
+func RespondToMissingQueryParameters(w http.ResponseWriter, r *http.Request, missingQueries []string, onSuccess func()) {
+	// if at least one query parameter exists return error code so the test will fail
+	if queryParam, ok := queryParamsMissing(r.URL.Query(), missingQueries); ok {
+		// if method is matching headers
+		onSuccess()
+	} else {
+		w.WriteHeader(http.StatusBadRequest)
+		WriteBody(w, fmt.Sprintf(`{
+			"error": {
+				"code": "from test",
+				"message": "test server found [%v] query parameter"
+			}}`, queryParam))
+	}
+}
+
 func WriteBody(w http.ResponseWriter, body string) {
 	_, _ = w.Write([]byte(body))
 }
@@ -62,3 +93,37 @@ func headerIsSubset(superset, subset http.Header) (string, bool) {
 
 	return "", true
 }
+
+func queryParamsAreSubset(superset, subset url.Values) (string, bool) {
+	for param, values := range subset {
+		superValues := make(map[string]bool)
+
+		strings, ok := superset[param]
+		if !ok {
+			return param, false
+		}
+
+		for _, v := range strings {
+			superValues[v] = true
+		}
+
+		for _, value := range values {
+			if _, found := superValues[value]; !found {
+				return param, false
+			}
+		}
+	}
+
+	return "", true
+}
+
+func queryParamsMissing(superset url.Values, missing []string) (string, bool) {
+	for _, param := range missing {
+		if superset.Has(param) {
+			// query was found, while should be missing
+			return param, false
+		}
+	}
+
+	return "", true
+}