Repository: amp-labs/connectors
Pull request #45 (2024-01-04 00:58:14)

chore: Update contacts lastmodifieddate field
* The `contacts` object silently fails to return objects according to criteria even if we use `hs_lastmodifieddate`. 
* According to https://community.hubspot.com/t5/APIs-Integrations/CRM-V3-API-Search-issue-with-Contacts-when-using-Filters/m-p/324617 (and testing), the right field to use is `lastmodifieddate` for contacts and `hs_lastmodifieddate` for other objects.

Also shifts a function that gets the last ID of a result into the connector, so that the server has minimal knowledge of the internal structure of a connector's response

## Concerns
* The check for which `lastmodifiedfield` to use is a direct string comparison, so we need to be careful (https://linear.app/ampersand/issue/ENG-520/how-do-we-handle-inconsistent-plural-object-name)

codesee-maps[bot]: <details open>
<summary><b>👇 <a href="https://app.codesee.io/r/reviews?pr=45&src=https%3A%2F%2Fgithub.com%2Famp-labs%2Fconnectors">Click on the image</a> for a new way to code review</b>
</summary>

####

[![Review these changes using an interactive CodeSee Map](https://s3.us-east-2.amazonaws.com/maps.codesee.io/images/github/amp-labs/connectors/45/892b084e/41519a7997e8a1244413d091dc97d161c8799c35.svg)](https://app.codesee.io/r/reviews?pr=45&src=https%3A%2F%2Fgithub.com%2Famp-labs%2Fconnectors)

#### Legend
<img src="https://app.codesee.io/Legend_v3.png" alt="CodeSee Map legend" width="220" height="156">
</details>

Patch:
diff --git a/go.mod b/go.mod
index 765beabb..113c3857 100644
--- a/go.mod
+++ b/go.mod
@@ -3,6 +3,8 @@ module github.com/amp-labs/connectors
 go 1.21
 
 require (
+	github.com/brianvoe/gofakeit/v6 v6.26.3
+	github.com/go-playground/validator v9.31.0+incompatible
 	github.com/joho/godotenv v1.5.1
 	github.com/spyzhov/ajson v0.9.0
 	github.com/stretchr/testify v1.8.4
@@ -10,16 +12,11 @@ require (
 )
 
 require (
-	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
 	github.com/go-playground/locales v0.14.1 // indirect
 	github.com/go-playground/universal-translator v0.18.1 // indirect
-	github.com/go-playground/validator v9.31.0+incompatible // indirect
-	github.com/go-playground/validator/v10 v10.16.0 // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/leodido/go-urn v1.2.4 // indirect
-	golang.org/x/crypto v0.16.0 // indirect
-	golang.org/x/sys v0.15.0 // indirect
-	golang.org/x/text v0.14.0 // indirect
+	gopkg.in/go-playground/assert.v1 v1.2.1 // indirect
 )
 
 require (
diff --git a/go.sum b/go.sum
index bce8eb3d..f7e958e7 100644
--- a/go.sum
+++ b/go.sum
@@ -1,16 +1,14 @@
+github.com/brianvoe/gofakeit/v6 v6.26.3 h1:3ljYrjPwsUNAUFdUIr2jVg5EhKdcke/ZLop7uVg1Er8=
+github.com/brianvoe/gofakeit/v6 v6.26.3/go.mod h1:Xj58BMSnFqcn/fAQeSK+/PLtC5kSb7FJIq4JyGa8vEs=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU=
-github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA=
 github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
 github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
 github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
 github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
 github.com/go-playground/validator v9.31.0+incompatible h1:UA72EPEogEnq76ehGdEDp4Mit+3FDh548oRqwVgNsHA=
 github.com/go-playground/validator v9.31.0+incompatible/go.mod h1:yrEkQXlcI+PugkyDjY2bRrL/UBU4f3rvrgkN3V8JEig=
-github.com/go-playground/validator/v10 v10.16.0 h1:x+plE831WK4vaKHO/jpgUGsvLKIqRRkz6M78GuJAfGE=
-github.com/go-playground/validator/v10 v10.16.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
 github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
@@ -35,20 +33,14 @@ github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY=
-golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c=
 golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U=
 golang.org/x/oauth2 v0.15.0 h1:s8pnnxNVzjWyrvYdFUQq5llS1PX2zhPXmccZv99h7uQ=
 golang.org/x/oauth2 v0.15.0/go.mod h1:q48ptWNTY5XWf+JNten23lcvHpLJ0ZSxF5ttTHKVCAM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
-golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
-golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
-golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
@@ -59,6 +51,8 @@ google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs
 google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM=
+gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
diff --git a/hubspot/parse.go b/hubspot/parse.go
index e66c84e8..58beac66 100644
--- a/hubspot/parse.go
+++ b/hubspot/parse.go
@@ -146,3 +146,21 @@ func getMarshaledData(records []map[string]interface{}, fields []string) ([]comm
 
 	return data, nil
 }
+
+// GetLastResultId returns the last row's id from a result.
+func GetLastResultId(result *common.ReadResult) string {
+	numRecords := len(result.Data)
+	if numRecords == 0 {
+		return ""
+	}
+
+	// Get the last row and get the hs_object_id field's value
+	lastRow := result.Data[numRecords-1]
+
+	lastRowId, ok := lastRow.Fields[string(ObjectFieldHsObjectId)].(string)
+	if !ok {
+		return ""
+	}
+
+	return lastRowId
+}
diff --git a/hubspot/read.go b/hubspot/read.go
index f7fa4d0a..ef803e39 100644
--- a/hubspot/read.go
+++ b/hubspot/read.go
@@ -30,7 +30,7 @@ func (c *Connector) Read(ctx context.Context, config common.ReadParams) (*common
 			FilterGroups: []FilterGroup{
 				{
 					Filters: []Filter{
-						BuildLastModifiedFilterGroup(config.Since),
+						BuildLastModifiedFilterGroup(&config),
 						// Add more filters to AND them together
 					},
 					// Add more filter groups to OR them together
diff --git a/hubspot/search.go b/hubspot/search.go
index ecc81200..ae719e75 100644
--- a/hubspot/search.go
+++ b/hubspot/search.go
@@ -35,11 +35,24 @@ func (c *Connector) Search(ctx context.Context, config SearchParams) (*common.Re
 }
 
 // BuildLastModifiedFilterGroup filters records modified since the given time.
-func BuildLastModifiedFilterGroup(since time.Time) Filter {
+// If the time is zero, it returns an empty filter. For contacts, it uses the
+// lastmodifieddate field. For other objects, it uses the hs_lastmodifieddate.
+// Read more: https://community.hubspot.com/t5/APIs-Integrations/CRM-V3-API-Search-issue-with-Contacts-when-using-Filters/m-p/324617
+func BuildLastModifiedFilterGroup(params *common.ReadParams) Filter {
+	if params.Since.IsZero() {
+		return Filter{}
+	}
+
+	// Use the lastmodifieddate field for contacts, and hs_lastmodifieddate for other objects.
+	lastModifiedField := ObjectFieldHsLastModifiedDate
+	if params.ObjectName == string(ObjectTypeContact) {
+		lastModifiedField = ObjectFieldLastModifiedDate
+	}
+
 	return Filter{
-		FieldName: string(ObjectFieldLastModifiedDate),
+		FieldName: string(lastModifiedField),
 		Operator:  FilterOperatorTypeGTE,
-		Value:     since.Format(time.RFC3339),
+		Value:     params.Since.Format(time.RFC3339),
 	}
 }
 
diff --git a/hubspot/types.go b/hubspot/types.go
index be61629d..5827ca62 100644
--- a/hubspot/types.go
+++ b/hubspot/types.go
@@ -58,6 +58,13 @@ const (
 type ObjectField string
 
 const (
-	ObjectFieldHsObjectId       ObjectField = "hs_object_id"
-	ObjectFieldLastModifiedDate ObjectField = "hs_lastmodifieddate"
+	ObjectFieldHsObjectId         ObjectField = "hs_object_id"
+	ObjectFieldHsLastModifiedDate ObjectField = "hs_lastmodifieddate"
+	ObjectFieldLastModifiedDate   ObjectField = "lastmodifieddate"
+)
+
+type ObjectType string
+
+const (
+	ObjectTypeContact ObjectType = "contacts"
 )


Test patch:
diff --git a/test/hubspot/main.go b/test/hubspot/main.go
new file mode 100644
index 00000000..60312a35
--- /dev/null
+++ b/test/hubspot/main.go
@@ -0,0 +1,76 @@
+package main
+
+import (
+	"context"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"github.com/amp-labs/connectors/common"
+	"github.com/amp-labs/connectors/test/utils"
+	"github.com/brianvoe/gofakeit/v6"
+)
+
+// Contact is a basic Hubspot contact.
+type Contact struct {
+	Email     string `json:"email"`
+	Phone     string `json:"phone"`
+	Company   string `json:"company"`
+	Website   string `json:"website"`
+	Lastname  string `json:"lastname"`
+	Firstname string `json:"firstname"`
+}
+
+// Running instructions
+//
+// Step 1: prepare creds.json file
+//
+//	e.g. {
+//	  "CLIENT_ID": "<client id goes here>",
+//	  "CLIENT_SECRET": "<client secret goes here>",
+//	  "ACCESS_TOKEN": "<access token goes here>",
+//	  "REFRESH_TOKEN": "<refresh token goes here>"
+//	}
+//
+// In 1password, you can find a Hubspot creds.json file in the "Shared" vault.
+// Look for the title "Hubspot Sample OAuth Credentials".
+// The 1password item has an attached file called "creds.json" that contains the JSON.
+//
+// Step 2: run the following commands
+//
+//	$> export CREDENTIALS_FILE=creds.json
+//	$> go run test/hubspot/main.go
+func main() {
+	// Handle Ctrl-C gracefully.
+	ctx, done := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
+	defer done()
+
+	// Set up slog logging.
+	utils.SetupLogging()
+
+	// Get the Hubspot connector.
+	hsConn := utils.GetHubspotConnector(ctx, "creds.json")
+	defer utils.Close(hsConn)
+
+	// Write an artificial contact to Hubspot.
+	result, err := hsConn.Write(ctx, common.WriteParams{
+		ObjectName: "contacts",
+		ObjectId:   "",
+		ObjectData: map[string]interface{}{
+			"properties": &Contact{
+				Email:     gofakeit.Email(),
+				Phone:     gofakeit.Phone(),
+				Company:   gofakeit.Company(),
+				Website:   gofakeit.URL(),
+				Lastname:  gofakeit.LastName(),
+				Firstname: gofakeit.FirstName(),
+			},
+		},
+	})
+	if err != nil {
+		utils.Fail("error writing to hubspot", "error", err)
+	}
+
+	// Dump the result.
+	utils.DumpJSON(result, os.Stdout)
+}
diff --git a/test/utils/close.go b/test/utils/close.go
new file mode 100644
index 00000000..3729ab53
--- /dev/null
+++ b/test/utils/close.go
@@ -0,0 +1,10 @@
+package utils
+
+import "io"
+
+// Close closes the given Closer and fails if there is an error.
+func Close(c io.Closer) {
+	if err := c.Close(); err != nil {
+		Fail("error closing: %w", err)
+	}
+}
diff --git a/test/utils/creds.go b/test/utils/creds.go
new file mode 100644
index 00000000..4f3a3341
--- /dev/null
+++ b/test/utils/creds.go
@@ -0,0 +1,23 @@
+package utils
+
+import (
+	"os"
+
+	"github.com/spyzhov/ajson"
+)
+
+// Credentials returns the credentials from the creds.json file.
+func Credentials(defaultFileName string) (*ajson.Node, error) {
+	fileName := defaultFileName
+
+	if fn, ok := os.LookupEnv("CREDENTIALS_FILE"); ok {
+		fileName = fn
+	}
+
+	byteValue, err := os.ReadFile(fileName)
+	if err != nil {
+		return nil, err
+	}
+
+	return ajson.Unmarshal(byteValue)
+}
diff --git a/test/utils/dump.go b/test/utils/dump.go
new file mode 100644
index 00000000..b1778cf5
--- /dev/null
+++ b/test/utils/dump.go
@@ -0,0 +1,19 @@
+package utils
+
+import (
+	"encoding/json"
+	"io"
+)
+
+// DumpJSON dumps the given value as JSON to the given writer.
+func DumpJSON(v any, w io.Writer) {
+	bts, err := json.MarshalIndent(v, "", "  ")
+	if err != nil {
+		Fail("error marshaling to JSON: %w", "error", err)
+	}
+
+	_, err = w.Write(bts)
+	if err != nil {
+		Fail("error writing to writer: %w", "error", err)
+	}
+}
diff --git a/test/utils/fail.go b/test/utils/fail.go
new file mode 100644
index 00000000..f825cd65
--- /dev/null
+++ b/test/utils/fail.go
@@ -0,0 +1,12 @@
+package utils
+
+import (
+	"log/slog"
+	"os"
+)
+
+// Fail logs the message and exits with code 1.
+func Fail(msg string, args ...any) {
+	slog.Error(msg, args...)
+	os.Exit(1)
+}
diff --git a/test/utils/hubspot.go b/test/utils/hubspot.go
new file mode 100644
index 00000000..b069ba9c
--- /dev/null
+++ b/test/utils/hubspot.go
@@ -0,0 +1,85 @@
+package utils
+
+import (
+	"context"
+	"net/http"
+	"time"
+
+	"github.com/amp-labs/connectors"
+	"github.com/amp-labs/connectors/hubspot"
+	"github.com/spyzhov/ajson"
+	"golang.org/x/oauth2"
+)
+
+// GetHubspotConnector returns a Hubspot connector.
+func GetHubspotConnector(ctx context.Context, defaultFileName string) *hubspot.Connector {
+	creds, err := Credentials(defaultFileName)
+	if err != nil {
+		Fail("error getting credentials", "error", err)
+	}
+
+	cfg, tok := GetOAuthInfo(creds)
+
+	conn, err := connectors.Hubspot(
+		hubspot.WithClient(ctx, http.DefaultClient, cfg, tok),
+		hubspot.WithModule(hubspot.ModuleCRM))
+	if err != nil {
+		Fail("error creating hubspot connector", "error", err)
+	}
+
+	return conn
+}
+
+// GetOAuthInfo returns the OAuth2 config and token from the creds.json file.
+func GetOAuthInfo(creds *ajson.Node) (*oauth2.Config, *oauth2.Token) {
+	config := creds.MustObject()
+
+	clientId, found := config["CLIENT_ID"]
+	if !found {
+		Fail("CLIENT_ID not found in creds")
+	}
+
+	clientSecret, found := config["CLIENT_SECRET"]
+
+	if !found {
+		Fail("CLIENT_SECRET not found in creds")
+	}
+
+	// Treat this as optional, since we can get a new one if needed.
+	accessToken, _ := config["ACCESS_TOKEN"]
+
+	refreshToken, found := config["REFRESH_TOKEN"]
+
+	if !found {
+		Fail("REFRESH_TOKEN not found in creds")
+	}
+
+	cfg := &oauth2.Config{
+		ClientID:     clientId.MustString(),
+		ClientSecret: clientSecret.MustString(),
+		RedirectURL:  "http://localhost:8080/callbacks/v1/oauth",
+		Endpoint: oauth2.Endpoint{
+			AuthURL:   "https://app.hubspot.com/oauth/authorize",
+			TokenURL:  "https://api.hubapi.com/oauth/v1/token",
+			AuthStyle: oauth2.AuthStyleInParams,
+		},
+		Scopes: []string{
+			"crm.objects.contacts.read",
+			"crm.objects.contacts.write",
+			"crm.objects.deals.read",
+			"crm.objects.line_items.read",
+			"oauth",
+			"crm.objects.companies.read",
+			"tickets",
+		},
+	}
+
+	tok := &oauth2.Token{
+		AccessToken:  accessToken.MustString(),
+		RefreshToken: refreshToken.MustString(),
+		TokenType:    "bearer",
+		Expiry:       time.Now().Add(-1 * time.Hour), // just pretend it's expired already, whatever, it'll fetch a new one.
+	}
+
+	return cfg, tok
+}
diff --git a/test/utils/logging.go b/test/utils/logging.go
new file mode 100644
index 00000000..c648b0b3
--- /dev/null
+++ b/test/utils/logging.go
@@ -0,0 +1,15 @@
+package utils
+
+import (
+	"log/slog"
+	"os"
+)
+
+// SetupLogging sets up logging for the test suite.
+func SetupLogging() {
+	handler := slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{
+		Level: slog.LevelDebug,
+	})
+	logger := slog.New(handler)
+	slog.SetDefault(logger)
+}