Repository: amp-labs/connectors
Pull request #1587 (2025-04-23 00:35:54)

[CON-1666] feat(insightly): Custom objects & fields
# Description
Custom fields on custom object or core object can be requested during the connector read operation.

The fields are flattened and moved to the fields. The raw response is not modified. But custom field querying is treated as equal to the core fields.

# Tests
## Contacts
![image](https://github.com/user-attachments/assets/eba8b1d0-82a6-4bb9-bb2a-3c81fcb15149)

## Fruits the Custom Object
![image](https://github.com/user-attachments/assets/4d3c3512-31c5-458f-b4ad-eeaa1740e8ff)


Cobalt0s: > [!WARNING]
> <b>This pull request is not mergeable via GitHub because a downstack PR is open. Once all requirements are satisfied, merge this PR as a stack <a href="https://app.graphite.dev/github/pr/amp-labs/connectors/1587?utm_source=stack-comment-downstack-mergeability-warning" >on Graphite</a>.</b>
> <a href="https://graphite.dev/docs/merge-pull-requests">Learn more</a>

* **#1587** <a href="https://app.graphite.dev/github/pr/amp-labs/connectors/1587?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a> 👈 <a href="https://app.graphite.dev/github/pr/amp-labs/connectors/1587?utm_source=stack-comment-view-in-graphite" target="_blank">(View in Graphite)</a>
* **#1579** <a href="https://app.graphite.dev/github/pr/amp-labs/connectors/1579?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a>
* **#1578** <a href="https://app.graphite.dev/github/pr/amp-labs/connectors/1578?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a>
* **#1577** <a href="https://app.graphite.dev/github/pr/amp-labs/connectors/1577?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a>
* **#1576** <a href="https://app.graphite.dev/github/pr/amp-labs/connectors/1576?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a>
* **#1575** <a href="https://app.graphite.dev/github/pr/amp-labs/connectors/1575?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a>
* **#1560** <a href="https://app.graphite.dev/github/pr/amp-labs/connectors/1560?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a>
* `main`


<h2></h2>

This stack of pull requests is managed by <a href="https://graphite.dev?utm-source=stack-comment"><b>Graphite</b></a>. Learn more about <a href="https://stacking.dev/?utm_source=stack-comment">stacking</a>.
<!-- Current dependencies on/for this PR: -->

Patch:
diff --git a/providers/insightly/custom.go b/providers/insightly/custom.go
new file mode 100644
index 00000000..ededa786
--- /dev/null
+++ b/providers/insightly/custom.go
@@ -0,0 +1,75 @@
+package insightly
+
+import (
+	"github.com/amp-labs/connectors/internal/jsonquery"
+	"github.com/spyzhov/ajson"
+)
+
+// Any object returned from READ operation may have a CUSTOMFIELDS array property.
+// This struct is it's schema representation.
+// nolint:tagliatelle
+type readCustomField struct {
+	Name  string `json:"FIELD_NAME"`
+	Value any    `json:"FIELD_VALUE"`
+}
+
+/*
+Provider object response:
+
+	  {
+		"RECORD_ID": 54840682,
+		"RECORD_NAME": "Banana",
+		"CUSTOMFIELDS": [
+		  {
+			"FIELD_NAME": "Color__c",
+			"FIELD_VALUE": null,
+			"CUSTOM_FIELD_ID": "Color__c"
+		  },
+		  {
+			"FIELD_NAME": "Weight__c",
+			"FIELD_VALUE": 3.2,
+			"CUSTOM_FIELD_ID": "Weight__c"
+		  }
+		]
+	  }
+
+Read fields:
+
+	  {
+		"RECORD_ID": 54840682,
+		"RECORD_NAME": "Banana",
+		"Color__c": null,
+		"Weight__c": 3.2
+		"CUSTOMFIELDS": [...]
+	  }
+*/
+func flattenCustomFields(node *ajson.Node) (map[string]any, error) {
+	root, err := jsonquery.Convertor.ObjectToMap(node)
+	if err != nil {
+		return nil, err
+	}
+
+	customFieldsArray, err := jsonquery.New(node).ArrayOptional("CUSTOMFIELDS")
+	if err != nil {
+		return nil, err
+	}
+
+	if len(customFieldsArray) == 0 {
+		// Nothing to move.
+		return root, nil
+	}
+
+	// Custom fields are converted from the object representing the property to
+	// key:value pair and moved to the top node level.
+	for _, customFieldObject := range customFieldsArray {
+		field, err := jsonquery.ParseNode[readCustomField](customFieldObject)
+		if err != nil {
+			return nil, err
+		}
+
+		root[field.Name] = field.Value
+	}
+
+	// Root level has adopted fields from custom fields.
+	return root, nil
+}
diff --git a/providers/insightly/handlers.go b/providers/insightly/handlers.go
index 200a3a05..5f474e0f 100644
--- a/providers/insightly/handlers.go
+++ b/providers/insightly/handlers.go
@@ -54,8 +54,6 @@ func (c *Connector) parseReadResponse(
 	request *http.Request,
 	response *common.JSONHTTPResponse,
 ) (*common.ReadResult, error) {
-	fieldName := metadata.Schemas.LookupArrayFieldName(c.Module(), params.ObjectName)
-
 	url, err := urlbuilder.FromRawURL(request.URL)
 	if err != nil {
 		return nil, err
@@ -63,13 +61,21 @@ func (c *Connector) parseReadResponse(
 
 	return common.ParseResult(
 		response,
-		common.ExtractRecordsFromPath(fieldName),
+		makeGetRecords(c.Module(), params.ObjectName),
 		nextRecordsURL(url),
-		common.GetMarshaledData,
+		common.MakeMarshaledDataFunc(flattenCustomFields),
 		params.Fields,
 	)
 }
 
+func makeGetRecords(moduleID common.ModuleID, objectName string) common.NodeRecordsFunc {
+	return func(node *ajson.Node) ([]*ajson.Node, error) {
+		responseFieldName := metadata.Schemas.LookupArrayFieldName(moduleID, objectName)
+
+		return jsonquery.New(node).ArrayOptional(responseFieldName)
+	}
+}
+
 func nextRecordsURL(url *urlbuilder.URL) func(*ajson.Node) (string, error) {
 	return func(node *ajson.Node) (string, error) {
 		skipStr, ok := url.GetFirstQueryParam("skip")
diff --git a/providers/insightly/url.go b/providers/insightly/url.go
index 8a995075..e6538a21 100644
--- a/providers/insightly/url.go
+++ b/providers/insightly/url.go
@@ -12,7 +12,9 @@ const apiVersion = "v3.1"
 func (c *Connector) getReadURL(objectName string) (*urlbuilder.URL, error) {
 	path, err := metadata.Schemas.LookupURLPath(common.ModuleRoot, objectName)
 	if err != nil {
-		return nil, err
+		// It is possible that an object is custom.
+		// Custom objects support Search which allows incremental reading.
+		path = objectName + "/Search"
 	}
 
 	return urlbuilder.New(c.ProviderInfo().BaseURL, apiVersion, path)


Test patch:
diff --git a/providers/insightly/read_test.go b/providers/insightly/read_test.go
index b9c2e3ad..6fa93d19 100644
--- a/providers/insightly/read_test.go
+++ b/providers/insightly/read_test.go
@@ -20,6 +20,8 @@ func TestRead(t *testing.T) { //nolint:funlen,gocognit,cyclop,maintidx
 	responseLeadsFirstPage := testutils.DataFromFile(t, "read/leads/1-first-page.json")
 	responseLeadsSecondPage := testutils.DataFromFile(t, "read/leads/2-second-page.json")
 	responseLeadsLastPage := testutils.DataFromFile(t, "read/leads/3-last-page.json")
+	responseContacts := testutils.DataFromFile(t, "read/contacts/list.json")
+	responseFruits := testutils.DataFromFile(t, "read/fruits-custom-object/list.json")
 
 	tests := []testroutines.Read{
 		{
@@ -180,6 +182,122 @@ func TestRead(t *testing.T) { //nolint:funlen,gocognit,cyclop,maintidx
 			},
 			ExpectedErrs: nil,
 		},
+		{
+			Name: "Contacts with custom fields",
+			Input: common.ReadParams{
+				ObjectName: "Contacts",
+				Fields:     connectors.Fields("CONTACT_ID", "EMAIL_ADDRESS", "Hobby__c", "Interests__c"),
+			},
+			Server: mockserver.Conditional{
+				Setup: mockserver.ContentJSON(),
+				If:    mockcond.PathSuffix("/v3.1/Contacts/Search"),
+				Then:  mockserver.Response(http.StatusOK, responseContacts),
+			}.Server(),
+			Comparator: testroutines.ComparatorSubsetRead,
+			Expected: &common.ReadResult{
+				Rows: 1,
+				Data: []common.ReadResultRow{{
+					Fields: map[string]any{
+						"email_address": "pamela@mail.com",
+						"contact_id":    float64(366638973),
+						"hobby__c":      "Skiing",
+						"interests__c":  "Music;Sports;Travel",
+					},
+					Raw: map[string]any{
+						"FIRST_NAME": "Pamela",
+						"CUSTOMFIELDS": []any{
+							map[string]any{
+								"FIELD_NAME":      "Hobby__c",
+								"FIELD_VALUE":     "Skiing",
+								"CUSTOM_FIELD_ID": "Hobby__c",
+							},
+							map[string]any{
+								"FIELD_NAME":      "Interests__c",
+								"FIELD_VALUE":     "Music;Sports;Travel",
+								"CUSTOM_FIELD_ID": "Interests__c",
+							},
+							map[string]any{
+								"FIELD_NAME":      "Newsletter_Subscription__c",
+								"FIELD_VALUE":     true,
+								"CUSTOM_FIELD_ID": "Newsletter_Subscription__c",
+							},
+							map[string]any{
+								"FIELD_NAME":      "Preferred_Contact_Method__c",
+								"FIELD_VALUE":     "SMS",
+								"CUSTOM_FIELD_ID": "Preferred_Contact_Method__c",
+							},
+						},
+					},
+				}},
+				NextPage: testroutines.URLTestServer +
+					"/v3.1/Contacts/Search?skip=500&top=500",
+				Done: false,
+			},
+			ExpectedErrs: nil,
+		},
+		{
+			Name: "Custom object fruits with custom fields",
+			Input: common.ReadParams{
+				ObjectName: "Fruit__c",
+				Fields:     connectors.Fields("RECORD_NAME", "RECORD_ID", "Color__c", "Weight__c"),
+			},
+			Server: mockserver.Conditional{
+				Setup: mockserver.ContentJSON(),
+				If:    mockcond.PathSuffix("/v3.1/Fruit__c/Search"),
+				Then:  mockserver.Response(http.StatusOK, responseFruits),
+			}.Server(),
+			Comparator: testroutines.ComparatorSubsetRead,
+			Expected: &common.ReadResult{
+				Rows: 2,
+				Data: []common.ReadResultRow{{
+					Fields: map[string]any{
+						"record_id":   float64(54840676),
+						"record_name": "Apple",
+						"color__c":    "Green",
+						"weight__c":   float64(12),
+					},
+					Raw: map[string]any{
+						"CUSTOMFIELDS": []any{
+							map[string]any{
+								"FIELD_NAME":      "Color__c",
+								"FIELD_VALUE":     "Green",
+								"CUSTOM_FIELD_ID": "Color__c",
+							},
+							map[string]any{
+								"FIELD_NAME":      "Weight__c",
+								"FIELD_VALUE":     float64(12),
+								"CUSTOM_FIELD_ID": "Weight__c",
+							},
+						},
+					},
+				}, {
+					Fields: map[string]any{
+						"record_id":   float64(54840682),
+						"record_name": "Banana",
+						"color__c":    nil,
+						"weight__c":   3.2,
+					},
+					Raw: map[string]any{
+						"CUSTOMFIELDS": []any{
+							map[string]any{
+								"FIELD_NAME":      "Color__c",
+								"FIELD_VALUE":     nil,
+								"CUSTOM_FIELD_ID": "Color__c",
+							},
+							map[string]any{
+								"FIELD_NAME":      "Weight__c",
+								"FIELD_VALUE":     3.2,
+								"CUSTOM_FIELD_ID": "Weight__c",
+							},
+						},
+					},
+				}},
+				NextPage: testroutines.URLTestServer +
+					"/v3.1/Fruit__c/Search?skip=500&top=500",
+				Done: false,
+			},
+			ExpectedErrs: nil,
+		},
 	}
 
 	for _, tt := range tests {
diff --git a/providers/insightly/test/read/contacts/list.json b/providers/insightly/test/read/contacts/list.json
new file mode 100644
index 00000000..44f033ef
--- /dev/null
+++ b/providers/insightly/test/read/contacts/list.json
@@ -0,0 +1,68 @@
+[
+  {
+    "CONTACT_ID": 366638973,
+    "SALUTATION": "Ms",
+    "FIRST_NAME": "Pamela",
+    "LAST_NAME": "Huber",
+    "IMAGE_URL": null,
+    "BACKGROUND": null,
+    "VISIBLE_TO": "EVERYONE",
+    "OWNER_USER_ID": 2240727,
+    "VISIBLE_TEAM_ID": null,
+    "DATE_CREATED_UTC": "2025-04-19 00:49:32",
+    "DATE_UPDATED_UTC": "2025-04-22 20:03:25",
+    "SOCIAL_LINKEDIN": null,
+    "SOCIAL_FACEBOOK": null,
+    "SOCIAL_TWITTER": null,
+    "DATE_OF_BIRTH": null,
+    "PHONE": null,
+    "PHONE_HOME": null,
+    "PHONE_MOBILE": null,
+    "PHONE_OTHER": null,
+    "PHONE_ASSISTANT": null,
+    "PHONE_FAX": null,
+    "EMAIL_ADDRESS": "pamela@mail.com",
+    "ASSISTANT_NAME": null,
+    "ADDRESS_MAIL_STREET": null,
+    "ADDRESS_MAIL_CITY": null,
+    "ADDRESS_MAIL_STATE": null,
+    "ADDRESS_MAIL_POSTCODE": null,
+    "ADDRESS_MAIL_COUNTRY": null,
+    "ADDRESS_OTHER_STREET": null,
+    "ADDRESS_OTHER_CITY": null,
+    "ADDRESS_OTHER_STATE": null,
+    "ADDRESS_OTHER_POSTCODE": null,
+    "ADDRESS_OTHER_COUNTRY": null,
+    "LAST_ACTIVITY_DATE_UTC": null,
+    "NEXT_ACTIVITY_DATE_UTC": null,
+    "CREATED_USER_ID": 2240727,
+    "ORGANISATION_ID": null,
+    "TITLE": null,
+    "EMAIL_OPTED_OUT": false,
+    "CUSTOMFIELDS": [
+      {
+        "FIELD_NAME": "Hobby__c",
+        "FIELD_VALUE": "Skiing",
+        "CUSTOM_FIELD_ID": "Hobby__c"
+      },
+      {
+        "FIELD_NAME": "Interests__c",
+        "FIELD_VALUE": "Music;Sports;Travel",
+        "CUSTOM_FIELD_ID": "Interests__c"
+      },
+      {
+        "FIELD_NAME": "Newsletter_Subscription__c",
+        "FIELD_VALUE": true,
+        "CUSTOM_FIELD_ID": "Newsletter_Subscription__c"
+      },
+      {
+        "FIELD_NAME": "Preferred_Contact_Method__c",
+        "FIELD_VALUE": "SMS",
+        "CUSTOM_FIELD_ID": "Preferred_Contact_Method__c"
+      }
+    ],
+    "TAGS": [],
+    "DATES": [],
+    "LINKS": []
+  }
+]
diff --git a/providers/insightly/test/read/fruits-custom-object/list.json b/providers/insightly/test/read/fruits-custom-object/list.json
new file mode 100644
index 00000000..f88e960d
--- /dev/null
+++ b/providers/insightly/test/read/fruits-custom-object/list.json
@@ -0,0 +1,46 @@
+[
+  {
+    "RECORD_ID": 54840676,
+    "RECORD_NAME": "Apple",
+    "OWNER_USER_ID": 2240727,
+    "DATE_CREATED_UTC": "2025-04-22 19:29:43",
+    "DATE_UPDATED_UTC": "2025-04-22 19:34:12",
+    "CREATED_USER_ID": 2240727,
+    "VISIBLE_TO": "EVERYONE",
+    "VISIBLE_TEAM_ID": null,
+    "CUSTOMFIELDS": [
+      {
+        "FIELD_NAME": "Color__c",
+        "FIELD_VALUE": "Green",
+        "CUSTOM_FIELD_ID": "Color__c"
+      },
+      {
+        "FIELD_NAME": "Weight__c",
+        "FIELD_VALUE": 12,
+        "CUSTOM_FIELD_ID": "Weight__c"
+      }
+    ]
+  },
+  {
+    "RECORD_ID": 54840682,
+    "RECORD_NAME": "Banana",
+    "OWNER_USER_ID": 2240727,
+    "DATE_CREATED_UTC": "2025-04-22 19:29:52",
+    "DATE_UPDATED_UTC": "2025-04-22 19:29:52",
+    "CREATED_USER_ID": 2240727,
+    "VISIBLE_TO": "EVERYONE",
+    "VISIBLE_TEAM_ID": null,
+    "CUSTOMFIELDS": [
+      {
+        "FIELD_NAME": "Color__c",
+        "FIELD_VALUE": null,
+        "CUSTOM_FIELD_ID": "Color__c"
+      },
+      {
+        "FIELD_NAME": "Weight__c",
+        "FIELD_VALUE": 3.2,
+        "CUSTOM_FIELD_ID": "Weight__c"
+      }
+    ]
+  }
+]
diff --git a/test/insightly/read/contacts/main.go b/test/insightly/read/contacts/main.go
index 1ca5bc0d..0651cab5 100644
--- a/test/insightly/read/contacts/main.go
+++ b/test/insightly/read/contacts/main.go
@@ -27,7 +27,7 @@ func main() {
 
 	res, err := conn.Read(ctx, common.ReadParams{
 		ObjectName: objectName,
-		Fields:     connectors.Fields("EMAIL_ADDRESS", "CONTACT_ID"),
+		Fields:     connectors.Fields("EMAIL_ADDRESS", "CONTACT_ID", "Hobby__c", "Interests__c"),
 	})
 	if err != nil {
 		utils.Fail("error reading from provider", "error", err)
diff --git a/test/insightly/read/customObject/main.go b/test/insightly/read/customObject/main.go
new file mode 100644
index 00000000..6ba46c51
--- /dev/null
+++ b/test/insightly/read/customObject/main.go
@@ -0,0 +1,38 @@
+package main
+
+import (
+	"context"
+	"log/slog"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"github.com/amp-labs/connectors"
+	"github.com/amp-labs/connectors/common"
+	connTest "github.com/amp-labs/connectors/test/insightly"
+	"github.com/amp-labs/connectors/test/utils"
+)
+
+const objectName = "Fruit__c"
+
+func main() {
+	// Handle Ctrl-C gracefully.
+	ctx, done := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
+	defer done()
+
+	// Set up slog logging.
+	utils.SetupLogging()
+
+	conn := connTest.GetInsightlyConnector(ctx)
+
+	res, err := conn.Read(ctx, common.ReadParams{
+		ObjectName: objectName,
+		Fields:     connectors.Fields("RECORD_ID", "RECORD_NAME", "Color__c"),
+	})
+	if err != nil {
+		utils.Fail("error reading from provider", "error", err)
+	}
+
+	slog.Info("Reading objects..")
+	utils.DumpJSON(res, os.Stdout)
+}
diff --git a/test/utils/mockutils/mockserver/switch.go b/test/utils/mockutils/mockserver/switch.go
index 465b04a2..6f357dd8 100644
--- a/test/utils/mockutils/mockserver/switch.go
+++ b/test/utils/mockutils/mockserver/switch.go
@@ -57,6 +57,8 @@ type Case struct {
 	Then http.HandlerFunc
 }
 
+type Cases []Case
+
 func (p *Case) isOpen(w http.ResponseWriter, r *http.Request) bool {
 	if p.If == nil {
 		return false