Repository: amp-labs/connectors
Pull request #1282 (2025-01-14 14:43:50)

fix: make next-page key optional in Close Connector
Before:
<img width="1207" alt="Screenshot 2025-01-14 at 17 38 47" src="https://github.com/user-attachments/assets/fe4a9443-7b94-4085-b7fb-7307b024f925" />


After:
<img width="1205" alt="Screenshot 2025-01-14 at 17 37 22" src="https://github.com/user-attachments/assets/08d9715c-ffd8-45f9-bd11-b67eae3d5e97" />



Patch:
diff --git a/providers/closecrm/parse.go b/providers/closecrm/parse.go
index 9021bd50..60db595e 100644
--- a/providers/closecrm/parse.go
+++ b/providers/closecrm/parse.go
@@ -34,21 +34,23 @@ var ErrSkipFailure = errors.New("error: failed to create next page url")
 func nextRecordsURL(url *urlbuilder.URL) common.NextPageFunc {
 	return func(node *ajson.Node) (string, error) {
 		// check if there is more items in the collection.
-		hasMore, err := jsonquery.New(node).Bool(hasMoreQuery, false)
+		hasMore, err := jsonquery.New(node).Bool(hasMoreQuery, true)
 		if err != nil {
 			return "", err
 		}
 
-		if *hasMore {
-			currSkip, exists := url.GetFirstQueryParam(skipQuery)
-			if !exists {
-				return "", ErrSkipFailure
-			}
+		if hasMore != nil {
+			if *hasMore {
+				currSkip, exists := url.GetFirstQueryParam(skipQuery)
+				if !exists {
+					return "", ErrSkipFailure
+				}
 
-			url.WithQueryParam(skipQuery, currSkip+defaultPageSize)
-			url.WithQueryParam(limitQuery, defaultPageSize)
+				url.WithQueryParam(skipQuery, currSkip+defaultPageSize)
+				url.WithQueryParam(limitQuery, defaultPageSize)
 
-			return url.String(), nil
+				return url.String(), nil
+			}
 		}
 
 		return "", nil


Test patch:
diff --git a/providers/pipedrive/read_test.go b/providers/pipedrive/read_test.go
index a08d4037..93db63e5 100644
--- a/providers/pipedrive/read_test.go
+++ b/providers/pipedrive/read_test.go
@@ -1,7 +1,7 @@
 package pipedrive
 
 import (
-	"fmt"
+	"errors"
 	"net/http"
 	"testing"
 
@@ -17,13 +17,7 @@ func TestRead(t *testing.T) { //nolint:funlen,gocognit,cyclop,maintidx
 
 	nextPageTest := testutils.DataFromFile(t, "activities.json")
 	leads := testutils.DataFromFile(t, "leads.json")
-
-	ErrResponseBody := `{
-		"success":false,
-		"error":"Scope and URL mismatch",
-		"errorCode":403,
-		"error_info":"Please check developers.pipedrive.com"
-	}`
+	ErrResponseBody := testutils.DataFromFile(t, "not-found.json")
 
 	server := mockserver.Fixed{
 		Setup:  mockserver.ContentJSON(),
@@ -46,6 +40,7 @@ func TestRead(t *testing.T) { //nolint:funlen,gocognit,cyclop,maintidx
 			Name:         "NextPage URL construction",
 			Input:        common.ReadParams{ObjectName: "activities", Fields: connectors.Fields("id")},
 			Server:       server,
+			Comparator:   testroutines.ComparatorSubsetRead,
 			ExpectedErrs: nil,
 			Expected: &common.ReadResult{
 				Rows: 1,
@@ -55,69 +50,24 @@ func TestRead(t *testing.T) { //nolint:funlen,gocognit,cyclop,maintidx
 							"id": float64(2),
 						},
 						Raw: map[string]any{
-							"id":                            float64(2),
-							"company_id":                    float64(13313052),
-							"user_id":                       float64(20580207),
-							"done":                          false,
-							"type":                          "call",
-							"reference_type":                nil,
-							"reference_id":                  nil,
-							"conference_meeting_client":     nil,
-							"conference_meeting_url":        nil,
-							"due_date":                      "2024-10-30",
-							"due_time":                      "",
-							"duration":                      "",
-							"busy_flag":                     false,
-							"add_time":                      "2024-10-16 12:16:02",
-							"marked_as_done_time":           "",
-							"last_notification_time":        nil,
-							"last_notification_user_id":     nil,
-							"notification_language_id":      nil,
-							"subject":                       "I usually can't come up with words",
-							"public_description":            "Demo activity",
-							"calendar_sync_include_context": nil,
-							"location":                      "Dar es salaam",
-							"org_id":                        nil,
-							"person_id":                     nil,
-							"deal_id":                       nil,
-							"lead_id":                       nil,
-							"active_flag":                   true,
-							"update_time":                   "2024-10-16 12:16:02",
-							"update_user_id":                nil,
-							"source_timezone":               nil,
-							"rec_rule":                      nil,
-							"rec_rule_extension":            nil,
-							"rec_master_activity_id":        nil,
-							"conference_meeting_id":         nil,
-							"original_start_time":           nil,
-							"private":                       false,
-							"priority":                      nil,
-							"note":                          nil,
-							"created_by_user_id":            float64(20580207),
-							"location_subpremise":           nil,
-							"location_street_number":        nil,
-							"location_route":                nil,
-							"location_sublocality":          nil,
-							"location_locality":             nil,
-							"location_admin_area_level_1":   nil,
-							"location_admin_area_level_2":   nil,
-							"location_country":              nil,
-							"location_postal_code":          nil,
-							"location_formatted_address":    nil,
-							"attendees":                     nil,
-							"participants":                  nil,
-							"series":                        nil,
-							"is_recurring":                  nil,
-							"org_name":                      nil,
-							"person_name":                   nil,
-							"deal_title":                    nil,
-							"lead_title":                    nil,
-							"owner_name":                    "Integration User",
-							"person_dropbox_bcc":            nil,
-							"deal_dropbox_bcc":              nil,
-							"assigned_to_user_id":           float64(20580207),
-							"type_name":                     "Call",
-							"lead":                          nil,
+							"id":                  float64(2),
+							"company_id":          float64(13313052),
+							"user_id":             float64(20580207),
+							"done":                false,
+							"type":                "call",
+							"due_date":            "2024-10-30",
+							"busy_flag":           false,
+							"add_time":            "2024-10-16 12:16:02",
+							"subject":             "I usually can't come up with words",
+							"public_description":  "Demo activity",
+							"location":            "Dar es salaam",
+							"active_flag":         true,
+							"update_time":         "2024-10-16 12:16:02",
+							"private":             false,
+							"created_by_user_id":  float64(20580207),
+							"owner_name":          "Integration User",
+							"assigned_to_user_id": float64(20580207),
+							"type_name":           "Call",
 						},
 					},
 				},
@@ -126,18 +76,20 @@ func TestRead(t *testing.T) { //nolint:funlen,gocognit,cyclop,maintidx
 			},
 		},
 		{
-			Name:  "Not Found Resource or Higher Suite Resource",
+			Name:  "Resource not found",
 			Input: common.ReadParams{ObjectName: "activitiess", Fields: connectors.Fields("id")},
 			Server: mockserver.Fixed{
 				Setup:  mockserver.ContentJSON(),
-				Always: mockserver.ResponseString(http.StatusForbidden, ErrResponseBody),
+				Always: mockserver.Response(http.StatusForbidden, ErrResponseBody),
 			}.Server(),
-			ExpectedErrs: []error{common.NewHTTPStatusError(http.StatusForbidden,
-				fmt.Errorf("%w: %s", common.ErrForbidden, ErrResponseBody))},
+			ExpectedErrs: []error{
+				common.ErrForbidden,
+				errors.New(string(ErrResponseBody)), //nolint:err113
+			},
 			Expected: nil,
 		},
 		{
-			Name: "Successful Read",
+			Name: "Successful read of leads",
 			Input: common.ReadParams{
 				ObjectName: "leads",
 				Fields:     connectors.Fields("channel", "id", "origin", "title"),
@@ -157,27 +109,22 @@ func TestRead(t *testing.T) { //nolint:funlen,gocognit,cyclop,maintidx
 						"title":   "Ampersand lead",
 					},
 					Raw: map[string]any{
-						"add_time":            "2024-10-15T09:33:33.213Z",
-						"cc_email":            "integrationuser-sandbox2+13313052+leadif7ZKQ7mhWz28LivD9BmE3@pipedrivemail.com",
-						"channel":             nil,
-						"channel_id":          nil,
-						"creator_id":          float64(20580207),
-						"expected_close_date": nil,
-						"id":                  "8ba414d0-8ad8-11ef-9e9e-07879dd74146",
-						"is_archived":         false,
-						"label_ids":           []any{},
-						"next_activity_id":    nil,
-						"organization_id":     float64(2),
-						"origin":              "ManuallyCreated",
-						"origin_id":           nil,
-						"owner_id":            float64(20580207),
-						"person_id":           float64(2),
-						"source_name":         "Manually created",
-						"title":               "Ampersand lead",
-						"update_time":         "2024-10-15T09:33:33.213Z",
-						"value":               nil,
-						"visible_to":          "3",
-						"was_seen":            true,
+						"add_time":        "2024-10-15T09:33:33.213Z",
+						"cc_email":        "integrationuser-sandbox2+13313052+leadif7ZKQ7mhWz28LivD9BmE3@pipedrivemail.com",
+						"creator_id":      float64(20580207),
+						"id":              "8ba414d0-8ad8-11ef-9e9e-07879dd74146",
+						"is_archived":     false,
+						"label_ids":       []any{},
+						"organization_id": float64(2),
+						"origin":          "ManuallyCreated",
+						"channel":         nil,
+						"owner_id":        float64(20580207),
+						"person_id":       float64(2),
+						"source_name":     "Manually created",
+						"title":           "Ampersand lead",
+						"update_time":     "2024-10-15T09:33:33.213Z",
+						"visible_to":      "3",
+						"was_seen":        true,
 					},
 				}},
 				NextPage: "",
diff --git a/providers/pipedrive/test/create-activity.json b/providers/pipedrive/test/create-activity.json
new file mode 100644
index 00000000..732b5a72
--- /dev/null
+++ b/providers/pipedrive/test/create-activity.json
@@ -0,0 +1,35 @@
+{
+  "success": true,
+  "data": {
+    "id": 9,
+    "company_id": 13313052,
+    "user_id": 20580207,
+    "done": false,
+    "type": "call",
+    "due_date": "2024-10-30",
+    "busy_flag": false,
+    "add_time": "2025-01-14 09:44:56",
+    "subject": "I usually can't come up with words",
+    "public_description": "Demo activity",
+    "location": "Dar es salaam",
+    "update_time": "2025-01-14 09:44:56",
+    "private": false,
+    "created_by_user_id": 20580207,
+    "owner_name": "Integration User",
+    "assigned_to_user_id": 20580207,
+    "type_name": "Call"
+  },
+  "additional_data": { "updates_story_id": null },
+  "related_objects": {
+    "user": {
+      "20580207": {
+        "id": 20580207,
+        "name": "Integration User",
+        "email": "integration.user+pipedrive@withampersand.com",
+        "has_pic": 0,
+        "pic_hash": null,
+        "active_flag": true
+      }
+    }
+  }
+}
diff --git a/providers/pipedrive/test/not-found.json b/providers/pipedrive/test/not-found.json
new file mode 100644
index 00000000..43593632
--- /dev/null
+++ b/providers/pipedrive/test/not-found.json
@@ -0,0 +1,6 @@
+{
+  "success": false,
+  "error": "Scope and URL mismatch",
+  "errorCode": 403,
+  "error_info": "Please check developers.pipedrive.com"
+}
diff --git a/providers/pipedrive/test/update-activity.json b/providers/pipedrive/test/update-activity.json
new file mode 100644
index 00000000..d1950ff8
--- /dev/null
+++ b/providers/pipedrive/test/update-activity.json
@@ -0,0 +1,36 @@
+{
+  "success": true,
+  "data": {
+    "id": 1,
+    "company_id": 13313052,
+    "user_id": 20580207,
+    "done": true,
+    "type": "call",
+    "reference_type": "salesphone",
+    "due_date": "2024-10-15",
+    "busy_flag": false,
+    "add_time": "2024-10-15 09:33:57",
+    "marked_as_done_time": "2024-10-16 12:22:47",
+    "subject": "Make call with sales people",
+    "active_flag": true,
+    "update_time": "2025-01-14 09:37:13",
+    "update_user_id": 20580207,
+    "private": false,
+    "created_by_user_id": 20580207,
+    "owner_name": "Integration User",
+    "assigned_to_user_id": 20580207,
+    "type_name": "Call"
+  },
+  "related_objects": {
+    "user": {
+      "20580207": {
+        "id": 20580207,
+        "name": "Integration User",
+        "email": "integration.user+pipedrive@withampersand.com",
+        "has_pic": 0,
+        "pic_hash": null,
+        "active_flag": true
+      }
+    }
+  }
+}
diff --git a/providers/pipedrive/write_test.go b/providers/pipedrive/write_test.go
new file mode 100644
index 00000000..e1f11cd3
--- /dev/null
+++ b/providers/pipedrive/write_test.go
@@ -0,0 +1,145 @@
+package pipedrive
+
+import (
+	"errors"
+	"net/http"
+	"testing"
+
+	"github.com/amp-labs/connectors"
+	"github.com/amp-labs/connectors/common"
+	"github.com/amp-labs/connectors/test/utils/mockutils/mockcond"
+	"github.com/amp-labs/connectors/test/utils/mockutils/mockserver"
+	"github.com/amp-labs/connectors/test/utils/testroutines"
+	"github.com/amp-labs/connectors/test/utils/testutils"
+)
+
+// nolint
+func TestWrite(t *testing.T) { // nolint:funlen,gocognit,cyclop
+	t.Parallel()
+
+	updateActivityResponse := testutils.DataFromFile(t, "update-activity.json")
+	unsupportedResponse := testutils.DataFromFile(t, "not-found.json")
+	createActivityResponse := testutils.DataFromFile(t, "create-activity.json")
+
+	tests := []testroutines.Write{
+		{
+			Name:         "Object Name is required",
+			Server:       mockserver.Dummy(),
+			ExpectedErrs: []error{common.ErrMissingObjects},
+		},
+		{
+			Name:         "RecordData is required",
+			Input:        common.WriteParams{ObjectName: "leads"},
+			Server:       mockserver.Dummy(),
+			ExpectedErrs: []error{common.ErrMissingRecordData},
+		},
+		{
+			Name:  "Unsupported object",
+			Input: common.WriteParams{ObjectName: "arsenal", RecordData: map[string]any{"test": "value"}},
+			Server: mockserver.Conditional{
+				Setup: mockserver.ContentJSON(),
+				If:    mockcond.MethodPOST(),
+				Then:  mockserver.Response(http.StatusNotFound, unsupportedResponse),
+			}.Server(),
+			ExpectedErrs: []error{
+				common.ErrRetryable,
+				errors.New(string(unsupportedResponse)), // nolint:err113
+			},
+		},
+		{
+			Name: "Successful creation of an activity",
+			Input: common.WriteParams{
+				ObjectName: "activities",
+				RecordData: map[string]any{
+					"due_date":           "2024-10-30",
+					"location":           "Dar es salaam",
+					"public_description": "Demo activity",
+					"subject":            "I usually can't come up with words",
+				},
+			},
+			Server: mockserver.Conditional{
+				Setup: mockserver.ContentJSON(),
+				If:    mockcond.MethodPOST(),
+				Then:  mockserver.Response(http.StatusOK, createActivityResponse),
+			}.Server(),
+			Comparator: testroutines.ComparatorSubsetWrite,
+			Expected: &common.WriteResult{
+				Success:  true,
+				RecordId: "9",
+				Data: map[string]any{
+					"add_time":            "2025-01-14 09:44:56",
+					"assigned_to_user_id": float64(20580207),
+					"busy_flag":           false,
+					"company_id":          float64(13313052),
+					"created_by_user_id":  float64(20580207),
+					"done":                false,
+					"due_date":            "2024-10-30",
+					"id":                  float64(9),
+					"location":            "Dar es salaam",
+					"owner_name":          "Integration User",
+					"private":             false,
+					"public_description":  "Demo activity",
+					"subject":             "I usually can't come up with words",
+					"type":                "call",
+					"type_name":           "Call",
+					"update_time":         "2025-01-14 09:44:56",
+					"user_id":             float64(20580207),
+				},
+			},
+			ExpectedErrs: nil,
+		},
+		{
+			Name: "Successful update an activity",
+			Input: common.WriteParams{
+				ObjectName: "activities",
+				RecordId:   "1",
+				RecordData: map[string]any{
+					"done": "1",
+				},
+			},
+			Server: mockserver.Conditional{
+				Setup: mockserver.ContentJSON(),
+				If:    mockcond.MethodPUT(),
+				Then:  mockserver.Response(http.StatusOK, updateActivityResponse),
+			}.Server(),
+			Comparator: testroutines.ComparatorSubsetWrite,
+			Expected: &common.WriteResult{
+				Success:  true,
+				RecordId: "1",
+				Data: map[string]any{
+					"active_flag":         true,
+					"add_time":            "2024-10-15 09:33:57",
+					"assigned_to_user_id": float64(20580207),
+					"busy_flag":           false,
+					"company_id":          float64(13313052),
+					"created_by_user_id":  float64(20580207),
+					"done":                true,
+					"due_date":            "2024-10-15",
+					"id":                  float64(1),
+					"marked_as_done_time": "2024-10-16 12:22:47",
+					"owner_name":          "Integration User",
+					"private":             false,
+					"reference_type":      "salesphone",
+					"subject":             "Make call with sales people",
+					"type":                "call",
+					"type_name":           "Call",
+					"update_time":         "2025-01-14 09:37:13",
+					"update_user_id":      float64(20580207),
+					"user_id":             float64(20580207),
+				},
+			},
+			ExpectedErrs: nil,
+		},
+	}
+
+	for _, tt := range tests {
+		// nolint:varnamelen
+		t.Run(tt.Name, func(t *testing.T) {
+			t.Parallel()
+
+			tt.Run(t, func() (connectors.WriteConnector, error) {
+				return constructTestConnector(tt.Server.URL)
+			})
+		})
+	}
+}