Repository: amp-labs/connectors
Pull request #1642 (2025-05-07 12:57:36)

[CON-1603] feat: Added read functionality for instantly v2 connector
### Sample Response

**For api-keys object**
![image](https://github.com/user-attachments/assets/ef82d8ae-64a9-4b5d-80cb-58d393dcfdc3)

**For custom-tags** object
![image](https://github.com/user-attachments/assets/91f31709-f388-423c-bb35-6ce2a22dbade)

**For lead-lists** object
![image](https://github.com/user-attachments/assets/110e26fd-d987-461d-941f-ad4cfecc88ac)

### TestCase Result
![image](https://github.com/user-attachments/assets/7942c6b6-4837-4732-a1b7-1fd39cfd0a55)


Patch:
diff --git a/providers/instantlyai/connector.go b/providers/instantlyai/connector.go
index 7e8422ba..7e266278 100644
--- a/providers/instantlyai/connector.go
+++ b/providers/instantlyai/connector.go
@@ -4,7 +4,10 @@ import (
 	_ "embed"
 
 	"github.com/amp-labs/connectors/common"
+	"github.com/amp-labs/connectors/common/interpreter"
 	"github.com/amp-labs/connectors/internal/components"
+	"github.com/amp-labs/connectors/internal/components/operations"
+	"github.com/amp-labs/connectors/internal/components/reader"
 	"github.com/amp-labs/connectors/internal/components/schema"
 	"github.com/amp-labs/connectors/providers"
 	"github.com/amp-labs/connectors/providers/instantlyai/metadata"
@@ -19,8 +22,11 @@ type Connector struct {
 
 	// Supported operations
 	components.SchemaProvider
+	components.Reader
 }
 
+const apiVersion = "v2"
+
 func NewConnector(params common.Parameters) (*Connector, error) {
 	// Create base connector with provider info
 	return components.Initialize(providers.InstantlyAI, params, constructor)
@@ -32,5 +38,23 @@ func constructor(base *components.Connector) (*Connector, error) {
 	// Set the metadata provider for the connector
 	connector.SchemaProvider = schema.NewOpenAPISchemaProvider(connector.ProviderContext.Module(), metadata.Schemas)
 
+	registry, err := components.NewEndpointRegistry(supportedOperations())
+	if err != nil {
+		return nil, err
+	}
+
+	connector.Reader = reader.NewHTTPReader(
+		connector.HTTPClient().Client,
+		registry,
+		connector.ProviderContext.Module(),
+		operations.ReadHandlers{
+			BuildRequest:  connector.buildReadRequest,
+			ParseResponse: connector.parseReadResponse,
+			ErrorHandler: interpreter.ErrorHandler{
+				JSON: interpreter.NewFaultyResponder(errorFormats, nil),
+			}.Handle,
+		},
+	)
+
 	return connector, nil
 }
diff --git a/providers/instantlyai/errors.go b/providers/instantlyai/errors.go
new file mode 100644
index 00000000..1f1a2b4f
--- /dev/null
+++ b/providers/instantlyai/errors.go
@@ -0,0 +1,29 @@
+package instantlyai
+
+import (
+	"fmt"
+
+	"github.com/amp-labs/connectors/common/interpreter"
+)
+
+var errorFormats = interpreter.NewFormatSwitch( // nolint:gochecknoglobals
+	[]interpreter.FormatTemplate{
+		{
+			MustKeys: nil,
+			Template: func() interpreter.ErrorDescriptor { return &ResponseError{} },
+		},
+	}...,
+)
+
+type ResponseError struct {
+	// Message usually has API Key query parameter. This shouldn't be returned.
+	Message string `json:"message"`
+	// Error is very general.
+	Error      string `json:"error"`
+	StatusCode int    `json:"statusCode"`
+}
+
+func (r ResponseError) CombineErr(base error) error {
+	// Error field is the safest to return, though not very useful.
+	return fmt.Errorf("%w: %v", base, r.Error)
+}
diff --git a/providers/instantlyai/handlers.go b/providers/instantlyai/handlers.go
new file mode 100644
index 00000000..e8d0edb7
--- /dev/null
+++ b/providers/instantlyai/handlers.go
@@ -0,0 +1,62 @@
+package instantlyai
+
+import (
+	"bytes"
+	"context"
+	"net/http"
+	"strconv"
+	"time"
+
+	"github.com/amp-labs/connectors/common"
+	"github.com/amp-labs/connectors/common/urlbuilder"
+)
+
+func (c *Connector) buildReadRequest(ctx context.Context, params common.ReadParams) (*http.Request, error) {
+	url, err := urlbuilder.New(c.ProviderInfo().BaseURL, apiVersion, params.ObjectName)
+	if err != nil {
+		return nil, err
+	}
+
+	if !directResponseEndpoints.Has(params.ObjectName) {
+		url.WithQueryParam("limit", strconv.Itoa(DefaultPageSize))
+	}
+
+	if sinceSupportedEndpoints.Has(params.ObjectName) {
+		url.WithQueryParam("start_date", params.Since.Format(time.DateOnly))
+	}
+
+	if len(params.NextPage) != 0 {
+		// Next page.
+		url, err = urlbuilder.New(params.NextPage.String())
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if postEndpointsOfRead.Has(params.ObjectName) {
+		return http.NewRequestWithContext(ctx, http.MethodPost, url.String(), bytes.NewReader([]byte("{}")))
+	}
+
+	return http.NewRequestWithContext(ctx, http.MethodGet, url.String(), nil)
+}
+
+func (c *Connector) parseReadResponse(
+	ctx context.Context,
+	params common.ReadParams,
+	request *http.Request,
+	response *common.JSONHTTPResponse,
+) (*common.ReadResult, error) {
+	path := "items"
+
+	if directResponseEndpoints.Has(params.ObjectName) {
+		path = ""
+	}
+
+	return common.ParseResult(
+		response,
+		common.ExtractRecordsFromPath(path),
+		makeNextRecordsURL(request.URL, params.ObjectName),
+		common.GetMarshaledData,
+		params.Fields,
+	)
+}
diff --git a/providers/instantlyai/supports.go b/providers/instantlyai/supports.go
new file mode 100644
index 00000000..b4d20606
--- /dev/null
+++ b/providers/instantlyai/supports.go
@@ -0,0 +1,23 @@
+package instantlyai
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/amp-labs/connectors/common"
+	"github.com/amp-labs/connectors/internal/components"
+	"github.com/amp-labs/connectors/providers/instantlyai/metadata"
+)
+
+func supportedOperations() components.EndpointRegistryInput {
+	readSupport := metadata.Schemas.ObjectNames().GetList(common.ModuleRoot)
+
+	return components.EndpointRegistryInput{
+		common.ModuleRoot: {
+			{
+				Endpoint: fmt.Sprintf("{%s}", strings.Join(readSupport, ",")),
+				Support:  components.ReadSupport,
+			},
+		},
+	}
+}
diff --git a/providers/instantlyai/utils.go b/providers/instantlyai/utils.go
new file mode 100644
index 00000000..0cf411ce
--- /dev/null
+++ b/providers/instantlyai/utils.go
@@ -0,0 +1,56 @@
+package instantlyai
+
+import (
+	"net/url"
+
+	"github.com/amp-labs/connectors/common"
+	"github.com/amp-labs/connectors/common/urlbuilder"
+	"github.com/amp-labs/connectors/internal/datautils"
+	"github.com/amp-labs/connectors/internal/jsonquery"
+	"github.com/spyzhov/ajson"
+)
+
+const DefaultPageSize = 100
+
+// directResponseEndpoints variable contains endpoint having direct
+// response which means response not included any objects like "items".
+var directResponseEndpoints = datautils.NewSet( //nolint:gochecknoglobals
+	"campaigns/analytics",
+	"campaigns/analytics/daily",
+	"campaigns/analytics/steps",
+)
+
+var postEndpointsOfRead = datautils.NewSet( //nolint:gochecknoglobals
+	"leads/list",
+)
+
+var sinceSupportedEndpoints = datautils.NewSet( //nolint:gochecknoglobals
+	"campaigns/analytics/daily",
+	"campaigns/analytics/steps",
+)
+
+func makeNextRecordsURL(reqLink *url.URL, objName string) common.NextPageFunc {
+	return func(node *ajson.Node) (string, error) {
+		if directResponseEndpoints.Has(objName) {
+			return "", nil
+		}
+
+		url, err := urlbuilder.FromRawURL(reqLink)
+		if err != nil {
+			return "", err
+		}
+
+		pagination, err := jsonquery.New(node).StringRequired("next_starting_after")
+		if err != nil {
+			return "", err
+		}
+
+		if pagination != "" {
+			url.WithQueryParam("starting_after", pagination)
+
+			return url.String(), nil
+		}
+
+		return "", nil
+	}
+}


Test patch:
diff --git a/providers/instantlyai/read_test.go b/providers/instantlyai/read_test.go
new file mode 100644
index 00000000..709c6b25
--- /dev/null
+++ b/providers/instantlyai/read_test.go
@@ -0,0 +1,124 @@
+package instantlyai
+
+import (
+	"net/http"
+	"testing"
+
+	"github.com/amp-labs/connectors"
+	"github.com/amp-labs/connectors/common"
+	"github.com/amp-labs/connectors/test/utils/mockutils/mockserver"
+	"github.com/amp-labs/connectors/test/utils/testroutines"
+	"github.com/amp-labs/connectors/test/utils/testutils"
+)
+
+func TestRead(t *testing.T) { // nolint:funlen,gocognit,cyclop
+	t.Parallel()
+
+	campaignsResponse := testutils.DataFromFile(t, "campaigns.json")
+	customTagsResponse := testutils.DataFromFile(t, "custom-tags.json")
+	leadListsResponse := testutils.DataFromFile(t, "lead-lists.json")
+
+	tests := []testroutines.Read{
+		{
+			Name:         "Read object must be included",
+			Server:       mockserver.Dummy(),
+			ExpectedErrs: []error{common.ErrMissingObjects},
+		},
+		{
+			Name:  "Read list of campaigns",
+			Input: common.ReadParams{ObjectName: "campaigns", Fields: connectors.Fields("")},
+			Server: mockserver.Fixed{
+				Setup:  mockserver.ContentJSON(),
+				Always: mockserver.Response(http.StatusOK, campaignsResponse),
+			}.Server(),
+			Comparator: testroutines.ComparatorPagination,
+			Expected: &common.ReadResult{
+				Rows: 1,
+				Data: []common.ReadResultRow{
+					{
+						Fields: map[string]any{},
+						Raw: map[string]any{
+							"id":                "0196d267-8d93-73c3-b563-1a6443e78eb8",
+							"name":              "My First Campaign",
+							"timestamp_created": "2025-05-15T05:25:23.987Z",
+							"timestamp_updated": "2025-05-15T05:25:23.987Z",
+							"organization":      "0196d267-8d93-73c3-b563-1a6685313ea8",
+						},
+					},
+				},
+				NextPage: testroutines.URLTestServer +
+					"/v2/campaigns?limit=100&starting_after=0196d267-98dd-7720-b740-f132c2c547d9",
+				Done: false,
+			},
+			ExpectedErrs: nil,
+		},
+		{
+			Name:  "Read list of custom tags",
+			Input: common.ReadParams{ObjectName: "custom-tags", Fields: connectors.Fields("")},
+			Server: mockserver.Fixed{
+				Setup:  mockserver.ContentJSON(),
+				Always: mockserver.Response(http.StatusOK, customTagsResponse),
+			}.Server(),
+			Comparator: testroutines.ComparatorPagination,
+			Expected: &common.ReadResult{
+				Rows: 1,
+				Data: []common.ReadResultRow{
+					{
+						Fields: map[string]any{},
+						Raw: map[string]any{
+							"id":                "01966b52-73c2-7a5d-9df3-ea5a95a7fb09",
+							"timestamp_created": "2025-04-25T05:01:27.874Z",
+							"timestamp_updated": "2025-04-25T05:01:27.875Z",
+							"organization_id":   "01966b52-73c3-7cc6-8081-efbe11b617f2",
+							"label":             "Important",
+							"description":       "Used for marking important items",
+						},
+					},
+				},
+				NextPage: testroutines.URLTestServer +
+					"/v2/custom-tags?limit=100&starting_after=01966b52-881d-76de-acb0-8b12432533f6",
+				Done: false,
+			},
+			ExpectedErrs: nil,
+		},
+		{
+			Name:  "Read list of lead lists",
+			Input: common.ReadParams{ObjectName: "lead-lists", Fields: connectors.Fields("")},
+			Server: mockserver.Fixed{
+				Setup:  mockserver.ContentJSON(),
+				Always: mockserver.Response(http.StatusOK, leadListsResponse),
+			}.Server(),
+			Comparator: testroutines.ComparatorPagination,
+			Expected: &common.ReadResult{
+				Rows: 1,
+				Data: []common.ReadResultRow{
+					{
+						Fields: map[string]any{},
+						Raw: map[string]any{
+							"id":                  "01966b52-73bf-79c0-bf01-933de40a96af",
+							"organization_id":     "01966b52-73bf-79c0-bf01-933e7e9d7123",
+							"has_enrichment_task": false,
+							"owned_by":            "01966b52-73bf-79c0-bf01-933fa9d5415a",
+							"name":                "My Lead List",
+							"timestamp_created":   "2025-04-25T05:01:27.871Z",
+						},
+					},
+				},
+				NextPage: testroutines.URLTestServer +
+					"/v2/lead-lists?limit=100&starting_after=01966b52-883f-760d-b7dc-cdd0fc3f7841",
+				Done: false,
+			},
+			ExpectedErrs: nil,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.Name, func(t *testing.T) {
+			t.Parallel()
+
+			tt.Run(t, func() (connectors.ReadConnector, error) {
+				return constructTestConnector(tt.Server.URL)
+			})
+		})
+	}
+}
diff --git a/providers/instantlyai/test/campaigns.json b/providers/instantlyai/test/campaigns.json
new file mode 100644
index 00000000..130c7656
--- /dev/null
+++ b/providers/instantlyai/test/campaigns.json
@@ -0,0 +1,12 @@
+{
+  "items": [
+    {
+      "id": "0196d267-8d93-73c3-b563-1a6443e78eb8",
+      "name": "My First Campaign",
+      "timestamp_created": "2025-05-15T05:25:23.987Z",
+      "timestamp_updated": "2025-05-15T05:25:23.987Z",
+      "organization": "0196d267-8d93-73c3-b563-1a6685313ea8"
+    }
+  ],
+  "next_starting_after": "0196d267-98dd-7720-b740-f132c2c547d9"
+}
\ No newline at end of file
diff --git a/test/instantlyai/read/read.go b/test/instantlyai/read/read.go
new file mode 100644
index 00000000..a67378e7
--- /dev/null
+++ b/test/instantlyai/read/read.go
@@ -0,0 +1,67 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/amp-labs/connectors"
+	"github.com/amp-labs/connectors/common"
+	ap "github.com/amp-labs/connectors/providers/instantlyai"
+	"github.com/amp-labs/connectors/test/instantlyai"
+)
+
+func main() {
+	os.Exit(MainFn())
+}
+
+func MainFn() int {
+	conn := instantlyai.GetInstantlyAIConnector(context.Background())
+
+	err := testRead(context.Background(), conn, "campaigns", []string{""}, time.Time{})
+	if err != nil {
+		return 1
+	}
+
+	err = testRead(context.Background(), conn, "custom-tags", []string{""}, time.Time{})
+	if err != nil {
+		return 1
+	}
+
+	err = testRead(context.Background(), conn, "lead-lists", []string{""}, time.Time{})
+	if err != nil {
+		return 1
+	}
+
+	err = testRead(context.Background(), conn, "campaigns/analytics/daily", []string{""}, time.Date(2025, time.May, 1, 0, 0, 0, 0, time.Local))
+	if err != nil {
+		return 1
+	}
+
+	return 0
+}
+
+func testRead(ctx context.Context, conn *ap.Connector, objName string, fields []string, since time.Time) error {
+	params := common.ReadParams{
+		ObjectName: objName,
+		Fields:     connectors.Fields(fields...),
+	}
+
+	res, err := conn.Read(ctx, params)
+	if err != nil {
+		return fmt.Errorf("failed to read %s: %w", objName, err)
+	}
+
+	// Print the results.
+	jsonStr, err := json.MarshalIndent(res, "", "  ")
+	if err != nil {
+		return fmt.Errorf("error marshalling JSON: %w", err)
+	}
+
+	_, _ = os.Stdout.Write(jsonStr)
+	_, _ = os.Stdout.WriteString("\n")
+
+	return nil
+}