Repository: amp-labs/connectors (go)

Issue:
Add Smartlead deep connector

Pull request:
feat: Deep connector Smartlead (Read)
Closes #911 

# Description
There is no pagination.

# Testing
![image](https://github.com/user-attachments/assets/c260c3aa-e8b9-4f34-947e-7cf367c2a9e5)


Patch:
diff --git a/smartlead/client.go b/smartlead/client.go
new file mode 100644
index 00000000..dd5fa7fa
--- /dev/null
+++ b/smartlead/client.go
@@ -0,0 +1,16 @@
+package smartlead
+
+import "github.com/amp-labs/connectors/common"
+
+// JSONHTTPClient returns the underlying JSON HTTP client.
+func (c *Connector) JSONHTTPClient() *common.JSONHTTPClient {
+	return c.Client
+}
+
+func (c *Connector) HTTPClient() *common.HTTPClient {
+	return c.Client.HTTPClient
+}
+
+func (c *Connector) Close() error {
+	return nil
+}
diff --git a/smartlead/connector.go b/smartlead/connector.go
new file mode 100644
index 00000000..dbb2a3a1
--- /dev/null
+++ b/smartlead/connector.go
@@ -0,0 +1,69 @@
+package smartlead
+
+import (
+	"fmt"
+
+	"github.com/amp-labs/connectors/common"
+	"github.com/amp-labs/connectors/common/interpreter"
+	"github.com/amp-labs/connectors/common/paramsbuilder"
+	"github.com/amp-labs/connectors/common/urlbuilder"
+	"github.com/amp-labs/connectors/providers"
+)
+
+const apiVersion = "v1"
+
+type Connector struct {
+	BaseURL string
+	Client  *common.JSONHTTPClient
+}
+
+func NewConnector(opts ...Option) (conn *Connector, outErr error) {
+	defer common.PanicRecovery(func(cause error) {
+		outErr = cause
+		conn = nil
+	})
+
+	params, err := paramsbuilder.Apply(parameters{}, opts)
+	if err != nil {
+		return nil, err
+	}
+
+	httpClient := params.Client.Caller
+	conn = &Connector{
+		Client: &common.JSONHTTPClient{
+			HTTPClient: httpClient,
+		},
+	}
+
+	providerInfo, err := providers.ReadInfo(conn.Provider())
+	if err != nil {
+		return nil, err
+	}
+
+	// connector and its client must mirror base url and provide its own error parser
+	conn.setBaseURL(providerInfo.BaseURL)
+	conn.Client.HTTPClient.ErrorHandler = interpreter.ErrorHandler{
+		JSON: interpreter.NewFaultyResponder(errorFormats, nil),
+		HTML: &interpreter.DirectFaultyResponder{Callback: conn.interpretHTMLError},
+	}.Handle
+
+	return conn, nil
+}
+
+func (c *Connector) Provider() providers.Provider {
+	return providers.Smartlead
+}
+
+func (c *Connector) String() string {
+	return fmt.Sprintf("%s.Connector", c.Provider())
+}
+
+// nolint:unused
+func (c *Connector) getURL(arg string) (*urlbuilder.URL, error) {
+	return constructURL(c.BaseURL, apiVersion, arg)
+}
+
+func (c *Connector) setBaseURL(newURL string) {
+	c.BaseURL = newURL
+	c.Client.HTTPClient.Base = newURL
+}
diff --git a/smartlead/errors.go b/smartlead/errors.go
new file mode 100644
index 00000000..41259e93
--- /dev/null
+++ b/smartlead/errors.go
@@ -0,0 +1,40 @@
+package smartlead
+
+import (
+	"bytes"
+	"fmt"
+	"net/http"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/amp-labs/connectors/common/interpreter"
+)
+
+var errorFormats = interpreter.NewFormatSwitch( // nolint:gochecknoglobals
+	[]interpreter.FormatTemplate{
+		{
+			MustKeys: nil,
+			Template: func() interpreter.ErrorDescriptor { return &ResponseError{} },
+		},
+	}...,
+)
+
+type ResponseError struct{}
+
+func (r ResponseError) CombineErr(base error) error {
+	return base
+}
+
+func (c *Connector) interpretHTMLError(res *http.Response, body []byte) error {
+	base := interpreter.DefaultStatusCodeMappingToErr(res, body)
+
+	document, err := goquery.NewDocumentFromReader(bytes.NewReader(body))
+	if err != nil {
+		// ignore HTML that cannot be understood
+		return base
+	}
+
+	// Message is located under the <pre></pre> tag
+	message := document.Find("pre").Text()
+
+	return fmt.Errorf("%w: %v", base, message)
+}
diff --git a/smartlead/params.go b/smartlead/params.go
new file mode 100644
index 00000000..0614724c
--- /dev/null
+++ b/smartlead/params.go
@@ -0,0 +1,60 @@
+package smartlead
+
+import (
+	"context"
+	"errors"
+	"net/http"
+
+	"github.com/amp-labs/connectors/common"
+	"github.com/amp-labs/connectors/common/paramsbuilder"
+	"github.com/amp-labs/connectors/providers"
+)
+
+// Option is a function which mutates the connector configuration.
+type Option = func(params *parameters)
+
+// parameters surface options by delegation.
+type parameters struct {
+	paramsbuilder.Client
+	// Error is set when any With<Method> fails, used for parameters validation.
+	setupError error
+}
+
+func (p parameters) ValidateParams() error {
+	if p.setupError != nil {
+		return p.setupError
+	}
+
+	return errors.Join(
+		p.Client.ValidateParams(),
+	)
+}
+
+func WithClient(ctx context.Context, client *http.Client,
+	apiKey string,
+	opts ...common.QueryParamAuthClientOption,
+) Option {
+	return func(params *parameters) {
+		info, err := providers.ReadInfo(providers.Smartlead)
+		if err != nil {
+			params.setupError = err
+
+			return
+		}
+
+		queryParam, err := info.GetApiKeyQueryParamName()
+		if err != nil {
+			params.setupError = err
+
+			return
+		}
+
+		params.WithApiKeyQueryParamClient(ctx, client, queryParam, apiKey, opts...)
+	}
+}
+
+func WithAuthenticatedClient(client common.AuthenticatedHTTPClient) Option {
+	return func(params *parameters) {
+		params.WithAuthenticatedClient(client)
+	}
+}
diff --git a/smartlead/parse.go b/smartlead/parse.go
new file mode 100644
index 00000000..39ac3592
--- /dev/null
+++ b/smartlead/parse.go
@@ -0,0 +1,38 @@
+package smartlead
+
+import (
+	"github.com/amp-labs/connectors/common"
+	"github.com/amp-labs/connectors/common/jsonquery"
+	"github.com/spyzhov/ajson"
+)
+
+func getTotalSize(node *ajson.Node) (int64, error) {
+	return jsonquery.New(node).ArraySize("")
+}
+
+func getRecords(node *ajson.Node) ([]map[string]any, error) {
+	arr, err := jsonquery.New(node).Array("", false)
+	if err != nil {
+		return nil, err
+	}
+
+	return jsonquery.Convertor.ArrayToMap(arr)
+}
+
+func getNextRecordsURL(_ *ajson.Node) (string, error) {
+	// Pagination is not supported for this provider.
+	return "", nil
+}
+
+func getMarshaledData(records []map[string]any, fields []string) ([]common.ReadResultRow, error) {
+	data := make([]common.ReadResultRow, len(records))
+
+	for i, record := range records {
+		data[i] = common.ReadResultRow{
+			Fields: common.ExtractLowercaseFieldsFromRaw(fields, record),
+			Raw:    record,
+		}
+	}
+
+	return data, nil
+}
diff --git a/smartlead/read.go b/smartlead/read.go
new file mode 100644
index 00000000..cf6633ab
--- /dev/null
+++ b/smartlead/read.go
@@ -0,0 +1,32 @@
+package smartlead
+
+import (
+	"context"
+
+	"github.com/amp-labs/connectors/common"
+)
+
+func (c *Connector) Read(ctx context.Context, config common.ReadParams) (*common.ReadResult, error) {
+	if len(config.ObjectName) == 0 {
+		return nil, common.ErrMissingObjects
+	}
+
+	link, err := c.getURL(config.ObjectName)
+	if err != nil {
+		return nil, err
+	}
+
+	rsp, err := c.Client.Get(ctx, link.String())
+	if err != nil {
+		return nil, err
+	}
+
+	return common.ParseResult(
+		rsp,
+		getTotalSize,
+		getRecords,
+		getNextRecordsURL,
+		getMarshaledData,
+		config.Fields,
+	)
+}
diff --git a/smartlead/url.go b/smartlead/url.go
new file mode 100644
index 00000000..3f11c702
--- /dev/null
+++ b/smartlead/url.go
@@ -0,0 +1,18 @@
+package smartlead
+
+import "github.com/amp-labs/connectors/common/urlbuilder"
+
+var queryEncodingExceptions = map[string]string{ //nolint:gochecknoglobals
+	// none
+}
+
+func constructURL(base string, path ...string) (*urlbuilder.URL, error) {
+	link, err := urlbuilder.New(base, path...)
+	if err != nil {
+		return nil, err
+	}
+
+	link.AddEncodingExceptions(queryEncodingExceptions)
+
+	return link, nil
+}


Test patch:
diff --git a/smartlead/read_test.go b/smartlead/read_test.go
new file mode 100644
index 00000000..41fe638b
--- /dev/null
+++ b/smartlead/read_test.go
@@ -0,0 +1,146 @@
+package smartlead
+
+import (
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/amp-labs/connectors"
+	"github.com/amp-labs/connectors/common"
+	"github.com/amp-labs/connectors/common/interpreter"
+	"github.com/amp-labs/connectors/common/jsonquery"
+	"github.com/amp-labs/connectors/test/utils/mockutils"
+	"github.com/amp-labs/connectors/test/utils/mockutils/mockserver"
+	"github.com/amp-labs/connectors/test/utils/testroutines"
+	"github.com/amp-labs/connectors/test/utils/testutils"
+)
+
+func TestRead(t *testing.T) { //nolint:funlen,gocognit,cyclop
+	t.Parallel()
+
+	responseCampaigns := testutils.DataFromFile(t, "read-campaign.json")
+	responseCampaignsEmpty := testutils.DataFromFile(t, "read-campaign-empty.json")
+	responseInvalidPath := testutils.DataFromFile(t, "read-invalid-path.html")
+
+	tests := []testroutines.Read{
+		{
+			Name:         "Read object must be included",
+			Input:        common.ReadParams{},
+			Server:       mockserver.Dummy(),
+			ExpectedErrs: []error{common.ErrMissingObjects},
+		},
+		{
+			Name:         "Mime response header expected",
+			Input:        common.ReadParams{ObjectName: "contact"},
+			Server:       mockserver.Dummy(),
+			ExpectedErrs: []error{interpreter.ErrMissingContentType},
+		},
+		{
+			Name:  "Correct error message is understood from HTML response",
+			Input: common.ReadParams{ObjectName: "contact"},
+			Server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				w.Header().Set("Content-Type", "text/html")
+				w.WriteHeader(http.StatusBadRequest)
+				_, _ = w.Write(responseInvalidPath)
+			})),
+			ExpectedErrs: []error{
+				common.ErrBadRequest,
+				errors.New("Cannot GET /api/v1/butterflies"), // nolint:goerr113
+			},
+		},
+		{
+			Name:  "Incorrect data type in payload",
+			Input: common.ReadParams{ObjectName: "contact"},
+			Server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				w.Header().Set("Content-Type", "application/json")
+				w.WriteHeader(http.StatusOK)
+				mockutils.WriteBody(w, `{}`)
+			})),
+			ExpectedErrs: []error{jsonquery.ErrNotArray},
+		},
+		{
+			Name:  "Empty read response",
+			Input: common.ReadParams{ObjectName: "campaigns"},
+			Server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				w.Header().Set("Content-Type", "application/json")
+				w.WriteHeader(http.StatusOK)
+				_, _ = w.Write(responseCampaignsEmpty)
+			})),
+			Expected: &common.ReadResult{
+				Rows:     0,
+				Data:     []common.ReadResultRow{},
+				NextPage: "",
+				Done:     true,
+			},
+			ExpectedErrs: nil,
+		},
+		{
+			Name:  "Successful read with chosen fields",
+			Input: common.ReadParams{ObjectName: "campaigns", Fields: []string{"name", "status"}},
+			Server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				w.Header().Set("Content-Type", "application/json")
+				w.WriteHeader(http.StatusOK)
+				_, _ = w.Write(responseCampaigns)
+			})),
+			Comparator: func(baseURL string, actual, expected *common.ReadResult) bool {
+				// custom comparison focuses on subset of fields to keep the test short
+				return mockutils.ReadResultComparator.SubsetFields(actual, expected) &&
+					mockutils.ReadResultComparator.SubsetRaw(actual, expected) &&
+					actual.NextPage.String() == expected.NextPage.String() &&
+					actual.Done == expected.Done
+			},
+			Expected: &common.ReadResult{
+				Rows: 3,
+				Data: []common.ReadResultRow{{
+					Fields: map[string]any{
+						"name":   "Monthly sales",
+						"status": "DRAFTED",
+					},
+					Raw: map[string]any{"id": float64(549768)},
+				}, {
+					Fields: map[string]any{
+						"name":   "Weekly events",
+						"status": "DRAFTED",
+					},
+					Raw: map[string]any{"id": float64(549772)},
+				}, {
+					Fields: map[string]any{
+						"name":   "Black Friday prep",
+						"status": "DRAFTED",
+					},
+					Raw: map[string]any{"id": float64(549773)},
+				}},
+				NextPage: "",
+				Done:     true,
+			},
+			ExpectedErrs: nil,
+		},
+	}
+
+	for _, tt := range tests {
+		// nolint:varnamelen
+		tt := tt // rebind, omit loop side effects for parallel goroutine
+		t.Run(tt.Name, func(t *testing.T) {
+			t.Parallel()
+
+			tt.Run(t, func() (connectors.ReadConnector, error) {
+				return constructTestConnector(tt.Server.URL)
+			})
+		})
+	}
+}
+
+func constructTestConnector(serverURL string) (*Connector, error) {
+	connector, err := NewConnector(
+		WithAuthenticatedClient(http.DefaultClient),
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	// for testing we want to redirect calls to our mock server
+	connector.setBaseURL(serverURL)
+
+	return connector, nil
+}
diff --git a/smartlead/test/read-campaign-empty.json b/smartlead/test/read-campaign-empty.json
new file mode 100644
index 00000000..fe51488c
--- /dev/null
+++ b/smartlead/test/read-campaign-empty.json
@@ -0,0 +1,1 @@
+[]
diff --git a/smartlead/test/read-campaign.json b/smartlead/test/read-campaign.json
new file mode 100644
index 00000000..2c745029
--- /dev/null
+++ b/smartlead/test/read-campaign.json
@@ -0,0 +1,59 @@
+[
+  {
+    "id": 549768,
+    "user_id": 59646,
+    "created_at": "2024-08-12T15:54:37.014Z",
+    "updated_at": "2024-08-12T15:54:37.014Z",
+    "status": "DRAFTED",
+    "name": "Monthly sales",
+    "track_settings": [],
+    "scheduler_cron_value": null,
+    "min_time_btwn_emails": 20,
+    "max_leads_per_day": 100,
+    "stop_lead_settings": "REPLY_TO_AN_EMAIL",
+    "enable_ai_esp_matching": false,
+    "send_as_plain_text": false,
+    "follow_up_percentage": 100,
+    "unsubscribe_text": null,
+    "parent_campaign_id": null,
+    "client_id": null
+  },
+  {
+    "id": 549772,
+    "user_id": 59646,
+    "created_at": "2024-08-12T15:55:07.452Z",
+    "updated_at": "2024-08-12T15:55:07.452Z",
+    "status": "DRAFTED",
+    "name": "Weekly events",
+    "track_settings": [],
+    "scheduler_cron_value": null,
+    "min_time_btwn_emails": 20,
+    "max_leads_per_day": 100,
+    "stop_lead_settings": "REPLY_TO_AN_EMAIL",
+    "enable_ai_esp_matching": false,
+    "send_as_plain_text": false,
+    "follow_up_percentage": 100,
+    "unsubscribe_text": null,
+    "parent_campaign_id": null,
+    "client_id": null
+  },
+  {
+    "id": 549773,
+    "user_id": 59646,
+    "created_at": "2024-08-12T15:55:35.435Z",
+    "updated_at": "2024-08-12T15:55:35.435Z",
+    "status": "DRAFTED",
+    "name": "Black Friday prep",
+    "track_settings": [],
+    "scheduler_cron_value": null,
+    "min_time_btwn_emails": 20,
+    "max_leads_per_day": 100,
+    "stop_lead_settings": "REPLY_TO_AN_EMAIL",
+    "enable_ai_esp_matching": false,
+    "send_as_plain_text": false,
+    "follow_up_percentage": 100,
+    "unsubscribe_text": null,
+    "parent_campaign_id": null,
+    "client_id": null
+  }
+]
diff --git a/smartlead/test/read-invalid-path.html b/smartlead/test/read-invalid-path.html
new file mode 100644
index 00000000..a194da89
--- /dev/null
+++ b/smartlead/test/read-invalid-path.html
@@ -0,0 +1,13 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+    <meta charset="utf-8">
+    <title>Error</title>
+</head>
+
+<body>
+<pre>Cannot GET /api/v1/butterflies</pre>
+</body>
+
+</html>
diff --git a/test/smartlead/connector.go b/test/smartlead/connector.go
new file mode 100644
index 00000000..9c98aad2
--- /dev/null
+++ b/test/smartlead/connector.go
@@ -0,0 +1,27 @@
+package smartlead
+
+import (
+	"context"
+	"net/http"
+
+	"github.com/amp-labs/connectors/common/scanning/credscanning"
+	"github.com/amp-labs/connectors/providers"
+	"github.com/amp-labs/connectors/smartlead"
+	"github.com/amp-labs/connectors/test/utils"
+)
+
+func GetSmartleadConnector(ctx context.Context) *smartlead.Connector {
+	filePath := credscanning.LoadPath(providers.Smartlead)
+	reader := utils.MustCreateProvCredJSON(filePath, false, false)
+
+	conn, err := smartlead.NewConnector(
+		smartlead.WithClient(ctx, http.DefaultClient,
+			reader.Get(credscanning.Fields.ApiKey),
+		),
+	)
+	if err != nil {
+		utils.Fail("error creating connector", "error", err)
+	}
+
+	return conn
+}
diff --git a/test/smartlead/read/main.go b/test/smartlead/read/main.go
new file mode 100644
index 00000000..fc0a973a
--- /dev/null
+++ b/test/smartlead/read/main.go
@@ -0,0 +1,42 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"github.com/amp-labs/connectors/common"
+	connTest "github.com/amp-labs/connectors/test/smartlead"
+	"github.com/amp-labs/connectors/test/utils"
+)
+
+var (
+	objectName = "campaigns" // nolint: gochecknoglobals
+)
+
+func main() {
+	// Handle Ctrl-C gracefully.
+	ctx, done := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
+	defer done()
+
+	// Set up slog logging.
+	utils.SetupLogging()
+
+	conn := connTest.GetSmartleadConnector(ctx)
+	defer utils.Close(conn)
+
+	res, err := conn.Read(ctx, common.ReadParams{
+		ObjectName: objectName,
+		Fields: []string{
+			"name", "status", "user_id",
+		},
+	})
+	if err != nil {
+		utils.Fail("error reading from Smartlead", "error", err)
+	}
+
+	fmt.Println("Reading campaign..")
+	utils.DumpJSON(res, os.Stdout)
+}