Repository: amp-labs/connectors
Pull request #1240 (2024-12-19 23:03:04)

[CON-807] Iterable: Read
# Testing

![image](https://github.com/user-attachments/assets/f86a0972-c611-46ff-869d-df13b1b27fee)


Cobalt0s: > [!WARNING]
> <b>This pull request is not mergeable via GitHub because a downstack PR is open. Once all requirements are satisfied, merge this PR as a stack <a href="https://app.graphite.dev/github/pr/amp-labs/connectors/1240?utm_source=stack-comment-downstack-mergeability-warning" >on Graphite</a>.</b>
> <a href="https://graphite.dev/docs/merge-pull-requests">Learn more</a>

* **#1242** <a href="https://app.graphite.dev/github/pr/amp-labs/connectors/1242?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a>
* **#1241** <a href="https://app.graphite.dev/github/pr/amp-labs/connectors/1241?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a>
* **#1240** <a href="https://app.graphite.dev/github/pr/amp-labs/connectors/1240?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a> 👈 <a href="https://app.graphite.dev/github/pr/amp-labs/connectors/1240?utm_source=stack-comment-view-in-graphite" target="_blank">(View in Graphite)</a>
* **#1239** <a href="https://app.graphite.dev/github/pr/amp-labs/connectors/1239?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a>
* `main`


<h2></h2>

This stack of pull requests is managed by <a href="https://graphite.dev?utm-source=stack-comment"><b>Graphite</b></a>. Learn more about <a href="https://stacking.dev/?utm_source=stack-comment">stacking</a>.
<!-- Current dependencies on/for this PR: -->
laurenzlong: Reassigning to @RajatPawar since Chris is OOO
Cobalt0s: Since, some time has passed, I verified that rerunning the script produces the same expected result.
![image](https://github.com/user-attachments/assets/439dac67-6770-4cc4-a74e-fdd33b9fe3e4)


Patch:
diff --git a/common/parse.go b/common/parse.go
index 127b2a60..dee6353c 100644
--- a/common/parse.go
+++ b/common/parse.go
@@ -107,20 +107,17 @@ func GetMarshaledData(records []map[string]any, fields []string) ([]ReadResultRo
 	return data, nil
 }
 
-func GetRecordsUnderJSONPath(jsonPath string) RecordsFunc {
-	return func(node *ajson.Node) ([]map[string]any, error) {
-		arr, err := jsonquery.New(node).Array(jsonPath, false)
-		if err != nil {
-			return nil, err
-		}
+func GetRecordsUnderJSONPath(jsonPath string, nestedPath ...string) RecordsFunc {
+	return getRecords(false, jsonPath, nestedPath...)
+}
 
-		return jsonquery.Convertor.ArrayToMap(arr)
-	}
+func GetOptionalRecordsUnderJSONPath(jsonPath string, nestedPath ...string) RecordsFunc {
+	return getRecords(true, jsonPath, nestedPath...)
 }
 
-func GetOptionalRecordsUnderJSONPath(jsonPath string) RecordsFunc {
+func getRecords(optional bool, jsonPath string, nestedPath ...string) RecordsFunc {
 	return func(node *ajson.Node) ([]map[string]any, error) {
-		arr, err := jsonquery.New(node).Array(jsonPath, true)
+		arr, err := jsonquery.New(node, nestedPath...).Array(jsonPath, optional)
 		if err != nil {
 			return nil, err
 		}
diff --git a/providers/iterable/client.go b/providers/iterable/client.go
new file mode 100644
index 00000000..ea3eec31
--- /dev/null
+++ b/providers/iterable/client.go
@@ -0,0 +1,14 @@
+package iterable
+
+import (
+	"github.com/amp-labs/connectors/common"
+)
+
+// JSONHTTPClient returns the underlying JSON HTTP client.
+func (c *Connector) JSONHTTPClient() *common.JSONHTTPClient {
+	return c.Client
+}
+
+func (c *Connector) HTTPClient() *common.HTTPClient {
+	return c.Client.HTTPClient
+}
diff --git a/providers/iterable/connector.go b/providers/iterable/connector.go
new file mode 100644
index 00000000..c6892abe
--- /dev/null
+++ b/providers/iterable/connector.go
@@ -0,0 +1,67 @@
+package iterable
+
+import (
+	"github.com/amp-labs/connectors/common"
+	"github.com/amp-labs/connectors/common/interpreter"
+	"github.com/amp-labs/connectors/common/paramsbuilder"
+	"github.com/amp-labs/connectors/common/urlbuilder"
+	"github.com/amp-labs/connectors/providers"
+	"github.com/amp-labs/connectors/providers/iterable/metadata"
+)
+
+type Connector struct {
+	BaseURL string
+	Client  *common.JSONHTTPClient
+	Module  common.Module
+}
+
+func NewConnector(opts ...Option) (conn *Connector, outErr error) {
+	params, err := paramsbuilder.Apply(parameters{}, opts)
+	if err != nil {
+		return nil, err
+	}
+
+	httpClient := params.Client.Caller
+	conn = &Connector{
+		Client: &common.JSONHTTPClient{
+			HTTPClient: httpClient,
+		},
+	}
+
+	// Read provider info
+	providerInfo, err := providers.ReadInfo(conn.Provider())
+	if err != nil {
+		return nil, err
+	}
+
+	// connector and its client must mirror base url and provide its own error parser
+	conn.setBaseURL(providerInfo.BaseURL)
+	conn.Client.HTTPClient.ErrorHandler = interpreter.ErrorHandler{
+		JSON: interpreter.NewFaultyResponder(errorFormats, nil),
+		HTML: &interpreter.DirectFaultyResponder{Callback: conn.interpretHTMLError},
+	}.Handle
+
+	return conn, nil
+}
+
+func (c *Connector) getReadURL(objectName string) (*urlbuilder.URL, error) {
+	path, err := metadata.Schemas.LookupURLPath(c.Module.ID, objectName)
+	if err != nil {
+		return nil, err
+	}
+
+	return urlbuilder.New(c.BaseURL, path)
+}
+
+func (c *Connector) setBaseURL(newURL string) {
+	c.BaseURL = newURL
+	c.Client.HTTPClient.Base = newURL
+}
+
+func (c *Connector) Provider() providers.Provider {
+	return providers.Iterable
+}
+
+func (c *Connector) String() string {
+	return c.Provider() + ".Connector"
+}
diff --git a/providers/iterable/errors.go b/providers/iterable/errors.go
new file mode 100644
index 00000000..c5313511
--- /dev/null
+++ b/providers/iterable/errors.go
@@ -0,0 +1,79 @@
+package iterable
+
+import (
+	"bytes"
+	"fmt"
+	"net/http"
+	"strings"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/amp-labs/connectors/common/interpreter"
+)
+
+var errorFormats = interpreter.NewFormatSwitch( // nolint:gochecknoglobals
+	[]interpreter.FormatTemplate{
+		{
+			MustKeys: nil,
+			Template: func() interpreter.ErrorDescriptor { return &ResponseError{} },
+		},
+	}...,
+)
+
+type ResponseError struct {
+	Message string `json:"msg"`
+	Code    string `json:"code"`
+	Params  any    `json:"params"`
+}
+
+func (r ResponseError) CombineErr(base error) error {
+	if len(r.Message) == 0 {
+		return base
+	}
+
+	return fmt.Errorf("%w: %v", base, r.Message)
+}
+
+func (c *Connector) interpretHTMLError(res *http.Response, body []byte) error {
+	base := interpreter.DefaultStatusCodeMappingToErr(res, body)
+
+	document, err := goquery.NewDocumentFromReader(bytes.NewReader(body))
+	if err != nil {
+		// ignore HTML that cannot be understood
+		return base
+	}
+
+	// Several HTML errors have different formats, given a small sample -- it is safe to use
+	// title as primary message.
+	// For examples refer to the `test` directory which has `.html` samples used in unit testing.
+	message := document.Find("title").Text()
+	h2 := document.Find("h2").Text()
+	detail := document.Find("#detail").Text()
+
+	// Detail element has better message than Header2.
+	errReason := h2
+	if len(detail) != 0 {
+		errReason = detail
+	}
+
+	if len(errReason) != 0 {
+		message += ": " + errReason
+	}
+
+	message = cleanMessage(message)
+
+	return fmt.Errorf("%w: %v", base, message)
+}
+
+func cleanMessage(message string) string {
+	// Remove new lines and double spaces.
+	r := strings.NewReplacer("\n", "", "  ", "")
+
+	size := 0
+	for size != len(message) {
+		// Replace until nothing changes.
+		size = len(message)
+		message = r.Replace(message)
+	}
+
+	return message
+}
diff --git a/providers/iterable/metadata/metadata.go b/providers/iterable/metadata/metadata.go
new file mode 100644
index 00000000..8e928f4b
--- /dev/null
+++ b/providers/iterable/metadata/metadata.go
@@ -0,0 +1,20 @@
+package metadata
+
+import (
+	_ "embed"
+
+	"github.com/amp-labs/connectors/tools/fileconv"
+	"github.com/amp-labs/connectors/tools/scrapper"
+)
+
+var (
+	// Static file containing a list of object metadata is embedded and can be served.
+	//
+	//go:embed schemas.json
+	schemas []byte
+
+	FileManager = scrapper.NewMetadataFileManager(schemas, fileconv.NewSiblingFileLocator()) // nolint:gochecknoglobals
+
+	// Schemas is cached Object schemas.
+	Schemas = FileManager.MustLoadSchemas() // nolint:gochecknoglobals
+)
diff --git a/providers/iterable/metadata/schemas.json b/providers/iterable/metadata/schemas.json
new file mode 100755
index 00000000..2bee797c
--- /dev/null
+++ b/providers/iterable/metadata/schemas.json
@@ -0,0 +1,162 @@
+{
+  "modules": {
+    "root": {
+      "id": "root",
+      "path": "/api",
+      "objects": {
+        "campaigns": {
+          "displayName": "Campaigns",
+          "path": "/campaigns",
+          "responseKey": "campaigns",
+          "fields": {
+            "campaignState": "campaignState",
+            "createdAt": "createdAt",
+            "createdByUserId": "createdByUserId",
+            "endedAt": "endedAt",
+            "id": "id",
+            "labels": "labels",
+            "listIds": "listIds",
+            "messageMedium": "messageMedium",
+            "name": "name",
+            "recurringCampaignId": "recurringCampaignId",
+            "sendSize": "sendSize",
+            "startAt": "startAt",
+            "suppressionListIds": "suppressionListIds",
+            "templateId": "templateId",
+            "type": "type",
+            "updatedAt": "updatedAt",
+            "updatedByUserId": "updatedByUserId",
+            "workflowId": "workflowId"
+          }
+        },
+        "catalogs": {
+          "displayName": "Catalogs",
+          "path": "/catalogs",
+          "responseKey": "catalogNames",
+          "fields": {
+            "name": "name"
+          }
+        },
+        "channels": {
+          "displayName": "Channels",
+          "path": "/channels",
+          "responseKey": "channels",
+          "fields": {
+            "channelType": "channelType",
+            "id": "id",
+            "messageMedium": "messageMedium",
+            "name": "name"
+          }
+        },
+        "jobs": {
+          "displayName": "Jobs",
+          "path": "/export/jobs",
+          "responseKey": "jobs",
+          "fields": {
+            "bytesExported": "bytesExported",
+            "dataTypeName": "dataTypeName",
+            "endTime": "endTime",
+            "id": "id",
+            "jobState": "jobState",
+            "scheduledStartTime": "scheduledStartTime"
+          }
+        },
+        "journeys": {
+          "displayName": "Journeys",
+          "path": "/journeys",
+          "responseKey": "journeys",
+          "fields": {
+            "createdAt": "createdAt",
+            "creatorUserId": "creatorUserId",
+            "description": "description",
+            "draft": "draft",
+            "enabled": "enabled",
+            "id": "id",
+            "isArchived": "isArchived",
+            "journeyType": "journeyType",
+            "lifetimeLimit": "lifetimeLimit",
+            "name": "name",
+            "simultaneousLimit": "simultaneousLimit",
+            "startTileId": "startTileId",
+            "triggerEventNames": "triggerEventNames",
+            "updatedAt": "updatedAt"
+          }
+        },
+        "lists": {
+          "displayName": "Lists",
+          "path": "/lists",
+          "responseKey": "lists",
+          "fields": {
+            "createdAt": "createdAt",
+            "description": "description",
+            "id": "id",
+            "listType": "listType",
+            "name": "name"
+          }
+        },
+        "messageTypes": {
+          "displayName": "Message Types",
+          "path": "/messageTypes",
+          "responseKey": "messageTypes",
+          "fields": {
+            "channelId": "channelId",
+            "createdAt": "createdAt",
+            "frequencyCap": "frequencyCap",
+            "id": "id",
+            "name": "name",
+            "rateLimitPerMinute": "rateLimitPerMinute",
+            "subscriptionPolicy": "subscriptionPolicy",
+            "updatedAt": "updatedAt"
+          }
+        },
+        "messages": {
+          "displayName": "Messages",
+          "path": "/embedded-messaging/messages",
+          "responseKey": "placements",
+          "fields": {
+            "embeddedMessages": "embeddedMessages",
+            "placementId": "placementId"
+          }
+        },
+        "metadata": {
+          "displayName": "Metadata",
+          "path": "/metadata",
+          "responseKey": "results",
+          "fields": {
+            "name": "name"
+          }
+        },
+        "templates": {
+          "displayName": "Templates",
+          "path": "/templates",
+          "responseKey": "templates",
+          "fields": {
+            "campaignId": "campaignId",
+            "clientTemplateId": "clientTemplateId",
+            "createdAt": "createdAt",
+            "creatorUserId": "creatorUserId",
+            "messageTypeId": "messageTypeId",
+            "name": "name",
+            "templateId": "templateId",
+            "updatedAt": "updatedAt"
+          }
+        },
+        "webhooks": {
+          "displayName": "Webhooks",
+          "path": "/webhooks",
+          "responseKey": "webhooks",
+          "fields": {
+            "authType": "authType",
+            "blastSendEnabled": "blastSendEnabled",
+            "channelIds": "channelIds",
+            "enabled": "enabled",
+            "endpoint": "endpoint",
+            "id": "id",
+            "messageTypeIds": "messageTypeIds",
+            "triggeredSendEnabled": "triggeredSendEnabled"
+          }
+        }
+      }
+    }
+  }
+}
\ No newline at end of file
diff --git a/providers/iterable/objectNames.go b/providers/iterable/objectNames.go
new file mode 100644
index 00000000..0b272c4f
--- /dev/null
+++ b/providers/iterable/objectNames.go
@@ -0,0 +1,24 @@
+package iterable
+
+import (
+	"github.com/amp-labs/connectors/internal/datautils"
+	"github.com/amp-labs/connectors/providers/iterable/metadata"
+)
+
+const (
+	objectNameCatalogs  = "catalogs"
+	objectNameJourneys  = "journeys"
+	objectNameTemplates = "templates"
+)
+
+var paginatedObjects = datautils.NewSet( //nolint:gochecknoglobals
+	objectNameCatalogs,
+	objectNameJourneys,
+)
+
+var incrementalReadObjects = datautils.NewSet( //nolint:gochecknoglobals
+	objectNameTemplates,
+)
+
+// Supported object names can be found under schemas.json.
+var supportedObjectsByRead = metadata.Schemas.ObjectNames() //nolint:gochecknoglobals
diff --git a/providers/iterable/params.go b/providers/iterable/params.go
new file mode 100644
index 00000000..ec5d3368
--- /dev/null
+++ b/providers/iterable/params.go
@@ -0,0 +1,41 @@
+package iterable
+
+import (
+	"context"
+	"errors"
+	"net/http"
+
+	"github.com/amp-labs/connectors/common"
+	"github.com/amp-labs/connectors/common/paramsbuilder"
+	"github.com/amp-labs/connectors/providers"
+)
+
+// DefaultPageSize is number of elements per page.
+const DefaultPageSize = 50
+
+// Option is a function which mutates the connector configuration.
+type Option = func(params *parameters)
+
+type parameters struct {
+	paramsbuilder.Client
+}
+
+func (p parameters) ValidateParams() error {
+	return errors.Join(
+		p.Client.ValidateParams(),
+	)
+}
+
+func WithClient(ctx context.Context, client *http.Client,
+	apiKey string, opts ...common.HeaderAuthClientOption,
+) Option {
+	return func(params *parameters) {
+		params.WithApiKeyHeaderClient(ctx, client, providers.Iterable, apiKey, opts...)
+	}
+}
+
+func WithAuthenticatedClient(client common.AuthenticatedHTTPClient) Option {
+	return func(params *parameters) {
+		params.WithAuthenticatedClient(client)
+	}
+}
diff --git a/providers/iterable/parse.go b/providers/iterable/parse.go
new file mode 100644
index 00000000..ac352bfd
--- /dev/null
+++ b/providers/iterable/parse.go
@@ -0,0 +1,49 @@
+package iterable
+
+import (
+	"github.com/amp-labs/connectors/common"
+	"github.com/amp-labs/connectors/common/jsonquery"
+	"github.com/amp-labs/connectors/providers/iterable/metadata"
+	"github.com/spyzhov/ajson"
+)
+
+func makeGetRecords(moduleID common.ModuleID, objectName string) common.RecordsFunc {
+	return func(node *ajson.Node) ([]map[string]any, error) {
+		responseFieldName := metadata.Schemas.LookupArrayFieldName(moduleID, objectName)
+
+		var nestedPath []string
+		if objectName == objectNameCatalogs {
+			// For some reason this is the only object that is nested.
+			// For reference, the parent object is "IterableApiResponse", which can be found under OpenAPI.
+			nestedPath = []string{"params"}
+		}
+
+		return common.GetOptionalRecordsUnderJSONPath(responseFieldName, nestedPath...)(node)
+	}
+}
+
+func makeNextRecordsURL(baseURL string) common.NextPageFunc {
+	return func(node *ajson.Node) (string, error) {
+		nextPage, err := jsonquery.New(node).StrWithDefault("nextPageUrl", "")
+		if err != nil {
+			return "", err
+		}
+
+		if len(nextPage) == 0 {
+			// Next page URL could be nested under params object.
+			nextPage, err = jsonquery.New(node, "params").StrWithDefault("nextPageUrl", "")
+			if err != nil {
+				return "", err
+			}
+		}
+
+		if len(nextPage) == 0 {
+			// Next page doesn't exist
+			return "", nil
+		}
+
+		fullURL := baseURL + nextPage
+
+		return fullURL, nil
+	}
+}
diff --git a/providers/iterable/read.go b/providers/iterable/read.go
new file mode 100644
index 00000000..52a0279c
--- /dev/null
+++ b/providers/iterable/read.go
@@ -0,0 +1,61 @@
+package iterable
+
+import (
+	"context"
+	"strconv"
+
+	"github.com/amp-labs/connectors/common"
+	"github.com/amp-labs/connectors/common/urlbuilder"
+	"github.com/amp-labs/connectors/internal/datautils"
+)
+
+func (c *Connector) Read(ctx context.Context, config common.ReadParams) (*common.ReadResult, error) {
+	if err := config.ValidateParams(true); err != nil {
+		return nil, err
+	}
+
+	if !supportedObjectsByRead[c.Module.ID].Has(config.ObjectName) {
+		return nil, common.ErrOperationNotSupportedForObject
+	}
+
+	url, err := c.buildReadURL(config)
+	if err != nil {
+		return nil, err
+	}
+
+	res, err := c.Client.Get(ctx, url.String())
+	if err != nil {
+		return nil, err
+	}
+
+	return common.ParseResult(res,
+		makeGetRecords(c.Module.ID, config.ObjectName),
+		makeNextRecordsURL(c.BaseURL),
+		common.GetMarshaledData,
+		config.Fields,
+	)
+}
+
+func (c *Connector) buildReadURL(config common.ReadParams) (*urlbuilder.URL, error) {
+	if len(config.NextPage) != 0 {
+		// Next page
+		return urlbuilder.New(config.NextPage.String())
+	}
+
+	// First page
+	url, err := c.getReadURL(config.ObjectName)
+	if err != nil {
+		return nil, err
+	}
+
+	if paginatedObjects.Has(config.ObjectName) {
+		url.WithQueryParam("pageSize", strconv.Itoa(DefaultPageSize))
+	}
+
+	if incrementalReadObjects.Has(config.ObjectName) && !config.Since.IsZero() {
+		sinceValue := datautils.Time.FormatRFC3339inUTC(config.Since)
+		url.WithQueryParam("startDateTime", sinceValue)
+	}
+
+	return url, nil
+}


Test patch:
diff --git a/providers/iterable/read_test.go b/providers/iterable/read_test.go
new file mode 100644
index 00000000..5c83aeb9
--- /dev/null
+++ b/providers/iterable/read_test.go
@@ -0,0 +1,185 @@
+package iterable
+
+import (
+	"errors"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/amp-labs/connectors"
+	"github.com/amp-labs/connectors/common"
+	"github.com/amp-labs/connectors/test/utils/mockutils/mockcond"
+	"github.com/amp-labs/connectors/test/utils/mockutils/mockserver"
+	"github.com/amp-labs/connectors/test/utils/testroutines"
+	"github.com/amp-labs/connectors/test/utils/testutils"
+)
+
+func TestRead(t *testing.T) { //nolint:funlen,gocognit,cyclop,maintidx
+	t.Parallel()
+
+	responseCatalogsFirst := testutils.DataFromFile(t, "read-catalogs-1-first-page.json")
+	responseCatalogsLast := testutils.DataFromFile(t, "read-catalogs-2-second-page.json")
+	errorTemplatesInvalidSince := testutils.DataFromFile(t, "read-templates-invalid-since.html")
+	responseTemplatesSince := testutils.DataFromFile(t, "read-templates-since.json")
+	errorInvalidRestMethod := testutils.DataFromFile(t, "error-wrong-method.html")
+
+	tests := []testroutines.Read{
+		{
+			Name:         "Read object must be included",
+			Input:        common.ReadParams{},
+			Server:       mockserver.Dummy(),
+			ExpectedErrs: []error{common.ErrMissingObjects},
+		},
+		{
+			Name:         "At least one field is requested",
+			Input:        common.ReadParams{ObjectName: "journeys"},
+			Server:       mockserver.Dummy(),
+			ExpectedErrs: []error{common.ErrMissingFields},
+		},
+		{
+			Name:     "Unknown object name is not supported",
+			Input:    common.ReadParams{ObjectName: "accounts", Fields: connectors.Fields("id")},
+			Server:   mockserver.Dummy(),
+			Expected: nil,
+			ExpectedErrs: []error{
+				common.ErrOperationNotSupportedForObject,
+			},
+		},
+		{
+			Name:  "Invalid usage of REST operation",
+			Input: common.ReadParams{ObjectName: "templates", Fields: connectors.Fields("templateId")},
+			Server: mockserver.Fixed{
+				Setup:  mockserver.ContentHTML(),
+				Always: mockserver.Response(http.StatusNotFound, errorInvalidRestMethod),
+			}.Server(),
+			ExpectedErrs: []error{
+				common.ErrBadRequest,
+				errors.New( // nolint:goerr113
+					"not found: Action not found: Oh no- that url doesn't exist."),
+			},
+		},
+		{
+			Name: "Catalogs first page has a link to the next",
+			Input: common.ReadParams{
+				ObjectName: "catalogs",
+				Fields:     connectors.Fields("name"),
+			},
+			Server: mockserver.Conditional{
+				Setup: mockserver.ContentJSON(),
+				If:    mockcond.PathSuffix("/api/catalogs"),
+				Then:  mockserver.Response(http.StatusOK, responseCatalogsFirst),
+			}.Server(),
+			Comparator: testroutines.ComparatorPagination,
+			Expected: &common.ReadResult{
+				Rows:     1,
+				Data:     []common.ReadResultRow{},
+				NextPage: testroutines.URLTestServer + "/api/catalogs?pageSize=1&page=2", // nolint:lll
+				Done:     false,
+			},
+			ExpectedErrs: nil,
+		},
+		{
+			Name: "Catalogs last page has no link to the next",
+			Input: common.ReadParams{
+				ObjectName: "catalogs",
+				Fields:     connectors.Fields("name"),
+			},
+			Server: mockserver.Conditional{
+				Setup: mockserver.ContentJSON(),
+				If:    mockcond.PathSuffix("/api/catalogs"),
+				Then:  mockserver.Response(http.StatusOK, responseCatalogsLast),
+			}.Server(),
+			Comparator: testroutines.ComparatorPagination,
+			Expected: &common.ReadResult{
+				Rows:     1,
+				Data:     []common.ReadResultRow{},
+				NextPage: "",
+				Done:     true,
+			},
+			ExpectedErrs: nil,
+		},
+		{
+			Name:  "Incremental read of templates with invalid time format",
+			Input: common.ReadParams{ObjectName: "templates", Fields: connectors.Fields("templateId")},
+			Server: mockserver.Fixed{
+				Setup:  mockserver.ContentHTML(),
+				Always: mockserver.Response(http.StatusNotFound, errorTemplatesInvalidSince),
+			}.Server(),
+			ExpectedErrs: []error{
+				common.ErrBadRequest,
+				errors.New( // nolint:goerr113
+					"Unable to bind 1734632773073 into a date time!Acceptable formats are ISO8601"),
+			},
+		},
+		{
+			Name: "Incremental read of templates with required query parameter",
+			Input: common.ReadParams{
+				ObjectName: "templates",
+				Fields:     connectors.Fields("templateId"),
+				Since:      time.Date(2024, 12, 19, 18, 28, 41, 0, time.UTC),
+			},
+			Server: mockserver.Conditional{
+				Setup: mockserver.ContentJSON(),
+				If: mockcond.And{
+					mockcond.PathSuffix("/api/templates"),
+					mockcond.QueryParam("startDateTime", "2024-12-19T18:28:41Z"),
+				},
+				Then: mockserver.Response(http.StatusOK, responseTemplatesSince),
+			}.Server(),
+			Comparator: testroutines.ComparatorSubsetRead,
+			Expected: &common.ReadResult{
+				Rows: 3,
+				Data: []common.ReadResultRow{{
+					Fields: map[string]any{
+						"templateid": float64(15870445),
+					},
+					Raw: map[string]any{
+						"creatorUserId": "iterablepartners@gmail.com",
+					},
+				}, {
+					Fields: map[string]any{
+						"templateid": float64(15870451),
+					},
+					Raw: map[string]any{
+						"creatorUserId": "john@gmail.com",
+					},
+				}, {
+					Fields: map[string]any{
+						"templateid": float64(15870455),
+					},
+					Raw: map[string]any{
+						"creatorUserId": "emily@gmail.com",
+					},
+				}},
+				NextPage: "",
+				Done:     true,
+			},
+			ExpectedErrs: nil,
+		},
+	}
+
+	for _, tt := range tests {
+		// nolint:varnamelen
+		t.Run(tt.Name, func(t *testing.T) {
+			t.Parallel()
+
+			tt.Run(t, func() (connectors.ReadConnector, error) {
+				return constructTestConnector(tt.Server.URL)
+			})
+		})
+	}
+}
+
+func constructTestConnector(serverURL string) (*Connector, error) {
+	connector, err := NewConnector(
+		WithAuthenticatedClient(http.DefaultClient),
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	// for testing we want to redirect calls to our mock server
+	connector.setBaseURL(serverURL)
+
+	return connector, nil
+}
diff --git a/providers/iterable/test/error-wrong-method.html b/providers/iterable/test/error-wrong-method.html
new file mode 100644
index 00000000..0720c786
--- /dev/null
+++ b/providers/iterable/test/error-wrong-method.html
@@ -0,0 +1,35 @@
+<!doctype html>
+<html>
+
+<head>
+    <meta charset="utf-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
+    <title ng-non-bindable>Action not found</title>
+    <meta name="description" content="Action not found" ng-non-bindable>
+    <meta name="viewport" content="width=device-width, maximum-scale=1, user-scalable=no">
+    <link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css" />
+    <link href='//fonts.googleapis.com/css?family=Inconsolata' rel='stylesheet' type='text/css'>
+    <link rel="stylesheet" href="https://use.typekit.net/adq8zms.css">
+
+
+
+
+
+    <link rel="stylesheet" ng-non-bindable media="screen"
+          href="https://assets.iterable.com/static/iterable-s3/vendor.a5119182066685c6b1f5.css.gz" />
+
+
+
+</head>
+
+<body>
+
+<div class="error-content" ng-non-bindable>
+    <h2> <img src="/assets/images/dead.svg" alt="Broken"/> Oh no- that url doesn't exist.</h2>
+    <h4>https://api.iterable.com/api/templates/push/upsert</h4>
+    <p>Please try again or contact the sender of this message.</p>
+</div>
+
+</body>
+
+</html>
diff --git a/providers/iterable/test/read-catalogs-1-first-page.json b/providers/iterable/test/read-catalogs-1-first-page.json
new file mode 100644
index 00000000..bee21c71
--- /dev/null
+++ b/providers/iterable/test/read-catalogs-1-first-page.json
@@ -0,0 +1,13 @@
+{
+  "msg": "",
+  "code": "Success",
+  "params": {
+    "catalogNames": [
+      {
+        "name": "catalogName1"
+      }
+    ],
+    "totalCatalogsCount": 2,
+    "nextPageUrl": "/api/catalogs?pageSize=1&page=2"
+  }
+}
diff --git a/providers/iterable/test/read-catalogs-2-second-page.json b/providers/iterable/test/read-catalogs-2-second-page.json
new file mode 100644
index 00000000..fa8c1409
--- /dev/null
+++ b/providers/iterable/test/read-catalogs-2-second-page.json
@@ -0,0 +1,13 @@
+{
+  "msg": "",
+  "code": "Success",
+  "params": {
+    "catalogNames": [
+      {
+        "name": "catalogName"
+      }
+    ],
+    "totalCatalogsCount": 2,
+    "previousPageUrl": "/api/catalogs?pageSize=1&page=1"
+  }
+}
diff --git a/providers/iterable/test/read-templates-invalid-since.html b/providers/iterable/test/read-templates-invalid-since.html
new file mode 100644
index 00000000..59237f39
--- /dev/null
+++ b/providers/iterable/test/read-templates-invalid-since.html
@@ -0,0 +1,51 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+    <title>Bad Request</title>
+
+    <style nonce="A1+mx8a2Ft9RByX7W6GA8Q==" type="text/css">
+        html,
+        body,
+        pre {
+            margin: 0;
+            padding: 0;
+            font-family: Monaco, 'Lucida Console', monospace;
+            background: #ECECEC;
+        }
+
+        h1 {
+            margin: 0;
+            background: #AD632A;
+            padding: 20px 45px;
+            color: #fff;
+            text-shadow: 1px 1px 1px rgba(0, 0, 0, .3);
+            border-bottom: 1px solid #9F5805;
+            font-size: 28px;
+        }
+
+        p#detail {
+            margin: 0;
+            padding: 15px 45px;
+            background: #F6A960;
+            border-top: 4px solid #D29052;
+            color: #733512;
+            text-shadow: 1px 1px 1px rgba(255, 255, 255, .3);
+            font-size: 14px;
+            border-bottom: 1px solid #BA7F5B;
+        }
+    </style>
+</head>
+
+<body>
+<h1>Bad Request</h1>
+
+<p id="detail">
+    For request 'GET /api/templates?startDateTime=1734632773073' [Unable to bind 1734632773073 into a date time!
+    Acceptable formats are ISO8601 with/without millis, and yyyy-MM-dd HH:mm:ss [ZZ] (Unable to parse 1734632773073
+    as a date in ES formats)]
+</p>
+
+</body>
+
+</html>
diff --git a/providers/iterable/test/read-templates-since.json b/providers/iterable/test/read-templates-since.json
new file mode 100644
index 00000000..6507c5fd
--- /dev/null
+++ b/providers/iterable/test/read-templates-since.json
@@ -0,0 +1,28 @@
+{
+  "templates": [
+    {
+      "templateId": 15870445,
+      "createdAt": 1734632955626,
+      "updatedAt": 1734632955626,
+      "name": "Template created on Thu, Dec 19 2024, 10:29 AM",
+      "creatorUserId": "iterablepartners@gmail.com",
+      "messageTypeId": 134467
+    },
+    {
+      "templateId": 15870451,
+      "createdAt": 1734632981784,
+      "updatedAt": 1734632981784,
+      "name": "Template created on Thu, Dec 19 2024, 10:29 AM",
+      "creatorUserId": "john@gmail.com",
+      "messageTypeId": 134467
+    },
+    {
+      "templateId": 15870455,
+      "createdAt": 1734632997386,
+      "updatedAt": 1734632997386,
+      "name": "Template created on Thu, Dec 19 2024, 10:29 AM",
+      "creatorUserId": "emily@gmail.com",
+      "messageTypeId": 134467
+    }
+  ]
+}
diff --git a/test/iterable/connector.go b/test/iterable/connector.go
new file mode 100644
index 00000000..b1babfdd
--- /dev/null
+++ b/test/iterable/connector.go
@@ -0,0 +1,27 @@
+package iterable
+
+import (
+	"context"
+	"net/http"
+
+	"github.com/amp-labs/connectors/common/scanning/credscanning"
+	"github.com/amp-labs/connectors/providers"
+	"github.com/amp-labs/connectors/providers/iterable"
+	"github.com/amp-labs/connectors/test/utils"
+)
+
+func GetIterableConnector(ctx context.Context) *iterable.Connector {
+	filePath := credscanning.LoadPath(providers.Iterable)
+	reader := utils.MustCreateProvCredJSON(filePath, false, false)
+
+	conn, err := iterable.NewConnector(
+		iterable.WithClient(ctx, http.DefaultClient,
+			reader.Get(credscanning.Fields.ApiKey),
+		),
+	)
+	if err != nil {
+		utils.Fail("error creating Iterable connector", "error", err)
+	}
+
+	return conn
+}
diff --git a/test/iterable/read/main.go b/test/iterable/read/main.go
new file mode 100644
index 00000000..eb61192e
--- /dev/null
+++ b/test/iterable/read/main.go
@@ -0,0 +1,36 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"github.com/amp-labs/connectors"
+	"github.com/amp-labs/connectors/common"
+	connTest "github.com/amp-labs/connectors/test/iterable"
+	"github.com/amp-labs/connectors/test/utils"
+)
+
+func main() {
+	// Handle Ctrl-C gracefully.
+	ctx, done := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
+	defer done()
+
+	// Set up slog logging.
+	utils.SetupLogging()
+
+	conn := connTest.GetIterableConnector(ctx)
+
+	res, err := conn.Read(ctx, common.ReadParams{
+		ObjectName: "channels",
+		Fields:     connectors.Fields("id", "name"),
+	})
+	if err != nil {
+		utils.Fail("error reading from Iterable", "error", err)
+	}
+
+	fmt.Println("Reading channels..")
+	utils.DumpJSON(res, os.Stdout)
+}