Repository: amp-labs/connectors
Pull request #232 (2024-04-18 13:05:22)

feat: Add outreach read connector
 This PR adds the Outreach read capability.

- Updated the jsonResponse to allow application/vnd+.json. All providers implementing JSONAPISpecification will have this as the Content-Type in the response.

Written test to read an Object from Outreach api. The test reads users object from the outreach API.

draft PR for #187 
   


Patch:
diff --git a/common/json.go b/common/json.go
index 4371eb50..f444b6be 100644
--- a/common/json.go
+++ b/common/json.go
@@ -97,8 +97,11 @@ func parseJSONResponse(res *http.Response, body []byte) (*JSONHTTPResponse, erro
 			return nil, fmt.Errorf("failed to parse content type: %w", err)
 		}
 
-		if mimeType != "application/json" {
-			return nil, fmt.Errorf("%w: expected content type to be application/json, got %s", ErrNotJSON, mimeType)
+		// Providers implementing JSONAPISpeicifcations returns application/vnd.api+json
+		if mimeType != "application/json" && mimeType != "application/vnd.api+json" {
+			return nil, fmt.Errorf("%w: expected content type to be application/json or application/vnd.api+json , got %s",
+				ErrNotJSON, mimeType,
+			)
 		}
 	}
 
diff --git a/connectors.go b/connectors.go
index 856a353e..38b54615 100644
--- a/connectors.go
+++ b/connectors.go
@@ -11,6 +11,7 @@ import (
 	"github.com/amp-labs/connectors/hubspot"
 	"github.com/amp-labs/connectors/microsoftdynamicscrm"
 	"github.com/amp-labs/connectors/mock"
+	"github.com/amp-labs/connectors/outreach"
 	"github.com/amp-labs/connectors/providers"
 	"github.com/amp-labs/connectors/salesforce"
 )
@@ -85,6 +86,9 @@ var MSDynamicsSales API[*microsoftdynamicscrm.Connector, microsoftdynamicscrm.Op
 // Mock is an API that returns a new Mock Connector.
 var Mock API[*mock.Connector, mock.Option] = mock.NewConnector //nolint:gochecknoglobals
 
+// Outreach is an API that returns a new Outreach Connector.
+var Outreach API[*outreach.Connector, outreach.Option] = outreach.NewConnector //nolint:gochecknoglobals
+
 // We re-export the following types so that they can be used by consumers of this library.
 type (
 	ReadParams               = common.ReadParams
@@ -129,6 +133,8 @@ func New(provider providers.Provider, opts map[string]any) (Connector, error) {
 		return newSalesforce(opts)
 	case providers.Hubspot:
 		return newHubspot(opts)
+	case providers.Outreach:
+		return newOutreach(opts)
 	default:
 		return nil, fmt.Errorf("%w: %s", ErrUnknownConnector, provider)
 	}
@@ -201,6 +207,18 @@ func newHubspot(opts map[string]any) (Connector, error) { //nolint:ireturn
 	return Hubspot.New(options...)
 }
 
+// newOutreach returns a new Outreach Connector, by unwrapping the options and passing them to the Outreach API.
+func newOutreach(opts map[string]any) (Connector, error) { //nolint:ireturn
+	var options []outreach.Option
+
+	c, valid := getParam[common.AuthenticatedHTTPClient](opts, "client")
+	if valid {
+		options = append(options, outreach.WithAuthenticatedClient(c))
+	}
+
+	return Outreach.New(options...)
+}
+
 // getParam returns the value of the given key, if present, safely cast to an assumed type.
 // If the key is not present, or the value is not of the assumed type, it returns the
 // zero value of the desired type, and false. In case of success, it returns the value and true.
diff --git a/outreach/README.md b/outreach/README.md
new file mode 100644
index 00000000..22145e31
--- /dev/null
+++ b/outreach/README.md
@@ -0,0 +1,24 @@
+# Outreach connector
+
+
+## Read
+Read is used to list all records of a given type. For example, if you want to list all users, you would use the `Read` method with the `users` object.
+
+### Example Usage
+
+```
+
+// Create the outreach connector instance 
+// This assumes you called the instance client
+
+// Call Read to list records in a users object
+res, err := client.Read(context.TODO(),common.ReadParams{
+		ObjectName: "users",
+        })
+if err != nil {
+	log.Fatal(err)
+}
+
+```
+
+
diff --git a/outreach/client.go b/outreach/client.go
new file mode 100644
index 00000000..05eb8122
--- /dev/null
+++ b/outreach/client.go
@@ -0,0 +1,16 @@
+package outreach
+
+import "github.com/amp-labs/connectors/common"
+
+// JSONHTTPClient returns the underlying JSON HTTP client.
+func (c *Connector) JSONHTTPClient() *common.JSONHTTPClient {
+	return c.Client
+}
+
+func (c *Connector) HTTPClient() *common.HTTPClient {
+	return c.Client.HTTPClient
+}
+
+func (c *Connector) Close() error {
+	return nil
+}
diff --git a/outreach/connector.go b/outreach/connector.go
new file mode 100644
index 00000000..43ec3901
--- /dev/null
+++ b/outreach/connector.go
@@ -0,0 +1,61 @@
+package outreach
+
+import (
+	"fmt"
+
+	"github.com/amp-labs/connectors/common"
+	"github.com/amp-labs/connectors/providers"
+)
+
+const (
+	providerOptionRestApiURL = "restAPIURL"
+)
+
+type Connector struct {
+	BaseURL string
+	Client  *common.JSONHTTPClient
+}
+
+func NewConnector(opts ...Option) (conn *Connector, outErr error) {
+	defer func() {
+		if re := recover(); re != nil {
+			tmp, ok := re.(error)
+			if !ok {
+				panic(re)
+			}
+
+			outErr = tmp
+			conn = nil
+		}
+	}()
+
+	params := &outreachParams{}
+	for _, opt := range opts {
+		opt(params)
+	}
+
+	var err error
+
+	params, err = params.prepare()
+	if err != nil {
+		return nil, err
+	}
+
+	// Read provider info
+	providerInfo, err := providers.ReadInfo(providers.Outreach, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	restApi, ok := providerInfo.GetOption(providerOptionRestApiURL)
+	if !ok {
+		return nil, fmt.Errorf("restAPIURL not set: %w", providers.ErrProviderOptionNotFound)
+	}
+
+	params.client.HTTPClient.Base = providerInfo.BaseURL
+
+	return &Connector{
+		Client:  params.client,
+		BaseURL: restApi,
+	}, nil
+}
diff --git a/outreach/errors.go b/outreach/errors.go
new file mode 100644
index 00000000..3ac48c17
--- /dev/null
+++ b/outreach/errors.go
@@ -0,0 +1,16 @@
+package outreach
+
+import (
+	"errors"
+)
+
+var (
+	ErrMissingClient = errors.New("JSON http client not set")
+	ErrNotArray      = errors.New("results data is not an array")
+	ErrNotObject     = errors.New("record is not an object")
+	ErrNotString     = errors.New("next is not a string")
+)
+
+func (c *Connector) HandleError(err error) error {
+	return err
+}
diff --git a/outreach/methods.go b/outreach/methods.go
new file mode 100644
index 00000000..f59d4b92
--- /dev/null
+++ b/outreach/methods.go
@@ -0,0 +1,16 @@
+package outreach
+
+import (
+	"context"
+
+	"github.com/amp-labs/connectors/common"
+)
+
+func (c *Connector) get(ctx context.Context, url string) (*common.JSONHTTPResponse, error) {
+	res, err := c.Client.Get(ctx, url)
+	if err != nil {
+		return nil, c.HandleError(err)
+	}
+
+	return res, nil
+}
diff --git a/outreach/params.go b/outreach/params.go
new file mode 100644
index 00000000..00a585fa
--- /dev/null
+++ b/outreach/params.go
@@ -0,0 +1,50 @@
+package outreach
+
+import (
+	"context"
+	"net/http"
+
+	"github.com/amp-labs/connectors/common"
+	"golang.org/x/oauth2"
+)
+
+type outreachParams struct {
+	client *common.JSONHTTPClient
+}
+
+type Option func(params *outreachParams)
+
+func WithClient(ctx context.Context, client *http.Client, config *oauth2.Config, token *oauth2.Token,
+) Option {
+	return func(params *outreachParams) {
+		oauthclient, err := common.NewOAuthHTTPClient(
+			ctx, common.WithClient(client),
+			common.WithOAuthConfig(config),
+			common.WithOAuthToken(token),
+		)
+		if err != nil {
+			panic(err)
+		}
+
+		WithAuthenticatedClient(oauthclient)(params)
+	}
+}
+
+func WithAuthenticatedClient(client common.AuthenticatedHTTPClient) Option {
+	return func(params *outreachParams) {
+		params.client = &common.JSONHTTPClient{
+			HTTPClient: &common.HTTPClient{
+				Client:       client,
+				ErrorHandler: common.InterpretError,
+			},
+		}
+	}
+}
+
+func (params *outreachParams) prepare() (*outreachParams, error) {
+	if params.client == nil {
+		return nil, ErrMissingClient
+	}
+
+	return params, nil
+}
diff --git a/outreach/parse.go b/outreach/parse.go
new file mode 100644
index 00000000..33fd1e84
--- /dev/null
+++ b/outreach/parse.go
@@ -0,0 +1,122 @@
+package outreach
+
+import (
+	"github.com/amp-labs/connectors/common"
+	"github.com/spyzhov/ajson"
+)
+
+// getNextRecords returns the "next" url for the next page of results,
+// If available, else returns an empty string.
+func getNextRecordsURL(node *ajson.Node) (string, error) {
+	var nextPage string
+
+	if node.HasKey("links") {
+		links, err := parsePagingNext(node)
+		if err != nil {
+			return "", err
+		}
+
+		nextPage, err = checkURL(links)
+		if err != nil {
+			return "", err
+		}
+	}
+
+	return nextPage, nil
+}
+
+// parsePagingNext is a helper to return the links node.
+func parsePagingNext(node *ajson.Node) (*ajson.Node, error) {
+	links, err := node.GetKey("links")
+	if err != nil {
+		return nil, err
+	}
+
+	if !links.IsObject() {
+		return nil, ErrNotObject
+	}
+
+	return links, nil
+}
+
+// checkURL is a helper function that returns the url of the  next page.
+func checkURL(node *ajson.Node) (string, error) {
+	if !node.HasKey("next") {
+		return "", nil
+	}
+
+	next, err := node.GetKey("next")
+	if err != nil {
+		return "", err
+	}
+
+	if !next.IsString() {
+		return "", ErrNotString
+	}
+
+	return next.String(), nil
+}
+
+// getRecords returns the records from the response.
+func getRecords(node *ajson.Node) ([]map[string]interface{}, error) {
+	records, err := node.GetKey("data")
+	if err != nil {
+		return nil, err
+	}
+
+	if !records.IsArray() {
+		return nil, ErrNotArray
+	}
+
+	arr := records.MustArray()
+
+	out := make([]map[string]interface{}, 0, len(arr))
+
+	for _, v := range arr {
+		if !v.IsObject() {
+			return nil, ErrNotObject
+		}
+
+		data, err := v.Unpack()
+		if err != nil {
+			return nil, err
+		}
+
+		m, ok := data.(map[string]interface{})
+		if !ok {
+			return nil, ErrNotObject
+		}
+
+		out = append(out, m)
+	}
+
+	return out, nil
+}
+
+// getTotalSize returns the total number of records that match the query.
+func getTotalSize(node *ajson.Node) (int64, error) {
+	node, err := node.GetKey("data")
+	if err != nil {
+		return 0, err
+	}
+
+	if !node.IsArray() {
+		return 0, ErrNotArray
+	}
+
+	return int64(node.Size()), nil
+}
+
+// getMarshaledData accepts a list of records and returns a list of structured data ([]ReadResultRow).
+func getMarshaledData(records []map[string]interface{}, fields []string) ([]common.ReadResultRow, error) {
+	data := make([]common.ReadResultRow, len(records))
+
+	for i, record := range records {
+		data[i] = common.ReadResultRow{
+			Fields: common.ExtractLowercaseFieldsFromRaw(fields, record),
+			Raw:    record,
+		}
+	}
+
+	return data, nil
+}
diff --git a/outreach/provider.go b/outreach/provider.go
new file mode 100644
index 00000000..fcf53bd7
--- /dev/null
+++ b/outreach/provider.go
@@ -0,0 +1,8 @@
+package outreach
+
+import "github.com/amp-labs/connectors/providers"
+
+// Provider returns the connector provider.
+func (c *Connector) Provider() providers.Provider {
+	return providers.Outreach
+}
diff --git a/outreach/read.go b/outreach/read.go
new file mode 100644
index 00000000..01a3d6bd
--- /dev/null
+++ b/outreach/read.go
@@ -0,0 +1,42 @@
+package outreach
+
+import (
+	"context"
+	"net/url"
+
+	"github.com/amp-labs/connectors/common"
+)
+
+func (c *Connector) Read(ctx context.Context, config common.ReadParams) (*common.ReadResult, error) {
+	var (
+		res    *common.JSONHTTPResponse
+		err    error
+		fields []string
+	)
+
+	if len(config.NextPage) > 0 {
+		// If NextPage is set, then we're reading the next page of results.
+		// The NextPage URL has all the necessary parameters.
+		res, err = c.get(ctx, config.NextPage.String())
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		fullURL, err := url.JoinPath(c.BaseURL, config.ObjectName)
+		if err != nil {
+			return nil, err
+		}
+
+		res, err = c.get(ctx, fullURL)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return common.ParseResult(res, getTotalSize,
+		getRecords,
+		getNextRecordsURL,
+		getMarshaledData,
+		fields,
+	)
+}
diff --git a/outreach/string.go b/outreach/string.go
new file mode 100644
index 00000000..dfd4cd51
--- /dev/null
+++ b/outreach/string.go
@@ -0,0 +1,5 @@
+package outreach
+
+func (c *Connector) String() string {
+	return c.Provider() + ".Connector"
+}
diff --git a/providers/catalog.go b/providers/catalog.go
index 54ca724f..cbb6bd74 100644
--- a/providers/catalog.go
+++ b/providers/catalog.go
@@ -216,6 +216,9 @@ var catalog = CatalogType{ // nolint:gochecknoglobals
 			Subscribe: false,
 			Write:     false,
 		},
+		ProviderOpts: ProviderOpts{
+			"restAPIURL": "https://api.outreach.io/api/v2",
+		},
 	},
 
 	// Pipedrive configuration
diff --git a/utils/credentials.go b/utils/credentials.go
index 6971b7a4..3f90fb46 100644
--- a/utils/credentials.go
+++ b/utils/credentials.go
@@ -127,3 +127,42 @@ func MSDynamics365SalesTokenFromRegistry(registry CredentialsRegistry) *oauth2.T
 
 	return tok
 }
+
+func OutreachOAuthConfigFromRegistry(registry CredentialsRegistry) *oauth2.Config {
+	clientId := registry.MustString(ClientId)
+	clientSecret := registry.MustString(ClientSecret)
+
+	cfg := &oauth2.Config{
+		ClientID:     clientId,
+		ClientSecret: clientSecret,
+		RedirectURL:  "https://dev-api.withampersand.com/callbacks/v1/oauth",
+		Endpoint: oauth2.Endpoint{
+			AuthURL:   "https://api.outreach.io/oauth/authorize",
+			TokenURL:  "https://api.outreach.io/oauth/token",
+			AuthStyle: oauth2.AuthStyleInParams,
+		},
+		Scopes: []string{
+			"users.all",
+			"accounts.read",
+			"calls.all",
+			"events.all",
+			"teams.all",
+		},
+	}
+
+	return cfg
+}
+
+func OutreachOauthTokenFromRegistry(registry CredentialsRegistry) *oauth2.Token {
+	accessToken := registry.MustString(AccessToken)
+	refreshToken := registry.MustString(RefreshToken)
+
+	tok := &oauth2.Token{
+		AccessToken:  accessToken,
+		RefreshToken: refreshToken,
+		TokenType:    "bearer",
+		Expiry:       time.Now().Add(-1 * time.Hour), // just pretend it's expired already, whatever, it'll fetch a new one.
+	}
+
+	return tok
+}


Test patch:
diff --git a/providers/catalog_test.go b/providers/catalog_test.go
index bb656166..5b149865 100644
--- a/providers/catalog_test.go
+++ b/providers/catalog_test.go
@@ -188,6 +188,9 @@ var testCases = []struct { // nolint
 				},
 			},
 			BaseURL: "https://api.outreach.io",
+			ProviderOpts: ProviderOpts{
+				"restAPIURL": "https://api.outreach.io/api/v2",
+			},
 		},
 		expectedErr: nil,
 	},
diff --git a/test/outreach/read/read.go b/test/outreach/read/read.go
new file mode 100644
index 00000000..cd2f6e06
--- /dev/null
+++ b/test/outreach/read/read.go
@@ -0,0 +1,85 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"log"
+	"net/http"
+	"os"
+
+	"github.com/amp-labs/connectors"
+	"github.com/amp-labs/connectors/outreach"
+	testUtils "github.com/amp-labs/connectors/test/utils"
+	"github.com/amp-labs/connectors/utils"
+)
+
+const (
+	DefaultCredsFile = "creds.json"
+)
+
+func GetOutreachConnector(ctx context.Context, filePath string) *outreach.Connector {
+	registry := utils.NewCredentialsRegistry()
+
+	readers := []utils.Reader{
+		&utils.JSONReader{
+			FilePath: filePath,
+			JSONPath: "$['clientId']",
+			CredKey:  "clientId",
+		},
+		&utils.JSONReader{
+			FilePath: filePath,
+			JSONPath: "$['clientSecret']",
+			CredKey:  "clientSecret",
+		},
+		&utils.JSONReader{
+			FilePath: filePath,
+			JSONPath: "$['refreshToken']",
+			CredKey:  "refreshToken",
+		},
+		&utils.JSONReader{
+			FilePath: filePath,
+			JSONPath: "$['accessToken']",
+			CredKey:  "accessToken",
+		},
+		&utils.JSONReader{
+			FilePath: filePath,
+			JSONPath: "$['provider']",
+			CredKey:  "provider",
+		},
+	}
+	registry.AddReaders(readers...)
+
+	cfg := utils.OutreachOAuthConfigFromRegistry(registry)
+	tok := utils.OutreachOauthTokenFromRegistry(registry)
+
+	conn, err := connectors.Outreach(
+		outreach.WithClient(ctx, http.DefaultClient, cfg, tok),
+	)
+	if err != nil {
+		testUtils.Fail("error creating outreach connector", "error", err)
+	}
+
+	return conn
+}
+
+func main() {
+	outreach := GetOutreachConnector(context.Background(), DefaultCredsFile)
+
+	config := connectors.ReadParams{
+		ObjectName: "users",
+		// NextPage:   "https://api.outreach.io/api/v2/users?page%5Blimit%5D=1\u0026page%5Boffset%5D=2",
+	}
+	result, err := outreach.Read(context.Background(), config)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// Print the results
+	jsonStr, err := json.MarshalIndent(result, "", "  ")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	_, _ = os.Stdout.Write(jsonStr)
+	_, _ = os.Stdout.WriteString("\n")
+}