Repository: amp-labs/connectors
Pull request #421 (2024-05-23 15:24:28)

feat: Intercom.ListMetadata return custom data fields
# Description

ListObjectMetadata returns schemas from static file as before.
But if `ListDataAttributes` returns successfully for certain ObjectName we will use these attributes and add to the set coming from schema files. Current hybrid behaviour:
* Right now `ListDataAttributes ` fields is a **subset** of fields from `static file`
* If custom fields are returned from this endpoint they will be surfaced (so impl better than raw static file)

# Naming
It doesn't matter what string format is ObjectName for ListObjectMetadata and Read. Under the hood concrete String data type is used. 


Patch:
diff --git a/common/naming/plural.go b/common/naming/plural.go
new file mode 100644
index 00000000..4c853ff4
--- /dev/null
+++ b/common/naming/plural.go
@@ -0,0 +1,36 @@
+package naming
+
+type PluralString struct {
+	text string
+}
+type PluralStrings []PluralString
+
+func NewPluralString(str string) PluralString {
+	return PluralString{text: pluralizer.Plural(str)}
+}
+
+func (s PluralString) String() string {
+	return s.text
+}
+
+func (s PluralString) Singular() SingularString {
+	return NewSingularString(s.String())
+}
+
+func NewPluralStrings(list []string) PluralStrings {
+	result := make(PluralStrings, len(list))
+	for i, str := range list {
+		result[i] = NewPluralString(str)
+	}
+
+	return result
+}
+
+func (s PluralStrings) Plural() SingularStrings {
+	result := make(SingularStrings, len(s))
+	for i, str := range s {
+		result[i] = str.Singular()
+	}
+
+	return result
+}
diff --git a/common/naming/singular.go b/common/naming/singular.go
new file mode 100644
index 00000000..0195fc46
--- /dev/null
+++ b/common/naming/singular.go
@@ -0,0 +1,40 @@
+package naming
+
+import "github.com/gertd/go-pluralize"
+
+var pluralizer = pluralize.NewClient() // nolint:gochecknoglobals
+
+type SingularString struct {
+	text string
+}
+type SingularStrings []SingularString
+
+func NewSingularString(str string) SingularString {
+	return SingularString{text: pluralizer.Singular(str)}
+}
+
+func (s SingularString) String() string {
+	return s.text
+}
+
+func (s SingularString) Plural() PluralString {
+	return NewPluralString(s.String())
+}
+
+func NewSingularStrings(list []string) SingularStrings {
+	result := make(SingularStrings, len(list))
+	for i, str := range list {
+		result[i] = NewSingularString(str)
+	}
+
+	return result
+}
+
+func (s SingularStrings) Plural() PluralStrings {
+	result := make(PluralStrings, len(s))
+	for i, str := range s {
+		result[i] = str.Plural()
+	}
+
+	return result
+}
diff --git a/common/scrapper/convertors.go b/common/scrapper/convertors.go
index 93779c2a..4fe09612 100644
--- a/common/scrapper/convertors.go
+++ b/common/scrapper/convertors.go
@@ -5,18 +5,20 @@ import (
 	"fmt"
 
 	"github.com/amp-labs/connectors/common"
+	"github.com/amp-labs/connectors/common/naming"
 )
 
 var ErrObjectNotFound = errors.New("object not found")
 
-func (r *ObjectMetadataResult) Select(objectNames []string) (*common.ListObjectMetadataResult, error) {
+func (r *ObjectMetadataResult) Select(objectNames naming.PluralStrings) (*common.ListObjectMetadataResult, error) {
 	list := &common.ListObjectMetadataResult{
 		Result: make(map[string]common.ObjectMetadata),
 		Errors: nil,
 	}
 
 	// Convert and return only listed objects
-	for _, name := range objectNames {
+	for _, n := range objectNames {
+		name := n.String()
 		if v, ok := r.Result[name]; ok {
 			// move metadata from scrapper object to common object
 			list.Result[name] = common.ObjectMetadata{
diff --git a/common/scrapper/models.go b/common/scrapper/models.go
index bde481cb..ec4ba0b0 100644
--- a/common/scrapper/models.go
+++ b/common/scrapper/models.go
@@ -51,7 +51,7 @@ func (r *ModelURLRegistry) Add(displayName, url string) {
 
 type ObjectMetadataResult struct {
 	// Result is a map of object names to object metadata
-	Result map[string]ObjectMetadata `json:"data"`
+	Result map[string]*ObjectMetadata `json:"data"`
 }
 
 type ObjectMetadata struct {
@@ -64,14 +64,14 @@ type ObjectMetadata struct {
 
 func NewObjectMetadataResult() *ObjectMetadataResult {
 	return &ObjectMetadataResult{
-		Result: make(map[string]ObjectMetadata),
+		Result: make(map[string]*ObjectMetadata),
 	}
 }
 
 func (r *ObjectMetadataResult) Add(objectName string, objectDisplayName string, fieldName string) {
 	data, ok := r.Result[objectName]
 	if !ok {
-		data = ObjectMetadata{
+		data = &ObjectMetadata{
 			DisplayName: objectDisplayName,
 			FieldsMap:   make(map[string]string),
 		}
diff --git a/intercom/metadata.go b/intercom/metadata.go
index 3d9ffba4..80262323 100644
--- a/intercom/metadata.go
+++ b/intercom/metadata.go
@@ -5,6 +5,10 @@ import (
 	"errors"
 
 	"github.com/amp-labs/connectors/common"
+	"github.com/amp-labs/connectors/common/jsonquery"
+	"github.com/amp-labs/connectors/common/naming"
+	"github.com/amp-labs/connectors/common/scrapper"
+	"github.com/amp-labs/connectors/common/urlbuilder"
 	"github.com/amp-labs/connectors/intercom/metadata"
 )
 
@@ -13,7 +17,13 @@ var ErrLoadFailure = errors.New("cannot load metadata")
 func (c *Connector) ListObjectMetadata(
 	ctx context.Context, objectNames []string,
 ) (*common.ListObjectMetadataResult, error) {
-	// Ensure that objectNames is not empty
+	return c.listObjectMetadata(ctx, naming.NewSingularStrings(objectNames))
+}
+
+func (c *Connector) listObjectMetadata(
+	ctx context.Context, objectNames naming.SingularStrings,
+) (*common.ListObjectMetadataResult, error) {
+	// Ensure that objectNames are not empty
 	if len(objectNames) == 0 {
 		return nil, common.ErrMissingObjects
 	}
@@ -23,5 +33,92 @@ func (c *Connector) ListObjectMetadata(
 		return nil, ErrLoadFailure
 	}
 
-	return schemas.Select(objectNames)
+	for _, objectName := range objectNames {
+		if err = c.substituteSchema(ctx, schemas, objectName); err != nil {
+			return nil, err
+		}
+	}
+
+	result, err := schemas.Select(objectNames.Plural())
+	if err != nil {
+		return nil, err
+	}
+
+	return result, nil
+}
+
+func (c *Connector) substituteSchema(ctx context.Context, schemas *scrapper.ObjectMetadataResult,
+	objectName naming.SingularString,
+) error {
+	url, err := c.buildMetaURL(objectName)
+	if err != nil {
+		return err
+	}
+
+	rsp, err := c.get(ctx, url.String(), common.Header{
+		Key:   "Intercom-Version",
+		Value: c.Module,
+	})
+	if err != nil {
+		if errors.Is(err, common.ErrBadRequest) {
+			// We have BadRequest. No data attributes available for this model.
+			// Nothing to replace in schema registry.
+			return nil
+		} else {
+			// critical error
+			return err
+		}
+	}
+
+	attributes, err := extractDataAttributes(rsp)
+	if err != nil {
+		return err
+	}
+	// Replace fields that we got from ListDataAttributes endpoint.
+	obj := schemas.Result[objectName.Plural().String()]
+	for k, v := range attributes {
+		obj.FieldsMap[k] = v
+	}
+
+	return nil
+}
+
+func extractDataAttributes(rsp *common.JSONHTTPResponse) (map[string]string, error) {
+	attributes := make(map[string]string)
+
+	arr, err := jsonquery.New(rsp.Body).Array("data", false)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, node := range arr {
+		name, err := jsonquery.New(node).Str("name", false)
+		if err != nil {
+			return nil, err
+		}
+
+		displayName, err := jsonquery.New(node).Str("label", false)
+		if err != nil {
+			return nil, err
+		}
+
+		attributes[*name] = *displayName
+	}
+
+	return attributes, nil
+}
+
+func (c *Connector) buildMetaURL(objectName naming.SingularString) (*urlbuilder.URL, error) {
+	// Metadata is described by the following endpoint:
+	//
+	// GET /data_attributes?model=objectName
+	//
+	url, err := c.getURL("data_attributes")
+	if err != nil {
+		return nil, err
+	}
+
+	url.WithQueryParam("model", objectName.String())
+
+	return url, nil
 }
diff --git a/intercom/read.go b/intercom/read.go
index df0f054a..23be1655 100644
--- a/intercom/read.go
+++ b/intercom/read.go
@@ -5,6 +5,7 @@ import (
 	"strconv"
 
 	"github.com/amp-labs/connectors/common"
+	"github.com/amp-labs/connectors/common/naming"
 	"github.com/amp-labs/connectors/common/urlbuilder"
 )
 
@@ -39,7 +40,7 @@ func (c *Connector) buildReadURL(config common.ReadParams) (*urlbuilder.URL, err
 	}
 
 	// First page
-	link, err := c.getURL(config.ObjectName)
+	link, err := c.getURL(naming.NewPluralString(config.ObjectName).String())
 	if err != nil {
 		return nil, err
 	}


Test patch:
diff --git a/intercom/metadata_test.go b/intercom/metadata_test.go
index 974dff6f..cfd9c48a 100644
--- a/intercom/metadata_test.go
+++ b/intercom/metadata_test.go
@@ -11,12 +11,16 @@ import (
 
 	"github.com/amp-labs/connectors/common"
 	"github.com/amp-labs/connectors/common/scrapper"
+	"github.com/amp-labs/connectors/test/utils/mockutils"
 	"github.com/go-test/deep"
 )
 
 func TestListObjectMetadata(t *testing.T) { // nolint:funlen,gocognit,cyclop
 	t.Parallel()
 
+	responseCompanyAttributes := mockutils.DataFromFile(t, "metadata-attributes-company.json")
+	responseUnsupportedModel := mockutils.DataFromFile(t, "unsupported-model.json")
+
 	tests := []struct {
 		name         string
 		input        []string
@@ -36,28 +40,48 @@ func TestListObjectMetadata(t *testing.T) { // nolint:funlen,gocognit,cyclop
 			name:  "Unknown object requested",
 			input: []string{"butterflies"},
 			server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-				w.WriteHeader(http.StatusTeapot)
+				w.Header().Set("Content-Type", "application/json")
+				w.WriteHeader(http.StatusBadRequest)
+				_, _ = w.Write(responseUnsupportedModel)
 			})),
 			expectedErrs: []error{scrapper.ErrObjectNotFound},
 		},
 		{
-			name:  "Successfully describe one object with metadata",
-			input: []string{"help_centers"},
+			// Some metadata is coming from ListDataAttributes and some from static file.
+			name:  "Successfully describe one object with hybrid metadata",
+			input: []string{"company"},
 			server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-				w.WriteHeader(http.StatusTeapot)
+				w.Header().Set("Content-Type", "application/json")
+				w.WriteHeader(http.StatusOK)
+				_, _ = w.Write(responseCompanyAttributes)
 			})),
 			expected: &common.ListObjectMetadataResult{
 				Result: map[string]common.ObjectMetadata{
-					"help_centers": {
-						DisplayName: "Help Centers",
+					"companies": {
+						DisplayName: "Companies",
 						FieldsMap: map[string]string{
-							"created_at":        "created_at",
-							"display_name":      "display_name",
-							"id":                "id",
-							"identifier":        "identifier",
-							"updated_at":        "updated_at",
-							"website_turned_on": "website_turned_on",
-							"workspace_id":      "workspace_id",
+							// Fields from static file
+							"custom_attributes": "custom_attributes",
+							"plan":              "plan",
+							"segments":          "segments",
+							"tags":              "tags",
+							"type":              "type",
+							// Fields from API endpoint (all present in static but different display)
+							"my_custom_field":   "Custom field for test purposes",
+							"last_request_at":   "Company last seen",
+							"name":              "Plan",
+							"company_id":        "Company ID",
+							"updated_at":        "Updated at",
+							"monthly_spend":     "Monthly Spend",
+							"size":              "Company size",
+							"industry":          "Company industry",
+							"website":           "Company website",
+							"id":                "Plan ID",
+							"created_at":        "Created at",
+							"app_id":            "App ID",
+							"remote_created_at": "Company created at",
+							"session_count":     "Company web sessions",
+							"user_count":        "People",
 						},
 					},
 				},
@@ -69,7 +93,9 @@ func TestListObjectMetadata(t *testing.T) { // nolint:funlen,gocognit,cyclop
 			name:  "Successfully describe multiple objects with metadata",
 			input: []string{"data_events", "teams"},
 			server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-				w.WriteHeader(http.StatusTeapot)
+				w.Header().Set("Content-Type", "application/json")
+				w.WriteHeader(http.StatusBadRequest)
+				_, _ = w.Write(responseUnsupportedModel)
 			})),
 			expected: &common.ListObjectMetadataResult{
 				Result: map[string]common.ObjectMetadata{
diff --git a/intercom/test/metadata-attributes-company.json b/intercom/test/metadata-attributes-company.json
new file mode 100644
index 00000000..aefc1fd6
--- /dev/null
+++ b/intercom/test/metadata-attributes-company.json
@@ -0,0 +1,243 @@
+{
+  "type": "list",
+  "data": [
+    {
+      "type": "data_attribute",
+      "name": "name",
+      "full_name": "name",
+      "label": "Company name",
+      "description": "The name of a company",
+      "data_type": "string",
+      "api_writable": true,
+      "ui_writable": true,
+      "messenger_writable": true,
+      "custom": false,
+      "archived": false,
+      "model": "company"
+    },
+    {
+      "type": "data_attribute",
+      "name": "company_id",
+      "full_name": "company_id",
+      "label": "Company ID",
+      "description": "A number identifying a company",
+      "data_type": "string",
+      "api_writable": false,
+      "ui_writable": false,
+      "messenger_writable": true,
+      "custom": false,
+      "archived": false,
+      "model": "company"
+    },
+    {
+      "type": "data_attribute",
+      "name": "last_request_at",
+      "full_name": "last_request_at",
+      "label": "Company last seen",
+      "description": "The last day anyone from a company visited your site or app",
+      "data_type": "date",
+      "api_writable": false,
+      "ui_writable": false,
+      "messenger_writable": true,
+      "custom": false,
+      "archived": false,
+      "model": "company"
+    },
+    {
+      "type": "data_attribute",
+      "name": "remote_created_at",
+      "full_name": "remote_created_at",
+      "label": "Company created at",
+      "description": "The day a company was added to Intercom",
+      "data_type": "date",
+      "api_writable": true,
+      "ui_writable": false,
+      "messenger_writable": true,
+      "custom": false,
+      "archived": false,
+      "model": "company"
+    },
+    {
+      "type": "data_attribute",
+      "name": "user_count",
+      "full_name": "user_count",
+      "label": "People",
+      "description": "The number of people in a company",
+      "data_type": "integer",
+      "api_writable": false,
+      "ui_writable": false,
+      "messenger_writable": true,
+      "custom": false,
+      "archived": false,
+      "model": "company"
+    },
+    {
+      "type": "data_attribute",
+      "name": "session_count",
+      "full_name": "session_count",
+      "label": "Company web sessions",
+      "description": "All visits from anyone in a company to your product's site or app",
+      "data_type": "integer",
+      "api_writable": false,
+      "ui_writable": false,
+      "messenger_writable": true,
+      "custom": false,
+      "archived": false,
+      "model": "company"
+    },
+    {
+      "type": "data_attribute",
+      "name": "name",
+      "full_name": "plan.name",
+      "label": "Plan",
+      "description": "A specific plan or level within your product that companies have signed up to",
+      "data_type": "string",
+      "api_writable": false,
+      "ui_writable": false,
+      "messenger_writable": true,
+      "custom": false,
+      "archived": false,
+      "model": "company"
+    },
+    {
+      "type": "data_attribute",
+      "name": "monthly_spend",
+      "full_name": "monthly_spend",
+      "label": "Monthly Spend",
+      "description": "The monthly revenue you receive from a company",
+      "data_type": "float",
+      "api_writable": true,
+      "ui_writable": false,
+      "messenger_writable": true,
+      "custom": false,
+      "archived": false,
+      "model": "company"
+    },
+    {
+      "type": "data_attribute",
+      "name": "size",
+      "full_name": "size",
+      "label": "Company size",
+      "description": "The number of people employed in this company, expressed as a single number",
+      "data_type": "integer",
+      "api_writable": true,
+      "ui_writable": true,
+      "messenger_writable": true,
+      "custom": false,
+      "archived": false,
+      "model": "company"
+    },
+    {
+      "type": "data_attribute",
+      "name": "industry",
+      "full_name": "industry",
+      "label": "Company industry",
+      "description": "The category or domain this company belongs to e.g. 'ecommerce' or 'SaaS'",
+      "data_type": "string",
+      "api_writable": true,
+      "ui_writable": true,
+      "messenger_writable": true,
+      "custom": false,
+      "archived": false,
+      "model": "company"
+    },
+    {
+      "type": "data_attribute",
+      "name": "website",
+      "full_name": "website",
+      "label": "Company website",
+      "description": "The web address for the company's primary marketing site",
+      "data_type": "string",
+      "api_writable": true,
+      "ui_writable": true,
+      "messenger_writable": true,
+      "custom": false,
+      "archived": false,
+      "model": "company"
+    },
+    {
+      "type": "data_attribute",
+      "name": "id",
+      "full_name": "id",
+      "label": "ID",
+      "description": "The Intercom defined id representing the company",
+      "data_type": "string",
+      "api_writable": false,
+      "ui_writable": false,
+      "messenger_writable": true,
+      "custom": false,
+      "archived": false,
+      "model": "company"
+    },
+    {
+      "type": "data_attribute",
+      "name": "created_at",
+      "full_name": "created_at",
+      "label": "Created at",
+      "description": "The time the company was added to Intercom",
+      "data_type": "date",
+      "api_writable": false,
+      "ui_writable": false,
+      "messenger_writable": true,
+      "custom": false,
+      "archived": false,
+      "model": "company"
+    },
+    {
+      "type": "data_attribute",
+      "name": "updated_at",
+      "full_name": "updated_at",
+      "label": "Updated at",
+      "description": "The last time the company was updated",
+      "data_type": "date",
+      "api_writable": false,
+      "ui_writable": false,
+      "messenger_writable": true,
+      "custom": false,
+      "archived": false,
+      "model": "company"
+    },
+    {
+      "type": "data_attribute",
+      "name": "id",
+      "full_name": "plan.id",
+      "label": "Plan ID",
+      "description": "The Intercom defined id representing the plan",
+      "data_type": "string",
+      "api_writable": false,
+      "ui_writable": false,
+      "messenger_writable": true,
+      "custom": false,
+      "archived": false,
+      "model": "company"
+    },
+    {
+      "type": "data_attribute",
+      "name": "app_id",
+      "full_name": "app_id",
+      "label": "App ID",
+      "description": "The Intercom defined id representing the app",
+      "data_type": "string",
+      "api_writable": false,
+      "ui_writable": false,
+      "messenger_writable": true,
+      "custom": false,
+      "archived": false,
+      "model": "company"
+    },
+    {
+      "type": "data_attribute",
+      "name": "my_custom_field",
+      "full_name": "my_custom_field",
+      "label": "Custom field for test purposes",
+      "description": "This was artificially added.",
+      "data_type": "string",
+      "api_writable": false,
+      "ui_writable": false,
+      "messenger_writable": true,
+      "custom": true,
+      "archived": false,
+      "model": "company"
+    }
+  ]
+}
diff --git a/intercom/test/unsupported-model.json b/intercom/test/unsupported-model.json
new file mode 100644
index 00000000..e3630073
--- /dev/null
+++ b/intercom/test/unsupported-model.json
@@ -0,0 +1,10 @@
+{
+  "type": "error.list",
+  "request_id": "0000vu89qmaaos7flru0",
+  "errors": [
+    {
+      "code": "parameter_invalid",
+      "message": "unsupported model parameter [someModelName]"
+    }
+  ]
+}
\ No newline at end of file
diff --git a/test/intercom/metadata/main.go b/test/intercom/metadata/main.go
index c5588bfb..e6d314e5 100644
--- a/test/intercom/metadata/main.go
+++ b/test/intercom/metadata/main.go
@@ -9,12 +9,13 @@ import (
 	"syscall"
 
 	"github.com/amp-labs/connectors/common"
+	"github.com/amp-labs/connectors/common/naming"
 	msTest "github.com/amp-labs/connectors/test/intercom"
 	"github.com/amp-labs/connectors/test/utils"
 )
 
 var (
-	objectName = "admins"
+	objectName = "company"
 )
 
 // we want to compare fields returned by read and schema properties provided by metadata methods
@@ -72,9 +73,9 @@ func compareFieldsMatch(metadata *common.ListObjectMetadataResult, response map[
 
 	mismatch := make([]error, 0)
 
-	for _, displayName := range metadata.Result[objectName].FieldsMap {
-		if _, found := fields[displayName]; found {
-			fields[displayName] = true
+	for name := range metadata.Result[naming.NewPluralString(objectName).String()].FieldsMap {
+		if _, found := fields[name]; found {
+			fields[name] = true
 		}
 	}