Repository: amp-labs/connectors
Pull request #1292 (2025-01-20 21:26:36)

[CON-972] feat(Hubspot): Support Lists object by ListObjectMetadata
# Problem

Lists is an object that lives under CRM module but is outside Objects APIs. Current implementation of `ListObjectMetadata` focusses on CRM Objects API that uses field discovery endpoint. This is not applicable for `Lists`.

# Solution
As with other connectors the approach is to make read request for 1 item and use response fields as metadata fields. As a fallback rely on static schema file for the object metadata. This same approach takes place in this PR. Lists object is added under schema.json for the fallback.

# Tests - ListObjectMetadata
Lists
![image](https://github.com/user-attachments/assets/7bc7ded3-11b7-4c3b-bb49-124803b58016)
![image](https://github.com/user-attachments/assets/eadd6e23-a4c9-46d5-9f76-10df01150112)
Contacts
![image](https://github.com/user-attachments/assets/6c7059f5-740f-44ba-ac66-cb0f60f6b264)
![image](https://github.com/user-attachments/assets/4d8c9829-d6ac-437a-9280-0dc0ea209e72)



Cobalt0s: > [!WARNING]
> <b>This pull request is not mergeable via GitHub because a downstack PR is open. Once all requirements are satisfied, merge this PR as a stack <a href="https://app.graphite.dev/github/pr/amp-labs/connectors/1292?utm_source=stack-comment-downstack-mergeability-warning" >on Graphite</a>.</b>
> <a href="https://graphite.dev/docs/merge-pull-requests">Learn more</a>

* **#1292** <a href="https://app.graphite.dev/github/pr/amp-labs/connectors/1292?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a> 👈 <a href="https://app.graphite.dev/github/pr/amp-labs/connectors/1292?utm_source=stack-comment-view-in-graphite" target="_blank">(View in Graphite)</a>
* **#1291** <a href="https://app.graphite.dev/github/pr/amp-labs/connectors/1291?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a>
* **#1283** <a href="https://app.graphite.dev/github/pr/amp-labs/connectors/1283?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a>
* **#1279** <a href="https://app.graphite.dev/github/pr/amp-labs/connectors/1279?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a>
* **#1278** <a href="https://app.graphite.dev/github/pr/amp-labs/connectors/1278?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a>
* `main`


<h2></h2>

This stack of pull requests is managed by <a href="https://graphite.dev?utm-source=stack-comment"><b>Graphite</b></a>. Learn more about <a href="https://stacking.dev/?utm_source=stack-comment">stacking</a>.
<!-- Current dependencies on/for this PR: -->

Patch:
diff --git a/providers/hubspot/metadata.go b/providers/hubspot/metadata.go
index a708444b..55df64bb 100644
--- a/providers/hubspot/metadata.go
+++ b/providers/hubspot/metadata.go
@@ -5,11 +5,12 @@ import (
 	"fmt"
 	"strconv"
 	"strings"
-	"time"
 
+	"github.com/amp-labs/connectors"
 	"github.com/amp-labs/connectors/common"
 	"github.com/amp-labs/connectors/common/logging"
 	"github.com/amp-labs/connectors/internal/datautils"
+	"github.com/amp-labs/connectors/providers/hubspot/metadata"
 )
 
 type objectMetadataResult struct {
@@ -42,7 +43,7 @@ func (c *Connector) ListObjectMetadata( // nolint:cyclop,funlen
 
 	for _, objectName := range objectNames {
 		go func(object string) {
-			objectMetadata, err := c.describeObject(ctx, object)
+			objectMetadata, err := c.getObjectMetadata(ctx, object)
 			if err != nil {
 				errChannel <- &objectMetadataError{
 					ObjectName: object,
@@ -78,8 +79,21 @@ func (c *Connector) ListObjectMetadata( // nolint:cyclop,funlen
 	return objectsMap, nil
 }
 
-// describeObject returns object metadata for the given object name.
-func (c *Connector) describeObject(ctx context.Context, objectName string) (*common.ObjectMetadata, error) {
+// getObjectMetadata returns object metadata for the given object name.
+func (c *Connector) getObjectMetadata(ctx context.Context, objectName string) (*common.ObjectMetadata, error) {
+	if crmObjectsOutsideThePropertiesAPI.Has(objectName) {
+		return c.getObjectMetadataFromObjectAPI(ctx, objectName)
+	}
+
+	return c.getObjectMetadataFromPropertyAPI(ctx, objectName)
+}
+
+// This method describes objects that are part of Objects API using properties endpoint.
+// There is a dedicated API endpoint that is used for discovery of object properties.
+// https://developers.hubspot.com/docs/guides/api/crm/properties
+func (c *Connector) getObjectMetadataFromPropertyAPI(
+	ctx context.Context, objectName string,
+) (*common.ObjectMetadata, error) {
 	relativeURL := strings.Join([]string{"properties", objectName}, "/")
 
 	u, err := c.getURL(relativeURL)
@@ -107,6 +121,42 @@ func (c *Connector) describeObject(ctx context.Context, objectName string) (*com
 	), nil
 }
 
+// Method focuses on acquiring object properties for those objects that are not part of CRM Properties API.
+// https://developers.hubspot.com/docs/guides/api/crm/objects/companies
+// There is no discovery endpoint to acquire object properties, therefore, manual read is used.
+func (c *Connector) getObjectMetadataFromObjectAPI(
+	ctx context.Context, objectName string,
+) (*common.ObjectMetadata, error) {
+	readResult, err := c.SearchCRM(ctx, SearchCRMParams{
+		ObjectName: objectName,
+		Fields:     connectors.Fields(""), // passed to satisfy validation
+		NextPage:   "",
+		PageSize:   1,
+	})
+	if err != nil {
+		// Ignore an error and fallback to static schema.
+		return metadata.Schemas.SelectOne(c.Module.ID, objectName)
+	}
+
+	if len(readResult.Data) == 0 {
+		// Read returned no rows.
+		return metadata.Schemas.SelectOne(c.Module.ID, objectName)
+	}
+
+	fields := make(map[string]common.FieldMetadata)
+	for fieldName := range readResult.Data[0].Raw {
+		fields[fieldName] = common.FieldMetadata{
+			DisplayName:  fieldName,
+			ValueType:    common.ValueTypeOther,
+			ProviderType: "", // not available
+			ReadOnly:     false,
+			Values:       nil,
+		}
+	}
+
+	return common.NewObjectMetadata(objectName, fields), nil
+}
+
 func (c *Connector) GetPostAuthInfo(
 	ctx context.Context,
 ) (*common.PostAuthInfo, error) {
@@ -153,38 +203,21 @@ type fieldDescriptionResponse struct {
 }
 
 type fieldDescription struct {
-	UpdatedAt            time.Time                 `json:"updatedAt"`
-	CreatedAt            time.Time                 `json:"createdAt"`
 	Name                 string                    `json:"name"`
 	Label                string                    `json:"label"`
 	Type                 string                    `json:"type"`
 	FieldType            string                    `json:"fieldType"`
-	Description          string                    `json:"description"`
-	GroupName            string                    `json:"groupName"`
 	Options              []fieldEnumerationOption  `json:"options"`
-	DisplayOrder         int                       `json:"displayOrder"`
-	Calculated           bool                      `json:"calculated"`
-	ExternalOptions      bool                      `json:"externalOptions"`
-	HasUniqueValue       bool                      `json:"hasUniqueValue"`
-	Hidden               bool                      `json:"hidden"`
-	HubspotDefined       bool                      `json:"hubspotDefined"`
 	ModificationMetadata fieldModificationMetadata `json:"modificationMetadata"`
-	FormField            bool                      `json:"formField"`
-	DataSensitivity      string                    `json:"dataSensitivity"`
 }
 
 type fieldEnumerationOption struct {
-	Label        string `json:"label"`
-	Value        string `json:"value"`
-	Description  string `json:"description"`
-	DisplayOrder int    `json:"displayOrder"`
-	Hidden       bool   `json:"hidden"`
+	Label string `json:"label"`
+	Value string `json:"value"`
 }
 
 type fieldModificationMetadata struct {
-	Archivable         bool `json:"archivable"`
-	ReadOnlyDefinition bool `json:"readOnlyDefinition"`
-	ReadOnlyValue      bool `json:"readOnlyValue"`
+	ReadOnlyValue bool `json:"readOnlyValue"`
 }
 
 func (r fieldDescriptionResponse) transformToFields() map[string]common.FieldMetadata {
diff --git a/providers/hubspot/metadata/metadata.go b/providers/hubspot/metadata/metadata.go
new file mode 100644
index 00000000..fd1bd3fb
--- /dev/null
+++ b/providers/hubspot/metadata/metadata.go
@@ -0,0 +1,22 @@
+package metadata
+
+import (
+	_ "embed"
+
+	"github.com/amp-labs/connectors/internal/staticschema"
+	"github.com/amp-labs/connectors/tools/fileconv"
+	"github.com/amp-labs/connectors/tools/scrapper"
+)
+
+var (
+	// Static file containing a list of object metadata is embedded and can be served.
+	//
+	//go:embed schemas.json
+	schemas []byte
+
+	FileManager = scrapper.NewMetadataFileManager[staticschema.FieldMetadataMapV2]( // nolint:gochecknoglobals
+		schemas, fileconv.NewSiblingFileLocator())
+
+	// Schemas is cached Object schemas.
+	Schemas = FileManager.MustLoadSchemas() // nolint:gochecknoglobals
+)
diff --git a/providers/hubspot/metadata/schemas.json b/providers/hubspot/metadata/schemas.json
new file mode 100755
index 00000000..30e60454
--- /dev/null
+++ b/providers/hubspot/metadata/schemas.json
@@ -0,0 +1,65 @@
+{
+  "modules": {
+    "CRM": {
+      "id": "CRM",
+      "path": "/crm/v3",
+      "objects": {
+        "lists": {
+          "displayName": "Lists",
+          "path": "/lists/search",
+          "responseKey": "data",
+          "fields": {
+            "additionalProperties": {
+              "displayName": "additionalProperties",
+              "valueType": "other"
+            },
+            "createdAt": {
+              "displayName": "createdAt",
+              "valueType": "other"
+            },
+            "createdById": {
+              "displayName": "createdById",
+              "valueType": "other"
+            },
+            "filtersUpdatedAt": {
+              "displayName": "filtersUpdatedAt",
+              "valueType": "other"
+            },
+            "listId": {
+              "displayName": "listId",
+              "valueType": "other"
+            },
+            "listVersion": {
+              "displayName": "listVersion",
+              "valueType": "other"
+            },
+            "name": {
+              "displayName": "name",
+              "valueType": "other"
+            },
+            "objectTypeId": {
+              "displayName": "objectTypeId",
+              "valueType": "other"
+            },
+            "processingStatus": {
+              "displayName": "processingStatus",
+              "valueType": "other"
+            },
+            "processingType": {
+              "displayName": "processingType",
+              "valueType": "other"
+            },
+            "updatedAt": {
+              "displayName": "updatedAt",
+              "valueType": "other"
+            },
+            "updatedById": {
+              "displayName": "updatedById",
+              "valueType": "other"
+            }
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/providers/hubspot/objectName.go b/providers/hubspot/objectName.go
index ad369eed..865f37e9 100644
--- a/providers/hubspot/objectName.go
+++ b/providers/hubspot/objectName.go
@@ -2,12 +2,12 @@ package hubspot
 
 import "github.com/amp-labs/connectors/internal/datautils"
 
-// This is a list of objectsNames that are part of Hubspot CRM Module but exist outside ObjectAPIs.
+// This is a list of objectsNames that are part of Hubspot CRM Module but exist outside Object Properties APIs.
 //
 // These objects cannot be accessed via `~/crm/v3/objects/{objectTypeId}/...`
 //
-// On the contrary those objects that are part of ObjectAPIs can be found
+// On the contrary those objects that are part of Object Properties APIs can be found
 // in the table here https://developers.hubspot.com/docs/guides/api/crm/understanding-the-crm#object-type-ids
-var crmObjectsOutsideTheObjectAPI = datautils.NewSet( //nolint:gochecknoglobals
+var crmObjectsOutsideThePropertiesAPI = datautils.NewSet( //nolint:gochecknoglobals
 	"lists",
 )
diff --git a/providers/hubspot/read.go b/providers/hubspot/read.go
index b18d2d55..7424822c 100644
--- a/providers/hubspot/read.go
+++ b/providers/hubspot/read.go
@@ -21,7 +21,7 @@ func (c *Connector) Read(ctx context.Context, config common.ReadParams) (*common
 		return nil, err
 	}
 
-	if crmObjectsOutsideTheObjectAPI.Has(config.ObjectName) {
+	if crmObjectsOutsideThePropertiesAPI.Has(config.ObjectName) {
 		// Objects outside ObjectAPI have different endpoint while both are part of CRM module.
 		// For instance Lists are fully returned only via Search endpoint.
 		return c.SearchCRM(ctx, SearchCRMParams{
diff --git a/providers/hubspot/types.go b/providers/hubspot/types.go
index dcbd4382..fe038b86 100644
--- a/providers/hubspot/types.go
+++ b/providers/hubspot/types.go
@@ -42,6 +42,7 @@ type SearchCRMParams struct {
 	Fields datautils.Set[string] // optional
 	// NextPage is an opaque token that can be used to get the next page of results.
 	NextPage common.NextPageToken // optional, only set this if you want to read the next page of results
+	PageSize int
 }
 
 func (p SearchCRMParams) ValidateParams() error {
@@ -68,9 +69,14 @@ func (p SearchCRMParams) Payload() (SearchCRMPayload, error) {
 		}
 	}
 
+	pageSize := DefaultPageSizeInt
+	if p.PageSize != 0 {
+		pageSize = p.PageSize
+	}
+
 	return SearchCRMPayload{
 		Offset: offset,
-		Count:  DefaultPageSizeInt,
+		Count:  pageSize,
 	}, nil
 }
 


Test patch:
diff --git a/providers/hubspot/metadata_test.go b/providers/hubspot/metadata_test.go
index 58fdbdbb..f5a3750e 100644
--- a/providers/hubspot/metadata_test.go
+++ b/providers/hubspot/metadata_test.go
@@ -19,6 +19,7 @@ func TestListObjectMetadata(t *testing.T) { // nolint:funlen,gocognit,cyclop,mai
 	metadataContactsPipelines := testutils.DataFromFile(t, "metadata-contacts-external-pipelines.json")
 	metadataDealsProperties := testutils.DataFromFile(t, "metadata-deals-properties-sampled.json")
 	metadataDealsPipelines := testutils.DataFromFile(t, "metadata-deals-external-pipelines.json")
+	responseLists := testutils.DataFromFile(t, "read-lists-1-first-page.json")
 
 	tests := []testroutines.Metadata{
 		{
@@ -293,6 +294,48 @@ func TestListObjectMetadata(t *testing.T) { // nolint:funlen,gocognit,cyclop,mai
 			},
 			ExpectedErrs: nil,
 		},
+		{
+			Name:  "Successfully describe lists, which is outside Properties API",
+			Input: []string{"lists"},
+			Server: mockserver.Conditional{
+				Setup: mockserver.ContentJSON(),
+				If:    mockcond.PathSuffix("/crm/v3/lists/search"),
+				Then:  mockserver.Response(http.StatusOK, responseLists),
+			}.Server(),
+			Comparator: testroutines.ComparatorSubsetMetadata,
+			Expected: &common.ListObjectMetadataResult{
+				Result: map[string]common.ObjectMetadata{
+					"lists": {
+						DisplayName: "lists",
+						Fields: map[string]common.FieldMetadata{
+							"additionalProperties": {
+								DisplayName:  "additionalProperties",
+								ValueType:    "other",
+								ProviderType: "",
+								ReadOnly:     false,
+								Values:       nil,
+							},
+							"name": {
+								DisplayName:  "name",
+								ValueType:    "other",
+								ProviderType: "",
+								ReadOnly:     false,
+								Values:       nil,
+							},
+							"updatedAt": {
+								DisplayName:  "updatedAt",
+								ValueType:    "other",
+								ProviderType: "",
+								ReadOnly:     false,
+								Values:       nil,
+							},
+						},
+					},
+				},
+				Errors: nil,
+			},
+			ExpectedErrs: nil,
+		},
 	}
 
 	for _, tt := range tests {
diff --git a/test/hubspot/metadata/lists/main.go b/test/hubspot/metadata/lists/main.go
new file mode 100644
index 00000000..38a18d7d
--- /dev/null
+++ b/test/hubspot/metadata/lists/main.go
@@ -0,0 +1,33 @@
+package main
+
+import (
+	"context"
+	"os"
+	"os/signal"
+	"syscall"
+
+	connTest "github.com/amp-labs/connectors/test/hubspot"
+	"github.com/amp-labs/connectors/test/utils"
+)
+
+var objectName = "lists"
+
+func main() {
+	// Handle Ctrl-C gracefully.
+	ctx, done := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
+	defer done()
+
+	// Set up slog logging.
+	utils.SetupLogging()
+
+	conn := connTest.GetHubspotConnector(ctx)
+
+	metadata, err := conn.ListObjectMetadata(ctx, []string{
+		objectName,
+	})
+	if err != nil {
+		utils.Fail("error listing metadata for Hubspot", "error", err)
+	}
+
+	utils.DumpJSON(metadata, os.Stdout)
+}