Repository: amp-labs/connectors
Pull request #1517 (2025-03-25 22:36:24)

[CON-1628] fix(Zendesk): Page sizes for each object
# Approach

* Invoked the Read operation for **each module and object**.
* Intentionally set a high `page_size` well beyond 1000 to find de facto upper limits.

Note: `page_size` was always included, with pagination type (cursor or offset) determining the query parameter used.

# Result

Many endpoints enforce a strict page_size limit and returned errors—explicit mappings were added for those.

Many endpoints are strict about the page size and would error out, for those the explicit mapping were added. The rest use 2000.

For example:
* `tickets` uses incremental endpoint which doesn't have an upper bound. (Handled by default 2000)
* `users` object on the other hand has a strict limit of `1000`. (Handled by explicit map)

# Additional Tidying

* Removed HelpCenter search endpoints requiring query parameters.
* HelpCenter always has a pagination of 100. Otherwise it would error out.
* Renamed `article_labels` to `article/labels` in HelpCenter. No similar renaming was needed in Ticketing (nor should it take place, given its apparent usage).


Cobalt0s: * **#1517** <a href="https://app.graphite.dev/github/pr/amp-labs/connectors/1517?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a> 👈 <a href="https://app.graphite.dev/github/pr/amp-labs/connectors/1517?utm_source=stack-comment-view-in-graphite" target="_blank">(View in Graphite)</a>
* `main`


<h2></h2>

This stack of pull requests is managed by <a href="https://graphite.dev?utm-source=stack-comment"><b>Graphite</b></a>. Learn more about <a href="https://stacking.dev/?utm_source=stack-comment">stacking</a>.
<!-- Current dependencies on/for this PR: -->

Patch:
diff --git a/providers/zendesksupport/metadata/metadata.go b/providers/zendesksupport/metadata/metadata.go
index e897a296..bea8abea 100644
--- a/providers/zendesksupport/metadata/metadata.go
+++ b/providers/zendesksupport/metadata/metadata.go
@@ -4,6 +4,7 @@ import (
 	_ "embed"
 
 	"github.com/amp-labs/connectors/common"
+	"github.com/amp-labs/connectors/internal/datautils"
 	"github.com/amp-labs/connectors/internal/staticschema"
 	"github.com/amp-labs/connectors/tools/fileconv"
 	"github.com/amp-labs/connectors/tools/scrapper"
@@ -58,3 +59,66 @@ func (s ZendeskSchemas) IsIncrementalRead(
 
 	return s.Modules[moduleID].Objects[objectName].Custom.Incremental
 }
+
+// nolint:lll
+var pageSizes = map[common.ModuleID]datautils.DefaultMap[string, string]{ //nolint:gochecknoglobals
+	"ticketing": datautils.NewDefaultMap(map[string]string{
+		// Every object below was verified.
+		// Increasing any of them would result in Bad Request error.
+		// Note some objects couldn't be tested due to account permissions.
+		// These are:
+		// * "custom_objects"		=> You don't have custom objects enabled on your account. To enable, go to Admin Center &gt; Objects and rules &gt; Custom objects.
+		// * "satisfaction_reasons"	=> [Forbidden]You do not have access to this page.
+		"custom_objects":       "100",
+		"satisfaction_reasons": "100",
+		// Objects with strict page size upper limit:
+		"activities":                 "100",
+		"audit_logs":                 "100",
+		"automations":                "100",
+		"brands":                     "100",
+		"deleted_tickets":            "100",
+		"deleted_users":              "100",
+		"email_notifications":        "100",
+		"groups":                     "100",
+		"group_memberships":          "100",
+		"job_statuses":               "100",
+		"macros":                     "100",
+		"organizations":              "1000",
+		"organization_fields":        "100",
+		"organization_memberships":   "100",
+		"organization_subscriptions": "100",
+		"recipient_addresses":        "1000",
+		"requests":                   "100",
+		"satisfaction_ratings":       "100",
+		"suspended_tickets":          "1000",
+		"tags":                       "1000",
+		"ticket_audits":              "100",
+		"ticket_fields":              "100",
+		"ticket_metrics":             "100",
+		"triggers":                   "100",
+		"trigger_categories":         "1000",
+		"users":                      "1000",
+		"user_fields":                "100",
+		"views":                      "100",
+	}, func(objectName string) string {
+		// Any large page size is accepted by API without an error.
+		return "2000"
+	}),
+	"help-center": datautils.NewDefaultMap(nil,
+		func(objectName string) string {
+			// Every object uses 100 as a page size.
+			return "100"
+		},
+	),
+}
+
+func (s ZendeskSchemas) PageSize(
+	moduleID common.ModuleID, objectName string,
+) string {
+	if len(moduleID) == 0 {
+		moduleID = staticschema.RootModuleID
+	}
+
+	// TODO the map should be part of schema.json.
+	return pageSizes[moduleID].Get(objectName)
+}
diff --git a/providers/zendesksupport/metadata/schemas.json b/providers/zendesksupport/metadata/schemas.json
index e5b6ef28..1b328ce3 100755
--- a/providers/zendesksupport/metadata/schemas.json
+++ b/providers/zendesksupport/metadata/schemas.json
@@ -4,7 +4,7 @@
       "id": "help-center",
       "path": "/api/v2",
       "objects": {
-        "article_labels": {
+        "articles/labels": {
           "displayName": "Article Labels",
           "path": "/help_center/articles/labels",
           "responseKey": "labels",
@@ -19,71 +19,6 @@
             "pagination": "cursor"
           }
         },
-        "articles": {
-          "displayName": "Articles",
-          "path": "/help_center/articles/search",
-          "responseKey": "results",
-          "fields": {
-            "author_id": "author_id",
-            "body": "body",
-            "comments_disabled": "comments_disabled",
-            "content_tag_ids": "content_tag_ids",
-            "created_at": "created_at",
-            "draft": "draft",
-            "edited_at": "edited_at",
-            "html_url": "html_url",
-            "id": "id",
-            "label_names": "label_names",
-            "locale": "locale",
-            "outdated": "outdated",
-            "outdated_locales": "outdated_locales",
-            "permission_group_id": "permission_group_id",
-            "position": "position",
-            "promoted": "promoted",
-            "section_id": "section_id",
-            "source_locale": "source_locale",
-            "title": "title",
-            "updated_at": "updated_at",
-            "url": "url",
-            "user_segment_id": "user_segment_id",
-            "user_segment_ids": "user_segment_ids",
-            "vote_count": "vote_count",
-            "vote_sum": "vote_sum"
-          },
-          "custom": {
-            "pagination": "offset"
-          }
-        },
-        "community_posts": {
-          "displayName": "Community Posts",
-          "path": "/help_center/community_posts/search",
-          "responseKey": "results",
-          "fields": {
-            "author_id": "author_id",
-            "closed": "closed",
-            "comment_count": "comment_count",
-            "content_tag_ids": "content_tag_ids",
-            "created_at": "created_at",
-            "details": "details",
-            "featured": "featured",
-            "follower_count": "follower_count",
-            "html_url": "html_url",
-            "id": "id",
-            "non_author_editor_id": "non_author_editor_id",
-            "non_author_updated_at": "non_author_updated_at",
-            "pinned": "pinned",
-            "status": "status",
-            "title": "title",
-            "topic_id": "topic_id",
-            "updated_at": "updated_at",
-            "url": "url",
-            "vote_count": "vote_count",
-            "vote_sum": "vote_sum"
-          },
-          "custom": {
-            "pagination": "offset"
-          }
-        },
         "posts": {
           "displayName": "Posts",
           "path": "/community/posts",
@@ -788,38 +723,6 @@
             "pagination": "offset"
           }
         },
-        "search": {
-          "displayName": "Search Results",
-          "path": "/search",
-          "responseKey": "results",
-          "fields": {
-            "created_at": "created_at",
-            "default": "default",
-            "deleted": "deleted",
-            "description": "description",
-            "id": "id",
-            "name": "name",
-            "result_type": "result_type",
-            "updated_at": "updated_at",
-            "url": "url"
-          },
-          "custom": {
-            "pagination": "offset"
-          }
-        },
-        "session": {
-          "displayName": "Session",
-          "path": "/users/me/session",
-          "responseKey": "session",
-          "fields": {
-            "authenticated_at": "authenticated_at",
-            "id": "id",
-            "last_seen_at": "last_seen_at",
-            "url": "url",
-            "user_id": "user_id"
-          },
-          "custom": {}
-        },
         "sessions": {
           "displayName": "Sessions",
           "path": "/sessions",
diff --git a/providers/zendesksupport/read.go b/providers/zendesksupport/read.go
index 577989bf..d14b5d60 100644
--- a/providers/zendesksupport/read.go
+++ b/providers/zendesksupport/read.go
@@ -40,8 +40,6 @@ func (c *Connector) Read(ctx context.Context, config common.ReadParams) (*common
 	)
 }
 
-const DefaultPageSizeStr = "100"
-
 func (c *Connector) buildReadURL(config common.ReadParams) (*urlbuilder.URL, error) {
 	if len(config.NextPage) != 0 {
 		// Next page
@@ -54,20 +52,22 @@ func (c *Connector) buildReadURL(config common.ReadParams) (*urlbuilder.URL, err
 		return nil, err
 	}
 
+	pageSizeStr := metadata.Schemas.PageSize(c.Module.ID, config.ObjectName)
 	isIncremental := metadata.Schemas.IsIncrementalRead(c.Module.ID, config.ObjectName)
+
 	if isIncremental {
 		// Incremental endpoints requires start query parameter.
 		// Even if no Since parameter is empty the start_time must be set to 0.
 		// This is effectively to say read everything since the beginning of time.
 		// https://developer.zendesk.com/api-reference/ticketing/ticket-management/incremental_exports/#start_time
 		url.WithQueryParam("start_time", formatStartTime(config))
-		url.WithQueryParam("per_page", DefaultPageSizeStr)
+		url.WithQueryParam("per_page", pageSizeStr)
 	} else {
 		// Different objects have different pagination types.
 		// https://developer.zendesk.com/api-reference/introduction/pagination/#using-offset-pagination
 		ptype := metadata.Schemas.LookupPaginationType(c.Module.ID, config.ObjectName)
 		if ptype == "cursor" {
-			url.WithQueryParam("page[size]", DefaultPageSizeStr)
+			url.WithQueryParam("page[size]", pageSizeStr)
 		}
 	}
 
diff --git a/scripts/openapi/zendesksupport/metadata/helpcenter/objects.go b/scripts/openapi/zendesksupport/metadata/helpcenter/objects.go
index 95e11167..53149291 100644
--- a/scripts/openapi/zendesksupport/metadata/helpcenter/objects.go
+++ b/scripts/openapi/zendesksupport/metadata/helpcenter/objects.go
@@ -13,23 +13,23 @@ var (
 	ignoreEndpoints = []string{ // nolint:gochecknoglobals
 		"/api/v2/guide/search", // -> this is unified search for Article, Post, ExternalRecord
 		"/api/v2/help_center/sessions",
+		// Requires additional query parameters.
+		"/api/v2/help_center/articles/search",
+		"/api/v2/help_center/community_posts/search",
 	}
 	objectEndpoints = map[string]string{ // nolint:gochecknoglobals
-		"/api/v2/help_center/articles/search":        "articles",
-		"/api/v2/help_center/articles/labels":        "article_labels",
-		"/api/v2/help_center/community_posts/search": "community_posts",
+		"/api/v2/help_center/articles/labels": "articles/labels",
+	}
+	displayNameOverride = map[string]string{ // nolint:gochecknoglobals
+		"articles/labels": "Article Labels",
 	}
 	objectNameToResponseField = datautils.NewDefaultMap(map[string]string{ // nolint:gochecknoglobals
-		"articles":        "results",
-		"article_labels":  "labels",
-		"community_posts": "results",
+		"articles/labels": "labels",
 	}, func(objectName string) (fieldName string) {
 		return objectName
 	})
 	objectNameToPagination = map[string]string{ // nolint:gochecknoglobals
-		"article_labels":  "cursor",
-		"articles":        "offset",
-		"community_posts": "offset",
+		"articles/labels": "cursor",
 		"posts":           "cursor",
 		"topics":          "cursor",
 		"user_segments":   "cursor",
@@ -47,7 +47,7 @@ func Objects() []metadatadef.ExtendedSchema[metadata.CustomProperties] {
 
 	objects, err := explorer.ReadObjectsGet(
 		api3.NewDenyPathStrategy(ignoreEndpoints),
-		objectEndpoints, nil,
+		objectEndpoints, displayNameOverride,
 		api3.CustomMappingObjectCheck(objectNameToResponseField),
 	)
 	goutils.MustBeNil(err)
diff --git a/scripts/openapi/zendesksupport/metadata/support/objects.go b/scripts/openapi/zendesksupport/metadata/support/objects.go
index cae89367..73b1c7a2 100644
--- a/scripts/openapi/zendesksupport/metadata/support/objects.go
+++ b/scripts/openapi/zendesksupport/metadata/support/objects.go
@@ -46,11 +46,13 @@ var (
 		"/api/v2/automations/search",
 		"/api/v2/views/search",
 		"/api/v2/triggers/search",
+		"/api/v2/search",
 		// Not applicable.
 		"/api/v2/channels/voice/tickets", // only POST method for create.
 		"/api/v2/imports/tickets",        // only POST method for create.
 		"/api/v2/custom_objects/limits/object_limit",
 		"/api/v2/users/me/session/renew",
+		"/api/v2/users/me/session",
 		"/api/v2/locales/current",
 		"/api/v2/locales/detect_best_locale",
 		"/api/v2/brands/check_host_mapping",
@@ -74,14 +76,12 @@ var (
 		"/api/v2/incremental/users/cursor":       "users",
 	}
 	displayNameOverride = map[string]string{ // nolint:gochecknoglobals
-		"search":               "Search Results",
 		"trigger_categories":   "Trigger Categories",
 		"satisfaction_reasons": "Satisfaction Rating Reasons",
 		"ticket_audits":        "Ticket Audits",
 	}
 	objectNameToResponseField = datautils.NewDefaultMap(map[string]string{ // nolint:gochecknoglobals
 		"ticket_audits":        "audits",
-		"search":               "results", // This is "/api/v2/search"
 		"satisfaction_reasons": "reasons",
 	}, func(objectName string) (fieldName string) {
 		return objectName
@@ -118,8 +118,6 @@ var (
 		"resource_collections":       "offset",
 		"satisfaction_ratings":       "cursor",
 		"satisfaction_reasons":       "offset",
-		"search":                     "offset",
-		"session":                    "",
 		"sessions":                   "",
 		"sharing_agreements":         "offset",
 		"suspended_tickets":          "cursor",


Test patch:
diff --git a/providers/zendesksupport/metadata_test.go b/providers/zendesksupport/metadata_test.go
index 68098318..53f16f4a 100644
--- a/providers/zendesksupport/metadata_test.go
+++ b/providers/zendesksupport/metadata_test.go
@@ -117,38 +117,18 @@ func TestListObjectMetadataHelpCenterModule(t *testing.T) { // nolint:funlen,goc
 		},
 		{
 			Name:   "Successfully describe one object with metadata",
-			Input:  []string{"articles"},
+			Input:  []string{"articles/labels"},
 			Server: mockserver.Dummy(),
 			Expected: &common.ListObjectMetadataResult{
 				Result: map[string]common.ObjectMetadata{
-					"articles": {
-						DisplayName: "Articles",
+					"articles/labels": {
+						DisplayName: "Article Labels",
 						FieldsMap: map[string]string{
-							"author_id":           "author_id",
-							"body":                "body",
-							"comments_disabled":   "comments_disabled",
-							"content_tag_ids":     "content_tag_ids",
-							"created_at":          "created_at",
-							"draft":               "draft",
-							"edited_at":           "edited_at",
-							"html_url":            "html_url",
-							"id":                  "id",
-							"label_names":         "label_names",
-							"locale":              "locale",
-							"outdated":            "outdated",
-							"outdated_locales":    "outdated_locales",
-							"permission_group_id": "permission_group_id",
-							"position":            "position",
-							"promoted":            "promoted",
-							"section_id":          "section_id",
-							"source_locale":       "source_locale",
-							"title":               "title",
-							"updated_at":          "updated_at",
-							"url":                 "url",
-							"user_segment_id":     "user_segment_id",
-							"user_segment_ids":    "user_segment_ids",
-							"vote_count":          "vote_count",
-							"vote_sum":            "vote_sum",
+							"id":         "id",
+							"name":       "name",
+							"url":        "url",
+							"created_at": "created_at",
+							"updated_at": "updated_at",
 						},
 					},
 				},
diff --git a/providers/zendesksupport/read_test.go b/providers/zendesksupport/read_test.go
index 799b4baf..93b40b90 100644
--- a/providers/zendesksupport/read_test.go
+++ b/providers/zendesksupport/read_test.go
@@ -151,7 +151,7 @@ func TestIncrementalReadZendeskSupportModule(t *testing.T) { //nolint:funlen,goc
 			Server: mockserver.Conditional{
 				Setup: mockserver.ContentJSON(),
 				If: mockcond.And{
-					mockcond.QueryParam("per_page", "100"),
+					mockcond.QueryParam("per_page", "2000"),
 					mockcond.QueryParam("start_time", "0"),
 					mockcond.PathSuffix("/api/v2/incremental/tickets/cursor"),
 				},
@@ -179,7 +179,7 @@ func TestIncrementalReadZendeskSupportModule(t *testing.T) { //nolint:funlen,goc
 			Server: mockserver.Conditional{
 				Setup: mockserver.ContentJSON(),
 				If: mockcond.And{
-					mockcond.QueryParam("per_page", "100"),
+					mockcond.QueryParam("per_page", "2000"),
 					mockcond.QueryParam("start_time", "1726674883"),
 					mockcond.PathSuffix("/api/v2/incremental/tickets/cursor"),
 				},
@@ -197,7 +197,7 @@ func TestIncrementalReadZendeskSupportModule(t *testing.T) { //nolint:funlen,goc
 			Server: mockserver.Conditional{
 				Setup: mockserver.ContentJSON(),
 				If: mockcond.And{
-					mockcond.QueryParam("per_page", "100"),
+					mockcond.QueryParam("per_page", "1000"),
 					mockcond.QueryParam("start_time", "0"),
 					mockcond.PathSuffix("/api/v2/incremental/users/cursor"),
 				},
@@ -224,7 +224,7 @@ func TestIncrementalReadZendeskSupportModule(t *testing.T) { //nolint:funlen,goc
 			Server: mockserver.Conditional{
 				Setup: mockserver.ContentJSON(),
 				If: mockcond.And{
-					mockcond.QueryParam("per_page", "100"),
+					mockcond.QueryParam("per_page", "1000"),
 					mockcond.QueryParam("start_time", "0"),
 					mockcond.PathSuffix("/api/v2/incremental/users/cursor"),
 				},
@@ -251,7 +251,7 @@ func TestIncrementalReadZendeskSupportModule(t *testing.T) { //nolint:funlen,goc
 			Server: mockserver.Conditional{
 				Setup: mockserver.ContentJSON(),
 				If: mockcond.And{
-					mockcond.QueryParam("per_page", "100"),
+					mockcond.QueryParam("per_page", "1000"),
 					mockcond.QueryParam("start_time", "0"),
 					mockcond.PathSuffix("/api/v2/incremental/organizations"),
 				},