Repository: cloudflare/cloudflare-go
Pull request #1206 (2023-02-10 19:02:03)

Add support for Pagination in Tunnels API
Signed-off-by: Haytham Abuelfutuh <haytham@afutuh.com>

## Description

[GET Tunnels API](https://api.cloudflare.com/#cloudflare-tunnel-list-cloudflare-tunnels) supports pagination but the cloudflare-go client doesn't. I copied the way it's done from Zone Contexts into this API. Generally speaking I think all APIs that support pagination should ideally support `ReqOption`s

## Example:
```golang
actual, err := client.Tunnels(context.Background(), AccountIdentifier(testAccountID),
		TunnelListParams{}, WithPagination(PaginationOptions{PerPage: 1, Page: 2}))
```
 
## Has your change been tested?

Using the fork, we have deployed a service that syncs the status of the channels to a local DB and now is able to iterate through all tunnels we've (100+ in total)

## Types of changes

What sort of change does your code introduce/modify?

- [ ] Bug fix (non-breaking change which fixes an issue)
- [X] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would cause existing functionality to change)

## Checklist:

- [X] My code follows the code style of this project.
- [ ] My change requires a change to the documentation.
- [ ] I have updated the documentation accordingly.
- [X] I have added tests to cover my changes.
- [X] All new and existing tests passed.
- [X] This change is using publicly documented (api.cloudflare.com or developers.cloudflare.com) and stable APIs.

[1]: https://help.github.com/articles/closing-issues-using-keywords/


Patch:
diff --git a/.changelog/1206.txt b/.changelog/1206.txt
new file mode 100644
index 000000000..fb12a08eb
--- /dev/null
+++ b/.changelog/1206.txt
@@ -0,0 +1,3 @@
+```release-note:enhancement
+tunnels: automatically paginate `ListTunnels`
+```
diff --git a/tunnel.go b/tunnel.go
index d933eedfc..8ae91d391 100644
--- a/tunnel.go
+++ b/tunnel.go
@@ -51,8 +51,12 @@ type TunnelConnection struct {
 type TunnelsDetailResponse struct {
 	Result []Tunnel `json:"result"`
 	Response
+	ResultInfo `json:"result_info"`
 }
 
+// listTunnelsDefaultPageSize represents the default per_page size of the API.
+var listTunnelsDefaultPageSize int = 100
+
 // TunnelDetailResponse is used for representing the API response payload for
 // a single tunnel.
 type TunnelDetailResponse struct {
@@ -171,29 +175,54 @@ type TunnelListParams struct {
 	UUID      string     `url:"uuid,omitempty"` // the tunnel ID
 	IsDeleted *bool      `url:"is_deleted,omitempty"`
 	ExistedAt *time.Time `url:"existed_at,omitempty"`
+
+	ResultInfo
 }
 
 // Tunnels lists all tunnels.
 //
 // API reference: https://api.cloudflare.com/#cloudflare-tunnel-list-cloudflare-tunnels
-func (api *API) Tunnels(ctx context.Context, rc *ResourceContainer, params TunnelListParams) ([]Tunnel, error) {
+func (api *API) Tunnels(ctx context.Context, rc *ResourceContainer, params TunnelListParams) ([]Tunnel, *ResultInfo, error) {
 	if rc.Identifier == "" {
-		return []Tunnel{}, ErrMissingAccountID
+		return []Tunnel{}, &ResultInfo{}, ErrMissingAccountID
 	}
 
-	uri := buildURI(fmt.Sprintf("/accounts/%s/cfd_tunnel", rc.Identifier), params)
+	autoPaginate := true
+	if params.PerPage >= 1 || params.Page >= 1 {
+		autoPaginate = false
+	}
 
-	res, err := api.makeRequestContext(ctx, http.MethodGet, uri, nil)
-	if err != nil {
-		return []Tunnel{}, err
+	if params.PerPage < 1 {
+		params.PerPage = listTunnelsDefaultPageSize
 	}
 
-	var argoDetailsResponse TunnelsDetailResponse
-	err = json.Unmarshal(res, &argoDetailsResponse)
-	if err != nil {
-		return []Tunnel{}, fmt.Errorf("%s: %w", errUnmarshalError, err)
+	if params.Page < 1 {
+		params.Page = 1
 	}
-	return argoDetailsResponse.Result, nil
+
+	var records []Tunnel
+	var listResponse TunnelsDetailResponse
+
+	for {
+		uri := buildURI(fmt.Sprintf("/accounts/%s/cfd_tunnel", rc.Identifier), params)
+		res, err := api.makeRequestContext(ctx, http.MethodGet, uri, nil)
+		if err != nil {
+			return []Tunnel{}, &ResultInfo{}, err
+		}
+
+		err = json.Unmarshal(res, &listResponse)
+		if err != nil {
+			return []Tunnel{}, &ResultInfo{}, fmt.Errorf("%s: %w", errUnmarshalError, err)
+		}
+
+		records = append(records, listResponse.Result...)
+		params.ResultInfo = listResponse.ResultInfo.Next()
+		if params.ResultInfo.Done() || !autoPaginate {
+			break
+		}
+	}
+
+	return records, &listResponse.ResultInfo, nil
 }
 
 // Tunnel returns a single Argo tunnel.


Test patch:
diff --git a/testdata/fixtures/tunnel/multiple_full.json b/testdata/fixtures/tunnel/multiple_full.json
index 409911a8f..586dd9003 100644
--- a/testdata/fixtures/tunnel/multiple_full.json
+++ b/testdata/fixtures/tunnel/multiple_full.json
@@ -20,5 +20,11 @@
         }
       ]
     }
-  ]
+  ],
+  "result_info": {
+    "count": 1,
+    "page": 1,
+    "per_page": 20,
+    "total_count": 1
+  }
 }
diff --git a/tunnel_test.go b/tunnel_test.go
index 48f8f386c..5e6291047 100644
--- a/tunnel_test.go
+++ b/tunnel_test.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"net/url"
 	"testing"
 	"time"
 
@@ -40,7 +41,57 @@ func TestTunnels(t *testing.T) {
 		}},
 	}}
 
-	actual, err := client.Tunnels(context.Background(), AccountIdentifier(testAccountID), TunnelListParams{UUID: "f174e90a-fafe-4643-bbbc-4a0ed4fc8415"})
+	actual, _, err := client.Tunnels(context.Background(), AccountIdentifier(testAccountID), TunnelListParams{UUID: "f174e90a-fafe-4643-bbbc-4a0ed4fc8415"})
+
+	if assert.NoError(t, err) {
+		assert.Equal(t, want, actual)
+	}
+}
+
+func TestTunnelsPagination(t *testing.T) {
+	setup()
+	defer teardown()
+
+	handler := func(w http.ResponseWriter, r *http.Request) {
+		assert.Equal(t, http.MethodGet, r.Method, "Expected method 'GET', got %s", r.Method)
+		w.Header().Set("content-type", "application/json")
+		qry, _ := url.Parse(r.RequestURI)
+		assert.Equal(t, "blog", qry.Query().Get("name"))
+		assert.Equal(t, "2", qry.Query().Get("page"))
+		assert.Equal(t, "1", qry.Query().Get("per_page"))
+		fmt.Fprint(w, loadFixture("tunnel", "multiple_full"))
+	}
+
+	mux.HandleFunc("/accounts/"+testAccountID+"/cfd_tunnel", handler)
+
+	createdAt, _ := time.Parse(time.RFC3339, "2009-11-10T23:00:00Z")
+	deletedAt, _ := time.Parse(time.RFC3339, "2009-11-10T23:00:00Z")
+	want := []Tunnel{
+		{
+			ID:        "f174e90a-fafe-4643-bbbc-4a0ed4fc8415",
+			Name:      "blog",
+			CreatedAt: &createdAt,
+			DeletedAt: &deletedAt,
+			Connections: []TunnelConnection{{
+				ColoName:           "DFW",
+				ID:                 "f174e90a-fafe-4643-bbbc-4a0ed4fc8415",
+				IsPendingReconnect: false,
+				ClientID:           "dc6472cc-f1ae-44a0-b795-6b8a0ce29f90",
+				ClientVersion:      "2022.2.0",
+				OpenedAt:           "2021-01-25T18:22:34.317854Z",
+				OriginIP:           "198.51.100.1",
+			}},
+		},
+	}
+
+	actual, _, err := client.Tunnels(context.Background(), AccountIdentifier(testAccountID),
+		TunnelListParams{
+			Name: "blog",
+			ResultInfo: ResultInfo{
+				Page:    2,
+				PerPage: 1,
+			},
+		})
 
 	if assert.NoError(t, err) {
 		assert.Equal(t, want, actual)