Repository: ucfopen/canvasapi
Pull request #614 (2023-04-26 18:00:05)

Update `PaginatedList`
Based on #605, `PaginatedList` could not process requests which return pagination info in the response body. This update checks for `Link` headers before checking the `meta` property in the response body.

Resolves #605
Related: #565 

codecov[bot]: ## [Codecov](https://codecov.io/gh/ucfopen/canvasapi/pull/614?src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=ucfopen) Report
> Merging [#614](https://codecov.io/gh/ucfopen/canvasapi/pull/614?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=ucfopen) (c52f236) into [develop](https://codecov.io/gh/ucfopen/canvasapi/commit/ac2fc96e27a7fad8263cd3a7d07b007cc7ab8616?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=ucfopen) (ac2fc96) will **not change** coverage.
> The diff coverage is `100.00%`.

```diff
@@            Coverage Diff            @@
##           develop      #614   +/-   ##
=========================================
  Coverage   100.00%   100.00%           
=========================================
  Files           72        72           
  Lines         3684      3688    +4     
=========================================
+ Hits          3684      3688    +4     
```


| [Impacted Files](https://codecov.io/gh/ucfopen/canvasapi/pull/614?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=ucfopen) | Coverage Δ | |
|---|---|---|
| [canvasapi/paginated\_list.py](https://codecov.io/gh/ucfopen/canvasapi/pull/614?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=ucfopen#diff-Y2FudmFzYXBpL3BhZ2luYXRlZF9saXN0LnB5) | `100.00% <100.00%> (ø)` | |

:mega: We’re building smart automated test selection to slash your CI/CD build times. [Learn more](https://about.codecov.io/iterative-testing/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=ucfopen)

bennettscience: I've made the changes requested, all tests passing still.

Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 39d8e44..00d1eb3 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,10 @@
 
 ## [Unreleased]
 
+### General
+
+- Added support for pagination with metadata when headers are missing (Thanks, [@bennettscience](https://github.com/bennettscience))
+
 ## [3.1.0] - 2023-04-21
 
 ### New Endpoint Coverage
diff --git a/canvasapi/paginated_list.py b/canvasapi/paginated_list.py
index ce14420..c568874 100644
--- a/canvasapi/paginated_list.py
+++ b/canvasapi/paginated_list.py
@@ -57,8 +57,22 @@ class PaginatedList(object):
         )
         data = response.json()
         self._next_url = None
+        # Check the response headers first. This is the normal Canvas convention
+        # for pagination, but there are endpoints which return a `meta` property
+        # for pagination instead.
+        # See https://github.com/ucfopen/canvasapi/discussions/605
+        if response.links:
+            next_link = response.links.get("next")
+        elif isinstance(data, dict) and "meta" in data:
+            # requests parses headers into dicts, this returns the same
+            # structure so the regex will still work.
+            try:
+                next_link = {"url": data["meta"]["pagination"]["next"], "rel": "next"}
+            except KeyError:
+                next_link = None
+        else:
+            next_link = None
 
-        next_link = response.links.get("next")
         regex = r"{}(.*)".format(re.escape(self._requester.base_url))
 
         self._next_url = (
@@ -73,8 +87,9 @@ class PaginatedList(object):
             try:
                 data = data[self._root]
             except KeyError:
-                # TODO: Fix this message to make more sense to an end user.
-                raise ValueError("Invalid root value specified.")
+                raise ValueError(
+                    "The key <{}> does not exist in the response.".format(self._root)
+                )
 
         for element in data:
             if element is not None:


Test patch:
diff --git a/tests/fixtures/paginated_list.json b/tests/fixtures/paginated_list.json
index e37006f..db7a3d4 100644
--- a/tests/fixtures/paginated_list.json
+++ b/tests/fixtures/paginated_list.json
@@ -114,5 +114,66 @@
 			}
 		],
 		"status_code": 200
-	}
-}
\ No newline at end of file
+	},
+	"no_header_4_2_pages_p1": {
+		"method": "ANY",
+		"endpoint": "no_header_four_objects_two_pages",
+		"data": {
+			"assessments": [
+				{
+					"id": "1",
+					"name": "object 1"
+				},
+				{
+					"id": "2",
+					"name": "object 2"
+				}
+			],
+			"meta": {
+				"pagination": {
+					"next": "https://example.com/api/v1/no_header_four_objects_two_pages?page=2"
+				}
+			}
+		},
+		"status_code": 200
+	},
+	"no_header_4_2_pages_p2": {
+		"method": "ANY",
+		"endpoint": "no_header_four_objects_two_pages?page=2",
+		"data": {
+			"assessments": [
+				{
+					"id": "3",
+					"name": "object 3"
+				},
+				{
+					"id": "4",
+					"name": "object 4"
+				}
+			]
+		},
+		"status_code": 200
+	},
+	"no_header_no_next_key": {
+		"method": "ANY",
+		"endpoint": "no_header_no_next_key",
+		"data": {
+			"assessments": [
+				{
+					"id": "1",
+					"name": "object 1"
+				},
+				{
+					"id": "2",
+					"name": "object 2"
+				}
+			],
+			"meta": {
+				"pagination": {
+					"prev": "https://example.com/api/v1/previous"
+				}
+			}
+		}
+	},
+	"status_code": 200
+}
diff --git a/tests/test_paginated_list.py b/tests/test_paginated_list.py
index 3b9e604..9783e31 100644
--- a/tests/test_paginated_list.py
+++ b/tests/test_paginated_list.py
@@ -156,6 +156,9 @@ class TestPaginatedList(unittest.TestCase):
 
         with self.assertRaises(ValueError):
             pag_list[0]
+            self.assertEqual(
+                pag_list[0], "The key <wrong> does not exist in the response."
+            )
 
     def test_root_element(self, m):
         register_uris({"account": ["get_enrollment_terms"]}, m)
@@ -202,3 +205,31 @@ class TestPaginatedList(unittest.TestCase):
 
         with self.assertRaises(IndexError):
             pag_list[:-1]
+
+    def test_paginated_list_no_header(self, m):
+        register_uris(
+            {"paginated_list": ["no_header_4_2_pages_p1", "no_header_4_2_pages_p2"]}, m
+        )
+
+        pag_list = PaginatedList(
+            User,
+            self.requester,
+            "GET",
+            "no_header_four_objects_two_pages",
+            _root="assessments",
+        )
+
+        self.assertIsInstance(pag_list, PaginatedList)
+        self.assertEqual(len(list(pag_list)), 4)
+        self.assertIsInstance(pag_list[0], User)
+
+    def test_paginated_list_no_header_no_next(self, m):
+        register_uris({"paginated_list": ["no_header_no_next_key"]}, m)
+
+        pag_list = PaginatedList(
+            User, self.requester, "GET", "no_header_no_next_key", _root="assessments"
+        )
+
+        self.assertIsInstance(pag_list, PaginatedList)
+        self.assertEqual(len(list(pag_list)), 2)
+        self.assertIsInstance(pag_list[0], User)