Repository: Kinto/kinto-http.py

Issue:
Expose methods for paginated lists
Currently there is no way to consume the content of individual pages using `get_records()`.

We could add a generator `get_paginated_records()` that would yield individual pages...

```py
for page_records in client.get_paginated_records(_limit=100):
     print(len(page_records))
```

Discussion:

Patch:
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index 0ac31c9..c982642 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -7,7 +7,9 @@ This document describes changes between each past release.
 10.2.0 (unreleased)
 ===================
 
-- Nothing changed yet.
+**New feature**
+
+- Created new method on client to get paginated records ``get_paginated_records``. (#175)
 
 
 10.1.1 (2018-11-13)
diff --git a/README.rst b/README.rst
index fa79a02..2052551 100644
--- a/README.rst
+++ b/README.rst
@@ -265,6 +265,11 @@ A record is a dict with the "permissions" and "data" keys.
     # Or retrieve all records.
     records = client.get_records(collection='todos', bucket='default')
 
+   # Or retrieve records page by page.
+   for page in client.get_paginated_records(collection='todos', bucket='default'):
+      # Do something with each page
+      print(page)
+
     # Or retrieve records timestamp.
     records_timestamp = client.get_records_timestamp(collection='todos', bucket='default')
 
diff --git a/kinto_http/__init__.py b/kinto_http/__init__.py
index c39e322..f8e8700 100644
--- a/kinto_http/__init__.py
+++ b/kinto_http/__init__.py
@@ -655,6 +655,29 @@ class Client(object):
                                      collection=collection)
         return self._paginated(endpoint, **kwargs)
 
+    def get_paginated_records(self, *, collection=None, bucket=None, **kwargs):
+        endpoint = self.get_endpoint('records',
+                                     bucket=bucket,
+                                     collection=collection)
+
+        return self._paginated_generator(endpoint, **kwargs)
+
+    def _paginated_generator(self, endpoint, *, if_none_match=None, **kwargs):
+        headers = {}
+        if if_none_match is not None:
+            headers['If-None-Match'] = utils.quote(if_none_match)
+
+        record_resp, headers = self.session.request(
+            'get', endpoint, headers=headers, params=kwargs)
+
+        if record_resp:
+            yield record_resp
+
+        if 'next-page' in map(str.lower, headers.keys()):
+            next_page = headers['Next-Page']
+            yield from self._paginated_generator(next_page,
+                                                 if_none_match=if_none_match)
+
     def get_record(self, *, id, collection=None, bucket=None):
         endpoint = self.get_endpoint('record', id=id,
                                      bucket=bucket,


Test patch:
diff --git a/kinto_http/tests/functional.py b/kinto_http/tests/functional.py
index e2351ee..15bc5c7 100644
--- a/kinto_http/tests/functional.py
+++ b/kinto_http/tests/functional.py
@@ -311,6 +311,19 @@ class FunctionalTest(unittest.TestCase):
         records = client.get_records()
         assert len(records) == 10
 
+    def test_records_generator_retrieval(self):
+        client = Client(server_url=self.server_url, auth=self.auth,
+                        bucket='mozilla', collection='payments')
+        client.create_bucket()
+        client.create_collection()
+        for i in range(10):
+            client.create_record(data={'foo': 'bar'},
+                                 permissions={'read': ['account:alexis']})
+
+        pages = list(client.get_paginated_records())
+
+        assert len(pages) == 2
+
     def test_single_record_save(self):
         client = Client(server_url=self.server_url, auth=self.auth,
                         bucket='mozilla', collection='payments')
diff --git a/kinto_http/tests/test_client.py b/kinto_http/tests/test_client.py
index 7d45230..f737096 100644
--- a/kinto_http/tests/test_client.py
+++ b/kinto_http/tests/test_client.py
@@ -833,6 +833,36 @@ class RecordTest(unittest.TestCase):
             {'id': '6', 'value': 'item6'},
         ]
 
+    def test_pagination_is_followed_generator(self):
+        # Mock the calls to request.
+        link = ('http://example.org/buckets/buck/collections/coll/records/'
+                '?token=1234')
+
+        response = [
+            # First one returns a list of items with a pagination token.
+            build_response(
+                [{'id': '1', 'value': 'item1'},
+                 {'id': '2', 'value': 'item2'}, ],
+                {'Next-Page': link}),
+            build_response(
+                [{'id': '3', 'value': 'item3'},
+                 {'id': '4', 'value': 'item4'}, ],
+                {'Next-Page': link}),
+            # Second one returns a list of items without a pagination token.
+            build_response(
+                [{'id': '5', 'value': 'item5'},
+                 {'id': '6', 'value': 'item6'}, ],
+            ),
+        ]
+
+        self.session.request.side_effect = response
+
+        # Build repsonses for assertion without next page
+        response = [record[0] for record in response]
+
+        for index, page_records in enumerate(self.client.get_paginated_records()):
+            assert response[index] == page_records
+
     def test_pagination_is_followed_for_number_of_pages(self):
         # Mock the calls to request.
         link = ('http://example.org/buckets/buck/collections/coll/records/'
@@ -911,6 +941,41 @@ class RecordTest(unittest.TestCase):
         self.session.request.assert_any_call(
             'get', link, headers={'If-None-Match': '"1234"'}, params={})
 
+    def test_pagination_generator_if_none_match(self):
+        link = ('http://example.org/buckets/buck/collections/coll/records/'
+                '?token=1234')
+
+        response = [
+            # First one returns a list of items with a pagination token.
+            build_response(
+                [{'id': '1', 'value': 'item1'},
+                 {'id': '2', 'value': 'item2'}, ],
+                {'Next-Page': link}),
+            build_response(
+                [{'id': '3', 'value': 'item3'},
+                 {'id': '4', 'value': 'item4'}, ],
+                {'Next-Page': link}),
+            # Second one returns a list of items without a pagination token.
+            build_response(
+                [{'id': '5', 'value': 'item5'},
+                 {'id': '6', 'value': 'item6'}, ],
+            ),
+        ]
+
+        self.session.request.side_effect = response
+
+        # Build repsonses for assertion without next page
+        response = [record[0] for record in response]
+
+        for index, page_records in enumerate(self.client.get_paginated_records(
+                                             if_none_match="1234")):
+            # Check that the If-None-Match header is present in the requests.
+            assert response[index] == page_records
+
+        # Check that the If-None-Match header is present in the requests.
+        self.session.request.assert_any_call(
+            'get', link, headers={'If-None-Match': '"1234"'}, params={})
+
     def test_collection_can_delete_a_record(self):
         mock_response(self.session, data={'id': 1234})
         resp = self.client.delete_record(id=1234)