Repository: Kinto/kinto

Issue:
Cannot retrieve more than `storage_max_fetch_size` records through pagination
It seems that the 10000 server limit is enforced on the number of records that have been queried, and not the number of records returned.

Eg: using `_limit=5000` will allow two requests to be made and "succeed" (with proper content and `Next-Page` headers). Using `_limit=100` will allow 100 requests. But using `_limit=10000` will only allow one.

https://github.com/mozilla-services/buildhub/issues/121#issuecomment-307729540
Cannot retrieve more than `storage_max_fetch_size` records through pagination
It seems that the 10000 server limit is enforced on the number of records that have been queried, and not the number of records returned.

Eg: using `_limit=5000` will allow two requests to be made and "succeed" (with proper content and `Next-Page` headers). Using `_limit=100` will allow 100 requests. But using `_limit=10000` will only allow one.

https://github.com/mozilla-services/buildhub/issues/121#issuecomment-307729540


Discussion:

Resolving patch:
diff --git a/kinto/core/resource/__init__.py b/kinto/core/resource/__init__.py
--- a/kinto/core/resource/__init__.py
+++ b/kinto/core/resource/__init__.py
@@ -904,12 +904,16 @@ def _extract_partial_fields(self):
     def _extract_limit(self):
         """Extract limit value from QueryString parameters."""
         paginate_by = self.request.registry.settings['paginate_by']
+        max_fetch_size = self.request.registry.settings['storage_max_fetch_size']
         limit = self.request.validated['querystring'].get('_limit', paginate_by)
 
         # If limit is higher than paginate_by setting, ignore it.
         if limit and paginate_by:
             limit = min(limit, paginate_by)
 
+        # If limit is higher than what storage can retrieve, ignore it.
+        limit = min(limit, max_fetch_size) if limit else max_fetch_size
+
         return limit
 
     def _extract_filters(self):
diff --git a/kinto/core/storage/postgresql/__init__.py b/kinto/core/storage/postgresql/__init__.py
--- a/kinto/core/storage/postgresql/__init__.py
+++ b/kinto/core/storage/postgresql/__init__.py
@@ -407,7 +407,7 @@ def delete_all(self, collection_id, parent_id, filters=None,
                               {conditions_filter}
                               {pagination_rules}
                         {sorting}
-                        {pagination_limit}
+                        LIMIT :pagination_limit
                 )
                 DELETE
                 FROM records
@@ -432,7 +432,7 @@ def delete_all(self, collection_id, parent_id, filters=None,
                           {conditions_filter}
                           {pagination_rules}
                     {sorting}
-                    {pagination_limit}
+                    LIMIT :pagination_limit
             )
             DELETE
             FROM records
@@ -480,9 +480,9 @@ def delete_all(self, collection_id, parent_id, filters=None,
             safeholders['pagination_rules'] = 'AND {}'.format(sql)
             placeholders.update(**holders)
 
-        if limit:
-            # We validate the limit value in the resource class as integer.
-            safeholders['pagination_limit'] = 'LIMIT {}'.format(limit)
+        # Limit the number of results (pagination).
+        limit = min(self._max_fetch_size, limit) if limit else self._max_fetch_size
+        placeholders['pagination_limit'] = limit
 
         with self.client.connect() as conn:
             result = conn.execute(query.format_map(safeholders), placeholders)
@@ -567,7 +567,6 @@ def get_all(self, collection_id, parent_id, filters=None, sorting=None,
              WHERE {parent_id_filter}
                AND collection_id = :collection_id
                {conditions_filter}
-             LIMIT {max_fetch_size}
         ),
         fake_deleted AS (
             SELECT (:deleted_field)::JSONB AS data
@@ -595,7 +594,7 @@ def get_all(self, collection_id, parent_id, filters=None, sorting=None,
           FROM paginated_records AS p JOIN all_records AS a ON (a.id = p.id),
                total_filtered
           {sorting}
-          {pagination_limit};
+          LIMIT :pagination_limit;
         """
         deleted_field = json.dumps(dict([(deleted_field, True)]))
 
@@ -606,7 +605,6 @@ def get_all(self, collection_id, parent_id, filters=None, sorting=None,
 
         # Safe strings
         safeholders = defaultdict(str)
-        safeholders['max_fetch_size'] = self._max_fetch_size
 
         # Handle parent_id as a regex only if it contains *
         if '*' in parent_id:
@@ -637,15 +635,15 @@ def get_all(self, collection_id, parent_id, filters=None, sorting=None,
             safeholders['pagination_rules'] = 'WHERE {}'.format(sql)
             placeholders.update(**holders)
 
-        if limit:
-            # We validate the limit value in the resource class as integer.
-            safeholders['pagination_limit'] = 'LIMIT {}'.format(limit)
+        # Limit the number of results (pagination).
+        limit = min(self._max_fetch_size, limit) if limit else self._max_fetch_size
+        placeholders['pagination_limit'] = limit
 
         with self.client.connect(readonly=True) as conn:
             result = conn.execute(query.format_map(safeholders), placeholders)
             retrieved = result.fetchmany(self._max_fetch_size)
 
-        if not len(retrieved):
+        if len(retrieved) == 0:
             return [], 0
 
         count_total = retrieved[0]['count_total']


Test patch:
diff --git a/kinto/core/storage/testing.py b/kinto/core/storage/testing.py
--- a/kinto/core/storage/testing.py
+++ b/kinto/core/storage/testing.py
@@ -652,34 +652,6 @@ def test_get_all_can_filter_with_wildchars(self):
         results, count = self.storage.get_all(filters=filters, **self.storage_kw)
         self.assertEqual(len(results), 2)
 
-    def test_get_all_handle_a_pagination_rules(self):
-        for x in range(10):
-            record = dict(self.record)
-            record["number"] = x % 3
-            self.create_record(record)
-
-        records, total_records = self.storage.get_all(
-            limit=5,
-            pagination_rules=[
-                [Filter('number', 1, utils.COMPARISON.GT)]
-            ], **self.storage_kw)
-        self.assertEqual(total_records, 10)
-        self.assertEqual(len(records), 3)
-
-    def test_get_all_handle_all_pagination_rules(self):
-        for x in range(10):
-            record = dict(self.record)
-            record["number"] = x % 3
-            last_record = self.create_record(record)
-
-        records, total_records = self.storage.get_all(
-            limit=5, pagination_rules=[
-                [Filter('number', 1, utils.COMPARISON.GT)],
-                [Filter('id', last_record['id'], utils.COMPARISON.EQ)],
-            ], **self.storage_kw)
-        self.assertEqual(total_records, 10)
-        self.assertEqual(len(records), 4)
-
 
 class TimestampsTest:
     def test_timestamp_are_incremented_on_create(self):
@@ -1106,15 +1078,6 @@ def test_delete_all_supports_sorting(self):
         self.assertEqual(count, 3)
         self.assertEqual(records[0]['foo'], 2)
 
-    def test_delete_all_supports_pagination_rules(self):
-        for i in range(6):
-            self.create_record({'foo': i})
-
-        pagination_rules = [[Filter('foo', 3, utils.COMPARISON.GT)]]
-        deleted = self.storage.delete_all(limit=4, pagination_rules=pagination_rules,
-                                          **self.storage_kw)
-        self.assertEqual(len(deleted), 2)
-
     def test_purge_deleted_remove_all_tombstones(self):
         self.create_record()
         self.create_record()
@@ -1358,6 +1321,43 @@ def test_pagination_rules_on_last_modified_apply_to_deleted_records(self):
         self.assertIn('deleted', records[0])
         self.assertNotIn('deleted', records[1])
 
+    def test_get_all_handle_a_pagination_rules(self):
+        for x in range(10):
+            record = dict(self.record)
+            record["number"] = x % 3
+            self.create_record(record)
+
+        records, total_records = self.storage.get_all(
+            limit=5,
+            pagination_rules=[
+                [Filter('number', 1, utils.COMPARISON.GT)]
+            ], **self.storage_kw)
+        self.assertEqual(total_records, 10)
+        self.assertEqual(len(records), 3)
+
+    def test_get_all_handle_all_pagination_rules(self):
+        for x in range(10):
+            record = dict(self.record)
+            record["number"] = x % 3
+            last_record = self.create_record(record)
+
+        records, total_records = self.storage.get_all(
+            limit=5, pagination_rules=[
+                [Filter('number', 1, utils.COMPARISON.GT)],
+                [Filter('id', last_record['id'], utils.COMPARISON.EQ)],
+            ], **self.storage_kw)
+        self.assertEqual(total_records, 10)
+        self.assertEqual(len(records), 4)
+
+    def test_delete_all_supports_pagination_rules(self):
+        for i in range(6):
+            self.create_record({'foo': i})
+
+        pagination_rules = [[Filter('foo', 3, utils.COMPARISON.GT)]]
+        deleted = self.storage.delete_all(limit=4, pagination_rules=pagination_rules,
+                                          **self.storage_kw)
+        self.assertEqual(len(deleted), 2)
+
 
 class ParentRecordAccessTest:
     def test_parent_cannot_access_other_parent_record(self):
diff --git a/tests/core/support.py b/tests/core/support.py
--- a/tests/core/support.py
+++ b/tests/core/support.py
@@ -6,6 +6,8 @@
 
 from kinto.core.authorization import PRIVATE
 from kinto.core import testing
+from kinto.core.storage.exceptions import BackendError
+from kinto.core.utils import sqlalchemy
 
 from .testapp import main as testapp
 
@@ -82,3 +84,52 @@ def wrapped(*args, **kwargs):
                 return f(*args, **kwargs)
         return wrapped
     return wrapper
+
+
+class PostgreSQLTest(BaseWebTest):
+
+    @classmethod
+    def get_app_settings(cls, extras=None):
+        settings = super().get_app_settings(extras)
+        if sqlalchemy is not None:
+            from .test_storage import PostgreSQLStorageTest
+            from .test_cache import PostgreSQLCacheTest
+            from .test_permission import PostgreSQLPermissionTest
+            settings.update(**PostgreSQLStorageTest.settings)
+            settings.update(**PostgreSQLCacheTest.settings)
+            settings.update(**PostgreSQLPermissionTest.settings)
+            settings.pop('storage_poolclass', None)
+            settings.pop('cache_poolclass', None)
+            settings.pop('permission_poolclass', None)
+        return settings
+
+    def run_failing_batch(self):
+        patch = mock.patch.object(
+            self.storage,
+            'delete_all',
+            side_effect=BackendError('boom'))
+        self.addCleanup(patch.stop)
+        patch.start()
+        request_create = {
+            'method': 'POST',
+            'path': '/mushrooms',
+            'body': {'data': {'name': 'Amanite'}}
+        }
+        request_delete = {
+            'method': 'DELETE',
+            'path': '/mushrooms'
+        }
+        body = {'requests': [request_create, request_create, request_delete]}
+        self.app.post_json('/batch', body, headers=self.headers, status=503)
+
+    def run_failing_post(self):
+        patch = mock.patch.object(
+            self.permission,
+            'add_principal_to_ace',
+            side_effect=BackendError('boom'))
+        self.addCleanup(patch.stop)
+        patch.start()
+        self.app.post_json('/psilos',
+                           {'data': {'name': 'Amanite'}},
+                           headers=self.headers,
+                           status=503)
diff --git a/tests/core/test_storage.py b/tests/core/test_storage.py
--- a/tests/core/test_storage.py
+++ b/tests/core/test_storage.py
@@ -129,6 +129,21 @@ def test_number_of_fetched_records_can_be_limited_in_settings(self):
         limited = self.backend.load_from_config(config)
 
         results, count = limited.get_all(**self.storage_kw)
+        self.assertEqual(count, 4)
+        self.assertEqual(len(results), 2)
+
+    def test_number_of_fetched_records_is_per_page(self):
+        for i in range(10):
+            self.create_record({'number': i})
+
+        settings = {**self.settings, 'storage_max_fetch_size': 2}
+        config = self._get_config(settings=settings)
+        backend = self.backend.load_from_config(config)
+
+        results, count = backend.get_all(pagination_rules=[
+                                             [Filter('number', 1, COMPARISON.GT)]
+                                         ], **self.storage_kw)
+        self.assertEqual(count, 10)
         self.assertEqual(len(results), 2)
 
     def test_connection_is_rolledback_if_error_occurs(self):
diff --git a/tests/core/test_views_postgresql.py b/tests/core/test_views_postgresql.py
new file mode 100644
--- /dev/null
+++ b/tests/core/test_views_postgresql.py
@@ -0,0 +1,39 @@
+import unittest
+
+from kinto.core.testing import skip_if_no_postgresql
+
+from .support import PostgreSQLTest
+
+
+@skip_if_no_postgresql
+class PaginationTest(PostgreSQLTest, unittest.TestCase):
+
+    @classmethod
+    def get_app_settings(cls, extras=None):
+        settings = super().get_app_settings(extras)
+        settings["storage_max_fetch_size"] = 4
+        return settings
+
+    def setUp(self):
+        super().setUp()
+        for i in range(10):
+            self.app.post_json('/mushrooms', {'data': {'name': str(i)}},
+                               headers=self.headers)
+
+    def test_storage_max_fetch_size_is_per_page(self):
+        resp = self.app.get('/mushrooms?_limit=6', headers=self.headers)
+        self.assertIn("Next-Page", resp.headers)
+        self.assertEqual(int(resp.headers["Total-Records"]), 10)
+        self.assertEqual(len(resp.json['data']), 4)
+
+        next_page_url = resp.headers["Next-Page"].replace("http://localhost/v0", "")
+        resp = self.app.get(next_page_url, headers=self.headers)
+        self.assertIn("Next-Page", resp.headers)
+        self.assertEqual(int(resp.headers["Total-Records"]), 10)
+        self.assertEqual(len(resp.json['data']), 4)
+
+        next_page_url = resp.headers["Next-Page"].replace("http://localhost/v0", "")
+        resp = self.app.get(next_page_url, headers=self.headers)
+        self.assertNotIn("Next-Page", resp.headers)
+        self.assertEqual(int(resp.headers["Total-Records"]), 10)
+        self.assertEqual(len(resp.json['data']), 2)
diff --git a/tests/core/test_views_transaction.py b/tests/core/test_views_transaction.py
--- a/tests/core/test_views_transaction.py
+++ b/tests/core/test_views_transaction.py
@@ -12,56 +12,7 @@
 from kinto.core import events
 from kinto.core.testing import skip_if_no_postgresql
 
-from .support import BaseWebTest, USER_PRINCIPAL
-
-
-class PostgreSQLTest(BaseWebTest):
-
-    @classmethod
-    def get_app_settings(cls, extras=None):
-        settings = super().get_app_settings(extras)
-        if sqlalchemy is not None:
-            from .test_storage import PostgreSQLStorageTest
-            from .test_cache import PostgreSQLCacheTest
-            from .test_permission import PostgreSQLPermissionTest
-            settings.update(**PostgreSQLStorageTest.settings)
-            settings.update(**PostgreSQLCacheTest.settings)
-            settings.update(**PostgreSQLPermissionTest.settings)
-            settings.pop('storage_poolclass', None)
-            settings.pop('cache_poolclass', None)
-            settings.pop('permission_poolclass', None)
-        return settings
-
-    def run_failing_batch(self):
-        patch = mock.patch.object(
-            self.storage,
-            'delete_all',
-            side_effect=BackendError('boom'))
-        self.addCleanup(patch.stop)
-        patch.start()
-        request_create = {
-            'method': 'POST',
-            'path': '/mushrooms',
-            'body': {'data': {'name': 'Amanite'}}
-        }
-        request_delete = {
-            'method': 'DELETE',
-            'path': '/mushrooms'
-        }
-        body = {'requests': [request_create, request_create, request_delete]}
-        self.app.post_json('/batch', body, headers=self.headers, status=503)
-
-    def run_failing_post(self):
-        patch = mock.patch.object(
-            self.permission,
-            'add_principal_to_ace',
-            side_effect=BackendError('boom'))
-        self.addCleanup(patch.stop)
-        patch.start()
-        self.app.post_json('/psilos',
-                           {'data': {'name': 'Amanite'}},
-                           headers=self.headers,
-                           status=503)
+from .support import PostgreSQLTest, USER_PRINCIPAL
 
 
 @skip_if_no_postgresql