Repository: ckan/ckan

Issue:
Implement paging in the API
## Problem

http://catalog.data.gov/api/3/action/package_list gives a time out (actually a 503 temp unavailable, but I think a time out is the real cause) because there are too many datasets in data.gov.
## Suggested solution

Implement paging in the API v3: the query to the database should be LIMIT-ed. When more rows are available, a HTTP header should be set linking towards the next page. The URL of the next page can be implemented using GET parameters: ?page=2 or ?limit=...&offset=... or ...



Discussion:

Resolving patch:
diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py
--- a/ckan/logic/action/get.py
+++ b/ckan/logic/action/get.py
@@ -64,6 +64,13 @@ def site_read(context,data_dict=None):
 def package_list(context, data_dict):
     '''Return a list of the names of the site's datasets (packages).
 
+    :param limit: if given, the list of datasets will be broken into pages of
+        at most ``limit`` datasets per page and only one page will be returned
+        at a time (optional)
+    :type limit: int
+    :param offset: when ``limit`` is given, the offset to start returning packages from
+    :type offset: int
+
     :rtype: list of strings
 
     '''
@@ -72,6 +79,12 @@ def package_list(context, data_dict):
 
     _check_access('package_list', context, data_dict)
 
+    schema = context.get('schema', logic.schema.default_pagination_schema())
+    data_dict, errors = _validate(data_dict, schema, context)
+    if errors:
+        raise ValidationError(errors)
+
+
     package_revision_table = model.package_revision_table
     col = (package_revision_table.c.id
         if api == 2 else package_revision_table.c.name)
@@ -79,6 +92,14 @@ def package_list(context, data_dict):
     query = query.where(_and_(package_revision_table.c.state=='active',
         package_revision_table.c.current==True))
     query = query.order_by(col)
+
+    limit = data_dict.get('limit')
+    if limit:
+        query = query.limit(limit)
+
+    offset = data_dict.get('offset')
+    if offset:
+        query = query.offset(offset)
     return list(zip(*query.execute())[0])
 
 def current_package_list_with_resources(context, data_dict):


Test patch:
diff --git a/ckan/tests/logic/test_action.py b/ckan/tests/logic/test_action.py
--- a/ckan/tests/logic/test_action.py
+++ b/ckan/tests/logic/test_action.py
@@ -60,8 +60,8 @@ def _add_basic_package(self, package_name=u'test_package', **kwargs):
         return json.loads(res.body)['result']
 
     def test_01_package_list(self):
-        postparams = '%s=1' % json.dumps({})
-        res = json.loads(self.app.post('/api/action/package_list', params=postparams).body)
+        res = json.loads(self.app.post('/api/action/package_list',
+                         headers={'content-type': 'application/json'}).body)
         assert res['success'] is True
         assert len(res['result']) == 2
         assert 'warandpeace' in res['result']
@@ -69,6 +69,13 @@ def test_01_package_list(self):
         assert res['help'].startswith(
             "Return a list of the names of the site's datasets (packages).")
 
+        postparams = '%s=1' % json.dumps({'limit': 1})
+        res = json.loads(self.app.post('/api/action/package_list',
+                         params=postparams).body)
+        assert res['success'] is True
+        assert len(res['result']) == 1
+        assert 'warandpeace' in res['result'] or 'annakarenina' in res['result']
+
 		# Test GET request
         res = json.loads(self.app.get('/api/action/package_list').body)
         assert len(res['result']) == 2