Repository: akvo/akvo-rsr

Issue:
Add offset parameter to API
The 'old' API used to support offsets. However, since we now route the requests through DRF instead of TastyPie, this isn't supported out of the box anymore.

For legacy reasons, we should continue supporting this.



Discussion:

Resolving patch:
diff --git a/akvo/rest/pagination.py b/akvo/rest/pagination.py
new file mode 100644
--- /dev/null
+++ b/akvo/rest/pagination.py
@@ -0,0 +1,109 @@
+# -*- coding: utf-8 -*-
+
+# Akvo RSR is covered by the GNU Affero General Public License.
+# See more details in the license.txt file located at the root folder of the Akvo RSR module.
+# For additional details on the GNU license please see < http://www.gnu.org/licenses/agpl.html >.
+
+from django.utils.six.moves.urllib import parse as urlparse
+
+from rest_framework import pagination, serializers
+from rest_framework.templatetags.rest_framework import replace_query_param
+
+
+def remove_query_param(url, key):
+    """
+    Given a URL and a key/val pair, remove an item in the query
+    parameters of the URL, and return the new URL.
+    """
+    (scheme, netloc, path, query, fragment) = urlparse.urlsplit(url)
+    query_dict = urlparse.parse_qs(query, keep_blank_values=True)
+    query_dict.pop(key, None)
+    query = urlparse.urlencode(sorted(list(query_dict.items())), doseq=True)
+    return urlparse.urlunsplit((scheme, netloc, path, query, fragment))
+
+
+class NextPageField(serializers.Field):
+    """
+    Field that returns a link to the next page in paginated results.
+    """
+    page_field = 'page'
+    offset_field = 'offset'
+
+    def to_native(self, value):
+        request = self.context.get('request')
+
+        if request and '/api/v1/' not in request.path:
+            if not value.has_next():
+                return None
+            page = value.next_page_number()
+            url = request and request.build_absolute_uri() or ''
+            new_url = remove_query_param(url, self.offset_field)
+            return replace_query_param(new_url, self.page_field, page)
+
+        elif request:
+            if not value.has_next():
+                return None
+            try:
+                offset = int(request.GET.get('offset') or 0)
+            except ValueError:
+                offset = 0
+            new_offset = offset + value.object_list.count()
+            url = request and request.build_absolute_uri() or ''
+            if new_offset > 0:
+                return replace_query_param(url, self.offset_field, new_offset)
+            else:
+                return remove_query_param(url, self.offset_field)
+
+
+class PreviousPageField(serializers.Field):
+    """
+    Field that returns a link to the next page in paginated results.
+    """
+    page_field = 'page'
+    offset_field = 'offset'
+
+    def to_native(self, value):
+        request = self.context.get('request')
+
+        if request and '/api/v1/' not in request.path:
+            if not value.has_previous():
+                return None
+            page = value.previous_page_number()
+            url = request and request.build_absolute_uri() or ''
+            new_url = remove_query_param(url, self.offset_field)
+            return replace_query_param(new_url, self.page_field, page)
+
+        elif request:
+            if not value.has_previous():
+                return None
+            try:
+                offset = int(request.GET.get('offset') or 0)
+            except ValueError:
+                offset = 0
+            new_offset = offset - value.object_list.count()
+            url = request and request.build_absolute_uri() or ''
+            if new_offset > 0:
+                return replace_query_param(url, self.offset_field, new_offset)
+            else:
+                return remove_query_param(url, self.offset_field)
+
+
+class OffsetPageField(serializers.Field):
+    """
+    Field that returns the offset.
+    """
+    def to_native(self, value):
+        request = self.context.get('request')
+        try:
+            return int(request.GET.get('offset') or 0)
+        except ValueError:
+            return 0
+
+
+class CustomPaginationSerializer(pagination.PaginationSerializer):
+    offset = OffsetPageField(source='*')
+    next = NextPageField(source='*')
+    previous = PreviousPageField(source='*')
+
+    def __init__(self, *args, **kwargs):
+        super(CustomPaginationSerializer, self).__init__(*args, **kwargs)
diff --git a/akvo/rest/renderers.py b/akvo/rest/renderers.py
--- a/akvo/rest/renderers.py
+++ b/akvo/rest/renderers.py
@@ -7,6 +7,8 @@
 from django.utils import six
 from django.utils.xmlutils import SimplerXMLGenerator
 
+from six import string_types
+
 from rest_framework.compat import StringIO, smart_text
 from rest_framework.renderers import BaseRenderer, BrowsableAPIRenderer, JSONRenderer, XMLRenderer
 
@@ -43,10 +45,15 @@ def _rename_fields(results):
                 if isinstance(result[object_id], list):
                     result_list, new_list = result.pop(object_id), []
                     for list_item in result_list:
-                        new_list.append(id_to_api[object_id].format(str(list_item)))
+                        if isinstance(list_item, string_types) and '/api/v1/' in list_item:
+                            new_list.append(list_item)
+                        else:
+                            new_list.append(id_to_api[object_id].format(str(list_item)))
                     result[object_id] = new_list
                 else:
-                    result[object_id] = id_to_api[object_id].format(str(result.pop(object_id)))
+                    if not (isinstance(result[object_id], string_types) and
+                            '/api/v1' in result[object_id]):
+                        result[object_id] = id_to_api[object_id].format(str(result.pop(object_id)))
     return results
 
 
@@ -68,6 +75,7 @@ def _convert_data_for_tastypie(request, view, data):
     response_data['meta'] = {
         'limit': view.get_paginate_by(),
         'total_count': data.get('count'),
+        'offset': data.get('offset'),
         'next': _remove_domain(request, data.get('next')),
         'previous': _remove_domain(request, data.get('previous')),
     }
@@ -95,12 +103,14 @@ def render(self, data, accepted_media_type=None, renderer_context=None):
         request = renderer_context.get('request')
 
         if '/api/v1/' in request.path:
-            if all(k in data.keys() for k in ['count', 'next', 'previous', 'results']):
+            if all(k in data.keys() for k in ['count', 'next', 'previous', 'offset', 'results']):
                 # Paginated result
                 data = _convert_data_for_tastypie(request, renderer_context['view'], data)
             else:
                 # Non-paginated result
                 data = _rename_fields([data])[0]
+        elif 'offset' in data.keys():
+            data.pop('offset')
 
         return super(CustomHTMLRenderer, self).render(data, accepted_media_type, renderer_context)
 
@@ -125,12 +135,14 @@ def render(self, data, accepted_media_type=None, renderer_context=None):
         request = renderer_context.get('request')
 
         if '/api/v1/' in request.path:
-            if all(k in data.keys() for k in ['count', 'next', 'previous', 'results']):
+            if all(k in data.keys() for k in ['count', 'next', 'previous', 'offset', 'results']):
                 # Paginated result
                 data = _convert_data_for_tastypie(request, renderer_context['view'], data)
             else:
                 # Non-paginated result
                 data = _rename_fields([data])[0]
+        elif 'offset' in data.keys():
+            data.pop('offset')
 
         return super(CustomJSONRenderer, self).render(data, accepted_media_type, renderer_context)
 
@@ -190,7 +202,7 @@ def render(self, data, accepted_media_type=None, renderer_context=None):
             xml = SimplerXMLGenerator(stream, self.charset)
             xml.startDocument()
 
-            if all(k in data.keys() for k in ['count', 'next', 'previous', 'results']):
+            if all(k in data.keys() for k in ['count', 'next', 'previous', 'offset', 'results']):
                 # Paginated result
                 xml.startElement("response", {})
                 data = _convert_data_for_tastypie(request, renderer_context['view'], data)
@@ -204,5 +216,7 @@ def render(self, data, accepted_media_type=None, renderer_context=None):
 
             xml.endDocument()
             return stream.getvalue()
+        elif 'offset' in data.keys():
+            data.pop('offset')
 
         return XMLRenderer().render(data, accepted_media_type, renderer_context)
diff --git a/akvo/rest/viewsets.py b/akvo/rest/viewsets.py
--- a/akvo/rest/viewsets.py
+++ b/akvo/rest/viewsets.py
@@ -7,6 +7,8 @@
 from django.db.models.fields import FieldDoesNotExist
 from django.db.models.fields.related import ForeignObject
 from django.core.exceptions import FieldError
+from django.core.paginator import InvalidPage
+from django.http import Http404
 
 from akvo.rest.models import TastyTokenAuthentication
 
@@ -14,6 +16,8 @@
 
 from .filters import RSRGenericFilterBackend
 
+import warnings
+
 
 class SafeMethodsPermissions(permissions.DjangoObjectPermissions):
     """
@@ -36,6 +40,70 @@ class BaseRSRViewSet(viewsets.ModelViewSet):
     filter_backends = (filters.OrderingFilter, RSRGenericFilterBackend,)
     ordering_fields = '__all__'
 
+    def paginate_queryset(self, queryset, page_size=None):
+        """
+        Paginate a queryset if required, either returning a page object,
+        or `None` if pagination is not configured for this view.
+        """
+        if '/rest/v1/' in self.request.path:
+            return super(BaseRSRViewSet, self).paginate_queryset(queryset, page_size)
+
+        deprecated_style = False
+        if page_size is not None:
+            warnings.warn('The `page_size` parameter to `paginate_queryset()` '
+                          'is deprecated. '
+                          'Note that the return style of this method is also '
+                          'changed, and will simply return a page object '
+                          'when called without a `page_size` argument.',
+                          DeprecationWarning, stacklevel=2)
+            deprecated_style = True
+        else:
+            # Determine the required page size.
+            # If pagination is not configured, simply return None.
+            page_size = self.get_paginate_by()
+            if not page_size:
+                return None
+
+        if not self.allow_empty:
+            warnings.warn(
+                'The `allow_empty` parameter is deprecated. '
+                'To use `allow_empty=False` style behavior, You should override '
+                '`get_queryset()` and explicitly raise a 404 on empty querysets.',
+                DeprecationWarning, stacklevel=2
+            )
+
+        paginator = self.paginator_class(queryset, page_size,
+                                         allow_empty_first_page=self.allow_empty)
+        offset_kwarg = self.kwargs.get('offset')
+        offset_query_param = self.request.QUERY_PARAMS.get('offset')
+
+        try:
+            offset = int(offset_kwarg or offset_query_param or 0)
+        except ValueError:
+            raise Http404(_("Offset cannot be converted to an int."))
+
+        page = int(offset / page_size) + 1
+
+        try:
+            page_number = paginator.validate_number(page)
+        except InvalidPage:
+            if page == 'last':
+                page_number = paginator.num_pages
+            else:
+                raise Http404(_("Page is not 'last', nor can it be converted to an int."))
+        try:
+            page = paginator.page(page_number)
+        except InvalidPage as exc:
+            error_format = _('Invalid page (%(page_number)s): %(message)s')
+            raise Http404(error_format % {
+                'page_number': page_number,
+                'message': str(exc)
+            })
+
+        if deprecated_style:
+            return (paginator, page, page.object_list, page.has_other_pages())
+        return page
+
     def get_queryset(self):
 
         def django_filter_filters(request):
@@ -45,8 +113,8 @@ def django_filter_filters(request):
             # query string keys reserved by the RSRGenericFilterBackend
             qs_params = ['filter', 'exclude', 'select_related', 'prefetch_related', ]
             # query string keys used by core DRF, OrderingFilter and Akvo custom views
-            exclude_params = ['limit', 'format', 'page', 'ordering', 'partner_type', 'sync_owner',
-                              'reporting_org', ]
+            exclude_params = ['limit', 'format', 'page', 'offset', 'ordering', 'partner_type',
+                              'sync_owner', 'reporting_org', ]
             filters = {}
             for key in request.QUERY_PARAMS.keys():
                 if key not in qs_params + exclude_params and not key.startswith('image_thumb_'):


Test patch: