Repository: NilCoalescing/djangochannelsrestframework (language: python)

Issue:
[Feature] Support Pagination
This should wrap the list response so that the items are embedded within a `pagination serializer`. 

`offset` and `limit` are logical args that can be passed as part of the request since not all DB backends support cursors for know that is out of scope.


Discussion / hints:

Fix patch:
diff --git a/djangochannelsrestframework/generics.py b/djangochannelsrestframework/generics.py
index 82a1300..939c142 100644
--- a/djangochannelsrestframework/generics.py
+++ b/djangochannelsrestframework/generics.py
@@ -1,10 +1,12 @@
-from typing import Any, Dict, Type, Optional
+from typing import Any, Dict, Type, Optional, List, OrderedDict, Union
 
 from django.db.models import QuerySet, Model
 from rest_framework.generics import get_object_or_404
 from rest_framework.serializers import Serializer
+from rest_framework.utils.serializer_helpers import ReturnDict, ReturnList
 
 from djangochannelsrestframework.consumers import AsyncAPIConsumer
+from djangochannelsrestframework.settings import api_settings
 
 
 class GenericAsyncAPIConsumer(AsyncAPIConsumer):
diff --git a/djangochannelsrestframework/mixins.py b/djangochannelsrestframework/mixins.py
index 1b42b83..0cf7507 100644
--- a/djangochannelsrestframework/mixins.py
+++ b/djangochannelsrestframework/mixins.py
@@ -1,8 +1,11 @@
-from typing import Tuple
+from typing import Any, Tuple, Dict, Optional, OrderedDict, Union
+from djangochannelsrestframework.observer.model_observer import Action
+from rest_framework.utils.serializer_helpers import ReturnDict, ReturnList
 from rest_framework import status
 from rest_framework.utils.serializer_helpers import ReturnDict, ReturnList
 
 from .decorators import action
+from djangochannelsrestframework.settings import api_settings
 
 
 class CreateModelMixin:
@@ -413,3 +416,71 @@ class LiveConsumer(DeleteModelMixin, GenericAsyncAPIConsumer):
 
     def perform_delete(self, instance, **kwargs):
         instance.delete()
+
+
+class PaginatedModelListMixin(ListModelMixin):
+
+    pagination_class = api_settings.DEFAULT_PAGINATION_CLASS
+
+    @action()
+    def list(self, **kwargs):
+        queryset = self.filter_queryset(self.get_queryset(**kwargs), **kwargs)
+        page = self.paginate_queryset(queryset, **kwargs)
+        if page is not None:
+            serializer = self.get_serializer(
+                instance=page, many=True, action_kwargs=kwargs
+            )
+            return self.get_paginated_response(serializer.data), status.HTTP_200_OK
+
+        serializer = self.get_serializer(
+            instance=queryset, many=True, action_kwargs=kwargs
+        )
+        return serializer.data, status.HTTP_200_OK
+
+    @property
+    def paginator(self) -> Optional[any]:
+        """Gets the paginator class
+
+        Returns:
+            Pagination class. Optional.
+        """
+        if not hasattr(self, "_paginator"):
+            if self.pagination_class is None:
+                self._paginator = None
+            else:
+                self._paginator = self.pagination_class()
+        return self._paginator
+
+    def paginate_queryset(
+        self, queryset, **kwargs: Dict
+    ) -> Optional[Any]:
+        if self.paginator is None:
+            return None
+        return self.paginator.paginate_queryset(
+            queryset, self.scope, view=self, **kwargs
+        )
+
+    def get_paginated_response(
+        self, data: Union[ReturnDict, ReturnList]
+    ) -> OrderedDict:
+        assert self.paginator is not None
+        return self.paginator.get_paginated_response(data)
+
+
+class StreamedPaginatedListMixin(PaginatedModelListMixin):
+    @action()
+    async def list(self, action, request_id, **kwargs):
+        data, status = await super().list(
+            action=action, request_id=request_id, **kwargs
+        )
+
+        await self.reply(action=action, data=data, status=status, request_id=request_id)
+
+        count = data.get("count", 0)
+        limit = data.get("limit", 0)
+        offset = data.get("offset", 0)
+
+        if offset < count:
+            kwargs["offset"] = limit + offset
+
+            await self.list(action=action, request_id=request_id, **kwargs)
diff --git a/djangochannelsrestframework/observer/model_observer.py b/djangochannelsrestframework/observer/model_observer.py
index 6b2dcd4..f5da6e0 100644
--- a/djangochannelsrestframework/observer/model_observer.py
+++ b/djangochannelsrestframework/observer/model_observer.py
@@ -37,7 +37,7 @@ class Action(Enum):
 
 
 class UnsupportedWarning(Warning):
-    """"""
+    """ """
 
 
 class ModelObserverInstanceState:
diff --git a/djangochannelsrestframework/pagination.py b/djangochannelsrestframework/pagination.py
new file mode 100644
index 0000000..e5c6317
--- /dev/null
+++ b/djangochannelsrestframework/pagination.py
@@ -0,0 +1,118 @@
+from collections import OrderedDict
+from typing import Any, Dict, List, Optional, Union
+
+from rest_framework.utils.serializer_helpers import ReturnDict, ReturnList
+from djangochannelsrestframework.settings import api_settings
+from rest_framework.response import Response
+from rest_framework.pagination import LimitOffsetPagination
+
+
+def _positive_int(integer_string, strict=False, cutoff=None):
+    """
+    Cast a string to a strictly positive integer.
+    """
+    ret = int(integer_string)
+    if ret < 0 or (ret == 0 and strict):
+        raise ValueError()
+    if cutoff:
+        return min(ret, cutoff)
+    return ret
+
+
+class WebsocketLimitOffsetPagination(LimitOffsetPagination):
+
+    default_limit = api_settings.PAGE_SIZE
+    count: int
+    limit: int
+    offset: int
+
+    def get_paginated_response(
+        self, data: Union[ReturnDict, ReturnList]
+    ) -> OrderedDict:
+        """Get the paginated response data
+
+        Args:
+            data: serializer data paginated.
+
+        Return:
+            Dictionary with the results and the count.
+        """
+        # TODO
+        return OrderedDict(
+            [
+                ("count", self.count),
+                ("results", data),
+                ("limit", self.limit),
+                ("offset", self.offset),
+            ]
+        )
+
+    def paginate_queryset(
+        self,
+        queryset,
+        scope: Dict[any, any],
+        view=None,
+        **kwargs: Dict[any, any]
+    ) -> Optional[List[Optional[Any]]]:
+        """Paginates a given queryset, based on the kwargs `limit` and `offset`.
+
+        Args:
+            queryset: database data.
+            scope: context.
+            view: ?
+            kwargs: keyworded argument dictionary.
+
+        Returns:
+            List of instances of the model.
+        """
+        self.count = self.get_count(queryset)
+        self.limit = self.get_limit(**kwargs)
+        if self.limit is None:
+            return None
+
+        self.offset = self.get_offset(**kwargs)
+        self.scope = scope
+        if self.count > self.limit and self.template is not None:
+            self.display_page_controls = True
+
+        if self.count == 0 or self.offset > self.count:
+            return []
+        return list(queryset[self.offset : self.offset + self.limit])
+
+    def get_limit(self, **kwargs: Dict) -> int:
+        """Gets the limit from the websocket message.
+
+        Args:
+            kwargs: keyworded argument dictionary.
+
+        Returns:
+            Limit results pagination.
+        """
+
+        limit_query_param = kwargs.get("limit", self.default_limit)
+        if self.limit_query_param:
+            try:
+                return _positive_int(
+                    limit_query_param, strict=True, cutoff=self.max_limit
+                )
+            except (KeyError, ValueError):
+                pass
+
+        return self.default_limit
+
+    def get_offset(self, **kwargs: Dict) -> int:
+        """Gets the offset from the websocket message
+
+        Args:
+            kwargs: keyworded argument dictionary.
+
+        Returns:
+            Offset value.
+        """
+        offset_query_param = kwargs.get("offset", 0)
+        try:
+            return _positive_int(
+                offset_query_param,
+            )
+        except (KeyError, ValueError):
+            return 0
diff --git a/djangochannelsrestframework/settings.py b/djangochannelsrestframework/settings.py
index 334d933..00e0bbb 100644
--- a/djangochannelsrestframework/settings.py
+++ b/djangochannelsrestframework/settings.py
@@ -5,6 +5,8 @@
 DEFAULTS = {
     "DEFAULT_PAGE_SIZE": 25,
     "DEFAULT_PERMISSION_CLASSES": ("djangochannelsrestframework.permissions.AllowAny",),
+    "DEFAULT_PAGINATION_CLASS": None,
+    "PAGE_SIZE": None,
 }
 IMPORT_STRINGS = ("DEFAULT_PERMISSION_CLASSES",)
 


Test patch:
diff --git a/tests/test_generic_consumer.py b/tests/test_generic_consumer.py
index 6f5aa3d..d486f48 100644
--- a/tests/test_generic_consumer.py
+++ b/tests/test_generic_consumer.py
@@ -3,6 +3,7 @@
 from channels.testing import WebsocketCommunicator
 from django.contrib.auth import get_user_model
 from rest_framework import serializers
+from djangochannelsrestframework.pagination import WebsocketLimitOffsetPagination
 
 from djangochannelsrestframework.decorators import action
 from djangochannelsrestframework.generics import GenericAsyncAPIConsumer
@@ -13,6 +14,8 @@
     UpdateModelMixin,
     PatchModelMixin,
     DeleteModelMixin,
+    PaginatedModelListMixin,
+    StreamedPaginatedListMixin,
 )
 
 
@@ -194,6 +197,213 @@ class AConsumer(ListModelMixin, GenericAsyncAPIConsumer):
     }
 
 
+@pytest.mark.django_db(transaction=True)
+@pytest.mark.asyncio
+async def test_list_mixin_consumer_with_pagination():
+    class UserSerializer(serializers.ModelSerializer):
+        class Meta:
+            model = get_user_model()
+            fields = (
+                "id",
+                "username",
+                "email",
+            )
+
+    class TempClass(WebsocketLimitOffsetPagination):
+        """Didn't found a way to override the PAGE_SIZE settings correctly"""
+
+        default_limit = 1
+
+    class AConsumer(PaginatedModelListMixin, GenericAsyncAPIConsumer):
+        queryset = get_user_model().objects.all()
+        serializer_class = UserSerializer
+        pagination_class = TempClass
+
+    assert not await database_sync_to_async(get_user_model().objects.all().exists)()
+
+    # Test a normal connection
+    communicator = WebsocketCommunicator(AConsumer(), "/testws/")
+    connected, _ = await communicator.connect()
+    assert connected
+
+    await communicator.send_json_to({"action": "list", "request_id": 1})
+
+    response = await communicator.receive_json_from()
+
+    assert response == {
+        "action": "list",
+        "errors": [],
+        "response_status": 200,
+        "request_id": 1,
+        "data": {
+            "results": [],
+            "count": 0,
+            "limit": 1,
+            "offset": 0,
+        },
+    }
+
+    u1 = await database_sync_to_async(get_user_model().objects.create)(
+        username="test1", email="42@example.com"
+    )
+    u2 = await database_sync_to_async(get_user_model().objects.create)(
+        username="test2", email="45@example.com"
+    )
+
+    await communicator.send_json_to(
+        {
+            "action": "list",
+            "request_id": 1,
+        }
+    )
+
+    response = await communicator.receive_json_from()
+
+    assert response == {
+        "action": "list",
+        "errors": [],
+        "response_status": 200,
+        "request_id": 1,
+        "data": {
+            "count": 2,
+            "limit": 1,
+            "offset": 0,
+            "results": [
+                {"email": "42@example.com", "id": u1.id, "username": "test1"},
+            ],
+        },
+    }
+    await communicator.send_json_to({"action": "list", "request_id": 1, "offset": 1})
+
+    response = await communicator.receive_json_from()
+
+    assert response == {
+        "action": "list",
+        "errors": [],
+        "response_status": 200,
+        "request_id": 1,
+        "data": {
+            "count": 2,
+            "limit": 1,
+            "offset": 1,
+            "results": [
+                {"email": "45@example.com", "id": u2.id, "username": "test2"},
+            ],
+        },
+    }
+    await communicator.send_json_to({"action": "list", "request_id": 1, "offset": 2})
+
+    response = await communicator.receive_json_from()
+
+    assert response == {
+        "action": "list",
+        "errors": [],
+        "response_status": 200,
+        "request_id": 1,
+        "data": {"count": 2, "limit": 1, "offset": 2, "results": []},
+    }
+
+
+@pytest.mark.django_db(transaction=True)
+@pytest.mark.asyncio
+async def test_stream_paginated_list_mixin():
+    class UserSerializer(serializers.ModelSerializer):
+        class Meta:
+            model = get_user_model()
+            fields = (
+                "id",
+                "username",
+                "email",
+            )
+
+    class TempClass(WebsocketLimitOffsetPagination):
+        """Didn't found a way to override the PAGE_SIZE settings correctly"""
+
+        default_limit = 1
+
+    class AConsumer(
+        StreamedPaginatedListMixin, ListModelMixin, GenericAsyncAPIConsumer
+    ):
+        queryset = get_user_model().objects.all()
+        serializer_class = UserSerializer
+        pagination_class = TempClass
+
+    assert not await database_sync_to_async(get_user_model().objects.all().exists)()
+
+    # Test a normal connection
+    communicator = WebsocketCommunicator(AConsumer(), "/testws/")
+    connected, _ = await communicator.connect()
+    assert connected
+
+    u1 = await database_sync_to_async(get_user_model().objects.create)(
+        username="test1", email="42@example.com"
+    )
+    u2 = await database_sync_to_async(get_user_model().objects.create)(
+        username="test2", email="45@example.com"
+    )
+    u3 = await database_sync_to_async(get_user_model().objects.create)(
+        username="test3", email="46@example.com"
+    )
+
+    await communicator.send_json_to(
+        {
+            "action": "list",
+            "request_id": 1,
+        }
+    )
+
+    response = await communicator.receive_json_from()
+
+    assert response == {
+        "action": "list",
+        "errors": [],
+        "response_status": 200,
+        "request_id": 1,
+        "data": {
+            "count": 3,
+            "limit": 1,
+            "offset": 0,
+            "results": [
+                {"email": "42@example.com", "id": u1.id, "username": "test1"},
+            ],
+        },
+    }
+
+    response = await communicator.receive_json_from()
+
+    assert response == {
+        "action": "list",
+        "errors": [],
+        "response_status": 200,
+        "request_id": 1,
+        "data": {
+            "count": 3,
+            "limit": 1,
+            "offset": 1,
+            "results": [
+                {"email": "45@example.com", "id": u2.id, "username": "test2"},
+            ],
+        },
+    }
+
+    response = await communicator.receive_json_from()
+
+    assert response == {
+        "action": "list",
+        "errors": [],
+        "response_status": 200,
+        "request_id": 1,
+        "data": {
+            "count": 3,
+            "limit": 1,
+            "offset": 2,
+            "results": [
+                {"email": "46@example.com", "id": u3.id, "username": "test3"},
+            ],
+        },
+    }
+
+
 @pytest.mark.django_db(transaction=True)
 @pytest.mark.asyncio
 async def test_retrieve_mixin_consumer():
diff --git a/tests/test_model_observer.py b/tests/test_model_observer.py
index 99a4419..57ab561 100644
--- a/tests/test_model_observer.py
+++ b/tests/test_model_observer.py
@@ -158,6 +158,8 @@ async def update_username(self, pk=None, username=None, **kwargs):
         "data": {"email": "42@example.com", "id": u1.id, "username": "thenewname"},
     }
 
+    u1_pk = u1.pk
+
     await database_sync_to_async(u1.delete)()
 
     response = await communicator.receive_json_from()
@@ -167,7 +169,7 @@ async def update_username(self, pk=None, username=None, **kwargs):
         "errors": [],
         "response_status": 204,
         "request_id": 4,
-        "data": {"pk": 13},
+        "data": {"pk": u1_pk},
     }
 
     await communicator.disconnect()