Repository: learningequality/kolibri

Issue:
Don't load all users for a facility at once in facility admin
## Observed behavior
As the number of users in a facility increases the time taken to load them all into the facility user administration page increases. When it gets to a very large number, this can take some considerable time.

## Expected behavior
Kolibri should paginate the users returned to the frontend.

In addition, this would require that the user search functionality be reimplemented, in order to get results from the backend, rather than doing filtering in the client side.

## User-facing consequences
Can lead to delays to delays loading the user page with large numbers of users (reports at ~30,000 users).
…

## Steps to reproduce
Make a large number (~30,000) users in a facility and load the Facility Admin user management page.



Discussion:

Resolving patch:
diff --git a/kolibri/core/auth/api.py b/kolibri/core/auth/api.py
--- a/kolibri/core/auth/api.py
+++ b/kolibri/core/auth/api.py
@@ -29,6 +29,7 @@
 from django.utils.timezone import now
 from django.views.decorators.csrf import ensure_csrf_cookie
 from django_filters.rest_framework import CharFilter
+from django_filters.rest_framework import ChoiceFilter
 from django_filters.rest_framework import DjangoFilterBackend
 from django_filters.rest_framework import FilterSet
 from django_filters.rest_framework import ModelChoiceFilter
@@ -75,9 +76,21 @@
 from kolibri.core.mixins import BulkDeleteMixin
 from kolibri.core.query import annotate_array_aggregate
 from kolibri.core.query import SQCount
+from kolibri.core.utils.pagination import ValuesViewsetPageNumberPagination
 from kolibri.plugins.app.utils import interface
 
 
+class OptionalPageNumberPagination(ValuesViewsetPageNumberPagination):
+    """
+    Pagination class that allows for page number-style pagination, when requested.
+    To activate, the `page_size` argument must be set. For example, to request the first 20 records:
+    `?page_size=20&page=1`
+    """
+
+    page_size = None
+    page_size_query_param = "page_size"
+
+
 class KolibriAuthPermissionsFilter(filters.BaseFilterBackend):
     """
     A Django REST Framework filter backend that limits results to those where the
@@ -186,16 +199,49 @@ def resetsettings(self, request, pk):
 
 class FacilityUserFilter(FilterSet):
 
+    USER_TYPE_CHOICES = (
+        ("learner", "learner"),
+        ("superuser", "superuser"),
+    ) + role_kinds.choices
+
     member_of = ModelChoiceFilter(
         method="filter_member_of", queryset=Collection.objects.all()
     )
+    user_type = ChoiceFilter(
+        choices=USER_TYPE_CHOICES,
+        method="filter_user_type",
+    )
+    exclude_member_of = ModelChoiceFilter(
+        method="filter_exclude_member_of", queryset=Collection.objects.all()
+    )
+    exclude_user_type = ChoiceFilter(
+        choices=USER_TYPE_CHOICES,
+        method="filter_exclude_user_type",
+    )
 
     def filter_member_of(self, queryset, name, value):
         return queryset.filter(Q(memberships__collection=value) | Q(facility=value))
 
+    def filter_user_type(self, queryset, name, value):
+        if value == "learner":
+            return queryset.filter(roles__isnull=True)
+        if value == "superuser":
+            return queryset.filter(devicepermissions__is_superuser=True)
+        return queryset.filter(roles__kind=value)
+
+    def filter_exclude_member_of(self, queryset, name, value):
+        return queryset.exclude(Q(memberships__collection=value) | Q(facility=value))
+
+    def filter_exclude_user_type(self, queryset, name, value):
+        if value == "learner":
+            return queryset.exclude(roles__isnull=True)
+        if value == "superuser":
+            return queryset.exclude(devicepermissions__is_superuser=True)
+        return queryset.exclude(roles__kind=value)
+
     class Meta:
         model = FacilityUser
-        fields = ["member_of"]
+        fields = ["member_of", "user_type", "exclude_member_of", "exclude_user_type"]
 
 
 class PublicFacilityUserViewSet(ReadOnlyValuesViewset):
@@ -246,10 +292,16 @@ def consolidate(self, items, queryset):
 
 class FacilityUserViewSet(ValuesViewset):
     permission_classes = (KolibriAuthPermissions,)
-    filter_backends = (KolibriAuthPermissionsFilter, DjangoFilterBackend)
+    pagination_class = OptionalPageNumberPagination
+    filter_backends = (
+        KolibriAuthPermissionsFilter,
+        DjangoFilterBackend,
+        filters.SearchFilter,
+    )
     queryset = FacilityUser.objects.all()
     serializer_class = FacilityUserSerializer
     filter_class = FacilityUserFilter
+    search_fields = ("username", "full_name")
 
     values = (
         "id",
diff --git a/kolibri/plugins/coach/class_summary_api.py b/kolibri/plugins/coach/class_summary_api.py
--- a/kolibri/plugins/coach/class_summary_api.py
+++ b/kolibri/plugins/coach/class_summary_api.py
@@ -18,6 +18,7 @@
 from kolibri.core.auth.models import AdHocGroup
 from kolibri.core.auth.models import Collection
 from kolibri.core.auth.models import FacilityUser
+from kolibri.core.content.api import OptionalPageNumberPagination  # added
 from kolibri.core.content.models import ContentNode
 from kolibri.core.exams.models import Exam
 from kolibri.core.lessons.models import Lesson
@@ -280,6 +281,7 @@ def has_permission(self, request, view):
 
 class ClassSummaryViewSet(viewsets.ViewSet):
     permission_classes = (permissions.IsAuthenticated, ClassSummaryPermissions)
+    pagination_class = OptionalPageNumberPagination
 
     def retrieve(self, request, pk):
         classroom = get_object_or_404(auth_models.Classroom, id=pk)


Test patch:
diff --git a/kolibri/plugins/facility/assets/src/views/UserPage/__test__/UserPage.spec.js b/kolibri/plugins/facility/assets/src/views/UserPage/__test__/UserPage.spec.js
--- a/kolibri/plugins/facility/assets/src/views/UserPage/__test__/UserPage.spec.js
+++ b/kolibri/plugins/facility/assets/src/views/UserPage/__test__/UserPage.spec.js
@@ -1,7 +1,12 @@
+import mock from 'xhr-mock';
 import { mount, RouterLinkStub } from '@vue/test-utils';
 import makeStore from '../../../../test/makeStore';
 import UserPage from '../index';
 
+jest.mock('kolibri.lib.logging');
+jest.mock('kolibri.urls');
+jest.mock('lockr');
+
 UserPage.computed.newUserLink = () => ({});
 
 function makeWrapper() {
@@ -20,6 +25,12 @@ const unicornUser = { id: '1', kind: 'UNICORN', username: 'unicorn', full_name:
 const coachUser = { id: '1', kind: 'coach', username: 'coach', full_name: 'coach' };
 
 describe('UserPage component', () => {
+  // replace the real XHR object with the mock XHR object before each test
+  beforeEach(() => mock.setup());
+
+  // put the real XHR object back and clear the mocks after each test
+  afterEach(() => mock.teardown());
+
   describe('message in empty states', () => {
     function getUserTableEmptyMessage(wrapper) {
       return wrapper.findComponent({ name: 'UserTable' }).props().emptyMessage;
@@ -46,12 +57,22 @@ describe('UserPage component', () => {
     });
 
     it('if a keyword filter is applied, the empty message is "no users match..."', async () => {
-      const { wrapper, store } = makeWrapper();
-      store.state.userManagement.facilityUsers = [{ ...coachUser }];
-      wrapper.setData({ roleFilter: { value: 'coach' } });
-      wrapper.findComponent({ name: 'PaginatedListContainer' }).setData({ filterInput: 'coachy' });
-      await wrapper.vm.$nextTick();
-      expect(getUserTableEmptyMessage(wrapper)).toEqual("No users match the filter: 'coachy'");
+      mock.get(/.*/, {
+        status: 200,
+        body: JSON.stringify({ results: [], page: 1, total_pages: 1, count: 0 }),
+        headers: { 'Content-Type': 'application/json' },
+      });
+
+      setTimeout(async () => {
+        const { wrapper, store } = makeWrapper();
+        store.state.userManagement.facilityUsers = [{ ...coachUser }];
+        wrapper.setData({ roleFilter: { value: 'coach' } });
+        wrapper
+          .findComponent({ name: 'PaginatedListContainer' })
+          .setData({ filterInput: 'coachy' });
+        await wrapper.vm.$nextTick();
+        expect(getUserTableEmptyMessage(wrapper)).toEqual("No users match the filter: 'coachy'");
+      }, 1000);
     });
   });
 });