Repository: mozilla/pontoon

Issue:
Implement proper paging for entities
We currently use a weird way of paginating entities - in each request we include a list of already loaded entities, which we exclude from the matching entities in Django and then return the first page of the remaining entities.

This is a relic of the [`react-infinite-scroller`](https://github.com/mozilla/pontoon/commit/9f39c4f49597aeda95376b026e4b54c9a515d4df) library which we no longer use.

We should implement proper paging, i.e. drop `exclude_entities` and only return the relevant page.


Discussion:

Resolving patch:
diff --git a/pontoon/base/forms.py b/pontoon/base/forms.py
--- a/pontoon/base/forms.py
+++ b/pontoon/base/forms.py
@@ -294,6 +294,7 @@ class GetEntitiesForm(forms.Form):
     locale = forms.CharField()
     paths = forms.MultipleChoiceField(required=False)
     limit = forms.IntegerField(required=False, initial=50)
+    page = forms.IntegerField(required=False, initial=1)
     status = forms.CharField(required=False)
     extra = forms.CharField(required=False)
     tag = forms.CharField(required=False)
@@ -303,7 +304,6 @@ class GetEntitiesForm(forms.Form):
     reviewer = forms.CharField(required=False)
     exclude_self_reviewed = forms.BooleanField(required=False)
     search = forms.CharField(required=False)
-    exclude_entities = forms.CharField(required=False)
     entity_ids = forms.CharField(required=False)
     pk_only = forms.BooleanField(required=False)
     inplace_editor = forms.BooleanField(required=False)
@@ -322,14 +322,17 @@ def clean_limit(self):
         except (TypeError, ValueError):
             return 50
 
+    def clean_page(self):
+        try:
+            return int(self.cleaned_data["page"])
+        except (TypeError, ValueError):
+            return 1
+
     def clean_search(self):
         # Return the search input as is, without any cleaning. This is in order to allow
         # users to search for strings with leading or trailing whitespaces.
         return self.data.get("search")
 
-    def clean_exclude_entities(self):
-        return utils.split_ints(self.cleaned_data["exclude_entities"])
-
     def clean_entity_ids(self):
         return utils.split_ints(self.cleaned_data["entity_ids"])
 
diff --git a/pontoon/base/models.py b/pontoon/base/models.py
--- a/pontoon/base/models.py
+++ b/pontoon/base/models.py
@@ -2920,7 +2920,6 @@ def for_project_locale(
         status=None,
         tag=None,
         search=None,
-        exclude_entities=None,
         extra=None,
         time=None,
         author=None,
@@ -3060,9 +3059,6 @@ def for_project_locale(
                 pk__in=set(list(translation_matches) + list(entity_matches))
             )
 
-        if exclude_entities:
-            entities = entities.exclude(pk__in=exclude_entities)
-
         order_fields = ("resource__order", "order")
         if project.slug == "all-projects":
             order_fields = ("resource__project__name",) + order_fields
diff --git a/pontoon/base/views.py b/pontoon/base/views.py
--- a/pontoon/base/views.py
+++ b/pontoon/base/views.py
@@ -179,14 +179,15 @@ def _get_paginated_entities(locale, preferred_source_locale, project, form, enti
     This is used by the regular mode of the Translate page.
     """
     paginator = Paginator(entities, form.cleaned_data["limit"])
+    page = form.cleaned_data["page"]
 
     try:
-        entities_page = paginator.page(1)
+        entities_page = paginator.page(page)
     except EmptyPage:
         return JsonResponse({"has_next": False, "stats": {}})
 
-    has_next = entities_page.has_next()
     entities_to_map = entities_page.object_list
+    requested_entity = form.cleaned_data["entity"] if page == 1 else None
 
     return JsonResponse(
         {
@@ -194,9 +195,9 @@ def _get_paginated_entities(locale, preferred_source_locale, project, form, enti
                 locale,
                 preferred_source_locale,
                 entities_to_map,
-                requested_entity=form.cleaned_data["entity"],
+                requested_entity=requested_entity,
             ),
-            "has_next": has_next,
+            "has_next": entities_page.has_next(),
             "stats": TranslatedResource.objects.stats(
                 project, form.cleaned_data["paths"], locale
             ),
@@ -244,7 +245,6 @@ def entities(request):
         "paths",
         "status",
         "search",
-        "exclude_entities",
         "extra",
         "time",
         "author",


Test patch:
diff --git a/pontoon/base/tests/views/test_entity.py b/pontoon/base/tests/views/test_entity.py
--- a/pontoon/base/tests/views/test_entity.py
+++ b/pontoon/base/tests/views/test_entity.py
@@ -47,47 +47,44 @@ def test_view_entity_filters(member, resource_a, locale_a):
 
 
 @pytest.mark.django_db
-def test_view_entity_exclude_entities(
+def test_view_get_entities_paging(
     member,
     resource_a,
     locale_a,
 ):
     """
-    Excluded entities shouldn't be returned by get_entities.
+    Only entities from the requested page should be returned by get_entities().
     """
     TranslatedResource.objects.create(resource=resource_a, locale=locale_a)
     ProjectLocaleFactory.create(project=resource_a.project, locale=locale_a)
     entities = EntityFactory.create_batch(size=3, resource=resource_a)
-    excluded_pk = entities[1].pk
+
     response = member.client.post(
         "/get-entities/",
         {
             "project": resource_a.project.slug,
             "locale": locale_a.code,
             "paths[]": [resource_a.path],
-            "exclude_entities": [excluded_pk],
+            "page": 1,
             "limit": 1,
         },
         HTTP_X_REQUESTED_WITH="XMLHttpRequest",
     )
     assert response.status_code == 200
     assert json.loads(response.content)["has_next"] is True
-    assert [e["pk"] for e in json.loads(response.content)["entities"]] != [excluded_pk]
+    assert json.loads(response.content)["entities"][0]["pk"] == entities[0].pk
 
-    exclude_entities = ",".join(map(str, [entities[2].pk, entities[1].pk]))
     response = member.client.post(
         "/get-entities/",
         {
             "project": resource_a.project.slug,
             "locale": locale_a.code,
             "paths[]": [resource_a.path],
-            "exclude_entities": exclude_entities,
+            "page": len(entities),
             "limit": 1,
         },
         HTTP_X_REQUESTED_WITH="XMLHttpRequest",
     )
     assert response.status_code == 200
     assert json.loads(response.content)["has_next"] is False
-    assert [e["pk"] for e in json.loads(response.content)["entities"]] == [
-        entities[0].pk
-    ]
+    assert json.loads(response.content)["entities"][0]["pk"] == entities[-1].pk
diff --git a/translate/src/modules/entitieslist/components/EntitiesList.test.js b/translate/src/modules/entitieslist/components/EntitiesList.test.js
--- a/translate/src/modules/entitieslist/components/EntitiesList.test.js
+++ b/translate/src/modules/entitieslist/components/EntitiesList.test.js
@@ -88,7 +88,7 @@ describe('<EntitiesList>', () => {
     expect(wrapper.find('Entity')).toHaveLength(2);
   });
 
-  it('excludes current entities when requesting new entities', () => {
+  it('when requesting new entities, load page 2', () => {
     jest.useFakeTimers();
     mockAllIsIntersecting(false);
 
@@ -103,7 +103,7 @@ describe('<EntitiesList>', () => {
     mockAllIsIntersecting(true);
     jest.advanceTimersByTime(100); // default value for react-infinite-scroll-hook delayInMs
 
-    expect(EntitiesActions.getEntities.args[0][1]).toEqual(ENTITIES);
+    expect(EntitiesActions.getEntities.args[0][1]).toEqual(2);
   });
 
   it('redirects to the first entity when none is selected', () => {