Repository: pypi/warehouse

Issue:
TransportError: TransportError(500, 'search_phase_execution_exception', 'Result window is too lar...
https://app.getsentry.com/python-software-foundation/warehouse-production/issues/118467761/

```
TransportError: TransportError(500, 'search_phase_execution_exception', 'Result window is too large, from + size must be less than or equal to: [10000] but was [10640]. See the scroll api for a more efficient way to request large data sets. This limit can be set by changing the [index.max_result_window] index level parameter.')
(18 additional frame(s) were not displayed)
...
  File "warehouse/utils/http.py", line 31, in wrapped
    return fn(context, request)
  File "warehouse/cache/origin/__init__.py", line 81, in wrapped
    return view(context, request)
  File "warehouse/views.py", line 179, in search
    url_maker=paginate_url_factory(request),
  File "warehouse/utils/paginate.py", line 43, in ElasticsearchPage
    return Page(*args, **kwargs)
  File "warehouse/utils/paginate.py", line 26, in __getitem__
    self.results = self.query[range].execute()
```



Discussion:

Resolving patch:
diff --git a/warehouse/utils/paginate.py b/warehouse/utils/paginate.py
--- a/warehouse/utils/paginate.py
+++ b/warehouse/utils/paginate.py
@@ -15,12 +15,30 @@
 
 class _ElasticsearchWrapper:
 
+    max_results = 10000
+
     def __init__(self, query):
         self.query = query
         self.results = None
         self.best_guess = None
 
     def __getitem__(self, range):
+        # If we're asking for a range that extends past our maximum results,
+        # then we need to clamp the start of our slice to our maximum results
+        # size, and make sure that the end of our slice >= to that to ensure a
+        # consistent slice.
+        if range.start > self.max_results:
+            range = slice(
+                self.max_results,
+                max(range.stop, self.max_results),
+                range.step,
+            )
+
+        # If we're being asked for a range that extends past our maximum result
+        # then we'll clamp it to the maximum result size and stop there.
+        if range.stop > self.max_results:
+            range = slice(range.start, self.max_results, range.step)
+
         if self.results is not None:
             raise RuntimeError("Cannot reslice after having already sliced.")
         self.results = self.query[range].execute()
@@ -35,7 +53,7 @@ def __getitem__(self, range):
     def __len__(self):
         if self.results is None:
             raise RuntimeError("Cannot get length until a slice.")
-        return self.results.hits.total
+        return min(self.results.hits.total, self.max_results)
 
 
 def ElasticsearchPage(*args, **kwargs):  # noqa
diff --git a/warehouse/views.py b/warehouse/views.py
--- a/warehouse/views.py
+++ b/warehouse/views.py
@@ -11,7 +11,7 @@
 # limitations under the License.
 
 from pyramid.httpexceptions import (
-    HTTPException, HTTPSeeOther, HTTPMovedPermanently,
+    HTTPException, HTTPSeeOther, HTTPMovedPermanently, HTTPNotFound,
 )
 from pyramid.view import (
     notfound_view_config, forbidden_view_config, view_config,
@@ -173,12 +173,16 @@ def search(request):
     if request.params.get("o"):
         query = query.sort(request.params["o"])
 
+    page_num = int(request.params.get("page", 1))
     page = ElasticsearchPage(
         query,
-        page=int(request.params.get("page", 1)),
+        page=page_num,
         url_maker=paginate_url_factory(request),
     )
 
+    if page_num > page.page_count:
+        raise HTTPNotFound
+
     return {
         "page": page,
         "term": request.params.get("q"),


Test patch:
diff --git a/tests/unit/test_views.py b/tests/unit/test_views.py
--- a/tests/unit/test_views.py
+++ b/tests/unit/test_views.py
@@ -15,6 +15,8 @@
 import pretend
 import pytest
 
+from pyramid.httpexceptions import HTTPNotFound
+
 from warehouse import views
 from warehouse.views import (
     forbidden, index, httpexception_view, robotstxt, current_user_indicator,
@@ -111,7 +113,7 @@ def test_with_a_query(self, monkeypatch, page):
             params=params,
         )
 
-        page_obj = pretend.stub()
+        page_obj = pretend.stub(page_count=(page or 1) + 10)
         page_cls = pretend.call_recorder(lambda *a, **kw: page_obj)
         monkeypatch.setattr(views, "ElasticsearchPage", page_cls)
 
@@ -167,7 +169,7 @@ def test_with_an_ordering(self, monkeypatch, page):
             params=params,
         )
 
-        page_obj = pretend.stub()
+        page_obj = pretend.stub(page_count=(page or 1) + 10)
         page_cls = pretend.call_recorder(lambda *a, **kw: page_obj)
         monkeypatch.setattr(views, "ElasticsearchPage", page_cls)
 
@@ -218,7 +220,7 @@ def test_without_a_query(self, monkeypatch, page):
             params=params,
         )
 
-        page_obj = pretend.stub()
+        page_obj = pretend.stub(page_count=(page or 1) + 10)
         page_cls = pretend.call_recorder(lambda *a, **kw: page_obj)
         monkeypatch.setattr(views, "ElasticsearchPage", page_cls)
 
@@ -235,3 +237,27 @@ def test_without_a_query(self, monkeypatch, page):
             pretend.call(query, url_maker=url_maker, page=page or 1),
         ]
         assert url_maker_factory.calls == [pretend.call(request)]
+
+    def test_raises_404_with_pagenum_too_high(self, monkeypatch):
+        params = {"page": 15}
+        query = pretend.stub()
+        request = pretend.stub(
+            es=pretend.stub(query=lambda: query),
+            params=params,
+        )
+
+        page_obj = pretend.stub(page_count=10)
+        page_cls = pretend.call_recorder(lambda *a, **kw: page_obj)
+        monkeypatch.setattr(views, "ElasticsearchPage", page_cls)
+
+        url_maker = pretend.stub()
+        url_maker_factory = pretend.call_recorder(lambda request: url_maker)
+        monkeypatch.setattr(views, "paginate_url_factory", url_maker_factory)
+
+        with pytest.raises(HTTPNotFound):
+            search(request)
+
+        assert page_cls.calls == [
+            pretend.call(query, url_maker=url_maker, page=15 or 1),
+        ]
+        assert url_maker_factory.calls == [pretend.call(request)]
diff --git a/tests/unit/utils/test_paginate.py b/tests/unit/utils/test_paginate.py
--- a/tests/unit/utils/test_paginate.py
+++ b/tests/unit/utils/test_paginate.py
@@ -94,6 +94,18 @@ def test_slices_and_length(self):
         assert wrapper[1:3] == [2, 3]
         assert len(wrapper) == 6
 
+    def test_slice_start_clamps_to_max(self):
+        wrapper = paginate._ElasticsearchWrapper(FakeQuery([1, 2, 3, 4, 5, 6]))
+        wrapper.max_results = 5
+        assert wrapper[6:10] == []
+        assert len(wrapper) == 5
+
+    def test_slice_end_clamps_to_max(self):
+        wrapper = paginate._ElasticsearchWrapper(FakeQuery([1, 2, 3, 4, 5, 6]))
+        wrapper.max_results = 5
+        assert wrapper[1:10] == [2, 3, 4, 5]
+        assert len(wrapper) == 5
+
     def test_second_slice_fails(self):
         wrapper = paginate._ElasticsearchWrapper(FakeQuery([1, 2, 3, 4, 5, 6]))
         wrapper[1:3]