Repository: ResonantGeoData/ResonantGeoData

Issue:
handle pagination in Rgdc.search
Solves part of #354 , replaces #359 

```Rgdc.search(limit=None)``` now reads and returns all pages instead of silently defaulting to `limit=100`.


Discussion:

Resolving patch:
diff --git a/rgdc/rgdc/rgdc.py b/rgdc/rgdc/rgdc.py
--- a/rgdc/rgdc/rgdc.py
+++ b/rgdc/rgdc/rgdc.py
@@ -11,7 +11,12 @@
 
 from .session import RgdcSession
 from .types import DATETIME_OR_STR_TUPLE, SEARCH_DATATYPE_CHOICE, SEARCH_PREDICATE_CHOICE
-from .utils import DEFAULT_RGD_API, datetime_to_str, download_checksum_file_to_path
+from .utils import (
+    DEFAULT_RGD_API,
+    datetime_to_str,
+    download_checksum_file_to_path,
+    limit_offset_pager,
+)
 
 
 @dataclass
@@ -185,7 +190,7 @@ def search(
         frame_rate: Optional[Tuple[int, int]] = None,
         limit: Optional[int] = None,
         offset: Optional[int] = None,
-    ) -> Iterator[Dict]:
+    ) -> List[Dict]:
         """
         Search for geospatial entries based on various criteria.
 
@@ -212,7 +217,7 @@ def search(
             offset: The number of results to skip.
 
         Returns:
-            An list of Spatial Entries.
+            A list of Spatial Entries.
         """
         # The dict that will be used to store params.
         # Initialize with queries that won't be additionally processed.
@@ -281,6 +286,4 @@ def search(
             params['frame_rate_min'] = frmin
             params['frame_rate_max'] = frmax
 
-        response = self.session.get('geosearch', params=params)
-        response.raise_for_status()
-        return [result for result in response.json()['results']]
+        return list(limit_offset_pager(self.session, 'geosearch', params=params))
diff --git a/rgdc/rgdc/utils.py b/rgdc/rgdc/utils.py
--- a/rgdc/rgdc/utils.py
+++ b/rgdc/rgdc/utils.py
@@ -1,6 +1,6 @@
 from datetime import datetime
 from pathlib import Path
-from typing import Dict, Iterator, List
+from typing import Dict, Generator, Iterator, List, Optional
 
 import requests
 from requests import Response, Session
@@ -20,6 +20,44 @@ def pager(session: Session, url: str, **kwargs) -> Iterator[Response]:
             break
 
 
+def limit_offset_pager(session: Session, url: str, **kwargs) -> Generator[Dict, None, None]:
+    """Exhaust a DRF Paginated list, respecting limit/offset."""
+    # Default params kwarg
+    if not kwargs.get('params'):
+        kwargs['params'] = {}
+
+    params: Optional[Dict] = kwargs['params']
+    total_limit = params.get('limit')
+
+    # Default offset
+    if params.get('offset') is None:
+        params['offset'] = 0
+
+    num_results = 0
+    while True:
+        # Update limit
+        if total_limit:
+            params['limit'] = total_limit - num_results
+
+        # Make request and raise exception if failed
+        r = session.get(url, **kwargs)
+        r.raise_for_status()
+
+        # Yield results
+        results = r.json()['results']
+        yield from results
+
+        # Update offset and num_results
+        params['offset'] += len(results)
+        num_results += len(results)
+
+        # Check if there is no more data, or if we've reached our limit
+        no_more_data = 'next' not in r.links or not results
+        limit_reached = total_limit and num_results >= total_limit
+        if no_more_data or limit_reached:
+            break
+
+
 def iterate_response_bytes(
     url: str, chunk_size: int = 1024 * 1024, raise_for_status: bool = True
 ) -> Iterator[bytes]:


Test patch: