Repository: rotki/rotki

Issue:
Infinite loop in FTX
## Problem Definition

A user in discord reported that FTX was not finishing querying the fills endpoint. @LefterisJP found that there is a potential infinite loop if the amount of trades in the timestamp range is greater than the limit https://github.com/rotki/rotki/blob/327b32db321f38dffa2b220a0cbf2a8c8a8e9467/rotkehlchen/exchanges/ftx.py#L331

Seems that they provide a `minId`  to filter but is not in their docs. Saw it in their python example https://github.com/ftexchange/ftx/blob/master/rest/client.py#L188

```
okay, i've got some logs. It seems at first that the back is trying to increment end_time :
[07/09/2022 15:55:38 UTC] DEBUG rotkehlchen.exchanges.ftx Greenlet-5: FTX API query request_url=/api/fills?limit=100&start_time=0&end_time=1655270237
[07/09/2022 15:55:38 UTC] DEBUG rotkehlchen.exchanges.ftx Greenlet-5: FTX API query request_url=/api/fills?limit=100&start_time=0&end_time=1655136685
[...] 
Until :
[07/09/2022 15:58:24 UTC] DEBUG rotkehlchen.exchanges.ftx Greenlet-5: FTX API query request_url=/api/fills?limit=100&start_time=0&end_time=1646145228

From there, it constantly print that request with the same end_time all the time
Also i don't a single line  "end rotki api GET 200" related to that greenlet-5 : 
```

https://discordapp.com/channels/657906918408585217/745172610240872508/1017103272043094167




Discussion:

Resolving patch:
diff --git a/rotkehlchen/exchanges/ftx.py b/rotkehlchen/exchanges/ftx.py
--- a/rotkehlchen/exchanges/ftx.py
+++ b/rotkehlchen/exchanges/ftx.py
@@ -67,6 +67,7 @@
 FTX_SUBACCOUNT_DB_SETTING = 'ftx_subaccount'
 FTX_BASE_URL = 'https://ftx.com'
 FTXUS_BASE_URL = 'https://ftx.us'
+FTX_QUERY_ORDER = 'asc'
 
 
 def trade_from_ftx(raw_trade: Dict[str, Any]) -> Optional[Trade]:
@@ -184,7 +185,9 @@ def _make_request(
         endpoint: str,
         start_time: Optional[Timestamp] = None,
         end_time: Optional[Timestamp] = None,
+        min_id: Optional[int] = None,
         limit: int = PAGINATION_LIMIT,
+        order: Optional[str] = FTX_QUERY_ORDER,
     ) -> Union[List[Dict[str, Any]], Dict[str, List[Any]]]:
         """Performs an FTX API Query for endpoint adding the needed information to
         authenticate user and handling errors.
@@ -197,11 +200,15 @@ def _make_request(
         # Use a while loop to retry request if rate limit is reached
         while True:
             request_url = '/api/' + endpoint
-            options = {'limit': limit}
+            options: Dict[str, Union[str, int]] = {'limit': limit}
             if start_time is not None:
                 options['start_time'] = start_time
             if end_time is not None:
                 options['end_time'] = end_time
+            if min_id is not None:
+                options['minId'] = min_id
+            if order is not None:
+                options['order'] = order
 
             if len(options) != 0:
                 request_url += '?' + urlencode(options)
@@ -216,6 +223,7 @@ def _make_request(
             })
 
             full_url = self.base_uri + request_url
+            log.debug(f'Querying {full_url}')
             try:
                 response = self.session.get(full_url, timeout=DEFAULT_TIMEOUT_TUPLE)
             except requests.exceptions.RequestException as e:
@@ -306,8 +314,10 @@ def _api_query(
             end_time: Optional[Timestamp] = None,
             limit: int = PAGINATION_LIMIT,
             paginate: bool = True,
+            min_id: Optional[int] = None,
     ) -> Union[List[Dict[str, Any]], Dict[str, List[Any]], Dict[str, Any]]:
-        """Query FTX endpoint and retrieve all available information if pagination
+        """
+        Query FTX endpoint and retrieve all available information if pagination
         is requested. In case of paginate being set to False only one request is made.
         Can raise:
         - RemoteError
@@ -318,22 +328,27 @@ def _api_query(
                 limit=limit,
                 start_time=start_time,
                 end_time=end_time,
+                min_id=min_id,
             )
             return final_data_no_pag
 
         # If there is pagination we follow the example from the official ftx python example
         # https://github.com/ftexchange/ftx/blob/master/rest/client.py#L163
-        # In this case the strategy is a while loop leaving fixed the start_time (lower bound)
-        # and decreasing end time (the upper bound) until we fetch all the available information
-        new_end_time = end_time
+        # In this case the strategy is a while loop leaving fixed the end_time (upper bound)
+        # and increasing start time (the lower bound) until we fetch all the available information
+        # The min_id is included for the fill endpoint to avoid the case where more than the
+        # page size trades are returned in the current window.
+        new_start_time = start_time
+        new_min_id = min_id
         ids = set()
         final_data: List[Dict[str, Any]] = []
         while True:
             step = self._make_request(
                 endpoint=endpoint,
                 limit=limit,
-                start_time=start_time,
-                end_time=new_end_time,
+                start_time=new_start_time,
+                end_time=end_time,
+                min_id=new_min_id,
             )
 
             if not isinstance(step, list):
@@ -342,6 +357,9 @@ def _api_query(
                     f'with start_time {start_time} and end_time {end_time}. Result was {step}.',
                 )
 
+            if len(step) == 0:
+                break
+
             # remove possible duplicates
             deduped = [
                 r for r in step if
@@ -350,9 +368,6 @@ def _api_query(
             ids |= {r['id'] for r in deduped}
             final_data.extend(deduped)
 
-            if len(step) == 0:
-                break
-
             # Avoid deserialization error if there is a bad date
             times = []
             for t in step:
@@ -362,7 +377,7 @@ def _api_query(
                     continue
 
             if len(times) != 0:
-                new_end_time = min(times)
+                new_start_time = max(times)
             else:
                 self.msg_aggregator.add_error(
                     f'Error processing FTX trade history. Query step '
@@ -375,6 +390,13 @@ def _api_query(
             if len(step) < limit:
                 break
 
+            if endpoint == 'fills':
+                # This adds a min_id field to the request to prevent the case where more trades
+                # than the max per page are in the current windows and we need to shift it.
+                # We do + 1 because the new_mind_id is included in the response so we want to
+                # skip it. As for the trades I checked the ids are an integer always increasing.
+                new_min_id = max(ids) + 1
+
         return final_data
 
     @protect_with_lock()


Test patch: