Repository: rotki/rotki

Issue:
Investigate our handling of Binance api's rate limits
## Problem Definition

Multiple users have reported they have gotten throttled by Binance during PnL report generation. We should investigate and see if anything changed on Binance's side and if we need to adjust Rotki's behavior.

## Task

Study binance api rate limiting docs: https://binance-docs.github.io/apidocs/spot/en/#limits and our implementation and see if we can improve anything.


Discussion:

Resolving patch:
diff --git a/rotkehlchen/exchanges/binance.py b/rotkehlchen/exchanges/binance.py
--- a/rotkehlchen/exchanges/binance.py
+++ b/rotkehlchen/exchanges/binance.py
@@ -225,8 +225,6 @@ def __init__(
             secret: ApiSecret,
             database: 'DBHandler',
             msg_aggregator: MessagesAggregator,
-            initial_backoff: int = 4,
-            backoff_limit: int = 180,
             uri: str = BINANCE_BASE_URL,
     ):
         exchange_name = str(Location.BINANCE)
@@ -240,8 +238,6 @@ def __init__(
             'X-MBX-APIKEY': self.api_key,
         })
         self.msg_aggregator = msg_aggregator
-        self.initial_backoff = initial_backoff
-        self.backoff_limit = backoff_limit
         self.nonce_lock = Semaphore()
         self.offset_ms = 0
 
@@ -394,6 +390,9 @@ def api_query(
                 # the ban is over.
                 # https://binance-docs.github.io/apidocs/spot/en/#limits
                 retry_after = int(response.headers.get('retry-after', '0'))
+                # Spoiler. They actually seem to always return 0 here. So we don't
+                # wait at all. Won't be much of an improvement but force 1 sec wait if 0 returns
+                retry_after = max(1, retry_after)  # wait at least 1 sec even if api says otherwise
                 log.debug(
                     f'Got status code {response.status_code} from {self.name}. Backing off',
                     seconds=retry_after,
@@ -401,11 +400,12 @@ def api_query(
                 if retry_after > RETRY_AFTER_LIMIT:
                     raise RemoteError(
                         '{} API request {} for {} failed with HTTP status '
-                        'code: {} due to a too long retry after value (> {})'.format(
+                        'code: {} due to a too long retry after value ({} > {})'.format(
                             self.name,
                             response.url,
                             method,
                             response.status_code,
+                            retry_after,
                             RETRY_AFTER_LIMIT,
                         ))
 
diff --git a/rotkehlchen/tasks/manager.py b/rotkehlchen/tasks/manager.py
--- a/rotkehlchen/tasks/manager.py
+++ b/rotkehlchen/tasks/manager.py
@@ -215,6 +215,8 @@ def _maybe_schedule_exchange_history_query(self) -> None:
         now = ts_now()
         queriable_exchanges = []
         for name, exchange in self.exchange_manager.connected_exchanges.items():
+            if name in ('binance', 'binance_us'):
+                continue  # skip binance due to the way their history is queried and rate limiting
             queried_range = self.database.get_used_query_range(f'{name}_trades')
             end_ts = queried_range[1] if queried_range else 0
             if now - max(self.last_exchange_query_ts[name], end_ts) > EXCHANGE_QUERY_FREQUENCY:


Test patch:
diff --git a/rotkehlchen/tests/exchanges/test_binance.py b/rotkehlchen/tests/exchanges/test_binance.py
--- a/rotkehlchen/tests/exchanges/test_binance.py
+++ b/rotkehlchen/tests/exchanges/test_binance.py
@@ -817,14 +817,10 @@ def mock_response(url):  # pylint: disable=unused-argument
         return next(get_response)
 
     get_response = get_mocked_response()
-
-    # force to break the loop after 2 requests by lowering the `self.backoff_limit`
-    backoff_patch = patch.object(binance, 'backoff_limit', new=7)
     offset_ms_patch = patch.object(binance, 'offset_ms', new=1000)
     binance_patch = patch.object(binance.session, 'get', side_effect=mock_response)
 
     with ExitStack() as stack:
-        stack.enter_context(backoff_patch)
         stack.enter_context(offset_ms_patch)
         binance_mock_get = stack.enter_context(binance_patch)
         with pytest.raises(RemoteError) as e: