Repository: rotki/rotki

Issue:
Binance trades log request failed due to invalid signature
## Problem Definition

Requesting Binance trades logs failed requests due to invalid signature. This was allegedly fixed in the 1.9.1 release, which included a fix for preventing the reuse of the signature on each [api_query() call](https://github.com/rotki/rotki/pull/1791/files#diff-a2460a3afc4ef931badf61357148e3d25a59d58f116bc302ead8910646396d8cR233).

Investigate why and fix it.

Consider if the error log is related with [this](https://github.com/rotki/rotki/blob/develop/rotkehlchen/exchanges/binance.py#L238).

Related issue: #1416
Related PR: #1791

## Logs

Reported by the user:

```shell
Failed query for Binance: Error: Binance API request https://api.binance.com/api/v3/myTrades?symbol=POABNB&fromId=0&limit=1000&recvWindow=10000&timestamp=1606900923763&signature=94f87ee92c42a2c3634248d1baf668833ae2d685c6b835bcf5e2bc76fbf1978e for myTrades failed with HTTP status code: 400, error code: -1022 and error message: Signature for this request is not valid.
```

```shell
Failed query for Binance: Error: Binance API request https://api.binance.com/api/v3/myTrades?symbol=ETHDAI&fromId=0&limit=1000&recvWindow=10000&timestamp=1606901461608&signature=02245ee1ebdb9c336e3b5901281128a2f339633dbeaeb60cc15287639a7036c0 for myTrades failed with HTTP status code: 400, error code: -1022 and error message: Signature for this request is not valid.
```

### System Description

Operating system: to be confirmed
Rotki version: 1.9.1

Closed by #1860 


Discussion:

Resolving patch:
diff --git a/rotkehlchen/exchanges/binance.py b/rotkehlchen/exchanges/binance.py
--- a/rotkehlchen/exchanges/binance.py
+++ b/rotkehlchen/exchanges/binance.py
@@ -64,6 +64,8 @@
     'withdrawHistory.html',
 )
 
+RETRY_AFTER_LIMIT = 60
+
 
 class BinancePair(NamedTuple):
     """A binance pair. Contains the symbol in the Binance mode e.g. "ETHBTC" and
@@ -231,13 +233,15 @@ def validate_api_key(self) -> Tuple[bool, str]:
 
     def api_query(self, method: str, options: Optional[Dict] = None) -> Union[List, Dict]:
         call_options = options.copy() if options else {}
-        backoff = self.initial_backoff
 
         while True:
             with self.nonce_lock:
                 # Protect this region with a lock since binance will reject
                 # non-increasing nonces. So if two greenlets come in here at
                 # the same time one of them will fail
+                if 'signature' in call_options:
+                    del call_options['signature']
+
                 if method in V3_ENDPOINTS or method in WAPI_ENDPOINTS:
                     api_version = 3
                     # Recommended recvWindows is 5000 but we get timeouts with it
@@ -257,15 +261,13 @@ def api_query(self, method: str, options: Optional[Dict] = None) -> Union[List,
                 apistr = 'wapi/' if method in WAPI_ENDPOINTS else 'api/'
                 request_url = f'{self.uri}{apistr}v{str(api_version)}/{method}?'
                 request_url += urlencode(call_options)
-
                 log.debug('Binance API request', request_url=request_url)
                 try:
                     response = self.session.get(request_url)
                 except requests.exceptions.RequestException as e:
                     raise RemoteError(f'Binance API request failed due to {str(e)}')
 
-            limit_ban = response.status_code == 429 and backoff > self.backoff_limit
-            if limit_ban or response.status_code not in (200, 429):
+            if response.status_code not in (200, 418, 429):
                 code = 'no code found'
                 msg = 'no message found'
                 try:
@@ -285,14 +287,29 @@ def api_query(self, method: str, options: Optional[Dict] = None) -> Union[List,
                         code,
                         msg,
                     ))
-            elif response.status_code == 429:
-                if backoff > self.backoff_limit:
-                    break
-                # Binance has limits and if we hit them we should backoff
-                # https://github.com/binance-exchange/binance-official-api-docs/blob/master/rest-api.md#limits
-                log.debug('Got 429 from Binance. Backing off', seconds=backoff)
-                gevent.sleep(backoff)
-                backoff = backoff * 2
+            elif response.status_code in (418, 429):
+                # Binance has limits and if we hit them we should backoff.
+                # A Retry-After header is sent with a 418 or 429 responses and
+                # will give the number of seconds required to wait, in the case
+                # of a 429, to prevent a ban, or, in the case of a 418, until
+                # the ban is over.
+                # https://binance-docs.github.io/apidocs/spot/en/#limits
+                retry_after = int(response.headers.get('retry-after', '0'))
+                log.debug(
+                    f'Got status code {response.status_code} from Binance. Backing off',
+                    seconds=retry_after,
+                )
+                if retry_after > RETRY_AFTER_LIMIT:
+                    raise RemoteError(
+                        'Binance API request {} for {} failed with HTTP status '
+                        'code: {} due to a too long retry after value (> {})'.format(
+                            response.url,
+                            method,
+                            response.status_code,
+                            RETRY_AFTER_LIMIT,
+                        ))
+
+                gevent.sleep(retry_after)
                 continue
             else:
                 # success


Test patch:
diff --git a/rotkehlchen/tests/api/test_ethereum_transactions.py b/rotkehlchen/tests/api/test_ethereum_transactions.py
--- a/rotkehlchen/tests/api/test_ethereum_transactions.py
+++ b/rotkehlchen/tests/api/test_ethereum_transactions.py
@@ -129,8 +129,7 @@
 def test_query_transactions(rotkehlchen_api_server):
     """Test that querying the ethereum transactions endpoint works as expected
 
-    This test uses real data. Found an ethereum address that has very few transactions
-    and hopefully won't have more. If it does we can adjust the test.
+    This test uses real data.
     """
     async_query = random.choice([False, True])
     rotki = rotkehlchen_api_server.rest_api.rotkehlchen
@@ -151,8 +150,19 @@ def test_query_transactions(rotkehlchen_api_server):
     expected_result = EXPECTED_AFB7_TXS + EXPECTED_4193_TXS
     expected_result.sort(key=lambda x: x['timestamp'])
     expected_result.reverse()
-    assert result['entries'] == expected_result
-    assert result['entries_found'] == len(expected_result)
+
+    # Make sure that all of the transactions we expect are there and in order
+    # There can be more transactions (since the address can make more)
+    # but this check ignores them
+    previous_index = 0
+    for entry in expected_result:
+        assert entry in result['entries']
+        entry_idx = result['entries'].index(entry)
+        if previous_index != 0:
+            assert entry_idx == previous_index + 1
+        previous_index = entry_idx
+
+    assert result['entries_found'] >= len(expected_result)
     assert result['entries_limit'] == FREE_ETH_TX_LIMIT
 
     # Check that transactions per address and in a specific time range can be
diff --git a/rotkehlchen/tests/exchanges/test_binance.py b/rotkehlchen/tests/exchanges/test_binance.py
--- a/rotkehlchen/tests/exchanges/test_binance.py
+++ b/rotkehlchen/tests/exchanges/test_binance.py
@@ -1,6 +1,10 @@
+import hashlib
+import hmac
 import warnings as test_warnings
+from contextlib import ExitStack
 from datetime import datetime
 from unittest.mock import call, patch
+from urllib.parse import urlencode
 
 import pytest
 
@@ -12,6 +16,7 @@
 from rotkehlchen.exchanges.binance import (
     API_TIME_INTERVAL_CONSTRAINT_TS,
     BINANCE_LAUNCH_TS,
+    RETRY_AFTER_LIMIT,
     Binance,
     trade_from_binance,
 )
@@ -23,6 +28,7 @@
 from rotkehlchen.tests.utils.mock import MockResponse
 from rotkehlchen.typing import AssetMovementCategory, Timestamp
 from rotkehlchen.user_messages import MessagesAggregator
+from rotkehlchen.utils.misc import ts_now_in_ms
 
 
 def test_name():
@@ -151,32 +157,6 @@ def test_trade_from_binance(function_scope_binance):
 }'''
 
 
-def test_binance_backoff_after_429(function_scope_binance):
-    count = 0
-    binance = function_scope_binance
-
-    def mock_429(url):  # pylint: disable=unused-argument
-        nonlocal count
-        if count < 2:
-            response = MockResponse(429, '{"code": -1103, "msg": "Too many requests"}')
-        else:
-            response = MockResponse(200, exchange_info_mock_text)
-        count += 1
-        return response
-
-    binance.initial_backoff = 0.5
-    binance.backoff_limit = 2
-    with patch.object(binance.session, 'get', side_effect=mock_429):
-        # test that after 2 429 cals we finally succeed in the API call
-        result = binance.api_query('exchangeInfo')
-        assert 'timezone' in result
-
-        # Test the backoff_limit properly returns an error when hit
-        count = -9999999
-        with pytest.raises(RemoteError):
-            binance.api_query('exchangeInfo')
-
-
 def test_binance_assets_are_known(
         database,
         inquirer,  # pylint: disable=unused-argument
@@ -790,3 +770,70 @@ def mock_get_deposit_withdrawal(url):  # pylint: disable=unused-argument
                 end_ts=Timestamp(end_ts),
             )
             assert mock_api_query_dict.call_args_list == expected_calls
+
+
+@pytest.mark.freeze_time(datetime(2020, 11, 24, 3, 14, 15))
+def test_api_query_retry_on_status_code_429(function_scope_binance):
+    """Test when Binance API returns 429 and the request is retried, the
+    signature is not polluted by any attribute from the previous call.
+
+    It also tests getting the `retry-after` seconds to backoff from the
+    response header.
+
+    NB: basically remove `call_options['signature']`.
+    """
+    binance = function_scope_binance
+    offset_ms = 1000
+    call_options = {
+        'fromId': 0,
+        'limit': 1000,
+        'symbol': 'BUSDUSDT',
+        'recvWindow': 10000,
+        'timestamp': str(ts_now_in_ms() + offset_ms),
+    }
+    signature = hmac.new(
+        binance.secret,
+        urlencode(call_options).encode('utf-8'),
+        hashlib.sha256,
+    ).hexdigest()
+    call_options['signature'] = signature
+    base_url = 'https://api.binance.com/api/v3/myTrades?'
+    exp_request_url = base_url + urlencode(call_options)
+
+    # NB: all calls must have the same signature (time frozen)
+    expected_calls = [call(exp_request_url), call(exp_request_url), call(exp_request_url)]
+
+    def get_mocked_response():
+        responses = [
+            MockResponse(429, '[]', headers={'retry-after': '1'}),
+            MockResponse(418, '[]', headers={'retry-after': '5'}),
+            MockResponse(418, '[]', headers={'retry-after': str(RETRY_AFTER_LIMIT + 1)}),
+        ]
+        for response in responses:
+            yield response
+
+    def mock_response(url):  # pylint: disable=unused-argument
+        return next(get_response)
+
+    get_response = get_mocked_response()
+
+    # force to break the loop after 2 requests by lowering the `self.backoff_limit`
+    backoff_patch = patch.object(binance, 'backoff_limit', new=7)
+    offset_ms_patch = patch.object(binance, 'offset_ms', new=1000)
+    binance_patch = patch.object(binance.session, 'get', side_effect=mock_response)
+
+    with ExitStack() as stack:
+        stack.enter_context(backoff_patch)
+        stack.enter_context(offset_ms_patch)
+        binance_mock_get = stack.enter_context(binance_patch)
+        with pytest.raises(RemoteError) as e:
+            binance.api_query(
+                method='myTrades',
+                options={
+                    'fromId': 0,
+                    'limit': 1000,
+                    'symbol': 'BUSDUSDT',
+                },
+            )
+    assert 'myTrades failed with HTTP status code: 418' in str(e.value)
+    assert binance_mock_get.call_args_list == expected_calls
diff --git a/rotkehlchen/tests/utils/mock.py b/rotkehlchen/tests/utils/mock.py
--- a/rotkehlchen/tests/utils/mock.py
+++ b/rotkehlchen/tests/utils/mock.py
@@ -6,11 +6,17 @@
 
 
 class MockResponse():
-    def __init__(self, status_code: int, text: str) -> None:
+    def __init__(
+            self,
+            status_code: int,
+            text: str,
+            headers: Dict['str', Any] = None,
+    ) -> None:
         self.status_code = status_code
         self.text = text
         self.content = text.encode()
         self.url = 'http://someurl.com'
+        self.headers = headers or {}
 
     def json(self) -> Dict[str, Any]:
         return json.loads(self.text)