Repository: rotki/rotki

Issue:
Binance Deposits & Withdrawals queries need to be fixed to conform to 90day window
## Problem Definition
As per the Binance API https://binance-docs.github.io/apidocs/spot/en/#withdraw-history-supporting-network-user_data & https://binance-docs.github.io/apidocs/spot/en/#deposit-history-user_data , the time interval for requested data must be 0-90 days. Currently rotki does not split queries into 90-day increments and thus the query returns 0 results.



```
[01/09/2020 10:05:24 W. Europe Daylight Time] rotkehlchen.exchanges.binance: Binance API request request_url=https://api.binance.com/wapi/v3/withdrawHistory.html?timestamp=1598947524970&startTime=1598947182000&endTime=1598947524000&recvWindow=10000&signature=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
[01/09/2020 10:05:25 W. Europe Daylight Time] rotkehlchen.exchanges.kraken: Kraken Ledgers Query Response with count:0
```

## Solution
Queries to Binance for withdrawals / deposits must be split into 90 day increments. 



Discussion:
Great. Just for the record when the driver was used the api docs were hosted here: https://github.com/binance-exchange/binance-official-api-docs/blob/master/wapi-api.md#deposit-history-user_data

They still are, and there is no such info on the 90 day period there. 

We'll fix this ofc in the next release :smile: 

Resolving patch:
diff --git a/rotkehlchen/exchanges/binance.py b/rotkehlchen/exchanges/binance.py
--- a/rotkehlchen/exchanges/binance.py
+++ b/rotkehlchen/exchanges/binance.py
@@ -8,6 +8,7 @@
 import gevent
 import requests
 from gevent.lock import Semaphore
+from typing_extensions import Literal
 
 from rotkehlchen.assets.converters import asset_from_binance
 from rotkehlchen.constants import BINANCE_BASE_URL
@@ -43,6 +44,10 @@
 logger = logging.getLogger(__name__)
 log = RotkehlchenLogsAdapter(logger)
 
+# Binance launched at 2017-07-14T04:00:00Z (12:00 GMT+8, Beijing Time)
+# https://www.binance.com/en/support/articles/115000599831-Binance-Exchange-Launched-Date-Set
+BINANCE_LAUNCH_TS = Timestamp(1500001200)
+API_TIME_INTERVAL_CONSTRAINT_TS = Timestamp(7776000)  # 90 days
 V3_ENDPOINTS = (
     'account',
     'myTrades',
@@ -225,9 +230,7 @@ def validate_api_key(self) -> Tuple[bool, str]:
         return True, ''
 
     def api_query(self, method: str, options: Optional[Dict] = None) -> Union[List, Dict]:
-        if not options:
-            options = {}
-
+        call_options = options.copy() if options else {}
         backoff = self.initial_backoff
 
         while True:
@@ -238,14 +241,14 @@ def api_query(self, method: str, options: Optional[Dict] = None) -> Union[List,
                 if method in V3_ENDPOINTS or method in WAPI_ENDPOINTS:
                     api_version = 3
                     # Recommended recvWindows is 5000 but we get timeouts with it
-                    options['recvWindow'] = 10000
-                    options['timestamp'] = str(ts_now_in_ms() + self.offset_ms)
+                    call_options['recvWindow'] = 10000
+                    call_options['timestamp'] = str(ts_now_in_ms() + self.offset_ms)
                     signature = hmac.new(
                         self.secret,
-                        urlencode(options).encode('utf-8'),
+                        urlencode(call_options).encode('utf-8'),
                         hashlib.sha256,
                     ).hexdigest()
-                    options['signature'] = signature
+                    call_options['signature'] = signature
                 elif method in V1_ENDPOINTS:
                     api_version = 1
                 else:
@@ -253,7 +256,7 @@ def api_query(self, method: str, options: Optional[Dict] = None) -> Union[List,
 
                 apistr = 'wapi/' if method in WAPI_ENDPOINTS else 'api/'
                 request_url = f'{self.uri}{apistr}v{str(api_version)}/{method}?'
-                request_url += urlencode(options)
+                request_url += urlencode(call_options)
 
                 log.debug('Binance API request', request_url=request_url)
                 try:
@@ -522,30 +525,94 @@ def _deserialize_asset_movement(self, raw_data: Dict[str, Any]) -> Optional[Asse
 
         return None
 
+    def _api_query_dict_within_time_delta(
+            self,
+            start_ts: Timestamp,
+            end_ts: Timestamp,
+            time_delta: Timestamp,
+            method: Literal['depositHistory.html', 'withdrawHistory.html'],
+    ) -> List[Dict[str, Any]]:
+        """Request via `api_query_dict()` from `start_ts` `end_ts` using a time
+        delta (offset) less than `time_delta`.
+
+        Be aware of:
+          - If `start_ts` equals zero, the Binance launch timestamp is used
+          (from BINANCE_LAUNCH_TS). This value is not stored in the
+          `used_query_ranges` table, but 0.
+          - Timestamps are converted to milliseconds.
+        """
+        if method == 'depositHistory.html':
+            query_schema = 'depositList'
+        elif method == 'withdrawHistory.html':
+            query_schema = 'withdrawList'
+        else:
+            raise AssertionError(f'Unexpected binance method case: {method}.')
+
+        results: List[Dict[str, Any]] = []
+
+        # Create required time references in milliseconds
+        start_ts = Timestamp(start_ts * 1000)
+        end_ts = Timestamp(end_ts * 1000)
+        offset = time_delta * 1000 - 1  # less than time_delta
+        if start_ts == Timestamp(0):
+            from_ts = BINANCE_LAUNCH_TS * 1000
+        else:
+            from_ts = start_ts
+
+        to_ts = (
+            from_ts + offset  # Case request with offset
+            if end_ts - from_ts > offset
+            else end_ts  # Case request without offset (1 request)
+        )
+        while True:
+            options = {
+                'timestamp': ts_now_in_ms(),
+                'startTime': from_ts,
+                'endTime': to_ts,
+            }
+            result = self.api_query_dict(method, options=options)
+            results.extend(result.get(query_schema, []))
+            # Case stop requesting
+            if to_ts >= end_ts:
+                break
+
+            from_ts = to_ts + 1
+            to_ts = min(to_ts + offset, end_ts)
+
+        return results
+
     def query_online_deposits_withdrawals(
             self,
             start_ts: Timestamp,
             end_ts: Timestamp,
     ) -> List[AssetMovement]:
-        # This does not check for any limits. Can there be any limits like with trades
-        # in the deposit/withdrawal binance api? Can't see anything in the docs:
-        # https://github.com/binance-exchange/binance-official-api-docs/blob/master/wapi-api.md#deposit-history-user_data
-        #
-        # Note that all timestamps should be in milliseconds, so we multiply by 1k
-        options = {
-            'timestamp': ts_now_in_ms(),
-            'startTime': start_ts * 1000,
-            'endTime': end_ts * 1000,
-        }
-        result = self.api_query_dict('depositHistory.html', options=options)
-        raw_data = result.get('depositList', [])
-        options['timestamp'] = ts_now_in_ms()
-        result = self.api_query_dict('withdrawHistory.html', options=options)
-        raw_data.extend(result.get('withdrawList', []))
-        log.debug('binance deposit/withdrawal history result', results_num=len(raw_data))
+        """
+        Be aware of:
+          - Timestamps must be in milliseconds.
+          - There must be less than 90 days between start and end timestamps.
+
+        Deposit & Withdraw history documentation:
+        https://binance-docs.github.io/apidocs/spot/en/#deposit-history-user_data
+        https://binance-docs.github.io/apidocs/spot/en/#withdraw-history-user_data
+        """
+        deposits = self._api_query_dict_within_time_delta(
+            start_ts=start_ts,
+            end_ts=end_ts,
+            time_delta=API_TIME_INTERVAL_CONSTRAINT_TS,
+            method='depositHistory.html',
+        )
+        log.debug('binance deposit history result', results_num=len(deposits))
+
+        withdraws = self._api_query_dict_within_time_delta(
+            start_ts=start_ts,
+            end_ts=end_ts,
+            time_delta=API_TIME_INTERVAL_CONSTRAINT_TS,
+            method='withdrawHistory.html',
+        )
+        log.debug('binance withdraw history result', results_num=len(withdraws))
 
         movements = []
-        for raw_movement in raw_data:
+        for raw_movement in deposits + withdraws:
             movement = self._deserialize_asset_movement(raw_movement)
             if movement:
                 movements.append(movement)


Test patch:
diff --git a/rotkehlchen/tests/exchanges/test_binance.py b/rotkehlchen/tests/exchanges/test_binance.py
--- a/rotkehlchen/tests/exchanges/test_binance.py
+++ b/rotkehlchen/tests/exchanges/test_binance.py
@@ -1,5 +1,6 @@
 import warnings as test_warnings
-from unittest.mock import patch
+from datetime import datetime
+from unittest.mock import call, patch
 
 import pytest
 
@@ -8,15 +9,19 @@
 from rotkehlchen.constants.assets import A_BTC, A_ETH
 from rotkehlchen.constants.misc import ZERO
 from rotkehlchen.errors import RemoteError, UnknownAsset, UnsupportedAsset
-from rotkehlchen.exchanges.binance import Binance, trade_from_binance
+from rotkehlchen.exchanges.binance import (
+    API_TIME_INTERVAL_CONSTRAINT_TS,
+    BINANCE_LAUNCH_TS,
+    Binance,
+    trade_from_binance,
+)
 from rotkehlchen.exchanges.data_structures import Location, Trade, TradeType
 from rotkehlchen.fval import FVal
 from rotkehlchen.tests.utils.constants import A_BNB, A_RDN, A_USDT, A_XMR
 from rotkehlchen.tests.utils.exchanges import BINANCE_BALANCES_RESPONSE, BINANCE_MYTRADES_RESPONSE
 from rotkehlchen.tests.utils.factories import make_api_key, make_api_secret
-from rotkehlchen.tests.utils.history import TEST_END_TS
 from rotkehlchen.tests.utils.mock import MockResponse
-from rotkehlchen.typing import AssetMovementCategory
+from rotkehlchen.typing import AssetMovementCategory, Timestamp
 from rotkehlchen.user_messages import MessagesAggregator
 
 
@@ -398,7 +403,7 @@ def query_binance_and_test(
             "address": "0x6915f16f8791d0a1cc2bf47c13a6b2a92000504b",
             "asset": "ETH",
             "txId": "0xdf33b22bdb2b28b1f75ccd201a4a4m6e7g83jy5fc5d5a9d1340961598cfcb0a1",
-            "applyTime": 1518192542000,
+            "applyTime": 1508488245000,
             "status": 4
         },
         {
@@ -408,7 +413,7 @@ def query_binance_and_test(
             "addressTag": "342341222",
             "txId": "b3c6219639c8ae3f9cf010cdc24fw7f7yt8j1e063f9b4bd1a05cb44c4b6e2509",
             "asset": "XMR",
-            "applyTime": 1529198732000,
+            "applyTime": 1508512521000,
             "status": 4
         }
     ],
@@ -417,7 +422,13 @@ def query_binance_and_test(
 
 
 def test_binance_query_deposits_withdrawals(function_scope_binance):
-    """Test the happy case of binance deposit withdrawal query"""
+    """Test the happy case of binance deposit withdrawal query
+
+    NB: set `start_ts` and `end_ts` with a difference less than 90 days to
+    prevent requesting with a time delta.
+    """
+    start_ts = 1508022000  # 2017-10-15
+    end_ts = 1508540400  # 2017-10-21 (less than 90 days since `start_ts`)
     binance = function_scope_binance
 
     def mock_get_deposit_withdrawal(url):  # pylint: disable=unused-argument
@@ -429,7 +440,10 @@ def mock_get_deposit_withdrawal(url):  # pylint: disable=unused-argument
         return MockResponse(200, response_str)
 
     with patch.object(binance.session, 'get', side_effect=mock_get_deposit_withdrawal):
-        movements = binance.query_online_deposits_withdrawals(start_ts=0, end_ts=TEST_END_TS)
+        movements = binance.query_online_deposits_withdrawals(
+            start_ts=Timestamp(start_ts),
+            end_ts=Timestamp(end_ts),
+        )
 
     errors = binance.msg_aggregator.consume_errors()
     warnings = binance.msg_aggregator.consume_warnings()
@@ -456,7 +470,7 @@ def mock_get_deposit_withdrawal(url):  # pylint: disable=unused-argument
 
     assert movements[2].location == Location.BINANCE
     assert movements[2].category == AssetMovementCategory.WITHDRAWAL
-    assert movements[2].timestamp == 1518192542
+    assert movements[2].timestamp == 1508488245
     assert isinstance(movements[2].asset, Asset)
     assert movements[2].asset == A_ETH
     assert movements[2].amount == FVal('1')
@@ -464,7 +478,7 @@ def mock_get_deposit_withdrawal(url):  # pylint: disable=unused-argument
 
     assert movements[3].location == Location.BINANCE
     assert movements[3].category == AssetMovementCategory.WITHDRAWAL
-    assert movements[3].timestamp == 1529198732
+    assert movements[3].timestamp == 1508512521
     assert isinstance(movements[3].asset, Asset)
     assert movements[3].asset == A_XMR
     assert movements[3].amount == FVal('850.1')
@@ -472,7 +486,13 @@ def mock_get_deposit_withdrawal(url):  # pylint: disable=unused-argument
 
 
 def test_binance_query_deposits_withdrawals_unexpected_data(function_scope_binance):
-    """Test that we handle unexpected deposit withdrawal query data gracefully"""
+    """Test that we handle unexpected deposit withdrawal query data gracefully
+
+    NB: set `start_ts` and `end_ts` with a difference less than 90 days to
+    prevent requesting with a time delta.
+    """
+    start_ts = 1508022000  # 2017-10-15
+    end_ts = 1508540400  # 2017-10-21 (less than 90 days since `start_ts`)
     binance = function_scope_binance
 
     def mock_binance_and_query(deposits, withdrawals, expected_warnings_num, expected_errors_num):
@@ -486,7 +506,10 @@ def mock_get_deposit_withdrawal(url):  # pylint: disable=unused-argument
             return MockResponse(200, response_str)
 
         with patch.object(binance.session, 'get', side_effect=mock_get_deposit_withdrawal):
-            movements = binance.query_online_deposits_withdrawals(start_ts=0, end_ts=TEST_END_TS)
+            movements = binance.query_online_deposits_withdrawals(
+                start_ts=Timestamp(start_ts),
+                end_ts=Timestamp(end_ts),
+            )
 
         if expected_errors_num == 0 and expected_warnings_num == 0:
             assert len(movements) == 1
@@ -628,3 +651,142 @@ def check_permutations_of_input_invalid_data(deposits, withdrawals):
         deposits=input_deposits,
         withdrawals=empty_withdrawals,
     )
+
+
+def test_binance_query_deposits_withdrawals_gte_90_days(function_scope_binance):
+    """Test the not so happy case of binance deposit withdrawal query
+
+    From `start_ts` to `end_ts` there is a difference gte 90 days, which forces
+    to request using a time delta (from API_TIME_INTERVAL_CONSTRAINT_TS). As
+    the `mock_get_deposit_withdrawal()` results are the same as in
+    `test_binance_query_deposits_withdrawals`, we only assert the number of
+    movements.
+
+    NB: this test implementation must mock 4 requests instead of 2.
+    """
+    start_ts = 0  # Defaults to BINANCE_LAUNCH_TS
+    end_ts = BINANCE_LAUNCH_TS + API_TIME_INTERVAL_CONSTRAINT_TS  # eq 90 days after
+    binance = function_scope_binance
+
+    def get_time_delta_deposit_result():
+        results = [
+            BINANCE_DEPOSITS_HISTORY_RESPONSE,
+            '{}',
+        ]
+        for result in results:
+            yield result
+
+    def get_time_delta_withdraw_result():
+        results = [
+            '{}',
+            BINANCE_WITHDRAWALS_HISTORY_RESPONSE,
+        ]
+        for result in results:
+            yield result
+
+    def mock_get_deposit_withdrawal(url):  # pylint: disable=unused-argument
+        if 'deposit' in url:
+            response_str = next(get_deposit_result)
+        else:
+            response_str = next(get_withdraw_result)
+
+        return MockResponse(200, response_str)
+
+    get_deposit_result = get_time_delta_deposit_result()
+    get_withdraw_result = get_time_delta_withdraw_result()
+
+    with patch.object(binance.session, 'get', side_effect=mock_get_deposit_withdrawal):
+        movements = binance.query_online_deposits_withdrawals(
+            start_ts=Timestamp(start_ts),
+            end_ts=Timestamp(end_ts),
+        )
+
+    errors = binance.msg_aggregator.consume_errors()
+    warnings = binance.msg_aggregator.consume_warnings()
+    assert len(errors) == 0
+    assert len(warnings) == 0
+
+    assert len(movements) == 4
+
+
+@pytest.mark.freeze_time(datetime(2020, 11, 24, 3, 14, 15))
+def test_api_query_dict_calls_with_time_delta(function_scope_binance):
+    """Test the `api_query_dict()` arguments when deposit/withdraw history
+    requests involve a time delta.
+
+    From `start_ts` to `end_ts` there is a difference gte 90 days, which forces
+    to request using a time delta (from API_TIME_INTERVAL_CONSTRAINT_TS).
+    """
+    now_ts_ms = int(datetime.utcnow().timestamp()) * 1000
+    start_ts = 0  # Defaults to BINANCE_LAUNCH_TS
+    end_ts = BINANCE_LAUNCH_TS + API_TIME_INTERVAL_CONSTRAINT_TS  # eq 90 days after
+    expected_calls = [
+        call(
+            'depositHistory.html',
+            options={
+                'timestamp': now_ts_ms,
+                'startTime': 1500001200000,
+                'endTime': 1507777199999,
+            },
+        ),
+        call(
+            'depositHistory.html',
+            options={
+                'timestamp': now_ts_ms,
+                'startTime': 1507777200000,
+                'endTime': 1507777200000,
+            },
+        ),
+        call(
+            'withdrawHistory.html',
+            options={
+                'timestamp': now_ts_ms,
+                'startTime': 1500001200000,
+                'endTime': 1507777199999,
+            },
+        ),
+        call(
+            'withdrawHistory.html',
+            options={
+                'timestamp': now_ts_ms,
+                'startTime': 1507777200000,
+                'endTime': 1507777200000,
+            },
+        ),
+    ]
+    binance = function_scope_binance
+
+    def get_time_delta_deposit_result():
+        results = [
+            BINANCE_DEPOSITS_HISTORY_RESPONSE,
+            '{}',
+        ]
+        for result in results:
+            yield result
+
+    def get_time_delta_withdraw_result():
+        results = [
+            '{}',
+            BINANCE_WITHDRAWALS_HISTORY_RESPONSE,
+        ]
+        for result in results:
+            yield result
+
+    def mock_get_deposit_withdrawal(url):  # pylint: disable=unused-argument
+        if 'deposit' in url:
+            response_str = next(get_deposit_result)
+        else:
+            response_str = next(get_withdraw_result)
+
+        return MockResponse(200, response_str)
+
+    get_deposit_result = get_time_delta_deposit_result()
+    get_withdraw_result = get_time_delta_withdraw_result()
+
+    with patch.object(binance.session, 'get', side_effect=mock_get_deposit_withdrawal):
+        with patch.object(binance, 'api_query_dict') as mock_api_query_dict:
+            binance.query_online_deposits_withdrawals(
+                start_ts=Timestamp(start_ts),
+                end_ts=Timestamp(end_ts),
+            )
+            assert mock_api_query_dict.call_args_list == expected_calls