Repository: rotki/rotki

Issue:
Migrate binance WAPI endpoints
Binance is making changes in their API and wapi endpoints are loosing support. We have to migrate to SAPI endpoints providing similar information.

https://www.binance.com/en/support/announcement/f45dde7da58b473aa885349946bed269

# Task

Do it!


Discussion:
Is this related to https://github.com/rotki/rotki/issues/2874?
@LefterisJP I discovered this migration looking for causes for that issue. The migration on the binance side is not finished and don't know if it is the reason for that error.

Resolving patch:
diff --git a/rotkehlchen/exchanges/binance.py b/rotkehlchen/exchanges/binance.py
--- a/rotkehlchen/exchanges/binance.py
+++ b/rotkehlchen/exchanges/binance.py
@@ -40,6 +40,7 @@
     deserialize_asset_amount_force_positive,
     deserialize_fee,
     deserialize_timestamp_from_binance,
+    deserialize_timestamp_from_date,
 )
 from rotkehlchen.typing import ApiKey, ApiSecret, AssetMovementCategory, Fee, Location, Timestamp
 from rotkehlchen.user_messages import MessagesAggregator
@@ -57,30 +58,24 @@
 # https://www.binance.com/en/support/articles/115000599831-Binance-Exchange-Launched-Date-Set
 BINANCE_LAUNCH_TS = Timestamp(1500001200)
 API_TIME_INTERVAL_CONSTRAINT_TS = Timestamp(7776000)  # 90 days
+
 V3_METHODS = (
     'account',
     'myTrades',
     'openOrders',
-)
-
-V1_METHODS = (
     'exchangeInfo',
     'time',
 )
-
-WAPI_METHODS = (
-    'depositHistory.html',
-    'withdrawHistory.html',
-)
-
+PUBLIC_METHODS = ('exchangeInfo', 'time')
 SAPI_METHODS = (
     'futures/loan/wallet',
     'lending/daily/token/position',
     'lending/daily/product/list',
     'lending/union/account',
     'bswap/liquidity',
+    'capital/deposit/hisrec',
+    'capital/withdraw/history',
 )
-
 FAPI_METHODS = (
     'balance',
     'account',
@@ -94,7 +89,7 @@
 REJECTED_MBX_KEY = -2015
 
 
-BINANCE_API_TYPE = Literal['api', 'sapi', 'wapi', 'dapi', 'fapi']
+BINANCE_API_TYPE = Literal['api', 'sapi', 'dapi', 'fapi']
 
 BINANCE_BASE_URL = 'binance.com/'
 BINANCEUS_BASE_URL = 'binance.us/'
@@ -317,19 +312,13 @@ def api_query(
 
             is_v3_api_method = api_type == 'api' and method in V3_METHODS
             is_new_futures_api = api_type in ('fapi', 'dapi')
-            call_needs_signature = (
-                (api_type == 'fapi' and method in FAPI_METHODS) or
-                (api_type == 'dapi' and method in FAPI_METHODS) or  # same as fapi
-                (api_type == 'sapi' and method in SAPI_METHODS) or
-                (api_type == 'wapi' and method in WAPI_METHODS) or
-                is_v3_api_method
-            )
-            if call_needs_signature:
+            api_version = 3  # public methos are v3
+            if method not in PUBLIC_METHODS:  # api call needs signature
                 if api_type in ('sapi', 'dapi'):
                     api_version = 1
                 elif api_type == 'fapi':
                     api_version = 2
-                elif api_type == 'wapi' or is_v3_api_method:
+                elif is_v3_api_method:
                     api_version = 3
                 else:
                     raise AssertionError(
@@ -346,10 +335,6 @@ def api_query(
                     hashlib.sha256,
                 ).hexdigest()
                 call_options['signature'] = signature
-            elif api_type == 'api' and method in V1_METHODS:
-                api_version = 1
-            else:
-                raise AssertionError(f'Unexpected {self.name} API method {method}')
 
             api_subdomain = api_type if is_new_futures_api else 'api'
             request_url = (
@@ -929,15 +914,19 @@ def _deserialize_asset_movement(self, raw_data: Dict[str, Any]) -> Optional[Asse
         try:
             if 'insertTime' in raw_data:
                 category = AssetMovementCategory.DEPOSIT
-                time_key = 'insertTime'
+                timestamp = deserialize_timestamp_from_binance(raw_data['insertTime'])
                 fee = Fee(ZERO)
             else:
                 category = AssetMovementCategory.WITHDRAWAL
-                time_key = 'applyTime'
+                timestamp = deserialize_timestamp_from_date(
+                    date=raw_data['applyTime'],
+                    formatstr='%Y-%m-%d %H:%M:%S',
+                    location='binance withdrawal',
+                    skip_milliseconds=True,
+                )
                 fee = Fee(deserialize_asset_amount(raw_data['transactionFee']))
 
-            timestamp = deserialize_timestamp_from_binance(raw_data[time_key])
-            asset = asset_from_binance(raw_data['asset'])
+            asset = asset_from_binance(raw_data['coin'])
             tx_id = get_key_if_has_val(raw_data, 'txId')
             internal_id = get_key_if_has_val(raw_data, 'id')
             link_str = str(internal_id) if internal_id else str(tx_id) if tx_id else ''
@@ -980,13 +969,13 @@ def _deserialize_asset_movement(self, raw_data: Dict[str, Any]) -> Optional[Asse
 
         return None
 
-    def _api_query_dict_within_time_delta(
+    def _api_query_list_within_time_delta(
             self,
             start_ts: Timestamp,
             end_ts: Timestamp,
             time_delta: Timestamp,
             api_type: BINANCE_API_TYPE,
-            method: Literal['depositHistory.html', 'withdrawHistory.html'],
+            method: Literal['capital/deposit/hisrec', 'capital/withdraw/history'],
     ) -> List[Dict[str, Any]]:
         """Request via `api_query_dict()` from `start_ts` `end_ts` using a time
         delta (offset) less than `time_delta`.
@@ -997,13 +986,6 @@ def _api_query_dict_within_time_delta(
           `used_query_ranges` table, but 0.
           - Timestamps are converted to milliseconds.
         """
-        if method == 'depositHistory.html':
-            query_schema = 'depositList'
-        elif method == 'withdrawHistory.html':
-            query_schema = 'withdrawList'
-        else:
-            raise AssertionError(f'Unexpected {self.name} method case: {method}.')
-
         results: List[Dict[str, Any]] = []
 
         # Create required time references in milliseconds
@@ -1026,8 +1008,8 @@ def _api_query_dict_within_time_delta(
                 'startTime': from_ts,
                 'endTime': to_ts,
             }
-            result = self.api_query_dict(api_type, method, options=options)
-            results.extend(result.get(query_schema, []))
+            result = self.api_query_list(api_type, method, options=options)
+            results.extend(result)
             # Case stop requesting
             if to_ts >= end_ts:
                 break
@@ -1051,21 +1033,21 @@ def query_online_deposits_withdrawals(
         https://binance-docs.github.io/apidocs/spot/en/#deposit-history-user_data
         https://binance-docs.github.io/apidocs/spot/en/#withdraw-history-user_data
         """
-        deposits = self._api_query_dict_within_time_delta(
+        deposits = self._api_query_list_within_time_delta(
             start_ts=start_ts,
             end_ts=end_ts,
             time_delta=API_TIME_INTERVAL_CONSTRAINT_TS,
-            api_type='wapi',
-            method='depositHistory.html',
+            api_type='sapi',
+            method='capital/deposit/hisrec',
         )
         log.debug(f'{self.name} deposit history result', results_num=len(deposits))
 
-        withdraws = self._api_query_dict_within_time_delta(
+        withdraws = self._api_query_list_within_time_delta(
             start_ts=start_ts,
             end_ts=end_ts,
             time_delta=API_TIME_INTERVAL_CONSTRAINT_TS,
-            api_type='wapi',
-            method='withdrawHistory.html',
+            api_type='sapi',
+            method='capital/withdraw/history',
         )
         log.debug(f'{self.name} withdraw history result', results_num=len(withdraws))
 


Test patch:
diff --git a/rotkehlchen/tests/exchanges/test_binance.py b/rotkehlchen/tests/exchanges/test_binance.py
--- a/rotkehlchen/tests/exchanges/test_binance.py
+++ b/rotkehlchen/tests/exchanges/test_binance.py
@@ -445,7 +445,7 @@ def check_permutations_of_input_invalid_data(deposits, withdrawals):
             new_withdrawals = withdrawals
         else:
             new_deposits = deposits
-            new_withdrawals = withdrawals.replace('1508198532000', '"dasdd"')
+            new_withdrawals = withdrawals.replace('"2017-10-17 00:02:12"', '"dasdd"')
         mock_binance_and_query(
             new_deposits,
             new_withdrawals,
@@ -525,37 +525,31 @@ def check_permutations_of_input_invalid_data(deposits, withdrawals):
 
     # To make the test easy to write the values for deposit/withdrawal attributes
     # are the same in the two examples below
-    empty_deposits = '{"success": true, "depositList": []}'
-    empty_withdrawals = '{"success": true, "withdrawList": []}'
-    input_withdrawals = """{
-    "success": true,
-    "withdrawList": [
-        {
-            "id":"7213fea8e94b4a5593d507237e5a555b",
-            "amount": 0.04670582,
-            "transactionFee": 0.01,
-            "address": "0x6915f16f8791d0a1cc2bf47c13a6b2a92000504b",
-            "asset": "ETH",
-            "txId": "0xdf33b22bdb2b28b1f75ccd201a4a4m6e7g83jy5fc5d5a9d1340961598cfcb0a1",
-            "applyTime": 1508198532000,
-            "status": 4
-        }]}"""
+    empty_deposits = '[]'
+    empty_withdrawals = '[]'
+    input_withdrawals = """[{
+        "id":"7213fea8e94b4a5593d507237e5a555b",
+        "amount": 0.04670582,
+        "transactionFee": 0.01,
+        "address": "0x6915f16f8791d0a1cc2bf47c13a6b2a92000504b",
+        "coin": "ETH",
+        "txId": "0xdf33b22bdb2b28b1f75ccd201a4a4m6e7g83jy5fc5d5a9d1340961598cfcb0a1",
+        "applyTime": "2017-10-17 00:02:12",
+        "status": 4
+        }]"""
     check_permutations_of_input_invalid_data(
         deposits=empty_deposits,
         withdrawals=input_withdrawals,
     )
 
-    input_deposits = """{
-    "success": true,
-    "depositList": [
-        {
-            "insertTime": 1508198532000,
-            "amount": 0.04670582,
-            "asset": "ETH",
-            "address": "0x6915f16f8791d0a1cc2bf47c13a6b2a92000504b",
-            "txId": "0xdf33b22bdb2b28b1f75ccd201a4a4m6e7g83jy5fc5d5a9d1340961598cfcb0a1",
-            "status": 1
-        }]}"""
+    input_deposits = """[{
+        "insertTime": 1508198532000,
+        "amount": 0.04670582,
+        "coin": "ETH",
+        "address": "0x6915f16f8791d0a1cc2bf47c13a6b2a92000504b",
+        "txId": "0xdf33b22bdb2b28b1f75ccd201a4a4m6e7g83jy5fc5d5a9d1340961598cfcb0a1",
+        "status": 1
+        }]"""
     check_permutations_of_input_invalid_data(
         deposits=input_deposits,
         withdrawals=empty_withdrawals,
@@ -580,14 +574,14 @@ def test_binance_query_deposits_withdrawals_gte_90_days(function_scope_binance):
     def get_time_delta_deposit_result():
         results = [
             BINANCE_DEPOSITS_HISTORY_RESPONSE,
-            '{}',
+            '[]',
         ]
         for result in results:
             yield result
 
     def get_time_delta_withdraw_result():
         results = [
-            '{}',
+            '[]',
             BINANCE_WITHDRAWALS_HISTORY_RESPONSE,
         ]
         for result in results:
@@ -619,8 +613,8 @@ def mock_get_deposit_withdrawal(url, **kwargs):  # pylint: disable=unused-argume
 
 
 @pytest.mark.freeze_time(datetime(2020, 11, 24, 3, 14, 15))
-def test_api_query_dict_calls_with_time_delta(function_scope_binance):
-    """Test the `api_query_dict()` arguments when deposit/withdraw history
+def test_api_query_list_calls_with_time_delta(function_scope_binance):
+    """Test the `api_query_list()` arguments when deposit/withdraw history
     requests involve a time delta.
 
     From `start_ts` to `end_ts` there is a difference gte 90 days, which forces
@@ -631,8 +625,8 @@ def test_api_query_dict_calls_with_time_delta(function_scope_binance):
     end_ts = BINANCE_LAUNCH_TS + API_TIME_INTERVAL_CONSTRAINT_TS  # eq 90 days after
     expected_calls = [
         call(
-            'wapi',
-            'depositHistory.html',
+            'sapi',
+            'capital/deposit/hisrec',
             options={
                 'timestamp': now_ts_ms,
                 'startTime': 1500001200000,
@@ -640,8 +634,8 @@ def test_api_query_dict_calls_with_time_delta(function_scope_binance):
             },
         ),
         call(
-            'wapi',
-            'depositHistory.html',
+            'sapi',
+            'capital/deposit/hisrec',
             options={
                 'timestamp': now_ts_ms,
                 'startTime': 1507777200000,
@@ -649,8 +643,8 @@ def test_api_query_dict_calls_with_time_delta(function_scope_binance):
             },
         ),
         call(
-            'wapi',
-            'withdrawHistory.html',
+            'sapi',
+            'capital/withdraw/history',
             options={
                 'timestamp': now_ts_ms,
                 'startTime': 1500001200000,
@@ -658,8 +652,8 @@ def test_api_query_dict_calls_with_time_delta(function_scope_binance):
             },
         ),
         call(
-            'wapi',
-            'withdrawHistory.html',
+            'sapi',
+            'capital/withdraw/history',
             options={
                 'timestamp': now_ts_ms,
                 'startTime': 1507777200000,
@@ -672,14 +666,14 @@ def test_api_query_dict_calls_with_time_delta(function_scope_binance):
     def get_time_delta_deposit_result():
         results = [
             BINANCE_DEPOSITS_HISTORY_RESPONSE,
-            '{}',
+            '[]',
         ]
         for result in results:
             yield result
 
     def get_time_delta_withdraw_result():
         results = [
-            '{}',
+            '[]',
             BINANCE_WITHDRAWALS_HISTORY_RESPONSE,
         ]
         for result in results:
@@ -697,12 +691,12 @@ def mock_get_deposit_withdrawal(url, **kwargs):  # pylint: disable=unused-argume
     get_withdraw_result = get_time_delta_withdraw_result()
 
     with patch.object(binance.session, 'get', side_effect=mock_get_deposit_withdrawal):
-        with patch.object(binance, 'api_query_dict') as mock_api_query_dict:
+        with patch.object(binance, 'api_query_list') as mock_api_query_list:
             binance.query_online_deposits_withdrawals(
                 start_ts=Timestamp(start_ts),
                 end_ts=Timestamp(end_ts),
             )
-            assert mock_api_query_dict.call_args_list == expected_calls
+            assert mock_api_query_list.call_args_list == expected_calls
 
 
 @pytest.mark.freeze_time(datetime(2020, 11, 24, 3, 14, 15))
diff --git a/rotkehlchen/tests/utils/exchanges.py b/rotkehlchen/tests/utils/exchanges.py
--- a/rotkehlchen/tests/utils/exchanges.py
+++ b/rotkehlchen/tests/utils/exchanges.py
@@ -292,57 +292,49 @@
     "isBestMatch": true
     }]"""
 
-BINANCE_DEPOSITS_HISTORY_RESPONSE = """{
-    "depositList": [
-        {
-            "insertTime": 1508198532000,
-            "amount": 0.04670582,
-            "asset": "ETH",
-            "address": "0x6915f16f8791d0a1cc2bf47c13a6b2a92000504b",
-            "txId": "0xef33b22bdb2b28b1f75ccd201a4a4m6e7g83jy5fc5d5a9d1340961598cfcb0a1",
-            "status": 1
-        },
-        {
-            "insertTime": 1508398632000,
-            "amount": 1000,
-            "asset": "XMR",
-            "address": "463tWEBn5XZJSxLU34r6g7h8jtxuNcDbjLSjkn3XAXHCbLrTTErJrBWYgHJQyrCwkNgYvV38",
-            "addressTag": "342341222",
-            "txId": "c3c6219639c8ae3f9cf010cdc24fw7f7yt8j1e063f9b4bd1a05cb44c4b6e2509",
-            "status": 1
-        }
-    ],
-    "success": true
-}"""
+BINANCE_DEPOSITS_HISTORY_RESPONSE = """[
+    {
+        "insertTime": 1508198532000,
+        "amount": 0.04670582,
+        "coin": "ETH",
+        "address": "0x6915f16f8791d0a1cc2bf47c13a6b2a92000504b",
+        "txId": "0xef33b22bdb2b28b1f75ccd201a4a4m6e7g83jy5fc5d5a9d1340961598cfcb0a1",
+        "status": 1
+    }, {
+        "insertTime": 1508398632000,
+        "amount": 1000,
+        "coin": "XMR",
+        "address": "463tWEBn5XZJSxLU34r6g7h8jtxuNcDbjLSjkn3XAXHCbLrTTErJrBWYgHJQyrCwkNgYvV38",
+        "addressTag": "342341222",
+        "txId": "c3c6219639c8ae3f9cf010cdc24fw7f7yt8j1e063f9b4bd1a05cb44c4b6e2509",
+        "status": 1
+    }
+]"""
 
-BINANCE_WITHDRAWALS_HISTORY_RESPONSE = """{
-    "withdrawList": [
-        {
-            "id":"7213fea8e94b4a5593d507237e5a555b",
-            "withdrawOrderId": null,
-            "amount": 0.99,
-            "transactionFee": 0.01,
-            "address": "0x6915f16f8791d0a1cc2bf47c13a6b2a92000504b",
-            "asset": "ETH",
-            "txId": "0xdf33b22bdb2b28b1f75ccd201a4a4m6e7g83jy5fc5d5a9d1340961598cfcb0a1",
-            "applyTime": 1508198532000,
-            "status": 4
-        },
+BINANCE_WITHDRAWALS_HISTORY_RESPONSE = """[
         {
-            "id":"7213fea8e94b4a5534ggsd237e5a555b",
-            "withdrawOrderId": "withdrawtest",
-            "amount": 999.9999,
-            "transactionFee": 0.0001,
-            "address": "463tWEBn5XZJSxLU34r6g7h8jtxuNcDbjLSjkn3XAXHCbLrTTErJrBWYgHJQyrCwkNgYvyV3z8zctJLPCZy24jvb3NiTcTJ",
-            "addressTag": "342341222",
-            "txId": "b3c6219639c8ae3f9cf010cdc24fw7f7yt8j1e063f9b4bd1a05cb44c4b6e2509",
-            "asset": "XMR",
-            "applyTime": 1508198532000,
-            "status": 4
-        }
-    ],
-    "success": true
-}"""  # noqa: E501
+        "id":"7213fea8e94b4a5593d507237e5a555b",
+        "withdrawOrderId": null,
+        "amount": 0.99,
+        "transactionFee": 0.01,
+        "address": "0x6915f16f8791d0a1cc2bf47c13a6b2a92000504b",
+        "coin": "ETH",
+        "txId": "0xdf33b22bdb2b28b1f75ccd201a4a4m6e7g83jy5fc5d5a9d1340961598cfcb0a1",
+        "applyTime": "2017-10-17 00:02:12",
+        "status": 4
+    }, {
+        "id":"7213fea8e94b4a5534ggsd237e5a555b",
+        "withdrawOrderId": "withdrawtest",
+        "amount": 999.9999,
+        "transactionFee": 0.0001,
+        "address": "463tWEBn5XZJSxLU34r6g7h8jtxuNcDbjLSjkn3XAXHCbLrTTErJrBWYgHJQyrCwkNgYvyV3z8zctJLPCZy24jvb3NiTcTJ",
+        "addressTag": "342341222",
+        "txId": "b3c6219639c8ae3f9cf010cdc24fw7f7yt8j1e063f9b4bd1a05cb44c4b6e2509",
+        "coin": "XMR",
+        "applyTime": "2017-10-17 00:02:12",
+        "status": 4
+    }
+]"""  # noqa: E501
 
 
 def assert_binance_balances_result(balances: Dict[str, Any]) -> None:
diff --git a/rotkehlchen/tests/utils/history.py b/rotkehlchen/tests/utils/history.py
--- a/rotkehlchen/tests/utils/history.py
+++ b/rotkehlchen/tests/utils/history.py
@@ -218,10 +218,10 @@ def mock_binance_api_queries(url, timeout):  # pylint: disable=unused-argument
                 "isMaker": false,
                 "isBestMatch": true
                 }]"""
-        elif 'depositHistory.html' in url:
-            payload = '{"success": true, "depositList": []}'
-        elif 'withdrawHistory.html' in url:
-            payload = '{"success": true, "withdrawList": []}'
+        elif 'capital/deposit' in url:
+            payload = '[]'
+        elif 'capital/withdraw' in url:
+            payload = '[]'
         else:
             raise RuntimeError(f'Binance test mock got unexpected/unmocked url {url}')