Repository: rotki/rotki

Issue:
Query balances of user in Binance Futures wallet
## Abstract

If a user has funds in Binance futures wallet they are not shown in Rotki.

They should be queriable by this endpoint: https://binance-docs.github.io/apidocs/spot/en/#cross-collateral-wallet-user_data

## Task

Query the binance futures wallet endpoint and if any are there include them in the user's binance balances



Discussion:

Resolving patch:
diff --git a/rotkehlchen/exchanges/binance.py b/rotkehlchen/exchanges/binance.py
--- a/rotkehlchen/exchanges/binance.py
+++ b/rotkehlchen/exchanges/binance.py
@@ -10,6 +10,7 @@
 from gevent.lock import Semaphore
 from typing_extensions import Literal
 
+from rotkehlchen.accounting.structures import Balance
 from rotkehlchen.assets.converters import asset_from_binance
 from rotkehlchen.constants import BINANCE_BASE_URL
 from rotkehlchen.constants.misc import ZERO
@@ -57,6 +58,7 @@
 V1_ENDPOINTS = (
     'exchangeInfo',
     'time',
+    'futures/loan/wallet',
 )
 
 WAPI_ENDPOINTS = (
@@ -64,6 +66,10 @@
     'withdrawHistory.html',
 )
 
+SAPI_ENDPOINTS = (
+    'futures/loan/wallet',
+)
+
 
 class BinancePair(NamedTuple):
     """A binance pair. Contains the symbol in the Binance mode e.g. "ETHBTC" and
@@ -238,8 +244,11 @@ def api_query(self, method: str, options: Optional[Dict] = None) -> Union[List,
                 # Protect this region with a lock since binance will reject
                 # non-increasing nonces. So if two greenlets come in here at
                 # the same time one of them will fail
-                if method in V3_ENDPOINTS or method in WAPI_ENDPOINTS:
-                    api_version = 3
+                if method in V3_ENDPOINTS or method in WAPI_ENDPOINTS or method in SAPI_ENDPOINTS:
+                    if method in SAPI_ENDPOINTS:
+                        api_version = 1
+                    else:
+                        api_version = 3
                     # Recommended recvWindows is 5000 but we get timeouts with it
                     call_options['recvWindow'] = 10000
                     call_options['timestamp'] = str(ts_now_in_ms() + self.offset_ms)
@@ -254,10 +263,14 @@ def api_query(self, method: str, options: Optional[Dict] = None) -> Union[List,
                 else:
                     raise ValueError('Unexpected binance api method {}'.format(method))
 
-                apistr = 'wapi/' if method in WAPI_ENDPOINTS else 'api/'
+                if method in WAPI_ENDPOINTS:
+                    apistr = 'wapi/'
+                elif method in SAPI_ENDPOINTS:
+                    apistr = 'sapi/'
+                else:
+                    apistr = 'api/'
                 request_url = f'{self.uri}{apistr}v{str(api_version)}/{method}?'
                 request_url += urlencode(call_options)
-
                 log.debug('Binance API request', request_url=request_url)
                 try:
                     response = self.session.get(request_url)
@@ -314,23 +327,8 @@ def api_query_list(self, method: str, options: Optional[Dict] = None) -> List:
         assert isinstance(result, List)
         return result
 
-    @protect_with_lock()
-    @cache_response_timewise()
-    def query_balances(self) -> Tuple[Optional[dict], str]:
-        self.first_connection()
-
-        try:
-            # account data returns a dict as per binance docs
-            account_data = self.api_query_dict('account')
-        except RemoteError as e:
-            msg = (
-                'Binance API request failed. Could not reach binance due '
-                'to {}'.format(e)
-            )
-            log.error(msg)
-            return None, msg
-
-        returned_balances = {}
+    def _query_spot_balances(self, balances: Dict) -> Dict:
+        account_data = self.api_query_dict('account')
         for entry in account_data['balances']:
             amount = entry['free'] + entry['locked']
             if amount == FVal(0):
@@ -369,21 +367,86 @@ def query_balances(self) -> Tuple[Optional[dict], str]:
             balance['amount'] = amount
             balance['usd_value'] = FVal(amount * usd_price)
 
-            if asset not in returned_balances:
-                returned_balances[asset] = balance
+            if asset not in balances:
+                balances[asset] = balance
             else:  # Some assets may appear twice in binance balance query for different locations
                 # Lending/staking for example
-                returned_balances[asset]['amount'] += balance['amount']
-                returned_balances[asset]['usd_value'] += balance['usd_value']
+                balances[asset]['amount'] += balance['amount']
+                balances[asset]['usd_value'] += balance['usd_value']
 
-            log.debug(
-                'binance balance query result',
-                sensitive_log=True,
-                asset=asset,
-                amount=amount,
-                usd_value=balance['usd_value'],
+        return balances
+
+    def _query_futures_balances(self, balances: Dict) -> Dict:
+        futures_response = self.api_query_dict('futures/loan/wallet')
+        try:
+            cross_collaterals = futures_response['crossCollaterals']
+            for entry in cross_collaterals:
+                try:
+                    asset = asset_from_binance(entry['collateralCoin'])
+                except UnsupportedAsset as e:
+                    self.msg_aggregator.add_warning(
+                        f'Found unsupported binance asset {e.asset_name}. '
+                        f' Ignoring its futures balance query.',
+                    )
+                    continue
+                except UnknownAsset as e:
+                    self.msg_aggregator.add_warning(
+                        f'Found unknown binance asset {e.asset_name}. '
+                        f' Ignoring its futures balance query.',
+                    )
+                    continue
+                except DeserializationError:
+                    self.msg_aggregator.add_error(
+                        f'Found binance asset with non-string type {type(entry["asset"])}. '
+                        f' Ignoring its futures balance query.',
+                    )
+                    continue
+                amount = FVal(entry['locked'])
+                try:
+                    usd_price = Inquirer().find_usd_price(asset)
+                except RemoteError as e:
+                    self.msg_aggregator.add_error(
+                        f'Error processing binance balance entry due to inability to '
+                        f'query USD price: {str(e)}. Skipping balance entry',
+                    )
+                    continue
+
+                balance = Balance(amount=amount, usd_value=amount * usd_price)
+                if asset not in balances:
+                    balances[asset] = balance.to_dict()
+                else:
+                    balances[asset]['amount'] += balance.amount
+                    balances[asset]['usd_value'] += balance.usd_value
+
+        except KeyError as e:
+            self.msg_aggregator.add_error(
+                f'At binance futures balance query did not find expected key '
+                f' {str(e)}. Skipping futures query ...',
+            )
+
+        return balances
+
+    @protect_with_lock()
+    @cache_response_timewise()
+    def query_balances(self) -> Tuple[Optional[Dict], str]:
+        self.first_connection()
+        returned_balances: Dict = {}
+        try:
+            returned_balances = self._query_spot_balances(returned_balances)
+            returned_balances = self._query_futures_balances(returned_balances)
+        except RemoteError as e:
+            msg = (
+                'Binance account API request failed. Could not reach binance due '
+                'to {}'.format(e)
             )
+            log.error(msg)
+            return None, msg
 
+        log.debug(
+            'binance balance query result',
+            sensitive_log=True,
+            balances=returned_balances,
+        )
         return returned_balances, ''
 
     def query_online_trade_history(


Test patch:
diff --git a/rotkehlchen/tests/api/test_async.py b/rotkehlchen/tests/api/test_async.py
--- a/rotkehlchen/tests/api/test_async.py
+++ b/rotkehlchen/tests/api/test_async.py
@@ -11,7 +11,10 @@
     assert_ok_async_response,
     assert_proper_response,
 )
-from rotkehlchen.tests.utils.exchanges import BINANCE_BALANCES_RESPONSE
+from rotkehlchen.tests.utils.exchanges import (
+    BINANCE_BALANCES_RESPONSE,
+    BINANCE_FUTURES_WALLET_RESPONSE,
+)
 from rotkehlchen.tests.utils.mock import MockResponse
 
 
@@ -31,7 +34,10 @@ def test_query_async_tasks(rotkehlchen_api_server_with_exchanges):
     binance = server.rest_api.rotkehlchen.exchange_manager.connected_exchanges['binance']
 
     def mock_binance_asset_return(url):  # pylint: disable=unused-argument
-        return MockResponse(200, BINANCE_BALANCES_RESPONSE)
+        if 'futures' in url:
+            return MockResponse(200, BINANCE_FUTURES_WALLET_RESPONSE)
+        else:
+            return MockResponse(200, BINANCE_BALANCES_RESPONSE)
 
     binance_patch = patch.object(binance.session, 'get', side_effect=mock_binance_asset_return)
 
@@ -49,38 +55,38 @@ def mock_binance_asset_return(url):  # pylint: disable=unused-argument
             "named_exchanges_balances_resource",
             name='binance',
         ), json={'async_query': True})
-    task_id = assert_ok_async_response(response)
+        task_id = assert_ok_async_response(response)
 
-    # now check that there is a task
-    response = requests.get(api_url_for(server, "asynctasksresource"))
-    assert_proper_response(response)
-    json_data = response.json()
-    assert json_data['message'] == ''
-    assert json_data['result'] == [task_id]
-
-    # now query for the task result and see it's still pending (test for task lists)
-    response = requests.get(
-        api_url_for(server, "specific_async_tasks_resource", task_id=task_id),
-    )
-    assert_proper_response(response)
-    json_data = response.json()
-    assert json_data['message'] == 'The task with id 0 is still pending'
-    assert json_data['result'] == {'status': 'pending', 'outcome': None}
+        # now check that there is a task
+        response = requests.get(api_url_for(server, "asynctasksresource"))
+        assert_proper_response(response)
+        json_data = response.json()
+        assert json_data['message'] == ''
+        assert json_data['result'] == [task_id]
 
-    while True:
-        # and now query for the task result and assert on it
+        # now query for the task result and see it's still pending (test for task lists)
         response = requests.get(
             api_url_for(server, "specific_async_tasks_resource", task_id=task_id),
         )
         assert_proper_response(response)
         json_data = response.json()
-        if json_data['result']['status'] == 'pending':
-            # context switch so that the greenlet to query balances can operate
-            gevent.sleep(1)
-        elif json_data['result']['status'] == 'completed':
-            break
-        else:
-            raise AssertionError(f"Unexpected status: {json_data['result']['status']}")
+        assert json_data['message'] == 'The task with id 0 is still pending'
+        assert json_data['result'] == {'status': 'pending', 'outcome': None}
+
+        while True:
+            # and now query for the task result and assert on it
+            response = requests.get(
+                api_url_for(server, "specific_async_tasks_resource", task_id=task_id),
+            )
+            assert_proper_response(response)
+            json_data = response.json()
+            if json_data['result']['status'] == 'pending':
+                # context switch so that the greenlet to query balances can operate
+                gevent.sleep(1)
+            elif json_data['result']['status'] == 'completed':
+                break
+            else:
+                raise AssertionError(f"Unexpected status: {json_data['result']['status']}")
 
     assert json_data['message'] == ''
     assert json_data['result']['status'] == 'completed'
diff --git a/rotkehlchen/tests/api/test_balances.py b/rotkehlchen/tests/api/test_balances.py
--- a/rotkehlchen/tests/api/test_balances.py
+++ b/rotkehlchen/tests/api/test_balances.py
@@ -244,6 +244,7 @@ def test_query_all_balances_ignore_cache(
         'query_ethereum_tokens',
         wraps=rotki.chain_manager.query_ethereum_tokens,
     )
+    original_binance_query_dict = binance.api_query_dict
     binance_query_patch = patch.object(binance, 'api_query_dict', wraps=binance.api_query_dict)
     poloniex_query_patch = patch.object(poloniex, 'api_query_dict', wraps=poloniex.api_query_dict)
 
@@ -273,7 +274,11 @@ def test_query_all_balances_ignore_cache(
             expected_data_in_db=True,
             setup=setup,
         )
-        assert all(fn.call_count == 1 for fn in function_call_counters)
+        for fn in function_call_counters:
+            if fn._mock_wraps == original_binance_query_dict:
+                assert fn.call_count == 2
+            else:
+                assert fn.call_count == 1
         full_query_etherscan_count = etherscan_mock.call_count
 
         # Query all balances second time and assert cache was used
@@ -291,7 +296,11 @@ def test_query_all_balances_ignore_cache(
             setup=setup,
         )
         msg = 'call count should stay the same since cache should have been used'
-        assert all(fn.call_count == 1 for fn in function_call_counters), msg
+        for fn in function_call_counters:
+            if fn._mock_wraps == original_binance_query_dict:
+                assert fn.call_count == 2, msg
+            else:
+                assert fn.call_count == 1, msg
         msg = 'etherscan call_count should have remained the same due to no token detection '
         assert etherscan_mock.call_count == full_query_etherscan_count, msg
 
@@ -310,7 +319,11 @@ def test_query_all_balances_ignore_cache(
             setup=setup,
         )
         msg = 'call count should increase since cache should have been ignored'
-        assert all(fn.call_count == 2 for fn in function_call_counters), msg
+        for fn in function_call_counters:
+            if fn._mock_wraps == original_binance_query_dict:
+                assert fn.call_count == 4, msg
+            else:
+                assert fn.call_count == 2, msg
         msg = 'etherscan call count should have doubled after forced token detection'
         expected_count = full_query_etherscan_count * 2 - len(ethereum_accounts)
         assert etherscan_mock.call_count == expected_count, msg
@@ -512,7 +525,7 @@ def test_multiple_balance_queries_not_concurrent(
             )
         assert eth.call_count == 1, 'eth balance query should only fire once'
         assert btc.call_count == 1, 'btc balance query should only happen once'
-        assert bn.call_count == 1, 'binance balance query should only happen once'
+        assert bn.call_count == 2, 'binance balance query should do 2 calls'
 
     assert_all_balances(
         result=outcome_all,
diff --git a/rotkehlchen/tests/api/test_ethereum_transactions.py b/rotkehlchen/tests/api/test_ethereum_transactions.py
--- a/rotkehlchen/tests/api/test_ethereum_transactions.py
+++ b/rotkehlchen/tests/api/test_ethereum_transactions.py
@@ -129,8 +129,7 @@
 def test_query_transactions(rotkehlchen_api_server):
     """Test that querying the ethereum transactions endpoint works as expected
 
-    This test uses real data. Found an ethereum address that has very few transactions
-    and hopefully won't have more. If it does we can adjust the test.
+    This test uses real data.
     """
     async_query = random.choice([False, True])
     rotki = rotkehlchen_api_server.rest_api.rotkehlchen
@@ -151,8 +150,19 @@ def test_query_transactions(rotkehlchen_api_server):
     expected_result = EXPECTED_AFB7_TXS + EXPECTED_4193_TXS
     expected_result.sort(key=lambda x: x['timestamp'])
     expected_result.reverse()
-    assert result['entries'] == expected_result
-    assert result['entries_found'] == len(expected_result)
+
+    # Make sure that all of the transactions we expect are there and in order
+    # There can be more transactions (since the address can make more)
+    # but this check ignores them
+    previous_index = 0
+    for entry in expected_result:
+        assert entry in result['entries']
+        entry_idx = result['entries'].index(entry)
+        if previous_index != 0:
+            assert entry_idx == previous_index + 1
+        previous_index = entry_idx
+
+    assert result['entries_found'] >= len(expected_result)
     assert result['entries_limit'] == FREE_ETH_TX_LIMIT
 
     # Check that transactions per address and in a specific time range can be
diff --git a/rotkehlchen/tests/api/test_exchanges.py b/rotkehlchen/tests/api/test_exchanges.py
--- a/rotkehlchen/tests/api/test_exchanges.py
+++ b/rotkehlchen/tests/api/test_exchanges.py
@@ -491,7 +491,7 @@ def test_exchange_query_balances_ignore_cache(rotkehlchen_api_server_with_exchan
         json_data = response.json()
         assert json_data['message'] == ''
         assert_binance_balances_result(json_data['result'])
-        assert bn.call_count == 1
+        assert bn.call_count == 2
         # Do the query again. Cache should be used.
         binance_patch = patch_binance_balances_query(binance)
         response = requests.get(api_url_for(
@@ -503,7 +503,7 @@ def test_exchange_query_balances_ignore_cache(rotkehlchen_api_server_with_exchan
         json_data = response.json()
         assert json_data['message'] == ''
         assert_binance_balances_result(json_data['result'])
-        assert bn.call_count == 1, 'call count should not have changed. Cache must have been used'
+        assert bn.call_count == 2, 'call count should not have changed. Cache must have been used'
         # Finally do the query and request ignoring of the cache
         binance_patch = patch_binance_balances_query(binance)
         response = requests.get(api_url_for(
@@ -515,7 +515,7 @@ def test_exchange_query_balances_ignore_cache(rotkehlchen_api_server_with_exchan
         json_data = response.json()
         assert json_data['message'] == ''
         assert_binance_balances_result(json_data['result'])
-        assert bn.call_count == 2, 'call count should have changed. Cache should have been ignored'
+        assert bn.call_count == 4, 'call count should have changed. Cache should have been ignored'
 
 
 @pytest.mark.parametrize('added_exchanges', [('binance', 'poloniex')])
diff --git a/rotkehlchen/tests/exchanges/test_binance.py b/rotkehlchen/tests/exchanges/test_binance.py
--- a/rotkehlchen/tests/exchanges/test_binance.py
+++ b/rotkehlchen/tests/exchanges/test_binance.py
@@ -17,8 +17,12 @@
 )
 from rotkehlchen.exchanges.data_structures import Location, Trade, TradeType
 from rotkehlchen.fval import FVal
-from rotkehlchen.tests.utils.constants import A_BNB, A_RDN, A_USDT, A_XMR
-from rotkehlchen.tests.utils.exchanges import BINANCE_BALANCES_RESPONSE, BINANCE_MYTRADES_RESPONSE
+from rotkehlchen.tests.utils.constants import A_BNB, A_BUSD, A_RDN, A_USDT, A_XMR
+from rotkehlchen.tests.utils.exchanges import (
+    BINANCE_BALANCES_RESPONSE,
+    BINANCE_FUTURES_WALLET_RESPONSE,
+    BINANCE_MYTRADES_RESPONSE,
+)
 from rotkehlchen.tests.utils.factories import make_api_key, make_api_secret
 from rotkehlchen.tests.utils.mock import MockResponse
 from rotkehlchen.typing import AssetMovementCategory, Timestamp
@@ -234,6 +238,31 @@ def mock_unknown_asset_return(url):  # pylint: disable=unused-argument
     assert 'unsupported binance asset ETF' in warnings[1]
 
 
+def test_binance_query_balances_include_features(function_scope_binance):
+    """Test that querying binance balances includes the futures wallet"""
+    binance = function_scope_binance
+
+    def mock_binance_response(url):  # pylint: disable=unused-argument
+        if 'futures' in url:
+            return MockResponse(200, BINANCE_FUTURES_WALLET_RESPONSE)
+        else:
+            return MockResponse(200, BINANCE_BALANCES_RESPONSE)
+
+    with patch.object(binance.session, 'get', side_effect=mock_binance_response):
+        balances, msg = binance.query_balances()
+
+    assert msg == ''
+    assert len(balances) == 3
+    assert balances[A_BTC]['amount'] == FVal('4723846.89208129')
+    assert balances[A_ETH]['amount'] == FVal('4763368.68006011')
+    assert balances[A_BUSD]['amount'] == FVal('5.82211108')
+
+    warnings = binance.msg_aggregator.consume_warnings()
+    assert len(warnings) == 2
+    assert 'unknown binance asset IDONTEXIST' in warnings[0]
+    assert 'unsupported binance asset ETF' in warnings[1]
+
+
 def test_binance_query_trade_history(function_scope_binance):
     """Test that turning a binance trade as returned by the server to our format works"""
     binance = function_scope_binance
diff --git a/rotkehlchen/tests/utils/constants.py b/rotkehlchen/tests/utils/constants.py
--- a/rotkehlchen/tests/utils/constants.py
+++ b/rotkehlchen/tests/utils/constants.py
@@ -27,6 +27,7 @@
 A_CNY = Asset('CNY')
 A_JPY = Asset('JPY')
 A_ZEC = Asset('ZEC')
+A_BUSD = Asset('BUSD')
 
 ETH_ADDRESS1 = deserialize_ethereum_address('0x5153493bB1E1642A63A098A65dD3913daBB6AE24')
 ETH_ADDRESS2 = deserialize_ethereum_address('0x4FED1fC4144c223aE3C1553be203cDFcbD38C581')
diff --git a/rotkehlchen/tests/utils/exchanges.py b/rotkehlchen/tests/utils/exchanges.py
--- a/rotkehlchen/tests/utils/exchanges.py
+++ b/rotkehlchen/tests/utils/exchanges.py
@@ -144,6 +144,31 @@
       "locked": "0.0"
 }]}"""
 
+
+BINANCE_FUTURES_WALLET_RESPONSE = """{
+    "totalCrossCollateral":"5.8238577133",
+    "totalBorrowed":"5.07000000",
+    "totalInterest":"0.0",
+    "asset": "USDT",
+    "crossCollaterals":[
+        {
+            "collateralCoin":"BUSD",
+            "locked":"5.82211108",
+            "loanAmount": "5.07",
+            "currentCollateralRate": "0.87168984",
+            "principalForInterest": "0.0",
+            "interest": "0.0"
+        }, {
+            "collateralCoin":"BTC",
+            "locked":"0",
+            "loanAmount": "0",
+            "currentCollateralRate": "0",
+            "principalForInterest": "0.0",
+            "interest": "0.0"
+        }
+    ]
+}"""
+
 BINANCE_MYTRADES_RESPONSE = """
 [
     {