Repository: rotki/rotki

Issue:
With a big number of BTC accounts the balance query gets stuck
## Problem Definition

If a big number of BTC accounts is given the balance query from blockchain.info seems to get stuck in a 429 wait retry loop. It's due to a prolonged ban.

![2020-04-28-184735_902x324_scrot](https://user-images.githubusercontent.com/1658405/80514414-c3adfc00-8980-11ea-8faf-c3f982339c85.png)

## Task

Offer alternative way of querying the balances. Perhaps the multiaddress from here: https://www.blockchain.com/api/blockchain_api or the blockcypher API: https://www.blockcypher.com/dev/bitcoin/#introduction



Discussion:

Resolving patch:
diff --git a/rotkehlchen/chain/manager.py b/rotkehlchen/chain/manager.py
--- a/rotkehlchen/chain/manager.py
+++ b/rotkehlchen/chain/manager.py
@@ -34,7 +34,7 @@
     cache_response_timewise,
     protect_with_lock,
 )
-from rotkehlchen.utils.misc import request_get_dict, request_get_direct, satoshis_to_btc
+from rotkehlchen.utils.misc import request_get, request_get_dict, satoshis_to_btc
 
 if TYPE_CHECKING:
     from rotkehlchen.chain.ethereum.manager import EthereumManager
@@ -223,31 +223,70 @@ def query_balances(
         return self.get_balances_update()
 
     @staticmethod
-    def query_btc_account_balance(account: BTCAddress) -> FVal:
+    def query_btc_accounts_balances(accounts: List[BTCAddress]) -> Dict[BTCAddress, FVal]:
         """Queries blockchain.info for the balance of account
 
         May raise:
         - RemotError if there is a problem querying blockchain.info or blockcypher
         """
+        source = 'blockchain.info'
+        balances = {}
         try:
-            if account.lower()[0:3] == 'bc1':
-                url = f'https://api.blockcypher.com/v1/btc/main/addrs/{account.lower()}/balance'
-                response_data = request_get_dict(url=url)
-                if 'balance' not in response_data:
-                    raise RemoteError(f'Unexpected blockcypher balance response: {response_data}')
-                btc_resp = response_data['balance']
+            if any(account.lower()[0:3] == 'bc1' for account in accounts):
+                # if 1 account is bech32 we have to query blockcypher. blockchaininfo won't work
+                source = 'blockcypher.com'
+                # the bech32 accounts have to be given lowercase to the
+                # blockcypher query. No idea why.
+                new_accounts = []
+                for x in accounts:
+                    lowered = x.lower()
+                    if lowered[0:3] == 'bc1':
+                        new_accounts.append(lowered)
+                    else:
+                        new_accounts.append(x)
+
+                # blockcypher's batching takes up as many api queries as the batch,
+                # and the api rate limit is 3 requests per second. So we should make
+                # sure each batch is of max size 3
+                # https://www.blockcypher.com/dev/bitcoin/#batching
+                batches = [new_accounts[x: x + 3] for x in range(0, len(new_accounts), 3)]
+                total_idx = 0
+                for batch in batches:
+                    params = ';'.join(batch)
+                    url = f'https://api.blockcypher.com/v1/btc/main/addrs/{params}/balance'
+                    response_data = request_get(url=url, handle_429=True, backoff_in_seconds=4)
+
+                    if isinstance(response_data, dict):
+                        # If only one account was requested put it in a list so the
+                        # rest of the code works
+                        response_data = [response_data]
+
+                    for idx, entry in enumerate(response_data):
+                        # we don't use the returned address as it may be lowercased
+                        balances[accounts[total_idx + idx]] = satoshis_to_btc(
+                            FVal(entry['final_balance']),
+                        )
+                    total_idx += len(batch)
             else:
-                btc_resp = request_get_direct(
-                    url='https://blockchain.info/q/addressbalance/%s' % account,
+                params = '|'.join(accounts)
+                btc_resp = request_get_dict(
+                    url=f'https://blockchain.info/multiaddr?active={params}',
                     handle_429=True,
                     # If we get a 429 then their docs suggest 10 seconds
                     # https://blockchain.info/q
                     backoff_in_seconds=10,
                 )
+                for idx, entry in enumerate(btc_resp['addresses']):
+                    balances[accounts[idx]] = satoshis_to_btc(FVal(entry['final_balance']))
         except (requests.exceptions.ConnectionError, UnableToDecryptRemoteData) as e:
             raise RemoteError(f'bitcoin external API request failed due to {str(e)}')
+        except KeyError as e:
+            raise RemoteError(
+                f'Malformed response when querying bitcoin blockchain via {source}.'
+                f'Did not find key {e}',
+            )
 
-        return satoshis_to_btc(FVal(btc_resp))  # result is in satoshis
+        return balances
 
     def query_btc_balances(self) -> None:
         """Queries blockchain.info for the balance of all BTC accounts
@@ -261,15 +300,13 @@ def query_btc_balances(self) -> None:
         self.balances.btc = {}
         btc_usd_price = Inquirer().find_usd_price(A_BTC)
         total = FVal(0)
-        for account in self.accounts.btc:
-            balance = self.query_btc_account_balance(account)
-
+        balances = self.query_btc_accounts_balances(self.accounts.btc)
+        for account, balance in balances.items():
             total += balance
             self.balances.btc[account] = Balance(
                 amount=balance,
                 usd_value=balance * btc_usd_price,
             )
-
         self.totals[A_BTC] = Balance(amount=total, usd_value=total * btc_usd_price)
 
     @overload
@@ -395,7 +432,8 @@ def modify_btc_account(
         # Query the balance of the account except for the case when it's removed
         # and there is no other account in the balances
         if append_or_remove == 'append' or remove_with_populated_balance:
-            balance = self.query_btc_account_balance(account)
+            balances = self.query_btc_accounts_balances([account])
+            balance = balances[account]
             usd_balance = balance * btc_usd_price
 
         if append_or_remove == 'append':
diff --git a/rotkehlchen/utils/misc.py b/rotkehlchen/utils/misc.py
--- a/rotkehlchen/utils/misc.py
+++ b/rotkehlchen/utils/misc.py
@@ -163,11 +163,17 @@ def retry_calls(
 
             if handle_429:
                 if result.status_code == HTTPStatus.TOO_MANY_REQUESTS and tries != 0:
+                    if tries == 0:
+                        raise RemoteError(
+                            f"{location} query for {method_name} failed after {times} tries",
+                        )
+
                     log.debug(
                         f'In retry_call for {location}-{method_name}. Got 429. Backing off for '
                         f'{backoff_in_seconds} seconds',
                     )
                     gevent.sleep(backoff_in_seconds)
+                    tries -= 1
                     continue
             return result
 


Test patch:
diff --git a/rotkehlchen/tests/api/test_blockchain.py b/rotkehlchen/tests/api/test_blockchain.py
--- a/rotkehlchen/tests/api/test_blockchain.py
+++ b/rotkehlchen/tests/api/test_blockchain.py
@@ -167,6 +167,51 @@ def test_query_blockchain_balances(
     )
 
 
+@pytest.mark.parametrize('number_of_eth_accounts', [0])
+@pytest.mark.parametrize('btc_accounts', [[
+    UNIT_BTC_ADDRESS1,
+    UNIT_BTC_ADDRESS2,
+    'bc1qhkje0xfvhmgk6mvanxwy09n45df03tj3h3jtnf',
+]])
+@pytest.mark.parametrize('owned_eth_tokens', [[]])
+def test_query_bitcoin_blockchain_bech32_balances(
+        rotkehlchen_api_server,
+        ethereum_accounts,
+        btc_accounts,
+        caplog,
+):
+    """Test that querying Bech32 bitcoin addresses works fine"""
+    caplog.set_level(logging.DEBUG)
+    # Disable caching of query results
+    rotki = rotkehlchen_api_server.rest_api.rotkehlchen
+    rotki.chain_manager.cache_ttl_secs = 0
+
+    btc_balances = ['111110', '3232223', '555555333']
+    setup = setup_balances(
+        rotki,
+        ethereum_accounts=ethereum_accounts,
+        btc_accounts=btc_accounts,
+        btc_balances=btc_balances,
+    )
+
+    # query all balances
+    with ExitStack() as stack:
+        setup.enter_blockchain_patches(stack)
+        response = requests.get(api_url_for(
+            rotkehlchen_api_server,
+            "blockchainbalancesresource",
+        ))
+    assert_proper_response(response)
+    json_data = response.json()
+    assert json_data['message'] == ''
+    assert_btc_balances_result(
+        json_data=json_data,
+        btc_accounts=btc_accounts,
+        btc_balances=setup.btc_balances,
+        also_eth=False,
+    )
+
+
 @pytest.mark.parametrize('number_of_eth_accounts', [2])
 @pytest.mark.parametrize('btc_accounts', [[UNIT_BTC_ADDRESS1, UNIT_BTC_ADDRESS2]])
 @pytest.mark.parametrize('owned_eth_tokens', [[A_RDN]])
diff --git a/rotkehlchen/tests/utils/blockchain.py b/rotkehlchen/tests/utils/blockchain.py
--- a/rotkehlchen/tests/utils/blockchain.py
+++ b/rotkehlchen/tests/utils/blockchain.py
@@ -505,9 +505,29 @@ def mock_bitcoin_balances_query(
 
     def mock_requests_get(url, *args, **kwargs):
         if 'blockchain.info' in url:
-            queried_addr = url.split('/')[-1]
-            response = btc_map.get(queried_addr, '0')
+            addresses = url.split('multiaddr?active=')[1].split('|')
+            response = '{"addresses":['
+            for idx, address in enumerate(addresses):
+                balance = btc_map.get(address, '0')
+                response += f'{{"address":"{address}", "final_balance":{balance}}}'
+                if idx < len(addresses) - 1:
+                    response += ','
+            response += ']}'
+        elif 'blockcypher.com' in url:
+            addresses = url.split('addrs/')[1].split('/balance')[0].split(';')
+            response = '['
+            for idx, address in enumerate(addresses):
+                balance = btc_map.get(address, '0')
+                entry = f'{{"address": "{address}", "final_balance": {balance}}}'
+                if len(addresses) == 1:
+                    response = entry
+                else:
+                    response += entry
+                if idx < len(addresses) - 1:
+                    response += ','
 
+            if len(addresses) > 1:
+                response += ']'
         else:
             return original_requests_get(url, *args, **kwargs)