Repository: rotki/rotki

Issue:
Switch to a different Bitcoin balance query api if blockchain.info fails
## Abstract

A user has reported that suddenly their bitcoin xpub balances are shown as 0. After inspection of the logs it seems that the blockchain.info call failed with invalid json.

Logs:

```
ERROR rotkehlchen.rotkehlchen: Querying blockchain balances failed due to: bitcoin external API request for balances failed due to https://blockchain.info/multiaddr?active=XXXXX|YYYYY|ZZZZZ....|FFFFF returned malformed json. Error: Expecting value: line 1 column 1 (char 0)
```

The user queried 170 addresses in 1 call. Which should not be a problem under normal circumstances if I recall. Also the docs don't mention it as a problem.

Trying it from the browswer you can a 502 error as of the writing of this issue.

![2020-11-27_23-38](https://user-images.githubusercontent.com/1658405/100487674-2d0a0f00-310a-11eb-9192-d76f865c611b.png)


## Task

Offer an alternative call via the blockstream api if the blockchain.info api call fails.


Discussion:
Is this still relevant or rather vice versa? In my rotki debug logs I can only see calls to blockstream.info.
@kilrau yes I think it still is. But perhaps should be renamed to generally try many sources when one fails. The reason you only see blockstream.info is that blockchain.info does not accept segwit bc1 addresses and I assume all your addresses, being a modern BTC user, are of that type.
True that. You really don't want to use blockchain.info as anything these days ;)

Mempool.space is another great service, never used their API but looks like the address endpoint has everything you'll need: https://mempool.space/api

Resolving patch:
diff --git a/rotkehlchen/chain/bitcoin/__init__.py b/rotkehlchen/chain/bitcoin/__init__.py
--- a/rotkehlchen/chain/bitcoin/__init__.py
+++ b/rotkehlchen/chain/bitcoin/__init__.py
@@ -3,7 +3,9 @@
 import requests
 
 from rotkehlchen.errors.misc import RemoteError, UnableToDecryptRemoteData
+from rotkehlchen.errors.serialization import DeserializationError
 from rotkehlchen.fval import FVal
+from rotkehlchen.serialization.deserialize import ensure_type
 from rotkehlchen.types import BTCAddress
 from rotkehlchen.utils.misc import satoshis_to_btc
 from rotkehlchen.utils.network import request_get_dict
@@ -13,65 +15,152 @@ def _have_bc1_accounts(accounts: List[BTCAddress]) -> bool:
     return any(account.lower()[0:3] == 'bc1' for account in accounts)
 
 
-def get_bitcoin_addresses_balances(accounts: List[BTCAddress]) -> Dict[BTCAddress, FVal]:
-    """Queries blockchain.info or blockstream for the balances of accounts
+def _query_blockstream_or_mempool(
+        accounts: List[BTCAddress],
+        base_url: str,
+) -> Dict[BTCAddress, FVal]:
+    """Queries balances from blockstream.info
+    May raise:
+    - RemoteError if got problems with querying the API
+    - KeyError if got unexpected json structure
+    - DeserializationError if got unexpected json values
+    """
+    balances = {}
+    for account in accounts:
+        url = base_url + account
+        response_data = request_get_dict(url=url, handle_429=True, backoff_in_seconds=4)
+        stats = response_data['chain_stats']
+        funded_txo_sum = satoshis_to_btc(
+            ensure_type(
+                symbol=stats['funded_txo_sum'],
+                expected_type=int,
+                location='blockstream funded_txo_sum',
+            ),
+        )
+        spent_txo_sum = satoshis_to_btc(
+            ensure_type(
+                symbol=stats['spent_txo_sum'],
+                expected_type=int,
+                location='blockstream spent_txo_sum',
+            ),
+        )
+        balance = funded_txo_sum - spent_txo_sum
+        balances[account] = balance
+    return balances
 
+
+def _query_blockstream_info(accounts: List[BTCAddress]) -> Dict[BTCAddress, FVal]:
+    return _query_blockstream_or_mempool(
+        accounts=accounts,
+        base_url='https://blockstream.info/api/address/',
+    )
+
+
+def _query_mempool_space(accounts: List[BTCAddress]) -> Dict[BTCAddress, FVal]:
+    return _query_blockstream_or_mempool(
+        accounts=accounts,
+        base_url='https://mempool.space/api/address/',
+    )
+
+
+def _query_blockchain_info(accounts: List[BTCAddress]) -> Dict[BTCAddress, FVal]:
+    """Queries balances from blockchain.info
     May raise:
-    - RemotError if there is a problem querying blockchain.info or blockstream
+    - RemoteError if got problems with querying the API
+    - KeyError if got unexpected json structure
+    - DeserializationError if got unexpected json values
     """
-    source = 'blockchain.info'
     balances: Dict[BTCAddress, FVal] = {}
-    try:
-        if _have_bc1_accounts(accounts):
-            # if 1 account is bech32 we have to query blockstream. blockchaininfo won't work
-            source = 'blockstream'
-            balances = {}
-            for account in accounts:
-                url = f'https://blockstream.info/api/address/{account}'
-                response_data = request_get_dict(url=url, handle_429=True, backoff_in_seconds=4)
-                stats = response_data['chain_stats']
-                balance = int(stats['funded_txo_sum']) - int(stats['spent_txo_sum'])
-                balances[account] = satoshis_to_btc(balance)
-        else:
-            # split the list of accounts into sublists of 80 addresses per list to overcome:
-            # https://github.com/rotki/rotki/issues/3037
-            accounts_chunks = [accounts[x:x + 80] for x in range(0, len(accounts), 80)]
-            for accounts_chunk in accounts_chunks:
-                params = '|'.join(accounts_chunk)
-                btc_resp = request_get_dict(
-                    url=f'https://blockchain.info/multiaddr?active={params}',
-                    handle_429=True,
-                    # If we get a 429 then their docs suggest 10 seconds
-                    # https://blockchain.info/q
-                    backoff_in_seconds=10,
-                )
-                for entry in btc_resp['addresses']:
-                    balances[entry['address']] = satoshis_to_btc(FVal(entry['final_balance']))
-    except (
+    accounts_chunks = [accounts[x:x + 80] for x in range(0, len(accounts), 80)]
+    for accounts_chunk in accounts_chunks:
+        params = '|'.join(accounts_chunk)
+        btc_resp = request_get_dict(
+            url=f'https://blockchain.info/multiaddr?active={params}',
+            handle_429=True,
+            # If we get a 429 then their docs suggest 10 seconds
+            # https://blockchain.info/q
+            backoff_in_seconds=10,
+        )
+        for entry in btc_resp['addresses']:
+            balances[entry['address']] = satoshis_to_btc(
+                ensure_type(
+                    symbol=entry['final_balance'],
+                    expected_type=int,
+                    location='blockchain.info "final_balance"',
+                ),
+            )
+    return balances
+
+
+def get_bitcoin_addresses_balances(
+        accounts: List[BTCAddress],
+) -> Dict[BTCAddress, FVal]:
+    """Queries bitcoin balance APIs for the balances of accounts
+
+    May raise:
+    - RemoteError couldn't query any of the bitcoin balance APIs
+    """
+    if _have_bc1_accounts(accounts) is True:
+        api_callbacks = {
+            'blockstream.info': _query_blockstream_info,
+            'mempool.space': _query_mempool_space,
+        }
+    else:
+        api_callbacks = {
+            'blockchain.info': _query_blockchain_info,
+            'blockstream.info': _query_blockstream_info,
+            'mempool.space': _query_mempool_space,
+        }
+    errors: Dict[str, str] = {}
+    for api_name, callback in api_callbacks.items():
+        try:
+            balances = callback(accounts)
+        except (
             requests.exceptions.RequestException,
             UnableToDecryptRemoteData,
             requests.exceptions.Timeout,
-    ) as e:
-        raise RemoteError(f'bitcoin external API request for balances failed due to {str(e)}') from e  # noqa: E501
-    except KeyError as e:
-        raise RemoteError(
-            f'Malformed response when querying bitcoin blockchain via {source}.'
-            f'Did not find key {e}',
-        ) from e
+            RemoteError,
+            DeserializationError,
+        ) as e:
+            errors[api_name] = str(e)
+            continue
+        except KeyError as e:
+            errors[api_name] = f'Got unexpected response from {api_name}. Couldn\'t find key {str(e)}'  # noqa: E501
+        else:
+            return balances
 
-    return balances
+    serialized_errors = ', '.join(f'{source} error is: "{error}"' for (source, error) in errors.items())  # noqa: E501
+    raise RemoteError(f'Bitcoin external API request for balances failed. {serialized_errors}')  # noqa: E501
 
 
 def _check_blockstream_for_transactions(
         accounts: List[BTCAddress],
 ) -> Dict[BTCAddress, Tuple[bool, FVal]]:
-    """May raise connection errors or KeyError"""
+    """May raise:
+    - RemoteError if couldn't query
+    - KeyError if response structure differs from the expected one
+    - DeserializationError if response values differ from the expected
+    """
     have_transactions = {}
     for account in accounts:
         url = f'https://blockstream.info/api/address/{account}'
         response_data = request_get_dict(url=url, handle_429=True, backoff_in_seconds=4)
         stats = response_data['chain_stats']
-        balance = satoshis_to_btc(int(stats['funded_txo_sum']) - int(stats['spent_txo_sum']))
+        funded_txo_sum = satoshis_to_btc(
+            ensure_type(
+                symbol=stats['funded_txo_sum'],
+                expected_type=int,
+                location='blockstream funded_txo_sum',
+            ),
+        )
+        spent_txo_sum = satoshis_to_btc(
+            ensure_type(
+                symbol=stats['spent_txo_sum'],
+                expected_type=int,
+                location='blockstream spent_txo_sum',
+            ),
+        )
+        balance = funded_txo_sum - spent_txo_sum
         have_txs = stats['tx_count'] != 0
         have_transactions[account] = (have_txs, balance)
 
@@ -124,5 +213,7 @@ def have_bitcoin_transactions(accounts: List[BTCAddress]) -> Dict[BTCAddress, Tu
             f'Malformed response when querying bitcoin blockchain via {source}.'
             f'Did not find key {str(e)}',
         ) from e
+    except DeserializationError as e:
+        raise RemoteError(f'Couldn\'t read data from the response due to {str(e)}') from e
 
     return have_transactions
diff --git a/rotkehlchen/serialization/deserialize.py b/rotkehlchen/serialization/deserialize.py
--- a/rotkehlchen/serialization/deserialize.py
+++ b/rotkehlchen/serialization/deserialize.py
@@ -1,5 +1,16 @@
 import logging
-from typing import TYPE_CHECKING, Any, Callable, Dict, Literal, Tuple, TypeVar, Union, overload
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    Dict,
+    Literal,
+    Tuple,
+    Type,
+    TypeVar,
+    Union,
+    overload,
+)
 
 from eth_utils import to_checksum_address
 
@@ -587,3 +598,12 @@ def deserialize_ethereum_transaction(
         raise DeserializationError(
             f'ethereum {"internal" if internal else ""}transaction from {source} missing expected key {str(e)}',  # noqa: E501
         ) from e
+
+
+def ensure_type(symbol: Any, expected_type: Type, location: str) -> Any:
+    if isinstance(symbol, expected_type) is True:
+        return symbol
+    raise DeserializationError(
+        f'Value "{symbol}" has type {type(symbol)} '
+        f'but expected {expected_type} at location "{location}"',
+    )


Test patch:
diff --git a/rotkehlchen/tests/unit/test_bitcoin.py b/rotkehlchen/tests/unit/test_bitcoin.py
--- a/rotkehlchen/tests/unit/test_bitcoin.py
+++ b/rotkehlchen/tests/unit/test_bitcoin.py
@@ -1,5 +1,8 @@
+from unittest.mock import MagicMock, patch
+
 import pytest
 
+from rotkehlchen.chain.bitcoin import get_bitcoin_addresses_balances
 from rotkehlchen.chain.bitcoin.hdkey import HDKey, XpubType
 from rotkehlchen.chain.bitcoin.utils import (
     is_valid_btc_address,
@@ -12,13 +15,15 @@
 )
 from rotkehlchen.chain.bitcoin.xpub import XpubData
 from rotkehlchen.chain.constants import NON_BITCOIN_CHAINS, SupportedBlockchain
-from rotkehlchen.errors.misc import XPUBError
+from rotkehlchen.constants import ZERO
+from rotkehlchen.errors.misc import RemoteError, XPUBError
 from rotkehlchen.tests.utils.ens import ENS_BRUNO_BTC_ADDR, ENS_BRUNO_BTC_BYTES
 from rotkehlchen.tests.utils.factories import (
     UNIT_BTC_ADDRESS1,
     UNIT_BTC_ADDRESS2,
     UNIT_BTC_ADDRESS3,
 )
+from rotkehlchen.types import BTCAddress
 
 
 def test_is_valid_btc_address():
@@ -350,3 +355,40 @@ def test_valid_bitcoin_chains():
     for blockchain in SupportedBlockchain:
         if blockchain not in (SupportedBlockchain.BITCOIN, SupportedBlockchain.BITCOIN_CASH):
             assert blockchain in NON_BITCOIN_CHAINS
+
+
+def test_bitcoin_balance_api_resolver():
+    # Test blockstream will be hit for bch
+    addresses = [
+        BTCAddress('3FZbgi29cpjq2GjdwV8eyHuJJnkLtktZc5'),
+        BTCAddress('34SjMcbLquZ7HmFmQiAHqEHY4mBEbvGeVL'),
+        BTCAddress('3J7sT2fbDaF3XrjpWM5GsUyaDr7i7psi88'),
+        BTCAddress('36Z62MQfJHF11DWqMMzc3rqLiDFGiVF8CB'),
+        BTCAddress('33k4CdyQJFwXQD9giSKyo36mTvE9Y6C9cP'),
+    ]
+    expected_balance = {
+        '34SjMcbLquZ7HmFmQiAHqEHY4mBEbvGeVL': ZERO,
+        '33k4CdyQJFwXQD9giSKyo36mTvE9Y6C9cP': ZERO,
+        '3FZbgi29cpjq2GjdwV8eyHuJJnkLtktZc5': ZERO,
+        '36Z62MQfJHF11DWqMMzc3rqLiDFGiVF8CB': ZERO,
+        '3J7sT2fbDaF3XrjpWM5GsUyaDr7i7psi88': ZERO,
+    }
+
+    # Test balances are returned properly
+    balances = get_bitcoin_addresses_balances(addresses)
+    assert balances == expected_balance
+
+    # One fails
+    with patch('rotkehlchen.chain.bitcoin._query_blockchain_info', MagicMock(side_effect=KeyError('someProperty'))):  # noqa: E501
+        balances = get_bitcoin_addresses_balances(addresses)
+        assert balances == expected_balance
+
+        # Two fail
+        with patch('rotkehlchen.chain.bitcoin._query_blockstream_info', MagicMock(side_effect=RemoteError('Epic fail'))):  # noqa: E501
+            balances = get_bitcoin_addresses_balances(addresses)
+            assert balances == expected_balance
+
+            # Three fail - FATALITY!!!
+            with patch('rotkehlchen.chain.bitcoin._query_mempool_space', MagicMock(side_effect=RemoteError('Fatality'))):  # noqa: E501
+                with pytest.raises(RemoteError):
+                    balances = get_bitcoin_addresses_balances(addresses)