Repository: rotki/rotki

Issue:
SNX and staked aave tokens are double counted
## Problem Definition

It seems that Rotki is counting SNX tokens as double since the token is also classified as a DeFi protocol by Zerion.

Also after the first run staked AAVE tokens seem to some times appear double.
## Task

Fix it


Discussion:

Resolving patch:
diff --git a/rotkehlchen/assets/utils.py b/rotkehlchen/assets/utils.py
--- a/rotkehlchen/assets/utils.py
+++ b/rotkehlchen/assets/utils.py
@@ -1,13 +1,12 @@
 import logging
 from typing import Optional, Union
 
-from rotkehlchen.errors import UnknownAsset
+from rotkehlchen.errors import DeserializationError, UnknownAsset
 from rotkehlchen.typing import ChecksumEthAddress
 
 from .asset import EthereumToken
 from .unknown_asset import UnknownEthereumToken
 
-
 log = logging.getLogger(__name__)
 
 
@@ -25,14 +24,14 @@ def get_ethereum_token(
 
     try:
         ethereum_token = EthereumToken(symbol)
-    except UnknownAsset:
+    except (UnknownAsset, DeserializationError):
         is_unknown_asset = True
     else:
         if ethereum_token.ethereum_address != ethereum_address:
             is_unknown_asset = True
 
     if is_unknown_asset:
-        log.error(
+        log.warning(
             f'Encountered unknown asset {symbol} with address '
             f'{ethereum_address}. Instantiating UnknownEthereumToken',
         )
diff --git a/rotkehlchen/chain/manager.py b/rotkehlchen/chain/manager.py
--- a/rotkehlchen/chain/manager.py
+++ b/rotkehlchen/chain/manager.py
@@ -46,6 +46,7 @@
 from rotkehlchen.premium.premium import Premium
 from rotkehlchen.serialization.deserialize import deserialize_ethereum_address
 from rotkehlchen.typing import (
+    AssetType,
     BTCAddress,
     ChecksumEthAddress,
     ListOfBlockchainAddresses,
@@ -71,12 +72,19 @@
 
 DEFI_BALANCES_REQUERY_SECONDS = 600
 
-DEFI_PROTOCOLS_TO_SKIP_ASSETS = (
-    'Aave',  # aTokens are already detected at token balance queries
-    'Compound',  # cTokens are already detected at token balance queries
-    'Chi Gastoken by 1inch',  # cTokens are already detected at token balance queries
-    'yearn.finance • Vaults',  # yearn vault balances are detected by the yTokens
-)
+# Mapping to token symbols to ignore. True means all
+DEFI_PROTOCOLS_TO_SKIP_ASSETS = {
+    # aTokens are already detected at token balance queries
+    'Aave': True,  # None means all
+    # cTokens are already detected at token balance queries
+    'Compound': True,  # None means all
+    # Chitoken is in our all_assets.json
+    'Chi Gastoken by 1inch': True,  # None means all
+    # yearn vault balances are detected by the yTokens
+    'yearn.finance • Vaults': True,
+    # Synthetix SNX token is in all_assets.json
+    'Synthetix': ['SNX'],
+}
 
 
 T = TypeVar('T')
@@ -375,7 +383,7 @@ def query_balances(
 
         return self.get_balances_update()
 
-    @protect_with_lock(arguments_matter=False)
+    @protect_with_lock()
     def query_btc_balances(self) -> None:
         """Queries blockchain.info/blockstream for the balance of all BTC accounts
 
@@ -770,9 +778,9 @@ def query_ethereum_tokens(self, force_detection: bool) -> None:
         client and the chain is not synced
         """
         # Clear out all previous token balances
-        for token in [x for x, _ in self.totals.assets.items() if isinstance(x, EthereumToken)]:
+        for token in [x for x, _ in self.totals.assets.items() if x.asset_type == AssetType.ETH_TOKEN]:  # noqa: E501
             del self.totals.assets[token]
-        for token in [x for x, _ in self.totals.liabilities.items() if isinstance(x, EthereumToken)]:  # noqa: E501
+        for token in [x for x, _ in self.totals.liabilities.items() if x.asset_type == AssetType.ETH_TOKEN]:  # noqa: E501
             del self.totals.liabilities[token]
 
         self._query_ethereum_tokens(action=AccountAction.QUERY, force_detection=force_detection)
@@ -799,7 +807,7 @@ def query_defi_balances(self) -> Dict[ChecksumEthAddress, List[DefiProtocolBalan
             self.defi_balances_last_query_ts = ts_now()
             return self.defi_balances
 
-    @protect_with_lock(arguments_matter=False)
+    @protect_with_lock()
     def query_ethereum_balances(self, force_token_detection: bool) -> None:
         """Queries all the ethereum balances and populates the state
 
@@ -874,8 +882,16 @@ def _add_account_defi_balances_to_token_and_totals(
     ) -> None:
         """Add a single account's defi balances to per account and totals"""
         for entry in balances:
+
+            skip_list = DEFI_PROTOCOLS_TO_SKIP_ASSETS.get(entry.protocol.name, None)
+            double_entry = (
+                entry.balance_type == 'Asset' and
+                skip_list and
+                (skip_list is True or entry.base_balance.token_symbol in skip_list)  # type: ignore
+            )
+
             # We have to filter out specific balances/protocols here to not get double entries
-            if entry.balance_type == 'Asset' and entry.protocol.name in DEFI_PROTOCOLS_TO_SKIP_ASSETS:  # noqa: E501
+            if double_entry:
                 continue
 
             if entry.balance_type == 'Asset' and entry.base_balance.token_symbol == 'ETH':
diff --git a/rotkehlchen/utils/interfaces.py b/rotkehlchen/utils/interfaces.py
--- a/rotkehlchen/utils/interfaces.py
+++ b/rotkehlchen/utils/interfaces.py
@@ -13,12 +13,12 @@
 def _function_sig_key(name: str, arguments_matter: bool, *args: Any, **kwargs: Any) -> int:
     """Return a unique int identifying a function's call signature"""
     function_sig = name
-    for arg in args:
-        function_sig += str(arg)
-
     if arguments_matter:
+        for arg in args:
+            function_sig += str(arg)
         for _, value in kwargs.items():
             function_sig += str(value)
+
     return hash(function_sig)
 
 
@@ -36,7 +36,7 @@ def __init__(self) -> None:
         self.cache_ttl_secs = CACHE_RESPONSE_FOR_SECS
 
 
-def cache_response_timewise() -> Callable:
+def cache_response_timewise(arguments_matter: bool = True) -> Callable:
     """ This is a decorator for caching results of functions of objects.
     The objects must adhere to the CachableOject interface.
 
@@ -52,7 +52,7 @@ def _cache_response_timewise(f: Callable) -> Callable:
         @wraps(f)
         def wrapper(wrappingobj: CacheableObject, *args: Any, **kwargs: Any) -> Any:
             ignore_cache = kwargs.pop('ignore_cache', False)
-            cache_key = _function_sig_key(f.__name__, False, *args, **kwargs)
+            cache_key = _function_sig_key(f.__name__, arguments_matter, *args, **kwargs)
             now = ts_now()
             if ignore_cache is False:
                 # Check the cache


Test patch:
diff --git a/rotkehlchen/tests/api/test_balances.py b/rotkehlchen/tests/api/test_balances.py
--- a/rotkehlchen/tests/api/test_balances.py
+++ b/rotkehlchen/tests/api/test_balances.py
@@ -491,3 +491,56 @@ def test_multiple_balance_queries_not_concurrent(
         token_balances=setup.token_balances,
         also_btc=True,
     )
+
+
+@pytest.mark.parametrize('number_of_eth_accounts', [1])
+def test_balances_caching_mixup(
+        rotkehlchen_api_server,
+        ethereum_accounts,
+):
+    """Test that querying the balances in a specific order does not mix up the caches.
+
+    This tests for the problem seen where the bitcoin balances being empty and
+    queried first returned an empty result for the ethereum balances.
+    """
+    rotki = rotkehlchen_api_server.rest_api.rotkehlchen
+    setup = setup_balances(
+        rotki,
+        ethereum_accounts=ethereum_accounts,
+        btc_accounts=None,
+        eth_balances=['1000000000000000000'],
+        token_balances={A_RDN: ['2000000000000000000']},
+        original_queries=['zerion'],
+    )
+
+    # Test all balances request by requesting to not save the data
+    with ExitStack() as stack:
+        setup.enter_blockchain_patches(stack)
+        response_btc = requests.get(api_url_for(
+            rotkehlchen_api_server,
+            "named_blockchain_balances_resource",
+            blockchain='BTC',
+        ), json={'async_query': True})
+        response_eth = requests.get(api_url_for(
+            rotkehlchen_api_server,
+            "named_blockchain_balances_resource",
+            blockchain='ETH',
+        ), json={'async_query': True})
+        task_id_btc = assert_ok_async_response(response_btc)
+        task_id_eth = assert_ok_async_response(response_eth)
+        result_btc = wait_for_async_task_with_result(
+            rotkehlchen_api_server,
+            task_id_btc,
+        )
+        result_eth = wait_for_async_task_with_result(
+            rotkehlchen_api_server,
+            task_id_eth,
+        )
+        assert result_eth['per_account']['ETH'][ethereum_accounts[0]]['assets']['ETH']['amount'] == '1'  # noqa: E501
+        assert result_eth['per_account']['ETH'][ethereum_accounts[0]]['assets']['RDN']['amount'] == '2'  # noqa: E501
+        assert result_eth['totals']['assets']['ETH']['amount'] == '1'
+        assert result_eth['totals']['assets']['RDN']['amount'] == '2'
+        assert result_eth['per_account']['ETH'][ethereum_accounts[0]]['assets']['RDN']['amount'] == '2'  # noqa: E501
+        assert result_btc['per_account'] == {}
+        assert result_btc['totals']['assets'] == {}
+        assert result_btc['totals']['liabilities'] == {}
diff --git a/rotkehlchen/tests/unit/test_assets.py b/rotkehlchen/tests/unit/test_assets.py
--- a/rotkehlchen/tests/unit/test_assets.py
+++ b/rotkehlchen/tests/unit/test_assets.py
@@ -7,6 +7,9 @@
 
 from rotkehlchen.assets.asset import Asset, EthereumToken
 from rotkehlchen.assets.resolver import AssetResolver, asset_type_mapping
+from rotkehlchen.assets.unknown_asset import UnknownEthereumToken
+from rotkehlchen.assets.utils import get_ethereum_token
+from rotkehlchen.constants.assets import A_DAI
 from rotkehlchen.errors import DeserializationError, UnknownAsset
 from rotkehlchen.externalapis.coingecko import Coingecko
 from rotkehlchen.typing import AssetType
@@ -237,3 +240,28 @@ def test_asset_with_unknown_type_does_not_crash(asset_resolver):  # pylint: disa
     # After the test runs we must reset the asset resolver so that it goes back to
     # the normal list of assets
     AssetResolver._AssetResolver__instance = None
+
+
+def test_get_ethereum_token():
+    assert A_DAI == get_ethereum_token(
+        symbol='DAI',
+        ethereum_address='0x6B175474E89094C44Da98b954EedeAC495271d0F',
+    )
+    unknown_token = UnknownEthereumToken(
+        symbol='DAI',
+        ethereum_address='0xA379B8204A49A72FF9703e18eE61402FAfCCdD60',
+        decimals=18,
+    )
+    assert unknown_token == get_ethereum_token(
+        symbol='DAI',
+        ethereum_address='0xA379B8204A49A72FF9703e18eE61402FAfCCdD60',
+    ), 'correct symbol but wrong address should result in unknown token'
+    unknown_token = UnknownEthereumToken(
+        symbol='DOT',
+        ethereum_address='0xA379B8204A49A72FF9703e18eE61402FAfCCdD60',
+        decimals=18,
+    )
+    assert unknown_token == get_ethereum_token(
+        symbol='DOT',
+        ethereum_address='0xA379B8204A49A72FF9703e18eE61402FAfCCdD60',
+    ), 'symbol of normal chain (polkadot here) should result in unknown token'
diff --git a/rotkehlchen/tests/unit/test_utils.py b/rotkehlchen/tests/unit/test_utils.py
--- a/rotkehlchen/tests/unit/test_utils.py
+++ b/rotkehlchen/tests/unit/test_utils.py
@@ -180,6 +180,7 @@ def __init__(self):
 
         self.do_sum_call_count = 0
         self.do_something_call_count = 0
+        self.do_something_arguments_dont_matter_count = 0
 
     @cache_response_timewise()
     def do_sum(self, arg1, arg2, **kwargs):  # pylint: disable=no-self-use, unused-argument
@@ -191,6 +192,11 @@ def do_something(self, **kwargs):  # pylint: disable=unused-argument
         self.do_something_call_count += 1
         return 5
 
+    @cache_response_timewise(arguments_matter=False)
+    def do_something_arguments_dont_matter(self, arg1, arg2, **kwargs):  # pylint: disable=unused-argument  # noqa: E501
+        self.do_something_arguments_dont_matter_count += 1
+        return arg1 + arg2
+
 
 def test_cache_response_timewise():
     """Test that cached value is called and not the function again"""
@@ -227,6 +233,19 @@ def test_cache_response_timewise_ignore_cache():
     assert instance.do_sum_call_count == 2
 
 
+def test_cache_response_timewise_with_arguments_matter_false():
+    """Test that arguments_matter works as expected and if false we always get same result"""
+    instance = Foo()
+
+    assert instance.do_something_arguments_dont_matter(5, 6) == 11
+    assert instance.do_something_arguments_dont_matter(1, 2) == 11
+    assert instance.do_something_arguments_dont_matter(3, 4) == 11
+    assert instance.do_something_arguments_dont_matter_count == 1
+
+    assert instance.do_something_arguments_dont_matter(1, 2, ignore_cache=True) == 3
+    assert instance.do_something_arguments_dont_matter_count == 2
+
+
 def test_convert_to_int():
     assert convert_to_int('5') == 5
     assert convert_to_int('37451082560000003241000000000003221111111111') == 37451082560000003241000000000003221111111111  # noqa: E501