Repository: rotki/rotki

Issue:
Omit ignored tokens from the token batch query
## Abstract

At this [point](https://github.com/rotki/rotki/blob/c6c6dc8298cc1806cb0c6a5230e276d964c305a6/rotkehlchen/chain/ethereum/tokens.py#L120) in the token querying code we create the batches of tokens to be queried. The ignored tokens are included in the batches and only end up getting ignored afterwards.

If there is a significant number of ignored tokens this may improve performance but for less than batch_size tokens it won't make a different.

This can also open up ways of ignoring big bunches of tokens, like say all uniswap v1,v2 and yearn vault related tokens in the future

## Task

Do it


Discussion:
I am taking this one up.

Resolving patch:
diff --git a/rotkehlchen/chain/ethereum/tokens.py b/rotkehlchen/chain/ethereum/tokens.py
--- a/rotkehlchen/chain/ethereum/tokens.py
+++ b/rotkehlchen/chain/ethereum/tokens.py
@@ -3,10 +3,12 @@
 from collections import defaultdict
 from typing import Dict, List, Optional, Sequence, Tuple
 
-from rotkehlchen.chain.ethereum.typing import string_to_ethereum_address
 from rotkehlchen.assets.asset import EthereumToken
 from rotkehlchen.chain.ethereum.manager import EthereumManager, NodeName
-from rotkehlchen.chain.ethereum.typing import CustomEthereumTokenWithIdentifier
+from rotkehlchen.chain.ethereum.typing import (
+    CustomEthereumTokenWithIdentifier,
+    string_to_ethereum_address,
+)
 from rotkehlchen.chain.ethereum.utils import token_normalized_value
 from rotkehlchen.constants.ethereum import ETH_SCAN
 from rotkehlchen.constants.misc import ZERO
@@ -118,13 +120,18 @@ def query_tokens_for_addresses(
             'Querying/detecting token balances for all addresses',
             force_detection=force_detection,
         )
-        all_tokens = GlobalDBHandler().get_ethereum_tokens(exceptions=[
+        ignored_assets = self.db.get_ignored_assets()
+        exceptions = [
             # Ignore the veCRV balance in token query. It's already detected by
             # defi SDK as part of locked CRV in Vote Escrowed CRV. Which is the right way
             # to approach it as there is no way to assign a price to 1 veCRV. It
             # can be 1 CRV locked for 4 years or 4 CRV locked for 1 year etc.
             string_to_ethereum_address('0x5f3b5DfEb7B28CDbD7FAba78963EE202a494e2A2'),
-        ])
+        ]
+        for asset in ignored_assets:  # don't query for the ignored tokens
+            if asset.is_eth_token():  # type ignore since we know asset is a token
+                exceptions.append(EthereumToken.from_asset(asset).ethereum_address)  # type: ignore
+        all_tokens = GlobalDBHandler().get_ethereum_tokens(exceptions=exceptions)
         # With etherscan with chunks > 120, we get request uri too large
         # so the limitation is not in the gas, but in the request uri length
         etherscan_chunks = list(get_chunks(all_tokens, n=ETHERSCAN_MAX_TOKEN_CHUNK_LENGTH))


Test patch:
diff --git a/rotkehlchen/tests/unit/test_tokens.py b/rotkehlchen/tests/unit/test_tokens.py
--- a/rotkehlchen/tests/unit/test_tokens.py
+++ b/rotkehlchen/tests/unit/test_tokens.py
@@ -85,3 +85,28 @@ def test_detected_tokens_cache(ethtokens, inquirer):  # pylint: disable=unused-a
             assert len(entry) == len(eth_map_entry)
             for token, val in entry.items():
                 assert token_normalized_value(eth_map_entry[token], token) == val
+
+
+@pytest.mark.parametrize('ignored_assets', [[A_GNO]])
+def test_ignored_tokens_in_query(ethtokens, inquirer):  # pylint: disable=unused-argument
+    """Test that if a token is ignored it's not included in the query"""
+    addr1 = make_ethereum_address()
+    addr2 = make_ethereum_address()
+    eth_map = {addr1: {A_GNO: 5000, A_MKR: 4000}, addr2: {A_MKR: 6000}}
+    etherscan_patch = mock_etherscan_query(
+        eth_map=eth_map,
+        etherscan=ethtokens.ethereum.etherscan,
+        original_queries=None,
+        original_requests_get=requests.get,
+        extra_flags=None,
+    )
+    ethtokens_max_chunks_patch = patch(
+        'rotkehlchen.chain.ethereum.tokens.ETHERSCAN_MAX_TOKEN_CHUNK_LENGTH',
+        new=800,
+    )
+
+    with ethtokens_max_chunks_patch, etherscan_patch:
+        result, _ = ethtokens.query_tokens_for_addresses([addr1, addr2], False)
+        assert len(result[addr1]) == 1
+        assert result[addr1][A_MKR] == FVal('4E-15')
+        assert len(result[addr2]) == 1