Repository: rotki/rotki

Issue:
Adding multiple ETH accounts simultaneously with one of them repeated show wrong balances
## Problem Definition

This beauty was spotted by @alexvansande.

Trying to add multiple ethereum accounts as a comma separated list with any of them repeated, will end up repeating the tokens of the repeated accounts, as many times as they are in the comma separated list.

This needs an empty account and will only show the duplicated balances during that rotki run. If rotki is restarted balances resume to normal.

As an example let's try the following:

```
0x1047344D5b5B28caeF6443848144c9a1e42Bb514,0x93D0DD7a89a9c477842bEcF30B8Ec18Bf285A19A,0x1047344D5b5B28caeF6443848144c9a1e42Bb514
```
At the time of writing `0x1047344D5b5B28caeF6443848144c9a1e42Bb514` has 32 USDC. If you repeat it once in the list as done above, you will see 64 USDC in the dashoard.

The frontend seems to make 3 calls. One of them correctly returns an error:

```
{"result": {"status": "completed", "outcome": {"result": null, "message": "Blockchain account/s ['0x1047344D5b5B28caeF6443848144c9a1e42Bb514'] already exist"}}, "message": ""}
```

If you try to repeat that address 4 times you get 32*4 = 128 USDC

```
0x1047344D5b5B28caeF6443848144c9a1e42Bb514,0x93D0DD7a89a9c477842bEcF30B8Ec18Bf285A19A,0x1047344D5b5B28caeF6443848144c9a1e42Bb514, 0x1047344D5b5B28caeF6443848144c9a1e42Bb514, 0x1047344D5b5B28caeF6443848144c9a1e42Bb514
```

## Task

Fix it


Discussion:

Resolving patch:
diff --git a/rotkehlchen/chain/manager.py b/rotkehlchen/chain/manager.py
--- a/rotkehlchen/chain/manager.py
+++ b/rotkehlchen/chain/manager.py
@@ -227,6 +227,20 @@ def is_queried(self, blockchain: SupportedBlockchain) -> bool:
         # else
         raise AssertionError('Invalid blockchain value')
 
+    def account_exists(
+            self,
+            blockchain: SupportedBlockchain,
+            account: Union[BTCAddress, ChecksumEthAddress, KusamaAddress],
+    ) -> bool:
+        if blockchain == SupportedBlockchain.ETHEREUM:
+            return account in self.eth
+        if blockchain == SupportedBlockchain.BITCOIN:
+            return account in self.btc
+        if blockchain == SupportedBlockchain.KUSAMA:
+            return account in self.ksm
+        # else
+        raise AssertionError('Invalid blockchain value')
+
 
 @dataclass(init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=True)
 class BlockchainBalancesUpdate:
@@ -274,6 +288,9 @@ def __init__(
 
         self.defi_lock = Semaphore()
         self.eth2_lock = Semaphore()
+        self.btc_lock = Semaphore()
+        self.eth_lock = Semaphore()
+        self.ksm_lock = Semaphore()
 
         # Per account balances
         self.balances = BlockchainBalances(db=database)
@@ -420,6 +437,22 @@ def get_balances_update(self) -> BlockchainBalancesUpdate:
             totals=self.totals.copy(),
         )
 
+    def check_accounts_exist(
+            self,
+            blockchain: SupportedBlockchain,
+            accounts: ListOfBlockchainAddresses,
+    ) -> None:
+        """Checks if any of the accounts already exist and if they do raises an input error"""
+        existing_accounts = []
+        for account in accounts:
+            if self.balances.account_exists(blockchain, account):
+                existing_accounts.append(account)
+
+        if len(existing_accounts) != 0:
+            raise InputError(
+                f'Blockchain account/s {",".join(existing_accounts)} already exist',
+            )
+
     @protect_with_lock(arguments_matter=True)
     @cache_response_timewise()
     def query_balances(
@@ -699,7 +732,8 @@ def add_blockchain_accounts(
 
         May Raise:
         - EthSyncError from modify_blockchain_accounts
-        - InputError if the given accounts list is empty, or if it contains invalid accounts
+        - InputError if the given accounts list is empty, or if it contains invalid accounts,
+          or if any account already exists.
         - RemoteError if an external service such as Etherscan is queried and
           there is a problem
         """
@@ -792,71 +826,80 @@ def modify_blockchain_accounts(
           as etherscan or blockchain.info
         """
         if blockchain == SupportedBlockchain.BITCOIN:
-            # we are adding/removing accounts, make sure query cache is flushed
-            self.flush_cache('query_btc_balances', arguments_matter=True)
-            self.flush_cache('query_balances', arguments_matter=True)
-            self.flush_cache('query_balances', arguments_matter=True, blockchain=SupportedBlockchain.BITCOIN)  # noqa: E501
-            for idx, account in enumerate(accounts):
-                a_balance = already_queried_balances[idx] if already_queried_balances else None
-                self.modify_btc_account(
-                    BTCAddress(account),
-                    append_or_remove,
-                    add_or_sub,
-                    already_queried_balance=a_balance,
-                )
-
-        elif blockchain == SupportedBlockchain.ETHEREUM:
-            # we are adding/removing accounts, make sure query cache is flushed
-            self.flush_cache('query_ethereum_balances', arguments_matter=True, force_token_detection=False)  # noqa: E501
-            self.flush_cache('query_ethereum_balances', arguments_matter=True, force_token_detection=True)  # noqa: E501
-            self.flush_cache('query_balances', arguments_matter=True)
-            self.flush_cache('query_balances', arguments_matter=True, blockchain=SupportedBlockchain.ETHEREUM)  # noqa: E501
-            for account in accounts:
-                address = deserialize_ethereum_address(account)
-                try:
-                    self.modify_eth_account(
-                        account=address,
-                        append_or_remove=append_or_remove,
-                    )
-                except BadFunctionCallOutput as e:
-                    log.error(
-                        'Assuming unsynced chain. Got web3 BadFunctionCallOutput '
-                        'exception: {}'.format(str(e)),
+            with self.btc_lock:
+                if append_or_remove == 'append':
+                    self.check_accounts_exist(blockchain, accounts)
+                # we are adding/removing accounts, make sure query cache is flushed
+                self.flush_cache('query_btc_balances', arguments_matter=True)
+                self.flush_cache('query_balances', arguments_matter=True)
+                self.flush_cache('query_balances', arguments_matter=True, blockchain=SupportedBlockchain.BITCOIN)  # noqa: E501
+                for idx, account in enumerate(accounts):
+                    a_balance = already_queried_balances[idx] if already_queried_balances else None
+                    self.modify_btc_account(
+                        BTCAddress(account),
+                        append_or_remove,
+                        add_or_sub,
+                        already_queried_balance=a_balance,
                     )
-                    raise EthSyncError(
-                        'Tried to use the ethereum chain of a local client to edit '
-                        'an eth account but the chain is not synced.',
-                    ) from e
 
-                # Also modify and take into account defi balances
+        elif blockchain == SupportedBlockchain.ETHEREUM:
+            with self.eth_lock:
                 if append_or_remove == 'append':
-                    balances = self.defichad.query_defi_balances([address])
-                    address_balances = balances.get(address, [])
-                    if len(address_balances) != 0:
-                        self.defi_balances[address] = address_balances
-                        self._add_account_defi_balances_to_token_and_totals(
+                    self.check_accounts_exist(blockchain, accounts)
+                # we are adding/removing accounts, make sure query cache is flushed
+                self.flush_cache('query_ethereum_balances', arguments_matter=True, force_token_detection=False)  # noqa: E501
+                self.flush_cache('query_ethereum_balances', arguments_matter=True, force_token_detection=True)  # noqa: E501
+                self.flush_cache('query_balances', arguments_matter=True)
+                self.flush_cache('query_balances', arguments_matter=True, blockchain=SupportedBlockchain.ETHEREUM)  # noqa: E501
+                for account in accounts:
+                    address = deserialize_ethereum_address(account)
+                    try:
+                        self.modify_eth_account(
                             account=address,
-                            balances=address_balances,
+                            append_or_remove=append_or_remove,
+                        )
+                    except BadFunctionCallOutput as e:
+                        log.error(
+                            'Assuming unsynced chain. Got web3 BadFunctionCallOutput '
+                            'exception: {}'.format(str(e)),
                         )
-                else:  # remove
-                    self.defi_balances.pop(address, None)
-                # For each module run the corresponding callback for the address
-                for _, module in self.iterate_modules():
+                        raise EthSyncError(
+                            'Tried to use the ethereum chain of a local client to edit '
+                            'an eth account but the chain is not synced.',
+                        ) from e
+
+                    # Also modify and take into account defi balances
                     if append_or_remove == 'append':
-                        module.on_account_addition(address)
+                        balances = self.defichad.query_defi_balances([address])
+                        address_balances = balances.get(address, [])
+                        if len(address_balances) != 0:
+                            self.defi_balances[address] = address_balances
+                            self._add_account_defi_balances_to_token_and_totals(
+                                account=address,
+                                balances=address_balances,
+                            )
                     else:  # remove
-                        module.on_account_removal(address)
+                        self.defi_balances.pop(address, None)
+                    # For each module run the corresponding callback for the address
+                    for _, module in self.iterate_modules():
+                        if append_or_remove == 'append':
+                            module.on_account_addition(address)
+                        else:  # remove
+                            module.on_account_removal(address)
 
         elif blockchain == SupportedBlockchain.KUSAMA:
-            # we are adding/removing accounts, make sure query cache is flushed
-            self.flush_cache('query_kusama_balances', arguments_matter=True)
-            self.flush_cache('query_balances', arguments_matter=True)
-            self.flush_cache('query_balances', arguments_matter=True, blockchain=SupportedBlockchain.KUSAMA)  # noqa: E501
-            for account in accounts:
-                self.modify_kusama_account(
-                    account=KusamaAddress(account),
-                    append_or_remove=append_or_remove,
-                )
+            with self.ksm_lock:
+                if append_or_remove == 'append':
+                    self.check_accounts_exist(blockchain, accounts)
+                # we are adding/removing accounts, make sure query cache is flushed
+                self.flush_cache('query_kusama_balances', arguments_matter=True)
+                self.flush_cache('query_balances', arguments_matter=True)
+                self.flush_cache('query_balances', arguments_matter=True, blockchain=SupportedBlockchain.KUSAMA)  # noqa: E501
+                for account in accounts:
+                    self.modify_kusama_account(
+                        account=KusamaAddress(account),
+                        append_or_remove=append_or_remove,
+                    )
         else:
             # That should not happen. Should be checked by marshmallow
             raise AssertionError(


Test patch:
diff --git a/rotkehlchen/tests/api/test_blockchain.py b/rotkehlchen/tests/api/test_blockchain.py
--- a/rotkehlchen/tests/api/test_blockchain.py
+++ b/rotkehlchen/tests/api/test_blockchain.py
@@ -4,11 +4,13 @@
 from http import HTTPStatus
 from unittest.mock import patch
 
+import gevent
 import pytest
 import requests
 from eth_utils import to_checksum_address
 
 from rotkehlchen.chain.substrate.typing import KusamaAddress
+from rotkehlchen.constants.assets import A_DAI
 from rotkehlchen.constants.misc import ZERO
 from rotkehlchen.fval import FVal
 from rotkehlchen.logging import RotkehlchenLogsAdapter
@@ -26,7 +28,7 @@
     assert_eth_balances_result,
     compare_account_data,
 )
-from rotkehlchen.tests.utils.constants import A_RDN
+from rotkehlchen.tests.utils.constants import A_GNO, A_RDN
 from rotkehlchen.tests.utils.ens import ENS_BRUNO, ENS_BRUNO_BTC_ADDR, ENS_BRUNO_KSM_ADDR
 from rotkehlchen.tests.utils.factories import (
     UNIT_BTC_ADDRESS1,
@@ -42,6 +44,7 @@
     SUBSTRATE_ACC2_KSM_ADDR,
 )
 from rotkehlchen.typing import BlockchainAccountData, SupportedBlockchain
+from rotkehlchen.utils.misc import from_wei
 
 logger = logging.getLogger(__name__)
 log = RotkehlchenLogsAdapter(logger)
@@ -503,6 +506,112 @@ def test_add_blockchain_accounts(
         )
 
 
+@pytest.mark.parametrize('number_of_eth_accounts', [0])
+@pytest.mark.parametrize('btc_accounts', [[]])
+def test_add_blockchain_accounts_concurrent(
+        rotkehlchen_api_server,
+):
+    """
+    Test reproducing the erroneous behaviour in token totals seen here:
+    https://github.com/rotki/rotki/issues/2603
+    """
+    ethereum_accounts = [make_ethereum_address(), make_ethereum_address(), make_ethereum_address()]
+    account_to_idx = {ethereum_accounts[0]: 0, ethereum_accounts[1]: 1, ethereum_accounts[2]: 2}
+    rotki = rotkehlchen_api_server.rest_api.rotkehlchen
+    eth_balances = ['1000000', '2000000', '3000000']
+    token_balances = {
+        A_RDN: ['0', '4000000', '0'],
+        A_GNO: ['555555', '0', '0'],
+        A_DAI: ['0', '0', '666666666'],
+    }
+    setup = setup_balances(
+        rotki,
+        ethereum_accounts=ethereum_accounts,
+        btc_accounts=None,
+        eth_balances=eth_balances,
+        token_balances=token_balances,
+        btc_balances=None,
+    )
+
+    task_ids = {}
+    query_accounts = ethereum_accounts.copy()
+    for _ in range(5):
+        query_accounts.extend(ethereum_accounts)
+    with ExitStack() as stack:
+        # Fire all requests almost concurrently. And don't wait for them
+        setup.enter_ethereum_patches(stack)
+        for idx, account in enumerate(query_accounts):
+            gevent.spawn_later(
+                0.01 * idx,
+                requests.put,
+                api_url_for(
+                    rotkehlchen_api_server,
+                    'blockchainsaccountsresource',
+                    blockchain='ETH',
+                ),
+                json={'accounts': [{'address': account}], 'async_query': True},
+            )
+        # We are making an assumption of sequential ids here. This may not always
+        # be the case so for that later down the test we will skip the task check
+        # if this happens. Can't think of a better way to do this at the moment
+        task_ids = {idx: account for idx, account in enumerate(query_accounts)}
+
+        # for task_id, account in task_ids.items():
+        with gevent.Timeout(ASYNC_TASK_WAIT_TIMEOUT):
+            while len(task_ids) != 0:
+                task_id, account = random.choice(list(task_ids.items()))
+                response = requests.get(
+                    api_url_for(
+                        rotkehlchen_api_server,
+                        'specific_async_tasks_resource',
+                        task_id=task_id,
+                    ),
+                )
+                if response.status_code == HTTPStatus.NOT_FOUND:
+                    gevent.sleep(.1)  # not started yet
+                    continue
+
+                assert_proper_response(response)
+                result = response.json()['result']
+                status = result['status']
+                if status == 'pending':
+                    gevent.sleep(.1)
+                    continue
+                if status == 'completed':
+                    result = result['outcome']
+                else:
+                    raise AssertionError('Should not happen at this point')
+
+                task_ids.pop(task_id)
+                if result['result'] is None:
+                    assert 'already exist' in result['message']
+                    continue
+
+                result = result['result']
+                per_acc = result['per_account']['ETH'].get(account, None)
+                totals = result['totals']['assets']
+                idx = account_to_idx[account]
+                if per_acc is None:
+                    # As mentioned above this may happen only if our expectation of sequential
+                    # task indices breaks. In that case ignore this entry
+                    continue
+
+                assert FVal(per_acc['assets']['ETH']['amount']) == from_wei(FVal(eth_balances[idx]))  # noqa: E501
+
+                rdn = from_wei(FVal(token_balances[A_RDN][idx]))
+                if rdn != ZERO:
+                    assert FVal(per_acc['assets']['RDN']['amount']) == rdn
+                    assert FVal(totals['RDN']['amount']) == rdn
+                gno = from_wei(FVal(token_balances[A_GNO][idx]))
+                if gno != ZERO:
+                    assert FVal(per_acc['assets']['GNO']['amount']) == gno
+                    assert FVal(totals['GNO']['amount']) == gno
+                dai = from_wei(FVal(token_balances[A_DAI][idx]))
+                if dai != ZERO:
+                    assert FVal(per_acc['assets']['DAI']['amount']) == dai
+                    assert FVal(totals['DAI']['amount']) == dai
+
+
 @pytest.mark.parametrize('include_etherscan_key', [False])
 @pytest.mark.parametrize('number_of_eth_accounts', [0])
 def test_no_etherscan_is_detected(rotkehlchen_api_server):