Repository: rotki/rotki

Issue:
Refreshing balances should derive new xpubkey addresses and check to see if there were deposits
## Abstract

When you refresh balances, you should check to see if there were coins sent to new addresses of xpubkeys

## Motivation

People with xpubkeys will occasionally deposit coins to new derived addresses. It's important that refreshing balances catches those.

## Specification





Discussion:
This is easy. Let's just implement it in next release

Resolving patch:
diff --git a/rotkehlchen/chain/bitcoin/xpub.py b/rotkehlchen/chain/bitcoin/xpub.py
--- a/rotkehlchen/chain/bitcoin/xpub.py
+++ b/rotkehlchen/chain/bitcoin/xpub.py
@@ -312,7 +312,7 @@ def check_for_new_xpub_addresses(
         """Checks all xpub addresses and sees if new addresses got used.
         If they did it adds them for tracking.
         """
-        log.debug('Starting task for derivation of new xpub addresses')
+        log.debug(f'Starting task for derivation of new {blockchain.value} xpub addresses')
         with self.db.conn.read_ctx() as cursor:
             xpubs = self.db.get_bitcoin_xpub_data(cursor)
         with self.lock:
diff --git a/rotkehlchen/chain/manager.py b/rotkehlchen/chain/manager.py
--- a/rotkehlchen/chain/manager.py
+++ b/rotkehlchen/chain/manager.py
@@ -26,7 +26,7 @@
 from rotkehlchen.chain.bitcoin import get_bitcoin_addresses_balances
 from rotkehlchen.chain.bitcoin.bch import get_bitcoin_cash_addresses_balances
 from rotkehlchen.chain.bitcoin.bch.utils import force_address_to_legacy_address
-from rotkehlchen.chain.bitcoin.xpub import XpubData
+from rotkehlchen.chain.bitcoin.xpub import XpubData, XpubManager
 from rotkehlchen.chain.ethereum.defi.chad import DefiChad
 from rotkehlchen.chain.ethereum.defi.structures import DefiProtocolBalances
 from rotkehlchen.chain.ethereum.modules import (
@@ -621,10 +621,27 @@ def query_balances(
             self.query_polkadot_balances(ignore_cache=ignore_cache)
         if should_query_avax:
             self.query_avax_balances(ignore_cache=ignore_cache)
+        if ignore_cache is True:
+            self.derive_new_addresses_from_xpubs(
+                should_derive_bch_xpubs=should_query_bch,
+                should_derive_btc_xpubs=should_query_btc,
+            )
 
         self.totals = self.balances.recalculate_totals()
         return self.get_balances_update()
 
+    def derive_new_addresses_from_xpubs(
+            self,
+            should_derive_btc_xpubs: bool = False,
+            should_derive_bch_xpubs: bool = False,
+    ) -> None:
+        """Derive new addresses from xpubs."""
+        xpub_manager = XpubManager(chain_manager=self)
+        if should_derive_btc_xpubs is True:
+            xpub_manager.check_for_new_xpub_addresses(blockchain=SupportedBlockchain.BITCOIN)
+        if should_derive_bch_xpubs is True:
+            xpub_manager.check_for_new_xpub_addresses(blockchain=SupportedBlockchain.BITCOIN_CASH)
+
     @protect_with_lock()
     @cache_response_timewise()
     def query_btc_balances(
diff --git a/rotkehlchen/tasks/manager.py b/rotkehlchen/tasks/manager.py
--- a/rotkehlchen/tasks/manager.py
+++ b/rotkehlchen/tasks/manager.py
@@ -8,7 +8,6 @@
 
 from rotkehlchen.api.websockets.typedefs import WSMessageType
 from rotkehlchen.assets.asset import Asset
-from rotkehlchen.chain.bitcoin.xpub import XpubManager
 from rotkehlchen.chain.manager import ChainManager
 from rotkehlchen.constants.assets import A_USD
 from rotkehlchen.db.dbhandler import DBHandler
@@ -28,14 +27,7 @@
 from rotkehlchen.logging import RotkehlchenLogsAdapter
 from rotkehlchen.premium.premium import Premium, premium_create_and_verify
 from rotkehlchen.premium.sync import PremiumSyncManager
-from rotkehlchen.types import (
-    ChecksumEvmAddress,
-    ExchangeLocationID,
-    Location,
-    Optional,
-    SupportedBlockchain,
-    Timestamp,
-)
+from rotkehlchen.types import ChecksumEvmAddress, ExchangeLocationID, Location, Optional, Timestamp
 from rotkehlchen.user_messages import MessagesAggregator
 from rotkehlchen.utils.misc import ts_now
 
@@ -226,17 +218,11 @@ def _maybe_schedule_xpub_derivation(self) -> None:
         log.debug('Scheduling task for Xpub derivation')
         self.greenlet_manager.spawn_and_track(
             after_seconds=None,
-            task_name='Derive new xpub addresses for BTC',
+            task_name='Derive new xpub addresses for BTC & BCH',
             exception_is_error=True,
-            method=XpubManager(self.chain_manager).check_for_new_xpub_addresses,
-            blockchain=SupportedBlockchain.BITCOIN,
-        )
-        self.greenlet_manager.spawn_and_track(
-            after_seconds=None,
-            task_name='Derive new xpub addresses for BCH',
-            exception_is_error=True,
-            method=XpubManager(self.chain_manager).check_for_new_xpub_addresses,
-            blockchain=SupportedBlockchain.BITCOIN_CASH,
+            method=self.chain_manager.derive_new_addresses_from_xpubs,
+            should_derive_btc_xpubs=True,
+            should_derive_bch_xpubs=True,
         )
         self.last_xpub_derivation_ts = now
 


Test patch:
diff --git a/rotkehlchen/tests/api/test_balances.py b/rotkehlchen/tests/api/test_balances.py
--- a/rotkehlchen/tests/api/test_balances.py
+++ b/rotkehlchen/tests/api/test_balances.py
@@ -266,6 +266,11 @@ def test_query_all_balances_ignore_cache(
     original_binance_query_dict = binance.api_query_dict
     binance_query_patch = patch.object(binance, 'api_query_dict', wraps=binance.api_query_dict)
     poloniex_query_patch = patch.object(poloniex, 'api_query_list', wraps=poloniex.api_query_list)
+    derive_new_addresses_from_xpubs_patch = patch.object(
+        rotki.chain_manager,
+        'derive_new_addresses_from_xpubs',
+        wraps=rotki.chain_manager.derive_new_addresses_from_xpubs,
+    )
 
     with ExitStack() as stack:
         stack.enter_context(setup.poloniex_patch)
@@ -274,12 +279,14 @@ def test_query_all_balances_ignore_cache(
         stack.enter_context(setup.bitcoin_patch)
         stack.enter_context(setup.ethtokens_max_chunks_patch)
         stack.enter_context(setup.beaconchain_patch)
-        function_call_counters = []
-        function_call_counters.append(stack.enter_context(eth_query_patch))
-        function_call_counters.append(stack.enter_context(btc_query_patch))
-        function_call_counters.append(stack.enter_context(tokens_query_patch))
-        function_call_counters.append(stack.enter_context(binance_query_patch))
-        function_call_counters.append(stack.enter_context(poloniex_query_patch))
+        function_call_counters = [
+            stack.enter_context(eth_query_patch),
+            stack.enter_context(btc_query_patch),
+            stack.enter_context(tokens_query_patch),
+            stack.enter_context(binance_query_patch),
+            stack.enter_context(poloniex_query_patch),
+            stack.enter_context(derive_new_addresses_from_xpubs_patch),
+        ]
 
         # Query all balances for the first time and test it works
         response = requests.get(
@@ -298,6 +305,9 @@ def test_query_all_balances_ignore_cache(
         for fn in function_call_counters:
             if fn._mock_wraps == original_binance_query_dict:
                 assert fn.call_count == 3
+            # addresses are not derived from xpubs when `ignore_cache` is False
+            elif fn == rotki.chain_manager.derive_new_addresses_from_xpubs:
+                assert fn.call_count == 0
             else:
                 assert fn.call_count == 1
         full_query_etherscan_count = etherscan_mock.call_count
@@ -320,6 +330,9 @@ def test_query_all_balances_ignore_cache(
         for fn in function_call_counters:
             if fn._mock_wraps == original_binance_query_dict:
                 assert fn.call_count == 3, msg
+            # addresses are not derived from xpubs when `ignore_cache` is False
+            elif fn == rotki.chain_manager.derive_new_addresses_from_xpubs:
+                assert fn.call_count == 0, msg
             else:
                 assert fn.call_count == 1, msg
         msg = 'etherscan call_count should have remained the same due to no token detection '
@@ -343,6 +356,9 @@ def test_query_all_balances_ignore_cache(
         for fn in function_call_counters:
             if fn._mock_wraps == original_binance_query_dict:
                 assert fn.call_count == 6, msg
+            # addresses are derived from xpubs when `ignore_cache` is True
+            elif fn == rotki.chain_manager.derive_new_addresses_from_xpubs:
+                assert fn.call_count == 1, msg
             else:
                 assert fn.call_count == 2, msg
         msg = 'etherscan call count should have doubled after forced token detection'