Repository: rotki/rotki

Issue:
Relieve pressure on tax report by maintaining backend tasks querying information beforehand
## Abstract

If the user has never done a tax report and has a lot of trades it takes a lot of time to create a tax report. And some time it may actually end up being rate limited by cryptocompare due to the sheer number of queries.

Additionally the data that are generated during a tax report such as profit/loss over time and the trades pulled are very useful and we would also need them in other cases.

## Task

Maintain some backend tasks that run every `X` minutes. They should query:

- [x] Cryptocompare hourly historical price data for all assets the user owns. These should be well spaced away from each other to not get the user rate limited.
- [x] Ethereum transactions of the user's accounts
- [x] Trades the user has made in each exchange


Discussion:

Resolving patch:
diff --git a/rotkehlchen/chain/ethereum/transactions.py b/rotkehlchen/chain/ethereum/transactions.py
--- a/rotkehlchen/chain/ethereum/transactions.py
+++ b/rotkehlchen/chain/ethereum/transactions.py
@@ -31,7 +31,7 @@ def __init__(
         self.msg_aggregator = msg_aggregator
         self.tx_per_address: Dict[ChecksumEthAddress, int] = defaultdict(int)
 
-    def _single_address_query_transactions(
+    def single_address_query_transactions(
             self,
             address: ChecksumEthAddress,
             start_ts: Timestamp,
@@ -128,7 +128,7 @@ def query(
             accounts = self.database.get_blockchain_accounts().eth
 
         for address in accounts:
-            new_transactions = self._single_address_query_transactions(
+            new_transactions = self.single_address_query_transactions(
                 address=address,
                 start_ts=from_ts,
                 end_ts=to_ts,
diff --git a/rotkehlchen/rotkehlchen.py b/rotkehlchen/rotkehlchen.py
--- a/rotkehlchen/rotkehlchen.py
+++ b/rotkehlchen/rotkehlchen.py
@@ -303,6 +303,7 @@ def unlock_user(
             cryptocompare=self.cryptocompare,
             premium_sync_manager=self.premium_sync_manager,
             chain_manager=self.chain_manager,
+            exchange_manager=self.exchange_manager,
         )
         self.user_is_logged_in = True
         log.debug('User unlocking complete')
diff --git a/rotkehlchen/tasks/manager.py b/rotkehlchen/tasks/manager.py
--- a/rotkehlchen/tasks/manager.py
+++ b/rotkehlchen/tasks/manager.py
@@ -1,16 +1,20 @@
-import gevent
-
 import logging
 import random
-from rotkehlchen.greenlets import GreenletManager
-from rotkehlchen.db.dbhandler import DBHandler
-from typing import NamedTuple, Set, List
+from collections import defaultdict
+from typing import DefaultDict, List, NamedTuple, Set
+
+import gevent
+
 from rotkehlchen.assets.asset import Asset
+from rotkehlchen.chain.bitcoin.xpub import XpubManager
+from rotkehlchen.chain.manager import ChainManager
+from rotkehlchen.db.dbhandler import DBHandler
+from rotkehlchen.exchanges.manager import ExchangeManager
 from rotkehlchen.externalapis.cryptocompare import Cryptocompare
-from rotkehlchen.utils.misc import ts_now
+from rotkehlchen.greenlets import GreenletManager
 from rotkehlchen.premium.sync import PremiumSyncManager
-from rotkehlchen.chain.manager import ChainManager
-from rotkehlchen.chain.bitcoin.xpub import XpubManager
+from rotkehlchen.typing import ChecksumEthAddress
+from rotkehlchen.utils.misc import ts_now
 
 logger = logging.getLogger(__name__)
 
@@ -19,6 +23,16 @@
 DEFAULT_MAX_TASKS_NUM = 2
 CRYPTOCOMPARE_HISTOHOUR_FREQUENCY = 240  # at least 4 mins apart
 XPUB_DERIVATION_FREQUENCY = 3600  # every hour
+ETH_TX_QUERY_FREQUENCY = 3600  # every hour
+EXCHANGE_QUERY_FREQUENCY = 3600  # every hour
+
+
+def noop_exchange_succes_cb(trades, margin, asset_movements, exchange_specific_data) -> None:  # type: ignore # noqa: E501
+    pass
+
+
+def exchange_fail_cb(error: str) -> None:
+    logger.error(error)
 
 
 class CCHistoQuery(NamedTuple):
@@ -37,16 +51,20 @@ def __init__(
             cryptocompare: Cryptocompare,
             premium_sync_manager: PremiumSyncManager,
             chain_manager: ChainManager,
+            exchange_manager: ExchangeManager,
     ) -> None:
         self.max_tasks_num = max_tasks_num
         self.greenlet_manager = greenlet_manager
         self.api_task_greenlets = api_task_greenlets
         self.database = database
         self.cryptocompare = cryptocompare
+        self.exchange_manager = exchange_manager
         self.premium_sync_manager = premium_sync_manager
         self.cryptocompare_queries: Set[CCHistoQuery] = set()
         self.chain_manager = chain_manager
         self.last_xpub_derivation_ts = 0
+        self.last_eth_tx_query_ts: DefaultDict[ChecksumEthAddress, int] = defaultdict(int)
+        self.last_exchange_query_ts: DefaultDict[str, int] = defaultdict(int)
         self.prepared_cryptocompare_query = False
         self.greenlet_manager.spawn_and_track(  # Needs to run in greenlet, is slow
             after_seconds=None,
@@ -59,6 +77,8 @@ def __init__(
             self._maybe_schedule_cryptocompare_query,
             self.premium_sync_manager.maybe_upload_data_to_server,
             self._maybe_schedule_xpub_derivation,
+            self._maybe_query_ethereum_transactions,
+            self._maybe_schedule_exchange_trade_query,
         ]
 
     def _prepare_cryptocompare_queries(self) -> None:
@@ -139,7 +159,7 @@ def _maybe_schedule_cryptocompare_query(self) -> bool:
     def _maybe_schedule_xpub_derivation(self) -> None:
         """Schedules the xpub derivation task if enough time has passed and if user has xpubs"""
         now = ts_now()
-        if self.last_xpub_derivation_ts - now < XPUB_DERIVATION_FREQUENCY:
+        if now - self.last_xpub_derivation_ts <= XPUB_DERIVATION_FREQUENCY:
             return
 
         xpubs = self.database.get_bitcoin_xpub_data()
@@ -154,6 +174,62 @@ def _maybe_schedule_xpub_derivation(self) -> None:
         )
         self.last_xpub_derivation_ts = now
 
+    def _maybe_query_ethereum_transactions(self) -> None:
+        """Schedules the ethereum transaction query task if enough time has passed"""
+        accounts = self.database.get_blockchain_accounts().eth
+        if len(accounts) == 0:
+            return
+
+        now = ts_now()
+        queriable_accounts = [
+            x for x in accounts if now - self.last_eth_tx_query_ts[x] > ETH_TX_QUERY_FREQUENCY
+        ]
+        if len(queriable_accounts) == 0:
+            return
+
+        address = random.choice(queriable_accounts)
+        task_name = f'Query ethereum transactions for {address}'
+        logger.debug(f'Scheduling task to {task_name}')
+        self.greenlet_manager.spawn_and_track(
+            after_seconds=None,
+            task_name=task_name,
+            exception_is_error=True,
+            method=self.chain_manager.ethereum.transactions.single_address_query_transactions,
+            address=address,
+            start_ts=0,
+            end_ts=now,
+            with_limit=False,
+        )
+        self.last_eth_tx_query_ts[address] = now
+
+    def _maybe_schedule_exchange_trade_query(self) -> None:
+        """Schedules the exchange trades history query task if enough time has passed"""
+        if len(self.exchange_manager.connected_exchanges) == 0:
+            return
+
+        now = ts_now()
+        queriable_exchanges = [
+            exchange for name, exchange in self.exchange_manager.connected_exchanges.items()
+            if now - self.last_exchange_query_ts[name] > EXCHANGE_QUERY_FREQUENCY
+        ]
+        if len(queriable_exchanges) == 0:
+            return
+
+        exchange = random.choice(queriable_exchanges)
+        task_name = f'Query trades of {exchange.name} exchange'
+        logger.debug(f'Scheduling task to {task_name}')
+        self.greenlet_manager.spawn_and_track(
+            after_seconds=None,
+            task_name=task_name,
+            exception_is_error=True,
+            method=exchange.query_history_with_callbacks,
+            start_ts=0,
+            end_ts=now,
+            success_callback=noop_exchange_succes_cb,
+            fail_callback=exchange_fail_cb,
+        )
+        self.last_exchange_query_ts[exchange.name] = now
+
     def schedule(self) -> None:
         """Schedules background tasks"""
         self.greenlet_manager.clear_finished()


Test patch:
diff --git a/rotkehlchen/tests/unit/test_tasks_manager.py b/rotkehlchen/tests/unit/test_tasks_manager.py
new file mode 100644
--- /dev/null
+++ b/rotkehlchen/tests/unit/test_tasks_manager.py
@@ -0,0 +1,151 @@
+from typing import List
+from unittest.mock import patch
+
+import gevent
+import pytest
+
+from rotkehlchen.chain.bitcoin.hdkey import HDKey
+from rotkehlchen.chain.bitcoin.xpub import XpubData
+from rotkehlchen.exchanges.manager import ExchangeManager
+from rotkehlchen.tasks.manager import TaskManager
+from rotkehlchen.utils.misc import ts_now
+
+
+class MockPremiumSyncManager():
+
+    def __init__(self):
+        pass
+
+    def maybe_upload_data_to_server(self) -> None:
+        pass
+
+
+@pytest.fixture(name='max_tasks_num')
+def fixture_max_tasks_num() -> int:
+    return 5
+
+
+@pytest.fixture(name='api_task_greenlets')
+def fixture_api_task_greenlets() -> List:
+    return []
+
+
+@pytest.fixture(name='exchange_manager')
+def fixture_exchange_manager(function_scope_messages_aggregator) -> ExchangeManager:
+    return ExchangeManager(msg_aggregator=function_scope_messages_aggregator)
+
+
+@pytest.fixture(name='task_manager')
+def fixture_task_manager(
+        database,
+        blockchain,
+        max_tasks_num,
+        greenlet_manager,
+        api_task_greenlets,
+        cryptocompare,
+        exchange_manager,
+) -> TaskManager:
+    task_manager = TaskManager(
+        max_tasks_num=max_tasks_num,
+        greenlet_manager=greenlet_manager,
+        api_task_greenlets=api_task_greenlets,
+        database=database,
+        cryptocompare=cryptocompare,
+        premium_sync_manager=MockPremiumSyncManager(),  # type: ignore
+        chain_manager=blockchain,
+        exchange_manager=exchange_manager,
+    )
+    return task_manager
+
+
+@pytest.mark.parametrize('number_of_eth_accounts', [2])
+def test_maybe_query_ethereum_transactions(task_manager, ethereum_accounts):
+    task_manager.potential_tasks = [task_manager._maybe_query_ethereum_transactions]
+    now = ts_now()
+
+    def tx_query_mock(address, start_ts, end_ts, with_limit):
+        assert address in ethereum_accounts
+        assert start_ts == 0
+        assert end_ts >= now
+        assert with_limit is False
+
+    tx_query_patch = patch.object(
+        task_manager.chain_manager.ethereum.transactions,
+        'single_address_query_transactions',
+        wraps=tx_query_mock,
+    )
+
+    timeout = 8
+    try:
+        with gevent.Timeout(timeout):
+            with tx_query_patch as tx_mock:
+                # First two calls to schedule should handle the addresses
+                for i in range(2):
+                    task_manager.schedule()
+                    while True:
+                        if tx_mock.call_count == i + 1:
+                            break
+                        gevent.sleep(.2)
+
+                task_manager.schedule()
+                gevent.sleep(.5)
+                assert tx_mock.call_count == 2, '3rd schedule should do nothing'
+
+    except gevent.Timeout as e:
+        raise AssertionError(f'The transaction query was not scheduled within {timeout} seconds') from e  # noqa: E501
+
+
+def test_maybe_schedule_xpub_derivation(task_manager, database):
+    xpub = 'xpub68V4ZQQ62mea7ZUKn2urQu47Bdn2Wr7SxrBxBDDwE3kjytj361YBGSKDT4WoBrE5htrSB8eAMe59NPnKrcAbiv2veN5GQUmfdjRddD1Hxrk'  # noqa: E501
+    xpub_data = XpubData(
+        xpub=HDKey.from_xpub(xpub=xpub, path='m'),
+        derivation_path='m/0/0',
+    )
+    database.add_bitcoin_xpub(xpub_data)
+    task_manager.potential_tasks = [task_manager._maybe_schedule_xpub_derivation]
+    xpub_derive_patch = patch(
+        'rotkehlchen.chain.bitcoin.xpub.XpubManager.check_for_new_xpub_addresses',
+        return_value=None,
+    )
+
+    timeout = 4
+    try:
+        with gevent.Timeout(timeout):
+            with xpub_derive_patch as xpub_mock:
+                task_manager.schedule()
+                while True:
+                    if xpub_mock.call_count == 1:
+                        break
+                    gevent.sleep(.2)
+
+    except gevent.Timeout as e:
+        raise AssertionError(f'xpub derivation query was not scheduled within {timeout} seconds') from e  # noqa: E501
+
+
+def test_maybe_schedule_exchange_query(task_manager, exchange_manager, poloniex):
+    now = ts_now()
+    task_manager.potential_tasks = [task_manager._maybe_schedule_exchange_trade_query]
+
+    def mock_query_history(start_ts, end_ts):
+        assert start_ts == 0
+        assert end_ts >= now
+
+    exchange_manager.connected_exchanges['poloniex'] = poloniex
+    poloniex_patch = patch.object(poloniex, 'query_trade_history', wraps=mock_query_history)
+
+    timeout = 5
+    try:
+        with gevent.Timeout(timeout):
+            with poloniex_patch as poloniex_mock:
+                task_manager.schedule()
+                while True:
+                    if poloniex_mock.call_count == 1:
+                        break
+                    gevent.sleep(.2)
+
+                task_manager.schedule()
+                gevent.sleep(.5)
+                assert poloniex_mock.call_count == 1, '2nd schedule should do nothing'
+
+    except gevent.Timeout as e:
+        raise AssertionError(f'exchange query was not scheduled within {timeout} seconds') from e  # noqa: E501