Repository: rotki/rotki

Issue:
Offer progress indicator during tax report generation
## Problem Definition

During the tax report generation, depending on the timerange, the number of taxable events in that period and the presence of price caches, it may take a long time for the tax report to be complete.

At the moment the only way to kind of watch the progress is to tail the log: `tail -f rotkehlchen.log`.

This can be improved on and we can provide a progress report to the user.

## Task

Provide periodic update to the user via the UI on the progress. This can be achieved through the [periodic_query_data()](https://github.com/rotkehlchenio/rotkehlchen/blob/855a4004b97c8ad6cb49579a9dbc11f33254988d/rotkehlchen/server.py#L246) function which is already querying the backend every `N` seconds.

A specific key can be added to the returned dictionary for when a tax report is underway. As the tax report is generated we can be updating a specific attribute of the `Accountant` class with the current timestamp and at every periodic query return the last processed timestamp.
Offer progress indicator during tax report generation
## Problem Definition

During the tax report generation, depending on the timerange, the number of taxable events in that period and the presence of price caches, it may take a long time for the tax report to be complete.

At the moment the only way to kind of watch the progress is to tail the log: `tail -f rotkehlchen.log`.

This can be improved on and we can provide a progress report to the user.

## Task

Provide periodic update to the user via the UI on the progress. This can be achieved through the [periodic_query_data()](https://github.com/rotkehlchenio/rotkehlchen/blob/855a4004b97c8ad6cb49579a9dbc11f33254988d/rotkehlchen/server.py#L246) function which is already querying the backend every `N` seconds.

A specific key can be added to the returned dictionary for when a tax report is underway. As the tax report is generated we can be updating a specific attribute of the `Accountant` class with the current timestamp and at every periodic query return the last processed timestamp.


Discussion:

Resolving patch:
diff --git a/rotkehlchen/accounting/accountant.py b/rotkehlchen/accounting/accountant.py
--- a/rotkehlchen/accounting/accountant.py
+++ b/rotkehlchen/accounting/accountant.py
@@ -100,6 +100,8 @@ def __init__(
         self.asset_movement_fees = FVal(0)
         self.last_gas_price = FVal(0)
 
+        self.currently_processed_timestamp = -1
+
         # Customizable Options
         self.ignored_assets = ignored_assets
         self.include_gas_costs = include_gas_costs
@@ -315,6 +317,7 @@ def process_history(
         self.eth_transactions_gas_costs = FVal(0)
         self.asset_movement_fees = FVal(0)
         self.csvexporter.reset_csv_lists()
+        self.currently_processed_timestamp = start_ts
 
         actions: List[TaxableAction] = list(trade_history)
         # If we got loans, we need to interleave them with the full history and re-sort
@@ -409,6 +412,8 @@ def process_action(
         if timestamp > end_ts:
             return False, prev_time, count
 
+        self.currently_processed_timestamp = timestamp
+
         action_type = action_get_type(action)
 
         asset1, asset2 = action_get_assets(action)
diff --git a/rotkehlchen/rotkehlchen.py b/rotkehlchen/rotkehlchen.py
--- a/rotkehlchen/rotkehlchen.py
+++ b/rotkehlchen/rotkehlchen.py
@@ -1,11 +1,11 @@
 #!/usr/bin/env python
 import logging
 import shutil
+from typing import Dict, Union
 
 import gevent
 from gevent.lock import Semaphore
 
-from rotkehlchen import typing
 from rotkehlchen.accounting.accountant import Accountant
 from rotkehlchen.binance import Binance
 from rotkehlchen.bitmex import Bitmex
@@ -22,6 +22,7 @@
 from rotkehlchen.logging import DEFAULT_ANONYMIZED_LOGS, LoggingSettings, RotkehlchenLogsAdapter
 from rotkehlchen.poloniex import Poloniex
 from rotkehlchen.premium import premium_create_and_verify
+from rotkehlchen.typing import ResultCache, Timestamp
 from rotkehlchen.utils import (
     accounts_result,
     combine_stat_dicts,
@@ -42,7 +43,7 @@ class Rotkehlchen(object):
     def __init__(self, args):
         self.lock = Semaphore()
         self.lock.acquire()
-        self.results_cache: typing.ResultCache = dict()
+        self.results_cache: ResultCache = dict()
         self.premium = None
         self.connected_exchanges = []
 
@@ -660,6 +661,14 @@ def remove_exchange(self, name):
         self.data.db.remove_exchange(name)
         return True, ''
 
+    def query_periodic_data(self) -> Dict[str, Union[bool, Timestamp]]:
+        """Query for frequently changing data"""
+        result = {}
+        result['last_balance_save'] = self.data.db.get_last_balance_save_time()
+        result['eth_node_connection'] = self.blockchain.ethchain.connected
+        result['history_process_current_ts'] = self.accountant.currently_processed_timestamp
+        return result
+
     def shutdown(self):
         log.info("Shutting Down")
         self.shutdown_event.set()
diff --git a/rotkehlchen/server.py b/rotkehlchen/server.py
--- a/rotkehlchen/server.py
+++ b/rotkehlchen/server.py
@@ -245,9 +245,7 @@ def query_balances_async(self, save_data=False):
 
     def query_periodic_data(self):
         """Will query for some client data that can change frequently"""
-        result = {}
-        result['last_balance_save'] = self.rotkehlchen.data.db.get_last_balance_save_time()
-        result['eth_node_connection'] = self.rotkehlchen.blockchain.ethchain.connected
+        result = self.rotkehlchen.query_periodic_data()
         return process_result(result)
 
     def get_eth_tokens(self):


Test patch:
diff --git a/rotkehlchen/tests/fixtures/rotkehlchen.py b/rotkehlchen/tests/fixtures/rotkehlchen.py
--- a/rotkehlchen/tests/fixtures/rotkehlchen.py
+++ b/rotkehlchen/tests/fixtures/rotkehlchen.py
@@ -27,8 +27,12 @@ def cli_args(data_dir):
 
 
 @pytest.fixture()
-def rotkehlchen_instance(cli_args, username, blockchain):
+def rotkehlchen_instance(cli_args, username, blockchain, accountant):
+    """A partially mocked rotkehlchen instance"""
     r = Rotkehlchen(cli_args)
     r.data.unlock(username, '123', create_new=True)
+    # Remember accountant fixture has a mocked accounting data dir
+    # different to the usual user one
+    r.accountant = accountant
     r.blockchain = blockchain
     return r
diff --git a/rotkehlchen/tests/test_api.py b/rotkehlchen/tests/test_api.py
--- a/rotkehlchen/tests/test_api.py
+++ b/rotkehlchen/tests/test_api.py
@@ -27,3 +27,16 @@ def test_add_remove_eth_tokens(rotkehlchen_instance):
     rotkehlchen_instance.remove_owned_eth_tokens(['STORJ', 'GNO'])
     db_tokens_list = rotkehlchen_instance.data.db.get_owned_tokens()
     assert len(db_tokens_list) == 1 and db_tokens_list[0] == 'RDN'
+
+
+def test_periodic_query(rotkehlchen_instance):
+    """Test that periodiq query returns expected dict values"""
+    result = rotkehlchen_instance.query_periodic_data()
+
+    assert len(result) == 3
+    assert result['last_balance_save'] == 0
+    assert (
+        isinstance(result['eth_node_connection'], bool) and
+        result['eth_node_connection'] is False
+    )
+    assert result['history_process_current_ts'] == -1