Repository: rotki/rotki

Issue:
Periodically store balances data in the DB
Currently the query_balances() api endpoint, queries all the balance data for the user, but does not save it in the DB unless manually specified. These data are really important and could be used in lots of interesting ways later.

- [ ] Create a settings option to specify the frequency at which this data should be saved
- [ ] If all the locations the user has configured for querying are succesfully queries and we are within the query period then save balances data in the DB


Discussion:

Resolving patch:
diff --git a/rotkehlchen/data_handler.py b/rotkehlchen/data_handler.py
--- a/rotkehlchen/data_handler.py
+++ b/rotkehlchen/data_handler.py
@@ -4,6 +4,7 @@
 import zlib
 import base64
 import hashlib
+import time
 from typing import Tuple, Dict, List, Optional, cast
 from eth_utils.address import to_checksum_address
 
@@ -53,6 +54,7 @@
     'premium_should_sync',
     'include_crypto2crypto',
     'taxfree_after_period',
+    'balance_save_frequency',
 )
 
 
@@ -225,6 +227,16 @@ def set_settings(
         self.db.set_settings(settings)
         return True, msg
 
+    def should_save_balances(self) -> bool:
+        """ Returns whether or not we can save data to the database depending on
+        the balance data saving frequency setting"""
+        last_save = self.db.get_last_balance_save_time()
+        settings = self.db.get_settings()
+        # Setting is saved in hours, convert to seconds here
+        period = settings['balance_save_frequency'] * 60 * 60
+        now = cast(typing.Timestamp, int(time.time()))
+        return now - last_save > period
+
     def get_eth_accounts(self) -> List[typing.EthAddress]:
         blockchain_accounts = self.db.get_blockchain_accounts()
         return blockchain_accounts[S_ETH] if S_ETH in blockchain_accounts else []
diff --git a/rotkehlchen/db/dbhandler.py b/rotkehlchen/db/dbhandler.py
--- a/rotkehlchen/db/dbhandler.py
+++ b/rotkehlchen/db/dbhandler.py
@@ -25,6 +25,7 @@
 DEFAULT_INCLUDE_CRYPTO2CRYPTO = True
 DEFAULT_START_DATE = "01/08/2015"
 DEFAULT_UI_FLOATING_PRECISION = 2
+DEFAULT_BALANCE_SAVE_FREQUENCY = 24
 KDF_ITER = 64000
 
 
@@ -220,6 +221,8 @@ def get_settings(self) -> DBSettings:
                 settings['ui_floating_precision'] = int(q[1])
             elif q[0] == 'taxfree_after_period':
                 settings['taxfree_after_period'] = int(q[1])
+            elif q[0] == 'balance_save_frequency':
+                settings['balance_save_frequency'] = int(q[1])
             else:
                 settings[q[0]] = q[1]
 
@@ -234,6 +237,8 @@ def get_settings(self) -> DBSettings:
             settings['include_crypto2crypto'] = DEFAULT_INCLUDE_CRYPTO2CRYPTO
         if 'taxfree_after_period' not in settings:
             settings['taxfree_after_period'] = DEFAULT_TAXFREE_AFTER_PERIOD
+        if 'balance_save_frequency' not in settings:
+            settings['balance_save_frequency'] = DEFAULT_BALANCE_SAVE_FREQUENCY
         return settings
 
     def get_main_currency(self) -> typing.FiatAsset:
@@ -305,6 +310,17 @@ def add_multiple_balances(self, balances: AssetBalances) -> None:
         self.conn.commit()
         self.update_last_write()
 
+    def get_last_balance_save_time(self) -> typing.Timestamp:
+        cursor = self.conn.cursor()
+        query = cursor.execute(
+            'SELECT MAX(time) from timed_location_data'
+        )
+        query = query.fetchall()
+        if len(query) == 0:
+            return cast(typing.Timestamp, 0)
+
+        return cast(typing.Timestamp, int(query[0][0]))
+
     def add_multiple_location_data(self, location_data: LocationData) -> None:
         """Execute addition of multiple location data in the DB
 
@@ -597,7 +613,11 @@ def get_external_trades(
             '  link,'
             '  notes FROM trades WHERE location="external" '
         )
-        bindings: Union[Tuple[typing.Timestamp], Tuple[typing.Timestamp, typing.Timestamp]] = ()
+        bindings: Union[
+            Tuple,
+            Tuple[typing.Timestamp],
+            Tuple[typing.Timestamp, typing.Timestamp],
+        ] = ()
         if from_ts:
             query += 'AND time >= ? '
             bindings = (from_ts,)
diff --git a/rotkehlchen/rotkehlchen.py b/rotkehlchen/rotkehlchen.py
--- a/rotkehlchen/rotkehlchen.py
+++ b/rotkehlchen/rotkehlchen.py
@@ -404,13 +404,14 @@ def query_fiat_balances(self):
 
         return result
 
-    def query_balances(self, save_data=False):
+    def query_balances(self, requested_save_data=False):
         balances = {}
+        problem_free = True
         for exchange in self.connected_exchanges:
             exchange_balances, msg = getattr(self, exchange).query_balances()
             # If we got an error, disregard that exchange but make sure we don't save data
             if not exchange_balances:
-                save_data = False
+                problem_free = False
             else:
                 balances[exchange] = exchange_balances
 
@@ -448,7 +449,8 @@ def query_balances(self, save_data=False):
 
         result_dict = merge_dicts(combined, stats)
 
-        if save_data:
+        allowed_to_save = requested_save_data or self.data.should_save_balances()
+        if problem_free and allowed_to_save:
             self.data.save_balances_data(result_dict)
 
         # After adding it to the saved file we can overlay additional data that
@@ -515,9 +517,6 @@ def usd_to_main_currency(self, amount):
 
         return self.usd_to_main_currency_rate * amount
 
-    def get_settings(self):
-        return self.data.settings
-
     def setup_exchange(self, name, api_key, api_secret):
         if name not in SUPPORTED_EXCHANGES:
             return False, 'Attempted to register unsupported exchange {}'.format(name)


Test patch:
diff --git a/rotkehlchen/tests/test_db.py b/rotkehlchen/tests/test_db.py
--- a/rotkehlchen/tests/test_db.py
+++ b/rotkehlchen/tests/test_db.py
@@ -1,5 +1,6 @@
 import os
 import pytest
+import time
 from pysqlcipher3 import dbapi2 as sqlcipher
 from eth_utils.address import to_checksum_address
 
@@ -7,7 +8,8 @@
 from rotkehlchen.db.dbhandler import (
     ROTKEHLCHEN_DB_VERSION,
     DEFAULT_START_DATE,
-    DEFAULT_UI_FLOATING_PRECISION
+    DEFAULT_UI_FLOATING_PRECISION,
+    DEFAULT_BALANCE_SAVE_FREQUENCY,
 )
 from rotkehlchen.data_handler import DataHandler
 from rotkehlchen.errors import AuthenticationError, InputError
@@ -130,6 +132,7 @@ def test_writting_fetching_data(data_dir, username):
         'db_version': ROTKEHLCHEN_DB_VERSION,
         'include_crypto2crypto': True,
         'taxfree_after_period': YEAR_IN_SECONDS,
+        'balance_save_frequency': DEFAULT_BALANCE_SAVE_FREQUENCY,
     }
 
     # Check setting non-existing settings. Should be ignored
@@ -276,6 +279,7 @@ def test_settings_entry_types(data_dir, username):
         'taxfree_after_period': 1,
         'historical_data_start': '01/08/2015',
         'eth_rpc_port': '8545',
+        'balance_save_frequency': 24,
     })
 
     res = data.db.get_settings()
@@ -287,3 +291,18 @@ def test_settings_entry_types(data_dir, username):
     assert isinstance(res['taxfree_after_period'], int)
     assert isinstance(res['historical_data_start'], str)
     assert isinstance(res['eth_rpc_port'], str)
+    assert isinstance(res['balance_save_frequency'], int)
+
+
+def test_balance_save_frequency_check(data_dir, username):
+    data = DataHandler(data_dir)
+    data.unlock(username, '123', create_new=True)
+
+    now = int(time.time())
+    data.db.add_multiple_location_data([(
+        now - 24 * 60 * 60 + 20, 'kraken', '1500',
+    )])
+
+    assert not data.should_save_balances()
+    data.db.set_settings({'balance_save_frequency': 5})
+    assert data.should_save_balances()