Repository: rotki/rotki

Issue:
 test_query_all_balances is flaky after save data change
## Problem Definition

Here is the trace:

```
 test_query_all_balances[added_exchanges0-owned_eth_tokens0-btc_accounts0-2-False] 

rotkehlchen_api_server_with_exchanges = <rotkehlchen.api.server.APIServer object at 0x7fe9475f32b0>
ethereum_accounts = ['0xe2f155E7c1C99b2b9DEcdC6615967820FF60135a', '0x12Debd85d4D8034770937047d08af87F0f827098']
btc_accounts = ['1BvBMSEYstWetqTFn5Au4m4GFg7xJaNVN2', '1CounterpartyXXXXXXXXXXXXXXXUWLpVr']

    @pytest.mark.parametrize('should_mock_current_price_queries', [False])
    @pytest.mark.parametrize('number_of_eth_accounts', [2])
    @pytest.mark.parametrize('btc_accounts', [[UNIT_BTC_ADDRESS1, UNIT_BTC_ADDRESS2]])
    @pytest.mark.parametrize('owned_eth_tokens', [[A_RDN]])
    @pytest.mark.parametrize('added_exchanges', [('binance', 'poloniex')])
    def test_query_all_balances(
            rotkehlchen_api_server_with_exchanges,
            ethereum_accounts,
            btc_accounts,
    ):
        """Test that using the query all balances endpoint works
    
        Test that balances from various sources are returned. Such as exchanges,
        blockchain and FIAT"""
        # Disable caching of query results
        rotki = rotkehlchen_api_server_with_exchanges.rest_api.rotkehlchen
        rotki.chain_manager.cache_ttl_secs = 0
        setup = setup_balances(rotki, ethereum_accounts, btc_accounts)
        # Test that all balances request saves data on a fresh account
        with ExitStack() as stack:
            setup.enter_all_patches(stack)
            response = requests.get(
                api_url_for(
                    rotkehlchen_api_server_with_exchanges,
                    "allbalancesresource",
                ),
            )
    
        assert_proper_response(response)
        json_data = response.json()
        assert_all_balances(
            data=json_data,
            db=rotki.data.db,
            expected_data_in_db=True,
            setup=setup,
        )
    
        last_save_timestamp = rotki.data.db.get_last_balance_save_time()
    
        # now do the same but check to see if the balance save frequency delay works
        # and thus data will not be saved
        with ExitStack() as stack:
            setup.enter_all_patches(stack)
            response = requests.get(
                api_url_for(
                    rotkehlchen_api_server_with_exchanges,
                    "allbalancesresource",
                ),
            )
        assert_proper_response(response)
        new_save_timestamp = rotki.data.db.get_last_balance_save_time()
        assert last_save_timestamp == new_save_timestamp
    
        # now do the same but test that balance are saved since the balance save frequency delay
        # is overriden via `save_data` = True
        with ExitStack() as stack:
            setup.enter_all_patches(stack)
            response = requests.get(
                api_url_for(
                    rotkehlchen_api_server_with_exchanges,
                    "allbalancesresource",
                ), json={'save_data': True},
            )
>       assert_proper_response(response)

rotkehlchen/tests/api/test_balances.py:186: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

response = <Response [500]>, status_code = <HTTPStatus.OK: 200>

    def assert_proper_response(
            response: Optional[requests.Response],
            status_code=HTTPStatus.OK,
    ) -> None:
        assert (
            response is not None and
>           response.status_code == status_code and
            response.headers["Content-Type"] == "application/json"
        )
E       AssertionError

rotkehlchen/tests/utils/api.py:56: AssertionError
travis_fold:start:py-8993.stdout-setup.0
---------------------------- Captured stdout setup -----------------------------
Rotki API server is running at: 127.0.0.1:29877
travis_fold:end:py-8993.stdout-setup.0
travis_fold:start:py-8993.log-setup.0
------------------------------ Captured log setup ------------------------------
WARNING  rotkehlchen.chain.ethereum.manager:manager.py:132 Could not connect to an ethereum node. Will use etherscan only
travis_fold:end:py-8993.log-setup.0
travis_fold:start:py-8993.log.0
------------------------------ Captured log call -------------------------------
WARNING  rotkehlchen.user_messages:user_messages.py:18 Found unknown binance asset IDONTEXIST.  Ignoring its balance query.
WARNING  rotkehlchen.user_messages:user_messages.py:18 Found unsupported binance asset ETF.  Ignoring its balance query.
WARNING  rotkehlchen.user_messages:user_messages.py:18 Found unknown poloniex asset IDONTEXIST.  Ignoring its balance query.
WARNING  rotkehlchen.user_messages:user_messages.py:18 Found unsupported poloniex asset CNOTE.  Ignoring its balance query.
ERROR    rotkehlchen.api.server:app.py:1891 Exception on /api/1/balances/ [GET]
Traceback (most recent call last):
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/flask/app.py", line 1949, in full_dispatch_request
    rv = self.dispatch_request()
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/flask/app.py", line 1935, in dispatch_request
    return self.view_functions[rule.endpoint](**req.view_args)
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/flask_restful/__init__.py", line 458, in wrapper
    resp = resource(*args, **kwargs)
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/flask/views.py", line 89, in view
    return self.dispatch_request(*args, **kwargs)
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/flask_restful/__init__.py", line 573, in dispatch_request
    resp = meth(*args, **kwargs)
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/webargs/core.py", line 366, in wrapper
    return func(*args, **kwargs)
  File "/home/travis/build/rotki/rotki/rotkehlchen/api/v1/resources.py", line 253, in get
    ignore_cache=ignore_cache,
  File "/home/travis/build/rotki/rotki/rotkehlchen/api/rest.py", line 108, in wrapper
    return f(wrappingobj, *args, **kwargs)
  File "/home/travis/build/rotki/rotki/rotkehlchen/api/rest.py", line 331, in query_all_balances
    response = self._query_all_balances(save_data=save_data, ignore_cache=ignore_cache)
  File "/home/travis/build/rotki/rotki/rotkehlchen/api/rest.py", line 313, in _query_all_balances
    ignore_cache=ignore_cache,
  File "/home/travis/build/rotki/rotki/rotkehlchen/rotkehlchen.py", line 593, in query_balances
    self.data.save_balances_data(data=result_dict, timestamp=timestamp)
  File "/home/travis/build/rotki/rotki/rotkehlchen/data_handler.py", line 117, in save_balances_data
    self.db.write_balances_data(data=data, timestamp=timestamp)
  File "/home/travis/build/rotki/rotki/rotkehlchen/db/dbhandler.py", line 1083, in write_balances_data
    self.add_multiple_balances(balances)
  File "/home/travis/build/rotki/rotki/rotkehlchen/db/dbhandler.py", line 626, in add_multiple_balances
    (entry.time, entry.asset.identifier, entry.amount, entry.usd_value),
pysqlcipher3.dbapi2.IntegrityError: UNIQUE constraint failed: timed_balances.time, timed_balances.currency
travis_fold:end:py-8993.log.0
```


Discussion:

Resolving patch:
diff --git a/rotkehlchen/db/dbhandler.py b/rotkehlchen/db/dbhandler.py
--- a/rotkehlchen/db/dbhandler.py
+++ b/rotkehlchen/db/dbhandler.py
@@ -619,12 +619,19 @@ def add_multiple_balances(self, balances: List[AssetBalance]) -> None:
         cursor = self.conn.cursor()
 
         for entry in balances:
-            cursor.execute(
-                'INSERT INTO timed_balances('
-                '    time, currency, amount, usd_value) '
-                ' VALUES(?, ?, ?, ?)',
-                (entry.time, entry.asset.identifier, entry.amount, entry.usd_value),
-            )
+            try:
+                cursor.execute(
+                    'INSERT INTO timed_balances('
+                    '    time, currency, amount, usd_value) '
+                    ' VALUES(?, ?, ?, ?)',
+                    (entry.time, entry.asset.identifier, entry.amount, entry.usd_value),
+                )
+            except sqlcipher.IntegrityError:  # pylint: disable=no-member
+                self.msg_aggregator.add_warning(
+                    f'Tried to add a timed_balance for {entry.asset.identifier} at'
+                    f' already existing timestamp {entry.time}. Skipping.',
+                )
+                continue
         self.conn.commit()
         self.update_last_write()
 
@@ -674,12 +681,20 @@ def add_multiple_location_data(self, location_data: List[LocationData]) -> None:
         """Execute addition of multiple location data in the DB"""
         cursor = self.conn.cursor()
         for entry in location_data:
-            cursor.execute(
-                'INSERT INTO timed_location_data('
-                '    time, location, usd_value) '
-                ' VALUES(?, ?, ?)',
-                (entry.time, entry.location, entry.usd_value),
-            )
+            try:
+                cursor.execute(
+                    'INSERT INTO timed_location_data('
+                    '    time, location, usd_value) '
+                    ' VALUES(?, ?, ?)',
+                    (entry.time, entry.location, entry.usd_value),
+                )
+            except sqlcipher.IntegrityError:  # pylint: disable=no-member
+                self.msg_aggregator.add_warning(
+                    f'Tried to add a timed_location_data for '
+                    f'{str(deserialize_location_from_db(entry.location))} at'
+                    f' already existing timestamp {entry.time}. Skipping.',
+                )
+                continue
         self.conn.commit()
         self.update_last_write()
 


Test patch:
diff --git a/rotkehlchen/tests/api/test_balances.py b/rotkehlchen/tests/api/test_balances.py
--- a/rotkehlchen/tests/api/test_balances.py
+++ b/rotkehlchen/tests/api/test_balances.py
@@ -2,6 +2,7 @@
 from http import HTTPStatus
 from unittest.mock import patch
 
+import gevent
 import pytest
 import requests
 
@@ -173,6 +174,9 @@ def test_query_all_balances(
     new_save_timestamp = rotki.data.db.get_last_balance_save_time()
     assert last_save_timestamp == new_save_timestamp
 
+    # wait for at least 1 second to make sure that new balances can be saved.
+    # Can't save balances again if it's the same timestamp
+    gevent.sleep(1)
     # now do the same but test that balance are saved since the balance save frequency delay
     # is overriden via `save_data` = True
     with ExitStack() as stack:
diff --git a/rotkehlchen/tests/db/test_db.py b/rotkehlchen/tests/db/test_db.py
--- a/rotkehlchen/tests/db/test_db.py
+++ b/rotkehlchen/tests/db/test_db.py
@@ -975,6 +975,51 @@ def test_can_unlock_db_with_disabled_taxfree_after_period(data_dir, username):
     assert settings.taxfree_after_period is None
 
 
+def test_multiple_location_data_and_balances_same_timestamp(data_dir, username):
+    """Test that adding location and balance data with same timestamp does not crash.
+
+    Regression test for https://github.com/rotki/rotki/issues/1043
+    """
+    msg_aggregator = MessagesAggregator()
+    username = 'foo'
+    userdata_dir = os.path.join(data_dir, username)
+    os.mkdir(userdata_dir)
+    db = DBHandler(userdata_dir, '123', msg_aggregator)
+
+    balances = [
+        AssetBalance(
+            time=1590676728,
+            asset=A_BTC,
+            amount='1.0',
+            usd_value='8500',
+        ), AssetBalance(
+            time=1590676728,
+            asset=A_BTC,
+            amount='1.1',
+            usd_value='9100',
+        ),
+    ]
+    db.add_multiple_balances(balances)
+    balances = db.query_timed_balances(from_ts=0, to_ts=1590676728, asset=A_BTC)
+    assert len(balances) == 1
+
+    locations = [
+        LocationData(
+            time=1590676728,
+            location='H',
+            usd_value='55',
+        ), LocationData(
+            time=1590676728,
+            location='H',
+            usd_value='56',
+        ),
+    ]
+    db.add_multiple_location_data(locations)
+    locations = db.get_latest_location_value_distribution()
+    assert len(locations) == 1
+    assert locations[0].usd_value == '55'
+
+
 def test_set_get_rotkehlchen_premium_credentials(data_dir, username):
     """Test that setting the premium credentials and getting them back from the DB works
     """