Repository: rotki/rotki

Issue:
Graph for multi chain assets is missing historical data
## Problem Definition

I have wstEth in optimism, ethereum and arbitrum but the graph for the asset only contains the last balance that is a virtual point added by the frontend. As it can bee seen in the console the response from the backend is empty.

Clicking in one of the chains shows correctly the graph of balance

## Logs

![image](https://github.com/rotki/rotki/assets/5068010/dc8bfcf2-5647-4f40-8b2b-606f469163bb)

![image](https://github.com/rotki/rotki/assets/5068010/6dc9f7cd-af99-4d8d-87b5-e8ef6eabbcc5)

The payload in the request was

```
{"from_timestamp":0,"to_timestamp":1696855724,"collection_id":217}
```

to http://127.0.0.1:4242/api/1/statistics/balance




### System Description


Operating system: Macos 
Rotki version: Happens in the latest release and too in develop



Discussion:
I have USDC across 2 chains in production (1.30.2) and I don't see any problems. MATIC in 2 chains also fine. DAI in 4 chains. Works fine.

@yabirgb I don't think this is a generic bug. I think you will have to debug this yourself.

Resolving patch:
diff --git a/rotkehlchen/globaldb/upgrades/v5_v6.py b/rotkehlchen/globaldb/upgrades/v5_v6.py
--- a/rotkehlchen/globaldb/upgrades/v5_v6.py
+++ b/rotkehlchen/globaldb/upgrades/v5_v6.py
@@ -1,6 +1,8 @@
 import logging
 from typing import TYPE_CHECKING
 
+from eth_utils.address import to_checksum_address
+
 from rotkehlchen.db.utils import update_table_schema
 from rotkehlchen.logging import RotkehlchenLogsAdapter
 from rotkehlchen.types import CacheType
@@ -69,9 +71,38 @@ def _update_multiasset_mappings(cursor: 'DBCursor') -> None:
     log.debug('Exit _upgrade_multiasset_mappings')
 
 
+def _fix_asset_in_multiasset_mappings(cursor: 'DBCursor') -> None:
+    """
+    Fix some assets that were not using checksummed addresses in the identifiers
+    https://github.com/rotki/rotki/issues/6717
+    """
+    log.debug('Enter _fix_asset_in_multiasset_mappings')
+    fixes = []
+    cursor.execute('SELECT asset FROM multiasset_mappings')
+    for (asset_id,) in cursor:
+        if not asset_id.startswith('eip155'):
+            continue
+        try:
+            address = asset_id.split(':')[-1]
+            checksummed_address = to_checksum_address(address)
+        except (IndexError, ValueError):
+            log.error(f'Unexpected asset identifier {asset_id} found in the multiassets mapping')
+            continue
+
+        if checksummed_address == address:
+            continue
+
+        new_id = asset_id.replace(address, checksummed_address)
+        fixes.append((new_id, asset_id))
+
+    cursor.executemany('UPDATE multiasset_mappings SET asset=? WHERE asset=?', fixes)
+    log.debug('Exit _fix_asset_in_multiasset_mappings')
+
+
 def migrate_to_v6(connection: 'DBConnection') -> None:
     """This globalDB upgrade does the following:
     - Adds the `unique_cache` table.
+    - Fixes the multiassets mappings ids to use checksummed addresses
     - Upgrades the multiasset_mappings to have unique collection_id+asset
 
     This upgrade takes place in v1.31.0
@@ -80,4 +111,5 @@ def migrate_to_v6(connection: 'DBConnection') -> None:
 
     with connection.write_ctx() as cursor:
         _create_and_populate_unique_cache_table(cursor)
+        _fix_asset_in_multiasset_mappings(cursor)
         _update_multiasset_mappings(cursor)


Test patch:
diff --git a/rotkehlchen/tests/unit/globaldb/test_correctness.py b/rotkehlchen/tests/unit/globaldb/test_correctness.py
new file mode 100644
--- /dev/null
+++ b/rotkehlchen/tests/unit/globaldb/test_correctness.py
@@ -0,0 +1,25 @@
+from eth_utils.address import to_checksum_address
+
+from rotkehlchen.globaldb.handler import GlobalDBHandler
+
+
+def test_checksummed_values(globaldb: GlobalDBHandler):
+    """Test that addresses and identifiers have checksummed addresses"""
+    cursor = globaldb.conn.cursor()
+    cursor.execute(
+        'SELECT evm_tokens.identifier, address, name, symbol FROM evm_tokens JOIN '
+        'common_asset_details ON evm_tokens.identifier=common_asset_details.identifier '
+        'JOIN assets ON evm_tokens.identifier=assets.identifier',
+    )
+    identifiers = set()
+    for identifier, address, name, symbol in cursor:
+        checksummed_address = to_checksum_address(address)
+        assert checksummed_address == address
+        assert checksummed_address in identifier
+        assert name != identifier
+        assert symbol != identifier
+        identifiers.add(identifier)
+
+    cursor.execute('SELECT asset FROM multiasset_mappings')
+    assert {x[0] for x in cursor if not x[0].startswith('eip155')}.issubset(identifiers)
+    cursor.close()
diff --git a/rotkehlchen/tests/unit/globaldb/test_upgrades.py b/rotkehlchen/tests/unit/globaldb/test_upgrades.py
--- a/rotkehlchen/tests/unit/globaldb/test_upgrades.py
+++ b/rotkehlchen/tests/unit/globaldb/test_upgrades.py
@@ -5,6 +5,7 @@
 from sqlite3 import IntegrityError
 
 import pytest
+from eth_utils.address import to_checksum_address
 from freezegun import freeze_time
 
 from rotkehlchen.assets.types import AssetType
@@ -73,11 +74,11 @@ def _count_sql_file_sentences(file_name: str, skip_statements: int = 0):
 @pytest.mark.parametrize('reload_user_assets', [False])
 @pytest.mark.parametrize('custom_globaldb', ['v2_global.db'])
 @pytest.mark.parametrize('target_globaldb_version', [2])
-def test_upgrade_v2_v3(globaldb):
+def test_upgrade_v2_v3(globaldb: GlobalDBHandler):
     """Test globalDB upgrade v2->v3"""
     # Check the state before upgrading
     with globaldb.conn.read_ctx() as cursor:
-        assert globaldb.get_setting_value('version', None) == 2
+        assert globaldb.get_setting_value('version', 0) == 2
         cursor.execute(
             'SELECT COUNT(*) FROM price_history WHERE from_asset=? OR to_asset=?',
             ('BIFI', 'BIFI'),
@@ -103,11 +104,11 @@ def test_upgrade_v2_v3(globaldb):
         patch_for_globaldb_upgrade_to(stack, 3)
         maybe_upgrade_globaldb(
             connection=globaldb.conn,
-            global_dir=globaldb._data_directory / 'global_data',
+            global_dir=globaldb._data_directory / 'global_data',  # type: ignore
             db_filename=GLOBAL_DB_FILENAME,
         )
 
-    assert globaldb.get_setting_value('version', None) == 3
+    assert globaldb.get_setting_value('version', 0) == 3
     assets_inserted_by_update = _count_sql_file_sentences('globaldb_v2_v3_assets.sql')
     with globaldb.conn.read_ctx() as cursor:
         # test that we have the same number of assets before and after the migration
@@ -190,11 +191,11 @@ def test_upgrade_v2_v3(globaldb):
 @pytest.mark.parametrize('custom_globaldb', ['v3_global.db'])
 @pytest.mark.parametrize('target_globaldb_version', [3])
 @pytest.mark.parametrize('reload_user_assets', [False])
-def test_upgrade_v3_v4(globaldb):
+def test_upgrade_v3_v4(globaldb: GlobalDBHandler):
     """Test the global DB upgrade from v3 to v4"""
     # Check the state before upgrading
     with globaldb.conn.read_ctx() as cursor:
-        assert globaldb.get_setting_value('version', None) == 3
+        assert globaldb.get_setting_value('version', 0) == 3
         cursor.execute(
             'SELECT COUNT(*) FROM sqlite_master WHERE type="table" and name IN (?, ?)',
             ('contract_abi', 'contract_data'),
@@ -208,11 +209,11 @@ def test_upgrade_v3_v4(globaldb):
         patch_for_globaldb_upgrade_to(stack, 4)
         maybe_upgrade_globaldb(
             connection=globaldb.conn,
-            global_dir=globaldb._data_directory / 'global_data',
+            global_dir=globaldb._data_directory / 'global_data',  # type: ignore
             db_filename=GLOBAL_DB_FILENAME,
         )
 
-    assert globaldb.get_setting_value('version', None) == 4
+    assert globaldb.get_setting_value('version', 0) == 4
     with globaldb.conn.read_ctx() as cursor:
         cursor.execute(
             'SELECT COUNT(*) FROM sqlite_master WHERE type="table" and name IN (?, ?)',
@@ -303,11 +304,11 @@ def test_upgrade_v3_v4(globaldb):
 @pytest.mark.parametrize('target_globaldb_version', [4])
 @pytest.mark.parametrize('reload_user_assets', [False])
 @freeze_time('2023-03-20')  # freezing time just to make sure comparisons of timestamps won't fail
-def test_upgrade_v4_v5(globaldb):
+def test_upgrade_v4_v5(globaldb: GlobalDBHandler):
     """Test the global DB upgrade from v4 to v5"""
     # Check the state before upgrading
     with globaldb.conn.read_ctx() as cursor:
-        assert globaldb.get_setting_value('version', None) == 4
+        assert globaldb.get_setting_value('version', 0) == 4
         cursor.execute(
             'SELECT COUNT(*) FROM sqlite_master WHERE type="table" and name=?',
             ('default_rpc_nodes',),
@@ -335,11 +336,11 @@ def test_upgrade_v4_v5(globaldb):
         patch_for_globaldb_upgrade_to(stack, 5)
         maybe_upgrade_globaldb(
             connection=globaldb.conn,
-            global_dir=globaldb._data_directory / 'global_data',
+            global_dir=globaldb._data_directory / 'global_data',  # type: ignore
             db_filename=GLOBAL_DB_FILENAME,
         )
 
-    assert globaldb.get_setting_value('version', None) == 5
+    assert globaldb.get_setting_value('version', 0) == 5
 
     with globaldb.conn.read_ctx() as cursor:
         cursor.execute(
@@ -383,7 +384,7 @@ def test_upgrade_v4_v5(globaldb):
 @pytest.mark.parametrize('custom_globaldb', ['v5_global.db'])
 @pytest.mark.parametrize('target_globaldb_version', [5])
 @pytest.mark.parametrize('reload_user_assets', [False])
-def test_upgrade_v5_v6(globaldb):
+def test_upgrade_v5_v6(globaldb: GlobalDBHandler):
     """Test the global DB upgrade from v5 to v6"""
     # Check the state before upgrading
     with globaldb.conn.read_ctx() as cursor:
@@ -407,7 +408,15 @@ def test_upgrade_v5_v6(globaldb):
             unique_keys_to_num_entries[key_part] = int(cursor.fetchone()[0])
             gen_cache_unique_key_content += unique_keys_to_num_entries[key_part]
 
-        old_multiasset_mappings = cursor.execute('SELECT * FROM multiasset_mappings ORDER BY collection_id, asset').fetchall()  # noqa: E501
+        # create a set of the old entries ensuring that the rows have checksummed addresses
+        # for the identifiers in multiasset_mappings
+        old_multiasset_mappings = set()
+        cursor.execute('SELECT * FROM multiasset_mappings ORDER BY collection_id, asset')
+        for collection_id, asset_id in cursor:
+            address = asset_id.split(':')[-1]
+            checksummed_address = to_checksum_address(address)
+            new_id = asset_id.replace(address, checksummed_address)
+            old_multiasset_mappings.add((collection_id, new_id))
 
     with globaldb.conn.write_ctx() as write_cursor:
         # add some dummy data into cache to verify behaviour during transfer.
@@ -439,10 +448,10 @@ def test_upgrade_v5_v6(globaldb):
         patch_for_globaldb_upgrade_to(stack, 6)
         maybe_upgrade_globaldb(
             connection=globaldb.conn,
-            global_dir=globaldb._data_directory / 'global_data',
+            global_dir=globaldb._data_directory / 'global_data',  # type: ignore
             db_filename=GLOBAL_DB_FILENAME,
         )
-    assert globaldb.get_setting_value('version', None) == 6
+    assert globaldb.get_setting_value('version', 0) == 6
     with globaldb.conn.read_ctx() as cursor:
         # check that unique_cache table is present in the database
         cursor.execute(
@@ -451,7 +460,7 @@ def test_upgrade_v5_v6(globaldb):
         )
         assert cursor.fetchone()[0] == 1
         # check that of dummy entry, only first value is transferred to unique_cache
-        value = globaldb_get_unique_cache_value(cursor, (next(iter(V5_V6_UPGRADE_UNIQUE_CACHE_KEYS)), 'test'))  # noqa: E501
+        value = globaldb_get_unique_cache_value(cursor, (next(iter(V5_V6_UPGRADE_UNIQUE_CACHE_KEYS)), 'test'))  # type: ignore  # noqa: E501
         assert value == values[0]
 
         with globaldb.conn.write_ctx() as write_cursor:
@@ -482,8 +491,9 @@ def test_upgrade_v5_v6(globaldb):
         gen_cache_content_after = cursor.execute('SELECT COUNT(*) FROM general_cache').fetchone()[0]  # noqa: E501
         assert gen_cache_unique_key_content == unique_cache_content
         assert gen_cache_content_before == gen_cache_content_after + gen_cache_unique_key_content
-
-        assert cursor.execute('SELECT * FROM multiasset_mappings ORDER BY collection_id, asset').fetchall() == old_multiasset_mappings  # noqa: E501
+        assert set(cursor.execute(
+            'SELECT * FROM multiasset_mappings ORDER BY collection_id, asset',
+        )) == old_multiasset_mappings
 
 
 @pytest.mark.parametrize('custom_globaldb', ['v2_global.db'])
@@ -522,19 +532,19 @@ def test_unfinished_upgrades(globaldb: GlobalDBHandler):
 @pytest.mark.parametrize('custom_globaldb', ['v2_global.db'])
 @pytest.mark.parametrize('target_globaldb_version', [2])
 @pytest.mark.parametrize('reload_user_assets', [False])
-def test_applying_all_upgrade(globaldb):
+def test_applying_all_upgrade(globaldb: GlobalDBHandler):
     """Test globalDB upgrade from v2 to latest"""
     # Check the state before upgrading
-    assert globaldb.get_setting_value('version', None) == 2
+    assert globaldb.get_setting_value('version', 0) == 2
     with globaldb.conn.cursor() as cursor:
-        assert cursor.execute('SELECT COUNT(*) from assets WHERE identifier="eip155:/erc20:0x32c6fcC9bC912C4A30cd53D2E606461e44B77AF2"')[0] == 0  # noqa: E501
+        assert cursor.execute('SELECT COUNT(*) from assets WHERE identifier="eip155:/erc20:0x32c6fcC9bC912C4A30cd53D2E606461e44B77AF2"').fetchone()[0] == 0  # noqa: E501
 
     maybe_upgrade_globaldb(
         connection=globaldb.conn,
-        global_dir=globaldb._data_directory / 'global_data',
+        global_dir=globaldb._data_directory / 'global_data',  # type: ignore
         db_filename=GLOBAL_DB_FILENAME,
     )
 
-    assert globaldb.get_setting_value('version', None) == GLOBAL_DB_VERSION
+    assert globaldb.get_setting_value('version', 0) == GLOBAL_DB_VERSION
     with globaldb.conn.cursor() as cursor:
-        assert cursor.execute('SELECT COUNT(*) from assets WHERE identifier="eip155:/erc20:0x32c6fcC9bC912C4A30cd53D2E606461e44B77AF2"')[0] == 1  # noqa: E501
+        assert cursor.execute('SELECT COUNT(*) from assets WHERE identifier="eip155:/erc20:0x32c6fcC9bC912C4A30cd53D2E606461e44B77AF2"').fetchone()[0] == 1  # noqa: E501
diff --git a/rotkehlchen/tests/utils/globaldb.py b/rotkehlchen/tests/utils/globaldb.py
--- a/rotkehlchen/tests/utils/globaldb.py
+++ b/rotkehlchen/tests/utils/globaldb.py
@@ -79,7 +79,7 @@ def create_initial_expected_globaldb_test_tokens():
 )
 
 
-def patch_for_globaldb_upgrade_to(stack: ExitStack, version: Literal[2, 3, 4, 6]) -> ExitStack:
+def patch_for_globaldb_upgrade_to(stack: ExitStack, version: Literal[2, 3, 4, 5, 6]) -> ExitStack:
     stack.enter_context(
         patch(
             'rotkehlchen.globaldb.upgrades.manager.GLOBAL_DB_VERSION',
diff --git a/rotkehlchen/tests/utils/history_base_entry.py b/rotkehlchen/tests/utils/history_base_entry.py
--- a/rotkehlchen/tests/utils/history_base_entry.py
+++ b/rotkehlchen/tests/utils/history_base_entry.py
@@ -64,8 +64,7 @@ def entry_to_input_dict(
         serialized['fee_recipient'] = serialized.pop('location_label')
         serialized['is_mev_reward'] = serialized.pop('event_subtype') == HistoryEventSubType.MEV_REWARD.serialize()  # noqa: E501
     elif entry.entry_type == HistoryBaseEntryType.ETH_DEPOSIT_EVENT:
-        if include_identifier is False:
-            # when creating a eth deposit event we don't include the event_identifier
+        if include_identifier is False:  # when creating an eth deposit event we don't include the event_identifier  # noqa: E501
             serialized.pop('event_identifier')
         serialized['depositor'] = serialized.pop('location_label')
     return serialized