Repository: rotki/rotki

Issue:
Remotely update EVM contracts information in the globaldb
## Abstract

Allow to update the set of contracts abi's as we do for updating the list of assets

## Motivation

At the moment if we want to add new contracts to the user's globaldb we have to make a globaldb migration and add them. We can avoid migrations by using an update sysmtem

Related issues are

- #5532
- #5236



Discussion:

Resolving patch:
diff --git a/rotkehlchen/db/updates.py b/rotkehlchen/db/updates.py
--- a/rotkehlchen/db/updates.py
+++ b/rotkehlchen/db/updates.py
@@ -2,7 +2,7 @@
 import logging
 import sys
 from enum import Enum
-from typing import TYPE_CHECKING, Any, Final
+from typing import TYPE_CHECKING, Any, Final, Literal, Union, overload
 
 import requests
 
@@ -26,6 +26,7 @@
 LAST_DATA_UPDATES_KEY: Final = 'last_data_updates_ts'
 SPAM_ASSETS_URL = 'https://raw.githubusercontent.com/rotki/data/{branch}/updates/spam_assets/v{version}.json'  # noqa: E501
 RPC_NODES_URL = 'https://raw.githubusercontent.com/rotki/data/{branch}/updates/rpc_nodes/v{version}.json'  # noqa: E501
+CONTRACTS_URL = 'https://raw.githubusercontent.com/rotki/data/{branch}/updates/contracts/v{version}.json'  # noqa: E501
 
 
 class UpdateType(Enum):
@@ -83,13 +84,13 @@ def _get_remote_info_json(self) -> dict[str, Any]:
 
         return json_data
 
-    def update_spam_assets(self, local_version: int, to_version: int) -> None:
+    def update_spam_assets(self, from_version: int, to_version: int) -> None:
         """
         Check the updates in the inclusive range [local_version + 1, to_version] and update the
         spam assets for those versions. Assets are also added to the globaldb if they don't exist
         locally
         """
-        for version in range(local_version + 1, to_version + 1):
+        for version in range(from_version + 1, to_version + 1):
             success, updated_assets = self._get_file_content(
                 file_url=SPAM_ASSETS_URL.format(branch=self.branch, version=version),
                 update_type=UpdateType.SPAM_ASSETS,
@@ -109,7 +110,7 @@ def update_spam_assets(self, local_version: int, to_version: int) -> None:
                     (UpdateType.SPAM_ASSETS.serialize(), version),
                 )
 
-    def update_rpc_nodes(self, to_version: int) -> None:
+    def update_rpc_nodes(self, from_version: int, to_version: int) -> None:  # pylint: disable=unused-argument  # from_version is here to have the same format as other updating functions  # noqa: E501
         """Checks the update for the latest version and update the default rpc nodes in the global db.
 
         It also updates the user db with these default nodes.
@@ -154,29 +155,89 @@ def update_rpc_nodes(self, to_version: int) -> None:
             new_default_nodes=new_default_nodes,
         )
 
+    def update_contracts(self, from_version: int, to_version: int) -> None:
+        """
+        Updates evm contracts and ABIs in globaldb.
+        If an ABI already exists in the db, it is reused.
+        If a contract (address + chain id) exists, it is replaced by the remote counterpart.
+        """
+        for version in range(from_version + 1, to_version + 1):
+            success, updated_contracts = self._get_file_content(
+                file_url=CONTRACTS_URL.format(branch=self.branch, version=version),
+                update_type=UpdateType.CONTRACTS,
+            )
+            if success is False:
+                return
+
+            log.info(f'Applying update for contracts to v{version}')
+            remote_id_to_local_id = {}
+            with GlobalDBHandler().conn.read_ctx() as cursor:
+                for single_abi_data in updated_contracts['abis_data']:
+                    serialized_abi = json.dumps(single_abi_data['value'], separators=(',', ':'))
+
+                    # check if the abi is already present in the database
+                    existing_abi_id = cursor.execute(
+                        'SELECT id FROM contract_abi WHERE value=?',
+                        (serialized_abi,),
+                    ).fetchone()
+
+                    if existing_abi_id is not None:
+                        abi_id = existing_abi_id[0]
+                    else:
+                        with GlobalDBHandler().conn.write_ctx() as write_cursor:
+                            write_cursor.execute(
+                                'INSERT INTO contract_abi(name, value) VALUES(?, ?)',
+                                (single_abi_data['name'], serialized_abi),
+                            )
+                            abi_id = write_cursor.lastrowid
+                    # store a mapping of the virtual id to the id in the user's globaldb
+                    remote_id_to_local_id[single_abi_data['id']] = abi_id
+
+            new_contracts_data = []
+            for single_contract_data in updated_contracts['contracts_data']:
+                if single_contract_data['abi'] not in remote_id_to_local_id:
+                    self.msg_aggregator.add_error(
+                        f'ABI with id {single_contract_data["abi"]} was missing in a contracts '
+                        f'update. Please report it to the rotki team.',
+                    )
+                    continue
+                new_contracts_data.append((
+                    single_contract_data['address'],
+                    single_contract_data['chain_id'],
+                    single_contract_data['name'],
+                    remote_id_to_local_id[single_contract_data['abi']],
+                    single_contract_data['deployed_block'],
+                ))
+
+            with GlobalDBHandler().conn.write_ctx() as write_cursor:
+                write_cursor.executemany(
+                    'INSERT OR REPLACE INTO contract_data(address, chain_id, name, abi, deployed_block) VALUES(?, ?, ?, ?, ?)',  # noqa: E501
+                    new_contracts_data,
+                )
+
+            with self.user_db.conn.write_ctx() as write_cursor:
+                write_cursor.execute(
+                    'INSERT OR REPLACE INTO settings(name, value) VALUES (?, ?)',
+                    (UpdateType.CONTRACTS.serialize(), version),
+                )
+
     def check_for_updates(self) -> None:
         """Retrieve the information about the latest available update"""
         remote_information = self._get_remote_info_json()
 
-        # Get latest applied versions
-        with self.user_db.conn.read_ctx() as cursor:
-            local_spam_assets_version = self._check_for_last_version(
-                cursor=cursor,
-                update_type=UpdateType.SPAM_ASSETS,
-            )
-            local_rpc_nodes_version = self._check_for_last_version(
-                cursor=cursor,
-                update_type=UpdateType.RPC_NODES,
-            )
-
-        # Update all remote data
-        latest_spam_assets_version = remote_information[UpdateType.SPAM_ASSETS.value]['latest']
-        if local_spam_assets_version < latest_spam_assets_version:
-            self.update_spam_assets(local_spam_assets_version, latest_spam_assets_version)
+        for update_type in UpdateType:
+            # Get latest applied version
+            with self.user_db.conn.read_ctx() as cursor:
+                local_version = self._check_for_last_version(
+                    cursor=cursor,
+                    update_type=update_type,
+                )
 
-        latest_rpc_nodes_version = remote_information[UpdateType.RPC_NODES.value]['latest']
-        if local_rpc_nodes_version < latest_rpc_nodes_version:
-            self.update_rpc_nodes(latest_rpc_nodes_version)
+            # Update all remote data
+            latest_version = remote_information[update_type.value]['latest']
+            if local_version < latest_version:
+                update_function = getattr(self, f'update_{update_type.value}')
+                update_function(from_version=local_version, to_version=latest_version)
 
         with self.user_db.user_write() as cursor:
             cursor.execute(  # remember last time data updates were detected
@@ -194,10 +255,26 @@ def _check_for_last_version(cursor: 'DBCursor', update_type: UpdateType) -> int:
         return int(found_version[0]) if found_version is not None else 0
 
     @staticmethod
+    @overload
     def _get_file_content(
             file_url: str,
-            update_type: UpdateType,
+            update_type: Literal[UpdateType.SPAM_ASSETS, UpdateType.RPC_NODES],
     ) -> tuple[bool, list[dict[str, Any]]]:
+        ...
+
+    @staticmethod
+    @overload
+    def _get_file_content(
+            file_url: str,
+            update_type: Literal[UpdateType.CONTRACTS],
+    ) -> tuple[bool, dict[str, Any]]:
+        ...
+
+    @staticmethod
+    def _get_file_content(
+            file_url: str,
+            update_type: UpdateType,
+    ) -> tuple[bool, Union[list[dict[str, Any]], dict[str, Any]]]:
         """Retrieves the content of data to be updated.
 
         Returns True/False for success along with the content.


Test patch:
diff --git a/rotkehlchen/tests/unit/test_data_updates.py b/rotkehlchen/tests/unit/test_data_updates.py
--- a/rotkehlchen/tests/unit/test_data_updates.py
+++ b/rotkehlchen/tests/unit/test_data_updates.py
@@ -16,59 +16,85 @@
 OPTIMISM_SPAM_ASSET_ADDRESS = make_evm_address()
 
 
-def mock_github_data_response(url, timeout):  # pylint: disable=unused-argument
-    if 'info' in url:
-        return MockResponse(200, '{"spam_assets":{"latest":1},"rpc_nodes":{"latest":1}}')
-    if 'spam_assets/v' in url:
-        data = {
-            'spam_assets': [
-                {
-                    'address': ETHEREUM_SPAM_ASSET_ADDRESS,
-                    'name': '$ ClaimUniLP.com',
-                    'symbol': '$ ClaimUniLP.com - Visit to claim',
-                    'decimals': 18,
+ABI_DATA = [
+    {'id': 1, 'name': 'ABI1', 'value': '[{"inputs":[{"internalType":"address","name":"_factory","type":"address"}]'},  # noqa: E501
+    {'id': 2, 'name': 'ABI2', 'value': '[{"name":"BasePoolAdded","inputs":[{"name":"base_pool","type":"address","indexed":false}],"anonymous":false,"type":"event"}]'},  # noqa: E501
+]
+CONTRACT_DATA = [
+    {'address': '0x4bBa290826C253BD854121346c370a9886d1bC26', 'chain_id': 1, 'name': 'CONTRACT1_ETH', 'abi': 1, 'deployed_block': 1000},  # noqa: E501
+    {'address': '0xc37b40ABdB939635068d3c5f13E7faF686F03B65', 'chain_id': 10, 'name': 'CONTRACT1_OP', 'abi': 1, 'deployed_block': 20},  # noqa: E501
+    {'address': '0x19e4057A38a730be37c4DA690b103267AAE1d75d', 'chain_id': 1, 'name': None, 'abi': 2, 'deployed_block': None},  # noqa: E501
+]
+
+
+def make_mock_github_data_response(target: UpdateType):
+    """Creates a mocking function for github requests."""
+    def mock_github_data_response(url, timeout):  # pylint: disable=unused-argument
+        if 'info' in url:
+            data = {
+                'spam_assets': {'latest': 1 if target == UpdateType.SPAM_ASSETS else 0},
+                'rpc_nodes': {'latest': 1 if target == UpdateType.RPC_NODES else 0},
+                'contracts': {'latest': 1 if target == UpdateType.CONTRACTS else 0},
+            }
+        elif 'spam_assets/v' in url:
+            data = {
+                'spam_assets': [
+                    {
+                        'address': ETHEREUM_SPAM_ASSET_ADDRESS,
+                        'name': '$ ClaimUniLP.com',
+                        'symbol': '$ ClaimUniLP.com - Visit to claim',
+                        'decimals': 18,
+                    },
+                    {
+                        'address': OPTIMISM_SPAM_ASSET_ADDRESS,
+                        'name': 'spooky-v3.xyz',
+                        'symbol': 'Visit https://spooky-v3.xyz and claim rewards',
+                        'decimals': 18,
+                        'chain': 'optimism',
+                    },
+                ],
+            }
+        elif 'rpc_nodes/v' in url:
+            data = {
+                'rpc_nodes': [
+                    {
+                        'name': 'pocket network',
+                        'endpoint': 'https://eth-mainnet.gateway.pokt.network/v1/5f3453978e354ab992c4da79',  # noqa: E501
+                        'weight': 0.50,
+                        'owned': False,
+                        'active': True,
+                        'blockchain': 'ETH',
+                    },
+                    {
+                        'name': 'alchemy free',
+                        'endpoint': 'https://mainnet.optimism.io',
+                        'weight': 0.50,
+                        'owned': False,
+                        'active': True,
+                        'blockchain': 'OPTIMISM',
+                    },
+                ],
+            }
+        elif 'contracts/v' in url:
+            data = {
+                'contracts': {
+                    'abis_data': ABI_DATA,
+                    'contracts_data': CONTRACT_DATA,
                 },
-                {
-                    'address': OPTIMISM_SPAM_ASSET_ADDRESS,
-                    'name': 'spooky-v3.xyz',
-                    'symbol': 'Visit https://spooky-v3.xyz and claim rewards',
-                    'decimals': 18,
-                    'chain': 'optimism',
-                },
-            ],
-        }
-        return MockResponse(200, json.dumps(data))
-    if 'rpc_nodes/v' in url:
-        data = {
-            'rpc_nodes': [
-                {
-                    'name': 'pocket network',
-                    'endpoint': 'https://eth-mainnet.gateway.pokt.network/v1/5f3453978e354ab992c4da79',  # noqa: E501
-                    'weight': 0.50,
-                    'owned': False,
-                    'active': True,
-                    'blockchain': 'ETH',
-                },
-                {
-                    'name': 'alchemy free',
-                    'endpoint': 'https://mainnet.optimism.io',
-                    'weight': 0.50,
-                    'owned': False,
-                    'active': True,
-                    'blockchain': 'OPTIMISM',
-                },
-            ],
-        }
+            }
+        else:
+            raise AssertionError(f'Unexpected url {url} called')
+
         return MockResponse(200, json.dumps(data))
 
-    raise AssertionError(f'Unexpected url {url} called')
+    return mock_github_data_response
 
 
 def mock_github_data_response_old_update(url, timeout):  # pylint: disable=unused-argument
     if 'info' not in url:
         raise AssertionError(f'Unexpected url {url} called')
 
-    return MockResponse(200, '{"spam_assets":{"latest":1}, "rpc_nodes": {"latest":1}}')
+    return MockResponse(200, '{"spam_assets":{"latest":1}, "rpc_nodes": {"latest":1}, "contracts": {"latest":1}}')  # noqa: E501
 
 
 @pytest.fixture(name='data_updater')
@@ -95,7 +121,7 @@ def test_update_spam_assets(data_updater: RotkiDataUpdater) -> None:
         EvmToken(op_spam_token_id)
 
     # set a high version of the globaldb to avoid conflicts with future changes
-    with patch('requests.get', wraps=mock_github_data_response):
+    with patch('requests.get', wraps=make_mock_github_data_response(UpdateType.SPAM_ASSETS)):
         data_updater.check_for_updates()
 
     ethereum_token = EvmToken(eth_spam_token_id)
@@ -120,6 +146,7 @@ def test_no_update_performed(data_updater: RotkiDataUpdater) -> None:
             [
                 (UpdateType.SPAM_ASSETS.serialize(), 999),
                 (UpdateType.RPC_NODES.serialize(), 999),
+                (UpdateType.CONTRACTS.serialize(), 999),
             ],
         )
 
@@ -127,10 +154,12 @@ def test_no_update_performed(data_updater: RotkiDataUpdater) -> None:
         patch('requests.get', wraps=mock_github_data_response_old_update),
         patch.object(data_updater, 'update_spam_assets') as spam_assets,
         patch.object(data_updater, 'update_rpc_nodes') as rpc_nodes,
+        patch.object(data_updater, 'update_contracts') as contracts,
     ):
         data_updater.check_for_updates()
         assert spam_assets.call_count == 0
         assert rpc_nodes.call_count == 0
+        assert contracts.call_count == 0
 
 
 def test_update_rpc_nodes(data_updater: RotkiDataUpdater) -> None:
@@ -154,7 +183,7 @@ def test_update_rpc_nodes(data_updater: RotkiDataUpdater) -> None:
         write_cursor.execute('SELECT COUNT(*) FROM rpc_nodes')
         assert write_cursor.fetchone()[0] == 11
 
-    with patch('requests.get', wraps=mock_github_data_response):
+    with patch('requests.get', wraps=make_mock_github_data_response(UpdateType.RPC_NODES)):
         data_updater.check_for_updates()
 
     # check the db state after updating
@@ -172,3 +201,44 @@ def test_update_rpc_nodes(data_updater: RotkiDataUpdater) -> None:
         (11, *custom_node_tuple),
         (12, 'pocket network', 'https://eth-mainnet.gateway.pokt.network/v1/5f3453978e354ab992c4da79', 0, 1, '0.5', 'ETH'),  # noqa: E501
     ]
+
+
+def test_update_contracts(data_updater: RotkiDataUpdater) -> None:
+    """Tests functionality for remotely updating contracts"""
+    with GlobalDBHandler().conn.read_ctx() as cursor:
+        # Check state before the update
+        cursor.execute('SELECT MAX(id) FROM contract_abi')
+        initial_abis = cursor.execute('SELECT * FROM contract_abi').fetchall()
+        assert len(initial_abis) > 0, 'There should be some abis in the db'
+        initial_contracts = cursor.execute('SELECT * FROM contract_data').fetchall()
+        assert len(initial_contracts) > 0, 'There should be some contracts in the db'
+
+    with patch('requests.get', wraps=make_mock_github_data_response(UpdateType.CONTRACTS)):  # noqa: E501
+        data_updater.check_for_updates()  # apply the update
+
+    remote_id_to_local_id = {}
+    with GlobalDBHandler().conn.read_ctx() as cursor:
+        for entry in ABI_DATA:
+            reduced_abi = json.dumps(entry['value'], separators=(',', ':'))
+            cursor.execute(
+                'SELECT id FROM contract_abi WHERE name=? AND value=?',
+                (entry['name'], reduced_abi),
+            )
+            local_id = cursor.fetchone()
+            assert local_id is not None
+            remote_id_to_local_id[entry['id']] = local_id[0]
+
+    expected_new_contracts = []
+    for contract_data in CONTRACT_DATA:
+        expected_new_contracts.append((
+            contract_data['address'],
+            contract_data['chain_id'],
+            contract_data['name'],
+            remote_id_to_local_id[contract_data['abi']],
+            contract_data['deployed_block'],
+        ))
+
+    with GlobalDBHandler().conn.read_ctx() as cursor:
+        # Check that the new abis and contracts were added and the old ones were not modified
+        contracts_after_update = cursor.execute('SELECT * FROM contract_data')
+        assert set(contracts_after_update) == set(initial_contracts) | set(expected_new_contracts)