Repository: rotki/rotki

Issue:
Provide remote updates for the global address book
## Abstract

We have a global name registry (address book) in the global DB. This is editable by the user. But unlike other global data such as assets, spam, assets, rpc nodes etc. we do not push data updates for it yet.

## Task

The approach should be same as in #5532 

Related issues: 

- #5594 
- #5532 
- #5236 



Discussion:

Resolving patch:
diff --git a/rotkehlchen/chain/ethereum/modules/makerdao/cache.py b/rotkehlchen/chain/ethereum/modules/makerdao/cache.py
--- a/rotkehlchen/chain/ethereum/modules/makerdao/cache.py
+++ b/rotkehlchen/chain/ethereum/modules/makerdao/cache.py
@@ -201,20 +201,11 @@ def update_ilk_registry(
                 log.error(f'Did not add ilk {ilk} due to no response for {join_address} metadata')
                 continue
 
-            serialized_abi = json.dumps(abi, separators=(',', ':'))
-            cursor.execute('SELECT id from contract_abi WHERE value=?', (serialized_abi,))
-            result = cursor.fetchone()
-            name = f'MAKERDAO_{ilk}_JOIN'
-            if result is None:
-                with GlobalDBHandler().conn.write_ctx() as write_cursor:
-                    write_cursor.execute(
-                        'INSERT INTO contract_abi(value, name) VALUES(?, ?)',
-                        (serialized_abi, name),
-                    )
-                    abi_id = write_cursor.lastrowid
-            else:
-                abi_id = result[0]
-
+            abi_name = f'MAKERDAO_{ilk}_JOIN'
+            abi_id = GlobalDBHandler().get_or_write_abi(
+                serialized_abi=json.dumps(abi, separators=(',', ':')),
+                abi_name=abi_name,
+            )
             cursor.execute(
                 'SELECT COUNT(*) FROM contract_data WHERE address=? and chain_id=?',
                 (join_address, 1),
@@ -226,7 +217,7 @@ def update_ilk_registry(
                 write_cursor.execute(
                     'INSERT INTO contract_data(address, chain_id, name, abi, deployed_block)'
                     ' VALUES(? ,? , ?, ?, ?)',
-                    (join_address, 1, name, abi_id, deployed_block),
+                    (join_address, 1, abi_name, abi_id, deployed_block),
                 )
 
             # if the underlying token does not exist, add it
diff --git a/rotkehlchen/db/updates.py b/rotkehlchen/db/updates.py
--- a/rotkehlchen/db/updates.py
+++ b/rotkehlchen/db/updates.py
@@ -8,11 +8,19 @@
 
 from rotkehlchen.assets.spam_assets import update_spam_assets
 from rotkehlchen.constants.timing import DEFAULT_TIMEOUT_TUPLE
+from rotkehlchen.db.addressbook import DBAddressbook
 from rotkehlchen.errors.misc import RemoteError
 from rotkehlchen.errors.serialization import DeserializationError
 from rotkehlchen.fval import FVal
 from rotkehlchen.globaldb.handler import GlobalDBHandler
 from rotkehlchen.logging import RotkehlchenLogsAdapter
+from rotkehlchen.serialization.deserialize import deserialize_evm_address
+from rotkehlchen.types import (
+    AddressbookEntry,
+    AddressbookType,
+    OptionalChainAddress,
+    SupportedBlockchain,
+)
 from rotkehlchen.utils.misc import ts_now
 from rotkehlchen.utils.network import query_file
 
@@ -27,12 +35,14 @@
 SPAM_ASSETS_URL = 'https://raw.githubusercontent.com/rotki/data/{branch}/updates/spam_assets/v{version}.json'  # noqa: E501
 RPC_NODES_URL = 'https://raw.githubusercontent.com/rotki/data/{branch}/updates/rpc_nodes/v{version}.json'  # noqa: E501
 CONTRACTS_URL = 'https://raw.githubusercontent.com/rotki/data/{branch}/updates/contracts/v{version}.json'  # noqa: E501
+GLOBAL_ADDRESSBOOK_URL = 'https://raw.githubusercontent.com/rotki/data/{branch}/updates/global_addressbook/v{version}.json'  # noqa: E501
 
 
 class UpdateType(Enum):
     SPAM_ASSETS = 'spam_assets'
     RPC_NODES = 'rpc_nodes'
     CONTRACTS = 'contracts'
+    GLOBAL_ADDRESSBOOK = 'global_addressbook'
 
     def serialize(self) -> str:
         """Serializes the update type for the DB and API"""
@@ -86,7 +96,7 @@ def _get_remote_info_json(self) -> dict[str, Any]:
 
     def update_spam_assets(self, from_version: int, to_version: int) -> None:
         """
-        Check the updates in the inclusive range [local_version + 1, to_version] and update the
+        Check the updates in the inclusive range [from_version + 1, to_version] and update the
         spam assets for those versions. Assets are also added to the globaldb if they don't exist
         locally
         """
@@ -110,7 +120,7 @@ def update_spam_assets(self, from_version: int, to_version: int) -> None:
                     (UpdateType.SPAM_ASSETS.serialize(), version),
                 )
 
-    def update_rpc_nodes(self, from_version: int, to_version: int) -> None:  # pylint: disable=unused-argument  # from_version is here to have the same format as other updating functions  # noqa: E501
+    def update_rpc_nodes(self, from_version: int, to_version: int) -> None:  # pylint: disable=unused-argument  # noqa: E501
         """Checks the update for the latest version and update the default rpc nodes in the global db.
 
         It also updates the user db with these default nodes.
@@ -171,27 +181,13 @@ def update_contracts(self, from_version: int, to_version: int) -> None:
 
             log.info(f'Applying update for contracts to v{version}')
             remote_id_to_local_id = {}
-            with GlobalDBHandler().conn.read_ctx() as cursor:
-                for single_abi_data in updated_contracts['abis_data']:
-                    serialized_abi = json.dumps(single_abi_data['value'], separators=(',', ':'))
-
-                    # check if the abi is already present in the database
-                    existing_abi_id = cursor.execute(
-                        'SELECT id FROM contract_abi WHERE value=?',
-                        (serialized_abi,),
-                    ).fetchone()
-
-                    if existing_abi_id is not None:
-                        abi_id = existing_abi_id[0]
-                    else:
-                        with GlobalDBHandler().conn.write_ctx() as write_cursor:
-                            write_cursor.execute(
-                                'INSERT INTO contract_abi(name, value) VALUES(?, ?)',
-                                (single_abi_data['name'], serialized_abi),
-                            )
-                            abi_id = write_cursor.lastrowid
-                    # store a mapping of the virtual id to the id in the user's globaldb
-                    remote_id_to_local_id[single_abi_data['id']] = abi_id
+            for single_abi_data in updated_contracts['abis_data']:
+                # store a mapping of the virtual id to the id in the user's globaldb
+                abi_id = GlobalDBHandler().get_or_write_abi(
+                    serialized_abi=single_abi_data['value'],
+                    abi_name=single_abi_data.get('name'),
+                )
+                remote_id_to_local_id[single_abi_data['id']] = abi_id
 
             new_contracts_data = []
             for single_contract_data in updated_contracts['contracts_data']:
@@ -221,6 +217,70 @@ def update_contracts(self, from_version: int, to_version: int) -> None:
                     (UpdateType.CONTRACTS.serialize(), version),
                 )
 
+    def update_global_addressbook(self, from_version: int, to_version: int) -> None:
+        """Downloads and applies global addressbook updates from from_version to to_version"""
+        db_addressbook = DBAddressbook(self.user_db)
+        for version in range(from_version + 1, to_version + 1):
+            log.debug(f'Applying global address book update version {version}')
+            success, new_raw_entries = self._get_file_content(
+                file_url=GLOBAL_ADDRESSBOOK_URL.format(branch=self.branch, version=version),
+                update_type=UpdateType.GLOBAL_ADDRESSBOOK,
+            )
+            if success is False:
+                log.error(f'Failed to download globaldb addressbook for version {version}')
+                continue
+
+            entries_to_add = []
+            for raw_entry in new_raw_entries:
+                try:
+                    address = deserialize_evm_address(raw_entry['address'])
+                    blockchain = SupportedBlockchain.deserialize(raw_entry['blockchain']) if raw_entry['blockchain'] else None  # noqa: E501
+                except DeserializationError as e:
+                    self.msg_aggregator.add_error(
+                        f'Could not deserialize address {raw_entry["address"]} or blockchain '
+                        f'{raw_entry["blockchain"]} that was seen in a global addressbook update. '
+                        f'Please report it to the rotki team. {str(e)}',
+                    )
+                    continue
+
+                entry = AddressbookEntry(
+                    address=address,
+                    name=raw_entry['new_name'],
+                    blockchain=blockchain,
+                )
+                existing_name = db_addressbook.get_addressbook_entry_name(
+                    book_type=AddressbookType.GLOBAL,
+                    chain_address=OptionalChainAddress(
+                        address=address,
+                        blockchain=blockchain,
+                    ),
+                )
+                if existing_name is None:
+                    entries_to_add.append(entry)
+                elif raw_entry.get('old_name') is not None and existing_name == raw_entry['old_name']:  # noqa: E501
+                    # If old_name is specified, replace the entry with the old name.
+                    db_addressbook.delete_addressbook_entries(
+                        book_type=AddressbookType.GLOBAL,
+                        chain_addresses=[OptionalChainAddress(
+                            address=address,
+                            blockchain=blockchain,
+                        )],
+                    )
+                    entries_to_add.append(entry)
+                else:
+                    log.debug(
+                        f'Global address book entry for address {address} and blockchain '
+                        f'{blockchain} already exists (probably was input by the user). '
+                        f'Update for this entry will not be applied.',
+                    )
+                    continue
+
+            with GlobalDBHandler().conn.write_ctx() as write_cursor:
+                db_addressbook.add_addressbook_entries(
+                    write_cursor=write_cursor,
+                    entries=entries_to_add,
+                )
+
     def check_for_updates(self) -> None:
         """Retrieve the information about the latest available update"""
         remote_information = self._get_remote_info_json()
@@ -258,7 +318,11 @@ def _check_for_last_version(cursor: 'DBCursor', update_type: UpdateType) -> int:
     @overload
     def _get_file_content(
             file_url: str,
-            update_type: Literal[UpdateType.SPAM_ASSETS, UpdateType.RPC_NODES],
+            update_type: Literal[
+                UpdateType.SPAM_ASSETS,
+                UpdateType.RPC_NODES,
+                UpdateType.GLOBAL_ADDRESSBOOK,
+            ],
     ) -> tuple[bool, list[dict[str, Any]]]:
         ...
 
diff --git a/rotkehlchen/globaldb/handler.py b/rotkehlchen/globaldb/handler.py
--- a/rotkehlchen/globaldb/handler.py
+++ b/rotkehlchen/globaldb/handler.py
@@ -1851,3 +1851,25 @@ def get_collection_main_asset(identifier: str) -> Optional[str]:
             result = cursor.fetchone()
 
         return result[0] if result is not None else None
+
+    @staticmethod
+    def get_or_write_abi(serialized_abi: str, abi_name: Optional[str] = None) -> int:
+        """
+        Finds and returns the id of the given abi.
+        If the abi doesn't exist in the db, inserts it there.
+        """
+        with GlobalDBHandler().conn.write_ctx() as cursor:
+            # check if the abi is already present in the database
+            existing_abi_id = cursor.execute(
+                'SELECT id FROM contract_abi WHERE value=?',
+                (serialized_abi,),
+            ).fetchone()
+
+            if existing_abi_id is not None:
+                return existing_abi_id[0]
+            else:
+                cursor.execute(
+                    'INSERT INTO contract_abi(name, value) VALUES(?, ?)',
+                    (abi_name, serialized_abi),
+                )
+                return cursor.lastrowid


Test patch:
diff --git a/rotkehlchen/tests/unit/test_data_updates.py b/rotkehlchen/tests/unit/test_data_updates.py
--- a/rotkehlchen/tests/unit/test_data_updates.py
+++ b/rotkehlchen/tests/unit/test_data_updates.py
@@ -5,16 +5,60 @@
 
 from rotkehlchen.assets.asset import EvmToken
 from rotkehlchen.constants.resolver import evm_address_to_identifier
+from rotkehlchen.db.addressbook import DBAddressbook
 from rotkehlchen.db.updates import RotkiDataUpdater, UpdateType
 from rotkehlchen.errors.asset import UnknownAsset
 from rotkehlchen.globaldb.handler import GlobalDBHandler
 from rotkehlchen.tests.utils.factories import make_evm_address
 from rotkehlchen.tests.utils.mock import MockResponse
-from rotkehlchen.types import SPAM_PROTOCOL, ChainID, EvmTokenKind
+from rotkehlchen.types import (
+    SPAM_PROTOCOL,
+    AddressbookEntry,
+    ChainID,
+    EvmTokenKind,
+    SupportedBlockchain,
+)
 
 ETHEREUM_SPAM_ASSET_ADDRESS = make_evm_address()
 OPTIMISM_SPAM_ASSET_ADDRESS = make_evm_address()
 
+ADDRESSBOOK_ADDRESS_1 = make_evm_address()
+ADDRESSBOOK_ADDRESS_2 = make_evm_address()
+
+REMOTE_ADDRESSBOOK = [
+    AddressbookEntry(
+        name='Remote name that doesnt conflict with anything',
+        address=ADDRESSBOOK_ADDRESS_1,
+        blockchain=SupportedBlockchain.ETHEREUM,
+    ), AddressbookEntry(
+        name='Remote name that has a conflict, and it should replace the local name',
+        address=ADDRESSBOOK_ADDRESS_2,
+        blockchain=SupportedBlockchain.OPTIMISM,
+    ), AddressbookEntry(
+        name='Remote name that has a conflict, but the local name should be preferred',
+        address=ADDRESSBOOK_ADDRESS_2,
+        blockchain=SupportedBlockchain.ETHEREUM,
+    ),
+]
+SERIALIZED_REMOTE_ADDRESSBOOK = [
+    {
+        'address': ADDRESSBOOK_ADDRESS_1,
+        'blockchain': 'ETH',
+        'new_name': 'Remote name that doesnt conflict with anything',
+    },
+    {
+        'address': ADDRESSBOOK_ADDRESS_2,
+        'blockchain': 'OPTIMISM',
+        'new_name': 'Remote name that has a conflict, and it should replace the local name',
+        'old_name': 'Local name that has a conflict and should be replaced by the remote name',
+    },
+    {
+        'address': ADDRESSBOOK_ADDRESS_2,
+        'blockchain': 'ETH',
+        'new_name': 'Remote name that has a conflict, but the local name should be preferred',
+    },
+]
+
 
 ABI_DATA = [
     {'id': 1, 'name': 'ABI1', 'value': '[{"inputs":[{"internalType":"address","name":"_factory","type":"address"}]'},  # noqa: E501
@@ -35,6 +79,7 @@ def mock_github_data_response(url, timeout):  # pylint: disable=unused-argument
                 'spam_assets': {'latest': 1 if target == UpdateType.SPAM_ASSETS else 0},
                 'rpc_nodes': {'latest': 1 if target == UpdateType.RPC_NODES else 0},
                 'contracts': {'latest': 1 if target == UpdateType.CONTRACTS else 0},
+                'global_addressbook': {'latest': 1 if target == UpdateType.GLOBAL_ADDRESSBOOK else 0},  # noqa: E501
             }
         elif 'spam_assets/v' in url:
             data = {
@@ -82,6 +127,8 @@ def mock_github_data_response(url, timeout):  # pylint: disable=unused-argument
                     'contracts_data': CONTRACT_DATA,
                 },
             }
+        elif 'global_addressbook/v' in url:
+            data = {'global_addressbook': SERIALIZED_REMOTE_ADDRESSBOOK}
         else:
             raise AssertionError(f'Unexpected url {url} called')
 
@@ -94,7 +141,12 @@ def mock_github_data_response_old_update(url, timeout):  # pylint: disable=unuse
     if 'info' not in url:
         raise AssertionError(f'Unexpected url {url} called')
 
-    return MockResponse(200, '{"spam_assets":{"latest":1}, "rpc_nodes": {"latest":1}, "contracts": {"latest":1}}')  # noqa: E501
+    return MockResponse(200, json.dumps({
+        'spam_assets': {'latest': 1},
+        'rpc_nodes': {'latest': 1},
+        'contracts': {'latest': 1},
+        'global_addressbook': {'latest': 1},
+    }))
 
 
 @pytest.fixture(name='data_updater')
@@ -147,6 +199,7 @@ def test_no_update_performed(data_updater: RotkiDataUpdater) -> None:
                 (UpdateType.SPAM_ASSETS.serialize(), 999),
                 (UpdateType.RPC_NODES.serialize(), 999),
                 (UpdateType.CONTRACTS.serialize(), 999),
+                (UpdateType.GLOBAL_ADDRESSBOOK.serialize(), 999),
             ],
         )
 
@@ -155,11 +208,13 @@ def test_no_update_performed(data_updater: RotkiDataUpdater) -> None:
         patch.object(data_updater, 'update_spam_assets') as spam_assets,
         patch.object(data_updater, 'update_rpc_nodes') as rpc_nodes,
         patch.object(data_updater, 'update_contracts') as contracts,
+        patch.object(data_updater, 'update_global_addressbook') as global_addressbook,
     ):
         data_updater.check_for_updates()
         assert spam_assets.call_count == 0
         assert rpc_nodes.call_count == 0
         assert contracts.call_count == 0
+        assert global_addressbook.call_count == 0
 
 
 def test_update_rpc_nodes(data_updater: RotkiDataUpdater) -> None:
@@ -219,10 +274,9 @@ def test_update_contracts(data_updater: RotkiDataUpdater) -> None:
     remote_id_to_local_id = {}
     with GlobalDBHandler().conn.read_ctx() as cursor:
         for entry in ABI_DATA:
-            reduced_abi = json.dumps(entry['value'], separators=(',', ':'))
             cursor.execute(
                 'SELECT id FROM contract_abi WHERE name=? AND value=?',
-                (entry['name'], reduced_abi),
+                (entry['name'], entry['value']),
             )
             local_id = cursor.fetchone()
             assert local_id is not None
@@ -242,3 +296,44 @@ def test_update_contracts(data_updater: RotkiDataUpdater) -> None:
         # Check that the new abis and contracts were added and the old ones were not modified
         contracts_after_update = cursor.execute('SELECT * FROM contract_data')
         assert set(contracts_after_update) == set(initial_contracts) | set(expected_new_contracts)
+
+
+@pytest.mark.parametrize('empty_global_addressbook', [True])
+def test_global_addressbook(data_updater: RotkiDataUpdater) -> None:
+    """Test that remote updates for global addressbook work"""
+    db_addressbook = DBAddressbook(data_updater.user_db)
+    # check state of the address book before updating
+    with GlobalDBHandler().conn.read_ctx() as cursor:
+        cursor.execute('SELECT COUNT(*) FROM address_book')
+        assert cursor.fetchone()[0] == 0
+
+    # Populate addressbook and see that preferring local / remote names works as expected.
+    initial_entries = [
+        AddressbookEntry(
+            address=ADDRESSBOOK_ADDRESS_1,
+            name='Local name that does not conflict with anything',
+            blockchain=SupportedBlockchain.AVALANCHE,
+        ), AddressbookEntry(
+            address=ADDRESSBOOK_ADDRESS_2,
+            name='Local name that has a conflict with remote, but should stay as is',
+            blockchain=SupportedBlockchain.ETHEREUM,
+        ), AddressbookEntry(
+            address=ADDRESSBOOK_ADDRESS_2,
+            name='Local name that has a conflict and should be replaced by the remote name',
+            blockchain=SupportedBlockchain.OPTIMISM,
+        ),
+    ]
+    with GlobalDBHandler().conn.write_ctx() as write_cursor:
+        db_addressbook.add_addressbook_entries(
+            write_cursor=write_cursor,
+            entries=initial_entries,
+        )
+
+    with patch('requests.get', wraps=make_mock_github_data_response(UpdateType.GLOBAL_ADDRESSBOOK)):  # noqa: E501
+        data_updater.check_for_updates()
+
+    # Assert state of the address book after the update
+    with GlobalDBHandler().conn.read_ctx() as cursor:
+        all_entries = db_addressbook.get_addressbook_entries(cursor=cursor)
+
+    assert set(all_entries) == set(initial_entries[:2] + REMOTE_ADDRESSBOOK[:2])