Repository: rotki/rotki

Issue:
Use savepoints during globaldb updates
At the moment we apply all the updates at once when the conflict method resolution has been chosen. This presents the issue where if the sql query is incorrect we can end up with missing assets. The reason for this is this delete statement https://github.com/rotki/rotki/blob/develop/rotkehlchen/globaldb/updates.py#L69 that is not rolled back at https://github.com/rotki/rotki/blob/develop/rotkehlchen/globaldb/updates.py#L353 and if we try to roll back we undo all the previous actions.

For this lefteris suggested that we use https://www.sqlite.org/lang_savepoint.html in the update process


Discussion:

Resolving patch:
diff --git a/rotkehlchen/db/drivers/gevent.py b/rotkehlchen/db/drivers/gevent.py
--- a/rotkehlchen/db/drivers/gevent.py
+++ b/rotkehlchen/db/drivers/gevent.py
@@ -8,7 +8,20 @@
 from enum import Enum, auto
 from pathlib import Path
 from types import TracebackType
-from typing import TYPE_CHECKING, Any, Dict, Generator, List, Optional, Sequence, Type, Union
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Dict,
+    Generator,
+    List,
+    Literal,
+    Optional,
+    Sequence,
+    Tuple,
+    Type,
+    Union,
+)
+from uuid import uuid4
 
 import gevent
 from pysqlcipher3 import dbapi2 as sqlcipher
@@ -25,6 +38,10 @@
 logger: 'RotkehlchenLogger' = logging.getLogger(__name__)  # type: ignore
 
 
+class ContextError(Exception):
+    """Intended to be raised when something is wrong with db context management"""
+
+
 class DBCursor:
 
     def __init__(self, connection: 'DBConnection', cursor: UnderlyingCursor) -> None:  # noqa: E501
@@ -214,6 +231,9 @@ def __init__(
         self.in_callback = gevent.lock.Semaphore()
         self.connection_type = connection_type
         self.sql_vm_instructions_cb = sql_vm_instructions_cb
+        # We need an ordered set. Python doesn't have such thing as a standalone object, but has
+        # `dict` which preserves the order of its keys. So we use dict with None values.
+        self.savepoints: Dict[str, None] = {}
         if connection_type == DBConnectionType.GLOBAL:
             self._conn = sqlite3.connect(path, check_same_thread=False)
         else:
@@ -295,6 +315,86 @@ def write_ctx(self) -> Generator['DBCursor', None, None]:
         finally:
             cursor.close()  # lgtm [py/should-use-with]
 
+    @contextmanager
+    def savepoint_ctx(
+            self,
+            savepoint_name: Optional[str] = None,
+    ) -> Generator['DBCursor', None, None]:
+        """
+        Creates a savepoint context with the provided name. If the code inside the savepoint fails,
+        rolls back this savepoint, otherwise releases it (aka forgets it -- this is not commited to the DB).  # noqa: E501
+        Savepoints work like nested transactions, more information here: https://www.sqlite.org/lang_savepoint.html  # noqa: E501
+        """
+        cursor, savepoint_name = self.enter_savepoint(savepoint_name)
+        try:
+            yield cursor
+        except Exception:
+            self.rollback_savepoint(savepoint_name)
+            raise
+        else:
+            self.release_savepoint(savepoint_name)
+        finally:
+            cursor.close()  # lgtm [py/should-use-with]
+
+    def enter_savepoint(self, savepoint_name: Optional[str] = None) -> Tuple['DBCursor', str]:
+        """
+        Creates an sqlite savepoint with the given name. If None is given, a uuid is created.
+        Returns cursor and savepoint's name.
+        May raise:
+        - ContextError if a savepoint with the same name already exists. Can only happen in case of
+        manually specified name.
+        """
+        if savepoint_name is None:
+            savepoint_name = str(uuid4())
+        if savepoint_name in self.savepoints:
+            raise ContextError(
+                f'Wanted to enter savepoint {savepoint_name} but a savepoint with the same name '
+                f'already exists. Current savepoints: {list(self.savepoints)}',
+            )
+        cursor = self.cursor()
+        cursor.execute(f'SAVEPOINT "{savepoint_name}"')
+        self.savepoints[savepoint_name] = None
+        return cursor, savepoint_name
+
+    def _modify_savepoint(
+            self,
+            rollback_or_release: Literal['ROLLBACK TO', 'RELEASE'],
+            savepoint_name: Optional[str],
+    ) -> None:
+        if len(self.savepoints) == 0:
+            raise ContextError(
+                f'Incorrect use of savepoints! Wanted to {rollback_or_release.lower()} savepoint '
+                f'{savepoint_name}, but the stack is empty.',
+            )
+        list_savepoints = list(self.savepoints)
+        if savepoint_name is None:
+            savepoint_name = list_savepoints[-1]
+        elif savepoint_name not in self.savepoints:
+            raise ContextError(
+                f'Incorrect use of savepoints! Wanted to {rollback_or_release.lower()} savepoint '
+                f'{savepoint_name}, but it is not present in the stack: {list_savepoints}',
+            )
+        self.execute(f'{rollback_or_release} SAVEPOINT "{savepoint_name}"')
+
+        # Rollback all savepoints until, and including, the one with name `savepoint_name`
+        self.savepoints = dict.fromkeys(list_savepoints[:list_savepoints.index(savepoint_name)])
+
+    def rollback_savepoint(self, savepoint_name: Optional[str] = None) -> None:
+        """
+        Rollbacks to `savepoint_name` if given and to the latest savepoint otherwise.
+        May raise:
+        - ContextError if savepoints stack is empty or given savepoint name is not in the stack
+        """
+        self._modify_savepoint(rollback_or_release='ROLLBACK TO', savepoint_name=savepoint_name)
+
+    def release_savepoint(self, savepoint_name: Optional[str] = None) -> None:
+        """
+        Releases (aka forgets) `savepoint_name` if given and the latest savepoint otherwise.
+        May raise:
+        - ContextError if savepoints stack is empty or given savepoint name is not in the stack
+        """
+        self._modify_savepoint(rollback_or_release='RELEASE', savepoint_name=savepoint_name)
+
     @property
     def total_changes(self) -> int:
         """total number of database rows that have been modified, inserted,
diff --git a/rotkehlchen/globaldb/updates.py b/rotkehlchen/globaldb/updates.py
--- a/rotkehlchen/globaldb/updates.py
+++ b/rotkehlchen/globaldb/updates.py
@@ -301,7 +301,7 @@ def _parse_full_insert(self, insert_text: str) -> AssetData:
 
     def _apply_single_version_update(
             self,
-            cursor: DBCursor,
+            connection: DBConnection,
             version: int,
             text: str,
             conflicts: Optional[Dict[Asset, Literal['remote', 'local']]],
@@ -327,13 +327,15 @@ def _apply_single_version_update(
                 pass
 
             try:
-                executeall(cursor, action)
+                with connection.savepoint_ctx() as cursor:
+                    executeall(cursor, action)
                 if local_asset is not None:
                     AssetResolver().clean_memory_cache(local_asset.identifier.lower())
             except sqlite3.Error:  # https://docs.python.org/3/library/sqlite3.html#exceptions
                 if local_asset is None:
                     try:  # if asset is not known then simply do an insertion
-                        executeall(cursor, full_insert)
+                        with connection.savepoint_ctx() as cursor:
+                            executeall(cursor, full_insert)
                     except sqlite3.Error as e:
                         self.msg_aggregator.add_warning(
                             f'Failed to add asset {remote_asset_data.identifier} in the '
@@ -349,7 +351,8 @@ def _apply_single_version_update(
                     continue
                 if resolution == 'remote':
                     try:
-                        _force_remote(cursor, local_asset, full_insert)
+                        with connection.savepoint_ctx() as cursor:
+                            _force_remote(cursor, local_asset, full_insert)
                     except sqlite3.Error as e:
                         self.msg_aggregator.add_warning(
                             f'Failed to resolve conflict for {remote_asset_data.identifier} in '
@@ -368,7 +371,7 @@ def _apply_single_version_update(
                 self.conflicts.append((local_data, remote_asset_data))
 
         # at the very end update the current version in the DB
-        cursor.execute(
+        connection.execute(
             'INSERT OR REPLACE INTO settings(name, value) VALUES(?, ?)',
             (ASSETS_VERSION_KEY, str(version)),
         )
@@ -435,7 +438,6 @@ def _perform_update(
         version = self.local_assets_version + 1
         target_version = min(up_to_version, self.last_remote_checked_version) if up_to_version else self.last_remote_checked_version   # type: ignore # noqa: E501
         # type ignore since due to check_for_updates we know last_remote_checked_version exists
-        cursor = connection.cursor()
         while version <= target_version:
             try:
                 min_schema_version = infojson['updates'][str(version)]['min_schema_version']
@@ -455,7 +457,7 @@ def _perform_update(
                         f'You will have to follow an alternative method to '
                         f'obtain the assets of this update. Easiest would be to reset global DB.',
                     )
-                    cursor.execute(
+                    connection.execute(
                         'INSERT OR REPLACE INTO settings(name, value) VALUES(?, ?)',
                         (ASSETS_VERSION_KEY, str(version)),
                     )
@@ -484,7 +486,7 @@ def _perform_update(
                 )
 
             self._apply_single_version_update(
-                cursor=cursor,
+                connection=connection,
                 version=version,
                 text=response.text,
                 conflicts=conflicts,


Test patch:
diff --git a/rotkehlchen/tests/db/test_savepoints.py b/rotkehlchen/tests/db/test_savepoints.py
new file mode 100644
--- /dev/null
+++ b/rotkehlchen/tests/db/test_savepoints.py
@@ -0,0 +1,75 @@
+import sqlite3
+
+import pytest
+
+from rotkehlchen.db.drivers.gevent import ContextError, DBConnection, DBConnectionType
+
+
+def test_unnamed_savepoints():
+    conn = DBConnection(
+        path=':memory:',
+        connection_type=DBConnectionType.GLOBAL,
+        sql_vm_instructions_cb=0,
+    )
+    conn.execute('CREATE TABLE a(b INTEGER PRIMARY KEY)')
+    with conn.savepoint_ctx() as cursor1:
+        assert len(conn.savepoints) == 1
+        savepoint1 = list(conn.savepoints)[0]
+        cursor1.execute('INSERT INTO a VALUES (1)')
+        cursor2, savepoint2 = conn.enter_savepoint()  # also check manual savepoints
+        assert list(conn.savepoints) == [savepoint1, savepoint2]
+        cursor2.execute('INSERT INTO a VALUES (2)')
+        # make sure that 2 was added
+        assert cursor1.execute('SELECT b FROM a').fetchall() == [(1,), (2,)]
+        conn.rollback_savepoint()
+        # check that the second savepoint was rolled back
+        assert list(conn.savepoints) == [savepoint1]
+        assert cursor1.execute('SELECT b FROM a').fetchall() == [(1,)]  # 2 should not be there
+        cursor1.execute('INSERT INTO a VALUES (3)')  # add one more value after the rollback
+    assert len(conn.savepoints) == 0  # check that we released successfully
+    # And make sure that the data is saved
+    assert conn.execute('SELECT b FROM a').fetchall() == [(1,), (3,)]
+
+
+def test_named_savepoints():
+    conn = DBConnection(
+        path=':memory:',
+        connection_type=DBConnectionType.GLOBAL,
+        sql_vm_instructions_cb=0,
+    )
+    conn.execute('CREATE TABLE a(b INTEGER PRIMARY KEY)')
+    savepoint1 = 'savepointXYZ'
+    with pytest.raises(sqlite3.Error):
+        with conn.savepoint_ctx(savepoint1) as cursor1:
+            assert list(conn.savepoints) == [savepoint1]
+            cursor1.execute('INSERT INTO a VALUES (1)')
+            _, savepoint2 = conn.enter_savepoint('point2')
+            cursor3, savepoint3 = conn.enter_savepoint()  # enter a couple more savepoints
+            assert list(conn.savepoints) == [savepoint1, savepoint2, savepoint3]
+            cursor3.execute('INSERT INTO a VALUES (2)')
+            # check that values are there
+            assert cursor3.execute('SELECT b FROM a').fetchall() == [(1,), (2,)]
+            # And now don't rollback manually entered savepoints,
+            # but rollback directly to the first one
+            cursor3.execute('INSERT INTO a VALUES (1)')  # make an error
+    # And check that everything was rolled back
+    assert len(conn.savepoints) == 0
+    assert conn.execute('SELECT b FROM a').fetchall() == []
+
+
+def test_savepoint_errors():
+    conn = DBConnection(
+        path=':memory:',
+        connection_type=DBConnectionType.GLOBAL,
+        sql_vm_instructions_cb=0,
+    )
+    with pytest.raises(ContextError):
+        conn.release_savepoint()
+
+    conn.enter_savepoint('point')
+    with pytest.raises(ContextError):
+        with conn.enter_savepoint('point'):
+            ...
+
+    with pytest.raises(ContextError):
+        conn.rollback_savepoint('abc')
diff --git a/rotkehlchen/tests/unit/test_globaldb_asset_updates.py b/rotkehlchen/tests/unit/test_globaldb_asset_updates.py
--- a/rotkehlchen/tests/unit/test_globaldb_asset_updates.py
+++ b/rotkehlchen/tests/unit/test_globaldb_asset_updates.py
@@ -3,6 +3,7 @@
 from rotkehlchen.assets.types import AssetData, AssetType
 from rotkehlchen.chain.ethereum.types import string_to_evm_address
 from rotkehlchen.errors.serialization import DeserializationError
+from rotkehlchen.globaldb.handler import GlobalDBHandler
 from rotkehlchen.globaldb.updates import AssetsUpdater
 from rotkehlchen.types import ChainID, EvmTokenKind, Timestamp
 
@@ -295,3 +296,45 @@ def test_parse_full_insert(assets_updater, text, expected_data, error_msg):
             assets_updater._parse_full_insert(text)
 
         assert error_msg in str(excinfo.value)
+
+
+def test_some_updates_are_malformed(assets_updater):
+    """
+    Checks the folowing cases:
+    1. If some of the updates are broken, others are not affected.
+    2. If an update is broken, info about asset stays the same in all tables as before the update.
+
+    Checks both insertion of a new asset and an update of an existing one.
+    """
+    # In "ETH" update swapped_for is incorrect
+    # In "BTC" update everything is good
+    # In "NEW-ASSET-1" insertion swapped_for is incorrect
+    # In "NEW-ASSET-2" insertion everything is good
+    update_text = """INSERT INTO assets(identifier, name, type) VALUES("ETH", "name1", "B"); INSERT INTO common_asset_details(identifier, symbol, coingecko, cryptocompare, forked, started, swapped_for) VALUES("ETH", "symbol1", "", "", NULL, NULL, "NONEXISTENT");
+*
+INSERT INTO assets(identifier, name, type) VALUES("BTC", "name2", "B"); INSERT INTO common_asset_details(identifier, symbol, coingecko, cryptocompare, forked, started, swapped_for) VALUES("BTC", "symbol2", "", "", NULL, NULL, NULL);
+*
+INSERT INTO assets(identifier, name, type) VALUES("NEW-ASSET-1", "name3", "B"); INSERT INTO common_asset_details(identifier, symbol, coingecko, cryptocompare, forked, started, swapped_for) VALUES("NEW-ASSET-1", "symbol3", "", "", NULL, NULL, "LOLKEK");
+*
+INSERT INTO assets(identifier, name, type) VALUES("NEW-ASSET-2", "name4", "B"); INSERT INTO common_asset_details(identifier, symbol, coingecko, cryptocompare, forked, started, swapped_for) VALUES("NEW-ASSET-2", "symbol4", "", "", NULL, NULL, NULL);
+*
+"""  # noqa: E501
+    connection = GlobalDBHandler().conn
+    assets_updater._apply_single_version_update(
+        connection=connection,
+        version=999,  # doesn't matter
+        text=update_text,
+        conflicts={'ETH': 'remote', 'BTC': 'remote'},
+    )
+
+    # Should have stayed unchanged since one of the insertions was incorrect
+    assert connection.execute('SELECT name FROM assets WHERE identifier="ETH"').fetchone()[0] == 'Ethereum'  # noqa: E501
+    assert connection.execute('SELECT symbol FROM common_asset_details WHERE identifier="ETH"').fetchone()[0] == 'ETH'  # noqa: E501
+    # BTC should have been updated since the insertion were correct
+    assert connection.execute('SELECT name FROM assets WHERE identifier="BTC"').fetchone()[0] == 'name2'  # noqa: E501
+    assert connection.execute('SELECT symbol FROM common_asset_details WHERE identifier="BTC"').fetchone()[0] == 'symbol2'  # noqa: E501
+    # NEW-ASSET-1 should have not been added since the insertions were incorrect
+    assert connection.execute('SELECT * FROM assets WHERE identifier="NEW-ASSET-1"').fetchone() is None  # noqa: E501
+    # NEW-ASSET-2 should have been added since the insertions were correct
+    assert connection.execute('SELECT name FROM assets WHERE identifier="NEW-ASSET-2"').fetchone()[0] == 'name4'  # noqa: E501
+    assert connection.execute('SELECT symbol FROM common_asset_details WHERE identifier="NEW-ASSET-2"').fetchone()[0] == 'symbol4'  # noqa: E501