Repository: rotki/rotki

Issue:
Editing/deleting old external trades does not work
## Problem Definition

It seems that editing/deleting external trades does not work. That is because of the introduction of a new trade id schema at 30/09. At the time upgrading the user's DB to upgrade the ids of all existing trades was omitted.

As a result whenever after that date the user attempts to edit or delete older external trades it fails with an id mismatch.

## Task

- [ ] Create a DB upgrade (v6 -> v7) that will bring all existing trades saved in the DB on par with the  current ID schema

- [ ] Write a test for this upgrade


Discussion:

Resolving patch:
diff --git a/rotkehlchen/db/settings.py b/rotkehlchen/db/settings.py
--- a/rotkehlchen/db/settings.py
+++ b/rotkehlchen/db/settings.py
@@ -7,7 +7,7 @@
 from rotkehlchen.typing import FiatAsset, Timestamp
 from rotkehlchen.user_messages import MessagesAggregator
 
-ROTKEHLCHEN_DB_VERSION = 6
+ROTKEHLCHEN_DB_VERSION = 7
 DEFAULT_TAXFREE_AFTER_PERIOD = YEAR_IN_SECONDS
 DEFAULT_INCLUDE_CRYPTO2CRYPTO = True
 DEFAULT_INCLUDE_GAS_COSTS = True
diff --git a/rotkehlchen/db/upgrade_manager.py b/rotkehlchen/db/upgrade_manager.py
--- a/rotkehlchen/db/upgrade_manager.py
+++ b/rotkehlchen/db/upgrade_manager.py
@@ -9,6 +9,7 @@
 from rotkehlchen.db.asset_rename import rename_assets_in_db
 from rotkehlchen.db.settings import ROTKEHLCHEN_DB_VERSION
 from rotkehlchen.db.upgrades.v5_v6 import upgrade_v5_to_v6
+from rotkehlchen.db.upgrades.v6_v7 import upgrade_v6_to_v7
 from rotkehlchen.errors import DBUpgradeError
 from rotkehlchen.logging import RotkehlchenLogsAdapter
 from rotkehlchen.typing import SupportedBlockchain
@@ -79,6 +80,10 @@ def _eth_rpc_port_to_eth_rpc_endpoint(db: 'DBHandler') -> None:
         from_version=5,
         function=upgrade_v5_to_v6,
     ),
+    UpgradeRecord(
+        from_version=6,
+        function=upgrade_v6_to_v7,
+    ),
 ]
 
 
diff --git a/rotkehlchen/db/upgrades/v6_v7.py b/rotkehlchen/db/upgrades/v6_v7.py
new file mode 100644
--- /dev/null
+++ b/rotkehlchen/db/upgrades/v6_v7.py
@@ -0,0 +1,150 @@
+from typing import TYPE_CHECKING
+
+from rotkehlchen.crypto import sha3
+from rotkehlchen.errors import DBUpgradeError
+from rotkehlchen.typing import Location, TradeType
+
+if TYPE_CHECKING:
+    from rotkehlchen.db.dbhandler import DBHandler
+
+
+def _v6_deserialize_location_from_db(symbol: str) -> Location:
+    """We copy the deserialize_location_from_db() function at v6
+
+    This is done in case the function ever changes in the future. Also another
+    difference is that instead of DeserializationError this throws a DBUpgradeError
+    """
+    if symbol == 'A':
+        return Location.EXTERNAL
+    elif symbol == 'B':
+        return Location.KRAKEN
+    elif symbol == 'C':
+        return Location.POLONIEX
+    elif symbol == 'D':
+        return Location.BITTREX
+    elif symbol == 'E':
+        return Location.BINANCE
+    elif symbol == 'F':
+        return Location.BITMEX
+    elif symbol == 'G':
+        return Location.COINBASE
+    elif symbol == 'H':
+        return Location.TOTAL
+    elif symbol == 'I':
+        return Location.BANKS
+    elif symbol == 'J':
+        return Location.BLOCKCHAIN
+    else:
+        raise DBUpgradeError(
+            f'Failed to deserialize location. Unknown symbol {symbol} for location found in DB',
+        )
+
+
+def _v6_deserialize_trade_type_from_db(symbol: str) -> TradeType:
+    """We copy the deserialize_trade_type_from_db() function at v6
+
+    This is done in case the function ever changes in the future. Also another
+    difference is that instead of DeserializationError this throws a DBUpgradeError
+    """
+    if symbol == 'A':
+        return TradeType.BUY
+    elif symbol == 'B':
+        return TradeType.SELL
+    elif symbol == 'C':
+        return TradeType.SETTLEMENT_BUY
+    elif symbol == 'D':
+        return TradeType.SETTLEMENT_SELL
+    else:
+        raise DBUpgradeError(
+            f'Failed to deserialize trade type. Unknown DB symbol {symbol} for trade type in DB',
+        )
+
+
+def _upgrade_trades_table(db: 'DBHandler') -> None:
+    cursor = db.conn.cursor()
+    # This is the data we need from trades table at v6
+    query = cursor.execute(
+        """SELECT time, location, pair, type, amount, rate, fee, fee_currency,
+        link, notes FROM trades;""",
+    )
+    trade_tuples = []
+    for result in query:
+        # for each trade get all the relevant data
+        time = result[0]
+        db_location = result[1]
+        pair = result[2]
+        db_trade_type = result[3]
+        amount = result[4]
+        rate = result[5]
+        fee = result[6]
+        fee_currency = result[7]
+        link = result[8]
+        notes = result[9]
+        # make sure to deserialize the db enums
+        location = _v6_deserialize_location_from_db(db_location)
+        trade_type = _v6_deserialize_trade_type_from_db(db_trade_type)
+
+        new_trade_id = sha3(
+            (str(location) + str(time) + str(trade_type) + pair + amount + rate + link).encode(),
+        ).hex()
+        trade_tuples.append((
+            new_trade_id,
+            time,
+            db_location,
+            pair,
+            db_trade_type,
+            amount,
+            rate,
+            fee,
+            fee_currency,
+            link,
+            notes,
+        ))
+
+    # We got all the external trades data. Now delete the old table and create
+    # the new one
+    cursor.execute('DROP TABLE trades;')
+    db.conn.commit()
+    # This is the scheme of the trades table at v7 from db/utils.py
+    cursor.execute("""
+    CREATE TABLE IF NOT EXISTS trades (
+    id TEXT PRIMARY KEY,
+    time INTEGER,
+    location CHAR(1) NOT NULL DEFAULT('A') REFERENCES location(location),
+    pair VARCHAR[24],
+    type CHAR(1) NOT NULL DEFAULT ('B') REFERENCES trade_type(type),
+    amount TEXT,
+    rate TEXT,
+    fee TEXT,
+    fee_currency VARCHAR[10],
+    link TEXT,
+    notes TEXT
+    );""")
+    db.conn.commit()
+
+    # and finally move the data to the new table
+    cursor.executemany(
+        'INSERT INTO trades('
+        '  id, '
+        '  time,'
+        '  location,'
+        '  pair,'
+        '  type,'
+        '  amount,'
+        '  rate,'
+        '  fee,'
+        '  fee_currency,'
+        '  link,'
+        '  notes)'
+        'VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
+        trade_tuples,
+    )
+    db.conn.commit()
+
+
+def upgrade_v6_to_v7(db: 'DBHandler') -> None:
+    """Upgrades the DB from v6 to v7
+
+    - upgrades trades table to use a new id scheme
+    """
+    _upgrade_trades_table(db)


Test patch:
diff --git a/rotkehlchen/tests/data/v6_rotkehlchen.db b/rotkehlchen/tests/data/v6_rotkehlchen.db
new file mode 100644
Binary files /dev/null and b/rotkehlchen/tests/data/v6_rotkehlchen.db differ
diff --git a/rotkehlchen/tests/db/test_db_upgrades.py b/rotkehlchen/tests/db/test_db_upgrades.py
--- a/rotkehlchen/tests/db/test_db_upgrades.py
+++ b/rotkehlchen/tests/db/test_db_upgrades.py
@@ -452,15 +452,80 @@ def test_upgrade_db_5_to_6(data_dir, username):
         '6395.62369468235924484',  # usd_value
     )]
     results = results.fetchall()
+
     assert results == expected_results
 
     # Also make sure the cache files were deleted
     for filename in fake_cache_files:
         assert not os.path.exists(filename)
-    # Finally slso make sure that we have updated to the target version
+    # Finally also make sure that we have updated to the target version
     assert db.get_version() == 6
 
 
+def test_upgrade_db_6_to_7(data_dir, username):
+    """Test upgrading the DB from version 6 to version 7.
+
+    Test that the trades tables has the new trade ids.
+    """
+    msg_aggregator = MessagesAggregator()
+    userdata_dir = os.path.join(data_dir, username)
+    os.mkdir(userdata_dir)
+    dir_path = os.path.dirname(os.path.realpath(__file__))
+    copyfile(
+        os.path.join(os.path.dirname(dir_path), 'data', 'v6_rotkehlchen.db'),
+        os.path.join(userdata_dir, 'rotkehlchen.db'),
+    )
+
+    with target_patch(target_version=7):
+        db = DBHandler(user_data_dir=userdata_dir, password='123', msg_aggregator=msg_aggregator)
+
+    cursor = db.conn.cursor()
+    query = (
+        'SELECT id,'
+        '  time,'
+        '  location,'
+        '  pair,'
+        '  type,'
+        '  amount,'
+        '  rate,'
+        '  fee,'
+        '  fee_currency,'
+        '  link,'
+        '  notes FROM trades ORDER BY time ASC;'
+    )
+    results = cursor.execute(query)
+    expected_results = [(
+        'fafd58fce59b9d5c5308be1994633e4adc35bce82ac04a9f793f8137ba4a6793',
+        1568928120,  # time
+        'A',  # external symbol for location Enum
+        'ETH_EUR',  # pair
+        'B',  # type sell
+        '10',  # amount
+        '196.6',  # rate
+        '0.001',  # fee
+        'ETH',  # fee currency
+        '',  # link
+        'Test Sell 1',  # notes
+    ), (
+        'dca097b6e0b5aa2b3d5fc26fd9d83527db7fe447e2a42b4000799ddb17bf1e83',
+        1569010800,  # time
+        'A',  # external symbol for location Enum
+        'BTC_EUR',  # pair
+        'A',  # type buy
+        '0.5',  # amount
+        '9240.1',  # rate
+        '0.1',  # fee
+        'EUR',  # fee currency
+        '',  # link
+        'Test Buy 1',  # notes
+    )]
+    results = results.fetchall()
+    assert results == expected_results
+
+    # Finally also make sure that we have updated to the target version
+    assert db.get_version() == 7
+
+
 def test_db_newer_than_software_raises_error(data_dir, username):
     """
     If the DB version is greater than the current known version in the
diff --git a/rotkehlchen/tests/test_utils.py b/rotkehlchen/tests/test_utils.py
--- a/rotkehlchen/tests/test_utils.py
+++ b/rotkehlchen/tests/test_utils.py
@@ -43,9 +43,6 @@ def test_tuple_in_process_result():
 
 
 def test_iso8601ts_to_timestamp():
-    # Get timezone offset to UTC
-    utc_offset = time.timezone if (time.localtime().tm_isdst == 0) else time.altzone
-
     assert iso8601ts_to_timestamp('2018-09-09T12:00:00.000Z') == 1536494400
     assert iso8601ts_to_timestamp('2011-01-01T04:13:22.220Z') == 1293855202
     assert iso8601ts_to_timestamp('1986-11-04T16:23:57.921Z') == 531505438
@@ -53,6 +50,9 @@ def test_iso8601ts_to_timestamp():
     timezone_ts_str = '1997-07-16T22:30'
     timezone_ts_at_utc = 869092200
     assert iso8601ts_to_timestamp(timezone_ts_str + 'Z') == timezone_ts_at_utc
+    # The utc offset for July should be time.altzone since it's in DST
+    # https://stackoverflow.com/questions/3168096/getting-computers-utc-offset-in-python
+    utc_offset = time.altzone
     assert iso8601ts_to_timestamp(timezone_ts_str) == timezone_ts_at_utc + utc_offset
     assert iso8601ts_to_timestamp('1997-07-16T22:30+01:00') == 869088600
     assert iso8601ts_to_timestamp('1997-07-16T22:30:45+01:00') == 869088645