Repository: rotki/rotki

Issue:
Disallow addition and editing of external trade that could result in invalid trade fees
## Problem Definition

At the moment it's possible to add an external trade through the API (or edit it) in a way that could result in invalid fees.

Specifically it seems that it's possible to have a fee but no fee currency which should not be possible.

We need to disallow this in the API and add a test for both add/edit that should fail.

## Task

The api code that adds a trade is here: https://github.com/rotki/rotki/blob/7f7478df40eb6b9397d3f0df945e23f0cc73a924/rotkehlchen/api/rest.py#L812

The api code that edits a trade is here:
https://github.com/rotki/rotki/blob/7f7478df40eb6b9397d3f0df945e23f0cc73a924/rotkehlchen/api/rest.py#L847

1. We need to make sure that if fee or fee_currency is given both of them are given and not only one. Also that fee is not ZERO.
2. Add tests for this.
3. Also add a changelog entry.
4. Add a DB upgrade to go through existing trades and if fee_currency is missing then also make fee as null.


Discussion:

Resolving patch:
diff --git a/rotkehlchen/api/v1/schemas.py b/rotkehlchen/api/v1/schemas.py
--- a/rotkehlchen/api/v1/schemas.py
+++ b/rotkehlchen/api/v1/schemas.py
@@ -32,7 +32,7 @@
     is_valid_kusama_address,
     is_valid_polkadot_address,
 )
-from rotkehlchen.constants.misc import ONE
+from rotkehlchen.constants.misc import ONE, ZERO
 from rotkehlchen.db.filtering import (
     AssetMovementsFilterQuery,
     Eth2DailyStatsFilterQuery,
@@ -517,6 +517,24 @@ class TradeSchema(Schema):
     link = fields.String(load_default=None)
     notes = fields.String(load_default=None)
 
+    @validates_schema
+    def validate_fee_and_fee_currency(  # pylint: disable=no-self-use
+        self,
+        data: Dict[str, Any],
+        **_kwargs: Any,
+    ) -> None:
+        """This validation checks that fee_currency is provided whenever fee is given and vice versa.
+         It also checks that fee is not a zero value when both fee and fee_currency are provided.
+        """
+        fee = data.get('fee')
+        fee_currency = data.get('fee_currency')
+
+        if not all([fee, fee_currency]) and any([fee, fee_currency]):
+            raise ValidationError('fee and fee_currency must be provided', field_name='fee')
+
+        if fee is not None and fee == ZERO:
+            raise ValidationError('fee cannot be zero', field_name='fee')
+
 
 class LedgerActionSchema(Schema):
     identifier = fields.Integer(load_default=None, required=False)
diff --git a/rotkehlchen/db/upgrades/v31_v32.py b/rotkehlchen/db/upgrades/v31_v32.py
--- a/rotkehlchen/db/upgrades/v31_v32.py
+++ b/rotkehlchen/db/upgrades/v31_v32.py
@@ -109,6 +109,11 @@ def _refactor_manual_balance_id(cursor: 'Cursor') -> None:
     )
 
 
+def _update_fee_for_existing_trades(cursor: 'Cursor') -> None:
+    cursor.execute('UPDATE trades SET fee = NULL WHERE fee_currency IS NULL')
+    cursor.execute('UPDATE trades SET fee_currency = NULL WHERE fee IS NULL')
+
+
 def upgrade_v31_to_v32(db: 'DBHandler') -> None:
     """Upgrades the DB from v31 to v32
     - use new identifiers for the history_events table. The id will be generated by sqlite
@@ -119,10 +124,13 @@ def upgrade_v31_to_v32(db: 'DBHandler') -> None:
 
     -Remove all gitcoin grant related data that was pulled from their API and saved in
     specific tables along with the tables themselves
+
+    -Sets fee to null for existing trades if fee_currency is missing.
     """
     cursor = db.conn.cursor()
     _upgrade_history_events(cursor)
     _remove_gitcoin(cursor)
     _add_new_tables(cursor)
     _refactor_manual_balance_id(cursor)
+    _update_fee_for_existing_trades(cursor)
     db.conn.commit()


Test patch:
diff --git a/frontend/app/tests/e2e/fixtures/history/trades.json b/frontend/app/tests/e2e/fixtures/history/trades.json
--- a/frontend/app/tests/e2e/fixtures/history/trades.json
+++ b/frontend/app/tests/e2e/fixtures/history/trades.json
@@ -6,7 +6,7 @@
     "quote": "EUR",
     "quote_id": "EUR",
     "rate": "268.678317859",
-    "fee": "0",
+    "fee": "1",
     "fee_currency": "DAI",
     "fee_id": "_ceth_0x6B175474E89094C44Da98b954EedeAC495271d0F",
     "amount": "82",
@@ -22,7 +22,7 @@
     "quote_id": "EUR",
     "rate": "4",
     "quote_amount": "160",
-    "fee": "0",
+    "fee": "1",
     "fee_currency": "DAI",
     "fee_id": "_ceth_0x6B175474E89094C44Da98b954EedeAC495271d0F",
     "amount": "40",
diff --git a/rotkehlchen/tests/api/test_trades.py b/rotkehlchen/tests/api/test_trades.py
--- a/rotkehlchen/tests/api/test_trades.py
+++ b/rotkehlchen/tests/api/test_trades.py
@@ -436,6 +436,57 @@ def test_add_trades(rotkehlchen_api_server):
         status_code=HTTPStatus.BAD_REQUEST,
     )
 
+    # Test with fee & without fee_currency
+    fee_and_no_fee_currency_trade = {
+        'timestamp': 1595640208,
+        'location': 'external',
+        'base_asset': 'ETH',
+        'quote_asset': 'USD',
+        'trade_type': 'buy',
+        'amount': '1.5541',
+        'rate': '22.1',
+        'fee': '0.55',
+    }
+
+    response = requests.put(
+        api_url_for(
+            rotkehlchen_api_server,
+            "tradesresource",
+        ), json=fee_and_no_fee_currency_trade,
+    )
+
+    assert_error_response(
+        response=response,
+        contained_in_msg='fee and fee_currency must be provided',
+        status_code=HTTPStatus.BAD_REQUEST,
+    )
+
+    # Test with fee is zero
+    fee_is_zero_trade = {
+        'timestamp': 1595640208,
+        'location': 'external',
+        'base_asset': 'ETH',
+        'quote_asset': 'USD',
+        'trade_type': 'buy',
+        'amount': '1.5541',
+        'rate': '22.1',
+        'fee': '0',
+        'fee_currency': 'USD',
+    }
+
+    response = requests.put(
+        api_url_for(
+            rotkehlchen_api_server,
+            "tradesresource",
+        ), json=fee_is_zero_trade,
+    )
+
+    assert_error_response(
+        response=response,
+        contained_in_msg='fee cannot be zero',
+        status_code=HTTPStatus.BAD_REQUEST,
+    )
+
 
 def assert_all_missing_fields_are_handled(correct_trade, server):
     fields = correct_trade.keys()
@@ -708,6 +759,20 @@ def test_add_trades_errors(rotkehlchen_api_server):
         contained_in_msg='Not a valid string',
         status_code=HTTPStatus.BAD_REQUEST,
     )
+    # Test that '0' value for fee is handled
+    broken_trade = correct_trade.copy()
+    broken_trade['fee'] = '0'
+    response = requests.put(
+        api_url_for(
+            rotkehlchen_api_server,
+            "tradesresource",
+        ), json=broken_trade,
+    )
+    assert_error_response(
+        response=response,
+        contained_in_msg='fee cannot be zero',
+        status_code=HTTPStatus.BAD_REQUEST,
+    )
 
 
 def _check_trade_is_edited(original_trade: Dict[str, Any], result_trade: Dict[str, Any]) -> None:
@@ -742,6 +807,38 @@ def test_edit_trades(rotkehlchen_api_server_with_exchanges):
     # get the binance trades
     original_binance_trades = [t['entry'] for t in trades if t['entry']['location'] == 'binance']
 
+    # get the poloniex trades
+    original_poloniex_trades = [t['entry'] for t in trades if t['entry']['location'] == 'poloniex']
+
+    # Test that setting '0' as value for `fee` fails
+    original_poloniex_trades[0]['fee'] = '0'
+    response = requests.patch(
+        api_url_for(
+            rotkehlchen_api_server_with_exchanges,
+            "tradesresource",
+        ), json=original_poloniex_trades[0],
+    )
+    assert_error_response(response, contained_in_msg='fee cannot be zero')
+
+    # Test that popping either of `fee` or `fee_currency` fails
+    original_poloniex_trades[1].pop('fee', None)
+    response = requests.patch(
+        api_url_for(
+            rotkehlchen_api_server_with_exchanges,
+            "tradesresource",
+        ), json=original_poloniex_trades[1],
+    )
+    assert_error_response(response, contained_in_msg='fee and fee_currency must be provided')
+
+    # Test that setting both `fee` and `fee_currency` passes
+    response = requests.patch(
+        api_url_for(
+            rotkehlchen_api_server_with_exchanges,
+            "tradesresource",
+        ), json=original_poloniex_trades[2],
+    )
+    assert_proper_response(response)
+
     for trade in original_binance_trades:
         # edit two fields of each binance trade
         trade['amount'] = '1337.5'
diff --git a/rotkehlchen/tests/data/v31_rotkehlchen.db b/rotkehlchen/tests/data/v31_rotkehlchen.db
Binary files a/rotkehlchen/tests/data/v31_rotkehlchen.db and b/rotkehlchen/tests/data/v31_rotkehlchen.db differ
diff --git a/rotkehlchen/tests/db/test_db_upgrades.py b/rotkehlchen/tests/db/test_db_upgrades.py
--- a/rotkehlchen/tests/db/test_db_upgrades.py
+++ b/rotkehlchen/tests/db/test_db_upgrades.py
@@ -2416,6 +2416,7 @@ def test_upgrade_db_31_to_32(user_data_dir):  # pylint: disable=unused-argument
 
     - Check that subtype is correctly updated
     - Check that gitcoin data is properly delete
+    - Check that trades with fee missing, sets fee_currency to NULL and vice versa
     """
     msg_aggregator = MessagesAggregator()
     _use_prepared_db(user_data_dir, 'v31_rotkehlchen.db')
@@ -2461,7 +2462,14 @@ def test_upgrade_db_31_to_32(user_data_dir):  # pylint: disable=unused-argument
         'manually_tracked_balances;',
     ).fetchall()
 
-    db_v31.logout()
+    # check that the trades with invalid fee/fee_currency are present at this point
+    trades_before = cursor.execute('SELECT * FROM trades WHERE id != ? AND id != ?', ("foo1", "foo2")).fetchall()  # noqa: E501
+    assert trades_before == [
+        ('1111111', 1595640208, 'external', 'ETH', 'USD', 'buy', '1.5541', '22.1', '3.4', 'USD', None, None),  # noqa: E501
+        ('1111112', 1595640208, 'external', 'ETH', 'USD', 'buy', '1.5541', '22.1', '3.4', None, None, None),  # noqa: E501
+        ('1111113', 1595640208, 'external', 'ETH', 'USD', 'buy', '1.5541', '22.1', None, 'USD', None, None),  # noqa: E501
+    ]
+
     # Execute upgrade
     db = _init_db_with_target_version(
         target_version=32,
@@ -2506,6 +2514,14 @@ def test_upgrade_db_31_to_32(user_data_dir):  # pylint: disable=unused-argument
 
     assert [1, 2, 3] == list(map(lambda x: x[0], manual_balance_ids))
 
+    # Check that trades with fee missing sets fee_currency to NULL and vice versa
+    trades_expected = cursor.execute('SELECT * FROM trades WHERE id != ? AND id != ?', ('foo1', 'foo2')).fetchall()  # noqa: E501
+    assert trades_expected == [
+        ('1111111', 1595640208, 'external', 'ETH', 'USD', 'buy', '1.5541', '22.1', '3.4', 'USD', None, None),  # noqa: E501
+        ('1111112', 1595640208, 'external', 'ETH', 'USD', 'buy', '1.5541', '22.1', None, None, None, None),  # noqa: E501
+        ('1111113', 1595640208, 'external', 'ETH', 'USD', 'buy', '1.5541', '22.1', None, None, None, None),  # noqa: E501
+    ]
+
 
 def test_latest_upgrade_adds_remove_tables(user_data_dir):
     """