Repository: rotki/rotki

Issue:
Trades and Deposits/withdrawals should not return events with ignored assets
## Problem Definition

At the moment the history events API towards which we want to go is by default not returning any events that contain an ignored_asset. This is tweakable behaviour via the `exclude_ignored_assets` API argument but it's always set to true.

We also have events in the old format in the trades API endpoing and the asset movements API endpoint. Those endpoints do not follow the same logic. They will return all events despite containing an ignored asset.

The plan is to move trades and deposit/withdrawals into history events and essentially remove those api endpoints, but until that happens we need to have common functionality between them and not show to the user events with ignored assets.

## Task

Backend: Add same functionality we have in the history events endpoint to trades and deposit/withdrawals. Essentially a default true `exclude_ignored_assets` argument. Add it to the filter and make sure ignored asset events are skipped. Add tests.
Frontend: Make it tweakable.


Discussion:
Api endpoint for history events that's doing it as we need: https://github.com/rotki/rotki/blob/65b60d27984aecf1abbeb3bc53e4dedda86df893/rotkehlchen/api/rest.py#L3454
The schema for said endpoint where the filter is initialized and the argument is given: https://github.com/rotki/rotki/blob/65b60d27984aecf1abbeb3bc53e4dedda86df893/rotkehlchen/api/v1/schemas.py#L683
And the filter in the DB where the DB filter is used: https://github.com/rotki/rotki/blob/65b60d27984aecf1abbeb3bc53e4dedda86df893/rotkehlchen/db/filtering.py#L955

now there is also the `get_trades` api endpoint: https://github.com/rotki/rotki/blob/65b60d27984aecf1abbeb3bc53e4dedda86df893/rotkehlchen/api/rest.py#L747 and the `get_asset_movements`: https://github.com/rotki/rotki/blob/65b60d27984aecf1abbeb3bc53e4dedda86df893/rotkehlchen/api/rest.py#L877 which utilize the `TradesFilterQuery` and the `AssetMovementsFilterQuery` respectively.

What you need to do is introduce the same `exclude_ignored_assets` argument in those endpoints and make it work down in the filtering. And also write tests for those.

Resolving patch:
diff --git a/rotkehlchen/api/v1/schemas.py b/rotkehlchen/api/v1/schemas.py
--- a/rotkehlchen/api/v1/schemas.py
+++ b/rotkehlchen/api/v1/schemas.py
@@ -374,6 +374,7 @@ class TradesQuerySchema(
     trade_type = SerializableEnumField(enum_class=TradeType, load_default=None)
     location = LocationField(load_default=None)
     include_ignored_trades = fields.Boolean(load_default=True)
+    exclude_ignored_assets = fields.Boolean(load_default=True)
 
     def __init__(self, db: 'DBHandler') -> None:
         super().__init__()
@@ -447,6 +448,7 @@ def make_trades_query(
             trade_type=[data['trade_type']] if data['trade_type'] is not None else None,
             location=data['location'],
             trades_idx_to_ignore=trades_idx_to_ignore,
+            exclude_ignored_assets=data['exclude_ignored_assets'],
         )
 
         return {
@@ -883,6 +885,7 @@ class AssetMovementsQuerySchema(
     asset = AssetField(expected_type=Asset, load_default=None)
     action = SerializableEnumField(enum_class=AssetMovementCategory, load_default=None)
     location = LocationField(load_default=None)
+    exclude_ignored_assets = fields.Boolean(load_default=True)
 
     def __init__(
             self,
@@ -943,6 +946,7 @@ def make_asset_movements_query(
             assets=asset_list,
             action=[data['action']] if data['action'] is not None else None,
             location=data['location'],
+            exclude_ignored_assets=data['exclude_ignored_assets'],
         )
         return {
             'async_query': data['async_query'],
diff --git a/rotkehlchen/db/filtering.py b/rotkehlchen/db/filtering.py
--- a/rotkehlchen/db/filtering.py
+++ b/rotkehlchen/db/filtering.py
@@ -608,6 +608,7 @@ def make(
             trade_type: list[TradeType] | None = None,
             location: Location | None = None,
             trades_idx_to_ignore: set[str] | None = None,
+            exclude_ignored_assets: bool = False,
     ) -> 'TradesFilterQuery':
         if order_by_rules is None:
             order_by_rules = [('timestamp', True)]
@@ -657,6 +658,18 @@ def make(
                 values=trades_idx_to_ignore,
             ))
 
+        if exclude_ignored_assets is True:
+            filters.append(DBIgnoredAssetsFilter(
+                and_op=True,
+                asset_key='base_asset',
+                operator='NOT IN',
+            ))
+            filters.append(DBIgnoredAssetsFilter(
+                and_op=True,
+                asset_key='quote_asset',
+                operator='NOT IN',
+            ))
+
         filter_query.timestamp_filter = DBTimestampFilter(
             and_op=True,
             from_ts=from_ts,
@@ -681,6 +694,7 @@ def make(
             assets: tuple[Asset, ...] | None = None,
             action: list[AssetMovementCategory] | None = None,
             location: Location | None = None,
+            exclude_ignored_assets: bool = False,
     ) -> 'AssetMovementsFilterQuery':
         if order_by_rules is None:
             order_by_rules = [('timestamp', True)]
@@ -709,6 +723,13 @@ def make(
             filter_query.location_filter = DBLocationFilter(and_op=True, location=location)
             filters.append(filter_query.location_filter)
 
+        if exclude_ignored_assets is True:
+            filters.append(DBIgnoredAssetsFilter(
+                and_op=True,
+                asset_key='asset',
+                operator='NOT IN',
+            ))
+
         filter_query.timestamp_filter = DBTimestampFilter(
             and_op=True,
             from_ts=from_ts,


Test patch:
diff --git a/rotkehlchen/tests/api/test_exchanges.py b/rotkehlchen/tests/api/test_exchanges.py
--- a/rotkehlchen/tests/api/test_exchanges.py
+++ b/rotkehlchen/tests/api/test_exchanges.py
@@ -33,6 +33,7 @@
 from rotkehlchen.fval import FVal
 from rotkehlchen.globaldb.binance import GlobalDBBinance
 from rotkehlchen.globaldb.handler import GlobalDBHandler
+from rotkehlchen.tests.utils.accounting import toggle_ignore_an_asset
 from rotkehlchen.tests.utils.api import (
     api_url_for,
     assert_error_response,
@@ -801,6 +802,40 @@ def assert_okay(response):
             api_url_for(server, 'assetmovementsresource') + '?' + urlencode(data))
         assert_okay(response)
 
+    # now test `exclude_ignored_assets` query param
+    with setup.polo_patch:
+        data = {'only_cache': True}
+        without_ignore_response = requests.get(
+            api_url_for(server, 'assetmovementsresource'),
+            json=data,
+        )
+        without_ignore_response = assert_proper_response_with_result(without_ignore_response)
+
+        toggle_ignore_an_asset(rotkehlchen_api_server_with_exchanges, A_BTC)
+        data |= {'exclude_ignored_assets': True}
+        ignored_response = requests.get(api_url_for(server, 'assetmovementsresource'), json=data)
+        ignored_response = assert_proper_response_with_result(ignored_response)
+        default_response = requests.get(api_url_for(server, 'assetmovementsresource'))
+        default_response = assert_proper_response_with_result(default_response)
+        assert ignored_response == default_response, '`exclude_ignored_assets` should be True by default'  # noqa: E501
+
+        assert all(
+            x['entry']['asset'] != 'BTC'
+            for x in ignored_response['entries']
+        ), 'BTC asset movements should have now been ignored for accounting'
+        assert len(ignored_response['entries']) == 6, 'Only 6 asset movements should have been returned'  # noqa: E501
+
+        data['exclude_ignored_assets'] = False
+        not_exclude_response = requests.get(
+            api_url_for(server, 'assetmovementsresource'),
+            json=data,
+        )
+        not_exclude_response = assert_proper_response_with_result(not_exclude_response)
+        assert not_exclude_response == without_ignore_response, '`exclude_ignored_assets` = False should not exclude any asset movements'  # noqa: E501
+
+        # reset the ignored status of BTC
+        toggle_ignore_an_asset(rotkehlchen_api_server_with_exchanges, A_BTC)
+
     # and now test pagination
     data = {'only_cache': True, 'offset': 1, 'limit': 2, 'location': 'kraken'}
     response = requests.get(api_url_for(server, 'assetmovementsresource'), json=data)
diff --git a/rotkehlchen/tests/api/test_trades.py b/rotkehlchen/tests/api/test_trades.py
--- a/rotkehlchen/tests/api/test_trades.py
+++ b/rotkehlchen/tests/api/test_trades.py
@@ -7,10 +7,11 @@
 from rotkehlchen.accounting.structures.types import ActionType
 from rotkehlchen.api.v1.schemas import TradeSchema
 from rotkehlchen.constants import ONE, ZERO
-from rotkehlchen.constants.assets import A_AAVE, A_BTC, A_DAI, A_EUR, A_WETH
+from rotkehlchen.constants.assets import A_AAVE, A_BTC, A_DAI, A_EUR, A_GUSD, A_WETH
 from rotkehlchen.constants.limits import FREE_TRADES_LIMIT
 from rotkehlchen.exchanges.data_structures import Trade
 from rotkehlchen.fval import FVal
+from rotkehlchen.tests.utils.accounting import toggle_ignore_an_asset
 from rotkehlchen.tests.utils.api import (
     api_url_for,
     assert_error_response,
@@ -1149,11 +1150,16 @@ def test_query_trades_associated_locations(rotkehlchen_api_server_with_exchanges
 
 
 def test_ignoring_trades(rotkehlchen_api_server):
-    """Check that ignoring trades filter works as expected."""
+    """Check that ignoring trades and ignoring assets filter works as expected."""
     rotki = rotkehlchen_api_server.rest_api.rotkehlchen
     trades = make_random_trades(3)
     trade_to_ignore = trades[0].identifier
 
+    trades[2].base_asset = A_AAVE
+    trades[2].quote_asset = A_GUSD
+    trade_with_asset_ignored = trades[2].identifier
+    asset_to_ignore = A_GUSD
+
     # populate db with trades
     with rotki.data.db.user_write() as cursor:
         rotki.data.db.add_trades(cursor, trades)
@@ -1170,7 +1176,7 @@ def test_ignoring_trades(rotkehlchen_api_server):
         result = rotki.data.db.get_ignored_action_ids(cursor, None)
     assert result[ActionType.TRADE] == {trade_to_ignore}
 
-    # now fetch trades and check for the behaviour of the filter
+    # now fetch trades and check for the behaviour of the `include_ignored_trades` filter
     response = requests.get(
         api_url_for(
             rotkehlchen_api_server,
@@ -1191,6 +1197,32 @@ def test_ignoring_trades(rotkehlchen_api_server):
     assert len(result['entries']) == len(trades) - 1
     assert trade_to_ignore not in {entry['entry']['trade_id'] for entry in result['entries']}
 
+    # ignore an asset
+    toggle_ignore_an_asset(rotkehlchen_api_server, asset_to_ignore)
+
+    # now fetch trades and check for the behaviour of the `exclude_ignored_assets` filter
+    response = requests.get(
+        api_url_for(
+            rotkehlchen_api_server,
+            'tradesresource',
+        ), json={'exclude_ignored_assets': 'True'},
+    )
+    result = assert_proper_response_with_result(response)
+    assert len(result['entries']) == len(trades) - 1
+    assert trade_with_asset_ignored not in {
+        entry['entry']['trade_id'] for entry in result['entries']
+    }
+
+    response = requests.get(
+        api_url_for(
+            rotkehlchen_api_server,
+            'tradesresource',
+        ), json={'exclude_ignored_assets': 'False'},
+    )
+    result = assert_proper_response_with_result(response)
+    assert len(result['entries']) == len(trades)
+    assert trade_with_asset_ignored in {entry['entry']['trade_id'] for entry in result['entries']}
+
 
 def test_ignoring_trades_with_pagination(rotkehlchen_api_server):
     """Check that pagination is respected when `include_ignored_trades` is True."""
diff --git a/rotkehlchen/tests/utils/accounting.py b/rotkehlchen/tests/utils/accounting.py
--- a/rotkehlchen/tests/utils/accounting.py
+++ b/rotkehlchen/tests/utils/accounting.py
@@ -5,16 +5,21 @@
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Optional
 
+import requests
+
 from rotkehlchen.accounting.export.csv import CSV_INDEX_OFFSET, FILENAME_ALL_CSV
 from rotkehlchen.accounting.mixins.event import AccountingEventMixin, AccountingEventType
 from rotkehlchen.accounting.pnl import PNL, PnlTotals
 from rotkehlchen.accounting.structures.processed_event import ProcessedAccountingEvent
+from rotkehlchen.api.server import APIServer
+from rotkehlchen.assets.asset import Asset
 from rotkehlchen.constants import ONE, ZERO
 from rotkehlchen.constants.assets import A_BTC, A_ETH, A_EUR, A_USDC, A_WBTC
 from rotkehlchen.db.filtering import ReportDataFilterQuery
 from rotkehlchen.db.reports import DBAccountingReports
 from rotkehlchen.exchanges.data_structures import Trade
 from rotkehlchen.fval import FVal
+from rotkehlchen.tests.utils.api import api_url_for
 from rotkehlchen.types import AssetAmount, Fee, Location, Price, Timestamp, TimestampMS, TradeType
 from rotkehlchen.utils.version_check import get_current_version
 
@@ -345,3 +350,47 @@ def assert_csv_export(
                 expected_csv_data=expected_csv_data,
                 expected_pnls=expected_pnls,
             )
+
+
+def toggle_ignore_an_asset(
+        rotkehlchen_api_server: APIServer,
+        asset_to_ignore: Asset,
+) -> None:
+    """Utility function to add/remove an ignored asset"""
+    rotki = rotkehlchen_api_server.rest_api.rotkehlchen
+    # check if the asset is already ignored
+    with rotki.data.db.conn.read_ctx() as cursor:
+        result = rotki.data.db.get_ignored_asset_ids(cursor)
+    already_ignored = asset_to_ignore.identifier in result
+
+    if already_ignored:
+        # remove the asset from the ignored list
+        response = requests.delete(
+            api_url_for(
+                rotkehlchen_api_server,
+                'ignoredassetsresource',
+            ), json={'assets': [asset_to_ignore.identifier]},
+        )
+        assert response.status_code == 200
+        response_result = response.json().get('result', [])
+        assert asset_to_ignore.identifier not in response_result
+
+        with rotki.data.db.conn.read_ctx() as cursor:
+            result = rotki.data.db.get_ignored_asset_ids(cursor)
+        assert asset_to_ignore.identifier not in result
+        return
+
+    # else add the asset to the ignored list
+    response = requests.put(
+        api_url_for(
+            rotkehlchen_api_server,
+            'ignoredassetsresource',
+        ), json={'assets': [asset_to_ignore.identifier]},
+    )
+    assert response.status_code == 200
+    response_result = response.json().get('result', [])
+    assert asset_to_ignore.identifier in response_result
+
+    with rotki.data.db.conn.read_ctx() as cursor:
+        result = rotki.data.db.get_ignored_asset_ids(cursor)
+    assert asset_to_ignore.identifier in result