Repository: rotki/rotki

Issue:
Don't ignore exchanges in PnL if API keys are removed
## Problem Definition

If a user removes exchange API keys from the app then trades, ledger actions, etc. from that exchange will no longer be used in PnL reports even though they are still stored in the database. That is a problem and should be fixed. The same problem does not appear for the trades view, as [query_trades](https://github.com/rotki/rotki/blob/c974019abddb4bf01d8742658c45d8fd8c6798bc/rotkehlchen/history/events.py#L135) does not check active exchanges but simply gets paginated trades in the DB.

The way to implement this would probably be by doing something similar like the trades query in the part of gathering history events for the PnL report.

## Task

- [ ] Make it so that trades from an exchange that is not tracked (no key in the app, so not part of the exchange manager) are taken into account in the PnL report.
- [ ] Write a test for it.


Discussion:

Resolving patch:
diff --git a/rotkehlchen/exchanges/exchange.py b/rotkehlchen/exchanges/exchange.py
--- a/rotkehlchen/exchanges/exchange.py
+++ b/rotkehlchen/exchanges/exchange.py
@@ -42,6 +42,7 @@
 ]
 
 ExchangeHistoryFailCallback = Callable[[str], None]
+ExchangeHistoryNewStepCallback = Callable[[str], None]
 
 
 class ExchangeInterface(CacheableMixIn, LockableQueryMixIn):
@@ -477,45 +478,48 @@ def query_history_with_callbacks(
             self,
             start_ts: Timestamp,
             end_ts: Timestamp,
-            success_callback: ExchangeHistorySuccessCallback,
             fail_callback: ExchangeHistoryFailCallback,
+            new_step_data: Optional[Tuple[ExchangeHistoryNewStepCallback, str]] = None,
     ) -> None:
         """Queries the historical event endpoints for this exchange and performs actions.
-
-        In case of success passes the result to success_callback.
+        The results are saved in the DB.
         In case of failure passes the error to failure_callback
+
+        `new_step_data` argument contains callback and exchange name to be used for steps.
         """
+        if new_step_data is not None:
+            new_step_callback, exchange_name = new_step_data
+            new_step_callback(f'Querying {exchange_name} trades history')
         try:
-            trades_history = self.query_trade_history(
+            self.query_trade_history(
                 start_ts=start_ts,
                 end_ts=end_ts,
                 only_cache=False,
             )
-            margin_history = self.query_margin_history(
+            if new_step_data is not None:
+                new_step_callback(f'Querying {exchange_name} margin history')
+            self.query_margin_history(
                 start_ts=start_ts,
                 end_ts=end_ts,
             )
-            asset_movements = self.query_deposits_withdrawals(
+            if new_step_data is not None:
+                new_step_callback(f'Querying {exchange_name} asset movements history')
+            self.query_deposits_withdrawals(
                 start_ts=start_ts,
                 end_ts=end_ts,
                 only_cache=False,
             )
-            # Query (and save in the DB) any ledger actions. They will be included in history l8er
+            if new_step_data is not None:
+                new_step_callback(f'Querying {exchange_name} ledger actions history')
             self.query_income_loss_expense(
                 start_ts=start_ts,
                 end_ts=end_ts,
                 only_cache=False,
             )
-            exchange_specific_data = self.query_exchange_specific_history(  # pylint: disable=assignment-from-none  # noqa: E501
+            # No new step for exchange_specific_history since it is not used in any exchange atm.
+            self.query_exchange_specific_history(
                 start_ts=start_ts,
                 end_ts=end_ts,
             )
-            success_callback(
-                trades_history,
-                margin_history,
-                asset_movements,
-                exchange_specific_data,
-            )
-
         except RemoteError as e:
             fail_callback(str(e))
diff --git a/rotkehlchen/history/events.py b/rotkehlchen/history/events.py
--- a/rotkehlchen/history/events.py
+++ b/rotkehlchen/history/events.py
@@ -1,6 +1,6 @@
 import logging
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, List, Tuple
+from typing import TYPE_CHECKING, List, Tuple
 
 from rotkehlchen.accounting.structures.base import HistoryBaseEntry
 from rotkehlchen.constants.misc import ZERO
@@ -14,11 +14,11 @@
 from rotkehlchen.db.history_events import DBHistoryEvents
 from rotkehlchen.db.ledger_actions import DBLedgerActions
 from rotkehlchen.errors.misc import RemoteError
-from rotkehlchen.exchanges.data_structures import AssetMovement, MarginPosition, Trade
+from rotkehlchen.exchanges.data_structures import AssetMovement, Trade
 from rotkehlchen.exchanges.manager import SUPPORTED_EXCHANGES, ExchangeManager
 from rotkehlchen.fval import FVal
 from rotkehlchen.logging import RotkehlchenLogsAdapter
-from rotkehlchen.types import EXTERNAL_LOCATION, Location, Timestamp
+from rotkehlchen.types import Location, Timestamp
 from rotkehlchen.user_messages import MessagesAggregator
 from rotkehlchen.utils.misc import timestamp_to_date
 
@@ -37,12 +37,12 @@
 # eth transactions
 # eth receipts
 # eth tx decoding
+# reading from the db trades, asset movements, margin positions
 # ledger actions
-# external location trades -> len(EXTERNAL_LOCATION)
 # eth2
 # base history entries
 # Please, update this number each time a history query step is either added or removed
-NUM_HISTORY_QUERY_STEPS_EXCL_EXCHANGES = 6 + len(EXTERNAL_LOCATION)
+NUM_HISTORY_QUERY_STEPS_EXCL_EXCHANGES = 7
 
 
 class EventsHistorian:
@@ -290,7 +290,10 @@ def get_history(
         """
         self._reset_variables()
         step = 0
-        total_steps = len(self.exchange_manager.connected_exchanges) + NUM_HISTORY_QUERY_STEPS_EXCL_EXCHANGES  # noqa: E501
+        total_steps = (
+            len(self.exchange_manager.connected_exchanges) * 4 +  # 4 substeps in each exchange
+            NUM_HISTORY_QUERY_STEPS_EXCL_EXCHANGES
+        )
         log.info(
             'Get/create trade history',
             start_ts=start_ts,
@@ -300,39 +303,54 @@ def get_history(
         history: List['AccountingEventMixin'] = []
         empty_or_error = ''
 
-        def populate_history_cb(
-                trades_history: List[Trade],
-                margin_history: List[MarginPosition],
-                result_asset_movements: List[AssetMovement],
-                exchange_specific_data: Any,
-        ) -> None:
-            """This callback will run for succesfull exchange history query
-
-            We don't include ledger actions here since we simply gather all of them at the end
-            """
-            history.extend(trades_history)
-            history.extend(margin_history)
-            history.extend(result_asset_movements)
-
-            if exchange_specific_data:
-                pass  # this used to be only for polo loans -- removed now. TODO: Think if needed
-
         def fail_history_cb(error_msg: str) -> None:
             """This callback will run for failure in exchange history query"""
             nonlocal empty_or_error
             empty_or_error += '\n' + error_msg
 
+        def new_step_cb(state_name: str) -> None:
+            """This callback will run for each new step in exchange history query"""
+            nonlocal step
+            step = self._increase_progress(step, total_steps)
+            self.processing_state_name = state_name
+
         for exchange in self.exchange_manager.iterate_exchanges():
             self.processing_state_name = f'Querying {exchange.name} exchange history'
             exchange.query_history_with_callbacks(
                 # We need to have history of exchanges since before the range
                 start_ts=Timestamp(0),
                 end_ts=end_ts,
-                success_callback=populate_history_cb,
                 fail_callback=fail_history_cb,
+                new_step_data=(new_step_cb, exchange.name),
             )
             step = self._increase_progress(step, total_steps)
 
+        # Query all trades, asset movements and margin positions from the DB for all
+        # possible locations.
+        self.processing_state_name = 'Reading trades, asset movements and margin positions from the DB'  # noqa: E501
+        with self.db.conn.read_ctx() as cursor:
+            # Include all trades
+            trades = self.db.get_trades(
+                cursor,
+                filter_query=TradesFilterQuery.make(),
+                has_premium=True,  # we need all trades for accounting -- limit happens later
+            )
+            history.extend(trades)
+
+            # Include all asset movements
+            asset_movements = self.db.get_asset_movements(
+                cursor,
+                filter_query=AssetMovementsFilterQuery.make(),
+                has_premium=True,  # we need all trades for accounting -- limit happens later
+            )
+            history.extend(asset_movements)
+
+            # Include all margin positions
+            margin_positions = self.db.get_margin_positions(cursor)
+            history.extend(margin_positions)
+
+        step = self._increase_progress(step, total_steps)
+
         self.processing_state_name = 'Querying ethereum transactions history'
         tx_filter_query = ETHTransactionsFilterQuery.make(
             limit=None,
@@ -365,18 +383,6 @@ def fail_history_cb(error_msg: str) -> None:
         self.eth_tx_decoder.get_and_decode_undecoded_transactions(limit=None)
         step = self._increase_progress(step, total_steps)
 
-        # Include all external trades and trades from external exchanges
-        for location in EXTERNAL_LOCATION:
-            self.processing_state_name = f'Querying {location} trades history'
-            with self.db.conn.read_ctx() as cursor:
-                external_trades = self.db.get_trades(
-                    cursor,
-                    filter_query=TradesFilterQuery.make(location=location),
-                    has_premium=True,  # we need all trades for accounting -- limit happens later
-                )
-            history.extend(external_trades)
-            step = self._increase_progress(step, total_steps)
-
         # include all ledger actions
         self.processing_state_name = 'Querying ledger actions history'
         ledger_actions, _ = self.query_ledger_actions(
diff --git a/rotkehlchen/tasks/manager.py b/rotkehlchen/tasks/manager.py
--- a/rotkehlchen/tasks/manager.py
+++ b/rotkehlchen/tasks/manager.py
@@ -59,10 +59,6 @@
 PREMIUM_CHECK_RETRY_LIMIT = 3
 
 
-def noop_exchange_success_cb(trades, margin, asset_movements, exchange_specific_data) -> None:  # type: ignore # noqa: E501
-    pass
-
-
 def exchange_fail_cb(error: str) -> None:
     log.error(error)
 
@@ -346,7 +342,6 @@ def _maybe_schedule_exchange_history_query(self) -> Optional[gevent.Greenlet]:
             method=exchange.query_history_with_callbacks,
             start_ts=0,
             end_ts=now,
-            success_callback=noop_exchange_success_cb,
             fail_callback=exchange_fail_cb,
         )
 


Test patch:
diff --git a/rotkehlchen/tests/api/test_history.py b/rotkehlchen/tests/api/test_history.py
--- a/rotkehlchen/tests/api/test_history.py
+++ b/rotkehlchen/tests/api/test_history.py
@@ -128,7 +128,7 @@ def test_query_history(rotkehlchen_api_server_with_exchanges, start_ts, end_ts):
         ),
     )
     result = assert_proper_response_with_result(response=response, status_code=HTTPStatus.OK)
-    assert len(result['missing_acquisitions']) == 10
+    assert len(result['missing_acquisitions']) == 9
     assert len(result['missing_prices']) == 0
     assert result['report_id'] == 1
 
diff --git a/rotkehlchen/tests/unit/accounting/test_exchanges.py b/rotkehlchen/tests/unit/accounting/test_exchanges.py
--- a/rotkehlchen/tests/unit/accounting/test_exchanges.py
+++ b/rotkehlchen/tests/unit/accounting/test_exchanges.py
@@ -3,11 +3,17 @@
 
 import pytest
 
+from rotkehlchen.accounting.ledger_actions import LedgerAction, LedgerActionType
 from rotkehlchen.accounting.mixins.event import AccountingEventType
+from rotkehlchen.constants.assets import A_BTC, A_ETH, A_LINK, A_USDT
+from rotkehlchen.constants.misc import ONE, ZERO
+from rotkehlchen.db.ledger_actions import DBLedgerActions
+from rotkehlchen.exchanges.data_structures import AssetMovement, MarginPosition, Trade
+from rotkehlchen.fval import FVal
 from rotkehlchen.tests.utils.accounting import accounting_create_and_process_history
 from rotkehlchen.tests.utils.exchanges import mock_normal_coinbase_query
 from rotkehlchen.tests.utils.history import prices
-from rotkehlchen.types import Location
+from rotkehlchen.types import AssetMovementCategory, Location, Timestamp, TradeType
 
 
 @pytest.mark.parametrize(
@@ -38,3 +44,119 @@ def test_account_for_coinbase_ledger_actions(rotkehlchen_api_server_with_exchang
     assert events_map[AccountingEventType.FEE] == 2
     assert events_map[AccountingEventType.ASSET_MOVEMENT] == 3
     assert sum(events_map.values()) == 11  # processed events
+
+
+@pytest.mark.parametrize('default_mock_price_value', [FVal(1.5)])
+@pytest.mark.parametrize('ethereum_accounts', [[]])
+@pytest.mark.parametrize('added_exchanges', [(Location.COINBASE,)])
+def test_exchanges_removed_api_keys(rotkehlchen_api_server_with_exchanges):
+    """
+    Test that if actions made on an exchange are stored in the DB but the API keys were removed
+    then the actions are still taken into account in accounting.
+    """
+    rotki = rotkehlchen_api_server_with_exchanges.rest_api.rotkehlchen
+    with rotki.data.db.user_write() as write_cursor:
+        rotki.data.db.add_trades(write_cursor, trades=[Trade(
+            timestamp=Timestamp(1611426201),
+            amount=ONE,
+            base_asset=A_ETH,
+            quote_asset=A_BTC,
+            trade_type=TradeType.BUY,
+            rate=FVal(1.5),
+            location=Location.COINBASE,
+        ), Trade(
+            timestamp=Timestamp(1611426201),
+            amount=FVal(7),
+            base_asset=A_USDT,
+            quote_asset=A_LINK,
+            trade_type=TradeType.SELL,
+            rate=FVal(7),
+            location=Location.EXTERNAL,
+        )])
+        rotki.data.db.add_asset_movements(write_cursor, asset_movements=[AssetMovement(
+            timestamp=Timestamp(1611426201),
+            location=Location.COINBASE,
+            category=AssetMovementCategory.DEPOSIT,
+            asset=A_BTC,
+            amount=ONE,
+            fee_asset=A_BTC,
+            fee=FVal(0.00001),
+            address=None,
+            transaction_id=None,
+            link='no link',
+        )])
+        rotki.data.db.add_margin_positions(write_cursor, margin_positions=[MarginPosition(
+            location=Location.COINBASE,
+            open_time=Timestamp(1611426200),
+            close_time=Timestamp(1611426201),
+            profit_loss=ONE,
+            pl_currency=A_BTC,
+            fee=ZERO,
+            fee_currency=A_BTC,
+            link='no link',
+            notes='no notes',
+        )])
+        DBLedgerActions(rotki.data.db, rotki.data.db.msg_aggregator).add_ledger_actions(
+            write_cursor,
+            actions=[LedgerAction(
+                identifier=0,
+                timestamp=Timestamp(1611426201),
+                action_type=LedgerActionType.INCOME,
+                location=Location.COINBASE,
+                amount=ONE,
+                asset=A_BTC,
+                rate=None,
+                rate_asset=None,
+                link='no link',
+                notes='no notes',
+            )],
+        )
+
+    rotki.remove_exchange(name='coinbase', location=Location.COINBASE)
+    _, events = accounting_create_and_process_history(rotki=rotki, start_ts=0, end_ts=1611426233)
+    assert len(events) == 7
+    event1 = events[0]
+    assert event1.type == AccountingEventType.TRADE
+    assert event1.location == Location.COINBASE
+    assert event1.taxable_amount == FVal(1.5)
+    assert event1.asset == A_BTC
+
+    event2 = events[1]
+    assert event2.type == AccountingEventType.TRADE
+    assert event2.location == Location.COINBASE
+    assert event2.free_amount == ONE
+    assert event2.asset == A_ETH
+
+    event3 = events[2]
+    assert event3.type == AccountingEventType.TRADE
+    assert event3.location == Location.EXTERNAL
+    assert event3.taxable_amount == FVal(7)
+    assert event3.asset == A_USDT
+
+    event4 = events[3]
+    assert event4.type == AccountingEventType.TRADE
+    assert event4.location == Location.EXTERNAL
+    assert event4.free_amount == FVal(49)
+    assert event4.asset == A_LINK
+
+    event5 = events[4]
+    assert event5.type == AccountingEventType.ASSET_MOVEMENT
+    assert event5.location == Location.COINBASE
+    assert event5.free_amount == ZERO
+    assert event5.taxable_amount == FVal(0.00001)
+    assert event5.asset == A_BTC
+    ...
+
+    event6 = events[5]
+    assert event6.type == AccountingEventType.MARGIN_POSITION
+    assert event6.location == Location.COINBASE
+    assert event6.free_amount == ZERO
+    assert event6.taxable_amount == ONE
+    assert event6.asset == A_BTC
+
+    event7 = events[6]
+    assert event7.type == AccountingEventType.LEDGER_ACTION
+    assert event7.location == Location.COINBASE
+    assert event7.free_amount == ZERO
+    assert event7.taxable_amount == ONE
+    assert event7.asset == A_BTC