Repository: rotki/rotki

Issue:
Trades from externally imported exchanges are not taken into account in historical accounting
## Problem Definition

It seems that trades from externally imported exchanges are not taken into account in historical accounting. Credit for finding it out @arosboro 

## Task

Fix it



Discussion:

Resolving patch:
diff --git a/rotkehlchen/api/rest.py b/rotkehlchen/api/rest.py
--- a/rotkehlchen/api/rest.py
+++ b/rotkehlchen/api/rest.py
@@ -77,13 +77,17 @@
 from rotkehlchen.exchanges.manager import ALL_SUPPORTED_EXCHANGES
 from rotkehlchen.fval import FVal
 from rotkehlchen.globaldb import GlobalDBHandler
-from rotkehlchen.history.events import FREE_LEDGER_ACTIONS_LIMIT
+from rotkehlchen.history.events import (
+    FREE_ASSET_MOVEMENTS_LIMIT,
+    FREE_LEDGER_ACTIONS_LIMIT,
+    FREE_TRADES_LIMIT,
+)
 from rotkehlchen.history.price import PriceHistorian
 from rotkehlchen.history.typing import NOT_EXPOSED_SOURCES, HistoricalPrice, HistoricalPriceOracle
 from rotkehlchen.inquirer import CurrentPriceOracle, Inquirer
 from rotkehlchen.logging import RotkehlchenLogsAdapter
 from rotkehlchen.premium.premium import PremiumCredentials
-from rotkehlchen.rotkehlchen import FREE_ASSET_MOVEMENTS_LIMIT, FREE_TRADES_LIMIT, Rotkehlchen
+from rotkehlchen.rotkehlchen import Rotkehlchen
 from rotkehlchen.serialization.serialize import process_result, process_result_list
 from rotkehlchen.typing import (
     AVAILABLE_MODULES_MAP,
@@ -716,7 +720,7 @@ def _get_trades(
             only_cache: bool,
     ) -> Dict[str, Any]:
         try:
-            trades = self.rotkehlchen.query_trades(
+            trades = self.rotkehlchen.events_historian.query_trades(
                 from_ts=from_ts,
                 to_ts=to_ts,
                 location=location,
@@ -877,7 +881,7 @@ def _get_asset_movements(
         status_code = HTTPStatus.OK
         result = None
         try:
-            movements = self.rotkehlchen.query_asset_movements(
+            movements = self.rotkehlchen.events_historian.query_asset_movements(
                 from_ts=from_timestamp,
                 to_ts=to_timestamp,
                 location=location,
diff --git a/rotkehlchen/history/events.py b/rotkehlchen/history/events.py
--- a/rotkehlchen/history/events.py
+++ b/rotkehlchen/history/events.py
@@ -1,17 +1,27 @@
 import logging
+from collections import defaultdict
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Union, cast
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast, overload
+
+from typing_extensions import Literal
 
 from rotkehlchen.chain.ethereum.trades import AMMTRADE_LOCATION_NAMES, AMMTrade, AMMTradeLocations
 from rotkehlchen.constants.misc import ZERO
 from rotkehlchen.db.ledger_actions import DBLedgerActions
 from rotkehlchen.errors import RemoteError
 from rotkehlchen.exchanges.data_structures import AssetMovement, Loan, MarginPosition, Trade
-from rotkehlchen.exchanges.manager import ExchangeManager
+from rotkehlchen.exchanges.exchange import ExchangeInterface
+from rotkehlchen.exchanges.manager import ALL_SUPPORTED_EXCHANGES, ExchangeManager
 from rotkehlchen.exchanges.poloniex import process_polo_loans
 from rotkehlchen.fval import FVal
 from rotkehlchen.logging import RotkehlchenLogsAdapter
-from rotkehlchen.typing import EthereumTransaction, Location, Timestamp
+from rotkehlchen.typing import (
+    EXTERNAL_EXCHANGES,
+    EXTERNAL_LOCATION,
+    EthereumTransaction,
+    Location,
+    Timestamp,
+)
 from rotkehlchen.user_messages import MessagesAggregator
 from rotkehlchen.utils.accounting import action_get_timestamp
 from rotkehlchen.utils.misc import timestamp_to_date
@@ -29,7 +39,8 @@
 # Number of steps excluding the connected exchanges. Current query steps:
 # eth transactions
 # ledger actions
-# external trades: balancer, uniswap
+# external location trades -> len(EXTERNAL_LOCATION)
+# amm trades len(AMMTradeLocations)
 # makerdao dsr
 # makerdao vaults
 # yearn vaults
@@ -38,7 +49,7 @@
 # aave lending
 # eth2
 # Please, update this number each time a history query step is either added or removed
-NUM_HISTORY_QUERY_STEPS_EXCL_EXCHANGES = 11
+NUM_HISTORY_QUERY_STEPS_EXCL_EXCHANGES = 9 + len(EXTERNAL_LOCATION) + len(AMMTradeLocations)
 FREE_LEDGER_ACTIONS_LIMIT = 50
 
 HistoryResult = Tuple[
@@ -50,6 +61,16 @@
     List['DefiEvent'],
     List['LedgerAction'],
 ]
+TRADES_LIST = List[Union[Trade, AMMTrade]]
+
+FREE_TRADES_LIMIT = 250
+FREE_ASSET_MOVEMENTS_LIMIT = 100
+
+LIMITS_MAPPING = {
+    'trade': FREE_TRADES_LIMIT,
+    'asset_movement': FREE_ASSET_MOVEMENTS_LIMIT,
+    'ledger_action': FREE_LEDGER_ACTIONS_LIMIT,
+}
 
 
 def limit_trade_list_to_period(
@@ -103,6 +124,11 @@ def timestamp_to_date(self, timestamp: Timestamp) -> str:
         )
 
     def _reset_variables(self) -> None:
+        # Keeps how many trades we have found per location. Used for free user limiting
+        self.actions_per_location: Dict[str, Dict[Location, int]] = {
+            'trade': defaultdict(int),
+            'asset_movement': defaultdict(int),
+        }
         self.processing_state_name = 'Starting query of historical events'
         self.progress = ZERO
         db_settings = self.db.get_settings()
@@ -114,6 +140,56 @@ def _increase_progress(self, step: int, total_steps: int) -> int:
         self.progress = FVal(step / total_steps) * 100
         return step
 
+    @overload
+    def _apply_actions_limit(
+            self,
+            location: Location,
+            action_type: Literal['trade'],
+            location_actions: TRADES_LIST,
+            all_actions: TRADES_LIST,
+    ) -> TRADES_LIST:
+        ...
+
+    @overload
+    def _apply_actions_limit(
+            self,
+            location: Location,
+            action_type: Literal['asset_movement'],
+            location_actions: List[AssetMovement],
+            all_actions: List[AssetMovement],
+    ) -> List[AssetMovement]:
+        ...
+
+    def _apply_actions_limit(
+            self,
+            location: Location,
+            action_type: Literal['trade', 'asset_movement'],
+            location_actions: Union[TRADES_LIST, List[AssetMovement]],
+            all_actions: Union[TRADES_LIST, List[AssetMovement]],
+    ) -> Union[TRADES_LIST, List[AssetMovement]]:
+        """Take as many actions from location actions and add them to all actions as the limit permits
+
+        Returns the modified (or not) all_actions
+        """
+        # If we are already at or above the limit return current actions disregarding this location
+        actions_mapping = self.actions_per_location[action_type]
+        current_num_actions = sum(x for _, x in actions_mapping.items())
+        limit = LIMITS_MAPPING[action_type]
+        if current_num_actions >= limit:
+            return all_actions
+
+        # Find out how many more actions can we return, and depending on that get
+        # the number of actions from the location actions and add them to the total
+        remaining_num_actions = limit - current_num_actions
+        if remaining_num_actions < 0:
+            remaining_num_actions = 0
+
+        num_actions_to_take = min(len(location_actions), remaining_num_actions)
+
+        actions_mapping[location] = num_actions_to_take
+        all_actions.extend(location_actions[0:num_actions_to_take])  # type: ignore
+        return all_actions
+
     def query_ledger_actions(
             self,
             has_premium: bool,
@@ -136,6 +212,265 @@ def query_ledger_actions(
 
         return actions, original_length
 
+    def query_trades(
+            self,
+            from_ts: Timestamp,
+            to_ts: Timestamp,
+            location: Optional[Location],
+            only_cache: bool,
+    ) -> TRADES_LIST:
+        """Queries trades for the given location and time range.
+        If no location is given then all external, all exchange and DEX trades are queried.
+
+        If only_cache is given then only trades cached in the DB are returned.
+        No service is queried.
+
+        DEX Trades are queried only if the user has premium
+        If the user does not have premium then a trade limit is applied.
+
+        May raise:
+        - RemoteError: If there are problems connecting to any of the remote exchanges
+        """
+        trades: TRADES_LIST
+        if location is not None:
+            # clear the trades queried for this location
+            self.actions_per_location['trade'][location] = 0
+            trades = self.query_location_trades(from_ts, to_ts, location, only_cache)
+        else:
+            for given_location in ALL_SUPPORTED_EXCHANGES + [Location.EXTERNAL]:
+                # clear the trades queried for this location
+                self.actions_per_location['trade'][given_location] = 0
+            trades = self.query_location_trades(from_ts, to_ts, Location.EXTERNAL, only_cache)
+            # Look for trades that might be imported from CSV files
+            for csv_location in EXTERNAL_EXCHANGES:
+                trades.extend(self.query_location_trades(
+                    from_ts=from_ts,
+                    to_ts=to_ts,
+                    location=csv_location,
+                    only_cache=only_cache,
+                ))
+
+            for exchange in self.exchange_manager.iterate_exchanges():
+                all_set = {x.identifier for x in trades}
+                exchange_trades = exchange.query_trade_history(
+                    start_ts=from_ts,
+                    end_ts=to_ts,
+                    only_cache=only_cache,
+                )
+                # TODO: Really dirty. Figure out a better way.
+                # Since some of the trades may already be in the DB if multiple
+                # keys are used for a single exchange.
+                exchange_trades = [x for x in exchange_trades if x.identifier not in all_set]
+                if self.chain_manager.premium is None:
+                    trades = self._apply_actions_limit(
+                        location=exchange.location,
+                        action_type='trade',
+                        location_actions=exchange_trades,
+                        all_actions=trades,
+                    )
+                else:
+                    trades.extend(exchange_trades)
+
+            # for all trades we also need the trades from the amm protocols
+            if self.chain_manager.premium is not None:
+                for amm_location in AMMTradeLocations:
+                    amm_module_name = cast(AMMTRADE_LOCATION_NAMES, str(amm_location))
+                    amm_module = self.chain_manager.get_module(amm_module_name)
+                    if amm_module is not None:
+                        trades.extend(
+                            amm_module.get_trades(
+                                addresses=self.chain_manager.queried_addresses_for_module(amm_module_name),  # noqa: E501
+                                from_timestamp=from_ts,
+                                to_timestamp=to_ts,
+                                only_cache=only_cache,
+                            ),
+                        )
+        # return trades with most recent first
+        trades.sort(key=lambda x: x.timestamp, reverse=True)
+        return trades
+
+    def query_location_trades(
+            self,
+            from_ts: Timestamp,
+            to_ts: Timestamp,
+            location: Location,
+            only_cache: bool,
+    ) -> TRADES_LIST:
+        location_trades: TRADES_LIST
+        if location in EXTERNAL_LOCATION:
+            location_trades = self.db.get_trades(  # type: ignore  # list invariance
+                from_ts=from_ts,
+                to_ts=to_ts,
+                location=location,
+            )
+        elif location in AMMTradeLocations:
+            if self.chain_manager.premium is not None:
+                amm_module_name = cast(AMMTRADE_LOCATION_NAMES, str(location))
+                amm_module = self.chain_manager.get_module(amm_module_name)
+                if amm_module is not None:
+                    location_trades = amm_module.get_trades(  # type: ignore  # list invariance
+                        addresses=self.chain_manager.queried_addresses_for_module(amm_module_name),
+                        from_timestamp=from_ts,
+                        to_timestamp=to_ts,
+                        only_cache=only_cache,
+                    )
+        else:
+            # should only be an exchange
+            exchanges_list = self.exchange_manager.connected_exchanges.get(location)
+            if exchanges_list is None:
+                log.warning(
+                    f'Tried to query trades from {str(location)} which is either not an '
+                    f'exchange or not an exchange the user has connected to',
+                )
+                return []
+
+            location_trades = []
+            for exchange in exchanges_list:
+                all_set = {x.identifier for x in location_trades}
+                new_trades = exchange.query_trade_history(
+                    start_ts=from_ts,
+                    end_ts=to_ts,
+                    only_cache=only_cache,
+                )
+                # TODO: Really dirty. Figure out a better way.
+                # Since some of the trades may already be in the DB if multiple
+                # keys are used for a single exchange.
+                new_trades = [x for x in new_trades if x.identifier not in all_set]
+                location_trades.extend(new_trades)
+
+        trades: TRADES_LIST = []
+        if self.chain_manager.premium is None:
+            trades = self._apply_actions_limit(
+                location=location,
+                action_type='trade',
+                location_actions=location_trades,
+                all_actions=trades,
+            )
+        else:
+            trades = location_trades
+
+        return trades
+
+    def _query_and_populate_exchange_asset_movements(
+            self,
+            from_ts: Timestamp,
+            to_ts: Timestamp,
+            all_movements: List[AssetMovement],
+            exchange: Union[ExchangeInterface, Location],
+            only_cache: bool,
+    ) -> List[AssetMovement]:
+        """Queries exchange for asset movements and adds it to all_movements"""
+        all_set = {x.identifier for x in all_movements}
+        if isinstance(exchange, ExchangeInterface):
+            location = exchange.location
+            location_movements = exchange.query_deposits_withdrawals(
+                start_ts=from_ts,
+                end_ts=to_ts,
+                only_cache=only_cache,
+            )
+            # TODO: Really dirty. Figure out a better way.
+            # Since some of the asset movements may already be in the DB if multiple
+            # keys are used for a single exchange.
+            location_movements = [x for x in location_movements if x.identifier not in all_set]
+        else:
+            assert isinstance(exchange, Location), 'only a location should make it here'
+            assert exchange in EXTERNAL_EXCHANGES, 'only csv supported exchanges should get here'  # noqa : E501
+            location = exchange
+            # We might have no exchange information but CSV imported information
+            self.actions_per_location['asset_movement'][location] = 0
+            location_movements = self.db.get_asset_movements(
+                from_ts=from_ts,
+                to_ts=to_ts,
+                location=location,
+            )
+
+        movements: List[AssetMovement] = []
+        if self.chain_manager.premium is None:
+            movements = self._apply_actions_limit(
+                location=location,
+                action_type='asset_movement',
+                location_actions=location_movements,
+                all_actions=all_movements,
+            )
+        else:
+            all_movements.extend(location_movements)
+            movements = all_movements
+
+        return movements
+
+    def query_asset_movements(
+            self,
+            from_ts: Timestamp,
+            to_ts: Timestamp,
+            location: Optional[Location],
+            only_cache: bool,
+    ) -> List[AssetMovement]:
+        """Queries AssetMovements for the given location and time range.
+
+        If no location is given then all exchange asset movements are queried.
+        If only_cache is True then only what is already in the DB is returned.
+        If the user does not have premium then a limit is applied.
+        May raise:
+        - RemoteError: If there are problems connecting to any of the remote exchanges
+        """
+        movements: List[AssetMovement] = []
+        if location is not None:
+            # clear the asset movements queried for this exchange
+            self.actions_per_location['asset_movement'][location] = 0
+            if location in EXTERNAL_EXCHANGES:
+                movements = self._query_and_populate_exchange_asset_movements(
+                    from_ts=from_ts,
+                    to_ts=to_ts,
+                    all_movements=movements,
+                    exchange=location,
+                    only_cache=only_cache,
+                )
+            else:
+                exchanges_list = self.exchange_manager.connected_exchanges.get(location)
+                if exchanges_list is None:
+                    log.warning(
+                        f'Tried to query deposits/withdrawals from {str(location)} which is '
+                        f'either not an exchange or not an exchange the user has connected to',
+                    )
+                    return []
+
+                # clear the asset movements queried for this exchange
+                self.actions_per_location['asset_movement'][location] = 0
+                for exchange in exchanges_list:
+                    self._query_and_populate_exchange_asset_movements(
+                        from_ts=from_ts,
+                        to_ts=to_ts,
+                        all_movements=movements,
+                        exchange=exchange,
+                        only_cache=only_cache,
+                    )
+        else:
+            for exchange_location in ALL_SUPPORTED_EXCHANGES:
+                # clear the asset movements queried for this exchange
+                self.actions_per_location['asset_movement'][exchange_location] = 0
+            # we may have DB entries due to csv import from supported locations
+            for external_location in EXTERNAL_EXCHANGES:
+
+                movements = self._query_and_populate_exchange_asset_movements(
+                    from_ts=from_ts,
+                    to_ts=to_ts,
+                    all_movements=movements,
+                    exchange=external_location,
+                    only_cache=only_cache,
+                )
+            for exchange in self.exchange_manager.iterate_exchanges():
+                self._query_and_populate_exchange_asset_movements(
+                    from_ts=from_ts,
+                    to_ts=to_ts,
+                    all_movements=movements,
+                    exchange=exchange,
+                    only_cache=only_cache,
+                )
+
+        # return movements with most recent first
+        movements.sort(key=lambda x: x.timestamp, reverse=True)
+        return movements
+
     def get_history(
             self,
             start_ts: Timestamp,
@@ -217,16 +552,18 @@ def fail_history_cb(error_msg: str) -> None:
             empty_or_error += '\n' + msg
         step = self._increase_progress(step, total_steps)
 
-        # Include the external trades in the history
-        self.processing_state_name = 'Querying external trades history'
-        external_trades = self.db.get_trades(
-            # We need to have history of trades since before the range
-            from_ts=Timestamp(0),
-            to_ts=end_ts,
-            location=Location.EXTERNAL,
-        )
-        history.extend(external_trades)
-        step = self._increase_progress(step, total_steps)
+        # Include all external trades and trades from external exchanges
+        for location in EXTERNAL_LOCATION:
+            self.processing_state_name = f'Querying {location} trades history'
+            external_trades = self.query_location_trades(
+                # We need to have history of trades since before the range
+                from_ts=Timestamp(0),
+                to_ts=end_ts,
+                location=location,
+                only_cache=True,
+            )
+            history.extend(external_trades)
+            step = self._increase_progress(step, total_steps)
 
         # include the ledger actions
         self.processing_state_name = 'Querying ledger actions history'
@@ -311,7 +648,7 @@ def fail_history_cb(error_msg: str) -> None:
                 to_timestamp=end_ts,
                 addresses=self.chain_manager.queried_addresses_for_module('aave'),
             ))
-        self._increase_progress(step, total_steps)
+        step = self._increase_progress(step, total_steps)
 
         # include eth2 staking events
         eth2 = self.chain_manager.get_module('eth2')
diff --git a/rotkehlchen/rotkehlchen.py b/rotkehlchen/rotkehlchen.py
--- a/rotkehlchen/rotkehlchen.py
+++ b/rotkehlchen/rotkehlchen.py
@@ -6,18 +6,7 @@
 import time
 from collections import defaultdict
 from pathlib import Path
-from typing import (
-    TYPE_CHECKING,
-    Any,
-    DefaultDict,
-    Dict,
-    List,
-    Optional,
-    Tuple,
-    Union,
-    cast,
-    overload,
-)
+from typing import TYPE_CHECKING, Any, DefaultDict, Dict, List, Optional, Tuple, Union
 
 import gevent
 from typing_extensions import Literal
@@ -33,7 +22,6 @@
     EthereumManager,
     NodeName,
 )
-from rotkehlchen.chain.ethereum.trades import AMMTRADE_LOCATION_NAMES, AMMTrade, AMMTradeLocations
 from rotkehlchen.chain.manager import BlockchainBalancesUpdate, ChainManager
 from rotkehlchen.chain.substrate.manager import SubstrateManager
 from rotkehlchen.chain.substrate.typing import SubstrateChain
@@ -53,9 +41,7 @@
     RemoteError,
     SystemPermissionError,
 )
-from rotkehlchen.exchanges.data_structures import AssetMovement, Trade
-from rotkehlchen.exchanges.exchange import ExchangeInterface
-from rotkehlchen.exchanges.manager import ALL_SUPPORTED_EXCHANGES, ExchangeManager
+from rotkehlchen.exchanges.manager import ExchangeManager
 from rotkehlchen.externalapis.beaconchain import BeaconChain
 from rotkehlchen.externalapis.coingecko import Coingecko
 from rotkehlchen.externalapis.covalent import Covalent, chains_id
@@ -65,7 +51,7 @@
 from rotkehlchen.globaldb import GlobalDBHandler
 from rotkehlchen.globaldb.updates import AssetsUpdater
 from rotkehlchen.greenlets import GreenletManager
-from rotkehlchen.history.events import FREE_LEDGER_ACTIONS_LIMIT, EventsHistorian
+from rotkehlchen.history.events import EventsHistorian
 from rotkehlchen.history.price import PriceHistorian
 from rotkehlchen.history.typing import HistoricalPriceOracle
 from rotkehlchen.icons import IconManager
@@ -75,8 +61,6 @@
 from rotkehlchen.premium.sync import PremiumSyncManager
 from rotkehlchen.tasks.manager import DEFAULT_MAX_TASKS_NUM, TaskManager
 from rotkehlchen.typing import (
-    EXTERNAL_EXCHANGES,
-    EXTERNAL_LOCATION,
     ApiKey,
     ApiSecret,
     BlockchainAccountData,
@@ -97,22 +81,12 @@
 log = RotkehlchenLogsAdapter(logger)
 
 MAIN_LOOP_SECS_DELAY = 10
-FREE_TRADES_LIMIT = 250
-FREE_ASSET_MOVEMENTS_LIMIT = 100
 
-LIMITS_MAPPING = {
-    'trade': FREE_TRADES_LIMIT,
-    'asset_movement': FREE_ASSET_MOVEMENTS_LIMIT,
-    'ledger_action': FREE_LEDGER_ACTIONS_LIMIT,
-}
 
 ICONS_BATCH_SIZE = 3
 ICONS_QUERY_SLEEP = 60
 
 
-TRADES_LIST = List[Union[Trade, AMMTrade]]
-
-
 class Rotkehlchen():
     def __init__(self, args: argparse.Namespace) -> None:
         """Initialize the Rotkehlchen object
@@ -161,11 +135,6 @@ def __init__(self, args: argparse.Namespace) -> None:
             cryptocompare=self.cryptocompare,
             coingecko=self.coingecko,
         )
-        # Keeps how many trades we have found per location. Used for free user limiting
-        self.actions_per_location: Dict[str, Dict[Location, int]] = {
-            'trade': defaultdict(int),
-            'asset_movement': defaultdict(int),
-        }
         self.task_manager: Optional[TaskManager] = None
         self.shutdown_event = gevent.event.Event()
 
@@ -605,195 +574,6 @@ def process_history(
         )
         return result, error_or_empty
 
-    @overload
-    def _apply_actions_limit(
-            self,
-            location: Location,
-            action_type: Literal['trade'],
-            location_actions: TRADES_LIST,
-            all_actions: TRADES_LIST,
-    ) -> TRADES_LIST:
-        ...
-
-    @overload
-    def _apply_actions_limit(
-            self,
-            location: Location,
-            action_type: Literal['asset_movement'],
-            location_actions: List[AssetMovement],
-            all_actions: List[AssetMovement],
-    ) -> List[AssetMovement]:
-        ...
-
-    def _apply_actions_limit(
-            self,
-            location: Location,
-            action_type: Literal['trade', 'asset_movement'],
-            location_actions: Union[TRADES_LIST, List[AssetMovement]],
-            all_actions: Union[TRADES_LIST, List[AssetMovement]],
-    ) -> Union[TRADES_LIST, List[AssetMovement]]:
-        """Take as many actions from location actions and add them to all actions as the limit permits
-
-        Returns the modified (or not) all_actions
-        """
-        # If we are already at or above the limit return current actions disregarding this location
-        actions_mapping = self.actions_per_location[action_type]
-        current_num_actions = sum(x for _, x in actions_mapping.items())
-        limit = LIMITS_MAPPING[action_type]
-        if current_num_actions >= limit:
-            return all_actions
-
-        # Find out how many more actions can we return, and depending on that get
-        # the number of actions from the location actions and add them to the total
-        remaining_num_actions = limit - current_num_actions
-        if remaining_num_actions < 0:
-            remaining_num_actions = 0
-
-        num_actions_to_take = min(len(location_actions), remaining_num_actions)
-
-        actions_mapping[location] = num_actions_to_take
-        all_actions.extend(location_actions[0:num_actions_to_take])  # type: ignore
-        return all_actions
-
-    def query_trades(
-            self,
-            from_ts: Timestamp,
-            to_ts: Timestamp,
-            location: Optional[Location],
-            only_cache: bool,
-    ) -> TRADES_LIST:
-        """Queries trades for the given location and time range.
-        If no location is given then all external, all exchange and DEX trades are queried.
-
-        If only_cache is given then only trades cached in the DB are returned.
-        No service is queried.
-
-        DEX Trades are queried only if the user has premium
-        If the user does not have premium then a trade limit is applied.
-
-        May raise:
-        - RemoteError: If there are problems connecting to any of the remote exchanges
-        """
-        trades: TRADES_LIST
-        if location is not None:
-            # clear the trades queried for this location
-            self.actions_per_location['trade'][location] = 0
-            trades = self.query_location_trades(from_ts, to_ts, location, only_cache)
-        else:
-            for given_location in ALL_SUPPORTED_EXCHANGES + [Location.EXTERNAL]:
-                # clear the trades queried for this location
-                self.actions_per_location['trade'][given_location] = 0
-            trades = self.query_location_trades(from_ts, to_ts, Location.EXTERNAL, only_cache)
-            # Look for trades that might be imported from CSV files
-            for csv_location in EXTERNAL_EXCHANGES:
-                trades.extend(self.query_location_trades(
-                    from_ts=from_ts,
-                    to_ts=to_ts,
-                    location=csv_location,
-                    only_cache=only_cache,
-                ))
-
-            for exchange in self.exchange_manager.iterate_exchanges():
-                all_set = {x.identifier for x in trades}
-                exchange_trades = exchange.query_trade_history(
-                    start_ts=from_ts,
-                    end_ts=to_ts,
-                    only_cache=only_cache,
-                )
-                # TODO: Really dirty. Figure out a better way.
-                # Since some of the trades may already be in the DB if multiple
-                # keys are used for a single exchange.
-                exchange_trades = [x for x in exchange_trades if x.identifier not in all_set]
-                if self.premium is None:
-                    trades = self._apply_actions_limit(
-                        location=exchange.location,
-                        action_type='trade',
-                        location_actions=exchange_trades,
-                        all_actions=trades,
-                    )
-                else:
-                    trades.extend(exchange_trades)
-
-            # for all trades we also need the trades from the amm protocols
-            if self.premium is not None:
-                for amm_location in AMMTradeLocations:
-                    amm_module_name = cast(AMMTRADE_LOCATION_NAMES, str(amm_location))
-                    amm_module = self.chain_manager.get_module(amm_module_name)
-                    if amm_module is not None:
-                        trades.extend(
-                            amm_module.get_trades(
-                                addresses=self.chain_manager.queried_addresses_for_module(amm_module_name),  # noqa: E501
-                                from_timestamp=from_ts,
-                                to_timestamp=to_ts,
-                                only_cache=only_cache,
-                            ),
-                        )
-        # return trades with most recent first
-        trades.sort(key=lambda x: x.timestamp, reverse=True)
-        return trades
-
-    def query_location_trades(
-            self,
-            from_ts: Timestamp,
-            to_ts: Timestamp,
-            location: Location,
-            only_cache: bool,
-    ) -> TRADES_LIST:
-        location_trades: TRADES_LIST
-        if location in EXTERNAL_LOCATION:
-            location_trades = self.data.db.get_trades(  # type: ignore  # list invariance
-                from_ts=from_ts,
-                to_ts=to_ts,
-                location=location,
-            )
-        elif location in AMMTradeLocations:
-            if self.premium is not None:
-                amm_module_name = cast(AMMTRADE_LOCATION_NAMES, str(location))
-                amm_module = self.chain_manager.get_module(amm_module_name)
-                if amm_module is not None:
-                    location_trades = amm_module.get_trades(  # type: ignore  # list invariance
-                        addresses=self.chain_manager.queried_addresses_for_module(amm_module_name),
-                        from_timestamp=from_ts,
-                        to_timestamp=to_ts,
-                        only_cache=only_cache,
-                    )
-        else:
-            # should only be an exchange
-            exchanges_list = self.exchange_manager.connected_exchanges.get(location)
-            if exchanges_list is None:
-                log.warning(
-                    f'Tried to query trades from {str(location)} which is either not an '
-                    f'exchange or not an exchange the user has connected to',
-                )
-                return []
-
-            location_trades = []
-            for exchange in exchanges_list:
-                all_set = {x.identifier for x in location_trades}
-                new_trades = exchange.query_trade_history(
-                    start_ts=from_ts,
-                    end_ts=to_ts,
-                    only_cache=only_cache,
-                )
-                # TODO: Really dirty. Figure out a better way.
-                # Since some of the trades may already be in the DB if multiple
-                # keys are used for a single exchange.
-                new_trades = [x for x in new_trades if x.identifier not in all_set]
-                location_trades.extend(new_trades)
-
-        trades: TRADES_LIST = []
-        if self.premium is None:
-            trades = self._apply_actions_limit(
-                location=location,
-                action_type='trade',
-                location_actions=location_trades,
-                all_actions=trades,
-            )
-        else:
-            trades = location_trades
-
-        return trades
-
     def query_balances(
             self,
             requested_save_data: bool = False,
@@ -917,126 +697,6 @@ def query_balances(
 
         return result_dict
 
-    def _query_and_populate_exchange_asset_movements(
-            self,
-            from_ts: Timestamp,
-            to_ts: Timestamp,
-            all_movements: List[AssetMovement],
-            exchange: Union[ExchangeInterface, Location],
-            only_cache: bool,
-    ) -> List[AssetMovement]:
-        """Queries exchange for asset movements and adds it to all_movements"""
-        all_set = {x.identifier for x in all_movements}
-        if isinstance(exchange, ExchangeInterface):
-            location = exchange.location
-            location_movements = exchange.query_deposits_withdrawals(
-                start_ts=from_ts,
-                end_ts=to_ts,
-                only_cache=only_cache,
-            )
-            # TODO: Really dirty. Figure out a better way.
-            # Since some of the asset movements may already be in the DB if multiple
-            # keys are used for a single exchange.
-            location_movements = [x for x in location_movements if x.identifier not in all_set]
-        else:
-            assert isinstance(exchange, Location), 'only a location should make it here'
-            assert exchange in EXTERNAL_EXCHANGES, 'only csv supported exchanges should get here'  # noqa : E501
-            location = exchange
-            # We might have no exchange information but CSV imported information
-            self.actions_per_location['asset_movement'][location] = 0
-            location_movements = self.data.db.get_asset_movements(
-                from_ts=from_ts,
-                to_ts=to_ts,
-                location=location,
-            )
-
-        movements: List[AssetMovement] = []
-        if self.premium is None:
-            movements = self._apply_actions_limit(
-                location=location,
-                action_type='asset_movement',
-                location_actions=location_movements,
-                all_actions=all_movements,
-            )
-        else:
-            all_movements.extend(location_movements)
-            movements = all_movements
-
-        return movements
-
-    def query_asset_movements(
-            self,
-            from_ts: Timestamp,
-            to_ts: Timestamp,
-            location: Optional[Location],
-            only_cache: bool,
-    ) -> List[AssetMovement]:
-        """Queries AssetMovements for the given location and time range.
-
-        If no location is given then all exchange asset movements are queried.
-        If only_cache is True then only what is already in the DB is returned.
-        If the user does not have premium then a limit is applied.
-        May raise:
-        - RemoteError: If there are problems connecting to any of the remote exchanges
-        """
-        movements: List[AssetMovement] = []
-        if location is not None:
-            # clear the asset movements queried for this exchange
-            self.actions_per_location['asset_movement'][location] = 0
-            if location in EXTERNAL_EXCHANGES:
-                movements = self._query_and_populate_exchange_asset_movements(
-                    from_ts=from_ts,
-                    to_ts=to_ts,
-                    all_movements=movements,
-                    exchange=location,
-                    only_cache=only_cache,
-                )
-            else:
-                exchanges_list = self.exchange_manager.connected_exchanges.get(location)
-                if exchanges_list is None:
-                    log.warning(
-                        f'Tried to query deposits/withdrawals from {str(location)} which is '
-                        f'either not an exchange or not an exchange the user has connected to',
-                    )
-                    return []
-
-                # clear the asset movements queried for this exchange
-                self.actions_per_location['asset_movement'][location] = 0
-                for exchange in exchanges_list:
-                    self._query_and_populate_exchange_asset_movements(
-                        from_ts=from_ts,
-                        to_ts=to_ts,
-                        all_movements=movements,
-                        exchange=exchange,
-                        only_cache=only_cache,
-                    )
-        else:
-            for exchange_location in ALL_SUPPORTED_EXCHANGES:
-                # clear the asset movements queried for this exchange
-                self.actions_per_location['asset_movement'][exchange_location] = 0
-            # we may have DB entries due to csv import from supported locations
-            for external_location in EXTERNAL_EXCHANGES:
-
-                movements = self._query_and_populate_exchange_asset_movements(
-                    from_ts=from_ts,
-                    to_ts=to_ts,
-                    all_movements=movements,
-                    exchange=external_location,
-                    only_cache=only_cache,
-                )
-            for exchange in self.exchange_manager.iterate_exchanges():
-                self._query_and_populate_exchange_asset_movements(
-                    from_ts=from_ts,
-                    to_ts=to_ts,
-                    all_movements=movements,
-                    exchange=exchange,
-                    only_cache=only_cache,
-                )
-
-        # return movements with most recent first
-        movements.sort(key=lambda x: x.timestamp, reverse=True)
-        return movements
-
     def set_settings(self, settings: ModifiableDBSettings) -> Tuple[bool, str]:
         """Tries to set new settings. Returns True in success or False with message if error"""
         if settings.eth_rpc_endpoint is not None:


Test patch:
diff --git a/rotkehlchen/tests/api/test_exchanges.py b/rotkehlchen/tests/api/test_exchanges.py
--- a/rotkehlchen/tests/api/test_exchanges.py
+++ b/rotkehlchen/tests/api/test_exchanges.py
@@ -16,7 +16,7 @@
 from rotkehlchen.exchanges.kucoin import API_KEY_ERROR_CODE_ACTION as KUCOIN_API_KEY_ERROR_CODE
 from rotkehlchen.exchanges.manager import SUPPORTED_EXCHANGES
 from rotkehlchen.fval import FVal
-from rotkehlchen.rotkehlchen import FREE_ASSET_MOVEMENTS_LIMIT, FREE_TRADES_LIMIT
+from rotkehlchen.history.events import FREE_ASSET_MOVEMENTS_LIMIT, FREE_TRADES_LIMIT
 from rotkehlchen.tests.utils.api import (
     api_url_for,
     assert_error_response,
diff --git a/rotkehlchen/tests/api/test_history.py b/rotkehlchen/tests/api/test_history.py
--- a/rotkehlchen/tests/api/test_history.py
+++ b/rotkehlchen/tests/api/test_history.py
@@ -322,6 +322,34 @@ def assert_csv_formulas_trades(row, profit_currency):
         )
 
 
+@pytest.mark.parametrize('ethereum_accounts', [[]])
+@pytest.mark.parametrize('mocked_price_queries', [prices])
+def test_query_history_external_exchanges(rotkehlchen_api_server):
+    """Test that history is processed for external exchanges too"""
+    start_ts = 0
+    end_ts = 1631455982
+
+    # import blockfi trades
+    dir_path = Path(__file__).resolve().parent.parent
+    filepath = dir_path / 'data' / 'blockfi-trades.csv'
+    json_data = {'source': 'blockfi-trades', 'file': str(filepath)}
+    response = requests.put(
+        api_url_for(
+            rotkehlchen_api_server,
+            'dataimportresource',
+        ), json=json_data,
+    )
+
+    # Query history processing to start the history processing
+    response = requests.get(
+        api_url_for(rotkehlchen_api_server, 'historyprocessingresource'),
+        json={'from_timestamp': start_ts, 'to_timestamp': end_ts},
+    )
+    outcome = assert_proper_response_with_result(response)
+    assert len(outcome['all_events']) == 2
+    assert FVal(outcome['overview']['total_taxable_profit_loss']) == FVal('5278.03086')
+
+
 def assert_csv_formulas_all_events(row, profit_currency):
     keys = list(row.keys())
     net_profit_loss = row['net_profit_or_loss']
diff --git a/rotkehlchen/tests/api/test_trades.py b/rotkehlchen/tests/api/test_trades.py
--- a/rotkehlchen/tests/api/test_trades.py
+++ b/rotkehlchen/tests/api/test_trades.py
@@ -8,7 +8,7 @@
 from rotkehlchen.constants.assets import A_AAVE, A_BTC, A_DAI, A_EUR, A_WETH
 from rotkehlchen.exchanges.data_structures import Trade
 from rotkehlchen.fval import FVal
-from rotkehlchen.rotkehlchen import FREE_TRADES_LIMIT
+from rotkehlchen.history.events import FREE_TRADES_LIMIT
 from rotkehlchen.tests.utils.api import (
     api_url_for,
     assert_error_response,
diff --git a/rotkehlchen/tests/fixtures/rotkehlchen.py b/rotkehlchen/tests/fixtures/rotkehlchen.py
--- a/rotkehlchen/tests/fixtures/rotkehlchen.py
+++ b/rotkehlchen/tests/fixtures/rotkehlchen.py
@@ -161,6 +161,7 @@ def mock_get_settings() -> DBSettings:
     if start_with_valid_premium:
         rotki.premium = Premium(rotki_premium_credentials)
         rotki.premium_sync_manager.premium = rotki.premium
+        rotki.chain_manager.premium = rotki.premium  # think this will conflict with develop
 
     if legacy_messages_via_websockets is False:
         rotki.msg_aggregator.rotki_notifier = None
diff --git a/rotkehlchen/tests/utils/history.py b/rotkehlchen/tests/utils/history.py
--- a/rotkehlchen/tests/utils/history.py
+++ b/rotkehlchen/tests/utils/history.py
@@ -6,7 +6,7 @@
 from rotkehlchen.accounting.structures import DefiEvent
 from rotkehlchen.api.v1.encoding import TradeSchema
 from rotkehlchen.chain.ethereum.trades import AMMTrade
-from rotkehlchen.constants.assets import A_BTC, A_ETH, A_LTC
+from rotkehlchen.constants.assets import A_BTC, A_ETH, A_LTC, A_USDC
 from rotkehlchen.constants.misc import ZERO
 from rotkehlchen.constants.resolver import strethaddress_to_identifier
 from rotkehlchen.exchanges.data_structures import AssetMovement, Loan, MarginPosition, Trade
@@ -135,6 +135,11 @@
             1624798800: FVal(1983.33),
         },
     },
+    A_USDC.identifier: {
+        'EUR': {
+            1612051199: FVal(0.8241),
+        },
+    },
     strethaddress_to_identifier('0x255Aa6DF07540Cb5d3d297f0D0D4D84cb52bc8e6'): {
         'EUR': {
             1512561942: ZERO,
@@ -149,6 +154,7 @@
         'EUR': {
             1493650800: FVal(14.07),
             1493737200: FVal(14.56),
+            1612051199: FVal(109.99),
         },
     },
     'XMR': {