Repository: rotki/rotki

Issue:
Research whether the Coinbase Pro API "fills" endpoint is worth it for our current trades history implementation
Our current get trades history implementation is based on reports. Research whether the fills endpoint could improve the current implementation or not.

[Trades](https://docs.pro.coinbase.com/#withdrawals)




Discussion:

Resolving patch:
diff --git a/rotkehlchen/exchanges/coinbasepro.py b/rotkehlchen/exchanges/coinbasepro.py
--- a/rotkehlchen/exchanges/coinbasepro.py
+++ b/rotkehlchen/exchanges/coinbasepro.py
@@ -1,28 +1,15 @@
 import binascii
-import csv
 import hashlib
 import hmac
 import json
 import logging
-import os
 import time
 from base64 import b64decode, b64encode
 from collections import defaultdict
 from http import HTTPStatus
 from json.decoder import JSONDecodeError
-from tempfile import TemporaryDirectory
-from typing import (
-    TYPE_CHECKING,
-    Any,
-    Callable,
-    DefaultDict,
-    Dict,
-    List,
-    Optional,
-    Tuple,
-    Union,
-    overload,
-)
+from typing import TYPE_CHECKING, Any, DefaultDict, Dict, List, Optional, Tuple, Union, Iterator
+from urllib.parse import urlencode
 
 import gevent
 import requests
@@ -32,6 +19,7 @@
 from rotkehlchen.assets.asset import Asset
 from rotkehlchen.assets.converters import asset_from_coinbase
 from rotkehlchen.constants.misc import ZERO
+from rotkehlchen.constants.assets import A_ETH
 from rotkehlchen.constants.timing import QUERY_RETRY_TIMES
 from rotkehlchen.errors import (
     DeserializationError,
@@ -52,11 +40,20 @@
     deserialize_price,
     deserialize_timestamp_from_date,
     deserialize_trade_type,
+    pair_get_assets,
+)
+from rotkehlchen.typing import (
+    ApiKey,
+    ApiSecret,
+    AssetMovementCategory,
+    Fee,
+    Location,
+    Timestamp,
+    TradePair,
+    AssetType,
 )
-from rotkehlchen.typing import ApiKey, ApiSecret, Fee, Location, Timestamp, TradePair
 from rotkehlchen.user_messages import MessagesAggregator
 from rotkehlchen.utils.interfaces import cache_response_timewise, protect_with_lock
-from rotkehlchen.utils.misc import timestamp_to_iso8601, ts_now
 from rotkehlchen.utils.serialization import rlk_jsonloads_dict, rlk_jsonloads_list
 
 if TYPE_CHECKING:
@@ -66,8 +63,7 @@
 log = RotkehlchenLogsAdapter(logger)
 
 
-SECS_TO_WAIT_FOR_REPORT = 300
-MINS_TO_WAIT_FOR_REPORT = SECS_TO_WAIT_FOR_REPORT / 60
+COINBASEPRO_PAGINATION_LIMIT = 100  # default + max limit
 
 
 def coinbasepro_to_worldpair(product: str) -> TradePair:
@@ -90,6 +86,20 @@ class CoinbaseProPermissionError(Exception):
     pass
 
 
+def coinbasepro_deserialize_timestamp(entry: Dict[str, Any], key: str) -> Timestamp:
+    """Deserialize a timestamp from coinbasepro
+
+    In case of an error raises:
+    - KeyError
+    - DeserializationError
+    """
+    raw_time = entry[key]
+    if raw_time.endswith('+00'):  # proper iso8601 needs + 00:00 for timezone
+        raw_time = raw_time.replace('+00', '+00:00')
+    timestamp = deserialize_timestamp_from_date(raw_time, 'iso8601', 'coinbasepro')
+    return timestamp
+
+
 class Coinbasepro(ExchangeInterface):  # lgtm[py/missing-call-to-init]
 
     def __init__(
@@ -103,6 +113,7 @@ def __init__(
         super().__init__('coinbasepro', api_key, secret, database)
         self.base_uri = 'https://api.pro.coinbase.com'
         self.msg_aggregator = msg_aggregator
+        self.account_to_currency: Optional[Dict[str, Asset]] = None
 
         self.session.headers.update({
             'Content-Type': 'Application/JSON',
@@ -138,43 +149,19 @@ def validate_api_key(self) -> Tuple[bool, str]:
 
         return True, ''
 
-    @overload  # noqa: F811
-    def _api_query(  # pylint: disable=no-self-use
-            self,
-            endpoint: Literal['accounts', 'products'],
-            request_method: Literal['GET'] = 'GET',
-            options: Optional[Dict[str, Any]] = None,
-    ) -> List[Any]:
-        ...
-
-    @overload  # noqa: F811
-    def _api_query(  # noqa: F811 pylint: disable=no-self-use
-            self,
-            endpoint: Literal['reports'],
-            request_method: Literal['GET', 'POST'] = 'GET',
-            options: Optional[Dict[str, Any]] = None,
-    ) -> Dict[str, Any]:
-        ...
-
-    @overload  # noqa: F811
-    def _api_query(  # noqa: F811 pylint: disable=no-self-use
-            self,
-            endpoint: str,
-            request_method: Literal['GET', 'POST'] = 'GET',
-            options: Optional[Dict[str, Any]] = None,
-    ) -> Union[List[Any], Dict[str, Any]]:
-        ...
-
-    def _api_query(  # noqa: F811
+    def _api_query(
             self,
             endpoint: str,
             request_method: Literal['GET', 'POST'] = 'GET',
             options: Optional[Dict[str, Any]] = None,
-    ) -> Union[List[Any], Dict[str, Any]]:
+            query_options: Optional[Dict[str, Any]] = None,
+    ) -> Tuple[List[Any], Optional[str]]:
         """Performs a coinbase PRO API Query for endpoint
 
         You can optionally provide extra arguments to the endpoint via the options argument.
 
+        Returns a tuple of the result and optional pagination cursor.
+
         Raises RemoteError if something went wrong with connecting or reading from the exchange
         Raises CoinbaseProPermissionError if the API Key does not have sufficient
         permissions for the endpoint
@@ -187,6 +174,10 @@ def _api_query(  # noqa: F811
         else:
             stringified_options = ''
             options = {}
+
+        if query_options:
+            request_url += '?' + urlencode(query_options)
+
         message = timestamp + request_method + request_url + stringified_options
         log.debug(
             'Coinbase Pro API query',
@@ -194,6 +185,7 @@ def _api_query(  # noqa: F811
             request_url=request_url,
             options=options,
         )
+
         if 'products' not in endpoint:
             try:
                 signature = hmac.new(
@@ -253,27 +245,52 @@ def _api_query(  # noqa: F811
                 f'status code: {response.status_code} and text: {response.text}',
             )
 
-        loading_function: Union[Callable[[str], Dict[str, Any]], Callable[[str], List[Any]]]
-        if any(x in endpoint for x in ('accounts', 'products')):
-            loading_function = rlk_jsonloads_list
-        else:
-            loading_function = rlk_jsonloads_dict
-
         try:
-            json_ret = loading_function(response.text)
+            json_ret = rlk_jsonloads_list(response.text)
         except JSONDecodeError as e:
             raise RemoteError(
                 f'Coinbase Pro {request_method} query at {full_url} '
                 f'returned invalid JSON response: {response.text}',
             ) from e
 
-        return json_ret
+        return json_ret, response.headers.get('cb-after', None)
+
+    def create_or_return_account_to_currency_map(self) -> Dict[str, Asset]:
+        if self.account_to_currency is not None:
+            return self.account_to_currency
+
+        accounts, _ = self._api_query('accounts')
+        self.account_to_currency = {}
+        for account in accounts:
+            try:
+                asset = asset_from_coinbase(account['currency'])
+                self.account_to_currency[account['id']] = asset
+            except UnsupportedAsset as e:
+                self.msg_aggregator.add_warning(
+                    f'Found coinbase pro account with unsupported asset '
+                    f'{e.asset_name}. Ignoring it.',
+                )
+                continue
+            except UnknownAsset as e:
+                self.msg_aggregator.add_warning(
+                    f'Found coinbase pro account result with unknown asset '
+                    f'{e.asset_name}. Ignoring it.',
+                )
+                continue
+            except KeyError as e:
+                self.msg_aggregator.add_warning(
+                    f'Found coinbase pro account entry with missing {str(e)} field. '
+                    f'Ignoring it',
+                )
+                continue
+
+        return self.account_to_currency
 
     @protect_with_lock()
     @cache_response_timewise()
     def query_balances(self) -> ExchangeQueryBalances:
         try:
-            accounts = self._api_query('accounts')
+            accounts, _ = self._api_query('accounts')
         except (CoinbaseProPermissionError, RemoteError) as e:
             msg = f'Coinbase Pro API request failed. {str(e)}'
             log.error(msg)
@@ -331,261 +348,111 @@ def query_balances(self) -> ExchangeQueryBalances:
 
         return dict(assets_balance), ''
 
-    def _get_products_ids(self) -> List[str]:
-        """Gets a list of all product ids (markets) offered by coinbase PRO
-
-        - Raises the same exceptions as _api_query()
-        - Can raise KeyError if the API does not return the expected response format.
-        """
-        products = self._api_query('products', request_method='GET')
-        return [product['id'] for product in products]
-
-    def _get_account_ids(self) -> List[str]:
-        """Gets a list of all account ids
-
-        - Raises the same exceptions as _api_query()
-        - Can raise KeyError if the API does not return the expected response format.
-        """
-        accounts = self._api_query('accounts')
-        return [account['id'] for account in accounts]
-
-    def _read_asset_movements(self, filepath: str) -> List[AssetMovement]:
-        """Reads a csv account type report and extracts the AssetMovements"""
-        with open(filepath, newline='') as csvfile:
-            reader = csv.DictReader(csvfile)
-            movements = []
-            for row in reader:
-                try:
-                    if row['type'] in ('withdrawal', 'deposit'):
-                        timestamp = deserialize_timestamp_from_date(
-                            row['time'],
-                            'iso8601',
-                            'coinbasepro',
-                        )
-                        asset = Asset(row['amount/balance unit'])
-                        movements.append(AssetMovement(
-                            location=Location.COINBASEPRO,
-                            category=deserialize_asset_movement_category(row['type']),
-                            address=None,  # can't get it from csv data
-                            transaction_id=None,  # can't get it from csv data
-                            timestamp=timestamp,
-                            asset=asset,
-                            amount=deserialize_asset_amount_force_positive(row['amount']),
-                            fee_asset=asset,
-                            # I don't see any fee in deposit withdrawals in coinbasepro
-                            fee=Fee(ZERO),
-                            link=str(row['transfer id']),
-                        ))
-                except UnknownAsset as e:
-                    self.msg_aggregator.add_warning(
-                        f'Found unknown Coinbasepro asset {e.asset_name}. '
-                        f'Ignoring its deposit/withdrawal.',
-                    )
-                    continue
-                except (DeserializationError, KeyError) as e:
-                    msg = str(e)
-                    if isinstance(e, KeyError):
-                        msg = f'Missing key entry for {msg}.'
-                    self.msg_aggregator.add_error(
-                        'Failed to deserialize a Coinbasepro deposit/withdrawal. '
-                        'Check logs for details. Ignoring it.',
-                    )
-                    log.error(
-                        'Error processing a coinbasepro  deposit/withdrawal.',
-                        raw_asset_movement=row,
-                        error=msg,
-                    )
-                    continue
-
-        return movements
-
-    def _read_trades(self, filepath: str) -> List[Trade]:
-        """Reads a csv fill type report and extracts the Trades"""
-        with open(filepath, newline='') as csvfile:
-            reader = csv.DictReader(csvfile)
-            trades = []
-            for row in reader:
-                try:
-                    timestamp = deserialize_timestamp_from_date(
-                        row['created at'],
-                        'iso8601',
-                        'coinbasepro',
-                    )
-                    trades.append(Trade(
-                        timestamp=timestamp,
-                        location=Location.COINBASEPRO,
-                        pair=coinbasepro_to_worldpair(row['product']),
-                        trade_type=deserialize_trade_type(row['side']),
-                        amount=deserialize_asset_amount(row['size']),
-                        rate=deserialize_price(row['price']),
-                        fee=deserialize_fee(row['fee']),
-                        fee_currency=Asset(row['price/fee/total unit']),
-                        link=row['trade id'],
-                        notes='',
-                    ))
-                except UnprocessableTradePair as e:
-                    self.msg_aggregator.add_warning(
-                        f'Found unprocessable Coinbasepro pair {e.pair}. Ignoring the trade.',
-                    )
-                    continue
-                except UnknownAsset as e:
-                    self.msg_aggregator.add_warning(
-                        f'Found unknown Coinbasepro asset {e.asset_name}. '
-                        f'Ignoring the trade.',
-                    )
-                    continue
-                except (DeserializationError, KeyError) as e:
-                    msg = str(e)
-                    if isinstance(e, KeyError):
-                        msg = f'Missing key entry for {msg}.'
-                    self.msg_aggregator.add_error(
-                        'Failed to deserialize a coinbasepro trade. '
-                        'Check logs for details. Ignoring it.',
-                    )
-                    log.error(
-                        'Error processing a coinbasepro trade.',
-                        raw_trade=row,
-                        error=msg,
-                    )
-                    continue
-
-        return trades
-
-    def _generate_reports(
+    def _paginated_query(
             self,
-            start_ts: Timestamp,
-            end_ts: Timestamp,
-            report_type: Literal['fills', 'account'],
-            tempdir: str,
-    ) -> List[str]:
-        """
-        Generates all the reports to get historical data from coinbase.
-
-        https://docs.pro.coinbase.com/#reports
-        There are 2 type of reports.
-        1. Fill reports which are per product id (market)
-        2. Account reports which are per account id
-
-        The fill reports have the following data format:
-        portfolio,trade id,product,side,created at,size,size unit,price,fee,
-        total,price/fee/total unit
-
-        The account reports have the following data format:
-        portfolio,type,time,amount,balance,amount/balance unit,transfer id,trade id,order id
-
-        Returns a list of filepaths where the reports were written.
-
-        - Raises the same exceptions as _api_query()
-        - Can raise KeyError if the API does not return the expected response format.
-        """
-        start_date = timestamp_to_iso8601(start_ts)
-        end_date = timestamp_to_iso8601(end_ts)
-
-        if report_type == 'fills':
-            account_or_product_ids = self._get_products_ids()
-            identifier_key = 'product_id'
-        else:
-            account_or_product_ids = self._get_account_ids()
-            identifier_key = 'account_id'
-
-        report_ids = []
-        options = {
-            'type': report_type,
-            'start_date': start_date,
-            'end_date': end_date,
-            'format': 'csv',
-            # The only way to disable emailing the report link is to give an invalid link
-            'email': 'some@invalidemail.com',
-        }
-        for identifier in account_or_product_ids:
-            options[identifier_key] = identifier
-            post_result = self._api_query('reports', request_method='POST', options=options)
-            report_ids.append(post_result['id'])
-
-        # At this point all reports must have been queued for creation at the server
-        # Now wait until they are ready and pull them one by one
-        report_paths = []
-        last_change_ts = ts_now()
+            endpoint: str,
+            query_options: Dict[str, Any] = None,
+            limit: int = COINBASEPRO_PAGINATION_LIMIT,
+    ) -> Iterator[List[Dict[str, Any]]]:
+        if query_options is None:
+            query_options = {}
+        query_options['limit'] = limit
         while True:
-            finished_ids_indices = []
-            for idx, report_id in enumerate(report_ids):
-                get_result = self._api_query(f'reports/{report_id}', request_method='GET')
-                # Have to add assert here for mypy since the endpoint string is
-                # a variable string and can't be overloaded and type checked
-                assert isinstance(get_result, dict)
-                if get_result['status'] != 'ready':
-                    continue
-                # a report is ready here so let's reset the timer
-                last_change_ts = ts_now()
-                file_url = get_result['file_url']
-                response = requests.get(file_url)
-                length = len(response.content)
-                # empty fill reports have length of 95, empty account reports 85
-                # So we assume a report of more than 100 chars has data.
-                if length > 100:
-                    log.debug(f'Got a populated report for id: {report_id}. Writing it to disk')
-                    filepath = os.path.join(tempdir, f'report_{report_id}.csv')
-                    with open(filepath, 'wb') as f:
-                        f.write(response.content)
-                    report_paths.append(filepath)
-                else:
-                    log.debug(f'Got report for id: {report_id} with length {length}. Skipping it')
-
-                finished_ids_indices.append(idx)
-
-            if ts_now() - last_change_ts > SECS_TO_WAIT_FOR_REPORT:
-                raise RemoteError(
-                    f'There has been no response from CoinbasePro reports for over '
-                    f' {MINS_TO_WAIT_FOR_REPORT} minutes. Bailing out.',
-                )
-
-            # Delete the report ids that have been downloaded. Note: reverse order
-            # so that we don't mess up the indices
-            for idx in reversed(finished_ids_indices):
-                del report_ids[idx]
-
-            # When there is no more ids to query break out of the loop
-            if len(report_ids) == 0:
+            result, after_cursor = self._api_query(endpoint=endpoint, query_options=query_options)
+            yield result
+            if after_cursor is None or len(result) < limit:
                 break
 
-        return report_paths
+            query_options['after'] = after_cursor
 
     def query_online_deposits_withdrawals(
             self,
             start_ts: Timestamp,
             end_ts: Timestamp,
     ) -> List[AssetMovement]:
-        """Queries coinbase pro for asset movements
-
-        1. Generates relevant reports and writes them to a temporary directory
-        2. Reads all files from that directory and extracts deposits/withdrawals
-        3. Temporary directory is removed
-        """
+        """Queries coinbase pro for asset movements"""
         log.debug('Query coinbasepro asset movements', start_ts=start_ts, end_ts=end_ts)
         movements = []
-        with TemporaryDirectory() as tempdir:
+        raw_movements = []
+        for batch in self._paginated_query(
+            endpoint='transfers',
+            query_options={'type': 'withdraw'},
+        ):
+            raw_movements.extend(batch)
+        for batch in self._paginated_query(
+            endpoint='transfers',
+            query_options={'type': 'deposit'},
+        ):
+            raw_movements.extend(batch)
+
+        account_to_currency = self.create_or_return_account_to_currency_map()
+        for entry in raw_movements:
             try:
-                filepaths = self._generate_reports(
-                    start_ts=start_ts,
-                    end_ts=end_ts,
-                    report_type='account',
-                    tempdir=tempdir,
+                timestamp = coinbasepro_deserialize_timestamp(entry, 'completed_at')
+                if timestamp < start_ts or timestamp > end_ts:
+                    continue
+
+                category = deserialize_asset_movement_category(entry['type'])
+                asset = account_to_currency.get(entry['account_id'], None)
+                if asset is None:
+                    log.warning(
+                        f'Skipping coinbase pro asset_movement {entry} due to inability to '
+                        f'match account id to an asset',
+                    )
+                    continue
+
+                address = None
+                transaction_id = None
+                fee = Fee(ZERO)
+                if category == AssetMovementCategory.DEPOSIT:
+                    try:
+                        address = entry['details']['crypto_address']
+                        transaction_id = entry['details']['crypto_transaction_hash']
+                    except KeyError:
+                        pass
+                else:  # withdrawal
+                    try:
+                        address = entry['details']['sent_to_address']
+                        transaction_id = entry['details']['crypto_transaction_hash']
+                        fee = deserialize_fee(entry['details']['fee'])
+                    except KeyError:
+                        pass
+
+                if transaction_id and (asset == A_ETH or asset.asset_type == AssetType.ETHEREUM_TOKEN):  # noqa: E501
+                    transaction_id = '0x' + transaction_id
+
+                movements.append(AssetMovement(
+                    location=Location.COINBASEPRO,
+                    category=category,
+                    address=address,
+                    transaction_id=transaction_id,
+                    timestamp=timestamp,
+                    asset=asset,
+                    amount=deserialize_asset_amount_force_positive(entry['amount']),
+                    fee_asset=asset,
+                    fee=fee,
+                    link=str(entry['id']),
+                ))
+            except UnknownAsset as e:
+                self.msg_aggregator.add_warning(
+                    f'Found unknown Coinbasepro asset {e.asset_name}. '
+                    f'Ignoring its deposit/withdrawal.',
                 )
-            except CoinbaseProPermissionError as e:
+                continue
+            except (DeserializationError, KeyError) as e:
+                msg = str(e)
+                if isinstance(e, KeyError):
+                    msg = f'Missing key entry for {msg}.'
                 self.msg_aggregator.add_error(
-                    f'Got permission error while querying Coinbasepro for '
-                    f'asset movements: {str(e)}',
+                    'Failed to deserialize a Coinbasepro deposit/withdrawal. '
+                    'Check logs for details. Ignoring it.',
                 )
-                return []
-            except RemoteError as e:
-                self.msg_aggregator.add_error(
-                    f'Got remote error while querying Coinbasepro for '
-                    f'asset movements: {str(e)}',
+                log.error(
+                    'Error processing a coinbasepro  deposit/withdrawal.',
+                    raw_asset_movement=entry,
+                    error=msg,
                 )
-                return []
-
-            for filepath in filepaths:
-                movements.extend(self._read_asset_movements(filepath))
+                continue
 
         return movements
 
@@ -594,35 +461,64 @@ def query_online_trade_history(
             start_ts: Timestamp,
             end_ts: Timestamp,
     ) -> List[Trade]:
-        """Queries coinbase pro for trades
-
-        1. Generates relevant reports and writes them to a temporary directory
-        2. Reads all files from that directory and extracts Trades
-        3. Temporary directory is removed
-        """
+        """Queries coinbase pro for trades"""
         log.debug('Query coinbasepro trade history', start_ts=start_ts, end_ts=end_ts)
+
         trades = []
-        with TemporaryDirectory() as tempdir:
+        raw_trades = []
+        for batch in self._paginated_query(
+            endpoint='orders',
+            query_options={'status': 'done'},
+        ):
+            raw_trades.extend(batch)
+
+        for entry in raw_trades:
+            timestamp = coinbasepro_deserialize_timestamp(entry, 'created_at')
+            if timestamp < start_ts or timestamp > end_ts:
+                continue
+
             try:
-                filepaths = self._generate_reports(
-                    start_ts=start_ts,
-                    end_ts=end_ts,
-                    report_type='fills',
-                    tempdir=tempdir,
+                pair = coinbasepro_to_worldpair(entry['product_id'])
+                # Fee currency seems to always be quote asset
+                # https://github.com/ccxt/ccxt/blob/ddf3a15cbff01541f0b37c35891aa143bb7f9d7b/python/ccxt/coinbasepro.py#L724  # noqa: E501
+                _, quote_asset = pair_get_assets(pair)
+                trades.append(Trade(
+                    timestamp=timestamp,
+                    location=Location.COINBASEPRO,
+                    pair=coinbasepro_to_worldpair(entry['product_id']),
+                    trade_type=deserialize_trade_type(entry['side']),
+                    amount=deserialize_asset_amount(entry['size']),
+                    rate=deserialize_price(entry['price']),
+                    fee=deserialize_fee(entry['fill_fees']),
+                    fee_currency=quote_asset,
+                    link=entry['id'],
+                    notes='',
+                ))
+            except UnprocessableTradePair as e:
+                self.msg_aggregator.add_warning(
+                    f'Found unprocessable Coinbasepro pair {e.pair}. Ignoring the trade.',
                 )
-            except CoinbaseProPermissionError as e:
-                self.msg_aggregator.add_error(
-                    f'Got permission error while querying Coinbasepro for trades: {str(e)}',
+                continue
+            except UnknownAsset as e:
+                self.msg_aggregator.add_warning(
+                    f'Found unknown Coinbasepro asset {e.asset_name}. '
+                    f'Ignoring the trade.',
                 )
-                return []
-            except RemoteError as e:
+                continue
+            except (DeserializationError, KeyError) as e:
+                msg = str(e)
+                if isinstance(e, KeyError):
+                    msg = f'Missing key entry for {msg}.'
                 self.msg_aggregator.add_error(
-                    f'Got remote error while querying Coinbasepro for trades: {str(e)}',
+                    'Failed to deserialize a coinbasepro trade. '
+                    'Check logs for details. Ignoring it.',
                 )
-                return []
-
-            for filepath in filepaths:
-                trades.extend(self._read_trades(filepath))
+                log.error(
+                    'Error processing a coinbasepro trade.',
+                    raw_trade=entry,
+                    error=msg,
+                )
+                continue
 
         return trades
 
diff --git a/rotkehlchen/serialization/deserialize.py b/rotkehlchen/serialization/deserialize.py
--- a/rotkehlchen/serialization/deserialize.py
+++ b/rotkehlchen/serialization/deserialize.py
@@ -612,7 +612,7 @@ def deserialize_asset_movement_category(symbol: str) -> AssetMovementCategory:
 
     if symbol.lower() == 'deposit':
         return AssetMovementCategory.DEPOSIT
-    if symbol.lower() == 'withdrawal':
+    if symbol.lower() in ('withdraw', 'withdrawal'):
         return AssetMovementCategory.WITHDRAWAL
 
     # else


Test patch:
diff --git a/rotkehlchen/tests/exchanges/test_coinbasepro.py b/rotkehlchen/tests/exchanges/test_coinbasepro.py
--- a/rotkehlchen/tests/exchanges/test_coinbasepro.py
+++ b/rotkehlchen/tests/exchanges/test_coinbasepro.py
@@ -1,81 +1,25 @@
-import json
+"""After removal of reports, this test file does not test the new way of querying
+trades, deposits and withdrawals.
+
+
+Lefteris tested with his own coinbase pro account at 21/03/2021 and they do work though.
+
+TODO: Make some mock tests at some point
+"""
+
+
 import warnings as test_warnings
 from enum import Enum
-from typing import Callable, Optional
 from unittest.mock import patch
 
-import requests
 from typing_extensions import Literal
 
 from rotkehlchen.constants.assets import A_ETH
-from rotkehlchen.constants.misc import ZERO
 from rotkehlchen.errors import UnknownAsset
 from rotkehlchen.exchanges.coinbasepro import Coinbasepro, coinbasepro_to_worldpair
-from rotkehlchen.exchanges.data_structures import AssetMovement, Location, Trade, TradeType
 from rotkehlchen.fval import FVal
 from rotkehlchen.tests.utils.constants import A_BAT
 from rotkehlchen.tests.utils.mock import MockResponse
-from rotkehlchen.typing import AssetAmount, AssetMovementCategory, Fee, Price, Timestamp, TradePair
-
-POST_REPORT_RESPONSE_TEMPLATE = """{{
-"id": "{}",
-"type": "fills",
-"status": "pending",
-"created_at": "2015-01-06T10:34:47.000Z",
-"completed_at": null,
-"expires_at": "2015-01-13T10:35:47.000Z",
-"file_url": null,
-"params": {{
-"start_date": "2014-11-01T00:00:00.000Z",
-"end_date": "2014-11-30T23:59:59.000Z"
-}}}}"""
-
-GET_REPORT_RESPONSE_TEMPLATE = """{{
-"id": "{}",
-"type": "fills",
-"status": "ready",
-"created_at": "2015-01-06T10:34:47.000Z",
-"completed_at": "2015-01-06T10:34:47.000Z",
-"expires_at": "2015-01-13T10:35:47.000Z",
-"file_url": "{}",
-"params": {{
-"start_date": "2014-11-01T00:00:00.000Z",
-"end_date": "2014-11-30T23:59:59.000Z"
-}}}}"""
-
-# Test fill report for the ETH-BAT product
-ETH_BAT_REPORT_ID = '0328b97b-bec1-129e-a94c-69232926778d'
-ETH_BAT_ROW = 'default,204623,BAT-ETH,SELL,2020-01-13T09:15:06.311Z,1.00000000,BAT,0.00131511,0.00000657555,0.00130853445,ETH'  # noqa: E501
-ETH_BAT_REPORT = f"""portfolio,trade id,product,side,created at,size,size unit,price,fee,total,price/fee/total unit
-{ETH_BAT_ROW}"""
-FILL_REPORT_UNKNOWN_ASSET = f"""portfolio,trade id,product,side,created at,size,size unit,price,fee,total,price/fee/total unit
-default,204623,UNKNOWN-ETH,SELL,2020-01-13T09:15:06.311Z,1.00000000,UNKNOWN,0.00131511,0.00000657555,0.00130853445,ETH
-{ETH_BAT_ROW}"""
-FILL_REPORT_KEY_ERROR = f"""portfolio,trade id,product,side,created at,size,size unit,price,fee,total,price/fee/total unit
-2020-01-13T09:15:06.311Z,1.00000000,0.00131511,0.00000657555,0.00130853445,ETH
-{ETH_BAT_ROW}"""
-
-# Test account reports
-ETH_ACCOUNT_REPORT_ID = '1328b97b-fec1-123e-b94c-69332926771d'
-ETH_ACCOUNT_ROWS = """default,match,2020-01-13T09:15:06.315Z,0.0013151100000000,0.0013151100000000,ETH,,204623,5abf20fe-2b79-2315-57d3-c143dd291654
-default,fee,2020-01-13T09:15:06.315Z,-0.0000065755500000,0.0013085344500000,ETH,,204623,5abf20fe-2b79-2315-57d3-c143dd291654
-default,withdrawal,2020-01-15T23:51:26.478Z,-0.0011085300000000,0.0002000044500000,ETH,fcc61b23-4b51-43f8-da1e-def2d5a217ad,,"""  # noqa: E501
-ETH_ACCOUNT_REPORT = f"""portfolio,type,time,amount,balance,amount/balance unit,transfer id,trade id,order id
-{ETH_ACCOUNT_ROWS}"""
-ETH_ACCOUNT_REPORT_UNKNOWN_ASSET = f"""portfolio,type,time,amount,balance,amount/balance unit,transfer id,trade id,order id
-default,withdrawal,2020-01-15T23:54:26.478Z,-0.0011085300000000,0.0002000044500000,UNKNOWN,fcc61b23-3b51-13f8-da1e-def2d5a217ad,,
-{ETH_ACCOUNT_ROWS}"""
-
-BAT_ACCOUNT_REPORT_ID = '9321b97c-fac2-224e-a94d-69332916772d'
-BAT_ACCOUNT_ROWS = """default,deposit,2020-01-12T23:26:44.073Z,14.2500000000000000,14.2500000000000000,BAT,dfdd574b-25ca-de01-asce-edc3c1f2e987,,
-default,deposit,2020-01-12T23:38:29.433Z,160.8000000000000000,175.0500000000000000,BAT,489f76g2-4dda-4ab8-3eac-6dffadaa57ba7,,
-default,deposit,2020-01-12T23:57:12.306Z,8.6500000000000000,183.7000000000000000,BAT,34c6d26c-d27d-4218-3d14-1493120543e9,,
-default,match,2020-01-13T09:15:06.315Z,-1.0000000000000000,182.7000000000000000,BAT,,204623,5abf20fe-2b79-2315-57d3-c143dd291654"""  # noqa: E501
-BAT_ACCOUNT_REPORT = f"""portfolio,type,time,amount,balance,amount/balance unit,transfer id,trade id,order id
-{BAT_ACCOUNT_ROWS}"""
-BAT_ACCOUNT_REPORT_WRONG_FORMAT = f"""portfolio,type,time,amount,balance,amount/balance unit,transfer id,trade id,order id
-default,deposit,2020-01-12T23:57:12.306Z,,foo,BAT,34c6d26c-d27d-4218-3d14-1493120543e9,,
-{BAT_ACCOUNT_ROWS}"""
 
 PRODUCTS_RESPONSE = """[{
 "id": "BAT-ETH",
@@ -143,44 +87,12 @@ class ErrorEmulation(Enum):
 def create_coinbasepro_query_mock(
         cb: Coinbasepro,
         emulate_errors: ErrorEmulation = ErrorEmulation.NONE,
-        original_get: Optional[Callable] = None,
 ):
 
-    def mock_get_request(url: str) -> MockResponse:
-        if 'download_report/' in url:
-            parts = url.split('download_report/')
-            assert len(parts) == 2
-            report_id = parts[1]
-            if report_id == ETH_ACCOUNT_REPORT_ID:
-                if emulate_errors == ErrorEmulation.UNKNOWN_ASSET:
-                    text = ETH_ACCOUNT_REPORT_UNKNOWN_ASSET
-                else:
-                    text = ETH_ACCOUNT_REPORT
-            elif report_id == BAT_ACCOUNT_REPORT_ID:
-                if emulate_errors == ErrorEmulation.ASSET_MOVEMENTS_WRONG_FORMAT:
-                    text = BAT_ACCOUNT_REPORT_WRONG_FORMAT
-                else:
-                    text = BAT_ACCOUNT_REPORT
-            elif report_id == ETH_BAT_REPORT_ID:
-                if emulate_errors == ErrorEmulation.UNKNOWN_ASSET:
-                    text = FILL_REPORT_UNKNOWN_ASSET
-                elif emulate_errors == ErrorEmulation.KEY_ERROR:
-                    text = FILL_REPORT_KEY_ERROR
-                else:
-                    text = ETH_BAT_REPORT
-            else:
-                raise AssertionError('Tried to download invalid coinbasepro report during tests')
-
-            return MockResponse(200, text)
-
-        # else
-        assert original_get, 'for mocked gets we need also the original function'
-        return original_get(url)
-
     def mock_coinbasepro_request(
-            request_method: Literal['get', 'post'],
+            request_method: Literal['get', 'post'],  # pylint: disable=unused-argument
             url: str,
-            data: str = '',
+            data: str = '',  # pylint: disable=unused-argument
             allow_redirects: bool = True,  # pylint: disable=unused-argument
     ) -> MockResponse:
         if 'products' in url:
@@ -195,58 +107,11 @@ def mock_coinbasepro_request(
                 text = KEYERROR_ACCOUNTS_RESPONSE
             else:
                 text = ACCOUNTS_RESPONSE
-        elif request_method == 'post' and 'reports' in url:
-            if emulate_errors == ErrorEmulation.INVALID_RESPONSE_POST_REPORT:
-                text = '{"foo": 5,,]'
-            else:
-                response_data = json.loads(data)
-                assert response_data['format'] == 'csv'
-                assert response_data['email'] == 'some@invalidemail.com'
-                if response_data['type'] == 'fills':
-                    assert 'account_id' not in response_data
-                    if response_data['product_id'] == 'BAT-ETH':
-                        text = POST_REPORT_RESPONSE_TEMPLATE.format(ETH_BAT_REPORT_ID)
-                    else:
-                        raise AssertionError(
-                            f'Unimplemented coinbasepro product_id '
-                            f'{response_data["product_id"]} in tests mock',
-                        )
-                elif response_data['type'] == 'account':
-                    assert 'product_id' not in response_data
-                    if response_data['account_id'] == ETH_ACCOUNT_ID:
-                        text = POST_REPORT_RESPONSE_TEMPLATE.format(ETH_ACCOUNT_REPORT_ID)
-                    elif response_data['account_id'] == BAT_ACCOUNT_ID:
-                        text = POST_REPORT_RESPONSE_TEMPLATE.format(BAT_ACCOUNT_REPORT_ID)
-                    else:
-                        raise AssertionError(
-                            f'Unimplemented coinbasepro account_id '
-                            f'{response_data["account_id"]} in tests mock',
-                        )
-                else:
-                    raise AssertionError(
-                        f'Unknown report type {response_data["type"]} '
-                        f'given to coinbasepro endpoint',
-                    )
-        elif request_method == 'get' and 'reports' in url:
-            if emulate_errors == ErrorEmulation.INVALID_RESPONSE_GET_REPORT:
-                text = '{"foo": 5,,]'
-            else:
-                parts = url.split('reports/')
-                assert len(parts) == 2
-                report_id = parts[1]
-                text = GET_REPORT_RESPONSE_TEMPLATE.format(
-                    report_id,
-                    f'http://download_report/{report_id}',
-                )
         else:
             raise AssertionError(f'Unknown url: {url} encountered during CoinbasePro mocking')
         return MockResponse(200, text)
 
     coinbasepro_mock = patch.object(cb.session, 'request', side_effect=mock_coinbasepro_request)
-    if original_get:
-        requests_get_mock = patch('requests.get', side_effect=mock_get_request)
-        return coinbasepro_mock, requests_get_mock
-
     return coinbasepro_mock
 
 
@@ -258,7 +123,7 @@ def test_name():
 def test_coverage_of_products():
     """Test that we can process all pairs and assets of the offered coinbasepro products"""
     exchange = Coinbasepro('a', b'a', object(), object(), '')
-    products = exchange._api_query('products', request_method='GET')
+    products, _ = exchange._api_query('products', request_method='GET')
     for product in products:
         try:
             # Make sure all products can be processed
@@ -346,220 +211,3 @@ def test_query_balances_keyerror_response(function_scope_coinbasepro):
     errors = cb.msg_aggregator.consume_errors()
     assert len(errors) == 1
     assert 'Error processing a coinbase pro account balance' in errors[0]
-
-
-EXPECTED_TRADE = Trade(
-    timestamp=Timestamp(1578906906),
-    location=Location.COINBASEPRO,
-    pair=TradePair('BAT_ETH'),
-    trade_type=TradeType.SELL,
-    amount=AssetAmount(FVal('1')),
-    rate=Price(FVal('0.00131511')),
-    fee=Fee(FVal('0.00000657555')),
-    fee_currency=A_ETH,
-    link='204623',
-)
-
-
-def test_query_trade_history(function_scope_coinbasepro):
-    """Test that querying trade data from coinbase pro works"""
-    cb = function_scope_coinbasepro
-    cb_query_mock, get_mock = create_coinbasepro_query_mock(
-        cb,
-        original_get=requests.get,
-    )
-    with cb_query_mock, get_mock:
-        trades = cb.query_trade_history(start_ts=0, end_ts=1579449769, only_cache=False)
-
-    assert len(trades) == 1
-    assert trades[0] == EXPECTED_TRADE
-    errors = cb.msg_aggregator.consume_errors()
-    assert len(errors) == 0
-
-
-def test_query_trade_history_unknown_assets(function_scope_coinbasepro):
-    """Test that unknown assets are handled when querying trade data from coinbase pro"""
-    cb = function_scope_coinbasepro
-    cb_query_mock, get_mock = create_coinbasepro_query_mock(
-        cb,
-        emulate_errors=ErrorEmulation.UNKNOWN_ASSET,
-        original_get=requests.get,
-    )
-    with cb_query_mock, get_mock:
-        trades = cb.query_trade_history(start_ts=0, end_ts=1579449769, only_cache=False)
-
-    assert len(trades) == 1
-    assert trades[0] == EXPECTED_TRADE
-
-    warnings = cb.msg_aggregator.consume_warnings()
-    assert len(warnings) == 1
-    assert 'Found unknown Coinbasepro asset UNKNOWN. Ignoring the trade' in warnings[0]
-    errors = cb.msg_aggregator.consume_errors()
-    assert len(errors) == 0
-
-
-def test_query_trade_history_wrong_format(function_scope_coinbasepro):
-    """Test that wrong data format are handled when querying trade data from coinbase pro"""
-    cb = function_scope_coinbasepro
-    cb_query_mock, get_mock = create_coinbasepro_query_mock(
-        cb,
-        emulate_errors=ErrorEmulation.KEY_ERROR,
-        original_get=requests.get,
-    )
-    with cb_query_mock, get_mock:
-        trades = cb.query_trade_history(start_ts=0, end_ts=1579449769, only_cache=False)
-
-    assert len(trades) == 1
-    assert trades[0] == EXPECTED_TRADE
-
-    warnings = cb.msg_aggregator.consume_warnings()
-    assert len(warnings) == 0
-
-    errors = cb.msg_aggregator.consume_errors()
-    assert len(errors) == 1
-    assert 'Failed to deserialize a coinbasepro trade. Check logs for details' in errors[0]
-
-
-def test_query_trade_history_invalid_response(function_scope_coinbasepro):
-    """Test that invalid response is handled when querying trade data from coinbase pro
-
-    We make the invalid response be in the POST report query in this test.
-"""
-    cb = function_scope_coinbasepro
-    cb_query_mock, get_mock = create_coinbasepro_query_mock(
-        cb,
-        emulate_errors=ErrorEmulation.INVALID_RESPONSE_POST_REPORT,
-        original_get=requests.get,
-    )
-    with cb_query_mock, get_mock:
-        trades = cb.query_trade_history(start_ts=0, end_ts=1579449769, only_cache=False)
-
-    assert len(trades) == 0
-
-    warnings = cb.msg_aggregator.consume_warnings()
-    assert len(warnings) == 0
-    errors = cb.msg_aggregator.consume_errors()
-    assert len(errors) == 1
-    assert 'returned invalid JSON response' in errors[0]
-
-
-EXPECTED_MOVEMENTS = [AssetMovement(
-    location=Location.COINBASEPRO,
-    category=AssetMovementCategory.DEPOSIT,
-    address=None,
-    transaction_id=None,
-    timestamp=Timestamp(1578871604),
-    asset=A_BAT,
-    amount=FVal('14.25'),
-    fee_asset=A_BAT,
-    fee=Fee(ZERO),
-    link='dfdd574b-25ca-de01-asce-edc3c1f2e987',
-), AssetMovement(
-    location=Location.COINBASEPRO,
-    category=AssetMovementCategory.DEPOSIT,
-    address=None,
-    transaction_id=None,
-    timestamp=Timestamp(1578872309),
-    asset=A_BAT,
-    amount=FVal('160.8'),
-    fee_asset=A_BAT,
-    fee=Fee(ZERO),
-    link='489f76g2-4dda-4ab8-3eac-6dffadaa57ba7',
-), AssetMovement(
-    location=Location.COINBASEPRO,
-    category=AssetMovementCategory.DEPOSIT,
-    address=None,
-    transaction_id=None,
-    timestamp=Timestamp(1578873432),
-    asset=A_BAT,
-    amount=FVal('8.65'),
-    fee_asset=A_BAT,
-    fee=Fee(ZERO),
-    link='34c6d26c-d27d-4218-3d14-1493120543e9',
-), AssetMovement(
-    location=Location.COINBASEPRO,
-    category=AssetMovementCategory.WITHDRAWAL,
-    address=None,
-    transaction_id=None,
-    timestamp=Timestamp(1579132286),
-    asset=A_ETH,
-    amount=FVal('0.0011085300000000'),
-    fee_asset=A_ETH,
-    fee=Fee(ZERO),
-    link='fcc61b23-4b51-43f8-da1e-def2d5a217ad',
-)]
-
-
-def test_query_asset_movements(function_scope_coinbasepro):
-    """Test that querying deposits/withdrawals from coinbase pro works"""
-    cb = function_scope_coinbasepro
-    cb_query_mock, get_mock = create_coinbasepro_query_mock(cb, original_get=requests.get)
-    with cb_query_mock, get_mock:
-        movements = cb.query_deposits_withdrawals(start_ts=0, end_ts=1579449769, only_cache=False)
-
-    assert movements == EXPECTED_MOVEMENTS
-    warnings = cb.msg_aggregator.consume_warnings()
-    assert len(warnings) == 0
-    errors = cb.msg_aggregator.consume_errors()
-    assert len(errors) == 0
-
-
-def test_query_asset_movements_unknown_assets(function_scope_coinbasepro):
-    """Test that unknown assets are handled in querying deposits/withdrawals from coinbase pro"""
-    cb = function_scope_coinbasepro
-    cb_query_mock, get_mock = create_coinbasepro_query_mock(
-        cb,
-        emulate_errors=ErrorEmulation.UNKNOWN_ASSET,
-        original_get=requests.get,
-    )
-    with cb_query_mock, get_mock:
-        movements = cb.query_deposits_withdrawals(start_ts=0, end_ts=1579449769, only_cache=False)
-
-    assert movements == EXPECTED_MOVEMENTS
-    warnings = cb.msg_aggregator.consume_warnings()
-    assert len(warnings) == 1
-    msg = 'Found unknown Coinbasepro asset UNKNOWN. Ignoring its deposit/withdrawal'
-    assert msg in warnings[0]
-    errors = cb.msg_aggregator.consume_errors()
-    assert len(errors) == 0
-
-
-def test_query_asset_movements_wrong_format(function_scope_coinbasepro):
-    """Test that invalid data are handled in querying deposits/withdrawals from coinbase pro"""
-    cb = function_scope_coinbasepro
-    cb_query_mock, get_mock = create_coinbasepro_query_mock(
-        cb,
-        emulate_errors=ErrorEmulation.ASSET_MOVEMENTS_WRONG_FORMAT,
-        original_get=requests.get,
-    )
-    with cb_query_mock, get_mock:
-        movements = cb.query_deposits_withdrawals(start_ts=0, end_ts=1579449769, only_cache=False)
-
-    assert movements == EXPECTED_MOVEMENTS
-    warnings = cb.msg_aggregator.consume_warnings()
-    assert len(warnings) == 0
-    errors = cb.msg_aggregator.consume_errors()
-    assert len(errors) == 1
-    assert 'Failed to deserialize a Coinbasepro deposit/withdrawal' in errors[0]
-
-
-def test_query_asset_movements_invalid_response(function_scope_coinbasepro):
-    """Test that invalid response is handled when querying deposits/withdrals from coinbase pro
-
-    We make the invalid response be in the GET report query in this test.
-"""
-    cb = function_scope_coinbasepro
-    cb_query_mock, get_mock = create_coinbasepro_query_mock(
-        cb,
-        emulate_errors=ErrorEmulation.INVALID_RESPONSE_GET_REPORT,
-        original_get=requests.get,
-    )
-    with cb_query_mock, get_mock:
-        movements = cb.query_deposits_withdrawals(start_ts=0, end_ts=1579449769, only_cache=False)
-
-    assert movements == []
-    warnings = cb.msg_aggregator.consume_warnings()
-    assert len(warnings) == 0
-    errors = cb.msg_aggregator.consume_errors()
-    assert len(errors) == 1
-    assert 'returned invalid JSON response' in errors[0]