Repository: rotki/rotki

Issue:
Detect crypto deposits/withdrawals for Bitpanda
## Abstract

A user reported that they are missing crypto deposits/withdrawals in rotki v1.22.2 and bitpanda. It seems they are right. We don't query those as they are on a different endpoint

## Task

Implement it. The endpoint should be this one: https://developers.bitpanda.com/platform/#/transactions-get

The user was kind enough to send example response.

```json
{
    "data": [
        {
            "type": "wallet_transaction",
            "attributes": {
                "amount": "6608.34105600",
                "recipient": "0xanaddress",
                "time": {
                    "date_iso8601": "2021-10-10T09:01:12+02:00",
                    "unix": "1633849272"
                },
                "confirmations": 56,
                "in_or_out": "incoming",
                "type": "deposit",
                "status": "finished",
                "amount_eur": "5718.43",
                "amount_eur_incl_fee": "5718.43",
                "wallet_id": "XXX",
                "confirmation_by": "not_required",
                "confirmed": false,
                "cryptocoin_id": "27",
                "cryptocoin_symbol": "USDT",
                "last_changed": {
                    "date_iso8601": "2021-10-10T09:15:45+02:00",
                    "unix": "1633850145"
                },
                "fee": "0.00000000",
                "current_fiat_id": "1",
                "current_fiat_amount": "5718.43",
                "tx_id": "0xtxhash",
                "is_metal_storage_fee": false,
                "tags": [],
                "public_status": "finished",
                "is_bfc": false,
                "is_card": false
            },
            "id": "XXX"
        },
        {
            "type": "wallet_transaction",
            "attributes": {
                "amount": "1.55165264",
                "recipient": "REDACTED",
                "time": {
                    "date_iso8601": "2020-08-10T17:10:46+02:00",
                    "unix": "1597072246"
                },
                "confirmations": 8,
                "in_or_out": "outgoing",
                "type": "withdrawal",
                "status": "finished",
                "amount_eur": "524.09",
                "amount_eur_incl_fee": "526.66",
                "purpose_text": "",
                "wallet_id": "XXX",
                "contact_id": "XXX",
                "confirmation_by": "email",
                "confirmed": true,
                "cryptocoin_id": "5",
                "cryptocoin_symbol": "ETH",
                "last_changed": {
                    "date_iso8601": "2020-08-10T17:12:24+02:00",
                    "unix": "1597072344"
                },
                "fee": "0.00762000",
                "current_fiat_id": "1",
                "current_fiat_amount": "524.09",
                "tx_id": "REDACTED",
                "is_metal_storage_fee": false,
                "tags": [],
                "public_status": "finished",
                "is_bfc": false,
                "is_card": false
            },
            "id": "XXX"
        }
    ],
    "meta": {
        "total_count": 191,
        "page": 1,
        "page_size": 10
    },
    "links": {
        "next": "?page=2&page_size=10",
        "last": "?page=20&page_size=10",
        "self": "?page=1&page_size=10"
    }
}
```


Discussion:

Resolving patch:
diff --git a/rotkehlchen/exchanges/bitpanda.py b/rotkehlchen/exchanges/bitpanda.py
--- a/rotkehlchen/exchanges/bitpanda.py
+++ b/rotkehlchen/exchanges/bitpanda.py
@@ -140,23 +140,28 @@ def validate_api_key(self) -> Tuple[bool, str]:
             return False, f'Error validating Bitpanda API Key due to {str(e)}'
         return True, ''
 
-    def _deserialize_fiattx(
+    def _deserialize_wallettx(
             self,
             entry: Dict[str, Any],
             from_ts: Timestamp,
             to_ts: Timestamp,
     ) -> Optional[AssetMovement]:
-        """Deserializes a bitpanda fiatwallets/transactions entry to a deposit/withdrawal
+        """Deserializes a bitpanda fiatwallets/transactions or wallets/transactions
+        entry to a deposit/withdrawal
 
         Returns None and logs error is there is a problem or simpy None if
         it's not a type of entry we are interested in
         """
         try:
-            if entry['type'] != 'fiat_wallet_transaction' or entry['attributes']['status'] != 'finished':  # noqa: E501
+            transaction_type = entry['type']
+            if (
+                transaction_type not in ('fiat_wallet_transaction', 'wallet_transaction') or
+                entry['attributes']['status'] != 'finished'
+            ):
                 return None
             time = Timestamp(deserialize_int_from_str(
                 symbol=entry['attributes']['time']['unix'],
-                location='bitpanda fiat wallet transaction',
+                location='bitpanda wallet transaction',
             ))
             if time < from_ts or time > to_ts:
                 # should we also stop querying from calling method?
@@ -169,26 +174,32 @@ def _deserialize_fiattx(
             except DeserializationError:
                 return None  # not a deposit/withdrawal
 
-            fiat_id = entry['attributes']['fiat_id']
-            fiat_asset = self.fiat_map.get(fiat_id)
-            if fiat_asset is None:
+            if transaction_type == 'fiat_wallet_transaction':
+                asset_id = entry['attributes']['fiat_id']
+                asset = self.fiat_map.get(asset_id)
+            else:
+                asset_id = entry['attributes']['cryptocoin_id']
+                asset = self.cryptocoin_map.get(asset_id)
+            if asset is None:
                 self.msg_aggregator.add_error(
                     f'While deserializing Bitpanda fiat transaction, could not find '
-                    f'bitpanda fiat with id {fiat_id} in the mapping',
+                    f'bitpanda asset with id {asset_id} in the mapping',
                 )
                 return None
             amount = deserialize_asset_amount(entry['attributes']['amount'])
             fee = deserialize_fee(entry['attributes']['fee'])
             tx_id = entry['id']
 
+            transaction_id = entry['attributes'].get('tx_id')
+            address = entry['attributes'].get('recipient')
         except (DeserializationError, KeyError) as e:
             msg = str(e)
             if isinstance(e, KeyError):
-                msg = f'Missing key {msg} for fiat transaction entry'
+                msg = f'Missing key {msg} for wallet transaction entry'
 
-            self.msg_aggregator.add_error(f'Error processing bitpanda fiat transaction entry due to {msg}')  # noqa: E501
+            self.msg_aggregator.add_error(f'Error processing bitpanda wallet transaction entry due to {msg}')  # noqa: E501
             log.error(
-                'Error processing bitpanda fiat transaction entry',
+                'Error processing bitpanda wallet transaction entry',
                 error=msg,
                 entry=entry,
             )
@@ -197,12 +208,12 @@ def _deserialize_fiattx(
         return AssetMovement(
             location=Location.BITPANDA,
             category=movement_category,
-            address=None,
-            transaction_id=None,
+            address=address,
+            transaction_id=transaction_id,
             timestamp=time,
-            asset=fiat_asset,
+            asset=asset,
             amount=amount,
-            fee_asset=fiat_asset,
+            fee_asset=asset,
             fee=fee,
             link=tx_id,
         )
@@ -297,7 +308,13 @@ def _deserialize_trade(
     @overload
     def _api_query(  # pylint: disable=no-self-use
             self,
-            endpoint: Literal['wallets', 'fiatwallets', 'trades', 'fiatwallets/transactions'],
+            endpoint: Literal[
+                'wallets',
+                'fiatwallets',
+                'trades',
+                'fiatwallets/transactions',
+                'wallets/transactions',
+            ],
             options: Optional[Dict[str, Any]] = None,
     ) -> Tuple[List[Any], Optional[Dict[str, Any]], Optional[Dict[str, Any]]]:
         ...
@@ -389,7 +406,7 @@ def _query_endpoint_until_end(  # pylint: disable=no-self-use
     @overload
     def _query_endpoint_until_end(  # pylint: disable=no-self-use
             self,
-            endpoint: Literal['fiatwallets/transactions'],
+            endpoint: Literal['fiatwallets/transactions', 'wallets/transactions'],
             from_ts: Optional[Timestamp],
             to_ts: Optional[Timestamp],
             options: Optional[Dict[str, Any]] = None,
@@ -398,7 +415,7 @@ def _query_endpoint_until_end(  # pylint: disable=no-self-use
 
     def _query_endpoint_until_end(
             self,
-            endpoint: Literal['trades', 'fiatwallets/transactions'],
+            endpoint: Literal['trades', 'fiatwallets/transactions', 'wallets/transactions'],
             from_ts: Optional[Timestamp],
             to_ts: Optional[Timestamp],
             options: Optional[Dict[str, Any]] = None,
@@ -413,7 +430,8 @@ def _query_endpoint_until_end(
         page = 1
         count_so_far = 0
         result = []
-        deserialize_fn = self._deserialize_trade if endpoint == 'trades' else self._deserialize_fiattx  # noqa: E501
+        deserialize_fn = self._deserialize_trade if endpoint == 'trades' else self._deserialize_wallettx  # noqa: E501
+
         while True:
             given_options['page_size'] = MAX_PAGE_SIZE
             given_options['page'] = page
@@ -529,6 +547,12 @@ def query_online_deposits_withdrawals(
             from_ts=start_ts,
             to_ts=end_ts,
         )
+        crypto_movements = self._query_endpoint_until_end(
+            endpoint='wallets/transactions',
+            from_ts=start_ts,
+            to_ts=end_ts,
+        )
+        movements.extend(crypto_movements)
         return movements
 
     def query_online_margin_history(


Test patch:
diff --git a/rotkehlchen/tests/exchanges/test_bitpanda.py b/rotkehlchen/tests/exchanges/test_bitpanda.py
--- a/rotkehlchen/tests/exchanges/test_bitpanda.py
+++ b/rotkehlchen/tests/exchanges/test_bitpanda.py
@@ -7,7 +7,7 @@
 import requests
 
 from rotkehlchen.assets.converters import asset_from_bitpanda
-from rotkehlchen.constants.assets import A_ADA, A_BEST, A_ETH, A_EUR, A_LTC
+from rotkehlchen.constants.assets import A_ADA, A_BEST, A_ETH, A_EUR, A_LTC, A_USDT
 from rotkehlchen.constants.misc import ZERO
 from rotkehlchen.errors import RemoteError, UnknownAsset
 from rotkehlchen.exchanges.data_structures import AssetMovement, Trade, TradeType
@@ -229,11 +229,16 @@ def mock_bitpanda_query(url: str, **kwargs):  # pylint: disable=unused-argument
 {"type":"fiat_wallet_transaction","attributes":{"fiat_wallet_id":"foo","user_id":"foo","fiat_id":"1","amount":"50.00","fee":"0.01","to_eur_rate":"1.00000000","time":{"date_iso8601":"2021-09-17T15:22:28+01:00","unix":"1631888548"},"in_or_out":"outgoing","type":"withdrawal","status":"finished","confirmation_by":"not_required","confirmed":true,"payment_option_id":"5","requires_2fa_approval":false,"is_savings":true,"last_changed":{"date_iso8601":"2021-09-17T15:22:28+01:00","unix":"1631888548"},"tags":[{"type":"tag","attributes":{"short_name":"savings.reserved_for_trade","name":"F\\u00fcr den Trade reserviert"}}],"public_status":"finished","is_index":false,"is_card":false,"is_card_order_charge":false},"id":"movementid2"}],
 "meta":{"total_count":2,"page":1,"page_size":100},"links":{"self":"?page=1&page_size=100"}}"""  # noqa: E501
 
+WALLET_TX_RESPONSE = """{"data":[{"type":"wallet_transaction","attributes":{"amount":"6608.34105600","recipient":"0x54dca71a34f498e3053cba240895e51da5f89d24","time":{"date_iso8601":"2021-10-10T09:01:12+02:00","unix":"1633849272"},"confirmations":56,"in_or_out":"incoming","type":"deposit","status":"finished","amount_eur":"5718.43","amount_eur_incl_fee":"5718.43","wallet_id":"XXX","confirmation_by":"not_required","confirmed":false,"cryptocoin_id":"27","cryptocoin_symbol":"USDT","last_changed":{"date_iso8601":"2021-10-10T09:15:45+02:00","unix":"1633850145"},"fee":"0.00000000","current_fiat_id":"1","current_fiat_amount":"5718.43","tx_id":"0x28cb2ba8ac14bdedb0ad021662b631952ce2514f1e3ff7870882ebe8a8c1b03f","is_metal_storage_fee":false,"tags":[],"public_status":"finished","is_bfc":false,"is_card":false},"id":"XXX"},{"type":"wallet_transaction","attributes":{"amount":"1.55165264","recipient":"0x54dca71a34f498e3053cba240895e51da5f89d24","time":{"date_iso8601":"2020-08-10T17:10:46+02:00","unix":"1597072246"},"confirmations":8,"in_or_out":"outgoing","type":"withdrawal","status":"finished","amount_eur":"524.09","amount_eur_incl_fee":"526.66","purpose_text":"","wallet_id":"XXX","contact_id":"XXX","confirmation_by":"email","confirmed":true,"cryptocoin_id":"5","cryptocoin_symbol":"ETH","last_changed":{"date_iso8601":"2020-08-10T17:12:24+02:00","unix":"1597072344"},"fee":"0.00762000","current_fiat_id":"1","current_fiat_amount":"524.09","tx_id":"0xe45c1befc0968d2dab0374bc8d1aa3e193136dc769596d42e4d3274475bc7c60","is_metal_storage_fee":false,"tags":[],"public_status":"finished","is_bfc":false,"is_card":false},"id":"XXX"}],
+"meta":{"total_count":2,"page":1,"page_size":10},"links":{"next":"?page=2&page_size=10","last":"?page=20&page_size=10","self":"?page=1&page_size=10"}}"""  # noqa: E501
+
 
 def test_asset_movements(mock_bitpanda):
     """Test that deposits/withdrawals are correctly queried"""
 
     def mock_bitpanda_query(url: str, **kwargs):  # pylint: disable=unused-argument
+        if '/wallets/transactions' in url:
+            return MockResponse(status_code=HTTPStatus.OK, text=WALLET_TX_RESPONSE)
         if '/wallets' in url:
             return MockResponse(status_code=HTTPStatus.OK, text=WALLETS_RESPONSE)
         if '/fiatwallets/transactions' in url:
@@ -277,5 +282,28 @@ def mock_bitpanda_query(url: str, **kwargs):  # pylint: disable=unused-argument
         amount=FVal('50'),
         fee_asset=A_EUR,
         fee=FVal('0.01'),
-        link='movementid2')]
+        link='movementid2',
+    ), AssetMovement(
+        location=Location.BITPANDA,
+        category=AssetMovementCategory.DEPOSIT,
+        address='0x54dca71a34f498e3053cba240895e51da5f89d24',
+        transaction_id='0x28cb2ba8ac14bdedb0ad021662b631952ce2514f1e3ff7870882ebe8a8c1b03f',
+        timestamp=1633849272,
+        asset=A_USDT,
+        amount=FVal('6608.34105600'),
+        fee_asset=A_USDT,
+        fee=ZERO,
+        link='XXX',
+    ), AssetMovement(
+        location=Location.BITPANDA,
+        category=AssetMovementCategory.WITHDRAWAL,
+        address='0x54dca71a34f498e3053cba240895e51da5f89d24',
+        transaction_id='0xe45c1befc0968d2dab0374bc8d1aa3e193136dc769596d42e4d3274475bc7c60',
+        timestamp=1597072246,
+        asset=A_ETH,
+        amount=FVal('1.55165264'),
+        fee_asset=A_ETH,
+        fee=FVal('0.00762000'),
+        link='XXX',
+    )]
     assert expected_movements == movements