Repository: rotki/rotki

Issue:
Add support for Bitstamp crypto transactions
As per [Bitstamp API v2 documentation](https://www.bitstamp.net/api/), the JSON returned by the user transactions endpoint does not include any chain-related data (e.g. txid, address). This data is obtained by requesting the crypto transactions endpoint.

Be aware the crypto transactions endpoint is not paginated like the user transactions one (e.g. it has no `since_id`).
 
Related #1856 and more specifically: https://github.com/rotki/rotki/pull/1856#discussion_r536031605



Discussion:

Resolving patch:
diff --git a/rotkehlchen/exchanges/bitstamp.py b/rotkehlchen/exchanges/bitstamp.py
--- a/rotkehlchen/exchanges/bitstamp.py
+++ b/rotkehlchen/exchanges/bitstamp.py
@@ -39,6 +39,7 @@
     ApiSecret,
     AssetAmount,
     ExchangeAuthCredentials,
+    Fee,
     Location,
     Timestamp,
 )
@@ -50,6 +51,7 @@
 
 if TYPE_CHECKING:
     from collections.abc import Callable
+
     from rotkehlchen.accounting.structures.base import HistoryEvent
     from rotkehlchen.db.dbhandler import DBHandler
     from rotkehlchen.fval import FVal
@@ -234,22 +236,135 @@ def query_online_deposits_withdrawals(
         }
         if start_ts != Timestamp(0):
             # Get latest link from the DB to know where to resume from
-            cursor = self.db.conn.cursor()
-            query_result = cursor.execute(
-                'SELECT link FROM asset_movements WHERE location=? AND timestamp <= ? ORDER BY timestamp DESC LIMIT 1',  # noqa: E501
-                (Location.BITSTAMP.serialize_for_db(), start_ts),
-            ).fetchone()
-            if query_result is not None:
-                since_id = int(query_result[0]) + 1
-                options.update({'since_id': since_id})
-
+            with self.db.conn.read_ctx() as cursor:
+                query_result = cursor.execute(
+                    'SELECT link FROM asset_movements WHERE location=? AND timestamp <= ? ORDER BY timestamp DESC LIMIT 1',  # noqa: E501
+                    (Location.BITSTAMP.serialize_for_db(), start_ts),
+                ).fetchone()
+                if query_result is not None:
+                    since_id = int(query_result[0]) + 1
+                    options.update({'since_id': since_id})
+
+        # get user transactions (which is deposits/withdrawals) with fees but not address/txid
         asset_movements: list[AssetMovement] = self._api_query_paginated(
             start_ts=start_ts,
             end_ts=end_ts,
             options=options,
             case='asset_movements',
         )
-        return asset_movements
+
+        # also query crypto transactions, to get address and transaction id (but no fee)
+        # TODO: The used query range should be replaced by cache for this: https://github.com/rotki/rotki/issues/5684  # noqa: E501
+        crypto_last_query_name = f'{self.location}_{self.name}_last_cryptotx_offset'
+        with self.db.conn.read_ctx() as cursor:
+            cursor.execute('SELECT end_ts FROM used_query_ranges WHERE name=?', (crypto_last_query_name,))  # noqa: E501
+            offset = 0
+            if (result := cursor.fetchone()) is not None:
+                offset = result[0]
+
+        crypto_asset_movements = self._query_crypto_transactions(offset)
+
+        # now the fun part. Figure out which asset movements from user transactions
+        # they correspond to so we can also have the fee taken into account
+        indices_to_delete = []
+        for asset_movement in asset_movements:
+            for idx, crypto_movement in enumerate(crypto_asset_movements):
+                if crypto_movement.timestamp == asset_movement.timestamp and crypto_movement.category == asset_movement.category and crypto_movement.asset == asset_movement.asset:  # noqa: E501
+                    asset_movement.address = crypto_movement.address
+                    asset_movement.transaction_id = crypto_movement.transaction_id
+                    indices_to_delete.append(idx)
+                    break
+
+        for idx in sorted(indices_to_delete, reverse=True):
+            del crypto_asset_movements[idx]  # remove the crypto asset movements whose data we already correlated  # noqa: E501
+
+        # even more fun. If somehow the endpoint is not called in order then we
+        # may end up with some crypto asset movements here that would need to
+        # check for corresponding asset movement in the DB.
+        serialized_location = Location.BITSTAMP.serialize_for_db()
+        indices_to_delete = []
+        with self.db.user_write() as write_cursor:
+            for idx, crypto_movement in enumerate(crypto_asset_movements):
+                write_cursor.execute(
+                    'SELECT id from asset_movements WHERE location=? AND category=? AND timestamp=? AND asset=?',  # noqa: E501
+                    (serialized_location, crypto_movement.category.serialize_for_db(), crypto_movement.timestamp, crypto_movement.asset.identifier),  # noqa: E501
+                )
+                if (result := write_cursor.fetchone()) is not None:
+                    write_cursor.execute(
+                        'UPDATE asset_movements SET address=?, transaction_id=? WHERE id=?',
+                        (crypto_movement.address, crypto_movement.transaction_id, result[0]),
+                    )
+                    indices_to_delete.append(idx)
+
+        for idx in sorted(indices_to_delete, reverse=True):
+            del crypto_asset_movements[idx]  # remove the crypto asset movements whose data we matched in the DB  # noqa: E501
+
+        return asset_movements + crypto_asset_movements
+
+    def _query_crypto_transactions(self, offset: int) -> list[AssetMovement]:
+        """Query crypto transactions to get address and transaction id.
+
+        Pagination here is unfortunately primitive. Can only use offset, so we rememmber the
+        last offset queried in the DB and after the query update it.
+
+        https://www.bitstamp.net/api/#tag/Transactions-private/operation/GetCryptoUserTransactions
+
+        May raise RemoteError
+        """
+        options = {}
+        options['limit'] = API_MAX_LIMIT
+        options['offset'] = offset
+        options['include_ious'] = False
+        total_asset_movements = []
+
+        while True:
+            response = self._api_query(
+                endpoint='crypto_transactions',
+                method='post',
+                options=options,
+            )
+            if response.status_code != HTTPStatus.OK:
+                return self._process_unsuccessful_response(
+                    response=response,
+                    case='crypto_transactions',
+                )
+
+            try:
+                response_dict = jsonloads_dict(response.text)
+            except JSONDecodeError:
+                msg = f'Bitstamp returned invalid JSON response: {response.text}.'
+                log.error(msg)
+                self.msg_aggregator.add_error(
+                    f'Got remote error while querying Bistamp crypto transactions: {msg}',
+                )
+                return []
+
+            asset_movements = [
+                self._deserialize_asset_movement_from_crypto_transaction(
+                    raw_movement=entry,
+                    category=category,
+                )
+                for entry_key, category in [
+                    ('deposits', AssetMovementCategory.DEPOSIT),
+                    ('withdrawals', AssetMovementCategory.WITHDRAWAL),
+                ]
+                for entry in response_dict.get(entry_key, {})
+            ]
+            options['offset'] += API_MAX_LIMIT  # add anyway so we can save new offset at end
+            total_asset_movements.extend(asset_movements)
+            if len(asset_movements) < API_MAX_LIMIT:
+                break
+
+        if options['offset'] != offset:
+            # write the new offset
+            crypto_last_query_name = f'{self.location}_{self.name}_last_cryptotx_offset'
+            with self.db.user_write() as write_cursor:
+                write_cursor.execute(  # TODO: Replace with cache from 5684 like above
+                    'INSERT OR REPLACE INTO used_query_ranges(name, start_ts, end_ts) VALUES(?, ?, ?) ',  # noqa: E501
+                    (crypto_last_query_name, 0, options['offset']),
+                )
+
+        return total_asset_movements
 
     def query_online_trade_history(
             self,
@@ -308,11 +423,13 @@ def validate_api_key(self) -> tuple[bool, str]:
 
     def _api_query(
             self,
-            endpoint: Literal['balance', 'user_transactions'],
+            endpoint: Literal['balance', 'user_transactions', 'crypto_transactions'],
             method: Literal['post'] = 'post',
             options: dict[str, Any] | None = None,
     ) -> Response:
         """Request a Bistamp API v2 endpoint (from `endpoint`).
+
+        May raise RemoteError
         """
         call_options = options.copy() if options else {}
         data = call_options or None
@@ -390,6 +507,7 @@ def _api_query_paginated(
             end_ts: Timestamp,
             options: dict[str, Any],
             case: Literal['trades', 'asset_movements'],
+            offset: int = 0,
     ) -> list[Trade] | (list[AssetMovement] | list):
         """Request a Bitstamp API v2 endpoint paginating via an options
         attribute.
@@ -418,7 +536,7 @@ def _api_query_paginated(
             raw_result_type_filter = USER_TRANSACTION_ASSET_MOVEMENT_TYPE
             response_case = 'asset_movements'
             case_pretty = 'asset movement'
-            deserialization_method = self._deserialize_asset_movement
+            deserialization_method = self._deserialize_asset_movement_from_user_transaction
         else:
             raise AssertionError(f'Unexpected Bitstamp case: {case}.')
 
@@ -506,7 +624,7 @@ def _api_query_paginated(
         return results
 
     @staticmethod
-    def _deserialize_asset_movement(
+    def _deserialize_asset_movement_from_user_transaction(
             raw_movement: dict[str, Any],
     ) -> AssetMovement:
         """Process a deposit/withdrawal user transaction from Bitstamp and
@@ -568,6 +686,42 @@ def _deserialize_asset_movement(
         )
         return asset_movement
 
+    @staticmethod
+    def _deserialize_asset_movement_from_crypto_transaction(
+            raw_movement: dict[str, Any],
+            category: AssetMovementCategory,
+    ) -> AssetMovement:
+        """Process a deposit/withdrawal crypto transaction from Bitstamp and
+        deserialize it.
+
+        Can raise DeserializationError.
+
+        https://www.bitstamp.net/api/#tag/Transactions-private/operation/GetCryptoUserTransactions
+        """
+
+        try:
+            timestamp = Timestamp(raw_movement['datetime'])
+            asset = asset_from_bitstamp(raw_movement['currency'])
+            amount = deserialize_asset_amount(raw_movement['amount'])
+            address = raw_movement['destinationAddress']
+            transaction_id = raw_movement['txid']
+            link = f'A {raw_movement["network"]} {category}'
+        except KeyError as e:
+            raise DeserializationError(f'Could not find key {e} in bitstramp crypto transaction') from e  # noqa: E501
+
+        return AssetMovement(
+            timestamp=timestamp,
+            location=Location.BITSTAMP,
+            category=category,
+            address=address,
+            transaction_id=transaction_id,
+            asset=asset,
+            amount=abs(amount),
+            fee_asset=asset,
+            fee=Fee(ZERO),
+            link=link,
+        )
+
     def _deserialize_trade(
             self,
             raw_trade: dict[str, Any],
@@ -679,14 +833,14 @@ def _process_unsuccessful_response(
     def _process_unsuccessful_response(
             self,
             response: Response,
-            case: Literal['asset_movements'],
+            case: Literal['asset_movements', 'crypto_transactions'],
     ) -> list[AssetMovement]:
         ...
 
     def _process_unsuccessful_response(
             self,
             response: Response,
-            case: Literal['validate_api_key', 'balances', 'trades', 'asset_movements'],
+            case: Literal['validate_api_key', 'balances', 'trades', 'asset_movements', 'crypto_transactions'],  # noqa: E501
     ) -> list | (tuple[bool, str] | ExchangeQueryBalances):
         """This function processes not successful responses for the following
         cases listed in `case`.
@@ -700,7 +854,7 @@ def _process_unsuccessful_response(
 
             if case in {'validate_api_key', 'balances'}:
                 return False, msg
-            if case in {'trades', 'asset_movements'}:
+            if case in {'trades', 'asset_movements', 'crypto_transactions'}:
                 self.msg_aggregator.add_error(
                     f'Got remote error while querying Bistamp {case_pretty}: {msg}',
                 )
@@ -721,7 +875,7 @@ def _process_unsuccessful_response(
 
         if case in {'validate_api_key', 'balances'}:
             return False, msg
-        if case in {'trades', 'asset_movements'}:
+        if case in {'trades', 'asset_movements', 'crypto_transactions'}:
             self.msg_aggregator.add_error(
                 f'Got remote error while querying Bistamp {case_pretty}: {msg}',
             )
diff --git a/rotkehlchen/exchanges/data_structures.py b/rotkehlchen/exchanges/data_structures.py
--- a/rotkehlchen/exchanges/data_structures.py
+++ b/rotkehlchen/exchanges/data_structures.py
@@ -62,7 +62,7 @@ def hash_id(hashable: str) -> TradeID:
 ]
 
 
-@dataclass(init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=True)
+@dataclass(init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False)
 class AssetMovement(AccountingEventMixin):
     location: Location
     category: AssetMovementCategory


Test patch:
diff --git a/rotkehlchen/tests/exchanges/test_bitstamp.py b/rotkehlchen/tests/exchanges/test_bitstamp.py
--- a/rotkehlchen/tests/exchanges/test_bitstamp.py
+++ b/rotkehlchen/tests/exchanges/test_bitstamp.py
@@ -106,7 +106,7 @@ def mock_api_query_response(endpoint, method='', options=None):  # pylint: disab
         movements = mock_bitstamp.query_online_deposits_withdrawals(0, 1)
         assert movements == []
         errors = mock_bitstamp.msg_aggregator.consume_errors()
-        assert len(errors) == 1
+        assert len(errors) == 2  # since we do 2 queries underneath
         assert API_ERR_AUTH_NONCE_MESSAGE in errors[0]
 
         trades, _ = mock_bitstamp.query_online_trade_history(0, 1)
@@ -831,7 +831,7 @@ def test_deserialize_asset_movement_deposit(mock_bitstamp):
         fee=Fee(FVal('0.0005')),
         link='2',
     )
-    expected_movement = mock_bitstamp._deserialize_asset_movement(raw_movement)
+    expected_movement = mock_bitstamp._deserialize_asset_movement_from_user_transaction(raw_movement)  # noqa: E501
     assert movement == expected_movement
 
     raw_movement = {
@@ -858,7 +858,7 @@ def test_deserialize_asset_movement_deposit(mock_bitstamp):
         fee=Fee(FVal('0.1')),
         link='3',
     )
-    expected_movement = mock_bitstamp._deserialize_asset_movement(raw_movement)
+    expected_movement = mock_bitstamp._deserialize_asset_movement_from_user_transaction(raw_movement)  # noqa: E501
     assert movement == expected_movement
 
     raw_movement = {
@@ -885,7 +885,7 @@ def test_deserialize_asset_movement_deposit(mock_bitstamp):
         fee=Fee(FVal('0.1')),
         link='3',
     )
-    expected_movement = mock_bitstamp._deserialize_asset_movement(raw_movement)
+    expected_movement = mock_bitstamp._deserialize_asset_movement_from_user_transaction(raw_movement)  # noqa: E501
     assert movement == expected_movement
 
 
@@ -914,7 +914,7 @@ def test_deserialize_asset_movement_withdrawal(mock_bitstamp):
         fee=Fee(FVal('50')),
         link='5',
     )
-    expected_movement = mock_bitstamp._deserialize_asset_movement(raw_movement)
+    expected_movement = mock_bitstamp._deserialize_asset_movement_from_user_transaction(raw_movement)  # noqa: E501
     assert movement == expected_movement
 
     raw_movement = {
@@ -941,7 +941,7 @@ def test_deserialize_asset_movement_withdrawal(mock_bitstamp):
         fee=Fee(FVal('0.1')),
         link='5',
     )
-    expected_movement = mock_bitstamp._deserialize_asset_movement(raw_movement)
+    expected_movement = mock_bitstamp._deserialize_asset_movement_from_user_transaction(raw_movement)  # noqa: E501
     assert movement == expected_movement