Repository: rotki/rotki

Issue:
Handle known kraken edge cases with negative amounts properly
## Problem Definition

Some times the kraken API for its ledger events returns weird combinations of events that rotki does not handle properly. We should identify the ones we know that can happen and handle them properly

## Task
The known cases are below

### Failed Withdrawals

As seen here: https://twitter.com/LefterisJP/status/1663530022416547841

Figure out what should be handled better. Perhaps combining the info with the withdrawals endpoint you can see more information (including that it's failed)

The way it's displayed now is not terrible, but we need to make sure that it's a no-op in accounting. As it most probably is not. If we can also display it in a better way that would be nice.

Whoever implements it this I can provide test data I suppose.

### Edge cases in the new earn/staking

Some earn/staking events after kraken's revamp appear wrongly in the history events, with negative amounts.

![2023-09-21_18-34](https://github.com/rotki/rotki/assets/1658405/9dd1f637-8756-4b50-a2dc-393496f4db55)

This needs to be fixed.

### A user reported negative amounts in trades

~~This will need more investigation.~~ Can't reproduce or get data. I think he misunderstood as he mentioned tiny amounts which sound like the kraken internal adjustments I am fixing.


Discussion:
Putting this in 1.31.0, let's see if we can do at least part of it, preferrably all. Since we are purging kraken events in 1.31.0

Resolving patch:
diff --git a/rotkehlchen/exchanges/kraken.py b/rotkehlchen/exchanges/kraken.py
--- a/rotkehlchen/exchanges/kraken.py
+++ b/rotkehlchen/exchanges/kraken.py
@@ -940,9 +940,14 @@ def process_kraken_raw_events(
             events: list[dict[str, Any]],
             events_source: str,
             save_skipped_events: bool,
-    ) -> list[HistoryEvent]:
+    ) -> tuple[list[HistoryEvent], set[str]]:
+        """Run through a list of raw kraken events with different refids and process them.
+
+        Returns a list of the newly created rotki events and a set of all processed refids
+        """
         # Group related events
         raw_events_grouped = defaultdict(list)
+        processed_refids = set()
         for raw_event in events:
             raw_events_grouped[raw_event['refid']].append(raw_event)
 
@@ -961,10 +966,15 @@ def process_kraken_raw_events(
                 log.error(f'Failed to read timestamp for {raw_events}')
                 continue
 
-            group_events, found_unknown_event = self.history_event_from_kraken(events=events, save_skipped_events=save_skipped_events)  # noqa: E501
+            group_events, skipped, found_unknown_event = self.history_event_from_kraken(
+                events=events,
+                save_skipped_events=save_skipped_events,
+            )
             if found_unknown_event:
                 for event in group_events:
                     event.event_type = HistoryEventType.INFORMATIONAL
+            if not skipped:
+                processed_refids.add(events[0]['refid'])
             new_events.extend(group_events)
 
         if len(new_events) != 0:
@@ -976,7 +986,7 @@ def process_kraken_raw_events(
                         f'Failed to save kraken events from {events_source} in database. {e!s}',
                     )
 
-        return new_events
+        return new_events, processed_refids
 
     @protect_with_lock()
     def query_kraken_ledgers(
@@ -1037,7 +1047,7 @@ def query_kraken_ledgers(
                 )
                 return True
 
-            new_events = self.process_kraken_raw_events(
+            new_events, _ = self.process_kraken_raw_events(
                 events=response,
                 events_source=f'{query_start_ts} to {query_end_ts}',
                 save_skipped_events=True,
@@ -1059,17 +1069,19 @@ def history_event_from_kraken(
             self,
             events: list[dict[str, Any]],
             save_skipped_events: bool,
-    ) -> tuple[list[HistoryEvent], bool]:
+    ) -> tuple[list[HistoryEvent], bool, bool]:
         """
         This function gets raw data from kraken and creates a list of related history events
-        to be used in the app. It returns a list of events
+        to be used in the app. All events passed to this functon have same refid.
+
+        It returns a list of events, a boolean indicating events are skipped
         and a boolean in the case that an unknown type is found.
 
         If `save_skipped_events` is True then any events that are skipped are saved
         in the DB for processing later.
         """
-        group_events = []
-        skipped_refids = set()
+        group_events: list[tuple[int, HistoryEvent]] = []
+        skipped = False
         # for receive/spend events they could be airdrops but they could also be instant swaps.
         # the only way to know if it was a trade is by finding a pair of receive/spend events.
         # This is why we collect them instead of directly pushing to group_events
@@ -1079,11 +1091,13 @@ def history_event_from_kraken(
 
         for idx, raw_event in enumerate(events):
             try:
+                if skipped:  # bad: Using exception for control flow. This function needs refactor
+                    raise DeserializationError('Hit a skipped event')
+
+                identifier = raw_event['refid']
                 timestamp = TimestampMS((deserialize_fval(
                     value=raw_event['time'], name='time', location='kraken ledger processing',
                 ) * 1000).to_int(exact=False))
-                if (identifier := raw_event['refid']) in skipped_refids:  # using exception for control flow is ugly. May make sense to change in a refactor  # noqa: E501
-                    raise DeserializationError(f'Hit a skipped refid {identifier}')
 
                 event_type = kraken_ledger_entry_type_to_ours(raw_event['type'])
                 asset = asset_from_kraken(raw_event['asset'])
@@ -1094,28 +1108,36 @@ def history_event_from_kraken(
                     name='event amount',
                     location='kraken ledger processing',
                 )
+
                 # If we don't know how to handle an event atm or we find an unsupported
                 # event type the logic will be to store it as unknown and if in the future
                 # we need some information from it we can take actions to process them
                 if event_type == HistoryEventType.TRANSFER:
-                    if raw_event['subtype'] == 'spottostaking':
+                    if raw_event['subtype'] == '':  # Internal kraken events
+                        # Lefteris has seen it in: Crediting airdrops/fork coins
+                        # such as ETC, BCH, BSV. OR the XXLM airdrop. Also for forced
+                        # removal of a coin due to delisting(negative amount), which is followed
+                        # by another similar transfer with positive amount. Also OTC
+                        # or other private deals seem to have this type
+                        event_type = HistoryEventType.ADJUSTMENT
+                        event_subtype = HistoryEventSubType.SPEND if raw_amount < ZERO else HistoryEventSubType.RECEIVE  # noqa: E501
+                    elif raw_event['subtype'] == 'spottostaking':
                         event_type = HistoryEventType.STAKING
                         event_subtype = HistoryEventSubType.DEPOSIT_ASSET
                     elif raw_event['subtype'] == 'stakingfromspot':
-                        event_type = HistoryEventType.STAKING
-                        event_subtype = HistoryEventSubType.RECEIVE_WRAPPED
+                        continue  # no need to have an event here. Covered by deposit_asset
                     elif raw_event['subtype'] == 'stakingtospot':
                         event_type = HistoryEventType.STAKING
                         event_subtype = HistoryEventSubType.REMOVE_ASSET
                     elif raw_event['subtype'] == 'spotfromstaking':
-                        event_type = HistoryEventType.STAKING
-                        event_subtype = HistoryEventSubType.RETURN_WRAPPED
-                elif event_type in (HistoryEventType.ADJUSTMENT, HistoryEventType.TRADE):
-                    if raw_amount < ZERO:
-                        event_subtype = HistoryEventSubType.SPEND
-                        raw_amount = -raw_amount
-                    else:
+                        continue  # no need to have an event here - covered by remove_asset
+                    elif raw_event['subtype'] == 'spotfromfutures':
+                        # at least for lefteris the credit of ETHW is this type
+                        event_type = HistoryEventType.ADJUSTMENT
                         event_subtype = HistoryEventSubType.RECEIVE
+
+                elif event_type in (HistoryEventType.ADJUSTMENT, HistoryEventType.TRADE):
+                    event_subtype = HistoryEventSubType.SPEND if raw_amount < ZERO else HistoryEventSubType.RECEIVE  # noqa: E501
                 elif event_type == HistoryEventType.STAKING:
                     # in the case of ETH.S after the activation of withdrawals rewards no longer
                     # compound unlike what happens with other assets and a virtual event with
@@ -1123,7 +1145,6 @@ def history_event_from_kraken(
                     if asset == A_ETH2 and raw_amount < ZERO:
                         event_type = HistoryEventType.INFORMATIONAL
                         notes = 'Automatic virtual conversion of staked ETH rewards to ETH'
-                        raw_amount = -raw_amount
                     else:
                         event_subtype = HistoryEventSubType.REWARD
                 elif event_type == HistoryEventType.INFORMATIONAL:
@@ -1132,10 +1153,19 @@ def history_event_from_kraken(
                     log.warning(
                         f'Encountered kraken historic event type we do not process. {raw_event}',
                     )
-                elif event_type == HistoryEventType.WITHDRAWAL:
-                    raw_amount = -raw_amount
 
                 fee_amount = deserialize_asset_amount(raw_event['fee'])
+                # check for failed events (events that cancel each other out -- like failed
+                if (  # withdrawals). Compare if amounts cancel themselves out (also fee if exists)
+                        len(events) == 2 and idx == 1 and
+                        events[0]['type'] == events[1]['type'] and
+                        events[0]['subtype'] == events[1]['subtype'] and
+                        abs(raw_amount) == group_events[0][1].balance.amount and (
+                            len(group_events) != 2 or
+                            abs(fee_amount) == group_events[1][1].balance.amount)
+                ):
+                    log.info(f'Skipping failed kraken events that cancel each other out: {events}')
+                    return [], skipped, False
 
                 # Make sure to not generate an event for KFEES that is not of type FEE
                 if asset != A_KFEE:
@@ -1146,8 +1176,8 @@ def history_event_from_kraken(
                         location=Location.KRAKEN,
                         location_label=self.name,
                         asset=asset,
-                        balance=Balance(
-                            amount=raw_amount,
+                        balance=Balance(  # amount sign was used above to determine types
+                            amount=abs(raw_amount),  # now enforce positive
                             usd_value=ZERO,
                         ),
                         notes=notes,
@@ -1167,7 +1197,7 @@ def history_event_from_kraken(
                         location_label=self.name,
                         asset=asset,
                         balance=Balance(
-                            amount=fee_amount,
+                            amount=abs(fee_amount),
                             usd_value=ZERO,
                         ),
                         notes=notes,
@@ -1178,8 +1208,7 @@ def history_event_from_kraken(
                     # fee and KFEE
                     current_fee_index += 1
             except (DeserializationError, KeyError, UnknownAsset) as e:
-                if (refid := raw_event.get('refid')) is not None:
-                    skipped_refids.add(refid)
+                skipped = True
                 msg = str(e)
                 if isinstance(e, KeyError):
                     msg = f'Keyrror {msg}'
@@ -1199,11 +1228,7 @@ def history_event_from_kraken(
         for event_set in receive_spend_events.values():
             if len(event_set) == 2:
                 for _, event in event_set:
-                    if event.event_type == HistoryEventType.RECEIVE:
-                        event.event_subtype = HistoryEventSubType.RECEIVE
-                    else:
-                        event.event_subtype = HistoryEventSubType.SPEND
-                        event.balance.amount = -event.balance.amount
+                    event.event_subtype = HistoryEventSubType.RECEIVE if event.event_type == HistoryEventType.RECEIVE else HistoryEventSubType.SPEND  # noqa: E501
                     event.event_type = HistoryEventType.TRADE
 
             # make sure to add all the events to group_events
@@ -1211,7 +1236,7 @@ def history_event_from_kraken(
 
         returned_events = []
         for raw_event_idx, event in group_events:
-            if event.event_identifier in skipped_refids:  # add it also to skipped events if another event with same refid had to be skipped  # noqa: E501
+            if skipped:  # add it also to skipped events if another event with same refid had to be skipped  # noqa: E501
                 with self.db.user_write() as write_cursor:
                     self.db.add_skipped_external_event(
                         write_cursor=write_cursor,
@@ -1220,10 +1245,14 @@ def history_event_from_kraken(
                         extra_data={'location_label': self.name},
                     )
                 continue
+            if event.event_type == HistoryEventType.TRADE and event.event_subtype == HistoryEventSubType.SPEND:  # noqa: E501
+                event.sequence_index = 0
+            elif event.event_type == HistoryEventType.TRADE and event.event_subtype == HistoryEventSubType.RECEIVE:  # noqa: E501
+                event.sequence_index = 1
 
             returned_events.append(event)
 
-        return returned_events, found_unknown_event
+        return returned_events, skipped, found_unknown_event
 
     def query_history_events(self) -> None:
         self.msg_aggregator.add_message(
@@ -1271,9 +1300,11 @@ def get_event_mappings() -> EventMappingType:
                 HistoryEventSubType.RECEIVE: EventCategory.RECEIVE,
             },
             HistoryEventType.WITHDRAWAL: {
+                HistoryEventSubType.NONE: EventCategory.WITHDRAW,
                 HistoryEventSubType.FEE: EventCategory.FEE,
             },
             HistoryEventType.DEPOSIT: {
+                HistoryEventSubType.NONE: EventCategory.DEPOSIT,
                 HistoryEventSubType.FEE: EventCategory.FEE,
             },
         }
diff --git a/rotkehlchen/history/skipped.py b/rotkehlchen/history/skipped.py
--- a/rotkehlchen/history/skipped.py
+++ b/rotkehlchen/history/skipped.py
@@ -104,21 +104,20 @@ def reprocess_skipped_external_events(rotki: 'Rotkehlchen') -> tuple[int, int]:
 
         total_num += len(raw_events)
         exchange = cast('Kraken', exchange)
-        processed_events = exchange.process_kraken_raw_events(
+        _, processed_refids = exchange.process_kraken_raw_events(
             events=[x[1] for x in raw_events],
             events_source='processing skipped events',
             save_skipped_events=False,
         )
-        processed_num += len(processed_events)
-
-        for processed_event in processed_events:
-            for identifier, raw_data in raw_events:
-                try:
-                    if raw_data['refid'] == processed_event.event_identifier:
-                        identifiers_to_delete.add(identifier)
-                except KeyError:
-                    log.error(f'Processing skipped kraken event could not find refid in {raw_data}')  # noqa: E501
-                    continue
+        processed_num = 0
+        for identifier, raw_data in raw_events:
+            try:
+                if raw_data['refid'] in processed_refids:
+                    identifiers_to_delete.add(identifier)
+                    processed_num += 1
+            except KeyError:  # should never really happen
+                log.error(f'Processing skipped kraken event could not find refid in {raw_data}')
+                continue
 
     if len(identifiers_to_delete) != 0:  # delete some skipped events if needed
         with rotki.data.db.user_write() as write_cursor:


Test patch:
diff --git a/rotkehlchen/tests/api/test_skipped_events.py b/rotkehlchen/tests/api/test_skipped_events.py
--- a/rotkehlchen/tests/api/test_skipped_events.py
+++ b/rotkehlchen/tests/api/test_skipped_events.py
@@ -16,20 +16,11 @@
     assert_simple_ok_response,
 )
 from rotkehlchen.tests.utils.exchanges import try_get_first_exchange
-from rotkehlchen.tests.utils.history import prices
 from rotkehlchen.tests.utils.kraken import KRAKEN_GENERAL_LEDGER_RESPONSE
 from rotkehlchen.types import Location
 
-TEST_PRICES = prices
-TEST_PRICES['NEWASSET'] = {
-    'USD': {
-        1636740198: ONE,
-        1640493374: ONE,
-    },
-}
-
 
-@pytest.mark.parametrize('mocked_price_queries', [prices])
+@pytest.mark.parametrize('default_mock_price_value', [ONE])
 @pytest.mark.parametrize('added_exchanges', [(Location.KRAKEN,)])
 def test_skipped_external_events(rotkehlchen_api_server_with_exchanges, globaldb, tmpdir_factory):
     server = rotkehlchen_api_server_with_exchanges
@@ -82,9 +73,20 @@ def test_skipped_external_events(rotkehlchen_api_server_with_exchanges, globaldb
             "amount": "0.0600000000",
             "fee": "0.0000000000",
             "balance": "0.0600000000"
+        },
+        "DDD": {
+            "refid": "ZZZZZZZZZ",
+            "time": 1636750198.9674,
+            "type": "staking",
+            "subtype": "stakingtospot",
+            "aclass": "currency",
+            "asset": "NEWASSET",
+            "amount": "0.0600000000",
+            "fee": "0.0000000000",
+            "balance": "0.0600000000"
         }
     },
-    "count": 4
+    "count": 5
     }
     """
     # Test that before populating we don't have any event
@@ -129,8 +131,8 @@ def test_skipped_external_events(rotkehlchen_api_server_with_exchanges, globaldb
         api_url_for(server, 'historyskippedexternaleventresource'),
     )
     result = assert_proper_response_with_result(response)
-    assert result['total'] == 5  # 5 is due to +2 unknown assets set up during fixture
-    assert result['locations']['kraken'] == 5
+    assert result['total'] == 6  # 5 is due to +2 unknown assets set up during fixture
+    assert result['locations']['kraken'] == 6
     assert len(result['locations']) == 1
 
     # now let's test the export skipped external events endpoint
@@ -154,7 +156,7 @@ def test_skipped_external_events(rotkehlchen_api_server_with_exchanges, globaldb
                 assert row['data'] in flat_general
             else:
                 assert row['data'] in flat_stake
-    assert count == 5
+    assert count == 6
 
     # now try to reprocess the skipped events after adding the asset to the kraken mapping
     globaldb.add_asset(CryptoAsset.initialize(
@@ -170,8 +172,8 @@ def test_skipped_external_events(rotkehlchen_api_server_with_exchanges, globaldb
         api_url_for(server, 'historyskippedexternaleventresource'),
     )
     result = assert_proper_response_with_result(response)
-    assert result['total'] == 5
-    assert result['successful'] == 3
+    assert result['total'] == 6
+    assert result['successful'] == 4
     response = requests.get(
         api_url_for(server, 'historyskippedexternaleventresource'),
     )
@@ -193,4 +195,4 @@ def test_skipped_external_events(rotkehlchen_api_server_with_exchanges, globaldb
     )
     result = assert_proper_response_with_result(response)
     events = result['entries']
-    assert len(events) == 4
+    assert len(events) == 3
diff --git a/rotkehlchen/tests/exchanges/test_kraken.py b/rotkehlchen/tests/exchanges/test_kraken.py
--- a/rotkehlchen/tests/exchanges/test_kraken.py
+++ b/rotkehlchen/tests/exchanges/test_kraken.py
@@ -10,6 +10,7 @@
 
 from rotkehlchen.accounting.mixins.event import AccountingEventType
 from rotkehlchen.accounting.structures.balance import Balance
+from rotkehlchen.accounting.structures.types import HistoryEventSubType, HistoryEventType
 from rotkehlchen.assets.asset import Asset
 from rotkehlchen.assets.converters import KRAKEN_TO_WORLD, asset_from_kraken
 from rotkehlchen.constants import ONE, ZERO
@@ -31,6 +32,8 @@
 )
 from rotkehlchen.constants.limits import FREE_HISTORY_EVENTS_LIMIT
 from rotkehlchen.constants.resolver import strethaddress_to_identifier
+from rotkehlchen.db.filtering import HistoryEventFilterQuery
+from rotkehlchen.db.history_events import DBHistoryEvents
 from rotkehlchen.db.settings import ModifiableDBSettings
 from rotkehlchen.errors.asset import UnknownAsset, UnprocessableTradePair
 from rotkehlchen.errors.serialization import DeserializationError
@@ -67,6 +70,18 @@
 from rotkehlchen.utils.misc import ts_now
 
 
+def _check_trade_history_events_order(db, expected):
+    """Check that the history events for the trades have the expected order"""
+    dbevents = DBHistoryEvents(db)
+    with db.conn.read_ctx() as cursor:
+        events = dbevents.get_history_events(cursor, HistoryEventFilterQuery.make(), True)
+        assert len(events) == len(expected)
+        for event in events:
+            assert event.sequence_index == expected[event.sequence_index][0]
+            assert event.event_type == expected[event.sequence_index][1]
+            assert event.event_subtype == expected[event.sequence_index][2]
+
+
 def test_name():
     exchange = Kraken('kraken1', 'a', b'a', object(), object())
     assert exchange.location == Location.KRAKEN
@@ -415,6 +430,11 @@ def test_kraken_trade_with_spend_receive(function_scope_kraken):
     assert trade.base_asset == A_ETH
     assert trade.quote_asset == A_EUR
     assert trade.fee == FVal(0.45)
+    _check_trade_history_events_order(kraken.db, {
+        0: (0, HistoryEventType.TRADE, HistoryEventSubType.SPEND),
+        1: (1, HistoryEventType.TRADE, HistoryEventSubType.RECEIVE),
+        2: (2, HistoryEventType.SPEND, HistoryEventSubType.FEE),
+    })
     errors = kraken.msg_aggregator.consume_errors()
     warnings = kraken.msg_aggregator.consume_warnings()
     assert len(errors) == 0
@@ -544,6 +564,51 @@ def test_kraken_trade_no_counterpart(function_scope_kraken):
     assert len(warnings) == 0
 
 
+@pytest.mark.parametrize('use_clean_caching_directory', [True])
+def test_kraken_failed_withdrawals(function_scope_kraken):
+    """Test that failed withdrawals are processed properly"""
+    kraken = function_scope_kraken
+    kraken.random_trade_data = False
+    kraken.random_ledgers_data = False
+    kraken.cache_ttl_secs = 0
+
+    test_events = """{
+        "ledger": {
+            "W1": {
+                "refid": "1",
+                "time": 1636406000.8555,
+                "type": "withdrawal",
+                "subtype": "",
+                "aclass": "currency",
+                "asset": "ZEUR",
+                "amount": "-1000.0",
+                "fee": "1.0",
+                "balance": "1"
+            },
+            "W2": {
+                "refid": "1",
+                "time": 1636508000.8555,
+                "type": "withdrawal",
+                "subtype": "",
+                "aclass": "currency",
+                "asset": "ZEUR",
+                "amount": "1000",
+                "fee": "-1.0",
+                "balance": "1"
+            }
+        },
+        "count": 2
+    }"""
+
+    target = 'rotkehlchen.tests.utils.kraken.KRAKEN_GENERAL_LEDGER_RESPONSE'
+    with patch(target, new=test_events):
+        withdrawals = kraken.query_online_deposits_withdrawals(
+            start_ts=0,
+            end_ts=Timestamp(1637406001),
+        )
+    assert len(withdrawals) == 0
+
+
 @pytest.mark.parametrize('use_clean_caching_directory', [True])
 def test_trade_from_kraken_unexpected_data(function_scope_kraken):
     """Test that getting unexpected data from kraken leads to skipping the trade
diff --git a/rotkehlchen/tests/utils/kraken.py b/rotkehlchen/tests/utils/kraken.py
--- a/rotkehlchen/tests/utils/kraken.py
+++ b/rotkehlchen/tests/utils/kraken.py
@@ -327,7 +327,7 @@ def get_random_kraken_asset() -> str:
 def generate_random_kraken_balance_response() -> dict[str, FVal]:
     kraken_assets = set(KRAKEN_TO_WORLD.keys()) - set(KRAKEN_DELISTED)
     number_of_assets = random.randrange(0, len(kraken_assets))
-    chosen_assets = random.sample(kraken_assets, number_of_assets)
+    chosen_assets = random.sample(list(kraken_assets), number_of_assets)
 
     balances = {}
     for asset in chosen_assets: