Repository: rotki/rotki

Issue:
Eth staking execution rewards are doubled in 1.32.0
## Problem Definition

The execution rewards in eth staking view after the 1.32.0 changes are 2 times the amount they should be.

## Task

Fix it.


Discussion:

Resolving patch:
diff --git a/rotkehlchen/chain/ethereum/modules/eth2/eth2.py b/rotkehlchen/chain/ethereum/modules/eth2/eth2.py
--- a/rotkehlchen/chain/ethereum/modules/eth2/eth2.py
+++ b/rotkehlchen/chain/ethereum/modules/eth2/eth2.py
@@ -163,7 +163,6 @@ def get_performance(
                 'entries_found': len(result['validators']),
             }
 
-        common_arguments: dict[str, Any] = {'from_ts': from_ts, 'to_ts': to_ts}
         index_to_pubkey, index_to_activation_ts, index_to_withdrawable_ts = {}, {}, {}
         dbeth2 = DBEth2(self.database)
         all_validator_indices = set()
@@ -201,14 +200,19 @@ def get_performance(
             to_query_indices = associated_indices if to_query_indices is None else to_query_indices | associated_indices  # noqa: E501
             # also add the to_filter_indices since this will enable deposit association
             # to validator index by address. We need to do this instead of adding
-            # addresses to common_arguments since then we would be ANDing the filters
+            # addresses to filter arguments since then we would be ANDing the filters
             # which would end up returning no values for many validators
             to_filter_indices = associated_indices if to_filter_indices is None else to_filter_indices | associated_indices  # noqa: E501
 
-        if to_filter_indices is not None:
-            common_arguments['validator_indices'] = list(to_filter_indices)
+        with self.database.conn.read_ctx() as cursor:
+            accounts = self.database.get_blockchain_accounts(cursor)
 
-        withdrawals_filter_query, exits_filter_query, execution_filter_query = create_profit_filter_queries(common_arguments)  # noqa: E501
+        withdrawals_filter_query, exits_filter_query, execution_filter_query = create_profit_filter_queries(  # noqa: E501
+            from_ts=from_ts,
+            to_ts=to_ts,
+            validator_indices=list(to_filter_indices) if to_filter_indices is not None else None,
+            tracked_addresses=accounts.eth,  # needed to exclude block recipients not tracked
+        )
 
         with self.database.conn.read_ctx() as cursor:
             withdrawals_amounts, exits_pnl, execution_rewards_amounts = dbeth2.get_validators_profit(  # noqa: E501
diff --git a/rotkehlchen/chain/ethereum/modules/eth2/utils.py b/rotkehlchen/chain/ethereum/modules/eth2/utils.py
--- a/rotkehlchen/chain/ethereum/modules/eth2/utils.py
+++ b/rotkehlchen/chain/ethereum/modules/eth2/utils.py
@@ -1,7 +1,7 @@
 import logging
 from collections.abc import Sequence
 from http import HTTPStatus
-from typing import Any, Literal
+from typing import Literal
 
 import gevent
 import requests
@@ -22,7 +22,7 @@
 from rotkehlchen.history.events.structures.base import HistoryBaseEntryType
 from rotkehlchen.history.events.structures.types import HistoryEventSubType, HistoryEventType
 from rotkehlchen.logging import RotkehlchenLogsAdapter
-from rotkehlchen.types import Eth2PubKey, Timestamp
+from rotkehlchen.types import ChecksumEvmAddress, Eth2PubKey, Timestamp
 from rotkehlchen.utils.misc import create_timestamp, get_chunks
 
 from .structures import ValidatorDailyStats
@@ -220,26 +220,38 @@ def calculate_query_chunks(
     return list(get_chunks(indices_or_pubkeys, n=chunk_size))
 
 
-def create_profit_filter_queries(common_arguments: dict[str, Any]) -> tuple[EthWithdrawalFilterQuery, EthWithdrawalFilterQuery, EthStakingEventFilterQuery]:  # noqa: E501
+def create_profit_filter_queries(
+        from_ts: Timestamp,
+        to_ts: Timestamp,
+        validator_indices: list[int] | None,
+        tracked_addresses: Sequence[ChecksumEvmAddress],
+) -> tuple[EthWithdrawalFilterQuery, EthWithdrawalFilterQuery, EthStakingEventFilterQuery]:
     """Create the Filter queries for withdrawal events and execution layer reward events"""
     withdrawals_filter_query = EthWithdrawalFilterQuery.make(
-        **common_arguments,
+        from_ts=from_ts,
+        to_ts=to_ts,
+        validator_indices=validator_indices,
         event_types=[HistoryEventType.STAKING],
         event_subtypes=[HistoryEventSubType.REMOVE_ASSET],
         entry_types=IncludeExcludeFilterData(values=[HistoryBaseEntryType.ETH_WITHDRAWAL_EVENT]),
         withdrawal_types_filter=WithdrawalTypesFilter.ONLY_PARTIAL,
     )
     exits_filter_query = EthWithdrawalFilterQuery.make(
-        **common_arguments,
+        from_ts=from_ts,
+        to_ts=to_ts,
+        validator_indices=validator_indices,
         event_types=[HistoryEventType.STAKING],
         event_subtypes=[HistoryEventSubType.REMOVE_ASSET],
         entry_types=IncludeExcludeFilterData(values=[HistoryBaseEntryType.ETH_WITHDRAWAL_EVENT]),
         withdrawal_types_filter=WithdrawalTypesFilter.ONLY_EXITS,
     )
     execution_filter_query = EthStakingEventFilterQuery.make(
-        **common_arguments,
+        from_ts=from_ts,
+        to_ts=to_ts,
+        validator_indices=validator_indices,
         event_types=[HistoryEventType.STAKING],
         event_subtypes=[HistoryEventSubType.BLOCK_PRODUCTION, HistoryEventSubType.MEV_REWARD],
         entry_types=IncludeExcludeFilterData(values=[HistoryBaseEntryType.ETH_BLOCK_EVENT]),
+        location_labels=tracked_addresses,  # type: ignore  # addresses are strings
     )
     return withdrawals_filter_query, exits_filter_query, execution_filter_query


Test patch:
diff --git a/rotkehlchen/tests/api/test_eth2.py b/rotkehlchen/tests/api/test_eth2.py
--- a/rotkehlchen/tests/api/test_eth2.py
+++ b/rotkehlchen/tests/api/test_eth2.py
@@ -241,10 +241,11 @@ def test_eth2_daily_stats(rotkehlchen_api_server):
 @pytest.mark.parametrize('ethereum_accounts', [[
     '0x61874850cC138e5e198d5756cF70e6EFED6aD464',  # withdrawal address of detected validator
     '0xbfEC7fc8DaC449a482b593Eb0aE28CfeAb49902c',  # withdrawal address of exited validator
+    '0x4675C7e5BaAFBFFbca748158bEcBA61ef3b0a263',  # execution fee recipient for 432840
 ]])
 @pytest.mark.parametrize('start_with_valid_premium', [True])
 @pytest.mark.parametrize('ethereum_modules', [['eth2']])
-@pytest.mark.freeze_time('2024-02-08 23:25:00 GMT')
+@pytest.mark.freeze_time('2024-02-17 17:00:00 GMT')
 @pytest.mark.parametrize('network_mocking', [False])
 def test_staking_performance(rotkehlchen_api_server, ethereum_accounts):
     rotki = rotkehlchen_api_server.rest_api.rotkehlchen
@@ -277,7 +278,7 @@ def test_staking_performance(rotkehlchen_api_server, ethereum_accounts):
         withdrawable_timestamp=Timestamp(1706386007),
         status=ValidatorStatus.EXITING,  # since we don't have withdrawals information yet
     )
-    total_validators, active_validators, exited_validators = 402, 259, 143
+    total_validators, active_validators, exited_validators = 402, 258, 144
     response = requests.get(
         api_url_for(
             rotkehlchen_api_server,
@@ -325,27 +326,26 @@ def test_staking_performance(rotkehlchen_api_server, ethereum_accounts):
 
     expected_validators_result = {
         'sums': {
-            'apr': '0.0443931202201263006106868447783673046350110514302790176714219659866470139396912',  # noqa: E501
-            'execution': '0.951964836013963505',
+            'apr': '0.0390711733712776643632006946387784579525834493111298798058751595316961517731974',  # noqa: E501
+            'execution': '0.5135720417808183',
             'exits': '0.0014143880000005993',
-            'outstanding_consensus_pnl': '0.009991843',
-            'sum': '3.2443113160139639043',
-            'withdrawals': '2.2809402489999998',
+            'outstanding_consensus_pnl': '0.011461867',
+            'sum': '2.8251388897808186993',
+            'withdrawals': '2.2986905929999998',
         },
         'validators': {
             '432840': {
-                'apr': '0.0585589969328525179226399380937759635877614420402413033269515587183469999638769',  # noqa: E501
-                'execution': '0.93361811418473',
+                'apr': '0.0488237003112313830083592875139977603583426651735722284434490481522956326987681',  # noqa: E501
+                'execution': '0.5135720417808183',
                 'exits': '0.0014143880000005993',
-                'sum': '2.5266283731847305993',
+                'sum': '2.1065823007808188993',
                 'withdrawals': '1.591595871',
             },
             '624729': {
-                'apr': '0.0302272435074000832987337514629586456822606608203167320158923732549470279155054',  # noqa: E501
-                'execution': '0.018346721829233505',
-                'outstanding_consensus_pnl': '0.009991843',
-                'sum': '0.717682942829233305',
-                'withdrawals': '0.6893443779999998',
+                'apr': '0.0293186464313239457180421017635591555468242334486875311683012709110966708476267',  # noqa: E501
+                'outstanding_consensus_pnl': '0.011461867',
+                'sum': '0.7185565889999998',
+                'withdrawals': '0.7070947219999998',
             },
         },
         'entries_found': 2,
diff --git a/rotkehlchen/tests/unit/test_eth2.py b/rotkehlchen/tests/unit/test_eth2.py
--- a/rotkehlchen/tests/unit/test_eth2.py
+++ b/rotkehlchen/tests/unit/test_eth2.py
@@ -616,14 +616,14 @@ def test_deposits_pubkey_re(eth2: 'Eth2', database):
 
 
 @pytest.mark.parametrize('ethereum_accounts', [['0x0fdAe061cAE1Ad4Af83b27A96ba5496ca992139b', '0xF4fEae08C1Fa864B64024238E33Bfb4A3Ea7741d']])  # noqa: E501
-def test_eth_validators_performance(eth2, database):
+def test_eth_validators_performance(eth2, database, ethereum_accounts):
     """Test that the performance of all multiple validators is returned fine"""
     dbevents = DBHistoryEvents(database)
     dbeth2 = DBEth2(database)
     vindex1 = 45555
-    vindex1_address = string_to_evm_address('0x0fdAe061cAE1Ad4Af83b27A96ba5496ca992139b')
+    vindex1_address = ethereum_accounts[0]
     vindex2 = 114543
-    vindex2_address = string_to_evm_address('0xF4fEae08C1Fa864B64024238E33Bfb4A3Ea7741d')
+    vindex2_address = ethereum_accounts[1]
     mev_builder_address = string_to_evm_address('0x690B9A9E9aa1C9dB991C7721a92d351Db4FaC990')
     block_number = 15824493
     tx_hash = deserialize_evm_tx_hash('0x8d0969db1e536969ba2e29abf8e8945e4304d49ae14523b66cbe9be5d52df804')  # noqa: E501
@@ -654,7 +654,7 @@ def test_eth_validators_performance(eth2, database):
                 validator_index=vindex1,
                 timestamp=timestampms,
                 balance=Balance(block_reward_1),
-                fee_recipient=mev_builder_address,
+                fee_recipient=vindex1_address,
                 block_number=block_number,
                 is_mev_reward=False,
             ), EthBlockEvent(
@@ -667,7 +667,7 @@ def test_eth_validators_performance(eth2, database):
             ), EthBlockEvent(
                 validator_index=vindex2,
                 timestamp=TimestampMS(timestampms + (2 * (HOUR_IN_SECONDS * 1000))),
-                balance=Balance(block_reward_2),
+                balance=Balance(block_reward_2),  # since mev builder gets it we shouldn't count it
                 fee_recipient=mev_builder_address,
                 block_number=block_number + 1,
                 is_mev_reward=False,
@@ -733,14 +733,14 @@ def check_performance_validator(performance, vindex, check_keys, expected_data,
 
     assert set(performance.keys()) == {'sums', 'validators', 'entries_found', 'entries_total'}
     expected_sums = {
-        'execution': block_reward_1 + mev_reward_1 + block_reward_2 + mev_reward_2 + no_mev_block_reward_2,  # noqa: E501
+        'execution': block_reward_1 + mev_reward_1 + mev_reward_2 + no_mev_block_reward_2,
         'exits': exit_1 - 32,
-        'sum': block_reward_1 + mev_reward_1 + withdrawal_1 + (exit_1 - 32) + block_reward_2 + mev_reward_2 + no_mev_block_reward_2,   # noqa: E501
+        'sum': block_reward_1 + mev_reward_1 + withdrawal_1 + (exit_1 - 32) + mev_reward_2 + no_mev_block_reward_2,   # noqa: E501
         'withdrawals': withdrawal_1,
     }
     for check_key in ('execution', 'exits', 'sum', 'withdrawals'):
         assert performance['sums'][check_key] == expected_sums[check_key]
-    assert performance['sums']['apr'].is_close(FVal('0.00412822186909020241811060649139005241062636710576993131056417104116726771999922'))  # noqa: E501
+    assert performance['sums']['apr'].is_close(FVal('0.00404161581589250586388450985171054082159224751613839429006282479554837399160762'))  # noqa: E501
 
     assert set(performance['validators'].keys()) == {vindex1, vindex2}
     expected_vindex1 = {
@@ -750,10 +750,10 @@ def check_performance_validator(performance, vindex, check_keys, expected_data,
     }
     check_performance_validator(performance, vindex1, ('execution', 'exits', 'sum'), expected_vindex1, FVal('0.00588921161744336568737457149820678805432013209494451739409154470208477353062825'))  # noqa: E501
     expected_vindex2 = {
-        'execution': block_reward_2 + mev_reward_2 + no_mev_block_reward_2,
-        'sum': block_reward_2 + mev_reward_2 + no_mev_block_reward_2,
+        'execution': mev_reward_2 + no_mev_block_reward_2,
+        'sum': mev_reward_2 + no_mev_block_reward_2,
     }
-    check_performance_validator(performance, vindex2, ('execution', 'sum'), expected_vindex2, FVal('0.00236723212073703914884664148457331676693260211659534522703679738024976190937018'))  # noqa: E501
+    check_performance_validator(performance, vindex2, ('execution', 'sum'), expected_vindex2, FVal('0.00219402001434164604039444820521429358886436293733227118603410488901197445258700'))  # noqa: E501
 
     # Check pagination and that cache works
     performance = eth2.get_performance(from_ts=Timestamp(0), to_ts=Timestamp(1706866836), limit=1, offset=0, ignore_cache=False)  # noqa: E501
@@ -762,14 +762,21 @@ def check_performance_validator(performance, vindex, check_keys, expected_data,
 
     performance = eth2.get_performance(from_ts=Timestamp(0), to_ts=Timestamp(1706866836), limit=2, offset=1, ignore_cache=False)  # noqa: E501
     assert set(performance['validators'].keys()) == {vindex2}
-    check_performance_validator(performance, vindex2, ('execution', 'sum'), expected_vindex2, FVal('0.00236723212073703914884664148457331676693260211659534522703679738024976190937018'))  # noqa: E501
+    check_performance_validator(performance, vindex2, ('execution', 'sum'), expected_vindex2, FVal('0.00219402001434164604039444820521429358886436293733227118603410488901197445258700'))  # noqa: E501
+
+    # check that filtering by an unknown address returns nothing
+    performance = eth2.get_performance(from_ts=Timestamp(0), to_ts=Timestamp(1706866836), limit=10, offset=0, addresses=[make_evm_address()], ignore_cache=True)  # noqa: E501
+    assert performance['entries_found'] == 0
+    assert performance['entries_total'] == 2
+    assert performance['sums'] == {}
+    assert performance['validators'] == {}
 
 
 @pytest.mark.vcr()
 @pytest.mark.freeze_time('2024-02-02 13:34:45 GMT')
 @pytest.mark.parametrize('network_mocking', [False])
 @pytest.mark.parametrize('ethereum_accounts', [['0x0fdAe061cAE1Ad4Af83b27A96ba5496ca992139b', '0xF4fEae08C1Fa864B64024238E33Bfb4A3Ea7741d']])  # noqa: E501
-def test_eth_validators_performance_recent(eth2, database):
+def test_eth_validators_performance_recent(eth2, database, ethereum_accounts):
     """Test that performance to recent time also takes into account outstanding consensus pnl"""
     dbevents = DBHistoryEvents(database)
     dbeth2 = DBEth2(database)
@@ -792,7 +799,7 @@ def test_eth_validators_performance_recent(eth2, database):
                 validator_index=vindex1,
                 timestamp=TimestampMS(1666693607000),
                 balance=Balance(block_reward_1),
-                fee_recipient=string_to_evm_address('0x690B9A9E9aa1C9dB991C7721a92d351Db4FaC990'),
+                fee_recipient=ethereum_accounts[0],
                 block_number=15824493,
                 is_mev_reward=False,
             ),