Repository: rotki/rotki

Issue:
Unhandled exception in web3 contract call
## Problem Definition

Seen only once while using v1.17.1

```
Traceback (most recent call last):
  File "site-packages/web3/contract.py", line 1507, in call_contract_function
  File "site-packages/eth_abi/codec.py", line 181, in decode_abi
  File "site-packages/eth_abi/decoding.py", line 127, in __call__
  File "site-packages/eth_utils/functional.py", line 45, in inner
  File "site-packages/eth_abi/decoding.py", line 173, in decode
  File "site-packages/eth_abi/decoding.py", line 127, in __call__
  File "site-packages/eth_abi/decoding.py", line 142, in decode
  File "site-packages/eth_abi/decoding.py", line 127, in __call__
  File "site-packages/eth_abi/decoding.py", line 198, in decode
  File "site-packages/eth_abi/decoding.py", line 308, in read_data_from_stream
eth_abi.exceptions.InsufficientDataBytes: Tried to read 32 bytes.  Only got 0 bytes

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "src/gevent/greenlet.py", line 906, in gevent._gevent_cgreenlet.Greenlet.run
  File "rotkehlchen/api/rest.py", line 271, in _do_query_async
  File "rotkehlchen/api/rest.py", line 638, in _query_blockchain_balances
  File "rotkehlchen/utils/mixins/lockable.py", line 40, in wrapper
  File "rotkehlchen/utils/mixins/cacheable.py", line 73, in wrapper
  File "rotkehlchen/chain/manager.py", line 482, in query_balances
  File "rotkehlchen/utils/mixins/lockable.py", line 40, in wrapper
  File "rotkehlchen/utils/mixins/cacheable.py", line 73, in wrapper
  File "rotkehlchen/chain/manager.py", line 1062, in query_ethereum_balances
  File "rotkehlchen/chain/manager.py", line 1031, in query_defi_balances
  File "rotkehlchen/chain/ethereum/defi/chad.py", line 30, in query_defi_balances
  File "rotkehlchen/chain/ethereum/defi/zerionsdk.py", line 264, in all_balances_for_account
  File "rotkehlchen/chain/ethereum/defi/zerionsdk.py", line 251, in _query_chain_for_all_balances
  File "rotkehlchen/chain/ethereum/contracts.py", line 33, in call
  File "rotkehlchen/chain/ethereum/manager.py", line 755, in call_contract
  File "rotkehlchen/chain/ethereum/manager.py", line 397, in query
  File "rotkehlchen/chain/ethereum/manager.py", line 784, in _call_contract
  File "site-packages/web3/contract.py", line 1449, in call_function
  File "site-packages/web3/contract.py", line 964, in call
  File "site-packages/web3/contract.py", line 1528, in call_contract_function
web3.exceptions.BadFunctionCallOutput: Could not decode contract function call getProtocolBalances return data b'' for output_types ['((string,string,string,string,uint256),((address,string),(((address,string,string,uint8),uint256),((address,string,string,uint8),uint256)[])[])[])[]']
2021-06-04T07:35:57Z <Greenlet at 0x7fc2329ca950: <bound method RestAPI._do_query_async of <rotkehlchen.api.rest.RestAPI object at 0x7fc2338a4a10>>('_query_blockchain_balances', 6, blockchain=<SupportedBlockchain.ETHEREUM: 'ETH'>, ignore_cache=False)> failed with BadFunctionCallOutput
```

## Task

Figure out why this may happen and handle more gracefully


Discussion:

Resolving patch:
diff --git a/rotkehlchen/chain/ethereum/manager.py b/rotkehlchen/chain/ethereum/manager.py
--- a/rotkehlchen/chain/ethereum/manager.py
+++ b/rotkehlchen/chain/ethereum/manager.py
@@ -17,6 +17,7 @@
 from web3._utils.contracts import find_matching_event_abi
 from web3._utils.filters import construct_event_filter_params
 from web3.datastructures import MutableAttributeDict
+from web3.exceptions import BadFunctionCallOutput
 from web3.middleware.exception_retry_request import http_retry_request_middleware
 from web3.types import FilterParams
 
@@ -782,7 +783,7 @@ def _call_contract(
         try:
             method = getattr(contract.caller, method_name)
             result = method(*arguments if arguments else [])
-        except ValueError as e:
+        except (ValueError, BadFunctionCallOutput) as e:
             raise BlockchainQueryError(
                 f'Error doing call on contract {contract_address}: {str(e)}',
             ) from e
diff --git a/rotkehlchen/exchanges/poloniex.py b/rotkehlchen/exchanges/poloniex.py
--- a/rotkehlchen/exchanges/poloniex.py
+++ b/rotkehlchen/exchanges/poloniex.py
@@ -3,21 +3,21 @@
 import hmac
 import logging
 import os
+from collections import defaultdict
 from json.decoder import JSONDecodeError
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, overload
+from typing import TYPE_CHECKING, Any, DefaultDict, Dict, List, Optional, Tuple, Union
 from urllib.parse import urlencode
 
 import gevent
 import requests
 from gevent.lock import Semaphore
-from typing_extensions import Literal
 
 from rotkehlchen.accounting.structures import Balance
 from rotkehlchen.assets.asset import Asset
 from rotkehlchen.assets.converters import asset_from_poloniex
 from rotkehlchen.constants.assets import A_LEND
 from rotkehlchen.constants.misc import ZERO
-from rotkehlchen.constants.timing import QUERY_RETRY_TIMES, DEFAULT_TIMEOUT_TUPLE
+from rotkehlchen.constants.timing import DEFAULT_TIMEOUT_TUPLE, QUERY_RETRY_TIMES
 from rotkehlchen.errors import (
     DeserializationError,
     RemoteError,
@@ -269,12 +269,18 @@ def validate_api_key(self) -> Tuple[bool, str]:
 
     def api_query_dict(self, command: str, req: Optional[Dict] = None) -> Dict:
         result = self._api_query(command, req)
-        assert isinstance(result, Dict)  # pylint: disable=isinstance-second-argument-not-valid-type  # noqa: E501
+        if not isinstance(result, Dict):
+            raise RemoteError(
+                f'Poloniex query for {command} did not return a dict result. Result: {result}',
+            )
         return result
 
     def api_query_list(self, command: str, req: Optional[Dict] = None) -> List:
         result = self._api_query(command, req)
-        assert isinstance(result, List)  # pylint: disable=isinstance-second-argument-not-valid-type  # noqa: E501
+        if not isinstance(result, List):
+            raise RemoteError(
+                f'Poloniex query for {command} did not return a list result. Result: {result}',
+            )
         return result
 
     def _single_query(self, command: str, req: Dict[str, Any]) -> Optional[requests.Response]:
@@ -414,39 +420,64 @@ def return_lending_history(
         response = self.api_query_list('returnLendingHistory', req)
         return response
 
-    @overload
-    def return_trade_history(  # pylint: disable=unused-argument, no-self-use
-            self,
-            currency_pair: Literal['all'],
-            start: Timestamp,
-            end: Timestamp,
-    ) -> Dict:
-        ...
-
-    @overload
-    def return_trade_history(  # pylint: disable=unused-argument, no-self-use
-            self,
-            currency_pair: Union[TradePair, str],
-            start: Timestamp,
-            end: Timestamp,
-    ) -> Union[Dict, List]:
-        ...
-
     def return_trade_history(
             self,
-            currency_pair: Union[TradePair, str],
             start: Timestamp,
             end: Timestamp,
-    ) -> Union[Dict, List]:
+    ) -> Dict[str, List[Dict[str, Any]]]:
         """If `currency_pair` is all, then it returns a dictionary with each key
         being a pair and each value a list of trades. If `currency_pair` is a specific
         pair then a list is returned"""
-        return self._api_query('returnTradeHistory', {
-            'currencyPair': currency_pair,
-            'start': start,
-            'end': end,
-            'limit': 10000,
-        })
+        limit = 10000
+        pair = 'all'
+        data: DefaultDict[str, List[Dict[str, Any]]] = defaultdict(list)
+        while True:
+            new_data = self.api_query_dict('returnTradeHistory', {
+                'currencyPair': pair,
+                'start': start,
+                'end': end,
+                'limit': limit,
+            })
+            results_length = 0
+            for _, v in new_data.items():
+                results_length += len(v)
+
+            if data == {} and results_length < limit:
+                return new_data  # simple case - only one query needed
+
+            latest_ts = start
+            # add results to data and prepare for next query
+            for market, trades in new_data.items():
+                existing_ids = {x['globalTradeID'] for x in data['market']}
+                for trade in trades:
+                    try:
+                        timestamp = deserialize_timestamp_from_poloniex_date(trade['date'])
+                        latest_ts = max(latest_ts, timestamp)
+                        # since we query again from last ts seen make sure no duplicates make it in
+                        if trade['globalTradeID'] not in existing_ids:
+                            data[market].append(trade)
+                    except (DeserializationError, KeyError) as e:
+                        msg = str(e)
+                        if isinstance(e, KeyError):
+                            msg = f'Missing key entry for {msg}.'
+                        self.msg_aggregator.add_warning(
+                            'Error deserializing a poloniex trade. Check the logs for details',
+                        )
+                        log.error(
+                            'Error deserializing poloniex trade',
+                            trade=trade,
+                            error=msg,
+                        )
+                        continue
+
+            if results_length < limit:
+                break  # last query has less than limit. We are done.
+
+            # otherwise we query again from the last ts seen in the last result
+            start = latest_ts
+            continue
+
+        return data
 
     def return_deposits_withdrawals(
             self,
@@ -544,7 +575,6 @@ def query_online_trade_history(
             end_ts: Timestamp,
     ) -> List[Trade]:
         raw_data = self.return_trade_history(
-            currency_pair='all',
             start=start_ts,
             end=end_ts,
         )
@@ -554,13 +584,6 @@ def query_online_trade_history(
             results_length += len(v)
 
         log.debug('Poloniex trade history query', results_num=results_length)
-
-        if results_length >= 10000:
-            raise ValueError(
-                'Poloniex api has a 10k limit to trade history. Have not implemented'
-                ' a solution for more than 10k trades at the moment',
-            )
-
         our_trades = []
         for pair, trades in raw_data.items():
             for trade in trades:
@@ -767,7 +790,7 @@ def _deserialize_asset_movement(
             )
             log.error(
                 f'Unexpected data encountered during deserialization of poloniex '
-                f'{str(movement_type)}: {movement_data}. Error was: {str(e)}',
+                f'{str(movement_type)}: {movement_data}. Error was: {msg}',
             )
 
         return None


Test patch: