Repository: rotki/rotki

Issue:
API request failed
## Problem Definition

I upgraded to version 1.2.0 and all my accounts from before were gone. I got 2 errors from the notification panel:

- Querying blockchain balances died because of: Etherscan returned error response: {'status': FVal(0.0), 'message': 'NOTOK', 'result': 'Maximum rate limit reached'}. Check the logs for more details.

- Task for startup of makerdao died with exception: Etherscan returned error response: {'status': FVal(0.0), 'message': 'NOTOK', 'result': 'Maximum rate limit reached'}. Check the logs for more details

I tried to add one account back manually and got this error:

- Operation failed due to Etherscan API request failed due to ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))

Plus this new error appears (note that I don't use cryptocompare and didn't add an API key for it):

- The backend query task died unexpectedly: HTTPSConnectionPool(host='min-api.cryptocompare.com', port=443): Read timed out. (read timeout=None)

## Relevant logs

08/03/2020 20:45:51 CET -- ERROR:rotkehlchen.greenlets:Task for startup of makerdao died with exception: Etherscan returned error response: {'status': FVal(0.0), 'message': 'NOTOK', 'result': 'Maximum rate limit reached'}.
Exception Name: <class 'rotkehlchen.errors.RemoteError'>
Exception Info: Etherscan returned error response: {'status': FVal(0.0), 'message': 'NOTOK', 'result': 'Maximum rate limit reached'}
Traceback:
   File "src/gevent/greenlet.py", line 818, in gevent._greenlet.Greenlet.run
  File "rotkehlchen/makerdao.py", line 556, in on_startup
  File "rotkehlchen/makerdao.py", line 387, in get_historical_dsr
  File "rotkehlchen/makerdao.py", line 337, in _historical_dsr_for_account
  File "rotkehlchen/makerdao.py", line 224, in _get_vat_move_event_value
  File "rotkehlchen/chain/ethereum.py", line 449, in get_logs
  File "rotkehlchen/externalapis/etherscan.py", line 436, in get_logs
  File "rotkehlchen/externalapis/etherscan.py", line 212, in _query

08/03/2020 22:07:26 CET -- ERROR:rotkehlchen.api.rest:Greenlet for task 3 dies with exception: HTTPSConnectionPool(host='min-api.cryptocompare.com', port=443): Read timed out. (read timeout=None).
Exception Name: <class 'requests.exceptions.ReadTimeout'>
Exception Info: HTTPSConnectionPool(host='min-api.cryptocompare.com', port=443): Read timed out. (read timeout=None)
Traceback:
   File "src/gevent/greenlet.py", line 818, in gevent._greenlet.Greenlet.run
  File "rotkehlchen/api/rest.py", line 206, in _do_query_async
  File "rotkehlchen/api/rest.py", line 415, in _query_exchange_balances
  File "rotkehlchen/utils/interfaces.py", line 110, in wrapper
  File "rotkehlchen/utils/interfaces.py", line 69, in wrapper
  File "rotkehlchen/exchanges/kraken.py", line 424, in query_balances
  File "rotkehlchen/inquirer.py", line 122, in find_usd_price
  File "rotkehlchen/externalapis/cryptocompare.py", line 237, in query_endpoint_price
  File "rotkehlchen/externalapis/cryptocompare.py", line 153, in _api_query
  File "requests/sessions.py", line 537, in get
  File "requests/sessions.py", line 524, in request
  File "requests/sessions.py", line 637, in send
  File "requests/adapters.py", line 529, in send


Discussion:

Resolving patch:
diff --git a/rotkehlchen/assets/asset.py b/rotkehlchen/assets/asset.py
--- a/rotkehlchen/assets/asset.py
+++ b/rotkehlchen/assets/asset.py
@@ -111,6 +111,7 @@
     'SC': 'SC',
     'PAXG': 'PAXG',
     'LSK': 'LSK',
+    'TRX': 'TRX',
 }
 
 WORLD_TO_BINANCE = {
diff --git a/rotkehlchen/externalapis/etherscan.py b/rotkehlchen/externalapis/etherscan.py
--- a/rotkehlchen/externalapis/etherscan.py
+++ b/rotkehlchen/externalapis/etherscan.py
@@ -2,6 +2,7 @@
 from json.decoder import JSONDecodeError
 from typing import Any, Dict, List, Optional, Union, overload
 
+import gevent
 import requests
 from eth_utils.address import to_checksum_address
 from typing_extensions import Literal
@@ -166,54 +167,86 @@ def _query(  # noqa: F811
 
         logger.debug(f'Querying etherscan: {query_str}')
 
-        try:
-            response = self.session.get(query_str)
-        except requests.exceptions.ConnectionError as e:
-            raise RemoteError(f'Etherscan API request failed due to {str(e)}')
+        backoff = 1
+        backoff_limit = 13
+        while backoff < backoff_limit:
+            try:
+                response = self.session.get(query_str)
+            except requests.exceptions.ConnectionError as e:
+                if 'Max retries exceeded with url' in str(e):
+                    log.debug(
+                        f'Got max retries exceeded from etherscan. Will '
+                        f'backoff for {backoff} seconds.',
+                    )
+                    gevent.sleep(backoff)
+                    backoff = backoff * 2
+                    if backoff >= backoff_limit:
+                        raise RemoteError(
+                            'Getting Etherscan max connections error even '
+                            'after we incrementally backed off',
+                        )
+                    continue
+
+                raise RemoteError(f'Etherscan API request failed due to {str(e)}')
+
+            if response.status_code != 200:
+                raise RemoteError(
+                    f'Etherscan API request {response.url} failed '
+                    f'with HTTP status code {response.status_code} and text '
+                    f'{response.text}',
+                )
 
-        if response.status_code != 200:
-            raise RemoteError(
-                f'Etherscan API request {response.url} failed '
-                f'with HTTP status code {response.status_code} and text '
-                f'{response.text}',
-            )
+            try:
+                json_ret = rlk_jsonloads_dict(response.text)
+            except JSONDecodeError:
+                raise RemoteError(f'Etherscan returned invalid JSON response: {response.text}')
 
-        try:
-            json_ret = rlk_jsonloads_dict(response.text)
-        except JSONDecodeError:
-            raise RemoteError(f'Etherscan returned invalid JSON response: {response.text}')
+            try:
+                result = json_ret['result']
+                if 'status' not in json_ret:
+                    # sucessful proxy calls do not include a status
+                    status = 1
+                else:
+                    status = json_ret['status'].to_int(exact=True)
+
+                if status != 1:
+                    if status == 0 and result == 'Maximum rate limit reached':
+                        log.debug(
+                            f'Got response: {response.text} from etherscan. Will '
+                            f'backoff for {backoff} seconds.',
+                        )
+                        gevent.sleep(backoff)
+                        backoff = backoff * 2
+                        if backoff >= backoff_limit:
+                            raise RemoteError(
+                                'Etherscan keeps returning rate limit errors even '
+                                'after we incrementally backed off',
+                            )
+                        continue
+
+                    transaction_endpoint_and_none_found = (
+                        status == 0 and
+                        json_ret['message'] == 'No transactions found' and
+                        'txlist' in action
+                    )
+                    logs_endpoint_and_none_found = (
+                        status == 0 and
+                        json_ret['message'] == 'No records found' and
+                        'getLogs' in action
+                    )
+                    if transaction_endpoint_and_none_found or logs_endpoint_and_none_found:
+                        # Can't realize that result is always a list here so we ignore mypy warning
+                        return []  # type: ignore
 
-        # TODO: Handle the errors mentioned here:
-        # https://medium.com/etherscan-blog/psa-for-developers-implementation-of-api-key-requirements-starting-from-february-15th-2020-b616870f3746
-        try:
-            result = json_ret['result']
-            if module == 'proxy':
-                # proxy calls do not include a status
-                status = 1
-            else:
-                status = json_ret['status']
-
-            if status != 1:
-                transaction_endpoint_and_none_found = (
-                    status == 0 and
-                    json_ret['message'] == 'No transactions found' and
-                    'txlist' in action
-                )
-                logs_endpoint_and_none_found = (
-                    status == 0 and
-                    json_ret['message'] == 'No records found' and
-                    'getLogs' in action
+                    # else
+                    raise RemoteError(f'Etherscan returned error response: {json_ret}')
+            except KeyError as e:
+                raise RemoteError(
+                    f'Unexpected format of Etherscan response. Missing key entry for {str(e)}',
                 )
-                if transaction_endpoint_and_none_found or logs_endpoint_and_none_found:
-                    # Can't realize that result is always a list here so we ignore mypy warning
-                    return []  # type: ignore
 
-                # else
-                raise RemoteError(f'Etherscan returned error response: {json_ret}')
-        except KeyError as e:
-            raise RemoteError(
-                f'Unexpected format of Etherscan response. Missing key entry for {str(e)}',
-            )
+            # success, break out of the loop and return result
+            return result
 
         return result
 


Test patch:
diff --git a/rotkehlchen/tests/external_apis/test_etherscan.py b/rotkehlchen/tests/external_apis/test_etherscan.py
new file mode 100644
--- /dev/null
+++ b/rotkehlchen/tests/external_apis/test_etherscan.py
@@ -0,0 +1,71 @@
+import os
+import sys
+import traceback
+
+import gevent
+import pytest
+
+from rotkehlchen.db.dbhandler import DBHandler
+from rotkehlchen.externalapis.etherscan import Etherscan
+from rotkehlchen.typing import ExternalService, ExternalServiceApiCredentials
+from rotkehlchen.user_messages import MessagesAggregator
+
+
+@pytest.fixture(scope='function')
+def temp_etherscan(database, inquirer, function_scope_messages_aggregator, tmpdir_factory):
+    api_key = os.environ.get('ETHERSCAN_API_KEY', None)
+    if not api_key:
+        pytest.fail('No ETHERSCAN_API_KEY environment variable found.')
+    directory = tmpdir_factory.mktemp('data')
+    msg_aggregator = MessagesAggregator()
+    db = DBHandler(user_data_dir=directory, password='123', msg_aggregator=msg_aggregator)
+    db.add_external_service_credentials(credentials=[
+        ExternalServiceApiCredentials(service=ExternalService.ETHERSCAN, api_key=api_key),
+    ])
+    etherscan = Etherscan(database=db, msg_aggregator=msg_aggregator)
+    return etherscan
+
+
+def _handle_killed_greenlets(greenlet: gevent.Greenlet) -> None:
+
+    tb = ''.join(traceback.format_tb(greenlet.exc_info[2]))
+    message = ('Greenlet died with exception: {}.\n'
+               'Exception Name: {}\nException Info: {}\nTraceback:\n {}'
+               .format(
+                   greenlet.exception,
+                   greenlet.exc_info[0],
+                   greenlet.exc_info[1],
+                   tb,
+               ))
+
+    print(message)
+    sys.exit(1)
+
+
+@pytest.mark.skipif(
+    'TRAVIS' in os.environ,
+    reason='no real etherscan tests in Travis yet due to API key',
+)
+def test_maximum_rate_limit_reached(temp_etherscan):
+    """
+    Test that we can handle etherscan's rate limit repsponse properly
+
+    Regression test for https://github.com/rotki/rotki/issues/772"
+    """
+    etherscan = temp_etherscan
+
+    # Spam with concurrent requests for a bit. This triggers the problem
+    count = 200
+    while count > 0:
+        greenlet = gevent.spawn(
+            etherscan.get_account_balance,
+            '0x25a63509FEF5D23FF226eb8004A3c1458D6F3AB8')
+        greenlet.link_exception(_handle_killed_greenlets)
+        greenlet = gevent.spawn(
+            etherscan.eth_call,
+            '0x4678f0a6958e4D2Bc4F1BAF7Bc52E8F3564f3fE4',
+            '0xc455279100000000000000000000000027a2eaaa8bebea8d23db486fb49627c165baacb5',
+        )
+        greenlet.link_exception(_handle_killed_greenlets)
+        gevent.sleep(0.001)
+        count -= 1