Repository: rotki/rotki

Issue:
Support the new GLM token and GNT -> GLM migration
## Abstract

https://blog.golemproject.net/gnt-to-glm-migration/
Add some additional open nodes as sources for ethereum data
## Abstract

There is some more open nodes that we could utilize in Rotki. We should add them for further redundancy if the user does not run his own node.

## Task

- Add 1inchexchange's node: https://twitter.com/1inchExchange/status/1326494153874747393
- Add cloudflare: https://www.cloudflare.com/distributed-web-gateway/#ethereum-gateway
- And perhaps more from the list that @dmihal is [putting up](https://twitter.com/dmihal/status/1326520031379853313) at https://ethereumnodes.com/


Discussion:

Resolving patch:
diff --git a/rotkehlchen/assets/asset.py b/rotkehlchen/assets/asset.py
--- a/rotkehlchen/assets/asset.py
+++ b/rotkehlchen/assets/asset.py
@@ -124,6 +124,9 @@
     'FIL': 'FIL',
     'UNI': 'UNI',
     'YFI': 'YFI',
+    'ANT': 'ANT',
+    'KEEP': 'KEEP',
+    'TBTC': 'TBTC',
 }
 
 WORLD_TO_BINANCE = {
diff --git a/rotkehlchen/chain/ethereum/manager.py b/rotkehlchen/chain/ethereum/manager.py
--- a/rotkehlchen/chain/ethereum/manager.py
+++ b/rotkehlchen/chain/ethereum/manager.py
@@ -133,23 +133,40 @@ def _query_web3_get_logs(
     return events
 
 
+# TODO: Ideally all these should become configurable
+# Taking LINKPOOL out since it's just really too slow and seems to not
+# respond to the batched calls almost at all. Combined with web3.py retries
+# this makes the tokens balance queries super slow.
 OPEN_NODES = (
     NodeName.MYCRYPTO,
     NodeName.BLOCKSCOUT,
     NodeName.AVADO_POOL,
+    NodeName.ONEINCH,
+    NodeName.MYETHERWALLET,
+    # NodeName.LINKPOOL,
+    NodeName.CLOUDFLARE_ETH,
     NodeName.ETHERSCAN,
 )
 ETHEREUM_NODES_TO_CONNECT_AT_START = (
     NodeName.OWN,
     NodeName.MYCRYPTO,
     NodeName.BLOCKSCOUT,
+    NodeName.ONEINCH,
     NodeName.AVADO_POOL,
+    NodeName.ONEINCH,
+    NodeName.MYETHERWALLET,
+    # NodeName.LINKPOOL,
+    NodeName.CLOUDFLARE_ETH,
 )
 OPEN_NODES_WEIGHT_MAP = {  # Probability with which to select each node
-    NodeName.ETHERSCAN: 0.5,
-    NodeName.MYCRYPTO: 0.25,
-    NodeName.BLOCKSCOUT: 0.2,
+    NodeName.ETHERSCAN: 0.3,
+    NodeName.MYCRYPTO: 0.15,
+    NodeName.BLOCKSCOUT: 0.1,
     NodeName.AVADO_POOL: 0.05,
+    NodeName.ONEINCH: 0.15,
+    NodeName.MYETHERWALLET: 0.15,
+    # NodeName.LINKPOOL: 0.05,
+    NodeName.CLOUDFLARE_ETH: 0.1,
 }
 
 
@@ -355,7 +372,7 @@ def query(self, method: Callable, call_order: Sequence[NodeName], **kwargs: Any)
 
             try:
                 result = method(web3, **kwargs)
-            except (RemoteError, BlockchainQueryError, requests.exceptions.HTTPError) as e:
+            except (RemoteError, BlockchainQueryError, requests.exceptions.RequestException) as e:
                 log.warning(f'Failed to query {node} for {str(method)} due to {str(e)}')
                 # Catch all possible errors here and just try next node call
                 continue
@@ -394,7 +411,7 @@ def query_eth_highest_block(self) -> BlockNumber:
         eth_resp: Optional[Dict[str, str]]
         try:
             eth_resp = request_get_dict(url)
-        except (RemoteError, UnableToDecryptRemoteData, requests.exceptions.ReadTimeout):
+        except (RemoteError, UnableToDecryptRemoteData, requests.exceptions.RequestException):
             eth_resp = None
 
         block_number: Optional[int]
diff --git a/rotkehlchen/chain/ethereum/typing.py b/rotkehlchen/chain/ethereum/typing.py
--- a/rotkehlchen/chain/ethereum/typing.py
+++ b/rotkehlchen/chain/ethereum/typing.py
@@ -2,11 +2,20 @@
 
 
 class NodeName(Enum):
+    """Various node types
+
+    Some open nodes taken from here: https://ethereumnodes.com/
+    Related issue: https://github.com/rotki/rotki/issues/1716
+    """
     OWN = 0
     ETHERSCAN = 1
     MYCRYPTO = 2
     BLOCKSCOUT = 3
     AVADO_POOL = 4
+    ONEINCH = 5
+    MYETHERWALLET = 6
+    LINKPOOL = 7
+    CLOUDFLARE_ETH = 8
 
     def __str__(self) -> str:
         if self == NodeName.OWN:
@@ -19,6 +28,14 @@ def __str__(self) -> str:
             return 'blockscout'
         elif self == NodeName.AVADO_POOL:
             return 'avado pool'
+        elif self == NodeName.ONEINCH:
+            return '1inch'
+        elif self == NodeName.MYETHERWALLET:
+            return 'myetherwallet'
+        elif self == NodeName.LINKPOOL:
+            return 'linkpool'
+        elif self == NodeName.CLOUDFLARE_ETH:
+            return 'cloudflare-eth'
 
         raise RuntimeError(f'Corrupt value {self} for NodeName -- Should never happen')
 
@@ -33,5 +50,13 @@ def endpoint(self, own_rpc_endpoint: str) -> str:
             return 'https://mainnet-nethermind.blockscout.com/'
         elif self == NodeName.AVADO_POOL:
             return 'https://mainnet.eth.cloud.ava.do/'
+        elif self == NodeName.ONEINCH:
+            return 'https://web3.1inch.exchange'
+        elif self == NodeName.MYETHERWALLET:
+            return 'https://nodes.mewapi.io/rpc/eth'
+        elif self == NodeName.LINKPOOL:
+            return 'https://main-rpc.linkpool.io/'
+        elif self == NodeName.CLOUDFLARE_ETH:
+            return 'https://cloudflare-eth.com/'
 
         raise RuntimeError(f'Corrupt value {self} for NodeName -- Should never happen')


Test patch:
diff --git a/rotkehlchen/tests/exchanges/test_kraken.py b/rotkehlchen/tests/exchanges/test_kraken.py
--- a/rotkehlchen/tests/exchanges/test_kraken.py
+++ b/rotkehlchen/tests/exchanges/test_kraken.py
@@ -21,10 +21,8 @@ def test_name():
 
 
 def test_coverage_of_kraken_balances(kraken):
-    # Since 05/08/2019 Kraken removed all delisted assets from their public API
-    # query except for BSV. No idea why or why this incosistency.
     got_assets = set(kraken.api_query('Assets').keys())
-    expected_assets = (set(KRAKEN_TO_WORLD.keys()) - set(KRAKEN_DELISTED)).union({'BSV'})
+    expected_assets = (set(KRAKEN_TO_WORLD.keys()) - set(KRAKEN_DELISTED))
     # Ignore the staking assets from the got assets
     got_assets.remove('XTZ.S')
     got_assets.remove('DOT.S')
@@ -32,6 +30,13 @@ def test_coverage_of_kraken_balances(kraken):
     got_assets.remove('EUR.M')
     got_assets.remove('USD.M')
     got_assets.remove('XBT.M')
+    got_assets.remove('KSM.S')
+    got_assets.remove('EUR.HOLD')
+    got_assets.remove('FLOW.S')
+    got_assets.remove('FLOWH.S')
+    # Ignore the following assets as well
+    got_assets.remove('FLOW')
+    got_assets.remove('FLOWH')
 
     diff = expected_assets.symmetric_difference(got_assets)
     if len(diff) != 0:
diff --git a/rotkehlchen/tests/unit/test_assets.py b/rotkehlchen/tests/unit/test_assets.py
--- a/rotkehlchen/tests/unit/test_assets.py
+++ b/rotkehlchen/tests/unit/test_assets.py
@@ -134,6 +134,26 @@ def test_coingecko_identifiers_are_reachable():
         'SGN',
         'SOAR',
         'YUP',
+        'AC',
+        'APIS',
+        'BITPARK',
+        'CO2',
+        'DAN',
+        'DEC',
+        'DLT',
+        'DROP',
+        'ERD',
+        'ETBS',
+        'GEN',
+        'STP',
+        'SYNC',
+        'TBT',
+        'TNT',
+        'WIC',
+        'XCN',
+        'XTP',
+        'FREC',
+        'PTC',
     ]
     coingecko = Coingecko()
     all_coins = coingecko.all_coins()
@@ -182,7 +202,7 @@ def test_coingecko_identifiers_are_reachable():
 def test_assets_json_meta():
     """Test that all_assets.json md5 matches and that if md5 changes since last
     time then version is also bumped"""
-    last_meta = {'md5': 'e327a3f7ffa1435ffb0fc481ab6dc63d', 'version': 26}
+    last_meta = {'md5': 'c9b48b7fac39090dbb3fe6c51abbaae7', 'version': 27}
     data_dir = Path(__file__).resolve().parent.parent.parent / 'data'
     data_md5 = file_md5(data_dir / 'all_assets.json')
 
diff --git a/rotkehlchen/tests/unit/test_ethereum_manager.py b/rotkehlchen/tests/unit/test_ethereum_manager.py
--- a/rotkehlchen/tests/unit/test_ethereum_manager.py
+++ b/rotkehlchen/tests/unit/test_ethereum_manager.py
@@ -1,12 +1,21 @@
+import os
+
 import pytest
 
-from rotkehlchen.chain.ethereum.manager import NodeName
+from rotkehlchen.chain.ethereum.manager import (
+    ETHEREUM_NODES_TO_CONNECT_AT_START,
+    OPEN_NODES,
+    OPEN_NODES_WEIGHT_MAP,
+    NodeName,
+)
 from rotkehlchen.constants.ethereum import (
     ATOKEN_ABI,
     ERC20TOKEN_ABI,
     YEARN_YCRV_VAULT,
     ZERO_ADDRESS,
 )
+from rotkehlchen.constants.misc import ONE, ZERO
+from rotkehlchen.fval import FVal
 from rotkehlchen.tests.utils.checks import assert_serialized_dicts_equal
 from rotkehlchen.tests.utils.ethereum import (
     ETHEREUM_TEST_PARAMETERS,
@@ -55,8 +64,8 @@ def test_get_transaction_receipt(ethereum_manager, call_order, ethereum_manager_
 @pytest.mark.parametrize('ethrpc_endpoint,ethereum_manager_connect_at_start,call_order', [
     (
         '',
-        (NodeName.MYCRYPTO, NodeName.AVADO_POOL, NodeName.BLOCKSCOUT),
-        (NodeName.MYCRYPTO, NodeName.AVADO_POOL, NodeName.BLOCKSCOUT),
+        [x for x in OPEN_NODES if x != NodeName.ETHERSCAN],
+        [x for x in OPEN_NODES if x != NodeName.ETHERSCAN],
     ),
 ])
 def test_use_open_nodes(ethereum_manager, call_order, ethereum_manager_connect_at_start):
@@ -192,3 +201,30 @@ def test_get_log_and_receipt_etherscan_bad_tx_index(
         call_order=call_order,
     )
     assert all(x['transactionIndex'] == 0 for x in result['logs'])
+
+
+def test_nodes_weight_map():
+    """Test the weight map has no duplicates and adds to 100%"""
+    nodes_set = set()
+    total = ZERO
+    for node, value in OPEN_NODES_WEIGHT_MAP.items():
+        assert node not in nodes_set, f'node {str(node)} appears more than once'
+        nodes_set.add(node)
+        total += FVal(value)
+
+    assert total == ONE
+
+
+def test_nodes_sets():
+    """Test that all nodes sets contain the nodes they should"""
+    assert set(OPEN_NODES) - set({NodeName.ETHERSCAN}) == set(ETHEREUM_NODES_TO_CONNECT_AT_START) - set({NodeName.OWN})  # noqa: E501
+    assert set(OPEN_NODES_WEIGHT_MAP.keys()) - set({NodeName.ETHERSCAN}) == set(ETHEREUM_NODES_TO_CONNECT_AT_START) - set({NodeName.OWN})  # noqa: E501
+
+
+@pytest.mark.skipif(
+    'CI' in os.environ,
+    reason='This test is only for us to figure out the speed of the open nodes',
+)
+def test_nodes_speed():
+    """TODO"""
+    pass