Repository: rotki/rotki

Issue:
Make Rotki pull changes to supported assets directly from Github
## Problem Definition

Users are annoyed that they have to wait for a new release for new tokens to be added, or for fixes that have to do with a token upgrade or what not.

It would be nice to offer a way to pull these updates directly from the source, that is Github.

## Specification

- [ ] Next to all_assets.json we will also be saving its md5hash in a file.
- [ ] When rotki starts a greenlet will run on the background that will pull the latest md5 hash of `all_assets.json` from github's develop branch and compare it with the one saved locally. If it's the same do nothing. If not pull the new all_assets.json and save it locally.
- [ ] Write tests that the all_assets md5 hash exists and that it's indeed the md5hash of the commited file.
- [ ] Write tests for the pulling behaviour


Discussion:

Resolving patch:
diff --git a/rotkehlchen/assets/resolver.py b/rotkehlchen/assets/resolver.py
--- a/rotkehlchen/assets/resolver.py
+++ b/rotkehlchen/assets/resolver.py
@@ -1,9 +1,15 @@
 import json
-import os
-from typing import Any, Dict, List, Optional
+import logging
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple
 
+import requests
+
+from rotkehlchen.errors import RemoteError
 from rotkehlchen.typing import AssetData, AssetType, ChecksumEthAddress, EthTokenInfo
 
+log = logging.getLogger(__name__)
+
 asset_type_mapping = {
     'fiat': AssetType.FIAT,
     'own chain': AssetType.OWN_CHAIN,
@@ -31,22 +37,132 @@
 }
 
 
+def _get_latest_assets(data_directory: Path) -> Dict[str, Any]:
+    """Gets the latest assets either locally or from the remote
+
+    Checks the remote (github) and if there is a newer file there it pulls it,
+    saves it and its md5 hash locally and returns the new assets.
+
+    If there is no new file (same hash) or if there is any problem contacting the remote
+    then the builtin assets file is used.
+    """
+    root_dir = Path(__file__).resolve().parent.parent
+    our_downloaded_meta = data_directory / 'assets' / 'all_assets.meta'
+    our_builtin_meta = root_dir / 'data' / 'all_assets.meta'
+    try:
+        response = requests.get('https://raw.githubusercontent.com/rotki/rotki/develop/rotkehlchen/data/all_assets.meta')  # noqa: E501
+        remote_meta = response.json()
+        if our_downloaded_meta.is_file():
+            local_meta_file = our_downloaded_meta
+        else:
+            local_meta_file = our_builtin_meta
+
+        with open(local_meta_file, 'r') as f:
+            local_meta = json.loads(f.read())
+
+        if local_meta['version'] < remote_meta['version']:
+            # we need to download and save the new assets from github
+            response = requests.get('https://raw.githubusercontent.com/rotki/rotki/develop/rotkehlchen/data/all_assets.json')  # noqa: E501
+            remote_asset_data = response.text
+
+            # Make sure directory exists
+            (data_directory / 'assets').mkdir(parents=True, exist_ok=True)
+            # Write the files
+            with open(data_directory / 'assets' / 'all_assets.meta', 'w') as f:
+                f.write(json.dumps(remote_meta))
+            with open(data_directory / 'assets' / 'all_assets.json', 'w') as f:
+                f.write(remote_asset_data)
+
+            log.info(
+                f'Found newer remote assets file with version: {remote_meta["version"]} '
+                f' and {remote_meta["md5"]} md5 hash. Replaced local file',
+            )
+            return json.loads(remote_asset_data)
+
+        # else, same as all error cases use the current one
+    except (RemoteError, KeyError, json.decoder.JSONDecodeError):
+        pass
+
+    if our_downloaded_meta.is_file():
+        assets_file = data_directory / 'assets' / 'all_assets.json'
+    else:
+        assets_file = root_dir / 'data' / 'all_assets.json'
+
+    with open(assets_file, 'r') as f:
+        return json.loads(f.read())
+
+
+def _attempt_initialization(
+        data_directory: Optional[Path],
+        saved_assets: Optional[Dict[str, Any]],
+) -> Tuple[Dict[str, Any], bool]:
+    """Reads the asset data either from builtin data or from the remote
+
+    1. If it's the very first time data is initialized (and data directory is not given)
+    then just get assets from the builtin file
+    2. If data directory is still not given but we have some saved assets return them directly
+    3. If data directory is given then we can finally do the comparison of local
+    saved and builtin file with the remote and return the most recent assets.
+
+    Returns a tuple of the most recent assets mapping it can get and a boolean denoting
+    if the remote check happened or not. If it did then we havethe most recent assets.
+    """
+    if not data_directory:
+        if saved_assets is not None:
+            # Do not read the all_assets file again if it has been already read
+            return saved_assets, False
+
+        # First initialization. Read the builtin all_assets.json
+        root_dir = Path(__file__).resolve().parent.parent
+        with open(root_dir / 'data' / 'all_assets.json', 'r') as f:
+            assets = json.loads(f.read())
+
+        return assets, False
+
+    # else we got the data directory so we can finally do the remote check
+    assets = _get_latest_assets(data_directory)
+    return assets, True
+
+
 class AssetResolver():
     __instance = None
+    remote_check_happened: bool = False
     assets: Dict[str, Dict[str, Any]] = {}
     eth_token_info: Optional[List[EthTokenInfo]] = None
 
-    def __new__(cls) -> 'AssetResolver':
-        if AssetResolver.__instance is not None:
-            return AssetResolver.__instance  # type: ignore
+    def __new__(
+            cls,
+            data_directory: Path = None,
+    ) -> 'AssetResolver':
+        """Lazily initializes AssetResolver
 
-        AssetResolver.__instance = object.__new__(cls)
+        As long as it's called without a data_directory path it uses the builtin
+        all_assets file. Once a data directory is given it attempts to see if a
+        newer file exists on the remote and uses that. Once that's done the remote
+        check flag is set to True.
 
-        dir_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
-        with open(os.path.join(dir_path, 'data', 'all_assets.json'), 'r') as f:
-            assets = json.loads(f.read())
+        From that point on all calls to AssetResolver() return the same data.
+        """
+        if AssetResolver.__instance is not None:
+            if AssetResolver.__instance.remote_check_happened:  # type: ignore
+                return AssetResolver.__instance
+
+            # else we still have not performed the remote check
+            assets, check_happened = _attempt_initialization(
+                data_directory=data_directory,
+                saved_assets=AssetResolver.__instance.assets,
+            )
+        else:
+            # first initialization
+            assets, check_happened = _attempt_initialization(
+                data_directory=data_directory,
+                saved_assets=None,
+            )
+            AssetResolver.__instance = object.__new__(cls)
 
         AssetResolver.__instance.assets = assets
+        AssetResolver.__instance.remote_check_happened = check_happened
+
         return AssetResolver.__instance
 
     @staticmethod
diff --git a/rotkehlchen/rotkehlchen.py b/rotkehlchen/rotkehlchen.py
--- a/rotkehlchen/rotkehlchen.py
+++ b/rotkehlchen/rotkehlchen.py
@@ -12,6 +12,7 @@
 from typing_extensions import Literal
 
 from rotkehlchen.accounting.accountant import Accountant
+from rotkehlchen.assets.resolver import AssetResolver
 from rotkehlchen.balances.manual import account_for_manually_tracked_balances
 from rotkehlchen.chain.ethereum.manager import (
     ETHEREUM_NODES_TO_CONNECT_AT_START,
@@ -99,6 +100,8 @@ def __init__(self, args: argparse.Namespace) -> None:
         self.msg_aggregator = MessagesAggregator()
         self.greenlet_manager = GreenletManager(msg_aggregator=self.msg_aggregator)
         self.exchange_manager = ExchangeManager(msg_aggregator=self.msg_aggregator)
+        # Initialize the AssetResolver singleton
+        AssetResolver(data_directory=self.data_dir)
         self.data = DataHandler(self.data_dir, self.msg_aggregator)
         self.cryptocompare = Cryptocompare(data_directory=self.data_dir, database=None)
         self.coingecko = Coingecko()
diff --git a/tools/asset_aggregator/__main__.py b/tools/asset_aggregator/__main__.py
--- a/tools/asset_aggregator/__main__.py
+++ b/tools/asset_aggregator/__main__.py
@@ -126,20 +126,20 @@ def main():
     arg_parser = aggregator_args()
     args = arg_parser.parse_args()
     msg_aggregator = MessagesAggregator()
+    data_directory = default_data_directory()
 
-    user_data_dir = default_data_directory() / args.db_user
+    user_data_dir = data_directory / args.db_user
     database = DBHandler(
         user_data_dir=user_data_dir,
         password=args.db_password,
         msg_aggregator=msg_aggregator,
         initial_settings=None,
     )
-    our_data = AssetResolver().assets
+    our_data = AssetResolver(data_directory).assets
     paprika = CoinPaprika()
     cmc = None
     cmc_list = None
     root_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
-    data_directory = default_data_directory()
     if args.cmc_api_key:
         cmc = Coinmarketcap(
             data_directory=data_directory,


Test patch:
diff --git a/rotkehlchen/tests/fixtures/__init__.py b/rotkehlchen/tests/fixtures/__init__.py
--- a/rotkehlchen/tests/fixtures/__init__.py
+++ b/rotkehlchen/tests/fixtures/__init__.py
@@ -2,6 +2,7 @@
 # flake8: noqa
 
 from rotkehlchen.tests.fixtures.accounting import *
+from rotkehlchen.tests.fixtures.assets import *
 from rotkehlchen.tests.fixtures.blockchain import *
 from rotkehlchen.tests.fixtures.db import *
 from rotkehlchen.tests.fixtures.exchanges.binance import *
diff --git a/rotkehlchen/tests/fixtures/accounting.py b/rotkehlchen/tests/fixtures/accounting.py
--- a/rotkehlchen/tests/fixtures/accounting.py
+++ b/rotkehlchen/tests/fixtures/accounting.py
@@ -33,6 +33,13 @@ def data_dir(use_clean_caching_directory, tmpdir_factory) -> Path:
 
     data_directory.mkdir(parents=True, exist_ok=True)
 
+    # do not keep pull github assets between tests. Can really confuse test results
+    # as we may end up with different set of assets in tests
+    try:
+        (data_directory / 'assets').unlink()
+    except FileNotFoundError:  # TODO: In python 3.8 we can add missing_ok=True to unlink
+        pass
+
     # Remove any old accounts. The only reason we keep this directory around is for
     # cached price queries, not for user DBs
     for x in data_directory.iterdir():
diff --git a/rotkehlchen/tests/fixtures/assets.py b/rotkehlchen/tests/fixtures/assets.py
new file mode 100644
--- /dev/null
+++ b/rotkehlchen/tests/fixtures/assets.py
@@ -0,0 +1,64 @@
+from unittest.mock import patch
+
+import pytest
+
+from rotkehlchen.assets.resolver import AssetResolver
+from rotkehlchen.tests.utils.mock import MockResponse
+
+
+@pytest.fixture()
+def mock_asset_meta_github_response() -> str:
+    return '{"md5": "", "version": 0}'
+
+
+@pytest.fixture()
+def mock_asset_github_response() -> str:
+    return '{}'
+
+
+@pytest.fixture()
+def query_github_for_assets() -> bool:
+    """If True, the default behavior of querying github for latest assets will occur"""
+    return False
+
+
+@pytest.fixture()
+def force_reinitialize_asset_resolver() -> bool:
+    """If True, the asset resolver instance will be force to start frm scratch"""
+    return False
+
+
+# We need auto-use here since the fixture needs to be included
+# everywhere so as to not have Asset() calls use a Resolver not
+# initialized from here which would take more time
+@pytest.fixture(autouse=True)
+def asset_resolver(
+        data_dir,
+        query_github_for_assets,
+        mock_asset_meta_github_response,
+        mock_asset_github_response,
+        force_reinitialize_asset_resolver,
+):
+    """Run the first initialization of the AssetResolver singleton
+
+    It's an autouse fixture so that it always gets initialized
+    """
+    if force_reinitialize_asset_resolver:
+        AssetResolver._AssetResolver__instance = None
+
+    if query_github_for_assets:
+        AssetResolver(data_dir)
+        return
+
+    # else mock the github request to return version lower than anything possible
+    def mock_get_request(url: str) -> MockResponse:
+        if url == 'https://raw.githubusercontent.com/rotki/rotki/develop/rotkehlchen/data/all_assets.meta':  # noqa: E501
+            return MockResponse(200, mock_asset_meta_github_response)
+        elif url == 'https://raw.githubusercontent.com/rotki/rotki/develop/rotkehlchen/data/all_assets.json':  # noqa: E501
+            return MockResponse(200, mock_asset_github_response)
+
+        raise AssertionError('This mock should receive no other urls')
+
+    get_patch = patch('requests.get', side_effect=mock_get_request)
+    with get_patch:
+        AssetResolver(data_dir)
diff --git a/rotkehlchen/tests/fixtures/rotkehlchen.py b/rotkehlchen/tests/fixtures/rotkehlchen.py
--- a/rotkehlchen/tests/fixtures/rotkehlchen.py
+++ b/rotkehlchen/tests/fixtures/rotkehlchen.py
@@ -134,7 +134,7 @@ def mock_get_settings() -> DBSettings:
 
 
 @pytest.fixture()
-def uninitialized_rotkehlchen(cli_args, inquirer):  # pylint: disable=unused-argument
+def uninitialized_rotkehlchen(cli_args, inquirer, asset_resolver):  # pylint: disable=unused-argument  # noqa: E501
     """A rotkehlchen instance that has only had __init__ run but is not unlocked
 
     Adding the inquirer fixture as a requirement to make sure that any mocking that
@@ -142,6 +142,8 @@ def uninitialized_rotkehlchen(cli_args, inquirer):  # pylint: disable=unused-arg
 
     For this to happen inquirer fixture must be initialized before Rotkehlchen so
     that the inquirer initialization in Rotkehlchen's __init__ uses the fixture's instance
+
+    Adding the AssetResolver as a requirement so that the first initialization happens here
     """
     # patch the constants to make sure that the periodic query for icons
     # does not run during tests
diff --git a/rotkehlchen/tests/unit/test_assets.py b/rotkehlchen/tests/unit/test_assets.py
--- a/rotkehlchen/tests/unit/test_assets.py
+++ b/rotkehlchen/tests/unit/test_assets.py
@@ -1,3 +1,6 @@
+import json
+from pathlib import Path
+
 import pytest
 from eth_utils import is_checksum_address
 
@@ -6,6 +9,7 @@
 from rotkehlchen.errors import DeserializationError, UnknownAsset
 from rotkehlchen.externalapis.coingecko import Coingecko
 from rotkehlchen.typing import AssetType
+from rotkehlchen.utils.hashing import file_md5
 
 
 def test_unknown_asset():
@@ -104,3 +108,44 @@ def test_coingecko_identifiers_are_reachable():
             for s in suggestions:
                 msg += f'\nSuggestion: id:{s[0]} name:{s[1]} symbol:{s[2]}'
         assert found, msg
+
+
+def test_assets_json_meta():
+    """Test that all_assets.json md5 matches and that if md5 changes since last
+    time then version is also bumped"""
+    last_meta = {'md5': 'd2076264e370ce731fa3d0b08f54f376', 'version': 1}
+    data_dir = Path(__file__).resolve().parent.parent.parent / 'data'
+    data_md5 = file_md5(data_dir / 'all_assets.json')
+
+    with open(data_dir / 'all_assets.meta', 'r') as f:
+        saved_meta = json.loads(f.read())
+
+    assert data_md5 == saved_meta['md5']
+
+    if data_md5 != last_meta['md5']:
+        msg = (
+            'The md5 has changed since the last time assets.json was edited '
+            'and the version has not been bumped',
+        )
+        assert saved_meta['version'] == last_meta['version'] + 1, msg
+
+
+@pytest.mark.parametrize('mock_asset_meta_github_response', ['{"md5": "", "version": 99999999}'])
+@pytest.mark.parametrize('use_clean_caching_directory', [True])
+@pytest.mark.parametrize('mock_asset_github_response', ["""{
+"COMPRLASSET": {
+    "coingecko": "",
+    "name": "Completely real asset, totally not for testing only",
+    "symbol": "COMPRLASSET",
+    "type": "own chain"
+}
+}"""])
+@pytest.mark.parametrize('force_reinitialize_asset_resolver', [True])
+def test_assets_pulling_from_github_works(asset_resolver):  # pylint: disable=unused-argument
+    """Test that pulling assets from mock github (due to super high version) makes the
+    pulled assets available to the local Rotki instance"""
+    new_asset = Asset("COMPRLASSET")
+    assert new_asset.name == 'Completely real asset, totally not for testing only'
+    # After the test runs we must reset the asset resolver so that it goes back to
+    # the normal list of assets
+    AssetResolver._AssetResolver__instance = None
diff --git a/rotkehlchen/tests/unit/test_tokens.py b/rotkehlchen/tests/unit/test_tokens.py
--- a/rotkehlchen/tests/unit/test_tokens.py
+++ b/rotkehlchen/tests/unit/test_tokens.py
@@ -21,7 +21,7 @@ def test_detect_tokens_for_addresses(ethtokens, inquirer):  # pylint: disable=un
     Autodetect tokens of two addresses
 
     This is going to be a bit slow test since it actually queries etherscan without any mocks.
-    By doing so we can test that the whole behaviou with etherscan works fine and our
+    By doing so we can test that the whole behavior with etherscan works fine and our
     chosen chunk length for it is also acceptable.
 
     USD price queries are mocked so we don't care about the result.
diff --git a/rotkehlchen/tests/utils/mock.py b/rotkehlchen/tests/utils/mock.py
--- a/rotkehlchen/tests/utils/mock.py
+++ b/rotkehlchen/tests/utils/mock.py
@@ -1,5 +1,6 @@
+import json
 from collections import namedtuple
-from typing import Dict
+from typing import Any, Dict
 
 from hexbytes import HexBytes
 
@@ -11,6 +12,9 @@ def __init__(self, status_code: int, text: str) -> None:
         self.content = text.encode()
         self.url = 'http://someurl.com'
 
+    def json(self) -> Dict[str, Any]:
+        return json.loads(self.text)
+
 
 class MockEth():