Repository: rotki/rotki

Issue:
Comp price at 1592625469 error
## Problem Definition

With some test accounts (yearn_test in my local pc) we get some errors when querying compound protocol historical events.

![2020-11-13_15-09](https://user-images.githubusercontent.com/1658405/99081047-7747a800-25c2-11eb-92a3-0dee75a4f1e1.png)

It seems that at 1592625469 cryptocompare has no price for COMP. Presumably since it's very close to the time it started trading

## Task

1. Demote it to a warning
2. Since for this case we know the edge case perhaps figure out the starting comp price and use that?


Discussion:

Resolving patch:
diff --git a/rotkehlchen/chain/ethereum/compound.py b/rotkehlchen/chain/ethereum/compound.py
--- a/rotkehlchen/chain/ethereum/compound.py
+++ b/rotkehlchen/chain/ethereum/compound.py
@@ -35,7 +35,6 @@
 COMPTROLLER_PROXY = EthereumConstants().contract('COMPTROLLER_PROXY')
 COMP_DEPLOYED_BLOCK = 9601359
 
-
 LEND_EVENTS_QUERY_PREFIX = """{graph_event_name}
 (where: {{blockTime_lte: $end_ts, blockTime_gte: $start_ts, {addr_position}: $address}}) {{
     id
diff --git a/rotkehlchen/externalapis/cryptocompare.py b/rotkehlchen/externalapis/cryptocompare.py
--- a/rotkehlchen/externalapis/cryptocompare.py
+++ b/rotkehlchen/externalapis/cryptocompare.py
@@ -145,6 +145,21 @@ class PriceHistoryData(NamedTuple):
     end_time: Timestamp
 
 
+class HistoHourAssetData(NamedTuple):
+    timestamp: Timestamp
+    usd_price: Price
+
+
+# Safest starting timestamp for requesting an asset price via histohour avoiding
+# 0 price. Be aware `usd_price` is from the 'close' price in USD.
+CRYPTOCOMPARE_SPECIAL_HISTOHOUR_CASES: Dict[Asset, HistoHourAssetData] = {
+    A_COMP: HistoHourAssetData(
+        timestamp=Timestamp(1592629200),
+        usd_price=Price(FVal('239.13')),
+    ),
+}
+
+
 def _dict_history_to_entries(data: List[Dict[str, Any]]) -> List[PriceHistoryEntry]:
     """Turns a list of dict of history entries to a list of proper objects"""
     return [
@@ -633,6 +648,42 @@ def get_historical_data(
 
         return self.price_history[cache_key].data
 
+    @staticmethod
+    def _check_and_get_special_histohour_price(
+            from_asset: Asset,
+            to_asset: Asset,
+            timestamp: Timestamp,
+    ) -> Price:
+        """For the given timestamp, check whether the from..to asset price
+        (or viceversa) is a special histohour API case. If so, return the price
+        based on the assets pair, otherwise return zero.
+
+        NB: special histohour API cases are the ones where this Cryptocompare
+        API returns zero prices per hour.
+        """
+        price = Price(ZERO)
+        if (
+            from_asset in CRYPTOCOMPARE_SPECIAL_HISTOHOUR_CASES and to_asset == A_USD or
+            from_asset == A_USD and to_asset in CRYPTOCOMPARE_SPECIAL_HISTOHOUR_CASES
+        ):
+            asset_data = (
+                CRYPTOCOMPARE_SPECIAL_HISTOHOUR_CASES[from_asset]
+                if to_asset == A_USD
+                else CRYPTOCOMPARE_SPECIAL_HISTOHOUR_CASES[to_asset]
+            )
+            if timestamp <= asset_data.timestamp:
+                price = (
+                    asset_data.usd_price
+                    if to_asset == A_USD
+                    else Price(FVal('1') / asset_data.usd_price)
+                )
+                log.warning(
+                    f'Query price of: {from_asset.identifier} in {to_asset.identifier} '
+                    f'at timestamp {timestamp} may return zero price. '
+                    f'Setting price to {price}, from timestamp {asset_data.timestamp}.',
+                )
+        return price
+
     def query_historical_price(
             self,
             from_asset: Asset,
@@ -651,6 +702,15 @@ def query_historical_price(
         - RemoteError if there is a problem reaching the cryptocompare server
         or with reading the response returned by the server
         """
+        # NB: check if the from..to asset price (or viceversa) is a special
+        # histohour API case.
+        price = self._check_and_get_special_histohour_price(
+            from_asset=from_asset,
+            to_asset=to_asset,
+            timestamp=timestamp,
+        )
+        if price != Price(ZERO):
+            return price
 
         try:
             data = self.get_historical_data(


Test patch:
diff --git a/rotkehlchen/tests/external_apis/test_cryptocompare.py b/rotkehlchen/tests/external_apis/test_cryptocompare.py
--- a/rotkehlchen/tests/external_apis/test_cryptocompare.py
+++ b/rotkehlchen/tests/external_apis/test_cryptocompare.py
@@ -1,14 +1,21 @@
 import os
+import warnings as test_warnings
 from unittest.mock import patch
 
 import pytest
 
 from rotkehlchen.assets.asset import Asset
-from rotkehlchen.constants.assets import A_BTC, A_USD
+from rotkehlchen.constants.assets import A_BTC, A_ETH, A_USD, A_USDT
+from rotkehlchen.constants.misc import ZERO
 from rotkehlchen.errors import NoPriceForGivenTimestamp
-from rotkehlchen.externalapis.cryptocompare import Cryptocompare
+from rotkehlchen.externalapis.cryptocompare import (
+    A_COMP,
+    CRYPTOCOMPARE_SPECIAL_HISTOHOUR_CASES,
+    Cryptocompare,
+)
 from rotkehlchen.fval import FVal
 from rotkehlchen.tests.utils.constants import A_SNGLS
+from rotkehlchen.typing import Price, Timestamp
 
 
 def test_cryptocompare_query_pricehistorical(cryptocompare):
@@ -177,3 +184,64 @@ def test_cryptocompare_query_compound_tokens(
         to_asset=A_USD,
     )
     assert price is not None
+
+
+@pytest.mark.parametrize('from_asset, to_asset, timestamp, expected_price', [
+    (A_ETH, A_USD, Timestamp(1592629200), Price(ZERO)),
+    (A_COMP, A_COMP, Timestamp(1592629200), Price(ZERO)),  # both assets COMP
+    (A_USD, A_USD, Timestamp(1592629200), Price(ZERO)),  # both assets USD
+    (A_COMP, A_USDT, Timestamp(1592629200), Price(ZERO)),  # to_asset USDT
+    (A_USDT, A_COMP, Timestamp(1592629200), Price(ZERO)),  # from_asset USDT
+    (A_COMP, A_USD, Timestamp(1592629200), Price(FVal('239.13'))),
+    (A_USD, A_COMP, Timestamp(1592629200), Price(FVal('0.004181825785137791159620290219'))),
+    (A_COMP, A_USD, Timestamp(1592629201), Price(ZERO)),  # timestamp gt
+    (A_USD, A_COMP, Timestamp(1592629201), Price(ZERO)),  # timestamp gt
+])
+def test_check_and_get_special_histohour_price(
+        cryptocompare,
+        from_asset,
+        to_asset,
+        timestamp,
+        expected_price,
+):
+    """
+    Test expected prices are returned for different combinations of
+    `from_asset`, `to_asset` and `timestamp`.
+    """
+    price = cryptocompare._check_and_get_special_histohour_price(
+        from_asset=from_asset,
+        to_asset=to_asset,
+        timestamp=timestamp,
+    )
+    assert price == expected_price
+
+
+def test_keep_special_histohour_cases_up_to_date(cryptocompare):
+    """Test CRYPTOCOMPARE_SPECIAL_HISTOHOUR_CASES assets timestamps are still
+    valid by checking that for a smaller timestamp the response contains
+    entries with all price attributes at zero.
+    """
+    def is_price_not_valid(hour_price_data):
+        return all(hour_price_data[attr] == 0 for attr in ('low', 'high', 'open', 'close'))
+
+    limit = 10
+    for asset, asset_data in CRYPTOCOMPARE_SPECIAL_HISTOHOUR_CASES.items():
+        # Call `query_endpoint_histohour()` for handling special assets
+        to_timestamp = Timestamp(asset_data.timestamp - 3600)
+        from_timestamp = Timestamp(to_timestamp - limit * 3600)
+        response = cryptocompare.query_endpoint_histohour(
+            from_asset=asset,
+            to_asset=A_USD,
+            limit=limit,
+            to_timestamp=to_timestamp,
+        )
+        try:
+            assert any(is_price_not_valid(price_data) for price_data in response['Data'])
+        except AssertionError:
+            warning_msg = (
+                f'Cryptocompare histohour API has non-zero prices for asset '
+                f'{asset.identifier} from {from_timestamp} to {to_timestamp}. '
+                f' Please, update CRYPTOCOMPARE_SPECIAL_HISTOHOUR_CASES dict '
+                f'with a smaller timestamp.'
+            )
+            test_warnings.warn(UserWarning(warning_msg))