Repository: rotki/rotki

Issue:
Kucoin query for historical trades is not working anymore
## Problem Definition

Kucoin has deprecated the endpoint to query for historical trades. 

![image](https://user-images.githubusercontent.com/5068010/156827772-3edd864f-3222-4482-b16b-fed12f014689.png)

As an alternative as described here https://github.com/Kucoin/kucoin-api-docs/issues/363 we can use the ledger endpoint https://docs.kucoin.com/#get-account-ledgers


### System Description

Operating system: 
Rotki version:



Discussion:
Since this is a bug I am adding it to 1.24.1

Resolving patch:
diff --git a/rotkehlchen/exchanges/kucoin.py b/rotkehlchen/exchanges/kucoin.py
--- a/rotkehlchen/exchanges/kucoin.py
+++ b/rotkehlchen/exchanges/kucoin.py
@@ -3,7 +3,7 @@
 import hmac
 import logging
 from collections import defaultdict
-from enum import Enum
+from enum import Enum, auto
 from functools import partial
 from http import HTTPStatus
 from json.decoder import JSONDecodeError
@@ -76,6 +76,7 @@
     400004: 'Invalid API passphrase.',
     400005: 'Invalid API secret.',
     400007: 'Provided KuCoin API key needs to have "General" permission activated.',
+    400100: 'Invalid query time',
     411100: 'Contact KuCoin support to unfreeze your account',
 }
 API_PAGE_SIZE_LIMIT = 500
@@ -90,12 +91,12 @@
 
 
 class KucoinCase(Enum):
-    API_KEY = 1
-    BALANCES = 2
-    TRADES = 3
-    OLD_TRADES = 4
-    DEPOSITS = 5
-    WITHDRAWALS = 6
+    API_KEY = auto()
+    BALANCES = auto()
+    TRADES = auto()
+    OLD_TRADES = auto()
+    DEPOSITS = auto()
+    WITHDRAWALS = auto()
 
     def __str__(self) -> str:
         if self == KucoinCase.API_KEY:
@@ -116,12 +117,6 @@ def __str__(self) -> str:
 PAGINATED_CASES = (KucoinCase.OLD_TRADES, KucoinCase.TRADES, KucoinCase.DEPOSITS, KucoinCase.WITHDRAWALS)  # noqa: E501
 
 
-def _serialize_ts(case: KucoinCase, time: int) -> int:
-    if case == KucoinCase.OLD_TRADES:
-        return time
-    return time * 1000
-
-
 def _deserialize_ts(case: KucoinCase, time: int) -> Timestamp:
     if case == KucoinCase.OLD_TRADES:
         return Timestamp(time)
@@ -225,11 +220,10 @@ def _api_query(
                 api_path = 'api/v1/withdrawals'
         elif case == KucoinCase.OLD_TRADES:
             assert isinstance(options, dict)
-            api_path = 'api/v1/hist-orders'
+            api_path = 'api/v1/orders'
         elif case == KucoinCase.TRADES:
             assert isinstance(options, dict)
             api_path = 'api/v1/fills'
-
         else:
             raise AssertionError(f'Unexpected case: {case}')
 
@@ -355,7 +349,7 @@ def _api_query_paginated(
             time_step = WEEK_IN_SECONDS
         elif case == KucoinCase.OLD_TRADES:
             deserialization_method = partial(self._deserialize_trade, case=case)
-            time_step = MONTH_IN_SECONDS  # does not get used anyway
+            time_step = WEEK_IN_SECONDS
         elif case in (KucoinCase.DEPOSITS, KucoinCase.WITHDRAWALS):
             deserialization_method = partial(
                 self._deserialize_asset_movement,
@@ -366,15 +360,15 @@ def _api_query_paginated(
             raise AssertionError(f'Unexpected case: {case}')
 
         call_options = options.copy()
-        call_options['startAt'] = _serialize_ts(case, max(start_ts, KUCOIN_LAUNCH_TS))
+        call_options['startAt'] = Timestamp(max(start_ts, KUCOIN_LAUNCH_TS) * 1000)
         while True:
             current_query_ts = _deserialize_ts(case, call_options['startAt'])
-            if case != KucoinCase.OLD_TRADES:
-                call_options['endAt'] = _serialize_ts(
-                    case=case,
-                    time=min(current_query_ts + time_step, end_ts),
-                )
-
+            call_options['endAt'] = Timestamp(
+                min(Timestamp(current_query_ts + time_step), end_ts) * 1000,
+            )
+            logger.debug(
+                f'Querying kucoin {case} from {current_query_ts} to {call_options["endAt"]}',
+            )
             response = self._api_query(
                 case=case,
                 options=call_options,
@@ -403,6 +397,12 @@ def _api_query_paginated(
             except KeyError as e:
                 msg = f'Kucoin {case} JSON response is missing key: {str(e)}'
                 log.error(msg, response_dict)
+                if case == KucoinCase.OLD_TRADES and '400100' == response_dict.get('code', ''):
+                    if current_query_ts + time_step >= end_ts:
+                        break  # end of time range query and last page. We are done.
+                    # else update query ts
+                    current_query_ts += time_step  # type: ignore
+                    continue
                 raise RemoteError(msg) from e
 
             for raw_result in raw_results:
@@ -441,7 +441,7 @@ def _api_query_paginated(
 
             is_last_page = total_page in (0, current_page)
             if is_last_page:
-                if case == KucoinCase.OLD_TRADES or current_query_ts + time_step >= end_ts:
+                if current_query_ts + time_step >= end_ts:
                     break  # end of time range query and last page. We are done.
                 # else update query ts
                 current_query_ts += time_step  # type: ignore
@@ -453,7 +453,7 @@ def _api_query_paginated(
                 call_options['currentPage'] = current_page + 1
             else:
                 call_options['currentPage'] = 1
-            call_options['startAt'] = _serialize_ts(case, current_query_ts)
+            call_options['startAt'] = Timestamp(current_query_ts * 1000)
 
         return results
 


Test patch:
diff --git a/rotkehlchen/tests/exchanges/test_kucoin.py b/rotkehlchen/tests/exchanges/test_kucoin.py
--- a/rotkehlchen/tests/exchanges/test_kucoin.py
+++ b/rotkehlchen/tests/exchanges/test_kucoin.py
@@ -11,10 +11,11 @@
 from rotkehlchen.assets.asset import WORLD_TO_KUCOIN
 from rotkehlchen.assets.converters import UNSUPPORTED_KUCOIN_ASSETS, asset_from_kucoin
 from rotkehlchen.constants.assets import A_BTC, A_ETH, A_LINK, A_USDT
+from rotkehlchen.constants.timing import WEEK_IN_SECONDS
 from rotkehlchen.errors.asset import UnknownAsset, UnsupportedAsset
 from rotkehlchen.errors.misc import RemoteError
 from rotkehlchen.exchanges.data_structures import AssetMovement, Trade, TradeType
-from rotkehlchen.exchanges.kucoin import Kucoin, KucoinCase
+from rotkehlchen.exchanges.kucoin import KUCOIN_LAUNCH_TS, Kucoin, KucoinCase
 from rotkehlchen.fval import FVal
 from rotkehlchen.tests.utils.constants import A_BSV, A_KCS, A_NANO
 from rotkehlchen.tests.utils.mock import MockResponse
@@ -411,8 +412,8 @@ def test_deserialize_asset_movement_withdrawal(mock_kucoin):
 
 @pytest.mark.skip('Fails with status code: 404 and text: KC-API-KEY not exists')
 @pytest.mark.parametrize('should_mock_current_price_queries', [True])
-def test_query_balances_sandbox(sandbox_kuckoin, inquirer):  # pylint: disable=unused-argument
-    assets_balance, msg = sandbox_kuckoin.query_balances()
+def test_query_balances_sandbox(sandbox_kucoin, inquirer):  # pylint: disable=unused-argument
+    assets_balance, msg = sandbox_kucoin.query_balances()
     assert assets_balance == {
         A_BTC: Balance(
             amount=FVal('2.61018067'),
@@ -436,7 +437,7 @@ def test_query_balances_sandbox(sandbox_kuckoin, inquirer):  # pylint: disable=u
 
 @pytest.mark.skip('Fails with status code: 404 and text: KC-API-KEY not exists')
 @pytest.mark.parametrize('should_mock_current_price_queries', [True])
-def test_query_trades_sandbox(sandbox_kuckoin, inquirer):  # pylint: disable=unused-argument
+def test_query_trades_sandbox(sandbox_kucoin, inquirer):  # pylint: disable=unused-argument
     """The sandbox account has 6 trades. Below a list of the trades and their
     timestamps in ascending mode.
     - trade 1: 1612556651 -> skipped
@@ -503,7 +504,7 @@ def test_query_trades_sandbox(sandbox_kuckoin, inquirer):  # pylint: disable=unu
             notes='',
         ),
     ]
-    trades, _ = sandbox_kuckoin.query_online_trade_history(
+    trades, _ = sandbox_kucoin.query_online_trade_history(
         start_ts=Timestamp(1612556693),
         end_ts=Timestamp(1612556765),
     )
@@ -512,7 +513,7 @@ def test_query_trades_sandbox(sandbox_kuckoin, inquirer):  # pylint: disable=unu
 
 @pytest.mark.parametrize('should_mock_current_price_queries', [True])
 def test_query_asset_movements_sandbox(
-        sandbox_kuckoin,
+        sandbox_kucoin,
         inquirer,  # pylint: disable=unused-argument
 ):
     """Unfortunately the sandbox environment does not support deposits and
@@ -750,16 +751,37 @@ def mock_api_query_response(case, options):  # pylint: disable=unused-argument
         new=2,
     )
     api_query_patch = patch.object(
-        target=sandbox_kuckoin,
+        target=sandbox_kucoin,
         attribute='_api_query',
         side_effect=mock_api_query_response,
     )
     with ExitStack() as stack:
         stack.enter_context(months_in_seconds_patch)
         stack.enter_context(api_query_patch)
-        asset_movements = sandbox_kuckoin.query_online_deposits_withdrawals(
+        asset_movements = sandbox_kucoin.query_online_deposits_withdrawals(
             start_ts=Timestamp(1612556651),
             end_ts=Timestamp(1612556654),
         )
 
     assert asset_movements == expected_asset_movements
+
+
+@pytest.mark.parametrize('should_mock_current_price_queries', [True])
+def test_query_old_trades_sandbox(sandbox_kucoin, inquirer):  # pylint: disable=unused-argument
+    """Test that the endpoint for old trades returns valid results from the api
+    in very old timestamps
+    """
+    trades, _ = sandbox_kucoin.query_online_trade_history(
+        start_ts=Timestamp(KUCOIN_LAUNCH_TS),
+        end_ts=Timestamp(KUCOIN_LAUNCH_TS + WEEK_IN_SECONDS * 4),
+    )
+    msg_aggregator = sandbox_kucoin.msg_aggregator
+    assert len(msg_aggregator.consume_errors()) == 0
+    assert len(msg_aggregator.consume_warnings()) == 0
+    assert trades == []
+
+    last_trades, _ = sandbox_kucoin.query_online_trade_history(
+        start_ts=Timestamp(1651425348),
+        end_ts=Timestamp(1654110961),
+    )
+    assert len(last_trades) == 24
diff --git a/rotkehlchen/tests/fixtures/exchanges/kucoin.py b/rotkehlchen/tests/fixtures/exchanges/kucoin.py
--- a/rotkehlchen/tests/fixtures/exchanges/kucoin.py
+++ b/rotkehlchen/tests/fixtures/exchanges/kucoin.py
@@ -45,19 +45,19 @@ def mock_kucoin(
 @pytest.fixture(name='kucoin_sandbox_api_key')
 def fixture_kucoin_sandbox_api_key():
     # General permission (aka read-only)
-    return '6023e471a2644e00063aa8bb'
+    return '6296162429c69200011e6d6f'
 
 
 @pytest.fixture(name='kucoin_sandbox_api_secret')
 def fixture_kucoin_sandbox_api_secret():
     # General permission (aka read-only)
-    return b'3a817593-eceb-47f3-90d4-26de79de588a'
+    return b'59df08af-5a6e-4893-9a8c-81e0d0a5b655'
 
 
 @pytest.fixture(name='kucoin_sandbox_passphrase')
 def fixture_kucoin_sandbox_passphrase():
     # General permission (aka read-only)
-    return 'rotkidev'
+    return 'rotkiapi'
 
 
 @pytest.fixture(name='kucoin_sandbox_base_uri')
@@ -66,7 +66,7 @@ def fixture_kucoin_sandbox_base_uri():
 
 
 @pytest.fixture
-def sandbox_kuckoin(
+def sandbox_kucoin(
         database,
         inquirer,  # pylint: disable=unused-argument
         function_scope_messages_aggregator,