Repository: rotki/rotki

Issue:
Support KuCoin csv import
## Motivation

While there does already exist an [exchange integration for KuCoin](https://github.com/rotki/rotki/blob/develop/rotkehlchen/exchanges/kucoin.py) on Rotki, generating an API key requires you to KYC. Not all users are willing to KYC with KuCoin (including myself) and so we should give these users the option to export their trades and then import them to Rotki

## Specification

Previous PRs for other exchanges/services are a good template to work off, as the imported csv just needs to be parsed and history events created from its rows.
- [Backend part](https://github.com/rotki/rotki/pull/6157)
- [Frontend part](https://github.com/rotki/rotki/pull/6179)
- [Full implementation](https://github.com/rotki/rotki/pull/6426)

### CSV Format
Currently KuCoin trades are exported in the following format
```
tradeCreatedAt,symbol,side,price,size,funds,fee,feeCurrency
2022-04-28 12:36:00,XYM-USDT,sell,0.1025,36.4479,3.73590975,0.00373590975,USDT
2022-04-28 15:46:07,XRP-USDT,buy,0.64924,432.59183198,280.8559209946952,0.2808559209946952,USDT
```

However they used to have this format
```
Time,Pair,Side,FilledPrice,FilledPriceCoin,Amount,AmountCoin,Volume,VolumeCoin,Fee,FeeCoin
2019/05/11 10:27:17,KCS/BTC,Buy,0.00015225,BTC,10.01,KCS,0.00152402,BTC,0.00000152,BTC
2019/05/11 10:27:17,KCS/BTC,Buy,0.00015225,BTC,10.01,KCS,0.00152402,BTC,0.00000152,BTC
```

MVP would be to support imports for the current format, but ideally we give the user the option of both.



Discussion:

Resolving patch:
diff --git a/rotkehlchen/data_import/importers/kucoin.py b/rotkehlchen/data_import/importers/kucoin.py
new file mode 100644
--- /dev/null
+++ b/rotkehlchen/data_import/importers/kucoin.py
@@ -0,0 +1,87 @@
+import csv
+from pathlib import Path
+from typing import Any
+
+from rotkehlchen.assets.converters import asset_from_kucoin
+from rotkehlchen.data_import.utils import BaseExchangeImporter
+from rotkehlchen.db.drivers.gevent import DBCursor
+from rotkehlchen.errors.misc import InputError
+from rotkehlchen.errors.serialization import DeserializationError
+from rotkehlchen.exchanges.data_structures import Trade
+from rotkehlchen.exchanges.utils import get_key_if_has_val
+from rotkehlchen.history.deserialization import deserialize_price
+from rotkehlchen.serialization.deserialize import (
+    deserialize_asset_amount,
+    deserialize_fee,
+    deserialize_timestamp_from_date,
+)
+from rotkehlchen.types import Location, TradeType
+
+
+class KucoinImporter(BaseExchangeImporter):
+    def _import_csv(
+            self,
+            write_cursor: DBCursor,
+            filepath: Path,
+            **kwargs: Any,
+    ) -> None:
+        """
+        Import trades from Kucoin. Find out which file
+        we are parsing depending on its format.
+
+        May raise:
+        - UnsupportedCSVEntry if operation not supported
+        - InputError if a column we need is missing
+        """
+        with open(filepath, encoding='utf-8-sig') as csvfile:
+            start_pos = csvfile.tell()
+            first_line = csvfile.readline()
+            if 'tradeCreatedAt,symbol,side,price,size,funds,fee,feeCurrency' in first_line:
+                date_key = 'tradeCreatedAt'
+                tokens_key, splitter = 'symbol', '-'
+                trade_type_key = 'side'
+                amount_key = 'size'
+                rate_key = 'price'
+                fee_key = 'fee'
+                fee_currency_key = 'feeCurrency'
+            elif 'Time,Pair,Side,FilledPrice,FilledPriceCoin,Amount,AmountCoin,Volume,VolumeCoin,Fee,FeeCoin' in first_line:  # noqa: E501
+                date_key = 'Time'
+                tokens_key, splitter = 'Pair', '/'
+                trade_type_key = 'Side'
+                amount_key = 'Amount'
+                rate_key = 'FilledPrice'
+                fee_key = 'Fee'
+                fee_currency_key = 'FeeCoin'
+            else:
+                raise InputError('The given kucoin CSV file format can not be recognized')
+
+            csvfile.seek(start_pos)  # go back to start of the file
+            for row in csv.DictReader(csvfile):
+                try:
+                    base, quote = row[tokens_key].split(splitter)
+                    fee_currency = get_key_if_has_val(row, fee_currency_key)
+                    self.add_trade(
+                        write_cursor=write_cursor,
+                        trade=Trade(
+                            timestamp=deserialize_timestamp_from_date(
+                                date=row[date_key],
+                                formatstr=kwargs.get('timestamp_format', '%Y-%m-%d %H:%M:%S'),
+                                location='Kucoin order history import',
+                            ),
+                            location=Location.KUCOIN,
+                            base_asset=asset_from_kucoin(base),
+                            quote_asset=asset_from_kucoin(quote),
+                            trade_type=TradeType.BUY if row[trade_type_key].lower() == 'buy' else TradeType.SELL,  # noqa: E501
+                            amount=deserialize_asset_amount(row[amount_key]),
+                            rate=deserialize_price(row[rate_key]),
+                            fee=deserialize_fee(get_key_if_has_val(row, fee_key)),
+                            fee_currency=asset_from_kucoin(fee_currency) if fee_currency else None,
+                        ),
+                    )
+                except DeserializationError as e:
+                    self.db.msg_aggregator.add_warning(
+                        f'Deserialization error during Kucoin CSV import. '
+                        f'{e!s}. Ignoring entry',
+                    )
+                except KeyError as e:
+                    raise InputError(f'Could not find key {e!s} in kucoin csv row {row!s}') from e
diff --git a/rotkehlchen/data_import/manager.py b/rotkehlchen/data_import/manager.py
--- a/rotkehlchen/data_import/manager.py
+++ b/rotkehlchen/data_import/manager.py
@@ -11,6 +11,7 @@
 from rotkehlchen.data_import.importers.blockfi_transactions import BlockfiTransactionsImporter
 from rotkehlchen.data_import.importers.cointracking import CointrackingImporter
 from rotkehlchen.data_import.importers.cryptocom import CryptocomImporter
+from rotkehlchen.data_import.importers.kucoin import KucoinImporter
 from rotkehlchen.data_import.importers.nexo import NexoImporter
 from rotkehlchen.data_import.importers.rotki_events import RotkiGenericEventsImporter
 from rotkehlchen.data_import.importers.rotki_trades import RotkiGenericTradesImporter
@@ -37,6 +38,7 @@ class DataImportSource(SerializableEnumNameMixin):
     BITMEX_WALLET_HISTORY = 13
     BITSTAMP = 14
     BITTREX = 15
+    KUCOIN = 16
 
     def get_importer_type(self) -> type[BaseExchangeImporter]:
         if self == DataImportSource.COINTRACKING:
@@ -69,6 +71,8 @@ def get_importer_type(self) -> type[BaseExchangeImporter]:
             return BitstampTransactionsImporter
         if self == DataImportSource.BITTREX:
             return BittrexImporter
+        if self == DataImportSource.KUCOIN:
+            return KucoinImporter
         raise AssertionError(f'Unknown DataImportSource value {self}')
 
 


Test patch:
diff --git a/rotkehlchen/tests/api/test_data_import.py b/rotkehlchen/tests/api/test_data_import.py
--- a/rotkehlchen/tests/api/test_data_import.py
+++ b/rotkehlchen/tests/api/test_data_import.py
@@ -28,6 +28,7 @@
     assert_cryptocom_import_results,
     assert_cryptocom_special_events_import_results,
     assert_custom_cointracking,
+    assert_kucoin_import_results,
     assert_nexo_results,
     assert_rotki_generic_events_import_results,
     assert_rotki_generic_trades_import_results,
@@ -651,3 +652,24 @@ def test_bittrex_history_import(rotkehlchen_api_server):
         assert assert_proper_response_with_result(response) is True
 
     assert_bittrex_import_results(rotki)
+
+
+def test_kucoin_history_import(rotkehlchen_api_server):
+    """Test that data import works for both kucoin csv files"""
+    rotki = rotkehlchen_api_server.rest_api.rotkehlchen
+    dir_path = Path(__file__).resolve().parent.parent
+    for filename, time_format in (
+        ('kucoin_order_history.csv', None),
+        ('kucoin_order_history_old.csv', '%Y/%m/%d %H:%M:%S'),
+    ):
+        filepath = dir_path / 'data' / filename
+        json_data = {'source': 'kucoin', 'file': str(filepath), 'timestamp_format': time_format}
+        response = requests.put(
+            api_url_for(
+                rotkehlchen_api_server,
+                'dataimportresource',
+            ), json=json_data,
+        )
+        assert assert_proper_response_with_result(response) is True
+
+    assert_kucoin_import_results(rotki)
diff --git a/rotkehlchen/tests/data/kucoin_order_history.csv b/rotkehlchen/tests/data/kucoin_order_history.csv
new file mode 100644
--- /dev/null
+++ b/rotkehlchen/tests/data/kucoin_order_history.csv
@@ -0,0 +1,3 @@
+tradeCreatedAt,symbol,side,price,size,funds,fee,feeCurrency
+2022-04-28 12:36:00,XTZ-USDT,sell,0.1025,36.4479,3.73590975,0.00373590975,USDT
+2022-04-28 15:46:07,XRP-USDT,buy,0.64924,432.59183198,280.8559209946952,0.2808559209946952,USDT
\ No newline at end of file
diff --git a/rotkehlchen/tests/data/kucoin_order_history_old.csv b/rotkehlchen/tests/data/kucoin_order_history_old.csv
new file mode 100644
--- /dev/null
+++ b/rotkehlchen/tests/data/kucoin_order_history_old.csv
@@ -0,0 +1,3 @@
+Time,Pair,Side,FilledPrice,FilledPriceCoin,Amount,AmountCoin,Volume,VolumeCoin,Fee,FeeCoin
+2019/05/11 10:27:17,KCS/BTC,Buy,0.00015225,BTC,10.01,KCS,0.00152402,BTC,0.00000152,BTC
+2019/05/11 10:27:18,KCS/BTC,Sell,0.00015226,BTC,10.02,KCS,0.00152403,BTC,0.00000153,BTC
\ No newline at end of file
diff --git a/rotkehlchen/tests/utils/dataimport.py b/rotkehlchen/tests/utils/dataimport.py
--- a/rotkehlchen/tests/utils/dataimport.py
+++ b/rotkehlchen/tests/utils/dataimport.py
@@ -41,7 +41,7 @@
 from rotkehlchen.history.events.structures.base import HistoryBaseEntry, HistoryEvent
 from rotkehlchen.history.events.structures.types import HistoryEventSubType, HistoryEventType
 from rotkehlchen.rotkehlchen import Rotkehlchen
-from rotkehlchen.tests.utils.constants import A_AXS, A_CRO, A_GBP, A_MCO, A_XMR
+from rotkehlchen.tests.utils.constants import A_AXS, A_CRO, A_GBP, A_KCS, A_MCO, A_XMR, A_XTZ
 from rotkehlchen.types import (
     AssetAmount,
     AssetMovementCategory,
@@ -2462,3 +2462,55 @@ def assert_bittrex_import_results(rotki: Rotkehlchen):
         fee=Fee(ZERO),
         link='Imported from bittrex CSV file',
     )]
+
+
+def assert_kucoin_import_results(rotki: Rotkehlchen):
+    """A utility function to help assert on correctness of importing data from kucoin"""
+    with rotki.data.db.conn.read_ctx() as cursor:
+        trades = rotki.data.db.get_trades(cursor, filter_query=TradesFilterQuery.make(), has_premium=True)  # noqa: E501
+    warnings = rotki.msg_aggregator.consume_warnings()
+    errors = rotki.msg_aggregator.consume_errors()
+    assert len(errors) == 0
+    assert len(warnings) == 0
+    expected_trades = [Trade(
+        timestamp=Timestamp(1557570437),
+        location=Location.KUCOIN,
+        base_asset=A_KCS,
+        quote_asset=A_BTC,
+        trade_type=TradeType.BUY,
+        amount=AssetAmount(FVal(10.01)),
+        rate=Price(FVal(0.00015225)),
+        fee=Fee(FVal(0.00000152)),
+        fee_currency=A_BTC,
+    ), Trade(
+        timestamp=Timestamp(1557570438),
+        location=Location.KUCOIN,
+        base_asset=A_KCS,
+        quote_asset=A_BTC,
+        trade_type=TradeType.SELL,
+        amount=AssetAmount(FVal(10.02)),
+        rate=Price(FVal(0.00015226)),
+        fee=Fee(FVal(0.00000153)),
+        fee_currency=A_BTC,
+    ), Trade(
+        timestamp=Timestamp(1651149360),
+        location=Location.KUCOIN,
+        base_asset=A_XTZ,
+        quote_asset=A_USDT,
+        trade_type=TradeType.SELL,
+        amount=AssetAmount(FVal(36.4479)),
+        rate=Price(FVal(0.1025)),
+        fee=Fee(FVal(0.00373590975)),
+        fee_currency=A_USDT,
+    ), Trade(
+        timestamp=Timestamp(1651160767),
+        location=Location.KUCOIN,
+        base_asset=A_XRP,
+        quote_asset=A_USDT,
+        trade_type=TradeType.BUY,
+        amount=AssetAmount(FVal(432.59183198)),
+        rate=Price(FVal(0.64924)),
+        fee=Fee(FVal(0.2808559209946952)),
+        fee_currency=A_USDT,
+    )]
+    assert trades == expected_trades