Repository: rotki/rotki

Issue:
Import CSV file from CoinTracking.info
## Abstract

Many people have already spent a lot of time tracking coins on other sites.  Hard to start over with rotkehlchen.  Would be very helpful if you could import csv file that other popular sites export.  Most popular right now is CoinTracking.info . 

## Motivation

Import csv file.  Almost all tracking sites allow you to export data to a csv file. 

## Specification

If the feature is techical in nature please write as detailed as possible a specification of what needs to be built.



Discussion:

Resolving patch:
diff --git a/rotkehlchen/data/importer.py b/rotkehlchen/data/importer.py
new file mode 100644
--- /dev/null
+++ b/rotkehlchen/data/importer.py
@@ -0,0 +1,163 @@
+import csv
+from typing import List
+
+from rotkehlchen.assets.asset import Asset
+from rotkehlchen.constants.assets import A_USD
+from rotkehlchen.constants.misc import ZERO
+from rotkehlchen.db.dbhandler import DBHandler
+from rotkehlchen.errors import DeserializationError, UnknownAsset
+from rotkehlchen.exchanges.data_structures import AssetMovement, AssetMovementCategory, Trade
+from rotkehlchen.serialization.deserialize import (
+    deserialize_asset_amount,
+    deserialize_asset_movement_category,
+    deserialize_timestamp_from_date,
+)
+from rotkehlchen.typing import Fee, Location, TradePair, TradeType
+
+
+class UnsupportedCointrackingEntry(Exception):
+    """Thrown for Cointracking CSV export entries we can't support to import"""
+
+
+def exchange_row_to_location(entry: str) -> Location:
+    """Takes the exchange row entry of Cointracking exported trades list and returns a location"""
+    if entry == 'no exchange':
+        return Location.EXTERNAL
+    elif entry == 'Kraken':
+        return Location.KRAKEN
+    elif entry == 'Poloniex':
+        return Location.POLONIEX
+    elif entry == 'Bittrex':
+        return Location.BITTREX
+    elif entry == 'Binance':
+        return Location.BINANCE
+    elif entry == 'Bitmex':
+        return Location.BITMEX
+    elif entry == 'Coinbase':
+        return Location.COINBASE
+    elif entry == 'ETH Transaction':
+        raise UnsupportedCointrackingEntry(
+            'Not importing ETH Transactions from Cointracking. Cointracking does not '
+            'export enough data for them. Simply enter your ethereum accounts and all '
+            'your transactions will be auto imported directly from the chain',
+        )
+    elif entry == 'BTC Transaction':
+        raise UnsupportedCointrackingEntry(
+            'Not importing BTC Transactions from Cointracking. Cointracking does not '
+            'export enough data for them. Simply enter your BTC accounts and all '
+            'your transactions will be auto imported directly from the chain',
+        )
+
+    raise UnsupportedCointrackingEntry(
+        f'Unknown Exchange "{entry}" encountered during a cointracking import. Ignoring it',
+    )
+
+
+class DataImporter():
+
+    def __init__(self, db: DBHandler) -> None:
+        self.db = db
+
+    def _consume_cointracking_entry(self, csv_row: List[str]):
+        """Consumes a cointracking entry row from the CSV and adds it into the database
+        Can raise:
+            - DeserializationError if something is wrong with the format of the expected values
+            - UnsupportedCointrackingEntry if importing of this entry is not supported.
+            - IndexError if the CSV file is corrupt
+            - UnknownAsset if one of the assets founds in the entry are not supported
+        """
+        row_type = csv_row[1]  # Type
+        timestamp = deserialize_timestamp_from_date(
+            date=csv_row[9],
+            formatstr='%d.%m.%Y %H:%M',
+            location='cointracking.info',
+        )
+        notes = csv_row[8]
+        location = exchange_row_to_location(csv_row[6])
+
+        if row_type == 'Gift/Tip' or row_type == 'Trade':
+            base_asset = Asset(csv_row[3])
+            quote_asset = None if csv_row[5] == '' else Asset(csv_row[5])
+            if not quote_asset and row_type != 'Gift/Tip':
+                raise DeserializationError('Got a trade entry with an empty quote asset')
+
+            if quote_asset is None:
+                # Really makes no difference as this is just a gift and the amount is zero
+                quote_asset = A_USD
+            pair = TradePair(f'{base_asset.identifier}_{quote_asset.identifier}')
+            base_amount_bought = deserialize_asset_amount(csv_row[2])
+            if csv_row[4] != '-':
+                quote_amount_sold = deserialize_asset_amount(csv_row[4])
+            else:
+                quote_amount_sold = ZERO
+            rate = quote_amount_sold / base_amount_bought
+
+            trade = Trade(
+                timestamp=timestamp,
+                location=location,
+                pair=pair,
+                trade_type=TradeType.BUY,  # It's always a buy during cointracking import
+                amount=base_amount_bought,
+                rate=rate,
+                fee=Fee(ZERO),  # There are no fees when import from cointracking
+                fee_currency=base_asset,
+                link='',
+                notes=notes,
+            )
+            self.db.add_trades([trade])
+        elif row_type == 'Deposit' or row_type == 'Withdrawal':
+            category = deserialize_asset_movement_category(row_type.lower())
+            if category == AssetMovementCategory.DEPOSIT:
+                amount = deserialize_asset_amount(csv_row[2])
+                asset = Asset(csv_row[3])
+            else:
+                amount = deserialize_asset_amount(csv_row[4])
+                asset = Asset(csv_row[5])
+
+            asset_movement = AssetMovement(
+                location=location,
+                category=category,
+                timestamp=timestamp,
+                asset=asset,
+                amount=amount,
+                fee_asset=asset,
+                fee=Fee(ZERO),
+                link='',
+            )
+            self.db.add_asset_movements([asset_movement])
+        else:
+            raise UnsupportedCointrackingEntry(
+                f'Unknown entrype type "{row_type}" encountered during cointracking '
+                f'data import. Ignoring entry',
+            )
+
+    def import_cointracking_csv(self, filepath: str) -> None:
+        with open(filepath, 'r') as csvfile:
+            data = csv.reader(csvfile, delimiter=',', quotechar='"')
+            next(data)  # skip header row
+            for row in data:
+                try:
+                    self._consume_cointracking_entry(row)
+                except UnknownAsset as e:
+                    self.db.msg_aggregator.add_warning(
+                        f'During cointracking CSV import found action with unknown '
+                        f'asset {e.asset_name}. Ignoring entry',
+                    )
+                    continue
+                except IndexError:
+                    self.db.msg_aggregator.add_warning(
+                        f'During cointracking CSV import found entry with '
+                        f'unexpected number of columns',
+                    )
+                    continue
+                except DeserializationError as e:
+                    self.db.msg_aggregator.add_warning(
+                        f'Error during cointracking CSV import deserialization. '
+                        f'Error was {str(e)}. Ignoring entry',
+                    )
+                    continue
+                except UnsupportedCointrackingEntry as e:
+                    self.db.msg_aggregator.add_warning(str(e))
+                    continue
+
+        return None
diff --git a/rotkehlchen/rotkehlchen.py b/rotkehlchen/rotkehlchen.py
--- a/rotkehlchen/rotkehlchen.py
+++ b/rotkehlchen/rotkehlchen.py
@@ -12,6 +12,7 @@
 from rotkehlchen.blockchain import Blockchain
 from rotkehlchen.constants import CACHE_RESPONSE_FOR_SECS
 from rotkehlchen.constants.assets import A_USD, S_USD
+from rotkehlchen.data.importer import DataImporter
 from rotkehlchen.data_handler import DataHandler
 from rotkehlchen.errors import (
     AuthenticationError,
@@ -137,6 +138,7 @@ def unlock_user(
         # unlock or create the DB
         self.password = password
         self.user_directory = self.data.unlock(user, password, create_new)
+        self.data_importer = DataImporter(db=self.data.db)
         self.last_data_upload_ts = self.data.db.get_last_data_upload_ts()
         self.premium_sync_manager = PremiumSyncManager(data=self.data, password=password)
         try:
@@ -214,6 +216,7 @@ def logout(self) -> None:
 
         del self.accountant
         del self.trades_historian
+        del self.data_importer
 
         if self.premium is not None:
             del self.premium
diff --git a/rotkehlchen/server.py b/rotkehlchen/server.py
--- a/rotkehlchen/server.py
+++ b/rotkehlchen/server.py
@@ -440,6 +440,20 @@ def version_check():
             url = result[2]
         return {'our_version': our_version, 'latest_version': latest_version, 'url': url}
 
+    def import_data_from(self, source: str, filepath: str):
+        """Imports data from external location `source` via file `filepath`"""
+        if not self.rotkehlchen.user_is_logged_in:
+            return {'result': False, 'message': 'Cant import data. No user is logged in'}
+
+        if not source == 'cointracking_info':
+            return {
+                'result': False,
+                'message': f'Cant import data from unknown location {source}',
+            }
+
+        self.rotkehlchen.data_importer.import_cointracking_csv(filepath)
+        return {'result': True, 'message': ''}
+
     def unlock_user(
             self,
             user: str,


Test patch:
diff --git a/rotkehlchen/tests/data/cointracking_trades_list.csv b/rotkehlchen/tests/data/cointracking_trades_list.csv
new file mode 100644
--- /dev/null
+++ b/rotkehlchen/tests/data/cointracking_trades_list.csv
@@ -0,0 +1,8 @@
+﻿"","Type","Buy","Cur.","Sell","Cur.","Exchange","Trade Group","Comment","Trade Date"
+"","Withdrawal","-","","0.00136700","ETH","ETH Transaction","","","20.09.2019 13:00"
+"","Deposit","0.02000000","ETH","-","","ETH Transaction","","","20.09.2019 02:01"
+"","Gift/Tip","0.00100000","BTC","-","","no exchange","what is trade group?","Just a small gift from someone","02.09.2019 10:00"
+"","Deposit","0.33676476","BTC","-","","BTC Transaction","","","02.09.2019 09:29"
+"","Deposit","5.00000000","XMR","-","","Poloniex","","","15.08.2019 05:57"
+"","Trade","0.05772716","ETH","10.99000000","EUR","Coinbase","","","24.08.2019 23:01"
+"","Withdrawal","-","","0.05770427","ETH","Coinbase","","","25.08.2019 09:42"
\ No newline at end of file
diff --git a/rotkehlchen/tests/fixtures/rotkehlchen.py b/rotkehlchen/tests/fixtures/rotkehlchen.py
--- a/rotkehlchen/tests/fixtures/rotkehlchen.py
+++ b/rotkehlchen/tests/fixtures/rotkehlchen.py
@@ -5,6 +5,7 @@
 
 import pytest
 
+from rotkehlchen.data.importer import DataImporter
 from rotkehlchen.history import TradesHistorian
 from rotkehlchen.premium.premium import Premium
 from rotkehlchen.premium.sync import PremiumSyncManager
@@ -75,6 +76,7 @@ def initialize_mock_rotkehlchen_instance(
         # Unlock must come after we have set the aggregator if we are to get the
         # messages caused by DB initialization
         rotki.data.unlock(username, db_password, create_new=True)
+        rotki.data_importer = DataImporter(db=rotki.data.db)
         rotki.password = db_password
         # Remember accountant fixture has a mocked accounting data dir
         # different to the usual user one
diff --git a/rotkehlchen/tests/test_data_import.py b/rotkehlchen/tests/test_data_import.py
new file mode 100644
--- /dev/null
+++ b/rotkehlchen/tests/test_data_import.py
@@ -0,0 +1,73 @@
+import os
+
+from rotkehlchen.constants.assets import A_BTC, A_ETH
+from rotkehlchen.constants.misc import ZERO
+from rotkehlchen.exchanges.data_structures import AssetMovement, Trade
+from rotkehlchen.fval import FVal
+from rotkehlchen.tests.utils.constants import A_XMR
+from rotkehlchen.typing import AssetMovementCategory, Location, TradeType
+
+
+def test_cointracking_data_import(rotkehlchen_server):
+    dir_path = os.path.dirname(os.path.realpath(__file__))
+    filepath = os.path.join(dir_path, 'data', 'cointracking_trades_list.csv')
+
+    # Check that for unknown source we get an error
+    response = rotkehlchen_server.import_data_from(source='other_source', filepath=filepath)
+    assert response['result'] is False
+    assert 'unknown location' in response['message']
+
+    # Check that the test cointracking data are imported succesfully
+    rotkehlchen_server.import_data_from(source='cointracking_info', filepath=filepath)
+    trades = rotkehlchen_server.rotkehlchen.data.db.get_trades()
+    asset_movements = rotkehlchen_server.rotkehlchen.data.db.get_asset_movements()
+    warnings = rotkehlchen_server.rotkehlchen.msg_aggregator.consume_warnings()
+    errors = rotkehlchen_server.rotkehlchen.msg_aggregator.consume_warnings()
+    assert len(errors) == 0
+    assert len(warnings) == 3
+
+    expected_trades = [Trade(
+        timestamp=1566687660,
+        location=Location.COINBASE,
+        pair='ETH_EUR',
+        trade_type=TradeType.BUY,
+        amount=FVal('0.05772716'),
+        rate=FVal('190.3783245183029963712055123'),
+        fee=ZERO,
+        fee_currency=A_ETH,
+        link='',
+        notes='',
+    ), Trade(
+        timestamp=1567418400,
+        location=Location.EXTERNAL,
+        pair='BTC_USD',
+        trade_type=TradeType.BUY,
+        amount=FVal('0.00100000'),
+        rate=ZERO,
+        fee=ZERO,
+        fee_currency=A_BTC,
+        link='',
+        notes='Just a small gift from someone',
+    )]
+    assert expected_trades == trades
+
+    expected_movements = [AssetMovement(
+        location=Location.POLONIEX,
+        category=AssetMovementCategory.DEPOSIT,
+        timestamp=1565848620,
+        asset=A_XMR,
+        amount=FVal('5'),
+        fee_asset=A_XMR,
+        fee=ZERO,
+        link='',
+    ), AssetMovement(
+        location=Location.COINBASE,
+        category=AssetMovementCategory.WITHDRAWAL,
+        timestamp=1566726120,
+        asset=A_ETH,
+        amount=FVal('0.05770427'),
+        fee_asset=A_ETH,
+        fee=ZERO,
+        link='',
+    )]
+    assert expected_movements == asset_movements