Repository: rotki/rotki

Issue:
Binance fiat payment can also return cryptoCurrency payments :man_facepalming: 
## Problem Definition

When processing payments in binance we look for key `cryptoCurrency` and the response has `fiatCurrency` as key

## Logs

With sensitive information removed

```
Error processing a binance fiat payment asset_movement={'orderNo': 'REDACTED', 'fiatCurrency': 'EUR', 'indicatedAmount': '1.00', 'amount': '0.80', 'totalFee': '0.20', 'method': 'card', 'status': 'Successful', 'createTime': REDACTED, 'updateTime': REDACTED}, error=Missing key entry for 'cryptoCurrency'.
```


### System Description


Operating system: 
Rotki version:

Binance fiat payment can also return cryptoCurrency payments :man_facepalming: 
## Problem Definition

When processing payments in binance we look for key `cryptoCurrency` and the response has `fiatCurrency` as key

## Logs

With sensitive information removed

```
Error processing a binance fiat payment asset_movement={'orderNo': 'REDACTED', 'fiatCurrency': 'EUR', 'indicatedAmount': '1.00', 'amount': '0.80', 'totalFee': '0.20', 'method': 'card', 'status': 'Successful', 'createTime': REDACTED, 'updateTime': REDACTED}, error=Missing key entry for 'cryptoCurrency'.
```


### System Description


Operating system: 
Rotki version:



Discussion:

Resolving patch:
diff --git a/rotkehlchen/exchanges/binance.py b/rotkehlchen/exchanges/binance.py
--- a/rotkehlchen/exchanges/binance.py
+++ b/rotkehlchen/exchanges/binance.py
@@ -42,14 +42,7 @@
     deserialize_timestamp_from_binance,
     deserialize_timestamp_from_date,
 )
-from rotkehlchen.typing import (
-    ApiKey,
-    ApiSecret,
-    Fee,
-    AssetMovementCategory,
-    Location,
-    Timestamp,
-)
+from rotkehlchen.typing import ApiKey, ApiSecret, AssetMovementCategory, Fee, Location, Timestamp
 from rotkehlchen.user_messages import MessagesAggregator
 from rotkehlchen.utils.misc import ts_now_in_ms
 from rotkehlchen.utils.mixins.cacheable import cache_response_timewise
@@ -913,7 +906,7 @@ def _query_online_fiat_payments(self, start_ts: Timestamp, end_ts: Timestamp) ->
             end_ts=end_ts,
             time_delta=API_TIME_INTERVAL_CONSTRAINT_TS,
             api_type='sapi',
-            method='fiat/orders',
+            method='fiat/payments',
             additional_options={'transactionType': 0},
         )
         log.debug(f'{self.name} fiat buys history result', results_num=len(fiat_buys))
@@ -922,7 +915,7 @@ def _query_online_fiat_payments(self, start_ts: Timestamp, end_ts: Timestamp) ->
             end_ts=end_ts,
             time_delta=API_TIME_INTERVAL_CONSTRAINT_TS,
             api_type='sapi',
-            method='fiat/orders',
+            method='fiat/payments',
             additional_options={'transactionType': 1},
         )
         log.debug(f'{self.name} fiat sells history result', results_num=len(fiat_sells))
@@ -946,7 +939,7 @@ def _deserialize_fiat_payment(
         Can log error/warning and return None if something went wrong at deserialization
         """
         try:
-            if 'status' not in raw_data or raw_data['status'] != 'Successful':
+            if 'status' not in raw_data or raw_data['status'] != 'Completed':
                 log.error(
                     f'Found {str(self.location)} fiat payment with failed status. Ignoring it.',
                 )
@@ -973,7 +966,7 @@ def _deserialize_fiat_payment(
                 amount=obtain_amount,
                 rate=rate,
                 fee=fee,
-                fee_currency=base_asset,
+                fee_currency=quote_asset,
                 link=link_str,
             )
 
@@ -1013,7 +1006,7 @@ def _deserialize_fiat_movement(
         Can log error/warning and return None if something went wrong at deserialization
         """
         try:
-            if 'status' not in raw_data or raw_data['status'] != 'Successful':
+            if 'status' not in raw_data or raw_data['status'] not in ('Successful', 'Finished'):
                 log.error(
                     f'Found {str(self.location)} fiat deposit/withdrawal with failed status. Ignoring it.',  # noqa: E501
                 )
@@ -1151,7 +1144,6 @@ def _api_query_list_within_time_delta(
           - Timestamps are converted to milliseconds.
         """
         results: List[Dict[str, Any]] = []
-
         # Create required time references in milliseconds
         start_ts = Timestamp(start_ts * 1000)
         end_ts = Timestamp(end_ts * 1000)


Test patch:
diff --git a/rotkehlchen/tests/exchanges/test_binance.py b/rotkehlchen/tests/exchanges/test_binance.py
--- a/rotkehlchen/tests/exchanges/test_binance.py
+++ b/rotkehlchen/tests/exchanges/test_binance.py
@@ -11,7 +11,7 @@
 
 from rotkehlchen.assets.asset import Asset
 from rotkehlchen.assets.converters import UNSUPPORTED_BINANCE_ASSETS, asset_from_binance
-from rotkehlchen.constants.assets import A_ADA, A_BNB, A_BTC, A_DOT, A_ETH, A_USDT, A_WBTC
+from rotkehlchen.constants.assets import A_ADA, A_BNB, A_BTC, A_DOT, A_ETH, A_EUR, A_USDT, A_WBTC
 from rotkehlchen.constants.timing import DEFAULT_TIMEOUT_TUPLE
 from rotkehlchen.db.dbhandler import DBHandler
 from rotkehlchen.errors import RemoteError, UnknownAsset, UnsupportedAsset
@@ -24,9 +24,13 @@
 )
 from rotkehlchen.exchanges.data_structures import Location, Trade, TradeType
 from rotkehlchen.fval import FVal
-from rotkehlchen.tests.utils.constants import A_BUSD, A_RDN
+from rotkehlchen.tests.utils.constants import A_BUSD, A_LUNA, A_RDN
 from rotkehlchen.tests.utils.exchanges import (
     BINANCE_DEPOSITS_HISTORY_RESPONSE,
+    BINANCE_FIATBUY_RESPONSE,
+    BINANCE_FIATDEPOSITS_RESPONSE,
+    BINANCE_FIATSELL_RESPONSE,
+    BINANCE_FIATWITHDRAWS_RESPONSE,
     BINANCE_MYTRADES_RESPONSE,
     BINANCE_WITHDRAWALS_HISTORY_RESPONSE,
     assert_binance_asset_movements_result,
@@ -223,22 +227,46 @@ def test_binance_query_balances_include_features(function_scope_binance):
     assert 'unsupported binance asset ETF' in warnings[1]
 
 
+TIMESTAMPS_RE = re.compile(r'.*&startTime\=(.*?)&endTime\=(.*?)&')
+
+
 def test_binance_query_trade_history(function_scope_binance):
     """Test that turning a binance trade as returned by the server to our format works"""
     binance = function_scope_binance
 
     def mock_my_trades(url, **kwargs):  # pylint: disable=unused-argument
-        if 'symbol=BNBBTC' in url:
-            text = BINANCE_MYTRADES_RESPONSE
+        if 'myTrades' in url:
+            if 'symbol=BNBBTC' in url:
+                text = BINANCE_MYTRADES_RESPONSE
+            else:
+                text = '[]'
+        elif 'fiat/payments' in url:
+            match = TIMESTAMPS_RE.search(url)
+            assert match
+            groups = match.groups()
+            assert len(groups) == 2
+            from_ts, to_ts = [int(x) for x in groups]
+            if 'transactionType=0' in url:
+                if from_ts < 1624529919000 < to_ts:
+                    text = BINANCE_FIATBUY_RESPONSE
+                else:
+                    text = '[]'
+            elif 'transactionType=1' in url:
+                if from_ts < 1628529919000 < to_ts:
+                    text = BINANCE_FIATSELL_RESPONSE
+                else:
+                    text = '[]'
+            else:
+                raise AssertionError('Unexpected binance request in test')
         else:
-            text = '[]'
+            raise AssertionError('Unexpected binance request in test')
 
         return MockResponse(200, text)
 
     with patch.object(binance.session, 'get', side_effect=mock_my_trades):
-        trades = binance.query_trade_history(start_ts=0, end_ts=1564301134, only_cache=False)
+        trades = binance.query_trade_history(start_ts=0, end_ts=1638529919, only_cache=False)
 
-    expected_trade = Trade(
+    expected_trades = [Trade(
         timestamp=1499865549,
         location=Location.BINANCE,
         base_asset=A_BNB,
@@ -249,10 +277,31 @@ def mock_my_trades(url, **kwargs):  # pylint: disable=unused-argument
         fee=FVal('10.10000000'),
         fee_currency=A_BNB,
         link='28457',
-    )
+    ), Trade(
+        timestamp=1624529919,
+        location=Location.BINANCE,
+        base_asset=A_LUNA,
+        quote_asset=A_EUR,
+        trade_type=TradeType.BUY,
+        amount=FVal('4.462'),
+        rate=FVal('4.437472'),
+        fee=FVal('0.2'),
+        fee_currency=A_EUR,
+        link='353fca443f06466db0c4dc89f94f027a',
+    ), Trade(
+        timestamp=1628529919,
+        location=Location.BINANCE,
+        base_asset=A_ETH,
+        quote_asset=A_EUR,
+        trade_type=TradeType.SELL,
+        amount=FVal('4.462'),
+        rate=FVal('4.437472'),
+        fee=FVal('0.2'),
+        fee_currency=A_EUR,
+        link='463fca443f06466db0c4dc89f94f027a',
+    )]
 
-    assert len(trades) == 1
-    assert trades[0] == expected_trade
+    assert trades == expected_trades
 
 
 def test_binance_query_trade_history_unexpected_data(function_scope_binance):
@@ -369,14 +418,42 @@ def test_binance_query_deposits_withdrawals(function_scope_binance):
     prevent requesting with a time delta.
     """
     start_ts = 1508022000  # 2017-10-15
-    end_ts = 1508540400  # 2017-10-21 (less than 90 days since `start_ts`)
+    # end_ts = 1508540400  # 2017-10-21 (less than 90 days since `start_ts`)
+    end_ts = 1636400907
     binance = function_scope_binance
 
     def mock_get_deposit_withdrawal(url, **kwargs):  # pylint: disable=unused-argument
-        if 'deposit' in url:
-            response_str = BINANCE_DEPOSITS_HISTORY_RESPONSE
+        match = TIMESTAMPS_RE.search(url)
+        assert match
+        groups = match.groups()
+        assert len(groups) == 2
+        from_ts, to_ts = [int(x) for x in groups]
+        if 'capital/deposit' in url:
+            if from_ts >= 1508022000000 and to_ts <= 1515797999999:
+                response_str = BINANCE_DEPOSITS_HISTORY_RESPONSE
+            else:
+                response_str = '[]'
+        elif 'capital/withdraw' in url:
+            if from_ts <= 1508198532000 <= to_ts:
+                response_str = BINANCE_WITHDRAWALS_HISTORY_RESPONSE
+            else:
+                response_str = '[]'
+        elif 'fiat/orders' in url:
+            from_ts, to_ts = [int(x) for x in groups]
+            if 'transactionType=0' in url:
+                if from_ts < 1626144956000 < to_ts:
+                    response_str = BINANCE_FIATDEPOSITS_RESPONSE
+                else:
+                    response_str = '[]'
+            elif 'transactionType=1' in url:
+                if from_ts < 1636144956000 < to_ts:
+                    response_str = BINANCE_FIATWITHDRAWS_RESPONSE
+                else:
+                    response_str = '[]'
+            else:
+                raise AssertionError('Unexpected binance request in test')
         else:
-            response_str = BINANCE_WITHDRAWALS_HISTORY_RESPONSE
+            raise AssertionError('Unexpected binance request in test')
 
         return MockResponse(200, response_str)
 
@@ -390,7 +467,11 @@ def mock_get_deposit_withdrawal(url, **kwargs):  # pylint: disable=unused-argume
     warnings = binance.msg_aggregator.consume_warnings()
     assert len(errors) == 0
     assert len(warnings) == 0
-    assert_binance_asset_movements_result(movements=movements, location=Location.BINANCE)
+    assert_binance_asset_movements_result(
+        movements=movements,
+        location=Location.BINANCE,
+        got_fiat=True,
+    )
 
 
 def test_binance_query_deposits_withdrawals_unexpected_data(function_scope_binance):
@@ -808,7 +889,7 @@ def test_binance_query_trade_history_custom_markets(function_scope_binance, user
 
     def mock_my_trades(url, timeout):  # pylint: disable=unused-argument
         nonlocal count
-        if '/fiat/orders' not in url:
+        if '/fiat/payments' not in url:
             count += 1
             market = p.search(url).group()[7:]
             assert market in markets and market not in seen
diff --git a/rotkehlchen/tests/exchanges/test_binance_us.py b/rotkehlchen/tests/exchanges/test_binance_us.py
--- a/rotkehlchen/tests/exchanges/test_binance_us.py
+++ b/rotkehlchen/tests/exchanges/test_binance_us.py
@@ -6,7 +6,7 @@
 from rotkehlchen.assets.converters import UNSUPPORTED_BINANCE_ASSETS, asset_from_binance
 from rotkehlchen.constants.assets import A_BNB, A_BTC
 from rotkehlchen.errors import UnknownAsset, UnsupportedAsset
-from rotkehlchen.exchanges.binance import Binance, BINANCEUS_BASE_URL
+from rotkehlchen.exchanges.binance import BINANCEUS_BASE_URL, Binance
 from rotkehlchen.exchanges.data_structures import Trade, TradeType
 from rotkehlchen.fval import FVal
 from rotkehlchen.tests.utils.exchanges import (
@@ -126,4 +126,8 @@ def mock_get_deposit_withdrawal(url, **kwargs):  # pylint: disable=unused-argume
     assert len(warnings) == 0
 
     assert len(movements) == 4
-    assert_binance_asset_movements_result(movements=movements, location=Location.BINANCEUS)
+    assert_binance_asset_movements_result(
+        movements=movements,
+        location=Location.BINANCEUS,
+        got_fiat=False,
+    )
diff --git a/rotkehlchen/tests/utils/constants.py b/rotkehlchen/tests/utils/constants.py
--- a/rotkehlchen/tests/utils/constants.py
+++ b/rotkehlchen/tests/utils/constants.py
@@ -27,6 +27,7 @@
 A_YAM_V1 = EthereumToken('0x0e2298E3B3390e3b945a5456fBf59eCc3f55DA16')
 
 
+A_LUNA = Asset('LUNA-2')
 A_AIR2 = Asset('AIR-2')
 A_SDC = Asset('SDC')
 A_DOGE = Asset('DOGE')
diff --git a/rotkehlchen/tests/utils/exchanges.py b/rotkehlchen/tests/utils/exchanges.py
--- a/rotkehlchen/tests/utils/exchanges.py
+++ b/rotkehlchen/tests/utils/exchanges.py
@@ -5,7 +5,7 @@
 from unittest.mock import patch
 
 from rotkehlchen.assets.asset import Asset
-from rotkehlchen.constants.assets import A_ETH
+from rotkehlchen.constants.assets import A_ETH, A_EUR
 from rotkehlchen.constants.misc import ZERO
 from rotkehlchen.db.dbhandler import DBHandler
 from rotkehlchen.exchanges.binance import (
@@ -275,6 +275,7 @@
     "totalFlexibleInUSDT": "77.13289230"
 }"""
 
+
 BINANCE_MYTRADES_RESPONSE = """
 [
     {
@@ -292,6 +293,47 @@
     "isBestMatch": true
     }]"""
 
+
+BINANCE_FIATBUY_RESPONSE = """{
+   "code": "000000",
+   "message": "success",
+   "data": [{
+      "orderNo": "353fca443f06466db0c4dc89f94f027a",
+      "sourceAmount": "20.0",
+      "fiatCurrency": "EUR",
+      "obtainAmount": "4.462",
+      "cryptoCurrency": "LUNA",
+      "totalFee": "0.2",
+      "price": "4.437472",
+      "status": "Completed",
+      "createTime": 1624529919000,
+      "updateTime": 1624529919000
+   }],
+   "total": 1,
+   "success": true
+}"""
+
+
+BINANCE_FIATSELL_RESPONSE = """{
+   "code": "000000",
+   "message": "success",
+   "data": [{
+      "orderNo": "463fca443f06466db0c4dc89f94f027a",
+      "sourceAmount": "20.0",
+      "fiatCurrency": "EUR",
+      "obtainAmount": "4.462",
+      "cryptoCurrency": "ETH",
+      "totalFee": "0.2",
+      "price": "4.437472",
+      "status": "Completed",
+      "createTime": 1628529919000,
+      "updateTime": 1628529919000
+   }],
+   "total": 1,
+   "success": true
+}"""
+
+
 BINANCE_DEPOSITS_HISTORY_RESPONSE = """[
     {
         "insertTime": 1508198532000,
@@ -337,6 +379,48 @@
 ]"""  # noqa: E501
 
 
+BINANCE_FIATDEPOSITS_RESPONSE = """{
+   "code": "000000",
+   "message": "success",
+   "data": [
+   {
+      "orderNo":"7d76d611-0568-4f43-afb6-24cac7767365",
+      "fiatCurrency": "EUR",
+      "indicatedAmount": "10.00",
+      "amount": "10.00",
+      "totalFee": "0.00",
+      "method": "BankAccount",
+      "status": "Successful",
+      "createTime": 1626144956000,
+      "updateTime": 1626400907000
+   }
+   ],
+   "total": 1,
+   "success": true
+}"""
+
+
+BINANCE_FIATWITHDRAWS_RESPONSE = """{
+   "code": "000000",
+   "message": "success",
+   "data": [
+   {
+      "orderNo":"8e76d611-0568-4f43-afb6-24cac7767365",
+      "fiatCurrency": "EUR",
+      "indicatedAmount": "10.00",
+      "amount": "10.00",
+      "totalFee": "0.02",
+      "method": "BankAccount",
+      "status": "Finished",
+      "createTime": 1636144956000,
+      "updateTime": 1636400907000
+   }
+   ],
+   "total": 1,
+   "success": true
+}"""
+
+
 def assert_binance_balances_result(balances: Dict[str, Any]) -> None:
     assert balances['BTC']['amount'] == '4723846.89208129'
     assert balances['BTC']['usd_value'] is not None
@@ -344,8 +428,12 @@ def assert_binance_balances_result(balances: Dict[str, Any]) -> None:
     assert balances['ETH']['usd_value'] is not None
 
 
-def assert_binance_asset_movements_result(movements: List[AssetMovement], location: Location) -> None:  # noqa: E501
-    assert len(movements) == 4
+def assert_binance_asset_movements_result(
+        movements: List[AssetMovement],
+        location: Location,
+        got_fiat: bool,
+) -> None:
+    assert len(movements) == 6 if got_fiat else 4
     assert movements[0].location == location
     assert movements[0].category == AssetMovementCategory.DEPOSIT
     assert movements[0].timestamp == 1508198532
@@ -378,6 +466,23 @@ def assert_binance_asset_movements_result(movements: List[AssetMovement], locati
     assert movements[3].amount == FVal('999.9999')
     assert movements[3].fee == FVal('0.0001')
 
+    if got_fiat:
+        assert movements[4].location == location
+        assert movements[4].category == AssetMovementCategory.DEPOSIT
+        assert movements[4].timestamp == 1626144956
+        assert isinstance(movements[4].asset, Asset)
+        assert movements[4].asset == A_EUR
+        assert movements[4].amount == FVal('10')
+        assert movements[4].fee == FVal('0')
+
+        assert movements[5].location == location
+        assert movements[5].category == AssetMovementCategory.WITHDRAWAL
+        assert movements[5].timestamp == 1636144956
+        assert isinstance(movements[5].asset, Asset)
+        assert movements[5].asset == A_EUR
+        assert movements[5].amount == FVal('10')
+        assert movements[5].fee == FVal('0.02')
+
 
 def assert_poloniex_balances_result(balances: Dict[str, Any]) -> None:
     assert balances['BTC']['amount'] == '5.5'