Repository: freqtrade/freqtrade

Issue:
FTX - Cancelled orders not being handled correctly
Operating system: Ubuntu 20.04
Python Version: 3.8
CCXT version: 1.27.19
Branch: Master
Last Commit ID: 2a9653cd4e75819e22ff66b3d2ce3c7c5a4ca842
  
Describe the problem:

Orders that time out on buy are being correctly cancelled on the exchange, but are not being cancelled by FT and remain open.

Steps to reproduce:

  1. Place buy order at bid price
  2. Wait for FT to timeout and cancel order
  3. Once order cancelled message is recieved on telegram, issue /status
  4. Order is still in list of open orders, but not found on exchange
  
Observed Results:

What happened?

- Timed out order was cancelled on exchange but still in open orders list in FT

What did you expect to happen?

- Timed out order removed from open orders list in FT

Relevant code exceptions or logs:

Log entries for the order cancellation:

```
2020-05-03 13:28:16,404 - freqtrade.freqtradebot - INFO - Found open order for Trade(id=3, pair=LINKBEAR/USD, amount=0.55232558, open_rate=10.32000000, open_since=2020-05-03 17:23:12)
2020-05-03 13:28:16,796 - freqtrade.freqtradebot - INFO - Buy order cancelled due to timeout for Trade(id=3, pair=LINKBEAR/USD, amount=0.55232558, open_rate=10.32000000, open_since=2020-05-03 17:23:12).
2020-05-03 13:28:16,796 - freqtrade.freqtradebot - INFO - Found open order for Trade(id=3, pair=LINKBEAR/USD, amount=0.55000000, open_rate=10.32000000, open_since=2020-05-03 17:23:12)
2020-05-03 13:28:17,485 - freqtrade.freqtradebot - INFO - Applying fee on amount for Trade(id=3, pair=LINKBEAR/USD, amount=0.55000000, open_rate=10.32000000, open_since=2020-05-03 17:23:12) failed: myTrade-Dict empty found
2020-05-03 13:28:17,485 - freqtrade.freqtradebot - INFO - Partial buy order timeout for Trade(id=3, pair=LINKBEAR/USD, amount=0.55000000, open_rate=10.32000000, open_since=2020-05-03 17:23:12).
2020-05-03 13:28:17,486 - freqtrade.rpc.rpc_manager - INFO - Sending rpc message: {'type': status, 'status': 'Remaining buy order for LINKBEAR/USD cancelled due to timeout'}
2020-05-03 13:28:17,918 - freqtrade.wallets - INFO - Wallets synced.
2020-05-03 13:28:17,918 - freqtrade.freqtradebot - INFO - Getting price from order book Bid side.
2020-05-03 13:28:18,108 - freqtrade.freqtradebot - INFO - ...top 1 order book buy rate 10.36000000
2020-05-03 13:28:18,109 - freqtrade.rpc.rpc_manager - INFO - Sending rpc message: {'type': buy_cancel, 'exchange': 'Ftx', 'pair': 'LINKBEAR/USD', 'limit': 10.32, 'order_type': 'limit', 'stake_amount': 5.676000000000001, 'stake_currency':
 'USD', 'fiat_currency': 'USD', 'amount': 0.55, 'open_date': datetime.datetime(2020, 5, 3, 17, 23, 12, 929777), 'current_rate': 10.36}
```
ccxt fetch_orders() result for this transaction after the cancellation is completed:

```
{
   'info': {
      'avgFillPrice': None,
      'clientId': None, 
      'createdAt': '2020-05-03T17:23:12.802235+00:00', 
      'filledSize': 0.0, 
      'future': None, 
      'id': 5047991688, 
      'ioc': False, 
      'liquidation': False, 
      'market': 'LINKBEAR/USD', 
      'postOnly': False, 
      'price': 10.3225, 
      'reduceOnly': False, 
      'remainingSize': 0.0, 
      'side': 'buy', 
      'size': 0.55, 
      'status': 'closed', 
      'type': 'limit'
   }, 
   'id': '5047991688', 
   'clientOrderId': None, 
   'timestamp': 1588526592802, 
   'datetime': '2020-05-03T17:23:12.802Z', 
   'lastTradeTimestamp': None, 
   'symbol': 'LINKBEAR/USD', 
   'type': 'limit', 
   'side': 'buy', 
   'price': 10.3225, 
   'amount': 0.55, 
   'cost': 0.0, 
   'average': None, 
   'filled': 0.0, 
   'remaining': 0.0, 
   'status': 'closed',
    'fee': None, 
   'trades': None
}
```
Note that the filled and remaining amounts are both == 0 in the ccxt results for this transaction...


Discussion:
As said on slack - all other exchanges i've looked at so far do fill "remaining" with "amount - filled"- but ftx does not.

Shouldn't be a big problem - but is an anoyance nonetheless, as they tend to deviate from the "default" in many occations (to be clear, it's NOT ccxt's fault - it's clearly visible in the 'info' dict that the info is comming from the exchange).
Added the following to freqtradeboy.py in handle_timeed_out_limit_buy() which seems to have solved the issue:

```
# Some exchanges (FTX) don't report the remaining value correctly, work around this...
if (corder['remaining'] == 0) and (corder['filled'] == 0):
   corder['remaining'] = corder['amount'] - corder['filled']
```

Resolving patch:
diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py
--- a/freqtrade/freqtradebot.py
+++ b/freqtrade/freqtradebot.py
@@ -898,7 +898,8 @@ def handle_timedout_limit_buy(self, trade: Trade, order: Dict) -> bool:
         Buy timeout - cancel order
         :return: True if order was fully cancelled
         """
-        if order['status'] != 'canceled':
+        # Cancelled orders may have the status of 'canceled' or 'closed'
+        if order['status'] not in ('canceled', 'closed'):
             reason = "cancelled due to timeout"
             corder = self.exchange.cancel_order_with_result(trade.open_order_id, trade.pair,
                                                             trade.amount)
@@ -909,7 +910,10 @@ def handle_timedout_limit_buy(self, trade: Trade, order: Dict) -> bool:
 
         logger.info('Buy order %s for %s.', reason, trade)
 
-        if safe_value_fallback(corder, order, 'remaining', 'remaining') == order['amount']:
+        # Using filled to determine the filled amount
+        filled_amount = safe_value_fallback(corder, order, 'filled', 'filled')
+
+        if isclose(filled_amount, 0.0, abs_tol=constants.MATH_CLOSE_PREC):
             logger.info('Buy order fully cancelled. Removing %s from database.', trade)
             # if trade is not partially completed, just delete the trade
             Trade.session.delete(trade)
@@ -921,8 +925,7 @@ def handle_timedout_limit_buy(self, trade: Trade, order: Dict) -> bool:
         # cancel_order may not contain the full order dict, so we need to fallback
         # to the order dict aquired before cancelling.
         # we need to fall back to the values from order if corder does not contain these keys.
-        trade.amount = order['amount'] - safe_value_fallback(corder, order,
-                                                             'remaining', 'remaining')
+        trade.amount = filled_amount
         trade.stake_amount = trade.amount * trade.open_rate
         self.update_trade_state(trade, corder, trade.amount)
 


Test patch:
diff --git a/tests/conftest.py b/tests/conftest.py
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -877,6 +877,99 @@ def limit_buy_order_old_partial_canceled(limit_buy_order_old_partial):
     return res
 
 
+@pytest.fixture(scope='function')
+def limit_buy_order_canceled_empty(request):
+    # Indirect fixture
+    # Documentation:
+    # https://docs.pytest.org/en/latest/example/parametrize.html#apply-indirect-on-particular-arguments
+
+    exchange_name = request.param
+    if exchange_name == 'ftx':
+        return {
+            'info': {},
+            'id': '1234512345',
+            'clientOrderId': None,
+            'timestamp': arrow.utcnow().shift(minutes=-601).timestamp,
+            'datetime': arrow.utcnow().shift(minutes=-601).isoformat(),
+            'lastTradeTimestamp': None,
+            'symbol': 'LTC/USDT',
+            'type': 'limit',
+            'side': 'buy',
+            'price': 34.3225,
+            'amount': 0.55,
+            'cost': 0.0,
+            'average': None,
+            'filled': 0.0,
+            'remaining': 0.0,
+            'status': 'closed',
+            'fee': None,
+            'trades': None
+        }
+    elif exchange_name == 'kraken':
+        return {
+            'info': {},
+            'id': 'AZNPFF-4AC4N-7MKTAT',
+            'clientOrderId': None,
+            'timestamp': arrow.utcnow().shift(minutes=-601).timestamp,
+            'datetime': arrow.utcnow().shift(minutes=-601).isoformat(),
+            'lastTradeTimestamp': None,
+            'status': 'canceled',
+            'symbol': 'LTC/USDT',
+            'type': 'limit',
+            'side': 'buy',
+            'price': 34.3225,
+            'cost': 0.0,
+            'amount': 0.55,
+            'filled': 0.0,
+            'average': 0.0,
+            'remaining': 0.55,
+            'fee': {'cost': 0.0, 'rate': None, 'currency': 'USDT'},
+            'trades': []
+        }
+    elif exchange_name == 'binance':
+        return {
+            'info': {},
+            'id': '1234512345',
+            'clientOrderId': 'alb1234123',
+            'timestamp': arrow.utcnow().shift(minutes=-601).timestamp,
+            'datetime': arrow.utcnow().shift(minutes=-601).isoformat(),
+            'lastTradeTimestamp': None,
+            'symbol': 'LTC/USDT',
+            'type': 'limit',
+            'side': 'buy',
+            'price': 0.016804,
+            'amount': 0.55,
+            'cost': 0.0,
+            'average': None,
+            'filled': 0.0,
+            'remaining': 0.55,
+            'status': 'canceled',
+            'fee': None,
+            'trades': None
+        }
+    else:
+        return {
+            'info': {},
+            'id': '1234512345',
+            'clientOrderId': 'alb1234123',
+            'timestamp': arrow.utcnow().shift(minutes=-601).timestamp,
+            'datetime': arrow.utcnow().shift(minutes=-601).isoformat(),
+            'lastTradeTimestamp': None,
+            'symbol': 'LTC/USDT',
+            'type': 'limit',
+            'side': 'buy',
+            'price': 0.016804,
+            'amount': 0.55,
+            'cost': 0.0,
+            'average': None,
+            'filled': 0.0,
+            'remaining': 0.55,
+            'status': 'canceled',
+            'fee': None,
+            'trades': None
+        }
+
+
 @pytest.fixture
 def limit_sell_order():
     return {
diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py
--- a/tests/test_freqtradebot.py
+++ b/tests/test_freqtradebot.py
@@ -2313,19 +2313,41 @@ def test_handle_timedout_limit_buy(mocker, caplog, default_conf, limit_buy_order
     Trade.session = MagicMock()
     trade = MagicMock()
     trade.pair = 'LTC/ETH'
-    limit_buy_order['remaining'] = limit_buy_order['amount']
+    limit_buy_order['filled'] = 0.0
+    limit_buy_order['status'] = 'open'
     assert freqtrade.handle_timedout_limit_buy(trade, limit_buy_order)
     assert cancel_order_mock.call_count == 1
 
     cancel_order_mock.reset_mock()
-    limit_buy_order['amount'] = 2
+    limit_buy_order['filled'] = 2
     assert not freqtrade.handle_timedout_limit_buy(trade, limit_buy_order)
     assert cancel_order_mock.call_count == 1
 
+    limit_buy_order['filled'] = 2
     mocker.patch('freqtrade.exchange.Exchange.cancel_order', side_effect=InvalidOrderException)
     assert not freqtrade.handle_timedout_limit_buy(trade, limit_buy_order)
 
 
+@pytest.mark.parametrize("limit_buy_order_canceled_empty", ['binance', 'ftx', 'kraken', 'bittrex'],
+                         indirect=['limit_buy_order_canceled_empty'])
+def test_handle_timedout_limit_buy_exchanges(mocker, caplog, default_conf,
+                                             limit_buy_order_canceled_empty) -> None:
+    patch_RPCManager(mocker)
+    patch_exchange(mocker)
+    cancel_order_mock = mocker.patch(
+        'freqtrade.exchange.Exchange.cancel_order_with_result',
+        return_value=limit_buy_order_canceled_empty)
+
+    freqtrade = FreqtradeBot(default_conf)
+
+    Trade.session = MagicMock()
+    trade = MagicMock()
+    trade.pair = 'LTC/ETH'
+    assert freqtrade.handle_timedout_limit_buy(trade, limit_buy_order_canceled_empty)
+    assert cancel_order_mock.call_count == 0
+    assert log_has_re(r'Buy order fully cancelled. Removing .* from database\.', caplog)
+
+
 @pytest.mark.parametrize('cancelorder', [
     {},
     {'remaining': None},
@@ -2347,12 +2369,14 @@ def test_handle_timedout_limit_buy_corder_empty(mocker, default_conf, limit_buy_
     Trade.session = MagicMock()
     trade = MagicMock()
     trade.pair = 'LTC/ETH'
-    limit_buy_order['remaining'] = limit_buy_order['amount']
+    limit_buy_order['filled'] = 0.0
+    limit_buy_order['status'] = 'open'
+
     assert freqtrade.handle_timedout_limit_buy(trade, limit_buy_order)
     assert cancel_order_mock.call_count == 1
 
     cancel_order_mock.reset_mock()
-    limit_buy_order['amount'] = 2
+    limit_buy_order['filled'] = 1.0
     assert not freqtrade.handle_timedout_limit_buy(trade, limit_buy_order)
     assert cancel_order_mock.call_count == 1