Repository: freqtrade/freqtrade

Issue:
Analyzed dataframe returns future row in backtesting.
  * Freqtrade Version: freqtrade docker-9e6ed5ad
  
In Freqtrade develop branch during backtesting in custom_sell() there is one row at the end of the analyzed dataframe that shouldn't be there.

I put a print statement in custom_sell() so it looks like this:
```
        df, _ = self.dp.get_analyzed_dataframe(pair, self.timeframe)

        print(current_time)
        print(df)

```
This is the output from develop.
```
2021-08-08 13:15:00+00:00

                         date      open      high       low     close       volume
73  2021-08-08 00:15:00+00:00  0.009823  0.011100  0.009592  0.009601  102195677.0
74  2021-08-08 00:30:00+00:00  0.009600  0.009662  0.009501  0.009600   12007259.0
75  2021-08-08 00:45:00+00:00  0.009621  0.009736  0.009540  0.009621   10980796.0
76  2021-08-08 01:00:00+00:00  0.009639  0.009643  0.009359  0.009370   22532044.0
77  2021-08-08 01:15:00+00:00  0.009374  0.009393  0.009219  0.009287    7555548.0
...
121 2021-08-08 12:15:00+00:00  0.009464  0.009513  0.009447  0.009510    1433469.0
122 2021-08-08 12:30:00+00:00  0.009500  0.009513  0.009467  0.009493     822540.0
123 2021-08-08 12:45:00+00:00  0.009479  0.009503  0.009467  0.009475     537343.0
124 2021-08-08 13:00:00+00:00  0.009489  0.009917  0.009472  0.009791   25718118.0
125 2021-08-08 13:15:00+00:00  0.009798  0.009950  0.009541  0.009555   17794229.0

```
And this from stable.
```
2021-08-08 13:15:00+00:00

                         date      open      high       low     close       volume
72  2021-08-08 00:00:00+00:00  0.009360  0.009837  0.009273  0.009823   26747579.0
73  2021-08-08 00:15:00+00:00  0.009823  0.011100  0.009592  0.009601  102195677.0
74  2021-08-08 00:30:00+00:00  0.009600  0.009662  0.009501  0.009600   12007259.0
75  2021-08-08 00:45:00+00:00  0.009621  0.009736  0.009540  0.009621   10980796.0
76  2021-08-08 01:00:00+00:00  0.009639  0.009643  0.009359  0.009370   22532044.0
...
120 2021-08-08 12:00:00+00:00  0.009309  0.009472  0.009309  0.009444    1663917.0
121 2021-08-08 12:15:00+00:00  0.009464  0.009513  0.009447  0.009510    1433469.0
122 2021-08-08 12:30:00+00:00  0.009500  0.009513  0.009467  0.009493     822540.0
123 2021-08-08 12:45:00+00:00  0.009479  0.009503  0.009467  0.009475     537343.0
124 2021-08-08 13:00:00+00:00  0.009489  0.009917  0.009472  0.009791   25718118.0

```


Discussion:

Resolving patch:
diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py
--- a/freqtrade/optimize/backtesting.py
+++ b/freqtrade/optimize/backtesting.py
@@ -466,6 +466,8 @@ def backtest(self, processed: Dict,
             for i, pair in enumerate(data):
                 row_index = indexes[pair]
                 try:
+                    # Row is treated as "current incomplete candle".
+                    # Buy / sell signals are shifted by 1 to compensate for this.
                     row = data[pair][row_index]
                 except IndexError:
                     # missing Data for one pair at the end.
@@ -476,8 +478,8 @@ def backtest(self, processed: Dict,
                 if row[DATE_IDX] > tmp:
                     continue
 
-                row_index += 1
                 self.dataprovider._set_dataframe_max_index(row_index)
+                row_index += 1
                 indexes[pair] = row_index
 
                 # without positionstacking, we can only have one open trade per pair.


Test patch:
diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py
--- a/tests/optimize/test_backtesting.py
+++ b/tests/optimize/test_backtesting.py
@@ -1,6 +1,7 @@
 # pragma pylint: disable=missing-docstring, W0212, line-too-long, C0103, unused-argument
 
 import random
+from datetime import timedelta
 from pathlib import Path
 from unittest.mock import MagicMock, PropertyMock
 
@@ -741,8 +742,13 @@ def test_backtest_alternate_buy_sell(default_conf, fee, mocker, testdatadir):
     # 100 buys signals
     results = result['results']
     assert len(results) == 100
-    # Cached data should be 200 (no change since required_startup is 0)
-    assert len(backtesting.dataprovider.get_analyzed_dataframe('UNITTEST/BTC', '1m')[0]) == 200
+    # Cached data should be 199 (missing 1 candle at the start)
+    analyzed_df = backtesting.dataprovider.get_analyzed_dataframe('UNITTEST/BTC', '1m')[0]
+    assert len(analyzed_df) == 199
+    # Expect last candle to be 1 below end date (as the last candle is assumed as "incomplete"
+    # during backtesting)
+    expected_last_candle_date = backtest_conf['end_date'] - timedelta(minutes=1)
+    assert analyzed_df.iloc[-1]['date'].to_pydatetime() == expected_last_candle_date
 
     # One trade was force-closed at the end
     assert len(results.loc[results['is_open']]) == 0
@@ -774,7 +780,8 @@ def _trend_alternate_hold(dataframe=None, metadata=None):
     data = trim_dictlist(data, -500)
 
     # Remove data for one pair from the beginning of the data
-    data[pair] = data[pair][tres:].reset_index()
+    if tres > 0:
+        data[pair] = data[pair][tres:].reset_index()
     default_conf['timeframe'] = '5m'
 
     backtesting = Backtesting(default_conf)
@@ -800,8 +807,11 @@ def _trend_alternate_hold(dataframe=None, metadata=None):
     assert len(evaluate_result_multi(results['results'], '5m', 3)) == 0
 
     # Cached data correctly removed amounts
-    removed_candles = len(data[pair]) - 1 - backtesting.strategy.startup_candle_count
+    offset = 2 if tres == 0 else 1
+    removed_candles = len(data[pair]) - offset - backtesting.strategy.startup_candle_count
     assert len(backtesting.dataprovider.get_analyzed_dataframe(pair, '5m')[0]) == removed_candles
+    assert len(backtesting.dataprovider.get_analyzed_dataframe(
+        'NXT/BTC', '5m')[0]) == len(data['NXT/BTC']) - 2 - backtesting.strategy.startup_candle_count
 
     backtest_conf = {
         'processed': processed,