Regressor Prediction Makes a Negative Y Offset

### Describe the bug

Hi, I've found a strange situation where regressor prediction makes a negative Y offset. See an orange line on my picture below.
Here is my py file and json data:
[test_scikit.zip](https://github.com/user-attachments/files/22069020/test_scikit.zip)

You will need to change JS_PATH  to your path:
JS_PATH = "D:/Projects/Crpt/CryptoMaiden/Bot/Base/Test/btc_data.json"

You will also need to install a poltly lib.

<img width="1625" height="921" alt="Image" src="https://github.com/user-attachments/assets/e8ac2c73-c556-4570-be28-6bfaedd7ba82" />

I'm new to the Scikit so I decided to report the issue.

I tried RandomForestRegressor, AdaBoostRegressor, GradientBoostingRegressor, HistGradientBoostingRegressor and all of them have this issue. 
To change a regressor in my code you can just comment/uncomment it on lines 89-95.

### Steps/Code to Reproduce

# See my attached ZIP file!

### Expected Results

No negative Y offset.

### Actual Results

Just run my py file!

### Versions

```shell
1.7.1
```

COMMENT:
@adrinjalali sure. Here is the script. All you need to do is just to run it:

```
import time
import numpy as np
import pandas as pd
from datetime import timedelta
import sklearn.ensemble as sklen
from sklearn.metrics import mean_squared_error, accuracy_score
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import MinMaxScaler
from plotly.subplots import make_subplots
import plotly.graph_objects as go
# import xgboost
import requests


# # Download historical data for BTC-USD and save to JSON (uncomment to refresh data)
JS_PATH = "https://gist.githubusercontent.com/mifth/e7a6038e0729c38a882f5d567c8bbbe0/raw/eb63fd24551da8dc74a0fc24554f4e5f37b50320/gistfile1.txt"

# ---- Main Logic Functions ----

def engineer_features(df_ohlc: pd.DataFrame, base_lags: int, indicator_lags: int) -> pd.DataFrame:
    # Use a dictionary to build all new features to avoid DataFrame fragmentation
    new_features = {}

    df: pd.DataFrame = df_ohlc.copy()

    # Add future bar timestamp
    last_timestamp = df.index[-1]
    time_delta = df.index[1] - df.index[0]
    future_timestamp = last_timestamp + time_delta

    future_row = pd.DataFrame(
        [df.iloc[-1].values], 
        columns=df.columns, 
        index=[future_timestamp]
    )

    df = pd.concat([df, future_row])

    # Lags for prices and volume
    for lag in range(1, base_lags + 1):
        new_features[f'close_lag_{lag}'] = df['Close'].shift(lag)
        new_features[f'open_lag_{lag}'] = df['Open'].shift(lag)
        new_features[f'high_lag_{lag}'] = df['High'].shift(lag)
        new_features[f'low_lag_{lag}'] = df['Low'].shift(lag)
        new_features[f'volume_lag_{lag}'] = df['Volume'].shift(lag)

        new_features[f"candle_weakness_lag_{lag}"] = (((df["Open"] - df["Close"]).abs() / (df["High"] - df["Low"])).shift(lag))
        new_features[f"is_green_candle_lag_{lag}"] = (df["Close"] > df["Open"]).astype(bool).shift(lag)

        # compare current candle and previous candle heights
        new_features[f"is_taller_candle_hl_lag_{lag}"] =  ((df["High"] - df["Low"]) - (df["High"] - df["Low"]).shift(1)).shift(lag)
        new_features[f"is_taller_candle_oc_lag_{lag}"] =  ((df["Open"] - df["Close"]).abs() - (df["Open"] - df["Close"]).abs().shift(1)).shift(lag)

        new_features[f"close_return_lag_{lag}"] = df["Close"].pct_change().shift(lag)

    # Day of week feature
    new_features['day_of_week'] = df.index.dayofweek

    # Create the new features DataFrame from the dictionary in one go
    new_features_df = pd.DataFrame(new_features)

    # Concatenate the original DataFrame with the new features
    df_combined = pd.concat([df, new_features_df], axis=1)

    # Targets: predict THIS bar from PREVIOUS bar
    df_combined["target_return"] = df_combined["Close"].pct_change()
    df_combined["target_direction"] = (df_combined["target_return"] > 0).astype(int)

    df_combined.dropna(inplace=True)
    return df_combined


def train_models(X_train, y_train_return, y_train_direction):
    # --- Voting Regressor Model for Price Return ---
    
    # Define your individual regressor pipelines
    # Each pipeline scales the data and then applies a different regression model.
    # reg1 = Pipeline([('scaler', MinMaxScaler()), ('model', sklen.HistGradientBoostingRegressor(max_iter=500, random_state=42))])
    reg2 = Pipeline([('scaler', MinMaxScaler()), ('model', sklen.RandomForestRegressor(n_estimators=500, random_state=35, n_jobs=-1))])
    # reg3 = Pipeline([('scaler', MinMaxScaler()), ('model', sklen.AdaBoostRegressor(n_estimators=500, random_state=42))])
    # reg4 = Pipeline([('scaler', MinMaxScaler()), ('model', sklen.GradientBoostingRegressor(n_estimators=500, random_state=42))])
    # reg5 = Pipeline([('scaler', MinMaxScaler()), ('model', xgboost.XGBRegressor(n_estimators=500, random_state=50))])
    # reg6 = Pipeline([('scaler', MinMaxScaler()), ('model', sklen.ExtraTreesRegressor(n_estimators=500, random_state=42))])

    # Create the VotingRegressor to combine all the models
    # It takes a list of estimators and averages their predictions.
    return_model = sklen.VotingRegressor(
        estimators=[
            # ('hgb', reg1), 
            # ('xgb', reg5),
            ('rf', reg2), 
            # ('ada', reg3), 
            # ('gb', reg4), 
            # ('et', reg6)
        ],
    )
    return_model.fit(X_train, y_train_return)

    # --- Voting Classifier Model for Price Direction ---
    
    # Define your individual classifier pipelines
    # Each pipeline scales the data and then applies a different classification model.
    # clf1 = Pipeline([('scaler', MinMaxScaler()), ('model', sklen.HistGradientBoostingClassifier(max_iter=500, random_state=42))])
    clf2 = Pipeline([('scaler', MinMaxScaler()), ('model', sklen.RandomForestClassifier(n_estimators=500, random_state=35, n_jobs=-1))])
    # clf3 = Pipeline([('scaler', MinMaxScaler()), ('model', sklen.AdaBoostClassifier(n_estimators=500, random_state=42))])
    # clf4 = Pipeline([('scaler', MinMaxScaler()), ('model', sklen.GradientBoostingClassifier(n_estimators=500, random_state=42))])
    # clf5 = Pipeline([('scaler', MinMaxScaler()), ('model', xgboost.XGBClassifier(n_estimators=500, random_state=50))])
    # clf6 = Pipeline([('scaler', MinMaxScaler()), ('model', sklen.ExtraTreesClassifier(n_estimators=500, random_state=42))])

    # Create the VotingClassifier to combine all the models
    direction_model = sklen.VotingClassifier(
        estimators=[
            # ('hgb', clf1), 
            # ('xgb', clf5),
            ('rf', clf2), 
            # ('ada', clf3), 
            # ('gb', clf4), 
            # ('et', clf6)
        ],
        voting='soft',  # Use 'soft' for probability-based voting
    )
    direction_model.fit(X_train, y_train_direction)
    
    return return_model, direction_model

def evaluate_and_visualize(df_original, df_train, df_test, return_model, direction_model, features):
    """
    Evaluates models on the test set and visualizes the results.
    This function does NOT perform any future predictions.
    """
    X_test = df_test[features]
    y_test_return = df_test["target_return"]
    y_test_direction = df_test["target_direction"]

    # Predict on test set using the regression model
    predicted_return_test = return_model.predict(X_test)
    df_test.loc[:, "predicted_return"] = predicted_return_test
    df_test.loc[:, "prob_up"] = direction_model.predict_proba(X_test)[:, 1]

    # # Calculate performance metrics
    # rmse = np.sqrt(mean_squared_error(y_test_return, predicted_return_test))
    # accuracy = accuracy_score(y_test_direction, direction_model.predict(X_test))
    # print(f"RMSE (current-bar return): {rmse:.6f}")
    # print(f"Accuracy (current-bar direction): {accuracy:.4f}")

    # --- Visualization ---
    fig = make_subplots(
        rows=4, cols=1,
        shared_xaxes=True,
        vertical_spacing=0.03,
        row_heights=[1.0, 0.2, 0.2, 0.2],
        subplot_titles=(
            "Candlestick & Predicted Close",
            "Volume",
            "Actual vs. Predicted Returns",
            "Predicted Probability of Up Move"
        )
    )

    # df_viz = pd.concat([df_train, df_test])

    # Candlestick trace (Row 1)
    fig.add_trace(go.Candlestick(
        x=df_original.index,
        open=df_original['Open'],
        high=df_original['High'],
        low=df_original['Low'],
        close=df_original['Close'],
        name='Candles'
    ), row=1, col=1)

    # Buy signals
    buy_ret_threshold = 0.003
    buy_prob_threshold = 0.7
    sell_ret_threshold = -buy_ret_threshold
    sell_prob_threshold = 1 - buy_prob_threshold
    buy_signals = (df_test['prob_up'] > buy_prob_threshold) & (df_test['predicted_return'] > buy_ret_threshold)
    fig.add_trace(go.Scatter(
        x=df_test[buy_signals].index,
        y=df_test[buy_signals]["Close"],
        mode="markers",
        marker=dict(color='green', size=8),
        name='Positive Predictions'
    ), row=1, col=1)

    # Sell signals
    sell_signals = (df_test['prob_up'] < sell_prob_threshold) & (df_test['predicted_return'] < sell_ret_threshold)
    fig.add_trace(go.Scatter(
        x=df_test[sell_signals].index,
        y=df_test[sell_signals]["Close"],
        mode="markers",
        marker=dict(color='red', size=8),
        name='Negative Predictions'
    ), row=1, col=1)

    # Volume (Row 2)
    fig.add_trace(go.Bar(
        x=df_original.index,
        y=df_original["Volume"],
        name='Volume',
        marker=dict(color='purple'),
    ), row=2, col=1)

    # Actual vs. Predicted Returns (Row 3)
    # fig.add_trace(go.Scatter(
    #     x=df_test.index,
    #     y=df_test["target_return"],
    #     name='Actual Return',
    #     line=dict(color='blue', width=1),
    #     mode="lines+markers",
    #     marker=dict(size=4, symbol='circle'),
    # ), row=3, col=1)
    
    fig.add_trace(go.Scatter(
        x=df_test.index,
        y=df_test["predicted_return"],
        name='Predicted Return',
        line=dict(color='orange', width=1),
        mode="lines+markers",
        marker=dict(size=4, symbol='circle'),
    ), row=3, col=1)

    fig.add_hline(y=0, line_dash="dash", line_color="gray", row=3, col=1)
    fig.add_hline(y=buy_ret_threshold, line_dash="dash", line_color="green", row=3, col=1)
    fig.add_hline(y=sell_ret_threshold, line_dash="dash", line_color="red", row=3, col=1)
    fig.update_yaxes(title_text="Return", row=3, col=1)

    # Predicted Probability of Up Move (Row 4)
    fig.add_trace(go.Scatter(
        x=df_test.index,
        y=df_test["prob_up"],
        name='Prob. Up',
        line=dict(color='green'),
        mode="lines+markers",
    ), row=4, col=1)
    fig.add_hline(y=0.5, line_dash="dash", line_color="gray", row=4, col=1)
    fig.add_hline(y=buy_prob_threshold, line_dash="dash", line_color="green", row=4, col=1)
    fig.add_hline(y=sell_prob_threshold, line_dash="dash", line_color="red", row=4, col=1)
    fig.update_yaxes(title_text="P(Up)", row=4, col=1, range=[0, 1])

    fig.update_layout(
        title="ETH-USD Candlestick, Predicted Close & Probability",
        xaxis_title="Date",
        yaxis_title="Price",
        xaxis_rangeslider_visible=False,
        showlegend=True,
        height=650
    )
    
    fig.update_yaxes(title_text="Price", row=1, col=1)
    fig.update_yaxes(title_text="Volume", row=2, col=1)
    fig.update_yaxes(title_text="Return", row=3, col=1)
    fig.update_yaxes(title_text="P(Up)", row=4, col=1)

    fig.show()

RESAMPLE_DICT: dict = {
    'Open': "first", 
    'High': "max",   
    'Low': "min",     
    'Close': "last",  
    'Volume': "sum"   
}

# ---- Main Execution (Revised) ---
def main():
    global JS_PATH
    
    # Load JSON as DataFrame
    resp = requests.get(JS_PATH)
    data = resp.json()

    df = pd.DataFrame(data).dropna()
    df.index = pd.to_datetime(df.index.astype("int64"), unit="ms")
    df: pd.DataFrame = (
        df
        .resample('6h')
        .agg(RESAMPLE_DICT)
        .dropna()
    )
    if 'Volume' in df.columns and df['Volume'].dtype == 'object':
        df['Volume'] = pd.to_numeric(df['Volume'])
    base_lags = 10
    indicator_lags = 10

    df_engineered = engineer_features(df, base_lags, indicator_lags)
    
    split_point = int(len(df_engineered) * 0.8)
    
    df_train = df_engineered.iloc[:split_point].copy()
    df_test = df_engineered.iloc[split_point:].copy()

    # Define the list of features to use for the model
    features = [
        "day_of_week"
    ]

    # Add dynamically generated lagged features
    for lag in range(1, base_lags + 1):
        features.extend([
            f'open_lag_{lag}', f'high_lag_{lag}', f'low_lag_{lag}', 
            f'close_lag_{lag}', f'volume_lag_{lag}', 
            f'candle_weakness_lag_{lag}', f'is_green_candle_lag_{lag}',
            f'is_taller_candle_hl_lag_{lag}', f'is_taller_candle_oc_lag_{lag}',
            f'close_return_lag_{lag}'
        ])
    
    X_train = df_train[features]
    y_train_return = df_train["target_return"]
    y_train_direction = df_train["target_direction"]
    
    start_time = time.time()

    return_model, direction_model = train_models(X_train, y_train_return, y_train_direction)
    evaluate_and_visualize(df, df_train, df_test, return_model, direction_model, features)
    
    end_time = time.time()
    print(f"Time spent: {end_time - start_time:.2f} seconds")

if __name__ == "__main__":
    main()
```

Labels: Bug, Needs Investigation