this is my code.. import pickle
import numpy as np
import pandas as pd
import logging
from typing import Tuple, Union
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score
from sklearn.model_selection import KFold, train_test_split
from keras import layers, models
from keras.callbacks import EarlyStopping
import keras_tuner as kt
from datetime import datetime, timedelta
import os
import schedule
import time

# Create the "ssn" directory if it doesn't exist
os.makedirs("exelentmodels", exist_ok=True)

# Configure logging
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')

# Function to evaluate model
def evaluate_model(model: models.Sequential, X: np.ndarray, y: np.ndarray) -> Tuple[float, float, float, float]:
    try:
        predictions = model.predict(X)
        mse = mean_squared_error(y, predictions)
        mae = mean_absolute_error(y, predictions)
        rmse = np.sqrt(mse)
        r_squared = r2_score(y, predictions)
        logging.info(f"MSE: {mse}")
        logging.info(f"MAE: {mae}")
        logging.info(f"RMSE: {rmse}")
        logging.info(f"R²: {r_squared}")
        return mse, mae, rmse, r_squared
    except Exception as e:
        logging.error(f"Error in evaluate_model: {str(e)}")
        return None, None, None, None

# Function to calculate RSI
def calculate_rsi(prices, window=14):
    delta = prices.diff()
    gain = delta.where(delta > 0, 0)
    loss = -delta.where(delta < 0, 0)
    avg_gain = gain.rolling(window=window).mean()
    avg_loss = loss.rolling(window=window).mean()
    rs = avg_gain / avg_loss
    rsi = 100 - (100 / (1 + rs))
    return rsi

# Load training data
try:
    data: pd.DataFrame = pd.read_csv("clean_spy_data_with_rsi.csv")
except Exception as e:
    logging.error(f"Error loading data: {str(e)}")
    raise

# Separate features and target
features: list[str] = ["Open", "High", "Low", "Volume", "RSI"]
target: str = "Close"
X: pd.DataFrame = data[features]
y: pd.Series = data[target]

# Data scaling
try:
    scaler: StandardScaler = StandardScaler()
    X_scaled: np.ndarray = scaler.fit_transform(X)
except Exception as e:
    logging.error(f"Error in data scaling: {str(e)}")
    raise

# Check for NaNs in scaled data
if np.isnan(X_scaled).any():
    raise ValueError("NaNs found in scaled data")

# Reshape the input data
X_scaled_reshaped: np.ndarray = X_scaled.reshape(-1, 1, X_scaled.shape[1])

# Split the data into train, validation, and test sets
X_train_val, X_test, y_train_val, y_test = train_test_split(X_scaled_reshaped, y, test_size=0.2, random_state=42)
X_train, X_val, y_train, y_val = train_test_split(X_train_val, y_train_val, test_size=0.25, random_state=42)

# Model creation functions
def create_lstm_model(hp):
    n_features = X_scaled.shape[1]
    n_hidden = hp.Int('n_hidden', min_value=32, max_value=256, step=32)
    dropout_rate = hp.Float('dropout_rate', min_value=0.1, max_value=0.5, step=0.1)

    model = models.Sequential([
        layers.Input(shape=(1, n_features)),
        layers.LSTM(n_hidden, return_sequences=True, activation='tanh', dropout=dropout_rate),
        layers.LSTM(n_hidden, activation='tanh', dropout=dropout_rate),
        layers.Dense(1, activation='linear')
    ])
    model.compile(loss="mse", optimizer="adam")
    return model

def create_mlp_model(hp):
    n_features = X_scaled.shape[1]
    n_hidden = hp.Int('n_hidden', min_value=32, max_value=256, step=32)
    dropout_rate = hp.Float('dropout_rate', min_value=0.1, max_value=0.5, step=0.1)

    model = models.Sequential([
        layers.Input(shape=(1, n_features)),
        layers.Flatten(),
        layers.Dense(n_hidden, activation='relu'),
        layers.Dropout(dropout_rate),
        layers.Dense(n_hidden, activation='relu'),
        layers.Dropout(dropout_rate),
        layers.Dense(1, activation='linear')
    ])
    model.compile(loss="mse", optimizer="adam")
    return model

# Define different architectures for the meta-learner
def create_meta_learner_1(input_dim: int) -> models.Sequential:
    model = models.Sequential([
        layers.Input(shape=(input_dim,)),
        layers.Dense(10, activation="relu"),
        layers.Dense(1, activation="linear")
    ])
    model.compile(loss="mse", optimizer="adam")
    return model

def create_meta_learner_2(input_dim: int) -> models.Sequential:
    model = models.Sequential([
        layers.Input(shape=(input_dim,)),
        layers.Dense(128, activation='relu'),
        layers.Dense(64, activation='relu'),
        layers.Dense(1, activation='linear')
    ])
    model.compile(loss="mse", optimizer="adam")
    return model

# Reset graph function
def reset_graph():
    import tensorflow as tf
    tf.compat.v1.reset_default_graph()

# Hyperparameter tuning for LSTM model
try:
    reset_graph()
    lstm_tuner = kt.RandomSearch(
        create_lstm_model,
        objective='val_loss',
        max_trials=5,
        directory='lstm_tuner',
        project_name='lstm_tuning'
    )

    lstm_tuner.search(X_train, y_train, epochs=100, validation_data=(X_val, y_val), callbacks=[EarlyStopping(patience=10)])
    best_lstm_model = lstm_tuner.get_best_models(num_models=1)[0]
except Exception as e:
    logging.error(f"Error in LSTM hyperparameter tuning: {str(e)}")
    raise

# Hyperparameter tuning for MLP model
try:
    reset_graph()
    mlp_tuner = kt.RandomSearch(
        create_mlp_model,
        objective='val_loss',
        max_trials=5,
        directory='mlp_tuner',
        project_name='mlp_tuning'
    )

    mlp_tuner.search(X_train, y_train, epochs=100, validation_data=(X_val, y_val), callbacks=[EarlyStopping(patience=9)])
    best_mlp_model = mlp_tuner.get_best_models(num_models=1)[0]
except Exception as e:
    logging.error(f"Error in MLP hyperparameter tuning: {str(e)}")
    raise

# Train the best models on all available data
try:
    reset_graph()
    best_lstm_model.fit(X_train, y_train, validation_data=(X_val, y_val), epochs=1000, verbose=1, callbacks=[EarlyStopping(patience=9)])
    evaluate_model(best_lstm_model, X_test, y_test)

    reset_graph()
    best_mlp_model.fit(X_train, y_train, validation_data=(X_val, y_val), epochs=1000, verbose=1, callbacks=[EarlyStopping(patience=9)])
    evaluate_model(best_mlp_model, X_test, y_test)
except Exception as e:
    logging.error(f"Error in model training: {str(e)}")
    raise

# Make ensemble predictions on training data
try:
    lstm_predictions = best_lstm_model.predict(X_train)
    mlp_predictions = best_mlp_model.predict(X_train)

    # Combine predictions into a new dataset
    ensemble_predictions = np.concatenate((lstm_predictions, mlp_predictions), axis=1)

    # Ensure the shapes of ensemble_predictions and y_train match
    assert ensemble_predictions.shape[0] == y_train.shape[0]
except Exception as e:
    logging.error(f"Error in making ensemble predictions: {str(e)}")
    raise

# Train the meta-learners on the combined predictions
meta_learner_1 = create_meta_learner_1(input_dim=ensemble_predictions.shape[1])
meta_learner_2 = create_meta_learner_2(input_dim=ensemble_predictions.shape[1])

logging.info("Training meta-learner 1...")
try:
    meta_learner_1.fit(ensemble_predictions, y_train, epochs=100, verbose=1)
except Exception as e:
    logging.error(f"Error occurred during training meta-learner 1: {str(e)}")

logging.info("Training meta-learner 2...")
try:
    meta_learner_2.fit(ensemble_predictions, y_train, epochs=100, verbose=1)
except Exception as e:
    logging.error(f"Error occurred during training meta-learner 2: {str(e)}")

# Save models and scaler for future use
try:
    best_lstm_model.save('exelentmodels/lstm_model.keras')
    best_mlp_model.save('exelentmodels/mlp_model.keras')
    meta_learner_1.save('exelentmodels/meta_learner_1.keras')
    meta_learner_2.save('exelentmodels/meta_learner_2.keras')
    with open("exelentmodels/scaler.pkl", "wb") as f:
        pickle.dump(scaler, f)
except Exception as e:
    logging.error(f"Error occurred during model and scaler saving: {str(e)}")

# Evaluate models on the test set
logging.info("Evaluating models on the held-out test set...")
evaluate_model(best_lstm_model, X_test, y_test)
evaluate_model(best_mlp_model, X_test, y_test)

# Make ensemble predictions on the test data
try:
    lstm_predictions_test = best_lstm_model.predict(X_test)
    mlp_predictions_test = best_mlp_model.predict(X_test)

    # Combine predictions into a new dataset
    ensemble_predictions_test = np.concatenate((lstm_predictions_test, mlp_predictions_test), axis=1)
except Exception as e:
    logging.error(f"Error in making ensemble predictions on test data: {str(e)}")
    raise

# Evaluate meta-learners on the test set
logging.info("Evaluating meta-learners on the held-out test set...")
evaluate_model(meta_learner_1, ensemble_predictions_test, y_test)
evaluate_model(meta_learner_2, ensemble_predictions_test, y_test)

# Finalize and save any results or models as needed
try:
    best_lstm_model.save('exelentmodels/final_lstm_model.keras')
    best_mlp_model.save('exelentmodels/final_mlp_model.keras')
    meta_learner_1.save('exelentmodels/final_meta_learner_1.keras')
    meta_learner_2.save('exelentmodels/final_meta_learner_2.keras')
    with open("exelentmodels/final_scaler.pkl", "wb") as f:
        pickle.dump(scaler, f)
    logging.info("Final models and scaler saved successfully.")
except Exception as e:
    logging.error(f"Error occurred during final model and scaler saving: {str(e)}")

# Function to retrain models
def retrain_models():
    try:
        # Load new data
        new_data = pd.read_csv("clean_spy_data_with_rsi.csv")
        X_new = new_data[features]
        y_new = new_data[target]
        
        # Scale new data
        X_new_scaled = scaler.transform(X_new)
        X_new_reshaped = X_new_scaled.reshape(-1, 1, X_new_scaled.shape[1])
        
        # Retrain LSTM model
        best_lstm_model.fit(X_new_reshaped, y_new, epochs=100, verbose=1, callbacks=[EarlyStopping(patience=10)])
        
        # Retrain MLP model
        best_mlp_model.fit(X_new_reshaped, y_new, epochs=100, verbose=1, callbacks=[EarlyStopping(patience=10)])
        
        # Make ensemble predictions on new data
        lstm_predictions = best_lstm_model.predict(X_new_reshaped)
        mlp_predictions = best_mlp_model.predict(X_new_reshaped)
        ensemble_predictions = np.concatenate((lstm_predictions, mlp_predictions), axis=1)
        
        # Retrain meta-learners
        meta_learner_1.fit(ensemble_predictions, y_new, epochs=100, verbose=1)
        meta_learner_2.fit(ensemble_predictions, y_new, epochs=100, verbose=1)
        
        logging.info("Models retrained successfully.")
        
        # Save retrained models
        best_lstm_model.save('exelentmodels/retrained_lstm_model.keras')
        best_mlp_model.save('exelentmodels/retrained_mlp_model.keras')
        meta_learner_1.save('exelentmodels/retrained_meta_learner_1.keras')
        meta_learner_2.save('exelentmodels/retrained_meta_learner_2.keras')
        with open("exelentmodels/retrained_scaler.pkl", "wb") as f:
            pickle.dump(scaler, f)
        logging.info("Retrained models and scaler saved successfully.")
    except Exception as e:
        logging.error(f"Error occurred during model retraining: {str(e)}")


and these are the results.. 2024-10-09 00:56:44,956 - INFO - Evaluating meta-learners on the held-out test set...
29/29 ━━━━━━━━━━━━━━━━━━━━ 0s 1ms/step                                    test set...
2024-10-09 00:56:45,055 - INFO - MSE: 5.264406198255105
2024-10-09 00:56:45,055 - INFO - MAE: 1.7683418824350234
2024-10-09 00:56:45,055 - INFO - RMSE: 2.2944293840201544
2024-10-09 00:56:45,055 - INFO - R²: 0.9996497162766704
29/29 ━━━━━━━━━━━━━━━━━━━━ 0s 1ms/step
2024-10-09 00:56:45,164 - INFO - MSE: 3.341379053104675
2024-10-09 00:56:45,164 - INFO - MAE: 1.2853619566102996
2024-10-09 00:56:45,165 - INFO - RMSE: 1.8279439414557206
2024-10-09 00:56:45,165 - INFO - R²: 0.9997776708993001
2024-10-09 00:56:45,216 - INFO - Final models and scaler saved successfully. ...So as you can see, everything looks great except the lstm model.. but the ensemble seems to not care and they have excellent results so that being said, would you move on to testing in a live paper trading environment or do you think that leaving that lstm could hurt the predictions in the long run? what would you do? would you replace it with something else?