code:
# %%
from sklearn.preprocessing import StandardScaler
import pandas as pd
import numpy as np
from tensorflow import keras
import joblib
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import LSTM,Dense,Dropout
import os

# %%
csv_directory = r"C:\Users\arisa\Desktop\day_spot_summary"
csv_files = [file for file in os.listdir(csv_directory) if file.endswith('.csv')]

# %%
from tensorflow.keras.layers import BatchNormalization
def build_lstm_model(input_shape):
    model = Sequential([
        LSTM(2716, activation='tanh', input_shape=input_shape, return_sequences=True),  # Adjusted for LSTM
        Dropout(0.20),
        BatchNormalization(),
        # LSTM(2716, activation='tanh', return_sequences=False),  # Additional LSTM layer
        # Dropout(0.10),
        Dense(2716, activation='relu'),
        Dropout(0.15),
        Dense(256, activation='relu'),
        Dropout(0.10),
        Dense(128, activation='relu'),
        Dense(64, activation='relu'),
        Dense(32, activation='relu'),
        Dense(12),
    ])
    
    model.compile(optimizer='adam',
                  loss='mse',  # Use Mean Squared Error for regression
                  metrics=['mae'])  # Mean Absolute Error as an additional metric
    return model

# %%
x_scaler_loaded = joblib.load('nn_x_minmaxscaler.sav')
y_scaler_loaded = joblib.load('nn_y_minmaxscaler.sav')

# %%
def data_generator_lstm( n_steps,x_scaler,y_scaler):
    
    while True:
        for csv_file in csv_files:
            # Read the CSV file
            file_path = os.path.join(csv_directory, csv_file)
            chunk = pd.read_csv(file_path)
        

            feature_data = chunk.drop([
                'y_High_1d', 'y_Low_1d', 'y_Priority_1d',
                 'y_High_2d', 'y_Low_2d', 'y_Priority_2d',
                  'y_High_3d', 'y_Low_3d', 'y_Priority_3d',
                   'y_High_5d', 'y_Low_5d', 'y_Priority_5d'], axis=1)
            target_data = chunk[[
                'y_High_1d','y_Low_1d', 'y_Priority_1d',
                 'y_High_2d', 'y_Low_2d', 'y_Priority_2d',
                  'y_High_3d', 'y_Low_3d', 'y_Priority_3d',
                   'y_High_5d', 'y_Low_5d', 'y_Priority_5d'
                   ]]
            
            feature_data_scaled = pd.DataFrame(x_scaler.transform(feature_data), columns=feature_data.columns)
            # Assuming target_data also needs to be scaled, apply scaler separately
            target_data_scaled = pd.DataFrame(y_scaler.transform(target_data), columns=target_data.columns)

            # ensuring end_ix does not go out of feature_data_scaled’s bounds
            num_samples = (len(feature_data_scaled) - n_steps) // 32
            for i in range(num_samples):
                start_ix = i * 32
                end_ix = start_ix + n_steps

                X = feature_data_scaled[start_ix:end_ix]
                # using .iloc to avoid KeyError, and selecting the corresponding outputs
                y = target_data_scaled.iloc[start_ix:end_ix].iloc[-1]

                yield X.reshape((1, n_steps, -1)), y.reshape((1, -1))
            
           


model = build_lstm_model((30, 2716))

model.summary()




# %%
import warnings
warnings.filterwarnings(action='ignore', message='X has feature names, but MinMaxScaler was fitted without feature names')
train_generator = data_generator_lstm(30,x_scaler_loaded,y_scaler_loaded)

# Update total_samples, train_samples, and val_samples according to your dataset after transformations

model.fit(
    train_generator,
    steps_per_epoch=50,
    epochs=75,
    # Add validation_data if you have a validation generator
)





error:
{
	"name": "AttributeError",
	"message": "'DataFrame' object has no attribute 'reshape'",
	"stack": "---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
~\\AppData\\Local\\Temp\\ipykernel_17980\\261283929.py in ?()
----> 7 import warnings
      8 warnings.filterwarnings(action='ignore', message='X has feature names, but MinMaxScaler was fitted without feature names')
      9 train_generator = data_generator_lstm(30,x_scaler_loaded,y_scaler_loaded)
     10 

c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\keras\\utils\\traceback_utils.py in ?(*args, **kwargs)
     68             # To get the full stack trace, call:
     69             # `tf.debugging.disable_traceback_filtering()`
     70             raise e.with_traceback(filtered_tb) from None
     71         finally:
---> 72             del filtered_tb

~\\AppData\\Local\\Temp\\ipykernel_17980\\994707971.py in ?(n_steps, x_scaler, y_scaler)
     32                 X = feature_data_scaled[start_ix:end_ix]
     33                 # using .iloc to avoid KeyError, and selecting the corresponding outputs
     34                 y = target_data_scaled.iloc[start_ix:end_ix].iloc[-1]
     35 
---> 36                 yield X.reshape((1, n_steps, -1)), y.reshape((1, -1))
     37 
     38             # Prepare sequences for features and targets
     39             # X, y = [], []

c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\pandas\\core\\generic.py in ?(self, name)
   6295             and name not in self._accessors
   6296             and self._info_axis._can_hold_identifiers_and_holds_name(name)
   6297         ):
   6298             return self[name]
-> 6299         return object.__getattribute__(self, name)

AttributeError: 'DataFrame' object has no attribute 'reshape'"
}