code:
# %%
from sklearn.preprocessing import StandardScaler
import pandas as pd
import numpy as np
from tensorflow import keras
import joblib
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import LSTM,Dense,Dropout
import os

# %%
csv_directory = r"C:\Users\arisa\Desktop\day_spot_summary"
csv_files = [file for file in os.listdir(csv_directory) if file.endswith('.csv')]

# %%
# %%
def build_lstm_model(input_shape):
    model = Sequential([
        LSTM(2716, activation='tanh', input_shape=input_shape, return_sequences=True),  # Adjusted for LSTM
        Dropout(0.20),
        LSTM(2716, activation='tanh', return_sequences=False),  # Additional LSTM layer
        # Dropout(0.10),
        # LSTM(2716, activation='tanh', return_sequences=False),  # Additional LSTM layer
        Dropout(0.10),
        Dense(128, activation='relu'),
        Dense(64, activation='relu'),
        Dense(32, activation='relu'),
        Dense(12),
    ])
    
    model.compile(optimizer='adam',
                  loss='mse',  # Use Mean Squared Error for regression
                  metrics=['mae'])  # Mean Absolute Error as an additional metric
    return model

# %%
def data_generator_lstm( n_steps):
    
    while True:
        for csv_file in csv_files:
            # Read the CSV file
            file_path = os.path.join(csv_directory, csv_file)
            chunk = pd.read_csv(file_path)
        

            feature_data = chunk.drop([
                'y_High_1d', 'y_Low_1d', 'y_Priority_1d',
                 'y_High_2d', 'y_Low_2d', 'y_Priority_2d',
                  'y_High_3d', 'y_Low_3d', 'y_Priority_3d',
                   'y_High_5d', 'y_Low_5d', 'y_Priority_5d'], axis=1)
            target_data = chunk[['y_High_1d'
                   , 'y_Low_1d', 'y_Priority_1d',
                 'y_High_2d', 'y_Low_2d', 'y_Priority_2d',
                  'y_High_3d', 'y_Low_3d', 'y_Priority_3d',
                   'y_High_5d', 'y_Low_5d', 'y_Priority_5d'
                   ]]
            
            # Prepare sequences for features and targets
            X, y = [], []
            for i in range(len(feature_data) - n_steps):
                X.append(feature_data[i:i + n_steps].to_numpy())  
                y.append(target_data[i + n_steps - 1].to_numpy())  

            X, y = np.array(X), np.array(y)
            yield X, y

# %%
from tensorflow.keras.mixed_precision import set_global_policy

# Enable mixed precision
set_global_policy('mixed_float16')

# %%


model = build_lstm_model((30, 2716,))

model.summary()




# %%
import warnings
warnings.filterwarnings(action='ignore', message='X has feature names, but StandardScaler was fitted without feature names')
train_generator = data_generator_lstm(30)

# Update total_samples, train_samples, and val_samples according to your dataset after transformations

model.fit(
    train_generator,
    steps_per_epoch=50,
    epochs=75,
    # Add validation_data if you have a validation generator
)





error:
{
	"name": "KeyError",
	"message": "29",
	"stack": "---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
Cell In[7], line 7
      3 train_generator = data_generator_lstm(30)
      5 # Update total_samples, train_samples, and val_samples according to your dataset after transformations
----> 7 model.fit(
      8     train_generator,
      9     steps_per_epoch=50,
     10     epochs=75,
     11     # Add validation_data if you have a validation generator
     12 )

File c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\keras\\utils\\traceback_utils.py:70, in filter_traceback.<locals>.error_handler(*args, **kwargs)
     67     filtered_tb = _process_traceback_frames(e.__traceback__)
     68     # To get the full stack trace, call:
     69     # `tf.debugging.disable_traceback_filtering()`
---> 70     raise e.with_traceback(filtered_tb) from None
     71 finally:
     72     del filtered_tb

Cell In[3], line 26, in data_generator_lstm(n_steps)
     24 for i in range(len(feature_data) - n_steps):
     25     X.append(feature_data[i:i + n_steps].to_numpy())  
---> 26     y.append(target_data[i + n_steps - 1].to_numpy())  
     28 X, y = np.array(X), np.array(y)
     29 yield X, y

File c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\pandas\\core\\frame.py:4102, in DataFrame.__getitem__(self, key)
   4100 if self.columns.nlevels > 1:
   4101     return self._getitem_multilevel(key)
-> 4102 indexer = self.columns.get_loc(key)
   4103 if is_integer(indexer):
   4104     indexer = [indexer]

File c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\pandas\\core\\indexes\\base.py:3812, in Index.get_loc(self, key)
   3807     if isinstance(casted_key, slice) or (
   3808         isinstance(casted_key, abc.Iterable)
   3809         and any(isinstance(x, slice) for x in casted_key)
   3810     ):
   3811         raise InvalidIndexError(key)
-> 3812     raise KeyError(key) from err
   3813 except TypeError:
   3814     # If we have a listlike key, _check_indexing_error will raise
   3815     #  InvalidIndexError. Otherwise we fall through and re-raise
   3816     #  the TypeError.
   3817     self._check_indexing_error(key)

KeyError: 29"
}