code:
# %%
from sklearn.preprocessing import StandardScaler
import pandas as pd
import numpy as np
from tensorflow import keras
import joblib
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import LSTM,Dense,Dropout
import os

# %%
def build_lstm_model(input_shape):
    model = Sequential([
        LSTM(2716, activation='tanh', input_shape=input_shape, return_sequences=True),  # Adjusted for LSTM
        Dropout(0.20),
        LSTM(2716, activation='tanh', return_sequences=False),  # Additional LSTM layer
        # Dropout(0.10),
        # LSTM(2716, activation='tanh', return_sequences=False),  # Additional LSTM layer
        Dropout(0.10),
        Dense(128, activation='relu'),
        Dense(64, activation='relu'),
        Dense(32, activation='relu'),
        Dense(12),
    ])
    
    model.compile(optimizer='adam',
                  loss='mse',  # Use Mean Squared Error for regression
                  metrics=['mae'])  # Mean Absolute Error as an additional metric
    return model

# %%
csv_directory = r"C:\Users\arisa\Desktop\day_spot_summary"
csv_files = [file for file in os.listdir(csv_directory) if file.endswith('.csv')]

# %%
def data_generator_lstm(x_scaler, y_scaler, n_steps):
    
    while True:
        for csv_file in csv_files:
            # Read the CSV file
            file_path = os.path.join(csv_directory, csv_file)
            chunk = pd.read_csv(file_path)
        

            feature_data = chunk.drop([
                'y_High_1d', 'y_Low_1d', 'y_Priority_1d',
                 'y_High_2d', 'y_Low_2d', 'y_Priority_2d',
                  'y_High_3d', 'y_Low_3d', 'y_Priority_3d',
                   'y_High_5d', 'y_Low_5d', 'y_Priority_5d'], axis=1)
            target_data = chunk[['y_High_1d'
                   , 'y_Low_1d', 'y_Priority_1d',
                 'y_High_2d', 'y_Low_2d', 'y_Priority_2d',
                  'y_High_3d', 'y_Low_3d', 'y_Priority_3d',
                   'y_High_5d', 'y_Low_5d', 'y_Priority_5d'
                   ]]
            
            feature_data_scaled = pd.DataFrame(x_scaler.transform(feature_data), columns=feature_data.columns)
            # Assuming target_data also needs to be scaled, apply scaler separately
            target_data_scaled = pd.DataFrame(y_scaler.transform(target_data), columns=target_data.columns)
            
            # Prepare sequences for features and targets
            X, y = [], []
            for i in range(len(feature_data_scaled) - n_steps):
                X.append(feature_data_scaled[i:i + n_steps])  
                y.append(target_data_scaled[i + n_steps - 1])  

            X, y = np.array(X), np.array(y)
            yield X, y

# %%
from tensorflow.keras.mixed_precision import set_global_policy

# Enable mixed precision
set_global_policy('mixed_float16')

# %%


model = build_lstm_model((30, 1500,))

model.summary()




# %%
x_scaler_loaded = joblib.load('nn_x_scaler.sav')
y_scaler_loaded = joblib.load('nn_y_scaler.sav')

# %%
import warnings
warnings.filterwarnings(action='ignore', message='X has feature names, but StandardScaler was fitted without feature names')
train_generator = data_generator_lstm(x_scaler_loaded, y_scaler_loaded, 30)

# Update total_samples, train_samples, and val_samples according to your dataset after transformations

model.fit(
    train_generator,
    steps_per_epoch=50,
    epochs=75,
    # Add validation_data if you have a validation generator
)





error:
{
	"name": "ValueError",
	"message": "operands could not be broadcast together with shapes (883,2716) (6427,) (883,2716) ",
	"stack": "---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
Cell In[18], line 7
      3 train_generator = data_generator_lstm(x_scaler_loaded, y_scaler_loaded, 30)
      5 # Update total_samples, train_samples, and val_samples according to your dataset after transformations
----> 7 model.fit(
      8     train_generator,
      9     steps_per_epoch=50,
     10     epochs=75,
     11     # Add validation_data if you have a validation generator
     12 )

File c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\keras\\utils\\traceback_utils.py:70, in filter_traceback.<locals>.error_handler(*args, **kwargs)
     67     filtered_tb = _process_traceback_frames(e.__traceback__)
     68     # To get the full stack trace, call:
     69     # `tf.debugging.disable_traceback_filtering()`
---> 70     raise e.with_traceback(filtered_tb) from None
     71 finally:
     72     del filtered_tb

Cell In[15], line 22, in data_generator_lstm(x_scaler, y_scaler, n_steps)
     10 feature_data = chunk.drop([
     11     'y_High_1d', 'y_Low_1d', 'y_Priority_1d',
     12      'y_High_2d', 'y_Low_2d', 'y_Priority_2d',
     13       'y_High_3d', 'y_Low_3d', 'y_Priority_3d',
     14        'y_High_5d', 'y_Low_5d', 'y_Priority_5d'], axis=1)
     15 target_data = chunk[['y_High_1d'
     16        , 'y_Low_1d', 'y_Priority_1d',
     17      'y_High_2d', 'y_Low_2d', 'y_Priority_2d',
     18       'y_High_3d', 'y_Low_3d', 'y_Priority_3d',
     19        'y_High_5d', 'y_Low_5d', 'y_Priority_5d'
     20        ]]
---> 22 feature_data_scaled = pd.DataFrame(x_scaler.transform(feature_data), columns=feature_data.columns)
     23 # Assuming target_data also needs to be scaled, apply scaler separately
     24 target_data_scaled = pd.DataFrame(y_scaler.transform(target_data), columns=target_data.columns)

File c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\sklearn\\utils\\_set_output.py:295, in _wrap_method_output.<locals>.wrapped(self, X, *args, **kwargs)
    293 @wraps(f)
    294 def wrapped(self, X, *args, **kwargs):
--> 295     data_to_wrap = f(self, X, *args, **kwargs)
    296     if isinstance(data_to_wrap, tuple):
    297         # only wrap the first output for cross decomposition
    298         return_tuple = (
    299             _wrap_data_with_container(method, data_to_wrap[0], X, self),
    300             *data_to_wrap[1:],
    301         )

File c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\sklearn\\preprocessing\\_data.py:1062, in StandardScaler.transform(self, X, copy)
   1060 else:
   1061     if self.with_mean:
-> 1062         X -= self.mean_
   1063     if self.with_std:
   1064         X /= self.scale_

ValueError: operands could not be broadcast together with shapes (883,2716) (6427,) (883,2716) "
}