code:
# %%
from sklearn.preprocessing import StandardScaler
import pandas as pd
import numpy as np
from tensorflow import keras
import joblib
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import LSTM,Dense,Dropout
import os

# %%
csv_directory = r"C:\Users\arisa\Desktop\day_spot_summary"
csv_files = [file for file in os.listdir(csv_directory) if file.endswith('.csv')]

# %%
# %%
def build_lstm_model(input_shape):
    model = Sequential([
        LSTM(2716, activation='tanh', input_shape=input_shape, return_sequences=True),  # Adjusted for LSTM
        Dropout(0.20),
        # LSTM(2716, activation='tanh', return_sequences=False),  # Additional LSTM layer
        # Dropout(0.10),
        # LSTM(2716, activation='tanh', return_sequences=False),  # Additional LSTM layer
        # Dropout(0.10),
        Dense(2716, activation='relu'),
        Dense(128, activation='relu'),
        Dense(64, activation='relu'),
        Dense(32, activation='relu'),
        Dense(12),
    ])
    
    model.compile(optimizer='adam',
                  loss='mse',  # Use Mean Squared Error for regression
                  metrics=['mae'])  # Mean Absolute Error as an additional metric
    return model

# %%
x_scaler_loaded = joblib.load('nn_x_minmaxscaler.sav')
y_scaler_loaded = joblib.load('nn_y_minmaxscaler.sav')

# %%
def data_generator_lstm( n_steps,x_scaler,y_scaler):
    
    while True:
        for csv_file in csv_files:
            # Read the CSV file
            file_path = os.path.join(csv_directory, csv_file)
            chunk = pd.read_csv(file_path)
        

            feature_data = chunk.drop([
                'y_High_1d', 'y_Low_1d', 'y_Priority_1d',
                 'y_High_2d', 'y_Low_2d', 'y_Priority_2d',
                  'y_High_3d', 'y_Low_3d', 'y_Priority_3d',
                   'y_High_5d', 'y_Low_5d', 'y_Priority_5d'], axis=1)
            target_data = chunk[['y_High_1d'
                   , 'y_Low_1d', 'y_Priority_1d',
                 'y_High_2d', 'y_Low_2d', 'y_Priority_2d',
                  'y_High_3d', 'y_Low_3d', 'y_Priority_3d',
                   'y_High_5d', 'y_Low_5d', 'y_Priority_5d'
                   ]]
            
            feature_data_scaled = pd.DataFrame(x_scaler.transform(feature_data), columns=feature_data.columns)
            # Assuming target_data also needs to be scaled, apply scaler separately
            target_data_scaled = pd.DataFrame(y_scaler.transform(target_data), columns=target_data.columns)
            
            # Prepare sequences for features and targets
            X, y = [], []
            for i in range(len(feature_data_scaled) - n_steps + 1):  # Correct range to prevent out-of-bounds access
                X.append(feature_data_scaled.iloc[i:i + n_steps].to_numpy())  # Use iloc for consistency, though not necessary for slicing
                # Make sure the index for y is correctly bounded within the target_data
                if i + n_steps - 1 < len(target_data_scaled):  # Adjust condition to prevent out-of-bounds
                    y.append(target_data_scaled.iloc[i + n_steps - 1].to_numpy())  # Correct indexing to match the condition
                else:
                    break  # Safety break (though should be unnecessary with corrected logic) 
                
            X, y = np.array(X), np.array(y)
            yield X, y

# %%
from tensorflow.keras.mixed_precision import set_global_policy

# Enable mixed precision
set_global_policy('mixed_float16')

# %%


model = build_lstm_model((30, 2716,))

model.summary()




# %%
import warnings
warnings.filterwarnings(action='ignore', message='X has feature names, but MinMaxScaler was fitted without feature names')
train_generator = data_generator_lstm(30,x_scaler_loaded,y_scaler_loaded)

# Update total_samples, train_samples, and val_samples according to your dataset after transformations

model.fit(
    train_generator,
    steps_per_epoch=50,
    epochs=75,
    # Add validation_data if you have a validation generator
)




error:
{
	"name": "InvalidArgumentError",
	"message": "Graph execution error:

Detected at node 'mean_squared_error/SquaredDifference' defined at (most recent call last):
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\runpy.py\", line 197, in _run_module_as_main
      return _run_code(code, main_globals, None,
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\runpy.py\", line 87, in _run_code
      exec(code, run_globals)
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\ipykernel_launcher.py\", line 18, in <module>
      app.launch_new_instance()
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\traitlets\\config\\application.py\", line 1075, in launch_instance
      app.start()
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\ipykernel\\kernelapp.py\", line 739, in start
      self.io_loop.start()
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\tornado\\platform\\asyncio.py\", line 205, in start
      self.asyncio_loop.run_forever()
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\asyncio\\base_events.py\", line 601, in run_forever
      self._run_once()
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\asyncio\\base_events.py\", line 1905, in _run_once
      handle._run()
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\asyncio\\events.py\", line 80, in _run
      self._context.run(self._callback, *self._args)
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 545, in dispatch_queue
      await self.process_one()
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 534, in process_one
      await dispatch(*args)
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 437, in dispatch_shell
      await result
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\ipykernel\\ipkernel.py\", line 359, in execute_request
      await super().execute_request(stream, ident, parent)
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 778, in execute_request
      reply_content = await reply_content
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\ipykernel\\ipkernel.py\", line 446, in do_execute
      res = shell.run_cell(
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\ipykernel\\zmqshell.py\", line 549, in run_cell
      return super().run_cell(*args, **kwargs)
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3048, in run_cell
      result = self._run_cell(
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3103, in _run_cell
      result = runner(coro)
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\IPython\\core\\async_helpers.py\", line 129, in _pseudo_sync_runner
      coro.send(None)
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3308, in run_cell_async
      has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3490, in run_ast_nodes
      if await self.run_code(code, result, async_=asy):
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3550, in run_code
      exec(code_obj, self.user_global_ns, self.user_ns)
    File \"C:\\Users\\arisa\\AppData\\Local\\Temp\\ipykernel_18304\\261283929.py\", line 7, in <module>
      model.fit(
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\keras\\utils\\traceback_utils.py\", line 65, in error_handler
      return fn(*args, **kwargs)
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\keras\\engine\\training.py\", line 1564, in fit
      tmp_logs = self.train_function(iterator)
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\keras\\engine\\training.py\", line 1160, in train_function
      return step_function(self, iterator)
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\keras\\engine\\training.py\", line 1146, in step_function
      outputs = model.distribute_strategy.run(run_step, args=(data,))
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\keras\\engine\\training.py\", line 1135, in run_step
      outputs = model.train_step(data)
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\keras\\engine\\training.py\", line 994, in train_step
      loss = self.compute_loss(x, y, y_pred, sample_weight)
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\keras\\engine\\training.py\", line 1052, in compute_loss
      return self.compiled_loss(
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\keras\\engine\\compile_utils.py\", line 265, in __call__
      loss_value = loss_obj(y_t, y_p, sample_weight=sw)
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\keras\\losses.py\", line 152, in __call__
      losses = call_fn(y_true, y_pred)
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\keras\\losses.py\", line 272, in call
      return ag_fn(y_true, y_pred, **self._fn_kwargs)
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\keras\\losses.py\", line 1486, in mean_squared_error
      return backend.mean(tf.math.squared_difference(y_pred, y_true), axis=-1)
Node: 'mean_squared_error/SquaredDifference'
required broadcastable shapes
\t [[{{node mean_squared_error/SquaredDifference}}]] [Op:__inference_train_function_8665]",
	"stack": "---------------------------------------------------------------------------
InvalidArgumentError                      Traceback (most recent call last)
Cell In[16], line 7
      3 train_generator = data_generator_lstm(30,x_scaler_loaded,y_scaler_loaded)
      5 # Update total_samples, train_samples, and val_samples according to your dataset after transformations
----> 7 model.fit(
      8     train_generator,
      9     steps_per_epoch=50,
     10     epochs=75,
     11     # Add validation_data if you have a validation generator
     12 )

File c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\keras\\utils\\traceback_utils.py:70, in filter_traceback.<locals>.error_handler(*args, **kwargs)
     67     filtered_tb = _process_traceback_frames(e.__traceback__)
     68     # To get the full stack trace, call:
     69     # `tf.debugging.disable_traceback_filtering()`
---> 70     raise e.with_traceback(filtered_tb) from None
     71 finally:
     72     del filtered_tb

File c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\tensorflow\\python\\eager\\execute.py:54, in quick_execute(op_name, num_outputs, inputs, attrs, ctx, name)
     52 try:
     53   ctx.ensure_initialized()
---> 54   tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,
     55                                       inputs, attrs, num_outputs)
     56 except core._NotOkStatusException as e:
     57   if name is not None:

InvalidArgumentError: Graph execution error:

Detected at node 'mean_squared_error/SquaredDifference' defined at (most recent call last):
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\runpy.py\", line 197, in _run_module_as_main
      return _run_code(code, main_globals, None,
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\runpy.py\", line 87, in _run_code
      exec(code, run_globals)
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\ipykernel_launcher.py\", line 18, in <module>
      app.launch_new_instance()
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\traitlets\\config\\application.py\", line 1075, in launch_instance
      app.start()
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\ipykernel\\kernelapp.py\", line 739, in start
      self.io_loop.start()
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\tornado\\platform\\asyncio.py\", line 205, in start
      self.asyncio_loop.run_forever()
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\asyncio\\base_events.py\", line 601, in run_forever
      self._run_once()
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\asyncio\\base_events.py\", line 1905, in _run_once
      handle._run()
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\asyncio\\events.py\", line 80, in _run
      self._context.run(self._callback, *self._args)
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 545, in dispatch_queue
      await self.process_one()
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 534, in process_one
      await dispatch(*args)
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 437, in dispatch_shell
      await result
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\ipykernel\\ipkernel.py\", line 359, in execute_request
      await super().execute_request(stream, ident, parent)
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 778, in execute_request
      reply_content = await reply_content
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\ipykernel\\ipkernel.py\", line 446, in do_execute
      res = shell.run_cell(
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\ipykernel\\zmqshell.py\", line 549, in run_cell
      return super().run_cell(*args, **kwargs)
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3048, in run_cell
      result = self._run_cell(
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3103, in _run_cell
      result = runner(coro)
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\IPython\\core\\async_helpers.py\", line 129, in _pseudo_sync_runner
      coro.send(None)
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3308, in run_cell_async
      has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3490, in run_ast_nodes
      if await self.run_code(code, result, async_=asy):
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3550, in run_code
      exec(code_obj, self.user_global_ns, self.user_ns)
    File \"C:\\Users\\arisa\\AppData\\Local\\Temp\\ipykernel_18304\\261283929.py\", line 7, in <module>
      model.fit(
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\keras\\utils\\traceback_utils.py\", line 65, in error_handler
      return fn(*args, **kwargs)
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\keras\\engine\\training.py\", line 1564, in fit
      tmp_logs = self.train_function(iterator)
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\keras\\engine\\training.py\", line 1160, in train_function
      return step_function(self, iterator)
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\keras\\engine\\training.py\", line 1146, in step_function
      outputs = model.distribute_strategy.run(run_step, args=(data,))
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\keras\\engine\\training.py\", line 1135, in run_step
      outputs = model.train_step(data)
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\keras\\engine\\training.py\", line 994, in train_step
      loss = self.compute_loss(x, y, y_pred, sample_weight)
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\keras\\engine\\training.py\", line 1052, in compute_loss
      return self.compiled_loss(
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\keras\\engine\\compile_utils.py\", line 265, in __call__
      loss_value = loss_obj(y_t, y_p, sample_weight=sw)
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\keras\\losses.py\", line 152, in __call__
      losses = call_fn(y_true, y_pred)
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\keras\\losses.py\", line 272, in call
      return ag_fn(y_true, y_pred, **self._fn_kwargs)
    File \"c:\\Users\\arisa\\.conda\\envs\\tf\\lib\\site-packages\\keras\\losses.py\", line 1486, in mean_squared_error
      return backend.mean(tf.math.squared_difference(y_pred, y_true), axis=-1)
Node: 'mean_squared_error/SquaredDifference'
required broadcastable shapes
\t [[{{node mean_squared_error/SquaredDifference}}]] [Op:__inference_train_function_8665]"
}