code:
# %%
import os
import numpy as np
import pandas as pd
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from tensorflow.keras.layers import Input, Dropout
from tensorflow.keras import Model

# %%
def data_generator(file_paths, batch_size, train=True, scaler=None):
    while True:  # Loop forever so the generator never terminates
        # Shuffle file paths to ensure random distribution of data across files
        np.random.shuffle(file_paths)
        
        for file_path in file_paths:
            data = pd.read_csv(file_path)

            # Split features and labels
            X = data.drop([
                'Date', 'Symbol',
                'y_High_1d', 'y_Low_1d', 'y_Priority_1d',
                 'y_High_2d', 'y_Low_2d', 'y_Priority_2d',
                  'y_High_3d', 'y_Low_3d', 'y_Priority_3d',
                   'y_High_5d', 'y_Low_5d', 'y_Priority_5d'], axis=1).values

            y_high_1d = data['y_High_1d'].values
            y_low_1d = data['y_Low_1d'].values
            y_priority_1d = data['y_Priority_1d'].values

            y_high_2d = data['y_High_2d'].values
            y_low_2d = data['y_Low_2d'].values
            y_priority_2d = data['y_Priority_2d'].values

            y_high_3d = data['y_High_3d'].values
            y_low_3d = data['y_Low_3d'].values
            y_priority_3d = data['y_Priority_3d'].values

            y_high_5d = data['y_High_5d'].values
            y_low_5d = data['y_Low_5d'].values
            y_priority_5d = data['y_Priority_5d'].values
            
            # Optionally, apply preprocessing here (e.g., Scaler transform)
            if scaler is not None and train:
                X = scaler.fit_transform(X)
            elif scaler is not None:
                X = scaler.transform(X)
                
            # Splitting data into batches
            for i in range(0, len(data), batch_size):
                end = i + batch_size
                X_batch = X[i:end]

                y_high_batch_1d = y_high_1d[i:end]
                y_low_batch_1d = y_low_1d[i:end]
                y_priority_batch_1d = y_priority_1d[i:end]

                y_high_batch_2d = y_high_2d[i:end]
                y_low_batch_2d = y_low_2d[i:end]
                y_priority_batch_2d = y_priority_2d[i:end]

                y_high_batch_3d = y_high_3d[i:end]
                y_low_batch_3d = y_low_3d[i:end]
                y_priority_batch_3d = y_priority_3d[i:end]

                y_high_batch_5d = y_high_5d[i:end]
                y_low_batch_5d = y_low_5d[i:end]
                y_priority_batch_5d = y_priority_5d[i:end]
                
                yield X_batch, (y_high_batch_1d, y_low_batch_1d,y_priority_batch_1d,
                y_high_batch_2d, y_low_batch_2d,y_priority_batch_2d,
                y_high_batch_3d, y_low_batch_3d, y_priority_batch_3d,
                y_high_batch_5d, y_low_batch_5d, y_priority_batch_5d)

# %%
dataset_dir = r'C:\Users\arisa\Desktop\day_spot'
file_paths = [os.path.join(dataset_dir, file_name) for file_name in os.listdir(dataset_dir)]
train_files, val_files = train_test_split(file_paths, test_size=0.06, random_state=42)

batch_size = 72

# %%
# Optional: Initialize a scaler for data normalization
scaler = StandardScaler()

# %%
# Create data generators
train_gen = data_generator(train_files, batch_size, train=True, scaler=scaler)
val_gen = data_generator(val_files, batch_size, train=False, scaler=scaler)


# %%
input_shape = (72, 6427)

inputs = Input(shape=input_shape)  # Corrected the input shape specification

x = Dense(6427, activation='relu')(inputs)
x = Dropout(0.25) (x)
x = Dense(3200, activation='relu') (x)
x = Dropout(0.20) (x)
x = Dense(1800, activation='relu') (x)
x = Dropout(0.15) (x)
x = Dense(1024, activation='relu') (x)
x = Dropout(0.10) (x)
x = Dense(512, activation='relu') (x)
x = Dropout(0.05) (x)
x = Dense(256, activation='relu') (x)
x = Dense(128, activation='relu') (x)
x = Dense(64, activation='relu') (x)
x = Dense(32, activation='relu')  (x)



# Defining three separate outputs

out_high_1d = Dense(1, name='high_output_1d')(x)  # No activation, linear output
out_low_1d = Dense(1, name='low_output_1d')(x)    # No activation, linear output
out_priority_1d = Dense(1, activation='sigmoid', name='priority_output_1d')(x)

out_high_2d = Dense(1, name='high_output_2d')(x)  # No activation, linear output
out_low_2d = Dense(1, name='low_output_2d')(x)    # No activation, linear output
out_priority_2d = Dense(1, activation='sigmoid', name='priority_output_2d')(x)

out_high_3d = Dense(1, name='high_output_3d')(x)  # No activation, linear output
out_low_3d = Dense(1, name='low_output_3d')(x)    # No activation, linear output
out_priority_3d = Dense(1, activation='sigmoid', name='priority_output_3d')(x)

out_high_5d = Dense(1, name='high_output_5d')(x)  # No activation, linear output
out_low_5d = Dense(1, name='low_output_5d')(x)    # No activation, linear output
out_priority_5d = Dense(1, activation='sigmoid', name='priority_output_5d')(x)

# Constructing the model
model = Model(inputs=inputs, outputs=[
    out_high_1d, out_low_1d, out_priority_1d,out_high_2d, out_low_2d, out_priority_2d,
    out_high_3d, out_low_3d, out_priority_3d,out_high_5d, out_low_5d, out_priority_5d])

model.compile(optimizer='adam',
              loss={
                'high_output_1d': 'mse', 'low_output_1d': 'mse', 'priority_output_1d': 'binary_crossentropy',
                'high_output_2d': 'mse', 'low_output_2d': 'mse', 'priority_output_2d': 'binary_crossentropy',
                'high_output_3d': 'mse', 'low_output_3d': 'mse', 'priority_output_3d': 'binary_crossentropy',
                'high_output_5d': 'mse', 'low_output_5d': 'mse', 'priority_output_5d': 'binary_crossentropy'
              },
              metrics={
                'high_output_1d': ['mae'], 'low_output_1d': ['mae'], 'priority_output_1d': ['accuracy'],
                'high_output_2d': ['mae'], 'low_output_2d': ['mae'], 'priority_output_2d': ['accuracy'],
                'high_output_3d': ['mae'], 'low_output_3d': ['mae'], 'priority_output_3d': ['accuracy'],
                'high_output_5d': ['mae'], 'low_output_5d': ['mae'], 'priority_output_5d': ['accuracy']
              },
              loss_weights={
                'high_output_1d': 1.0, 'low_output_1d': 1.0, 'priority_output_1d': 1.0,
                'high_output_2d': 1.0, 'low_output_2d': 1.0, 'priority_output_2d': 1.0,
                'high_output_3d': 1.0, 'low_output_3d': 1.0, 'priority_output_3d': 1.0,
                'high_output_5d': 1.0, 'low_output_5d': 1.0, 'priority_output_5d': 1.0
              }
              )

# %%
import os
import pandas as pd

def count_samples(file_paths):
    total_samples = 0
    for file_path in file_paths:
        df = pd.read_csv(file_path)
        samples = len(df)
        total_samples += samples
    return total_samples

number_of_training_samples = count_samples(train_files)
number_of_validation_samples = count_samples(val_files)

print(f'Number of training samples: {number_of_training_samples}')
print(f'Number of validation samples: {number_of_validation_samples}')

# %%
import tensorflow as tf
output_signature = (
    tf.TensorSpec(shape=(72, X_batch.shape[1]), dtype=tf.float32),  # for X_batch
    (
        tf.TensorSpec(shape=(72,), dtype=tf.float32),  # For each y_batch
        tf.TensorSpec(shape=(72,), dtype=tf.float32),
        tf.TensorSpec(shape=(72,), dtype=tf.float32),
        tf.TensorSpec(shape=(72,), dtype=tf.float32),
        tf.TensorSpec(shape=(72,), dtype=tf.float32),
        tf.TensorSpec(shape=(72,), dtype=tf.float32),
        tf.TensorSpec(shape=(72,), dtype=tf.float32),
        tf.TensorSpec(shape=(72,), dtype=tf.float32),
        tf.TensorSpec(shape=(72,), dtype=tf.float32),
        tf.TensorSpec(shape=(72,), dtype=tf.float32),
        tf.TensorSpec(shape=(72,), dtype=tf.float32),
        tf.TensorSpec(shape=(72,), dtype=tf.float32),
    )
)

train_dataset = tf.data.Dataset.from_generator(
    lambda: train_gen,
    output_signature=output_signature
)

val_dataset = tf.data.Dataset.from_generator(
    lambda: val_gen,
    output_signature=output_signature
)

# %%

# You'll need to determine steps_per_epoch and validation_steps based on your data size and batch size
steps_per_epoch = np.ceil(number_of_training_samples / batch_size).astype(int)  # Replace number_of_training_samples
validation_steps = np.ceil(number_of_validation_samples / batch_size).astype(int)  # Replace number_of_validation_samples

# model.fit(train_gen,
#           steps_per_epoch=steps_per_epoch,
#           validation_data=val_gen,
#           validation_steps=validation_steps,
#           epochs=1000)

# Then use these datasets in your model.fit call
model.fit(train_dataset,
          steps_per_epoch=steps_per_epoch,
          validation_data=val_dataset,
          validation_steps=validation_steps,
          epochs=1000)





error:
{
	"name": "AttributeError",
	"message": "'NoneType' object has no attribute 'items'",
	"stack": "---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
Cell In[46], line 12
      3 validation_steps = np.ceil(number_of_validation_samples / batch_size).astype(int)  # Replace number_of_validation_samples
      5 # model.fit(train_gen,
      6 #           steps_per_epoch=steps_per_epoch,
      7 #           validation_data=val_gen,
   (...)
     10 
     11 # Then use these datasets in your model.fit call
---> 12 model.fit(train_dataset,
     13           steps_per_epoch=steps_per_epoch,
     14           validation_data=val_dataset,
     15           validation_steps=validation_steps,
     16           epochs=1000)

File c:\\Users\\arisa\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\keras\\src\\utils\\traceback_utils.py:122, in filter_traceback.<locals>.error_handler(*args, **kwargs)
    119     filtered_tb = _process_traceback_frames(e.__traceback__)
    120     # To get the full stack trace, call:
    121     # `keras.config.disable_traceback_filtering()`
--> 122     raise e.with_traceback(filtered_tb) from None
    123 finally:
    124     del filtered_tb

File c:\\Users\\arisa\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\keras\\src\\trainers\\trainer.py:898, in Trainer._pythonify_logs(self, logs)
    896 def _pythonify_logs(self, logs):
    897     result = {}
--> 898     for key, value in sorted(logs.items()):
    899         if isinstance(value, dict):
    900             result.update(self._pythonify_logs(value))

AttributeError: 'NoneType' object has no attribute 'items'"
}