i have following code to train my NN model :
# %%
from sklearn.preprocessing import StandardScaler
import pandas as pd
import numpy as np
from tensorflow import keras
import joblib

def calculate_targets_scaling_params(file_path):
scaler = StandardScaler()
for chunk in pd.read_csv(file_path, chunksize=10000): # Adjust chunksize based on your memory capacity
filtered_chunk = chunk[[‘y_High_5d’,‘y_Low_5d’,‘y_Priority_5d’]]


scaler.partial_fit(filtered_chunk) # Accumulate means and variances

return scaler.mean_, scaler.var_


# %%
x_scaler_loaded = joblib.load(‘nn_x_scaler.sav’)
y_scaler_loaded = joblib.load(‘nn_y_hlp5_scaler.sav’)


def data_generator(file_path, batch_size, x_scaler, y_scaler):
chunksize = batch_size

while True: # Loop forever, so the generator never terminates
for chunk in pd.read_csv(file_path, chunksize=chunksize):

filtered_c = chunk.drop([‘Date’, ‘Symbol’], axis=1)

feature_data = filtered_c.drop([
‘y_High_1d’, ‘y_Low_1d’, ‘y_Priority_1d’,
‘y_High_2d’, ‘y_Low_2d’, ‘y_Priority_2d’,
‘y_High_3d’, ‘y_Low_3d’, ‘y_Priority_3d’,
‘y_High_5d’, ‘y_Low_5d’, ‘y_Priority_5d’], axis=1)
target_data = filtered_c[[‘y_High_5d’, ‘y_Low_5d’, ‘y_Priority_5d’]]

feature_data_scaled = pd.DataFrame(x_scaler.transform(feature_data), columns=feature_data.columns)
# Assuming target_data also needs to be scaled, apply scaler separately
target_data_scaled = pd.DataFrame(y_scaler.transform(target_data), columns=target_data.columns)

# Now, feature_data_scaled and target_data_scaled are both DataFrames, scaled and ready to use
yield feature_data_scaled.values, target_data_scaled.values



# %%

from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout, Input
import tensorflow as tf
from tensorflow.keras.layers import BatchNormalization

def build_model():
input_shape = (6427,)

model = Sequential([
Dense(6427, activation=‘relu’, input_shape = input_shape),
Dropout(0.17),
BatchNormalization(),
Dense(4096, activation=‘relu’),
Dropout(0.15),
BatchNormalization(),
Dense(2048, activation=‘relu’),
Dropout(0.12),
BatchNormalization(),
Dense(1024, activation=‘relu’),
Dropout(0.10),
BatchNormalization(),
Dense(512, activation=‘relu’),
Dropout(0.05),
BatchNormalization(),
Dense(256, activation=‘relu’),
BatchNormalization(),
Dense(128, activation=‘relu’),
BatchNormalization(),
Dense(64, activation=‘relu’),
BatchNormalization(),
Dense(32, activation=‘relu’),
BatchNormalization(),
Dense(3),
])

optimizer = keras.optimizers.Adam(learning_rate=0.006)
model.compile(optimizer=‘adam’,
loss=‘mse’, # Use Mean Squared Error for regression
metrics=[‘mae’]) # Mean Absolute Error as an additional metric
return model


# %%


# Instantiate the model
model = build_model()

model.summary()

# %%
import warnings
warnings.filterwarnings(action=‘ignore’, message=‘X has feature names, but StandardScaler was fitted without feature names’)
train_generator = data_generator(file_path, batch_size,x_scaler=x_scaler_loaded,y_scaler=y_scaler_loaded)

total_samples = 301617 # Assuming same example size
train_samples = int(total_samples)

steps_per_epoch = train_samples // batch_size


# Modify the model fitting call to include validation data
model.fit(
train_generator,
steps_per_epoch=steps_per_epoch,
epochs=45,
)



currently my model not using any validation dataset when training
update the code properly and implement val set