Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<import_modules>
pd.options.display.max_columns = None
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
from itertools import product
from sklearn.preprocessing import LabelEncoder
import lightgbm as lgbm
from lightgbm import plot_importance
from sklearn.metrics import mean_squared_error
import joblib
from math import sqrt
from time import time<load_from_csv>
train_features = pd.read_csv('/kaggle/input/lish-moa/train_features.csv')
 test_features = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 train_features['dataset'] = 'train'
test_features['dataset'] = 'test'
df = pd.concat([train_features, test_features] )
train_sales = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/sales_train.csv')
 train_sales['date'] = pd.to_datetime(train_sales['date'], dayfirst = True)
 test = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/test.csv' ).set_index('ID')
 df_items = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/items.csv')
 df_shops = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/shops.csv')
 df_categories = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/item_categories.csv' )<define_variables>
sample_submission = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 sample_submission
max_cnt = train_sales["item_cnt_day"].quantile(0.999)
 max_price = train_sales["item_price"].quantile(0.999)
 print(max_cnt)
 train_sales = train_sales[train_sales["item_cnt_day"]<=max_cnt]
print(max_price)
 train_sales = train_sales[(train_sales["item_price"]>0)&(train_sales["item_price"]<=max_price)]
print(train_sales.shape )<merge>
cp_width = 500
cp_height = 400
scatter_size = 600
group_sales = train_sales.groupby(['date_block_num', 'shop_id', 'item_id'],
as_index=False)[['item_cnt_day']].sum()
group_sales = group_sales.rename(columns={"item_cnt_day":"item_cnt_month"})
 train_monthly = train_monthly.merge(group_sales,
on=['date_block_num', 'shop_id', 'item_id'],
how='left')
 train_monthly['item_cnt_month'] = train_monthly['item_cnt_month'].fillna(0 ).clip(0, 20 )<concatenate>
plt.figure(figsize=(15,5))
sns.distplot(train_features['cp_time'], color='red', bins=5)
 plt.title("Train: Treatment duration ", fontsize=15, weight='bold')
 plt.show()
test['date_block_num'] = 34
train_test_monthly = pd.concat([train_monthly, test] ).reset_index(drop = True )<merge>
train_columns = train_features.columns.to_list()
g_list = [i for i in train_columns if i.startswith('g-')]
c_list = [i for i in train_columns if i.startswith('c-')]
prices = train_sales.groupby(by = ['item_id'],
as_index=False ).agg({'item_price':['mean']})
 prices.columns = ['item_id', 'item_price_mean']
prices.fillna(0, inplace = True)
 train_test_monthly = train_test_monthly.merge(prices,
on=['item_id'],
how='left' )<feature_engineering>
all_columns = list(set(all_columns))
print('Number of columns:', len(all_columns))
train_test_monthly.loc[train_test_monthly['shop_id']==0, 'shop_id'] = 57
train_test_monthly.loc[train_test_monthly['shop_id']==1, 'shop_id'] = 58
train_test_monthly.loc[train_test_monthly['shop_id']==39, 'shop_id'] = 40<feature_engineering>
train_targets = pd.read_csv(".. /input/lish-moa/train_targets_scored.csv")
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 print('Number of rows: ', train_targets.shape[0])
 print('Number of cols: ', train_targets.shape[1])
 train_targets.head()
df_shops.loc[df_shops.shop_name == 'Сергиев Посад ТЦ "7Я"', 'shop_name'] = 'СергиевПосад ТЦ "7Я"'
df_shops['city'] = df_shops['shop_name'].str.split(' ' ).map(lambda x: x[0])
 df_shops.loc[df_shops['city'] == '!Якутск', 'city'] = 'Якутск'<merge>
%%time
correlation_matrix = pd.DataFrame()
for t_col in train_targets.columns:
corr_list = list()
if t_col == 'sig_id':
continue
for col in columns:
res = train_features[col].corr(train_targets[t_col])
 corr_list.append(res)
 correlation_matrix[t_col] = corr_list
train_test_monthly = train_test_monthly.merge(df_shops[['shop_id', 'city']],
how = 'left',
on = 'shop_id' )<categorify>
correlation_matrix['train_features'] = columns
correlation_matrix = correlation_matrix.set_index('train_features')
 correlation_matrix
train_test_monthly['city'] = LabelEncoder().fit_transform(train_test_monthly['city'] )<feature_engineering>
col_df = pd.DataFrame()
tr_cols = list()
tar_cols = list()
for col in correlation_matrix.columns:
tar_cols.append(col)
 tr_cols.append(
 correlation_matrix[col].abs().sort_values(ascending=False ).reset_index() ['train_features'].head(1 ).values[0] 
)
 col_df['column'] = tar_cols
col_df['train_best_column'] = tr_cols
total_scores = pd.merge(high_scores, col_df)
 total_scores
df_categories['type'] = df_categories['item_category_name'].str.\
split('-' ).map(lambda x: x[0].strip())
 df_categories['sub_type'] = df_categories['item_category_name'].str.\
split('-' ).map(lambda x: x[1].strip() if len(x)> 1
else x[0].strip())
 df_categories.head()<merge>
col_df = pd.DataFrame()
tr_first_cols = list()
tr_second_cols = list()
tar_cols = list()
for col in current_corr.columns:
tar_cols.append(col)
 tr_first_cols.append(
 current_corr[col].abs().sort_values(ascending=False ).reset_index() ['train_features'].head(2 ).values[0] 
)
 tr_second_cols.append(
 current_corr[col].abs().sort_values(ascending=False ).reset_index() ['train_features'].head(2 ).values[1] 
)
 col_df['column'] = tar_cols
col_df['train_1_column'] = tr_first_cols
col_df['train_2_column'] = tr_second_cols
col_df
df_items = df_items.merge(df_categories[['item_category_id', 'type', 'sub_type']],
on = 'item_category_id',
how = 'left' )<merge>
for_analysis = [
target_columns[np.random.randint(0, len(target_columns)-1)] for i in range(5)
 ]
current_corr = correlation_matrix[for_analysis]
col_df = pd.DataFrame()
tr_first_cols = list()
tr_second_cols = list()
tr_third_cols = list()
tar_cols = list()
for col in current_corr.columns:
tar_cols.append(col)
 tr_first_cols.append(
 current_corr[col].abs().sort_values(ascending=False ).reset_index() ['train_features'].head(3 ).values[0] 
)
 tr_second_cols.append(
 current_corr[col].abs().sort_values(ascending=False ).reset_index() ['train_features'].head(3 ).values[1] 
)
 tr_third_cols.append(
 current_corr[col].abs().sort_values(ascending=False ).reset_index() ['train_features'].head(3 ).values[2] 
)
 col_df['column'] = tar_cols
col_df['train_1_column'] = tr_first_cols
col_df['train_2_column'] = tr_second_cols
col_df['train_3_column'] = tr_third_cols
col_df
train_test_monthly = train_test_monthly.merge(df_items[['item_id', 'item_category_id', 'type', 'sub_type']],
on = 'item_id',
how = 'left' )<categorify>
def plot_3dscatter(col_df, index):
analysis = pd.DataFrame()
analysis['color'] = train_targets[col_df.iloc[index]['column']]
analysis['x'] = train_features[col_df.iloc[index]['train_1_column']]
analysis['y'] = train_features[col_df.iloc[index]['train_2_column']]
analysis['z'] = train_features[col_df.iloc[index]['train_3_column']]
analysis.columns = [
'color',
col_df.iloc[index]['train_1_column'],
col_df.iloc[index]['train_2_column'],
col_df.iloc[index]['train_3_column']
]
analysis['size'] = 1
analysis.loc[analysis['color'] == 1, 'size'] = 20
fig = px.scatter_3d(
 analysis,
x=col_df.iloc[index]['train_1_column'],
y=col_df.iloc[index]['train_2_column'],
z=col_df.iloc[index]['train_3_column'],
color="color",
size='size',
height=scatter_size,
width=scatter_size,
title='Scatter plot for ' + col_df.iloc[index]['column'] 
)
 fig.show()
train_test_monthly['type'] = LabelEncoder().fit_transform(train_test_monthly['type'])
 train_test_monthly['sub_type'] = LabelEncoder().fit_transform(train_test_monthly['sub_type'] )<feature_engineering>
answer = list()
for group in last_term.group.tolist() :
agent_list = list()
for item in target_columns:
if item.split('_')[-1] == group:
agent_list.append(item)
 agent_df = train_targets[agent_list]
data = agent_df.astype(bool ).sum(axis=1 ).reset_index()
answer.append(data[0].max() )
train_test_monthly['month'] =(train_test_monthly['date_block_num'] % 12)+ 1<data_type_conversions>
categories = train_features[['cp_type', 'cp_time', 'cp_dose']]
tar = train_targets.copy()
tar = tar.drop(['sig_id'], axis=1)
 analysis = pd.concat([categories, tar], axis=1 )
train_test_monthly['shop_item_id'] = train_test_monthly['shop_id'].astype(str)+ '_' +\
train_test_monthly['item_id'].astype(str)
 train_test_monthly['shop_cat_id'] = train_test_monthly['shop_id'].astype(str)+ '_' +\
train_test_monthly['item_category_id'].astype(str)
 train_test_monthly['shop_type_id'] = train_test_monthly['shop_id'].astype(str)+ '_' +\
train_test_monthly['type'].astype(str)
 train_test_monthly['item_city_id'] = train_test_monthly['item_id'].astype(str)+ '_' +\
train_test_monthly['city'].astype(str)
 train_test_monthly['item_city_month'] = train_test_monthly['item_city_id'].astype(str)+ '_' +\
train_test_monthly['month'].astype(str)
 train_test_monthly['shop_item_month'] = train_test_monthly['shop_item_id'].astype(str)+ '_' +\
train_test_monthly['month'].astype(str)
 train_test_monthly['shop_cat_month'] = train_test_monthly['shop_cat_id'].astype(str)+ '_' +\
train_test_monthly['month'].astype(str)
 train_test_monthly['shop_type_month'] = train_test_monthly['shop_type_id'].astype(str)+ '_' +\
train_test_monthly['month'].astype(str )<drop_column>
for category in analysis['cp_dose'].unique().tolist() :
number = 0
cols = list()
for col in analysis.columns:
if col in ['cp_type', 'cp_time', 'cp_dose']:
continue
if len(analysis[analysis['cp_dose'] == category][col].value_counts())== 1:
number += 1
cols.append(col)
 print(category, '.Number of columns with 1 unique value: ', number, '.Columns: ', cols )
train_test_monthly.drop(columns=['month'], inplace=True )<categorify>
analysis[analysis['cp_dose'] == 'D2']['atp-sensitive_potassium_channel_antagonist'].value_counts()
train_test_monthly['shop_item_id'] = LabelEncoder().fit_transform(train_test_monthly['shop_item_id'])
 train_test_monthly['shop_cat_id'] = LabelEncoder().fit_transform(train_test_monthly['shop_cat_id'])
 train_test_monthly['shop_type_id'] = LabelEncoder().fit_transform(train_test_monthly['shop_type_id'])
 train_test_monthly['item_city_id'] = LabelEncoder().fit_transform(train_test_monthly['item_city_id'])
 train_test_monthly['item_city_month'] = LabelEncoder().fit_transform(train_test_monthly['item_city_month'])
 train_test_monthly['shop_item_month'] = LabelEncoder().fit_transform(train_test_monthly['shop_item_month'])
 train_test_monthly['shop_cat_month'] = LabelEncoder().fit_transform(train_test_monthly['shop_cat_month'])
 train_test_monthly['shop_type_month'] = LabelEncoder().fit_transform(train_test_monthly['shop_type_month'] )<count_missing_values>
analysis[analysis['cp_dose']=='D2']['erbb2_inhibitor'].value_counts()
train_test_monthly.isna().sum()<groupby>
for category in analysis['cp_time'].unique().tolist() :
number = 0
cols = list()
for col in analysis.columns:
if col in ['cp_type', 'cp_time', 'cp_dose']:
continue
if len(analysis[analysis['cp_time']==category][col].value_counts())== 1:
number += 1
cols.append(col)
 print(category, '.Number of columns with 1 unique value: ', number, '.Columns: ', cols )
group_mean_type = train_test_monthly.groupby('type')['item_price_mean'].mean()
train_test_monthly.loc[train_test_monthly['item_price_mean'].isna() , 'item_price_mean'] =\
train_test_monthly.loc[train_test_monthly['item_price_mean'].isna() , 'type'].map(group_mean_type )<count_unique_values>
analysis[analysis['cp_time'] == 24]['erbb2_inhibitor'].value_counts()
print('%i cities'%(train_test_monthly[train_test_monthly['date_block_num']<34]['city'].nunique()))
print('%i types'%(train_test_monthly[train_test_monthly['date_block_num']<34]['type'].nunique()))
print('%i sub_types'%(train_test_monthly[train_test_monthly['date_block_num']<34]['sub_type'].nunique()))<count_unique_values>
analysis[analysis['cp_time'] == 72]['erbb2_inhibitor'].value_counts()
print('%i cities'%(train_test_monthly[train_test_monthly['date_block_num']==34]['city'].nunique()))
print('%i types'%(train_test_monthly[train_test_monthly['date_block_num']==34]['type'].nunique()))
print('%i sub_types'%(train_test_monthly[train_test_monthly['date_block_num']==34]['sub_type'].nunique()))<groupby>
analysis[analysis['cp_time'] == 24]['atp-sensitive_potassium_channel_antagonist'].value_counts()
def col_months_lag(col, dataset, date_block_mean=True, use_similar=False):
lag_months = [1, 2, 3, 12]
df = dataset.copy()
if date_block_mean:
group_mean = df.groupby(['date_block_num',
col],
as_index=False)['item_cnt_month'].mean().fillna(0)
 group_mean.rename(columns={'item_cnt_month':col+'_date_block_mean'},
inplace = True)
 for lag in lag_months:
group_mean_lag = group_mean.copy()
group_mean_lag['date_block_num'] += lag
group_mean_lag.rename(columns={col+'_date_block_mean':col+'_date_block_mean_lag_'+str(lag)},
inplace = True)
 df = df.merge(group_mean_lag,
on=['date_block_num', col],
how='left')
 df[col+'_date_block_mean_lag_'+str(lag)].fillna(0, inplace = True)
 df[col+'_3_months_mean'] = df[[col+'_date_block_mean_lag_1',
col+'_date_block_mean_lag_2',
col+'_date_block_mean_lag_3']].mean(axis = 1)
 df[col+'_3_months_std'] = df[[col+'_date_block_mean_lag_1',
col+'_date_block_mean_lag_2',
col+'_date_block_mean_lag_3']].std(axis = 1)
 df[col+'_diff_1_2_months'] = df[col+'_date_block_mean_lag_1'] - df[col+'_date_block_mean_lag_2']
df.drop(columns=[col+'_date_block_mean_lag_3'],
inplace=True)
 else:
for lag in lag_months:
df[col+'_month_lag_'+str(lag)] =\
df.groupby([col])['item_cnt_month'].shift(lag)
 df[col+'_month_lag_'+str(lag)].fillna(0, inplace = True)
 df[col+'_3_months_mean'] = df[[col+'_month_lag_1',
col+'_month_lag_2',
col+'_month_lag_3']].mean(axis = 1)
 df[col+'_3_months_std'] = df[[col+'_month_lag_1',
col+'_month_lag_2',
col+'_month_lag_3']].std(axis = 1)
 df[col+'_diff_1_2_months'] = df[col+'_month_lag_1'] - df[col+'_month_lag_2']
df.drop(columns=[col+'_month_lag_3'],
inplace=True)
 return df<categorify>
analysis[analysis['cp_time'] == 72]['atp-sensitive_potassium_channel_antagonist'].value_counts()
train_test_monthly = col_months_lag('shop_item_id', train_test_monthly, False)
 train_test_monthly = col_months_lag('shop_type_id', train_test_monthly)
 train_test_monthly = col_months_lag('item_id', train_test_monthly )<groupby>
for category in analysis['cp_type'].unique().tolist() :
number = 0
cols = list()
for col in analysis.columns:
if col in ['cp_type', 'cp_time', 'cp_dose']:
continue
if len(analysis[analysis['cp_type'] == category][col].value_counts())== 1:
number += 1
cols.append(col)
 print(category, '.Number of columns with 1 unique value: ', number, '.Columns: ', cols )
similar_group_mean = train_test_monthly.groupby(by=['date_block_num',
'shop_id',
'item_id'],
as_index=False)['item_cnt_month'].last().fillna(0)
 similar_group_mean.rename(columns={'item_cnt_month':'shop_similar_item_id_month_lag_1'},
inplace = True)
 similar_group_mean['item_id'] += 1
similar_group_mean['date_block_num'] += 1
train_test_monthly = train_test_monthly.merge(similar_group_mean,
on=['date_block_num', 'shop_id', 'item_id'],
how='left' )<compute_test_metric>
analysis[analysis['cp_type']=='ctl_vehicle']['igf-1_inhibitor'].value_counts()
def RMSE(y_true, y_pred):
return sqrt(mean_squared_error(y_true, y_pred))<split>
data = train_features.append(test_features )
def split_date_block_num(df, date_block_num):
split_1 = df[df['date_block_num'] < date_block_num].copy()
split_2 = df[df['date_block_num'] == date_block_num].copy().reset_index(drop=True)
 return split_1, split_2<split>
def preprocess(df):
df = df.copy()
df.drop(['cp_type'], axis=1, inplace=True)
 df.drop(['dataset'],axis=1,inplace=True)
 df.loc[:, 'cp_dose'] = df.loc[:, 'cp_dose'].map({'D1': 0, 'D2': 1})
 df.loc[:, 'cp_time'] = df.loc[:, 'cp_time'].map({24: 0, 48: 1, 72: 2})
 df = pd.get_dummies(df, columns=['cp_time','cp_dose'])
 del df['sig_id']
return df
train = preprocess(train_features)
 test = preprocess(test_features)
 del train_targets['sig_id']
train_sales, test = split_date_block_num(train_test_monthly, 34 )<filter>
scaler = preprocessing.MinMaxScaler()
scaler.fit(train.append(test))
train_trans = scaler.transform(train)
 test_trans = scaler.transform(test)
 train = pd.DataFrame(train_trans, columns=train.columns)
 test = pd.DataFrame(test_trans, columns=test.columns )
train_sales = train_sales[train_sales['date_block_num']>2]<define_variables>
somthing_rate = 1e-15
P_MIN = somthing_rate
P_MAX = 1 - P_MIN
def loss_fn(yt, yp):
yp = np.clip(yp, P_MIN, P_MAX)
 return log_loss(yt, yp, labels=[0,1])
select_feats = ['shop_id',
'item_id',
'shop_item_id',
'shop_cat_id',
'shop_type_id',
'item_city_month',
'shop_item_month',
'shop_item_id_month_lag_1',
'shop_item_id_month_lag_2',
'shop_item_id_month_lag_12',
'shop_item_id_3_months_mean',
'shop_item_id_3_months_std',
'shop_item_id_diff_1_2_months',
'shop_type_id_date_block_mean_lag_1',
'shop_type_id_date_block_mean_lag_2',
'shop_type_id_date_block_mean_lag_12',
'shop_type_id_3_months_mean',
'shop_type_id_3_months_std',
'shop_type_id_diff_1_2_months',
'item_id_date_block_mean_lag_1',
'item_id_date_block_mean_lag_2',
'item_id_date_block_mean_lag_12',
'item_id_3_months_mean',
'item_id_3_months_std',
'item_id_diff_1_2_months',
'shop_similar_item_id_month_lag_1'
]<init_hyperparams>
def create_model(num_columns, actv='relu'):
model = tf.keras.Sequential([tf.keras.layers.Input(num_columns)])
 model.add(tf.keras.layers.BatchNormalization())
 model.add(tf.keras.layers.Dropout(0.2))
model.add(tfa.layers.WeightNormalization(tf.keras.layers.Dense(1024, activation=actv)))
 if actv == 'elu':
model.add(tf.keras.layers.BatchNormalization())
 model.add(tf.keras.layers.AlphaDropout(0.2))
model.add(tfa.layers.WeightNormalization(tf.keras.layers.Dense(512, kernel_initializer='lecun_normal', activation='selu')))
 else:
model.add(tf.keras.layers.BatchNormalization())
 model.add(tf.keras.layers.Dropout(0.2))
model.add(tfa.layers.WeightNormalization(tf.keras.layers.Dense(1024, activation=actv)))
 model.add(tf.keras.layers.BatchNormalization())
 model.add(tfa.layers.WeightNormalization(tf.keras.layers.Dense(206, activation="sigmoid")))
 model.compile(optimizer=tfa.optimizers.AdamW(lr = 1e-3, weight_decay = 1e-5, clipvalue = 756),
loss=BinaryCrossentropy(label_smoothing=somthing_rate), 
)
 return model
params = {
'num_iterations':140,
'max_depth':15,
'objective': 'rmse',
'metric': 'rmse',
'num_leaves':(2 ** 11)- 1,
'learning_rate': 0.025,
'feature_fraction': 0.4,
'bagging_fraction': 0.2,
'bagging_freq': 5,
'seed': 1,
'verbose': 1
}
cat_cols = ['shop_id', 'item_id', 'shop_item_id', 'shop_cat_id',
'shop_type_id', 'item_city_month', 'shop_item_month']<split>
def metric(y_true, y_pred):
metrics = []
for _target in train_targets.columns:
metrics.append(loss_fn(y_true.loc[:, _target], y_pred.loc[:, _target].astype(float)))
 return np.mean(metrics )
errors = []
for block_num in range(24, 34):
train, validation = split_date_block_num(train_sales, block_num)
 x_train = train[select_feats].values
y_train = train['item_cnt_month'].values
x_val = validation[select_feats].values
y_val = validation['item_cnt_month'].values
del train
del validation
print('TRAINING %i MONTHS:'%(block_num))
start = time()
lgbmr_train = lgbm.Dataset(x_train, y_train)
 lgbmr_val = lgbm.Dataset(x_val, y_val, reference=lgbmr_train)
 lgbmr_trained = lgbm.train(params,
lgbmr_train,
feature_name=select_feats,
categorical_feature=cat_cols)
 end = time()
del lgbmr_train
del lgbmr_val
y_train_pred = lgbmr_trained.predict(x_train ).clip(0, 20)
 train_rmse = RMSE(y_train, y_train_pred)
 print('Train RMSE = %.5f'%(train_rmse))
y_val_pred = lgbmr_trained.predict(x_val ).clip(0, 20)
 val_rmse = RMSE(y_val, y_val_pred)
 print('Validation RMSE = %.5f'%(val_rmse))
errors.append(val_rmse)
 print('Training time = %.2f s'%(end - start))
print()
del x_train
del y_train
del x_val
del y_val
del lgbmr_trained
del y_train_pred
del y_val_pred<compute_test_metric>
N_STARTS = 14
S_STARTS = int(N_STARTS/2)
 res_relu = train_targets.copy()
res_elu = train_targets.copy()
res_relu.loc[:, train_targets.columns] = 0
res_elu.loc[:, train_targets.columns] = 0
ss_relu = sample_submission.copy()
ss_elu = sample_submission.copy()
ss_relu.loc[:, train_targets.columns] = 0
ss_elu.loc[:, train_targets.columns] = 0
ss_dict = {}
historys = dict()
tf.random.set_seed(42)
 for seed in range(N_STARTS):
for n,(tr, te)in enumerate(MultilabelStratifiedKFold(n_splits=7, random_state=seed, shuffle=True ).split(train_targets, train_targets)) :
print(f"======{train_targets.values[tr].shape}========{train_targets.values[te].shape}=====")
 if seed < S_STARTS:
print(f'Seed: {seed} => Fold: {n} ==>(RELU MODEL)')
 model = create_model(len(top_feats), actv='relu')
 else:
print(f'Seed: {seed} => Fold: {n} ==>(ELU MODEL)')
 model = create_model(len(top_feats), actv='elu')
 checkpoint_path = f'repeat:{seed}_Fold:{n}.hdf5'
reduce_lr_loss = ReduceLROnPlateau(monitor='val_loss', factor=0.2, min_lr=1e-6, patience=4, verbose=1, mode='auto')
 cb_checkpt = ModelCheckpoint(checkpoint_path, monitor = 'val_loss', verbose = 1, save_best_only = True,
save_weights_only = True, mode = 'auto')
 early = EarlyStopping(monitor="val_loss", mode="min", restore_best_weights=True, patience= 10, verbose = 1)
 history = model.fit(train.values[tr][:, top_feats],
train_targets.values[tr],
validation_data=(train.values[te][:, top_feats], train_targets.values[te]),
epochs=60, batch_size=128,
callbacks=[reduce_lr_loss, cb_checkpt, early], verbose=2 
)
 historys[f'history_{seed+1}'] = history
print("Model History Saved.")
 model.load_weights(checkpoint_path)
 test_predict = model.predict(test.values[:, top_feats])
 val_predict = model.predict(train.values[te][:, top_feats])
 if seed < S_STARTS:
ss_relu.loc[:, train_targets.columns] += test_predict
res_relu.loc[te, train_targets.columns] += val_predict
else:
ss_elu.loc[:, train_targets.columns] += test_predict
res_elu.loc[te, train_targets.columns] += val_predict
print(f'OOF Metric For SEED {seed} => FOLD {n} : {metric(train_targets.loc[te, train_targets.columns], pd.DataFrame(val_predict, columns=train_targets.columns)) }')
 print('+-' * 10)
 ss_relu.loc[:, train_targets.columns] /=(( n+1)* S_STARTS)
 res_relu.loc[:, train_targets.columns] /= S_STARTS
ss_elu.loc[:, train_targets.columns] /=(( n+1)* S_STARTS)
 res_elu.loc[:, train_targets.columns] /= S_STARTS
print('RMSE = %.5f +/- %.5f'%(np.array(errors ).mean() , np.array(errors ).std()))<prepare_x_and_y>
print(f'OOF Metric(relu): {metric(train_targets, res_relu)}')
 print(f'OOF Metric(elu): {metric(train_targets, res_elu)}' )
x_train = train_sales[select_feats].values
y_train = train_sales['item_cnt_month'].values
x_test = test[select_feats].values<train_model>
ss_elu.to_csv('submission_elu.csv', index=False )
<predict_on_test><EOS> 
ss_relu.to_csv('submission.csv', index=False )