Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<save_to_csv>
random.seed(42)
 init_notebook_mode(connected=True)
sample['target'] = y_pred
sample.to_csv("submission.csv", index=False )<import_modules>
df_train = pd.read_csv('.. /input/train.csv')
 df_comp = pd.read_csv('.. /input/test.csv' )
import pandas as pd
import numpy as np
import datetime
import gc
import os
import random
import time
import warnings
import pandas as pd
import numpy as np
import lightgbm as lgb
import xgboost
import catboost
import seaborn as sns
from pandas import DataFrame
from sklearn.metrics import roc_auc_score, f1_score, precision_recall_curve, auc
from sklearn.model_selection import StratifiedKFold, train_test_split, KFold
import matplotlib.pyplot as plt
from sklearn.preprocessing import KBinsDiscretizer, OneHotEncoder
from statsmodels.gam.tests.test_gam import sigmoid
from tqdm import tqdm 
<load_from_csv>
df_train.isnull().sum().sum()
train = pd.read_csv('/kaggle/input/tabular-playground-series-jan-2021/train.csv')
 test = pd.read_csv('/kaggle/input/tabular-playground-series-jan-2021/test.csv')
 train.shape,test.shape<init_hyperparams>
from sklearn.model_selection import train_test_split
label = 'target'
seed = 0
local_test = True
def seed_everything(seed):
random.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 seed_everything(seed)
 params = {
'objective': 'regression',
'boosting_type': 'gbdt',
'metric': 'rmse',
'n_jobs': -1,
'learning_rate': 0.006,
'num_leaves': 2 ** 8,
'max_depth': 8,
'tree_learner': 'serial',
'colsample_bytree': 0.8,
'subsample_freq': 1,
'subsample': 0.8,
'max_bin': 255,
'verbose': -1,
'seed': seed,
}
base_features = [x for x in train.columns if 'cont' in x]
remove_features = [label,'id']<define_variables>
from sklearn.model_selection import train_test_split
def make_test(new_features):
features = base_features + new_features
oof_predictions = np.zeros(len(train))
final_predictions = np.zeros(len(test))
cv = KFold(n_splits=10,shuffle=True,random_state=seed)
 if local_test:
n_estimators=1000
else:
n_estimators = 10000
lgb = LGBMRegressor(**params,n_estimators=n_estimators,device='GPU')
 for n,(trn_id,val_id)in enumerate(cv.split(train[features],train[label])) :
print(f"===== training fold {n+1} =====")
 trn_x,trn_y = train.loc[trn_id,features],train.loc[trn_id,label]
val_x,val_y = train.loc[val_id,features],train.loc[val_id,label]
lgb.fit(trn_x,trn_y,eval_set=[(val_x,val_y)],early_stopping_rounds=100,verbose=-1)
 oof_predictions[val_id] = lgb.predict(val_x)
 final_predictions += lgb.predict(test[features])/ cv.n_splits
mse_score = np.sqrt(mean_squared_error(y_true=val_y,y_pred=oof_predictions[val_id]))
del trn_x,trn_y,val_x,val_y
gc.collect()
cur_mse_score = np.sqrt(mean_squared_error(y_true=train[label],y_pred=oof_predictions))
cur_mae_score = mean_absolute_error(y_true=train[label],y_pred=oof_predictions)
 print(f"global mse score {cur_mse_score}")
 print(f"global mae score {cur_mae_score}")
 print(f"diff with previous version {score[0] - cur_mse_score}")
 print(f"diff with previous version {score[1] - cur_mae_score}")
 if not local_test:
test[label] = final_predictions
test[['id',label]].to_csv(f'sub_{np.round(cur_mse_score,4)}.csv',index=False)
 return [cur_mse_score,cur_mae_score]<define_variables>
Y = df_train.label
X = df_train.drop('label', axis=1)
 X = X / 255
X_comp = df_comp / 255
X_train, X_cross, Y_train, Y_cross = train_test_split(X, Y,test_size=0.1, random_state=42)
 X_valid, X_test, Y_valid, Y_test = train_test_split(X_cross, Y_cross, test_size=0.5, random_state=42 )
local_test=False
score = [0.6970820000536615, 0.5829603998473519]
make_test([] )<import_modules>
from keras.models import Sequential, load_model
from keras.layers import Dense, Conv2D, MaxPool2D, Dropout, Flatten
from keras.utils import plot_model, to_categorical
from keras.utils.vis_utils import model_to_dot
from keras.preprocessing.image import ImageDataGenerator
from sklearn.metrics import confusion_matrix, accuracy_score
import os
import joblib
import numpy as np
import pandas as pd
from sklearn.linear_model import LinearRegression, Ridge
from sklearn.tree import DecisionTreeRegressor
from sklearn.ensemble import RandomForestRegressor, AdaBoostRegressor
from sklearn.preprocessing import StandardScaler, MinMaxScaler
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import cross_val_score, cross_val_predict,RandomizedSearchCV, KFold
from lightgbm import LGBMRegressor
from xgboost import XGBRegressor
from catboost import CatBoostRegressor
import matplotlib
import matplotlib.pyplot as plt
import seaborn as sns
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
from tensorflow.keras.layers.experimental import preprocessing
import optuna<load_from_csv>
X_train = X_train.values.reshape(X_train.shape[0],28,28,1)
 X_valid = X_valid.values.reshape(X_valid.shape[0],28,28,1)
 X_test = X_test.values.reshape(X_test.shape[0],28,28,1)
 X_comp = X_comp.values.reshape(X_comp.shape[0],28,28,1)
 Y_train = to_categorical(Y_train)
 Y_valid = to_categorical(Y_valid)
 Y_test = to_categorical(Y_test )
train_df = pd.read_csv('/kaggle/input/tabular-playground-series-jan-2021/train.csv')
 test_df = pd.read_csv('/kaggle/input/tabular-playground-series-jan-2021/test.csv')
 submission = pd.read_csv('/kaggle/input/tabular-playground-series-jan-2021/sample_submission.csv' )<prepare_x_and_y>
datagen = ImageDataGenerator(height_shift_range=0.1,
width_shift_range=0.1,
rotation_range=10,
zoom_range=0.1,
fill_mode='constant',
cval=0 
)
 datagen.fit(X_train )
features = [feature for feature in train_df.columns if feature not in ['id', 'target']]
X_train = train_df[features]
y_train = train_df['target']
X_test = test_df[features]<count_missing_values>
model = Sequential()
droprate = 0.175
model.add(Conv2D(kernel_size=(2,2), filters=128, strides=(1,1), padding='same',activation='relu', input_shape=(28,28,1)))
 model.add(Conv2D(kernel_size=(2,2), filters=128, strides=(1,1), padding='same',activation='relu'))
model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))
 model.add(Dropout(droprate))
model.add(Conv2D(kernel_size=(2,2), filters=64, strides=(1,1), padding='same',activation='relu'))
model.add(Conv2D(kernel_size=(2,2), filters=64, strides=(1,1), padding='same',activation='relu'))
model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))
 model.add(Dropout(droprate))
model.add(Flatten())
 model.add(Dense(256, activation='relu'))
model.add(Dropout(droprate))
model.add(Dense(10, activation='softmax'))
model1 = model
model1.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'] )
print('Missing value in train dataset:', sum(train_df.isnull().sum()))
print('Missing value in test dataset:', sum(test_df.isnull().sum()))<choose_model_class>
epochsN = 25
batch_sizeN = 63
history1 = model1.fit_generator(datagen.flow(X_train, Y_train, batch_size=batch_sizeN), validation_data=(X_valid, Y_valid), steps_per_epoch=len(X_train)/batch_sizeN, epochs=epochsN, verbose=2 )
cv = KFold(n_splits=5, shuffle=True, random_state=42 )<compute_train_metric>
model1.evaluate(X_test, Y_test, verbose=0 )
%%time
lin_reg = LinearRegression()
scores = cross_val_score(lin_reg, X_train, y_train, scoring='neg_mean_squared_error', cv=cv, n_jobs=-1)
 lin_rmse_scores = np.sqrt(-scores)
 print('Linear Regression performance:', lin_rmse_scores )<compute_train_metric>
model1.save('model_1.h5' )
%%time
tree_reg = DecisionTreeRegressor(random_state=42)
 scores = cross_val_score(tree_reg, X_train, y_train, scoring='neg_mean_squared_error', cv=cv, n_jobs=-1)
 tree_rmse_scores = np.sqrt(-scores)
 print('Decision Tree Regressor performance:', tree_rmse_scores )<compute_train_metric>
del model
model = Sequential()
droprate = 0.15
model.add(Conv2D(kernel_size=(2,2), filters=128, strides=(1,1), padding='same',activation='relu', input_shape=(28,28,1)))
 model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))
 model.add(Dropout(droprate))
model.add(Conv2D(kernel_size=(2,2), filters=64, strides=(1,1), padding='same',activation='relu'))
model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))
 model.add(Dropout(droprate))
model.add(Conv2D(kernel_size=(2,2), filters=32, strides=(1,1), padding='same',activation='relu'))
model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))
 model.add(Dropout(droprate))
model.add(Flatten())
 model.add(Dense(256, activation='relu'))
model.add(Dropout(droprate))
model.add(Dense(10, activation='softmax'))
model2 = model
model2.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'] )
%%time
forest_reg = RandomForestRegressor(random_state=42, n_jobs=-1)
 scores = cross_val_score(forest_reg, X_train, y_train, scoring='neg_mean_squared_error', cv=cv, n_jobs=-1)
 forest_rmse_scores = np.sqrt(-scores)
 print('Random Forest performance:', forest_rmse_scores )<compute_train_metric>
epochsN = 35
batch_sizeN = 63
history2 = model2.fit_generator(datagen.flow(X_train, Y_train, batch_size=batch_sizeN), validation_data=(X_valid, Y_valid), steps_per_epoch=len(X_train)/batch_sizeN, epochs=epochsN, verbose=2 )
%%time
lgbm_reg = LGBMRegressor(random_state=42)
 scores = cross_val_score(lgbm_reg, X_train, y_train, scoring='neg_mean_squared_error', cv=cv, n_jobs=-1)
 lgbm_rmse_scores = np.sqrt(-scores)
 print('LGBM performance:', lgbm_rmse_scores )<compute_train_metric>
model2.evaluate(X_test, Y_test, verbose=0 )
%%time
xgb_reg = XGBRegressor(random_state=42)
 scores = cross_val_score(xgb_reg, X_train, y_train, scoring='neg_mean_squared_error', cv=cv, n_jobs=-1)
 xgb_rmse_scores = np.sqrt(-scores)
 print('XGBoost performance:', xgb_rmse_scores )<compute_train_metric>
model2.save('model_2.h5' )
%%time
cb_reg = CatBoostRegressor(random_state=42, verbose=False)
 scores = cross_val_score(cb_reg, X_train, y_train, scoring='neg_mean_squared_error', cv=cv, n_jobs=-1)
 cb_rmse_scores = np.sqrt(-scores)
 print('CatBoost performance:', cb_rmse_scores )<compute_train_metric>
del model
model = Sequential()
droprate = 0.2
model.add(Conv2D(kernel_size=(2,2), filters=128, strides=(1,1), padding='same',activation='relu', input_shape=(28,28,1)))
 model.add(Conv2D(kernel_size=(2,2), filters=128, strides=(1,1), padding='same',activation='relu'))
model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))
 model.add(Dropout(droprate))
model.add(Conv2D(kernel_size=(2,2), filters=64, strides=(1,1), padding='same',activation='relu'))
model.add(Conv2D(kernel_size=(2,2), filters=64, strides=(1,1), padding='same',activation='relu'))
model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))
 model.add(Dropout(droprate))
model.add(Conv2D(kernel_size=(2,2), filters=32, strides=(1,1), padding='same',activation='relu'))
model.add(Conv2D(kernel_size=(2,2), filters=32, strides=(1,1), padding='same',activation='relu'))
model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))
 model.add(Dropout(droprate))
model.add(Flatten())
 model.add(Dense(256, activation='relu'))
model.add(Dropout(droprate))
model.add(Dense(10, activation='softmax'))
model3 = model
model3.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'] )
%%time
ab_reg = AdaBoostRegressor(random_state=42)
 scores = cross_val_score(ab_reg, X_train, y_train, scoring='neg_mean_squared_error', cv=cv, n_jobs=-1)
 ab_rmse_scores = np.sqrt(-scores)
 print('AdaBoost performance:', ab_rmse_scores )<choose_model_class>
epochsN = 40
batch_sizeN = 63
history3 = model3.fit_generator(datagen.flow(X_train, Y_train, batch_size=batch_sizeN), validation_data=(X_valid, Y_valid), steps_per_epoch=len(X_train)/batch_sizeN, epochs=epochsN, verbose=2 )
def build_and_compile_model(norm):
model = keras.Sequential([
norm,
layers.Dense(64, activation='relu'),
layers.Dense(64, activation='relu'),
layers.Dense(1)])
 model.compile(loss='mean_squared_error',
optimizer=tf.keras.optimizers.Adam(0.001))
return model<train_model>
model3.evaluate(X_test, Y_test, verbose=0 )
%%time
normalizer = preprocessing.Normalization()
normalizer.adapt(np.array(X_train))
dnn_model = build_and_compile_model(normalizer)
 history = dnn_model.fit(X_train, y_train, validation_split=0.2,
verbose=0, epochs=100 )<predict_on_test>
model3.save('model_3.h5' )
%%time
lin_reg = LinearRegression()
y_predict = cross_val_predict(lin_reg, X_train, y_train, cv=cv, n_jobs=-1 )<predict_on_test>
del model
model = Sequential()
droprate = 0.20
model.add(Conv2D(kernel_size=(2,2), filters=128, strides=(1,1), padding='same',activation='relu', input_shape=(28,28,1)))
 model.add(Conv2D(kernel_size=(2,2), filters=128, strides=(1,1), padding='same',activation='relu'))
model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))
 model.add(Dropout(droprate))
model.add(Conv2D(kernel_size=(2,2), filters=64, strides=(1,1), padding='same',activation='relu'))
model.add(Conv2D(kernel_size=(2,2), filters=64, strides=(1,1), padding='same',activation='relu'))
model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))
 model.add(Dropout(droprate))
model.add(Conv2D(kernel_size=(2,2), filters=32, strides=(1,1), padding='same',activation='relu'))
model.add(Conv2D(kernel_size=(2,2), filters=32, strides=(1,1), padding='same',activation='relu'))
model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))
 model.add(Dropout(droprate))
model.add(Conv2D(kernel_size=(2,2), filters=16, strides=(1,1), padding='same',activation='relu'))
model.add(Conv2D(kernel_size=(2,2), filters=16, strides=(1,1), padding='same',activation='relu'))
model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))
 model.add(Dropout(droprate))
model.add(Flatten())
 model.add(Dense(256, activation='relu'))
model.add(Dropout(droprate))
model.add(Dense(10, activation='softmax'))
model4 = model
model4.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'] )
%%time
tree_reg = DecisionTreeRegressor(random_state=42)
 y_predict = cross_val_predict(tree_reg, X_train, y_train, cv=cv, n_jobs=-1 )<predict_on_test>
epochsN = 90
batch_sizeN = 63
history4 = model4.fit_generator(datagen.flow(X_train, Y_train, batch_size=batch_sizeN), validation_data=(X_valid, Y_valid), steps_per_epoch=len(X_train)/batch_sizeN, epochs=epochsN, verbose=2 )
%%time
forest_reg = RandomForestRegressor(random_state=42, n_jobs=-1)
 y_predict = cross_val_predict(forest_reg, X_train, y_train, cv=cv, n_jobs=-1 )<predict_on_test>
model4.evaluate(X_test, Y_test, verbose=0 )
%%time
lgbm_reg = LGBMRegressor(random_state=42)
 y_predict = cross_val_predict(lgbm_reg, X_train, y_train, cv=cv, n_jobs=-1 )<predict_on_test>
model4.save('model_4.h5' )
%%time
xgb_reg = XGBRegressor(random_state=42)
 y_predict = cross_val_predict(xgb_reg, X_train, y_train, cv=cv, n_jobs=-1 )<predict_on_test>
del model
model = Sequential()
droprate = 0.1
model.add(Conv2D(kernel_size=(2,2), filters=64, strides=(1,1), padding='same',activation='relu', input_shape=(28,28,1)))
 model.add(Conv2D(kernel_size=(2,2), filters=64, strides=(2,2), padding='valid',activation='relu'))
model.add(Dropout(droprate))
model.add(Conv2D(kernel_size=(2,2), filters=32, strides=(1,1), padding='same',activation='relu'))
model.add(Conv2D(kernel_size=(2,2), filters=32, strides=(2,2), padding='valid',activation='relu'))
model.add(Dropout(droprate))
model.add(Conv2D(kernel_size=(2,2), filters=16, strides=(1,1), padding='same',activation='relu'))
model.add(Conv2D(kernel_size=(3,3), filters=16, strides=(2,2), padding='valid',activation='relu'))
model.add(Flatten())
 model.add(Dropout(droprate))
model.add(Dense(128, activation='relu'))
model.add(Dropout(droprate))
model.add(Dense(10, activation='softmax'))
model5 = model
model5.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'] )
%%time
cb_reg = CatBoostRegressor(random_state=42, verbose=False)
 y_predict = cross_val_predict(cb_reg, X_train, y_train, cv=cv, n_jobs=-1 )<predict_on_test>
epochsN = 90
batch_sizeN = 63
history5 = model5.fit_generator(datagen.flow(X_train, Y_train, batch_size=batch_sizeN), validation_data=(X_valid, Y_valid), steps_per_epoch=len(X_train)/batch_sizeN, epochs=epochsN, verbose=2 )
%%time
ab_reg = AdaBoostRegressor(random_state=42)
 y_predict = cross_val_predict(ab_reg, X_train, y_train, cv=cv, n_jobs=-1 )<compute_train_metric>
model5.evaluate(X_test, Y_test, verbose=0 )
def objective(trial):
params = {
'random_state': 42,
'max_depth': trial.suggest_int('max_depth', 1, 14),
'learning_rate': trial.suggest_float('learning_rate', 0.01, 1.0)
 }
lgbm_reg = LGBMRegressor()
lgbm_reg.set_params(**params)
 scores = cross_val_score(lgbm_reg, X_train, y_train, scoring='neg_mean_squared_error', cv=cv, n_jobs=-1)
 rmse = np.sqrt(-scores)
 return np.mean(rmse )<find_best_params>
model5.save('model_5.h5' )
study = optuna.create_study(direction = 'minimize')
 study.optimize(objective, n_trials = 1)
 best_params = study.best_trial.params<compute_train_metric>
del model
model = Sequential()
droprate = 0.15
model.add(Conv2D(kernel_size=(2,2), filters=64, strides=(1,1), padding='same',activation='relu', input_shape=(28,28,1)))
 model.add(Conv2D(kernel_size=(2,2), filters=64, strides=(1,1), padding='same',activation='relu'))
model.add(Conv2D(kernel_size=(2,2), filters=64, strides=(1,1), padding='same',activation='relu'))
model.add(Conv2D(kernel_size=(2,2), filters=64, strides=(2,2), padding='valid',activation='relu'))
model.add(Dropout(droprate))
model.add(Conv2D(kernel_size=(2,2), filters=32, strides=(1,1), padding='same',activation='relu'))
model.add(Conv2D(kernel_size=(2,2), filters=32, strides=(1,1), padding='same',activation='relu'))
model.add(Conv2D(kernel_size=(2,2), filters=32, strides=(1,1), padding='same',activation='relu'))
model.add(Conv2D(kernel_size=(2,2), filters=16, strides=(2,2), padding='valid',activation='relu'))
model.add(Flatten())
 model.add(Dropout(droprate))
model.add(Dense(256, activation='relu'))
model.add(Dropout(droprate))
model.add(Dense(10, activation='softmax'))
model6 = model
model6.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'] )
%%time
lgbm_reg = LGBMRegressor()
lgbm_reg.set_params(**best_params)
 scores = cross_val_score(lgbm_reg, X_train, y_train, scoring='neg_mean_squared_error', cv=cv, n_jobs=-1)
 lgbm_rmse_scores = np.sqrt(-scores)
 print('LGBM performance:', lgbm_rmse_scores )<compute_train_metric>
epochsN = 45
batch_sizeN = 63
history6 = model6.fit_generator(datagen.flow(X_train, Y_train, batch_size=batch_sizeN), validation_data=(X_valid, Y_valid), steps_per_epoch=len(X_train)/batch_sizeN, epochs=epochsN, verbose=2 )
def objective(trial):
params = {
'random_state': 42,
'max_depth': trial.suggest_int('max_depth', 1, 14),
'eta': trial.suggest_float('eta', 0.01, 1.0),
}
xgb_reg = XGBRegressor()
xgb_reg.set_params(**params)
 scores = cross_val_score(xgb_reg, X_train, y_train, scoring='neg_mean_squared_error', cv=cv, n_jobs=-1)
 rmse = np.sqrt(-scores)
 return np.mean(rmse )<find_best_params>
model6.evaluate(X_test, Y_test, verbose=0 )
study = optuna.create_study(direction = 'minimize')
 study.optimize(objective, n_trials = 1)
 best_params = study.best_trial.params<compute_train_metric>
model6.save('model_6.h5' )
%%time
xgb_reg = XGBRegressor()
xgb_reg.set_params(**best_params)
 scores = cross_val_score(xgb_reg, X_train, y_train, scoring='neg_mean_squared_error', cv=cv, n_jobs=-1)
 xgb_rmse_scores = np.sqrt(-scores)
 print('XGBoost performance:', xgb_rmse_scores )<feature_engineering>
del model
model = Sequential()
droprate = 0.35
model.add(Conv2D(kernel_size=(2,2), filters=64, strides=(1,1), padding='same',activation='relu', input_shape=(28,28,1)))
 model.add(Conv2D(kernel_size=(2,2), filters=64, strides=(1,1), padding='same',activation='relu'))
model.add(Conv2D(kernel_size=(2,2), filters=64, strides=(2,2), padding='valid',activation='relu'))
model.add(Dropout(droprate))
model.add(Conv2D(kernel_size=(3,3), filters=128, strides=(1,1), padding='same',activation='relu'))
model.add(Conv2D(kernel_size=(3,3), filters=128, strides=(1,1), padding='same',activation='relu'))
model.add(Conv2D(kernel_size=(2,2), filters=128, strides=(2,2), padding='valid',activation='relu'))
model.add(Dropout(droprate))
model.add(Conv2D(kernel_size=(3,3), filters=256, strides=(1,1), padding='valid',activation='relu'))
model.add(Conv2D(kernel_size=(3,3), filters=256, strides=(1,1), padding='valid',activation='relu'))
model.add(Conv2D(kernel_size=(3,3), filters=256, strides=(2,2), padding='valid',activation='relu'))
model.add(Dropout(droprate))
model.add(Flatten())
 model.add(Dense(10, activation='softmax'))
model7 = model
model7.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'] )
X_train['below8'] = np.where(y_train < 8, 1, 0 )<compute_train_metric>
epochsN = 60
batch_sizeN = 63
history7 = model7.fit_generator(datagen.flow(X_train, Y_train, batch_size=batch_sizeN), validation_data=(X_valid, Y_valid), steps_per_epoch=len(X_train)/batch_sizeN, epochs=epochsN, verbose=2 )
%%time
cb_reg = CatBoostRegressor(random_state=42, verbose=False)
 scores = cross_val_score(cb_reg, X_train, y_train, scoring='neg_mean_squared_error', cv=5)
 cb_rmse_scores = np.sqrt(-scores)
 print('CatBoost performance:', cb_rmse_scores )<import_modules>
model7.evaluate(X_test, Y_test, verbose=0 )
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
from xgboost import XGBRegressor
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error<load_from_csv>
model7.save('model_7.h5' )
train = pd.read_csv('.. /input/tabular-playground-series-jan-2021/train.csv', index_col='id')
 test = pd.read_csv('.. /input/tabular-playground-series-jan-2021/test.csv', index_col='id' )<set_options>
trained_models = [model1, model2, model3, model4, model5, model6, model7]
plt.style.use('ggplot')
 plt.rcParams['axes.titlesize'] = 16
plt.rcParams['axes.labelsize'] = 12
plt.rcParams['xtick.labelsize'] = 'large'<count_missing_values>
acc_scores = pd.Series()
for num, model in enumerate(trained_models):
acc_scores.loc['Model ' + str(num + 1)] = accuracy_score(np.argmax(Y_test, axis=1), np.argmax(model.predict(X_test), axis=1))
print('Missing values on the train data:', train.isnull().sum().sum())
 print('Missing values on the test data:', test.isnull().sum().sum() )<count_duplicates>
def summing_classifier(data, model_list):
total_pred_prob = model_list[0].predict(data)
 for model in model_list[1:]:
total_pred_prob += model.predict(data)
 return np.argmax(total_pred_prob, axis=1 )
print('Duplicated rows on the train data:', train.duplicated().sum())
 print('Duplicated rows on the test data:', test.duplicated().sum() )<define_variables>
acc_scores.loc['Summing Classifier'] = accuracy_score(np.argmax(Y_test, axis=1), summing_classifier(X_test, trained_models))
acc_scores.loc['Summing Classifier']
q1 = train.quantile(0.25)
 q3 = train.quantile(0.75)
 iqr = q3 - q1
mask =(train >=(q1 - 1.5*iqr)) &(train <= q3 + 1.5*iqr)
 train = train[mask.apply(all, axis=1)]
print('Train set without outliers shape:', train.shape )<split>
def voting_classifier(data, model_list):
pred_list = np.argmax(model_list[0].predict(data), axis=1 ).reshape(( 1,len(data)))
 for model in model_list[1:]:
pred_list = np.append(pred_list, [np.argmax(model.predict(data), axis=1)], axis=0)
 return np.array(list(map(lambda x: np.bincount(x ).argmax() , pred_list.T)) )
X_train, X_val, y_train, y_val = train_test_split(train[predictors],
train[target],
test_size = 0.2,
random_state=2021 )<choose_model_class>
acc_scores.loc['Voting Classifier'] = accuracy_score(np.argmax(Y_test, axis=1), voting_classifier(X_test, trained_models))
acc_scores.loc['Voting Classifier']
model = XGBRegressor(objective='reg:squarederror',
booster = "gbtree",
eval_metric = "rmse",
tree_method = "gpu_hist",
n_estimators = 1000,
learning_rate = 0.04,
eta = 0.1,
max_depth = 7,
subsample=0.85,
colsample_bytree = 0.85,
colsample_bylevel = 0.8,
alpha = 0,
random_state = 2021 )<train_model>
best_model_results = pd.DataFrame({'Label' : np.argmax(trained_models[ind_best_model].predict(X_comp), axis=1)})
 best_model_results = best_model_results.reset_index().rename(columns={'index' : 'ImageId'})
 best_model_results['ImageId'] = best_model_results['ImageId'] + 1
best_model_results.to_csv('best_model_result_kaggle.csv', index=False )
%time model.fit(X_train, y_train )<predict_on_test>
esmbl_sum_results = pd.DataFrame({'Label' : summing_classifier(X_comp, trained_models)})
 esmbl_sum_results = esmbl_sum_results.reset_index().rename(columns={'index' : 'ImageId'})
 esmbl_sum_results['ImageId'] = esmbl_sum_results['ImageId'] + 1
esmbl_sum_results.to_csv('esmbl_sum_result_kaggle.csv', index=False )
<save_to_csv><EOS> 
esmbl_vote_results = pd.DataFrame({'Label' : voting_classifier(X_comp, trained_models)})
 esmbl_vote_results = esmbl_vote_results.reset_index().rename(columns={'index' : 'ImageId'})
 esmbl_vote_results['ImageId'] = esmbl_vote_results['ImageId'] + 1
esmbl_vote_results.to_csv('esmbl_vote_result_kaggle.csv', index=False )