Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<define_variables>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix, roc_curve, auc, roc_auc_score
from sklearn.preprocessing import label_binarize
from sklearn.svm import SVC
from itertools import cycle
from scipy import interp
from keras.optimizers import Adam
from keras.callbacks import Callback, EarlyStopping, ModelCheckpoint
from keras import backend as K
from keras.layers import Conv2D, MaxPooling2D
from keras.layers import Dense, Input, Dropout, Flatten
from keras.models import Model, load_model
from keras.preprocessing.image import ImageDataGenerator
numerical_features = [f"cont{i}" for i in range(13 + 1)]<concatenate>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )
train_x = pd.concat([
df_train[numerical_features],
train_ohe,
train_or
], axis=1 )<concatenate>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )
test_x = pd.concat([
df_test[numerical_features],
test_ohe,
test_or
], axis=1 )<prepare_x_and_y>
X_train, X_valid, y_train, y_valid = train_test_split(train.drop(['label'], axis=1), train['label'], random_state = 0 )
train_y = df_train["target"]<create_dataframe>
X_train = X_train.values.reshape(-1,28,28,1)
 X_valid = X_valid.values.reshape(-1,28,28,1)
 y_train = label_binarize(y_train, classes=range(10))
y_valid = label_binarize(y_valid, classes=range(10))
oplgb_train_data = oplgb.Dataset(train_x, train_y )<init_hyperparams>
class RocAucEvaluation(Callback):
def __init__(self, validation_data=() , interval=1):
super(Callback, self ).__init__()
self.interval = interval
self.X_val, self.y_val = validation_data
def on_epoch_end(self, epoch, logs={}):
if epoch % self.interval == 0:
y_pred = self.model.predict(self.X_val, verbose=0)
 score = roc_auc_score(self.y_val, y_pred)
 print("
ROC-AUC - epoch: {:d} - score: {:.6f}".format(epoch+1, score))
oplgb_params = {
"objective": "regression",
"metric": "root_mean_squared_error",
"verbosity": -1,
"learning_rate": 0.01
}<choose_model_class>
class CNN:
def __init__(self):
self.arguments = {
'batch_size': 64,
'epochs': 100,
'learning_rate': 1e-3,
'learning_rate_decay': 0,
'units': 128,
'drop_out_rate': 0.2,
'checkpoint_path': 'best_bilstm_model.hdf5',
'early_stop_patience': 10,
}
print('Building CNN Models...')
 print(self.arguments)
 def fit(self, X_train, y_train, X_valid, y_valid):
file_path = self.arguments['checkpoint_path']
check_point = ModelCheckpoint(file_path, monitor = "val_loss", verbose = 1,
save_best_only = True, mode = "min")
 ra_val = RocAucEvaluation(validation_data=(X_valid, y_valid), interval = 1)
 early_stop = EarlyStopping(monitor = "val_loss", mode = "min", patience = self.arguments['early_stop_patience'])
 inp = Input(shape=(28,28,1))
x = Conv2D(filters=20, kernel_size =(5, 5), activation="relu" )(inp)
 max_pool_x = MaxPooling2D(pool_size=(2,2))(x)
 y = Conv2D(filters=20, kernel_size =(5, 5), activation="relu" )(max_pool_x)
 max_pool_y = MaxPooling2D(pool_size=(2,2))(y)
 flat = Flatten()(max_pool_y)
 z = Dense(100, activation="relu" )(flat)
 z = Dropout(rate = self.arguments['drop_out_rate'] )(z)
 z = Dense(100, activation="relu" )(z)
 z = Dropout(rate = self.arguments['drop_out_rate'] )(z)
 output = Dense(10, activation="softmax" )(z)
 self.model = Model(inputs = inp, outputs = output)
 self.model.compile(loss = "categorical_crossentropy", optimizer = Adam(lr = self.arguments['learning_rate'],
decay = self.arguments['learning_rate_decay']), metrics = ["accuracy"])
 history = self.model.fit(X_train, y_train, batch_size = self.arguments['batch_size'], epochs = self.arguments['epochs'],
validation_data =(X_valid, y_valid), verbose = 1, callbacks = [ra_val, check_point, early_stop])
 self.model = load_model(file_path)
 print('Finished Building CNN Model as class attribute class.model')
 return self
def predict(self, X, batch_size, verbose):
return self.model.predict(X, batch_size = batch_size, verbose = verbose )
folds = KFold(n_splits=5, shuffle=True, random_state=2021 )<choose_model_class>
cnn = CNN()
cnn.fit(X_train, y_train, X_valid, y_valid)
 cnn_pred = cnn.predict(X_valid, batch_size = cnn.arguments['batch_size'], verbose = 1 )
tuner_cv = oplgb.LightGBMTunerCV(oplgb_params, oplgb_train_data, num_boost_round=1000, early_stopping_rounds=100, folds=folds, verbose_eval=100, time_budget=21600)
 tuner_cv.run()<find_best_params>
datagen = ImageDataGenerator(
 rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
)
tuner_cv.best_params<train_model>
train_gen = datagen.flow(X_train, y_train, batch_size=64)
 valid_gen = datagen.flow(X_valid, y_valid, batch_size=64 )
class FoldsAverageLGBM:
def __init__(self, folds):
self.folds = folds
self.models = []
def fit(self, lgb_params, train_x, train_y):
oof_preds = np.zeros_like(train_y)
 self.train_x = train_x
self.train_y = train_y.values
for tr_idx, va_idx in tqdm(folds.split(train_x)) :
tr_x, va_x = self.train_x.iloc[tr_idx], self.train_x.iloc[va_idx]
tr_y, va_y = self.train_y[tr_idx], self.train_y[va_idx]
lgb_train_dataset = lgb.Dataset(tr_x, tr_y)
 lgb_valid_dataset = lgb.Dataset(va_x, va_y)
 model = lgb.train(lgb_params, lgb_train_dataset, valid_sets=[lgb_valid_dataset], verbose_eval=100)
 self.models.append(model)
 oof_pred = model.predict(va_x)
 oof_preds[va_idx] = oof_pred
self.oof_preds = oof_preds
def predict(self, test_x):
preds = []
for model in tqdm(self.models):
pred = model.predict(test_x)
 preds.append(pred)
 preds = np.mean(preds, axis=0)
 return preds
def get_feature_importance(self, importance_type="gain"):
feature_names = self.models[0].feature_name()
feature_importances_list = [model.feature_importance(importance_type)for model in self.models]
out_df = pd.DataFrame()
for i, name in enumerate(feature_names):
out_df[name] = [v[i] for v in feature_importances_list]
return out_df<init_hyperparams>
class CNN_Generator:
def __init__(self):
self.arguments = {
'batch_size': 64,
'epochs': 100,
'learning_rate': 1e-3,
'learning_rate_decay': 0,
'units': 128,
'drop_out_rate': 0.2,
'checkpoint_path': 'best_bilstm_model.hdf5',
'early_stop_patience': 10,
}
print('Building CNN_Generator Models...')
 print(self.arguments)
 def fit(self, train_gen, valid_gen):
file_path = self.arguments['checkpoint_path']
check_point = ModelCheckpoint(file_path, monitor = "val_loss", verbose = 1,
save_best_only = True, mode = "min")
 ra_val = RocAucEvaluation(validation_data=(X_valid, y_valid), interval = 1)
 early_stop = EarlyStopping(monitor = "val_loss", mode = "min", patience = self.arguments['early_stop_patience'])
 inp = Input(shape=(28,28,1))
x = Conv2D(filters=20, kernel_size =(5, 5), activation="relu" )(inp)
 max_pool_x = MaxPooling2D(pool_size=(2,2))(x)
 y = Conv2D(filters=20, kernel_size =(5, 5), activation="relu" )(max_pool_x)
 max_pool_y = MaxPooling2D(pool_size=(2,2))(y)
 flat = Flatten()(max_pool_y)
 z = Dense(100, activation="relu" )(flat)
 z = Dropout(rate = self.arguments['drop_out_rate'] )(z)
 z = Dense(100, activation="relu" )(z)
 z = Dropout(rate = self.arguments['drop_out_rate'] )(z)
 output = Dense(10, activation="softmax" )(z)
 self.model = Model(inputs = inp, outputs = output)
 self.model.compile(loss = "categorical_crossentropy", optimizer = Adam(lr = self.arguments['learning_rate'],
decay = self.arguments['learning_rate_decay']), metrics = ["accuracy"])
 history = self.model.fit_generator(train_gen, epochs = self.arguments['epochs'],
steps_per_epoch = X_train.shape[0] // self.arguments['batch_size'],
validation_steps = X_train.shape[0] // self.arguments['batch_size'],
validation_data = valid_gen, verbose = 1, callbacks = [ra_val, check_point, early_stop])
 self.model = load_model(file_path)
 print('Finished Building CNN Model as class attribute class.model')
 return self
def predict(self, X, batch_size, verbose):
return self.model.predict(X, batch_size = batch_size, verbose = verbose )
lgb_params = dict(tuner_cv.best_params)
 lgb_params["learning_rate"] = 0.001
lgb_params["early_stopping_round"] = 1000
lgb_params["num_iterations"] = 20000<statistical_test>
cnn_gen = CNN_Generator()
cnn_gen.fit(train_gen, valid_gen)
 cnn_gen_pred = cnn_gen.predict(X_valid, batch_size = cnn.arguments['batch_size'], verbose = 1 )
folds_average_lgbm = FoldsAverageLGBM(folds )<train_model>
train_predictions = []
valid_predictions = []
test_predictions = []
for i in range(5):
cnn = CNN_Generator()
cnn.fit(train_gen, valid_gen)
 train_predictions += [cnn.predict(X_train, batch_size = cnn.arguments['batch_size'], verbose = 0)]
valid_predictions += [cnn.predict(X_valid, batch_size = cnn.arguments['batch_size'], verbose = 1)]
test_predictions += [cnn.predict(test.values.reshape(-1,28,28,1), batch_size = cnn.arguments['batch_size'], verbose = 1)]
folds_average_lgbm.fit(lgb_params, train_x, train_y )<compute_test_metric>
train_pred = np.concatenate(train_predictions, axis=1)
 valid_pred = np.concatenate(valid_predictions, axis=1)
 test_pred = np.concatenate(test_predictions, axis=1)
 svm = SVC(probability=True, gamma='scale' ).fit(train_pred, np.argmax(y_train, axis=1))
pred = svm.predict(valid_pred)
 print("val_acc: ", round(np.sum(pred == np.argmax(y_valid, axis=1)) /y_valid.shape[0], 4))
print("val_ROC-AUC: ", round(roc_auc_score(y_valid, svm.predict_proba(valid_pred)) , 6))
<predict_on_test><EOS> 
sample_submission = pd.read_csv(".. /input/sample_submission.csv")
 sample_submission['Label'] = svm.predict(test_pred)
 sample_submission.head()
sample_submission.to_csv('submission.csv', index=False )