Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<train_model>
print(os.listdir(".. /input"))
train_oof = np.zeros(( 300000,))
test_preds = 0
train_oof.shape
NUM_FOLDS = 5
kf = KFold(n_splits=NUM_FOLDS, shuffle=True, random_state=42)
 for f,(train_ind, val_ind)in tqdm(enumerate(kf.split(xmod, ymod))):
tmp_train_df, tmp_val_df = xmod.iloc[train_ind][feature_cols_mod], xmod.iloc[val_ind][feature_cols_mod]
train_target, val_target = ymod[train_ind], ymod[val_ind]
model = XGBRegressor(**xgb_params)
 model.fit(
 tmp_train_df,
train_target,
eval_set=[(tmp_val_df, val_target)],
early_stopping_rounds = 50,
verbose = False 
)
 temp_oof = model.predict(tmp_val_df)
 temp_test = model.predict(mod_test_df[feature_cols_mod])
 train_oof[val_ind] = temp_oof
test_preds += temp_test/NUM_FOLDS
print(mean_squared_error(temp_oof, val_target, squared=False))<save_to_csv>
img_rows, img_cols = 28, 28
sub_df['target'] = test_preds
sub_df.to_csv('submission_xgb_mod_cv_optimized.csv', index=False)
 sub_df.head()
sub_xgb_mod_cv_optimized = test_preds<split>
def load_dataset(train_path,test_path):
global train,test,trainX,trainY,nb_classes
train = pd.read_csv(train_path ).values
test = pd.read_csv(test_path ).values
print("Train Shape :",train.shape)
 trainX = train[:, 1:].reshape(train.shape[0], img_rows, img_cols, 1)
 trainX = trainX.astype(float)
 trainX /= 255.0
trainY = kutils.to_categorical(train[:, 0])
 nb_classes = trainY.shape[1]
print("TrainX Shape : ",trainX.shape)
 print("Trainy shape : ",trainY.shape)
 testX = test.reshape(test.shape[0], 28, 28, 1)
 testX = testX.astype(float)
 testX /= 255.0
trainY = kutils.to_categorical(train[:, 0])
 return train,test,trainX,trainY,testX,nb_classes
def objective(trial,data=x,target=y):
train_x, test_x, train_y, test_y = train_test_split(data, target, test_size=0.15,random_state=42)
 param = {
'device':'gpu',
'metric': 'rmse',
'random_state': 42,
'reg_lambda': trial.suggest_loguniform(
 'reg_lambda', 1e-3, 10.0 
),
'reg_alpha': trial.suggest_loguniform(
 'reg_alpha', 1e-3, 10.0 
),
'colsample_bytree': trial.suggest_categorical(
 'colsample_bytree', [0.3,0.5,0.6,0.7,0.8,0.9,1.0] 
),
'subsample': trial.suggest_categorical(
 'subsample', [0.6,0.7,0.8,1.0] 
),
'learning_rate': trial.suggest_categorical(
 'learning_rate', [0.008,0.009,0.01,0.012,0.014,0.016,0.018, 0.02] 
),
'n_estimators': trial.suggest_categorical(
 "n_estimators", [150, 200, 300, 3000] 
),
'max_depth': trial.suggest_categorical(
 'max_depth', [4,5,7,9,11,13,15,17,20] 
),
'min_child_samples': trial.suggest_int(
 'min_child_samples', 1, 300 
),
'num_leaves': trial.suggest_int(
 'num_leaves', 15, 120 
),
}
model = LGBMRegressor(**param)
 model.fit(train_x,train_y,eval_set=[(test_x,test_y)], early_stopping_rounds=300, verbose=False)
 preds = model.predict(test_x)
 rmse = mean_squared_error(test_y, preds,squared=False)
 return rmse<train_model>
def createModel(inp_shape,nClasses):
model = models.Sequential()
model.add(Conv2D(32,(3, 3), padding='same', activation='relu', input_shape=inp_shape))
model.add(Conv2D(32,(3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.25))
model.add(Conv2D(64,(3, 3), padding='same', activation='relu'))
model.add(Conv2D(64,(3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.25))
model.add(Conv2D(64,(3, 3), padding='same', activation='relu'))
model.add(Conv2D(64,(3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(128, activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.25))
model.add(Dense(nClasses, activation='softmax'))
optimizer1 = RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0)
 model.compile(optimizer=optimizer1, loss='categorical_crossentropy', metrics=['accuracy'])
 return model
study = optuna.create_study(direction='minimize')
 study.optimize(objective, n_trials=5)
 print('Number of finished trials:', len(study.trials))
print('Best trial:', study.best_trial.params )<find_best_params>
def submission(prediction):
np.savetxt('mnist-submission.csv', np.c_[range(1,len(prediction)+1),prediction], delimiter=',', header = 'ImageId,Label', comments = '', fmt='%d' )
study.best_params<compute_train_metric>
def classification_report(X_test,test):
predicted_classes = model.predict_classes(X_test)
 y_true = test.iloc[:, 0]
correct = np.nonzero(predicted_classes==y_true)[0]
incorrect = np.nonzero(predicted_classes!=y_true)[0]
target_names = ["Class {}".format(i)for i in range(num_classes)]
print(classification_report(y_true, predicted_classes, target_names=target_names))
best_params = {
'reg_lambda': 0.015979956459638782,
'reg_alpha': 9.103977313355028,
'colsample_bytree': 0.3,
'subsample': 1.0,
'learning_rate': 0.009,
'n_estimators': 3000,
'max_depth': 15,
'min_child_samples': 142,
'num_leaves': 84,
'random_state': 42,
'device': 'gpu',
}
clf = LGBMRegressor(**best_params)
 clf.fit(x_train, y_train)
 predictions = clf.predict(x_test)
 score_rmse = math.sqrt(mean_squared_error(y_test, predictions))
print(Fore.GREEN + 'Base LGBM RMSE: {}'.format(score_rmse))<save_to_csv>
train_path=".. /input/train.csv"
test_path=".. /input/test.csv"
train,test,trainX,trainY,testX,nb_classes=load_dataset(train_path,test_path)
 X_train, X_test, y_train, y_test = train_test_split(trainX,trainY,test_size=0.1, random_state=21)
 inp_shape=(28,28,1)
 model=createModel(inp_shape,nb_classes)
 imgaug=False
batch_size=128
nb_epochs=30
learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001)
 if imgaug==True:
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(trainX)
 out = model.fit_generator(datagen.flow(trainX,trainY, batch_size=batch_size),
epochs = nb_epochs, validation_data =(X_test,y_test),
verbose = 2, steps_per_epoch=batch_size // batch_size
, callbacks=[learning_rate_reduction])
 else:
out=model.fit(trainX, trainY, batch_size=batch_size, nb_epoch=nb_epochs, verbose=1,
validation_data=(X_test, y_test))
yPred = model.predict_classes(testX)
 print("Predictions : ",yPred)
 submission(yPred)
 result_visualization(out)
sub_preds = clf.predict(test_df[feature_cols])
 sub_df['target'] = sub_preds
sub_df.to_csv('submission_lgbm_optuna.csv', index=False)
 sub_df.head()
sub_lgbm_optuna = sub_preds<init_hyperparams>