Kaggle competition: Digit Recognizer
cat_cols = [feature for feature in train.columns if 'cat' in feature]
def label_encoder(df):
for feature in cat_cols:
le = LabelEncoder()
le.fit(df[feature])
 df[feature] = le.transform(df[feature])
 return df<categorify>
trainRaw = pd.read_csv('.. /input/train.csv')
 testRaw = pd.read_csv('.. /input/test.csv' )
X_train = label_encoder(X_train)
 X_test = label_encoder(X_test )<choose_model_class>
train = trainRaw.copy()
test_imagesKaggle = testRaw.copy()
train_labelsKaggle = trainRaw['label']
print("train with Labels ", train.shape)
 print("train_labelsKaggle ", train_labelsKaggle.shape)
 print("_"*50)
 train.drop(['label'],axis=1, inplace=True)
 train_imagesKaggle = train
print("train_imagesKaggle without Labels ", train_imagesKaggle.shape)
 print("_"*50)
 print("test_imagesKaggle ", test_imagesKaggle.shape )
split = KFold(n_splits=5, random_state=2 )<train_model>
train4Display = np.array(train_imagesKaggle ).reshape(42000,28,28)
 test4Display = np.array(test_imagesKaggle ).reshape(28000,28,28)
 z = 4056
print("train image")
 print(train_labelsKaggle[z])
 digit = train4Display[z]
plt.imshow(digit, cmap=plt.cm.binary)
 plt.show()
print("test image")
 digit = test4Display[z]
plt.imshow(digit, cmap=plt.cm.binary)
 plt.show()
def objective(trial, X, y, name='xgb'):
params = {'max_depth':trial.suggest_int('max_depth', 5, 50),
'n_estimators':200000,
'subsample': trial.suggest_uniform('subsample', 0.2, 1.0),
'colsample_bytree':trial.suggest_uniform('colsample_bytree', 0.2, 1.0),
'learning_rate':trial.suggest_uniform('learning_rate', 0.007, 0.02),
'reg_lambda':trial.suggest_uniform('reg_lambda', 0.01, 50),
'reg_alpha':trial.suggest_uniform('reg_alpha', 0.01, 50),
'min_child_samples':trial.suggest_int('min_child_samples', 5, 100),
'num_leaves':trial.suggest_int('num_leaves', 10, 200),
'n_jobs' : -1,
'metric':'rmse',
'max_bin':trial.suggest_int('max_bin', 300, 1000),
'cat_smooth':trial.suggest_int('cat_smooth', 5, 100),
'cat_l2':trial.suggest_loguniform('cat_l2', 1e-3, 100)}
model = LGBMRegressor(**params)
 X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=0)
 model.fit(X_train, y_train, eval_set=[(X_val, y_val)],
eval_metric=['rmse'],
early_stopping_rounds=250,
categorical_feature=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
verbose=0)
 train_score = np.round(np.sqrt(mean_squared_error(y_train, model.predict(X_train))), 5)
 test_score = np.round(np.sqrt(mean_squared_error(y_val, model.predict(X_val))), 5)
 print(f'TRAIN RMSE : {train_score} || TEST RMSE : {test_score}')
 return test_score<train_model>
train_imagesKaggle = train4Display.reshape(42000,28,28,1)
 test_imagesKaggle = test4Display.reshape(28000,28,28,1)
 train_imagesKaggle = train_imagesKaggle.astype('float32')/ 255
test_imagesKaggle = test_imagesKaggle.astype('float32')/ 255
print("train_imagesKaggle ",train_imagesKaggle.shape)
 print("test_imagesKaggle ", test_imagesKaggle.shape)
 print("_"*50)
 train_labelsKaggle = to_categorical(train_labelsKaggle)
 print("train_labelsKaggle ",train_labelsKaggle.shape )
optimize = partial(objective, X=X_train, y=y_train)
 study_lgbm = optuna.create_study(direction='minimize')
<init_hyperparams>
( train_imagesRaw, train_labelsRaw),(test_imagesRaw, test_labelsRaw)= mnist.load_data()
lgbm_params = {'max_depth': 16,
'subsample': 0.8032697250789377,
'colsample_bytree': 0.21067140508531404,
'learning_rate': 0.009867383057779643,
'reg_lambda': 10.987474846877767,
'reg_alpha': 17.335285595031994,
'min_child_samples': 31,
'num_leaves': 66,
'max_bin': 522,
'cat_smooth': 81,
'cat_l2': 0.029690334194270022,
'metric': 'rmse',
'n_jobs': -1,
'n_estimators': 20000}<train_model>
train_imagesKeras = train_imagesRaw.copy()
train_labelsKeras = train_labelsRaw.copy()
test_imagesKeras = test_imagesRaw.copy()
test_labelsKeras = test_labelsRaw.copy()
train_imagesKeras = train_imagesKeras.reshape(60000,28,28,1)
 test_imagesKeras = test_imagesKeras.reshape(10000,28,28,1)
 print("train_imagesKeras ",train_imagesKeras.shape)
 print("train_labelsKeras ",train_labelsKeras.shape)
 print("test_imagesKeras ", test_imagesKeras.shape)
 print("test_labelsKeras ", test_labelsKeras.shape)
 train_imagesKeras = train_imagesKeras.astype('float32')/ 255
test_imagesKeras = test_imagesKeras.astype('float32')/ 255
print("_"*50)
 train_labelsKeras = to_categorical(train_labelsKeras)
 test_labelsKeras = to_categorical(test_labelsKeras)
 print("train_labelsKeras ",train_labelsKeras.shape)
 print("test_labelsKeras ", test_labelsKeras.shape )
preds_list_base = []
preds_list_final_iteration = []
preds_list_all = []
for train_idx, val_idx in split.split(X_train):
X_tr = X_train.iloc[train_idx]
X_val = X_train.iloc[val_idx]
y_tr = y_train.iloc[train_idx]
y_val = y_train.iloc[val_idx]
Model = LGBMRegressor(**lgbm_params ).fit(X_tr, y_tr, eval_set=[(X_val, y_val)],
eval_metric=['rmse'],
early_stopping_rounds=250,
categorical_feature=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
verbose=0)
 preds_list_base.append(Model.predict(X_test))
preds_list_all.append(Model.predict(X_test))
print(f'RMSE for Base model is {np.sqrt(mean_squared_error(y_val, Model.predict(X_val)))}')
 first_rmse = np.sqrt(mean_squared_error(y_val, Model.predict(X_val)))
 params = lgbm_params.copy()
for i in range(1, 8):
if i >2:
params['reg_lambda'] *= 0.9
params['reg_alpha'] *= 0.9
params['num_leaves'] += 40
params['learning_rate'] = 0.003
Model = LGBMRegressor(**params ).fit(X_tr, y_tr, eval_set=[(X_val, y_val)],
eval_metric=['rmse'],
early_stopping_rounds=200,
categorical_feature=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
verbose=0,
init_model=Model)
 preds_list_all.append(Model.predict(X_test))
print(f'RMSE for Incremental trial {i} model is {np.sqrt(mean_squared_error(y_val, Model.predict(X_val)))}')
 last_rmse = np.sqrt(mean_squared_error(y_val, Model.predict(X_val)))
 print('',end='
')
 print(f'Improvement of : {first_rmse - last_rmse}')
 print('-' * 100)
 preds_list_final_iteration.append(Model.predict(X_test))<prepare_x_and_y>
train_images = np.concatenate(( train_imagesKeras,train_imagesKaggle), axis=0)
 print("new Concatenated train_images ", train_images.shape)
 print("_"*50)
 train_labels = np.concatenate(( train_labelsKeras,train_labelsKaggle), axis=0)
 print("new Concatenated train_labels ", train_labels.shape )
y_preds_base = np.array(preds_list_base ).mean(axis=0)
 y_preds_base<prepare_output>
model = models.Sequential()
model.add(layers.Conv2D(32,(3, 3), activation='relu', input_shape=(28, 28, 1)))
 model.add(layers.Dropout(0.5))
model.add(layers.MaxPooling2D(( 2, 2)))
 model.add(layers.Conv2D(64,(3, 3), activation='relu'))
model.add(layers.Dropout(0.5))
model.add(layers.MaxPooling2D(( 2, 2)))
 model.add(layers.Conv2D(64,(3, 3), activation='relu'))
model.add(layers.Dropout(0.5))
model.add(layers.Flatten())
 model.add(layers.Dense(64, activation='relu'))
model.add(layers.Dense(10, activation='softmax'))
model.compile(optimizer='rmsprop',loss='categorical_crossentropy',metrics=['accuracy'])
 print(model.summary() )
y_preds_all = np.array(preds_list_all ).mean(axis=0)
 y_preds_all<prepare_output>
num_epochs = 30
BatchSize = 2048
model.fit(train_images, train_labels, epochs=num_epochs, batch_size=BatchSize)
 test_loss, test_acc = model.evaluate(test_imagesKeras, test_labelsKeras)
 print("_"*80)
 print("Accuracy on test ", test_acc )
y_preds_final_iteration = np.array(preds_list_final_iteration ).mean(axis=0)
 y_preds_final_iteration<create_dataframe>
def build_model() :
model = models.Sequential()
model.add(layers.Conv2D(32,(3, 3), activation='relu', input_shape=(28, 28, 1)))
 model.add(layers.Dropout(0.5))
model.add(layers.MaxPooling2D(( 2, 2)))
 model.add(layers.Conv2D(64,(3, 3), activation='relu'))
model.add(layers.Dropout(0.5))
model.add(layers.MaxPooling2D(( 2, 2)))
 model.add(layers.Conv2D(64,(3, 3), activation='relu'))
model.add(layers.Dropout(0.5))
model.add(layers.Flatten())
 model.add(layers.Dense(64, activation='relu'))
model.add(layers.Dense(10, activation='softmax'))
model.compile(optimizer='rmsprop',loss='categorical_crossentropy',metrics=['accuracy'])
 return model
submission = pd.DataFrame({'id':test.id,
'target':y_preds_final_iteration} )<save_to_csv>
train_data = train_images
train_targets = train_labels
k = 4
num_val_samples = len(train_data)// k
all_mae_histories = []
for i in range(k):
print('processing fold
val_data = train_data[i * num_val_samples:(i + 1)* num_val_samples]
val_targets = train_targets[i * num_val_samples:(i + 1)* num_val_samples]
partial_train_data = np.concatenate(
 [train_data[:i * num_val_samples],
train_data[(i + 1)* num_val_samples:]],
axis=0)
 partial_train_targets = np.concatenate(
 [train_targets[:i * num_val_samples],
train_targets[(i + 1)* num_val_samples:]],
axis=0)
 model = build_model()
history = model.fit(partial_train_data, partial_train_targets,
validation_data=(val_data, val_targets),
epochs=num_epochs, batch_size=BatchSize, verbose=0)
 mae_history = history.history['acc']
all_mae_histories.append(mae_history)
 print("Done CV k-fold" )
submission.to_csv('submission.csv', index=False )<save_to_csv>
train_imagesFin = np.concatenate(( train_images,test_imagesKeras), axis=0)
 print("train_imagesFin ", train_imagesFin.shape)
 print("_"*50)
 train_labelsFin = np.concatenate(( train_labels,test_labelsKeras), axis=0)
 print("train_labelsFin ", train_labelsFin.shape )
submission.to_csv('submission.csv', index=False )<load_from_csv>
model = build_model()
model.fit(train_imagesFin, train_labelsFin, epochs=num_epochs, batch_size=BatchSize )
pd.read_csv('submission.csv' )<train_model>
RawPred = model.predict(test_imagesKaggle)
 pred = []
numTest = RawPred.shape[0]
for i in range(numTest):
pred.append(np.argmax(RawPred[i]))
predictions = np.array(pred )
<find_best_model_class><EOS> 
sample_submission = pd.read_csv('.. /input/sample_submission.csv')
 result=pd.DataFrame({'ImageId':sample_submission.ImageId, 'Label':predictions})
 result.to_csv("submission.csv",index=False)
 print(result )