Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<categorify>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 Y_train = train["label"]
X_train = train.drop(labels = ["label"],axis = 1)
 X_train = X_train / 255.0
X_test = test / 255.0
X_train = X_train.values.reshape(-1,28,28,1)
 X_test = X_test.values.reshape(-1,28,28,1)
 Y_train = to_categorical(Y_train, num_classes = 10)
 annealer = LearningRateScheduler(lambda x: 1e-3 * 0.95 ** x, verbose=0)
 styles=[':','-.','--','-',':','-.','--','-',':','-.','--','-']
def lencode(df):
for i in df.columns:
for k in cat_features.columns:
if i ==k:
df[i]= le.fit_transform(df[i])
<prepare_x_and_y>
nets = 3
model = [0] *nets
for j in range(3):
model[j] = Sequential()
model[j].add(Conv2D(24,kernel_size=5,padding='same',activation='relu',
input_shape=(28,28,1)))
 model[j].add(MaxPool2D())
 if j>0:
model[j].add(Conv2D(48,kernel_size=5,padding='same',activation='relu'))
model[j].add(MaxPool2D())
 if j>1:
model[j].add(Conv2D(64,kernel_size=5,padding='same',activation='relu'))
model[j].add(MaxPool2D(padding='same'))
model[j].add(Flatten())
 model[j].add(Dense(256, activation='relu'))
model[j].add(Dense(10, activation='softmax'))
model[j].compile(optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"] )
X_train = df[:300000]
Xf_test = df[300000:]
y = train_df['target']<split>
X_train2, X_val2, Y_train2, Y_val2 = train_test_split(X_train, Y_train, test_size = 0.333)
 history = [0] * nets
names = ["(C-P)x1","(C-P)x2","(C-P)x3"]
epochs = 20
for j in range(nets):
history[j] = model[j].fit(X_train2,Y_train2, batch_size=80, epochs = epochs,
validation_data =(X_val2,Y_val2), callbacks=[annealer], verbose=0)
 print("CNN {0}: Epochs={1:d}, Train accuracy={2:.5f}, Validation accuracy={3:.5f}".format(
 names[j],epochs,max(history[j].history['acc']),max(history[j].history['val_acc'])) )
X_train, X_test, y_train, y_test = train_test_split(X_train, y, test_size=0.1, random_state=42 )<choose_model_class>
nets = 6
model = [0] *nets
for j in range(6):
model[j] = Sequential()
model[j].add(Conv2D(j*8+8,kernel_size=5,activation='relu',input_shape=(28,28,1)))
 model[j].add(MaxPool2D())
 model[j].add(Conv2D(j*16+16,kernel_size=5,activation='relu'))
model[j].add(MaxPool2D())
 model[j].add(Flatten())
 model[j].add(Dense(256, activation='relu'))
model[j].add(Dense(10, activation='softmax'))
model[j].compile(optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"] )
linear = LinearRegression()
random = RandomForestRegressor(n_estimators=100,random_state=42,n_jobs=-1,max_features=4)
<choose_model_class>
X_train2, X_val2, Y_train2, Y_val2 = train_test_split(X_train, Y_train, test_size = 0.333)
 history = [0] * nets
names = ["8 maps","16 maps","24 maps","32 maps","48 maps","64 maps"]
epochs = 20
for j in range(nets):
history[j] = model[j].fit(X_train2,Y_train2, batch_size=80, epochs = epochs,
validation_data =(X_val2,Y_val2), callbacks=[annealer], verbose=0)
 print("CNN {0}: Epochs={1:d}, Train accuracy={2:.5f}, Validation accuracy={3:.5f}".format(
 names[j],epochs,max(history[j].history['acc']),max(history[j].history['val_acc'])) )
xgbr = xgb.XGBRegressor(tree_method='gpu_hist')
<find_best_params>
nets = 8
model = [0] *nets
for j in range(8):
model[j] = Sequential()
model[j].add(Conv2D(32,kernel_size=5,activation='relu',input_shape=(28,28,1)))
 model[j].add(MaxPool2D())
 model[j].add(Conv2D(64,kernel_size=5,activation='relu'))
model[j].add(MaxPool2D())
 model[j].add(Flatten())
 if j>0:
model[j].add(Dense(2**(j+4), activation='relu'))
model[j].add(Dense(10, activation='softmax'))
model[j].compile(optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"] )
def objective(trial):
params = {
'random_state': 0,
'n_estimators': trial.suggest_categorical('n_estimators', [10000]),
'max_depth': trial.suggest_int('max_depth', 3, 8),
'learning_rate': trial.suggest_float('learning_rate', 0.001, 1.0),
'reg_lambda': trial.suggest_float('reg_lambda', 0.0, 10),
'reg_alpha': trial.suggest_float('reg_alpha', 0.0, 10),
'gamma': trial.suggest_float('gamma', 0.0, 10),
'subsample': trial.suggest_categorical('subsample', [0.8, 0.9, 1.0]),
'colsample_bytree': trial.suggest_categorical('colsample_bytree', [0.1, 0.2, 0.3, 0.4, 0.5]),
'tree_method':'gpu_hist'
}
model = xgb.XGBRegressor(**params)
 model.fit(X_train, y_train, eval_set=[(X_test,y_test)], early_stopping_rounds=1000, verbose=0)
 y_pred = model.predict(X_test)
 rmse = mean_squared_error(y_test, y_pred, squared=False)
 return rmse<find_best_params>
X_train2, X_val2, Y_train2, Y_val2 = train_test_split(X_train, Y_train, test_size = 0.333)
 history = [0] * nets
names = ["0N","32N","64N","128N","256N","512N","1024N","2048N"]
epochs = 20
for j in range(nets):
history[j] = model[j].fit(X_train2,Y_train2, batch_size=80, epochs = epochs,
validation_data =(X_val2,Y_val2), callbacks=[annealer], verbose=0)
 print("CNN {0}: Epochs={1:d}, Train accuracy={2:.5f}, Validation accuracy={3:.5f}".format(
 names[j],epochs,max(history[j].history['acc']),max(history[j].history['val_acc'])) )
%%time
study = optuna.create_study(direction='minimize',sampler=optuna.samplers.TPESampler(seed=0))
study.optimize(objective, n_trials=100)
 print('Number of finished trials:', len(study.trials))
print('Best parameters:', study.best_trial.params)
 print('Best RMSE:', study.best_trial.value )<train_model>
nets = 8
model = [0] *nets
for j in range(8):
model[j] = Sequential()
model[j].add(Conv2D(32,kernel_size=5,activation='relu',input_shape=(28,28,1)))
 model[j].add(MaxPool2D())
 model[j].add(Dropout(j*0.1))
model[j].add(Conv2D(64,kernel_size=5,activation='relu'))
model[j].add(MaxPool2D())
 model[j].add(Dropout(j*0.1))
model[j].add(Flatten())
 model[j].add(Dense(128, activation='relu'))
model[j].add(Dropout(j*0.1))
model[j].add(Dense(10, activation='softmax'))
model[j].compile(optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"] )
params = study.best_params
params['random_state'] = 0
params['n_estimators'] = 10000
params['tree_method'] = 'gpu_hist'
model3 = xgb.XGBRegressor(**params)
 model3.fit(X_train,y_train,eval_set=[(X_test, y_test)],early_stopping_rounds=1000,verbose=2)
<train_model>
X_train2, X_val2, Y_train2, Y_val2 = train_test_split(X_train, Y_train, test_size = 0.333)
 history = [0] * nets
names = ["D=0","D=0.1","D=0.2","D=0.3","D=0.4","D=0.5","D=0.6","D=0.7"]
epochs = 30
for j in range(nets):
history[j] = model[j].fit(X_train2,Y_train2, batch_size=80, epochs = epochs,
validation_data =(X_val2,Y_val2), callbacks=[annealer], verbose=0)
 print("CNN {0}: Epochs={1:d}, Train accuracy={2:.5f}, Validation accuracy={3:.5f}".format(
 names[j],epochs,max(history[j].history['acc']),max(history[j].history['val_acc'])) )
model = make_pipeline(linear)
 model.fit(X_train,y_train)
 model2 = make_pipeline(xgbr)
 model2.fit(X_train,y_train )<choose_model_class>
nets = 5
model = [0] *nets
j=0
model[j] = Sequential()
model[j].add(Conv2D(32,kernel_size=5,activation='relu',input_shape=(28,28,1)))
 model[j].add(MaxPool2D())
 model[j].add(Dropout(0.4))
model[j].add(Conv2D(64,kernel_size=5,activation='relu'))
model[j].add(MaxPool2D())
 model[j].add(Dropout(0.4))
model[j].add(Flatten())
 model[j].add(Dense(128, activation='relu'))
model[j].add(Dropout(0.4))
model[j].add(Dense(10, activation='softmax'))
model[j].compile(optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"])
 j=1
model[j] = Sequential()
model[j].add(Conv2D(32,kernel_size=3,activation='relu',input_shape=(28,28,1)))
 model[j].add(Conv2D(32,kernel_size=3,activation='relu'))
model[j].add(MaxPool2D())
 model[j].add(Dropout(0.4))
model[j].add(Conv2D(64,kernel_size=3,activation='relu'))
model[j].add(Conv2D(64,kernel_size=3,activation='relu'))
model[j].add(MaxPool2D())
 model[j].add(Dropout(0.4))
model[j].add(Flatten())
 model[j].add(Dense(128, activation='relu'))
model[j].add(Dropout(0.4))
model[j].add(Dense(10, activation='softmax'))
model[j].compile(optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"])
 j=2
model[j] = Sequential()
model[j].add(Conv2D(32,kernel_size=5,activation='relu',input_shape=(28,28,1)))
 model[j].add(Conv2D(32,kernel_size=5,strides=2,padding='same',activation='relu'))
model[j].add(Dropout(0.4))
model[j].add(Conv2D(64,kernel_size=5,activation='relu'))
model[j].add(Conv2D(64,kernel_size=5,strides=2,padding='same',activation='relu'))
model[j].add(Dropout(0.4))
model[j].add(Flatten())
 model[j].add(Dense(128, activation='relu'))
model[j].add(Dropout(0.4))
model[j].add(Dense(10, activation='softmax'))
model[j].compile(optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"])
 j=3
model[j] = Sequential()
model[j].add(Conv2D(32,kernel_size=3,activation='relu',input_shape=(28,28,1)))
 model[j].add(BatchNormalization())
 model[j].add(Conv2D(32,kernel_size=3,activation='relu'))
model[j].add(BatchNormalization())
 model[j].add(Conv2D(32,kernel_size=5,strides=2,padding='same',activation='relu'))
model[j].add(BatchNormalization())
 model[j].add(Dropout(0.4))
model[j].add(Conv2D(64,kernel_size=3,activation='relu'))
model[j].add(BatchNormalization())
 model[j].add(Conv2D(64,kernel_size=3,activation='relu'))
model[j].add(BatchNormalization())
 model[j].add(Conv2D(64,kernel_size=5,strides=2,padding='same',activation='relu'))
model[j].add(BatchNormalization())
 model[j].add(Dropout(0.4))
model[j].add(Flatten())
 model[j].add(Dense(128, activation='relu'))
model[j].add(Dropout(0.4))
model[j].add(Dense(10, activation='softmax'))
model[j].compile(optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"] )
models ={'linear':model}
models['xgbr'] = model2
models['optuna_xbg'] =model3<compute_test_metric>
j=4
model[j] = Sequential()
model[j].add(Conv2D(32,kernel_size=3,activation='relu',input_shape=(28,28,1)))
 model[j].add(BatchNormalization())
 model[j].add(Conv2D(32,kernel_size=3,activation='relu'))
model[j].add(BatchNormalization())
 model[j].add(Conv2D(32,kernel_size=5,strides=2,padding='same',activation='relu'))
model[j].add(BatchNormalization())
 model[j].add(Dropout(0.4))
model[j].add(Conv2D(64,kernel_size=3,activation='relu'))
model[j].add(BatchNormalization())
 model[j].add(Conv2D(64,kernel_size=3,activation='relu'))
model[j].add(BatchNormalization())
 model[j].add(Conv2D(64,kernel_size=5,strides=2,padding='same',activation='relu'))
model[j].add(BatchNormalization())
 model[j].add(Dropout(0.4))
model[j].add(Flatten())
 model[j].add(Dense(128, activation='relu'))
model[j].add(BatchNormalization())
 model[j].add(Dropout(0.4))
model[j].add(Dense(10, activation='softmax'))
model[j].compile(optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"] )
def rmse(y_test,y_pred):
rmse= np.sqrt(mean_squared_error(y_test,y_pred))
return rmse 
<find_best_model_class>
X_train2, X_val2, Y_train2, Y_val2 = train_test_split(X_train, Y_train, test_size = 0.2)
 history = [0] * nets
names = ["basic","32C3-32C3","32C5S2","both+BN","both+BN+DA"]
epochs = 35
for j in range(nets-1):
history[j] = model[j].fit(X_train2,Y_train2, batch_size=64, epochs = epochs,
validation_data =(X_val2,Y_val2), callbacks=[annealer], verbose=0)
 print("CNN {0}: Epochs={1:d}, Train accuracy={2:.5f}, Validation accuracy={3:.5f}".format(
 names[j],epochs,max(history[j].history['acc']),max(history[j].history['val_acc'])))
 datagen = ImageDataGenerator(
 rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1)
 j = nets-1
history[j] = model[j].fit_generator(datagen.flow(X_train2,Y_train2, batch_size=64),
epochs = epochs, steps_per_epoch = X_train2.shape[0]//64,
validation_data =(X_val2,Y_val2), callbacks=[annealer], verbose=0)
 print("CNN {0}: Epochs={1:d}, Train accuracy={2:.5f}, Validation accuracy={3:.5f}".format(
 names[j],epochs,max(history[j].history['acc']),max(history[j].history['val_acc'])) )
def cross_val_rmse(model):
model =clone(model)
 five_fold = KFold(n_splits=5)
 rmse_val =[]
for tr_ind,val_ind in five_fold.split(X_train):
model.fit(X_train.iloc[tr_ind,:],y.iloc[tr_ind])
 rmse_val.append(rmse(y.iloc[val_ind],model.predict(X_train.iloc[val_ind,:])))
 return np.mean(rmse_val )<compute_test_metric>
annealer = LearningRateScheduler(lambda x: 1e-3 * 0.95 **(x+epochs))
model[4].fit_generator(datagen.flow(X_train,Y_train, batch_size=64), epochs = 25,
steps_per_epoch = X_train.shape[0]//64, callbacks=[annealer], verbose=0)
 results = model[4].predict(X_test)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results,name="Label")
 submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("MNIST-CNN.csv",index=False )