Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<create_dataframe>
final_models = {"keras1":"", "keras2":""}
<feature_engineering>
train_data = pd.read_csv(".. /input/train.csv")
 x_train_all = np.array(train_data.drop(["label"], axis=1))
print(x_train_all[0][x_train_all[0] > 0])
 x_train_all = x_train_all/255
x_train_all = stretch_image(x_train_all)
 y_train_all = np.array(train_data["label"])
 idx_all = range(x_train_all.shape[0])
 if 1==2:
bad_data_idx = [28290, 16301,14101,15065, 6389,7764,28611,20954,2316, 37056, 37887, 36569, 40257]
plot_bad_data(x_train_all[bad_data_idx], y_train_all[bad_data_idx])
 x_train_all = np.delete(x_train_all, bad_data_idx, axis=0)
 y_train_all = np.delete(y_train_all, bad_data_idx)
 idx_all = np.delete(idx_all, bad_data_idx)
 y_train_all = keras.utils.to_categorical(y_train_all, num_classes=10)
 x_train, x_valid, y_train, y_valid, idx_train, idx_valid = skm.train_test_split(x_train_all, y_train_all,idx_all, test_size=0.2)
 test_data = pd.read_csv(".. /input/test.csv")
 x_test = np.array(test_data)
 x_test = x_test/255
x_test = stretch_image(x_test)
<define_search_space>
datagen = keras.preprocessing.image.ImageDataGenerator(featurewise_center=False,
featurewise_std_normalization=False,
rotation_range=10,
width_shift_range=0.1,
height_shift_range=0.1,
zoom_range=0.1,
horizontal_flip=False, shear_range=0.2)
 datagen.fit(x_train.reshape(-1,28,28,1))
train_data.loc[[128,699,715,75]]<drop_column>
learning_rate_reduction = keras.callbacks.ReduceLROnPlateau(monitor='acc',
patience=1,
verbose=1,
factor=0.5,
min_lr=0.000001 )
train_data = train_data.drop('PassengerId',axis=1)
 dt = train_data 
<filter>
def tensorflow_keras_model(x_train, y_train, x_valid, y_valid, num_classes,\
num_epochs, learning_rate):
keras_model = keras.models.Sequential()
keras_model.add(keras.layers.Conv2D(filters=32,kernel_size=(6,6),strides=(1,1), \
padding="same",input_shape=(28,28,1)))
 keras_model.add(keras.layers.BatchNormalization())
 keras_model.add(keras.layers.Activation('relu'))
keras_model.add(keras.layers.MaxPool2D(strides=(2,2), padding="same"))
keras_model.add(keras.layers.Dropout(rate=0.05))
keras_model.add(keras.layers.Conv2D(filters=64,kernel_size=(6,6),strides=(1,1), \
padding="same"))
keras_model.add(keras.layers.BatchNormalization())
 keras_model.add(keras.layers.Activation('relu'))
keras_model.add(keras.layers.MaxPool2D(strides=(2,2), padding="same"))
keras_model.add(keras.layers.Dropout(rate=0.05))
keras_model.add(keras.layers.Flatten())
 keras_model.add(keras.layers.Dense(units=1024))
keras_model.add(keras.layers.BatchNormalization())
 keras_model.add(keras.layers.Activation('relu'))
keras_model.add(keras.layers.Dropout(rate=0.05))
keras_model.add(keras.layers.Dense(units=num_classes))
keras_model.add(keras.layers.BatchNormalization())
 keras_model.add(keras.layers.Activation('softmax'))
opt = keras.optimizers.RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0)
 keras_model.compile(optimizer=opt,loss="categorical_crossentropy", metrics=["accuracy"])
 keras_model.fit(x_train.reshape(-1,28,28,1), y_train, epochs=num_epochs, callbacks=[learning_rate_reduction])
 final_models["keras1"] = keras_model
keras_model.save("Model_MNIST_Keras_Lenet.h5")
 cur_y_pred = keras_model.predict(x_valid.reshape(-1,28,28,1))
y_valid_argmax = np.argmax(y_valid, 1)
 y_pred_argmax = np.argmax(cur_y_pred, 1)
 y_correct = np.equal(y_valid_argmax, y_pred_argmax)
 acc = y_correct.sum() /y_pred_argmax.shape[0]
return cur_y_pred, acc, y_valid_argmax,y_pred_argmax
<drop_column>
def tensorflow_keras_model_2(x_train, y_train, x_valid, y_valid, num_classes,\
num_epochs, learning_rate):
keras_model = keras.models.Sequential()
keras_model.add(keras.layers.Conv2D(filters=32,kernel_size=(3,3),strides=(1,1), \
padding="same",activation='relu', input_shape=(28,28,1)))
 keras_model.add(keras.layers.Conv2D(filters=32,kernel_size=(3,3),strides=(1,1), \
padding="same"))
keras_model.add(keras.layers.BatchNormalization())
 keras_model.add(keras.layers.Activation('relu'))
keras_model.add(keras.layers.Conv2D(filters=32,kernel_size=(5,5),strides=(2,2), \
padding="same"))
keras_model.add(keras.layers.Dropout(rate=0.05))
keras_model.add(keras.layers.Conv2D(filters=64,kernel_size=(3,3),strides=(1,1), \
padding="same",activation='relu',input_shape=(28,28,1)))
 keras_model.add(keras.layers.Conv2D(filters=64,kernel_size=(3,3),strides=(1,1), \
padding="same"))
keras_model.add(keras.layers.BatchNormalization())
 keras_model.add(keras.layers.Activation('relu'))
keras_model.add(keras.layers.Conv2D(filters=64,kernel_size=(5,5),strides=(2,2), \
padding="same"))
keras_model.add(keras.layers.Dropout(rate=0.050))
keras_model.add(keras.layers.Conv2D(filters=128,kernel_size=(3,3),strides=(1,1), \
padding="same",activation='relu'))
keras_model.add(keras.layers.Conv2D(filters=128,kernel_size=(3,3),strides=(1,1), \
padding="same"))
keras_model.add(keras.layers.BatchNormalization())
 keras_model.add(keras.layers.Activation('relu'))
keras_model.add(keras.layers.Conv2D(filters=128,kernel_size=(5,5),strides=(2,2), \
padding="same"))
keras_model.add(keras.layers.Dropout(rate=0.075))
keras_model.add(keras.layers.Flatten())
 keras_model.add(keras.layers.Dense(units=1024))
keras_model.add(keras.layers.BatchNormalization())
 keras_model.add(keras.layers.Activation('relu'))
keras_model.add(keras.layers.Dropout(rate=0.100))
keras_model.add(keras.layers.Dense(units=128))
keras_model.add(keras.layers.BatchNormalization())
 keras_model.add(keras.layers.Activation('relu'))
keras_model.add(keras.layers.Dropout(rate=0.100))
keras_model.add(keras.layers.Dense(units=num_classes))
keras_model.add(keras.layers.BatchNormalization())
 keras_model.add(keras.layers.Activation('softmax'))
opt = keras.optimizers.RMSprop(lr=0.0001, rho=0.9, epsilon=1e-08, decay=0.0)
 keras_model.compile(optimizer=opt,loss="categorical_crossentropy", metrics=["accuracy"])
 keras_model.fit_generator(datagen.flow(x_train.reshape(-1,28,28,1), y_train, batch_size=32),steps_per_epoch=len(x_train)/ 32, \
epochs=num_epochs, callbacks=[learning_rate_reduction])
 final_models["keras2"] = keras_model
keras_model.save("Model_MNIST_Keras_Resnet.h5")
 cur_y_pred = keras_model.predict(x_valid.reshape(-1,28,28,1))
y_valid_argmax = np.argmax(y_valid, 1)
 y_pred_argmax = np.argmax(cur_y_pred, 1)
 y_correct = np.equal(y_valid_argmax, y_pred_argmax)
 acc = y_correct.sum() /y_pred_argmax.shape[0]
return cur_y_pred, acc, y_valid_argmax,y_pred_argmax , keras_model
train_data = train_data.drop('Cabin',axis=1)
 test_data = test_data.drop('Cabin',axis=1 )<split>
model2_list = []
num_model = 12
def exec_tensorflow_keras_model_2() :
num_epochs = 30
learning_rate=0.00001
num_rows, num_features, num_classes = x_train.shape[0], x_train.shape[1], 10
for imodel in range(num_model):
final_pred_base_model, acc,y_valid_argmax,y_pred_argmax, keras_model = \
tensorflow_keras_model_2(x_train, y_train, x_valid, y_valid, num_classes, num_epochs, learning_rate)
 model2_list.append(keras_model)
 print("Num Epoch:", num_epochs, " Accuracy:", acc)
 plot_y_true_vs_y_pred(x_valid, y_valid_argmax.reshape(len(y_valid)) , y_pred_argmax.reshape(len(y_valid)) , idx_valid )
train, test = train_test_split(train_data, test_size=0.33, random_state=42)
 features_train = train.drop(['Survived'],axis=1)
 target_train = train['Survived']
features_test = test.drop(['Survived'],axis=1)
 target_test = test['Survived']<train_model>
def tensorflow_keras_model_3(x_train, y_train, x_valid, y_valid, num_classes,\
num_epochs, learning_rate):
keras_model = keras.models.Sequential()
keras_model.add(keras.layers.Conv2D(filters=6,kernel_size=(6,6),strides=(1,1), \
padding="same",input_shape=(28,28,1)))
 keras_model.add(keras.layers.BatchNormalization())
 keras_model.add(keras.layers.Activation('relu'))
keras_model.add(keras.layers.MaxPool2D(strides=(2,2), padding="same"))
keras_model.add(keras.layers.Dropout(rate=0.05))
keras_model.add(keras.layers.Conv2D(filters=16,kernel_size=(6,6),strides=(1,1), \
padding="same"))
keras_model.add(keras.layers.BatchNormalization())
 keras_model.add(keras.layers.Activation('relu'))
keras_model.add(keras.layers.MaxPool2D(strides=(2,2), padding="same"))
keras_model.add(keras.layers.Dropout(rate=0.05))
keras_model.add(keras.layers.Conv2D(filters=120,kernel_size=(6,6),strides=(1,1), \
padding="same"))
keras_model.add(keras.layers.BatchNormalization())
 keras_model.add(keras.layers.Activation('relu'))
keras_model.add(keras.layers.MaxPool2D(strides=(2,2), padding="same"))
keras_model.add(keras.layers.Dropout(rate=0.05))
keras_model.add(keras.layers.Flatten())
 keras_model.add(keras.layers.Dense(units=120))
keras_model.add(keras.layers.BatchNormalization())
 keras_model.add(keras.layers.Activation('relu'))
keras_model.add(keras.layers.Dropout(rate=0.05))
keras_model.add(keras.layers.Dense(units=120))
keras_model.add(keras.layers.BatchNormalization())
 keras_model.add(keras.layers.Activation('relu'))
keras_model.add(keras.layers.Dropout(rate=0.05))
keras_model.add(keras.layers.Dense(units=num_classes))
keras_model.add(keras.layers.BatchNormalization())
 keras_model.add(keras.layers.Activation('softmax'))
opt = keras.optimizers.RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0)
 keras_model.compile(optimizer=opt,loss="categorical_crossentropy", metrics=["accuracy"])
 keras_model.fit(x_train.reshape(-1,28,28,1), y_train, epochs=num_epochs, callbacks=[learning_rate_reduction])
 final_models["keras1"] = keras_model
keras_model.save("Model_MNIST_Keras_Lenet.h5")
 cur_y_pred = keras_model.predict(x_valid.reshape(-1,28,28,1))
y_valid_argmax = np.argmax(y_valid, 1)
 y_pred_argmax = np.argmax(cur_y_pred, 1)
 y_correct = np.equal(y_valid_argmax, y_pred_argmax)
 acc = y_correct.sum() /y_pred_argmax.shape[0]
return cur_y_pred, acc, y_valid_argmax,y_pred_argmax
model = CatBoostClassifier(iterations=200,
depth=9,learning_rate=0.05,l2_leaf_reg=10,random_seed=42,
loss_function='Logloss',grow_policy='Lossguide',
max_leaves=39,
verbose=True,eval_metric='Accuracy',nan_mode='Min',cat_features=['Pclass','Sex','Q','S'])
 model.fit(features_train, target_train,text_features=['Ticket','Name'],
plot=True,eval_set=(features_test,target_test),verbose=False)
 print(accuracy_score(model.predict(features_train),target_train))
print(accuracy_score(model.predict(features_test),target_test))<drop_column>
model2 = final_models["keras2"]
x_valid = x_valid.reshape(-1,28,28,1)
 y_pred_valid2 = np.zeros(( x_valid.shape[0],10))
for imodel in range(num_model):
y_pred_valid2 = y_pred_valid2 + model2_list[imodel].predict(x_valid)
 y_pred_valid = y_pred_valid2
y_pred_valid_final = np.argmax(y_pred_valid, axis=1)
 y_correct = np.equal(y_pred_valid_final, np.argmax(y_valid, axis=1))
acc_final = y_correct.sum() /y_valid.shape[0]
print(acc_final )
test_data = test_data.drop('PassengerId',axis=1)
 dt = test_data
test_data<save_to_csv>
x_test = x_test.reshape(-1,28,28,1)
 y_pred_test2 = np.zeros(( x_test.shape[0],10))
for imodel in range(num_model):
y_pred_test2 = y_pred_test2 + model2_list[imodel].predict(x_test)
submission=pd.DataFrame(model.predict(test_data),columns=['Survived'])
 submission['PassengerId'] = hui
submission.Survived = submission.Survived.astype(int)
 filename = 'Titanic Predictions1.csv'
submission.to_csv(filename,index=False)
 print('Saved file: ' + filename )<load_from_csv>
y_pred_test = y_pred_test2
y_pred_test_final = np.argmax(y_pred_test, axis=1)
train = pd.read_csv("/kaggle/input/titanic/train.csv")
 test = pd.read_csv("/kaggle/input/titanic/test.csv")
 print(train.shape)
 train.head()<count_missing_values>
dictionary_data = {"ImageId":np.arange(1, x_test.shape[0]+1), "Label":y_pred_test_final}
df_final = pd.DataFrame(dictionary_data)
 df_final.to_csv("submission.csv", index=False )