Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<compute_train_metric>
warnings.filterwarnings('ignore' )
<compute_test_metric>
train_data = pd.read_csv('.. /input/digit-recognizer/train.csv')
 test_data = pd.read_csv('.. /input/digit-recognizer/test.csv')
 train_data.shape, test_data.shape
<prepare_output>
train_data = train_data / 255.0
test_data = test_data / 255.0
x_train = train_data.values.reshape(-1, 28, 28, 1)
 x_test = test_data.values.reshape(-1, 28, 28,1 )
sub = pd.DataFrame()
sub["PassengerId"] = ids
sub["Survived"] = knn_gd.predict(df_t)
 sub["Survived"] = sub["Survived"].astype(int)
 sub<save_to_csv>
y_train = to_categorical(train_labels, 10 )
sub.to_csv("submission.csv",index=None)
 print("End!" )<define_variables>
x_train, x_val, y_train, y_val = train_test_split(x_train, y_train, test_size = 0.1, random_state = 1)
 x_train.shape, y_train.shape, x_val.shape, y_val.shape
TRAIN_PATH = ".. /input/titanic/train.csv"
TEST_PATH = ".. /input/titanic/test.csv"
SAMPLE_SUBMISSION_PATH = ".. /input/titanic/gender_submission.csv"
SUBMISSION_PATH = "submission.csv"
ID = "PassengerId"
TARGET = "Survived"
SEED = 123
N_SPLITS = 5
GRID_SEARCH_CV_NUM = 5
SCORING = "roc_auc"<load_pretrained>
model = Sequential()
model.add(Conv2D(filters = 32, kernel_size =(5, 5), padding = 'SAME', activation = 'relu', input_shape =(28, 28, 1)))
 model.add(Conv2D(filters = 32, kernel_size =(5, 5), padding = 'SAME', activation = 'relu'))
model.add(MaxPool2D(2, 2))
model.add(Dropout(0.25))
model.add(Conv2D(filters = 64, kernel_size =(3, 3), padding = 'SAME', activation = 'relu'))
model.add(Conv2D(filters = 64, kernel_size =(3, 3), padding = 'SAME', activation = 'relu'))
model.add(MaxPool2D(pool_size =(2, 2), strides =(2, 2)))
 model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(256, activation = 'relu'))
model.add(Dropout(0.5))
model.add(Dense(10, activation = 'softmax'))
copyfile(src = ".. /input/titanic-preprocess/titanic_preprocess_py.py",
dst = ".. /working/titanic_preprocess_py.py")
 train,test = loadAndPreprocess(TRAIN_PATH,TEST_PATH)
 train.head()<prepare_x_and_y>
model.compile(optimizer=optimizer, loss = 'categorical_crossentropy', metrics=['accuracy'] )
y = train[TARGET]
X = train.drop([ID,TARGET],axis=1)
 X_test = test.drop([ID,TARGET],axis=1)
 stk = StratifiedKFold(n_splits=N_SPLITS,random_state=SEED,shuffle=True)
 for train_index,val_index in stk.split(X,y):
X_train,y_train = X.iloc[train_index],y.iloc[train_index]
X_val,y_val = X.iloc[val_index],y.iloc[val_index]
knn = KNeighborsClassifier()
knn.fit(X_train,y_train)
 pred_knn = knn.predict(X_val)
 print("knn recall:",round(recall_score(y_val,pred_knn),2))
print("knn precision:",round(precision_score(y_val,pred_knn),2))
print("knn f1_score:",round(f1_score(y_val,pred_knn),2))
print("knn rou_auc_score:",round(roc_auc_score(y_val,pred_knn),2))
params = {'algorithm': ['auto'],
'weights': ['uniform', 'distance'],
'leaf_size': range(1,30),
'n_neighbors': range(3,20)}
gs =GridSearchCV(knn, param_grid = params,
verbose=True,
cv=GRID_SEARCH_CV_NUM,
scoring = SCORING 
)
 gs.fit(X_train, y_train)
 print(gs.best_score_)
 print(gs.best_estimator_)
 print(gs.best_params_)
 pred_val = gs.predict(X_val)
 print("gs recall:",round(recall_score(y_val,pred_val),2))
print("gs precision:",round(precision_score(y_val,pred_val),2))
print("gs f1_score:",round(f1_score(y_val,pred_val),2))
print("gs rou_auc_score:",round(roc_auc_score(y_val,pred_val),2))<save_to_csv>
LR_reducer = ReduceLROnPlateau(monitor='val_acc', factor=0.5, patience=3, verbose=1, min_lr=0.00001 )
sub = pd.read_csv(SAMPLE_SUBMISSION_PATH)
 sub[TARGET] = gs.predict(X_test ).astype(int)
 sub.to_csv(SUBMISSION_PATH,index=False)
 sub.head(10 )<import_modules>
epochs = 30
batch_size = 86
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import classification_report, confusion_matrix, roc_auc_score<load_from_csv>
image_data = ImageDataGenerator(
 featurewise_center = False,
samplewise_center = False,
featurewise_std_normalization = False,
samplewise_std_normalization = False,
zca_whitening = False,
rotation_range = 10,
width_shift_range = 0.1,
height_shift_range = 0.1,
zoom_range = 0.1,
horizontal_flip = False,
vertical_flip= False
)
train = pd.read_csv(".. /input/titanic/train.csv", index_col="PassengerId")
 X_test = pd.read_csv(".. /input/titanic/test.csv", index_col="PassengerId")
 print(train.info())
 print(X_test.info() )<count_missing_values>
image_data.fit(x_train )
print("TrainData")
 print("---------------")
 print(train.isna().sum())
 print("TestData")
 print("---------------")
 print(X_test.isna().sum() ,"
" )<data_type_conversions>
train_generator = image_data.flow(x_train, y_train, batch_size = batch_size)
 trained_model = model.fit_generator(train_generator, epochs=epochs, validation_data=(x_val, y_val), verbose=1, steps_per_epoch=len(x_train)//batch_size,
callbacks=[LR_reducer] )
train.loc[~train.isna().Cabin, "Cabin"] = 1
train.Cabin.fillna(0, inplace=True)
 X_test.loc[~X_test.isna().Cabin, "Cabin"] = 1
X_test.Cabin.fillna(0, inplace=True )<data_type_conversions>
val_pred = model.predict(x_val)
 val_pred_classes = np.argmax(val_pred, axis = 1)
 val_true_classes = np.argmax(y_val, axis = 1)
 results = confusion_matrix(val_true_classes, val_pred_classes)
 print(results )
train.Age = train.groupby(["Pclass",pd.cut(train.Fare, np.arange(0, 300, 2)) ],group_keys=False ).Age.apply(lambda g: g.fillna(g.mean()))
train.Age.fillna(train.Age.mean() , inplace=True)
 train = train.astype({"Age":"int64"})
 X_test.Age = X_test.groupby(["Pclass",pd.cut(X_test.Fare, np.arange(0, 300, 2)) ],group_keys=False ).Age.apply(lambda g: g.fillna(g.mean()))
X_test.Age.fillna(X_test.Age.mean() , inplace=True)
 X_test = X_test.astype({"Age":"int64"})
 print(train.Age.isna().sum() , X_test.Age.isna().sum() )<correct_missing_values>
results = model.predict(x_test)
 results_classes = np.argmax(results, axis = 1 )
X_test.Fare.fillna(X_test[X_test.Pclass==3].Fare.mean() , inplace=True )<categorify>
result_data = {'ImageID': [i for i in range(1, len(results_classes)+1)],
'Label': results_classes
}
results_df = pd.DataFrame(result_data)
 results_df.head()
<split><EOS> 
results_df.to_csv('MNIST_data_output_using_CNN_30_epochs.csv', index = False )