Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<predict_on_test>
%matplotlib inline
init_notebook_mode(connected=True)
 warnings.filterwarnings("ignore", category=ConvergenceWarning)
 print(os.listdir(".. /input"))
threshold=best_thresh
wiki_y_test_pre=wiki_model.predict(test_padded, batch_size=BATCH_SIZE, verbose=1)
 wiki_y_test_pre=(wiki_y_test_pre>thresh ).astype(int )<drop_column>
img_rows, img_cols = 28, 28
np.random.seed(5)
del wiki_embed, wiki_embedding_mat, wiki_history, best_score, best_thresh
gc.collect()<compute_test_metric>
def get_best_score(model):
print(model.best_score_)
 print(model.best_params_)
 print(model.best_estimator_)
 return model.best_score_
y_pre=0.25*(google_y_pre + glove_y_pre + para_y_pre + wiki_y_pre)
 best_score = 0
best_thresh = 0
for thresh in np.arange(0.1,0.5,0.01):
if(best_score < metrics.f1_score(val.target,(y_pre>thresh ).astype(int))):
best_score = metrics.f1_score(val.target,(y_pre>thresh))
best_thresh = round(thresh, 2)
 print("threshold {0:2.2f} f1 score:{1:2.3f}".format(thresh,metrics.f1_score(val.target,(y_pre>thresh ).astype(int))))
print("\033[1mBest result {0:2.3f} in thresh_hold {1:2.2f}\033[0m".format(best_score, best_thresh))<save_to_csv>
def print_validation_report(y_true, y_pred):
print("Classification Report")
 print(classification_report(y_true, y_pred))
acc_sc = accuracy_score(y_true, y_pred)
 print("Accuracy : "+ str(acc_sc))
return acc_sc
y_test_pre = 0.2*google_y_test_pre + 0.3*glove_y_test_pre + 0.2*para_y_test_pre + 0.3*wiki_y_test_pre
y_test_pre=(y_test_pre>thresh ).astype(int)
 submit=pd.DataFrame()
submit["qid"]=df_test.qid
submit["prediction"]=y_test_pre
submit.to_csv("submission.csv",index=False )<load_from_disk>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )
!unzip -n.. /input/quora-insincere-questions-classification/embeddings.zip<import_modules>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )
tqdm.pandas()<import_modules>
y = train["label"]
X = train.drop(["label"],axis = 1)
 X_test = test
print(f"tf version: {tf.__version__}")
 print(f"keras version: {keras.__version__}" )<define_variables>
X = X/255.0
X_test = X_test/255.0
max_length = 60
embedding_size = 600
learning_rate = 0.001
batch_size = 512
num_epoch = 4<load_from_csv>
mode = "edit"
mode = "commit"
if mode == "edit" :
nr_samples = 1200
if mode == "commit" :
nr_samples = 30000
y_train=y[:nr_samples]
X_train=X[:nr_samples]
start_ix_val = nr_samples
end_ix_val = nr_samples + int(nr_samples/3)
 y_val=y[start_ix_val:end_ix_val]
X_val=X[start_ix_val:end_ix_val]
print("nr_samples train data:", nr_samples)
 print("start_ix_val:", start_ix_val)
 print("end_ix_val:", end_ix_val )
train = pd.read_csv(".. /input/quora-insincere-questions-classification/train.csv")
 test = pd.read_csv(".. /input/quora-insincere-questions-classification/test.csv" )<categorify>
y.value_counts()
regex = re.compile("["+re.escape(string.punctuation)+ "0-9\\r\\t\
]")
 def clean_text(x):
return regex.sub("",x )<define_variables>
li_idxs = []
for i in range(10):
for nr in range(10):
ix = y[y==nr].index[i]
li_idxs.append(ix )
mispell_dict = {'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', "mastrubating": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', "whst": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization'}
def correct_spelling(x):
for word in mispell_dict.keys() :
x = x.replace(word, mispell_dict[word])
 return x<feature_engineering>
clf_Perceptron = Perceptron(random_state=0)
 param_grid = { 'penalty': ['l1','l2'], 'tol': [0.05, 0.1] }
GridCV_Perceptron = GridSearchCV(clf_Perceptron, param_grid, verbose=1, cv=5)
 GridCV_Perceptron.fit(X_train,y_train)
 score_grid_Perceptron = get_best_score(GridCV_Perceptron )
train["question_text"] = train["question_text"].progress_apply(lambda x: x.lower())
 test["question_text"] = test["question_text"].progress_apply(lambda x: x.lower())
 train["question_text"] = train["question_text"].progress_apply(clean_text)
 test["question_text"] = test["question_text"].progress_apply(clean_text)
 train["question_text"] = train["question_text"].progress_apply(correct_spelling)
 test["question_text"] = test["question_text"].progress_apply(correct_spelling )<prepare_x_and_y>
pred_val_perc = GridCV_Perceptron.predict(X_val )
train_X = train["question_text"].values
test_X = test["question_text"].values<prepare_x_and_y>
acc_perc = print_validation_report(y_val, pred_val_perc )
tokenizer = Tokenizer()
tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=max_length)
 test_X = pad_sequences(test_X, maxlen=max_length)
 train_y = train['target'].values<split>
clf_LR = LogisticRegression(random_state=0)
 param_grid = {'C': [0.014,0.012], 'multi_class': ['multinomial'],
'penalty': ['l1'],'solver': ['saga'], 'tol': [0.1] }
GridCV_LR = GridSearchCV(clf_LR, param_grid, verbose=1, cv=5)
 GridCV_LR.fit(X_train,y_train)
 score_grid_LR = get_best_score(GridCV_LR )
embedding_files={"glove":r"glove.840B.300d/glove.840B.300d.txt","para":r"paragram_300_sl999/paragram_300_sl999.txt","wiki":r"wiki-news-300d-1M/wiki-news-300d-1M.vec"}
def get_coefs(word,*arr):
return word, np.asarray(arr, dtype='float32')
 def load_data(embedding,word_index):
embedding_file=embedding_files.get(embedding)
 if embedding=="glove":
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(embedding_file))
elif embedding=="para":
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(embedding_file, encoding="utf8", errors='ignore')if len(o)>100)
 elif embedding=="wiki":
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(embedding_file)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = len(word_index)+1
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
return embedding_matrix 
<choose_model_class>
pred_val_lr = GridCV_LR.predict(X_val)
 acc_lr = print_validation_report(y_val, pred_val_lr )
def build_model(embedding_matrix, nb_words, embedding_size=300):
inp = Input(shape=(max_length,))
x = Embedding(nb_words, embedding_size, weights=[embedding_matrix], trainable=False )(inp)
 x = SpatialDropout1D(0.3 )(x)
 x1 = Bidirectional(LSTM(256, return_sequences=True))(x)
 x2 = Bidirectional(GRU(128, return_sequences=True))(x1)
 max_pool1 = GlobalMaxPooling1D()(x1)
 max_pool2 = GlobalMaxPooling1D()(x2)
 conc = Concatenate()([max_pool1, max_pool2])
 predictions = Dense(1, activation='sigmoid' )(conc)
 model = Model(inputs=inp, outputs=predictions)
 adam = optimizers.Adam(lr=learning_rate)
 model.compile(optimizer=adam, loss='binary_crossentropy', metrics=['accuracy'])
 return model<load_pretrained>
clf_knn = KNeighborsClassifier(n_neighbors=10)
 clf_knn.fit(X_train,y_train )
train_pred = np.zeros(( len(train_X),), dtype=np.float32)
 pred_prob = np.zeros(( len(test_X),), dtype=np.float32)
 print("Loading embedding matrix...")
 embedding_matrix_wiki = load_data("wiki",tokenizer.word_index)
 embedding_matrix_glove = load_data("glove",tokenizer.word_index)
 embedding_matrix_para = load_data("para",tokenizer.word_index )<train_model>
pred_val_knn = clf_knn.predict(X_val)
 acc_knn = print_validation_report(y_val, pred_val_knn )
embedding_matrix = np.concatenate(( embedding_matrix_glove, embedding_matrix_wiki), axis=1)
 model = build_model(embedding_matrix, len(embedding_matrix), embedding_size)
 model.fit(train_X, train_y, batch_size=batch_size, epochs=num_epoch,validation_split=0.01 ,verbose=1)
 train_pred += 0.5*np.squeeze(model.predict(train_X, batch_size=batch_size, verbose=1))
pred_prob += 0.5*np.squeeze(model.predict(test_X, batch_size=batch_size, verbose=1))<train_model>
clf_RF = RandomForestClassifier(random_state=0)
 param_grid = {'max_depth': [15], 'max_features': [100],
'min_samples_split': [5],'n_estimators' : [50] }
GridCV_RF = GridSearchCV(clf_RF, param_grid, verbose=1, cv=5)
 GridCV_RF.fit(X_train,y_train)
 score_grid_RF = get_best_score(GridCV_RF )
embedding_matrix = np.concatenate(( embedding_matrix_glove, embedding_matrix_para), axis=1)
 model = build_model(embedding_matrix, len(embedding_matrix), embedding_size)
 model.fit(train_X, train_y, batch_size=batch_size, epochs=num_epoch,validation_split=0.01 ,verbose=1)
 train_pred += 0.5*np.squeeze(model.predict(train_X, batch_size=batch_size, verbose=1))
pred_prob += 0.5*np.squeeze(model.predict(test_X, batch_size=batch_size, verbose=1))<compute_test_metric>
pred_val_rf = GridCV_RF.predict(X_val )
i=0.01
maxi=0.99
maxf1=0
while i<0.99:
pred_train =(train_pred > i ).astype(np.int)
 f1=f1_score(train_y, pred_train)
 if f1>maxf1:
maxi=i
maxf1=f1
i+=0.01
print(f"threshold: {maxi}")
 print(f"max f1 score: {maxf1}" )<save_to_csv>
acc_rf = print_validation_report(y_val, pred_val_rf )
submission = pd.DataFrame.from_dict({'qid': test['qid']})
 submission['prediction'] =(pred_prob>maxi ).astype(int)
 submission.to_csv('submission.csv', index=False )<load_from_csv>
clf_svm = SVC(C=5, gamma=0.05, kernel='rbf', random_state=0)
 clf_svm.fit(X_train,y_train )
df_train = pd.read_csv(".. /input/titanic/train.csv")
 df_test = pd.read_csv(".. /input/titanic/test.csv" )<prepare_x_and_y>
pred_val_svm = clf_svm.predict(X_val)
 acc_svm = print_validation_report(y_val, pred_val_svm )
X_train = df_train.copy()
X_test = df_test.copy()<drop_column>
batchsize = int(nr_samples/15 )
X_train = X_train.set_index('PassengerId')
 X_test = X_test.set_index('PassengerId' )<count_missing_values>
clf_mlp = MLPClassifier(activation = "logistic", hidden_layer_sizes=(200,), random_state=0)
 param_grid = { 'batch_size' : [batchsize] , 'max_iter': [600], 'alpha': [1e-4],
'solver': ['sgd'], 'learning_rate_init': [0.05,0.06],'tol': [1e-4] }
GridCV_MLP = GridSearchCV(clf_mlp, param_grid, verbose=1, cv=3)
 GridCV_MLP.fit(X_train,y_train)
 score_grid_MLP = get_best_score(GridCV_MLP )
X_train_null = X_train.isnull().sum()
X_train_null<feature_engineering>
pred_val_mlp = GridCV_MLP.predict(X_val )
X_train_null/len(X_train)* 100<count_missing_values>
acc_mlp = print_validation_report(y_val, pred_val_mlp )
X_test_null = X_test.isnull().sum()
X_test_null<feature_engineering>
y_train = to_categorical(y_train, 10)
 y_val_10 = to_categorical(y_val, 10 )
X_test_null/len(X_test)* 100<feature_engineering>
def dense_model_0() :
model = Sequential()
model.add(Dense(10, input_dim=784, activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model
X_train['Age'] = X_train['Age'].fillna(X_train['Age'].mean())
 X_test['Age'] = X_test['Age'].fillna(X_test['Age'].mean() )<count_values>
model_dense_0.fit(X_train, y_train, epochs=50, batch_size=batchsize )
X_train['Embarked'].value_counts()<data_type_conversions>
pred_val_dense0 = model_dense_0.predict_classes(X_val )
X_train['Embarked'] = X_train['Embarked'].fillna('S')
<drop_column>
acc_fc0 = print_validation_report(y_val, pred_val_dense0 )
X_train = X_train.drop('Cabin', axis=1)
 X_test = X_test.drop('Cabin', axis=1 )<feature_engineering>
def dense_model_1() :
model = Sequential()
model.add(Dense(100, input_dim=784, activation='relu'))
model.add(Dense(10, activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model
X_test['Fare'] = X_test['Fare'].fillna(X_test['Fare'].mean() )<count_missing_values>
history_dense_1 = model_dense_1.fit(X_train, y_train, validation_data=(X_val,y_val_10),
epochs=50, batch_size=batchsize )
X_train_null = X_train.isnull().sum()
X_train_null<count_missing_values>
pred_val_dense1 = model_dense_1.predict_classes(X_val)
 plot_confusion_matrix(y_val, pred_val_dense1)
 print(classification_report(y_val, pred_val_dense1))
acc_fc1 = accuracy_score(y_val, pred_val_dense1)
 print(acc_fc1 )
X_test_null = X_test.isnull().sum()
X_test_null<drop_column>
def dense_model_2() :
model = Sequential()
model.add(Dense(100, input_dim=784, activation='relu'))
model.add(Dense(200, activation='relu'))
model.add(Dense(10, activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model
X_train = X_train.drop(['Name', 'Ticket', 'Survived'], axis = 1)
 X_train<drop_column>
history_dense_2 = model_dense_2.fit(X_train, y_train, validation_data=(X_val,y_val_10),
epochs=50, batch_size=batchsize )
X_test = X_test.drop(['Name', 'Ticket'], axis = 1)
 X_test<drop_column>
pred_val_dense2 = model_dense_2.predict_classes(X_val)
 plot_confusion_matrix(y_val, pred_val_dense2)
 print(classification_report(y_val, pred_val_dense2))
acc_fc2 = accuracy_score(y_val, pred_val_dense2)
 print(acc_fc2 )
X_test['family'] = X_test['SibSp'] + X_test['Parch']
X_test = X_test.drop(['SibSp','Parch'], axis=1)
 X_test<drop_column>
def dense_model_3() :
model = Sequential()
model.add(Dense(100, activation='relu', input_dim=784))
model.add(Dense(200, activation='relu'))
model.add(Dense(100, activation='relu'))
model.add(Dense(10, activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model
X_train['family'] = X_train['SibSp'] + X_train['Parch']
X_train = X_train.drop(['SibSp','Parch'], axis=1)
 X_train<prepare_x_and_y>
history_dense_3 = model_dense_3.fit(X_train, y_train, validation_data=(X_val,y_val_10),
epochs=50, batch_size=batchsize )
Y_train = df_train['Survived']
Y_train<count_missing_values>
pred_val_dense3 = model_dense_3.predict_classes(X_val)
 plot_confusion_matrix(y_val, pred_val_dense3)
 print(classification_report(y_val, pred_val_dense3))
acc_fc3 = accuracy_score(y_val, pred_val_dense3)
 print(acc_fc3 )
X_train_null = X_train.isnull().sum()
X_train_null<count_missing_values>
X_train = X_train.values.reshape(X_train.shape[0], img_rows, img_cols, 1)
 X_val = X_val.values.reshape(X_val.shape[0], img_rows, img_cols, 1)
 input_shape =(img_rows, img_cols, 1 )
X_test_null = X_test.isnull().sum()
X_test_null<categorify>
batchsize = 128
epochs = 12
X_train = pd.get_dummies(X_train, columns=['Sex', 'Embarked'], drop_first=True)
 X_train<categorify>
activation = 'relu'
adadelta = Adadelta()
loss = categorical_crossentropy
X_test = pd.get_dummies(X_test, columns=['Sex', 'Embarked'], drop_first=True)
 X_test<import_modules>
def cnn_model_1(activation):
model = Sequential()
model.add(Conv2D(32, kernel_size=(3, 3), activation=activation, input_shape=input_shape))
model.add(Conv2D(64,(3, 3), activation=activation))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(128, activation=activation))
model.add(Dropout(0.5))
model.add(Dense(10, activation='softmax'))
model.compile(loss=loss, optimizer=adadelta, metrics=['accuracy'])
 return model
from sklearn.model_selection import GridSearchCV
from sklearn.ensemble import RandomForestClassifier<choose_model_class>
model_cnn_1 = cnn_model_1(activation)
 model_cnn_1.summary()
Rf = RandomForestClassifier()<define_search_space>
history_cnn_1 = model_cnn_1.fit(X_train, y_train, validation_data=(X_val,y_val_10),
epochs=epochs, batch_size=batchsize, verbose=1 )
param_grid = {"n_estimators":[20, 40, 100, 120, 150],'max_depth': np.arange(1, 20), 'max_features': ['auto', 'sqrt'],
'criterion' :['gini', 'entropy']}<choose_model_class>
pred_val_cnn1 = model_cnn_1.predict_classes(X_val)
 plot_confusion_matrix(y_val, pred_val_cnn1)
 print(classification_report(y_val, pred_val_cnn1))
acc_cnn1 = accuracy_score(y_val, pred_val_cnn1)
 print(acc_cnn1)
RF_grid = GridSearchCV(Rf, param_grid=param_grid, cv=10, scoring='accuracy' )<train_model>
batch_size=90
epochs=30
RF_grid.fit(X_train, Y_train )<find_best_params>
def cnn_model_2(optimizer,loss):
model = Sequential()
model.add(Conv2D(32,(3, 3), padding = 'Same', activation="relu", input_shape=input_shape))
model.add(MaxPooling2D(pool_size =(2, 2)))
 model.add(Conv2D(32,(3, 3), activation="relu"))
model.add(MaxPooling2D(pool_size =(2, 2)))
 model.add(Flatten())
 model.add(Dense(256, activation=activation))
model.add(Dense(10, activation='softmax'))
model.compile(optimizer = optimizer, loss = loss, metrics = ['accuracy'])
 return model
RF_grid.best_params_<find_best_params>
model_cnn_2 = cnn_model_2(adadelta, categorical_crossentropy)
 model_cnn_2.summary()
RF_cv_result = RF_grid.cv_results_['mean_test_score']<import_modules>
history_cnn_2 = model_cnn_2.fit(X_train, y_train, validation_data=(X_val,y_val_10),
epochs=epochs, batch_size=batchsize, verbose=1 )
from xgboost import XGBClassifier
from sklearn.model_selection import StratifiedKFold
from sklearn.model_selection import RandomizedSearchCV<choose_model_class>
pred_val_cnn2 = model_cnn_2.predict_classes(X_val)
 plot_confusion_matrix(y_val, pred_val_cnn2)
 print(classification_report(y_val, pred_val_cnn2))
acc_cnn2 = accuracy_score(y_val, pred_val_cnn2)
 print(acc_cnn2 )
xgb_model = XGBClassifier()<define_search_space>
sample_submission = pd.read_csv('.. /input/sample_submission.csv')
 if mode == "edit" :
X = X[:nr_samples//2]
y = y[:nr_samples//2]
X_test = X_test[:nr_samples//2]
sample_submission = sample_submission[:nr_samples//2]
distributions = {"n_estimators":[20, 40, 100, 120, 150],'max_depth': np.arange(1, 20),
'criterion' :['gini', 'entropy'],
'learning_rate' : [0.0001, 0.001, 0.01, 0.1, 0.2, 0.3]}<choose_model_class>
print(GridCV_Perceptron.best_params_)
 GridCV_Perceptron.best_estimator_.fit(X,y )
kfold = StratifiedKFold(n_splits=10, shuffle=True, random_state=7)
 XG_grid = RandomizedSearchCV(xgb_model, distributions, scoring="neg_log_loss", n_jobs=-1, cv=kfold )<train_model>
pred_test_perc = GridCV_Perceptron.best_estimator_.predict(X_test)
 result_perc = pd.DataFrame({'ImageId':sample_submission.ImageId, 'Label':pred_test_perc})
 result_perc.to_csv("subm_perc.csv",index=False )
XG_grid.fit(X_train, Y_train )<find_best_params>
print(GridCV_LR.best_params_)
 GridCV_LR.best_estimator_.fit(X,y )
XG_grid.best_params_<find_best_params>
pred_test_lr = GridCV_LR.best_estimator_.predict(X_test)
 result_lr = pd.DataFrame({'ImageId':sample_submission.ImageId, 'Label':pred_test_lr})
 result_lr.to_csv("subm_lr.csv",index=False )
XG_grid_result = XG_grid.cv_results_['mean_test_score']
XG_grid_result<predict_on_test>
clf_knn.fit(X,y )
predictions = RF_grid.predict(X_test)
 predictions<save_to_csv>
pred_test_knn = clf_knn.predict(X_test)
 result_knn = pd.DataFrame({'ImageId':sample_submission.ImageId, 'Label':pred_test_knn})
 result_knn.to_csv("subm_knn.csv",index=False )
output = pd.DataFrame({'PassengerId': df_test.PassengerId, 'Survived': predictions})
 output.to_csv('my_submission.csv', index=False)
 print("Your submission was successfully saved!" )<load_from_csv>
print(GridCV_RF.best_params_)
 GridCV_RF.best_estimator_.fit(X,y )
df_train = pd.read_csv('.. /input/titanic/train.csv')
 df_test = pd.read_csv('.. /input/titanic/test.csv')
 data = df_train.append(df_test )<count_missing_values>
pred_test_rf = GridCV_RF.best_estimator_.predict(X_test)
 result_rf = pd.DataFrame({'ImageId':sample_submission.ImageId, 'Label':pred_test_rf})
 result_rf.to_csv("subm_rf.csv",index=False )
df_train.isnull().sum()<count_missing_values>
clf_svm.fit(X,y )
df_test.isnull().sum()<count_duplicates>
pred_test_svm = clf_svm.predict(X_test)
 result_svm = pd.DataFrame({'ImageId':sample_submission.ImageId, 'Label':pred_test_svm})
 result_svm.to_csv("subm_svm.csv",index=False )
df_train.duplicated().sum()<train_model>
print(GridCV_MLP.best_params_)
 GridCV_MLP.best_estimator_.fit(X,y )
print("Training set size",len(df_train))
print("Test set size",len(df_train))<feature_engineering>
pred_test_mlp = GridCV_MLP.best_estimator_.predict(X_test)
 result_mlp = pd.DataFrame({'ImageId':sample_submission.ImageId, 'Label':pred_test_mlp})
 result_mlp.to_csv("subm_mlp.csv",index=False )
knn = KNNImputer(n_neighbors=3)
 data["Age"] = knn.fit_transform(data["Age"].values.reshape(-1,1)).ravel()<categorify>
y = to_categorical(y, 10 )
data["Fare"] = data["Fare"].fillna(data[:891]["Fare"].mode() [0])
 data["Embarked"] = data["Embarked"].fillna(data[:891]["Embarked"].mode() [0] )<feature_engineering>
model_dense_1.fit(X,y)
 pred_test_fc1 = model_dense_1.predict_classes(X_test)
 result_fc1 = pd.DataFrame({'ImageId':sample_submission.ImageId, 'Label':pred_test_fc1})
 result_fc1.to_csv("dense_1.csv",index=False )
data["Ticket_num"] = data["Ticket"].apply(lambda x:"".join(re.findall(r"\d+",x)))
 data["Name_last"] = data["Name"].apply(lambda x:x.split(",",1)[0] )<categorify>
model_dense_2.fit(X,y)
 pred_test_fc2 = model_dense_2.predict_classes(X_test)
 result_fc2 = pd.DataFrame({'ImageId':sample_submission.ImageId, 'Label':pred_test_fc2})
 result_fc2.to_csv("dense_2.csv",index=False )
Def = 0.5
data["family_survived"] = Def
da_group = data.groupby(["Name_last","Fare"])
 for group,all_data in da_group:
if(len(all_data)>= 1):
for index, row in all_data.iterrows() :
if(row['family_survived'] == 0)or(row['family_survived']== 0.5):
surviv_max = all_data.drop(index)['Survived'].max()
surviv_min = all_data.drop(index)['Survived'].min()
ids = row['PassengerId']
if(surviv_max == 1.0):
data.loc[data['PassengerId'] == ids, 'family_survived'] = 1
elif(surviv_min==0.0):
data.loc[data['PassengerId'] == ids, 'family_survived'] = 0
print("family survival:", data.loc[data['family_survived'] == 1].shape[0])
<groupby>
model_dense_3.fit(X,y)
 pred_test_fc3 = model_dense_3.predict_classes(X_test)
 result_fc3 = pd.DataFrame({'ImageId':sample_submission.ImageId, 'Label':pred_test_fc3})
 result_fc3.to_csv("dense_3.csv",index=False )
da_group = data[["PassengerId","Name_last","Survived","family","Fare","Ticket_num"]].groupby(["Name_last","Ticket_num"])
 for group,all_data in da_group:
if(len(all_data)>= 1):
for index, row in all_data.iterrows() :
surviv_max = all_data.drop(index)['Survived'].max()
surviv_min = all_data.drop(index)['Survived'].min()
ids = row['PassengerId']
if(surviv_max == 1.0):
data.loc[data['PassengerId'] == ids, 'family_survived'] = 1
elif(surviv_min==0.0):
data.loc[data['PassengerId'] == ids, 'family_survived'] = 0
print("family survival:", data.loc[data['family_survived'] == 1].shape[0] )<groupby>
X = X.values.reshape(X.shape[0], img_rows, img_cols, 1)
 X_test = X_test.values.reshape(X_test.shape[0], img_rows, img_cols, 1)
da_group = data.groupby(["Ticket_num"])
 for group,all_data in da_group:
if(len(all_data)>= 1):
for index, row in all_data.iterrows() :
if(row['family_survived'] == 0)or(row['family_survived']== 0.5):
surviv_max = all_data.drop(index)['Survived'].max()
surviv_min = all_data.drop(index)['Survived'].min()
ids = row['PassengerId']
if(surviv_max == 1.0):
data.loc[data['PassengerId'] == ids, 'family_survived'] = 1
elif(surviv_min==0.0):
data.loc[data['PassengerId'] == ids, 'family_survived'] = 0
print("family survival:", data.loc[data['family_survived'] == 1].shape[0] )<drop_column>
batchsize = 128
epochs = 12
model_cnn_1 = cnn_model_1('relu')
 model_cnn_1.fit(X, y, epochs=epochs, batch_size=batchsize, verbose=0 )
data = data.drop(columns=["Name","Ticket","Ticket_num","Name_last"] )<feature_engineering>
pred_test_cnn_1 = model_cnn_1.predict(X_test)
 pred_test_cnn_1 = np.argmax(pred_test_cnn_1,axis=1)
 result_cnn_1 = pd.DataFrame({'ImageId':sample_submission.ImageId, 'Label':pred_test_cnn_1})
 result_cnn_1.to_csv("subm_cnn_1.csv",index=False )
data["Age"] = pd.qcut(data["Age"],7 )<feature_engineering>
batch_size=90
epochs=30
model_cnn_2 = cnn_model_2(adadelta, categorical_crossentropy)
 model_cnn_2.fit(X, y, epochs=epochs, batch_size=batchsize, verbose=0 )
data["Fare"] = pd.qcut(data["Fare"],7 )<feature_engineering>
pred_test_cnn_2 = model_cnn_2.predict(X_test)
 pred_test_cnn_2 = np.argmax(pred_test_cnn_2,axis=1)
 result_cnn_2 = pd.DataFrame({'ImageId':sample_submission.ImageId, 'Label':pred_test_cnn_2})
 result_cnn_2.to_csv("subm_cnn_2_adadelta.csv",index=False )
data["Cabin"] = data["Cabin"].apply(lambda x:str(x)[0] )<categorify>
predictions = {'PERC': pred_test_perc, 'LR': pred_test_lr, 'KNN': pred_test_knn,
'RF': pred_test_rf, 'SVM': pred_test_svm, 'MLP': pred_test_mlp,
'DENSE1': pred_test_fc1, 'DENSE2': pred_test_fc2, 'DENSE3': pred_test_fc3,
'CNN1': pred_test_cnn_1, 'CNN2': pred_test_cnn_2}
df_predictions = pd.DataFrame(data=predictions)
 df_predictions.corr()
la = LabelEncoder()
la.fit(data["Sex"])
 data["Sex"] = la.transform(data["Sex"])
 la.fit(data["Age"])
 data["Age"] = la.transform(data["Age"])
 la.fit(data["Fare"])
 data["Fare"] = la.transform(data["Fare"])
 la.fit(data["Embarked"])
 data["Embarked"] = la.transform(data["Embarked"])
 la.fit(data["Cabin"])
 data["Cabin"] = la.transform(data["Cabin"])
 all_col = ["Sex","Age","Fare","Embarked","Pclass","family","family_survived","Cabin"]
for i in all_col:
std = StandardScaler()
std.fit(data[:891].loc[:,i].values.reshape(-1, 1))
data[i] = std.transform(data.loc[:,i].values.reshape(-1, 1))<split>
list_classifiers = ['PERC','LR','KNN','RF','SVM',
'MLP','DENSE1','DENSE2','DENSE3',
'CNN1','CNN2']
df_train = data[:891]
df_test = data[891:]<count_values>
val_scores = [acc_perc, acc_lr, acc_knn, acc_rf,
acc_svm, acc_mlp, acc_fc1, acc_fc2,
acc_fc3, acc_cnn1, acc_cnn2]
df_train["Survived"].value_counts(normalize=True )<import_modules>
score_perc = 0.88057
score_lr = 0.88700
score_knn = 0.96557
score_rf = 0.96028
score_svm = 0.98100
score_mlp = 0.96985
score_dns_1 = 0.95971
score_dns_2 = 0.96228
score_dns_3 = 0.96128
score_cnn_1 = 0.98928
score_cnn_2 = 0.99028
from sklearn.model_selection import StratifiedKFold,cross_val_score,RandomizedSearchCV,GridSearchCV
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier,AdaBoostClassifier,GradientBoostingClassifier,StackingClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.metrics import f1_score,recall_score,precision_score,roc_auc_score
from xgboost import XGBClassifier<prepare_x_and_y>
test_scores = [score_perc, score_lr, score_knn, score_rf, score_svm, score_mlp,
score_dns_1, score_dns_2, score_dns_3, score_cnn_1, score_cnn_2]
target = df_train.iloc[:,1]
feature = df_train.iloc[:,2:]
stk = StratifiedKFold(n_splits=5,random_state=123,shuffle=True)
 for train_index,test_index in stk.split(feature,target):
x_train,y_train = feature.iloc[train_index],target.iloc[train_index]
x_test,y_test = feature.iloc[test_index],target.iloc[test_index]<compute_test_metric>
model_cnn_2.optimizer
gbdt = GradientBoostingClassifier(random_state=123)
 gbdt.fit(x_train,y_train)
 gbdt_pre = gbdt.predict(x_test)
 print("gbdt recall:",round(recall_score(y_test,gbdt_pre),2))
print("gbdt precision:",round(precision_score(y_test,gbdt_pre),2))
print("gbdt f1_score:",round(f1_score(y_test,gbdt_pre),2))
print("gbdt rou_auc_score:",round(roc_auc_score(y_test,gbdt_pre),2))<train_on_grid>
model_cnn_2_rmsprop = cnn_model_2(RMSprop() , categorical_crossentropy)
 model_cnn_2_rmsprop.optimizer
params = {
"max_depth":(2,8),
"learning_rate":(0.01,0.1),
"min_samples_leaf":(1,3),
"n_estimators":(100,300)
 }
gbdt_gd = RandomizedSearchCV(gbdt,param_distributions=params,cv=5,scoring="roc_auc")
 gbdt_gd.fit(x_train,y_train)
 print(gbdt_gd.best_score_)
 print(gbdt_gd.best_estimator_)
 print(gbdt_gd.best_params_ )<compute_test_metric>
model_cnn_2.fit(X, y, epochs=epochs, batch_size=batchsize, verbose=0)
 pred_test_cnn_2 = model_cnn_2.predict(X_test)
 pred_test_cnn_2 = np.argmax(pred_test_cnn_2,axis=1)
 result_cnn_2 = pd.DataFrame({'ImageId':sample_submission.ImageId, 'Label':pred_test_cnn_2})
 result_cnn_2.to_csv("subm_cnn_2_rmsprop.csv",index=False )
gbdt_pre = gbdt_gd.predict(x_test)
 print("gbdt_gd recall:",round(recall_score(y_test,gbdt_pre),2))
print("gbdt_gd precision:",round(precision_score(y_test,gbdt_pre),2))
print("gbdt_gd f1_score:",round(f1_score(y_test,gbdt_pre),2))
print("gbdt_gd rou_auc_score:",round(roc_auc_score(y_test,gbdt_pre),2))<compute_test_metric>
model_cnn_2_adam = cnn_model_2(Adam() , categorical_crossentropy)
 model_cnn_2_adam.optimizer
adaboost = AdaBoostClassifier(random_state=123)
 adaboost.fit(x_train,y_train)
 adaboost_pre = gbdt.predict(x_test)
 print("adaboost recall:",round(recall_score(y_test,adaboost_pre),2))
print("adaboost precision:",round(precision_score(y_test,adaboost_pre),2))
print("adaboost f1_score:",round(f1_score(y_test,adaboost_pre),2))
print("adaboost rou_auc_score:",round(roc_auc_score(y_test,adaboost_pre),2))<train_on_grid>
model_cnn_2.fit(X, y, epochs=epochs, batch_size=batchsize, verbose=0)
 pred_test_cnn_2 = model_cnn_2.predict(X_test)
 pred_test_cnn_2 = np.argmax(pred_test_cnn_2,axis=1)
 result_cnn_2 = pd.DataFrame({'ImageId':sample_submission.ImageId, 'Label':pred_test_cnn_2})
 result_cnn_2.to_csv("subm_cnn_2_adam.csv",index=False )
<compute_test_metric><EOS> 
arr_y_val = y_val.values
false_cnn2 = pred_val_cnn2 != arr_y_val