Kaggle competition: Titanic - Machine Learning from Disaster
def run_cv(param, n_split = 3, verbose = 1):
preds_test = []
scores = []
oof = np.zeros([len(train), 21])
 kfold = StratifiedKFold(n_splits=n_split, shuffle = True, random_state=42)
 for i_fold,(train_idx, valid_idx)in enumerate(kfold.split(train, y_train)) :
print(f"--------fold {i_fold}-------")
 x_tr_wv = train_wv[train_idx]
x_tr = train_x[train_idx]
y_tr = y_train_ohe[train_idx]
x_va_wv = train_wv[valid_idx]
x_va = train_x[valid_idx]
y_va = y_train_ohe[valid_idx]
y_va_label = y_train[valid_idx]
model = trainNN(param, [x_tr_wv, x_tr], y_tr, [x_va_wv, x_va], y_va, y_va_label, verbose = verbose)
 pred_val = model.predict([x_va_wv, x_va])
 oof[valid_idx] = pred_val
score = get_score(y_va_label, pred_val)
 print(score)
 scores.append(score)
 pred_test = model.predict([test_wv, test_x])
 preds_test.append(pred_test)
 del model
score_df = pd.DataFrame(scores)
 return score_df, oof, preds_test<categorify>
train = pd.read_csv(".. /input/titanic/train.csv")
 test = pd.read_csv(".. /input/titanic/test.csv")
 submission_df = pd.read_csv('/kaggle/input/titanic/gender_submission.csv' )
<compute_train_metric>
df = pd.concat([train, test], axis=0)
 df = df.set_index('PassengerId')
 df.info()
params = {'decay': 0.0002561250212980425, 'batch_size': 276, 'n_unit': 2048, 'n_layer': 1, 'n_unit_scale': 0.6165090828302822, 'n_unit_fc': 225, 'n_layer_fc': 1, 'n_unit_fc_scale': 0.5835791877627263, 'p_drop': 0.57113050869184264, 'p_drop_fc': 0.3911374259982809}
score_df, oof, preds_test = run_cv(params, n_split = 10, verbose = False )<prepare_output>
df = df.drop(['Name', 'Ticket', 'Cabin'], axis=1)
pred_test_mean = np.array(preds_test ).mean(axis = 0)
 pred_test_final = np.argmax(pred_test_mean, axis = 1 )<save_to_csv>
df = df.drop('Fare', axis=1 )
sub["department_id"] = pred_test_final
sub.to_csv("submission.csv", index = False)
 sub.head()<categorify>
def fillna_using_knn_imputer(df):
imputer = KNNImputer(n_neighbors=5,
weights='uniform',
metric='nan_euclidean')
 df['Sex'] = df['Sex'].factorize() [0]
df['Embarked'] = df['Embarked'].factorize() [0]
df['Kind'] = df['Kind'].factorize() [0]
X_train = df[~df['Survived'].isnull() ].drop('Survived', axis=1)
 X_test = df[df['Survived'].isnull() ].drop('Survived', axis=1)
 X_train_trans = pd.DataFrame(imputer.fit_transform(X_train), columns=X_train.columns, index=X_train.index)
 X_test_trans = pd.DataFrame(imputer.transform(X_test), columns=X_test.columns, index=X_test.index)
 dff = pd.concat([X_train_trans, X_test_trans], axis=0)
 dff['Survived'] = df['Survived']
dff = dff.sort_index()
return dff
df = fillna_using_knn_imputer(df)
 df.info()
pd.to_pickle(
 {"train" : train, "test" : test, "oof" : oof, "pred_test" : pred_test_mean, "sub" : sub}, "data.pkl" )<load_from_csv>
def encode_cols(df, cols=['Pclass', 'Embarked']):
for col in cols:
dumm = pd.get_dummies(data=df[col], prefix=col)
 df = pd.concat([df, dumm], axis=1)
 df = df.drop(col, axis=1)
 return df
def scale_all_features(df):
X = df.drop('Survived', axis=1)
 scaler = MinMaxScaler()
X = pd.DataFrame(scaler.fit_transform(X), columns=X.columns, index=X.index)
 df = pd.concat([X, df['Survived']], axis=1)
 return df, scaler
df = encode_cols(df)
 df, _ = scale_all_features(df)
 df.info()
train = pd.read_csv(".. /input/ykc-cup-2nd/train.csv")
 test = pd.read_csv(".. /input/ykc-cup-2nd/test.csv")
 sub = pd.read_csv(".. /input/ykc-cup-2nd/sample_submission.csv")
 train.shape, test.shape, sub.shape<concatenate>
def split_to_train_test_X_y(df):
X = df[~df['Survived'].isna() ].drop('Survived', axis=1)
 y = df[~df['Survived'].isna() ]['Survived']
X_test = df[df['Survived'].isna() ].drop('Survived', axis=1)
 return X, y, X_test
X, y, X_test = split_to_train_test_X_y(df )
df = pd.concat([train, test])
 df = df.reset_index(drop=True)
 df.shape<choose_model_class>
class ML_Classifier_Switcher(object):
def pick_model(self, model_name):

 self.param_grid = None
method_name = str(model_name)
 method = getattr(self, method_name, lambda: "Invalid ML Model")
 return method()
def SVM(self):
self.param_grid = {'kernel': ['rbf', 'sigmoid', 'linear'],
'C': np.logspace(-2, 2, 10),
'gamma': np.logspace(-5, 1, 14)}
return SVC()
def XGR(self):
self.param_grid = {'gamma': np.logspace(-5, 1, 7),
'subsample': [0.5, 0.75, 1.0],
'colsample_bytree': [0.5, 0.75, 1.0],
'eta': [0.1, 0.5, 0.9],
'max_depth': [3, 5]}
return XGBClassifier(random_state=42, nthread=7, use_label_encoder=False,
eval_metric='error', tree_method = "hist")
 def RF(self):
self.param_grid = {
'n_estimators': [50, 100, 200, 300],
'max_features': ['auto'],
'criterion': ['entropy'],
'max_depth': [5, 10],
'min_samples_split': [5],
'min_samples_leaf': [1]
}
return RandomForestClassifier(random_state=42, n_jobs=-1)
 def LR(self):
self.param_grid = {'solver': ['newton-cg', 'lbfgs', 'liblinear'],
'penalty': ['l2'],
'C': [100, 10, 1.0, 0.1, 0.01]}
return LogisticRegression(n_jobs=None, random_state=42)
 def KNN(self):
self.param_grid = {
'n_neighbors': list(range(1, 5)) ,
'weights': ['uniform', 'distance'],
'algorithm': ['auto', 'ball_tree', 'kd_tree', 'brute'],
'leaf_size': list(range(1, 10)) ,
'p': [1, 2]
}
return KNeighborsClassifier()
target = "department_id"
n_split = 5
kfold = StratifiedKFold(n_splits=n_split, shuffle = True, random_state=42 )<feature_engineering>
def cross_validate(X, y, model_name='RF', cv=5, scoring='accuracy',
gridsearch=True):

 switcher = ML_Classifier_Switcher()
model = switcher.pick_model(model_name)
 if gridsearch:
gr = GridSearchCV(model, switcher.param_grid,
scoring=scoring, cv=cv, n_jobs=-1)
 gr.fit(X, y)
 model = gr.best_estimator_
cvr = cross_val_score(model, X, y, cv=cv, scoring=scoring)
 cvr = pd.DataFrame(cvr ).T
cvr.index = [model_name]
cvr.columns = ['fold_{}'.format(x+1)for x in cvr.columns]
cvr['mean'] = cvr.mean(axis=1)
 cvr['std'] = cvr.std(axis=1)
 return cvr, model
df["product_name"] = df["product_name"].apply(lambda words : words.lower().replace(",", "" ).replace("&", "" ).split(" "))
df.head()<define_variables>
def cross_validate_models(X, y, cv=5, scoring='accuracy'):
models = ['LR', 'KNN', 'SVM', 'RF']
cvrs = []
bests = []
for model_name in models:
print('Optimizing {} model'.format(model_name))
cvr, best = cross_validate(X, y, model_name=model_name, cv=cv, scoring=scoring,
gridsearch=True)
 cvrs.append(cvr)
 bests.append(best)
 cvr = pd.concat(cvrs, axis=0)
 return cvr, bests
model_names = ["crawl-300d-2M", "crawl-300d-2M-subword", "wiki-news-300d-1M", "wiki-news-300d-1M-subword"]<load_pretrained>
cvr, bests = cross_validate_models(X,y)
 print(cvr )
fasttext_pretrain_cols = []
unused_words = defaultdict(int)
 for model_name in model_names:
model = pd.read_pickle(f".. /input/fasttext/{model_name}.pkl")
 def to_mean_vec(x, model):
v = np.zeros(model.vector_size)
 for w in x:
try:
v += model[w]
except:
unused_words[w] += 1
v = v /(np.sqrt(np.sum(v ** 2)) + 1e-16)
 return v
def to_max_vec(x, model):
v = np.zeros(model.vector_size)
 for w in x:
try:
v = np.maximum(v, model[w])
 except:
pass
return v
mean_vecs = df["product_name"].apply(lambda x : to_mean_vec(x, model))
mean_vecs = np.vstack(mean_vecs)
 cols = [f"fasttext_pretrain_{model_name}_mean_vec{k}" for k in range(mean_vecs.shape[1])]
fasttext_pretrain_cols += cols
mean_vec_df = pd.DataFrame(mean_vecs, columns=cols)
 df = pd.concat([df, mean_vec_df], axis = 1)
 max_vecs = df["product_name"].apply(lambda x : to_max_vec(x, model))
max_vecs = np.vstack(max_vecs)
 cols = [f"fasttext_pretrain_{model_name}_max_vec{k}" for k in range(max_vecs.shape[1])]
fasttext_pretrain_cols += cols
max_vec_df = pd.DataFrame(max_vecs, columns=cols)
 df = pd.concat([df, max_vec_df], axis = 1)
 df.head()<split>
def predict_on_test(X_test, model, submission_df, target='Survived'):
preds_test = model.predict(X_test)
 submission_df.loc[:, target] = [int(x)for x in preds_test]
submission_df.to_csv('submission.csv', index=False)
 return
predict_on_test(X_test, bests[-1], submission_df )