Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<prepare_x_and_y>
np.random.seed(2018)
 train = pd.read_csv('.. /input/train.csv', usecols =['Survived','PassengerId','Name'])
 test = pd.read_csv('.. /input/test.csv', usecols =['PassengerId','Name'])
 test['Survived'] = np.nan
comb = pd.concat([ train, test ])
 comb.head()
y_train = train['isFraud'].copy()<set_options>
def clean_name(x):
x = x.replace(',',' ' ).replace('.',' ' ).replace('(',' ' ).replace(')',' ' ).replace('"',' ' ).replace('-',' ')
 return ' '.join([ w for w in x.split() if len(w)> 1] )
gc.collect()<drop_column>
count_vect = CountVectorizer(preprocessor=clean_name, min_df=2 )
del train_transaction, train_identity, test_transaction, test_identity
gc.collect()<data_type_conversions>
count_vect.fit(comb['Name'])
 for i,k in enumerate(count_vect.vocabulary_):
if i>5:
break
else:
print(k, count_vect.vocabulary_[k] )
train['nulls'] = train.isna().sum(axis=1)
 test['nulls'] = test.isna().sum(axis=1 )<feature_engineering>
v = count_vect.transform(comb['Name'])
 va = np.array(v.toarray())
 va.shape
train['D9'] =(train['TransactionDT']%(3600*24)/3600//1)/24.0
test['D9'] =(test['TransactionDT']%(3600*24)/3600//1)/24.0<define_variables>
pca = PCA(n_components = 6, random_state = 2018, whiten = True)
 va = pca.fit_transform(va)
 va_survived = va[comb.index[comb.loc[:,'Survived']==1],...]
va_perished = va[comb.index[comb.loc[:,'Survived']==0],...]
va_na = va[comb.index[comb.loc[:,'Survived']== np.nan],...]
train_cols = [c for c in train.columns if c not in list(set(['TransactionDT', 'id_30','id_31','id_33', 'id_35', "id_32", "D15", "D10", "D4", "C13", "D11", "C9", "C11", 'V300', 'V309', 'V111', 'C3', 'V124', 'V106', 'V125', 'V315', 'V134', 'V102', 'V123', 'V316', 'V113', 'V136', 'V305', 'V110', 'V299', 'V289', 'V286', 'V318', 'V103', 'V304', 'V116', 'V298', 'V284', 'V293', 'V137', 'V295', 'V301', 'V104', 'V311', 'V115', 'V109', 'V119', 'V321', 'V114', 'V133', 'V122', 'V319', 'V105', 'V112', 'V118', 'V117', 'V121', 'V108', 'V135', 'V320', 'V303', 'V297', 'V120',"isFraud", "id_32", "D15", 'D9','D15', 'V107', 'V28', 'V305', 'V68', 'V117', 'V119', 'V241', 'V27', 'V65', 'V89', 'V122', 'V88'])) ]<feature_engineering>
feature_names = ['Nv' + str(i)for i in range(va.shape[1])]
NameVect = pd.DataFrame(data = va, index = comb.index, columns = feature_names)
 NameVect.head()
for c in ['P_emaildomain', 'R_emaildomain']:
train[c + '_bin'] = train[c].map(emails)
 test[c + '_bin'] = test[c].map(emails)
 train[c + '_suffix'] = train[c].map(lambda x: str(x ).split('.')[-1])
 test[c + '_suffix'] = test[c].map(lambda x: str(x ).split('.')[-1])
 train[c + '_suffix'] = train[c + '_suffix'].map(lambda x: x if str(x)not in us_emails else 'us')
 test[c + '_suffix'] = test[c + '_suffix'].map(lambda x: x if str(x)not in us_emails else 'us')
 gc.collect()<define_variables>
comb2 = pd.concat([comb[['Survived']], NameVect],axis =1)
 comb2.head()
comb2.to_csv('name_only_df.csv',index=False )
cat_cols = train.select_dtypes("object" ).columns.tolist()<categorify>
df_train = comb2.loc[comb2['Survived'].isin([np.nan])== False]
df_test = comb2.loc[comb2['Survived'].isin([np.nan])== True]
print(df_train.shape)
 df_train.head()
%%time
for f in cat_cols:
lbl = preprocessing.LabelEncoder()
lbl.fit(list(train[f].values)+ list(test[f].values))
train[f] = lbl.transform(list(train[f].values))
test[f] = lbl.transform(list(test[f].values))
gc.collect()<choose_model_class>
from sklearn.model_selection import GridSearchCV
n_fold = 6
folds = KFold(n_fold, shuffle=False, random_state=11 )<init_hyperparams>
from sklearn.model_selection import GridSearchCV
params = {'tree_method': 'hist',
'silent': 1,
'colsample_bytree': 0.8999504397295506,
'subsample': 0.7446134812140273,
'learning_rate': 0.05,
'max_leaves': 72,
'objective': 'binary:logistic',
'max_depth': 0,
'reg_alpha': 0.7726783188295172,
'min_child_weight': 2,
'eval_metric': 'auc',
'grow_policy':'lossguide'}
params<prepare_x_and_y>
knclass = KNeighborsClassifier(n_neighbors=11, metric = 'manhattan')
 param_grid =({'n_neighbors':[3,4,5,6,7,8,9,11,12,13],'metric':['manhattan','minkowski'],'p':[1,2]})
 grs = GridSearchCV(knclass, param_grid, cv = 28, n_jobs=1, return_train_score = True, iid = False)
 grs.fit(np.array(df_train[feature_names]), np.array(df_train['Survived']))
%%time
oof_preds = np.zeros(train.shape[0])
 sub_preds = np.zeros(test.shape[0])
 feature_importance_df = pd.DataFrame()
for n_fold,(trn_idx, val_idx)in enumerate(folds.split(train, y_train)) :
trn_x, trn_y = train[train_cols].iloc[trn_idx], y_train.iloc[trn_idx]
val_x, val_y = train[train_cols].iloc[val_idx], y_train.iloc[val_idx]
gc.collect()
dtrain = xgb.DMatrix(trn_x, trn_y, feature_names=trn_x.columns)
 dval = xgb.DMatrix(val_x, val_y, feature_names=val_x.columns)
 gc.collect()
clf = xgb.train(params=params, dtrain=dtrain, num_boost_round=1500, evals=[(dtrain, "Train"),(dval, "Val")],
verbose_eval= 250, early_stopping_rounds=100)
 gc.collect()
oof_preds[val_idx] = clf.predict(xgb.DMatrix(val_x))
sub_preds += clf.predict(xgb.DMatrix(test[train_cols])) / folds.n_splits
gc.collect()
xgbfir.saveXgbFI(clf, feature_names=trn_x.columns, OutputXlsxFile='ieee_xgbfir_%sFold.xlsx'%str(n_fold+1), MaxInteractionDepth=9, MaxHistograms=15)
 gc.collect()
fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = pd.DataFrame.from_dict(data=clf.get_fscore() , orient="index", columns=["FScore"])["FScore"].index
fold_importance_df["fscore"] = pd.DataFrame.from_dict(data=clf.get_fscore() , orient="index", columns=["FScore"])["FScore"].values
fold_importance_df["fold"] = n_fold + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 gc.collect()
print('
Fold %2d AUC %.6f & std %.6f' %(n_fold + 1, roc_auc_score(val_y, oof_preds[val_idx]), np.std([oof_preds[val_idx]])))
 gc.collect()
print('
CV AUC score %.6f & std %.6f' %(roc_auc_score(y_train, oof_preds), np.std(( oof_preds))))<set_options>
print("Best parameters " + str(grs.best_params_))
gpd = pd.DataFrame(grs.cv_results_)
 print("Estimated accuracy of this model for unseen data:{0:1.4f}".format(gpd['mean_test_score'][grs.best_index_]))
<sort_values><EOS> 
pred_knn = grs.predict(np.array(df_test[feature_names]))
sub = pd.DataFrame({'PassengerId':test['PassengerId'],'Survived':pred_knn})
 sub.to_csv('name_only_knn.csv', index = False, float_format='%1d')
 sub.head()