Kaggle competition: Don't Get Kicked!
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=testset_size, stratify=y,random_state=rstate)
 X_train.shape, y_train.shape, X_test.shape, y_test.shape
cv = StratifiedShuffleSplit(n_splits=n_splits, test_size=0.2, random_state=rstate )<find_best_model_class>
train_data=pd.read_csv('/kaggle/input/DontGetKicked/training.csv')
 train_data.head()
def save(model, modelname):

 global results
model.best_estimator_.fit(X_train, y_train)
 scores = cross_val_score(model.best_estimator_, X_train, y_train, cv=5,
scoring=scoring, verbose =0)
 CV_scores = scores.mean()
STDev = scores.std()
Test_scores = model.score(X_test, y_test)
 results = results.append({'Model': modelname,'Para': model.best_params_,'Test_Score': Test_scores,
'CV Mean':CV_scores, 'CV STDEV': STDev}, ignore_index=True)
 ensemble_models[modelname] = model.best_estimator_
print("
Evaluation Method: {}".format(scoring))
print("Optimal Model Parameters: {}".format(grid.best_params_))
print("Train CV Accuracy: %0.2f(+/- %0.2f)[%s]" %(CV_scores, STDev, modelname))
print('Test_Score:', Test_scores)
 model.best_estimator_.fit(X_train, y_train)
 pred = model.predict(X_test)
 skplt.metrics.plot_confusion_matrix(y_test, pred, title="{} Confusion Matrix".format(modelname),
normalize=True,figsize=(6,6),text_fontsize='large')
 plt.show()
def norm_save(model,score, modelname):
global results
model.fit(X, y)
 submission = model.predict(test_df)
 df = pd.DataFrame({'PassengerId':test_df.index,
'Survived':submission})
 CV_Score = score.mean()
Test_scores = model.score(X_test, y_test)
 STDev = score.std()
Test_Score = model.score(X_test, y_test)
 results = results.append({'Model': modelname,'Para': model,'Test_Score': Test_scores,
'CV Mean': CV_Score, 'CV STDEV': STDev}, ignore_index=True)
 ensemble_models[modelname] = model
print("
Evaluation Method: {}".format(scoring))
print("Train CV Accuracy: %0.2f(+/- %0.2f)[%s]" %(CV_Score, STDev, modelname))
print('Test_Score:', Test_scores)
 model.fit(X_train, y_train)
 pred = model.predict(X_test)
 skplt.metrics.plot_confusion_matrix(y_test, pred, title="{} Confusion Matrix".format(modelname),
normalize=True,figsize=(6,6),text_fontsize='large')
 plt.show()
def eval_plot(model):
skplt.metrics.plot_roc_curve(y_test, model.predict_proba(X_test))
plt.show()<train_on_grid>
test_data = pd.read_csv('/kaggle/input/DontGetKicked/test.csv')
 test_data.head()
param_grid ={'n_neighbors': st.randint(1,40),
'weights':['uniform','distance']
}
grid = RandomizedSearchCV(KNeighborsClassifier() ,
param_grid,
cv=cv,
scoring=scoring,
verbose=1,
n_iter=n_iter,
random_state=rstate)
 grid.fit(X_train, y_train)
 save(grid, "KNN" )<train_on_grid>
test_data.isnull().sum()
param_grid ={'loss':["hinge","log","modified_huber","epsilon_insensitive","squared_epsilon_insensitive"]}
grid = GridSearchCV(SGDClassifier(max_iter=5, tol=None),
param_grid,cv=cv, scoring=scoring,
verbose=1)
 grid.fit(X_train, y_train)
 save(grid, "StochasticGradientDescent" )<find_best_params>
train_data.isnull().sum()
DecisionTreeClassifier().get_params().keys()<compute_train_metric>
train_data['IsBadBuy'].value_counts()
tree = DecisionTreeClassifier()
print("Mean CV Accuracy:",cross_val_score(tree, X, y, cv=cv, scoring=scoring ).mean())
 feature_imp(tree )<train_on_grid>
train_data['Model'].value_counts()
param_grid ={'n_estimators': n_tree_range}
tree = DecisionTreeClassifier()
grid = RandomizedSearchCV(BaggingClassifier(tree),
param_grid, cv=cv, scoring=scoring,
verbose=1,n_iter=n_iter, random_state=rstate)
 grid.fit(X_train, y_train)
 save(grid, "Bagger_ensemble" )<compute_train_metric>
train_data.drop('Model',axis=1,inplace=True)
 test_data.drop("Model",axis=1,inplace=True )
model = RandomForestClassifier()
print("Mean CV Accuracy:",cross_val_score(model, X, y, cv=cv, scoring=scoring ).mean())
 feature_imp(model )<train_on_grid>
train_data['Trim'].value_counts()
param_grid ={
'max_depth': st.randint(3, 11),
'max_features':["sqrt", "log2"],
'max_leaf_nodes':st.randint(6, 10)
 }
model= RandomForestClassifier()
grid = RandomizedSearchCV(model,
param_grid, cv=cv,
scoring=scoring,
verbose=1,n_iter=n_iter, random_state=rstate)
 grid.fit(X_train, y_train)
 save(grid, "Random_Forest" )<find_best_params>
train_data.drop('Trim',inplace=True,axis=1)
 test_data.drop('Trim',inplace=True,axis=1 )
AdaBoostClassifier().get_params().keys()<train_on_grid>
train_data['SubModel'].value_counts()
param_grid ={'n_estimators':n_tree_range,
'learning_rate':np.arange (.1, 4,.5)}
grid = RandomizedSearchCV(AdaBoostClassifier() ,
param_grid,cv=cv, scoring=scoring,
verbose=1, n_iter=n_iter, random_state=rstate)
 grid.fit(X_train, y_train)
 save(grid, "AdaBoost_Ensemble" )<find_best_params>
train_data.drop('SubModel',inplace=True,axis=1)
 test_data.drop('SubModel',inplace=True,axis=1 )
GradientBoostingClassifier().get_params().keys()<train_on_grid>
train_data['Color'].value_counts()
param_grid ={'n_estimators': n_tree_range,
'loss': ['deviance', 'exponential'],
'learning_rate': [0.1, 0.01,0.05,0.001],
'max_depth': np.arange(2, 12, 1)}
grid = RandomizedSearchCV(GradientBoostingClassifier() ,
param_grid,cv=cv,
scoring=scoring,
verbose=1, n_iter=n_iter, random_state=rstate)
 grid.fit(X_train, y_train)
 save(grid, "Gradient_Boosting" )<train_on_grid>
test_data['Color'].value_counts()
params = {
"n_estimators": n_tree_range,
"max_depth": st.randint(3, 15),
"learning_rate": [0.05]
}
xgbreg = XGBClassifier(objective= 'binary:logistic', eval_metric="auc",
nthreads=2)
 grid = RandomizedSearchCV(xgbreg, params, n_jobs=1, verbose=1, n_iter=n_iter,
random_state=rstate, scoring=scoring)
 grid.fit(X_train,y_train, verbose=False)
 save(grid, "Sci_kit XGB" )<train_model>
train_data['Color'].fillna(value='Color_Unknown',inplace=True)
 test_data['Color'].fillna(value='Color_Unknown',inplace=True )
model = XGBClassifier(n_estimators = num_rounds,
objective= 'binary:logistic',
learning_rate=0.01,
scale_pos_weight = imbalance_weight,
random_state=rstate,
scoring=scoring,
verbose_eval=10)
 model.fit(X_train,y_train, early_stopping_rounds=50, eval_set=[(X_test,
y_test)], verbose=False)
 score = cross_val_score(model, X_train,y_train, cv=cv)
 print("
xgBoost - CV Train : %.2f" % score.mean())
 print("xgBoost - Train : %.2f" % metrics.accuracy_score(model.predict(X_train), y_train))
print("xgBoost - Test : %.2f" % metrics.accuracy_score(model.predict(X_test), y_test))
norm_save(model,score, "XGBsklearn" )<train_model>
print("Number of null values in Color column "+str(train_data['Color'].isnull().sum()))
print("Number of null values in Color column "+str(test_data['Color'].isnull().sum()))
xgtrain = xgb.DMatrix(X_train, label=y_train)
 xgtest = xgb.DMatrix(X_test, label=y_test)
 xgtesting = xgb.DMatrix(test_df)
 param = {'eta': 0.05,
'max_depth': 8,
'subsample': 0.8,
'colsample_bytree': 0.75,
'objective': 'binary:logistic',
'eval_metric': 'logloss',
'scale_pos_weight': imbalance_weight,
'seed': 23,
'lambda': 1.5,
'alpha':.6
}
clf_xgb_cv = xgb.cv(param, xgtrain, num_rounds,
stratified=True,
nfold=n_splits,
early_stopping_rounds=50,
verbose_eval=20)
 print("Optimal number of trees/estimators is %i" % clf_xgb_cv.shape[0])
 watchlist = [(xgtrain,'train'),(xgtest,'test')]
clf_xgb = xgb.train(param, xgtrain,clf_xgb_cv.shape[0], watchlist,verbose_eval=0)
 p = clf_xgb.predict(xgtest)
 fpr, tpr, thresholds = metrics.roc_curve(y_test,p)
 optimal_idx = np.argmax(tpr - fpr)
 optimal_threshold = thresholds[optimal_idx]
print("
Optimal Threshold: ",optimal_threshold)
 y_train_pred =(clf_xgb.predict(xgtrain, ntree_limit=clf_xgb.best_iteration)> optimal_threshold ).astype(int)
 y_test_pred =(clf_xgb.predict(xgtest, ntree_limit=clf_xgb.best_iteration)> optimal_threshold ).astype(int)
 score= metrics.accuracy_score(y_test_pred, y_test)
 print("
XGB - CV Train : %.2f" % score)
 train_score= metrics.accuracy_score(y_train_pred, y_train)
 print("XGB - Train : %.2f" % train_score)
 results = results.append({'Model': "BESTXGBOOST",'Para': param,'Test_Score': score,
'CV Mean':train_score, 'CV STDEV': None}, ignore_index=True)
 xgboostprobpred = clf_xgb.predict(xgtesting, ntree_limit=clf_xgb.best_iteration)
 xgboostpred =(xgboostprobpred > optimal_threshold ).astype(int)
 submission = pd.DataFrame({'PassengerId':test_df.index,'Survived':xgboostpred})
 submission.to_csv('BESTXGBOOST.csv',index=False )<choose_model_class>
train_data['Transmission'].value_counts()
model = CatBoostClassifier(
 eval_metric='Logloss',
iterations = num_rounds,
use_best_model=True,
od_wait = 100,
random_seed=42,
logging_level = "Verbose",
metric_period = 100
)<train_model>
test_data['Transmission'].value_counts()
model = CatBoostClassifier(eval_metric='Accuracy',
iterations = 500,
scale_pos_weight= imbalance_weight,
random_seed=42,
logging_level = "Verbose",
metric_period = 100 
)
 model.fit(X,y,cat_features=categorical_features_pos )<find_best_params>
train_data[train_data['Transmission']=='Manual']
model.get_params()<create_dataframe>
train_data['Transmission'].replace("Manual","MANUAL",inplace=True )
lgtrain = lgb.Dataset(X_train, y_train,
categorical_feature=categorical_features 
)
 lgvalid = lgb.Dataset(X_test, y_test,
categorical_feature=categorical_features 
)
 lgbm_params = {
'boosting_type': 'gbdt',
'objective': 'binary',
'metric': 'auc',
'max_depth': 12,
'learning_rate': 0.01,
'feature_fraction': 0.80,
'bagging_fraction': 0.80,
'bagging_freq': 5,
'max_bin':300,
'lambda_l2': 1.5,
'is_unbalance': True
}
modelstart = time.time()
lgb_clf = lgb.train(
 lgbm_params,
lgtrain,
num_boost_round=num_rounds,
valid_sets=[lgtrain,lgvalid],
valid_names=['train','valid'],
early_stopping_rounds=50,
verbose_eval=50 
)
 print("
Model Runtime: %0.2f Minutes"%(( time.time() - modelstart)/60))<compute_train_metric>
train_data['Transmission'].value_counts()
p = lgb_clf.predict(X_test)
 fpr, tpr, thresholds = metrics.roc_curve(y_test,p)
 optimal_idx = np.argmax(tpr - fpr)
 optimal_threshold = thresholds[optimal_idx]
print("Optimal Threshold: ",optimal_threshold)
 y_train_pred =(lgb_clf.predict(X_train)> optimal_threshold ).astype(int)
 y_test_pred =(lgb_clf.predict(X_test)> optimal_threshold ).astype(int)
 score= metrics.accuracy_score(y_test_pred, y_test)
 print("
XGB - CV Train : %.2f" % score)
 train_score = metrics.accuracy_score(y_train_pred, y_train)
 print("XGB - Train : %.2f" % train_score)
 results = results.append({'Model': "LGBM",'Para': lgbm_params,'Test_Score': score,
'CV Mean':train_score, 'CV STDEV': None}, ignore_index=True)
 lgbmprobpred = lgb_clf.predict(test_df)
 lgbmpred =(lgbmprobpred > optimal_threshold ).astype(int)
 submission = pd.DataFrame({'PassengerId':test_df.index,'Survived':lgbmpred})
 submission.to_csv('LGBM.csv',index=False )<save_to_csv>
train_data['Transmission'].fillna(value="Transmission_unk",inplace=True)
 test_data['Transmission'].fillna(value="Transmission_unk",inplace=True )
modeblend = pd.DataFrame([xgboostpred,lgbmpred, lgbmpred] ).T
modeblend.head()
modeblend["Mode"] = modeblend.mode(axis=1)
 submission = pd.DataFrame({'PassengerId':test_df.index,'Survived':modeblend["Mode"]})
 submission.to_csv('Mode_GBM_Blend.csv',index=False)
 submission.head()<save_to_csv>
train_data['WheelTypeID'].value_counts()
predblend = pd.DataFrame(np.array([xgboostprobpred,lgbmprobpred,lgbmpred])).T
predblend.columns = ["XG","LGBM","CAT"]
predblend["meanblend"] = predblend.mean(axis=1)
 predblend["blendout"] =(predblend["meanblend"] > 0.5 ).astype(int)
 submission = pd.DataFrame({'PassengerId':test_df.index,'Survived':predblend["blendout"]})
 submission.to_csv('mean_GBM_blend.csv',index=False)
 submission.head()<choose_model_class>
train_data.drop('WheelTypeID',inplace=True,axis=1)
 test_data.drop('WheelTypeID',inplace=True,axis=1 )
cv = StratifiedShuffleSplit(n_splits=n_splits, test_size=0.2, random_state=rstate )<compute_train_metric>
train_data['WheelType'].value_counts()
model= LogisticRegression()
score = cross_val_score(model, X_train, y_train, cv=cv, scoring='accuracy')
 norm_save(LogisticRegression() ,score, "Logistic_Regression" )<train_on_grid>
test_data['WheelType'].value_counts()
logit = sm.OLS(y,sm.add_constant(X)).fit()
logit.summary()<choose_model_class>
train_data['WheelType'].fillna(value='WheelType_unk',inplace=True)
 test_data['WheelType'].fillna(value='WheelType_unk',inplace=True )
param_grid ={'max_iter': np.logspace(1, 5, 10 ).astype("int32"),
'hidden_layer_sizes': np.logspace(2, 3, 4 ).astype("int32"),
'activation':['identity', 'logistic', 'tanh', 'relu'],
'learning_rate': ['adaptive'],
'early_stopping': [True],
'alpha': np.logspace(2, 3, 4 ).astype("int32")
 }
model = MLPClassifier()
grid = RandomizedSearchCV(model,
param_grid, cv=cv, scoring=scoring,
verbose=1, n_iter=n_iter, random_state=rstate)
 grid.fit(X_train, y_train)
 save(grid, "FFNeural_Net" )<compute_train_metric>
train_data['WheelType'].value_counts()
model = LinearSVC()
scores= cross_val_score(model, X_train, y_train, cv=cv, scoring=scoring)
 norm_save(model, scores, "LinearSV" )<train_on_grid>
train_data['Nationality'].value_counts()
svc = SVC(kernel= 'rbf', probability=True)
 model = Pipeline(steps=[('svc', svc)])
 param_grid = {'svc__C': st.randint(1,10000),
'svc__gamma': np.logspace(1, -7, 10)}
grid = RandomizedSearchCV(model, param_grid,
cv=cv, verbose=1, scoring=scoring,
n_iter=n_iter, random_state=rstate)
 grid.fit(X_train, y_train)
 save(grid, "SVCrbf" )<choose_model_class>
test_data['Nationality'].value_counts()
pca = PCA()
svc = SVC(kernel= 'rbf',probability=True)
 model = Pipeline(steps=[('pca',pca), 
('svc', svc)])
 param_grid = {'svc__C': st.randint(1,10000),
'svc__gamma': np.logspace(1, -7, 10),
'pca__n_components': st.randint(1,len(X.columns)) }
grid = RandomizedSearchCV(model, param_grid,
cv=cv, verbose=1,
n_iter=n_iter, random_state=rstate, scoring=scoring)
 grid.fit(X_train, y_train)
 save(grid, "PCA_SVC" )<compute_train_metric>
train_data['Nationality'].fillna(value='Nationality_unk',inplace=True)
 test_data['Nationality'].fillna(value='Nationality_unk',inplace=True )
model = GaussianNB()
score = cross_val_score(model, X_train, y_train, cv=cv, scoring=scoring)
 norm_save(model,score, "Gaussian" )<sort_values>
train_data['Size'].value_counts()
results.sort_values(by=["Test_Score"], ascending=False, inplace=True)
 results<create_dataframe>
test_data['Size'].value_counts()
ignore = ["Catboost","BESTXGBOOST","LGBM", "Logistic GAM"]
not_proba_list = ['LinearSV','StochasticGradientDescent']
not_proba = results.query("Model in @not_proba_list")
 hard_models = results.query("Model not in @ignore")
 prob_models = results.query("Model not in @not_proba_list & Model not in @ignore")
 test_hard_pred_matrix = pd.DataFrame()
train_hard_pred_matrix = pd.DataFrame()
valid_hard_pred_matrix = pd.DataFrame()
test_soft_pred_matrix = pd.DataFrame()
train_soft_pred_matrix = pd.DataFrame()
valid_soft_pred_matrix = pd.DataFrame()<compute_train_metric>
train_data['Size'].fillna(value='Size_unk',inplace=True)
 test_data['Size'].fillna(value="Size_unk",inplace=True )
models = list(zip([ensemble_models[x] for x in not_proba.Model], not_proba.Model))
clfs = []
print('5-fold cross validation:
')
 for clf, label in models:
scores = cross_val_score(clf, X_train, y_train, cv=cv, scoring=scoring, verbose=0)
 print("Train CV Accuracy: %0.2f(+/- %0.2f)[%s]" %(scores.mean() , scores.std() , label))
md = clf.fit(X_train, y_train)
 valid_hard_pred_matrix = pd.concat(( valid_hard_pred_matrix, pd.DataFrame({label: md.predict(X_test)})) , axis=1)
 clfs.append(md)
 print("Test Accuracy: %0.2f
" %(metrics.accuracy_score(clf.predict(X_test), y_test)))
 md = clf.fit(X,y)
 submission = md.predict(test_df)
 df = pd.DataFrame({'PassengerId':test_df.index,
'Survived':submission})
 train_hard_pred_matrix = pd.concat(( train_hard_pred_matrix, pd.DataFrame({label: md.predict(X)})) , axis=1)
 test_hard_pred_matrix = pd.concat(( test_hard_pred_matrix, pd.DataFrame({label: submission})) , axis=1)
 df.to_csv("{}.csv".format(label),header=True,index=False)
 del clfs<find_best_model_class>
train_data['TopThreeAmericanName'].value_counts()
warnings.filterwarnings(action='ignore', category=DeprecationWarning)
 models = list(zip([ensemble_models[x] for x in prob_models.Model], prob_models.Model))
plt.figure()
print('5-fold cross validation:
')
 clfs = []
for clf, model_label in models:
scores = cross_val_score(clf, X_train, y_train,cv=5, scoring=scoring, verbose=0)
 print("Train CV Accuracy: %0.2f(+/- %0.2f)[%s]" %(scores.mean() , scores.std() , model_label))
md = clf.fit(X_train, y_train)
 valid_soft_pred_matrix = pd.concat(( valid_soft_pred_matrix, pd.DataFrame({model_label: md.predict_proba(X_test)[:,1]})) , axis=1)
 fpr, tpr, _ = roc_curve(y_test, md.predict_proba(X_test)[:,1])
 roc_auc = auc(fpr, tpr)
 print('ROC AUC: %0.2f' % roc_auc)
 plt.plot(fpr, tpr, label='{} ROC curve(area = {:.2})'.format(model_label, roc_auc))
clfs.append(md)
 print("Test Accuracy: %0.2f
" %(metrics.accuracy_score(clf.predict(X_test), y_test)))
 md = clf.fit(X,y)
 submission = md.predict(test_df)
 df = pd.DataFrame({'PassengerId':test_df.index,
'Survived':submission})
 train_hard_pred_matrix = pd.concat(( train_hard_pred_matrix, pd.DataFrame({model_label: md.predict(X)})) , axis=1)
 test_hard_pred_matrix = pd.concat(( test_hard_pred_matrix, pd.DataFrame({model_label: submission})) , axis=1)
 train_soft_pred_matrix = pd.concat(( train_soft_pred_matrix, pd.DataFrame({model_label: md.predict_proba(X)[:,1]})) , axis=1)
 test_soft_pred_matrix = pd.concat(( test_soft_pred_matrix, pd.DataFrame({model_label: md.predict_proba(test_df)[:,1]})) , axis=1)
 df.to_csv("{}.csv".format(model_label),header=True,index=False)
 plt.plot([0, 1], [0, 1], 'k--')
 plt.xlim([0.0, 1.0])
 plt.ylim([0.0, 1.05])
 plt.xlabel('False Positive Rate')
 plt.ylabel('True Positive Rate')
 plt.title('ROC Curve')
 plt.legend(loc="lower right")
 plt.show()<define_search_model>
test_data['TopThreeAmericanName'].value_counts()
plt.figure()
voters = {}
for x in [2,3,5,7,10]:
ECH = EnsembleVoteClassifier([ensemble_models.get(key)for key in hard_models.Model[:x]], voting='hard')
 ECS = EnsembleVoteClassifier([ensemble_models.get(key)for key in prob_models.Model[:x]], voting='soft')
 print('
{}-Voting Models: 5-fold cross validation:
'.format(x))
for clf, model_label in zip([ECS, ECH],
['{}-VM-Ensemble Soft Voting'.format(x),
'{}-VM-Ensemble Hard Voting'.format(x)]):
scores = cross_val_score(clf, X_train, y_train, cv=5, scoring='accuracy')
 print("Train CV Accuracy: %0.2f(+/- %0.2f)[%s]" %(scores.mean() , scores.std() , model_label))
md = clf.fit(X_train, y_train)
 clfs.append(md)
 Test_Score = metrics.accuracy_score(clf.predict(X_test), y_test)
 print("Test Accuracy: %0.2f " % Test_Score)
 CV_Score = scores.mean()
STDev = scores.std()
global results
results = results.append({'Model': model_label,'Para': clf, 'CV Mean': CV_Score,
'Test_Score':Test_Score,'CV STDEV': STDev}, ignore_index=True)
 voters[model_label] = clf
md = clf.fit(X,y)
 submission = md.predict(test_df)
 df = pd.DataFrame({'PassengerId':test_df.index,'Survived':submission})
 df.to_csv("{}.csv".format(model_label),header=True,index=False)
 if clf is ECH:
train_hard_pred_matrix = pd.concat(( train_hard_pred_matrix, pd.DataFrame({model_label: md.predict(X)})) , axis=1)
 test_hard_pred_matrix = pd.concat(( test_hard_pred_matrix, pd.DataFrame({model_label: submission})) , axis=1)
 elif clf is ECS:
fpr, tpr, _ = roc_curve(y_test, md.predict_proba(X_test)[:,1])
 roc_auc = auc(fpr, tpr)
 print('ROC AUC: %0.2f' % roc_auc)
 plt.plot(fpr, tpr, label='{} ROC curve(area = {:.2})'.format(model_label, roc_auc))
train_soft_pred_matrix = pd.concat(( train_soft_pred_matrix, pd.DataFrame({model_label: md.predict_proba(X)[:,1]})) , axis=1)
 test_soft_pred_matrix = pd.concat(( test_soft_pred_matrix, pd.DataFrame({model_label: md.predict_proba(test_df)[:,1]})) , axis=1)
 plt.plot([0, 1], [0, 1], 'k--')
 plt.xlim([0.0, 1.0])
 plt.ylim([0.0, 1.05])
 plt.xlabel('False Positive Rate')
 plt.ylabel('True Positive Rate')
 plt.title('Soft Model ROC Curve')
 plt.legend(loc="lower right")
 plt.show()<set_options>
train_data['TopThreeAmericanName'].fillna(value='Top_unk',inplace=True)
 test_data['TopThreeAmericanName'].fillna(value='Top_unk',inplace=True )
voters.get('10-VM-Ensemble Hard Voting' )<create_dataframe>
train_data['PRIMEUNIT'].value_counts()
Xstack = X.copy()
ystack = y.copy()
X_trainstack = X_train.copy()
X_teststack = X_test.copy()
y_trainstack = y_train.copy()
y_teststack = y_test.copy()<import_modules>
train_data['PRIMEUNIT'].fillna(value="Prime_unk",inplace=True)
 test_data['PRIMEUNIT'].fillna(value="Prime_unk",inplace=True )
from sklearn.model_selection import cross_val_score<find_best_model_class>
train_data['AUCGUART'].replace("AGREEN","GREEN",inplace=True)
 test_data['AUCGUART'].replace("ARED","RED",inplace=True)
 train_data['AUCGUART'].fillna(value="AUC_unk",inplace=True)
 test_data['AUCGUART'].fillna(value="AUC_unk",inplace=True )
num_trees = 10
verbose = True
clfs = [ensemble_models.get('KNN'),
ensemble_models.get('Sci_kit XGB')]
dataset_blend_train = np.zeros(( X_trainstack.shape[0], len(clfs)))
 dataset_blend_test = np.zeros(( X_teststack.shape[0], len(clfs)))
 dataset_blend_test_df = np.zeros(( test_df.shape[0], len(clfs)))
 print('5-fold cross validation:')
 for i, clf in enumerate(clfs):
scores = cross_val_score(clf, X_trainstack, y_trainstack, cv=4, scoring='accuracy')
 print("
print("Train CV Accuracy: %0.2f(+/- %0.2f)" %(scores.mean() , scores.std()))
clf.fit(X_trainstack, y_trainstack)
 print("Train Accuracy: %0.2f " %(metrics.accuracy_score(clf.predict(X_trainstack), y_trainstack)))
 dataset_blend_train[:,i] = clf.predict_proba(X_trainstack)[:, 1]
dataset_blend_test[:,i] = clf.predict_proba(X_teststack)[:, 1]
dataset_blend_test_df[:,i] = clf.predict_proba(test_df)[:, 1]
print("Test Accuracy: %0.2f
" %(metrics.accuracy_score(clf.predict(X_teststack), y_teststack)))
 print("
clf = LogisticRegression()
scores = cross_val_score(clf, dataset_blend_train, y_trainstack, cv=4, scoring=scoring)
 clf.fit(dataset_blend_train, y_trainstack)
 print("Train CV Accuracy: %0.2f(+/- %0.2f)" %(scores.mean() , scores.std()))
print("Train Accuracy: %0.2f " %(metrics.accuracy_score(clf.predict(dataset_blend_train), y_trainstack)))
 print("Test Accuracy: %0.2f " %(metrics.accuracy_score(clf.predict(dataset_blend_test), y_teststack)))
 if save == True:
pd.DataFrame({'PassengerId':test_df.index,
'Survived':clf.predict(dataset_blend_test_df)} ).to_csv(
 "{}.csv".format("Stacked"),header=True,index=False )<compute_train_metric>
train_data.drop(['MMRAcquisitionAuctionAveragePrice','MMRAcquisitionAuctionCleanPrice',
'MMRAcquisitionRetailAveragePrice','MMRAcquisitonRetailCleanPrice',
'MMRCurrentAuctionAveragePrice','MMRCurrentAuctionCleanPrice',
'MMRCurrentRetailAveragePrice','MMRCurrentRetailCleanPrice'],
inplace=True,axis=1)
 test_data.drop(['MMRAcquisitionAuctionAveragePrice','MMRAcquisitionAuctionCleanPrice',
'MMRAcquisitionRetailAveragePrice','MMRAcquisitonRetailCleanPrice',
'MMRCurrentAuctionAveragePrice','MMRCurrentAuctionCleanPrice',
'MMRCurrentRetailAveragePrice','MMRCurrentRetailCleanPrice'],
inplace=True,axis=1)
 train_data.drop('PurchDate',axis=1,inplace=True)
 test_data.drop("PurchDate",axis=1,inplace=True )
score = cross_val_score(clf, X, y, cv=cv, scoring=scoring)
 norm_save(clf, score, "stacked")
 eval_plot(clf )<concatenate>
train_data.drop(['RefId','IsBadBuy'],axis=1 ).dtypes!='object'
train_hard_pred_matrix = train_hard_pred_matrix.set_index([traindex])
 train_hard_pred_matrix = pd.concat([train_hard_pred_matrix, Survived], axis=1)
 valid_hard_pred_matrix = valid_hard_pred_matrix.set_index([X_test.index])
 valid_hard_pred_matrix = pd.concat([valid_hard_pred_matrix, Survived[X_test.index]], axis=1)
 train_soft_pred_matrix = train_soft_pred_matrix.set_index([traindex])
 train_soft_pred_matrix = pd.concat([train_soft_pred_matrix, Survived], axis=1)
 valid_soft_pred_matrix = valid_soft_pred_matrix.set_index([X_test.index])
 valid_soft_pred_matrix = pd.concat([valid_soft_pred_matrix, Survived[X_test.index]], axis=1)
 test_hard_pred_matrix = test_hard_pred_matrix.set_index([testdex])
 valid_hard_pred_matrix = valid_hard_pred_matrix.set_index([X_test.index])
 test_soft_pred_matrix = test_soft_pred_matrix.set_index([testdex])
 valid_soft_pred_matrix = valid_soft_pred_matrix.set_index([X_test.index])
 test_hard_pred_matrix.to_csv("test_hard_pred_matrix.csv")
 test_soft_pred_matrix.to_csv("test_soft_pred_matrix.csv")
 valid_soft_pred_matrix.to_csv("valid_soft_pred_matrix.csv")
 train_hard_pred_matrix.to_csv("train_hard_pred_matrix.csv")
 train_soft_pred_matrix.to_csv("train_soft_pred_matrix.csv")
 valid_soft_pred_matrix.to_csv("valid_soft_pred_matrix.csv" )<save_to_csv>
not_categorical=train_data.drop(['RefId','IsBadBuy'],axis=1 ).columns[train_data.drop(['RefId','IsBadBuy'],axis=1 ).dtypes!='object']
results.sort_values(by=["Test_Score"], ascending=False, inplace=True)
 results.to_csv("titanic_clf_results.csv", index=False)
 results<sort_values>
not_categorical=train_data.drop(['RefId','IsBadBuy'],axis=1 ).columns[train_data.drop(['RefId','IsBadBuy'],axis=1 ).dtypes!='object']
top_models = [x for x in valid_soft_pred_matrix.columns
if x in list(results.sort_values(by="CV Mean",ascending = False ).Model[:15])]<feature_engineering>
for i in not_categorical:
maximum=np.max(train_data[i])
 train_data[i]=train_data[i]/maximum
maximum_test=np.max(test_data[i])
 test_data[i]=test_data[i]/maximum_test
top_models = [x for x in valid_soft_pred_matrix.columns
if x in list(results.sort_values(by="CV Mean",ascending = False ).Model[:15])]
print("Models Chosen: ",top_models)
 valid_soft_pred_matrix = valid_soft_pred_matrix[top_models]
test_soft_pred_matrix = test_soft_pred_matrix[top_models]
pred_cols = list(valid_soft_pred_matrix.columns)
 def stack_features(tempdf):
tempdf['max'] = np.max(np.array([tempdf[col] for col in pred_cols]),axis=0)
 tempdf['min'] = np.min(np.array([tempdf[col] for col in pred_cols]),axis=0)
 tempdf['avg'] = np.mean(np.array([tempdf[col] for col in pred_cols]),axis=0)
 tempdf['med'] = np.median(np.array([tempdf[col] for col in pred_cols]),axis=0)
 tempdf['std'] = np.std(np.array([tempdf[col] for col in pred_cols]),axis=0)
 for p1, p2 in itertools.combinations(pred_cols, 2):
tempdf['difference_%s__%s'%(p1,p2)] = tempdf[p2] - tempdf[p1]
tempdf['sums_%s__%s'%(p1,p2)] = tempdf[p2] + tempdf[p1]
return tempdf<concatenate>
categorical=train_data.drop(['RefId','IsBadBuy'],axis=1 ).columns[train_data.drop(['RefId','IsBadBuy'],axis=1 ).dtypes=='object']
all_soft_preds = pd.concat([valid_soft_pred_matrix, test_soft_pred_matrix],axis=0)
 all_soft_preds = stack_features(all_soft_preds)
 valid_soft_pred_matrix = all_soft_preds.loc[X_test.index, :]
test_soft_pred_matrix = all_soft_preds.loc[testdex, :]
X_stack = pd.concat([X_test, valid_soft_pred_matrix], axis=1)
 test_stack = pd.concat([test_df, test_soft_pred_matrix],axis=1)
 print("Train Shape: ", X_stack.shape)
 print("Test Shape: ", test_stack.shape)
 y_stack = y[X_test.index]
X_stack_train, X_stack_test, y_stack_train, y_stack_test = train_test_split(X_stack, y_stack, test_size=.30, stratify=y_stack,random_state=23)
 print(X_stack_train.shape, y_stack_train.shape, X_stack_test.shape, y_stack_test.shape)
 lgtrain = lgb.Dataset(data = X_stack_train,label = y_stack_train.values, categorical_feature = categorical_features)
 lgb_results = pd.DataFrame(columns = ["Rounds","Score","STDV", "LB", "Parameters"])
 fulllgtrain = lgb.Dataset(data = X_stack,label = y_stack.values, categorical_feature = categorical_features )<save_to_csv>
train_data[categorical[0]]
model = LogisticRegression()
print("Mean CV Accuracy:",cross_val_score(model, X_stack, y_stack, cv=5, scoring="accuracy" ).mean())
 model.fit(X_stack, y_stack)
 logregpred = model.predict(test_stack)
 submission = pd.DataFrame({'PassengerId':test_stack.index,'Survived':logregpred})
 submission.to_csv('LogRegStack.csv',index=False )<init_hyperparams>
pd.get_dummies(train_data[categorical[0]] )
print("Light Gradient Boosting Classifier: ")
 lgbm_params = {
'task': 'train',
'boosting_type': 'gbdt',
'objective': 'binary',
'metric': ['binary_logloss'],
"learning_rate": 0.05,
"num_leaves": 20,
"max_depth": 5,
'is_unbalance': True
}<find_best_params>
for i in categorical:
dummies=pd.get_dummies(train_data[i])
 dummies.columns=str(i)+'_'+dummies.columns
train_data=pd.concat([train_data,dummies],axis=1)
 train_data.drop(i,inplace=True,axis=1)
 dummies=pd.get_dummies(test_data[i])
 dummies.columns=str(i)+'_'+dummies.columns
test_data=pd.concat([test_data,dummies],axis=1)
 test_data.drop(i,inplace=True,axis=1 )
lgb_cv = lgb.cv(
 params = lgbm_params,
train_set = lgtrain,
num_boost_round=10000,
stratified=True,
nfold = 5,
verbose_eval=25,
seed = 23,
early_stopping_rounds=75,
categorical_feature= categorical_features)
 loss = lgbm_params["metric"][0]
optimal_rounds = np.argmin(lgb_cv[str(loss)+ '-mean'])
 best_cv_score = round(min(lgb_cv[str(loss)+ '-mean']), 7)
 best_std_score = round(lgb_cv[str(loss)+ '-stdv'][optimal_rounds],7)
 print("
Optimal Round: {}
Optimal Score: {} + {}".format(
 optimal_rounds,best_cv_score,best_std_score))
lgb_results = lgb_results.append({"Rounds": optimal_rounds,
"Score": best_cv_score,
"STDV": best_std_score,
"LB": None,
"Parameters": lgbm_params}, ignore_index=True )<sort_values>
for i in train_data.drop('IsBadBuy',axis=1 ).columns:
if i not in test_data.columns:
test_data[i]=np.zeros(len(test_data))
display(lgb_results.sort_values(by="Score",ascending = True))<find_best_params>
for i in test_data.columns:
if i not in train_data.columns:
train_data[i]=np.zeros(len(train_data))
final_model_params = lgb_results.iloc[lgb_results["Score"].idxmin() ,:]["Parameters"]
optimal_rounds = lgb_results.iloc[lgb_results["Score"].idxmin() ,:]["Rounds"]
print("Parameters for Final Models:
",final_model_params)
 print("Score: {} +/- {}".format(lgb_results.iloc[lgb_results["Score"].idxmin() ,:]["Score"],lgb_results.iloc[lgb_results["Score"].idxmin() ,:]["STDV"]))
print("Rounds: ", optimal_rounds )<find_best_model_class>
test_data=test_data[train_data.drop("IsBadBuy",axis=1 ).columns]
print("Find Optimal Cutoff")
 lgb_clf = lgb.train(
 final_model_params,
lgtrain,
num_boost_round = optimal_rounds + 1,
verbose_eval=100,
categorical_feature= categorical_features)
 p = lgb_clf.predict(X_stack_test)
 fpr, tpr, thresholds = metrics.roc_curve(y_stack_test,p)
 optimal_idx = np.argmax(tpr - fpr)
 optimal_threshold = thresholds[optimal_idx]
print("Optimal Threshold: ",optimal_threshold)
 y_stack_train_pred =(lgb_clf.predict(X_stack_train)> optimal_threshold ).astype(int)
 y_stack_test_pred =(lgb_clf.predict(X_stack_test)> optimal_threshold ).astype(int)
 score= metrics.accuracy_score(y_stack_test_pred, y_stack_test)
 print("
LGBM STACKER - CV Train : %.2f" % score )<prepare_output>
X=train_data.drop(['RefId','IsBadBuy'],axis=1)
 y=train_data['IsBadBuy']
allmodelstart= time.time()
all_feature_importance_df = pd.DataFrame()
modelstart= time.time()
lgb_clf = lgb.train(
 final_model_params,
fulllgtrain,
num_boost_round = optimal_rounds + 1,
verbose_eval=100,
categorical_feature= categorical_features)
 fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = X_stack.columns
fold_importance_df["importance"] = lgb_clf.feature_importance()
all_feature_importance_df = pd.concat([all_feature_importance_df, fold_importance_df], axis=0)
 lgbmprobpred = lgb_clf.predict(test_stack)
 lgbmpred =(lgbmprobpred > optimal_threshold ).astype(int)
 submission = pd.DataFrame({'PassengerId':test_stack.index,'Survived':lgbmpred})
 submission.to_csv('LGBM.csv',index=False)
 print("Model Runtime: %0.2f Minutes"%(( time.time() - modelstart)/60))
cols = all_feature_importance_df[["feature", "importance"]].groupby("feature" ).mean().sort_values(
 by="importance", ascending=False)[:50].index
best_features = all_feature_importance_df.loc[all_feature_importance_df.feature.isin(cols)]
plt.figure(figsize=(10,8))
sns.barplot(x="importance", y="feature",
data=best_features.sort_values(by="importance", ascending=False))
plt.title('LightGBM Features')
 plt.tight_layout()
plt.savefig('lgbm_importances.png' )<train_model>
X_train,X_test,y_train,y_test=train_test_split(X,y,random_state=42 )
end = time.time()
print("Notebook took %0.2f minutes to Run"%(( end - start)/60))<set_options>
KNN=KNeighborsClassifier(n_neighbors=11)
 KNN.fit(X_train,y_train )
warnings.filterwarnings('ignore')
 pd.set_option('precision', 3)
 %config InlineBackend.figure_format = 'retina'
sns.set_style('dark')
 mpl.rcParams['axes.labelsize'] = 14
mpl.rcParams['axes.titlesize'] = 15
mpl.rcParams['xtick.labelsize'] = 12
mpl.rcParams['ytick.labelsize'] = 12
mpl.rcParams['legend.fontsize'] = 12
print('Libraries Loaded!' )<load_from_csv>
KNN.score(X_test,y_test )
train_df = pd.read_csv('.. /input/titanic/train.csv')
 test_df = pd.read_csv('.. /input/titanic/test.csv')
 print('Dataframes loaded!')
 print('Training set: {} rows and {} columns'.format(train_df.shape[0], train_df.shape[1]))
print(' Test set: {} rows and {} columns'.format(test_df.shape[0], test_df.shape[1]))<count_duplicates>
predict=KNN.predict(test_data.drop('RefId',axis=1))
all_data = pd.concat([train_df, test_df])
 print('Combined set: {} rows and {} columns'.format(all_data.shape[0], all_data.shape[1]))
print('
Survived?: ')
 all_data['Survived'].value_counts(dropna = False )<drop_column>
Submission=pd.DataFrame(data=predict,columns=['IsBadBuy'])
 Submission.head()
train_df.drop('PassengerId', axis = 1, inplace = True )<define_variables>
Submission['RefId']=test_data['RefId']
Submission.set_index('RefId',inplace=True )
men = train_df[train_df['Sex'] == 'male']
women = train_df[train_df['Sex'] == 'female']<feature_engineering>
Submission.head()
Submission.to_csv('Submission.csv' )