Kaggle competition: Titanic - Machine Learning from Disaster
tree_model = {
'DecisionTree': DecisionTreeClassifier() ,
'ExtraTree': ExtraTreeClassifier()
}<train_model>
df_train = pd.read_csv('.. /input/titanic/train.csv')
 df_test = pd.read_csv('.. /input/titanic/test.csv')
 data = df_train.append(df_test )
train_model(tree_model )<predict_on_test>
df_train.isnull().sum()
evaluation.append(preds(tree_model))<install_modules>
df_test.isnull().sum()
!pip install catboost 
<choose_model_class>
df_train.duplicated().sum()
boosting_model = {
'CatBoost': CatBoostClassifier(eval_metric='F1', logging_level='Silent'),
'XGBoost': XGBClassifier() ,
'LGBM': LGBMClassifier() ,
'AdaBoost': AdaBoostClassifier()
}<train_model>
print("Training set size",len(df_train))
print("Test set size",len(df_train))
train_model(boosting_model )<predict_on_test>
knn = KNNImputer(n_neighbors=3)
 data["Age"] = knn.fit_transform(data["Age"].values.reshape(-1,1)).ravel()
evaluation.append(preds(boosting_model))<import_modules>
data["Fare"] = data["Fare"].fillna(data[:891]["Fare"].mode() [0])
 data["Embarked"] = data["Embarked"].fillna(data[:891]["Embarked"].mode() [0] )
from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier<choose_model_class>
data["Ticket_num"] = data["Ticket"].apply(lambda x:"".join(re.findall(r"\d+",x)))
 data["Name_last"] = data["Name"].apply(lambda x:x.split(",",1)[0] )
ensemble_model = {
'RandomForest': RandomForestClassifier() ,
'ExtraTrees': ExtraTreesClassifier()
}<train_model>
Def = 0.5
data["family_survived"] = Def
da_group = data.groupby(["Name_last","Fare"])
 for group,all_data in da_group:
if(len(all_data)>= 1):
for index, row in all_data.iterrows() :
if(row['family_survived'] == 0)or(row['family_survived']== 0.5):
surviv_max = all_data.drop(index)['Survived'].max()
surviv_min = all_data.drop(index)['Survived'].min()
ids = row['PassengerId']
if(surviv_max == 1.0):
data.loc[data['PassengerId'] == ids, 'family_survived'] = 1
elif(surviv_min==0.0):
data.loc[data['PassengerId'] == ids, 'family_survived'] = 0
print("family survival:", data.loc[data['family_survived'] == 1].shape[0])
train_model(ensemble_model )<predict_on_test>
da_group = data[["PassengerId","Name_last","Survived","family","Fare","Ticket_num"]].groupby(["Name_last","Ticket_num"])
 for group,all_data in da_group:
if(len(all_data)>= 1):
for index, row in all_data.iterrows() :
surviv_max = all_data.drop(index)['Survived'].max()
surviv_min = all_data.drop(index)['Survived'].min()
ids = row['PassengerId']
if(surviv_max == 1.0):
data.loc[data['PassengerId'] == ids, 'family_survived'] = 1
elif(surviv_min==0.0):
data.loc[data['PassengerId'] == ids, 'family_survived'] = 0
print("family survival:", data.loc[data['family_survived'] == 1].shape[0] )
evaluation.append(preds(ensemble_model))<sort_values>
da_group = data.groupby(["Ticket_num"])
 for group,all_data in da_group:
if(len(all_data)>= 1):
for index, row in all_data.iterrows() :
if(row['family_survived'] == 0)or(row['family_survived']== 0.5):
surviv_max = all_data.drop(index)['Survived'].max()
surviv_min = all_data.drop(index)['Survived'].min()
ids = row['PassengerId']
if(surviv_max == 1.0):
data.loc[data['PassengerId'] == ids, 'family_survived'] = 1
elif(surviv_min==0.0):
data.loc[data['PassengerId'] == ids, 'family_survived'] = 0
print("family survival:", data.loc[data['family_survived'] == 1].shape[0] )
eval = pd.concat(evaluation ).sort_values('val_f1_score',ascending=False ).reset_index(drop=True)
 eval<import_modules>
data = data.drop(columns=["Name","Ticket","Ticket_num","Name_last"] )
from sklearn.model_selection import GridSearchCV<import_modules>
data["Age"] = pd.qcut(data["Age"],7 )
from sklearn.model_selection import GridSearchCV<define_search_space>
data["Fare"] = pd.qcut(data["Fare"],7 )
params = {
'C': np.logspace(-2,2,20),
'solver': ['liblinear', 'lbfgs'],
'penalty': ['l2']
}<choose_model_class>
data["Cabin"] = data["Cabin"].apply(lambda x:str(x)[0] )
tuning_model = GridSearchCV(estimator = linear_models['LogisticRegression'],
param_grid = params,
verbose=True,
scoring = 'f1',
n_jobs=5 )<predict_on_test>
la = LabelEncoder()
la.fit(data["Sex"])
 data["Sex"] = la.transform(data["Sex"])
 la.fit(data["Age"])
 data["Age"] = la.transform(data["Age"])
 la.fit(data["Fare"])
 data["Fare"] = la.transform(data["Fare"])
 la.fit(data["Embarked"])
 data["Embarked"] = la.transform(data["Embarked"])
 la.fit(data["Cabin"])
 data["Cabin"] = la.transform(data["Cabin"])
 all_col = ["Sex","Age","Fare","Embarked","Pclass","family","family_survived","Cabin"]
for i in all_col:
std = StandardScaler()
std.fit(data[:891].loc[:,i].values.reshape(-1, 1))
data[i] = std.transform(data.loc[:,i].values.reshape(-1, 1))
tuning_model.fit(x, y)
 predict = pd.DataFrame(tuning_model.best_estimator_.predict(new_test))
predict[0].value_counts()<find_best_params>
df_train = data[:891]
df_test = data[891:]
print(tuning_model.best_estimator_)
 print(tuning_model.best_params_ )<prepare_x_and_y>
df_train["Survived"].value_counts(normalize=True )
y_pred = predict[0]<data_type_conversions>
from sklearn.model_selection import StratifiedKFold,cross_val_score,RandomizedSearchCV,GridSearchCV
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier,AdaBoostClassifier,GradientBoostingClassifier,StackingClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.metrics import f1_score,recall_score,precision_score,roc_auc_score
from xgboost import XGBClassifier
<sort_values>
target = df_train.iloc[:,1]
feature = df_train.iloc[:,2:]
stk = StratifiedKFold(n_splits=5,random_state=123,shuffle=True)
 for train_index,test_index in stk.split(feature,target):
x_train,y_train = feature.iloc[train_index],target.iloc[train_index]
x_test,y_test = feature.iloc[test_index],target.iloc[test_index]
def compare(col):
count_diff = 0
for i, row in comparison.iterrows() :
if(row['prediction'] != row[col]):
count_diff += 1
return count_diff<load_from_csv>
gbdt = GradientBoostingClassifier(random_state=123)
 gbdt.fit(x_train,y_train)
 gbdt_pre = gbdt.predict(x_test)
 print("gbdt recall:",round(recall_score(y_test,gbdt_pre),2))
print("gbdt precision:",round(precision_score(y_test,gbdt_pre),2))
print("gbdt f1_score:",round(f1_score(y_test,gbdt_pre),2))
print("gbdt rou_auc_score:",round(roc_auc_score(y_test,gbdt_pre),2))
idx = pd.read_csv('.. /input/seleksidukungaib/test.csv')['idx']<data_type_conversions>
params = {
"max_depth":(2,8),
"learning_rate":(0.01,0.1),
"min_samples_leaf":(1,3),
"n_estimators":(100,300)
 }
gbdt_gd = RandomizedSearchCV(gbdt,param_distributions=params,cv=5,scoring="roc_auc")
 gbdt_gd.fit(x_train,y_train)
 print(gbdt_gd.best_score_)
 print(gbdt_gd.best_estimator_)
 print(gbdt_gd.best_params_ )
submission = pd.DataFrame({
'idx': idx,
'isChurned': y_pred.astype(int)
 } )<count_values>
gbdt_pre = gbdt_gd.predict(x_test)
 print("gbdt_gd recall:",round(recall_score(y_test,gbdt_pre),2))
print("gbdt_gd precision:",round(precision_score(y_test,gbdt_pre),2))
print("gbdt_gd f1_score:",round(f1_score(y_test,gbdt_pre),2))
print("gbdt_gd rou_auc_score:",round(roc_auc_score(y_test,gbdt_pre),2))
submission['isChurned'].value_counts()<save_to_csv>
adaboost = AdaBoostClassifier(random_state=123)
 adaboost.fit(x_train,y_train)
 adaboost_pre = gbdt.predict(x_test)
 print("adaboost recall:",round(recall_score(y_test,adaboost_pre),2))
print("adaboost precision:",round(precision_score(y_test,adaboost_pre),2))
print("adaboost f1_score:",round(f1_score(y_test,adaboost_pre),2))
print("adaboost rou_auc_score:",round(roc_auc_score(y_test,adaboost_pre),2))
submission.to_csv('submission.csv', index=False )<save_to_csv>
params = {
"learning_rate":(0.01,0.2),
"n_estimators":(50,300)
 }
ada_gd = RandomizedSearchCV(adaboost,param_distributions=params,cv=5,scoring="roc_auc")
 ada_gd.fit(x_train,y_train)
 print(ada_gd.best_score_)
 print(ada_gd.best_estimator_)
 print(ada_gd.best_params_ )
submission.to_csv('submission.csv', index=False )<load_from_csv>
ada_pre = ada_gd.predict(x_test)
 print("ada_gd recall:",round(recall_score(y_test,ada_pre),2))
print("ada_gd precision:",round(precision_score(y_test,ada_pre),2))
print("ada_gd f1_score:",round(f1_score(y_test,ada_pre),2))
print("ada_gd rou_auc_score:",round(roc_auc_score(y_test,ada_pre),2))
!kaggle competitions submit -c seleksidukungaib -f submission.csv -m "No Comment"<set_options>
rf = RandomForestClassifier(random_state=123)
 rf.fit(x_train,y_train)
 rf_pre = rf.predict(x_test)
 print("rf recall:",round(recall_score(y_test,rf_pre),2))
print("rf precision:",round(precision_score(y_test,rf_pre),2))
print("rf f1_score:",round(f1_score(y_test,rf_pre),2))
print("rf rou_auc_score:",round(roc_auc_score(y_test,rf_pre),2))
sns.set()
%matplotlib inline
warnings.filterwarnings('ignore')
 %load_ext tensorboard.notebook
pd.set_option('display.max_columns', None)
<load_from_csv>
params = {
"max_depth":(3,10),
"n_estimators":(50,300),
"min_samples_leaf":(1,5),
"min_samples_split":(0.1,0.2),
"min_impurity_decrease":(0,0.2),
}
rf_gd = RandomizedSearchCV(rf,param_distributions=params,cv=5,scoring="roc_auc")
 rf_gd.fit(x_train,y_train)
 print(rf_gd.best_score_)
 print(rf_gd.best_estimator_)
 print(rf_gd.best_params_ )
odf=pd.read_csv('/kaggle/input/equipfails/equip_failures_training_set.csv',index_col=0)
 odft=pd.read_csv('/kaggle/input/equipfails/equip_failures_test_set.csv',index_col=0 )<data_type_conversions>
rfgd_pre = rf_gd.predict(x_test)
 print("rf_gd recall:",round(recall_score(y_test,rfgd_pre),2))
print("rf_gd precision:",round(precision_score(y_test,rfgd_pre),2))
print("rf_gd f1_score:",round(f1_score(y_test,rfgd_pre),2))
print("rf_gd rou_auc_score:",round(roc_auc_score(y_test,rfgd_pre),2))
df=odf.replace({'na':-999999})
 Xt=odft.replace({'na':-999999})
 Xt=Xt.astype(float)
 df=df.astype(float)
 df['target']=df['target'].astype(int )<prepare_x_and_y>
knn = KNeighborsClassifier()
knn.fit(x_train,y_train)
 knn_pre = knn.predict(x_test)
 print("knn recall:",round(recall_score(y_test,knn_pre),2))
print("knn precision:",round(precision_score(y_test,knn_pre),2))
print("knn f1_score:",round(f1_score(y_test,knn_pre),2))
print("knn rou_auc_score:",round(roc_auc_score(y_test,knn_pre),2))
X=df.iloc[:,1:]
y=df.iloc[:,0]
Xn=normalize(X)
 Xtn=normalize(Xt )<split>
params = {'algorithm': ['auto'], 'weights': ['uniform', 'distance'], 'leaf_size': list(range(1,50,5)) ,
'n_neighbors': [3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,20,22]}
knn_gd=GridSearchCV(knn, param_grid = params, verbose=True,
cv=10, scoring = "roc_auc")
 knn_gd.fit(x_train, y_train)
 print(knn_gd.best_score_)
 print(knn_gd.best_estimator_)
 print(knn_gd.best_params_ )
X_train, X_test, y_train, y_test = train_test_split(Xn, y, test_size=0.2, random_state=8 )<import_modules>
knngd_pre = knn_gd.predict(x_test)
 print("knngd_pre recall:",round(recall_score(y_test,knngd_pre),2))
print("knngd_pre precision:",round(precision_score(y_test,knngd_pre),2))
print("knngd_pre f1_score:",round(f1_score(y_test,knngd_pre),2))
print("knngd_pre rou_auc_score:",round(roc_auc_score(y_test,knngd_pre),2))
from sklearn.metrics import f1_score
from sklearn.ensemble import RandomForestClassifier as RFC
from sklearn.model_selection import StratifiedKFold<compute_train_metric>
xgb = XGBClassifier(random_state=123,eval_metric="logloss",use_label_encoder=False)
 xgb.fit(x_train,y_train)
 xgb_pre = xgb.predict(x_test)
 print("xgb recall:",round(recall_score(y_test,xgb_pre),2))
print("xgb precision:",round(precision_score(y_test,xgb_pre),2))
print("xgb f1_score:",round(f1_score(y_test,xgb_pre),2))
print("xgb rou_auc_score:",round(roc_auc_score(y_test,xgb_pre),2))
def cross_validate_random_forest(X, y, Xt, yt, criterion, max_height, n_folds):
train_acc=[]
test_acc=[]
general_acc=[]
all_feature_importance=[]
f1s=[]
f1su=0
skf=StratifiedKFold(n_splits=n_folds, shuffle=True)
 for train_index, test_index in skf.split(X, y):
Xtrain, Xtest = X.iloc[train_index], X.iloc[test_index]
ytrain, ytest = y.iloc[train_index].values.ravel() , y.iloc[test_index].values.ravel()
Xtrain_min, Xtrain_max = Xtrain.min() , Xtrain.max()
Xtrain =(Xtrain - Xtrain_min)/(Xtrain_max-Xtrain_min)
 Xtest =(Xtest - Xtrain_min)/(Xtrain_max-Xtrain_min)
 Xtrain = Xtrain[Xtrain.columns[~(Xtrain_min==Xtrain_max)]]
Xtest = Xtest [Xtest.columns[~(Xtrain_min==Xtrain_max)]]
Xt_min, Xt_max = Xt.min() , Xt.max()
Xt=(Xt - Xt_min)/(Xt_max-Xt_min)
 Xt = Xt[Xt.columns[~(Xt_min==Xt_max)]]
dt=RFC(n_estimators=100,criterion=criterion,max_depth=max_height)
 dt.fit(Xtrain, ytrain)
 pred = dt.predict(Xtest)
 general_acc.append(accuracy_score(ytest,pred))
f1s.append(f1_score(ytest,pred))
f1su+=f1_score(ytest,pred)
 pred = dt.predict(Xtrain)
 train_acc.append(accuracy_score(ytrain,pred))
if Xt is not None and yt is not None:
pred = dt.predict(Xt)
 test_acc.append(accuracy_score(yt,pred))
all_feature_importance.append(dt.feature_importances_)
 return [[np.mean(general_acc),np.std(general_acc)],[np.mean(train_acc),np.std(train_acc)],[np.mean(test_acc),np.std(test_acc)],all_feature_importance,f1s,f1su]<create_dataframe>
params ={
"max_depth":(2,8),
"n_estimators":(100,400),
"learning_rate":(0.5,1),
"gamma":(0,0.2),
"reg_lambda":(0.1,0.3),
"reg_alpha":(0.1,0.3),
"colsample_bytree":(0.8,1),
}
xgb_gd = RandomizedSearchCV(xgb,param_distributions=params,cv=5,scoring="roc_auc")
 xgb_gd.fit(x_train,y_train)
 print(xgb_gd.best_score_)
 print(xgb_gd.best_estimator_)
 print(xgb_gd.best_params_ )
n_folds=10
criterion='gini'
max_height=None
results=cross_validate_random_forest(pd.DataFrame(X_train),pd.DataFrame(y_train),pd.DataFrame(X_test),pd.DataFrame(y_test),criterion, max_height, n_folds )<train_model>
xgbgd_pre = xgb_gd.predict(x_test)
 print("xgb_gd recall:",round(recall_score(y_test,xgbgd_pre),2))
print("xgb_gd precision:",round(precision_score(y_test,xgbgd_pre),2))
print("xgb_gd f1_score:",round(f1_score(y_test,xgbgd_pre),2))
print("xgb_gd rou_auc_score:",round(roc_auc_score(y_test,xgbgd_pre),2))
print('General Accuracy: ',results[0][0], 'STD: ',results[0][1])
 print('Train Accuracy: ',results[1][0])
 print('Test Accuracy: ',results[2][0])
 print('F1 Avg: ',results[-1]/10 )<compute_train_metric>
logstic = LogisticRegression(random_state=123)
 logstic.fit(x_train,y_train)
 logstic_pre = logstic.predict(x_test)
 print("logstic recall:",round(recall_score(y_test,logstic_pre),2))
print("logstic precision:",round(precision_score(y_test,logstic_pre),2))
print("logstic f1_score:",round(f1_score(y_test,logstic_pre),2))
print("logstic rou_auc_score:",round(roc_auc_score(y_test,logstic_pre),2))
my_class = ExtraTreesClassifier(random_state=0)
 my_class.fit(X_train, y_train)
 y_pred= my_class.predict(X_test)
 print('accuracy: {}'.format(accuracy_score(y_test, y_pred)))
 print(f'F1: {f1_score(y_test,y_pred)}')
 confusion_matrix(y_test,y_pred )<compute_test_metric>
prams = {
"C":[0.1,0.2,0.3,0.4,0.5],
"solver":["newton-cg","lbfgs","sag","saga"]
}
logstic_gd=GridSearchCV(logstic, param_grid = prams, verbose=True,
cv=10, scoring = "roc_auc")
 logstic_gd.fit(x_train, y_train)
 print(logstic_gd.best_score_)
 print(logstic_gd.best_estimator_)
 print(logstic_gd.best_params_ )
my_class = AdaBoostClassifier(random_state=0)
 my_class.fit(X_train, y_train)
 y_pred= my_class.predict(X_test)
 print('accuracy: {}'.format(accuracy_score(y_test, y_pred)))
 print(f'F1: {f1_score(y_test,y_pred)}')
 confusion_matrix(y_test,y_pred )<compute_train_metric>
logsticgd_pre = logstic_gd.predict(x_test)
 print("logstic recall:",round(recall_score(y_test,logsticgd_pre),2))
print("logstic precision:",round(precision_score(y_test,logsticgd_pre),2))
print("logstic f1_score:",round(f1_score(y_test,logsticgd_pre),2))
print("logstic rou_auc_score:",round(roc_auc_score(y_test,logsticgd_pre),2))
lg = LogisticRegression(solver='lbfgs', random_state=18)
 lg.fit(X_train, y_train)
 logistic_prediction = lg.predict(X_test)
 score = metrics.accuracy_score(y_test, logistic_prediction)
 print(score)
 confusion_matrix(y_test,logistic_prediction )<train_model>
df_t = df_test.iloc[:,2:]
ids = df_test.iloc[:,0]
df_t
data_dmatrix = xgb.DMatrix(data=Xn,label=y)
 xgc = xgb.XGBClassifier(objective ='reg:logistic', colsample_bytree = 0.15,
learning_rate = 0.1,
max_depth = 20, alpha = 12, n_estimators = 700)
 xgc.fit(X,y )<predict_on_test>
pred_train=xgc.predict(X)
 pred_train.sum()<predict_on_test>
pred_test=xgc.predict(Xt)
 pred_test.sum()<create_dataframe>
sub = pd.DataFrame()
sub["PassengerId"] = ids
sub["Survived"] = knn_gd.predict(df_t)
 sub["Survived"] = sub["Survived"].astype(int)
 sub
<save_to_csv><EOS> 
sub.to_csv("submission.csv",index=None)
 print("End!" )