Kaggle competition: Titanic - Machine Learning from Disaster
def hyper_parameter_tuning(clf_model,Xtrain,ytrain,parameters):

 precision_scorer = make_scorer(precision_score)
 grid_obj = GridSearchCV(estimator=clf_model, param_grid=parameters, scoring=precision_scorer,cv = 5, n_jobs = -1, verbose = 2)
 grid_obj = grid_obj.fit(Xtrain, ytrain)
 model = grid_obj.best_estimator_
return model 
<choose_model_class>
%matplotlib inline
def recursiveFeature_elimination(df,model):

 features = df.drop('Zielvariable', axis=1)
 label = df['Zielvariable']
print('original dataset shape %s' % Counter(label))
sm = SMOTE(random_state=43,sampling_strategy='auto')
 features_resampled, labels_resampled =sm.fit_resample(features, label)
 print('Resampled dataset shape %s' % Counter(labels_resampled))
clf = model
RFE = RFECV(clf, step=1, cv=5,scoring='accuracy')
 RFE.fit(features_resampled,labels_resampled)
 support=RFE.support_
feature_names = np.array(features.columns.tolist())
 features_to_use= feature_names[support].tolist()
print(len(feature_names))
print(len(features_to_use))
print(features_to_use)
 return features_to_use 
<train_model>
train = pd.read_csv('.. /input/titanic/train.csv')
 test = pd.read_csv('.. /input/titanic/test.csv' )
def run_kfold(model,X_all,y_all):

 kf = StratifiedKFold(n_splits=10, random_state=43, shuffle=True)
 Precisionscore_list = []
accuracyscore_list=[]
F1_score_list= []
fold = 0
for train_index, test_index in kf.split(X_all,y_all):
fold += 1
X_train, X_test = X_all.values[train_index], X_all.values[test_index]
y_train, y_test = y_all.values[train_index], y_all.values[test_index]
model.fit(X_train, y_train)
 predictions = model.predict(X_test)
 precisionscore = precision_score(y_test, predictions)
 accuracy = accuracy_score(y_test, predictions)
 F1score= f1_score(y_test,predictions)
 Precisionscore_list.append(precisionscore)
 accuracyscore_list.append(accuracy)
 F1_score_list.append(F1score)
 print("Fold {0} precision_score: {1}".format(fold, precisionscore))
print("Fold {0} accuracy_score: {1}".format(fold, accuracy))
print("Fold {0} F1_score: {1}".format(fold, F1score))
mean_precisionscore = np.mean(Precisionscore_list)
 mean_accuracy = np.mean(accuracyscore_list)
 mean_F1= np.mean(F1_score_list)
 print("Mean precision_score: {0}".format(mean_precisionscore))
print("Mean accuracy_score: {0}".format(mean_accuracy))
print("Mean F1_score: {0}".format(mean_F1))
return 
<choose_model_class>
train.set_index('PassengerId', inplace=True)
 test.set_index('PassengerId', inplace=True )
classification_model=LogisticRegression(random_state=1)
 parameters = { 'C': [0.8,0.9,1.0,1.2],
'class_weight': [None, 'balanced'],
'solver': ['liblinear','sag', 'saga'],
}
best_model=hyper_parameter_tuning(classification_model,train_features_all,train_labels_all,parameters)
 features_to_use=recursiveFeature_elimination(df_train,best_model)
 df_filtered_features = df_train[features_to_use]
df_filtered_label =df_train.Zielvariable
run_kfold(best_model,df_filtered_features,df_filtered_label)
 sm_obj = SMOTE(random_state=43,sampling_strategy='auto')
 features_resampled, labels_resampled =sm_obj.fit_resample(df_filtered_features, df_filtered_label)
 best_model.fit(features_resampled,labels_resampled)
 df_test= test_data[features_to_use]
df_test= encode_categorical_features(df_test)
 if 'Dauer' in df_test.columns:
df_test[['Dauer']] = df_test[['Dauer']].apply(np.sqrt)
 scaler = MinMaxScaler(feature_range=(0, 1))
df_test[['Dauer']] = scaler.fit_transform(df_test[['Dauer']])
 test_features_all = df_test
test_labels_all =test_data.Zielvariable
predictions=best_model.predict(test_features_all)
 prediction_proba =best_model.predict_proba(test_features_all)
 pos_label_proba= prediction_proba[:,0] 
<save_to_csv>
train.drop(['Ticket','Cabin'], axis=1, inplace=True)
 test.drop(['Ticket','Cabin'], axis=1, inplace=True )
y = predictions
pred = pos_label_proba
fpr, tpr, thresholds = metrics.roc_curve(y, pred, pos_label=0)
 AUC_score= metrics.auc(fpr, tpr)
 df_submission=pd.DataFrame()
df_submission['ID']=submission_data['ID']
df_submission['Expected']=pos_label_proba
df_submission.to_csv('Logistc_Regression_prediction.csv',index=False)
<choose_model_class>
train['Embarked'].fillna('S', inplace=True )
classification_model=GradientBoostingClassifier(random_state=1)
 parameters = { 'n_estimators': [100, 200, 300],
'max_features': ['sqrt','auto'],
'max_depth': [2, 3, 5],
}
best_model=hyper_parameter_tuning(classification_model,train_features_all,train_labels_all,parameters)
 features_to_use=recursiveFeature_elimination(df_train,best_model)
 df_filtered_features = df_train[features_to_use]
df_filtered_label =df_train.Zielvariable
run_kfold(best_model,df_filtered_features,df_filtered_label)
 sm_obj = SMOTE(random_state=43,sampling_strategy='auto')
 features_resampled, labels_resampled =sm_obj.fit_resample(df_filtered_features, df_filtered_label)
 best_model.fit(features_resampled,labels_resampled)
 df_test= test_data[features_to_use]
df_test= encode_categorical_features(df_test)
 if 'Dauer' in df_test.columns:
df_test[['Dauer']] = df_test[['Dauer']].apply(np.sqrt)
 scaler = MinMaxScaler(feature_range=(0, 1))
df_test[['Dauer']] = scaler.fit_transform(df_test[['Dauer']])
 test_features_all = df_test
test_labels_all =test_data.Zielvariable
predictions=best_model.predict(test_features_all)
 prediction_proba =best_model.predict_proba(test_features_all)
 pos_label_proba= prediction_proba[:,0] 
<save_to_csv>
test[test['Fare'].isnull() ]
y = predictions
pred = pos_label_proba
fpr, tpr, thresholds = metrics.roc_curve(y, pred, pos_label=0)
 AUC_score= metrics.auc(fpr, tpr)
 AUC_score
df_submission=pd.DataFrame()
df_submission['ID']=submission_data['ID']
df_submission['Expected']=pos_label_proba
df_submission.to_csv('Gradient_Boosting_prediction.csv',index=False)
<choose_model_class>
test['Fare'].fillna(7.9, inplace=True )
classification_model=DecisionTreeClassifier(random_state=1)
 parameters = { 'class_weight': [None, 'balanced'],
'max_features': ['sqrt','auto','log2'],
'max_depth': [None, 2, 3, 5],
}
best_model=hyper_parameter_tuning(classification_model,train_features_all,train_labels_all,parameters)
 features_to_use=recursiveFeature_elimination(df_train,best_model)
 df_filtered_features = df_train[features_to_use]
df_filtered_label =df_train.Zielvariable
run_kfold(best_model,df_filtered_features,df_filtered_label)
 sm_obj = SMOTE(random_state=43,sampling_strategy='auto')
 features_resampled, labels_resampled =sm_obj.fit_resample(df_filtered_features, df_filtered_label)
 best_model.fit(features_resampled,labels_resampled)
 df_test= test_data[features_to_use]
df_test= encode_categorical_features(df_test)
 if 'Dauer' in df_test.columns:
df_test[['Dauer']] = df_test[['Dauer']].apply(np.sqrt)
 scaler = MinMaxScaler(feature_range=(0, 1))
df_test[['Dauer']] = scaler.fit_transform(df_test[['Dauer']])
 test_features_all = df_test
test_labels_all =test_data.Zielvariable
predictions=best_model.predict(test_features_all)
 prediction_proba =best_model.predict_proba(test_features_all)
 pos_label_proba= prediction_proba[:,0]
y = predictions
pred = pos_label_proba
fpr, tpr, thresholds = metrics.roc_curve(y, pred, pos_label=0)
 AUC_score= metrics.auc(fpr, tpr)
 df_submission=pd.DataFrame()
df_submission['ID']=submission_data['ID']
df_submission['Expected']=pos_label_proba
df_submission.to_csv('Decision_tree_prediction.csv',index=False)
<choose_model_class>
train['Sex'] = preprocessing.LabelEncoder().fit_transform(train['Sex'].values)
 test['Sex'] = preprocessing.LabelEncoder().fit_transform(test['Sex'].values)
 train['Embarked'] = preprocessing.LabelEncoder().fit_transform(train['Embarked'].values)
 test['Embarked'] = preprocessing.LabelEncoder().fit_transform(test['Embarked'].values )
classification_model=RandomForestClassifier(random_state=1)
 parameters = { 'class_weight': [None, 'balanced'],
'n_estimators': [100, 200, 300],
'max_features': ['sqrt','auto'],
'max_depth': [None, 2, 3, 5],
}
best_model=hyper_parameter_tuning(classification_model,train_features_all,train_labels_all,parameters)
 features_to_use=recursiveFeature_elimination(df_train,best_model)
 df_filtered_features = df_train[features_to_use]
df_filtered_label =df_train.Zielvariable
run_kfold(best_model,df_filtered_features,df_filtered_label)
 sm_obj = SMOTE(random_state=43,sampling_strategy='auto')
 features_resampled, labels_resampled =sm_obj.fit_resample(df_filtered_features, df_filtered_label)
 best_model.fit(features_resampled,labels_resampled)
 df_test= test_data[features_to_use]
df_test= encode_categorical_features(df_test)
 if 'Dauer' in df_test.columns:
df_test[['Dauer']] = df_test[['Dauer']].apply(np.sqrt)
 scaler = MinMaxScaler(feature_range=(0, 1))
df_test[['Dauer']] = scaler.fit_transform(df_test[['Dauer']])
 test_features_all = df_test
test_labels_all =test_data.Zielvariable
predictions=best_model.predict(test_features_all)
 prediction_proba =best_model.predict_proba(test_features_all)
 pos_label_proba= prediction_proba[:,0]
y = predictions
pred = pos_label_proba
fpr, tpr, thresholds = metrics.roc_curve(y, pred, pos_label=0)
 AUC_score= metrics.auc(fpr, tpr)
 df_submission=pd.DataFrame()
df_submission['ID']=submission_data['ID']
df_submission['Expected']=pos_label_proba
df_submission.to_csv('Random_Forest_Classifier_prediction.csv',index=False)
<import_modules>
for dataset in train, test:
dataset.loc[ dataset['Fare'] <= 7.91, 'Fare'] = 0
dataset.loc[(dataset['Fare'] > 7.91)&(dataset['Fare'] <= 14.454), 'Fare'] = 1
dataset.loc[(dataset['Fare'] > 14.454)&(dataset['Fare'] <= 31), 'Fare'] = 2
dataset.loc[ dataset['Fare'] > 31, 'Fare'] = 3
dataset['Fare'] = dataset['Fare'].astype(int )
import lightgbm as lgb
from sklearn.model_selection import GroupKFold<load_from_csv>
train["Age"] = train[['Pclass', 'Sex', 'Age']].groupby(['Pclass', 'Sex'] ).transform(lambda x: x.fillna(x.median()))
test["Age"] = test[['Pclass', 'Sex', 'Age']].groupby(['Pclass', 'Sex'] ).transform(lambda x: x.fillna(x.median()))
df = pd.read_csv('.. /input/Training_data.csv')
 df_test = pd.read_csv('.. /input/Testing_data.csv')
 df_sub = pd.read_csv('.. /input/Submission.csv' )<prepare_x_and_y>
for dataset in train, test:
dataset.loc[ dataset['Age'] <= 16, 'Age'] = 0
dataset.loc[(dataset['Age'] > 16)&(dataset['Age'] <= 32), 'Age'] = 1
dataset.loc[(dataset['Age'] > 32)&(dataset['Age'] <= 48), 'Age'] = 2
dataset.loc[(dataset['Age'] > 48)&(dataset['Age'] <= 64), 'Age'] = 3
dataset.loc[ dataset['Age'] > 64, 'Age'] = 4
dataset['Age'] = dataset['Age'].astype(int )
features = df.columns[1:-1]
target = df.target<count_unique_values>
train.drop('Name', axis=1, inplace=True)
 test.drop('Name', axis=1, inplace=True )
def get_folds(df=None, n_splits=5, index = 'index'):

 unique_vis = np.array(sorted(df[index].unique()))
folds = GroupKFold(n_splits=n_splits)
 fold_ids = []
ids = np.arange(df.shape[0])
 for trn_vis, val_vis in folds.split(X=unique_vis, y=unique_vis, groups=unique_vis):
fold_ids.append(
 [
ids[df[index].isin(unique_vis[trn_vis])],
ids[df[index].isin(unique_vis[val_vis])]
] 
)
 return fold_ids<prepare_x_and_y>
y_train = train["Survived"]
train.drop('Survived', axis=1, inplace=True)
 train.shape, y_train.shape, test.shape
X = df[features]
y = target<init_hyperparams>
random_forest = RandomForestClassifier()
random_forest.fit(train, y_train)
 scores = cross_validate(random_forest, train, y_train, scoring='roc_auc', cv=5, return_train_score=True)
 scores
params = {
'boosting_type': 'gbdt',
'objective': 'binary',
'metric': 'auc',
'num_leaves': 63,
'max_depth': 6,
'learning_rate': 0.03,
"bagging_fraction" : 0.9,
"feature_fraction" : 0.9,
'random_state':1
}
sub_reg_preds = np.zeros(df_test.shape[0])
 folds = get_folds(df = df.reset_index() ,index = 'index', n_splits=5)
 for fold_,(trn_, val_)in enumerate(folds):
print("-"* 20 + "Fold :"+str(fold_+1)+ "-"* 20)
 trn_x, trn_y = pd.DataFrame(X ).iloc[trn_], pd.DataFrame(y ).iloc[trn_]
val_x, val_y = pd.DataFrame(X ).iloc[val_], pd.DataFrame(y ).iloc[val_]
lgb_train = lgb.Dataset(trn_x,label=trn_y)
 lgb_eval = lgb.Dataset(val_x,label=val_y)
 model = lgb.train(params, lgb_train, 10000, valid_sets=[lgb_eval], early_stopping_rounds=50, verbose_eval=100)
 _preds = model.predict(df_test[features], num_iteration=model.best_iteration)
 _preds[_preds < 0] = 0
sub_reg_preds += _preds / len(folds)
 print("
modeling finished!!
" )<save_to_csv>
n_est = [int(x)for x in np.linspace(start = 200, stop = 2000, num = 10)]
max_feat = ['auto', 'sqrt']
max_depth = [int(x)for x in np.linspace(5, 110, num = 11)]
max_depth.append(None)
 min_samples_split = [2, 5, 10]
min_samples_leaf = [1, 2, 4]
bootstrap = [True, False]
random_grid = {'n_estimators': n_est,
'max_features': max_feat,
'max_depth': max_depth,
'criterion' :['gini', 'entropy'],
'min_samples_split': min_samples_split,
'min_samples_leaf': min_samples_leaf,
'bootstrap': bootstrap}
random_grid
df_sub['Probability'] = sub_reg_preds
df_sub.to_csv('sub.csv',index=False )<set_options>
model = RandomForestClassifier(bootstrap=False, criterion='entropy', max_depth=5,
min_samples_leaf=2, min_samples_split=10,
n_estimators=1800)
 model.fit(train, y_train)
 y_pred = model.predict(test )
os.environ['TF_CPP_MIN_LOG_LEVEL']='2'<init_hyperparams>
submission = pd.DataFrame({
"PassengerId": test.index,
"Survived": y_pred
})
 submission.to_csv('./submission.csv', index=False )