Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<compute_test_metric>
input_io_dir=".. /input/titanic/"
original_train_data=pd.read_csv(input_io_dir+"train.csv")
 original_test_data=pd.read_csv(input_io_dir+"test.csv")
 print('original_train_data',original_train_data.shape)
 print('original_test_data',original_test_data.shape )
labels = df_submission.columns[1:].tolist()
label_size = len(labels)
 def calc_P_R_AP(y_true, y_pred):
P = [None] * label_size
R = [None] * label_size
AP = np.zeros(label_size)
 for i in range(label_size):
P[i], R[i], _ = precision_recall_curve(y_true[:,i], y_pred[:,i])
 AP[i] = average_precision_score(y_true[:,i], y_pred[:,i])
 df_ap = pd.DataFrame(data=AP, index=labels, columns=['AP'])
 return P, R, AP, df_ap<predict_on_test>
input_io_dir='.. /input/titanic-competition-feature-engineering-1/'
def PrepareDataSets() :
passengerId=pd.read_csv(input_io_dir+"passengerId.csv",header=None)
 train_features=pd.read_csv(input_io_dir+"train_features.csv",header=0)
 train_labels=pd.read_csv(input_io_dir+"train_labels.csv",header=None)
 test_features=pd.read_csv(input_io_dir+"test_features.csv",header=0)
 print('PrepareDataSets: passengerId loaded(%d)'% len(passengerId))
print('PrepareDataSets: train_features loaded(%d)'% len(train_features))
print('PrepareDataSets: train_labels loaded(%d)'% len(train_labels))
print('PrepareDataSets: test_features loaded(%d)'% len(test_features))
return passengerId,train_features,train_labels, test_features
def ModelSelection(clf_list,name_list,train_features,train_labels,scoring='accuracy'):
best_score=0
for clf, name in zip(clf_list,name_list):
scores = model_selection.cross_val_score(clf, train_features.values.astype(float), train_labels.values.ravel().astype(float), cv=10, scoring=scoring)
 print("ModelSelection: Scoring %0.2f +/- %0.2f(%s 95%% CI)" %(scores.mean() , scores.std() *2, name))
reference_score=scores.mean() +scores.std()
if(reference_score>best_score):
best_clf=name
best_score=reference_score
learning_model=clf
print("ModelSelection: Best model - "+best_clf)
 return learning_model
def ConfigureLearningModelsForBinaryClassification() :
xgb_clf = XGBClassifier(n_estimators=100,max_depth=40, random_state=42)
 dt_clf = DecisionTreeClassifier(random_state=42)
 rf_clf = ensemble.RandomForestClassifier(n_estimators=100, random_state=42)
 et_clf = ensemble.ExtraTreesClassifier(n_estimators=100, random_state=42)
 gb_clf = ensemble.GradientBoostingClassifier(n_estimators=100, random_state=42)
 ada_clf = ensemble.AdaBoostClassifier(n_estimators=100, random_state=42)
 svm_clf = svm.LinearSVC(C=0.1,random_state=42)
 lg_clf = LogisticRegression(solver='lbfgs', multi_class='auto', max_iter=400,random_state=42)
 e_clf = ensemble.VotingClassifier(estimators=[('xgb', xgb_clf),('dt', dt_clf),('rf',rf_clf),('et',et_clf),('gbc',gb_clf),('ada',ada_clf),('svm',svm_clf),('lg',lg_clf)])
 clf_list = [xgb_clf, dt_clf, rf_clf, et_clf, gb_clf, ada_clf, svm_clf,lg_clf,e_clf]
name_list = ['XGBoost', 'Decision Trees','Random Forest', 'Extra Trees', 'Gradient Boosted', 'AdaBoost', 'Support Vector Machine', 'LogisticRegression','Ensemble']
return clf_list,name_list
passengerId,train_features,train_labels, test_features=PrepareDataSets()
clf_list,name_list=ConfigureLearningModelsForBinaryClassification()
learning_model=ModelSelection(clf_list,name_list,train_features,train_labels)
def normalize_predict(y):
min_pred = y.min(axis=1 ).reshape(-1,1)
 max_pred = y.max(axis=1 ).reshape(-1,1)
 return(y - min_pred)/(max_pred - min_pred )<choose_model_class>
warnings.filterwarnings("ignore", category=DeprecationWarning)
 def FineTuneLearningModel(learning_model, param_grid, train_features,train_labels,scoring='accuracy'):
grid_search = GridSearchCV(learning_model, param_grid, scoring,cv=10)
 grid_search.fit(train_features.values.astype(float),train_labels.values.ravel().astype(float))
cvres = grid_search.cv_results_
for mean_score,std_score, params in zip(cvres["mean_test_score"], cvres["std_test_score"],cvres["params"]):
print('FineTuneLearningModel:',mean_score,'+-',std_score, params)
 print('FineTuneLearningModel: Best params - '+str(grid_search.best_params_))
return grid_search.best_estimator_
passengerId,train_features,train_labels, test_features=PrepareDataSets()
learning_model = XGBClassifier(objective='binary:logistic')
 print('FineTuneLearningModel: round 1 - booster type')
 print('---------------------------------------------')
 param_grid = [
{'booster':['gbtree','gblinear'],'n_estimators': [10,30,100],'learning_rate':[0.1]}
]
learning_model=FineTuneLearningModel(learning_model, param_grid,train_features,train_labels)
 print('Current model params:'+str(learning_model))
print('FineTuneLearningModel: round 2 - complexity params')
 print('--------------------------------------------------')
 param_grid = [
{'max_depth':range(3,10,2),'min_child_weight':range(1,6,2),'gamma':[i/10.0 for i in range(0,5)]}
]
learning_model=FineTuneLearningModel(learning_model, param_grid,train_features,train_labels)
 print('Current model params:'+str(learning_model))
print('FineTuneLearningModel: round 3 - robustness params')
 print('--------------------------------------------------')
 param_grid = [
{ 'subsample':[1e-5,1e-2,0.1,0.2,0.5,0.8,1], 'colsample_bytree':[1e-5,1e-2,0.1,0.2,0.5,0.8,1]}
]
learning_model=FineTuneLearningModel(learning_model, param_grid,train_features,train_labels)
 print('Current model params:'+str(learning_model))
print('FineTuneLearningModel: round 4 - regularisation')
 print('-----------------------------------------------')
 param_grid = [
{ 'reg_alpha':[1e-5, 1e-2, 0.1, 1, 10,50],'reg_lambda':[0.1,0.5, 1, 2,5,10,50]}
]
learning_model=FineTuneLearningModel(learning_model, param_grid,train_features,train_labels)
 print('Current model params:'+str(learning_model))
print('FineTuneLearningModel: round 5 - reduce learning rate as it prevents overfitting')
 print('--------------------------------------------------------------------------------')
 param_grid = [
{ 'learning_rate': [1e-5,0.01],'n_estimators': [10,100,200,500,1000]}
]
learning_model=FineTuneLearningModel(learning_model, param_grid,train_features,train_labels)
 print('Current model params:'+str(learning_model))
def borrowed_model(pretrained=False, **kwargs):
return Classifier(**kwargs)
 if TRAIN_MODE:
f_score = partial(fbeta, thresh=0.2)
 learn = cnn_learner(
 data,
borrowed_model, pretrained=False,
metrics=[lwlrap],
loss_func=nn.MultiLabelSoftMarginLoss() 
)
 if MIXMATCH_SSL > 0:
if CONTINUOUS_TRAIN:
learn.mixmatch(noisy_data, temperature=0.5, n_augment=MIXMATCH_SSL, alpha=0.4, lambda_u=50., rampup=0)
 else:
learn.mixmatch(noisy_data, temperature=0.5, n_augment=MIXMATCH_SSL, alpha=0.4, lambda_u=10., rampup=1000)
 else:
learn.mixup(alpha=0.4, stack_y=False)
 if CONTINUOUS_TRAIN: learn.load(Path('.. /.. ')/SAVED_WEIGHT)
 learn.unfreeze()<load_from_csv>
def TrainModelAndGeneratePredictionsOnTestSet(learning_model,train_features,train_labels,test_features, threshold=-1):
learning_model.fit(train_features.values.astype(float),train_labels.values.ravel().astype(float))
if threshold==-1:
predictions = learning_model.predict(test_features.values.astype(float))
else:
if hasattr(learning_model,"decision_function"):
y_scores=learning_model.decision_function(test_features.values.astype(float))
else:
y_proba=learning_model.predict_proba(test_features.values.astype(float))
y_scores=y_proba[:,1]
predictions=(y_scores>threshold ).astype(float)
 pred=pd.Series(predictions)
 return pred.apply(lambda x: 1 if x>0 else 0)
 predictions=TrainModelAndGeneratePredictionsOnTestSet(learning_model,train_features,train_labels,test_features)
 print('TrainModelAndGeneratePredictionsOnTestSet: predictions ready' )
if TRAIN_MODE and CONTINUOUS_TRAIN:
df_ap = pd.read_csv(LAST_WEIGHTS/'labels_ap.csv', index_col=0)
 loss_weights = torch.FloatTensor(( 1/df_ap.AP ).values ** 4 ).cuda()
print(loss_weights)
 learn.loss_func = nn.MultiLabelSoftMarginLoss(weight=loss_weights)
 else:
loss_weights = None<find_best_params>
def GenerateOutputFile(passengerId,predictions):
output = pd.DataFrame({ 'PassengerId': passengerId,
'Survived': predictions })
 output.to_csv("output.csv", index=False)
 passengerId = original_test_data['PassengerId']
GenerateOutputFile(passengerId,predictions )
if TRAIN_MODE:
learn.lr_find()
learn.recorder.plot(suggestion=True )<train_model>
training_predictions = pd.DataFrame(learning_model.predict(train_features.values.astype(float)))
 training_predictions.iloc[:,0]=training_predictions.iloc[:,0].astype(int )
if TRAIN_MODE:
gc.collect()
callbacks = [
SaveModelCallback(learn, every='improvement', monitor='lwlrap', name='best'),
]
if CONTINUOUS_TRAIN:
learn.fit_one_cycle(50, slice(1e-7,1e-4), callbacks=callbacks)
 else:
learn.fit_one_cycle(300, 2e-2, callbacks=callbacks )<predict_on_test>
result=original_train_data.join(training_predictions!=train_labels)
<save_model><EOS> 
result.rename(columns={0:'Error'},inplace=True )