Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<prepare_x_and_y>
import numpy as np
import pandas as pd
import tensorflow as tf
import os
import math
from IPython.display import clear_output
import matplotlib.pyplot as plt
from sklearn.utils import shuffle
folds = get_folds(df=train, n_splits=5)
 train_features = [_f for _f in train.columns if _f not in excluded_features]
print(train_features)
 importances = pd.DataFrame()
oof_reg_preds = np.zeros(train.shape[0])
 sub_reg_preds = np.zeros(test.shape[0])
 for fold_,(trn_, val_)in enumerate(folds):
trn_x, trn_y = train[train_features].iloc[trn_], y_reg.iloc[trn_]
val_x, val_y = train[train_features].iloc[val_], y_reg.iloc[val_]
reg = lgb.LGBMRegressor(
 num_leaves=31,
learning_rate=0.03,
n_estimators=1000,
subsample=.9,
colsample_bytree=.9,
random_state=1 
)
 reg.fit(
 trn_x, np.log1p(trn_y),
eval_set=[(val_x, np.log1p(val_y)) ],
early_stopping_rounds=50,
verbose=100,
eval_metric='rmse' 
)
 imp_df = pd.DataFrame()
imp_df['feature'] = train_features
imp_df['gain'] = reg.booster_.feature_importance(importance_type='gain')
 imp_df['fold'] = fold_ + 1
importances = pd.concat([importances, imp_df], axis=0, sort=False)
 oof_reg_preds[val_] = reg.predict(val_x, num_iteration=reg.best_iteration_)
 oof_reg_preds[oof_reg_preds < 0] = 0
_preds = reg.predict(test[train_features], num_iteration=reg.best_iteration_)
 _preds[_preds < 0] = 0
sub_reg_preds += np.expm1(_preds)/ len(folds)
 mean_squared_error(np.log1p(y_reg), oof_reg_preds)**.5<prepare_output>
training_data_raw = pd.read_csv("/kaggle/input/titanic/train.csv")
 testing_data_raw = pd.read_csv("/kaggle/input/titanic/test.csv")
 print(training_data_raw.head())
 print(testing_data_raw.head() )
train['predictions'] = np.expm1(oof_reg_preds)
 test['predictions'] = sub_reg_preds<groupby>
training_data = training_data_raw[['Sex','Name','Age','Pclass','SibSp','Parch','Fare','Survived']]
testing_data = testing_data_raw[['Sex','Name','Age','Pclass','SibSp','Parch','Fare']]
trn_data = train[train_features + ['fullVisitorId']].groupby('fullVisitorId' ).mean()<groupby>
print(( training_data[['Name','Sex','Age','Pclass','SibSp','Parch','Fare','Survived']].isnull() ).sum())
 print(( testing_data[['Name','Sex','Age','Pclass','SibSp','Parch','Fare']].isnull() ).sum() )
%%time
trn_pred_list = train[['fullVisitorId', 'predictions']].groupby('fullVisitorId')\ 
.apply(lambda df: list(df.predictions)) \ 
.apply(lambda x: {'pred_'+str(i): pred for i, pred in enumerate(x)} )<prepare_output>
def handle_nans(dataframe):
dataframe['HasNAN'] = 0
for column in dataframe:
if dataframe[column].dtypes != 'int32' and dataframe[column].dtypes != 'float32' and \
dataframe[column].dtypes != 'int64' and dataframe[column].dtypes != 'float64':
continue
for row, value in dataframe[column].iteritems() :
if math.isnan(value):
dataframe.loc[row,'HasNAN'] = 1
if dataframe[column].isnull().sum() != 0:
temp = testing_data[column]
median = temp.append(training_data[column] ).median()
dataframe[column] = dataframe[column].fillna(median)
 return dataframe
trn_all_predictions = pd.DataFrame(list(trn_pred_list.values), index=trn_data.index)
 trn_feats = trn_all_predictions.columns
trn_all_predictions['t_mean'] = np.log1p(trn_all_predictions[trn_feats].mean(axis=1))
trn_all_predictions['t_median'] = np.log1p(trn_all_predictions[trn_feats].median(axis=1))
trn_all_predictions['t_sum_log'] = np.log1p(trn_all_predictions[trn_feats] ).sum(axis=1)
 trn_all_predictions['t_sum_act'] = np.log1p(trn_all_predictions[trn_feats].fillna(0 ).sum(axis=1))
trn_all_predictions['t_nb_sess'] = trn_all_predictions[trn_feats].isnull().sum(axis=1)
 full_data = pd.concat([trn_data, trn_all_predictions], axis=1)
 del trn_data, trn_all_predictions
gc.collect()
full_data.shape<groupby>
training_data = handle_nans(training_data)
 testing_data = handle_nans(testing_data)
%%time
sub_pred_list = test[['fullVisitorId', 'predictions']].groupby('fullVisitorId')\ 
.apply(lambda df: list(df.predictions)) \ 
.apply(lambda x: {'pred_'+str(i): pred for i, pred in enumerate(x)} )<prepare_output>
def get_title(name):
first_part_name = name.split(",")[1]
title = first_part_name.split(" ")[1]
return title
def age_class_dummies(dataframe):
dataframe['Sex'] = pd.factorize(dataframe['Sex'])[0]
dataframe['Name'] = dataframe['Name'].apply(get_title)
 dataframe['Name'] = pd.factorize(dataframe['Name'])[0]
dataframe = dataframe.rename(columns={'Name':'Title'})
 dataframe = pd.get_dummies(dataframe, columns=['Title'])
 dataframe = dataframe.astype('int32')
 return dataframe
training_data = age_class_dummies(training_data)
 testing_data = age_class_dummies(testing_data)
 for column in training_data:
if(column not in testing_data)and(column != 'Survived'):
print("Column not in testing data: ", column)
 testing_data[column] = 0
for column in testing_data:
if column not in training_data:
print("Column not in training data: ", column)
 training_data[column] = 0
training_data = training_data.reindex(sorted(training_data.columns), axis=1)
 testing_data = testing_data.reindex(sorted(testing_data.columns), axis=1)
sub_data = test[train_features + ['fullVisitorId']].groupby('fullVisitorId' ).mean()
sub_all_predictions = pd.DataFrame(list(sub_pred_list.values), index=sub_data.index)
 for f in trn_feats:
if f not in sub_all_predictions.columns:
sub_all_predictions[f] = np.nan
sub_all_predictions['t_mean'] = np.log1p(sub_all_predictions[trn_feats].mean(axis=1))
sub_all_predictions['t_median'] = np.log1p(sub_all_predictions[trn_feats].median(axis=1))
sub_all_predictions['t_sum_log'] = np.log1p(sub_all_predictions[trn_feats] ).sum(axis=1)
 sub_all_predictions['t_sum_act'] = np.log1p(sub_all_predictions[trn_feats].fillna(0 ).sum(axis=1))
sub_all_predictions['t_nb_sess'] = sub_all_predictions[trn_feats].isnull().sum(axis=1)
 sub_full_data = pd.concat([sub_data, sub_all_predictions], axis=1)
 del sub_data, sub_all_predictions
gc.collect()
sub_full_data.shape<groupby>
print(len(training_data))
validation_split = int(len(training_data)/ 2)
 def shuffle_data() :
global train_x, train_y, validation_x, validation_y, test_x
shuffled_training_data = shuffle(training_data)
 shuffled_training_data.reset_index(inplace=True, drop=True)
 train_x = shuffled_training_data.drop(columns=['Survived'] ).to_numpy()
train_y = shuffled_training_data['Survived'].to_numpy()
validation_x = train_x[validation_split + 1:]
validation_y = train_y[validation_split + 1:]
train_x = train_x[0:validation_split]
train_y = train_y[0:validation_split]
test_x = testing_data.to_numpy()
train['target'] = y_reg
trn_user_target = train[['fullVisitorId', 'target']].groupby('fullVisitorId' ).sum()<prepare_x_and_y>
from xgboost import XGBRegressor, XGBClassifier
from sklearn.metrics import mean_absolute_error
folds = get_folds(df=full_data[['totals.pageviews']].reset_index() , n_splits=5)
 oof_preds = np.zeros(full_data.shape[0])
 oof_preds1 = np.zeros(full_data.shape[0])
 both_oof = np.zeros(full_data.shape[0])
 sub_preds = np.zeros(sub_full_data.shape[0])
 vis_importances = pd.DataFrame()
for fold_,(trn_, val_)in enumerate(folds):
print("-"* 20 + "Fold :"+str(fold_)+ "-"* 20)
 trn_x, trn_y = full_data.iloc[trn_], trn_user_target['target'].iloc[trn_]
val_x, val_y = full_data.iloc[val_], trn_user_target['target'].iloc[val_]
xg = XGBRegressor(**xgb_params, n_estimators=1000)
 reg = lgb.LGBMRegressor(
 num_leaves=31,
learning_rate=0.03,
n_estimators=1000,
subsample=.9,
colsample_bytree=.9,
random_state=1 
)
 print("-"* 20 + "LightGBM Training" + "-"* 20)
 reg.fit(
 trn_x, np.log1p(trn_y),
eval_set=[(trn_x, np.log1p(trn_y)) ,(val_x, np.log1p(val_y)) ],
eval_names=['TRAIN', 'VALID'],
early_stopping_rounds=50,
eval_metric='rmse',
verbose=100 
)
 print("-"* 20 + "Xgboost Training" + "-"* 20)
 xg.fit(
 trn_x, np.log1p(trn_y),
eval_set=[(trn_x, np.log1p(trn_y)) ,(val_x, np.log1p(val_y)) ],
early_stopping_rounds=50,
eval_metric='rmse',
verbose=100 
)
 imp_df = pd.DataFrame()
imp_df['feature'] = trn_x.columns
imp_df['gain'] = reg.booster_.feature_importance(importance_type='gain')
 imp_df['fold'] = fold_ + 1
vis_importances = pd.concat([vis_importances, imp_df], axis=0, sort=False)
 oof_preds[val_] = reg.predict(val_x, num_iteration=reg.best_iteration_)
 oof_preds1[val_] = xg.predict(val_x)
 oof_preds[oof_preds < 0] = 0
oof_preds1[oof_preds1 < 0] = 0
both_oof[val_] = oof_preds[val_] * 0.6 + oof_preds1[val_] * 0.4
_preds = reg.predict(sub_full_data[full_data.columns], num_iteration=reg.best_iteration_)
 _preds[_preds < 0] = 0
pre = xg.predict(sub_full_data[full_data.columns])
 pre[pre<0]=0
sub_preds +=(_preds / len(folds)) * 0.6 +(pre / len(folds)) * 0.4
print("LGB ", mean_squared_error(np.log1p(trn_user_target['target']), oof_preds)**.5)
 print("XGB ", mean_squared_error(np.log1p(trn_user_target['target']), oof_preds1)**.5)
 print("Combine ", mean_squared_error(np.log1p(trn_user_target['target']), both_oof)**.5 )<define_variables>
loss_log = pd.DataFrame(columns=['loss'])
 best_model = None
best_score = 99
for i in range(0,50):
shuffle_data()
model = XGBClassifier(n_estimators=2000, learning_rate=0.001)
 model.fit(train_x, train_y, early_stopping_rounds=150,
eval_set=[(train_x, train_y)], verbose=False)
 predictions = model.predict(validation_x)
 mae = mean_absolute_error(predictions, validation_y)
 if mae < best_score:
best_score = mae
loss_log.loc[len(loss_log)] = [mae]
loss_log.plot(figsize=[20,4])
 plt.show()
tips = 2<save_to_csv>
test_y = model.predict(test_x)
 test_y = test_y.flatten()
<set_options><EOS> 
final_prediction = pd.DataFrame(columns=['PassengerId','Survived'])
 final_prediction['Survived'] = test_y
final_prediction['PassengerId'] = testing_data_raw['PassengerId']
print(final_prediction.head())
 final_prediction.to_csv('submission.csv', index=False )