Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<prepare_output>
plt.style.use('classic')
 %matplotlib inline
sb.set()
warnings.filterwarnings('ignore' )
train['predictions'] = np.expm1(oof_reg_preds)
 test['predictions'] = sub_reg_preds<groupby>
train = pd.read_csv('.. /input/titanic/train.csv')
 test = pd.read_csv('.. /input/titanic/test.csv')
 train_len = len(train)
 data_all = train.append(test, sort=False, ignore_index=True)
 data_all.isnull().sum()
trn_data = train[train_features + ['fullVisitorId']].groupby('fullVisitorId' ).mean()<groupby>
data_all['Age'] = data_all.groupby(['Pclass', 'Sex'])['Age'].apply(lambda x: x.fillna(x.median()))
data_all['Fare'] = data_all.groupby('Pclass')['Fare'].apply(lambda x: x.fillna(x.median()))
data_all['Embarked'] = data_all['Embarked'].fillna('S' )
%%time
trn_pred_list = train[['fullVisitorId', 'predictions']].groupby('fullVisitorId')\ 
.apply(lambda df: list(df.predictions)) \ 
.apply(lambda x: {'pred_'+str(i): pred for i, pred in enumerate(x)} )<feature_engineering>
data_all['Pclass'].replace([1, 2, 3], [3, 2, 1],inplace=True )
%%time
with warnings.catch_warnings() :
warnings.simplefilter("ignore")
 trn_all_predictions = pd.DataFrame(list(trn_pred_list.values), index=trn_data.index)
 trn_feats = trn_all_predictions.columns
trn_all_predictions['t_mean'] = np.log1p(trn_all_predictions[trn_feats].mean(axis=1))
trn_all_predictions['t_median'] = np.log1p(trn_all_predictions[trn_feats].median(axis=1))
trn_all_predictions['t_sum_log'] = np.log1p(trn_all_predictions[trn_feats] ).sum(axis=1)
 trn_all_predictions['t_sum_act'] = np.log1p(trn_all_predictions[trn_feats].fillna(0 ).sum(axis=1))
trn_all_predictions['t_nb_sess'] = trn_all_predictions[trn_feats].isnull().sum(axis=1)
 trn_all_predictions.to_csv('trn_all_predictions.csv', index=False)
 full_data = pd.concat([trn_data, trn_all_predictions], axis=1)
 del trn_data, trn_all_predictions
gc.collect()
full_data.shape<groupby>
for i in data_all:
data_all['Title'] = [x[1].split(".")[0].strip(" ")for x in data_all['Name'].str.split(",")]
print(pd.crosstab(data_all['Title'], data_all['Sex']))
%%time
sub_pred_list = test[['fullVisitorId', 'predictions']].groupby('fullVisitorId')\ 
.apply(lambda df: list(df.predictions)) \ 
.apply(lambda x: {'pred_'+str(i): pred for i, pred in enumerate(x)} )<feature_engineering>
for i in data_all:
data_all['Title'] = data_all['Title'].replace(['Capt', 'Col', 'Don', 'Major', 'Rev', 'Sir', 'Jonkheer'], 'Mr')
 data_all['Title'] = data_all['Title'].replace(['Mlle', 'Ms'], 'Miss')
 data_all['Title'] = data_all['Title'].replace(['Dona', 'Lady', 'the Countess', 'Mme'], 'Mrs')
 for i in np.where(pd.notnull(data_all.Title)&(data_all['Title'] == 'Dr')&(data_all['Sex'] == 'M')) :
data_all.at[i, 'Title'] = 'Mr'
for i in np.where(pd.notnull(data_all.Title)&(data_all['Title'] == 'Dr')&(data_all['Sex'] == 'F')) :
data_all.at[i, 'Title'] = 'Mrs'
bar = sb.barplot(x='Title', y='Survived' , data=data_all )
%%time
with warnings.catch_warnings() :
warnings.simplefilter("ignore")
 sub_data = test[train_features + ['fullVisitorId']].groupby('fullVisitorId' ).mean()
sub_all_predictions = pd.DataFrame(list(sub_pred_list.values), index=sub_data.index)
 for f in trn_feats:
if f not in sub_all_predictions.columns:
sub_all_predictions[f] = np.nan
sub_all_predictions['t_mean'] = np.log1p(sub_all_predictions[trn_feats].mean(axis=1))
sub_all_predictions['t_median'] = np.log1p(sub_all_predictions[trn_feats].median(axis=1))
sub_all_predictions['t_sum_log'] = np.log1p(sub_all_predictions[trn_feats] ).sum(axis=1)
 sub_all_predictions['t_sum_act'] = np.log1p(sub_all_predictions[trn_feats].fillna(0 ).sum(axis=1))
sub_all_predictions['t_nb_sess'] = sub_all_predictions[trn_feats].isnull().sum(axis=1)
 sub_all_predictions.to_csv('sub_all_predictions.csv',index = False)
 sub_full_data = pd.concat([sub_data, sub_all_predictions], axis=1)
 del sub_data, sub_all_predictions
gc.collect()
sub_full_data.shape<groupby>
numeric = ['Pclass', 'Fare', 'FamilySize']
data_num = data_all[numeric]
train['target'] = y_reg
trn_user_target = train[['fullVisitorId', 'target']].groupby('fullVisitorId' ).sum()<init_hyperparams>
categorical = ['SexAge', 'Embarked', 'Sex', 'Title']
data_cat = data_all[categorical]
data_cat = pd.get_dummies(data_cat )
params={'learning_rate': 0.03,
'objective':'regression',
'metric':'rmse',
'num_leaves': 31,
'verbose': 1,
"subsample": 0.99,
"colsample_bytree": 0.99,
"random_state":42,
'max_depth': 15,
'lambda_l2': 0.02085548700474218,
'lambda_l1': 0.004107624022751344,
'bagging_fraction': 0.7934712636944741,
'feature_fraction': 0.686612409641711,
'min_child_samples': 21
}
xgb_params = {
'objective': 'reg:linear',
'booster': 'gbtree',
'learning_rate': 0.02,
'max_depth': 22,
'min_child_weight': 57,
'gamma' : 1.45,
'alpha': 0.0,
'lambda': 0.0,
'subsample': 0.67,
'colsample_bytree': 0.054,
'colsample_bylevel': 0.50,
'n_jobs': -1,
'random_state': 456
}<prepare_x_and_y>
y = train.Survived
df_all = pd.concat([data_num, data_cat], axis = 1)
 features =(data_num.columns ).append(data_cat.columns)
 df_train = df_all[:891]
df_test = df_all[891:]
target = 'Survived'
X = df_train[features]
X_test = df_test[features]
warnings.simplefilter(action='ignore', category=FutureWarning)
 folds = get_folds(df=full_data[['totals.pageviews']].reset_index() , n_splits=5)
 oof_preds = np.zeros(full_data.shape[0])
 sub_preds = np.zeros(sub_full_data.shape[0])
 vis_importances = pd.DataFrame()
for fold_,(trn_, val_)in enumerate(folds):
trn_x, trn_y = full_data.iloc[trn_], trn_user_target['target'].iloc[trn_]
val_x, val_y = full_data.iloc[val_], trn_user_target['target'].iloc[val_]
xg = XGBRegressor(**xgb_params, n_estimators=1000)
 reg = lgb.LGBMRegressor(**params,
n_estimators=1500, 
)
 xg.fit(
 trn_x, np.log1p(trn_y),
eval_set=[(trn_x, np.log1p(trn_y)) ,(val_x, np.log1p(val_y)) ],
early_stopping_rounds=50,
eval_metric='rmse',
verbose=100 
)
 reg.fit(
 trn_x, np.log1p(trn_y),
eval_set=[(trn_x, np.log1p(trn_y)) ,(val_x, np.log1p(val_y)) ],
eval_names=['TRAIN', 'VALID'],
early_stopping_rounds=50,
eval_metric='rmse',
verbose=100 
)
 imp_df = pd.DataFrame()
imp_df['feature'] = trn_x.columns
imp_df['gain'] = reg.booster_.feature_importance(importance_type='gain')
 imp_df['fold'] = fold_ + 1
vis_importances = pd.concat([vis_importances, imp_df], axis=0, sort=False)
 oof_preds[val_] = reg.predict(val_x, num_iteration=reg.best_iteration_)
 oof_preds[oof_preds < 0] = 0
_preds = reg.predict(sub_full_data[full_data.columns], num_iteration=reg.best_iteration_)
 _preds[_preds < 0] = 0
pre = xg.predict(sub_full_data[full_data.columns])
 pre[pre<0]=0
sub_preds +=(_preds / len(folds)) * 0.6 +(pre / len(folds)) * 0.4
mean_squared_error(np.log1p(trn_user_target['target']), oof_preds)**.5<save_to_csv>
kfold = StratifiedKFold(n_splits=5)
 def auc_cv(model):
roc_auc = cross_val_score(model, X, y, scoring='roc_auc', cv=kfold)
 return(roc_auc )
sub_full_data['PredictedLogRevenue'] = sub_preds
sub_full_data[['PredictedLogRevenue']].to_csv('final_sub.csv', index=True )<set_options>
logreg2 = LogisticRegression(C =.4, penalty='l2')
 auc_cv_logreg = auc_cv(logreg2)
 logreg2.fit(X, y)
 print('LogisticRegression CV score min: ' + str(auc_cv_logreg.min())+ ' mean: ' + str(auc_cv_logreg.mean())
 + ' max: ' + str(auc_cv_logreg.max()))
gc.enable()
warnings.simplefilter('ignore')
 sns.set()
%matplotlib inline<load_from_csv>
gbct = GradientBoostingClassifier(n_estimators=1000, learning_rate=.01, max_depth=3, max_features=3, min_samples_split=4,
min_samples_leaf=7, loss='exponential', subsample=.6, random_state=0)
 auc_cv_gbct = auc_cv(gbct)
 gbct.fit(X, y)
 print('GBoost CV score min: ' + str(auc_cv_gbct.min())+ ' mean: ' + str(auc_cv_gbct.mean())
 + ' max: ' + str(auc_cv_gbct.max()))
train_df = pd.read_csv('.. /input/data-flattened/train-flattened/train-flattened.csv', dtype={'fullVisitorId': 'str'})
 test_df = pd.read_csv('.. /input/data-flattened/test-flattened/test-flattened.csv', dtype={'fullVisitorId': 'str'} )<count_unique_values>
def blend_models(X):
return(( logreg2.predict(X)) +(gbct.predict(X)))/2
logreg2_preds = logreg2.predict(X_test)
 gbct_preds = gbct.predict(X_test)
 logreg2_gbct_preds = blend_models(X_test ).astype(int)
 titanic_logreg2_gbct = pd.DataFrame({'PassengerId':test.PassengerId, 'Survived':logreg2_gbct_preds})
 titanic_logreg2_gbct.to_csv('solution.csv', index = False)
 print('Version 3 submitted on', datetime.now() )