Kaggle competition: Titanic - Machine Learning from Disaster
sub = pd.DataFrame()
sub['TransactionID'] = X_test.index
sub['isFraud'] = y_preds
print(sub)
 sub.to_csv("submission_lgb.csv", index=False)
<set_options>
train_path = '/kaggle/input/titanic/train.csv'
gender_path = '/kaggle/input/titanic/gender_submission.csv'
test_path = '/kaggle/input/titanic/test.csv'
train = pd.read_csv(train_path)
 gender = pd.read_csv(gender_path)
 test =pd.read_csv(test_path)
 test['Survived'] = np.empty(( len(test), 0)).tolist()
Total = train.append(test)
 Total.set_index('PassengerId', inplace = True)
 Total
del sub,clf
gc.collect()<choose_model_class>
for column in Total:
if Total[column].isnull().sum() != 0:
print('Missing values in',column,':', Total[column].isnull().sum() )
xgb_model = xgb.XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=1, gamma=10,
learning_rate=0.05, max_delta_step=0, max_depth=5,
min_child_weight=6, missing=None, n_estimators=1000, n_jobs=1,
nthread=None, objective='binary:logistic', random_state=0,
reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,
silent=None, subsample=1, verbosity=1,early_stopping_rounds=10)
 xgb_model.fit(X_temp,y_temp )<compute_train_metric>
Total = Total.drop(['Ticket','Cabin'], axis = 1)
 fare_mode = Total['Fare'].mode().iat[0]
Total['Fare'].fillna(fare_mode,inplace = True)
 embarked_mode = Total['Embarked'].mode().iat[0]
Total['Embarked'].fillna(embarked_mode, inplace = True)
 for column in Total:
if Total[column].isnull().sum() != 0:
print('Missing values in',column,':', Total[column].isnull().sum() )
y_pred = xgb_model.predict_proba(X)[:,1]
print(roc_auc_score(y,y_pred))<save_to_csv>
sex_dummy = pd.get_dummies(Total['Sex'])
 Total = Total.drop('Sex', axis = 1)
 Total = pd.concat([Total, sex_dummy], axis = 1)
 embark_dummy = pd.get_dummies(Total['Embarked'])
 Total = Total.drop('Embarked', axis = 1)
 Total = pd.concat([Total, embark_dummy], axis = 1)
 Total
sub = pd.DataFrame()
sub['TransactionID'] = X_test.index
sub['isFraud'] = y_preds
sub.to_csv("submission_xgboost.csv", index=False )<set_options>
title = lambda x: x.split(',')[1].split('.')[0].strip()
Total['Title']=Total['Name'].map(title)
 Total = Total.drop('Name', axis = 1)
 Total
del sub
gc.collect()<set_options>
title_dummy = pd.get_dummies(Total['Title'])
 title_dummy
title_dummy['Mil'] = title_dummy['Capt']+title_dummy['Col']+title_dummy['Major']
title_dummy = title_dummy.drop(['Capt','Col','Major'], axis = 1)
 title_dummy['Senior Male Honorific'] = title_dummy['Don']+title_dummy['Sir']
title_dummy = title_dummy.drop(['Don','Sir'], axis = 1)
 title_dummy['Senior Female Honorific'] = title_dummy['Dona']+title_dummy['Mme']+title_dummy['Lady']
title_dummy = title_dummy.drop(['Dona','Mme','Lady'], axis = 1)
 title_dummy['Ms+Mlle'] = title_dummy['Ms']+title_dummy['Mlle']
title_dummy = title_dummy.drop(['Ms','Mlle'], axis = 1)
 title_dummy['Fancy'] = title_dummy['Jonkheer']+title_dummy['the Countess']
title_dummy = title_dummy.drop(['Jonkheer','the Countess'], axis = 1)
 Total = Total.drop('Title', axis = 1)
 Total = pd.concat([Total, title_dummy], axis = 1)
 Total
warnings.filterwarnings('ignore' )<define_variables>
Age_Total = Total
Age_Total = Age_Total.dropna()
Age_Total
def nans(df): return df[df.isnull().any(axis=1)]
Age_Total_Predict = nans(Total)
 Age_Total_Predict
def seed_everything(seed=0):
random.seed(seed)
 np.random.seed(seed )<define_variables>
col = list(Age_Total.columns)
 col.remove('Age')
 col.remove('Survived')
 Age_Total[col]
SEED = 42
seed_everything(SEED)
 TARGET = 'isFraud'
START_DATE = datetime.datetime.strptime('2017-11-30', '%Y-%m-%d' )<init_hyperparams>
x = Age_Total[col]
x_test = Age_Total_Predict[col]
y_train = Age_Total['Age']
x_train = StandardScaler().fit(x ).transform(x)
 x_test = StandardScaler().fit(x_test ).transform(x_test )
lgb_params = {
'objective':'binary',
'boosting_type':'gbdt',
'metric':'auc',
'n_jobs':-1,
'learning_rate':0.01,
'num_leaves': 2**8,
'max_depth':-1,
'tree_learner':'serial',
'colsample_bytree': 0.8,
'subsample_freq':1,
'subsample':0.7,
'n_estimators':22000,
'max_bin':384,
'verbose':-1,
'seed': SEED,
'early_stopping_rounds':110,
}<feature_engineering>
lasso = LassoCV(alphas = [0.01, 0.05, 0.1, 0.15,0.5] ).fit(x_train, y_train)
 alpha = 0.1
lasso_tuned = LassoCV(alphas = [0.8*alpha, 0.9*alpha, alpha, 1.1*alpha, 1.2*alpha] ).fit(x_train, y_train )
print('Load Data')
 train_df = pd.read_pickle('.. /input/ieee-data-minification/train_transaction.pkl')
 train_df['DT_M'] = train_df['TransactionDT'].apply(lambda x:(START_DATE + datetime.timedelta(seconds = x)))
 train_df['DT_M'] =(train_df['DT_M'].dt.year-2017)*12 + train_df['DT_M'].dt.month
test_df = train_df[train_df['DT_M']==train_df['DT_M'].max() ].reset_index(drop=True)
 train_df = train_df[train_df['DT_M']<(train_df['DT_M'].max())].reset_index(drop=True)
 print('Shape control:', train_df.shape, test_df.shape )<categorify>
svm_age = svm.SVR(kernel = 'rbf')
 svm_age.fit(x_train, y_train )
for col in list(train_df):
if train_df[col].dtype=='O':
print(col)
 train_df[col] = train_df[col].fillna('unseen_before_label')
 test_df[col] = test_df[col].fillna('unseen_before_label')
 train_df[col] = train_df[col].astype(str)
 test_df[col] = test_df[col].astype(str)
 le = LabelEncoder()
le.fit(list(train_df[col])+list(test_df[col]))
train_df[col] = le.transform(train_df[col])
 test_df[col] = le.transform(test_df[col])
 train_df[col] = train_df[col].astype('category')
 test_df[col] = test_df[col].astype('category' )<define_variables>
ridge = Ridge()
parameters = {'alpha': [1e-15, 1e-10, 1e-8, 1e-4, 1e-3, 1e-2, 1, 5, 10, 20]}
ridge_age = GridSearchCV(ridge, parameters, scoring = 'neg_root_mean_squared_error', cv=5)
 ridge_age.fit(x_train, y_train)
 print(ridge_age.best_params_)
 print(-ridge_age.best_score_ )
rm_cols = [
'TransactionID','TransactionDT',
TARGET,
'DT_M'
]
rm_cols += ['V'+str(i)for i in range(1,340)]
features_columns = [col for col in list(train_df)if col not in rm_cols] 
<define_variables>
print('The average R squared score for the lasso model is:', -cross_val_score(lasso_tuned, x_train, y_train, scoring ='neg_root_mean_squared_error' ).mean())
 print('The average R squared score for the svm model is:', -cross_val_score(svm_age, x_train, y_train, scoring ='neg_root_mean_squared_error' ).mean())
 print('The average R squared score for the ridge model is:', -ridge_age.best_score_)
RESULTS = test_df[['TransactionID',TARGET]]
N_SPLITS = 4<train_model>
predicted_ages = pd.DataFrame(lasso_tuned.predict(x_test), columns=['Age'])
 Age_Total_Predict.drop('Age',axis = 1)
 Age_Total_Predict = Age_Total_Predict.assign(Age = lasso_tuned.predict(x_test ).round())
 Age_Total_Predict['PassengerId'] = Age_Total_Predict.index
Age_Total_Predict = Age_Total_Predict.reset_index(drop=True)
 Age_Total_Predict
X,y = train_df[features_columns], train_df[TARGET]
P = test_df[features_columns]
for n_rounds in [1000,2500,5000, 10000]:
print('
print('No Validation training...', n_rounds, 'boosting rounds')
 corrected_lgb_params = lgb_params.copy()
corrected_lgb_params['n_estimators'] = n_rounds
corrected_lgb_params['early_stopping_rounds'] = None
train_data = lgb.Dataset(X, label=y)
 estimator = lgb.train(
 corrected_lgb_params,
train_data 
)
 RESULTS['no_validation_'+str(n_rounds)] = estimator.predict(P)
 print('AUC score', metrics.roc_auc_score(RESULTS[TARGET], RESULTS['no_validation_'+str(n_rounds)]))
print(' 
<prepare_x_and_y>
Age_Total['PassengerId'] = Age_Total.index
Age_Total = Age_Total.reset_index(drop=True)
 Total_Processed = pd.concat([Age_Total_Predict,Age_Total])
 Total_Processed.set_index('PassengerId', inplace = True)
 Total_Processed = Total_Processed.sort_index()
fix_neg = lambda x: x + 11.148738330664404 if x < 0 else x
Total_Processed['Age'] = Total_Processed['Age'].apply(fix_neg)
 Total_Processed
print('
print('KFold training...')
 folds = KFold(n_splits=N_SPLITS, shuffle=True, random_state=SEED)
 X,y = train_df[features_columns], train_df[TARGET]
P = test_df[features_columns]
RESULTS['kfold'] = 0
for fold_,(trn_idx, val_idx)in enumerate(folds.split(X, y)) :
print('Fold:',fold_+1)
 tr_x, tr_y = X.iloc[trn_idx,:], y[trn_idx]
vl_x, v_y = X.iloc[val_idx,:], y[val_idx]
train_data = lgb.Dataset(tr_x, label=tr_y)
 valid_data = lgb.Dataset(vl_x, label=v_y)
 estimator = lgb.train(
 lgb_params,
train_data,
valid_sets = [train_data, valid_data],
verbose_eval = 1000, 
)
 RESULTS['kfold'] = estimator.predict(P)
 print('AUC score', metrics.roc_auc_score(RESULTS[TARGET], RESULTS['kfold']))
print(' 
<split>
Train_Processed = Total_Processed[:len(train)]
Test_Processed = Total_Processed[len(train):]
print('
print('StratifiedKFold training...')
 folds = StratifiedKFold(n_splits=N_SPLITS, shuffle=True, random_state=SEED)
 X,y = train_df[features_columns], train_df[TARGET]
P = test_df[features_columns]
RESULTS['stratifiedkfold'] = 0
for fold_,(trn_idx, val_idx)in enumerate(folds.split(X, y, groups=y)) :
print('Fold:',fold_+1)
 tr_x, tr_y = X.iloc[trn_idx,:], y[trn_idx]
vl_x, v_y = X.iloc[val_idx,:], y[val_idx]
train_data = lgb.Dataset(tr_x, label=tr_y)
 valid_data = lgb.Dataset(vl_x, label=v_y)
 estimator = lgb.train(
 lgb_params,
train_data,
valid_sets = [train_data, valid_data],
verbose_eval = 1000, 
)
 RESULTS['stratifiedkfold'] += estimator.predict(P)/N_SPLITS
print('AUC score', metrics.roc_auc_score(RESULTS[TARGET], RESULTS['stratifiedkfold']))
print(' 
<feature_engineering>
col = list(Train_Processed.columns)
 col.remove('Survived')
 x_train = Train_Processed[col]
y_train = Train_Processed['Survived']
y_train = y_train.astype('int')
 x_final = Test_Processed[col]
x_train = preprocessing.StandardScaler().fit(x_train ).transform(x_train)
 x_final= preprocessing.StandardScaler().fit(x_final ).transform(x_final )
print('
print('LBO training...')
 train_df['DT_M'] = train_df['TransactionDT'].apply(lambda x:(START_DATE + datetime.timedelta(seconds = x)))
 train_df['DT_M'] =(train_df['DT_M'].dt.year-2017)*12 + train_df['DT_M'].dt.month
main_train_set = train_df[train_df['DT_M']<(train_df['DT_M'].max())].reset_index(drop=True)
 validation_set = train_df[train_df['DT_M']==train_df['DT_M'].max() ].reset_index(drop=True)
 folds = KFold(n_splits=N_SPLITS, shuffle=True, random_state=SEED)
 X,y = main_train_set[features_columns], main_train_set[TARGET]
v_X, v_y = validation_set[features_columns], validation_set[TARGET]
estimators_bestround = []
for fold_,(trn_idx, val_idx)in enumerate(folds.split(X, y)) :
print('Fold:',fold_+1)
 tr_x, tr_y = X.iloc[trn_idx,:], y[trn_idx]
train_data = lgb.Dataset(tr_x, label=tr_y)
 valid_data = lgb.Dataset(v_X, label=v_y)
 estimator = lgb.train(
 lgb_params,
train_data,
valid_sets = [train_data, valid_data],
verbose_eval = 1000, 
)
 estimators_bestround.append(estimator.current_iteration())
 corrected_lgb_params = lgb_params.copy()
corrected_lgb_params['n_estimators'] = int(np.mean(estimators_bestround))
corrected_lgb_params['early_stopping_rounds'] = None
print('
print('Mean Best round:', corrected_lgb_params['n_estimators'])
 X,y = train_df[features_columns], train_df[TARGET]
P = test_df[features_columns]
RESULTS['lbo'] = 0
for fold_,(trn_idx, val_idx)in enumerate(folds.split(X, y)) :
print('Fold:',fold_+1)
 tr_x, tr_y = X.iloc[trn_idx,:], y[trn_idx]
train_data = lgb.Dataset(tr_x, label=tr_y)
 estimator = lgb.train(
 corrected_lgb_params,
train_data 
)
 RESULTS['lbo'] += estimator.predict(P)/N_SPLITS
print('AUC score', metrics.roc_auc_score(RESULTS[TARGET], RESULTS['lbo']))
print('<feature_engineering>
k_score = []
k_range = range(1,16)
 for k in k_range:
neigh = KNeighborsClassifier(n_neighbors = k ).fit(x_train,y_train)
 score = -cross_val_score(neigh, x_train, y_train, scoring ='neg_root_mean_squared_error' ).mean()
k_score.append(score)
 Scores = pd.DataFrame(k_score, columns = ['Score'])
 Scores.index += 1
Scores.sort_values('Score' )
print('
print('GroupKFold timeblocks split training...')
 folds = GroupKFold(n_splits=N_SPLITS)
 train_df['groups'] = train_df['TransactionDT'].apply(lambda x:(START_DATE + datetime.timedelta(seconds = x)))
 train_df['groups'] =(train_df['groups'].dt.year-2017)*12 + train_df['groups'].dt.month
X,y = train_df[features_columns], train_df[TARGET]
split_groups = train_df['groups']
P = test_df[features_columns]
RESULTS['groupkfold_timeblocks'] = 0
for fold_,(trn_idx, val_idx)in enumerate(folds.split(X, y, groups=split_groups)) :
print('Fold:',fold_+1)
 tr_x, tr_y = X.iloc[trn_idx,:], y[trn_idx]
vl_x, v_y = X.iloc[val_idx,:], y[val_idx]
train_data = lgb.Dataset(tr_x, label=tr_y)
 valid_data = lgb.Dataset(vl_x, label=v_y)
 estimator = lgb.train(
 lgb_params,
train_data,
valid_sets = [train_data, valid_data],
verbose_eval = 1000, 
)
 RESULTS['groupkfold_timeblocks'] += estimator.predict(P)/N_SPLITS
print('AUC score', metrics.roc_auc_score(RESULTS[TARGET], RESULTS['groupkfold_timeblocks']))
print('<prepare_x_and_y>
neigh = KNeighborsClassifier(n_neighbors = 8 ).fit(x_train,y_train)
 y_KNN = pd.DataFrame(neigh.predict(x_final), columns = ['Survived'])
 y_KNN = y_KNN.assign(PassengerId = Test_Processed.index)
 cols = y_KNN.columns.tolist()
cols = cols[-1:] + cols[:-1]
y_KNN = y_KNN[cols]
filename = 'Titanic Predictions KNN.csv'
y_KNN.to_csv(filename,index=False)
 print('Saved file: ' + filename )
print('
print('GroupKFold uID split training...')
 folds = GroupKFold(n_splits=N_SPLITS)
 train_df['groups'] = ''
for col in ['card1','card2','card3','card5','addr1','addr2',]:
train_df['groups'] = '_' + train_df[col].astype(str)
 X,y = train_df[features_columns], train_df[TARGET]
split_groups = train_df['groups']
P = test_df[features_columns]
RESULTS['groupkfold_uid'] = 0
for fold_,(trn_idx, val_idx)in enumerate(folds.split(X, y, groups=split_groups)) :
print('Fold:',fold_+1)
 tr_x, tr_y = X.iloc[trn_idx,:], y[trn_idx]
vl_x, v_y = X.iloc[val_idx,:], y[val_idx]
train_data = lgb.Dataset(tr_x, label=tr_y)
 valid_data = lgb.Dataset(vl_x, label=v_y)
 estimator = lgb.train(
 lgb_params,
train_data,
valid_sets = [train_data, valid_data],
verbose_eval = 1000, 
)
 RESULTS['groupkfold_uid'] += estimator.predict(P)/N_SPLITS
print('AUC score', metrics.roc_auc_score(RESULTS[TARGET], RESULTS['groupkfold_uid']))
print('<create_dataframe>
svm_lin = svm.SVC(kernel='linear' ).fit(x_train, y_train)
 print('The average R squared score for the svm_lin is:', cross_val_score(svm_lin, x_train, y_train, scoring ='accuracy' ).mean())
 svm_rbf = svm.SVC(kernel='rbf' ).fit(x_train, y_train)
 print('The average R squared score for the svm_rbf is:', cross_val_score(svm_rbf, x_train, y_train, scoring ='accuracy' ).mean() )
print('
print('Intermediate results...')
 final_df = []
for current_strategy in list(RESULTS.iloc[:,2:]):
auc_score = metrics.roc_auc_score(RESULTS[TARGET], RESULTS[current_strategy])
 final_df.append([current_strategy, auc_score])
 final_df = pd.DataFrame(final_df, columns=['Stategy', 'Result'])
 final_df.sort_values(by=['Result'], ascending=False, inplace=True)
 print(final_df )<feature_engineering>
gammas = [0.01,0.1, 1, 10, 100]
for gamma in gammas:
svm_rbf = svm.SVC(kernel='rbf' ).fit(x_train, y_train)
 print('The average R squared score for the svm_rbf with a gamma of',gamma,' is:', cross_val_score(svm_rbf, x_train, y_train, scoring ='accuracy' ).mean() )
print('
print('LBO full set training...')
 train_df['DT_M'] = train_df['TransactionDT'].apply(lambda x:(START_DATE + datetime.timedelta(seconds = x)))
 train_df['DT_M'] =(train_df['DT_M'].dt.year-2017)*12 + train_df['DT_M'].dt.month
main_train_set = train_df[train_df['DT_M']<(train_df['DT_M'].max())].reset_index(drop=True)
 validation_set = train_df[train_df['DT_M']==train_df['DT_M'].max() ].reset_index(drop=True)
 X,y = main_train_set[features_columns], main_train_set[TARGET]
v_X, v_y = validation_set[features_columns], validation_set[TARGET]
estimators_bestround = []
for current_model in range(3):
print('Model:',current_model+1)
 SEED += 1
seed_everything(SEED)
 corrected_lgb_params = lgb_params.copy()
corrected_lgb_params['seed'] = SEED
train_data = lgb.Dataset(X, label=y)
 valid_data = lgb.Dataset(v_X, label=v_y)
 estimator = lgb.train(
 corrected_lgb_params,
train_data,
valid_sets = [train_data, valid_data],
verbose_eval = 1000, 
)
 estimators_bestround.append(estimator.current_iteration())
 corrected_lgb_params = lgb_params.copy()
corrected_lgb_params['n_estimators'] = int(np.mean(estimators_bestround))
corrected_lgb_params['early_stopping_rounds'] = None
print('
print('Mean Best round:', corrected_lgb_params['n_estimators'])
 X,y = train_df[features_columns], train_df[TARGET]
P = test_df[features_columns]
RESULTS['lbo_full'] = 0
NUMBER_OF_MODELS = 3
for current_model in range(NUMBER_OF_MODELS):
print('Model:',current_model+1)
 SEED += 1
seed_everything(SEED)
 train_data = lgb.Dataset(X, label=y)
 estimator = lgb.train(
 corrected_lgb_params,
train_data 
)
 RESULTS['lbo_full'] += estimator.predict(P)/NUMBER_OF_MODELS
print('AUC score', metrics.roc_auc_score(RESULTS[TARGET], RESULTS['lbo_full']))
print('<create_dataframe>
cs = [0.65,0.67]
for c in cs:
svm_rbf = svm.SVC(kernel='rbf', C=c ).fit(x_train, y_train)
 print('The average R squared score for the svm_rbf with a C of',c,' is:', cross_val_score(svm_rbf, x_train, y_train, scoring ='accuracy' ).mean() )
print('
print('Intermediate results...')
 final_df = []
for current_strategy in list(RESULTS.iloc[:,2:]):
auc_score = metrics.roc_auc_score(RESULTS[TARGET], RESULTS[current_strategy])
 final_df.append([current_strategy, auc_score])
 final_df = pd.DataFrame(final_df, columns=['Stategy', 'Result'])
 final_df.sort_values(by=['Result'], ascending=False, inplace=True)
 print(final_df )<feature_engineering>
svm_rbf = svm.SVC(kernel='rbf' ).fit(x_train, y_train)
 y_SVM = pd.DataFrame(svm_rbf.predict(x_final), columns = ['Survived'])
 y_SVM = y_SVM.assign(PassengerId = Test_Processed.index)
 cols = y_SVM.columns.tolist()
cols = cols[-1:] + cols[:-1]
y_SVM = y_SVM[cols]
filename = 'Titanic Predictions SVM.csv'
y_SVM.to_csv(filename,index=False)
 print('Saved file: ' + filename )
<save_to_csv><EOS> 
print('The average accuracy score for the SVM model is:', cross_val_score(svm_rbf, x_train, y_train, scoring ='accuracy' ).mean())