Kaggle competition: Titanic - Machine Learning from Disaster
train_df, test_df = reverse(train_df, test_df )<concatenate>
pd.set_option('display.max_columns', 500)
 pd.set_option('display.max_rows', 50)
real_idx = np.sort(np.concatenate(( public, private)))
 real_idx<filter>
import plotly.express as px
import plotly.graph_objects as go
import plotly.figure_factory as ff
from plotly.colors import n_colors
from plotly.subplots import make_subplots
from sklearn.preprocessing import MinMaxScaler
test_real = test_df.iloc[real_idx]
test_real.head()<feature_engineering>
train_data = pd.read_csv("/kaggle/input/titanic/train.csv")
 test_data = pd.read_csv("/kaggle/input/titanic/test.csv")
 train_data["Age"] = train_data["Age"].fillna(train_data.describe() ["Age"]["mean"])
 test_data["Age"] = test_data["Age"].fillna(test_data.describe() ["Age"]["mean"])
 train_data['Age']=train_data['Age'].astype(np.float64)
 train_data["Fare"] = train_data["Fare"].fillna(train_data.describe() ["Fare"]["mean"])
 test_data["Fare"] = test_data["Fare"].fillna(test_data.describe() ["Fare"]["mean"])
 train_data['Fare']=train_data['Fare'].astype(int)
 train_data.head(5 )
%%time
concat_df = pd.concat(( train_df, test_real))
features = train_df.columns[1:]
if debug:
features = features[:3]
for i, feature in enumerate(features):
print('Calculating var_{}'.format(i), end='\r')
 n_dups_dict = concat_df.loc[:, feature].value_counts().to_dict()
train_df.loc[:, 'count_{}'.format(feature)] = [n_dups_dict.get(value, 1)* np.sign(value)for value in train_df.loc[:, feature]]
test_df.loc[:, 'count_{}'.format(feature)] = [n_dups_dict.get(value, 1)* np.sign(value)for value in test_df.loc[:, feature]]
train_df.loc[:, '{}_x_count_{}'.format(feature, feature)] = train_df.loc[:, feature] * np.absolute(train_df.loc[:, 'count_{}'.format(feature)])
 test_df.loc[:, '{}_x_count_{}'.format(feature, feature)] = test_df.loc[:, feature] * np.absolute(test_df.loc[:, 'count_{}'.format(feature)])
 train_df.loc[:, 'count_inverse_{}'.format(feature)] = [(1 / n_dups_dict.get(value, 1)) * np.sign(value)for value in train_df.loc[:, feature]]
test_df.loc[:, 'count_inverse_{}'.format(feature)] = [(1 / n_dups_dict.get(value, 1)) * np.sign(value)for value in test_df.loc[:, feature]]
train_df.loc[:, '{}_x_count_inverse_{}'.format(feature, feature)] = train_df.loc[:, feature] * np.absolute(train_df.loc[:, 'count_inverse_{}'.format(feature)])
 test_df.loc[:, '{}_x_count_inverse_{}'.format(feature, feature)] = test_df.loc[:, feature] * np.absolute(test_df.loc[:, 'count_inverse_{}'.format(feature)])
 print(train_df.shape )<set_options>
women = train_data[train_data.Sex == 'female']["Survived"]
men = train_data[train_data.Sex == 'male']["Survived"]
print("Survival rate for women is {:.2f} and for men is {:.2f}".format(( sum(women)/len(women)) *100,(sum(men)/len(men)) *100))
gc.collect()<init_hyperparams>
data = train_data
data['Died'] = 1 - data['Survived']
lgb_params = {
"objective" : "binary",
"metric" : "auc",
"boosting": 'gbdt',
"max_depth" : -1,
"num_leaves" : 7,
"learning_rate" : 0.01,
"bagging_freq": 5,
"bagging_fraction" : 1,
"feature_fraction" : 0.3,
"min_data_in_leaf": 80,
"min_sum_hessian_in_leaf" : 10,
"tree_learner": "serial",
"boost_from_average": "false",
"bagging_seed" : random_state,
"verbosity" : 1,
"seed": random_state
}<split>
x_train = train_data.drop(['Survived', 'Died'], axis=1)
 y_train = train_data['Survived']
x_test = test_data
df_combined = x_train.append(x_test)
 df_combined.shape
%%time
df_train = train_df.reset_index()
df_test = test_df.reset_index()
if debug:
df_train = df_train[:1000]
df_test = df_test[:1000]
target = target[:1000]
skf = StratifiedKFold(n_splits = 5, shuffle=True, random_state=random_state)
 oof = df_train[['ID_code', 'target']]
oof['predict'] = 0
predictions = df_test[['ID_code']]
val_aucs = []
feature_importance_df = pd.DataFrame()
features = [col for col in df_train.columns if col not in ['target', 'ID_code']]
X_test = df_test[features].values
for fold,(trn_idx, val_idx)in enumerate(skf.split(df_train, target)) :
X_train, y_train = df_train.iloc[trn_idx][features], df_train.iloc[trn_idx]['target']
X_valid, y_valid = df_train.iloc[val_idx][features], df_train.iloc[val_idx]['target']
print("shape of training data: ", X_train.shape)
 trn_data = lgb.Dataset(X_train, label=y_train)
 val_data = lgb.Dataset(X_valid, label=y_valid)
 evals_result = {}
lgb_clf = lgb.train(lgb_params,
trn_data,
100000,
valid_sets = [trn_data, val_data],
early_stopping_rounds=1000,
verbose_eval = 1000,
evals_result=evals_result 
)
 p_valid = lgb_clf.predict(X_valid, num_iteration=lgb_clf.best_iteration)
 yp = lgb_clf.predict(X_test, num_iteration=lgb_clf.best_iteration)
 fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = [col for col in X_train.columns]
fold_importance_df["importance"] = lgb_clf.feature_importance()
fold_importance_df["fold"] = fold + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 oof['predict'][val_idx] = p_valid
val_score = roc_auc_score(y_valid, p_valid)
 val_aucs.append(val_score)
 predictions['fold{}'.format(fold+1)] = yp<set_options>
def family_size() :
global df_combined
df_combined['FamilySize'] = df_combined['Parch'] + df_combined['SibSp'] + 1
df_combined['Singleton'] = df_combined['FamilySize'].map(lambda s: 1 if s == 1 else 0)
 df_combined['SmallFamily'] = df_combined['FamilySize'].map(lambda s: 1 if 2 <= s <= 4 else 0)
 df_combined['LargeFamily'] = df_combined['FamilySize'].map(lambda s: 1 if 5 <= s <= 8 else 0)
 return df_combined
gc.collect()<compute_test_metric>
def embarked() :
global df_combined
df_combined['Embarked'].fillna('S', inplace=True)
 df_dummies = pd.get_dummies(df_combined['Embarked'], prefix='Embarked')
 df_combined = pd.concat([df_combined, df_dummies], axis=1)
 df_combined.drop('Embarked', axis=1, inplace=True)
 return df_combined
mean_auc = np.mean(val_aucs)
 std_auc = np.std(val_aucs)
 all_auc = roc_auc_score(oof['target'], oof['predict'])
 print("Mean auc: %.5f, std: %.5f.All auc: %.5f." %(mean_auc, std_auc, all_auc))<save_to_csv>
def cabin() :
global df_combined
df_combined['Cabin'].fillna('U', inplace=True)
 df_combined['Cabin'] = df_combined['Cabin'].map(lambda ca: ca[0])
 cabin_dummies = pd.get_dummies(df_combined['Cabin'], prefix='Cabin')
 df_combined = pd.concat([df_combined, cabin_dummies], axis=1)
 df_combined.drop('Cabin', inplace=True, axis=1)
 return df_combined
predictions['target'] = np.mean(predictions[[col for col in predictions.columns if col not in ['ID_code', 'target']]].values, axis=1)
 predictions.to_csv('lgb_all_predictions.csv', index=None)
 sub_df = pd.DataFrame({"ID_code":df_test["ID_code"].values})
 sub_df["target"] = predictions['target']
sub_df.to_csv("submission.csv", index=False)
 oof.to_csv('lgb_oof.csv', index=False )<load_from_csv>
df_combined['Age'] = df_combined['Age'].astype(int)
 df_combined.loc[df_combined['Age'] <= 11, 'Age'] = 0
df_combined.loc[(df_combined['Age'] > 11)&(df_combined['Age'] <= 18), 'Age'] = 1
df_combined.loc[(df_combined['Age'] > 18)&(df_combined['Age'] <= 22), 'Age'] = 2
df_combined.loc[(df_combined['Age'] > 22)&(df_combined['Age'] <= 27), 'Age'] = 3
df_combined.loc[(df_combined['Age'] > 27)&(df_combined['Age'] <= 33), 'Age'] = 4
df_combined.loc[(df_combined['Age'] > 33)&(df_combined['Age'] <= 46), 'Age'] = 5
df_combined.loc[(df_combined['Age'] > 46)&(df_combined['Age'] <= 66), 'Age'] = 6
df_combined.loc[df_combined['Age'] >= 66, 'Age'] = 7
df_combined['Age'].value_counts()
train_df = pd.read_csv('.. /input/santander-customer-transaction-prediction/train.csv')
 test_df = pd.read_csv('.. /input/santander-customer-transaction-prediction/test.csv' )<drop_column>
df_combined.loc[df_combined['Fare'] <= 7, 'Fare'] = 0
df_combined.loc[(df_combined['Fare'] > 7)&(df_combined['Fare'] <= 14), 'Fare'] = 1
df_combined.loc[(df_combined['Fare'] > 14)&(df_combined['Fare'] <= 31), 'Fare'] = 2
df_combined.loc[(df_combined['Fare'] > 31)&(df_combined['Fare'] <= 99), 'Fare'] = 3
df_combined.loc[(df_combined['Fare'] > 99)&(df_combined['Fare'] <= 250), 'Fare'] = 4
df_combined.loc[df_combined['Fare'] >= 250, 'Fare'] = 5
df_combined['Fare'] = df_combined['Fare'].astype(int)
 df_combined['Fare'].value_counts()
train_features = train_df.drop(['target','ID_code'], axis = 1)
 test_features = test_df.drop(['ID_code'],axis = 1)
 train_target = train_df['target']<set_options>
common_titles = ["Mr", "Mrs", "Miss", "Master"]
titles = []
for name in df_combined['Name']:
title = name.split(',')[1].split('.')[0].strip()
if title in common_titles:
titles.append(title)
 elif title=="Mlle":
titles.append("Miss")
 elif title=="Mme":
titles.append("Mrs")
 else:
titles.append("Rare")
 df_titles = pd.DataFrame(titles, columns=['Titles'])
 title_dummies = pd.get_dummies(df_titles['Titles'], prefix='Title')
 title_dummies = title_dummies.reset_index(drop=True)
 df_combined = df_combined.reset_index(drop=True)
 df_combined = pd.concat([df_combined, title_dummies], axis=1 )
gc.collect()<concatenate>
gender_dummies = pd.get_dummies(df_combined['Sex'])
 df_combined = pd.concat([df_combined, gender_dummies], axis=1)
 df_combined.drop('Sex', axis=1, inplace=True )
train_all = pd.concat(( train_features,test_features),axis = 0 )<data_type_conversions>
df_combined.drop(['PassengerId',"Name","Ticket"], axis=1, inplace=True )
for f in train_all.columns:
train_all[f+'_duplicate'] = train_all.duplicated(f,False ).astype(int)
<feature_engineering>
x_train = df_combined[:891].copy()
x_test = df_combined[891:].copy()
x_test.reset_index(inplace=True, drop=True)
 x_train.shape, x_test.shape
for f in train_all.columns[0:200]:
train_all[f+'duplicate_value'] = train_all[f]*train_all[f+'_duplicate']<split>
random_forest = RandomForestClassifier()
random_forest.fit(x_train, y_train)
 y_pred_random_forest = random_forest.predict(x_test)
 random_forest_accuracy = round(random_forest.score(x_train, y_train)*100, 2 )
train_features = train_all.iloc[:200000]
test_features = train_all.iloc[200000:400000]<set_options>
log_regres = LogisticRegression()
log_regres.fit(x_train, y_train)
 y_pred_log_regres = log_regres.predict(x_test)
 log_regres_accuracy = round(log_regres.score(x_train, y_train)*100, 2 )
del train_all
gc.collect()<split>
knn = KNeighborsClassifier(n_neighbors=3)
 knn.fit(x_train, y_train)
 y_pred_knn = knn.predict(x_test)
 knn_accuracy = round(knn.score(x_train, y_train)*100, 2 )
n_splits = 7
splits = list(StratifiedKFold(n_splits=n_splits, shuffle=True ).split(train_features, train_target))
splits[:3]<init_hyperparams>
gaussian = GaussianNB()
gaussian.fit(x_train, y_train)
 y_pred_gaussian = gaussian.predict(x_test)
 gaussian_accuracy = round(gaussian.score(x_train, y_train)*100, 2 )
cat_params = {
'learning_rate':0.01,
'max_depth':2,
'eval_metric': 'AUC',
'bootstrap_type': 'Bayesian',
'bagging_temperature': 1,
'objective': 'Logloss',
'od_type': 'Iter',
'l2_leaf_reg': 2,
'allow_writing_files': False}<prepare_x_and_y>
perceptron = Perceptron()
perceptron.fit(x_train, y_train)
 y_pred_perceptron = perceptron.predict(x_test)
 perceptron_accuracy = round(perceptron.score(x_train, y_train)*100, 2 )
oof_cb = np.zeros(len(train_features))
predictions_cb = np.zeros(len(test_features))
for i,(train_idx, valid_idx)in enumerate(splits):
print(f'Fold {i + 1}')
 x_train = np.array(train_features)
 y_train = np.array(train_target)
 trn_x = x_train[train_idx.astype(int)]
trn_y = y_train[train_idx.astype(int)]
val_x = x_train[valid_idx.astype(int)]
val_y = y_train[valid_idx.astype(int)]
num_round = 100000
clf = CatBoostClassifier(num_round, task_type='GPU', early_stopping_rounds=1000,**cat_params,)
 clf.fit(trn_x, trn_y, eval_set=(val_x, val_y), cat_features=[], use_best_model=True, verbose=500)
 oof_cb[valid_idx] = clf.predict_proba(val_x)[:,1]
predictions_cb += clf.predict_proba(test_features)[:,1] / 5
print("CV score: {:<8.5f}".format(roc_auc_score(train_target, oof_cb)))
 gc.collect()<init_hyperparams>
svc = LinearSVC()
svc.fit(x_train, y_train)
 y_pred_svc = svc.predict(x_test)
 svc_accuracy = round(svc.score(x_train, y_train)*100, 2 )
param = {
'bagging_freq': 5,
'bagging_fraction': 0.33,
'boost_from_average':'false',
'boost': 'gbdt',
'feature_fraction': 0.05,
'learning_rate': 0.01,
'max_depth': -1,
'metric':'auc',
'min_data_in_leaf': 80,
'min_sum_hessian_in_leaf': 10.0,
'num_leaves': 13,
'num_threads': 12,
'tree_learner': 'serial',
'objective': 'binary',
'verbosity': 1
}<prepare_x_and_y>
tree = DecisionTreeClassifier()
tree.fit(x_train, y_train)
 y_pred_tree = tree.predict(x_train)
 tree_accuracy = round(tree.score(x_train, y_train)*100, 2 )
oof = np.zeros(len(train_features))
predictions = np.zeros(len(test_features))
for i,(train_idx, valid_idx)in enumerate(splits):
print(f'Fold {i + 1}')
 x_train = np.array(train_features)
 y_train = np.array(train_target)
 trn_data = lgb.Dataset(x_train[train_idx.astype(int)], label=y_train[train_idx.astype(int)])
 val_data = lgb.Dataset(x_train[valid_idx.astype(int)], label=y_train[valid_idx.astype(int)])
 num_round = 100000
clf = lgb.train(param, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=1000, early_stopping_rounds = 3000)
 oof[valid_idx] = clf.predict(x_train[valid_idx], num_iteration=clf.best_iteration)
 predictions += clf.predict(test_features, num_iteration=clf.best_iteration)/ 5
print("CV score: {:<8.5f}".format(roc_auc_score(train_target, oof)))
 gc.collect()<compute_test_metric>
d_x_train = xgb.DMatrix(x_train, label=y_train )
esemble_lgbm_cat = 0.5*oof_cb+0.5*oof
print('LightBGM auc = {:<8.5f}'.format(roc_auc_score(train_target, oof)))
 print('catboost auc = {:<8.5f}'.format(roc_auc_score(train_target, oof_cb)))
 print('LightBGM+catboost auc = {:<8.5f}'.format(roc_auc_score(train_target, esemble_lgbm_cat)) )<define_variables>
param = {
'eta': 0.5,
'max_depth': 16,
'objective': 'multi:softprob',
'num_class': 3}
steps = 20
esemble_pred_lgbm_cat = 0.5*predictions+0.5*predictions_cb<define_variables>
xgb_model = xgb.train(param, d_x_train, steps)
 y_pred_xgb = xgb_model.predict(d_x_train)
 y_pred_xgb_new = np.asarray([np.argmax(line)for line in y_pred_xgb] )
id_code_test = test_df['ID_code']<create_dataframe>
xgb_model_accuracy = round(accuracy_score(y_train, y_pred_xgb_new)*100, 2 )
my_submission_lbgm = pd.DataFrame({"ID_code" : id_code_test, "target" : predictions})
 my_submission_cat = pd.DataFrame({"ID_code" : id_code_test, "target" : predictions_cb})
 my_submission_esemble_lgbm_cat = pd.DataFrame({"ID_code" : id_code_test, "target" : esemble_pred_lgbm_cat} )<save_to_csv>
rf = RandomForestClassifier()
scores = cross_val_score(rf, x_train, y_train, cv=10, scoring='accuracy')
 print("Scores:", scores)
 print("Mean:", scores.mean())
 print("Standard Deviation:", scores.std() )
my_submission_lbgm.to_csv('submission_lbgm.csv', index = False, header = True)
 my_submission_cat.to_csv('submission_cb.csv', index = False, header = True)
 my_submission_esemble_lgbm_cat.to_csv('my_submission_esemble_lgbm_cat.csv', index = False, header = True )<set_options>
for i in range(23, importances_df.shape[0]):
column = importances_df['Feature'][i]
df_combined.drop([column], inplace=True, axis=1)
 df_combined.head()
%matplotlib inline
print(os.listdir(".. /input")) 
<load_from_csv>
x_train = df_combined[:891].copy()
x_test = df_combined[891:].copy()
x_test.reset_index(inplace=True, drop=True )
train = pd.read_csv(r'.. /input/train.csv',low_memory=True,index_col='ID_code')
 print(train.head(1))
train=reduce_mem_usage(train )<drop_column>
random_forest = RandomForestClassifier()
random_forest.fit(x_train, y_train)
 y_prediction = random_forest.predict(x_test)
 accuracy_random_forest = round(random_forest.score(x_train, y_train)*100, 2)
 print("The accuracy after removing least important Features is {}, which is same as before removing.".format(accuracy_random_forest))
train.replace(np.nan,0,inplace=True)
<load_from_csv>
tree = DecisionTreeClassifier()
tree.fit(x_train, y_train)
 y_pred_tree = tree.predict(x_test)
 tree_accuracy = round(tree.score(x_train, y_train)*100, 2)
 print("The accuracy after removing least important Features is {}, which is same as before removing.".format(tree_accuracy))
test = pd.read_csv(r'.. /input/test.csv',low_memory=True,index_col='ID_code')
 test=reduce_mem_usage(test )<drop_column>
d_x_train = xgb.DMatrix(x_train, label=y_train)
 d_x_test = xgb.DMatrix(x_test )
train = reduce_mem_usage(train )<choose_model_class>
xgb_model = xgb.train(param, d_x_train, steps)
 y_pred_xgb = xgb_model.predict(d_x_test)
 y_pred_xgb_new = np.asarray([np.argmax(line)for line in y_pred_xgb])
<init_hyperparams><EOS> 
output = pd.DataFrame({'PassengerID': test_data.PassengerId,'Survived':y_pred_xgb_new})
 output.to_csv("my_submission.csv", index=False)
 print("Submission successfully saved!!!" )