Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<merge>
import numpy as np
import pandas as pd
import tensorflow as tf
from IPython.display import clear_output
from sklearn.model_selection import train_test_split
train = train_transaction.merge(train_identity, how='left', left_index=True, right_index=True)
 test = test_transaction.merge(test_identity, how='left', left_index=True, right_index=True)
 print('Data was successfully merged!
')
 del train_identity, train_transaction, test_identity, test_transaction
print(f'Train dataset has {train.shape[0]} rows and {train.shape[1]} columns.')
 print(f'Test dataset has {test.shape[0]} rows and {test.shape[1]} columns.')
 gc.collect()<drop_column>
print(tf.__version__ )
train.set_index(keys='TransactionID_x',inplace=True)
 test.set_index(keys='TransactionID_x',inplace=True )<define_variables>
train_data = pd.read_csv('/kaggle/input/titanic/train.csv')
 test_data = pd.read_csv("/kaggle/input/titanic/test.csv")
 train_data.head(100 )
useful_features = ['TransactionAmt', 'ProductCD', 'card1', 'card2', 'card3', 'card4', 'card5', 'card6', 'addr1', 'addr2', 'dist1',
'P_emaildomain', 'R_emaildomain', 'C1', 'C2', 'C4', 'C5', 'C6', 'C7', 'C8', 'C9', 'C10', 'C11', 'C12', 'C13',
'C14', 'D1', 'D2', 'D3', 'D4', 'D5', 'D6', 'D8', 'D9', 'D10', 'D11', 'D12', 'D13', 'D14', 'D15', 'M2', 'M3',
'M4', 'M5', 'M6', 'M7', 'M8', 'M9', 'V3', 'V4', 'V5', 'V6', 'V7', 'V8', 'V9', 'V10', 'V11', 'V12', 'V13', 'V17',
'V19', 'V20', 'V29', 'V30', 'V33', 'V34', 'V35', 'V36', 'V37', 'V38', 'V40', 'V44', 'V45', 'V46', 'V47', 'V48',
'V49', 'V51', 'V52', 'V53', 'V54', 'V56', 'V58', 'V59', 'V60', 'V61', 'V62', 'V63', 'V64', 'V69', 'V70', 'V71',
'V72', 'V73', 'V74', 'V75', 'V76', 'V78', 'V80', 'V81', 'V82', 'V83', 'V84', 'V85', 'V87', 'V90', 'V91', 'V92',
'V93', 'V94', 'V95', 'V96', 'V97', 'V99', 'V100', 'V126', 'V127', 'V128', 'V130', 'V131', 'V138', 'V139', 'V140',
'V143', 'V145', 'V146', 'V147', 'V149', 'V150', 'V151', 'V152', 'V154', 'V156', 'V158', 'V159', 'V160', 'V161',
'V162', 'V163', 'V164', 'V165', 'V166', 'V167', 'V169', 'V170', 'V171', 'V172', 'V173', 'V175', 'V176', 'V177',
'V178', 'V180', 'V182', 'V184', 'V187', 'V188', 'V189', 'V195', 'V197', 'V200', 'V201', 'V202', 'V203', 'V204',
'V205', 'V206', 'V207', 'V208', 'V209', 'V210', 'V212', 'V213', 'V214', 'V215', 'V216', 'V217', 'V219', 'V220',
'V221', 'V222', 'V223', 'V224', 'V225', 'V226', 'V227', 'V228', 'V229', 'V231', 'V233', 'V234', 'V238', 'V239',
'V242', 'V243', 'V244', 'V245', 'V246', 'V247', 'V249', 'V251', 'V253', 'V256', 'V257', 'V258', 'V259', 'V261',
'V262', 'V263', 'V264', 'V265', 'V266', 'V267', 'V268', 'V270', 'V271', 'V272', 'V273', 'V274', 'V275', 'V276',
'V277', 'V278', 'V279', 'V280', 'V282', 'V283', 'V285', 'V287', 'V288', 'V289', 'V291', 'V292', 'V294', 'V303',
'V304', 'V306', 'V307', 'V308', 'V310', 'V312', 'V313', 'V314', 'V315', 'V317', 'V322', 'V323', 'V324', 'V326',
'V329', 'V331', 'V332', 'V333', 'V335', 'V336', 'V338', 'id_01', 'id_02', 'id_03', 'id_05', 'id_06', 'id_09',
'id_11', 'id_12', 'id_13', 'id_14', 'id_15', 'id_17', 'id_19', 'id_20', 'id_30', 'id_31', 'id_32', 'id_33',
'id_36', 'id_37', 'id_38', 'DeviceType', 'DeviceInfo', 'device_name', 'device_version', 'OS_id_30', 'version_id_30',
'browser_id_31', 'version_id_31', 'screen_width', 'screen_height', 'had_id']<drop_column>
def clean_data(df):
df['Age'].fillna(0, inplace = True)
 df['Embarked'].fillna('S', inplace = True)
 df["Fare"] = df["Fare"].fillna(df["Fare"].median())
 df['Family_Size'] = df['SibSp'] + df['Parch'] + 1
df['Title'] = df.Name.str.extract('([A-Za-z]+)\.', expand=False)
 df['Title'] = df['Title'].replace(['Lady', 'Countess','Capt', 'Col','Don',\
'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona', 'Dr'], 'Other')
 df['Title'] = df['Title'].replace('Mlle', 'Miss')
 df['Title'] = df['Title'].replace('Ms', 'Miss')
 df['Title'] = df['Title'].replace('Mme', 'Mrs')
 df['Title'].fillna('Unknown', inplace = True)
 df['Title'] = df['Title'].astype(str)
 df['Cabin'].fillna('U', inplace = True)
 df['Deck'] = df["Cabin"].str.slice(0,1)
 df['Last_Name'] = df['Name'].apply(lambda x: str.split(x, ",")[0])
 Age_Title_Group = df.groupby('Title')['Age'].mean()
df['Title_Age'] = df['Age']
Titles = df['Title'].unique()
for i in range(len(Titles)) :
df.loc[(df['Age']==0)&(df['Title']== Titles[i]), 'Title_Age'] = Age_Title_Group[Titles[i]]
df['Age'] = df['Title_Age'].astype(int)
 return df
cols_to_drop = [col for col in train.columns if col not in useful_features]
cols_to_drop.remove('isFraud')
 cols_to_drop.remove('TransactionDT' )<drop_column>
train_data = clean_data(train_data)
 test_data = clean_data(test_data )
train = train.drop(cols_to_drop, axis=1)
 test = test.drop(cols_to_drop, axis=1 )<categorify>
df_all = train_data.append(test_data)
 DEFAULT_SURVIVAL_VALUE = 'Alone'
df_all['Family_Have_Survival'] = DEFAULT_SURVIVAL_VALUE
for grp, grp_df in df_all[['Survived','Name', 'Last_Name', 'Fare', 'Ticket', 'PassengerId',
'SibSp', 'Parch', 'Title_Age', 'Cabin']].groupby(['Last_Name', 'Fare']):
if(len(grp_df)!= 1):
for ind, row in grp_df.iterrows() :
smax = grp_df.drop(ind)['Survived'].max()
smin = grp_df.drop(ind)['Survived'].min()
scount = grp_df.drop(ind)['Survived'].count()
ssum = grp_df.drop(ind)['Survived'].sum()
passID = row['PassengerId']
if(smax == 1.0):
df_all.loc[df_all['PassengerId'] == passID, 'Family_Have_Survival'] = 'True'
elif(smin==0.0):
df_all.loc[df_all['PassengerId'] == passID, 'Family_Have_Survival'] = 'False'
else:
df_all.loc[df_all['PassengerId'] == passID, 'Family_Have_Survival'] = 'Unknown'
print("Number of passengers with family survival information:",
df_all.loc[df_all['Family_Have_Survival']!='Alone'].shape[0])
 df_all['Group_Size'] = 1
for _, grp_df in df_all.groupby('Ticket'):
if(len(grp_df)> 1):
for ind, row in grp_df.iterrows() :
passID = row['PassengerId']
df_all.loc[df_all['PassengerId'] == passID, 'Group_Size'] = len(grp_df)
 if(row['Family_Have_Survival'] == 'False')|(row['Family_Have_Survival']== 'Alone'):
smax = grp_df.drop(ind)['Survived'].max()
smin = grp_df.drop(ind)['Survived'].min()
scount = grp_df.drop(ind)['Survived'].count()
ssum = grp_df.drop(ind)['Survived'].sum()
if(smax == 1.0):
df_all.loc[df_all['PassengerId'] == passID, 'Family_Have_Survival'] = 'True'
elif(smin==0.0):
df_all.loc[df_all['PassengerId'] == passID, 'Family_Have_Survival'] = 'False'
else:
df_all.loc[df_all['PassengerId'] == passID, 'Family_Have_Survival'] = 'Unknown'
print("Number of passenger with family/group survival information: "
+str(df_all[df_all['Family_Have_Survival']!='Alone'].shape[0]))
print("Number of passenger buy ticket in a group information: "
+str(df_all[df_all['Group_Size']>1].shape[0]))
df_all['Fare_PP'] = df_all['Fare']/df_all['Group_Size']
train_data['Family_Have_Survival'] = df_all['Family_Have_Survival'][:len(train_data)]
test_data['Family_Have_Survival'] = df_all['Family_Have_Survival'][len(train_data):]
train_data['Group_Size'] = df_all['Group_Size'][:len(train_data)]
test_data['Group_Size'] = df_all['Group_Size'][len(train_data):]
train_data['Fare_PP'] = df_all['Fare_PP'][:len(train_data)]
test_data['Fare_PP'] = df_all['Fare_PP'][len(train_data):]
columns_a = ['TransactionAmt', 'id_02', 'D15']
columns_b = ['card1', 'card4', 'addr1']
for col_a in columns_a:
for col_b in columns_b:
for df in [train, test]:
df[f'{col_a}_to_mean_{col_b}'] = df[col_a] / df.groupby([col_b])[col_a].transform('mean')
 df[f'{col_a}_to_std_{col_b}'] = df[col_a] / df.groupby([col_b])[col_a].transform('std' )<feature_engineering>
train_data.drop(['Name', 'Ticket','SibSp', 'Parch', 'Title_Age', 'Cabin', 'Last_Name', 'Fare'], axis = 1, inplace = True)
 test_data.drop(['Name', 'Ticket', 'SibSp', 'Parch', 'Title_Age', 'Cabin', 'Last_Name', 'Fare'], axis = 1, inplace = True )
train['TransactionAmt_Log'] = np.log(train['TransactionAmt'])
 test['TransactionAmt_Log'] = np.log(test['TransactionAmt'])
 train['TransactionAmt_decimal'] =(( train['TransactionAmt'] - train['TransactionAmt'].astype(int)) * 1000 ).astype(int)
 test['TransactionAmt_decimal'] =(( test['TransactionAmt'] - test['TransactionAmt'].astype(int)) * 1000 ).astype(int)
 train['Transaction_day_of_week'] = np.floor(( train['TransactionDT'] /(3600 * 24)- 1)% 7)
 test['Transaction_day_of_week'] = np.floor(( test['TransactionDT'] /(3600 * 24)- 1)% 7)
 train['Transaction_hour'] = np.floor(train['TransactionDT'] / 3600)% 24
test['Transaction_hour'] = np.floor(test['TransactionDT'] / 3600)% 24
for feature in ['id_02__id_20', 'id_02__D8', 'D11__DeviceInfo', 'DeviceInfo__P_emaildomain', 'P_emaildomain__C2',
'card2__dist1', 'card1__card5', 'card2__id_20', 'card5__P_emaildomain', 'addr1__card1']:
f1, f2 = feature.split('__')
 train[feature] = train[f1].astype(str)+ '_' + train[f2].astype(str)
 test[feature] = test[f1].astype(str)+ '_' + test[f2].astype(str)
 le = LabelEncoder()
le.fit(list(train[feature].astype(str ).values)+ list(test[feature].astype(str ).values))
train[feature] = le.transform(list(train[feature].astype(str ).values))
test[feature] = le.transform(list(test[feature].astype(str ).values))
for feature in ['card1', 'card2', 'card3', 'card4', 'card5', 'card6', 'id_36']:
train[feature + '_count_full'] = train[feature].map(pd.concat([train[feature], test[feature]], ignore_index=True ).value_counts(dropna=False))
test[feature + '_count_full'] = test[feature].map(pd.concat([train[feature], test[feature]], ignore_index=True ).value_counts(dropna=False))
for feature in ['id_01', 'id_31', 'id_33', 'id_36']:
train[feature + '_count_dist'] = train[feature].map(train[feature].value_counts(dropna=False))
test[feature + '_count_dist'] = test[feature].map(test[feature].value_counts(dropna=False))<data_type_conversions>
train_data, val_data = train_test_split(train_data, test_size=0.2)
 print(len(train_data), 'train examples')
 print(len(val_data), 'validation examples')
 print(len(test_data), 'test examples' )
%%time
for col in train.columns:
if train[col].dtype == 'object':
le = LabelEncoder()
le.fit(list(train[col].astype(str ).values)+ list(test[col].astype(str ).values))
train[col] = le.transform(list(train[col].astype(str ).values))
test[col] = le.transform(list(test[col].astype(str ).values))<drop_column>
def df_to_dataset(df, shuffle, test, n_epochs =None):
def input_fn() :
dataframe = df.copy()
NUM_EXAMPLES = len(dataframe)
 if test:
ds = tf.data.Dataset.from_tensor_slices(dict(dataframe))
else:
labels = dataframe.pop('Survived')
 ds = tf.data.Dataset.from_tensor_slices(( dict(dataframe), labels))
if shuffle:
ds.shuffle(buffer_size=NUM_EXAMPLES)
 ds = ds.repeat(n_epochs)
 ds = ds.batch(NUM_EXAMPLES)
 return ds
return input_fn
%%time
train = reduce_mem_usage(train)
 test = reduce_mem_usage(test )<prepare_x_and_y>
train_ds = df_to_dataset(train_data, shuffle=True, test=False)
 val_ds = df_to_dataset(val_data, shuffle=False, test=False, n_epochs=1)
 test_ds = df_to_dataset(test_data, shuffle=False, test=True, n_epochs=1 )
X = train.sort_values('TransactionDT' ).drop(['isFraud', 'TransactionDT'], axis=1)
 y = train.sort_values('TransactionDT')['isFraud']
X_test = test.drop(['TransactionDT'], axis=1)
 del train, test
gc.collect()<init_hyperparams>
n_batches = 1
est = tf.estimator.BoostedTreesClassifier(feature_columns,
n_batches_per_layer=n_batches)
 est.train(train_ds, max_steps=100)
 result = est.evaluate(val_ds)
 clear_output()
print(pd.Series(result))
params = {'num_leaves': 491,
'min_child_weight': 0.03454472573214212,
'feature_fraction': 0.3797454081646243,
'bagging_fraction': 0.4181193142567742,
'min_data_in_leaf': 106,
'objective': 'binary',
'max_depth': -1,
'learning_rate': 0.006883242363721497,
"boosting_type": "gbdt",
"bagging_seed": 11,
"metric": 'auc',
"verbosity": -1,
'reg_alpha': 0.3899927210061127,
'reg_lambda': 0.6485237330340494,
'random_state': 47,
}<split>
predictions = est.predict(test_ds)
 probs = pd.Series([pred['probabilities'][1] for pred in predictions])
 print(probs)
 predictions =(probs >= 0.5 ).astype(int )
X_temp = X.iloc[:10000]
y_temp = y.iloc[:10000] 
<prepare_x_and_y>
predictions_dataframe = test_data[["PassengerId"]]
predictions_dataframe["Survived"] = predictions
NFOLDS = 2
folds = KFold(n_splits=NFOLDS)
 columns = X.columns
splits = folds.split(X, y)
 y_preds = np.zeros(X_test.shape[0])
 y_oof = np.zeros(X.shape[0])
 score = 0
feature_importances = pd.DataFrame()
feature_importances['feature'] = columns
for fold_n,(train_index, valid_index)in enumerate(splits):
X_train, X_valid = X[columns].iloc[train_index], X[columns].iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
dtrain = lgb.Dataset(X_train, label=y_train)
 dvalid = lgb.Dataset(X_valid, label=y_valid)
 clf = lgb.train(params, dtrain, 10000, valid_sets = [dtrain, dvalid], verbose_eval=200, early_stopping_rounds=500)
 feature_importances[f'fold_{fold_n + 1}'] = clf.feature_importance()
y_pred_valid = clf.predict(X_valid)
 y_oof[valid_index] = y_pred_valid
print(f"Fold {fold_n + 1} | AUC: {roc_auc_score(y_valid, y_pred_valid)}")
 score += roc_auc_score(y_valid, y_pred_valid)/ NFOLDS
y_preds += clf.predict(X_test)/ NFOLDS
del X_train, X_valid, y_train, y_valid
gc.collect()
print(f"
Mean AUC = {score}")
 print(f"Out of folds AUC = {roc_auc_score(y, y_oof)}" )<save_to_csv>
predictions_dataframe.to_csv("my_submission.csv",index=False )