Kaggle competition: Titanic - Machine Learning from Disaster
%%time
X_train['cents'] =(X_train['TransactionAmt'] - np.floor(X_train['TransactionAmt'])).astype('float32')
 X_test['cents'] =(X_test['TransactionAmt'] - np.floor(X_test['TransactionAmt'])).astype('float32')
 print('cents, ', end='')
 encode_FE(X_train,X_test,['addr1','card1','card2','card3','P_emaildomain'])
 encode_CB('card1','addr1')
 encode_CB('card1_addr1','P_emaildomain')
 encode_FE(X_train,X_test,['card1_addr1','card1_addr1_P_emaildomain'])
 encode_AG(['TransactionAmt','D9','D11'],['card1','card1_addr1','card1_addr1_P_emaildomain'],['mean','std'],usena=True )<drop_column>
titanic_data=pd.read_csv("/kaggle/input/titanic/train.csv")
 titanic_data.head()
cols = list(X_train.columns)
 cols.remove('TransactionDT')
 for c in ['D6','D7','D8','D9','D12','D13','D14']:
cols.remove(c)
 for c in ['C3','M5','id_08','id_33']:
cols.remove(c)
 for c in ['card4','id_07','id_14','id_21','id_30','id_32','id_34']:
cols.remove(c)
 for c in ['id_'+str(x)for x in range(22,28)]:
cols.remove(c )<prepare_x_and_y>
test_data=pd.read_csv("/kaggle/input/titanic/test.csv")
 test_data.head()
idxT = X_train.index[:3*len(X_train)//4]
idxV = X_train.index[3*len(X_train)//4:] 
<train_model>
!pip install --upgrade pip
!pip install nameparser
titanic_data['NameTitle']=titanic_data['Name'].apply(lambda x: HumanName(x ).title)
 test_data['NameTitle']=test_data['Name'].apply(lambda x: HumanName(x ).title)
print("XGBoost version:", xgb.__version__)
 if BUILD95:
clf = xgb.XGBClassifier(
 n_estimators=2000,
max_depth=12,
learning_rate=0.02,
subsample=0.8,
colsample_bytree=0.4,
missing=-1,
eval_metric='auc',
tree_method='gpu_hist' 
)
 h = clf.fit(X_train.loc[idxT,cols], y_train[idxT],
eval_set=[(X_train.loc[idxV,cols],y_train[idxV])],
verbose=50, early_stopping_rounds=100 )<feature_engineering>
titanic_data.groupby(['NameTitle'] ).agg('count')
START_DATE = datetime.datetime.strptime('2017-11-30', '%Y-%m-%d')
 X_train['DT_M'] = X_train['TransactionDT'].apply(lambda x:(START_DATE + datetime.timedelta(seconds = x)))
 X_train['DT_M'] =(X_train['DT_M'].dt.year-2017)*12 + X_train['DT_M'].dt.month
X_test['DT_M'] = X_test['TransactionDT'].apply(lambda x:(START_DATE + datetime.timedelta(seconds = x)))
 X_test['DT_M'] =(X_test['DT_M'].dt.year-2017)*12 + X_test['DT_M'].dt.month<train_on_grid>
feature_list=["Pclass","Sex","Age","SibSp","Parch","Fare","NameTitle", "Survived"]
feature_list_minus_y=["Pclass","Sex","Age","SibSp","Parch","Fare","NameTitle"]
titanic_data=titanic_data[feature_list]
titanic_test=test_data
test_data=test_data[feature_list_minus_y]
if BUILD95:
oof = np.zeros(len(X_train))
preds = np.zeros(len(X_test))
skf = GroupKFold(n_splits=6)
 for i,(idxT, idxV)in enumerate(skf.split(X_train, y_train, groups=X_train['DT_M'])) :
month = X_train.iloc[idxV]['DT_M'].iloc[0]
print('Fold',i,'withholding month',month)
 print(' rows of train =',len(idxT),'rows of holdout =',len(idxV))
clf = xgb.XGBClassifier(
 n_estimators=5000,
max_depth=12,
learning_rate=0.02,
subsample=0.8,
colsample_bytree=0.4,
missing=-1,
eval_metric='auc',
tree_method='gpu_hist' 
)
 h = clf.fit(X_train[cols].iloc[idxT], y_train.iloc[idxT],
eval_set=[(X_train[cols].iloc[idxV],y_train.iloc[idxV])],
verbose=100, early_stopping_rounds=200)
 oof[idxV] += clf.predict_proba(X_train[cols].iloc[idxV])[:,1]
preds += clf.predict_proba(X_test[cols])[:,1]/skf.n_splits
del h, clf
x=gc.collect()
print('
print('XGB95 OOF CV=',roc_auc_score(y_train,oof))<save_to_csv>
def fillna_Values(df):
df['Age'] = df.groupby(['Sex', 'NameTitle'])['Age'].apply(lambda x: x.fillna(x.mean()))
df['Age'] = df.groupby(['Sex'])['Age'].apply(lambda x: x.fillna(x.mean()))
df['Fare'] = df.groupby(['Pclass'])['Fare'].apply(lambda x: x.fillna(x.mean()))
return df
if BUILD95:
plt.hist(oof,bins=100)
 plt.ylim(( 0,5000))
plt.title('XGB OOF')
 plt.show()
X_train['oof'] = oof
X_train.reset_index(inplace=True)
 X_train[['TransactionID','oof']].to_csv('oof_xgb_95.csv')
 X_train.set_index('TransactionID',drop=True,inplace=True)
 else: X_train['oof'] = 0<save_to_csv>
def standardizeData(X):
X=X.replace(to_replace =["Capt.","Col.","Sir."],
value ="Mr.")
 X=X.replace(to_replace =["Lady.","Mlle.","Mme.","Ms.","the Countess.of"],
value ="Mrs.")
 train_numerical_features = ["Age","SibSp","Parch","Fare"]
ss_scaler = MinMaxScaler(feature_range=(0,1))
X_ss = pd.DataFrame(data = X)
 X_ss[train_numerical_features] = ss_scaler.fit_transform(X_ss[train_numerical_features])
 X_ss=pd.get_dummies(X_ss, columns=["Pclass","Sex","NameTitle"])
 return X_ss
if BUILD95:
sample_submission = pd.read_csv('.. /input/ieee-fraud-detection/sample_submission.csv')
 sample_submission.isFraud = preds
sample_submission.to_csv('sub_xgb_95.csv',index=False)
 plt.hist(sample_submission.isFraud,bins=100)
 plt.ylim(( 0,5000))
plt.title('XGB95 Submission')
 plt.show()<data_type_conversions>
titanic_data=standardizeData(titanic_data)
 test_data=standardizeData(test_data )
X_train['day'] = X_train.TransactionDT /(24*60*60)
 X_train['uid'] = X_train.card1_addr1.astype(str)+'_'+np.floor(X_train.day-X_train.D1 ).astype(str)
 X_test['day'] = X_test.TransactionDT /(24*60*60)
 X_test['uid'] = X_test.card1_addr1.astype(str)+'_'+np.floor(X_test.day-X_test.D1 ).astype(str )<categorify>
train_data, mytest_data = train_test_split(titanic_data, test_size=0.2 )
%%time
encode_FE(X_train,X_test,['uid'])
 encode_AG(['TransactionAmt','D4','D9','D10','D15'],['uid'],['mean','std'],fillna=True,usena=True)
 encode_AG(['C'+str(x)for x in range(1,15)if x!=3],['uid'],['mean'],X_train,X_test,fillna=True,usena=True)
 encode_AG(['M'+str(x)for x in range(1,10)],['uid'],['mean'],fillna=True,usena=True)
 encode_AG2(['P_emaildomain','dist1','DT_M','id_02','cents'], ['uid'], train_df=X_train, test_df=X_test)
 encode_AG(['C14'],['uid'],['std'],X_train,X_test,fillna=True,usena=True)
 encode_AG2(['C13','V314'], ['uid'], train_df=X_train, test_df=X_test)
 encode_AG2(['V127','V136','V309','V307','V320'], ['uid'], train_df=X_train, test_df=X_test)
 X_train['outsider15'] =(np.abs(X_train.D1-X_train.D15)>3 ).astype('int8')
 X_test['outsider15'] =(np.abs(X_test.D1-X_test.D15)>3 ).astype('int8')
 print('outsider15' )<drop_column>
y_train=train_data["Survived"]
y_test=mytest_data["Survived"]
X_train=train_data.drop("Survived",axis=1)
 X_test=mytest_data.drop("Survived",axis=1 )
cols = list(X_train.columns)
 cols.remove('TransactionDT')
 for c in ['D6','D7','D8','D9','D12','D13','D14']:
cols.remove(c)
 for c in ['oof','DT_M','day','uid']:
cols.remove(c)
 for c in ['C3','M5','id_08','id_33']:
cols.remove(c)
 for c in ['card4','id_07','id_14','id_21','id_30','id_32','id_34']:
cols.remove(c)
 for c in ['id_'+str(x)for x in range(22,28)]:
cols.remove(c )<prepare_x_and_y>
model=GradientBoostingClassifier(n_estimators=100,max_depth=5,random_state=1)
 model.fit(X_train,y_train )
idxT = X_train.index[:3*len(X_train)//4]
idxV = X_train.index[3*len(X_train)//4:] 
<train_model>
mypredictions=model.predict(X_test)
 print(accuracy_score(y_test, mypredictions))
print('F1 score:',f1_score(y_test, mypredictions))
if BUILD96:
clf = xgb.XGBClassifier(
 n_estimators=2000,
max_depth=12,
learning_rate=0.02,
subsample=0.8,
colsample_bytree=0.4,
missing=-1,
eval_metric='auc',
tree_method='gpu_hist' 
)
 h = clf.fit(X_train.loc[idxT,cols], y_train[idxT],
eval_set=[(X_train.loc[idxV,cols],y_train[idxV])],
verbose=50, early_stopping_rounds=100 )<train_on_grid>
tf.random.set_seed(seed_value)
 nmodel=tf.keras.models.Sequential([tf.keras.layers.InputLayer(input_shape=(16,)) ,
tf.keras.layers.Dense(8,activation=tf.nn.relu),
tf.keras.layers.Dropout(0.4),
tf.keras.layers.Dense(8,activation=tf.nn.relu),
tf.keras.layers.Dropout(0.4),
tf.keras.layers.Dense(8,activation=tf.nn.relu),
tf.keras.layers.Dropout(0.4),
tf.keras.layers.Dense(1,activation='sigmoid')])
if BUILD96:
oof = np.zeros(len(X_train))
preds = np.zeros(len(X_test))
skf = GroupKFold(n_splits=6)
 for i,(idxT, idxV)in enumerate(skf.split(X_train, y_train, groups=X_train['DT_M'])) :
month = X_train.iloc[idxV]['DT_M'].iloc[0]
print('Fold',i,'withholding month',month)
 print(' rows of train =',len(idxT),'rows of holdout =',len(idxV))
clf = xgb.XGBClassifier(
 n_estimators=5000,
max_depth=12,
learning_rate=0.02,
subsample=0.8,
colsample_bytree=0.4,
missing=-1,
eval_metric='auc',
tree_method='gpu_hist' 
)
 h = clf.fit(X_train[cols].iloc[idxT], y_train.iloc[idxT],
eval_set=[(X_train[cols].iloc[idxV],y_train.iloc[idxV])],
verbose=100, early_stopping_rounds=200)
 oof[idxV] += clf.predict_proba(X_train[cols].iloc[idxV])[:,1]
preds += clf.predict_proba(X_test[cols])[:,1]/skf.n_splits
del h, clf
x=gc.collect()
print('
print('XGB96 OOF CV=',roc_auc_score(y_train,oof))<save_to_csv>
nmodel.compile(optimizer='Adam',
loss='mean_squared_error',metrics=['accuracy'])
 nmodel.fit(X_train,y_train,batch_size=32,epochs=50 )
if BUILD96:
plt.hist(oof,bins=100)
 plt.ylim(( 0,5000))
plt.title('XGB OOF')
 plt.show()
X_train['oof'] = oof
X_train.reset_index(inplace=True)
 X_train[['TransactionID','oof']].to_csv('oof_xgb_96.csv')
 X_train.set_index('TransactionID',drop=True,inplace=True )<save_to_csv>
y_pred=nmodel.predict(X_test)
 y_pred = [np.round(x)for x in y_pred]
print('Accuracy:',accuracy_score(y_test, y_pred))
score=nmodel.evaluate(X_test,y_test,verbose=1)
if BUILD96:
sample_submission = pd.read_csv('.. /input/ieee-fraud-detection/sample_submission.csv')
 sample_submission.isFraud = preds
sample_submission.to_csv('sub_xgb_96.csv',index=False)
 plt.hist(sample_submission.isFraud,bins=100)
 plt.ylim(( 0,5000))
plt.title('XGB96 Submission')
 plt.show()<load_from_csv>
y_train=titanic_data["Survived"]
X_train=titanic_data
X_train=X_train.drop("Survived",axis=1 )
X_test['isFraud'] = sample_submission.isFraud.values
X_train['isFraud'] = y_train.values
comb = pd.concat([X_train[['isFraud']],X_test[['isFraud']]],axis=0)
 uids = pd.read_csv('/kaggle/input/ieee-submissions-and-uids/uids_v4_no_multiuid_cleaning.. csv',usecols=['TransactionID','uid'] ).rename({'uid':'uid2'},axis=1)
 comb = comb.merge(uids,on='TransactionID',how='left')
 mp = comb.groupby('uid2' ).isFraud.agg(['mean'])
 comb.loc[comb.uid2>0,'isFraud'] = comb.loc[comb.uid2>0].uid2.map(mp['mean'])
 uids = pd.read_csv('/kaggle/input/ieee-submissions-and-uids/uids_v1_no_multiuid_cleaning.csv',usecols=['TransactionID','uid'] ).rename({'uid':'uid3'},axis=1)
 comb = comb.merge(uids,on='TransactionID',how='left')
 mp = comb.groupby('uid3' ).isFraud.agg(['mean'])
 comb.loc[comb.uid3>0,'isFraud'] = comb.loc[comb.uid3>0].uid3.map(mp['mean'])
 sample_submission.isFraud = comb.iloc[len(X_train):].isFraud.values
sample_submission.to_csv('sub_xgb_96_PP.csv',index=False )<set_options>
final_df=titanic_test
final_df["Survived"]=nmodel.predict(test_data)
 final_df["Survived"]=titanic_test["Survived"].apply(lambda x: round(x,0)).astype('int')
%matplotlib inline
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename)) 
<load_from_csv>
output=pd.DataFrame({'PassengerId':final_df.PassengerId,'Survived':final_df.Survived})
 output.to_csv('my_submission.csv',index=False)
 print("Your submission was successfuly saved!" )