Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<import_modules>
!pip install category_encoders
import numpy as np
import pandas as pd
from sklearn.preprocessing import LabelEncoder
import xgboost as xgb<load_from_csv>
!pip install natsort
train_transaction = pd.read_csv('.. /input/ieee-fraud-detection/train_transaction.csv', index_col='TransactionID')
 test_transaction = pd.read_csv('.. /input/ieee-fraud-detection/test_transaction.csv', index_col='TransactionID')
 train_identity = pd.read_csv('.. /input/ieee-fraud-detection/train_identity.csv', index_col='TransactionID')
 test_identity = pd.read_csv('.. /input/ieee-fraud-detection/test_identity.csv', index_col='TransactionID')
 sample_submission = pd.read_csv('.. /input/ieee-fraud-detection/sample_submission.csv', index_col='TransactionID' )<merge>
import category_encoders as ce
train = train_transaction.merge(train_identity, how='left', left_index=True, right_index=True)
 test = test_transaction.merge(test_identity, how='left', left_index=True, right_index=True )<prepare_x_and_y>
%matplotlib inline
warnings.filterwarnings('ignore' )
y_train = train['isFraud'].copy()<drop_column>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 train.describe(include="all" )
del train_transaction, train_identity, test_transaction, test_identity<prepare_x_and_y>
print(pd.isnull(train ).sum() )
X_train = train.drop('isFraud', axis=1)
 X_test = test.copy()<drop_column>
print(pd.isnull(test ).sum() )
del train, test<data_type_conversions>
train['family'] = train['SibSp'] + train['Parch']+1
test['family'] = test['SibSp'] + test['Parch']+1
X_train = X_train.fillna(-999)
 X_test = X_test.fillna(-999 )<categorify>
train['Fare_log'] = train['Fare'].apply(lambda x: np.log(x+1))
test['Fare'] = test['Fare'].fillna(test['Fare'].mean())
 test['Fare_log'] = test['Fare'].apply(lambda x: np.log(x+1))
for f in X_train.columns:
if X_train[f].dtype=='object' or X_test[f].dtype=='object':
lbl = LabelEncoder()
lbl.fit(list(X_train[f].values)+ list(X_test[f].values))
X_train[f] = lbl.transform(list(X_train[f].values))
X_test[f] = lbl.transform(list(X_test[f].values))<choose_model_class>
train['cabin_ini'] = train['Cabin'].apply(lambda x:str(x)[0])
 test['cabin_ini'] = test['Cabin'].apply(lambda x:str(x)[0])
 train.groupby('cabin_ini' ).mean()
clf = xgb.XGBClassifier(
 n_estimators=500,
max_depth=9,
learning_rate=0.05,
subsample=0.9,
colsample_bytree=0.9,
missing=-999,
random_state=2019,
tree_method='gpu_hist'
)<train_model>
ticket_num = {key:i for i,key in enumerate(natsorted(pd.concat([train['Ticket'], test['Ticket']])))}
train['ticket_id'] = train['Ticket'].map(ticket_num)/1308
test['ticket_id'] = test['Ticket'].map(ticket_num)/1308
%time clf.fit(X_train, y_train )<save_to_csv>
train['nick'] = train['Name'].apply(lambda x: x.split('.')[0].split(', ')[-1])
 test['nick'] = test['Name'].apply(lambda x: x.split('.')[0].split(', ')[-1])
 pd.concat([train['nick'], test['nick']] ).value_counts()
sample_submission['isFraud'] = clf.predict_proba(X_test)[:,1]
sample_submission.to_csv('simple_xgboost.csv' )<import_modules>
train['nick'] = train['nick'].replace(['Lady', 'Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Jonkheer', 'Dona', 'Sir', 'the Countess'], 'rare' ).replace(['Ms', 'Mme', 'Mlle'],'Miss')
 test['nick'] = test['nick'].replace(['Lady', 'Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Jonkheer', 'Dona', 'Sir', 'the Countess'], 'rare' ).replace(['Ms', 'Mme', 'Mlle'],'Miss' )
import os
import numpy as np
import pandas as pd
from sklearn import preprocessing
import xgboost as xgb<load_from_csv>
train['Embarked'] = train['Embarked'].fillna('S' )
%%time
train_transaction = pd.read_csv('.. /input/train_transaction.csv', index_col='TransactionID')
 test_transaction = pd.read_csv('.. /input/test_transaction.csv', index_col='TransactionID')
 train_identity = pd.read_csv('.. /input/train_identity.csv', index_col='TransactionID')
 test_identity = pd.read_csv('.. /input/test_identity.csv', index_col='TransactionID')
 sample_submission = pd.read_csv('.. /input/sample_submission.csv', index_col='TransactionID')
 train = train_transaction.merge(train_identity, how='left', left_index=True, right_index=True)
 test = test_transaction.merge(test_identity, how='left', left_index=True, right_index=True)
 print(train.shape)
 print(test.shape)
 y_train = train['isFraud'].copy()
del train_transaction, train_identity, test_transaction, test_identity
X_train = train.drop('isFraud', axis=1)
 X_test = test.copy()
del train, test
for f in X_train.columns:
if X_train[f].dtype=='object' or X_test[f].dtype=='object':
lbl = preprocessing.LabelEncoder()
lbl.fit(list(X_train[f].values)+ list(X_test[f].values))
X_train[f] = lbl.transform(list(X_train[f].values))
X_test[f] = lbl.transform(list(X_test[f].values)) 
<split>
list_cols = ['Sex', 'Embarked', 'cabin_ini', 'nick']
target_columns = ['Pclass', 'Sex', 'Age', 'SibSp', 'Parch', 'Embarked','family', 'Fare_log', 'cabin_ini', 'ticket_id', 'nick']
ce_ohe = ce.OneHotEncoder(cols=list_cols)
 train_onehot = ce_ohe.fit_transform(train[target_columns] )
%%time
EPOCHS = 3
kf = KFold(n_splits = EPOCHS, shuffle = True)
 y_preds = np.zeros(sample_submission.shape[0])
 y_oof = np.zeros(X_train.shape[0])
 for tr_idx, val_idx in kf.split(X_train, y_train):
clf = xgb.XGBClassifier(
 n_estimators=1000,
max_depth=15,
learning_rate=0.08,
subsample=0.9,
colsample_bytree=0.9,
tree_method='gpu_hist' 
)
 X_tr, X_vl = X_train.iloc[tr_idx, :], X_train.iloc[val_idx, :]
y_tr, y_vl = y_train.iloc[tr_idx], y_train.iloc[val_idx]
clf.fit(X_tr, y_tr)
 y_pred_train = clf.predict_proba(X_vl)[:,1]
y_oof[val_idx] = y_pred_train
print('ROC AUC {}'.format(roc_auc_score(y_vl, y_pred_train)))
 y_preds+= clf.predict_proba(X_test)[:,1] / EPOCHS 
<save_to_csv>
target = train["Survived"]
x_train, x_val, y_train, y_val = train_test_split(train_onehot, target, test_size = 0.2, random_state = 0 )
sample_submission['isFraud'] = y_preds
sample_submission.to_csv('simple_xgboost.csv' )<define_variables>
gbk = GradientBoostingClassifier()
gbk.fit(x_train, y_train)
 y_pred = gbk.predict(x_val)
 acc_gbk = round(accuracy_score(y_pred, y_val)* 100, 2)
 print(acc_gbk )
BUILD95 = True
BUILD96 = True
str_type = ['ProductCD', 'card4', 'card6', 'P_emaildomain', 'R_emaildomain','M1', 'M2', 'M3', 'M4','M5',
'M6', 'M7', 'M8', 'M9', 'id_12', 'id_15', 'id_16', 'id_23', 'id_27', 'id_28', 'id_29', 'id_30',
'id_31', 'id_33', 'id_34', 'id_35', 'id_36', 'id_37', 'id_38', 'DeviceType', 'DeviceInfo']
str_type += ['id-12', 'id-15', 'id-16', 'id-23', 'id-27', 'id-28', 'id-29', 'id-30',
'id-31', 'id-33', 'id-34', 'id-35', 'id-36', 'id-37', 'id-38']
cols = ['TransactionID', 'TransactionDT', 'TransactionAmt',
'ProductCD', 'card1', 'card2', 'card3', 'card4', 'card5', 'card6',
'addr1', 'addr2', 'dist1', 'dist2', 'P_emaildomain', 'R_emaildomain',
'C1', 'C2', 'C3', 'C4', 'C5', 'C6', 'C7', 'C8', 'C9', 'C10', 'C11',
'C12', 'C13', 'C14', 'D1', 'D2', 'D3', 'D4', 'D5', 'D6', 'D7', 'D8',
'D9', 'D10', 'D11', 'D12', 'D13', 'D14', 'D15', 'M1', 'M2', 'M3', 'M4',
'M5', 'M6', 'M7', 'M8', 'M9']
v = [1, 3, 4, 6, 8, 11]
v += [13, 14, 17, 20, 23, 26, 27, 30]
v += [36, 37, 40, 41, 44, 47, 48]
v += [54, 56, 59, 62, 65, 67, 68, 70]
v += [76, 78, 80, 82, 86, 88, 89, 91]
v += [107, 108, 111, 115, 117, 120, 121, 123]
v += [124, 127, 129, 130, 136]
v += [138, 139, 142, 147, 156, 162]
v += [165, 160, 166]
v += [178, 176, 173, 182]
v += [187, 203, 205, 207, 215]
v += [169, 171, 175, 180, 185, 188, 198, 210, 209]
v += [218, 223, 224, 226, 228, 229, 235]
v += [240, 258, 257, 253, 252, 260, 261]
v += [264, 266, 267, 274, 277]
v += [220, 221, 234, 238, 250, 271]
v += [294, 284, 285, 286, 291, 297]
v += [303, 305, 307, 309, 310, 320]
v += [281, 283, 289, 296, 301, 314]
cols += ['V'+str(x)for x in v]
dtypes = {}
for c in cols+['id_0'+str(x)for x in range(1,10)]+['id_'+str(x)for x in range(10,34)]+\
['id-0'+str(x)for x in range(1,10)]+['id-'+str(x)for x in range(10,34)]:
dtypes[c] = 'float32'
for c in str_type: dtypes[c] = 'category'<load_from_csv>
gbk = GradientBoostingClassifier()
gbk.fit(x_train, y_train)
 y_pred = gbk.predict(x_val)
 acc_gbk = round(accuracy_score(y_pred, y_val)* 100, 2)
 print(acc_gbk )
%%time
X_train = pd.read_csv('.. /input/ieee-fraud-detection/train_transaction.csv',index_col='TransactionID', dtype=dtypes, usecols=cols+['isFraud'])
 train_id = pd.read_csv('.. /input/ieee-fraud-detection/train_identity.csv',index_col='TransactionID', dtype=dtypes)
 X_train = X_train.merge(train_id, how='left', left_index=True, right_index=True)
 X_test = pd.read_csv('.. /input/ieee-fraud-detection/test_transaction.csv',index_col='TransactionID', dtype=dtypes, usecols=cols)
 test_id = pd.read_csv('.. /input/ieee-fraud-detection/test_identity.csv',index_col='TransactionID', dtype=dtypes)
 fix = {o:n for o, n in zip(test_id.columns, train_id.columns)}
test_id.rename(columns=fix, inplace=True)
 X_test = X_test.merge(test_id, how='left', left_index=True, right_index=True)
 y_train = X_train['isFraud'].copy()
del train_id, test_id, X_train['isFraud']; x = gc.collect()
print('Train shape',X_train.shape,'test shape',X_test.shape )<feature_engineering>
rfc = RandomForestClassifier()
rfc.fit(x_train, y_train)
 y_pred = rfc.predict(x_val)
 acc_rfc = round(accuracy_score(y_pred, y_val)* 100, 2)
 print(acc_rfc )
for i in range(1,16):
if i in [1,2,3,5,9]: continue
X_train['D'+str(i)] = X_train['D'+str(i)] - X_train.TransactionDT/np.float32(24*60*60)
 X_test['D'+str(i)] = X_test['D'+str(i)] - X_test.TransactionDT/np.float32(24*60*60 )<data_type_conversions>
mlp = MLPClassifier(max_iter=1000, hidden_layer_sizes=(100,),
activation='relu', solver='sgd', learning_rate_init=0.01)
 mlp.fit(x_train, y_train)
 y_pred = mlp.predict(x_val)
 acc_rfc = round(accuracy_score(y_pred, y_val)* 100, 2)
 print(acc_rfc )
%%time
for i,f in enumerate(X_train.columns):
if(np.str(X_train[f].dtype)=='category')|(X_train[f].dtype=='object'):
df_comb = pd.concat([X_train[f],X_test[f]],axis=0)
 df_comb,_ = df_comb.factorize(sort=True)
 if df_comb.max() >32000: print(f,'needs int32')
 X_train[f] = df_comb[:len(X_train)].astype('int16')
 X_test[f] = df_comb[len(X_train):].astype('int16')
 elif f not in ['TransactionAmt','TransactionDT']:
mn = np.min(( X_train[f].min() ,X_test[f].min()))
X_train[f] -= np.float32(mn)
 X_test[f] -= np.float32(mn)
 X_train[f].fillna(-1,inplace=True)
 X_test[f].fillna(-1,inplace=True )<categorify>
ids = test['PassengerId']
predictions_ens = np.array([gbk.predict(test_onehot), rfc.predict(test_onehot), mlp.predict(test_onehot)])
 predictions = stats.mode(predictions_ens, axis=0)[0][0]
def encode_FE(df1, df2, cols):
for col in cols:
df = pd.concat([df1[col],df2[col]])
 vc = df.value_counts(dropna=True, normalize=True ).to_dict()
vc[-1] = -1
nm = col+'_FE'
df1[nm] = df1[col].map(vc)
 df1[nm] = df1[nm].astype('float32')
 df2[nm] = df2[col].map(vc)
 df2[nm] = df2[nm].astype('float32')
 print(nm,', ',end='')
 def encode_LE(col,train=X_train,test=X_test,verbose=True):
df_comb = pd.concat([train[col],test[col]],axis=0)
 df_comb,_ = df_comb.factorize(sort=True)
 nm = col
if df_comb.max() >32000:
train[nm] = df_comb[:len(train)].astype('int32')
 test[nm] = df_comb[len(train):].astype('int32')
 else:
train[nm] = df_comb[:len(train)].astype('int16')
 test[nm] = df_comb[len(train):].astype('int16')
 del df_comb; x=gc.collect()
if verbose: print(nm,', ',end='')
 def encode_AG(main_columns, uids, aggregations=['mean'], train_df=X_train, test_df=X_test,
fillna=True, usena=False):
for main_column in main_columns:
for col in uids:
for agg_type in aggregations:
new_col_name = main_column+'_'+col+'_'+agg_type
temp_df = pd.concat([train_df[[col, main_column]], test_df[[col,main_column]]])
 if usena: temp_df.loc[temp_df[main_column]==-1,main_column] = np.nan
temp_df = temp_df.groupby([col])[main_column].agg([agg_type] ).reset_index().rename(
 columns={agg_type: new_col_name})
 temp_df.index = list(temp_df[col])
 temp_df = temp_df[new_col_name].to_dict()
train_df[new_col_name] = train_df[col].map(temp_df ).astype('float32')
 test_df[new_col_name] = test_df[col].map(temp_df ).astype('float32')
 if fillna:
train_df[new_col_name].fillna(-1,inplace=True)
 test_df[new_col_name].fillna(-1,inplace=True)
 print("'"+new_col_name+"'",', ',end='')
 def encode_CB(col1,col2,df1=X_train,df2=X_test):
nm = col1+'_'+col2
df1[nm] = df1[col1].astype(str)+'_'+df1[col2].astype(str)
 df2[nm] = df2[col1].astype(str)+'_'+df2[col2].astype(str)
 encode_LE(nm,verbose=False)
 print(nm,', ',end='')
 def encode_AG2(main_columns, uids, train_df=X_train, test_df=X_test):
for main_column in main_columns:
for col in uids:
comb = pd.concat([train_df[[col]+[main_column]],test_df[[col]+[main_column]]],axis=0)
 mp = comb.groupby(col)[main_column].agg(['nunique'])['nunique'].to_dict()
train_df[col+'_'+main_column+'_ct'] = train_df[col].map(mp ).astype('float32')
 test_df[col+'_'+main_column+'_ct'] = test_df[col].map(mp ).astype('float32')
 print(col+'_'+main_column+'_ct, ',end='' )<categorify>
output = pd.DataFrame({ 'PassengerId' : ids, 'Survived': predictions })
 output.to_csv('submission.csv', index=False )