Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<data_type_conversions>
import numpy as np
import pandas as pd
import pandas_summary as ps
from category_encoders import WOEEncoder
train['sex'] = train['sex'].astype("category" ).cat.codes +1
train['anatom_site_general_challenge'] = train['anatom_site_general_challenge'].astype("category" ).cat.codes +1
train.head()<data_type_conversions>
warnings.filterwarnings("ignore")
 %matplotlib inline
test['sex'] = test['sex'].astype("category" ).cat.codes +1
test['anatom_site_general_challenge'] = test['anatom_site_general_challenge'].astype("category" ).cat.codes +1
test.head()<prepare_x_and_y>
folder = '.. /input/titanic/'
train_df = pd.read_csv(folder + 'train.csv')
 test_df = pd.read_csv(folder + 'test.csv')
 sub_df = pd.read_csv(folder + 'gender_submission.csv' )
x_train = train[['sex', 'age_approx','anatom_site_general_challenge']]
y_train = train['target']
x_test = test[['sex', 'age_approx','anatom_site_general_challenge']]
train_DMatrix = xgb.DMatrix(x_train, label= y_train)
 test_DMatrix = xgb.DMatrix(x_test )<init_hyperparams>
train_df['Sex'] = train_df['Sex'].apply(lambda x: 1 if str(x)== 'male' else 0)
 test_df['Sex'] = test_df['Sex'].apply(lambda x: 1 if str(x)== 'male' else 0 )
param = {
'booster':'gbtree',
'eta': 0.3,
'num_class': 2,
'max_depth': 8
}
epochs = 100<choose_model_class>
train_df.drop('PassengerId', axis=1, inplace=True)
 test_df.drop('PassengerId', axis=1, inplace=True )
clf = xgb.XGBClassifier(n_estimators=1000,
max_depth=8,
objective='multi:softprob',
seed=0,
nthread=-1,
learning_rate=0.015,
num_class = 2,
scale_pos_weight =(32542/584))<train_model>
y = train_df['Survived']
train_df.drop('Survived', axis=1, inplace=True )
clf.fit(x_train, y_train)
<predict_on_test>
y.value_counts()
pred_train = clf.predict(x_train )<compute_test_metric>
y.value_counts(normalize=True )
accuracy_score(pred_train, y_train )<predict_on_test>
for data in(train_df, test_df):
data['FamilySize'] = data['Parch'] + data['SibSp']
sub.target = clf.predict_proba(x_test)[:,1]
sub_tabular = sub.copy()<load_from_csv>
for data in(train_df, test_df):
data['Parch'] = data['Parch'] / data['FamilySize']
data['Parch'].fillna(-1, inplace=True)
 data['SibSp'] = data['SibSp'] / data['FamilySize']
data['SibSp'].fillna(-1, inplace=True )
sub_public_merge = pd.read_csv('/kaggle/input/incredible-tpus-finetune-effnetb0-b6-at-once/submission_models_blended.csv' )<feature_engineering>
for data in(train_df, test_df):
data['Fare'].fillna(train_df.groupby(['Embarked', 'Pclass'])['Fare'].transform('median'), inplace=True)
 for data in(train_df, test_df):
data['Embarked'].fillna(train_df['Embarked'].mode() , inplace=True )
sub.target = sub_public_merge.target *0.80 + sub_tabular.target *0.20<save_to_csv>
class AgeFeature(BaseEstimator, TransformerMixin):

 def fit(self, X, y=None):
return self
def transform(self, X, y=None):
X['Initial'] = 0
for i in X:
X['Initial'] = X.Name.str.extract('([A-Za-z]+)\.')
 X['Initial'].replace(
 ['Dona','Mlle','Mme','Ms','Dr','Major','Lady','Countess','Jonkheer','Col','Rev','Capt','Sir','Don'],
['Miss','Miss','Miss','Miss','Mr','Mr','Mrs','Mrs','Other','Other','Other','Mr','Mr','Mr'],
inplace=True 
)
 print(X.groupby('Initial')['Age'].median())
 X.loc[(X.Age.isnull())&(X.Initial=='Mr'), 'Age'] = 30
X.loc[(X.Age.isnull())&(X.Initial=='Mrs'), 'Age'] = 35
X.loc[(X.Age.isnull())&(X.Initial=='Master'), 'Age'] = 3.5
X.loc[(X.Age.isnull())&(X.Initial=='Miss'), 'Age'] = 21.5
X.loc[(X.Age.isnull())&(X.Initial=='Other'), 'Age'] = 51
return X['Age'].as_matrix().reshape(-1, 1)
 def fit_transform(self, X, y=None):
return self.fit(X ).transform(X)
 ager = AgeFeature().fit(train_df)
 train_df['Age'] = ager.transform(train_df)
 test_df['Age'] = ager.transform(test_df )
sub.to_csv('submission.csv', index = False )<set_options>
for data in(train_df, test_df):
data['Is_Married'] = 0
data['Is_Married'].loc[data['Initial'] == 'Mrs'] = 1
%matplotlib inline
warnings.filterwarnings(action='ignore', category=DeprecationWarning, module='sklearn')
 warnings.simplefilter('ignore')
 def seed_everything(SEED):
np.random.seed(SEED)
 os.environ['PYTHONHASHSEED'] = str(SEED )<define_variables>
train_df['Cabin'] = train_df['Cabin'].apply(lambda x: str(x)[0] ).apply(lambda x: 'n' if x == 'T' else x)
 test_df['Cabin'] = test_df['Cabin'].apply(lambda x: str(x)[0] ).apply(lambda x: 'n' if x == 'T' else x)
 train_df['Cabin'].unique() , test_df['Cabin'].unique()
FOLDS = 3
SEED = 123
Setup_Parameters = True
seed_everything(SEED)
 file_add_list = [1,2,3,4,5]
pesudo_label = True
test_pipeline = True<load_from_csv>
woe_encoder = WOEEncoder(cols=['Cabin', 'Embarked', 'Initial'])
 woe_encoder.fit(train_df, y)
 train_df = woe_encoder.transform(train_df)
 test_df = woe_encoder.transform(test_df )
BASE_PATH = '.. /input/siim-isic-melanoma-classification'
train_metadata = pd.read_csv(os.path.join(BASE_PATH, 'train.csv'))
test_metadata = pd.read_csv(os.path.join(BASE_PATH, 'test.csv'))
sample_submission = pd.read_csv(os.path.join(BASE_PATH, 'sample_submission.csv'))
tfrecord_number_df = pd.read_csv('.. /input/stacking-data/Image_Name_TFRecord_number.csv' )<count_values>
train_df.drop(['Name', 'Ticket'], axis=1, inplace=True)
 test_df.drop(['Name', 'Ticket'], axis=1, inplace=True )
print('Unique values in column with frequency : ')
 print('
sex : ', dict(train_metadata.sex.value_counts()))
print('
age_approx : ', dict(train_metadata.age_approx.value_counts()))
print('
anatom_site_general_challenge : ', dict(train_metadata.anatom_site_general_challenge.value_counts()))
print('
diagnosis : ', dict(train_metadata.diagnosis.value_counts()))
print('
benign_malignant : ', dict(train_metadata.benign_malignant.value_counts()))
print('
target : ', dict(train_metadata.target.value_counts()))<count_values>
train_df.columns.to_list()
print('Unique values in column with frequency : ')
 print('
sex : ', dict(test_metadata.sex.value_counts()))
print('
age_approx : ', dict(test_metadata.age_approx.value_counts()))
print('
anatom_site_general_challenge : ', dict(test_metadata.anatom_site_general_challenge.value_counts()))<categorify>
train_df.columns.to_list()
train = train_metadata.copy()
train['age_approx'] = train['age_approx'].fillna(train.age_approx.mean())
 sex_code = pd.get_dummies(train.sex, prefix='sex')
 anatom_site_general_challenge_code = pd.get_dummies(train.anatom_site_general_challenge, prefix='anatom_site')
 age_aprox_normalized =(train.age_approx-train.age_approx.mean())/train.age_approx.std()
train_coded = pd.concat([train.image_name, sex_code, age_aprox_normalized, anatom_site_general_challenge_code , train.target], axis=1)
 print('Shape : ',train_coded.shape)
 train_coded.tail()<save_to_csv>
dfs = ps.DataFrameSummary(train_df)
 dfs.summary()
def add_OOF_pred(train_coded,num):
for n in file_add_list:
df_ = pd.read_csv(f'.. /input/95-cv-oof-submission/oof_{n}.csv')
 train_coded = pd.merge(train_coded, df_[['image_name','pred']], on="image_name",how='right')
 train_coded.rename({'pred': f'pred_{n}'}, axis=1, inplace=True)
 return train_coded
train_coded = pd.merge(tfrecord_number_df, train_coded, on="image_name",how='left')
 train_coded = add_OOF_pred(train_coded,5)
 train_coded.to_csv('train_coded.csv',index=False)
 train_coded.tail()<categorify>
dfs = ps.DataFrameSummary(test_df)
 dfs.summary()
test = test_metadata.copy()
test['age_approx'] = test['age_approx'].fillna(test.age_approx.mean())
 sex_code = pd.get_dummies(test.sex, prefix='sex')
 anatom_site_general_challenge_code = pd.get_dummies(test.anatom_site_general_challenge, prefix='anatom_site')
 age_aprox_normalized =(test.age_approx-test.age_approx.mean())/test.age_approx.std()
test_coded = pd.concat([test.image_name, sex_code, age_aprox_normalized , anatom_site_general_challenge_code], axis=1)
 print('Shape : ',test_coded.shape)
 test_coded.tail()<save_to_csv>
scaler = StandardScaler().fit(train_df)
 scaled_train_df = scaler.transform(train_df)
 scaled_test_df = scaler.transform(test_df )
def add_submission_pred(test_coded,num):
for n in file_add_list:
df_ = pd.read_csv(f'.. /input/95-cv-oof-submission/submission_{n}.csv')
 test_coded = pd.merge(test_coded, df_[['image_name','target']], on="image_name",how='right')
 test_coded.rename({'target': f'pred_{n}'}, axis=1, inplace=True)
 return test_coded
test_coded = add_submission_pred(test_coded,5)
 test_coded.to_csv('test_coded.csv',index=False)
 test_coded.tail()<compute_train_metric>
tsne = TSNE(perplexity=30)
 train_tsne_transformed = tsne.fit_transform(scaled_train_df)
 test_tsne_transformed = tsne.fit_transform(scaled_test_df )
def crossValidate(CLF,X=train_coded,X_test=test_coded,FOLDS = 5,SEED = 123,show_roc_curve = False,pesudo_label = False):
print(Fore.YELLOW)
 print('
model_name = type(CLF ).__name__
print('
print('
CV_Score = []
Val_preds = []
Val_imagenames = []
val_targets = []
CV_Score_pesudo = []
Val_preds_pesudo = []
Val_imagenames_pesudo = []
val_targets_pesudo = []
skf = KFold(n_splits=FOLDS,shuffle=True,random_state=SEED)
 for fold,(idxT,idxV)in enumerate(skf.split(np.arange(15))):
idxT, idxV = X.tfrecord.isin(idxT), X.tfrecord.isin(idxV)
 X_train_main, y_train_main = X[idxT], X[idxT]
X_val_main, y_val_main = X[idxV], X[idxV]
print(Fore.MAGENTA)
 print('
print(Fore.BLUE)
 print('FOLD : ',fold)
 print('Train TFrecords : ',X_train_main.tfrecord.unique())
 print('Validation TFrecords : ',X_val_main.tfrecord.unique())
 image_names = list(X_val_main['image_name'])
 X_train = X_train_main.drop(['target','tfrecord'],axis=1 ).iloc[:,1:]
y_train = y_train_main['target']
X_val = X_val_main.drop(['target','tfrecord'],axis=1 ).iloc[:,1:]
y_val = y_val_main['target']
CLF_pesudo = CLF
CLF.fit(X_train, y_train)
 try:
y_train_pred = CLF.predict_proba(X_train)[:,1]
except:
y_train_pred = CLF.predict(X_train)
 print('Train AUC : ', roc_auc_score(y_train,y_train_pred))
try:
Val_pred = CLF.predict_proba(X_val)[:,1]
except:
Val_pred = CLF.predict(X_val)
 Val_auc = roc_auc_score(y_val,Val_pred)
 print('Val AUC : ', Val_auc)
 CV_Score.append(Val_auc)
 Val_preds.append(Val_pred)
 Val_imagenames.append(image_names)
 val_targets.append(list(y_val))
if pesudo_label:
train2_Pesudo = X_train_main.copy()
train2_Pesudo['target_label'] = y_train_main['target']
test2_Pesudo = X_val_main.copy()
test2_Pesudo['target_label'] = Val_pred
test2_Pesudo = test2_Pesudo[(test2_Pesudo['target_label'] >= 0.99)| 
(test2_Pesudo['target_label'] >= 0.01)]
test2_Pesudo.loc[ test2_Pesudo['target_label']>=0.5, 'target_label' ] = 1
test2_Pesudo.loc[ test2_Pesudo['target_label']<0.5, 'target_label' ] = 0
print(Fore.CYAN)
 print('Number of Pesudo Labeled Data added : ',len(test2_Pesudo))
print('target_label = 1 : ',len(test2_Pesudo[test2_Pesudo['target_label'] == 1]))
print('target_label = 0 : ',len(test2_Pesudo[test2_Pesudo['target_label'] == 0]))
train_pesudo = pd.concat([train2_Pesudo,test2_Pesudo],axis=0)
 X_train_pesudo = train_pesudo.drop(['target_label','target','tfrecord'],axis=1 ).iloc[:,1:]
y_train_pesudo = train_pesudo['target_label']
CLF_pesudo.fit(X_train_pesudo, y_train_pesudo)
 try:
y_train_pred_pesudo = CLF_pesudo.predict_proba(X_train_pesudo)[:,1]
except:
y_train_pred_pesudo = CLF_pesudo.predict(X_train_pesudo)
 print('Pesudo Train AUC : ', roc_auc_score(y_train_pesudo,y_train_pred_pesudo))
try:
Val_pred_pesudo = CLF_pesudo.predict_proba(X_val)[:,1]
except:
Val_pred_pesudo = CLF_pesudo.predict(X_val)
 Val_auc_pesudo = roc_auc_score(y_val,Val_pred_pesudo)
 print('Pesudo Val AUC : ', Val_auc_pesudo)
 print(Style.RESET_ALL)
 CV_Score_pesudo.append(Val_auc_pesudo)
 Val_preds_pesudo.append(Val_pred_pesudo)
 Val_imagenames_pesudo.append(image_names)
 val_targets_pesudo.append(list(y_val))
valtargets = np.concatenate(val_targets)
 valpreds = np.concatenate(Val_preds)
 valimagenames = np.concatenate(Val_imagenames)
 auc_score = roc_auc_score(valtargets,valpreds)
 print(Fore.YELLOW)
 print('
print('
CV(auc_score): ',auc_score)
 print(f'Mean CV : {np.mean(CV_Score)} +/- {np.std(CV_Score)}
')
 oof = pd.DataFrame()
oof['image_name'] = valimagenames
oof['pred'] = valpreds
oof['target'] = valtargets
Test_imagenames = X_test['image_name']
X_test = X_test.iloc[:,1:]
try:
test_pred = CLF.predict_proba(X_test)[:,1]
except:
test_pred = CLF.predict(X_test)
 submission = pd.DataFrame()
submission['image_name'] = Test_imagenames
submission['target'] = test_pred
if pesudo_label:
valtargets_pesudo = np.concatenate(val_targets_pesudo)
 valpreds_pesudo = np.concatenate(Val_preds_pesudo)
 valimagenames_pesudo = np.concatenate(Val_imagenames_pesudo)
 auc_score_pesudo = roc_auc_score(valtargets_pesudo,valpreds_pesudo)
 print('Pesudo CV(auc_score): ',auc_score_pesudo)
 print(f'Pesudo Mean CV : {np.mean(CV_Score_pesudo)} +/- {np.std(CV_Score_pesudo)}
')
 oof_pesudo = pd.DataFrame()
oof_pesudo['image_name'] = valimagenames_pesudo
oof_pesudo['pred'] = valpreds_pesudo
oof_pesudo['target'] = valtargets_pesudo
try:
test_pred_pesudo = CLF_pesudo.predict_proba(X_test)[:,1]
except:
test_pred_pesudo = CLF_pesudo.predict(X_test)
 submission_pesudo = pd.DataFrame()
submission_pesudo['image_name'] = Test_imagenames
submission_pesudo['target'] = test_pred_pesudo
print('
if show_roc_curve:
fpr, tpr, _ = roc_curve(valtargets,valpreds)
 if pesudo_label:
fpr_p, tpr_p, _ = roc_curve(valtargets_pesudo,valpreds_pesudo)
 plt.figure()
lw = 2
if pesudo_label:
plt.plot(fpr_p, tpr_p, color='red',
lw=lw, label=f'Pesudo ROC curve(area = {auc_score_pesudo:0.4f})')
 plt.plot(fpr, tpr, color='darkorange',
lw=lw, label=f'ROC curve(area = {auc_score:0.4f})')
 plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')
 plt.xlim([0.0, 1.0])
 plt.ylim([0.0, 1.05])
 plt.xlabel('False Positive Rate')
 plt.ylabel('True Positive Rate')
 plt.title(f'ROC Curve by : {model_name}')
 plt.legend(loc="lower right")
 plt.show()
if pesudo_label:
return oof , submission, auc_score, model_name, oof_pesudo, submission_pesudo, auc_score_pesudo, str(model_name)+ '_Pesudo'
return oof , submission, auc_score, model_name<choose_model_class>
p = 0.9
k = 350
cat=[]
models = dict()
trains = []
preds = []
folds = []
valid_scores =dict()
skf = StratifiedKFold(n_splits=9, shuffle=True, random_state=7)
 for num, spl in enumerate(skf.split(train_df, y)) :
train_index, val_index = spl[0], spl[1]
x_train_1, x_valid_1 = train_df.iloc[train_index, :], train_df.iloc[val_index, :]
y_train_1, y_valid_1 = y.iloc[train_index], y.iloc[val_index]
print('fold: ', num+1)
 def score(params):
w=[]
best_iter = []
skf_1 = StratifiedKFold(n_splits=2, shuffle=True, random_state=77)
 for train_index_2, val_index_2 in skf_1.split(x_train_1, y_train_1):
x_train_2, x_valid_2 = x_train_1.iloc[train_index_2, :], x_train_1.iloc[val_index_2, :]
y_train_2, y_valid_2 = y_train_1.iloc[train_index_2], y_train_1.iloc[val_index_2]
train_data = lgb.Dataset(x_train_2, label=y_train_2,)
 val_data = lgb.Dataset(x_valid_2, label=y_valid_2, reference=train_data)
 gbm = lgb.train(params, train_data, valid_sets = [train_data, val_data], valid_names=['train', 'val'],
num_boost_round = 5800, verbose_eval = False,)
 w.append([gbm.best_score['train']['auc'], gbm.best_score['val']['auc']])
 best_iter.append(gbm.best_iteration)
 nrounds = np.mean(best_iter)
 res = list(np.mean(w, axis=0))
return {'loss': -res[1]+np.power(np.square(res[0]-res[1]), p), 'status': STATUS_OK,
'mean_auc_train': res[0], 'mean_auc_test': res[1], 'best_iter': int(nrounds)}
def optimize(trials):
space = {
'max_depth': -1,
'max_bin': scope.int(hp.uniform('max_bin', 2, 200)) ,
'num_leaves': scope.int(hp.uniform('num_leaves', 2, 16)) ,
'min_data_in_leaf': scope.int(hp.uniform('min_data_in_leaf', 2, 75)) ,
'lambda_l1': hp.quniform('lambda_l1', 0, 6.5, 0.25),
'lambda_l2': hp.quniform('lambda_l2', 0, 10.5, 0.25),
'learning_rate': hp.quniform('learning_rate', 0.01 , 0.05, 0.005),
'min_child_weight': hp.quniform('min_child_weight', 0.05, 25.95, 0.05),
'feature_fraction': hp.quniform('feature_fraction', 0.4, 0.8, 0.1),
'bagging_fraction': hp.quniform('bagging_fraction', 0.45, 0.95, 0.05),
'metric':('auc',),
'boosting_type': 'gbdt',
'objective': 'binary',
'nthread': 8,
'early_stopping_rounds': 10,
'silent':1,
}
best = fmin(score, space, algo=tpe.suggest, trials=trials, max_evals=k)
 trials = Trials()
optimize(trials)
 params_for = trials.best_trial['misc']['vals']
params_for['num_leaves'] = int(params_for['num_leaves'][0])
 params_for['max_bin'] = int(params_for['max_bin'][0])
 params_for['min_data_in_leaf'] = int(params_for['min_data_in_leaf'][0])
 params_for['objective'] = 'binary'
params_for['metric'] = {'auc'}
train_data = lgb.Dataset(x_train_1, label=y_train_1)
 val_data = lgb.Dataset(x_valid_1, label=y_valid_1, reference=train_data)
 gbm_main = lgb.train(params_for, train_data, valid_sets = [train_data, val_data], valid_names=['train', 'val'],
num_boost_round = 10000, verbose_eval = 100, early_stopping_rounds = 20)
 trains.append(gbm_main.predict(train_df))
preds.append(gbm_main.predict(test_df))
folds.append(gbm_main.best_score['val']['auc'])
 print('best score: ', gbm_main.best_score['val']['auc'])
 print()
valid_scores[num] = gbm_main.best_score['val']['auc']
models['lgb_fold'+str(num+1)] = gbm_main
clf1 = lgb.LGBMClassifier(max_depth=5,
metric="auc",
n_estimators=100,
num_leaves=5,
boosting_type="gbdt",
learning_rate=0.1,
feature_fraction=0.05,
colsample_bytree=0.1,
bagging_fraction=0.8,
bagging_freq=2,
reg_lambda=0.2 )<choose_model_class>
model = LogisticRegression(penalty='l1', solver='saga')
 model.fit(pd.DataFrame(np.array(trains)).T, y)
 target = model.predict(pd.DataFrame(np.array(preds)).T )
clf2 = LogisticRegression(
 C= 1.0,
class_weight=None,
dual= False,
fit_intercept= True,
intercept_scaling= 1,
l1_ratio= None,
max_iter= 100,
multi_class= 'auto',
n_jobs= None,
penalty= 'l2',
random_state= None,
solver= 'lbfgs',
tol= 0.0001,
verbose= 0,
warm_start= False )<choose_model_class>
sub_df['Survived'] = target
clf3 = XGBRegressor(base_score=0.5,
booster=None,
colsample_bylevel=1,
colsample_bynode=1,
colsample_bytree=0.8,
gamma=1,
gpu_id=-1,
importance_type='gain',
interaction_constraints=None,
learning_rate=0.002,
max_delta_step=0,
max_depth=10,
min_child_weight=1,
missing=None,
monotone_constraints=None,
n_estimators=700,
n_jobs=-1,
nthread=-1,
num_parallel_tree=1,
objective='binary:logistic',
random_state=0,
reg_alpha=0,
reg_lambda=1,
scale_pos_weight=1,
subsample=0.8,
tree_method=None,
validate_parameters=False,
verbosity=None )<choose_model_class>
sub_df.to_csv("submission.csv", index=False )