Kaggle competition: SIIM-ISIC Melanoma Classification
<SOS> metric: MCAUC Kaggle data source: siim-isic-melanoma-classification<data_type_conversions>
%matplotlib inline
warnings.filterwarnings(action='ignore', category=DeprecationWarning, module='sklearn')
 warnings.simplefilter('ignore')
 def seed_everything(SEED):
np.random.seed(SEED)
 os.environ['PYTHONHASHSEED'] = str(SEED )
all_data["MasVnrType"] = all_data["MasVnrType"].fillna("None")
 all_data["MasVnrArea"] = all_data["MasVnrArea"].fillna(0 )<categorify>
FOLDS = 3
SEED = 123
Setup_Parameters = True
seed_everything(SEED)
 file_add_list = [1,2,3,4,5]
pesudo_label = True
test_pipeline = True
for col in("MSZoning","Functional", "Electrical","KitchenQual", "Exterior1st","Exterior2nd","SaleType"):
all_data[col] = all_data[col].fillna(all_data[col].mode() [0] )<count_values>
BASE_PATH = '.. /input/siim-isic-melanoma-classification'
train_metadata = pd.read_csv(os.path.join(BASE_PATH, 'train.csv'))
test_metadata = pd.read_csv(os.path.join(BASE_PATH, 'test.csv'))
sample_submission = pd.read_csv(os.path.join(BASE_PATH, 'sample_submission.csv'))
tfrecord_number_df = pd.read_csv('.. /input/stacking-data/Image_Name_TFRecord_number.csv' )
all_data.Utilities.value_counts()<drop_column>
print('Unique values in column with frequency : ')
 print('
sex : ', dict(train_metadata.sex.value_counts()))
print('
age_approx : ', dict(train_metadata.age_approx.value_counts()))
print('
anatom_site_general_challenge : ', dict(train_metadata.anatom_site_general_challenge.value_counts()))
print('
diagnosis : ', dict(train_metadata.diagnosis.value_counts()))
print('
benign_malignant : ', dict(train_metadata.benign_malignant.value_counts()))
print('
target : ', dict(train_metadata.target.value_counts()))
all_data = all_data.drop(["Utilities"], axis=1 )<feature_engineering>
print('Unique values in column with frequency : ')
 print('
sex : ', dict(test_metadata.sex.value_counts()))
print('
age_approx : ', dict(test_metadata.age_approx.value_counts()))
print('
anatom_site_general_challenge : ', dict(test_metadata.anatom_site_general_challenge.value_counts()))
all_data["Age_of_house"] = all_data.YrSold - all_data.YearBuilt<drop_column>
train = train_metadata.copy()
train['age_approx'] = train['age_approx'].fillna(train.age_approx.mean())
 sex_code = pd.get_dummies(train.sex, prefix='sex')
 anatom_site_general_challenge_code = pd.get_dummies(train.anatom_site_general_challenge, prefix='anatom_site')
 age_aprox_normalized =(train.age_approx-train.age_approx.mean())/train.age_approx.std()
train_coded = pd.concat([train.image_name, sex_code, age_aprox_normalized, anatom_site_general_challenge_code , train.target], axis=1)
 print('Shape : ',train_coded.shape)
 train_coded.tail()
for col in all_data.columns[all_data.dtypes == "object"]:
all_data[col] = all_data[col].factorize() [0]
all_data = all_data.drop(["Id"], axis=1 )<split>
def add_OOF_pred(train_coded,num):
for n in file_add_list:
df_ = pd.read_csv(f'.. /input/95-cv-oof-submission/oof_{n}.csv')
 train_coded = pd.merge(train_coded, df_[['image_name','pred']], on="image_name",how='right')
 train_coded.rename({'pred': f'pred_{n}'}, axis=1, inplace=True)
 return train_coded
train_coded = pd.merge(tfrecord_number_df, train_coded, on="image_name",how='left')
 train_coded = add_OOF_pred(train_coded,5)
 train_coded.to_csv('train_coded.csv',index=False)
 train_coded.tail()
data_train = all_data.iloc[:ntrain]
data_test = all_data.iloc[ntrain:]<split>
test = test_metadata.copy()
test['age_approx'] = test['age_approx'].fillna(test.age_approx.mean())
 sex_code = pd.get_dummies(test.sex, prefix='sex')
 anatom_site_general_challenge_code = pd.get_dummies(test.anatom_site_general_challenge, prefix='anatom_site')
 age_aprox_normalized =(test.age_approx-test.age_approx.mean())/test.age_approx.std()
test_coded = pd.concat([test.image_name, sex_code, age_aprox_normalized , anatom_site_general_challenge_code], axis=1)
 print('Shape : ',test_coded.shape)
 test_coded.tail()
train_x, val_x, train_y, val_y = train_test_split(data_train, train_label, test_size=0.3, random_state=42 )<train_model>
def add_submission_pred(test_coded,num):
for n in file_add_list:
df_ = pd.read_csv(f'.. /input/95-cv-oof-submission/submission_{n}.csv')
 test_coded = pd.merge(test_coded, df_[['image_name','target']], on="image_name",how='right')
 test_coded.rename({'target': f'pred_{n}'}, axis=1, inplace=True)
 return test_coded
test_coded = add_submission_pred(test_coded,5)
 test_coded.to_csv('test_coded.csv',index=False)
 test_coded.tail()
def model_tuning(n_jobs=4,colsample_bylevel = 1, colsample_bytree=1, gamma=0.0,
learning_rate=0.1, max_depth= 3, min_child_weight=1, n_estimators=300, reg_alpha=0, reg_lambda=1,subsample=1):
model = XGBRegressor(n_jobs=n_jobs, objective='reg:linear', seed=42,silent=True,colsample_bylevel = colsample_bylevel, colsample_bytree=colsample_bytree,
gamma=gamma,learning_rate=learning_rate, max_depth= max_depth,
min_child_weight=min_child_weight, n_estimators=n_estimators, reg_alpha=reg_alpha, reg_lambda=reg_lambda,subsample=subsample)
 model_cv = cross_val_score(model, train_x, train_y, cv=5, scoring="neg_mean_squared_log_error")
 return np.sqrt(-1*model_cv.mean() )<train_model>
def crossValidate(CLF,X=train_coded,X_test=test_coded,FOLDS = 5,SEED = 123,show_roc_curve = False,pesudo_label = False):
print(Fore.YELLOW)
 print('
model_name = type(CLF ).__name__
print('
print('
CV_Score = []
Val_preds = []
Val_imagenames = []
val_targets = []
CV_Score_pesudo = []
Val_preds_pesudo = []
Val_imagenames_pesudo = []
val_targets_pesudo = []
skf = KFold(n_splits=FOLDS,shuffle=True,random_state=SEED)
 for fold,(idxT,idxV)in enumerate(skf.split(np.arange(15))):
idxT, idxV = X.tfrecord.isin(idxT), X.tfrecord.isin(idxV)
 X_train_main, y_train_main = X[idxT], X[idxT]
X_val_main, y_val_main = X[idxV], X[idxV]
print(Fore.MAGENTA)
 print('
print(Fore.BLUE)
 print('FOLD : ',fold)
 print('Train TFrecords : ',X_train_main.tfrecord.unique())
 print('Validation TFrecords : ',X_val_main.tfrecord.unique())
 image_names = list(X_val_main['image_name'])
 X_train = X_train_main.drop(['target','tfrecord'],axis=1 ).iloc[:,1:]
y_train = y_train_main['target']
X_val = X_val_main.drop(['target','tfrecord'],axis=1 ).iloc[:,1:]
y_val = y_val_main['target']
CLF_pesudo = CLF
CLF.fit(X_train, y_train)
 try:
y_train_pred = CLF.predict_proba(X_train)[:,1]
except:
y_train_pred = CLF.predict(X_train)
 print('Train AUC : ', roc_auc_score(y_train,y_train_pred))
try:
Val_pred = CLF.predict_proba(X_val)[:,1]
except:
Val_pred = CLF.predict(X_val)
 Val_auc = roc_auc_score(y_val,Val_pred)
 print('Val AUC : ', Val_auc)
 CV_Score.append(Val_auc)
 Val_preds.append(Val_pred)
 Val_imagenames.append(image_names)
 val_targets.append(list(y_val))
if pesudo_label:
train2_Pesudo = X_train_main.copy()
train2_Pesudo['target_label'] = y_train_main['target']
test2_Pesudo = X_val_main.copy()
test2_Pesudo['target_label'] = Val_pred
test2_Pesudo = test2_Pesudo[(test2_Pesudo['target_label'] >= 0.99)| 
(test2_Pesudo['target_label'] >= 0.01)]
test2_Pesudo.loc[ test2_Pesudo['target_label']>=0.5, 'target_label' ] = 1
test2_Pesudo.loc[ test2_Pesudo['target_label']<0.5, 'target_label' ] = 0
print(Fore.CYAN)
 print('Number of Pesudo Labeled Data added : ',len(test2_Pesudo))
print('target_label = 1 : ',len(test2_Pesudo[test2_Pesudo['target_label'] == 1]))
print('target_label = 0 : ',len(test2_Pesudo[test2_Pesudo['target_label'] == 0]))
train_pesudo = pd.concat([train2_Pesudo,test2_Pesudo],axis=0)
 X_train_pesudo = train_pesudo.drop(['target_label','target','tfrecord'],axis=1 ).iloc[:,1:]
y_train_pesudo = train_pesudo['target_label']
CLF_pesudo.fit(X_train_pesudo, y_train_pesudo)
 try:
y_train_pred_pesudo = CLF_pesudo.predict_proba(X_train_pesudo)[:,1]
except:
y_train_pred_pesudo = CLF_pesudo.predict(X_train_pesudo)
 print('Pesudo Train AUC : ', roc_auc_score(y_train_pesudo,y_train_pred_pesudo))
try:
Val_pred_pesudo = CLF_pesudo.predict_proba(X_val)[:,1]
except:
Val_pred_pesudo = CLF_pesudo.predict(X_val)
 Val_auc_pesudo = roc_auc_score(y_val,Val_pred_pesudo)
 print('Pesudo Val AUC : ', Val_auc_pesudo)
 print(Style.RESET_ALL)
 CV_Score_pesudo.append(Val_auc_pesudo)
 Val_preds_pesudo.append(Val_pred_pesudo)
 Val_imagenames_pesudo.append(image_names)
 val_targets_pesudo.append(list(y_val))
valtargets = np.concatenate(val_targets)
 valpreds = np.concatenate(Val_preds)
 valimagenames = np.concatenate(Val_imagenames)
 auc_score = roc_auc_score(valtargets,valpreds)
 print(Fore.YELLOW)
 print('
print('
CV(auc_score): ',auc_score)
 print(f'Mean CV : {np.mean(CV_Score)} +/- {np.std(CV_Score)}
')
 oof = pd.DataFrame()
oof['image_name'] = valimagenames
oof['pred'] = valpreds
oof['target'] = valtargets
Test_imagenames = X_test['image_name']
X_test = X_test.iloc[:,1:]
try:
test_pred = CLF.predict_proba(X_test)[:,1]
except:
test_pred = CLF.predict(X_test)
 submission = pd.DataFrame()
submission['image_name'] = Test_imagenames
submission['target'] = test_pred
if pesudo_label:
valtargets_pesudo = np.concatenate(val_targets_pesudo)
 valpreds_pesudo = np.concatenate(Val_preds_pesudo)
 valimagenames_pesudo = np.concatenate(Val_imagenames_pesudo)
 auc_score_pesudo = roc_auc_score(valtargets_pesudo,valpreds_pesudo)
 print('Pesudo CV(auc_score): ',auc_score_pesudo)
 print(f'Pesudo Mean CV : {np.mean(CV_Score_pesudo)} +/- {np.std(CV_Score_pesudo)}
')
 oof_pesudo = pd.DataFrame()
oof_pesudo['image_name'] = valimagenames_pesudo
oof_pesudo['pred'] = valpreds_pesudo
oof_pesudo['target'] = valtargets_pesudo
try:
test_pred_pesudo = CLF_pesudo.predict_proba(X_test)[:,1]
except:
test_pred_pesudo = CLF_pesudo.predict(X_test)
 submission_pesudo = pd.DataFrame()
submission_pesudo['image_name'] = Test_imagenames
submission_pesudo['target'] = test_pred_pesudo
print('
if show_roc_curve:
fpr, tpr, _ = roc_curve(valtargets,valpreds)
 if pesudo_label:
fpr_p, tpr_p, _ = roc_curve(valtargets_pesudo,valpreds_pesudo)
 plt.figure()
lw = 2
if pesudo_label:
plt.plot(fpr_p, tpr_p, color='red',
lw=lw, label=f'Pesudo ROC curve(area = {auc_score_pesudo:0.4f})')
 plt.plot(fpr, tpr, color='darkorange',
lw=lw, label=f'ROC curve(area = {auc_score:0.4f})')
 plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')
 plt.xlim([0.0, 1.0])
 plt.ylim([0.0, 1.05])
 plt.xlabel('False Positive Rate')
 plt.ylabel('True Positive Rate')
 plt.title(f'ROC Curve by : {model_name}')
 plt.legend(loc="lower right")
 plt.show()
if pesudo_label:
return oof , submission, auc_score, model_name, oof_pesudo, submission_pesudo, auc_score_pesudo, str(model_name)+ '_Pesudo'
return oof , submission, auc_score, model_name
model = XGBRegressor(max_depth=3, subsample=0.8, colsample_bytree=0.2, colsample_bylevel=1.0, n_jobs=4, n_estimators=600, learning_rate=0.03, seed=42)
 model.fit(train_x, train_y )<compute_train_metric>
clf1 = lgb.LGBMClassifier(max_depth=5,
metric="auc",
n_estimators=100,
num_leaves=5,
boosting_type="gbdt",
learning_rate=0.1,
feature_fraction=0.05,
colsample_bytree=0.1,
bagging_fraction=0.8,
bagging_freq=2,
reg_lambda=0.2 )
print("val acc", np.sqrt(mean_squared_log_error(model.predict(val_x), val_y)))
 print("train acc", np.sqrt(mean_squared_log_error(model.predict(train_x), train_y)) )<choose_model_class>
clf2 = LogisticRegression(
 C= 1.0,
class_weight=None,
dual= False,
fit_intercept= True,
intercept_scaling= 1,
l1_ratio= None,
max_iter= 100,
multi_class= 'auto',
n_jobs= None,
penalty= 'l2',
random_state= None,
solver= 'lbfgs',
tol= 0.0001,
verbose= 0,
warm_start= False )
def model_tuner(max_depth=4, subsample=0.68,
colsample_bytree=0.2, colsample_bylevel=1.0,
n_estimators=600, learning_rate=0.03,
min_child_weight=4.5, reg_lambda=5.9):
model = XGBRegressor(max_depth=max_depth, subsample=subsample,
colsample_bytree=colsample_bytree, colsample_bylevel=colsample_bylevel,
n_jobs=4, n_estimators=n_estimators, learning_rate=learning_rate, seed=42,
min_child_weight=min_child_weight, reg_lambda=reg_lambda)
 model.fit(train_x, train_y)
 train_accuracy = np.sqrt(mean_squared_log_error(model.predict(train_x), train_y))
test_accuracy = np.sqrt(mean_squared_log_error(model.predict(val_x), val_y))
return train_accuracy, test_accuracy<train_model>
clf3 = XGBRegressor(base_score=0.5,
booster=None,
colsample_bylevel=1,
colsample_bynode=1,
colsample_bytree=0.8,
gamma=1,
gpu_id=-1,
importance_type='gain',
interaction_constraints=None,
learning_rate=0.002,
max_delta_step=0,
max_depth=10,
min_child_weight=1,
missing=None,
monotone_constraints=None,
n_estimators=700,
n_jobs=-1,
nthread=-1,
num_parallel_tree=1,
objective='binary:logistic',
random_state=0,
reg_alpha=0,
reg_lambda=1,
scale_pos_weight=1,
subsample=0.8,
tree_method=None,
validate_parameters=False,
verbosity=None )
model = XGBRegressor(max_depth=4, subsample=0.68, colsample_bytree=0.2,
colsample_bylevel=1.0, n_estimators=5000,
learning_rate=0.01, min_child_weight=4.5,
reg_lambda=5.9, n_jobs=4, seed=42)
 history = model.fit(train_x, train_y,
eval_set=[(val_x, val_y)], verbose=False )<compute_train_metric>
clf4 = GaussianNB(
 priors= None,
var_smoothing= 1e-09 )
model = XGBRegressor(max_depth=4, subsample=0.68, colsample_bytree=0.2,
colsample_bylevel=1.0, n_estimators=3000,
learning_rate=0.01, min_child_weight=4.5,
reg_lambda=5.9, n_jobs=4, seed=42)
 accuracy = cross_val_score(model, data_train, train_label, cv=5, scoring="neg_mean_squared_log_error")
 print("Accuracy", np.sqrt(-accuracy.mean()))<train_model>
clf5 = RandomForestClassifier(
 bootstrap= True,
ccp_alpha= 0.0,
class_weight= None,
criterion= 'gini',
max_depth= 5,
max_features= 'auto',
max_leaf_nodes= 30,
max_samples= None,
min_impurity_decrease= 0.0,
min_impurity_split= None,
min_samples_leaf= 2,
min_samples_split= 100,
min_weight_fraction_leaf= 0.0,
n_estimators= 300,
n_jobs= None,
oob_score= False,
random_state= None,
verbose= 0,
warm_start= False )
model.fit(data_train, train_label )<save_to_csv>
clf9 = KNeighborsRegressor(algorithm= 'auto',
leaf_size= 30,
metric= 'minkowski',
metric_params= None,
n_jobs= None,
n_neighbors= 10,
p= 5,
weights= 'uniform' )
y_pred = model.predict(data_test)
 submission = pd.read_csv(".. /input/sample_submission.csv")
 submission["SalePrice"] = y_pred
submission.to_csv('submission.csv', index=False )<set_options>
clf10 = DecisionTreeRegressor(ccp_alpha= 0.0,
criterion= 'mse',
max_depth= 5,
max_features= 'auto',
max_leaf_nodes= 30,
min_impurity_decrease= 0.0,
min_impurity_split= None,
min_samples_leaf= 2,
min_samples_split= 100,
min_weight_fraction_leaf= 0.0,
presort= 'deprecated',
random_state= SEED,
splitter= 'best' )
sns.set_style("whitegrid")
 pd.set_option('display.max_columns', None)
 pd.set_option('display.max_rows', 200)
 warnings.filterwarnings('ignore')
 print(device_lib.list_local_devices())
 config = tf.ConfigProto(device_count={"CPU": 1, "GPU" : 1})
 session = tf.Session(config=config)
 K.set_session(session )<load_from_csv>
clf11 = GradientBoostingRegressor()
test = pd.read_csv(".. /input/test.csv", ",")
 train = pd.read_csv(".. /input/train.csv", ",")
 test["is_test"] = True
train["is_test"] = False
common = pd.concat([test, train],axis=0 ).loc[:,["PassengerId", "Survived", "is_test",
"Age", "Cabin", "Embarked",
"Fare", "Name", "Parch", "Pclass",
"Sex", "SibSp", "Ticket"]]<count_unique_values>
SCF = StackingClassifier(classifiers=[clf1, clf5],
meta_classifier=clf2,
use_probas=True,
average_probas=True )
common["Ticket"].count() - len(common["Ticket"].unique() )<merge>
if test_pipeline:
pipelines = []
pipelines.append(( 'LGBMClassifier', Pipeline([('LGBMClassifier',clf1)])))
 pipelines.append(( 'LogisticRegression', Pipeline([('LogisticRegression',clf2)])))
 pipelines.append(( 'XGBRegressor', Pipeline([('XGBRegressor',clf3)])))
 pipelines.append(( 'GaussianNB', Pipeline([('GaussianNB',clf4)])))
 pipelines.append(( 'RandomForestClassifier', Pipeline([('RandomForestClassifier',clf5)])))
 pipelines.append(( 'KNeighborsRegressor', Pipeline([('KNeighborsRegressor', clf9)])))
 pipelines.append(( 'DecisionTreeRegressor', Pipeline([('DecisionTreeRegressor', clf10)])))
 pipelines.append(( 'GradientBoostingRegressor', Pipeline([('GradientBoostingRegressor', clf11)])))
 pipelines.append(( 'StackingClassifier', Pipeline([('StackingClassifier', SCF)])))
 X_train = train_coded.drop('target',axis=1 ).iloc[:,1:]
y_train = train_coded['target']
M_name = []
M_auc_score = []
for name, model in pipelines:
cv_results = cross_val_score(model, X_train, y_train, cv=FOLDS, scoring='roc_auc')
 M_auc_score.append(np.mean(cv_results))
M_name.append(name)
 print("%s: %f +/- %f" %(name, cv_results.mean() , cv_results.std()))
t = train.groupby(by="Ticket", as_index=False ).agg({"PassengerId" : 'count', "Sex" : lambda x : x[x=="female"].count() })
 t.columns = ["Ticket", "SameTicket", "FemalesOnTicket"]
common = pd.merge(common, t, how="left", on="Ticket")
 common["TicketDigits"] = pd.to_numeric(common["Ticket"].str.split(" " ).str[-1], errors="coerce" ).astype(np.str ).str.len()
common["TicketIsNumber"] = ~common["Ticket"].str.contains("[A-Za-z]", regex = True)
 common["FemalesPerTicketPart"] = common["FemalesOnTicket"]/common["SameTicket"]<feature_engineering>
select_classifier = clf2
select_classifier.get_params()
common["DoubleName"] = common["Name"].str.contains("\(")
 common["NameLen"] = common["Name"].str.len()<feature_engineering>
if Setup_Parameters:
X_train = train_coded.drop('target',axis=1 ).iloc[:,1:]
y_train = train_coded['target']
params = dict(
 C = [0.001, 0.01, 0.1, 1, 10],
max_iter = [100,150,50] 
)
 grid = GridSearchCV(estimator=select_classifier,
param_grid=params,
cv=FOLDS,
scoring='roc_auc',
refit='AUC',
n_jobs = -1)
 grid.fit(X_train, y_train)
 for r, _ in enumerate(grid.cv_results_['mean_test_score']):
print("AUC : %0.5f +/- %0.5f %r"
%(grid.cv_results_['mean_test_score'][r],
grid.cv_results_['std_test_score'][r] / 2.0,
grid.cv_results_['params'][r]))
print(f'
Best parameters: {grid.best_params_}')
 oof, submission, auc_score, model_name = crossValidate(grid,X=train_coded,X_test=test_coded,FOLDS=FOLDS,SEED=SEED,show_roc_curve=True )