Kaggle competition: SIIM-ISIC Melanoma Classification
<SOS> metric: MCAUC Kaggle data source: siim-isic-melanoma-classification<count_missing_values>
!pip install iterative-stratification
!pip install efficientnet-pytorch
imputer = SimpleImputer(strategy='most_frequent')
 def impute_df(df):
df_imputed = pd.DataFrame(imputer.fit_transform(df))
df_imputed.columns = df.columns
df = df_imputed.copy()
return df
print(f'Total NA in train: {train.isna().sum().sum() }')
 print(f'Total NA in test: {test.isna().sum().sum() }' )<feature_engineering>
import random
import numpy as np
import pandas as pd
import torch
import PIL.Image as pil
import matplotlib.pyplot as plt
from fastai.vision import *
from efficientnet_pytorch import EfficientNet
from sklearn.model_selection import StratifiedKFold
import os
for df in [train, test]:
df['Family'] = df['Parch'] + df['SibSp']<drop_column>
warnings.filterwarnings("ignore", category=UserWarning, module="torch.nn.functional" )
for df in [train,test]:
df.drop(['Age','Fare'],inplace=True,axis=1 )<split>
!pip install torch==1.4.0 torchvision==0.5.0
X = train.copy()
X.drop('Survived',inplace=True,axis=1)
 y = train['Survived'].copy()
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2 )<train_on_grid>
%reload_ext autoreload
%autoreload 2
%matplotlib inline
def gridsearch(X,y,model,grid,cv):

 CV = GridSearchCV(estimator=model,param_grid=grid,cv=cv)
 CV.fit(X, y)
 print(CV.best_params_)
 print('Best parameters returned for use')
 return(CV.best_params_ )<choose_model_class>
seed = 42
def random_seed(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 os.environ['PYTHONHASHSEED'] = str(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
random_seed(seed )
xgb = XGBClassifier(learning_rate=xgb_params['learning_rate'],max_depth=xgb_params['max_depth'],
n_estimators=xgb_params['n_estimators'],subsample=xgb_params['subsample'])
 forest = RandomForestClassifier(max_depth=forest_params['max_depth'],max_leaf_nodes=forest_params['max_leaf_nodes'],
n_estimators=forest_params['n_estimators'] )<compute_train_metric>
path = '/kaggle/input/siim-isic-melanoma-classification'
path
def cross_val(model_name,model,X,y,cv):

 scores = cross_val_score(model, X, y, cv=cv)
 print(f'{model_name} Scores:')
 for i in scores:
print(round(i,2))
print(f'Average {model_name} score: {round(scores.mean() ,2)}' )<predict_on_test>
img_path = '/kaggle/input/melanoma-merged-external-data-512x512-jpeg'
img_path
xgb.fit(X,y)
 preds = xgb.predict(test)
 test.PassengerId.astype(int )<save_to_csv>
train_df = pd.read_csv(img_path + '/folds_13062020.csv')
 train_df.head()
output = pd.DataFrame({'PassengerId': test.PassengerId.astype(int), 'Survived': preds.astype(int)})
 output.to_csv('my_submission.csv', index=False)
 print("Your submission was successfully saved!" )<set_options>
test_df = pd.read_csv(path + '/test.csv')
 test_df.head()
%matplotlib inline
sns.set()
rcParams['figure.figsize'] = 12,8 
<load_from_csv>
sample_df = pd.read_csv(path + '/sample_submission.csv')
 sample_df.head()
train_df = pd.read_csv("/kaggle/input/titanic/train.csv")
 test_df = pd.read_csv('/kaggle/input/titanic/test.csv')
 train_df.head()<count_unique_values>
tfms = get_transforms(flip_vert=True, max_rotate=15, max_zoom=1.2, max_lighting=0.3, max_warp=0, p_affine=0, p_lighting=0.8 )
print('Train dataset has only {} unique tickets'.format(len(train_df['Ticket'].unique())))
 print('-'*40)
 print('Test dataset has only {} unique tickets'.format(len(test_df['Ticket'].unique())) )<create_dataframe>
class FocalLoss(nn.Module):
def __init__(self, gamma=2., reduction='mean'):
super().__init__()
self.gamma = gamma
self.reduction = reduction
def forward(self, inputs, targets):
CE_loss = nn.CrossEntropyLoss(reduction='none' )(inputs, targets)
 pt = torch.exp(-CE_loss)
 F_loss =(( 1 - pt)**self.gamma)* CE_loss
if self.reduction == 'sum':
return F_loss.sum()
elif self.reduction == 'mean':
return F_loss.mean()
new_train_df = train_df.copy()
new_test_df = test_df.copy()<groupby>
submission_ver = '0002'
arch = [EfficientNet.from_pretrained('efficientnet-b0', num_classes=2)]
fc_size = 1280
lin_size = 1000
n_folds = 5
size = [256]
bs = 32
stage_1_epochs = 3
lr1 = [1e-1]
lr_eff_1 = [1e-3]
is_stage_2 = False
stage_2_epochs = 4
lr2 = [slice(1e-7, 1e-4)]
lr_eff_2 = [slice(1e-4, 1e-3)]
custom_loss = True
loss_func = FocalLoss()
w_decay = 0.01
use_fp16 = True
num_wkrs = os.cpu_count()
use_tta = True
submit_after_train = False
oversampling_flag = True
oversampling_size = 5
preds_to_int = False
smooth_preds = False
smooth_alpha = 0.01
def total_famil_members(df):
return df['SibSp']+df['Parch']+1<feature_engineering>
num_classes = len(np.unique(train_df['target']))
num_classes
for data in [new_train_df, new_test_df]:
data['Members'] = total_famil_members(data)
 data['Adjusted_Fare'] = data['Fare']/data['Members']
data['Title'] = data['Name'].apply(get_title )<feature_engineering>
test_df['image_name'] = '512x512-test/512x512-test/' + test_df['image_name'] + '.jpg'
for data in [new_train_df, new_test_df]:
data['Title'] = data['Title'].replace(['Lady', 'Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 data['Title'] = data['Title'].replace('Mlle', 'Miss')
 data['Title'] = data['Title'].replace('Ms', 'Miss')
 data['Title'] = data['Title'].replace('Mme', 'Mrs')
<data_type_conversions>
test_data = ImageList.from_df(test_df, img_path)
 test_data
title_mapping = {"Mr": 1, "Miss": 2, "Mrs": 3, "Master": 4, "Rare": 5}
for data in [new_train_df, new_test_df]:
data['Title'] = data['Title'].map(title_mapping)
 data['Title'] = data['Title'].fillna(0)
 data['Embarked'] = data['Embarked'].fillna('S')
 data['Sex'] = data['Sex'].map({'female': 0, 'male': 1} ).astype(int)
<data_type_conversions>
labels_df = train_df[['image_id', 'target']].copy()
labels_df.head()
for data in [new_train_df, new_test_df]:
data['Embarked'] = data['Embarked'].map({'S': 0, 'C': 1, 'Q': 2} ).astype(int )<split>
def k_fold(df, num_fld, seed = seed):
for fold in range(num_fld):
df.loc[df.fold == fold, f'is_valid_{fold}'] = True
df.loc[df.fold != fold, f'is_valid_{fold}'] = False
data_train = new_train_df[~new_train_df['Age'].isnull() ]
data_test = new_test_df[~new_test_df['Age'].isnull() ]<prepare_x_and_y>
k_fold(train_df, n_folds, seed )
X_train = data_train[['Pclass','Sex','SibSp','Parch','Title']]
y_train = data_train['Age']
X_test = data_test[['Pclass','Sex','SibSp','Parch','Title']]
y_test = data_test['Age']
X_train.head()<train_model>
def oversample(fld, df, os_size, num_fld=5):
train_df_fld = df.loc[df['fold'] != fld]
valid_df_fld = df.loc[df['fold'] == fld]
train_df_md = train_df_fld.loc[train_df_fld['target'] == 1]
if os_size == 'auto':
os_size = int(np.floor(train_df_fld.loc[train_df_fld['target'] == 0]['target'].value_counts() [0]/train_df_fld.loc[train_df_fld['target'] == 1]['target'].value_counts() [1]))
train_df_md = train_df_md.append([train_df_md] *(os_size - 1))
full_df_fld = pd.concat([train_df_fld, train_df_md, valid_df_fld] ).sample(frac=1.0, random_state=seed ).reset_index(drop=True)
 return full_df_fld
for x in range(n_folds):
fold_df = train_df.copy()
if oversampling_flag:
fold_df = oversample(x, train_df, oversampling_size)
 globals() ['src_%s' %x] =(ImageList.from_df(fold_df, img_path + '/512x512-dataset-melanoma', folder='512x512-dataset-melanoma', suffix='.jpg' ).split_from_df(col =(3 + x)))
model_age_prediction = RandomForestRegressor(n_estimators=900, max_depth=6, min_samples_leaf=0.001, random_state=100)
 model_age_prediction.fit(X_train, y_train )<predict_on_test>
def get_data(fold, size, bs, padding_mode='reflection'):
return(globals() ['src_%s' %fold].label_from_df(cols='target')
.add_test(test_data)
.transform(tfms, size=size, padding_mode=padding_mode)
.databunch(bs=bs, num_workers = num_wkrs ).normalize(imagenet_stats))
y_predict = model_age_prediction.predict(X_test )<compute_test_metric>
def preds_smoothing(encodings , alpha):
K = encodings.shape[1]
y_ls =(1 - alpha)* encodings + alpha / K
return y_ls
print('test score is: {}'.format(r2_score(y_test, y_predict)))
 print('training score is: {}'.format(r2_score(y_train, model_age_prediction.predict(X_train))))<predict_on_test>
def print_metrics(val_preds, val_labels):
targs, preds = LongTensor([]), Tensor([])
 val_preds = F.softmax(val_preds, dim=1)[:,-1]
preds = torch.cat(( preds, val_preds.cpu()))
targs = torch.cat(( targs, val_labels.cpu().long()))
print('AUCROC = ' + str(auc_roc_score(preds, targs ).item()))
train_missing_predicted = model_age_prediction.predict(new_train_df[new_train_df['Age'].isnull() ][['Pclass','Sex','SibSp','Parch','Title']])
 test_missing_predicted = model_age_prediction.predict(new_test_df[new_test_df['Age'].isnull() ][['Pclass','Sex','SibSp','Parch','Title']] )<filter>
gc.collect()
<filter><EOS> 
for model in arch:
if hasattr(model, '__name__'):
model_name = model.__name__
else:
model_name = "EfficientNet"
globals() [model_name + '___val_preds'] = []
globals() [model_name + '___val_labels'] = []
globals() [model_name + '___test_preds'] = []
print(f'/////////////////////////////////////////////////////')
 print(f'//////////////// MODEL: {model_name} ////////////////')
 print(f'/////////////////////////////////////////////////////
')
 for fld in range(n_folds):
print(f'
//////////////// FOLD {fld} ////////////////
')
 for sz in size:
print(f'-------- Size: {sz} --------
')
 print("Preparing data & applying settings...
 ")
 data = get_data(fld, sz, bs)
 gc.collect()
if sz == size[0]:
if model_name != "EfficientNet":
learn = cnn_learner(data, model, metrics=[AUROC() ], bn_final=True)
 else:
model._fc = nn.Sequential(nn.Linear(fc_size, lin_size, bias=True),
nn.ReLU() ,
nn.Dropout(p=0.5),
nn.Linear(lin_size, num_classes, bias = True))
learn = Learner(data, model, metrics=[AUROC() ])
 learn = learn.split([learn.model._conv_stem,learn.model._blocks,learn.model._conv_head])
 else:
learn.data = data
if custom_loss:
learn.loss_func = loss_func
if use_fp16:
learn = learn.to_fp16()
if model_name != "EfficientNet":
lr = lr1[size.index(sz)]
else:
lr = lr_eff_1[size.index(sz)]
print("Data is ready.Learning - Stage 1...")
 learn.fit_one_cycle(stage_1_epochs, slice(lr), wd=w_decay)
 if is_stage_2:
print("Stage 1 complete.Stage 2...")
 if model_name != "EfficientNet":
lr = lr2[size.index(sz)]
else:
lr = lr_eff_2[size.index(sz)]
learn.unfreeze()
learn.fit_one_cycle(stage_2_epochs, lr, wd=w_decay)
 print(f"Final learning is over for size {sz}
")
 val_preds, val_labels = learn.get_preds()
print_metrics(val_preds, val_labels)
 globals() [model_name + '___val_preds'].append(val_preds)
 globals() [model_name + '___val_labels'].append(val_labels)
 if use_tta == False:
print(f'
Saving test results for fold {fld}...')
 test_preds, _ = learn.get_preds(DatasetType.Test)
 globals() [model_name + '___test_preds'].append(test_preds[:, 1])
 else:
print(f'
Saving test TTA results for fold {fld}...')
 test_preds, _ = learn.TTA(ds_type=DatasetType.Test)
 globals() [model_name + '___test_preds'].append(test_preds[:, 1])
 print("Done!")
 gc.collect()
print("All folds are trained successfully
")
 print_metrics(torch.cat(globals() [model_name + '___val_preds']), torch.cat(globals() [model_name + '___val_labels']))
print("
Writing submission file...")
 test_df_output = pd.concat([test_df, pd.DataFrame(np.mean(np.stack(globals() [model_name + '___test_preds']), axis=0), columns=['target'])], axis=1)
 sample_df.iloc[:,1:] = test_df_output.iloc[:,5]
sample_df.to_csv(f'submission_v{submission_ver}.csv', index=False)
 print(f'File is ready to submit
')
 if submit_after_train:
print("Submitting to Kaggle
")
 !kaggle competitions submit -c siim-isic-melanoma-classification -f 'submission_v{submission_ver}.csv' -m 'Md: {model_name}, Fd: {n_folds}, Bs: {bs}, Sz: {size[0]}, Os: {oversampling_flag}, TTa: {use_tta}'
print('
')