Kaggle competition: SIIM-ISIC Melanoma Classification
<SOS> metric: MCAUC Kaggle data source: siim-isic-melanoma-classification<feature_engineering>
!pip install iterative-stratification
!pip install efficientnet-pytorch
train_df['Survived'] = train['Survived']<feature_engineering>
import random
import numpy as np
import pandas as pd
import torch
import PIL.Image as pil
import matplotlib.pyplot as plt
from fastai.vision import *
from efficientnet_pytorch import EfficientNet
from sklearn.model_selection import StratifiedKFold
import os
train_df['Name_len'] = train_df.Name.apply(lambda x: len(x))<feature_engineering>
warnings.filterwarnings("ignore", category=UserWarning, module="torch.nn.functional" )
train_df['FamilySize'] = train_df['SibSp'] + train_df['Parch']<filter>
!pip install torch==1.4.0 torchvision==0.5.0
train_df.loc[(train_df['Pclass']==2)&(train_df['Fare'] == 0)]<feature_engineering>
%reload_ext autoreload
%autoreload 2
%matplotlib inline
train_test['Last_Name'] = train_test.Name.apply(lambda x: x.split('(')[0].split('"')[0].split(' ')[-1] )<feature_engineering>
seed = 42
def random_seed(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 os.environ['PYTHONHASHSEED'] = str(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
random_seed(seed )
train_test['NameLen'] = train_test.Name.apply(lambda x : len(x))<data_type_conversions>
path = '/kaggle/input/siim-isic-melanoma-classification'
path
train_test['Sex'] = train_test['Sex'].map({'female': 1, 'male': 0} ).astype(int )<feature_engineering>
img_path = '/kaggle/input/melanoma-merged-external-data-512x512-jpeg'
img_path
train_test['FamilySize'] = train_test['Parch'] + train_test['SibSp']<feature_engineering>
train_df = pd.read_csv(img_path + '/folds_13062020.csv')
 train_df.head()
train_test['Alone']=0
train_test.loc[(train_test.FamilySize==0),'Alone']=1<feature_engineering>
test_df = pd.read_csv(path + '/test.csv')
 test_df.head()
train_test['Small_Family']=0
train_test.loc[(train_test.FamilySize<=3)&(train_test.FamilySize!=0),'Small_Family']=1<feature_engineering>
sample_df = pd.read_csv(path + '/sample_submission.csv')
 sample_df.head()
train_test['Big_Family']=0
train_test.loc[(train_test.FamilySize>=4),'Big_Family']=1<drop_column>
tfms = get_transforms(flip_vert=True, max_rotate=15, max_zoom=1.2, max_lighting=0.3, max_warp=0, p_affine=0, p_lighting=0.8 )
train_test.drop(['Name','SibSp','Parch','Ticket','FamilySize'],axis=1,inplace=True )<categorify>
class FocalLoss(nn.Module):
def __init__(self, gamma=2., reduction='mean'):
super().__init__()
self.gamma = gamma
self.reduction = reduction
def forward(self, inputs, targets):
CE_loss = nn.CrossEntropyLoss(reduction='none' )(inputs, targets)
 pt = torch.exp(-CE_loss)
 F_loss =(( 1 - pt)**self.gamma)* CE_loss
if self.reduction == 'sum':
return F_loss.sum()
elif self.reduction == 'mean':
return F_loss.mean()
train_test = pd.get_dummies(train_test )<drop_column>
num_classes = len(np.unique(train_df['target']))
num_classes
train = train_test[train_test.is_train == 1].drop(['is_train'],axis=1)
 test = train_test[train_test.is_train == 0].drop(['is_train'],axis=1 )<import_modules>
test_df['image_name'] = '512x512-test/512x512-test/' + test_df['image_name'] + '.jpg'
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier
from sklearn import svm
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.svm import LinearSVC
from sklearn import metrics
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix<split>
test_data = ImageList.from_df(test_df, img_path)
 test_data
X_train, X_test, Y_train, Y_test=train_test_split(train,train_df['Survived'],test_size=0.33,random_state=3 )<compute_train_metric>
labels_df = train_df[['image_id', 'target']].copy()
labels_df.head()
model = LogisticRegression(random_state=51)
 model.fit(X_train, Y_train)
 prediction=model.predict(X_test)
 print('Accuracy for rbf LogisticRegression is ',np.mean(cross_val_score(model, train, train_df['Survived'], cv=3)) )<find_best_model_class>
def k_fold(df, num_fld, seed = seed):
for fold in range(num_fld):
df.loc[df.fold == fold, f'is_valid_{fold}'] = True
df.loc[df.fold != fold, f'is_valid_{fold}'] = False
model = RandomForestClassifier(n_estimators=30,random_state=51)
 model.fit(X_train, Y_train)
 prediction=model.predict(X_test)
 print('Accuracy for rbf RandomForestClassifier is ',np.mean(cross_val_score(model, train, train_df['Survived'], cv=3)) )<find_best_model_class>
k_fold(train_df, n_folds, seed )
model=svm.SVC(kernel='linear',C=0.1,gamma=0.1,random_state=51)
 model.fit(X_train,Y_train)
 prediction=model.predict(X_test)
 print('Accuracy for linear SVM is',np.mean(cross_val_score(model, train, train_df['Survived'], cv=3)) )<compute_train_metric>
def oversample(fld, df, os_size, num_fld=5):
train_df_fld = df.loc[df['fold'] != fld]
valid_df_fld = df.loc[df['fold'] == fld]
train_df_md = train_df_fld.loc[train_df_fld['target'] == 1]
if os_size == 'auto':
os_size = int(np.floor(train_df_fld.loc[train_df_fld['target'] == 0]['target'].value_counts() [0]/train_df_fld.loc[train_df_fld['target'] == 1]['target'].value_counts() [1]))
train_df_md = train_df_md.append([train_df_md] *(os_size - 1))
full_df_fld = pd.concat([train_df_fld, train_df_md, valid_df_fld] ).sample(frac=1.0, random_state=seed ).reset_index(drop=True)
 return full_df_fld
for x in range(n_folds):
fold_df = train_df.copy()
if oversampling_flag:
fold_df = oversample(x, train_df, oversampling_size)
 globals() ['src_%s' %x] =(ImageList.from_df(fold_df, img_path + '/512x512-dataset-melanoma', folder='512x512-dataset-melanoma', suffix='.jpg' ).split_from_df(col =(3 + x)))
model=LinearSVC(random_state=51)
 model.fit(X_train, Y_train)
 prediction=model.predict(X_test)
 print('The accuracy of the NaiveBayes is',np.mean(cross_val_score(model, train, train_df['Survived'], cv=3)) )<save_to_csv>
def get_data(fold, size, bs, padding_mode='reflection'):
return(globals() ['src_%s' %fold].label_from_df(cols='target')
.add_test(test_data)
.transform(tfms, size=size, padding_mode=padding_mode)
.databunch(bs=bs, num_workers = num_wkrs ).normalize(imagenet_stats))
model=RandomForestClassifier(n_estimators=30)
 model.fit(X_train, Y_train)
 prediction = model.predict(test)
 submission = pd.read_csv('.. /input/gender_submission.csv')
 submission.Survived = prediction
submission.to_csv('submission.csv',index=False )<import_modules>
def preds_smoothing(encodings , alpha):
K = encodings.shape[1]
y_ls =(1 - alpha)* encodings + alpha / K
return y_ls
GridSearchCV,
RandomizedSearchCV,
train_test_split 
)
 FunctionTransformer,
OneHotEncoder,
OrdinalEncoder,
QuantileTransformer, 
)
<compute_test_metric>
def print_metrics(val_preds, val_labels):
targs, preds = LongTensor([]), Tensor([])
 val_preds = F.softmax(val_preds, dim=1)[:,-1]
preds = torch.cat(( preds, val_preds.cpu()))
targs = torch.cat(( targs, val_labels.cpu().long()))
print('AUCROC = ' + str(auc_roc_score(preds, targs ).item()))
pd.set_option('display.max_columns', None)
 sns.set()
def md(text: str):
display(Markdown(text))
def rmsle(y_true, y_pred):
return np.sqrt(mean_squared_log_error(y_true, y_pred))
def prepend_keys(dict_: dict, prepend: str)-> dict:
return {f"{prepend}{key}": value for key, value in dict_.items() }<load_from_csv>
gc.collect()
data_dir = "/kaggle/input/house-prices-advanced-regression-techniques"
data = pd.read_csv(os.path.join(data_dir, "train.csv"), index_col="Id")
 test = pd.read_csv(os.path.join(data_dir, "test.csv"), index_col="Id")
 sample_submission = pd.read_csv(os.path.join(data_dir, "sample_submission.csv"))
X = data.drop(columns=["SalePrice"])
 y = data["SalePrice"]
md(f"The total number of training observations is: `{len(X)}`")
 X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33)
 md(f"The number of observations with which we do hyperparameter tuning is: `{len(X_train)}`" )<create_dataframe>
for model in arch:
if hasattr(model, '__name__'):
model_name = model.__name__
else:
model_name = "EfficientNet"
globals() [model_name + '___val_preds'] = []
globals() [model_name + '___val_labels'] = []
globals() [model_name + '___test_preds'] = []
print(f'/////////////////////////////////////////////////////')
 print(f'//////////////// MODEL: {model_name} ////////////////')
 print(f'/////////////////////////////////////////////////////
')
 for fld in range(n_folds):
print(f'
//////////////// FOLD {fld} ////////////////
')
 for sz in size:
print(f'-------- Size: {sz} --------
')
 print("Preparing data & applying settings...
 ")
 data = get_data(fld, sz, bs)
 gc.collect()
if sz == size[0]:
if model_name != "EfficientNet":
learn = cnn_learner(data, model, metrics=[AUROC() ], bn_final=True)
 else:
model._fc = nn.Sequential(nn.Linear(fc_size, lin_size, bias=True),
nn.ReLU() ,
nn.Dropout(p=0.5),
nn.Linear(lin_size, num_classes, bias = True))
learn = Learner(data, model, metrics=[AUROC() ])
 learn = learn.split([learn.model._conv_stem,learn.model._blocks,learn.model._conv_head])
 else:
learn.data = data
if custom_loss:
learn.loss_func = loss_func
if use_fp16:
learn = learn.to_fp16()
if model_name != "EfficientNet":
lr = lr1[size.index(sz)]
else:
lr = lr_eff_1[size.index(sz)]
print("Data is ready.Learning - Stage 1...")
 learn.fit_one_cycle(stage_1_epochs, slice(lr), wd=w_decay)
 if is_stage_2:
print("Stage 1 complete.Stage 2...")
 if model_name != "EfficientNet":
lr = lr2[size.index(sz)]
else:
lr = lr_eff_2[size.index(sz)]
learn.unfreeze()
learn.fit_one_cycle(stage_2_epochs, lr, wd=w_decay)
 print(f"Final learning is over for size {sz}
")
 val_preds, val_labels = learn.get_preds()
print_metrics(val_preds, val_labels)
 globals() [model_name + '___val_preds'].append(val_preds)
 globals() [model_name + '___val_labels'].append(val_labels)
 if use_tta == False:
print(f'
Saving test results for fold {fld}...')
 test_preds, _ = learn.get_preds(DatasetType.Test)
 globals() [model_name + '___test_preds'].append(test_preds[:, 1])
 else:
print(f'
Saving test TTA results for fold {fld}...')
 test_preds, _ = learn.TTA(ds_type=DatasetType.Test)
 globals() [model_name + '___test_preds'].append(test_preds[:, 1])
 print("Done!")
 gc.collect()
print("All folds are trained successfully
")
 print_metrics(torch.cat(globals() [model_name + '___val_preds']), torch.cat(globals() [model_name + '___val_labels']))
print("
Writing submission file...")
 test_df_output = pd.concat([test_df, pd.DataFrame(np.mean(np.stack(globals() [model_name + '___test_preds']), axis=0), columns=['target'])], axis=1)
 sample_df.iloc[:,1:] = test_df_output.iloc[:,5]
sample_df.to_csv(f'submission_v{submission_ver}.csv', index=False)
 print(f'File is ready to submit
')
 if submit_after_train:
print("Submitting to Kaggle
")
 !kaggle competitions submit -c siim-isic-melanoma-classification -f 'submission_v{submission_ver}.csv' -m 'Md: {model_name}, Fd: {n_folds}, Bs: {bs}, Sz: {size[0]}, Os: {oversampling_flag}, TTa: {use_tta}'
print('
')