Kaggle competition: SIIM-ISIC Melanoma Classification
<SOS> metric: MCAUC Kaggle data source: siim-isic-melanoma-classification<train_on_grid>
!pip install efficientnet-pytorch
svm_linear_grid = {
'epsilon' : np.linspace(0, 0.5, num=200),
}
svm_linear_model = SVR(kernel='linear')
 svm_linear_best = RandomizedSearchCV(svm_linear_model, svm_linear_grid, verbose=2, cv=3, n_jobs=-1,
scoring='neg_mean_squared_error' ).fit(X_train_cleaned, y_train ).best_estimator_
svm_linear_best.fit(X_train_cleaned, y_train)
 analys_model(svm_linear_best)
<train_on_grid>
from pathlib import Path
import pandas as pd
from torch.utils.data import Dataset,DataLoader
from PIL import Image
from torchvision import transforms as T
import torch.nn as nn
import torch
import torch.nn.functional as F
from sklearn.model_selection import GroupKFold
import numpy as np
from fastprogress.fastprogress import master_bar, progress_bar
from sklearn.metrics import accuracy_score, roc_auc_score
from efficientnet_pytorch import EfficientNet
from torchvision import models
import pdb
import albumentations as A
from albumentations.pytorch.transforms import ToTensor
import matplotlib.pyplot as plt
import pickle
svm_poly_grid = {
'epsilon' : np.linspace(0, 0.5, num=200),
}
svm_poly_model = SVR(kernel='poly')
 svm_poly_best = RandomizedSearchCV(svm_poly_model, svm_poly_grid, verbose=2, cv=3, n_jobs=-1,
scoring='neg_mean_squared_error' ).fit(X_train_cleaned, y_train ).best_estimator_
svm_poly_best.fit(X_train_cleaned, y_train)
 analys_model(svm_poly_best )<train_on_grid>
path = Path('.. /input/jpeg-melanoma-256x256/')
 df_path = Path('.. /input/melanoma-256x256/')
 im_sz = 256
bs = 16
dt_model = DecisionTreeRegressor()
dt_model.fit(X_train_cleaned, y_train)
 max_features = [int(x)for x in np.linspace(1, 270, num = 30)]
max_depth = [1, 2, 4, 5, 6, 9, 10, 12 , None]
min_samples_leaf = [int(x)for x in np.linspace(1, 10, num = 5)]
random_grid = {'max_features': max_features,
'max_depth': max_depth,
'min_samples_leaf': min_samples_leaf}
random_search = RandomizedSearchCV(estimator = dt_model, param_distributions = random_grid,
n_iter = 100, cv = 3, verbose=2, random_state=42, n_jobs = -1, scoring="neg_mean_squared_error")
 random_search.fit(X_train_cleaned, y_train)
 dt_best = random_search.best_estimator_
dt_best.fit(X_train_cleaned, y_train)
 analys_model(dt_best )<train_model>
train_fnames = list_files(path/'train')
 df = pd.read_csv(df_path/'train.csv')
 df.head()
dt_ada_model = AdaBoostRegressor(dt_best, n_estimators=200, learning_rate=0.5)
 dt_ada_model.fit(X_train_cleaned, y_train)
 analys_model(dt_ada_model )<define_search_space>
df.target.value_counts() ,df.shape
n_estimators = [int(x)for x in np.linspace(start = 100, stop = 2000, num = 5)]
max_features = ['auto', 10, 20, 40, 90, 140, 200, 250]
max_depth = [int(x)for x in np.linspace(1, 1000, num = 20)]
max_depth.append(None)
 min_samples_leaf = [5, 10, 15]
bootstrap = [True, False]
random_grid = {'n_estimators': n_estimators,
'max_features': max_features,
'max_depth': max_depth,
'min_samples_leaf': min_samples_leaf,
'bootstrap': bootstrap}
rf = RandomForestRegressor()<train_on_grid>
def get_train_val_split(df):
df = df[df.tfrecord != -1].reset_index(drop=True)
 train_tf_records = list(range(len(df.tfrecord.unique())))[:12]
split_cond = df.tfrecord.apply(lambda x: x in train_tf_records)
 train_df = df[split_cond].reset_index()
valid_df = df[~split_cond].reset_index()
return train_df,valid_df
random_search = RandomizedSearchCV(estimator = rf, param_distributions = random_grid,
n_iter = 100, cv = 3, verbose=2, random_state=42, n_jobs = -1, scoring="neg_mean_squared_error")
 random_search.fit(X_train_cleaned, y_train )<train_model>
class MelanomaDataset(Dataset):
def __init__(self,df,im_path,transforms=None,is_test=False):
self.df = df
self.im_path = im_path
self.transforms = transforms
self.is_test = is_test
def __getitem__(self,idx):
img_path = f"{self.im_path}/{self.df.iloc[idx]['image_name']}.jpg"
img = Image.open(img_path)
 if self.transforms:
img = self.transforms(**{"image": np.array(img)})["image"]
if self.is_test:
return img
target = self.df.iloc[idx]['target']
return img,torch.tensor([target],dtype=torch.float32)
 def __len__(self):
return self.df.shape[0]
rf_best = random_search.best_estimator_
rf_best.fit(X_train_cleaned, y_train)
 analys_model(rf_best )<train_model>
class MelanomaEfficientNet(nn.Module):
def __init__(self,model_name='efficientnet-b0',pool_type=F.adaptive_avg_pool2d):
super().__init__()
self.pool_type = pool_type
self.backbone = EfficientNet.from_pretrained(model_name)
 in_features = getattr(self.backbone,'_fc' ).in_features
self.classifier = nn.Linear(in_features,1)
 def forward(self,x):
features = self.pool_type(self.backbone.extract_features(x),1)
 features = features.view(x.size(0),-1)
 return self.classifier(features )
dt_ada_model = AdaBoostRegressor(rf_best, n_estimators=10, learning_rate=0.5)
 dt_ada_model.fit(X_train_cleaned, y_train)
 analys_model(dt_ada_model )<train_model>
def get_device() :
return torch.device("cuda")if torch.cuda.is_available() else torch.device("cpu")
 def get_model(model_name='efficientnet-b0',lr=1e-5,wd=0.01,freeze_backbone=False,opt_fn=torch.optim.AdamW,device=None):
device = device if device else get_device()
model = MelanomaEfficientNet(model_name=model_name)
 if freeze_backbone:
for parameter in model.backbone.parameters() :
parameter.requires_grad = False
opt = opt_fn(model.parameters() ,lr=lr,weight_decay=wd)
 model = model.to(device)
 return model, opt
def training_step(xb,yb,model,loss_fn,opt,device,scheduler):
xb,yb = xb.to(device), yb.to(device)
 out = model(xb)
 opt.zero_grad()
loss = loss_fn(out,yb)
 loss.backward()
opt.step()
scheduler.step()
return loss.item()
def validation_step(xb,yb,model,loss_fn,device):
xb,yb = xb.to(device), yb.to(device)
 out = model(xb)
 loss = loss_fn(out,yb)
 out = torch.sigmoid(out)
 return loss.item() ,out
def get_data(train_df,valid_df,train_tfms,test_tfms,bs):
train_ds = MelanomaDataset(df=train_df,im_path=path/'train',transforms=train_tfms)
 valid_ds = MelanomaDataset(df=valid_df,im_path=path/'train',transforms=test_tfms)
 train_dl = DataLoader(dataset=train_ds,batch_size=bs,shuffle=True,num_workers=4)
 valid_dl = DataLoader(dataset=valid_ds,batch_size=bs*2,shuffle=False,num_workers=4)
 return train_dl,valid_dl
voting_model = VotingRegressor(
 estimators=[('ridge', ridge_best),('lasso', lasso_best),('elastic', elastic_best),('svm', svm_linear_best), 
('rf', gradient_best),('dt', dt_ada_model)],
n_jobs=-1 
)
 voting_model.fit(X_train_cleaned, y_train)
 analys_model(voting_model )<train_model>
def fit(epochs,model,train_dl,valid_dl,opt,device=None,loss_fn=F.binary_cross_entropy_with_logits):
device = device if device else get_device()
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(opt, len(train_dl)*epochs)
 val_rocs = []
mb = master_bar(range(epochs))
mb.write(['epoch','train_loss','valid_loss','val_roc'],table=True)
 for epoch in mb:
trn_loss,val_loss = 0.0,0.0
val_preds = np.zeros(( len(valid_dl.dataset),1))
val_targs = np.zeros(( len(valid_dl.dataset),1))
model.train()
for xb,yb in progress_bar(train_dl,parent=mb):
trn_loss += training_step(xb,yb,model,loss_fn,opt,device,scheduler)
 trn_loss /= mb.child.total
model.eval()
with torch.no_grad() :
for i,(xb,yb)in enumerate(progress_bar(valid_dl,parent=mb)) :
loss,out = validation_step(xb,yb,model,loss_fn,device)
 val_loss += loss
bs = xb.shape[0]
val_preds[i*bs:i*bs+bs] = out.cpu().numpy()
val_targs[i*bs:i*bs+bs] = yb.cpu().numpy()
val_loss /= mb.child.total
val_roc = roc_auc_score(val_targs.reshape(-1),val_preds.reshape(-1))
val_rocs.append(val_roc)
 mb.write([epoch,f'{trn_loss:.6f}',f'{val_loss:.6f}',f'{val_roc:.6f}'],table=True)
 return model,val_rocs
class StackingAveragedModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, base_models, meta_model, n_folds=5):
self.base_models = base_models
self.meta_model = meta_model
self.n_folds = n_folds
def fit(self, X, y=None):
X = X.values
y = y.values
self.base_models_ = [list() for x in self.base_models]
self.meta_model_ = clone(self.meta_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=42)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(X, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 instance.fit(X[train_index], y[train_index])
 y_pred = instance.predict(X[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
self.meta_model_.fit(out_of_fold_predictions, y)
 return self
def get_metafeatures(self, X):
return np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_ ])
 def predict(self, X):
meta_features = np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_ ])
 return self.meta_model_.predict(meta_features )<train_model>
df = pd.read_csv(df_path/'train.csv')
 train_df,valid_df = get_train_val_split(df)
 train_tfms,test_tfms = get_augmentations(p=0.5)
 train_dl,valid_dl = get_data(train_df,valid_df,train_tfms,test_tfms,bs)
 model, opt = get_model(model_name='efficientnet-b5',lr=1e-4,wd=1e-4 )
stacked_averaged_models = StackingAveragedModels(base_models = [gradient_best, dt_ada_model, elastic_best, lasso_lars_best, svm_linear_best],
meta_model = LinearRegression())
 stacked_averaged_models.fit(X_train_cleaned, y_train )<create_dataframe>
device = torch.device("cuda")if torch.cuda.is_available() else torch.device("cpu" )
meta_features = pd.DataFrame(stacked_averaged_models.get_metafeatures(X_train_cleaned)) 
<predict_on_test>
test_tfms = A.Compose([
A.RandomRotate90(p=p),
A.Flip(p=p),
A.OneOf([
A.RandomBrightnessContrast(brightness_limit=0.2,
contrast_limit=0.2, 
),
A.HueSaturationValue(
 hue_shift_limit=20,
sat_shift_limit=50,
val_shift_limit=50)
 ], p=p),
A.OneOf([
A.IAAAdditiveGaussianNoise() ,
A.GaussNoise() ,
], p=p),
ToTensor(normalize=imagenet_stats)
 ] )
X_test_clean = full_pipeline.transform(X_test)
 predictions = stacked_averaged_models.predict(X_test_clean)
 final_prediction = pd.DataFrame({'Id': X_test['Id'],
'SalePrice': np.expm1(predictions)} )<save_to_csv>
test_df = pd.read_csv(path/'test.csv')
 model, opt = get_model(model_name='efficientnet-b5',lr=1e-4,wd=1e-4)
 model.load_state_dict(torch.load(f'.. /input/melanomaefficientnetb5/effb5.pth',map_location=device))
test_ds = MelanomaDataset(df=test_df,im_path=path/'test',transforms=test_tfms,is_test=True)
 test_dl = DataLoader(dataset=test_ds,batch_size=bs*2,shuffle=False,num_workers=4 )
final_prediction.to_csv('prediction.csv', index=False )<set_options>
def get_preds(model,device=None,tta=3):
if device is None:
device = torch.device("cuda")if torch.cuda.is_available() else torch.device("cpu")
 preds = np.zeros(len(test_ds))
for tta_id in range(tta):
test_preds = []
with torch.no_grad() :
for xb in test_dl:
xb = xb.to(device)
 out = model(xb)
 out = torch.sigmoid(out)
 test_preds.extend(out.cpu().numpy())
 preds += np.array(test_preds ).reshape(-1)
 print(f'TTA {tta_id}')
 preds /= tta
return preds
preds = get_preds(model,tta=25 )
<load_from_csv><EOS> 
subm = pd.read_csv(path/'sample_submission.csv')
 subm.target = preds
subm.to_csv('submission.csv',index=False )