Kaggle competition: APTOS 2019 Blindness Detection
<SOS> metric: QuadraticWeightedKappa Kaggle data source: aptos-2019-blindness-detection<set_options>
t_start = time.time()
%matplotlib inline<set_options>
%reload_ext autoreload
%autoreload 2
%matplotlib inline
warnings.simplefilter('ignore')
 torch.manual_seed(47)
 np.random.seed(47 )<set_options>
from fastai.vision import *
from fastai.metrics import error_rate
from fastai.callbacks import *
device = torch.device("cuda" if torch.cuda.is_available() else "cpu" )<create_dataframe>
from functools import partial
from sklearn import metrics
from collections import Counter
class MelanomaDataset(Dataset):
def __init__(self, df: pd.DataFrame, imfolder: str, train: bool = True, transforms = None, meta_features = None):

 self.df = df
self.imfolder = imfolder
self.transforms = transforms
self.train = train
self.meta_features = meta_features
def __getitem__(self, index):
im_path = os.path.join(self.imfolder, self.df.iloc[index]['image_name'] + '.jpg')
 x = cv2.imread(im_path)
 meta = np.array(self.df.iloc[index][self.meta_features].values, dtype=np.float32)
 if self.transforms:
x = self.transforms(x)
 if self.train:
y = self.df.iloc[index]['target']
return(x, meta), y
else:
return(x, meta)
 def __len__(self):
return len(self.df)
 class Net(nn.Module):
def __init__(self, arch, n_meta_features: int):
super(Net, self ).__init__()
self.arch = arch
if 'ResNet' in str(arch.__class__):
self.arch.fc = nn.Linear(in_features=512, out_features=500, bias=True)
 if 'EfficientNet' in str(arch.__class__):
self.arch._fc = nn.Linear(in_features=1280, out_features=500, bias=True)
 self.meta = nn.Sequential(nn.Linear(n_meta_features, 500),
nn.BatchNorm1d(500),
nn.ReLU() ,
nn.Dropout(p=0.2),
nn.Linear(500, 250),
nn.BatchNorm1d(250),
nn.ReLU() ,
nn.Dropout(p=0.2))
self.ouput = nn.Linear(500 + 250, 1)
 def forward(self, inputs):

 x, meta = inputs
cnn_features = self.arch(x)
 meta_features = self.meta(meta)
 features = torch.cat(( cnn_features, meta_features), dim=1)
 output = self.ouput(features)
 return output<categorify>
import os
import PIL
import cv2
class AdvancedHairAugmentation:

 def __init__(self, hairs: int = 5, hairs_folder: str = ""):
self.hairs = hairs
self.hairs_folder = hairs_folder
def __call__(self, img):

 n_hairs = random.randint(0, self.hairs)
 if not n_hairs:
return img
height, width, _ = img.shape
hair_images = [im for im in os.listdir(self.hairs_folder)if 'png' in im]
for _ in range(n_hairs):
hair = cv2.imread(os.path.join(self.hairs_folder, random.choice(hair_images)))
 hair = cv2.flip(hair, random.choice([-1, 0, 1]))
hair = cv2.rotate(hair, random.choice([0, 1, 2]))
h_height, h_width, _ = hair.shape
roi_ho = random.randint(0, img.shape[0] - hair.shape[0])
 roi_wo = random.randint(0, img.shape[1] - hair.shape[1])
 roi = img[roi_ho:roi_ho + h_height, roi_wo:roi_wo + h_width]
img2gray = cv2.cvtColor(hair, cv2.COLOR_BGR2GRAY)
 ret, mask = cv2.threshold(img2gray, 10, 255, cv2.THRESH_BINARY)
 mask_inv = cv2.bitwise_not(mask)
 img_bg = cv2.bitwise_and(roi, roi, mask=mask_inv)
 hair_fg = cv2.bitwise_and(hair, hair, mask=mask)
 dst = cv2.add(img_bg, hair_fg)
 img[roi_ho:roi_ho + h_height, roi_wo:roi_wo + h_width] = dst
return img
def __repr__(self):
return f'{self.__class__.__name__}(hairs={self.hairs}, hairs_folder="{self.hairs_folder}")'<categorify>
dat_path = Path('.. /input/aptos2019-blindness-detection')
 trn_path = Path('.. /input/aptos2019-blindness-detection/train_images')
(dat_path,trn_path )
class Microscope:

 def __init__(self, p: float = 0.5):
self.p = p
def __call__(self, img):

 if random.random() < self.p:
circle = cv2.circle(( np.ones(img.shape)* 255 ).astype(np.uint8), 
(img.shape[0]//2, img.shape[1]//2),
random.randint(img.shape[0]//2 - 3, img.shape[0]//2 + 15), 
(0, 0, 0),
-1)
 mask = circle - 255
img = np.multiply(img, mask)
 return img
def __repr__(self):
return f'{self.__class__.__name__}(p={self.p})'<choose_model_class>
train_df = pd.read_csv(dat_path/'train.csv')
 train_df.shape
arch = EfficientNet.from_pretrained('efficientnet-b1')
 skf = GroupKFold(n_splits=5 )<load_from_csv>
IMG_SIZE = 512
def _load_format(path, convert_mode, after_open)->Image:
image = cv2.imread(path)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
 image = crop_image_from_gray(image)
 image = cv2.resize(image,(IMG_SIZE, IMG_SIZE))
image=cv2.addWeighted(image,4, cv2.GaussianBlur(image ,(0,0), 10),-4 ,128)
 return Image(pil2tensor(image, np.float32 ).div_(255))
vision.data.open_image = _load_format
train_df = pd.read_csv('/kaggle/input/melanoma-external-malignant-256/train_concat.csv')
 test_df = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/test.csv' )<categorify>
tfms = get_transforms(do_flip=True, flip_vert=True, max_rotate=0.10, max_zoom=1.3, max_warp=0.0, max_lighting=0.2 )
concat = pd.concat([train_df['anatom_site_general_challenge'], test_df['anatom_site_general_challenge']], ignore_index=True)
 dummies = pd.get_dummies(concat, dummy_na=True, dtype=np.uint8, prefix='site')
 train_df = pd.concat([train_df, dummies.iloc[:train_df.shape[0]]], axis=1)
 test_df = pd.concat([test_df, dummies.iloc[train_df.shape[0]:].reset_index(drop=True)], axis=1)
 train_df['sex'] = train_df['sex'].map({'male': 1, 'female': 0})
 test_df['sex'] = test_df['sex'].map({'male': 1, 'female': 0})
 train_df['sex'] = train_df['sex'].fillna(-1)
 test_df['sex'] = test_df['sex'].fillna(-1)
 mean_age, std_age = train_df['age_approx'].mean() , train_df['age_approx'].std()
train_df['age_approx'] =(train_df['age_approx'] - mean_age)/(1e-7 + std_age)
 test_df['age_approx'] =(test_df['age_approx'] - mean_age)/(1e-7 + std_age)
 train_df['age_approx'] = train_df['age_approx'].fillna(0)
 test_df['age_approx'] = test_df['age_approx'].fillna(0)
 train_df['patient_id'] = train_df['patient_id'].fillna(0 )<drop_column>
bs = 64
meta_features = ['sex', 'age_approx'] + [col for col in train_df.columns if 'site_' in col]
meta_features.remove('anatom_site_general_challenge' )<create_dataframe>
data = ImageDataBunch.from_df(path=dat_path, folder='train_images', df=train_df, suffix='.png',
fn_col=0, label_col=1,valid_pct=0.2, size=256, ds_tfms=tfms ).normalize(imagenet_stats )
test = MelanomaDataset(df=test_df,
imfolder='/kaggle/input/melanoma-external-malignant-256/test/test/',
train=False,
transforms=train_transform,
meta_features=meta_features )<define_variables>
kappa = KappaScore()
kappa.weights = "quadratic"
epochs = 10
model_path = 'model.pth'
es_patience = 3
TTA = 3
oof = np.zeros(( len(train_df), 1))
preds = torch.zeros(( len(test), 1), dtype=torch.float32, device=device)
 for fold,(train_idx, val_idx)in enumerate(skf.split(X=np.zeros(len(train_df)) , y=train_df['target'], groups=train_df['patient_id'].tolist()), 1):
print('=' * 20, 'Fold', fold, '=' * 20)
 best_val = None
patience = es_patience
arch = EfficientNet.from_pretrained('efficientnet-b1')
 model = Net(arch=arch, n_meta_features=len(meta_features))
model = model.to(device)
 optim = torch.optim.Adam(model.parameters() , lr=0.001)
 scheduler = ReduceLROnPlateau(optimizer=optim, mode='max', patience=1, verbose=True, factor=0.2)
 criterion = nn.BCEWithLogitsLoss()
train = MelanomaDataset(df=train_df.iloc[train_idx].reset_index(drop=True),
imfolder='/kaggle/input/melanoma-external-malignant-256/train/train/',
train=True,
transforms=train_transform,
meta_features=meta_features)
 val = MelanomaDataset(df=train_df.iloc[val_idx].reset_index(drop=True),
imfolder='/kaggle/input/melanoma-external-malignant-256/train/train/',
train=True,
transforms=test_transform,
meta_features=meta_features)
 train_loader = DataLoader(dataset=train, batch_size=64, shuffle=True, num_workers=2)
 val_loader = DataLoader(dataset=val, batch_size=16, shuffle=False, num_workers=2)
 test_loader = DataLoader(dataset=test, batch_size=16, shuffle=False, num_workers=2)
 for epoch in range(epochs):
start_time = time.time()
correct = 0
epoch_loss = 0
model.train()
for x, y in train_loader:
x[0] = torch.tensor(x[0], device=device, dtype=torch.float32)
 x[1] = torch.tensor(x[1], device=device, dtype=torch.float32)
 y = torch.tensor(y, device=device, dtype=torch.float32)
 optim.zero_grad()
z = model(x)
 loss = criterion(z, y.unsqueeze(1))
loss.backward()
optim.step()
pred = torch.round(torch.sigmoid(z))
correct +=(pred.cpu() == y.cpu().unsqueeze(1)).sum().item()
epoch_loss += loss.item()
train_acc = correct / len(train_idx)
 model.eval()
val_preds = torch.zeros(( len(val_idx), 1), dtype=torch.float32, device=device)
 with torch.no_grad() :
for j,(x_val, y_val)in enumerate(val_loader):
x_val[0] = torch.tensor(x_val[0], device=device, dtype=torch.float32)
 x_val[1] = torch.tensor(x_val[1], device=device, dtype=torch.float32)
 y_val = torch.tensor(y_val, device=device, dtype=torch.float32)
 z_val = model(x_val)
 val_pred = torch.sigmoid(z_val)
 val_preds[j*x_val[0].shape[0]:j*x_val[0].shape[0] + x_val[0].shape[0]] = val_pred
val_acc = accuracy_score(train_df.iloc[val_idx]['target'].values, torch.round(val_preds.cpu()))
val_roc = roc_auc_score(train_df.iloc[val_idx]['target'].values, val_preds.cpu())
 print('Epoch {:03}: | Loss: {:.3f} | Train acc: {:.3f} | Val acc: {:.3f} | Val roc_auc: {:.3f} | Training time: {}'.format(
 epoch + 1,
epoch_loss,
train_acc,
val_acc,
val_roc,
str(datetime.timedelta(seconds=time.time() - start_time)) [:7]))
scheduler.step(val_roc)
 if not best_val:
best_val = val_roc
torch.save(model, model_path)
 continue
if val_roc >= best_val:
best_val = val_roc
patience = es_patience
torch.save(model, model_path)
 else:
patience -= 1
if patience == 0:
print('Early stopping.Best Val roc_auc: {:.3f}'.format(best_val))
break
model = torch.load(model_path)
 model.eval()
val_preds = torch.zeros(( len(val_idx), 1), dtype=torch.float32, device=device)
 with torch.no_grad() :
for j,(x_val, y_val)in enumerate(val_loader):
x_val[0] = torch.tensor(x_val[0], device=device, dtype=torch.float32)
 x_val[1] = torch.tensor(x_val[1], device=device, dtype=torch.float32)
 y_val = torch.tensor(y_val, device=device, dtype=torch.float32)
 z_val = model(x_val)
 val_pred = torch.sigmoid(z_val)
 val_preds[j*x_val[0].shape[0]:j*x_val[0].shape[0] + x_val[0].shape[0]] = val_pred
oof[val_idx] = val_preds.cpu().numpy()
for _ in range(TTA):
for i, x_test in enumerate(test_loader):
x_test[0] = torch.tensor(x_test[0], device=device, dtype=torch.float32)
 x_test[1] = torch.tensor(x_test[1], device=device, dtype=torch.float32)
 z_test = model(x_test)
 z_test = torch.sigmoid(z_test)
 preds[i*x_test[0].shape[0]:i*x_test[0].shape[0] + x_test[0].shape[0]] += z_test
preds /= TTA
del train, val, train_loader, val_loader, x, y, x_val, y_val
gc.collect()
preds /= skf.n_splits<compute_test_metric>
learn = cnn_learner(data, models.resnet50, metrics=[error_rate, kappa] )
print('OOF: {:.3f}'.format(roc_auc_score(train_df['target'], oof)) )<save_to_csv>
learn.fit_one_cycle(1 )
pd.Series(oof.reshape(-1,)).to_csv('oof.csv', index=False )<save_to_csv>
learn.fit_one_cycle(1)
sub = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv')
 sub['target'] = preds.cpu().numpy().reshape(-1,)
 sub.to_csv('submission.csv', index=False )<import_modules>
learn.unfreeze()
learn.fit_one_cycle(9, max_lr=slice(1e-4,1e-3))
import os
import random
import math
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import sklearn
from sklearn.model_selection import train_test_split, cross_val_score, StratifiedKFold
from sklearn.metrics import auc, roc_auc_score
from lightgbm import LGBMClassifier
import lightgbm as lgb
from hyperopt import hp, tpe, Trials, fmin 
<load_from_csv>
learn.model_dir = Path('/kaggle/working/' )
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 submission = pd.read_csv(".. /input/sample_submission.csv" )<count_values>
learn.save('/kaggle/working/FastAI_APTOS_epoch_11')
 learn.export('/kaggle/working/FastAI_APTOS_epoch_11.pkl' )
np.sum(train['target'])/ len(train['target'] )<split>
test_df = pd.read_csv(dat_path/'sample_submission.csv' )
X_train, X_test, y_train, y_test = train_test_split(
 train.drop(['ID_code', 'target'], axis = 1),
train['target'],
test_size = 0.15,
random_state = 42,
stratify = train['target']
)<feature_engineering>
learn.data.add_test(ImageList.from_df(test_df,dat_path,folder='test_images',suffix='.png'))
np.sum(y_train)/ len(y_train), np.sum(y_test)/ len(y_test )<choose_model_class>
tta_params = {'beta':0.12, 'scale':1.0}
def objective(params):
n_folds = 3
params = {
'n_estimators': int(params['n_estimators']),
'num_leaves': int(params['num_leaves']),
'learning_rate': float(params['learning_rate']),
'subsample_for_bin': int(params['subsample_for_bin']),
'min_child_samples': int(params['min_child_samples']),
'reg_alpha': float(params['reg_alpha']),
'reg_lambda': float(params['reg_lambda'])
 }
clf = LGBMClassifier(**params)
 score1 = cross_val_score(clf, X_train, y_train, scoring = 'roc_auc', cv = StratifiedKFold(n_splits = n_folds)).mean()
clf.fit(
 X_train, y_train,
eval_set = [(X_test, y_test)],
eval_metric = 'auc',
early_stopping_rounds = 3,
verbose = False, 
)
 score2 = roc_auc_score(y_test, clf.predict_proba(X_test)[:, 1])
 score = 2 /(( 1 / score1)+(1 / score2))
return 1 - score<init_hyperparams>
preds,y = learn.get_preds(DatasetType.Test)
 preds,y = learn.TTA(ds_type=DatasetType.Test, **tta_params )
space = {
'n_estimators': hp.quniform('n_estimators', 50, 1500, 25),
'num_leaves': hp.quniform('num_leaves', 30, 150, 1),
'learning_rate': hp.loguniform('learning_rate', np.log(0.01), np.log(0.2)) ,
'subsample_for_bin': hp.quniform('subsample_for_bin', 20000, 300000, 20000),
'min_child_samples': hp.quniform('min_child_samples', 20, 500, 5),
'reg_alpha': hp.uniform('reg_alpha', 0.0, 1.0),
'reg_lambda': hp.uniform('reg_lambda', 0.0, 1.0)
 }
tpe_algo = tpe.suggest
tpe_trials = Trials() 
<init_hyperparams>
test_df.diagnosis = preds.argmax(1)
 test_df.head()
<compute_train_metric><EOS> 
test_df.to_csv('submission.csv',index=False )