Kaggle competition: APTOS 2019 Blindness Detection
ATTENTION_MODELS = []
ATTENTION_MODELS_NAMES = []
for k in MODEL_DICT['stitch_reg']:
atten_ckpt = MODEL_DICT['stitch_reg'][k]['atten_ckpt']
atten_output = MODEL_DICT['stitch_reg'][k]['atten_output']
state_dict = torch.load(atten_ckpt,map_location=lambda storage, loc: storage)
 attention_model = EfficientModel(c_out=atten_output,
n_tiles=num_attention_tiles,
tile_size=tile_size,
name='efficientnet-b0',
strategy='bag',
head='attention')
 attention_model.load_state_dict(state_dict,strict=True)
 attention_model.float()
attention_model.eval()
attention_model.cuda()
del state_dict
ATTENTION_MODELS.append(attention_model)
 ATTENTION_MODELS_NAMES.append(atten_ckpt)
<choose_model_class>
def scale_radius(src, img_size, padding=False):
x = src[src.shape[0] // 2,...].sum(axis=1)
 r =(x > x.mean() / 10 ).sum() // 2
yx = src.sum(axis=2)
 region_props = measure.regionprops(( yx > yx.mean() / 10 ).astype('uint8'))
yc, xc = np.round(region_props[0].centroid ).astype('int')
 x1 = max(xc - r, 0)
 x2 = min(xc + r, src.shape[1] - 1)
 y1 = max(yc - r, 0)
 y2 = min(yc + r, src.shape[0] - 1)
 dst = src[y1:y2, x1:x2]
dst = cv2.resize(dst, dsize=None, fx=img_size/(2*r), fy=img_size/(2*r))
if padding:
pad_x =(img_size - dst.shape[1])// 2
pad_y =(img_size - dst.shape[0])// 2
dst = np.pad(dst,(( pad_y, pad_y),(pad_x, pad_x),(0, 0)) , 'constant')
 return dst
class Dataset(torch.utils.data.Dataset):
def __init__(self, img_paths, labels, transform=None, img_size=576, save_img=True):
self.img_paths = img_paths
self.labels = labels
self.transform = transform
self.img_size = img_size
self.save_img = save_img
def __getitem__(self, index):
img_path, label = self.img_paths[index], self.labels[index]
if 'train' in img_path:
img = cv2.imread('.. /input/aptos2019-dataset/images_288_scaled/images_288_scaled/%s' %os.path.basename(img_path))
else:
if os.path.exists('processed/%s' %os.path.basename(img_path)) :
img = cv2.imread('processed/%s' %os.path.basename(img_path))
else:
img = cv2.imread(img_path)
 try:
img = scale_radius(img, img_size=self.img_size, padding=False)
 except Exception as e:
img = img
if self.save_img:
cv2.imwrite('processed/%s' %os.path.basename(img_path), img)
 img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
 img = Image.fromarray(img)
 if self.transform is not None:
img = self.transform(img)
 return img, label
def __len__(self):
return len(self.img_paths)
 def get_model(model_name='resnet18', num_outputs=None, pretrained=True,
freeze_bn=False, dropout_p=0, **kwargs):
pretrained = 'imagenet' if pretrained else None
model = pretrainedmodels.__dict__[model_name](num_classes=1000,
pretrained=pretrained)
 if 'dpn' in model_name:
in_channels = model.last_linear.in_channels
model.last_linear = nn.Conv2d(in_channels, num_outputs,
kernel_size=1, bias=True)
 else:
if 'resnet' in model_name:
model.avgpool = nn.AdaptiveAvgPool2d(1)
 else:
model.avg_pool = nn.AdaptiveAvgPool2d(1)
 in_features = model.last_linear.in_features
if dropout_p == 0:
model.last_linear = nn.Linear(in_features, num_outputs)
 else:
model.last_linear = nn.Sequential(
 nn.Dropout(p=dropout_p),
nn.Linear(in_features, num_outputs), 
)
 if freeze_bn:
for m in model.modules() :
if isinstance(m, nn.BatchNorm2d):
m.weight.requires_grad = False
m.bias.requires_grad = False
return model
class RAdam(Optimizer):
def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0):
defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay)
 self.buffer = [[None, None, None] for ind in range(10)]
super(RAdam, self ).__init__(params, defaults)
 def __setstate__(self, state):
super(RAdam, self ).__setstate__(state)
 def step(self, closure=None):
loss = None
if closure is not None:
loss = closure()
for group in self.param_groups:
for p in group['params']:
if p.grad is None:
continue
grad = p.grad.data.float()
if grad.is_sparse:
raise RuntimeError('RAdam does not support sparse gradients')
 p_data_fp32 = p.data.float()
state = self.state[p]
if len(state)== 0:
state['step'] = 0
state['exp_avg'] = torch.zeros_like(p_data_fp32)
 state['exp_avg_sq'] = torch.zeros_like(p_data_fp32)
 else:
state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32)
 state['exp_avg_sq'] = state['exp_avg_sq'].type_as(p_data_fp32)
 exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']
beta1, beta2 = group['betas']
exp_avg_sq.mul_(beta2 ).addcmul_(1 - beta2, grad, grad)
 exp_avg.mul_(beta1 ).add_(1 - beta1, grad)
 state['step'] += 1
buffered = self.buffer[int(state['step'] % 10)]
if state['step'] == buffered[0]:
N_sma, step_size = buffered[1], buffered[2]
else:
buffered[0] = state['step']
beta2_t = beta2 ** state['step']
N_sma_max = 2 /(1 - beta2)- 1
N_sma = N_sma_max - 2 * state['step'] * beta2_t /(1 - beta2_t)
 buffered[1] = N_sma
if N_sma >= 5:
step_size = group['lr'] * math.sqrt(( 1 - beta2_t)*(N_sma - 4)/(N_sma_max - 4)*(N_sma - 2)/ N_sma * N_sma_max /(N_sma_max - 2)) /(1 - beta1 ** state['step'])
 else:
step_size = group['lr'] /(1 - beta1 ** state['step'])
 buffered[2] = step_size
if group['weight_decay'] != 0:
p_data_fp32.add_(-group['weight_decay'] * group['lr'], p_data_fp32)
 if N_sma >= 5:
denom = exp_avg_sq.sqrt().add_(group['eps'])
 p_data_fp32.addcdiv_(-step_size, exp_avg, denom)
 else:
p_data_fp32.add_(-step_size, exp_avg)
 p.data.copy_(p_data_fp32)
 return loss
class AverageMeter(object):

 def __init__(self):
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count
def quadratic_weighted_kappa(y_pred, y_true):
if torch.is_tensor(y_pred):
y_pred = y_pred.data.cpu().numpy()
if torch.is_tensor(y_true):
y_true = y_true.data.cpu().numpy()
if y_pred.shape[1] == 1:
y_pred = y_pred[:, 0]
else:
y_pred = np.argmax(y_pred, axis=1)
 return metrics.cohen_kappa_score(y_pred, y_true, weights='quadratic')
 def train(train_loader, model, criterion, optimizer, epoch):
losses = AverageMeter()
scores = AverageMeter()
model.train()
for i,(input, target)in tqdm(enumerate(train_loader), total=len(train_loader)) :
input = input.cuda()
target = target.cuda()
output = model(input)
 loss = criterion(output.view(-1), target.float())
 optimizer.zero_grad()
loss.backward()
optimizer.step()
thrs = [0.5, 1.5, 2.5, 3.5]
output[output < thrs[0]] = 0
output[(output >= thrs[0])&(output < thrs[1])] = 1
output[(output >= thrs[1])&(output < thrs[2])] = 2
output[(output >= thrs[2])&(output < thrs[3])] = 3
output[output >= thrs[3]] = 4
target[target < thrs[0]] = 0
target[(target >= thrs[0])&(target < thrs[1])] = 1
target[(target >= thrs[1])&(target < thrs[2])] = 2
target[(target >= thrs[2])&(target < thrs[3])] = 3
target[target >= thrs[3]] = 4
score = quadratic_weighted_kappa(output, target)
 losses.update(loss.item() , input.size(0))
scores.update(score, input.size(0))
return losses.avg, scores.avg
def validate(val_loader, model, criterion):
losses = AverageMeter()
scores = AverageMeter()
model.eval()
with torch.no_grad() :
for i,(input, target)in tqdm(enumerate(val_loader), total=len(val_loader)) :
input = input.cuda()
target = target.cuda()
output = model(input)
 loss = criterion(output.view(-1), target.float())
 thrs = [0.5, 1.5, 2.5, 3.5]
output[output < thrs[0]] = 0
output[(output >= thrs[0])&(output < thrs[1])] = 1
output[(output >= thrs[1])&(output < thrs[2])] = 2
output[(output >= thrs[2])&(output < thrs[3])] = 3
output[output >= thrs[3]] = 4
score = quadratic_weighted_kappa(output, target)
 losses.update(loss.item() , input.size(0))
scores.update(score, input.size(0))
return losses.avg, scores.avg
MODELS = []
for k in MODEL_DICT['stitch_reg']:
ckpt = MODEL_DICT['stitch_reg'][k]['ckpt']
n_tiles = MODEL_DICT['stitch_reg'][k]['num_tiles']
state_dict = torch.load(ckpt,map_location=lambda storage, loc: storage)
 model = regnet.Regnet(1)
 model.load_state_dict(state_dict,strict=True)
 model.float()
model.eval()
model.cuda()
del state_dict
MODELS.append(model )<categorify>
pseudo_probs = {}
aptos2019_df = pd.read_csv('.. /input/aptos2019-blindness-detection/train.csv')
 aptos2019_img_paths = '.. /input/aptos2019-blindness-detection/train_images/' + aptos2019_df['id_code'].values + '.png'
aptos2019_labels = aptos2019_df['diagnosis'].values
test_df = pd.read_csv('.. /input/aptos2019-blindness-detection/test.csv')
 dir_name = '.. /input/aptos2019-blindness-detection/test_images'
test_img_paths = dir_name + '/' + test_df['id_code'].values + '.png'
test_labels = np.zeros(len(test_img_paths))
test_transform = transforms.Compose([
transforms.Resize(( 512, 512)) ,
transforms.ToTensor() ,
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
])
 test_set = Dataset(
 test_img_paths,
test_labels,
transform=test_transform)
 test_loader = torch.utils.data.DataLoader(
 test_set,
batch_size=16,
shuffle=False,
num_workers=2 )
def preprocess(img,sz,num_tiles):
shape = img.shape
pad0,pad1 =(sz - shape[0]%sz)%sz,(sz - shape[1]%sz)%sz
img = np.pad(img,[[pad0//2,pad0-pad0//2],[pad1//2,pad1-pad1//2],[0,0]],
constant_values=255)
 img = img.reshape(img.shape[0]//sz,sz,img.shape[1]//sz,sz,3)
 img = img.transpose(0,2,1,3,4 ).reshape(-1,sz,sz,3)
 if len(img)< num_tiles:
img = np.pad(img,[[0,num_tiles-len(img)],[0,0],[0,0],[0,0]],constant_values=255)
 idxs = np.argsort(img.reshape(img.shape[0],-1 ).sum(-1)) [:num_tiles]
img = img[idxs]
return img
class StitchPandaDataset(Dataset):
def __init__(self,path,test_df,num_tiles,attention_model):
self.path = path
self.names = list(test_df.image_id)
 self.attention_model = attention_model
self.num_tiles = num_tiles
def __len__(self):
return len(self.names)
 def __getitem__(self,idx):
name = self.names[idx]
image = skimage.io.MultiImage(os.path.join(self.path,name+'.tiff')) [1]
best_tiles = self.get_best_tiles(image)
 images = self.make_square(best_tiles)
 return torch.tensor(images ).float() ,name
def make_square(self, tiles):
idxes = list(range(self.num_tiles))
image_size = tiles[0].shape[0]
n_rows = int(self.num_tiles**0.5)
 images = np.zeros(( image_size * n_rows, image_size * n_rows, 3))
for h in range(n_rows):
for w in range(n_rows):
i = h * n_rows + w
this_img = tiles[idxes[i]]
this_img = 255 - this_img
h1 = h * image_size
w1 = w * image_size
images[h1:h1 + image_size, w1:w1 + image_size] = this_img
images = images.astype(np.float32)
 images /= 255
images = images.transpose(2, 0, 1)
 images = torch.tensor(images)
 return images
def get_best_tiles(self,imgs):
tiles = preprocess(imgs,tile_size,num_attention_tiles)
 tiles_bag = self.make_bag(tiles)
 with torch.no_grad() :
features = self.attention_model.feature_extractor(tiles_bag.cuda())
 features = features.view(-1, num_attention_tiles, features.shape[-1])
 weights = self.attention_model.head.attention_pool.compute_weights(features)
 weights = weights.reshape(-1 ).argsort(0 ).cpu().numpy() [::-1]
best_tiles = [tiles[w] for w in weights[:num_tiles]]
return best_tiles
def make_bag(self,tiles):
images = []
for tile in tiles:
tile = tile.astype(np.float32)
 images.append(tile)
 images = np.stack(images,axis=0)
 images = images.astype(np.float32)
 images = images.transpose(0,3,1,2)
 images /= 255
images = torch.tensor(images)
 return images<load_from_csv>
model = get_model(model_name='se_resnext50_32x4d',
num_outputs=1,
pretrained=False,
freeze_bn=True,
dropout_p=0)
 model = model.cuda()
model.eval()
probs = []
for fold in range(5):
print('Fold [%d/%d]' %(fold+1, 5))
model.load_state_dict(torch.load('.. /input/se-resnext50-32x4d-080922/model_%d.pth' %(fold+1)))
 probs_fold = []
with torch.no_grad() :
for i,(input, _)in tqdm(enumerate(test_loader), total=len(test_loader)) :
input = input.cuda()
output = model(input)
 probs_fold.extend(output.data.cpu().numpy() [:, 0])
 probs_fold = np.array(probs_fold)
 probs.append(probs_fold)
 probs = np.mean(probs, axis=0)
 pseudo_probs['se_resnext50_32x4d'] = probs
del model
torch.cuda.empty_cache()
!nvidia-smi
sub_df = pd.read_csv(SAMPLE)
 if os.path.exists(DATA):
for model, threshold, atten_model, atten_model_name in zip(MODELS,THRES, ATTENTION_MODELS, ATTENTION_MODELS_NAMES):
ds = StitchPandaDataset(image_folder,df,16,attention_model=atten_model)
 if ATTENTION_MEMORIES[atten_model_name] is not None:
ds.attention_memory = ATTENTION_MEMORIES[atten_model_name]
preds = []
names = []
with torch.no_grad() :
for x,y in tqdm(ds):
x = x.cuda()
x = torch.stack([x,x.flip(-1),x.flip(-2),x.flip(-1,-2),
x.transpose(-1,-2),x.transpose(-1,-2 ).flip(-1),
x.transpose(-1,-2 ).flip(-2),x.transpose(-1,-2 ).flip(-1,-2)],0)
 p = model(x)
 p = p.view(-1 ).mean().cpu()
names.append(y)
 preds.append(p)
 final_preds = torch.tensor(preds ).numpy()
final_preds = optimized_rounder.predict(final_preds,threshold)
 final_preds = final_preds.astype(int)
 PREDS.append(final_preds)
 NAMES = names
ATTENTION_MEMORIES[atten_model_name] = ds.attention_memory.copy()<import_modules>
model = get_model(model_name='se_resnext101_32x4d',
num_outputs=1,
pretrained=False,
freeze_bn=True,
dropout_p=0, 
)
 model = model.cuda()
model.eval()
probs = []
for fold in range(5):
print('Fold [%d/%d]' %(fold+1, 5))
model.load_state_dict(torch.load('.. /input/se-resnext101-32x4d-081208/model_%d.pth' %(fold+1)))
 probs_fold = []
with torch.no_grad() :
for i,(input, _)in tqdm(enumerate(test_loader), total=len(test_loader)) :
input = input.cuda()
output = model(input)
 probs_fold.extend(output.data.cpu().numpy() [:, 0])
 probs_fold = np.array(probs_fold)
 probs.append(probs_fold)
 probs = np.mean(probs, axis=0)
 pseudo_probs['se_resnext101_32x4d'] = probs
del model
torch.cuda.empty_cache()
!nvidia-smi
import numpy as np
import cv2
import pandas as pd
import torch
import torch.nn as nn
import torch.utils.data as tdata
import os
import pickle
from hubconf import *
from mish_activation import Mish
import skimage.io
from scipy.stats import mode
from efficientnet_pytorch.model import EfficientNet<define_variables>
model = get_model(model_name='senet154',
num_outputs=1,
pretrained=False,
freeze_bn=True,
dropout_p=0)
 model = model.cuda()
model.eval()
probs = []
for fold in range(5):
print('Fold [%d/%d]' %(fold+1, 5))
model.load_state_dict(torch.load('.. /input/senet154-082510/model_%d.pth' %(fold+1)))
 probs_fold = []
with torch.no_grad() :
for i,(input, _)in tqdm(enumerate(test_loader), total=len(test_loader)) :
input = input.cuda()
output = model(input)
 probs_fold.extend(output.data.cpu().numpy() [:, 0])
 probs_fold = np.array(probs_fold)
 probs.append(probs_fold)
 probs = np.mean(probs, axis=0)
 pseudo_probs['senet154'] = probs
del model
torch.cuda.empty_cache()
!nvidia-smi
ATTENTION_MODELS = [f'.. /input/panda-localtrain/attention_gcp/fold_1/epoch28-kappa0.8734.ckpt']
MODELS = [f'.. /input/panda-localtrain/efficient0-20200618-064502/efficient0-20200618-064502/fold_1/epoch23-kappa0.9014.ckpt',
f'.. /input/panda-localtrain/efficient0-20200618-064502/efficient0-20200618-064502/fold_4/epoch26-kappa0.9034.ckpt',
f'.. /input/panda-localtrain/efficient0-20200709-212941/efficient0-20200709-212941/fold_1/epoch33-kappa0.9021.ckpt',
f'.. /input/panda-localtrain/efficient0-20200628-213437/efficient0-20200628-213437/fold_1/epoch27-kappa0.8957.ckpt',
f'.. /input/panda-localtrain/efficient0-20200616-200111/efficient0-20200616-200111/fold_1/epoch26-kappa0.8911.ckpt',
f'.. /input/panda-localtrain/25tiles-gcp/25tiles-gcp/fold_1/epoch29-kappa0.9046.ckpt',
f'.. /input/panda-localtrain/efficient0-20200708-205659/efficient0-20200708-205659/fold_1/epoch29-kappa0.8975.ckpt',
f'.. /input/panda-localtrain/efficient0-20200708-205659/efficient0-20200708-205659/fold_1/epoch27-kappa0.8982.ckpt']
attention_tiles_num = 128<normalization>
aptos2019_df = pd.read_csv('.. /input/aptos2019-blindness-detection/train.csv')
 aptos2019_img_paths = '.. /input/aptos2019-blindness-detection/train_images/' + aptos2019_df['id_code'].values + '.png'
aptos2019_labels = aptos2019_df['diagnosis'].values
test_df = pd.read_csv('.. /input/aptos2019-blindness-detection/test.csv')
 test_img_paths = '.. /input/aptos2019-blindness-detection/test_images/' + test_df['id_code'].values + '.png'
test_labels = 0.4 * pseudo_probs['se_resnext50_32x4d'] + 0.3 * pseudo_probs['se_resnext101_32x4d'] + 0.3 * pseudo_probs['senet154']
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=41)
 kf = KFold(n_splits=5, shuffle=True, random_state=41)
 img_paths = []
labels = []
for(train_idx1, val_idx1),(train_idx2, val_idx2)in zip(skf.split(aptos2019_img_paths, aptos2019_labels), kf.split(test_img_paths)) :
img_paths.append(( np.hstack(( aptos2019_img_paths[train_idx1], test_img_paths[val_idx2])) , aptos2019_img_paths[val_idx1]))
labels.append(( np.hstack(( aptos2019_labels[train_idx1], test_labels[val_idx2])) , aptos2019_labels[val_idx1]))
class AdaptiveConcatPool2d(nn.Module):
def __init__(self):
super().__init__()
self.avg = nn.AdaptiveAvgPool2d(output_size=(1, 1))
self.max = nn.AdaptiveMaxPool2d(output_size=(1, 1))
def forward(self, x):
avg_x = self.avg(x)
 max_x = self.max(x)
 return torch.cat([avg_x, max_x], dim=1 ).squeeze(2 ).squeeze(2)
 class Flatten(nn.Module):
def forward(self, x):
return x.view(x.shape[0], -1)
 class EfficientModel(nn.Module):
def __init__(self, c_out=5, n_tiles=36, tile_size=224, name='efficientnet-b0', strategy='stitched', head='basic'):
super().__init__()
m = EfficientNet.from_name(name, num_classes=c_out, in_channels=3)
 c_feature = m._fc.in_features
m._fc = nn.Identity()
self.feature_extractor = m
self.n_tiles = n_tiles
self.tile_size = tile_size
if strategy == 'stitched':
if head == 'basic':
self.head = nn.Linear(c_feature, c_out)
 elif head == 'concat':
m._avg_pooling = AdaptiveConcatPool2d()
self.head = nn.Linear(c_feature * 2, c_out)
 elif head == 'gem':
m._avg_pooling = GeM()
self.head = nn.Linear(c_feature, c_out)
 elif strategy == 'bag':
if head == 'basic':
self.head = BasicHead(c_feature, c_out, n_tiles)
 elif head == 'attention':
self.head = AttentionHead(c_feature, c_out, n_tiles)
 self.strategy = strategy
def forward(self, x):
if self.strategy == 'bag':
x = x.view(-1, 3, self.tile_size, self.tile_size)
 h = self.feature_extractor(x)
 h = self.head(h)
 return h
class BasicHead(nn.Module):
def __init__(self, c_in, c_out, n_tiles):
self.n_tiles = n_tiles
super().__init__()
self.fc = nn.Sequential(AdaptiveConcatPool2d() ,
nn.Linear(c_in * 2, c_out))
def forward(self, x):
bn, c = x.shape
h = x.view(-1, self.n_tiles, c, 1, 1 ).permute(0, 2, 1, 3, 4)\ 
.contiguous().view(-1, c, 1 * self.n_tiles, 1)
 h = self.fc(h)
 return h
class AttentionHead(nn.Module):
def __init__(self, c_in, c_out, n_tiles):
self.n_tiles = n_tiles
super().__init__()
self.attention_pool = AttentionPool(c_in, c_in//2)
 self.fc = nn.Linear(c_in, c_out)
 def forward(self, x):
bn, c = x.shape
h = x.view(-1, self.n_tiles, c)
 h = self.attention_pool(h)
 h = self.fc(h)
 return h
class AttentionPool(nn.Module):
def __init__(self, c_in, d):
super().__init__()
self.lin_V = nn.Linear(c_in, d)
 self.lin_w = nn.Linear(d, 1)
 def compute_weights(self, x):
key = self.lin_V(x)
 weights = self.lin_w(torch.tanh(key))
weights = torch.softmax(weights, dim=1)
 return weights
def forward(self, x):
weights = self.compute_weights(x)
 pooled = torch.matmul(x.transpose(1, 2), weights ).squeeze(2)
 return pooled<normalization>
model = get_model(model_name='se_resnext50_32x4d',
num_outputs=1,
pretrained=False,
freeze_bn=True,
dropout_p=0, 
)
 model = model.cuda()
criterion = nn.MSELoss().cuda()
best_losses = []
best_scores = []
for fold,(( train_img_paths, val_img_paths),(train_labels, val_labels)) in enumerate(zip(img_paths, labels)) :
print('Fold [%d/%d]' %(fold+1, len(img_paths)))
 train_set = Dataset(
 train_img_paths,
train_labels,
transform=train_transform,
img_size=576,
save_img=True)
 train_loader = torch.utils.data.DataLoader(
 train_set,
batch_size=16,
shuffle=True,
num_workers=2, 
)
 val_set = Dataset(
 val_img_paths,
val_labels,
transform=val_transform,
save_img=True)
 val_loader = torch.utils.data.DataLoader(
 val_set,
batch_size=16,
shuffle=False,
num_workers=2)
 model.load_state_dict(torch.load('.. /input/se-resnext50-32x4d-080922/model_%d.pth' %(fold+1)))
 optimizer = RAdam(
 filter(lambda p: p.requires_grad, model.parameters()), lr=1e-3)
 scheduler = lr_scheduler.CosineAnnealingLR(
 optimizer, T_max=10, eta_min=1e-5)
 os.makedirs('se_resnext50_32x4d', exist_ok=True)
 best_loss = float('inf')
 best_score = 0
for epoch in range(10):
print('Epoch [%d/%d]' %(epoch + 1, 10))
train_loss, train_score = train(
 train_loader, model, criterion, optimizer, epoch)
 val_loss, val_score = validate(val_loader, model, criterion)
 scheduler.step()
print('loss %.4f - score %.4f - val_loss %.4f - val_score %.4f'
%(train_loss, train_score, val_loss, val_score))
if val_loss < best_loss:
torch.save(model.state_dict() , 'se_resnext50_32x4d/model_%d.pth' %(fold+1))
best_loss = val_loss
best_score = val_score
print("=> saved best model")
 print('val_loss: %f' % best_loss)
 print('val_score: %f' % best_score)
 del model
torch.cuda.empty_cache()
!nvidia-smi
class TileMaker:
def __init__(self, size, number, scale):
self.size = size
self.number = number
self.scale = scale
def __pad(self, image, constant_values=255):
h, w, c = image.shape
horizontal_pad = 0 if(w % self.size)== 0 else self.size -(w % self.size)
 vertical_pad = 0 if(h % self.size)== 0 else self.size -(h % self.size)
 image = np.pad(image, pad_width=(( vertical_pad // 2, vertical_pad - vertical_pad // 2), 
(horizontal_pad // 2, horizontal_pad - horizontal_pad // 2), 
(0, 0)) ,
mode='constant', constant_values=constant_values)
 return image
def __call__(self, image, constant_values=255):
image = self.__pad(image, constant_values=constant_values)
 h, w, c = image.shape
image = image.reshape(h // self.size, self.size, w // self.size, self.size, c)
 image = image.swapaxes(1, 2 ).reshape(-1, self.size, self.size, c)
 if image.shape[0] < self.number:
image = np.pad(image, pad_width=(( 0, self.number - image.shape[0]),(0, 0),(0, 0),(0, 0)) ,
mode='constant', constant_values=constant_values)
 sorted_tiles = np.argsort(np.sum(image, axis=(1, 2, 3)))
 sorted_tiles = sorted_tiles[:self.number]
image = image[sorted_tiles]
return image<categorify>
test_df = pd.read_csv('.. /input/aptos2019-blindness-detection/test.csv')
 dir_name = '.. /input/aptos2019-blindness-detection/test_images'
test_img_paths = dir_name + '/' + test_df['id_code'].values + '.png'
test_labels = np.zeros(len(test_img_paths))
test_transform = transforms.Compose([
transforms.Resize(( 512, 512)) ,
transforms.ToTensor() ,
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
])
 test_set = Dataset(
 test_img_paths,
test_labels,
transform=test_transform)
 test_loader = torch.utils.data.DataLoader(
 test_set,
batch_size=16,
shuffle=False,
num_workers=2)
 model = get_model(model_name='se_resnext50_32x4d',
num_outputs=1,
pretrained=False,
freeze_bn=True,
dropout_p=0)
 model = model.cuda()
model.eval()
probs = []
for fold in range(5):
print('Fold [%d/%d]' %(fold+1, 5))
model.load_state_dict(torch.load('se_resnext50_32x4d/model_%d.pth' %(fold+1)))
 probs_fold = []
with torch.no_grad() :
for i,(input, _)in tqdm(enumerate(test_loader), total=len(test_loader)) :
input = input.cuda()
output = model(input)
 probs_fold.extend(output.data.cpu().numpy() [:, 0])
 probs_fold = np.array(probs_fold)
 probs.append(probs_fold)
 probs = np.mean(probs, axis=0)
 l1_probs['se_resnext50_32x4d'] = probs
del model
torch.cuda.empty_cache()
!nvidia-smi
def dict_to_args(d):
args = argparse.Namespace()
def dict_to_args_recursive(args, d, prefix=''):
for k, v in d.items() :
if type(v)== dict:
dict_to_args_recursive(args, v, prefix=k)
 elif type(v)in [tuple, list]:
continue
else:
if prefix:
args.__setattr__(prefix + '_' + k, v)
 else:
args.__setattr__(k, v)
 dict_to_args_recursive(args, d)
 return args<import_modules>
aptos2019_df = pd.read_csv('.. /input/aptos2019-blindness-detection/train.csv')
 aptos2019_img_paths = '.. /input/aptos2019-blindness-detection/train_images/' + aptos2019_df['id_code'].values + '.png'
aptos2019_labels = aptos2019_df['diagnosis'].values
test_df = pd.read_csv('.. /input/aptos2019-blindness-detection/test.csv')
 test_img_paths = '.. /input/aptos2019-blindness-detection/test_images/' + test_df['id_code'].values + '.png'
test_labels = 0.4 * pseudo_probs['se_resnext50_32x4d'] + 0.3 * pseudo_probs['se_resnext101_32x4d'] + 0.3 * pseudo_probs['senet154']
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=41)
 kf = KFold(n_splits=5, shuffle=True, random_state=42)
 img_paths = []
labels = []
for(train_idx1, val_idx1),(train_idx2, val_idx2)in zip(skf.split(aptos2019_img_paths, aptos2019_labels), kf.split(test_img_paths)) :
img_paths.append(( np.hstack(( aptos2019_img_paths[train_idx1], test_img_paths[val_idx2])) , aptos2019_img_paths[val_idx1]))
labels.append(( np.hstack(( aptos2019_labels[train_idx1], test_labels[val_idx2])) , aptos2019_labels[val_idx1]))
class Model(nn.Module):
def __init__(self, model):
super().__init__()
self.model = model
def forward(self, x):
return model(x )<choose_model_class>
model = get_model(model_name='se_resnext101_32x4d',
num_outputs=1,
pretrained=False,
freeze_bn=True,
dropout_p=0)
 model = model.cuda()
criterion = nn.MSELoss().cuda()
best_losses = []
best_scores = []
for fold,(( train_img_paths, val_img_paths),(train_labels, val_labels)) in enumerate(zip(img_paths, labels)) :
print('Fold [%d/%d]' %(fold+1, len(img_paths)))
 train_set = Dataset(
 train_img_paths,
train_labels,
transform=train_transform,
img_size=576,
save_img=True)
 train_loader = torch.utils.data.DataLoader(
 train_set,
batch_size=16,
shuffle=True,
num_workers=2)
 val_set = Dataset(
 val_img_paths,
val_labels,
transform=val_transform,
save_img=True)
 val_loader = torch.utils.data.DataLoader(
 val_set,
batch_size=16,
shuffle=False,
num_workers=2)
 model.load_state_dict(torch.load('.. /input/se-resnext101-32x4d-081208/model_%d.pth' %(fold+1)))
 optimizer = RAdam(
 filter(lambda p: p.requires_grad, model.parameters()), lr=1e-3)
 scheduler = lr_scheduler.CosineAnnealingLR(
 optimizer, T_max=10, eta_min=1e-5)
 os.makedirs('se_resnext101_32x4d', exist_ok=True)
 best_loss = float('inf')
 best_score = 0
for epoch in range(10):
print('Epoch [%d/%d]' %(epoch + 1, 10))
train_loss, train_score = train(
 train_loader, model, criterion, optimizer, epoch)
 val_loss, val_score = validate(val_loader, model, criterion)
 scheduler.step()
print('loss %.4f - score %.4f - val_loss %.4f - val_score %.4f'
%(train_loss, train_score, val_loss, val_score))
if val_loss < best_loss:
torch.save(model.state_dict() , 'se_resnext101_32x4d/model_%d.pth' %(fold+1))
best_loss = val_loss
best_score = val_score
print("=> saved best model")
 print('val_loss: %f' % best_loss)
 print('val_score: %f' % best_score)
 del model
torch.cuda.empty_cache()
!nvidia-smi
attention_models = []
for attention_model_path in ATTENTION_MODELS:
device = torch.device('cuda')if torch.cuda.is_available() else torch.device('cpu')
 ckpt = torch.load(attention_model_path)
 hparams = dict_to_args(ckpt['hparams'])
 attention_model = EfficientModel(c_out=5,
n_tiles=attention_tiles_num,
tile_size=hparams.tile_size,
name=hparams.backbone,
strategy=hparams.strategy,
head=hparams.head 
)
 attention_model = Model(attention_model)
 attention_model.load_state_dict(ckpt['state_dict'])
 attention_model = attention_model.model.to(device)
 attention_model.eval() ;
attention_models.append(attention_model )<choose_model_class>
test_df = pd.read_csv('.. /input/aptos2019-blindness-detection/test.csv')
 dir_name = '.. /input/aptos2019-blindness-detection/test_images'
test_img_paths = dir_name + '/' + test_df['id_code'].values + '.png'
test_labels = np.zeros(len(test_img_paths))
test_transform = transforms.Compose([
transforms.Resize(( 512,512)) ,
transforms.ToTensor() ,
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
])
 test_set = Dataset(
 test_img_paths,
test_labels,
transform=test_transform)
 test_loader = torch.utils.data.DataLoader(
 test_set,
batch_size=16,
shuffle=False,
num_workers=2)
 model = get_model(model_name='se_resnext101_32x4d',
num_outputs=1,
pretrained=False,
freeze_bn=True,
dropout_p=0)
 model = model.cuda()
model.eval()
probs = []
for fold in range(5):
print('Fold [%d/%d]' %(fold+1, 5))
model.load_state_dict(torch.load('se_resnext101_32x4d/model_%d.pth' %(fold+1)))
 probs_fold = []
with torch.no_grad() :
for i,(input, _)in tqdm(enumerate(test_loader), total=len(test_loader)) :
input = input.cuda()
output = model(input)
 probs_fold.extend(output.data.cpu().numpy() [:, 0])
 probs_fold = np.array(probs_fold)
 probs.append(probs_fold)
 probs = np.mean(probs, axis=0)
 l1_probs['se_resnext101_32x4d'] = probs
del model
torch.cuda.empty_cache()
!nvidia-smi
models = []
hparams_list = []
for model_path in MODELS:
ckpt = torch.load(model_path)
 try:
hparams = dict_to_args(ckpt['hyper_parameters'])
 except:
hparams = dict_to_args(ckpt['hparams'])
 if hparams.loss == 'mse':
c_out = 1
elif hparams.loss == 'bce':
c_out = 5
model = EfficientModel(c_out=c_out,
n_tiles=hparams.n_tiles,
tile_size=hparams.tile_size,
name=hparams.backbone,
strategy=hparams.strategy,
head=hparams.head 
)
 model = Model(model)
 model.load_state_dict(ckpt['state_dict'])
 model = model.model.to(device)
 model.eval() ;
models.append(model)
 hparams_list.append(hparams)
 print(hparams )<choose_model_class>
preds1 = 0.0 * l1_probs['se_resnext50_32x4d'] + 1.0 * l1_probs['se_resnext101_32x4d']
preds2 = 0.5 * l1_probs['se_resnext50_32x4d'] + 0.5 * l1_probs['se_resnext101_32x4d']
<categorify><EOS> 
thrs = [0.5, 1.5, 2.5, 3.5]
preds1[preds1 < thrs[0]] = 0
preds1[(preds1 >= thrs[0])&(preds1 < thrs[1])] = 1
preds1[(preds1 >= thrs[1])&(preds1 < thrs[2])] = 2
preds1[(preds1 >= thrs[2])&(preds1 < thrs[3])] = 3
preds1[preds1 >= thrs[3]] = 4
preds1 = preds1.astype('int')
 preds2[preds2 < thrs[0]] = 0
preds2[(preds2 >= thrs[0])&(preds2 < thrs[1])] = 1
preds2[(preds2 >= thrs[1])&(preds2 < thrs[2])] = 2
preds2[(preds2 >= thrs[2])&(preds2 < thrs[3])] = 3
preds2[preds2 >= thrs[3]] = 4
preds2 = preds2.astype('int')
 test_df['diagnosis'] = preds1
test_df.to_csv('submission.csv', index=False)
 test_df['diagnosis'] = preds2
test_df.to_csv('submission2.csv', index=False )