Kaggle competition: APTOS 2019 Blindness Detection
ATTENTION_MEMORIES = {'.. /input/pandachallengemodel/newcv_tiles_eb0_attention_reg_64_256_f1.pth':None,
'.. /input/pandachallengemodel/tiles_eb0_attention_cls_64_256_f1.pth':None}<install_modules>
import torch
from torch.utils.data import Dataset
from torch.utils.data import Subset
from torch.utils.data import DataLoader
import torch.nn as nn
import torch.nn.functional as F
from torch.optim import Adam
from torch.nn import CosineSimilarity
from torchvision.transforms import ToTensor
from torchvision import transforms
import torchvision.models as models
from pathlib import Path
import PIL.Image
import random
import math
from tqdm import tqdm_notebook as tqdm
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from torch.optim.lr_scheduler import ExponentialLR
import os
import glob
import cv2
import numpy as np
import PIL
from sklearn.metrics import cohen_kappa_score
!pip install.. /input/pytorch-lightning/pytorch_lightning-0.8.5-py3-none-any.whl<define_variables>
random.seed(42)
 torch.manual_seed(42)
 np.random.seed(42 )
sys.path = [
'.. /input/efficientnet-package/EfficientNet-PyTorch/',
'.. /input/efficienet062020/EfficientNet-PyTorch-master/',
'.. /input/regnet/'
] + sys.path<set_options>
path = Path('.. /input/aptos2019-blindness-detection' )
warnings.filterwarnings("ignore")
<load_from_csv>
class CircleCrop:
def __init__(self, radius):
self.radius = radius
circle_img = np.zeros(( 2 * radius, 2 * radius), np.uint8)
 cv2.circle(circle_img,(radius, radius), int(radius), 1, thickness=-1)
 self.mask = torch.tensor(circle_img, dtype=torch.float32)
 def __call__(self, img):
return torch.mul(img, self.mask)
 def __repr__(self):
return self.__class__.__name__ + '(radius={0})'.format(self.radius )
DATA = '.. /input/prostate-cancer-grade-assessment/'
SAMPLE = '.. /input/prostate-cancer-grade-assessment/sample_submission.csv'
df_train = pd.read_csv(os.path.join(DATA,'train.csv'))
df_test = pd.read_csv(os.path.join(DATA,'test.csv'))
image_folder = os.path.join(DATA,'test_images')
 is_test = os.path.exists(image_folder)
 image_folder = image_folder if is_test else os.path.join(DATA,'train_images')
 df = df_test if is_test else df_train.loc[:5]
tile_size = 256
batch_size = 2
num_tiles = 16
num_attention_tiles = 64
nworkers = 4<choose_model_class>
tr_df = pd.read_csv(path/'train.csv' )
class NetVLAD(nn.Module):
def __init__(self, feature_size, max_frames,cluster_size, add_bn=False, truncate=True):
super(NetVLAD, self ).__init__()
self.feature_size = feature_size / 2 if truncate else feature_size
self.max_frames = max_frames
self.cluster_size = cluster_size
self.batch_norm = nn.BatchNorm1d(cluster_size, eps=1e-3, momentum=0.01)
 self.linear = nn.Linear(self.feature_size, self.cluster_size)
 self.softmax = nn.Softmax(dim=1)
 self.cluster_weights2 = nn.Parameter(torch.FloatTensor(1, self.feature_size,
self.cluster_size))
self.add_bn = add_bn
self.truncate = truncate
self.first = True
self.init_parameters()
def init_parameters(self):
init.normal(self.cluster_weights2, std=1 / math.sqrt(self.feature_size))
def forward(self, reshaped_input):
random_idx = torch.bernoulli(torch.Tensor([0.5]))
if self.truncate:
if self.training == True:
reshaped_input = reshaped_input[:, :self.feature_size].contiguous() if random_idx[0]==0 else reshaped_input[:, self.feature_size:].contiguous()
else:
if self.first == True:
reshaped_input = reshaped_input[:, :self.feature_size].contiguous()
else:
reshaped_input = reshaped_input[:, self.feature_size:].contiguous()
activation = self.linear(reshaped_input)
 if self.add_bn:
activation = self.batch_norm(activation)
 activation = self.softmax(activation ).view([-1, self.max_frames, self.cluster_size])
 a_sum = activation.sum(-2 ).unsqueeze(1)
 a = torch.mul(a_sum, self.cluster_weights2)
 activation = activation.permute(0, 2, 1 ).contiguous()
reshaped_input = reshaped_input.view([-1, self.max_frames, self.feature_size])
 vlad = torch.matmul(activation, reshaped_input ).permute(0, 2, 1 ).contiguous()
vlad = vlad.sub(a ).view([-1, self.cluster_size * self.feature_size])
 if self.training == False:
self.first = 1 - self.first
return vlad
class Model(nn.Module):
def __init__(self, num_clusters,num_tiles,num_classes=1,arch='efficientnet-b0'):
super().__init__()
self.base = enet.EfficientNet.from_name(arch)
 self.nc = self.base._fc.in_features
self.tile = nn.Sequential(
 nn.BatchNorm2d(self.nc,eps=0.001,momentum=0.01),
nn.AdaptiveAvgPool2d(output_size=1),
nn.Flatten() ,
nn.Dropout(p=0.2), 
)
 self.netvlad = NetVLAD(cluster_size=num_clusters,max_frames=num_tiles,
feature_size=self.nc,truncate=False)
 self.fc = nn.Linear(num_clusters*self.nc,num_classes)
 def forward(self, x):

 batch = x.shape[0]
shape = x[0].shape
n = shape[0]
x = x.reshape(-1,shape[1],shape[2],shape[3])
 x = self.base.extract_features(x)
 x = self.tile(x)
 x = x.view(batch,n,self.nc)
 x = self.netvlad(x)
 x = self.fc(x)
 return x<normalization>
tr_ds = APTOSDataset(path/'train_images', tr_df, tr_df['diagnosis'].values, augmentation=True, img_size=256 )
class AdaptiveConcatPool2d(nn.Module):
def __init__(self):
super().__init__()
self.avg = nn.AdaptiveAvgPool2d(output_size=(1, 1))
self.max = nn.AdaptiveMaxPool2d(output_size=(1, 1))
def forward(self, x):
avg_x = self.avg(x)
 max_x = self.max(x)
 return torch.cat([avg_x, max_x], dim=1 ).squeeze(2 ).squeeze(2)
 class Flatten(nn.Module):
def forward(self, x):
return x.view(x.shape[0], -1)
 class EfficientModel(nn.Module):
def __init__(self, c_out=6, n_tiles=36, tile_size=224, name='efficientnet-b0', strategy='stitched', head='basic'):
super().__init__()
m = EfficientNet.from_name(name, num_classes=c_out, in_channels=3)
 c_feature = m._fc.in_features
m._fc = nn.Identity()
self.feature_extractor = m
self.n_tiles = n_tiles
self.tile_size = tile_size
if strategy == 'stitched':
if head == 'basic':
self.head = nn.Linear(c_feature, c_out)
 elif head == 'concat':
m._avg_pooling = AdaptiveConcatPool2d()
self.head = nn.Linear(c_feature * 2, c_out)
 elif head == 'gem':
m._avg_pooling = GeM()
self.head = nn.Linear(c_feature, c_out)
 elif strategy == 'bag':
if head == 'basic':
self.head = BasicHead(c_feature, c_out, n_tiles)
 elif head == 'attention':
self.head = AttentionHead(c_feature, c_out, n_tiles)
 self.strategy = strategy
def forward(self, x):
if self.strategy == 'bag':
x = x.view(-1, 3, self.tile_size, self.tile_size)
 h = self.feature_extractor(x)
 h = self.head(h)
 return h
class BasicHead(nn.Module):
def __init__(self, c_in, c_out, n_tiles):
self.n_tiles = n_tiles
super().__init__()
self.fc = nn.Sequential(AdaptiveConcatPool2d() ,
nn.Linear(c_in * 2, c_out))
def forward(self, x):
bn, c = x.shape
h = x.view(-1, self.n_tiles, c, 1, 1 ).permute(0, 2, 1, 3, 4)\ 
.contiguous().view(-1, c, 1 * self.n_tiles, 1)
 h = self.fc(h)
 return h
class AttentionHead(nn.Module):
def __init__(self, c_in, c_out, n_tiles):
self.n_tiles = n_tiles
super().__init__()
self.attention_pool = AttentionPool(c_in, c_in//2)
 self.fc = nn.Linear(c_in, c_out)
 def forward(self, x):
bn, c = x.shape
h = x.view(-1, self.n_tiles, c)
 h = self.attention_pool(h)
 h = self.fc(h)
 return h
class AttentionPool(nn.Module):
def __init__(self, c_in, d):
super().__init__()
self.lin_V = nn.Linear(c_in, d)
 self.lin_w = nn.Linear(d, 1)
 def compute_weights(self, x):
key = self.lin_V(x)
 weights = self.lin_w(torch.tanh(key))
weights = torch.softmax(weights, dim=1)
 return weights
def forward(self, x):
weights = self.compute_weights(x)
 pooled = torch.matmul(x.transpose(1, 2), weights ).squeeze(2)
 return pooled<compute_test_metric>
te_df = pd.read_csv(path/'test.csv' )
class OptimizedRounder() :
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
elif pred >= coef[3] and pred < coef[4]:
X_p[i] = 4
else:
X_p[i] = 5
ll = quadratic_weighted_kappa(y, X_p)
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5, 4.5]
self.coef_ = scipy.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
elif pred >= coef[3] and pred < coef[4]:
X_p[i] = 4
else:
X_p[i] = 5
return X_p
def coefficients(self):
return self.coef_['x']
optimized_rounder = OptimizedRounder()<normalization>
te_ds = APTOSDataset(path/'test_images', te_df, te_df['id_code'], augmentation=True, img_size=256 )
def preprocess(img,sz,num_tiles):
shape = img.shape
pad0,pad1 =(sz - shape[0]%sz)%sz,(sz - shape[1]%sz)%sz
img = np.pad(img,[[pad0//2,pad0-pad0//2],[pad1//2,pad1-pad1//2],[0,0]],
constant_values=255)
 img = img.reshape(img.shape[0]//sz,sz,img.shape[1]//sz,sz,3)
 img = img.transpose(0,2,1,3,4 ).reshape(-1,sz,sz,3)
 idxs = np.argsort(img.reshape(img.shape[0],-1 ).sum(-1)) [:num_tiles]
img = img[idxs]
return img
class PandaDataset(Dataset):
def __init__(self,path,test_df,num_tiles,attention_model=None):
self.path = path
self.names = list(test_df.image_id)
 self.num_tiles = num_tiles
self.attention_model = attention_model
self.attention_memory = {}
def __len__(self):
return len(self.names)
 def __getitem__(self,idx):
name = self.names[idx]
image = skimage.io.MultiImage(os.path.join(self.path,name+'.tiff')) [1]
tiles = preprocess(image,tile_size,num_attention_tiles)
 if self.attention_model:
images = np.zeros(( num_attention_tiles,3,tile_size,tile_size),np.float32)
 else:
tiles = self.get_sum_tiles(tiles)
 images = np.zeros(( self.num_tiles,3,tile_size,tile_size),np.float32)
 for i,tile in enumerate(tiles):
tile = tile.astype(np.float32)
 tile /=255.
 tile = tile.transpose(2,0,1)
 images[i,:,:,:] = tile
if self.attention_model:
images = self.get_best_tiles(images, name)
 return torch.tensor(images ).float() ,name
def get_sum_tiles(self,tiles):
idxs = np.argsort(tiles.reshape(tiles.shape[0],-1 ).sum(-1)) [:self.num_tiles]
tiles = tiles[idxs]
return tiles
def get_best_tiles(self,tiles,name):
if name in self.attention_memory:
weights = self.attention_memory[name]
else:
tile_bags = torch.tensor(tiles.copy())
 with torch.no_grad() :
features = self.attention_model.feature_extractor(tile_bags.cuda())
 features = features.view(-1, num_attention_tiles, features.shape[-1])
 weights = self.attention_model.head.attention_pool.compute_weights(features)
 weights = weights.reshape(-1 ).argsort(0 ).cpu().numpy() [::-1]
self.attention_memory[name] = weights
best_tiles = [tiles[w] for w in weights[:num_tiles]]
return best_tiles<define_variables>
bs = 50
nw = 4
PREDS = []<load_pretrained>
tr_dl = DataLoader(tr_ds, batch_size=bs, num_workers=nw, drop_last=False, pin_memory=True, shuffle=True )
MODELS = []
THRES = []
for k in MODEL_DICT['reg']:
ckpt = MODEL_DICT['reg'][k]['ckpt']
n_tiles = MODEL_DICT['reg'][k]['num_tiles']
backbone = MODEL_DICT['reg'][k]['backbone']
threshold = MODEL_DICT['reg'][k]['threshold']
state_dict = torch.load(ckpt,map_location=lambda storage, loc: storage)
 model = Model(num_clusters=6,num_tiles=n_tiles,num_classes=1,arch=backbone)
 model.load_state_dict(state_dict,strict=True)
 model.float()
model.eval()
model.cuda()
del state_dict
MODELS.append(model)
 THRES.append(threshold )<load_from_csv>
te_dl = DataLoader(te_ds, batch_size=bs, num_workers=nw, drop_last=False, pin_memory=True, shuffle=False )
sub_df = pd.read_csv(SAMPLE)
 if os.path.exists(DATA):
ds = PandaDataset(image_folder,df,36,attention_model=None)
 for model,threshold in zip(MODELS,THRES):
preds = []
names = []
with torch.no_grad() :
for x,y in tqdm(ds):
x = x.cuda()
x = torch.stack([x,x.flip(-1),x.flip(-2),x.flip(-1,-2),
x.transpose(-1,-2),x.transpose(-1,-2 ).flip(-1),
x.transpose(-1,-2 ).flip(-2),x.transpose(-1,-2 ).flip(-1,-2)],0)
 p = model(x)
 p = p.view(-1 ).mean().cpu()
names.append(y)
 preds.append(p)
 final_preds = torch.tensor(preds ).numpy()
final_preds = optimized_rounder.predict(final_preds,threshold)
 final_preds = final_preds.astype(int)
 PREDS.append(final_preds)
 NAMES = names<drop_column>
class Bias(nn.Module):
def __init__(self, num_out):
super(Bias, self ).__init__()
self.bias = nn.Parameter(torch.zeros(num_out ).float())
 def forward(self, x):
return x + self.bias
for model in MODELS:
del model<load_pretrained>
num_classes = 5
ATTENTION_MODELS = []
ATTENTION_MODELS_NAMES = []
for k in MODEL_DICT['reg_with_attention']:
atten_ckpt = MODEL_DICT['reg_with_attention'][k]['atten_ckpt']
atten_output = MODEL_DICT['reg_with_attention'][k]['atten_output']
state_dict = torch.load(atten_ckpt,map_location=lambda storage, loc: storage)
 attention_model = EfficientModel(c_out=atten_output,
n_tiles=num_attention_tiles,
tile_size=tile_size,
name='efficientnet-b0',
strategy='bag',
head='attention')
 attention_model.load_state_dict(state_dict,strict=True)
 attention_model.float()
attention_model.eval()
attention_model.cuda()
del state_dict
ATTENTION_MODELS.append(attention_model)
 ATTENTION_MODELS_NAMES.append(atten_ckpt)
<load_pretrained>
model = models.densenet201(pretrained=True)
 for param in model.parameters() :
param.requires_grad = False
model.classifier = nn.Sequential(
 nn.BatchNorm1d(1920),
nn.Dropout(p=0.5, inplace=False),
nn.Linear(in_features=1920, out_features=512),
nn.ReLU(inplace=True),
nn.BatchNorm1d(512),
nn.Dropout(p=0.5, inplace=False),
nn.Linear(in_features=512, out_features=num_classes),
nn.Linear(in_features=num_classes, out_features=1, bias=False),
Bias(num_classes - 1)
)
 model = model.cuda()
MODELS = []
THRES = []
for k in MODEL_DICT['reg_with_attention']:
ckpt = MODEL_DICT['reg_with_attention'][k]['ckpt']
n_tiles = MODEL_DICT['reg_with_attention'][k]['num_tiles']
backbone = MODEL_DICT['reg_with_attention'][k]['backbone']
threshold = MODEL_DICT['reg_with_attention'][k]['threshold']
state_dict = torch.load(ckpt,map_location=lambda storage, loc: storage)
 model = Model(num_clusters=6,num_tiles=n_tiles,num_classes=1,arch=backbone)
 model.load_state_dict(state_dict,strict=True)
 model.float()
model.eval()
model.cuda()
del state_dict
MODELS.append(model)
 THRES.append(threshold )<load_from_csv>
checkpoint = torch.load('.. /input/aptos-pytorch/checkpoint.tar')
 model.load_state_dict(checkpoint['model'])
sub_df = pd.read_csv(SAMPLE)
 if os.path.exists(DATA):
for model, threshold, atten_model, atten_model_name in zip(MODELS, THRES, ATTENTION_MODELS, ATTENTION_MODELS_NAMES):
ds = PandaDataset(image_folder,df,16,attention_model=atten_model)
 if ATTENTION_MEMORIES[atten_model_name] is not None:
ds.attention_memory = ATTENTION_MEMORIES[atten_model_name]
preds = []
names = []
with torch.no_grad() :
for x,y in tqdm(ds):
x = x.cuda()
x = torch.stack([x,x.flip(-1),x.flip(-2),x.flip(-1,-2),
x.transpose(-1,-2),x.transpose(-1,-2 ).flip(-1),
x.transpose(-1,-2 ).flip(-2),x.transpose(-1,-2 ).flip(-1,-2)],0)
 p = model(x)
 p = p.view(-1 ).mean().cpu()
names.append(y)
 preds.append(p)
 final_preds = torch.tensor(preds ).numpy()
final_preds = optimized_rounder.predict(final_preds,threshold)
 final_preds = final_preds.astype(int)
 PREDS.append(final_preds)
 NAMES = names
ATTENTION_MEMORIES[atten_model_name] = ds.attention_memory.copy()<drop_column>
tta_count = 1
for model in ATTENTION_MODELS:
del model
for model in MODELS:
del model<load_pretrained>
diagnoses = defaultdict(list)
 with torch.no_grad() :
for i in tqdm(range(tta_count)) :
id_code = []
diagnosis = []
for inputs, id_codes in tqdm(te_dl):
inputs = inputs.cuda()
preds = model(inputs ).tolist()
diagnosis.extend(preds)
 id_code.extend(id_codes)
 for id_c, diag in zip(id_code, diagnosis):
diagnoses[id_c].append(diag )
atten_ckpt = MODEL_DICT['ordinal_reg']['model1']['atten_ckpt']
atten_output = MODEL_DICT['ordinal_reg']['model1']['atten_output']
state_dict = torch.load(atten_ckpt,map_location=lambda storage, loc: storage)
 attention_model = EfficientModel(c_out=atten_output,
n_tiles=num_attention_tiles,
tile_size=tile_size,
name='efficientnet-b0',
strategy='bag',
head='attention')
 attention_model.load_state_dict(state_dict,strict=True)
 attention_model.float()
attention_model.eval()
attention_model.cuda()<load_pretrained>
def coral_out_to_class(out):
out = torch.sigmoid(torch.tensor(out))
out = out > 0.5
return torch.sum(out, dim=1 )
MODELS = []
for k in MODEL_DICT['ordinal_reg']:
ckpt = MODEL_DICT['ordinal_reg'][k]['ckpt']
n_tiles = MODEL_DICT['ordinal_reg'][k]['num_tiles']
backbone = MODEL_DICT['ordinal_reg'][k]['backbone']
state_dict = torch.load(ckpt,map_location=lambda storage, loc: storage)
 model = Model(num_clusters=6,num_tiles=n_tiles,num_classes=5,arch=backbone)
 model.load_state_dict(state_dict,strict=True)
 model.float()
model.eval()
model.cuda()
del state_dict
MODELS.append(model )<load_from_csv>
id_code = []
diagnosis = []
for k, v in diagnoses.items() :
id_code.append(k)
 diagnosis.append(np.argmax(np.bincount(coral_out_to_class(v ).numpy())) )
sub_df = pd.read_csv(SAMPLE)
 atten_model_name = atten_ckpt
if os.path.exists(DATA):
ds = PandaDataset(image_folder,df,16,attention_model)
 if ATTENTION_MEMORIES[atten_model_name] is not None:
ds.attention_memory = ATTENTION_MEMORIES[atten_model_name]
for model in MODELS:
names,preds = [],[]
with torch.no_grad() :
for x,y in tqdm(ds):
x = x.cuda()
x = torch.stack([x,x.flip(-1),x.flip(-2),x.flip(-1,-2),
x.transpose(-1,-2),x.transpose(-1,-2 ).flip(-1),
x.transpose(-1,-2 ).flip(-2),x.transpose(-1,-2 ).flip(-1,-2)],0)
 p = model(x)
 p = torch.sigmoid(p)
 p = p.view(8,5 ).mean(0 ).sum().round().cpu()
names.append(y)
 preds.append(p)
 final_preds = torch.tensor(preds ).numpy()
final_preds = final_preds.astype(int)
 PREDS.append(final_preds)
 NAMES = names
ATTENTION_MEMORIES[atten_model_name] = ds.attention_memory.copy()<load_pretrained>
df = pd.DataFrame({'id_code':id_code, 'diagnosis':diagnosis})
 df.to_csv('submission.csv', index=False )