Kaggle competition: APTOS 2019 Blindness Detection
x_train, x_test, y_train, train_features, test_features, word_index = load_and_prec()
print('Time elapsed:', time.time() - kernel_start_time )<split>
import torch
import torch.nn as nn
import torch.optim as optim
from torch.optim import lr_scheduler
import torch.nn.functional as F
import numpy as np
import torchvision
from torchvision import datasets, models, transforms
import matplotlib.pyplot as plt
import time
import os
import copy
from sklearn.metrics import confusion_matrix
from torch.utils.data import random_split, Dataset
from torch.utils.data.sampler import SubsetRandomSampler
import glob
from PIL import Image
import glob
import cv2
import gc
if USE_DEV_SET:
x_train, y_train, train_features, x_dev, y_dev, dev_features = add_dev_set(x_train, y_train, train_features )<define_variables>
transform = transforms.Compose(
 [transforms.Resize(( 320,320)) ,
transforms.ToTensor() ,
transforms.Normalize([0.460, 0.247, 0.080], [0.249, 0.138, 0.081])
 ])
 class APTOSDataset(Dataset):

 def __init__(self, csv_file, filetype, transform=None):
self.eye_frame = pd.read_csv(csv_file)
 self.filetype = filetype
self.transform = transform
def __len__(self):
return len(self.eye_frame)
 def __getitem__(self, idx):
if self.filetype == 'train':
img_name = os.path.join('.. /input/aptos2019-blindness-detection/train_images',
self.eye_frame.loc[idx,'id_code'] + '.png')
 image = Image.open(img_name)
 if self.transform:
image = self.transform(image)
 else:
image = transforms.ToTensor()(image)
 return image,self.eye_frame.diagnosis[idx]
else:
img_name = os.path.join('.. /input/aptos2019-blindness-detection/test_images',
self.eye_frame.loc[idx,'id_code'] + '.png')
 image = Image.open(img_name)
 if self.transform:
image = self.transform(image)
 else:
image = transforms.ToTensor()(image)
 return image, self.eye_frame.loc[idx,'id_code']
embedding_data = {
'glove': {
'file': '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt',
'emb_mean': -0.005838499,
'emb_std': 0.48782197,
'file_kwargs': {
'encoding': 'utf8',
},
},
'fasttext': {
'file': '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec',
'emb_mean': -0.0033469985,
'emb_std': 0.109855495,
},
'paragram': {
'file': '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt',
'emb_mean': -0.0053247833,
'emb_std': 0.49346462,
'file_kwargs': {
'encoding': 'utf8',
'errors': 'ignore',
},
},
}
def load_embedding(
 type, word_index,
normalize=NORMALIZE_EMBEDDINGS,
try_capitals=TRY_CAPITALS, try_upper=TRY_UPPER, 
):
assert type in embedding_data
data = embedding_data.get(type)
 emb_mean, emb_std = data['emb_mean'], data['emb_std']
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, model_input_size))
all_words = set(k for k, i in word_index.items() if i <= max_features)
 oov_candidates = {}
def set_row(i, vecstring):
extra_features = []
embedding_vector = np.asarray(vec.split(' ')+ extra_features, dtype='float32')[:model_input_size]
if len(embedding_vector)== model_input_size:
embedding_matrix[i] = embedding_vector
return True
with open(data['file'], 'r', **data.get('file_kwargs', {})) as f:
for line in f:
word, vec = line.split(' ', 1)
 i = word_index.get(word)
 if i is None and try_capitals:
oov_candidates[word] = vec
if i is None or i >= max_features:
continue
all_words.discard(word)
 set_row(i, vec)
 oov_words = [(w, i)for w, i in word_index.items() if w in all_words]
if try_capitals or TRY_UPPER:
n_added = 0
for word, i in oov_words:
capitalized = oov_candidates.get(word[0].upper() + word[1:])if try_capitals else None
upper = TRY_UPPER and(capitalized == None)and(len(word)< 7)and oov_candidates.get(word.upper())
 replacement = capitalized or upper
if replacement:
if set_row(i, replacement):
n_added += 1
continue
print(n_added, 'added')
 if normalize:
embedding_matrix = embedding_matrix / emb_std
return embedding_matrix, oov_words<load_pretrained>
test_dataset = APTOSDataset(csv_file='.. /input/aptos2019-blindness-detection/test.csv', filetype='test',transform=transform)
 test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=24, shuffle=False, num_workers=4)
 device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
def load_all_embeddings(seed, load_embeddings=['glove', 'paragram', 'fasttext']):
seed_everything(seed)
 embeddings = []
for embedding_name in load_embeddings:
matrix, oov = load_embedding(embedding_name, word_index)
 embeddings.append(matrix)
 print(embedding_name, 'OOV', len(oov))
return embeddings
embeddings = load_all_embeddings(1 )<choose_model_class>
model0 = torchvision.models.resnet152(pretrained=False)
 state = torch.load('.. /input/resnet/FinalResnet152_0.pt')
 num_ftrs = model0.fc.in_features
model0.fc = nn.Linear(num_ftrs,5)
 model0.load_state_dict(state)
 model0 = model0.to(device)
 model1 = torchvision.models.resnet101(pretrained=False)
 state = torch.load('.. /input/resnet/FinalResnet02.pt')
 num_ftrs = model1.fc.in_features
model1.fc = nn.Linear(num_ftrs,5)
 model1.load_state_dict(state)
 model1 = model1.to(device)
 model2 = torchvision.models.resnet101(pretrained=False)
 state = torch.load('.. /input/resnet/FinalResnet01.pt')
 num_ftrs = model2.fc.in_features
model2.fc = nn.Linear(num_ftrs,5)
 model2.load_state_dict(state)
 model2 = model2.to(device)
 model3 = torchvision.models.resnet101(pretrained=False)
 state = torch.load('.. /input/resnet/FinalResnet00.pt')
 num_ftrs = model3.fc.in_features
model3.fc = nn.Linear(num_ftrs,5)
 model3.load_state_dict(state)
 model3 = model3.to(device)
 model4 = torchvision.models.resnet101(pretrained=False)
 state = torch.load('.. /input/resnet/FinalResnet0.pt')
 num_ftrs = model4.fc.in_features
model4.fc = nn.Linear(num_ftrs,5)
 model4.load_state_dict(state)
 model4 = model4.to(device)
class CyclicLR(object):
def __init__(self, optimizer, base_lr=1e-3, max_lr=6e-3,
step_size=2000, mode='triangular', gamma=1.,
scale_fn=None, scale_mode='cycle', last_batch_iteration=-1):
if not isinstance(optimizer, Optimizer):
raise TypeError('{} is not an Optimizer'.format(
 type(optimizer ).__name__))
self.optimizer = optimizer
if isinstance(base_lr, list)or isinstance(base_lr, tuple):
if len(base_lr)!= len(optimizer.param_groups):
raise ValueError("expected {} base_lr, got {}".format(
 len(optimizer.param_groups), len(base_lr)))
 self.base_lrs = list(base_lr)
 else:
self.base_lrs = [base_lr] * len(optimizer.param_groups)
 if isinstance(max_lr, list)or isinstance(max_lr, tuple):
if len(max_lr)!= len(optimizer.param_groups):
raise ValueError("expected {} max_lr, got {}".format(
 len(optimizer.param_groups), len(max_lr)))
 self.max_lrs = list(max_lr)
 else:
self.max_lrs = [max_lr] * len(optimizer.param_groups)
 self.step_size = step_size
if mode not in ['triangular', 'triangular2', 'exp_range'] \
and scale_fn is None:
raise ValueError('mode is invalid and scale_fn is None')
 self.mode = mode
self.gamma = gamma
if scale_fn is None:
if self.mode == 'triangular':
self.scale_fn = self._triangular_scale_fn
self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = self._triangular2_scale_fn
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = self._exp_range_scale_fn
self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.in_final_stage = False
self.batch_step(last_batch_iteration + 1)
 self.last_batch_iteration = last_batch_iteration
def batch_step(self, batch_iteration=None):
if batch_iteration is None:
batch_iteration = self.last_batch_iteration + 1
self.last_batch_iteration = batch_iteration
for param_group, lr in zip(self.optimizer.param_groups, self.get_lr()):
param_group['lr'] = lr
def enter_final_stage(self, remaining_iterations):
self.in_final_stage = True
self.remaining_iterations = remaining_iterations
self.final_lr = self.base_lrs[0] / 100
self.final_iterations = 0
def _triangular_scale_fn(self, x):
return 1.
 def _triangular2_scale_fn(self, x):
return 1 /(2.**(x - 1))
def _exp_range_scale_fn(self, x):
return self.gamma**(x)
 def get_lr(self):
if self.in_final_stage:
lrs = []
param_lrs = zip(self.optimizer.param_groups, self.base_lrs, self.max_lrs)
 for param_group, base_lr, max_lr in param_lrs:
step =(base_lr - self.final_lr)/ self.remaining_iterations
lr = base_lr - step * self.final_iterations
if self.final_iterations < self.remaining_iterations:
self.final_iterations += 1
lrs.append(lr)
 return lrs
step_size = float(self.step_size)
 cycle = np.floor(1 + self.last_batch_iteration /(2 * step_size))
x = np.abs(self.last_batch_iteration / step_size - 2 * cycle + 1)
 lrs = []
param_lrs = zip(self.optimizer.param_groups, self.base_lrs, self.max_lrs)
 for param_group, base_lr, max_lr in param_lrs:
base_height =(max_lr - base_lr)* np.maximum(0,(1 - x))
if self.scale_mode == 'cycle':
lr = base_lr + base_height * self.scale_fn(cycle)
 else:
lr = base_lr + base_height * self.scale_fn(self.last_batch_iteration)
 lrs.append(lr)
 return lrs<feature_engineering>
def compute_predictions(model, model_type, data_loader, device):
if model_type == 'train':
predictions = []
correct_pred, num_examples = 0, 0
tqdm_notebook()
for i,(inputs, labels)in enumerate(tqdm_notebook(data_loader)) :
inputs = inputs.to(device)
 labels = labels.to(device)
 outputs = model(inputs)
 _, preds = torch.max(outputs, 1)
 predictions.append(preds)
 num_examples += labels.size(0)
 correct_pred +=(preds==labels ).sum()
return predictions, correct_pred.item() /num_examples*100
else:
predictions = []
img_ids = []
out = []
tqdm_notebook()
for i,(inputs, img_id)in enumerate(tqdm_notebook(data_loader)) :
inputs = inputs.to(device)
 outputs = model(inputs)
 _, preds = torch.max(outputs, 1)
 predictions.extend(preds)
 img_ids.extend(img_id)
 out.extend(outputs)
 predictions = [pred.item() for pred in predictions]
final_predictions = pd.DataFrame(np.array([img_ids, predictions])).transpose()
final_predictions.columns = ['id_code', 'diagnosis']
final_predictions_df = final_predictions.copy()
return final_predictions_df, out, img_ids
class Attention(nn.Module):
def __init__(self, feature_dim, step_dim, bias=True, **kwargs):
super(Attention, self ).__init__(**kwargs)
 self.supports_masking = True
self.bias = bias
self.feature_dim = feature_dim
self.step_dim = step_dim
self.features_dim = 0
weight = torch.zeros(feature_dim, 1)
 nn.init.xavier_uniform_(weight)
 self.weight = nn.Parameter(weight)
 if bias:
self.b = nn.Parameter(torch.zeros(step_dim))
def forward(self, x, mask=None):
feature_dim = self.feature_dim
step_dim = self.step_dim
eij = torch.mm(
 x.contiguous().view(-1, feature_dim),
self.weight
).view(-1, step_dim)
 if self.bias:
eij = eij + self.b
eij = torch.tanh(eij)
 a = torch.exp(eij)
 if mask is not None:
a = a * mask
a = a / torch.sum(a, 1, keepdim=True)+ 1e-10
weighted_input = x * torch.unsqueeze(a, -1)
 return torch.sum(weighted_input, 1 )<choose_model_class>
with torch.set_grad_enabled(False):
model0.eval()
model1.eval()
model2.eval()
model3.eval()
model4.eval()
print('Computing Test Predictions')
 test_predictions0,out0,ids0 = compute_predictions(model0, 'test', test_loader, device)
 test_predictions1,out1,ids1 = compute_predictions(model1, 'test', test_loader, device)
 test_predictions2,out2,ids2 = compute_predictions(model2, 'test', test_loader, device)
 test_predictions3,out3,ids3 = compute_predictions(model3, 'test', test_loader, device)
 test_predictions4,out4,ids4 = compute_predictions(model4, 'test', test_loader, device)
<choose_model_class><EOS> 
out =(torch.stack(out0)+torch.stack(out1)+torch.stack(out2)+torch.stack(out3)+torch.stack(out4)) /5
img_ids = np.array(ids0)
 _, predictions = torch.max(out, 1)
 predictions = predictions.cpu().numpy()
final_predictions = pd.DataFrame(np.array([img_ids, predictions])).transpose()
final_predictions.columns = ['id_code', 'diagnosis']
final_predictions_df = final_predictions.copy()
final_predictions.to_csv("submission.csv",index=False )