Kaggle competition: APTOS 2019 Blindness Detection
<SOS> metric: QuadraticWeightedKappa Kaggle data source: aptos-2019-blindness-detection<compute_train_metric>
%matplotlib inline
test_df = pd.read_csv('.. /input/aptos2019-blindness-detection/test.csv')
models = {
'gru': NeuralNetGRU,
'big': NeuralNetBig,
}
class MyDataset(Dataset):
def __init__(self,dataset):
self.dataset = dataset
def __getitem__(self, index):
data, target = self.dataset[index]
return data, target, index
def __len__(self):
return len(self.dataset)
 def sigmoid(x):
return 1 /(1 + np.exp(-x))
def best_thresshold(y_true, y_proba):
precision, recall, thresholds = precision_recall_curve(y_true, y_proba)
 thresholds = np.append(thresholds, 1.001)
 F = 2 /(1/precision + 1/recall)
 best_score = np.max(F)
 best_th = thresholds[np.argmax(F)]
return best_th, best_score<create_dataframe>
N = test_df.shape[0]
x_test = np.empty(( N, 224, 224, 3), dtype=np.uint8)
 for i, image_id in enumerate(tqdm(test_df['id_code'])) :
x_test[i, :, :, :] = preprocess_image(
 f'.. /input/aptos2019-blindness-detection/test_images/{image_id}.png'
)
x_test_cuda = torch.tensor(x_test, dtype=torch.long ).cuda()
test = torch.utils.data.TensorDataset(x_test_cuda)
 if USE_DEV_SET:
x_dev_cuda = torch.tensor(x_dev, dtype=torch.long ).cuda()
dev = torch.utils.data.TensorDataset(x_dev_cuda )<init_hyperparams>
model=load_model('.. /input/model211/val_model2_11.h5' )
def ensemble_embeddings(settings, split_i):
global embeddings
mode = settings.get('embedding_mode', 'avg')
 weights = settings.get('embedding_weights', [1, 1, 1, 0])
 assert mode in ['avg', 'sample', 'concat']
if mode == 'avg':
embedding = np.average(embeddings, axis=0, weights=weights)
 elif mode == 'sample':
sample_embedding_size = settings['embedding_sample']
weight_sum = np.array(weights ).sum()
emb_list = []
for e, w in zip(embeddings, weights):
count = int(round(( w / weight_sum)* sample_embedding_size))
emb_list.append(
 e[:, np.random.randint(e.shape[1], size=count)] 
)
 embedding = np.concatenate(emb_list, axis=1)
 elif mode == 'concat':
embedding = np.concatenate(embeddings, axis=1)
 if settings.get('pca', 0):
pca = PCA(n_components=settings['pca'])
 embedding = pca.fit_transform(embedding)
 if settings.get('set_random_columns', 0):
n_random_cols = settings['set_random_columns']
random_cols = np.random.normal(0, 1,(len(embedding), n_random_cols))
embedding[:, np.random.randint(embedding.shape[1], size=n_random_cols)] = random_cols
if settings.get('add_random_columns', 0):
n_random_cols = settings['add_random_columns']
random_cols = np.random.normal(0, 1,(len(embedding), n_random_cols))
embedding = np.concatenate([embedding, random_cols], -1)
 if settings.get('final_sample'):
embedding = embedding[:, np.random.randint(embedding.shape[1], size=settings['final_sample'])]
return embedding<train_model>
for layer in model.layers[:]:
print(layer )
def training(n_splits, schedule, base_seed=1):
global embeddings
training_start = time.time()
base_seed += SEED
train_preds = np.zeros(( len(x_train)))
 test_preds = np.zeros(( len(df_test)))
 if USE_DEV_SET:
dev_preds = np.zeros(( len(x_dev)))
 seed_everything(base_seed)
 avg_losses_f = []
avg_val_losses_f = []
splits = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED+base_seed-1 ).split(x_train, y_train)
 while len(schedule)< n_splits:
schedule *= 2
if MEASURE_CORRELATION:
all_test_preds = np.zeros(( len(x_test), n_splits))
for i,(train_idx, valid_idx)in enumerate(splits):
settings = {
'gamma': 0.99994,
**schedule[i]
}
init_seed = settings.get('init_seed', base_seed + i * 100)
 seed_everything(init_seed)
 model_class = models[settings['model']]
batch_size = settings['batch_size']
n_epochs = settings['n_epochs']
test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)
 if USE_DEV_SET:
dev_loader = torch.utils.data.DataLoader(dev, batch_size=batch_size, shuffle=False)
 x_train_fold = torch.tensor(x_train[train_idx.astype(int)], dtype=torch.long ).cuda()
y_train_fold = torch.tensor(y_train[train_idx.astype(int), np.newaxis], dtype=torch.float32 ).cuda()
x_val_fold = torch.tensor(x_train[valid_idx.astype(int)], dtype=torch.long ).cuda()
y_val_fold = torch.tensor(y_train[valid_idx.astype(int), np.newaxis], dtype=torch.float32 ).cuda()
kfold_X_features = train_features[train_idx.astype(int)]
kfold_X_valid_features = train_features[valid_idx.astype(int)]
random_sample_weights_range = settings.get('random_sample_weights_range', 0)
 if settings.get('add_sample_weights')and random_sample_weights_range != 0:
kfold_weights = np.random.uniform(
 1 - random_sample_weights_range / 2,
1 + random_sample_weights_range / 2, 
(len(x_train_fold),), 
)
 embedding = ensemble_embeddings(settings, i)
 model = model_class(embedding=embedding, config=settings['nn_config'])
 model.cuda()
loss_factor = 1
loss_fn = torch.nn.BCEWithLogitsLoss(reduction='none')
 train = torch.utils.data.TensorDataset(x_train_fold, y_train_fold)
 train = MyDataset(train)
 train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)
 updates_per_epoch = math.ceil(len(train_idx)/ batch_size)
 step_size = round(updates_per_epoch *(( n_epochs * settings['n_cycles'])/ 2))
base_lr, max_lr = settings['base_lr'], settings['max_lr']
optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=base_lr)
 scheduler = CyclicLR(optimizer, base_lr=base_lr, max_lr=max_lr,
step_size=step_size, mode='exp_range',
gamma=settings['gamma'])
 valid = torch.utils.data.TensorDataset(x_val_fold, y_val_fold)
 valid = MyDataset(valid)
 valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)
 print(f'Fold {i + 1}')
 for epoch in range(n_epochs +(1 if settings.get('add_final_stage', 0)> 0 else 0)) :
seed_everything(init_seed + epoch)
 start_time = time.time()
model.train()
is_final_stage = False
steps_per_epoch = len(train_loader)
 if settings.get('add_final_stage')and epoch == n_epochs:
is_final_stage = True
steps_per_epoch = settings['add_final_stage']
scheduler.enter_final_stage(settings['add_final_stage'])
 avg_loss = 0.
 for batch_i,(x_batch, y_batch, index)in enumerate(train_loader):
f = kfold_X_features[index]
y_pred = model([x_batch, f])
 if scheduler:
scheduler.batch_step()
if settings['add_sample_weights']:
loss_weights = torch.tensor(kfold_weights[index], dtype=torch.float ).unsqueeze(1 ).cuda()
losses = loss_fn(y_pred, y_batch)
 losses *= loss_weights
else:
losses = loss_fn(y_pred, y_batch)
 loss = losses.sum()
optimizer.zero_grad()
loss.backward()
optimizer.step()
avg_loss +=(( loss.item() / steps_per_epoch)/ batch_size)* loss_factor
if is_final_stage and batch_i >= settings['add_final_stage']:
break
model.eval()
test_preds_fold = np.zeros(( len(df_test)))
 avg_val_loss = 0
val_f1 = 0
valid_preds_fold = np.zeros(( x_val_fold.size(0)))
 avg_val_loss = 0.
 for bi,(x_batch, y_batch, index)in enumerate(valid_loader):
f = kfold_X_valid_features[index]
y_pred = model([x_batch,f] ).detach()
avg_val_loss +=(( loss_fn(y_pred, y_batch ).sum().item() / len(valid_loader)) / batch_size)* loss_factor
valid_preds_fold[bi * batch_size:(bi+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
threshold, val_f1 = best_thresshold(y_val_fold.cpu().numpy() , valid_preds_fold)
 elapsed_time = time.time() - start_time
print('Epoch {}/{} \t loss={:.4f} \t val_loss={:.4f} \t val_f1={:.4f} \t thresh={:.4f} \t time={:.2f}s'.format(
 epoch + 1, n_epochs, avg_loss, avg_val_loss, val_f1, threshold, elapsed_time))
ensemble_weight = settings.get('ensemble_weight', 1 / n_splits)
 avg_losses_f.append(avg_loss)
 avg_val_losses_f.append(avg_val_loss)
 for bi,(x_batch,)in enumerate(test_loader):
f = test_features[bi * batch_size:(bi+1)* batch_size]
y_pred = model([x_batch,f] ).detach()
test_preds_fold[bi * batch_size:(bi+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
if USE_DEV_SET:
dev_preds_fold = np.zeros(( len(x_dev)))
 for bi,(x_batch,)in enumerate(dev_loader):
f = dev_features[bi * batch_size:(bi+1)* batch_size]
y_pred = model([x_batch, f] ).detach()
dev_preds_fold[bi * batch_size:(bi+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
dev_preds += dev_preds_fold * ensemble_weight
if MEASURE_CORRELATION:
all_test_preds[:, i] = test_preds_fold
train_preds[valid_idx] = valid_preds_fold
test_preds += test_preds_fold * ensemble_weight
correlation = 0
if MEASURE_CORRELATION:
correlation_matrix = pd.DataFrame(all_test_preds ).corr()
print('
correlation matrix')
 print(correlation_matrix)
 print()
correlation = correlation_matrix.values[np.triu_indices_from(correlation_matrix.values, 1)].mean()
threshold, val_f1 = best_thresshold(y_train, train_preds)
 average_val_loss = np.average(avg_val_losses_f)
 overfit = average_val_loss / np.average(avg_losses_f)
 training_time = time.time() - training_start
dev_threshold, dev_f1 = 0, 0
if USE_DEV_SET:
dev_threshold, dev_f1 = best_thresshold(y_dev, dev_preds)
 return test_preds, val_f1, average_val_loss, correlation, training_time, overfit, threshold, dev_f1, dev_threshold<init_hyperparams>
y_test = model.predict(x_test)> 0.5
y_test = y_test.astype(int ).sum(axis=1)- 1
test_df['diagnosis'] = y_test
schedule = [
{
"embedding_weights": [
0.44439510338875615,
0.3859913694202824,
0.1696135271909615,
],
"model": "big",
"base_lr": 0.001,
"max_lr": 0.006,
"n_epochs": 4,
"n_cycles": 1,
"batch_size": 1024,
"add_final_stage": 5,
"nn_config": {
"hidden_size_1": 123,
"hidden_size_2": 73,
"embedding_dropout": 0.04055337160498571,
"dropout": 0.15174998980548882,
"fc_size": 77
},
"add_sample_weights": True,
'set_random_columns': 5,
'add_random_columns': 5,
'random_sample_weights_range': 0.25,
},
{
"embedding_weights": [
0.6303467699905284,
0.09360214396255329,
0.27605108604691825,
],
"model": "big",
"base_lr": 0.0006013507213308578,
"max_lr": 0.006102420734804078,
"n_epochs": 4,
"n_cycles": 1,
"batch_size": 1024,
"add_final_stage": 0,
"nn_config": {
"hidden_size_1": 103,
"hidden_size_2": 140,
"embedding_dropout": 0.0728942043640125,
"dropout": 0.1812402857338244,
"fc_size": 19
},
"add_sample_weights": True,
'set_random_columns': 5,
'add_random_columns': 5,
'random_sample_weights_range': 0.25,
},
{
"embedding_weights": [
0.5497754648703559,
0.28986474829626435,
0.16035978683337976,
],
"model": "big",
"base_lr": 0.0007647926420277793,
"max_lr": 0.006060292392943227,
"n_epochs": 4,
"n_cycles": 1,
"batch_size": 1024,
"add_final_stage": 10,
"nn_config": {
"hidden_size_1": 89,
"hidden_size_2": 74,
"embedding_dropout": 0.041249174666613736,
"dropout": 0.19393867662442568,
"fc_size": 87
},
"add_sample_weights": True,
'set_random_columns': 5,
'add_random_columns': 5,
'random_sample_weights_range': 0.25,
},
{
"embedding_weights": [
0.38162653503923166,
0.34041829363892023,
0.27795517132184816,
],
"model": "gru",
"base_lr": 0.000765667042217169,
"max_lr": 0.006078156994754822,
"n_epochs": 3,
"n_cycles": 1,
"batch_size": 1024,
"nn_config": {
"hidden_size": [
154,
81
],
"fc_size": 0
},
"add_sample_weights": True,
'set_random_columns': 5,
'add_random_columns': 5,
'random_sample_weights_range': 0.25,
"add_final_stage": 10,
"reload_embeddings": True,
},
{
"embedding_weights": [
0.09924183199312638,
0.3664584186963535,
0.53429974931052,
],
"model": "gru",
"base_lr": 0.0012809083457068925,
"max_lr": 0.006110334179517582,
"n_epochs": 4,
"n_cycles": 1,
"batch_size": 1024,
"nn_config": {
"hidden_size": [
157,
60
],
"fc_size": 0
},
"add_sample_weights": True,
'set_random_columns': 5,
'add_random_columns': 5,
'random_sample_weights_range': 0.25,
"add_final_stage": 0
},
]<train_model>
with open("submission.csv", "w")as f:
f.write(test_df.to_csv(index=False))
test_preds, val_f1, average_val_loss, correlation, training_time, overfit, threshold, dev_f1, dev_threshold = training(n_splits, schedule )<train_model>
test_df['diagnosis'].value_counts()
print('val_f1', val_f1)
 print('average_val_loss', average_val_loss)
 print('overfit', overfit)
 print('threshold', threshold)
 print('correlation', correlation)
 print('dev f1', dev_f1)
 print('dev threshold', dev_threshold)
 print('training_time', training_time )<save_to_csv>
data = np.asarray(x_test[1], dtype="int32")
 inputImg = np.expand_dims(data,0 ).astype(np.float32 )
<categorify><EOS> 
input_details = mobilenetV2F_interpreter.get_input_details()
mobilenetV2F_interpreter.set_tensor(input_details[0]['index'], inputImg)
 mobilenetV2F_interpreter.invoke()
output_details = mobilenetV2F_interpreter.get_output_details()
output_data = mobilenetV2F_interpreter.get_tensor(output_details[0]['index'] )