Kaggle competition: APTOS 2019 Blindness Detection
<SOS> metric: QuadraticWeightedKappa Kaggle data source: aptos-2019-blindness-detection<choose_model_class>
%reload_ext autoreload
%autoreload 2
%matplotlib inline
class New_GRU_Layer(nn.Module):
def __init__(self):
super(New_GRU_Layer, self ).__init__()
self.gru = nn.GRU(input_size=hidden_size*2,
hidden_size=hidden_size,
bidirectional=True,
batch_first=True)

def init_weights(self):
ih =(param.data for name, param in self.named_parameters() if 'weight_ih' in name)
 hh =(param.data for name, param in self.named_parameters() if 'weight_hh' in name)
 b =(param.data for name, param in self.named_parameters() if 'bias' in name)
 for k in ih:
nn.init.xavier_uniform_(k)
 for k in hh:
nn.init.orthogonal_(k)
 for k in b:
nn.init.constant_(k, 0)
 def forward(self, x):
return self.gru(x)
 class New_LSTM_Layer(nn.Module):
def __init__(self):
super(New_LSTM_Layer, self ).__init__()
self.lstm = nn.LSTM(input_size=embed_size,
hidden_size=hidden_size,
bidirectional=True,
batch_first=True)

def init_weights(self):
ih =(param.data for name, param in self.named_parameters() if 'weight_ih' in name)
 hh =(param.data for name, param in self.named_parameters() if 'weight_hh' in name)
 b =(param.data for name, param in self.named_parameters() if 'bias' in name)
 for k in ih:
nn.init.xavier_uniform_(k)
 for k in hh:
nn.init.orthogonal_(k)
 for k in b:
nn.init.constant_(k, 0)
 def forward(self, x):
return self.lstm(x )<choose_model_class>
from fastai import *
from fastai.vision import *
import pandas as pd
import matplotlib.pyplot as plt
class Linear(nn.Module):
def __init__(self):
super(Linear, self ).__init__()
fc_layer1 = 16
self.linear = nn.Linear(hidden_size * 8 + 3, fc_layer1)
 self.relu = nn.ReLU()
def forward(self, x):
return self.relu(self.linear(x))<normalization>
print('Make sure cudnn is enabled:', torch.backends.cudnn.enabled )
class Attention(nn.Module):
def __init__(self, feature_dim, step_dim, bias=True, **kwargs):
super(Attention, self ).__init__(**kwargs)
 self.supports_masking = True
self.bias = bias
self.feature_dim = feature_dim
self.step_dim = step_dim
self.features_dim = 0
weight = torch.zeros(feature_dim, 1)
 nn.init.xavier_uniform_(weight)
 self.weight = nn.Parameter(weight)
 if bias:
self.b = nn.Parameter(torch.zeros(step_dim))
def forward(self, x, mask=None):
feature_dim = self.feature_dim
step_dim = self.step_dim
eij = torch.mm(
 x.contiguous().view(-1, feature_dim),
self.weight
).view(-1, step_dim)
 if self.bias:
eij = eij + self.b
eij = torch.tanh(eij)
 a = torch.exp(eij)
 if mask is not None:
a = a * mask
a = a / torch.sum(a, 1, keepdim=True)+ 1e-10
weighted_input = x * torch.unsqueeze(a, -1)
 return torch.sum(weighted_input, 1)
 class NeuralNet(nn.Module):
def __init__(self):
super(NeuralNet, self ).__init__()
fc_layer = 16
fc_layer1 = 16
self.embedding = nn.Embedding(max_features, embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = nn.Dropout2d(0.1)
 self.lstm = WeightDrop(nn.LSTM(input_size=embed_size, hidden_size=hidden_size, bidirectional=True, batch_first=True), ['weight_hh_l0'], dropout=0.045)
 self.gru = WeightDrop(nn.GRU(input_size=hidden_size*2, hidden_size=hidden_size, bidirectional=True, batch_first=True), ['weight_hh_l0'], dropout=0.045)
 self.lstm2 = nn.LSTM(hidden_size * 2, hidden_size, bidirectional=True, batch_first=True)
 self.lstm_attention = Attention(hidden_size * 2, maxlen)
 self.gru_attention = Attention(hidden_size * 2, maxlen)
 self.linear = nn.Linear(hidden_size * 8 + 3, fc_layer1)
 self.relu = nn.ReLU()
self.dropout = nn.Dropout(0.0925)
 self.fc = nn.Linear(fc_layer**2,fc_layer)
 self.out = nn.Linear(fc_layer1, 1)
 self.lincaps = nn.Linear(Num_capsule * Dim_capsule, 1)
 self.caps_layer = Caps_Layer()
def forward(self, x):
h_embedding = self.embedding(x[0])
 h_embedding = torch.squeeze(
 self.embedding_dropout(torch.unsqueeze(h_embedding, 0)))
 h_lstm, _ = self.lstm(h_embedding)
 h_gru, _ = self.gru(h_lstm)
 content3 = self.caps_layer(h_gru)
 content3 = self.dropout(content3)
 batch_size = content3.size(0)
 content3 = content3.view(batch_size, -1)
 content3 = self.relu(self.lincaps(content3))
h_lstm_atten = self.lstm_attention(h_lstm)
 h_gru_atten = self.gru_attention(h_gru)
 avg_pool = torch.mean(h_gru, 1)
 max_pool, _ = torch.max(h_gru, 1)
 f = torch.tensor(x[1], dtype=torch.float ).cuda()
conc = torch.cat(( h_lstm_atten, h_gru_atten, content3, avg_pool, max_pool, f), 1)
 conc = self.relu(self.linear(conc))
conc = self.dropout(conc)
 out = self.out(conc)
 return out<choose_model_class>
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
SEED = 999
seed_everything(SEED )
class Padam(optim.Optimizer):

 def __init__(self, params, lr=1e-1, betas=(0.9, 0.999), eps=1e-8, weight_decay=0, amsgrad=True, partial = 1/4):
if not 0.0 <= betas[0] < 1.0:
raise ValueError("Invalid beta parameter at index 0: {}".format(betas[0]))
if not 0.0 <= betas[1] < 1.0:
raise ValueError("Invalid beta parameter at index 1: {}".format(betas[1]))
defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay, amsgrad=amsgrad, partial = partial)
 super(Padam, self ).__init__(params, defaults)
 def step(self, closure=None):

 loss = None
if closure is not None:
loss = closure()
for group in self.param_groups:
for p in group['params']:
if p.grad is None:
continue
grad = p.grad.data
if grad.is_sparse:
raise RuntimeError('Adam does not support sparse gradients, please consider SparseAdam instead')
 amsgrad = group['amsgrad']
partial = group['partial']
state = self.state[p]
if len(state)== 0:
state['step'] = 0
state['exp_avg'] = torch.zeros_like(p.data)
 state['exp_avg_sq'] = torch.zeros_like(p.data)
 if amsgrad:
state['max_exp_avg_sq'] = torch.zeros_like(p.data)
 exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']
if amsgrad:
max_exp_avg_sq = state['max_exp_avg_sq']
beta1, beta2 = group['betas']
state['step'] += 1
if group['weight_decay'] != 0:
grad = grad.add(group['weight_decay'], p.data)
 exp_avg.mul_(beta1 ).add_(1 - beta1, grad)
 exp_avg_sq.mul_(beta2 ).addcmul_(1 - beta2, grad, grad)
 if amsgrad:
torch.max(max_exp_avg_sq, exp_avg_sq, out=max_exp_avg_sq)
 denom = max_exp_avg_sq.sqrt().add_(group['eps'])
 else:
denom = exp_avg_sq.sqrt().add_(group['eps'])
 bias_correction1 = 1 - beta1 ** state['step']
bias_correction2 = 1 - beta2 ** state['step']
step_size = group['lr'] * np.sqrt(bias_correction2)/ bias_correction1
p.data.addcdiv_(-step_size, exp_avg, denom**(partial*2))
return loss<define_variables>
base_image_dir = os.path.join('.. ', 'input/aptos2019-blindness-detection/')
 train_dir = os.path.join(base_image_dir,'train_images/')
 df = pd.read_csv(os.path.join(base_image_dir, 'train.csv'))
df['path'] = df['id_code'].map(lambda x: os.path.join(train_dir,'{}.png'.format(x)))
 df = df.drop(columns=['id_code'])
 df = df.sample(frac=1 ).reset_index(drop=True)
 df.head(10 )
def sigmoid(x):
return 1 /(1 + np.exp(-x))
train_preds = np.zeros(( len(x_train)))
 test_preds = np.zeros(( len(df_test)))
 seed_everything()

 x_test_cuda = torch.tensor(x_test, dtype=torch.long ).cuda()
test = torch.utils.data.TensorDataset(x_test_cuda)
 test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)
 avg_losses_f = []
avg_val_losses_f = []<categorify>
im = Image.open(df['path'][1])
 width, height = im.size
print(width,height)
 im.show()
class MyDataset(Dataset):
def __init__(self,dataset):
self.dataset = dataset
def __getitem__(self, index):
data, target = self.dataset[index]
return data, target, index
def __len__(self):
return len(self.dataset )<data_type_conversions>
bs = 64
sz=224
for i,(train_idx, valid_idx)in enumerate(splits):
x_train = np.array(x_train)
 y_train = np.array(y_train)
 features = np.array(features)
 x_train_fold = torch.tensor(x_train[train_idx.astype(int)], dtype=torch.long ).cuda()
y_train_fold = torch.tensor(y_train[train_idx.astype(int), np.newaxis], dtype=torch.float32 ).cuda()
kfold_X_features = features[train_idx.astype(int)]
kfold_X_valid_features = features[valid_idx.astype(int)]
x_val_fold = torch.tensor(x_train[valid_idx.astype(int)], dtype=torch.long ).cuda()
y_val_fold = torch.tensor(y_train[valid_idx.astype(int), np.newaxis], dtype=torch.float32 ).cuda()
model = NeuralNet()
model.cuda()
loss_fn = torch.nn.BCEWithLogitsLoss(reduction='sum')
 step_size = 300
base_lr, max_lr = 1e-3, 3e-3
optimizer = Padam(filter(lambda p: p.requires_grad, model.parameters()),
lr=2e-3, partial=0.4, eps=2e-8)
 scheduler = CyclicLR(optimizer, base_lr=base_lr, max_lr=max_lr,
step_size=step_size, mode='exp_range',
gamma=0.999999999)
 train = torch.utils.data.TensorDataset(x_train_fold, y_train_fold)
 valid = torch.utils.data.TensorDataset(x_val_fold, y_val_fold)
 train = MyDataset(train)
 valid = MyDataset(valid)
 train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)
 print(f'Fold {i + 1}')
 for epoch in range(n_epochs):
start_time = time.time()
model.train()
avg_loss = 0.
 for i,(x_batch, y_batch, index)in enumerate(train_loader):
f = kfold_X_features[index]
y_pred = model([x_batch,f])
 if scheduler:
scheduler.batch_step()
loss = loss_fn(y_pred, y_batch)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
avg_loss += loss.item() / len(train_loader)
 model.eval()
valid_preds_fold = np.zeros(( x_val_fold.size(0)))
 test_preds_fold = np.zeros(( len(df_test)))
 avg_val_loss = 0.
 for i,(x_batch, y_batch, index)in enumerate(valid_loader):
f = kfold_X_valid_features[index]
y_pred = model([x_batch,f] ).detach()
avg_val_loss += loss_fn(y_pred, y_batch ).item() / len(valid_loader)
 valid_preds_fold[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
elapsed_time = time.time() - start_time
print('Epoch {}/{} \t loss={:.4f} \t val_loss={:.4f} \t time={:.2f}s'.format(
 epoch + 1, n_epochs, avg_loss, avg_val_loss, elapsed_time))
avg_losses_f.append(avg_loss)
 avg_val_losses_f.append(avg_val_loss)
 for i,(x_batch,)in enumerate(test_loader):
f = test_features[i * batch_size:(i+1)* batch_size]
y_pred = model([x_batch,f] ).detach()
test_preds_fold[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
train_preds[valid_idx] = valid_preds_fold
test_preds += test_preds_fold / len(splits)
 print('All \t loss={:.4f} \t val_loss={:.4f} \t '.format(np.average(avg_losses_f),np.average(avg_val_losses_f)))
<compute_test_metric>
data.show_batch(rows=3, figsize=(7,6))
def bestThresshold(y_train,train_preds):
tmp = [0,0,0]
delta = 0
for tmp[0] in tqdm(np.arange(0.2, 0.402, 0.001)) :
tmp[1] = f1_score(y_train, np.array(train_preds)>tmp[0])
 if tmp[1] > tmp[2]:
delta = tmp[0]
tmp[2] = tmp[1]
print('best threshold is {:.4f} with F1 score: {:.4f}'.format(delta, tmp[2]))
return delta
delta = bestThresshold(y_train,train_preds )<save_to_csv>
def quadratic_kappa(y_hat, y):
return torch.tensor(cohen_kappa_score(torch.round(y_hat), y, weights='quadratic'),device='cuda:0' )
submission = df_test[['qid']].copy()
submission['prediction'] =(test_preds > delta ).astype(int)
 submission.to_csv('submission.csv', index=False )<set_options>
learn = cnn_learner(data, base_arch=models.resnet50, metrics = [quadratic_kappa] )
%reload_ext autoreload
%autoreload 2<import_modules>
learn.fit_one_cycle(4,max_lr = 1e-2 )
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.utils.data<set_options>
learn.fit_one_cycle(6, max_lr=slice(1e-6,1e-3))
KAGGLE_RUN =(not os.path.exists('/opt/conda/home/.history'))
if KAGGLE_RUN: print('Kaggle run')
 pd.options.display.float_format = '{:.6f}'.format
def tf_seed_everything(seed):
os.environ['PYTHONHASHSEED'] = str(seed)
 random.seed(seed)
 np.random.seed(seed)
 tf.set_random_seed(seed)
 def f1_curve(target, preds, t_min=0.01, t_max=0.99, steps=99):
curve = {}
for t in np.linspace(t_min, t_max, steps):
with warnings.catch_warnings() :
warnings.simplefilter("ignore")
 curve[t] = sklearn.metrics.f1_score(target, preds >= t)
 return pd.Series(curve ).sort_index() 
<define_variables>
learn.export()
learn.save('stage-2' )
class QuoraData:
def __init__(self):
self.paths = {
'glove': '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt',
'news': '.. /input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin',
'paragram': '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt',
'wiki': '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec',
}
def glove(self): return QuoraEmbedding(self.paths['glove'])
 def news(self): return QuoraEmbedding(self.paths['news'])
 def paragram(self): return QuoraEmbedding(self.paths['paragram'])
 def wiki(self): return QuoraEmbedding(self.paths['wiki'])
 def convert_start(self, embeddings):

 self.convert_pids = []
for name in embeddings:
path = self.paths[name]
if path.endswith('.npy'): continue
if os.path.exists(path + '.npy'): continue
if os.path.exists(name + '.npy'): continue
if KAGGLE_RUN:
out_path = name + '.npy'
else:
out_path = path + '.npy'
print(f'Converting {path} -> {out_path}', flush=True)
 pid = os.fork()
if pid == 0:
emb = QuoraEmbedding(path)
 emb.save_npy(out_path)
 os._exit(0)
 self.convert_pids.append(pid)
 self.paths[name] = out_path
def convert_wait(self):

 for pid in self.convert_pids:
try:
os.waitpid(pid, 0)
 except:
pass
def read_train(self):
return pd.read_csv('.. /input/train.csv')
 def read_test(self):
return pd.read_csv('.. /input/test.csv')
 def read_input(self):
return pd.concat([self.read_train() , self.read_test() ], axis=0, copy=False, sort=False, ignore_index=True)
 class QuoraEmbedding:

 def __init__(self, filename, vectors=None, vocab=None):
self.filename = filename
if vectors is None:
vectors, vocab = self._read(filename)
 self.vectors = vectors
self.index2word = vocab
self.name = re.findall('^\w+', os.path.basename(filename)) [0]
self.shape = self.vectors.shape
self.num_words, self.dim = self.shape
assert len(self.index2word)== self.num_words
self.word2index = { self.index2word[i]: i for i in reversed(range(self.num_words)) }
self.lword2index = { self.index2word[i].lower() : i for i in reversed(range(self.num_words)) }
def __repr__(self):
return f'QuoraEmbedding({self.name}, {self.shape[0]}x{self.shape[1]}, {self.vectors.nbytes/(1024**3):.1f}GiB)'
def lookup(self, word, lower=False):
if lower:
idx = self.lword2index.get(word.lower() , -1)
 else:
idx = self.word2index.get(word, -1)
 return self.vectors[idx] if idx != -1 else None
@functools.lru_cache(1)
 def mean(self): return self.vectors.mean()
@functools.lru_cache(1)
 def std(self): return self.vectors.std()
def save_npy(self, filename):
assert filename.endswith('.npy')
 np.save(filename, self.vectors)
 with open(filename[:-4] + '.vocab', 'wb')as fp:
pickle.dump(self.index2word, fp)
 @classmethod
def _read(cls, filename):

 if filename.endswith('.npy'):
return cls._read_npy(filename)
 elif os.path.exists(filename + '.npy'):
return cls._read_npy(filename + '.npy')
 elif os.path.exists(re.sub('[.]\w+$', '.npy', filename)) :
return cls._read_npy(re.sub('[.]\w+$', '.npy', filename))
elif filename.endswith('.bin'):
return cls._read_bin(filename)
 else:
return cls._read_txt(filename)
 @classmethod
def _read_npy(cls, filename):
assert filename.endswith('.npy')
 vectors = np.load(filename, 'r')
 with open(filename[:-4] + '.vocab', 'rb')as fp:
vocab = pickle.load(fp)
 return vectors, vocab
@classmethod
def _read_txt(cls, filename):
vectors = []
vocab = []
dim = -1
for line_num, line in enumerate(open(filename, 'rb')) :
try:
line = line.decode('utf-8')
 except:
line = line.decode('latin_1')
 print(f'Bad word on line {line_num+1}: {line[:20]}...')
 word, line = line.split(' ', maxsplit=1)
 vec = np.fromstring(line, np.float32, count=dim, sep=' ')
 if len(vec)== 1 and len(vectors)== 0:
dim = int(vec[0])
 continue
if dim != -1 and dim != len(vec):
raise Exception(f'Mismatching vector lengths: {dim} vs {len(vec)}')
 dim = len(vec)
 vectors.append(vec)
 vocab.append(word)
 vectors = np.stack(vectors)
 vocab = '
'.join(vocab ).split('
')
 return vectors, vocab
@classmethod
def _read_bin(cls, filename):
kv = KeyedVectors.load_word2vec_format(filename, binary=True)
 return kv.vectors, kv.index2word
class QuoraPreprocessor:

 punct = r
's 'm 'd 'll 're 've n't 'em o'clock
i.e.e.g.vs.U.S.U.K.[A-Za-z].a.m.p.m.
 e-mail t-shirt
: :):D :-):);):-)=);-)
 [0-9]+,[0-9]{3}

 [A-Za-z].Mr.Mrs.Dr.a.m.p.m.

\.{3,5} [?!]{1,3}
['’`‘´`′′] Embeds tokens with given pretrained embedding, initializes missing with random vectors .<set_options>
interp = ClassificationInterpretation.from_learner(learn)
 losses,idxs = interp.top_losses()
len(data.valid_ds)==len(losses)==len(idxs )
def torch_seed(seed):
os.environ['PYTHONHASHSEED'] = str(seed)
 random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
def torch_gc() :
gc.collect()
torch.cuda.empty_cache()
def keras_rnn_init(m):
for name, param in m.named_parameters() :
if 'weight_ih' in name: torch.nn.init.xavier_uniform_(param)
 if 'weight_hh' in name: torch.nn.init.orthogonal_(param)
 if 'bias_' in name: torch.nn.init.constant_(param, 0)
 class Fold:
def __init__(self, data, X, seed=42, valid_frac=0.1, holdout_frac=0.0432, holdout_seed=42):
self.data = data
self.X = X
self.y = data.target.values.astype(np.float32)
 self.seed = seed
tmask = data.target.notnull().values
self.all_idx = data.index.values.astype(np.int32)
 self.test_idx = self.all_idx[~tmask]
trainval_idx, self.holdout_idx = train_test_split(
 self.all_idx[tmask],
stratify=self.y[tmask],
test_size=holdout_frac,
random_state=holdout_seed,
shuffle=True)
 self.train_idx, self.valid_idx = train_test_split(
 trainval_idx,
stratify=self.y[trainval_idx],
test_size=valid_frac,
random_state=seed,
shuffle=True)
 self.train_idx.sort()
self.valid_idx.sort()
self.holdout_idx.sort()
self.oob_idx = self.all_idx[~np.isin(self.all_idx, self.train_idx, assume_unique=True)] 
<feature_engineering>
valid_preds = learn.get_preds(ds_type=DatasetType.Valid )
%%time
torch_seed(42)
 qd = QuoraData()
qd.convert_start(['glove', 'wiki'])
 prep = QuoraPreprocessor()
input_df = qd.read_input()
input_df['question_text'] = prep.transform(input_df.question_text )<categorify>
import numpy as np
import pandas as pd
import os
import scipy as sp
from functools import partial
from sklearn import metrics
from collections import Counter
import json
%%time
qfe = QuoraFeatureExtractor(num_words=95000, max_len=70)
 input_X = qfe.fit_transform(input_df.question_text, fit_mask=input_df.target.notnull().values)
 print({ k: v.shape for(k, v)in input_X.items() } )<load_pretrained>
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = metrics.cohen_kappa_score(y, X_p, weights='quadratic')
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 print(-loss_partial(self.coef_['x']))
def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']
%%time
qd.convert_wait()
torch_seed(42)
 emb_glove = qfe.embedding_weights(qd.glove())
 torch_seed(43)
 emb_wiki = qfe.embedding_weights(qd.wiki())
 emb_glovewiki = np.concatenate([emb_glove, emb_wiki], axis=1)
 gc.collect()
os.system('rm -f glove.npy glove.vocab wiki.npy wiki.vocab' )<choose_model_class>
optR = OptimizedRounder()
optR.fit(valid_preds[0],valid_preds[1] )
class Arch1b_LstmGru(torch.nn.Module):
def __init__(self, emb_weights):
super().__init__()
self.embedding = nn.Embedding(emb_weights.shape[0], emb_weights.shape[1])
 self.embedding.weight = nn.Parameter(torch.Tensor(emb_weights), requires_grad=False)
 self.lstm = nn.LSTM(self.embedding.embedding_dim, 128, bidirectional=True, batch_first=True)
 self.gru = nn.GRU(self.lstm.hidden_size*2, 64, bidirectional=True, batch_first=True)
 self.fc = nn.Linear(128, 1)
 def dropout1d(self, x, p):
x = x.permute(0, 2, 1)
 x = F.dropout2d(x, p, training=self.training)
 x = x.permute(0, 2, 1)
 return x
def forward(self, x_tok):
x = self.embedding(x_tok)
 x = self.dropout1d(x, 0.2)
 x = F.dropout2d(x, 0.05, training=self.training)
 x, _ = self.lstm(x)
 x, c = self.gru(x)
 x = x.max(1)[0]
x = F.dropout(x, 0.1, training=self.training)
 x = self.fc(x)
 return x<prepare_x_and_y>
sample_df = pd.read_csv('.. /input/aptos2019-blindness-detection/sample_submission.csv')
 sample_df.head()
class Learner:
def __init__(self, fold, model, seed=None, batch_size=512, grad_clip=1):
self.__dict__.update(fold.__dict__)
 self.fold = fold
self.model = model
self.dataset = torch.utils.data.TensorDataset(
 torch.LongTensor(self.X['tokens']),
torch.FloatTensor(self.y[:, np.newaxis]))
self.train_dl = self.make_loader(self.train_idx, batch_size=batch_size, shuffle=True)
 self.valid_dl = self.make_loader(self.valid_idx)
 opt_params = [p for p in self.model.parameters() if p.requires_grad]
self.opt = torch.optim.Adam(opt_params)
 self.grad_clip = grad_clip
self.loss_fn = nn.BCEWithLogitsLoss(reduction='sum')
 self.steps = 0
self.epochs = 0
params_sz = sum([p.detach().cpu().numpy().nbytes for p in opt_params])
 print(f'Training on {len(self.train_dl.dataset)} examples, '
f'validating on {len(self.valid_dl.dataset)} examples.'
f'Parameters: {params_sz/1048576:.2f}MiB', flush=True)
 self.epoch_params = [ self.get_params() ]
self.val_metrics = []
def make_loader(self, idx, batch_size=1024, shuffle=False):
ds = torch.utils.data.Subset(self.dataset, idx)
 return torch.utils.data.DataLoader(
 ds, batch_size=batch_size, shuffle=shuffle, pin_memory=True)
 def get_params(self):
return { name: param.detach().cpu().numpy().copy()
for(name, param)in self.model.named_parameters()
if param.requires_grad }
def restore_params(self, prev):
for name, param in self.model.named_parameters() :
if name in prev:
param.data.copy_(torch.from_numpy(prev[name]))
def restore_epoch(self, epoch):
self.restore_params(self.epoch_params[epoch])
 def predict(self, idx=None, dl=None):
if dl is None:
dl = self.make_loader(idx, shuffle=False)
 preds = []
with torch.no_grad() :
self.model.eval()
for batch in dl:
batch = [t.cuda() for t in batch[:-1]]
pred = torch.sigmoid(self.model(*batch))
preds.append(pred.cpu().numpy().flatten())
 self.model.train()
return np.concatenate(preds)
 def eval(self):
y_pred = self.predict(dl=self.valid_dl)
 y_true = self.y[self.valid_idx]
curve = f1_curve(y_true, y_pred)
 metrics = {
'epochs': self.epochs,
'steps': self.steps,
'f1': curve.max() ,
'roc-auc': sklearn.metrics.roc_auc_score(y_true, y_pred),
'pr-auc': sklearn.metrics.average_precision_score(y_true, y_pred),
'loss': sklearn.metrics.log_loss(y_true, y_pred),
'thresh': curve.idxmax() ,
}
self.val_metrics.append(metrics)
 print(' '.join('%s=%.6g' %(k, float(v)) for(k, v)in metrics.items()), flush=True)
 def train(self, epochs=1, eval_steps=0):
self.model.cuda()
self.model.train()
for ep in range(epochs):
t_start = time.time()
if not KAGGLE_RUN and eval_steps > 0:
it = enumerate(tqdm_notebook(self.train_dl, leave=False))
else:
it = enumerate(self.train_dl)
 for i, batch in it:
batch = [t.cuda() for t in batch]
y_batch = batch.pop()
self.opt.zero_grad()
y_pred = self.model(*batch)
 loss = self.loss_fn(y_pred, y_batch)
 loss.backward()
if self.grad_clip is not None:
nn.utils.clip_grad_norm_(self.model.parameters() , self.grad_clip)
 self.opt.step()
self.steps += 1
if eval_steps and i % eval_steps == 0 and i > 0: self.eval()
self.epoch_params.append(self.get_params())
 self.epochs += 1
print('t=%.2fs' %(time.time() - t_start), end=' ', flush=True)
 self.eval()
return self<choose_model_class>
learn.data.add_test(ImageList.from_df(sample_df,'.. /input/aptos2019-blindness-detection',folder='test_images',suffix='.png'))
%%time
fold = Fold(input_df, input_X, seed=1005); torch_seed(fold.seed)
 learn1 = Learner(fold, Arch1b_LstmGru(emb_glovewiki)).train(5 )<choose_model_class>
preds,y = learn.TTA(ds_type=DatasetType.Test )
%%time
fold = Fold(input_df, input_X, seed=555); torch_seed(fold.seed)
 learn2 = Learner(fold, Arch1b_LstmGru(emb_glovewiki)).train(5 )<choose_model_class>
test_predictions = optR.predict(preds, coefficients )
%%time
fold = Fold(input_df, input_X, seed=1111); torch_seed(fold.seed)
 learn3 = Learner(fold, Arch1b_LstmGru(emb_glovewiki)).train(5 )<choose_model_class>
sample_df.diagnosis = test_predictions.astype(int)
 sample_df.head()
<find_best_params><EOS> 
sample_df.to_csv('submission.csv',index=False )