Kaggle competition: APTOS 2019 Blindness Detection
<SOS> metric: QuadraticWeightedKappa Kaggle data source: aptos-2019-blindness-detection<load_pretrained>
%reload_ext autoreload
%autoreload 2
%matplotlib inline
x_train = np.load("x_train.npy")
 x_test = np.load("x_test.npy")
 y_train = np.load("y_train.npy")
 features = np.load("features.npy")
 test_features = np.load("test_features.npy")
 word_index = np.load("word_index.npy" ).item()<normalization>
from fastai import *
from fastai.vision import *
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import os
import scipy as sp
from functools import partial
from sklearn import metrics
from collections import Counter
from fastai.callbacks import *
import PIL
import cv2
seed_everything()
glove_embeddings = load_glove(word_index)
 paragram_embeddings = load_para(word_index)
 fasttext_embeddings = load_fasttext(word_index)
 embedding_matrix = np.mean([glove_embeddings, paragram_embeddings, fasttext_embeddings], axis=0)
 del glove_embeddings, paragram_embeddings, fasttext_embeddings
gc.collect()
np.shape(embedding_matrix )<split>
def seed_everything(seed=1358):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything()
splits = list(StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED ).split(x_train, y_train))
splits[:3]<choose_model_class>
PATH = Path('.. /input/aptos2019-blindness-detection' )
class CyclicLR(object):
def __init__(self, optimizer, base_lr=1e-3, max_lr=6e-3,
step_size=2000, mode='triangular', gamma=1.,
scale_fn=None, scale_mode='cycle', last_batch_iteration=-1):
if not isinstance(optimizer, Optimizer):
raise TypeError('{} is not an Optimizer'.format(
 type(optimizer ).__name__))
self.optimizer = optimizer
if isinstance(base_lr, list)or isinstance(base_lr, tuple):
if len(base_lr)!= len(optimizer.param_groups):
raise ValueError("expected {} base_lr, got {}".format(
 len(optimizer.param_groups), len(base_lr)))
 self.base_lrs = list(base_lr)
 else:
self.base_lrs = [base_lr] * len(optimizer.param_groups)
 if isinstance(max_lr, list)or isinstance(max_lr, tuple):
if len(max_lr)!= len(optimizer.param_groups):
raise ValueError("expected {} max_lr, got {}".format(
 len(optimizer.param_groups), len(max_lr)))
 self.max_lrs = list(max_lr)
 else:
self.max_lrs = [max_lr] * len(optimizer.param_groups)
 self.step_size = step_size
if mode not in ['triangular', 'triangular2', 'exp_range'] \
and scale_fn is None:
raise ValueError('mode is invalid and scale_fn is None')
 self.mode = mode
self.gamma = gamma
if scale_fn is None:
if self.mode == 'triangular':
self.scale_fn = self._triangular_scale_fn
self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = self._triangular2_scale_fn
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = self._exp_range_scale_fn
self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.batch_step(last_batch_iteration + 1)
 self.last_batch_iteration = last_batch_iteration
def batch_step(self, batch_iteration=None):
if batch_iteration is None:
batch_iteration = self.last_batch_iteration + 1
self.last_batch_iteration = batch_iteration
for param_group, lr in zip(self.optimizer.param_groups, self.get_lr()):
param_group['lr'] = lr
def _triangular_scale_fn(self, x):
return 1.
 def _triangular2_scale_fn(self, x):
return 1 /(2.**(x - 1))
def _exp_range_scale_fn(self, x):
return self.gamma**(x)
 def get_lr(self):
step_size = float(self.step_size)
 cycle = np.floor(1 + self.last_batch_iteration /(2 * step_size))
x = np.abs(self.last_batch_iteration / step_size - 2 * cycle + 1)
 lrs = []
param_lrs = zip(self.optimizer.param_groups, self.base_lrs, self.max_lrs)
 for param_group, base_lr, max_lr in param_lrs:
base_height =(max_lr - base_lr)* np.maximum(0,(1 - x))
if self.scale_mode == 'cycle':
lr = base_lr + base_height * self.scale_fn(cycle)
 else:
lr = base_lr + base_height * self.scale_fn(self.last_batch_iteration)
 lrs.append(lr)
 return lrs 
<choose_model_class>
df = pd.read_csv(PATH/'train.csv')
 df.head()
embedding_dim = 300
embedding_path = '.. /save/embedding_matrix.npy'
use_pretrained_embedding = True
hidden_size = 60
gru_len = hidden_size
Routings = 4
Num_capsule = 5
Dim_capsule = 5
dropout_p = 0.25
rate_drop_dense = 0.28
LR = 0.001
T_epsilon = 1e-7
num_classes = 30
class Embed_Layer(nn.Module):
def __init__(self, embedding_matrix=None, vocab_size=None, embedding_dim=300):
super(Embed_Layer, self ).__init__()
self.encoder = nn.Embedding(vocab_size + 1, embedding_dim)
 if use_pretrained_embedding:
self.encoder.weight.data.copy_(t.from_numpy(embedding_matrix))
def forward(self, x, dropout_p=0.25):
return nn.Dropout(p=dropout_p )(self.encoder(x))
class GRU_Layer(nn.Module):
def __init__(self):
super(GRU_Layer, self ).__init__()
self.gru = nn.GRU(input_size=300,
hidden_size=gru_len,
bidirectional=True)

def init_weights(self):
ih =(param.data for name, param in self.named_parameters() if 'weight_ih' in name)
 hh =(param.data for name, param in self.named_parameters() if 'weight_hh' in name)
 b =(param.data for name, param in self.named_parameters() if 'bias' in name)
 for k in ih:
nn.init.xavier_uniform_(k)
 for k in hh:
nn.init.orthogonal_(k)
 for k in b:
nn.init.constant_(k, 0)
 def forward(self, x):
return self.gru(x)
 class Caps_Layer(nn.Module):
def __init__(self, input_dim_capsule=gru_len * 2, num_capsule=Num_capsule, dim_capsule=Dim_capsule, \
routings=Routings, kernel_size=(9, 1), share_weights=True,
activation='default', **kwargs):
super(Caps_Layer, self ).__init__(**kwargs)
 self.num_capsule = num_capsule
self.dim_capsule = dim_capsule
self.routings = routings
self.kernel_size = kernel_size
self.share_weights = share_weights
if activation == 'default':
self.activation = self.squash
else:
self.activation = nn.ReLU(inplace=True)
 if self.share_weights:
self.W = nn.Parameter(
 nn.init.xavier_normal_(t.empty(1, input_dim_capsule, self.num_capsule * self.dim_capsule)))
 else:
self.W = nn.Parameter(
 t.randn(BATCH_SIZE, input_dim_capsule, self.num_capsule * self.dim_capsule))
def forward(self, x):
if self.share_weights:
u_hat_vecs = t.matmul(x, self.W)
 else:
print('add later')
 batch_size = x.size(0)
 input_num_capsule = x.size(1)
 u_hat_vecs = u_hat_vecs.view(( batch_size, input_num_capsule,
self.num_capsule, self.dim_capsule))
u_hat_vecs = u_hat_vecs.permute(0, 2, 1, 3)
 b = t.zeros_like(u_hat_vecs[:, :, :, 0])
 for i in range(self.routings):
b = b.permute(0, 2, 1)
 c = F.softmax(b, dim=2)
 c = c.permute(0, 2, 1)
 b = b.permute(0, 2, 1)
 outputs = self.activation(t.einsum('bij,bijk->bik',(c, u_hat_vecs)))
 if i < self.routings - 1:
b = t.einsum('bik,bijk->bij',(outputs, u_hat_vecs))
return outputs
def squash(self, x, axis=-1):
s_squared_norm =(x ** 2 ).sum(axis, keepdim=True)
 scale = t.sqrt(s_squared_norm + T_epsilon)
 return x / scale
class Capsule_Main(nn.Module):
def __init__(self, embedding_matrix=None, vocab_size=None):
super(Capsule_Main, self ).__init__()
self.embed_layer = Embed_Layer(embedding_matrix, vocab_size)
 self.gru_layer = GRU_Layer()
self.gru_layer.init_weights()
self.caps_layer = Caps_Layer()
self.dense_layer = Dense_Layer()
def forward(self, content):
content1 = self.embed_layer(content)
 content2, _ = self.gru_layer(
 content1)
 content3 = self.caps_layer(content2)
 output = self.dense_layer(content3)
 return output 
<normalization>
df.diagnosis.value_counts()
class Attention(nn.Module):
def __init__(self, feature_dim, step_dim, bias=True, **kwargs):
super(Attention, self ).__init__(**kwargs)
 self.supports_masking = True
self.bias = bias
self.feature_dim = feature_dim
self.step_dim = step_dim
self.features_dim = 0
weight = torch.zeros(feature_dim, 1)
 nn.init.xavier_uniform_(weight)
 self.weight = nn.Parameter(weight)
 if bias:
self.b = nn.Parameter(torch.zeros(step_dim))
def forward(self, x, mask=None):
feature_dim = self.feature_dim
step_dim = self.step_dim
eij = torch.mm(
 x.contiguous().view(-1, feature_dim),
self.weight
).view(-1, step_dim)
 if self.bias:
eij = eij + self.b
eij = torch.tanh(eij)
 a = torch.exp(eij)
 if mask is not None:
a = a * mask
a = a / torch.sum(a, 1, keepdim=True)+ 1e-10
weighted_input = x * torch.unsqueeze(a, -1)
 return torch.sum(weighted_input, 1)
 class NeuralNet(nn.Module):
def __init__(self):
super(NeuralNet, self ).__init__()
fc_layer = 16
fc_layer1 = 16
self.embedding = nn.Embedding(max_features, embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = nn.Dropout2d(0.1)
 self.lstm = nn.LSTM(embed_size, hidden_size, bidirectional=True, batch_first=True)
 self.gru = nn.GRU(hidden_size * 2, hidden_size, bidirectional=True, batch_first=True)
 self.lstm2 = nn.LSTM(hidden_size * 2, hidden_size, bidirectional=True, batch_first=True)
 self.lstm_attention = Attention(hidden_size * 2, maxlen)
 self.gru_attention = Attention(hidden_size * 2, maxlen)
 self.bn = nn.BatchNorm1d(16, momentum=0.5)
 self.linear = nn.Linear(hidden_size*8+3, fc_layer1)
 self.relu = nn.ReLU()
self.dropout = nn.Dropout(0.1)
 self.fc = nn.Linear(fc_layer**2,fc_layer)
 self.out = nn.Linear(fc_layer, 1)
 self.lincaps = nn.Linear(Num_capsule * Dim_capsule, 1)
 self.caps_layer = Caps_Layer()
def forward(self, x):
h_embedding = self.embedding(x[0])
 h_embedding = torch.squeeze(
 self.embedding_dropout(torch.unsqueeze(h_embedding, 0)))
 h_lstm, _ = self.lstm(h_embedding)
 h_gru, _ = self.gru(h_lstm)
 content3 = self.caps_layer(h_gru)
 content3 = self.dropout(content3)
 batch_size = content3.size(0)
 content3 = content3.view(batch_size, -1)
 content3 = self.relu(self.lincaps(content3))
h_lstm_atten = self.lstm_attention(h_lstm)
 h_gru_atten = self.gru_attention(h_gru)
 avg_pool = torch.mean(h_gru, 1)
 max_pool, _ = torch.max(h_gru, 1)
 f = torch.tensor(x[1], dtype=torch.float ).cuda()
conc = torch.cat(( h_lstm_atten, h_gru_atten,content3, avg_pool, max_pool,f), 1)
 conc = self.relu(self.linear(conc))
conc = self.bn(conc)
 conc = self.dropout(conc)
 out = self.out(conc)
 return out<categorify>
tfms = get_transforms(do_flip=True, flip_vert=True, max_rotate=0.10, max_zoom=1.3, max_warp=0.0, max_lighting=0.2 )
class MyDataset(Dataset):
def __init__(self,dataset):
self.dataset = dataset
def __getitem__(self, index):
data, target = self.dataset[index]
return data, target, index
def __len__(self):
return len(self.dataset )<define_variables>
data =(
 src.transform(tfms,size=128)
.databunch() 
.normalize(imagenet_stats)
)
 data
def sigmoid(x):
return 1 /(1 + np.exp(-x))
train_preds = np.zeros(( len(x_train)))
 test_preds = np.zeros(( len(df_test)))
 seed_everything()
x_test_cuda = torch.tensor(x_test, dtype=torch.long ).cuda()
test = torch.utils.data.TensorDataset(x_test_cuda)
 test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)
 avg_losses_f = []
avg_val_losses_f = []<data_type_conversions>
def quadratic_kappa(y_hat, y):
return torch.tensor(cohen_kappa_score(torch.round(y_hat), y, weights='quadratic'),device='cuda:0')
 learn = cnn_learner(data, base_arch=models.resnet50 ,metrics=[quadratic_kappa],model_dir='/kaggle',pretrained=True )
for i,(train_idx, valid_idx)in enumerate(splits):
x_train = np.array(x_train)
 y_train = np.array(y_train)
 features = np.array(features)
 x_train_fold = torch.tensor(x_train[train_idx.astype(int)], dtype=torch.long ).cuda()
y_train_fold = torch.tensor(y_train[train_idx.astype(int), np.newaxis], dtype=torch.float32 ).cuda()
kfold_X_features = features[train_idx.astype(int)]
kfold_X_valid_features = features[valid_idx.astype(int)]
x_val_fold = torch.tensor(x_train[valid_idx.astype(int)], dtype=torch.long ).cuda()
y_val_fold = torch.tensor(y_train[valid_idx.astype(int), np.newaxis], dtype=torch.float32 ).cuda()
model = NeuralNet()
model.cuda()
loss_fn = torch.nn.BCEWithLogitsLoss(reduction='sum')
 step_size = 300
base_lr, max_lr = 0.001, 0.003
optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()),
lr=max_lr)
 scheduler = CyclicLR(optimizer, base_lr=base_lr, max_lr=max_lr,
step_size=step_size, mode='exp_range',
gamma=0.99994)
 train = torch.utils.data.TensorDataset(x_train_fold, y_train_fold)
 valid = torch.utils.data.TensorDataset(x_val_fold, y_val_fold)
 train = MyDataset(train)
 valid = MyDataset(valid)
 train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)
 print(f'Fold {i + 1}')
 for epoch in range(n_epochs):
start_time = time.time()
model.train()
avg_loss = 0.
 for i,(x_batch, y_batch, index)in enumerate(train_loader):
f = kfold_X_features[index]
y_pred = model([x_batch,f])
 if scheduler:
scheduler.batch_step()
loss = loss_fn(y_pred, y_batch)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
avg_loss += loss.item() / len(train_loader)
 model.eval()
valid_preds_fold = np.zeros(( x_val_fold.size(0)))
 test_preds_fold = np.zeros(( len(df_test)))
 avg_val_loss = 0.
 for i,(x_batch, y_batch, index)in enumerate(valid_loader):
f = kfold_X_valid_features[index]
y_pred = model([x_batch,f] ).detach()
avg_val_loss += loss_fn(y_pred, y_batch ).item() / len(valid_loader)
 valid_preds_fold[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
elapsed_time = time.time() - start_time
print('Epoch {}/{} \t loss={:.4f} \t val_loss={:.4f} \t time={:.2f}s'.format(
 epoch + 1, n_epochs, avg_loss, avg_val_loss, elapsed_time))
avg_losses_f.append(avg_loss)
 avg_val_losses_f.append(avg_val_loss)
 for i,(x_batch,)in enumerate(test_loader):
f = test_features[i * batch_size:(i+1)* batch_size]
y_pred = model([x_batch,f] ).detach()
test_preds_fold[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
train_preds[valid_idx] = valid_preds_fold
test_preds += test_preds_fold / len(splits)
 print('All \t loss={:.4f} \t val_loss={:.4f} \t '.format(np.average(avg_losses_f),np.average(avg_val_losses_f)))
<compute_test_metric>
learn.lr_find()
learn.recorder.plot()
def bestThresshold(y_train,train_preds):
tmp = [0,0,0]
delta = 0
for tmp[0] in tqdm(np.arange(0.1, 0.501, 0.01)) :
tmp[1] = f1_score(y_train, np.array(train_preds)>tmp[0])
 if tmp[1] > tmp[2]:
delta = tmp[0]
tmp[2] = tmp[1]
print('best threshold is {:.4f} with F1 score: {:.4f}'.format(delta, tmp[2]))
return delta
delta = bestThresshold(y_train,train_preds )<save_to_csv>
lr = 1e-2
learn.fit_one_cycle(3, lr )
submission = df_test[['qid']].copy()
submission['prediction'] =(test_preds > delta ).astype(int)
 submission.to_csv('submission.csv', index=False )<define_variables>
learn.data = data =(
 src.transform(tfms,size=224)
.databunch() 
.normalize(imagenet_stats)
)
 learn.lr_find()
learn.recorder.plot()
PATTERN = re.compile(r"[\w']+|[\.\!\?\:\(\)"]+")
 WORD_PATTERN = re.compile(r"[\w']+")
 DIGIT_PATTERN = re.compile(r'\d')
 MATH_PATTERN = re.compile(r'\[\/?math\]|\\\w+' )<load_from_csv>
lr = 1e-2
learn.fit_one_cycle(5, lr )
train_data = pd.read_csv('.. /input/train.csv')
 test_data = pd.read_csv('.. /input/test.csv')
 y_train = train_data.target.values.astype('float32' )<feature_engineering>
learn.unfreeze()
learn.lr_find()
learn.recorder.plot()
for corpus in(train_data, test_data):
for _, row in corpus.iterrows() :
row['question_text'] = MATH_PATTERN.sub('', row.question_text )<load_pretrained>
learn.fit_one_cycle(5, slice(1e-6,1e-3))
w2v = gensim.models.KeyedVectors.load_word2vec_format(
 '.. /input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin',
binary=True 
)
 stemmer = SnowballStemmer('english' )<define_variables>
valid_preds = learn.get_preds(ds_type=DatasetType.Valid )
W2V_SLICE = slice(0, 300)
 END_INDEX = 300
START_INDEX = 301
UNKNOWN_INDEX = 302
NUM_1_TO_5_INDEX = 303
NUM_9_TO_18_INDEX = 304
NUM_YEAR_INDEX = 305
NUM_OTHER_INDEX = 306
JEW_OPEN_INDEX = 307
JEW_CLOSE_INDEX = 308
QUESTION_INDEX = 309
EXCLAMATION_INDEX = 310
ELLIPSIS_INDEX = 311
PERIOD_INDEX = 312
COLON_INDEX = 313
NUM_FEATURES = 314
END_ID = 0
START_ID = 1
UNKNOWN_ID = 2
NUM_1_TO_5_ID = 3
NUM_9_TO_18_ID = 4
NUM_YEAR_ID = 5
NUM_OTHER_ID = 6
JEW_OPEN_ID = 7
JEW_CLOSE_ID = 8
QUESTION_ID = 9
EXCLAMATION_ID = 10
ELLIPSIS_ID = 11
PERIOD_ID = 12
COLON_ID = 13
NUM_SPECIAL_IDS = 14
WORD_TO_ID = dict()
def lookup_word(words_list, passthrough_tokens):

 word = words_list[0]
if len(words_list)> 1:
pair = word + '_' + words_list[1]
if pair in w2v:
return pair, 2
if word.endswith("'s"):
word = word[:-2]
if word in w2v:
return word, 1
word = word.lower()
if word in w2v:
return word, 1
return(word if passthrough_tokens else None), 1
def lookup_words(words_list, passthrough_tokens):
while words_list:
word, size = lookup_word(words_list, passthrough_tokens)
 if word is not None:
yield word
words_list = words_list[size:]
for dataset in(train_data, test_data):
for _, row in dataset.iterrows() :
tokens = [
match.group(0 ).strip("'")
 for match in PATTERN.finditer(row.question_text)
 ]
for word in lookup_words(tokens, passthrough_tokens=False):
if word is None:
continue
word_id = WORD_TO_ID.get(word)
 if word_id is None:
word_id = len(WORD_TO_ID)+ NUM_SPECIAL_IDS
if word in w2v:
WORD_TO_ID[word] = word_id
NUM_IDS = len(WORD_TO_ID)+ NUM_SPECIAL_IDS<feature_engineering>
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = metrics.cohen_kappa_score(y, X_p, weights='quadratic')
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 print(-loss_partial(self.coef_['x']))
def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']
ID_TO_VEC = np.zeros(( NUM_IDS, NUM_FEATURES), dtype='float32')
 ID_TO_VEC[END_ID, END_INDEX] = 1
ID_TO_VEC[START_ID, START_INDEX] = 1
ID_TO_VEC[UNKNOWN_ID, UNKNOWN_INDEX] = 1
ID_TO_VEC[NUM_1_TO_5_ID, NUM_1_TO_5_INDEX] = 1
ID_TO_VEC[NUM_9_TO_18_ID, NUM_9_TO_18_INDEX] = 1
ID_TO_VEC[NUM_YEAR_ID, NUM_YEAR_INDEX] = 1
ID_TO_VEC[NUM_OTHER_ID, NUM_OTHER_INDEX] = 1
ID_TO_VEC[JEW_OPEN_ID, JEW_OPEN_INDEX] = 1
ID_TO_VEC[JEW_CLOSE_ID, JEW_CLOSE_INDEX] = 1
ID_TO_VEC[QUESTION_ID, QUESTION_INDEX] = 1
ID_TO_VEC[EXCLAMATION_ID, EXCLAMATION_INDEX] = 1
ID_TO_VEC[ELLIPSIS_ID, ELLIPSIS_INDEX] = 1
ID_TO_VEC[PERIOD_ID, PERIOD_INDEX] = 1
ID_TO_VEC[COLON_ID, COLON_INDEX] = 1
for word, word_id in sorted(WORD_TO_ID.items() , key=lambda kv: kv[1]):
ID_TO_VEC[word_id, W2V_SLICE] = w2v[word]<categorify>
optR = OptimizedRounder()
optR.fit(valid_preds[0],valid_preds[1] )
def word_to_id(word):
if WORD_PATTERN.match(word):
word_id = WORD_TO_ID.get(word)
 if word_id is not None:
return word_id
else:
match = DIGIT_PATTERN.search(word)
 if match:
num = int(match.group(0))
if 1 <= num <= 5:
return NUM_1_TO_5_ID
elif 9 <= num <= 18:
return NUM_9_TO_18_ID
elif 1200 <= num <= 2500:
return NUM_YEAR_ID
else:
return NUM_OTHER_ID
else:
return UNKNOWN_ID
elif word.startswith('((('):
return JEW_OPEN_ID
elif word.startswith(')))'):
return JEW_CLOSE_ID
elif word.startswith('?'):
return QUESTION_ID
elif word.startswith('!'):
return EXCLAMATION_ID
elif word == '...':
return ELLIPSIS_ID
elif word.startswith('.'):
return PERIOD_ID
elif word == ':':
return COLON_ID
else:
return None
def text_to_ids(text):
tokens = [
match.group(0 ).strip("'")
 for match in PATTERN.finditer(text)
 ]
word_ids =(
 word_to_id(word)
 for word in lookup_words(tokens, passthrough_tokens=True)
)
 return [START_ID] + [
word_id
for word_id in word_ids
if word_id is not None
] + [END_ID]
X_train = [
text_to_ids(text)
 for text in train_data.question_text
]
X_test = [
text_to_ids(text)
 for text in test_data.question_text
]
y_train = train_data.target.values<split>
sample_df = pd.read_csv(PATH/'sample_submission.csv')
 sample_df.head()
X_train_train, X_train_val, \
y_train_train, y_train_val = train_test_split(
 X_train, y_train, test_size=0.1
)<categorify>
learn.data.add_test(ImageList.from_df(sample_df,PATH,folder='test_images',suffix='.png'))
def div_up(a, b):
return(a + b - 1)// b
def batch_slice(index, size):
return slice(index * size,(index + 1)* size)
 def pad_samples(list_list_ids):
num_words = 8 * div_up(max(map(len, list_list_ids)) , 8)
 samples = np.zeros(( len(list_list_ids), num_words), dtype='int32')
 for i, list_ids in enumerate(list_list_ids):
samples[i, 0:len(list_ids)] = list_ids
return samples
class DataGenerator(keras.utils.Sequence):
def __init__(
 self,
list_list_ids,
labels=None,
batch_size=64,
num_mini_epochs=1,
shuffle=True, 
):
self.labels = None if labels is None else labels.astype('float32')
 self.list_list_ids = list_list_ids
self.batch_size = batch_size
self.shuffle = shuffle
self.indices = list(range(len(list_list_ids)))
 self.num_mini_epochs = num_mini_epochs
self.mini_epoch = -1
self.on_epoch_end()
def __len__(self):
return div_up(len(self.mini_epoch_indices), self.batch_size)
 def __getitem__(self, index):
batch_indices = self.mini_epoch_indices[batch_slice(index, self.batch_size)]
samples = pad_samples([self.list_list_ids[i] for i in batch_indices])
 if self.labels is None:
return samples
else:
return samples, self.labels[batch_indices]
def on_epoch_end(self):
self.mini_epoch =(self.mini_epoch + 1)% self.num_mini_epochs
if self.mini_epoch == 0 and self.shuffle:
np.random.shuffle(self.indices)
 mini_epoch_size = div_up(len(self.indices), self.num_mini_epochs)
 self.mini_epoch_indices = self.indices[batch_slice(self.mini_epoch, mini_epoch_size)]<choose_model_class>
preds,y = learn.get_preds(DatasetType.Test)
def combine_stuff(input_layer, *layer_specs, pool=2, unpool=False):
layer = input_layer
for size, filters in layer_specs:
layer = K.Conv1D(filters, size, padding='same' )(layer)
 layer = K.BatchNormalization()(layer)
 layer = K.LeakyReLU(0.01 )(layer)
 layer = K.MaxPooling1D(pool )(layer)
 if unpool:
layer = K.UpSampling1D(pool )(layer)
 return layer
def build_model() :
input_layer = K.Input(( None,), dtype='int32')
 manual_embed_layer = K.Embedding(
 ID_TO_VEC.shape[0],
NUM_FEATURES - 300,
weights=[ID_TO_VEC[:, 300:]],
trainable=False,
)(input_layer)
 pretrained_embed_layer = K.Embedding(
 ID_TO_VEC.shape[0],
300,
weights=[ID_TO_VEC[:, :300]],
trainable=False,
)(input_layer)
 tuning_embed_layer = K.Embedding(
 ID_TO_VEC.shape[0],
100,
embeddings_initializer='lecun_normal',
)(input_layer)
 embedded_tuning_embed_layer = K.Conv1D(300, 1 )(tuning_embed_layer)
 embed_layer = K.concatenate([
manual_embed_layer,
K.add([pretrained_embed_layer, embedded_tuning_embed_layer])
 ])
 basic_layer = combine_stuff(embed_layer,(2, 200),(2, 300), pool=4, unpool=True)
 e_basic_layer = K.concatenate([manual_embed_layer, pretrained_embed_layer, basic_layer])
 linkage_layer_4 = combine_stuff(e_basic_layer,(4, 400),(4, 500), pool=8)
 linkage_layer_6 = combine_stuff(e_basic_layer,(6, 400),(6, 500), pool=8)
 linkage_layer = K.add([linkage_layer_4, linkage_layer_6])
 sentence_layer = combine_stuff(linkage_layer,(2, 500))
layer = K.GlobalMaxPooling1D()(sentence_layer)
 layer = K.Dense(256 )(layer)
 layer = K.BatchNormalization()(layer)
 layer = K.LeakyReLU(0.01 )(layer)
 layer = K.Dense(1, activation='sigmoid' )(layer)
 return Model(input_layer, layer)
 keras_model = build_model()
keras_model.summary()<train_model>
test_predictions = optR.predict(preds, coefficients )
keras_model.compile(
 optimizer='adam',
loss='binary_crossentropy', 
)
 num_mini_epochs = 5
train_generator = DataGenerator(X_train_train, y_train_train, num_mini_epochs=num_mini_epochs)
 val_generator = DataGenerator(X_train_val, y_train_val, batch_size=256)
 keras_model.fit_generator(
 train_generator,
validation_data=val_generator,
epochs=2 * num_mini_epochs,
use_multiprocessing=True,
workers=2,
callbacks=[keras.callbacks.EarlyStopping(patience=2, restore_best_weights=True)],
verbose=2,
)<predict_on_test>
sample_df.diagnosis = test_predictions.astype(int)
 sample_df.head()
<compute_test_metric><EOS> 
sample_df.to_csv('submission.csv',index=False )