Kaggle competition: APTOS 2019 Blindness Detection
<SOS> metric: QuadraticWeightedKappa Kaggle data source: aptos-2019-blindness-detection<choose_model_class>
warnings.filterwarnings('always')
 warnings.filterwarnings('ignore')
 %matplotlib inline
style.use('fivethirtyeight')
 sns.set(style='whitegrid', color_codes=True)
 !ls.. /input/*
class ExponentialMovingAverage(Callback):
def __init__(self, model, decay=0.999, mode='epoch', n=100):

 self.decay = decay
self.mode = mode
self.ema_model = clone_model(model)
 self.ema_model.set_weights(model.get_weights())
 self.n = n
if self.mode is 'batch':
self.cnt = 0
self.ema_weights = [K.get_value(w)for w in model.trainable_weights]
self.n_weights = len(self.ema_weights)
 super(ExponentialMovingAverage, self ).__init__()
def on_batch_end(self, batch, logs={}):
if self.mode is 'batch':
self.cnt += 1
if self.cnt % self.n == 0:
self.update_weights()
def on_epoch_end(self, epoch, logs={}):
if self.mode is 'epoch':
self.update_weights()
for var, w in zip(self.ema_model.trainable_weights, self.ema_weights):
K.set_value(var, w)
 def update_weights(self):
for w_old, var_new in zip(self.ema_weights, self.model.trainable_weights):
w_old +=(1 - self.decay)*(K.get_value(var_new)- w_old )<choose_model_class>
fastai.__version__
def create_rnn_model(rnn, maxlen, embedding, max_features, embed_size,
rnn_dim=64, dense1_dim=100, dense2_dim=50,
embed_trainable=False, seed=123):
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding],
trainable=embed_trainable )(inp)
 x = Dense(dense1_dim, activation='relu',
kernel_initializer=glorot_uniform(seed=seed))(x)
 x = Bidirectional(rnn(rnn_dim, return_sequences=True,
kernel_initializer=glorot_uniform(seed=seed)) )(x)
 x = GlobalMaxPooling1D()(x)
 x = Dense(dense2_dim, activation='relu',
kernel_initializer=glorot_uniform(seed=seed))(x)
 x = Dense(1, activation='sigmoid',
kernel_initializer=glorot_uniform(seed=seed))(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam')
 return model<find_best_params>
def is_interactive() :
return 'runtime' in get_ipython().config.IPKernelApp.connection_file
print('Interactive?', is_interactive() )
def f1_best(y_val, pred_val):
best_f1 = 0
best_thresh = 0
for thresh in np.linspace(0.2, 0.4, 41):
f1 = metrics.f1_score(y_val,(pred_val > thresh ).astype(int))
if f1 > best_f1:
best_f1 = f1
best_thresh = thresh
return best_f1, best_thresh
embed_ids = [list(range(300)) , list(range(300, 600)) ,
list(range(600, 900)) , list(range(900, 1200)) ]
embed_ids_dict = {1: [embed_ids[0], embed_ids[1], embed_ids[2], embed_ids[3]],
2: [embed_ids[0] + embed_ids[1],
embed_ids[0] + embed_ids[2],
embed_ids[0] + embed_ids[3],
embed_ids[1] + embed_ids[2],
embed_ids[1] + embed_ids[3],
embed_ids[2] + embed_ids[3]],
3: [embed_ids[0] + embed_ids[1] + embed_ids[2],
embed_ids[0] + embed_ids[1] + embed_ids[3],
embed_ids[0] + embed_ids[2] + embed_ids[3],
embed_ids[1] + embed_ids[2] + embed_ids[3]],
4: [embed_ids[0] + embed_ids[1] + embed_ids[2] + embed_ids[3]]}
embed_ids_lst = embed_ids_dict[2]
embed_size = 600
rnn = CuDNNGRU
embed_trainable = False
n_models = 6
epochs = 7
batch_size = 512
dense1_dim = rnn_dim = 128
dense2_dim = 2 * rnn_dim
ema_n = int(len(train_y)/ batch_size / 10)
 decay = 0.9
scores = []
pred_avg = np.zeros(( len(val_y), 1))
pred_test_avg = np.zeros(( test_df.shape[0], 1))
for i in range(n_models):
t1 = time.time()
seed = 101 + 11 * i
cols_in_use = embed_ids_lst[i % len(embed_ids_lst)]
model = create_rnn_model(rnn, maxlen, embedding_matrix[:, cols_in_use],
max_features, embed_size,
rnn_dim=rnn_dim,
dense1_dim=dense1_dim,
dense2_dim=dense2_dim,
embed_trainable=embed_trainable,
seed=seed)
 ema = ExponentialMovingAverage(model, decay=decay, mode='batch', n=ema_n)
 model.fit(train_X, train_y, batch_size=batch_size, epochs=epochs,
callbacks=[ema], verbose=0)
 m = ema.ema_model
t_per_epoch =(time.time() - t1)/ epochs
pred = m.predict([val_X])
 pred_avg += pred
pred_test = m.predict([test_X])
 pred_test_avg += pred_test
f1_one, thresh_one = f1_best(val_y, pred)
 f1_avg, thresh_avg = f1_best(val_y, pred_avg /(i + 1))
nll_one = metrics.log_loss(val_y, pred)
 nll_avg = metrics.log_loss(val_y, pred_avg /(i + 1))
auc_one = metrics.roc_auc_score(val_y, pred)
 auc_avg = metrics.roc_auc_score(val_y, pred_avg)
 print(f' n_model:{i + 1} epoch:{epochs} F1:{f1_avg:.4f} th:{thresh_avg:.3f} ' +
f'AUC:{auc_avg:.4f} NLL:{nll_avg:.4f} One:{f1_one:.4f} {auc_one:.4f} {nll_one:.4f} ' +
f'Time:{time.time() - t1:.1f}s {t_per_epoch:.1f}s/epoch')
<save_to_csv>
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(42 )
pred_test_avg /= n_models
pred_test_avg =(pred_test_avg>thresh_avg ).astype(int)
 out_df = pd.DataFrame({"qid":test_df["qid"].values})
 out_df['prediction'] = pred_test_avg
out_df.to_csv("submission.csv", index=False)
 print(f'Done:{time.time() - t0:.1f}s' )<import_modules>
Path('/tmp/.cache/torch/checkpoints/' ).mkdir(exist_ok=True, parents=True)
 model_path = '/tmp/.cache/torch/checkpoints/efficientNet.pth'
!cp.. /input/efficientnet*/efficientNet_*.pth {model_path}
tqdm.pandas(desc='Progress')
<define_variables>
PATH = Path('.. /input/aptos2019-blindness-detection')
 df_train = pd.read_csv(PATH/'train.csv')
 df_test = pd.read_csv(PATH/'test.csv')
 _ = df_train.hist()
embed_size = 300
max_features = 120000
maxlen = 70
batch_size = 512
n_epochs = 5
n_splits = 5
SEED = 1029<set_options>
aptos19_stats =([0.42, 0.22, 0.075], [0.27, 0.15, 0.081])
 data = ImageDataBunch.from_df(df=df_train,
path=PATH, folder='train_images', suffix='.png',
valid_pct=0.1,
ds_tfms=get_transforms(flip_vert=True, max_warp=0.1, max_zoom=1.15, max_rotate=45.) ,
size=224,
bs=32,
num_workers=os.cpu_count()
).normalize(aptos19_stats )
def seed_everything(seed=1029):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
seed_everything()<features_selection>
print(f'Classes: {data.classes}' )
def load_glove(word_index):
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')[:300]
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = -0.005838499,0.48782197
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_fasttext(word_index):
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_para(word_index):
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = -0.0053247833,0.49346462
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix<load_from_csv>
data.show_batch(rows=3, figsize=(7,6))
df_train = pd.read_csv(".. /input/train.csv")
 df_test = pd.read_csv(".. /input/test.csv")
 df = pd.concat([df_train ,df_test],sort=True )<feature_engineering>
package_path = '.. /input/efficientnet-pytorch/efficientnet-pytorch/EfficientNet-PyTorch-master'
sys.path.append(package_path)
def build_vocab(texts):
sentences = texts.apply(lambda x: x.split() ).values
vocab = {}
for sentence in sentences:
for word in sentence:
try:
vocab[word] += 1
except KeyError:
vocab[word] = 1
return vocab
vocab = build_vocab(df['question_text'] )<define_variables>
def EfficientNetB4(pretrained=True):

 model = EfficientNet.from_name('efficientnet-b4', override_params={'num_classes':5})
 if pretrained:
model_state = torch.load(model_path)
 if '_fc.weight' in model_state.keys() :
model_state.pop('_fc.weight')
 model_state.pop('_fc.bias')
 res = model.load_state_dict(model_state, strict=False)
 assert str(res.missing_keys)== str(['_fc.weight', '_fc.bias']), 'issue loading pretrained weights'
else:
mapping = { i:o for i,o in zip(model_state.keys() , model.state_dict().keys())}
mapped_model_state = OrderedDict([ 
(mapping[k], v)for k,v in model_state.items() if not mapping[k].startswith('_fc')
 ])
 res = model.load_state_dict(mapped_model_state, strict=False)
 print(res)
 return model
sin = len(df_train[df_train["target"]==0])
 insin = len(df_train[df_train["target"]==1])
 persin =(sin/(sin+insin)) *100
perinsin =(insin/(sin+insin)) *100
print("
print("<feature_engineering>
model = EfficientNetB4(pretrained=True)
 model
def build_vocab(texts):
sentences = texts.apply(lambda x: x.split() ).values
vocab = {}
for sentence in sentences:
for word in sentence:
try:
vocab[word] += 1
except KeyError:
vocab[word] = 1
return vocab
def known_contractions(embed):
known = []
for contract in contraction_mapping:
if contract in embed:
known.append(contract)
 return known
def clean_contractions(text, mapping):
specials = ["’", "‘", "´", "`"]
for s in specials:
if s in text:
text = text.replace(s, "'")
 text = ' '.join([mapping[t] if t in mapping else t for t in text.split(" ")])
 return text
def correct_spelling(x, dic):
for word in dic.keys() :
if word in dic:
x = x.replace(word, dic[word])
 return x
def unknown_punct(embed, punct):
unknown = ''
for p in punct:
if p not in embed:
unknown += p
unknown += ' '
return unknown
def clean_numbers(x):
x = re.sub('[0-9]{5,}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}', '
return x
def clean_special_chars(text, punct, mapping):
for p in mapping:
if p in text:
text = text.replace(p, mapping[p])
 for p in punct:
if p in text:
text = text.replace(p, f' {p} ')
 specials = {'\u200b': ' ', '…': '...', '\ufeff': '', 'करना': '', 'है': ''}
for s in specials:
if s in text:
text = text.replace(s, specials[s])
 return text
def replace(word, pos=None):

 antonyms = set()
for syn in wordnet.synsets(word, pos=pos):
for lemma in syn.lemmas() :
for antonym in lemma.antonyms() :
antonyms.add(antonym.name())
 if len(antonyms)== 1:
return antonyms.pop()
else:
return None
def replace_negations(text):

 i, l = 0, len(text)
 words = []
while i < l:
word = text[i]
if word == 'not' and i+1 < l:
ant = replace(text[i+1])
 if ant:
words.append(ant)
 i += 2
continue
words.append(word)
 i += 1
return words



 def add_lower(embedding, vocab):
count = 0
for word in vocab:
if word in embedding and word.lower() not in embedding:
embedding[word.lower() ] = embedding[word]
count += 1
print(f"Added {count} words to embedding" )<define_variables>
class FocalLoss(nn.Module):
def __init__(self, gamma=3., reduction='mean'):
super().__init__()
self.gamma = gamma
self.reduction = reduction
def forward(self, inputs, targets):
CE_loss = nn.CrossEntropyLoss(reduction='none' )(inputs, targets)
 pt = torch.exp(-CE_loss)
 F_loss =(( 1 - pt)**self.gamma)* CE_loss
if self.reduction == 'sum':
return F_loss.sum()
elif self.reduction == 'mean':
return F_loss.mean()
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
def clean_text(x):
x = str(x)
 for punct in puncts:
if punct in x:
x = x.replace(punct, f' {punct} ')
 return x
def clean_numbers(x):
x = re.sub('[0-9]{5,}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}', '
return x
mispell_dict = {"ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because", "could've": "could have", "couldn't": "could not", "didn't": "did not", "doesn't": "does not", "don't": "do not", "hadn't": "had not", "hasn't": "has not", "haven't": "have not", "he'd": "he would","he'll": "he will", "he's": "he is", "how'd": "how did", "how'd'y": "how do you", "how'll": "how will", "how's": "how is", "I'd": "I would", "I'd've": "I would have", "I'll": "I will", "I'll've": "I will have","I'm": "I am", "I've": "I have", "i'd": "i would", "i'd've": "i would have", "i'll": "i will", "i'll've": "i will have","i'm": "i am", "i've": "i have", "isn't": "is not", "it'd": "it would", "it'd've": "it would have", "it'll": "it will", "it'll've": "it will have","it's": "it is", "let's": "let us", "ma'am": "madam", "mayn't": "may not", "might've": "might have","mightn't": "might not","mightn't've": "might not have", "must've": "must have", "mustn't": "must not", "mustn't've": "must not have", "needn't": "need not", "needn't've": "need not have","o'clock": "of the clock", "oughtn't": "ought not", "oughtn't've": "ought not have", "shan't": "shall not", "sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would", "she'd've": "she would have", "she'll": "she will", "she'll've": "she will have", "she's": "she is", "should've": "should have", "shouldn't": "should not", "shouldn't've": "should not have", "so've": "so have","so's": "so as", "this's": "this is","that'd": "that would", "that'd've": "that would have", "that's": "that is", "there'd": "there would", "there'd've": "there would have", "there's": "there is", "here's": "here is","they'd": "they would", "they'd've": "they would have", "they'll": "they will", "they'll've": "they will have", "they're": "they are", "they've": "they have", "to've": "to have", "wasn't": "was not", "we'd": "we would", "we'd've": "we would have", "we'll": "we will", "we'll've": "we will have", "we're": "we are", "we've": "we have", "weren't": "were not", "what'll": "what will", "what'll've": "what will have", "what're": "what are", "what's": "what is", "what've": "what have", "when's": "when is", "when've": "when have", "where'd": "where did", "where's": "where is", "where've": "where have", "who'll": "who will", "who'll've": "who will have", "who's": "who is", "who've": "who have", "why's": "why is", "why've": "why have", "will've": "will have", "won't": "will not", "won't've": "will not have", "would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have", "y'all": "you all", "y'all'd": "you all would","y'all'd've": "you all would have","y'all're": "you all are","y'all've": "you all have","you'd": "you would", "you'd've": "you would have", "you'll": "you will", "you'll've": "you will have", "you're": "you are", "you've": "you have", 'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', "mastrubating": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', "whst": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization', 'oher':'other', 'fuckin':'fucking', }
def _get_mispell(mispell_dict):
mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))
return mispell_dict, mispell_re
mispellings, mispellings_re = _get_mispell(mispell_dict)
 def replace_typical_misspell(text):
def replace(match):
return mispellings[match.group(0)]
return mispellings_re.sub(replace, text )<feature_engineering>
class OverSamplingCallback(LearnerCallback):
def __init__(self,learn:Learner, weights:torch.Tensor=None):
super().__init__(learn)
 self.labels = self.learn.data.train_dl.dataset.y.items
_, counts = np.unique(self.labels, return_counts=True)
 self.weights =(weights if weights is not None else
torch.DoubleTensor(( 1/counts)[self.labels]))
def on_train_begin(self, **kwargs):
self.learn.data.train_dl.dl.batch_sampler = BatchSampler(
 WeightedRandomSampler(self.weights, len(self.learn.data.train_dl.dataset)) ,
self.learn.data.train_dl.batch_size,False )
def add_features(df):
df['question_text'] = df['question_text'].progress_apply(lambda x:str(x))
df['total_length'] = df['question_text'].progress_apply(len)
 df['capitals'] = df['question_text'].progress_apply(lambda comment: sum(1 for c in comment if c.isupper()))
df['caps_vs_length'] = df.progress_apply(lambda row: float(row['capitals'])/float(row['total_length']),
axis=1)
 df['num_words'] = df.question_text.str.count('\S+')
 df['num_unique_words'] = df['question_text'].progress_apply(lambda comment: len(set(w for w in comment.split())))
 df['words_vs_unique'] = df['num_unique_words'] / df['num_words']
return df
def load_and_prec() :
train_df_original = pd.read_csv(".. /input/train.csv")
 test_df_original = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df_original.shape)
 print("Test shape : ",test_df_original.shape)
 train_df = train_df_original
test_df = test_df_original
train_df["question_text"] = train_df["question_text"].apply(lambda x: x.lower())
 test_df["question_text"] = test_df["question_text"].apply(lambda x: x.lower())
 train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: clean_text(x))
test_df["question_text"] = test_df["question_text"].apply(lambda x: clean_text(x))
train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: clean_numbers(x))
test_df["question_text"] = test_df["question_text"].apply(lambda x: clean_numbers(x))
train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: replace_typical_misspell(x))
test_df["question_text"] = test_df["question_text"].apply(lambda x: replace_typical_misspell(x))
train_X = train_df["question_text"].fillna("_
test_X = test_df["question_text"].fillna("_
train = add_features(train_df_original)
 test = add_features(test_df_original)
 features = train[['caps_vs_length', 'words_vs_unique']].fillna(0)
 test_features = test[['caps_vs_length', 'words_vs_unique']].fillna(0)
 ss = StandardScaler()
ss.fit(np.vstack(( features, test_features)))
 features = ss.transform(features)
 test_features = ss.transform(test_features)
 train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: replace_negations(x))
test_df["question_text"] = test_df["question_text"].apply(lambda x: replace_negations(x))
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
np.random.seed(SEED)
 trn_idx = np.random.permutation(len(train_X))
train_X = train_X[trn_idx]
train_y = train_y[trn_idx]
return train_X, test_X, train_y, features, test_features, tokenizer.word_index 
<train_model>
learn = Learner(data, model,
loss_func=FocalLoss() ,
metrics=[accuracy, KappaScore(weights="quadratic")],
callback_fns=[BnFreeze,
partial(SaveModelCallback, monitor='kappa_score', name='best_kappa')] 
)
 learn.split(lambda m:(model._conv_head,))
learn.freeze()
learn.model_dir = '/tmp/'
x_train, x_test, y_train, features, test_features, word_index = load_and_prec() 
<define_variables>
learn.freeze()
learn.lr_find(start_lr=1e-5, end_lr=1e1, wd=5e-3)
 learn.recorder.plot(suggestion=True )
questions = list(df_train['question_text'])
 targets = list(df_train['target'] )<save_model>
learn.fit_one_cycle(1, max_lr=3e-3, pct_start=0.1, div_factor=10, final_div=30, wd=5e-3, moms=(0.9, 0.8))
learn.save('stage-1')
 learn.recorder.plot_losses()
np.save("x_train",x_train)
 np.save("x_test",x_test)
 np.save("y_train",y_train)
 np.save("features",features)
 np.save("test_features",test_features)
 np.save("word_index.npy",word_index )<load_pretrained>
learn.fit_one_cycle(3, max_lr=slice(1e-4, 1e-3), div_factor=50, final_div=100, wd=1e-3, moms=(0.9, 0.8))
learn.save('stage-2')
 learn.recorder.plot_losses()
learn.recorder.plot_lr(show_moms=True )
x_train = np.load("x_train.npy")
 x_test = np.load("x_test.npy")
 y_train = np.load("y_train.npy")
 features = np.load("features.npy")
 test_features = np.load("test_features.npy")
 word_index = np.load("word_index.npy" ).item()<set_options>
tta_params = {'beta':0.12, 'scale':1.0}
warnings.filterwarnings("ignore" )<normalization>
sample_df = pd.read_csv('.. /input/aptos2019-blindness-detection/sample_submission.csv')
 sample_df.head()
seed_everything()
glove_embeddings = load_glove(word_index)
 paragram_embeddings = load_para(word_index)
 embedding_matrix = np.mean([glove_embeddings, paragram_embeddings], axis=0)
 del glove_embeddings, paragram_embeddings
gc.collect()<split>
learn.data.add_test(ImageList.from_df(
 sample_df, PATH,
folder='test_images',
suffix='.png' 
))
splits = list(StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED ).split(x_train, y_train))
splits[:3]<choose_model_class>
preds,y = learn.TTA(ds_type=DatasetType.Test, **tta_params )
class CyclicLR(object):
def __init__(self, optimizer, base_lr=1e-3, max_lr=6e-3,
step_size=2000, mode='triangular', gamma=1.,
scale_fn=None, scale_mode='cycle', last_batch_iteration=-1):
if not isinstance(optimizer, Optimizer):
raise TypeError('{} is not an Optimizer'.format(
 type(optimizer ).__name__))
self.optimizer = optimizer
if isinstance(base_lr, list)or isinstance(base_lr, tuple):
if len(base_lr)!= len(optimizer.param_groups):
raise ValueError("expected {} base_lr, got {}".format(
 len(optimizer.param_groups), len(base_lr)))
 self.base_lrs = list(base_lr)
 else:
self.base_lrs = [base_lr] * len(optimizer.param_groups)
 if isinstance(max_lr, list)or isinstance(max_lr, tuple):
if len(max_lr)!= len(optimizer.param_groups):
raise ValueError("expected {} max_lr, got {}".format(
 len(optimizer.param_groups), len(max_lr)))
 self.max_lrs = list(max_lr)
 else:
self.max_lrs = [max_lr] * len(optimizer.param_groups)
 self.step_size = step_size
if mode not in ['triangular', 'triangular2', 'exp_range'] \
and scale_fn is None:
raise ValueError('mode is invalid and scale_fn is None')
 self.mode = mode
self.gamma = gamma
if scale_fn is None:
if self.mode == 'triangular':
self.scale_fn = self._triangular_scale_fn
self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = self._triangular2_scale_fn
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = self._exp_range_scale_fn
self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.batch_step(last_batch_iteration + 1)
 self.last_batch_iteration = last_batch_iteration
def batch_step(self, batch_iteration=None):
if batch_iteration is None:
batch_iteration = self.last_batch_iteration + 1
self.last_batch_iteration = batch_iteration
for param_group, lr in zip(self.optimizer.param_groups, self.get_lr()):
param_group['lr'] = lr
def _triangular_scale_fn(self, x):
return 1.
 def _triangular2_scale_fn(self, x):
return 1 /(2.**(x - 1))
def _exp_range_scale_fn(self, x):
return self.gamma**(x)
 def get_lr(self):
step_size = float(self.step_size)
 cycle = np.floor(1 + self.last_batch_iteration /(2 * step_size))
x = np.abs(self.last_batch_iteration / step_size - 2 * cycle + 1)
 lrs = []
param_lrs = zip(self.optimizer.param_groups, self.base_lrs, self.max_lrs)
 for param_group, base_lr, max_lr in param_lrs:
base_height =(max_lr - base_lr)* np.maximum(0,(1 - x))
if self.scale_mode == 'cycle':
lr = base_lr + base_height * self.scale_fn(cycle)
 else:
lr = base_lr + base_height * self.scale_fn(self.last_batch_iteration)
 lrs.append(lr)
 return lrs<choose_model_class>
sample_df.diagnosis = preds.argmax(1)
 sample_df.head()
<categorify><EOS> 
sample_df.to_csv('submission.csv',index=False)
 _ = sample_df.hist()