Kaggle competition: Quora Insincere Questions Classification
model = load_model(PATH_BASE+MODEL_BASE+'fine_tune_weights.hdf5', custom_objects={'f1':f1, 'binary_focal_loss': binary_focal_loss} )<train_model>
embed_size = 600
max_features = None
maxlen = 57
for layer in model.layers[:]:
layer.trainable =True
model.compile(
 loss=[binary_focal_loss],
optimizer=Adam(1e-4),
metrics=['acc', f1])
 checkpointer = ModelCheckpoint('fine_tune_weights.hdf5', verbose=2, monitor='val_loss', save_best_only=True, mode='max')
 generator = data_generator()
train_generator = generator.batch_train(train_idx, BATCH_SIZE, SHAPE, augment=True)
 validation_generator = generator.batch_train(test_idx, 620, SHAPE, augment=False )<train_model>
import os
import time
import numpy as np
import pandas as pd
from tqdm import tqdm
import math
from sklearn.model_selection import train_test_split
from sklearn import metrics
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Input, CuDNNLSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D
from keras.layers import Bidirectional, GlobalMaxPool1D, GlobalMaxPooling1D, GlobalAveragePooling1D
from keras.layers import Input, Embedding, Dense, Conv2D, MaxPool2D, concatenate, Lambda
from keras.layers import Reshape, Flatten, Concatenate, Dropout, SpatialDropout1D, BatchNormalization
from keras.optimizers import Adam, Nadam
from keras.models import Model
from keras import backend as K
from keras.callbacks import Callback
from keras import initializers, regularizers, constraints, optimizers, layers
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau
from keras.engine.topology import Layer
import gc, re
from sklearn import metrics
from sklearn.model_selection import GridSearchCV, StratifiedKFold
from sklearn.metrics import f1_score, roc_auc_score, confusion_matrix, auc, precision_recall_curve
history = model.fit_generator(
 train_generator,
steps_per_epoch=100,
validation_data=next(validation_generator),
epochs=80,
verbose=1,
callbacks=[checkpointer] )<define_variables>
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
def clean_text(x):
x = str(x)
 for punct in puncts:
if punct in x:
x = x.replace(punct, f' {punct} ')
 return x
def clean_numbers(x):
x = re.sub('[0-9]{5,}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}', '
return x
def _get_mispell(mispell_dict):
mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))
return mispell_dict, mispell_re
mispell_dict = {"aren't" : "are not",
"can't" : "cannot",
"couldn't" : "could not",
"didn't" : "did not",
"doesn't" : "does not",
"don't" : "do not",
"hadn't" : "had not",
"hasn't" : "has not",
"haven't" : "have not",
"he'd" : "he would",
"he'll" : "he will",
"he's" : "he is",
"i'd" : "I would",
"i'd" : "I had",
"i'll" : "I will",
"i'm" : "I am",
"isn't" : "is not",
"it's" : "it is",
"it'll":"it will",
"i've" : "I have",
"let's" : "let us",
"mightn't" : "might not",
"mustn't" : "must not",
"shan't" : "shall not",
"she'd" : "she would",
"she'll" : "she will",
"she's" : "she is",
"shouldn't" : "should not",
"that's" : "that is",
"there's" : "there is",
"they'd" : "they would",
"they'll" : "they will",
"they're" : "they are",
"they've" : "they have",
"we'd" : "we would",
"we're" : "we are",
"weren't" : "were not",
"we've" : "we have",
"what'll" : "what will",
"what're" : "what are",
"what's" : "what is",
"what've" : "what have",
"where's" : "where is",
"who'd" : "who would",
"who'll" : "who will",
"who're" : "who are",
"who's" : "who is",
"who've" : "who have",
"won't" : "will not",
"wouldn't" : "would not",
"you'd" : "you would",
"you'll" : "you will",
"you're" : "you are",
"you've" : "you have",
"'re": " are",
"wasn't": "was not",
"we'll":" will",
"didn't": "did not"}
mispellings, mispellings_re = _get_mispell(mispell_dict)
 def replace_typical_misspell(text):
def replace(match):
return mispellings[match.group(0)]
return mispellings_re.sub(replace, text )
n_list = np.arange(0.1,0.5,0.02)
 for idx in test_idx[:3]:
name0 = idx['name']
print(idx)
 print(idx['name'])
 print(name0 )<init_hyperparams>
def load_and_prec() :
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape)
 train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: clean_text(x))
test_df["question_text"] = test_df["question_text"].progress_apply(lambda x: clean_text(x))
train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: clean_numbers(x))
test_df["question_text"] = test_df["question_text"].progress_apply(lambda x: clean_numbers(x))
train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: replace_typical_misspell(x))
test_df["question_text"] = test_df["question_text"].progress_apply(lambda x: replace_typical_misspell(x))
train_X = train_df["question_text"].fillna("_
splits = list(StratifiedKFold(n_splits=10,random_state=2018 ).split(train_X,train_df['target'].values))
test_X = test_df["question_text"].fillna("_
train_X, val_X = train_X[splits[0][0]],train_X[splits[0][1]]
train_y = train_df['target'].values[splits[0][0]]
val_y = train_df['target'].values[splits[0][1]]
tokenizer = Tokenizer(num_words=max_features, filters='', lower=True)
 tokenizer.fit_on_texts(list(train_X)+list(test_X))
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 val_X = pad_sequences(val_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 np.random.seed(SEED)
 trn_idx = np.random.permutation(len(train_X))
val_idx = np.random.permutation(len(val_X))
train_X = train_X[trn_idx]
val_X = val_X[val_idx]
train_y = train_y[trn_idx]
val_y = val_y[val_idx]
return train_X, val_X, test_X, train_y, val_y, tokenizer.word_index
%%time
TP_data = {}
FP_data = {}
FN_data = {}
F1_best = 0
F1_ther = 0
for threshold in tqdm(n_list):
F1_sum = 0
TP_datai = {}
FP_datai = {}
FN_datai = {}
for i in range(28):
TP_datai[i] = 0
FP_datai[i] = 0
FN_datai[i] = 0
for idx in test_idx[:500]:
name0 = idx['name']
generator = data_generator()
image = generator.load_img(name0, SHAPE)
 score_predict = model.predict(image[np.newaxis,:])
 score_predict = np.array(score_predict)[0]
label_predict = np.arange(28)[score_predict>=threshold]
true_label = idx['label']
true_label = np.array(true_label ).astype(int)
 label_predict = set(label_predict)
 true_label = set(true_label)
 TP = sum(1 for num in label_predict if num in true_label)
 FP = sum(1 for num in label_predict if not num in true_label)
 FN = sum(1 for num in true_label if not num in label_predict)
 TN = 28 -(TP+FP+FN)
 F1_sum += 2*TP/(2*TP+FN+FP)
 for num in label_predict:
if num in true_label:
TP_datai[num] += 1
if num not in true_label:
FP_datai[num] += 1
for num in true_label:
if num not in label_predict:
FN_datai[num] += 1
if F1_sum>F1_best:
F1_best = F1_sum
F1_thre = threshold
TP_data = TP_datai
FP_data = FP_datai
FN_data = FN_datai
print('F1_score_sum: ', F1_sum, 'at threshold: ', threshold)
 TP_data = SortedDict(TP_data)
 FP_data = SortedDict(FP_data)
 FN_data = SortedDict(FN_data)
 print('F1_best ', F1_best, ' F1_thre ', F1_thre)
 print('TP_data ', TP_data)
 print('FP_data ', FP_data)
 print('FN_data ', FN_data )<load_from_csv>
def load_glove(word_index):
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
emb_mean,emb_std = -0.005838499,0.48782197
embed_size = 300
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
with open(EMBEDDING_FILE, 'r', encoding="utf8")as f:
for line in f:
word, vec = line.split(' ', 1)
 if word not in word_index:
continue
i = word_index[word]
if i >= nb_words:
continue
embedding_vector = np.asarray(vec.split(' '), dtype='float32')[:300]
if len(embedding_vector)== 300:
embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_fasttext(word_index):
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: break
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_para(word_index):
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
emb_mean,emb_std = -0.0053247833,0.49346462
embed_size = 300
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
with open(EMBEDDING_FILE, 'r', encoding="utf8", errors='ignore')as f:
for line in f:
word, vec = line.split(' ', 1)
 if word not in word_index:
continue
i = word_index[word]
if i >= nb_words:
continue
embedding_vector = np.asarray(vec.split(' '), dtype='float32')[:300]
if len(embedding_vector)== 300:
embedding_matrix[i] = embedding_vector
return embedding_matrix
submit = pd.read_csv(PATH_BASE+TRAIN_BASE+'sample_submission.csv' )<predict_on_test>
def threshold_search(y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in [i * 0.01 for i in range(100)]:
score = f1_score(y_true=y_true, y_pred=y_proba > threshold)
 if score > best_score:
best_threshold = threshold
best_score = score
rocauc = roc_auc_score(y_true, y_proba)
 p, r, _ = precision_recall_curve(y_true, y_proba)
 prauc = auc(r, p)
 search_result = {'threshold': best_threshold, 'f1': best_score, 'rocauc': rocauc, 'prauc': prauc}
return search_result
%%time
PATH_TRAIN = PATH_BASE+TRAIN_BASE+'test/'
generator = data_generator()
predicted = []
for name in tqdm(submit['Id']):
image = generator.load_img(name, SHAPE)
 score_predict = model.predict(image[np.newaxis,:])[0]
label_predict = np.arange(28)[score_predict>=F1_thre]
str_predict_label = ' '.join(str(l)for l in label_predict)
 predicted.append(str_predict_label )<save_to_csv>
class CyclicLR(Callback):

 def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',
gamma=1., scale_fn=None, scale_mode='cycle'):
super(CyclicLR, self ).__init__()
self.base_lr = base_lr
self.max_lr = max_lr
self.step_size = step_size
self.mode = mode
self.gamma = gamma
if scale_fn == None:
if self.mode == 'triangular':
self.scale_fn = lambda x: 1.
 self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = lambda x: 1/(2.**(x-1))
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = lambda x: gamma**(x)
 self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.clr_iterations = 0.
 self.trn_iterations = 0.
 self.history = {}
self._reset()
def _reset(self, new_base_lr=None, new_max_lr=None,
new_step_size=None):

 if new_base_lr != None:
self.base_lr = new_base_lr
if new_max_lr != None:
self.max_lr = new_max_lr
if new_step_size != None:
self.step_size = new_step_size
self.clr_iterations = 0.
 def clr(self):
cycle = np.floor(1+self.clr_iterations/(2*self.step_size))
x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)
 if self.scale_mode == 'cycle':
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(cycle)
 else:
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(self.clr_iterations)
 def on_train_begin(self, logs={}):
logs = logs or {}
if self.clr_iterations == 0:
K.set_value(self.model.optimizer.lr, self.base_lr)
 else:
K.set_value(self.model.optimizer.lr, self.clr())
 def on_batch_end(self, epoch, logs=None):
logs = logs or {}
self.trn_iterations += 1
self.clr_iterations += 1
self.history.setdefault('lr', [] ).append(K.get_value(self.model.optimizer.lr))
self.history.setdefault('iterations', [] ).append(self.trn_iterations)
 for k, v in logs.items() :
self.history.setdefault(k, [] ).append(v)
 K.set_value(self.model.optimizer.lr, self.clr() )
submit['Predicted'] = predicted
submit.to_csv('4 channel V2 with rare T2 plus threshold.csv', index=False )<set_options>
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
%matplotlib inline 
<load_pretrained>
def model_gru_conv_3(embedding_matrix):
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = SpatialDropout1D(0.2 )(x)
 x0 = Bidirectional(CuDNNLSTM(128, kernel_initializer=initializers.glorot_uniform(seed = 2018), return_sequences=True))(x)
 x1 = Bidirectional(CuDNNGRU(64, kernel_initializer=initializers.glorot_uniform(seed = 2018), return_sequences=True))(x0)
 z = Conv1D(64, kernel_size = 1, kernel_initializer=initializers.he_uniform(seed=2018), activation = "tanh" )(x1)
 y1 = GlobalMaxPooling1D()(x1)
 y2 = GlobalMaxPooling1D()(z)
 x = concatenate([y1,y2])
 outp= Dense(1, kernel_initializer=initializers.he_uniform(seed=2018), activation='sigmoid' )(x)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy', optimizer=Adam() , metrics=['accuracy'])
 return model
try:
shutil.copytree(".. /input/keras-pretrained-models", "/tmp/.keras/models")
 except FileExistsError:
pass<import_modules>
SEED=2018
def model_RCNN(embedding_matrix, hidden_dim_1=128, hidden_dim_2=64,max_features=max_features):
embedding_matrix = np.concatenate([embedding_matrix,np.zeros(( 1,np.shape(embedding_matrix)[1])) ])
 print(np.shape(embedding_matrix))
left_context = Input(shape=(maxlen,))
document = Input(shape=(maxlen,))
right_context = Input(shape=(maxlen,))
embedder = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False)
 doc_embedding = embedder(document)
 doc_embedding = SpatialDropout1D(0.2 )(doc_embedding)
 l_embedding = embedder(left_context)
 l_embedding = SpatialDropout1D(0.2 )(l_embedding)
 r_embedding = embedder(right_context)
 r_embedding = SpatialDropout1D(0.2 )(r_embedding)
 forward = CuDNNLSTM(hidden_dim_1, return_sequences = True )(l_embedding)
 backward = CuDNNLSTM(hidden_dim_1, return_sequences = True )(r_embedding)
 backward = Lambda(lambda x: K.reverse(x, axes = 1))(backward)
 together = concatenate([forward, doc_embedding, backward], axis = 2)
 semantic = Conv1D(hidden_dim_2*8, kernel_size = 1, kernel_initializer=initializers.he_uniform(seed=SEED), activation = "tanh" )(together)
 pool_rnn = GlobalMaxPool1D()(semantic)
 output = Dense(1, kernel_initializer=initializers.he_uniform(seed=SEED), activation = "sigmoid" )(pool_rnn)
 model = Model(inputs = [left_context, document, right_context], outputs = output)
 model.compile(optimizer = Adam() , loss = 'binary_crossentropy', metrics = ["accuracy"])
 return model
Conv1D, Conv2D, Conv3D, \
Conv2DTranspose, \
AveragePooling1D, AveragePooling2D, \
MaxPooling1D, MaxPooling2D, MaxPooling3D, \
GlobalAveragePooling1D, \
GlobalMaxPooling1D, GlobalMaxPooling2D, GlobalMaxPooling3D, \
LocallyConnected1D, LocallyConnected2D, \
concatenate, Flatten, Average, Activation, \
RepeatVector, Permute, Reshape, Dot, \
multiply, dot, add, \
PReLU, \
Bidirectional, TimeDistributed, \
SpatialDropout1D, \
BatchNormalization 
<import_modules>
def model_lstm_atten(embedding_matrix):
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = SpatialDropout1D(0.2 )(x)
 x0 = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x2 = Bidirectional(CuDNNGRU(96, return_sequences=True))(x0)
 y2 = GlobalMaxPooling1D()(x2)
 y2 = Dropout(0.1 )(y2)
 x = Dense(1, activation="sigmoid" )(y2)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=Adam() , metrics=['accuracy'])
 return model
from PIL import Image
from zipfile import ZipFile
import h5py
import cv2
from tqdm import tqdm<define_variables>
def model_lstm_max(embedding_matrix):
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = SpatialDropout1D(0.2 )(x)
 x0 = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x1 = Bidirectional(CuDNNGRU(64, kernel_initializer=initializers.glorot_uniform(seed = 2018), return_sequences=True))(x0)
 z = Conv1D(64, kernel_size = 1, kernel_initializer=initializers.he_uniform(seed=2018), activation = "tanh" )(x1)
 y2 = Attention(maxlen )(z)
 x = Dense(1, activation="sigmoid" )(y2)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=Adam() , metrics=['accuracy'])
 return model
src_dir = '.. /input/human-protein-atlas-image-classification'<load_from_csv>
def get_train_list(train_X):
return [np.concatenate(( np.ones(( np.shape(train_X)[0],1)) *max_features+1,train_X[:,1:]),1),train_X,np.concatenate(( np.ones(( np.shape(train_X)[0],1)) *max_features+1,train_X[:,::-1][:,1:]),1)]
def RCNN_train_pred(model, epochs=2):
train_X_list = get_train_list(train_X)
 test_X_list=get_train_list(test_X)
 val_X_list=get_train_list(val_X)
 for e in range(epochs):
model.fit(train_X_list, train_y, batch_size=512, epochs=1, validation_data=(val_X_list, val_y),callbacks=[clr])
 pred_val_y = model.predict(val_X_list, batch_size=1024, verbose=0)
 pred_test_y = model.predict(test_X_list, batch_size=1024, verbose=0)
 return pred_val_y, pred_test_y
train_labels = pd.read_csv(os.path.join(src_dir, "train.csv"))
print(train_labels.shape)
 train_labels.head(10 )<load_from_csv>
def train_pred(model, epochs=2):
for e in range(epochs):
model.fit(train_X, train_y, batch_size=512, epochs=1, validation_data=(val_X, val_y),verbose=1,callbacks=[clr])
 pred_val_y = model.predict([val_X], batch_size=1024, verbose=0)
 pred_test_y = model.predict([test_X], batch_size=1024, verbose=0)
 return pred_val_y, pred_test_y
test_labels = pd.read_csv(os.path.join(src_dir, "sample_submission.csv"))
print(test_labels.shape)
 test_labels.head()<define_variables>
train_X, val_X, test_X, train_y, val_y, word_index = load_and_prec()
max_features = len(word_index)
 print(max_features)
 embedding_matrix_1 = load_glove(word_index)
 embedding_matrix_2 = load_fasttext(word_index)
 embedding_matrix = np.concatenate([embedding_matrix_1, embedding_matrix_2], axis = 1)
 np.shape(embedding_matrix )
SH =(139, 139)
 ID_LIST_TRAIN = train_labels.Id.tolist()<load_pretrained>
outputs = []
img_cache_train = np.load('.. /input/139x139-resized-numpy-array/img_cache_train_resize_139x139.npz')['x']<data_type_conversions>
clr = CyclicLR(base_lr=0.001, max_lr=0.003,step_size=300., mode='exp_range', gamma=0.99994 )
def get_arr(Id, test=False):
if test:
arr = get_arr0(Id, test=True)
 arr = cv2.resize(arr, SH ).astype('float32')
 else:
ii = ID_LIST_TRAIN.index(Id)
 arr = img_cache_train[ii]
arr = arr.astype('float32')
 return arr<split>
pred_val_y, pred_test_y = RCNN_train_pred(model_RCNN(embedding_matrix, hidden_dim_1=128, hidden_dim_2=64,max_features=max_features+1), epochs = 5)
 outputs.append([pred_val_y, pred_test_y, 'RCNN'])
 results = threshold_search(val_y, pred_val_y)
 print(results)
 print(confusion_matrix(val_y,pred_val_y>results['threshold']))
y_cat_train_dic = {}
for icat in range(28):
target = str(icat)
 y_cat_train_5 = np.array([int(target in ee.split())for ee in train_labels.Target.tolist() ])
 y_cat_train_dic[icat] = y_cat_train_5<feature_engineering>
clr = CyclicLR(base_lr=0.001, max_lr=0.003,step_size=300., mode='exp_range', gamma=0.99994 )
up_sample = {}
for k in y_cat_train_dic:
v = y_cat_train_dic[k].sum()
up_sample[k] = np.ceil(( train_labels.shape[0]/28)/ v)
 up_sample<sort_values>
pred_val_y, pred_test_y = train_pred(model_lstm_atten(embedding_matrix), epochs = 4)
 outputs.append([pred_val_y, pred_test_y, 'LSTM w/ max'])
 results = threshold_search(val_y, pred_val_y)
 print(results)
 print(confusion_matrix(val_y,pred_val_y>results['threshold']))
up_sample2 = list(zip(*sorted(list(up_sample.items()), key=lambda x: x[0])))[1]
up_sample2 = np.array(up_sample2)
 up_sample2<define_variables>
clr = CyclicLR(base_lr=0.001, max_lr=0.003,step_size=300., mode='exp_range', gamma=0.99994 )
class Seq(object):
sections = None
index = None
def __init__(self, df, extend=False, aug=False, test=False, batch_size=32):
self.shaffle = None
self.extend = extend
self.aug = aug
self.test = test
self.batch_size = batch_size
self.df = df
self.ids = self.df.Id.tolist()
self.reversed = sorted(range(SH[0]), reverse=True)
 self.initialize_it()
self.len = 1
for _ in self.it:
self.len += 1
self.initialize_it()
def initialize_it(self):
if self.shaffle:

 raise NotImplementedError
self.it = iter(range(0, len(self.ids), self.batch_size))
self.idx_next = self.it.__next__()
def __len__(self):
return self.len
def __iter__(self):
return self
def __next__(self):
idx = self.idx_next
self.ids_part = self.ids[idx:(( idx+self.batch_size)if idx+self.batch_size<len(self.ids)else len(self.ids)) ]
res = self.getpart(self.ids_part)
 try:
self.idx_next = self.it.__next__()
except StopIteration:
self.initialize_it()
return res
def __getitem__(self, id0):
arr, tgts = self.get_data(id0)
 cat = self.convert_tgts(tgts)
 return arr, cat
k_list = list(range(4))
def random_transform(self, arr):
k = random.choice(self.k_list)
 arr0 = np.rot90(arr, k=k)
 if random.randint(0,1):
arr0 = arr0[self.reversed,:,:]
if random.randint(0,1):
arr0 = arr0[:,self.reversed,:]
return arr0
def convert_tgts(self, tgts):
try:
cats = to_categorical(tgts, num_classes=28)
 cat = cats.sum(axis=0)
 except TypeError:
cat = np.zeros(( 28,))
return cat
def get_data(self, id0):
arr = get_arr(id0, test=self.test)
 try:
y0 =(self.df.Target[self.df.Id == id0] ).tolist() [0]
y1 = y0.split()
y = [int(ee)for ee in y1]
except AttributeError:
y = None
return arr, y
def getpart(self, ids):
xs = []
ys = []
for id0 in ids:
self.extend_data(id0, xs, ys)
 x = np.stack(xs)
 y = np.stack(ys)
 x_dummy = np.zeros(( len(x), 1))
x_ret = {
'input': x,
'input_cls': y,
}
y_ret = {
'path_fit_cls_img': x_dummy,
'path_cls_img_cls': y,
'path_fit_imgA': x_dummy,
'path_fit_img_cls_img': x_dummy,
'path_cls_cls': y,
'path_img_cls': y,
'path_img_img_cls': y,
'path_fit_cls_img_imgE': x_dummy,
}
return(x_ret, y_ret)
 def split(self, arr, sections=sections):
res0 = np.vsplit(arr, sections)
 res = [np.hsplit(ee, sections)for ee in res0]
res = list(chain.from_iterable(res))
return res
def extend_data(self, id0, xs, ys):
arr0, cat = self[id0]
if self.extend:
mm = up_sample2[cat==1].max()
mm = int(mm)
 for ii in range(mm):
if self.aug:
img = self.random_transform(arr0)
 else:
img = arr0
xs.append(img.flatten())
 ys.append(cat)
 else:
if self.aug:
img = self.random_transform(arr0)
 else:
img = arr0
xs.append(img.flatten())
 ys.append(cat )<import_modules>
pred_val_y, pred_test_y = train_pred(model_gru_conv_3(embedding_matrix), epochs = 4)
 outputs.append([pred_val_y, pred_test_y, 'LSTM conv 3'])
 results = threshold_search(val_y, pred_val_y)
 print(results)
 print(confusion_matrix(val_y,pred_val_y>results['threshold']))
from keras import applications<choose_model_class>
clr = CyclicLR(base_lr=0.001, max_lr=0.003,step_size=300., mode='exp_range', gamma=0.99994 )
model_resnet = applications.inception_resnet_v2.InceptionResNetV2(
 include_top=False,
weights='imagenet',
input_tensor=None,
input_shape=(139,139,3),
pooling='avg',
classes=None )<categorify>
pred_val_y, pred_test_y = train_pred(model_lstm_max(embedding_matrix), epochs = 4)
 outputs.append([pred_val_y, pred_test_y, 'LSTM w/ atten'])
 results = threshold_search(val_y, pred_val_y)
 print(results)
 print(confusion_matrix(val_y,pred_val_y>results['threshold']))
def make_trainable_false(model_resnet, trainable=False):
layers = model_resnet.layers
for ilayer in layers:
ilayer.trainable = trainable
return<load_pretrained>
coefs = [0.35,0.25,0.2,0.2]
pred_val_y = np.sum([outputs[i][0]*coefs[i] for i in range(len(outputs)) ], axis = 0)
 results = threshold_search(val_y, pred_val_y)
 print(results)
 print(confusion_matrix(val_y,pred_val_y>results['threshold']))
<choose_model_class><EOS> 
coefs = [0.35,0.25,0.2,0.2]
pred_test_y = np.sum([outputs[i][1]*coefs[i] for i in range(len(coefs)) ], axis = 0)
 pred_test_y =(pred_test_y > results['threshold'] ).astype(int)
 test_df = pd.read_csv(".. /input/test.csv", usecols=["qid"])
 out_df = pd.DataFrame({"qid":test_df["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False )