Kaggle competition: Quora Insincere Questions Classification
lengths = torch.from_numpy(np.array([len(x)for x in x_train]))
maxlen = 300
x_train_padded = torch.from_numpy(sequence.pad_sequences(x_train, maxlen=maxlen))
x_train_padded.shape<categorify>
dim = 300
num_words = 50000
max_len = 100
print('Fiting tokenizer')
 tokenizer = Tokenizer(num_words=num_words)
 tokenizer.fit_on_texts(df['question_text'])
 print('text to sequence')
 x_train = tokenizer.texts_to_sequences(df['question_text'])
 print('pad sequence')
 x_train = pad_sequences(x_train,maxlen=max_len)
 y_train = df['target'].values
print(x_train.shape)
 print(y_train.shape )
class SequenceBucketCollator() :
def __init__(self, choose_length, sequence_index, length_index, label_index=None):
self.choose_length = choose_length
self.sequence_index = sequence_index
self.length_index = length_index
self.label_index = label_index
def __call__(self, batch):
batch = [torch.stack(x)for x in list(zip(*batch)) ]
sequences = batch[self.sequence_index]
lengths = batch[self.length_index]
length = self.choose_length(lengths)
 mask = torch.arange(start=maxlen, end=0, step=-1)< length
padded_sequences = sequences[:, mask]
batch[self.sequence_index] = padded_sequences
if self.label_index is not None:
return [x for i, x in enumerate(batch)if i != self.label_index], batch[self.label_index]
return batch<choose_model_class>
print('Glove...')
 def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open('.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
print(len(all_embs))
word_index = tokenizer.word_index
embedding_matrix_glov = np.random.normal(emb_mean, emb_std,(num_words, dim))
count=0
for word, i in word_index.items() :
if i >= num_words:
break
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_glov[i] = embedding_vector
else:
count += 1
print('embedding matrix size:',embedding_matrix_glov.shape)
 print('Number of words not in vocab:',count)
 del embeddings_index,all_embs
gc.collect()
class NeuralNet(nn.Module):
def __init__(self, embedding_matrix, num_aux_targets):
super(NeuralNet, self ).__init__()
embed_size = embedding_matrix.shape[1]
self.embedding = nn.Embedding(max_features, embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = SpatialDropout(0.3)
 self.lstm1 = nn.LSTM(embed_size, LSTM_UNITS, bidirectional=True, batch_first=True)
 self.lstm2 = nn.LSTM(LSTM_UNITS * 2, LSTM_UNITS, bidirectional=True, batch_first=True)
 self.linear1 = nn.Linear(DENSE_HIDDEN_UNITS, DENSE_HIDDEN_UNITS)
 self.linear2 = nn.Linear(DENSE_HIDDEN_UNITS, DENSE_HIDDEN_UNITS)
 self.linear_out = nn.Linear(DENSE_HIDDEN_UNITS, 1)
 self.linear_aux_out = nn.Linear(DENSE_HIDDEN_UNITS, num_aux_targets)
 def forward(self, x, lengths=None):
h_embedding = self.embedding(x.long())
 h_embedding = self.embedding_dropout(h_embedding)
 h_lstm1, _ = self.lstm1(h_embedding)
 h_lstm2, _ = self.lstm2(h_lstm1)
 avg_pool = torch.mean(h_lstm2, 1)
 max_pool, _ = torch.max(h_lstm2, 1)
 h_conc = torch.cat(( max_pool, avg_pool), 1)
 h_conc_linear1 = F.relu(self.linear1(h_conc))
h_conc_linear2 = F.relu(self.linear2(h_conc))
hidden = h_conc + h_conc_linear1 + h_conc_linear2
result = self.linear_out(hidden)
 aux_result = self.linear_aux_out(hidden)
 out = torch.cat([result, aux_result], 1)
 return out<categorify>
print('Para...')
 EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
print(len(all_embs))
word_index = tokenizer.word_index
embedding_matrix_para = np.random.normal(emb_mean, emb_std,(num_words, dim))
count=0
for word, i in word_index.items() :
if i >= num_words:
break
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_para[i] = embedding_vector
else:
count += 1
print('embedding matrix size:',embedding_matrix_glov.shape)
 print('Number of words not in vocab:',count)
 del embeddings_index,all_embs
gc.collect()
test_lengths = torch.from_numpy(np.array([len(x)for x in x_test]))
x_test_padded = torch.from_numpy(sequence.pad_sequences(x_test, maxlen=maxlen))<create_dataframe>
matrixes = [embedding_matrix_glov,embedding_matrix_para]
matrix = np.mean(matrixes,axis=0)
 del embedding_matrix_glov,embedding_matrix_para
gc.collect()
batch_size = 512
test_dataset = data.TensorDataset(x_test_padded, test_lengths)
 train_dataset = data.TensorDataset(x_train_padded, lengths, y_train_torch)
 valid_dataset = data.Subset(train_dataset, indices=[0, 1])
 train_collator = SequenceBucketCollator(lambda lenghts: lenghts.max() ,
sequence_index=0,
length_index=1,
label_index=2)
 test_collator = SequenceBucketCollator(lambda lenghts: lenghts.max() , sequence_index=0, length_index=1)
 train_loader = data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True, collate_fn=train_collator)
 valid_loader = data.DataLoader(valid_dataset, batch_size=batch_size, shuffle=False, collate_fn=train_collator)
 test_loader = data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False, collate_fn=test_collator)
 databunch = DataBunch(train_dl=train_loader, valid_dl=valid_loader, collate_fn=train_collator )<compute_test_metric>
print('Loading test data...')
 df_final = pd.read_csv('.. /input/test.csv')
 df_final["question_text"].fillna("_
x_test=tokenizer.texts_to_sequences(df_final['question_text'])
 x_test = pad_sequences(x_test,maxlen=max_len)
 print('Test data loaded:',x_test.shape )
def custom_loss(data, targets):

 bce_loss_1 = nn.BCEWithLogitsLoss(weight=targets[:,1:2] )(data[:,:1],targets[:,:1])
 bce_loss_2 = nn.BCEWithLogitsLoss()(data[:,1:],targets[:,2:])
 return(bce_loss_1 * loss_weight)+ bce_loss_2<choose_model_class>
class CyclicLR(Callback):

 def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',
gamma=1., scale_fn=None, scale_mode='cycle'):
super(CyclicLR, self ).__init__()
self.base_lr = base_lr
self.max_lr = max_lr
self.step_size = step_size
self.mode = mode
self.gamma = gamma
if scale_fn == None:
if self.mode == 'triangular':
self.scale_fn = lambda x: 1.
 self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = lambda x: 1/(2.**(x-1))
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = lambda x: gamma**(x)
 self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.clr_iterations = 0.
 self.trn_iterations = 0.
 self.history = {}
self._reset()
def _reset(self, new_base_lr=None, new_max_lr=None,
new_step_size=None):

 if new_base_lr != None:
self.base_lr = new_base_lr
if new_max_lr != None:
self.max_lr = new_max_lr
if new_step_size != None:
self.step_size = new_step_size
self.clr_iterations = 0.
 def clr(self):
cycle = np.floor(1+self.clr_iterations/(2*self.step_size))
x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)
 if self.scale_mode == 'cycle':
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(cycle)
 else:
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(self.clr_iterations)
 def on_train_begin(self, logs={}):
logs = logs or {}
if self.clr_iterations == 0:
K.set_value(self.model.optimizer.lr, self.base_lr)
 else:
K.set_value(self.model.optimizer.lr, self.clr())
 def on_batch_end(self, epoch, logs=None):
logs = logs or {}
self.trn_iterations += 1
self.clr_iterations += 1
self.history.setdefault('lr', [] ).append(K.get_value(self.model.optimizer.lr))
self.history.setdefault('iterations', [] ).append(self.trn_iterations)
 for k, v in logs.items() :
self.history.setdefault(k, [] ).append(v)
 K.set_value(self.model.optimizer.lr, self.clr())
 class Metrics(Callback):
def on_train_begin(self, logs={}):
self.val_f1s = []
def _threshold_search(self,y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in [i * 0.01 for i in range(100)]:
score = f1_score(y_true=y_true, y_pred=y_proba > threshold)
 if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'f1': best_score}
return search_result
def on_epoch_end(self, epoch, logs={}):
y_val = self.validation_data[1]
y_pred = self.model.predict(self.validation_data[0])
 with warnings.catch_warnings() :
warnings.simplefilter("ignore")
 search_result = self._threshold_search(y_val, y_pred)
 _val_f1 = search_result['f1']
self.val_f1s.append(_val_f1)
 print('— val_f1_on_epoch_end: %f '%(_val_f1))
return
def get_model(adam, trainable=False):
inp1 = Input(shape=(max_len,))
emb = Embedding(num_words, dim, weights=[matrix],trainable = trainable, )(inp1)
 filters = []
for f in [3,4,5]:
conv = Conv1D(64,f, activation='elu' )(emb)
 pool = GlobalMaxPooling1D()(conv)
 filters.append(pool)
 x = concatenate(filters)
 x = Dense(max_len, activation="relu" )(x)
 x = Dropout(0.5 )(x)
 x = Dense(1, activation='sigmoid',kernel_regularizer=regularizers.l2(0.0001))(x)
 model = Model(inputs=inp1, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=adam, metrics=['accuracy'])
 return model
def get_opt() :
adam = optimizers.Adam()
print('LR:',K.eval(adam.lr))
return adam
def get_batch_size(opt):
batch_size = int(x_train.shape[0]*K.eval(opt.lr))
print('Batch size = ',batch_size)
 return batch_size
patience = 2
best_model=None
all_results = {}
splits = list(StratifiedKFold(n_splits=5, shuffle=True, random_state=14 ).split(x_train, y_train))
for idx,(train_idx, valid_idx)in enumerate(splits):
print('----'+str(idx)+'-----')
 X_train1 = x_train[train_idx]
y_train1 = y_train[train_idx]
X_val = x_train[valid_idx]
y_val = y_train[valid_idx]
model_file = 'model_'+str(idx)+'.h5'
modelcheck = ModelCheckpoint(model_file,save_best_only=True)
 stop = EarlyStopping(patience=patience)
 reduce = CyclicLR(base_lr=0.001,
max_lr=0.004,
step_size=300.,
mode='exp_range',
gamma=0.99994)
 metrics = Metrics()
adam = get_opt()
batch_size = get_batch_size(adam)
 model = get_model(adam)
 history = model.fit(X_train1,y_train1,
batch_size=batch_size,
validation_data=(X_val,y_val),
epochs=100,
callbacks=[modelcheck,stop,reduce,metrics],
verbose=2)
 print('Pretraining finished, unfreezing embeddings layer...')
 model = get_model(adam,True)
 model.load_weights(model_file)
 modelcheck = ModelCheckpoint(model_file,save_best_only=True)
 stop = EarlyStopping(patience=patience)
 reduce = CyclicLR(base_lr=0.001,
max_lr=0.004,
step_size=300.,
mode='exp_range',
gamma=0.99994)
 metrics = Metrics()
history = model.fit(X_train1,y_train1,
batch_size=batch_size,
validation_data=(X_val,y_val),
epochs=100,
callbacks=[modelcheck,stop,reduce,metrics],
verbose=2)
 print('training finished...')
 model.load_weights(model_file)
 y_pred = model.predict(X_val,batch_size=batch_size, verbose=1)
 search_result = threshold_search(y_val, y_pred)
 print(search_result)
 y_pred = y_pred>search_result['threshold']
y_pred = y_pred.astype(int)
 print('RESULTS ON VALIDATION SET:
',classification_report(y_val,y_pred))
all_results[model_file] = search_result['f1']
if best_model is None or best_model['f1'] < search_result['f1']:
best_model={'model':model_file,'f1':search_result['f1']}
print('-'*80)
 print(all_results)
 print('-'*80)
 print(best_model)
 print('-'*80 )
all_test_preds = []
for model_idx in range(NUM_MODELS):
print('Model ', model_idx)
 seed_everything(1 + model_idx)
 model = NeuralNet(embedding_matrix, y_aux_train.shape[-1])
 learn = Learner(databunch, model, loss_func=custom_loss)
 test_preds = train_model(learn,test_dataset,output_dim=7)
 all_test_preds.append(test_preds )<save_to_csv>
print('using models to generate meta...')
 train_meta = []
test_meta = []
for model_file in all_results.keys() :
model = get_model(adam,True)
 model.load_weights(model_file)
 train_meta.append(model.predict(x_train,batch_size=batch_size, verbose=1))
test_meta.append(model.predict(x_test,batch_size=batch_size, verbose=1))
train_meta = np.mean(train_meta,axis=0)
 test_meta = np.mean(test_meta,axis=0)
 print(y_train.shape)
 print(train_meta.shape)
 print(test_meta.shape)
submission = pd.DataFrame.from_dict({
'id': test['id'],
'prediction': np.mean(all_test_preds, axis=0)[:, 0]
})
 submission.to_csv('submission.csv', index=False )<set_options>
search_result = threshold_search(y_train, train_meta)
 print(search_result)
 test_meta = test_meta>search_result['threshold']
test_meta = test_meta.astype(int)
 df_subm = pd.DataFrame()
df_subm['qid'] = df_final.qid
df_subm['prediction'] = test_meta
print(df_subm.head())
 df_subm.to_csv('submission.csv', index=False )