Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<train_model>
import logging
from nltk import word_tokenize
import pandas as pd
import numpy as np
import gc
import os
import torch
from torch import nn
from torch.autograd import Variable
import torch.nn.functional as F
from torch.nn.utils.rnn import pad_sequence
from sklearn.metrics import f1_score
from torch import optim
import torchtext
import random
%%time
checkpoint_predictions = []
weights = []
for model_idx in range(NUM_MODELS):
seed_everything(SEED + model_idx)
 model = build_model(embedding_matrix, y_aux_train.shape[-1])
 for global_epoch in range(EPOCHS):
print('model {} epoch {}'.format(model_idx + 1, global_epoch + 1))
model.fit(
 x_train,
[y_train, y_aux_train],
batch_size=BATCH_SIZE,
epochs=1,
verbose=2,
sample_weight=[sample_weights.values, np.ones_like(sample_weights)],
callbacks=[
LearningRateScheduler(lambda _: 1e-3 *(0.55 ** global_epoch))
] 
)
 checkpoint_predictions.append(model.predict(x_test, batch_size=2048)[0].flatten())
 weights.append(2 ** global_epoch)
 predictions = np.average(checkpoint_predictions, weights=weights, axis=0)
<prepare_output>
text = torchtext.data.Field(lower=True, batch_first=True, tokenize=word_tokenize)
 qid = torchtext.data.Field()
target = torchtext.data.Field(sequential=False, use_vocab=False, is_target=True)
 train = torchtext.data.TabularDataset(path='.. /input/train.csv', format='csv',
fields={'question_text':('text',text),
'target':('target',target)})
 test = torchtext.data.TabularDataset(path='.. /input/test.csv', format='csv',
fields={'qid':('qid', qid),
'question_text':('text', text)} )
submission = pd.DataFrame.from_dict({
'id': test_df.id,
'prediction': predictions
})
 submission.head()<save_to_csv>
text.build_vocab(train, test, min_freq=3)
 qid.build_vocab(test)
 text.vocab.load_vectors(torchtext.vocab.Vectors('.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'))
print(text.vocab.vectors.shape )
submission.to_csv('submission.csv', index=False)
 print(os.listdir("."))<import_modules>
random.seed(2018)
 train, val = train.split(split_ratio=0.9, random_state=random.getstate() )
from keras.models import Model
from keras.layers import Input, Dense, Embedding, SpatialDropout1D, Dropout, add, concatenate
from keras.layers import CuDNNLSTM, Bidirectional, GlobalMaxPooling1D, GlobalAveragePooling1D
from keras.preprocessing import text, sequence
from keras.callbacks import LearningRateScheduler
from keras.losses import binary_crossentropy
from keras import backend as K<define_variables>
class BiLSTM(nn.Module):
def __init__(self, pretrained_lm, padding_idx, static=True, hidden_dim=128, lstm_layer=2, dropout=0.2):
super(BiLSTM, self ).__init__()
self.hidden_dim = hidden_dim
self.dropout = nn.Dropout(p=dropout)
 self.embedding = nn.Embedding.from_pretrained(pretrained_lm)
 self.embedding.padding_idx = padding_idx
if static:
self.embedding.weight.requires_grad = False
self.lstm = nn.LSTM(input_size=self.embedding.embedding_dim,
hidden_size=hidden_dim,
num_layers=lstm_layer,
dropout = dropout,
bidirectional=True)
 self.hidden2label = nn.Linear(hidden_dim*lstm_layer*2, 1)
 def forward(self, sents):
x = self.embedding(sents)
 x = torch.transpose(x, dim0=1, dim1=0)
 lstm_out,(h_n, c_n)= self.lstm(x)
 y = self.hidden2label(self.dropout(torch.cat([c_n[i,:, :] for i in range(c_n.shape[0])], dim=1)))
 return y
EMBEDDING_FILES = [
'.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec',
'.. /input/glove840b300dtxt/glove.840B.300d.txt'
]<define_variables>
def training(epoch, model, eval_every, loss_func, optimizer, train_iter, val_iter, early_stop=1, warmup_epoch=2):
step = 0
max_loss = 1e5
no_improve_epoch = 0
no_improve_in_previous_epoch = False
fine_tuning = False
train_record = []
val_record = []
losses = []
for e in range(epoch):
if e >= warmup_epoch:
if no_improve_in_previous_epoch:
no_improve_epoch += 1
if no_improve_epoch >= early_stop:
break
else:
no_improve_epoch = 0
no_improve_in_previous_epoch = True
if not fine_tuning and e >= warmup_epoch:
model.embedding.weight.requires_grad = True
fine_tuning = True
train_iter.init_epoch()
for train_batch in iter(train_iter):
step += 1
model.train()
x = train_batch.text.cuda()
y = train_batch.target.type(torch.Tensor ).cuda()
model.zero_grad()
pred = model.forward(x ).view(-1)
 loss = loss_function(pred, y)
 losses.append(loss.cpu().data.numpy())
 train_record.append(loss.cpu().data.numpy())
 loss.backward()
optimizer.step()
if step % eval_every == 0:
model.eval()
model.zero_grad()
val_loss = []
for val_batch in iter(val_iter):
val_x = val_batch.text.cuda()
val_y = val_batch.target.type(torch.Tensor ).cuda()
val_pred = model.forward(val_x ).view(-1)
 val_loss.append(loss_function(val_pred, val_y ).cpu().data.numpy())
 val_record.append({'step': step, 'loss': np.mean(val_loss)})
 print('epcoh {:02} - step {:06} - train_loss {:.4f} - val_loss {:.4f} '.format(
 e, step, np.mean(losses), val_record[-1]['loss']))
if e >= warmup_epoch:
if val_record[-1]['loss'] <= max_loss:
save(m=model, info={'step': step, 'epoch': e, 'train_loss': np.mean(losses),
'val_loss': val_record[-1]['loss']})
 max_loss = val_record[-1]['loss']
no_improve_in_previous_epoch = False
def save(m, info):
torch.save(info, 'best_model.info')
 torch.save(m, 'best_model.m')
 def load() :
m = torch.load('best_model.m')
 info = torch.load('best_model.info')
 return m, info
NUM_MODELS = 2
BATCH_SIZE = 512
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
EPOCHS = 4
MAX_LEN = 220<compute_test_metric>
batch_size = 128
train_iter = torchtext.data.BucketIterator(dataset=train,
batch_size=batch_size,
sort_key=lambda x: x.text.__len__() ,
shuffle=True,
sort=False)
 val_iter = torchtext.data.BucketIterator(dataset=val,
batch_size=batch_size,
sort_key=lambda x: x.text.__len__() ,
train=False,
sort=False)
 model = BiLSTM(text.vocab.vectors, lstm_layer=2, padding_idx=text.vocab.stoi[text.pad_token], hidden_dim=128 ).cuda()
loss_function = nn.BCEWithLogitsLoss()
optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()),
lr=1e-3 )
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32' )<load_pretrained>
training(model=model, epoch=20, eval_every=500,
loss_func=loss_function, optimizer=optimizer, train_iter=train_iter,
val_iter=val_iter, warmup_epoch=3, early_stop=2 )
def load_embeddings(path):
with open(path)as f:
return dict(get_coefs(*line.strip().split(' ')) for line in f )<categorify>
model, m_info = load()
m_info
def build_matrix(word_index, path):
embedding_index = load_embeddings(path)
 embedding_matrix = np.zeros(( len(word_index)+ 1, 300))
for word, i in word_index.items() :
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
pass
return embedding_matrix<compute_test_metric>
model.lstm.flatten_parameters()
def custom_loss(y_true, y_pred):
return binary_crossentropy(K.reshape(y_true[:,0],(-1,1)) , y_pred)* y_true[:,1]<choose_model_class>
model.eval()
val_pred = []
val_true = []
val_iter.init_epoch()
for val_batch in iter(val_iter):
val_x = val_batch.text.cuda()
val_true += val_batch.target.data.numpy().tolist()
val_pred += torch.sigmoid(model.forward(val_x ).view(-1)).cpu().data.numpy().tolist()
def build_model(embedding_matrix, num_aux_targets, loss_weight):
words = Input(shape=(MAX_LEN,))
x = Embedding(*embedding_matrix.shape, weights=[embedding_matrix], trainable=False )(words)
 x = SpatialDropout1D(0.3 )(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 hidden = concatenate([
GlobalMaxPooling1D()(x),
GlobalAveragePooling1D()(x),
])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 result = Dense(1, activation='sigmoid' )(hidden)
 aux_result = Dense(num_aux_targets, activation='sigmoid' )(hidden)
 model = Model(inputs=words, outputs=[result, aux_result])
 model.compile(loss=[custom_loss,'binary_crossentropy'], loss_weights=[loss_weight, 1.0], optimizer='adam')
 return model<categorify>
tmp = [0,0,0]
delta = 0
for tmp[0] in np.arange(0.1, 0.501, 0.01):
tmp[1] = f1_score(val_true, np.array(val_pred)>tmp[0])
 if tmp[1] > tmp[2]:
delta = tmp[0]
tmp[2] = tmp[1]
print('best threshold is {:.4f} with F1 score: {:.4f}'.format(delta, tmp[2]))
def preprocess(data):

 punct = "/-'?!.,
def clean_special_chars(text, punct):
for p in punct:
text = text.replace(p, ' ')
 return text
data = data.astype(str ).apply(lambda x: clean_special_chars(x, punct))
return data<load_from_csv>
model.eval()
model.zero_grad()
test_iter = torchtext.data.BucketIterator(dataset=test,
batch_size=batch_size,
sort_key=lambda x: x.text.__len__() ,
sort=True)
 test_pred = []
test_id = []
for test_batch in iter(test_iter):
test_x = test_batch.text.cuda()
test_pred += torch.sigmoid(model.forward(test_x ).view(-1)).cpu().data.numpy().tolist()
test_id += test_batch.qid.view(-1 ).data.numpy().tolist()
train = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv')
 test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv' )<define_variables>
sub_df =pd.DataFrame()
sub_df['qid'] = [qid.vocab.itos[i] for i in test_id]
sub_df['prediction'] =(np.array(test_pred)>= delta ).astype(int )
x_train = preprocess(train['comment_text'])
 identity_columns = [
'male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish',
'muslim', 'black', 'white', 'psychiatric_or_mental_illness']<data_type_conversions>
sub_df.to_csv("submission.csv", index=False )