Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<choose_model_class>
import re
import os
import sys
import math
import string
import random
import datetime
import numpy as np
import pandas as pd
from time import time
from tqdm import tqdm, trange
from sklearn import metrics
import keras.layers
from keras import backend as K
from keras import initializers, regularizers, constraints, optimizers, layers
from keras.models import Sequential, Model
from keras.layers import CuDNNLSTM, CuDNNGRU, Dense, Bidirectional, Dropout, BatchNormalization, Activation, Input
from keras.optimizers import Adam
from keras.regularizers import l1_l2
from keras.engine.topology import Layer
from keras.preprocessing.sequence import pad_sequences
OUTPUT_DIM = 5
model = BERTGRUSentiment(bert, OUTPUT_DIM )<choose_model_class>
print(f'Starting at {datetime.datetime.now() }' )
<train_model>
df=pd.read_csv('.. /input/train.csv')
 df=df[['question_text','target']]
df=df.dropna()
max_word_len=60
def count_parameters(model):
return sum(p.numel() for p in model.parameters() if p.requires_grad)
 print(f'The model has {count_parameters(model):,} trainable parameters' )<train_model>
embedding_path='glove.840B.300d/glove.840B.300d.txt'
vocab={}
with open('.. /input/embeddings/' + embedding_path,'r')as f:
for line_number,line in enumerate(tqdm(f)) :
key,values=line.split(' ')[0],line.split(' ')[1:]
if not any(char in string.punctuation for char in key):
vocab[key]=np.asarray(values,dtype='float32')
 print(f'Total of {len(vocab.keys())} words in vocabulary' )
for name, param in model.named_parameters() :
if name.startswith('bert'):
param.requires_grad = False
print(f'The model has {count_parameters(model):,} trainable parameters' )<train_model>
trans_table={key:' ' for key in string.punctuation}
unknown_word=np.zeros(300 )
for name, param in model.named_parameters() :
if param.requires_grad:
print(name )<choose_model_class>
def str_prep(s):
s=s.translate(str.maketrans(trans_table)).lower().strip()
s=re.sub(' +',' ',s)
 s=s.split(' ')
 ar=np.asarray([vocab.get(x,unknown_word)for x in s],dtype='float32')
 return ar
optimizer = optim.Adam(model.parameters() )<choose_model_class>
def batch_gen(df,min_batch_size=1024,transformation_func=str_prep, training_mode=True):
n_batches=math.ceil(len(df)/min_batch_size)
 while True:
df=df.sample(frac=1 ).reset_index(drop=True)
 for batch in range(n_batches):
start=batch*min_batch_size
end=start+min_batch_size
if batch==n_batches-1:
X=df['question_text'][start:].apply(transformation_func)
 X=pad_sequences(X,maxlen=60,dtype='float32',padding='post')
 y=np.array(df['target'][start:])
 else:
X=df['question_text'][start:end].apply(transformation_func)
 X=pad_sequences(X,maxlen=60,dtype='float32',padding='post')
 y=np.array(df['target'][start:end])
 yield X, y
criterion = nn.CrossEntropyLoss()<categorify>
def train_val_split(df,frac=0.2):
df=df.sample(frac=1 ).reset_index(drop=True)
 df_sincere=df[df['target']==0]
df_taxic=df[df['target']==1]
sincere_border=int(len(df_sincere)*(1-frac))
taxic_border=int(len(df_taxic)*(1-frac))
df_train=pd.concat([df_sincere[:sincere_border],df_taxic[:taxic_border]])
 df_val=pd.concat([df_sincere[sincere_border:],df_taxic[taxic_border:]])
 print('Training data peppered(prepared)!')
 return df_train.sample(frac=1 ).reset_index(drop=True), df_val.sample(frac=1 ).reset_index(drop=True )
model = model.to(DEVICE)
 criterion = criterion.to(DEVICE )<compute_test_metric>
class Attention(Layer):
def __init__(self, step_dim, W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None, bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],), initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],), initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
def accuracy(prediction, label):

 prediction = torch.argmax(nn.functional.softmax(prediction, dim=1), dim=1)
 acc = torch.sum(prediction == label ).float() / len(prediction == label)
 return acc<train_model>
def train(model, iterator, optimizer, criterion):
epoch_loss = 0
epoch_acc = 0
model.train()
for batch in iterator:
optimizer.zero_grad()
data = batch.Phrase
label = batch.Sentiment
prediction = model(data)
 loss = criterion(prediction, label)
 acc = accuracy(prediction, label)
 loss.backward()
optimizer.step()
epoch_loss += loss.item()
epoch_acc += acc.item()
return epoch_loss / len(iterator), epoch_acc / len(iterator )<compute_train_metric>
def get_predictions(df, model, min_batch_size=1024,transformation_func=str_prep):
predictions=np.ndarray(shape=(0,1))
print(f'Total values to predict: {len(df)}')
 n_batches=math.ceil(len(df)/min_batch_size)
 for batch in trange(n_batches):
start=batch*min_batch_size
end=start+min_batch_size
if batch==n_batches-1:
X=df['question_text'][start:].apply(transformation_func)
 X=pad_sequences(X,maxlen=60,dtype='float32',padding='post')
 y_predicted=model.predict(X)
 else:
X=df['question_text'][start:end].apply(transformation_func)
 X=pad_sequences(X,maxlen=60,dtype='float32',padding='post')
 y_predicted=model.predict(X)
 predictions=np.append(predictions,y_predicted,axis=0)
 return predictions
def evaluate(model, iterator, criterion):
epoch_loss = 0
epoch_acc = 0
model.eval()
with torch.no_grad() :
for batch in iterator:
data = batch.Phrase
label = batch.Sentiment
prediction = model(data)
 loss = criterion(prediction, label)
 acc = accuracy(prediction, label)
 epoch_loss += loss.item()
epoch_acc += acc.item()
return epoch_loss / len(iterator), epoch_acc / len(iterator )<train_model>
def select_best_threshold(y_true, y_predicted):
thresholds = []
for thresh in np.arange(0.0, 1, 0.001):
thresh = np.round(thresh, 3)
 res=metrics.f1_score(y_true,(y_predicted>thresh ).astype(int))
thresholds.append([thresh, res])
 thresholds.sort(key=lambda x: x[1], reverse=True)
 best_thresh = thresholds[0][0]
print("Best threshold: ", best_thresh)
 return best_thresh
def epoch_time(start_time, end_time):
elapsed_time = end_time - start_time
elapsed_mins = int(elapsed_time / 60)
 elapsed_secs = int(elapsed_time -(elapsed_mins * 60))
return elapsed_mins, elapsed_secs<train_model>
stack_size=3
model_stack={}
t_start=time()
for ind in range(stack_size):
print(f'
=== Model number {ind+1} ===
')
 train_df, val_df=train_val_split(df,frac=0.2)
 model=train_model(train_df,val_df, n_epochs=5)
 y_predicted=get_predictions(val_df,model)
 y_true=val_df['target'].values
best_thresh=select_best_threshold(y_true, y_predicted)
 model_stack[ind]=(model, best_thresh)
 print(f'Model number {ind+1} finished!')
 t_finish=time()
print(f'Total time for training and calculating thresholds is {t_finish-t_start:.2f} seconds' )
N_EPOCHS = 10
best_epoch = 0
best_valid_loss = float('inf')
 for epoch in range(N_EPOCHS):
start_time = time.time()
train_loss, train_acc = train(model, train_iter, optimizer, criterion)
 valid_loss, valid_acc = evaluate(model, valid_iter, criterion)
 end_time = time.time()
epoch_mins, epoch_secs = epoch_time(start_time, end_time)
 if valid_loss < best_valid_loss:
best_epoch = epoch + 1
best_valid_loss = valid_loss
torch.save(model.state_dict() , 'model.pt')
 print(f'Epoch: {epoch+1:02} | Epoch Time: {epoch_mins}m {epoch_secs}s')
 print(f'\tTrain Loss: {train_loss:.3f} | Train Acc: {train_acc*100:.2f}%')
 print(f'\t Val.Loss: {valid_loss:.3f} | Val.Acc: {valid_acc*100:.2f}%' )<create_dataframe>
test_set=pd.read_csv('.. /input/test.csv' )
train_full = data.TabularDataset('./data/train.tsv', format='tsv', skip_header=True,
fields=[('PhraseId', None),('SentenceId', None), 
('Phrase', TEXT),('Sentiment', LABEL)])
 train_full_iter = data.BucketIterator(train_full, batch_size=BATCH_SIZE, shuffle=True, device=DEVICE)
 model = BERTGRUSentiment(bert, OUTPUT_DIM)
 optimizer = optim.Adam(model.parameters())
 criterion = nn.CrossEntropyLoss()
model = model.to(DEVICE)
 criterion = criterion.to(DEVICE)
 for epoch in range(best_epoch + 1):
start_time = time.time()
train_loss, train_acc = train(model, train_full_iter, optimizer, criterion)
 end_time = time.time()
epoch_mins, epoch_secs = epoch_time(start_time, end_time)
 if epoch == best_epoch:
torch.save(model.state_dict() , 'model_full.pt')
 print(f'Epoch: {epoch+1:02} | Epoch Time: {epoch_mins}m {epoch_secs}s')
 print(f'\tTrain Loss: {train_loss:.3f} | Train Acc: {train_acc*100:.2f}%' )<predict_on_test>
predictions={}
for key,(model, threshold)in model_stack.items() :
predictions[key]=(get_predictions(test_set,model)>threshold ).astype(int )
def predict(model, iterator):
model.eval()
predictions = []
with torch.no_grad() :
for batch in iterator:
data = batch.Phrase
prediction = model(data)
 prediction = torch.argmax(nn.functional.softmax(prediction, dim=1), dim=1)
 predictions.extend(prediction.tolist())
 return predictions<save_to_csv>
preds=np.stack([value for value in predictions.values() ],axis=1 ).reshape(-1,stack_size)
 preds=np.average(preds,axis=1)
 preds=np.around(preds,decimals=0 ).astype(int )
predictions = predict(model, test_iter)
 submission = pd.read_csv('.. /input/sentiment-analysis-on-movie-reviews/sampleSubmission.csv')
 submission['Sentiment'] = predictions
submission.to_csv('submissionBERTGRU.csv', index=False )<install_modules>
test_set['prediction']=preds
to_submit=test_set[['qid','prediction']]
!pip install -U sentence-transformers<feature_engineering>
to_submit.to_csv("submission.csv", index=False)
 print('Submissions saved to file!' )
PRETRAINED_MODEL='roberta-large-nli-stsb-mean-tokens'
model = SentenceTransformer(PRETRAINED_MODEL)
<import_modules>
print(f'Finishing at {datetime.datetime.now() }' )