Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<install_modules>
with tf.Session() as sess:
devices = sess.list_devices()
devices
%%capture
!pip install -U scikit-learn<install_modules>
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
def load_and_prec(maxlen=300):
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape)
 train_df, val_df = train_test_split(train_df, test_size=0.2, random_state=2018)
 print('Creating train, val, and test sets')
 train_X = train_df["question_text"].values
val_X = val_df["question_text"].values
test_X = test_df["question_text"].values
print('Tokenizing the sentences')
 tokenizer = Tokenizer()
tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 print('Padding Sequences for input datasets')
 train_X = pad_sequences(train_X, maxlen=maxlen, padding='pre')
 val_X = pad_sequences(val_X, maxlen=maxlen, padding='pre')
 test_X = pad_sequences(test_X, maxlen=maxlen, padding='pre')
 train_y = train_df['target'].values
val_y = val_df['target'].values
print('Shuffling training and validation sets')
 np.random.seed(2018)
 trn_idx = np.random.permutation(len(train_X))
val_idx = np.random.permutation(len(val_X))
train_X = train_X[trn_idx]
val_X = val_X[val_idx]
train_y = train_y[trn_idx]
val_y = val_y[val_idx]
print('Complete - Returning Values')
 return train_X, val_X, test_X, train_y, val_y, tokenizer.word_index
def load_all_embeddings(vocabulary,max_features,embed_size):
def load_embedding(embed_file):
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 if 'glove' in embed_file:
embed_file = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
embeddings_index = dict(get_coefs(*o.split(" ")) for o\
in open(embed_file))
if 'wiki' in embed_file:
embed_file = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
embeddings_index = dict(get_coefs(*o.split(" ")) for o\
in open(embed_file)if len(o)>100)
 if 'paragram' in embed_file:
embed_file = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
embeddings_index = dict(get_coefs(*o.split(" ")) for o\
in open(embed_file, encoding="utf8", errors='ignore')\
if len(o)>100)
 if 'Google' in embed_file:
embed_file = '.. /input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin'
embeddings_index = KeyedVectors.load_word2vec_format(embed_file, binary=True)
 nb_words = min(max_features, len(vocabulary))
embedding_matrix =(np.random.rand(nb_words, embed_size)- 0.5)/ 5.0
for word, i in vocabulary.items() :
if i >= max_features: continue
if word in embeddings_index:
embedding_vector = embeddings_index.get_vector(word)
 embedding_matrix[i] = embedding_vector
del embeddings_index; gc.collect()
return embedding_matrix
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
nb_words = min(max_features, len(vocabulary))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in vocabulary.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
del embeddings_index; gc.collect()
return embedding_matrix
embed_list = os.listdir('.. /input/embeddings/')
 embedding_matrices = []
for embed in embed_list:
embedding_matrices.append(load_embedding(embed))
print('Loaded %s' % embed)
 average_embedding_matrix = np.mean(embedding_matrices,axis=0)
 print(np.shape(average_embedding_matrix))
return average_embedding_matrix
%%capture
!pip install ktrain<import_modules>
def joined_model(embedding_matrix, maxlen, max_features,embed_size):
inp = Input(shape=(maxlen,))
emb = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(emb)
 x = Dropout(0.2 )(x)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = Dropout(0.2 )(x)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = Dense(128, activation="relu" )(x)
 y = Bidirectional(CuDNNLSTM(128, return_sequences=True))(emb)
 y = Bidirectional(CuDNNLSTM(128, return_sequences=True))(y)
 y = Attention(maxlen )(y)
 z = Bidirectional(CuDNNLSTM(128, return_sequences=True))(emb)
 z = Bidirectional(CuDNNLSTM(128, return_sequences=True))(z)
 z = GlobalMaxPool1D()(z)
 z = Dense(256, activation="sigmoid" )(z)
 w = Bidirectional(CuDNNGRU(64, return_sequences=True))(emb)
 avg_pool = GlobalAveragePooling1D()(w)
 max_pool = GlobalMaxPooling1D()(w)
 w_conc = concatenate([avg_pool, max_pool])
 w_conc = Dense(64, activation="relu" )(w_conc)
 w_conc = Dropout(0.1 )(w_conc)
 output = concatenate([x,y,z,w_conc])
 output = Dense(1, activation="sigmoid" )(output)
 model = Model(inputs=inp, outputs=output)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model
def train_pred(model, train_X, train_y, val_X, val_y, test_X, epochs=2):
file_path = "model-checkpoint.hdf5"
checkpoint = ModelCheckpoint(file_path, monitor='val_loss', verbose=1,save_best_only=True, mode='min')
 early_stop = EarlyStopping(monitor="val_loss", mode="min", patience=1)
 model.fit(
 train_X,
train_y,
batch_size=1024,
epochs=epochs,
validation_data=(val_X, val_y),
callbacks=[checkpoint,early_stop] 
)
 model.load_weights(file_path)
 pred_val_y = model.predict([val_X], batch_size=256, verbose=0)
 best_thresh = 0.5
best_score = 0.0
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 score = metrics.f1_score(val_y,(pred_val_y > thresh ).astype(int))
if score > best_score:
best_thresh = thresh
best_score = score
print("Val F1 Score: {:.4f}".format(best_score))
pred_test_y = model.predict([test_X], batch_size=1024, verbose=0)
 return pred_val_y, pred_test_y, best_score
import numpy as np
import pandas as pd
import tensorflow as tf
import ktrain
import plotnine
import seaborn as sns
from ktrain import text
from transformers import *
from plotnine import *
from plotnine.options import figure_size
from sklearn.model_selection import StratifiedShuffleSplit
from sklearn.metrics import roc_curve, auc, accuracy_score, f1_score
from pylab import rcParams<load_from_csv>
embed_size = 300
max_features = 75000
maxlen = 65
train_X, val_X, test_X, train_y, val_y, vocabulary =\
load_and_prec(maxlen)
 embed_matrix = load_all_embeddings(vocabulary,max_features,embed_size )
input_dir = '/kaggle/input/nlp-getting-started/'
checkpoint_dir = '/kaggle/input/checkpoint-nlp-disaster/'
output_dir = '/kaggle/working/'
df_fit = pd.read_csv(input_dir + 'train.csv')
 df_sub = pd.read_csv(input_dir + 'test.csv')
 df_fit.head()<split>
model = joined_model(embed_matrix, maxlen, max_features, embed_size)
 pred_val_y, pred_test_y, best_score =\
train_pred(model, train_X, train_y, val_X, val_y, test_X, epochs = 8 )
sss = StratifiedShuffleSplit(n_splits=1, random_state=42)
 train_index, valid_index = next(sss.split(df_fit, df_fit['target']))
df_trn = df_fit.iloc[train_index, :]
df_tst = df_fit.iloc[valid_index, :]<define_variables>
thresholds = []
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 res = metrics.f1_score(val_y,(pred_val_y > thresh ).astype(int))
thresholds.append([thresh, res])
 print("F1 score at threshold {0} is {1}".format(thresh, res))
thresholds.sort(key=lambda x: x[1], reverse=True)
 best_thresh = thresholds[0][0]
print("Best threshold: ", best_thresh )
benchmark_dict = dict()
model_list = []
if len(model_list)> 0:
nrow_trn, nrow_tst, nrow_sub = df_trn.shape[0], df_tst.shape[0], df_sub.shape[0]
dict_benchmark = {'Model': [], 'Type': [], 'Id': [], 'Prediction': [], 'Target': []}
dict_submission = {'Model': [], 'Id': [], 'Proba': []}
for curr_model in model_list:
curr_benchmark = dict()
t = text.Transformer(curr_model, maxlen=140, classes=[0, 1])
 trn = t.preprocess_train(df_trn['text'].tolist() , df_trn['target'].tolist())
 val = t.preprocess_train(df_tst['text'].tolist() , df_tst['target'].tolist())
 model = t.get_classifier()
learner = ktrain.get_learner(model, train_data=trn, val_data=val, batch_size=16)
 learner.fit_onecycle(2e-6, 3)
 predictor = ktrain.get_predictor(model, t)
 curr_predict_trn = predictor.predict_proba(df_trn['text'].tolist())[:, 1].tolist()
curr_predict_tst = predictor.predict_proba(df_tst['text'].tolist())[:, 1].tolist()
curr_predict_sub = predictor.predict_proba(df_sub['text'].tolist())[:, 1].tolist()
dict_benchmark['Model'] +=(nrow_trn + nrow_tst)* [curr_model]
dict_benchmark['Type'] +=(nrow_trn * ['Train'])+(nrow_tst * ['Test'])
 dict_benchmark['Id'] += df_trn['id'].tolist() + df_tst['id'].tolist()
dict_benchmark['Prediction'] += curr_predict_trn + curr_predict_tst
dict_benchmark['Target'] += df_trn['target'].tolist() + df_tst['target'].tolist()
dict_submission['Model'] += nrow_sub * [curr_model]
dict_submission['Id'] += df_sub['id'].tolist()
dict_submission['Proba'] += curr_predict_sub
df_benchmark, df_submission = pd.DataFrame(dict_benchmark), pd.DataFrame(dict_submission)
 df_benchmark.to_csv(output_dir + 'benchmark.csv'), df_submission.to_csv('submission.csv')
 else:
benchmark_list, submission_list = [], []
for model_name in ['albert', 'bert', 'roberta', 'xlnet']:
benchmark_list.append(pd.read_csv(checkpoint_dir + 'benchmark_' + model_name + '.csv'))
submission_list.append(pd.read_csv(checkpoint_dir + 'submission_' + model_name + '.csv'))
df_benchmark = pd.concat(benchmark_list, ignore_index=True)
 df_submission = pd.concat(submission_list, ignore_index=True )<find_best_params>
pred_test_y =(pred_test_y > best_thresh ).astype(int)
 test_df = pd.read_csv(".. /input/test.csv", usecols=["qid"])
 out_df = pd.DataFrame({"qid":test_df["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False )