Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<define_variables>
import os
import time
import numpy as np
import pandas as pd
from tqdm import tqdm
import math
from sklearn.model_selection import train_test_split
from sklearn import metrics
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Input, CuDNNLSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D
from keras.layers import Bidirectional, GlobalMaxPool1D
from keras.optimizers import Adam
from keras.models import Model
from keras import backend as K
from keras.engine.topology import Layer
from keras import initializers, regularizers, constraints, optimizers, layers
%env JOBLIB_TEMP_FOLDER=/tmp
plt.style.use('seaborn')
 sns.set_style('whitegrid')
 %matplotlib inline
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename)) 
<load_from_csv>
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape )
a=pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 test=pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')
 submission= pd.read_csv('/kaggle/input/nlp-getting-started/sample_submission.csv')
 train_data=a.text.values
train_labels=a.target.values
test_data=test.text.values
a.head() 
<data_type_conversions>
train_df.isna().sum()
NAcols=a.columns
for col in NAcols:
if a[col].dtype == "object":
a[col] = a[col].fillna("None" )<filter>
test_df.isna().sum()
no_dis=a[a['target']==0]
yes_dis=a[a['target']==1]<string_transform>
train_df, val_df = train_test_split(train_df, test_size=0.2, random_state=2018)
 embed_size = 300
max_features = 50000
maxlen = 120
train_X = train_df["question_text"].fillna("_na_" ).values
val_X = val_df["question_text"].fillna("_na_" ).values
test_X = test_df["question_text"].fillna("_na_" ).values
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 val_X = pad_sequences(val_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
val_y = val_df['target'].values
def word_counter(data):
df=data['text'].str.lower()
swords=[ "i'm",'the', 'a', 'to', 'in', 'of', 'and', 'I', 'you', 'he', 'she', 'it', 'is', 'on','-','The',
'my', 'with', 'that', 'at','by','it','from', 'be', 'was', 'have', 'are', 'this', 'like',
'A', 'as', 'just', 'your', 'up', 'but', 'me', 'so', 'not', 'has', 'out', '??', 'will',
'via','after', 'an', 'about', 'been', 'get', 'or', 'when', 'all', 'no', 'into', 'over', 'In',
'who', 'we', 'if', 'I', 'can', 'The', 'how', 'them', 'But', 'So', 'Too', 'too', 'did', 'much',
'for', 'his', 'her', 'To', "it's", 'You', 'there', 'If', 'what', 'i', '2', 'they', 'had', 'their',
'one', 'got', 'us', 'man', 'our', 'two', '&amp;', 'were', 'than']
b= collections.Counter([y for x in df.values.flatten() for y in x.split() if y not in swords])
 ca= b.most_common(20)
 c=pd.DataFrame(ca)
 c= c.rename(columns={0: "word", 1: "n_word"})
 fig, ax = plt.subplots(figsize=(6,6))
title_cnt=c.n_word.sort_values(ascending=False ).reset_index()
n= ax.barh(title_cnt.iloc[:,0], title_cnt.iloc[:,1], color='lightgreen', edgecolor='green', height=0.7)
 for s in ['top','bottom','left','right']:
ax.spines[s].set_visible(False)
 ax.xaxis.set_ticks_position('none')
 ax.yaxis.set_ticks_position('none')
 ax.xaxis.set_tick_params(pad=5)
 ax.yaxis.set_tick_params(pad=10)
 ax.grid(b=True, color='grey', linestyle='-.', linewidth=1, alpha=0.3)
 ax.invert_yaxis()
ax.set_title('Disaster tweets: MOST FREQUENT WORDS',
loc='center', pad=10, fontsize=16)
 ax.set_xlabel('
for i in ax.patches:
ax.text(i.get_width() +5, i.get_y() +0.5, str(round(( i.get_width()), 2)) ,
fontsize=10, fontweight='bold', color='grey')
 r=range(20)
 plt.yticks(r, c.word ,weight='bold')
 return plt.show() 
<count_values>
np.random.seed(2018)
 trn_idx = np.random.permutation(len(train_X))
val_idx = np.random.permutation(len(val_X))
train_X = train_X[trn_idx]
val_X = val_X[val_idx]
train_y = train_y[trn_idx]
val_y = val_y[val_idx]
word_counter(no_dis )<count_values>
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * 2*a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
word_counter(yes_dis )<choose_model_class>
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = Dense(64, activation="relu" )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-3), metrics=['accuracy'] )
hub_layer = hub.KerasLayer("https://tfhub.dev/google/Wiki-words-500-with-normalization/2",
input_shape=[], dtype=tf.string)
 model = tf.keras.Sequential()
model.add(hub_layer)
 model.add(tf.keras.layers.Dense(128, activation='relu'))
model.add(tf.keras.layers.Dropout(0.4))
model.add(tf.keras.layers.BatchNormalization())
 model.add(tf.keras.layers.Dense(32, activation='relu'))
model.add(tf.keras.layers.Dropout(0.4))
model.add(tf.keras.layers.BatchNormalization())
 model.add(tf.keras.layers.Dense(1, activation='sigmoid'))
model.summary()<choose_model_class>
model.fit(train_X, train_y, batch_size=512, epochs=3, validation_data=(val_X, val_y))
model.compile(Adam(lr=0.0002), loss='binary_crossentropy', metrics=['accuracy'] )<train_model>
pred_glove_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_glove_val_y>thresh ).astype(int))))
history = model.fit(
 train_data, train_labels,
validation_split=0.3,
epochs=22,
callbacks=callbacks_list,
batch_size=32
)<save_to_csv>
pred_glove_test_y = model.predict([test_X], batch_size=1024, verbose=1 )
model.load_weights('my_model_file.h5')
 prediction = model.predict(test_data)
 submission['target'] = prediction.round().astype(int)
 submission.to_csv('submission.csv', index=False)
<set_options>
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )
%reload_ext autoreload
%autoreload 2
%matplotlib inline 
<set_options>
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = Dense(64, activation="relu" )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-3), metrics=['accuracy'] )
print(torch.cuda.is_available() , torch.backends.cudnn.enabled )<load_from_csv>
model.fit(train_X, train_y, batch_size=512, epochs=3, validation_data=(val_X, val_y))
train_df = pd.read_csv(INPUT/"train.csv")
 train_df.head(3 )<load_from_csv>
pred_fasttext_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_fasttext_val_y>thresh ).astype(int))))
test_df = pd.read_csv(INPUT/"test.csv")
 test_df.head(3 )<define_variables>
pred_fasttext_test_y = model.predict([test_X], batch_size=1024, verbose=1 )
TRAIN = Path(".. /train")
 TEST = Path(".. /test" )<sort_values>
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )
sorted(os.listdir(TRAIN))<train_model>
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = Dense(64, activation="relu" )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-3), metrics=['accuracy'] )
if os.path.isdir(TRAIN):
print('Train directory has been created')
 else:
print('Train directory creation failed.')
 if os.path.isdir(TEST):
print('Test directory has been created')
 else:
print('Test directory creation failed.' )<import_modules>
model.fit(train_X, train_y, batch_size=512, epochs=3, validation_data=(val_X, val_y))
from PIL import Image<data_type_conversions>
pred_paragram_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_paragram_val_y>thresh ).astype(int))))
def pix2img(pix_data, filepath):
img_mat = pix_data.reshape(28,28)
 img_mat = img_mat.astype(np.uint8())
 img_dat = Image.fromarray(img_mat)
 img_dat.save(filepath)
<feature_engineering>
pred_paragram_test_y = model.predict([test_X], batch_size=1024, verbose=1 )
tfms = get_transforms(do_flip = False)
<train_model>
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )
print('test : ',TEST)
 print('train: ', TRAIN)
 print(type(TEST))<define_variables>
pred_val_y = 0.39*pred_glove_val_y + 0.41*pred_fasttext_val_y + 0.39*pred_paragram_val_y
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_val_y>thresh ).astype(int))))
data = ImageDataBunch.from_folder(
 path =(".. /train"),
test =(".. /test"),
valid_pct = 0.1,
bs = 256,
size = 28,
num_workers = 0,
ds_tfms = tfms
)<normalization>
pred_test_y = 0.35*pred_glove_test_y + 0.42*pred_fasttext_test_y + 0.35*pred_paragram_test_y
pred_test_y =(pred_test_y>0.42 ).astype(int)
 out_df = pd.DataFrame({"qid":test_df["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False )