Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<predict_on_test>
%matplotlib inline
lgb_pred = model_lgb.predict(X_test ).clip(0, 20)
<save_to_csv>
max_seq_len = 60
embed_size = 300
max_features = 50000
EMBEDDING = 'glove.840B.300d'
MODEL = 'attention'
embedding_matrix = 'None'
embeddings_idx = 'None'
checkpoint = ModelCheckpoint('./checkpoints/', monitor='val_acc', verbose=0, save_best_only=True)
 earlystop = EarlyStopping(monitor='val_acc', min_delta=0, patience=1, verbose=0)
 tensorboard = TensorBoard(log_dir='./logs', histogram_freq=0, batch_size=32, write_graph=True, write_grads=True, write_images=True)
 reducelr = ReduceLROnPlateau(monitor='val_acc', factor=0.1, patience=3, verbose=0, mode='auto', min_delta=0.0001, cooldown=0, min_lr=0)
 thresh = 0.4
result = pd.DataFrame({
"ID": df_test["ID"],
"item_cnt_month": lgb_pred.clip(0.,20.)
})
 result.to_csv("submission.csv", index=False )<import_modules>
train_set = pd.read_csv('.. /input/train.csv')
 test_set = pd.read_csv('.. /input/test.csv')
 train_set.head()
import gc, itertools
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.preprocessing import LabelEncoder
import xgboost as xgb
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error<set_options>
x = train_set['target'].value_counts(dropna=False)
 print(x)
 sincere_examples = x[0]
insincere_examples = x[1]
pd.options.display.max_rows = 10<load_from_csv>
print(len(lengths_without_puncs)- np.count_nonzero(lengths_without_puncs))
df_train = pd.read_csv('.. /input/sales_train.csv')
 df_test = pd.read_csv('.. /input/test.csv' )<load_from_csv>
contraction_mapping = {"ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because", "could've": "could have", "couldn't": "could not", "didn't": "did not", "doesn't": "does not", "don't": "do not", "hadn't": "had not", "hasn't": "has not", "haven't": "have not", "he'd": "he would","he'll": "he will", "he's": "he is", "how'd": "how did", "how'd'y": "how do you", "how'll": "how will", "how's": "how is", "I'd": "I would", "I'd've": "I would have", "I'll": "I will", "I'll've": "I will have","I'm": "I am", "I've": "I have", "i'd": "i would", "i'd've": "i would have", "i'll": "i will", "i'll've": "i will have","i'm": "i am", "i've": "i have", "isn't": "is not", "it'd": "it would", "it'd've": "it would have", "it'll": "it will", "it'll've": "it will have","it's": "it is", "let's": "let us", "ma'am": "madam", "mayn't": "may not", "might've": "might have","mightn't": "might not","mightn't've": "might not have", "must've": "must have", "mustn't": "must not", "mustn't've": "must not have", "needn't": "need not", "needn't've": "need not have","o'clock": "of the clock", "oughtn't": "ought not", "oughtn't've": "ought not have", "shan't": "shall not", "sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would", "she'd've": "she would have", "she'll": "she will", "she'll've": "she will have", "she's": "she is", "should've": "should have", "shouldn't": "should not", "shouldn't've": "should not have", "so've": "so have","so's": "so as", "this's": "this is","that'd": "that would", "that'd've": "that would have", "that's": "that is", "there'd": "there would", "there'd've": "there would have", "there's": "there is", "here's": "here is","they'd": "they would", "they'd've": "they would have", "they'll": "they will", "they'll've": "they will have", "they're": "they are", "they've": "they have", "to've": "to have", "wasn't": "was not", "we'd": "we would", "we'd've": "we would have", "we'll": "we will", "we'll've": "we will have", "we're": "we are", "we've": "we have", "weren't": "were not", "what'll": "what will", "what'll've": "what will have", "what're": "what are", "what's": "what is", "what've": "what have", "when's": "when is", "when've": "when have", "where'd": "where did", "where's": "where is", "where've": "where have", "who'll": "who will", "who'll've": "who will have", "who's": "who is", "who've": "who have", "why's": "why is", "why've": "why have", "will've": "will have", "won't": "will not", "won't've": "will not have", "would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have", "y'all": "you all", "y'all'd": "you all would","y'all'd've": "you all would have","y'all're": "you all are","y'all've": "you all have","you'd": "you would", "you'd've": "you would have", "you'll": "you will", "you'll've": "you will have", "you're": "you are", "you've": "you have" }
punct = "/-'?!.,
punct_mapping = {"‘": "'", "₹": "e", "´": "'", "°": "", "€": "e", "™": "tm", "√": " sqrt ", "×": "x", "²": "2", "—": "-", "–": "-", "’": "'", "_": "-", "`": "'", '“': '"', '”': '"', '“': '"', "£": "e", '∞': 'infinity', 'θ': 'theta', '÷': '/', 'α': 'alpha', '•': '.', 'à': 'a', '−': '-', 'β': 'beta', '∅': '', '³': '3', 'π': 'pi', }
mispell_dict = {'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', "mastrubating": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', "whst": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization'}
def clean_contractions(text, mapping=contraction_mapping):
specials = ["’", "‘", "´", "`"]
for s in specials:
text = text.replace(s, "'")
 text = ' '.join([mapping[t] if t in mapping else t for t in text.split(" ")])
 return text
def clean_special_chars(text, punct=punct, mapping=punct_mapping):
for p in mapping:
text = text.replace(p, mapping[p])
 for p in punct:
text = text.replace(p, f' {p} ')
 specials = {'\u200b': ' ', '…': '...', '\ufeff': '', 'करना': '', 'है': ''}
for s in specials:
text = text.replace(s, specials[s])
 return text
def correct_spelling(x, dictionary=mispell_dict):
for word in dictionary.keys() :
x = x.replace(word, dictionary[word])
 return x
def clean(text):
text = text.lower()
text = clean_contractions(text)
 text = clean_special_chars(text)
 text = correct_spelling(text)
 return text
df_items = pd.read_csv('.. /input/items.csv')
 df_categories = pd.read_csv('.. /input/item_categories.csv')
 df_shops = pd.read_csv('.. /input/shops.csv' )<feature_engineering>
sincere_counts = Counter()
insincere_counts = Counter()
word_dict = Counter()
sincere_to_insincere_ratio = Counter()
def prepare_dicts() :
qs = [clean(i)for i in train_set['question_text']]
lbl = [j for j in train_set['target']]
for i,j in zip(qs,lbl):
words = i.split()
for word in words:
word_dict[word] += 1
if j == 0:
sincere_counts[word] += 1
elif j == 1:
insincere_counts[word] += 1
tst_qs = [clean(i)for i in test_set['question_text']]
for i in tst_qs:
i = i.split()
for j in i:
word_dict[j] += 1
print('Words in sincere Questions: {}'.format(len(sincere_counts)))
 print('Words in insincere Questions: {}'.format(len(insincere_counts)))
 print('Total Words in corpus: {}'.format(len(word_dict)))
 print('Most Common Words in Sincere Questions : ')
 print(sincere_counts.most_common() [:10])
 print('Most Common Words in Insincere Questions : ')
 print(insincere_counts.most_common() [:10])
 for i in sincere_counts:
if sincere_counts[i] >= 100:
sincere_to_insincere_ratio[i] = np.log(sincere_counts[i]/(insincere_counts[i] + 1))
print('The Most Sincere Words : ')
 print(sincere_to_insincere_ratio.most_common() [:10])
 print('The Most Insincere Words : ')
 print(list(reversed(sincere_to_insincere_ratio.most_common())) [:10])
df_train['datatype'], df_test['datatype'] = 'training', 'testing'
df_train.insert(0, 'ID', np.nan)
 df_test.insert(1, 'date', '01.11.2015')
 df_test.insert(2, 'date_block_num', 34)
 df_test.insert(df_test.shape[1] - 1, 'item_price', np.nan)
 df_test.insert(df_test.shape[1] - 1, 'item_cnt_day', np.nan)
 df_data = pd.concat([df_train, df_test], ignore_index=False )<feature_engineering>
prepare_dicts()
df_data.loc[df_data['shop_id'] == 0, 'shop_id'] = 57
df_data.loc[df_data['shop_id'] == 1, 'shop_id'] = 58
df_data.loc[df_data['shop_id'] == 11, 'shop_id'] = 10<feature_engineering>
train_x = list(train_set['question_text'].fillna("_na_" ).values)
 train_y = train_set['target'].values
test_x = list(test_set['question_text'].fillna("_na_" ).values)
 train_x, val_x, train_y, val_y = train_test_split(train_x, train_y, test_size=0.2)
 train_x = [clean(i)for i in train_x]
val_x = [clean(i)for i in val_x]
test_x = [clean(i)for i in test_x]
print('An Example From Train Set: ')
 print(train_x[0])
 tokenizer = text.Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_x))
train_X = tokenizer.texts_to_sequences(train_x)
 val_X = tokenizer.texts_to_sequences(val_x)
 test_X = tokenizer.texts_to_sequences(test_x)
 print('After Tokenizing: ')
 print(train_X[0])
 train_X = seq.pad_sequences(train_X, maxlen=max_seq_len)
 val_X = seq.pad_sequences(val_X, maxlen=max_seq_len)
 test_X = seq.pad_sequences(test_X, maxlen=max_seq_len)
 print('After Padding: ')
 print(train_X[0])
 print(np.shape(train_X), np.shape(train_y), np.shape(val_X), np.shape(val_y))
df_data.loc[df_data['item_price'] < 0, 'item_price'] = df_data.loc[(df_data['date_block_num'] == 4)&(df_data['shop_id'] == 32)&(df_data['item_price'] > 0), 'item_price'].median()
df_data = df_data[(( df_data['item_price'] >= 0)&(df_data['item_price'] <= 100000)) |(df_data['item_price'].isna())]<filter>
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 def get_embeddings(embedding_name, mode='new'):
filePath = '.. /input/embeddings/{0}/{0}.txt'.format(embedding_name)
 if mode == 'new':
embeddings_idx = dict(get_coefs(*i.split(" ")) for i in open(filePath))
all_embs = np.stack(embeddings_idx.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_idx.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embeddings_idx, embedding_matrix
df_data = df_data[(df_data['item_cnt_day'] <= 1000)|(df_data['item_cnt_day'].isna())]<create_dataframe>
def check_coverage(vocab, embeddings_index):
known_words = {}
unknown_words = {}
nb_known_words = 0
nb_unknown_words = 0
for word in vocab.keys() :
try:
known_words[word] = embeddings_index[word]
nb_known_words += vocab[word]
except:
unknown_words[word] = vocab[word]
nb_unknown_words += vocab[word]
pass
print('Found embeddings for {:.2%} of vocab'.format(len(known_words)/ len(vocab)))
 print('Found embeddings for {:.2%} of all text'.format(nb_known_words /(nb_known_words + nb_unknown_words)))
 unknown_words = sorted(unknown_words.items() , key=operator.itemgetter(1)) [::-1]
return unknown_words
df_block = df_data.groupby(['date_block_num', 'shop_id', 'item_id'], as_index=False ).agg({
'item_price': 'mean', 'item_cnt_day': sum
} ).rename(columns={'item_cnt_day': 'item_cnt_month'} )<merge>
embedding_idxs, embedding_mtx = get_embeddings(EMBEDDING, 'new')
 unk_wrds = check_coverage(word_dict, embedding_idxs )
df_block = pd.merge(df_cross, df_block, how='left', left_on=['date_block_num', 'shop_id', 'item_id'], right_on=['date_block_num', 'shop_id', 'item_id'] )<feature_engineering>
def get_model(model_type):
if model_type == 'nb':
model = NaiveBayes()
elif model_type == 'svm':
model = __SVC__()
elif model_type == 'rnn':
inp = Input(shape=(max_seq_len,))
layer = Embedding(max_features, embed_size, weights=[embedding_mtx], trainable=False )(inp)
 layer = SimpleRNN(32, return_sequences=True )(layer)
 layer = GlobalMaxPool1D()(layer)
 layer = Dense(16, activation='relu' )(layer)
 layer = Dropout(0.1 )(layer)
 layer = Dense(1, activation='sigmoid' )(layer)
 model = Model(inputs=inp, outputs=layer)
 elif model_type == 'lstm':
inp = Input(shape=(max_seq_len,))
layer = Embedding(max_features, embed_size, weights=[embedding_mtx], trainable=False )(inp)
 layer = Bidirectional(CuDNNLSTM(32, return_sequences=True))(layer)
 layer = GlobalMaxPool1D()(layer)
 layer = Dense(16, activation='relu' )(layer)
 layer = Dropout(0.1 )(layer)
 layer = Dense(1, activation='sigmoid' )(layer)
 model = Model(inputs=inp, outputs=layer)
 elif model_type == 'gru':
inp = Input(shape=(max_seq_len,))
layer = Embedding(max_features, embed_size, weights=[embedding_mtx], trainable=False )(inp)
 layer = Bidirectional(CuDNNGRU(32, return_sequences=True))(layer)
 layer = GlobalMaxPool1D()(layer)
 layer = Dense(16, activation='relu' )(layer)
 layer = Dropout(0.1 )(layer)
 layer = Dense(1, activation='sigmoid' )(layer)
 model = Model(inputs=inp, outputs=layer)
 elif model_type == 'attention':
inp = Input(shape=(max_seq_len,))
layer = Embedding(max_features, embed_size, weights=[embedding_mtx], trainable=False )(inp)
 layer = Bidirectional(CuDNNLSTM(32, return_sequences=True))(layer)
 layer = Attention(max_seq_len )(layer)
 layer = Dense(16, activation='relu' )(layer)
 layer = Dropout(0.1 )(layer)
 layer = Dense(1, activation='sigmoid' )(layer)
 model = Model(inputs=inp, outputs=layer)
 return model
df_block['item_cnt_month'] = df_block['item_cnt_month'].fillna(0 )<merge>
class NaiveBayes() :
def __init__(self):
self.sincere_example_count = sincere_examples
self.insincere_example_count = insincere_examples
self.total_examples = x[0]+x[1]
self.sincere_dict = sincere_counts
self.insincere_dict = insincere_counts
self.word_dict= word_dict
self.sincere_word_count = np.sum(list(sincere_counts.values()))
self.insincere_word_count = np.sum(list(insincere_counts.values()))
self.sincere_prob = self.sincere_example_count / self.total_examples
self.insincere_prob = self.insincere_example_count / self.total_examples
def summary(self):
print('Positive Examples : {}, Negative Examples : {}, Total Examples : {}'.format(self.sincere_example_count, self.insincere_example_count, self.total_examples))
def predict(self, x_test):
print('Predicting...')
 predictions = []
for example in x_test:
p_words = np.prod([word_dict[j]/np.sum(list(word_dict.values())) for j in example.split() ])
 p_words += 2
sincere_prob_num = np.prod([sincere_counts[j]/self.sincere_word_count for j in example.split() ])* self.sincere_prob
insincere_prob_num = np.prod([insincere_counts[j]/self.insincere_word_count for j in example.split() ])* self.insincere_prob
sincere_prob = sincere_prob_num/p_words
insincere_prob = insincere_prob_num/p_words
predictions.append(np.argmax([sincere_prob, insincere_prob]))
return predictions
class __SVC__(SVC):
def __init__(self):
super(__SVC__,self ).__init__(verbose=True)
 print('initializing...')
 def summary(self):
print(self.__dict__)
 def prepare_data(self):
self.X_train = [embedding_mtx[i] for example in train_X for i in example]
self.X_val = [embedding_mtx[i] for example in val_X for i in example]
self.X_test = [embedding_mtx[i] for example in test_X for i in example]
df_block = pd.merge(df_block, df_items, how='left', left_on='item_id', right_on='item_id' )<merge>
class Attention(Layer):
def __init__(self, step_dim, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.features_dim = 0
self.step_dim = step_dim
self.bias = True
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name))
self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name), 
)
 else:
self.b = None
self.built = True
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
df_block = pd.merge(df_block, df_categories, how='left', left_on='item_category_id', right_on='item_category_id' )<merge>
def print_f1s(predictions):
for threshold in np.arange(0.1, 0.501, 0.01):
threshold = np.round(threshold, 2)
 print("F1 score at threshold {0} is {1}".format(threshold, f1_score(val_y,(predictions>threshold ).astype(int))))
df_block = pd.merge(df_block, df_shops, how='left', left_on='shop_id', right_on='shop_id' )<count_values>
df_block['item_id'].value_counts()<count_values>
rnn = get_model('rnn')
 rnn.summary()
rnn.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-3), metrics=['accuracy'])
 rnn.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y), callbacks=[earlystop, reducelr])
 predictions_rnn_real = rnn.predict(test_X)
 predictions_rnn =(predictions_rnn_real >= thresh ).astype(int)
 predictions_val_rnn = rnn.predict(val_X, batch_size=1024)
 print_f1s(predictions_val_rnn)
 del rnn
gc.collect()
time.sleep(10 )
df_block['item_category_id'].value_counts()<count_values>
gru = get_model('gru')
 gru.summary()
gru.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-3), metrics=['accuracy'])
 gru.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y), callbacks=[earlystop, reducelr])
 predictions_gru_real = gru.predict(test_X)
 predictions_gru =(predictions_gru_real >= thresh ).astype(int)
 predictions_val_gru = gru.predict(val_X, batch_size=1024)
 print_f1s(predictions_val_gru)
 prediction_gru = pd.DataFrame({"qid":test_set["qid"].values})
 prediction_gru['prediction'] = predictions_gru
prediction_gru.to_csv("submission.csv", index=False)
 del gru
gc.collect()
time.sleep(10 )
df_block['shop_id'].value_counts()<filter>
lstm = get_model('lstm')
 lstm.summary()
lstm.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-3), metrics=['accuracy'])
 lstm.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y), callbacks=[earlystop, reducelr])
 predictions_lstm_real = lstm.predict(test_X)
 predictions_lstm =(predictions_lstm_real >= thresh ).astype(int)
 predictions_val_lstm = lstm.predict(val_X, batch_size=1024)
 print_f1s(predictions_val_lstm)
 del lstm
gc.collect()
time.sleep(10 )
df_items.loc[df_items['item_id'] == 20949, 'item_category_id']<filter>
attention = get_model('attention')
 attention.summary()
attention.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-3), metrics=['accuracy'])
 attention.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y), callbacks=[earlystop, reducelr])
 predictions_attention_real = attention.predict(test_X)
 predictions_attention =(predictions_attention_real >= thresh ).astype(int)
 predictions_val_attention = attention.predict(val_X, batch_size=1024)
 print_f1s(predictions_val_attention)
 del attention
gc.collect()
time.sleep(10)
df_items.loc[df_items['item_category_id'] == 71, 'item_id']<count_values>
val_preds = 0.50*predictions_val_gru + 0.25*predictions_val_lstm + 0.25*predictions_val_attention
val_preds =(val_preds > thresh ).astype(int)
 print_f1s(val_preds )
<count_values><EOS> 
final_preds = 0.50*predictions_gru_real + 0.25*predictions_lstm_real + 0.25*predictions_attention_real
final_preds =(final_preds > thresh ).astype(int)
 final_prediction = pd.DataFrame({"qid":test_set["qid"].values})
 final_prediction['prediction'] = final_preds
final_prediction.to_csv("submission.csv", index=False )