Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<merge>
import os
import time
import numpy as np
import pandas as pd
from tqdm import tqdm
import math
from sklearn.model_selection import train_test_split
from sklearn import metrics
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D
from keras.layers import Bidirectional, GlobalMaxPool1D
from keras.models import Model
from keras import initializers, regularizers, constraints, optimizers, layers
df=pd.merge(df,titles,how='outer',left_index=True,right_index=True)
 df.head()<data_type_conversions>
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape )
nonull_df=df[(df['Age'].notnull())&(df['Age'].notnull())&(df['Embarked'].notnull())&(df['block'].notnull())&(df['CabinOccupancy'].notnull())]
nonull_df.head()<count_values>
train_df, val_df = train_test_split(train_df, test_size=0.1, random_state=2018)
 embed_size = 300
max_features = 50000
maxlen = 100
train_X = train_df["question_text"].fillna("_na_" ).values
val_X = val_df["question_text"].fillna("_na_" ).values
test_X = test_df["question_text"].fillna("_na_" ).values
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 val_X = pad_sequences(val_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
val_y = val_df['target'].values
nonull_df.count() ['PassengerId']<categorify>
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size )(inp)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = GlobalMaxPool1D()(x)
 x = Dense(16, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 print(model.summary() )
cat_feats=['Embarked','Sex','Titles','block','Pclass']
nonull_df_train = pd.get_dummies(nonull_df,columns=cat_feats,drop_first=False)
 nonull_df_train=nonull_df_train[['Age','Fare','Parch','Pclass_1','Pclass_2','Pclass_3','SibSp','Embarked_S','Sex_male','Titles_ Mr']]
nonull_df_train.head() 
<prepare_x_and_y>
model.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))
X_nonull_df_train = nonull_df_train.drop('Fare',axis=1)
 y_nonull_df_train = nonull_df_train['Fare']<filter>
pred_noemb_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_noemb_val_y>thresh ).astype(int))))
nullfare=df[df['Fare'].isnull() ]
nullfare<categorify>
pred_noemb_test_y = model.predict([test_X], batch_size=1024, verbose=1 )
nullfare=df[df['Fare'].isnull() ]
cat_feats=['Embarked','Sex','block','Titles','Pclass']
nullfare_test = pd.get_dummies(nullfare,columns=cat_feats,drop_first=False)
 nullfare_test=nullfare_test.drop(columns=['Name','Ticket','PassengerId','Cabin','Survived','CabinOccupancy'])
 nullfare_test.head() 
<drop_column>
del model, inp, x
time.sleep(10 )
X_nullfare_test = nullfare_test<drop_column>
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = GlobalMaxPool1D()(x)
 x = Dense(16, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 print(model.summary() )
X_nonull_df_train=X_nonull_df_train.drop(columns=['Pclass_1','Pclass_2'])
 X_nullfare_test=X_nullfare_test.drop(columns=['Fare'])
<train_model>
model.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))
dtreeReg = DecisionTreeRegressor()
dtreeReg.fit(X_nonull_df_train,y_nonull_df_train )<predict_on_test>
pred_glove_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_glove_val_y>thresh ).astype(int))))
predictions_nullfare = dtreeReg.predict(X_nullfare_test)
 predictions_nullfare<feature_engineering>
pred_glove_test_y = model.predict([test_X], batch_size=1024, verbose=1 )
nullfare['Fare']=np.where(nullfare['Fare'].isnull() ,93.5,nullfare['Fare'])
 nullfare.head()<feature_engineering>
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )
df['Fare']=np.where(df['Fare'].isnull() ,93.5,df['Fare'])
 df[df['PassengerId']==1044].head()<categorify>
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = GlobalMaxPool1D()(x)
 x = Dense(16, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'] )
cat_feats=['Embarked','Sex','Titles','block','Pclass']
nonull_df_train = pd.get_dummies(nonull_df,columns=cat_feats,drop_first=False)
 nonull_df_train=nonull_df_train[['Age','Fare','Parch','SibSp',
'Embarked_S','Sex_male','Titles_ Mr',
'block_A','block_B','block_C','block_D',
'block_E','block_F','block_G','Pclass_1','Pclass_2','Pclass_3']]
nonull_df_train.head() 
<prepare_x_and_y>
model.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))
X_nonull_df_train = nonull_df_train.drop(['block_A','block_B','block_C','block_D','block_E','block_F','block_G'],axis=1)
 y_nonull_df_train = nonull_df_train[['block_A','block_B','block_C','block_D','block_E','block_F','block_G']]<categorify>
pred_fasttext_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_fasttext_val_y>thresh ).astype(int))))
nullblock=nullfare
cat_feats=['Embarked','Sex','block','Titles']
nullblock_test = pd.get_dummies(nullblock,columns=cat_feats,drop_first=False)
 nullblock_test=nullblock_test.drop(columns=['Name','Ticket','PassengerId','Cabin','Survived'])
 nullblock_test.head() 
<drop_column>
pred_fasttext_test_y = model.predict([test_X], batch_size=1024, verbose=1 )
X_nullblock_test = nullblock_test<drop_column>
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )
X_nonull_df_train=X_nonull_df_train.drop(columns=['Pclass_1','Pclass_2'])
 X_nullblock_test=X_nullblock_test.drop(columns=['CabinOccupancy'])
<train_model>
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = GlobalMaxPool1D()(x)
 x = Dense(16, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'] )
dtreeClass = DecisionTreeClassifier()
dtreeClass.fit(X_nonull_df_train,y_nonull_df_train )<predict_on_test>
model.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))
predictions_nullblock = dtreeClass.predict(X_nullblock_test)
 predictions_nullblock<feature_engineering>
pred_paragram_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_paragram_val_y>thresh ).astype(int))))
nullblock['block']=np.where(nullfare['block'].isnull() ,'C',nullblock['block'])
 nullblock.head()<feature_engineering>
pred_paragram_test_y = model.predict([test_X], batch_size=1024, verbose=1 )
df['block']=np.where(( df['block'].isnull())&(df['PassengerId']==1044),'C',df['block'])
 df[df['PassengerId']==1044].head()<categorify>
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )
cat_feats=['Embarked','Sex','Titles','block','Pclass','CabinOccupancy']
nonull_df_train = pd.get_dummies(nonull_df,columns=cat_feats,drop_first=False)
 nonull_df_train=nonull_df_train[['Age','Fare','Parch','Pclass_1','Pclass_2','Pclass_3','SibSp',
'Embarked_S','Sex_male','block_C','Titles_ Mr','CabinOccupancy_1',
'CabinOccupancy_2','CabinOccupancy_3','CabinOccupancy_4','CabinOccupancy_5','CabinOccupancy_6']]
nonull_df_train.head() 
<prepare_x_and_y>
pred_val_y = 0.33*pred_glove_val_y + 0.33*pred_fasttext_val_y + 0.34*pred_paragram_val_y
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_val_y>thresh ).astype(int))))
X_nonull_df_train = nonull_df_train.drop(columns=['CabinOccupancy_1','CabinOccupancy_2','CabinOccupancy_3','CabinOccupancy_4','CabinOccupancy_5','CabinOccupancy_6'],axis=1)
 y_nonull_df_train = nonull_df_train[['CabinOccupancy_1','CabinOccupancy_2','CabinOccupancy_3','CabinOccupancy_4','CabinOccupancy_5','CabinOccupancy_6']]<categorify>
pred_test_y = 0.33*pred_glove_test_y + 0.33*pred_fasttext_test_y + 0.34*pred_paragram_test_y
pred_test_y =(pred_test_y>0.35 ).astype(int)
 out_df = pd.DataFrame({"qid":test_df["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False )