Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<set_options>
import os
import time
import numpy as np
import pandas as pd
from tqdm import tqdm
import math
from sklearn.model_selection import train_test_split
from sklearn import metrics
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D
from keras.layers import Bidirectional, GlobalMaxPool1D
from keras.models import Model
from keras import initializers, regularizers, constraints, optimizers, layers
import matplotlib.pyplot as plt
%matplotlib inline
sns.set_style('darkgrid')
 plt.rcParams['figure.figsize'] = 7, 5<load_from_csv>
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape )
house_data = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv', index_col='Id')
 y = house_data['SalePrice']
X = house_data.drop('SalePrice', axis=1 )<load_from_csv>
train_df, val_df = train_test_split(train_df, test_size=0.1, random_state=2018)
 embed_size = 300
max_features = 50000
maxlen = 100
train_X = train_df["question_text"].fillna("_na_" ).values
val_X = val_df["question_text"].fillna("_na_" ).values
test_X = test_df["question_text"].fillna("_na_" ).values
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 val_X = pad_sequences(val_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
val_y = val_df['target'].values
X_test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv', index_col='Id' )<count_values>
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size )(inp)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = GlobalMaxPool1D()(x)
 x = Dense(16, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 print(model.summary() )
X.dtypes.value_counts()<groupby>
model.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))
top_ten_corr_features = corr_matrix.nlargest(n=11, columns='SalePrice')['SalePrice'][1:]
top_ten_corr_features<categorify>
pred_noemb_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_noemb_val_y>thresh ).astype(int))))
def impute(df, column, value):

 df[column] = df[column].fillna(value )<count_missing_values>
print('Accuracy withour using pretrained Embedings =', metrics.accuracy_score(val_y, [round(float(i)) for i in pred_noemb_val_y]))
X.isna().sum() [X.isna().sum() > 0]<count_missing_values>
pred_noemb_test_y = model.predict([test_X], batch_size=1024, verbose=1 )
X_test.isna().sum() [X_test.isna().sum() > 0]<data_type_conversions>
del model, inp, x
time.sleep(10 )
print('Variable with NA in training but not test set: ', set(X.columns[X.isna().any() ].tolist())- set(X_test.columns[X_test.isna().any() ].tolist()))
print('Variables with NA in test but not training set: ', set(X_test.columns[X_test.isna().any() ].tolist())- set(X.columns[X.isna().any() ].tolist()))<count_values>
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = GlobalMaxPool1D()(x)
 x = Dense(16, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 print(model.summary() )
impute(X, 'MSZoning', X['MSZoning'].value_counts().idxmax())
 impute(X_test, 'MSZoning', X['MSZoning'].value_counts().idxmax() )<drop_column>
model.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))
X['LotFrontage'] = X['LotFrontage'].fillna(X.groupby('Neighborhood')['LotFrontage'].transform('median'))
lotfrontage_median = pd.DataFrame(X.groupby('Neighborhood')['LotFrontage'].median() ).rename(columns={'LotFrontage':'LotFrontage_median'})
 X_test_tmp = X_test.reset_index().merge(lotfrontage_median, on='Neighborhood' ).set_index('Id')
 X_test_tmp['LotFrontage'] = X_test_tmp['LotFrontage'].fillna(X_test_tmp['LotFrontage_median'])
 X_test = X_test_tmp.drop('LotFrontage_median', axis=1 )<count_values>
pred_glove_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_glove_val_y>thresh ).astype(int))))
impute(X, 'Utilities', X['Utilities'].value_counts().idxmax())
 impute(X_test, 'Utilities', X['Utilities'].value_counts().idxmax() )<count_values>
pred_glove_test_y = model.predict([test_X], batch_size=1024, verbose=1 )
impute(X, 'Exterior1st', X['Exterior1st'].value_counts().idxmax())
 impute(X_test, 'Exterior1st', X['Exterior1st'].value_counts().idxmax())
 impute(X, 'Exterior2nd', X['Exterior2nd'].value_counts().idxmax())
 impute(X_test, 'Exterior2nd', X['Exterior2nd'].value_counts().idxmax() )<count_values>
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )
impute(X, 'KitchenQual', X['KitchenQual'].value_counts().idxmax())
 impute(X_test, 'KitchenQual', X['KitchenQual'].value_counts().idxmax() )<count_values>
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = GlobalMaxPool1D()(x)
 x = Dense(16, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'] )
impute(X, 'Functional', X['Functional'].value_counts().idxmax())
 impute(X_test, 'Functional', X['Functional'].value_counts().idxmax() )<count_values>
model.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))
impute(X, 'SaleType', X['SaleType'].value_counts().idxmax())
 impute(X_test, 'SaleType', X['SaleType'].value_counts().idxmax() )<count_values>
pred_fasttext_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_fasttext_val_y>thresh ).astype(int))))
impute(X, 'Electrical', X['Electrical'].value_counts().idxmax())
 impute(X_test, 'Electrical', X['Electrical'].value_counts().idxmax() )<count_missing_values>
pred_fasttext_test_y = model.predict([test_X], batch_size=1024, verbose=1 )
print('Number of NA in training set after imputation: ', X.isna().sum().sum())
 print('Number of NA in test set after imputation: ', X_test.isna().sum().sum() )<feature_engineering>
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )
def create_features(X):
X['TotalSF'] = X['TotalBsmtSF'] + X['GrLivArea']
X['TotalPorchSF'] = X['OpenPorchSF'] + X['EnclosedPorch'] + X['3SsnPorch'] + X['ScreenPorch']
X['TotalBath'] = X['BsmtFullBath'] + 0.5 * X['BsmtHalfBath'] + X['FullBath'] + 0.5 * X['HalfBath']
X['TotalRooms'] = X['TotRmsAbvGrd'] + X['FullBath'] + 0.5 * X['HalfBath']
X['HasGarage'] = X['GarageArea'] > 0
X['HasPool'] = X['PoolArea'] > 0
X['HasFireplace'] = X['Fireplaces'] > 0
X['HasBasement'] = X['TotalBsmtSF'] > 0
X['Has2ndFloor'] = X['2ndFlrSF'] > 0
X['IsRemodeled'] = X['YearBuilt'] != X['YearRemodAdd']
return X
X = create_features(X)
 X_test = create_features(X_test )<feature_engineering>
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = GlobalMaxPool1D()(x)
 x = Dense(16, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'] )
<data_type_conversions>
model.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))
num_cols_to_cat = ['MSSubClass', 'MoSold', 'YrSold']
X[num_cols_to_cat] = X[num_cols_to_cat].astype(str)
 X_test[num_cols_to_cat] = X_test[num_cols_to_cat].astype(str )<define_variables>
pred_paragram_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_paragram_val_y>thresh ).astype(int))))
cat_cols = [col for col in X.columns if X.dtypes[col] == 'object']
num_cols = [col for col in X.columns if X.dtypes[col] in ['int64', 'float64']]<categorify>
pred_paragram_test_y = model.predict([test_X], batch_size=1024, verbose=1 )
def ordinal_encoder(X):
X = X.replace({
'Street': {'Pave':1, 'Grvl':2},
'Alley': {'None':0, 'Pave':1, 'Grvl':2},
'LotShape': {'IR3':1, 'IR2':2, 'IR1':3, 'Reg':4},
'Utilities': {'ELO':1, 'NoSeWa':2, 'NoSewr':3, 'AllPub':4},
'LandSlope': {'Sev':1, 'Mod':2, 'Gtl':3},
'ExterQual': {'Po':1, 'Fa':2, 'TA':3, 'Gd':4, 'Ex':5},
'ExterCond': {'Po':1, 'Fa':2, 'TA':3, 'Gd':4, 'Ex':5},
'BsmtQual': {'None':0, 'Po':1, 'Fa':2, 'TA':3, 'Gd':4, 'Ex':5},
'BsmtCond': {'None':0, 'Po':1, 'Fa':2, 'TA':3, 'Gd':4, 'Ex':5},
'BsmtExposure': {'None':0, 'No':1, 'Mn':2, 'Av':3, 'Gd':4},
'BsmtFinType1': {'None':0, 'Unf':1, 'LwQ':2, 'Rec':3, 'BLQ':4, 'ALQ':5, 'GLQ':6},
'BsmtFinType2': {'None':0, 'Unf':1, 'LwQ':2, 'Rec':3, 'BLQ':4, 'ALQ':5, 'GLQ':6},
'HeatingQC': {'Po':1, 'Fa':2, 'TA':3, 'Gd':4, 'Ex':5},
'KitchenQual': {'Po':1, 'Fa':2, 'TA':3, 'Gd':4, 'Ex':5},
'Functional': {'Sal':1, 'Sev':2, 'Maj2':3, 'Maj1':4, 'Mod':5, 'Min2':6, 'Min1':7, 'Typ':8},
'FireplaceQu': {'None':0, 'Po':1, 'Fa':2, 'TA':3, 'Gd':4, 'Ex':5},
'GarageFinish': {'None':0, 'Unf':1, 'RFn':2, 'Fin':3},
'GarageQual': {'None':0, 'Po':1, 'Fa':2, 'TA':3, 'Gd':4, 'Ex':5},
'GarageCond': {'None':0, 'Po':1, 'Fa':2, 'TA':3, 'Gd':4, 'Ex':5},
'PavedDrive': {'N':0, 'P':1, 'Y':2},
'PoolQC': {'None':0, 'Fa':1, 'TA':2, 'Gd':3, 'Ex':4},
'Fence': {'None':0, 'MnWw':1, 'GdWo':2, 'MnPrv':3, 'GdPrv':4},
'Electrical':{'Mix':1, 'FuseP':2, 'FuseF':3, 'FuseA':4, 'SBrkr':5}
})
 return X
X = ordinal_encoder(X)
 X_test = ordinal_encoder(X_test )<categorify>
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )
X = pd.get_dummies(X)
 X_test = pd.get_dummies(X_test )<feature_engineering>
pred_val_y = 0.33*pred_glove_val_y + 0.33*pred_fasttext_val_y + 0.34*pred_paragram_val_y
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_val_y>thresh ).astype(int))))
skewness = X[num_cols].apply(lambda x: skew(x))
skewed_cols = skewness[abs(skewness)> 0.5].index
X[skewed_cols] = np.log1p(X[skewed_cols])
 X_test[skewed_cols] = np.log1p(X_test[skewed_cols] )<train_on_grid>
out_df = pd.DataFrame({"qid":test_df["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False )