Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<feature_engineering>
import os
import time
import numpy as np
import pandas as pd
from tqdm import tqdm
import math
from sklearn.model_selection import train_test_split
from sklearn import metrics
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Input, CuDNNLSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D
from keras.layers import Bidirectional, GlobalMaxPool1D, GlobalMaxPooling1D, GlobalAveragePooling1D
from keras.layers import Input, Embedding, Dense, Conv2D, MaxPool2D, concatenate
from keras.layers import Reshape, Flatten, Concatenate, Dropout, SpatialDropout1D
from keras.optimizers import Adam
from keras.models import Model
from keras import backend as K
from keras.engine.topology import Layer
from keras import initializers, regularizers, constraints, optimizers, layers
from keras.layers import *
from keras.models import *
from keras import initializers, regularizers, constraints, optimizers, layers
from keras.initializers import *
from keras.optimizers import *
import keras.backend as K
from keras.callbacks import *
import tensorflow as tf
import os
import time
import gc
import re
import glob
train_data['Age'] = train_data[['Age','Pclass']].apply(inpute,axis=1)
 test_data['Age'] = test_data[['Age','Pclass']].apply(inpute,axis=1 )<drop_column>
max_features = 100000
maxlen = 72
embed_size = 300
train_data.drop('Cabin',axis=1,inplace=True )<data_type_conversions>
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
def clean_text(x):
x = str(x)
 for punct in puncts:
x = x.replace(punct, f' {punct} ')
 return x
def load_and_prec() :
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape)
 train_df["question_text"] = train_df["question_text"].apply(lambda x: clean_text(x))
test_df["question_text"] = test_df["question_text"].apply(lambda x: clean_text(x))
train_df, val_df = train_test_split(train_df, test_size=0.08, random_state=2018)
 train_X = train_df["question_text"].fillna("_
val_X = val_df["question_text"].fillna("_
test_X = test_df["question_text"].fillna("_

 tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X)

train_X = pad_sequences(train_X, maxlen=maxlen)
 val_X = pad_sequences(val_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
val_y = val_df['target'].values
np.random.seed(2018)
 trn_idx = np.random.permutation(len(train_X))
val_idx = np.random.permutation(len(val_X))
train_X = train_X[trn_idx]
val_X = val_X[val_idx]
train_y = train_y[trn_idx]
val_y = val_y[val_idx]
return train_X, val_X, test_X, train_y, val_y, tokenizer.word_index
train_data.Age=train_data.Age.astype(int)
 test_data.Age = test_data.Age.astype(int )<feature_engineering>
train_X, val_X, test_X, train_y, val_y, word_index = load_and_prec()
for dataset in combine:
dataset.loc[ dataset['Age'] <= 16, 'Age'] = 0
dataset.loc[(dataset['Age'] > 16)&(dataset['Age'] <= 32), 'Age'] = 1
dataset.loc[(dataset['Age'] > 32)&(dataset['Age'] <= 48), 'Age'] = 2
dataset.loc[(dataset['Age'] > 48)&(dataset['Age'] <= 64), 'Age'] = 3
dataset.loc[ dataset['Age'] > 64, 'Age'] = 4
train_data.head()<save_to_csv>
embedding_matrix = load_glove(word_index )
y = train_data["Survived"]
features = ["Pclass", "Sex", "Age", "Parch",'SibSp']
X = pd.get_dummies(train_data[features])
 X_test = pd.get_dummies(test_data[features])
 model = RandomForestClassifier(n_estimators=100, max_depth=5, random_state=0)
 model.fit(X, y)
 predictions = model.predict(X_test)
 acc_random_forest = round(model.score(X,y)* 100, 2)
 acc_random_forest
output = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': predictions})
 output.to_csv('submission.csv', index=False)
 print("Your submission was successfully saved!" )<set_options>
def model_lstm_du(embedding_matrix):
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)

x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 avg_pool = GlobalAveragePooling1D()(x)
 max_pool = GlobalMaxPooling1D()(x)
 conc = concatenate([avg_pool, max_pool])
 conc = Dense(64, activation="relu" )(conc)
 conc = Dropout(0.1 )(conc)
 outp = Dense(1, activation="sigmoid" )(conc)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model
%matplotlib inline
color = sns.color_palette()
sns.set_style('darkgrid')
 def ignore_warn(*args, **kwargs):
pass
warnings.warn = ignore_warn
pd.set_option('display.float_format', lambda x: '{:.3f}'.format(x))
train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 train_ID = train['Id']
test_ID = test['Id']
train.drop("Id", axis = 1, inplace = True)
 test.drop("Id", axis = 1, inplace = True )<prepare_x_and_y>
def train_pred(model, epochs=2):
filepath="weights_best.h5"
checkpoint = ModelCheckpoint(filepath, monitor='val_loss', verbose=2, save_best_only=True, mode='min')
 reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=1, min_lr=0.0001, verbose=2)
 earlystopping = EarlyStopping(monitor='val_loss', min_delta=0.0001, patience=2, verbose=2, mode='auto')
 callbacks = [checkpoint, reduce_lr]
for e in range(epochs):
model.fit(train_X, train_y, batch_size=512, epochs=1, validation_data=(val_X, val_y),callbacks=callbacks)
 model.load_weights(filepath)
 pred_val_y = model.predict([val_X], batch_size=1024, verbose=0)
 pred_test_y = model.predict([test_X], batch_size=1024, verbose=0)
 return pred_val_y, pred_test_y
train.drop(train[train["GrLivArea"] > 4000].index, inplace=True)
 test.loc[1116, "GarageType"] = np.nan
train["SalePrice"] = np.log1p(train["SalePrice"])
 y_train = train.SalePrice.values
all_data = pd.concat(( train, test)).reset_index(drop=True ).drop(['SalePrice'], axis=1 )<data_type_conversions>
pred_val_y, pred_test_y = train_pred(model, epochs=8 )
def fillNA(data, var, fill=None, custom=False):
if not fill:
data_map = {
"float64": 0,
"int64": 0,
"object": "None"
}
data[var] = data[var].fillna(data_map[str(data[var].dtype)])
 else:
if custom:
data[var] = data[var].fillna(fill)
 elif fill == 'mode':
data[var] = data[var].fillna(data[var].mode() [0])
 return data
impute_vals = ["BedroomAbvGr", "Alley", "GarageYrBlt", "GarageArea", "GarageCars", 'BsmtFinSF1', 'PoolArea', 'MSSubClass',
'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'HalfBath', 'KitchenAbvGr', 'WoodDeckSF',
'BsmtHalfBath', 'MasVnrType', 'MasVnrArea', 'MiscVal', 'OpenPorchSF', 'ScreenPorch', 'TotRmsAbvGrd', "EnclosedPorch"]
for var in impute_vals:
all_data = fillNA(all_data, var)
 impute_mode = ['MSZoning', 'Electrical', 'Exterior1st', 'Exterior2nd', 'SaleType']
for var in impute_mode:
all_data = fillNA(all_data, var, fill='mode')
 impute_custom = {
"CentralAir": "N",
"Condition1": "Norm",
"Condition1": "Norm",
"Functional": "Typ",
"HeatingQC" : "TA",
"LotShape" : "Reg",
"PavedDrive": "N",
"PoolQC" : "No",
"BsmtQual" : "No",
"BsmtCond" : "No",
"BsmtExposure" : "No",
"BsmtFinType1" : "No",
"BsmtFinType2" : "No",
"ExterCond" : "No",
"ExterQual" : "No",
"Fence" : "No",
"FireplaceQu" : "No",
"GarageType" : "No",
"GarageFinish" : "No",
"GarageQual" : "No",
"GarageCond" : "No",
"KitchenQual" : "TA",
"MiscFeature" : "No",
"PavedDrive" : "N",
"SaleCondition": "Normal"
}
for var in impute_custom:
all_data = fillNA(all_data, var, impute_custom[var], custom=True)
 all_data["Functional"] = all_data["Functional"].fillna("Typ")
 all_data = all_data.drop(['Utilities'], axis=1)
 all_data["LotFrontage"] = all_data.groupby("Neighborhood")["LotFrontage"].transform(lambda x: x.fillna(x.median()))<data_type_conversions>
f1, threshold = f1_smart(val_y, pred_val_y)
 print('Optimal F1: {} at threshold: {}'.format(f1, threshold))
for var in ['MSSubClass', 'YrSold', 'OverallCond', 'MoSold']:
all_data[var] = all_data[var].astype(str)
 all_data = all_data.replace({"Alley" : {"No" : 0, "Grvl" : 1, "Pave" : 2},
"BsmtCond" : {"No" : 0, "Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
"BsmtExposure" : {"No" : 0, "Mn" : 1, "Av": 2, "Gd" : 3},
"BsmtFinType1" : {"No" : 0, "Unf" : 1, "LwQ": 2, "Rec" : 3, "BLQ" : 4, "ALQ" : 5, "GLQ" : 6},
"BsmtFinType2" : {"No" : 0, "Unf" : 1, "LwQ": 2, "Rec" : 3, "BLQ" : 4, "ALQ" : 5, "GLQ" : 6},
"BsmtQual" : {"No" : 0, "Po" : 1, "Fa" : 2, "TA": 3, "Gd" : 4, "Ex" : 5},
"ExterCond" : {"No" : 0, "Po" : 1, "Fa" : 2, "TA": 3, "Gd": 4, "Ex" : 5},
"ExterQual" : {"No" : 0, "Po" : 1, "Fa" : 2, "TA": 3, "Gd": 4, "Ex" : 5},
"FireplaceQu" : {"No" : 0, "Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
"Functional" : {"No" : 0, "Sal" : 1, "Sev" : 2, "Maj2" : 3, "Maj1" : 4, "Mod": 5, "Min2" : 6, "Min1" : 7, "Typ" : 8},
"GarageCond" : {"No" : 0, "Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
"GarageQual" : {"No" : 0, "Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
"HeatingQC" : {"No" : 0, "Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
"KitchenQual" : {"No" : 0, "Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
"LandSlope" : {"Sev" : 1, "Mod" : 2, "Gtl" : 3},
"LotShape" : {"IR3" : 1, "IR2" : 2, "IR1" : 3, "Reg" : 4},
"PavedDrive" : {"N" : 0, "P" : 1, "Y" : 2},
"PoolQC" : {"No" : 0, "Fa" : 1, "TA" : 2, "Gd" : 3, "Ex" : 4},
"Street" : {"Grvl" : 1, "Pave" : 2},
"Utilities" : {"ELO" : 1, "NoSeWa" : 2, "NoSewr" : 3, "AllPub" : 4}} 
)
 all_data["SimplOverallQual"] = all_data.OverallQual.replace({1 : 1, 2 : 1, 3 : 1, 4 : 2, 5 : 2, 6 : 2,7 : 3, 8 : 3, 9 : 3, 10 : 3})
 all_data["SimplOverallCond"] = all_data.OverallCond.replace({1 : 1, 2 : 1, 3 : 1, 4 : 2, 5 : 2, 6 : 2,7 : 3, 8 : 3, 9 : 3, 10 : 3})
 all_data["SimplPoolQC"] = all_data.PoolQC.replace({1 : 1, 2 : 1,3 : 2, 4 : 2})
 all_data["SimplGarageCond"] = all_data.GarageCond.replace({1 : 1, 4 : 2, 5 : 2})
 all_data["SimplGarageQual"] = all_data.GarageQual.replace({1 : 1, 4 : 2, 5 : 2})
 all_data["SimplFireplaceQu"] = all_data.FireplaceQu.replace({1 : 1, 4 : 2, 5 : 2})
 all_data["SimplFireplaceQu"] = all_data.FireplaceQu.replace({1 : 1, 4 : 2, 5 : 2})
 all_data["SimplFunctional"] = all_data.Functional.replace({1 : 1, 2 : 1,3 : 2, 4 : 2,5 : 3, 6 : 3, 7 : 3, 8 : 4})
 all_data["SimplKitchenQual"] = all_data.KitchenQual.replace({1 : 1, 4 : 2, 5 : 2})
 all_data["SimplHeatingQC"] = all_data.HeatingQC.replace({1 : 1,4 : 2, 5 : 2})
 all_data["SimplBsmtFinType1"] = all_data.BsmtFinType1.replace({1 : 1, 2 : 1, 3 : 1, 4 : 2, 5 : 2, 6 : 2 })
 all_data["SimplBsmtFinType2"] = all_data.BsmtFinType2.replace({1 : 1, 2 : 1, 3 : 1, 4 : 2, 5 : 2, 6 : 2})
 all_data["SimplBsmtCond"] = all_data.BsmtCond.replace({1 : 1, 4 : 2, 5 : 2})
 all_data["SimplBsmtQual"] = all_data.BsmtQual.replace({1 : 1,4 : 2, 5 : 2})
 all_data["SimplExterCond"] = all_data.ExterCond.replace({1 : 1,4 : 2, 5 : 2})
 all_data["SimplExterQual"] = all_data.ExterQual.replace({1 : 1,4 : 2, 5 : 2})
 all_data["OverallGrade"] = all_data["OverallQual"] * all_data["OverallCond"]
all_data["GarageGrade"] = all_data["GarageQual"] * all_data["GarageCond"]
all_data["ExterGrade"] = all_data["ExterQual"] * all_data["ExterCond"]
all_data["KitchenScore"] = all_data["KitchenAbvGr"] * all_data["KitchenQual"]
all_data["FireplaceScore"] = all_data["Fireplaces"] * all_data["FireplaceQu"]
all_data["GarageScore"] = all_data["GarageArea"] * all_data["GarageQual"]
all_data["PoolScore"] = all_data["PoolArea"] * all_data["PoolQC"]
all_data["SimplOverallGrade"] = all_data["SimplOverallQual"] * all_data["SimplOverallCond"]
all_data["SimplExterGrade"] = all_data["SimplExterQual"] * all_data["SimplExterCond"]
all_data["SimplPoolScore"] = all_data["PoolArea"] * all_data["SimplPoolQC"]
all_data["SimplGarageScore"] = all_data["GarageArea"] * all_data["SimplGarageQual"]
all_data["SimplFireplaceScore"] = all_data["Fireplaces"] * all_data["SimplFireplaceQu"]
all_data["SimplKitchenScore"] = all_data["KitchenAbvGr"] * all_data["SimplKitchenQual"]
all_data["TotalBath"] = all_data["BsmtFullBath"] +(0.5 * all_data["BsmtHalfBath"])+ \
all_data["FullBath"] +(0.5 * all_data["HalfBath"])
 all_data["AllSF"] = all_data["GrLivArea"] + all_data["TotalBsmtSF"]
all_data["AllFlrsSF"] = all_data["1stFlrSF"] + all_data["2ndFlrSF"]
all_data["AllPorchSF"] = all_data["OpenPorchSF"] + all_data["EnclosedPorch"] + \
all_data["3SsnPorch"] + all_data["ScreenPorch"]
all_data["HasMasVnr"] = all_data.MasVnrType.replace({"BrkCmn" : 1, "BrkFace" : 1, "CBlock" : 1, "Stone" : 1, "None" : 0})
 all_data["BoughtOffPlan"] = all_data.SaleCondition.replace({"Abnorml" : 0, "Alloca" : 0, "AdjLand" : 0, "Family" : 0, "Normal" : 0, "Partial" : 1})
 all_data["OverallQual-s2"] = all_data["OverallQual"] ** 2
all_data["OverallQual-s3"] = all_data["OverallQual"] ** 3
all_data["OverallQual-Sq"] = np.sqrt(all_data["OverallQual"])
 all_data["AllSF-2"] = all_data["AllSF"] ** 2
all_data["AllSF-3"] = all_data["AllSF"] ** 3
all_data["AllSF-Sq"] = np.sqrt(all_data["AllSF"])
 all_data["AllFlrsSF-2"] = all_data["AllFlrsSF"] ** 2
all_data["AllFlrsSF-3"] = all_data["AllFlrsSF"] ** 3
all_data["AllFlrsSF-Sq"] = np.sqrt(all_data["AllFlrsSF"])
 all_data["GrLivArea-2"] = all_data["GrLivArea"] ** 2
all_data["GrLivArea-3"] = all_data["GrLivArea"] ** 3
all_data["GrLivArea-Sq"] = np.sqrt(all_data["GrLivArea"])
 all_data["SimplOverallQual-s2"] = all_data["SimplOverallQual"] ** 2
all_data["SimplOverallQual-s3"] = all_data["SimplOverallQual"] ** 3
all_data["SimplOverallQual-Sq"] = np.sqrt(all_data["SimplOverallQual"])
 all_data["ExterQual-2"] = all_data["ExterQual"] ** 2
all_data["ExterQual-3"] = all_data["ExterQual"] ** 3
all_data["ExterQual-Sq"] = np.sqrt(all_data["ExterQual"])
 all_data["GarageCars-2"] = all_data["GarageCars"] ** 2
all_data["GarageCars-3"] = all_data["GarageCars"] ** 3
all_data["GarageCars-Sq"] = np.sqrt(all_data["GarageCars"])
 all_data["TotalBath-2"] = all_data["TotalBath"] ** 2
all_data["TotalBath-3"] = all_data["TotalBath"] ** 3
all_data["TotalBath-Sq"] = np.sqrt(all_data["TotalBath"])
 all_data["KitchenQual-2"] = all_data["KitchenQual"] ** 2
all_data["KitchenQual-3"] = all_data["KitchenQual"] ** 3
all_data["KitchenQual-Sq"] = np.sqrt(all_data["KitchenQual"])
 all_data["GarageScore-2"] = all_data["GarageScore"] ** 2
all_data["GarageScore-3"] = all_data["GarageScore"] ** 3
all_data["GarageScore-Sq"] = np.sqrt(all_data["GarageScore"])
 all_data['TotalSF'] = all_data['TotalBsmtSF'] + all_data['1stFlrSF'] + all_data['2ndFlrSF']
categorical_features = all_data.select_dtypes(include = ["object"] ).columns
numerical_features = all_data.select_dtypes(exclude = ["object"] ).columns
train_num = all_data[numerical_features]
train_cat = all_data[categorical_features]
skewness = train_num.apply(lambda x: skew(x))
skewness = skewness[abs(skewness)> 0.75]
skewed_features = skewness.index
train_num[skewed_features] = boxcox1p(train_num[skewed_features], 0.15)
 train_cat = pd.get_dummies(train_cat)
 all_data = pd.concat([train_num, train_cat], axis = 1)
 train = all_data[:train.shape[0]]
test = all_data[train.shape[0]:]
scale = StandardScaler()
train.loc[:, numerical_features] = scale.fit_transform(train.loc[:, numerical_features])
 test.loc[:, numerical_features] = scale.transform(test.loc[:, numerical_features])
 drop_cols = ["Exterior1st_ImStucc", "Exterior1st_Stone", "Exterior2nd_Other","HouseStyle_2.5Fin",
"RoofMatl_Membran", "RoofMatl_Metal", "RoofMatl_Roll",
"Condition2_RRAe", "Condition2_RRAn", "Condition2_RRNn",
"Heating_Floor", "Heating_OthW", "MSSubClass_150",
"Electrical_Mix", "MiscFeature_TenC", "Condition2_PosN", "MSZoning_C(all)", "MSSubClass_160"
]
train.drop(drop_cols, axis=1, inplace=True)
 test.drop(drop_cols, axis=1, inplace=True)
 print(train.shape, test.shape )<train_on_grid>
pred_test_y =(pred_test_y >threshold ).astype(int)
 test_df = pd.read_csv(".. /input/test.csv", usecols=["qid"])
 out_df = pd.DataFrame({"qid":test_df["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False )