Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<feature_engineering>
import numpy as np
import pandas as pd
import os
from tqdm import tqdm, trange
import string
import re
import sys
import math
from keras.models import Sequential
from keras.layers import CuDNNLSTM, Dense, Bidirectional, Dropout, BatchNormalization, Activation
import random
from keras.preprocessing.sequence import pad_sequences
from time import time
from sklearn import metrics
from keras.optimizers import Adam
from keras.regularizers import l1_l2
train_data['PoolStatus'] = train_data['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 train_data['SeFlrStatus'] = train_data['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 train_data['GarageStatus'] = train_data['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 train_data['BsmtStatus'] = train_data['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 train_data['FirePlaceStatus'] = train_data['Fireplaces'].apply(lambda x: 1 if x > 0 else 0)
 test_data['PoolStatus'] = test_data['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 test_data['SeFlrStatus'] = test_data['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 test_data['GarageStatus'] = test_data['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 test_data['BsmtStatus'] = test_data['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 test_data['FirePlaceStatus'] = test_data['Fireplaces'].apply(lambda x: 1 if x > 0 else 0)
 train_data['RemodNBuild']=train_data['YearBuilt']+train_data['YearRemodAdd']
train_data['SF']=train_data['TotalBsmtSF'] + train_data['1stFlrSF'] + train_data['2ndFlrSF']
train_data['AreaInFt'] =(train_data['BsmtFinSF1'] + train_data['BsmtFinSF2'] +
train_data['1stFlrSF'] + train_data['2ndFlrSF'])
 train_data['NumBathroom'] =(train_data['FullBath'] +(0.5 * train_data['HalfBath'])+
train_data['BsmtFullBath'] +(0.5 * train_data['BsmtHalfBath']))
train_data['Total_porch_sf'] =(train_data['OpenPorchSF'] + train_data['3SsnPorch'] +
train_data['EnclosedPorch'] + train_data['ScreenPorch'] +
train_data['WoodDeckSF'])
 test_data['RemodNBuild']=test_data['YearBuilt']+test_data['YearRemodAdd']
test_data['SF']=test_data['TotalBsmtSF'] + test_data['1stFlrSF'] + test_data['2ndFlrSF']
test_data['AreaInFt'] =(test_data['BsmtFinSF1'] + test_data['BsmtFinSF2'] +
test_data['1stFlrSF'] + test_data['2ndFlrSF'])
 test_data['NumBathroom'] =(test_data['FullBath'] +(0.5 * test_data['HalfBath'])+
test_data['BsmtFullBath'] +(0.5 * test_data['BsmtHalfBath']))
test_data['Total_porch_sf'] =(test_data['OpenPorchSF'] + test_data['3SsnPorch'] +
test_data['EnclosedPorch'] + test_data['ScreenPorch'] +
test_data['WoodDeckSF'] )<drop_column>
df=pd.read_csv('.. /input/train.csv')
 df=df[['question_text','target']]
df=df.dropna()
ID = test_data['Id']
del test_data['Id']<drop_column>
pos_pct=df['target'].value_counts() [1]/len(df)
 print(f'Caustic questions percentage is {pos_pct*100:.2f}%' )
del train_data['Id']<import_modules>
trans_table={key:' ' for key in string.punctuation}
def get_words(s):

 s=s.translate(str.maketrans(trans_table)).lower().strip()
s=re.sub(' +',' ',s)
 s=s.split(' ')
 return len(s )
from sklearn_pandas import DataFrameMapper, CategoricalImputer
from sklearn.pipeline import Pipeline, FeatureUnion
import xgboost as xgb
from sklearn.preprocessing import Imputer, FunctionTransformer, MinMaxScaler
from sklearn.impute import SimpleImputer
from sklearn.model_selection import cross_val_score
from sklearn.feature_extraction import DictVectorizer<count_values>
df['n_words']=df['question_text'].apply(get_words )
quasi_constant = []
for i in train_data.columns:
counts = train_data[i].value_counts()
zeros = counts.iloc[0]
if zeros / len(train_data)* 100 > 99.94:
quasi_constant.append(i)
 train_data = train_data.drop(quasi_constant, axis=1)
 test_data = test_data.drop(quasi_constant, axis=1 )<define_variables>
df=df[df['n_words']<=60]
list_of_numerical = []
list_of_categorical = []
for i in train_data.columns:
if train_data[i].dtypes == 'object':
list_of_categorical.append(i)
 else:
if i == 'SalePrice':
pass
else:
list_of_numerical.append(i )<categorify>
embedding_path='glove.840B.300d/glove.840B.300d.txt'
vocab={}
with open('.. /input/embeddings/' + embedding_path,'r')as f:
for line_number,line in enumerate(tqdm(f)) :
key,values=line.split(' ')[0],line.split(' ')[1:]
if not any(char in string.punctuation for char in key):
vocab[key]=np.asarray(values,dtype='float32')
 print(f'Total of {len(vocab.keys())} words in vocabulary' )
transformers = []
transformers.extend([([num_feat],SimpleImputer(strategy='constant',fill_value=-1)) for num_feat in list_of_numerical])
 transformers.extend([(cat_feat,CategoricalImputer())for cat_feat in list_of_categorical])
 combined_pipeline = DataFrameMapper(transformers,
input_df=True,
df_out=True )<categorify>
trans_table={key:' ' for key in string.punctuation}
unknown_word=np.zeros(300)
 def str_prep(s):
s=s.translate(str.maketrans(trans_table)).lower().strip()
s=re.sub(' +',' ',s)
 s=s.split(' ')
 ar=np.asarray([vocab.get(x,unknown_word)for x in s],dtype='float32')
 return ar
def return_dict(blob):
return blob.to_dict("records" )<split>
def batch_gen(df,min_batch_size=1024,transformation_func=str_prep, training_mode=True):
n_batches=math.ceil(len(df)/min_batch_size)
 while True:
df=df.sample(frac=1 ).reset_index(drop=True)
 for batch in range(n_batches):
start=batch*min_batch_size
end=start+min_batch_size
if batch==n_batches-1:
X=df['question_text'][start:].apply(transformation_func)
 X=pad_sequences(X,maxlen=60,dtype='float32',padding='post')
 y=np.array(df['target'][start:])
 else:
X=df['question_text'][start:end].apply(transformation_func)
 X=pad_sequences(X,maxlen=60,dtype='float32',padding='post')
 y=np.array(df['target'][start:end])
 yield X, y
X_train, X_test, y_train, y_test = train_test_split(
 train_data.drop(['SalePrice'],axis=1), train_data['SalePrice'], test_size=0.15, random_state=42 )<choose_model_class>
def train_val_split(df,frac=0.2):
df=df.sample(frac=1 ).reset_index(drop=True)
 df_sincere=df[df['target']==0]
df_taxic=df[df['target']==1]
sincere_border=int(len(df_sincere)*(1-frac))
taxic_border=int(len(df_taxic)*(1-frac))
df_train=pd.concat([df_sincere[:sincere_border],df_taxic[:taxic_border]])
 df_val=pd.concat([df_sincere[sincere_border:],df_taxic[taxic_border:]])
 print('Training data peppered(prepared)!')
 return df_train.sample(frac=1 ).reset_index(drop=True), df_val.sample(frac=1 ).reset_index(drop=True )
pipeline1 = Pipeline([ 
('featureunion',combined_pipeline), 
('dictifier',FunctionTransformer(func=return_dict,validate=False)) , 
('vectorizer',DictVectorizer(sort=False,sparse=False)) , 
('reg',xgb.XGBRegressor(objective="reg:squarederror"))
] )<compute_train_metric>
def train_model(train_df,val_df,n_epochs=8,batch_size=1024):
model = Sequential()
model.add(Bidirectional(CuDNNLSTM(64, return_sequences=True),
input_shape=(60, 300)))
 model.add(Bidirectional(CuDNNLSTM(32)))
 model.add(Dense(64))
model.add(Activation('relu'))
model.add(Dense(32))
model.add(Activation('relu'))
model.add(Dense(16))
model.add(Activation('relu'))
model.add(Dense(1,activation='sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
 model.fit_generator(batch_gen(train_df, min_batch_size=batch_size), epochs=n_epochs,
steps_per_epoch=math.ceil(len(train_df)/batch_size),
validation_data=batch_gen(val_df, min_batch_size=batch_size),
validation_steps=math.ceil(len(val_df)/batch_size),
verbose=True)
 return model
mse1 = cross_val_score(pipeline1, X_train, y_train, scoring="neg_mean_squared_log_error", cv=5 )<compute_test_metric>
def get_predictions(df, model, min_batch_size=1024,transformation_func=str_prep):
predictions=np.ndarray(shape=(0,1))
print(f'Total values to predict: {len(df)}')
 n_batches=math.ceil(len(df)/min_batch_size)
 for batch in trange(n_batches):
start=batch*min_batch_size
end=start+min_batch_size
if batch==n_batches-1:
X=df['question_text'][start:].apply(transformation_func)
 X=pad_sequences(X,maxlen=60,dtype='float32',padding='post')
 y_predicted=model.predict(X)
 else:
X=df['question_text'][start:end].apply(transformation_func)
 X=pad_sequences(X,maxlen=60,dtype='float32',padding='post')
 y_predicted=model.predict(X)
 predictions=np.append(predictions,y_predicted,axis=0)
 return predictions
print(np.sqrt(-mse1))<import_modules>
def select_best_threshold(y_true, y_predicted):
thresholds = []
for thresh in np.arange(0.0, 1, 0.001):
thresh = np.round(thresh, 3)
 res=metrics.f1_score(y_true,(y_predicted>thresh ).astype(int))
thresholds.append([thresh, res])
 thresholds.sort(key=lambda x: x[1], reverse=True)
 best_thresh = thresholds[0][0]
print("Best threshold: ", best_thresh)
 return best_thresh
from sklearn.model_selection import RandomizedSearchCV<define_search_space>
stack_size=3
model_stack={}
for ind in range(stack_size):
print(f'
=== Model number {ind+1} ===
')
 train_df, val_df=train_val_split(df, frac=0.1)
 model=train_model(train_df,val_df, n_epochs=5)
 y_predicted=get_predictions(val_df,model)
 y_true=val_df['target'].values
best_thresh=select_best_threshold(y_true, y_predicted)
 model_stack[ind]=(model, best_thresh)
 print(f'Model number {ind+1} finished!' )
params = {
'reg__max_depth' : [5,10,15,20],
'reg__gamma' : [.1,.2,.3,.4,.5,.6],
'reg__colsample_bytree' : [.1,.2,.3,.4,.5,.6,.7],
'reg__n_estimators' : [50,100,200]
}<choose_model_class>
test_set=pd.read_csv('.. /input/test.csv' )
grid_pipe_xgboost = RandomizedSearchCV(estimator=pipeline1,param_distributions=params,cv=4,
scoring="neg_mean_squared_log_error",n_iter=15 )<train_model>
predictions={}
for key,(model, threshold)in model_stack.items() :
predictions[key]=(get_predictions(test_set,model)>threshold ).astype(int )
%%time
grid_pipe_xgboost.fit(X_train,y_train )<find_best_params>
preds=np.stack([value for value in predictions.values() ],axis=1 ).reshape(-1,stack_size)
 preds=np.average(preds,axis=1)
 preds=np.around(preds,decimals=0 ).astype(int )
<find_best_score><EOS> 
test_set['prediction']=preds
to_submit=test_set[['qid','prediction']]
to_submit.to_csv("submission.csv", index=False)
 print('Submissions saved to file!' )