Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<filter>
import numpy as np
import pandas as pd
import matplotlib as mp
import matplotlib.pyplot as plt
import os
from sklearn.metrics import confusion_matrix,f1_score
from sklearn.model_selection import GridSearchCV,RandomizedSearchCV,cross_validate,train_test_split,StratifiedKFold
from datetime import datetime
def basmentbath(df):
No_BsmtFullBath=df[(df['BsmtFullBath']==0)&(df['BsmtHalfBath']==0)].index
Only_halfBath=df[(df['BsmtFullBath']==0)&(df['BsmtHalfBath']==1)].index
Two_halfBath =df[(df['BsmtFullBath']==0)&(df['BsmtHalfBath']==2)].index
Only_FullBath=df[(df['BsmtFullBath']==1)&(df['BsmtHalfBath']==0)].index
One_Full_Half = df[(df['BsmtFullBath']==1)&(df['BsmtHalfBath']==1)].index
Two_Full = df[(df['BsmtFullBath']==2)&(df['BsmtHalfBath']==0)].index
Three_Full = df[(df['BsmtFullBath']==3)&(df['BsmtHalfBath']==0)].index
df['BsmtBathCat'] = 0
df.loc[No_BsmtFullBath, 'BsmtBathCat'] = 0
df.loc[Only_halfBath, 'BsmtBathCat'] = 1
df.loc[Two_halfBath, 'BsmtBathCat'] = 2
df.loc[Only_FullBath, 'BsmtBathCat'] = 3
df.loc[One_Full_Half, 'BsmtBathCat'] = 4
df.loc[Two_Full, 'BsmtBathCat'] = 5
df.loc[Three_Full, 'BsmtBathCat'] = 6
df.drop(['BsmtFullBath','BsmtHalfBath'],axis= 1 , inplace=True )<feature_engineering>
from keras.models import Sequential
from keras.layers import *
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.models import Model
from keras.wrappers.scikit_learn import KerasClassifier
integer['2ndFloor']=integer['2ndFlrSF'].apply(lambda x: 0 if x==0 else 1)
 tinteger['2ndFloor']=tinteger['2ndFlrSF'].apply(lambda x: 0 if x==0 else 1)
 integer.drop(['1stFlrSF','2ndFlrSF'],axis =1, inplace = True)
 tinteger.drop(['1stFlrSF','2ndFlrSF'],axis =1, inplace = True )<feature_engineering>
train=pd.read_csv(".. /input/train.csv",sep=',')
 test=pd.read_csv(".. /input/test.csv",sep=',' )
tinteger= tinteger.join(train['Neighborhood'])
 lot_mean = tinteger.groupby("Neighborhood")["LotFrontage"].mean().round().to_dict()
tinteger.loc[(tinteger['LotFrontage']==0)&(tinteger['Neighborhood'].isin(lot_mean.keys())) , 'LotFrontage'] = tinteger['Neighborhood'].map(lot_mean)
 tinteger.drop('Neighborhood',axis=1 , inplace=True)
 integer= integer.join(data['Neighborhood'])
 lot_mean = integer.groupby("Neighborhood")["LotFrontage"].mean().round().to_dict()
integer.loc[(integer['LotFrontage']==0)&(integer['Neighborhood'].isin(lot_mean.keys())) , 'LotFrontage'] = integer['Neighborhood'].map(lot_mean)
 integer.drop('Neighborhood',axis=1 , inplace=True )<drop_column>
vocabulary_size=50000
vec_size=70
emb_size=300
integer.drop(['LowQualFinSF','MiscVal'],axis =1, inplace = True)
 tinteger.drop(['LowQualFinSF','MiscVal'],axis =1, inplace = True )<drop_column>
tokenizer = Tokenizer(num_words=vocabulary_size)
 tokenizer.fit_on_texts(train.question_text)
 sequences = tokenizer.texts_to_sequences(train.question_text )
integer.drop(['MoSold','YrSold','KitchenAbvGr'],axis =1, inplace = True)
 tinteger.drop(['MoSold','YrSold','KitchenAbvGr'],axis =1, inplace = True )<define_variables>
tokenizer.word_index.get('trump' )
categorical = data[['KitchenQual','FireplaceQu','GarageQual','GarageFinish','GarageCond','BsmtCond','BsmtExposure','BsmtQual','BsmtFinType2',
'BsmtFinType1','ExterCond','ExterQual','HeatingQC','LandSlope','LotShape','PavedDrive','Street',
'BldgType','Functional','SaleType','GarageType','Electrical','Foundation','LandContour','RoofMatl','RoofStyle']]
tcategorical = train[['SalePrice','KitchenQual','FireplaceQu','GarageQual','GarageFinish','GarageCond','BsmtCond','BsmtExposure','BsmtQual','BsmtFinType2',
'BsmtFinType1','ExterCond','ExterQual','HeatingQC','LandSlope','LotShape','PavedDrive','Street',
'BldgType','Functional','SaleType','GarageType','Electrical','Foundation','LandContour','RoofMatl','RoofStyle']] 
<categorify>
tokenizer.word_counts.get('trump' )
kit ={"Ex":5,"Gd":4,"TA":3,"Fa":2,"Po":1,"None":0}
fire ={"Ex":5,"Gd":4,"TA":3,"Fa":2,"Po":1,"None":0}
garagequa ={"Ex":5,"Gd":4,"TA":3,"Fa":2,"Po":1,"None":0}
garage_finish = {"Fin":3,"RFn":2,"Unf":1,"None":0}
garagecond= {"Ex":5,"Gd":4,"TA":3,"Fa":2,"Po":1,"None":0}
bsmtcond= {"Gd":4,"TA":3,"Fa":2,"Po":1,"None":0}
bsmtexposure = {"Gd":4,"Av":3,"Mn":2,"No":1,"None":0}
bsmtqual = {"Ex":4,"Gd":3,"TA":2,"Fa":1,"None":0}
bsmtype2={"GLQ":6,"ALQ":5,"BLQ":4,"Rec":3,"LwQ":2,"Unf":1,"None":0}
bsmtype1 = {"GLQ":6,"ALQ":5,"BLQ":4,"Rec":3,"LwQ":2,"Unf":1,"None":0}
extercond = {"Ex":5,"Gd":4,"TA":3,"Fa":2,"Po":1}
exterqual = {"Ex":5,"Gd":4,"TA":3,"Fa":2,"Po":1}
heatingqc = {"Ex":5,"Gd":4,"TA":3,"Fa":2,"Po":1}
landslope={"Gtl":3,"Mod":2,"Sev":1}
lotshape={"Reg":4,"IR1":3,"IR2":2,"IR3":1}
pavedrive= {"Y":3,"P":2,"N":1}
street={"Grvl":2,"Pave":1}
bldgtype = {'1Fam': 1, '2fmCon': 2, 'Duplex': 3, 'TwnhsE': 4, 'Twnhs': 5}
functional = {'Typ': 8,'Min1': 7,'Min2': 6,'Mod': 5,'Maj1': 4,'Maj2': 3,'Sev': 2,'Sal': 1,'None':1}
saletype ={'WD': 10,'CWD': 9,'VWD': 8,'New': 7,'COD': 6,'Con': 5,'ConLw': 4,'ConLI': 3,'ConLD': 2,'Oth': 1,'None':0}
garage={'2Types': 6, 'Attchd': 5, 'Basment': 4, 'BuiltIn': 3, 'CarPort': 2, 'Detchd': 1, 'None': 0}
electrical={'SBrkr': 5, 'FuseA': 4, 'FuseF': 3, 'FuseP': 2, 'Mix': 1,'None':0}
foundation={'BrkTil': 6, 'CBlock': 5, 'PConc': 4, 'Slab': 3, 'Stone': 2, 'Wood': 1}
landcontour={'Lvl': 4, 'Bnk': 3, 'HLS': 2, 'Low': 1}
roofmatl={'ClyTile': 8, 'CompShg': 7, 'Membran': 6, 'Metal': 5, 'Roll': 4, 'Tar&Grv': 3, 'WdShake': 2, 'WdShngl': 1}
roofstyle ={'Flat': 6, 'Gable': 5, 'Gambrel': 4, 'Hip': 3, 'Mansard': 2, 'Shed': 1}<categorify>
X = pad_sequences(sequences, maxlen=vec_size )
for i in tcategorical:
tcategorical[i].fillna('None',inplace=True)
 for i in categorical:
categorical[i].fillna('None',inplace=True )<categorify>
def get_emb_index(filepath):
embeddings_index = dict()
f = open(filepath,errors='ignore')
 for line in f:
if len(line)>100 :
values = line.split(" ")
 word = values[0]
coefs = np.asarray(values[1:], dtype='float32')
 embeddings_index[word] = coefs
f.close()
return embeddings_index
list_dict =[kit,fire,garagequa,garage_finish,garagecond,bsmtcond,bsmtexposure,bsmtqual,bsmtype2,
bsmtype1,extercond,exterqual,heatingqc,landslope,lotshape,pavedrive,street,
bldgtype,functional,saletype,garage,electrical,foundation,landcontour,roofmatl,roofstyle]
def replace_t(i,diccionaty):
tcategorical[i].replace(diccionaty, inplace=True)
 def replace(i,diccionaty):
categorical[i].replace(diccionaty, inplace=True)
 for i,j in zip(tcategorical.columns[1:].to_list() ,list_dict):
replace_t(i,j)
 for i,j in zip(categorical.columns.to_list() ,list_dict):
replace(i,j)
 for i in categorical:
v= categorical[i].unique()
print(i, v )<count_values>
def get_emb_matrix(embeddings_index):
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
emb_mean,emb_std
embedding_matrix = np.random.normal(emb_mean, emb_std,(vocabulary_size, emb_size))
for word, index in tokenizer.word_index.items() :
if index > vocabulary_size - 1:
break
else:
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[index] = embedding_vector
embedding_matrix=(embedding_matrix-emb_mean)/emb_std
return embedding_matrix
for i in tcategorical:
v = tcategorical[i].value_counts().to_frame().reset_index().iloc[0].to_list()
w= tcategorical[i].value_counts().reset_index().iloc[0][0]
if w == 0:
print(i,v )<drop_column>
%%time
glove_emb_index=get_emb_index('.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt')
 glove_emb_matrix= get_emb_matrix(glove_emb_index )
tcategorical.drop('FireplaceQu',axis=1, inplace=True)
 categorical.drop('FireplaceQu',axis=1, inplace=True )<categorify>
%%time
wiki_emb_index=get_emb_index('.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec')
 wiki_emb_matrix= get_emb_matrix(wiki_emb_index )
for i in nominal:
encoding = pd.get_dummies(nominal[i], prefix=i, drop_first= True)
 nominal= pd.concat([nominal,encoding],axis=1)
 nominal.drop(i,axis=1, inplace=True)
 nominal<drop_column>
%%time
paragram_emb_index=get_emb_index('.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt')
 paragram_emb_matrix= get_emb_matrix(paragram_emb_index )
all_features = nominal.keys()
nominal = nominal.drop(nominal.loc[:,(nominal==0 ).sum() >=(nominal.shape[0]*0.9994)],axis=1)
 nominal = nominal.drop(nominal.loc[:,(nominal==1 ).sum() >=(nominal.shape[0]*0.9994)],axis=1)
 remain_features = nominal.keys()
remov_features = [st for st in all_features if st not in remain_features]
print(len(remov_features), 'features were removed:', remov_features )<concatenate>
pd.DataFrame({'glove':glove_emb_index.get('trump'),'paragram' : paragram_emb_index.get('trump'),'wikiNews':wiki_emb_index.get('trump')} ).T
new_data = pd.concat([integer,categorical,nominal], axis = 1 , sort=False)
 new_data<split>
avg_emb_matrix =(glove_emb_matrix+paragram_emb_matrix+wiki_emb_matrix)/3
train_data =new_data.iloc[:1460].reset_index(drop=True)
 test_data = new_data.iloc[1460:].reset_index(drop=True )<save_to_csv>
%%time
def get_model_gru(dropout=0.2,gru_units=64,dense_units=16,kernel_initializer='he_normal', embedding='avg'):
inp = Input(shape=(vec_size,))
if embedding =='avg':
x = Embedding(vocabulary_size, emb_size, weights=[avg_emb_matrix] )(inp)
 elif embedding=='glove' :
x = Embedding(vocabulary_size, emb_size, weights=[glove_emb_matrix] )(inp)
 elif embedding=='paragram':
x = Embedding(vocabulary_size, emb_size, weights=[paragram_emb_matrix] )(inp)
 elif embedding =='wiki':
x = Embedding(vocabulary_size, emb_size, weights=[wiki_emb_matrix] )(inp)
 x = SpatialDropout1D(dropout )(x)
 x = Bidirectional(CuDNNGRU(gru_units, return_sequences=True))(x)
 x = GlobalMaxPool1D()(x)
 x = Dense(dense_units,kernel_initializer=kernel_initializer, activation="relu" )(x)
 x = Dropout(dropout )(x)
 x = Dense(1, kernel_initializer=kernel_initializer,activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model
X = train_data
y = train['SalePrice']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)
 LR = LinearRegression().fit(X_train,y_train)
 y_pred_LR = LR.predict(test_data)
 test_Id=test['Id']
my_submission = pd.DataFrame({'Id': test_Id, 'SalePrice': np.expm1(y_pred_LR)})
 my_submission.to_csv('KaggelsubmissionLR.csv', index=False )<split>
param = {
'dropout': [0.1,0.2,0.5],
'gru_units': [16, 32, 64],
'dense_units': [16, 32, 64],
'kernel_initializer': ['he_normal','uniform'],
'embedding': ['avg','glove','paragram','wiki']
}
train_data = train_data.astype(int)
 test_data = test_data.astype(int)
 X = train_data
y = train['SalePrice']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42 )<choose_model_class>
model = KerasClassifier(build_fn=get_model_gru, epochs=3, batch_size=2000,verbose=False)
 rnn_model=RandomizedSearchCV(model,n_iter=10, param_distributions=param,cv=3,scoring='f1',return_train_score=True )
classifier = xgboost.XGBRegressor(base_score=0.25, booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=1, gamma=0,
importance_type='gain', learning_rate=0.05, max_delta_step=0,
max_depth=2, min_child_weight=4, missing=None, n_estimators=900,
n_jobs=1, nthread=None, objective='reg:linear', random_state=0,
reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,
silent=None, subsample=1, verbosity=0)
 classifier = classifier.fit(X_train,y_train)
 y_pred_XG = classifier.predict(test_data)
 sub = pd.DataFrame()
sub = pd.DataFrame()
sub['Id'] = test['Id']
sub['SalePrice'] = np.expm1(y_pred_XG)
 sub.to_csv('KaggelXBoostsubmission.csv',index=False )<load_from_csv>
%%time
rnn_model.fit(X,train.target )
pd.read_csv("/kaggle/input/titanic/train.csv" )<load_from_csv>
pred_train=rnn_model.predict(X )
train_data = pd.read_csv("/kaggle/input/titanic/train.csv")
 test_data = pd.read_csv("/kaggle/input/titanic/test.csv")
 train_data.head()<feature_engineering>
to=0.50
print('F1 score :%s
'% f1_score(train.target,pred_train>to))
print('Confusion matrix
%s'%confusion_matrix(train.target,pred_train))
train_data['Log_Fare']=np.log(train_data['Fare']+1)
 diagnostic_plots(train_data,'Log_Fare')
<feature_engineering>
train[train.target!=pred_train[:,0]]
train_data['Rec'] = 1/(train_data['Fare']+1 )<feature_engineering>
sequences_t = tokenizer.texts_to_sequences(test.question_text )
train_data['Fare'] = np.log(train_data['Fare']+1 )<categorify>
X_t= pad_sequences(sequences_t, maxlen=vec_size )
<feature_engineering><EOS> 
result=pd.read_csv(".. /input/sample_submission.csv",sep=',')
 pred_res=rnn_model.predict(X_t)>to
result['prediction']=pd.DataFrame(pred_res)
 result.to_csv('submission.csv',index=False )