Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<concatenate>
import os
import time
import numpy as np
import pandas as pd
from tqdm import tqdm
import math
from sklearn.model_selection import train_test_split
from sklearn import metrics
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Input, CuDNNLSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D
from keras.layers import Bidirectional, GlobalMaxPool1D
from keras.optimizers import Adam
from keras.models import Model
from keras import backend as K
from keras.engine.topology import Layer
from keras import initializers, regularizers, constraints, optimizers, layers
train=pd.concat([train,extratrain,extr,xx],axis=1)
 test=pd.concat([test,extratest,exte,xte],axis=1)
<prepare_x_and_y>
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape )
xtrain=train.iloc[:,1:].values
xtest=test.values
ytrain=train.iloc[:,0].values 
<normalization>
train_df, val_df = train_test_split(train_df, test_size=0.1, random_state=2018)
 embed_size = 300
max_features = 60000
maxlen = 120
train_X = train_df["question_text"].fillna("_na_" ).values
val_X = val_df["question_text"].fillna("_na_" ).values
test_X = test_df["question_text"].fillna("_na_" ).values
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 val_X = pad_sequences(val_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
val_y = val_df['target'].values
sc_x=StandardScaler()
xtrain=sc_x.fit_transform(xtrain)
 xtest=sc_x.transform(xtest)
<train_model>
np.random.seed(2018)
 trn_idx = np.random.permutation(len(train_X))
val_idx = np.random.permutation(len(val_X))
train_X = train_X[trn_idx]
val_X = val_X[val_idx]
train_y = train_y[trn_idx]
val_y = val_y[val_idx]
regressor=LogisticRegression(C=1,solver='saga')
 regressor.fit(xtrain,ytrain )<train_model>
regressor2=SVC(C=1,gamma=0.01,kernel='rbf')
 regressor2.fit(xtrain,ytrain )<train_model>
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
regressor3=RandomForestClassifier(criterion='entropy',n_estimators=500)
 regressor3.fit(xtrain,ytrain )<compute_test_metric>
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = Dense(64, activation="relu" )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-3), metrics=['accuracy'] )
regressor3.score(xtrain,ytrain )<predict_on_test>
model.fit(train_X, train_y, batch_size=512, epochs=3, validation_data=(val_X, val_y))
test['Survived']=regressor.predict(xtest )<define_search_space>
pred_glove_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_glove_val_y>thresh ).astype(int))))
param_svm=[
{
'kernel':['rbf'],
'C':[0.1,0.01,1,5,10],
'gamma':[1,0.1,0.01]
},
{
'kernel':['linear'],
'C':[0.1,0.01,1,5,10],
'gamma':[1,0.1,0.01,5,10]
},
{
'kernel':['sigmoid'],
'C':[0.1,0.01,1,5,10],
'gamma':[1,0.1,0.01]
}
]
param_rf=[
{ 'n_estimators':[10,100,300,600,500], 'criterion':['gini'],'max_depth':[2,5,10,20,50,70,100,150],'max_leaf_nodes':[2,5,10,20,50]},
{ 'n_estimators':[10,100,300,600,500], 'criterion':['entropy'],'max_depth':[2,5,10,20,50,70,100,150],'max_leaf_nodes':[2,5,10,20,50]}
]
param_logistic=[
{'solver':['newton-cg'],
'C':[10,0.1,0.01,1]
},
{'solver':['lbfgs'],
'C':[10,0.1,0.01,1]
},
{'solver':['liblinear'],
'C':[10,0.1,0.01]
},
{'solver':['sag'],'C':[10,0.1,0.01,1]
}
]
gc_svm=GridSearchCV(regressor2,param_grid=param_svm,scoring='accuracy',cv=10 ).fit(xtrain,ytrain)
 gc_logistic=GridSearchCV(regressor,param_grid=param_logistic,scoring='accuracy',cv=10 ).fit(xtrain,ytrain)
 gc_rf=GridSearchCV(regressor3,param_grid=param_rf,scoring='accuracy',cv=10 ).fit(xtrain,ytrain)
 regressor3=gc_rf.best_estimator_
regressor3.fit(xtrain,ytrain)
<find_best_params>
pred_glove_test_y = model.predict([test_X], batch_size=1024, verbose=1 )
print("SVM")
 print(gc_svm.best_params_)
 print('-----------------------------------------------------------------------------------')
 print('logistic regression')
 print(gc_logistic.best_params_)
 print('-----------------------------------------------------------------------------------')
 print('random forest')
 print(gc_rf.best_params_ )<find_best_score>
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )
print("SVM")
 print(gc_svm.best_score_)
 print('-----------------------------------------------------------------------------------')
 print('logistic regression')
 print(gc_logistic.best_score_)
 print('-----------------------------------------------------------------------------------')
 print('random forest')
 print(gc_rf.best_score_ )<predict_on_test>
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = Dense(64, activation="relu" )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-3), metrics=['accuracy'] )
output_train=pd.DataFrame([regressor.predict(xtrain),regressor2.predict(xtrain)] ).apply(lambda x: x.mode() ).iloc[0].values
output=pd.DataFrame([regressor.predict(xtest),regressor2.predict(xtest)] ).apply(lambda x: x.mode() )<compute_test_metric>
model.fit(train_X, train_y, batch_size=512, epochs=3, validation_data=(val_X, val_y))
print(classification_report(ytrain,output_train)) 
<load_from_csv>
pred_fasttext_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_fasttext_val_y>thresh ).astype(int))))
submission=pd.read_csv('.. /input/gender_submission.csv')
 submission['Survived']=output.iloc[0].values 
<save_to_csv>
pred_fasttext_test_y = model.predict([test_X], batch_size=1024, verbose=1 )
submission.to_csv('finaloutput.csv',index=False )<import_modules>
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )
%matplotlib inline
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename)) 
<load_from_csv>
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = Dense(64, activation="relu" )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-3), metrics=['accuracy'] )
sns.set(style="darkgrid")
 train_full = pd.read_csv('.. /input/titanic/train.csv', index_col='PassengerId')
 test_full = pd.read_csv('.. /input/titanic/test.csv', index_col='PassengerId')
 train_NA = train_full.isna().sum()
test_NA = test_full.isna().sum()
pd.concat([train_NA, test_NA], axis=1, sort = False, keys=['Train NA', 'Test NA'] ).transpose()<feature_engineering>
model.fit(train_X, train_y, batch_size=512, epochs=3, validation_data=(val_X, val_y))
null_fare = test_full[test_full.Fare.isnull() ].index[0]
test_full.loc[null_fare, 'Fare'] = 0<prepare_x_and_y>
pred_paragram_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_paragram_val_y>thresh ).astype(int))))
y = train_full.Survived
train_X_full = train_full[train_full.columns.drop('Survived')]
combined = pd.concat([train_full, test_full], axis=0 )<categorify>
pred_paragram_test_y = model.predict([test_X], batch_size=1024, verbose=1 )
def alone(df):
if('SibSp' in df.columns)and('Parch' in df.columns):
df['Family'] = df['SibSp'] + df['Parch'] + 1
le = LabelEncoder()
df['le_Ticket'] = le.fit_transform(df['Ticket'])
 df['Same_Ticket'] = df.duplicated(['le_Ticket'])
 df['Alone'] = np.where(( df['Family'] > 1)|(df['Same_Ticket']), False, True)
 plt.figure(figsize=(20, 12))
sns.catplot(x="Alone", kind="count", palette="ch:.25", data=df)
 plt.title('Number of passengers travelling alone or not')
 df = df[df.columns.drop('SibSp', errors='ignore')]
df = df[df.columns.drop('Parch', errors='ignore')]
df = df[df.columns.drop('le_Ticket', errors='ignore')]
df = df[df.columns.drop('Same_Ticket', errors='ignore')]
return df<prepare_x_and_y>
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )
combined = alone(combined)
 train_X_full = combined[combined.Survived.notnull() ]
test_full = combined[combined.Survived.isnull() ]<categorify>
pred_val_y = 0.39*pred_glove_val_y + 0.41*pred_fasttext_val_y + 0.39*pred_paragram_val_y
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_val_y>thresh ).astype(int))))
def fix_cabin(df):
t = df.Cabin.fillna('U')
 df['Cabin'] = t.str.slice(0,1)
 plt.figure(figsize=(20, 15))
sns.catplot(x="Cabin", kind="count", palette="ch:.25", data=df)
 plt.title('Number of passengers per cabin')
<split>
pred_test_y = 0.35*pred_glove_test_y + 0.42*pred_fasttext_test_y + 0.35*pred_paragram_test_y
pred_test_y =(pred_test_y>0.42 ).astype(int)
 out_df = pd.DataFrame({"qid":test_df["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False )