Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<find_best_params>
eng_stopwords = set(stopwords.words("english"))
def hyperparameter_analysis(searcher, top_values=5):
tested_hyperparameters=pd.DataFrame()
for i in range(len(searcher.cv_results_['params'])) :
tested_hyperparameters = tested_hyperparameters.append(searcher.cv_results_['params'][i], ignore_index=True)
 tested_hyperparameters['train score in %']=(searcher.cv_results_['mean_train_score']*100 ).round(3)
 tested_hyperparameters['test score in %']=(searcher.cv_results_['mean_test_score']*100 ).round(3)
 tested_hyperparameters['test std in %']=(searcher.cv_results_['std_test_score']*100 ).round(3)
 tested_hyperparameters.index=searcher.cv_results_['rank_test_score']
return(tested_hyperparameters.sort_index().head(top_values))
def feature_importance(searcher_best_estimator, list_feature_names, top_values=5):
fi_norm = 1/abs(searcher_best_estimator.feature_importances_ ).sum() *abs(searcher_best_estimator.feature_importances_)
 fi_df= pd.DataFrame(data=(fi_norm*100 ).round(2), index=list_feature_names, columns=['feature importance in %'])
 return(fi_df.sort_values(['feature importance in %'], ascending=[False] ).head(top_values))
def feature_importance_ann(perm, list_feature_names, top_values=5, neg_values=False):
if neg_values==True:
fi_df= pd.DataFrame(data=(perm.feature_importances_*100 ).round(2), index=list_feature_names, columns=['score reduction in % points'])
 return(fi_df[fi_df['score reduction in % points']<0] ).sort_values(['score reduction in % points'], ascending=[True] ).head(top_values)
 else:
fi_df= pd.DataFrame(data=(perm.feature_importances_*100 ).round(2), index=list_feature_names, columns=['score reduction in % points'])
 return(fi_df.sort_values(['score reduction in % points'], ascending=[False] ).head(top_values))
def metric_scores(y_reference, y_prediction):
precision = precision_score(y_reference, y_prediction)
 recall = recall_score(y_reference, y_prediction)
 F1_score = f1_score(y_reference, y_prediction)
 print('precision: '+ str(precision.round(3)))
 print('recall: '+ str(recall.round(3)))
 print('F1_score: '+ str(F1_score.round(3)))
 return(precision, recall, F1_score)
 def plot_confusion_matrix(cm, classes, title='Confusion matrix', cmap=plt.cm.Blues):
plt.imshow(cm, interpolation='nearest', cmap=cmap)
 plt.title(title)
 tick_marks = np.arange(len(classes))
plt.xticks(tick_marks, classes, rotation=45)
 plt.yticks(tick_marks, classes)
 fmt = 'd'
thresh = cm.max() / 2.
 for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])) :
plt.text(j, i, format(cm[i, j], fmt),
horizontalalignment="center",
color="white" if cm[i, j] > thresh else "black")
 plt.ylabel('True label')
 plt.xlabel('Predicted label')
 plt.tight_layout()
def pred_to_df(y_predicted):
survived = pd.DataFrame(y_predicted,index=test.index, columns=['Survived'])
 survived['PassengerId'] = test.PassengerId
columns_titles = ["PassengerId","Survived"]
survived=survived.reindex(columns=columns_titles)
 return(survived)
 def save_to_csv(survived, filename,index=False):
survived.to_csv(filename + '.csv',index=False, header=survived.columns)
 def plot_roc_curve(fpr, tpr, label=None):
plt.plot(fpr, tpr, linewidth=2, label=label)
 plt.plot([0, 1], [0, 1], 'k--')
 plt.axis([0, 1, 0, 1])
 plt.xlabel('False Positive Rate', fontsize=16)
 plt.ylabel('True Positive Rate', fontsize=16)
<choose_model_class>
train_df = pd.read_csv(".. /input/train.csv")
 gc.collect()
def create_model(input_shape=X_train.shape[1:],
number_hidden=2,
neurons_per_hidden=10,
hidden_drop_rate= 0.2,
hidden_activation = 'selu',
hidden_initializer="lecun_normal",
output_activation ='sigmoid',
loss='binary_crossentropy',
optimizer = Nadam(lr=0.0005), 
):
model = Sequential()
model.add(Input(shape=input_shape)) ,
for layer in range(number_hidden):
model.add(Dense(neurons_per_hidden, activation = hidden_activation ,kernel_initializer=hidden_initializer))
model.add(Dense(1, activation = output_activation))
model.compile(loss=loss,
optimizer = Nadam(lr=0.0005),
metrics = ['accuracy'])
 return model<train_model>
test_df = pd.read_csv(".. /input/test.csv" )
keras.backend.clear_session()
np.random.seed(42)
 tf.random.set_seed(42)
 dnn_clf=create_model()
history = dnn_clf.fit(X_train, y_train, epochs=30, batch_size=30, verbose=0 )<train_model>
train_df["question_text"].isna().sum() , test_df["question_text"].isna().sum() ,
print('Training score: ' + str(( pd.DataFrame(history.history)['accuracy'].max() *100)) + '%' )<predict_on_test>
train_df, val_df = train_test_split(train_df, test_size=0.1, random_state = 1001 )
y_train_pred_dnn = dnn_clf.predict_classes(X_train )<compute_test_metric>
max_features = 95000
max_len = 72
embed_size = 300
metric_scores(y_train, y_train_pred_dnn )<predict_on_test>
%%time
embeddings_index = {}
f = open('.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt')
 for line in tqdm(f):
values = line.split(" ")
 word = values[0]
coefs = np.asarray(values[1:], dtype='float32')
 embeddings_index[word] = coefs
f.close()
gc.collect()
print('Found %s word vectors.' % len(embeddings_index))
y_train_proba_dnn = dnn_clf.predict(X_train )<create_dataframe>
def text_to_array(text, zeros = 300, split_val = max_len):
empyt_emb = np.zeros(zeros)
 text = text[:-1].split() [:split_val]
embeds = [embeddings_index.get(x, empyt_emb)for x in text]
embeds+= [empyt_emb] *(split_val - len(embeds))
return np.array(embeds )
proba_df=pd.DataFrame(y_train_proba_dnn, columns=['probabilities'])*100
proba_df['false_predictions']=(pd.DataFrame(y_train_pred_dnn)-pd.DataFrame(y_train)) [0]
proba_df['probabilities'][proba_df['false_predictions']==0].hist()<prepare_output>
val_vects = np.array([text_to_array(X_text)for X_text in tqdm(val_df["question_text"][:5000])])
 val_y = np.array(val_df["target"][:5000] )
check_df=data[:891].copy()
check_df['false_predictions']=proba_df['false_predictions']
check_df['probabilities']=proba_df['probabilities']
check_df[check_df['PassengerId']==18]<train_model>
batch_size = 256
def batch_gen(train_df):
n_batches = math.ceil(len(train_df)/ batch_size)
 while True:
train_df = train_df.sample(frac=1.)
for i in range(n_batches):
texts = train_df.iloc[i*batch_size:(i+1)*batch_size, 1]
text_arr = np.array([text_to_array(text)for text in texts])
 yield text_arr, np.array(train_df["target"][i*batch_size:(i+1)*batch_size])
np.random.seed(42)
 tf.random.set_seed(42)
 dnn_clf_fi=KerasClassifier(build_fn = create_model)
 dnn_clf_fi.fit(X_train, y_train, epochs=30, batch_size=30)
 perm = PermutationImportance(dnn_clf_fi, random_state=42 ).fit(X_train,y_train)
<compute_train_metric>
from keras.models import Sequential,Model
from keras.layers import CuDNNLSTM, Dense, Bidirectional, Input,Dropout, CuDNNGRU
from keras.engine.topology import Layer
from keras import initializers, regularizers, constraints
from keras.callbacks import Callback, EarlyStopping, ModelCheckpoint
from sklearn.metrics import f1_score
from keras.layers.normalization import BatchNormalization
from keras import backend as K
from keras.engine.topology import Layer, InputSpec
feature_importance_ann(perm, feature_names, top_values=10, neg_values=False )<compute_train_metric>
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('normal')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
feature_importance_ann(perm, feature_names, top_values=20, neg_values=True )<prepare_x_and_y>
class F1Evaluation(Callback):
def __init__(self, validation_data=() , interval=1):
super(Callback, self ).__init__()
self.interval = interval
self.X_val, self.y_val = validation_data
def on_epoch_end(self, epoch, logs={}):
if epoch % self.interval == 0:
y_pred = self.model.predict(self.X_val, verbose=0)
 y_pred =(y_pred > 0.5 ).astype(int)
 score = f1_score(self.y_val, y_pred)
 print("
F1 - Epoch: %d - Score: %.6f
" %(epoch+1, score))
k = 17
sel = SelectFromModel(perm, threshold=-np.inf, max_features=k, prefit=True)
 best_festures_index=np.where(sel.get_support() ==True)
 X_train_bf = data_post[:len(train['Survived']),best_festures_index[0]].copy()
X_test_bf = data_post[len(train['Survived']):,best_festures_index[0]].copy()<predict_on_test>
esr = EarlyStopping(verbose=2, patience=3 )
y_pred_dnn = dnn_clf.predict_classes(X_test )<prepare_output>
f1 = F1Evaluation(validation_data=(val_vects, val_y), interval=1 )
my_submission=pred_to_df(y_pred_dnn)
 my_submission.head()<save_to_csv>
inp = Input(shape=(max_len,300))
x = Bidirectional(CuDNNGRU(192, return_sequences=True))(inp)
 x = Bidirectional(CuDNNGRU(64,return_sequences=True))(x)
 x = Attention(max_len )(x)
 x = Dense(32, activation="relu" )(x)
 x = Dropout(0.25 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'] )
save_to_csv(my_submission,'submission' )<split>
model_name = 'gru_model'
print(model_name )
X_train_gs=X_train.copy()
X_test_gs=X_test.copy() 
<define_search_space>
gc.collect()
keras.backend.clear_session()
np.random.seed(42)
 tf.random.set_seed(42)
 dnn_clf_gs = KerasClassifier(build_fn = create_model, verbose = 0)
 param_grid = {
'input_shape': X_train_gs.shape[1:]
}
grid_search_dnn = GridSearchCV(dnn_clf_gs, param_grid, cv=5, n_jobs=-1, verbose=0, return_train_score=True)
 training_gs = grid_search_dnn.fit(X_train_gs, y_train,
epochs = 30,
batch_size = 30,
validation_split=0.2, shuffle=True, 
)
<compute_test_metric>
np.random.seed(2018)
 mg = batch_gen(train_df)
 hist = model.fit_generator(mg, epochs=30,
steps_per_epoch=512,
validation_data=(val_vects, val_y), callbacks=[f1, esr, model_checkpoint],
verbose=2 )
print('Mean training score: ' + str(( grid_search_dnn.cv_results_['mean_train_score'][grid_search_dnn.best_index_]*100 ).round(2)) +'%(' + str(( grid_search_dnn.cv_results_['std_train_score'][grid_search_dnn.best_index_]*100 ).round(2)) + '%)')
 print('Mean validation score: ' + str(( grid_search_dnn.cv_results_['mean_test_score'][grid_search_dnn.best_index_]*100 ).round(2)) +'%(' + str(( grid_search_dnn.cv_results_['std_test_score'][grid_search_dnn.best_index_]*100 ).round(2)) + '%)' )<train_on_grid>
model.load_weights(bst_model_path )
hyperparameter_analysis(grid_search_dnn, 10 )<predict_on_test>
import matplotlib.pyplot as plt
y_pred_dnn_opti = grid_search_dnn.best_estimator_.predict(X_test_gs )<prepare_output>
gc.collect()
survived_dnn_opti=pred_to_df(y_pred_dnn_opti)
 survived_dnn_opti.head()<save_to_csv>
batch_size = 256
def batch_gen(test_df):
n_batches = math.ceil(len(test_df)/ batch_size)
 for i in range(n_batches):
texts = test_df.iloc[i*batch_size:(i+1)*batch_size, 1]
text_arr = np.array([text_to_array(text)for text in texts])
 yield text_arr
all_preds = []
for x in tqdm(batch_gen(test_df)) :
all_preds.extend(model.predict(x ).flatten() )
save_to_csv(survived_dnn_opti,'survived_dnn_opti' )<train_on_grid>
%%time
val_preds = []
for x in batch_gen(val_df):
val_preds.extend(model.predict(x ).flatten() )
param_distribs = {
'n_estimators': randint(low=50, high=150),
'max_features': randint(low=5, high=15),
'min_samples_split': randint(low=10, high=30),
}
forest_clf = RandomForestClassifier(random_state=42)
 rnd_search_rf = RandomizedSearchCV(forest_clf, param_distributions=param_distribs, n_jobs=-1,
n_iter=50, cv=5, scoring="accuracy", random_state=42, return_train_score=True)
 rnd_search_rf.fit(X_train, y_train )<compute_test_metric>
gc.collect()
print('Best score: ' + str(( rnd_search_rf.best_score_*100 ).round(2)) + '%' )<compute_test_metric>
_thresh = []
for thresh in np.arange(0.1, 0.501, 0.01):
_thresh.append([thresh, f1_score(val_df["target"],(val_preds>thresh ).astype(int)) ])
 print("F1 score at threshold {0} is {1}".format(thresh, f1_score(val_df["target"],(val_preds>thresh ).astype(int))))
print('Mean training score: ' + str(( rnd_search_rf.cv_results_['mean_train_score'][rnd_search_rf.best_index_]*100 ).round(2)) +'%(' + str(( rnd_search_rf.cv_results_['std_train_score'][0]*100 ).round(2)) + '%)')
 print('Mean validation score: ' + str(( rnd_search_rf.cv_results_['mean_test_score'][rnd_search_rf.best_index_]*100 ).round(2)) +'%(' + str(( rnd_search_rf.cv_results_['std_test_score'][0]*100 ).round(2)) + '%)' )<create_dataframe>
_thresh = np.array(_thresh)
 best_id = _thresh[:,1].argmax()
best_thresh = _thresh[best_id][0]
best_thresh
<train_on_grid><EOS> 
y_te =(np.array(all_preds)> best_thresh ).astype(np.int)
 submit_df = pd.DataFrame({"qid": test_df["qid"], "prediction": y_te})
 submit_df.to_csv("submission.csv", index=False )