Kaggle competition: Quora Insincere Questions Classification
nullocc=nullfare
cat_feats=['Embarked','Sex','block','Titles','Pclass']
nullocc_test = pd.get_dummies(nullocc,columns=cat_feats,drop_first=False)
 nullocc_test=nullocc_test.drop(columns=['Name','Ticket','PassengerId','Cabin','Survived'])
 nullocc_test.head() 
<prepare_x_and_y>
MAX_SEQUENCE_LENGTH = 60
MAX_WORDS = 45000
EMBEDDINGS_LOADED_DIMENSIONS = 300
X_nullocc_test = nullocc_test<drop_column>
df_train = pd.read_csv(".. /input/train.csv")
 df_test = pd.read_csv(".. /input/test.csv" )
X_nonull_df_train=X_nonull_df_train.drop(columns=['Pclass_1','Pclass_2'])
 X_nullocc_test=X_nullocc_test.drop(columns=['CabinOccupancy'] )<train_model>
BATCH_SIZE = 256
Q_FRACTION = 1
questions = df_train.sample(frac=Q_FRACTION)
 question_texts = questions["question_text"].values
question_targets = questions["target"].values
test_texts = df_test["question_text"].fillna("_na_" ).values
print(f"Working on {len(questions)} questions" )
dtreeClass.fit(X_nonull_df_train,y_nonull_df_train )<predict_on_test>
def load_embeddings(file):
embeddings = {}
with open(file, encoding="utf8", errors='ignore')as f:
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings = dict(get_coefs(*line.split(" ")) for line in f)
 print('Found %s word vectors.' % len(embeddings))
return embeddings
predictions_nullocc = dtreeClass.predict(X_nullocc_test)
 predictions_nullocc<feature_engineering>
tokenizer = Tokenizer(num_words=MAX_WORDS)
 tokenizer.fit_on_texts(list(df_train["question_text"].values))
nullocc['CabinOccupancy']=np.where(nullocc['CabinOccupancy'].isnull() ,'2',nullocc['CabinOccupancy'])
 nullocc.head()<feature_engineering>
def create_embedding_weights(tokenizer, embeddings, dimensions):
not_embedded = defaultdict(int)
 word_index = tokenizer.word_index
words_count = min(len(word_index), MAX_WORDS)
 embeddings_matrix = np.zeros(( words_count, dimensions))
for word, i in word_index.items() :
if i >= MAX_WORDS:
continue
if word not in embeddings:
continue
embedding_vector = embeddings[word]
if embedding_vector is not None:
embeddings_matrix[i] = embedding_vector
return embeddings_matrix
df['CabinOccupancy']=np.where(( df['CabinOccupancy'].isnull())&(df['PassengerId']==1044),'2',df['CabinOccupancy'])
 df[df['PassengerId']==1044].head()<filter>
THRESHOLD = 0.35
class EpochMetricsCallback(keras.callbacks.Callback):
def on_train_begin(self, logs={}):
self.f1s = []
self.precisions = []
self.recalls = []
def on_epoch_end(self, epoch, logs={}):
predictions = self.model.predict(self.validation_data[0])
 predictions =(predictions > THRESHOLD ).astype(int)
 predictions = np.asarray(predictions)
 targets = self.validation_data[1]
f1 = metrics.f1_score(targets, predictions)
 precision = metrics.precision_score(targets, predictions)
 recall = metrics.recall_score(targets, predictions)
 print(" - F1 score: {0:.4f}, Precision: {1:.4f}, Recall: {2:.4f}" 
.format(f1, precision, recall))
self.f1s.append(f1)
 self.precisions.append(precision)
 self.recalls.append(recall)
 return
def display_model_history(history):
data = pd.DataFrame(data={'Train': history.history['loss'], 'Test': history.history['val_loss']})
 ax = sns.lineplot(data=data, palette="pastel", linewidth=2.5, dashes=False)
 ax.set(xlabel='Epoch', ylabel='Loss', title='Loss')
 ax.xaxis.set_major_locator(MaxNLocator(integer=True))
sns.despine()
plt.show()
def display_model_epoch_metrics(epoch_callback):
data = pd.DataFrame(data = {
'F1': epoch_callback.f1s,
'Precision': epoch_callback.precisions,
'Recall': epoch_callback.recalls})
 ax = sns.lineplot(data=data, palette='muted', linewidth=2.5, dashes=False)
 ax.set(xlabel='Epoch', title='Epoch metrics')
 ax.xaxis.set_major_locator(MaxNLocator(integer=True))
sns.despine()
plt.show()
nonull_df=df[(df['Fare'].notnull())&(df['Age'].notnull())&(df['Embarked'].notnull())&(df['block'].notnull())&(df['CabinOccupancy'].notnull())]
nonull_df.count() ['PassengerId']<categorify>
X = pad_sequences(tokenizer.texts_to_sequences(question_texts), maxlen=MAX_SEQUENCE_LENGTH)
 Y = question_targets
test_word_tokens = pad_sequences(tokenizer.texts_to_sequences(test_texts), maxlen=MAX_SEQUENCE_LENGTH )
cat_feats=['Embarked','Sex','Titles','block','Pclass']
nonull_df_train = pd.get_dummies(nonull_df,columns=cat_feats,drop_first=False)
 nonull_df_train = nonull_df_train[['Age','Fare','Parch','Pclass_1','Pclass_2','Pclass_3','SibSp','CabinOccupancy',
'Embarked_S','Embarked_Q','Embarked_C','Sex_female','Titles_ Miss','Titles_ Mrs','block_B']]
nonull_df_train.head()<prepare_x_and_y>
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
X_nonull_df_train = nonull_df_train.drop(['Embarked_S','Embarked_Q','Embarked_C'],axis=1)
 y_nonull_df_train = nonull_df_train[['Embarked_S','Embarked_Q','Embarked_C']]<categorify>
def make_model(emb_weights):
tokenized_input = Input(shape=(MAX_SEQUENCE_LENGTH,), name="tokenized_input")
 embedding = Embedding(MAX_WORDS, EMBEDDINGS_LOADED_DIMENSIONS,
weights=[emb_weights],
trainable=False )(tokenized_input)
 embedding = SpatialDropout1D(0.15 )(embedding)
 lstm = Bidirectional(CuDNNGRU(64, return_sequences=True))(embedding)
 lstm = SpatialDropout1D(0.15 )(lstm)
 lstm = Bidirectional(CuDNNGRU(32, return_sequences=True))(lstm)
 a = Attention(MAX_SEQUENCE_LENGTH )(lstm)
 d1 = Dense(32 )(a)
 d1 = Dropout(0.15 )(d1)
 b = BatchNormalization()(d1)
 out = Dense(1, activation='sigmoid' )(b)
 model = Model(inputs=[tokenized_input], outputs=out)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 model.summary()
return model
cat_feats=['Embarked','Sex','block','Titles','Pclass']
nullembark_test = pd.get_dummies(nullembark,columns=cat_feats,drop_first=False)
 nullembark_test=nullembark_test.drop(columns=['Name','Ticket','PassengerId','Cabin','Survived'])
 nullembark_test.head() 
<prepare_x_and_y>
train_X, test_X, train_Y, test_Y = train_test_split(X, Y, test_size=0.01)
 test_predictions = []
kaggle_predictions = []
embedding_files = [
".. /input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin",
".. /input/embeddings/glove.840B.300d/glove.840B.300d.txt",
".. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec",
".. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt"]
load_embedding_functions = [
lambda: load_embeddings(embedding_files[1]),
lambda: load_embeddings(embedding_files[2]),
lambda: load_embeddings(embedding_files[3])]
for index, load_embeddings_fn in enumerate(load_embedding_functions):
print(f"Training {embedding_files[index]}")
 pretrained_embeddings = load_embeddings_fn()
pretrained_emb_weights = create_embedding_weights(tokenizer, pretrained_embeddings, EMBEDDINGS_LOADED_DIMENSIONS)
 epoch_callback = EpochMetricsCallback()
model = make_model(pretrained_emb_weights)
 history = model.fit(x=train_X, y=train_Y, validation_split=0.015,
batch_size=BATCH_SIZE, epochs=5, verbose=2,
callbacks=[epoch_callback])
 display_model_history(history)
 display_model_epoch_metrics(epoch_callback)
 kaggle_predictions.append(model.predict([test_word_tokens], batch_size=1024, verbose=2))
test_predictions.append(model.predict([test_X]))
X_nullembark_test = nullembark_test<drop_column>
avg = np.average(kaggle_predictions, axis=0)
 df_out = pd.DataFrame({"qid":df_test["qid"].values})
 df_out['prediction'] =(avg > THRESHOLD ).astype(int)
 df_out.to_csv("submission.csv", index=False )
X_nonull_df_train=X_nonull_df_train.drop(columns=['Pclass_1','Pclass_2'])
<train_model>
avg = np.average(test_predictions, axis=0)
 f1s = []
precisions = []
recalls = []
Ts = [x * 0.01 for x in range(20, 60)]
for t in Ts:
pred =(avg > t ).astype(int)
 f1s.append(metrics.f1_score(test_Y, pred))
precisions.append(metrics.precision_score(test_Y, pred))
recalls.append(metrics.recall_score(test_Y, pred))
data = pd.DataFrame(data = {'F1': f1s,
'Precision': precisions,
'Recall': recalls},
index=Ts)
 ax = sns.lineplot(data=data, palette='muted', linewidth=2.5, dashes=False)
 ax.set(xlabel='Threshold', ylabel='Value', title='Threshold levels')
 sns.despine()
plt.show()
thresh = Ts[np.argmax(f1s)]
pred =(avg > thresh ).astype(int)
 f1 = metrics.f1_score(test_Y, pred)
 print("Test F1 {0:.4f} at threshold {1:.3f}".format(f1, thresh))
thresh = THRESHOLD
pred =(avg > thresh ).astype(int)
 f1 = metrics.f1_score(test_Y, pred)
 print("Test F1 {0:.4f} at threshold {1:.3f}".format(f1, thresh))
<predict_on_test><EOS> 
thresh = Ts[np.argmax(f1s)]
avg = np.average(kaggle_predictions, axis=0)
 df_out = pd.DataFrame({"qid":df_test["qid"].values})
 df_out['prediction'] =(avg > thresh ).astype(int)
 df_out.to_csv("submission.csv", index=False )