Kaggle competition: Quora Insincere Questions Classification
for base_layer in model.layers[:-1]:
base_layer.trainable = False
model.compile(optimizer=Adam(lr=1e-4), loss='binary_crossentropy')
 history_0 = model.fit_generator(generator=data_generator_train,
validation_data=data_generator_val,
epochs=1,
callbacks=[train_metric_callback, val_callback],
workers=num_cores,
verbose=1
)<train_model>
MAX_SEQUENCE_LENGTH = 60
MAX_WORDS = 45000
EMBEDDINGS_LOADED_DIMENSIONS = 300
for base_layer in model.layers[:-1]:
base_layer.trainable = True
model.compile(optimizer=Adam(lr=1e-5), loss='binary_crossentropy')
 history_1 = model.fit_generator(generator=data_generator_train,
validation_data=data_generator_val,
epochs=2,
callbacks=[train_metric_callback, val_callback],
workers=num_cores,
verbose=1,
initial_epoch=1
)<load_pretrained>
df_train = pd.read_csv(".. /input/train.csv")
 df_test = pd.read_csv(".. /input/test.csv" )
model = load_model('.. /input/clouds-classifier-files/classifier_densenet169_epoch_21_val_pr_auc_0.8365921057512743.h5' )<define_variables>
BATCH_SIZE = 512
Q_FRACTION = 1
questions = df_train.sample(frac=Q_FRACTION)
 question_texts = questions["question_text"].values
question_targets = questions["target"].values
test_texts = df_test["question_text"].fillna("_na_" ).values
print(f"Working on {len(questions)} questions" )
Image(".. /input/clouds-classifier-files/loss_hist_densenet169.png" )<define_variables>
def load_embeddings(file):
embeddings = {}
with open(file)as f:
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings = dict(get_coefs(*line.split(" ")) for line in f)
 print('Found %s word vectors.' % len(embeddings))
return embeddings
%time pretrained_embeddings = load_embeddings(".. /input/embeddings/glove.840B.300d/glove.840B.300d.txt" )
Image(".. /input/clouds-classifier-files/pr_auc_hist_densenet169.png" )<find_best_params>
tokenizer = Tokenizer(num_words=MAX_WORDS)
 %time tokenizer.fit_on_texts(list(df_train["question_text"].values))
class_names = ['Fish', 'Flower', 'Sugar', 'Gravel']
def get_threshold_for_recall(y_true, y_pred, class_i, recall_threshold=0.95, precision_threshold=0.94, plot=False):
precision, recall, thresholds = precision_recall_curve(y_true[:, class_i], y_pred[:, class_i])
 i = len(thresholds)- 1
best_recall_threshold = None
while best_recall_threshold is None:
next_threshold = thresholds[i]
next_recall = recall[i]
if next_recall >= recall_threshold:
best_recall_threshold = next_threshold
i -= 1
best_precision_threshold = [thres for prec, thres in zip(precision, thresholds)if prec >= precision_threshold][0]
if plot:
plt.figure(figsize=(10, 7))
plt.step(recall, precision, color='r', alpha=0.3, where='post')
 plt.fill_between(recall, precision, alpha=0.3, color='r')
 plt.axhline(y=precision[i + 1])
 recall_for_prec_thres = [rec for rec, thres in zip(recall, thresholds)
 if thres == best_precision_threshold][0]
plt.axvline(x=recall_for_prec_thres, color='g')
 plt.xlabel('Recall')
 plt.ylabel('Precision')
 plt.ylim([0.0, 1.05])
 plt.xlim([0.0, 1.0])
 plt.legend(['PR curve',
f'Precision {precision[i + 1]:.2f} corresponding to selected recall threshold',
f'Recall {recall_for_prec_thres:.2f} corresponding to selected precision threshold'])
 plt.title(f'Precision-Recall curve for Class {class_names[class_i]}')
 return best_recall_threshold, best_precision_threshold
y_pred = model.predict_generator(data_generator_val, workers=num_cores)
 y_true = data_generator_val.get_labels()
recall_thresholds = dict()
precision_thresholds = dict()
for i, class_name in tqdm(enumerate(class_names)) :
recall_thresholds[class_name], precision_thresholds[class_name] = get_threshold_for_recall(y_true, y_pred, i, plot=True )<predict_on_test>
def create_embedding_weights(tokenizer, embeddings, dimensions):
not_embedded = defaultdict(int)
 word_index = tokenizer.word_index
words_count = min(len(word_index), MAX_WORDS)
 embeddings_matrix = np.zeros(( words_count, dimensions))
for word, i in word_index.items() :
if i >= MAX_WORDS:
continue
embedding_vector = embeddings.get(word)
 if embedding_vector is not None:
embeddings_matrix[i] = embedding_vector
return embeddings_matrix
pretrained_emb_weights = create_embedding_weights(tokenizer, pretrained_embeddings, EMBEDDINGS_LOADED_DIMENSIONS )
data_generator_test = DataGenenerator(folder_imgs=test_imgs_folder, shuffle=False)
 y_pred_test = model.predict_generator(data_generator_test, workers=num_cores )<define_variables>
THRESHOLD = 0.35
class EpochMetricsCallback(keras.callbacks.Callback):
def on_train_begin(self, logs={}):
self.f1s = []
self.precisions = []
self.recalls = []
def on_epoch_end(self, epoch, logs={}):
predictions = self.model.predict(self.validation_data[0])
 predictions =(predictions > THRESHOLD ).astype(int)
 predictions = np.asarray(predictions)
 targets = self.validation_data[1]
f1 = metrics.f1_score(targets, predictions)
 precision = metrics.precision_score(targets, predictions)
 recall = metrics.recall_score(targets, predictions)
 print(" - F1 score: {0:.4f}, Precision: {1:.4f}, Recall: {2:.4f}" 
.format(f1, precision, recall))
self.f1s.append(f1)
 self.precisions.append(precision)
 self.recalls.append(recall)
 return
def display_model_history(history):
data = pd.DataFrame(data={'Train': history.history['loss'], 'Test': history.history['val_loss']})
 ax = sns.lineplot(data=data, palette="pastel", linewidth=2.5, dashes=False)
 ax.set(xlabel='Epoch', ylabel='Loss', title='Loss')
 sns.despine()
plt.show()
def display_model_epoch_metrics(epoch_callback):
data = pd.DataFrame(data = {
'F1': epoch_callback.f1s,
'Precision': epoch_callback.precisions,
'Recall': epoch_callback.recalls})
 sns.lineplot(data=data, palette='muted', linewidth=2.5, dashes=False)
 sns.despine()
plt.show()
image_labels_empty = set()
for i,(img, predictions)in enumerate(zip(os.listdir(test_imgs_folder), y_pred_test)) :
for class_i, class_name in enumerate(class_names):
if predictions[class_i] < recall_thresholds[class_name]:
image_labels_empty.add(f'{img}_{class_name}' )<load_from_csv>
%time X = pad_sequences(tokenizer.texts_to_sequences(question_texts), maxlen=MAX_SEQUENCE_LENGTH)
 %time Y = question_targets
%time test_word_tokens = pad_sequences(tokenizer.texts_to_sequences(test_texts), maxlen=MAX_SEQUENCE_LENGTH )
submission = pd.read_csv('.. /input/efficient-net-b4-unet-clouds/submission.csv')
 submission.head()<filter>
def make_model() :
tokenized_input = Input(shape=(MAX_SEQUENCE_LENGTH,), name="tokenized_input")
 embedding = Embedding(MAX_WORDS, EMBEDDINGS_LOADED_DIMENSIONS,
weights=[pretrained_emb_weights],
trainable=False )(tokenized_input)
 d0 = SpatialDropout1D(0.1 )(embedding)
 lstm = Bidirectional(LSTM(128, return_sequences=True))(d0)
 lstm = Bidirectional(LSTM(64, return_sequences=False))(lstm)
 d1 = Dropout(0.15 )(lstm)
 d1 = Dense(64 )(d1)
 b = BatchNormalization()(d1)
 out = Dense(1, activation='sigmoid' )(b)
 model = Model(inputs=[tokenized_input], outputs=out)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 model.summary()
return model
predictions_nonempty = set(submission.loc[~submission['EncodedPixels'].isnull() , 'Image_Label'].values )<concatenate>
train_X, test_X, train_Y, test_Y = train_test_split(X, Y, test_size=0.01)
 epoch_callback = EpochMetricsCallback()
model = make_model()
history = model.fit(x=train_X, y=train_Y, validation_split=0.015,
batch_size=BATCH_SIZE, epochs=4, verbose=2,
callbacks=[epoch_callback] )
print(f'{len(image_labels_empty.intersection(predictions_nonempty)) } masks would be removed' )<save_to_csv>
test_word_tokens = pad_sequences(tokenizer.texts_to_sequences(test_texts), maxlen=MAX_SEQUENCE_LENGTH)
 kaggle_predictions =(model.predict([test_word_tokens], batch_size=1024, verbose=2))
df_out = pd.DataFrame({"qid":df_test["qid"].values})
 df_out['prediction'] =(kaggle_predictions > THRESHOLD ).astype(int)
 df_out.to_csv("submission.csv", index=False )