Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<split>
tqdm.pandas()
X_train_count, X_test_count, y_train_count, y_test_count = train_test_split(count_vectorized, df_train.author, train_size=0.75)
 X_train_tfidf, X_test_tfidf, y_train_tfidf, y_test_tfidf = train_test_split(tfidf_vectorized, df_train.author, train_size=0.75 )<train_on_grid>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 print(f"Train shape: {train.shape}")
 print(f"Test shape: {test.shape}")
 train.sample()
model_count_NB = MultinomialNB()
model_tfidf_NB = MultinomialNB()
model_count_NB.fit(X_train_count, y_train_count)
 model_tfidf_NB.fit(X_train_tfidf, y_train_tfidf )<predict_on_test>
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr):
return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
predictions_count = model_count_NB.predict(X_test_count)
 predictions_tfidf = model_tfidf_NB.predict(X_test_tfidf )<compute_test_metric>
def check_coverage(vocab,embeddings_index):
a, oov, k, i = {}, {}, 0, 0
for word in vocab:
try:
a[word] = embeddings_index[word]
k += vocab[word]
except:
oov[word] = vocab[word]
i += vocab[word]
pass
print(f'Found embeddings for {(len(a)/ len(vocab)) :.2%} of vocab')
 print(f'Found embeddings for {(k /(k + i)) :.2%} of all text')
 sorted_x = sorted(oov.items() , key=(lambda x: x[1]), reverse=True)
 return sorted_x
def get_vocab(question_series):
sentences = question_series.str.split().values
words = [item for sublist in sentences for item in sublist]
return dict(Counter(words))
accuracy_count = accuracy_score(y_test_count, predictions_count)
 accuracy_tfidf = accuracy_score(y_test_tfidf, predictions_tfidf)
 print('Count Vectorized Words Accuracy:', accuracy_count)
 print('TfIdf Vectorized Words Accuracy:', accuracy_tfidf )<train_model>
vocab = get_vocab(train["question_text"])
 out_of_vocab = check_coverage(vocab, embeddings_index)
 out_of_vocab[:10]
model_NB = MultinomialNB()
model_NB.fit(count_vectorized, df_train.author )<categorify>
punct = set('?!.,"
embed_punct = punct & set(embeddings_index.keys())
 def clean_punctuation(txt):
for p in "/-":
txt = txt.replace(p, ' ')
 for p in "'`‘":
txt = txt.replace(p, '')
 for p in punct:
txt = txt.replace(p, f' {p} ' if p in embed_punct else ' _punct_ ')
 return txt
X_test = count_vect.transform(df_test.text )<predict_on_test>
train["question_text"] = train["question_text"].map(lambda x: clean_punctuation(x)).str.replace('\d+', '
test["question_text"] = test["question_text"].map(lambda x: clean_punctuation(x)).str.replace('\d+', '
vocab = get_vocab(train["question_text"])
 out_of_vocab = check_coverage(vocab, embeddings_index )
predicted_values = model_NB.predict_proba(X_test )<find_best_params>
train, validation = train_test_split(train, test_size=0.08, random_state=20181224)
 embed_size = 300
vocab_size = 95000
maxlen = 100
train_X = train["question_text"].fillna("_
val_X = validation["question_text"].fillna("_
test_X = test["question_text"].fillna("_
tokenizer = Tokenizer(num_words=vocab_size, filters='', lower=False)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 val_X = pad_sequences(val_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train['target'].values
val_y = validation['target'].values
model_NB.classes_<save_to_csv>
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
df_submission = pd.DataFrame({
'id': df_test.id.values,
'EAP': predicted_values[:,0],
'HPL': predicted_values[:,1],
'MWS': predicted_values[:,2]
})
 datetime = time.strftime("%Y%m%d_%H%M%S")
 filename = 'submission_' + datetime + '_acc_' + str(accuracy_count)+ '.csv'
df_submission.to_csv(filename, index=False)
 print('File',filename,'created.' )<set_options>
word_index = tokenizer.word_index
nb_words = min(vocab_size, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
num_missed = 0
for word, i in word_index.items() :
if i >= vocab_size: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
else: num_missed += 1
print(num_missed )
seed(10)
 set_random_seed(10)
 %matplotlib inline<define_variables>
hidden_layer_size = 100
BATCH_SIZE = 64
tf.reset_default_graph()
X = tf.placeholder(tf.int32, [None, maxlen], name='X')
 Y = tf.placeholder(tf.float32, [None], name='Y')
 batch_size = tf.placeholder(tf.int64 )
test_imgs_folder = '.. /input/understanding_cloud_organization/test_images/'
train_imgs_folder = '.. /input/understanding_cloud_organization/train_images/'
num_cores = multiprocessing.cpu_count()<load_from_csv>
dataset = tf.data.Dataset.from_tensor_slices(( X, Y)).shuffle(buffer_size=1000 ).batch(batch_size ).repeat()
test_dataset = tf.data.Dataset.from_tensor_slices(( X, Y)).batch(batch_size)
 iterator = tf.data.Iterator.from_structure(dataset.output_types,
dataset.output_shapes)
 train_init_op = iterator.make_initializer(dataset)
 test_init_op = iterator.make_initializer(test_dataset)
 questions, labels = iterator.get_next()
train_df = pd.read_csv('.. /input/understanding_cloud_organization/train.csv')
 train_df.head()<feature_engineering>
embeddings = tf.get_variable(name="embeddings", shape=embedding_matrix.shape,
initializer=tf.constant_initializer(np.array(embedding_matrix)) ,
trainable=False)
 embed = tf.nn.embedding_lookup(embeddings, questions )
train_df = train_df[~train_df['EncodedPixels'].isnull() ]
train_df['Image'] = train_df['Image_Label'].map(lambda x: x.split('_')[0])
 train_df['Class'] = train_df['Image_Label'].map(lambda x: x.split('_')[1])
 classes = train_df['Class'].unique()
train_df = train_df.groupby('Image')['Class'].agg(set ).reset_index()
for class_name in classes:
train_df[class_name] = train_df['Class'].map(lambda x: 1 if class_name in x else 0)
 train_df.head()<define_variables>
lstm_cell= tf.nn.rnn_cell.LSTMCell(hidden_layer_size)
 _, final_state = tf.nn.dynamic_rnn(lstm_cell, embed, dtype=tf.float32)
 last_layer = tf.layers.dense(final_state.h, 1)
 prediction = tf.nn.sigmoid(last_layer)
 prediction = tf.squeeze(prediction, [1] )
img_2_ohe_vector = {img:vec for img, vec in zip(train_df['Image'], train_df.iloc[:, 2:].values)}<split>
learning_rate=0.001
loss = tf.nn.sigmoid_cross_entropy_with_logits(logits=tf.squeeze(last_layer), labels=labels)
 loss = tf.reduce_mean(loss)
 optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate ).minimize(loss )
train_imgs, val_imgs = train_test_split(train_df['Image'].values,
test_size=0.2,
stratify=train_df['Class'].map(lambda x: str(sorted(list(x)))) ,
random_state=10 )<define_variables>
with tf.name_scope('metrics'):
F1, f1_update = tf.contrib.metrics.f1_score(labels=labels, predictions=prediction, name='my_metric')
 running_vars = tf.get_collection(tf.GraphKeys.LOCAL_VARIABLES, scope="my_metric")
 reset_op = tf.variables_initializer(var_list=running_vars )
class DataGenenerator(Sequence):
def __init__(self, images_list=None, folder_imgs=train_imgs_folder,
batch_size=32, shuffle=True, augmentation=None,
resized_height=224, resized_width=224, num_channels=3):
self.batch_size = batch_size
self.shuffle = shuffle
self.augmentation = augmentation
if images_list is None:
self.images_list = os.listdir(folder_imgs)
 else:
self.images_list = deepcopy(images_list)
 self.folder_imgs = folder_imgs
self.len = len(self.images_list)// self.batch_size
self.resized_height = resized_height
self.resized_width = resized_width
self.num_channels = num_channels
self.num_classes = 4
self.is_test = not 'train' in folder_imgs
if not shuffle and not self.is_test:
self.labels = [img_2_ohe_vector[img] for img in self.images_list[:self.len*self.batch_size]]
def __len__(self):
return self.len
def on_epoch_start(self):
if self.shuffle:
random.shuffle(self.images_list)
 def __getitem__(self, idx):
current_batch = self.images_list[idx * self.batch_size:(idx + 1)* self.batch_size]
X = np.empty(( self.batch_size, self.resized_height, self.resized_width, self.num_channels))
y = np.empty(( self.batch_size, self.num_classes))
for i, image_name in enumerate(current_batch):
path = os.path.join(self.folder_imgs, image_name)
 img = cv2.resize(cv2.imread(path),(self.resized_height, self.resized_width)).astype(np.float32)
 if not self.augmentation is None:
augmented = self.augmentation(image=img)
 img = augmented['image']
X[i, :, :, :] = img/255.0
if not self.is_test:
y[i, :] = img_2_ohe_vector[image_name]
return X, y
def get_labels(self):
if self.shuffle:
images_current = self.images_list[:self.len*self.batch_size]
labels = [img_2_ohe_vector[img] for img in images_current]
else:
labels = self.labels
return np.array(labels )<concatenate>
num_epochs = 10
seed = 3
sess = tf.Session()
sess.run(tf.global_variables_initializer())
 sess.run(tf.local_variables_initializer())
 costs, f1 = [], []
albumentations_train = Compose([
VerticalFlip() , HorizontalFlip() , Rotate(limit=30), GridDistortion()
], p=1 )<choose_model_class>
start = time.time()
end = 0
max_time = 6600
sess.run(train_init_op, feed_dict={X:train_X, Y:train_y, batch_size:BATCH_SIZE})
 num_iter = 1000
num_batches = int(train_X.shape[0] / BATCH_SIZE)
 for epoch in range(1, num_epochs+1):
seed += seed
tf.set_random_seed(seed)
 iter_cost = 0.
 prev_iter = 0.
 for i in range(num_batches):
_ , batch_loss, _ = sess.run([optimizer, loss, f1_update])
 iter_cost += batch_loss
end = time.time()
if(end-start > max_time):
break
if(i % num_iter == 0 and i > 0):
iter_cost /=(i-prev_iter)
 prev_iter = i
cur_f1 = sess.run(F1)
 sess.run(reset_op)
 f1.append(cur_f1)
 costs.append(iter_cost)
 print(f"Epoch {epoch} Iteration {i:5} cost: {iter_cost:.10f} f1: {cur_f1:.10f} time: {end-start:4.4f}")
 batch_cost = 0 .
data_generator_train = DataGenenerator(train_imgs, augmentation=albumentations_train)
 data_generator_train_eval = DataGenenerator(train_imgs, shuffle=False)
 data_generator_val = DataGenenerator(val_imgs, shuffle=False )<load_pretrained>
sz = 90
tf.set_random_seed(2018)
 sess.run(test_init_op, feed_dict={X: val_X, Y: val_y, batch_size: sz})
 val_pred = np.concatenate([sess.run(prediction)for _ in range(int(val_X.shape[0]/sz)) ] )
class PrAucCallback(Callback):
def __init__(self, data_generator, num_workers=num_cores,
early_stopping_patience=5,
plateau_patience=3, reduction_rate=0.5,
stage='train', checkpoints_path='checkpoints/'):
super(Callback, self ).__init__()
self.data_generator = data_generator
self.num_workers = num_workers
self.class_names = ['Fish', 'Flower', 'Sugar', 'Gravel']
self.history = [[] for _ in range(len(self.class_names)+ 1)]
self.early_stopping_patience = early_stopping_patience
self.plateau_patience = plateau_patience
self.reduction_rate = reduction_rate
self.stage = stage
self.best_pr_auc = -float('inf')
 if not os.path.exists(checkpoints_path):
os.makedirs(checkpoints_path)
 self.checkpoints_path = checkpoints_path
def compute_pr_auc(self, y_true, y_pred):
pr_auc_mean = 0
print(f"
{'
")
 for class_i in range(len(self.class_names)) :
precision, recall, _ = precision_recall_curve(y_true[:, class_i], y_pred[:, class_i])
 pr_auc = auc(recall, precision)
 pr_auc_mean += pr_auc/len(self.class_names)
 print(f"PR AUC {self.class_names[class_i]}, {self.stage}: {pr_auc:.3f}
")
 self.history[class_i].append(pr_auc)
 print(f"
{'
PR AUC mean, {self.stage}: {pr_auc_mean:.3f}
{'
")
 self.history[-1].append(pr_auc_mean)
 return pr_auc_mean
def is_patience_lost(self, patience):
if len(self.history[-1])> patience:
best_performance = max(self.history[-1][-(patience + 1):-1])
 return best_performance == self.history[-1][-(patience + 1)] and best_performance >= self.history[-1][-1]
def early_stopping_check(self, pr_auc_mean):
if self.is_patience_lost(self.early_stopping_patience):
self.model.stop_training = True
def model_checkpoint(self, pr_auc_mean, epoch):
if pr_auc_mean > self.best_pr_auc:
for checkpoint in glob.glob(os.path.join(self.checkpoints_path, 'classifier_densenet169_epoch_*')) :
os.remove(checkpoint)
 self.best_pr_auc = pr_auc_mean
self.model.save(os.path.join(self.checkpoints_path, f'classifier_densenet169_epoch_{epoch}_val_pr_auc_{pr_auc_mean}.h5'))
print(f"
{'
Saved new checkpoint
{'
")
 def reduce_lr_on_plateau(self):
if self.is_patience_lost(self.plateau_patience):
new_lr = float(keras.backend.get_value(self.model.optimizer.lr)) * self.reduction_rate
keras.backend.set_value(self.model.optimizer.lr, new_lr)
 print(f"
{'
Reduced learning rate to {new_lr}.
 {'
")
 def on_epoch_end(self, epoch, logs={}):
y_pred = self.model.predict_generator(self.data_generator, workers=self.num_workers)
 y_true = self.data_generator.get_labels()
pr_auc_mean = self.compute_pr_auc(y_true, y_pred)
 if self.stage == 'val':
self.early_stopping_check(pr_auc_mean)
 self.model_checkpoint(pr_auc_mean, epoch)
 self.reduce_lr_on_plateau()
def get_pr_auc_history(self):
return self.history<groupby>
thresh = thresholds[np.argmax(scores)]
print(f"Best Validation F1 Score is {max(scores):.4f} at threshold {thresh}" )
train_metric_callback = PrAucCallback(data_generator_train_eval)
 val_callback = PrAucCallback(data_generator_val, stage='val' )<choose_model_class>
sz=30
temp_y = val_y[:test_X.shape[0]]
sub = test[['qid']]
sess.run(test_init_op, feed_dict={X: test_X, Y: temp_y, batch_size:sz})
 sub['prediction'] = np.concatenate([sess.run(prediction)for _ in range(int(test_X.shape[0]/sz)) ] )
def get_model() :
K.clear_session()
base_model = DenseNet169(weights='imagenet', include_top=False, pooling='avg', input_shape=(224, 224, 3))
x = base_model.output
y_pred = Dense(4, activation='sigmoid' )(x)
 return Model(inputs=base_model.input, outputs=y_pred)
 model = get_model()<train_model>
sub['prediction'] =(sub['prediction'] > thresh ).astype(np.int16)
 sub.to_csv("submission.csv", index=False)
 sub.sample()