Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<categorify>
os.environ['PYTHONHASHSEED'] = '10000'
np.random.seed(10001)
 random.seed(10002)
 tf.set_random_seed(10003)
 wordnet_lemmatizer = WordNetLemmatizer()
stop_words = set(stopwords.words('english'))
test_labels1_1, test_labels1_2 = make_test(pred_test )<save_to_csv>
path = '.. /input/'
embedding_path = '.. /input/embeddings/'
cores = 4
max_text_length=50
do_submission = False
min_df_one=1
keep_only_words_in_embedding = True
keep_unknown_words_in_keras_sequence_as_zeros = True
contraction_mapping = {u"ain't": u"is not", u"aren't": u"are not",u"can't": u"cannot", u"'cause": u"because",
u"could've": u"could have", u"couldn't": u"could not", u"didn't": u"did not",
u"doesn't": u"does not", u"don't": u"do not", u"hadn't": u"had not",
u"hasn't": u"has not", u"haven't": u"have not", u"he'd": u"he would",
u"he'll": u"he will", u"he's": u"he is", u"how'd": u"how did", u"how'd'y": u"how do you",
u"how'll": u"how will", u"how's": u"how is", u"I'd": u"I would",
u"I'd've": u"I would have", u"I'll": u"I will", u"I'll've": u"I will have",
u"I'm": u"I am", u"I've": u"I have", u"i'd": u"i would", u"i'd've": u"i would have",
u"i'll": u"i will", u"i'll've": u"i will have",u"i'm": u"i am", u"i've": u"i have",
u"isn't": u"is not", u"it'd": u"it would", u"it'd've": u"it would have",
u"it'll": u"it will", u"it'll've": u"it will have",u"it's": u"it is",
u"let's": u"let us", u"ma'am": u"madam", u"mayn't": u"may not",
u"might've": u"might have",u"mightn't": u"might not",u"mightn't've": u"might not have",
u"must've": u"must have", u"mustn't": u"must not", u"mustn't've": u"must not have",
u"needn't": u"need not", u"needn't've": u"need not have",u"o'clock": u"of the clock",
u"oughtn't": u"ought not", u"oughtn't've": u"ought not have", u"shan't": u"shall not",
u"sha'n't": u"shall not", u"shan't've": u"shall not have", u"she'd": u"she would",
u"she'd've": u"she would have", u"she'll": u"she will", u"she'll've": u"she will have",
u"she's": u"she is", u"should've": u"should have", u"shouldn't": u"should not",
u"shouldn't've": u"should not have", u"so've": u"so have",u"so's": u"so as",
u"this's": u"this is",u"that'd": u"that would", u"that'd've": u"that would have",
u"that's": u"that is", u"there'd": u"there would", u"there'd've": u"there would have",
u"there's": u"there is", u"here's": u"here is",u"they'd": u"they would",
u"they'd've": u"they would have", u"they'll": u"they will",
u"they'll've": u"they will have", u"they're": u"they are", u"they've": u"they have",
u"to've": u"to have", u"wasn't": u"was not", u"we'd": u"we would",
u"we'd've": u"we would have", u"we'll": u"we will", u"we'll've": u"we will have",
u"we're": u"we are", u"we've": u"we have", u"weren't": u"were not",
u"what'll": u"what will", u"what'll've": u"what will have", u"what're": u"what are",
u"what's": u"what is", u"what've": u"what have", u"when's": u"when is",
u"when've": u"when have", u"where'd": u"where did", u"where's": u"where is",
u"where've": u"where have", u"who'll": u"who will", u"who'll've": u"who will have",
u"who's": u"who is", u"who've": u"who have", u"why's": u"why is", u"why've": u"why have",
u"will've": u"will have", u"won't": u"will not", u"won't've": u"will not have",
u"would've": u"would have", u"wouldn't": u"would not", u"wouldn't've": u"would not have",
u"y'all": u"you all", u"y'all'd": u"you all would",u"y'all'd've": u"you all would have",
u"y'all're": u"you all are",u"y'all've": u"you all have",u"you'd": u"you would",
u"you'd've": u"you would have", u"you'll": u"you will", u"you'll've": u"you will have",
u"you're": u"you are", u"you've": u"you have" }
test_labels1_2.to_csv('InceptionResNetV1_2.csv', index=False )<categorify>
def load_glove_words() :
EMBEDDING_FILE = embedding_path+'glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, 1
embeddings_index1 = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
EMBEDDING_FILE = embedding_path+'paragram_300_sl999/paragram_300_sl999.txt'
embeddings_index2 = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding='utf-8', errors='ignore'))
return(set(list(embeddings_index1.keys())) ).union(set(list(embeddings_index2.keys())))
 embeddings_index = load_glove_words()
print(len(embeddings_index))
No = 1
models['model_cnvt'].save_weights('model_{}_cnvt.h5'.format(No))
models['model_resnet'].save_weights('model_{}_resnet.h5'.format(No))
models['model_classifier'].save_weights('model_{}_classifier.h5'.format(No))<train_model>
def clean_str(text):
text = re.sub(u"\[math\].*\[\/math\]", u" math ", text)
 text = re.sub(u"\S*@\S*\.\S*", u" email ", text)
 text = u" ".join(re.sub(u"^\d+(?:[.,]\d*)?$", u"number", w)for w in text.split(" "))
specials = [u"’", u"‘", u"´", u"`", u"\u2019"]
for s in specials:
text = u" ".join(w.replace(s, u"'")for w in text.split(" "))
text = u" ".join([w.strip() if w in embeddings_index else clean(w ).strip() for w in text.split(" ")])
 text = re.sub(u"\s+", u" ", text ).strip()
text = u" ".join([w.strip() for w in text.split(" ")if w in embeddings_index ])
 return text
def clean(text):
try:
if text in contraction_mapping:
return contraction_mapping[text]
for i, j in [(u"é", u"e"),(u"ē", u"e"),(u"è", u"e"),(u"ê", u"e"),(u"à", u"a"), 
(u"â", u"a"),(u"ô", u"o"),(u"ō", u"o"),(u"ü", u"u"),(u"ï", u"i"), 
(u"ç", u"c"),(u"\xed", u"i")]:
text = re.sub(i, j, text)
 if text in embeddings_index:
return text
text = text.lower()
if text in embeddings_index:
return text
text = re.sub(u"[^a-z\s0-9]", u" ", text)
 text = u" ".join(re.sub(u"^\d+(?:[.,]\d*)?$", u"number", w)for w in text.split(" "))
text = re.sub(u"[^a-z\s]", u" ", text)
 text = re.sub(u"\s+", u" ", text ).strip()
text = u" ".join([ wordnet_lemmatizer.lemmatize(w)if w not in embeddings_index else w for w in text.split() ])
 except:
print('ERROR')
 text = ''
return text
def parallelize_dataframe(df, func):
df_split = np.array_split(df, cores)
 pool = Pool(cores)
 df = pd.concat(pool.map(func, df_split))
pool.close()
pool.join()
return df
def clean_str_df(df):
return df.apply(lambda s : clean_str(s))
def prepare_data(df_data, train=True):
print('Prepare data.... ')
 df_data['question_text'] = parallelize_dataframe(df_data['question_text'], clean_str_df)
 return df_data
def create_vocabulary(df):
start_time = time.time()
word_frequency_dc=defaultdict(np.uint32)
 def word_count(text):
text = set(text.split(' '))
if keep_only_words_in_embedding:
for w in text:
if w in embeddings_index:
word_frequency_dc[w]+=1
else:
for w in text:
word_frequency_dc[w]+=1
df['question_text'].apply(lambda x : word_count(x))
print('[{}] Finished COUNTING WORDS FOR question_text...'.format(time.time() - start_time))
start_time = time.time()
vocabulary_dc = word_frequency_dc.copy()
cpt=1
for key in vocabulary_dc:
vocabulary_dc[key]=cpt
cpt+=1
print('[{}] Finished CREATING VOCABULARY...'.format(time.time() - start_time))
return word_frequency_dc, vocabulary_dc
def tokenize(text):
return [w for w in text.split() ]
def load_glove(word_index):
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 EMBEDDING_FILE = embedding_path+'glove.840B.300d/glove.840B.300d.txt'
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean1,emb_std1 = all_embs.mean() , all_embs.std()
EMBEDDING_FILE = embedding_path+'paragram_300_sl999/paragram_300_sl999.txt'
embeddings_index2 = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding='utf-8', errors='ignore'))
all_embs = np.stack(embeddings_index2.values())
 emb_mean2,emb_std2 = all_embs.mean() , all_embs.std()
embedding_matrix = np.zeros(( len(word_index)+2, 300), dtype=np.float32)
 for word, i in word_index.items() :
embedding_vector = None
if(word in embeddings_index)&(word in embeddings_index2):
embedding_vector =(embeddings_index.get(word)+(embeddings_index2.get(word)-emb_mean2+emb_mean1)) /2.
 else:
if(word in embeddings_index):
embedding_vector = embeddings_index.get(word)
 else:
if(word in embeddings_index2):
embedding_vector = embeddings_index2.get(word)-emb_mean2+emb_mean1
if embedding_vector is not None: embedding_matrix[i] = embedding_vector.astype(np.float32)
 return embedding_matrix
def preprocess_keras(text):
if keep_unknown_words_in_keras_sequence_as_zeros:
return [ vocabulary_dc[w] if(( w in embeddings_index)&(word_frequency_dc[w]>=min_df_one)) else 0 for w in(text.split())]
else:
return [ vocabulary_dc[w] for w in(text.split())if w in embeddings_index ]
def preprocess_keras_df(df):
return df.apply(preprocess_keras)
 def get_keras_data(df):
X = {
'num_data' : num_FE(df),
'question_text_in_glove': pad_sequences(df['seq_question_text_in_glove'], maxlen=max_text_length, value=0, padding='post'),
'question_text_mask_in_glove':(pad_sequences(df['seq_question_text_in_glove'], maxlen=max_text_length, value=0, padding='post')>0 ).astype(int ).reshape(-1,max_text_length,1),
'starting_words': pad_sequences(df['seq_question_text_in_glove'].apply(lambda x : x[:10]), maxlen=10, value=0, padding='post'),
'ending_words': pad_sequences(df['seq_question_text_in_glove'].apply(lambda x : x[-10:]), maxlen=10, value=0, padding='post')
 }
return X
def num_FE(temp_df):
df = pd.DataFrame()
df['total_length'] = temp_df['question_text_original'].apply(len)
 df['capitals'] = temp_df['question_text_original'].apply(lambda comment: sum(1 for c in comment if c.isupper()))
df['caps_vs_length'] = df.apply(lambda row: float(row['capitals'])/float(row['total_length']),
axis=1)
 df['num_exclamation_marks'] = temp_df['question_text_original'].apply(lambda comment: comment.count('!'))
df['num_question_marks'] = temp_df['question_text_original'].apply(lambda comment: comment.count('?'))
df['num_punctuation'] = temp_df['question_text_original'].apply(
 lambda comment: sum(comment.count(w)for w in '.,;:'))
df['num_symbols'] = temp_df['question_text_original'].apply(
 lambda comment: sum(comment.count(w)for w in '*
df['num_words'] = temp_df['question_text_original'].apply(lambda comment: len(comment.split()))
df['num_unique_words'] = temp_df['question_text_original'].apply(
 lambda comment: len(set(w for w in comment.split())))
 df['words_vs_unique'] = df['num_unique_words'] / df['num_words']
return df.values.astype(np.float32 )
ls -la<choose_model_class>
def threshold_search(y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in [i * 0.01 for i in range(10,70)]:
score = f1_score(y_true=y_true, y_pred=y_proba > threshold)
 if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'f1': best_score}
return search_result
class My_Callback(Callback):
def __init__(self, save_path, factor=0.5, patience_reduce=10, patience_stop=30, min_lr=1e-4):
self.scores_list = []
self.scores_list_stop = []
self.thresh_list = []
self.all_preds = None
self.best_score = 0.
 self.best_threshold = 0.5
self.best_epoch = 0
self.best_lr = 0
self.save_path = save_path
self.factor = factor
self.min_lr = min_lr
self.patience_reduce = patience_reduce
self.patience_stop = patience_stop
def on_epoch_end(self, epoch, logs={}):
all_preds = self.all_preds
ep = epoch
best_score = self.best_score
preds_valid = self.model.predict(test_keras , batch_size=2048)
 thresholds = threshold_search(dtest_y, preds_valid)
 best_thres = thresholds['threshold']
score_all = thresholds['f1']
print(ep+1, ' F1 : ', "{0:.4f}".format(score_all), "{0:.4f}".format(best_thres),
' AUC : ', "{0:.4f}".format(roc_auc_score(dtest_y, preds_valid>=best_thres)))
 self.scores_list.append(score_all)
 self.thresh_list.append(best_thres)
 self.scores_list_stop.append(score_all)
 if score_all > best_score:
self.best_score = score_all
self.best_threshold = best_thres
self.best_epoch = ep+1
self.best_lr = np.float32(K.get_value(self.model.optimizer.lr))
self.model.save_weights(self.save_path, overwrite=True)
 print("Score improved from ", "{0:.4f}".format(best_score), ' to ', "{0:.4f}".format(score_all),\
'threshold : ', "{0:.4f}".format(self.best_threshold), ' lr : ', self.best_lr, ' epoch : ', self.best_epoch)
 else:
print("Score didnt improve, current best score is : ", "{0:.4f}".format(best_score),\
'with threshold : ', "{0:.4f}".format(self.best_threshold), ' lr : ', self.best_lr, ' epoch : ', self.best_epoch)
 pos = np.argmax(self.scores_list)
 if len(self.scores_list)- pos>self.patience_reduce:
old_lr = np.float32(K.get_value(self.model.optimizer.lr))
if old_lr > self.min_lr:
new_lr = old_lr * self.factor
new_lr = max(new_lr, self.min_lr)
 self.model.load_weights(self.save_path)
 K.set_value(self.model.optimizer.lr, new_lr)
 self.scores_list = self.scores_list[:pos+1]
self.thresh_list = self.thresh_list[:pos+1]
print('*'*70)
 print('Reducing LR from ', old_lr, ' to ', new_lr)
 print('Loading Last best score : ', self.scores_list[-1], 'with threshold : ',self.thresh_list[-1])
 print('*'*70)
 pos = np.argmax(self.scores_list_stop)
 if len(self.scores_list_stop)- pos>self.patience_stop:
print('TRAINING STOP, NO MORE IMPROVEMENT')
 print('LOADING BEST WEIGHTS.... ')
 self.model.load_weights(self.save_path)
 self.stopped_epoch = ep
self.model.stop_training = True
print('')
 return
class DataGenerator(keras.utils.Sequence):
'Generates data for Keras'
def __init__(self, data, labels, shuffle=True, seed=None):
self.epoch = 0
self.seed = seed
if self.seed is None:
self.seed = self.epoch
np.random.seed(self.seed)
 self.list_IDs = np.random.permutation(len(labels))
self.data_num = data['num_data']
self.data_start = data['starting_words']
self.data_end = data['ending_words']
self.data_glove = data['question_text_in_glove']
self.data_mask = data['question_text_mask_in_glove']
self.labels = labels
self.shuffle = shuffle
self.on_epoch_end()
def __len__(self):
'Denotes the number of batches per epoch'
return int(np.floor(len(self.list_IDs)/ batch_size))
def __getitem__(self, index):
'Generate one batch of data'
indexes = self.indexes[index*batch_size:(index+1)*batch_size]
list_IDs_temp = [self.list_IDs[k] for k in indexes]
X, y = self.__data_generation(list_IDs_temp)
 return(X, y)
 def on_epoch_end(self):
'Updates indexes after each epoch'
self.epoch += 1
self.seed += 1
np.random.seed(self.seed)
 self.list_IDs = np.random.permutation(len(self.labels))
self.indexes = np.arange(len(self.list_IDs))
if self.shuffle == True:
np.random.shuffle(self.indexes)
 def __data_generation(self, list_IDs_temp):
X = {
'num_data' : self.data_num[list_IDs_temp],
'starting_words': self.data_start[list_IDs_temp],
'ending_words': self.data_end[list_IDs_temp],
'question_text_in_glove': self.data_glove[list_IDs_temp],
'question_text_mask_in_glove': self.data_mask[list_IDs_temp].reshape(-1,max_text_length,1),
}
return X , self.labels[list_IDs_temp]
def new_rnn_model() :
sequence_in_glove = Input(shape=[train_keras["question_text_in_glove"].shape[1]], name="question_text_in_glove")
 mask_in_glove = Input(shape=[train_keras["question_text_mask_in_glove"].shape[1],1], name="question_text_mask_in_glove")
 starting_words = Input(shape=[train_keras["starting_words"].shape[1]], name="starting_words")
 ending_words = Input(shape=[train_keras["ending_words"].shape[1]], name="ending_words")
 num_data = Input(shape=[train_keras["num_data"].shape[1]], name="num_data")
 num_data_normed = BatchNormalization()(num_data)
 shared_embedding = Embedding(glove_weights.shape[0], glove_weights.shape[1], weights=[glove_weights], trainable=False, name='emb')
 emb_sequence_in_glove = shared_embedding(sequence_in_glove)
 emb_sequence_in_glove = multiply([emb_sequence_in_glove, mask_in_glove])
 emb_sequence_in_glove = SpatialDropout1D(0.1 )(emb_sequence_in_glove)
 shared_embedding2 = Embedding(glove_weights.shape[0], 20, trainable=True, name='emb2')
 emb_sequence_in_glove2 = shared_embedding(sequence_in_glove)
 emb_sequence_in_glove2 = multiply([emb_sequence_in_glove2, mask_in_glove])
 emb_sequence_in_glove2 = SpatialDropout1D(0.2 )(emb_sequence_in_glove2)
 lstm2 = CuDNNLSTM(64, return_sequences=True )(emb_sequence_in_glove2)
 lstm2 = SpatialDropout1D(0.2 )(lstm2)
 max_lstm2 = GlobalMaxPooling1D()(lstm2)
 avg_lstm2 = Lambda(lambda x : K.sum(x, axis=1))(lstm2)
 avg_lstm2 = BatchNormalization()(avg_lstm2)
 starting_emb = shared_embedding(starting_words)
 ending_emb = shared_embedding(ending_words)
 avg_emb_start = Lambda(lambda x : K.sum(x, axis=1))(starting_emb)
 avg_emb_start = BatchNormalization()(avg_emb_start)
 avg_emb_end = Lambda(lambda x : K.sum(x, axis=1))(ending_emb)
 avg_emb_end = BatchNormalization()(avg_emb_end)
 avg_emb = Lambda(lambda x : K.sum(x, axis=1))(emb_sequence_in_glove)
 avg_emb = BatchNormalization()(avg_emb)
 lstm = CuDNNLSTM(128, return_sequences=True )(emb_sequence_in_glove)
 lstm = SpatialDropout1D(0.1 )(lstm)
 max_lstm = GlobalMaxPooling1D()(lstm)
 avg_lstm = Lambda(lambda x : K.sum(x, axis=1))(lstm)
 avg_lstm = BatchNormalization()(avg_lstm)
 main_l = concatenate([ max_lstm, avg_lstm, num_data_normed, avg_emb])
 main_l = Dense(196 )(main_l)
 main_l = Activation('relu' )(main_l)
 main_l = Dense(64 )(main_l)
 main_l = Activation('relu' )(main_l)
 output = Dense(1, activation="sigmoid" )(main_l)
 model = Model([sequence_in_glove, mask_in_glove, starting_words, ending_words, num_data], output)
 return model
model_resnet = applications.inception_resnet_v2.InceptionResNetV2(
 include_top=False,
weights='imagenet',
input_tensor=None,
input_shape=(139,139,3),
pooling='avg',
classes=None)
<choose_model_class>
df_sub = pd.read_csv(path+'test.csv', encoding='utf-8', engine='python')
 df_sub['target'] = -99
df_sub['question_text'].fillna(u'unknownstring', inplace=True)
 df_train_all = pd.read_csv(path+'train.csv', encoding='utf-8', engine='python')
 df_train_all['question_text'].fillna(u'unknownstring', inplace=True)
 df_sub['question_text_original'] = df_sub['question_text'].copy()
df_sub = prepare_data(df_sub)
 df_train_all['question_text_original'] = df_train_all['question_text'].copy()
df_train_all = prepare_data(df_train_all)
 word_frequency_dc, vocabulary_dc = create_vocabulary(df_train_all[['question_text']].append(df_sub[['question_text']]))
print(len(vocabulary_dc))
glove_weights = load_glove(vocabulary_dc)
 print(glove_weights.shape )
models['model'].compile(loss=focal_loss,
optimizer='adam',
metrics=['categorical_accuracy', 'binary_accuracy'] )<train_model>
df_train_all['seq_question_text_in_glove'] = parallelize_dataframe(df_train_all['question_text'], preprocess_keras_df)
 train_keras = get_keras_data(df_train_all)
 df_sub['seq_question_text_in_glove'] = parallelize_dataframe(df_sub['question_text'], preprocess_keras_df)
 sub_keras = get_keras_data(df_sub)
seq = Seq(train_labels, aug=True, batch_size=64)
 print(len(seq))
hst = models['model'].fit_generator(seq, epochs=5,
steps_per_epoch=len(seq),
callbacks=[] )<load_pretrained>
scale = 1.
 BATCH_SIZE = int(scale*1024)
 lr1 = scale*2e-3
lr2 = scale*1e-3
batch_size = BATCH_SIZE
epochs = 5
save_model_name='./model32.h5'
all_preds_sub = [ ]
print("Fitting RNN model...")
 for bag in range(7):
train_generator = DataGenerator(train_keras, df_train_all['target'].values, shuffle=True, seed=bag)
 rnn_model = new_rnn_model()
rnn_model.get_layer('emb' ).trainable=False
optimizer = Adam(lr=lr1)
 rnn_model.compile(loss="binary_crossentropy", optimizer=optimizer)
 rnn_model.fit_generator(generator=train_generator, workers=3, verbose=2, epochs=epochs)
 finetune=True
if finetune:
rnn_model.get_layer('emb' ).trainable=True
optimizer = Adam(lr=lr2)
 rnn_model.compile(loss="binary_crossentropy", optimizer=optimizer)
 rnn_model.fit_generator(generator=train_generator, workers=3, verbose=2,epochs=1)
 preds_sub = rnn_model.predict(sub_keras , batch_size=2048 ).squeeze()
all_preds_sub.append(preds_sub)
 del rnn_model
gc.collect()
print('*******************************************************')
<predict_on_test><EOS> 
subThreshold = 0.33
mean_preds = np.array(all_preds_sub ).transpose()
mean_preds = mean_preds.mean(axis=1)
 sub_df = pd.DataFrame()
sub_df['qid'] = df_sub.qid.values
sub_df['prediction'] =(mean_preds>subThreshold ).astype(int)
 sub_df.to_csv('submission.csv', index=False )