Kaggle competition: Quora Question Pairs
ct = test_df.shape[0]
input_ids_t = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(test_df.shape[0]):
text1 = " "+" ".join(test_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[test_df.loc[k,'sentiment']]
input_ids_t[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask_t[k,:len(enc.ids)+5] = 1<load_pretrained>
df_train = pd.read_csv('.. /input/train.csv')
 df_train.head()
def build_model() :
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 config = RobertaConfig.from_pretrained(PATH+'config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained(PATH+'pretrained-roberta-base.h5',config=config)
 x = bert_model(ids,attention_mask=att,token_type_ids=tok)
 x1 = tf.keras.layers.Dropout(0.1 )(x[0])
 x1 = tf.keras.layers.Conv1D(128, 2,padding='same' )(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.Conv1D(64, 2,padding='same' )(x1)
 x1 = tf.keras.layers.Dense(2 )(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.Dense(1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.1 )(x[0])
 x2 = tf.keras.layers.Conv1D(128, 2, padding='same' )(x2)
 x2 = tf.keras.layers.LeakyReLU()(x2)
 x2 = tf.keras.layers.Conv1D(64, 2, padding='same' )(x2)
 x2 = tf.keras.layers.Dense(2 )(x2)
 x2 = tf.keras.layers.LeakyReLU()(x2)
 x2 = tf.keras.layers.Dense(1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)
 model.compile(loss='binary_crossentropy', optimizer=optimizer)
 return model<define_variables>
p = df_train['is_duplicate'].mean()
print('Predicted score:', log_loss(df_train['is_duplicate'], np.zeros_like(df_train['is_duplicate'])+ p))
df_test = pd.read_csv('.. /input/test.csv')
 sub = pd.DataFrame({'test_id': df_test['test_id'], 'is_duplicate': p})
 sub.to_csv('naive_submission.csv', index=False)
 sub.head()
n_splits = 4<choose_model_class>
df_test = pd.read_csv('.. /input/test.csv')
 df_test.head()
jac = []; VER='v1'; DISPLAY=1
oof_start = np.zeros(( input_ids.shape[0],MAX_LEN))
oof_end = np.zeros(( input_ids.shape[0],MAX_LEN))
skf = StratifiedKFold(n_splits=n_splits,shuffle=True,random_state=777)
 for fold,(idxT,idxV)in enumerate(skf.split(input_ids,train_df.sentiment.values)) :
print('
print('
print('
K.clear_session()
model = build_model()
reduce_lr = tf.keras.callbacks.LearningRateScheduler(scheduler)
 sv = tf.keras.callbacks.ModelCheckpoint(
 '%s-roberta-%i.h5'%(VER,fold), monitor='val_loss', verbose=1, save_best_only=True,
save_weights_only=True, mode='auto', save_freq='epoch')
 hist = model.fit([input_ids[idxT,], attention_mask[idxT,], token_type_ids[idxT,]], [start_tokens[idxT,], end_tokens[idxT,]],
epochs=4, batch_size=8, verbose=DISPLAY, callbacks=[sv, reduce_lr],
validation_data=([input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]],
[start_tokens[idxV,], end_tokens[idxV,]]))
print('Loading model...')
 model.load_weights('./v1-roberta-%i.h5'%(fold))
print('Predicting OOF...')
 oof_start[idxV,],oof_end[idxV,] = model.predict([input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]],verbose=DISPLAY)
 all = []
for k in idxV:
start_sorted_indexes = np.argsort(-oof_start[k,], kind='quicksort', order=None)
 end_sorted_indexes = np.argsort(-oof_end[k,], kind='quicksort', order=None)
 if start_sorted_indexes[0] <= end_sorted_indexes[0]:
a = start_sorted_indexes[0]
b = end_sorted_indexes[0]
text1 = " "+" ".join(train_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 elif start_sorted_indexes[0] <= end_sorted_indexes[1]:
a = start_sorted_indexes[0]
b = end_sorted_indexes[1]
text1 = " "+" ".join(train_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 elif start_sorted_indexes[1] <= end_sorted_indexes[0]:
a = start_sorted_indexes[1]
b = end_sorted_indexes[0]
text1 = " "+" ".join(train_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 elif start_sorted_indexes[1] <= end_sorted_indexes[1]:
a = start_sorted_indexes[1]
b = end_sorted_indexes[1]
text1 = " "+" ".join(train_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 else:
st = train_df.loc[k,'text']
all.append(jaccard(st,train_df.loc[k,'selected_text']))
jac.append(np.mean(all))
print('>>>> FOLD %i Jaccard ='%(fold+1),np.mean(all))
print()<predict_on_test>
print('Total number of question pairs for testing: {}'.format(len(df_test)) )
preds_start = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids_t.shape[0],MAX_LEN))
DISPLAY=1
for i in range(n_splits):
print('
print('
print('
K.clear_session()
model = build_model()
model.load_weights('./v1-roberta-%i.h5'%(i))
print('Predicting Test...')
 preds = model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=DISPLAY)
 preds_start += preds[0]/n_splits
preds_end += preds[1]/n_splits<string_transform>
qmarks = np.mean(train_qs.apply(lambda x: '?' in x))
math = np.mean(train_qs.apply(lambda x: '[math]' in x))
fullstop = np.mean(train_qs.apply(lambda x: '.' in x))
capital_first = np.mean(train_qs.apply(lambda x: x[0].isupper()))
capitals = np.mean(train_qs.apply(lambda x: max([y.isupper() for y in x])))
 numbers = np.mean(train_qs.apply(lambda x: max([y.isdigit() for y in x])))
 print('Questions with question marks: {:.2f}%'.format(qmarks * 100))
print('Questions with [math] tags: {:.2f}%'.format(math * 100))
print('Questions with full stops: {:.2f}%'.format(fullstop * 100))
print('Questions with capitalised first letters: {:.2f}%'.format(capital_first * 100))
print('Questions with capital letters: {:.2f}%'.format(capitals * 100))
print('Questions with numbers: {:.2f}%'.format(numbers * 100))
all = []
counter = 0
for k in range(input_ids_t.shape[0]):
start_sorted_indexes = np.argsort(-preds_start[k,], kind='quicksort', order=None)
 end_sorted_indexes = np.argsort(-preds_end[k,], kind='quicksort', order=None)
 if start_sorted_indexes[0] <= end_sorted_indexes[0]:
a = start_sorted_indexes[0]
b = end_sorted_indexes[0]
text1 = " "+" ".join(test_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 elif start_sorted_indexes[0] <= end_sorted_indexes[1]:
a = start_sorted_indexes[0]
b = end_sorted_indexes[1]
text1 = " "+" ".join(test_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 elif start_sorted_indexes[1] <= end_sorted_indexes[0]:
a = start_sorted_indexes[1]
b = end_sorted_indexes[0]
text1 = " "+" ".join(test_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 elif start_sorted_indexes[1] <= end_sorted_indexes[1]:
a = start_sorted_indexes[1]
b = end_sorted_indexes[1]
text1 = " "+" ".join(test_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 else:
counter += 1
st = test_df.loc[k,'text']
all.append(st)
 print(counter, " row cant predicted on test set" )<save_to_csv>
stops = set(stopwords.words("english"))
def word_match_share(row):
q1words = {}
q2words = {}
for word in str(row['question1'] ).lower().split() :
if word not in stops:
q1words[word] = 1
for word in str(row['question2'] ).lower().split() :
if word not in stops:
q2words[word] = 1
if len(q1words)== 0 or len(q2words)== 0:
return 0
shared_words_in_q1 = [w for w in q1words.keys() if w in q2words]
shared_words_in_q2 = [w for w in q2words.keys() if w in q1words]
R =(len(shared_words_in_q1)+ len(shared_words_in_q2)) /(len(q1words)+ len(q2words))
return R
plt.figure(figsize=(15, 5))
train_word_match = df_train.apply(word_match_share, axis=1, raw=True)
 plt.hist(train_word_match[df_train['is_duplicate'] == 0], bins=20, normed=True, label='Not Duplicate')
 plt.hist(train_word_match[df_train['is_duplicate'] == 1], bins=20, normed=True, alpha=0.7, label='Duplicate')
 plt.legend()
plt.title('Label distribution over word_match_share', fontsize=15)
 plt.xlabel('word_match_share', fontsize=15 )
test_df['selected_text'] = all
test_df[['textID','selected_text']].to_csv('submission.csv',index=False )<predict_on_test>
def get_weight(count, eps=10000, min_count=2):
if count < min_count:
return 0
else:
return 1 /(count + eps)
 eps = 5000
words =(" ".join(train_qs)).lower().split()
counts = Counter(words)
 weights = {word: get_weight(count)for word, count in counts.items() }
train_preds_start = np.zeros(( input_ids.shape[0],MAX_LEN))
train_preds_end = np.zeros(( input_ids.shape[0],MAX_LEN))
DISPLAY=1
for i in range(n_splits):
print('
print('
print('
K.clear_session()
model = build_model()
model.load_weights('./v1-roberta-%i.h5'%(i))
print('Predicting Train...')
 train_preds = model.predict([input_ids,attention_mask,token_type_ids],verbose=DISPLAY)
 train_preds_start += train_preds[0]/n_splits
train_preds_end += train_preds[1]/n_splits<string_transform>
print('Most common words and weights:
')
 print(sorted(weights.items() , key=lambda x: x[1] if x[1] > 0 else 9999)[:10])
 print('
Least common words and weights: ')
(sorted(weights.items() , key=lambda x: x[1], reverse=True)[:10] )
all = []
counter = 0
for k in range(input_ids.shape[0]):
start_sorted_indexes = np.argsort(-train_preds_start[k,], kind='quicksort', order=None)
 end_sorted_indexes = np.argsort(-train_preds_end[k,], kind='quicksort', order=None)
 if start_sorted_indexes[0] <= end_sorted_indexes[0]:
a = start_sorted_indexes[0]
b = end_sorted_indexes[0]
text1 = " "+" ".join(train_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 elif start_sorted_indexes[0] <= end_sorted_indexes[1]:
a = start_sorted_indexes[0]
b = end_sorted_indexes[1]
text1 = " "+" ".join(train_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 elif start_sorted_indexes[1] <= end_sorted_indexes[0]:
a = start_sorted_indexes[1]
b = end_sorted_indexes[0]
text1 = " "+" ".join(train_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 elif start_sorted_indexes[1] <= end_sorted_indexes[1]:
a = start_sorted_indexes[1]
b = end_sorted_indexes[1]
text1 = " "+" ".join(train_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 else:
counter += 1
st = train_df.loc[k,'text']
all.append(st)
 print(counter, " row cant predicted on train set" )<feature_engineering>
def tfidf_word_match_share(row):
q1words = {}
q2words = {}
for word in str(row['question1'] ).lower().split() :
if word not in stops:
q1words[word] = 1
for word in str(row['question2'] ).lower().split() :
if word not in stops:
q2words[word] = 1
if len(q1words)== 0 or len(q2words)== 0:
return 0
shared_weights = [weights.get(w, 0)for w in q1words.keys() if w in q2words] + [weights.get(w, 0)for w in q2words.keys() if w in q1words]
total_weights = [weights.get(w, 0)for w in q1words] + [weights.get(w, 0)for w in q2words]
R = np.sum(shared_weights)/ np.sum(total_weights)
 return R
train_df['predicted_text'] = all<predict_on_test>
print('Original AUC:', roc_auc_score(df_train['is_duplicate'], train_word_match))
print(' TFIDF AUC:', roc_auc_score(df_train['is_duplicate'], tfidf_train_word_match.fillna(0)) )
validation_preds_start = np.zeros(( input_ids_v.shape[0],MAX_LEN))
validation_preds_end = np.zeros(( input_ids_v.shape[0],MAX_LEN))
DISPLAY=1
for i in range(n_splits):
print('
print('
print('
K.clear_session()
model = build_model()
model.load_weights('./v1-roberta-%i.h5'%(i))
print('Predicting Validation...')
 validation_preds = model.predict([input_ids_v,attention_mask_v,token_type_ids_v],verbose=DISPLAY)
 validation_preds_start += validation_preds[0]/n_splits
validation_preds_end += validation_preds[1]/n_splits<string_transform>
x_train = pd.DataFrame()
x_test = pd.DataFrame()
x_train['word_match'] = train_word_match
x_train['tfidf_word_match'] = tfidf_train_word_match
x_test['word_match'] = df_test.apply(word_match_share, axis=1, raw=True)
 x_test['tfidf_word_match'] = df_test.apply(tfidf_word_match_share, axis=1, raw=True)
 y_train = df_train['is_duplicate'].values
all = []
counter = 0
for k in range(input_ids_v.shape[0]):
start_sorted_indexes = np.argsort(-validation_preds_start[k,], kind='quicksort', order=None)
 end_sorted_indexes = np.argsort(-validation_preds_end[k,], kind='quicksort', order=None)
 if start_sorted_indexes[0] <= end_sorted_indexes[0]:
a = start_sorted_indexes[0]
b = end_sorted_indexes[0]
text1 = " "+" ".join(validation_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 elif start_sorted_indexes[0] <= end_sorted_indexes[1]:
a = start_sorted_indexes[0]
b = end_sorted_indexes[1]
text1 = " "+" ".join(validation_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 elif start_sorted_indexes[1] <= end_sorted_indexes[0]:
a = start_sorted_indexes[1]
b = end_sorted_indexes[0]
text1 = " "+" ".join(validation_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 elif start_sorted_indexes[1] <= end_sorted_indexes[1]:
a = start_sorted_indexes[1]
b = end_sorted_indexes[1]
text1 = " "+" ".join(validation_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 else:
counter += 1
st = validation_df.loc[k,'text']
all.append(st)
 print(counter, " row cant predicted on validation set" )<feature_engineering>
pos_train = x_train[y_train == 1]
neg_train = x_train[y_train == 0]
p = 0.165
scale =(( len(pos_train)/(len(pos_train)+ len(neg_train)))/ p)- 1
while scale > 1:
neg_train = pd.concat([neg_train, neg_train])
 scale -=1
neg_train = pd.concat([neg_train, neg_train[:int(scale * len(neg_train)) ]])
 print(len(pos_train)/(len(pos_train)+ len(neg_train)))
 x_train = pd.concat([pos_train, neg_train])
 y_train =(np.zeros(len(pos_train)) + 1 ).tolist() + np.zeros(len(neg_train)).tolist()
del pos_train, neg_train
validation_df['predicted_text'] = all<compute_test_metric>
x_train, x_valid, y_train, y_valid = train_test_split(x_train, y_train, test_size=0.2, random_state=4242 )
cumulativeJaccard = []
for index, row in validation_df.iterrows() :
cumulativeJaccard.append(jaccard(row["predicted_text"],row['selected_text']))
print("Roberta Jaccard on validation = ", np.mean(cumulativeJaccard), " Expected value is ~0.711" )<string_transform>
params = {}
params['objective'] = 'binary:logistic'
params['eval_metric'] = 'logloss'
params['eta'] = 0.02
params['max_depth'] = 4
d_train = xgb.DMatrix(x_train, label=y_train)
 d_valid = xgb.DMatrix(x_valid, label=y_valid)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
bst = xgb.train(params, d_train, 400, watchlist, early_stopping_rounds=50, verbose_eval=10 )
text_se = []
selected_text_se = []
sentiment_se = []
position = []
for index, row in train_df.iterrows() :
text = " "+" ".join(row['text'].split())
 selected_text = " ".join(row["selected_text"].split())
 selected_text_start_word = selected_text.split() [0]
selected_text_end_word = selected_text.split() [-1]
selected_text_start_char = text.find(selected_text)
 selected_text_end_char = selected_text_start_char + len(selected_text)
 selected_text_full_start_word = None
selected_text_full_end_word = None
char_counter = 0
for word in row["text"].split() :
char_counter += len(word)+1
if selected_text_start_char < char_counter:
selected_text_full_start_word = word
break
char_counter = 0
for word in row["text"].split() :
char_counter += len(word)+1
if selected_text_end_char <= char_counter:
selected_text_full_end_word = word
break
if row["sentiment"] == "neutral":
position.append("Start-Neutral")
 text_se.append(selected_text_full_start_word)
 selected_text_se.append(selected_text_start_word)
 position.append("End-Neutral")
 text_se.append(selected_text_full_end_word)
 selected_text_se.append(selected_text_end_word)
 sentiment_se.append(row["sentiment"])
 sentiment_se.append(row["sentiment"])
 else:
position.append("Start")
 text_se.append(selected_text_full_start_word)
 selected_text_se.append(selected_text_start_word)
 position.append("End")
 text_se.append(selected_text_full_end_word)
 selected_text_se.append(selected_text_end_word)
 sentiment_se.append(row["sentiment"])
 sentiment_se.append(row["sentiment"])
 train_df_se = pd.DataFrame(data={"text_se":text_se, "selected_text_se":selected_text_se, "sentiment":sentiment_se, "position":position})
 train_df_se<compute_test_metric>
d_test = xgb.DMatrix(x_test)
 p_test = bst.predict(d_test)
 sub = pd.DataFrame()
sub['test_id'] = df_test['test_id']
sub['is_duplicate'] = p_test
sub.to_csv('simple_xgb.csv', index=False )