Kaggle competition: Quora Question Pairs
<SOS> metric: LogLoss Kaggle data source: quora-question-pairs<feature_engineering>
stops = eng_stopwords = set(stopwords.words('english'))
color = sns.color_palette()
%matplotlib inline
pd.options.mode.chained_assignment = None
class TweetDataset:
def __init__(self, tweet, sentiment, selected_text):
self.tweet = tweet
self.sentiment = sentiment
self.selected_text = selected_text
self.tokenizer = config.TOKENIZER
self.max_len = config.MAX_LEN
def __len__(self):
return len(self.tweet)
 def __getitem__(self, item):
data = process_data(
 self.tweet[item],
self.selected_text[item],
self.sentiment[item],
self.tokenizer,
self.max_len 
)
 return {
'ids': torch.tensor(data["ids"], dtype=torch.long),
'mask': torch.tensor(data["mask"], dtype=torch.long),
'token_type_ids': torch.tensor(data["token_type_ids"], dtype=torch.long),
'targets_start': torch.tensor(data["targets_start"], dtype=torch.long),
'targets_end': torch.tensor(data["targets_end"], dtype=torch.long),
'orig_tweet': data["orig_tweet"],
'orig_selected': data["orig_selected"],
'sentiment': data["sentiment"],
'offsets': torch.tensor(data["offsets"], dtype=torch.long)
 }<load_pretrained>
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print(train_df.shape)
 print(test_df.shape )
class TweetModel(transformers.BertPreTrainedModel):
def __init__(self, conf):
super(TweetModel, self ).__init__(conf)
 self.roberta = transformers.RobertaModel.from_pretrained(config.ROBERTA_PATH, config=conf)
 self.drop_out = nn.Dropout(0.1)

self.l0 = nn.Linear(768 * 2, 2)
 torch.nn.init.normal_(self.l0.weight, std=0.02)
 def forward(self, ids, mask, token_type_ids):
_, _, out = self.roberta(
 ids,
attention_mask=mask,
token_type_ids=token_type_ids 
)

out = torch.cat(( out[-1], out[-2]), dim=-1)
 out = self.drop_out(out)
 logits = self.l0(out)
 start_logits, end_logits = logits.split(1, dim=-1)
 start_logits = start_logits.squeeze(-1)
 end_logits = end_logits.squeeze(-1)
 return start_logits, end_logits<compute_test_metric>
train_qs = pd.Series(train_df['question1'].tolist() + train_df['question2'].tolist() ).astype(str)
 def word_match_share(row):
q1words = {}
q2words = {}
for word in str(row['question1'] ).lower().split() :
if word not in stops:
q1words[word] = 1
for word in str(row['question2'] ).lower().split() :
if word not in stops:
q2words[word] = 1
if len(q1words)== 0 or len(q2words)== 0:
return 0
shared_words_in_q1 = [w for w in q1words.keys() if w in q2words]
shared_words_in_q2 = [w for w in q2words.keys() if w in q1words]
R =(len(shared_words_in_q1)+ len(shared_words_in_q2)) /(len(q1words)+ len(q2words))
return R
def get_weight(count, eps=10000, min_count=2):
if count < min_count:
return 0
else:
return 1 /(count + eps)
 eps = 5000
words =(" ".join(train_qs)).lower().split()
counts = Counter(words)
 weights = {word: get_weight(count)for word, count in counts.items() }
def tfidf_word_match_share(row):
q1words = {}
q2words = {}
for word in str(row['question1'] ).lower().split() :
if word not in stops:
q1words[word] = 1
for word in str(row['question2'] ).lower().split() :
if word not in stops:
q2words[word] = 1
if len(q1words)== 0 or len(q2words)== 0:
return 0
shared_weights = [weights.get(w, 0)for w in q1words.keys() if w in q2words] + [weights.get(w, 0)for w in q2words.keys() if w in q1words]
total_weights = [weights.get(w, 0)for w in q1words] + [weights.get(w, 0)for w in q2words]
R = np.sum(shared_weights)/ np.sum(total_weights)
 return R
def diff_ratios(st1, st2):
seq = difflib.SequenceMatcher()
seq.set_seqs(str(st1 ).lower() , str(st2 ).lower())
 return seq.ratio()
def calculate_jaccard_score(
 original_tweet,
target_string,
sentiment_val,
idx_start,
idx_end,
offsets,
verbose=False):
if idx_end < idx_start:
idx_end = idx_start
filtered_output = ""

 for ix in range(idx_start, idx_end + 1):
filtered_output += original_tweet[offsets[ix][0]: offsets[ix][1]]
if(ix+1)< len(offsets)and offsets[ix][1] < offsets[ix+1][0]:
filtered_output += " "
if len(original_tweet.split())< 2:
filtered_output = original_tweet
jac = utils.jaccard(target_string.strip() , filtered_output.strip())
 return jac, filtered_output<load_from_csv>
def feature_extraction(row):
que1 = str(row['question1'])
 que2 = str(row['question2'])
 out_list = []
unigrams_que1 = [word for word in que1.lower().split() if word not in eng_stopwords]
unigrams_que2 = [word for word in que2.lower().split() if word not in eng_stopwords]
common_unigrams_len = len(set(unigrams_que1 ).intersection(set(unigrams_que2)))
 common_unigrams_ratio = float(common_unigrams_len)/ max(len(set(unigrams_que1 ).union(set(unigrams_que2))),1)
 out_list.extend([common_unigrams_len, common_unigrams_ratio])
 bigrams_que1 = [i for i in ngrams(unigrams_que1, 2)]
bigrams_que2 = [i for i in ngrams(unigrams_que2, 2)]
common_bigrams_len = len(set(bigrams_que1 ).intersection(set(bigrams_que2)))
 common_bigrams_ratio = float(common_bigrams_len)/ max(len(set(bigrams_que1 ).union(set(bigrams_que2))),1)
 out_list.extend([common_bigrams_len, common_bigrams_ratio])
 trigrams_que1 = [i for i in ngrams(unigrams_que1, 3)]
trigrams_que2 = [i for i in ngrams(unigrams_que2, 3)]
common_trigrams_len = len(set(trigrams_que1 ).intersection(set(trigrams_que2)))
 common_trigrams_ratio = float(common_trigrams_len)/ max(len(set(trigrams_que1 ).union(set(trigrams_que2))),1)
 out_list.extend([common_trigrams_len, common_trigrams_ratio])
 return out_list
df_test = pd.read_csv(".. /input/tweet-sentiment-extraction/test.csv")
 df_test.loc[:, "selected_text"] = df_test.text.values<load_pretrained>
data = pd.DataFrame()
data['match_ratio'] = train_df.apply(lambda r: diff_ratios(r.question1, r.question2), axis=1)
 print('6')
 data['word_match_share'] = train_df.apply(lambda row: word_match_share(row), axis=1)
 print('5')
 data['tfidf_word_match_share'] = train_df.apply(lambda row: tfidf_word_match_share(row), axis=1)
 z =
ROBERTA_PATH = ".. /input/roberta-base"
model_config = transformers.RobertaConfig.from_pretrained(ROBERTA_PATH)
 model_config.output_hidden_states = True
device = torch.device("cuda" )<load_pretrained>
data['fuzz_partial_ratio'] = train_df.apply(lambda x: fuzz.partial_ratio(str(x['question1']), str(x['question2'])) , axis=1)
 data['fuzz_token_set_ratio'] = train_df.apply(lambda x: fuzz.token_set_ratio(str(x['question1']), str(x['question2'])) , axis=1)
 data['fuzz_token_sort_ratio'] = train_df.apply(lambda x: fuzz.token_sort_ratio(str(x['question1']), str(x['question2'])) , axis=1)
 data = data.fillna(0)
 data.head()
%%time
model1 = TweetModel(conf=model_config)
 model1.to(device)
 model1.load_state_dict(torch.load(".. /input/train-tweet-fold/model_0.bin"))
model1.eval()
model2 = TweetModel(conf=model_config)
 model2.to(device)
 model2.load_state_dict(torch.load(".. /input/train-tweet-fold/model_1.bin"))
model2.eval()
model3 = TweetModel(conf=model_config)
 model3.to(device)
 model3.load_state_dict(torch.load(".. /input/train-tweet-fold/model_2.bin"))
model3.eval()
model4 = TweetModel(conf=model_config)
 model4.to(device)
 model4.load_state_dict(torch.load(".. /input/train-tweet-fold/model_3.bin"))
model4.eval()
model5 = TweetModel(conf=model_config)
 model5.to(device)
 model5.load_state_dict(torch.load(".. /input/train-tweet-fold/model_4.bin"))
model5.eval()<create_dataframe>
s1 = data.as_matrix()
s2 = np.vstack(np.array(train_df.apply(lambda row: feature_extraction(row), axis=1)))
 argh = pd.DataFrame(np.hstack(( s1, s2)))
 argh = argh.fillna(0)
 final = argh.as_matrix()
pd.isnull(argh ).any(1 ).nonzero() [0]
print(final )
final_output = []
TEST_BATCH_SIZE = 32
test_dataset = TweetDataset(
 tweet=df_test.text.values,
sentiment=df_test.sentiment.values,
selected_text=df_test.selected_text.values 
)
 data_loader = torch.utils.data.DataLoader(
 test_dataset,
shuffle=False,
batch_size=TEST_BATCH_SIZE,
num_workers=1 
)
 with torch.no_grad() :
tk0 = tqdm(data_loader, total=len(data_loader))
for bi, d in enumerate(tk0):
ids = d["ids"]
token_type_ids = d["token_type_ids"]
mask = d["mask"]
sentiment = d["sentiment"]
orig_selected = d["orig_selected"]
orig_tweet = d["orig_tweet"]
targets_start = d["targets_start"]
targets_end = d["targets_end"]
offsets = d["offsets"].numpy()
ids = ids.to(device, dtype=torch.long)
 token_type_ids = token_type_ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 targets_start = targets_start.to(device, dtype=torch.long)
 targets_end = targets_end.to(device, dtype=torch.long)
 outputs_start1, outputs_end1 = model1(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 outputs_start2, outputs_end2 = model2(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 outputs_start3, outputs_end3 = model3(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 outputs_start4, outputs_end4 = model4(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 outputs_start5, outputs_end5 = model5(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 outputs_start =(outputs_start1 + outputs_start2 + outputs_start3 + outputs_start4 +
outputs_start5)/ 5
outputs_end =(outputs_end1 + outputs_end2 + outputs_end3 + outputs_end4 +
outputs_end5)/ 5
outputs_start = torch.softmax(outputs_start, dim=1 ).cpu().detach().numpy()
outputs_end = torch.softmax(outputs_end, dim=1 ).cpu().detach().numpy()
jaccard_scores = []
for px, tweet in enumerate(orig_tweet):
selected_tweet = orig_selected[px]
tweet_sentiment = sentiment[px]
_, output_sentence = calculate_jaccard_score(
 original_tweet=tweet,
target_string=selected_tweet,
sentiment_val=tweet_sentiment,
idx_start=np.argmax(outputs_start[px, :]),
idx_end=np.argmax(outputs_end[px, :]),
offsets=offsets[px] 
)
 final_output.append(output_sentence )<save_to_csv>
ALPHA = 0.2
BETA = 0.45
ETA = 0.4
PHI = 0.2
DELTA = 0.85
brown_freqs = dict()
N = 0
def get_best_synset_pair(word_1, word_2):

 max_sim = -1.0
synsets_1 = wn.synsets(word_1)
 synsets_2 = wn.synsets(word_2)
 if len(synsets_1)== 0 or len(synsets_2)== 0:
return None, None
else:
max_sim = -1.0
best_pair = None, None
for synset_1 in synsets_1:
for synset_2 in synsets_2:
sim = wn.wup_similarity(synset_1, synset_2)
 if not sim:
continue
if sim > max_sim:
max_sim = sim
best_pair = synset_1, synset_2
return best_pair
def length_dist(synset_1, synset_2):

 l_dist = sys.maxsize
if synset_1 is None or synset_2 is None:
return 0.0
if synset_1 == synset_2:
l_dist = 0.0
else:
wset_1 = set([str(x.name())for x in synset_1.lemmas() ])
 wset_2 = set([str(x.name())for x in synset_2.lemmas() ])
 if len(wset_1.intersection(wset_2)) > 0:
l_dist = 1.0
else:
l_dist = synset_1.shortest_path_distance(synset_2)
 if l_dist is None:
l_dist = 0.0
return math.exp(-ALPHA * l_dist)
 def hierarchy_dist(synset_1, synset_2):

 h_dist = sys.maxsize
if synset_1 is None or synset_2 is None:
return h_dist
if synset_1 == synset_2:
h_dist = max([x[1] for x in synset_1.hypernym_distances() ])
 else:
hypernyms_1 = {x[0]:x[1] for x in synset_1.hypernym_distances() }
hypernyms_2 = {x[0]:x[1] for x in synset_2.hypernym_distances() }
lcs_candidates = set(hypernyms_1.keys() ).intersection(
 set(hypernyms_2.keys()))
if len(lcs_candidates)> 0:
lcs_dists = []
for lcs_candidate in lcs_candidates:
lcs_d1 = 0
if lcs_candidate in hypernyms_1:
lcs_d1 = hypernyms_1[lcs_candidate]
lcs_d2 = 0
if lcs_candidate in hypernyms_2:
lcs_d2 = hypernyms_2[lcs_candidate]
lcs_dists.append(max([lcs_d1, lcs_d2]))
h_dist = max(lcs_dists)
 else:
h_dist = 0
return(( math.exp(BETA * h_dist)- math.exp(-BETA * h_dist)) / 
(math.exp(BETA * h_dist)+ math.exp(-BETA * h_dist)))
 def word_similarity(word_1, word_2):
synset_pair = get_best_synset_pair(word_1, word_2)
 return(length_dist(synset_pair[0], synset_pair[1])*
hierarchy_dist(synset_pair[0], synset_pair[1]))
def most_similar_word(word, word_set):

 max_sim = -1.0
sim_word = ""
for ref_word in word_set:
sim = word_similarity(word, ref_word)
 if sim > max_sim:
max_sim = sim
sim_word = ref_word
return sim_word, max_sim
def info_content(lookup_word):

 global N
if N == 0:
for sent in brown.sents() :
for word in sent:
word = word.lower()
if word not in brown_freqs:
brown_freqs[word] = 0
brown_freqs[word] = brown_freqs[word] + 1
N = N + 1
lookup_word = lookup_word.lower()
n = 0 if lookup_word not in brown_freqs else brown_freqs[lookup_word]
return 1.0 -(math.log(n + 1)/ math.log(N + 1))
def semantic_vector(words, joint_words, info_content_norm):

 sent_set = set(words)
 semvec = np.zeros(len(joint_words))
i = 0
for joint_word in joint_words:
if joint_word in sent_set:
semvec[i] = 1.0
if info_content_norm:
semvec[i] = semvec[i] * math.pow(info_content(joint_word), 2)
 else:
sim_word, max_sim = most_similar_word(joint_word, sent_set)
 semvec[i] = PHI if max_sim > PHI else 0.0
if info_content_norm:
semvec[i] = semvec[i] * info_content(joint_word)* info_content(sim_word)
 i = i + 1
return semvec
def semantic_similarity(sentence_1, sentence_2, info_content_norm):

 words_1 = nltk.word_tokenize(sentence_1)
 words_2 = nltk.word_tokenize(sentence_2)
 joint_words = set(words_1 ).union(set(words_2))
vec_1 = semantic_vector(words_1, joint_words, info_content_norm)
 vec_2 = semantic_vector(words_2, joint_words, info_content_norm)
 return np.dot(vec_1, vec_2.T)/(np.linalg.norm(vec_1)* np.linalg.norm(vec_2))
def word_order_vector(words, joint_words, windex):

 wovec = np.zeros(len(joint_words))
i = 0
wordset = set(words)
 for joint_word in joint_words:
if joint_word in wordset:
wovec[i] = windex[joint_word]
else:
sim_word, max_sim = most_similar_word(joint_word, wordset)
 if max_sim > ETA:
wovec[i] = windex[sim_word]
else:
wovec[i] = 0
i = i + 1
return wovec
def word_order_similarity(sentence_1, sentence_2):

 words_1 = nltk.word_tokenize(sentence_1)
 words_2 = nltk.word_tokenize(sentence_2)
 joint_words = list(set(words_1 ).union(set(words_2)))
 windex = {x[1]: x[0] for x in enumerate(joint_words)}
r1 = word_order_vector(words_1, joint_words, windex)
 r2 = word_order_vector(words_2, joint_words, windex)
 return 1.0 -(np.linalg.norm(r1 - r2)/ np.linalg.norm(r1 + r2))
def similarity(sentence_1, sentence_2, info_content_norm):

 return DELTA * semantic_similarity(sentence_1, sentence_2, info_content_norm)+ \ 
(1.0 - DELTA)* word_order_similarity(sentence_1, sentence_2 )
sample = pd.read_csv(".. /input/tweet-sentiment-extraction/sample_submission.csv")
 sample.loc[:, 'selected_text'] = final_output
sample.to_csv("submission.csv", index=False)
 sample.head()<import_modules>
for i in [5,7,11,12,13,15,16,18]:
print(train_df['question1'][i])
 print(train_df['question2'][i])
 print(train_df['is_duplicate'][i])
 print(similarity(train_df['question1'][i], train_df['question2'][i], False))
print(similarity(train_df['question1'][i], train_df['question2'][i], True))
import numpy as np
import pandas as pd
import os
import re
import string
import tokenizers
import transformers
import torch
import torch.nn as nn
from sklearn import model_selection
from torch.nn import functional as F
from tqdm import tqdm
from transformers import AdamW
from transformers import get_linear_schedule_with_warmup<define_variables>
def runXGB(train_X, train_y, test_X, test_y=None, feature_names=None, seed_val=0):
params = {}
params["objective"] = "binary:logistic"
params['eval_metric'] = 'logloss'
params["eta"] = 0.02
params["subsample"] = 0.7
params["min_child_weight"] = 1
params["colsample_bytree"] = 0.7
params["max_depth"] = 4
params["silent"] = 1
params["seed"] = seed_val
num_rounds = 250
plst = list(params.items())
 xgtrain = xgb.DMatrix(train_X, label=train_y)
 if test_y is not None:
xgtest = xgb.DMatrix(test_X, label=test_y)
 watchlist = [(xgtrain,'train'),(xgtest, 'test')]
model = xgb.train(plst, xgtrain, num_rounds, watchlist, early_stopping_rounds=100, verbose_eval=10)
 else:
xgtest = xgb.DMatrix(test_X)
 model = xgb.train(plst, xgtrain, num_rounds)
 pred_test_y = model.predict(xgtest)
 loss = 1
if test_y is not None:
loss = log_loss(test_y, pred_test_y)
 return pred_test_y, loss, model
else:
return pred_test_y, loss, model
MAX_LEN = 192
TEST_BATCH_SIZE = 32
EPOCHS = 5
ROBERTA_PATH = ".. /input/roberta-base"
TOKENIZER = tokenizers.ByteLevelBPETokenizer(
 vocab_file=f"{ROBERTA_PATH}/vocab.json",
merges_file=f"{ROBERTA_PATH}/merges.txt",
lowercase=True,
add_prefix_space=True 
)
 DEVICE = 'cuda'
TEST_PATH = '.. /input/tweet-sentiment-extraction/test.csv'
FOLDS = 5<load_from_csv>
GradientBoostingClassifier)
 def fit(X_train, y_train, X_test, y_test):
n_estimator = 100
rt = RandomTreesEmbedding(max_depth=3, n_estimators=n_estimator)
 rt_lm = LogisticRegression()
pipeline = make_pipeline(rt, rt_lm)
 rf = RandomForestClassifier(max_depth=3, n_estimators=n_estimator)
 rf_enc = OneHotEncoder()
rf_lm = LogisticRegression()
grd = GradientBoostingClassifier(n_estimators=n_estimator)
 grd_enc = OneHotEncoder()
grd_lm = LogisticRegression()
X_train, X_train_lr, y_train, y_train_lr = train_test_split(X_train,
y_train,
test_size=0.5)
 print('random trees + lt')
 pipeline.fit(X_train, y_train)
 y_pred_rt = pipeline.predict_proba(X_test)[:, 1]
fpr_rt_lm, tpr_rt_lm, _ = roc_curve(y_test, y_pred_rt)
 print(log_loss(y_test, pipeline.predict_proba(X_test)))
 print('random forest + lt')
 rf.fit(X_train, y_train)
 rf_enc.fit(rf.apply(X_train))
rf_lm.fit(rf_enc.transform(rf.apply(X_train_lr)) , y_train_lr)
 y_pred_rf_lm = rf_lm.predict_proba(rf_enc.transform(rf.apply(X_test)))[:, 1]
fpr_rf_lm, tpr_rf_lm, _ = roc_curve(y_test, y_pred_rf_lm)
 print(log_loss(y_test, rf_lm.predict_proba(rf_enc.transform(rf.apply(X_test)))))
 print('gradient boosting + lt')
 grd.fit(X_train, y_train)
 grd_enc.fit(grd.apply(X_train)[:, :, 0])
 grd_lm.fit(grd_enc.transform(grd.apply(X_train_lr)[:, :, 0]), y_train_lr)
 y_pred_grd_lm = grd_lm.predict_proba(
 grd_enc.transform(grd.apply(X_test)[:, :, 0])) [:, 1]
fpr_grd_lm, tpr_grd_lm, _ = roc_curve(y_test, y_pred_grd_lm)
 print(log_loss(y_test, grd_lm.predict_proba(
 grd_enc.transform(grd.apply(X_test)[:, :, 0]))))
print('gradient boosting')
 y_pred_grd = grd.predict_proba(X_test)[:, 1]
fpr_grd, tpr_grd, _ = roc_curve(y_test, y_pred_grd)
 print(log_loss(y_test, grd.predict_proba(X_test)))
 print('random forest')
 y_pred_rf = rf.predict_proba(X_test)[:, 1]
fpr_rf, tpr_rf, _ = roc_curve(y_test, y_pred_rf)
 print(log_loss(y_test, rf.predict_proba(X_test)))
 plt.figure(1)
 plt.plot([0, 1], [0, 1], 'k--')
 plt.plot(fpr_rt_lm, tpr_rt_lm, label='RT + LR')
 plt.plot(fpr_rf, tpr_rf, label='RF')
 plt.plot(fpr_rf_lm, tpr_rf_lm, label='RF + LR')
 plt.plot(fpr_grd, tpr_grd, label='GBT')
 plt.plot(fpr_grd_lm, tpr_grd_lm, label='GBT + LR')
 plt.xlabel('False positive rate')
 plt.ylabel('True positive rate')
 plt.title('ROC curve')
 plt.legend(loc='best')
 plt.show()
test_df = pd.read_csv(TEST_PATH)
 test_df<init_hyperparams>
GradientBoostingClassifier)
 def fit(X_train, y_train, X_test, y_test):
ens = pd.DataFrame()
n_estimator = 100
rt = RandomTreesEmbedding(max_depth=3, n_estimators=n_estimator)
 rt_lm = LogisticRegression()
pipeline = make_pipeline(rt, rt_lm)
 rf = RandomForestClassifier(max_depth=3, n_estimators=n_estimator)
 rf_enc = OneHotEncoder()
rf_lm = LogisticRegression()
grd = GradientBoostingClassifier(n_estimators=n_estimator)
 grd_enc = OneHotEncoder()
grd_lm = LogisticRegression()
X_train, X_train_lr, y_train, y_train_lr = train_test_split(X_train,
y_train,
test_size=0.5)
 print('random trees + lt')
 pipeline.fit(X_train, y_train)
 y_pred_rt = pipeline.predict_proba(X_test)[:, 1]
fpr_rt_lm, tpr_rt_lm, _ = roc_curve(y_test, y_pred_rt)
 ens['rt_lt'] = pipeline.predict(X_test)
 print(log_loss(y_test, pipeline.predict_proba(X_test)))
 print('random forest + lt')
 rf.fit(X_train, y_train)
 rf_enc.fit(rf.apply(X_train))
rf_lm.fit(rf_enc.transform(rf.apply(X_train_lr)) , y_train_lr)
 y_pred_rf_lm = rf_lm.predict_proba(rf_enc.transform(rf.apply(X_test)))[:, 1]
fpr_rf_lm, tpr_rf_lm, _ = roc_curve(y_test, y_pred_rf_lm)
 ens['rf_lm'] = rf_lm.predict(rf_enc.transform(rf.apply(X_test)))
 print(log_loss(y_test, rf_lm.predict_proba(rf_enc.transform(rf.apply(X_test)))))
 print('gradient boosting + lt')
 grd.fit(X_train, y_train)
 grd_enc.fit(grd.apply(X_train)[:, :, 0])
 grd_lm.fit(grd_enc.transform(grd.apply(X_train_lr)[:, :, 0]), y_train_lr)
 y_pred_grd_lm = grd_lm.predict_proba(
 grd_enc.transform(grd.apply(X_test)[:, :, 0])) [:, 1]
fpr_grd_lm, tpr_grd_lm, _ = roc_curve(y_test, y_pred_grd_lm)
 ens['grd_lm'] = grd_lm.predict(grd_enc.transform(grd.apply(X_test)[:, :, 0]))
print(log_loss(y_test, grd_lm.predict_proba(
 grd_enc.transform(grd.apply(X_test)[:, :, 0]))))
print('gradient boosting')
 y_pred_grd = grd.predict_proba(X_test)[:, 1]
fpr_grd, tpr_grd, _ = roc_curve(y_test, y_pred_grd)
 ens['grd'] = grd.predict(X_test)
 print(log_loss(y_test, grd.predict_proba(X_test)))
 print('random forest')
 y_pred_rf = rf.predict_proba(X_test)[:, 1]
fpr_rf, tpr_rf, _ = roc_curve(y_test, y_pred_rf)
 ens['rf'] = rf.predict(X_test)
 print(log_loss(y_test, rf.predict_proba(X_test)))
 return ens
class AverageMeter:

 def __init__(self):
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count
class EarlyStopping:
def __init__(self, patience=7, mode="max", delta=0.001):
self.patience = patience
self.counter = 0
self.mode = mode
self.best_score = None
self.early_stop = False
self.delta = delta
if self.mode == "min":
self.val_score = np.Inf
else:
self.val_score = -np.Inf
def __call__(self, epoch_score, model, model_path):
if self.mode == "min":
score = -1.0 * epoch_score
else:
score = np.copy(epoch_score)
 if self.best_score is None:
self.best_score = score
self.save_checkpoint(epoch_score, model, model_path)
 elif score < self.best_score + self.delta:
self.counter += 1
print('EarlyStopping counter: {} out of {}'.format(self.counter, self.patience))
if self.counter >= self.patience:
self.early_stop = True
else:
self.best_score = score
self.save_checkpoint(epoch_score, model, model_path)
 self.counter = 0
def save_checkpoint(self, epoch_score, model, model_path):
if epoch_score not in [-np.inf, np.inf, -np.nan, np.nan]:
print('Validation score improved({} --> {} ).Saving model!'.format(self.val_score, epoch_score))
torch.save(model.state_dict() , model_path)
 self.val_score = epoch_score
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))<load_pretrained>
train_X = data.as_matrix()
train_y = np.array(train_df["is_duplicate"])
 test_id = np.array(test_df["test_id"] )
class TweetModel(transformers.BertPreTrainedModel):
def __init__(self, conf):
super(TweetModel, self ).__init__(conf)
 config = transformers.RobertaConfig.from_pretrained(
 '.. /input/roberta-base/config.json', output_hidden_states=True)
 self.roberta = transformers.RobertaModel.from_pretrained(
 '.. /input/roberta-base/pytorch_model.bin', config=config)
 self.drop_out = nn.Dropout(0.3)
 self.fc = nn.Linear(768, 2)
 nn.init.normal_(self.fc.weight, std=0.02)
 nn.init.normal_(self.fc.bias, 0)
 def forward(self, ids, mask):
_, _, out = self.roberta(
 ids,
attention_mask = mask 
)
 out = torch.stack([out[-1], out[-2], out[-3], out[-4]])
 out = torch.mean(out, 0)
 out = self.drop_out(out)
 logits = self.fc(out)
 start_logits, end_logits = logits.split(1, dim = -1)
 start_logits = start_logits.squeeze(-1)
 end_logits = end_logits.squeeze(-1)
 return start_logits, end_logits<categorify>
train_X_dup = train_X[train_y==1]
train_X_non_dup = train_X[train_y==0]
train_X = np.vstack([train_X_non_dup, train_X_dup, train_X_non_dup, train_X_non_dup])
 train_y = np.array([0]*train_X_non_dup.shape[0] + [1]*train_X_dup.shape[0] + [0]*train_X_non_dup.shape[0] + [0]*train_X_non_dup.shape[0])
 del train_X_dup
del train_X_non_dup
print("Mean target rate : ",train_y.mean() )
class TweetDataset:
def __init__(self, tweet, sentiment):
self.tweet = tweet
self.sentiment = sentiment
self.tokenizer = TOKENIZER
self.max_len = MAX_LEN
def __len__(self):
return len(self.tweet)
 def __getitem__(self, item):
data = process_data(
 self.tweet[item],
self.sentiment[item],
self.tokenizer,
self.max_len 
)
 return {
'ids': torch.tensor(data['ids'], dtype = torch.long),
'mask': torch.tensor(data['mask'], dtype = torch.long),
'token_type_ids': torch.tensor(data['token_type_ids'], dtype = torch.long),
'orig_tweet': data['orig_tweet'],
'sentiment': data['sentiment'],
'offsets': torch.tensor(data['offsets'], dtype = torch.long)
 }<split>
kf = KFold(n_splits=5, shuffle=True)
 for dev_index, val_index in kf.split(range(train_X.shape[0])) :
dev_X, val_X = train_X[dev_index,:], train_X[val_index,:]
dev_y, val_y = train_y[dev_index], train_y[val_index]
ens = fit(dev_X, dev_y, val_X, val_y)
 ens['avg'] = ens.mean(axis=1)
 ens['median'] = ens.median(axis=1)
 break
def calculate_jaccard_score(
 original_tweet,
sentiment_val,
idx_start,
idx_end,
offsets,
verbose=False):
if idx_end < idx_start:
idx_end = idx_start
filtered_output = ""
if sentiment_val == "neutral" or len(original_tweet.split())< 2:
filtered_output = original_tweet
else:
for ix in range(idx_start, idx_end + 1):
filtered_output += original_tweet[offsets[ix][0]: offsets[ix][1]]
if(ix+1)< len(offsets)and offsets[ix][1] < offsets[ix+1][0]:
filtered_output += " "
return filtered_output<load_pretrained>
print(ens.mean(axis=0))
print(accuracy_score(ens['median'].as_matrix().astype(int),val_y))
model_config = transformers.RobertaConfig.from_pretrained(ROBERTA_PATH)
 model_config.output_hidden_states = True
model1 = TweetModel(conf=model_config)
 model1.to(DEVICE)
 model1.load_state_dict(torch.load(".. /input/roberta-sentiment-extraction/model_0.bin"))
model1.eval()
model2 = TweetModel(conf=model_config)
 model2.to(DEVICE)
 model2.load_state_dict(torch.load(".. /input/roberta-sentiment-extraction/model_1.bin"))
model2.eval()
model3 = TweetModel(conf=model_config)
 model3.to(DEVICE)
 model3.load_state_dict(torch.load(".. /input/roberta-sentiment-extraction/model_2.bin"))
model3.eval()
model4 = TweetModel(conf=model_config)
 model4.to(DEVICE)
 model4.load_state_dict(torch.load(".. /input/roberta-sentiment-extraction/model_3.bin"))
model4.eval()
model5 = TweetModel(conf=model_config)
 model5.to(DEVICE)
 model5.load_state_dict(torch.load(".. /input/roberta-sentiment-extraction/model_4.bin"))
model5.eval()<categorify>
testz = pd.DataFrame()
testz['word_match_share'] = test_df.apply(lambda row: word_match_share(row), axis=1)
 print('5')
 testz['tfidf_word_match_share'] = test_df.apply(lambda row: tfidf_word_match_share(row), axis=1)
 print('0')
 testz['len_q1'] = test_df.question1.apply(lambda x: len(str(x)))
 testz['len_q2'] = test_df.question2.apply(lambda x: len(str(x)))
 testz['diff_len'] = data.len_q1 - data.len_q2
print('1')
 testz['len_char_q1'] = test_df.question1.apply(lambda x: len(''.join(set(str(x ).replace(' ', '')))))
 testz['len_char_q2'] = test_df.question2.apply(lambda x: len(''.join(set(str(x ).replace(' ', '')))))
 testz['len_word_q1'] = test_df.question1.apply(lambda x: len(str(x ).split()))
testz['len_word_q2'] = test_df.question2.apply(lambda x: len(str(x ).split()))
print('2')
 testz['common_words'] = test_df.apply(lambda x: len(set(str(x['question1'] ).lower().split() ).intersection(set(str(x['question2'] ).lower().split()))), axis=1)
 print('3' )
def eval_fn(data_loader, device):
final_outputs = []
with torch.no_grad() :
tk0 = tqdm(data_loader, total = len(data_loader))
for bi, d in enumerate(tk0):
ids = d["ids"]
mask = d["mask"]
token_type_ids = d["token_type_ids"]
orig_tweet = d["orig_tweet"]
sentiment = d["sentiment"]
offsets = d["offsets"]
ids = ids.to(device, dtype=torch.long)
 token_type_ids = token_type_ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 predicted_start1, predicted_end1 = model1(ids, mask)
 predicted_start2, predicted_end2 = model2(ids, mask)
 predicted_start3, predicted_end3 = model3(ids, mask)
 predicted_start4, predicted_end4 = model4(ids, mask)
 predicted_start5, predicted_end5 = model5(ids, mask)
 predicted_start =(predicted_start1 + predicted_start2 + predicted_start3 + predicted_start4 + predicted_start5)/ 5
predicted_end =(predicted_end1 + predicted_end2 + predicted_end3 + predicted_end4 + predicted_end5)/ 5
predicted_start = torch.softmax(predicted_start, dim = 1 ).cpu().detach().numpy()
predicted_end = torch.softmax(predicted_end, dim = 1 ).cpu().detach().numpy()
for i, tweet in enumerate(orig_tweet):
tweet_sentiment = sentiment[i]
output_sentence = calculate_jaccard_score(
 original_tweet = tweet,
sentiment_val = tweet_sentiment,
idx_start = np.argmax(predicted_start[i, :]),
idx_end = np.argmax(predicted_end[i, :]),
offsets = offsets[i] 
)
 final_outputs.append(output_sentence)
 return final_outputs<predict_on_test>
zzz=
def predict() :
df_test = test_df
test_dataset = TweetDataset(
 tweet = df_test.text.values,
sentiment = df_test.sentiment.values 
)
 test_dataloader = torch.utils.data.DataLoader(
 test_dataset,
batch_size = TEST_BATCH_SIZE,
num_workers = 0 
)
 outputs = eval_fn(test_dataloader, DEVICE)
 sample = pd.read_csv(".. /input/tweet-sentiment-extraction/sample_submission.csv")
 sample.loc[:, 'selected_text'] = outputs
sample.to_csv("submission.csv", index=False )<predict_on_test>
s3 = testz.as_matrix()
s4 = np.vstack(np.array(test_df.apply(lambda row: feature_extraction(row), axis=1)))
 print(s3.shape, s4.shape)
 final_test = np.hstack(( s3, s4))
predict()<set_options>
xgtest = xgb.DMatrix(final_test)
 preds = model.predict(xgtest)
 out_df = pd.DataFrame({"test_id":test_id, "is_duplicate":preds})
 out_df.to_csv("idkhowtosubmit.csv", index=False )