Kaggle competition: TensorFlow 2.0 Question Answering
GradientBoostingClassifier)
 def fit(X_train, y_train, X_test, y_test):
n_estimator = 100
rt = RandomTreesEmbedding(max_depth=3, n_estimators=n_estimator)
 rt_lm = LogisticRegression()
pipeline = make_pipeline(rt, rt_lm)
 rf = RandomForestClassifier(max_depth=3, n_estimators=n_estimator)
 rf_enc = OneHotEncoder()
rf_lm = LogisticRegression()
grd = GradientBoostingClassifier(n_estimators=n_estimator)
 grd_enc = OneHotEncoder()
grd_lm = LogisticRegression()
X_train, X_train_lr, y_train, y_train_lr = train_test_split(X_train,
y_train,
test_size=0.5)
 print('random trees + lt')
 pipeline.fit(X_train, y_train)
 y_pred_rt = pipeline.predict_proba(X_test)[:, 1]
fpr_rt_lm, tpr_rt_lm, _ = roc_curve(y_test, y_pred_rt)
 print(log_loss(y_test, pipeline.predict_proba(X_test)))
 print('random forest + lt')
 rf.fit(X_train, y_train)
 rf_enc.fit(rf.apply(X_train))
rf_lm.fit(rf_enc.transform(rf.apply(X_train_lr)) , y_train_lr)
 y_pred_rf_lm = rf_lm.predict_proba(rf_enc.transform(rf.apply(X_test)))[:, 1]
fpr_rf_lm, tpr_rf_lm, _ = roc_curve(y_test, y_pred_rf_lm)
 print(log_loss(y_test, rf_lm.predict_proba(rf_enc.transform(rf.apply(X_test)))))
 print('gradient boosting + lt')
 grd.fit(X_train, y_train)
 grd_enc.fit(grd.apply(X_train)[:, :, 0])
 grd_lm.fit(grd_enc.transform(grd.apply(X_train_lr)[:, :, 0]), y_train_lr)
 y_pred_grd_lm = grd_lm.predict_proba(
 grd_enc.transform(grd.apply(X_test)[:, :, 0])) [:, 1]
fpr_grd_lm, tpr_grd_lm, _ = roc_curve(y_test, y_pred_grd_lm)
 print(log_loss(y_test, grd_lm.predict_proba(
 grd_enc.transform(grd.apply(X_test)[:, :, 0]))))
print('gradient boosting')
 y_pred_grd = grd.predict_proba(X_test)[:, 1]
fpr_grd, tpr_grd, _ = roc_curve(y_test, y_pred_grd)
 print(log_loss(y_test, grd.predict_proba(X_test)))
 print('random forest')
 y_pred_rf = rf.predict_proba(X_test)[:, 1]
fpr_rf, tpr_rf, _ = roc_curve(y_test, y_pred_rf)
 print(log_loss(y_test, rf.predict_proba(X_test)))
 plt.figure(1)
 plt.plot([0, 1], [0, 1], 'k--')
 plt.plot(fpr_rt_lm, tpr_rt_lm, label='RT + LR')
 plt.plot(fpr_rf, tpr_rf, label='RF')
 plt.plot(fpr_rf_lm, tpr_rf_lm, label='RF + LR')
 plt.plot(fpr_grd, tpr_grd, label='GBT')
 plt.plot(fpr_grd_lm, tpr_grd_lm, label='GBT + LR')
 plt.xlabel('False positive rate')
 plt.ylabel('True positive rate')
 plt.title('ROC curve')
 plt.legend(loc='best')
 plt.show() 
<train_model>
on_kaggle_server = os.path.exists('/kaggle')
 nq_test_file = '.. /input/tensorflow2-question-answering/simplified-nq-test.jsonl'
nq_train_file = '.. /input/tensorflow2-question-answering/simplified-nq-train.jsonl'
public_dataset = os.path.getsize(nq_test_file)<20_000_000
private_dataset = os.path.getsize(nq_test_file)>=20_000_000
GradientBoostingClassifier)
 def fit(X_train, y_train, X_test, y_test):
ens = pd.DataFrame()
n_estimator = 100
rt = RandomTreesEmbedding(max_depth=3, n_estimators=n_estimator)
 rt_lm = LogisticRegression()
pipeline = make_pipeline(rt, rt_lm)
 rf = RandomForestClassifier(max_depth=3, n_estimators=n_estimator)
 rf_enc = OneHotEncoder()
rf_lm = LogisticRegression()
grd = GradientBoostingClassifier(n_estimators=n_estimator)
 grd_enc = OneHotEncoder()
grd_lm = LogisticRegression()
X_train, X_train_lr, y_train, y_train_lr = train_test_split(X_train,
y_train,
test_size=0.5)
 print('random trees + lt')
 pipeline.fit(X_train, y_train)
 y_pred_rt = pipeline.predict_proba(X_test)[:, 1]
fpr_rt_lm, tpr_rt_lm, _ = roc_curve(y_test, y_pred_rt)
 ens['rt_lt'] = pipeline.predict(X_test)
 print(log_loss(y_test, pipeline.predict_proba(X_test)))
 print('random forest + lt')
 rf.fit(X_train, y_train)
 rf_enc.fit(rf.apply(X_train))
rf_lm.fit(rf_enc.transform(rf.apply(X_train_lr)) , y_train_lr)
 y_pred_rf_lm = rf_lm.predict_proba(rf_enc.transform(rf.apply(X_test)))[:, 1]
fpr_rf_lm, tpr_rf_lm, _ = roc_curve(y_test, y_pred_rf_lm)
 ens['rf_lm'] = rf_lm.predict(rf_enc.transform(rf.apply(X_test)))
 print(log_loss(y_test, rf_lm.predict_proba(rf_enc.transform(rf.apply(X_test)))))
 print('gradient boosting + lt')
 grd.fit(X_train, y_train)
 grd_enc.fit(grd.apply(X_train)[:, :, 0])
 grd_lm.fit(grd_enc.transform(grd.apply(X_train_lr)[:, :, 0]), y_train_lr)
 y_pred_grd_lm = grd_lm.predict_proba(
 grd_enc.transform(grd.apply(X_test)[:, :, 0])) [:, 1]
fpr_grd_lm, tpr_grd_lm, _ = roc_curve(y_test, y_pred_grd_lm)
 ens['grd_lm'] = grd_lm.predict(grd_enc.transform(grd.apply(X_test)[:, :, 0]))
print(log_loss(y_test, grd_lm.predict_proba(
 grd_enc.transform(grd.apply(X_test)[:, :, 0]))))
print('gradient boosting')
 y_pred_grd = grd.predict_proba(X_test)[:, 1]
fpr_grd, tpr_grd, _ = roc_curve(y_test, y_pred_grd)
 ens['grd'] = grd.predict(X_test)
 print(log_loss(y_test, grd.predict_proba(X_test)))
 print('random forest')
 y_pred_rf = rf.predict_proba(X_test)[:, 1]
fpr_rf, tpr_rf, _ = roc_curve(y_test, y_pred_rf)
 ens['rf'] = rf.predict(X_test)
 print(log_loss(y_test, rf.predict_proba(X_test)))
 return ens<prepare_x_and_y>
if True:
importlib.reload(bert_utils )
train_X = data.as_matrix()
train_y = np.array(train_df["is_duplicate"])
 test_id = np.array(test_df["test_id"] )<prepare_x_and_y>
with open('.. /input/bert-joint-baseline/bert_config.json','r')as f:
config = json.load(f)
 print(json.dumps(config,indent=4))
train_X_dup = train_X[train_y==1]
train_X_non_dup = train_X[train_y==0]
train_X = np.vstack([train_X_non_dup, train_X_dup, train_X_non_dup, train_X_non_dup])
 train_y = np.array([0]*train_X_non_dup.shape[0] + [1]*train_X_dup.shape[0] + [0]*train_X_non_dup.shape[0] + [0]*train_X_non_dup.shape[0])
 del train_X_dup
del train_X_non_dup
print("Mean target rate : ",train_y.mean() )<train_model>
class TDense(tf.keras.layers.Layer):
def __init__(self,
output_size,
kernel_initializer=None,
bias_initializer="zeros",
**kwargs):
super().__init__(**kwargs)
 self.output_size = output_size
self.kernel_initializer = kernel_initializer
self.bias_initializer = bias_initializer
def build(self,input_shape):
dtype = tf.as_dtype(self.dtype or tf.keras.backend.floatx())
 if not(dtype.is_floating or dtype.is_complex):
raise TypeError("Unable to build `TDense` layer with "
"non-floating point(and non-complex)"
"dtype %s" %(dtype,))
input_shape = tf.TensorShape(input_shape)
 if tf.compat.dimension_value(input_shape[-1])is None:
raise ValueError("The last dimension of the inputs to "
"`TDense` should be defined."
"Found `None`.")
 last_dim = tf.compat.dimension_value(input_shape[-1])
 self.input_spec = tf.keras.layers.InputSpec(min_ndim=3, axes={-1: last_dim})
 self.kernel = self.add_weight(
 "kernel",
shape=[self.output_size,last_dim],
initializer=self.kernel_initializer,
dtype=self.dtype,
trainable=True)
 self.bias = self.add_weight(
 "bias",
shape=[self.output_size],
initializer=self.bias_initializer,
dtype=self.dtype,
trainable=True)
 super(TDense, self ).build(input_shape)
 def call(self,x):
return tf.matmul(x,self.kernel,transpose_b=True)+self.bias
def mk_model(config):
seq_len = config['max_position_embeddings']
unique_id = tf.keras.Input(shape=(1,),dtype=tf.int64,name='unique_id')
 input_ids = tf.keras.Input(shape=(seq_len,),dtype=tf.int32,name='input_ids')
 input_mask = tf.keras.Input(shape=(seq_len,),dtype=tf.int32,name='input_mask')
 segment_ids = tf.keras.Input(shape=(seq_len,),dtype=tf.int32,name='segment_ids')
 BERT = modeling.BertModel(config=config,name='bert')
 pooled_output, sequence_output = BERT(input_word_ids=input_ids,
input_mask=input_mask,
input_type_ids=segment_ids)
 logits = TDense(2,name='logits' )(sequence_output)
 start_logits,end_logits = tf.split(logits,axis=-1,num_or_size_splits= 2,name='split')
 start_logits = tf.squeeze(start_logits,axis=-1,name='start_squeeze')
 end_logits = tf.squeeze(end_logits, axis=-1,name='end_squeeze')
 ans_type = TDense(5,name='ans_type' )(pooled_output)
 return tf.keras.Model([input_ for input_ in [unique_id,input_ids,input_mask,segment_ids]
if input_ is not None],
[unique_id,start_logits,end_logits,ans_type],
name='bert-baseline' )
kf = KFold(n_splits=5, shuffle=True)
 for dev_index, val_index in kf.split(range(train_X.shape[0])) :
dev_X, val_X = train_X[dev_index,:], train_X[val_index,:]
dev_y, val_y = train_y[dev_index], train_y[val_index]
ens = fit(dev_X, dev_y, val_X, val_y)
 ens['avg'] = ens.mean(axis=1)
 ens['median'] = ens.median(axis=1)
 break<data_type_conversions>
small_config = config.copy()
small_config['vocab_size']=16
small_config['hidden_size']=64
small_config['max_position_embeddings'] = 32
small_config['num_hidden_layers'] = 4
small_config['num_attention_heads'] = 4
small_config['intermediate_size'] = 256
small_config
print(ens.mean(axis=0))
print(accuracy_score(ens['median'].as_matrix().astype(int),val_y))<feature_engineering>
model= mk_model(config )
testz = pd.DataFrame()
testz['word_match_share'] = test_df.apply(lambda row: word_match_share(row), axis=1)
 print('5')
 testz['tfidf_word_match_share'] = test_df.apply(lambda row: tfidf_word_match_share(row), axis=1)
 print('0')
 testz['len_q1'] = test_df.question1.apply(lambda x: len(str(x)))
 testz['len_q2'] = test_df.question2.apply(lambda x: len(str(x)))
 testz['diff_len'] = data.len_q1 - data.len_q2
print('1')
 testz['len_char_q1'] = test_df.question1.apply(lambda x: len(''.join(set(str(x ).replace(' ', '')))))
 testz['len_char_q2'] = test_df.question2.apply(lambda x: len(''.join(set(str(x ).replace(' ', '')))))
 testz['len_word_q1'] = test_df.question1.apply(lambda x: len(str(x ).split()))
testz['len_word_q2'] = test_df.question2.apply(lambda x: len(str(x ).split()))
print('2')
 testz['common_words'] = test_df.apply(lambda x: len(set(str(x['question1'] ).lower().split() ).intersection(set(str(x['question2'] ).lower().split()))), axis=1)
 print('3' )<feature_engineering>
class DummyObject:
def __init__(self,**kwargs):
self.__dict__.update(kwargs)
 FLAGS=DummyObject(skip_nested_contexts=True,
max_position=50,
max_contexts=48,
max_query_length=64,
max_seq_length=512,
doc_stride=128,
include_unknowns=-1.0,
n_best_size=20,
max_answer_length=30 )
zzz=<concatenate>
eval_records = ".. /input/bert-joint-baseline/nq-test.tfrecords"
if on_kaggle_server and private_dataset:
eval_records='nq-test.tfrecords'
if not os.path.exists(eval_records):
eval_writer = bert_utils.FeatureWriter(
 filename=os.path.join(eval_records),
is_training=False)
 tokenizer = tokenization.FullTokenizer(vocab_file='.. /input/bert-joint-baseline/vocab-nq.txt',
do_lower_case=True)
 features = []
convert = bert_utils.ConvertExamples2Features(tokenizer=tokenizer,
is_training=False,
output_fn=eval_writer.process_feature,
collect_stat=False)
 n_examples = 0
tqdm_notebook= tqdm.tqdm_notebook if not on_kaggle_server else None
for examples in bert_utils.nq_examples_iter(input_file=nq_test_file,
is_training=False,
tqdm=tqdm_notebook):
for example in examples:
n_examples += convert(example)
 eval_writer.close()
print('number of test examples: %d, written to file: %d' %(n_examples,eval_writer.num_features))
s3 = testz.as_matrix()
s4 = np.vstack(np.array(test_df.apply(lambda row: feature_extraction(row), axis=1)))
 print(s3.shape, s4.shape)
 final_test = np.hstack(( s3, s4))<save_to_csv>
seq_length = FLAGS.max_seq_length
name_to_features = {
"unique_id": tf.io.FixedLenFeature([], tf.int64),
"input_ids": tf.io.FixedLenFeature([seq_length], tf.int64),
"input_mask": tf.io.FixedLenFeature([seq_length], tf.int64),
"segment_ids": tf.io.FixedLenFeature([seq_length], tf.int64),
}
def _decode_record(record, name_to_features=name_to_features):

 example = tf.io.parse_single_example(serialized=record, features=name_to_features)
 for name in list(example.keys()):
t = example[name]
if name != 'unique_id':
t = tf.cast(t, dtype=tf.int32)
 example[name] = t
return example
def _decode_tokens(record):
return tf.io.parse_single_example(serialized=record,
features={
"unique_id": tf.io.FixedLenFeature([], tf.int64),
"token_map" : tf.io.FixedLenFeature([seq_length], tf.int64)
 })
xgtest = xgb.DMatrix(final_test)
 preds = model.predict(xgtest)
 out_df = pd.DataFrame({"test_id":test_id, "is_duplicate":preds})
 out_df.to_csv("idkhowtosubmit.csv", index=False )<load_from_csv>
raw_ds = tf.data.TFRecordDataset(eval_records)
 token_map_ds = raw_ds.map(_decode_tokens)
 decoded_ds = raw_ds.map(_decode_record)
 ds = decoded_ds.batch(batch_size=16,drop_remainder=False )
df_train = pd.read_csv('.. /input/train.csv' )<load_from_csv>
result=model.predict_generator(ds,verbose=1 )
df_test = pd.read_csv('.. /input/test.csv' )<count_duplicates>
np.savez_compressed('bert-joint-baseline-output.npz',
**dict(zip(['uniqe_id','start_logits','end_logits','answer_type_logits'],
result)) )
print('Duplicate pairs in small train: {}%'.format(round(small_train['is_duplicate'].mean() *100, 2)))
 print('Duplicate pairs in full df_train: {}%'.format(round(df_train['is_duplicate'].mean() *100, 2)) )<string_transform>
Span = collections.namedtuple("Span", ["start_token_idx", "end_token_idx"] )
stops = set(stopwords.words("english"))
def word_match_share(row):
q1words = {}
q2words = {}
for word in str(row['question1'] ).lower().split() :
if word not in stops:
q1words[word] = 1
for word in str(row['question2'] ).lower().split() :
if word not in stops:
q2words[word] = 1
if len(q1words)== 0 or len(q2words)== 0:
return 0
shared_words_in_q1 = [w for w in q1words.keys() if w in q2words]
shared_words_in_q2 = [w for w in q2words.keys() if w in q1words]
R =(len(shared_words_in_q1)+ len(shared_words_in_q2)) /(len(q1words)+ len(q2words))
return R
train_word_match = df_train.apply(word_match_share, axis=1, raw=True )<init_hyperparams>
class ScoreSummary(object):
def __init__(self):
self.predicted_label = None
self.short_span_score = None
self.cls_token_score = None
self.answer_type_logits = None
def get_weight(count, eps=1000, min_count=2):
if count < min_count:
return 0
else:
return 1 /(count + eps)
 eps = 5000
words =(" ".join(train_qs)).lower().split()
counts = Counter(words)
 weights = {word: get_weight(count)for word, count in counts.items() }<sort_values>
class EvalExample(object):

 def __init__(self, example_id, candidates):
self.example_id = example_id
self.candidates = candidates
self.results = {}
self.features = {}
print('Most common words and weights:
')
 print(sorted(weights.items() , key=lambda x: x[1] if x[1] > 0 else 9999)[:10])
 print('
Least common words and weights: ')
(sorted(weights.items() , key=lambda x: x[1], reverse=True)[:10] )<feature_engineering>
def get_best_indexes(logits, n_best_size):

 index_and_score = sorted(
 enumerate(logits[1:], 1), key=lambda x: x[1], reverse=True)
 best_indexes = []
for i in range(len(index_and_score)) :
if i >= n_best_size:
break
best_indexes.append(index_and_score[i][0])
 return best_indexes
def top_k_indices(logits,n_best_size,token_map):
indices = np.argsort(logits[1:])+1
indices = indices[token_map[indices]!=-1]
return indices[-n_best_size:]
def tfidf_word_match_share(row):
q1words = {}
q2words = {}
for word in str(row['question1'] ).lower().split() :
if word not in stops:
q1words[word] = 1
for word in str(row['question2'] ).lower().split() :
if word not in stops:
q2words[word] = 1
if len(q1words)== 0 or len(q2words)== 0:
return 0
shared_weights = [weights.get(w, 0)for w in q1words.keys() if w in q2words] + [weights.get(w, 0)for w in q2words.keys() if w in q1words]
total_weights = [weights.get(w, 0)for w in q1words] + [weights.get(w, 0)for w in q2words]
R = np.sum(shared_weights)/ np.sum(total_weights)
 return R<compute_test_metric>
def compute_predictions(example):

 predictions = []
n_best_size = FLAGS.n_best_size
max_answer_length = FLAGS.max_answer_length
i = 0
for unique_id, result in example.results.items() :
if unique_id not in example.features:
raise ValueError("No feature found with unique_id:", unique_id)
 token_map = np.array(example.features[unique_id]["token_map"])
 start_indexes = top_k_indices(result.start_logits,n_best_size,token_map)
 if len(start_indexes)==0:
continue
end_indexes = top_k_indices(result.end_logits,n_best_size,token_map)
 if len(end_indexes)==0:
continue
indexes = np.array(list(np.broadcast(start_indexes[None],end_indexes[:,None])))
 indexes = indexes[(indexes[:,0]<indexes[:,1])*(indexes[:,1]-indexes[:,0]<max_answer_length)]
for start_index,end_index in indexes:
summary = ScoreSummary()
summary.short_span_score =(
 result.start_logits[start_index] +
result.end_logits[end_index])
 summary.cls_token_score =(
 result.start_logits[0] + result.end_logits[0])
 summary.answer_type_logits = result.answer_type_logits-result.answer_type_logits.mean()
start_span = token_map[start_index]
end_span = token_map[end_index] + 1
score = summary.short_span_score - summary.cls_token_score
predictions.append(( score, i, summary, start_span, end_span))
i += 1
score = -10000.0
short_span = Span(-1, -1)
 long_span = Span(-1, -1)
 summary = ScoreSummary()
if predictions:
score, _, summary, start_span, end_span = sorted(predictions, reverse=True)[0]
short_span = Span(start_span, end_span)
 for c in example.candidates:
start = short_span.start_token_idx
end = short_span.end_token_idx
if c["top_level"] and c["start_token"] <= start and c["end_token"] >= end:
long_span = Span(c["start_token"], c["end_token"])
 break
summary.predicted_label = {
"example_id": int(example.example_id),
"long_answer": {
"start_token": int(long_span.start_token_idx),
"end_token": int(long_span.end_token_idx),
"start_byte": -1,
"end_byte": -1
},
"long_answer_score": float(score),
"short_answers": [{
"start_token": int(short_span.start_token_idx),
"end_token": int(short_span.end_token_idx),
"start_byte": -1,
"end_byte": -1
}],
"short_answer_score": float(score),
"yes_no_answer": "NONE",
"answer_type_logits": summary.answer_type_logits.tolist() ,
"answer_type": int(np.argmax(summary.answer_type_logits))
}
return summary
print('Original AUC:', roc_auc_score(df_train['is_duplicate'], train_word_match))
print(' TFIDF AUC:', roc_auc_score(df_train['is_duplicate'], tfidf_train_word_match.fillna(0)) )<string_transform>
def read_candidates_from_one_split(input_path):

 candidates_dict = {}
print("Reading examples from: %s" % input_path)
 if input_path.endswith(".gz"):
with gzip.GzipFile(fileobj=tf.io.gfile.GFile(input_path, "rb")) as input_file:
for index, line in enumerate(input_file):
e = json.loads(line)
 candidates_dict[e["example_id"]] = e["long_answer_candidates"]
else:
with tf.io.gfile.GFile(input_path, "r")as input_file:
for index, line in enumerate(input_file):
e = json.loads(line)
 candidates_dict[e["example_id"]] = e["long_answer_candidates"]
return candidates_dict
train_q1s = pd.Series(df_train['question1'].tolist() ).astype(str)
 train_q2s = pd.Series(df_train['question2'].tolist() ).astype(str)
 train_q1_words = train_q1s.apply(lambda x: len(x.split(' ')))
 train_q2_words = train_q1s.apply(lambda x: len(x.split(' ')))
 train_word_count_diff=train_q1_words-train_q2_words
test_q1s = pd.Series(df_test['question1'].tolist() ).astype(str)
 test_q2s = pd.Series(df_test['question2'].tolist() ).astype(str)
 test_q1_words = test_q1s.apply(lambda x: len(x.split(' ')))
 test_q2_words = test_q2s.apply(lambda x: len(x.split(' ')))
 test_word_count_diff=test_q1_words-test_q2_words<prepare_x_and_y>
def read_candidates(input_pattern):

 input_paths = tf.io.gfile.glob(input_pattern)
 final_dict = {}
for input_path in input_paths:
final_dict.update(read_candidates_from_one_split(input_path))
return final_dict
x_train = pd.DataFrame()
x_test = pd.DataFrame()
x_train['word_match'] = train_word_match
x_train['tfidf_word_match'] = tfidf_train_word_match
x_train['word_count_diff']=train_word_count_diff
x_test['word_match'] = df_test.apply(word_match_share, axis=1, raw=True)
 x_test['tfidf_word_match'] = df_test.apply(tfidf_word_match_share, axis=1, raw=True)
 x_test['word_count_diff']=train_word_count_diff
y_train = df_train['is_duplicate'].values<concatenate>
all_results = [bert_utils.RawResult(*x)for x in zip(*result)]
print("Going to candidates file")
 candidates_dict = read_candidates('.. /input/tensorflow2-question-answering/simplified-nq-test.jsonl')
 print("setting up eval features")
 eval_features = list(token_map_ds)
 print("compute_pred_dict")
 tqdm_notebook= tqdm.tqdm_notebook
nq_pred_dict = compute_pred_dict(candidates_dict,
eval_features,
all_results,
tqdm=tqdm_notebook)
 predictions_json = {"predictions": list(nq_pred_dict.values())}
print("writing json")
 with tf.io.gfile.GFile('predictions.json', "w")as f:
json.dump(predictions_json, f, indent=4 )
pos_train = x_train[y_train == 1]
neg_train = x_train[y_train == 0]
p = 0.165
scale =(( len(pos_train)/(len(pos_train)+ len(neg_train)))/ p)- 1
while scale > 1:
neg_train = pd.concat([neg_train, neg_train])
 scale -=1
neg_train = pd.concat([neg_train, neg_train[:int(scale * len(neg_train)) ]])
 print(len(pos_train)/(len(pos_train)+ len(neg_train)))
 x_train = pd.concat([pos_train, neg_train])
 y_train =(np.zeros(len(pos_train)) + 1 ).tolist() + np.zeros(len(neg_train)).tolist()
del pos_train, neg_train<split>
def create_short_answer(entry):
answer = []
if entry['answer_type'] == 0:
return ""
elif entry['answer_type'] == 1:
return 'YES'
elif entry['answer_type'] == 2:
return 'NO'
elif entry["short_answer_score"] < 1.5:
return ""
else:
for short_answer in entry["short_answers"]:
if short_answer["start_token"] > -1:
answer.append(str(short_answer["start_token"])+ ":" + str(short_answer["end_token"]))
return " ".join(answer)
 def create_long_answer(entry):
answer = []
if entry['answer_type'] == 0:
return ''
elif entry["long_answer_score"] < 1.5:
return ""
elif entry["long_answer"]["start_token"] > -1:
answer.append(str(entry["long_answer"]["start_token"])+ ":" + str(entry["long_answer"]["end_token"]))
return " ".join(answer )
x_train, x_valid, y_train, y_valid = train_test_split(x_train, y_train, test_size=0.2, random_state=4242 )<train_model>
test_answers_df = pd.read_json(".. /working/predictions.json")
 for var_name in ['long_answer_score','short_answer_score','answer_type']:
test_answers_df[var_name] = test_answers_df['predictions'].apply(lambda q: q[var_name])
 test_answers_df["long_answer"] = test_answers_df["predictions"].apply(create_long_answer)
 test_answers_df["short_answer"] = test_answers_df["predictions"].apply(create_short_answer)
 test_answers_df["example_id"] = test_answers_df["predictions"].apply(lambda q: str(q["example_id"]))
long_answers = dict(zip(test_answers_df["example_id"], test_answers_df["long_answer"]))
short_answers = dict(zip(test_answers_df["example_id"], test_answers_df["short_answer"]))
test_answers_df.head()
params = {}
params['objective'] = 'binary:logistic'
params['eval_metric'] = 'logloss'
params['eta'] = 0.02
params['max_depth'] = 4
d_train = xgb.DMatrix(x_train, label=y_train)
 d_valid = xgb.DMatrix(x_valid, label=y_valid)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
bst = xgb.train(params, d_train, 400, watchlist, early_stopping_rounds=50, verbose_eval=10 )<save_to_csv>
sample_submission = pd.read_csv(".. /input/tensorflow2-question-answering/sample_submission.csv")
 long_prediction_strings = sample_submission[sample_submission["example_id"].str.contains("_long")].apply(lambda q: long_answers[q["example_id"].replace("_long", "")], axis=1)
 short_prediction_strings = sample_submission[sample_submission["example_id"].str.contains("_short")].apply(lambda q: short_answers[q["example_id"].replace("_short", "")], axis=1)
 sample_submission.loc[sample_submission["example_id"].str.contains("_long"), "PredictionString"] = long_prediction_strings
sample_submission.loc[sample_submission["example_id"].str.contains("_short"), "PredictionString"] = short_prediction_strings
d_test = xgb.DMatrix(x_test)
 p_test = bst.predict(d_test)
 sub = pd.DataFrame()
sub['test_id'] = df_test['test_id']
sub['is_duplicate'] = p_test
sub.to_csv('simple_xgb.csv', index=False )<load_from_csv>
sample_submission.to_csv('submission.csv', index=False )
df_train = pd.read_csv('.. /input/train.csv')
 df_train.head()<save_to_csv>
yes_answers = sample_submission[sample_submission['PredictionString'] == 'YES']
yes_answers
p = df_train['is_duplicate'].mean()
print('Predicted score:', log_loss(df_train['is_duplicate'], np.zeros_like(df_train['is_duplicate'])+ p))
df_test = pd.read_csv('.. /input/test.csv')
 sub = pd.DataFrame({'test_id': df_test['test_id'], 'is_duplicate': p})
 sub.to_csv('naive_submission.csv', index=False)
 sub.head()<load_from_csv>
no_answers = sample_submission[sample_submission['PredictionString'] == 'NO']
no_answers
df_test = pd.read_csv('.. /input/test.csv')
 df_test.head()<define_variables>
blank_answers = sample_submission[sample_submission['PredictionString'] == '']
blank_answers.head()
print('Total number of question pairs for testing: {}'.format(len(df_test)) )<feature_engineering>
blank_answers.count()