Kaggle competition: TensorFlow 2.0 Question Answering
train_data=train_csv[cols]
test_data=test_csv[cols]<feature_engineering>
on_kaggle_server = os.path.exists('/kaggle')
 nq_test_file = '.. /input/tensorflow2-question-answering/simplified-nq-test.jsonl'
nq_train_file = '.. /input/tensorflow2-question-answering/simplified-nq-train.jsonl'
public_dataset = os.path.getsize(nq_test_file)<20_000_000
private_dataset = os.path.getsize(nq_test_file)>=20_000_000
for df in [train_data,test_data]:
df["sum"]=df.sum(axis=1)
 df["median"]=df.median(axis=1)
 df["mean"]=df.mean(axis=1)
 df["std"]=df.std(axis=1)
 df["max"]=df.max(axis=1)
 df["min"]=df.min(axis=1)
 df["kurtosis"]=df.kurtosis(axis=1)
 print(train_data.shape,test_data.shape )<import_modules>
if True:
importlib.reload(bert_utils )
from sklearn.model_selection import KFold, cross_val_score, train_test_split
from sklearn.metrics import mean_squared_error
import xgboost as xgb
import lightgbm as lgb<compute_train_metric>
config = {'attention_probs_dropout_prob':0.1,
'hidden_act':'gelu',
'hidden_dropout_prob':0.1,
'hidden_size':1024,
'initializer_range':0.02,
'intermediate_size':4096,
'max_position_embeddings':512,
'num_attention_heads':16,
'num_hidden_layers':24,
'type_vocab_size':2,
'vocab_size':30522}
y_target=np.log1p(target)
 NUM_FOLDS = 5
def rmsle_cv(model):
kf = KFold(NUM_FOLDS, shuffle=True, random_state=42 ).get_n_splits(train_data.values)
 rmse= np.sqrt(-cross_val_score(model, train_data, y_target, scoring="neg_mean_squared_error", cv = kf))
return(rmse )<split>
class TDense(tf.keras.layers.Layer):
def __init__(self,
output_size,
kernel_initializer=None,
bias_initializer="zeros",
**kwargs):
super().__init__(**kwargs)
 self.output_size = output_size
self.kernel_initializer = kernel_initializer
self.bias_initializer = bias_initializer
def build(self,input_shape):
dtype = tf.as_dtype(self.dtype or tf.keras.backend.floatx())
 if not(dtype.is_floating or dtype.is_complex):
raise TypeError("Unable to build `TDense` layer with "
"non-floating point(and non-complex)"
"dtype %s" %(dtype,))
input_shape = tf.TensorShape(input_shape)
 if tf.compat.dimension_value(input_shape[-1])is None:
raise ValueError("The last dimension of the inputs to "
"`TDense` should be defined."
"Found `None`.")
 last_dim = tf.compat.dimension_value(input_shape[-1])
 self.input_spec = tf.keras.layers.InputSpec(min_ndim=3, axes={-1: last_dim})
 self.kernel = self.add_weight(
 "kernel",
shape=[self.output_size,last_dim],
initializer=self.kernel_initializer,
dtype=self.dtype,
trainable=True)
 self.bias = self.add_weight(
 "bias",
shape=[self.output_size],
initializer=self.bias_initializer,
dtype=self.dtype,
trainable=True)
 super(TDense, self ).build(input_shape)
 def call(self,x):
return tf.matmul(x,self.kernel,transpose_b=True)+self.bias
def mk_model(config):
seq_len = config['max_position_embeddings']
unique_id = tf.keras.Input(shape=(1,),dtype=tf.int64,name='unique_id')
 input_ids = tf.keras.Input(shape=(seq_len,),dtype=tf.int32,name='input_ids')
 input_mask = tf.keras.Input(shape=(seq_len,),dtype=tf.int32,name='input_mask')
 segment_ids = tf.keras.Input(shape=(seq_len,),dtype=tf.int32,name='segment_ids')
 BERT = modeling.BertModel(config=config,name='bert')
 pooled_output, sequence_output = BERT(input_word_ids=input_ids,
input_mask=input_mask,
input_type_ids=segment_ids)
 logits = TDense(2,name='logits' )(sequence_output)
 start_logits,end_logits = tf.split(logits,axis=-1,num_or_size_splits= 2,name='split')
 start_logits = tf.squeeze(start_logits,axis=-1,name='start_squeeze')
 end_logits = tf.squeeze(end_logits, axis=-1,name='end_squeeze')
 ans_type = TDense(5,name='ans_type' )(pooled_output)
 return tf.keras.Model([input_ for input_ in [unique_id,input_ids,input_mask,segment_ids]
if input_ is not None],
[unique_id,start_logits,end_logits,ans_type],
name='bert-baseline' )
x_train, x_val, y_train, y_val = train_test_split(train_data, y_target, test_size=0.15, random_state=42 )<create_dataframe>
model= mk_model(config )
params = {
"objective" : "regression",
"metric" : "rmse",
"num_leaves" : 50,
"max_depth" : 6,
"learning_rate" : 0.0005,
"bagging_fraction" : 0.7,
"feature_fraction" : 0.8,
"bagging_frequency" : 5,
"bagging_seed" : 42,
"verbosity" : -1,
"lambda" : 0.01,
"random_seed": 42
}
lgtrain = lgb.Dataset(x_train, label=y_train)
 lgval = lgb.Dataset(x_val, label=y_val)
 evals_result = {}
model = lgb.train(params, lgtrain, 200000,
valid_sets=[lgval],
early_stopping_rounds=1500,
verbose_eval=50,
evals_result=evals_result )<predict_on_test>
class DummyObject:
def __init__(self,**kwargs):
self.__dict__.update(kwargs)
 FLAGS=DummyObject(skip_nested_contexts=True,
max_position=50,
max_contexts=48,
max_query_length=64,
max_seq_length=512,
doc_stride=128,
include_unknowns=0.1,
n_best_size=20,
max_answer_length=30 )
pred_test_y = np.expm1(model.predict(test_data, num_iteration=model.best_iteration))<train_model>
eval_records = ".. /input/bert-joint-baseline/nq-test.tfrecords"
if on_kaggle_server and private_dataset:
eval_records='nq-test.tfrecords'
if not os.path.exists(eval_records):
eval_writer = bert_utils.FeatureWriter(
 filename=os.path.join(eval_records),
is_training=False)
 tokenizer = tokenization.FullTokenizer(vocab_file='.. /input/bert-joint-baseline/vocab-nq.txt',
do_lower_case=True)
 features = []
convert = bert_utils.ConvertExamples2Features(tokenizer=tokenizer,
is_training=False,
output_fn=eval_writer.process_feature,
collect_stat=False)
 n_examples = 0
tqdm_notebook= tqdm.tqdm_notebook if not on_kaggle_server else None
for examples in bert_utils.nq_examples_iter(input_file=nq_test_file,
is_training=False,
tqdm=tqdm_notebook):
for example in examples:
n_examples += convert(example)
 eval_writer.close()
params = {'objective': 'reg:linear',
'eval_metric': 'rmse',
'eta': 0.0005,
'max_depth':6,
'subsample': 0.7,
'colsample_bytree': 0.5,
'alpha':0,
'random_state': 42,
'silent': True}
tr_data = xgb.DMatrix(x_train,y_train)
 va_data = xgb.DMatrix(x_val,y_val)
 watchlist = [(tr_data, 'train'),(va_data, 'valid')]
model_xgb = xgb.train(params, tr_data, 200000, watchlist, maximize=False, early_stopping_rounds = 3000, verbose_eval=100 )<predict_on_test>
seq_length = FLAGS.max_seq_length
name_to_features = {
"unique_id": tf.io.FixedLenFeature([], tf.int64),
"input_ids": tf.io.FixedLenFeature([seq_length], tf.int64),
"input_mask": tf.io.FixedLenFeature([seq_length], tf.int64),
"segment_ids": tf.io.FixedLenFeature([seq_length], tf.int64),
}
def _decode_record(record, name_to_features=name_to_features):

 example = tf.io.parse_single_example(serialized=record, features=name_to_features)
 for name in list(example.keys()):
t = example[name]
if name != 'unique_id':
t = tf.cast(t, dtype=tf.int64)
 example[name] = t
return example
def _decode_tokens(record):
return tf.io.parse_single_example(serialized=record,
features={
"unique_id": tf.io.FixedLenFeature([], tf.int64),
"token_map" : tf.io.FixedLenFeature([seq_length], tf.int64)
 })
dtest = xgb.DMatrix(test_data)
 xgb_pred_y = np.expm1(model_xgb.predict(dtest, ntree_limit=model_xgb.best_ntree_limit))<save_to_csv>
raw_ds = tf.data.TFRecordDataset(eval_records)
 token_map_ds = raw_ds.map(_decode_tokens)
 decoded_ds = raw_ds.map(_decode_record)
 ds = decoded_ds.batch(batch_size=16,drop_remainder=False )
sub = pd.DataFrame()
sub['ID'] = test_id
sub['target'] = pred_test_y
sub.to_csv('submission_lgb.csv',index=False )<save_to_csv>
result=model.predict_generator(ds,verbose=1 )
sub = pd.DataFrame()
sub['ID'] = test_id
sub['target'] = xgb_pred_y
sub.to_csv('submission_xgb.csv',index=False )<save_to_csv>
np.savez_compressed('bert-joint-baseline-output.npz',
**dict(zip(['uniqe_id','start_logits','end_logits','answer_type_logits'],
result)) )
sub = pd.DataFrame()
sub['ID'] = test_id
sub['target'] = np.mean([pred_test_y,xgb_pred_y])
 sub.to_csv('submission_comb_xlgb.csv',index=False )<set_options>
Span = collections.namedtuple("Span", ["start_token_idx", "end_token_idx", "score"] )
eng_stopwords = set(stopwords.words('english'))
color = sns.color_palette()
%matplotlib inline
pd.options.mode.chained_assignment = None<load_from_csv>
class ScoreSummary(object):
def __init__(self):
self.predicted_label = None
self.short_span_score = None
self.cls_token_score = None
self.answer_type_logits = None
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv" )<create_dataframe>
class EvalExample(object):

 def __init__(self, example_id, candidates):
self.example_id = example_id
self.candidates = candidates
self.results = {}
self.features = {}
question_df=pd.DataFrame(pd.concat([train_df['question1'],train_df['question2']]))
question_df.columns=["questions"]<feature_engineering>
def get_best_indexes(logits, n_best_size):

 index_and_score = sorted(
 enumerate(logits[1:], 1), key=lambda x: x[1], reverse=True)
 best_indexes = []
for i in range(len(index_and_score)) :
if i >= n_best_size:
break
best_indexes.append(index_and_score[i][0])
 return best_indexes
def top_k_indices(logits,n_best_size,token_map):
indices = np.argsort(logits[1:])+1
indices = indices[token_map[indices]!=-1]
return indices[-n_best_size:]
question_df = pd.DataFrame(pd.concat([train_df['question1'], train_df['question2']]))
question_df.columns = ["questions"]
question_df["No_of_Words"] = question_df["questions"].apply(lambda x : len(str(x ).split()))<feature_engineering>
def remove_duplicates(span):
start_end = []
for s in span:
cont = 0
if not start_end:
start_end.append(Span(s[0], s[1], s[2]))
cont += 1
else:
for i in range(len(start_end)) :
if start_end[i][0] == s[0] and start_end[i][1] == s[1]:
cont += 1
if cont == 0:
start_end.append(Span(s[0], s[1], s[2]))
return start_end
def get_unigrams(que):
return [word for word in word_tokenize(que.lower())if word not in eng_stopwords]
def get_common_unigrams(row):
return len(set(row["unigrams_ques1"] ).intersection(set(row["unigrams_ques2"])))
 def get_common_unigram_ratio(row):
return float(row["unigrams_common_count"])/ max(len(set(row["unigrams_ques1"] ).union(set(row["unigrams_ques2"]))),1)
 train_df["unigrams_ques1"] = train_df['question1'].apply(lambda x: get_unigrams(str(x)))
 train_df["unigrams_ques2"] = train_df['question2'].apply(lambda x: get_unigrams(str(x)))
 train_df["unigrams_common_count"] = train_df.apply(lambda row: get_common_unigrams(row),axis=1)
 train_df["unigrams_common_ratio"] = train_df.apply(lambda row: get_common_unigram_ratio(row), axis=1)
<feature_engineering>
def get_short_long_span(predictions, example):
sorted_predictions = sorted(predictions, reverse=True)
 short_span = []
long_span = []
for prediction in sorted_predictions:
score, _, summary, start_span, end_span = prediction
if score > 0:
short_span.append(Span(int(start_span), int(end_span), float(score)))
 short_span = remove_duplicates(short_span)
 for s in range(len(short_span)) :
for c in example.candidates:
start = short_span[s].start_token_idx
end = short_span[s].end_token_idx
if c["top_level"] and c["start_token"] <= start and c["end_token"] >= end:
long_span.append(Span(int(c["start_token"]), int(c["end_token"]), float(short_span[s].score)))
 long_span = remove_duplicates(long_span)
 long_span.sort(key = operator.itemgetter(2), reverse=True)
 if not long_span:
long_span = [Span(-1, -1, -10000.0)]
if not short_span:
short_span = [Span(-1, -1, -10000.0)]
return short_span, long_span
def get_bigrams(que):
return [word for word in word_tokenize(que.lower())if word not in eng_stopwords]
def get_common_bigrams(row):
return len(set(row["bigrams_ques1"] ).intersection(set(row["bigrams_ques2"])))
 def get_common_bigram_ratio(row):
return float(row["bigrams_common_count"])/ max(len(set(row["bigrams_ques1"] ).union(set(row["bigrams_ques2"]))),1)
 train_df["bigrams_ques1"] = train_df['unigrams_ques1'].apply(lambda x: get_unigrams(str(x)))
 train_df["bigrams_ques2"] = train_df['unigrams_ques2'].apply(lambda x: get_unigrams(str(x)))
 train_df["bigrams_common_count"] = train_df.apply(lambda row: get_common_unigrams(row),axis=1)
 train_df["bigrams_common_ratio"] = train_df.apply(lambda row: get_common_unigram_ratio(row), axis=1)
<string_transform>
def compute_predictions(example):

 predictions = []
n_best_size = FLAGS.n_best_size
max_answer_length = 1024
i = 0
for unique_id, result in example.results.items() :
if unique_id not in example.features:
raise ValueError("No feature found with unique_id:", unique_id)
 token_map = np.array(example.features[unique_id]["token_map"])
 start_indexes = top_k_indices(result.start_logits,n_best_size,token_map)
 if len(start_indexes)==0:
continue
end_indexes = top_k_indices(result.end_logits,n_best_size,token_map)
 if len(end_indexes)==0:
continue
indexes = np.array(list(np.broadcast(start_indexes[None],end_indexes[:,None])))
 indexes = indexes[(indexes[:,0]<indexes[:,1])*(indexes[:,1]-indexes[:,0]<max_answer_length)]
for _,(start_index,end_index)in enumerate(indexes):
summary = ScoreSummary()
summary.short_span_score =(
 result.start_logits[start_index] +
result.end_logits[end_index])
 summary.cls_token_score =(
 result.start_logits[0] + result.end_logits[0])
 summary.answer_type_logits = result.answer_type_logits-result.answer_type_logits.mean()
start_span = token_map[start_index]
end_span = token_map[end_index] + 1
score = summary.short_span_score - summary.cls_token_score
predictions.append(( score, i, summary, start_span, end_span))
i += 1
short_span = [Span(-1, -1, -10000.0)]
long_span = [Span(-1, -1, -10000.0)]
summary = ScoreSummary()
if predictions:
short_span, long_span = get_short_long_span(predictions, example)
 summary.predicted_label = {
"example_id": int(example.example_id),
"long_answers": {
"tokens_and_score": long_span,
"start_byte": -1,
"end_byte": -1
},
"short_answers": {
"tokens_and_score": short_span,
"start_byte": -1,
"end_byte": -1,
"yes_no_answer": "NONE"
}
}
return summary
def feature_extraction(row):
que1 = str(row['question1'])
 que2 = str(row['question2'])
 out_list = []
unigrams_que1 = [word for word in que1.lower().split() if word not in eng_stopwords]
unigrams_que2 = [word for word in que2.lower().split() if word not in eng_stopwords]
common_unigrams_len = len(set(unigrams_que1 ).intersection(set(unigrams_que2)))
 common_unigrams_ratio = float(common_unigrams_len)/ max(len(set(unigrams_que1 ).union(set(unigrams_que2))),1)
 out_list.extend([common_unigrams_len, common_unigrams_ratio])
 bigrams_que1 = [i for i in ngrams(unigrams_que1, 2)]
bigrams_que2 = [i for i in ngrams(unigrams_que2, 2)]
common_bigrams_len = len(set(bigrams_que1 ).intersection(set(bigrams_que2)))
 common_bigrams_ratio = float(common_bigrams_len)/ max(len(set(bigrams_que1 ).union(set(bigrams_que2))),1)
 out_list.extend([common_bigrams_len, common_bigrams_ratio])
 trigrams_que1 = [i for i in ngrams(unigrams_que1, 3)]
trigrams_que2 = [i for i in ngrams(unigrams_que2, 3)]
common_trigrams_len = len(set(trigrams_que1 ).intersection(set(trigrams_que2)))
 common_trigrams_ratio = float(common_trigrams_len)/ max(len(set(trigrams_que1 ).union(set(trigrams_que2))),1)
 out_list.extend([common_trigrams_len, common_trigrams_ratio])
 return out_list<train_model>
def read_candidates_from_one_split(input_path):

 candidates_dict = {}
print("Reading examples from: %s" % input_path)
 if input_path.endswith(".gz"):
with gzip.GzipFile(fileobj=tf.io.gfile.GFile(input_path, "rb")) as input_file:
for index, line in enumerate(input_file):
e = json.loads(line)
 candidates_dict[e["example_id"]] = e["long_answer_candidates"]
else:
with tf.io.gfile.GFile(input_path, "r")as input_file:
for index, line in enumerate(input_file):
e = json.loads(line)
 candidates_dict[e["example_id"]] = e["long_answer_candidates"]
return candidates_dict
def runXGB(train_X, train_y, test_X, test_y=None, feature_names=None, seed_val=0):
params = {}
params["objective"] = "binary:logistic"
params['eval_metric'] = 'logloss'
params["eta"] = 0.02
params["subsample"] = 0.7
params["min_child_weight"] = 1
params["colsample_bytree"] = 0.7
params["max_depth"] = 4
params["silent"] = 1
params["seed"] = seed_val
num_rounds = 300
plst = list(params.items())
 xgtrain = xgb.DMatrix(train_X, label=train_y)
 if test_y is not None:
xgtest = xgb.DMatrix(test_X, label=test_y)
 watchlist = [(xgtrain,'train'),(xgtest, 'test')]
model = xgb.train(plst, xgtrain, num_rounds, watchlist, early_stopping_rounds=100, verbose_eval=10)
 else:
xgtest = xgb.DMatrix(test_X)
 model = xgb.train(plst, xgtrain, num_rounds)
 pred_test_y = model.predict(xgtest)
 loss = 1
if test_y is not None:
loss = log_loss(test_y, pred_test_y)
 return pred_test_y, loss, model
else:
return pred_test_y, loss, model<prepare_x_and_y>
def read_candidates(input_pattern):

 input_paths = tf.io.gfile.glob(input_pattern)
 final_dict = {}
for input_path in input_paths:
final_dict.update(read_candidates_from_one_split(input_path))
return final_dict
train_X = np.vstack(np.array(train_df.apply(lambda row: feature_extraction(row), axis=1)))
 test_X = np.vstack(np.array(test_df.apply(lambda row: feature_extraction(row), axis=1)))
 train_y = np.array(train_df["is_duplicate"])
 test_id = np.array(test_df["test_id"] )<prepare_x_and_y>
all_results = [bert_utils.RawResult(*x)for x in zip(*result)]
print("Going to candidates file")
 candidates_dict = read_candidates('.. /input/tensorflow2-question-answering/simplified-nq-test.jsonl')
 print("setting up eval features")
 eval_features = list(token_map_ds)
 print("compute_pred_dict")
 tqdm_notebook= tqdm.tqdm_notebook
nq_pred_dict = compute_pred_dict(candidates_dict,
eval_features,
all_results,
tqdm=tqdm_notebook)
 predictions_json = {"predictions": list(nq_pred_dict.values())}
print("writing json")
 with tf.io.gfile.GFile('predictions.json', "w")as f:
json.dump(predictions_json, f, indent=4)
 print('done!' )
train_X_dup = train_X[train_y==1]
train_X_non_dup = train_X[train_y==0]
train_X = np.vstack([train_X_non_dup, train_X_dup, train_X_non_dup, train_X_non_dup])
 train_y = np.array([0]*train_X_non_dup.shape[0] + [1]*train_X_dup.shape[0] + [0]*train_X_non_dup.shape[0] + [0]*train_X_non_dup.shape[0])
 del train_X_dup
del train_X_non_dup
print("Mean target rate : ",train_y.mean() )<compute_train_metric>
answers_df = pd.read_json(".. /working/predictions.json")
 answers_df.head()
kf = KFold(n_splits=5, shuffle=True, random_state=2016)
 for dev_index, val_index in kf.split(range(train_X.shape[0])) :
dev_X, val_X = train_X[dev_index,:], train_X[val_index,:]
dev_y, val_y = train_y[dev_index], train_y[val_index]
preds, lloss, model = runXGB(dev_X, dev_y, val_X, val_y)
 break<save_to_csv>
def df_long_index_score(df):
answers = []
cont = 0
for e in df['long_answers']['tokens_and_score']:
if e[2] >= 2.5:
index = {}
index['start'] = e[0]
index['end'] = e[1]
index['score'] = e[2]
answers.append(index)
 cont += 1
if cont == 1:
break
return answers
def df_short_index_score(df):
answers = []
cont = 0
for e in df['short_answers']['tokens_and_score']:
if e[2] >= 6.5:
index = {}
index['start'] = e[0]
index['end'] = e[1]
index['score'] = e[2]
answers.append(index)
 cont += 1
if cont == 1:
break
return answers
def df_example_id(df):
return df['example_id']
xgtest = xgb.DMatrix(test_X)
 preds = model.predict(xgtest)
 out_df = pd.DataFrame({"test_id":test_id, "is_duplicate":preds})
 out_df.to_csv("xgb_starter.csv", index=False )<set_options>
answers_df['example_id'] = answers_df['predictions'].apply(df_example_id)
 answers_df['long_indexes_and_scores'] = answers_df['predictions'].apply(df_long_index_score)
 answers_df['short_indexes_and_scores'] = answers_df['predictions'].apply(df_short_index_score)
 answers_df.head()
stops = eng_stopwords = set(stopwords.words('english'))
color = sns.color_palette()
%matplotlib inline
pd.options.mode.chained_assignment = None<load_from_csv>
answers_df = answers_df.drop(['predictions'], axis=1)
 answers_df.head()
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print(train_df.shape)
 print(test_df.shape )<feature_engineering>
def create_answer(entry):
answer = []
for e in entry:
answer.append(str(e['start'])+ ':'+ str(e['end']))
if not answer:
answer = ""
return ", ".join(answer)
train_qs = pd.Series(train_df['question1'].tolist() + train_df['question2'].tolist() ).astype(str)
 def word_match_share(row):
q1words = {}
q2words = {}
for word in str(row['question1'] ).lower().split() :
if word not in stops:
q1words[word] = 1
for word in str(row['question2'] ).lower().split() :
if word not in stops:
q2words[word] = 1
if len(q1words)== 0 or len(q2words)== 0:
return 0
shared_words_in_q1 = [w for w in q1words.keys() if w in q2words]
shared_words_in_q2 = [w for w in q2words.keys() if w in q1words]
R =(len(shared_words_in_q1)+ len(shared_words_in_q2)) /(len(q1words)+ len(q2words))
return R
def get_weight(count, eps=10000, min_count=2):
if count < min_count:
return 0
else:
return 1 /(count + eps)
 eps = 5000
words =(" ".join(train_qs)).lower().split()
counts = Counter(words)
 weights = {word: get_weight(count)for word, count in counts.items() }
def tfidf_word_match_share(row):
q1words = {}
q2words = {}
for word in str(row['question1'] ).lower().split() :
if word not in stops:
q1words[word] = 1
for word in str(row['question2'] ).lower().split() :
if word not in stops:
q2words[word] = 1
if len(q1words)== 0 or len(q2words)== 0:
return 0
shared_weights = [weights.get(w, 0)for w in q1words.keys() if w in q2words] + [weights.get(w, 0)for w in q2words.keys() if w in q1words]
total_weights = [weights.get(w, 0)for w in q1words] + [weights.get(w, 0)for w in q2words]
R = np.sum(shared_weights)/ np.sum(total_weights)
 return R
def diff_ratios(st1, st2):
seq = difflib.SequenceMatcher()
seq.set_seqs(str(st1 ).lower() , str(st2 ).lower())
 return seq.ratio()<string_transform>
answers_df["long_answer"] = answers_df['long_indexes_and_scores'].apply(create_answer)
 answers_df["short_answer"] = answers_df['short_indexes_and_scores'].apply(create_answer)
 answers_df["example_id"] = answers_df['example_id'].apply(lambda q: str(q))
long_answers = dict(zip(answers_df["example_id"], answers_df["long_answer"]))
short_answers = dict(zip(answers_df["example_id"], answers_df["short_answer"]))
answers_df.head()
def feature_extraction(row):
que1 = str(row['question1'])
 que2 = str(row['question2'])
 out_list = []
unigrams_que1 = [word for word in que1.lower().split() if word not in eng_stopwords]
unigrams_que2 = [word for word in que2.lower().split() if word not in eng_stopwords]
common_unigrams_len = len(set(unigrams_que1 ).intersection(set(unigrams_que2)))
 common_unigrams_ratio = float(common_unigrams_len)/ max(len(set(unigrams_que1 ).union(set(unigrams_que2))),1)
 out_list.extend([common_unigrams_len, common_unigrams_ratio])
 bigrams_que1 = [i for i in ngrams(unigrams_que1, 2)]
bigrams_que2 = [i for i in ngrams(unigrams_que2, 2)]
common_bigrams_len = len(set(bigrams_que1 ).intersection(set(bigrams_que2)))
 common_bigrams_ratio = float(common_bigrams_len)/ max(len(set(bigrams_que1 ).union(set(bigrams_que2))),1)
 out_list.extend([common_bigrams_len, common_bigrams_ratio])
 trigrams_que1 = [i for i in ngrams(unigrams_que1, 3)]
trigrams_que2 = [i for i in ngrams(unigrams_que2, 3)]
common_trigrams_len = len(set(trigrams_que1 ).intersection(set(trigrams_que2)))
 common_trigrams_ratio = float(common_trigrams_len)/ max(len(set(trigrams_que1 ).union(set(trigrams_que2))),1)
 out_list.extend([common_trigrams_len, common_trigrams_ratio])
 return out_list<feature_engineering>
answers_df = answers_df.drop(['long_indexes_and_scores', 'short_indexes_and_scores'], axis=1)
 answers_df.head()
data = pd.DataFrame()
data['match_ratio'] = train_df.apply(lambda r: diff_ratios(r.question1, r.question2), axis=1)
 print('6')
 data['word_match_share'] = train_df.apply(lambda row: word_match_share(row), axis=1)
 print('5')
 data['tfidf_word_match_share'] = train_df.apply(lambda row: tfidf_word_match_share(row), axis=1)
 z =<feature_engineering>
sample_submission = pd.read_csv(".. /input/tensorflow2-question-answering/sample_submission.csv")
 long_prediction_strings = sample_submission[sample_submission["example_id"].str.contains("_long")].apply(lambda q: long_answers[q["example_id"].replace("_long", "")], axis=1)
 short_prediction_strings = sample_submission[sample_submission["example_id"].str.contains("_short")].apply(lambda q: short_answers[q["example_id"].replace("_short", "")], axis=1)
 sample_submission.loc[sample_submission["example_id"].str.contains("_long"), "PredictionString"] = long_prediction_strings
sample_submission.loc[sample_submission["example_id"].str.contains("_short"), "PredictionString"] = short_prediction_strings
<prepare_x_and_y><EOS> 
sample_submission.to_csv('submission.csv', index=False )