Kaggle competition: TensorFlow 2.0 Question Answering
cv_conv=train_conv[train_mask]
test_conv=train_conv[test_mask]<prepare_x_and_y>
on_kaggle_server = os.path.exists('/kaggle')
 nq_test_file = '.. /input/tensorflow2-question-answering/simplified-nq-test.jsonl'
nq_train_file = '.. /input/tensorflow2-question-answering/simplified-nq-train.jsonl'
public_dataset = os.path.getsize(nq_test_file)<20_000_000
private_dataset = os.path.getsize(nq_test_file)>=20_000_000
Y_c = np.zeros(( train_runn.shape[0], 199))
for idx, target in enumerate(train_runn['Yards']):
Y_c[idx][99 + target] = 1
cv_Y_c=Y_c[train_mask]
test_Y_c=Y_c[test_mask]<import_modules>
if True:
importlib.reload(bert_utils )
from keras.layers import Dense
from keras.models import Sequential,Model
from keras.callbacks import Callback, EarlyStopping
from keras.layers import Dropout, PReLU, BatchNormalization, concatenate,ELU,AveragePooling2D, GaussianNoise, Activation,Dense, Conv2D, Flatten,MaxPooling2D,Embedding,Input,Reshape,Concatenate
from keras.optimizers import Adam
from keras.losses import categorical_crossentropy 
<compute_test_metric>
config = {'attention_probs_dropout_prob':0.1,
'hidden_act':'gelu',
'hidden_dropout_prob':0.1,
'hidden_size':1024,
'initializer_range':0.02,
'intermediate_size':4096,
'max_position_embeddings':512,
'num_attention_heads':16,
'num_hidden_layers':24,
'type_vocab_size':2,
'vocab_size':30522}
def crps_loss(y_true, y_pred):
return K.mean(K.square(K.clip(K.cumsum(y_true, axis=1),0,1)- K.clip(K.cumsum(y_pred, axis=1),0,1)) , axis=1 )<compute_test_metric>
class TDense(tf.keras.layers.Layer):
def __init__(self,
output_size,
kernel_initializer=None,
bias_initializer="zeros",
**kwargs):
super().__init__(**kwargs)
 self.output_size = output_size
self.kernel_initializer = kernel_initializer
self.bias_initializer = bias_initializer
def build(self,input_shape):
dtype = tf.as_dtype(self.dtype or tf.keras.backend.floatx())
 if not(dtype.is_floating or dtype.is_complex):
raise TypeError("Unable to build `TDense` layer with "
"non-floating point(and non-complex)"
"dtype %s" %(dtype,))
input_shape = tf.TensorShape(input_shape)
 if tf.compat.dimension_value(input_shape[-1])is None:
raise ValueError("The last dimension of the inputs to "
"`TDense` should be defined."
"Found `None`.")
 last_dim = tf.compat.dimension_value(input_shape[-1])
 self.input_spec = tf.keras.layers.InputSpec(min_ndim=3, axes={-1: last_dim})
 self.kernel = self.add_weight(
 "kernel",
shape=[self.output_size,last_dim],
initializer=self.kernel_initializer,
dtype=self.dtype,
trainable=True)
 self.bias = self.add_weight(
 "bias",
shape=[self.output_size],
initializer=self.bias_initializer,
dtype=self.dtype,
trainable=True)
 super(TDense, self ).build(input_shape)
 def call(self,x):
return tf.matmul(x,self.kernel,transpose_b=True)+self.bias
def mk_model(config):
seq_len = config['max_position_embeddings']
unique_id = tf.keras.Input(shape=(1,),dtype=tf.int64,name='unique_id')
 input_ids = tf.keras.Input(shape=(seq_len,),dtype=tf.int32,name='input_ids')
 input_mask = tf.keras.Input(shape=(seq_len,),dtype=tf.int32,name='input_mask')
 segment_ids = tf.keras.Input(shape=(seq_len,),dtype=tf.int32,name='segment_ids')
 BERT = modeling.BertModel(config=config,name='bert')
 pooled_output, sequence_output = BERT(input_word_ids=input_ids,
input_mask=input_mask,
input_type_ids=segment_ids)
 logits = TDense(2,name='logits' )(sequence_output)
 start_logits,end_logits = tf.split(logits,axis=-1,num_or_size_splits= 2,name='split')
 start_logits = tf.squeeze(start_logits,axis=-1,name='start_squeeze')
 end_logits = tf.squeeze(end_logits, axis=-1,name='end_squeeze')
 ans_type = TDense(5,name='ans_type' )(pooled_output)
 return tf.keras.Model([input_ for input_ in [unique_id,input_ids,input_mask,segment_ids]
if input_ is not None],
[unique_id,start_logits,end_logits,ans_type],
name='bert-baseline' )
def ordinal_loss(y_true, y_pred):
weights = K.cast(K.abs(K.argmax(y_true, axis=1)- K.argmax(y_pred, axis=1)) /(K.int_shape(y_pred)[1] - 1), dtype='float32')
 return(1.0 + weights)* categorical_crossentropy(y_true, y_pred )<compute_test_metric>
model= mk_model(config )
def ordinal_loss_2(y_true, y_pred):
weights = K.cast(K.square(K.argmax(y_true, axis=1)- K.argmax(y_pred, axis=1)) /(K.int_shape(y_pred)[1] - 1), dtype='float32')
 return(1.0 + weights)* losses.categorical_crossentropy(y_true, y_pred )<compute_test_metric>
class DummyObject:
def __init__(self,**kwargs):
self.__dict__.update(kwargs)
 FLAGS=DummyObject(skip_nested_contexts=True,
max_position=50,
max_contexts=48,
max_query_length=64,
max_seq_length=512,
doc_stride=128,
include_unknowns=0.02,
n_best_size=5,
max_answer_length=30,
warmup_proportion=0.1,
learning_rate=1e-5,
num_train_epochs=3.0,
train_batch_size=32,
num_train_steps=100000,
num_warmup_steps=10000,
max_eval_steps=100,
use_tpu=False,
eval_batch_size=8,
max_predictions_per_seq=20 )
def ordinal_loss_h(y_true, y_pred,delta=10):
weights = K.cast(K.abs(K.argmax(y_true, axis=1)- K.argmax(y_pred, axis=1)) /(K.int_shape(y_pred)[1] - 1), dtype='float32')
 mae=(1.0 + weights)* categorical_crossentropy(y_true, y_pred)
 quadratic=K.minimum(mae, delta)
 linear=mae - quadratic
return 0.5 * K.square(quadratic)+ delta * linear<count_unique_values>
eval_records = ".. /input/bert-joint-baseline/nq-test.tfrecords"
if on_kaggle_server and private_dataset:
eval_records='nq-test.tfrecords'
if not os.path.exists(eval_records):
eval_writer = bert_utils.FeatureWriter(
 filename=os.path.join(eval_records),
is_training=False)
 tokenizer = tokenization.FullTokenizer(vocab_file='.. /input/bert-joint-baseline/vocab-nq.txt',
do_lower_case=True)
 features = []
convert = bert_utils.ConvertExamples2Features(tokenizer=tokenizer,
is_training=False,
output_fn=eval_writer.process_feature,
collect_stat=False)
 n_examples = 0
tqdm_notebook= tqdm.tqdm_notebook if not on_kaggle_server else None
for examples in bert_utils.nq_examples_iter(input_file=nq_test_file,
is_training=False,
tqdm=tqdm_notebook):
for example in examples:
n_examples += convert(example)
 eval_writer.close()
print('number of test examples: %d, written to file: %d' %(n_examples,eval_writer.num_features))
cat_col_map=dict(train_runn[cat_cols].nunique() )<choose_model_class>
seq_length = FLAGS.max_seq_length
name_to_features = {
"unique_id": tf.io.FixedLenFeature([], tf.int64),
"input_ids": tf.io.FixedLenFeature([seq_length], tf.int64),
"input_mask": tf.io.FixedLenFeature([seq_length], tf.int64),
"segment_ids": tf.io.FixedLenFeature([seq_length], tf.int64),
}
def _decode_record(record, name_to_features=name_to_features):

 example = tf.io.parse_single_example(serialized=record, features=name_to_features)
 for name in list(example.keys()):
t = example[name]
if name != 'unique_id':
t = tf.cast(t, dtype=tf.int64)
 example[name] = t
return example
def _decode_tokens(record):
return tf.io.parse_single_example(serialized=record,
features={
"unique_id": tf.io.FixedLenFeature([], tf.int64),
"token_map" : tf.io.FixedLenFeature([seq_length], tf.int64)
 })
def cont_model(cat_cols=cat_cols,cont_cols=cont_cols):
inputs=[]
embeddings=[]
for i in cat_col_map.keys() :
input=Input(shape=(1,))
vals=cat_col_map[i]
embs=int(np.sqrt(vals))
embedding=Embedding(vals,embs,input_length=1 )(input)
 embedding=Reshape(target_shape=(embs,))(embedding)
 inputs.append(input)
 embeddings.append(embedding)
 cont_input=Input(shape=(len(cont_cols),))
inputs.append(cont_input)
 embeddings.append(cont_input)
 cont_x=Concatenate()(embeddings)
 cont_x=Dense(32,input_dim=inputs, activation=None )(cont_x)
 cont_x=BatchNormalization()(cont_x)
 cont_x=PReLU()(cont_x)
 cont_x=Dropout(0.7 )(cont_x)
 cont_x=GaussianNoise(0.2 )(cont_x)
 cont_x=Dense(32, activation=None )(cont_x)
 cont_x=BatchNormalization()(cont_x)
 cont_x=PReLU()(cont_x)
 cont_x=Dropout(0.5 )(cont_x)
 cont_x=GaussianNoise(0.2 )(cont_x)
 cont_model=Model(inputs,cont_x)
 return cont_model<choose_model_class>
raw_ds = tf.data.TFRecordDataset(eval_records)
 token_map_ds = raw_ds.map(_decode_tokens)
 decoded_ds = raw_ds.map(_decode_record)
 ds = decoded_ds.batch(batch_size=32,drop_remainder=False )
def conv_model2(cols):
conv_input=Input(shape=(cols,22,1))
conv_x=conv_input
conv_3=Conv2D(64, kernel_size=(cols,3), activation='relu',
input_shape=(cols,22,1),padding='same' )(conv_x)
 conv_5=Conv2D(64, kernel_size=(cols,5), activation='relu',
input_shape=(cols,22,1),padding='same' )(conv_x)
 conv_7=Conv2D(64, kernel_size=(cols,7), activation='relu',
input_shape=(cols,22,1),padding='same' )(conv_x)
 conv_out=concatenate([conv_3,conv_5,conv_7],axis=3)
 conv_out=Flatten()(conv_out)
 conv_out=BatchNormalization()(conv_out)
 conv_out=Dropout(0.7 )(conv_out)
 conv_out=GaussianNoise(0.2 )(conv_out)
 conv_out=Dense(32, activation=None )(conv_out)
 conv_out=BatchNormalization()(conv_out)
 conv_out=PReLU()(conv_out)
 conv_model=Model(conv_input,conv_out)
 return conv_model 
<choose_model_class>
result=model.predict_generator(ds,verbose=1 )
def conv_model(cols,player_width):
conv_input=Input(shape=(cols,22,1))
conv_x=conv_input
conv_x=Conv2D(64, kernel_size=(cols,player_width), activation='relu',
input_shape=(cols,22,1),padding='same' )(conv_x)
 conv_x=BatchNormalization(axis=1 )(conv_x)
 conv_x=AveragePooling2D(pool_size=(1,player_width),strides=1 )(conv_x)
 conv_x=Conv2D(32, kernel_size=(1,player_width), activation='relu',
padding='same' )(conv_x)
 conv_x=BatchNormalization(axis=1 )(conv_x)
 conv_x=AveragePooling2D(pool_size=(1,player_width),strides=1 )(conv_x)
 conv_x=Conv2D(32, kernel_size=(1,player_width), activation='relu',
padding='same' )(conv_x)
 conv_x=BatchNormalization(axis=1 )(conv_x)
 conv_x=AveragePooling2D(pool_size=(1,player_width),strides=1 )(conv_x)
 conv_x=Flatten()(conv_x)
 conv_x=BatchNormalization()(conv_x)
 conv_x=Dropout(0.5 )(conv_x)
 conv_x=GaussianNoise(0.2 )(conv_x)
 conv_x=Dense(32, activation=None )(conv_x)
 conv_x=BatchNormalization()(conv_x)
 conv_x=PReLU()(conv_x)
 conv_model=Model(conv_input,conv_x)
 return conv_model 
<define_variables>
np.savez_compressed('bert-joint-baseline-output.npz',
**dict(zip(['uniqe_id','start_logits','end_logits','answer_type_logits'],
result)) )
cont_inputs=train_cont.shape[1]
cont_inputs<choose_model_class>
Span = collections.namedtuple("Span", ["start_token_idx", "end_token_idx", "score"] )
models=concatenate([cont_model_.output,conv_model_.output])
 models=BatchNormalization()(models)
 models=Dropout(0.7 )(models)
 models=GaussianNoise(0.2 )(models)
 combout=Dense(32,activation=None )(models)
 combout=BatchNormalization()(combout)
 combout=PReLU()(combout)
 combout=Dropout(0.7 )(combout)
 combout=GaussianNoise(0.2 )(combout)
 combout=Dense(32,activation=None )(models)
 combout=BatchNormalization()(combout)
 combout=PReLU()(combout)
 combout=Dropout(0.5 )(combout)
 combout=GaussianNoise(0.2 )(combout)
 combout=Dense(199, activation='softmax' )(combout )<choose_model_class>
class ScoreSummary(object):
def __init__(self):
self.predicted_label = None
self.short_span_score = None
self.cls_token_score = None
self.answer_type_logits = None
big_model=Model(inputs=cont_model_.input+[conv_model_.input],outputs=combout )<define_variables>
class EvalExample(object):

 def __init__(self, example_id, candidates):
self.example_id = example_id
self.candidates = candidates
self.results = {}
self.features = {}
np.random.seed(214)
<train_model>
def get_best_indexes(logits, n_best_size):

 index_and_score = sorted(
 enumerate(logits[1:], 1), key=lambda x: x[1], reverse=True)
 best_indexes = []
for i in range(len(index_and_score)) :
if i >= n_best_size:
break
best_indexes.append(index_and_score[i][0])
 return best_indexes
def top_k_indices(logits,n_best_size,token_map):
indices = np.argsort(logits[1:])+1
indices = indices[token_map[indices]!=-1]
return indices[-n_best_size:]
big_model.compile(optimizer=Adam(learning_rate=0.005,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-07,
amsgrad=False), loss=ordinal_loss,
metrics=[crps_loss] 
)
 es=EarlyStopping(patience=10, min_delta=5e-5, restore_best_weights=True, monitor='val_crps_loss')
 big_model.fit(cv_cat+[cv_cont,cv_conv], cv_Y_c, callbacks=[es], epochs=100, batch_size=512,
verbose=True,validation_data=(test_cat+[test_cont,test_conv],test_Y_c)) 
<predict_on_test>
def remove_duplicates(span):
start_end = []
for s in span:
cont = 0
if not start_end:
start_end.append(Span(s[0], s[1], s[2]))
cont += 1
else:
for i in range(len(start_end)) :
if start_end[i][0] == s[0] and start_end[i][1] == s[1]:
cont += 1
if cont == 0:
start_end.append(Span(s[0], s[1], s[2]))
return start_end
train_preds=big_model.predict(train_catt+[train_cont,train_conv])
<define_variables>
def get_short_long_span(predictions, example):
sorted_predictions = sorted(predictions, reverse=True)
 short_span = []
long_span = []
for prediction in sorted_predictions:
score, _, summary, start_span, end_span = prediction
if score > 0:
short_span.append(Span(int(start_span), int(end_span), float(score)))
 short_span = remove_duplicates(short_span)
 for s in range(len(short_span)) :
for c in example.candidates:
start = short_span[s].start_token_idx
end = short_span[s].end_token_idx
if c["top_level"] and c["start_token"] <= start and c["end_token"] >= end:
long_span.append(Span(int(c["start_token"]), int(c["end_token"]), float(short_span[s].score)))
 break
long_span = remove_duplicates(long_span)
 if not long_span:
long_span = [Span(-1, -1, -10000.0)]
if not short_span:
short_span = [Span(-1, -1, -10000.0)]
return short_span, long_span
train_predslon=np.sum(train_preds*np.arange(-99,100),axis=1)
<feature_engineering>
def compute_predictions(example):

 predictions = []
n_best_size = FLAGS.n_best_size
max_answer_length = FLAGS.max_answer_length
i = 0
for unique_id, result in example.results.items() :
if unique_id not in example.features:
raise ValueError("No feature found with unique_id:", unique_id)
 token_map = np.array(example.features[unique_id]["token_map"])
 start_indexes = top_k_indices(result.start_logits,n_best_size,token_map)
 if len(start_indexes)==0:
continue
end_indexes = top_k_indices(result.end_logits,n_best_size,token_map)
 if len(end_indexes)==0:
continue
indexes = np.array(list(np.broadcast(start_indexes[None],end_indexes[:,None])))
 indexes = indexes[(indexes[:,0]<indexes[:,1])*(indexes[:,1]-indexes[:,0]<max_answer_length)]
for _,(start_index,end_index)in enumerate(indexes):
summary = ScoreSummary()
summary.short_span_score =(
 result.start_logits[start_index] +
result.end_logits[end_index])
 summary.cls_token_score =(
 result.start_logits[0] + result.end_logits[0])
 summary.answer_type_logits = result.answer_type_logits-result.answer_type_logits.mean()
start_span = token_map[start_index]
end_span = token_map[end_index] + 1
score = summary.short_span_score - summary.cls_token_score
predictions.append(( score, i, summary, start_span, end_span))
i += 1
short_span = [Span(-1, -1, -10000.0)]
long_span = [Span(-1, -1, -10000.0)]
summary = ScoreSummary()
if predictions:
short_span, long_span = get_short_long_span(predictions, example)
 summary.predicted_label = {
"example_id": int(example.example_id),
"long_answers": {
"tokens_and_score": long_span,
"start_byte": -1,
"end_byte": -1
},
"short_answers": {
"tokens_and_score": short_span,
"start_byte": -1,
"end_byte": -1,
"yes_no_answer": "NONE"
}
}
return summary
train_runn['pred']=train_predslon 
<compute_test_metric>
def read_candidates_from_one_split(input_path):

 candidates_dict = {}
print("Reading examples from: %s" % input_path)
 if input_path.endswith(".gz"):
with gzip.GzipFile(fileobj=tf.io.gfile.GFile(input_path, "rb")) as input_file:
for index, line in enumerate(input_file):
e = json.loads(line)
 candidates_dict[e["example_id"]] = e["long_answer_candidates"]
else:
with tf.io.gfile.GFile(input_path, "r")as input_file:
for index, line in enumerate(input_file):
e = json.loads(line)
 candidates_dict[e["example_id"]] = e["long_answer_candidates"]
return candidates_dict
lossess=np.mean(np.square(np.clip(np.cumsum(train_preds, axis=1),0,1)- np.clip(np.cumsum(Y_c, axis=1),0,1)) , axis=1)
 lossess.mean()<feature_engineering>
def read_candidates(input_pattern):

 input_paths = tf.io.gfile.glob(input_pattern)
 final_dict = {}
for input_path in input_paths:
final_dict.update(read_candidates_from_one_split(input_path))
return final_dict
train_runn['loss']=lossess<define_search_space>
all_results = [bert_utils.RawResult(*x)for x in zip(*result)]
print("Going to candidates file")
 candidates_dict = read_candidates('.. /input/tensorflow2-question-answering/simplified-nq-test.jsonl')
 print("setting up eval features")
 eval_features = list(token_map_ds)
 print("compute_pred_dict")
 tqdm_notebook= tqdm.tqdm_notebook
nq_pred_dict = compute_pred_dict(candidates_dict,
eval_features,
all_results,
tqdm=tqdm_notebook)
 predictions_json = {"predictions": list(nq_pred_dict.values())}
print("writing json")
 with tf.io.gfile.GFile('predictions.json', "w")as f:
json.dump(predictions_json, f, indent=4)
 print('done!' )
train_runn.loss.quantile([.1,.25,.5,.75,.8,.9,.95,1] )<import_modules>
answers_df = pd.read_json(".. /working/predictions.json")
 answers_df.head()
from sklearn.metrics import r2_score<compute_test_metric>
def df_long_index_score(df):
answers = []
cont = 0
for e in df['long_answers']['tokens_and_score']:
if e[2] > 3:
index = {}
index['start'] = e[0]
index['end'] = e[1]
index['score'] = e[2]
answers.append(index)
 cont += 1
if cont == 1:
break
return answers
def df_short_index_score(df):
answers = []
cont = 0
for e in df['short_answers']['tokens_and_score']:
if e[2] > 8:
index = {}
index['start'] = e[0]
index['end'] = e[1]
index['score'] = e[2]
answers.append(index)
 cont += 1
if cont == 1:
break
return answers
def df_example_id(df):
return df['example_id']
r2_score(train_runn['Yards'],train_runn['pred'] )<sort_values>
answers_df['example_id'] = answers_df['predictions'].apply(df_example_id)
 answers_df['long_indexes_and_scores'] = answers_df['predictions'].apply(df_long_index_score)
 answers_df['short_indexes_and_scores'] = answers_df['predictions'].apply(df_short_index_score)
 answers_df.head()
big_preds=train_runn.sort_values('pred',ascending=False)[['PlayId','PossTeam','PossScore','DefTeam','DefScore','DisplayName','Quarter','GameClock','Down','Distance','Yards','pred','loss']].head(10)
 big_preds<sort_values>
answers_df = answers_df.drop(['predictions'], axis=1)
 answers_df.head()
small_preds=train_runn.sort_values('pred',ascending=True)[['PlayId','PossTeam','PossScore','DefTeam','DefScore','DisplayName','Quarter','GameClock','Down','Distance','Yards','pred','loss']].head(10)
 small_preds<split>
def create_answer(entry):
answer = []
for e in entry:
answer.append(str(e['start'])+ ':'+ str(e['end']))
if not answer:
answer = ""
return ", ".join(answer)
iter_test=env.iter_test()<categorify>
answers_df["long_answer"] = answers_df['long_indexes_and_scores'].apply(create_answer)
 answers_df["short_answer"] = answers_df['short_indexes_and_scores'].apply(create_answer)
 answers_df["example_id"] = answers_df['example_id'].apply(lambda q: str(q))
long_answers = dict(zip(answers_df["example_id"], answers_df["long_answer"]))
short_answers = dict(zip(answers_df["example_id"], answers_df["short_answer"]))
answers_df.head()
def clipper(yl,predsin):
lind=-yl+99
rind=100-yl+99
predsin=np.clip(np.cumsum(predsin),0,1)
 predsin[:lind]=0
predsin[rind:]=1
return predsin<predict_on_test>
answers_df = answers_df.drop(['long_indexes_and_scores', 'short_indexes_and_scores'], axis=1)
 answers_df.head()
for(test_df, sample_prediction_df)in iter_test:
testt,test_runn,test_conv,test_cont,test_cat,tall_pipe,cont_pipe,cat_pipe=feature_magic(test_df,lefty=False,training=False)
 test_pred=big_model.predict(test_cat+[test_cont,test_conv])
 test_pred=clipper(test_runn.YardsFromOwnGoal.values[0],test_pred)
 pred_df=pd.DataFrame(data=[test_pred],columns=sample_prediction_df.columns)
 env.predict(pred_df )<save_to_csv>
sample_submission = pd.read_csv(".. /input/tensorflow2-question-answering/sample_submission.csv")
 long_prediction_strings = sample_submission[sample_submission["example_id"].str.contains("_long")].apply(lambda q: long_answers[q["example_id"].replace("_long", "")], axis=1)
 short_prediction_strings = sample_submission[sample_submission["example_id"].str.contains("_short")].apply(lambda q: short_answers[q["example_id"].replace("_short", "")], axis=1)
 sample_submission.loc[sample_submission["example_id"].str.contains("_long"), "PredictionString"] = long_prediction_strings
sample_submission.loc[sample_submission["example_id"].str.contains("_short"), "PredictionString"] = short_prediction_strings
<import_modules><EOS> 
sample_submission.to_csv('submission.csv', index=False )