Kaggle competition: TensorFlow 2.0 Question Answering
for df in(joined, joined_test):
df['CompetitionOpenSince'] = pd.to_datetime(dict(year=df.CompetitionOpenSinceYear,
month=df.CompetitionOpenSinceMonth,
day=15))
df['CompetitionDaysOpen'] = df.Date.subtract(df.CompetitionOpenSince ).dt.days<feature_engineering>
use_wta = True
on_kaggle_server = os.path.exists('/kaggle')
 nq_test_file = '.. /input/tensorflow2-question-answering/simplified-nq-test.jsonl'
public_dataset = os.path.getsize(nq_test_file)<20_000_000
private_dataset = os.path.getsize(nq_test_file)>=20_000_000
model_path = '.. /input/tpu-2020-01-22/'
for k in ['on_kaggle_server','nq_test_file','public_dataset','private_dataset']:
print(k,globals() [k],sep=': ' )
for df in(joined, joined_test):
df.loc[df.CompetitionDaysOpen<0, "CompetitionDaysOpen"] = 0
df.loc[df.CompetitionOpenSinceYear<1990, "CompetitionDaysOpen"] = 0<feature_engineering>
model = tf.saved_model.load(model_path )
for df in(joined,joined_test):
df["CompetitionMonthsOpen"] = df["CompetitionDaysOpen"]//30
df.loc[df.CompetitionMonthsOpen>24, "CompetitionMonthsOpen"] = 24
joined.CompetitionMonthsOpen.unique()<data_type_conversions>
to_pkl.jsonl_to_pkl(source=nq_test_file,output='features.pkl',
vocab=model_path +'assets/vocab-nq.txt',
max_contexts=-1,lower_case=True )
for df in(joined, joined_test):
df["Promo2Since"] = pd.to_datetime(df.apply(
 lambda x: Week(x.Promo2SinceYear, x.Promo2SinceWeek ).monday() , axis=1))
df["Promo2Days"] = df.Date.subtract(df['Promo2Since'] ).dt.days<feature_engineering>
pkl_to_tfrecords._convert(source='features.pkl',output='all.tfrecords',meta_data='meta_data',shuffle=False,shuffle_size=0,yield_segment_variant='nolabels' )
for df in(joined,joined_test):
df.loc[df.Promo2Days<0, "Promo2Days"] = 0
df.loc[df.Promo2SinceYear<1990, "Promo2Days"] = 0
df["Promo2Weeks"] = df["Promo2Days"]//7
df.loc[df.Promo2Weeks<0, "Promo2Weeks"] = 0
df.loc[df.Promo2Weeks>25, "Promo2Weeks"] = 25
df.Promo2Weeks.unique()<data_type_conversions>
def input_fn(input_file_pattern,seq_length=512,batch_size=4):
def mk_labels(ex):
qlen = ex.pop('question_len')
 dlen = ex.pop('data_len')
 input_mask = tf.sequence_mask(dlen,seq_length,dtype=tf.int32)
 ex['input_mask'] = input_mask
ex['segment_ids'] = tf.minimum(input_mask,1-tf.sequence_mask(qlen,seq_length,dtype=tf.int32))
return ex
name_to_features = {
'input_ids' : tf.io.FixedLenFeature([seq_length], tf.int64),
'question_len': tf.io.FixedLenFeature([], tf.int64),
'data_len' : tf.io.FixedLenFeature([], tf.int64),
}
name_to_features['unique_id'] = tf.io.FixedLenFeature([2], tf.int64)
 def decode(record):
ex = tf.io.parse_single_example(record, name_to_features)
 for k,v in ex.items() :
if k!='unique_id':
ex[k] = tf.cast(v,tf.int32)
 return ex
input_files = tf.io.gfile.glob(input_file_pattern)
 d = tf.data.TFRecordDataset(input_files)
 d = d.map(decode)
 d = d.batch(batch_size,drop_remainder=False)
 d = d.prefetch(128)
 return d
def get_elapsed(fld, pre):
day1 = np.timedelta64(1, 'D')
 last_date = np.datetime64()
last_store = 0
res = []
for s,v,d in zip(df.Store.values, df[fld].values, df.Date.values):
if s != last_store:
last_date = np.datetime64()
last_store = s
if v: last_date = d
res.append(((d-last_date ).astype('timedelta64[D]')/ day1))
df[pre+fld] = res<define_variables>
def output_fn() :
def _output_fn(unique_id,model_output,n_keep=100):
pos_logits,ans_logits,long_mask,short_mask,cross = model_output
long_span_logits = pos_logits
mask = tf.cast(tf.expand_dims(long_mask,-1),long_span_logits.dtype)
 long_span_logits = long_span_logits-10000*mask
long_p = tf.nn.softmax(long_span_logits,axis=1)
 short_span_logits = pos_logits
short_span_logits -= 10000*tf.cast(tf.expand_dims(short_mask,-1),short_span_logits.dtype)
 start_logits,end_logits = short_span_logits[:,:,0],short_span_logits[:,:,1]
batch_size,seq_length = short_span_logits.shape[0],short_span_logits.shape[1]
seq = tf.range(seq_length)
 i_leq_j_mask = tf.cast(tf.expand_dims(seq,1)>tf.expand_dims(seq,0),short_span_logits.dtype)
 i_leq_j_mask = tf.expand_dims(i_leq_j_mask,0)
 logits = tf.expand_dims(start_logits,2)+tf.expand_dims(end_logits,1)+cross
logits -= 10000*i_leq_j_mask
logits = tf.reshape(logits, [batch_size,seq_length*seq_length])
 short_p = tf.nn.softmax(logits)
 indices = tf.argsort(short_p,axis=1,direction='DESCENDING')[:,:n_keep]
short_p = tf.gather(short_p,indices,batch_dims=1)
 return dict(unique_id = unique_id,
ans_logits= ans_logits,
long_p = long_p,
short_p = short_p,
short_p_indices = indices)
 return _output_fn
columns = ["Date", "Store", "Promo", "StateHoliday", "SchoolHoliday"]<concatenate>
d = input_fn('all.tfrecords',batch_size=64)
 if public_dataset:
d = d.take(3)
 if not on_kaggle_server:
d = tqdm.notebook.tqdm(d)
 results = []
output = output_fn()
for b in d:
unique_id = b.pop('unique_id' ).numpy()
b = [b['data_len'],b['input_ids'],b['question_len']]
out_dict = output(unique_id,model(b,training=False))
for k,v in out_dict.items() :
if isinstance(v,tf.Tensor):
out_dict[k] = v.numpy()
results.append(out_dict)
 raw_results = postprocess.read_rawresult(results)
df = train[columns].append(test[columns])
 df.head(3 )<sort_values>
examples = postprocess.compute_examples(raw_results,records )
fld = 'SchoolHoliday'
df = df.sort_values(['Store', 'Date'])
 get_elapsed(fld, 'After')
 df = df.sort_values(['Store', 'Date'], ascending=[True, False])
 get_elapsed(fld, 'Before' )<sort_values>
index = pd.read_csv('.. /input/tensorflow2-question-answering/sample_submission.csv' ).example_id
submission = postprocess.create_submission_df(predictions,index=index,
long_threshold=0.94 if use_wta else 0.77,
short_threshold=0.94 if use_wta else 0.77 ,
yes_no_threshold=0.6 )
<sort_values><EOS> 
submission.to_csv('submission.csv' )