Kaggle competition: TensorFlow 2.0 Question Answering
<SOS> metric: NQMicroF1 Kaggle data source: tensorflow-2.0-question-answering<load_from_csv>
import time
train = pd.read_csv('.. /input/rossmann-store-sales/train.csv')
 test = pd.read_csv('.. /input/rossmann-store-sales/test.csv')
 store = pd.read_csv('.. /input/rossmann-store-sales/store.csv' )<filter>
%%time
!pip install.. /input/tf-20/tensorflow_gpu-2.0.0-cp36-cp36m-manylinux2010_x86_64.whl
train = train.loc[train.Open != 0]
train = train.loc[train.Sales > 0].reset_index(drop=True )<filter>
%%time
!pip install.. /input/tf-115-dependencies/natural_questions-1.0.4-py2.py3-none-any.whl --no-dependencies
train[train.isnull().values==True]<filter>
PATH = '.. /input/tf2qa-sub/tf2qa_sub'
test[test.isnull().values==True]<merge>
GPU_id = '0'
os.environ['CUDA_VISIBLE_DEVICES'] = GPU_id
train = pd.merge(train, store, on='Store')
 test = pd.merge(test, store, on='Store' )<feature_engineering>
print('tensorflow',tf.__version__ )
def build_features(features, data):
features.extend(['Store','CompetitionDistance','CompetitionOpenSinceMonth','StateHoliday','StoreType','Assortment',
'SchoolHoliday','CompetitionOpenSinceYear', 'Promo', 'Promo2', 'Promo2SinceWeek', 'Promo2SinceYear'])
 features.extend(['Year','Month','Day','DayOfWeek','WeekOfYear'])
 data['Year'] = data.Date.dt.year
data['Month'] = data.Date.dt.month
data['Day'] = data.Date.dt.day
data['DayOfWeek'] = data.Date.dt.dayofweek
data['WeekOfYear'] = data.Date.dt.weekofyear
features.extend(['CompetitionOpen','PromoOpen'])
 data['CompetitionOpen'] = 12*(data.Year-data.CompetitionOpenSinceYear)+(data.Month-data.CompetitionOpenSinceMonth)
 data['PromoOpen'] = 12*(data.Year-data.Promo2SinceYear)+(data.WeekOfYear-data.Promo2SinceWeek)/4.0
data['CompetitionOpen'] = data.CompetitionOpen.apply(lambda x: x if x > 0 else 0)
 data['PromoOpen'] = data.PromoOpen.apply(lambda x: x if x > 0 else 0)
 features.append('IsPromoMonth')
 month2str = {1:'Jan', 2:'Feb', 3:'Mar', 4:'Apr', 5:'May', 6:'Jun', 7:'Jul', 8:'Aug', 9:'Sept', 10:'Oct', 11:'Nov', 12:'Dec'}
data['monthStr'] = data.Month.map(month2str)
 data.loc[data.PromoInterval==0, 'PromoInterval'] = ''
data['IsPromoMonth'] = 0
for interval in data.PromoInterval.unique() :
if interval != '':
for month in interval.split(','):
data.loc[(data.monthStr == month)&(data.PromoInterval == interval), 'IsPromoMonth'] = 1
mappings = {'0':0, 'a':1, 'b':2, 'c':3, 'd':4}
data.StoreType.replace(mappings, inplace=True)
 data.Assortment.replace(mappings, inplace=True)
 data.StateHoliday.replace(mappings, inplace=True)
 data['StoreType'] = data['StoreType'].astype(int)
 data['Assortment'] = data['Assortment'].astype(int)
 data['StateHoliday'] = data['StateHoliday'].astype(int )<data_type_conversions>
%%time
gpus = tf.config.experimental.list_physical_devices('GPU')
 if gpus:
try:
for gpu in gpus:
tf.config.experimental.set_memory_growth(gpu, True)
 logical_gpus = tf.config.experimental.list_logical_devices('GPU')
 print(len(gpus), "Physical GPUs,", len(logical_gpus), "Logical GPUs")
 except RuntimeError as e:
print(e )
train.Date = pd.to_datetime(train.Date, errors='coerce')
 test.Date = pd.to_datetime(test.Date, errors='coerce')
 features = []
build_features(features, train)
 build_features([], test)
 print(features )<compute_test_metric>
sys.path.append(f"{PATH}/models-2.0")
 sys.path.append(f"{PATH}/jb_2.0")
def ToWeight(y):
w = np.zeros(y.shape, dtype=float)
 ind = y != 0
w[ind] = 1./(y[ind]**2)
 return w
def rmspe(yhat, y):
w = ToWeight(y)
 rmspe = np.sqrt(np.mean(w *(y-yhat)**2))
return rmspe
def rmspe_xg(yhat, y):
y = y.get_label()
y = np.expm1(y)
 yhat = np.expm1(yhat)
 w = ToWeight(y)
 rmspe = np.sqrt(np.mean(w *(y-yhat)**2))
return "rmspe", rmspe
def neg_rmspe(yhat, y):
y = np.expm1(y)
 yhat = np.expm1(yhat)
 w = ToWeight(y)
 rmspe = np.sqrt(np.mean(w *(y-yhat)**2))
return -rmspe<train_on_grid>
INPUT_DIR = '.. /input/tensorflow2-question-answering'
MODEL_DIR='.. /input/tf2qa-sub/tf2qa_sub'
BERT_FILE='.. /input/tf2qa-sub/tf2qa_sub/bert_config.json'
BATCH_SIZE = 32
CKPT = 'ctl_step_5158.ckpt-12'
regressor = DecisionTreeRegressor(random_state=2)
 cv_sets = ShuffleSplit(n_splits=5, test_size=0.2)
 params = {'max_depth':range(10,40,2)}
scoring_fnc = make_scorer(neg_rmspe)
 grid = GridSearchCV(regressor,params,scoring_fnc,cv=cv_sets)
 grid = grid.fit(train[features], np.log1p(train.Sales))
DTR = grid.best_estimator_<find_best_params>
def create_test() :
input_jsonl = f'{INPUT_DIR}/simplified-nq-test.jsonl'
output_tfrecord = 'simplified-nq-test.tfrecords'
meta_path = 'simplified-nq-test.meta'
vocab_file = f'{MODEL_DIR}/vocab-nq.txt'
if os.path.exists(meta_path):
print(meta_path,'exists.')
 return
tokenizer = tokenization.FullTokenizer(
 vocab_file=vocab_file, do_lower_case=FLAGS.do_lower_case)
 eval_examples = read_nq_examples(
 input_file=input_jsonl, is_training=False)
 eval_writer = FeatureWriter(
 filename=output_tfrecord,
is_training=False)
 real_examples = []
all_examples = []
def append_feature(feature,is_padding=False):
if not is_padding:
real_examples.append(1)
 eval_writer.process_feature(feature)
 all_examples.append(1)
 return len(real_examples)
 num_spans_to_ids = convert_examples_to_features(
 examples=eval_examples,
tokenizer=tokenizer,
is_training=False,
output_fn=append_feature,
predict_batch_size = BATCH_SIZE)
 eval_writer.close()
num_orig_examples = len(eval_examples)
 num_real_examples = len(real_examples)
 num_all_examples = len(all_examples)
 del eval_examples,real_examples,all_examples
meta_data = {
"task_type": "jb_nq",
"num_orig_examples": num_orig_examples,
"num_real_examples": num_real_examples,
"num_all_examples": num_all_examples,
"max_query_length": FLAGS.max_query_length,
"max_seq_length": FLAGS.max_seq_length,
"doc_stride": FLAGS.doc_stride,
}
with tf.io.gfile.GFile(meta_path, "w")as writer:
writer.write(json.dumps(meta_data, indent=4)+ "
" )
DTR.get_params()<save_to_csv>
def predict_jb() :
bert_config_file= BERT_FILE
model_dir=MODEL_DIR
input_meta_data_path='simplified-nq-test.meta'
predict_batch_size=BATCH_SIZE
predict_file='simplified-nq-test.tfrecords'
result_path=f'sub_{CKPT}'
use_checkpoint=f'{MODEL_DIR}/{CKPT}'
strategy_type = 'mirror'
tpu = False
strategy = get_strategy(strategy_type,tpu)
 input_meta_data = read_metas(input_meta_data_path)
 num_eval_examples = input_meta_data['num_orig_examples']
dataset_size = input_meta_data['num_all_examples']
print("***** Running predictions *****")
 print(" Num orig examples = %d" % num_eval_examples)
 print(" Num split examples = %d" % dataset_size)
 print(" Batch size = %d" % predict_batch_size)
 all_results = predict_jb_customized(strategy, input_meta_data,
bert_config_file,
predict_file,
model_dir,
use_tpu = tpu,
predict_batch_size = predict_batch_size,
use_checkpoint = use_checkpoint)
 with open(f'{result_path}.p','wb')as f:
pickle.dump(all_results,f)
 del all_results
submission = pd.DataFrame({"Id": test["Id"], "Sales": np.expm1(DTR.predict(test[features])) })
 submission.to_csv("benchmark.csv", index=False )<init_hyperparams>
%%time
predict_jb()
params = {'objective': 'reg:linear',
'eta': 0.01,
'max_depth': 11,
'subsample': 0.5,
'colsample_bytree': 0.5,
'silent': 1,
'seed': 1
}
num_trees = 10000<train_model>
def create_eval_nq_dataset(file_path, seq_length):

 name_to_features = {
'unique_ids': tf.io.FixedLenFeature([], tf.int64),
'input_ids': tf.io.FixedLenFeature([seq_length], tf.int64),
'input_mask': tf.io.FixedLenFeature([seq_length], tf.int64),
'segment_ids': tf.io.FixedLenFeature([seq_length], tf.int64),
'token_map': tf.io.FixedLenFeature([seq_length], tf.int64),
}
input_fn = input_pipeline.file_based_input_fn_builder(file_path, name_to_features)
 return input_fn()
def post_process() :
raw_prediction_file = f'sub_{CKPT}.p'
predict_file = f'{INPUT_DIR}/simplified-nq-test.jsonl'
predict_tfrecord = 'simplified-nq-test.tfrecords'
with open(raw_prediction_file,'rb')as f:
all_results = pickle.load(f)
 candidates_dict = read_candidates(predict_file)
 eval_features = [
r for r in create_eval_nq_dataset(predict_tfrecord,FLAGS.max_seq_length)
 ]
print(len(eval_features),len(all_results))
results = [r._asdict() for c,r in enumerate(all_results)if c<len(eval_features)]
nq_pred_dict,nbest_summary_dict = compute_pred_dict(candidates_dict, eval_features,results)
 output = {"predictions": list(nq_pred_dict.values())}
return output
X_train, X_test = train_test_split(train, test_size=0.2, random_state=2)
 dtrain = xgb.DMatrix(X_train[features], np.log1p(X_train.Sales))
dvalid = xgb.DMatrix(X_test[features], np.log1p(X_test.Sales))
dtest = xgb.DMatrix(test[features])
 watchlist = [(dtrain, 'train'),(dvalid, 'eval')]
gbm = xgb.train(params, dtrain, num_trees, evals=watchlist, early_stopping_rounds=50, feval=rmspe_xg, verbose_eval=False )<save_to_csv>
%%time
output = post_process()
test_probs = gbm.predict(xgb.DMatrix(test[features]), ntree_limit=gbm.best_ntree_limit)
 indices = test_probs < 0
test_probs[indices] = 0
submission = pd.DataFrame({"Id": test["Id"], "Sales": np.expm1(test_probs)})
 submission.to_csv("xgboost.csv", index=False )<set_options>
def create_short_answer(entry):
if entry["short_answers_score"] < short_thr:
return ""
answer = []
for short_answer in entry["short_answers"]:
if short_answer["start_token"] > -1:
answer.append(str(short_answer["start_token"])+ ":" + str(short_answer["end_token"]))
if entry["yes_no_answer"] != "NONE":
answer.append(entry["yes_no_answer"])
 return " ".join(answer)
 def create_long_answer(entry):
if entry["long_answer_score"] < long_thr:
return ""
answer = []
if entry["long_answer"]["start_token"] > -1:
answer.append(str(entry["long_answer"]["start_token"])+ ":" + str(entry["long_answer"]["end_token"]))
return " ".join(answer)
 def write_sub(test_answers_df):
path = f'{INPUT_DIR}/sample_submission.csv'
test_answers_df["long_answer_score"] = test_answers_df["predictions"].apply(lambda q: q["long_answer_score"])
 test_answers_df["short_answer_score"] = test_answers_df["predictions"].apply(lambda q: q["short_answers_score"])
 test_answers_df["long_answer"] = test_answers_df["predictions"].apply(create_long_answer)
 test_answers_df["short_answer"] = test_answers_df["predictions"].apply(create_short_answer)
 test_answers_df["example_id"] = test_answers_df["predictions"].apply(lambda q: str(q["example_id"]))
long_answers = dict(zip(test_answers_df["example_id"], test_answers_df["long_answer"]))
short_answers = dict(zip(test_answers_df["example_id"], test_answers_df["short_answer"]))
sample_submission = pd.read_csv(path)
 long_prediction_strings = sample_submission[sample_submission["example_id"].str.contains("_long")].apply(lambda q: long_answers[q["example_id"].replace("_long", "")], axis=1)
 short_prediction_strings = sample_submission[sample_submission["example_id"].str.contains("_short")].apply(lambda q: short_answers[q["example_id"].replace("_short", "")], axis=1)
 sample_submission.loc[sample_submission["example_id"].str.contains("_long"), "PredictionString"] = long_prediction_strings
sample_submission.loc[sample_submission["example_id"].str.contains("_short"), "PredictionString"] = short_prediction_strings
sample_submission.to_csv('submission.csv',index=False)
%matplotlib inline<load_from_csv>
%%time
df = pd.DataFrame(output)
 long_thr = 2.7368
short_thr = 6.8347
write_sub(df )
train = pd.read_csv('.. /input/rossmann-store-sales/train.csv')
 test = pd.read_csv('.. /input/rossmann-store-sales/test.csv')
 store = pd.read_csv('.. /input/rossmann-store-sales/store.csv' )<filter>
pd.read_csv('submission.csv' ).count()
<filter><EOS> 
FLAGS.__dict__