Kaggle competition: TensorFlow 2.0 Question Answering
<SOS> metric: NQMicroF1 Kaggle data source: tensorflow-2.0-question-answering<split>
import os
import sys
import json
import tensorflow as tf
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import absl
import datetime
kf = model_selection.KFold(n_splits=5, shuffle=True, random_state=2017)
 pred_test_full = 0
for dev_index, val_index in kf.split(train_X):
dev_X, val_X = train_X.loc[dev_index,:], train_X.loc[val_index,:]
dev_y, val_y = train_y[dev_index], train_y[val_index]
pred_test, model, evals_result = run_lgb(dev_X, dev_y, val_X, val_y, test_X)
 pred_test_full += pred_test
pred_test_full /= 5.
 pred_test_full = np.expm1(pred_test_full )<save_to_csv>
IS_KAGGLE = True
INPUT_DIR = "/kaggle/input/"
BERT_JOINT_BASE_DIR = os.path.join(INPUT_DIR, "bertjointbaseline")
 NQ_DIR = os.path.join(INPUT_DIR, "nq-competition")
 MY_OWN_NQ_DIR = NQ_DIR
if not os.path.isdir(INPUT_DIR):
IS_KAGGLE = False
INPUT_DIR = "./"
NQ_DIR = "./"
MY_OWN_NQ_DIR = "./"
for dirname, _, filenames in os.walk(INPUT_DIR):
for filename in filenames:
print(os.path.join(dirname, filename))
INPUT_DIR
sub_df = pd.DataFrame({"ID":test_df["ID"].values})
 sub_df["target"] = pred_test_full
sub_df.to_csv("baseline_lgb.csv", index=False )<import_modules>
PRETRAINED_MODELS = {
"BERT": [
'bert-base-uncased',
'bert-large-uncased',
'bert-base-cased',
'bert-large-cased',
'bert-base-multilingual-uncased',
'bert-base-multilingual-cased',
'bert-base-chinese',
'bert-base-german-cased',
'bert-large-uncased-whole-word-masking',
'bert-large-cased-whole-word-masking',
'bert-large-uncased-whole-word-masking-finetuned-squad',
'bert-large-cased-whole-word-masking-finetuned-squad',
'bert-base-cased-finetuned-mrpc'
],
"DISTILBERT": [
'distilbert-base-uncased',
'distilbert-base-uncased-distilled-squad'
]
}
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
from keras.models import Model
from keras.layers import Dense, Input
from keras.layers.merge import concatenate
from keras.optimizers import SGD<sort_values>
flags = absl.flags
del_all_flags(flags.FLAGS)
 flags.DEFINE_bool(
 "do_lower_case", True,
"Whether to lower case the input text.Should be True for uncased "
"models and False for cased models.")
 vocab_file = os.path.join(NQ_DIR, "vocab-nq.txt")
 flags.DEFINE_string("vocab_file", vocab_file,
"The vocabulary file that the BERT model was trained on.")
 flags.DEFINE_integer(
 "max_seq_length_for_training", 512,
"The maximum total input sequence length after WordPiece tokenization for training examples."
"Sequences longer than this will be truncated, and sequences shorter "
"than this will be padded.")
 flags.DEFINE_integer(
 "max_seq_length", 512,
"The maximum total input sequence length after WordPiece tokenization."
"Sequences longer than this will be truncated, and sequences shorter "
"than this will be padded.")
 flags.DEFINE_integer(
 "doc_stride", 128,
"When splitting up a long document into chunks, how much stride to "
"take between chunks.")
 flags.DEFINE_float(
 "include_unknowns_for_training", 0.02,
"If positive, for converting training dataset, probability of including answers of type `UNKNOWN`.")
 flags.DEFINE_float(
 "include_unknowns", -1.0,
"If positive, probability of including answers of type `UNKNOWN`.")
 flags.DEFINE_boolean(
 "skip_nested_contexts", True,
"Completely ignore context that are not top level nodes in the page.")
 flags.DEFINE_integer("max_contexts", 48,
"Maximum number of contexts to output for an example.")
 flags.DEFINE_integer(
 "max_position", 50,
"Maximum context position for which to generate special tokens.")
 flags.DEFINE_integer(
 "max_query_length", 64,
"The maximum number of tokens for the question.Questions longer than "
"this will be truncated to this length." )
def organize_data(X):
max_non_null_terms = 0
num_rows = X.shape[0]
for i in range(0,num_rows):
temp = np.array(sorted(X[i,:],reverse=True))
X[i,:] = temp
non_null_terms = np.sum(X[i,:] > 0)
 if non_null_terms > max_non_null_terms:
max_non_null_terms = non_null_terms
X = X[:,0:max_non_null_terms]
return X<categorify>
IS_SUBMITTING = False
test_answers_df = pd.read_csv(FLAGS.sample_submission_csv)
 if IS_KAGGLE and len(test_answers_df)!= 692:
IS_SUBMITTING = True
FLAGS.do_train = False
FLAGS.do_valid = False
FLAGS.do_predict = True
def normalize_data(X):
num_rows = X.shape[0]
num_cols = X.shape[1]
X_normalized = np.zeros(( num_rows,num_cols), dtype = np.float32)
 for i in range(0,num_rows):
for j in range(0,num_cols):
if X[i,j] > 0:
X_normalized[i,j] = np.log10(X[i,j])
 return X_normalized<prepare_x_and_y>
print(len(test_answers_df))
print(IS_SUBMITTING)
 print(FLAGS.do_train)
 print(FLAGS.do_valid)
 print(FLAGS.do_predict )
train_data = pd.read_csv(".. /input/train.csv")
 train_val = train_data.values
Y = train_val[:,1].astype(np.float32)
 X = train_val[:,2:].astype(np.float32)
 test_data = pd.read_csv(".. /input/test.csv")
 test_val = test_data.values
X_test = test_val[:,1:].astype(np.float32 )<concatenate>
NB_ANSWER_TYPES = 5
X_all = np.vstack(( X, X_test))
X_all = organize_data(X_all)
 num_features = X_all.shape[1]
X_all = normalize_data(X_all)
 Y = Y.reshape(Y.shape[0],1)
 Y_normalized = normalize_data(Y )<feature_engineering>
def jsonl_iterator(jsonl_files, to_json=False):
for file_path in jsonl_files:
with open(file_path, "r", encoding="UTF-8")as fp:
for jsonl in fp:
raw_example = jsonl
if to_json:
raw_example = json.loads(jsonl)
 yield raw_example
creator = TFExampleCreator(is_training=False)
 nq_lines = jsonl_iterator([FLAGS.predict_file])
 creator.process_nq_lines(nq_lines=nq_lines, output_tfrecord=FLAGS.test_tf_record, max_examples=0, collect_nq_features=False )
num_new_features = 5
X_add = np.zeros(( X_all.shape[0],num_features + num_new_features), dtype = np.float32)
 for i in range(0,X_all.shape[0]):
X_add[i,0] = np.mean(X_all[i,X_all[i,:]!=0])
 X_add[i,1] = np.min(X_all[i,X_all[i,:]!=0])
 X_add[i,2] = np.max(X_all[i,:])
 X_add[i,3] = np.sum(X_all[i,:] > 0)
 X_add[i,4] = np.std(X_all[i,:])
 X_add[:,5:] = X_all
num_features = num_features + num_new_features<split>
def get_dataset(tf_record_file, seq_length, batch_size=1, shuffle_buffer_size=0, is_training=False):
if is_training:
features = {
"unique_ids": tf.io.FixedLenFeature([], tf.int64),
"input_ids": tf.io.FixedLenFeature([seq_length], tf.int64),
"input_mask": tf.io.FixedLenFeature([seq_length], tf.int64),
"segment_ids": tf.io.FixedLenFeature([seq_length], tf.int64),
"start_positions": tf.io.FixedLenFeature([], tf.int64),
"end_positions": tf.io.FixedLenFeature([], tf.int64),
"answer_types": tf.io.FixedLenFeature([], tf.int64)
 }
else:
features = {
"unique_ids": tf.io.FixedLenFeature([], tf.int64),
"input_ids": tf.io.FixedLenFeature([seq_length], tf.int64),
"input_mask": tf.io.FixedLenFeature([seq_length], tf.int64),
"segment_ids": tf.io.FixedLenFeature([seq_length], tf.int64),
"token_map": tf.io.FixedLenFeature([seq_length], tf.int64)
 }
def decode_record(record, features):

 example = tf.io.parse_single_example(record, features)
 for name in list(example.keys()):
t = example[name]
if t.dtype == tf.int64:
t = tf.cast(t, tf.int32)
 example[name] = t
return example
def select_data_from_record(record):
x = {
'unique_ids': record['unique_ids'],
'input_ids': record['input_ids'],
'input_mask': record['input_mask'],
'segment_ids': record['segment_ids']
}
if not is_training:
x['token_map'] = record['token_map']
if is_training:
y = {
'start_positions': record['start_positions'],
'end_positions': record['end_positions'],
'answer_types': record['answer_types']
}
return(x, y)
 return x
dataset = tf.data.TFRecordDataset(tf_record_file)
 dataset = dataset.map(lambda record: decode_record(record, features))
dataset = dataset.map(select_data_from_record)
 if shuffle_buffer_size > 0:
dataset = dataset.shuffle(shuffle_buffer_size)
 dataset = dataset.batch(batch_size)
 return dataset
X_train_final = X_add[0:X.shape[0],:]
X_test_final = X_add[X.shape[0]:,:]<choose_model_class>
valid_tf_record = FLAGS.valid_tf_record
if FLAGS.smaller_valid_dataset:
valid_tf_record = FLAGS.valid_small_tf_record
valid_tf_record_with_labels = FLAGS.valid_tf_record_with_labels
if FLAGS.smaller_valid_dataset:
valid_tf_record_with_labels = FLAGS.valid_small_tf_record_with_labels
train_dataset = get_dataset(FLAGS.train_tf_record,
seq_length=FLAGS.max_seq_length_for_training,
batch_size=2,
shuffle_buffer_size=FLAGS.shuffle_buffer_size,
is_training=True 
)
 validation_dataset = get_dataset(os.path.join(NQ_DIR, valid_tf_record),
seq_length=FLAGS.max_seq_length,
batch_size=2,
is_training=False 
)
 validation_dataset_with_labels = get_dataset(os.path.join(NQ_DIR, valid_tf_record_with_labels),
seq_length=FLAGS.max_seq_length,
batch_size=2,
is_training=True 
)
 test_dataset = get_dataset(FLAGS.test_tf_record,
seq_length=FLAGS.max_seq_length,
batch_size=2,
is_training=False 
)
 features, targets = next(iter(train_dataset))
print(features)
 print(targets)
 features = next(iter(validation_dataset))
print(features)
 features, labels = next(iter(validation_dataset_with_labels))
print(features)
 features = next(iter(test_dataset))
print(features )
for cycles in [1,5,10]:
for neurons in [10,50,100,150]:
visible = Input(shape=(num_features,))
hid1 = Dense(neurons, kernel_initializer='normal', activation = "tanh")
 hid2 = Dense(neurons, kernel_initializer='normal', activation = "relu")
 hidpar2 = Dense(neurons, kernel_initializer='normal', activation = "tanh")
 hid3 = Dense(neurons, kernel_initializer='normal', activation = "relu")
 hidpar3 = Dense(neurons, kernel_initializer='normal', activation = "tanh")
 hid1_ = Dense(neurons, kernel_initializer='normal', activation = "relu")
 hid2_ = Dense(neurons, kernel_initializer='normal', activation = "tanh")
 hidpar2_ = Dense(neurons, kernel_initializer='normal', activation = "relu")
 hid3_ = Dense(neurons, kernel_initializer='normal', activation = "tanh")
 hidpar3_ = Dense(neurons, kernel_initializer='normal', activation = "relu")
 feedback_hid = Dense(num_features, kernel_initializer='normal', activation = "relu")
 x = visible
L = []
LP = []
for i in range(0,cycles):
L.append(hid1(x))
L.append(hid2(L[0]))
L.append(hidpar2(L[1]))
L.append(hid3(L[2]))
L.append(hidpar3(L[3]))
L.append(concatenate([L[3],L[4]]))
LP.append(hid1_(x))
LP.append(hid2_(LP[0]))
LP.append(hidpar2_(LP[1]))
LP.append(hid3_(LP[2]))
LP.append(hidpar3_(LP[3]))
LP.append(concatenate([LP[3],LP[4]]))
final_merge = concatenate([L[-1],LP[-1]])
 x = feedback_hid(final_merge)
 master = Dense(neurons, kernel_initializer='normal', activation='tanh' )(x)
 output = Dense(1, kernel_initializer='normal', activation='softplus' )(master)
 model = Model(inputs=visible, outputs=output)
 sgd = SGD(lr=0.01, momentum=0.1, decay=0.0, nesterov=False)
 model.compile(loss='mean_squared_error', optimizer=sgd)
 history = model.fit(X_train_final, Y_normalized,validation_split=0.33, epochs=150, batch_size=100, verbose=0)
 print('Cycles =',cycles)
 print('Neurons =', neurons)
 print('
Loss of predictions in train dataset:')
 predictions = model.predict(X_train_final)
 predictions = 10**predictions
print(np.sqrt(1/Y.shape[0])*np.linalg.norm(( np.log(predictions+1)-np.log(Y+1)) ,2))
plt.plot(history.history['loss'][2:])
 plt.plot(history.history['val_loss'][2:])
 plt.title('model loss')
 plt.ylabel('loss')
 plt.xlabel('epoch')
 plt.legend(['train', 'test'], loc='upper left')
 plt.show()<choose_model_class>
validation_dataset = get_dataset(os.path.join(NQ_DIR, valid_tf_record),
seq_length=FLAGS.max_seq_length,
batch_size=FLAGS.predict_batch_size,
is_training=False 
)
 validation_dataset_with_labels = get_dataset(os.path.join(NQ_DIR, valid_tf_record_with_labels),
seq_length=FLAGS.max_seq_length,
batch_size=FLAGS.predict_batch_size,
is_training=True 
)
 test_dataset = get_dataset(FLAGS.test_tf_record,
seq_length=FLAGS.max_seq_length,
batch_size=FLAGS.predict_batch_size,
is_training=False
)
cycles = 5
neurons = 150
visible = Input(shape=(num_features,))
hid1 = Dense(neurons, kernel_initializer='normal', activation = "tanh")
 hid2 = Dense(neurons, kernel_initializer='normal', activation = "relu")
 hidpar2 = Dense(neurons, kernel_initializer='normal', activation = "tanh")
 hid3 = Dense(neurons, kernel_initializer='normal', activation = "relu")
 hidpar3 = Dense(neurons, kernel_initializer='normal', activation = "tanh")
 hid1_ = Dense(neurons, kernel_initializer='normal', activation = "relu")
 hid2_ = Dense(neurons, kernel_initializer='normal', activation = "tanh")
 hidpar2_ = Dense(neurons, kernel_initializer='normal', activation = "relu")
 hid3_ = Dense(neurons, kernel_initializer='normal', activation = "tanh")
 hidpar3_ = Dense(neurons, kernel_initializer='normal', activation = "relu")
 feedback_hid = Dense(num_features, kernel_initializer='normal', activation = "relu")
 mast = Dense(neurons, kernel_initializer='normal', activation='tanh')
 out = Dense(1, kernel_initializer='normal', activation='softplus')
 x = visible
L = []
LP = []
for i in range(0,cycles):
L.append(hid1(x))
L.append(hid2(L[0]))
L.append(hidpar2(L[1]))
L.append(hid3(L[2]))
L.append(hidpar3(L[3]))
L.append(concatenate([L[3],L[4]]))
LP.append(hid1_(x))
LP.append(hid2_(LP[0]))
LP.append(hidpar2_(LP[1]))
LP.append(hid3_(LP[2]))
LP.append(hidpar3_(LP[3]))
LP.append(concatenate([LP[3],LP[4]]))
final_merge = concatenate([L[-1],LP[-1]])
 x = feedback_hid(final_merge)
 master = mast(x)
 output = out(master)
 model = Model(inputs=visible, outputs=output)
 sgd = SGD(lr=0.01, momentum=0.1, decay=0.0, nesterov=False)
 model.compile(loss='mean_squared_error', optimizer=sgd )<save_to_csv>
validation_features =(tf.train.Example.FromString(r.numpy())for r in tf.data.TFRecordDataset(valid_tf_record))
test_features =(tf.train.Example.FromString(r.numpy())for r in tf.data.TFRecordDataset(FLAGS.test_tf_record))
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission["target"] = predictions
submission.to_csv('submission.csv', index=False)
 print(submission.head() )<import_modules>
class TFNQModel:
def __init__(self, config, *inputs, **kwargs):

 self.backend = None
self.seq_output_dropout = tf.keras.layers.Dropout(kwargs.get('seq_output_dropout_prob', 0.05))
self.pooled_output_dropout = tf.keras.layers.Dropout(kwargs.get('pooled_output_dropout_prob', 0.05))
self.pos_classifier = tf.keras.layers.Dense(2,
kernel_initializer=get_initializer(config.initializer_range),
name='pos_classifier')
 self.answer_type_classifier = tf.keras.layers.Dense(NB_ANSWER_TYPES,
kernel_initializer=get_initializer(config.initializer_range),
name='answer_type_classifier')
 def backend_call(self, inputs, **kwargs):

 raise NotImplementedError
def call(self, inputs, **kwargs):
sequence_output, pooled_output = self.backend_call(inputs, **kwargs)
 sequence_output = self.seq_output_dropout(sequence_output, training=kwargs.get('training', False))
pooled_output = self.pooled_output_dropout(pooled_output, training=kwargs.get('training', False))
pos_logits = self.pos_classifier(sequence_output)
 start_pos_logits = pos_logits[:, :, 0]
end_pos_logits = pos_logits[:, :, 1]
answer_type_logits = self.answer_type_classifier(pooled_output)
 outputs =(start_pos_logits, end_pos_logits, answer_type_logits)
 return outputs
class TFBertForNQ(TFNQModel, TFBertPreTrainedModel):
def __init__(self, config, *inputs, **kwargs):
TFBertPreTrainedModel.__init__(self, config, *inputs, **kwargs)
 TFNQModel.__init__(self, config)
 self.bert = TFBertMainLayer(config, name='bert')
 def backend_call(self, inputs, **kwargs):
outputs = self.bert(inputs, **kwargs)
 sequence_output, pooled_output = outputs[0], outputs[1]
return sequence_output, pooled_output
class TFDistilBertForNQ(TFNQModel, TFDistilBertPreTrainedModel):
def __init__(self, config, *inputs, **kwargs):
TFDistilBertPreTrainedModel.__init__(self, config, *inputs, **kwargs)
 TFNQModel.__init__(self, config)
 self.backend = TFDistilBertMainLayer(config, name="distilbert")
 def backend_call(self, inputs, **kwargs):
if isinstance(inputs, tuple):
inputs = inputs[:2]
else:
inputs = inputs
outputs = self.backend(inputs, **kwargs)
 sequence_output = outputs[0]
pooled_output = sequence_output[:, 0, :]
return sequence_output, pooled_output
model_mapping = {
"bert": TFBertForNQ,
"distilbert": TFDistilBertForNQ
}
def get_pretrained_model(model_name):
pretrained_path = os.path.join(FLAGS.model_dir, model_name)
 tokenizer = BertTokenizer.from_pretrained(pretrained_path)
 model_type = model_name.split("-")[0]
if model_type not in model_mapping:
raise ValueError("Model definition not found.")
 model_class = model_mapping[model_type]
model = model_class.from_pretrained(pretrained_path)
 return tokenizer, model
import numpy as np
import pandas as pd
from xgboost import XGBRegressor
from sklearn.metrics import mean_squared_error, mean_squared_log_error
from sklearn.preprocessing import StandardScaler, MinMaxScaler, MaxAbsScaler
from sklearn.model_selection import KFold<load_from_csv>
bert_tokenizer, bert_for_nq = get_pretrained_model('bert-base-uncased')
 _, distil_bert_for_nq = get_pretrained_model('distilbert-base-uncased-distilled-squad')
 input_ids = tf.constant(bert_tokenizer.encode("Hello, my dog is cute")) [None, :]
input_masks = tf.constant(0, shape=input_ids.shape)
 segment_ids = tf.constant(0, shape=input_ids.shape)
 inputs =(input_ids, input_masks, segment_ids)
 outputs = bert_for_nq(inputs)
(start_pos_logits, end_pos_logits, answer_type_logits)= outputs
print(start_pos_logits.shape)
 print(end_pos_logits.shape)
 print(answer_type_logits.shape)
 len(bert_for_nq.trainable_variables)
 outputs = distil_bert_for_nq(inputs)
(start_pos_logits, end_pos_logits, answer_type_logits)= outputs
print(start_pos_logits.shape)
 print(end_pos_logits.shape)
 print(answer_type_logits.shape)
 len(distil_bert_for_nq.trainable_variables )
%%time
data = pd.read_csv('.. /input/santander-value-prediction-challenge/train.csv')
 target = np.log1p(data['target'])
 data.drop(['ID', 'target'], axis=1, inplace=True )<load_from_csv>
class TDense(tf.keras.layers.Layer):
def __init__(self,
output_size,
kernel_initializer=None,
bias_initializer="zeros",
**kwargs):
super().__init__(**kwargs)
 self.output_size = output_size
self.kernel_initializer = kernel_initializer
self.bias_initializer = bias_initializer
def build(self,input_shape):
dtype = tf.as_dtype(self.dtype or tf.keras.backend.floatx())
 if not(dtype.is_floating or dtype.is_complex):
raise TypeError("Unable to build `TDense` layer with "
"non-floating point(and non-complex)"
"dtype %s" %(dtype,))
input_shape = tf.TensorShape(input_shape)
 if tf.compat.dimension_value(input_shape[-1])is None:
raise ValueError("The last dimension of the inputs to "
"`TDense` should be defined."
"Found `None`.")
 last_dim = tf.compat.dimension_value(input_shape[-1])
 self.input_spec = tf.keras.layers.InputSpec(min_ndim=3, axes={-1: last_dim})
 self.kernel = self.add_weight(
 "kernel",
shape=[self.output_size,last_dim],
initializer=self.kernel_initializer,
dtype=self.dtype,
trainable=True)
 self.bias = self.add_weight(
 "bias",
shape=[self.output_size],
initializer=self.bias_initializer,
dtype=self.dtype,
trainable=True)
 super(TDense, self ).build(input_shape)
 def call(self,x):
return tf.matmul(x,self.kernel,transpose_b=True)+self.bias
def mk_model(config):
seq_len = config['max_position_embeddings']
input_ids = tf.keras.Input(shape=(seq_len,),dtype=tf.int32,name='input_ids')
 input_mask = tf.keras.Input(shape=(seq_len,),dtype=tf.int32,name='input_mask')
 segment_ids = tf.keras.Input(shape=(seq_len,),dtype=tf.int32,name='segment_ids')
 BERT = modeling.BertModel(config=config,name='bert')
 pooled_output, sequence_output = BERT(input_word_ids=input_ids,
input_mask=input_mask,
input_type_ids=segment_ids)
 logits = TDense(2,name='logits' )(sequence_output)
 start_logits,end_logits = tf.split(logits,axis=-1,num_or_size_splits= 2,name='split')
 start_logits = tf.squeeze(start_logits,axis=-1,name='start_squeeze')
 end_logits = tf.squeeze(end_logits, axis=-1,name='end_squeeze')
 ans_type = TDense(5,name='ans_type' )(pooled_output)
 return tf.keras.Model([input_ for input_ in [input_ids,input_mask,segment_ids]
if input_ is not None],
[start_logits,end_logits,ans_type],
name='bert-baseline' )
%%time
leak = pd.read_csv('.. /input/breaking-lb-fresh-start-with-lag-selection/train_leak.csv')
 data['leak'] = leak['compiled_leak'].values
data['log_leak'] = np.log1p(leak['compiled_leak'].values )<train_on_grid>
bert_tokenizer, bert_nq = get_pretrained_model(FLAGS.model_name )
%%time
def rmse(y_true, y_pred):
return mean_squared_error(y_true, y_pred)**.5
reg = XGBRegressor(n_estimators=1000)
 folds = KFold(4, True, 134259)
 fold_idx = [(trn_, val_)for trn_, val_ in folds.split(data)]
scores = []
nb_values = data.nunique(dropna=False)
 nb_zeros =(data == 0 ).astype(np.uint8 ).sum(axis=0)
 features = [f for f in data.columns if f not in ['log_leak', 'leak', 'target', 'ID']]
for _f in features:
score = 0
for trn_, val_ in fold_idx:
reg.fit(
 data[['log_leak', _f]].iloc[trn_], target.iloc[trn_],
eval_set=[(data[['log_leak', _f]].iloc[val_], target.iloc[val_])],
eval_metric='rmse',
early_stopping_rounds=50,
verbose=False 
)
 score += rmse(target.iloc[val_], reg.predict(data[['log_leak', _f]].iloc[val_], ntree_limit=reg.best_ntree_limit)) / folds.n_splits
scores.append(( _f, score))<create_dataframe>
with open('.. /input/bert-joint-baseline/bert_config.json','r')as f:
config = json.load(f)
 print(json.dumps(config,indent=4))
base_bert_nq= mk_model(config )
report = pd.DataFrame(scores, columns=['feature', 'rmse'] ).set_index('feature')
 report['nb_zeros'] = nb_zeros
report['nunique'] = nb_values
report.sort_values(by='rmse', ascending=True, inplace=True )<save_to_csv>
bert_nq.load_weights('.. /input/bertjointfinetuned/bert-base-e4.h5' )
report.to_csv('feature_report.csv', index=True )<filter>
cpkt = tf.train.Checkpoint(model=base_bert_nq)
 cpkt.restore('.. /input/bert-joint-baseline/model_cpkt-1' ).assert_consumed()
good_features = report.loc[report['rmse'] <= 0.7955].index
rmses = report.loc[report['rmse'] <= 0.7955, 'rmse'].values
good_features<load_from_csv>
Span = collections.namedtuple("Span", ["start_token_idx", "end_token_idx", "score"])
 class EvalExample(object):

 def __init__(self, example_id, candidates):
self.example_id = example_id
self.candidates = candidates
self.results = {}
self.features = {}
class ScoreSummary(object):
def __init__(self):
self.predicted_label = None
self.short_span_score = None
self.cls_token_score = None
self.answer_type_logits = None
self.start_prob = None
self.end_prob = None
self.answer_type_prob_dist = None
def read_candidates_from_one_split(input_path):

 candidates_dict = {}
if input_path.endswith(".gz"):
with gzip.GzipFile(fileobj=tf.io.gfile.GFile(input_path, "rb")) as input_file:
print("Reading examples from: {}".format(input_path))
for index, line in enumerate(input_file):
e = json.loads(line)
 candidates_dict[e["example_id"]] = e["long_answer_candidates"]
else:
with tf.io.gfile.GFile(input_path, "r")as input_file:
print("Reading examples from: {}".format(input_path))
for index, line in enumerate(input_file):
e = json.loads(line)
 candidates_dict[e["example_id"]] = e["long_answer_candidates"]
return candidates_dict
def read_candidates(input_pattern):

 input_paths = tf.io.gfile.glob(input_pattern)
 final_dict = {}
for input_path in input_paths:
final_dict.update(read_candidates_from_one_split(input_path))
return final_dict
def get_best_indexes(logits, n_best_size, token_map=None):
index_and_score = sorted(enumerate(logits[1:], 1), key=lambda x: x[1], reverse=True)
 best_indexes = []
for i in range(len(index_and_score)) :
idx = index_and_score[i][0]
if token_map is not None and token_map[idx] == -1:
continue
best_indexes.append(idx)
 if len(best_indexes)>= n_best_size:
break
return best_indexes
test = pd.read_csv('.. /input/santander-value-prediction-challenge/test.csv' )<load_from_csv>
def remove_duplicates(span):
start_end = []
for s in span:
cont = 0
if not start_end:
start_end.append(Span(s[0], s[1], s[2]))
cont += 1
else:
for i in range(len(start_end)) :
if start_end[i][0] == s[0] and start_end[i][1] == s[1]:
cont += 1
if cont == 0:
start_end.append(Span(s[0], s[1], s[2]))
return start_end
def get_short_long_span(predictions, example):
sorted_predictions = sorted(predictions, key=lambda x:(x[0], x[2], x[3]), reverse=True)
 short_span = []
long_span = []
for prediction in sorted_predictions:
score, summary, start_span, end_span = prediction
if score > 0:
short_span.append(Span(int(start_span), int(end_span), float(score)))
 short_span = remove_duplicates(short_span)
 for s in range(len(short_span)) :
for c in example.candidates:
start = short_span[s].start_token_idx
end = short_span[s].end_token_idx
if c["top_level"] and c["start_token"] <= start and c["end_token"] >= end:
long_span.append(Span(int(c["start_token"]), int(c["end_token"]), float(short_span[s].score)))
 break
long_span = remove_duplicates(long_span)
 if not long_span:
long_span = [Span(-1, -1, -10000.0)]
if not short_span:
short_span = [Span(-1, -1, -10000.0)]
return short_span, long_span
%%time
tst_leak = pd.read_csv('.. /input/breaking-lb-fresh-start-with-lag-selection/test_leak.csv')
 test['leak'] = tst_leak['compiled_leak']
test['log_leak'] = np.log1p(tst_leak['compiled_leak'] )<feature_engineering>
def compute_predictions(example):

 predictions = []
max_answer_length = FLAGS.max_answer_length
null_entry_threshold = -0.3
score_null = 1000000
for unique_id, result in example.results.items() :
if unique_id not in example.features:
raise ValueError("No feature found with unique_id:", unique_id)
 token_map = example.features[unique_id]["token_map"].int64_list.value
feature_null_score = result["start_logits"][0] + result["end_logits"][0]
score_null = min(score_null, feature_null_score)
 for start_index, start_logit, start_prob in zip(result["start_indexes"], result["start_logits"], result["start_pos_prob_dist"]):
if token_map[start_index] == -1:
continue
for end_index, end_logit, end_prob in zip(result["end_indexes"], result["end_logits"], result["end_pos_prob_dist"]):
if token_map[end_index] == -1:
continue
if end_index < start_index:
continue
length = end_index - start_index + 1
if length > max_answer_length:
continue
summary = ScoreSummary()
summary.instance_id = unique_id
summary.short_span_score = start_logit + end_logit
summary.cls_token_score = result["cls_start_logit"] + result["cls_end_logit"]
summary.answer_type_logits = result["answer_type_logits"]
summary.start_indexes = result["start_indexes"]
summary.end_indexes = result["end_indexes"]
summary.start_logits = result["start_logits"]
summary.end_logits = result["end_logits"]
summary.start_pos_prob_dist = result["start_pos_prob_dist"]
summary.end_pos_prob_dist = result["end_pos_prob_dist"]
summary.start_index = start_index
summary.end_index = end_index
summary.start_logit = start_logit
summary.end_logit = end_logit
answer_type_prob_dist = result["answer_type_prob_dist"]
summary.start_prob = start_prob
summary.end_prob = end_prob
summary.answer_type_prob_dist = {
"unknown": answer_type_prob_dist[0],
"yes": answer_type_prob_dist[1],
"no": answer_type_prob_dist[2],
"short": answer_type_prob_dist[3],
"long": answer_type_prob_dist[4]
}
start_span = token_map[start_index]
end_span = token_map[end_index] + 1
score = summary.short_span_score - summary.cls_token_score
predictions.append(( score, summary, start_span, end_span))
all_summaries = []
short_span = [Span(-1, -1, 0)]
long_span = [Span(-1, -1, 0)]
score = 0
summary = ScoreSummary()
if predictions:
short_span, long_span = get_short_long_span(predictions, example)
 if len(all_summaries)== 0:
summary.predicted_label = {
"example_id": example.example_id,
"instance_id": None,
"long_answer": {
"tokens_and_score": long_span,
"start_byte": -1,
"end_byte": -1
},
"long_answer_score": score,
"short_answers": [{
"tokens_and_score": short_span,
"start_byte": -1,
"end_byte": -1
}],
"short_answers_score": score,
"yes_no_answer": "NONE",
"no_answer": True
}
all_summaries.append(summary)
 all_summaries = all_summaries[:min(FLAGS.n_best_size, len(all_summaries)) ]
return all_summaries
def compute_pred_dict(candidates_dict, dev_features, raw_results):

 raw_results_by_id = [(int(res["unique_id"]), 1, res, None)for res in raw_results]
examples_by_id = [(int(tf.cast(int(k), dtype=tf.int32)) , 0, v, k)for k, v in candidates_dict.items() ]
features_by_id = [(int(tf.cast(f.features.feature["unique_ids"].int64_list.value[0], dtype=tf.int32)) , 2, f.features.feature, None)for f in dev_features]
print('merging examples...')
 merged = sorted(examples_by_id + raw_results_by_id + features_by_id)
 print('done.')
 examples = []
for idx, type_, datum, orig_example_id in merged:
if type_ == 0:
examples.append(EvalExample(orig_example_id, datum))
elif type_ == 2:
examples[-1].features[idx] = datum
else:
examples[-1].results[idx] = datum
summary_dict = {}
nq_pred_dict = {}
for e in examples:
all_summaries = compute_predictions(e)
 summary_dict[e.example_id] = all_summaries
nq_pred_dict[e.example_id] = [summary.predicted_label for summary in all_summaries]
if len(nq_pred_dict)% 100 == 0:
print("Examples processed: %d" % len(nq_pred_dict))
return nq_pred_dict
folds = KFold(n_splits=5, shuffle=True, random_state=1)
 features = [f for f in data if f not in ['ID', 'leak', 'log_leak', 'target']]
data.replace(0, np.nan, inplace=True)
 data['log_of_mean'] = np.log1p(data[features].replace(0, np.nan ).mean(axis=1))
data['mean_of_log'] = np.log1p(data[features] ).replace(0, np.nan ).mean(axis=1)
 data['log_of_median'] = np.log1p(data[features].replace(0, np.nan ).median(axis=1))
data['nb_nans'] = data[features].isnull().sum(axis=1)
 data['the_sum'] = np.log1p(data[features].sum(axis=1))
data['the_std'] = data[features].std(axis=1)
 data['the_kur'] = data[features].kurtosis(axis=1)
 test.replace(0, np.nan, inplace=True)
 test['log_of_mean'] = np.log1p(test[features].replace(0, np.nan ).mean(axis=1))
test['mean_of_log'] = np.log1p(test[features] ).replace(0, np.nan ).mean(axis=1)
 test['log_of_median'] = np.log1p(test[features].replace(0, np.nan ).median(axis=1))
test['nb_nans'] = test[features].isnull().sum(axis=1)
 test['the_sum'] = np.log1p(test[features].sum(axis=1))
test['the_std'] = test[features].std(axis=1)
 test['the_kur'] = test[features].kurtosis(axis=1)
 features = good_features.tolist()
features = features + ['log_leak', 'log_of_mean', 'mean_of_log', 'log_of_median', 'nb_nans', 'the_sum', 'the_std', 'the_kur']
dtrain = lgb.Dataset(data=data[features],
label=target, free_raw_data=False)
 test['target'] = 0
dtrain.construct()
oof_preds = np.zeros(data.shape[0])
 for trn_idx, val_idx in folds.split(data):
lgb_params = {
'objective': 'regression',
'num_leaves': 58,
'subsample': 0.6143,
'colsample_bytree': 0.6453,
'min_split_gain': np.power(10, -2.5988),
'reg_alpha': np.power(10, -2.2887),
'reg_lambda': np.power(10, 1.7570),
'min_child_weight': np.power(10, -0.1477),
'verbose': -1,
'seed': 3,
'boosting_type': 'gbdt',
'max_depth': -1,
'learning_rate': 0.05,
'metric': 'l2',
}
clf = lgb.train(
 params=lgb_params,
train_set=dtrain.subset(trn_idx),
valid_sets=dtrain.subset(val_idx),
num_boost_round=10000,
early_stopping_rounds=100,
verbose_eval=0 
)
 oof_preds[val_idx] = clf.predict(dtrain.data.iloc[val_idx])
 test['target'] += clf.predict(test[features])/ folds.n_splits
print(mean_squared_error(target.iloc[val_idx],
oof_preds[val_idx])**.5)
 data['predictions'] = oof_preds
data.loc[data['leak'].notnull() , 'predictions'] = np.log1p(data.loc[data['leak'].notnull() , 'leak'])
 print('OOF SCORE : %9.6f'
%(mean_squared_error(target, oof_preds)**.5))
print('OOF SCORE with LEAK : %9.6f'
%(mean_squared_error(target, data['predictions'])**.5)) 
<save_to_csv>
def get_prediction_json(mode, max_nb_pos_logits=-1):
if mode == 'valid':
dataset = validation_dataset
if FLAGS.smaller_valid_dataset:
predict_file = FLAGS.validation_predict_file_small
prediction_output_file = FLAGS.validation_small_prediction_output_file
else:
predict_file = FLAGS.validation_predict_file
prediction_output_file = FLAGS.validation_prediction_output_file
eval_features = validation_features
else:
dataset = test_dataset
predict_file = FLAGS.predict_file
eval_features = test_features
prediction_output_file = FLAGS.prediction_output_file
print(predict_file)
 print(prediction_output_file)
 all_results = []
prediction_start_time = datetime.datetime.now()
for(batch_idx, features)in enumerate(dataset):
batch_start_time = datetime.datetime.now()
unique_ids = features['unique_ids']
token_maps = features['token_map'] 
(input_ids, input_masks, segment_ids)=(features['input_ids'], features['input_mask'], features['segment_ids'])
 nq_inputs =(input_ids, input_masks, segment_ids)
 nq_logits = bert_nq(nq_inputs, training=False)
 base_nq_logits = base_bert_nq(nq_inputs, training=False)
(start_pos_logits, end_pos_logits, answer_type_logits)= nq_logits 
(base_start_pos_logits, base_end_pos_logits, baseanswer_type_logits)= base_nq_logits
start_pos_logits =(0.2 * start_pos_logits + 0.8 * base_start_pos_logits)
 end_pos_logits =(0.2 * end_pos_logits + 0.8 * base_end_pos_logits)
 answer_type_logits =(0.2 * answer_type_logits + 0.8 * baseanswer_type_logits)
 unique_ids = unique_ids.numpy().tolist()
token_maps = token_maps.numpy().tolist()
start_pos_prob_dist = tf.nn.softmax(start_pos_logits, axis=-1 ).numpy().tolist()
end_pos_prob_dist = tf.nn.softmax(end_pos_logits, axis=-1 ).numpy().tolist()
answer_type_prob_dist = tf.nn.softmax(answer_type_logits, axis=-1 ).numpy().tolist()
start_pos_logits = start_pos_logits.numpy().tolist()
end_pos_logits = end_pos_logits.numpy().tolist()
answer_type_logits = answer_type_logits.numpy().tolist()
for uid, token_map, s, e, a, sp, ep, ap in zip(unique_ids, token_maps, start_pos_logits, end_pos_logits, answer_type_logits, start_pos_prob_dist, end_pos_prob_dist, answer_type_prob_dist):
if max_nb_pos_logits < 0:
max_nb_pos_logits = len(start_pos_logits)
 cls_start_logit = s[0]
cls_end_logit = e[0]
start_indexes = get_best_indexes(s, max_nb_pos_logits, token_map)
 end_indexes = get_best_indexes(e, max_nb_pos_logits, token_map)
 s = [s[idx] for idx in start_indexes]
e = [e[idx] for idx in end_indexes]
sp = [sp[idx] for idx in start_indexes]
ep = [ep[idx] for idx in end_indexes]
raw_result = {
"unique_id": uid,
"start_indexes": start_indexes,
"end_indexes": end_indexes,
"start_logits": s,
"end_logits": e,
"answer_type_logits": a,
"start_pos_prob_dist": sp,
"end_pos_prob_dist": ep,
"answer_type_prob_dist": ap,
"cls_start_logit": cls_start_logit,
"cls_end_logit": cls_end_logit
}
all_results.append(raw_result)
 batch_end_time = datetime.datetime.now()
batch_elapsed_time =(batch_end_time - batch_start_time ).total_seconds()
if(batch_idx + 1)% 100 == 0:
print('Batch {} | Elapsed Time {}'.format(
 batch_idx + 1,
batch_elapsed_time 
))
prediction_end_time = datetime.datetime.now()
prediction_elapsed_time =(prediction_end_time - prediction_start_time ).total_seconds()
print('
Time taken for prediction: {} secs
'.format(prediction_elapsed_time))
print("-" * 80 + "
")
 print("Going to candidates file")
 candidates_dict = read_candidates(predict_file)
 print("setting up eval features")
 print("compute_pred_dict")
 nq_pred_dict = compute_pred_dict(candidates_dict, eval_features, all_results)
 predictions_json = {"predictions": list(nq_pred_dict.values())}
print("writing json")
 with tf.io.gfile.GFile(prediction_output_file, "w")as f:
json.dump(predictions_json, f, indent=4)
 return predictions_json
test['target'] = np.expm1(test['target'])
 test.loc[test['leak'].notnull() , 'target'] = test.loc[test['leak'].notnull() , 'leak']
test[['ID', 'target']].to_csv('leaky_submission.csv', index=False, float_format='%.2f' )<load_from_csv>
def create_answer_from_token_indices(answer):
if answer["start_token"] == -1 or answer["end_token"] == -1:
return ""
else:
return str(answer["start_token"])+ ":" + str(answer["end_token"])
 def create_long_answer_from_1_pred(pred):

 if pred["long_answer_score"] < 1.5 or pred['answer_type'] == 0:
return ""
long_answer = create_answer_from_token_indices(pred["long_answer"])
 return long_answer
def create_short_answers_from_1_pred(pred):

 short_answers = []
for predicted_short_answer in pred["short_answers"]:
short_answer = create_answer_from_token_indices(predicted_short_answer)
 if "answer_type_prob_dist" in pred:
if pred["short_answers_score"] < 1.5 or pred['answer_type'] == 0:
if short_answer not in ["YES", "NO"]:
short_answer = ""
short_answers.append(short_answer)
 return short_answers
def is_pred_ok(pred, annotations):

 long_labels = []
for annotation in annotations:
long_label = create_answer_from_token_indices(annotation["long_answer"])
 long_labels.append(long_label)
 non_null_long_labels = [x for x in long_labels if x != ""]
has_long_label = len(non_null_long_labels)> 1
long_pred = create_long_answer_from_1_pred(pred)
 has_long_pred =(long_pred != "")
 short_label_lists = []
for annotation in annotations:
if len(annotation["short_answers"])== 0:
if annotation["yes_no_answer"] == "YES":
short_label_lists.append(["YES"])
 elif annotation["yes_no_answer"] == "NO":
short_label_lists.append(["NO"])
 else:
short_label_lists.append([""])
 else:
short_labels = []
for anno_short_answer in annotation["short_answers"]:
short_label = create_answer_from_token_indices(anno_short_answer)
 short_labels.append(short_label)
 short_label_lists.append(short_labels)
 non_null_short_label_lists = [x for x in short_label_lists if x != [""]]
has_short_label = len(non_null_short_label_lists)> 1
short_preds = create_short_answers_from_1_pred(pred)
 has_short_pred =(short_preds != [""])
 is_long_pred_correct = False
is_short_pred_correct = False
for long_label in long_labels:
if has_long_label and long_label == "":
continue
if not has_long_label and long_label != "":
continue
if long_pred == long_label:
is_long_pred_correct = True
break
for short_labels in short_label_lists:
if has_short_label and short_labels == [""]:
continue
if not has_short_label and short_labels != [""]:
continue
if has_short_label:
if short_labels == ["YES"] or short_labels == ["NO"]:
if short_preds == short_labels:
is_short_pred_correct = True
break
else:
if short_preds[0] in short_labels:
is_short_pred_correct = True
break
else:
if short_preds == short_labels:
is_short_pred_correct = True
break
return has_long_label, has_short_label, has_long_pred, has_short_pred, is_long_pred_correct, is_short_pred_correct
def compute_f1_scores(predictions_json, gold_jsonl_file):
predictions = predictions_json["predictions"]
golden_nq_lines = jsonl_iterator([gold_jsonl_file])
 golden_dict = dict()
for nq_line in golden_nq_lines:
nq_data = json.loads(nq_line)
 golden = dict()
golden["example_id"] = nq_data["example_id"]
golden["annotations"] = nq_data["annotations"]
golden_dict[golden["example_id"]] = golden
long_labels = []
long_preds = []
short_labels = []
short_preds = []
for preds in predictions:
pred = preds[0]
example_id = pred["example_id"]
assert example_id in golden_dict
golden = golden_dict[example_id]
assert example_id == golden["example_id"]
has_long_label, has_short_label, has_long_pred, has_short_pred, is_long_correct, is_short_correct = is_pred_ok(pred, golden["annotations"])
 if has_long_label or has_long_pred:
if is_long_correct:
long_labels.append(1)
 long_preds.append(1)
 else:
long_labels.append(1)
 long_preds.append(0)
 if has_short_label or has_short_pred:
if is_short_correct:
short_labels.append(1)
 short_preds.append(1)
 else:
short_labels.append(1)
 short_preds.append(0)
 f1 = f1_score(long_labels + short_labels, long_preds + short_preds)
 long_f1 = f1_score(long_labels, long_preds)
 short_f1 = f1_score(short_labels, short_preds)
 return f1, long_f1, short_f1
train_csv=pd.read_csv(".. /input/train.csv")
 test_csv=pd.read_csv(".. /input/test.csv" )<drop_column>
def df_long_index_score(df):
answers = []
cont = 0
for e in df[0]['long_answer']['tokens_and_score']:
if e[2] >= 1.5:
index = {}
index['start'] = e[0]
index['end'] = e[1]
index['score'] = e[2]
answers.append(str(index['start'])+ ":" + str(index['end']))
cont += 1
if cont == 1:
break
if len(answers)== 0:
return ""
return answers[0]
def df_short_index_score(df):
answers = []
cont = 0
for e in df[0]['short_answers'][0]['tokens_and_score']:
if e[2] >= 3.0:
index = {}
index['start'] = e[0]
index['end'] = e[1]
index['score'] = e[2]
answers.append(str(index['start'])+ ":" + str(index['end']))
cont += 1
if cont == 1:
break
if len(answers)== 0:
return ""
return answers[0]
train_csv=train_csv.drop(['ID'],axis=1)
 test_id=test_csv['ID']
test_csv.drop(['ID'],axis=1,inplace=True)
 target=train_csv['target']
train_csv.drop(['target'],axis=1,inplace=True)
 target_log=np.log10(target)
 target_vals=np.log1p(target_log)
 print(train_csv.shape,test_csv.shape )<train_model>
if FLAGS.do_predict:
test_features =(tf.train.Example.FromString(r.numpy())for r in tf.data.TFRecordDataset(FLAGS.test_tf_record))
predictions_json = get_prediction_json(mode='test', max_nb_pos_logits=FLAGS.n_best_size )
forest = RandomForestRegressor(n_estimators = 1000, random_state = 42)
 forest.fit(train_csv,target_vals)
<features_selection>
def create_long_answer(preds):

 return create_long_answer_from_1_pred(preds[0])
 def create_short_answer(preds):

 return create_short_answers_from_1_pred(preds[0])[0]
if FLAGS.do_predict:
test_answers_df = pd.read_json(FLAGS.prediction_output_file)
 test_answers_df["long_answer_score"] = test_answers_df["predictions"].apply(lambda q: q[0]["long_answer_score"])
 test_answers_df["short_answer_score"] = test_answers_df["predictions"].apply(lambda q: q[0]["short_answers_score"])
 test_answers_df["long_answer_score"].describe()
sample_submission = pd.read_csv(FLAGS.sample_submission_csv)
 sample_submission.to_csv("submission.csv", index=False)
 test_answers_df["long_answer"] = test_answers_df["predictions"].apply(df_long_index_score)
 test_answers_df["short_answer"] = test_answers_df["predictions"].apply(df_short_index_score)
 test_answers_df["example_id"] = test_answers_df["predictions"].apply(lambda q: str(q[0]["example_id"]))
long_answers = dict(zip(test_answers_df["example_id"], test_answers_df["long_answer"]))
short_answers = dict(zip(test_answers_df["example_id"], test_answers_df["short_answer"]))
long_prediction_strings = sample_submission[sample_submission["example_id"].str.contains("_long")].apply(lambda q: long_answers[q["example_id"].replace("_long", "")], axis=1)
 short_prediction_strings = sample_submission[sample_submission["example_id"].str.contains("_short")].apply(lambda q: short_answers[q["example_id"].replace("_short", "")], axis=1)
 sample_submission.loc[sample_submission["example_id"].str.contains("_long"), "PredictionString"] = long_prediction_strings
sample_submission.loc[sample_submission["example_id"].str.contains("_short"), "PredictionString"] = short_prediction_strings
sample_submission.to_csv("submission.csv", index=False )
importances = forest.feature_importances_
feature_importance = 100.0 *(importances / importances.max())
 sorted_idx = np.argsort(feature_importance)
 feature_names = list(train_csv.columns.values)
 feature_names_sort = [feature_names[indice] for indice in sorted_idx]
pos = np.arange(sorted_idx.shape[0])+.5
print('Top 40 features are: ')
 cols=[]
for feature in feature_names_sort[::-1][:40]:
cols.append(feature)
 print(sorted(cols))<load_from_csv>
df = pd.read_csv('submission.csv')
 df.head()