Kaggle competition: House Prices - Advanced Regression Techniques
def jsonl_iterator(jsonl_files, to_json=False):
for file_path in jsonl_files:
with open(file_path, "r", encoding="UTF-8")as fp:
for jsonl in fp:
raw_example = jsonl
if to_json:
raw_example = json.loads(jsonl)
 yield raw_example
creator = TFExampleCreator(is_training=False)
 nq_lines = jsonl_iterator([FLAGS.predict_file])
 creator.process_nq_lines(nq_lines=nq_lines, output_tfrecord=FLAGS.test_tf_record, max_examples=0, collect_nq_features=False )<define_variables>
train=pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv',index_col=0)
 test=pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv',index_col=0)
 submission=pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/sample_submission.csv')
 Id_test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv' ).values[:,0]
data=train
print(train.shape, test.shape, submission.shape, Id_test.shape )
def get_dataset(tf_record_file, seq_length, batch_size=1, shuffle_buffer_size=0, is_training=False):
if is_training:
features = {
"unique_ids": tf.io.FixedLenFeature([], tf.int64),
"input_ids": tf.io.FixedLenFeature([seq_length], tf.int64),
"input_mask": tf.io.FixedLenFeature([seq_length], tf.int64),
"segment_ids": tf.io.FixedLenFeature([seq_length], tf.int64),
"start_positions": tf.io.FixedLenFeature([], tf.int64),
"end_positions": tf.io.FixedLenFeature([], tf.int64),
"answer_types": tf.io.FixedLenFeature([], tf.int64)
 }
else:
features = {
"unique_ids": tf.io.FixedLenFeature([], tf.int64),
"input_ids": tf.io.FixedLenFeature([seq_length], tf.int64),
"input_mask": tf.io.FixedLenFeature([seq_length], tf.int64),
"segment_ids": tf.io.FixedLenFeature([seq_length], tf.int64),
"token_map": tf.io.FixedLenFeature([seq_length], tf.int64)
 }
def decode_record(record, features):

 example = tf.io.parse_single_example(record, features)
 for name in list(example.keys()):
t = example[name]
if t.dtype == tf.int64:
t = tf.cast(t, tf.int32)
 example[name] = t
return example
def select_data_from_record(record):
x = {
'unique_ids': record['unique_ids'],
'input_ids': record['input_ids'],
'input_mask': record['input_mask'],
'segment_ids': record['segment_ids']
}
if not is_training:
x['token_map'] = record['token_map']
if is_training:
y = {
'start_positions': record['start_positions'],
'end_positions': record['end_positions'],
'answer_types': record['answer_types']
}
return(x, y)
 return x
dataset = tf.data.TFRecordDataset(tf_record_file)
 dataset = dataset.map(lambda record: decode_record(record, features))
dataset = dataset.map(select_data_from_record)
 if shuffle_buffer_size > 0:
dataset = dataset.shuffle(shuffle_buffer_size)
 dataset = dataset.batch(batch_size)
 return dataset<create_dataframe>
%matplotlib inline
valid_tf_record = FLAGS.valid_tf_record
if FLAGS.smaller_valid_dataset:
valid_tf_record = FLAGS.valid_small_tf_record
valid_tf_record_with_labels = FLAGS.valid_tf_record_with_labels
if FLAGS.smaller_valid_dataset:
valid_tf_record_with_labels = FLAGS.valid_small_tf_record_with_labels
train_dataset = get_dataset(FLAGS.train_tf_record,
seq_length=FLAGS.max_seq_length_for_training,
batch_size=2,
shuffle_buffer_size=FLAGS.shuffle_buffer_size,
is_training=True 
)
 validation_dataset = get_dataset(os.path.join(NQ_DIR, valid_tf_record),
seq_length=FLAGS.max_seq_length,
batch_size=2,
is_training=False 
)
 validation_dataset_with_labels = get_dataset(os.path.join(NQ_DIR, valid_tf_record_with_labels),
seq_length=FLAGS.max_seq_length,
batch_size=2,
is_training=True 
)
 test_dataset = get_dataset(FLAGS.test_tf_record,
seq_length=FLAGS.max_seq_length,
batch_size=2,
is_training=False 
)
 features, targets = next(iter(train_dataset))
print(features)
 print(targets)
 features = next(iter(validation_dataset))
print(features)
 features, labels = next(iter(validation_dataset_with_labels))
print(features)
 features = next(iter(test_dataset))
print(features )<create_dataframe>
train=train.drop(train[(train['GrLivArea']>4000)&(train['SalePrice']<300000)].index )
validation_dataset = get_dataset(os.path.join(NQ_DIR, valid_tf_record),
seq_length=FLAGS.max_seq_length,
batch_size=FLAGS.predict_batch_size,
is_training=False 
)
 validation_dataset_with_labels = get_dataset(os.path.join(NQ_DIR, valid_tf_record_with_labels),
seq_length=FLAGS.max_seq_length,
batch_size=FLAGS.predict_batch_size,
is_training=True 
)
 test_dataset = get_dataset(FLAGS.test_tf_record,
seq_length=FLAGS.max_seq_length,
batch_size=FLAGS.predict_batch_size,
is_training=False
)<data_type_conversions>
Ytrain=train['SalePrice']
train=train[list(test)]
all_data=pd.concat(( train, test), axis=0)
 print(all_data.shape)
 Ytrain=np.log(Ytrain+1 )
validation_features =(tf.train.Example.FromString(r.numpy())for r in tf.data.TFRecordDataset(valid_tf_record))
test_features =(tf.train.Example.FromString(r.numpy())for r in tf.data.TFRecordDataset(FLAGS.test_tf_record))<import_modules>
cols=list(all_data)
 for col in list(all_data):
if(all_data[col].isnull().sum())==0:
cols.remove(col)
 else:
pass
print(len(cols))
class TFNQModel:
def __init__(self, config, *inputs, **kwargs):

 self.backend = None
self.seq_output_dropout = tf.keras.layers.Dropout(kwargs.get('seq_output_dropout_prob', 0.05))
self.pooled_output_dropout = tf.keras.layers.Dropout(kwargs.get('pooled_output_dropout_prob', 0.05))
self.pos_classifier = tf.keras.layers.Dense(2,
kernel_initializer=get_initializer(config.initializer_range),
name='pos_classifier')
 self.answer_type_classifier = tf.keras.layers.Dense(NB_ANSWER_TYPES,
kernel_initializer=get_initializer(config.initializer_range),
name='answer_type_classifier')
 def backend_call(self, inputs, **kwargs):

 raise NotImplementedError
def call(self, inputs, **kwargs):
sequence_output, pooled_output = self.backend_call(inputs, **kwargs)
 sequence_output = self.seq_output_dropout(sequence_output, training=kwargs.get('training', False))
pooled_output = self.pooled_output_dropout(pooled_output, training=kwargs.get('training', False))
pos_logits = self.pos_classifier(sequence_output)
 start_pos_logits = pos_logits[:, :, 0]
end_pos_logits = pos_logits[:, :, 1]
answer_type_logits = self.answer_type_classifier(pooled_output)
 outputs =(start_pos_logits, end_pos_logits, answer_type_logits)
 return outputs
class TFBertForNQ(TFNQModel, TFBertPreTrainedModel):
def __init__(self, config, *inputs, **kwargs):
TFBertPreTrainedModel.__init__(self, config, *inputs, **kwargs)
 TFNQModel.__init__(self, config)
 self.bert = TFBertMainLayer(config, name='bert')
 def backend_call(self, inputs, **kwargs):
outputs = self.bert(inputs, **kwargs)
 sequence_output, pooled_output = outputs[0], outputs[1]
return sequence_output, pooled_output
class TFDistilBertForNQ(TFNQModel, TFDistilBertPreTrainedModel):
def __init__(self, config, *inputs, **kwargs):
TFDistilBertPreTrainedModel.__init__(self, config, *inputs, **kwargs)
 TFNQModel.__init__(self, config)
 self.backend = TFDistilBertMainLayer(config, name="distilbert")
 def backend_call(self, inputs, **kwargs):
if isinstance(inputs, tuple):
inputs = inputs[:2]
else:
inputs = inputs
outputs = self.backend(inputs, **kwargs)
 sequence_output = outputs[0]
pooled_output = sequence_output[:, 0, :]
return sequence_output, pooled_output
model_mapping = {
"bert": TFBertForNQ,
"distilbert": TFDistilBertForNQ
}
def get_pretrained_model(model_name):
pretrained_path = os.path.join(FLAGS.model_dir, model_name)
 tokenizer = BertTokenizer.from_pretrained(pretrained_path)
 model_type = model_name.split("-")[0]
if model_type not in model_mapping:
raise ValueError("Model definition not found.")
 model_class = model_mapping[model_type]
model = model_class.from_pretrained(pretrained_path)
 return tokenizer, model<categorify>
for col in('PoolQC', 'MiscFeature', 'Alley', 'Fence', 'FireplaceQu', 'GarageType', 'GarageFinish', 'GarageQual', 'GarageCond', 'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2', 'MasVnrType', 'MSSubClass'):
all_data[col] = all_data[col].fillna('None')
 for col in('GarageYrBlt', 'GarageArea', 'GarageCars', 'BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath', 'MasVnrArea','LotFrontage'):
all_data[col] = all_data[col].fillna(0)
 for col in('MSZoning', 'Electrical', 'KitchenQual', 'Exterior1st', 'Exterior2nd', 'SaleType', 'Functional', 'Utilities'):
all_data[col] = all_data[col].fillna(all_data[col].mode() [0])
 print(f"Total count of missing values in all_data : {all_data.isnull().sum().sum() }" )
bert_tokenizer, bert_for_nq = get_pretrained_model('bert-base-uncased')
 _, distil_bert_for_nq = get_pretrained_model('distilbert-base-uncased-distilled-squad')
 input_ids = tf.constant(bert_tokenizer.encode("Hello, my dog is cute")) [None, :]
input_masks = tf.constant(0, shape=input_ids.shape)
 segment_ids = tf.constant(0, shape=input_ids.shape)
 inputs =(input_ids, input_masks, segment_ids)
 outputs = bert_for_nq(inputs)
(start_pos_logits, end_pos_logits, answer_type_logits)= outputs
print(start_pos_logits.shape)
 print(end_pos_logits.shape)
 print(answer_type_logits.shape)
 len(bert_for_nq.trainable_variables)
 outputs = distil_bert_for_nq(inputs)
(start_pos_logits, end_pos_logits, answer_type_logits)= outputs
print(start_pos_logits.shape)
 print(end_pos_logits.shape)
 print(answer_type_logits.shape)
 len(distil_bert_for_nq.trainable_variables )<choose_model_class>
all_data['TotalSF']=all_data['TotalBsmtSF'] + all_data['1stFlrSF'] + all_data['2ndFlrSF']
all_data['No2ndFlr']=(all_data['2ndFlrSF']==0 )
class TDense(tf.keras.layers.Layer):
def __init__(self,
output_size,
kernel_initializer=None,
bias_initializer="zeros",
**kwargs):
super().__init__(**kwargs)
 self.output_size = output_size
self.kernel_initializer = kernel_initializer
self.bias_initializer = bias_initializer
def build(self,input_shape):
dtype = tf.as_dtype(self.dtype or tf.keras.backend.floatx())
 if not(dtype.is_floating or dtype.is_complex):
raise TypeError("Unable to build `TDense` layer with "
"non-floating point(and non-complex)"
"dtype %s" %(dtype,))
input_shape = tf.TensorShape(input_shape)
 if tf.compat.dimension_value(input_shape[-1])is None:
raise ValueError("The last dimension of the inputs to "
"`TDense` should be defined."
"Found `None`.")
 last_dim = tf.compat.dimension_value(input_shape[-1])
 self.input_spec = tf.keras.layers.InputSpec(min_ndim=3, axes={-1: last_dim})
 self.kernel = self.add_weight(
 "kernel",
shape=[self.output_size,last_dim],
initializer=self.kernel_initializer,
dtype=self.dtype,
trainable=True)
 self.bias = self.add_weight(
 "bias",
shape=[self.output_size],
initializer=self.bias_initializer,
dtype=self.dtype,
trainable=True)
 super(TDense, self ).build(input_shape)
 def call(self,x):
return tf.matmul(x,self.kernel,transpose_b=True)+self.bias
def mk_model(config):
seq_len = config['max_position_embeddings']
input_ids = tf.keras.Input(shape=(seq_len,),dtype=tf.int32,name='input_ids')
 input_mask = tf.keras.Input(shape=(seq_len,),dtype=tf.int32,name='input_mask')
 segment_ids = tf.keras.Input(shape=(seq_len,),dtype=tf.int32,name='segment_ids')
 BERT = modeling.BertModel(config=config,name='bert')
 pooled_output, sequence_output = BERT(input_word_ids=input_ids,
input_mask=input_mask,
input_type_ids=segment_ids)
 logits = TDense(2,name='logits' )(sequence_output)
 start_logits,end_logits = tf.split(logits,axis=-1,num_or_size_splits= 2,name='split')
 start_logits = tf.squeeze(start_logits,axis=-1,name='start_squeeze')
 end_logits = tf.squeeze(end_logits, axis=-1,name='end_squeeze')
 ans_type = TDense(5,name='ans_type' )(pooled_output)
 return tf.keras.Model([input_ for input_ in [input_ids,input_mask,segment_ids]
if input_ is not None],
[start_logits,end_logits,ans_type],
name='bert-baseline' )<load_pretrained>
all_data['TotalBath']=all_data['BsmtFullBath'] + all_data['FullBath'] +(all_data['BsmtHalfBath']/2)+(all_data['HalfBath']/2 )
bert_tokenizer, bert_nq = get_pretrained_model(FLAGS.model_name )<load_pretrained>
all_data['YrBltAndRemod']=all_data['YearBuilt']+all_data['YearRemodAdd']
with open('.. /input/bert-joint-baseline/bert_config.json','r')as f:
config = json.load(f)
 print(json.dumps(config,indent=4))
base_bert_nq= mk_model(config )<load_pretrained>
Basement = ['BsmtCond', 'BsmtExposure', 'BsmtFinSF1', 'BsmtFinSF2', 'BsmtFinType1', 'BsmtFinType2', 'BsmtQual', 'BsmtUnfSF', 'TotalBsmtSF']
Bsmt=all_data[Basement]
bert_nq.load_weights('.. /input/bertjointfinetuned/bert-base-e4.h5' )<train_model>
Bsmt=Bsmt.replace(to_replace='Po', value=1)
 Bsmt=Bsmt.replace(to_replace='Fa', value=2)
 Bsmt=Bsmt.replace(to_replace='TA', value=3)
 Bsmt=Bsmt.replace(to_replace='Gd', value=4)
 Bsmt=Bsmt.replace(to_replace='Ex', value=5)
 Bsmt=Bsmt.replace(to_replace='None', value=0)
 Bsmt=Bsmt.replace(to_replace='No', value=1)
 Bsmt=Bsmt.replace(to_replace='Mn', value=2)
 Bsmt=Bsmt.replace(to_replace='Av', value=3)
 Bsmt=Bsmt.replace(to_replace='Gd', value=4)
 Bsmt=Bsmt.replace(to_replace='Unf', value=1)
 Bsmt=Bsmt.replace(to_replace='LwQ', value=2)
 Bsmt=Bsmt.replace(to_replace='Rec', value=3)
 Bsmt=Bsmt.replace(to_replace='BLQ', value=4)
 Bsmt=Bsmt.replace(to_replace='ALQ', value=5)
 Bsmt=Bsmt.replace(to_replace='GLQ', value=6 )
cpkt = tf.train.Checkpoint(model=base_bert_nq)
 cpkt.restore('.. /input/bert-joint-baseline/model_cpkt-1' ).assert_consumed()<init_hyperparams>
Bsmt['BsmtScore']= Bsmt['BsmtQual'] * Bsmt['BsmtCond'] * Bsmt['TotalBsmtSF']
all_data['BsmtScore']=Bsmt['BsmtScore']
Bsmt['BsmtFin'] =(Bsmt['BsmtFinSF1'] * Bsmt['BsmtFinType1'])+(Bsmt['BsmtFinSF2'] * Bsmt['BsmtFinType2'])
 all_data['BsmtFinScore']=Bsmt['BsmtFin']
all_data['BsmtDNF']=(all_data['BsmtFinScore']==0 )
Span = collections.namedtuple("Span", ["start_token_idx", "end_token_idx", "score"])
 class EvalExample(object):

 def __init__(self, example_id, candidates):
self.example_id = example_id
self.candidates = candidates
self.results = {}
self.features = {}
class ScoreSummary(object):
def __init__(self):
self.predicted_label = None
self.short_span_score = None
self.cls_token_score = None
self.answer_type_logits = None
self.start_prob = None
self.end_prob = None
self.answer_type_prob_dist = None
def read_candidates_from_one_split(input_path):

 candidates_dict = {}
if input_path.endswith(".gz"):
with gzip.GzipFile(fileobj=tf.io.gfile.GFile(input_path, "rb")) as input_file:
print("Reading examples from: {}".format(input_path))
for index, line in enumerate(input_file):
e = json.loads(line)
 candidates_dict[e["example_id"]] = e["long_answer_candidates"]
else:
with tf.io.gfile.GFile(input_path, "r")as input_file:
print("Reading examples from: {}".format(input_path))
for index, line in enumerate(input_file):
e = json.loads(line)
 candidates_dict[e["example_id"]] = e["long_answer_candidates"]
return candidates_dict
def read_candidates(input_pattern):

 input_paths = tf.io.gfile.glob(input_pattern)
 final_dict = {}
for input_path in input_paths:
final_dict.update(read_candidates_from_one_split(input_path))
return final_dict
def get_best_indexes(logits, n_best_size, token_map=None):
index_and_score = sorted(enumerate(logits[1:], 1), key=lambda x: x[1], reverse=True)
 best_indexes = []
for i in range(len(index_and_score)) :
idx = index_and_score[i][0]
if token_map is not None and token_map[idx] == -1:
continue
best_indexes.append(idx)
 if len(best_indexes)>= n_best_size:
break
return best_indexes<count_duplicates>
garage=['GarageArea','GarageCars','GarageCond','GarageFinish','GarageQual','GarageType','GarageYrBlt']
Garage=all_data[garage]
all_data['NoGarage']=(all_data['GarageArea']==0 )
def remove_duplicates(span):
start_end = []
for s in span:
cont = 0
if not start_end:
start_end.append(Span(s[0], s[1], s[2]))
cont += 1
else:
for i in range(len(start_end)) :
if start_end[i][0] == s[0] and start_end[i][1] == s[1]:
cont += 1
if cont == 0:
start_end.append(Span(s[0], s[1], s[2]))
return start_end
def get_short_long_span(predictions, example):
sorted_predictions = sorted(predictions, key=lambda x:(x[0], x[2], x[3]), reverse=True)
 short_span = []
long_span = []
for prediction in sorted_predictions:
score, summary, start_span, end_span = prediction
if score > 0:
short_span.append(Span(int(start_span), int(end_span), float(score)))
 short_span = remove_duplicates(short_span)
 for s in range(len(short_span)) :
for c in example.candidates:
start = short_span[s].start_token_idx
end = short_span[s].end_token_idx
if c["top_level"] and c["start_token"] <= start and c["end_token"] >= end:
long_span.append(Span(int(c["start_token"]), int(c["end_token"]), float(short_span[s].score)))
 break
long_span = remove_duplicates(long_span)
 if not long_span:
long_span = [Span(-1, -1, -10000.0)]
if not short_span:
short_span = [Span(-1, -1, -10000.0)]
return short_span, long_span<define_variables>
Garage=Garage.replace(to_replace='Po', value=1)
 Garage=Garage.replace(to_replace='Fa', value=2)
 Garage=Garage.replace(to_replace='TA', value=3)
 Garage=Garage.replace(to_replace='Gd', value=4)
 Garage=Garage.replace(to_replace='Ex', value=5)
 Garage=Garage.replace(to_replace='None', value=0)
 Garage=Garage.replace(to_replace='Unf', value=1)
 Garage=Garage.replace(to_replace='RFn', value=2)
 Garage=Garage.replace(to_replace='Fin', value=3)
 Garage=Garage.replace(to_replace='CarPort', value=1)
 Garage=Garage.replace(to_replace='Basment', value=4)
 Garage=Garage.replace(to_replace='Detchd', value=2)
 Garage=Garage.replace(to_replace='2Types', value=3)
 Garage=Garage.replace(to_replace='Basement', value=5)
 Garage=Garage.replace(to_replace='Attchd', value=6)
 Garage=Garage.replace(to_replace='BuiltIn', value=7)
 Garage['GarageScore']=(Garage['GarageArea'])*(Garage['GarageCars'])*(Garage['GarageFinish'])*(Garage['GarageQual'])*(Garage['GarageType'])
 all_data['GarageScore']=Garage['GarageScore']
def compute_predictions(example):

 predictions = []
max_answer_length = FLAGS.max_answer_length
null_entry_threshold = -0.3
score_null = 1000000
for unique_id, result in example.results.items() :
if unique_id not in example.features:
raise ValueError("No feature found with unique_id:", unique_id)
 token_map = example.features[unique_id]["token_map"].int64_list.value
feature_null_score = result["start_logits"][0] + result["end_logits"][0]
score_null = min(score_null, feature_null_score)
 for start_index, start_logit, start_prob in zip(result["start_indexes"], result["start_logits"], result["start_pos_prob_dist"]):
if token_map[start_index] == -1:
continue
for end_index, end_logit, end_prob in zip(result["end_indexes"], result["end_logits"], result["end_pos_prob_dist"]):
if token_map[end_index] == -1:
continue
if end_index < start_index:
continue
length = end_index - start_index + 1
if length > max_answer_length:
continue
summary = ScoreSummary()
summary.instance_id = unique_id
summary.short_span_score = start_logit + end_logit
summary.cls_token_score = result["cls_start_logit"] + result["cls_end_logit"]
summary.answer_type_logits = result["answer_type_logits"]
summary.start_indexes = result["start_indexes"]
summary.end_indexes = result["end_indexes"]
summary.start_logits = result["start_logits"]
summary.end_logits = result["end_logits"]
summary.start_pos_prob_dist = result["start_pos_prob_dist"]
summary.end_pos_prob_dist = result["end_pos_prob_dist"]
summary.start_index = start_index
summary.end_index = end_index
summary.start_logit = start_logit
summary.end_logit = end_logit
answer_type_prob_dist = result["answer_type_prob_dist"]
summary.start_prob = start_prob
summary.end_prob = end_prob
summary.answer_type_prob_dist = {
"unknown": answer_type_prob_dist[0],
"yes": answer_type_prob_dist[1],
"no": answer_type_prob_dist[2],
"short": answer_type_prob_dist[3],
"long": answer_type_prob_dist[4]
}
start_span = token_map[start_index]
end_span = token_map[end_index] + 1
score = summary.short_span_score - summary.cls_token_score
predictions.append(( score, summary, start_span, end_span))
all_summaries = []
short_span = [Span(-1, -1, 0)]
long_span = [Span(-1, -1, 0)]
score = 0
summary = ScoreSummary()
if predictions:
short_span, long_span = get_short_long_span(predictions, example)
 if len(all_summaries)== 0:
summary.predicted_label = {
"example_id": example.example_id,
"instance_id": None,
"long_answer": {
"tokens_and_score": long_span,
"start_byte": -1,
"end_byte": -1
},
"long_answer_score": score,
"short_answers": [{
"tokens_and_score": short_span,
"start_byte": -1,
"end_byte": -1
}],
"short_answers_score": score,
"yes_no_answer": "NONE",
"no_answer": True
}
all_summaries.append(summary)
 all_summaries = all_summaries[:min(FLAGS.n_best_size, len(all_summaries)) ]
return all_summaries
def compute_pred_dict(candidates_dict, dev_features, raw_results):

 raw_results_by_id = [(int(res["unique_id"]), 1, res, None)for res in raw_results]
examples_by_id = [(int(tf.cast(int(k), dtype=tf.int32)) , 0, v, k)for k, v in candidates_dict.items() ]
features_by_id = [(int(tf.cast(f.features.feature["unique_ids"].int64_list.value[0], dtype=tf.int32)) , 2, f.features.feature, None)for f in dev_features]
print('merging examples...')
 merged = sorted(examples_by_id + raw_results_by_id + features_by_id)
 print('done.')
 examples = []
for idx, type_, datum, orig_example_id in merged:
if type_ == 0:
examples.append(EvalExample(orig_example_id, datum))
elif type_ == 2:
examples[-1].features[idx] = datum
else:
examples[-1].results[idx] = datum
summary_dict = {}
nq_pred_dict = {}
for e in examples:
all_summaries = compute_predictions(e)
 summary_dict[e.example_id] = all_summaries
nq_pred_dict[e.example_id] = [summary.predicted_label for summary in all_summaries]
if len(nq_pred_dict)% 100 == 0:
print("Examples processed: %d" % len(nq_pred_dict))
return nq_pred_dict<define_variables>
all_data=all_data.drop(columns=['PoolArea','PoolQC'] )
def get_prediction_json(mode, max_nb_pos_logits=-1):
if mode == 'valid':
dataset = validation_dataset
if FLAGS.smaller_valid_dataset:
predict_file = FLAGS.validation_predict_file_small
prediction_output_file = FLAGS.validation_small_prediction_output_file
else:
predict_file = FLAGS.validation_predict_file
prediction_output_file = FLAGS.validation_prediction_output_file
eval_features = validation_features
else:
dataset = test_dataset
predict_file = FLAGS.predict_file
eval_features = test_features
prediction_output_file = FLAGS.prediction_output_file
print(predict_file)
 print(prediction_output_file)
 all_results = []
prediction_start_time = datetime.datetime.now()
for(batch_idx, features)in enumerate(dataset):
batch_start_time = datetime.datetime.now()
unique_ids = features['unique_ids']
token_maps = features['token_map'] 
(input_ids, input_masks, segment_ids)=(features['input_ids'], features['input_mask'], features['segment_ids'])
 nq_inputs =(input_ids, input_masks, segment_ids)
 nq_logits = bert_nq(nq_inputs, training=False)
 base_nq_logits = base_bert_nq(nq_inputs, training=False)
(start_pos_logits, end_pos_logits, answer_type_logits)= nq_logits 
(base_start_pos_logits, base_end_pos_logits, baseanswer_type_logits)= base_nq_logits
start_pos_logits =(0.2 * start_pos_logits + 0.8 * base_start_pos_logits)
 end_pos_logits =(0.2 * end_pos_logits + 0.8 * base_end_pos_logits)
 answer_type_logits =(0.2 * answer_type_logits + 0.8 * baseanswer_type_logits)
 unique_ids = unique_ids.numpy().tolist()
token_maps = token_maps.numpy().tolist()
start_pos_prob_dist = tf.nn.softmax(start_pos_logits, axis=-1 ).numpy().tolist()
end_pos_prob_dist = tf.nn.softmax(end_pos_logits, axis=-1 ).numpy().tolist()
answer_type_prob_dist = tf.nn.softmax(answer_type_logits, axis=-1 ).numpy().tolist()
start_pos_logits = start_pos_logits.numpy().tolist()
end_pos_logits = end_pos_logits.numpy().tolist()
answer_type_logits = answer_type_logits.numpy().tolist()
for uid, token_map, s, e, a, sp, ep, ap in zip(unique_ids, token_maps, start_pos_logits, end_pos_logits, answer_type_logits, start_pos_prob_dist, end_pos_prob_dist, answer_type_prob_dist):
if max_nb_pos_logits < 0:
max_nb_pos_logits = len(start_pos_logits)
 cls_start_logit = s[0]
cls_end_logit = e[0]
start_indexes = get_best_indexes(s, max_nb_pos_logits, token_map)
 end_indexes = get_best_indexes(e, max_nb_pos_logits, token_map)
 s = [s[idx] for idx in start_indexes]
e = [e[idx] for idx in end_indexes]
sp = [sp[idx] for idx in start_indexes]
ep = [ep[idx] for idx in end_indexes]
raw_result = {
"unique_id": uid,
"start_indexes": start_indexes,
"end_indexes": end_indexes,
"start_logits": s,
"end_logits": e,
"answer_type_logits": a,
"start_pos_prob_dist": sp,
"end_pos_prob_dist": ep,
"answer_type_prob_dist": ap,
"cls_start_logit": cls_start_logit,
"cls_end_logit": cls_end_logit
}
all_results.append(raw_result)
 batch_end_time = datetime.datetime.now()
batch_elapsed_time =(batch_end_time - batch_start_time ).total_seconds()
if(batch_idx + 1)% 100 == 0:
print('Batch {} | Elapsed Time {}'.format(
 batch_idx + 1,
batch_elapsed_time 
))
prediction_end_time = datetime.datetime.now()
prediction_elapsed_time =(prediction_end_time - prediction_start_time ).total_seconds()
print('
Time taken for prediction: {} secs
'.format(prediction_elapsed_time))
print("-" * 80 + "
")
 print("Going to candidates file")
 candidates_dict = read_candidates(predict_file)
 print("setting up eval features")
 print("compute_pred_dict")
 nq_pred_dict = compute_pred_dict(candidates_dict, eval_features, all_results)
 predictions_json = {"predictions": list(nq_pred_dict.values())}
print("writing json")
 with tf.io.gfile.GFile(prediction_output_file, "w")as f:
json.dump(predictions_json, f, indent=4)
 return predictions_json<categorify>
all_data['MSSubClass']=all_data['MSSubClass'].astype(str)
 all_data['MoSold']=all_data['MoSold'].astype(str)
 all_data['YrSold']=all_data['YrSold'].astype(str )
def create_answer_from_token_indices(answer):
if answer["start_token"] == -1 or answer["end_token"] == -1:
return ""
else:
return str(answer["start_token"])+ ":" + str(answer["end_token"])
 def create_long_answer_from_1_pred(pred):

 if pred["long_answer_score"] < 1.5 or pred['answer_type'] == 0:
return ""
long_answer = create_answer_from_token_indices(pred["long_answer"])
 return long_answer
def create_short_answers_from_1_pred(pred):

 short_answers = []
for predicted_short_answer in pred["short_answers"]:
short_answer = create_answer_from_token_indices(predicted_short_answer)
 if "answer_type_prob_dist" in pred:
if pred["short_answers_score"] < 1.5 or pred['answer_type'] == 0:
if short_answer not in ["YES", "NO"]:
short_answer = ""
short_answers.append(short_answer)
 return short_answers
def is_pred_ok(pred, annotations):

 long_labels = []
for annotation in annotations:
long_label = create_answer_from_token_indices(annotation["long_answer"])
 long_labels.append(long_label)
 non_null_long_labels = [x for x in long_labels if x != ""]
has_long_label = len(non_null_long_labels)> 1
long_pred = create_long_answer_from_1_pred(pred)
 has_long_pred =(long_pred != "")
 short_label_lists = []
for annotation in annotations:
if len(annotation["short_answers"])== 0:
if annotation["yes_no_answer"] == "YES":
short_label_lists.append(["YES"])
 elif annotation["yes_no_answer"] == "NO":
short_label_lists.append(["NO"])
 else:
short_label_lists.append([""])
 else:
short_labels = []
for anno_short_answer in annotation["short_answers"]:
short_label = create_answer_from_token_indices(anno_short_answer)
 short_labels.append(short_label)
 short_label_lists.append(short_labels)
 non_null_short_label_lists = [x for x in short_label_lists if x != [""]]
has_short_label = len(non_null_short_label_lists)> 1
short_preds = create_short_answers_from_1_pred(pred)
 has_short_pred =(short_preds != [""])
 is_long_pred_correct = False
is_short_pred_correct = False
for long_label in long_labels:
if has_long_label and long_label == "":
continue
if not has_long_label and long_label != "":
continue
if long_pred == long_label:
is_long_pred_correct = True
break
for short_labels in short_label_lists:
if has_short_label and short_labels == [""]:
continue
if not has_short_label and short_labels != [""]:
continue
if has_short_label:
if short_labels == ["YES"] or short_labels == ["NO"]:
if short_preds == short_labels:
is_short_pred_correct = True
break
else:
if short_preds[0] in short_labels:
is_short_pred_correct = True
break
else:
if short_preds == short_labels:
is_short_pred_correct = True
break
return has_long_label, has_short_label, has_long_pred, has_short_pred, is_long_pred_correct, is_short_pred_correct
def compute_f1_scores(predictions_json, gold_jsonl_file):
predictions = predictions_json["predictions"]
golden_nq_lines = jsonl_iterator([gold_jsonl_file])
 golden_dict = dict()
for nq_line in golden_nq_lines:
nq_data = json.loads(nq_line)
 golden = dict()
golden["example_id"] = nq_data["example_id"]
golden["annotations"] = nq_data["annotations"]
golden_dict[golden["example_id"]] = golden
long_labels = []
long_preds = []
short_labels = []
short_preds = []
for preds in predictions:
pred = preds[0]
example_id = pred["example_id"]
assert example_id in golden_dict
golden = golden_dict[example_id]
assert example_id == golden["example_id"]
has_long_label, has_short_label, has_long_pred, has_short_pred, is_long_correct, is_short_correct = is_pred_ok(pred, golden["annotations"])
 if has_long_label or has_long_pred:
if is_long_correct:
long_labels.append(1)
 long_preds.append(1)
 else:
long_labels.append(1)
 long_preds.append(0)
 if has_short_label or has_short_pred:
if is_short_correct:
short_labels.append(1)
 short_preds.append(1)
 else:
short_labels.append(1)
 short_preds.append(0)
 f1 = f1_score(long_labels + short_labels, long_preds + short_preds)
 long_f1 = f1_score(long_labels, long_preds)
 short_f1 = f1_score(short_labels, short_preds)
 return f1, long_f1, short_f1<feature_engineering>
non_numeric=all_data.select_dtypes(include='object')
 def onehot(col_list):
global all_data
while len(col_list)!=0:
col=col_list.pop(0)
 data_encoded=pd.get_dummies(all_data[col], prefix=col)
 all_data=pd.merge(all_data, data_encoded, on='Id')
 all_data=all_data.drop(columns=col)
 print(all_data.shape)
 onehot(list(non_numeric))
def df_long_index_score(df):
answers = []
cont = 0
for e in df[0]['long_answer']['tokens_and_score']:
if e[2] >= 1.5:
index = {}
index['start'] = e[0]
index['end'] = e[1]
index['score'] = e[2]
answers.append(str(index['start'])+ ":" + str(index['end']))
cont += 1
if cont == 1:
break
if len(answers)== 0:
return ""
return answers[0]
def df_short_index_score(df):
answers = []
cont = 0
for e in df[0]['short_answers'][0]['tokens_and_score']:
if e[2] >= 3.0:
index = {}
index['start'] = e[0]
index['end'] = e[1]
index['score'] = e[2]
answers.append(str(index['start'])+ ":" + str(index['end']))
cont += 1
if cont == 1:
break
if len(answers)== 0:
return ""
return answers[0]<prepare_output>
numeric=all_data.select_dtypes(exclude='object')
 def log_transform(col_list):
transformed_col=[]
while len(col_list)!=0:
col=col_list.pop(0)
 if all_data[col].skew() > 0.5:
all_data[col]=np.log(all_data[col]+1)
 transformed_col.append(col)
 else:
pass
print(f"{len(transformed_col)} features had been tranformed")
 print(all_data.shape)
 log_transform(list(numeric))
if FLAGS.do_predict:
test_features =(tf.train.Example.FromString(r.numpy())for r in tf.data.TFRecordDataset(FLAGS.test_tf_record))
predictions_json = get_prediction_json(mode='test', max_nb_pos_logits=FLAGS.n_best_size )<categorify>
from sklearn.model_selection import KFold, cross_val_score, GridSearchCV
from sklearn.metrics import mean_squared_error
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from xgboost import XGBRegressor
from lightgbm import LGBMRegressor
def create_long_answer(preds):

 return create_long_answer_from_1_pred(preds[0])
 def create_short_answer(preds):

 return create_short_answers_from_1_pred(preds[0])[0]
if FLAGS.do_predict:
test_answers_df = pd.read_json(FLAGS.prediction_output_file)
 test_answers_df["long_answer_score"] = test_answers_df["predictions"].apply(lambda q: q[0]["long_answer_score"])
 test_answers_df["short_answer_score"] = test_answers_df["predictions"].apply(lambda q: q[0]["short_answers_score"])
 test_answers_df["long_answer_score"].describe()
sample_submission = pd.read_csv(FLAGS.sample_submission_csv)
 sample_submission.to_csv("submission.csv", index=False)
 test_answers_df["long_answer"] = test_answers_df["predictions"].apply(df_long_index_score)
 test_answers_df["short_answer"] = test_answers_df["predictions"].apply(df_short_index_score)
 test_answers_df["example_id"] = test_answers_df["predictions"].apply(lambda q: str(q[0]["example_id"]))
long_answers = dict(zip(test_answers_df["example_id"], test_answers_df["long_answer"]))
short_answers = dict(zip(test_answers_df["example_id"], test_answers_df["short_answer"]))
long_prediction_strings = sample_submission[sample_submission["example_id"].str.contains("_long")].apply(lambda q: long_answers[q["example_id"].replace("_long", "")], axis=1)
 short_prediction_strings = sample_submission[sample_submission["example_id"].str.contains("_short")].apply(lambda q: short_answers[q["example_id"].replace("_short", "")], axis=1)
 sample_submission.loc[sample_submission["example_id"].str.contains("_long"), "PredictionString"] = long_prediction_strings
sample_submission.loc[sample_submission["example_id"].str.contains("_short"), "PredictionString"] = short_prediction_strings
sample_submission.to_csv("submission.csv", index=False )<load_from_csv>
kfold = KFold(n_splits=4 )
df = pd.read_csv('submission.csv')
 df.head()<define_variables>
GBC = GradientBoostingRegressor()
gb_param_grid = {'n_estimators' : [100, 200, 500],
'learning_rate': [0.01, 0.05, 0.1, 0.2],
'max_depth': [3, 5, 10, 15],
'min_samples_leaf': [100,150],
'max_features': [0.3, 0.1]
}
gsGBC = GridSearchCV(GBC,param_grid = gb_param_grid, cv=kfold,
scoring="neg_mean_squared_error", n_jobs= 4, verbose = 1)
 gsGBC.fit(Xtrain,Ytrain)
 GBC_best = gsGBC.best_estimator_
gsGBC.best_score_
on_kaggle_server = os.path.exists('/kaggle')
 nq_test_file = '.. /input/tensorflow2-question-answering/simplified-nq-test.jsonl'
nq_train_file = '.. /input/tensorflow2-question-answering/simplified-nq-train.jsonl'
public_dataset = os.path.getsize(nq_test_file)<20_000_000
private_dataset = os.path.getsize(nq_test_file)>=20_000_000<load_pretrained>
XGB=XGBRegressor()
xgb_param_grid = {'learning_rate':[0.01, 0.05, 0.1, 0.2],
'n_estimators':[100, 1000, 2000],
'max_depth':[3, 5, 10, 15]}
gsXGB = GridSearchCV(XGB,param_grid = xgb_param_grid,
cv=kfold, scoring="neg_mean_squared_error",
n_jobs=4, verbose =1)
 gsXGB.fit(Xtrain,Ytrain)
 XGB_best = gsXGB.best_estimator_
gsXGB.best_score_
if True:
importlib.reload(bert_utils )<init_hyperparams>
LGB = LGBMRegressor()
lgb_param_grid = {
'num_leaves' : [1,5,10],
'learning_rate': [1,0.1,0.01,0.001],
'n_estimators': [50, 100, 200, 500, 1000,5000],
'max_depth': [15,20,25],
'num_leaves': [50, 100, 200],
'min_split_gain': [0.3, 0.4]
}
gsLGB = GridSearchCV(LGB,param_grid = lgb_param_grid,
cv=kfold, scoring="neg_mean_squared_error",
n_jobs= 4, verbose = 1)
 gsLGB.fit(Xtrain,Ytrain)
 LGB_best = gsLGB.best_estimator_
gsLGB.best_score_
config = {'attention_probs_dropout_prob':0.1,
'hidden_act':'gelu',
'hidden_dropout_prob':0.1,
'hidden_size':1024,
'initializer_range':0.02,
'intermediate_size':4096,
'max_position_embeddings':512,
'num_attention_heads':16,
'num_hidden_layers':24,
'type_vocab_size':2,
'vocab_size':30522}<choose_model_class>
from sklearn.ensemble import VotingRegressor
class TDense(tf.keras.layers.Layer):
def __init__(self,
output_size,
kernel_initializer=None,
bias_initializer="zeros",
**kwargs):
super().__init__(**kwargs)
 self.output_size = output_size
self.kernel_initializer = kernel_initializer
self.bias_initializer = bias_initializer
def build(self,input_shape):
dtype = tf.as_dtype(self.dtype or tf.keras.backend.floatx())
 if not(dtype.is_floating or dtype.is_complex):
raise TypeError("Unable to build `TDense` layer with "
"non-floating point(and non-complex)"
"dtype %s" %(dtype,))
input_shape = tf.TensorShape(input_shape)
 if tf.compat.dimension_value(input_shape[-1])is None:
raise ValueError("The last dimension of the inputs to "
"`TDense` should be defined."
"Found `None`.")
 last_dim = tf.compat.dimension_value(input_shape[-1])
 self.input_spec = tf.keras.layers.InputSpec(min_ndim=3, axes={-1: last_dim})
 self.kernel = self.add_weight(
 "kernel",
shape=[self.output_size,last_dim],
initializer=self.kernel_initializer,
dtype=self.dtype,
trainable=True)
 self.bias = self.add_weight(
 "bias",
shape=[self.output_size],
initializer=self.bias_initializer,
dtype=self.dtype,
trainable=True)
 super(TDense, self ).build(input_shape)
 def call(self,x):
return tf.matmul(x,self.kernel,transpose_b=True)+self.bias
def mk_model(config):
seq_len = config['max_position_embeddings']
unique_id = tf.keras.Input(shape=(1,),dtype=tf.int64,name='unique_id')
 input_ids = tf.keras.Input(shape=(seq_len,),dtype=tf.int32,name='input_ids')
 input_mask = tf.keras.Input(shape=(seq_len,),dtype=tf.int32,name='input_mask')
 segment_ids = tf.keras.Input(shape=(seq_len,),dtype=tf.int32,name='segment_ids')
 BERT = modeling.BertModel(config=config,name='bert')
 pooled_output, sequence_output = BERT(input_word_ids=input_ids,
input_mask=input_mask,
input_type_ids=segment_ids)
 logits = TDense(2,name='logits' )(sequence_output)
 start_logits,end_logits = tf.split(logits,axis=-1,num_or_size_splits= 2,name='split')
 start_logits = tf.squeeze(start_logits,axis=-1,name='start_squeeze')
 end_logits = tf.squeeze(end_logits, axis=-1,name='end_squeeze')
 ans_type = TDense(5,name='ans_type' )(pooled_output)
 return tf.keras.Model([input_ for input_ in [unique_id,input_ids,input_mask,segment_ids]
if input_ is not None],
[unique_id,start_logits,end_logits,ans_type],
name='bert-baseline' )<categorify>
votingC = VotingRegressor(estimators=[('LGB', LGB_best), 
('GBC',GBC_best), 
('XGB', XGB_best)], n_jobs=4)
 votingC = votingC.fit(Xtrain, Ytrain)
model= mk_model(config )<categorify>
test_SalePrice = pd.Series(votingC.predict(Xtest), name="SalePrice" )
class DummyObject:
def __init__(self,**kwargs):
self.__dict__.update(kwargs)
 FLAGS=DummyObject(skip_nested_contexts=True,
max_position=50,
max_contexts=48,
max_query_length=64,
max_seq_length=512,
doc_stride=128,
include_unknowns=0.1,
n_best_size=20,
max_answer_length=30 )<define_variables>
submission = pd.DataFrame({
"Id" :Id_test,
"SalePrice": test_SalePrice
})
 submission.head()
eval_records = ".. /input/bert-joint-baseline/nq-test.tfrecords"
if on_kaggle_server and private_dataset:
eval_records='nq-test.tfrecords'
if not os.path.exists(eval_records):
eval_writer = bert_utils.FeatureWriter(
 filename=os.path.join(eval_records),
is_training=False)
 tokenizer = tokenization.FullTokenizer(vocab_file='.. /input/bert-joint-baseline/vocab-nq.txt',
do_lower_case=True)
 features = []
convert = bert_utils.ConvertExamples2Features(tokenizer=tokenizer,
is_training=False,
output_fn=eval_writer.process_feature,
collect_stat=False)
 n_examples = 0
tqdm_notebook= tqdm.tqdm_notebook if not on_kaggle_server else None
for examples in bert_utils.nq_examples_iter(input_file=nq_test_file,
is_training=False,
tqdm=tqdm_notebook):
for example in examples:
n_examples += convert(example)
 eval_writer.close()<categorify>
submission.to_csv('submission_voting.csv', index=False )