Kaggle competition: House Prices - Advanced Regression Techniques
seq_length = FLAGS.max_seq_length
name_to_features = {
"unique_id": tf.io.FixedLenFeature([], tf.int64),
"input_ids": tf.io.FixedLenFeature([seq_length], tf.int64),
"input_mask": tf.io.FixedLenFeature([seq_length], tf.int64),
"segment_ids": tf.io.FixedLenFeature([seq_length], tf.int64),
}
def _decode_record(record, name_to_features=name_to_features):

 example = tf.io.parse_single_example(serialized=record, features=name_to_features)
 for name in list(example.keys()):
t = example[name]
if name != 'unique_id':
t = tf.cast(t, dtype=tf.int64)
 example[name] = t
return example
def _decode_tokens(record):
return tf.io.parse_single_example(serialized=record,
features={
"unique_id": tf.io.FixedLenFeature([], tf.int64),
"token_map" : tf.io.FixedLenFeature([seq_length], tf.int64)
 })
<categorify>
train = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/train.csv")
 test = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/test.csv")
 print("train shape:" + str(train.shape))
print("test shape:" + str(test.shape))
raw_ds = tf.data.TFRecordDataset(eval_records)
 token_map_ds = raw_ds.map(_decode_tokens)
 decoded_ds = raw_ds.map(_decode_record)
 ds = decoded_ds.batch(batch_size=16,drop_remainder=False )<predict_on_test>
from scipy.stats import skew, norm
import scipy.stats as stats
from scipy.special import boxcox1p
from scipy.stats import boxcox_normmax
result=model.predict_generator(ds,verbose=1 )<load_pretrained>
train[(train['GrLivArea'] > 4500)&(train['SalePrice'] < 250000)] [cols]
np.savez_compressed('bert-joint-baseline-output.npz',
**dict(zip(['uniqe_id','start_logits','end_logits','answer_type_logits'],
result)) )<create_dataframe>
cols_outliers = cols[:]
cols_outliers.extend(['YrSold', 'Neighborhood'])
 train[(train['OverallQual'] >= 9)&(train['GrLivArea'] > 3000)] [cols_outliers]
Span = collections.namedtuple("Span", ["start_token_idx", "end_token_idx", "score"] )<import_modules>
cols_outliers_test = cols_outliers[:]
cols_outliers_test.remove('SalePrice')
 test[(test['OverallQual'] >= 9)&(test['GrLivArea'] > 3000)] [cols_outliers_test]
class ScoreSummary(object):
def __init__(self):
self.predicted_label = None
self.short_span_score = None
self.cls_token_score = None
self.answer_type_logits = None<prepare_output>
train.drop([523, 1298], axis = 0, inplace = True )
class EvalExample(object):

 def __init__(self, example_id, candidates):
self.example_id = example_id
self.candidates = candidates
self.results = {}
self.features = {}<find_best_params>
train[(train['OverallQual'] == 4)&(train['SalePrice'] > 250000)] [cols]
def get_best_indexes(logits, n_best_size):

 index_and_score = sorted(
 enumerate(logits[1:], 1), key=lambda x: x[1], reverse=True)
 best_indexes = []
for i in range(len(index_and_score)) :
if i >= n_best_size:
break
best_indexes.append(index_and_score[i][0])
 return best_indexes
def top_k_indices(logits,n_best_size,token_map):
indices = np.argsort(logits[1:])+1
indices = indices[token_map[indices]!=-1]
return indices[-n_best_size:]<count_duplicates>
pd.DataFrame(train[(train['OverallQual'] == 4)][cols].mean() ).T
def remove_duplicates(span):
start_end = []
for s in span:
cont = 0
if not start_end:
start_end.append(Span(s[0], s[1], s[2]))
cont += 1
else:
for i in range(len(start_end)) :
if start_end[i][0] == s[0] and start_end[i][1] == s[1]:
cont += 1
if cont == 0:
start_end.append(Span(s[0], s[1], s[2]))
return start_end<define_variables>
train[(train['OverallQual'] == 8)&(train['SalePrice'] > 500000)] [cols]
def get_short_long_span(predictions, example):
sorted_predictions = sorted(predictions, reverse=True)
 short_span = []
long_span = []
for prediction in sorted_predictions:
score, _, summary, start_span, end_span = prediction
if score > 0:
short_span.append(Span(int(start_span), int(end_span), float(score)))
 short_span = remove_duplicates(short_span)
 for s in range(len(short_span)) :
for c in example.candidates:
start = short_span[s].start_token_idx
end = short_span[s].end_token_idx
if c["top_level"] and c["start_token"] <= start and c["end_token"] >= end:
long_span.append(Span(int(c["start_token"]), int(c["end_token"]), float(short_span[s].score)))
 long_span = remove_duplicates(long_span)
 long_span.sort(key = operator.itemgetter(2), reverse=True)
 if not long_span:
long_span = [Span(-1, -1, -10000.0)]
if not short_span:
short_span = [Span(-1, -1, -10000.0)]
return short_span, long_span<find_best_params>
pd.DataFrame(train[(train['OverallQual'] == 8)][cols].mean() ).T
def compute_predictions(example):

 predictions = []
n_best_size = FLAGS.n_best_size
max_answer_length = 1024
i = 0
for unique_id, result in example.results.items() :
if unique_id not in example.features:
raise ValueError("No feature found with unique_id:", unique_id)
 token_map = np.array(example.features[unique_id]["token_map"])
 start_indexes = top_k_indices(result.start_logits,n_best_size,token_map)
 if len(start_indexes)==0:
continue
end_indexes = top_k_indices(result.end_logits,n_best_size,token_map)
 if len(end_indexes)==0:
continue
indexes = np.array(list(np.broadcast(start_indexes[None],end_indexes[:,None])))
 indexes = indexes[(indexes[:,0]<indexes[:,1])*(indexes[:,1]-indexes[:,0]<max_answer_length)]
for _,(start_index,end_index)in enumerate(indexes):
summary = ScoreSummary()
summary.short_span_score =(
 result.start_logits[start_index] +
result.end_logits[end_index])
 summary.cls_token_score =(
 result.start_logits[0] + result.end_logits[0])
 summary.answer_type_logits = result.answer_type_logits-result.answer_type_logits.mean()
start_span = token_map[start_index]
end_span = token_map[end_index] + 1
score = summary.short_span_score - summary.cls_token_score
predictions.append(( score, i, summary, start_span, end_span))
i += 1
short_span = [Span(-1, -1, -10000.0)]
long_span = [Span(-1, -1, -10000.0)]
summary = ScoreSummary()
if predictions:
short_span, long_span = get_short_long_span(predictions, example)
 summary.predicted_label = {
"example_id": int(example.example_id),
"long_answers": {
"tokens_and_score": long_span,
"start_byte": -1,
"end_byte": -1
},
"short_answers": {
"tokens_and_score": short_span,
"start_byte": -1,
"end_byte": -1,
"yes_no_answer": "NONE"
}
}
return summary<load_pretrained>
data_sets = [train, test]
data_sets_null = []
for data_set in data_sets:
n_houses = data_set.shape[0]
data_set = data_set.isnull().sum().sort_values(ascending = False)
 data_set = pd.DataFrame(data_set[(data_set != 0)], columns = ['NullSum'])
 data_set['NullPercent'] =(data_set['NullSum']/n_houses)*100
data_sets_null.append(data_set)
 train_null, test_null = data_sets_null
def read_candidates_from_one_split(input_path):

 candidates_dict = {}
print("Reading examples from: %s" % input_path)
 if input_path.endswith(".gz"):
with gzip.GzipFile(fileobj=tf.io.gfile.GFile(input_path, "rb")) as input_file:
for index, line in enumerate(input_file):
e = json.loads(line)
 candidates_dict[e["example_id"]] = e["long_answer_candidates"]
else:
with tf.io.gfile.GFile(input_path, "r")as input_file:
for index, line in enumerate(input_file):
e = json.loads(line)
 candidates_dict[e["example_id"]] = e["long_answer_candidates"]
return candidates_dict<string_transform>
train_null_tidy = train_null.add_prefix('Train_');
test_null_tidy = test_null.add_prefix('Test_');
total_null_tidy = pd.concat([train_null_tidy, test_null_tidy], axis=1 ).fillna(0)
 total_null_tidy
def read_candidates(input_pattern):

 input_paths = tf.io.gfile.glob(input_pattern)
 final_dict = {}
for input_path in input_paths:
final_dict.update(read_candidates_from_one_split(input_path))
return final_dict<load_pretrained>
print("Train features with Na = " + str(( train.isnull().sum() != 0 ).sum()))
print("Test features with Na = " + str(( test.isnull().sum() != 0 ).sum()))
all_results = [bert_utils.RawResult(*x)for x in zip(*result)]
print("Going to candidates file")
 candidates_dict = read_candidates('.. /input/tensorflow2-question-answering/simplified-nq-test.jsonl')
 print("setting up eval features")
 eval_features = list(token_map_ds)
 print("compute_pred_dict")
 tqdm_notebook= tqdm.tqdm_notebook
nq_pred_dict = compute_pred_dict(candidates_dict,
eval_features,
all_results,
tqdm=tqdm_notebook)
 predictions_json = {"predictions": list(nq_pred_dict.values())}
print("writing json")
 with tf.io.gfile.GFile('predictions.json', "w")as f:
json.dump(predictions_json, f, indent=4)
 print('done!' )<load_from_disk>
def handle_missing(data_sets):
for data_set in data_sets:
data_set['MSSubClass'] = data_set ['MSSubClass'].apply(str)
 data_set['YrSold'] = data_set['YrSold'].astype(str)
 data_set['MoSold'] = data_set['MoSold'].astype(str)
 for col in('PoolQC', 'MiscFeature', 'Alley', 'Fence', 'FireplaceQu', 'MasVnrType', 'Utilities'):
data_set[col] = data_set[col].fillna('None')
 data_set['MasVnrArea'] = data_set['MasVnrArea'].fillna(0)
 for col in('Exterior1st', 'Exterior2nd', 'SaleType'):
data_set[col] = data_set[col].fillna(data_set[col].mode() [0])
 data_set['MSZoning'] = data_set.groupby('MSSubClass')['MSZoning'].transform(lambda x: x.fillna(x.mode() [0]))
data_set['LotFrontage'] = data_set.groupby("Neighborhood")['LotFrontage'].transform(lambda x: x.fillna(x.median()))
data_set['Electrical'] = data_set['Electrical'].fillna("SBrkr")
 data_set['Functional'] = data_set['Functional'].fillna('Typ')
 data_set['KitchenQual'] = data_set['KitchenQual'].fillna('TA')
 for col in ['GarageType', 'GarageFinish', 'GarageQual', 'GarageCond']:
data_set[col] = data_set[col].fillna('None')
 for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
data_set[col] = data_set[col].fillna(0)
 for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
data_set[col] = data_set[col].fillna('None')
 for col in('BsmtFullBath', 'BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF', 'BsmtHalfBath'):
data_set[col] = data_set[col].fillna(0 )
answers_df = pd.read_json(".. /working/predictions.json")
 answers_df.head()<feature_engineering>
print("Train features with Na = " + str(( train.isnull().sum() != 0 ).sum()))
print("Test features with Na = " + str(( test.isnull().sum() != 0 ).sum()))
def df_long_index_score(df):
answers = []
cont = 0
for e in df['long_answers']['tokens_and_score']:
if e[2] >= 2.5:
index = {}
index['start'] = e[0]
index['end'] = e[1]
index['score'] = e[2]
answers.append(index)
 cont += 1
if cont == 1:
break
return answers
def df_short_index_score(df):
answers = []
cont = 0
for e in df['short_answers']['tokens_and_score']:
if e[2] >= 6.5:
index = {}
index['start'] = e[0]
index['end'] = e[1]
index['score'] = e[2]
answers.append(index)
 cont += 1
if cont == 1:
break
return answers
def df_example_id(df):
return df['example_id']<feature_engineering>
train['SalePrice_norm'] = np.log1p(train['SalePrice'] )
answers_df['example_id'] = answers_df['predictions'].apply(df_example_id)
 answers_df['long_indexes_and_scores'] = answers_df['predictions'].apply(df_long_index_score)
 answers_df['short_indexes_and_scores'] = answers_df['predictions'].apply(df_short_index_score)
 answers_df.head()<drop_column>
train_Y = train['SalePrice_norm']
train.drop(['SalePrice', 'SalePrice_norm'], axis = 1, inplace = True)
 all_data = pd.concat([train, test], axis=0, ignore_index = True)
answers_df = answers_df.drop(['predictions'], axis=1)
 answers_df.head()<string_transform>
numeric_dtypes = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
numeric = []
for i in all_data.columns:
if all_data[i].dtype in numeric_dtypes:
numeric.append(i)
 skew_features = all_data[numeric].apply(lambda x: skew(x)).sort_values(ascending=False)
 high_skew = skew_features[abs(skew_features)> 0.5]
skew_index = high_skew.index
print("There are {} numerical features with Skew > 0.5 :".format(high_skew.shape[0]))
skewness = pd.DataFrame({'Skew' :high_skew})
 high_skew
def create_answer(entry):
answer = []
for e in entry:
answer.append(str(e['start'])+ ':'+ str(e['end']))
if not answer:
answer = ""
return ", ".join(answer)
<feature_engineering>
for i in skew_index:
all_data[i] = boxcox1p(all_data[i], boxcox_normmax(all_data[i] + 1))
answers_df["long_answer"] = answers_df['long_indexes_and_scores'].apply(create_answer)
 answers_df["short_answer"] = answers_df['short_indexes_and_scores'].apply(create_answer)
 answers_df["example_id"] = answers_df['example_id'].apply(lambda q: str(q))
long_answers = dict(zip(answers_df["example_id"], answers_df["long_answer"]))
short_answers = dict(zip(answers_df["example_id"], answers_df["short_answer"]))
answers_df.head()<drop_column>
all_data['TotalSF'] = all_data['TotalBsmtSF'] + all_data['1stFlrSF'] + all_data['2ndFlrSF']
answers_df = answers_df.drop(['long_indexes_and_scores', 'short_indexes_and_scores'], axis=1)
 answers_df.head()<feature_engineering>
new_data = all_data.copy()
sample_submission = pd.read_csv(".. /input/tensorflow2-question-answering/sample_submission.csv")
 long_prediction_strings = sample_submission[sample_submission["example_id"].str.contains("_long")].apply(lambda q: long_answers[q["example_id"].replace("_long", "")], axis=1)
 short_prediction_strings = sample_submission[sample_submission["example_id"].str.contains("_short")].apply(lambda q: short_answers[q["example_id"].replace("_short", "")], axis=1)
 sample_submission.loc[sample_submission["example_id"].str.contains("_long"), "PredictionString"] = long_prediction_strings
sample_submission.loc[sample_submission["example_id"].str.contains("_short"), "PredictionString"] = short_prediction_strings 
<save_to_csv>
ordinal_label=['LandSlope','YearBuilt','YearRemodAdd',
'CentralAir','GarageYrBlt','PavedDrive',
'YrSold']
for field in ordinal_label:
le = LabelEncoder()
new_data[field] = le.fit_transform(new_data[field].values )
sample_submission.to_csv('submission.csv', index=False )<define_variables>
ordinal_features = [
'MSSubClass','ExterQual','LotShape','BsmtQual','BsmtCond',
'BsmtExposure','BsmtFinType1', 'BsmtFinType2','HeatingQC',
'Functional', 'FireplaceQu','KitchenQual', 'GarageFinish',
'GarageQual','GarageCond', 'PoolQC', 'Fence'
]
orders=[
['20','30','40','45','50','60','70','75','80','85', '90','120','150','160','180','190'],
['Po','Fa','TA','Gd','Ex'],
['Reg','IR1' ,'IR2','IR3'],
['None','Fa','TA','Gd','Ex'],
['None','Po','Fa','TA','Gd','Ex'],
['None','No','Mn','Av','Gd'],
['None','Unf','LwQ', 'Rec','BLQ','ALQ' , 'GLQ' ],
['None','Unf','LwQ', 'Rec','BLQ','ALQ' , 'GLQ' ],
['Po','Fa','TA','Gd','Ex'],
['Sev','Maj2','Maj1','Mod','Min2','Min1','Typ'],
['None','Po','Fa','TA','Gd','Ex'],
['Fa','TA','Gd','Ex'],
['None','Unf','RFn','Fin'],
['None','Po','Fa','TA','Gd','Ex'],
['None','Po','Fa','TA','Gd','Ex'],
['None','Fa','Gd','Ex'],
['None','MnWw','GdWo','MnPrv','GdPrv'] ]
for field in ordinal_features:
new_data[field] = new_data[field].astype(str)
 for i in range(len(orders)) :
ord_en = OrdinalEncoder(categories = [orders[i]])
 new_data[ordinal_features[i]] = pd.DataFrame(ord_en.fit_transform(new_data[[ordinal_features[i]]]))
pred_path = ".. /input/nq-sample-csv/first50.csv"<load_from_disk>
scale = StandardScaler()
new_data.drop("Id", axis = 1, inplace = True)
 numerical_cols = new_data._get_numeric_data().columns
new_data[numerical_cols]= scale.fit_transform(new_data[numerical_cols] )
df = pd.read_json('.. /input/tensorflow2-question-answering/simplified-nq-test.jsonl', lines = True, dtype={'example_id':'Object'})
 submission = pd.DataFrame(index=pd.concat([df.example_id + '_short', df.example_id + '_long']), columns=['PredictionString'] ).sort_index()<feature_engineering>
categorical_cols = list(set(new_data.columns)- set(numerical_cols))
categorical_cols
updates = pd.read_csv(pred_path, na_filter=False ).set_index('example_id' ).sort_index()
submission.loc[updates.index.intersection(submission.index),'PredictionString'] = updates['PredictionString']<save_to_csv>
new_data=pd.get_dummies(new_data )
submission.to_csv('submission.csv' )<define_variables>
import shap
import xgboost as xgb
from catboost import Pool
from sklearn.svm import SVR
from catboost import CatBoostRegressor
from lightgbm import LGBMRegressor
from sklearn.naive_bayes import GaussianNB
from sklearn.tree import DecisionTreeRegressor
from mlxtend.regressor import StackingRegressor
from sklearn.linear_model import LinearRegression, BayesianRidge
from sklearn.model_selection import RepeatedKFold
from sklearn.model_selection import KFold, cross_val_score
from sklearn.model_selection import train_test_split
from sklearn.ensemble import GradientBoostingRegressor, RandomForestRegressor
from sklearn.model_selection import GridSearchCV, RandomizedSearchCV
from sklearn.metrics import mean_squared_error, mean_absolute_error, mean_squared_log_error
from sklearn.preprocessing import RobustScaler
from sklearn.pipeline import make_pipeline
from sklearn.linear_model import Ridge, RidgeCV
from mlxtend.regressor import StackingCVRegressor
from lightgbm import LGBMRegressor
on_kaggle_server = os.path.exists('/kaggle')
 nq_test_file = '.. /input/tensorflow2-question-answering/simplified-nq-test.jsonl'
nq_train_file = '.. /input/tensorflow2-question-answering/simplified-nq-train.jsonl'
public_dataset = os.path.getsize(nq_test_file)<20_000_000
private_dataset = os.path.getsize(nq_test_file)>=20_000_000<load_pretrained>
train_X = new_data.iloc[:len(train), :]
test_X = new_data.iloc[len(train):, :]
test_id = test['Id']

 def rmse(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
def cv_rmse(model):
rmse = np.sqrt(-cross_val_score(model, train_X, train_Y, scoring="neg_mean_squared_error", cv=kf))
return(rmse )
if True:
importlib.reload(bert_utils )<load_from_disk>
cv_scores = []
cv_std = []
kf = KFold(n_splits = 10, shuffle = True, random_state = 1)
 baseline_models = ['Linear_Reg.','Bayesian_Ridge_Reg.','LGBM_Reg.','SVR',
'Dec_Tree_Reg.','Random_Forest_Reg.', 'XGB_Reg.',
'Grad_Boost_Reg.','Cat_Boost_Reg.','Stacked_Reg.']
lreg = LinearRegression()
score_lreg = cv_rmse(lreg)
 cv_scores.append(score_lreg.mean())
 cv_std.append(score_lreg.std())
 brr = BayesianRidge(compute_score=True)
 score_brr = cv_rmse(brr)
 cv_scores.append(score_brr.mean())
 cv_std.append(score_brr.std())
 l_gbm = LGBMRegressor(objective='regression')
 score_l_gbm = cv_rmse(l_gbm)
 cv_scores.append(score_l_gbm.mean())
 cv_std.append(score_l_gbm.std())
 svr = SVR()
score_svr = cv_rmse(svr)
 cv_scores.append(score_svr.mean())
 cv_std.append(score_svr.std())
 dtr = DecisionTreeRegressor()
score_dtr = cv_rmse(dtr)
 cv_scores.append(score_dtr.mean())
 cv_std.append(score_dtr.std())
 rfr = RandomForestRegressor()
score_rfr = cv_rmse(rfr)
 cv_scores.append(score_rfr.mean())
 cv_std.append(score_rfr.std())
 xgb = xgb.XGBRegressor()
score_xgb = cv_rmse(xgb)
 cv_scores.append(score_xgb.mean())
 cv_std.append(score_xgb.std())
 gbr = GradientBoostingRegressor()
score_gbr = cv_rmse(gbr)
 cv_scores.append(score_gbr.mean())
 cv_std.append(score_gbr.std())
 catb = CatBoostRegressor()
score_catb = cv_rmse(catb)
 cv_scores.append(score_catb.mean())
 cv_std.append(score_catb.std())
 stack_gen = StackingRegressor(regressors=(CatBoostRegressor() ,
LinearRegression() ,
BayesianRidge() ,
GradientBoostingRegressor()),
meta_regressor = CatBoostRegressor() ,
use_features_in_secondary = True)
 score_stack_gen = cv_rmse(stack_gen)
 cv_scores.append(score_stack_gen.mean())
 cv_std.append(score_stack_gen.std())
 final_cv_score = pd.DataFrame(baseline_models, columns = ['Regressors'])
 final_cv_score['RMSE_mean'] = cv_scores
final_cv_score['RMSE_std'] = cv_std
with open('.. /input/bert-joint-baseline/bert_config.json','r')as f:
config = json.load(f)
 print(json.dumps(config,indent=4)) 
<choose_model_class>
feat_imp = gbr.get_feature_importance(prettified=True)
 feat_imp
class TDense(tf.keras.layers.Layer):
def __init__(self,
output_size,
kernel_initializer=None,
bias_initializer="zeros",
**kwargs):
super().__init__(**kwargs)
 self.output_size = output_size
self.kernel_initializer = kernel_initializer
self.bias_initializer = bias_initializer
def build(self,input_shape):
dtype = tf.as_dtype(self.dtype or tf.keras.backend.floatx())
 if not(dtype.is_floating or dtype.is_complex):
raise TypeError("Unable to build `TDense` layer with "
"non-floating point(and non-complex)"
"dtype %s" %(dtype,))
input_shape = tf.TensorShape(input_shape)
 if tf.compat.dimension_value(input_shape[-1])is None:
raise ValueError("The last dimension of the inputs to "
"`TDense` should be defined."
"Found `None`.")
 last_dim = tf.compat.dimension_value(input_shape[-1])
 self.input_spec = tf.keras.layers.InputSpec(min_ndim=3, axes={-1: last_dim})
 self.kernel = self.add_weight(
 "kernel",
shape=[self.output_size,last_dim],
initializer=self.kernel_initializer,
dtype=self.dtype,
trainable=True)
 self.bias = self.add_weight(
 "bias",
shape=[self.output_size],
initializer=self.bias_initializer,
dtype=self.dtype,
trainable=True)
 super(TDense, self ).build(input_shape)
 def call(self,x):
return tf.matmul(x,self.kernel,transpose_b=True)+self.bias
def mk_model(config):
seq_len = config['max_position_embeddings']
unique_id = tf.keras.Input(shape=(1,),dtype=tf.int64,name='unique_id')
 input_ids = tf.keras.Input(shape=(seq_len,),dtype=tf.int32,name='input_ids')
 input_mask = tf.keras.Input(shape=(seq_len,),dtype=tf.int32,name='input_mask')
 segment_ids = tf.keras.Input(shape=(seq_len,),dtype=tf.int32,name='segment_ids')
 BERT = modeling.BertModel(config=config,name='bert')
 pooled_output, sequence_output = BERT(input_word_ids=input_ids,
input_mask=input_mask,
input_type_ids=segment_ids)
 logits = TDense(2,name='logits' )(sequence_output)
 start_logits,end_logits = tf.split(logits,axis=-1,num_or_size_splits= 2,name='split')
 start_logits = tf.squeeze(start_logits,axis=-1,name='start_squeeze')
 end_logits = tf.squeeze(end_logits, axis=-1,name='end_squeeze')
 ans_type = TDense(5,name='ans_type' )(pooled_output)
 return tf.keras.Model([input_ for input_ in [unique_id,input_ids,input_mask,segment_ids]
if input_ is not None],
[unique_id,start_logits,end_logits,ans_type],
name='bert-baseline' )<define_variables>
small_config = config.copy()
small_config['vocab_size']=16
small_config['hidden_size']=64
small_config['max_position_embeddings'] = 32
small_config['num_hidden_layers'] = 4
small_config['num_attention_heads'] = 4
small_config['intermediate_size'] = 256
small_config<categorify>
model= mk_model(config )<choose_model_class>
stack_gen.fit(train_X, train_Y)
 test_pred = stack_gen.predict(test_X)
 submission = pd.DataFrame(test_id, columns = ['Id'])
 test_pred = np.expm1(test_pred)
 submission['SalePrice'] = test_pred
submission.head()
class DummyObject:
def __init__(self,**kwargs):
self.__dict__.update(kwargs)
 FLAGS=DummyObject(skip_nested_contexts=True,
max_position=50,
max_contexts=48,
max_query_length=64,
max_seq_length=512,
doc_stride=128,
include_unknowns=-1.0,
n_best_size=20,
max_answer_length=30 )<feature_engineering>
submission.loc[1089, 'SalePrice'] = 125000
submission.to_csv("result.csv", index = False, header = True )
eval_records = ".. /input/bert-joint-baseline/nq-test.tfrecords"
if on_kaggle_server and private_dataset:
eval_records='nq-test.tfrecords'
if not os.path.exists(eval_records):
eval_writer = bert_utils.FeatureWriter(
 filename=os.path.join(eval_records),
is_training=False)
 tokenizer = tokenization.FullTokenizer(vocab_file='.. /input/bert-joint-baseline/vocab-nq.txt',
do_lower_case=True)
 features = []
convert = bert_utils.ConvertExamples2Features(tokenizer=tokenizer,
is_training=False,
output_fn=eval_writer.process_feature,
collect_stat=False)
 n_examples = 0
tqdm_notebook= tqdm.tqdm_notebook if not on_kaggle_server else None
for examples in bert_utils.nq_examples_iter(input_file=nq_test_file,
is_training=False,
tqdm=tqdm_notebook):
for example in examples:
n_examples += convert(example)
 eval_writer.close()
print('number of test examples: %d, written to file: %d' %(n_examples,eval_writer.num_features))<categorify>
submission[submission['Id'] == 2550]