Kaggle competition: House Prices - Advanced Regression Techniques
seq_length = FLAGS.max_seq_length
name_to_features = {
"unique_id": tf.io.FixedLenFeature([], tf.int64),
"input_ids": tf.io.FixedLenFeature([seq_length], tf.int64),
"input_mask": tf.io.FixedLenFeature([seq_length], tf.int64),
"segment_ids": tf.io.FixedLenFeature([seq_length], tf.int64),
}
def _decode_record(record, name_to_features=name_to_features):

 example = tf.io.parse_single_example(serialized=record, features=name_to_features)
 for name in list(example.keys()):
t = example[name]
if name != 'unique_id':
t = tf.cast(t, dtype=tf.int32)
 example[name] = t
return example
def _decode_tokens(record):
return tf.io.parse_single_example(serialized=record,
features={
"unique_id": tf.io.FixedLenFeature([], tf.int64),
"token_map" : tf.io.FixedLenFeature([seq_length], tf.int64)
 })
<categorify>
d_train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 d_test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv' )
raw_ds = tf.data.TFRecordDataset(eval_records)
 token_map_ds = raw_ds.map(_decode_tokens)
 decoded_ds = raw_ds.map(_decode_record)
 ds = decoded_ds.batch(batch_size=16,drop_remainder=False )<predict_on_test>
pd.set_option('display.max_columns', None)
 pd.set_option('display.max_rows', None )
result=model.predict_generator(ds,verbose=1 )<load_pretrained>
warnings.filterwarnings('ignore' )
np.savez_compressed('bert-joint-baseline-output.npz',
**dict(zip(['uniqe_id','start_logits','end_logits','answer_type_logits'],
result)) )<create_dataframe>
d_train.isnull().sum()
Span = collections.namedtuple("Span", ["start_token_idx", "end_token_idx"] )<import_modules>
d_test.isnull().sum()
class ScoreSummary(object):
def __init__(self):
self.predicted_label = None
self.short_span_score = None
self.cls_token_score = None
self.answer_type_logits = None<prepare_output>
d_train.corr() ['SalePrice'].sort_values(ascending=False )
class EvalExample(object):

 def __init__(self, example_id, candidates):
self.example_id = example_id
self.candidates = candidates
self.results = {}
self.features = {}<find_best_params>
d_train = d_train.drop(d_train[(d_train['SalePrice']>740000)&(d_train['SalePrice']<756000)].index ).reset_index(drop=True )
def get_best_indexes(logits, n_best_size):

 index_and_score = sorted(
 enumerate(logits[1:], 1), key=lambda x: x[1], reverse=True)
 best_indexes = []
for i in range(len(index_and_score)) :
if i >= n_best_size:
break
best_indexes.append(index_and_score[i][0])
 return best_indexes
def top_k_indices(logits,n_best_size,token_map):
indices = np.argsort(logits[1:])+1
indices = indices[token_map[indices]!=-1]
return indices[-n_best_size:]<find_best_params>
d_train = d_train.drop(d_train[(d_train['1stFlrSF']>4690)&(d_train['1stFlrSF']<4700)].index ).reset_index(drop=True )
def compute_predictions(example):

 predictions = []
n_best_size = FLAGS.n_best_size
max_answer_length = FLAGS.max_answer_length
i = 0
for unique_id, result in example.results.items() :
if unique_id not in example.features:
raise ValueError("No feature found with unique_id:", unique_id)
 token_map = np.array(example.features[unique_id]["token_map"])
 start_indexes = top_k_indices(result.start_logits,n_best_size,token_map)
 if len(start_indexes)==0:
continue
end_indexes = top_k_indices(result.end_logits,n_best_size,token_map)
 if len(end_indexes)==0:
continue
indexes = np.array(list(np.broadcast(start_indexes[None],end_indexes[:,None])))
 indexes = indexes[(indexes[:,0]<indexes[:,1])*(indexes[:,1]-indexes[:,0]<max_answer_length)]
for start_index,end_index in indexes:
summary = ScoreSummary()
summary.short_span_score =(
 result.start_logits[start_index] +
result.end_logits[end_index])
 summary.cls_token_score =(
 result.start_logits[0] + result.end_logits[0])
 summary.answer_type_logits = result.answer_type_logits-result.answer_type_logits.mean()
start_span = token_map[start_index]
end_span = token_map[end_index] + 1
score = summary.short_span_score - summary.cls_token_score
predictions.append(( score, i, summary, start_span, end_span))
i += 1
score = -10000.0
short_span = Span(-1, -1)
 long_span = Span(-1, -1)
 summary = ScoreSummary()
if predictions:
score, _, summary, start_span, end_span = sorted(predictions, reverse=True)[0]
short_span = Span(start_span, end_span)
 for c in example.candidates:
start = short_span.start_token_idx
end = short_span.end_token_idx
if c["top_level"] and c["start_token"] <= start and c["end_token"] >= end:
long_span = Span(c["start_token"], c["end_token"])
 break
summary.predicted_label = {
"example_id": int(example.example_id),
"long_answer": {
"start_token": int(long_span.start_token_idx),
"end_token": int(long_span.end_token_idx),
"start_byte": -1,
"end_byte": -1
},
"long_answer_score": float(score),
"short_answers": [{
"start_token": int(short_span.start_token_idx),
"end_token": int(short_span.end_token_idx),
"start_byte": -1,
"end_byte": -1
}],
"short_answer_score": float(score),
"yes_no_answer": "NONE",
"answer_type_logits": summary.answer_type_logits.tolist() ,
"answer_type": int(np.argmax(summary.answer_type_logits))
}
return summary<load_pretrained>
d_train = d_train.drop(d_train[(d_train['GrLivArea']>4000)&(d_train['SalePrice']<250000)].index ).reset_index(drop=True )
def read_candidates_from_one_split(input_path):

 candidates_dict = {}
print("Reading examples from: %s" % input_path)
 if input_path.endswith(".gz"):
with gzip.GzipFile(fileobj=tf.io.gfile.GFile(input_path, "rb")) as input_file:
for index, line in enumerate(input_file):
e = json.loads(line)
 candidates_dict[e["example_id"]] = e["long_answer_candidates"]
else:
with tf.io.gfile.GFile(input_path, "r")as input_file:
for index, line in enumerate(input_file):
e = json.loads(line)
 candidates_dict[e["example_id"]] = e["long_answer_candidates"]
return candidates_dict<string_transform>
d_train = d_train.drop(d_train[(d_train['GarageCars']>3)&(d_train['SalePrice']<290000)].index ).reset_index(drop=True )
def read_candidates(input_pattern):

 input_paths = tf.io.gfile.glob(input_pattern)
 final_dict = {}
for input_path in input_paths:
final_dict.update(read_candidates_from_one_split(input_path))
return final_dict<load_pretrained>
d_train = d_train.drop(d_train[(d_train['GarageArea']>1240)&(d_train['GarageArea']<1400)].index ).reset_index(drop=True )
all_results = [bert_utils.RawResult(*x)for x in zip(*result)]
print("Going to candidates file")
 candidates_dict = read_candidates('.. /input/tensorflow2-question-answering/simplified-nq-test.jsonl')
 print("setting up eval features")
 eval_features = list(token_map_ds)
 print("compute_pred_dict")
 tqdm_notebook= tqdm.tqdm_notebook
nq_pred_dict = compute_pred_dict(candidates_dict,
eval_features,
all_results,
tqdm=tqdm_notebook)
 predictions_json = {"predictions": list(nq_pred_dict.values())}
print("writing json")
 with tf.io.gfile.GFile('predictions.json', "w")as f:
json.dump(predictions_json, f, indent=4 )<define_variables>
def Series_stats(var, category, prop1, prop2):
s1 = d_train[(d_train[category]==prop1)][var]
s2 = d_train[(d_train[category]==prop2)][var]
t, p = ttest_ind(s1,s2,equal_var = False)
 print("Two-sample t-test: t={}, p={}".format(round(t,5),p))
if(( p < 0.05)and(np.abs(t)> 1.96)) :
print("
REJECT the Null Hypothesis and state that:
at 5% significance level, the mean {} of {}-{} and {}-{} are not equal.".format(var, prop1, category, prop2, category))
print("
YES, the {} of {}-{} differ significantly from {}-{} in the current dataset.".format(var, prop1, category, prop2, category))
print("
The mean value of {} for {}-{} is {} and for {}-{} is {}".format(var, prop1, category, round(s1.mean() ,2), prop2, category, round(s2.mean() ,2)))
 else:
print("
FAIL to Reject the Null Hypothesis and state that:
at 5% significance level, the mean {} of {} - {} and {} - {} are equal.".format(var, prop1, category, prop2, category))
print("
NO, the {} of {}-{} NOT differ significantly from {}-{} in the current dataset".format(var, prop1, category, prop2, category))
print("
The mean value of {} for {}-{} is {} and for {}-{} is {}".format(var, prop1, category, round(s1.mean() ,2), prop2, category, round(s2.mean() ,2)) )
def create_short_answer(entry):
answer = []
if entry['answer_type'] == 0:
return ""
elif entry['answer_type'] == 1:
return 'YES'
elif entry['answer_type'] == 2:
return 'NO'
elif entry["short_answer_score"] < 1.5:
return ""
else:
for short_answer in entry["short_answers"]:
if short_answer["start_token"] > -1:
answer.append(str(short_answer["start_token"])+ ":" + str(short_answer["end_token"]))
return " ".join(answer)
 def create_long_answer(entry):
answer = []
if entry['answer_type'] == 0:
return ''
elif entry["long_answer_score"] < 1.5:
return ""
elif entry["long_answer"]["start_token"] > -1:
answer.append(str(entry["long_answer"]["start_token"])+ ":" + str(entry["long_answer"]["end_token"]))
return " ".join(answer )<feature_engineering>
d_train['Exterior1st'].fillna("VinylSd", inplace=True)
 d_train['Exterior2nd'].fillna("VinylSd", inplace=True)
 d_train['KitchenQual'].fillna("TA", inplace=True)
 d_train['Functional'].fillna("Typ", inplace=True)
 d_train['SaleType'].fillna("WD", inplace=True)
 d_test['MSZoning'].fillna('RL', inplace=True)
 d_test['Utilities'].fillna('NoSeWa', inplace=True)
 d_test['Exterior1st'].fillna("VinylSd", inplace=True)
 d_test['Exterior2nd'].fillna("VinylSd", inplace=True)
 d_test['KitchenQual'].fillna("TA", inplace=True)
 d_test['Functional'].fillna("Typ", inplace=True)
 d_test['SaleType'].fillna("WD", inplace=True)
 d_test['Electrical'].fillna("SBrkr")
 d_train = d_train.fillna("None")
 d_test = d_test.fillna("None" )
test_answers_df = pd.read_json(".. /working/predictions.json")
 for var_name in ['long_answer_score','short_answer_score','answer_type']:
test_answers_df[var_name] = test_answers_df['predictions'].apply(lambda q: q[var_name])
 test_answers_df["long_answer"] = test_answers_df["predictions"].apply(create_long_answer)
 test_answers_df["short_answer"] = test_answers_df["predictions"].apply(create_short_answer)
 test_answers_df["example_id"] = test_answers_df["predictions"].apply(lambda q: str(q["example_id"]))
long_answers = dict(zip(test_answers_df["example_id"], test_answers_df["long_answer"]))
short_answers = dict(zip(test_answers_df["example_id"], test_answers_df["short_answer"]))
test_answers_df.head()<feature_engineering>
d_train.isnull().sum()
sample_submission = pd.read_csv(".. /input/tensorflow2-question-answering/sample_submission.csv")
 long_prediction_strings = sample_submission[sample_submission["example_id"].str.contains("_long")].apply(lambda q: long_answers[q["example_id"].replace("_long", "")], axis=1)
 short_prediction_strings = sample_submission[sample_submission["example_id"].str.contains("_short")].apply(lambda q: short_answers[q["example_id"].replace("_short", "")], axis=1)
 sample_submission.loc[sample_submission["example_id"].str.contains("_long"), "PredictionString"] = long_prediction_strings
sample_submission.loc[sample_submission["example_id"].str.contains("_short"), "PredictionString"] = short_prediction_strings 
<save_to_csv>
d_test.isnull().sum()
sample_submission.to_csv('submission.csv', index=False )<filter>
train=copy.deepcopy(d_train)
 test=copy.deepcopy(d_test)
 cols=np.array(d_train.columns[d_train.dtypes != object])
 for i in train.columns:
if i not in cols:
train[i]=train[i].map(str)
 test[i]=test[i].map(str)
 train.drop(columns=cols,inplace=True)
 test.drop(columns=np.delete(cols,len(cols)-1),inplace=True )
yes_answers = sample_submission[sample_submission['PredictionString'] == 'YES']
yes_answers<filter>
cols = np.array(d_train.columns[d_train.dtypes != object])
 d = defaultdict(LabelEncoder)
 train = train.apply(lambda x: d[x.name].fit_transform(x))
test = test.apply(lambda x: d[x.name].transform(x))
train[cols] = d_train[cols]
test[np.delete(cols,len(cols)-1)]=d_test[np.delete(cols,len(cols)-1)]
no_answers = sample_submission[sample_submission['PredictionString'] == 'NO']
no_answers<define_variables>
train["O_Style"] = train["Condition1"] + train["Condition2"] + train["BldgType"] + train["HouseStyle"]+ train["RoofStyle"] + train["MasVnrType"]
train["O_Neighbor"] = train["Street"] + train["LotShape"] + train["Neighborhood"] * train["Condition1"]
train["O_Street"] = train["Street"] * train["LotShape"] + train["LandContour"] * train["LotConfig"]
train["SqFtPerRoom"] = train["GrLivArea"] /(train["TotRmsAbvGrd"] + train["FullBath"] +train["HalfBath"] + train["KitchenAbvGr"])
 train["HighQualSF"] = train["1stFlrSF"] + train["2ndFlrSF"] + train["TotalBsmtSF"]
train["Years_Years"] = train["YrSold"] - train["YearRemodAdd"]
train['Total_Home_Quality'] = train['OverallQual'] + train['OverallCond']
train['Total_Bathrooms'] =(train['FullBath'] +(0.5 * train['HalfBath'])+ train['BsmtFullBath'] +(0.5 * train['BsmtHalfBath']))
test["O_Style"] = test["Condition1"] + test["Condition2"] + test["BldgType"] + test["HouseStyle"]+ test["RoofStyle"] + test["MasVnrType"]
test["O_Neighbor"] = test["Street"] + test["LotShape"] + test["Neighborhood"] * test["Condition1"]
test["O_Street"] = test["Street"] * test["LotShape"] + test["LandContour"] * test["LotConfig"]
test["SqFtPerRoom"] = test["GrLivArea"] /(test["TotRmsAbvGrd"] + test["FullBath"] +test["HalfBath"] + test["KitchenAbvGr"])
 test["HighQualSF"] = test["1stFlrSF"] + test["2ndFlrSF"] + test["TotalBsmtSF"]
test["Years_Years"] = test["YrSold"] - test["YearRemodAdd"]
test['Total_Home_Quality'] = test['OverallQual'] + test['OverallCond']
test['Total_Bathrooms'] =(test['FullBath'] +(0.5 * test['HalfBath'])+ test['BsmtFullBath'] +(0.5 * test['BsmtHalfBath']))
blank_answers = sample_submission[sample_submission['PredictionString'] == '']
blank_answers.head()<count_values>
train = train.drop(['Id'], axis=1)
 test = test.drop(['Id'], axis=1 )
blank_answers.count()<define_variables>
from sklearn import metrics
from sklearn import model_selection
from sklearn import preprocessing
from sklearn.datasets import make_classification
from sklearn.ensemble import ExtraTreesRegressor
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.ensemble import RandomForestRegressor
from sklearn.metrics import accuracy_score
from sklearn.metrics import classification_report, confusion_matrix
from sklearn.metrics import mean_squared_error, mean_absolute_error
from sklearn.metrics import plot_confusion_matrix
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import train_test_split
from sklearn.neighbors import KNeighborsRegressor
from sklearn.tree import DecisionTreeRegressor
from xgboost.sklearn import XGBRegressor
from catboost import CatBoostRegressor
from sklearn.ensemble import GradientBoostingRegressor,AdaBoostRegressor,BaggingRegressor, RandomForestRegressor
import xgboost as xgb
import lightgbm as lgb
on_kaggle_server = os.path.exists('/kaggle')
 nq_test_file = '.. /input/tensorflow2-question-answering/simplified-nq-test.jsonl'
nq_train_file = '.. /input/tensorflow2-question-answering/simplified-nq-train.jsonl'
public_dataset = os.path.getsize(nq_test_file)<20_000_000
private_dataset = os.path.getsize(nq_test_file)>=20_000_000<load_pretrained>
X = train.drop(columns=['SalePrice'] ).values
y = np.log1p(train["SalePrice"])
 Z = test.values
X_train, X_test1, y_train, y_test1 = train_test_split(X, y, test_size = 0.0001, random_state = 42)
 X_train1, X_test, y_train1, y_test = train_test_split(X, y, test_size = 0.5, random_state = 42 )
if True:
importlib.reload(bert_utils )<init_hyperparams>
Acc = pd.DataFrame(index=None, columns=['model','Root Mean Squared Error','Accuracy on Traing set','Accuracy on Testing set'])
 regressors = [['DecisionTreeRegressor',DecisionTreeRegressor() ],
['XGBRegressor', XGBRegressor() ],
['CatBoostRegressor', CatBoostRegressor(verbose= False)],
['LGBMRegressor',lgb.LGBMRegressor() ],
['GradientBoostingRegressor',GradientBoostingRegressor() ],
['ExtraTreesRegressor',ExtraTreesRegressor() ]]
for mod in regressors:
name = mod[0]
model = mod[1]
model.fit(X_train1,y_train1)
 ATrS = model.score(X_train1,y_train1)
 ATeS = model.score(X_test,y_test)
 RMSE = mean_squared_error(y_test, model.predict(X_test))
Acc = Acc.append(pd.Series({'model':name, 'Root Mean Squared Error': RMSE,'Accuracy on Traing set':ATrS,'Accuracy on Testing set':ATeS}),ignore_index=True)
 Acc.sort_values(by='Root Mean Squared Error' )
config = {'attention_probs_dropout_prob':0.1,
'hidden_act':'gelu',
'hidden_dropout_prob':0.1,
'hidden_size':1024,
'initializer_range':0.02,
'intermediate_size':4096,
'max_position_embeddings':512,
'num_attention_heads':16,
'num_hidden_layers':24,
'type_vocab_size':2,
'vocab_size':30522}<choose_model_class>
def modelFitter(maxDepth, num_leaves, learning_rate, n_estimators):
model = lgb.LGBMRegressor(learning_rate=learning_rate,
num_leaves=num_leaves.astype("int32"),
max_depth=maxDepth.astype("int32"),
bagging_freq=bagging_freq.astype("int32"))
evalSet = [(X_test, y_test)]
model.fit(X_train, y_train, eval_metric="rmse", eval_set=evalSet, early_stopping_rounds=50, verbose=False)
 bestScore = model.best_score_[list(model.best_score_.keys())[0]]['rmse']
return -bestScore
pbounds = {'maxDepth':(1,5),'num_leaves':(10, 35),'learning_rate':(0.005, 0.01), 'bagging_freq':(4, 6)}
class TDense(tf.keras.layers.Layer):
def __init__(self,
output_size,
kernel_initializer=None,
bias_initializer="zeros",
**kwargs):
super().__init__(**kwargs)
 self.output_size = output_size
self.kernel_initializer = kernel_initializer
self.bias_initializer = bias_initializer
def build(self,input_shape):
dtype = tf.as_dtype(self.dtype or tf.keras.backend.floatx())
 if not(dtype.is_floating or dtype.is_complex):
raise TypeError("Unable to build `TDense` layer with "
"non-floating point(and non-complex)"
"dtype %s" %(dtype,))
input_shape = tf.TensorShape(input_shape)
 if tf.compat.dimension_value(input_shape[-1])is None:
raise ValueError("The last dimension of the inputs to "
"`TDense` should be defined."
"Found `None`.")
 last_dim = tf.compat.dimension_value(input_shape[-1])
 self.input_spec = tf.keras.layers.InputSpec(min_ndim=3, axes={-1: last_dim})
 self.kernel = self.add_weight(
 "kernel",
shape=[self.output_size,last_dim],
initializer=self.kernel_initializer,
dtype=self.dtype,
trainable=True)
 self.bias = self.add_weight(
 "bias",
shape=[self.output_size],
initializer=self.bias_initializer,
dtype=self.dtype,
trainable=True)
 super(TDense, self ).build(input_shape)
 def call(self,x):
return tf.matmul(x,self.kernel,transpose_b=True)+self.bias
def mk_model(config):
seq_len = config['max_position_embeddings']
unique_id = tf.keras.Input(shape=(1,),dtype=tf.int64,name='unique_id')
 input_ids = tf.keras.Input(shape=(seq_len,),dtype=tf.int32,name='input_ids')
 input_mask = tf.keras.Input(shape=(seq_len,),dtype=tf.int32,name='input_mask')
 segment_ids = tf.keras.Input(shape=(seq_len,),dtype=tf.int32,name='segment_ids')
 BERT = modeling.BertModel(config=config,name='bert')
 pooled_output, sequence_output = BERT(input_word_ids=input_ids,
input_mask=input_mask,
input_type_ids=segment_ids)
 logits = TDense(2,name='logits' )(sequence_output)
 start_logits,end_logits = tf.split(logits,axis=-1,num_or_size_splits= 2,name='split')
 start_logits = tf.squeeze(start_logits,axis=-1,name='start_squeeze')
 end_logits = tf.squeeze(end_logits, axis=-1,name='end_squeeze')
 ans_type = TDense(5,name='ans_type' )(pooled_output)
 return tf.keras.Model([input_ for input_ in [unique_id,input_ids,input_mask,segment_ids]
if input_ is not None],
[unique_id,start_logits,end_logits,ans_type],
name='bert-baseline' )<categorify>
LGBMR = lgb.LGBMRegressor(objective='regression', num_leaves=6,learning_rate=0.02, n_estimators=2000,max_bin=100,
bagging_fraction=0.8,bagging_freq=4, bagging_seed=8,feature_fraction=0.2,feature_fraction_seed=8,
min_sum_hessian_in_leaf = 11,verbose=-1,random_state=42)
 LGBMR.fit(X_train,y_train)
 ATrS = LGBMR.score(X_train,y_train)
 ATeS = LGBMR.score(X_test,y_test)
 RMSE = mean_squared_error(y_test, LGBMR.predict(X_test))
print("Root Mean Squared: {}, Accuracy Train set: {},Accuracy Test set: {}".format(RMSE, ATrS, ATeS))
model= mk_model(config )<categorify>
gbr = GradientBoostingRegressor()
params = {'loss': ['huber'],
'learning_rate': [0.012, 0.015, 0.02],
'max_depth': [3, 4, 5],
'min_samples_leaf' : [10, 12, 15],
'min_samples_split' : [3, 5, 7]}
class DummyObject:
def __init__(self,**kwargs):
self.__dict__.update(kwargs)
 FLAGS=DummyObject(skip_nested_contexts=True,
max_position=50,
max_contexts=48,
max_query_length=64,
max_seq_length=512,
doc_stride=128,
include_unknowns=0.02,
n_best_size=5,
max_answer_length=30,
warmup_proportion=0.1,
learning_rate=1e-5,
num_train_epochs=3.0,
train_batch_size=32,
num_train_steps=100000,
num_warmup_steps=10000,
max_eval_steps=100,
use_tpu=False,
eval_batch_size=8,
max_predictions_per_seq=20 )<define_variables>
GBR = GradientBoostingRegressor(n_estimators=1000,learning_rate=0.012,max_depth=6,max_features='sqrt',min_samples_leaf=10,
min_samples_split=4,loss='huber',random_state=12)
 GBR.fit(X_train,y_train)
 ATrS = GBR.score(X_train,y_train)
 ATeS = GBR.score(X_test,y_test)
 RMSE = mean_squared_error(y_test, GBR.predict(X_test))
print("Root Mean Squared: {}, Accuracy Train set: {},Accuracy Test set: {}".format(RMSE, ATrS, ATeS))
eval_records = ".. /input/bert-joint-baseline/nq-test.tfrecords"
if on_kaggle_server and private_dataset:
eval_records='nq-test.tfrecords'
if not os.path.exists(eval_records):
eval_writer = bert_utils.FeatureWriter(
 filename=os.path.join(eval_records),
is_training=False)
 tokenizer = tokenization.FullTokenizer(vocab_file='.. /input/bert-joint-baseline/vocab-nq.txt',
do_lower_case=True)
 features = []
convert = bert_utils.ConvertExamples2Features(tokenizer=tokenizer,
is_training=False,
output_fn=eval_writer.process_feature,
collect_stat=False)
 n_examples = 0
tqdm_notebook= tqdm.tqdm_notebook if not on_kaggle_server else None
for examples in bert_utils.nq_examples_iter(input_file=nq_test_file,
is_training=False,
tqdm=tqdm_notebook):
for example in examples:
n_examples += convert(example)
 eval_writer.close()
print('number of test examples: %d, written to file: %d' %(n_examples,eval_writer.num_features))<categorify>
train_data = X_train
train_labels = y_train
model = CatBoostRegressor()
grid = {'iterations': [4000, 7500],
'learning_rate': [0.003, 0.005, 0.007],
'depth': [3, 7, 10],
'l2_leaf_reg': [1, 2],
'random_seed': [12]}
seq_length = FLAGS.max_seq_length
name_to_features = {
"unique_id": tf.io.FixedLenFeature([], tf.int64),
"input_ids": tf.io.FixedLenFeature([seq_length], tf.int64),
"input_mask": tf.io.FixedLenFeature([seq_length], tf.int64),
"segment_ids": tf.io.FixedLenFeature([seq_length], tf.int64),
}
def _decode_record(record, name_to_features=name_to_features):

 example = tf.io.parse_single_example(serialized=record, features=name_to_features)
 for name in list(example.keys()):
t = example[name]
if name != 'unique_id':
t = tf.cast(t, dtype=tf.int64)
 example[name] = t
return example
def _decode_tokens(record):
return tf.io.parse_single_example(serialized=record,
features={
"unique_id": tf.io.FixedLenFeature([], tf.int64),
"token_map" : tf.io.FixedLenFeature([seq_length], tf.int64)
 })
<categorify>
params = {'iterations': 5500,
'learning_rate': 0.005,
'loss_function':'RMSE',
'depth': 8,
'l2_leaf_reg': 1,
'eval_metric':'RMSE',
'verbose': False,
'random_seed': 12}
CBR = CatBoostRegressor(**params)
 CBR.fit(X_train,y_train)
 ATrS = CBR.score(X_train,y_train)
 ATeS = CBR.score(X_test,y_test)
 RMSE = mean_squared_error(y_test, CBR.predict(X_test))
print("Root Mean Squared: {}, Accuracy Train set: {},Accuracy Test set: {}".format(RMSE, ATrS, ATeS))
raw_ds = tf.data.TFRecordDataset(eval_records)
 token_map_ds = raw_ds.map(_decode_tokens)
 decoded_ds = raw_ds.map(_decode_record)
 ds = decoded_ds.batch(batch_size=32,drop_remainder=False )<predict_on_test>
dtrain = xgb.DMatrix(X_train, label=y_train)
 dtest = xgb.DMatrix(X_test)
 def xgb_evaluate(learning_rate, max_depth, gamma, subsample, colsample_bytree, reg_alpha):
params = {'learning_rate':learning_rate,
'max_depth': int(max_depth),
'gamma': gamma,
'subsample':subsample,
'colsample_bytree': colsample_bytree,
'reg_alpha':reg_alpha}
cv_result = xgb.cv(params, dtrain, num_boost_round=100, nfold=3)
 return -1.0 * cv_result['test-rmse-mean'].iloc[-1]
result=model.predict_generator(ds,verbose=1 )<load_pretrained>
XGBR = xgb.XGBRegressor(colsample_bytree=0.5149443835418306, gamma=0.0,learning_rate=0.03,max_depth=4,
reg_alpha=0.005412343201815549, subsample=0.7119381043613345, n_estimators=1200,
min_child_weight=0,nthread=-1,scale_pos_weight=1,seed=27,random_state=42)
 XGBR.fit(X_train,y_train)
 ATrS = XGBR.score(X_train,y_train)
 ATeS = XGBR.score(X_test,y_test)
 RMSE = mean_squared_error(y_test, XGBR.predict(X_test))
print("Root Mean Squared: {}, Accuracy Train set: {},Accuracy Test set: {}".format(RMSE, ATrS, ATeS))
<create_dataframe><EOS> 
result =(np.expm1(GBR.predict(Z)) + np.expm1(LGBMR.predict(Z)) + np.expm1(CBR.predict(Z)) + np.expm1(XGBR.predict(Z)))/4
sub = pd.DataFrame()
sub = pd.DataFrame({'Id':d_test.Id,'SalePrice':result})
 sub.to_csv('submission.csv',index=False)
 sub.head(2 )