Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<import_modules>
warnings.filterwarnings(action="ignore")
 target_name = 'SalePrice'
dataset_train_raw = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 dataset_test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 dataset_train_raw
class ScoreSummary(object):
def __init__(self):
self.predicted_label = None
self.short_span_score = None
self.cls_token_score = None
self.answer_type_logits = None<prepare_output>
ignore_feature = ['Id']
y_train = dataset_train_raw[target_name]
dataset_train = dataset_train_raw.drop([target_name] + ignore_feature, axis=1, inplace=False)
 dataset_test.drop(ignore_feature, axis=1, inplace=True)
 all_data = pd.concat([dataset_train, dataset_test], axis=0, sort=False)
 all_data
class EvalExample(object):

 def __init__(self, example_id, candidates):
self.example_id = example_id
self.candidates = candidates
self.results = {}
self.features = {}<find_best_params>
specially_missed = ['Alley',
'PoolQC',
'MiscFeature',
'Fence',
'FireplaceQu',
'GarageType',
'GarageFinish',
'GarageQual',
'GarageCond',
'BsmtQual',
'BsmtCond',
'BsmtExposure',
'BsmtFinType1',
'BsmtFinType2',
'MasVnrType']
for feature in specially_missed:
all_data[feature] = all_data[feature].fillna('None' )
def get_best_indexes(logits, n_best_size):

 index_and_score = sorted(
 enumerate(logits[1:], 1), key=lambda x: x[1], reverse=True)
 best_indexes = []
for i in range(len(index_and_score)) :
if i >= n_best_size:
break
best_indexes.append(index_and_score[i][0])
 return best_indexes
def top_k_indices(logits,n_best_size,token_map):
indices = np.argsort(logits[1:])+1
indices = indices[token_map[indices]!=-1]
return indices[-n_best_size:]<count_duplicates>
numeric_missed = ['BsmtFinSF1',
'BsmtFinSF2',
'BsmtUnfSF',
'TotalBsmtSF',
'BsmtFullBath',
'BsmtHalfBath',
'GarageYrBlt',
'GarageArea',
'GarageCars',
'MasVnrArea']
for feature in numeric_missed:
all_data[feature] = all_data[feature].fillna(0 )
def remove_duplicates(span):
start_end = []
for s in span:
cont = 0
if not start_end:
start_end.append(Span(s[0], s[1], s[2]))
cont += 1
else:
for i in range(len(start_end)) :
if start_end[i][0] == s[0] and start_end[i][1] == s[1]:
cont += 1
if cont == 0:
start_end.append(Span(s[0], s[1], s[2]))
return start_end<define_variables>
all_data['MSSubClass'] = all_data['MSSubClass'].astype(str)
 all_data['YrSold'] = all_data['YrSold'].astype(str)
 all_data['MoSold'] = all_data['MoSold'].astype(str )
def get_short_long_span(predictions, example):
sorted_predictions = sorted(predictions, reverse=True)
 short_span = []
long_span = []
for prediction in sorted_predictions:
score, _, summary, start_span, end_span = prediction
if score > 0:
short_span.append(Span(int(start_span), int(end_span), float(score)))
 short_span = remove_duplicates(short_span)
 for s in range(len(short_span)) :
for c in example.candidates:
start = short_span[s].start_token_idx
end = short_span[s].end_token_idx
if c["top_level"] and c["start_token"] <= start and c["end_token"] >= end:
long_span.append(Span(int(c["start_token"]), int(c["end_token"]), float(short_span[s].score)))
 break
long_span = remove_duplicates(long_span)
 if not long_span:
long_span = [Span(-1, -1, -10000.0)]
if not short_span:
short_span = [Span(-1, -1, -10000.0)]
return short_span, long_span<find_best_params>
all_data['Functional'] = all_data['Functional'].fillna('Typ')
 all_data['Utilities'] = all_data['Utilities'].fillna('AllPub')
 all_data['KitchenQual'] = all_data['KitchenQual'].fillna('TA')
 all_data['Electrical'] = all_data['Electrical'].fillna('SBrkr')
 all_data['Exterior1st'] = all_data['Exterior1st'].fillna(all_data['Exterior1st'].mode() [0])
 all_data['Exterior2nd'] = all_data['Exterior2nd'].fillna(all_data['Exterior2nd'].mode() [0])
 all_data['SaleType'] = all_data['SaleType'].fillna(all_data['SaleType'].mode() [0])
 all_data['LotFrontage'] = all_data.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.mean()))
all_data['MSZoning'] = all_data.groupby('MSSubClass')['MSZoning'].transform(lambda x: x.fillna(x.mode() [0]))
def compute_predictions(example):

 predictions = []
n_best_size = FLAGS.n_best_size
max_answer_length = FLAGS.max_answer_length
i = 0
for unique_id, result in example.results.items() :
if unique_id not in example.features:
raise ValueError("No feature found with unique_id:", unique_id)
 token_map = np.array(example.features[unique_id]["token_map"])
 start_indexes = top_k_indices(result.start_logits,n_best_size,token_map)
 if len(start_indexes)==0:
continue
end_indexes = top_k_indices(result.end_logits,n_best_size,token_map)
 if len(end_indexes)==0:
continue
indexes = np.array(list(np.broadcast(start_indexes[None],end_indexes[:,None])))
 indexes = indexes[(indexes[:,0]<indexes[:,1])*(indexes[:,1]-indexes[:,0]<max_answer_length)]
for _,(start_index,end_index)in enumerate(indexes):
summary = ScoreSummary()
summary.short_span_score =(
 result.start_logits[start_index] +
result.end_logits[end_index])
 summary.cls_token_score =(
 result.start_logits[0] + result.end_logits[0])
 summary.answer_type_logits = result.answer_type_logits-result.answer_type_logits.mean()
start_span = token_map[start_index]
end_span = token_map[end_index] + 1
score = summary.short_span_score - summary.cls_token_score
predictions.append(( score, i, summary, start_span, end_span))
i += 1
short_span = [Span(-1, -1, -10000.0)]
long_span = [Span(-1, -1, -10000.0)]
summary = ScoreSummary()
if predictions:
short_span, long_span = get_short_long_span(predictions, example)
 summary.predicted_label = {
"example_id": int(example.example_id),
"long_answers": {
"tokens_and_score": long_span,
"start_byte": -1,
"end_byte": -1
},
"short_answers": {
"tokens_and_score": short_span,
"start_byte": -1,
"end_byte": -1,
"yes_no_answer": "NONE"
}
}
return summary<load_pretrained>
numeric_feats = all_data.dtypes[all_data.dtypes != 'object'].index
skewed_feats = all_data[numeric_feats].apply(lambda x: skew(x)).sort_values(ascending=False)
 high_skew = skewed_feats[abs(skewed_feats)> 0.5]
high_skew
def read_candidates_from_one_split(input_path):

 candidates_dict = {}
print("Reading examples from: %s" % input_path)
 if input_path.endswith(".gz"):
with gzip.GzipFile(fileobj=tf.io.gfile.GFile(input_path, "rb")) as input_file:
for index, line in enumerate(input_file):
e = json.loads(line)
 candidates_dict[e["example_id"]] = e["long_answer_candidates"]
else:
with tf.io.gfile.GFile(input_path, "r")as input_file:
for index, line in enumerate(input_file):
e = json.loads(line)
 candidates_dict[e["example_id"]] = e["long_answer_candidates"]
return candidates_dict<string_transform>
for feature in high_skew.index:
all_data[feature] = np.log1p(all_data[feature] )
def read_candidates(input_pattern):

 input_paths = tf.io.gfile.glob(input_pattern)
 final_dict = {}
for input_path in input_paths:
final_dict.update(read_candidates_from_one_split(input_path))
return final_dict<load_pretrained>
all_data['TotalSF'] = all_data['TotalBsmtSF'] + all_data['1stFlrSF'] + all_data['2ndFlrSF']
all_data['SqFtPerRoom'] = all_data['GrLivArea'] /(all_data['TotRmsAbvGrd'] + all_data['FullBath'] +
all_data['HalfBath'] + all_data['KitchenAbvGr'])
 all_data['TotalHomeQuality'] = all_data['OverallQual'] + all_data['OverallCond']
all_data['TotalBathrooms'] =(all_data['FullBath'] +(0.5 * all_data['HalfBath'])+
all_data['BsmtFullBath'] +(0.5 * all_data['BsmtHalfBath']))
all_results = [bert_utils.RawResult(*x)for x in zip(*result)]
print("Going to candidates file")
 candidates_dict = read_candidates('.. /input/tensorflow2-question-answering/simplified-nq-test.jsonl')
 print("setting up eval features")
 eval_features = list(token_map_ds)
 print("compute_pred_dict")
 tqdm_notebook= tqdm.tqdm_notebook
nq_pred_dict = compute_pred_dict(candidates_dict,
eval_features,
all_results,
tqdm=tqdm_notebook)
 predictions_json = {"predictions": list(nq_pred_dict.values())}
print("writing json")
 with tf.io.gfile.GFile('predictions.json', "w")as f:
json.dump(predictions_json, f, indent=4)
 print('done!' )<load_from_disk>
X_all = pd.get_dummies(all_data)
 X_train = X_all[:len(y_train)]
X_test = X_all[len(y_train):]
X_train
answers_df = pd.read_json(".. /working/predictions.json")
 answers_df.head()<feature_engineering>
kf = KFold(n_splits=8, random_state=42, shuffle=True)
 def cv_rmse(model):
return -cross_val_score(model, X_train, y_train, scoring='neg_root_mean_squared_error', cv=kf)
 models = ['Linear', 'SVR', 'Random_Forest', 'XGBR', 'Cat_Boost', 'Ridge', 'Elastic_Net', 'Lasso', 'Stack']
scores = []
lin = LinearRegression()
score_lin = cv_rmse(lin)
 scores.append(score_lin.mean())
 svr = SVR()
score_svr = cv_rmse(svr)
 scores.append(score_svr.mean())
 rfr = RandomForestRegressor()
score_rfr = cv_rmse(rfr)
 scores.append(score_rfr.mean())
 xgb = xg.XGBRegressor()
score_xgb = cv_rmse(xgb)
 scores.append(score_xgb.mean())
 catb = CatBoostRegressor(verbose=0, allow_writing_files=False)
 score_catb = cv_rmse(catb)
 scores.append(score_catb.mean())
 rid = Ridge()
score_rid = cv_rmse(rid)
 scores.append(score_rid.mean())
 el = ElasticNet()
score_el = cv_rmse(el)
 scores.append(score_el.mean())
 las = Lasso()
score_las = cv_rmse(las)
 scores.append(score_las.mean())
 stack_gen = StackingRegressor(regressors=(CatBoostRegressor(verbose=0, allow_writing_files=False),
Ridge() ,
xg.XGBRegressor() ,
RandomForestRegressor()),
meta_regressor=CatBoostRegressor(verbose=0, allow_writing_files=False),
use_features_in_secondary=True)
 score_stack_gen = cv_rmse(stack_gen)
 scores.append(score_stack_gen.mean())
 cv_score = pd.DataFrame(models, columns=['Regressors'])
 cv_score['RMSE_mean'] = scores
cv_score
def df_long_index_score(df):
answers = []
cont = 0
for e in df['long_answers']['tokens_and_score']:
if e[2] > 3:
index = {}
index['start'] = e[0]
index['end'] = e[1]
index['score'] = e[2]
answers.append(index)
 cont += 1
if cont == 1:
break
return answers
def df_short_index_score(df):
answers = []
cont = 0
for e in df['short_answers']['tokens_and_score']:
if e[2] > 8:
index = {}
index['start'] = e[0]
index['end'] = e[1]
index['score'] = e[2]
answers.append(index)
 cont += 1
if cont == 1:
break
return answers
def df_example_id(df):
return df['example_id']<feature_engineering>
predictions = {}
def xgbr(X_train, y_train, X_test):
xgbrM = xg.XGBRegressor()
params = {'max_depth': [3, 4, 5, 6, 7, 8],
'min_child_weight': [0, 4, 5, 6, 7, 8],
'learning_rate': [0.01, 0.05, 0.1, 0.2, 0.25, 0.8, 1],
'n_estimators': [10, 30, 50, 100, 200, 400, 1000]}
grid_search_xg = RandomizedSearchCV(estimator=xgbrM, scoring='neg_root_mean_squared_error', param_distributions=params, n_iter=200, cv=4, verbose=2,
random_state=42, n_jobs=-1)
 grid_search_xg.fit(X_train, y_train)
 xgbrModel = grid_search_xg.best_estimator_
print('Best params(XGBR):',grid_search_xg.best_params_)
 print('RMSE(XGBR):', -grid_search_xg.best_score_)
 return xgbrModel
xgbrModel = xg.XGBRegressor(n_estimators=400, min_child_weight=5, max_depth=7, learning_rate=0.05)
 xgbrModel.fit(X_train, y_train)
 predictions['XGBR'] = xgbrModel.predict(X_test )
answers_df['example_id'] = answers_df['predictions'].apply(df_example_id)
 answers_df['long_indexes_and_scores'] = answers_df['predictions'].apply(df_long_index_score)
 answers_df['short_indexes_and_scores'] = answers_df['predictions'].apply(df_short_index_score)
 answers_df.head()<drop_column>
def ridge(X_train, y_train, X_test):
alpha_ridge = {'alpha': [-3, -2, -1, 1e-15, 1e-10, 1e-8, 1e-5, 1e-4, 1e-3, 1e-2, 0.5, 1, 1.5, 2, 3, 4, 5, 10, 20, 30, 40]}
rd = Ridge()
grid_search_rd = GridSearchCV(estimator=rd, scoring='neg_root_mean_squared_error', param_grid=alpha_ridge, cv=4, n_jobs=-1, verbose=3)
 grid_search_rd.fit(X_train, y_train)
 ridgeModel = grid_search_rd.best_estimator_
print('Best params(Ridge):', grid_search_rd.best_params_)
 print('RMSE(Ridge):', -grid_search_rd.best_score_)
 return ridgeModel
ridgeModel = Ridge(alpha=10)
 ridgeModel.fit(X_train, y_train)
 predictions['Ridge'] = ridgeModel.predict(X_test )
answers_df = answers_df.drop(['predictions'], axis=1)
 answers_df.head()<string_transform>
def catBoost(X_train, y_train, X_test):
catM = CatBoostRegressor(verbose=0, allow_writing_files=False)
 params = {'learning_rate': [0.01, 0.05, 0.005, 0.0005],
'depth': [4, 6, 10],
'l2_leaf_reg': [1, 2, 3, 5, 9]}
grid_search_cat = RandomizedSearchCV(estimator=catM, scoring='neg_root_mean_squared_error', param_distributions=params, n_iter=10, cv=4, verbose=2,
random_state=42, n_jobs=-1)
 grid_search_cat.fit(X_train, y_train)
 catModel = grid_search_cat.best_estimator_
print('Best params(CatBoost):',grid_search_cat.best_params_)
 print('RMSE(CatBoost):', -grid_search_cat.best_score_)
 return catModel
catModel = CatBoostRegressor(verbose=0, allow_writing_files=False, learning_rate=0.05, l2_leaf_reg=2, depth=4)
 catModel.fit(X_train, y_train)
 predictions['CatBoost'] = catModel.predict(X_test )
def create_answer(entry):
answer = []
for e in entry:
answer.append(str(e['start'])+ ':'+ str(e['end']))
if not answer:
answer = ""
return ", ".join(answer)
<feature_engineering>
final_prediction = 0.25 * predictions['XGBR'] + 0.35 * predictions['CatBoost'] + 0.4 * predictions['Ridge']
<drop_column><EOS> 
result = pd.DataFrame([len(y_train)+ 1 + i for i in range(len(X_test)) ], columns=['Id'])
 result[target_name] = np.expm1(final_prediction)
 result.to_csv('result.csv', index=False, header=True)
 result