Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<train_model>
warnings.filterwarnings('ignore' )
def cross_validate_transformer_hparams(df, kf, hparams):
errors = np.zeros(0)
 input_ids, attention_mask = encode_with_tokinizer(df.text, tokenizers[hparams['model_name']], MAX_LENGTH)
 for train_index, val_index in kf.split(df):
init_tpu(tpu)
 model = create_model(models[hparams['model_name']]() , hparams)
 model.fit(
 x=[input_ids[train_index], attention_mask[train_index]],
y=df.iloc[train_index].target.values,
validation_data=([input_ids[val_index], attention_mask[val_index]], df.iloc[val_index].target.values),
verbose=2,
epochs=EPOCHS,
batch_size=BATCH_SIZE 
)
 _, a = model.evaluate(x=[input_ids[val_index], attention_mask[val_index]], y=df.iloc[val_index].target.values, verbose=2)
 er = 1 - a
errors = np.append(errors, er)
 del model
mean_error = errors.mean()
print(f'Hparams: {hparams}')
 print(f'Mean error: {mean_error}')
 print('--------------------------------------')
 del input_ids
del attention_mask
return mean_error<define_variables>
train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 total = [train, test]
max_evals = 100<init_hyperparams>
train.isnull().sum().sort_values(ascending=False)[train.isnull().sum().sort_values(ascending=False)> 0] / train.shape[0] * 100
space = {
'layers': ho.hp.choice('layers', [node_params(n)for n in [n for n in range(4)]]),
'lr_rate': ho.hp.loguniform("lr_rate", np.log(0.00001), np.log(0.001)) ,
'model_name': ho.hp.choice('model_name', ['bert', 'roberta', 'distilbert']),
'hidden_states_size': ho.hp.choice('hidden_states_size', [n for n in range(1, 5)])
 } 
<define_variables>
test.isnull().sum().sort_values(ascending=False)[test.isnull().sum().sort_values(ascending=False)> 0] / test.shape[0] * 100
max_evals = 200<find_best_params>
for dataset in total:
dataset['MSSubClass'] = dataset['MSSubClass'].astype(np.object)
 dataset['MoSold'] = dataset['MoSold'].astype(np.object)
 dataset['YrSold'] = dataset['YrSold'].astype(np.object )
trials.trials<define_variables>
skew_feats = []
for feat in numeric_features:
if train[feat].skew() > 0.75:
skew_feats.append(feat)
 for dataset in total:
for feat in skew_feats:
dataset[feat] = dataset[feat].apply(np.log1p )
EPOCHS = 3<split>
mapping = {'Ex': 5, 'Gd': 4, 'TA': 3, 'Fa': 2, 'Po': 1, np.nan:0}
mapping_1 = {'Gd': 4, 'Av': 3, 'Mn':2, 'No':1, np.nan: 0}
mapping_2 = {'GLQ': 6, 'ALQ': 5, 'BLQ': 4, 'Rec': 3, 'LwQ': 2, 'Unf': 1, np.nan: 0}
mapping_3 = {'Typ': 7, 'Min1': 6, 'Min2': 5, 'Mod': 4, 'Maj1': 3, 'Maj2': 2, 'Sev': 1, 'Sal': 0}
mapping_4 = {'Fin': 3, 'RFn': 2, 'Unf': 1, np.nan: 0}
mapping_5 = {'GdPrv': 4, 'MnPrv': 3, 'GdWo': 2, 'MnWw': 1, np.nan: 0}
for dataset in total:
for column in ['ExterQual', 'ExterCond', 'BsmtQual', 'BsmtCond', 'HeatingQC', 'KitchenQual', 'FireplaceQu', 'GarageQual', 'GarageCond', 'PoolQC']:
dataset[column] = dataset[column].map(mapping)
 dataset['BsmtExposure'] = dataset['BsmtExposure'].map(mapping_1)
 dataset['BsmtFinType1'] = dataset['BsmtFinType1'].map(mapping_2)
 dataset['BsmtFinType2'] = dataset['BsmtFinType2'].map(mapping_2)
 dataset['Functional'] = dataset['Functional'].map(mapping_3)
 dataset['GarageFinish'] = dataset['GarageFinish'].map(mapping_4)
 dataset['Fence'] = dataset['Fence'].map(mapping_5)
 dataset[['LotFrontage','GarageYrBlt','MasVnrArea','BsmtFullBath','BsmtHalfBath']] = dataset[['LotFrontage','GarageYrBlt','MasVnrArea','BsmtFullBath','BsmtHalfBath']].fillna(0)
 dataset[['MiscFeature','Alley','GarageType']] = dataset[['MiscFeature','Alley','GarageType']].fillna('No')
 dataset['MasVnrType'] = dataset['MasVnrType'].fillna('None')
 train['Electrical'] = train['Electrical'].fillna(train['Electrical'].mode() [0])
 test.at[1150, 'MasVnrType'] = 'BrkFace'
test.at[1116, 'GarageCars'] = 0
test.at[1116, 'GarageArea'] = 0
test.at[1116, 'GarageType'] = 0
test['BsmtFinSF1'] = test['BsmtFinSF1'].fillna(0)
 test['BsmtUnfSF'] = test['BsmtUnfSF'].fillna(0)
 test['BsmtFinSF2'] = test['BsmtFinSF2'].fillna(0)
 test['TotalBsmtSF'] = test['TotalBsmtSF'].fillna(0)
 test['MSZoning'] = test['MSZoning'].fillna(train['MSZoning'].mode() [0])
 test['Utilities'] = test['Utilities'].fillna(train['Utilities'].mode() [0])
 test['Functional'] = test['Functional'].fillna(train['Functional'].mode() [0])
 test['SaleType'] = test['SaleType'].fillna(train['SaleType'].mode() [0])
 test['Exterior1st'] = test['Exterior1st'].fillna(train['Exterior1st'].mode() [0])
 test['Exterior2nd'] = test['Exterior2nd'].fillna(train['Exterior2nd'].mode() [0] )
tr, hold, val = np.split(train_df, [int (.7*len(train_df)) , int (.9*len(train_df)) ])
 tr.shape, hold.shape, val.shape<choose_model_class>
for dataset in total:
dataset['HasPool'] = dataset['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 dataset['Has2ndFloor'] = dataset['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 dataset['HasGarage'] = dataset['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 dataset['HasBsmt'] = dataset['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 dataset['HasFireplace'] = dataset['Fireplaces'].apply(lambda x: 1 if x > 0 else 0)
 dataset['BltSoldYrDiff'] = dataset['YrSold'].astype(np.int64)- dataset['YearBuilt']
dataset['TotalSF'] = dataset['TotalBsmtSF'] + dataset['1stFlrSF'] + dataset['2ndFlrSF']
dataset['TotalBathr'] = dataset['FullBath'] + 0.5 * dataset['HalfBath'] + dataset['BsmtFullBath'] + 0.5 * dataset['BsmtHalfBath']
dataset['TotalPorchSF'] = dataset['OpenPorchSF'] + dataset['3SsnPorch'] + dataset['EnclosedPorch'] + dataset['ScreenPorch'] + dataset['WoodDeckSF']
def create_final_bert_model(train, val):
input_ids = tf.keras.Input(shape=(MAX_LENGTH,),dtype='int32')
 attention_mask = tf.keras.Input(shape=(MAX_LENGTH,), dtype='int32')
 transformer = models['bert']()([input_ids, attention_mask])
 hidden_states = transformer[1]
output = hidden_states[-1]
output = tf.keras.layers.Dense(175, activation='relu' )(output)
 output = tf.keras.layers.BatchNormalization()(output)
 output = tf.keras.layers.Dropout(0.5 )(output)
 output = tf.keras.layers.Dense(1750, activation='relu' )(output)
 output = tf.keras.layers.BatchNormalization()(output)
 output = tf.keras.layers.Dropout(0.2 )(output)
 output = tf.keras.layers.Dense(1, activation='sigmoid' )(output)
 final_model = tf.keras.models.Model(inputs = [input_ids, attention_mask], outputs = output)
 final_model.compile(tf.keras.optimizers.Adam(lr=1.0210068320376741e-05), loss='binary_crossentropy', metrics=['accuracy'])
 train_input_ids, train_attention_mask = encode_with_tokinizer(train.text.values, tokenizers['bert'], MAX_LENGTH)
 val_input_ids, val_attention_mask = encode_with_tokinizer(val.text.values, tokenizers['bert'], MAX_LENGTH)
 final_model.fit(
 x=[train_input_ids, train_attention_mask],
y=train.target.values,
validation_data=([val_input_ids, val_attention_mask], val.target.values),
verbose=1,
epochs=EPOCHS,
batch_size=BATCH_SIZE 
)
 _, a = final_model.evaluate(x=[val_input_ids, val_attention_mask], y=val.target.values, verbose=2)
 er = 1 - a
print(er)
 return final_model<choose_model_class>
numeric_features = ['LotArea', 'OverallQual', 'OverallCond', 'YearBuilt',
'YearRemodAdd', 'BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF',
'1stFlrSF', '2ndFlrSF', 'LowQualFinSF', 'GrLivArea', 'BsmtFullBath',
'BsmtHalfBath', 'FullBath', 'HalfBath', 'BedroomAbvGr', 'KitchenAbvGr',
'TotRmsAbvGrd', 'Fireplaces', 'GarageCars', 'GarageArea', 'WoodDeckSF',
'OpenPorchSF', 'EnclosedPorch', '3SsnPorch', 'ScreenPorch', 'PoolArea',
'MiscVal', 'LotFrontage', 'MasVnrArea', 'GarageYrBlt', 'SalePrice', 'BltSoldYrDiff',
'TotalSF', 'TotalBathr', 'TotalPorchSF']
def create_final_roberta_model(train, val):
input_ids = tf.keras.Input(shape=(MAX_LENGTH,),dtype='int32')
 attention_mask = tf.keras.Input(shape=(MAX_LENGTH,), dtype='int32')
 transformer = models['roberta']()([input_ids, attention_mask])
 hidden_states = transformer[1]
hiddes_states_ind = list(range(-3, 0, 1))
output = tf.keras.layers.concatenate(tuple([hidden_states[i] for i in hiddes_states_ind]))
output = tf.keras.layers.Dense(1850, activation='relu' )(output)
 output = tf.keras.layers.BatchNormalization()(output)
 output = tf.keras.layers.Dropout(0.55 )(output)
 output = tf.keras.layers.Dense(1975, activation='relu' )(output)
 output = tf.keras.layers.BatchNormalization()(output)
 output = tf.keras.layers.Dropout(0.6 )(output)
 output = tf.keras.layers.Dense(50, activation='relu' )(output)
 output = tf.keras.layers.BatchNormalization()(output)
 output = tf.keras.layers.Dropout(0.2 )(output)
 output = tf.keras.layers.Dense(1, activation='sigmoid' )(output)
 final_model = tf.keras.models.Model(inputs = [input_ids, attention_mask], outputs = output)
 final_model.compile(tf.keras.optimizers.Adam(lr=1.0470485274874683e-05), loss='binary_crossentropy', metrics=['accuracy'])
 train_input_ids, train_attention_mask = encode_with_tokinizer(train.text.values, tokenizers['roberta'], MAX_LENGTH)
 val_input_ids, val_attention_mask = encode_with_tokinizer(val.text.values, tokenizers['roberta'], MAX_LENGTH)
 final_model.fit(
 x=[train_input_ids, train_attention_mask],
y=train.target.values,
validation_data=([val_input_ids, val_attention_mask], val.target.values),
verbose=1,
epochs=EPOCHS,
batch_size=BATCH_SIZE 
)
 _, a = final_model.evaluate(x=[val_input_ids, val_attention_mask], y=val.target.values, verbose=2)
 er = 1 - a
print(er)
 return final_model<compute_test_metric>
train.isnull().sum().sort_values(ascending=False)[train.isnull().sum().sort_values(ascending=False)> 0]
def get_predicitons_with_ensemble(x, ensemble_of_classifiers, y=None):
bert_pred = get_prediciton_with_tokenizer(x, ensemble_of_classifiers['bert'], tokenizers['bert'])
 roberta_pred = get_prediciton_with_tokenizer(x, ensemble_of_classifiers['roberta'], tokenizers['roberta'])
 if y is not None:
print(f'bert accuracy: {m.accuracy_score(y, np.round(bert_pred ).astype(int)) }')
 print(f'roberta accuracy: {m.accuracy_score(y, np.round(roberta_pred ).astype(int)) }')
 pred_results = np.array([bert_pred, roberta_pred])
 pred_results = np.swapaxes(pred_results,0,1)
 return pred_results<predict_on_test>
test.isnull().sum().sort_values(ascending=False)[test.isnull().sum().sort_values(ascending=False)> 0]
def get_prediciton_with_tokenizer(x, classifier, tokenizer):
pred_results = pd.DataFrame()
input_ids, attention_mask = encode_with_tokinizer(x, tokenizer, MAX_LENGTH)
 y_pred = classifier.predict([input_ids, attention_mask])[:, 0].reshape(-1)
 return y_pred 
<categorify>
df_dummies = pd.get_dummies(data=pd.concat([train, test]), columns=columns )
with tpu_strategy.scope() :
bert_model = create_final_bert_model(tr, hold)
 roberta_model = create_final_roberta_model(tr, hold )<define_variables>
train = df_dummies.iloc[:train.shape[0]]
test = df_dummies.iloc[train.shape[0]:].drop('SalePrice', axis=1 )
ensemble_of_classifiers = {'bert': bert_model, 'roberta': roberta_model}<predict_on_test>
X = pd.DataFrame(StandardScaler().fit_transform(train), columns=train.columns ).drop(['Id', 'SalePrice'], axis=1)
 y = train['SalePrice']
scaled_test = pd.DataFrame(StandardScaler().fit_transform(test),columns=test.columns ).drop('Id', axis=1 )
hold_ens_prediction = get_predicitons_with_ensemble(hold.text.values, ensemble_of_classifiers, hold.target.values )<prepare_x_and_y>
reg = RidgeCV()
reg.fit(X, y)
 print("Best alpha using built-in RidgeCV: %f" % reg.alpha_)
 print("Best score using built-in RidgeCV: %f" %reg.score(X,y))
coef = pd.Series(reg.coef_, index = X.columns)
 print("Ridge picked " + str(sum(coef != 0)) + " variables and eliminated the other " + str(sum(coef == 0)) + " variables" )
X_stack_train = hold_ens_prediction
y_stack_train = hold.target.values<compute_train_metric>
reg = ElasticNetCV()
reg.fit(X, y)
 print("Best alpha using built-in ElasticNetCV: %f" % reg.alpha_)
 print("Best score using built-in ElasticNetCV: %f" %reg.score(X,y))
coef = pd.Series(reg.coef_, index = X.columns)
 print("ElasticNet picked " + str(sum(coef != 0)) + " variables and eliminated the other " + str(sum(coef == 0)) + " variables")
 ridge_coef = coef[coef != 0]
def cross_validate_xgb_hparams(hparams, x, y):
estimator = xgboost.XGBClassifier(learning_rate=hparams['learning_rate'], max_depth=hparams['max_depth'], n_estimators=int(hparams['n_estimators']))
cv_results = ms.cross_validate(estimator, x, y, cv=5, scoring='accuracy', n_jobs=3)
 mean_acc = np.mean(cv_results['test_score'])
 mean_error = 1 - mean_acc
print(hparams)
 print(f'Mean error: {mean_error}')
 print('----------------------------------------')
 return mean_error<define_search_model>
reg = LassoCV()
reg.fit(X, y)
 print("Best alpha using built-in LassoCV: %f" % reg.alpha_)
 print("Best score using built-in LassoCV: %f" %reg.score(X,y))
coef = pd.Series(reg.coef_, index = X.columns)
 print("Lasso picked " + str(sum(coef != 0)) + " variables and eliminated the other " + str(sum(coef == 0)) + " variables" )
trials_xgb = ho.Trials()<define_variables>
models = [('DTR', DecisionTreeRegressor()), 
('RFR', RandomForestRegressor()), 
('KNR', KNeighborsRegressor()), 
('GBR', GradientBoostingRegressor()), 
('LR', LinearRegression()), 
('XGB', XGBRegressor()), 
('LGBM', LGBMRegressor()), 
('SVR', SVR()), 
('Ridge', Ridge(alpha=10)) , 
('Lasso', Lasso(alpha=0.003487)) , 
('ENet', ElasticNet(alpha=0.006974)) ]
max_evals_xgb = 50<define_search_space>
results = []
names = []
for coef in [ridge_coef.index, enet_coef.index, lasso_coef.index]:
for name, model in models:
kfold = KFold(n_splits=10, random_state=21)
 cv_results = cross_val_score(model, X[coef], y, cv=kfold, scoring='neg_mean_squared_error')
 results.append(cv_results)
 names.append(name)
 msg = "%s: %f(%f)" %(name, cv_results.mean() , cv_results.std())
 print(msg )
learning_rate_xgb_arr = [0.1, 0.05, 0.0025, 0.01, 0.005, 0.0025, 0.001, 0.0005, 0.00025, 0.0001]
max_depth_xgb_arr = [2, 3, 4, 5, 6]<find_best_model_class>
params = {
'alpha': [0.001]
}
reg = Lasso()
rs = GridSearchCV(estimator = reg, param_grid = params,
cv = 10, verbose= 5, n_jobs = -1, scoring='neg_mean_squared_error')
 rs.fit(X[lasso_coef.index],y)
 print(rs.best_score_)
 print(rs.best_estimator_)
 lasso = rs.best_estimator_
space_xgb = {
'learning_rate': ho.hp.choice('learning_rate', learning_rate_xgb_arr),
'max_depth': ho.hp.choice('max_depth', max_depth_xgb_arr),
'n_estimators': ho.hp.quniform('n_estimators', 50, 2000, 5),
}
ho.fmin(fn=partial(cross_validate_xgb_hparams, x=X_stack_train, y=y_stack_train), space=space_xgb, algo=ho.tpe.suggest, max_evals=max_evals_xgb, trials=trials_xgb )<define_variables>
params = {
'learning_rate': [0.01],
'n_estimators': [2000],
'max_depth': [11],
'min_samples_split': [200],
'min_samples_leaf': [10],
'max_features': ['sqrt'],
'subsample': [0.85]
}
reg = GradientBoostingRegressor()
rs = GridSearchCV(estimator = reg, param_grid = params,
cv = 10, verbose= 5, n_jobs = -1, scoring='neg_mean_squared_error')
 rs.fit(X[lasso_coef.index],y)
 print(rs.best_score_)
 print(rs.best_estimator_)
 gbr = rs.best_estimator_
max_evals_xgb = 100<compute_train_metric>
params = {
'alpha': [0.001]
}
reg = ElasticNet()
rs = GridSearchCV(estimator = reg, param_grid = params,
cv = 10, verbose= 5, n_jobs = -1, scoring='neg_mean_squared_error')
 rs.fit(X[lasso_coef.index],y)
 print(rs.best_score_)
 print(rs.best_estimator_)
 elasticnet = rs.best_estimator_
best_xgb = ho.fmin(fn=partial(cross_validate_xgb_hparams, x=X_stack_train, y=y_stack_train), space=space_xgb, algo=ho.anneal.suggest, max_evals=max_evals_xgb, trials=trials_xgb )<find_best_params>
params = {
'learning_rate': [0.01],
'n_estimators': [3000],
'max_depth': [3],
'min_child_weight': [5],
'gamma': [0],
'colsample_bytree': [0.65],
'subsample': [0.6],
'reg_alpha':[1e-6]
}
reg = XGBRegressor()
rs = GridSearchCV(estimator = reg, param_grid = params,
cv = 10, verbose= 5, n_jobs = -1, scoring='neg_mean_squared_error')
 rs.fit(X[lasso_coef.index],y)
 print(rs.best_score_)
 print(rs.best_estimator_)
 xgboost = rs.best_estimator_
learning_rate_xgb = learning_rate_xgb_arr[best_xgb['learning_rate']]
max_depth_xgb = max_depth_xgb_arr[best_xgb['max_depth']]
n_estimators_xgb = int(best_xgb['n_estimators'] )<find_best_params>
params = {
'learning_rate': [0.01],
'n_estimators': [3000],
'max_depth': [3],
'min_child_weight': [1],
'gamma': [0],
'colsample_bytree': [0.8],
'subsample': [0.6],
}
reg = LGBMRegressor()
rs = GridSearchCV(estimator = reg, param_grid = params,
cv = 10, verbose= 5, n_jobs = -1, scoring='neg_mean_squared_error')
 rs.fit(X[lasso_coef.index],y)
 print(rs.best_score_)
 print(rs.best_estimator_ )
learning_rate_xgb, max_depth_xgb, n_estimators_xgb<train_model>
level_0 = [('ENet',elasticnet),('GBR', gbr)]
level_1 = lasso
model = StackingRegressor(estimators=level_0, final_estimator=level_1, cv=10)
 cv = KFold(n_splits=10, random_state=21)
 scores = cross_val_score(model, X[lasso_coef.index],y,cv=cv,scoring='neg_mean_absolute_error')
 print(scores.mean() )
metalearner = xgboost.XGBClassifier(
 learning_rate=learning_rate_xgb,
max_depth=max_depth_xgb,
n_estimators=n_estimators_xgb 
)
 metalearner.fit(X_stack_train, y_stack_train )<predict_on_test>
model.fit(X[lasso_coef.index],y )
val_ens_predictions = get_predicitons_with_ensemble(val.text.values, ensemble_of_classifiers, val.target.values )<predict_on_test>
pred_stacked = model.predict(scaled_test[lasso_coef.index])
 pred = np.expm1(pred_stacked)
 sub = test[['Id']]
sub['SalePrice'] = pred
sub[['Id', 'SalePrice']].to_csv('pred_submission.csv', index=False, encoding='utf-8' )