Kaggle competition: House Prices - Advanced Regression Techniques
OUTPUT_CHANNELS = 3
def downsample(filters, size, apply_instancenorm=True):
initializer = tf.random_normal_initializer(0., 0.02)
 gamma_init = keras.initializers.RandomNormal(mean=0.0, stddev=0.02)
 result = keras.Sequential()
result.add(layers.Conv2D(filters, size, strides=2, padding='same',
kernel_initializer=initializer, use_bias=False))
if apply_instancenorm:
result.add(tfa.layers.InstanceNormalization(gamma_initializer=gamma_init))
result.add(layers.LeakyReLU())
 return result<define_search_model>
train = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv', index_col='Id')
 test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv', index_col='Id')
 train.head()
def upsample(filters, size, apply_dropout=False):
initializer = tf.random_normal_initializer(0., 0.02)
 gamma_init = keras.initializers.RandomNormal(mean=0.0, stddev=0.02)
 result = keras.Sequential()
result.add(layers.Conv2DTranspose(filters, size, strides=2,
padding='same',
kernel_initializer=initializer,
use_bias=False))
result.add(tfa.layers.InstanceNormalization(gamma_initializer=gamma_init))
if apply_dropout:
result.add(layers.Dropout(0.5))
result.add(layers.ReLU())
 return result<define_search_space>
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=1)
 print('==Testing model R^2(higher is better)==')
 linr = LinearRegression() ; cv = cross_val_score(linr, X_train, y_train, cv=5, scoring='r2'); print("LinReg 5-fold CV score: {:.2f}".format(np.mean(cv)))
 elas = ElasticNet(alpha=0.01); cv = cross_val_score(elas, X_train, y_train, cv=5); print("ElasticNet 5-fold CV score: {:.3f}".format(np.mean(cv)))
 ridge = Ridge() ; cv = cross_val_score(ridge, X_train, y_train, cv=5); print("Ridge 5-fold CV score: {:.3f}".format(np.mean(cv)))
 br = BayesianRidge() ; cv = cross_val_score(br, X_train, y_train, cv=5); print("Bayesian Ridge 5-fold CV score: {:.3f}".format(np.mean(cv)))
 lasso = Lasso(alpha=0.001); cv = cross_val_score(lasso, X_train, y_train, cv=5); print("Lasso 5-fold CV score: {:.3f}".format(np.mean(cv))); lasso_coef = lasso.fit(X_train, y_train ).coef_
knn = KNeighborsRegressor() ; cv = cross_val_score(knn, X_train, y_train, cv=5); print("knn 5-fold CV score: {:.3f}".format(np.mean(cv)))
 svr = SVR() ; cv = cross_val_score(svr, X_train, y_train, cv=5); print("SVMReg 5-fold CV score: {:.3f}".format(np.mean(cv)))
 dtree = DecisionTreeRegressor() ; cv = cross_val_score(dtree, X_train, y_train, cv=5); print("Dec.tree 5-fold CV score: {:.3f}".format(np.mean(cv)))
 rftree = RandomForestRegressor() ; cv = cross_val_score(rftree, X_train, y_train, cv=5); print("RF 5-fold CV score: {:.3f}".format(np.mean(cv)))
 ada = AdaBoostRegressor() ; cv = cross_val_score(ada, X_train, y_train, cv=5); print("Adaboosted 5-fold CV score: {:.3f}".format(np.mean(cv)))
 grad = GradientBoostingRegressor() ; cv = cross_val_score(grad, X_train, y_train, cv=5); print("Grad.boosted 5-fold CV score: {:.3f}".format(np.mean(cv)))
 estimators = [('LinReg',linr),('Ridge',ridge),('Lasso',lasso),('BayesianRidge', br),('SVMREg', svr),('RandomForest',rftree),('Gradboosted',grad)]
voting = VotingRegressor(estimators=estimators, n_jobs=-1); cv = cross_val_score(voting, X_train, y_train, cv=5); print("...all combined?! {:.3f}".format(np.mean(cv)))
 _ = plt.figure(figsize=(30,10)) ; _ = plt.plot(range(len(train.columns)) ,lasso_coef[0:80]); _ = plt.xticks(range(len(train.columns)) , train.columns, rotation=60); _ = plt.title('Most important predictors')
 print('
==Testing some of the basic models on data(lower RMSE indicated better fit)==');
grad.fit(X_train, y_train); y_pred = grad.predict(X_test); rmse_grad = np.sqrt(mean_squared_error(y_test, y_pred))
print('RMSE of grad.boosted regressor on test data: {:.3f}'.format(rmse_grad))
voting.fit(X_train, y_train); y_pred = voting.predict(X_test); rmse_vot = np.sqrt(mean_squared_error(y_test, y_pred))
print('RMSE of ensemble voting on test data: {:.3f}'.format(rmse_vot))
xg_reg = xgb.XGBRegressor(objective='reg:squarederror', n_estimators=10); xg_reg.fit(X_train, y_train); pred = xg_reg.predict(X_test); print('RMSE of XGBoost regressor: {:.3f}'.format(np.sqrt(mean_squared_error(y_test,pred))))
DM_train = xgb.DMatrix(data=X_train, label=y_train); DM_test = xgb.DMatrix(data=X_test, label=y_test); params = {'booster':'gblinear','objective':'reg:squarederror'}
cv_results = xgb.cv(dtrain=DM_train, params=params, nfold=5, num_boost_round=50, early_stopping_rounds=10, metrics="rmse", as_pandas=True)
 ',cv_results)
 xg_reg = xgb.train(params=params, dtrain=DM_train, num_boost_round=10); preds = xg_reg.predict(DM_test); print('RMSE of XGBoosted base learner: {:.3f}'.format(np.sqrt(mean_squared_error(y_test,preds))))
def Generator() :
inputs = layers.Input(shape=[256,256,3])
 down_stack = [
downsample(64, 4, apply_instancenorm=False),
downsample(128, 4),
downsample(256, 4),
downsample(512, 4),
downsample(512, 4),
downsample(512, 4),
downsample(512, 4),
downsample(512, 4),
]
up_stack = [
upsample(512, 4, apply_dropout=True),
upsample(512, 4, apply_dropout=True),
upsample(512, 4, apply_dropout=True),
upsample(512, 4),
upsample(256, 4),
upsample(128, 4),
upsample(64, 4),
]
initializer = tf.random_normal_initializer(0., 0.02)
 last = layers.Conv2DTranspose(OUTPUT_CHANNELS, 4,
strides=2,
padding='same',
kernel_initializer=initializer,
activation='tanh')
 x = inputs
skips = []
for down in down_stack:
x = down(x)
 skips.append(x)
 skips = reversed(skips[:-1])
 for up, skip in zip(up_stack, skips):
x = up(x)
 x = layers.Concatenate()([x, skip])
 x = last(x)
 return keras.Model(inputs=inputs, outputs=x )<define_search_model>
grad.fit(X, y)
 predict = np.expm1(grad.predict(test))
sub = pd.DataFrame({'Id':test.index, 'SalePrice':predict})
 sub.head()
def Discriminator() :
initializer = tf.random_normal_initializer(0., 0.02)
 gamma_init = keras.initializers.RandomNormal(mean=0.0, stddev=0.02)
 inp = layers.Input(shape=[256, 256, 3], name='input_image')
 x = inp
down1 = downsample(64, 4, False )(x)
 down2 = downsample(128, 4 )(down1)
 down3 = downsample(256, 4 )(down2)
 zero_pad1 = layers.ZeroPadding2D()(down3)
 conv = layers.Conv2D(512, 4, strides=1,
kernel_initializer=initializer,
use_bias=False )(zero_pad1)
 norm1 = tfa.layers.InstanceNormalization(gamma_initializer=gamma_init )(conv)
 leaky_relu = layers.LeakyReLU()(norm1)
 zero_pad2 = layers.ZeroPadding2D()(leaky_relu)
 last = layers.Conv2D(1, 4, strides=1,
kernel_initializer=initializer )(zero_pad2)
 return tf.keras.Model(inputs=inp, outputs=last)
<define_search_model>
params = {'alpha': [0.0001,0.001,0.01,0.1,1,5,10]}; lasso = Lasso()
lasso_mse = GridSearchCV(estimator=lasso, param_grid=params, scoring='neg_mean_squared_error', cv=4, n_jobs=-1)
 lasso_mse.fit(X,y); print("Best Lasso paramaters found:",lasso_mse.best_params_);print("Lowest RMSE found:", np.sqrt(np.abs(lasso_mse.best_score_)))
 params = {'alpha_1': [0.0000001,0.000001,0.00001,0.0001],
'alpha_2': [0.0000001,0.000001,0.00001,0.0001],
'lambda_1': [0.0000001,0.000001,0.00001,0.0001],
'lambda_2': [0.0000001,0.000001,0.00001,0.0001]}; br = BayesianRidge()
br_mse = GridSearchCV(estimator=br, param_grid=params, scoring='neg_mean_squared_error', cv=4, n_jobs=-1)
 br_mse.fit(X,y); print("Best Bayesian Ridge paramaters found:",br_mse.best_params_);print("Lowest RMSE found:", np.sqrt(np.abs(br_mse.best_score_)))
 gbm_param_grid = {'learning_rate':[0.001,0.01,0.1,0.2,0.3,0.5],
'max_depth': np.arange(3,12,1),
'n_estimators': [50,100,200,300],
'subsample': [0.1,0.2,0.3,0.5,0.75],
'colsample_bytree': np.arange(0.1,0.8,0.1)}
gbm = xgb.XGBRegressor(objective ='reg:squarederror')
 random_mse = RandomizedSearchCV(param_distributions=gbm_param_grid, n_iter=10, scoring='neg_mean_squared_error', cv=5, estimator=gbm, verbose=1)
 random_mse.fit(X,y); print("Best XGB paramaters found:",random_mse.best_params_);print("Lowest RMSE found:", np.sqrt(np.abs(random_mse.best_score_)))
<choose_model_class><EOS> 
predict = np.expm1(br_mse.predict(test))
sub = pd.DataFrame({'Id':test.index, 'SalePrice':predict})
 print(sub.head())
 br = BayesianRidge(alpha_1=0.0001, alpha_2=1e-07, lambda_1=1e-07, lambda_2=0.0001)
 br.fit(X, y)
 predict = np.expm1(br.predict(test))
sub = pd.DataFrame({'Id':test.index, 'SalePrice':predict})
 print(sub.head())
 xgb = xgb.XGBRegressor(learning_rate=0.1, max_depth=3, n_estimators=300, subsample=0.75)
 xgb.fit(X, y)
 predict = np.expm1(xgb.predict(test))
sub = pd.DataFrame({'Id':test.index, 'SalePrice':predict})
 print(sub.head())
 sub.to_csv('./submission.csv', index = False )