Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<define_search_model>
warnings.filterwarnings('ignore')
 train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv')
 print("Skewness: %f" % train_df['SalePrice'].skew())
 print("Kurtosis: %f" % train_df['SalePrice'].kurt())
 train_ID = train_df['Id']
test_ID = test_df['Id']
train_df.drop("Id", axis = 1, inplace = True)
 test_df.drop("Id", axis = 1, inplace = True)
 train_df.head()
base_models = list()
base_models.append(( 'ridge', ridge_model))
base_models.append(( 'lasso', lasso_model))
base_models.append(( 'rf', rf_model_en))
learner = LinearRegression()
stack1 = StackingRegressor(estimators=base_models, final_estimator=learner, cv=10)
 stack1.fit(X, y )<train_model>
total = full_data.isnull().sum().sort_values(ascending=False)
 percent =(full_data.isnull().sum() /full_data.isnull().count() ).sort_values(ascending=False)
 missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 missing_data.head(20)
 full_data["LotFrontage"] = full_data.groupby("Neighborhood")["LotFrontage"].transform(lambda x: x.fillna(x.median()))
full_data.fillna(full_data.mean() , inplace=True)
 full_data.fillna(full_data.mode().iloc[0], inplace=True)
 full_data.isnull().sum().max()
base_model = list()
base_model.append(( 'rf1', rf_model))
base_model.append(( 'rf2', rf_model_en))
base_model.append(( 'rf3', RandomForestRegressor(max_depth=8, max_features=0.1, min_samples_leaf=3,
min_samples_split=2, n_estimators=250, n_jobs=-1, oob_score=False)))
 learner = LinearRegression()
stack2 = StackingRegressor(estimators=base_model, final_estimator=learner, cv=10)
 stack2.fit(X, y )<save_to_csv>
submission = pd.DataFrame(columns=['Id','Prediction'])
 submission['Id'] = test_df['Id']
ridge_pred = ridge_model.predict(test_df.drop('Id', axis=1))
submission['Prediction'] = np.expm1(ridge_pred)
 submission.to_csv('submission_ridge.csv',index=False)
 lasso_pred = lasso_model.predict(test_df.drop('Id', axis=1))
submission['Prediction'] = np.expm1(lasso_pred)
 submission.to_csv('submission_lasso.csv',index=False)
 elastic_pred = el_model.predict(test_df.drop('Id', axis=1))
submission['Prediction'] = np.expm1(elastic_pred)
 submission.to_csv('submission_elastic.csv',index=False)
 knn_pred = knn_model.predict(test_df.drop('Id', axis=1))
submission['Prediction'] = np.expm1(knn_pred)
 submission.to_csv('submission_knn.csv',index=False)
 rf_pred = rf_model.predict(test_df.drop('Id', axis=1))
submission['Prediction'] = np.expm1(rf_pred)
 submission.to_csv('submission_rf.csv',index=False)
 lgb_pred = lgb_model.predict(test_df.drop('Id', axis=1))
submission['Prediction'] = np.expm1(lgb_pred)
 submission.to_csv('submission_lgb.csv',index=False)
 xgb_pred = xgb_model.predict(test_df.drop('Id', axis=1))
submission['Prediction'] = np.expm1(xgb_pred)
 submission.to_csv('submission_xgb.csv',index=False)
 stack_pred1 = stack1.predict(test_df.drop('Id', axis=1))
submission['Prediction'] = np.expm1(stack_pred1)
 submission.to_csv('submission_stack1.csv',index=False)
 stack_pred2 = stack2.predict(test_df.drop('Id', axis=1))
submission['Prediction'] = np.expm1(stack_pred2)
 submission.to_csv('submission_stack2.csv',index=False )<import_modules>
train_df['SalePrice'] = np.log1p(train_df['SalePrice'])
 numeric_feats = full_data.dtypes[full_data.dtypes != "object"].index
skewed_feats = full_data[numeric_feats].apply(lambda x: skew(x.dropna()))
skewed_feats = skewed_feats[skewed_feats > 0.75]
skew_index = skewed_feats.index
print(skew_index)
 for i in skew_index:
full_data[i] = boxcox1p(full_data[i], boxcox_normmax(full_data[i] + 1))
import pandas as pd
import numpy as np 
<load_from_csv>
full_data = pd.get_dummies(full_data)
 full_data.head()
data = pd.read_csv('/kaggle/input/restaurant-revenue-prediction/train.csv.zip')
 test_data = pd.read_csv('/kaggle/input/restaurant-revenue-prediction/test.csv.zip')
 data.head() 
<feature_engineering>
X_train = full_data[:ntrain]
X_test = full_data[ntrain:]
y = train_df.SalePrice.values
kf = KFold(n_splits=12, random_state=42, shuffle=True)
 def rmse_cv(model):
rmse= np.sqrt(-cross_val_score(model, X_train.values, y, scoring="neg_mean_squared_error", cv = kf))
return(rmse)
 model_ridge = Ridge()
alphas = [0.05, 0.1, 0.3, 1, 3, 5, 10, 15, 30, 50, 75]
cv_ridge = [rmse_cv(Ridge(alpha = alpha)).mean() for alpha in alphas]
cv_ridge = pd.Series(cv_ridge, index = alphas)
 cv_ridge.plot(title = "Validation")
 plt.xlabel("alpha")
 plt.ylabel("rmse")
 print(cv_ridge.min())
 model_lasso = LassoCV(alphas = [1, 0.1, 0.001, 0.0005] ).fit(X_train, y)
 print(rmse_cv(model_lasso ).mean())
 matplotlib.rcParams['figure.figsize'] =(6.0, 6.0)
 preds = pd.DataFrame({"preds":model_lasso.predict(X_train), "true":y})
 preds["residuals"] = preds["true"] - preds["preds"]
preds.plot(x = "preds", y = "residuals",kind = "scatter")
 dtrain = xgb.DMatrix(X_train, label = y)
 params = {"max_depth":2, "eta":0.1}
model = xgb.cv(params, dtrain, num_boost_round=500, early_stopping_rounds=100)
 model.loc[30:,["test-rmse-mean", "train-rmse-mean"]].plot()
data['Type_IL'] = np.where(data['Type'] == 'IL', 1, 0)
 data['Type_FC'] = np.where(data['Type'] == 'FC', 1, 0)
 data['Type_DT'] = np.where(data['Type'] == 'DT', 1, 0)
 data['Big_Cities'] = np.where(data['City Group'] == 'Big Cities', 1, 0)
 data['Days_Open'] =(pd.to_datetime('2015-03-23')- pd.to_datetime(data['Open Date'])).dt.days
data = data.drop('Type', axis=1)
 data = data.drop('City Group', axis=1)
 data = data.drop('City', axis=1)
 data = data.drop('Open Date', axis=1)
 test_data['Type_IL'] = np.where(test_data['Type'] == 'IL', 1, 0)
 test_data['Type_FC'] = np.where(test_data['Type'] == 'FC', 1, 0)
 test_data['Type_DT'] = np.where(test_data['Type'] == 'DT', 1, 0)
 test_data['Big_Cities'] = np.where(test_data['City Group'] == 'Big Cities', 1, 0)
 test_data['Days_Open'] =(pd.to_datetime('2015-03-23')- pd.to_datetime(test_data['Open Date'])).dt.days
test_data = test_data.drop('Type', axis=1)
 test_data = test_data.drop('City Group', axis=1)
 test_data = test_data.drop('City', axis=1)
 test_data = test_data.drop('Open Date', axis=1 )<prepare_x_and_y>
lightgbm = LGBMRegressor(objective='regression',
num_leaves=6,
learning_rate=0.01,
n_estimators=7000,
max_bin=200,
bagging_fraction=0.8,
bagging_freq=4,
bagging_seed=8,
feature_fraction=0.2,
feature_fraction_seed=8,
min_sum_hessian_in_leaf = 11,
verbose=-1,
random_state=42)
 xgboost = XGBRegressor(learning_rate=0.01,
n_estimators=6000,
max_depth=4,
min_child_weight=0,
gamma=0.6,
subsample=0.7,
colsample_bytree=0.7,
objective='reg:linear',
nthread=-1,
scale_pos_weight=1,
seed=27,
reg_alpha=0.00006,
random_state=42)
 ridge_alphas = [1e-15, 1e-10, 1e-8, 9e-4, 7e-4, 5e-4, 3e-4, 1e-4, 1e-3, 5e-2, 1e-2, 0.1, 0.3, 1, 3, 5, 10, 15, 18, 20, 30, 50, 75, 100]
ridge = make_pipeline(RobustScaler() , RidgeCV(alphas=ridge_alphas, cv=kf))
svr = make_pipeline(RobustScaler() , SVR(C= 20, epsilon= 0.008, gamma=0.0003))
gbr = GradientBoostingRegressor(n_estimators=6000,
learning_rate=0.01,
max_depth=4,
max_features='sqrt',
min_samples_leaf=15,
min_samples_split=10,
loss='huber',
random_state=42)
 rf = RandomForestRegressor(n_estimators=1200,
max_depth=15,
min_samples_split=5,
min_samples_leaf=5,
max_features=None,
oob_score=True,
random_state=42)
 stack_gen = StackingCVRegressor(regressors=(xgboost, lightgbm, svr, ridge, gbr, rf),
meta_regressor=xgboost,
use_features_in_secondary=True)
 scores = {}
score = rmse_cv(lightgbm)
 print("lightgbm: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['lgb'] =(score.mean() , score.std())
 score = rmse_cv(xgboost)
 print("xgboost: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['xgb'] =(score.mean() , score.std())
 score = rmse_cv(svr)
 print("SVR: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['svr'] =(score.mean() , score.std())
 score = rmse_cv(ridge)
 print("ridge: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['ridge'] =(score.mean() , score.std())
 score = rmse_cv(rf)
 print("rf: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['rf'] =(score.mean() , score.std())
 score = rmse_cv(gbr)
 print("gbr: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['gbr'] =(score.mean() , score.std())
 sns.set_style("white")
 fig = plt.figure(figsize=(24, 12))
ax = sns.pointplot(x=list(scores.keys()), y=[score for score, _ in scores.values() ], markers=['o'], linestyles=['-'])
 for i, score in enumerate(scores.values()):
ax.text(i, score[0] + 0.002, '{:.6f}'.format(score[0]), horizontalalignment='left', size='large', color='black', weight='semibold')
 plt.ylabel('Score(RMSE)', size=20, labelpad=12.5)
 plt.xlabel('Model', size=20, labelpad=12.5)
 plt.tick_params(axis='x', labelsize=13.5)
 plt.tick_params(axis='y', labelsize=12.5)
 plt.title('Scores of Models', size=20)
 plt.show()
X = data.drop(['Id', 'revenue'], axis=1)
 Y = data.revenue 
<not_enough_vertices>
class StackingAveragedModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, base_models, meta_model, n_folds=5):
self.base_models = base_models
self.meta_model = meta_model
self.n_folds = n_folds
def fit(self, X, y):
self.base_models_ = [list() for x in self.base_models]
self.meta_model_ = clone(self.meta_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=156)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(X, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 instance.fit(X[train_index], y[train_index])
 y_pred = instance.predict(X[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
self.meta_model_.fit(out_of_fold_predictions, y)
 return self
def predict(self, X):
meta_features = np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_ ])
 return self.meta_model_.predict(meta_features)
def check_rmse(X, Y, alpha):
RMSE_lasso = []
RMSE_ridge = []
for i in alpha:
lasso = Lasso(alpha=i)
 lasso.fit(X, Y)
 ridge = Ridge(alpha=i)
 ridge.fit(X, Y)
 RMSE_lasso.append(metrics.mean_squared_error(Y, lasso.predict(X)))
 RMSE_ridge.append(metrics.mean_squared_error(Y, ridge.predict(X)))
 return(RMSE_lasso, RMSE_ridge)
<train_model>
def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
print('stack_gen')
 stack_gen_model = stack_gen.fit(np.array(X_train.values), np.array(y))
print('lightgbm')
 lgb_model_full_data = lightgbm.fit(X_train, y)
 print('xgboost')
 xgb_model_full_data = xgboost.fit(X_train, y)
 print('Svr')
 svr_model_full_data = svr.fit(X_train, y)
 print('Ridge')
 ridge_model_full_data = ridge.fit(X_train, y)
 print('RandomForest')
 rf_model_full_data = rf.fit(X_train, y)
 print('GradientBoosting')
 gbr_model_full_data = gbr.fit(X_train, y)
 def blended_predictions(X):
return(( 0.1 * ridge_model_full_data.predict(X)) + \ 
(0.2 * svr_model_full_data.predict(X)) + \ 
(0.1 * gbr_model_full_data.predict(X)) + \ 
(0.1 * xgb_model_full_data.predict(X)) + \ 
(0.1 * lgb_model_full_data.predict(X)) + \ 
(0.05 * rf_model_full_data.predict(X)) + \ 
(0.35 * stack_gen_model.predict(np.array(X))))
blended_score = rmsle(y, blended_predictions(X_train))
scores['blended'] =(blended_score, 0)
 print('RMSLE score on train data:')
 print(blended_score )
lasso = Lasso(alpha=5.5)
 lasso.fit(X, Y)
 metrics.mean_squared_error(Y, lasso.predict(X))<save_to_csv>
predictions = blended_predictions(X_test)
 submission = pd.DataFrame()
submission['Id'] = test_ID
submission['SalePrice'] = np.floor(np.expm1(predictions))
submission.to_csv('submission.csv',index=False )