Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<find_best_params>
def ignore_warn(*args, **kwargs):
pass
warnings.warn = ignore_warn
lasso = make_pipeline(RobustScaler() , Lasso(alpha = 0.00042, max_iter=100000, random_state=1))
score = rmsle_cv(lasso)
 print('Lasso Score = {:.4f}(std = {:.4f})'.format(score.mean() , score.std()))
models.loc[len(models)] = ['lasso',lasso,score.mean() ,score.std() ]<compute_train_metric>
X_train = pd.read_csv('.. /input/train.csv')
 X_test = pd.read_csv('.. /input/test.csv' )
e_net = make_pipeline(RobustScaler() , ElasticNet(alpha = 0.00045, l1_ratio=0.9, random_state=1))
score = rmsle_cv(e_net)
 print('Elastic Net score = {:.4f}(std = {:.4f})'.format(score.mean() , score.std()))
models.loc[len(models)] = ['e_net',e_net,score.mean() ,score.std() ]<find_best_params>
X_train.drop(X_train[(X_train['GrLivArea']>4000)&(X_train['SalePrice']<300000)].index, inplace=True )
kr = make_pipeline(RobustScaler() , KernelRidge(alpha=0.04, kernel='polynomial', degree=1, coef0=2.5))
score = rmsle_cv(kr)
 print('Kernel Ridge score = {:.4f}(std = {:.4f})'.format(score.mean() , score.std()))
models.loc[len(models)] = ['kr',kr,score.mean() ,score.std() ]<compute_train_metric>
y = X_train['SalePrice']
test_id = X_test['Id']
X_train.drop('SalePrice', 1, inplace=True)
 X_train.drop("Id", axis = 1, inplace = True)
 X_test.drop("Id", axis = 1, inplace = True)
 X_train_idx = X_train.shape[0]
X_test_idx = X_test.shape[0]
data = pd.concat(( X_train,X_test)).reset_index(drop=True)
 print('Train index: %s
Test index: %s
Shape of our whole data(rows,columns): %s'%(X_train_idx, X_test_idx, data.shape))
dtr = make_pipeline(RobustScaler() , DecisionTreeRegressor(random_state=0, max_depth=20))
score = rmsle_cv(dtr)
 print('Decision Tree score = {:.4f}(std = {:.4f})'.format(score.mean() , score.std()))
models.loc[len(models)] = ['dtr',dtr,score.mean() ,score.std() ]<compute_train_metric>
numeric_cols = []
non_numeric_cols = []
for key,val in data.isnull().sum().sort_values(ascending=False ).items() :
if val > 0:
print(key, val, data[key].dtype)
 if data[key].dtype != 'object':numeric_cols.append(key)
 elif data[key].dtype == 'object':non_numeric_cols.append(key)
gbr = GradientBoostingRegressor(n_estimators=3000,
learning_rate=0.05, max_depth=4, max_features='sqrt',
min_samples_leaf=1, min_samples_split=2, loss='huber',
random_state=5,)
 score = rmsle_cv(gbr)
 print('Gradient Boosting score = {:.4f}(std = {:.4f})'.format(score.mean() , score.std()))
models.loc[len(models)] = ['gbr',gbr,score.mean() ,score.std() ]<compute_train_metric>
fill_zero_cols = ['BsmtHalfBath', 'BsmtFullBath', 'BsmtFinSF2', 'GarageCars']
fill_median_cols = ['GarageArea','TotalBsmtSF', 'MasVnrArea', 'BsmtFinSF1', 'LotFrontage', 'BsmtUnfSF', 'GarageYrBlt']
[data[i].fillna(0,inplace=True)for i in fill_zero_cols]
[data[j].fillna(data[j].median() ,inplace=True)for j in fill_median_cols];
lgbr = lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720, max_bin = 55,
bagging_fraction = 0.8, bagging_freq = 5,
feature_fraction = 0.2319, feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11)
 score = rmsle_cv(lgbr)
 print('LightGBM score = {:.4f}(std = {:.4f})'.format(score.mean() , score.std()))
models.loc[len(models)] = ['lgbr',lgbr,score.mean() ,score.std() ]<compute_train_metric>
data.fillna('none', inplace=True )
xgbr = xgb.XGBRegressor(colsample_bytree=0.4603, gamma=0.0468,
learning_rate=0.05, max_depth=3, min_child_weight=1.7817,
n_estimators=2200, reg_alpha=0.4640, reg_lambda=0.8571,
subsample=0.5213, silent=True, random_state =7, nthread = -1)
 score = rmsle_cv(xgbr)
 print('XGBoost score = {:.4f}(std = {:.4f})'.format(score.mean() , score.std()))
models.loc[len(models)] = ['xgbr',xgbr,score.mean() ,score.std() ]<sort_values>
data.isnull().sum().sort_values(ascending=False )
models.sort_values(by='score_mean',inplace=True)
 models.reset_index(inplace=True,drop=True)
 models<train_model>
y = np.log1p(y)
 numeric_feats = data.dtypes[data.dtypes != "object"].index
skewed_feats = X_train[numeric_feats].apply(lambda x: skew(x.dropna()))
skewed_feats = skewed_feats[skewed_feats > 0.75]
skewed_feats = skewed_feats.index
data[skewed_feats] = np.log1p(data[skewed_feats] )
class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models):
self.models = models
def fit(self, X, y):
self.models_ = [clone(x)for x in self.models]
for model in self.models_:
model.fit(X, y)
 return self
def predict(self, X):
predictions = np.column_stack([
model.predict(X)for model in self.models_
])
 return np.mean(predictions, axis=1 )<count_unique_values>
data = pd.get_dummies(data )
def subset(lst, count):
return list(set(combinations(lst, count)))
 model_list = list(models[models['score_mean']<0.11]['model_name'])
 combo = list()
for i in range(1,len(model_list)) :
combo = combo + subset(model_list, i)
 print('There are {:.0f} combinations.First 20 include:'.format(len(combo)))
 combo[:20]<compute_train_metric>
X_train = data[:X_train_idx]
X_test = data[X_train_idx:]
simple_avg_final = AveragingModels(models =(lasso, gbr, lgbr, kr))
score = rmsle_cv(simple_avg_final)
 print('Simple Average score = {:.4f}(std = {:.4f})'.format(score.mean() , score.std()))<compute_train_metric>
from sklearn.linear_model import Lasso, ElasticNet, Ridge, LinearRegression
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import PolynomialFeatures
from sklearn.model_selection import cross_val_score
from sklearn.preprocessing import StandardScaler
stacked = StackingCVRegressor(regressors=(lasso, gbr, lgbr, kr),
meta_regressor=lasso,
use_features_in_secondary=True)
 score = rmsle_cv(stacked)
 print('Stacked score = {:.8f}(std = {:.4f})'.format(score.mean() , score.std()))<train_model>
elastic_net = ElasticNet(alpha=.5, l1_ratio=.5)
 scores = cross_val_score(elastic_net, X_train, y, scoring='neg_mean_squared_error', cv=5)
 rmse_scores = np.sqrt(-scores)
 display_scores(rmse_scores )
class StackingCVRegressor_Scratch(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, base_models, meta_model, n_folds=5):
self.base_models = base_models
self.meta_model = meta_model
self.n_folds = n_folds
def fit(self, X, y):
self.base_models_ = [list() for x in self.base_models]
self.meta_model_ = clone(self.meta_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=42)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(X, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 instance.fit(X[train_index], y[train_index])
 y_pred = instance.predict(X[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
self.meta_model_.fit(out_of_fold_predictions, y)
 return self
def predict(self, X):
meta_features = np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_])
 return self.meta_model_.predict(meta_features )<compute_train_metric>
param_grid = [
{'alpha': [.0004,.0005], 'l1_ratio': [.5,.8, 1]},
]
grid_search = GridSearchCV(ElasticNet() , param_grid, cv=5,
scoring='neg_mean_squared_error',
return_train_score=True )
stacked_scratch = StackingCVRegressor_Scratch(base_models =(lasso, gbr, lgbr, kr),
meta_model = lasso)
 score = rmsle_cv(stacked_scratch)
 print('Stacked(Scratch)score = {:.8f}(std = {:.4f})'.format(score.mean() , score.std()))<compute_train_metric>
grid_search.fit(X_train, y )
stacked_final = StackingCVRegressor(regressors=(svr, ridg, xgbr),
meta_regressor=e_net,
use_features_in_secondary=True)
 score = rmsle_cv(stacked_final)
 print('stacked_final score = {:.8f}(std = {:.4f})'.format(score.mean() , score.std()))<predict_on_test>
cvres = grid_search.cv_results_
for mean_score, params in zip(cvres["mean_test_score"], cvres["params"]):
print(np.sqrt(-mean_score), params )
model_1 = simple_avg_final
model_2 = stacked_final
mod_1_share =.5
mod_2_share =.5
model_1.fit(train.values, Y_train)
 model_1_test_predictions = np.expm1(model_1.predict(test.values))
model_2.fit(train.values, Y_train)
 model_2_test_predictions = np.expm1(model_2.predict(test.values))
test_predictions = mod_1_share * model_1_test_predictions + mod_2_share * model_2_test_predictions<save_to_csv>
lasso = Lasso(alpha=.1 ).fit(X_train,y)
 scores = cross_val_score(lasso,X_train,y,scoring='neg_mean_squared_error',cv=5)
 rmse_scores = np.sqrt(-scores)
 display_scores(rmse_scores )
test_id = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')[['Id']]
test_id['SalePrice'] = np.round(test_predictions,2)
 test_id.to_csv('out51_simple(lasso,gbr,lgbr,kr)_meta(e_net,svr,ridg,xgbr ).csv',index=False )<load_from_csv>
lasso = Lasso(alpha=.0005 ).fit(X_train,y)
 scores = cross_val_score(lasso,X_train,y,scoring='neg_mean_squared_error',cv=5)
 rmse_scores = np.sqrt(-scores)
 display_scores(rmse_scores )
<create_dataframe><EOS> 
preds = lasso.predict(X_test)
 lasso_predictions = pd.DataFrame(np.exp(preds), test_id, columns=['SalePrice'] ).to_csv('lasso_predictions.csv' )