Kaggle competition: House Prices - Advanced Regression Techniques
score = rmsle_cv(model_lgb)
 print("LGBM score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<predict_on_test>
train=pd.read_csv("/kaggle/input/train.csv")
 train.head()
class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models):
self.models = models
def fit(self, X, y):
self.models_ = [clone(x)for x in self.models]
for model in self.models_:
model.fit(X, y)
 return self
def predict(self, X):
predictions = np.column_stack([
model.predict(X)for model in self.models_
])
 return np.mean(predictions, axis=1 )<compute_train_metric>
train=train.drop(index=[523,1298],axis=0 )
averaged_models = AveragingModels(models =(ENet, GBoost, KRR, lasso))
score = rmsle_cv(averaged_models)
 print(" Averaged base models score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<train_model>
test=pd.read_csv("/kaggle/input/test.csv" )
class StackingAveragedModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, base_models, meta_model, n_folds=5):
self.base_models = base_models
self.meta_model = meta_model
self.n_folds = n_folds
def fit(self, X, y):
self.base_models_ = [list() for x in self.base_models]
self.meta_model_ = clone(self.meta_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=156)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(X, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 instance.fit(X[train_index], y[train_index])
 y_pred = instance.predict(X[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
self.meta_model_.fit(out_of_fold_predictions, y)
 return self
def predict(self, X):
meta_features = np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_ ])
 return self.meta_model_.predict(meta_features )<compute_train_metric>
data=pd.concat([train.iloc[:,:-1],test],axis=0)
 print('tha data has {} rows and {} features'.format(data.shape[0],data.shape[1]))
stacked_averaged_models = StackingAveragedModels(base_models =(ENet, GBoost, KRR),
meta_model = lasso)
 score = rmsle_cv(stacked_averaged_models)
 print("Stacking Averaged models score: {:.4f}({:.4f})".format(score.mean() , score.std()))<compute_test_metric>
num_features=data.select_dtypes(include=['int64','float64'])
 categorical_features=data.select_dtypes(include='object' )
def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))<predict_on_test>
data.isnull().sum().sort_values(ascending=False)[:34]
stacked_averaged_models.fit(train.values, y_train)
 stacked_train_pred = stacked_averaged_models.predict(train.values)
 stacked_pred = np.expm1(stacked_averaged_models.predict(test.values))
print(rmsle(y_train, stacked_train_pred))<predict_on_test>
f = open("/kaggle/input/data_description.txt", "r")
model_xgb.fit(train, y_train)
 xgb_train_pred = model_xgb.predict(train)
 xgb_pred = np.expm1(model_xgb.predict(test))
print(rmsle(y_train, xgb_train_pred))<predict_on_test>
data = data.drop(columns=['Id','Street','PoolQC','Utilities'],axis=1 )
model_lgb.fit(train, y_train)
 lgb_train_pred = model_lgb.predict(train)
 lgb_pred = np.expm1(model_lgb.predict(test.values))
print(rmsle(y_train, lgb_train_pred))<compute_test_metric>
data['LotFrontage'] = data.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.median()))
print('RMSLE score on train data:')
 print(rmsle(y_train,stacked_train_pred*0.70 +
xgb_train_pred*0.15 + lgb_train_pred*0.15))<define_variables>
data['LotFrontage'].isnull().sum()
ensemble = stacked_pred*0.75 + xgb_pred*0.10 + lgb_pred*0.15<save_to_csv>
features=['Electrical','KitchenQual','SaleType','Exterior2nd','Exterior1st','Alley','Fence', 'MiscFeature','FireplaceQu','GarageCond','GarageQual','GarageFinish','GarageType','BsmtCond','BsmtExposure','BsmtQual','BsmtFinType2','BsmtFinType1','MasVnrType']
for name in features:
data[name].fillna('Other',inplace=True )
sub = pd.DataFrame()
sub['Id'] = test_ID
sub['SalePrice'] = ensemble
sub.to_csv('submission.csv',index=False )<set_options>
data[features].isnull().sum()
warnings.filterwarnings('ignore' )<load_from_csv>
data['MSZoning'] = data.groupby('MSSubClass')['MSZoning'].transform(lambda x: x.fillna(x.mode() [0]))
train_df = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test_df = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv' )<drop_column>
data['Functional']=data['Functional'].fillna('typ' )
train_df.drop(['Id'],axis=1,inplace=True)
 test_df.drop(['Id'],axis=1,inplace=True )<drop_column>
zero=['GarageArea','GarageYrBlt','MasVnrArea','BsmtHalfBath','BsmtHalfBath','BsmtFullBath','BsmtFinSF1','BsmtFinSF2','BsmtUnfSF','TotalBsmtSF','GarageCars']
for name in zero:
data[name].fillna(0,inplace=True )
train_df.drop(train_df[(train_df['SalePrice'] < 300000)& 
(train_df['GrLivArea'] > 4000)].index,
inplace=True)
 train_df.drop(train_df[(train_df['TotalBsmtSF'] > 6000)| 
(train_df['1stFlrSF'] > 4000)].index,
inplace=True )<concatenate>
data.isnull().sum().sum()
df = pd.concat([train_df.drop(['SalePrice'],axis=1),
test_df] ).reset_index(drop=True)
 df.shape<compute_test_metric>
data.loc[data['MSSubClass']==60, 'MSSubClass']=0
data.loc[(data['MSSubClass']==20)|(data['MSSubClass']==120), 'MSSubClass']=1
data.loc[data['MSSubClass']==75, 'MSSubClass']=2
data.loc[(data['MSSubClass']==40)|(data['MSSubClass']==70)|(data['MSSubClass']==80), 'MSSubClass']=3
data.loc[(data['MSSubClass']==50)|(data['MSSubClass']==85)|(data['MSSubClass']==90)|(data['MSSubClass']==160)|(data['MSSubClass']==190), 'MSSubClass']=4
data.loc[(data['MSSubClass']==30)|(data['MSSubClass']==45)|(data['MSSubClass']==180), 'MSSubClass']=5
data.loc[(data['MSSubClass']==150), 'MSSubClass']=6
msno.matrix(test_df )<sort_values>
def dummies(d):
dummies_df=pd.DataFrame()
object_features = d.select_dtypes(include='object' ).columns
for name in object_features:
dummies = pd.get_dummies(d[name], drop_first=False)
 dummies = dummies.add_prefix("{}_".format(name))
dummies_df=pd.concat([dummies_df,dummies],axis=1)
 return dummies_df
def missing_percentage(df):
total = df.isnull().sum().sort_values(ascending = False)[df.isnull().sum().sort_values(ascending = False)!= 0]
percent = round(df.isnull().sum().sort_values(ascending = False)/len(df)*100,2)[round(df.isnull().sum().sort_values(ascending = False)/len(df)*100,2)!= 0]
return pd.concat([total, percent], axis=1, keys=['Total','Percent'])
 missing_percentage(df )<data_type_conversions>
dummies_data=dummies(data)
 dummies_data.shape
for i in ['BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2',
'PoolQC', 'MiscFeature', 'Alley', 'Fence', 'GarageType', 'GarageFinish',
'GarageQual', 'GarageCond', 'MasVnrType', 'FireplaceQu', 'MSSubClass']:
df[i] = df[i].fillna('None')
 for i in ['GarageYrBlt', 'GarageArea', 'GarageCars', 'BsmtFinSF1', 'BsmtFinSF2',
'BsmtUnfSF', 'TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath', 'MasVnrArea']:
df[i] = df[i].fillna(0)
 for i in ['Exterior1st', 'Exterior2nd', 'KitchenQual', 'Electrical', 'MSZoning',
'SaleType', 'Functional']:
df[i] = df[i].fillna(df[i].mode() [0])
 df['LotFrontage'] = df.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.median()))
df.drop(['Utilities'],inplace=True,axis=1 )<count_missing_values>
data=data.drop(columns=object_features,axis=1)
 data.columns
missing_percentage(df )<data_type_conversions>
final_data=pd.concat([data,dummies_data],axis=1)
 final_data.shape
df['MSSubClass'] = df['MSSubClass'].astype(str)
 df['OverallCond'] = df['OverallCond'].astype(str)
 df['YrSold'] = df['YrSold'].astype(str)
 df['MoSold'] = df['MoSold'].astype(str )<categorify>
X=train_data
y=train.loc[:,'SalePrice']
var = ['FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond',
'ExterQual', 'ExterCond','HeatingQC', 'PoolQC', 'KitchenQual',
'BsmtFinType1', 'BsmtFinType2', 'Functional', 'Fence',
'BsmtExposure', 'GarageFinish', 'LandSlope', 'LotShape',
'PavedDrive', 'Street', 'Alley', 'CentralAir', 'MSSubClass',
'OverallCond', 'YrSold', 'MoSold']
for i in var:
mdl = LabelEncoder().fit(list(df[i].values))
df[i] = mdl.transform(list(df[i].values))
df[var].head()<feature_engineering>
from sklearn.linear_model import Ridge, RidgeCV, LassoCV, ElasticNet
df['Total_SF_Main'] = df['TotalBsmtSF'] + df['1stFlrSF'] + df['2ndFlrSF'] 
<feature_engineering>
model_las_cv = LassoCV(alphas=(0.0001, 0.0005, 0.001, 0.01, 0.05, 0.1, 0.3, 1, 3, 5, 10))
model_las_cv.fit(X,y)
 las_cv_preds=model_las_cv.predict(test_data )
numeric_var_skews = pd.DataFrame(df.dtypes[df.dtypes != 'object'].index,columns=['Numeric_Variables'])
 numeric_var_skews['Skew'] = numeric_var_skews['Numeric_Variables'].apply(lambda x: df[x].skew())
 numeric_var_skews.sort_values('Skew',ascending=False,inplace=True)
 numeric_var_skews.reset_index(inplace=True,drop=True)
 display(numeric_var_skews )<feature_engineering>
model_ridge_cv = RidgeCV(alphas=(0.01, 0.05, 0.1, 0.3, 1, 3, 5, 10))
model_ridge_cv.fit(X, y)
 ridge_cv_preds=model_ridge_cv.predict(test_data )
high_skew = numeric_var_skews[abs(numeric_var_skews['Skew'])> 0.75]
high_skew_vars = high_skew['Numeric_Variables']
for var in high_skew_vars:
df[var] = boxcox1p(df[var], 0.15,
)<categorify>
model_ridge = Ridge(alpha=10, solver='auto')
 model_ridge.fit(X, y)
 ridge_preds=model_ridge.predict(test_data )
df_dummy = pd.get_dummies(df, 
)
 df_dummy.shape<categorify>
model_en = ElasticNet(random_state=1, alpha=0.00065, max_iter=3000)
 model_en.fit(X, y)
 en_preds=model_en.predict(test_data )
def overfit_reducer(df):

 overfit = []
for i in df.columns:
counts = df[i].value_counts()
zeros = counts.iloc[0]
if zeros / len(df)* 100 > 99.94:
overfit.append(i)
 overfit = list(overfit)
 return overfit
overfitted_features = overfit_reducer(df_dummy[:train_df.shape[0]])
 df_dummy = df_dummy.drop(overfitted_features, axis=1 )<train_model>
import xgboost as xgb
train = df_dummy[:train_df.shape[0]]
Y_train = train_df['SalePrice'].values
ols = sm.OLS(endog = Y_train,
exog = train)
 fit = ols.fit()
test2 = fit.outlier_test() ['bonf(p)']
outliers = list(test2[test2<1e-2].index)
 print('There were {:.0f} outliers at indices:'.format(len(outliers)))
 print(outliers)
 train_df = train_df.drop(train_df.index[outliers])
 df_dummy = df_dummy.drop(df_dummy.index[outliers])
 df_dummy.shape<prepare_x_and_y>
model_xgb = xgb.XGBRegressor(learning_rate=0.01,n_estimators=3460,
max_depth=3, min_child_weight=0,
gamma=0, subsample=0.7,
colsample_bytree=0.7,
objective='reg:linear', nthread=-1,
scale_pos_weight=1, seed=27,
reg_alpha=0.00006)
 model_xgb.fit(X, y)
 xgb_preds=model_xgb.predict(test_data )
train = df_dummy[:train_df.shape[0]]
test = df_dummy[train_df.shape[0]:]
Y_train = train_df['SalePrice'].values
def rmsle_cv(model):
kf = KFold(5, shuffle=True, random_state=42 ).get_n_splits(train.values)
 rmse = np.sqrt(-cross_val_score(model, train.values, Y_train,
scoring='neg_mean_squared_error', cv=kf))
return(rmse )<create_dataframe>
from sklearn.ensemble import GradientBoostingRegressor
models = pd.DataFrame([],columns=['model_name','model_object','score_mean','score_std'] )<find_best_params>
model_gbr = GradientBoostingRegressor(n_estimators=3000,
learning_rate=0.05,
max_depth=4,
max_features='sqrt',
min_samples_leaf=15,
min_samples_split=10,
loss='huber',
random_state =42)
 model_gbr.fit(X, y)
 gbr_preds=model_gbr.predict(test_data )
knr = KNeighborsRegressor(9, weights='distance')
 score = rmsle_cv(knr)
 print('KNN Regression score = {:.4f}(std = {:.4f})'.format(score.mean() , score.std()))
models.loc[len(models)] = ['knr',knr,score.mean() ,score.std() ]<find_best_params>
from lightgbm import LGBMRegressor
sgd = make_pipeline(RobustScaler() , SGDRegressor(alpha=1000000000000000,l1_ratio=1))
score = rmsle_cv(sgd)
 print('SGD score = {:.4f}(std = {:.4f})'.format(score.mean() , score.std()))
models.loc[len(models)] = ['sgd',sgd,score.mean() ,score.std() ]<compute_train_metric>
model_lgbm = LGBMRegressor(objective='regression',
num_leaves=4,
learning_rate=0.01,
n_estimators=5000,
max_bin=200,
bagging_fraction=0.75,
bagging_freq=5,
bagging_seed=7,
feature_fraction=0.2,
feature_fraction_seed=7,
verbose=-1, 
)
 model_lgbm.fit(X, y)
 lgbm_preds=model_lgbm.predict(test_data )
rfr = RandomForestRegressor()
score = rmsle_cv(rfr)
 print('Random Forest score = {:.4f}(std = {:.4f})'.format(score.mean() , score.std()))
models.loc[len(models)] = ['rfr',rfr,score.mean() ,score.std() ]<compute_train_metric>
final_predictions = 0.3 * lgbm_preds + 0.3 * gbr_preds + 0.1 * xgb_preds + 0.3 * ridge_cv_preds
lnr = LinearRegression()
score = rmsle_cv(lnr)
 print('Linear Regression score = {:.4f}(std = {:.4f})'.format(score.mean() , score.std()))
models.loc[len(models)] = ['lnr',lnr,score.mean() ,score.std() ]<compute_train_metric>
submission = {
'Id': test.Id.values,
'SalePrice': final_predictions + 0.007 * final_predictions
}
solution = pd.DataFrame(submission)
 solution.head()
<compute_train_metric><EOS> 
solution.to_csv('submission.csv',index=False )