Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<save_to_csv>


plt.style.use('bmh')
 sns.set_style({'axes.grid':False})
 %matplotlib inline

 init_notebook_mode(connected = True)

wrn.filterwarnings('ignore', category = DeprecationWarning)
 wrn.filterwarnings('ignore', category = FutureWarning)
 wrn.filterwarnings('ignore', category = UserWarning )
test_df = test_df.join(pd.DataFrame(y_test, columns=[
'any', 'epidural', 'intraparenchymal', 'intraventricular', 'subarachnoid', 'subdural'
]))
test_df = test_df.melt(id_vars=['filename'])
 test_df['ID'] = test_df.filename.apply(lambda x: x.replace('.png', '')) + '_' + test_df.variable
test_df['Label'] = test_df['value']
test_df[['ID', 'Label']].to_csv('submission.csv', index=False )<import_modules>

 train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv' )
import numpy as np
import pandas as pd
from sklearn.ensemble import GradientBoostingRegressor<load_from_csv>

 merged = pd.concat([train, test], axis=0, sort=True)
 display(merged.dtypes.value_counts())
 print('Dimensions of data:', merged.shape )
train = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/train.csv")
 test = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/test.csv" )<feature_engineering>

 merged.loc[:,['MSSubClass', 'OverallQual', 'OverallCond', 'MoSold', 'YrSold']] = merged.loc[:,['MSSubClass', 'OverallQual', 'OverallCond', 'MoSold', 'YrSold']].astype('object' )
train['totalSF'] =(train['TotalBsmtSF'] + train['1stFlrSF'] + train['2ndFlrSF'])
 train['total_bathrooms'] =(train['BsmtFullBath'] + 0.5*train['BsmtHalfBath'] + train['FullBath'] + 0.5*train['HalfBath'])
 train['ageHouse'] =(train['YrSold'] - train['YearBuilt'])
 test['totalSF'] =(test['TotalBsmtSF'] + test['1stFlrSF'] + test['2ndFlrSF'])
 test['total_bathrooms'] =(test['BsmtFullBath'] + 0.5*test['BsmtHalfBath'] + test['FullBath'] + 0.5*test['HalfBath'])
 test['ageHouse'] =(test['YrSold'] - test['YearBuilt'] )<prepare_x_and_y>

 merged.dtypes.value_counts()
train.drop(['Id','Utilities','PoolQC','MiscFeature','Alley','Fence','GarageYrBlt'] , axis=1 , inplace=True)
 train.drop(['TotalBsmtSF' , '1stFlrSF' ,'2ndFlrSF'] , axis = 1 , inplace =True)
 train.drop(['BsmtFullBath' , 'BsmtHalfBath' , 'FullBath' , 'HalfBath'] , axis=1 , inplace=True)
 train = train.drop(columns = [ 'YearRemodAdd', 'GarageQual', 'GarageCond','YrSold','MoSold', 'YearBuilt', 'Heating'], axis = 1)
 Y = train['SalePrice']
train = train.drop(columns = ['SalePrice'], axis = 1)
 Id = test['Id']
test.drop(['Id','Utilities','PoolQC','MiscFeature','Alley','Fence','GarageYrBlt'] , axis=1 , inplace=True)
 test.drop(['TotalBsmtSF' , '1stFlrSF' ,'2ndFlrSF'] , axis = 1 , inplace =True)
 test.drop(['BsmtFullBath' , 'BsmtHalfBath' , 'FullBath' , 'HalfBath'] , axis=1 , inplace=True)
 test = test.drop(columns = [ 'YearRemodAdd', 'GarageQual', 'GarageCond','YrSold','MoSold', 'YearBuilt', 'Heating'], axis = 1 )<categorify>

 train.drop(train[train.GrLivArea>4000].index, inplace = True)
 train.reset_index(drop = True, inplace = True )
mode = ['MasVnrArea' , 'Electrical' , 'MSZoning' , 'SaleType','Exterior1st','Exterior2nd','KitchenQual']
for col in mode:
train[col] = train[col].fillna(train[col].mode() [0])
 for col in mode:
test[col] = test[col].fillna(test[col].mode() [0])
 No = ['GarageType','GarageFinish',
'BsmtExposure','BsmtFinType2','BsmtFinType1','BsmtCond','BsmtQual',
'FireplaceQu','MasVnrType']
for col in No:
train[col]=train[col].fillna('None')
 for col in No:
test[col]=test[col].fillna('None')
 train['Functional'] = train['Functional'].fillna('Typ')
 train['LotFrontage'] = train['LotFrontage'].fillna(train['LotFrontage'].median())
 test['Functional'] = test['Functional'].fillna('Typ')
 test['LotFrontage'] = test['LotFrontage'].fillna(test['LotFrontage'].median())
 zero = ['total_bathrooms','totalSF','BsmtFinSF1','BsmtFinSF2','BsmtUnfSF','GarageArea','GarageCars' ]
for col in zero:
train[col] = train[col].fillna(0)
 for col in zero:
test[col] = test[col].fillna(0)
 train.head()<correct_missing_values>

 train.drop(train[train.TotalBsmtSF>3000].index, inplace = True)
 train.reset_index(drop = True, inplace = True )
train.dropna(inplace = True)
 test.dropna(inplace = True )<categorify>

 train.drop(train[train.YearBuilt<1900].index, inplace = True)
 train.reset_index(drop = True, inplace = True )
Label = ['MSZoning', 'Street', 'LotShape', 'LandContour', 'LotConfig', 'LandSlope', 'Condition1', 'RoofStyle', 'RoofMatl', 'MasVnrType', 'ExterCond', 'Electrical', 'Functional', 'PavedDrive', 'SaleType', 'Exterior1st', 'HeatingQC', 'BsmtCond', 'Foundation', 'SaleCondition', 'CentralAir', 'ExterQual' ,'BsmtQual', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2', 'Neighborhood', 'OverallQual', 'FireplaceQu', 'KitchenQual' , 'Condition2', 'Exterior2nd', 'GarageType', 'GarageFinish', 'BldgType', 'HouseStyle']
for feature in Label:
ohe = pd.get_dummies(train[feature], prefix=feature)
 train = pd.concat([train, ohe], axis=1)
 train = train.drop(columns = Label, axis = 1)
 for feature in Label:
ohe = pd.get_dummies(test[feature], prefix=feature)
 test = pd.concat([test, ohe], axis=1)
 test = test.drop(columns = Label, axis = 1 )<drop_column>

 y_train = train.SalePrice

 train.drop('SalePrice', axis = 1, inplace = True)

df_merged = pd.concat([train, test], axis = 0)

df_merged.shape
train = train.drop(columns = ['HouseStyle_2.5Fin','RoofMatl_Metal', 'RoofMatl_Roll', 'RoofMatl_Membran', 'RoofMatl_ClyTile', 'Electrical_Mix', 'Exterior1st_Stone', 'Exterior1st_ImStucc', 'Condition2_RRNn', 'Condition2_RRAn', 'Condition2_RRAe', 'Exterior2nd_Other'], axis = 1 )<train_model>

 df_merged.loc[:,['MSSubClass', 'OverallQual', 'OverallCond', 'MoSold', 'YrSold']] = df_merged.loc[:,['MSSubClass', 'OverallQual', 'OverallCond', 'MoSold', 'YrSold']].astype('object')
 df_merged.dtypes.value_counts()
model = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
max_depth=4, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state =5)
 model.fit(X = train, y = Y )<save_to_csv>

 missing_columns = df_merged.columns[df_merged.isnull().any() ].values

 total_missing_columns = np.count_nonzero(df_merged.isnull().sum())
 print('We have ' , total_missing_columns , 'features with missing values and those features(with missing values)are:
' , missing_columns )
test['SalePrice'] = model.predict(test)
 test['Id'] = Id
test[['Id', 'SalePrice']].to_csv('price_submission.csv', index=False )<install_modules>

 to_impute_by_mode = df_merged.loc[:, ['Electrical', 'MSZoning','Utilities','Exterior1st','Exterior2nd','KitchenQual','Functional', 'SaleType']]
for i in to_impute_by_mode.columns:
df_merged[i].fillna(df_merged[i].mode() [0], inplace = True )
!pip install dabl<set_options>

 df = df_merged.drop(columns=['Id','LotFrontage'], axis=1)
 le = LabelEncoder()
df = df.apply(le.fit_transform)
 df.head(2 )
warnings.filterwarnings('ignore')
 %matplotlib inline
plt.style.use('ggplot')
 warnings.filterwarnings(action="ignore")
 pd.options.display.max_seq_items = 8000
pd.options.display.max_rows = 8000
pd.options.display.max_columns = 8000<import_modules>
df['LotFrontage'] = df_merged['LotFrontage']
df = df.set_index('LotFrontage' ).reset_index()
df.head(2 )
from sklearn.model_selection import train_test_split
from sklearn.model_selection import cross_val_score, GridSearchCV, KFold
from sklearn.preprocessing import RobustScaler, StandardScaler
from sklearn.pipeline import make_pipeline
from collections import Counter
from sklearn.preprocessing import scale
from sklearn.preprocessing import LabelEncoder 
<load_from_csv>

 df_merged['LotFrontage'] = df_merged.groupby(['BldgType'])['LotFrontage'].transform(lambda x: x.fillna(x.median()))
file_path = '.. /input/house-prices-advanced-regression-techniques'
train=pd.read_csv(f'{file_path}/train.csv')
 test= pd.read_csv(f'{file_path}/test.csv' )<count_missing_values>

 df_merged_num_scaled = pd.DataFrame(data = df_merged_num_scaled, columns = df_merged_num.columns, index = df_merged_num.index)
train.isna().sum()<count_missing_values>

 df_merged_cat = df_merged.select_dtypes(include = ['object'] ).astype('category')

df_merged_cat.LotShape.replace(to_replace = ['IR3', 'IR2', 'IR1', 'Reg'], value = [0, 1, 2, 3], inplace = True)
 df_merged_cat.LandContour.replace(to_replace = ['Low', 'Bnk', 'HLS', 'Lvl'], value = [0, 1, 2, 3], inplace = True)
 df_merged_cat.Utilities.replace(to_replace = ['NoSeWa', 'AllPub'], value = [0, 1], inplace = True)
 df_merged_cat.LandSlope.replace(to_replace = ['Sev', 'Mod', 'Gtl'], value = [0, 1, 2], inplace = True)
 df_merged_cat.ExterQual.replace(to_replace = ['Fa', 'TA', 'Gd', 'Ex'], value = [0, 1, 2, 3], inplace = True)
 df_merged_cat.ExterCond.replace(to_replace = ['Po', 'Fa', 'TA', 'Gd', 'Ex'], value = [0, 1, 2, 3, 4], inplace = True)
 df_merged_cat.BsmtQual.replace(to_replace = ['None', 'Fa', 'TA', 'Gd', 'Ex'], value = [0, 1, 2, 3, 4], inplace = True)
 df_merged_cat.BsmtCond.replace(to_replace = ['None', 'Po', 'Fa', 'TA', 'Gd'], value = [0, 1, 2, 3, 4], inplace = True)
 df_merged_cat.BsmtExposure.replace(to_replace = ['None', 'No', 'Mn', 'Av', 'Gd'], value = [0, 1, 2, 3, 4], inplace = True)
 df_merged_cat.BsmtFinType1.replace(to_replace = ['None', 'Unf', 'LwQ', 'Rec', 'BLQ', 'ALQ', 'GLQ'], value = [0, 1, 2, 3, 4, 5, 6], inplace = True)
 df_merged_cat.BsmtFinType2.replace(to_replace = ['None', 'Unf', 'LwQ', 'Rec', 'BLQ', 'ALQ', 'GLQ'], value = [0, 1, 2, 3, 4, 5, 6], inplace = True)
 df_merged_cat.HeatingQC.replace(to_replace = ['Po', 'Fa', 'TA', 'Gd', 'Ex'], value = [0, 1, 2, 3, 4], inplace = True)
 df_merged_cat.Electrical.replace(to_replace = ['Mix', 'FuseP', 'FuseF', 'FuseA', 'SBrkr'], value = [0, 1, 2, 3, 4], inplace = True)
 df_merged_cat.KitchenQual.replace(to_replace = ['Fa', 'TA', 'Gd', 'Ex'], value = [0, 1, 2, 3], inplace = True)
 df_merged_cat.Functional.replace(to_replace = ['Sev', 'Maj2', 'Maj1', 'Mod', 'Min2', 'Min1', 'Typ'], value = [0, 1, 2, 3, 4, 5, 6], inplace = True)
 df_merged_cat.FireplaceQu.replace(to_replace = ['None', 'Po', 'Fa', 'TA', 'Gd', 'Ex'], value = [0, 1, 2, 3, 4, 5], inplace = True)
 df_merged_cat.GarageFinish.replace(to_replace = ['None', 'Unf', 'RFn', 'Fin'], value = [0, 1, 2, 3], inplace = True)
 df_merged_cat.GarageQual.replace(to_replace = ['None', 'Po', 'Fa', 'TA', 'Gd', 'Ex'], value = [0, 1, 2, 3, 4, 5], inplace = True)
 df_merged_cat.GarageCond.replace(to_replace = ['None', 'Po', 'Fa', 'TA', 'Gd', 'Ex'], value = [0, 1, 2, 3, 4, 5], inplace = True)
 df_merged_cat.PavedDrive.replace(to_replace = ['N', 'P', 'Y'], value = [0, 1, 2], inplace = True)
 df_merged_cat.PoolQC.replace(to_replace = ['None', 'Fa', 'Gd', 'Ex'], value = [0, 1, 2, 3], inplace = True)
 df_merged_cat.Fence.replace(to_replace = ['None', 'MnWw', 'GdWo', 'MnPrv', 'GdPrv'], value = [0, 1, 2, 3, 4], inplace = True )
test.isna().sum()<create_dataframe>

 df_merged_cat.loc[:, ['OverallQual', 'OverallCond']] = df_merged_cat.loc[:, ['OverallQual', 'OverallCond']].astype('int64')

df_merged_label_encoded = df_merged_cat.select_dtypes(include = ['int64'] )
train_data=train.copy()<drop_column>

 df_merged_one_hot = df_merged_cat.select_dtypes(include=['category'])

df_merged_one_hot = pd.get_dummies(df_merged_one_hot, drop_first=True )
train_data.drop(train_data[(train_data['OverallQual']<5)&(train_data['SalePrice']>200000)].index, inplace=True)
 train_data.drop(train_data[(train_data['GrLivArea']>4500)&(train_data['SalePrice']<300000)].index, inplace=True)
 train_data.reset_index(drop=True, inplace=True )<create_dataframe>

 seed = 44


lr = LinearRegression(n_jobs = -1)
 lasso = Lasso(random_state = seed)
 ridge = Ridge(random_state = seed)
 elnt = ElasticNet(random_state = seed)
 kr = KernelRidge()
dt = DecisionTreeRegressor(random_state = seed)
 svr = SVR()
knn = KNeighborsRegressor(n_jobs= -1)
 pls = PLSRegression()
rf = RandomForestRegressor(n_jobs = -1, random_state = seed)
 et = ExtraTreesRegressor(n_jobs = -1, random_state = seed)
 ab = AdaBoostRegressor(random_state = seed)
 gb = GradientBoostingRegressor(random_state = seed)
 xgb = XGBRegressor(n_jobs = -1, random_state = seed)
 lgb = LGBMRegressor(n_jobs = -1, random_state = seed )
train=train_data.copy()<feature_engineering>

 def tune_hyperparameters(model, param_grid):
global best_params, best_score

 grid = GridSearchCV(model, param_grid, cv = 10, verbose = 1, scoring = 'neg_mean_squared_error', n_jobs = -1)
 grid.fit(df_train_final, y_train)
 best_params = grid.best_params_
best_score = np.round(np.sqrt(-1 * grid.best_score_), 5)
 return best_params, best_score
train['SalePrice']=np.log(train['SalePrice'] )<prepare_x_and_y>
alpha = [0.0001, 0.0002, 0.00025, 0.0003, 0.00031, 0.00032, 0.00033, 0.00034, 0.00035, 0.00036, 0.00037, 0.00038,
0.0004, 0.00045, 0.0005, 0.00055, 0.0006, 0.0008, 0.001, 0.002, 0.005, 0.007, 0.008, 0.01]
lasso_params = {'alpha': alpha,
'random_state':[seed]}
tune_hyperparameters(lasso, lasso_params)
 lasso_best_params, lasso_best_score = best_params, best_score
print('Lasso best params:{} & best_score:{:0.5f}'.format(lasso_best_params, lasso_best_score))
ntrain = train.shape[0]
ntest = test.shape[0]
y_train = train.SalePrice.values
df = pd.concat(( train, test)).reset_index(drop=True )<prepare_x_and_y>
KernelRidge_param_grid = {'alpha':[0.1, 0.15, 0.23, 0.25, 0.3,1],
'kernel': ['linear', 'polynomial'],
'degree': [2,3],
'coef0': [1.5,2,3]}
tune_hyperparameters(kr, KernelRidge_param_grid)
 kr_best_params, kr_best_score = best_params, best_score
print('Kernel Ridge best params:{} & best_score: {:0.5f}'.format(kr_best_params, kr_best_score))
y=df['SalePrice'].copy()<create_dataframe>
elastic_params_grid = {'alpha': [0.0001,0.0002, 0.0003, 0.01,0.1,2],
'l1_ratio': [0.2, 0.85, 0.95,0.98,10],
'random_state':[seed]}
tune_hyperparameters(elnt, elastic_params_grid)
 elastic_best_params, elastic_best_score = best_params, best_score
print('Elastic Net best params:{} & best_score:{:0.5f}'.format(elastic_best_params, elastic_best_score))
df1 =(df.isnull().sum() / len(df)) * 100
df1 = df1.drop(df1[df1 == 0].index ).sort_values(ascending=False)[:30]
missing_data = pd.DataFrame({'Missing Ratio' :df1})
 missing_data.head(20 )<create_dataframe>
svr_params_grid = {'kernel':['linear', 'poly', 'rbf'],
'C':[2,4,5],
'gamma':[0.01,0.001,0.0001]}
tune_hyperparameters(svr, svr_params_grid)
 svr_best_params, svr_best_score = best_params, best_score
print('SVR best params:{} & best_score:{:0.5f}'.format(svr_best_params, svr_best_score))
data=df.copy()<create_dataframe>
rf_params_grid = {'n_estimators':[1,5,50,100],
'max_depth':[1,2],
'min_samples_split':[3,4],
'min_samples_leaf':[2,4],
'random_state':[seed]}
tune_hyperparameters(rf, rf_params_grid)
 rf_best_params, rf_best_score = best_params, best_score
print('RF best params:{} & best_score:{:0.5f}'.format(rf_best_params, rf_best_score))
test_data=test.copy()<drop_column>
xgb_params_grid = {'min_child_weight': [5, 10],
'gamma': [0.04, 0.1, 1.5],
'subsample': [0.6, 0.8, 1.0],
'colsample_bytree': [0.46, 1.0],
'max_depth': [3, 4]}
xgb_opt = XGBRegressor(learning_rate = 0.03, reg_alpha = 0.4640, reg_lambda = 0.8571, n_estimators = 1000,
silent = 1, nthread = -1, random_state = 101)
 tune_hyperparameters(xgb_opt, xgb_params_grid)
 xgb_best_params, xgb_best_score = best_params, best_score
print('XGB best params:{} & best_score:{:0.5f}'.format(xgb_best_params, xgb_best_score))
data = data.drop(['Alley', 'PoolQC', 'Fence', 'MiscFeature'], axis=1)
 data.head()<train_model>


gb_opt = GradientBoostingRegressor(n_estimators = 3000, learning_rate = 0.05,
max_depth = 4, max_features = 'sqrt',
min_samples_leaf = 15, min_samples_split = 10,
loss = 'huber', random_state = seed)

lgb_opt = LGBMRegressor(objective = 'regression', num_leaves = 5,
learning_rate=0.05, n_estimators = 660,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed = 9, bagging_seed = 9,
min_data_in_leaf = 6, min_sum_hessian_in_leaf = 11)

gb_best_score = cross_validation(gb_opt)
 lgb_best_score = cross_validation(lgb_opt )
<define_variables>

 lasso_opt = Lasso(**lasso_best_params)
 ridge_opt = Ridge(**ridge_best_params)
 kernel_ridge_opt = KernelRidge(**kr_best_params)
 elastic_net_opt = ElasticNet(**elastic_best_params)
 rf_opt = RandomForestRegressor(**rf_best_params)
 svm_opt = SVR(**svr_best_params)
 xgb_opt = XGBRegressor(**xgb_best_params)
 gb_opt = gb_opt
lgb_opt = lgb_opt
<data_type_conversions><EOS> 

 def predict_with_optimized_models(model):
model.fit(df_train_final, y_train)
 y_pred = np.expm1(model.predict(df_test_final))
submission = pd.DataFrame()
submission['Id']= test.Id
submission['SalePrice'] = y_pred
return submission

 predict_with_optimized_models(lasso_opt ).to_csv('lasso_optimized.csv', index = False)
 predict_with_optimized_models(ridge_opt ).to_csv('ridge_optimized.csv', index = False)
 predict_with_optimized_models(kernel_ridge_opt ).to_csv('kernel_ridge_optimized.csv', index = False)
 predict_with_optimized_models(elastic_net_opt ).to_csv('elastic_net_optimized.csv', index = False)
 predict_with_optimized_models(rf_opt ).to_csv('rf_opt_optimized.csv', index = False)
 predict_with_optimized_models(svm_opt ).to_csv('svm_opt_optimized.csv', index = False)
 predict_with_optimized_models(xgb_opt ).to_csv('xgb_optimized.csv', index = False)
 predict_with_optimized_models(gb_opt ).to_csv('gb_optimized.csv', index = False)
 predict_with_optimized_models(lgb_opt ).to_csv('lgb_optimized.csv', index = False )