Kaggle competition: House Prices - Advanced Regression Techniques
data.drop(labels=dominant_features[dominant_features['Percentage'] > 80].index,axis=1, inplace=True )<count_missing_values>
def save_file(predictions):

 output = pd.DataFrame({'Id': sample_submission_file.Id,
'SalePrice': predictions})
 output.to_csv('submission.csv', index=False)
 print("Submission file is saved")
 def calculate_root_mean_squared_log_error(y_true, y_pred):

 if len(y_pred)!=len(y_true): return 'error_mismatch'
y_pred_new = [math.log(x+1)for x in y_pred]
y_true_new = [math.log(x+1)for x in y_true]
return mean_squared_error(y_true_new, y_pred_new, squared=False )
num_data = data.select_dtypes(include=['integer', 'float'])
 for features in num_data.isna().sum() [num_data.isna().sum() != 0].index:
data[features].fillna(data[features].mean() , inplace=True)
 cat_data = data.select_dtypes(include=['object'])
 for features in cat_data.isna().sum() [cat_data.isna().sum() != 0].index:
data[features].fillna(data[features].mode() [0], inplace=True)
 data.isna().sum() [data.isna().sum() != 0]<define_variables>
train_data = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv',
index_col='Id')
 X_test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv',
index_col='Id')
 X = train_data.copy()
X.dropna(axis=0, subset=['SalePrice'], inplace=True)
 y = X.SalePrice
X.drop(['SalePrice', 'Utilities'], axis=1, inplace=True)
 X_test.drop(['Utilities'], axis=1, inplace=True)
 X_train, X_valid, y_train, y_valid = train_test_split(X, y, train_size=0.8, test_size=0.2,
random_state=0)
 sample_submission_file = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/sample_submission.csv")
 print('Data is OK' )
ms_zoning = {'C(all)':0, 'RM':1, 'RH':1, 'RL':2, 'FV':3}
lot_shape = {'Reg':0, 'IR1':1, 'IR2':2, 'IR3':2}
lot_config = {'Inside':0, 'FR2':1, 'Corner':2, 'CulDSac':3, 'FR3':3}
neighbor_hood = {'MeadowV':0, 'IDOTRR':0, 'BrDale':0, 'BrkSide':1, 'Edwards':1, 'OldTown':1,
'Sawyer':2, 'Blueste':2, 'SWISU':2, 'NPkVill':2, 'NAmes':3, 'Mitchel':3, 'SawyerW':4,
'NWAmes':5, 'Gilbert':5, 'Blmngtn':5, 'CollgCr':6, 'Crawfor':6, 'ClearCr':7,
'Somerst':7, 'Veenker':8, 'Timber':8, 'StoneBr':9, 'NridgHt':9, 'NoRidge':10}
house_style = {'1.5Unf':0, 'SFoyer':1, '1.5Fin':1, '2.5Unf':1, 'SLvl':1, '1Story':2, '2Story':3, '2.5Fin':3}
roof_style = {'Gambrel':0, 'Gable':0, 'Mansard':1, 'Flat':1, 'Hip':1, 'Shed':1}
exterior_1st = {'BrkComm':0, 'AsphShn':0, 'CBlock':0, 'AsbShng':0, 'MetalSd':1, 'Wd Sdng':1,
'WdShing':2, 'Stucco':3, 'HdBoard':3, 'Plywood':3, 'BrkFace':4, 'VinylSd':5,
'CemntBd':6, 'Stone':6, 'ImStucc':6}
exterior_2nd = {'CBlock':0, 'AsbShng':0, 'Brk Cmn':0, 'AsphShn':0, 'Wd Sdng':1, 'MetalSd':1,
'Stucco':2, 'Stone':2, 'Wd Shng':2, 'HdBoard':3, 'Plywood':3, 'BrkFace':4,
'VinylSd':4, 'CmentBd':5, 'ImStucc':5, 'Other':5}
masvnr_type = {'BrkCmn':0, 'None':0, 'BrkFace':1, 'Stone':2}
exter_qual = {'Fa':0, 'TA':1, 'Gd':2, 'Ex':3}
foundation = {'Slab':0, 'BrkTil':1, 'CBlock':2, 'Stone':3, 'Wood':3, 'PConc':3}
bsmt_qual = {'Fa':0, 'TA':1, 'Gd':2, 'Ex':3}
bsmt_exposure = {'No':0, 'Mn':1, 'Av':2, 'Gd':3}
bsmtfin_type1 = {'Rec':0, 'BLQ':0, 'LwQ':1, 'ALQ':1, 'Unf':2, 'GLQ':3}
heat_qc = {'Po':0, 'Fa':0, 'TA':1, 'Gd':2, 'Ex':3}
kitchen_qual = {'Fa':0, 'TA':1, 'Gd':2, 'Ex':3}
garage_type = {'CarPort':0, 'Detchd':0, '2Types':1, 'Basment':1, 'Attchd':2, 'BuiltIn':3}
garage_finish = {'Unf':0, 'RFn':1, 'Fin':2}<categorify>
categorical_cols = [cname for cname in X_train.columns if X_train[cname].dtype == "object"]
numerical_cols = [cname for cname in X_train.columns if X_train[cname].dtype in ['int64',
'float64']]
missing_val_count_by_column_train =(X_train.isnull().sum())
 print("Number of missing values in each column:")
 print(missing_val_count_by_column_train[missing_val_count_by_column_train > 0] )
data['MSZoning'] = data['MSZoning'].map(ms_zoning)
 data['LotShape'] = data['LotShape'].map(lot_shape)
 data['LotConfig'] = data['LotConfig'].map(lot_config)
 data['Neighborhood'] = data['Neighborhood'].map(neighbor_hood)
 data['HouseStyle'] = data['HouseStyle'].map(house_style)
 data['RoofStyle'] = data['RoofStyle'].map(roof_style)
 data['Exterior1st'] = data['Exterior1st'].map(exterior_1st)
 data['Exterior2nd'] = data['Exterior2nd'].map(exterior_2nd)
 data['MasVnrType'] = data['MasVnrType'].map(masvnr_type)
 data['ExterQual'] = data['ExterQual'].map(exter_qual)
 data['Foundation'] = data['Foundation'].map(foundation)
 data['BsmtQual'] = data['BsmtQual'].map(bsmt_qual)
 data['BsmtExposure'] = data['BsmtExposure'].map(bsmt_exposure)
 data['BsmtFinType1'] = data['BsmtFinType1'].map(bsmtfin_type1)
 data['HeatingQC'] = data['HeatingQC'].map(heat_qc)
 data['KitchenQual'] = data['KitchenQual'].map(kitchen_qual)
 data['GarageType'] = data['GarageType'].map(garage_type)
 data['GarageFinish'] = data['GarageFinish'].map(garage_finish )<prepare_x_and_y>
missing_val_count_by_column_numeric =(X_train[numerical_cols].isnull().sum())
 print("Number of missing values in numerical columns:")
 print(missing_val_count_by_column_numeric[missing_val_count_by_column_numeric > 0] )
outlier_data = data[:][:1460]
outlier_data['SalePrice'] = y
outlier_data.drop(labels=outlier_data[outlier_data['LotFrontage']>300].index, axis=0, inplace=True)
 outlier_data.drop(labels=outlier_data[(outlier_data['YearBuilt']<1900)&(outlier_data['SalePrice']>300000)].index, axis=0, inplace=True)
 outlier_data.drop(labels=outlier_data[(outlier_data['YearBuilt']<2000)&(outlier_data['SalePrice']>600000)].index, axis=0, inplace=True)
 outlier_data.drop(labels=outlier_data[outlier_data['MasVnrArea']>1160].index, axis=0, inplace=True)
 outlier_data.drop(labels=outlier_data[outlier_data['SalePrice']>700000].index, axis=0, inplace=True)
 outlier_data.drop(labels=outlier_data[(outlier_data['SalePrice']>700000)|(outlier_data['BsmtFinSF1']>5000)].index, axis=0, inplace=True)
 outlier_data.drop(labels=outlier_data[(outlier_data['TotalBsmtSF']>5000)].index, axis=0, inplace=True)
 outlier_data.drop(labels=outlier_data[(outlier_data['GrLivArea']>4000)&(outlier_data['SalePrice']<300000)].index, axis=0, inplace=True)
 outlier_data.drop(labels=outlier_data[(outlier_data['GarageArea']>1200)&(outlier_data['SalePrice']<300000)].index, axis=0, inplace=True)
 outlier_data.drop(labels=outlier_data[outlier_data['SalePrice']>700000].index, axis=0, inplace=True )<prepare_x_and_y>
constant_num_cols = ['GarageYrBlt', 'MasVnrArea']
mean_num_cols = list(set(numerical_cols ).difference(set(constant_num_cols)))
 constant_categorical_cols = ['Alley', 'MasVnrType', 'BsmtQual', 'BsmtCond', 'BsmtExposure',
'BsmtFinType1', 'BsmtFinType2', 'FireplaceQu','GarageType',
'GarageFinish', 'GarageQual', 'GarageCond', 'PoolQC', 'Fence',
'MiscFeature']
mf_categorical_cols = list(set(categorical_cols ).difference(set(constant_categorical_cols)))
 my_cols = constant_num_cols + mean_num_cols + constant_categorical_cols + mf_categorical_cols
y = outlier_data['SalePrice']
train_data = outlier_data.drop(labels='SalePrice', axis=1)
 test_data = data[1460:]<prepare_x_and_y>
numerical_transformer_m = Pipeline(steps=[ 
('imputer', SimpleImputer(strategy='mean')) , 
('scaler', StandardScaler())])
 numerical_transformer_c = Pipeline(steps=[ 
('imputer', SimpleImputer(strategy='constant', fill_value=0)) , 
('scaler', StandardScaler())])
 categorical_transformer_mf = Pipeline(steps=[ 
('imputer', SimpleImputer(strategy='most_frequent')) , 
('onehot', OneHotEncoder(handle_unknown = 'ignore', sparse = False))
])
 categorical_transformer_c = Pipeline(steps=[ 
('imputer', SimpleImputer(strategy='constant', fill_value='NA')) , 
('onehot', OneHotEncoder(handle_unknown = 'ignore', sparse = False))
])
 preprocessor = ColumnTransformer(
 transformers=[ 
('num_mean', numerical_transformer_m, mean_num_cols), 
('num_constant', numerical_transformer_c, constant_num_cols), 
('cat_mf', categorical_transformer_mf, mf_categorical_cols), 
('cat_c', categorical_transformer_c, constant_categorical_cols)
 ] )
scaler = StandardScaler()
y = pd.DataFrame(data=scaler.fit_transform(np.array(y ).reshape(1448,1)) , index=y.index, columns=['SalePrice'])
 y = np.ravel(y )<compute_train_metric>
model = XGBRegressor(learning_rate = 0.01,
n_estimators=2500,
max_depth=4,
min_child_weight=1,
gamma=0,
subsample=0.7,
colsample_bytree=0.6,
reg_alpha = 0.1,
reg_lambda = 1.25 )
def cross_val(model, x):
score = -cross_val_score(estimator=model, X=x, y=y, scoring='neg_root_mean_squared_error', cv=10, verbose=1, n_jobs=2 ).mean()
return score<define_search_model>
X_valid_eval = preprocessor.fit(X_train, y_train ).transform(X_valid )
linear_reg = make_pipeline(StandardScaler() , LinearRegression())
 poly_feat = PolynomialFeatures(degree=2)
 poly_reg = make_pipeline(poly_feat, StandardScaler() , LinearRegression())
 ridge_reg = make_pipeline(StandardScaler() , RidgeCV(alphas=np.linspace(start=0.00001, stop=100, num=350), scoring='r2', cv=8))
lasso_reg = make_pipeline(StandardScaler() , LassoCV(alphas=np.linspace(start=0.00001, stop=100, num=350), cv=8))
elasticnet_reg = make_pipeline(StandardScaler() , ElasticNetCV(l1_ratio=[.2,.4,.5,.6,.8],
alphas=np.linspace(start=0.00001, stop=100, num=350), cv=8, max_iter=1000))
svr = make_pipeline(StandardScaler() , SVR(C=10, epsilon=0.0001))
decision_tree = DecisionTreeRegressor(ccp_alpha=0.0, criterion='mse', max_depth=15,
max_features=None, max_leaf_nodes=None,
min_impurity_decrease=0.0, min_impurity_split=None,
min_samples_leaf=10, min_samples_split=2,
min_weight_fraction_leaf=0, presort='deprecated',
random_state=None, splitter='best')
 rfr = RandomForestRegressor(bootstrap=True, ccp_alpha=0.0, criterion='mse',
max_depth=40, max_features='auto', max_leaf_nodes=None,
max_samples=None, min_impurity_decrease=0.0,
min_impurity_split=None, min_samples_leaf=1,
min_samples_split=2, min_weight_fraction_leaf=0,
n_estimators=100, n_jobs=None, oob_score=False,
random_state=None, verbose=0, warm_start=False)
 adaboost_reg = AdaBoostRegressor(base_estimator=DecisionTreeRegressor(ccp_alpha=0.0,
criterion='mse', max_depth=None, max_features=None,
max_leaf_nodes=None, min_impurity_decrease=0.0,
min_impurity_split=None, min_samples_leaf=1,
min_samples_split=2, min_weight_fraction_leaf=0.0,
presort='deprecated', random_state=None,
splitter='best'),
learning_rate=1, loss='square', n_estimators=350,
random_state=None)
 gbr = GradientBoostingRegressor(alpha=0.9, ccp_alpha=0.0, criterion='friedman_mse',
init=None, learning_rate=0.02, loss='ls', max_depth=7,
max_features='sqrt', max_leaf_nodes=25,
min_impurity_decrease=0.0, min_impurity_split=None,
min_samples_leaf=1, min_samples_split=2,
min_weight_fraction_leaf=0.0, n_estimators=550,
n_iter_no_change=None, presort='deprecated',
random_state=None, subsample=0.65, tol=0.0001,
validation_fraction=0.1, verbose=0, warm_start=False)
 lgbr = LGBMRegressor(boosting='gbdt', boosting_type='gbdt', class_weight=None,
colsample_bytree=1.0, importance_type='split', subsample_freq=0,
learning_rate=0.02, max_depth=7, min_child_samples=20,
min_child_weight=0.001, min_split_gain=0.0, n_estimators=450,
n_jobs=-1, num_leaves=18, objective=None, random_state=None,
reg_alpha=0.0, reg_lambda=0.0, silent=True, subsample=1.0,
subsample_for_bin=200000)
 xgbr = XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=1, gamma=0.1,
importance_type='gain', learning_rate=0.02, max_delta_step=0,
max_depth=3, min_child_weight=1, missing=None, n_estimators=750,
n_jobs=-1, nthread=None, objective='reg:linear', random_state=0,
reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,
silent=None, subsample=0.65, verbosity=1 )<choose_model_class>
fit_params = {"model__early_stopping_rounds": 50,
"model__eval_set": [(X_valid_eval, y_valid)],
"model__verbose": False,
"model__eval_metric" : "rmsle"}
estimator_list = {'Linear Regressor': linear_reg,
'Polynomial Regressor': poly_reg,
'Ridge Regressor': ridge_reg,
'Lasso Regressor': lasso_reg,
'ElasticNet Regressor': elasticnet_reg,
'Support Vector Regressor': svr,
'Decision Tree': decision_tree,
'Random Forest Regressor': rfr,
'Adaboost Regressor': adaboost_reg,
'Gradient Boosting Regressor': gbr,
'Light Gradient Boosting Regressor': lgbr,
'Extreme Gradient Boosting Regressor': xgbr}
score = {}
def results() :
for model in estimator_list.keys() :
score[model] = cross_val(model= estimator_list[model], x= train_data)
 results()<train_model>
my_pipeline = Pipeline(steps=[('preprocessor', preprocessor), 
('model', model)
 ])
 my_pipeline.fit(X_train, y_train, **fit_params)
 preds = my_pipeline.predict(X_valid)
 score = calculate_root_mean_squared_log_error(y_valid,preds)
 print("Score: {}".format(score))
for name in estimator_list.keys() :
print(name.upper() , ':')
 estimator_list[name].fit(train_data, y)
 print('\t\tDONE !
' )<choose_model_class>
X_cv = X[my_cols].copy()
X_sub = X_test[my_cols].copy()
stacked_reg = StackingRegressor(estimators=[('Linear Regressor', linear_reg), 
('Polynomial Regressor', poly_reg), 
('Ridge Regressor', ridge_reg), 
('Lasso Regressor', lasso_reg), 
('ElasticNet Regressor', elasticnet_reg), 
('Support Vector Regressor', svr), 
('Decision Tree', decision_tree), 
('Random Forest Regressor', rfr), 
('Adaboost Regressor', adaboost_reg), 
('Gradient Boosting Regressor', gbr), 
('Light Gradient Boosting Regressor', lgbr), 
('Extreme Gradient Boosting Regressor', xgbr)],
cv=7, n_jobs=-1, verbose=1 )<train_on_grid>
scores = -1 * cross_val_score(my_pipeline, X_cv, y,
cv=5,
scoring=make_scorer(calculate_root_mean_squared_log_error,
greater_is_better=False))
print("Score:
", scores)
 print("Score mean: {}".format(scores.mean()))
print("Score std: {}".format(scores.std()))
params={'final_estimator':[estimator_list[name] for name in estimator_list.keys() ]}
search = GridSearchCV(estimator=stacked_reg, param_grid=params,
scoring='neg_root_mean_squared_error', cv=7, verbose=1, n_jobs=-1)
 search.fit(train_data, y )<find_best_params>
my_pipeline.fit(X_cv, y)
 preds = my_pipeline.predict(X_sub )
search.best_estimator_<find_best_score>
save_file(preds )